media: Add AV1 uAPI
PATCH v7 1/1 media: Add AV1 uAPI
This patch adds the AOMedia Video 1 (AV1) kernel uAPI.
This design is based on currently available AV1 API implementations and aims to support the development of AV1 stateless video codecs on Linux.
---
Changes from v6:
- Fix remaining padding issues.
pahole output for arm64:
struct v4l2_ctrl_av1_sequence {
__u32 flags; /* 0 4 */ __u8 seq_profile; /* 4 1 */ __u8 order_hint_bits; /* 5 1 */ __u8 bit_depth; /* 6 1 */ __u8 reserved; /* 7 1 */ __u16 max_frame_width_minus_1; /* 8 2 */ __u16 max_frame_height_minus_1; /* 10 2 */
/* size: 12, cachelines: 1, members: 7 */
/* last cacheline: 12 bytes */
};
struct v4l2_ctrl_av1_tile_group_entry {
__u32 tile_offset; /* 0 4 */
__u32 tile_size; /* 4 4 */
__u32 tile_row; /* 8 4 */
__u32 tile_col; /* 12 4 */ /* size: 16, cachelines: 1, members: 4 */
/* last cacheline: 16 bytes */
};
struct v4l2_av1_global_motion {
__u8 flags[8]; /* 0 8 */
enum v4l2_av1_warp_model type[8]; /* 8 32 */
__s32 params[8][6]; /* 40 192 */
/* --- cacheline 3 boundary (192 bytes) was 40 bytes ago --- */
__u8 invalid; /* 232 1 */
__u8 reserved[3]; /* 233 3 */
/* size: 236, cachelines: 4, members: 5 */
/* last cacheline: 44 bytes */
};
struct v4l2_av1_loop_restoration {
__u8 flags; /* 0 1 */
__u8 lr_unit_shift; /* 1 1 */
__u8 lr_uv_shift; /* 2 1 */
__u8 reserved; /* 3 1 */
enum v4l2_av1_frame_restoration_type frame_restoration_type[3]; /* 4 12 */
__u32 loop_restoration_size[3]; /* 16 12 */
/* size: 28, cachelines: 1, members: 6 */
/* last cacheline: 28 bytes */
};
struct v4l2_av1_cdef {
__u8 damping_minus_3; /* 0 1 */
__u8 bits; /* 1 1 */
__u8 y_pri_strength[8]; /* 2 8 */
__u8 y_sec_strength[8]; /* 10 8 */
__u8 uv_pri_strength[8]; /* 18 8 */
__u8 uv_sec_strength[8]; /* 26 8 */ /* size: 34, cachelines: 1, members: 6 */
/* last cacheline: 34 bytes */
};
struct v4l2_av1_segmentation {
__u8 flags; /* 0 1 */
__u8 last_active_seg_id; /* 1 1 */
__u8 feature_enabled[8]; /* 2 8 */
__s16 feature_data[8][8]; /* 10 128 */
/* size: 138, cachelines: 3, members: 4 */
/* last cacheline: 10 bytes */
};
struct v4l2_av1_loop_filter {
__u8 flags; /* 0 1 */
__u8 level[4]; /* 1 4 */
__u8 sharpness; /* 5 1 */
__s8 ref_deltas[8]; /* 6 8 */
__s8 mode_deltas[2]; /* 14 2 */
__u8 delta_lf_res; /* 16 1 */ /* size: 17, cachelines: 1, members: 6 */
/* last cacheline: 17 bytes */
};
struct v4l2_av1_quantization {
__u8 flags; /* 0 1 */
__u8 base_q_idx; /* 1 1 */
__s8 delta_q_y_dc; /* 2 1 */
__s8 delta_q_u_dc; /* 3 1 */
__s8 delta_q_u_ac; /* 4 1 */
__s8 delta_q_v_dc; /* 5 1 */
__s8 delta_q_v_ac; /* 6 1 */
__u8 qm_y; /* 7 1 */
__u8 qm_u; /* 8 1 */
__u8 qm_v; /* 9 1 */
__u8 delta_q_res; /* 10 1 */ /* size: 11, cachelines: 1, members: 11 */
/* last cacheline: 11 bytes */
};
struct v4l2_av1_tile_info {
__u8 flags; /* 0 1 */
__u8 context_update_tile_id; /* 1 1 */
__u8 tile_cols; /* 2 1 */
__u8 tile_rows; /* 3 1 */
__u32 mi_col_starts[65]; /* 4 260 */
/* --- cacheline 4 boundary (256 bytes) was 8 bytes ago --- */
__u32 mi_row_starts[65]; /* 264 260 */
/* --- cacheline 8 boundary (512 bytes) was 12 bytes ago --- */
__u32 width_in_sbs_minus_1[64]; /* 524 256 */
/* --- cacheline 12 boundary (768 bytes) was 12 bytes ago --- */
__u32 height_in_sbs_minus_1[64]; /* 780 256 */
/* --- cacheline 16 boundary (1024 bytes) was 12 bytes ago --- */
__u8 tile_size_bytes; /* 1036 1 */
__u8 padding[3]; /* 1037 3 */
/* size: 1040, cachelines: 17, members: 10 */
/* last cacheline: 16 bytes */
};
struct v4l2_ctrl_av1_frame {
struct v4l2_av1_tile_info tile_info; /* 0 1040 */
/* --- cacheline 16 boundary (1024 bytes) was 16 bytes ago --- */
struct v4l2_av1_quantization quantization; /* 1040 11 */
__u8 superres_denom; /* 1051 1 */
struct v4l2_av1_segmentation segmentation; /* 1052 138 */
/* --- cacheline 18 boundary (1152 bytes) was 38 bytes ago --- */
struct v4l2_av1_loop_filter loop_filter; /* 1190 17 */
struct v4l2_av1_cdef cdef; /* 1207 34 */
/* --- cacheline 19 boundary (1216 bytes) was 25 bytes ago --- */
__u8 skip_mode_frame[2]; /* 1241 2 */
__u8 primary_ref_frame; /* 1243 1 */
struct v4l2_av1_loop_restoration loop_restoration; /* 1244 28 */
struct v4l2_av1_global_motion global_motion; /* 1272 236 */
/* --- cacheline 23 boundary (1472 bytes) was 36 bytes ago --- */
__u32 flags; /* 1508 4 */
enum v4l2_av1_frame_type frame_type; /* 1512 4 */
__u32 order_hint; /* 1516 4 */
__u32 upscaled_width; /* 1520 4 */
enum v4l2_av1_interpolation_filter interpolation_filter; /* 1524 4 */
enum v4l2_av1_tx_mode tx_mode; /* 1528 4 */
__u32 frame_width_minus_1; /* 1532 4 */
/* --- cacheline 24 boundary (1536 bytes) --- */
__u32 frame_height_minus_1; /* 1536 4 */
__u16 render_width_minus_1; /* 1540 2 */
__u16 render_height_minus_1; /* 1542 2 */
__u32 current_frame_id; /* 1544 4 */
__u32 buffer_removal_time[32]; /* 1548 128 */
/* --- cacheline 26 boundary (1664 bytes) was 12 bytes ago --- */
__u8 reserved[4]; /* 1676 4 */
__u32 order_hints[8]; /* 1680 32 */
__u64 reference_frame_ts[8]; /* 1712 64 */
/* --- cacheline 27 boundary (1728 bytes) was 48 bytes ago --- */
__s8 ref_frame_idx[7]; /* 1776 7 */
__u8 refresh_frame_flags; /* 1783 1 */
/* size: 1784, cachelines: 28, members: 27 */
/* last cacheline: 56 bytes */
};
struct v4l2_ctrl_av1_film_grain {
__u8 flags; /* 0 1 */
__u8 cr_mult; /* 1 1 */
__u16 grain_seed; /* 2 2 */
__u8 film_grain_params_ref_idx; /* 4 1 */
__u8 num_y_points; /* 5 1 */
__u8 point_y_value[16]; /* 6 16 */
__u8 point_y_scaling[16]; /* 22 16 */
__u8 num_cb_points; /* 38 1 */
__u8 point_cb_value[16]; /* 39 16 */
__u8 point_cb_scaling[16]; /* 55 16 */ /* --- cacheline 1 boundary (64 bytes) was 7 bytes ago --- */
__u8 num_cr_points; /* 71 1 */
__u8 point_cr_value[16]; /* 72 16 */
__u8 point_cr_scaling[16]; /* 88 16 */
__u8 grain_scaling_minus_8; /* 104 1 */
__u8 ar_coeff_lag; /* 105 1 */
__u8 ar_coeffs_y_plus_128[25]; /* 106 25 */
/* --- cacheline 2 boundary (128 bytes) was 3 bytes ago --- */
__u8 ar_coeffs_cb_plus_128[25]; /* 131 25 */
__u8 ar_coeffs_cr_plus_128[25]; /* 156 25 */
__u8 ar_coeff_shift_minus_6; /* 181 1 */
__u8 grain_scale_shift; /* 182 1 */
__u8 cb_mult; /* 183 1 */
__u8 cb_luma_mult; /* 184 1 */
__u8 cr_luma_mult; /* 185 1 */
__u16 cb_offset; /* 186 2 */
__u16 cr_offset; /* 188 2 */
__u8 reserved[4]; /* 190 4 */
/* size: 194, cachelines: 4, members: 26 */ /* last cacheline: 2 bytes */
};
pahole output for x64:
struct v4l2_ctrl_av1_sequence {
__u32 flags; /* 0 4 */ __u8 seq_profile; /* 4 1 */ __u8 order_hint_bits; /* 5 1 */ __u8 bit_depth; /* 6 1 */ __u8 reserved; /* 7 1 */ __u16 max_frame_width_minus_1; /* 8 2 */ __u16 max_frame_height_minus_1; /* 10 2 */
/* size: 12, cachelines: 1, members: 7 */
/* last cacheline: 12 bytes */
};
struct v4l2_ctrl_av1_tile_group_entry {
__u32 tile_offset; /* 0 4 */
__u32 tile_size; /* 4 4 */
__u32 tile_row; /* 8 4 */
__u32 tile_col; /* 12 4 */ /* size: 16, cachelines: 1, members: 4 */
/* last cacheline: 16 bytes */
};
struct v4l2_av1_global_motion {
__u8 flags[8]; /* 0 8 */
enum v4l2_av1_warp_model type[8]; /* 8 32 */
__s32 params[8][6]; /* 40 192 */
/* --- cacheline 3 boundary (192 bytes) was 40 bytes ago --- */
__u8 invalid; /* 232 1 */
__u8 reserved[3]; /* 233 3 */
/* size: 236, cachelines: 4, members: 5 */
/* last cacheline: 44 bytes */
};
struct v4l2_av1_loop_restoration {
__u8 flags; /* 0 1 */
__u8 lr_unit_shift; /* 1 1 */
__u8 lr_uv_shift; /* 2 1 */
__u8 reserved; /* 3 1 */
enum v4l2_av1_frame_restoration_type frame_restoration_type[3]; /* 4 12 */
__u32 loop_restoration_size[3]; /* 16 12 */
/* size: 28, cachelines: 1, members: 6 */
/* last cacheline: 28 bytes */
};
struct v4l2_av1_cdef {
__u8 damping_minus_3; /* 0 1 */
__u8 bits; /* 1 1 */
__u8 y_pri_strength[8]; /* 2 8 */
__u8 y_sec_strength[8]; /* 10 8 */
__u8 uv_pri_strength[8]; /* 18 8 */
__u8 uv_sec_strength[8]; /* 26 8 */ /* size: 34, cachelines: 1, members: 6 */
/* last cacheline: 34 bytes */
};
struct v4l2_av1_segmentation {
__u8 flags; /* 0 1 */
__u8 last_active_seg_id; /* 1 1 */
__u8 feature_enabled[8]; /* 2 8 */
__s16 feature_data[8][8]; /* 10 128 */
/* size: 138, cachelines: 3, members: 4 */
/* last cacheline: 10 bytes */
};
struct v4l2_av1_loop_filter {
__u8 flags; /* 0 1 */
__u8 level[4]; /* 1 4 */
__u8 sharpness; /* 5 1 */
__s8 ref_deltas[8]; /* 6 8 */
__s8 mode_deltas[2]; /* 14 2 */
__u8 delta_lf_res; /* 16 1 */ /* size: 17, cachelines: 1, members: 6 */
/* last cacheline: 17 bytes */
};
struct v4l2_av1_quantization {
__u8 flags; /* 0 1 */
__u8 base_q_idx; /* 1 1 */
__s8 delta_q_y_dc; /* 2 1 */
__s8 delta_q_u_dc; /* 3 1 */
__s8 delta_q_u_ac; /* 4 1 */
__s8 delta_q_v_dc; /* 5 1 */
__s8 delta_q_v_ac; /* 6 1 */
__u8 qm_y; /* 7 1 */
__u8 qm_u; /* 8 1 */
__u8 qm_v; /* 9 1 */
__u8 delta_q_res; /* 10 1 */ /* size: 11, cachelines: 1, members: 11 */
/* last cacheline: 11 bytes */
};
struct v4l2_av1_tile_info {
__u8 flags; /* 0 1 */
__u8 context_update_tile_id; /* 1 1 */
__u8 tile_cols; /* 2 1 */
__u8 tile_rows; /* 3 1 */
__u32 mi_col_starts[65]; /* 4 260 */
/* --- cacheline 4 boundary (256 bytes) was 8 bytes ago --- */
__u32 mi_row_starts[65]; /* 264 260 */
/* --- cacheline 8 boundary (512 bytes) was 12 bytes ago --- */
__u32 width_in_sbs_minus_1[64]; /* 524 256 */
/* --- cacheline 12 boundary (768 bytes) was 12 bytes ago --- */
__u32 height_in_sbs_minus_1[64]; /* 780 256 */
/* --- cacheline 16 boundary (1024 bytes) was 12 bytes ago --- */
__u8 tile_size_bytes; /* 1036 1 */
__u8 padding[3]; /* 1037 3 */
/* size: 1040, cachelines: 17, members: 10 */
/* last cacheline: 16 bytes */
};
struct v4l2_ctrl_av1_frame {
struct v4l2_av1_tile_info tile_info; /* 0 1040 */
/* --- cacheline 16 boundary (1024 bytes) was 16 bytes ago --- */
struct v4l2_av1_quantization quantization; /* 1040 11 */
__u8 superres_denom; /* 1051 1 */
struct v4l2_av1_segmentation segmentation; /* 1052 138 */
/* --- cacheline 18 boundary (1152 bytes) was 38 bytes ago --- */
struct v4l2_av1_loop_filter loop_filter; /* 1190 17 */
struct v4l2_av1_cdef cdef; /* 1207 34 */
/* --- cacheline 19 boundary (1216 bytes) was 25 bytes ago --- */
__u8 skip_mode_frame[2]; /* 1241 2 */
__u8 primary_ref_frame; /* 1243 1 */
struct v4l2_av1_loop_restoration loop_restoration; /* 1244 28 */
struct v4l2_av1_global_motion global_motion; /* 1272 236 */
/* --- cacheline 23 boundary (1472 bytes) was 36 bytes ago --- */
__u32 flags; /* 1508 4 */
enum v4l2_av1_frame_type frame_type; /* 1512 4 */
__u32 order_hint; /* 1516 4 */
__u32 upscaled_width; /* 1520 4 */
enum v4l2_av1_interpolation_filter interpolation_filter; /* 1524 4 */
enum v4l2_av1_tx_mode tx_mode; /* 1528 4 */
__u32 frame_width_minus_1; /* 1532 4 */
/* --- cacheline 24 boundary (1536 bytes) --- */
__u32 frame_height_minus_1; /* 1536 4 */
__u16 render_width_minus_1; /* 1540 2 */
__u16 render_height_minus_1; /* 1542 2 */
__u32 current_frame_id; /* 1544 4 */
__u32 buffer_removal_time[32]; /* 1548 128 */
/* --- cacheline 26 boundary (1664 bytes) was 12 bytes ago --- */
__u8 reserved[4]; /* 1676 4 */
__u32 order_hints[8]; /* 1680 32 */
__u64 reference_frame_ts[8]; /* 1712 64 */
/* --- cacheline 27 boundary (1728 bytes) was 48 bytes ago --- */
__s8 ref_frame_idx[7]; /* 1776 7 */
__u8 refresh_frame_flags; /* 1783 1 */
/* size: 1784, cachelines: 28, members: 27 */
/* last cacheline: 56 bytes */
};
struct v4l2_ctrl_av1_film_grain {
__u8 flags; /* 0 1 */
__u8 cr_mult; /* 1 1 */
__u16 grain_seed; /* 2 2 */
__u8 film_grain_params_ref_idx; /* 4 1 */
__u8 num_y_points; /* 5 1 */
__u8 point_y_value[16]; /* 6 16 */
__u8 point_y_scaling[16]; /* 22 16 */
__u8 num_cb_points; /* 38 1 */
__u8 point_cb_value[16]; /* 39 16 */
__u8 point_cb_scaling[16]; /* 55 16 */ /* --- cacheline 1 boundary (64 bytes) was 7 bytes ago --- */
__u8 num_cr_points; /* 71 1 */
__u8 point_cr_value[16]; /* 72 16 */
__u8 point_cr_scaling[16]; /* 88 16 */
__u8 grain_scaling_minus_8; /* 104 1 */
__u8 ar_coeff_lag; /* 105 1 */
__u8 ar_coeffs_y_plus_128[25]; /* 106 25 */
/* --- cacheline 2 boundary (128 bytes) was 3 bytes ago --- */
__u8 ar_coeffs_cb_plus_128[25]; /* 131 25 */
__u8 ar_coeffs_cr_plus_128[25]; /* 156 25 */
__u8 ar_coeff_shift_minus_6; /* 181 1 */
__u8 grain_scale_shift; /* 182 1 */
__u8 cb_mult; /* 183 1 */
__u8 cb_luma_mult; /* 184 1 */
__u8 cr_luma_mult; /* 185 1 */
__u16 cb_offset; /* 186 2 */
__u16 cr_offset; /* 188 2 */
__u8 reserved[4]; /* 190 4 */
/* size: 194, cachelines: 4, members: 26 */ /* last cacheline: 2 bytes */
};
pahole output for x86:
struct v4l2_ctrl_av1_sequence {
__u32 flags; /* 0 4 */ __u8 seq_profile; /* 4 1 */ __u8 order_hint_bits; /* 5 1 */ __u8 bit_depth; /* 6 1 */ __u8 reserved; /* 7 1 */ __u16 max_frame_width_minus_1; /* 8 2 */ __u16 max_frame_height_minus_1; /* 10 2 */
/* size: 12, cachelines: 1, members: 7 */
/* last cacheline: 12 bytes */
};
struct v4l2_ctrl_av1_tile_group_entry {
__u32 tile_offset; /* 0 4 */
__u32 tile_size; /* 4 4 */
__u32 tile_row; /* 8 4 */
__u32 tile_col; /* 12 4 */ /* size: 16, cachelines: 1, members: 4 */
/* last cacheline: 16 bytes */
};
struct v4l2_av1_global_motion {
__u8 flags[8]; /* 0 8 */
enum v4l2_av1_warp_model type[8]; /* 8 32 */
__s32 params[8][6]; /* 40 192 */
/* --- cacheline 3 boundary (192 bytes) was 40 bytes ago --- */
__u8 invalid; /* 232 1 */
__u8 reserved[3]; /* 233 3 */
/* size: 236, cachelines: 4, members: 5 */
/* last cacheline: 44 bytes */
};
struct v4l2_av1_loop_restoration {
__u8 flags; /* 0 1 */
__u8 lr_unit_shift; /* 1 1 */
__u8 lr_uv_shift; /* 2 1 */
__u8 reserved; /* 3 1 */
enum v4l2_av1_frame_restoration_type frame_restoration_type[3]; /* 4 12 */
__u32 loop_restoration_size[3]; /* 16 12 */
/* size: 28, cachelines: 1, members: 6 */
/* last cacheline: 28 bytes */
};
struct v4l2_av1_cdef {
__u8 damping_minus_3; /* 0 1 */
__u8 bits; /* 1 1 */
__u8 y_pri_strength[8]; /* 2 8 */
__u8 y_sec_strength[8]; /* 10 8 */
__u8 uv_pri_strength[8]; /* 18 8 */
__u8 uv_sec_strength[8]; /* 26 8 */ /* size: 34, cachelines: 1, members: 6 */
/* last cacheline: 34 bytes */
};
struct v4l2_av1_segmentation {
__u8 flags; /* 0 1 */
__u8 last_active_seg_id; /* 1 1 */
__u8 feature_enabled[8]; /* 2 8 */
__s16 feature_data[8][8]; /* 10 128 */
/* size: 138, cachelines: 3, members: 4 */
/* last cacheline: 10 bytes */
};
struct v4l2_av1_loop_filter {
__u8 flags; /* 0 1 */
__u8 level[4]; /* 1 4 */
__u8 sharpness; /* 5 1 */
__s8 ref_deltas[8]; /* 6 8 */
__s8 mode_deltas[2]; /* 14 2 */
__u8 delta_lf_res; /* 16 1 */ /* size: 17, cachelines: 1, members: 6 */
/* last cacheline: 17 bytes */
};
struct v4l2_av1_quantization {
__u8 flags; /* 0 1 */
__u8 base_q_idx; /* 1 1 */
__s8 delta_q_y_dc; /* 2 1 */
__s8 delta_q_u_dc; /* 3 1 */
__s8 delta_q_u_ac; /* 4 1 */
__s8 delta_q_v_dc; /* 5 1 */
__s8 delta_q_v_ac; /* 6 1 */
__u8 qm_y; /* 7 1 */
__u8 qm_u; /* 8 1 */
__u8 qm_v; /* 9 1 */
__u8 delta_q_res; /* 10 1 */ /* size: 11, cachelines: 1, members: 11 */
/* last cacheline: 11 bytes */
};
struct v4l2_av1_tile_info {
__u8 flags; /* 0 1 */
__u8 context_update_tile_id; /* 1 1 */
__u8 tile_cols; /* 2 1 */
__u8 tile_rows; /* 3 1 */
__u32 mi_col_starts[65]; /* 4 260 */
/* --- cacheline 4 boundary (256 bytes) was 8 bytes ago --- */
__u32 mi_row_starts[65]; /* 264 260 */
/* --- cacheline 8 boundary (512 bytes) was 12 bytes ago --- */
__u32 width_in_sbs_minus_1[64]; /* 524 256 */
/* --- cacheline 12 boundary (768 bytes) was 12 bytes ago --- */
__u32 height_in_sbs_minus_1[64]; /* 780 256 */
/* --- cacheline 16 boundary (1024 bytes) was 12 bytes ago --- */
__u8 tile_size_bytes; /* 1036 1 */
__u8 padding[3]; /* 1037 3 */
/* size: 1040, cachelines: 17, members: 10 */
/* last cacheline: 16 bytes */
};
struct v4l2_ctrl_av1_frame {
struct v4l2_av1_tile_info tile_info; /* 0 1040 */
/* --- cacheline 16 boundary (1024 bytes) was 16 bytes ago --- */
struct v4l2_av1_quantization quantization; /* 1040 11 */
__u8 superres_denom; /* 1051 1 */
struct v4l2_av1_segmentation segmentation; /* 1052 138 */
/* --- cacheline 18 boundary (1152 bytes) was 38 bytes ago --- */
struct v4l2_av1_loop_filter loop_filter; /* 1190 17 */
struct v4l2_av1_cdef cdef; /* 1207 34 */
/* --- cacheline 19 boundary (1216 bytes) was 25 bytes ago --- */
__u8 skip_mode_frame[2]; /* 1241 2 */
__u8 primary_ref_frame; /* 1243 1 */
struct v4l2_av1_loop_restoration loop_restoration; /* 1244 28 */
struct v4l2_av1_global_motion global_motion; /* 1272 236 */
/* --- cacheline 23 boundary (1472 bytes) was 36 bytes ago --- */
__u32 flags; /* 1508 4 */
enum v4l2_av1_frame_type frame_type; /* 1512 4 */
__u32 order_hint; /* 1516 4 */
__u32 upscaled_width; /* 1520 4 */
enum v4l2_av1_interpolation_filter interpolation_filter; /* 1524 4 */
enum v4l2_av1_tx_mode tx_mode; /* 1528 4 */
__u32 frame_width_minus_1; /* 1532 4 */
/* --- cacheline 24 boundary (1536 bytes) --- */
__u32 frame_height_minus_1; /* 1536 4 */
__u16 render_width_minus_1; /* 1540 2 */
__u16 render_height_minus_1; /* 1542 2 */
__u32 current_frame_id; /* 1544 4 */
__u32 buffer_removal_time[32]; /* 1548 128 */
/* --- cacheline 26 boundary (1664 bytes) was 12 bytes ago --- */
__u8 reserved[4]; /* 1676 4 */
__u32 order_hints[8]; /* 1680 32 */
__u64 reference_frame_ts[8]; /* 1712 64 */
/* --- cacheline 27 boundary (1728 bytes) was 48 bytes ago --- */
__s8 ref_frame_idx[7]; /* 1776 7 */
__u8 refresh_frame_flags; /* 1783 1 */
/* size: 1784, cachelines: 28, members: 27 */
/* last cacheline: 56 bytes */
};
struct v4l2_ctrl_av1_film_grain {
__u8 flags; /* 0 1 */
__u8 cr_mult; /* 1 1 */
__u16 grain_seed; /* 2 2 */
__u8 film_grain_params_ref_idx; /* 4 1 */
__u8 num_y_points; /* 5 1 */
__u8 point_y_value[16]; /* 6 16 */
__u8 point_y_scaling[16]; /* 22 16 */
__u8 num_cb_points; /* 38 1 */
__u8 point_cb_value[16]; /* 39 16 */
__u8 point_cb_scaling[16]; /* 55 16 */ /* --- cacheline 1 boundary (64 bytes) was 7 bytes ago --- */
__u8 num_cr_points; /* 71 1 */
__u8 point_cr_value[16]; /* 72 16 */
__u8 point_cr_scaling[16]; /* 88 16 */
__u8 grain_scaling_minus_8; /* 104 1 */
__u8 ar_coeff_lag; /* 105 1 */
__u8 ar_coeffs_y_plus_128[25]; /* 106 25 */
/* --- cacheline 2 boundary (128 bytes) was 3 bytes ago --- */
__u8 ar_coeffs_cb_plus_128[25]; /* 131 25 */
__u8 ar_coeffs_cr_plus_128[25]; /* 156 25 */
__u8 ar_coeff_shift_minus_6; /* 181 1 */
__u8 grain_scale_shift; /* 182 1 */
__u8 cb_mult; /* 183 1 */
__u8 cb_luma_mult; /* 184 1 */
__u8 cr_luma_mult; /* 185 1 */
__u16 cb_offset; /* 186 2 */
__u16 cr_offset; /* 188 2 */
__u8 reserved[4]; /* 190 4 */
/* size: 194, cachelines: 4, members: 26 */ /* last cacheline: 2 bytes */
};
- Made sure all structs show no difference in size for the archs above.
- Added a padding field to the film grain control, to be on the safer side.
- Fixed more issues with the docs (thanks, Nicolas)
Changes from v5:
- Fix kernel-doc issues from scripts/kernel-doc -none include/uapi/linux/v4l2-controls.h
Changes from v4:
- Again tested on MT8195/RK3588.
- Ran pahole, made sure there are no holes or size differences between different archs. Updated .rst docs as needed.
- Fixed doc issues pointed out by Hans in his last review (Thanks, Hans!)
- Remove uniform_tile_spacing from the frame flags. That flag was duplicated given the flag in the tile_info structure with the same name.
- Fixed whitespace issues (hopefully)
- Fixed a small typo in the docs as pointed out by Brian Norris. Thanks, Brian!
- Add mention to full vs limited range quantization where applicable.
- Fix incorrect validation for the loop filter flags. Previous validation code accepted up to 0b111, but there are four possible flags defined.
- Fixed issue with the validation of feature_data as pointed out by Hans.
- Fixed checkpatch warnings (ran with --strict)
Changes from v3:
- Dropped RFC tag.
- Tested on MT8195 and RK3588 SoCs
- Remove NUM_REF_FRAMES. This field was duplicated.
- Address Chen-Yu's review of v3. Thanks, Chen-Yu!
- Change unicode dash to ascii in the AV1 documentation. This would trigger some warnings from time to time.
- Change ref_frame_idx from u8 to s8. ref_frame_idx is initialized to -1 in a few places in the spec.
- Move AV1 profile and level CIDs to MPEG class. These can be used by a stateful implementation.
- Fix order_hints docs, replace it with something more descriptive.
- Change gm_params to s32
- Remove [last|golden]_frame_idx. These are not needed in light of ref_frame_idx.
Changes from v2:
- Bump AV1 CIDs to HEVC+100
In light of the HEVC upstreaming, bump the AV1 CIDs to the values used by HEVC by 100, in order to avoid conflicts. The range previously used by AV1 was taken by HEVC, as it was merged first.
- Fix typo in enum v4l2_av1_frame_restoration_type
- Change u8 to __u8 in struct v4l2_av1_loop_restoration
- Rename v4l2_ctrl_av1_frame_header to v4l2_ctrl_av1_frame
- Fix a small typo
- Also include 6.8.16 in the loop filter docs
- delta_lf_multi is now a flag
- Stop using the BIT macro
- Move the AV1 CIDs below the VP9 ones
- Remove useless break
- Fix wrong docs for ref_frame_idx
- Remove tile_group control.
Data in this control is not currently used. The fact that VA-API does not have a similar Buffer in its API hints that it probably will not be needed in the future as well.
- Move av1 CIDs to the bottom of the stateless block
- Move V4L2_CTRL_TYPE_AV1_* to the end of the stateless block
- Remove stale film_grain doc from av1 frame
Changes from v1:
- Fixed the issues pointed out by Hans in his review (Thanks, Hans)
- Fixed the sign for ref_deltas and mode_deltas
These are signed values and will otherwise fail the validation in v4l2-ctrls-core.c.
- Fix film grain y/cb/cr points validation
We were controlling the entire array, but there is a subset of points being set, which lead to the validation to fail. Fix this by limiting the test to the number of points. The number of point is already controlled in the previous check so should be safe.
- Fix validation for Su(n) syntax
The Su(n) AV1 syntax ranges from -2^n/2 to 2^n/2 - 1, while the previous code was only validating values from -2^n/2 + 1 to 2^n/2 - 1.
- Fix order_hints and buffer_removal_time
The buffer_removal_time should have been 32 bit and the ref_order_hint got confused with order_hints. We want the
second one, as the first one is only used as a backup to
fill the second on missing last frame.
- Fix off by one error in validation code
There are 5 segmentation flags, so any code that sets the most significant flag would automatically fail.
- Feature data can be negative
Feature data can be negative if Segmentation_Feature_Signed[j] == 1, in which case it is read with the Su(n) syntax.
Reflect that by changing the data type to s16.
- Remove large scale mode controls
Large scale mode is not supported. Remove all controls associated with it. This means that all TILE_LIST_* controls are gone.
- Split v4l2_av1_film_grain into its own control
Split v4l2_av1_film_grain into its own control, as not all accelerators might support it. This control is now optional.
- Fix broken docs for v4l2_av1_interpolation_filter
Docs mention VP9 instead of AV1. This is a typo.
- Fix broken docs for v4l2_ctrl_av1_tile_group_entry
Fix a small typo in the docs.
- Fix AV1 reference lists
We need both all the references and the per frame references. This is because unlike VP9, show_existing_frame that are key frame will refresh the frame slots, which prevents the driver from keeping track since these frame are not signalled to the driver.
Instead mimic other API (like VA and DXVA) and provide a complete list of active references. The driver can then cleanup any data it was storing for references that are no longer in use, saving previous memory.
- Fix broken docs in ext-ctrls-codec-stateless.rst :c:type:struct v4l2_ctrl_av1_tile_group_entry would not resolve, it would also produce a warning when making the docs. Fix this by changing this to :c:type:v4l2_ctrl_av1_tile_group_entry instead.
--- .../userspace-api/media/v4l/biblio.rst | 9 + .../media/v4l/ext-ctrls-codec-stateless.rst | 1209 ++++++++++++++++- .../media/v4l/pixfmt-compressed.rst | 16 + .../media/v4l/vidioc-g-ext-ctrls.rst | 16 + .../media/v4l/vidioc-queryctrl.rst | 24 + .../media/videodev2.h.rst.exceptions | 4 + drivers/media/v4l2-core/v4l2-ctrls-core.c | 258 ++++ drivers/media/v4l2-core/v4l2-ctrls-defs.c | 61 + drivers/media/v4l2-core/v4l2-ioctl.c | 1 + include/media/v4l2-ctrls.h | 8 + include/uapi/linux/v4l2-controls.h | 721 ++++++++++ include/uapi/linux/videodev2.h | 10 + 12 files changed, 2335 insertions(+), 2 deletions(-)
Documentation/userspace-api/media/v4l/biblio.rst +9 -0
@@ -427,3 +427,12 @@ VP9 :title: VP9 Bitstream & Decoding Process Specification :author: Adrian Grange (Google), Peter de Rivaz (Argon Design), Jonathan Hunt (Argon Design) + +.. _av1: + +AV1 +=== + +:title: AV1 Bitstream & Decoding Process Specification + +:author: Peter de Rivaz, Argon Design Ltd, Jack Haughton, Argon Design Ltd
Documentation/userspace-api/media/v4l/ext-ctrls-codec-stateless.rst +1207 -2
1,221 lines not on this page · raw message
Documentation/userspace-api/media/v4l/pixfmt-compressed.rst +16 -0
@@ -233,6 +233,22 @@ Compressed Formats through the ``V4L2_CID_STATELESS_FWHT_PARAMS`` control. See the :ref:`associated Codec Control ID <codec-stateless-fwht>`. + * .. _V4L2-PIX-FMT-AV1-FRAME: + + - ``V4L2_PIX_FMT_AV1_FRAME`` + - 'AV1F' + - AV1 parsed frame, including the frame header, as extracted from the container. + This format is adapted for stateless video decoders that implement a AV1 + pipeline with the :ref:`stateless_decoder`. Metadata associated with the + frame to decode is required to be passed through the + ``V4L2_CID_STATELESS_AV1_SEQUENCE``, ``V4L2_CID_STATELESS_AV1_FRAME``, + and ``V4L2_CID_STATELESS_AV1_TILE_GROUP_ENTRY`` controls. + See the :ref:`associated Codec Control IDs <v4l2-codec-stateless-av1>`. + Exactly one output and one capture buffer must be provided for use with + this pixel format. The output buffer must contain the appropriate number + of macroblocks to decode a full corresponding frame to the matching + capture buffer. + .. raw:: latex \normalsize
Documentation/userspace-api/media/v4l/vidioc-g-ext-ctrls.rst +16 -0
@@ -269,6 +269,22 @@ still cause this situation. - ``p_hevc_decode_params`` - A pointer to a struct :c:type:`v4l2_ctrl_hevc_decode_params`. Valid if this control is of type ``V4L2_CTRL_TYPE_HEVC_DECODE_PARAMS``. + * - struct :c:type:`v4l2_ctrl_av1_sequence` * + - ``p_av1_sequence`` + - A pointer to a struct :c:type:`v4l2_ctrl_av1_sequence`. Valid if this control is + of type ``V4L2_CTRL_TYPE_AV1_SEQUENCE``. + * - struct :c:type:`v4l2_ctrl_av1_tile_group_entry` * + - ``p_av1_tile_group_entry`` + - A pointer to a struct :c:type:`v4l2_ctrl_av1_tile_group_entry`. Valid if this control is + of type ``V4L2_CTRL_TYPE_AV1_TILE_GROUP_ENTRY``. + * - struct :c:type:`v4l2_ctrl_av1_frame` * + - ``p_av1_frame`` + - A pointer to a struct :c:type:`v4l2_ctrl_av1_frame`. Valid if this control is + of type ``V4L2_CTRL_TYPE_AV1_FRAME``. + * - struct :c:type:`v4l2_ctrl_av1_film_grain` * + - ``p_av1_film_grain`` + - A pointer to a struct :c:type:`v4l2_ctrl_av1_film_grain`. Valid if this control is + of type ``V4L2_CTRL_TYPE_AV1_FILM_GRAIN``. * - void * - ``ptr`` - A pointer to a compound type which can be an N-dimensional array
Documentation/userspace-api/media/v4l/vidioc-queryctrl.rst +24 -0
@@ -525,6 +525,30 @@ See also the examples in :ref:`control`. - n/a - A struct :c:type:`v4l2_ctrl_vp9_frame`, containing VP9 frame decode parameters for stateless video decoders. + * - ``V4L2_CTRL_TYPE_AV1_SEQUENCE`` + - n/a + - n/a + - n/a + - A struct :c:type:`v4l2_ctrl_av1_sequence`, containing AV1 Sequence OBU + decoding parameters for stateless video decoders. + * - ``V4L2_CTRL_TYPE_AV1_TILE_GROUP_ENTRY`` + - n/a + - n/a + - n/a + - A struct :c:type:`v4l2_ctrl_av1_tile_group_entry`, containing AV1 Tile Group + OBU decoding parameters for stateless video decoders. + * - ``V4L2_CTRL_TYPE_AV1_FRAME`` + - n/a + - n/a + - n/a + - A struct :c:type:`v4l2_ctrl_av1_frame`, containing AV1 Frame/Frame + Header OBU decoding parameters for stateless video decoders. + * - ``V4L2_CTRL_TYPE_AV1_FILM_GRAIN`` + - n/a + - n/a + - n/a + - A struct :c:type:`v4l2_ctrl_av1_film_grain`, containing AV1 Film Grain + parameters for stateless video decoders. .. raw:: latex
Documentation/userspace-api/media/videodev2.h.rst.exceptions +4 -0
@@ -161,6 +161,10 @@ replace symbol V4L2_CTRL_TYPE_HEVC_PPS :c:type:`v4l2_ctrl_type` replace symbol V4L2_CTRL_TYPE_HEVC_SLICE_PARAMS :c:type:`v4l2_ctrl_type` replace symbol V4L2_CTRL_TYPE_HEVC_SCALING_MATRIX :c:type:`v4l2_ctrl_type` replace symbol V4L2_CTRL_TYPE_HEVC_DECODE_PARAMS :c:type:`v4l2_ctrl_type` +replace symbol V4L2_CTRL_TYPE_AV1_SEQUENCE :c:type:`v4l2_ctrl_type` +replace symbol V4L2_CTRL_TYPE_AV1_TILE_GROUP_ENTRY :c:type:`v4l2_ctrl_type` +replace symbol V4L2_CTRL_TYPE_AV1_FRAME :c:type:`v4l2_ctrl_type` +replace symbol V4L2_CTRL_TYPE_AV1_FILM_GRAIN :c:type:`v4l2_ctrl_type` # V4L2 capability defines replace define V4L2_CAP_VIDEO_CAPTURE device-capabilities
drivers/media/v4l2-core/v4l2-ctrls-core.c +258 -0
282 lines not on this page · raw message
drivers/media/v4l2-core/v4l2-ctrls-defs.c +61 -0
97 lines not on this page · raw message
drivers/media/v4l2-core/v4l2-ioctl.c +1 -0
@@ -1495,6 +1495,7 @@ static void v4l_fill_fmtdesc(struct v4l2_fmtdesc *fmt)case V4L2_PIX_FMT_QC08C : descr ="QCOM Compressed 8-bit Format" ;break ;case V4L2_PIX_FMT_QC10C : descr ="QCOM Compressed 10-bit Format" ;break ;case V4L2_PIX_FMT_AJPG : descr ="Aspeed JPEG" ;break ; +case V4L2_PIX_FMT_AV1_FRAME : descr ="AV1 Frame" ;break ;default :if (fmt->description[0 ])return ;
include/media/v4l2-ctrls.h +8 -0
@@ -52,6 +52,10 @@ struct video_device;* @p_hdr10_cll: Pointer to an HDR10 Content Light Level structure. * @p_hdr10_mastering: Pointer to an HDR10 Mastering Display structure. * @p_area: Pointer to an area. +* @p_av1_sequence: Pointer to an AV1 sequence structure. +* @p_av1_tile_group_entry: Pointer to an AV1 tile group entry structure. +* @p_av1_frame: Pointer to an AV1 frame structure. +* @p_av1_film_grain: Pointer to an AV1 film grain structure. * @p: Pointer to a compound value. * @p_const: Pointer to a constant compound value. */
@@ -81,6 +85,10 @@ union v4l2_ctrl_ptr {struct v4l2_ctrl_hdr10_cll_info *p_hdr10_cll;struct v4l2_ctrl_hdr10_mastering_display *p_hdr10_mastering;struct v4l2_area *p_area; +struct v4l2_ctrl_av1_sequence *p_av1_sequence; +struct v4l2_ctrl_av1_tile_group_entry *p_av1_tile_group_entry; +struct v4l2_ctrl_av1_frame *p_av1_frame; +struct v4l2_ctrl_av1_film_grain *p_av1_film_grain;void *p;const void *p_const; };
include/uapi/linux/v4l2-controls.h +721 -0
733 lines not on this page · raw message
include/uapi/linux/videodev2.h +10 -0
@@ -739,6 +739,7 @@ struct v4l2_pix_format {#define V4L2_PIX_FMT_FWHT_STATELESS v4l2_fourcc('S', 'F', 'W', 'H')/* Stateless FWHT (vicodec) */ #define V4L2_PIX_FMT_H264_SLICE v4l2_fourcc('S', '2', '6', '4')/* H264 parsed slices */ #define V4L2_PIX_FMT_HEVC_SLICE v4l2_fourcc('S', '2', '6', '5')/* HEVC parsed slices */ +#define V4L2_PIX_FMT_AV1_FRAME v4l2_fourcc('A', 'V', '1', 'F')/* AV1 parsed frame */ /* Vendor-specific formats */ #define V4L2_PIX_FMT_CPIA1 v4l2_fourcc('C', 'P', 'I', 'A')/* cpia1 YUV */
@@ -1804,6 +1805,10 @@ struct v4l2_ext_control {struct v4l2_ctrl_hevc_slice_params __user *p_hevc_slice_params;struct v4l2_ctrl_hevc_scaling_matrix __user *p_hevc_scaling_matrix;struct v4l2_ctrl_hevc_decode_params __user *p_hevc_decode_params; +struct v4l2_ctrl_av1_sequence __user *p_av1_sequence; +struct v4l2_ctrl_av1_tile_group_entry __user *p_av1_tile_group_entry; +struct v4l2_ctrl_av1_frame __user *p_av1_frame; +struct v4l2_ctrl_av1_film_grain __user *p_av1_film_grain;void __user *ptr; }; } __attribute__ ((packed));
@@ -1877,6 +1882,11 @@ enum v4l2_ctrl_type {V4L2_CTRL_TYPE_HEVC_SLICE_PARAMS =0x0272 ,V4L2_CTRL_TYPE_HEVC_SCALING_MATRIX =0x0273 ,V4L2_CTRL_TYPE_HEVC_DECODE_PARAMS =0x0274 , + +V4L2_CTRL_TYPE_AV1_SEQUENCE =0x280 , +V4L2_CTRL_TYPE_AV1_TILE_GROUP_ENTRY =0x281 , +V4L2_CTRL_TYPE_AV1_FRAME =0x282 , + V4L2_CTRL_TYPE_AV1_FILM_GRAIN = 0x283, };/* Used in the VIDIOC_QUERYCTRL ioctl for querying controls */
--
2.39.2