Skip to contentSkip to conversation

media: Add AV1 uAPI

PATCH New Daniel Almeida 1 participant 1 message Mar 6, 2023 Media (V4L/DVB)

lore
Share & tools
lore
#0 lore raw

PATCH v7 1/1 media: Add AV1 uAPI

This patch adds the AOMedia Video 1 (AV1) kernel uAPI.

This design is based on currently available AV1 API implementations and aims to support the development of AV1 stateless video codecs on Linux.

---
Changes from v6:
- Fix remaining padding issues.

pahole output for arm64:

struct v4l2_ctrl_av1_sequence {

	__u32                      flags;                /*     0     4 */
	__u8                       seq_profile;          /*     4     1 */
	__u8                       order_hint_bits;      /*     5     1 */
	__u8                       bit_depth;            /*     6     1 */
	__u8                       reserved;             /*     7     1 */
	__u16                      max_frame_width_minus_1; /*     8     2 */
	__u16                      max_frame_height_minus_1; /*    10     2 */
	/* size: 12, cachelines: 1, members: 7 */
	/* last cacheline: 12 bytes */
};
struct v4l2_ctrl_av1_tile_group_entry {
	__u32                      tile_offset;          /*     0     4 */
	__u32                      tile_size;            /*     4     4 */
	__u32                      tile_row;             /*     8     4 */
	__u32                      tile_col;             /*    12     4 */
	/* size: 16, cachelines: 1, members: 4 */
	/* last cacheline: 16 bytes */
};
struct v4l2_av1_global_motion {
	__u8                       flags[8];             /*     0     8 */
	enum v4l2_av1_warp_model   type[8];              /*     8    32 */
	__s32                      params[8][6];         /*    40   192 */
	/* --- cacheline 3 boundary (192 bytes) was 40 bytes ago --- */
	__u8                       invalid;              /*   232     1 */
	__u8                       reserved[3];          /*   233     3 */

	/* size: 236, cachelines: 4, members: 5 */
	/* last cacheline: 44 bytes */
};
struct v4l2_av1_loop_restoration {
	__u8                       flags;                /*     0     1 */
	__u8                       lr_unit_shift;        /*     1     1 */
	__u8                       lr_uv_shift;          /*     2     1 */
	__u8                       reserved;             /*     3     1 */
	enum v4l2_av1_frame_restoration_type frame_restoration_type[3]; /*     4    12 */
	__u32                      loop_restoration_size[3]; /*    16    12 */

	/* size: 28, cachelines: 1, members: 6 */
	/* last cacheline: 28 bytes */
};
struct v4l2_av1_cdef {
	__u8                       damping_minus_3;      /*     0     1 */
	__u8                       bits;                 /*     1     1 */
	__u8                       y_pri_strength[8];    /*     2     8 */
	__u8                       y_sec_strength[8];    /*    10     8 */
	__u8                       uv_pri_strength[8];   /*    18     8 */
	__u8                       uv_sec_strength[8];   /*    26     8 */
	/* size: 34, cachelines: 1, members: 6 */
	/* last cacheline: 34 bytes */
};
struct v4l2_av1_segmentation {
	__u8                       flags;                /*     0     1 */
	__u8                       last_active_seg_id;   /*     1     1 */
	__u8                       feature_enabled[8];   /*     2     8 */
	__s16                      feature_data[8][8];   /*    10   128 */

	/* size: 138, cachelines: 3, members: 4 */
	/* last cacheline: 10 bytes */
};
struct v4l2_av1_loop_filter {
	__u8                       flags;                /*     0     1 */
	__u8                       level[4];             /*     1     4 */
	__u8                       sharpness;            /*     5     1 */
	__s8                       ref_deltas[8];        /*     6     8 */
	__s8                       mode_deltas[2];       /*    14     2 */
	__u8                       delta_lf_res;         /*    16     1 */
	/* size: 17, cachelines: 1, members: 6 */
	/* last cacheline: 17 bytes */
};
struct v4l2_av1_quantization {
	__u8                       flags;                /*     0     1 */
	__u8                       base_q_idx;           /*     1     1 */
	__s8                       delta_q_y_dc;         /*     2     1 */
	__s8                       delta_q_u_dc;         /*     3     1 */
	__s8                       delta_q_u_ac;         /*     4     1 */
	__s8                       delta_q_v_dc;         /*     5     1 */
	__s8                       delta_q_v_ac;         /*     6     1 */
	__u8                       qm_y;                 /*     7     1 */
	__u8                       qm_u;                 /*     8     1 */
	__u8                       qm_v;                 /*     9     1 */
	__u8                       delta_q_res;          /*    10     1 */
	/* size: 11, cachelines: 1, members: 11 */
	/* last cacheline: 11 bytes */
};
struct v4l2_av1_tile_info {
	__u8                       flags;                /*     0     1 */
	__u8                       context_update_tile_id; /*     1     1 */
	__u8                       tile_cols;            /*     2     1 */
	__u8                       tile_rows;            /*     3     1 */
	__u32                      mi_col_starts[65];    /*     4   260 */
	/* --- cacheline 4 boundary (256 bytes) was 8 bytes ago --- */
	__u32                      mi_row_starts[65];    /*   264   260 */
	/* --- cacheline 8 boundary (512 bytes) was 12 bytes ago --- */
	__u32                      width_in_sbs_minus_1[64]; /*   524   256 */
	/* --- cacheline 12 boundary (768 bytes) was 12 bytes ago --- */
	__u32                      height_in_sbs_minus_1[64]; /*   780   256 */
	/* --- cacheline 16 boundary (1024 bytes) was 12 bytes ago --- */
	__u8                       tile_size_bytes;      /*  1036     1 */
	__u8                       padding[3];           /*  1037     3 */

	/* size: 1040, cachelines: 17, members: 10 */
	/* last cacheline: 16 bytes */
};
struct v4l2_ctrl_av1_frame {
	struct v4l2_av1_tile_info  tile_info;            /*     0  1040 */
	/* --- cacheline 16 boundary (1024 bytes) was 16 bytes ago --- */
	struct v4l2_av1_quantization quantization;       /*  1040    11 */
	__u8                       superres_denom;       /*  1051     1 */
	struct v4l2_av1_segmentation segmentation;       /*  1052   138 */
	/* --- cacheline 18 boundary (1152 bytes) was 38 bytes ago --- */
	struct v4l2_av1_loop_filter loop_filter;         /*  1190    17 */
	struct v4l2_av1_cdef       cdef;                 /*  1207    34 */
	/* --- cacheline 19 boundary (1216 bytes) was 25 bytes ago --- */
	__u8                       skip_mode_frame[2];   /*  1241     2 */
	__u8                       primary_ref_frame;    /*  1243     1 */
	struct v4l2_av1_loop_restoration loop_restoration; /*  1244    28 */
	struct v4l2_av1_global_motion global_motion;     /*  1272   236 */
	/* --- cacheline 23 boundary (1472 bytes) was 36 bytes ago --- */
	__u32                      flags;                /*  1508     4 */
	enum v4l2_av1_frame_type   frame_type;           /*  1512     4 */
	__u32                      order_hint;           /*  1516     4 */
	__u32                      upscaled_width;       /*  1520     4 */
	enum v4l2_av1_interpolation_filter interpolation_filter; /*  1524     4 */
	enum v4l2_av1_tx_mode      tx_mode;              /*  1528     4 */
	__u32                      frame_width_minus_1;  /*  1532     4 */
	/* --- cacheline 24 boundary (1536 bytes) --- */
	__u32                      frame_height_minus_1; /*  1536     4 */
	__u16                      render_width_minus_1; /*  1540     2 */
	__u16                      render_height_minus_1; /*  1542     2 */
	__u32                      current_frame_id;     /*  1544     4 */
	__u32                      buffer_removal_time[32]; /*  1548   128 */
	/* --- cacheline 26 boundary (1664 bytes) was 12 bytes ago --- */
	__u8                       reserved[4];          /*  1676     4 */
	__u32                      order_hints[8];       /*  1680    32 */
	__u64                      reference_frame_ts[8]; /*  1712    64 */
	/* --- cacheline 27 boundary (1728 bytes) was 48 bytes ago --- */
	__s8                       ref_frame_idx[7];     /*  1776     7 */
	__u8                       refresh_frame_flags;  /*  1783     1 */

	/* size: 1784, cachelines: 28, members: 27 */
	/* last cacheline: 56 bytes */
};
struct v4l2_ctrl_av1_film_grain {
	__u8                       flags;                /*     0     1 */
	__u8                       cr_mult;              /*     1     1 */
	__u16                      grain_seed;           /*     2     2 */
	__u8                       film_grain_params_ref_idx; /*     4     1 */
	__u8                       num_y_points;         /*     5     1 */
	__u8                       point_y_value[16];    /*     6    16 */
	__u8                       point_y_scaling[16];  /*    22    16 */
	__u8                       num_cb_points;        /*    38     1 */
	__u8                       point_cb_value[16];   /*    39    16 */
	__u8                       point_cb_scaling[16]; /*    55    16 */

/* --- cacheline 1 boundary (64 bytes) was 7 bytes ago --- */
__u8 num_cr_points; /* 71 1 */
__u8 point_cr_value[16]; /* 72 16 */
__u8 point_cr_scaling[16]; /* 88 16 */
__u8 grain_scaling_minus_8; /* 104 1 */
__u8 ar_coeff_lag; /* 105 1 */
__u8 ar_coeffs_y_plus_128[25]; /* 106 25 */
/* --- cacheline 2 boundary (128 bytes) was 3 bytes ago --- */
__u8 ar_coeffs_cb_plus_128[25]; /* 131 25 */
__u8 ar_coeffs_cr_plus_128[25]; /* 156 25 */
__u8 ar_coeff_shift_minus_6; /* 181 1 */
__u8 grain_scale_shift; /* 182 1 */
__u8 cb_mult; /* 183 1 */
__u8 cb_luma_mult; /* 184 1 */
__u8 cr_luma_mult; /* 185 1 */
__u16 cb_offset; /* 186 2 */
__u16 cr_offset; /* 188 2 */
__u8 reserved[4]; /* 190 4 */

/* size: 194, cachelines: 4, members: 26 */ /* last cacheline: 2 bytes */
};

pahole output for x64:

struct v4l2_ctrl_av1_sequence {

	__u32                      flags;                /*     0     4 */
	__u8                       seq_profile;          /*     4     1 */
	__u8                       order_hint_bits;      /*     5     1 */
	__u8                       bit_depth;            /*     6     1 */
	__u8                       reserved;             /*     7     1 */
	__u16                      max_frame_width_minus_1; /*     8     2 */
	__u16                      max_frame_height_minus_1; /*    10     2 */
	/* size: 12, cachelines: 1, members: 7 */
	/* last cacheline: 12 bytes */
};
struct v4l2_ctrl_av1_tile_group_entry {
	__u32                      tile_offset;          /*     0     4 */
	__u32                      tile_size;            /*     4     4 */
	__u32                      tile_row;             /*     8     4 */
	__u32                      tile_col;             /*    12     4 */
	/* size: 16, cachelines: 1, members: 4 */
	/* last cacheline: 16 bytes */
};
struct v4l2_av1_global_motion {
	__u8                       flags[8];             /*     0     8 */
	enum v4l2_av1_warp_model   type[8];              /*     8    32 */
	__s32                      params[8][6];         /*    40   192 */
	/* --- cacheline 3 boundary (192 bytes) was 40 bytes ago --- */
	__u8                       invalid;              /*   232     1 */
	__u8                       reserved[3];          /*   233     3 */

	/* size: 236, cachelines: 4, members: 5 */
	/* last cacheline: 44 bytes */
};
struct v4l2_av1_loop_restoration {
	__u8                       flags;                /*     0     1 */
	__u8                       lr_unit_shift;        /*     1     1 */
	__u8                       lr_uv_shift;          /*     2     1 */
	__u8                       reserved;             /*     3     1 */
	enum v4l2_av1_frame_restoration_type frame_restoration_type[3]; /*     4    12 */
	__u32                      loop_restoration_size[3]; /*    16    12 */

	/* size: 28, cachelines: 1, members: 6 */
	/* last cacheline: 28 bytes */
};
struct v4l2_av1_cdef {
	__u8                       damping_minus_3;      /*     0     1 */
	__u8                       bits;                 /*     1     1 */
	__u8                       y_pri_strength[8];    /*     2     8 */
	__u8                       y_sec_strength[8];    /*    10     8 */
	__u8                       uv_pri_strength[8];   /*    18     8 */
	__u8                       uv_sec_strength[8];   /*    26     8 */
	/* size: 34, cachelines: 1, members: 6 */
	/* last cacheline: 34 bytes */
};
struct v4l2_av1_segmentation {
	__u8                       flags;                /*     0     1 */
	__u8                       last_active_seg_id;   /*     1     1 */
	__u8                       feature_enabled[8];   /*     2     8 */
	__s16                      feature_data[8][8];   /*    10   128 */

	/* size: 138, cachelines: 3, members: 4 */
	/* last cacheline: 10 bytes */
};
struct v4l2_av1_loop_filter {
	__u8                       flags;                /*     0     1 */
	__u8                       level[4];             /*     1     4 */
	__u8                       sharpness;            /*     5     1 */
	__s8                       ref_deltas[8];        /*     6     8 */
	__s8                       mode_deltas[2];       /*    14     2 */
	__u8                       delta_lf_res;         /*    16     1 */
	/* size: 17, cachelines: 1, members: 6 */
	/* last cacheline: 17 bytes */
};
struct v4l2_av1_quantization {
	__u8                       flags;                /*     0     1 */
	__u8                       base_q_idx;           /*     1     1 */
	__s8                       delta_q_y_dc;         /*     2     1 */
	__s8                       delta_q_u_dc;         /*     3     1 */
	__s8                       delta_q_u_ac;         /*     4     1 */
	__s8                       delta_q_v_dc;         /*     5     1 */
	__s8                       delta_q_v_ac;         /*     6     1 */
	__u8                       qm_y;                 /*     7     1 */
	__u8                       qm_u;                 /*     8     1 */
	__u8                       qm_v;                 /*     9     1 */
	__u8                       delta_q_res;          /*    10     1 */
	/* size: 11, cachelines: 1, members: 11 */
	/* last cacheline: 11 bytes */
};
struct v4l2_av1_tile_info {
	__u8                       flags;                /*     0     1 */
	__u8                       context_update_tile_id; /*     1     1 */
	__u8                       tile_cols;            /*     2     1 */
	__u8                       tile_rows;            /*     3     1 */
	__u32                      mi_col_starts[65];    /*     4   260 */
	/* --- cacheline 4 boundary (256 bytes) was 8 bytes ago --- */
	__u32                      mi_row_starts[65];    /*   264   260 */
	/* --- cacheline 8 boundary (512 bytes) was 12 bytes ago --- */
	__u32                      width_in_sbs_minus_1[64]; /*   524   256 */
	/* --- cacheline 12 boundary (768 bytes) was 12 bytes ago --- */
	__u32                      height_in_sbs_minus_1[64]; /*   780   256 */
	/* --- cacheline 16 boundary (1024 bytes) was 12 bytes ago --- */
	__u8                       tile_size_bytes;      /*  1036     1 */
	__u8                       padding[3];           /*  1037     3 */

	/* size: 1040, cachelines: 17, members: 10 */
	/* last cacheline: 16 bytes */
};
struct v4l2_ctrl_av1_frame {
	struct v4l2_av1_tile_info  tile_info;            /*     0  1040 */
	/* --- cacheline 16 boundary (1024 bytes) was 16 bytes ago --- */
	struct v4l2_av1_quantization quantization;       /*  1040    11 */
	__u8                       superres_denom;       /*  1051     1 */
	struct v4l2_av1_segmentation segmentation;       /*  1052   138 */
	/* --- cacheline 18 boundary (1152 bytes) was 38 bytes ago --- */
	struct v4l2_av1_loop_filter loop_filter;         /*  1190    17 */
	struct v4l2_av1_cdef       cdef;                 /*  1207    34 */
	/* --- cacheline 19 boundary (1216 bytes) was 25 bytes ago --- */
	__u8                       skip_mode_frame[2];   /*  1241     2 */
	__u8                       primary_ref_frame;    /*  1243     1 */
	struct v4l2_av1_loop_restoration loop_restoration; /*  1244    28 */
	struct v4l2_av1_global_motion global_motion;     /*  1272   236 */
	/* --- cacheline 23 boundary (1472 bytes) was 36 bytes ago --- */
	__u32                      flags;                /*  1508     4 */
	enum v4l2_av1_frame_type   frame_type;           /*  1512     4 */
	__u32                      order_hint;           /*  1516     4 */
	__u32                      upscaled_width;       /*  1520     4 */
	enum v4l2_av1_interpolation_filter interpolation_filter; /*  1524     4 */
	enum v4l2_av1_tx_mode      tx_mode;              /*  1528     4 */
	__u32                      frame_width_minus_1;  /*  1532     4 */
	/* --- cacheline 24 boundary (1536 bytes) --- */
	__u32                      frame_height_minus_1; /*  1536     4 */
	__u16                      render_width_minus_1; /*  1540     2 */
	__u16                      render_height_minus_1; /*  1542     2 */
	__u32                      current_frame_id;     /*  1544     4 */
	__u32                      buffer_removal_time[32]; /*  1548   128 */
	/* --- cacheline 26 boundary (1664 bytes) was 12 bytes ago --- */
	__u8                       reserved[4];          /*  1676     4 */
	__u32                      order_hints[8];       /*  1680    32 */
	__u64                      reference_frame_ts[8]; /*  1712    64 */
	/* --- cacheline 27 boundary (1728 bytes) was 48 bytes ago --- */
	__s8                       ref_frame_idx[7];     /*  1776     7 */
	__u8                       refresh_frame_flags;  /*  1783     1 */

	/* size: 1784, cachelines: 28, members: 27 */
	/* last cacheline: 56 bytes */
};
struct v4l2_ctrl_av1_film_grain {
	__u8                       flags;                /*     0     1 */
	__u8                       cr_mult;              /*     1     1 */
	__u16                      grain_seed;           /*     2     2 */
	__u8                       film_grain_params_ref_idx; /*     4     1 */
	__u8                       num_y_points;         /*     5     1 */
	__u8                       point_y_value[16];    /*     6    16 */
	__u8                       point_y_scaling[16];  /*    22    16 */
	__u8                       num_cb_points;        /*    38     1 */
	__u8                       point_cb_value[16];   /*    39    16 */
	__u8                       point_cb_scaling[16]; /*    55    16 */

/* --- cacheline 1 boundary (64 bytes) was 7 bytes ago --- */
__u8 num_cr_points; /* 71 1 */
__u8 point_cr_value[16]; /* 72 16 */
__u8 point_cr_scaling[16]; /* 88 16 */
__u8 grain_scaling_minus_8; /* 104 1 */
__u8 ar_coeff_lag; /* 105 1 */
__u8 ar_coeffs_y_plus_128[25]; /* 106 25 */
/* --- cacheline 2 boundary (128 bytes) was 3 bytes ago --- */
__u8 ar_coeffs_cb_plus_128[25]; /* 131 25 */
__u8 ar_coeffs_cr_plus_128[25]; /* 156 25 */
__u8 ar_coeff_shift_minus_6; /* 181 1 */
__u8 grain_scale_shift; /* 182 1 */
__u8 cb_mult; /* 183 1 */
__u8 cb_luma_mult; /* 184 1 */
__u8 cr_luma_mult; /* 185 1 */
__u16 cb_offset; /* 186 2 */
__u16 cr_offset; /* 188 2 */
__u8 reserved[4]; /* 190 4 */

/* size: 194, cachelines: 4, members: 26 */ /* last cacheline: 2 bytes */
};

pahole output for x86:

struct v4l2_ctrl_av1_sequence {

	__u32                      flags;                /*     0     4 */
	__u8                       seq_profile;          /*     4     1 */
	__u8                       order_hint_bits;      /*     5     1 */
	__u8                       bit_depth;            /*     6     1 */
	__u8                       reserved;             /*     7     1 */
	__u16                      max_frame_width_minus_1; /*     8     2 */
	__u16                      max_frame_height_minus_1; /*    10     2 */
	/* size: 12, cachelines: 1, members: 7 */
	/* last cacheline: 12 bytes */
};
struct v4l2_ctrl_av1_tile_group_entry {
	__u32                      tile_offset;          /*     0     4 */
	__u32                      tile_size;            /*     4     4 */
	__u32                      tile_row;             /*     8     4 */
	__u32                      tile_col;             /*    12     4 */
	/* size: 16, cachelines: 1, members: 4 */
	/* last cacheline: 16 bytes */
};
struct v4l2_av1_global_motion {
	__u8                       flags[8];             /*     0     8 */
	enum v4l2_av1_warp_model   type[8];              /*     8    32 */
	__s32                      params[8][6];         /*    40   192 */
	/* --- cacheline 3 boundary (192 bytes) was 40 bytes ago --- */
	__u8                       invalid;              /*   232     1 */
	__u8                       reserved[3];          /*   233     3 */

	/* size: 236, cachelines: 4, members: 5 */
	/* last cacheline: 44 bytes */
};
struct v4l2_av1_loop_restoration {
	__u8                       flags;                /*     0     1 */
	__u8                       lr_unit_shift;        /*     1     1 */
	__u8                       lr_uv_shift;          /*     2     1 */
	__u8                       reserved;             /*     3     1 */
	enum v4l2_av1_frame_restoration_type frame_restoration_type[3]; /*     4    12 */
	__u32                      loop_restoration_size[3]; /*    16    12 */

	/* size: 28, cachelines: 1, members: 6 */
	/* last cacheline: 28 bytes */
};
struct v4l2_av1_cdef {
	__u8                       damping_minus_3;      /*     0     1 */
	__u8                       bits;                 /*     1     1 */
	__u8                       y_pri_strength[8];    /*     2     8 */
	__u8                       y_sec_strength[8];    /*    10     8 */
	__u8                       uv_pri_strength[8];   /*    18     8 */
	__u8                       uv_sec_strength[8];   /*    26     8 */
	/* size: 34, cachelines: 1, members: 6 */
	/* last cacheline: 34 bytes */
};
struct v4l2_av1_segmentation {
	__u8                       flags;                /*     0     1 */
	__u8                       last_active_seg_id;   /*     1     1 */
	__u8                       feature_enabled[8];   /*     2     8 */
	__s16                      feature_data[8][8];   /*    10   128 */

	/* size: 138, cachelines: 3, members: 4 */
	/* last cacheline: 10 bytes */
};
struct v4l2_av1_loop_filter {
	__u8                       flags;                /*     0     1 */
	__u8                       level[4];             /*     1     4 */
	__u8                       sharpness;            /*     5     1 */
	__s8                       ref_deltas[8];        /*     6     8 */
	__s8                       mode_deltas[2];       /*    14     2 */
	__u8                       delta_lf_res;         /*    16     1 */
	/* size: 17, cachelines: 1, members: 6 */
	/* last cacheline: 17 bytes */
};
struct v4l2_av1_quantization {
	__u8                       flags;                /*     0     1 */
	__u8                       base_q_idx;           /*     1     1 */
	__s8                       delta_q_y_dc;         /*     2     1 */
	__s8                       delta_q_u_dc;         /*     3     1 */
	__s8                       delta_q_u_ac;         /*     4     1 */
	__s8                       delta_q_v_dc;         /*     5     1 */
	__s8                       delta_q_v_ac;         /*     6     1 */
	__u8                       qm_y;                 /*     7     1 */
	__u8                       qm_u;                 /*     8     1 */
	__u8                       qm_v;                 /*     9     1 */
	__u8                       delta_q_res;          /*    10     1 */
	/* size: 11, cachelines: 1, members: 11 */
	/* last cacheline: 11 bytes */
};
struct v4l2_av1_tile_info {
	__u8                       flags;                /*     0     1 */
	__u8                       context_update_tile_id; /*     1     1 */
	__u8                       tile_cols;            /*     2     1 */
	__u8                       tile_rows;            /*     3     1 */
	__u32                      mi_col_starts[65];    /*     4   260 */
	/* --- cacheline 4 boundary (256 bytes) was 8 bytes ago --- */
	__u32                      mi_row_starts[65];    /*   264   260 */
	/* --- cacheline 8 boundary (512 bytes) was 12 bytes ago --- */
	__u32                      width_in_sbs_minus_1[64]; /*   524   256 */
	/* --- cacheline 12 boundary (768 bytes) was 12 bytes ago --- */
	__u32                      height_in_sbs_minus_1[64]; /*   780   256 */
	/* --- cacheline 16 boundary (1024 bytes) was 12 bytes ago --- */
	__u8                       tile_size_bytes;      /*  1036     1 */
	__u8                       padding[3];           /*  1037     3 */

	/* size: 1040, cachelines: 17, members: 10 */
	/* last cacheline: 16 bytes */
};
struct v4l2_ctrl_av1_frame {
	struct v4l2_av1_tile_info  tile_info;            /*     0  1040 */
	/* --- cacheline 16 boundary (1024 bytes) was 16 bytes ago --- */
	struct v4l2_av1_quantization quantization;       /*  1040    11 */
	__u8                       superres_denom;       /*  1051     1 */
	struct v4l2_av1_segmentation segmentation;       /*  1052   138 */
	/* --- cacheline 18 boundary (1152 bytes) was 38 bytes ago --- */
	struct v4l2_av1_loop_filter loop_filter;         /*  1190    17 */
	struct v4l2_av1_cdef       cdef;                 /*  1207    34 */
	/* --- cacheline 19 boundary (1216 bytes) was 25 bytes ago --- */
	__u8                       skip_mode_frame[2];   /*  1241     2 */
	__u8                       primary_ref_frame;    /*  1243     1 */
	struct v4l2_av1_loop_restoration loop_restoration; /*  1244    28 */
	struct v4l2_av1_global_motion global_motion;     /*  1272   236 */
	/* --- cacheline 23 boundary (1472 bytes) was 36 bytes ago --- */
	__u32                      flags;                /*  1508     4 */
	enum v4l2_av1_frame_type   frame_type;           /*  1512     4 */
	__u32                      order_hint;           /*  1516     4 */
	__u32                      upscaled_width;       /*  1520     4 */
	enum v4l2_av1_interpolation_filter interpolation_filter; /*  1524     4 */
	enum v4l2_av1_tx_mode      tx_mode;              /*  1528     4 */
	__u32                      frame_width_minus_1;  /*  1532     4 */
	/* --- cacheline 24 boundary (1536 bytes) --- */
	__u32                      frame_height_minus_1; /*  1536     4 */
	__u16                      render_width_minus_1; /*  1540     2 */
	__u16                      render_height_minus_1; /*  1542     2 */
	__u32                      current_frame_id;     /*  1544     4 */
	__u32                      buffer_removal_time[32]; /*  1548   128 */
	/* --- cacheline 26 boundary (1664 bytes) was 12 bytes ago --- */
	__u8                       reserved[4];          /*  1676     4 */
	__u32                      order_hints[8];       /*  1680    32 */
	__u64                      reference_frame_ts[8]; /*  1712    64 */
	/* --- cacheline 27 boundary (1728 bytes) was 48 bytes ago --- */
	__s8                       ref_frame_idx[7];     /*  1776     7 */
	__u8                       refresh_frame_flags;  /*  1783     1 */

	/* size: 1784, cachelines: 28, members: 27 */
	/* last cacheline: 56 bytes */
};
struct v4l2_ctrl_av1_film_grain {
	__u8                       flags;                /*     0     1 */
	__u8                       cr_mult;              /*     1     1 */
	__u16                      grain_seed;           /*     2     2 */
	__u8                       film_grain_params_ref_idx; /*     4     1 */
	__u8                       num_y_points;         /*     5     1 */
	__u8                       point_y_value[16];    /*     6    16 */
	__u8                       point_y_scaling[16];  /*    22    16 */
	__u8                       num_cb_points;        /*    38     1 */
	__u8                       point_cb_value[16];   /*    39    16 */
	__u8                       point_cb_scaling[16]; /*    55    16 */

/* --- cacheline 1 boundary (64 bytes) was 7 bytes ago --- */
__u8 num_cr_points; /* 71 1 */
__u8 point_cr_value[16]; /* 72 16 */
__u8 point_cr_scaling[16]; /* 88 16 */
__u8 grain_scaling_minus_8; /* 104 1 */
__u8 ar_coeff_lag; /* 105 1 */
__u8 ar_coeffs_y_plus_128[25]; /* 106 25 */
/* --- cacheline 2 boundary (128 bytes) was 3 bytes ago --- */
__u8 ar_coeffs_cb_plus_128[25]; /* 131 25 */
__u8 ar_coeffs_cr_plus_128[25]; /* 156 25 */
__u8 ar_coeff_shift_minus_6; /* 181 1 */
__u8 grain_scale_shift; /* 182 1 */
__u8 cb_mult; /* 183 1 */
__u8 cb_luma_mult; /* 184 1 */
__u8 cr_luma_mult; /* 185 1 */
__u16 cb_offset; /* 186 2 */
__u16 cr_offset; /* 188 2 */
__u8 reserved[4]; /* 190 4 */

/* size: 194, cachelines: 4, members: 26 */ /* last cacheline: 2 bytes */
};

- Made sure all structs show no difference in size for the archs above.
- Added a padding field to the film grain control, to be on the safer side.
- Fixed more issues with the docs (thanks, Nicolas)

Changes from v5:

- Fix kernel-doc issues from scripts/kernel-doc -none include/uapi/linux/v4l2-controls.h

Changes from v4:

- Again tested on MT8195/RK3588.
- Ran pahole, made sure there are no holes or size differences between different archs. Updated .rst docs as needed.
- Fixed doc issues pointed out by Hans in his last review (Thanks, Hans!)
- Remove uniform_tile_spacing from the frame flags. That flag was duplicated given the flag in the tile_info structure with the same name.
- Fixed whitespace issues (hopefully)
- Fixed a small typo in the docs as pointed out by Brian Norris. Thanks, Brian!
- Add mention to full vs limited range quantization where applicable.
- Fix incorrect validation for the loop filter flags. Previous validation code accepted up to 0b111, but there are four possible flags defined.
- Fixed issue with the validation of feature_data as pointed out by Hans.
- Fixed checkpatch warnings (ran with --strict)

Changes from v3:

- Dropped RFC tag.
- Tested on MT8195 and RK3588 SoCs
- Remove NUM_REF_FRAMES. This field was duplicated.
- Address Chen-Yu's review of v3. Thanks, Chen-Yu!
- Change unicode dash to ascii in the AV1 documentation. This would trigger some warnings from time to time.
- Change ref_frame_idx from u8 to s8. ref_frame_idx is initialized to -1 in a few places in the spec.
- Move AV1 profile and level CIDs to MPEG class. These can be used by a stateful implementation.
- Fix order_hints docs, replace it with something more descriptive.
- Change gm_params to s32
- Remove [last|golden]_frame_idx. These are not needed in light of ref_frame_idx.

Changes from v2:

- Bump AV1 CIDs to HEVC+100

In light of the HEVC upstreaming, bump the AV1 CIDs to the values used by HEVC by 100, in order to avoid conflicts. The range previously used by AV1 was taken by HEVC, as it was merged first.

- Fix typo in enum v4l2_av1_frame_restoration_type

- Change u8 to __u8 in struct v4l2_av1_loop_restoration

- Rename v4l2_ctrl_av1_frame_header to v4l2_ctrl_av1_frame

- Fix a small typo

- Also include 6.8.16 in the loop filter docs

- delta_lf_multi is now a flag

- Stop using the BIT macro

- Move the AV1 CIDs below the VP9 ones

- Remove useless break

- Fix wrong docs for ref_frame_idx

- Remove tile_group control.

Data in this control is not currently used. The fact that VA-API does not have a similar Buffer in its API hints that it probably will not be needed in the future as well.

- Move av1 CIDs to the bottom of the stateless block

- Move V4L2_CTRL_TYPE_AV1_* to the end of the stateless block

- Remove stale film_grain doc from av1 frame

Changes from v1:

- Fixed the issues pointed out by Hans in his review (Thanks, Hans)
- Fixed the sign for ref_deltas and mode_deltas
These are signed values and will otherwise fail the validation in v4l2-ctrls-core.c.
- Fix film grain y/cb/cr points validation
We were controlling the entire array, but there is a subset of points being set, which lead to the validation to fail. Fix this by limiting the test to the number of points. The number of point is already controlled in the previous check so should be safe.
- Fix validation for Su(n) syntax
The Su(n) AV1 syntax ranges from -2^n/2 to 2^n/2 - 1, while the previous code was only validating values from -2^n/2 + 1 to 2^n/2 - 1.
- Fix order_hints and buffer_removal_time
The buffer_removal_time should have been 32 bit and the ref_order_hint got confused with order_hints. We want the
second one, as the first one is only used as a backup to
fill the second on missing last frame.
- Fix off by one error in validation code
There are 5 segmentation flags, so any code that sets the most significant flag would automatically fail.
- Feature data can be negative
Feature data can be negative if Segmentation_Feature_Signed[j] == 1, in which case it is read with the Su(n) syntax.
Reflect that by changing the data type to s16.
- Remove large scale mode controls
Large scale mode is not supported. Remove all controls associated with it. This means that all TILE_LIST_* controls are gone.
- Split v4l2_av1_film_grain into its own control
Split v4l2_av1_film_grain into its own control, as not all accelerators might support it. This control is now optional.
- Fix broken docs for v4l2_av1_interpolation_filter
Docs mention VP9 instead of AV1. This is a typo.
- Fix broken docs for v4l2_ctrl_av1_tile_group_entry
Fix a small typo in the docs.
- Fix AV1 reference lists
We need both all the references and the per frame references. This is because unlike VP9, show_existing_frame that are key frame will refresh the frame slots, which prevents the driver from keeping track since these frame are not signalled to the driver.

Instead mimic other API (like VA and DXVA) and provide a complete list of active references. The driver can then cleanup any data it was storing for references that are no longer in use, saving previous memory.
- Fix broken docs in ext-ctrls-codec-stateless.rst :c:type:struct v4l2_ctrl_av1_tile_group_entry would not resolve, it would also produce a warning when making the docs. Fix this by changing this to :c:type:v4l2_ctrl_av1_tile_group_entry instead.

---
 .../userspace-api/media/v4l/biblio.rst        |    9 +
 .../media/v4l/ext-ctrls-codec-stateless.rst   | 1209 ++++++++++++++++-
 .../media/v4l/pixfmt-compressed.rst           |   16 +
 .../media/v4l/vidioc-g-ext-ctrls.rst          |   16 +
 .../media/v4l/vidioc-queryctrl.rst            |   24 +
 .../media/videodev2.h.rst.exceptions          |    4 +
 drivers/media/v4l2-core/v4l2-ctrls-core.c     |  258 ++++
 drivers/media/v4l2-core/v4l2-ctrls-defs.c     |   61 +
 drivers/media/v4l2-core/v4l2-ioctl.c          |    1 +
 include/media/v4l2-ctrls.h                    |    8 +
 include/uapi/linux/v4l2-controls.h            |  721 ++++++++++
 include/uapi/linux/videodev2.h                |   10 +
 12 files changed, 2335 insertions(+), 2 deletions(-)
Documentation/userspace-api/media/v4l/biblio.rst +9 -0
index 9cd18c153d19..72aef1759b60 100644
@@ -427,3 +427,12 @@ VP9
 :title:     VP9 Bitstream & Decoding Process Specification
 
 :author:    Adrian Grange (Google), Peter de Rivaz (Argon Design), Jonathan Hunt (Argon Design)
+
+.. _av1:
+
+AV1
+===
+
+:title:     AV1 Bitstream & Decoding Process Specification
+
+:author:    Peter de Rivaz, Argon Design Ltd, Jack Haughton, Argon Design Ltd
Documentation/userspace-api/media/v4l/ext-ctrls-codec-stateless.rst +1207 -2

1,221 lines not on this page · raw message

Documentation/userspace-api/media/v4l/pixfmt-compressed.rst +16 -0
index 506dd3c98884..d7bf21bffa30 100644
@@ -233,6 +233,22 @@ Compressed Formats
         through the ``V4L2_CID_STATELESS_FWHT_PARAMS`` control.
 	See the :ref:`associated Codec Control ID <codec-stateless-fwht>`.
 
+    * .. _V4L2-PIX-FMT-AV1-FRAME:
+
+      - ``V4L2_PIX_FMT_AV1_FRAME``
+      - 'AV1F'
+      - AV1 parsed frame, including the frame header, as extracted from the container.
+        This format is adapted for stateless video decoders that implement a AV1
+        pipeline with the :ref:`stateless_decoder`. Metadata associated with the
+        frame to decode is required to be passed through the
+        ``V4L2_CID_STATELESS_AV1_SEQUENCE``, ``V4L2_CID_STATELESS_AV1_FRAME``,
+        and ``V4L2_CID_STATELESS_AV1_TILE_GROUP_ENTRY`` controls.
+        See the :ref:`associated Codec Control IDs <v4l2-codec-stateless-av1>`.
+        Exactly one output and one capture buffer must be provided for use with
+        this pixel format. The output buffer must contain the appropriate number
+        of macroblocks to decode a full corresponding frame to the matching
+        capture buffer.
+
 .. raw:: latex
 
     \normalsize
Documentation/userspace-api/media/v4l/vidioc-g-ext-ctrls.rst +16 -0
index 892cfeb8b988..4c44910a5c19 100644
@@ -269,6 +269,22 @@ still cause this situation.
       - ``p_hevc_decode_params``
       - A pointer to a struct :c:type:`v4l2_ctrl_hevc_decode_params`. Valid if this
         control is of type ``V4L2_CTRL_TYPE_HEVC_DECODE_PARAMS``.
+    * - struct :c:type:`v4l2_ctrl_av1_sequence` *
+      - ``p_av1_sequence``
+      - A pointer to a struct :c:type:`v4l2_ctrl_av1_sequence`. Valid if this control is
+        of type ``V4L2_CTRL_TYPE_AV1_SEQUENCE``.
+    * - struct :c:type:`v4l2_ctrl_av1_tile_group_entry` *
+      - ``p_av1_tile_group_entry``
+      - A pointer to a struct :c:type:`v4l2_ctrl_av1_tile_group_entry`. Valid if this control is
+        of type ``V4L2_CTRL_TYPE_AV1_TILE_GROUP_ENTRY``.
+    * - struct :c:type:`v4l2_ctrl_av1_frame` *
+      - ``p_av1_frame``
+      - A pointer to a struct :c:type:`v4l2_ctrl_av1_frame`. Valid if this control is
+        of type ``V4L2_CTRL_TYPE_AV1_FRAME``.
+    * - struct :c:type:`v4l2_ctrl_av1_film_grain` *
+      - ``p_av1_film_grain``
+      - A pointer to a struct :c:type:`v4l2_ctrl_av1_film_grain`. Valid if this control is
+        of type ``V4L2_CTRL_TYPE_AV1_FILM_GRAIN``.
     * - void *
       - ``ptr``
       - A pointer to a compound type which can be an N-dimensional array
Documentation/userspace-api/media/v4l/vidioc-queryctrl.rst +24 -0
index a20dfa2a933b..4d38acafe8e1 100644
@@ -525,6 +525,30 @@ See also the examples in :ref:`control`.
       - n/a
       - A struct :c:type:`v4l2_ctrl_vp9_frame`, containing VP9
 	frame decode parameters for stateless video decoders.
+    * - ``V4L2_CTRL_TYPE_AV1_SEQUENCE``
+      - n/a
+      - n/a
+      - n/a
+      - A struct :c:type:`v4l2_ctrl_av1_sequence`, containing AV1 Sequence OBU
+	decoding parameters for stateless video decoders.
+    * - ``V4L2_CTRL_TYPE_AV1_TILE_GROUP_ENTRY``
+      - n/a
+      - n/a
+      - n/a
+      - A struct :c:type:`v4l2_ctrl_av1_tile_group_entry`, containing AV1 Tile Group
+	OBU decoding parameters for stateless video decoders.
+    * - ``V4L2_CTRL_TYPE_AV1_FRAME``
+      - n/a
+      - n/a
+      - n/a
+      - A struct :c:type:`v4l2_ctrl_av1_frame`, containing AV1 Frame/Frame
+	Header OBU decoding parameters for stateless video decoders.
+    * - ``V4L2_CTRL_TYPE_AV1_FILM_GRAIN``
+      - n/a
+      - n/a
+      - n/a
+      - A struct :c:type:`v4l2_ctrl_av1_film_grain`, containing AV1 Film Grain
+        parameters for stateless video decoders.
 
 .. raw:: latex
 
Documentation/userspace-api/media/videodev2.h.rst.exceptions +4 -0
index 2a589d34b80e..3e58aac4ef0b 100644
@@ -161,6 +161,10 @@ replace symbol V4L2_CTRL_TYPE_HEVC_PPS :c:type:`v4l2_ctrl_type`
 replace symbol V4L2_CTRL_TYPE_HEVC_SLICE_PARAMS :c:type:`v4l2_ctrl_type`
 replace symbol V4L2_CTRL_TYPE_HEVC_SCALING_MATRIX :c:type:`v4l2_ctrl_type`
 replace symbol V4L2_CTRL_TYPE_HEVC_DECODE_PARAMS :c:type:`v4l2_ctrl_type`
+replace symbol V4L2_CTRL_TYPE_AV1_SEQUENCE :c:type:`v4l2_ctrl_type`
+replace symbol V4L2_CTRL_TYPE_AV1_TILE_GROUP_ENTRY :c:type:`v4l2_ctrl_type`
+replace symbol V4L2_CTRL_TYPE_AV1_FRAME :c:type:`v4l2_ctrl_type`
+replace symbol V4L2_CTRL_TYPE_AV1_FILM_GRAIN :c:type:`v4l2_ctrl_type`
 
 # V4L2 capability defines
 replace define V4L2_CAP_VIDEO_CAPTURE device-capabilities
drivers/media/v4l2-core/v4l2-ctrls-core.c +258 -0

282 lines not on this page · raw message

drivers/media/v4l2-core/v4l2-ctrls-defs.c +61 -0

97 lines not on this page · raw message

drivers/media/v4l2-core/v4l2-ioctl.c +1 -0
index b26da2650289..4b49f79bae8d 100644
@@ -1495,6 +1495,7 @@ static void v4l_fill_fmtdesc(struct v4l2_fmtdesc *fmt)
 		case V4L2_PIX_FMT_QC08C:	descr = "QCOM Compressed 8-bit Format"; break;
 		case V4L2_PIX_FMT_QC10C:	descr = "QCOM Compressed 10-bit Format"; break;
 		case V4L2_PIX_FMT_AJPG:		descr = "Aspeed JPEG"; break;
+		case V4L2_PIX_FMT_AV1_FRAME:	descr = "AV1 Frame"; break;
 		default:
 			if (fmt->description[0])
 				return;
include/media/v4l2-ctrls.h +8 -0
index e59d9a234631..90257040c7c9 100644
@@ -52,6 +52,10 @@ struct video_device;
  * @p_hdr10_cll:		Pointer to an HDR10 Content Light Level structure.
  * @p_hdr10_mastering:		Pointer to an HDR10 Mastering Display structure.
  * @p_area:			Pointer to an area.
+ * @p_av1_sequence:		Pointer to an AV1 sequence structure.
+ * @p_av1_tile_group_entry:	Pointer to an AV1 tile group entry structure.
+ * @p_av1_frame:		Pointer to an AV1 frame structure.
+ * @p_av1_film_grain:		Pointer to an AV1 film grain structure.
  * @p:				Pointer to a compound value.
  * @p_const:			Pointer to a constant compound value.
  */
@@ -81,6 +85,10 @@ union v4l2_ctrl_ptr {
 	struct v4l2_ctrl_hdr10_cll_info *p_hdr10_cll;
 	struct v4l2_ctrl_hdr10_mastering_display *p_hdr10_mastering;
 	struct v4l2_area *p_area;
+	struct v4l2_ctrl_av1_sequence *p_av1_sequence;
+	struct v4l2_ctrl_av1_tile_group_entry *p_av1_tile_group_entry;
+	struct v4l2_ctrl_av1_frame *p_av1_frame;
+	struct v4l2_ctrl_av1_film_grain *p_av1_film_grain;
 	void *p;
 	const void *p_const;
 };
include/uapi/linux/v4l2-controls.h +721 -0

733 lines not on this page · raw message

include/uapi/linux/videodev2.h +10 -0
index 1befd181a4cc..3dec002cb420 100644
@@ -739,6 +739,7 @@ struct v4l2_pix_format {
 #define V4L2_PIX_FMT_FWHT_STATELESS     v4l2_fourcc('S', 'F', 'W', 'H') /* Stateless FWHT (vicodec) */
 #define V4L2_PIX_FMT_H264_SLICE v4l2_fourcc('S', '2', '6', '4') /* H264 parsed slices */
 #define V4L2_PIX_FMT_HEVC_SLICE v4l2_fourcc('S', '2', '6', '5') /* HEVC parsed slices */
+#define V4L2_PIX_FMT_AV1_FRAME v4l2_fourcc('A', 'V', '1', 'F') /* AV1 parsed frame */
 
 /*  Vendor-specific formats   */
 #define V4L2_PIX_FMT_CPIA1    v4l2_fourcc('C', 'P', 'I', 'A') /* cpia1 YUV */
@@ -1804,6 +1805,10 @@ struct v4l2_ext_control {
 		struct v4l2_ctrl_hevc_slice_params __user *p_hevc_slice_params;
 		struct v4l2_ctrl_hevc_scaling_matrix __user *p_hevc_scaling_matrix;
 		struct v4l2_ctrl_hevc_decode_params __user *p_hevc_decode_params;
+		struct v4l2_ctrl_av1_sequence __user *p_av1_sequence;
+		struct v4l2_ctrl_av1_tile_group_entry __user *p_av1_tile_group_entry;
+		struct v4l2_ctrl_av1_frame __user *p_av1_frame;
+		struct v4l2_ctrl_av1_film_grain __user *p_av1_film_grain;
 		void __user *ptr;
 	};
 } __attribute__ ((packed));
@@ -1877,6 +1882,11 @@ enum v4l2_ctrl_type {
 	V4L2_CTRL_TYPE_HEVC_SLICE_PARAMS	= 0x0272,
 	V4L2_CTRL_TYPE_HEVC_SCALING_MATRIX	= 0x0273,
 	V4L2_CTRL_TYPE_HEVC_DECODE_PARAMS	= 0x0274,
+
+	V4L2_CTRL_TYPE_AV1_SEQUENCE	    = 0x280,
+	V4L2_CTRL_TYPE_AV1_TILE_GROUP_ENTRY = 0x281,
+	V4L2_CTRL_TYPE_AV1_FRAME	    = 0x282,
+	V4L2_CTRL_TYPE_AV1_FILM_GRAIN	    = 0x283,
 };
 
 /*  Used in the VIDIOC_QUERYCTRL ioctl for querying controls */

--
2.39.2