media: amphion: Start decoding job when both queue are on
Ming Qian
maintainerPATCH 1/1 media: amphion: Start decoding job when both queue are on
Start the decoding job when both queue are on, except the for the initialization sequence.
Especially when seeking, the capture streamon may be called after output streamon, driver will start to decode job immediately after output streamo, if seek to a new resolution, then the source change flow may be mixed with the seek, it will cause confusion, then may led to pipeline hang.
When both output and capture queue are on, it's ready to start the decoding job, and it can avoid the above potential problem.
--- drivers/media/platform/amphion/vdec.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-)
drivers/media/platform/amphion/vdec.c +17 -1
@@ -1363,6 +1363,12 @@ static int vdec_process_output(struct vpu_inst *inst, struct vb2_buffer *vb)if (inst->state ==VPU_CODEC_STATE_STARTED )vdec_update_state (inst,VPU_CODEC_STATE_ACTIVE ,0 ); +if (vdec->seq_hdr_found && + !vb2_start_streaming_called ((v4l2_m2m_get_dst_vq (inst->fh.m2m_ctx)))) { +vpu_trace (inst->dev,"[%d] capture is not ready, pend input frame\n" , inst->id); +return -EINVAL ; + } + ret =vpu_iface_get_stream_buffer_desc (inst, &desc);if (ret)return ret;
@@ -1555,6 +1561,16 @@ static int vdec_start(struct vpu_inst *inst)return ret; } +static void vdec_enqueue_pending_frames (struct vpu_inst *inst) +{ +int i; + +for (i =0 ; i <v4l2_m2m_num_src_bufs_ready (inst->fh.m2m_ctx); i++) { +if (vpu_process_output_buffer (inst)) +break ; + } +} +static int vdec_start_session (struct vpu_inst *inst,u32 type) {struct vdec_t *vdec = inst->priv;
@@ -1573,10 +1589,10 @@ static int vdec_start_session(struct vpu_inst *inst, u32 type)if (V4L2_TYPE_IS_OUTPUT (type)) {vdec_update_state (inst, vdec->state,1 ); vdec->eos_received =0 ; -vpu_process_output_buffer (inst); }else {vdec_cmd_start (inst); } +vdec_enqueue_pending_frames (inst);if (inst->state ==VPU_CODEC_STATE_ACTIVE )vdec_response_fs_request (inst, false);
Signature · 1 line
2.43.0-rc1
Ming Qian
maintainerPATCH 1/1 media: amphion: Add H264 and HEVC profile and level control
For format H264 and HEVC, the firmware can report the parsed profile idc and level idc to driver, there information may be useful. Implement the H264 and HEVC profile and level control to report them.
--- drivers/media/platform/amphion/vdec.c | 60 +++++++++++++ drivers/media/platform/amphion/vpu_defs.h | 1 + drivers/media/platform/amphion/vpu_helpers.c | 93 ++++++++++++++++++++ drivers/media/platform/amphion/vpu_helpers.h | 5 ++ drivers/media/platform/amphion/vpu_malone.c | 3 +- 5 files changed, 161 insertions(+), 1 deletion(-)
drivers/media/platform/amphion/vdec.c +60 -0
@@ -232,6 +232,36 @@ static int vdec_ctrl_init(struct vpu_inst *inst)V4L2_CID_MPEG_VIDEO_DEC_DISPLAY_DELAY_ENABLE , 0, 1, 1, 0); +v4l2_ctrl_new_std_menu (&inst->ctrl_handler,NULL , +V4L2_CID_MPEG_VIDEO_H264_PROFILE , +V4L2_MPEG_VIDEO_H264_PROFILE_MULTIVIEW_HIGH , + ~((1 <<V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE ) | + (1 <<V4L2_MPEG_VIDEO_H264_PROFILE_MAIN ) | + (1 <<V4L2_MPEG_VIDEO_H264_PROFILE_EXTENDED ) | + (1 <<V4L2_MPEG_VIDEO_H264_PROFILE_HIGH ) | + (1 <<V4L2_MPEG_VIDEO_H264_PROFILE_MULTIVIEW_HIGH ) | + (1 <<V4L2_MPEG_VIDEO_H264_PROFILE_STEREO_HIGH )), +V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE ); + +v4l2_ctrl_new_std_menu (&inst->ctrl_handler,NULL , +V4L2_CID_MPEG_VIDEO_H264_LEVEL , +V4L2_MPEG_VIDEO_H264_LEVEL_6_2 , +0 , +V4L2_MPEG_VIDEO_H264_LEVEL_4_0 ); + +v4l2_ctrl_new_std_menu (&inst->ctrl_handler,NULL , +V4L2_CID_MPEG_VIDEO_HEVC_PROFILE , +V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN_10 , + ~((1 <<V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN ) | + (1 <<V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN_10 )), +V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN ); + +v4l2_ctrl_new_std_menu (&inst->ctrl_handler,NULL , +V4L2_CID_MPEG_VIDEO_HEVC_LEVEL , +V4L2_MPEG_VIDEO_HEVC_LEVEL_6_2 , +0 , +V4L2_MPEG_VIDEO_HEVC_LEVEL_4 ); + ctrl =v4l2_ctrl_new_std (&inst->ctrl_handler, &vdec_ctrl_ops,V4L2_CID_MIN_BUFFERS_FOR_CAPTURE ,1 ,32 ,1 ,2 );if (ctrl)
@@ -1166,6 +1196,35 @@ static void vdec_clear_slots(struct vpu_inst *inst) } } +static void vdec_update_v4l2_ctrl (struct vpu_inst *inst,u32 id,u32 val) +{ +struct v4l2_ctrl *ctrl =v4l2_ctrl_find (&inst->ctrl_handler, id); + +if (ctrl) +v4l2_ctrl_s_ctrl (ctrl, val); +} + +static void vdec_update_v4l2_profile_level (struct vpu_inst *inst,struct vpu_dec_codec_info *hdr) +{ +switch (inst->out_format.pixfmt) { +case V4L2_PIX_FMT_H264 : +case V4L2_PIX_FMT_H264_MVC : +vdec_update_v4l2_ctrl (inst,V4L2_CID_MPEG_VIDEO_H264_PROFILE , +vpu_get_h264_v4l2_profile (hdr->profile_idc)); +vdec_update_v4l2_ctrl (inst,V4L2_CID_MPEG_VIDEO_H264_LEVEL , +vpu_get_h264_v4l2_level (hdr->level_idc)); +break ; +case V4L2_PIX_FMT_HEVC : +vdec_update_v4l2_ctrl (inst,V4L2_CID_MPEG_VIDEO_HEVC_PROFILE , +vpu_get_hevc_v4l2_profile (hdr->profile_idc)); +vdec_update_v4l2_ctrl (inst,V4L2_CID_MPEG_VIDEO_HEVC_LEVEL , +vpu_get_hevc_v4l2_level (hdr->level_idc)); +break ; +default : +return ; + } +} +static void vdec_event_seq_hdr (struct vpu_inst *inst,struct vpu_dec_codec_info *hdr) {struct vdec_t *vdec = inst->priv;
@@ -1189,6 +1248,7 @@ static void vdec_event_seq_hdr(struct vpu_inst *inst, struct vpu_dec_codec_infovdec_init_crop (inst);vdec_init_mbi (inst);vdec_init_dcp (inst); +vdec_update_v4l2_profile_level (inst, hdr);if (!vdec->seq_hdr_found) { vdec->seq_tag = vdec->codec_info.tag;if (vdec->is_source_changed) {
drivers/media/platform/amphion/vpu_defs.h +1 -0
@@ -134,6 +134,7 @@ struct vpu_dec_codec_info {u32 decoded_height;struct v4l2_fract frame_rate;u32 dsp_asp_ratio; +u32 profile_idc;u32 level_idc;u32 bit_depth_luma;u32 bit_depth_chroma;
drivers/media/platform/amphion/vpu_helpers.c +93 -0
96 lines not on this page · raw message
drivers/media/platform/amphion/vpu_helpers.h +5 -0
@@ -70,4 +70,9 @@ int vpu_color_get_default(u32 primaries, u32 *ptransfers, u32 *pmatrix, u32 *pfuint vpu_find_dst_by_src (struct vpu_pair *pairs,u32 cnt,u32 src);int vpu_find_src_by_dst (struct vpu_pair *pairs,u32 cnt,u32 dst); + +u32 vpu_get_h264_v4l2_profile (u32 idc); +u32 vpu_get_h264_v4l2_level (u32 idc); +u32 vpu_get_hevc_v4l2_profile (u32 idc); +u32 vpu_get_hevc_v4l2_level (u32 idc);#endif
drivers/media/platform/amphion/vpu_malone.c +2 -1
@@ -889,7 +889,8 @@ static void vpu_malone_unpack_seq_hdr(struct vpu_rpc_event *pkt, info->frame_rate.numerator =1000 ; info->frame_rate.denominator = pkt->data[8 ]; info->dsp_asp_ratio = pkt->data[9 ]; - info->level_idc = pkt->data[10 ]; + info->profile_idc = (pkt->data[10 ] >>8 ) &0xff ; + info->level_idc = pkt->data[10 ] &0xff ; info->bit_depth_luma = pkt->data[13 ]; info->bit_depth_chroma = pkt->data[14 ]; info->chroma_fmt = pkt->data[15 ];
Signature · 1 line
2.43.0-rc1
Hi,
Le vendredi 19 juillet 2024 à 16:50 +0900, Ming Qian a écrit :
Quoted 11 lines
Start the decoding job when both queue are on, except the for the initialization sequence.
Especially when seeking, the capture streamon may be called after output streamon, driver will start to decode job immediately after output streamo, if seek to a new resolution, then the source change flow may be mixed with the seek, it will cause confusion, then may led to pipeline hang.
When both output and capture queue are on, it's ready to start the decoding job, and it can avoid the above potential problem.
This commit message needs some work and I'm unsure I understand its meaning. After reading the change, I am under the impression that you simply say that once the seq_hdr is found, the driver should keep delaying the processing of output buffer until the capture queue is ready ?
Quoted 14 lines
Signed-off-by Ming Qian <[email protected]>---
drivers/media/platform/amphion/vdec.c | 18 +++++++++++++++++-1 file changed, 17 insertions(+), 1 deletion(-)drivers/media/platform/amphion/vdec.c +0
-0@@ -1363,6 +1363,12 @@ static int vdec_process_output(struct vpu_inst *inst, struct vb2_buffer *vb) if (inst->state == VPU_CODEC_STATE_STARTED) vdec_update_state(inst, VPU_CODEC_STATE_ACTIVE, 0);
drivers/media/platform/amphion/vdec.c static int vdec_process_output(struct vpu_inst *inst, struct vb2_buffer *vb) · line 1369
Quoted 4 lines
drivers/media/platform/amphion/vdec.c +4
-0+if (vdec->seq_hdr_found && + !vb2_start_streaming_called((v4l2_m2m_get_dst_vq(inst->fh.m2m_ctx)))) { +vpu_trace (inst->dev,"[%d] capture is not ready, pend input frame\n" , inst->id); +return -EINVAL ;
I got really confused by this error return value. Its clearly not an error, but just a delay. I think before we add more on top, you should turn all these function for which the return value is unused to void. And use a plain "return;" here. That applies to all similar ops.
Quoted 9 lines
drivers/media/platform/amphion/vdec.c +2
-0+ } +ret =
vpu_iface_get_stream_buffer_desc(inst, &desc); if (ret)
return ret;drivers/media/platform/amphion/vdec.c +0
-0@@ -1555,6 +1561,16 @@ static int vdec_start(struct vpu_inst *inst) return ret; }
drivers/media/platform/amphion/vdec.c static int vdec_start(struct vpu_inst *inst) · line 1570
Quoted 7 lines
drivers/media/platform/amphion/vdec.c +7
-0+static void vdec_enqueue_pending_frames (struct vpu_inst *inst) +{ +int i; + +for (i =0 ; i <v4l2_m2m_num_src_bufs_ready (inst->fh.m2m_ctx); i++) { +if (vpu_process_output_buffer (inst)) +break ;
How well does this work ? Previous you made good care of interleaving the processing output and capture, which I could imaging prevents the hardware from starving ?
Quoted 7 lines
drivers/media/platform/amphion/vdec.c +3
-0+ } +} +static int
vdec_start_session(struct vpu_inst *inst, u32 type){
structvdec_t*vdec =inst->priv;
drivers/media/platform/amphion/vdec.c static int vdec_start_session(struct vpu_inst *inst, u32 type) · line 1595
Quoted 8 lines
drivers/media/platform/amphion/vdec.c +1
-1@@ -1573,10 +1589,10 @@ static int vdec_start_session(struct vpu_inst *inst, u32 type) if (V4L2_TYPE_IS_OUTPUT(type)) { vdec_update_state(inst, vdec->state, 1); vdec->eos_received = 0; -vpu_process_output_buffer (inst); } else { vdec_cmd_start(inst); } +vdec_enqueue_pending_frames (inst);
Was this intentional to reverse the STREAMON(CAPTURE) call flow to:
- process_capture() (inside vdec_cmd_start())
- proces_output() x n
Nicolas
Quoted 3 lines
drivers/media/platform/amphion/vdec.c +0
-0if (inst->state == VPU_CODEC_STATE_ACTIVE) vdec_response_fs_request(inst, false);
Hi Nicolas,
Quoted 12 lines · Nicolas Dufresne
Hi,
Quoted 9 lines · Ming Qian
Start the decoding job when both queue are on, except the for the initialization sequence.
Especially when seeking, the capture streamon may be called after output streamon, driver will start to decode job immediately after output streamo, if seek to a new resolution, then the source change flow may be mixed with the seek, it will cause confusion, then may led to pipeline hang.
Quoted 9 lines
Quoted 2 lines
When both output and capture queue are on, it's ready to start the decoding job, and it can avoid the above potential problem.
This commit message needs some work and I'm unsure I understand its meaning. After reading the change, I am under the impression that you simply say that once the
seq_hdris found, the driver should keep delaying the processing of output buffer until the capture queue is ready ?
I just want to do something like v4l2_m2m, that device_run() is only scheduled when both queues are ready.
I admit that this patch does not have any substantial functional changes.
I think this patch can be discarded, as it is confusing.
Thanks a lot for the review and comments.
Regards,
Ming
Quoted 72 lines
Quoted 11 lines
Signed-off-by Ming Qian <[email protected]>--- drivers/media/platform/amphion/vdec.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-)drivers/media/platform/amphion/vdec.c +0
-0drivers/media/platform/amphion/vdec.c static int vdec_process_output(struct vpu_inst *inst, struct vb2_buffer *vb) · line 1369
Quoted 7 lines
drivers/media/platform/amphion/vdec.c +4
-0@@ -1363,6 +1363,12 @@ static int vdec_process_output(struct vpu_inst *inst, struct vb2_buffer *vb)if (inst->state ==VPU_CODEC_STATE_STARTED )vdec_update_state (inst,VPU_CODEC_STATE_ACTIVE ,0 ); +if (vdec->seq_hdr_found && + !vb2_start_streaming_called ((v4l2_m2m_get_dst_vq (inst->fh.m2m_ctx)))) { +vpu_trace (inst->dev,"[%d] capture is not ready, pend input frame\n" , inst->id); +return -EINVAL ;I got really confused by this error return value. Its clearly not an error, but just a delay. I think before we add more on top, you should turn all these function for which the return value is unused to void. And use a plain "return;" here. That applies to all similar ops.
Quoted 6 lines
drivers/media/platform/amphion/vdec.c +2
-0+ } + ret =vpu_iface_get_stream_buffer_desc (inst, &desc);if (ret)return ret;drivers/media/platform/amphion/vdec.c static int vdec_start(struct vpu_inst *inst) · line 1570
Quoted 10 lines
drivers/media/platform/amphion/vdec.c +7
-0@@ -1555,6 +1561,16 @@ static int vdec_start(struct vpu_inst *inst)return ret; } +static void vdec_enqueue_pending_frames (struct vpu_inst *inst) +{ +int i; + +for (i =0 ; i <v4l2_m2m_num_src_bufs_ready (inst->fh.m2m_ctx); i++) { +if (vpu_process_output_buffer (inst)) +break ;How well does this work ? Previous you made good care of interleaving the processing output and capture, which I could imaging prevents the hardware from starving ?
Quoted 7 lines
drivers/media/platform/amphion/vdec.c +3
-0+ } +} +static int vdec_start_session (struct vpu_inst *inst,u32 type) {struct vdec_t *vdec = inst->priv;drivers/media/platform/amphion/vdec.c static int vdec_start_session(struct vpu_inst *inst, u32 type) · line 1595
Quoted 8 lines
drivers/media/platform/amphion/vdec.c +1
-1@@ -1573,10 +1589,10 @@ static int vdec_start_session(struct vpu_inst *inst, u32 type)if (V4L2_TYPE_IS_OUTPUT (type)) {vdec_update_state (inst, vdec->state,1 ); vdec->eos_received =0 ; -vpu_process_output_buffer (inst); }else {vdec_cmd_start (inst); } +vdec_enqueue_pending_frames (inst);Was this intentional to reverse the STREAMON(CAPTURE) call flow to:
-
process_capture()(insidevdec_cmd_start())
-proces_output()x nNicolas
Quoted 3 lines
drivers/media/platform/amphion/vdec.c +0
-0if (inst->state ==VPU_CODEC_STATE_ACTIVE )vdec_response_fs_request (inst, false);