media: v4l-ctrl: add control for variable frame rate
Subsystems (2)
PATCH 1/1 media: v4l-ctrl: add control for variable frame rate
These flag are added
* V4L2_CID_MPEG_VIDEO_VFR_ENABLE
* V4L2_CID_MPEG_VIDEO_VFR_MIN_FRAMERATE
One is to set variable frame rate enable or not, another is to control minimal frame rate that video encoder should keep per second.
--- Documentation/media/uapi/v4l/ext-ctrls-codec.rst | 9 +++++++++ drivers/media/v4l2-core/v4l2-ctrls.c | 3 +++ include/uapi/linux/v4l2-controls.h | 3 +++ 3 files changed, 15 insertions(+)
Documentation/media/uapi/v4l/ext-ctrls-codec.rst +9 -0
@@ -620,7 +620,16 @@ enum v4l2_mpeg_video_bitrate_mode - * - Bit 24:31 - Must be zero. +.. _v4l2-mpeg-video-variable-framerate: +``V4L2_CID_MPEG_VIDEO_VFR_ENABLE (boolean)`` + Set variable framerate enable or not. When enabled, video encoder is able + to drop frames based on some mechanism, like frame similarity or bitrate + control (default disable). + +``V4L2_CID_MPEG_VIDEO_VFR_MIN_FRAMERATE (integer)`` + Minimal kept frame per second when variable framerate is enabled (default + is input framerate). .. _v4l2-mpeg-video-dec-pts:
drivers/media/v4l2-core/v4l2-ctrls.c +3 -0
@@ -786,6 +786,8 @@ const char *v4l2_ctrl_get_name(u32 id)case V4L2_CID_MPEG_VIDEO_MB_RC_ENABLE :return "H264 MB Level Rate Control" ;case V4L2_CID_MPEG_VIDEO_HEADER_MODE :return "Sequence Header Mode" ;case V4L2_CID_MPEG_VIDEO_MAX_REF_PIC :return "Max Number of Reference Pics" ; +case V4L2_CID_MPEG_VIDEO_VFR_ENABLE :return "Variable Frame Rate Enable" ; +case V4L2_CID_MPEG_VIDEO_VFR_MIN_FRAMERATE :return "VFR Minimal Frame Rate" ;case V4L2_CID_MPEG_VIDEO_H263_I_FRAME_QP :return "H263 I-Frame QP Value" ;case V4L2_CID_MPEG_VIDEO_H263_P_FRAME_QP :return "H263 P-Frame QP Value" ;case V4L2_CID_MPEG_VIDEO_H263_B_FRAME_QP :return "H263 B-Frame QP Value" ;
@@ -1108,6 +1110,7 @@ void v4l2_ctrl_fill(u32 id, const char **name, enum v4l2_ctrl_type *type,case V4L2_CID_MPEG_VIDEO_DECODER_SLICE_INTERFACE :case V4L2_CID_MPEG_VIDEO_FRAME_RC_ENABLE :case V4L2_CID_MPEG_VIDEO_MB_RC_ENABLE : +case V4L2_CID_MPEG_VIDEO_VFR_ENABLE :case V4L2_CID_MPEG_VIDEO_H264_8X8_TRANSFORM :case V4L2_CID_MPEG_VIDEO_H264_VUI_SAR_ENABLE :case V4L2_CID_MPEG_VIDEO_MPEG4_QPEL :
include/uapi/linux/v4l2-controls.h +3 -0
@@ -404,6 +404,9 @@ enum v4l2_mpeg_video_multi_slice_mode {#define V4L2_CID_MPEG_VIDEO_MV_V_SEARCH_RANGE (V4L2_CID_MPEG_BASE+228)#define V4L2_CID_MPEG_VIDEO_FORCE_KEY_FRAME (V4L2_CID_MPEG_BASE+229) +#define V4L2_CID_MPEG_VIDEO_VFR_ENABLE (V4L2_CID_MPEG_BASE + 250) +#define V4L2_CID_MPEG_VIDEO_VFR_MIN_FRAMERATE (V4L2_CID_MPEG_BASE + 251) +#define V4L2_CID_MPEG_VIDEO_H263_I_FRAME_QP (V4L2_CID_MPEG_BASE+300)#define V4L2_CID_MPEG_VIDEO_H263_P_FRAME_QP (V4L2_CID_MPEG_BASE+301)#define V4L2_CID_MPEG_VIDEO_H263_B_FRAME_QP (V4L2_CID_MPEG_BASE+302)
Signature · 1 line
2.21.0.225.g810b269d1ac-goog
Quoted 21 lines · Fish Lin
These flag are added
*V4L2_CID_MPEG_VIDEO_VFR_ENABLE
*V4L2_CID_MPEG_VIDEO_VFR_MIN_FRAMERATE
One is to set variable frame rate enable or not, another is to control minimal frame rate that video encoder should keep per second.Signed-off-by Fish Lin <[email protected]>--- Documentation/media/uapi/v4l/ext-ctrls-codec.rst | 9 +++++++++ drivers/media/v4l2-core/v4l2-ctrls.c | 3 +++ include/uapi/linux/v4l2-controls.h | 3 +++ 3 files changed, 15 insertions(+)Documentation/media/uapi/v4l/ext-ctrls-codec.rst +0
-0@@ -620,7 +620,16 @@ enum v4l2_mpeg_video_bitrate_mode - * - Bit 24:31 - Must be zero.
Documentation/media/uapi/v4l/ext-ctrls-codec.rst enum v4l2_mpeg_video_bitrate_mode - · line 632
Quoted 10 lines
Documentation/media/uapi/v4l/ext-ctrls-codec.rst +9
-0+.. _v4l2-mpeg-video-variable-framerate: +``V4L2_CID_MPEG_VIDEO_VFR_ENABLE (boolean)`` + Set variable framerate enable or not. When enabled, video encoder is able + to drop frames based on some mechanism, like frame similarity or bitrate + control (default disable). + +``V4L2_CID_MPEG_VIDEO_VFR_MIN_FRAMERATE (integer)`` + Minimal kept frame per second when variable framerate is enabled (default + is input framerate).
You don't really have a concept of frames per second when encoding. What you are probably trying to say is something like: "don't drop more than one frame for every X frames" or something like that.
Which codecs support this? A decoder needs to know that a frame is missing, right?
What's the use-case for this?
Sorry, just trying to understand this better.
Regards,
Hans
Quoted 39 lines
Documentation/media/uapi/v4l/ext-ctrls-codec.rst +0
-0.. _v4l2-mpeg-video-dec-pts:drivers/media/v4l2-core/v4l2-ctrls.c +3
-0@@ -786,6 +786,8 @@ const char *v4l2_ctrl_get_name(u32 id)case V4L2_CID_MPEG_VIDEO_MB_RC_ENABLE :return "H264 MB Level Rate Control" ;case V4L2_CID_MPEG_VIDEO_HEADER_MODE :return "Sequence Header Mode" ;case V4L2_CID_MPEG_VIDEO_MAX_REF_PIC :return "Max Number of Reference Pics" ; +case V4L2_CID_MPEG_VIDEO_VFR_ENABLE :return "Variable Frame Rate Enable" ; +case V4L2_CID_MPEG_VIDEO_VFR_MIN_FRAMERATE :return "VFR Minimal Frame Rate" ;case V4L2_CID_MPEG_VIDEO_H263_I_FRAME_QP :return "H263 I-Frame QP Value" ;case V4L2_CID_MPEG_VIDEO_H263_P_FRAME_QP :return "H263 P-Frame QP Value" ;case V4L2_CID_MPEG_VIDEO_H263_B_FRAME_QP :return "H263 B-Frame QP Value" ;@@ -1108,6 +1110,7 @@ void v4l2_ctrl_fill(u32 id, const char **name, enum v4l2_ctrl_type *type,case V4L2_CID_MPEG_VIDEO_DECODER_SLICE_INTERFACE :case V4L2_CID_MPEG_VIDEO_FRAME_RC_ENABLE :case V4L2_CID_MPEG_VIDEO_MB_RC_ENABLE : +case V4L2_CID_MPEG_VIDEO_VFR_ENABLE :case V4L2_CID_MPEG_VIDEO_H264_8X8_TRANSFORM :case V4L2_CID_MPEG_VIDEO_H264_VUI_SAR_ENABLE :case V4L2_CID_MPEG_VIDEO_MPEG4_QPEL :include/uapi/linux/v4l2-controls.h +3
-0@@ -404,6 +404,9 @@ enum v4l2_mpeg_video_multi_slice_mode {#define V4L2_CID_MPEG_VIDEO_MV_V_SEARCH_RANGE (V4L2_CID_MPEG_BASE+228)#define V4L2_CID_MPEG_VIDEO_FORCE_KEY_FRAME (V4L2_CID_MPEG_BASE+229) +#define V4L2_CID_MPEG_VIDEO_VFR_ENABLE (V4L2_CID_MPEG_BASE + 250) +#define V4L2_CID_MPEG_VIDEO_VFR_MIN_FRAMERATE (V4L2_CID_MPEG_BASE + 251) +#define V4L2_CID_MPEG_VIDEO_H263_I_FRAME_QP (V4L2_CID_MPEG_BASE+300)#define V4L2_CID_MPEG_VIDEO_H263_P_FRAME_QP (V4L2_CID_MPEG_BASE+301)#define V4L2_CID_MPEG_VIDEO_H263_B_FRAME_QP (V4L2_CID_MPEG_BASE+302)
The idea for variable framerate is to reduce bitrate by encode less frames. Sometimes the motion is too much and encoder can encode in really low bitrate, will be able to dynamic drop some frames to archive requirement.
Codec like H.264 and HEVC can support VFR, and lots of containers also support.
https://en.wikipedia.org/wiki/Comparison_of_video_container_formats
Yes, the decoder need to know if video is CFR or VFR, not all decoder support VFR, especially for video editing.
There are also some tools to convert VFR video to CFR video.
Some reference:
https://en.wikipedia.org/wiki/Variable_frame_rate
https://www.bandicam.com/support/tips/vfr-cfr/
https://pro-av.panasonic.net/manual/html/AJ-PX270P_PJ_PX_EN_AN(VQT5J83A-8(E))/chapter04_08.htm
Hans Verkuil <[email protected]> 於 2019年3月28日 週四 下午10:57寫道：
Quoted 88 lines · Fish Lin
Quoted 21 lines
These flag are added
*V4L2_CID_MPEG_VIDEO_VFR_ENABLE
*V4L2_CID_MPEG_VIDEO_VFR_MIN_FRAMERATE
One is to set variable frame rate enable or not, another is to control minimal frame rate that video encoder should keep per second.Signed-off-by Fish Lin <[email protected]>--- Documentation/media/uapi/v4l/ext-ctrls-codec.rst | 9 +++++++++ drivers/media/v4l2-core/v4l2-ctrls.c | 3 +++ include/uapi/linux/v4l2-controls.h | 3 +++ 3 files changed, 15 insertions(+)Documentation/media/uapi/v4l/ext-ctrls-codec.rst +0
-0@@ -620,7 +620,16 @@ enum v4l2_mpeg_video_bitrate_mode - * - Bit 24:31 - Must be zero.Documentation/media/uapi/v4l/ext-ctrls-codec.rst enum v4l2_mpeg_video_bitrate_mode - · line 632
Quoted 10 lines
Documentation/media/uapi/v4l/ext-ctrls-codec.rst +9
-0+.. _v4l2-mpeg-video-variable-framerate: +``V4L2_CID_MPEG_VIDEO_VFR_ENABLE (boolean)`` + Set variable framerate enable or not. When enabled, video encoder is able + to drop frames based on some mechanism, like frame similarity or bitrate + control (default disable). + +``V4L2_CID_MPEG_VIDEO_VFR_MIN_FRAMERATE (integer)`` + Minimal kept frame per second when variable framerate is enabled (default + is input framerate).You don't really have a concept of frames per second when encoding. What you are probably trying to say is something like: "don't drop more than one frame for every X frames" or something like that.
Which codecs support this? A decoder needs to know that a frame is missing, right?
What's the use-case for this?
Sorry, just trying to understand this better.
Regards,
Hans
Quoted 39 lines
Documentation/media/uapi/v4l/ext-ctrls-codec.rst +0
-0.. _v4l2-mpeg-video-dec-pts:drivers/media/v4l2-core/v4l2-ctrls.c +3
-0@@ -786,6 +786,8 @@ const char *v4l2_ctrl_get_name(u32 id)case V4L2_CID_MPEG_VIDEO_MB_RC_ENABLE :return "H264 MB Level Rate Control" ;case V4L2_CID_MPEG_VIDEO_HEADER_MODE :return "Sequence Header Mode" ;case V4L2_CID_MPEG_VIDEO_MAX_REF_PIC :return "Max Number of Reference Pics" ; +case V4L2_CID_MPEG_VIDEO_VFR_ENABLE :return "Variable Frame Rate Enable" ; +case V4L2_CID_MPEG_VIDEO_VFR_MIN_FRAMERATE :return "VFR Minimal Frame Rate" ;case V4L2_CID_MPEG_VIDEO_H263_I_FRAME_QP :return "H263 I-Frame QP Value" ;case V4L2_CID_MPEG_VIDEO_H263_P_FRAME_QP :return "H263 P-Frame QP Value" ;case V4L2_CID_MPEG_VIDEO_H263_B_FRAME_QP :return "H263 B-Frame QP Value" ;@@ -1108,6 +1110,7 @@ void v4l2_ctrl_fill(u32 id, const char **name, enum v4l2_ctrl_type *type,case V4L2_CID_MPEG_VIDEO_DECODER_SLICE_INTERFACE :case V4L2_CID_MPEG_VIDEO_FRAME_RC_ENABLE :case V4L2_CID_MPEG_VIDEO_MB_RC_ENABLE : +case V4L2_CID_MPEG_VIDEO_VFR_ENABLE :case V4L2_CID_MPEG_VIDEO_H264_8X8_TRANSFORM :case V4L2_CID_MPEG_VIDEO_H264_VUI_SAR_ENABLE :case V4L2_CID_MPEG_VIDEO_MPEG4_QPEL :include/uapi/linux/v4l2-controls.h +3
-0@@ -404,6 +404,9 @@ enum v4l2_mpeg_video_multi_slice_mode {#define V4L2_CID_MPEG_VIDEO_MV_V_SEARCH_RANGE (V4L2_CID_MPEG_BASE+228)#define V4L2_CID_MPEG_VIDEO_FORCE_KEY_FRAME (V4L2_CID_MPEG_BASE+229) +#define V4L2_CID_MPEG_VIDEO_VFR_ENABLE (V4L2_CID_MPEG_BASE + 250) +#define V4L2_CID_MPEG_VIDEO_VFR_MIN_FRAMERATE (V4L2_CID_MPEG_BASE + 251) +#define V4L2_CID_MPEG_VIDEO_H263_I_FRAME_QP (V4L2_CID_MPEG_BASE+300)#define V4L2_CID_MPEG_VIDEO_H263_P_FRAME_QP (V4L2_CID_MPEG_BASE+301)#define V4L2_CID_MPEG_VIDEO_H263_B_FRAME_QP (V4L2_CID_MPEG_BASE+302)
Hi,
Quoted 4 lines · Fish Lin
The idea for variable framerate is to reduce bitrate by encode less frames. Sometimes the motion is too much and encoder can encode in really low bitrate, will be able to dynamic drop some frames to archive requirement.
This looks like an interesting optimization to support!
Quoted 6 lines
Codec like H.264 and HEVC can support VFR, and lots of containers also support.
https://en.wikipedia.org/wiki/Comparison_of_video_container_formatsYes, the decoder need to know if video is CFR or VFR, not all decoder support VFR, especially for video editing.
There are also some tools to convert VFR video to CFR video.
Do you have some idea of what using this feature requires in terms of bitstream elements? We are currently working on the H.264 stateless decoding spec for hardware decoders and it would be useful to have a somewhat precise idea of how we could fit support for this feature into it.
Cheers,
Paul
Quoted 94 lines
Some reference:
https://en.wikipedia.org/wiki/Variable_frame_rate
https://www.bandicam.com/support/tips/vfr-cfr/
https://pro-av.panasonic.net/manual/html/AJ-PX270P_PJ_PX_EN_AN(VQT5J83A-8(E))/chapter04_08.htmHans Verkuil <[email protected]> 於 2019年3月28日 週四 下午10:57寫道：
Quoted 88 lines · Fish Lin
Quoted 21 lines
These flag are added
*V4L2_CID_MPEG_VIDEO_VFR_ENABLE
*V4L2_CID_MPEG_VIDEO_VFR_MIN_FRAMERATE
One is to set variable frame rate enable or not, another is to control minimal frame rate that video encoder should keep per second.Signed-off-by Fish Lin <[email protected]>--- Documentation/media/uapi/v4l/ext-ctrls-codec.rst | 9 +++++++++ drivers/media/v4l2-core/v4l2-ctrls.c | 3 +++ include/uapi/linux/v4l2-controls.h | 3 +++ 3 files changed, 15 insertions(+)Documentation/media/uapi/v4l/ext-ctrls-codec.rst +0
-0@@ -620,7 +620,16 @@ enum v4l2_mpeg_video_bitrate_mode - * - Bit 24:31 - Must be zero.Documentation/media/uapi/v4l/ext-ctrls-codec.rst enum v4l2_mpeg_video_bitrate_mode - · line 632
Quoted 10 lines
Documentation/media/uapi/v4l/ext-ctrls-codec.rst +9
-0+.. _v4l2-mpeg-video-variable-framerate: +``V4L2_CID_MPEG_VIDEO_VFR_ENABLE (boolean)`` + Set variable framerate enable or not. When enabled, video encoder is able + to drop frames based on some mechanism, like frame similarity or bitrate + control (default disable). + +``V4L2_CID_MPEG_VIDEO_VFR_MIN_FRAMERATE (integer)`` + Minimal kept frame per second when variable framerate is enabled (default + is input framerate).You don't really have a concept of frames per second when encoding. What you are probably trying to say is something like: "don't drop more than one frame for every X frames" or something like that.
Which codecs support this? A decoder needs to know that a frame is missing, right?
What's the use-case for this?
Sorry, just trying to understand this better.
Regards,
Hans
Quoted 39 lines
Documentation/media/uapi/v4l/ext-ctrls-codec.rst +0
-0.. _v4l2-mpeg-video-dec-pts:drivers/media/v4l2-core/v4l2-ctrls.c +3
-0@@ -786,6 +786,8 @@ const char *v4l2_ctrl_get_name(u32 id)case V4L2_CID_MPEG_VIDEO_MB_RC_ENABLE :return "H264 MB Level Rate Control" ;case V4L2_CID_MPEG_VIDEO_HEADER_MODE :return "Sequence Header Mode" ;case V4L2_CID_MPEG_VIDEO_MAX_REF_PIC :return "Max Number of Reference Pics" ; +case V4L2_CID_MPEG_VIDEO_VFR_ENABLE :return "Variable Frame Rate Enable" ; +case V4L2_CID_MPEG_VIDEO_VFR_MIN_FRAMERATE :return "VFR Minimal Frame Rate" ;case V4L2_CID_MPEG_VIDEO_H263_I_FRAME_QP :return "H263 I-Frame QP Value" ;case V4L2_CID_MPEG_VIDEO_H263_P_FRAME_QP :return "H263 P-Frame QP Value" ;case V4L2_CID_MPEG_VIDEO_H263_B_FRAME_QP :return "H263 B-Frame QP Value" ;@@ -1108,6 +1110,7 @@ void v4l2_ctrl_fill(u32 id, const char **name, enum v4l2_ctrl_type *type,case V4L2_CID_MPEG_VIDEO_DECODER_SLICE_INTERFACE :case V4L2_CID_MPEG_VIDEO_FRAME_RC_ENABLE :case V4L2_CID_MPEG_VIDEO_MB_RC_ENABLE : +case V4L2_CID_MPEG_VIDEO_VFR_ENABLE :case V4L2_CID_MPEG_VIDEO_H264_8X8_TRANSFORM :case V4L2_CID_MPEG_VIDEO_H264_VUI_SAR_ENABLE :case V4L2_CID_MPEG_VIDEO_MPEG4_QPEL :include/uapi/linux/v4l2-controls.h +3
-0@@ -404,6 +404,9 @@ enum v4l2_mpeg_video_multi_slice_mode {#define V4L2_CID_MPEG_VIDEO_MV_V_SEARCH_RANGE (V4L2_CID_MPEG_BASE+228)#define V4L2_CID_MPEG_VIDEO_FORCE_KEY_FRAME (V4L2_CID_MPEG_BASE+229) +#define V4L2_CID_MPEG_VIDEO_VFR_ENABLE (V4L2_CID_MPEG_BASE + 250) +#define V4L2_CID_MPEG_VIDEO_VFR_MIN_FRAMERATE (V4L2_CID_MPEG_BASE + 251) +#define V4L2_CID_MPEG_VIDEO_H263_I_FRAME_QP (V4L2_CID_MPEG_BASE+300)#define V4L2_CID_MPEG_VIDEO_H263_P_FRAME_QP (V4L2_CID_MPEG_BASE+301)#define V4L2_CID_MPEG_VIDEO_H263_B_FRAME_QP (V4L2_CID_MPEG_BASE+302)
Signature · 3 lines
Paul Kocialkowski, Bootlin Embedded Linux and kernel engineering https://bootlin.com