Skip to contentSkip to conversation

media: iris: add FPS calculation and VPP FW overhead in frequency formula

PATCH Superseded Vishnu Reddy 3 participants 4 messages Mar 5 – Mar 25

Subsystems (4)

v2 of 6 Superseded Latest is v6 →

lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 media: iris: add FPS calculation and VPP FW overhead in frequency formula

The driver was using a fixed default FPS value when calculating the VPU frequency. This caused wrong frequency requests for high‑frame‑rate streams, for example 4K at 240 FPS. Because of this, the hardware was running at a lower frequency than needed.

Add the FPS measurement based on the decoder input buffer arrival rate. The measured FPS is stored per instance and used in frequency calculation instead of the fixed default FPS. The value is clamped so that it does not exceed platform limits. Add a VPP firmware overhead when running in STAGE_2.

---
Changes in v2:
- Replaced div_u64 with mult_frac
- Link to v1: https://lore.kernel.org/r/20260304-update_fps_calculation-v1-1-4eeac373a504@oss.qualcomm.com

---
 drivers/media/platform/qcom/iris/iris_instance.h   |  2 ++
 drivers/media/platform/qcom/iris/iris_vdec.c       | 21 +++++++++++++++++++++
 drivers/media/platform/qcom/iris/iris_vpu_common.c |  6 +++++-
 3 files changed, 28 insertions(+), 1 deletion(-)
drivers/media/platform/qcom/iris/iris_instance.h +2 -0
index 16965150f427..180cba36a7f2 100644
@@ -109,6 +109,8 @@ struct iris_inst {
 	u32				metadata_idx;
 	u32				codec;
 	bool				last_buffer_dequeued;
+	u64				last_buf_recv_time_ns;
+	u32				frame_count;
 	u32				frame_rate;
 	u32				operating_rate;
 	u32				hfi_rc_type;
drivers/media/platform/qcom/iris/iris_vdec.c +21 -0
index 719217399a30..88820060e22a 100644
@@ -369,6 +369,8 @@ int iris_vdec_streamon_input(struct iris_inst *inst)
 	if (ret)
 		return ret;
 
+	inst->frame_count = 0;
+
 	return iris_process_streamon_input(inst);
 }
 
@@ -411,6 +413,7 @@ int iris_vdec_qbuf(struct iris_inst *inst, struct vb2_v4l2_buffer *vbuf)
 {
 	struct iris_buffer *buf = to_iris_buffer(vbuf);
 	struct vb2_buffer *vb2 = &vbuf->vb2_buf;
+	u64 cur_buf_recv_time_ns, time_delta_ns;
 	struct vb2_queue *q;
 	int ret;
 
@@ -427,6 +430,24 @@ int iris_vdec_qbuf(struct iris_inst *inst, struct vb2_v4l2_buffer *vbuf)
 		return 0;
 	}
 
+	if (buf->type == BUF_INPUT) {
+		cur_buf_recv_time_ns = ktime_get_ns();
+
+		if (inst->frame_count == 0) {
+			inst->last_buf_recv_time_ns = cur_buf_recv_time_ns;
+			inst->frame_rate = MAXIMUM_FPS;
+		}
+		time_delta_ns = cur_buf_recv_time_ns - inst->last_buf_recv_time_ns;
+
+		if (time_delta_ns >= NSEC_PER_SEC) {
+			inst->frame_rate = clamp_t(u32, inst->frame_count, DEFAULT_FPS,
+						   MAXIMUM_FPS);
+			inst->last_buf_recv_time_ns = cur_buf_recv_time_ns;
+			inst->frame_count = 0;
+		}
+		inst->frame_count++;
+	}
+
 	iris_scale_power(inst);
 
 	return iris_queue_buffer(inst, buf);
drivers/media/platform/qcom/iris/iris_vpu_common.c +5 -1
index 548e5f1727fd..d621ccffa868 100644
@@ -416,7 +416,7 @@ u64 iris_vpu3x_vpu4x_calculate_frequency(struct iris_inst *inst, size_t data_siz
 	u32 height, width, mbs_per_second, mbpf;
 	u64 fw_cycles, fw_vpp_cycles;
 	u64 vsp_cycles, vpp_cycles;
-	u32 fps = DEFAULT_FPS;
+	u32 fps = inst->frame_rate;
 
 	width = max(inp_f->fmt.pix_mp.width, inst->crop.width);
 	height = max(inp_f->fmt.pix_mp.height, inst->crop.height);
@@ -435,6 +435,10 @@ u64 iris_vpu3x_vpu4x_calculate_frequency(struct iris_inst *inst, size_t data_siz
 	if (inst->fw_caps[PIPE].value > 1)
 		vpp_cycles += div_u64(vpp_cycles * 59, 1000);
 
+	/* 1.05 is VPP FW overhead */
+	if (inst->fw_caps[STAGE].value == STAGE_2)
+		vpp_cycles += mult_frac(vpp_cycles, 5, 100);
+
 	vsp_cycles = fps * data_size * 8;
 	vsp_cycles = div_u64(vsp_cycles, 2);
 	/* VSP FW overhead 1.05 */

---
base-commit: f505e978d1a0442adbbde48aed38c084ddea6d6e
change-id: 20260304-update_fps_calculation-98ee7f7507b1

Best regards,

Signature · 1 line
Vishnu Reddy <[email protected]>
maintainer
#1 lore raw
Quoted 68 lines · Vishnu Reddy

The driver was using a fixed default FPS value when calculating the VPU frequency. This caused wrong frequency requests for high‑frame‑rate streams, for example 4K at 240 FPS. Because of this, the hardware was running at a lower frequency than needed.

Add the FPS measurement based on the decoder input buffer arrival rate. The measured FPS is stored per instance and used in frequency calculation instead of the fixed default FPS. The value is clamped so that it does not exceed platform limits. Add a VPP firmware overhead when running in STAGE_2.

---
Changes in v2:
- Replaced div_u64 with mult_frac
- Link to v1: https://lore.kernel.org/r/20260304-update_fps_calculation-v1-1-4eeac373a504@oss.qualcomm.com

---
  drivers/media/platform/qcom/iris/iris_instance.h   |  2 ++
  drivers/media/platform/qcom/iris/iris_vdec.c       | 21 +++++++++++++++++++++
  drivers/media/platform/qcom/iris/iris_vpu_common.c |  6 +++++-
  3 files changed, 28 insertions(+), 1 deletion(-)
drivers/media/platform/qcom/iris/iris_instance.h +2 -0
index 16965150f427..180cba36a7f2 100644
@@ -109,6 +109,8 @@ struct iris_inst {
  	u32				metadata_idx;
  	u32				codec;
  	bool				last_buffer_dequeued;
+	u64				last_buf_recv_time_ns;
+	u32				frame_count;
  	u32				frame_rate;
  	u32				operating_rate;
  	u32				hfi_rc_type;
drivers/media/platform/qcom/iris/iris_vdec.c +11 -0
index 719217399a30..88820060e22a 100644
@@ -369,6 +369,8 @@ int iris_vdec_streamon_input(struct iris_inst *inst)
  	if (ret)
  		return ret;
 
+	inst->frame_count = 0;
+
  	return iris_process_streamon_input(inst);
  }
 
@@ -411,6 +413,7 @@ int iris_vdec_qbuf(struct iris_inst *inst, struct vb2_v4l2_buffer *vbuf)
  {
  	struct iris_buffer *buf = to_iris_buffer(vbuf);
  	struct vb2_buffer *vb2 = &vbuf->vb2_buf;
+	u64 cur_buf_recv_time_ns, time_delta_ns;
  	struct vb2_queue *q;
  	int ret;
 
@@ -427,6 +430,24 @@ int iris_vdec_qbuf(struct iris_inst *inst, struct vb2_v4l2_buffer *vbuf)
  		return 0;
  	}
 
+	if (buf->type == BUF_INPUT) {
+		cur_buf_recv_time_ns = ktime_get_ns();
+
+		if (inst->frame_count == 0) {
+			inst->last_buf_recv_time_ns = cur_buf_recv_time_ns;
+			inst->frame_rate = MAXIMUM_FPS;
+		}
+		time_delta_ns = cur_buf_recv_time_ns - inst->last_buf_recv_time_ns;

drivers/media/platform/qcom/iris/iris_vdec.c int iris_vdec_qbuf(struct iris_inst *inst, struct vb2_v4l2_buffer *vbuf) · line 450

Quoted 10 lines
drivers/media/platform/qcom/iris/iris_vdec.c +10 -0
+
+		if (time_delta_ns >= NSEC_PER_SEC) {
+			inst->frame_rate = clamp_t(u32, inst->frame_count, DEFAULT_FPS,
+						   MAXIMUM_FPS);
+			inst->last_buf_recv_time_ns = cur_buf_recv_time_ns;
+			inst->frame_count = 0;
+		}
+		inst->frame_count++;
+	}
+

simplifying it ?

    u64 now_ns, delta_ns;

    if (buf->type == BUF_INPUT) {
    	now_ns = ktime_get_ns();

        if (!inst->input_frame_count)
            inst->buf_fps_start_ts = now_ns;

	inst->input_frame_count++;
        delta_ns = now_ns - inst->buf_fps_start_ts;
        if (delta_ns < NSEC_PER_SEC)
            //skip the further logic;

        inst->frame_rate = clamp_t(u32, input_frame_count, DEFAULT_FPS, 
MAXIMUM_FPS);
       inst->input_frame_count = 0;
    }
}
Quoted 33 lines
drivers/media/platform/qcom/iris/iris_vdec.c +0 -0
  	iris_scale_power(inst);
 
  	return iris_queue_buffer(inst, buf);
drivers/media/platform/qcom/iris/iris_vpu_common.c +5 -2
index 548e5f1727fd..d621ccffa868 100644
@@ -416,7 +416,7 @@ u64 iris_vpu3x_vpu4x_calculate_frequency(struct iris_inst *inst, size_t data_siz
  	u32 height, width, mbs_per_second, mbpf;
  	u64 fw_cycles, fw_vpp_cycles;
  	u64 vsp_cycles, vpp_cycles;
-	u32 fps = DEFAULT_FPS;
+	u32 fps = inst->frame_rate;
 
  	width = max(inp_f->fmt.pix_mp.width, inst->crop.width);
  	height = max(inp_f->fmt.pix_mp.height, inst->crop.height);
@@ -435,6 +435,10 @@ u64 iris_vpu3x_vpu4x_calculate_frequency(struct iris_inst *inst, size_t data_siz
  	if (inst->fw_caps[PIPE].value > 1)
  		vpp_cycles += div_u64(vpp_cycles * 59, 1000);
 
+	/* 1.05 is VPP FW overhead */
+	if (inst->fw_caps[STAGE].value == STAGE_2)
+		vpp_cycles += mult_frac(vpp_cycles, 5, 100);
+
  	vsp_cycles = fps * data_size * 8;
  	vsp_cycles = div_u64(vsp_cycles, 2);
  	/* VSP FW overhead 1.05 */
 
---

base-commit: f505e978d1a0442adbbde48aed38c084ddea6d6e
change-id: 20260304-update_fps_calculation-98ee7f7507b1

Best regards,

#2 lore raw
Quoted 83 lines · Vikash Garodia
Quoted 70 lines · Vishnu Reddy

The driver was using a fixed default FPS value when calculating the VPU frequency. This caused wrong frequency requests for high‑frame‑rate streams, for example 4K at 240 FPS. Because of this, the hardware was running at a lower frequency than needed.

Add the FPS measurement based on the decoder input buffer arrival rate. The measured FPS is stored per instance and used in frequency calculation instead of the fixed default FPS. The value is clamped so that it does not exceed platform limits. Add a VPP firmware overhead when running in STAGE_2.

---
Changes in v2:
- Replaced div_u64 with mult_frac
- Link to v1:
https://lore.kernel.org/r/20260304-update_fps_calculation-v1-1-4eeac373a504@oss.qualcomm.com
---
  drivers/media/platform/qcom/iris/iris_instance.h   |  2 ++ drivers/media/platform/qcom/iris/iris_vdec.c       | 21
+++++++++++++++++++++
  drivers/media/platform/qcom/iris/iris_vpu_common.c |  6 +++++-

  3 files changed, 28 insertions(+), 1 deletion(-)
drivers/media/platform/qcom/iris/iris_instance.h +2 -0
index 16965150f427..180cba36a7f2 100644
@@ -109,6 +109,8 @@ struct iris_inst {
       u32                metadata_idx;
       u32                codec;
       bool                last_buffer_dequeued;
+    u64                last_buf_recv_time_ns;
+    u32                frame_count;
       u32                frame_rate;
       u32                operating_rate;
       u32                hfi_rc_type;
drivers/media/platform/qcom/iris/iris_vdec.c +1 -0
index 719217399a30..88820060e22a 100644
@@ -369,6 +369,8 @@ int iris_vdec_streamon_input(struct iris_inst *inst)
       if (ret)
           return ret;
   +    inst->frame_count = 0;
+
       return iris_process_streamon_input(inst);
   }
   @@ -411,6 +413,7 @@ int iris_vdec_qbuf(struct iris_inst *inst, struct vb2_v4l2_buffer *vbuf)

  {
      struct iris_buffer *buf = to_iris_buffer(vbuf);
      struct vb2_buffer *vb2 = &vbuf->vb2_buf;
+    u64 cur_buf_recv_time_ns, time_delta_ns;
      struct vb2_queue *q;
      int ret;
  @@ -427,6 +430,24 @@ int iris_vdec_qbuf(struct iris_inst *inst, struct vb2_v4l2_buffer *vbuf) return 0;
      }
  +    if (buf->type == BUF_INPUT) {
+        cur_buf_recv_time_ns = ktime_get_ns();
+
+        if (inst->frame_count == 0) {
+            inst->last_buf_recv_time_ns = cur_buf_recv_time_ns;
+            inst->frame_rate = MAXIMUM_FPS;
+        }
+        time_delta_ns = cur_buf_recv_time_ns -
inst->last_buf_recv_time_ns;
+
+        if (time_delta_ns >= NSEC_PER_SEC) {
+            inst->frame_rate = clamp_t(u32, inst->frame_count,

Quoted 8 lines

DEFAULT_FPS,
+                           MAXIMUM_FPS);
+            inst->last_buf_recv_time_ns = cur_buf_recv_time_ns; +            inst->frame_count = 0;
+        }
+        inst->frame_count++;
+    }
+

simplifying it ?

Quoted 7 lines

    u64 now_ns, delta_ns;

    if (buf->type == BUF_INPUT) {
        now_ns = ktime_get_ns();

        if (!inst->input_frame_count)
            inst->buf_fps_start_ts = now_ns;

Here, the actual required FPS will be found after 1 second. until that, the initial FPS need to set for max.

Quoted 2 lines

    inst->input_frame_count++;

If we increment the frame_count here, once frame_count becomes zero (after 1 sec), In next buffer queue, again the above if condition will be true and FPS will be set to max. I can add the code like below, Let me know your opinion.

if (buf->type == BUF_INPUT) {
    now_ns = ktime_get_ns();

    if (!inst->frame_count) {
        inst->last_buf_recv_time_ns = now_ns;
        inst->frame_rate = MAXIMUM_FPS;
    }
    delta_ns = now_ns - inst->buf_fps_start_ts;

    if (delta_ns >= NSEC_PER_SEC) {
        inst->frame_rate = clamp_t(u32, inst->frame_count, DEFAULT_FPS, MAXIMUM_FPS); inst->buf_fps_start_ts = now_ns;
        inst->frame_count = 0;
    }
    inst->frame_count++;
}

Regards,
Vishnu Reddy

Quoted 42 lines

delta_ns = now_ns - inst->buf_fps_start_ts;
        if (delta_ns < NSEC_PER_SEC)
            //skip the further logic;

        inst->frame_rate = clamp_t(u32, input_frame_count, DEFAULT_FPS, MAXIMUM_FPS); inst->input_frame_count = 0;
    }
}

Quoted 32 lines

      iris_scale_power(inst);
        return iris_queue_buffer(inst, buf);

drivers/media/platform/qcom/iris/iris_vpu_common.c +0 -0
index 548e5f1727fd..d621ccffa868 100644
@@ -416,7 +416,7 @@ u64 iris_vpu3x_vpu4x_calculate_frequency(struct 

iris_inst *inst, size_t data_siz
      u32 height, width, mbs_per_second, mbpf;
      u64 fw_cycles, fw_vpp_cycles;
      u64 vsp_cycles, vpp_cycles;
-    u32 fps = DEFAULT_FPS;
+    u32 fps = inst->frame_rate;
        width = max(inp_f->fmt.pix_mp.width, inst->crop.width); height = max(inp_f->fmt.pix_mp.height, inst->crop.height);

drivers/media/platform/qcom/iris/iris_vpu_common.c +0 -0
@@ -435,6 +435,10 @@ u64 iris_vpu3x_vpu4x_calculate_frequency(struct 

iris_inst *inst, size_t data_siz
      if (inst->fw_caps[PIPE].value > 1) vpp_cycles += div_u64(vpp_cycles * 59, 1000); +    /* 1.05 is VPP FW overhead */
+    if (inst->fw_caps[STAGE].value == STAGE_2) +        vpp_cycles += mult_frac(vpp_cycles, 5, 100); +
      vsp_cycles = fps * data_size * 8; vsp_cycles = div_u64(vsp_cycles, 2);
      /* VSP FW overhead 1.05 */

---
base-commit: f505e978d1a0442adbbde48aed38c084ddea6d6e
change-id: 20260304-update_fps_calculation-98ee7f7507b1

Best regards,

maintainer
#3 lore raw
Quoted 91 lines · Vishnu Reddy
Quoted 89 lines · Vikash Garodia
Quoted 74 lines · Vishnu Reddy

The driver was using a fixed default FPS value when calculating the VPU frequency. This caused wrong frequency requests for high‑frame‑rate streams, for example 4K at 240 FPS. Because of this, the hardware was running at a lower frequency than needed.

Add the FPS measurement based on the decoder input buffer arrival rate. The measured FPS is stored per instance and used in frequency calculation instead of the fixed default FPS. The value is clamped so that it does not exceed platform limits. Add a VPP firmware overhead when running in STAGE_2.

---
Changes in v2:
- Replaced div_u64 with mult_frac
- Link to v1:
https://lore.kernel.org/r/20260304-update_fps_calculation-v1-1-4eeac373a504@oss.qualcomm.com
---
  drivers/media/platform/qcom/iris/iris_instance.h   |  2 ++ drivers/media/platform/qcom/iris/iris_vdec.c       | 21
+++++++++++++++++++++
  drivers/media/platform/qcom/iris/iris_vpu_common.c |  6 +++++-

  3 files changed, 28 insertions(+), 1 deletion(-)
drivers/media/platform/qcom/iris/iris_instance.h +2 -0
index 16965150f427..180cba36a7f2 100644
@@ -109,6 +109,8 @@ struct iris_inst {
       u32                metadata_idx;
       u32                codec;
       bool                last_buffer_dequeued;
+    u64                last_buf_recv_time_ns;
+    u32                frame_count;
       u32                frame_rate;
       u32                operating_rate;
       u32                hfi_rc_type;
drivers/media/platform/qcom/iris/iris_vdec.c +1 -0
index 719217399a30..88820060e22a 100644
@@ -369,6 +369,8 @@ int iris_vdec_streamon_input(struct iris_inst *inst)
       if (ret)
           return ret;
   +    inst->frame_count = 0;
+
       return iris_process_streamon_input(inst);
   }

  @@ -411,6 +413,7 @@ int iris_vdec_qbuf(struct iris_inst *inst, struct vb2_v4l2_buffer *vbuf)
  {
      struct iris_buffer *buf = to_iris_buffer(vbuf);
      struct vb2_buffer *vb2 = &vbuf->vb2_buf;
+    u64 cur_buf_recv_time_ns, time_delta_ns;
      struct vb2_queue *q;
      int ret;
  @@ -427,6 +430,24 @@ int iris_vdec_qbuf(struct iris_inst *inst, struct vb2_v4l2_buffer *vbuf)
          return 0;
      }
  +    if (buf->type == BUF_INPUT) {
+        cur_buf_recv_time_ns = ktime_get_ns();
+
+        if (inst->frame_count == 0) {
+            inst->last_buf_recv_time_ns = cur_buf_recv_time_ns; +            inst->frame_rate = MAXIMUM_FPS;
+        }
+        time_delta_ns = cur_buf_recv_time_ns - inst->last_buf_recv_time_ns;
+
+        if (time_delta_ns >= NSEC_PER_SEC) {
+            inst->frame_rate = clamp_t(u32, inst->frame_count,

Quoted 8 lines

DEFAULT_FPS,
+                           MAXIMUM_FPS);
+            inst->last_buf_recv_time_ns = cur_buf_recv_time_ns; +            inst->frame_count = 0;
+        }
+        inst->frame_count++;
+    }
+

simplifying it ?

    u64 now_ns, delta_ns;

Quoted 8 lines
Quoted 5 lines

    if (buf->type == BUF_INPUT) {
        now_ns = ktime_get_ns();

        if (!inst->input_frame_count)
            inst->buf_fps_start_ts = now_ns;

Here, the actual required FPS will be found after 1 second. until that, the initial FPS need to set for max.

why max and not default? setting to max would over vote always in the start of session.

- Dikshita

Quoted 74 lines
Quoted 3 lines

    inst->input_frame_count++;

If we increment the frame_count here, once frame_count becomes zero (after 1 sec),
In next buffer queue, again the above if condition will be true and FPS will be set to max.
I can add the code like below, Let me know your opinion.

if (buf->type == BUF_INPUT) {
    now_ns = ktime_get_ns();

    if (!inst->frame_count) {
        inst->last_buf_recv_time_ns = now_ns;
        inst->frame_rate = MAXIMUM_FPS;
    }
    delta_ns = now_ns - inst->buf_fps_start_ts;

    if (delta_ns >= NSEC_PER_SEC) {
        inst->frame_rate = clamp_t(u32, inst->frame_count, DEFAULT_FPS, MAXIMUM_FPS);
        inst->buf_fps_start_ts = now_ns;
        inst->frame_count = 0;
    }
    inst->frame_count++;
}

Regards,
Vishnu Reddy

Quoted 44 lines

delta_ns = now_ns - inst->buf_fps_start_ts;
        if (delta_ns < NSEC_PER_SEC)
            //skip the further logic;

        inst->frame_rate = clamp_t(u32, input_frame_count, DEFAULT_FPS, MAXIMUM_FPS);
       inst->input_frame_count = 0;
    }
}

Quoted 33 lines

      iris_scale_power(inst);
        return iris_queue_buffer(inst, buf);

drivers/media/platform/qcom/iris/iris_vpu_common.c +0 -0
index 548e5f1727fd..d621ccffa868 100644
@@ -416,7 +416,7 @@ u64 iris_vpu3x_vpu4x_calculate_frequency(struct

iris_inst *inst, size_t data_siz
      u32 height, width, mbs_per_second, mbpf;
      u64 fw_cycles, fw_vpp_cycles;
      u64 vsp_cycles, vpp_cycles;
-    u32 fps = DEFAULT_FPS;
+    u32 fps = inst->frame_rate;
        width = max(inp_f->fmt.pix_mp.width, inst->crop.width); height = max(inp_f->fmt.pix_mp.height, inst->crop.height);

drivers/media/platform/qcom/iris/iris_vpu_common.c +0 -0
@@ -435,6 +435,10 @@ u64 iris_vpu3x_vpu4x_calculate_frequency(struct

iris_inst *inst, size_t data_siz
      if (inst->fw_caps[PIPE].value > 1) vpp_cycles += div_u64(vpp_cycles * 59, 1000); +    /* 1.05 is VPP FW overhead */
+    if (inst->fw_caps[STAGE].value == STAGE_2) +        vpp_cycles += mult_frac(vpp_cycles, 5, 100); +
      vsp_cycles = fps * data_size * 8; vsp_cycles = div_u64(vsp_cycles, 2);
      /* VSP FW overhead 1.05 */

---
base-commit: f505e978d1a0442adbbde48aed38c084ddea6d6e
change-id: 20260304-update_fps_calculation-98ee7f7507b1

Best regards,