Skip to contentSkip to conversation

drm/msm/a6xx: Fix excessive stack usage

PATCH In review Akhil P Oommen 4 participants 13 messages Oct 27, 2024 – Oct 28, 2024

Subsystems (4)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 drm/msm/a6xx: Fix excessive stack usage

Clang-19 and above sometimes end up with multiple copies of the large a6xx_hfi_msg_bw_table structure on the stack. The problem is that a6xx_hfi_send_bw_table() calls a number of device specific functions to fill the structure, but these create another copy of the structure on the stack which gets copied to the first.

If the functions get inlined, that busts the warning limit:

drivers/gpu/drm/msm/adreno/a6xx_hfi.c:631:12: error: stack frame size (1032) exceeds limit (1024) in 'a6xx_hfi_send_bw_table' [-Werror,-Wframe-larger-than]

Fix this by kmalloc-ating struct a6xx_hfi_msg_bw_table instead of using the stack. Also, use this opportunity to skip re-initializing this table to optimize gpu wake up latency.

Cc Arnd Bergmann <[email protected]>
---
 drivers/gpu/drm/msm/adreno/a6xx_gmu.h |  1 +
 drivers/gpu/drm/msm/adreno/a6xx_hfi.c | 34 ++++++++++++++++++++++------------
 2 files changed, 23 insertions(+), 12 deletions(-)
drivers/gpu/drm/msm/adreno/a6xx_gmu.h +1 -0
index 94b6c5cab6f4..b4a79f88ccf4 100644
@@ -99,6 +99,7 @@ struct a6xx_gmu {
 	struct completion pd_gate;
 
 	struct qmp *qmp;
+	struct a6xx_hfi_msg_bw_table *bw_table;
 };
 
 static inline u32 gmu_read(struct a6xx_gmu *gmu, u32 offset)
drivers/gpu/drm/msm/adreno/a6xx_hfi.c +22 -12
index cdb3f6e74d3e..55e51c81be1f 100644
@@ -630,32 +630,42 @@ static void a6xx_build_bw_table(struct a6xx_hfi_msg_bw_table *msg)
 
 static int a6xx_hfi_send_bw_table(struct a6xx_gmu *gmu)
 {
-	struct a6xx_hfi_msg_bw_table msg = { 0 };
+	struct a6xx_hfi_msg_bw_table *msg;
 	struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
 	struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
 
+	if (gmu->bw_table)
+		goto send;
+
+	msg = devm_kzalloc(gmu->dev, sizeof(*msg), GFP_KERNEL);
+	if (!msg)
+		return -ENOMEM;
+
 	if (adreno_is_a618(adreno_gpu))
-		a618_build_bw_table(&msg);
+		a618_build_bw_table(msg);
 	else if (adreno_is_a619(adreno_gpu))
-		a619_build_bw_table(&msg);
+		a619_build_bw_table(msg);
 	else if (adreno_is_a640_family(adreno_gpu))
-		a640_build_bw_table(&msg);
+		a640_build_bw_table(msg);
 	else if (adreno_is_a650(adreno_gpu))
-		a650_build_bw_table(&msg);
+		a650_build_bw_table(msg);
 	else if (adreno_is_7c3(adreno_gpu))
-		adreno_7c3_build_bw_table(&msg);
+		adreno_7c3_build_bw_table(msg);
 	else if (adreno_is_a660(adreno_gpu))
-		a660_build_bw_table(&msg);
+		a660_build_bw_table(msg);
 	else if (adreno_is_a690(adreno_gpu))
-		a690_build_bw_table(&msg);
+		a690_build_bw_table(msg);
 	else if (adreno_is_a730(adreno_gpu))
-		a730_build_bw_table(&msg);
+		a730_build_bw_table(msg);
 	else if (adreno_is_a740_family(adreno_gpu))
-		a740_build_bw_table(&msg);
+		a740_build_bw_table(msg);
 	else
-		a6xx_build_bw_table(&msg);
+		a6xx_build_bw_table(msg);
+
+	gmu->bw_table = msg;
 
-	return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_BW_TABLE, &msg, sizeof(msg),
+send:
+	return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_BW_TABLE, gmu->bw_table, sizeof(*(gmu->bw_table)),
 		NULL, 0);
 }
 

---
base-commit: 74c374648ed08efb2ef339656f2764c28c046956
change-id: 20241024-stack-size-fix-28af7abd3fab

Best regards,

Signature · 1 line
Akhil P Oommen <[email protected]>
#1 lore raw
Quoted 12 lines · Akhil P Oommen

Clang-19 and above sometimes end up with multiple copies of the large a6xx_hfi_msg_bw_table structure on the stack. The problem is that a6xx_hfi_send_bw_table() calls a number of device specific functions to fill the structure, but these create another copy of the structure on the stack which gets copied to the first.

If the functions get inlined, that busts the warning limit:

drivers/gpu/drm/msm/adreno/a6xx_hfi.c:631:12: error: stack frame size (1032) exceeds limit (1024) in 'a6xx_hfi_send_bw_table' [-Werror,-Wframe-larger-than]

Quoted 5 lines

Fix this by kmalloc-ating struct a6xx_hfi_msg_bw_table instead of using the stack. Also, use this opportunity to skip re-initializing this table to optimize gpu wake up latency.

Cc Arnd Bergmann <[email protected]>

Please change this to "Reported-by:"

The patch looks correct to me, just one idea for improvement.

drivers/gpu/drm/msm/adreno/a6xx_gmu.h struct a6xx_gmu { · line 103

Quoted 10 lines

b/drivers/gpu/drm/msm/adreno/a6xx_gmu.h
index 94b6c5cab6f4..b4a79f88ccf4 100644

drivers/gpu/drm/msm/adreno/a6xx_gmu.h +1 -0
@@ -99,6 +99,7 @@ struct a6xx_gmu {
 	struct completion pd_gate;
 
 	struct qmp *qmp;
+	struct a6xx_hfi_msg_bw_table *bw_table;
 };

I think the bw_table is better just embedded
in here rather than referenced as a pointer:

drivers/gpu/drm/msm/adreno/a6xx_gmu.h struct a6xx_gmu {

Quoted 6 lines
drivers/gpu/drm/msm/adreno/a6xx_gmu.h +6 -0
+	if (gmu->bw_table)
+		goto send;
+
+	msg = devm_kzalloc(gmu->dev, sizeof(*msg), GFP_KERNEL);
+	if (!msg)
+		return -ENOMEM;

It looked like it's always allocated here when the device is up, so you can avoid the extra overhead for keeping track of the allocation.

Arnd

#2 lore raw
Quoted 13 lines · Arnd Bergmann
Quoted 12 lines · Akhil P Oommen

Clang-19 and above sometimes end up with multiple copies of the large a6xx_hfi_msg_bw_table structure on the stack. The problem is that a6xx_hfi_send_bw_table() calls a number of device specific functions to fill the structure, but these create another copy of the structure on the stack which gets copied to the first.

If the functions get inlined, that busts the warning limit:

drivers/gpu/drm/msm/adreno/a6xx_hfi.c:631:12: error: stack frame size (1032) exceeds limit (1024) in 'a6xx_hfi_send_bw_table' [-Werror,-Wframe-larger-than]

Quoted 7 lines
Quoted 5 lines

Fix this by kmalloc-ating struct a6xx_hfi_msg_bw_table instead of using the stack. Also, use this opportunity to skip re-initializing this table to optimize gpu wake up latency.

Cc Arnd Bergmann <[email protected]>

Please change this to "Reported-by:"

Sure.

Quoted 10 lines

The patch looks correct to me, just one idea for improvement.

Quoted 7 lines

b/drivers/gpu/drm/msm/adreno/a6xx_gmu.h
index 94b6c5cab6f4..b4a79f88ccf4 100644

drivers/gpu/drm/msm/adreno/a6xx_gmu.h +0 -0
@@ -99,6 +99,7 @@ struct a6xx_gmu {
 	struct completion pd_gate;
 
Quoted 7 lines

drivers/gpu/drm/msm/adreno/a6xx_gmu.h struct a6xx_gmu { · line 103

Quoted 3 lines
drivers/gpu/drm/msm/adreno/a6xx_gmu.h +1 -0
 	struct qmp *qmp;
+	struct a6xx_hfi_msg_bw_table *bw_table;
 };

I think the bw_table is better just embedded
in here rather than referenced as a pointer:

There are some low tier chipsets with relatively lower RAM size that doesn't require this table. So, dynamically allocating this here helps to save 640 bytes (minus the overhead of tracking).

-Akhil

Quoted 12 lines

drivers/gpu/drm/msm/adreno/a6xx_gmu.h struct a6xx_gmu {

Quoted 6 lines
drivers/gpu/drm/msm/adreno/a6xx_gmu.h +6 -0
+	if (gmu->bw_table)
+		goto send;
+
+	msg = devm_kzalloc(gmu->dev, sizeof(*msg), GFP_KERNEL);
+	if (!msg)
+		return -ENOMEM;

It looked like it's always allocated here when the device is up, so you can avoid the extra overhead for keeping track of the allocation.

Arnd

#3 lore raw
Quoted 41 lines · Akhil P Oommen
Quoted 13 lines · Arnd Bergmann
Quoted 12 lines · Akhil P Oommen

Clang-19 and above sometimes end up with multiple copies of the large a6xx_hfi_msg_bw_table structure on the stack. The problem is that a6xx_hfi_send_bw_table() calls a number of device specific functions to fill the structure, but these create another copy of the structure on the stack which gets copied to the first.

If the functions get inlined, that busts the warning limit:

drivers/gpu/drm/msm/adreno/a6xx_hfi.c:631:12: error: stack frame size (1032) exceeds limit (1024) in 'a6xx_hfi_send_bw_table' [-Werror,-Wframe-larger-than]

Quoted 7 lines
Quoted 5 lines

Fix this by kmalloc-ating struct a6xx_hfi_msg_bw_table instead of using the stack. Also, use this opportunity to skip re-initializing this table to optimize gpu wake up latency.

Cc Arnd Bergmann <[email protected]>

Please change this to "Reported-by:"

Sure.

Quoted 10 lines

The patch looks correct to me, just one idea for improvement.

Quoted 7 lines

b/drivers/gpu/drm/msm/adreno/a6xx_gmu.h
index 94b6c5cab6f4..b4a79f88ccf4 100644

drivers/gpu/drm/msm/adreno/a6xx_gmu.h +0 -0
@@ -99,6 +99,7 @@ struct a6xx_gmu {
 	struct completion pd_gate;
 
Quoted 7 lines

drivers/gpu/drm/msm/adreno/a6xx_gmu.h struct a6xx_gmu { · line 103

Quoted 3 lines
drivers/gpu/drm/msm/adreno/a6xx_gmu.h +1 -0
 	struct qmp *qmp;
+	struct a6xx_hfi_msg_bw_table *bw_table;
 };

I think the bw_table is better just embedded
in here rather than referenced as a pointer:

Quoted 3 lines

There are some low tier chipsets with relatively lower RAM size that doesn't require this table. So, dynamically allocating this here helps to save 640 bytes (minus the overhead of tracking).

I'd second this, said chipsets often ship with 1-2 GiB of RAM (which is still a lot in comparison, but you know.. every little bit counts)

Konrad

#4 lore raw
Quoted 42 lines · Konrad Dybcio
Quoted 41 lines · Akhil P Oommen
Quoted 13 lines · Arnd Bergmann
Quoted 12 lines · Akhil P Oommen

Clang-19 and above sometimes end up with multiple copies of the large a6xx_hfi_msg_bw_table structure on the stack. The problem is that a6xx_hfi_send_bw_table() calls a number of device specific functions to fill the structure, but these create another copy of the structure on the stack which gets copied to the first.

If the functions get inlined, that busts the warning limit:

drivers/gpu/drm/msm/adreno/a6xx_hfi.c:631:12: error: stack frame size (1032) exceeds limit (1024) in 'a6xx_hfi_send_bw_table' [-Werror,-Wframe-larger-than]

Quoted 7 lines
Quoted 5 lines

Fix this by kmalloc-ating struct a6xx_hfi_msg_bw_table instead of using the stack. Also, use this opportunity to skip re-initializing this table to optimize gpu wake up latency.

Cc Arnd Bergmann <[email protected]>

Please change this to "Reported-by:"

Sure.

Quoted 17 lines

The patch looks correct to me, just one idea for improvement.

drivers/gpu/drm/msm/adreno/a6xx_gmu.h struct a6xx_gmu { · line 103

Quoted 10 lines

b/drivers/gpu/drm/msm/adreno/a6xx_gmu.h
index 94b6c5cab6f4..b4a79f88ccf4 100644

drivers/gpu/drm/msm/adreno/a6xx_gmu.h +1 -0
@@ -99,6 +99,7 @@ struct a6xx_gmu {
 	struct completion pd_gate;
 
 	struct qmp *qmp;
+	struct a6xx_hfi_msg_bw_table *bw_table;
 };

I think the bw_table is better just embedded
in here rather than referenced as a pointer:

Quoted 6 lines
Quoted 3 lines

There are some low tier chipsets with relatively lower RAM size that doesn't require this table. So, dynamically allocating this here helps to save 640 bytes (minus the overhead of tracking).

I'd second this, said chipsets often ship with 1-2 GiB of RAM (which is still a lot in comparison, but you know.. every little bit counts)

Okay from my side. Yeah, poor Gnome runnning on top of 1 GiB device is very sad.

Signature · 2 lines
With best wishes
Dmitry
#5 lore raw
In reply to Akhil P Oommen
Quoted 41 lines · Akhil P Oommen

Clang-19 and above sometimes end up with multiple copies of the large a6xx_hfi_msg_bw_table structure on the stack. The problem is that a6xx_hfi_send_bw_table() calls a number of device specific functions to fill the structure, but these create another copy of the structure on the stack which gets copied to the first.

If the functions get inlined, that busts the warning limit:

drivers/gpu/drm/msm/adreno/a6xx_hfi.c:631:12: error: stack frame size (1032) exceeds limit (1024) in 'a6xx_hfi_send_bw_table' [-Werror,-Wframe-larger-than]

Fix this by kmalloc-ating struct a6xx_hfi_msg_bw_table instead of using the stack. Also, use this opportunity to skip re-initializing this table to optimize gpu wake up latency.

Cc Arnd Bergmann <[email protected]>
---
 drivers/gpu/drm/msm/adreno/a6xx_gmu.h |  1 +
 drivers/gpu/drm/msm/adreno/a6xx_hfi.c | 34 ++++++++++++++++++++++------------
 2 files changed, 23 insertions(+), 12 deletions(-)
drivers/gpu/drm/msm/adreno/a6xx_gmu.h +1 -0
index 94b6c5cab6f4..b4a79f88ccf4 100644
@@ -99,6 +99,7 @@ struct a6xx_gmu {
 	struct completion pd_gate;
 
 	struct qmp *qmp;
+	struct a6xx_hfi_msg_bw_table *bw_table;
 };
 
 static inline u32 gmu_read(struct a6xx_gmu *gmu, u32 offset)
drivers/gpu/drm/msm/adreno/a6xx_hfi.c +0 -0
index cdb3f6e74d3e..55e51c81be1f 100644
@@ -630,32 +630,42 @@ static void a6xx_build_bw_table(struct a6xx_hfi_msg_bw_table *msg)
 
 static int a6xx_hfi_send_bw_table(struct a6xx_gmu *gmu)

drivers/gpu/drm/msm/adreno/a6xx_hfi.c static void a6xx_build_bw_table(struct a6xx_hfi_msg_bw_table *msg) · line 640

Quoted 10 lines
drivers/gpu/drm/msm/adreno/a6xx_hfi.c +5 -1
 {
-	struct a6xx_hfi_msg_bw_table msg = { 0 };
+	struct a6xx_hfi_msg_bw_table *msg;
 	struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
 	struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
 
+	if (gmu->bw_table)
+		goto send;
+
+	msg = devm_kzalloc(gmu->dev, sizeof(*msg), GFP_KERNEL);

Is it necessary after being sent? Isn't it better to just kzalloc() it and then kfree() it at the end of the function?

Quoted 51 lines
drivers/gpu/drm/msm/adreno/a6xx_hfi.c +17 -12
+	if (!msg)
+		return -ENOMEM;
+
 	if (adreno_is_a618(adreno_gpu))
-		a618_build_bw_table(&msg);
+		a618_build_bw_table(msg);
 	else if (adreno_is_a619(adreno_gpu))
-		a619_build_bw_table(&msg);
+		a619_build_bw_table(msg);
 	else if (adreno_is_a640_family(adreno_gpu))
-		a640_build_bw_table(&msg);
+		a640_build_bw_table(msg);
 	else if (adreno_is_a650(adreno_gpu))
-		a650_build_bw_table(&msg);
+		a650_build_bw_table(msg);
 	else if (adreno_is_7c3(adreno_gpu))
-		adreno_7c3_build_bw_table(&msg);
+		adreno_7c3_build_bw_table(msg);
 	else if (adreno_is_a660(adreno_gpu))
-		a660_build_bw_table(&msg);
+		a660_build_bw_table(msg);
 	else if (adreno_is_a690(adreno_gpu))
-		a690_build_bw_table(&msg);
+		a690_build_bw_table(msg);
 	else if (adreno_is_a730(adreno_gpu))
-		a730_build_bw_table(&msg);
+		a730_build_bw_table(msg);
 	else if (adreno_is_a740_family(adreno_gpu))
-		a740_build_bw_table(&msg);
+		a740_build_bw_table(msg);
 	else
-		a6xx_build_bw_table(&msg);
+		a6xx_build_bw_table(msg);
+
+	gmu->bw_table = msg;
 
-	return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_BW_TABLE, &msg, sizeof(msg),
+send:
+	return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_BW_TABLE, gmu->bw_table, sizeof(*(gmu->bw_table)),
 		NULL, 0);
 }
 
 
---

base-commit: 74c374648ed08efb2ef339656f2764c28c046956
change-id: 20241024-stack-size-fix-28af7abd3fab

Best regards,

Signature · 1 line
Akhil P Oommen <[email protected]>
Signature · 2 lines
With best wishes
Dmitry
#6 lore raw
Quoted 48 lines · Dmitry Baryshkov
Quoted 47 lines · Akhil P Oommen

Clang-19 and above sometimes end up with multiple copies of the large a6xx_hfi_msg_bw_table structure on the stack. The problem is that a6xx_hfi_send_bw_table() calls a number of device specific functions to fill the structure, but these create another copy of the structure on the stack which gets copied to the first.

If the functions get inlined, that busts the warning limit:

drivers/gpu/drm/msm/adreno/a6xx_hfi.c:631:12: error: stack frame size (1032) exceeds limit (1024) in 'a6xx_hfi_send_bw_table' [-Werror,-Wframe-larger-than]

Fix this by kmalloc-ating struct a6xx_hfi_msg_bw_table instead of using the stack. Also, use this opportunity to skip re-initializing this table to optimize gpu wake up latency.

Cc Arnd Bergmann <[email protected]>
---
 drivers/gpu/drm/msm/adreno/a6xx_gmu.h |  1 +
 drivers/gpu/drm/msm/adreno/a6xx_hfi.c | 34 ++++++++++++++++++++++------------
 2 files changed, 23 insertions(+), 12 deletions(-)
drivers/gpu/drm/msm/adreno/a6xx_gmu.h +1 -0
index 94b6c5cab6f4..b4a79f88ccf4 100644
@@ -99,6 +99,7 @@ struct a6xx_gmu {
 	struct completion pd_gate;
 
 	struct qmp *qmp;
+	struct a6xx_hfi_msg_bw_table *bw_table;
 };
 
 static inline u32 gmu_read(struct a6xx_gmu *gmu, u32 offset)
drivers/gpu/drm/msm/adreno/a6xx_hfi.c +1 -1
index cdb3f6e74d3e..55e51c81be1f 100644
@@ -630,32 +630,42 @@ static void a6xx_build_bw_table(struct a6xx_hfi_msg_bw_table *msg)
 
 static int a6xx_hfi_send_bw_table(struct a6xx_gmu *gmu)
 {
-	struct a6xx_hfi_msg_bw_table msg = { 0 };
+	struct a6xx_hfi_msg_bw_table *msg;
 	struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
 	struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
 
Quoted 7 lines

drivers/gpu/drm/msm/adreno/a6xx_hfi.c static void a6xx_build_bw_table(struct a6xx_hfi_msg_bw_table *msg) · line 640

Quoted 4 lines
drivers/gpu/drm/msm/adreno/a6xx_hfi.c +4 -0
+	if (gmu->bw_table)
+		goto send;
+
+	msg = devm_kzalloc(gmu->dev, sizeof(*msg), GFP_KERNEL);

Is it necessary after being sent? Isn't it better to just kzalloc() it and then kfree() it at the end of the function?

Keeping it around will help to cut down unnecessary work during subsequent gpu wake ups.

-Akhil.

Quoted 53 lines
Quoted 52 lines
drivers/gpu/drm/msm/adreno/a6xx_hfi.c +17 -12
+	if (!msg)
+		return -ENOMEM;
+
 	if (adreno_is_a618(adreno_gpu))
-		a618_build_bw_table(&msg);
+		a618_build_bw_table(msg);
 	else if (adreno_is_a619(adreno_gpu))
-		a619_build_bw_table(&msg);
+		a619_build_bw_table(msg);
 	else if (adreno_is_a640_family(adreno_gpu))
-		a640_build_bw_table(&msg);
+		a640_build_bw_table(msg);
 	else if (adreno_is_a650(adreno_gpu))
-		a650_build_bw_table(&msg);
+		a650_build_bw_table(msg);
 	else if (adreno_is_7c3(adreno_gpu))
-		adreno_7c3_build_bw_table(&msg);
+		adreno_7c3_build_bw_table(msg);
 	else if (adreno_is_a660(adreno_gpu))
-		a660_build_bw_table(&msg);
+		a660_build_bw_table(msg);
 	else if (adreno_is_a690(adreno_gpu))
-		a690_build_bw_table(&msg);
+		a690_build_bw_table(msg);
 	else if (adreno_is_a730(adreno_gpu))
-		a730_build_bw_table(&msg);
+		a730_build_bw_table(msg);
 	else if (adreno_is_a740_family(adreno_gpu))
-		a740_build_bw_table(&msg);
+		a740_build_bw_table(msg);
 	else
-		a6xx_build_bw_table(&msg);
+		a6xx_build_bw_table(msg);
+
+	gmu->bw_table = msg;
 
-	return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_BW_TABLE, &msg, sizeof(msg),
+send:
+	return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_BW_TABLE, gmu->bw_table, sizeof(*(gmu->bw_table)),
 		NULL, 0);
 }
 
 
---

base-commit: 74c374648ed08efb2ef339656f2764c28c046956
change-id: 20241024-stack-size-fix-28af7abd3fab

Best regards,

Signature · 1 line
Akhil P Oommen <[email protected]>
#7 lore raw
Quoted 58 lines · Akhil P Oommen
Quoted 48 lines · Dmitry Baryshkov
Quoted 47 lines · Akhil P Oommen

Clang-19 and above sometimes end up with multiple copies of the large a6xx_hfi_msg_bw_table structure on the stack. The problem is that a6xx_hfi_send_bw_table() calls a number of device specific functions to fill the structure, but these create another copy of the structure on the stack which gets copied to the first.

If the functions get inlined, that busts the warning limit:

drivers/gpu/drm/msm/adreno/a6xx_hfi.c:631:12: error: stack frame size (1032) exceeds limit (1024) in 'a6xx_hfi_send_bw_table' [-Werror,-Wframe-larger-than]

Fix this by kmalloc-ating struct a6xx_hfi_msg_bw_table instead of using the stack. Also, use this opportunity to skip re-initializing this table to optimize gpu wake up latency.

Cc Arnd Bergmann <[email protected]>
---
 drivers/gpu/drm/msm/adreno/a6xx_gmu.h |  1 +
 drivers/gpu/drm/msm/adreno/a6xx_hfi.c | 34 ++++++++++++++++++++++------------
 2 files changed, 23 insertions(+), 12 deletions(-)
drivers/gpu/drm/msm/adreno/a6xx_gmu.h +1 -0
index 94b6c5cab6f4..b4a79f88ccf4 100644
@@ -99,6 +99,7 @@ struct a6xx_gmu {
     struct completion pd_gate;
 
     struct qmp *qmp;
+    struct a6xx_hfi_msg_bw_table *bw_table;
 };
 
 static inline u32 gmu_read(struct a6xx_gmu *gmu, u32 offset)
drivers/gpu/drm/msm/adreno/a6xx_hfi.c +1 -1
index cdb3f6e74d3e..55e51c81be1f 100644
@@ -630,32 +630,42 @@ static void a6xx_build_bw_table(struct a6xx_hfi_msg_bw_table *msg)
 
 static int a6xx_hfi_send_bw_table(struct a6xx_gmu *gmu)
 {
-    struct a6xx_hfi_msg_bw_table msg = { 0 };
+    struct a6xx_hfi_msg_bw_table *msg;
     struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
     struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
 
Quoted 7 lines

drivers/gpu/drm/msm/adreno/a6xx_hfi.c static void a6xx_build_bw_table(struct a6xx_hfi_msg_bw_table *msg) · line 640

Quoted 4 lines
drivers/gpu/drm/msm/adreno/a6xx_hfi.c +4 -0
+    if (gmu->bw_table)
+            goto send;
+
+    msg = devm_kzalloc(gmu->dev, sizeof(*msg), GFP_KERNEL);

Is it necessary after being sent? Isn't it better to just kzalloc() it and then kfree() it at the end of the function?

Quoted 2 lines

Keeping it around will help to cut down unnecessary work during subsequent gpu wake ups.

Then, I'd say, it is better to make it a part of the a6xx_gpu struct.

Signature · 2 lines
With best wishes
Dmitry
#8 lore raw
Quoted 59 lines · Dmitry Baryshkov
Quoted 58 lines · Akhil P Oommen
Quoted 55 lines · Dmitry Baryshkov
Quoted 41 lines · Akhil P Oommen

Clang-19 and above sometimes end up with multiple copies of the large a6xx_hfi_msg_bw_table structure on the stack. The problem is that a6xx_hfi_send_bw_table() calls a number of device specific functions to fill the structure, but these create another copy of the structure on the stack which gets copied to the first.

If the functions get inlined, that busts the warning limit:

drivers/gpu/drm/msm/adreno/a6xx_hfi.c:631:12: error: stack frame size (1032) exceeds limit (1024) in 'a6xx_hfi_send_bw_table' [-Werror,-Wframe-larger-than]

Fix this by kmalloc-ating struct a6xx_hfi_msg_bw_table instead of using the stack. Also, use this opportunity to skip re-initializing this table to optimize gpu wake up latency.

Cc Arnd Bergmann <[email protected]>
---
 drivers/gpu/drm/msm/adreno/a6xx_gmu.h |  1 +
 drivers/gpu/drm/msm/adreno/a6xx_hfi.c | 34 ++++++++++++++++++++++------------
 2 files changed, 23 insertions(+), 12 deletions(-)
drivers/gpu/drm/msm/adreno/a6xx_gmu.h +1 -0
index 94b6c5cab6f4..b4a79f88ccf4 100644
@@ -99,6 +99,7 @@ struct a6xx_gmu {
     struct completion pd_gate;
 
     struct qmp *qmp;
+    struct a6xx_hfi_msg_bw_table *bw_table;
 };
 
 static inline u32 gmu_read(struct a6xx_gmu *gmu, u32 offset)
drivers/gpu/drm/msm/adreno/a6xx_hfi.c +0 -0
index cdb3f6e74d3e..55e51c81be1f 100644
@@ -630,32 +630,42 @@ static void a6xx_build_bw_table(struct a6xx_hfi_msg_bw_table *msg)
 
 static int a6xx_hfi_send_bw_table(struct a6xx_gmu *gmu)

drivers/gpu/drm/msm/adreno/a6xx_hfi.c static void a6xx_build_bw_table(struct a6xx_hfi_msg_bw_table *msg) · line 640

Quoted 10 lines
drivers/gpu/drm/msm/adreno/a6xx_hfi.c +5 -1
 {
-    struct a6xx_hfi_msg_bw_table msg = { 0 };
+    struct a6xx_hfi_msg_bw_table *msg;
     struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
     struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
 
+    if (gmu->bw_table)
+            goto send;
+
+    msg = devm_kzalloc(gmu->dev, sizeof(*msg), GFP_KERNEL);

Is it necessary after being sent? Isn't it better to just kzalloc() it and then kfree() it at the end of the function?

Quoted 4 lines
Quoted 2 lines

Keeping it around will help to cut down unnecessary work during subsequent gpu wake ups.

Then, I'd say, it is better to make it a part of the a6xx_gpu struct.

I think a6xx_gmu makes more logical sense here.

FWIW, the driver allocates both _gmu and _gpu for all GPUs regardless

Konrad

#9 lore raw
Quoted 60 lines · Konrad Dybcio
Quoted 59 lines · Dmitry Baryshkov
Quoted 58 lines · Akhil P Oommen
Quoted 55 lines · Dmitry Baryshkov
Quoted 41 lines · Akhil P Oommen

Clang-19 and above sometimes end up with multiple copies of the large a6xx_hfi_msg_bw_table structure on the stack. The problem is that a6xx_hfi_send_bw_table() calls a number of device specific functions to fill the structure, but these create another copy of the structure on the stack which gets copied to the first.

If the functions get inlined, that busts the warning limit:

drivers/gpu/drm/msm/adreno/a6xx_hfi.c:631:12: error: stack frame size (1032) exceeds limit (1024) in 'a6xx_hfi_send_bw_table' [-Werror,-Wframe-larger-than]

Fix this by kmalloc-ating struct a6xx_hfi_msg_bw_table instead of using the stack. Also, use this opportunity to skip re-initializing this table to optimize gpu wake up latency.

Cc Arnd Bergmann <[email protected]>
---
 drivers/gpu/drm/msm/adreno/a6xx_gmu.h |  1 +
 drivers/gpu/drm/msm/adreno/a6xx_hfi.c | 34 ++++++++++++++++++++++------------
 2 files changed, 23 insertions(+), 12 deletions(-)
drivers/gpu/drm/msm/adreno/a6xx_gmu.h +1 -0
index 94b6c5cab6f4..b4a79f88ccf4 100644
@@ -99,6 +99,7 @@ struct a6xx_gmu {
     struct completion pd_gate;
 
     struct qmp *qmp;
+    struct a6xx_hfi_msg_bw_table *bw_table;
 };
 
 static inline u32 gmu_read(struct a6xx_gmu *gmu, u32 offset)
drivers/gpu/drm/msm/adreno/a6xx_hfi.c +0 -0
index cdb3f6e74d3e..55e51c81be1f 100644
@@ -630,32 +630,42 @@ static void a6xx_build_bw_table(struct a6xx_hfi_msg_bw_table *msg)
 
 static int a6xx_hfi_send_bw_table(struct a6xx_gmu *gmu)

drivers/gpu/drm/msm/adreno/a6xx_hfi.c static void a6xx_build_bw_table(struct a6xx_hfi_msg_bw_table *msg) · line 640

Quoted 10 lines
drivers/gpu/drm/msm/adreno/a6xx_hfi.c +5 -1
 {
-    struct a6xx_hfi_msg_bw_table msg = { 0 };
+    struct a6xx_hfi_msg_bw_table *msg;
     struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
     struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
 
+    if (gmu->bw_table)
+            goto send;
+
+    msg = devm_kzalloc(gmu->dev, sizeof(*msg), GFP_KERNEL);

Is it necessary after being sent? Isn't it better to just kzalloc() it and then kfree() it at the end of the function?

Quoted 8 lines
Quoted 4 lines
Quoted 2 lines

Keeping it around will help to cut down unnecessary work during subsequent gpu wake ups.

Then, I'd say, it is better to make it a part of the a6xx_gpu struct.

I think a6xx_gmu makes more logical sense here.

FWIW, the driver allocates both _gmu and _gpu for all GPUs regardless

Hmm, are we expected to handle / perform BW requests in case of GMU-less devices?

Signature · 2 lines
With best wishes
Dmitry
#10 lore raw
Quoted 64 lines · Dmitry Baryshkov
Quoted 63 lines · Konrad Dybcio
Quoted 62 lines · Dmitry Baryshkov
Quoted 60 lines · Akhil P Oommen
Quoted 48 lines · Dmitry Baryshkov
Quoted 47 lines · Akhil P Oommen

Clang-19 and above sometimes end up with multiple copies of the large a6xx_hfi_msg_bw_table structure on the stack. The problem is that a6xx_hfi_send_bw_table() calls a number of device specific functions to fill the structure, but these create another copy of the structure on the stack which gets copied to the first.

If the functions get inlined, that busts the warning limit:

drivers/gpu/drm/msm/adreno/a6xx_hfi.c:631:12: error: stack frame size (1032) exceeds limit (1024) in 'a6xx_hfi_send_bw_table' [-Werror,-Wframe-larger-than]

Fix this by kmalloc-ating struct a6xx_hfi_msg_bw_table instead of using the stack. Also, use this opportunity to skip re-initializing this table to optimize gpu wake up latency.

Cc Arnd Bergmann <[email protected]>
---
 drivers/gpu/drm/msm/adreno/a6xx_gmu.h |  1 +
 drivers/gpu/drm/msm/adreno/a6xx_hfi.c | 34 ++++++++++++++++++++++------------
 2 files changed, 23 insertions(+), 12 deletions(-)
drivers/gpu/drm/msm/adreno/a6xx_gmu.h +1 -0
index 94b6c5cab6f4..b4a79f88ccf4 100644
@@ -99,6 +99,7 @@ struct a6xx_gmu {
     struct completion pd_gate;
 
     struct qmp *qmp;
+    struct a6xx_hfi_msg_bw_table *bw_table;
 };
 
 static inline u32 gmu_read(struct a6xx_gmu *gmu, u32 offset)
drivers/gpu/drm/msm/adreno/a6xx_hfi.c +1 -1
index cdb3f6e74d3e..55e51c81be1f 100644
@@ -630,32 +630,42 @@ static void a6xx_build_bw_table(struct a6xx_hfi_msg_bw_table *msg)
 
 static int a6xx_hfi_send_bw_table(struct a6xx_gmu *gmu)
 {
-    struct a6xx_hfi_msg_bw_table msg = { 0 };
+    struct a6xx_hfi_msg_bw_table *msg;
     struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
     struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
 
Quoted 7 lines

drivers/gpu/drm/msm/adreno/a6xx_hfi.c static void a6xx_build_bw_table(struct a6xx_hfi_msg_bw_table *msg) · line 640

Quoted 4 lines
drivers/gpu/drm/msm/adreno/a6xx_hfi.c +4 -0
+    if (gmu->bw_table)
+            goto send;
+
+    msg = devm_kzalloc(gmu->dev, sizeof(*msg), GFP_KERNEL);

Is it necessary after being sent? Isn't it better to just kzalloc() it and then kfree() it at the end of the function?

Keeping it around will help to cut down unnecessary work during subsequent gpu wake ups.

Quoted 8 lines
Quoted 5 lines
Quoted 1 line

Then, I'd say, it is better to make it a part of the a6xx_gpu struct.

I think a6xx_gmu makes more logical sense here.

FWIW, the driver allocates both _gmu and _gpu for all GPUs regardless

Hmm, are we expected to handle / perform BW requests in case of GMU-less devices?

opp-table does that for us

In case of no gmu ("gmu wrapper"), Linux is the only entity that controls things

Konrad

#11 lore raw
Quoted 71 lines · Konrad Dybcio
Quoted 70 lines · Dmitry Baryshkov
Quoted 68 lines · Konrad Dybcio
Quoted 59 lines · Dmitry Baryshkov
Quoted 58 lines · Akhil P Oommen
Quoted 55 lines · Dmitry Baryshkov
Quoted 41 lines · Akhil P Oommen

Clang-19 and above sometimes end up with multiple copies of the large a6xx_hfi_msg_bw_table structure on the stack. The problem is that a6xx_hfi_send_bw_table() calls a number of device specific functions to fill the structure, but these create another copy of the structure on the stack which gets copied to the first.

If the functions get inlined, that busts the warning limit:

drivers/gpu/drm/msm/adreno/a6xx_hfi.c:631:12: error: stack frame size (1032) exceeds limit (1024) in 'a6xx_hfi_send_bw_table' [-Werror,-Wframe-larger-than]

Fix this by kmalloc-ating struct a6xx_hfi_msg_bw_table instead of using the stack. Also, use this opportunity to skip re-initializing this table to optimize gpu wake up latency.

Cc Arnd Bergmann <[email protected]>
---
 drivers/gpu/drm/msm/adreno/a6xx_gmu.h |  1 +
 drivers/gpu/drm/msm/adreno/a6xx_hfi.c | 34 ++++++++++++++++++++++------------
 2 files changed, 23 insertions(+), 12 deletions(-)
drivers/gpu/drm/msm/adreno/a6xx_gmu.h +1 -0
index 94b6c5cab6f4..b4a79f88ccf4 100644
@@ -99,6 +99,7 @@ struct a6xx_gmu {
     struct completion pd_gate;
 
     struct qmp *qmp;
+    struct a6xx_hfi_msg_bw_table *bw_table;
 };
 
 static inline u32 gmu_read(struct a6xx_gmu *gmu, u32 offset)
drivers/gpu/drm/msm/adreno/a6xx_hfi.c +0 -0
index cdb3f6e74d3e..55e51c81be1f 100644
@@ -630,32 +630,42 @@ static void a6xx_build_bw_table(struct a6xx_hfi_msg_bw_table *msg)
 
 static int a6xx_hfi_send_bw_table(struct a6xx_gmu *gmu)

drivers/gpu/drm/msm/adreno/a6xx_hfi.c static void a6xx_build_bw_table(struct a6xx_hfi_msg_bw_table *msg) · line 640

Quoted 10 lines
drivers/gpu/drm/msm/adreno/a6xx_hfi.c +5 -1
 {
-    struct a6xx_hfi_msg_bw_table msg = { 0 };
+    struct a6xx_hfi_msg_bw_table *msg;
     struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
     struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
 
+    if (gmu->bw_table)
+            goto send;
+
+    msg = devm_kzalloc(gmu->dev, sizeof(*msg), GFP_KERNEL);

Is it necessary after being sent? Isn't it better to just kzalloc() it and then kfree() it at the end of the function?

Quoted 4 lines
Quoted 2 lines

Keeping it around will help to cut down unnecessary work during subsequent gpu wake ups.

Then, I'd say, it is better to make it a part of the a6xx_gpu struct.

I think a6xx_gmu makes more logical sense here.

FWIW, the driver allocates both _gmu and _gpu for all GPUs regardless

Quoted 7 lines
Quoted 2 lines

Hmm, are we expected to handle / perform BW requests in case of GMU-less devices?

opp-table does that for us

In case of no gmu ("gmu wrapper"), Linux is the only entity that controls things

Ack

Signature · 2 lines
With best wishes
Dmitry
Reviewed-by on v1 1/1
#12 lore raw
In reply to Akhil P Oommen
Quoted 8 lines · Akhil P Oommen

Clang-19 and above sometimes end up with multiple copies of the large a6xx_hfi_msg_bw_table structure on the stack. The problem is that a6xx_hfi_send_bw_table() calls a number of device specific functions to fill the structure, but these create another copy of the structure on the stack which gets copied to the first.

If the functions get inlined, that busts the warning limit:

Quoted 8 lines

drivers/gpu/drm/msm/adreno/a6xx_hfi.c:631:12: error: stack frame size (1032) exceeds limit (1024) in 'a6xx_hfi_send_bw_table' [-Werror,-Wframe-larger-than]

Fix this by kmalloc-ating struct a6xx_hfi_msg_bw_table instead of using the stack. Also, use this opportunity to skip re-initializing this table to optimize gpu wake up latency.

Cc Arnd Bergmann <[email protected]>

Please no empty lines between tags.

Quoted 1 line

After all the discussions:

Quoted 57 lines
---
 drivers/gpu/drm/msm/adreno/a6xx_gmu.h |  1 +
 drivers/gpu/drm/msm/adreno/a6xx_hfi.c | 34 ++++++++++++++++++++++------------
 2 files changed, 23 insertions(+), 12 deletions(-)
drivers/gpu/drm/msm/adreno/a6xx_gmu.h +1 -0
index 94b6c5cab6f4..b4a79f88ccf4 100644
@@ -99,6 +99,7 @@ struct a6xx_gmu {
 	struct completion pd_gate;
 
 	struct qmp *qmp;
+	struct a6xx_hfi_msg_bw_table *bw_table;
 };
 
 static inline u32 gmu_read(struct a6xx_gmu *gmu, u32 offset)
drivers/gpu/drm/msm/adreno/a6xx_hfi.c +14 -8
index cdb3f6e74d3e..55e51c81be1f 100644
@@ -630,32 +630,42 @@ static void a6xx_build_bw_table(struct a6xx_hfi_msg_bw_table *msg)
 
 static int a6xx_hfi_send_bw_table(struct a6xx_gmu *gmu)
 {
-	struct a6xx_hfi_msg_bw_table msg = { 0 };
+	struct a6xx_hfi_msg_bw_table *msg;
 	struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
 	struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
 
+	if (gmu->bw_table)
+		goto send;
+
+	msg = devm_kzalloc(gmu->dev, sizeof(*msg), GFP_KERNEL);
+	if (!msg)
+		return -ENOMEM;
+
 	if (adreno_is_a618(adreno_gpu))
-		a618_build_bw_table(&msg);
+		a618_build_bw_table(msg);
 	else if (adreno_is_a619(adreno_gpu))
-		a619_build_bw_table(&msg);
+		a619_build_bw_table(msg);
 	else if (adreno_is_a640_family(adreno_gpu))
-		a640_build_bw_table(&msg);
+		a640_build_bw_table(msg);
 	else if (adreno_is_a650(adreno_gpu))
-		a650_build_bw_table(&msg);
+		a650_build_bw_table(msg);
 	else if (adreno_is_7c3(adreno_gpu))
-		adreno_7c3_build_bw_table(&msg);
+		adreno_7c3_build_bw_table(msg);
 	else if (adreno_is_a660(adreno_gpu))
-		a660_build_bw_table(&msg);
+		a660_build_bw_table(msg);
 	else if (adreno_is_a690(adreno_gpu))
-		a690_build_bw_table(&msg);

drivers/gpu/drm/msm/adreno/a6xx_hfi.c static void a6xx_build_bw_table(struct a6xx_hfi_msg_bw_table *msg) · line 663

Quoted 10 lines
drivers/gpu/drm/msm/adreno/a6xx_hfi.c +4 -3
+		a690_build_bw_table(msg);
 	else if (adreno_is_a730(adreno_gpu))
-		a730_build_bw_table(&msg);
+		a730_build_bw_table(msg);
 	else if (adreno_is_a740_family(adreno_gpu))
-		a740_build_bw_table(&msg);
+		a740_build_bw_table(msg);
 	else
-		a6xx_build_bw_table(&msg);
+		a6xx_build_bw_table(msg);

Note for the future improvement: this begs to be migrated to the catalog data, adding device-specific callback instead of this if/else series.

Quoted 18 lines
drivers/gpu/drm/msm/adreno/a6xx_hfi.c +4 -2
+
+	gmu->bw_table = msg;
 
-	return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_BW_TABLE, &msg, sizeof(msg),
+send:
+	return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_BW_TABLE, gmu->bw_table, sizeof(*(gmu->bw_table)),
 		NULL, 0);
 }
 
 
---

base-commit: 74c374648ed08efb2ef339656f2764c28c046956
change-id: 20241024-stack-size-fix-28af7abd3fab

Best regards,

Signature · 1 line
Akhil P Oommen <[email protected]>
Signature · 2 lines
With best wishes
Dmitry