Skip to contentSkip to conversation

amdgpu: ratelimit on vm faults to avoid spaming console

PATCH Superseded Edward O'Callaghan 2 participants 5 messages Nov 7, 2016 3 messages not in the archive

Subsystems (2)

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

These are rather minor however should help stop some folks machines grinding to a halt when a userspace application somehow gets the GPU into some horrible state causing the console to fill very quickly. Applies on top of master.

Please kindly review,

Edward O'Callaghan (2):
[PATCH 1/2] amdgpu: Use dev_err() over vanilla printk() in [PATCH 2/2] amdgpu: Use 'dev_err_ratelimited()' over 'dev_err()' on
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 1478490470-27089-1-git-send-email-funfunctor-dczkZgxz+BNUPWh3PAxdjQ@public.gmane.org

#2 lore raw

PATCH 1/2 amdgpu: Use dev_err() over vanilla printk() in vm_decode_fault()

---
 drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c | 2 +-
 drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c +1 -1
index aa0c4b9..a0df431 100644
@@ -711,7 +711,7 @@ static void gmc_v7_0_vm_decode_fault(struct amdgpu_device *adev,
 	mc_id = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
 			      MEMORY_CLIENT_ID);
 
-	printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
+	dev_err(adev->dev, "VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
 	       protections, vmid, addr,
 	       REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
 			     MEMORY_CLIENT_RW) ?
drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c +1 -1
index a16b220..7285294 100644
@@ -837,7 +837,7 @@ static void gmc_v8_0_vm_decode_fault(struct amdgpu_device *adev,
 	mc_id = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
 			      MEMORY_CLIENT_ID);
 
-	printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
+	dev_err(adev->dev, "VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
 	       protections, vmid, addr,
 	       REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
 			     MEMORY_CLIENT_RW) ?
Signature · 6 lines
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
#3 lore raw

PATCH 2/2 amdgpu: Use 'dev_err_ratelimited()' over 'dev_err()' on vm faults

In reply to a message not in the archive

It can be the case that upon GPU page faults we start trashing the logs, and so let us ratelimit here to avoid that.

---
 drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c | 8 ++++----
 drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c | 8 ++++----
 drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c | 8 ++++----
 3 files changed, 12 insertions(+), 12 deletions(-)
drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c +4 -4
index b13c8aa..79af3f2 100644
@@ -549,7 +549,7 @@ static void gmc_v6_0_vm_decode_fault(struct amdgpu_device *adev,
 	mc_id = REG_GET_FIELD(status, mmVM_CONTEXT1_PROTECTION_FAULT_STATUS,
 			      xxMEMORY_CLIENT_ID);
 
-	dev_err(adev->dev, "VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
+	dev_err_ratelimited(adev->dev, "VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
 	       protections, vmid, addr,
 	       REG_GET_FIELD(status, mmVM_CONTEXT1_PROTECTION_FAULT_STATUS,
 			     xxMEMORY_CLIENT_RW) ?
@@ -1007,11 +1007,11 @@ static int gmc_v6_0_process_interrupt(struct amdgpu_device *adev,
 	if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_FIRST)
 		gmc_v6_0_set_fault_enable_default(adev, false);
 
-	dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n",
+	dev_err_ratelimited(adev->dev, "GPU fault detected: %d 0x%08x\n",
 		entry->src_id, entry->src_data);
-	dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
+	dev_err_ratelimited(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
 		addr);
-	dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
+	dev_err_ratelimited(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
 		status);
 	gmc_v6_0_vm_decode_fault(adev, status, addr, 0);
 
drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c +4 -4
index a0df431..4ed97c4 100644
@@ -711,7 +711,7 @@ static void gmc_v7_0_vm_decode_fault(struct amdgpu_device *adev,
 	mc_id = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
 			      MEMORY_CLIENT_ID);
 
-	dev_err(adev->dev, "VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
+	dev_err_ratelimited(adev->dev, "VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
 	       protections, vmid, addr,
 	       REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
 			     MEMORY_CLIENT_RW) ?
@@ -1198,11 +1198,11 @@ static int gmc_v7_0_process_interrupt(struct amdgpu_device *adev,
 	if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_FIRST)
 		gmc_v7_0_set_fault_enable_default(adev, false);
 
-	dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n",
+	dev_err_ratelimited(adev->dev, "GPU fault detected: %d 0x%08x\n",
 		entry->src_id, entry->src_data);
-	dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
+	dev_err_ratelimited(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
 		addr);
-	dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
+	dev_err_ratelimited(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
 		status);
 	gmc_v7_0_vm_decode_fault(adev, status, addr, mc_client);
 
drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c +4 -4
index 7285294..050a884 100644
@@ -837,7 +837,7 @@ static void gmc_v8_0_vm_decode_fault(struct amdgpu_device *adev,
 	mc_id = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
 			      MEMORY_CLIENT_ID);
 
-	dev_err(adev->dev, "VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
+	dev_err_ratelimited(adev->dev, "VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
 	       protections, vmid, addr,
 	       REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
 			     MEMORY_CLIENT_RW) ?
@@ -1242,11 +1242,11 @@ static int gmc_v8_0_process_interrupt(struct amdgpu_device *adev,
 	if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_FIRST)
 		gmc_v8_0_set_fault_enable_default(adev, false);
 
-	dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n",
+	dev_err_ratelimited(adev->dev, "GPU fault detected: %d 0x%08x\n",
 		entry->src_id, entry->src_data);
-	dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
+	dev_err_ratelimited(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
 		addr);
-	dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
+	dev_err_ratelimited(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
 		status);
 	gmc_v8_0_vm_decode_fault(adev, status, addr, mc_client);
 
Signature · 6 lines
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 1478490470-27089-3-git-send-email-funfunctor-dczkZgxz+BNUPWh3PAxdjQ@public.gmane.org

#5 lore raw

Re: [PATCH 2/2] amdgpu: Use 'dev_err_ratelimited()' over 'dev_err()' on vm faults

Quoted 27 lines · Edward O'Callaghan

It can be the case that upon GPU page faults we start trashing the logs, and so let us ratelimit here to avoid that.

---
 drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c | 8 ++++----
 drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c | 8 ++++----
 drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c | 8 ++++----
 3 files changed, 12 insertions(+), 12 deletions(-)
drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c +1 -1
index b13c8aa..79af3f2 100644
@@ -549,7 +549,7 @@ static void gmc_v6_0_vm_decode_fault(struct amdgpu_device *adev,
 	mc_id = REG_GET_FIELD(status, mmVM_CONTEXT1_PROTECTION_FAULT_STATUS,
 			      xxMEMORY_CLIENT_ID);
 
-	dev_err(adev->dev, "VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
+	dev_err_ratelimited(adev->dev, "VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
 	       protections, vmid, addr,
 	       REG_GET_FIELD(status, mmVM_CONTEXT1_PROTECTION_FAULT_STATUS,
 			     xxMEMORY_CLIENT_RW) ?
@@ -1007,11 +1007,11 @@ static int gmc_v6_0_process_interrupt(struct amdgpu_device *adev,
 	if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_FIRST)
 		gmc_v6_0_set_fault_enable_default(adev, false);
 

drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c static int gmc_v6_0_process_interrupt(struct amdgpu_device *adev, · line 1016

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c +3 -3
-	dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n",
+	dev_err_ratelimited(adev->dev, "GPU fault detected: %d 0x%08x\n",
 		entry->src_id, entry->src_data);
-	dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
+	dev_err_ratelimited(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
 		addr);
-	dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
+	dev_err_ratelimited(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
 		status);
 	gmc_v6_0_vm_decode_fault(adev, status, addr, 0);

If calling dev_err_ratelimited separately for each line means that some lines corresponding to a single VM fault may or may not appear depending on the rate, it'd be better to use a single call per VM fault.

Signature · 6 lines
Earthling Michel Dänzer               |               http://www.amd.com
Libre software enthusiast             |             Mesa and X developer
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): e46fd0b0-7d0d-d502-77e2-400c38cef605-otUistvHUpPR7s880joybQ@public.gmane.org

#7 lore raw

Re: [PATCH 2/2] amdgpu: Use 'dev_err_ratelimited()' over 'dev_err()' on vm faults

Quoted 28 lines · Michel Dänzer
Quoted 27 lines · Edward O'Callaghan

It can be the case that upon GPU page faults we start trashing the logs, and so let us ratelimit here to avoid that.

---
 drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c | 8 ++++----
 drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c | 8 ++++----
 drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c | 8 ++++----
 3 files changed, 12 insertions(+), 12 deletions(-)
drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c +1 -1
index b13c8aa..79af3f2 100644
@@ -549,7 +549,7 @@ static void gmc_v6_0_vm_decode_fault(struct amdgpu_device *adev,
 	mc_id = REG_GET_FIELD(status, mmVM_CONTEXT1_PROTECTION_FAULT_STATUS,
 			      xxMEMORY_CLIENT_ID);
 
-	dev_err(adev->dev, "VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
+	dev_err_ratelimited(adev->dev, "VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
 	       protections, vmid, addr,
 	       REG_GET_FIELD(status, mmVM_CONTEXT1_PROTECTION_FAULT_STATUS,
 			     xxMEMORY_CLIENT_RW) ?
@@ -1007,11 +1007,11 @@ static int gmc_v6_0_process_interrupt(struct amdgpu_device *adev,
 	if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_FIRST)
 		gmc_v6_0_set_fault_enable_default(adev, false);
 
Quoted 14 lines

drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c static int gmc_v6_0_process_interrupt(struct amdgpu_device *adev, · line 1016

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c +3 -3
-	dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n",
+	dev_err_ratelimited(adev->dev, "GPU fault detected: %d 0x%08x\n",
 		entry->src_id, entry->src_data);
-	dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
+	dev_err_ratelimited(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
 		addr);
-	dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
+	dev_err_ratelimited(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
 		status);
 	gmc_v6_0_vm_decode_fault(adev, status, addr, 0);

If calling dev_err_ratelimited separately for each line means that some lines corresponding to a single VM fault may or may not appear depending on the rate, it'd be better to use a single call per VM fault.

Ah ok, yes I was not 100% that was definitely the case - easy to fix, I will resend a update to address it. Thanks, Ed.

Quoted 2 lines

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx