drm/nouveau/subdev/bus: Ratelimit logging for fault errors
Subsystems (3)
Lyude Paul
maintainerPATCH 1/1 drm/nouveau/subdev/bus: Ratelimit logging for fault errors
There's plenty of ways to fudge the GPU when developing on nouveau by mistake, some of which can result in nouveau seriously spamming dmesg with fault errors. This can be somewhat annoying, as it can quickly overrun the message buffer (or your terminal emulator's buffer) and get rid of actually useful feedback from the driver. While working on my new atomic only MST branch, I ran into this issue a couple of times.
So, let's fix this by adding nvkm_error_ratelimited(), and using it to ratelimit errors from faults. This should be fine for developers, since it's nearly always only the first few faults that we care about seeing. Plus, you can turn off rate limiting in the kernel if you really need to.
--- drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h | 2 ++ drivers/gpu/drm/nouveau/nvkm/subdev/bus/gf100.c | 14 +++++++------- drivers/gpu/drm/nouveau/nvkm/subdev/bus/nv31.c | 6 +++--- drivers/gpu/drm/nouveau/nvkm/subdev/bus/nv50.c | 6 +++--- 4 files changed, 15 insertions(+), 13 deletions(-)
drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h +2 -0
@@ -62,4 +62,6 @@ void nvkm_subdev_intr(struct nvkm_subdev *); #define nvkm_debug(s,f,a...) nvkm_printk((s), DEBUG, info, f, ##a) #define nvkm_trace(s,f,a...) nvkm_printk((s), TRACE, info, f, ##a) #define nvkm_spam(s,f,a...) nvkm_printk((s), SPAM, dbg, f, ##a) + +#define nvkm_error_ratelimited(s,f,a...) nvkm_printk((s), ERROR, err_ratelimited, f, ##a)#endif
drivers/gpu/drm/nouveau/nvkm/subdev/bus/gf100.c +7 -7
@@ -35,13 +35,13 @@ gf100_bus_intr(struct nvkm_bus *bus)u32 addr =nvkm_rd32 (device,0x009084 );u32 data =nvkm_rd32 (device,0x009088 ); -nvkm_error (subdev, -"MMIO %s of %08x FAULT at %06x [ %s%s%s]\n" , - (addr &0x00000002 ) ?"write" :"read" , data, - (addr &0x00fffffc ), - (stat &0x00000002 ) ?"!ENGINE " :"" , - (stat &0x00000004 ) ?"PRIVRING " :"" , - (stat &0x00000008 ) ?"TIMEOUT " :"" ); +nvkm_error_ratelimited (subdev, +"MMIO %s of %08x FAULT at %06x [ %s%s%s]\n" , + (addr &0x00000002 ) ?"write" :"read" , data, + (addr &0x00fffffc ), + (stat &0x00000002 ) ?"!ENGINE " :"" , + (stat &0x00000004 ) ?"PRIVRING " :"" , + (stat &0x00000008 ) ?"TIMEOUT " :"" );nvkm_wr32 (device,0x009084 ,0x00000000 );nvkm_wr32 (device,0x001100 , (stat &0x0000000e ));
drivers/gpu/drm/nouveau/nvkm/subdev/bus/nv31.c +3 -3
@@ -45,9 +45,9 @@ nv31_bus_intr(struct nvkm_bus *bus)u32 addr =nvkm_rd32 (device,0x009084 );u32 data =nvkm_rd32 (device,0x009088 ); -nvkm_error (subdev,"MMIO %s of %08x FAULT at %06x\n" , - (addr &0x00000002 ) ?"write" :"read" , data, - (addr &0x00fffffc )); +nvkm_error_ratelimited (subdev,"MMIO %s of %08x FAULT at %06x\n" , + (addr &0x00000002 ) ?"write" :"read" , data, + (addr &0x00fffffc )); stat &= ~0x00000008 ;nvkm_wr32 (device,0x001100 ,0x00000008 );
drivers/gpu/drm/nouveau/nvkm/subdev/bus/nv50.c +3 -3
@@ -60,9 +60,9 @@ nv50_bus_intr(struct nvkm_bus *bus)u32 addr =nvkm_rd32 (device,0x009084 );u32 data =nvkm_rd32 (device,0x009088 ); -nvkm_error (subdev,"MMIO %s of %08x FAULT at %06x\n" , - (addr &0x00000002 ) ?"write" :"read" , data, - (addr &0x00fffffc )); +nvkm_error_ratelimited (subdev,"MMIO %s of %08x FAULT at %06x\n" , + (addr &0x00000002 ) ?"write" :"read" , data, + (addr &0x00fffffc )); stat &= ~0x00000008 ;nvkm_wr32 (device,0x001100 ,0x00000008 );
Signature · 1 line
2.35.1
Quoted 8 lines · Lyude Paul
There's plenty of ways to fudge the GPU when developing on nouveau by mistake, some of which can result in nouveau seriously spamming dmesg with fault errors. This can be somewhat annoying, as it can quickly overrun the message buffer (or your terminal emulator's buffer) and get rid of actually useful feedback from the driver. While working on my new atomic only MST branch, I ran into this issue a couple of times.
Quoted 5 lines
So, let's fix this by adding
nvkm_error_ratelimited(), and using it to ratelimit errors from faults. This should be fine for developers, since it's nearly always only the first few faults that we care about seeing. Plus, you can turn off rate limiting in the kernel if you really need to.
good idea :)
Although I think we ultimately need a detection of "GPU state is trashed" where we cease to do anything and disable interrupts as some of those issues actually cause the kernel to get DOSed by nouveau spamming/handling interrupts. Not sure if we should go with this patch in the meantime, or if we should just go straight ahead and disable interrupts, which would also stop the spam.
Do you know if this patch here is enough to get the system to reboot normally or is it only spamming less?
Quoted 81 lines
Signed-off-by Lyude Paul <[email protected]>--- drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h | 2 ++ drivers/gpu/drm/nouveau/nvkm/subdev/bus/gf100.c | 14 +++++++------- drivers/gpu/drm/nouveau/nvkm/subdev/bus/nv31.c | 6 +++--- drivers/gpu/drm/nouveau/nvkm/subdev/bus/nv50.c | 6 +++--- 4 files changed, 15 insertions(+), 13 deletions(-)drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h +2
-0@@ -62,4 +62,6 @@ void nvkm_subdev_intr(struct nvkm_subdev *); #define nvkm_debug(s,f,a...) nvkm_printk((s), DEBUG, info, f, ##a) #define nvkm_trace(s,f,a...) nvkm_printk((s), TRACE, info, f, ##a) #define nvkm_spam(s,f,a...) nvkm_printk((s), SPAM, dbg, f, ##a) + +#define nvkm_error_ratelimited(s,f,a...) nvkm_printk((s), ERROR, err_ratelimited, f, ##a)#endif drivers/gpu/drm/nouveau/nvkm/subdev/bus/gf100.c +7
-7@@ -35,13 +35,13 @@ gf100_bus_intr(struct nvkm_bus *bus)u32 addr =nvkm_rd32 (device,0x009084 );u32 data =nvkm_rd32 (device,0x009088 ); -nvkm_error (subdev, -"MMIO %s of %08x FAULT at %06x [ %s%s%s]\n" , - (addr &0x00000002 ) ?"write" :"read" , data, - (addr &0x00fffffc ), - (stat &0x00000002 ) ?"!ENGINE " :"" , - (stat &0x00000004 ) ?"PRIVRING " :"" , - (stat &0x00000008 ) ?"TIMEOUT " :"" ); +nvkm_error_ratelimited (subdev, +"MMIO %s of %08x FAULT at %06x [ %s%s%s]\n" , + (addr &0x00000002 ) ?"write" :"read" , data, + (addr &0x00fffffc ), + (stat &0x00000002 ) ?"!ENGINE " :"" , + (stat &0x00000004 ) ?"PRIVRING " :"" , + (stat &0x00000008 ) ?"TIMEOUT " :"" );nvkm_wr32 (device,0x009084 ,0x00000000 );nvkm_wr32 (device,0x001100 , (stat &0x0000000e ));drivers/gpu/drm/nouveau/nvkm/subdev/bus/nv31.c +3
-3@@ -45,9 +45,9 @@ nv31_bus_intr(struct nvkm_bus *bus)u32 addr =nvkm_rd32 (device,0x009084 );u32 data =nvkm_rd32 (device,0x009088 ); -nvkm_error (subdev,"MMIO %s of %08x FAULT at %06x\n" , - (addr &0x00000002 ) ?"write" :"read" , data, - (addr &0x00fffffc )); +nvkm_error_ratelimited (subdev,"MMIO %s of %08x FAULT at %06x\n" , + (addr &0x00000002 ) ?"write" :"read" , data, + (addr &0x00fffffc )); stat &= ~0x00000008 ;nvkm_wr32 (device,0x001100 ,0x00000008 );drivers/gpu/drm/nouveau/nvkm/subdev/bus/nv50.c +3
-4@@ -60,9 +60,9 @@ nv50_bus_intr(struct nvkm_bus *bus)u32 addr =nvkm_rd32 (device,0x009084 );u32 data =nvkm_rd32 (device,0x009088 ); -nvkm_error (subdev,"MMIO %s of %08x FAULT at %06x\n" , - (addr &0x00000002 ) ?"write" :"read" , data, - (addr &0x00fffffc )); +nvkm_error_ratelimited (subdev,"MMIO %s of %08x FAULT at %06x\n" , + (addr &0x00000002 ) ?"write" :"read" , data, + (addr &0x00fffffc )); stat &= ~0x00000008 ;nvkm_wr32 (device,0x001100 ,0x00000008 ); --2.35.1
Lyude Paul
maintainerQuoted 25 lines · Karol Herbst
Quoted 9 lines · Lyude Paul
There's plenty of ways to fudge the GPU when developing on nouveau by mistake, some of which can result in nouveau seriously spamming dmesg with fault errors. This can be somewhat annoying, as it can quickly overrun the message buffer (or your terminal emulator's buffer) and get rid of actually
useful feedback from the driver. While working on my new atomic only MST branch, I ran into this issue a couple of times.Quoted 5 lines
So, let's fix this by adding
nvkm_error_ratelimited(), and using it to ratelimit errors from faults. This should be fine for developers, since it's nearly always only the first few faults that we care about seeing. Plus, you can turn off rate limiting in the kernel if you really need to.good idea :)
Although I think we ultimately need a detection of "GPU state is trashed" where we cease to do anything and disable interrupts as some of those issues actually cause the kernel to get DOSed by nouveau spamming/handling interrupts. Not sure if we should go with this patch in the meantime, or if we should just go straight ahead and disable interrupts, which would also stop the spam.
Quoted 2 lines
Do you know if this patch here is enough to get the system to reboot normally or is it only spamming less?
I'd have to try to come up with a setup that can hit this again, although it might not be that hard - last time I hit this it was because I was programming the wrong PBN values for MST payloads into nvidia's hardware.
So I can't remember for sure if I could reboot, but I do know I could actually log into the system easily and do stuff with ratelimiting applied so it's definitely better then the current situation.
Quoted 97 lines
Quoted 96 lines
Signed-off-by Lyude Paul <[email protected]>---
drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h | 2 ++ drivers/gpu/drm/nouveau/nvkm/subdev/bus/gf100.c | 14 +++++++------- drivers/gpu/drm/nouveau/nvkm/subdev/bus/nv31.c | 6 +++--- drivers/gpu/drm/nouveau/nvkm/subdev/bus/nv50.c | 6 +++---4 files changed, 15 insertions(+), 13 deletions(-)drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h +2
-0@@ -62,4 +62,6 @@ void nvkm_subdev_intr(struct nvkm_subdev *); #define nvkm_debug(s,f,a...) nvkm_printk((s), DEBUG, info, f, ##a) #define nvkm_trace(s,f,a...) nvkm_printk((s), TRACE, info, f, ##a) #define nvkm_spam(s,f,a...) nvkm_printk((s), SPAM, dbg, f, ##a) + +#define nvkm_error_ratelimited(s,f,a...) nvkm_printk((s), ERROR, err_ratelimited, f, ##a)#endif
drivers/gpu/drm/nouveau/nvkm/subdev/bus/gf100.c +6
-7@@ -35,13 +35,13 @@ gf100_bus_intr(struct nvkm_bus *bus) u32 addr = nvkm_rd32(device, 0x009084); u32 data = nvkm_rd32(device, 0x009088); - nvkm_error(subdev, - "MMIO %s of %08x FAULT at %06x [ %s%s%s]\n", - (addr & 0x00000002) ? "write" : "read", data, - (addr & 0x00fffffc), - (stat & 0x00000002) ? "!ENGINE " : "", - (stat & 0x00000004) ? "PRIVRING " : "", - (stat & 0x00000008) ? "TIMEOUT " : ""); + nvkm_error_ratelimited(subdev, + "MMIO %s of %08x FAULT at %06x [ %s%s%s]\n", + (addr & 0x00000002) ? "write" : "read", data, + (addr & 0x00fffffc), + (stat & 0x00000002) ? "!ENGINE " : "", + (stat & 0x00000004) ? "PRIVRING " :"",
+ (stat &0x00000008) ? "TIMEOUT " :
"");
nvkm_wr32(device, 0x009084, 0x00000000);nvkm_wr32(device, 0x001100, (stat & 0x0000000e));drivers/gpu/drm/nouveau/nvkm/subdev/bus/nv31.c +3
-3@@ -45,9 +45,9 @@ nv31_bus_intr(struct nvkm_bus *bus) u32 addr = nvkm_rd32(device, 0x009084); u32 data = nvkm_rd32(device, 0x009088); - nvkm_error(subdev, "MMIO %s of %08x FAULT at %06x\n", - (addr & 0x00000002) ? "write" : "read", data, - (addr & 0x00fffffc)); + nvkm_error_ratelimited(subdev, "MMIO %s of %08x FAULT at %06x\n", + (addr & 0x00000002) ? "write" :"read" , data, + (addr & 0x00fffffc));stat &= ~0x00000008;
nvkm_wr32(device, 0x001100, 0x00000008);drivers/gpu/drm/nouveau/nvkm/subdev/bus/nv50.c +3
-3@@ -60,9 +60,9 @@ nv50_bus_intr(struct nvkm_bus *bus) u32 addr = nvkm_rd32(device, 0x009084); u32 data = nvkm_rd32(device, 0x009088); - nvkm_error(subdev, "MMIO %s of %08x FAULT at %06x\n", - (addr & 0x00000002) ? "write" : "read", data, - (addr & 0x00fffffc)); + nvkm_error_ratelimited(subdev, "MMIO %s of %08x FAULT at %06x\n", + (addr & 0x00000002) ? "write" :"read" , data, + (addr & 0x00fffffc));stat &= ~0x00000008;
nvkm_wr32(device, 0x001100, 0x00000008); --
2.35.1
Signature · 3 lines
Cheers, Lyude Paul (she/her) Software Engineer at Red Hat
Quoted 30 lines · Lyude Paul
Quoted 25 lines · Karol Herbst
Quoted 9 lines · Lyude Paul
There's plenty of ways to fudge the GPU when developing on nouveau by mistake, some of which can result in nouveau seriously spamming dmesg with fault errors. This can be somewhat annoying, as it can quickly overrun the message buffer (or your terminal emulator's buffer) and get rid of actually
useful feedback from the driver. While working on my new atomic only MST branch, I ran into this issue a couple of times.Quoted 5 lines
So, let's fix this by adding
nvkm_error_ratelimited(), and using it to ratelimit errors from faults. This should be fine for developers, since it's nearly always only the first few faults that we care about seeing. Plus, you can turn off rate limiting in the kernel if you really need to.good idea :)
Although I think we ultimately need a detection of "GPU state is trashed" where we cease to do anything and disable interrupts as some of those issues actually cause the kernel to get DOSed by nouveau spamming/handling interrupts. Not sure if we should go with this patch in the meantime, or if we should just go straight ahead and disable interrupts, which would also stop the spam.
Quoted 2 lines
Do you know if this patch here is enough to get the system to reboot normally or is it only spamming less?
Quoted 8 lines
I'd have to try to come up with a setup that can hit this again, although it might not be that hard - last time I hit this it was because I was programming the wrong PBN values for MST payloads into nvidia's hardware.
So I can't remember for sure if I could reboot, but I do know I could actually log into the system easily and do stuff with ratelimiting applied so it's definitely better then the current situation.
I can play around with your patch a little, as with broken multithreading and OpenCL I can trash my entire machine within split seconds quite reliably :)
Quoted 103 lines
Quoted 91 lines
Quoted 91 lines
Signed-off-by Lyude Paul <[email protected]>--- drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h | 2 ++ drivers/gpu/drm/nouveau/nvkm/subdev/bus/gf100.c | 14 +++++++------- drivers/gpu/drm/nouveau/nvkm/subdev/bus/nv31.c | 6 +++--- drivers/gpu/drm/nouveau/nvkm/subdev/bus/nv50.c | 6 +++--- 4 files changed, 15 insertions(+), 13 deletions(-)drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h +2
-0@@ -62,4 +62,6 @@ void nvkm_subdev_intr(struct nvkm_subdev *); #define nvkm_debug(s,f,a...) nvkm_printk((s), DEBUG, info, f, ##a) #define nvkm_trace(s,f,a...) nvkm_printk((s), TRACE, info, f, ##a) #define nvkm_spam(s,f,a...) nvkm_printk((s), SPAM, dbg, f, ##a) + +#define nvkm_error_ratelimited(s,f,a...) nvkm_printk((s), ERROR, err_ratelimited, f, ##a)#endif
drivers/gpu/drm/nouveau/nvkm/subdev/bus/gf100.c +6
-7@@ -35,13 +35,13 @@ gf100_bus_intr(struct nvkm_bus *bus)u32 addr =nvkm_rd32 (device,0x009084 );u32 data =nvkm_rd32 (device,0x009088 ); -nvkm_error (subdev, -"MMIO %s of %08x FAULT at %06x [ %s%s%s]\n" , - (addr &0x00000002 ) ?"write" :"read" , data, - (addr &0x00fffffc ), - (stat &0x00000002 ) ?"!ENGINE " :"" , - (stat &0x00000004 ) ?"PRIVRING " :"" , - (stat &0x00000008 ) ?"TIMEOUT " :"" ); + nvkm_error_ratelimited(subdev, + "MMIO %s of %08x FAULT at %06x [ %s%s%s]\n", + (addr & 0x00000002) ? "write" : "read", data, + (addr & 0x00fffffc), + (stat & 0x00000002) ? "!ENGINE " : "", + (stat & 0x00000004) ? "PRIVRING " :"",
+ (stat &0x00000008) ? "TIMEOUT " :
"");nvkm_wr32(device, 0x009084, 0x00000000); nvkm_wr32(device, 0x001100, (stat & 0x0000000e));drivers/gpu/drm/nouveau/nvkm/subdev/bus/nv31.c +3
-3@@ -45,9 +45,9 @@ nv31_bus_intr(struct nvkm_bus *bus)u32 addr =nvkm_rd32 (device,0x009084 );u32 data =nvkm_rd32 (device,0x009088 ); -nvkm_error (subdev,"MMIO %s of %08x FAULT at %06x\n" , - (addr &0x00000002 ) ?"write" :"read" , data, - (addr &0x00fffffc )); + nvkm_error_ratelimited(subdev, "MMIO %s of %08x FAULT at %06x\n", + (addr & 0x00000002) ? "write" :"read" , data, + (addr & 0x00fffffc));stat &= ~0x00000008; nvkm_wr32(device, 0x001100, 0x00000008);drivers/gpu/drm/nouveau/nvkm/subdev/bus/nv50.c +3
-3@@ -60,9 +60,9 @@ nv50_bus_intr(struct nvkm_bus *bus)u32 addr =nvkm_rd32 (device,0x009084 );u32 data =nvkm_rd32 (device,0x009088 ); -nvkm_error (subdev,"MMIO %s of %08x FAULT at %06x\n" , - (addr &0x00000002 ) ?"write" :"read" , data, - (addr &0x00fffffc )); + nvkm_error_ratelimited(subdev, "MMIO %s of %08x FAULT at %06x\n", + (addr & 0x00000002) ? "write" :"read" , data, + (addr & 0x00fffffc));Quoted 6 lines
Quoted 5 lines
stat &= ~0x00000008; nvkm_wr32(device, 0x001100, 0x00000008);--
2.35.1--
Cheers,
Lyude Paul (she/her)
Software Engineer at Red Hat
not able to hit any error on my machine, but regardless:
I suspect there are more places where we could put it, but we can add those later.
Anyway, I think it's valuable to push it through fixes, not sure how far back we want to CC stable though.
Quoted 94 lines · Lyude Paul
There's plenty of ways to fudge the GPU when developing on nouveau by mistake, some of which can result in nouveau seriously spamming dmesg with fault errors. This can be somewhat annoying, as it can quickly overrun the message buffer (or your terminal emulator's buffer) and get rid of actually useful feedback from the driver. While working on my new atomic only MST branch, I ran into this issue a couple of times.
So, let's fix this by adding
nvkm_error_ratelimited(), and using it to ratelimit errors from faults. This should be fine for developers, since it's nearly always only the first few faults that we care about seeing. Plus, you can turn off rate limiting in the kernel if you really need to.Signed-off-by Lyude Paul <[email protected]>--- drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h | 2 ++ drivers/gpu/drm/nouveau/nvkm/subdev/bus/gf100.c | 14 +++++++------- drivers/gpu/drm/nouveau/nvkm/subdev/bus/nv31.c | 6 +++--- drivers/gpu/drm/nouveau/nvkm/subdev/bus/nv50.c | 6 +++--- 4 files changed, 15 insertions(+), 13 deletions(-)drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h +2
-0@@ -62,4 +62,6 @@ void nvkm_subdev_intr(struct nvkm_subdev *); #define nvkm_debug(s,f,a...) nvkm_printk((s), DEBUG, info, f, ##a) #define nvkm_trace(s,f,a...) nvkm_printk((s), TRACE, info, f, ##a) #define nvkm_spam(s,f,a...) nvkm_printk((s), SPAM, dbg, f, ##a) + +#define nvkm_error_ratelimited(s,f,a...) nvkm_printk((s), ERROR, err_ratelimited, f, ##a)#endif drivers/gpu/drm/nouveau/nvkm/subdev/bus/gf100.c +7
-7@@ -35,13 +35,13 @@ gf100_bus_intr(struct nvkm_bus *bus)u32 addr =nvkm_rd32 (device,0x009084 );u32 data =nvkm_rd32 (device,0x009088 ); -nvkm_error (subdev, -"MMIO %s of %08x FAULT at %06x [ %s%s%s]\n" , - (addr &0x00000002 ) ?"write" :"read" , data, - (addr &0x00fffffc ), - (stat &0x00000002 ) ?"!ENGINE " :"" , - (stat &0x00000004 ) ?"PRIVRING " :"" , - (stat &0x00000008 ) ?"TIMEOUT " :"" ); +nvkm_error_ratelimited (subdev, +"MMIO %s of %08x FAULT at %06x [ %s%s%s]\n" , + (addr &0x00000002 ) ?"write" :"read" , data, + (addr &0x00fffffc ), + (stat &0x00000002 ) ?"!ENGINE " :"" , + (stat &0x00000004 ) ?"PRIVRING " :"" , + (stat &0x00000008 ) ?"TIMEOUT " :"" );nvkm_wr32 (device,0x009084 ,0x00000000 );nvkm_wr32 (device,0x001100 , (stat &0x0000000e ));drivers/gpu/drm/nouveau/nvkm/subdev/bus/nv31.c +3
-3@@ -45,9 +45,9 @@ nv31_bus_intr(struct nvkm_bus *bus)u32 addr =nvkm_rd32 (device,0x009084 );u32 data =nvkm_rd32 (device,0x009088 ); -nvkm_error (subdev,"MMIO %s of %08x FAULT at %06x\n" , - (addr &0x00000002 ) ?"write" :"read" , data, - (addr &0x00fffffc )); +nvkm_error_ratelimited (subdev,"MMIO %s of %08x FAULT at %06x\n" , + (addr &0x00000002 ) ?"write" :"read" , data, + (addr &0x00fffffc )); stat &= ~0x00000008 ;nvkm_wr32 (device,0x001100 ,0x00000008 );drivers/gpu/drm/nouveau/nvkm/subdev/bus/nv50.c +3
-4@@ -60,9 +60,9 @@ nv50_bus_intr(struct nvkm_bus *bus)u32 addr =nvkm_rd32 (device,0x009084 );u32 data =nvkm_rd32 (device,0x009088 ); -nvkm_error (subdev,"MMIO %s of %08x FAULT at %06x\n" , - (addr &0x00000002 ) ?"write" :"read" , data, - (addr &0x00fffffc )); +nvkm_error_ratelimited (subdev,"MMIO %s of %08x FAULT at %06x\n" , + (addr &0x00000002 ) ?"write" :"read" , data, + (addr &0x00fffffc )); stat &= ~0x00000008 ;nvkm_wr32 (device,0x001100 ,0x00000008 ); --2.35.1
Actually, there is another place we have to put this, we can spam "SCHED_ERROR" things which are triggered within multiple drivers/gpu/drm/nouveau/nvkm/engine/fifo/ files.
Quoted 106 lines · Karol Herbst
not able to hit any error on my machine, but regardless:
Reviewed-by Karol Herbst <[email protected]>I suspect there are more places where we could put it, but we can add those later.
Anyway, I think it's valuable to push it through fixes, not sure how far back we want to CC stable though.
Quoted 94 lines · Lyude Paul
There's plenty of ways to fudge the GPU when developing on nouveau by mistake, some of which can result in nouveau seriously spamming dmesg with fault errors. This can be somewhat annoying, as it can quickly overrun the message buffer (or your terminal emulator's buffer) and get rid of actually useful feedback from the driver. While working on my new atomic only MST branch, I ran into this issue a couple of times.
So, let's fix this by adding
nvkm_error_ratelimited(), and using it to ratelimit errors from faults. This should be fine for developers, since it's nearly always only the first few faults that we care about seeing. Plus, you can turn off rate limiting in the kernel if you really need to.Signed-off-by Lyude Paul <[email protected]>--- drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h | 2 ++ drivers/gpu/drm/nouveau/nvkm/subdev/bus/gf100.c | 14 +++++++------- drivers/gpu/drm/nouveau/nvkm/subdev/bus/nv31.c | 6 +++--- drivers/gpu/drm/nouveau/nvkm/subdev/bus/nv50.c | 6 +++--- 4 files changed, 15 insertions(+), 13 deletions(-)drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h +2
-0@@ -62,4 +62,6 @@ void nvkm_subdev_intr(struct nvkm_subdev *); #define nvkm_debug(s,f,a...) nvkm_printk((s), DEBUG, info, f, ##a) #define nvkm_trace(s,f,a...) nvkm_printk((s), TRACE, info, f, ##a) #define nvkm_spam(s,f,a...) nvkm_printk((s), SPAM, dbg, f, ##a) + +#define nvkm_error_ratelimited(s,f,a...) nvkm_printk((s), ERROR, err_ratelimited, f, ##a)#endif drivers/gpu/drm/nouveau/nvkm/subdev/bus/gf100.c +7
-7@@ -35,13 +35,13 @@ gf100_bus_intr(struct nvkm_bus *bus)u32 addr =nvkm_rd32 (device,0x009084 );u32 data =nvkm_rd32 (device,0x009088 ); -nvkm_error (subdev, -"MMIO %s of %08x FAULT at %06x [ %s%s%s]\n" , - (addr &0x00000002 ) ?"write" :"read" , data, - (addr &0x00fffffc ), - (stat &0x00000002 ) ?"!ENGINE " :"" , - (stat &0x00000004 ) ?"PRIVRING " :"" , - (stat &0x00000008 ) ?"TIMEOUT " :"" ); +nvkm_error_ratelimited (subdev, +"MMIO %s of %08x FAULT at %06x [ %s%s%s]\n" , + (addr &0x00000002 ) ?"write" :"read" , data, + (addr &0x00fffffc ), + (stat &0x00000002 ) ?"!ENGINE " :"" , + (stat &0x00000004 ) ?"PRIVRING " :"" , + (stat &0x00000008 ) ?"TIMEOUT " :"" );nvkm_wr32 (device,0x009084 ,0x00000000 );nvkm_wr32 (device,0x001100 , (stat &0x0000000e ));drivers/gpu/drm/nouveau/nvkm/subdev/bus/nv31.c +3
-3@@ -45,9 +45,9 @@ nv31_bus_intr(struct nvkm_bus *bus)u32 addr =nvkm_rd32 (device,0x009084 );u32 data =nvkm_rd32 (device,0x009088 ); -nvkm_error (subdev,"MMIO %s of %08x FAULT at %06x\n" , - (addr &0x00000002 ) ?"write" :"read" , data, - (addr &0x00fffffc )); +nvkm_error_ratelimited (subdev,"MMIO %s of %08x FAULT at %06x\n" , + (addr &0x00000002 ) ?"write" :"read" , data, + (addr &0x00fffffc )); stat &= ~0x00000008 ;nvkm_wr32 (device,0x001100 ,0x00000008 );drivers/gpu/drm/nouveau/nvkm/subdev/bus/nv50.c +3
-4@@ -60,9 +60,9 @@ nv50_bus_intr(struct nvkm_bus *bus)u32 addr =nvkm_rd32 (device,0x009084 );u32 data =nvkm_rd32 (device,0x009088 ); -nvkm_error (subdev,"MMIO %s of %08x FAULT at %06x\n" , - (addr &0x00000002 ) ?"write" :"read" , data, - (addr &0x00fffffc )); +nvkm_error_ratelimited (subdev,"MMIO %s of %08x FAULT at %06x\n" , + (addr &0x00000002 ) ?"write" :"read" , data, + (addr &0x00fffffc )); stat &= ~0x00000008 ;nvkm_wr32 (device,0x001100 ,0x00000008 ); --2.35.1
Lyude Paul
maintainerQuoted 9 lines · Karol Herbst
not able to hit any error on my machine, but regardless:
Reviewed-by Karol Herbst <[email protected]>I suspect there are more places where we could put it, but we can add those later.
Anyway, I think it's valuable to push it through fixes, not sure how far back we want to CC stable though.
JFYI - dim fixes can figure this out if you just give it the commit that introduced the issue.
Anyway, will push this upstream in a moment
Quoted 112 lines · Lyude Paul
Quoted 109 lines
There's plenty of ways to fudge the GPU when developing on nouveau by mistake, some of which can result in nouveau seriously spamming dmesg with fault errors. This can be somewhat annoying, as it can quickly overrun the message buffer (or your terminal emulator's buffer) and get rid of actually
useful feedback from the driver. While working on my new atomic only MST branch, I ran into this issue a couple of times.So, let's fix this by adding
nvkm_error_ratelimited(), and using it to ratelimit errors from faults. This should be fine for developers, since it's nearly always only the first few faults that we care about seeing. Plus, you can turn off rate limiting in the kernel if you really need to.Signed-off-by Lyude Paul <[email protected]>---
drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h | 2 ++ drivers/gpu/drm/nouveau/nvkm/subdev/bus/gf100.c | 14 +++++++------- drivers/gpu/drm/nouveau/nvkm/subdev/bus/nv31.c | 6 +++--- drivers/gpu/drm/nouveau/nvkm/subdev/bus/nv50.c | 6 +++---4 files changed, 15 insertions(+), 13 deletions(-)drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h +2
-0@@ -62,4 +62,6 @@ void nvkm_subdev_intr(struct nvkm_subdev *); #define nvkm_debug(s,f,a...) nvkm_printk((s), DEBUG, info, f, ##a) #define nvkm_trace(s,f,a...) nvkm_printk((s), TRACE, info, f, ##a) #define nvkm_spam(s,f,a...) nvkm_printk((s), SPAM, dbg, f, ##a) + +#define nvkm_error_ratelimited(s,f,a...) nvkm_printk((s), ERROR, err_ratelimited, f, ##a)#endif
drivers/gpu/drm/nouveau/nvkm/subdev/bus/gf100.c +6
-7@@ -35,13 +35,13 @@ gf100_bus_intr(struct nvkm_bus *bus) u32 addr = nvkm_rd32(device, 0x009084); u32 data = nvkm_rd32(device, 0x009088); - nvkm_error(subdev, - "MMIO %s of %08x FAULT at %06x [ %s%s%s]\n", - (addr & 0x00000002) ? "write" : "read", data, - (addr & 0x00fffffc), - (stat & 0x00000002) ? "!ENGINE " : "", - (stat & 0x00000004) ? "PRIVRING " : "", - (stat & 0x00000008) ? "TIMEOUT " : ""); + nvkm_error_ratelimited(subdev, + "MMIO %s of %08x FAULT at %06x [ %s%s%s]\n", + (addr & 0x00000002) ? "write" : "read", data, + (addr & 0x00fffffc), + (stat & 0x00000002) ? "!ENGINE " : "", + (stat & 0x00000004) ? "PRIVRING " :"",
+ (stat &0x00000008) ? "TIMEOUT " :
"");
nvkm_wr32(device, 0x009084, 0x00000000);nvkm_wr32(device, 0x001100, (stat & 0x0000000e));drivers/gpu/drm/nouveau/nvkm/subdev/bus/nv31.c +3
-3@@ -45,9 +45,9 @@ nv31_bus_intr(struct nvkm_bus *bus) u32 addr = nvkm_rd32(device, 0x009084); u32 data = nvkm_rd32(device, 0x009088); - nvkm_error(subdev, "MMIO %s of %08x FAULT at %06x\n", - (addr & 0x00000002) ? "write" : "read", data, - (addr & 0x00fffffc)); + nvkm_error_ratelimited(subdev, "MMIO %s of %08x FAULT at %06x\n", + (addr & 0x00000002) ? "write" :"read" , data, + (addr & 0x00fffffc));stat &= ~0x00000008;
nvkm_wr32(device, 0x001100, 0x00000008);drivers/gpu/drm/nouveau/nvkm/subdev/bus/nv50.c +3
-3@@ -60,9 +60,9 @@ nv50_bus_intr(struct nvkm_bus *bus) u32 addr = nvkm_rd32(device, 0x009084); u32 data = nvkm_rd32(device, 0x009088); - nvkm_error(subdev, "MMIO %s of %08x FAULT at %06x\n", - (addr & 0x00000002) ? "write" : "read", data, - (addr & 0x00fffffc)); + nvkm_error_ratelimited(subdev, "MMIO %s of %08x FAULT at %06x\n", + (addr & 0x00000002) ? "write" :"read" , data, + (addr & 0x00fffffc));stat &= ~0x00000008;
nvkm_wr32(device, 0x001100, 0x00000008); --
2.35.1
Signature · 3 lines
Cheers, Lyude Paul (she/her) Software Engineer at Red Hat
Lyude Paul
maintainerMhhh - looks like it goes far back enough I probably should just Cc it without a version tag and see how far it applies for greg. Will do that
Quoted 135 lines · Lyude Paul
Quoted 9 lines · Karol Herbst
not able to hit any error on my machine, but regardless:
Reviewed-by Karol Herbst <[email protected]>I suspect there are more places where we could put it, but we can add those later.
Anyway, I think it's valuable to push it through fixes, not sure how far back we want to CC stable though.
JFYI - dim fixes can figure this out if you just give it the commit that introduced the issue.
Anyway, will push this upstream in a moment
Quoted 118 lines · Lyude Paul
Quoted 115 lines
There's plenty of ways to fudge the GPU when developing on nouveau by mistake, some of which can result in nouveau seriously spamming dmesg with
fault errors. This can be somewhat annoying, as it can quickly overrun the
message buffer (or your terminal emulator's buffer) and get rid of actually
useful feedback from the driver. While working on my new atomic only MST branch, I ran into this issue a couple of times.So, let's fix this by adding
nvkm_error_ratelimited(), and using it to ratelimit errors from faults. This should be fine for developers, since it's nearly always only the first few faults that we care about seeing. Plus, you can turn off rate limiting in the kernel if you really need to.Signed-off-by Lyude Paul <[email protected]>---
drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h | 2 ++ drivers/gpu/drm/nouveau/nvkm/subdev/bus/gf100.c | 14 +++++++------- drivers/gpu/drm/nouveau/nvkm/subdev/bus/nv31.c | 6 +++--- drivers/gpu/drm/nouveau/nvkm/subdev/bus/nv50.c | 6 +++---4 files changed, 15 insertions(+), 13 deletions(-)drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h +2
-0@@ -62,4 +62,6 @@ void nvkm_subdev_intr(struct nvkm_subdev *); #define nvkm_debug(s,f,a...) nvkm_printk((s), DEBUG, info, f, ##a) #define nvkm_trace(s,f,a...) nvkm_printk((s), TRACE, info, f, ##a) #define nvkm_spam(s,f,a...) nvkm_printk((s), SPAM, dbg, f, ##a) + +#define nvkm_error_ratelimited(s,f,a...) nvkm_printk((s), ERROR, err_ratelimited, f, ##a)#endif
drivers/gpu/drm/nouveau/nvkm/subdev/bus/gf100.c +5
-7@@ -35,13 +35,13 @@ gf100_bus_intr(struct nvkm_bus *bus) u32 addr = nvkm_rd32(device, 0x009084); u32 data = nvkm_rd32(device, 0x009088); - nvkm_error(subdev, - "MMIO %s of %08x FAULT at %06x [ %s%s%s]\n", - (addr & 0x00000002) ? "write" : "read", data, - (addr & 0x00fffffc), - (stat & 0x00000002) ? "!ENGINE " : "", - (stat & 0x00000004) ? "PRIVRING " : "", - (stat & 0x00000008) ? "TIMEOUT " : ""); + nvkm_error_ratelimited(subdev, + "MMIO %s of %08x FAULT at %06x [ %s%s%s]\n", + (addr & 0x00000002) ? "write" : "read", data, + (addr & 0x00fffffc), + (stat & 0x00000002) ? "!ENGINE ":
"",
+ (stat &0x00000004) ? "PRIVRING " :
"",
+ (stat &0x00000008) ? "TIMEOUT " :
"");
nvkm_wr32(device, 0x009084, 0x00000000);nvkm_wr32(device, 0x001100, (stat & 0x0000000e));drivers/gpu/drm/nouveau/nvkm/subdev/bus/nv31.c +3
-3@@ -45,9 +45,9 @@ nv31_bus_intr(struct nvkm_bus *bus) u32 addr = nvkm_rd32(device, 0x009084); u32 data = nvkm_rd32(device, 0x009088); - nvkm_error(subdev, "MMIO %s of %08x FAULT at %06x\n", - (addr & 0x00000002) ? "write" : "read", data, - (addr & 0x00fffffc)); + nvkm_error_ratelimited(subdev, "MMIO %s of %08x FAULT at %06x\n", + (addr & 0x00000002) ? "write" :"read" , data, + (addr & 0x00fffffc));stat &= ~0x00000008;
nvkm_wr32(device, 0x001100, 0x00000008);drivers/gpu/drm/nouveau/nvkm/subdev/bus/nv50.c +3
-3@@ -60,9 +60,9 @@ nv50_bus_intr(struct nvkm_bus *bus) u32 addr = nvkm_rd32(device, 0x009084); u32 data = nvkm_rd32(device, 0x009088); - nvkm_error(subdev, "MMIO %s of %08x FAULT at %06x\n", - (addr & 0x00000002) ? "write" : "read", data, - (addr & 0x00fffffc)); + nvkm_error_ratelimited(subdev, "MMIO %s of %08x FAULT at %06x\n", + (addr & 0x00000002) ? "write" :"read" , data, + (addr & 0x00fffffc));stat &= ~0x00000008;
nvkm_wr32(device, 0x001100, 0x00000008); --
2.35.1
Signature · 3 lines
Cheers, Lyude Paul (she/her) Software Engineer at Red Hat