Skip to contentSkip to conversation

nouveau: fix disabling the nonstall irq due to storm code.

PATCH In review Dave Airlie 2 participants 4 messages Aug 28, 2025 – Aug 29, 2025

Subsystems (3)
lore
Share & tools
lore

Dave Airlie

author maintainer
#0 lore raw

PATCH 1/2 nouveau: fix disabling the nonstall irq due to storm code.

From: Dave Airlie <[email protected]>

Nouveau has code that when it gets an IRQ with no allowed handler it disables it to avoid storms.

However with nonstall interrupts, we often disable them from the drm driver, but still request their emission via the push submission.

Just don't disable nonstall irqs ever in normal operation, the event handling code will filter them out, and the driver will just enable/disable them at load time.

This fixes timeouts we've been seeing on/off for a long time, but they became a lot more noticable on Blackwell.

This doesn't fix all of them, there is a subsequent fence emission fix to fix the last few.

Fixes 3ebd64aa3c4f ("drm/nouveau/intr: support multiple trees, and explicit interfaces")
---
 .../gpu/drm/nouveau/nvkm/engine/fifo/base.c   |  2 ++
 .../gpu/drm/nouveau/nvkm/engine/fifo/ga100.c  | 22 ++++++++++++-------
 .../gpu/drm/nouveau/nvkm/engine/fifo/priv.h   |  2 ++
 .../nouveau/nvkm/subdev/gsp/rm/r535/fifo.c    |  2 +-
 4 files changed, 19 insertions(+), 9 deletions(-)
drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c +2 -0
index fdffa0391b31..6fd4e60634fb 100644
@@ -350,6 +350,8 @@ nvkm_fifo_dtor(struct nvkm_engine *engine)
 	nvkm_chid_unref(&fifo->chid);
 
 	nvkm_event_fini(&fifo->nonstall.event);
+	if (fifo->func->nonstall_dtor)
+		fifo->func->nonstall_dtor(fifo);
 	mutex_destroy(&fifo->mutex);
 
 	if (fifo->func->dtor)
drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga100.c +14 -8
index e74493a4569e..81beae473122 100644
@@ -517,19 +517,11 @@ ga100_fifo_nonstall_intr(struct nvkm_inth *inth)
 static void
 ga100_fifo_nonstall_block(struct nvkm_event *event, int type, int index)
 {
-	struct nvkm_fifo *fifo = container_of(event, typeof(*fifo), nonstall.event);
-	struct nvkm_runl *runl = nvkm_runl_get(fifo, index, 0);
-
-	nvkm_inth_block(&runl->nonstall.inth);
 }
 
 static void
 ga100_fifo_nonstall_allow(struct nvkm_event *event, int type, int index)
 {
-	struct nvkm_fifo *fifo = container_of(event, typeof(*fifo), nonstall.event);
-	struct nvkm_runl *runl = nvkm_runl_get(fifo, index, 0);
-
-	nvkm_inth_allow(&runl->nonstall.inth);
 }
 
 const struct nvkm_event_func
@@ -564,12 +556,25 @@ ga100_fifo_nonstall_ctor(struct nvkm_fifo *fifo)
 		if (ret)
 			return ret;
 
+		nvkm_inth_allow(&runl->nonstall.inth);
+
 		nr = max(nr, runl->id + 1);
 	}
 
 	return nr;
 }
 
+void
+ga100_fifo_nonstall_dtor(struct nvkm_fifo *fifo)
+{
+	struct nvkm_runl *runl;
+	nvkm_runl_foreach(runl, fifo) {
+		if (runl->nonstall.vector < 0)
+			continue;
+		nvkm_inth_block(&runl->nonstall.inth);
+	}
+}
+
 int
 ga100_fifo_runl_ctor(struct nvkm_fifo *fifo)
 {
@@ -599,6 +604,7 @@ ga100_fifo = {
 	.runl_ctor = ga100_fifo_runl_ctor,
 	.mmu_fault = &tu102_fifo_mmu_fault,
 	.nonstall_ctor = ga100_fifo_nonstall_ctor,
+	.nonstall_dtor = ga100_fifo_nonstall_dtor,
 	.nonstall = &ga100_fifo_nonstall,
 	.runl = &ga100_runl,
 	.runq = &ga100_runq,
drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h +2 -0
index 5e81ae195329..fff1428ef267 100644
@@ -41,6 +41,7 @@ struct nvkm_fifo_func {
 	void (*start)(struct nvkm_fifo *, unsigned long *);
 
 	int (*nonstall_ctor)(struct nvkm_fifo *);
+	void (*nonstall_dtor)(struct nvkm_fifo *);
 	const struct nvkm_event_func *nonstall;
 
 	const struct nvkm_runl_func *runl;
@@ -200,6 +201,7 @@ u32 tu102_chan_doorbell_handle(struct nvkm_chan *);
 
 int ga100_fifo_runl_ctor(struct nvkm_fifo *);
 int ga100_fifo_nonstall_ctor(struct nvkm_fifo *);
+void ga100_fifo_nonstall_dtor(struct nvkm_fifo *);
 extern const struct nvkm_event_func ga100_fifo_nonstall;
 extern const struct nvkm_runl_func ga100_runl;
 extern const struct nvkm_runq_func ga100_runq;
drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/fifo.c +1 -1
index 1ac5628c5140..b8be0a872e7a 100644
@@ -601,7 +601,7 @@ r535_fifo_new(const struct nvkm_fifo_func *hw, struct nvkm_device *device,
 	rm->chan.func = &r535_chan;
 	rm->nonstall = &ga100_fifo_nonstall;
 	rm->nonstall_ctor = ga100_fifo_nonstall_ctor;
-
+	rm->nonstall_dtor = ga100_fifo_nonstall_dtor;
 	return nvkm_fifo_new_(rm, device, type, inst, pfifo);
 }
 
Signature · 1 line
2.50.1
author maintainer
#1 lore raw

PATCH 2/2 nouveau: Membar before between semaphore writes and the interrupt

From: Faith Ekstrand <[email protected]>

This ensures that the memory write and the interrupt are properly ordered and we won't wake up the kernel before the semaphore write has hit memory.

Fixes b1ca384772b6 ("drm/nouveau/gv100-: switch to volta semaphore methods")
---
 drivers/gpu/drm/nouveau/gv100_fence.c         |  7 +-
 .../drm/nouveau/include/nvhw/class/clc36f.h   | 85 +++++++++++++++++++
 2 files changed, 91 insertions(+), 1 deletion(-)
drivers/gpu/drm/nouveau/gv100_fence.c +6 -1
index cccdeca72002..317e516c4ec7 100644
@@ -18,7 +18,7 @@ gv100_fence_emit32(struct nouveau_channel *chan, u64 virtual, u32 sequence)
 	struct nvif_push *push = &chan->chan.push;
 	int ret;
 
-	ret = PUSH_WAIT(push, 8);
+	ret = PUSH_WAIT(push, 13);
 	if (ret)
 		return ret;
 
@@ -32,6 +32,11 @@ gv100_fence_emit32(struct nouveau_channel *chan, u64 virtual, u32 sequence)
 		  NVDEF(NVC36F, SEM_EXECUTE, PAYLOAD_SIZE, 32BIT) |
 		  NVDEF(NVC36F, SEM_EXECUTE, RELEASE_TIMESTAMP, DIS));
 
+	PUSH_MTHD(push, NVC36F, MEM_OP_A, 0,
+				MEM_OP_B, 0,
+				MEM_OP_C, NVDEF(NVC36F, MEM_OP_C, MEMBAR_TYPE, SYS_MEMBAR),
+				MEM_OP_D, NVDEF(NVC36F, MEM_OP_D, OPERATION, MEMBAR));
+
 	PUSH_MTHD(push, NVC36F, NON_STALL_INTERRUPT, 0);
 
 	PUSH_KICK(push);
drivers/gpu/drm/nouveau/include/nvhw/class/clc36f.h +85 -0

91 lines not on this page · raw message

Signature · 1 line
2.50.1
#2 lore raw
In reply to Dave Airlie
Quoted 93 lines · Dave Airlie

From: Dave Airlie <[email protected]>

Nouveau has code that when it gets an IRQ with no allowed handler it disables it to avoid storms.

However with nonstall interrupts, we often disable them from the drm driver, but still request their emission via the push submission.

Just don't disable nonstall irqs ever in normal operation, the event handling code will filter them out, and the driver will just enable/disable them at load time.

This fixes timeouts we've been seeing on/off for a long time, but they became a lot more noticable on Blackwell.

This doesn't fix all of them, there is a subsequent fence emission fix to fix the last few.

Fixes 3ebd64aa3c4f ("drm/nouveau/intr: support multiple trees, and explicit interfaces")
---
 .../gpu/drm/nouveau/nvkm/engine/fifo/base.c   |  2 ++
 .../gpu/drm/nouveau/nvkm/engine/fifo/ga100.c  | 22 ++++++++++++-------
 .../gpu/drm/nouveau/nvkm/engine/fifo/priv.h   |  2 ++
 .../nouveau/nvkm/subdev/gsp/rm/r535/fifo.c    |  2 +-
 4 files changed, 19 insertions(+), 9 deletions(-)
drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c +2 -0
index fdffa0391b31..6fd4e60634fb 100644
@@ -350,6 +350,8 @@ nvkm_fifo_dtor(struct nvkm_engine *engine)
        nvkm_chid_unref(&fifo->chid);
 
        nvkm_event_fini(&fifo->nonstall.event);
+       if (fifo->func->nonstall_dtor)
+               fifo->func->nonstall_dtor(fifo);
        mutex_destroy(&fifo->mutex);
 
        if (fifo->func->dtor)
drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga100.c +13 -8
index e74493a4569e..81beae473122 100644
@@ -517,19 +517,11 @@ ga100_fifo_nonstall_intr(struct nvkm_inth *inth)
 static void
 ga100_fifo_nonstall_block(struct nvkm_event *event, int type, int index)
 {
-       struct nvkm_fifo *fifo = container_of(event, typeof(*fifo), nonstall.event);
-       struct nvkm_runl *runl = nvkm_runl_get(fifo, index, 0);
-
-       nvkm_inth_block(&runl->nonstall.inth);
 }
 
 static void
 ga100_fifo_nonstall_allow(struct nvkm_event *event, int type, int index)
 {
-       struct nvkm_fifo *fifo = container_of(event, typeof(*fifo), nonstall.event);
-       struct nvkm_runl *runl = nvkm_runl_get(fifo, index, 0);
-
-       nvkm_inth_allow(&runl->nonstall.inth);
 }
 
 const struct nvkm_event_func
@@ -564,12 +556,25 @@ ga100_fifo_nonstall_ctor(struct nvkm_fifo *fifo)
                if (ret)
                        return ret;
 
+               nvkm_inth_allow(&runl->nonstall.inth);
+
                nr = max(nr, runl->id + 1);
        }
 
        return nr;
 }
 
+void
+ga100_fifo_nonstall_dtor(struct nvkm_fifo *fifo)
+{
+       struct nvkm_runl *runl;
+       nvkm_runl_foreach(runl, fifo) {
+               if (runl->nonstall.vector < 0)
+                       continue;
+               nvkm_inth_block(&runl->nonstall.inth);
+       }
+}
+
 int
 ga100_fifo_runl_ctor(struct nvkm_fifo *fifo)
 {

drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga100.c ga100_fifo = { · line 607

Quoted 4 lines
drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga100.c +1 -0
@@ -599,6 +604,7 @@ ga100_fifo = {
        .runl_ctor = ga100_fifo_runl_ctor,
        .mmu_fault = &tu102_fifo_mmu_fault,
        .nonstall_ctor = ga100_fifo_nonstall_ctor,
+       .nonstall_dtor = ga100_fifo_nonstall_dtor,

You're missing the corresponding update in ga102.c, which is what actually covers most of the drivers. Honestly, I'm not even sure why there are two files. They look identical to me.

~Faith

Quoted 39 lines
drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga100.c +0 -0
        .nonstall = &ga100_fifo_nonstall,
        .runl = &ga100_runl,
        .runq = &ga100_runq,
drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h +2 -0
index 5e81ae195329..fff1428ef267 100644
@@ -41,6 +41,7 @@ struct nvkm_fifo_func {
        void (*start)(struct nvkm_fifo *, unsigned long *);
 
        int (*nonstall_ctor)(struct nvkm_fifo *);
+       void (*nonstall_dtor)(struct nvkm_fifo *);
        const struct nvkm_event_func *nonstall;
 
        const struct nvkm_runl_func *runl;
@@ -200,6 +201,7 @@ u32 tu102_chan_doorbell_handle(struct nvkm_chan *);
 
 int ga100_fifo_runl_ctor(struct nvkm_fifo *);
 int ga100_fifo_nonstall_ctor(struct nvkm_fifo *);
+void ga100_fifo_nonstall_dtor(struct nvkm_fifo *);
 extern const struct nvkm_event_func ga100_fifo_nonstall;
 extern const struct nvkm_runl_func ga100_runl;
 extern const struct nvkm_runq_func ga100_runq;
drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/fifo.c +1 -2
index 1ac5628c5140..b8be0a872e7a 100644
@@ -601,7 +601,7 @@ r535_fifo_new(const struct nvkm_fifo_func *hw, struct nvkm_device *device,
        rm->chan.func = &r535_chan;
        rm->nonstall = &ga100_fifo_nonstall;
        rm->nonstall_ctor = ga100_fifo_nonstall_ctor;
-
+       rm->nonstall_dtor = ga100_fifo_nonstall_dtor;
        return nvkm_fifo_new_(rm, device, type, inst, pfifo);
 }
 
--

2.50.1

author maintainer
#3 lore raw
Quoted 6 lines
Quoted 6 lines
drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga100.c +2 -0
+}
+
 int
 ga100_fifo_runl_ctor(struct nvkm_fifo *fifo)
 {
@@ -599,6 +604,7 @@ ga100_fifo = {
Quoted 8 lines

drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga100.c ga100_fifo = { · line 607

Quoted 4 lines
drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga100.c +1 -0
        .runl_ctor = ga100_fifo_runl_ctor,
        .mmu_fault = &tu102_fifo_mmu_fault,
        .nonstall_ctor = ga100_fifo_nonstall_ctor,
+       .nonstall_dtor = ga100_fifo_nonstall_dtor,

You're missing the corresponding update in ga102.c, which is what actually covers most of the drivers. Honestly, I'm not even sure why there are two files. They look identical to me.

Indeed, the r535 change is all that really matters since it's the GSP path, and we shouldn't use non-gsp on these paths.

I've fixed it up anyways in a v2.

Thanks,
Dave.