Skip to contentSkip to conversation

drm/amdgpu/mes12: drop queue state on RESET_QUEUES unmap

PATCH In review Jesse Zhang 3 participants 6 messages Jun 20 – Jun 23

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

PATCH 1/3 drm/amdgpu/mes12: drop queue state on RESET_QUEUES unmap

Set remove_queue_after_reset=1 (MES >= 0x5a) so MES drops its internal state instead of re-unmapping an already MMIO-reset queue, which can timeout into a GPU reset.

---
 drivers/gpu/drm/amd/amdgpu/mes_v12_0.c | 11 +++++++++++
 1 file changed, 11 insertions(+)
drivers/gpu/drm/amd/amdgpu/mes_v12_0.c +11 -0
index d80a983b1b6c..20f4fd57b1da 100644
@@ -749,6 +749,17 @@ static int mes_v12_0_unmap_legacy_queue(struct amdgpu_mes *mes,
 		mes_remove_queue_pkt.unmap_legacy_queue = 1;
 		mes_remove_queue_pkt.queue_type =
 			convert_to_mes_queue_type(input->queue_type);
+		/*
+		 * A reset-time unmap: the queue was already reset via MMIO while
+		 * gangs are suspended and it is on the MES hung/fail list. Tell
+		 * MES to just drop its internal state for it. Without this flag
+		 * MES asks CP to unmap the already-reset (still wedged) queue
+		 * again, which times out and forces a GPU reset.
+		 */
+		if (input->action == RESET_QUEUES &&
+		    (mes->sched_version & AMDGPU_MES_VERSION_MASK) >= 0x5a)
+			mes_remove_queue_pkt.remove_queue_after_reset = 1;
+
 	}
 
 	if (mes->adev->enable_uni_mes) {
Signature · 1 line
2.49.0
#1 lore raw

PATCH 2/3 drm/amdkfd: flush MES queue on reset-time queue removal

Pass flush_mes_queue=true in reset_queue_mes() to match the GFX post-reset drop semantics.

---
 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +1 -1
index 166dfa808cfa..9f28974f25b4 100644
@@ -427,7 +427,7 @@ static int reset_queue_mes(struct device_queue_manager *dqm, struct queue *q,
 	if (r)
 		return r;
 	/* Proceed remove_queue with reset=true */
-	remove_queue_mes_on_reset_option(dqm, q, &pdd->qpd, true, false);
+	remove_queue_mes_on_reset_option(dqm, q, &pdd->qpd, true, true);
 	set_queue_as_reset(dqm, q, &pdd->qpd);
 	return 0;
 }
Signature · 1 line
2.49.0
#2 lore raw

PATCH 3/3 drm/amdgpu: defer KCQ remap until after MES resume in reset flow

In reply to Jesse Zhang

Split amdgpu_gfx_mes_reset_queue_start() into reset+unmap now and queue reinit later, and do the remap only after amdgpu_mes_resume(). Avoids re-adding legacy queues while MES gangs are still suspended.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 70 +++++++++++++++++++------
 drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h |  1 +
 2 files changed, 55 insertions(+), 16 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c +54 -16
index 982b41606d48..a5b835d0c166 100644
@@ -1989,10 +1989,24 @@ static ssize_t amdgpu_gfx_get_compute_reset_mask(struct device *dev,
 	return amdgpu_show_reset_mask(buf, adev->gfx.compute_supported_reset);
 }
 
+static int amdgpu_gfx_mes_reset_queue_reinit(struct amdgpu_ring *ring)
+{
+	struct amdgpu_device *adev = ring->adev;
+	int r;
+
+	amdgpu_gfx_mqd_reset_restore(ring);
+
+	r = amdgpu_mes_map_legacy_queue(adev, ring, 0);
+	if (r)
+		dev_err(adev->dev, "failed to remap kgq\n");
+
+	return r;
+}
+
 static int amdgpu_gfx_mes_reset_queue_start(struct amdgpu_ring *ring,
 					     unsigned int vmid,
 					     struct amdgpu_fence *timedout_fence,
-					     bool use_mmio)
+					     bool use_mmio, bool *need_reinit)
 {
 	struct amdgpu_device *adev = ring->adev;
 	bool reinit_queue;
@@ -2007,6 +2021,9 @@ static int amdgpu_gfx_mes_reset_queue_start(struct amdgpu_ring *ring,
 	else
 		reinit_queue = use_mmio;
 
+	if (need_reinit)
+		*need_reinit = false;
+
 	amdgpu_ring_reset_helper_begin(ring, timedout_fence);
 
 	r = amdgpu_mes_reset_legacy_queue(ring->adev, ring, vmid, use_mmio, 0);
@@ -2018,13 +2035,9 @@ static int amdgpu_gfx_mes_reset_queue_start(struct amdgpu_ring *ring,
 						  RESET_QUEUES, 0, 0, 0);
 		if (r)
 			return r;
-		amdgpu_gfx_mqd_reset_restore(ring);
 
-		r = amdgpu_mes_map_legacy_queue(adev, ring, 0);
-		if (r) {
-			dev_err(adev->dev, "failed to remap kgq\n");
-			return r;
-		}
+		if (need_reinit)
+			*need_reinit = true;
 	}
 	return 0;
 }
@@ -2034,12 +2047,19 @@ int amdgpu_gfx_mes_reset_queue(struct amdgpu_ring *ring,
 			       struct amdgpu_fence *timedout_fence,
 			       bool use_mmio)
 {
+	bool need_reinit;
 	int r;
 
+	/* Single-queue reset (no suspend/resume): re-add the queue inline. */
 	r = amdgpu_gfx_mes_reset_queue_start(ring, vmid, timedout_fence,
-					      use_mmio);
+					      use_mmio, &need_reinit);
 	if (r)
 		return r;
+	if (need_reinit) {
+		r = amdgpu_gfx_mes_reset_queue_reinit(ring);
+		if (r)
+			return r;
+	}
 	return amdgpu_ring_reset_helper_end(ring, timedout_fence);
 }
 
@@ -2239,7 +2259,8 @@ static int amdgpu_gfx_reset_mes_kcq(struct amdgpu_device *adev,
 				    struct amdgpu_ring *guilty_ring,
 				    unsigned int db,
 				    struct amdgpu_ring **out_ring,
-				    struct amdgpu_fence **out_fence)
+				    struct amdgpu_fence **out_fence,
+				    bool *out_reinit)
 {
 	bool use_mmio = adev->gfx.mec.use_mmio_for_reset;
 	struct amdgpu_fence *fence;
@@ -2248,14 +2269,16 @@ static int amdgpu_gfx_reset_mes_kcq(struct amdgpu_device *adev,
 
 	*out_ring = NULL;
 	*out_fence = NULL;
+	*out_reinit = false;
 	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
 		ring = &adev->gfx.compute_ring[i];
 		if (ring == guilty_ring)
 			continue;
 		if (ring->doorbell_index == db) {
 			fence = amdgpu_ring_find_guilty_fence(ring);
+			/* reset + unmap now; re-add (map) is deferred to after resume */
 			r = amdgpu_gfx_mes_reset_queue_start(ring, 0, fence,
-							      use_mmio);
+							      use_mmio, out_reinit);
 			if (r)
 				return r;
 			*out_ring = ring;
@@ -2306,12 +2329,16 @@ int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev,
 fence_reset:
 	/* reset the queue this came from if specified */
 	if (ring) {
+		bool reinit = false;
+
+		/* reset + unmap now; re-add (map) is deferred to after resume */
 		r = amdgpu_gfx_mes_reset_queue_start(ring, 0, guilty_fence,
-						      use_mmio);
+						      use_mmio, &reinit);
 		if (r)
 			goto out;
 		deferred_end[n_deferred].ring = ring;
 		deferred_end[n_deferred].fence = guilty_fence;
+		deferred_end[n_deferred].reinit = reinit;
 		n_deferred++;
 	}
 	if (uq) {
@@ -2322,6 +2349,7 @@ int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev,
 	for (i = 0; i < num_hung; i++) {
 		struct amdgpu_ring *hr = NULL;
 		struct amdgpu_fence *hf = NULL;
+		bool hr_reinit = false;
 
 		pipe = hqd_info[i].pipe_index;
 		queue = hqd_info[i].queue_index;
@@ -2330,12 +2358,13 @@ int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev,
 		/* reset any KCQs */
 		r = amdgpu_gfx_reset_mes_kcq(adev, ring,
 					     adev->gfx.mec.mes_hung_db_array[i],
-					     &hr, &hf);
+					     &hr, &hf, &hr_reinit);
 		if (r)
 			goto out;
 		if (hr) {
 			deferred_end[n_deferred].ring = hr;
 			deferred_end[n_deferred].fence = hf;
+			deferred_end[n_deferred].reinit = hr_reinit;
 			n_deferred++;
 		}
 		/* reset any KFD queues */
@@ -2372,12 +2401,21 @@ int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev,
 	/* resume all will enable the non-hung queues */
 	amdgpu_mes_resume(adev, 0);
 
-	/* Now CP is running again — replay backed-up commands and ring
-	 * doorbells on each reset queue.
+	/* Now CP is running again — for queues that were unmapped during the
+	 * reset, re-add (map) them only now that MES is resumed and back to a
+	 * normal state, then replay backed-up commands and ring doorbells on
+	 * each reset queue.
 	 */
 	for (i = 0; i < n_deferred; i++) {
-		int er = amdgpu_ring_reset_helper_end(deferred_end[i].ring,
-						      deferred_end[i].fence);
+		int er;
+
+		if (deferred_end[i].reinit) {
+			er = amdgpu_gfx_mes_reset_queue_reinit(deferred_end[i].ring);
+			if (er && !r)
+				r = er;
+		}
+		er = amdgpu_ring_reset_helper_end(deferred_end[i].ring,
+						  deferred_end[i].fence);
 		if (er && !r)
 			r = er;
 	}
drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h +1 -0
index aefd4f03b443..9432107c96a1 100644
@@ -550,6 +550,7 @@ struct amdgpu_gfx {
 struct amdgpu_gfx_deferred_entry {
 	struct amdgpu_ring	*ring;
 	struct amdgpu_fence	*fence;
+	bool			reinit;
 };
 
 struct amdgpu_gfx_ras_reg_entry {
Signature · 1 line
2.49.0
Acked-by on v1 3/3
#3 lore raw

Re: [PATCH 3/3] drm/amdgpu: defer KCQ remap until after MES resume in reset flow

Quoted 7 lines · Jesse Zhang

Split amdgpu_gfx_mes_reset_queue_start() into reset+unmap now and queue reinit later, and do the remap only after amdgpu_mes_resume(). Avoids re-adding legacy queues while MES gangs are still suspended.

The series is:

Is there a way we can clean this up in a future patch set? It's getting really complicated.

Alex

Quoted 191 lines
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 70 +++++++++++++++++++------
 drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h |  1 +
 2 files changed, 55 insertions(+), 16 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c +54 -16
index 982b41606d48..a5b835d0c166 100644
@@ -1989,10 +1989,24 @@ static ssize_t amdgpu_gfx_get_compute_reset_mask(struct device *dev,
        return amdgpu_show_reset_mask(buf, adev->gfx.compute_supported_reset);
 }
 
+static int amdgpu_gfx_mes_reset_queue_reinit(struct amdgpu_ring *ring)
+{
+       struct amdgpu_device *adev = ring->adev;
+       int r;
+
+       amdgpu_gfx_mqd_reset_restore(ring);
+
+       r = amdgpu_mes_map_legacy_queue(adev, ring, 0);
+       if (r)
+               dev_err(adev->dev, "failed to remap kgq\n");
+
+       return r;
+}
+
 static int amdgpu_gfx_mes_reset_queue_start(struct amdgpu_ring *ring,
                                             unsigned int vmid,
                                             struct amdgpu_fence *timedout_fence,
-                                            bool use_mmio)
+                                            bool use_mmio, bool *need_reinit)
 {
        struct amdgpu_device *adev = ring->adev;
        bool reinit_queue;
@@ -2007,6 +2021,9 @@ static int amdgpu_gfx_mes_reset_queue_start(struct amdgpu_ring *ring,
        else
                reinit_queue = use_mmio;
 
+       if (need_reinit)
+               *need_reinit = false;
+
        amdgpu_ring_reset_helper_begin(ring, timedout_fence);
 
        r = amdgpu_mes_reset_legacy_queue(ring->adev, ring, vmid, use_mmio, 0);
@@ -2018,13 +2035,9 @@ static int amdgpu_gfx_mes_reset_queue_start(struct amdgpu_ring *ring,
                                                  RESET_QUEUES, 0, 0, 0);
                if (r)
                        return r;
-               amdgpu_gfx_mqd_reset_restore(ring);
 
-               r = amdgpu_mes_map_legacy_queue(adev, ring, 0);
-               if (r) {
-                       dev_err(adev->dev, "failed to remap kgq\n");
-                       return r;
-               }
+               if (need_reinit)
+                       *need_reinit = true;
        }>         return 0;
 }
@@ -2034,12 +2047,19 @@ int amdgpu_gfx_mes_reset_queue(struct amdgpu_ring *ring,
                               struct amdgpu_fence *timedout_fence,
                               bool use_mmio)
 {
+       bool need_reinit;
        int r;
 
+       /* Single-queue reset (no suspend/resume): re-add the queue inline. */
        r = amdgpu_gfx_mes_reset_queue_start(ring, vmid, timedout_fence,
-                                             use_mmio);
+                                             use_mmio, &need_reinit);
        if (r)
                return r;
+       if (need_reinit) {
+               r = amdgpu_gfx_mes_reset_queue_reinit(ring);
+               if (r)
+                       return r;
+       }
        return amdgpu_ring_reset_helper_end(ring, timedout_fence);
 }
 
@@ -2239,7 +2259,8 @@ static int amdgpu_gfx_reset_mes_kcq(struct amdgpu_device *adev,
                                    struct amdgpu_ring *guilty_ring,
                                    unsigned int db,
                                    struct amdgpu_ring **out_ring,
-                                   struct amdgpu_fence **out_fence)
+                                   struct amdgpu_fence **out_fence,
+                                   bool *out_reinit)
 {
        bool use_mmio = adev->gfx.mec.use_mmio_for_reset;
        struct amdgpu_fence *fence;
@@ -2248,14 +2269,16 @@ static int amdgpu_gfx_reset_mes_kcq(struct amdgpu_device *adev,
 
        *out_ring = NULL;
        *out_fence = NULL;
+       *out_reinit = false;
        for (i = 0; i < adev->gfx.num_compute_rings; i++) {
                ring = &adev->gfx.compute_ring[i];
                if (ring == guilty_ring)
                        continue;
                if (ring->doorbell_index == db) {
                        fence = amdgpu_ring_find_guilty_fence(ring);
+                       /* reset + unmap now; re-add (map) is deferred to after resume */
                        r = amdgpu_gfx_mes_reset_queue_start(ring, 0, fence,
-                                                             use_mmio);
+                                                             use_mmio, out_reinit);
                        if (r)
                                return r;
                        *out_ring = ring;
@@ -2306,12 +2329,16 @@ int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev,
 fence_reset:
        /* reset the queue this came from if specified */
        if (ring) {
+               bool reinit = false;
+
+               /* reset + unmap now; re-add (map) is deferred to after resume */
                r = amdgpu_gfx_mes_reset_queue_start(ring, 0, guilty_fence,
-                                                     use_mmio);
+                                                     use_mmio, &reinit);
                if (r)
                        goto out;
                deferred_end[n_deferred].ring = ring;
                deferred_end[n_deferred].fence = guilty_fence;
+               deferred_end[n_deferred].reinit = reinit;
                n_deferred++;
        }
        if (uq) {
@@ -2322,6 +2349,7 @@ int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev,
        for (i = 0; i < num_hung; i++) {
                struct amdgpu_ring *hr = NULL;
                struct amdgpu_fence *hf = NULL;
+               bool hr_reinit = false;
 
                pipe = hqd_info[i].pipe_index;
                queue = hqd_info[i].queue_index;
@@ -2330,12 +2358,13 @@ int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev,
                /* reset any KCQs */
                r = amdgpu_gfx_reset_mes_kcq(adev, ring,
                                             adev->gfx.mec.mes_hung_db_array[i],
-                                            &hr, &hf);
+                                            &hr, &hf, &hr_reinit);
                if (r)
                        goto out;
                if (hr) {
                        deferred_end[n_deferred].ring = hr;
                        deferred_end[n_deferred].fence = hf;
+                       deferred_end[n_deferred].reinit = hr_reinit;
                        n_deferred++;
                }
                /* reset any KFD queues */
@@ -2372,12 +2401,21 @@ int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev,
        /* resume all will enable the non-hung queues */
        amdgpu_mes_resume(adev, 0);
 
-       /* Now CP is running again — replay backed-up commands and ring
-        * doorbells on each reset queue.
+       /* Now CP is running again — for queues that were unmapped during the
+        * reset, re-add (map) them only now that MES is resumed and back to a
+        * normal state, then replay backed-up commands and ring doorbells on
+        * each reset queue.
         */
        for (i = 0; i < n_deferred; i++) {
-               int er = amdgpu_ring_reset_helper_end(deferred_end[i].ring,
-                                                     deferred_end[i].fence);
+               int er;
+
+               if (deferred_end[i].reinit) {
+                       er = amdgpu_gfx_mes_reset_queue_reinit(deferred_end[i].ring);
+                       if (er && !r)
+                               r = er;
+               }
+               er = amdgpu_ring_reset_helper_end(deferred_end[i].ring,
+                                                 deferred_end[i].fence);
                if (er && !r)
                        r = er;
        }
drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h +1 -1
index aefd4f03b443..9432107c96a1 100644
@@ -550,6 +550,7 @@ struct amdgpu_gfx {
 struct amdgpu_gfx_deferred_entry {
        struct amdgpu_ring      *ring;
        struct amdgpu_fence     *fence;
+       bool                    reinit;
 };
 
 struct amdgpu_gfx_ras_reg_entry {
--

2.49.0

#4 lore raw
In reply to Jesse Zhang

AMD General

Shouldn't this apply to mes v11 as well ?

Shaoyun.liu

-----Original Message-----
From: Jesse Zhang <[email protected]>
Sent: Saturday, June 20, 2026 11:38 AM
To: [email protected]
Cc: Deucher, Alexander <[email protected]>; Koenig, Christian <[email protected]>; Liu, Shaoyun <[email protected]>; Lin, Amber <[email protected]>; Zhang, Jesse(Jie) <[email protected]>
Subject: [PATCH 1/3] drm/amdgpu/mes12: drop queue state on RESET_QUEUES unmap

Set remove_queue_after_reset=1 (MES >= 0x5a) so MES drops its internal state instead of re-unmapping an already MMIO-reset queue, which can timeout into a GPU reset.

---
 drivers/gpu/drm/amd/amdgpu/mes_v12_0.c | 11 +++++++++++
 1 file changed, 11 insertions(+)
drivers/gpu/drm/amd/amdgpu/mes_v12_0.c +11 -0
index d80a983b1b6c..20f4fd57b1da 100644
@@ -749,6 +749,17 @@ static int mes_v12_0_unmap_legacy_queue(struct amdgpu_mes *mes,
                mes_remove_queue_pkt.unmap_legacy_queue = 1;
                mes_remove_queue_pkt.queue_type =
                        convert_to_mes_queue_type(input->queue_type);
+               /*
+                * A reset-time unmap: the queue was already reset via MMIO while
+                * gangs are suspended and it is on the MES hung/fail list. Tell
+                * MES to just drop its internal state for it. Without this flag
+                * MES asks CP to unmap the already-reset (still wedged) queue
+                * again, which times out and forces a GPU reset.
+                */
+               if (input->action == RESET_QUEUES &&
+                   (mes->sched_version & AMDGPU_MES_VERSION_MASK) >= 0x5a)
+                       mes_remove_queue_pkt.remove_queue_after_reset = 1;
+
        }
 
        if (mes->adev->enable_uni_mes) {

--
2.49.0

#5 lore raw

AMD General

Quoted 10 lines

-----Original Message-----
From: Liu, Shaoyun <[email protected]>
Sent: Tuesday, June 23, 2026 12:22 AM
To: Zhang, Jesse(Jie) <[email protected]>; [email protected]
Cc: Deucher, Alexander <[email protected]>; Koenig, Christian <[email protected]>; Lin, Amber <[email protected]>; Zhang, Jesse(Jie) <[email protected]>
Subject: RE: [PATCH 1/3] drm/amdgpu/mes12: drop queue state on RESET_QUEUES unmap

Quoted 3 lines

AMD General

Shouldn't this apply to mes v11 as well ?

Thanks Shaoyun. I will prepare the patch for mev v11 .

Quoted 51 lines

Shaoyun.liu

-----Original Message-----
From: Jesse Zhang <[email protected]>
Sent: Saturday, June 20, 2026 11:38 AM
To: [email protected]
Cc: Deucher, Alexander <[email protected]>; Koenig, Christian <[email protected]>; Liu, Shaoyun <[email protected]>; Lin, Amber <[email protected]>; Zhang, Jesse(Jie) <[email protected]>
Subject: [PATCH 1/3] drm/amdgpu/mes12: drop queue state on RESET_QUEUES unmap

Set remove_queue_after_reset=1 (MES >= 0x5a) so MES drops its internal state instead of re-unmapping an already MMIO-reset queue, which can timeout into a GPU reset.

---
 drivers/gpu/drm/amd/amdgpu/mes_v12_0.c | 11 +++++++++++
 1 file changed, 11 insertions(+)
drivers/gpu/drm/amd/amdgpu/mes_v12_0.c +11 -0
index d80a983b1b6c..20f4fd57b1da 100644
@@ -749,6 +749,17 @@ static int mes_v12_0_unmap_legacy_queue(struct
 amdgpu_mes *mes,
                mes_remove_queue_pkt.unmap_legacy_queue = 1;
                mes_remove_queue_pkt.queue_type =
                        convert_to_mes_queue_type(input->queue_type);
+               /*
+                * A reset-time unmap: the queue was already reset via MMIO while
+                * gangs are suspended and it is on the MES hung/fail list. Tell
+                * MES to just drop its internal state for it. Without this flag
+                * MES asks CP to unmap the already-reset (still wedged) queue
+                * again, which times out and forces a GPU reset.
+                */
+               if (input->action == RESET_QUEUES &&
+                   (mes->sched_version & AMDGPU_MES_VERSION_MASK) >=
 0x5a)
+                       mes_remove_queue_pkt.remove_queue_after_reset = 1;
+
        }
        if (mes->adev->enable_uni_mes) {

--
2.49.0