drm/amdgpu/mes12: drop queue state on RESET_QUEUES unmap
Subsystems (3)
PATCH 1/3 drm/amdgpu/mes12: drop queue state on RESET_QUEUES unmap
Set remove_queue_after_reset=1 (MES >= 0x5a) so MES drops its internal state instead of re-unmapping an already MMIO-reset queue, which can timeout into a GPU reset.
--- drivers/gpu/drm/amd/amdgpu/mes_v12_0.c | 11 +++++++++++ 1 file changed, 11 insertions(+)
drivers/gpu/drm/amd/amdgpu/mes_v12_0.c +11 -0
@@ -749,6 +749,17 @@ static int mes_v12_0_unmap_legacy_queue(struct amdgpu_mes *mes, mes_remove_queue_pkt.unmap_legacy_queue =1 ; mes_remove_queue_pkt.queue_type =convert_to_mes_queue_type (input->queue_type); +/* +* A reset-time unmap: the queue was already reset via MMIO while +* gangs are suspended and it is on the MES hung/fail list. Tell +* MES to just drop its internal state for it. Without this flag +* MES asks CP to unmap the already-reset (still wedged) queue +* again, which times out and forces a GPU reset. +*/ +if (input->action ==RESET_QUEUES && + (mes->sched_version &AMDGPU_MES_VERSION_MASK ) >=0x5a ) + mes_remove_queue_pkt.remove_queue_after_reset =1 ; + }if (mes->adev->enable_uni_mes) {
Signature · 1 line
2.49.0
PATCH 2/3 drm/amdkfd: flush MES queue on reset-time queue removal
Pass flush_mes_queue=true in reset_queue_mes() to match the GFX post-reset drop semantics.
--- drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +1 -1
@@ -427,7 +427,7 @@ static int reset_queue_mes(struct device_queue_manager *dqm, struct queue *q,if (r)return r;/* Proceed remove_queue with reset=true */ -remove_queue_mes_on_reset_option (dqm, q, &pdd->qpd, true, false); +remove_queue_mes_on_reset_option (dqm, q, &pdd->qpd, true, true);set_queue_as_reset (dqm, q, &pdd->qpd);return 0 ; }
Signature · 1 line
2.49.0
PATCH 3/3 drm/amdgpu: defer KCQ remap until after MES resume in reset flow
In reply to Jesse ZhangSplit amdgpu_gfx_mes_reset_queue_start() into reset+unmap now and queue reinit later, and do the remap only after amdgpu_mes_resume(). Avoids re-adding legacy queues while MES gangs are still suspended.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 70 +++++++++++++++++++------ drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h | 1 + 2 files changed, 55 insertions(+), 16 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c +54 -16
@@ -1989,10 +1989,24 @@ static ssize_t amdgpu_gfx_get_compute_reset_mask(struct device *dev,return amdgpu_show_reset_mask (buf, adev->gfx.compute_supported_reset); } +static int amdgpu_gfx_mes_reset_queue_reinit (struct amdgpu_ring *ring) +{ +struct amdgpu_device *adev = ring->adev; +int r; + +amdgpu_gfx_mqd_reset_restore (ring); + + r =amdgpu_mes_map_legacy_queue (adev, ring,0 ); +if (r) +dev_err (adev->dev,"failed to remap kgq\n" ); + +return r; +} +static int amdgpu_gfx_mes_reset_queue_start (struct amdgpu_ring *ring,unsigned int vmid,struct amdgpu_fence *timedout_fence, -bool use_mmio) +bool use_mmio,bool *need_reinit) {struct amdgpu_device *adev = ring->adev;bool reinit_queue;
@@ -2007,6 +2021,9 @@ static int amdgpu_gfx_mes_reset_queue_start(struct amdgpu_ring *ring,else reinit_queue = use_mmio; +if (need_reinit) + *need_reinit = false; +amdgpu_ring_reset_helper_begin (ring, timedout_fence); r =amdgpu_mes_reset_legacy_queue (ring->adev, ring, vmid, use_mmio,0 );
@@ -2018,13 +2035,9 @@ static int amdgpu_gfx_mes_reset_queue_start(struct amdgpu_ring *ring, RESET_QUEUES, 0, 0, 0);if (r)return r; -amdgpu_gfx_mqd_reset_restore (ring); - r =amdgpu_mes_map_legacy_queue (adev, ring,0 ); -if (r) { -dev_err (adev->dev,"failed to remap kgq\n" ); -return r; - } +if (need_reinit) + *need_reinit = true; }return 0 ; }
@@ -2034,12 +2047,19 @@ int amdgpu_gfx_mes_reset_queue(struct amdgpu_ring *ring,struct amdgpu_fence *timedout_fence, bool use_mmio) { + bool need_reinit;int r; +/* Single-queue reset (no suspend/resume): re-add the queue inline. */ r =amdgpu_gfx_mes_reset_queue_start (ring, vmid, timedout_fence, - use_mmio); + use_mmio, &need_reinit);if (r)return r; +if (need_reinit) { + r =amdgpu_gfx_mes_reset_queue_reinit (ring); +if (r) +return r; + }return amdgpu_ring_reset_helper_end (ring, timedout_fence); }
@@ -2239,7 +2259,8 @@ static int amdgpu_gfx_reset_mes_kcq(struct amdgpu_device *adev,struct amdgpu_ring *guilty_ring, unsigned int db, struct amdgpu_ring **out_ring, - struct amdgpu_fence **out_fence) + struct amdgpu_fence **out_fence, + bool *out_reinit) { bool use_mmio = adev->gfx.mec.use_mmio_for_reset;struct amdgpu_fence *fence;
@@ -2248,14 +2269,16 @@ static int amdgpu_gfx_reset_mes_kcq(struct amdgpu_device *adev, *out_ring =NULL ; *out_fence =NULL ; + *out_reinit = false;for (i =0 ; i < adev->gfx.num_compute_rings; i++) { ring = &adev->gfx.compute_ring[i];if (ring == guilty_ring)continue ;if (ring->doorbell_index == db) { fence =amdgpu_ring_find_guilty_fence (ring); +/* reset + unmap now; re-add (map) is deferred to after resume */ r =amdgpu_gfx_mes_reset_queue_start (ring,0 , fence, - use_mmio); + use_mmio, out_reinit);if (r)return r; *out_ring = ring;
@@ -2306,12 +2329,16 @@ int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev, fence_reset:/* reset the queue this came from if specified */ if (ring) { +bool reinit = false; + +/* reset + unmap now; re-add (map) is deferred to after resume */ r =amdgpu_gfx_mes_reset_queue_start (ring,0 , guilty_fence, - use_mmio); + use_mmio, &reinit);if (r) goto out; deferred_end[n_deferred].ring = ring; deferred_end[n_deferred].fence = guilty_fence; + deferred_end[n_deferred].reinit = reinit; n_deferred++; }if (uq) {
@@ -2322,6 +2349,7 @@ int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev,for (i =0 ; i < num_hung; i++) {struct amdgpu_ring *hr =NULL ;struct amdgpu_fence *hf =NULL ; +bool hr_reinit = false; pipe = hqd_info[i].pipe_index; queue = hqd_info[i].queue_index;
@@ -2330,12 +2358,13 @@ int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev,/* reset any KCQs */ r =amdgpu_gfx_reset_mes_kcq (adev, ring, adev->gfx.mec.mes_hung_db_array[i], - &hr, &hf); + &hr, &hf, &hr_reinit);if (r) goto out;if (hr) { deferred_end[n_deferred].ring = hr; deferred_end[n_deferred].fence = hf; + deferred_end[n_deferred].reinit = hr_reinit; n_deferred++; }/* reset any KFD queues */
@@ -2372,12 +2401,21 @@ int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev,/* resume all will enable the non-hung queues */ amdgpu_mes_resume (adev,0 ); -/* Now CP is running again — replay backed-up commands and ring -* doorbells on each reset queue. +/* Now CP is running again — for queues that were unmapped during the +* reset, re-add (map) them only now that MES is resumed and back to a +* normal state, then replay backed-up commands and ring doorbells on +* each reset queue. */ for (i =0 ; i < n_deferred; i++) { -int er =amdgpu_ring_reset_helper_end (deferred_end[i].ring, - deferred_end[i].fence); +int er; + +if (deferred_end[i].reinit) { + er =amdgpu_gfx_mes_reset_queue_reinit (deferred_end[i].ring); +if (er && !r) + r = er; + } + er =amdgpu_ring_reset_helper_end (deferred_end[i].ring, + deferred_end[i].fence);if (er && !r) r = er; }
drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h +1 -0
@@ -550,6 +550,7 @@ struct amdgpu_gfx {struct amdgpu_gfx_deferred_entry {struct amdgpu_ring *ring;struct amdgpu_fence *fence; +bool reinit; };struct amdgpu_gfx_ras_reg_entry {
Signature · 1 line
2.49.0
Re: [PATCH 3/3] drm/amdgpu: defer KCQ remap until after MES resume in reset flow
Quoted 7 lines · Jesse Zhang
Split
amdgpu_gfx_mes_reset_queue_start()into reset+unmap now and queue reinit later, and do the remap only afteramdgpu_mes_resume(). Avoids re-adding legacy queues while MES gangs are still suspended.Suggested-by Shaoyun Liu <[email protected]>Signed-off-by Jesse Zhang <[email protected]>
The series is:
Is there a way we can clean this up in a future patch set? It's getting really complicated.
Alex
Quoted 191 lines
--- drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 70 +++++++++++++++++++------ drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h | 1 + 2 files changed, 55 insertions(+), 16 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c +54
-16@@ -1989,10 +1989,24 @@ static ssize_t amdgpu_gfx_get_compute_reset_mask(struct device *dev,return amdgpu_show_reset_mask (buf, adev->gfx.compute_supported_reset); } +static int amdgpu_gfx_mes_reset_queue_reinit (struct amdgpu_ring *ring) +{ +struct amdgpu_device *adev = ring->adev; +int r; + +amdgpu_gfx_mqd_reset_restore (ring); + + r =amdgpu_mes_map_legacy_queue (adev, ring,0 ); +if (r) +dev_err (adev->dev,"failed to remap kgq\n" ); + +return r; +} +static int amdgpu_gfx_mes_reset_queue_start (struct amdgpu_ring *ring,unsigned int vmid,struct amdgpu_fence *timedout_fence, -bool use_mmio) +bool use_mmio,bool *need_reinit) {struct amdgpu_device *adev = ring->adev;bool reinit_queue;@@ -2007,6 +2021,9 @@ static int amdgpu_gfx_mes_reset_queue_start(struct amdgpu_ring *ring,else reinit_queue = use_mmio; +if (need_reinit) + *need_reinit = false; +amdgpu_ring_reset_helper_begin (ring, timedout_fence); r =amdgpu_mes_reset_legacy_queue (ring->adev, ring, vmid, use_mmio,0 );@@ -2018,13 +2035,9 @@ static int amdgpu_gfx_mes_reset_queue_start(struct amdgpu_ring *ring, RESET_QUEUES, 0, 0, 0);if (r)return r; -amdgpu_gfx_mqd_reset_restore (ring); - r =amdgpu_mes_map_legacy_queue (adev, ring,0 ); -if (r) { -dev_err (adev->dev,"failed to remap kgq\n" ); -return r; - } +if (need_reinit) + *need_reinit = true; }> return 0; }@@ -2034,12 +2047,19 @@ int amdgpu_gfx_mes_reset_queue(struct amdgpu_ring *ring,struct amdgpu_fence *timedout_fence, bool use_mmio) { + bool need_reinit;int r; +/* Single-queue reset (no suspend/resume): re-add the queue inline. */ r =amdgpu_gfx_mes_reset_queue_start (ring, vmid, timedout_fence, - use_mmio); + use_mmio, &need_reinit);if (r)return r; +if (need_reinit) { + r =amdgpu_gfx_mes_reset_queue_reinit (ring); +if (r) +return r; + }return amdgpu_ring_reset_helper_end (ring, timedout_fence); }@@ -2239,7 +2259,8 @@ static int amdgpu_gfx_reset_mes_kcq(struct amdgpu_device *adev,struct amdgpu_ring *guilty_ring, unsigned int db, struct amdgpu_ring **out_ring, - struct amdgpu_fence **out_fence) + struct amdgpu_fence **out_fence, + bool *out_reinit) { bool use_mmio = adev->gfx.mec.use_mmio_for_reset;struct amdgpu_fence *fence;@@ -2248,14 +2269,16 @@ static int amdgpu_gfx_reset_mes_kcq(struct amdgpu_device *adev, *out_ring =NULL ; *out_fence =NULL ; + *out_reinit = false;for (i =0 ; i < adev->gfx.num_compute_rings; i++) { ring = &adev->gfx.compute_ring[i];if (ring == guilty_ring)continue ;if (ring->doorbell_index == db) { fence =amdgpu_ring_find_guilty_fence (ring); +/* reset + unmap now; re-add (map) is deferred to after resume */ r =amdgpu_gfx_mes_reset_queue_start (ring,0 , fence, - use_mmio); + use_mmio, out_reinit);if (r)return r; *out_ring = ring;@@ -2306,12 +2329,16 @@ int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev, fence_reset:/* reset the queue this came from if specified */ if (ring) { +bool reinit = false; + +/* reset + unmap now; re-add (map) is deferred to after resume */ r =amdgpu_gfx_mes_reset_queue_start (ring,0 , guilty_fence, - use_mmio); + use_mmio, &reinit);if (r) goto out; deferred_end[n_deferred].ring = ring; deferred_end[n_deferred].fence = guilty_fence; + deferred_end[n_deferred].reinit = reinit; n_deferred++; }if (uq) {@@ -2322,6 +2349,7 @@ int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev,for (i =0 ; i < num_hung; i++) {struct amdgpu_ring *hr =NULL ;struct amdgpu_fence *hf =NULL ; +bool hr_reinit = false; pipe = hqd_info[i].pipe_index; queue = hqd_info[i].queue_index;@@ -2330,12 +2358,13 @@ int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev,/* reset any KCQs */ r =amdgpu_gfx_reset_mes_kcq (adev, ring, adev->gfx.mec.mes_hung_db_array[i], - &hr, &hf); + &hr, &hf, &hr_reinit);if (r) goto out;if (hr) { deferred_end[n_deferred].ring = hr; deferred_end[n_deferred].fence = hf; + deferred_end[n_deferred].reinit = hr_reinit; n_deferred++; }/* reset any KFD queues */ @@ -2372,12 +2401,21 @@ int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev,/* resume all will enable the non-hung queues */ amdgpu_mes_resume (adev,0 ); -/* Now CP is running again — replay backed-up commands and ring -* doorbells on each reset queue. +/* Now CP is running again — for queues that were unmapped during the +* reset, re-add (map) them only now that MES is resumed and back to a +* normal state, then replay backed-up commands and ring doorbells on +* each reset queue. */ for (i =0 ; i < n_deferred; i++) { -int er =amdgpu_ring_reset_helper_end (deferred_end[i].ring, - deferred_end[i].fence); +int er; + +if (deferred_end[i].reinit) { + er =amdgpu_gfx_mes_reset_queue_reinit (deferred_end[i].ring); +if (er && !r) + r = er; + } + er =amdgpu_ring_reset_helper_end (deferred_end[i].ring, + deferred_end[i].fence);if (er && !r) r = er; }drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h +1
-1@@ -550,6 +550,7 @@ struct amdgpu_gfx {struct amdgpu_gfx_deferred_entry {struct amdgpu_ring *ring;struct amdgpu_fence *fence; +bool reinit; };struct amdgpu_gfx_ras_reg_entry { --2.49.0
AMD General
Shouldn't this apply to mes v11 as well ?
Shaoyun.liu
-----Original Message-----
From: Jesse Zhang <[email protected]>
Sent: Saturday, June 20, 2026 11:38 AM
To: [email protected]
Cc: Deucher, Alexander <[email protected]>; Koenig, Christian <[email protected]>; Liu, Shaoyun <[email protected]>; Lin, Amber <[email protected]>; Zhang, Jesse(Jie) <[email protected]>
Subject: [PATCH 1/3] drm/amdgpu/mes12: drop queue state on RESET_QUEUES unmap
Set remove_queue_after_reset=1 (MES >= 0x5a) so MES drops its internal state instead of re-unmapping an already MMIO-reset queue, which can timeout into a GPU reset.
--- drivers/gpu/drm/amd/amdgpu/mes_v12_0.c | 11 +++++++++++ 1 file changed, 11 insertions(+)
drivers/gpu/drm/amd/amdgpu/mes_v12_0.c +11 -0
@@ -749,6 +749,17 @@ static int mes_v12_0_unmap_legacy_queue(struct amdgpu_mes *mes, mes_remove_queue_pkt.unmap_legacy_queue =1 ; mes_remove_queue_pkt.queue_type =convert_to_mes_queue_type (input->queue_type); +/* +* A reset-time unmap: the queue was already reset via MMIO while +* gangs are suspended and it is on the MES hung/fail list. Tell +* MES to just drop its internal state for it. Without this flag +* MES asks CP to unmap the already-reset (still wedged) queue +* again, which times out and forces a GPU reset. +*/ +if (input->action ==RESET_QUEUES && + (mes->sched_version &AMDGPU_MES_VERSION_MASK ) >=0x5a ) + mes_remove_queue_pkt.remove_queue_after_reset =1 ; + }if (mes->adev->enable_uni_mes) {
--
2.49.0
AMD General
Quoted 10 lines
-----Original Message-----
From: Liu, Shaoyun <[email protected]>
Sent: Tuesday, June 23, 2026 12:22 AM
To: Zhang, Jesse(Jie) <[email protected]>; [email protected]
Cc: Deucher, Alexander <[email protected]>; Koenig, Christian <[email protected]>; Lin, Amber <[email protected]>; Zhang, Jesse(Jie) <[email protected]>
Subject: RE: [PATCH 1/3] drm/amdgpu/mes12: drop queue state onRESET_QUEUESunmap
Quoted 3 lines
AMD General
Shouldn't this apply to mes v11 as well ?
Thanks Shaoyun. I will prepare the patch for mev v11 .
Quoted 51 lines
Shaoyun.liu
-----Original Message-----
From: Jesse Zhang <[email protected]>
Sent: Saturday, June 20, 2026 11:38 AM
To: [email protected]
Cc: Deucher, Alexander <[email protected]>; Koenig, Christian <[email protected]>; Liu, Shaoyun <[email protected]>; Lin, Amber <[email protected]>; Zhang, Jesse(Jie) <[email protected]>
Subject: [PATCH 1/3] drm/amdgpu/mes12: drop queue state onRESET_QUEUESunmapSet remove_queue_after_reset=1 (MES >=
0x5a) so MES drops its internal state instead of re-unmapping an already MMIO-reset queue, which can timeout into a GPU reset.Suggested-by Shaoyun Liu <[email protected]>Signed-off-by Jesse Zhang <[email protected]>--- drivers/gpu/drm/amd/amdgpu/mes_v12_0.c | 11 +++++++++++ 1 file changed, 11 insertions(+)drivers/gpu/drm/amd/amdgpu/mes_v12_0.c +11
-0@@ -749,6 +749,17 @@ static int mes_v12_0_unmap_legacy_queue(struct amdgpu_mes *mes, mes_remove_queue_pkt.unmap_legacy_queue =1 ; mes_remove_queue_pkt.queue_type =convert_to_mes_queue_type (input->queue_type); +/* +* A reset-time unmap: the queue was already reset via MMIO while +* gangs are suspended and it is on the MES hung/fail list. Tell +* MES to just drop its internal state for it. Without this flag +* MES asks CP to unmap the already-reset (still wedged) queue +* again, which times out and forces a GPU reset. +*/ +if (input->action ==RESET_QUEUES && + (mes->sched_version &AMDGPU_MES_VERSION_MASK ) >=0x5a ) + mes_remove_queue_pkt.remove_queue_after_reset =1 ; + }if (mes->adev->enable_uni_mes) {--
2.49.0