drm/amdgpu/userq: add reset helper and identify guilty user queue
Subsystems (2)
PATCH v3 1/5 drm/amdgpu/userq: add reset helper and identify guilty user queue
If we get an interrupt for a bad user queue (bad opcode, etc.), add a helper to handle the reset for user queues.
v2: squash in fixes
v3:
- schedule the reset via amdgpu_userq_start_hang_detect_work() instead of open-coding mod_delayed_work()
- drop the per-queue guilty flag; always reset the queue the hang detect work belongs to, matching the non-compute reset path
--- drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 22 +++++++++++++++++++++- drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h | 11 +++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c +21 -1
@@ -141,7 +141,8 @@ static void amdgpu_userq_hang_detect_work(struct work_struct *work)int r;if (queue->queue_type ==AMDGPU_HW_IP_COMPUTE ) - r =amdgpu_gfx_reset_mes_compute (adev,NULL ,NULL ,NULL ,NULL ,NULL ); + r =amdgpu_gfx_reset_mes_compute (adev,NULL ,NULL , + queue,NULL ,NULL );else r = userq_funcs->reset (queue);if (r)
@@ -671,6 +672,7 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args) } queue->doorbell_index = index; + queue->doorbell_offset = (u32 )args->in.doorbell_offset; r = uq_funcs->mqd_create (queue, &args->in);if (r) {drm_file_err (uq_mgr->file,"Failed to create Queue\n" );
@@ -1112,6 +1114,24 @@ static void amdgpu_userq_restore_worker(struct work_struct *work)dma_fence_put (ev_fence); } +void amdgpu_userq_process_reset_irq (struct amdgpu_device *adev, +u32 pasid,u32 doorbell_offset) +{ +struct xarray *xa = &adev->userq_doorbell_xa; +struct amdgpu_usermode_queue *queue; +unsigned long flags, idx; + +xa_lock_irqsave (xa, flags); +xa_for_each (xa, idx, queue) { +if (queue->vm && queue->vm->pasid == pasid && + queue->doorbell_offset == doorbell_offset) { +amdgpu_userq_start_hang_detect_work (queue); +break ; + } + } +xa_unlock_irqrestore (xa, flags); +} +static int amdgpu_userq_evict_all (struct amdgpu_userq_mgr *uq_mgr) {
drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h +11 -0
@@ -53,6 +53,7 @@ struct amdgpu_usermode_queue {enum amdgpu_userq_state state;uint64_t doorbell_handle;uint64_t doorbell_index; +u32 doorbell_offset;uint64_t flags;struct amdgpu_mqd_prop *userq_prop;struct amdgpu_userq_mgr *userq_mgr;
@@ -178,6 +179,16 @@ int amdgpu_userq_post_reset(struct amdgpu_device *adev, bool vram_lost);void amdgpu_userq_start_hang_detect_work (struct amdgpu_usermode_queue *queue);void amdgpu_userq_process_fence_irq (struct amdgpu_device *adev,u32 doorbell); +/* +* CP packs the per-process doorbell_id of the queue in +* CTXID0[9:0] on priv-fault (same encoding KFD uses via +* KFD_CTXID0_DOORBELL_ID_MASK) +*/ +#define AMDGPU_CTXID0_DOORBELL_ID_MASK 0x3ff + +void amdgpu_userq_process_reset_irq (struct amdgpu_device *adev, +u32 pasid,u32 doorbell_offset); +int amdgpu_userq_input_va_validate (struct amdgpu_device *adev,struct amdgpu_usermode_queue *queue,u64 addr,u64 expected_size,u64 *va_out);
Signature · 1 line
2.49.0
PATCH v3 2/5 drm/amdgpu/gfx11: handle error interrupts for userqs
Call the new userq reset helper, and dispatch KQs first by ring_id before falling back to the user-queue lookup.
v2: squash in fixes
--- drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c | 32 ++++++++++++++++++-------- 1 file changed, 23 insertions(+), 9 deletions(-)
drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c +23 -9
@@ -6683,22 +6683,29 @@ static int gfx_v11_0_set_priv_inst_fault_state(struct amdgpu_device *adev,static void gfx_v11_0_handle_priv_fault (struct amdgpu_device *adev,struct amdgpu_iv_entry *entry) { -u8 me_id, pipe_id, queue_id; -struct amdgpu_ring *ring; -int i; - - me_id = (entry->ring_id &0x0c ) >>2 ; - pipe_id = (entry->ring_id &0x03 ) >>0 ; - queue_id = (entry->ring_id &0x70 ) >>4 ; +u32 doorbell_offset = entry->src_data[0 ] &AMDGPU_CTXID0_DOORBELL_ID_MASK ; +/* +* Try KQ first by ring_id (HW slot is authoritative). The +* KMD compute_hqd_mask contract guarantees KCQ and user queues +* never share a HW slot. +*/ if (!adev->gfx.disable_kq) { +u8 me_id = (entry->ring_id &0x0c ) >>2 ; +u8 pipe_id = (entry->ring_id &0x03 ) >>0 ; +u8 queue_id = (entry->ring_id &0x70 ) >>4 ; +struct amdgpu_ring *ring; +int i; +switch (me_id) {case 0 :for (i =0 ; i < adev->gfx.num_gfx_rings; i++) { ring = &adev->gfx.gfx_ring[i];if (ring->me == me_id && ring->pipe == pipe_id && - ring->queue == queue_id) + ring->queue == queue_id) {drm_sched_fault (&ring->sched); +return ; + } }break ;case 1 :
@@ -6706,8 +6713,10 @@ static void gfx_v11_0_handle_priv_fault(struct amdgpu_device *adev,for (i =0 ; i < adev->gfx.num_compute_rings; i++) { ring = &adev->gfx.compute_ring[i];if (ring->me == me_id && ring->pipe == pipe_id && - ring->queue == queue_id) + ring->queue == queue_id) {drm_sched_fault (&ring->sched); +return ; + } }break ;default :
@@ -6715,6 +6724,11 @@ static void gfx_v11_0_handle_priv_fault(struct amdgpu_device *adev,break ; } } + +/* No KQ matched: HW slot is a MES-scheduled user queue. */ +if (adev->enable_mes && doorbell_offset) +amdgpu_userq_process_reset_irq (adev, entry->pasid, + doorbell_offset); } static int gfx_v11_0_priv_reg_irq(struct amdgpu_device *adev,
Signature · 1 line
2.49.0
PATCH v3 3/5 drm/amdgpu/gfx12: handle error interrupts for userqs
In reply to Jesse ZhangCall the new userq reset helper, and dispatch KQs first by ring_id before falling back to the user-queue lookup.
v2: squash in fixes
--- drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c | 33 +++++++++++++++++++------- 1 file changed, 24 insertions(+), 9 deletions(-)
drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c +24 -9
@@ -5011,22 +5011,30 @@ static int gfx_v12_0_set_priv_inst_fault_state(struct amdgpu_device *adev,static void gfx_v12_0_handle_priv_fault (struct amdgpu_device *adev,struct amdgpu_iv_entry *entry) { -u8 me_id, pipe_id, queue_id; -struct amdgpu_ring *ring; -int i; - - me_id = (entry->ring_id &0x0c ) >>2 ; - pipe_id = (entry->ring_id &0x03 ) >>0 ; - queue_id = (entry->ring_id &0x70 ) >>4 ; +u32 doorbell_offset = entry->src_data[0 ] &AMDGPU_CTXID0_DOORBELL_ID_MASK ; +/* +* Try KQ first by ring_id; UQ as fallback. KCQ and UQ never share +* a HW slot (compute_hqd_mask contract). +*/ if (!adev->gfx.disable_kq) { +u8 me_id, pipe_id, queue_id; +struct amdgpu_ring *ring; +int i; + + me_id = (entry->ring_id &0x0c ) >>2 ; + pipe_id = (entry->ring_id &0x03 ) >>0 ; + queue_id = (entry->ring_id &0x70 ) >>4 ; +switch (me_id) {case 0 :for (i =0 ; i < adev->gfx.num_gfx_rings; i++) { ring = &adev->gfx.gfx_ring[i];if (ring->me == me_id && ring->pipe == pipe_id && - ring->queue == queue_id) + ring->queue == queue_id) {drm_sched_fault (&ring->sched); +return ; + } }break ;case 1 :
@@ -5034,8 +5042,10 @@ static void gfx_v12_0_handle_priv_fault(struct amdgpu_device *adev,for (i =0 ; i < adev->gfx.num_compute_rings; i++) { ring = &adev->gfx.compute_ring[i];if (ring->me == me_id && ring->pipe == pipe_id && - ring->queue == queue_id) + ring->queue == queue_id) {drm_sched_fault (&ring->sched); +return ; + } }break ;default :
@@ -5043,6 +5053,11 @@ static void gfx_v12_0_handle_priv_fault(struct amdgpu_device *adev,break ; } } + +/* No KQ matched: HW slot is a MES-scheduled user queue. */ +if (adev->enable_mes && doorbell_offset) +amdgpu_userq_process_reset_irq (adev, entry->pasid, + doorbell_offset); } static int gfx_v12_0_priv_reg_irq(struct amdgpu_device *adev,
Signature · 1 line
2.49.0
PATCH v3 4/5 drm/amdgpu/gfx11: fix EOP interrupt routing for KQ and userq
In reply to Jesse ZhangTry KQ by ring_id first (KCQ and UQ never share a HW slot); fall back to amdgpu_userq_process_fence_irq() on miss, since KQ EOPs were misrouted into the userq fence path when enable_mes is true.
Require a strict (me,pipe,queue) match in the gfx case, then userq gfx EOPs fall through to amdgpu_userq_process_fence_irq().
--- drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c | 43 +++++++++++++++++--------- 1 file changed, 29 insertions(+), 14 deletions(-)
drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c +29 -14
@@ -6517,25 +6517,33 @@ static int gfx_v11_0_eop_irq(struct amdgpu_device *adev, struct amdgpu_iv_entry *entry) {u32 doorbell_offset = entry->src_data[0 ]; -u8 me_id, pipe_id, queue_id; -struct amdgpu_ring *ring; -int i;DRM_DEBUG ("IH: CP EOP\n" ); -if (adev->enable_mes && doorbell_offset) { -amdgpu_userq_process_fence_irq (adev, doorbell_offset); - }else { - me_id = (entry->ring_id &0x0c ) >>2 ; - pipe_id = (entry->ring_id &0x03 ) >>0 ; - queue_id = (entry->ring_id &0x70 ) >>4 ; +if (!adev->gfx.disable_kq) { +u8 me_id = (entry->ring_id &0x0c ) >>2 ; +u8 pipe_id = (entry->ring_id &0x03 ) >>0 ; +u8 queue_id = (entry->ring_id &0x70 ) >>4 ; +struct amdgpu_ring *ring; +int i;switch (me_id) {case 0 : -if (pipe_id ==0 ) -amdgpu_fence_process (&adev->gfx.gfx_ring[0 ]); -else -amdgpu_fence_process (&adev->gfx.gfx_ring[1 ]); +/* +* MES splits gfx HQDs per (me,pipe): KGQ owns queue=0, +* userq gfx owns queue>=1 (see amdgpu_mes_get_hqd_mask). +* Require a strict (me,pipe,queue) match so userq gfx +* EOPs fall through to amdgpu_userq_process_fence_irq(). +*/ +for (i =0 ; i < adev->gfx.num_gfx_rings; i++) { + ring = &adev->gfx.gfx_ring[i]; +if ((ring->me == me_id) && + (ring->pipe == pipe_id) && + (ring->queue == queue_id)) { +amdgpu_fence_process (ring); +return 0 ; + } + }break ;case 1 :case 2 :
@@ -6547,13 +6555,20 @@ static int gfx_v11_0_eop_irq(struct amdgpu_device *adev,*/ if ((ring->me == me_id) && (ring->pipe == pipe_id) && - (ring->queue == queue_id)) + (ring->queue == queue_id)) {amdgpu_fence_process (ring); +return 0 ; + } }break ; +default : +break ; } } +if (adev->enable_mes && doorbell_offset) +amdgpu_userq_process_fence_irq (adev, doorbell_offset); +return 0 ; }
Signature · 1 line
2.49.0
PATCH v3 5/5 drm/amdgpu/gfx12: fix EOP interrupt routing for KQ and userq
In reply to Jesse ZhangTry KQ by ring_id first (KCQ and UQ never share a HW slot); fall back to amdgpu_userq_process_fence_irq() on miss, since KCQ EOPs were misrouted into the userq fence path when enable_mes is true.
Require a strict (me,pipe,queue) match in the gfx case, then userq gfx EOPs fall through to amdgpu_userq_process_fence_irq().
--- drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c | 43 +++++++++++++++++--------- 1 file changed, 29 insertions(+), 14 deletions(-)
drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c +29 -14
@@ -4845,25 +4845,33 @@ static int gfx_v12_0_eop_irq(struct amdgpu_device *adev, struct amdgpu_iv_entry *entry) {u32 doorbell_offset = entry->src_data[0 ]; -u8 me_id, pipe_id, queue_id; -struct amdgpu_ring *ring; -int i;DRM_DEBUG ("IH: CP EOP\n" ); -if (adev->enable_mes && doorbell_offset) { -amdgpu_userq_process_fence_irq (adev, doorbell_offset); - }else { - me_id = (entry->ring_id &0x0c ) >>2 ; - pipe_id = (entry->ring_id &0x03 ) >>0 ; - queue_id = (entry->ring_id &0x70 ) >>4 ; +if (!adev->gfx.disable_kq) { +u8 me_id = (entry->ring_id &0x0c ) >>2 ; +u8 pipe_id = (entry->ring_id &0x03 ) >>0 ; +u8 queue_id = (entry->ring_id &0x70 ) >>4 ; +struct amdgpu_ring *ring; +int i;switch (me_id) {case 0 : -if (pipe_id ==0 ) -amdgpu_fence_process (&adev->gfx.gfx_ring[0 ]); -else -amdgpu_fence_process (&adev->gfx.gfx_ring[1 ]); +/* +* MES splits gfx HQDs per (me,pipe): KGQ owns queue=0, +* userq gfx owns queue>=1 (see amdgpu_mes_get_hqd_mask). +* Require a strict (me,pipe,queue) match so userq gfx +* EOPs fall through to amdgpu_userq_process_fence_irq(). +*/ +for (i =0 ; i < adev->gfx.num_gfx_rings; i++) { + ring = &adev->gfx.gfx_ring[i]; +if ((ring->me == me_id) && + (ring->pipe == pipe_id) && + (ring->queue == queue_id)) { +amdgpu_fence_process (ring); +return 0 ; + } + }break ;case 1 :case 2 :
@@ -4875,13 +4883,20 @@ static int gfx_v12_0_eop_irq(struct amdgpu_device *adev,*/ if ((ring->me == me_id) && (ring->pipe == pipe_id) && - (ring->queue == queue_id)) + (ring->queue == queue_id)) {amdgpu_fence_process (ring); +return 0 ; + } }break ; +default : +break ; } } +if (adev->enable_mes && doorbell_offset) +amdgpu_userq_process_fence_irq (adev, doorbell_offset); +return 0 ; }
Signature · 1 line
2.49.0
Quoted 11 lines · Jesse Zhang
If we get an interrupt for a bad user queue (bad opcode, etc.), add a helper to handle the reset for user queues.
v2: squash in fixes
v3:
- schedule the reset viaamdgpu_userq_start_hang_detect_work()instead of open-codingmod_delayed_work()
- drop the per-queue guilty flag; always reset the queue the hang detect work belongs to, matching the non-compute reset path
Quoted 3 lines
Co-developed-by Alex Deucher <[email protected]>Signed-off-by Alex Deucher <[email protected]>Signed-off-by Jesse Zhang <[email protected]>
Series is:
Quoted 84 lines
--- drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 22 +++++++++++++++++++++- drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h | 11 +++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-)drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c +21
-1@@ -141,7 +141,8 @@ static void amdgpu_userq_hang_detect_work(struct work_struct *work)int r;if (queue->queue_type ==AMDGPU_HW_IP_COMPUTE ) - r =amdgpu_gfx_reset_mes_compute (adev,NULL ,NULL ,NULL ,NULL ,NULL ); + r =amdgpu_gfx_reset_mes_compute (adev,NULL ,NULL , + queue,NULL ,NULL );else r = userq_funcs->reset (queue);if (r)@@ -671,6 +672,7 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args) } queue->doorbell_index = index; + queue->doorbell_offset = (u32 )args->in.doorbell_offset; r = uq_funcs->mqd_create (queue, &args->in);if (r) {drm_file_err (uq_mgr->file,"Failed to create Queue\n" );@@ -1112,6 +1114,24 @@ static void amdgpu_userq_restore_worker(struct work_struct *work)dma_fence_put (ev_fence); } +void amdgpu_userq_process_reset_irq (struct amdgpu_device *adev, +u32 pasid,u32 doorbell_offset) +{ +struct xarray *xa = &adev->userq_doorbell_xa; +struct amdgpu_usermode_queue *queue; +unsigned long flags, idx; + +xa_lock_irqsave (xa, flags); +xa_for_each (xa, idx, queue) { +if (queue->vm && queue->vm->pasid == pasid && + queue->doorbell_offset == doorbell_offset) { +amdgpu_userq_start_hang_detect_work (queue); +break ; + } + } +xa_unlock_irqrestore (xa, flags); +} +static int amdgpu_userq_evict_all (struct amdgpu_userq_mgr *uq_mgr) {drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h +11
-1@@ -53,6 +53,7 @@ struct amdgpu_usermode_queue {enum amdgpu_userq_state state;uint64_t doorbell_handle;uint64_t doorbell_index; +u32 doorbell_offset;uint64_t flags;struct amdgpu_mqd_prop *userq_prop;struct amdgpu_userq_mgr *userq_mgr;@@ -178,6 +179,16 @@ int amdgpu_userq_post_reset(struct amdgpu_device *adev, bool vram_lost);void amdgpu_userq_start_hang_detect_work (struct amdgpu_usermode_queue *queue);void amdgpu_userq_process_fence_irq (struct amdgpu_device *adev,u32 doorbell); +/* +* CP packs the per-process doorbell_id of the queue in +* CTXID0[9:0] on priv-fault (same encoding KFD uses via +* KFD_CTXID0_DOORBELL_ID_MASK) +*/ +#define AMDGPU_CTXID0_DOORBELL_ID_MASK 0x3ff + +void amdgpu_userq_process_reset_irq (struct amdgpu_device *adev, +u32 pasid,u32 doorbell_offset); +int amdgpu_userq_input_va_validate (struct amdgpu_device *adev,struct amdgpu_usermode_queue *queue,u64 addr,u64 expected_size,u64 *va_out); --2.49.0