drm/amdgpu: cleanup GPU recovery check a bit
Subsystems (2)
PATCH 1/5 drm/amdgpu: cleanup GPU recovery check a bit
Check if we should call the function instead of providing the forced flag.
--- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 3 ++- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 38 ++++++++++++++++++++---------- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c | 4 ++-- drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 3 ++- drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c | 4 ++-- drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c | 3 ++- 7 files changed, 36 insertions(+), 21 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +2 -1
@@ -1158,8 +1158,9 @@ int emu_soc_asic_init(struct amdgpu_device *adev);#define amdgpu_asic_need_full_reset (adev) (adev)->asic_funcs->need_full_reset((adev))/* Common functions */ +bool amdgpu_device_should_recover_gpu (struct amdgpu_device *adev);int amdgpu_device_gpu_recover (struct amdgpu_device *adev, -struct amdgpu_job * job,bool force); +struct amdgpu_job * job);void amdgpu_device_pci_config_reset (struct amdgpu_device *adev);bool amdgpu_device_need_post (struct amdgpu_device *adev);
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +25 -13
@@ -3244,32 +3244,44 @@ static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,return r; } +/** +* amdgpu_device_should_recover_gpu - check if we should try GPU recovery +* +* @adev: amdgpu device pointer +* +* Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover +* a hung GPU. +*/ +bool amdgpu_device_should_recover_gpu (struct amdgpu_device *adev) +{ +if (!amdgpu_device_ip_check_soft_reset (adev)) { +DRM_INFO ("Timeout, but no hardware hang detected.\n" ); +return false; + } + +if (amdgpu_gpu_recovery ==0 || (amdgpu_gpu_recovery ==-1 && + !amdgpu_sriov_vf (adev))) { +DRM_INFO ("GPU recovery disabled.\n" ); +return false; + } + +return true; +} +/** * amdgpu_device_gpu_recover - reset the asic and recover scheduler * * @adev: amdgpu device pointer * @job: which job trigger hang -* @force: forces reset regardless of amdgpu_gpu_recovery * * Attempt to reset the GPU if it has hung (all asics). * Returns 0 for success or an error on failure. */ int amdgpu_device_gpu_recover (struct amdgpu_device *adev, -struct amdgpu_job *job,bool force) +struct amdgpu_job *job) {int i, r, resched; -if (!force && !amdgpu_device_ip_check_soft_reset (adev)) { -DRM_INFO ("No hardware hang detected. Did some blocks stall?\n" ); -return 0 ; - } - -if (!force && (amdgpu_gpu_recovery ==0 || - (amdgpu_gpu_recovery ==-1 && !amdgpu_sriov_vf (adev)))) { -DRM_INFO ("GPU recovery disabled.\n" ); -return 0 ; - } -dev_info (adev->dev,"GPU reset begin!\n" );mutex_lock (&adev->lock_reset);
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +1 -1
@@ -702,7 +702,7 @@ static int amdgpu_debugfs_gpu_recover(struct seq_file *m, void *data)struct amdgpu_device *adev = dev->dev_private;seq_printf (m,"gpu recover\n" ); -amdgpu_device_gpu_recover (adev,NULL , true); +amdgpu_device_gpu_recover (adev,NULL );return 0 ; }
drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c +2 -2
@@ -105,8 +105,8 @@ static void amdgpu_irq_reset_work_func(struct work_struct *work) struct amdgpu_device *adev = container_of(work, struct amdgpu_device, reset_work); -if (!amdgpu_sriov_vf (adev)) -amdgpu_device_gpu_recover (adev,NULL , false); +if (!amdgpu_sriov_vf (adev) &&amdgpu_device_should_recover_gpu (adev)) +amdgpu_device_gpu_recover (adev,NULL ); }/**
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +2 -1
@@ -37,7 +37,8 @@ static void amdgpu_job_timedout(struct drm_sched_job *s_job) job->base.sched->name,atomic_read (&ring->fence_drv.last_seq), ring->fence_drv.sync_seq); -amdgpu_device_gpu_recover (ring->adev, job, false); +if (amdgpu_device_should_recover_gpu (ring->adev)) +amdgpu_device_gpu_recover (ring->adev, job); } int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs,
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +2 -2
@@ -266,8 +266,8 @@ static void xgpu_ai_mailbox_flr_work(struct work_struct *work) }/* Trigger recovery for world switch failure if no TDR */ -if (amdgpu_lockup_timeout ==0 ) -amdgpu_device_gpu_recover (adev,NULL , true); +if (amdgpu_device_should_recover_gpu (adev)) +amdgpu_device_gpu_recover (adev,NULL ); } static int xgpu_ai_set_mailbox_rcv_irq(struct amdgpu_device *adev,
drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c +2 -1
@@ -521,7 +521,8 @@ static void xgpu_vi_mailbox_flr_work(struct work_struct *work) }/* Trigger recovery due to world switch failure */ -amdgpu_device_gpu_recover (adev,NULL , false); +if (amdgpu_device_should_recover_gpu (adev)) +amdgpu_device_gpu_recover (adev,NULL ); } static int xgpu_vi_set_mailbox_rcv_irq(struct amdgpu_device *adev,
Signature · 6 lines
2.14.1 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): 20180822100531.52863-1-christian.koenig-5C7GfCeVMHo@public.gmane.org
PATCH 2/5 drm/amdgpu: add ring soft recovery v2
Instead of hammering hard on the GPU try a soft recovery first.
v2: reorder code a bit
--- drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 6 ++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c | 24 ++++++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h | 4 ++++ 3 files changed, 34 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +6 -0
@@ -33,6 +33,12 @@ static void amdgpu_job_timedout(struct drm_sched_job *s_job)struct amdgpu_ring *ring =to_amdgpu_ring (s_job->sched);struct amdgpu_job *job =to_amdgpu_job (s_job); +if (amdgpu_ring_soft_recovery (ring, job->vmid, s_job->s_fence->parent)) { +DRM_ERROR ("ring %s timeout, but soft recovered\n" , + s_job->sched->name); +return ; + } +DRM_ERROR ("ring %s timeout, signaled seq=%u, emitted seq=%u\n" , job->base.sched->name,atomic_read (&ring->fence_drv.last_seq), ring->fence_drv.sync_seq);
drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c +24 -0
@@ -383,6 +383,30 @@ void amdgpu_ring_emit_reg_write_reg_wait_helper(struct amdgpu_ring *ring,amdgpu_ring_emit_reg_wait (ring, reg1, mask, mask); } +/** +* amdgpu_ring_soft_recovery - try to soft recover a ring lockup +* +* @ring: ring to try the recovery on +* @vmid: VMID we try to get going again +* @fence: timedout fence +* +* Tries to get a ring proceeding again when it is stuck. +*/ +bool amdgpu_ring_soft_recovery (struct amdgpu_ring *ring,unsigned int vmid, +struct dma_fence *fence) +{ +ktime_t deadline =ktime_add_us (ktime_get (),1000 ); + +if (!ring->funcs->soft_recovery) +return false; + +while (!dma_fence_is_signaled (fence) && +ktime_to_ns (ktime_sub (deadline,ktime_get ())) >0 ) + ring->funcs->soft_recovery (ring, vmid); + +return dma_fence_is_signaled (fence); +} +/* * Debugfs info */
drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +4 -0
@@ -168,6 +168,8 @@ struct amdgpu_ring_funcs {/* priority functions */ void (*set_priority) (struct amdgpu_ring *ring,enum drm_sched_priority priority); +/* Try to soft recover the ring to make the fence signal */ +void (*soft_recovery)(struct amdgpu_ring *ring,unsigned vmid); };struct amdgpu_ring {
@@ -260,6 +262,8 @@ void amdgpu_ring_fini(struct amdgpu_ring *ring);void amdgpu_ring_emit_reg_write_reg_wait_helper (struct amdgpu_ring *ring,uint32_t reg0,uint32_t val0,uint32_t reg1,uint32_t val1); +bool amdgpu_ring_soft_recovery (struct amdgpu_ring *ring,unsigned int vmid, +struct dma_fence *fence);static inline void amdgpu_ring_clear_ring (struct amdgpu_ring *ring) {
Signature · 6 lines
2.14.1 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): 20180822100531.52863-2-christian.koenig-5C7GfCeVMHo@public.gmane.org
Re: [PATCH 2/5] drm/amdgpu: add ring soft recovery v2
Quoted 17 lines · Christian König
Instead of hammering hard on the GPU try a soft recovery first.
v2: reorder code a bit
Signed-off-by Christian König <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 6 ++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c | 24 ++++++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h | 4 ++++ 3 files changed, 34 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +0
-0
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c static void amdgpu_job_timedout(struct drm_sched_job *s_job) · line 40
Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +5
-0@@ -33,6 +33,12 @@ static void amdgpu_job_timedout(struct drm_sched_job *s_job)struct amdgpu_ring *ring =to_amdgpu_ring (s_job->sched);struct amdgpu_job *job =to_amdgpu_job (s_job); +if (amdgpu_ring_soft_recovery (ring, job->vmid, s_job->s_fence->parent)) { +DRM_ERROR ("ring %s timeout, but soft recovered\n" , + s_job->sched->name); +return ; + }
I think we should still bubble up the error to userspace even if we can recover. Data is lost when the wave is killed. We should treat it like a GPU reset.
Alex
Quoted 64 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +1
-0+DRM_ERROR ("ring %s timeout, signaled seq=%u, emitted seq=%u\n" , job->base.sched->name,atomic_read (&ring->fence_drv.last_seq), ring->fence_drv.sync_seq);drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c +24
-0@@ -383,6 +383,30 @@ void amdgpu_ring_emit_reg_write_reg_wait_helper(struct amdgpu_ring *ring,amdgpu_ring_emit_reg_wait (ring, reg1, mask, mask); } +/** +* amdgpu_ring_soft_recovery - try to soft recover a ring lockup +* +* @ring: ring to try the recovery on +* @vmid: VMID we try to get going again +* @fence: timedout fence +* +* Tries to get a ring proceeding again when it is stuck. +*/ +bool amdgpu_ring_soft_recovery (struct amdgpu_ring *ring,unsigned int vmid, +struct dma_fence *fence) +{ +ktime_t deadline =ktime_add_us (ktime_get (),1000 ); + +if (!ring->funcs->soft_recovery) +return false; + +while (!dma_fence_is_signaled (fence) && +ktime_to_ns (ktime_sub (deadline,ktime_get ())) >0 ) + ring->funcs->soft_recovery (ring, vmid); + +return dma_fence_is_signaled (fence); +} +/* * Debugfs info */ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +4
-1@@ -168,6 +168,8 @@ struct amdgpu_ring_funcs {/* priority functions */ void (*set_priority) (struct amdgpu_ring *ring,enum drm_sched_priority priority); +/* Try to soft recover the ring to make the fence signal */ +void (*soft_recovery)(struct amdgpu_ring *ring,unsigned vmid); };struct amdgpu_ring {@@ -260,6 +262,8 @@ void amdgpu_ring_fini(struct amdgpu_ring *ring);void amdgpu_ring_emit_reg_write_reg_wait_helper (struct amdgpu_ring *ring,uint32_t reg0,uint32_t val0,uint32_t reg1,uint32_t val1); +bool amdgpu_ring_soft_recovery (struct amdgpu_ring *ring,unsigned int vmid, +struct dma_fence *fence);static inline void amdgpu_ring_clear_ring (struct amdgpu_ring *ring) { --2.14.1
Quoted 4 lines
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): CADnq5_PU5TT4Q9oE+egAd2S4UgDENLu-Las_fnUZrNJUS=rmCw-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org
Re: [PATCH 2/5] drm/amdgpu: add ring soft recovery v2
Quoted 23 lines · Alex Deucher
Quoted 20 lines · Christian König
Instead of hammering hard on the GPU try a soft recovery first.
v2: reorder code a bit
Signed-off-by Christian König <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 6 ++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c | 24 ++++++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h | 4 ++++ 3 files changed, 34 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +0
-0@@ -33,6 +33,12 @@ static void amdgpu_job_timedout(struct drm_sched_job *s_job)struct amdgpu_ring *ring =to_amdgpu_ring (s_job->sched);struct amdgpu_job *job =to_amdgpu_job (s_job);
Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c static void amdgpu_job_timedout(struct drm_sched_job *s_job) · line 40
Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +5
-0+if (amdgpu_ring_soft_recovery (ring, job->vmid, s_job->s_fence->parent)) { +DRM_ERROR ("ring %s timeout, but soft recovered\n" , + s_job->sched->name); +return ; + }I think we should still bubble up the error to userspace even if we can recover. Data is lost when the wave is killed. We should treat it like a GPU reset.
Yes, please increment gpu_reset_counter, so that we are compliant with OpenGL. Being able to recover from infinite loops is great, but test suites also expect this to be properly reported to userspace via the per-context query.
Also please bump the deadline to 1 second. Even you if you kill all shaders, the IB can also contain CP DMA, which may take longer than 1 ms.
Marek
Marek
Quoted 67 lines
Alex
Quoted 64 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +1
-0+DRM_ERROR ("ring %s timeout, signaled seq=%u, emitted seq=%u\n" , job->base.sched->name,atomic_read (&ring->fence_drv.last_seq), ring->fence_drv.sync_seq);drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c +24
-0@@ -383,6 +383,30 @@ void amdgpu_ring_emit_reg_write_reg_wait_helper(struct amdgpu_ring *ring,amdgpu_ring_emit_reg_wait (ring, reg1, mask, mask); } +/** +* amdgpu_ring_soft_recovery - try to soft recover a ring lockup +* +* @ring: ring to try the recovery on +* @vmid: VMID we try to get going again +* @fence: timedout fence +* +* Tries to get a ring proceeding again when it is stuck. +*/ +bool amdgpu_ring_soft_recovery (struct amdgpu_ring *ring,unsigned int vmid, +struct dma_fence *fence) +{ +ktime_t deadline =ktime_add_us (ktime_get (),1000 ); + +if (!ring->funcs->soft_recovery) +return false; + +while (!dma_fence_is_signaled (fence) && +ktime_to_ns (ktime_sub (deadline,ktime_get ())) >0 ) + ring->funcs->soft_recovery (ring, vmid); + +return dma_fence_is_signaled (fence); +} +/* * Debugfs info */ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +4
-1@@ -168,6 +168,8 @@ struct amdgpu_ring_funcs {/* priority functions */ void (*set_priority) (struct amdgpu_ring *ring,enum drm_sched_priority priority); +/* Try to soft recover the ring to make the fence signal */ +void (*soft_recovery)(struct amdgpu_ring *ring,unsigned vmid); };struct amdgpu_ring {@@ -260,6 +262,8 @@ void amdgpu_ring_fini(struct amdgpu_ring *ring);void amdgpu_ring_emit_reg_write_reg_wait_helper (struct amdgpu_ring *ring,uint32_t reg0,uint32_t val0,uint32_t reg1,uint32_t val1); +bool amdgpu_ring_soft_recovery (struct amdgpu_ring *ring,unsigned int vmid, +struct dma_fence *fence);static inline void amdgpu_ring_clear_ring (struct amdgpu_ring *ring) { --2.14.1
Quoted 8 lines
Quoted 4 lines
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): CAAxE2A5L5UjsWVMSBGVe82GyVS3LAPtVh-5QBb5Ope5ZVEHKig-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org
Re: [PATCH 2/5] drm/amdgpu: add ring soft recovery v2
Am 22.08.2018 um 21:32 schrieb Marek Olšák:
Quoted 30 lines · Alex Deucher
Quoted 21 lines
Quoted 19 lines · Christian König
Instead of hammering hard on the GPU try a soft recovery first.
v2: reorder code a bit
Signed-off-by Christian König <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 6 ++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c | 24 ++++++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h | 4 ++++ 3 files changed, 34 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +0
-0@@ -33,6 +33,12 @@ static void amdgpu_job_timedout(struct drm_sched_job *s_job)struct amdgpu_ring *ring =to_amdgpu_ring (s_job->sched);struct amdgpu_job *job =to_amdgpu_job (s_job);Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c static void amdgpu_job_timedout(struct drm_sched_job *s_job) · line 40
Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +5
-0+if (amdgpu_ring_soft_recovery (ring, job->vmid, s_job->s_fence->parent)) { +DRM_ERROR ("ring %s timeout, but soft recovered\n" , + s_job->sched->name); +return ; + }I think we should still bubble up the error to userspace even if we can recover. Data is lost when the wave is killed. We should treat it like a GPU reset.
Quoted 4 lines
Yes, please increment
gpu_reset_counter, so that we are compliant with OpenGL. Being able to recover from infinite loops is great, but test suites also expect this to be properly reported to userspace via the per-context query.
Sure that shouldn't be a problem.
Quoted 3 lines
Also please bump the deadline to 1 second. Even you if you kill all shaders, the IB can also contain CP DMA, which may take longer than 1 ms.
Is there any way we can get a feedback from the SQ if the kill was successfully?
1 second is way to long, since in the case of a blocked MC we need to start up hard reset relative fast.
Regards,
Christian.
Quoted 79 lines
Marek
Marek
Quoted 74 lines
Alex
Quoted 64 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +1
-0+DRM_ERROR ("ring %s timeout, signaled seq=%u, emitted seq=%u\n" , job->base.sched->name,atomic_read (&ring->fence_drv.last_seq), ring->fence_drv.sync_seq);drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c +24
-0@@ -383,6 +383,30 @@ void amdgpu_ring_emit_reg_write_reg_wait_helper(struct amdgpu_ring *ring,amdgpu_ring_emit_reg_wait (ring, reg1, mask, mask); } +/** +* amdgpu_ring_soft_recovery - try to soft recover a ring lockup +* +* @ring: ring to try the recovery on +* @vmid: VMID we try to get going again +* @fence: timedout fence +* +* Tries to get a ring proceeding again when it is stuck. +*/ +bool amdgpu_ring_soft_recovery (struct amdgpu_ring *ring,unsigned int vmid, +struct dma_fence *fence) +{ +ktime_t deadline =ktime_add_us (ktime_get (),1000 ); + +if (!ring->funcs->soft_recovery) +return false; + +while (!dma_fence_is_signaled (fence) && +ktime_to_ns (ktime_sub (deadline,ktime_get ())) >0 ) + ring->funcs->soft_recovery (ring, vmid); + +return dma_fence_is_signaled (fence); +} +/* * Debugfs info */ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +4
-1@@ -168,6 +168,8 @@ struct amdgpu_ring_funcs {/* priority functions */ void (*set_priority) (struct amdgpu_ring *ring,enum drm_sched_priority priority); +/* Try to soft recover the ring to make the fence signal */ +void (*soft_recovery)(struct amdgpu_ring *ring,unsigned vmid); };struct amdgpu_ring {@@ -260,6 +262,8 @@ void amdgpu_ring_fini(struct amdgpu_ring *ring);void amdgpu_ring_emit_reg_write_reg_wait_helper (struct amdgpu_ring *ring,uint32_t reg0,uint32_t val0,uint32_t reg1,uint32_t val1); +bool amdgpu_ring_soft_recovery (struct amdgpu_ring *ring,unsigned int vmid, +struct dma_fence *fence);static inline void amdgpu_ring_clear_ring (struct amdgpu_ring *ring) { --2.14.1
Quoted 4 lines
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): b4b062be-2bc8-035b-e5db-2d17de698826-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org
Re: [PATCH 2/5] drm/amdgpu: add ring soft recovery v2
Quoted 37 lines · Christian König
Am 22.08.2018 um 21:32 schrieb Marek Olšák:
Quoted 30 lines · Alex Deucher
Quoted 21 lines
Quoted 19 lines · Christian König
Instead of hammering hard on the GPU try a soft recovery first.
v2: reorder code a bit
Signed-off-by Christian König <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 6 ++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c | 24 ++++++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h | 4 ++++ 3 files changed, 34 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +0
-0@@ -33,6 +33,12 @@ static void amdgpu_job_timedout(struct drm_sched_job *s_job)struct amdgpu_ring *ring =to_amdgpu_ring (s_job->sched);struct amdgpu_job *job =to_amdgpu_job (s_job);Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c static void amdgpu_job_timedout(struct drm_sched_job *s_job) · line 40
Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +5
-0+if (amdgpu_ring_soft_recovery (ring, job->vmid, s_job->s_fence->parent)) { +DRM_ERROR ("ring %s timeout, but soft recovered\n" , + s_job->sched->name); +return ; + }I think we should still bubble up the error to userspace even if we can recover. Data is lost when the wave is killed. We should treat it like a GPU reset.
Quoted 4 lines
Yes, please increment
gpu_reset_counter, so that we are compliant with OpenGL. Being able to recover from infinite loops is great, but test suites also expect this to be properly reported to userspace via the per-context query.
Quoted 8 lines
Sure that shouldn't be a problem.
Quoted 3 lines
Also please bump the deadline to 1 second. Even you if you kill all shaders, the IB can also contain CP DMA, which may take longer than 1 ms.
Is there any way we can get a feedback from the SQ if the kill was successfully?
I don't think so. The kill should be finished pretty quickly, but more waves with infinite loops may be waiting to be launched, so you still need to repeat the kill command. And we should ideally repeat it for 1 second.
The reason is that vertex shader waves take a lot of time to launch. A very very very large draw call can keep launching new waves for 1 second with the same infinite loop. You would have to soft-reset all VGTs to stop that.
Quoted 3 lines
1 second is way to long, since in the case of a blocked MC we need to start up hard reset relative fast.
10 seconds have already passed.
I think that some hangs from corrupted descriptors may still be recoverable just by killing waves.
Marek
Quoted 84 lines
Regards,
Christian.Quoted 79 lines
Marek
Marek
Quoted 74 lines
Alex
Quoted 64 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +1
-0+DRM_ERROR ("ring %s timeout, signaled seq=%u, emitted seq=%u\n" , job->base.sched->name,atomic_read (&ring->fence_drv.last_seq), ring->fence_drv.sync_seq);drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c +24
-0@@ -383,6 +383,30 @@ void amdgpu_ring_emit_reg_write_reg_wait_helper(struct amdgpu_ring *ring,amdgpu_ring_emit_reg_wait (ring, reg1, mask, mask); } +/** +* amdgpu_ring_soft_recovery - try to soft recover a ring lockup +* +* @ring: ring to try the recovery on +* @vmid: VMID we try to get going again +* @fence: timedout fence +* +* Tries to get a ring proceeding again when it is stuck. +*/ +bool amdgpu_ring_soft_recovery (struct amdgpu_ring *ring,unsigned int vmid, +struct dma_fence *fence) +{ +ktime_t deadline =ktime_add_us (ktime_get (),1000 ); + +if (!ring->funcs->soft_recovery) +return false; + +while (!dma_fence_is_signaled (fence) && +ktime_to_ns (ktime_sub (deadline,ktime_get ())) >0 ) + ring->funcs->soft_recovery (ring, vmid); + +return dma_fence_is_signaled (fence); +} +/* * Debugfs info */ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +4
-1@@ -168,6 +168,8 @@ struct amdgpu_ring_funcs {/* priority functions */ void (*set_priority) (struct amdgpu_ring *ring,enum drm_sched_priority priority); +/* Try to soft recover the ring to make the fence signal */ +void (*soft_recovery)(struct amdgpu_ring *ring,unsigned vmid); };struct amdgpu_ring {@@ -260,6 +262,8 @@ void amdgpu_ring_fini(struct amdgpu_ring *ring);void amdgpu_ring_emit_reg_write_reg_wait_helper (struct amdgpu_ring *ring,uint32_t reg0,uint32_t val0,uint32_t reg1,uint32_t val1); +bool amdgpu_ring_soft_recovery (struct amdgpu_ring *ring,unsigned int vmid, +struct dma_fence *fence);static inline void amdgpu_ring_clear_ring (struct amdgpu_ring *ring) { --2.14.1
Quoted 4 lines
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): CAAxE2A4TKbg-Z07u9nsr=k3ujzA7p8m-2NEiAFmHP8Lab_Zy4g-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org
Re: [PATCH 2/5] drm/amdgpu: add ring soft recovery v2
Am 23.08.2018 um 22:21 schrieb Marek Olšák:
Quoted 49 lines
Quoted 35 lines · Christian König
Am 22.08.2018 um 21:32 schrieb Marek Olšák:
Quoted 30 lines · Alex Deucher
Quoted 21 lines
Quoted 19 lines · Christian König
Instead of hammering hard on the GPU try a soft recovery first.
v2: reorder code a bit
Signed-off-by Christian König <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 6 ++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c | 24 ++++++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h | 4 ++++ 3 files changed, 34 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +0
-0@@ -33,6 +33,12 @@ static void amdgpu_job_timedout(struct drm_sched_job *s_job)struct amdgpu_ring *ring =to_amdgpu_ring (s_job->sched);struct amdgpu_job *job =to_amdgpu_job (s_job);Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c static void amdgpu_job_timedout(struct drm_sched_job *s_job) · line 40
Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +5
-0+if (amdgpu_ring_soft_recovery (ring, job->vmid, s_job->s_fence->parent)) { +DRM_ERROR ("ring %s timeout, but soft recovered\n" , + s_job->sched->name); +return ; + }I think we should still bubble up the error to userspace even if we can recover. Data is lost when the wave is killed. We should treat it like a GPU reset.
Quoted 4 lines
Yes, please increment
gpu_reset_counter, so that we are compliant with OpenGL. Being able to recover from infinite loops is great, but test suites also expect this to be properly reported to userspace via the per-context query.Quoted 7 lines
Sure that shouldn't be a problem.
Quoted 3 lines
Also please bump the deadline to 1 second. Even you if you kill all shaders, the IB can also contain CP DMA, which may take longer than 1 ms.
Is there any way we can get a feedback from the SQ if the kill was successfully?
I don't think so. The kill should be finished pretty quickly, but more waves with infinite loops may be waiting to be launched, so you still need to repeat the kill command. And we should ideally repeat it for 1 second.
Quoted 8 lines
The reason is that vertex shader waves take a lot of time to launch. A very very very large draw call can keep launching new waves for 1 second with the same infinite loop. You would have to soft-reset all VGTs to stop that.
Quoted 2 lines
1 second is way to long, since in the case of a blocked MC we need to start up hard reset relative fast.
10 seconds have already passed.
That is a good argument, but I still find 1 second way to long.
Keep in mind that we hammer on the SQ here to kill the waves as soon as they are launched.
Quoted 2 lines
I think that some hangs from corrupted descriptors may still be recoverable just by killing waves.
Agreed, especially since Vega10 the memory paths became much more robust to error handling.
Christian.
Quoted 84 lines
Marek
Quoted 81 lines
Regards,
Christian.Quoted 78 lines
Marek
Marek
Quoted 74 lines
Alex
Quoted 64 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +1
-0+DRM_ERROR ("ring %s timeout, signaled seq=%u, emitted seq=%u\n" , job->base.sched->name,atomic_read (&ring->fence_drv.last_seq), ring->fence_drv.sync_seq);drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c +24
-0@@ -383,6 +383,30 @@ void amdgpu_ring_emit_reg_write_reg_wait_helper(struct amdgpu_ring *ring,amdgpu_ring_emit_reg_wait (ring, reg1, mask, mask); } +/** +* amdgpu_ring_soft_recovery - try to soft recover a ring lockup +* +* @ring: ring to try the recovery on +* @vmid: VMID we try to get going again +* @fence: timedout fence +* +* Tries to get a ring proceeding again when it is stuck. +*/ +bool amdgpu_ring_soft_recovery (struct amdgpu_ring *ring,unsigned int vmid, +struct dma_fence *fence) +{ +ktime_t deadline =ktime_add_us (ktime_get (),1000 ); + +if (!ring->funcs->soft_recovery) +return false; + +while (!dma_fence_is_signaled (fence) && +ktime_to_ns (ktime_sub (deadline,ktime_get ())) >0 ) + ring->funcs->soft_recovery (ring, vmid); + +return dma_fence_is_signaled (fence); +} +/* * Debugfs info */ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +4
-1@@ -168,6 +168,8 @@ struct amdgpu_ring_funcs {/* priority functions */ void (*set_priority) (struct amdgpu_ring *ring,enum drm_sched_priority priority); +/* Try to soft recover the ring to make the fence signal */ +void (*soft_recovery)(struct amdgpu_ring *ring,unsigned vmid); };struct amdgpu_ring {@@ -260,6 +262,8 @@ void amdgpu_ring_fini(struct amdgpu_ring *ring);void amdgpu_ring_emit_reg_write_reg_wait_helper (struct amdgpu_ring *ring,uint32_t reg0,uint32_t val0,uint32_t reg1,uint32_t val1); +bool amdgpu_ring_soft_recovery (struct amdgpu_ring *ring,unsigned int vmid, +struct dma_fence *fence);static inline void amdgpu_ring_clear_ring (struct amdgpu_ring *ring) { --2.14.1
Quoted 4 lines
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Quoted 4 lines
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Re: [PATCH 2/5] drm/amdgpu: add ring soft recovery v2
In reply to a message not in the archiveQuoted 22 lines · Alex Deucher
Quoted 20 lines · Christian König
Instead of hammering hard on the GPU try a soft recovery first.
v2: reorder code a bit
Signed-off-by Christian König <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 6 ++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c | 24 ++++++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h | 4 ++++ 3 files changed, 34 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +0
-0@@ -33,6 +33,12 @@ static void amdgpu_job_timedout(struct drm_sched_job *s_job)struct amdgpu_ring *ring =to_amdgpu_ring (s_job->sched);struct amdgpu_job *job =to_amdgpu_job (s_job);
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c static void amdgpu_job_timedout(struct drm_sched_job *s_job) · line 40
Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +5
-0+if (amdgpu_ring_soft_recovery (ring, job->vmid, s_job->s_fence->parent)) { +DRM_ERROR ("ring %s timeout, but soft recovered\n" , + s_job->sched->name); +return ; + }I think we should still bubble up the error to userspace even if we can recover. Data is lost when the wave is killed. We should treat it like a GPU reset.
May I know what does the wavefront stand for? Why we can do the "light" recover than reset here?
Thanks,
Ray
Quoted 66 lines
Alex
Quoted 64 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +1
-0+DRM_ERROR ("ring %s timeout, signaled seq=%u, emitted seq=%u\n" , job->base.sched->name,atomic_read (&ring->fence_drv.last_seq), ring->fence_drv.sync_seq);drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c +24
-0@@ -383,6 +383,30 @@ void amdgpu_ring_emit_reg_write_reg_wait_helper(struct amdgpu_ring *ring,amdgpu_ring_emit_reg_wait (ring, reg1, mask, mask); } +/** +* amdgpu_ring_soft_recovery - try to soft recover a ring lockup +* +* @ring: ring to try the recovery on +* @vmid: VMID we try to get going again +* @fence: timedout fence +* +* Tries to get a ring proceeding again when it is stuck. +*/ +bool amdgpu_ring_soft_recovery (struct amdgpu_ring *ring,unsigned int vmid, +struct dma_fence *fence) +{ +ktime_t deadline =ktime_add_us (ktime_get (),1000 ); + +if (!ring->funcs->soft_recovery) +return false; + +while (!dma_fence_is_signaled (fence) && +ktime_to_ns (ktime_sub (deadline,ktime_get ())) >0 ) + ring->funcs->soft_recovery (ring, vmid); + +return dma_fence_is_signaled (fence); +} +/* * Debugfs info */ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +4
-1@@ -168,6 +168,8 @@ struct amdgpu_ring_funcs {/* priority functions */ void (*set_priority) (struct amdgpu_ring *ring,enum drm_sched_priority priority); +/* Try to soft recover the ring to make the fence signal */ +void (*soft_recovery)(struct amdgpu_ring *ring,unsigned vmid); };struct amdgpu_ring {@@ -260,6 +262,8 @@ void amdgpu_ring_fini(struct amdgpu_ring *ring);void amdgpu_ring_emit_reg_write_reg_wait_helper (struct amdgpu_ring *ring,uint32_t reg0,uint32_t val0,uint32_t reg1,uint32_t val1); +bool amdgpu_ring_soft_recovery (struct amdgpu_ring *ring,unsigned int vmid, +struct dma_fence *fence);static inline void amdgpu_ring_clear_ring (struct amdgpu_ring *ring) { --2.14.1
Quoted 8 lines
Quoted 4 lines
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Re: [PATCH 2/5] drm/amdgpu: add ring soft recovery v2
Am 23.08.2018 um 09:17 schrieb Huang Rui:
Quoted 31 lines · Alex Deucher
Quoted 21 lines
Quoted 19 lines · Christian König
Instead of hammering hard on the GPU try a soft recovery first.
v2: reorder code a bit
Signed-off-by Christian König <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 6 ++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c | 24 ++++++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h | 4 ++++ 3 files changed, 34 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +0
-0@@ -33,6 +33,12 @@ static void amdgpu_job_timedout(struct drm_sched_job *s_job)struct amdgpu_ring *ring =to_amdgpu_ring (s_job->sched);struct amdgpu_job *job =to_amdgpu_job (s_job);Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c static void amdgpu_job_timedout(struct drm_sched_job *s_job) · line 40
Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +5
-0+if (amdgpu_ring_soft_recovery (ring, job->vmid, s_job->s_fence->parent)) { +DRM_ERROR ("ring %s timeout, but soft recovered\n" , + s_job->sched->name); +return ; + }I think we should still bubble up the error to userspace even if we can recover. Data is lost when the wave is killed. We should treat it like a GPU reset.
Quoted 2 lines
May I know what does the wavefront stand for? Why we can do the "light" recover than reset here?
Wavefront means a running shader in the SQ.
Basically this only covers the case when the application sends down a shader with an endless loop to the hardware. Here we just kill the shader and try to continue.
When you run into a hang because of a corrupted resource descriptor you need usually need a full ASIC reset to get out of that again.
Regards,
Christian.
Quoted 78 lines
Thanks,
RayQuoted 74 lines
Alex
Quoted 64 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +1
-0+DRM_ERROR ("ring %s timeout, signaled seq=%u, emitted seq=%u\n" , job->base.sched->name,atomic_read (&ring->fence_drv.last_seq), ring->fence_drv.sync_seq);drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c +24
-0@@ -383,6 +383,30 @@ void amdgpu_ring_emit_reg_write_reg_wait_helper(struct amdgpu_ring *ring,amdgpu_ring_emit_reg_wait (ring, reg1, mask, mask); } +/** +* amdgpu_ring_soft_recovery - try to soft recover a ring lockup +* +* @ring: ring to try the recovery on +* @vmid: VMID we try to get going again +* @fence: timedout fence +* +* Tries to get a ring proceeding again when it is stuck. +*/ +bool amdgpu_ring_soft_recovery (struct amdgpu_ring *ring,unsigned int vmid, +struct dma_fence *fence) +{ +ktime_t deadline =ktime_add_us (ktime_get (),1000 ); + +if (!ring->funcs->soft_recovery) +return false; + +while (!dma_fence_is_signaled (fence) && +ktime_to_ns (ktime_sub (deadline,ktime_get ())) >0 ) + ring->funcs->soft_recovery (ring, vmid); + +return dma_fence_is_signaled (fence); +} +/* * Debugfs info */ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +4
-1@@ -168,6 +168,8 @@ struct amdgpu_ring_funcs {/* priority functions */ void (*set_priority) (struct amdgpu_ring *ring,enum drm_sched_priority priority); +/* Try to soft recover the ring to make the fence signal */ +void (*soft_recovery)(struct amdgpu_ring *ring,unsigned vmid); };struct amdgpu_ring {@@ -260,6 +262,8 @@ void amdgpu_ring_fini(struct amdgpu_ring *ring);void amdgpu_ring_emit_reg_write_reg_wait_helper (struct amdgpu_ring *ring,uint32_t reg0,uint32_t val0,uint32_t reg1,uint32_t val1); +bool amdgpu_ring_soft_recovery (struct amdgpu_ring *ring,unsigned int vmid, +struct dma_fence *fence);static inline void amdgpu_ring_clear_ring (struct amdgpu_ring *ring) { --2.14.1
Quoted 4 lines
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): 11e0af50-f589-b2e1-3431-8125fe2014ad-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org
Re: [PATCH 2/5] drm/amdgpu: add ring soft recovery v2
Quoted 35 lines · Christian König
Am 23.08.2018 um 09:17 schrieb Huang Rui:
Quoted 31 lines · Alex Deucher
Quoted 21 lines
Quoted 19 lines · Christian König
Instead of hammering hard on the GPU try a soft recovery first.
v2: reorder code a bit
Signed-off-by Christian König <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 6 ++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c | 24 ++++++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h | 4 ++++ 3 files changed, 34 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +0
-0@@ -33,6 +33,12 @@ static void amdgpu_job_timedout(struct drm_sched_job *s_job)struct amdgpu_ring *ring =to_amdgpu_ring (s_job->sched);struct amdgpu_job *job =to_amdgpu_job (s_job);Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c static void amdgpu_job_timedout(struct drm_sched_job *s_job) · line 40
Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +5
-0+if (amdgpu_ring_soft_recovery (ring, job->vmid, s_job->s_fence->parent)) { +DRM_ERROR ("ring %s timeout, but soft recovered\n" , + s_job->sched->name); +return ; + }I think we should still bubble up the error to userspace even if we can recover. Data is lost when the wave is killed. We should treat it like a GPU reset.
Quoted 2 lines
May I know what does the wavefront stand for? Why we can do the "light" recover than reset here?
Quoted 9 lines
Wavefront means a running shader in the SQ.
Basically this only covers the case when the application sends down a shader with an endless loop to the hardware. Here we just kill the shader and try to continue.
When you run into a hang because of a corrupted resource descriptor you need usually need a full ASIC reset to get out of that again.
Good to know this, thank you.
Series are Acked-by: Huang Rui <[email protected]>
Quoted 82 lines
Regards,
Christian.Quoted 78 lines
Thanks,
RayQuoted 74 lines
Alex
Quoted 64 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +1
-0+DRM_ERROR ("ring %s timeout, signaled seq=%u, emitted seq=%u\n" , job->base.sched->name,atomic_read (&ring->fence_drv.last_seq), ring->fence_drv.sync_seq);drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c +24
-0@@ -383,6 +383,30 @@ void amdgpu_ring_emit_reg_write_reg_wait_helper(struct amdgpu_ring *ring,amdgpu_ring_emit_reg_wait (ring, reg1, mask, mask); } +/** +* amdgpu_ring_soft_recovery - try to soft recover a ring lockup +* +* @ring: ring to try the recovery on +* @vmid: VMID we try to get going again +* @fence: timedout fence +* +* Tries to get a ring proceeding again when it is stuck. +*/ +bool amdgpu_ring_soft_recovery (struct amdgpu_ring *ring,unsigned int vmid, +struct dma_fence *fence) +{ +ktime_t deadline =ktime_add_us (ktime_get (),1000 ); + +if (!ring->funcs->soft_recovery) +return false; + +while (!dma_fence_is_signaled (fence) && +ktime_to_ns (ktime_sub (deadline,ktime_get ())) >0 ) + ring->funcs->soft_recovery (ring, vmid); + +return dma_fence_is_signaled (fence); +} +/* * Debugfs info */ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +4
-1@@ -168,6 +168,8 @@ struct amdgpu_ring_funcs {/* priority functions */ void (*set_priority) (struct amdgpu_ring *ring,enum drm_sched_priority priority); +/* Try to soft recover the ring to make the fence signal */ +void (*soft_recovery)(struct amdgpu_ring *ring,unsigned vmid); };struct amdgpu_ring {@@ -260,6 +262,8 @@ void amdgpu_ring_fini(struct amdgpu_ring *ring);void amdgpu_ring_emit_reg_write_reg_wait_helper (struct amdgpu_ring *ring,uint32_t reg0,uint32_t val0,uint32_t reg1,uint32_t val1); +bool amdgpu_ring_soft_recovery (struct amdgpu_ring *ring,unsigned int vmid, +struct dma_fence *fence);static inline void amdgpu_ring_clear_ring (struct amdgpu_ring *ring) { --2.14.1
Quoted 4 lines
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
PATCH 3/5 drm/amdgpu: implement soft_recovery for GFX7
In reply to a message not in the archiveTry to kill waves on the SQ.
--- drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c | 13 +++++++++++++ 1 file changed, 13 insertions(+)
drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c +13 -0
@@ -4212,6 +4212,18 @@ static void gfx_v7_0_ring_emit_gds_switch(struct amdgpu_ring *ring,amdgpu_ring_write (ring, (1 << (oa_size + oa_base)) - (1 << oa_base)); } +static void gfx_v7_0_ring_soft_recovery (struct amdgpu_ring *ring,unsigned vmid) +{ +struct amdgpu_device *adev = ring->adev; +uint32_t value =0 ; + + value =REG_SET_FIELD (value,SQ_CMD ,CMD ,0x03 ); + value =REG_SET_FIELD (value,SQ_CMD ,MODE ,0x01 ); + value =REG_SET_FIELD (value,SQ_CMD ,CHECK_VMID ,1 ); + value =REG_SET_FIELD (value,SQ_CMD ,VM_ID , vmid); +WREG32 (mmSQ_CMD, value); +} +static uint32_t wave_read_ind (struct amdgpu_device *adev,uint32_t simd,uint32_t wave,uint32_t address) { WREG32(mmSQ_IND_INDEX,
@@ -5088,6 +5100,7 @@ static const struct amdgpu_ring_funcs gfx_v7_0_ring_funcs_gfx = { .pad_ib = amdgpu_ring_generic_pad_ib, .emit_cntxcntl = gfx_v7_ring_emit_cntxcntl, .emit_wreg = gfx_v7_0_ring_emit_wreg, + .soft_recovery = gfx_v7_0_ring_soft_recovery, };static const struct amdgpu_ring_funcs gfx_v7_0_ring_funcs_compute = {
Signature · 6 lines
2.14.1 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
PATCH 4/5 drm/amdgpu: implement soft_recovery for GFX8 v2
In reply to a message not in the archiveTry to kill waves on the SQ.
v2: only for the GFX ring for now.
--- drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c | 13 +++++++++++++ 1 file changed, 13 insertions(+)
drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +13 -0
@@ -6714,6 +6714,18 @@ static void gfx_v8_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,amdgpu_ring_write (ring, val); } +static void gfx_v8_0_ring_soft_recovery (struct amdgpu_ring *ring,unsigned vmid) +{ +struct amdgpu_device *adev = ring->adev; +uint32_t value =0 ; + + value =REG_SET_FIELD (value,SQ_CMD ,CMD ,0x03 ); + value =REG_SET_FIELD (value,SQ_CMD ,MODE ,0x01 ); + value =REG_SET_FIELD (value,SQ_CMD ,CHECK_VMID ,1 ); + value =REG_SET_FIELD (value,SQ_CMD ,VM_ID , vmid); +WREG32 (mmSQ_CMD, value); +} +static void gfx_v8_0_set_gfx_eop_interrupt_state (struct amdgpu_device *adev,enum amdgpu_interrupt_state state) {
@@ -7171,6 +7183,7 @@ static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_gfx = { .init_cond_exec = gfx_v8_0_ring_emit_init_cond_exec, .patch_cond_exec = gfx_v8_0_ring_emit_patch_cond_exec, .emit_wreg = gfx_v8_0_ring_emit_wreg, + .soft_recovery = gfx_v8_0_ring_soft_recovery, };static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_compute = {
Signature · 6 lines
2.14.1 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
PATCH 5/5 drm/amdgpu: implement soft_recovery for GFX9
In reply to a message not in the archiveTry to kill waves on the SQ.
--- drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c | 13 +++++++++++++ 1 file changed, 13 insertions(+)
drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c +13 -0
@@ -4421,6 +4421,18 @@ static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring, ref, mask); } +static void gfx_v9_0_ring_soft_recovery (struct amdgpu_ring *ring,unsigned vmid) +{ +struct amdgpu_device *adev = ring->adev; +uint32_t value =0 ; + + value =REG_SET_FIELD (value,SQ_CMD ,CMD ,0x03 ); + value =REG_SET_FIELD (value,SQ_CMD ,MODE ,0x01 ); + value =REG_SET_FIELD (value,SQ_CMD ,CHECK_VMID ,1 ); + value =REG_SET_FIELD (value,SQ_CMD ,VM_ID , vmid); +WREG32 (mmSQ_CMD, value); +} +static void gfx_v9_0_set_gfx_eop_interrupt_state (struct amdgpu_device *adev,enum amdgpu_interrupt_state state) {
@@ -4743,6 +4755,7 @@ static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = { .emit_wreg = gfx_v9_0_ring_emit_wreg, .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, + .soft_recovery = gfx_v9_0_ring_soft_recovery, };static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = {
Signature · 6 lines
2.14.1 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Series is Acked-by: Andrey Grodzovsky <[email protected]>
Andrey
Quoted 153 lines · Christian König
Check if we should call the function instead of providing the forced flag.
Signed-off-by Christian König <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 3 ++- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 38 ++++++++++++++++++++---------- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c | 4 ++-- drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 3 ++- drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c | 4 ++-- drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c | 3 ++- 7 files changed, 36 insertions(+), 21 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu.h +2
-1@@ -1158,8 +1158,9 @@ int emu_soc_asic_init(struct amdgpu_device *adev);#define amdgpu_asic_need_full_reset (adev) (adev)->asic_funcs->need_full_reset((adev))/* Common functions */ +bool amdgpu_device_should_recover_gpu (struct amdgpu_device *adev);int amdgpu_device_gpu_recover (struct amdgpu_device *adev, -struct amdgpu_job * job,bool force); +struct amdgpu_job * job);void amdgpu_device_pci_config_reset (struct amdgpu_device *adev);bool amdgpu_device_need_post (struct amdgpu_device *adev);drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +25
-13@@ -3244,32 +3244,44 @@ static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,return r; } +/** +* amdgpu_device_should_recover_gpu - check if we should try GPU recovery +* +* @adev: amdgpu device pointer +* +* Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover +* a hung GPU. +*/ +bool amdgpu_device_should_recover_gpu (struct amdgpu_device *adev) +{ +if (!amdgpu_device_ip_check_soft_reset (adev)) { +DRM_INFO ("Timeout, but no hardware hang detected.\n" ); +return false; + } + +if (amdgpu_gpu_recovery ==0 || (amdgpu_gpu_recovery ==-1 && + !amdgpu_sriov_vf (adev))) { +DRM_INFO ("GPU recovery disabled.\n" ); +return false; + } + +return true; +} +/** * amdgpu_device_gpu_recover - reset the asic and recover scheduler * * @adev: amdgpu device pointer * @job: which job trigger hang -* @force: forces reset regardless of amdgpu_gpu_recovery * * Attempt to reset the GPU if it has hung (all asics). * Returns 0 for success or an error on failure. */ int amdgpu_device_gpu_recover (struct amdgpu_device *adev, -struct amdgpu_job *job,bool force) +struct amdgpu_job *job) {int i, r, resched; -if (!force && !amdgpu_device_ip_check_soft_reset (adev)) { -DRM_INFO ("No hardware hang detected. Did some blocks stall?\n" ); -return 0 ; - } - -if (!force && (amdgpu_gpu_recovery ==0 || - (amdgpu_gpu_recovery ==-1 && !amdgpu_sriov_vf (adev)))) { -DRM_INFO ("GPU recovery disabled.\n" ); -return 0 ; - } -dev_info (adev->dev,"GPU reset begin!\n" );mutex_lock (&adev->lock_reset);drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +1
-1@@ -702,7 +702,7 @@ static int amdgpu_debugfs_gpu_recover(struct seq_file *m, void *data)struct amdgpu_device *adev = dev->dev_private;seq_printf (m,"gpu recover\n" ); -amdgpu_device_gpu_recover (adev,NULL , true); +amdgpu_device_gpu_recover (adev,NULL );return 0 ; }drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c +2
-2@@ -105,8 +105,8 @@ static void amdgpu_irq_reset_work_func(struct work_struct *work) struct amdgpu_device *adev = container_of(work, struct amdgpu_device, reset_work); -if (!amdgpu_sriov_vf (adev)) -amdgpu_device_gpu_recover (adev,NULL , false); +if (!amdgpu_sriov_vf (adev) &&amdgpu_device_should_recover_gpu (adev)) +amdgpu_device_gpu_recover (adev,NULL ); }/** drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +2
-1@@ -37,7 +37,8 @@ static void amdgpu_job_timedout(struct drm_sched_job *s_job) job->base.sched->name,atomic_read (&ring->fence_drv.last_seq), ring->fence_drv.sync_seq); -amdgpu_device_gpu_recover (ring->adev, job, false); +if (amdgpu_device_should_recover_gpu (ring->adev)) +amdgpu_device_gpu_recover (ring->adev, job); } int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs,drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +2
-2@@ -266,8 +266,8 @@ static void xgpu_ai_mailbox_flr_work(struct work_struct *work) }/* Trigger recovery for world switch failure if no TDR */ -if (amdgpu_lockup_timeout ==0 ) -amdgpu_device_gpu_recover (adev,NULL , true); +if (amdgpu_device_should_recover_gpu (adev)) +amdgpu_device_gpu_recover (adev,NULL ); } static int xgpu_ai_set_mailbox_rcv_irq(struct amdgpu_device *adev,drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c +0
-0
drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c static void xgpu_vi_mailbox_flr_work(struct work_struct *work) · line 528
Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c +2
-1@@ -521,7 +521,8 @@ static void xgpu_vi_mailbox_flr_work(struct work_struct *work) }/* Trigger recovery due to world switch failure */ -amdgpu_device_gpu_recover (adev,NULL , false); +if (amdgpu_device_should_recover_gpu (adev)) +amdgpu_device_gpu_recover (adev,NULL ); } static int xgpu_vi_set_mailbox_rcv_irq(struct amdgpu_device *adev,
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx