drm/amd/amdgpu: customized the reset to skip soft recovery
Subsystems (2)
PATCH 1/2 drm/amd/amdgpu: customized the reset to skip soft recovery
customized the reset to skip soft recovery
--- drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +1 -1
@@ -548,7 +548,7 @@ module_param_named(compute_multipipe, amdgpu_compute_multipipe, int, 0444);* DOC: gpu_recovery (int) * Set to enable GPU recovery mechanism (1 = enable, 0 = disable). The default is -1 (auto, disabled except SRIOV). */ -MODULE_PARM_DESC (gpu_recovery,"Enable GPU recovery mechanism, (1 = enable, 0 = disable, -1 = auto)" ); +MODULE_PARM_DESC (gpu_recovery,"Enable GPU recovery mechanism, (1 = enable, 0 = disable, -1 = auto, 2 = customized)" );module_param_named (gpu_recovery, amdgpu_gpu_recovery, int,0444 );/**
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +1 -1
@@ -51,7 +51,7 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job) adev->job_hang = true; -if (amdgpu_gpu_recovery && +if (amdgpu_gpu_recovery && amdgpu_gpu_recovery !=2 &&amdgpu_ring_soft_recovery (ring, job->vmid, s_job->s_fence->parent)) {DRM_ERROR ("ring %s timeout, but soft recovered\n" , s_job->sched->name);
Signature · 1 line
2.34.1
PATCH 2/2 drm/amd/amdgpu: use the default reset for ras recovery
use the default reset for ras recovery
--- drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c | 7 +++++++ 1 file changed, 7 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +7 -0
@@ -2437,6 +2437,7 @@ static void amdgpu_ras_do_recovery(struct work_struct *work)struct amdgpu_device *adev = ras->adev;struct list_head device_list, *device_list_handle =NULL ;struct amdgpu_hive_info *hive =amdgpu_get_xgmi_hive (adev); +int save_reset_method = amdgpu_reset_method;if (hive) {atomic_set (&hive->ras_recovery,1 );
@@ -2501,7 +2502,13 @@ static void amdgpu_ras_do_recovery(struct work_struct *work) } } +if (amdgpu_gpu_recovery ==2 ) + amdgpu_reset_method =-1 ; +amdgpu_device_gpu_recover (ras->adev,NULL , &reset_context); + +if (amdgpu_gpu_recovery ==2 ) + amdgpu_reset_method = save_reset_method; }atomic_set (&ras->in_recovery,0 );if (hive) {
Signature · 1 line
2.34.1
RE: [PATCH 2/2] drm/amd/amdgpu: use the default reset for ras recovery
[AMD Official Use Only - General]
+@Zhang, GuoQing (Sam)
-----Original Message-----
From: Kenneth Feng <[email protected]>
Sent: Monday, April 29, 2024 3:32 PM
To: [email protected]
Cc: Zhang, Owen(SRDC) <[email protected]>; Aldabagh, Maad <[email protected]>; Ma, Qing (Mark) <[email protected]>; Feng, Kenneth <[email protected]>
Subject: [PATCH 2/2] drm/amd/amdgpu: use the default reset for ras recovery
use the default reset for ras recovery
--- drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c | 7 +++++++ 1 file changed, 7 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +7 -0
@@ -2437,6 +2437,7 @@ static void amdgpu_ras_do_recovery(struct work_struct *work)struct amdgpu_device *adev = ras->adev;struct list_head device_list, *device_list_handle =NULL ;struct amdgpu_hive_info *hive =amdgpu_get_xgmi_hive (adev); +int save_reset_method = amdgpu_reset_method;if (hive) {atomic_set (&hive->ras_recovery,1 );
@@ -2501,7 +2502,13 @@ static void amdgpu_ras_do_recovery(struct work_struct *work) } } +if (amdgpu_gpu_recovery ==2 ) + amdgpu_reset_method =-1 ; +amdgpu_device_gpu_recover (ras->adev,NULL , &reset_context); + +if (amdgpu_gpu_recovery ==2 ) + amdgpu_reset_method = save_reset_method; }atomic_set (&ras->in_recovery,0 );if (hive) {
--
2.34.1
Re: [PATCH 2/2] drm/amd/amdgpu: use the default reset for ras recovery
[AMD Official Use Only - General]
Hi @Deucher, Alexander<mailto:[email protected]> and @Koenig, Christian<mailto:[email protected]>
Could you help review this patch?
Without this patch, when customer set reset_method=3 modprobe param to use mode2 reset, ras recovery will also use mode2 reset and skip mode1 reset. When ECC error happens, GPU can’t be recovered with mode2 reset and mode1 reset is skipped, this will cause GPU reset failure.
This patch is to always use mode1 reset for ras recovery (ECC error) when setting reset_method=3.
Thanks
Sam
From: Feng, Kenneth <[email protected]>
Date: Monday, April 29, 2024 at 16:15
To: Feng, Kenneth <[email protected]>, [email protected] <[email protected]>, Zhang, GuoQing (Sam) <[email protected]>
Cc: Zhang, Owen(SRDC) <[email protected]>, Aldabagh, Maad <[email protected]>, Ma, Qing (Mark) <[email protected]>
Subject: RE: [PATCH 2/2] drm/amd/amdgpu: use the default reset for ras recovery
[AMD Official Use Only - General]
+@Zhang, GuoQing (Sam)
-----Original Message-----
From: Kenneth Feng <[email protected]>
Sent: Monday, April 29, 2024 3:32 PM
To: [email protected]
Cc: Zhang, Owen(SRDC) <[email protected]>; Aldabagh, Maad <[email protected]>; Ma, Qing (Mark) <[email protected]>; Feng, Kenneth <[email protected]>
Subject: [PATCH 2/2] drm/amd/amdgpu: use the default reset for ras recovery
use the default reset for ras recovery
--- drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c | 7 +++++++ 1 file changed, 7 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +7 -0
@@ -2437,6 +2437,7 @@ static void amdgpu_ras_do_recovery(struct work_struct *work)struct amdgpu_device *adev = ras->adev;struct list_head device_list, *device_list_handle =NULL ;struct amdgpu_hive_info *hive =amdgpu_get_xgmi_hive (adev); +int save_reset_method = amdgpu_reset_method;if (hive) {atomic_set (&hive->ras_recovery,1 );
@@ -2501,7 +2502,13 @@ static void amdgpu_ras_do_recovery(struct work_struct *work) } } +if (amdgpu_gpu_recovery ==2 ) + amdgpu_reset_method =-1 ; +amdgpu_device_gpu_recover (ras->adev,NULL , &reset_context); + +if (amdgpu_gpu_recovery ==2 ) + amdgpu_reset_method = save_reset_method; }atomic_set (&ras->in_recovery,0 );if (hive) {
--
2.34.1
Re: [PATCH 2/2] drm/amd/amdgpu: use the default reset for ras recovery
In reply to Kenneth FengQuoted 21 lines · Kenneth Feng
use the default reset for ras recovery
Signed-off-by Kenneth Feng <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c | 7 +++++++ 1 file changed, 7 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +1
-0@@ -2437,6 +2437,7 @@ static void amdgpu_ras_do_recovery(struct work_struct *work)struct amdgpu_device *adev = ras->adev;struct list_head device_list, *device_list_handle =NULL ;struct amdgpu_hive_info *hive =amdgpu_get_xgmi_hive (adev); +int save_reset_method = amdgpu_reset_method;if (hive) {atomic_set (&hive->ras_recovery,1 );
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c static void amdgpu_ras_do_recovery(struct work_struct *work) · line 2511
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +6
-0@@ -2501,7 +2502,13 @@ static void amdgpu_ras_do_recovery(struct work_struct *work) } } +if (amdgpu_gpu_recovery ==2 ) + amdgpu_reset_method =-1 ; +amdgpu_device_gpu_recover (ras->adev,NULL , &reset_context); + +if (amdgpu_gpu_recovery ==2 ) + amdgpu_reset_method = save_reset_method;
This is racy. amdgpu_gpu_recovery is a global variable and will be referenced by all of the AMD GPUs in the system that are using amdgpu. To handle this properly, we should store the selected reset method in the adev structure and set that based on the module parameter at driver bind time. Then at runtime if we need to change the reset method, we can change the device specific one in adev. Maybe it would be better to have two variable in adev. E.g., default_reset_method and override_reset_method. In cases where have to use the default method, we can use that. In other cases, we can use the override method.
Alex
Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +0
-1}atomic_set (&ras->in_recovery,0 );if (hive) { --2.34.1
[AMD Official Use Only - General]
+@Zhang, GuoQing (Sam)
-----Original Message-----
From: Kenneth Feng <[email protected]>
Sent: Monday, April 29, 2024 3:32 PM
To: [email protected]
Cc: Zhang, Owen(SRDC) <[email protected]>; Aldabagh, Maad <[email protected]>; Ma, Qing (Mark) <[email protected]>; Feng, Kenneth <[email protected]>
Subject: [PATCH 1/2] drm/amd/amdgpu: customized the reset to skip soft recovery
customized the reset to skip soft recovery
--- drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +2 -1
@@ -548,7 +548,7 @@ module_param_named(compute_multipipe, amdgpu_compute_multipipe, int, 0444);* DOC: gpu_recovery (int) * Set to enable GPU recovery mechanism (1 = enable, 0 = disable). The default is -1 (auto, disabled except SRIOV). */ -MODULE_PARM_DESC (gpu_recovery,"Enable GPU recovery mechanism, (1 = enable, 0 = disable, -1 = auto)" ); +MODULE_PARM_DESC(gpu_recovery, "Enable GPU recovery mechanism, (1 = +enable, 0 = disable, -1 = auto, 2 = customized)");module_param_named (gpu_recovery, amdgpu_gpu_recovery, int,0444 );
/**
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +1 -1
@@ -51,7 +51,7 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job) adev->job_hang = true; -if (amdgpu_gpu_recovery && +if (amdgpu_gpu_recovery && amdgpu_gpu_recovery !=2 &&amdgpu_ring_soft_recovery (ring, job->vmid, s_job->s_fence->parent)) {DRM_ERROR ("ring %s timeout, but soft recovered\n" , s_job->sched->name);
--
2.34.1
[AMD Official Use Only - General]
Hi @Deucher, Alexander<mailto:[email protected]> and @Koenig, Christian<mailto:[email protected]>
Could you help review this patch? Customer needs this toggle to turn off software recovery and use mode2 reset directly. This toggle is also needed when we test mode2 reset using quark tool.
Thanks
Sam
From: Feng, Kenneth <[email protected]>
Date: Monday, April 29, 2024 at 16:14
To: Feng, Kenneth <[email protected]>, [email protected] <[email protected]>, Zhang, GuoQing (Sam) <[email protected]>
Cc: Zhang, Owen(SRDC) <[email protected]>, Aldabagh, Maad <[email protected]>, Ma, Qing (Mark) <[email protected]>
Subject: RE: [PATCH 1/2] drm/amd/amdgpu: customized the reset to skip soft recovery
[AMD Official Use Only - General]
+@Zhang, GuoQing (Sam)
-----Original Message-----
From: Kenneth Feng <[email protected]>
Sent: Monday, April 29, 2024 3:32 PM
To: [email protected]
Cc: Zhang, Owen(SRDC) <[email protected]>; Aldabagh, Maad <[email protected]>; Ma, Qing (Mark) <[email protected]>; Feng, Kenneth <[email protected]>
Subject: [PATCH 1/2] drm/amd/amdgpu: customized the reset to skip soft recovery
customized the reset to skip soft recovery
--- drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +2 -1
@@ -548,7 +548,7 @@ module_param_named(compute_multipipe, amdgpu_compute_multipipe, int, 0444);* DOC: gpu_recovery (int) * Set to enable GPU recovery mechanism (1 = enable, 0 = disable). The default is -1 (auto, disabled except SRIOV). */ -MODULE_PARM_DESC (gpu_recovery,"Enable GPU recovery mechanism, (1 = enable, 0 = disable, -1 = auto)" ); +MODULE_PARM_DESC(gpu_recovery, "Enable GPU recovery mechanism, (1 = +enable, 0 = disable, -1 = auto, 2 = customized)");module_param_named (gpu_recovery, amdgpu_gpu_recovery, int,0444 );
/**
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +1 -1
@@ -51,7 +51,7 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job) adev->job_hang = true; -if (amdgpu_gpu_recovery && +if (amdgpu_gpu_recovery && amdgpu_gpu_recovery !=2 &&amdgpu_ring_soft_recovery (ring, job->vmid, s_job->s_fence->parent)) {DRM_ERROR ("ring %s timeout, but soft recovered\n" , s_job->sched->name);
--
2.34.1