drm/amdkfd: restore_process_worker race with GPU reset
Subsystems (3)
PATCH 1/1 drm/amdkfd: restore_process_worker race with GPU reset
If GPU reset kick in while KFD restore_process_worker running, this may causes different issues, for example below rcu stall warning, because restore work may move BOs and evict queues under VRAM pressure.
Fix this race by taking adev reset_domain read semaphore to prevent GPU reset in restore_process_worker, the reset read semaphore can be taken recursively if adev have multiple partitions.
Then there is live locking issue if CP hangs while restore_process_worker runs, then GPU reset wait for semaphore to start and restore_process_worker cannot finish to release semaphore. We need signal eviction fence to solve the live locking if evict queue return -ETIMEOUT (for MES path) or -ETIME (for HWS path) because CP hangs,
amdgpu 0000:af:00.0: amdgpu: GPU reset(21) succeeded!
rcu: INFO: rcu_sched self-detected stall on CPU Workqueue: kfd_restore_wq restore_process_worker [amdgpu] Call Trace: update_process_times+0x94/0xd0 RIP: 0010:amdgpu_vm_handle_moved+0x9a/0x210 [amdgpu] amdgpu_amdkfd_gpuvm_restore_process_bos+0x3d6/0x7d0 [amdgpu] restore_process_helper+0x27/0x80 [amdgpu]
--- drivers/gpu/drm/amd/amdkfd/kfd_process.c | 56 +++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-)
drivers/gpu/drm/amd/amdkfd/kfd_process.c +55 -1
@@ -35,6 +35,7 @@#include <linux/pm_runtime.h> #include "amdgpu_amdkfd.h" #include "amdgpu.h" +#include "amdgpu_reset.h" struct mm_struct ;
@@ -1972,8 +1973,14 @@ static void evict_process_worker(struct work_struct *work)kfd_process_restore_queues (p);pr_debug ("Finished evicting pasid 0x%x\n" , p->pasid); - }else + }else if (ret == -ETIMEDOUT || ret == -ETIME ) { +/* If CP hangs, signal the eviction fence, then restore_bo_worker +* can finish to up_read GPU reset semaphore to start GPU reset. +*/ +signal_eviction_fence (p); + }else {pr_err ("Failed to evict queues of pasid 0x%x\n" , p->pasid); + } }static int restore_process_helper (struct kfd_process *p)
@@ -1997,6 +2004,45 @@ static int restore_process_helper(struct kfd_process *p)return ret; } +/* +* kfd_hold_devices_reset_semaphore +* +* return: +* true : hold reset domain semaphore to prevent device reset +* false: one of the device is resetting or already reset +* +*/ +static bool kfd_hold_devices_reset_semaphore (struct kfd_process *p) +{ +struct amdgpu_device *adev; +int i; + +for (i =0 ; i < p->n_pdds; i++) { + adev = p->pdds[i]->dev->adev; +if (!down_read_trylock (&adev->reset_domain->sem)) + goto out_upread; + } +return true; + +out_upread: +while (i--) { + adev = p->pdds[i]->dev->adev; +up_read (&adev->reset_domain->sem); + } +return false; +} + +static void kfd_unhold_devices_reset_semaphore (struct kfd_process *p) +{ +struct amdgpu_device *adev; +int i; + +for (i =0 ; i < p->n_pdds; i++) { + adev = p->pdds[i]->dev->adev; +up_read (&adev->reset_domain->sem); + } +} +static void restore_process_worker (struct work_struct *work) {struct delayed_work *dwork;
@@ -2009,6 +2055,12 @@ static void restore_process_worker(struct work_struct *work)* lifetime of this thread, kfd_process p will be valid */ p = container_of(dwork, struct kfd_process, restore_work); + +if (!kfd_hold_devices_reset_semaphore (p)) { +pr_debug ("GPU resetting, restore bo and queue skipped\n" ); +return ; + } +pr_debug ("Started restoring pasid 0x%x\n" , p->pasid);/* Setting last_restore_timestamp before successful restoration.
@@ -2031,6 +2083,8 @@ static void restore_process_worker(struct work_struct *work) msecs_to_jiffies(PROCESS_RESTORE_TIME_MS)))kfd_process_restore_queues (p); } + +kfd_unhold_devices_reset_semaphore (p); }void kfd_suspend_all_processes (void )
Signature · 1 line
2.43.2
Felix Kuehling
maintainerQuoted 7 lines · Philip Yang
If GPU reset kick in while KFD
restore_process_workerrunning, this may causes different issues, for example below rcu stall warning, because restore work may move BOs and evict queues under VRAM pressure.Fix this race by taking adev
reset_domainread semaphore to prevent GPU reset inrestore_process_worker, the reset read semaphore can be taken recursively if adev have multiple partitions.
Are you sure that an rw_sem can be read-locked recursively in the same thread. I can't find any evidence that this is true.
Regards,
Felix
Quoted 118 lines
Then there is live locking issue if CP hangs while
restore_process_workerruns, then GPU reset wait for semaphore to start andrestore_process_workercannot finish to release semaphore. We need signal eviction fence to solve the live locking if evict queue return-ETIMEOUT(for MES path) or-ETIME(for HWS path) because CP hangs,amdgpu 0000:af:00.0: amdgpu: GPU reset(21) succeeded!
rcu: INFO: rcu_sched self-detected stall on CPU Workqueue: kfd_restore_wq restore_process_worker [amdgpu] Call Trace: update_process_times+0x94/0xd0 RIP: 0010:amdgpu_vm_handle_moved+0x9a/0x210 [amdgpu] amdgpu_amdkfd_gpuvm_restore_process_bos+0x3d6/0x7d0 [amdgpu] restore_process_helper+0x27/0x80 [amdgpu]Signed-off-by Philip Yang <[email protected]>--- drivers/gpu/drm/amd/amdkfd/kfd_process.c | 56 +++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-)drivers/gpu/drm/amd/amdkfd/kfd_process.c +55
-1@@ -35,6 +35,7 @@#include <linux/pm_runtime.h> #include "amdgpu_amdkfd.h" #include "amdgpu.h" +#include "amdgpu_reset.h" struct mm_struct ;@@ -1972,8 +1973,14 @@ static void evict_process_worker(struct work_struct *work)kfd_process_restore_queues (p);pr_debug ("Finished evicting pasid 0x%x\n" , p->pasid); - }else + }else if (ret == -ETIMEDOUT || ret == -ETIME ) { +/* If CP hangs, signal the eviction fence, then restore_bo_worker +* can finish to up_read GPU reset semaphore to start GPU reset. +*/ +signal_eviction_fence (p); + }else {pr_err ("Failed to evict queues of pasid 0x%x\n" , p->pasid); + } }static int restore_process_helper (struct kfd_process *p)@@ -1997,6 +2004,45 @@ static int restore_process_helper(struct kfd_process *p)return ret; } +/* +* kfd_hold_devices_reset_semaphore +* +* return: +* true : hold reset domain semaphore to prevent device reset +* false: one of the device is resetting or already reset +* +*/ +static bool kfd_hold_devices_reset_semaphore (struct kfd_process *p) +{ +struct amdgpu_device *adev; +int i; + +for (i =0 ; i < p->n_pdds; i++) { + adev = p->pdds[i]->dev->adev; +if (!down_read_trylock (&adev->reset_domain->sem)) + goto out_upread; + } +return true; + +out_upread: +while (i--) { + adev = p->pdds[i]->dev->adev; +up_read (&adev->reset_domain->sem); + } +return false; +} + +static void kfd_unhold_devices_reset_semaphore (struct kfd_process *p) +{ +struct amdgpu_device *adev; +int i; + +for (i =0 ; i < p->n_pdds; i++) { + adev = p->pdds[i]->dev->adev; +up_read (&adev->reset_domain->sem); + } +} +static void restore_process_worker (struct work_struct *work) {struct delayed_work *dwork;@@ -2009,6 +2055,12 @@ static void restore_process_worker(struct work_struct *work)* lifetime of this thread, kfd_process p will be valid */ p = container_of(dwork, struct kfd_process, restore_work); + +if (!kfd_hold_devices_reset_semaphore (p)) { +pr_debug ("GPU resetting, restore bo and queue skipped\n" ); +return ; + } +pr_debug ("Started restoring pasid 0x%x\n" , p->pasid);/* Setting last_restore_timestamp before successful restoration. @@ -2031,6 +2083,8 @@ static void restore_process_worker(struct work_struct *work) msecs_to_jiffies(PROCESS_RESTORE_TIME_MS)))kfd_process_restore_queues (p); } + +kfd_unhold_devices_reset_semaphore (p); }void kfd_suspend_all_processes (void )
On 2024-08-28 18:01, Felix Kuehling wrote:
On 2024-08-23 15:49, Philip Yang wrote:
If GPU reset kick in while KFD restore_process_worker running, this may causes different issues, for example below rcu stall warning, because restore work may move BOs and evict queues under VRAM pressure.
Fix this race by taking adev reset_domain read semaphore to prevent GPU reset in restore_process_worker, the reset read semaphore can be taken recursively if adev have multiple partitions.
Are you sure that an rw_sem can be read-locked recursively in the same thread. I can't find any evidence that this is true.
yes, down_read_trylock(&adev->reset_domain->sem) still return 1 for successful inside down_read_trylock(&adev->reset_domain->sem).
This works fine in many path now, for example execute_queues_cpsch down read lock reset_domain->sem -> unmap_queues_cpsch down read lock reset_domain->sem again.
Regards,
Philip
Regards,
Felix
Then there is live locking issue if CP hangs while restore_process_worker runs, then GPU reset wait for semaphore to start and restore_process_worker cannot finish to release semaphore. We need signal eviction fence to solve the live locking if evict queue return -ETIMEOUT (for MES path) or -ETIME (for HWS path) because CP hangs,
amdgpu 0000:af:00.0: amdgpu: GPU reset(21) succeeded!
rcu: INFO: rcu_sched self-detected stall on CPU Workqueue: kfd_restore_wq restore_process_worker [amdgpu] Call Trace: update_process_times+0x94/0xd0 RIP: 0010:amdgpu_vm_handle_moved+0x9a/0x210 [amdgpu] amdgpu_amdkfd_gpuvm_restore_process_bos+0x3d6/0x7d0 [amdgpu] restore_process_helper+0x27/0x80 [amdgpu]
---
drivers/gpu/drm/amd/amdkfd/kfd_process.c | 56 +++++++++++++++++++++++-
1 file changed, 55 insertions(+), 1 deletion(-)
drivers/gpu/drm/amd/amdkfd/kfd_process.c +0 -0
@@ -35,6 +35,7 @@
#include <linux/pm_runtime.h>
#include "amdgpu_amdkfd.h"
#include "amdgpu.h"
+#include "amdgpu_reset.h"
struct mm_struct;
@@ -1972,8 +1973,14 @@ static void evict_process_worker(struct work_struct *work) kfd_process_restore_queues(p);
pr_debug("Finished evicting pasid 0x%x\n", p->pasid);
- } else
+ } else if (ret == -ETIMEDOUT || ret == -ETIME) {
+ /* If CP hangs, signal the eviction fence, then restore_bo_worker + * can finish to up_read GPU reset semaphore to start GPU reset. + */
+ signal_eviction_fence(p);
+ } else {
pr_err("Failed to evict queues of pasid 0x%x\n", p->pasid); + }
}
static int restore_process_helper(struct kfd_process *p)
@@ -1997,6 +2004,45 @@ static int restore_process_helper(struct kfd_process *p) return ret;
}
+/*
+ * kfd_hold_devices_reset_semaphore
+ *
+ * return:
+ * true : hold reset domain semaphore to prevent device reset
+ * false: one of the device is resetting or already reset
+ *
+ */
+static bool kfd_hold_devices_reset_semaphore(struct kfd_process *p) +{
+ struct amdgpu_device *adev;
+ int i;
+
+ for (i = 0; i < p->n_pdds; i++) {
+ adev = p->pdds[i]->dev->adev;
+ if (!down_read_trylock(&adev->reset_domain->sem))
+ goto out_upread;
+ }
+ return true;
+
+out_upread:
+ while (i--) {
+ adev = p->pdds[i]->dev->adev;
+ up_read(&adev->reset_domain->sem);
+ }
+ return false;
+}
+
+static void kfd_unhold_devices_reset_semaphore(struct kfd_process *p) +{
+ struct amdgpu_device *adev;
+ int i;
+
+ for (i = 0; i < p->n_pdds; i++) {
+ adev = p->pdds[i]->dev->adev;
+ up_read(&adev->reset_domain->sem);
+ }
+}
+
static void restore_process_worker(struct work_struct *work)
{
struct delayed_work *dwork;
@@ -2009,6 +2055,12 @@ static void restore_process_worker(struct work_struct *work)
* lifetime of this thread, kfd_process p will be valid
*/
p = container_of(dwork, struct kfd_process, restore_work);
+
+ if (!kfd_hold_devices_reset_semaphore(p)) {
+ pr_debug("GPU resetting, restore bo and queue skipped\n");
+ return;
+ }
+
pr_debug("Started restoring pasid 0x%x\n", p->pasid);
/* Setting last_restore_timestamp before successful restoration. @@ -2031,6 +2083,8 @@ static void restore_process_worker(struct work_struct *work) msecs_to_jiffies(PROCESS_RESTORE_TIME_MS))) kfd_process_restore_queues(p);
}
+
+ kfd_unhold_devices_reset_semaphore(p);
}
void kfd_suspend_all_processes(void)
Felix Kuehling
maintainerQuoted 24 lines · Philip Yang
If GPU reset kick in while KFD
restore_process_workerrunning, this may causes different issues, for example below rcu stall warning, because restore work may move BOs and evict queues under VRAM pressure.Fix this race by taking adev
reset_domainread semaphore to prevent GPU reset inrestore_process_worker, the reset read semaphore can be taken recursively if adev have multiple partitions.Then there is live locking issue if CP hangs while
restore_process_workerruns, then GPU reset wait for semaphore to start andrestore_process_workercannot finish to release semaphore. We need signal eviction fence to solve the live locking if evict queue return-ETIMEOUT(for MES path) or-ETIME(for HWS path) because CP hangs,amdgpu 0000:af:00.0: amdgpu: GPU reset(21) succeeded!
rcu: INFO: rcu_sched self-detected stall on CPU Workqueue: kfd_restore_wq restore_process_worker [amdgpu] Call Trace: update_process_times+0x94/0xd0 RIP: 0010:amdgpu_vm_handle_moved+0x9a/0x210 [amdgpu] amdgpu_amdkfd_gpuvm_restore_process_bos+0x3d6/0x7d0 [amdgpu] restore_process_helper+0x27/0x80 [amdgpu]
Quoted 1 line
Signed-off-by Philip Yang <[email protected]>
See comments inline. I'd also like Christian to take a look at this patch since he's the expert on the reset locking stuff.
Quoted 35 lines
--- drivers/gpu/drm/amd/amdkfd/kfd_process.c | 56 +++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-)drivers/gpu/drm/amd/amdkfd/kfd_process.c +8
-1@@ -35,6 +35,7 @@#include <linux/pm_runtime.h> #include "amdgpu_amdkfd.h" #include "amdgpu.h" +#include "amdgpu_reset.h" struct mm_struct ;@@ -1972,8 +1973,14 @@ static void evict_process_worker(struct work_struct *work)kfd_process_restore_queues (p);pr_debug ("Finished evicting pasid 0x%x\n" , p->pasid); - }else + }else if (ret == -ETIMEDOUT || ret == -ETIME ) { +/* If CP hangs, signal the eviction fence, then restore_bo_worker +* can finish to up_read GPU reset semaphore to start GPU reset. +*/ +signal_eviction_fence (p); + }else {pr_err ("Failed to evict queues of pasid 0x%x\n" , p->pasid); + } }static int restore_process_helper (struct kfd_process *p)@@ -1997,6 +2004,45 @@ static int restore_process_helper(struct kfd_process *p)return ret; }
drivers/gpu/drm/amd/amdkfd/kfd_process.c static int restore_process_helper(struct kfd_process *p) · line 2015
Quoted 10 lines
drivers/gpu/drm/amd/amdkfd/kfd_process.c +9
-0+/* +* kfd_hold_devices_reset_semaphore +* +* return: +* true : hold reset domain semaphore to prevent device reset +* false: one of the device is resetting or already reset +* +*/ +static bool kfd_hold_devices_reset_semaphore (struct kfd_process *p)
I find the function naming of these functions (hold/unhold) a bit weird. I'd suggest kfd_process_trylock_reset_sems/kfd_process_unlock_reset_sems.
Quoted 34 lines
drivers/gpu/drm/amd/amdkfd/kfd_process.c +30
-0+{ +struct amdgpu_device *adev; +int i; + +for (i =0 ; i < p->n_pdds; i++) { + adev = p->pdds[i]->dev->adev; +if (!down_read_trylock (&adev->reset_domain->sem)) + goto out_upread; + } +return true; + +out_upread: +while (i--) { + adev = p->pdds[i]->dev->adev; +up_read (&adev->reset_domain->sem); + } +return false; +} + +static void kfd_unhold_devices_reset_semaphore (struct kfd_process *p) +{ +struct amdgpu_device *adev; +int i; + +for (i =0 ; i < p->n_pdds; i++) { + adev = p->pdds[i]->dev->adev; +up_read (&adev->reset_domain->sem); + } +} +static void restore_process_worker (struct work_struct *work) {struct delayed_work *dwork;
drivers/gpu/drm/amd/amdkfd/kfd_process.c static void restore_process_worker(struct work_struct *work) · line 2060
Quoted 6 lines
drivers/gpu/drm/amd/amdkfd/kfd_process.c +3
-0@@ -2009,6 +2055,12 @@ static void restore_process_worker(struct work_struct *work)* lifetime of this thread, kfd_process p will be valid */ p = container_of(dwork, struct kfd_process, restore_work); + +if (!kfd_hold_devices_reset_semaphore (p)) { +pr_debug ("GPU resetting, restore bo and queue skipped\n" );
Should we reschedule the restore worker to make sure it runs again after the reset is done?
Thanks,
Felix
Quoted 15 lines
drivers/gpu/drm/amd/amdkfd/kfd_process.c +5
-0+return ; + } +pr_debug ("Started restoring pasid 0x%x\n" , p->pasid);/* Setting last_restore_timestamp before successful restoration. @@ -2031,6 +2083,8 @@ static void restore_process_worker(struct work_struct *work) msecs_to_jiffies(PROCESS_RESTORE_TIME_MS)))kfd_process_restore_queues (p); } + +kfd_unhold_devices_reset_semaphore (p); }void kfd_suspend_all_processes (void )
On 2024-08-29 17:15, Felix Kuehling wrote:
On 2024-08-23 15:49, Philip Yang wrote:
If GPU reset kick in while KFD restore_process_worker running, this may causes different issues, for example below rcu stall warning, because restore work may move BOs and evict queues under VRAM pressure.
Fix this race by taking adev reset_domain read semaphore to prevent GPU reset in restore_process_worker, the reset read semaphore can be taken recursively if adev have multiple partitions.
Then there is live locking issue if CP hangs while restore_process_worker runs, then GPU reset wait for semaphore to start and restore_process_worker cannot finish to release semaphore. We need signal eviction fence to solve the live locking if evict queue return -ETIMEOUT (for MES path) or -ETIME (for HWS path) because CP hangs,
amdgpu 0000:af:00.0: amdgpu: GPU reset(21) succeeded!
rcu: INFO: rcu_sched self-detected stall on CPU Workqueue: kfd_restore_wq restore_process_worker [amdgpu] Call Trace: update_process_times+0x94/0xd0 RIP: 0010:amdgpu_vm_handle_moved+0x9a/0x210 [amdgpu] amdgpu_amdkfd_gpuvm_restore_process_bos+0x3d6/0x7d0 [amdgpu] restore_process_helper+0x27/0x80 [amdgpu]
See comments inline. I'd also like Christian to take a look at this patch since he's the expert on the reset locking stuff.
---
drivers/gpu/drm/amd/amdkfd/kfd_process.c | 56 +++++++++++++++++++++++-
1 file changed, 55 insertions(+), 1 deletion(-)
drivers/gpu/drm/amd/amdkfd/kfd_process.c +0 -0
@@ -35,6 +35,7 @@
#include <linux/pm_runtime.h>
#include "amdgpu_amdkfd.h"
#include "amdgpu.h"
+#include "amdgpu_reset.h"
struct mm_struct;
@@ -1972,8 +1973,14 @@ static void evict_process_worker(struct work_struct *work) kfd_process_restore_queues(p);
pr_debug("Finished evicting pasid 0x%x\n", p->pasid);
- } else
+ } else if (ret == -ETIMEDOUT || ret == -ETIME) {
+ /* If CP hangs, signal the eviction fence, then restore_bo_worker + * can finish to up_read GPU reset semaphore to start GPU reset. + */
+ signal_eviction_fence(p);
+ } else {
pr_err("Failed to evict queues of pasid 0x%x\n", p->pasid); + }
}
static int restore_process_helper(struct kfd_process *p)
@@ -1997,6 +2004,45 @@ static int restore_process_helper(struct kfd_process *p) return ret;
}
+/*
+ * kfd_hold_devices_reset_semaphore
+ *
+ * return:
+ * true : hold reset domain semaphore to prevent device reset
+ * false: one of the device is resetting or already reset
+ *
+ */
+static bool kfd_hold_devices_reset_semaphore(struct kfd_process *p)
I find the function naming of these functions (hold/unhold) a bit weird. I'd suggest kfd_process_trylock_reset_sems/kfd_process_unlock_reset_sems.
ok
+{
+ struct amdgpu_device *adev;
+ int i;
+
+ for (i = 0; i < p->n_pdds; i++) {
+ adev = p->pdds[i]->dev->adev;
+ if (!down_read_trylock(&adev->reset_domain->sem))
+ goto out_upread;
+ }
+ return true;
+
+out_upread:
+ while (i--) {
+ adev = p->pdds[i]->dev->adev;
+ up_read(&adev->reset_domain->sem);
+ }
+ return false;
+}
+
+static void kfd_unhold_devices_reset_semaphore(struct kfd_process *p) +{
+ struct amdgpu_device *adev;
+ int i;
+
+ for (i = 0; i < p->n_pdds; i++) {
+ adev = p->pdds[i]->dev->adev;
+ up_read(&adev->reset_domain->sem);
+ }
+}
+
static void restore_process_worker(struct work_struct *work)
{
struct delayed_work *dwork;
@@ -2009,6 +2055,12 @@ static void restore_process_worker(struct work_struct *work)
* lifetime of this thread, kfd_process p will be valid
*/
p = container_of(dwork, struct kfd_process, restore_work);
+
+ if (!kfd_hold_devices_reset_semaphore(p)) {
+ pr_debug("GPU resetting, restore bo and queue skipped\n");
Should we reschedule the restore worker to make sure it runs again after the reset is done?
After GPU mode1 reset, user processes already aborted, it is meaningless to reschedule restore worker to update GPU mapping.
Regards,
Philip
Thanks,
Felix
+ return;
+ }
+
pr_debug("Started restoring pasid 0x%x\n", p->pasid);
/* Setting last_restore_timestamp before successful restoration. @@ -2031,6 +2083,8 @@ static void restore_process_worker(struct work_struct *work) msecs_to_jiffies(PROCESS_RESTORE_TIME_MS))) kfd_process_restore_queues(p);
}
+
+ kfd_unhold_devices_reset_semaphore(p);
}
void kfd_suspend_all_processes(void)
Quoted 120 lines · Philip Yang
Quoted 64 lines · Felix Kuehling
Quoted 24 lines · Philip Yang
If GPU reset kick in while KFD
restore_process_workerrunning, this may causes different issues, for example below rcu stall warning, because restore work may move BOs and evict queues under VRAM pressure.Fix this race by taking adev
reset_domainread semaphore to prevent GPU reset inrestore_process_worker, the reset read semaphore can be taken recursively if adev have multiple partitions.Then there is live locking issue if CP hangs while
restore_process_workerruns, then GPU reset wait for semaphore to start andrestore_process_workercannot finish to release semaphore. We need signal eviction fence to solve the live locking if evict queue return-ETIMEOUT(for MES path) or-ETIME(for HWS path) because CP hangs,amdgpu 0000:af:00.0: amdgpu: GPU reset(21) succeeded!
rcu: INFO: rcu_sched self-detected stall on CPU Workqueue: kfd_restore_wq restore_process_worker [amdgpu] Call Trace: update_process_times+0x94/0xd0 RIP: 0010:amdgpu_vm_handle_moved+0x9a/0x210 [amdgpu] amdgpu_amdkfd_gpuvm_restore_process_bos+0x3d6/0x7d0 [amdgpu] restore_process_helper+0x27/0x80 [amdgpu]Quoted 1 line
Signed-off-by Philip Yang <[email protected]>See comments inline. I'd also like Christian to take a look at this patch since he's the expert on the reset locking stuff.
Quoted 34 lines
---
drivers/gpu/drm/amd/amdkfd/kfd_process.c| 56 +++++++++++++++++++++++-1 file changed, 55 insertions(+), 1 deletion(-)drivers/gpu/drm/amd/amdkfd/kfd_process.c +1
-0@@ -35,6 +35,7 @@ #include <linux/pm_runtime.h> #include "amdgpu_amdkfd.h" #include "amdgpu.h" +#include "amdgpu_reset.h" struct mm_struct; @@ -1972,8 +1973,14 @@ static void evict_process_worker(struct work_struct *work) kfd_process_restore_queues(p);
pr_debug("Finished evicting pasid 0x%x\n", p->pasid); - } else
+ } else if (ret ==-ETIMEDOUT|| ret ==-ETIME) {
+ /* If CP hangs, signal the eviction fence, thenrestore_bo_worker+ * can finish toup_readGPU reset semaphore to start GPU reset. + */
+signal_eviction_fence(p);
+ } else {
pr_err("Failed to evict queues of pasid 0x%x\n", p->pasid); + }
}
static intrestore_process_helper(struct kfd_process *p)drivers/gpu/drm/amd/amdkfd/kfd_process.c +0
-0@@ -1997,6 +2004,45 @@ static int restore_process_helper(struct kfd_process *p) return ret; }+/*
+ *kfd_hold_devices_reset_semaphore
+ *Quoted 8 lines
Quoted 6 lines
+ * return:
+ * true : hold reset domain semaphore to prevent device reset
+ * false: one of the device is resetting or already reset
+ *
+ */
+static boolkfd_hold_devices_reset_semaphore(struct kfd_process *p)I find the function naming of these functions (hold/unhold) a bit weird. I'd suggest kfd_process_trylock_reset_sems/kfd_process_unlock_reset_sems.
ok
Quoted 36 lines
Quoted 36 lines
drivers/gpu/drm/amd/amdkfd/kfd_process.c +30
-0+{ + struct amdgpu_device *adev; + int i; + + for (i = 0; i < p->n_pdds; i++) { + adev = p->pdds[i]->dev->adev; + if (!down_read_trylock(&adev->reset_domain->sem)) + goto out_upread; + } + return true; + +out_upread: + while (i--) { + adev = p->pdds[i]->dev->adev; + up_read(&adev->reset_domain->sem); + } + return false; +} + +static void kfd_unhold_devices_reset_semaphore (struct kfd_process *p) +{ + struct amdgpu_device *adev; + int i; + + for (i = 0; i < p->n_pdds; i++) { + adev = p->pdds[i]->dev->adev; + up_read(&adev->reset_domain->sem); + } +} +static void
restore_process_worker(struct work_struct *work){
structdelayed_work*dwork;drivers/gpu/drm/amd/amdkfd/kfd_process.c +0
-0@@ -2009,6 +2055,12 @@ static void restore_process_worker(struct work_struct *work)Quoted 8 lines
drivers/gpu/drm/amd/amdkfd/kfd_process.c static void restore_process_worker(struct work_struct *work) · line 2060
Quoted 6 lines
drivers/gpu/drm/amd/amdkfd/kfd_process.c +3
-0* lifetime of this thread, kfd_process p will be valid */ p = container_of(dwork, struct kfd_process, restore_work); + + if (!kfd_hold_devices_reset_semaphore(p)) { + pr_debug("GPU resetting, restore bo and queue skipped\n");Should we reschedule the restore worker to make sure it runs again after the reset is done?
Quoted 1 line
After GPU mode1 reset, user processes already aborted, it is meaningless to reschedule restore worker to update GPU mapping.
OK. With the fixed function names, the patch is
Quoted 23 lines
Regards,
Philip
Quoted 18 lines
Thanks,
FelixQuoted 13 lines
drivers/gpu/drm/amd/amdkfd/kfd_process.c +5
-0+ return; + } + pr_debug("Started restoring pasid 0x%x\n", p->pasid); /* Setting last_restore_timestamp before successful restoration.@@ -2031,6 +2083,8 @@ static void restore_process_worker(struct work_struct *work) msecs_to_jiffies(PROCESS_RESTORE_TIME_MS))) kfd_process_restore_queues(p); } + + kfd_unhold_devices_reset_semaphore(p); } void kfd_suspend_all_processes(void)