drm/amdkfd: Ensure mm remain valid in svm deferred_list work
Subsystems (3)
PATCH v2 1/2 drm/amdkfd: Ensure mm remain valid in svm deferred_list work
svm_deferred_list work should continue to handle deferred_range_list which maybe split to child range to avoid child range leak, and remove ranges mmu interval notifier to avoid mm mm_count leak. So taking mm reference when adding range to deferred list, to ensure mm is valid in the scheduled deferred_list_work, and drop the mm referrence after range is handled.
--- drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 62 ++++++++++++++++------------ 1 file changed, 36 insertions(+), 26 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +36 -26
@@ -1985,10 +1985,9 @@ svm_range_update_notifier_and_interval_tree(struct mm_struct *mm, }static void -svm_range_handle_list_op (struct svm_range_list *svms,struct svm_range *prange) +svm_range_handle_list_op (struct svm_range_list *svms,struct svm_range *prange, +struct mm_struct *mm) { -struct mm_struct *mm = prange->work_item.mm; -switch (prange->work_item.op) {case SVM_OP_NULL : pr_debug("NULL OP 0x%p prange 0x%p [0x%lx 0x%lx]\n",
@@ -2071,34 +2070,41 @@ static void svm_range_deferred_list_work(struct work_struct *work)pr_debug ("enter svms 0x%p\n" , svms); p = container_of(svms, struct kfd_process, svms); -/* Avoid mm is gone when inserting mmu notifier */ - mm =get_task_mm (p->lead_thread); -if (!mm) { -pr_debug ("svms 0x%p process mm gone\n" , svms); -return ; - } -retry: -mmap_write_lock (mm); - -/* Checking for the need to drain retry faults must be inside -* mmap write lock to serialize with munmap notifiers. -*/ -if (unlikely (atomic_read (&svms->drain_pagefaults))) { -mmap_write_unlock (mm); -svm_range_drain_retry_fault (svms); - goto retry; - }spin_lock (&svms->deferred_list_lock);while (!list_empty (&svms->deferred_range_list)) { prange =list_first_entry (&svms->deferred_range_list, struct svm_range, deferred_list); -list_del_init (&prange->deferred_list);spin_unlock (&svms->deferred_list_lock);pr_debug ("prange 0x%p [0x%lx 0x%lx] op %d\n" , prange, prange->start, prange->last, prange->work_item.op); + mm = prange->work_item.mm; +retry: +mmap_write_lock (mm); + +/* Checking for the need to drain retry faults must be inside +* mmap write lock to serialize with munmap notifiers. +*/ +if (unlikely (atomic_read (&svms->drain_pagefaults))) { +mmap_write_unlock (mm); +svm_range_drain_retry_fault (svms); + goto retry; + } + +/* Remove from deferred_list must be inside mmap write lock, for +* two race cases: +* 1. unmap_from_cpu may change work_item.op and add the range +* to deferred_list again, cause use after free bug. +* 2. svm_range_list_lock_and_flush_work may hold mmap write +* lock and continue because deferred_list is empty, but +* deferred_list work is actually waiting for mmap lock. +*/ +spin_lock (&svms->deferred_list_lock); +list_del_init (&prange->deferred_list); +spin_unlock (&svms->deferred_list_lock); +mutex_lock (&svms->lock);mutex_lock (&prange->migrate_mutex);while (!list_empty (&prange->child_list)) {
@@ -2109,19 +2115,20 @@ static void svm_range_deferred_list_work(struct work_struct *work)pr_debug ("child prange 0x%p op %d\n" , pchild, pchild->work_item.op);list_del_init (&pchild->child_list); -svm_range_handle_list_op (svms, pchild); +svm_range_handle_list_op (svms, pchild, mm); }mutex_unlock (&prange->migrate_mutex); -svm_range_handle_list_op (svms, prange); +svm_range_handle_list_op (svms, prange, mm);mutex_unlock (&svms->lock); +mmap_write_unlock (mm); + +/* Pairs with mmget in svm_range_add_list_work */ +mmput (mm);spin_lock (&svms->deferred_list_lock); }spin_unlock (&svms->deferred_list_lock); - -mmap_write_unlock (mm); -mmput (mm);pr_debug ("exit svms 0x%p\n" , svms); }
@@ -2139,6 +2146,9 @@ svm_range_add_list_work(struct svm_range_list *svms, struct svm_range *prange, prange->work_item.op = op; }else { prange->work_item.op = op; + +/* Pairs with mmput in deferred_list_work */ +mmget (mm); prange->work_item.mm = mm;list_add_tail (&prange->deferred_list, &prange->svms->deferred_range_list);
Signature · 1 line
2.17.1
PATCH v2 2/2 drm/amdkfd: svm range restore work deadlock when process exit
kfd_process_notifier_release flush svm_range_restore_work which calls svm_range_list_lock_and_flush_work to flush deferred_list work, but if deferred_list work mmput release the last user, it will call exit_mmap -> notifier_release, it is deadlock with below backtrace.
Move flush svm_range_restore_work to kfd_process_wq_release to avoid deadlock. Then svm_range_restore_work take task->mm ref to avoid mm is gone while validating and mapping ranges to GPU.
Workqueue: events svm_range_deferred_list_work [amdgpu] Call Trace: wait_for_completion+0x94/0x100 __flush_work+0x12a/0x1e0 __cancel_work_timer+0x10e/0x190 cancel_delayed_work_sync+0x13/0x20 kfd_process_notifier_release+0x98/0x2a0 [amdgpu] __mmu_notifier_release+0x74/0x1f0 exit_mmap+0x170/0x200 mmput+0x5d/0x130 svm_range_deferred_list_work+0x104/0x230 [amdgpu] process_one_work+0x220/0x3c0
--- drivers/gpu/drm/amd/amdkfd/kfd_process.c | 1 - drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 15 +++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_process.c +0 -1
@@ -1150,7 +1150,6 @@ static void kfd_process_notifier_release(struct mmu_notifier *mn,cancel_delayed_work_sync (&p->eviction_work);cancel_delayed_work_sync (&p->restore_work); -cancel_delayed_work_sync (&p->svms.restore_work);mutex_lock (&p->mutex);
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +9 -6
@@ -1643,13 +1643,14 @@ static void svm_range_restore_work(struct work_struct *work)pr_debug ("restore svm ranges\n" ); -/* kfd_process_notifier_release destroys this worker thread. So during -* the lifetime of this thread, kfd_process and mm will be valid. -*/ p = container_of(svms, struct kfd_process, svms); - mm = p->mm; -if (!mm) + +/* Keep mm reference when svm_range_validate_and_map ranges */ + mm =get_task_mm (p->lead_thread); +if (!mm) { +pr_debug ("svms 0x%p process mm gone\n" , svms);return ; + }svm_range_list_lock_and_flush_work (svms, mm);mutex_lock (&svms->lock);
@@ -1703,6 +1704,7 @@ static void svm_range_restore_work(struct work_struct *work) out_reschedule:mutex_unlock (&svms->lock);mmap_write_unlock (mm); +mmput (mm);/* If validation failed, reschedule another attempt */ if (evicted_ranges) {
@@ -2840,6 +2842,8 @@ void svm_range_list_fini(struct kfd_process *p)pr_debug ("pasid 0x%x svms 0x%p\n" , p->pasid, &p->svms); +cancel_delayed_work_sync (&p->svms.restore_work); +/* Ensure list work is finished before process is destroyed */ flush_work (&p->svms.deferred_list_work);
@@ -2850,7 +2854,6 @@ void svm_range_list_fini(struct kfd_process *p)atomic_inc (&p->svms.drain_pagefaults);svm_range_drain_retry_fault (&p->svms); -list_for_each_entry_safe (prange, next, &p->svms.list, list) {svm_range_unlink (prange);svm_range_remove_notifier (prange);
Signature · 1 line
2.17.1
Am 2022-01-24 um 16:11 schrieb Philip Yang:
Quoted 6 lines
svm_deferred_listwork should continue to handledeferred_range_listwhich maybe split to child range to avoid child range leak, and remove ranges mmu interval notifier to avoid mmmm_countleak. So taking mm reference when adding range to deferred list, to ensure mm is valid in the scheduleddeferred_list_work, and drop the mm referrence after range is handled.
So this reverts much of your previous patch f0b45b5196de ("drm/amdkfd: process exit and retry fault race").
One one hand it feels wasteful to have the mm pointer in each work item, when we know that the mm pointer will be the same in all of them. On the other hand the pointers now properly represent references to the mm struct, and it's cleaner to track them per deferred list entry. Also, finer-grained locking is not necessarily a bad thing.
The series is
Quoted 120 lines
Signed-off-by Philip Yang <[email protected]>Reported-by Ruili Ji <[email protected]>--- drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 62 ++++++++++++++++------------ 1 file changed, 36 insertions(+), 26 deletions(-)drivers/gpu/drm/amd/amdkfd/kfd_svm.c +36
-26@@ -1985,10 +1985,9 @@ svm_range_update_notifier_and_interval_tree(struct mm_struct *mm, }static void -svm_range_handle_list_op (struct svm_range_list *svms,struct svm_range *prange) +svm_range_handle_list_op (struct svm_range_list *svms,struct svm_range *prange, +struct mm_struct *mm) { -struct mm_struct *mm = prange->work_item.mm; -switch (prange->work_item.op) {case SVM_OP_NULL : pr_debug("NULL OP 0x%p prange 0x%p [0x%lx 0x%lx]\n",@@ -2071,34 +2070,41 @@ static void svm_range_deferred_list_work(struct work_struct *work)pr_debug ("enter svms 0x%p\n" , svms); p = container_of(svms, struct kfd_process, svms); -/* Avoid mm is gone when inserting mmu notifier */ - mm =get_task_mm (p->lead_thread); -if (!mm) { -pr_debug ("svms 0x%p process mm gone\n" , svms); -return ; - } -retry: -mmap_write_lock (mm); - -/* Checking for the need to drain retry faults must be inside -* mmap write lock to serialize with munmap notifiers. -*/ -if (unlikely (atomic_read (&svms->drain_pagefaults))) { -mmap_write_unlock (mm); -svm_range_drain_retry_fault (svms); - goto retry; - }spin_lock (&svms->deferred_list_lock);while (!list_empty (&svms->deferred_range_list)) { prange =list_first_entry (&svms->deferred_range_list, struct svm_range, deferred_list); -list_del_init (&prange->deferred_list);spin_unlock (&svms->deferred_list_lock);pr_debug ("prange 0x%p [0x%lx 0x%lx] op %d\n" , prange, prange->start, prange->last, prange->work_item.op); + mm = prange->work_item.mm; +retry: +mmap_write_lock (mm); + +/* Checking for the need to drain retry faults must be inside +* mmap write lock to serialize with munmap notifiers. +*/ +if (unlikely (atomic_read (&svms->drain_pagefaults))) { +mmap_write_unlock (mm); +svm_range_drain_retry_fault (svms); + goto retry; + } + +/* Remove from deferred_list must be inside mmap write lock, for +* two race cases: +* 1. unmap_from_cpu may change work_item.op and add the range +* to deferred_list again, cause use after free bug. +* 2. svm_range_list_lock_and_flush_work may hold mmap write +* lock and continue because deferred_list is empty, but +* deferred_list work is actually waiting for mmap lock. +*/ +spin_lock (&svms->deferred_list_lock); +list_del_init (&prange->deferred_list); +spin_unlock (&svms->deferred_list_lock); +mutex_lock (&svms->lock);mutex_lock (&prange->migrate_mutex);while (!list_empty (&prange->child_list)) {@@ -2109,19 +2115,20 @@ static void svm_range_deferred_list_work(struct work_struct *work)pr_debug ("child prange 0x%p op %d\n" , pchild, pchild->work_item.op);list_del_init (&pchild->child_list); -svm_range_handle_list_op (svms, pchild); +svm_range_handle_list_op (svms, pchild, mm); }mutex_unlock (&prange->migrate_mutex); -svm_range_handle_list_op (svms, prange); +svm_range_handle_list_op (svms, prange, mm);mutex_unlock (&svms->lock); +mmap_write_unlock (mm); + +/* Pairs with mmget in svm_range_add_list_work */ +mmput (mm);spin_lock (&svms->deferred_list_lock); }spin_unlock (&svms->deferred_list_lock); - -mmap_write_unlock (mm); -mmput (mm);pr_debug ("exit svms 0x%p\n" , svms); }@@ -2139,6 +2146,9 @@ svm_range_add_list_work(struct svm_range_list *svms, struct svm_range *prange, prange->work_item.op = op; }else { prange->work_item.op = op; + +/* Pairs with mmput in deferred_list_work */ +mmget (mm); prange->work_item.mm = mm;list_add_tail (&prange->deferred_list, &prange->svms->deferred_range_list);