drm/amdkfd: Change kfd/svm page fault drain handling
Subsystems (3)
PATCH v2 1/1 drm/amdkfd: Change kfd/svm page fault drain handling
From: Xiaogang Chen <[email protected]>
When app unmap vm ranges(munmap) kfd/svm starts drain pending page fault and not handle any incoming pages fault of this process until a deferred work item got executed by default system wq. The time period of "not handle page fault" can be long and is unpredicable. That is advese to kfd performance on page faults recovery.
This patch uses time stamp of incoming page page to decide to drop or handle page fault. When app unmap vm ranges kfd records each gpu device's ih ring current time stamp. These time stamps are used at kfd page fault recovery routine.
Any page fault happens on unmapped ranges after unmap events is app bug that accesses vm range after unmap. It is not driver work to cover that.
By using time stamp of page fault do not need drain page faults at deferred work. So, the time period that kfd does not handle page faults is reduced and can be controlled.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 4 +- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 2 +- drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c | 3 +- drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c | 4 +- drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 5 +- drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 111 +++++++++++++++++-------- drivers/gpu/drm/amd/amdkfd/kfd_svm.h | 2 +- 7 files changed, 88 insertions(+), 43 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +2 -2
@@ -2763,7 +2763,7 @@ int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)* shouldn't be reported any more. */ bool amdgpu_vm_handle_fault (struct amdgpu_device *adev,u32 pasid, -u32 vmid,u32 node_id,uint64_t addr, +u32 vmid,u32 node_id,uint64_t addr,uint64_t ts,bool write_fault) {bool is_compute_context = false;
@@ -2789,7 +2789,7 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid, addr /=AMDGPU_GPU_PAGE_SIZE ;if (is_compute_context && !svm_range_restore_pages (adev, pasid, vmid, - node_id, addr, write_fault)) { + node_id, addr, ts, write_fault)) {amdgpu_bo_unref (&root);return true; }
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +1 -1
@@ -548,7 +548,7 @@ amdgpu_vm_get_task_info_vm(struct amdgpu_vm *vm);void amdgpu_vm_put_task_info (struct amdgpu_task_info *task_info);bool amdgpu_vm_handle_fault (struct amdgpu_device *adev,u32 pasid, -u32 vmid,u32 node_id,uint64_t addr, +u32 vmid,u32 node_id,uint64_t addr,uint64_t ts,bool write_fault);void amdgpu_vm_set_task_info (struct amdgpu_vm *vm);
drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c +2 -1
@@ -132,7 +132,8 @@ static int gmc_v10_0_process_interrupt(struct amdgpu_device *adev,/* Try to handle the recoverable page faults by filling page * tables */ -if (amdgpu_vm_handle_fault (adev, entry->pasid,0 ,0 , addr, write_fault)) +if (amdgpu_vm_handle_fault (adev, entry->pasid,0 ,0 , addr, + entry->timestamp, write_fault))return 1 ; }
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +2 -2
@@ -595,7 +595,7 @@ static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev, cam_index = entry->src_data[2 ] &0x3ff ; ret =amdgpu_vm_handle_fault (adev, entry->pasid, entry->vmid, node_id, - addr, write_fault); + addr, entry->timestamp, write_fault);WDOORBELL32 (adev->irq.retry_cam_doorbell_index, cam_index);if (ret)return 1 ;
@@ -618,7 +618,7 @@ static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev,* tables */ if (amdgpu_vm_handle_fault (adev, entry->pasid, entry->vmid, node_id, - addr, write_fault)) + addr, entry->timestamp, write_fault))return 1 ; } }
drivers/gpu/drm/amd/amdkfd/kfd_priv.h +4 -1
@@ -850,10 +850,13 @@ struct svm_range_list {struct list_head criu_svm_metadata_list;spinlock_t deferred_list_lock;atomic_t evicted_ranges; -atomic_t drain_pagefaults; +/* stop page fault recovery for this process */ +atomic_t stop_pf_recovery;struct delayed_work restore_work;DECLARE_BITMAP (bitmap_supported,MAX_GPU_INSTANCE );struct task_struct *faulting_task; +/* check point ts decides if page fault recovery need be dropped */ +uint64_t checkpoint_ts[MAX_GPU_INSTANCE ]; };/* Process data */
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +76 -35
193 lines not on this page · raw message
drivers/gpu/drm/amd/amdkfd/kfd_svm.h +1 -1
@@ -173,7 +173,7 @@ int svm_range_vram_node_new(struct kfd_node *node, struct svm_range *prange, bool clear);void svm_range_vram_node_free (struct svm_range *prange);int svm_range_restore_pages (struct amdgpu_device *adev,unsigned int pasid, -uint32_t vmid,uint32_t node_id,uint64_t addr, +uint32_t vmid,uint32_t node_id,uint64_t addr,uint64_t ts,bool write_fault);int svm_range_schedule_evict_svm_bo (struct amdgpu_amdkfd_fence *fence); void svm_range_add_list_work(struct svm_range_list *svms,
Signature · 1 line
2.25.1
On 2024-07-19 18:17, Xiaogang.Chen wrote:
From: Xiaogang Chen <[email protected]> When app unmap vm ranges(munmap) kfd/svm starts drain pending page fault and not handle any incoming pages fault of this process until a deferred work item got executed by default system wq. The time period of "not handle page fault" can be long and is unpredicable. That is advese to kfd performance on page faults recovery. This patch uses time stamp of incoming page page to decide to drop or handle page fault. When app unmap vm ranges kfd records each gpu device's ih ring current time stamp. These time stamps are used at kfd page fault recovery routine. Any page fault happens on unmapped ranges after unmap events is app bug that accesses vm range after unmap. It is not driver work to cover that. By using time stamp of page fault do not need drain page faults at deferred work. So, the time period that kfd does not handle page faults is reduced and can be controlled. This simplify the retry fault draining and support the multiple processes correctly now, some nitpick below.
return 1; } diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c index 350f6b6676f1..ac08d9424feb 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c @@ -595,7 +595,7 @@ static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev, cam_index = entry->src_data[2] & 0x3ff; ret = amdgpu_vm_handle_fault(adev, entry->pasid, entry->vmid, node_id, - addr, write_fault); + addr, entry->timestamp, write_fault); WDOORBELL32(adev->irq.retry_cam_doorbell_index, cam_index); if (ret) return 1; @@ -618,7 +618,7 @@ static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev, * tables */ if (amdgpu_vm_handle_fault(adev, entry->pasid, entry->vmid, node_id, - addr, write_fault)) + addr, entry->timestamp, write_fault)) return 1; } } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h index c51e908f6f19..8b8d5ab9da76 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h @@ -850,10 +850,13 @@ struct svm_range_list { struct list_head criu_svm_metadata_list; spinlock_t deferred_list_lock; atomic_t evicted_ranges; - atomic_t drain_pagefaults; + /* stop page fault recovery for this process */ + atomic_t stop_pf_recovery; This flag seems redundant, as timestamp is 48bit increasing and amdgpu_ih_ts_after handle the wrap around, so always use timestamp to drop retry fault.
struct delayed_work restore_work; DECLARE_BITMAP(bitmap_supported, MAX_GPU_INSTANCE); struct task_struct *faulting_task; + /* check point ts decides if page fault recovery need be dropped */ + uint64_t checkpoint_ts[MAX_GPU_INSTANCE]; I think it is safe to not use atomic64_t.
}; /* Process data */ diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c index 407636a68814..9dd28d06ea2a 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c @@ -2263,16 +2263,10 @@ static void svm_range_drain_retry_fault(struct svm_range_list *svms) { struct kfd_process_device *pdd; struct kfd_process *p; - int drain; uint32_t i; p = container_of(svms, struct kfd_process, svms); -restart: - drain = atomic_read(&svms->drain_pagefaults); - if (!drain) - return; - for_each_set_bit(i, svms->bitmap_supported, p->n_pdds) { pdd = p->pdds[i]; if (!pdd) @@ -2292,8 +2286,6 @@ static void svm_range_drain_retry_fault(struct svm_range_list *svms) pr_debug("drain retry fault gpu %d svms 0x%p done\n", i, svms); } - if (atomic_cmpxchg(&svms->drain_pagefaults, drain, 0) != drain) - goto restart; } static void svm_range_deferred_list_work(struct work_struct *work) @@ -2315,17 +2307,8 @@ static void svm_range_deferred_list_work(struct work_struct *work) prange->start, prange->last, prange->work_item.op); mm = prange->work_item.mm; -retry: - mmap_write_lock(mm); - /* Checking for the need to drain retry faults must be inside - * mmap write lock to serialize with munmap notifiers. - */ - if (unlikely(atomic_read(&svms->drain_pagefaults))) { - mmap_write_unlock(mm); - svm_range_drain_retry_fault(svms); - goto retry; - } + mmap_write_lock(mm); /* Remove from deferred_list must be inside mmap write lock, for * two race cases: @@ -2446,6 +2429,7 @@ svm_range_unmap_from_cpu(struct mm_struct *mm, struct svm_range *prange, struct kfd_process *p; unsigned long s, l; bool unmap_parent; + uint32_t i; p = kfd_lookup_process_by_mm(mm); if (!p) @@ -2455,11 +2439,49 @@ svm_range_unmap_from_cpu(struct mm_struct *mm, struct svm_range *prange, pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx] [0x%lx 0x%lx]\n", svms, prange, prange->start, prange->last, start, last); - /* Make sure pending page faults are drained in the deferred worker - * before the range is freed to avoid straggler interrupts on - * unmapped memory causing "phantom faults". + /* first stop kfd page fault handling for this process */ + atomic_set(&svms->stop_pf_recovery, 1); this flag is not needed.
+ /* calculate time stamps that are used to decide which page faults need be + * dropped or handled before unmap pages from gpu vm */ - atomic_inc(&svms->drain_pagefaults); + for_each_set_bit(i, svms->bitmap_supported, p->n_pdds) { + struct kfd_process_device *pdd; + struct amdgpu_device *adev; + struct amdgpu_ih_ring *ih; + uint32_t checkpoint_wptr; + uint64_t checkpoint_ts = 0; + + svms->checkpoint_ts[i] = 0; don't reset it to zero.
+ pdd = p->pdds[i]; + if (!pdd) + continue; + + adev = pdd->dev->adev; + ih = adev->irq.retry_cam_enabled ? &adev->irq.ih : &adev->irq.ih1; + if (!ih->enabled || adev->shutdown) + continue; + + checkpoint_wptr = amdgpu_ih_get_wptr(adev, ih); + /* Order wptr with ring data. */ + rmb(); + checkpoint_ts = amdgpu_ih_decode_iv_ts(adev, ih, checkpoint_wptr, -1); + + if (adev->irq.retry_cam_enabled) { + ih = &adev->irq.ih_soft; + + if (!ih->enabled || adev->shutdown) + continue; this check can be removed.
+ + checkpoint_wptr = amdgpu_ih_get_wptr(adev, ih); + rmb(); + checkpoint_ts = max(checkpoint_ts, + amdgpu_ih_decode_iv_ts(adev, ih, checkpoint_wptr, -1)); + } + svms->checkpoint_ts[i] = checkpoint_ts; WRITE_ONCE(svms->checkpoint_ts[i], checkpoint_ts); Use WRITE_ONCE here and restore_page use READ_ONCE to sync up the timestamp update.
+ } + /* resume kfd page fault handing with time stamp checking */ + atomic_set(&svms->stop_pf_recovery, 0); unmap_parent = start <= prange->start && last >= prange->last; @@ -2900,7 +2922,7 @@ svm_fault_allowed(struct vm_area_struct *vma, bool write_fault) int svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid, uint32_t vmid, uint32_t node_id, - uint64_t addr, bool write_fault) + uint64_t addr, uint64_t ts, bool write_fault) { unsigned long start, last, size; struct mm_struct *mm = NULL; @@ -2910,7 +2932,7 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid, ktime_t timestamp = ktime_get_boottime(); struct kfd_node *node; int32_t best_loc; - int32_t gpuidx = MAX_GPU_INSTANCE; + int32_t gpuid, gpuidx = MAX_GPU_INSTANCE; bool write_locked = false; struct vm_area_struct *vma; bool migration = false; @@ -2930,7 +2952,29 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid, pr_debug("restoring svms 0x%p fault address 0x%llx\n", svms, addr); - if (atomic_read(&svms->drain_pagefaults)) { + /* kfd page fault recovery is disabled */ + if (atomic_read(&svms->stop_pf_recovery)) { + pr_debug("page fault handing disabled, drop fault 0x%llx\n", addr); + r = 0; + goto out; + } + + node = kfd_node_by_irq_ids(adev, node_id, vmid); + if (!node) { + pr_debug("kfd node does not exist node_id: %d, vmid: %d\n", node_id, + vmid); + r = -EFAULT; + goto out; + } + + if (kfd_process_gpuid_from_node(p, node, &gpuid, &gpuidx)) { + pr_debug("failed to get gpuid/gpuidex for node_id: %d \n", node_id); + r = -EFAULT; + goto out; + } + + /* check if this page fault time stamp is before svms->checkpoint_ts */ + if (amdgpu_ih_ts_after(ts, svms->checkpoint_ts[gpuidx])) { if (amdgpu_ih_ts_after(ts, READ_ONCE(svms->checkpoint_ts[gpuidx]))) { pr_debug("draining retry fault, drop fault 0x%llx\n", addr); r = 0; goto out; @@ -2952,13 +2996,6 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid, goto out; } - node = kfd_node_by_irq_ids(adev, node_id, vmid); - if (!node) { - pr_debug("kfd node does not exist node_id: %d, vmid: %d\n", node_id, - vmid); - r = -EFAULT; - goto out; - } mmap_read_lock(mm); retry_write_locked: mutex_lock(&svms->lock); @@ -3173,9 +3210,11 @@ void svm_range_list_fini(struct kfd_process *p) /* * Ensure no retry fault comes in afterwards, as page fault handler will * not find kfd process and take mm lock to recover fault. + * stop kfd page fault handing, then wait pending page faults got drained */ - atomic_inc(&p->svms.drain_pagefaults); + atomic_set(&p->svms.stop_pf_recovery, 1); svm_range_drain_retry_fault(&p->svms); + atomic_set(&p->svms.stop_pf_recovery, 0); list_for_each_entry_safe(prange, next, &p->svms.list, list) { svm_range_unlink(prange); @@ -3197,16 +3236,18 @@ int svm_range_list_init(struct kfd_process *p) mutex_init(&svms->lock); INIT_LIST_HEAD(&svms->list); atomic_set(&svms->evicted_ranges, 0); - atomic_set(&svms->drain_pagefaults, 0); + atomic_set(&svms->stop_pf_recovery, 0); INIT_DELAYED_WORK(&svms->restore_work, svm_range_restore_work); INIT_WORK(&svms->deferred_list_work, svm_range_deferred_list_work); INIT_LIST_HEAD(&svms->deferred_range_list); INIT_LIST_HEAD(&svms->criu_svm_metadata_list); spin_lock_init(&svms->deferred_list_lock); - for (i = 0; i < p->n_pdds; i++) + for (i = 0; i < p->n_pdds; i++) { + svms->checkpoint_ts[i] = 0; not need to init 0 as kfd_process structure is from kzalloc. Regards,
Philip
if (KFD_IS_SVM_API_SUPPORTED(p->pdds[i]->dev->adev)) bitmap_set(svms->bitmap_supported, i, 1); + } return 0; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h index 70c1776611c4..5f447c3642cb 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h @@ -173,7 +173,7 @@ int svm_range_vram_node_new(struct kfd_node *node, struct svm_range *prange, bool clear); void svm_range_vram_node_free(struct svm_range *prange); int svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid, - uint32_t vmid, uint32_t node_id, uint64_t addr, + uint32_t vmid, uint32_t node_id, uint64_t addr, uint64_t ts, bool write_fault); int svm_range_schedule_evict_svm_bo(struct amdgpu_amdkfd_fence *fence); void svm_range_add_list_work(struct svm_range_list *svms,
Quoted 68 lines · Philip Yang
Quoted 13 lines · Xiaogang.Chen
From: Xiaogang Chen<[email protected]>
When app unmap vm ranges(munmap) kfd/svm starts drain pending page fault and not handle any incoming pages fault of this process until a deferred work item got executed by default system wq. The time period of "not handle page fault" can be long and is unpredicable. That is advese to kfd performance on page faults recovery.
This patch uses time stamp of incoming page page to decide to drop or handle page fault. When app unmap vm ranges kfd records each gpu device's ih ring current time stamp. These time stamps are used at kfd page fault recovery routine.
Quoted 6 lines
Any page fault happens on unmapped ranges after unmap events is app bug that accesses vm range after unmap. It is not driver work to cover that.
By using time stamp of page fault do not need drain page faults at deferred work. So, the time period that kfd does not handle page faults is reduced and can be controlled.
This simplify the retry fault draining and support the multiple processes correctly now, some nitpick below.
Quoted 45 lines
Signed-off-by Xiaogang.Chen<[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 4 +- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 2 +- drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c | 3 +- drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c | 4 +- drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 5 +- drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 111 +++++++++++++++++-------- drivers/gpu/drm/amd/amdkfd/kfd_svm.h | 2 +- 7 files changed, 88 insertions(+), 43 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +2
-2@@ -2763,7 +2763,7 @@ int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)* shouldn't be reported any more. */ bool amdgpu_vm_handle_fault (struct amdgpu_device *adev,u32 pasid, -u32 vmid,u32 node_id,uint64_t addr, +u32 vmid,u32 node_id,uint64_t addr,uint64_t ts,bool write_fault) {bool is_compute_context = false;@@ -2789,7 +2789,7 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid, addr /=AMDGPU_GPU_PAGE_SIZE ;if (is_compute_context && !svm_range_restore_pages (adev, pasid, vmid, - node_id, addr, write_fault)) { + node_id, addr, ts, write_fault)) {amdgpu_bo_unref (&root);return true; }drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +1
-1@@ -548,7 +548,7 @@ amdgpu_vm_get_task_info_vm(struct amdgpu_vm *vm);void amdgpu_vm_put_task_info (struct amdgpu_task_info *task_info);bool amdgpu_vm_handle_fault (struct amdgpu_device *adev,u32 pasid, -u32 vmid,u32 node_id,uint64_t addr, +u32 vmid,u32 node_id,uint64_t addr,uint64_t ts,bool write_fault);
Quoted 13 lines
drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c static int gmc_v10_0_process_interrupt(struct amdgpu_device *adev, · line 136
Quoted 12 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +0
-0void amdgpu_vm_set_task_info (struct amdgpu_vm *vm);drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c +2
-1@@ -132,7 +132,8 @@ static int gmc_v10_0_process_interrupt(struct amdgpu_device *adev,/* Try to handle the recoverable page faults by filling page * tables */ -if (amdgpu_vm_handle_fault (adev, entry->pasid,0 ,0 , addr, write_fault)) +if (amdgpu_vm_handle_fault (adev, entry->pasid,0 ,0 , addr, + entry->timestamp, write_fault))indent should align to the start bracket.
ok.
Quoted 29 lines
Quoted 29 lines
drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c +0
-0return 1 ; }drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +2
-2@@ -595,7 +595,7 @@ static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev, cam_index = entry->src_data[2 ] &0x3ff ; ret =amdgpu_vm_handle_fault (adev, entry->pasid, entry->vmid, node_id, - addr, write_fault); + addr, entry->timestamp, write_fault);WDOORBELL32 (adev->irq.retry_cam_doorbell_index, cam_index);if (ret)return 1 ;@@ -618,7 +618,7 @@ static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev,* tables */ if (amdgpu_vm_handle_fault (adev, entry->pasid, entry->vmid, node_id, - addr, write_fault)) + addr, entry->timestamp, write_fault))return 1 ; } }drivers/gpu/drm/amd/amdkfd/kfd_priv.h +0
-0
Quoted 8 lines
drivers/gpu/drm/amd/amdkfd/kfd_priv.h struct svm_range_list { · line 854
Quoted 7 lines
drivers/gpu/drm/amd/amdkfd/kfd_priv.h +2
-1@@ -850,10 +850,13 @@ struct svm_range_list {struct list_head criu_svm_metadata_list;spinlock_t deferred_list_lock;atomic_t evicted_ranges; -atomic_t drain_pagefaults; +/* stop page fault recovery for this process */ +atomic_t stop_pf_recovery;This flag seems redundant, as timestamp is 48bit increasing and
amdgpu_ih_ts_afterhandle the wrap around, so always use timestamp to drop retry fault.
This flag notifies page fault restore routine to not process page fault for this process. It is used here when getting current time stamp at each gpu's IH ring, then unset it. To make sure page fault will not be processed during time period when get current time stamps, though it is very short.
And this flag maybe used in future for general purpose that we need stop process page fault recovery for some reasons.
Quoted 6 lines
drivers/gpu/drm/amd/amdkfd/kfd_priv.h struct svm_range_list { · line 859
Quoted 5 lines
drivers/gpu/drm/amd/amdkfd/kfd_priv.h +2
-0struct delayed_work restore_work;DECLARE_BITMAP (bitmap_supported,MAX_GPU_INSTANCE );struct task_struct *faulting_task; +/* check point ts decides if page fault recovery need be dropped */ +uint64_t checkpoint_ts[MAX_GPU_INSTANCE ];I think it is safe to not use
atomic64_t.
not sure what you mean here: you do not want use uint64_4, or atomic64_4? I use unit64_4 here. And as you mention below, will use WRITE_ONCE/READ_ONCE on svms->checkpoint_ts[i].
Quoted 64 lines
Quoted 64 lines
drivers/gpu/drm/amd/amdkfd/kfd_priv.h +0
-0};/* Process data */ drivers/gpu/drm/amd/amdkfd/kfd_svm.c +2
-18@@ -2263,16 +2263,10 @@ static void svm_range_drain_retry_fault(struct svm_range_list *svms) {struct kfd_process_device *pdd;struct kfd_process *p; -int drain;uint32_t i; p = container_of(svms, struct kfd_process, svms); -restart: - drain =atomic_read (&svms->drain_pagefaults); -if (!drain) -return ; -for_each_set_bit (i, svms->bitmap_supported, p->n_pdds) { pdd = p->pdds[i];if (!pdd)@@ -2292,8 +2286,6 @@ static void svm_range_drain_retry_fault(struct svm_range_list *svms)pr_debug ("drain retry fault gpu %d svms 0x%p done\n" , i, svms); } -if (atomic_cmpxchg (&svms->drain_pagefaults, drain,0 ) != drain) - goto restart; }static void svm_range_deferred_list_work (struct work_struct *work)@@ -2315,17 +2307,8 @@ static void svm_range_deferred_list_work(struct work_struct *work) prange->start, prange->last, prange->work_item.op); mm = prange->work_item.mm; -retry: -mmap_write_lock (mm); -/* Checking for the need to drain retry faults must be inside -* mmap write lock to serialize with munmap notifiers. -*/ -if (unlikely (atomic_read (&svms->drain_pagefaults))) { -mmap_write_unlock (mm); -svm_range_drain_retry_fault (svms); - goto retry; - } +mmap_write_lock (mm);/* Remove from deferred_list must be inside mmap write lock, for * two race cases: @@ -2446,6 +2429,7 @@ svm_range_unmap_from_cpu(struct mm_struct *mm, struct svm_range *prange,struct kfd_process *p;unsigned long s, l;bool unmap_parent; +uint32_t i; p =kfd_lookup_process_by_mm (mm);if (!p)@@ -2455,11 +2439,49 @@ svm_range_unmap_from_cpu(struct mm_struct *mm, struct svm_range *prange,pr_debug ("svms 0x%p prange 0x%p [0x%lx 0x%lx] [0x%lx 0x%lx]\n" , svms, prange, prange->start, prange->last, start, last);
Quoted 6 lines
drivers/gpu/drm/amd/amdkfd/kfd_svm.c svm_range_unmap_from_cpu(struct mm_struct *mm, struct svm_range *prange, · line 2443
Quoted 5 lines
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +2
-3-/* Make sure pending page faults are drained in the deferred worker -* before the range is freed to avoid straggler interrupts on -* unmapped memory causing "phantom faults". +/* first stop kfd page fault handling for this process */ +atomic_set (&svms->stop_pf_recovery,1 );this flag is not needed.
As above, this flag used here to not process page fault during time period that get current time stamp of each gpu IH ring.
Quoted 5 lines
Quoted 5 lines
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +3
-1+/* calculate time stamps that are used to decide which page faults need be +* dropped or handled before unmap pages from gpu vm */ -atomic_inc (&svms->drain_pagefaults); +for_each_set_bit (i, svms->bitmap_supported, p->n_pdds) {
Quoted 8 lines
drivers/gpu/drm/amd/amdkfd/kfd_svm.c svm_range_unmap_from_cpu(struct mm_struct *mm, struct svm_range *prange, · line 2454
Quoted 7 lines
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +7
-0+struct kfd_process_device *pdd; +struct amdgpu_device *adev; +struct amdgpu_ih_ring *ih; +uint32_t checkpoint_wptr; +uint64_t checkpoint_ts =0 ; + + svms->checkpoint_ts[i] =0 ;don't reset it to zero.
ok
Quoted 12 lines
Quoted 12 lines
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +12
-0+ pdd = p->pdds[i]; +if (!pdd) +continue ; + + adev = pdd->dev->adev; + ih = adev->irq.retry_cam_enabled ? &adev->irq.ih : &adev->irq.ih1; +if (!ih->enabled || adev->shutdown) +continue ; + + checkpoint_wptr =amdgpu_ih_get_wptr (adev, ih); +/* Order wptr with ring data. */ +rmb ();
Quoted 8 lines
drivers/gpu/drm/amd/amdkfd/kfd_svm.c svm_range_unmap_from_cpu(struct mm_struct *mm, struct svm_range *prange, · line 2474
Quoted 7 lines
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +7
-0+ checkpoint_ts =amdgpu_ih_decode_iv_ts (adev, ih, checkpoint_wptr,-1 ); + +if (adev->irq.retry_cam_enabled) { + ih = &adev->irq.ih_soft; + +if (!ih->enabled || adev->shutdown) +continue ;this check can be removed.
ok
Quoted 9 lines
drivers/gpu/drm/amd/amdkfd/kfd_svm.c svm_range_unmap_from_cpu(struct mm_struct *mm, struct svm_range *prange, · line 2481
Quoted 7 lines
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +7
-0+ + checkpoint_wptr =amdgpu_ih_get_wptr (adev, ih); +rmb (); + checkpoint_ts =max (checkpoint_ts, +amdgpu_ih_decode_iv_ts (adev, ih, checkpoint_wptr,-1 )); + } + svms->checkpoint_ts[i] = checkpoint_ts;
WRITE_ONCE(svms->checkpoint_ts[i], checkpoint_ts);
UseWRITE_ONCEhere andrestore_pageuseREAD_ONCEto sync up the timestamp update.
ok
Quoted 95 lines
Quoted 42 lines
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +18
-3+ } +/* resume kfd page fault handing with time stamp checking */ +atomic_set (&svms->stop_pf_recovery,0 ); unmap_parent = start <= prange->start && last >= prange->last;@@ -2900,7 +2922,7 @@ svm_fault_allowed(struct vm_area_struct *vma, bool write_fault)int svm_range_restore_pages (struct amdgpu_device *adev,unsigned int pasid,uint32_t vmid,uint32_t node_id, -uint64_t addr,bool write_fault) +uint64_t addr,uint64_t ts,bool write_fault) {unsigned long start, last, size;struct mm_struct *mm =NULL ;@@ -2910,7 +2932,7 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,ktime_t timestamp =ktime_get_boottime ();struct kfd_node *node;int32_t best_loc; -int32_t gpuidx =MAX_GPU_INSTANCE ; +int32_t gpuid, gpuidx =MAX_GPU_INSTANCE ;bool write_locked = false;struct vm_area_struct *vma;bool migration = false;@@ -2930,7 +2952,29 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,pr_debug ("restoring svms 0x%p fault address 0x%llx\n" , svms, addr); -if (atomic_read (&svms->drain_pagefaults)) { +/* kfd page fault recovery is disabled */ +if (atomic_read (&svms->stop_pf_recovery)) { +pr_debug ("page fault handing disabled, drop fault 0x%llx\n" , addr); + r =0 ; + goto out; + } + + node =kfd_node_by_irq_ids (adev, node_id, vmid); +if (!node) { +pr_debug ("kfd node does not exist node_id: %d, vmid: %d\n" , node_id, + vmid); + r = -EFAULT ; + goto out;drivers/gpu/drm/amd/amdkfd/kfd_svm.c svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid, · line 2977
Quoted 10 lines
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +10
-0+ } + +if (kfd_process_gpuid_from_node (p, node, &gpuid, &gpuidx)) { +pr_debug ("failed to get gpuid/gpuidex for node_id: %d \n" , node_id); + r = -EFAULT ; + goto out; + } + +/* check if this page fault time stamp is before svms->checkpoint_ts */ +if (amdgpu_ih_ts_after (ts, svms->checkpoint_ts[gpuidx])) {if (
amdgpu_ih_ts_after(ts, READ_ONCE(svms->checkpoint_ts[gpuidx]))) {Quoted 42 lines
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +4
-9pr_debug ("draining retry fault, drop fault 0x%llx\n" , addr); r =0 ; goto out;@@ -2952,13 +2996,6 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid, goto out; } - node =kfd_node_by_irq_ids (adev, node_id, vmid); -if (!node) { -pr_debug ("kfd node does not exist node_id: %d, vmid: %d\n" , node_id, - vmid); - r = -EFAULT ; - goto out; - }mmap_read_lock (mm); retry_write_locked:mutex_lock (&svms->lock);@@ -3173,9 +3210,11 @@ void svm_range_list_fini(struct kfd_process *p)/* * Ensure no retry fault comes in afterwards, as page fault handler will * not find kfd process and take mm lock to recover fault. +* stop kfd page fault handing, then wait pending page faults got drained */ -atomic_inc (&p->svms.drain_pagefaults); +atomic_set (&p->svms.stop_pf_recovery,1 );svm_range_drain_retry_fault (&p->svms); +atomic_set (&p->svms.stop_pf_recovery,0 );list_for_each_entry_safe (prange, next, &p->svms.list, list) {svm_range_unlink (prange);@@ -3197,16 +3236,18 @@ int svm_range_list_init(struct kfd_process *p)mutex_init (&svms->lock);INIT_LIST_HEAD (&svms->list);atomic_set (&svms->evicted_ranges,0 ); -atomic_set (&svms->drain_pagefaults,0 ); +atomic_set (&svms->stop_pf_recovery,0 );INIT_DELAYED_WORK (&svms->restore_work, svm_range_restore_work);INIT_WORK (&svms->deferred_list_work, svm_range_deferred_list_work);INIT_LIST_HEAD (&svms->deferred_range_list);INIT_LIST_HEAD (&svms->criu_svm_metadata_list);spin_lock_init (&svms->deferred_list_lock);
Quoted 6 lines
drivers/gpu/drm/amd/amdkfd/kfd_svm.c int svm_range_list_init(struct kfd_process *p) · line 3247
Quoted 3 lines
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +2
-1-for (i =0 ; i < p->n_pdds; i++) +for (i =0 ; i < p->n_pdds; i++) { + svms->checkpoint_ts[i] =0 ;not need to init 0 as
kfd_processstructure is from kzalloc.
ok
Regards
Xiaogang
Quoted 23 lines
Regards,
Philip
Quoted 19 lines
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +1
-0if (KFD_IS_SVM_API_SUPPORTED (p->pdds[i]->dev->adev))bitmap_set (svms->bitmap_supported, i,1 ); + }return 0 ; }drivers/gpu/drm/amd/amdkfd/kfd_svm.h +1
-1@@ -173,7 +173,7 @@ int svm_range_vram_node_new(struct kfd_node *node, struct svm_range *prange, bool clear);void svm_range_vram_node_free (struct svm_range *prange);int svm_range_restore_pages (struct amdgpu_device *adev,unsigned int pasid, -uint32_t vmid,uint32_t node_id,uint64_t addr, +uint32_t vmid,uint32_t node_id,uint64_t addr,uint64_t ts,bool write_fault);int svm_range_schedule_evict_svm_bo (struct amdgpu_amdkfd_fence *fence); void svm_range_add_list_work(struct svm_range_list *svms,
On 2024-07-24 09:58, Chen, Xiaogang wrote:
On 7/23/2024 4:02 PM, Philip Yang wrote:
On 2024-07-19 18:17, Xiaogang.Chen wrote:
From: Xiaogang Chen <[email protected]> When app unmap vm ranges(munmap) kfd/svm starts drain pending page fault and not handle any incoming pages fault of this process until a deferred work item got executed by default system wq. The time period of "not handle page fault" can be long and is unpredicable. That is advese to kfd performance on page faults recovery. This patch uses time stamp of incoming page page to decide to drop or handle page fault. When app unmap vm ranges kfd records each gpu device's ih ring current time stamp. These time stamps are used at kfd page fault recovery routine. Any page fault happens on unmapped ranges after unmap events is app bug that accesses vm range after unmap. It is not driver work to cover that. By using time stamp of page fault do not need drain page faults at deferred work. So, the time period that kfd does not handle page faults is reduced and can be controlled. This simplify the retry fault draining and support the multiple processes correctly now, some nitpick below.
ok.
return 1; } diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c index 350f6b6676f1..ac08d9424feb 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c @@ -595,7 +595,7 @@ static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev, cam_index = entry->src_data[2] & 0x3ff; ret = amdgpu_vm_handle_fault(adev, entry->pasid, entry->vmid, node_id, - addr, write_fault); + addr, entry->timestamp, write_fault); WDOORBELL32(adev->irq.retry_cam_doorbell_index, cam_index); if (ret) return 1; @@ -618,7 +618,7 @@ static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev, * tables */ if (amdgpu_vm_handle_fault(adev, entry->pasid, entry->vmid, node_id, - addr, write_fault)) + addr, entry->timestamp, write_fault)) return 1; } } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h index c51e908f6f19..8b8d5ab9da76 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h @@ -850,10 +850,13 @@ struct svm_range_list { struct list_head criu_svm_metadata_list; spinlock_t deferred_list_lock; atomic_t evicted_ranges; - atomic_t drain_pagefaults; + /* stop page fault recovery for this process */ + atomic_t stop_pf_recovery; This flag seems redundant, as timestamp is 48bit increasing and amdgpu_ih_ts_after handle the wrap around, so always use timestamp to drop retry fault. This flag notifies page fault restore routine to not process page fault for this process. It is used here when getting current time stamp at each gpu's IH ring, then unset it. To make sure page fault will not be processed during time period when get current time stamps, though it is very short.
don't need unset checkpoint_ts, what we need is to ensure updating checkpoint_ts with current wptr timestamp, and then free prange. If there is issue, probably because missing write_once/read_once or have to use atomic64_t for checkpoint_ts.
And this flag maybe used in future for general purpose that we need stop process page fault recovery for some reasons.
Then this could be another flag to stop retry fault recovery in future, not related to drain retry fault.
struct delayed_work restore_work; DECLARE_BITMAP(bitmap_supported, MAX_GPU_INSTANCE); struct task_struct *faulting_task; + /* check point ts decides if page fault recovery need be dropped */ + uint64_t checkpoint_ts[MAX_GPU_INSTANCE]; I think it is safe to not use atomic64_t.
not sure what you mean here: you do not want use uint64_4, or atomic64_4? I use unit64_4 here. And as you mention below, will use WRITE_ONCE/READ_ONCE on svms->checkpoint_ts[i]. uint64_t update is not multiple thread safe even on 64bit cpu, we should add mutex/spinlock or use atomic64_t. checkpoint_ts only update in mmu notifier unmap callback, this is serialized by mmap lock, so it is safe to use uint64_t.
Regards,
Philip
}; /* Process data */ diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c index 407636a68814..9dd28d06ea2a 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c @@ -2263,16 +2263,10 @@ static void svm_range_drain_retry_fault(struct svm_range_list *svms) { struct kfd_process_device *pdd; struct kfd_process *p; - int drain; uint32_t i; p = container_of(svms, struct kfd_process, svms); -restart: - drain = atomic_read(&svms->drain_pagefaults); - if (!drain) - return; - for_each_set_bit(i, svms->bitmap_supported, p->n_pdds) { pdd = p->pdds[i]; if (!pdd) @@ -2292,8 +2286,6 @@ static void svm_range_drain_retry_fault(struct svm_range_list *svms) pr_debug("drain retry fault gpu %d svms 0x%p done\n", i, svms); } - if (atomic_cmpxchg(&svms->drain_pagefaults, drain, 0) != drain) - goto restart; } static void svm_range_deferred_list_work(struct work_struct *work) @@ -2315,17 +2307,8 @@ static void svm_range_deferred_list_work(struct work_struct *work) prange->start, prange->last, prange->work_item.op); mm = prange->work_item.mm; -retry: - mmap_write_lock(mm); - /* Checking for the need to drain retry faults must be inside - * mmap write lock to serialize with munmap notifiers. - */ - if (unlikely(atomic_read(&svms->drain_pagefaults))) { - mmap_write_unlock(mm); - svm_range_drain_retry_fault(svms); - goto retry; - } + mmap_write_lock(mm); /* Remove from deferred_list must be inside mmap write lock, for * two race cases: @@ -2446,6 +2429,7 @@ svm_range_unmap_from_cpu(struct mm_struct *mm, struct svm_range *prange, struct kfd_process *p; unsigned long s, l; bool unmap_parent; + uint32_t i; p = kfd_lookup_process_by_mm(mm); if (!p) @@ -2455,11 +2439,49 @@ svm_range_unmap_from_cpu(struct mm_struct *mm, struct svm_range *prange, pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx] [0x%lx 0x%lx]\n", svms, prange, prange->start, prange->last, start, last); - /* Make sure pending page faults are drained in the deferred worker - * before the range is freed to avoid straggler interrupts on - * unmapped memory causing "phantom faults". + /* first stop kfd page fault handling for this process */ + atomic_set(&svms->stop_pf_recovery, 1); this flag is not needed.
As above, this flag used here to not process page fault during time period that get current time stamp of each gpu IH ring.
+ /* calculate time stamps that are used to decide which page faults need be + * dropped or handled before unmap pages from gpu vm */ - atomic_inc(&svms->drain_pagefaults); + for_each_set_bit(i, svms->bitmap_supported, p->n_pdds) { + struct kfd_process_device *pdd; + struct amdgpu_device *adev; + struct amdgpu_ih_ring *ih; + uint32_t checkpoint_wptr; + uint64_t checkpoint_ts = 0; + + svms->checkpoint_ts[i] = 0; don't reset it to zero.
ok
+ pdd = p->pdds[i]; + if (!pdd) + continue; + + adev = pdd->dev->adev; + ih = adev->irq.retry_cam_enabled ? &adev->irq.ih : &adev->irq.ih1; + if (!ih->enabled || adev->shutdown) + continue; + + checkpoint_wptr = amdgpu_ih_get_wptr(adev, ih); + /* Order wptr with ring data. */ + rmb(); + checkpoint_ts = amdgpu_ih_decode_iv_ts(adev, ih, checkpoint_wptr, -1); + + if (adev->irq.retry_cam_enabled) { + ih = &adev->irq.ih_soft; + + if (!ih->enabled || adev->shutdown) + continue; this check can be removed.
ok
+ + checkpoint_wptr = amdgpu_ih_get_wptr(adev, ih); + rmb(); + checkpoint_ts = max(checkpoint_ts, + amdgpu_ih_decode_iv_ts(adev, ih, checkpoint_wptr, -1)); + } + svms->checkpoint_ts[i] = checkpoint_ts; WRITE_ONCE(svms->checkpoint_ts[i], checkpoint_ts); Use WRITE_ONCE here and restore_page use READ_ONCE to sync up the timestamp update.
ok
+ } + /* resume kfd page fault handing with time stamp checking */ + atomic_set(&svms->stop_pf_recovery, 0); unmap_parent = start <= prange->start && last >= prange->last; @@ -2900,7 +2922,7 @@ svm_fault_allowed(struct vm_area_struct *vma, bool write_fault) int svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid, uint32_t vmid, uint32_t node_id, - uint64_t addr, bool write_fault) + uint64_t addr, uint64_t ts, bool write_fault) { unsigned long start, last, size; struct mm_struct *mm = NULL; @@ -2910,7 +2932,7 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid, ktime_t timestamp = ktime_get_boottime(); struct kfd_node *node; int32_t best_loc; - int32_t gpuidx = MAX_GPU_INSTANCE; + int32_t gpuid, gpuidx = MAX_GPU_INSTANCE; bool write_locked = false; struct vm_area_struct *vma; bool migration = false; @@ -2930,7 +2952,29 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid, pr_debug("restoring svms 0x%p fault address 0x%llx\n", svms, addr); - if (atomic_read(&svms->drain_pagefaults)) { + /* kfd page fault recovery is disabled */ + if (atomic_read(&svms->stop_pf_recovery)) { + pr_debug("page fault handing disabled, drop fault 0x%llx\n", addr); + r = 0; + goto out; + } + + node = kfd_node_by_irq_ids(adev, node_id, vmid); + if (!node) { + pr_debug("kfd node does not exist node_id: %d, vmid: %d\n", node_id, + vmid); + r = -EFAULT; + goto out; + } + + if (kfd_process_gpuid_from_node(p, node, &gpuid, &gpuidx)) { + pr_debug("failed to get gpuid/gpuidex for node_id: %d \n", node_id); + r = -EFAULT; + goto out; + } + + /* check if this page fault time stamp is before svms->checkpoint_ts */ + if (amdgpu_ih_ts_after(ts, svms->checkpoint_ts[gpuidx])) { if (amdgpu_ih_ts_after(ts, READ_ONCE(svms->checkpoint_ts[gpuidx]))) { pr_debug("draining retry fault, drop fault 0x%llx\n", addr); r = 0; goto out; @@ -2952,13 +2996,6 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid, goto out; } - node = kfd_node_by_irq_ids(adev, node_id, vmid); - if (!node) { - pr_debug("kfd node does not exist node_id: %d, vmid: %d\n", node_id, - vmid); - r = -EFAULT; - goto out; - } mmap_read_lock(mm); retry_write_locked: mutex_lock(&svms->lock); @@ -3173,9 +3210,11 @@ void svm_range_list_fini(struct kfd_process *p) /* * Ensure no retry fault comes in afterwards, as page fault handler will * not find kfd process and take mm lock to recover fault. + * stop kfd page fault handing, then wait pending page faults got drained */ - atomic_inc(&p->svms.drain_pagefaults); + atomic_set(&p->svms.stop_pf_recovery, 1); svm_range_drain_retry_fault(&p->svms); + atomic_set(&p->svms.stop_pf_recovery, 0); list_for_each_entry_safe(prange, next, &p->svms.list, list) { svm_range_unlink(prange); @@ -3197,16 +3236,18 @@ int svm_range_list_init(struct kfd_process *p) mutex_init(&svms->lock); INIT_LIST_HEAD(&svms->list); atomic_set(&svms->evicted_ranges, 0); - atomic_set(&svms->drain_pagefaults, 0); + atomic_set(&svms->stop_pf_recovery, 0); INIT_DELAYED_WORK(&svms->restore_work, svm_range_restore_work); INIT_WORK(&svms->deferred_list_work, svm_range_deferred_list_work); INIT_LIST_HEAD(&svms->deferred_range_list); INIT_LIST_HEAD(&svms->criu_svm_metadata_list); spin_lock_init(&svms->deferred_list_lock); - for (i = 0; i < p->n_pdds; i++) + for (i = 0; i < p->n_pdds; i++) { + svms->checkpoint_ts[i] = 0; not need to init 0 as kfd_process structure is from kzalloc. ok
Regards
Xiaogang
Regards,
Philip
if (KFD_IS_SVM_API_SUPPORTED(p->pdds[i]->dev->adev)) bitmap_set(svms->bitmap_supported, i, 1); + } return 0; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h index 70c1776611c4..5f447c3642cb 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h @@ -173,7 +173,7 @@ int svm_range_vram_node_new(struct kfd_node *node, struct svm_range *prange, bool clear); void svm_range_vram_node_free(struct svm_range *prange); int svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid, - uint32_t vmid, uint32_t node_id, uint64_t addr, + uint32_t vmid, uint32_t node_id, uint64_t addr, uint64_t ts, bool write_fault); int svm_range_schedule_evict_svm_bo(struct amdgpu_amdkfd_fence *fence); void svm_range_add_list_work(struct svm_range_list *svms,
Felix Kuehling
maintainerQuoted 179 lines · Xiaogang.Chen
From: Xiaogang Chen <[email protected]>
When app unmap vm ranges(munmap) kfd/svm starts drain pending page fault and not handle any incoming pages fault of this process until a deferred work item got executed by default system wq. The time period of "not handle page fault" can be long and is unpredicable. That is advese to kfd performance on page faults recovery.
This patch uses time stamp of incoming page page to decide to drop or handle page fault. When app unmap vm ranges kfd records each gpu device's ih ring current time stamp. These time stamps are used at kfd page fault recovery routine.
Any page fault happens on unmapped ranges after unmap events is app bug that accesses vm range after unmap. It is not driver work to cover that.
By using time stamp of page fault do not need drain page faults at deferred work. So, the time period that kfd does not handle page faults is reduced and can be controlled.
Signed-off-by Xiaogang.Chen <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 4 +- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 2 +- drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c | 3 +- drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c | 4 +- drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 5 +- drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 111 +++++++++++++++++-------- drivers/gpu/drm/amd/amdkfd/kfd_svm.h | 2 +- 7 files changed, 88 insertions(+), 43 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +2
-2@@ -2763,7 +2763,7 @@ int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)* shouldn't be reported any more. */ bool amdgpu_vm_handle_fault (struct amdgpu_device *adev,u32 pasid, -u32 vmid,u32 node_id,uint64_t addr, +u32 vmid,u32 node_id,uint64_t addr,uint64_t ts,bool write_fault) {bool is_compute_context = false;@@ -2789,7 +2789,7 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid, addr /=AMDGPU_GPU_PAGE_SIZE ;if (is_compute_context && !svm_range_restore_pages (adev, pasid, vmid, - node_id, addr, write_fault)) { + node_id, addr, ts, write_fault)) {amdgpu_bo_unref (&root);return true; }drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +1
-1@@ -548,7 +548,7 @@ amdgpu_vm_get_task_info_vm(struct amdgpu_vm *vm);void amdgpu_vm_put_task_info (struct amdgpu_task_info *task_info);bool amdgpu_vm_handle_fault (struct amdgpu_device *adev,u32 pasid, -u32 vmid,u32 node_id,uint64_t addr, +u32 vmid,u32 node_id,uint64_t addr,uint64_t ts,bool write_fault);void amdgpu_vm_set_task_info (struct amdgpu_vm *vm);drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c +2
-1@@ -132,7 +132,8 @@ static int gmc_v10_0_process_interrupt(struct amdgpu_device *adev,/* Try to handle the recoverable page faults by filling page * tables */ -if (amdgpu_vm_handle_fault (adev, entry->pasid,0 ,0 , addr, write_fault)) +if (amdgpu_vm_handle_fault (adev, entry->pasid,0 ,0 , addr, + entry->timestamp, write_fault))return 1 ; }drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +2
-2@@ -595,7 +595,7 @@ static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev, cam_index = entry->src_data[2 ] &0x3ff ; ret =amdgpu_vm_handle_fault (adev, entry->pasid, entry->vmid, node_id, - addr, write_fault); + addr, entry->timestamp, write_fault);WDOORBELL32 (adev->irq.retry_cam_doorbell_index, cam_index);if (ret)return 1 ;@@ -618,7 +618,7 @@ static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev,* tables */ if (amdgpu_vm_handle_fault (adev, entry->pasid, entry->vmid, node_id, - addr, write_fault)) + addr, entry->timestamp, write_fault))return 1 ; } }drivers/gpu/drm/amd/amdkfd/kfd_priv.h +4
-1@@ -850,10 +850,13 @@ struct svm_range_list {struct list_head criu_svm_metadata_list;spinlock_t deferred_list_lock;atomic_t evicted_ranges; -atomic_t drain_pagefaults; +/* stop page fault recovery for this process */ +atomic_t stop_pf_recovery;struct delayed_work restore_work;DECLARE_BITMAP (bitmap_supported,MAX_GPU_INSTANCE );struct task_struct *faulting_task; +/* check point ts decides if page fault recovery need be dropped */ +uint64_t checkpoint_ts[MAX_GPU_INSTANCE ]; };/* Process data */ drivers/gpu/drm/amd/amdkfd/kfd_svm.c +2
-18@@ -2263,16 +2263,10 @@ static void svm_range_drain_retry_fault(struct svm_range_list *svms) {struct kfd_process_device *pdd;struct kfd_process *p; -int drain;uint32_t i; p = container_of(svms, struct kfd_process, svms); -restart: - drain =atomic_read (&svms->drain_pagefaults); -if (!drain) -return ; -for_each_set_bit (i, svms->bitmap_supported, p->n_pdds) { pdd = p->pdds[i];if (!pdd)@@ -2292,8 +2286,6 @@ static void svm_range_drain_retry_fault(struct svm_range_list *svms)pr_debug ("drain retry fault gpu %d svms 0x%p done\n" , i, svms); } -if (atomic_cmpxchg (&svms->drain_pagefaults, drain,0 ) != drain) - goto restart; }static void svm_range_deferred_list_work (struct work_struct *work)@@ -2315,17 +2307,8 @@ static void svm_range_deferred_list_work(struct work_struct *work) prange->start, prange->last, prange->work_item.op); mm = prange->work_item.mm; -retry: -mmap_write_lock (mm); -/* Checking for the need to drain retry faults must be inside -* mmap write lock to serialize with munmap notifiers. -*/ -if (unlikely (atomic_read (&svms->drain_pagefaults))) { -mmap_write_unlock (mm); -svm_range_drain_retry_fault (svms); - goto retry; - } +mmap_write_lock (mm);/* Remove from deferred_list must be inside mmap write lock, for * two race cases: @@ -2446,6 +2429,7 @@ svm_range_unmap_from_cpu(struct mm_struct *mm, struct svm_range *prange,struct kfd_process *p;unsigned long s, l;bool unmap_parent; +uint32_t i; p =kfd_lookup_process_by_mm (mm);if (!p)
drivers/gpu/drm/amd/amdkfd/kfd_svm.c svm_range_unmap_from_cpu(struct mm_struct *mm, struct svm_range *prange, · line 2443
Quoted 8 lines
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +2
-3@@ -2455,11 +2439,49 @@ svm_range_unmap_from_cpu(struct mm_struct *mm, struct svm_range *prange,pr_debug ("svms 0x%p prange 0x%p [0x%lx 0x%lx] [0x%lx 0x%lx]\n" , svms, prange, prange->start, prange->last, start, last); -/* Make sure pending page faults are drained in the deferred worker -* before the range is freed to avoid straggler interrupts on -* unmapped memory causing "phantom faults". +/* first stop kfd page fault handling for this process */ +atomic_set (&svms->stop_pf_recovery,1 );
Why is this needed?
Quoted 8 lines
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +6
-1+/* calculate time stamps that are used to decide which page faults need be +* dropped or handled before unmap pages from gpu vm */ -atomic_inc (&svms->drain_pagefaults); +for_each_set_bit (i, svms->bitmap_supported, p->n_pdds) { +struct kfd_process_device *pdd; +struct amdgpu_device *adev; +struct amdgpu_ih_ring *ih;
drivers/gpu/drm/amd/amdkfd/kfd_svm.c svm_range_unmap_from_cpu(struct mm_struct *mm, struct svm_range *prange, · line 2460
Quoted 10 lines
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +10
-0+uint32_t checkpoint_wptr; +uint64_t checkpoint_ts =0 ; + + svms->checkpoint_ts[i] =0 ; + pdd = p->pdds[i]; +if (!pdd) +continue ; + + adev = pdd->dev->adev; + ih = adev->irq.retry_cam_enabled ? &adev->irq.ih : &adev->irq.ih1;
Rerouting interrupts to ih1 doesn't work on all GPUs. I think a better heuristic to determine which ih ring to drain is to look at whether ih1 is empty. If it is empty, drain ih_soft instead. If both are empty, there is nothing to drain and you skip this.
Quoted 9 lines
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +9
-0+ +if (!ih->enabled || adev->shutdown) +continue ; + + checkpoint_wptr =amdgpu_ih_get_wptr (adev, ih); +/* Order wptr with ring data. */ +rmb (); + checkpoint_ts =amdgpu_ih_decode_iv_ts (adev, ih, checkpoint_wptr,-1 ); +
drivers/gpu/drm/amd/amdkfd/kfd_svm.c svm_range_unmap_from_cpu(struct mm_struct *mm, struct svm_range *prange, · line 2479
Quoted 10 lines
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +10
-0+if (adev->irq.retry_cam_enabled) { + ih = &adev->irq.ih_soft; + +if (!ih->enabled || adev->shutdown) +continue ; + + checkpoint_wptr =amdgpu_ih_get_wptr (adev, ih); +rmb (); + checkpoint_ts =max (checkpoint_ts, +amdgpu_ih_decode_iv_ts (adev, ih, checkpoint_wptr,-1 ));
This won't work correctly when the timestamp wraps around. Use amdgpu_ih_ts_after to compare timestamps in a way that works with wraparound (as long as the timestamps aren't too far apart).
That said, I don't think you need to look at two IH rings. It should be enough to look at one IH ring, either IH1 or IH_SOFT. svm_range_restore_pages never processes interrupts from IH because it always runs in a worker thread that looks at one of the other IH rings.
Quoted 44 lines
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +20
-3+ } + svms->checkpoint_ts[i] = checkpoint_ts; + } +/* resume kfd page fault handing with time stamp checking */ +atomic_set (&svms->stop_pf_recovery,0 ); unmap_parent = start <= prange->start && last >= prange->last;@@ -2900,7 +2922,7 @@ svm_fault_allowed(struct vm_area_struct *vma, bool write_fault)int svm_range_restore_pages (struct amdgpu_device *adev,unsigned int pasid,uint32_t vmid,uint32_t node_id, -uint64_t addr,bool write_fault) +uint64_t addr,uint64_t ts,bool write_fault) {unsigned long start, last, size;struct mm_struct *mm =NULL ;@@ -2910,7 +2932,7 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,ktime_t timestamp =ktime_get_boottime ();struct kfd_node *node;int32_t best_loc; -int32_t gpuidx =MAX_GPU_INSTANCE ; +int32_t gpuid, gpuidx =MAX_GPU_INSTANCE ;bool write_locked = false;struct vm_area_struct *vma;bool migration = false;@@ -2930,7 +2952,29 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,pr_debug ("restoring svms 0x%p fault address 0x%llx\n" , svms, addr); -if (atomic_read (&svms->drain_pagefaults)) { +/* kfd page fault recovery is disabled */ +if (atomic_read (&svms->stop_pf_recovery)) { +pr_debug ("page fault handing disabled, drop fault 0x%llx\n" , addr); + r =0 ; + goto out; + } + + node =kfd_node_by_irq_ids (adev, node_id, vmid); +if (!node) { +pr_debug ("kfd node does not exist node_id: %d, vmid: %d\n" , node_id, + vmid); + r = -EFAULT ; + goto out;
drivers/gpu/drm/amd/amdkfd/kfd_svm.c svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid, · line 2977
Quoted 10 lines
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +10
-0+ } + +if (kfd_process_gpuid_from_node (p, node, &gpuid, &gpuidx)) { +pr_debug ("failed to get gpuid/gpuidex for node_id: %d \n" , node_id); + r = -EFAULT ; + goto out; + } + +/* check if this page fault time stamp is before svms->checkpoint_ts */ +if (amdgpu_ih_ts_after (ts, svms->checkpoint_ts[gpuidx])) {
When we're not draining page faults, you should set svms->checkpoint_ts[gpuidx] to some invalid value to avoid comparing ts against an outdated timestamp. If it has the invalid value, skip the draining check here. Otherwise, when ts wraps around, you will get the wrong result and drain faults when you shouldn't.
Regards,
Felix
Quoted 64 lines
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +7
-10pr_debug ("draining retry fault, drop fault 0x%llx\n" , addr); r =0 ; goto out;@@ -2952,13 +2996,6 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid, goto out; } - node =kfd_node_by_irq_ids (adev, node_id, vmid); -if (!node) { -pr_debug ("kfd node does not exist node_id: %d, vmid: %d\n" , node_id, - vmid); - r = -EFAULT ; - goto out; - }mmap_read_lock (mm); retry_write_locked:mutex_lock (&svms->lock);@@ -3173,9 +3210,11 @@ void svm_range_list_fini(struct kfd_process *p)/* * Ensure no retry fault comes in afterwards, as page fault handler will * not find kfd process and take mm lock to recover fault. +* stop kfd page fault handing, then wait pending page faults got drained */ -atomic_inc (&p->svms.drain_pagefaults); +atomic_set (&p->svms.stop_pf_recovery,1 );svm_range_drain_retry_fault (&p->svms); +atomic_set (&p->svms.stop_pf_recovery,0 );list_for_each_entry_safe (prange, next, &p->svms.list, list) {svm_range_unlink (prange);@@ -3197,16 +3236,18 @@ int svm_range_list_init(struct kfd_process *p)mutex_init (&svms->lock);INIT_LIST_HEAD (&svms->list);atomic_set (&svms->evicted_ranges,0 ); -atomic_set (&svms->drain_pagefaults,0 ); +atomic_set (&svms->stop_pf_recovery,0 );INIT_DELAYED_WORK (&svms->restore_work, svm_range_restore_work);INIT_WORK (&svms->deferred_list_work, svm_range_deferred_list_work);INIT_LIST_HEAD (&svms->deferred_range_list);INIT_LIST_HEAD (&svms->criu_svm_metadata_list);spin_lock_init (&svms->deferred_list_lock); -for (i =0 ; i < p->n_pdds; i++) +for (i =0 ; i < p->n_pdds; i++) { + svms->checkpoint_ts[i] =0 ;if (KFD_IS_SVM_API_SUPPORTED (p->pdds[i]->dev->adev))bitmap_set (svms->bitmap_supported, i,1 ); + }return 0 ; }drivers/gpu/drm/amd/amdkfd/kfd_svm.h +1
-1@@ -173,7 +173,7 @@ int svm_range_vram_node_new(struct kfd_node *node, struct svm_range *prange, bool clear);void svm_range_vram_node_free (struct svm_range *prange);int svm_range_restore_pages (struct amdgpu_device *adev,unsigned int pasid, -uint32_t vmid,uint32_t node_id,uint64_t addr, +uint32_t vmid,uint32_t node_id,uint64_t addr,uint64_t ts,bool write_fault);int svm_range_schedule_evict_svm_bo (struct amdgpu_amdkfd_fence *fence); void svm_range_add_list_work(struct svm_range_list *svms,