drm/amdkfd: Evict SVM BOs synchronously from TTM eviction
Subsystems (3)
PATCH 1/2 drm/amdkfd: Evict SVM BOs synchronously from TTM eviction
svm_range_evict_svm_bo_worker() migrated an SVM BO's pages back to system memory from a work item that took mmap_read_lock. When an mmap writer was pending, that read lock blocked behind the writer while the thread allocating a new migration VRAM BO waited on this BO's eviction fence - a circular wait that hung the SVM workers.
Evict the SVM BO synchronously from the TTM eviction path (amdgpu_ttm_bo_eviction_valuable) instead of deferring to a work item. The BO is already reserved and the lock order is mmap_lock -> BO reservation, so only trylock the owning process's mmap lock; on contention return -EBUSY so TTM skips this BO. This removes the eviction work item and the enable_signaling path, so no worker can block on mmap_read_lock.
The SVM BO uses AMDGPU_GEM_CREATE_DISCARDABLE, so ttm_bo_evict takes the pipeline_gutting path and skips allocating a system memory placement. That would be wasted work, since svm_migrate_vram_to_ram allocates the system pages and copies the data back itself.
Eviction now migrates ranges directly, so it must serialize with the owning process: it trylocks migrate_mutex under svm_bo->list_lock before unlinking the range, and svm_range_free() unlinks the range then waits on migrate_mutex, so a concurrent eviction cannot free a range under it.
Drop the mm reference with mmput_async so exit_mmap() does not run under the BO reservation.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 22 +++++ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 1 + .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c | 3 - drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 16 ++++ drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 88 ++++++++++++------- drivers/gpu/drm/amd/amdkfd/kfd_svm.h | 10 +-- 6 files changed, 100 insertions(+), 40 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +22 -0
@@ -39,6 +39,7 @@#if IS_ENABLED (CONFIG_HSA_AMD )#include "kfd_priv.h" #endif +#include "kfd_svm.h" /* Total memory size in system memory and all GPU VRAM. Used to * estimate worst case amount of memory to reserve for page tables
@@ -972,3 +973,24 @@ int amdgpu_amdkfd_reset_mes_queue(struct amdgpu_device *adev,return kgd2kfd_reset_mes_queue (adev->kfd.dev, node_id, queue_type, pipe, queue, db); } + +int amdgpu_amdkfd_evict_svm_bo (struct amdgpu_bo *bo) +{ +struct dma_resv_iter cursor; +struct dma_fence *fence; +int r =0 ; + +dma_resv_iter_begin (&cursor, bo->tbo.base.resv,DMA_RESV_USAGE_BOOKKEEP ); +dma_resv_for_each_fence_unlocked (&cursor, fence) { +struct amdgpu_amdkfd_fence *f =to_amdgpu_amdkfd_fence (fence); + +if (f && f->svm_bo) { + r =svm_range_evict_svm_bo (f->svm_bo); +if (r) +break ; + } + } +dma_resv_iter_end (&cursor); + +return r; +}
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +1 -0
@@ -286,6 +286,7 @@ int amdgpu_amdkfd_reset_mes_queue(struct amdgpu_device *adev,int queue_type, int pipe, int queue, unsigned int db); +int amdgpu_amdkfd_evict_svm_bo (struct amdgpu_bo *bo);/* Read user wptr from a specified user address space with page fault * disabled. The memory must be pinned and mapped to the hardware when
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c +0 -3
@@ -135,9 +135,6 @@ static bool amdkfd_fence_enable_signaling(struct dma_fence *f)if (!fence->svm_bo) {if (!kgd2kfd_schedule_evict_and_restore_process (fence->mm, fence->context_id, f))return true; - }else { -if (!svm_range_schedule_evict_svm_bo (fence)) -return true; }return false; }
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +16 -0
@@ -1488,6 +1488,7 @@ static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo, const struct ttm_place *place) {struct dma_resv_iter resv_cursor; +struct amdgpu_bo *abo;struct dma_fence *f;if (!amdgpu_bo_is_amdgpu_bo (bo))
@@ -1497,6 +1498,21 @@ static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,if (bo->resource->mem_type ==TTM_PL_SYSTEM )return true; + abo =ttm_to_amdgpu_bo (bo); +if (abo->flags &AMDGPU_GEM_CREATE_DISCARDABLE ) { +/* +* SVM BOs are migrated to system memory synchronously in this +* TTM eviction context. The migration needs the owning +* process's mmap lock, but the normal lock order is +* mmap_lock -> BO reservation and the BO is already reserved +* here. svm_range_evict_svm_bo() only trylocks the mmap lock; +* if the eviction fails for any reason, we return false so TTM +* skips this BO instead of risking a deadlock. +*/ +if (amdgpu_amdkfd_evict_svm_bo (abo) <0 ) +return false; + } +if (bo->type == ttm_bo_type_kernel && !amdgpu_vm_evictable (ttm_to_amdgpu_bo (bo)))return false;
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +57 -31
151 lines not on this page · raw message
drivers/gpu/drm/amd/amdkfd/kfd_svm.h +4 -6
@@ -44,7 +44,6 @@ struct svm_range_bo {struct list_head range_list;/* all svm ranges shared this bo */ spinlock_t list_lock;struct amdgpu_amdkfd_fence *eviction_fence; -struct work_struct eviction_work;uint32_t evicting;struct work_struct release_work;struct kfd_node *node;
@@ -175,7 +174,8 @@ void svm_range_vram_node_free(struct svm_range *prange);int svm_range_restore_pages (struct amdgpu_device *adev,unsigned int pasid,uint32_t vmid,uint32_t node_id,uint64_t addr,uint64_t ts,bool write_fault); -int svm_range_schedule_evict_svm_bo (struct amdgpu_amdkfd_fence *fence); +int svm_range_evict_svm_bo (struct svm_range_bo *svm_bo); +void svm_range_add_list_work (struct svm_range_list *svms,struct svm_range *prange,struct mm_struct *mm,enum svm_work_list_ops op);
@@ -229,11 +229,9 @@ static inline int svm_range_restore_pages(struct amdgpu_device *adev,return -EFAULT ; } -static inline int svm_range_schedule_evict_svm_bo ( -struct amdgpu_amdkfd_fence *fence) +static inline int svm_range_evict_svm_bo (struct svm_range_bo *svm_bo) { -WARN_ONCE (1 ,"SVM eviction fence triggered, but SVM is disabled" ); -return -EINVAL ; +return 0 ; } static inline void svm_range_get_info(struct kfd_process *p,
Signature · 1 line
2.50.1
PATCH 2/2 drm/amdkfd: Remove svm_bo eviction fence
SVM BOs are now migrated back to system memory synchronously from the TTM eviction path (svm_range_evict_svm_bo), so the per-svm_bo eviction fence is no longer used.
Remove the eviction fence from svm_range_bo, drop the amdgpu_amdkfd_fence->svm_bo back pointer and the amdgpu_amdkfd_evict_svm_bo() helper, and stop special-casing svm_bo fences in the KFD fence enable_signaling and check_mm paths. Embed struct amdgpu_bo directly in svm_range_bo with a dedicated svm_range_bo_destroy() callback, and keep the owning mm via mmgrab()/mmdrop() instead of through the fence.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 21 ---- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 3 - .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c | 15 +-- .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 4 +- drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 4 +- drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 6 +- drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 4 +- drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 109 +++++++----------- drivers/gpu/drm/amd/amdkfd/kfd_svm.h | 13 ++- 9 files changed, 64 insertions(+), 115 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +0 -21
@@ -973,24 +973,3 @@ int amdgpu_amdkfd_reset_mes_queue(struct amdgpu_device *adev,return kgd2kfd_reset_mes_queue (adev->kfd.dev, node_id, queue_type, pipe, queue, db); } - -int amdgpu_amdkfd_evict_svm_bo (struct amdgpu_bo *bo) -{ -struct dma_resv_iter cursor; -struct dma_fence *fence; -int r =0 ; - -dma_resv_iter_begin (&cursor, bo->tbo.base.resv,DMA_RESV_USAGE_BOOKKEEP ); -dma_resv_for_each_fence_unlocked (&cursor, fence) { -struct amdgpu_amdkfd_fence *f =to_amdgpu_amdkfd_fence (fence); - -if (f && f->svm_bo) { - r =svm_range_evict_svm_bo (f->svm_bo); -if (r) -break ; - } - } -dma_resv_iter_end (&cursor); - -return r; -}
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +0 -3
@@ -99,7 +99,6 @@ struct amdgpu_amdkfd_fence {struct mm_struct *mm;spinlock_t lock;char timeline_name[TASK_COMM_LEN ]; -struct svm_range_bo *svm_bo;uint16_t context_id; };
@@ -194,7 +193,6 @@ int amdgpu_queue_mask_bit_to_set_resource_bit(struct amdgpu_device *adev,struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create (u64 context,struct mm_struct *mm, -struct svm_range_bo *svm_bo,u16 context_id);int amdgpu_amdkfd_drm_client_create (struct amdgpu_device *adev);
@@ -286,7 +284,6 @@ int amdgpu_amdkfd_reset_mes_queue(struct amdgpu_device *adev,int queue_type, int pipe, int queue, unsigned int db); -int amdgpu_amdkfd_evict_svm_bo (struct amdgpu_bo *bo);/* Read user wptr from a specified user address space with page fault * disabled. The memory must be pinned and mapped to the hardware when
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c +3 -12
@@ -62,7 +62,6 @@ static atomic_t fence_seq = ATOMIC_INIT(0);struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create (u64 context,struct mm_struct *mm, -struct svm_range_bo *svm_bo,u16 context_id) {struct amdgpu_amdkfd_fence *fence;
@@ -76,7 +75,6 @@ struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context, fence->mm = mm;get_task_comm (fence->timeline_name, current);spin_lock_init (&fence->lock); - fence->svm_bo = svm_bo; fence->context_id = context_id;dma_fence_init (&fence->base, &amdkfd_fence_ops, &fence->lock, context,atomic_inc_return (&fence_seq));
@@ -128,14 +126,8 @@ static bool amdkfd_fence_enable_signaling(struct dma_fence *f)if (dma_fence_is_signaled (f))return true; -/* if fence->svm_bo is NULL, means this fence is created through -* init_kfd_vm() or amdgpu_amdkfd_gpuvm_restore_process_bos(). -* Therefore, this fence is amdgpu_amdkfd_fence->eviction_fence. -*/ -if (!fence->svm_bo) { -if (!kgd2kfd_schedule_evict_and_restore_process (fence->mm, fence->context_id, f)) -return true; - } +if (!kgd2kfd_schedule_evict_and_restore_process (fence->mm, fence->context_id, f)) +return true;return false; }
@@ -169,7 +161,6 @@ static void amdkfd_fence_release(struct dma_fence *f)* * Check if @mm is same as that of the fence @f, if same return TRUE else * return FALSE. -* For svm bo, which support vram overcommitment, always return FALSE. */ bool amdkfd_fence_check_mm (struct dma_fence *f,struct mm_struct *mm) {
@@ -177,7 +168,7 @@ bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm)if (!fence)return false; -else if (fence->mm == mm && !fence->svm_bo) +else if (fence->mm == mm)return true;return false;
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +2 -2
@@ -1423,7 +1423,7 @@ static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info, info->eviction_fence =amdgpu_amdkfd_fence_create (dma_fence_context_alloc (1 ), current->mm, -NULL , process->context_id); + process->context_id);if (!info->eviction_fence) {pr_err ("Failed to create eviction fence\n" ); ret = -ENOMEM ;
@@ -3093,7 +3093,7 @@ int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence __rcu *amdgpu_amdkfd_fence_create ( process_info->eviction_fence->base.context, process_info->eviction_fence->mm, -NULL , process_info->context_id); + process_info->context_id);if (!new_fence) {pr_err ("Failed to create eviction fence\n" );
drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +3 -1
@@ -43,6 +43,7 @@#include "amdgpu_vram_mgr.h" #include "amdgpu_vm.h" #include "amdgpu_dma_buf.h" +#include "kfd_svm.h" /** * DOC: amdgpu_object
@@ -93,7 +94,8 @@ static void amdgpu_bo_user_destroy(struct ttm_buffer_object *tbo)bool amdgpu_bo_is_amdgpu_bo (struct ttm_buffer_object *bo) {if (bo->destroy == &amdgpu_bo_destroy || - bo->destroy == &amdgpu_bo_user_destroy) + bo->destroy == &amdgpu_bo_user_destroy || + bo->destroy == &svm_range_bo_destroy)return true;return false;
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +4 -2
@@ -60,6 +60,7 @@#include "amdgpu_atomfirmware.h" #include "amdgpu_res_cursor.h" #include "bif/bif_4_1_d.h" +#include "kfd_svm.h" MODULE_IMPORT_NS ("DMA_BUF" );
@@ -1499,7 +1500,8 @@ static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,return true; abo =ttm_to_amdgpu_bo (bo); -if (abo->flags &AMDGPU_GEM_CREATE_DISCARDABLE ) { +if ((abo->flags &AMDGPU_GEM_CREATE_DISCARDABLE ) && + bo->destroy == &svm_range_bo_destroy) {/* * SVM BOs are migrated to system memory synchronously in this * TTM eviction context. The migration needs the owning
@@ -1509,7 +1511,7 @@ static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,* if the eviction fails for any reason, we return false so TTM * skips this BO instead of risking a deadlock. */ -if (amdgpu_amdkfd_evict_svm_bo (abo) <0 ) +if (svm_range_evict_svm_bo (abo) <0 )return false; }
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +2 -2
@@ -954,12 +954,12 @@ static vm_fault_t svm_migrate_to_ram(struct vm_fault *vmf)pr_debug ("failed get device page at addr 0x%lx\n" , addr);return VM_FAULT_SIGBUS ; } -if (!mmget_not_zero (svm_bo->eviction_fence->mm)) { +if (!mmget_not_zero (svm_bo->mm)) {pr_debug ("addr 0x%lx of process mm is destroyed\n" , addr);return VM_FAULT_SIGBUS ; } - mm = svm_bo->eviction_fence->mm; + mm = svm_bo->mm;if (mm != vmf->vma->vm_mm)pr_debug ("addr 0x%lx is COW mapping in child process\n" , addr);
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +41 -68
228 lines not on this page · raw message
drivers/gpu/drm/amd/amdkfd/kfd_svm.h +9 -4
@@ -39,11 +39,11 @@ ((adev)->hive ? (void *)(adev)->hive : (void *)(adev))struct svm_range_bo { -struct amdgpu_bo *bo; +struct amdgpu_bo bo;struct kref kref;struct list_head range_list;/* all svm ranges shared this bo */ spinlock_t list_lock; -struct amdgpu_amdkfd_fence *eviction_fence; +struct mm_struct *mm;uint32_t evicting;struct work_struct release_work;struct kfd_node *node;
@@ -168,13 +168,14 @@ struct svm_range *svm_range_from_addr(struct svm_range_list *svms, struct svm_range **parent);struct kfd_node *svm_range_get_node_by_id (struct svm_range *prange,uint32_t gpu_id); +void svm_range_bo_destroy (struct ttm_buffer_object *tbo);int svm_range_vram_node_new (struct kfd_node *node,struct svm_range *prange,bool clear);void svm_range_vram_node_free (struct svm_range *prange);int svm_range_restore_pages (struct amdgpu_device *adev,unsigned int pasid,uint32_t vmid,uint32_t node_id,uint64_t addr,uint64_t ts,bool write_fault); -int svm_range_evict_svm_bo (struct svm_range_bo *svm_bo); +int svm_range_evict_svm_bo (struct amdgpu_bo *bo); void svm_range_add_list_work(struct svm_range_list *svms, struct svm_range *prange, struct mm_struct *mm,
@@ -229,7 +230,11 @@ static inline int svm_range_restore_pages(struct amdgpu_device *adev,return -EFAULT ; } -static inline int svm_range_evict_svm_bo (struct svm_range_bo *svm_bo) +static inline void svm_range_bo_destroy (struct ttm_buffer_object *tbo) +{ +} + +static inline int svm_range_evict_svm_bo (struct amdgpu_bo *bo) {return 0 ; }
Signature · 1 line
2.50.1