Skip to contentSkip to conversation

drm/amdkfd: Evict SVM BOs synchronously from TTM eviction

PATCH New Philip Yang 1 participant 2 messages Jul 27

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

PATCH 1/2 drm/amdkfd: Evict SVM BOs synchronously from TTM eviction

svm_range_evict_svm_bo_worker() migrated an SVM BO's pages back to system memory from a work item that took mmap_read_lock. When an mmap writer was pending, that read lock blocked behind the writer while the thread allocating a new migration VRAM BO waited on this BO's eviction fence - a circular wait that hung the SVM workers.

Evict the SVM BO synchronously from the TTM eviction path (amdgpu_ttm_bo_eviction_valuable) instead of deferring to a work item. The BO is already reserved and the lock order is mmap_lock -> BO reservation, so only trylock the owning process's mmap lock; on contention return -EBUSY so TTM skips this BO. This removes the eviction work item and the enable_signaling path, so no worker can block on mmap_read_lock.

The SVM BO uses AMDGPU_GEM_CREATE_DISCARDABLE, so ttm_bo_evict takes the pipeline_gutting path and skips allocating a system memory placement. That would be wasted work, since svm_migrate_vram_to_ram allocates the system pages and copies the data back itself.

Eviction now migrates ranges directly, so it must serialize with the owning process: it trylocks migrate_mutex under svm_bo->list_lock before unlinking the range, and svm_range_free() unlinks the range then waits on migrate_mutex, so a concurrent eviction cannot free a range under it.

Drop the mm reference with mmput_async so exit_mmap() does not run under the BO reservation.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c    | 22 +++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h    |  1 +
 .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c  |  3 -
 drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c       | 16 ++++
 drivers/gpu/drm/amd/amdkfd/kfd_svm.c          | 88 ++++++++++++-------
 drivers/gpu/drm/amd/amdkfd/kfd_svm.h          | 10 +--
 6 files changed, 100 insertions(+), 40 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +22 -0
index 121282dd30c1..be764b6802b5 100644
@@ -39,6 +39,7 @@
 #if IS_ENABLED(CONFIG_HSA_AMD)
 #include "kfd_priv.h"
 #endif
+#include "kfd_svm.h"
 
 /* Total memory size in system memory and all GPU VRAM. Used to
  * estimate worst case amount of memory to reserve for page tables
@@ -972,3 +973,24 @@ int amdgpu_amdkfd_reset_mes_queue(struct amdgpu_device *adev,
 	return kgd2kfd_reset_mes_queue(adev->kfd.dev, node_id, queue_type,
 				       pipe, queue, db);
 }
+
+int amdgpu_amdkfd_evict_svm_bo(struct amdgpu_bo *bo)
+{
+	struct dma_resv_iter cursor;
+	struct dma_fence *fence;
+	int r = 0;
+
+	dma_resv_iter_begin(&cursor, bo->tbo.base.resv, DMA_RESV_USAGE_BOOKKEEP);
+	dma_resv_for_each_fence_unlocked(&cursor, fence) {
+		struct amdgpu_amdkfd_fence *f = to_amdgpu_amdkfd_fence(fence);
+
+		if (f && f->svm_bo) {
+			r = svm_range_evict_svm_bo(f->svm_bo);
+			if (r)
+				break;
+		}
+	}
+	dma_resv_iter_end(&cursor);
+
+	return r;
+}
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +1 -0
index 338412a750ed..b4840ee36f2b 100644
@@ -286,6 +286,7 @@ int amdgpu_amdkfd_reset_mes_queue(struct amdgpu_device *adev,
 				  int queue_type,
 				  int pipe, int queue,
 				  unsigned int db);
+int amdgpu_amdkfd_evict_svm_bo(struct amdgpu_bo *bo);
 
 /* Read user wptr from a specified user address space with page fault
  * disabled. The memory must be pinned and mapped to the hardware when
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c +0 -3
index b0299d861903..553d26c2744e 100644
@@ -135,9 +135,6 @@ static bool amdkfd_fence_enable_signaling(struct dma_fence *f)
 	if (!fence->svm_bo) {
 		if (!kgd2kfd_schedule_evict_and_restore_process(fence->mm, fence->context_id, f))
 			return true;
-	} else {
-		if (!svm_range_schedule_evict_svm_bo(fence))
-			return true;
 	}
 	return false;
 }
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +16 -0
index 03c1e5e3580c..746290216aae 100644
@@ -1488,6 +1488,7 @@ static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
 					    const struct ttm_place *place)
 {
 	struct dma_resv_iter resv_cursor;
+	struct amdgpu_bo *abo;
 	struct dma_fence *f;
 
 	if (!amdgpu_bo_is_amdgpu_bo(bo))
@@ -1497,6 +1498,21 @@ static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
 	if (bo->resource->mem_type == TTM_PL_SYSTEM)
 		return true;
 
+	abo = ttm_to_amdgpu_bo(bo);
+	if (abo->flags & AMDGPU_GEM_CREATE_DISCARDABLE) {
+		/*
+		 * SVM BOs are migrated to system memory synchronously in this
+		 * TTM eviction context. The migration needs the owning
+		 * process's mmap lock, but the normal lock order is
+		 * mmap_lock -> BO reservation and the BO is already reserved
+		 * here. svm_range_evict_svm_bo() only trylocks the mmap lock;
+		 * if the eviction fails for any reason, we return false so TTM
+		 * skips this BO instead of risking a deadlock.
+		 */
+		if (amdgpu_amdkfd_evict_svm_bo(abo) < 0)
+			return false;
+	}
+
 	if (bo->type == ttm_bo_type_kernel &&
 	    !amdgpu_vm_evictable(ttm_to_amdgpu_bo(bo)))
 		return false;
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +57 -31

151 lines not on this page · raw message

drivers/gpu/drm/amd/amdkfd/kfd_svm.h +4 -6
index a63dfc95b602..4232c47422e6 100644
@@ -44,7 +44,6 @@ struct svm_range_bo {
 	struct list_head		range_list; /* all svm ranges shared this bo */
 	spinlock_t			list_lock;
 	struct amdgpu_amdkfd_fence	*eviction_fence;
-	struct work_struct		eviction_work;
 	uint32_t			evicting;
 	struct work_struct		release_work;
 	struct kfd_node			*node;
@@ -175,7 +174,8 @@ void svm_range_vram_node_free(struct svm_range *prange);
 int svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,
 			    uint32_t vmid, uint32_t node_id, uint64_t addr, uint64_t ts,
 			    bool write_fault);
-int svm_range_schedule_evict_svm_bo(struct amdgpu_amdkfd_fence *fence);
+int svm_range_evict_svm_bo(struct svm_range_bo *svm_bo);
+
 void svm_range_add_list_work(struct svm_range_list *svms,
 			     struct svm_range *prange, struct mm_struct *mm,
 			     enum svm_work_list_ops op);
@@ -229,11 +229,9 @@ static inline int svm_range_restore_pages(struct amdgpu_device *adev,
 	return -EFAULT;
 }
 
-static inline int svm_range_schedule_evict_svm_bo(
-		struct amdgpu_amdkfd_fence *fence)
+static inline int svm_range_evict_svm_bo(struct svm_range_bo *svm_bo)
 {
-	WARN_ONCE(1, "SVM eviction fence triggered, but SVM is disabled");
-	return -EINVAL;
+	return 0;
 }
 
 static inline void svm_range_get_info(struct kfd_process *p,
Signature · 1 line
2.50.1
#1 lore raw

PATCH 2/2 drm/amdkfd: Remove svm_bo eviction fence

SVM BOs are now migrated back to system memory synchronously from the TTM eviction path (svm_range_evict_svm_bo), so the per-svm_bo eviction fence is no longer used.

Remove the eviction fence from svm_range_bo, drop the amdgpu_amdkfd_fence->svm_bo back pointer and the amdgpu_amdkfd_evict_svm_bo() helper, and stop special-casing svm_bo fences in the KFD fence enable_signaling and check_mm paths. Embed struct amdgpu_bo directly in svm_range_bo with a dedicated svm_range_bo_destroy() callback, and keep the owning mm via mmgrab()/mmdrop() instead of through the fence.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c    |  21 ----
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h    |   3 -
 .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c  |  15 +--
 .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c  |   4 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_object.c    |   4 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c       |   6 +-
 drivers/gpu/drm/amd/amdkfd/kfd_migrate.c      |   4 +-
 drivers/gpu/drm/amd/amdkfd/kfd_svm.c          | 109 +++++++-----------
 drivers/gpu/drm/amd/amdkfd/kfd_svm.h          |  13 ++-
 9 files changed, 64 insertions(+), 115 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +0 -21
index be764b6802b5..816d8817f0b2 100644
@@ -973,24 +973,3 @@ int amdgpu_amdkfd_reset_mes_queue(struct amdgpu_device *adev,
 	return kgd2kfd_reset_mes_queue(adev->kfd.dev, node_id, queue_type,
 				       pipe, queue, db);
 }
-
-int amdgpu_amdkfd_evict_svm_bo(struct amdgpu_bo *bo)
-{
-	struct dma_resv_iter cursor;
-	struct dma_fence *fence;
-	int r = 0;
-
-	dma_resv_iter_begin(&cursor, bo->tbo.base.resv, DMA_RESV_USAGE_BOOKKEEP);
-	dma_resv_for_each_fence_unlocked(&cursor, fence) {
-		struct amdgpu_amdkfd_fence *f = to_amdgpu_amdkfd_fence(fence);
-
-		if (f && f->svm_bo) {
-			r = svm_range_evict_svm_bo(f->svm_bo);
-			if (r)
-				break;
-		}
-	}
-	dma_resv_iter_end(&cursor);
-
-	return r;
-}
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +0 -3
index b4840ee36f2b..1b7dc0d3963b 100644
@@ -99,7 +99,6 @@ struct amdgpu_amdkfd_fence {
 	struct mm_struct *mm;
 	spinlock_t lock;
 	char timeline_name[TASK_COMM_LEN];
-	struct svm_range_bo *svm_bo;
 	uint16_t context_id;
 };
 
@@ -194,7 +193,6 @@ int amdgpu_queue_mask_bit_to_set_resource_bit(struct amdgpu_device *adev,
 
 struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context,
 				struct mm_struct *mm,
-				struct svm_range_bo *svm_bo,
 				u16 context_id);
 
 int amdgpu_amdkfd_drm_client_create(struct amdgpu_device *adev);
@@ -286,7 +284,6 @@ int amdgpu_amdkfd_reset_mes_queue(struct amdgpu_device *adev,
 				  int queue_type,
 				  int pipe, int queue,
 				  unsigned int db);
-int amdgpu_amdkfd_evict_svm_bo(struct amdgpu_bo *bo);
 
 /* Read user wptr from a specified user address space with page fault
  * disabled. The memory must be pinned and mapped to the hardware when
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c +3 -12
index 553d26c2744e..9b10d015671c 100644
@@ -62,7 +62,6 @@ static atomic_t fence_seq = ATOMIC_INIT(0);
 
 struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context,
 				struct mm_struct *mm,
-				struct svm_range_bo *svm_bo,
 				u16 context_id)
 {
 	struct amdgpu_amdkfd_fence *fence;
@@ -76,7 +75,6 @@ struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context,
 	fence->mm = mm;
 	get_task_comm(fence->timeline_name, current);
 	spin_lock_init(&fence->lock);
-	fence->svm_bo = svm_bo;
 	fence->context_id = context_id;
 	dma_fence_init(&fence->base, &amdkfd_fence_ops, &fence->lock,
 		   context, atomic_inc_return(&fence_seq));
@@ -128,14 +126,8 @@ static bool amdkfd_fence_enable_signaling(struct dma_fence *f)
 	if (dma_fence_is_signaled(f))
 		return true;
 
-	/* if fence->svm_bo is NULL, means this fence is created through
-	 * init_kfd_vm() or amdgpu_amdkfd_gpuvm_restore_process_bos().
-	 * Therefore, this fence is amdgpu_amdkfd_fence->eviction_fence.
-	 */
-	if (!fence->svm_bo) {
-		if (!kgd2kfd_schedule_evict_and_restore_process(fence->mm, fence->context_id, f))
-			return true;
-	}
+	if (!kgd2kfd_schedule_evict_and_restore_process(fence->mm, fence->context_id, f))
+		return true;
 	return false;
 }
 
@@ -169,7 +161,6 @@ static void amdkfd_fence_release(struct dma_fence *f)
  *
  * Check if @mm is same as that of the fence @f, if same return TRUE else
  * return FALSE.
- * For svm bo, which support vram overcommitment, always return FALSE.
  */
 bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm)
 {
@@ -177,7 +168,7 @@ bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm)
 
 	if (!fence)
 		return false;
-	else if (fence->mm == mm  && !fence->svm_bo)
+	else if (fence->mm == mm)
 		return true;
 
 	return false;
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +2 -2
index 451d7f7ef5c1..1e71829e0fc6 100644
@@ -1423,7 +1423,7 @@ static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info,
 		info->eviction_fence =
 			amdgpu_amdkfd_fence_create(dma_fence_context_alloc(1),
 						   current->mm,
-						   NULL, process->context_id);
+						   process->context_id);
 		if (!info->eviction_fence) {
 			pr_err("Failed to create eviction fence\n");
 			ret = -ENOMEM;
@@ -3093,7 +3093,7 @@ int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence __rcu *
 			amdgpu_amdkfd_fence_create(
 				process_info->eviction_fence->base.context,
 				process_info->eviction_fence->mm,
-				NULL, process_info->context_id);
+				process_info->context_id);
 
 		if (!new_fence) {
 			pr_err("Failed to create eviction fence\n");
drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +3 -1
index d4a9d5e8fb42..5d9d137209b6 100644
@@ -43,6 +43,7 @@
 #include "amdgpu_vram_mgr.h"
 #include "amdgpu_vm.h"
 #include "amdgpu_dma_buf.h"
+#include "kfd_svm.h"
 
 /**
  * DOC: amdgpu_object
@@ -93,7 +94,8 @@ static void amdgpu_bo_user_destroy(struct ttm_buffer_object *tbo)
 bool amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object *bo)
 {
 	if (bo->destroy == &amdgpu_bo_destroy ||
-	    bo->destroy == &amdgpu_bo_user_destroy)
+	    bo->destroy == &amdgpu_bo_user_destroy ||
+	    bo->destroy == &svm_range_bo_destroy)
 		return true;
 
 	return false;
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +4 -2
index 746290216aae..0e7e586b217a 100644
@@ -60,6 +60,7 @@
 #include "amdgpu_atomfirmware.h"
 #include "amdgpu_res_cursor.h"
 #include "bif/bif_4_1_d.h"
+#include "kfd_svm.h"
 
 MODULE_IMPORT_NS("DMA_BUF");
 
@@ -1499,7 +1500,8 @@ static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
 		return true;
 
 	abo = ttm_to_amdgpu_bo(bo);
-	if (abo->flags & AMDGPU_GEM_CREATE_DISCARDABLE) {
+	if ((abo->flags & AMDGPU_GEM_CREATE_DISCARDABLE) &&
+	    bo->destroy == &svm_range_bo_destroy) {
 		/*
 		 * SVM BOs are migrated to system memory synchronously in this
 		 * TTM eviction context. The migration needs the owning
@@ -1509,7 +1511,7 @@ static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
 		 * if the eviction fails for any reason, we return false so TTM
 		 * skips this BO instead of risking a deadlock.
 		 */
-		if (amdgpu_amdkfd_evict_svm_bo(abo) < 0)
+		if (svm_range_evict_svm_bo(abo) < 0)
 			return false;
 	}
 
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +2 -2
index ed3649a81332..f5af1dd3b70e 100644
@@ -954,12 +954,12 @@ static vm_fault_t svm_migrate_to_ram(struct vm_fault *vmf)
 		pr_debug("failed get device page at addr 0x%lx\n", addr);
 		return VM_FAULT_SIGBUS;
 	}
-	if (!mmget_not_zero(svm_bo->eviction_fence->mm)) {
+	if (!mmget_not_zero(svm_bo->mm)) {
 		pr_debug("addr 0x%lx of process mm is destroyed\n", addr);
 		return VM_FAULT_SIGBUS;
 	}
 
-	mm = svm_bo->eviction_fence->mm;
+	mm = svm_bo->mm;
 	if (mm != vmf->vma->vm_mm)
 		pr_debug("addr 0x%lx is COW mapping in child process\n", addr);
 
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +41 -68

228 lines not on this page · raw message

drivers/gpu/drm/amd/amdkfd/kfd_svm.h +9 -4
index 4232c47422e6..284c4cd5aa0f 100644
@@ -39,11 +39,11 @@
 			((adev)->hive ? (void *)(adev)->hive : (void *)(adev))
 
 struct svm_range_bo {
-	struct amdgpu_bo		*bo;
+	struct amdgpu_bo		bo;
 	struct kref			kref;
 	struct list_head		range_list; /* all svm ranges shared this bo */
 	spinlock_t			list_lock;
-	struct amdgpu_amdkfd_fence	*eviction_fence;
+	struct mm_struct		*mm;
 	uint32_t			evicting;
 	struct work_struct		release_work;
 	struct kfd_node			*node;
@@ -168,13 +168,14 @@ struct svm_range *svm_range_from_addr(struct svm_range_list *svms,
 				      struct svm_range **parent);
 struct kfd_node *svm_range_get_node_by_id(struct svm_range *prange,
 					  uint32_t gpu_id);
+void svm_range_bo_destroy(struct ttm_buffer_object *tbo);
 int svm_range_vram_node_new(struct kfd_node *node, struct svm_range *prange,
 			    bool clear);
 void svm_range_vram_node_free(struct svm_range *prange);
 int svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,
 			    uint32_t vmid, uint32_t node_id, uint64_t addr, uint64_t ts,
 			    bool write_fault);
-int svm_range_evict_svm_bo(struct svm_range_bo *svm_bo);
+int svm_range_evict_svm_bo(struct amdgpu_bo *bo);
 
 void svm_range_add_list_work(struct svm_range_list *svms,
 			     struct svm_range *prange, struct mm_struct *mm,
@@ -229,7 +230,11 @@ static inline int svm_range_restore_pages(struct amdgpu_device *adev,
 	return -EFAULT;
 }
 
-static inline int svm_range_evict_svm_bo(struct svm_range_bo *svm_bo)
+static inline void svm_range_bo_destroy(struct ttm_buffer_object *tbo)
+{
+}
+
+static inline int svm_range_evict_svm_bo(struct amdgpu_bo *bo)
 {
 	return 0;
 }
Signature · 1 line
2.50.1