Skip to contentSkip to conversation

drm/amdkfd: Add helper function svm_range_need_access_gpus

PATCH New Philip Yang 1 participant 7 messages Jan 15, 2024

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

PATCH v4 1/7 drm/amdkfd: Add helper function svm_range_need_access_gpus

Add the helper function to get all GPUs bitmap that need access the svm range. This helper will be used in the following patch to check if prange is mapped to all gpus.

Refactor svm_range_validate_and_map to use the helper function, no functional change.

---
 drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 74 ++++++++++++++++------------
 1 file changed, 43 insertions(+), 31 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +43 -31
index 18f8c82a849c..14dbc0fd51a9 100644
@@ -1169,6 +1169,44 @@ svm_range_add_child(struct svm_range *prange, struct mm_struct *mm,
 	list_add_tail(&pchild->child_list, &prange->child_list);
 }
 
+static int
+svm_range_need_access_gpus(unsigned long *bitmap, struct svm_range *prange)
+{
+	struct kfd_process *p = container_of(prange->svms, struct kfd_process, svms);
+	u32 gpuidx;
+
+	if (p->xnack_enabled) {
+		bitmap_copy(bitmap, prange->bitmap_aip, MAX_GPU_INSTANCE);
+
+		/* If prefetch range to GPU, or GPU retry fault migrate range to
+		 * GPU, which has ACCESS attribute to the range, create mapping
+		 * on that GPU.
+		 */
+		if (prange->actual_loc) {
+			gpuidx = kfd_process_gpuidx_from_gpuid(p, prange->actual_loc);
+			if (gpuidx < 0)
+				return -EINVAL;
+
+			if (test_bit(gpuidx, prange->bitmap_access))
+				bitmap_set(bitmap, gpuidx, 1);
+		}
+
+		/*
+		 * If prange is already mapped or with always mapped flag,
+		 * update mapping on GPUs with ACCESS attribute
+		 */
+		if (bitmap_empty(bitmap, MAX_GPU_INSTANCE)) {
+			if (prange->mapped_to_gpu ||
+			    prange->flags & KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED)
+				bitmap_copy(bitmap, prange->bitmap_access, MAX_GPU_INSTANCE);
+		}
+	} else {
+		bitmap_or(bitmap, prange->bitmap_access,
+			  prange->bitmap_aip, MAX_GPU_INSTANCE);
+	}
+	return 0;
+}
+
 static bool
 svm_nodes_in_same_hive(struct kfd_node *node_a, struct kfd_node *node_b)
 {
@@ -1609,38 +1647,12 @@ static int svm_range_validate_and_map(struct mm_struct *mm,
 	if (gpuidx < MAX_GPU_INSTANCE) {
 		bitmap_zero(ctx->bitmap, MAX_GPU_INSTANCE);
 		bitmap_set(ctx->bitmap, gpuidx, 1);
-	} else if (ctx->process->xnack_enabled) {
-		bitmap_copy(ctx->bitmap, prange->bitmap_aip, MAX_GPU_INSTANCE);
-
-		/* If prefetch range to GPU, or GPU retry fault migrate range to
-		 * GPU, which has ACCESS attribute to the range, create mapping
-		 * on that GPU.
-		 */
-		if (prange->actual_loc) {
-			gpuidx = kfd_process_gpuidx_from_gpuid(ctx->process,
-							prange->actual_loc);
-			if (gpuidx < 0) {
-				WARN_ONCE(1, "failed get device by id 0x%x\n",
-					 prange->actual_loc);
-				r = -EINVAL;
-				goto free_ctx;
-			}
-			if (test_bit(gpuidx, prange->bitmap_access))
-				bitmap_set(ctx->bitmap, gpuidx, 1);
-		}
-
-		/*
-		 * If prange is already mapped or with always mapped flag,
-		 * update mapping on GPUs with ACCESS attribute
-		 */
-		if (bitmap_empty(ctx->bitmap, MAX_GPU_INSTANCE)) {
-			if (prange->mapped_to_gpu ||
-			    prange->flags & KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED)
-				bitmap_copy(ctx->bitmap, prange->bitmap_access, MAX_GPU_INSTANCE);
-		}
 	} else {
-		bitmap_or(ctx->bitmap, prange->bitmap_access,
-			  prange->bitmap_aip, MAX_GPU_INSTANCE);
+		r = svm_range_need_access_gpus(ctx->bitmap, prange);
+		if (r) {
+			WARN_ONCE(1, "failed get device by id 0x%x\n", prange->actual_loc);
+			goto free_ctx;
+		}
 	}
 
 	if (bitmap_empty(ctx->bitmap, MAX_GPU_INSTANCE)) {
Signature · 1 line
2.35.1
#1 lore raw

PATCH v4 2/7 drm/amdkfd: Add helper function align range start last

Calculate range start, last address aligned to the range granularity size. This removes the duplicate code, and the helper function will be used in the future patch to handle map, unmap to GPU based on range granularity. No functional change.

---
 drivers/gpu/drm/amd/amdkfd/kfd_migrate.c |  4 ++--
 drivers/gpu/drm/amd/amdkfd/kfd_svm.c     | 10 ++++------
 drivers/gpu/drm/amd/amdkfd/kfd_svm.h     | 10 ++++++++++
 3 files changed, 16 insertions(+), 8 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +2 -2
index dae05f70257b..64eb9023d66b 100644
@@ -986,8 +986,8 @@ static vm_fault_t svm_migrate_to_ram(struct vm_fault *vmf)
 
 	/* Align migration range start and size to granularity size */
 	size = 1UL << prange->granularity;
-	start = max(ALIGN_DOWN(addr, size), prange->start);
-	last = min(ALIGN(addr + 1, size) - 1, prange->last);
+	start = svm_range_align_start(addr, prange->start, size);
+	last = svm_range_align_last(addr, prange->last, size);
 
 	r = svm_migrate_vram_to_ram(prange, vmf->vma->vm_mm, start, last,
 				    KFD_MIGRATE_TRIGGER_PAGEFAULT_CPU, vmf->page);
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +4 -6
index 14dbc0fd51a9..a2c96f5760ff 100644
@@ -2698,10 +2698,8 @@ svm_range_get_range_boundaries(struct kfd_process *p, int64_t addr,
 			 (vma->vm_start <= vma->vm_mm->start_stack &&
 			  vma->vm_end >= vma->vm_mm->start_stack);
 
-	start_limit = max(vma->vm_start >> PAGE_SHIFT,
-		      (unsigned long)ALIGN_DOWN(addr, 2UL << 8));
-	end_limit = min(vma->vm_end >> PAGE_SHIFT,
-		    (unsigned long)ALIGN(addr + 1, 2UL << 8));
+	start_limit = svm_range_align_start(addr, vma->vm_start >> PAGE_SHIFT, 2UL << 8);
+	end_limit = svm_range_align_last(addr, (vma->vm_end >> PAGE_SHIFT) - 1, 2UL << 8) + 1;
 	/* First range that starts after the fault address */
 	node = interval_tree_iter_first(&p->svms.objects, addr + 1, ULONG_MAX);
 	if (node) {
@@ -3043,8 +3041,8 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,
 
 	/* Align migration range start and size to granularity size */
 	size = 1UL << prange->granularity;
-	start = max_t(unsigned long, ALIGN_DOWN(addr, size), prange->start);
-	last = min_t(unsigned long, ALIGN(addr + 1, size) - 1, prange->last);
+	start = svm_range_align_start(addr, prange->start, size);
+	last = svm_range_align_last(addr, prange->last, size);
 	if (prange->actual_loc != 0 || best_loc != 0) {
 		migration = true;
 
drivers/gpu/drm/amd/amdkfd/kfd_svm.h +10 -0
index 026863a0abcd..806bcac6d101 100644
@@ -159,6 +159,16 @@ static inline struct svm_range_bo *svm_range_bo_ref(struct svm_range_bo *svm_bo)
 	return svm_bo;
 }
 
+static inline u64 svm_range_align_start(u64 addr, u64 range_start, u64 align_size)
+{
+	return max(ALIGN_DOWN(addr, align_size), range_start);
+}
+
+static inline u64 svm_range_align_last(u64 addr, u64 range_last, u64 align_size)
+{
+	return min(ALIGN(addr + 1, align_size) - 1, range_last);
+}
+
 int svm_range_list_init(struct kfd_process *p);
 void svm_range_list_fini(struct kfd_process *p);
 int svm_ioctl(struct kfd_process *p, enum kfd_ioctl_svm_op op, uint64_t start,
Signature · 1 line
2.35.1
#2 lore raw

PATCH v4 3/7 drm/amdkfd: Add granularity size based bitmap map flag

In reply to Philip Yang

Replace prange->mapped_to_gpu with prange->bitmap_map[], which is per GPU flag and use bitmap bits based on prange granularity. Align map to GPU or unmap from GPU range size to granularity size and update the corresponding bitmap_map flag bits. This will optimize multiple GPU map, unmap and retry fault recover.

svm_range_partial_mapped is false only if no part of the range mapping on any GPUs.

---
 drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 258 ++++++++++++++++++++++-----
 drivers/gpu/drm/amd/amdkfd/kfd_svm.h |   7 +-
 2 files changed, 219 insertions(+), 46 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +213 -45
index a2c96f5760ff..a003406db067 100644
@@ -307,12 +307,12 @@ static void svm_range_free(struct svm_range *prange, bool do_unmap)
 					KFD_IOC_ALLOC_MEM_FLAGS_USERPTR, 0);
 	}
 
-	/* free dma_addr array for each gpu */
+	/* free dma_addr array, bitmap_map for each gpu */
 	for (gpuidx = 0; gpuidx < MAX_GPU_INSTANCE; gpuidx++) {
-		if (prange->dma_addr[gpuidx]) {
+		if (prange->dma_addr[gpuidx])
 			kvfree(prange->dma_addr[gpuidx]);
-			prange->dma_addr[gpuidx] = NULL;
-		}
+		if (prange->bitmap_map[gpuidx])
+			bitmap_free(prange->bitmap_map[gpuidx]);
 	}
 
 	mutex_destroy(&prange->lock);
@@ -338,19 +338,38 @@ svm_range *svm_range_new(struct svm_range_list *svms, uint64_t start,
 	uint64_t size = last - start + 1;
 	struct svm_range *prange;
 	struct kfd_process *p;
-
-	prange = kzalloc(sizeof(*prange), GFP_KERNEL);
-	if (!prange)
-		return NULL;
+	unsigned int nbits;
+	u32 gpuidx;
 
 	p = container_of(svms, struct kfd_process, svms);
 	if (!p->xnack_enabled && update_mem_usage &&
 	    amdgpu_amdkfd_reserve_mem_limit(NULL, size << PAGE_SHIFT,
 				    KFD_IOC_ALLOC_MEM_FLAGS_USERPTR, 0)) {
 		pr_info("SVM mapping failed, exceeds resident system memory limit\n");
-		kfree(prange);
 		return NULL;
 	}
+
+	prange = kzalloc(sizeof(*prange), GFP_KERNEL);
+	if (!prange)
+		return NULL;
+
+	svm_range_set_default_attributes(&prange->preferred_loc,
+					 &prange->prefetch_loc,
+					 &prange->granularity, &prange->flags);
+
+	nbits = svm_range_map_nbits(start, last, prange->granularity);
+	pr_debug("prange 0x%p [0x%llx 0x%llx] bitmap_map nbits %d\n", prange,
+		 start, last, nbits);
+	for_each_set_bit(gpuidx, p->svms.bitmap_supported, p->n_pdds) {
+		prange->bitmap_map[gpuidx] = bitmap_zalloc(nbits, GFP_KERNEL);
+		if (!prange->bitmap_map[gpuidx]) {
+			while (gpuidx--)
+				bitmap_free(prange->bitmap_map[gpuidx]);
+			kfree(prange);
+			return NULL;
+		}
+	}
+
 	prange->npages = size;
 	prange->svms = svms;
 	prange->start = start;
@@ -369,10 +388,6 @@ svm_range *svm_range_new(struct svm_range_list *svms, uint64_t start,
 		bitmap_copy(prange->bitmap_access, svms->bitmap_supported,
 			    MAX_GPU_INSTANCE);
 
-	svm_range_set_default_attributes(&prange->preferred_loc,
-					 &prange->prefetch_loc,
-					 &prange->granularity, &prange->flags);
-
 	pr_debug("svms 0x%p [0x%llx 0x%llx]\n", svms, start, last);
 
 	return prange;
@@ -1017,6 +1032,51 @@ svm_range_split_nodes(struct svm_range *new, struct svm_range *old,
 	return 0;
 }
 
+static int
+svm_range_split_bitmap_map(struct svm_range *new, struct svm_range *old,
+			   u64 start, u64 last)
+{
+	struct kfd_process *p = container_of(new->svms, struct kfd_process, svms);
+	u32 new_nbits, old_nbits, old_nbits2;
+	unsigned long *bits;
+	u32 gpuidx;
+
+	new_nbits = svm_range_map_nbits(new->start, new->last, new->granularity);
+	old_nbits = svm_range_map_nbits(old->start, old->last, old->granularity);
+	old_nbits2 = svm_range_map_nbits(start, last, old->granularity);
+
+	pr_debug("old 0x%p [0x%lx 0x%lx] => [0x%llx 0x%llx] nbits %d => %d\n",
+		 old, old->start, old->last, start, last, old_nbits, old_nbits2);
+	pr_debug("new 0x%p [0x%lx 0x%lx] nbits %d\n", new, new->start, new->last,
+		 new_nbits);
+
+	for_each_set_bit(gpuidx, p->svms.bitmap_supported, p->n_pdds) {
+		bits = bitmap_alloc(old_nbits2, GFP_KERNEL);
+		if (!bits)
+			return -ENOMEM;
+
+		if (start == old->start) {
+			bitmap_copy(bits, old->bitmap_map[gpuidx], old_nbits2);
+			bitmap_shift_right(old->bitmap_map[gpuidx],
+					   old->bitmap_map[gpuidx],
+					   old_nbits - new_nbits, old_nbits);
+			bitmap_copy(new->bitmap_map[gpuidx],
+				    old->bitmap_map[gpuidx], new_nbits);
+		} else {
+			bitmap_copy(new->bitmap_map[gpuidx],
+				    old->bitmap_map[gpuidx], new_nbits);
+			bitmap_shift_right(old->bitmap_map[gpuidx],
+					   old->bitmap_map[gpuidx],
+					   old_nbits - old_nbits2, old_nbits);
+			bitmap_copy(bits, old->bitmap_map[gpuidx], old_nbits2);
+		}
+		bitmap_free(old->bitmap_map[gpuidx]);
+		old->bitmap_map[gpuidx] = bits;
+	}
+
+	return 0;
+}
+
 /**
  * svm_range_split_adjust - split range and adjust
  *
@@ -1057,6 +1117,10 @@ svm_range_split_adjust(struct svm_range *new, struct svm_range *old,
 			return r;
 	}
 
+	r = svm_range_split_bitmap_map(new, old, start, last);
+	if (r)
+		return r;
+
 	old->npages = last - start + 1;
 	old->start = start;
 	old->last = last;
@@ -1064,7 +1128,6 @@ svm_range_split_adjust(struct svm_range *new, struct svm_range *old,
 	new->preferred_loc = old->preferred_loc;
 	new->prefetch_loc = old->prefetch_loc;
 	new->granularity = old->granularity;
-	new->mapped_to_gpu = old->mapped_to_gpu;
 	bitmap_copy(new->bitmap_access, old->bitmap_access, MAX_GPU_INSTANCE);
 	bitmap_copy(new->bitmap_aip, old->bitmap_aip, MAX_GPU_INSTANCE);
 
@@ -1169,6 +1232,65 @@ svm_range_add_child(struct svm_range *prange, struct mm_struct *mm,
 	list_add_tail(&pchild->child_list, &prange->child_list);
 }
 
+/**
+ * svm_range_partial_mapped_dev - check if prange mapped on the specific GPU
+ *
+ * @gpuidx: the GPU index to check
+ * @prange: prange to check
+ * @start: the start address in pages
+ * @last: the last address in pages
+ *
+ * Return:
+ * true: if any part of the range within [start, last] mapped on the GPU
+ * false: if the entire range [start, last] not mapped on the GPU
+ */
+static bool
+svm_range_partial_mapped_dev(u32 gpuidx, struct svm_range *prange,
+			     u64 start, u64 last)
+{
+	u64 start_index, last_index;
+
+	start = max_t(u64, start, prange->start);
+	last = min_t(u64, last, prange->last);
+	if (last < start)
+		return false;
+
+	start_index = (start >> prange->granularity) - (prange->start >> prange->granularity);
+	last_index = (last >> prange->granularity) - (prange->start >> prange->granularity);
+	return find_next_bit(prange->bitmap_map[gpuidx], last_index + 1,
+			     start_index) <= last_index;
+}
+
+/**
+ * svm_range_partial_mapped - check if prange mapped on any GPU
+ *
+ * @prange: prange to check
+ * @start: the start address in pages
+ * @last: the last address in pages
+ *
+ * Return:
+ * true: if any part of prange mapped on any GPU currently
+ * false: if the entire range is not mapped on any GPU
+ */
+static bool
+svm_range_partial_mapped(struct svm_range *prange, u64 start, u64 last)
+{
+	struct kfd_process *p = container_of(prange->svms, struct kfd_process, svms);
+	struct svm_range *pchild;
+	u32 gpuidx;
+
+	for_each_set_bit(gpuidx, p->svms.bitmap_supported, p->n_pdds) {
+		list_for_each_entry(pchild, &prange->child_list, child_list) {
+			if (svm_range_partial_mapped_dev(gpuidx, pchild, start, last))
+				return true;
+		}
+
+		if (svm_range_partial_mapped_dev(gpuidx, prange, start, last))
+			return true;
+	}
+	return false;
+}
+
 static int
 svm_range_need_access_gpus(unsigned long *bitmap, struct svm_range *prange)
 {
@@ -1196,7 +1318,7 @@ svm_range_need_access_gpus(unsigned long *bitmap, struct svm_range *prange)
 		 * update mapping on GPUs with ACCESS attribute
 		 */
 		if (bitmap_empty(bitmap, MAX_GPU_INSTANCE)) {
-			if (prange->mapped_to_gpu ||
+			if (svm_range_partial_mapped(prange, prange->start, prange->last) ||
 			    prange->flags & KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED)
 				bitmap_copy(bitmap, prange->bitmap_access, MAX_GPU_INSTANCE);
 		}
@@ -1336,6 +1458,48 @@ svm_range_unmap_from_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 				      fence);
 }
 
+/**
+ * svm_range_complete_mapped - check if prange mapped on all GPUs completely
+ *
+ * @prange: prange to check
+ *
+ * Return:
+ * true: if the entire prange mapped completely on all GPUs that need access
+ * otherwise return false
+ */
+static bool svm_range_complete_mapped(struct svm_range *prange)
+{
+	int nbits = svm_range_map_nbits(prange->start, prange->last, prange->granularity);
+	DECLARE_BITMAP(bitmap, MAX_GPU_INSTANCE);
+	u32 gpuidx;
+	int r;
+
+	r = svm_range_need_access_gpus(bitmap, prange);
+	if (r)
+		return false;
+
+	for_each_set_bit(gpuidx, bitmap, MAX_GPU_INSTANCE)
+		if (!bitmap_full(prange->bitmap_map[gpuidx], nbits))
+			return false;
+	return true;
+}
+
+static void
+svm_range_update_mapped(u32 gpuidx, struct svm_range *prange,
+			u64 start, u64 last, bool mapped)
+{
+	u64 index, nbits;
+
+	index = (start >>  prange->granularity) - (prange->start >> prange->granularity);
+	nbits = svm_range_map_nbits(start, last, prange->granularity);
+	if (mapped)
+		bitmap_set(prange->bitmap_map[gpuidx], index, nbits);
+	else
+		bitmap_clear(prange->bitmap_map[gpuidx], index, nbits);
+	pr_debug("prange 0x%p [0x%llx 0x%llx] update mapped %d nbits %lld gpu %d\n",
+		 prange, start, last, mapped, nbits, gpuidx);
+}
+
 static int
 svm_range_unmap_from_gpus(struct svm_range *prange, unsigned long start,
 			  unsigned long last, uint32_t trigger)
@@ -1347,29 +1511,28 @@ svm_range_unmap_from_gpus(struct svm_range *prange, unsigned long start,
 	uint32_t gpuidx;
 	int r = 0;
 
-	if (!prange->mapped_to_gpu) {
-		pr_debug("prange 0x%p [0x%lx 0x%lx] not mapped to GPU\n",
-			 prange, prange->start, prange->last);
-		return 0;
-	}
-
-	if (prange->start == start && prange->last == last) {
-		pr_debug("unmap svms 0x%p prange 0x%p\n", prange->svms, prange);
-		prange->mapped_to_gpu = false;
-	}
-
 	bitmap_or(bitmap, prange->bitmap_access, prange->bitmap_aip,
 		  MAX_GPU_INSTANCE);
 	p = container_of(prange->svms, struct kfd_process, svms);
 
 	for_each_set_bit(gpuidx, bitmap, MAX_GPU_INSTANCE) {
-		pr_debug("unmap from gpu idx 0x%x\n", gpuidx);
 		pdd = kfd_process_device_from_gpuidx(p, gpuidx);
 		if (!pdd) {
 			pr_debug("failed to find device idx %d\n", gpuidx);
-			return -EINVAL;
+			continue;
 		}
 
+		if (!svm_range_partial_mapped_dev(gpuidx, prange, start, last)) {
+			pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx] not mapped on gpu %d\n",
+				 prange->svms, prange, start, last, gpuidx);
+			continue;
+		}
+
+		svm_range_update_mapped(gpuidx, prange, start, last, false);
+
+		pr_debug("unmap svms 0x%p prange 0x%p [0x%lx 0x%lx] from gpu %d\n",
+			 prange->svms, prange, start, last, gpuidx);
+
 		kfd_smi_event_unmap_from_gpu(pdd->dev, p->lead_thread->pid,
 					     start, last, trigger);
 
@@ -1510,6 +1673,10 @@ svm_range_map_to_gpus(struct svm_range *prange, unsigned long offset,
 		if (r)
 			break;
 
+		if (!r)
+			svm_range_update_mapped(gpuidx, prange, prange->start + offset,
+						prange->start + offset + npages - 1, true);
+
 		if (fence) {
 			r = dma_fence_wait(fence, false);
 			dma_fence_put(fence);
@@ -1745,9 +1912,6 @@ static int svm_range_validate_and_map(struct mm_struct *mm,
 			}
 		}
 
-		if (!r && next == end)
-			prange->mapped_to_gpu = true;
-
 		svm_range_unlock(prange);
 
 		addr = next;
@@ -1916,12 +2080,11 @@ svm_range_evict(struct svm_range *prange, struct mm_struct *mm,
 	if (!p->xnack_enabled ||
 	    (prange->flags & KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED)) {
 		int evicted_ranges;
-		bool mapped = prange->mapped_to_gpu;
+
+		if (!svm_range_partial_mapped(prange, start, last))
+			return r;
 
 		list_for_each_entry(pchild, &prange->child_list, child_list) {
-			if (!pchild->mapped_to_gpu)
-				continue;
-			mapped = true;
 			mutex_lock_nested(&pchild->lock, 1);
 			if (pchild->start <= last && pchild->last >= start) {
 				pr_debug("increment pchild invalid [0x%lx 0x%lx]\n",
@@ -1930,10 +2093,6 @@ svm_range_evict(struct svm_range *prange, struct mm_struct *mm,
 			}
 			mutex_unlock(&pchild->lock);
 		}
-
-		if (!mapped)
-			return r;
-
 		if (prange->start <= last && prange->last >= start)
 			atomic_inc(&prange->invalid);
 
@@ -1982,7 +2141,10 @@ svm_range_evict(struct svm_range *prange, struct mm_struct *mm,
 
 static struct svm_range *svm_range_clone(struct svm_range *old)
 {
+	struct kfd_process *p = container_of(old->svms, struct kfd_process, svms);
 	struct svm_range *new;
+	u32 gpuidx;
+	int nbits;
 
 	new = svm_range_new(old->svms, old->start, old->last, false);
 	if (!new)
@@ -2004,8 +2166,13 @@ static struct svm_range *svm_range_clone(struct svm_range *old)
 	new->prefetch_loc = old->prefetch_loc;
 	new->actual_loc = old->actual_loc;
 	new->granularity = old->granularity;
-	new->mapped_to_gpu = old->mapped_to_gpu;
 	new->vram_pages = old->vram_pages;
+	nbits = svm_range_map_nbits(new->start, new->last, new->granularity);
+	for_each_set_bit(gpuidx, p->svms.bitmap_supported, p->n_pdds) {
+		bitmap_copy(new->bitmap_map[gpuidx],
+			    old->bitmap_map[gpuidx],
+			    nbits);
+	};
 	bitmap_copy(new->bitmap_access, old->bitmap_access, MAX_GPU_INSTANCE);
 	bitmap_copy(new->bitmap_aip, old->bitmap_aip, MAX_GPU_INSTANCE);
 
@@ -2126,7 +2293,7 @@ svm_range_add(struct kfd_process *p, uint64_t start, uint64_t size,
 		next_start = min(node->last, last) + 1;
 
 		if (svm_range_is_same_attrs(p, prange, nattr, attrs) &&
-		    prange->mapped_to_gpu) {
+		    svm_range_complete_mapped(prange)) {
 			/* nothing to do */
 		} else if (node->start < start || node->last > last) {
 			/* node intersects the update range and its attributes
@@ -3611,7 +3778,7 @@ svm_range_set_attr(struct kfd_process *p, struct mm_struct *mm,
 
 		if (migrated && (!p->xnack_enabled ||
 		    (prange->flags & KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED)) &&
-		    prange->mapped_to_gpu) {
+		    svm_range_partial_mapped(prange, prange->start, prange->last)) {
 			pr_debug("restore_work will update mappings of GPUs\n");
 			mutex_unlock(&prange->migrate_mutex);
 			continue;
@@ -3622,8 +3789,8 @@ svm_range_set_attr(struct kfd_process *p, struct mm_struct *mm,
 			continue;
 		}
 
-		flush_tlb = !migrated && update_mapping && prange->mapped_to_gpu;
-
+		flush_tlb = !migrated && update_mapping &&
+			    svm_range_partial_mapped(prange, prange->start, prange->last);
 		r = svm_range_validate_and_map(mm, prange->start, prange->last, prange,
 					       MAX_GPU_INSTANCE, true, true, flush_tlb);
 		if (r)
@@ -3639,8 +3806,9 @@ svm_range_set_attr(struct kfd_process *p, struct mm_struct *mm,
 		pr_debug("Remapping prange 0x%p [0x%lx 0x%lx]\n",
 			 prange, prange->start, prange->last);
 		mutex_lock(&prange->migrate_mutex);
+		flush_tlb = svm_range_partial_mapped(prange, prange->start, prange->last);
 		r = svm_range_validate_and_map(mm,  prange->start, prange->last, prange,
-					       MAX_GPU_INSTANCE, true, true, prange->mapped_to_gpu);
+					       MAX_GPU_INSTANCE, true, true, flush_tlb);
 		if (r)
 			pr_debug("failed %d on remap svm range\n", r);
 		mutex_unlock(&prange->migrate_mutex);
drivers/gpu/drm/amd/amdkfd/kfd_svm.h +6 -1
index 806bcac6d101..a10eeb77f83e 100644
@@ -136,7 +136,7 @@ struct svm_range {
 	struct list_head		child_list;
 	DECLARE_BITMAP(bitmap_access, MAX_GPU_INSTANCE);
 	DECLARE_BITMAP(bitmap_aip, MAX_GPU_INSTANCE);
-	bool				mapped_to_gpu;
+	unsigned long			*bitmap_map[MAX_GPU_INSTANCE];
 };
 
 static inline void svm_range_lock(struct svm_range *prange)
@@ -169,6 +169,11 @@ static inline u64 svm_range_align_last(u64 addr, u64 range_last, u64 align_size)
 	return min(ALIGN(addr + 1, align_size) - 1, range_last);
 }
 
+static inline u32 svm_range_map_nbits(u64 start, u64 last, u8 granularity)
+{
+	return (last >> granularity) - (start >> granularity) + 1;
+}
+
 int svm_range_list_init(struct kfd_process *p);
 void svm_range_list_fini(struct kfd_process *p);
 int svm_ioctl(struct kfd_process *p, enum kfd_ioctl_svm_op op, uint64_t start,
Signature · 1 line
2.35.1
#3 lore raw

PATCH v4 4/7 amd/amdkfd: Unmap range from GPU based on granularity

In reply to Philip Yang

When MMU notifier invalidate the range, align the start and last address to range granularity to unmap from GPU and update bitmap_map flag. Skip unmap from GPU if range is already unmapped based on bitmap_map flag. This avoids unmap 1 page from GPU and flush TLB, also solve the rocgdb CWSR migration related issue.

Unmap the range from cpu will remove the range and split the range, this cannot align the start and last address to range granularity. Change to split the range and bitmap_map flag first, then unmap the range from GPU. If unmapping from GPU first, the bitmap_map flag is updated, split range may get incorrect bitmap_map for the remaining ranges.

---
 drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 42 +++++++++++++++++++---------
 1 file changed, 29 insertions(+), 13 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +29 -13
index a003406db067..7a30c3e58234 100644
@@ -2114,6 +2114,13 @@ svm_range_evict(struct svm_range *prange, struct mm_struct *mm,
 	} else {
 		unsigned long s, l;
 		uint32_t trigger;
+		u64 size = 1UL << prange->granularity;
+
+		if (!svm_range_partial_mapped(prange, start, last)) {
+			pr_debug("svms 0x%p [0x%lx 0x%lx] unmapped already\n",
+				 prange->svms, start, last);
+			return 0;
+		}
 
 		if (event == MMU_NOTIFY_MIGRATE)
 			trigger = KFD_SVM_UNMAP_TRIGGER_MMU_NOTIFY_MIGRATE;
@@ -2122,16 +2129,17 @@ svm_range_evict(struct svm_range *prange, struct mm_struct *mm,
 
 		pr_debug("invalidate unmap svms 0x%p [0x%lx 0x%lx] from GPUs\n",
 			 prange->svms, start, last);
+
 		list_for_each_entry(pchild, &prange->child_list, child_list) {
 			mutex_lock_nested(&pchild->lock, 1);
-			s = max(start, pchild->start);
-			l = min(last, pchild->last);
+			s = svm_range_align_start(start, pchild->start, size);
+			l = svm_range_align_last(last, pchild->last, size);
 			if (l >= s)
 				svm_range_unmap_from_gpus(pchild, s, l, trigger);
 			mutex_unlock(&pchild->lock);
 		}
-		s = max(start, prange->start);
-		l = min(last, prange->last);
+		s = svm_range_align_start(start, prange->start, size);
+		l = svm_range_align_last(last, prange->last, size);
 		if (l >= s)
 			svm_range_unmap_from_gpus(prange, s, l, trigger);
 	}
@@ -2645,24 +2653,32 @@ svm_range_unmap_from_cpu(struct mm_struct *mm, struct svm_range *prange,
 
 	list_for_each_entry(pchild, &prange->child_list, child_list) {
 		mutex_lock_nested(&pchild->lock, 1);
-		s = max(start, pchild->start);
-		l = min(last, pchild->last);
-		if (l >= s)
-			svm_range_unmap_from_gpus(pchild, s, l, trigger);
 		svm_range_unmap_split(mm, prange, pchild, start, last);
 		mutex_unlock(&pchild->lock);
 	}
-	s = max(start, prange->start);
-	l = min(last, prange->last);
-	if (l >= s)
-		svm_range_unmap_from_gpus(prange, s, l, trigger);
 	svm_range_unmap_split(mm, prange, prange, start, last);
-
 	if (unmap_parent)
 		svm_range_add_list_work(svms, prange, mm, SVM_OP_UNMAP_RANGE);
 	else
 		svm_range_add_list_work(svms, prange, mm,
 					SVM_OP_UPDATE_RANGE_NOTIFIER);
+
+	list_for_each_entry(pchild, &prange->child_list, child_list) {
+		if (pchild->work_item.op != SVM_OP_UNMAP_RANGE)
+			continue;
+
+		s = max(start, pchild->start);
+		l = min(last, pchild->last);
+		if (l >= s)
+			svm_range_unmap_from_gpus(pchild, s, l, trigger);
+	}
+	if (prange->work_item.op == SVM_OP_UNMAP_RANGE) {
+		s = max(start, prange->start);
+		l = min(last, prange->last);
+		if (l >= s)
+			svm_range_unmap_from_gpus(prange, s, l, trigger);
+	}
+
 	schedule_deferred_list_work(svms);
 
 	kfd_unref_process(p);
Signature · 1 line
2.35.1
#4 lore raw

PATCH v4 5/7 drm/amdkfd: Change range granularity update bitmap_map

In reply to Philip Yang

When changing the svm range granularity, update the svm range bitmap_map based on new range granularity.

---
 drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 49 +++++++++++++++++++++++++++-
 1 file changed, 48 insertions(+), 1 deletion(-)
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +48 -1
index 7a30c3e58234..ebc4cce801bf 100644
@@ -757,6 +757,53 @@ svm_range_check_attr(struct kfd_process *p,
 	return 0;
 }
 
+static void
+svm_range_change_granularity(struct svm_range *prange, u8 value)
+{
+	struct kfd_process *p = container_of(prange->svms, struct kfd_process, svms);
+	u32 new_nbits, old_nbits, i, n;
+	unsigned long *new_bits, *old_bits;
+	u32 gpuidx;
+
+	if (prange->granularity == value)
+		return;
+
+	old_nbits = svm_range_map_nbits(prange->start, prange->last, prange->granularity);
+	new_nbits = svm_range_map_nbits(prange->start, prange->last, value);
+	if (new_nbits > old_nbits) {
+		n = new_nbits / old_nbits;
+		if (new_nbits % old_nbits)
+			n++;
+	} else {
+		n = old_nbits / new_nbits;
+		if (old_nbits % new_nbits)
+			n++;
+	}
+
+	pr_debug("prange 0x%p [0x%lx 0x%lx] bitmap_map nbits %d -> %d\n",
+		 prange, prange->start, prange->last, old_nbits, new_nbits);
+
+	for_each_set_bit(gpuidx, p->svms.bitmap_supported, p->n_pdds) {
+		old_bits = prange->bitmap_map[gpuidx];
+		if (bitmap_empty(old_bits, old_nbits))
+			continue;
+
+		new_bits = bitmap_zalloc(new_nbits, GFP_KERNEL);
+		if (!new_bits)
+			return;
+
+		for_each_set_bit(i, old_bits, old_nbits) {
+			if (new_nbits > old_nbits)
+				bitmap_set(new_bits, i * n, n);
+			else
+				bitmap_set(new_bits, i / n, 1);
+		}
+		prange->bitmap_map[gpuidx] = new_bits;
+		bitmap_free(old_bits);
+	}
+	prange->granularity = value;
+}
+
 static void
 svm_range_apply_attrs(struct kfd_process *p, struct svm_range *prange,
 		      uint32_t nattr, struct kfd_ioctl_svm_attribute *attrs,
@@ -801,7 +848,7 @@ svm_range_apply_attrs(struct kfd_process *p, struct svm_range *prange,
 			prange->flags &= ~attrs[i].value;
 			break;
 		case KFD_IOCTL_SVM_ATTR_GRANULARITY:
-			prange->granularity = min_t(uint32_t, attrs[i].value, 0x3F);
+			svm_range_change_granularity(prange, min_t(u32, attrs[i].value, 0x3F));
 			break;
 		default:
 			WARN_ONCE(1, "svm_range_check_attrs wasn't called?");
Signature · 1 line
2.35.1
#5 lore raw

PATCH v4 6/7 drm/amdkfd: Check bitmap_map flag to skip retry fault

In reply to Philip Yang

Remove prange validate_timestamp which is not accurate for multiple GPUs.

Use the bitmap_map flag to skip the retry fault from different pages of the same granularity range if the granularity range is already mapped on the specific GPU.

---
 drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 24 ++++++++----------------
 drivers/gpu/drm/amd/amdkfd/kfd_svm.h |  1 -
 2 files changed, 8 insertions(+), 17 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +8 -16
index ebc4cce801bf..b36d997e7a3d 100644
@@ -45,10 +45,6 @@
 
 #define AMDGPU_SVM_RANGE_RESTORE_DELAY_MS 1
 
-/* Long enough to ensure no retry fault comes after svm range is restored and
- * page table is updated.
- */
-#define AMDGPU_SVM_RANGE_RETRY_FAULT_PENDING	(2UL * NSEC_PER_MSEC)
 #if IS_ENABLED(CONFIG_DYNAMIC_DEBUG)
 #define dynamic_svm_range_dump(svms) \
 	_dynamic_func_call_no_desc("svm_range_dump", svm_range_debug_dump, svms)
@@ -380,7 +376,6 @@ svm_range *svm_range_new(struct svm_range_list *svms, uint64_t start,
 	INIT_LIST_HEAD(&prange->deferred_list);
 	INIT_LIST_HEAD(&prange->child_list);
 	atomic_set(&prange->invalid, 0);
-	prange->validate_timestamp = 0;
 	mutex_init(&prange->migrate_mutex);
 	mutex_init(&prange->lock);
 
@@ -1965,8 +1960,6 @@ static int svm_range_validate_and_map(struct mm_struct *mm,
 	}
 
 	svm_range_unreserve_bos(ctx);
-	if (!r)
-		prange->validate_timestamp = ktime_get_boottime();
 
 free_ctx:
 	kfree(ctx);
@@ -3226,15 +3219,6 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,
 		goto out_unlock_mm;
 	}
 
-	/* skip duplicate vm fault on different pages of same range */
-	if (ktime_before(timestamp, ktime_add_ns(prange->validate_timestamp,
-				AMDGPU_SVM_RANGE_RETRY_FAULT_PENDING))) {
-		pr_debug("svms 0x%p [0x%lx %lx] already restored\n",
-			 svms, prange->start, prange->last);
-		r = 0;
-		goto out_unlock_mm;
-	}
-
 	/* __do_munmap removed VMA, return success as we are handling stale
 	 * retry fault.
 	 */
@@ -3260,6 +3244,14 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,
 		goto out_unlock_mm;
 	}
 
+	/* skip duplicate vm fault on different pages of same granularity range */
+	if (svm_range_partial_mapped_dev(gpuidx, prange, addr, addr)) {
+		pr_debug("svms 0x%p [0x%lx %lx] addr 0x%llx already mapped on gpu %d\n",
+			 svms, prange->start, prange->last, addr, gpuidx);
+		r = 0;
+		goto out_unlock_mm;
+	}
+
 	mutex_lock(&prange->migrate_mutex);
 
 	pr_debug("svms %p [0x%lx 0x%lx] best restore 0x%x, actual loc 0x%x\n",
drivers/gpu/drm/amd/amdkfd/kfd_svm.h +0 -1
index a10eeb77f83e..5a9688d5c18c 100644
@@ -129,7 +129,6 @@ struct svm_range {
 	uint32_t			actual_loc;
 	uint8_t				granularity;
 	atomic_t			invalid;
-	ktime_t				validate_timestamp;
 	struct mmu_interval_notifier	notifier;
 	struct svm_work_list_item	work_item;
 	struct list_head		deferred_list;
Signature · 1 line
2.35.1
#6 lore raw

PATCH v4 7/7 drm/amdkfd: Wait update sdma fence before tlb flush

In reply to Philip Yang

If using sdma update GPU page table, kfd flush tlb does nothing if vm update fence callback doesn't update vm->tlb_seq. This works now because retry fault will come and update page table again and flush tlb finally.

With the bitmap_map flag, the retry fault recover will only update GPU page table once, have to wait sdma udate fence and then flush tlb.

No change if using CPU update GPU page table for large bar because no vm update fence.

Remove wait parameter in svm_range_validate_and_map because it is always called with true now.

---
 drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +8 -9
index b36d997e7a3d..9e5f6e12c498 100644
@@ -1677,7 +1677,7 @@ svm_range_map_to_gpu(struct kfd_process_device *pdd, struct svm_range *prange,
 static int
 svm_range_map_to_gpus(struct svm_range *prange, unsigned long offset,
 		      unsigned long npages, bool readonly,
-		      unsigned long *bitmap, bool wait, bool flush_tlb)
+		      unsigned long *bitmap, bool flush_tlb)
 {
 	struct kfd_process_device *pdd;
 	struct amdgpu_device *bo_adev = NULL;
@@ -1710,8 +1710,7 @@ svm_range_map_to_gpus(struct svm_range *prange, unsigned long offset,
 
 		r = svm_range_map_to_gpu(pdd, prange, offset, npages, readonly,
 					 prange->dma_addr[gpuidx],
-					 bo_adev, wait ? &fence : NULL,
-					 flush_tlb);
+					 bo_adev, &fence, flush_tlb);
 		if (r)
 			break;
 
@@ -1837,7 +1836,7 @@ static void *kfd_svm_page_owner(struct kfd_process *p, int32_t gpuidx)
 static int svm_range_validate_and_map(struct mm_struct *mm,
 				      unsigned long map_start, unsigned long map_last,
 				      struct svm_range *prange, int32_t gpuidx,
-				      bool intr, bool wait, bool flush_tlb)
+				      bool intr, bool flush_tlb)
 {
 	struct svm_validate_context *ctx;
 	unsigned long start, end, addr;
@@ -1950,7 +1949,7 @@ static int svm_range_validate_and_map(struct mm_struct *mm,
 				offset = map_start_vma - prange->start;
 				npages = map_last_vma - map_start_vma + 1;
 				r = svm_range_map_to_gpus(prange, offset, npages, readonly,
-							  ctx->bitmap, wait, flush_tlb);
+							  ctx->bitmap, flush_tlb);
 			}
 		}
 
@@ -2041,7 +2040,7 @@ static void svm_range_restore_work(struct work_struct *work)
 		mutex_lock(&prange->migrate_mutex);
 
 		r = svm_range_validate_and_map(mm, prange->start, prange->last, prange,
-					       MAX_GPU_INSTANCE, false, true, false);
+					       MAX_GPU_INSTANCE, false, false);
 		if (r)
 			pr_debug("failed %d to map 0x%lx to gpus\n", r,
 				 prange->start);
@@ -3303,7 +3302,7 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,
 	mmap_read_lock(mm);
 
 	r = svm_range_validate_and_map(mm, start, last, prange, gpuidx, false,
-				       false, false);
+				       false);
 	if (r)
 		pr_debug("failed %d to map svms 0x%p [0x%lx 0x%lx] to gpus\n",
 			 r, svms, start, last);
@@ -3847,7 +3846,7 @@ svm_range_set_attr(struct kfd_process *p, struct mm_struct *mm,
 		flush_tlb = !migrated && update_mapping &&
 			    svm_range_partial_mapped(prange, prange->start, prange->last);
 		r = svm_range_validate_and_map(mm, prange->start, prange->last, prange,
-					       MAX_GPU_INSTANCE, true, true, flush_tlb);
+					       MAX_GPU_INSTANCE, true, flush_tlb);
 		if (r)
 			pr_debug("failed %d to map svm range\n", r);
 
@@ -3863,7 +3862,7 @@ svm_range_set_attr(struct kfd_process *p, struct mm_struct *mm,
 		mutex_lock(&prange->migrate_mutex);
 		flush_tlb = svm_range_partial_mapped(prange, prange->start, prange->last);
 		r = svm_range_validate_and_map(mm,  prange->start, prange->last, prange,
-					       MAX_GPU_INSTANCE, true, true, flush_tlb);
+					       MAX_GPU_INSTANCE, true, flush_tlb);
 		if (r)
 			pr_debug("failed %d on remap svm range\n", r);
 		mutex_unlock(&prange->migrate_mutex);
Signature · 1 line
2.35.1