drm/amdgpu: use HMM mirror callback to replace mmu notifier v6

Use HMM helper function hmm_vma_fault() to get physical pages backing userptr and start CPU page table update track of those pages. Then use hmm_vma_range_done() to check if those pages are updated before amdgpu_cs_submit for gfx or before user queues are resumed for kfd.

If userptr pages are updated, for gfx, amdgpu_cs_ioctl will restart from scratch, for kfd, restore worker is rescheduled to retry.

HMM simplify the CPU page table concurrent update check, so remove guptasklock, mmu_invalidations, last_set_pages fields from amdgpu_ttm_tt struct.

HMM does not pin the page (increase page ref count), so remove related operations like release_pages(), put_page(), mark_page_dirty().

Change-Id Id2d3130378b44a774e0d77156102a20a203b5036

Signed-off-by Philip Yang <[email protected]>

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h    |   1 -
 .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c  |  81 ++------
 drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c   |   2 -
 drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.h   |   3 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c        | 188 +++++++++---------
 drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c       |  14 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c        |  34 +++-
 drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h        |   7 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c       | 160 ++++++---------
 drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h       |   4 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c        |   1 -
 11 files changed, 206 insertions(+), 289 deletions(-)

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +0 -1

index bcf587b4ba98..b492dd9e541a 100644
@@ -62,7 +62,6 @@ struct kgd_mem {
 
 	atomic_t invalid;
 	struct amdkfd_process_info *process_info;
-	struct page **user_pages;
 
 	struct amdgpu_sync sync;
 

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +5 -50

index b29ef088fa14..75833b40507b 100644
@@ -580,28 +580,12 @@ static int init_user_pages(struct kgd_mem *mem, struct mm_struct *mm,
 		goto out;
 	}
 
-	/* If no restore worker is running concurrently, user_pages
-	 * should not be allocated
-	 */
-	WARN(mem->user_pages, "Leaking user_pages array");
-
-	mem->user_pages = kvmalloc_array(bo->tbo.ttm->num_pages,
-					   sizeof(struct page *),
-					   GFP_KERNEL | __GFP_ZERO);
-	if (!mem->user_pages) {
-		pr_err("%s: Failed to allocate pages array\n", __func__);
-		ret = -ENOMEM;
-		goto unregister_out;
-	}
-
-	ret = amdgpu_ttm_tt_get_user_pages(bo->tbo.ttm, mem->user_pages);
+	ret = amdgpu_ttm_tt_get_user_pages(bo->tbo.ttm, bo->tbo.ttm->pages);
 	if (ret) {
 		pr_err("%s: Failed to get user pages: %d\n", __func__, ret);
-		goto free_out;
+		goto unregister_out;
 	}
 
-	amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm, mem->user_pages);
-
 	ret = amdgpu_bo_reserve(bo, true);
 	if (ret) {
 		pr_err("%s: Failed to reserve BO\n", __func__);

@@ -614,11 +598,7 @@ static int init_user_pages(struct kgd_mem *mem, struct mm_struct *mm,
 	amdgpu_bo_unreserve(bo);
 
 release_out:
-	if (ret)
-		release_pages(mem->user_pages, bo->tbo.ttm->num_pages);
-free_out:
-	kvfree(mem->user_pages);
-	mem->user_pages = NULL;
+	amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm);
 unregister_out:
 	if (ret)
 		amdgpu_mn_unregister(bo);

@@ -677,7 +657,6 @@ static int reserve_bo_and_vm(struct kgd_mem *mem,
 	ctx->kfd_bo.priority = 0;
 	ctx->kfd_bo.tv.bo = &bo->tbo;
 	ctx->kfd_bo.tv.num_shared = 1;
-	ctx->kfd_bo.user_pages = NULL;
 	list_add(&ctx->kfd_bo.tv.head, &ctx->list);
 
 	amdgpu_vm_get_pd_bo(vm, &ctx->list, &ctx->vm_pd[0]);

@@ -741,7 +720,6 @@ static int reserve_bo_and_cond_vms(struct kgd_mem *mem,
 	ctx->kfd_bo.priority = 0;
 	ctx->kfd_bo.tv.bo = &bo->tbo;
 	ctx->kfd_bo.tv.num_shared = 1;
-	ctx->kfd_bo.user_pages = NULL;
 	list_add(&ctx->kfd_bo.tv.head, &ctx->list);
 
 	i = 0;

@@ -1329,15 +1307,6 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
 	list_del(&bo_list_entry->head);
 	mutex_unlock(&process_info->lock);
 
-	/* Free user pages if necessary */
-	if (mem->user_pages) {
-		pr_debug("%s: Freeing user_pages array\n", __func__);
-		if (mem->user_pages[0])
-			release_pages(mem->user_pages,
-					mem->bo->tbo.ttm->num_pages);
-		kvfree(mem->user_pages);
-	}
-
 	ret = reserve_bo_and_cond_vms(mem, NULL, BO_VM_ALL, &ctx);
 	if (unlikely(ret))
 		return ret;

@@ -1751,25 +1720,11 @@ static int update_invalid_user_pages(struct amdkfd_process_info *process_info,
 
 		bo = mem->bo;
 
-		if (!mem->user_pages) {
-			mem->user_pages =
-				kvmalloc_array(bo->tbo.ttm->num_pages,
-						 sizeof(struct page *),
-						 GFP_KERNEL | __GFP_ZERO);
-			if (!mem->user_pages) {
-				pr_err("%s: Failed to allocate pages array\n",
-				       __func__);
-				return -ENOMEM;
-			}
-		} else if (mem->user_pages[0]) {
-			release_pages(mem->user_pages, bo->tbo.ttm->num_pages);
-		}
-
 		/* Get updated user pages */
 		ret = amdgpu_ttm_tt_get_user_pages(bo->tbo.ttm,
-						   mem->user_pages);
+						   bo->tbo.ttm->pages);
 		if (ret) {
-			mem->user_pages[0] = NULL;
+			bo->tbo.ttm->pages[0] = NULL;
 			pr_info("%s: Failed to get user pages: %d\n",
 				__func__, ret);
 			/* Pretend it succeeded. It will fail later

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +9 -11

 	}
 
 	return 0;

@@ -1863,10 +1812,8 @@ static int validate_invalid_user_pages(struct amdkfd_process_info *process_info)
 
 		bo = mem->bo;
 
-		/* Copy pages array and validate the BO if we got user pages */
-		if (mem->user_pages[0]) {
-			amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm,
-						     mem->user_pages);
+		/* Validate the BO if we got user pages */
+		if (bo->tbo.ttm->pages[0]) {
 			amdgpu_bo_placement_from_domain(bo, mem->domain);
 			ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 			if (ret) {

@@ -1875,16 +1822,16 @@ static int validate_invalid_user_pages(struct amdkfd_process_info *process_info)
 			}
 		}
 
-		/* Validate succeeded, now the BO owns the pages, free
-		 * our copy of the pointer array. Put this BO back on
-		 * the userptr_valid_list. If we need to revalidate
-		 * it, we need to start from scratch.
-		 */
-		kvfree(mem->user_pages);
-		mem->user_pages = NULL;
 		list_move_tail(&mem->validate_list.head,
 			       &process_info->userptr_valid_list);
 
+		/* Stop HMM track the userptr update. We dont check the return
+		 * value for concurrent CPU page table update because we will
+		 * reschedule the restore worker if process_info->evicted_bos
+		 * is updated.
+		 */
+		amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm);
+
 		/* Update mapping. If the BO was not validated
 		 * (because we couldn't get user pages), this will
 		 * clear the page table entries, which will result in

drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c +0 -2

index 5c79da8e1150..3842183970cc 100644
@@ -200,8 +200,6 @@ void amdgpu_bo_list_get_list(struct amdgpu_bo_list *list,
 
 		if (!bo->parent)
 			list_add_tail(&e->tv.head, &bucket[priority]);
-
-		e->user_pages = NULL;
 	}
 
 	/* Connect the sorted buckets in the output list. */

drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.h +1 -2

index 7c5f5d1601e6..4beab2de09b3 100644
@@ -35,8 +35,7 @@ struct amdgpu_bo_list_entry {
 	struct ttm_validate_buffer	tv;
 	struct amdgpu_bo_va		*bo_va;
 	uint32_t			priority;
-	struct page			**user_pages;
-	int				user_invalidated;
+	bool				user_invalidated;
 };
 
 struct amdgpu_bo_list {

drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +0 -0

drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +0 -0

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +0 -2

index 5d518d2bb9be..c8002ee1d415 100644
@@ -229,8 +229,6 @@ static void amdgpu_mn_invalidate_node(struct amdgpu_mn_node *node,
 			true, false, MAX_SCHEDULE_TIMEOUT);
 		if (r <= 0)
 			DRM_ERROR("(%ld) failed to wait for user bo\n", r);
-
-		amdgpu_ttm_tt_mark_user_pages(bo->tbo.ttm);
 	}
 }
 

@@ -353,15 +351,16 @@ static struct hmm_mirror_ops amdgpu_hmm_mirror_ops[] = {
 /**
  * amdgpu_mn_get - create HMM mirror context

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +25 -2

+				struct amdgpu_device *adev,
 				enum amdgpu_mn_type type)
 {
-	struct mm_struct *mm = current->mm;
 	struct amdgpu_mn *amn;
 	unsigned long key = AMDGPU_MN_KEY(mm, type);
 	int r;

@@ -431,7 +430,7 @@ int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)
 	struct list_head bos;
 	struct interval_tree_node *it;
 
-	amn = amdgpu_mn_get(adev, type);
+	amn = amdgpu_mn_get(current->mm, adev, type);
 	if (IS_ERR(amn))
 		return PTR_ERR(amn);
 

@@ -513,3 +512,26 @@ void amdgpu_mn_unregister(struct amdgpu_bo *bo)
 	mutex_unlock(&adev->mn_lock);
 }
 
+/* flags used by HMM internal, not related to CPU/GPU PTE flags */
+static const uint64_t hmm_range_flags[HMM_PFN_FLAG_MAX] = {
+		(1 << 0), /* HMM_PFN_VALID */
+		(1 << 1), /* HMM_PFN_WRITE */
+		0 /* HMM_PFN_DEVICE_PRIVATE */
+};
+
+static const uint64_t hmm_range_values[HMM_PFN_VALUE_MAX] = {
+		0xfffffffffffffffeUL, /* HMM_PFN_ERROR */
+		0, /* HMM_PFN_NONE */
+		0xfffffffffffffffcUL /* HMM_PFN_SPECIAL */
+};
+
+void amdgpu_hmm_init_range(struct hmm_range *range)
+{
+	if (range) {
+		range->flags = hmm_range_flags;
+		range->values = hmm_range_values;
+		range->pfn_shift = PAGE_SHIFT;
+		range->pfns = NULL;
+		INIT_LIST_HEAD(&range->list);
+	}
+}

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h +5 -2

index 0a51fd00021c..4c5e86847a3b 100644
@@ -25,9 +25,10 @@
 #define __AMDGPU_MN_H__
 
 /*
- * MMU Notifier
+ * HMM mirror
  */
 struct amdgpu_mn;
+struct hmm_range;
 
 enum amdgpu_mn_type {
 	AMDGPU_MN_TYPE_GFX,

@@ -37,10 +38,12 @@ enum amdgpu_mn_type {
 #if defined(CONFIG_HMM_MIRROR)
 void amdgpu_mn_lock(struct amdgpu_mn *mn);
 void amdgpu_mn_unlock(struct amdgpu_mn *mn);
-struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev,
+struct amdgpu_mn *amdgpu_mn_get(struct mm_struct *mm,
+				struct amdgpu_device *adev,
 				enum amdgpu_mn_type type);
 int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr);
 void amdgpu_mn_unregister(struct amdgpu_bo *bo);
+void amdgpu_hmm_init_range(struct hmm_range *range);
 #else
 static inline void amdgpu_mn_lock(struct amdgpu_mn *mn) {}
 static inline void amdgpu_mn_unlock(struct amdgpu_mn *mn) {}

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +9 -15

index c91ec3101d00..33057d6114c7 100644
@@ -43,6 +43,7 @@
 #include <linux/pagemap.h>
 #include <linux/debugfs.h>
 #include <linux/iommu.h>
+#include <linux/hmm.h>
 #include "amdgpu.h"
 #include "amdgpu_object.h"
 #include "amdgpu_trace.h"

@@ -705,11 +706,6 @@ static unsigned long amdgpu_ttm_io_mem_pfn(struct ttm_buffer_object *bo,
 /*
  * TTM backend functions.
  */
-struct amdgpu_ttm_gup_task_list {
-	struct list_head	list;
-	struct task_struct	*task;
-};
-
 struct amdgpu_ttm_tt {
 	struct ttm_dma_tt	ttm;
 	u64			offset;

@@ -718,107 +714,105 @@ struct amdgpu_ttm_tt {
 	uint32_t		userflags;
 	spinlock_t              guptasklock;
 	struct list_head        guptasks;
-	atomic_t		mmu_invalidations;
-	uint32_t		last_set_pages;
+	struct hmm_range	range;
 };
 
 /**
- * amdgpu_ttm_tt_get_user_pages - Pin pages of memory pointed to by a USERPTR
- * pointer to memory
+ * amdgpu_ttm_tt_get_user_pages - get device accessible pages that back user
+ * memory and start HMM tracking CPU page table update
  *
- * Called by amdgpu_gem_userptr_ioctl() and amdgpu_cs_parser_bos().
- * This provides a wrapper around the get_user_pages() call to provide
- * device accessible pages that back user memory.
+ * Calling function must call amdgpu_ttm_tt_userptr_range_done() once and only
+ * once afterwards to stop HMM tracking
  */
 int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages)
 {
 	struct amdgpu_ttm_tt *gtt = (void *)ttm;
 	struct mm_struct *mm = gtt->usertask->mm;
-	unsigned int flags = 0;
-	unsigned pinned = 0;
-	int r;
+	unsigned long end = gtt->userptr + ttm->num_pages * PAGE_SIZE;
+	struct hmm_range *range = &gtt->range;
+	int r, i;
 

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +3 -10

 	if (gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) {
 		/*
 		 * check that we only use anonymous memory to prevent problems
 		 * with writeback
 		 */
-		unsigned long end = gtt->userptr + ttm->num_pages * PAGE_SIZE;
-		struct vm_area_struct *vma;
+		struct vm_area_struct *vma = range->vma;
 
-		vma = find_vma(mm, gtt->userptr);
-		if (!vma || vma->vm_file || vma->vm_end < end) {
-			up_read(&mm->mmap_sem);
-			return -EPERM;
-		}
+		if (!vma || vma->vm_file || vma->vm_end < end)
+			range->vma = NULL;
 	}
 
-	/* loop enough times using contiguous pages of memory */
-	do {
-		unsigned num_pages = ttm->num_pages - pinned;

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +9 -6

+		goto out;
+	}
 
-		if (mm == current->mm)
-			r = get_user_pages(userptr, num_pages, flags, p, NULL);
-		else
-			r = get_user_pages_remote(gtt->usertask,
-					mm, userptr, num_pages,
-					flags, p, NULL, NULL);
+	range->pfns = kvmalloc_array(ttm->num_pages, sizeof(uint64_t),
+				     GFP_KERNEL);
+	if (range->pfns == NULL) {
+		r = -ENOMEM;
+		goto out;
+	}
+	range->start = gtt->userptr;

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +11 -7

-		if (r < 0)
-			goto release_pages;
+	/* This may trigger page table update */
+	r = hmm_vma_fault(range, true);
+	if (r)
+		goto out_free_pfns;
 
-		pinned += r;
+	up_read(&mm->mmap_sem);
 
-	} while (pinned < ttm->num_pages);
+	for (i = 0; i < ttm->num_pages; i++)
+		pages[i] = hmm_pfn_to_page(range, range->pfns[i]);
 
-	up_read(&mm->mmap_sem);
 	return 0;
 
-release_pages:
-	release_pages(pages, pinned);
+out_free_pfns:
+	kvfree(range->pfns);
+	range->pfns = NULL;
+out:
 	up_read(&mm->mmap_sem);
 	return r;
 }

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +1 -1

  */
-void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page **pages)
+bool amdgpu_ttm_tt_get_user_pages_done(struct ttm_tt *ttm)
 {

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +4 -38

 }
 
 /**

@@ -903,9 +897,6 @@ static void amdgpu_ttm_tt_unpin_userptr(struct ttm_tt *ttm)
 	/* unmap the pages mapped to the device */
 	dma_unmap_sg(adev->dev, ttm->sg->sgl, ttm->sg->nents, direction);
 
-	/* mark the pages as dirty */
-	amdgpu_ttm_tt_mark_user_pages(ttm);
-
 	sg_free_table(ttm->sg);
 }
 

@@ -1207,7 +1198,7 @@ static void amdgpu_ttm_tt_unpopulate(struct ttm_tt *ttm)
 	bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
 
 	if (gtt && gtt->userptr) {
-		amdgpu_ttm_tt_set_user_pages(ttm, NULL);
+		memset(ttm->pages, 0, ttm->num_pages * sizeof(struct page *));
 		kfree(ttm->sg);
 		ttm->page_flags &= ~TTM_PAGE_FLAG_SG;
 		return;

@@ -1258,8 +1249,6 @@ int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,
 
 	spin_lock_init(&gtt->guptasklock);
 	INIT_LIST_HEAD(&gtt->guptasks);
-	atomic_set(&gtt->mmu_invalidations, 0);
-	gtt->last_set_pages = 0;
 
 	return 0;
 }

@@ -1289,7 +1278,6 @@ bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,
 				  unsigned long end)
 {
 	struct amdgpu_ttm_tt *gtt = (void *)ttm;
-	struct amdgpu_ttm_gup_task_list *entry;
 	unsigned long size;
 
 	if (gtt == NULL || !gtt->userptr)

@@ -1302,48 +1290,20 @@ bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,
 	if (gtt->userptr > end || gtt->userptr + size <= start)
 		return false;
 
-	/* Search the lists of tasks that hold this mapping and see
-	 * if current is one of them.  If it is return false.
-	 */
-	spin_lock(&gtt->guptasklock);
-	list_for_each_entry(entry, &gtt->guptasks, list) {
-		if (entry->task == current) {
-			spin_unlock(&gtt->guptasklock);
-			return false;
-		}
-	}
-	spin_unlock(&gtt->guptasklock);
-
-	atomic_inc(&gtt->mmu_invalidations);
-
 	return true;
 }
 
 /**
- * amdgpu_ttm_tt_userptr_invalidated - Has the ttm_tt object been invalidated?
+ * amdgpu_ttm_tt_is_userptr - Have the pages backing by userptr?
  */
-bool amdgpu_ttm_tt_userptr_invalidated(struct ttm_tt *ttm,
-				       int *last_invalidated)
-{
-	struct amdgpu_ttm_tt *gtt = (void *)ttm;
-	int prev_invalidated = *last_invalidated;
-
-	*last_invalidated = atomic_read(&gtt->mmu_invalidations);
-	return prev_invalidated != *last_invalidated;
-}
-
-/**
- * amdgpu_ttm_tt_userptr_needs_pages - Have the pages backing this ttm_tt object
- * been invalidated since the last time they've been set?
- */
-bool amdgpu_ttm_tt_userptr_needs_pages(struct ttm_tt *ttm)
+bool amdgpu_ttm_tt_is_userptr(struct ttm_tt *ttm)
 {
 	struct amdgpu_ttm_tt *gtt = (void *)ttm;
 
 	if (gtt == NULL || !gtt->userptr)
 		return false;
 
-	return atomic_read(&gtt->mmu_invalidations) != gtt->last_set_pages;
+	return true;
 }
 
 /**

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h +2 -2

index b5b2d101f7db..c5a1e78a09b1 100644
@@ -102,7 +102,7 @@ int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo);
 int amdgpu_ttm_recover_gart(struct ttm_buffer_object *tbo);
 
 int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages);
-void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page **pages);
+bool amdgpu_ttm_tt_get_user_pages_done(struct ttm_tt *ttm);
 void amdgpu_ttm_tt_mark_user_pages(struct ttm_tt *ttm);
 int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,
 				     uint32_t flags);

@@ -112,7 +112,7 @@ bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,
 				  unsigned long end);
 bool amdgpu_ttm_tt_userptr_invalidated(struct ttm_tt *ttm,
 				       int *last_invalidated);
-bool amdgpu_ttm_tt_userptr_needs_pages(struct ttm_tt *ttm);
+bool amdgpu_ttm_tt_is_userptr(struct ttm_tt *ttm);
 bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm);
 uint64_t amdgpu_ttm_tt_pde_flags(struct ttm_tt *ttm, struct ttm_mem_reg *mem);
 uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +0 -0

index f059ec314f2b..ab8fa84b27a6 100644

This is a nice improvement from the last version. I still see some potential problems. See inline ...

I'm skipping over the CS and GEM parts. I hope Christian can review those parts.

Quoted 152 lines

Use HMM helper function hmm_vma_fault() to get physical pages backing userptr and start CPU page table update track of those pages. Then use hmm_vma_range_done() to check if those pages are updated before amdgpu_cs_submit for gfx or before user queues are resumed for kfd.

If userptr pages are updated, for gfx, amdgpu_cs_ioctl will restart from scratch, for kfd, restore worker is rescheduled to retry.

HMM simplify the CPU page table concurrent update check, so remove guptasklock, mmu_invalidations, last_set_pages fields from amdgpu_ttm_tt struct.

HMM does not pin the page (increase page ref count), so remove related operations like release_pages(), put_page(), mark_page_dirty().

Change-Id Id2d3130378b44a774e0d77156102a20a203b5036

Signed-off-by Philip Yang <[email protected]>

---
  drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h    |   1 -
  .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c  |  81 ++------
  drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c   |   2 -
  drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.h   |   3 +-
  drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c        | 188 +++++++++---------
  drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c       |  14 +-
  drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c        |  34 +++-
  drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h        |   7 +-
  drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c       | 160 ++++++---------
  drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h       |   4 +-
  drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c        |   1 -
  11 files changed, 206 insertions(+), 289 deletions(-)

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +0 -1

index bcf587b4ba98..b492dd9e541a 100644
@@ -62,7 +62,6 @@ struct kgd_mem {
 
  	atomic_t invalid;
  	struct amdkfd_process_info *process_info;
-	struct page **user_pages;
 
  	struct amdgpu_sync sync;
 

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +5 -50

index b29ef088fa14..75833b40507b 100644
@@ -580,28 +580,12 @@ static int init_user_pages(struct kgd_mem *mem, struct mm_struct *mm,
  		goto out;
  	}
 
-	/* If no restore worker is running concurrently, user_pages
-	 * should not be allocated
-	 */
-	WARN(mem->user_pages, "Leaking user_pages array");
-
-	mem->user_pages = kvmalloc_array(bo->tbo.ttm->num_pages,
-					   sizeof(struct page *),
-					   GFP_KERNEL | __GFP_ZERO);
-	if (!mem->user_pages) {
-		pr_err("%s: Failed to allocate pages array\n", __func__);
-		ret = -ENOMEM;
-		goto unregister_out;
-	}
-
-	ret = amdgpu_ttm_tt_get_user_pages(bo->tbo.ttm, mem->user_pages);
+	ret = amdgpu_ttm_tt_get_user_pages(bo->tbo.ttm, bo->tbo.ttm->pages);
  	if (ret) {
  		pr_err("%s: Failed to get user pages: %d\n", __func__, ret);
-		goto free_out;
+		goto unregister_out;
  	}
 
-	amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm, mem->user_pages);
-
  	ret = amdgpu_bo_reserve(bo, true);
  	if (ret) {
  		pr_err("%s: Failed to reserve BO\n", __func__);

@@ -614,11 +598,7 @@ static int init_user_pages(struct kgd_mem *mem, struct mm_struct *mm,
  	amdgpu_bo_unreserve(bo);
 
  release_out:
-	if (ret)
-		release_pages(mem->user_pages, bo->tbo.ttm->num_pages);
-free_out:
-	kvfree(mem->user_pages);
-	mem->user_pages = NULL;
+	amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm);
  unregister_out:
  	if (ret)
  		amdgpu_mn_unregister(bo);

@@ -677,7 +657,6 @@ static int reserve_bo_and_vm(struct kgd_mem *mem,
  	ctx->kfd_bo.priority = 0;
  	ctx->kfd_bo.tv.bo = &bo->tbo;
  	ctx->kfd_bo.tv.num_shared = 1;
-	ctx->kfd_bo.user_pages = NULL;
  	list_add(&ctx->kfd_bo.tv.head, &ctx->list);
 
  	amdgpu_vm_get_pd_bo(vm, &ctx->list, &ctx->vm_pd[0]);

@@ -741,7 +720,6 @@ static int reserve_bo_and_cond_vms(struct kgd_mem *mem,
  	ctx->kfd_bo.priority = 0;
  	ctx->kfd_bo.tv.bo = &bo->tbo;
  	ctx->kfd_bo.tv.num_shared = 1;
-	ctx->kfd_bo.user_pages = NULL;
  	list_add(&ctx->kfd_bo.tv.head, &ctx->list);
 
  	i = 0;

@@ -1329,15 +1307,6 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
  	list_del(&bo_list_entry->head);
  	mutex_unlock(&process_info->lock);
 
-	/* Free user pages if necessary */
-	if (mem->user_pages) {
-		pr_debug("%s: Freeing user_pages array\n", __func__);
-		if (mem->user_pages[0])
-			release_pages(mem->user_pages,
-					mem->bo->tbo.ttm->num_pages);
-		kvfree(mem->user_pages);
-	}
-
  	ret = reserve_bo_and_cond_vms(mem, NULL, BO_VM_ALL, &ctx);
  	if (unlikely(ret))
  		return ret;

@@ -1751,25 +1720,11 @@ static int update_invalid_user_pages(struct amdkfd_process_info *process_info,
 
  		bo = mem->bo;
 
-		if (!mem->user_pages) {
-			mem->user_pages =
-				kvmalloc_array(bo->tbo.ttm->num_pages,
-						 sizeof(struct page *),
-						 GFP_KERNEL | __GFP_ZERO);
-			if (!mem->user_pages) {
-				pr_err("%s: Failed to allocate pages array\n",
-				       __func__);
-				return -ENOMEM;
-			}
-		} else if (mem->user_pages[0]) {
-			release_pages(mem->user_pages, bo->tbo.ttm->num_pages);
-		}
-
  		/* Get updated user pages */
  		ret = amdgpu_ttm_tt_get_user_pages(bo->tbo.ttm,
-						   mem->user_pages);
+						   bo->tbo.ttm->pages);
  		if (ret) {
-			mem->user_pages[0] = NULL;
+			bo->tbo.ttm->pages[0] = NULL;
  			pr_info("%s: Failed to get user pages: %d\n",
  				__func__, ret);
  			/* Pretend it succeeded. It will fail later

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c static int update_invalid_user_pages(struct amdkfd_process_info *process_info, · line 1786

Quoted 9 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +0 -6

@@ -1778,12 +1733,6 @@ static int update_invalid_user_pages(struct amdkfd_process_info *process_info,
  			 * stalled user mode queues.
  			 */
  		}
-
-		/* Mark the BO as valid unless it was invalidated
-		 * again concurrently
-		 */
-		if (atomic_cmpxchg(&mem->invalid, invalid, 0) != invalid)
-			return -EAGAIN;

I think you're removing this to make sure we exactly pair up amdgpu_ttm_tt_get_user_pages and amdgpu_ttm_tt_get_user_pages_done calls. But there are other error conditions in validate_invalid_user_pages that would skip the amdgpu_ttm_tt_get_user_pages_done calls. That will need to be handled properly in the error-handling code paths in validate_invalid_user_pages.

Quoted 68 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +9 -11

  	}
 
  	return 0;

@@ -1863,10 +1812,8 @@ static int validate_invalid_user_pages(struct amdkfd_process_info *process_info)
 
  		bo = mem->bo;
 
-		/* Copy pages array and validate the BO if we got user pages */
-		if (mem->user_pages[0]) {
-			amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm,
-						     mem->user_pages);
+		/* Validate the BO if we got user pages */
+		if (bo->tbo.ttm->pages[0]) {
  			amdgpu_bo_placement_from_domain(bo, mem->domain);
  			ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
  			if (ret) {

@@ -1875,16 +1822,16 @@ static int validate_invalid_user_pages(struct amdkfd_process_info *process_info)
  			}
  		}
 
-		/* Validate succeeded, now the BO owns the pages, free
-		 * our copy of the pointer array. Put this BO back on
-		 * the userptr_valid_list. If we need to revalidate
-		 * it, we need to start from scratch.
-		 */
-		kvfree(mem->user_pages);
-		mem->user_pages = NULL;
  		list_move_tail(&mem->validate_list.head,
  			       &process_info->userptr_valid_list);
 
+		/* Stop HMM track the userptr update. We dont check the return
+		 * value for concurrent CPU page table update because we will
+		 * reschedule the restore worker if process_info->evicted_bos
+		 * is updated.
+		 */
+		amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm);
+
  		/* Update mapping. If the BO was not validated
  		 * (because we couldn't get user pages), this will
  		 * clear the page table entries, which will result in

drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c +0 -2

index 5c79da8e1150..3842183970cc 100644
@@ -200,8 +200,6 @@ void amdgpu_bo_list_get_list(struct amdgpu_bo_list *list,
 
  		if (!bo->parent)
  			list_add_tail(&e->tv.head, &bucket[priority]);
-
-		e->user_pages = NULL;
  	}
 
  	/* Connect the sorted buckets in the output list. */

drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.h +1 -2

index 7c5f5d1601e6..4beab2de09b3 100644
@@ -35,8 +35,7 @@ struct amdgpu_bo_list_entry {
  	struct ttm_validate_buffer	tv;
  	struct amdgpu_bo_va		*bo_va;
  	uint32_t			priority;
-	struct page			**user_pages;
-	int				user_invalidated;
+	bool				user_invalidated;
  };
 
  struct amdgpu_bo_list {

drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +0 -0

[snip]

Quoted 1 line

drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +0 -0

[snip]

Quoted 16 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +0 -2

index 5d518d2bb9be..c8002ee1d415 100644
@@ -229,8 +229,6 @@ static void amdgpu_mn_invalidate_node(struct amdgpu_mn_node *node,
  			true, false, MAX_SCHEDULE_TIMEOUT);
  		if (r <= 0)
  			DRM_ERROR("(%ld) failed to wait for user bo\n", r);
-
-		amdgpu_ttm_tt_mark_user_pages(bo->tbo.ttm);
  	}
  }
 

@@ -353,15 +351,16 @@ static struct hmm_mirror_ops amdgpu_hmm_mirror_ops[] = {
  /**
   * amdgpu_mn_get - create HMM mirror context

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c static struct hmm_mirror_ops amdgpu_hmm_mirror_ops[] = { · line 360

Quoted 10 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +3 -2

   *
+ * @mm: the mm struct
   * @adev: amdgpu device pointer
   * @type: type of MMU notifier context
   *
- * Creates a HMM mirror context for current->mm.
+ * Creates a HMM mirror context for mm.
   */
-struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev,
+struct amdgpu_mn *amdgpu_mn_get(struct mm_struct *mm,

I think this change isn't needed any more. All callers of this function specify mm as current->mm anyway.

Quoted 129 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +25 -2

+				struct amdgpu_device *adev,
  				enum amdgpu_mn_type type)
  {
-	struct mm_struct *mm = current->mm;
  	struct amdgpu_mn *amn;
  	unsigned long key = AMDGPU_MN_KEY(mm, type);
  	int r;

@@ -431,7 +430,7 @@ int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)
  	struct list_head bos;
  	struct interval_tree_node *it;
 
-	amn = amdgpu_mn_get(adev, type);
+	amn = amdgpu_mn_get(current->mm, adev, type);
  	if (IS_ERR(amn))
  		return PTR_ERR(amn);
 

@@ -513,3 +512,26 @@ void amdgpu_mn_unregister(struct amdgpu_bo *bo)
  	mutex_unlock(&adev->mn_lock);
  }
 
+/* flags used by HMM internal, not related to CPU/GPU PTE flags */
+static const uint64_t hmm_range_flags[HMM_PFN_FLAG_MAX] = {
+		(1 << 0), /* HMM_PFN_VALID */
+		(1 << 1), /* HMM_PFN_WRITE */
+		0 /* HMM_PFN_DEVICE_PRIVATE */
+};
+
+static const uint64_t hmm_range_values[HMM_PFN_VALUE_MAX] = {
+		0xfffffffffffffffeUL, /* HMM_PFN_ERROR */
+		0, /* HMM_PFN_NONE */
+		0xfffffffffffffffcUL /* HMM_PFN_SPECIAL */
+};
+
+void amdgpu_hmm_init_range(struct hmm_range *range)
+{
+	if (range) {
+		range->flags = hmm_range_flags;
+		range->values = hmm_range_values;
+		range->pfn_shift = PAGE_SHIFT;
+		range->pfns = NULL;
+		INIT_LIST_HEAD(&range->list);
+	}
+}

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h +5 -2

index 0a51fd00021c..4c5e86847a3b 100644
@@ -25,9 +25,10 @@
  #define __AMDGPU_MN_H__
 
  /*
- * MMU Notifier
+ * HMM mirror
   */
  struct amdgpu_mn;
+struct hmm_range;
 
  enum amdgpu_mn_type {
  	AMDGPU_MN_TYPE_GFX,

@@ -37,10 +38,12 @@ enum amdgpu_mn_type {
  #if defined(CONFIG_HMM_MIRROR)
  void amdgpu_mn_lock(struct amdgpu_mn *mn);
  void amdgpu_mn_unlock(struct amdgpu_mn *mn);
-struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev,
+struct amdgpu_mn *amdgpu_mn_get(struct mm_struct *mm,
+				struct amdgpu_device *adev,
  				enum amdgpu_mn_type type);
  int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr);
  void amdgpu_mn_unregister(struct amdgpu_bo *bo);
+void amdgpu_hmm_init_range(struct hmm_range *range);
  #else
  static inline void amdgpu_mn_lock(struct amdgpu_mn *mn) {}
  static inline void amdgpu_mn_unlock(struct amdgpu_mn *mn) {}

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +9 -15

index c91ec3101d00..33057d6114c7 100644
@@ -43,6 +43,7 @@
  #include <linux/pagemap.h>
  #include <linux/debugfs.h>
  #include <linux/iommu.h>
+#include <linux/hmm.h>
  #include "amdgpu.h"
  #include "amdgpu_object.h"
  #include "amdgpu_trace.h"

@@ -705,11 +706,6 @@ static unsigned long amdgpu_ttm_io_mem_pfn(struct ttm_buffer_object *bo,
  /*
   * TTM backend functions.
   */
-struct amdgpu_ttm_gup_task_list {
-	struct list_head	list;
-	struct task_struct	*task;
-};
-
  struct amdgpu_ttm_tt {
  	struct ttm_dma_tt	ttm;
  	u64			offset;

@@ -718,107 +714,105 @@ struct amdgpu_ttm_tt {
  	uint32_t		userflags;
  	spinlock_t              guptasklock;
  	struct list_head        guptasks;
-	atomic_t		mmu_invalidations;
-	uint32_t		last_set_pages;
+	struct hmm_range	range;
  };
 
  /**
- * amdgpu_ttm_tt_get_user_pages - Pin pages of memory pointed to by a USERPTR
- * pointer to memory
+ * amdgpu_ttm_tt_get_user_pages - get device accessible pages that back user
+ * memory and start HMM tracking CPU page table update
   *
- * Called by amdgpu_gem_userptr_ioctl() and amdgpu_cs_parser_bos().
- * This provides a wrapper around the get_user_pages() call to provide
- * device accessible pages that back user memory.
+ * Calling function must call amdgpu_ttm_tt_userptr_range_done() once and only
+ * once afterwards to stop HMM tracking
   */
  int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages)
  {
  	struct amdgpu_ttm_tt *gtt = (void *)ttm;
  	struct mm_struct *mm = gtt->usertask->mm;
-	unsigned int flags = 0;
-	unsigned pinned = 0;
-	int r;
+	unsigned long end = gtt->userptr + ttm->num_pages * PAGE_SIZE;
+	struct hmm_range *range = &gtt->range;
+	int r, i;
 

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c struct amdgpu_ttm_tt { · line 742

Quoted 10 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +2 -2

  	if (!mm) /* Happens during process shutdown */
  		return -ESRCH;
 
-	if (!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY))
-		flags |= FOLL_WRITE;
+	amdgpu_hmm_init_range(range);
 
  	down_read(&mm->mmap_sem);
 
+	range->vma = find_vma(mm, gtt->userptr);

There is still an assumption here that our userptr BOs cover only a single VM. This is a new limitation with HMM that the previous get_user_pages-based implementation did not have. We're probably OK with this for now. But we may run into regressions if there are any ROCm application that register user memory spanning multiple VMAs. If that happens, we may need to generalize this so that each userptr BO can have multiple HMM ranges.

Quoted 21 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +3 -10

  	if (gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) {
  		/*
  		 * check that we only use anonymous memory to prevent problems
  		 * with writeback
  		 */
-		unsigned long end = gtt->userptr + ttm->num_pages * PAGE_SIZE;
-		struct vm_area_struct *vma;
+		struct vm_area_struct *vma = range->vma;
 
-		vma = find_vma(mm, gtt->userptr);
-		if (!vma || vma->vm_file || vma->vm_end < end) {
-			up_read(&mm->mmap_sem);
-			return -EPERM;
-		}
+		if (!vma || vma->vm_file || vma->vm_end < end)
+			range->vma = NULL;
  	}
 
-	/* loop enough times using contiguous pages of memory */
-	do {
-		unsigned num_pages = ttm->num_pages - pinned;

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c struct amdgpu_ttm_tt { · line 755

Quoted 10 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +2 -8

-		uint64_t userptr = gtt->userptr + pinned * PAGE_SIZE;
-		struct page **p = pages + pinned;
-		struct amdgpu_ttm_gup_task_list guptask;
-
-		guptask.task = current;
-		spin_lock(&gtt->guptasklock);
-		list_add(&guptask.list, &gtt->guptasks);
-		spin_unlock(&gtt->guptasklock);
+	if (!range->vma) {
+		r = -EPERM;

You're treating permissions errors and errors finding a VMA the same way. And I think you don't detect failures to find a VMA properly in the first place. find_vma may return a VMA that doesn't include the address provided. It's documented as "Look up the first VMA which satisfies addr < vm_end,  NULL if none." You should also check that the VMA covers the entire address range of the userptr BO. There is a helper function in linux/mm.h that does that: range_in_vma(range->vma, gtt->userptr, gtt->userptr + gtt->userptr + ttm->num_pages * PAGE_SIZE). You should check this condition in all cases.

There should be two distinct types of errors. If find_vma fails to find a matching VMA, that should be -EFAULT. If the gtt->userflags don't match the VMA properties, that should be -EPERM.

Quoted 16 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +9 -6

+		goto out;
+	}
 
-		if (mm == current->mm)
-			r = get_user_pages(userptr, num_pages, flags, p, NULL);
-		else
-			r = get_user_pages_remote(gtt->usertask,
-					mm, userptr, num_pages,
-					flags, p, NULL, NULL);
+	range->pfns = kvmalloc_array(ttm->num_pages, sizeof(uint64_t),
+				     GFP_KERNEL);
+	if (range->pfns == NULL) {
+		r = -ENOMEM;
+		goto out;
+	}
+	range->start = gtt->userptr;

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c struct amdgpu_ttm_tt { · line 772

Quoted 10 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +6 -3

+	range->end = end;
 
-		spin_lock(&gtt->guptasklock);
-		list_del(&guptask.list);
-		spin_unlock(&gtt->guptasklock);
+	for (i = 0; i < ttm->num_pages; i++) {
+		range->pfns[i] = range->flags[HMM_PFN_VALID];
+		range->pfns[i] |= amdgpu_ttm_tt_is_readonly(ttm) ?
+					0 : range->flags[HMM_PFN_WRITE];
+	}

You could optimize this slightly. You don't need to call amdgpu_ttm_tt_is_readonly in each loop iteration. The flags are the same for all pfns. So do the conditional thing on range->pfns[0], and then just copy that to all the remaining pfns.

Quoted 27 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +11 -7

-		if (r < 0)
-			goto release_pages;
+	/* This may trigger page table update */
+	r = hmm_vma_fault(range, true);
+	if (r)
+		goto out_free_pfns;
 
-		pinned += r;
+	up_read(&mm->mmap_sem);
 
-	} while (pinned < ttm->num_pages);
+	for (i = 0; i < ttm->num_pages; i++)
+		pages[i] = hmm_pfn_to_page(range, range->pfns[i]);
 
-	up_read(&mm->mmap_sem);
  	return 0;
 
-release_pages:
-	release_pages(pages, pinned);
+out_free_pfns:
+	kvfree(range->pfns);
+	range->pfns = NULL;
+out:
  	up_read(&mm->mmap_sem);
  	return r;
  }

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c struct amdgpu_ttm_tt { · line 798

Quoted 10 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +3 -4

 
  /**
- * amdgpu_ttm_tt_set_user_pages - Copy pages in, putting old pages as necessary.
+ * amdgpu_ttm_tt_userptr_range_done - stop HMM track the CPU page table change
+ * Check if the pages backing this ttm range have been invalidated
   *
- * Called by amdgpu_cs_list_validate(). This creates the page list
- * that backs user memory and will ultimately be mapped into the device
- * address space.
+ * Returns: true if pages are invalidated since the last time they've been set

I still don't like that you're inverting the return value of hmm_vma_range_done. This will get confusing in the future. Could this function instead return "true if pages are still valid"?

Quoted 4 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +1 -1

   */
-void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page **pages)
+bool amdgpu_ttm_tt_get_user_pages_done(struct ttm_tt *ttm)
  {

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c struct amdgpu_ttm_tt { · line 806

Quoted 10 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +3 -5

  	struct amdgpu_ttm_tt *gtt = (void *)ttm;
-	unsigned i;
+	bool r;
 
-	gtt->last_set_pages = atomic_read(&gtt->mmu_invalidations);
-	for (i = 0; i < ttm->num_pages; ++i) {
-		if (ttm->pages[i])
-			put_page(ttm->pages[i]);
+	if (gtt == NULL || !gtt->userptr)
+		return false;

Use "!gtt" instead of "gtt == NULL". Also, returning "false" here means the pages are still valid. That seems like the wrong answer for this kind of failure. "false" would make sense here if you followed my suggestion above to return true, if the pages are still valid.

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c struct amdgpu_ttm_tt { · line 815

Quoted 10 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +7 -1

-		ttm->pages[i] = pages ? pages[i] : NULL;
+	r = hmm_vma_range_done(&gtt->range);
+
+	if (gtt->range.pfns) {
+		kvfree(gtt->range.pfns);
+		gtt->range.pfns = NULL;
  	}
+
+	return !r;

I still don't like that you're inverting the return value of hmm_vma_range_done. This will get confusing in the future.

Regards,
  Felix

Quoted 121 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +4 -38

  }
 
  /**

@@ -903,9 +897,6 @@ static void amdgpu_ttm_tt_unpin_userptr(struct ttm_tt *ttm)
  	/* unmap the pages mapped to the device */
  	dma_unmap_sg(adev->dev, ttm->sg->sgl, ttm->sg->nents, direction);
 
-	/* mark the pages as dirty */
-	amdgpu_ttm_tt_mark_user_pages(ttm);
-
  	sg_free_table(ttm->sg);
  }
 

@@ -1207,7 +1198,7 @@ static void amdgpu_ttm_tt_unpopulate(struct ttm_tt *ttm)
  	bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
 
  	if (gtt && gtt->userptr) {
-		amdgpu_ttm_tt_set_user_pages(ttm, NULL);
+		memset(ttm->pages, 0, ttm->num_pages * sizeof(struct page *));
  		kfree(ttm->sg);
  		ttm->page_flags &= ~TTM_PAGE_FLAG_SG;
  		return;

@@ -1258,8 +1249,6 @@ int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,
 
  	spin_lock_init(&gtt->guptasklock);
  	INIT_LIST_HEAD(&gtt->guptasks);
-	atomic_set(&gtt->mmu_invalidations, 0);
-	gtt->last_set_pages = 0;
 
  	return 0;
  }

@@ -1289,7 +1278,6 @@ bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,
  				  unsigned long end)
  {
  	struct amdgpu_ttm_tt *gtt = (void *)ttm;
-	struct amdgpu_ttm_gup_task_list *entry;
  	unsigned long size;
 
  	if (gtt == NULL || !gtt->userptr)

@@ -1302,48 +1290,20 @@ bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,
  	if (gtt->userptr > end || gtt->userptr + size <= start)
  		return false;
 
-	/* Search the lists of tasks that hold this mapping and see
-	 * if current is one of them.  If it is return false.
-	 */
-	spin_lock(&gtt->guptasklock);
-	list_for_each_entry(entry, &gtt->guptasks, list) {
-		if (entry->task == current) {
-			spin_unlock(&gtt->guptasklock);
-			return false;
-		}
-	}
-	spin_unlock(&gtt->guptasklock);
-
-	atomic_inc(&gtt->mmu_invalidations);
-
  	return true;
  }
 
  /**
- * amdgpu_ttm_tt_userptr_invalidated - Has the ttm_tt object been invalidated?
+ * amdgpu_ttm_tt_is_userptr - Have the pages backing by userptr?
   */
-bool amdgpu_ttm_tt_userptr_invalidated(struct ttm_tt *ttm,
-				       int *last_invalidated)
-{
-	struct amdgpu_ttm_tt *gtt = (void *)ttm;
-	int prev_invalidated = *last_invalidated;
-
-	*last_invalidated = atomic_read(&gtt->mmu_invalidations);
-	return prev_invalidated != *last_invalidated;
-}
-
-/**
- * amdgpu_ttm_tt_userptr_needs_pages - Have the pages backing this ttm_tt object
- * been invalidated since the last time they've been set?
- */
-bool amdgpu_ttm_tt_userptr_needs_pages(struct ttm_tt *ttm)
+bool amdgpu_ttm_tt_is_userptr(struct ttm_tt *ttm)
  {
  	struct amdgpu_ttm_tt *gtt = (void *)ttm;
 
  	if (gtt == NULL || !gtt->userptr)
  		return false;
 
-	return atomic_read(&gtt->mmu_invalidations) != gtt->last_set_pages;
+	return true;
  }
 
  /**

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h +2 -2

index b5b2d101f7db..c5a1e78a09b1 100644
@@ -102,7 +102,7 @@ int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo);
  int amdgpu_ttm_recover_gart(struct ttm_buffer_object *tbo);
 
  int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages);
-void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page **pages);
+bool amdgpu_ttm_tt_get_user_pages_done(struct ttm_tt *ttm);
  void amdgpu_ttm_tt_mark_user_pages(struct ttm_tt *ttm);
  int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,
  				     uint32_t flags);

@@ -112,7 +112,7 @@ bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,
  				  unsigned long end);
  bool amdgpu_ttm_tt_userptr_invalidated(struct ttm_tt *ttm,
  				       int *last_invalidated);
-bool amdgpu_ttm_tt_userptr_needs_pages(struct ttm_tt *ttm);
+bool amdgpu_ttm_tt_is_userptr(struct ttm_tt *ttm);
  bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm);
  uint64_t amdgpu_ttm_tt_pde_flags(struct ttm_tt *ttm, struct ttm_mem_reg *mem);
  uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +0 -0

index f059ec314f2b..ab8fa84b27a6 100644
@@ -619,7 +619,6 @@ void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
  	entry->tv.bo = &vm->root.base.bo->tbo;
  	/* One for the VM updates, one for TTM and one for the CS job */
  	entry->tv.num_shared = 3;

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm, · line 624

Quoted 4 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +0 -1

-	entry->user_pages = NULL;
  	list_add(&entry->tv.head, validated);
  }
 

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

This is a nice improvement from the last version. I still see some potential problems. See inline ...

I'm skipping over the CS and GEM parts. I hope Christian can review those parts.

Quoted 159 lines · Philip

Use HMM helper function hmm_vma_fault() to get physical pages backing userptr and start CPU page table update track of those pages. Then use hmm_vma_range_done() to check if those pages are updated before amdgpu_cs_submit for gfx or before user queues are resumed for kfd.

If userptr pages are updated, for gfx, amdgpu_cs_ioctl will restart from scratch, for kfd, restore worker is rescheduled to retry.

HMM simplify the CPU page table concurrent update check, so remove guptasklock, mmu_invalidations, last_set_pages fields from amdgpu_ttm_tt struct.

HMM does not pin the page (increase page ref count), so remove related operations like release_pages(), put_page(), mark_page_dirty().

Change-Id Id2d3130378b44a774e0d77156102a20a203b5036

Signed-off-by Philip Yang <[email protected]>

---
  drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h    |   1 -
  .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c  |  81 ++------
  drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c   |   2 -
  drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.h   |   3 +-
  drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c        | 188 +++++++++---------
  drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c       |  14 +-
  drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c        |  34 +++-
  drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h        |   7 +-
  drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c       | 160 ++++++---------
  drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h       |   4 +-
  drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c        |   1 -
  11 files changed, 206 insertions(+), 289 deletions(-)

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +0 -1

index bcf587b4ba98..b492dd9e541a 100644
@@ -62,7 +62,6 @@ struct kgd_mem {
 
  	atomic_t invalid;
  	struct amdkfd_process_info *process_info;
-	struct page **user_pages;
 
  	struct amdgpu_sync sync;
 

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +5 -54

index b29ef088fa14..75833b40507b 100644
@@ -580,28 +580,12 @@ static int init_user_pages(struct kgd_mem *mem, struct mm_struct *mm,
  		goto out;
  	}
 
-	/* If no restore worker is running concurrently, user_pages
-	 * should not be allocated
-	 */
-	WARN(mem->user_pages, "Leaking user_pages array");
-
-	mem->user_pages = kvmalloc_array(bo->tbo.ttm->num_pages,
-					   sizeof(struct page *),
-					   GFP_KERNEL | __GFP_ZERO);
-	if (!mem->user_pages) {
-		pr_err("%s: Failed to allocate pages array\n", __func__);
-		ret = -ENOMEM;
-		goto unregister_out;
-	}
-
-	ret = amdgpu_ttm_tt_get_user_pages(bo->tbo.ttm, mem->user_pages);
+	ret = amdgpu_ttm_tt_get_user_pages(bo->tbo.ttm, bo->tbo.ttm->pages);
  	if (ret) {
  		pr_err("%s: Failed to get user pages: %d\n", __func__, ret);
-		goto free_out;
+		goto unregister_out;
  	}
 
-	amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm, mem->user_pages);
-
  	ret = amdgpu_bo_reserve(bo, true);
  	if (ret) {
  		pr_err("%s: Failed to reserve BO\n", __func__);

@@ -614,11 +598,7 @@ static int init_user_pages(struct kgd_mem *mem, struct mm_struct *mm,
  	amdgpu_bo_unreserve(bo);
 
  release_out:
-	if (ret)
-		release_pages(mem->user_pages, bo->tbo.ttm->num_pages);
-free_out:
-	kvfree(mem->user_pages);
-	mem->user_pages = NULL;
+	amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm);
  unregister_out:
  	if (ret)
  		amdgpu_mn_unregister(bo);

@@ -677,7 +657,6 @@ static int reserve_bo_and_vm(struct kgd_mem *mem,
  	ctx->kfd_bo.priority = 0;
  	ctx->kfd_bo.tv.bo = &bo->tbo;
  	ctx->kfd_bo.tv.num_shared = 1;
-	ctx->kfd_bo.user_pages = NULL;
  	list_add(&ctx->kfd_bo.tv.head, &ctx->list);
 
  	amdgpu_vm_get_pd_bo(vm, &ctx->list, &ctx->vm_pd[0]);

@@ -741,7 +720,6 @@ static int reserve_bo_and_cond_vms(struct kgd_mem *mem,
  	ctx->kfd_bo.priority = 0;
  	ctx->kfd_bo.tv.bo = &bo->tbo;
  	ctx->kfd_bo.tv.num_shared = 1;
-	ctx->kfd_bo.user_pages = NULL;
  	list_add(&ctx->kfd_bo.tv.head, &ctx->list);
 
  	i = 0;

@@ -1329,15 +1307,6 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
  	list_del(&bo_list_entry->head);
  	mutex_unlock(&process_info->lock);
 
-	/* Free user pages if necessary */
-	if (mem->user_pages) {
-		pr_debug("%s: Freeing user_pages array\n", __func__);
-		if (mem->user_pages[0])
-			release_pages(mem->user_pages,
-					mem->bo->tbo.ttm->num_pages);
-		kvfree(mem->user_pages);
-	}
-
  	ret = reserve_bo_and_cond_vms(mem, NULL, BO_VM_ALL, &ctx);
  	if (unlikely(ret))
  		return ret;

@@ -1751,25 +1720,11 @@ static int update_invalid_user_pages(struct amdkfd_process_info *process_info,
 
  		bo = mem->bo;
 
-		if (!mem->user_pages) {
-			mem->user_pages =
-				kvmalloc_array(bo->tbo.ttm->num_pages,
-						 sizeof(struct page *),
-						 GFP_KERNEL | __GFP_ZERO);
-			if (!mem->user_pages) {
-				pr_err("%s: Failed to allocate pages array\n",
-				       __func__);
-				return -ENOMEM;
-			}
-		} else if (mem->user_pages[0]) {
-			release_pages(mem->user_pages, bo->tbo.ttm->num_pages);
-		}
-
  		/* Get updated user pages */
  		ret = amdgpu_ttm_tt_get_user_pages(bo->tbo.ttm,
-						   mem->user_pages);
+						   bo->tbo.ttm->pages);
  		if (ret) {
-			mem->user_pages[0] = NULL;
+			bo->tbo.ttm->pages[0] = NULL;
  			pr_info("%s: Failed to get user pages: %d\n",
  				__func__, ret);
  			/* Pretend it succeeded. It will fail later

@@ -1778,12 +1733,6 @@ static int update_invalid_user_pages(struct amdkfd_process_info *process_info,
  			 * stalled user mode queues.
  			 */
  		}
-
-		/* Mark the BO as valid unless it was invalidated
-		 * again concurrently
-		 */

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c static int update_invalid_user_pages(struct amdkfd_process_info *process_info, · line 1786

Quoted 2 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +0 -2

-		if (atomic_cmpxchg(&mem->invalid, invalid, 0) != invalid)
-			return -EAGAIN;

I think you're removing this to make sure we exactly pair up amdgpu_ttm_tt_get_user_pages and amdgpu_ttm_tt_get_user_pages_done calls. But there are other error conditions in validate_invalid_user_pages that would skip the amdgpu_ttm_tt_get_user_pages_done calls. That will need to be handled properly in the error-handling code paths in validate_invalid_user_pages.

Quoted 68 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +9 -11

  	}
 
  	return 0;

@@ -1863,10 +1812,8 @@ static int validate_invalid_user_pages(struct amdkfd_process_info *process_info)
 
  		bo = mem->bo;
 
-		/* Copy pages array and validate the BO if we got user pages */
-		if (mem->user_pages[0]) {
-			amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm,
-						     mem->user_pages);
+		/* Validate the BO if we got user pages */
+		if (bo->tbo.ttm->pages[0]) {
  			amdgpu_bo_placement_from_domain(bo, mem->domain);
  			ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
  			if (ret) {

@@ -1875,16 +1822,16 @@ static int validate_invalid_user_pages(struct amdkfd_process_info *process_info)
  			}
  		}
 
-		/* Validate succeeded, now the BO owns the pages, free
-		 * our copy of the pointer array. Put this BO back on
-		 * the userptr_valid_list. If we need to revalidate
-		 * it, we need to start from scratch.
-		 */
-		kvfree(mem->user_pages);
-		mem->user_pages = NULL;
  		list_move_tail(&mem->validate_list.head,
  			       &process_info->userptr_valid_list);
 
+		/* Stop HMM track the userptr update. We dont check the return
+		 * value for concurrent CPU page table update because we will
+		 * reschedule the restore worker if process_info->evicted_bos
+		 * is updated.
+		 */
+		amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm);
+
  		/* Update mapping. If the BO was not validated
  		 * (because we couldn't get user pages), this will
  		 * clear the page table entries, which will result in

drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c +0 -2

index 5c79da8e1150..3842183970cc 100644
@@ -200,8 +200,6 @@ void amdgpu_bo_list_get_list(struct amdgpu_bo_list *list,
 
  		if (!bo->parent)
  			list_add_tail(&e->tv.head, &bucket[priority]);
-
-		e->user_pages = NULL;
  	}
 
  	/* Connect the sorted buckets in the output list. */

drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.h +1 -2

index 7c5f5d1601e6..4beab2de09b3 100644
@@ -35,8 +35,7 @@ struct amdgpu_bo_list_entry {
  	struct ttm_validate_buffer	tv;
  	struct amdgpu_bo_va		*bo_va;
  	uint32_t			priority;
-	struct page			**user_pages;
-	int				user_invalidated;
+	bool				user_invalidated;
  };
 
  struct amdgpu_bo_list {

drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +0 -0

[snip]

Quoted 1 line

drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +0 -0

[snip]

Quoted 13 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +0 -2

index 5d518d2bb9be..c8002ee1d415 100644
@@ -229,8 +229,6 @@ static void amdgpu_mn_invalidate_node(struct amdgpu_mn_node *node,
  			true, false, MAX_SCHEDULE_TIMEOUT);
  		if (r <= 0)
  			DRM_ERROR("(%ld) failed to wait for user bo\n", r);
-
-		amdgpu_ttm_tt_mark_user_pages(bo->tbo.ttm);
  	}
  }
 

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c static struct hmm_mirror_ops amdgpu_hmm_mirror_ops[] = { · line 360

Quoted 13 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +3 -2

@@ -353,15 +351,16 @@ static struct hmm_mirror_ops amdgpu_hmm_mirror_ops[] = {
  /**
   * amdgpu_mn_get - create HMM mirror context
   *
+ * @mm: the mm struct
   * @adev: amdgpu device pointer
   * @type: type of MMU notifier context
   *
- * Creates a HMM mirror context for current->mm.
+ * Creates a HMM mirror context for mm.
   */
-struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev,
+struct amdgpu_mn *amdgpu_mn_get(struct mm_struct *mm,

I think this change isn't needed any more. All callers of this function specify mm as current->mm anyway.

Quoted 138 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +25 -2

+				struct amdgpu_device *adev,
  				enum amdgpu_mn_type type)
  {
-	struct mm_struct *mm = current->mm;
  	struct amdgpu_mn *amn;
  	unsigned long key = AMDGPU_MN_KEY(mm, type);
  	int r;

@@ -431,7 +430,7 @@ int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)
  	struct list_head bos;
  	struct interval_tree_node *it;
 
-	amn = amdgpu_mn_get(adev, type);
+	amn = amdgpu_mn_get(current->mm, adev, type);
  	if (IS_ERR(amn))
  		return PTR_ERR(amn);
 

@@ -513,3 +512,26 @@ void amdgpu_mn_unregister(struct amdgpu_bo *bo)
  	mutex_unlock(&adev->mn_lock);
  }
 
+/* flags used by HMM internal, not related to CPU/GPU PTE flags */
+static const uint64_t hmm_range_flags[HMM_PFN_FLAG_MAX] = {
+		(1 << 0), /* HMM_PFN_VALID */
+		(1 << 1), /* HMM_PFN_WRITE */
+		0 /* HMM_PFN_DEVICE_PRIVATE */
+};
+
+static const uint64_t hmm_range_values[HMM_PFN_VALUE_MAX] = {
+		0xfffffffffffffffeUL, /* HMM_PFN_ERROR */
+		0, /* HMM_PFN_NONE */
+		0xfffffffffffffffcUL /* HMM_PFN_SPECIAL */
+};
+
+void amdgpu_hmm_init_range(struct hmm_range *range)
+{
+	if (range) {
+		range->flags = hmm_range_flags;
+		range->values = hmm_range_values;
+		range->pfn_shift = PAGE_SHIFT;
+		range->pfns = NULL;
+		INIT_LIST_HEAD(&range->list);
+	}
+}

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h +5 -2

index 0a51fd00021c..4c5e86847a3b 100644
@@ -25,9 +25,10 @@
  #define __AMDGPU_MN_H__
 
  /*
- * MMU Notifier
+ * HMM mirror
   */
  struct amdgpu_mn;
+struct hmm_range;
 
  enum amdgpu_mn_type {
  	AMDGPU_MN_TYPE_GFX,

@@ -37,10 +38,12 @@ enum amdgpu_mn_type {
  #if defined(CONFIG_HMM_MIRROR)
  void amdgpu_mn_lock(struct amdgpu_mn *mn);
  void amdgpu_mn_unlock(struct amdgpu_mn *mn);
-struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev,
+struct amdgpu_mn *amdgpu_mn_get(struct mm_struct *mm,
+				struct amdgpu_device *adev,
  				enum amdgpu_mn_type type);
  int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr);
  void amdgpu_mn_unregister(struct amdgpu_bo *bo);
+void amdgpu_hmm_init_range(struct hmm_range *range);
  #else
  static inline void amdgpu_mn_lock(struct amdgpu_mn *mn) {}
  static inline void amdgpu_mn_unlock(struct amdgpu_mn *mn) {}

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +10 -17

index c91ec3101d00..33057d6114c7 100644
@@ -43,6 +43,7 @@
  #include <linux/pagemap.h>
  #include <linux/debugfs.h>
  #include <linux/iommu.h>
+#include <linux/hmm.h>
  #include "amdgpu.h"
  #include "amdgpu_object.h"
  #include "amdgpu_trace.h"

@@ -705,11 +706,6 @@ static unsigned long amdgpu_ttm_io_mem_pfn(struct ttm_buffer_object *bo,
  /*
   * TTM backend functions.
   */
-struct amdgpu_ttm_gup_task_list {
-	struct list_head	list;
-	struct task_struct	*task;
-};
-
  struct amdgpu_ttm_tt {
  	struct ttm_dma_tt	ttm;
  	u64			offset;

@@ -718,107 +714,105 @@ struct amdgpu_ttm_tt {
  	uint32_t		userflags;
  	spinlock_t              guptasklock;
  	struct list_head        guptasks;
-	atomic_t		mmu_invalidations;
-	uint32_t		last_set_pages;
+	struct hmm_range	range;
  };
 
  /**
- * amdgpu_ttm_tt_get_user_pages - Pin pages of memory pointed to by a USERPTR
- * pointer to memory
+ * amdgpu_ttm_tt_get_user_pages - get device accessible pages that back user
+ * memory and start HMM tracking CPU page table update
   *
- * Called by amdgpu_gem_userptr_ioctl() and amdgpu_cs_parser_bos().
- * This provides a wrapper around the get_user_pages() call to provide
- * device accessible pages that back user memory.
+ * Calling function must call amdgpu_ttm_tt_userptr_range_done() once and only
+ * once afterwards to stop HMM tracking
   */
  int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages)
  {
  	struct amdgpu_ttm_tt *gtt = (void *)ttm;
  	struct mm_struct *mm = gtt->usertask->mm;
-	unsigned int flags = 0;
-	unsigned pinned = 0;
-	int r;
+	unsigned long end = gtt->userptr + ttm->num_pages * PAGE_SIZE;
+	struct hmm_range *range = &gtt->range;
+	int r, i;
 
  	if (!mm) /* Happens during process shutdown */
  		return -ESRCH;
 
-	if (!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY))
-		flags |= FOLL_WRITE;
+	amdgpu_hmm_init_range(range);
 
  	down_read(&mm->mmap_sem);
 

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c struct amdgpu_ttm_tt { · line 742

Quoted 1 line

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +1 -0

+	range->vma = find_vma(mm, gtt->userptr);

There is still an assumption here that our userptr BOs cover only a single VM. This is a new limitation with HMM that the previous get_user_pages-based implementation did not have. We're probably OK with this for now. But we may run into regressions if there are any ROCm application that register user memory spanning multiple VMAs. If that happens, we may need to generalize this so that each userptr BO can have multiple HMM ranges.

Quoted 22 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +3 -10

  	if (gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) {
  		/*
  		 * check that we only use anonymous memory to prevent problems
  		 * with writeback
  		 */
-		unsigned long end = gtt->userptr + ttm->num_pages * PAGE_SIZE;
-		struct vm_area_struct *vma;
+		struct vm_area_struct *vma = range->vma;
 
-		vma = find_vma(mm, gtt->userptr);
-		if (!vma || vma->vm_file || vma->vm_end < end) {
-			up_read(&mm->mmap_sem);
-			return -EPERM;
-		}
+		if (!vma || vma->vm_file || vma->vm_end < end)
+			range->vma = NULL;
  	}
 
-	/* loop enough times using contiguous pages of memory */
-	do {
-		unsigned num_pages = ttm->num_pages - pinned;

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c struct amdgpu_ttm_tt { · line 755

Quoted 10 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +2 -8

-		uint64_t userptr = gtt->userptr + pinned * PAGE_SIZE;
-		struct page **p = pages + pinned;
-		struct amdgpu_ttm_gup_task_list guptask;
-
-		guptask.task = current;
-		spin_lock(&gtt->guptasklock);
-		list_add(&guptask.list, &gtt->guptasks);
-		spin_unlock(&gtt->guptasklock);
+	if (!range->vma) {
+		r = -EPERM;

You're treating permissions errors and errors finding a VMA the same way. And I think you don't detect failures to find a VMA properly in the first place. find_vma may return a VMA that doesn't include the address provided. It's documented as "Look up the first VMA which satisfies addr < vm_end,  NULL if none." You should also check that the VMA covers the entire address range of the userptr BO. There is a helper function in linux/mm.h that does that: range_in_vma(range->vma, gtt->userptr, gtt->userptr + gtt->userptr + ttm->num_pages * PAGE_SIZE). You should check this condition in all cases.

There should be two distinct types of errors. If find_vma fails to find a matching VMA, that should be -EFAULT. If the gtt->userflags don't match the VMA properties, that should be -EPERM.

Quoted 18 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +10 -6

+		goto out;
+	}
 
-		if (mm == current->mm)
-			r = get_user_pages(userptr, num_pages, flags, p, NULL);
-		else
-			r = get_user_pages_remote(gtt->usertask,
-					mm, userptr, num_pages,
-					flags, p, NULL, NULL);
+	range->pfns = kvmalloc_array(ttm->num_pages, sizeof(uint64_t),
+				     GFP_KERNEL);
+	if (range->pfns == NULL) {
+		r = -ENOMEM;
+		goto out;
+	}
+	range->start = gtt->userptr;
+	range->end = end;
 

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c struct amdgpu_ttm_tt { · line 772

Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +5 -3

-		spin_lock(&gtt->guptasklock);
-		list_del(&guptask.list);
-		spin_unlock(&gtt->guptasklock);
+	for (i = 0; i < ttm->num_pages; i++) {
+		range->pfns[i] = range->flags[HMM_PFN_VALID];
+		range->pfns[i] |= amdgpu_ttm_tt_is_readonly(ttm) ?
+					0 : range->flags[HMM_PFN_WRITE];
+	}

You could optimize this slightly. You don't need to call amdgpu_ttm_tt_is_readonly in each loop iteration. The flags are the same for all pfns. So do the conditional thing on range->pfns[0], and then just copy that to all the remaining pfns.

Quoted 28 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +11 -7

-		if (r < 0)
-			goto release_pages;
+	/* This may trigger page table update */
+	r = hmm_vma_fault(range, true);
+	if (r)
+		goto out_free_pfns;
 
-		pinned += r;
+	up_read(&mm->mmap_sem);
 
-	} while (pinned < ttm->num_pages);
+	for (i = 0; i < ttm->num_pages; i++)
+		pages[i] = hmm_pfn_to_page(range, range->pfns[i]);
 
-	up_read(&mm->mmap_sem);
  	return 0;
 
-release_pages:
-	release_pages(pages, pinned);
+out_free_pfns:
+	kvfree(range->pfns);
+	range->pfns = NULL;
+out:
  	up_read(&mm->mmap_sem);
  	return r;
  }
 

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c struct amdgpu_ttm_tt { · line 798

Quoted 9 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +3 -4

  /**
- * amdgpu_ttm_tt_set_user_pages - Copy pages in, putting old pages as necessary.
+ * amdgpu_ttm_tt_userptr_range_done - stop HMM track the CPU page table change
+ * Check if the pages backing this ttm range have been invalidated
   *
- * Called by amdgpu_cs_list_validate(). This creates the page list
- * that backs user memory and will ultimately be mapped into the device
- * address space.
+ * Returns: true if pages are invalidated since the last time they've been set

I still don't like that you're inverting the return value of hmm_vma_range_done. This will get confusing in the future. Could this function instead return "true if pages are still valid"?

Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +2 -2

   */
-void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page **pages)
+bool amdgpu_ttm_tt_get_user_pages_done(struct ttm_tt *ttm)
  {
  	struct amdgpu_ttm_tt *gtt = (void *)ttm;
-	unsigned i;
+	bool r;
 

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c struct amdgpu_ttm_tt { · line 806

Quoted 6 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +2 -4

-	gtt->last_set_pages = atomic_read(&gtt->mmu_invalidations);
-	for (i = 0; i < ttm->num_pages; ++i) {
-		if (ttm->pages[i])
-			put_page(ttm->pages[i]);
+	if (gtt == NULL || !gtt->userptr)
+		return false;

Use "!gtt" instead of "gtt == NULL". Also, returning "false" here means the pages are still valid. That seems like the wrong answer for this kind of failure. "false" would make sense here if you followed my suggestion above to return true, if the pages are still valid.

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c struct amdgpu_ttm_tt { · line 815

Quoted 11 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +7 -1

-		ttm->pages[i] = pages ? pages[i] : NULL;
+	r = hmm_vma_range_done(&gtt->range);
+
+	if (gtt->range.pfns) {
+		kvfree(gtt->range.pfns);
+		gtt->range.pfns = NULL;
  	}
+
+	return !r;

I still don't like that you're inverting the return value of hmm_vma_range_done. This will get confusing in the future.

Regards,
  Felix

Quoted 117 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +4 -38

  }
 
  /**

@@ -903,9 +897,6 @@ static void amdgpu_ttm_tt_unpin_userptr(struct ttm_tt *ttm)
  	/* unmap the pages mapped to the device */
  	dma_unmap_sg(adev->dev, ttm->sg->sgl, ttm->sg->nents, direction);
 
-	/* mark the pages as dirty */
-	amdgpu_ttm_tt_mark_user_pages(ttm);
-
  	sg_free_table(ttm->sg);
  }
 

@@ -1207,7 +1198,7 @@ static void amdgpu_ttm_tt_unpopulate(struct ttm_tt *ttm)
  	bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
 
  	if (gtt && gtt->userptr) {
-		amdgpu_ttm_tt_set_user_pages(ttm, NULL);
+		memset(ttm->pages, 0, ttm->num_pages * sizeof(struct page *));
  		kfree(ttm->sg);
  		ttm->page_flags &= ~TTM_PAGE_FLAG_SG;
  		return;

@@ -1258,8 +1249,6 @@ int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,
 
  	spin_lock_init(&gtt->guptasklock);
  	INIT_LIST_HEAD(&gtt->guptasks);
-	atomic_set(&gtt->mmu_invalidations, 0);
-	gtt->last_set_pages = 0;
 
  	return 0;
  }

@@ -1289,7 +1278,6 @@ bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,
  				  unsigned long end)
  {
  	struct amdgpu_ttm_tt *gtt = (void *)ttm;
-	struct amdgpu_ttm_gup_task_list *entry;
  	unsigned long size;
 
  	if (gtt == NULL || !gtt->userptr)

@@ -1302,48 +1290,20 @@ bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,
  	if (gtt->userptr > end || gtt->userptr + size <= start)
  		return false;
 
-	/* Search the lists of tasks that hold this mapping and see
-	 * if current is one of them.  If it is return false.
-	 */
-	spin_lock(&gtt->guptasklock);
-	list_for_each_entry(entry, &gtt->guptasks, list) {
-		if (entry->task == current) {
-			spin_unlock(&gtt->guptasklock);
-			return false;
-		}
-	}
-	spin_unlock(&gtt->guptasklock);
-
-	atomic_inc(&gtt->mmu_invalidations);
-
  	return true;
  }
 
  /**
- * amdgpu_ttm_tt_userptr_invalidated - Has the ttm_tt object been invalidated?
+ * amdgpu_ttm_tt_is_userptr - Have the pages backing by userptr?
   */
-bool amdgpu_ttm_tt_userptr_invalidated(struct ttm_tt *ttm,
-				       int *last_invalidated)
-{
-	struct amdgpu_ttm_tt *gtt = (void *)ttm;
-	int prev_invalidated = *last_invalidated;
-
-	*last_invalidated = atomic_read(&gtt->mmu_invalidations);
-	return prev_invalidated != *last_invalidated;
-}
-
-/**
- * amdgpu_ttm_tt_userptr_needs_pages - Have the pages backing this ttm_tt object
- * been invalidated since the last time they've been set?
- */
-bool amdgpu_ttm_tt_userptr_needs_pages(struct ttm_tt *ttm)
+bool amdgpu_ttm_tt_is_userptr(struct ttm_tt *ttm)
  {
  	struct amdgpu_ttm_tt *gtt = (void *)ttm;
 
  	if (gtt == NULL || !gtt->userptr)
  		return false;
 
-	return atomic_read(&gtt->mmu_invalidations) != gtt->last_set_pages;
+	return true;
  }
 
  /**

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h +2 -2

index b5b2d101f7db..c5a1e78a09b1 100644
@@ -102,7 +102,7 @@ int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo);
  int amdgpu_ttm_recover_gart(struct ttm_buffer_object *tbo);
 
  int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages);
-void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page **pages);
+bool amdgpu_ttm_tt_get_user_pages_done(struct ttm_tt *ttm);
  void amdgpu_ttm_tt_mark_user_pages(struct ttm_tt *ttm);
  int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,
  				     uint32_t flags);

@@ -112,7 +112,7 @@ bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,
  				  unsigned long end);
  bool amdgpu_ttm_tt_userptr_invalidated(struct ttm_tt *ttm,
  				       int *last_invalidated);
-bool amdgpu_ttm_tt_userptr_needs_pages(struct ttm_tt *ttm);
+bool amdgpu_ttm_tt_is_userptr(struct ttm_tt *ttm);
  bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm);
  uint64_t amdgpu_ttm_tt_pde_flags(struct ttm_tt *ttm, struct ttm_mem_reg *mem);
  uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +0 -0

index f059ec314f2b..ab8fa84b27a6 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm, · line 624

Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +0 -1

@@ -619,7 +619,6 @@ void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
  	entry->tv.bo = &vm->root.base.bo->tbo;
  	/* One for the VM updates, one for TTM and one for the CS job */
  	entry->tv.num_shared = 3;
-	entry->user_pages = NULL;
  	list_add(&entry->tv.head, validated);
  }
 

-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Yang, Philip
Sent: Friday, December 07, 2018 5:03 AM
To: [email protected]
Cc: Yang, Philip <[email protected]>
Subject: [PATCH 1/3] drm/amdgpu: use HMM mirror callback to replace mmu notifier v6

Replace our MMU notifier with hmm_mirror_ops.sync_cpu_device_pagetables callback. Enable CONFIG_HMM and CONFIG_HMM_MIRROR as a dependency in DRM_AMDGPU_USERPTR Kconfig.

It supports both KFD userptr and gfx userptr paths.

The depdent HMM patchset from Jérôme Glisse are all merged into 4.20.0 kernel now.

Change-Id Ie62c3c5e3c5b8521ab3b438d1eff2aa2a003835e

Signed-off-by Philip Yang <[email protected]>

---
 drivers/gpu/drm/amd/amdgpu/Kconfig     |   6 +-
 drivers/gpu/drm/amd/amdgpu/Makefile    |   2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c | 122 ++++++++++-------------
--
 drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h |   2 +-
 4 files changed, 55 insertions(+), 77 deletions(-)

drivers/gpu/drm/amd/amdgpu/Kconfig +0 -0

index 9221e5489069..960a63355705 100644
@@ -26,10 +26,10 @@ config DRM_AMDGPU_CIK  config

DRM_AMDGPU_USERPTR

drivers/gpu/drm/amd/amdgpu/Kconfig +3 -3

 	bool "Always enable userptr write support"
 	depends on DRM_AMDGPU
-	select MMU_NOTIFIER
+	select HMM_MIRROR
 	help
-	  This option selects CONFIG_MMU_NOTIFIER if it isn't already
-	  selected to enabled full userptr support.
+	  This option selects CONFIG_HMM and CONFIG_HMM_MIRROR if it
+	  isn't already selected to enabled full userptr support.
 
 config DRM_AMDGPU_GART_DEBUGFS
 	bool "Allow GART access through debugfs"

drivers/gpu/drm/amd/amdgpu/Makefile +1 -1

index f76bcb9c45e4..675efc850ff4 100644
@@ -172,7 +172,7 @@ endif
 amdgpu-$(CONFIG_COMPAT) += amdgpu_ioc32.o
 amdgpu-$(CONFIG_VGA_SWITCHEROO) += amdgpu_atpx_handler.o
 amdgpu-$(CONFIG_ACPI) += amdgpu_acpi.o
-amdgpu-$(CONFIG_MMU_NOTIFIER) += amdgpu_mn.o
+amdgpu-$(CONFIG_HMM_MIRROR) += amdgpu_mn.o
 
 include $(FULL_AMD_PATH)/powerplay/Makefile
 

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +1 -2

index e55508b39496..5d518d2bb9be 100644
@@ -45,7 +45,7 @@
 
 #include <linux/firmware.h>
 #include <linux/module.h>
-#include <linux/mmu_notifier.h>
+#include <linux/hmm.h>
 #include <linux/interval_tree.h>
 #include <drm/drmP.h>
 #include <drm/drm.h>

@@ -58,7 +58,6 @@
  *
  * @adev: amdgpu device pointer
  * @mm: process address space
- * @mn: MMU notifier structure
  * @type: type of MMU notifier
  * @work: destruction work item
  * @node: hash table node to find structure by adev and mn @@ -66,6 +65,7

@@
* @objects: interval tree containing amdgpu_mn_nodes
* @read_lock: mutex for recursive locking of @lock
* @recursion: depth of recursion
+ * @mirror: HMM mirror function support
*
* Data for each amdgpu device and process address space. */

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +12 -11

@@ -73,7 +73,6 @@ struct amdgpu_mn {
 	/* constant after initialisation */
 	struct amdgpu_device	*adev;
 	struct mm_struct	*mm;
-	struct mmu_notifier	mn;
 	enum amdgpu_mn_type	type;
 
 	/* only used on destruction */

@@ -87,6 +86,9 @@ struct amdgpu_mn {
 	struct rb_root_cached	objects;
 	struct mutex		read_lock;
 	atomic_t		recursion;
+
+	/* HMM mirror */
+	struct hmm_mirror	mirror;
 };
 
 /**

@@ -103,7 +105,7 @@ struct amdgpu_mn_node {  };
 
 /**
- * amdgpu_mn_destroy - destroy the MMU notifier
+ * amdgpu_mn_destroy - destroy the HMM mirror
  *
  * @work: previously sheduled work item
  *

@@ -129,28 +131,26 @@ static void amdgpu_mn_destroy(struct work_struct
 *work)
 	}
 	up_write(&amn->lock);
 	mutex_unlock(&adev->mn_lock);
-	mmu_notifier_unregister_no_release(&amn->mn, amn->mm);
+
+	hmm_mirror_unregister(&amn->mirror);
 	kfree(amn);
 }
 
 /**
- * amdgpu_mn_release - callback to notify about mm destruction
+ * amdgpu_hmm_mirror_release - callback to notify about mm destruction
  *
- * @mn: our notifier
- * @mm: the mm this callback is about
+ * @mirror: the HMM mirror (mm) this callback is about
  *
- * Shedule a work item to lazy destroy our notifier.
+ * Shedule a work item to lazy destroy HMM mirror.
  */
-static void amdgpu_mn_release(struct mmu_notifier *mn,
-			      struct mm_struct *mm)
+static void amdgpu_hmm_mirror_release(struct hmm_mirror *mirror)
 {
-	struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn,
 mn);
+	struct amdgpu_mn *amn = container_of(mirror, struct amdgpu_mn,
+mirror);
 
 	INIT_WORK(&amn->work, amdgpu_mn_destroy);
 	schedule_work(&amn->work);
 }
 
-
 /**
  * amdgpu_mn_lock - take the write side lock for this notifier
  *

@@ -237,21 +237,19 @@ static void amdgpu_mn_invalidate_node(struct
 amdgpu_mn_node *node,
 /**
  * amdgpu_mn_invalidate_range_start_gfx - callback to notify about mm

change
*
- * @mn: our notifier
- * @mm: the mm this callback is about
- * @start: start of updated range
- * @end: end of updated range
+ * @mirror: the hmm_mirror (mm) is about to update
+ * @update: the update start, end address
*
* Block for operations on BOs to finish and mark pages as accessed and
* potentially dirty.
*/

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +2 -6

-static int amdgpu_mn_invalidate_range_start_gfx(struct mmu_notifier *mn,
-						 struct mm_struct *mm,
-						 unsigned long start,
-						 unsigned long end,
-						 bool blockable)
+static int amdgpu_mn_sync_pagetables_gfx(struct hmm_mirror *mirror,
+			const struct hmm_update *update)
 {
-	struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn,

mn);

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +1 -0

+	struct amdgpu_mn *amn = container_of(mirror, struct amdgpu_mn,

mirror);

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +4 -0

+	unsigned long start = update->start;
+	unsigned long end = update->end;
+	bool blockable = update->blockable;
 	struct interval_tree_node *it;
 
 	/* notification is exclusive, but interval is inclusive */ @@ -278,28
+276,28 @@ static int amdgpu_mn_invalidate_range_start_gfx(struct

mmu_notifier *mn,

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +2 -0

 		amdgpu_mn_invalidate_node(node, start, end);
 	}
 
+	amdgpu_mn_read_unlock(amn);
+
 	return 0;
 }
 
 /**
  * amdgpu_mn_invalidate_range_start_hsa - callback to notify about mm

change
*
- * @mn: our notifier
- * @mm: the mm this callback is about
- * @start: start of updated range
- * @end: end of updated range
+ * @mirror: the hmm_mirror (mm) is about to update
+ * @update: the update start, end address
*
* We temporarily evict all BOs between start and end. This
* necessitates evicting all user-mode queues of the process. The BOs
* are restorted in amdgpu_mn_invalidate_range_end_hsa.
*/
-static int amdgpu_mn_invalidate_range_start_hsa(struct mmu_notifier *mn,

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +2 -5

-						 struct mm_struct *mm,
-						 unsigned long start,
-						 unsigned long end,
-						 bool blockable)
+static int amdgpu_mn_sync_pagetables_hsa(struct hmm_mirror *mirror,
+			const struct hmm_update *update)
 {
-	struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn,

mn);

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +1 -0

+	struct amdgpu_mn *amn = container_of(mirror, struct amdgpu_mn,

mirror);

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +4 -0

+	unsigned long start = update->start;
+	unsigned long end = update->end;
+	bool blockable = update->blockable;
 	struct interval_tree_node *it;
 
 	/* notification is exclusive, but interval is inclusive */ @@ -326,59
+324,39 @@ static int amdgpu_mn_invalidate_range_start_hsa(struct

mmu_notifier *mn,

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +1 -1

 			if (amdgpu_ttm_tt_affect_userptr(bo->tbo.ttm,
 							 start, end))
-				amdgpu_amdkfd_evict_userptr(mem, mm);
+				amdgpu_amdkfd_evict_userptr(mem, amn-

Quoted 1 line

mm);

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +5 -15

 		}
 	}
 
+	amdgpu_mn_read_unlock(amn);
+
 	return 0;
 }
 
-/**
- * amdgpu_mn_invalidate_range_end - callback to notify about mm change
- *
- * @mn: our notifier
- * @mm: the mm this callback is about
- * @start: start of updated range
- * @end: end of updated range
- *
- * Release the lock again to allow new command submissions.
+/* Low bits of any reasonable mm pointer will be unused due to struct
+ * alignment. Use these bits to make a unique key from the mm pointer
+ * and notifier type.
  */
-static void amdgpu_mn_invalidate_range_end(struct mmu_notifier *mn,
-					   struct mm_struct *mm,
-					   unsigned long start,
-					   unsigned long end)
-{
-	struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn,

mn);

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +2 -6

-
-	amdgpu_mn_read_unlock(amn);
-}
+#define AMDGPU_MN_KEY(mm, type) ((unsigned long)(mm) + (type))
 
-static const struct mmu_notifier_ops amdgpu_mn_ops[] = {
+static struct hmm_mirror_ops amdgpu_hmm_mirror_ops[] = {
 	[AMDGPU_MN_TYPE_GFX] = {
-		.release = amdgpu_mn_release,
-		.invalidate_range_start =

amdgpu_mn_invalidate_range_start_gfx,

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +1 -1

-		.invalidate_range_end = amdgpu_mn_invalidate_range_end,
+		.sync_cpu_device_pagetables =

amdgpu_mn_sync_pagetables_gfx,

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +1 -2

+		.release = amdgpu_hmm_mirror_release
 	},
 	[AMDGPU_MN_TYPE_HSA] = {
-		.release = amdgpu_mn_release,
-		.invalidate_range_start =

amdgpu_mn_invalidate_range_start_hsa,

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +1 -1

-		.invalidate_range_end = amdgpu_mn_invalidate_range_end,
+		.sync_cpu_device_pagetables =

amdgpu_mn_sync_pagetables_hsa,

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +6 -11

+		.release = amdgpu_hmm_mirror_release
 	},
 };
 
-/* Low bits of any reasonable mm pointer will be unused due to struct
- * alignment. Use these bits to make a unique key from the mm pointer
- * and notifier type.
- */
-#define AMDGPU_MN_KEY(mm, type) ((unsigned long)(mm) + (type))
-
 /**
- * amdgpu_mn_get - create notifier context
+ * amdgpu_mn_get - create HMM mirror context
  *
  * @adev: amdgpu device pointer
  * @type: type of MMU notifier context
  *
- * Creates a notifier context for current->mm.
+ * Creates a HMM mirror context for current->mm.
  */
 struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev,
 				enum amdgpu_mn_type type)

@@ -408,12 +386,12 @@ struct amdgpu_mn *amdgpu_mn_get(struct
 amdgpu_device *adev,
 	amn->mm = mm;
 	init_rwsem(&amn->lock);
 	amn->type = type;
-	amn->mn.ops = &amdgpu_mn_ops[type];
 	amn->objects = RB_ROOT_CACHED;
 	mutex_init(&amn->read_lock);
 	atomic_set(&amn->recursion, 0);
 
-	r = __mmu_notifier_register(&amn->mn, mm);
+	amn->mirror.ops = &amdgpu_hmm_mirror_ops[type];
+	r = hmm_mirror_register(&amn->mirror, mm);
 	if (r)
 		goto free_amn;

@@ -439,7 +417,7 @@ struct amdgpu_mn *amdgpu_mn_get(struct
 amdgpu_device *adev,
  * @bo: amdgpu buffer object
  * @addr: userptr addr we should monitor
  *
- * Registers an MMU notifier for the given BO at the specified address.
+ * Registers an HMM mirror for the given BO at the specified address.
  * Returns 0 on success, -ERRNO if anything goes wrong.
  */

int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr) @@ -
495,11 +473,11 @@ int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr) }

/**
- * amdgpu_mn_unregister - unregister a BO for notifier updates
+ * amdgpu_mn_unregister - unregister a BO for HMM mirror updates *
* @bo: amdgpu buffer object
*
- * Remove any registration of MMU notifier updates from the buffer object.
+ * Remove any registration of HMM mirror updates from the buffer object. */
void amdgpu_mn_unregister(struct amdgpu_bo *bo) { diff --git
a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h
index eb0f432f78fe..0a51fd00021c 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h +1 -1

@@ -34,7 +34,7 @@ enum amdgpu_mn_type {
 	AMDGPU_MN_TYPE_HSA,
 };
 
-#if defined(CONFIG_MMU_NOTIFIER)
+#if defined(CONFIG_HMM_MIRROR)
 void amdgpu_mn_lock(struct amdgpu_mn *mn);  void

amdgpu_mn_unlock(struct amdgpu_mn *mn); struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev,
--
2.17.1

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Even you should rename amdgpu_mn.c/h to amdgpu_hmm.c/h.

-David

Quoted 372 lines

-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Yang, Philip
Sent: Friday, December 07, 2018 5:03 AM
To: [email protected]
Cc: Yang, Philip <[email protected]>
Subject: [PATCH 1/3] drm/amdgpu: use HMM mirror callback to replace mmu notifier v6

Replace our MMU notifier with hmm_mirror_ops.sync_cpu_device_pagetables callback. Enable CONFIG_HMM and CONFIG_HMM_MIRROR as a dependency in DRM_AMDGPU_USERPTR Kconfig.

It supports both KFD userptr and gfx userptr paths.

The depdent HMM patchset from Jérôme Glisse are all merged into 4.20.0 kernel now.

Change-Id Ie62c3c5e3c5b8521ab3b438d1eff2aa2a003835e

Signed-off-by Philip Yang <[email protected]>

---
  drivers/gpu/drm/amd/amdgpu/Kconfig     |   6 +-
  drivers/gpu/drm/amd/amdgpu/Makefile    |   2 +-
  drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c | 122 ++++++++++-------------
--
  drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h |   2 +-
  4 files changed, 55 insertions(+), 77 deletions(-)

drivers/gpu/drm/amd/amdgpu/Kconfig +0 -0

index 9221e5489069..960a63355705 100644
@@ -26,10 +26,10 @@ config DRM_AMDGPU_CIK  config

DRM_AMDGPU_USERPTR
bool "Always enable userptr write support" depends on DRM_AMDGPU

drivers/gpu/drm/amd/amdgpu/Kconfig +3 -3

-	select MMU_NOTIFIER
+	select HMM_MIRROR
  	help
-	  This option selects CONFIG_MMU_NOTIFIER if it isn't already
-	  selected to enabled full userptr support.
+	  This option selects CONFIG_HMM and CONFIG_HMM_MIRROR if it
+	  isn't already selected to enabled full userptr support.
 
  config DRM_AMDGPU_GART_DEBUGFS
  	bool "Allow GART access through debugfs"

drivers/gpu/drm/amd/amdgpu/Makefile +1 -1

index f76bcb9c45e4..675efc850ff4 100644
@@ -172,7 +172,7 @@ endif
  amdgpu-$(CONFIG_COMPAT) += amdgpu_ioc32.o
  amdgpu-$(CONFIG_VGA_SWITCHEROO) += amdgpu_atpx_handler.o
  amdgpu-$(CONFIG_ACPI) += amdgpu_acpi.o
-amdgpu-$(CONFIG_MMU_NOTIFIER) += amdgpu_mn.o
+amdgpu-$(CONFIG_HMM_MIRROR) += amdgpu_mn.o
 
  include $(FULL_AMD_PATH)/powerplay/Makefile
 

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +1 -2

index e55508b39496..5d518d2bb9be 100644
@@ -45,7 +45,7 @@
 
  #include <linux/firmware.h>
  #include <linux/module.h>
-#include <linux/mmu_notifier.h>
+#include <linux/hmm.h>
  #include <linux/interval_tree.h>
  #include <drm/drmP.h>
  #include <drm/drm.h>

@@ -58,7 +58,6 @@
   *
   * @adev: amdgpu device pointer
   * @mm: process address space
- * @mn: MMU notifier structure
   * @type: type of MMU notifier
   * @work: destruction work item
   * @node: hash table node to find structure by adev and mn @@ -66,6 +65,7

@@
* @objects: interval tree containing amdgpu_mn_nodes
* @read_lock: mutex for recursive locking of @lock
* @recursion: depth of recursion
+ * @mirror: HMM mirror function support
*
* Data for each amdgpu device and process address space. */

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +12 -11

@@ -73,7 +73,6 @@ struct amdgpu_mn {
  	/* constant after initialisation */
  	struct amdgpu_device	*adev;
  	struct mm_struct	*mm;
-	struct mmu_notifier	mn;
  	enum amdgpu_mn_type	type;
 
  	/* only used on destruction */

@@ -87,6 +86,9 @@ struct amdgpu_mn {
  	struct rb_root_cached	objects;
  	struct mutex		read_lock;
  	atomic_t		recursion;
+
+	/* HMM mirror */
+	struct hmm_mirror	mirror;
  };
 
  /**

@@ -103,7 +105,7 @@ struct amdgpu_mn_node {  };
 
  /**
- * amdgpu_mn_destroy - destroy the MMU notifier
+ * amdgpu_mn_destroy - destroy the HMM mirror
   *
   * @work: previously sheduled work item
   *

@@ -129,28 +131,26 @@ static void amdgpu_mn_destroy(struct work_struct
 *work)
  	}
  	up_write(&amn->lock);
  	mutex_unlock(&adev->mn_lock);
-	mmu_notifier_unregister_no_release(&amn->mn, amn->mm);
+
+	hmm_mirror_unregister(&amn->mirror);
  	kfree(amn);
  }
 
  /**
- * amdgpu_mn_release - callback to notify about mm destruction
+ * amdgpu_hmm_mirror_release - callback to notify about mm destruction
   *
- * @mn: our notifier
- * @mm: the mm this callback is about
+ * @mirror: the HMM mirror (mm) this callback is about
   *
- * Shedule a work item to lazy destroy our notifier.
+ * Shedule a work item to lazy destroy HMM mirror.
   */
-static void amdgpu_mn_release(struct mmu_notifier *mn,
-			      struct mm_struct *mm)
+static void amdgpu_hmm_mirror_release(struct hmm_mirror *mirror)
  {
-	struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn,
 mn);
+	struct amdgpu_mn *amn = container_of(mirror, struct amdgpu_mn,
+mirror);
 
  	INIT_WORK(&amn->work, amdgpu_mn_destroy);
  	schedule_work(&amn->work);
  }
 
-
  /**
   * amdgpu_mn_lock - take the write side lock for this notifier
   *

@@ -237,21 +237,19 @@ static void amdgpu_mn_invalidate_node(struct
 amdgpu_mn_node *node,
  /**
   * amdgpu_mn_invalidate_range_start_gfx - callback to notify about mm

change
*
- * @mn: our notifier
- * @mm: the mm this callback is about
- * @start: start of updated range
- * @end: end of updated range
+ * @mirror: the hmm_mirror (mm) is about to update
+ * @update: the update start, end address
*
* Block for operations on BOs to finish and mark pages as accessed and
* potentially dirty.
*/

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +2 -6

-static int amdgpu_mn_invalidate_range_start_gfx(struct mmu_notifier *mn,
-						 struct mm_struct *mm,
-						 unsigned long start,
-						 unsigned long end,
-						 bool blockable)
+static int amdgpu_mn_sync_pagetables_gfx(struct hmm_mirror *mirror,
+			const struct hmm_update *update)
  {
-	struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn,

mn);

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +1 -0

+	struct amdgpu_mn *amn = container_of(mirror, struct amdgpu_mn,

mirror);

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +4 -0

+	unsigned long start = update->start;
+	unsigned long end = update->end;
+	bool blockable = update->blockable;
  	struct interval_tree_node *it;
 
  	/* notification is exclusive, but interval is inclusive */ @@ -278,28
+276,28 @@ static int amdgpu_mn_invalidate_range_start_gfx(struct

mmu_notifier *mn,
amdgpu_mn_invalidate_node(node, start, end);
}

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +2 -0

+	amdgpu_mn_read_unlock(amn);
+
  	return 0;
  }
 
  /**
   * amdgpu_mn_invalidate_range_start_hsa - callback to notify about mm

change
*
- * @mn: our notifier
- * @mm: the mm this callback is about
- * @start: start of updated range
- * @end: end of updated range
+ * @mirror: the hmm_mirror (mm) is about to update
+ * @update: the update start, end address
*
* We temporarily evict all BOs between start and end. This
* necessitates evicting all user-mode queues of the process. The BOs
* are restorted in amdgpu_mn_invalidate_range_end_hsa.
*/
-static int amdgpu_mn_invalidate_range_start_hsa(struct mmu_notifier *mn,

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +2 -5

-						 struct mm_struct *mm,
-						 unsigned long start,
-						 unsigned long end,
-						 bool blockable)
+static int amdgpu_mn_sync_pagetables_hsa(struct hmm_mirror *mirror,
+			const struct hmm_update *update)
  {
-	struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn,

mn);

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +1 -0

+	struct amdgpu_mn *amn = container_of(mirror, struct amdgpu_mn,

mirror);

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +4 -0

+	unsigned long start = update->start;
+	unsigned long end = update->end;
+	bool blockable = update->blockable;
  	struct interval_tree_node *it;
 
  	/* notification is exclusive, but interval is inclusive */ @@ -326,59
+324,39 @@ static int amdgpu_mn_invalidate_range_start_hsa(struct

mmu_notifier *mn,

if (amdgpu_ttm_tt_affect_userptr(bo->tbo.ttm,
start, end))

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +1 -1

-				amdgpu_amdkfd_evict_userptr(mem, mm);
+				amdgpu_amdkfd_evict_userptr(mem, amn-

Quoted 1 line

mm);

}
}

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +5 -15

+	amdgpu_mn_read_unlock(amn);
+
  	return 0;
  }
 
-/**
- * amdgpu_mn_invalidate_range_end - callback to notify about mm change
- *
- * @mn: our notifier
- * @mm: the mm this callback is about
- * @start: start of updated range
- * @end: end of updated range
- *
- * Release the lock again to allow new command submissions.
+/* Low bits of any reasonable mm pointer will be unused due to struct
+ * alignment. Use these bits to make a unique key from the mm pointer
+ * and notifier type.
   */
-static void amdgpu_mn_invalidate_range_end(struct mmu_notifier *mn,
-					   struct mm_struct *mm,
-					   unsigned long start,
-					   unsigned long end)
-{
-	struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn,

mn);

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +2 -6

-
-	amdgpu_mn_read_unlock(amn);
-}
+#define AMDGPU_MN_KEY(mm, type) ((unsigned long)(mm) + (type))
 
-static const struct mmu_notifier_ops amdgpu_mn_ops[] = {
+static struct hmm_mirror_ops amdgpu_hmm_mirror_ops[] = {
  	[AMDGPU_MN_TYPE_GFX] = {
-		.release = amdgpu_mn_release,
-		.invalidate_range_start =

amdgpu_mn_invalidate_range_start_gfx,

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +1 -1

-		.invalidate_range_end = amdgpu_mn_invalidate_range_end,
+		.sync_cpu_device_pagetables =

amdgpu_mn_sync_pagetables_gfx,

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +1 -2

+		.release = amdgpu_hmm_mirror_release
  	},
  	[AMDGPU_MN_TYPE_HSA] = {
-		.release = amdgpu_mn_release,
-		.invalidate_range_start =

amdgpu_mn_invalidate_range_start_hsa,

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +1 -1

-		.invalidate_range_end = amdgpu_mn_invalidate_range_end,
+		.sync_cpu_device_pagetables =

amdgpu_mn_sync_pagetables_hsa,

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +6 -11

+		.release = amdgpu_hmm_mirror_release
  	},
  };
 
-/* Low bits of any reasonable mm pointer will be unused due to struct
- * alignment. Use these bits to make a unique key from the mm pointer
- * and notifier type.
- */
-#define AMDGPU_MN_KEY(mm, type) ((unsigned long)(mm) + (type))
-
  /**
- * amdgpu_mn_get - create notifier context
+ * amdgpu_mn_get - create HMM mirror context
   *
   * @adev: amdgpu device pointer
   * @type: type of MMU notifier context
   *
- * Creates a notifier context for current->mm.
+ * Creates a HMM mirror context for current->mm.
   */
  struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev,
  				enum amdgpu_mn_type type)

@@ -408,12 +386,12 @@ struct amdgpu_mn *amdgpu_mn_get(struct
 amdgpu_device *adev,
  	amn->mm = mm;
  	init_rwsem(&amn->lock);
  	amn->type = type;
-	amn->mn.ops = &amdgpu_mn_ops[type];
  	amn->objects = RB_ROOT_CACHED;
  	mutex_init(&amn->read_lock);
  	atomic_set(&amn->recursion, 0);
 
-	r = __mmu_notifier_register(&amn->mn, mm);
+	amn->mirror.ops = &amdgpu_hmm_mirror_ops[type];
+	r = hmm_mirror_register(&amn->mirror, mm);
  	if (r)
  		goto free_amn;

@@ -439,7 +417,7 @@ struct amdgpu_mn *amdgpu_mn_get(struct
 amdgpu_device *adev,
   * @bo: amdgpu buffer object
   * @addr: userptr addr we should monitor
   *
- * Registers an MMU notifier for the given BO at the specified address.
+ * Registers an HMM mirror for the given BO at the specified address.
   * Returns 0 on success, -ERRNO if anything goes wrong.
   */

int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr) @@ -
495,11 +473,11 @@ int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr) }

/**
- * amdgpu_mn_unregister - unregister a BO for notifier updates
+ * amdgpu_mn_unregister - unregister a BO for HMM mirror updates *
* @bo: amdgpu buffer object
*
- * Remove any registration of MMU notifier updates from the buffer object.
+ * Remove any registration of HMM mirror updates from the buffer object. */
void amdgpu_mn_unregister(struct amdgpu_bo *bo) { diff --git
a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h
index eb0f432f78fe..0a51fd00021c 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h +1 -1

@@ -34,7 +34,7 @@ enum amdgpu_mn_type {
  	AMDGPU_MN_TYPE_HSA,
  };
 
-#if defined(CONFIG_MMU_NOTIFIER)
+#if defined(CONFIG_HMM_MIRROR)
  void amdgpu_mn_lock(struct amdgpu_mn *mn);  void

amdgpu_mn_unlock(struct amdgpu_mn *mn); struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev,
--
2.17.1

Quoted 4 lines

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx