Skip to contentSkip to conversation

Use HMM to replace get_user_pages

PATCH In review Yang, Philip 2 participants 7 messages Feb 4, 2019 4 messages not in the archive

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

cover Use HMM to replace get_user_pages

Hi Christian,

This patch is rebased to lastest HMM. Please review the GEM and CS part changes in patch 3/3.

Regards,

Philip Yang (3):
drm/amdgpu: use HMM mirror callback to replace mmu notifier v6 drm/amdkfd: avoid HMM change cause circular lock dependency v2 drm/amdgpu: replace get_user_pages with HMM address mirror helpers v6

 drivers/gpu/drm/amd/amdgpu/Kconfig            |   6 +-
 drivers/gpu/drm/amd/amdgpu/Makefile           |   2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h    |   1 -
 .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c  |  95 +++-------
 drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.h   |   2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c        | 158 +++++++---------
 drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c       |  14 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c        | 164 +++++++++--------
 drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h        |   6 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c       | 173 ++++++++----------
 drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h       |   3 +-
 .../drm/amd/amdkfd/kfd_device_queue_manager.c |  32 ++--
 12 files changed, 282 insertions(+), 374 deletions(-)
Signature · 6 lines
2.17.1

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): [email protected]

#2 lore raw

PATCH 1/3 drm/amdgpu: use HMM mirror callback to replace mmu notifier v6

Replace our MMU notifier with hmm_mirror_ops.sync_cpu_device_pagetables callback. Enable CONFIG_HMM and CONFIG_HMM_MIRROR as a dependency in DRM_AMDGPU_USERPTR Kconfig.

It supports both KFD userptr and gfx userptr paths.

The depdent HMM patchset from Jérôme Glisse are all merged into 4.20.0 kernel now.

Change-Id Ie62c3c5e3c5b8521ab3b438d1eff2aa2a003835e
---
 drivers/gpu/drm/amd/amdgpu/Kconfig     |   6 +-
 drivers/gpu/drm/amd/amdgpu/Makefile    |   2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c | 139 +++++++++++--------------
 drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h |   2 +-
 4 files changed, 67 insertions(+), 82 deletions(-)
drivers/gpu/drm/amd/amdgpu/Kconfig +3 -3
index 9221e5489069..960a63355705 100644
@@ -26,10 +26,10 @@ config DRM_AMDGPU_CIK
 config DRM_AMDGPU_USERPTR
 	bool "Always enable userptr write support"
 	depends on DRM_AMDGPU
-	select MMU_NOTIFIER
+	select HMM_MIRROR
 	help
-	  This option selects CONFIG_MMU_NOTIFIER if it isn't already
-	  selected to enabled full userptr support.
+	  This option selects CONFIG_HMM and CONFIG_HMM_MIRROR if it
+	  isn't already selected to enabled full userptr support.
 
 config DRM_AMDGPU_GART_DEBUGFS
 	bool "Allow GART access through debugfs"
drivers/gpu/drm/amd/amdgpu/Makefile +1 -1
index 466da5954a68..851001ced5e8 100644
@@ -172,7 +172,7 @@ endif
 amdgpu-$(CONFIG_COMPAT) += amdgpu_ioc32.o
 amdgpu-$(CONFIG_VGA_SWITCHEROO) += amdgpu_atpx_handler.o
 amdgpu-$(CONFIG_ACPI) += amdgpu_acpi.o
-amdgpu-$(CONFIG_MMU_NOTIFIER) += amdgpu_mn.o
+amdgpu-$(CONFIG_HMM_MIRROR) += amdgpu_mn.o
 
 include $(FULL_AMD_PATH)/powerplay/Makefile
 
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +62 -77

301 lines not on this page · raw message

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h +1 -1
index eb0f432f78fe..0a51fd00021c 100644
@@ -34,7 +34,7 @@ enum amdgpu_mn_type {
 	AMDGPU_MN_TYPE_HSA,
 };
 
-#if defined(CONFIG_MMU_NOTIFIER)
+#if defined(CONFIG_HMM_MIRROR)
 void amdgpu_mn_lock(struct amdgpu_mn *mn);
 void amdgpu_mn_unlock(struct amdgpu_mn *mn);
 struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev,
Signature · 6 lines
2.17.1

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): [email protected]

#4 lore raw

Re: [PATCH 1/3] drm/amdgpu: use HMM mirror callback to replace mmu notifier v6

Am 04.02.19 um 16:06 schrieb Yang, Philip:

Quoted 87 lines

Replace our MMU notifier with hmm_mirror_ops.sync_cpu_device_pagetables callback. Enable CONFIG_HMM and CONFIG_HMM_MIRROR as a dependency in DRM_AMDGPU_USERPTR Kconfig.

It supports both KFD userptr and gfx userptr paths.

The depdent HMM patchset from Jérôme Glisse are all merged into 4.20.0 kernel now.

Change-Id Ie62c3c5e3c5b8521ab3b438d1eff2aa2a003835e
---
  drivers/gpu/drm/amd/amdgpu/Kconfig     |   6 +-
  drivers/gpu/drm/amd/amdgpu/Makefile    |   2 +-
  drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c | 139 +++++++++++--------------
  drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h |   2 +-
  4 files changed, 67 insertions(+), 82 deletions(-)
drivers/gpu/drm/amd/amdgpu/Kconfig +3 -3
index 9221e5489069..960a63355705 100644
@@ -26,10 +26,10 @@ config DRM_AMDGPU_CIK
  config DRM_AMDGPU_USERPTR
  	bool "Always enable userptr write support"
  	depends on DRM_AMDGPU
-	select MMU_NOTIFIER
+	select HMM_MIRROR
  	help
-	  This option selects CONFIG_MMU_NOTIFIER if it isn't already
-	  selected to enabled full userptr support.
+	  This option selects CONFIG_HMM and CONFIG_HMM_MIRROR if it
+	  isn't already selected to enabled full userptr support.
 
  config DRM_AMDGPU_GART_DEBUGFS
  	bool "Allow GART access through debugfs"
drivers/gpu/drm/amd/amdgpu/Makefile +1 -1
index 466da5954a68..851001ced5e8 100644
@@ -172,7 +172,7 @@ endif
  amdgpu-$(CONFIG_COMPAT) += amdgpu_ioc32.o
  amdgpu-$(CONFIG_VGA_SWITCHEROO) += amdgpu_atpx_handler.o
  amdgpu-$(CONFIG_ACPI) += amdgpu_acpi.o
-amdgpu-$(CONFIG_MMU_NOTIFIER) += amdgpu_mn.o
+amdgpu-$(CONFIG_HMM_MIRROR) += amdgpu_mn.o
 
  include $(FULL_AMD_PATH)/powerplay/Makefile
 
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +2 -3
index 3e6823fdd939..5d518d2bb9be 100644
@@ -45,7 +45,7 @@
 
  #include <linux/firmware.h>
  #include <linux/module.h>
-#include <linux/mmu_notifier.h>
+#include <linux/hmm.h>
  #include <linux/interval_tree.h>
  #include <drm/drmP.h>
  #include <drm/drm.h>
@@ -58,7 +58,6 @@
   *
   * @adev: amdgpu device pointer
   * @mm: process address space
- * @mn: MMU notifier structure
   * @type: type of MMU notifier
   * @work: destruction work item
   * @node: hash table node to find structure by adev and mn
@@ -66,6 +65,7 @@
   * @objects: interval tree containing amdgpu_mn_nodes
   * @read_lock: mutex for recursive locking of @lock
   * @recursion: depth of recursion
+ * @mirror: HMM mirror function support
   *
   * Data for each amdgpu device and process address space.
   */
@@ -73,7 +73,6 @@ struct amdgpu_mn {
  	/* constant after initialisation */
  	struct amdgpu_device	*adev;
  	struct mm_struct	*mm;
-	struct mmu_notifier	mn;
  	enum amdgpu_mn_type	type;
 
  	/* only used on destruction */

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c struct amdgpu_mn { · line 89

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +0 -0
@@ -87,6 +86,9 @@ struct amdgpu_mn {
  	struct rb_root_cached	objects;
  	struct mutex		read_lock;
 
  	atomic_t		recursion;

With HMM we don't need this any more. Please remove it and simplify amdgpu_mn_read_lock() and amdgpu_mn_read_unlock().

Apart from that looks good to me,
Christian.

Quoted 280 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +3 -0
+
+	/* HMM mirror */
+	struct hmm_mirror	mirror;
  };
 

/**

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +57 -74
@@ -103,7 +105,7 @@ struct amdgpu_mn_node {
  };
 
  /**
- * amdgpu_mn_destroy - destroy the MMU notifier
+ * amdgpu_mn_destroy - destroy the HMM mirror
   *
   * @work: previously sheduled work item
   *
@@ -129,28 +131,26 @@ static void amdgpu_mn_destroy(struct work_struct *work)
  	}
  	up_write(&amn->lock);
  	mutex_unlock(&adev->mn_lock);
-	mmu_notifier_unregister_no_release(&amn->mn, amn->mm);
+
+	hmm_mirror_unregister(&amn->mirror);
  	kfree(amn);
  }
 
  /**
- * amdgpu_mn_release - callback to notify about mm destruction
+ * amdgpu_hmm_mirror_release - callback to notify about mm destruction
   *
- * @mn: our notifier
- * @mm: the mm this callback is about
+ * @mirror: the HMM mirror (mm) this callback is about
   *
- * Shedule a work item to lazy destroy our notifier.
+ * Shedule a work item to lazy destroy HMM mirror.
   */
-static void amdgpu_mn_release(struct mmu_notifier *mn,
-			      struct mm_struct *mm)
+static void amdgpu_hmm_mirror_release(struct hmm_mirror *mirror)
  {
-	struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn, mn);
+	struct amdgpu_mn *amn = container_of(mirror, struct amdgpu_mn, mirror);
 
  	INIT_WORK(&amn->work, amdgpu_mn_destroy);
  	schedule_work(&amn->work);
  }
 
-
  /**
   * amdgpu_mn_lock - take the write side lock for this notifier
   *
@@ -237,141 +237,126 @@ static void amdgpu_mn_invalidate_node(struct amdgpu_mn_node *node,
  /**
   * amdgpu_mn_invalidate_range_start_gfx - callback to notify about mm change
   *
- * @mn: our notifier
- * @range: mmu notifier context
+ * @mirror: the hmm_mirror (mm) is about to update
+ * @update: the update start, end address
   *
   * Block for operations on BOs to finish and mark pages as accessed and
   * potentially dirty.
   */
-static int amdgpu_mn_invalidate_range_start_gfx(struct mmu_notifier *mn,
-			const struct mmu_notifier_range *range)
+static int amdgpu_mn_sync_pagetables_gfx(struct hmm_mirror *mirror,
+			const struct hmm_update *update)
  {
-	struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn, mn);
+	struct amdgpu_mn *amn = container_of(mirror, struct amdgpu_mn, mirror);
+	unsigned long start = update->start;
+	unsigned long end = update->end;
+	bool blockable = update->blockable;
  	struct interval_tree_node *it;
-	unsigned long end;
 
  	/* notification is exclusive, but interval is inclusive */
-	end = range->end - 1;
+	end -= 1;
 
  	/* TODO we should be able to split locking for interval tree and
  	 * amdgpu_mn_invalidate_node
  	 */
-	if (amdgpu_mn_read_lock(amn, range->blockable))
+	if (amdgpu_mn_read_lock(amn, blockable))
  		return -EAGAIN;
 
-	it = interval_tree_iter_first(&amn->objects, range->start, end);
+	it = interval_tree_iter_first(&amn->objects, start, end);
  	while (it) {
  		struct amdgpu_mn_node *node;
 
-		if (!range->blockable) {
+		if (!blockable) {
  			amdgpu_mn_read_unlock(amn);
  			return -EAGAIN;
  		}
 
  		node = container_of(it, struct amdgpu_mn_node, it);
-		it = interval_tree_iter_next(it, range->start, end);
+		it = interval_tree_iter_next(it, start, end);
 
-		amdgpu_mn_invalidate_node(node, range->start, end);
+		amdgpu_mn_invalidate_node(node, start, end);
  	}
 
+	amdgpu_mn_read_unlock(amn);
+
  	return 0;
  }
 
  /**
   * amdgpu_mn_invalidate_range_start_hsa - callback to notify about mm change
   *
- * @mn: our notifier
- * @mm: the mm this callback is about
- * @start: start of updated range
- * @end: end of updated range
+ * @mirror: the hmm_mirror (mm) is about to update
+ * @update: the update start, end address
   *
   * We temporarily evict all BOs between start and end. This
   * necessitates evicting all user-mode queues of the process. The BOs
   * are restorted in amdgpu_mn_invalidate_range_end_hsa.
   */
-static int amdgpu_mn_invalidate_range_start_hsa(struct mmu_notifier *mn,
-			const struct mmu_notifier_range *range)
+static int amdgpu_mn_sync_pagetables_hsa(struct hmm_mirror *mirror,
+			const struct hmm_update *update)
  {
-	struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn, mn);
+	struct amdgpu_mn *amn = container_of(mirror, struct amdgpu_mn, mirror);
+	unsigned long start = update->start;
+	unsigned long end = update->end;
+	bool blockable = update->blockable;
  	struct interval_tree_node *it;
-	unsigned long end;
 
  	/* notification is exclusive, but interval is inclusive */
-	end = range->end - 1;
+	end -= 1;
 
-	if (amdgpu_mn_read_lock(amn, range->blockable))
+	if (amdgpu_mn_read_lock(amn, blockable))
  		return -EAGAIN;
 
-	it = interval_tree_iter_first(&amn->objects, range->start, end);
+	it = interval_tree_iter_first(&amn->objects, start, end);
  	while (it) {
  		struct amdgpu_mn_node *node;
  		struct amdgpu_bo *bo;
 
-		if (!range->blockable) {
+		if (!blockable) {
  			amdgpu_mn_read_unlock(amn);
  			return -EAGAIN;
  		}
 
  		node = container_of(it, struct amdgpu_mn_node, it);
-		it = interval_tree_iter_next(it, range->start, end);
+		it = interval_tree_iter_next(it, start, end);
 
  		list_for_each_entry(bo, &node->bos, mn_list) {
  			struct kgd_mem *mem = bo->kfd_bo;
 
  			if (amdgpu_ttm_tt_affect_userptr(bo->tbo.ttm,
-							 range->start,
-							 end))
-				amdgpu_amdkfd_evict_userptr(mem, range->mm);
+							 start, end))
+				amdgpu_amdkfd_evict_userptr(mem, amn->mm);
  		}
  	}
 
+	amdgpu_mn_read_unlock(amn);
+
  	return 0;
  }
 
-/**
- * amdgpu_mn_invalidate_range_end - callback to notify about mm change
- *
- * @mn: our notifier
- * @mm: the mm this callback is about
- * @start: start of updated range
- * @end: end of updated range
- *
- * Release the lock again to allow new command submissions.
+/* Low bits of any reasonable mm pointer will be unused due to struct
+ * alignment. Use these bits to make a unique key from the mm pointer
+ * and notifier type.
   */
-static void amdgpu_mn_invalidate_range_end(struct mmu_notifier *mn,
-			const struct mmu_notifier_range *range)
-{
-	struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn, mn);
-
-	amdgpu_mn_read_unlock(amn);
-}
+#define AMDGPU_MN_KEY(mm, type) ((unsigned long)(mm) + (type))
 
-static const struct mmu_notifier_ops amdgpu_mn_ops[] = {
+static struct hmm_mirror_ops amdgpu_hmm_mirror_ops[] = {
  	[AMDGPU_MN_TYPE_GFX] = {
-		.release = amdgpu_mn_release,
-		.invalidate_range_start = amdgpu_mn_invalidate_range_start_gfx,
-		.invalidate_range_end = amdgpu_mn_invalidate_range_end,
+		.sync_cpu_device_pagetables = amdgpu_mn_sync_pagetables_gfx,
+		.release = amdgpu_hmm_mirror_release
  	},
  	[AMDGPU_MN_TYPE_HSA] = {
-		.release = amdgpu_mn_release,
-		.invalidate_range_start = amdgpu_mn_invalidate_range_start_hsa,
-		.invalidate_range_end = amdgpu_mn_invalidate_range_end,
+		.sync_cpu_device_pagetables = amdgpu_mn_sync_pagetables_hsa,
+		.release = amdgpu_hmm_mirror_release
  	},
  };
 
-/* Low bits of any reasonable mm pointer will be unused due to struct
- * alignment. Use these bits to make a unique key from the mm pointer
- * and notifier type.
- */
-#define AMDGPU_MN_KEY(mm, type) ((unsigned long)(mm) + (type))
-
  /**
- * amdgpu_mn_get - create notifier context
+ * amdgpu_mn_get - create HMM mirror context
   *
   * @adev: amdgpu device pointer
   * @type: type of MMU notifier context
   *
- * Creates a notifier context for current->mm.
+ * Creates a HMM mirror context for current->mm.
   */
  struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev,
  				enum amdgpu_mn_type type)
@@ -401,12 +386,12 @@ struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev,
  	amn->mm = mm;
  	init_rwsem(&amn->lock);
  	amn->type = type;
-	amn->mn.ops = &amdgpu_mn_ops[type];
  	amn->objects = RB_ROOT_CACHED;
  	mutex_init(&amn->read_lock);
  	atomic_set(&amn->recursion, 0);
 
-	r = __mmu_notifier_register(&amn->mn, mm);
+	amn->mirror.ops = &amdgpu_hmm_mirror_ops[type];
+	r = hmm_mirror_register(&amn->mirror, mm);
  	if (r)
  		goto free_amn;
 
@@ -432,7 +417,7 @@ struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev,
   * @bo: amdgpu buffer object
   * @addr: userptr addr we should monitor
   *
- * Registers an MMU notifier for the given BO at the specified address.
+ * Registers an HMM mirror for the given BO at the specified address.
   * Returns 0 on success, -ERRNO if anything goes wrong.
   */
  int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)
@@ -488,11 +473,11 @@ int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)
  }
 
  /**
- * amdgpu_mn_unregister - unregister a BO for notifier updates
+ * amdgpu_mn_unregister - unregister a BO for HMM mirror updates
   *
   * @bo: amdgpu buffer object
   *
- * Remove any registration of MMU notifier updates from the buffer object.
+ * Remove any registration of HMM mirror updates from the buffer object.
   */
  void amdgpu_mn_unregister(struct amdgpu_bo *bo)
  {
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h +0 -0
index eb0f432f78fe..0a51fd00021c 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h enum amdgpu_mn_type { · line 40

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h +1 -1
@@ -34,7 +34,7 @@ enum amdgpu_mn_type {
  	AMDGPU_MN_TYPE_HSA,
  };
 
-#if defined(CONFIG_MMU_NOTIFIER)
+#if defined(CONFIG_HMM_MIRROR)
  void amdgpu_mn_lock(struct amdgpu_mn *mn);
  void amdgpu_mn_unlock(struct amdgpu_mn *mn);
  struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev,

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 71e6094c-a257-7aa3-854c-9a9dff163b01-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org

#6 lore raw

Re: [PATCH 1/3] drm/amdgpu: use HMM mirror callback to replace mmu notifier v6

Quoted 89 lines · Christian König

Am 04.02.19 um 16:06 schrieb Yang, Philip:

Quoted 87 lines

Replace our MMU notifier with hmm_mirror_ops.sync_cpu_device_pagetables callback. Enable CONFIG_HMM and CONFIG_HMM_MIRROR as a dependency in DRM_AMDGPU_USERPTR Kconfig.

It supports both KFD userptr and gfx userptr paths.

The depdent HMM patchset from Jérôme Glisse are all merged into 4.20.0 kernel now.

Change-Id Ie62c3c5e3c5b8521ab3b438d1eff2aa2a003835e

---
  drivers/gpu/drm/amd/amdgpu/Kconfig     |   6 +- drivers/gpu/drm/amd/amdgpu/Makefile    |   2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c | 139 +++++++++++-------------- drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h |   2 +-

  4 files changed, 67 insertions(+), 82 deletions(-)
drivers/gpu/drm/amd/amdgpu/Kconfig +0 -0
index 9221e5489069..960a63355705 100644
@@ -26,10 +26,10 @@ config DRM_AMDGPU_CIK

  config DRM_AMDGPU_USERPTR
      bool "Always enable userptr write support" depends on DRM_AMDGPU
-    select MMU_NOTIFIER
+    select HMM_MIRROR
      help
-      This option selects CONFIG_MMU_NOTIFIER if it isn't already -      selected to enabled full userptr support.
+      This option selects CONFIG_HMM and CONFIG_HMM_MIRROR if it +      isn't already selected to enabled full userptr support. config DRM_AMDGPU_GART_DEBUGFS
      bool "Allow GART access through debugfs"

drivers/gpu/drm/amd/amdgpu/Makefile +0 -0
index 466da5954a68..851001ced5e8 100644
@@ -172,7 +172,7 @@ endif

  amdgpu-$(CONFIG_COMPAT) += amdgpu_ioc32.o amdgpu-$(CONFIG_VGA_SWITCHEROO) += amdgpu_atpx_handler.o amdgpu-$(CONFIG_ACPI) += amdgpu_acpi.o

drivers/gpu/drm/amd/amdgpu/Makefile +1 -1
-amdgpu-$(CONFIG_MMU_NOTIFIER) += amdgpu_mn.o
+amdgpu-$(CONFIG_HMM_MIRROR) += amdgpu_mn.o

  include $(FULL_AMD_PATH)/powerplay/Makefile

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +2 -3
index 3e6823fdd939..5d518d2bb9be 100644
@@ -45,7 +45,7 @@
   #include <linux/firmware.h>
   #include <linux/module.h>
-#include <linux/mmu_notifier.h>
+#include <linux/hmm.h>
   #include <linux/interval_tree.h>
   #include <drm/drmP.h>
   #include <drm/drm.h>
@@ -58,7 +58,6 @@
    *
    * @adev: amdgpu device pointer
    * @mm: process address space
- * @mn: MMU notifier structure
    * @type: type of MMU notifier
    * @work: destruction work item
    * @node: hash table node to find structure by adev and mn
@@ -66,6 +65,7 @@
    * @objects: interval tree containing amdgpu_mn_nodes
    * @read_lock: mutex for recursive locking of @lock
    * @recursion: depth of recursion
+ * @mirror: HMM mirror function support
    *
    * Data for each amdgpu device and process address space.
    */
@@ -73,7 +73,6 @@ struct amdgpu_mn {
       /* constant after initialisation */
       struct amdgpu_device    *adev;
       struct mm_struct    *mm;
-    struct mmu_notifier    mn;
       enum amdgpu_mn_type    type;
       /* only used on destruction */
@@ -87,6 +86,9 @@ struct amdgpu_mn {
       struct rb_root_cached    objects;
       struct mutex        read_lock;
Quoted 5 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c struct amdgpu_mn { · line 88

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +0 -0
       atomic_t        recursion;

With HMM we don't need this any more. Please remove it and simplify amdgpu_mn_read_lock() and amdgpu_mn_read_unlock().

Thanks, this makes sense because HMM uses hmm->mirror_sem to serialize invalidate range.

amn->read_lock is also not needed anymore, this was used to protect atomic operations for atomic_inc_return(amn->recursion) and down_read(amn->lock).

Just one amn->lock is needed to sync amdgpu_cs_submit and userptr update. I will submit another patch.

Philip

Quoted 267 lines

Apart from that looks good to me,
Christian.

Quoted 264 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +11 -10
+
+    /* HMM mirror */
+    struct hmm_mirror    mirror;
   };
   /**
@@ -103,7 +105,7 @@ struct amdgpu_mn_node {
   };
   /**
- * amdgpu_mn_destroy - destroy the MMU notifier
+ * amdgpu_mn_destroy - destroy the HMM mirror
    *
    * @work: previously sheduled work item
    *
@@ -129,28 +131,26 @@ static void amdgpu_mn_destroy(struct work_struct 
 *work)
       }
       up_write(&amn->lock);
       mutex_unlock(&adev->mn_lock);
-    mmu_notifier_unregister_no_release(&amn->mn, amn->mm);
+
+    hmm_mirror_unregister(&amn->mirror);
       kfree(amn);
   }
   /**
- * amdgpu_mn_release - callback to notify about mm destruction
+ * amdgpu_hmm_mirror_release - callback to notify about mm destruction
    *
- * @mn: our notifier
- * @mm: the mm this callback is about
+ * @mirror: the HMM mirror (mm) this callback is about
    *
- * Shedule a work item to lazy destroy our notifier.
+ * Shedule a work item to lazy destroy HMM mirror.
    */
-static void amdgpu_mn_release(struct mmu_notifier *mn,
-                  struct mm_struct *mm)
+static void amdgpu_hmm_mirror_release(struct hmm_mirror *mirror)
   {
-    struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn, mn);
+    struct amdgpu_mn *amn = container_of(mirror, struct amdgpu_mn, 
 mirror);
       INIT_WORK(&amn->work, amdgpu_mn_destroy);
       schedule_work(&amn->work);
   }
-
   /**
    * amdgpu_mn_lock - take the write side lock for this notifier
    *
@@ -237,141 +237,126 @@ static void amdgpu_mn_invalidate_node(struct 
 amdgpu_mn_node *node,
   /**
    * amdgpu_mn_invalidate_range_start_gfx - callback to notify about 

mm change
   *
- * @mn: our notifier
- * @range: mmu notifier context
+ * @mirror: the hmm_mirror (mm) is about to update
+ * @update: the update start, end address
   *
   * Block for operations on BOs to finish and mark pages as accessed and
   * potentially dirty.
   */

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +2 -2
-static int amdgpu_mn_invalidate_range_start_gfx(struct mmu_notifier *mn,
-            const struct mmu_notifier_range *range)
+static int amdgpu_mn_sync_pagetables_gfx(struct hmm_mirror *mirror,
+            const struct hmm_update *update)

  {
-    struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn, mn); +    struct amdgpu_mn *amn = container_of(mirror, struct amdgpu_mn, mirror);
+    unsigned long start = update->start;
+    unsigned long end = update->end;
+    bool blockable = update->blockable;
      struct interval_tree_node *it;
-    unsigned long end;
      /* notification is exclusive, but interval is inclusive */ -    end = range->end - 1;
+    end -= 1;
      /* TODO we should be able to split locking for interval tree and
       * amdgpu_mn_invalidate_node
       */
-    if (amdgpu_mn_read_lock(amn, range->blockable))
+    if (amdgpu_mn_read_lock(amn, blockable))
          return -EAGAIN;
-    it = interval_tree_iter_first(&amn->objects, range->start, end); +    it = interval_tree_iter_first(&amn->objects, start, end); while (it) {
          struct amdgpu_mn_node *node;
-        if (!range->blockable) {
+        if (!blockable) {
              amdgpu_mn_read_unlock(amn);
              return -EAGAIN;
          }
          node = container_of(it, struct amdgpu_mn_node, it); -        it = interval_tree_iter_next(it, range->start, end); +        it = interval_tree_iter_next(it, start, end);
-        amdgpu_mn_invalidate_node(node, range->start, end); +        amdgpu_mn_invalidate_node(node, start, end);
      }
+    amdgpu_mn_read_unlock(amn);
+
      return 0;
  }
  /**
   * amdgpu_mn_invalidate_range_start_hsa - callback to notify about mm change
   *
- * @mn: our notifier
- * @mm: the mm this callback is about
- * @start: start of updated range
- * @end: end of updated range
+ * @mirror: the hmm_mirror (mm) is about to update
+ * @update: the update start, end address
   *
   * We temporarily evict all BOs between start and end. This
   * necessitates evicting all user-mode queues of the process. The BOs
   * are restorted in amdgpu_mn_invalidate_range_end_hsa.
   */

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +2 -2
-static int amdgpu_mn_invalidate_range_start_hsa(struct mmu_notifier *mn,
-            const struct mmu_notifier_range *range)
+static int amdgpu_mn_sync_pagetables_hsa(struct hmm_mirror *mirror,
+            const struct hmm_update *update)

  {
-    struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn, mn); +    struct amdgpu_mn *amn = container_of(mirror, struct amdgpu_mn, mirror);
+    unsigned long start = update->start;
+    unsigned long end = update->end;
+    bool blockable = update->blockable;
      struct interval_tree_node *it;
-    unsigned long end;
      /* notification is exclusive, but interval is inclusive */ -    end = range->end - 1;
+    end -= 1;
-    if (amdgpu_mn_read_lock(amn, range->blockable))
+    if (amdgpu_mn_read_lock(amn, blockable))
          return -EAGAIN;
-    it = interval_tree_iter_first(&amn->objects, range->start, end); +    it = interval_tree_iter_first(&amn->objects, start, end); while (it) {
          struct amdgpu_mn_node *node;
          struct amdgpu_bo *bo;
-        if (!range->blockable) {
+        if (!blockable) {
              amdgpu_mn_read_unlock(amn);
              return -EAGAIN;
          }
          node = container_of(it, struct amdgpu_mn_node, it); -        it = interval_tree_iter_next(it, range->start, end); +        it = interval_tree_iter_next(it, start, end); list_for_each_entry(bo, &node->bos, mn_list) { struct kgd_mem *mem = bo->kfd_bo;
              if (amdgpu_ttm_tt_affect_userptr(bo->tbo.ttm, -                             range->start,
-                             end))
-                amdgpu_amdkfd_evict_userptr(mem, range->mm); +                             start, end))
+                amdgpu_amdkfd_evict_userptr(mem, amn->mm); }
      }
+    amdgpu_mn_read_unlock(amn);
+
      return 0;
  }

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +3 -9
-/**
- * amdgpu_mn_invalidate_range_end - callback to notify about mm change
- *
- * @mn: our notifier
- * @mm: the mm this callback is about
- * @start: start of updated range
- * @end: end of updated range
- *
- * Release the lock again to allow new command submissions.
+/* Low bits of any reasonable mm pointer will be unused due to struct
+ * alignment. Use these bits to make a unique key from the mm pointer
+ * and notifier type.

   */

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +2 -8
-static void amdgpu_mn_invalidate_range_end(struct mmu_notifier *mn,
-            const struct mmu_notifier_range *range)
-{
-    struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn, mn);
-
-    amdgpu_mn_read_unlock(amn);
-}
+#define AMDGPU_MN_KEY(mm, type) ((unsigned long)(mm) + (type))
-static const struct mmu_notifier_ops amdgpu_mn_ops[] = {
+static struct hmm_mirror_ops amdgpu_hmm_mirror_ops[] = {

      [AMDGPU_MN_TYPE_GFX] = {
-        .release = amdgpu_mn_release,
-        .invalidate_range_start = amdgpu_mn_invalidate_range_start_gfx, -        .invalidate_range_end = amdgpu_mn_invalidate_range_end, +        .sync_cpu_device_pagetables = amdgpu_mn_sync_pagetables_gfx, +        .release = amdgpu_hmm_mirror_release
      },
      [AMDGPU_MN_TYPE_HSA] = {
-        .release = amdgpu_mn_release,
-        .invalidate_range_start = amdgpu_mn_invalidate_range_start_hsa, -        .invalidate_range_end = amdgpu_mn_invalidate_range_end, +        .sync_cpu_device_pagetables = amdgpu_mn_sync_pagetables_hsa, +        .release = amdgpu_hmm_mirror_release
      },
  };

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +0 -6
-/* Low bits of any reasonable mm pointer will be unused due to struct
- * alignment. Use these bits to make a unique key from the mm pointer
- * and notifier type.
- */
-#define AMDGPU_MN_KEY(mm, type) ((unsigned long)(mm) + (type))
-

  /**
- * amdgpu_mn_get - create notifier context
+ * amdgpu_mn_get - create HMM mirror context
   *
   * @adev: amdgpu device pointer
   * @type: type of MMU notifier context
   *
- * Creates a notifier context for current->mm.
+ * Creates a HMM mirror context for current->mm. */
  struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev, enum amdgpu_mn_type type)

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +3 -3
@@ -401,12 +386,12 @@ struct amdgpu_mn *amdgpu_mn_get(struct 
 amdgpu_device *adev,
       amn->mm = mm;
       init_rwsem(&amn->lock);
       amn->type = type;
-    amn->mn.ops = &amdgpu_mn_ops[type];
       amn->objects = RB_ROOT_CACHED;
       mutex_init(&amn->read_lock);
       atomic_set(&amn->recursion, 0);
-    r = __mmu_notifier_register(&amn->mn, mm);
+    amn->mirror.ops = &amdgpu_hmm_mirror_ops[type];
+    r = hmm_mirror_register(&amn->mirror, mm);
       if (r)
           goto free_amn;
@@ -432,7 +417,7 @@ struct amdgpu_mn *amdgpu_mn_get(struct 
 amdgpu_device *adev,
    * @bo: amdgpu buffer object
    * @addr: userptr addr we should monitor
    *
- * Registers an MMU notifier for the given BO at the specified address.
+ * Registers an HMM mirror for the given BO at the specified address.
    * Returns 0 on success, -ERRNO if anything goes wrong.
    */

  int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +1 -2
@@ -488,11 +473,11 @@ int amdgpu_mn_register(struct amdgpu_bo *bo, 
 unsigned long addr)
   }
   /**
- * amdgpu_mn_unregister - unregister a BO for notifier updates
+ * amdgpu_mn_unregister - unregister a BO for HMM mirror updates
    *
    * @bo: amdgpu buffer object
    *
- * Remove any registration of MMU notifier updates from the buffer 

object.
+ * Remove any registration of HMM mirror updates from the buffer object. */
  void amdgpu_mn_unregister(struct amdgpu_bo *bo)
  {

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h +0 -0
index eb0f432f78fe..0a51fd00021c 100644
Quoted 9 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h enum amdgpu_mn_type { · line 39

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h +1 -1
@@ -34,7 +34,7 @@ enum amdgpu_mn_type {
       AMDGPU_MN_TYPE_HSA,
   };
-#if defined(CONFIG_MMU_NOTIFIER)
+#if defined(CONFIG_HMM_MIRROR)
   void amdgpu_mn_lock(struct amdgpu_mn *mn);
   void amdgpu_mn_unlock(struct amdgpu_mn *mn);
   struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev,

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): ee46d8e7-cf64-eb8c-cfd6-793db28c6c69-5C7GfCeVMHo@public.gmane.org

#8 lore raw

Re: [PATCH 1/3] drm/amdgpu: use HMM mirror callback to replace mmu notifier v6

Am 04.02.19 um 18:17 schrieb Yang, Philip:

Quoted 97 lines · Christian König
Quoted 83 lines

Am 04.02.19 um 16:06 schrieb Yang, Philip:

Quoted 82 lines

Replace our MMU notifier with hmm_mirror_ops.sync_cpu_device_pagetables callback. Enable CONFIG_HMM and CONFIG_HMM_MIRROR as a dependency in DRM_AMDGPU_USERPTR Kconfig.

It supports both KFD userptr and gfx userptr paths.

The depdent HMM patchset from Jérôme Glisse are all merged into 4.20.0 kernel now.

Change-Id Ie62c3c5e3c5b8521ab3b438d1eff2aa2a003835e
---
   drivers/gpu/drm/amd/amdgpu/Kconfig     |   6 +-
   drivers/gpu/drm/amd/amdgpu/Makefile    |   2 +-
   drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c | 139 +++++++++++--------------
   drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h |   2 +-
   4 files changed, 67 insertions(+), 82 deletions(-)
drivers/gpu/drm/amd/amdgpu/Kconfig +3 -3
index 9221e5489069..960a63355705 100644
@@ -26,10 +26,10 @@ config DRM_AMDGPU_CIK
   config DRM_AMDGPU_USERPTR
       bool "Always enable userptr write support"
       depends on DRM_AMDGPU
-    select MMU_NOTIFIER
+    select HMM_MIRROR
       help
-      This option selects CONFIG_MMU_NOTIFIER if it isn't already
-      selected to enabled full userptr support.
+      This option selects CONFIG_HMM and CONFIG_HMM_MIRROR if it
+      isn't already selected to enabled full userptr support.
   config DRM_AMDGPU_GART_DEBUGFS
       bool "Allow GART access through debugfs"
drivers/gpu/drm/amd/amdgpu/Makefile +1 -1
index 466da5954a68..851001ced5e8 100644
@@ -172,7 +172,7 @@ endif
   amdgpu-$(CONFIG_COMPAT) += amdgpu_ioc32.o
   amdgpu-$(CONFIG_VGA_SWITCHEROO) += amdgpu_atpx_handler.o
   amdgpu-$(CONFIG_ACPI) += amdgpu_acpi.o
-amdgpu-$(CONFIG_MMU_NOTIFIER) += amdgpu_mn.o
+amdgpu-$(CONFIG_HMM_MIRROR) += amdgpu_mn.o
   include $(FULL_AMD_PATH)/powerplay/Makefile
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +2 -3
index 3e6823fdd939..5d518d2bb9be 100644
@@ -45,7 +45,7 @@
   #include <linux/firmware.h>
   #include <linux/module.h>
-#include <linux/mmu_notifier.h>
+#include <linux/hmm.h>
   #include <linux/interval_tree.h>
   #include <drm/drmP.h>
   #include <drm/drm.h>
@@ -58,7 +58,6 @@
    *
    * @adev: amdgpu device pointer
    * @mm: process address space
- * @mn: MMU notifier structure
    * @type: type of MMU notifier
    * @work: destruction work item
    * @node: hash table node to find structure by adev and mn
@@ -66,6 +65,7 @@
    * @objects: interval tree containing amdgpu_mn_nodes
    * @read_lock: mutex for recursive locking of @lock
    * @recursion: depth of recursion
+ * @mirror: HMM mirror function support
    *
    * Data for each amdgpu device and process address space.
    */
@@ -73,7 +73,6 @@ struct amdgpu_mn {
       /* constant after initialisation */
       struct amdgpu_device    *adev;
       struct mm_struct    *mm;
-    struct mmu_notifier    mn;
Quoted 9 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c struct amdgpu_mn { · line 88

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +0 -0
       enum amdgpu_mn_type    type;
       /* only used on destruction */
@@ -87,6 +86,9 @@ struct amdgpu_mn {
       struct rb_root_cached    objects;
       struct mutex        read_lock;
       atomic_t        recursion;

With HMM we don't need this any more. Please remove it and simplify amdgpu_mn_read_lock() and amdgpu_mn_read_unlock().

Thanks, this makes sense because HMM uses hmm->mirror_sem to serialize invalidate range.

Quoted 6 lines

amn->read_lock is also not needed anymore, this was used to protect atomic operations for atomic_inc_return(amn->recursion) and down_read(amn->lock).

Just one amn->lock is needed to sync amdgpu_cs_submit and userptr update. I will submit another patch.

Sounds good. I will take a closer look at the other patches tomorrow.

Thanks,
Christian.

Quoted 274 lines

Philip

Quoted 271 lines

Apart from that looks good to me,
Christian.

Quoted 268 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +11 -10
+
+    /* HMM mirror */
+    struct hmm_mirror    mirror;
   };
   /**
@@ -103,7 +105,7 @@ struct amdgpu_mn_node {
   };
   /**
- * amdgpu_mn_destroy - destroy the MMU notifier
+ * amdgpu_mn_destroy - destroy the HMM mirror
    *
    * @work: previously sheduled work item
    *
@@ -129,28 +131,26 @@ static void amdgpu_mn_destroy(struct work_struct
 *work)
       }
       up_write(&amn->lock);
       mutex_unlock(&adev->mn_lock);
-    mmu_notifier_unregister_no_release(&amn->mn, amn->mm);
+
+    hmm_mirror_unregister(&amn->mirror);
       kfree(amn);
   }
   /**
- * amdgpu_mn_release - callback to notify about mm destruction
+ * amdgpu_hmm_mirror_release - callback to notify about mm destruction
    *
- * @mn: our notifier
- * @mm: the mm this callback is about
+ * @mirror: the HMM mirror (mm) this callback is about
    *
- * Shedule a work item to lazy destroy our notifier.
+ * Shedule a work item to lazy destroy HMM mirror.
    */
-static void amdgpu_mn_release(struct mmu_notifier *mn,
-                  struct mm_struct *mm)
+static void amdgpu_hmm_mirror_release(struct hmm_mirror *mirror)
   {
-    struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn, mn);
+    struct amdgpu_mn *amn = container_of(mirror, struct amdgpu_mn,
 mirror);
       INIT_WORK(&amn->work, amdgpu_mn_destroy);
       schedule_work(&amn->work);
   }
-
   /**
    * amdgpu_mn_lock - take the write side lock for this notifier
    *
@@ -237,141 +237,126 @@ static void amdgpu_mn_invalidate_node(struct
 amdgpu_mn_node *node,
   /**
    * amdgpu_mn_invalidate_range_start_gfx - callback to notify about

mm change
   *
- * @mn: our notifier
- * @range: mmu notifier context
+ * @mirror: the hmm_mirror (mm) is about to update
+ * @update: the update start, end address
   *
   * Block for operations on BOs to finish and mark pages as accessed and
   * potentially dirty.
   */

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +3 -3
-static int amdgpu_mn_invalidate_range_start_gfx(struct mmu_notifier *mn,
-            const struct mmu_notifier_range *range)
+static int amdgpu_mn_sync_pagetables_gfx(struct hmm_mirror *mirror,
+            const struct hmm_update *update)
   {
-    struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn, mn);
+    struct amdgpu_mn *amn = container_of(mirror, struct amdgpu_mn,

mirror);
+    unsigned long start = update->start;
+    unsigned long end = update->end;
+    bool blockable = update->blockable;
      struct interval_tree_node *it;
-    unsigned long end;
      /* notification is exclusive, but interval is inclusive */
-    end = range->end - 1;
+    end -= 1;
      /* TODO we should be able to split locking for interval tree and
       * amdgpu_mn_invalidate_node
       */
-    if (amdgpu_mn_read_lock(amn, range->blockable))
+    if (amdgpu_mn_read_lock(amn, blockable))
          return -EAGAIN;
-    it = interval_tree_iter_first(&amn->objects, range->start, end); +    it = interval_tree_iter_first(&amn->objects, start, end);
      while (it) {
          struct amdgpu_mn_node *node;
-        if (!range->blockable) {
+        if (!blockable) {
              amdgpu_mn_read_unlock(amn);
              return -EAGAIN;
          }
          node = container_of(it, struct amdgpu_mn_node, it);
-        it = interval_tree_iter_next(it, range->start, end); +        it = interval_tree_iter_next(it, start, end);
-        amdgpu_mn_invalidate_node(node, range->start, end); +        amdgpu_mn_invalidate_node(node, start, end);
      }
+    amdgpu_mn_read_unlock(amn);
+
      return 0;
  }
  /**
   * amdgpu_mn_invalidate_range_start_hsa - callback to notify about
mm change
   *
- * @mn: our notifier
- * @mm: the mm this callback is about
- * @start: start of updated range
- * @end: end of updated range
+ * @mirror: the hmm_mirror (mm) is about to update
+ * @update: the update start, end address
   *
   * We temporarily evict all BOs between start and end. This
   * necessitates evicting all user-mode queues of the process. The BOs
   * are restorted in amdgpu_mn_invalidate_range_end_hsa. */

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +3 -3
-static int amdgpu_mn_invalidate_range_start_hsa(struct mmu_notifier *mn,
-            const struct mmu_notifier_range *range)
+static int amdgpu_mn_sync_pagetables_hsa(struct hmm_mirror *mirror,
+            const struct hmm_update *update)
   {
-    struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn, mn);
+    struct amdgpu_mn *amn = container_of(mirror, struct amdgpu_mn,

mirror);
+    unsigned long start = update->start;
+    unsigned long end = update->end;
+    bool blockable = update->blockable;
      struct interval_tree_node *it;
-    unsigned long end;
      /* notification is exclusive, but interval is inclusive */
-    end = range->end - 1;
+    end -= 1;
-    if (amdgpu_mn_read_lock(amn, range->blockable))
+    if (amdgpu_mn_read_lock(amn, blockable))
          return -EAGAIN;
-    it = interval_tree_iter_first(&amn->objects, range->start, end); +    it = interval_tree_iter_first(&amn->objects, start, end);
      while (it) {
          struct amdgpu_mn_node *node;
          struct amdgpu_bo *bo;
-        if (!range->blockable) {
+        if (!blockable) {
              amdgpu_mn_read_unlock(amn);
              return -EAGAIN;
          }
          node = container_of(it, struct amdgpu_mn_node, it);
-        it = interval_tree_iter_next(it, range->start, end); +        it = interval_tree_iter_next(it, start, end);
          list_for_each_entry(bo, &node->bos, mn_list) { struct kgd_mem *mem = bo->kfd_bo;
              if (amdgpu_ttm_tt_affect_userptr(bo->tbo.ttm,
-                             range->start,
-                             end))
-                amdgpu_amdkfd_evict_userptr(mem, range->mm); +                             start, end))
+                amdgpu_amdkfd_evict_userptr(mem, amn->mm);
          }
      }
+    amdgpu_mn_read_unlock(amn);

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +15 -34
+
       return 0;
   }
-/**
- * amdgpu_mn_invalidate_range_end - callback to notify about mm change
- *
- * @mn: our notifier
- * @mm: the mm this callback is about
- * @start: start of updated range
- * @end: end of updated range
- *
- * Release the lock again to allow new command submissions.
+/* Low bits of any reasonable mm pointer will be unused due to struct
+ * alignment. Use these bits to make a unique key from the mm pointer
+ * and notifier type.
    */
-static void amdgpu_mn_invalidate_range_end(struct mmu_notifier *mn,
-            const struct mmu_notifier_range *range)
-{
-    struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn, mn);
-
-    amdgpu_mn_read_unlock(amn);
-}
+#define AMDGPU_MN_KEY(mm, type) ((unsigned long)(mm) + (type))
-static const struct mmu_notifier_ops amdgpu_mn_ops[] = {
+static struct hmm_mirror_ops amdgpu_hmm_mirror_ops[] = {
       [AMDGPU_MN_TYPE_GFX] = {
-        .release = amdgpu_mn_release,
-        .invalidate_range_start = amdgpu_mn_invalidate_range_start_gfx,
-        .invalidate_range_end = amdgpu_mn_invalidate_range_end,
+        .sync_cpu_device_pagetables = amdgpu_mn_sync_pagetables_gfx,
+        .release = amdgpu_hmm_mirror_release
       },
       [AMDGPU_MN_TYPE_HSA] = {
-        .release = amdgpu_mn_release,
-        .invalidate_range_start = amdgpu_mn_invalidate_range_start_hsa,
-        .invalidate_range_end = amdgpu_mn_invalidate_range_end,
+        .sync_cpu_device_pagetables = amdgpu_mn_sync_pagetables_hsa,
+        .release = amdgpu_hmm_mirror_release
       },
   };
-/* Low bits of any reasonable mm pointer will be unused due to struct
- * alignment. Use these bits to make a unique key from the mm pointer
- * and notifier type.
- */
-#define AMDGPU_MN_KEY(mm, type) ((unsigned long)(mm) + (type))
-
   /**
- * amdgpu_mn_get - create notifier context
+ * amdgpu_mn_get - create HMM mirror context
    *
    * @adev: amdgpu device pointer
    * @type: type of MMU notifier context
    *
- * Creates a notifier context for current->mm.
+ * Creates a HMM mirror context for current->mm.
    */
   struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev,
                   enum amdgpu_mn_type type)
@@ -401,12 +386,12 @@ struct amdgpu_mn *amdgpu_mn_get(struct
 amdgpu_device *adev,
       amn->mm = mm;
       init_rwsem(&amn->lock);
       amn->type = type;
-    amn->mn.ops = &amdgpu_mn_ops[type];
       amn->objects = RB_ROOT_CACHED;
       mutex_init(&amn->read_lock);
       atomic_set(&amn->recursion, 0);
-    r = __mmu_notifier_register(&amn->mn, mm);
+    amn->mirror.ops = &amdgpu_hmm_mirror_ops[type];
+    r = hmm_mirror_register(&amn->mirror, mm);
       if (r)
           goto free_amn;
@@ -432,7 +417,7 @@ struct amdgpu_mn *amdgpu_mn_get(struct
 amdgpu_device *adev,
    * @bo: amdgpu buffer object
    * @addr: userptr addr we should monitor
    *
- * Registers an MMU notifier for the given BO at the specified address.
+ * Registers an HMM mirror for the given BO at the specified address.
    * Returns 0 on success, -ERRNO if anything goes wrong.
    */

  int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +1 -2
@@ -488,11 +473,11 @@ int amdgpu_mn_register(struct amdgpu_bo *bo,
 unsigned long addr)
   }
   /**
- * amdgpu_mn_unregister - unregister a BO for notifier updates
+ * amdgpu_mn_unregister - unregister a BO for HMM mirror updates
    *
    * @bo: amdgpu buffer object
    *
- * Remove any registration of MMU notifier updates from the buffer

object.
+ * Remove any registration of HMM mirror updates from the buffer object. */
  void amdgpu_mn_unregister(struct amdgpu_bo *bo)
  {

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h +0 -1
index eb0f432f78fe..0a51fd00021c 100644
@@ -34,7 +34,7 @@ enum amdgpu_mn_type {
       AMDGPU_MN_TYPE_HSA,
   };
-#if defined(CONFIG_MMU_NOTIFIER)
Quoted 8 lines
Quoted 4 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h enum amdgpu_mn_type { · line 40

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h +1 -0
+#if defined(CONFIG_HMM_MIRROR)
   void amdgpu_mn_lock(struct amdgpu_mn *mn);
   void amdgpu_mn_unlock(struct amdgpu_mn *mn);
   struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev,

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#9 lore raw

PATCH 2/3 drm/amdkfd: avoid HMM change cause circular lock dependency v2

In reply to a message not in the archive

There is circular lock between gfx and kfd path with HMM change:
lock(dqm) -> bo::reserve -> amdgpu_mn_lock

To avoid this, move init/unint_mqd() out of lock(dqm), to remove nested locking between mmap_sem and bo::reserve. The locking order
is: bo::reserve -> amdgpu_mn_lock(p->mn)

Change-Id I2ec09a47571f6b4c8eaef93f22c0a600f5f70153
---
 .../drm/amd/amdkfd/kfd_device_queue_manager.c | 32 ++++++++++---------
 1 file changed, 17 insertions(+), 15 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +17 -15
index 8372556b52eb..efe0d3c0215b 100644
@@ -1156,21 +1156,17 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
 	int retval;
 	struct mqd_manager *mqd_mgr;
 
-	retval = 0;
-
-	dqm_lock(dqm);
-
 	if (dqm->total_queue_count >= max_num_of_queues_per_device) {
 		pr_warn("Can't create new usermode queue because %d queues were already created\n",
 				dqm->total_queue_count);
 		retval = -EPERM;
-		goto out_unlock;
+		goto out;
 	}
 
 	if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
 		retval = allocate_sdma_queue(dqm, &q->sdma_id);
 		if (retval)
-			goto out_unlock;
+			goto out;
 		q->properties.sdma_queue_id =
 			q->sdma_id / get_num_sdma_engines(dqm);
 		q->properties.sdma_engine_id =
@@ -1181,6 +1177,9 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
 	if (retval)
 		goto out_deallocate_sdma_queue;
 
+	/* Do init_mqd before dqm_lock(dqm) to avoid circular locking order:
+	 * lock(dqm) -> bo::reserve
+	 */
 	mqd_mgr = dqm->ops.get_mqd_manager(dqm,
 			get_mqd_type_from_queue_type(q->properties.type));
 
@@ -1188,6 +1187,7 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
 		retval = -ENOMEM;
 		goto out_deallocate_doorbell;
 	}
+
 	/*
 	 * Eviction state logic: we only mark active queues as evicted
 	 * to avoid the overhead of restoring inactive queues later
@@ -1196,9 +1196,7 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
 		q->properties.is_evicted = (q->properties.queue_size > 0 &&
 					    q->properties.queue_percent > 0 &&
 					    q->properties.queue_address != 0);
-
 	dqm->asic_ops.init_sdma_vm(dqm, q, qpd);
-
 	q->properties.tba_addr = qpd->tba_addr;
 	q->properties.tma_addr = qpd->tma_addr;
 	retval = mqd_mgr->init_mqd(mqd_mgr, &q->mqd, &q->mqd_mem_obj,
@@ -1206,6 +1204,8 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
 	if (retval)
 		goto out_deallocate_doorbell;
 
+	dqm_lock(dqm);
+
 	list_add(&q->list, &qpd->queues_list);
 	qpd->queue_count++;
 	if (q->properties.is_active) {
@@ -1233,9 +1233,7 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
 out_deallocate_sdma_queue:
 	if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
 		deallocate_sdma_queue(dqm, q->sdma_id);
-out_unlock:
-	dqm_unlock(dqm);
-
+out:
 	return retval;
 }
 
@@ -1398,8 +1396,6 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm,
 			qpd->reset_wavefronts = true;
 	}
 
-	mqd_mgr->uninit_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
-
 	/*
 	 * Unconditionally decrement this counter, regardless of the queue's
 	 * type
@@ -1410,6 +1406,9 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm,
 
 	dqm_unlock(dqm);
 
+	/* Do uninit_mqd after dqm_unlock(dqm) to avoid circular locking */
+	mqd_mgr->uninit_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
+
 	return retval;
 
 failed:
@@ -1631,7 +1630,11 @@ static int process_termination_cpsch(struct device_queue_manager *dqm,
 		qpd->reset_wavefronts = false;
 	}
 
-	/* lastly, free mqd resources */
+	dqm_unlock(dqm);
+
+	/* Lastly, free mqd resources.
+	 * Do uninit_mqd() after dqm_unlock to avoid circular locking.
+	 */
 	list_for_each_entry_safe(q, next, &qpd->queues_list, list) {
 		mqd_mgr = dqm->ops.get_mqd_manager(dqm,
 			get_mqd_type_from_queue_type(q->properties.type));
@@ -1645,7 +1648,6 @@ static int process_termination_cpsch(struct device_queue_manager *dqm,
 	}
 
 out:
-	dqm_unlock(dqm);
 	return retval;
 }
 
Signature · 6 lines
2.17.1

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
#10 lore raw

PATCH 3/3 drm/amdgpu: replace get_user_pages with HMM address mirror helpers v6

In reply to a message not in the archive

Use HMM helper function hmm_vma_fault() to get physical pages backing userptr and start CPU page table update track of those pages. Then use hmm_vma_range_done() to check if those pages are updated before amdgpu_cs_submit for gfx or before user queues are resumed for kfd.

If userptr pages are updated, for gfx, amdgpu_cs_ioctl will restart from scratch, for kfd, restore worker is rescheduled to retry.

HMM simplify the CPU page table concurrent update check, so remove guptasklock, mmu_invalidations, last_set_pages fields from amdgpu_ttm_tt struct.

HMM does not pin the page (increase page ref count), so remove related operations like release_pages(), put_page(), mark_page_dirty().

Change-Id I2e8c0c6f0d8c21e5596a32d7fc91762778bc9e67
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h    |   1 -
 .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c  |  95 +++-------
 drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.h   |   2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c        | 158 +++++++---------
 drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c       |  14 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c        |  25 ++-
 drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h        |   4 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c       | 173 ++++++++----------
 drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h       |   3 +-
 9 files changed, 198 insertions(+), 277 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +0 -1
index 0b31a1859023..0e1711a75b68 100644
@@ -61,7 +61,6 @@ struct kgd_mem {
 
 	atomic_t invalid;
 	struct amdkfd_process_info *process_info;
-	struct page **user_pages;
 
 	struct amdgpu_sync sync;
 
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +25 -70

178 lines not on this page · raw message

drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.h +1 -1
index 7c5f5d1601e6..a130e766cbdb 100644
@@ -36,7 +36,7 @@ struct amdgpu_bo_list_entry {
 	struct amdgpu_bo_va		*bo_va;
 	uint32_t			priority;
 	struct page			**user_pages;
-	int				user_invalidated;
+	bool				user_invalidated;
 };
 
 struct amdgpu_bo_list {
drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +66 -92

233 lines not on this page · raw message

drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +6 -8
index d21dd2f369da..555285e329ed 100644
@@ -329,26 +329,24 @@ int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
 
 		r = amdgpu_bo_reserve(bo, true);
 		if (r)
-			goto free_pages;
+			goto user_pages_done;
 
 		amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
 		r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 		amdgpu_bo_unreserve(bo);
 		if (r)
-			goto free_pages;
+			goto user_pages_done;
 	}
 
 	r = drm_gem_handle_create(filp, gobj, &handle);
-	/* drop reference from allocate - handle holds it now */
-	drm_gem_object_put_unlocked(gobj);
 	if (r)
-		return r;
+		goto user_pages_done;
 
 	args->handle = handle;
-	return 0;
 
-free_pages:
-	release_pages(bo->tbo.ttm->pages, bo->tbo.ttm->num_pages);
+user_pages_done:
+	if (args->flags & AMDGPU_GEM_USERPTR_VALIDATE)
+		amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm);
 
 release_object:
 	drm_gem_object_put_unlocked(gobj);
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +23 -2
index 5d518d2bb9be..9d40f3001f3c 100644
@@ -229,8 +229,6 @@ static void amdgpu_mn_invalidate_node(struct amdgpu_mn_node *node,
 			true, false, MAX_SCHEDULE_TIMEOUT);
 		if (r <= 0)
 			DRM_ERROR("(%ld) failed to wait for user bo\n", r);
-
-		amdgpu_ttm_tt_mark_user_pages(bo->tbo.ttm);
 	}
 }
 
@@ -513,3 +511,26 @@ void amdgpu_mn_unregister(struct amdgpu_bo *bo)
 	mutex_unlock(&adev->mn_lock);
 }
 
+/* flags used by HMM internal, not related to CPU/GPU PTE flags */
+static const uint64_t hmm_range_flags[HMM_PFN_FLAG_MAX] = {
+		(1 << 0), /* HMM_PFN_VALID */
+		(1 << 1), /* HMM_PFN_WRITE */
+		0 /* HMM_PFN_DEVICE_PRIVATE */
+};
+
+static const uint64_t hmm_range_values[HMM_PFN_VALUE_MAX] = {
+		0xfffffffffffffffeUL, /* HMM_PFN_ERROR */
+		0, /* HMM_PFN_NONE */
+		0xfffffffffffffffcUL /* HMM_PFN_SPECIAL */
+};
+
+void amdgpu_hmm_init_range(struct hmm_range *range)
+{
+	if (range) {
+		range->flags = hmm_range_flags;
+		range->values = hmm_range_values;
+		range->pfn_shift = PAGE_SHIFT;
+		range->pfns = NULL;
+		INIT_LIST_HEAD(&range->list);
+	}
+}
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h +3 -1
index 0a51fd00021c..4803e216e174 100644
@@ -25,9 +25,10 @@
 #define __AMDGPU_MN_H__
 
 /*
- * MMU Notifier
+ * HMM mirror
  */
 struct amdgpu_mn;
+struct hmm_range;
 
 enum amdgpu_mn_type {
 	AMDGPU_MN_TYPE_GFX,
@@ -41,6 +42,7 @@ struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev,
 				enum amdgpu_mn_type type);
 int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr);
 void amdgpu_mn_unregister(struct amdgpu_bo *bo);
+void amdgpu_hmm_init_range(struct hmm_range *range);
 #else
 static inline void amdgpu_mn_lock(struct amdgpu_mn *mn) {}
 static inline void amdgpu_mn_unlock(struct amdgpu_mn *mn) {}
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +72 -101

263 lines not on this page · raw message

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h +2 -1
index b5b2d101f7db..8988c87fff9d 100644
@@ -102,6 +102,7 @@ int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo);
 int amdgpu_ttm_recover_gart(struct ttm_buffer_object *tbo);
 
 int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages);
+bool amdgpu_ttm_tt_get_user_pages_done(struct ttm_tt *ttm);
 void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page **pages);
 void amdgpu_ttm_tt_mark_user_pages(struct ttm_tt *ttm);
 int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,
@@ -112,7 +113,7 @@ bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,
 				  unsigned long end);
 bool amdgpu_ttm_tt_userptr_invalidated(struct ttm_tt *ttm,
 				       int *last_invalidated);
-bool amdgpu_ttm_tt_userptr_needs_pages(struct ttm_tt *ttm);
+bool amdgpu_ttm_tt_is_userptr(struct ttm_tt *ttm);
 bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm);
 uint64_t amdgpu_ttm_tt_pde_flags(struct ttm_tt *ttm, struct ttm_mem_reg *mem);
 uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,
Signature · 6 lines
2.17.1

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx