Use HMM to replace get_user_pages
Subsystems (3)
cover Use HMM to replace get_user_pages
Hi Christian,
This patch is rebased to lastest HMM. Please review the GEM and CS part changes in patch 3/3.
Regards,
Philip Yang (3):
drm/amdgpu: use HMM mirror callback to replace mmu notifier v6 drm/amdkfd: avoid HMM change cause circular lock dependency v2 drm/amdgpu: replace get_user_pages with HMM address mirror helpers v6
drivers/gpu/drm/amd/amdgpu/Kconfig | 6 +- drivers/gpu/drm/amd/amdgpu/Makefile | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 1 - .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 95 +++------- drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.h | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c | 158 +++++++--------- drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c | 14 +- drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c | 164 +++++++++-------- drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h | 6 +- drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 173 ++++++++---------- drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 3 +- .../drm/amd/amdkfd/kfd_device_queue_manager.c | 32 ++-- 12 files changed, 282 insertions(+), 374 deletions(-)
Signature · 6 lines
2.17.1 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): [email protected]
PATCH 1/3 drm/amdgpu: use HMM mirror callback to replace mmu notifier v6
Replace our MMU notifier with hmm_mirror_ops.sync_cpu_device_pagetables callback. Enable CONFIG_HMM and CONFIG_HMM_MIRROR as a dependency in DRM_AMDGPU_USERPTR Kconfig.
It supports both KFD userptr and gfx userptr paths.
The depdent HMM patchset from Jérôme Glisse are all merged into 4.20.0 kernel now.
--- drivers/gpu/drm/amd/amdgpu/Kconfig | 6 +- drivers/gpu/drm/amd/amdgpu/Makefile | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c | 139 +++++++++++-------------- drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h | 2 +- 4 files changed, 67 insertions(+), 82 deletions(-)
drivers/gpu/drm/amd/amdgpu/Kconfig +3 -3
@@ -26,10 +26,10 @@ config DRM_AMDGPU_CIK config DRM_AMDGPU_USERPTR bool "Always enable userptr write support" depends on DRM_AMDGPU - select MMU_NOTIFIER + select HMM_MIRROR help - This option selects CONFIG_MMU_NOTIFIER if it isn't already - selected to enabled full userptr support. + This option selects CONFIG_HMM and CONFIG_HMM_MIRROR if it + isn't already selected to enabled full userptr support. config DRM_AMDGPU_GART_DEBUGFS bool "Allow GART access through debugfs"
drivers/gpu/drm/amd/amdgpu/Makefile +1 -1
@@ -172,7 +172,7 @@ endif amdgpu-$(CONFIG_COMPAT) += amdgpu_ioc32.o amdgpu-$(CONFIG_VGA_SWITCHEROO) += amdgpu_atpx_handler.o amdgpu-$(CONFIG_ACPI) += amdgpu_acpi.o -amdgpu-$(CONFIG_MMU_NOTIFIER) += amdgpu_mn.o +amdgpu-$(CONFIG_HMM_MIRROR) += amdgpu_mn.o include $(FULL_AMD_PATH)/powerplay/Makefile
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +62 -77
301 lines not on this page · raw message
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h +1 -1
@@ -34,7 +34,7 @@ enum amdgpu_mn_type { AMDGPU_MN_TYPE_HSA, }; -#if defined(CONFIG_MMU_NOTIFIER ) +#if defined(CONFIG_HMM_MIRROR )void amdgpu_mn_lock (struct amdgpu_mn *mn);void amdgpu_mn_unlock (struct amdgpu_mn *mn); struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev,
Signature · 6 lines
2.17.1 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): [email protected]
Re: [PATCH 1/3] drm/amdgpu: use HMM mirror callback to replace mmu notifier v6
Am 04.02.19 um 16:06 schrieb Yang, Philip:
Quoted 87 lines
Replace our MMU notifier with hmm_mirror_ops.sync_cpu_device_pagetables callback. Enable
CONFIG_HMMandCONFIG_HMM_MIRRORas a dependency inDRM_AMDGPU_USERPTRKconfig.It supports both KFD userptr and gfx userptr paths.
The depdent HMM patchset from Jérôme Glisse are all merged into 4.20.0 kernel now.
Change-Id Ie62c3c5e3c5b8521ab3b438d1eff2aa2a003835eSigned-off-by Philip Yang <[email protected]>--- drivers/gpu/drm/amd/amdgpu/Kconfig | 6 +- drivers/gpu/drm/amd/amdgpu/Makefile | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c | 139 +++++++++++-------------- drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h | 2 +- 4 files changed, 67 insertions(+), 82 deletions(-)drivers/gpu/drm/amd/amdgpu/Kconfig +3
-3@@ -26,10 +26,10 @@ config DRM_AMDGPU_CIK config DRM_AMDGPU_USERPTR bool "Always enable userptr write support" depends on DRM_AMDGPU - select MMU_NOTIFIER + select HMM_MIRROR help - This option selects CONFIG_MMU_NOTIFIER if it isn't already - selected to enabled full userptr support. + This option selects CONFIG_HMM and CONFIG_HMM_MIRROR if it + isn't already selected to enabled full userptr support. config DRM_AMDGPU_GART_DEBUGFS bool "Allow GART access through debugfs"drivers/gpu/drm/amd/amdgpu/Makefile +1
-1@@ -172,7 +172,7 @@ endif amdgpu-$(CONFIG_COMPAT) += amdgpu_ioc32.o amdgpu-$(CONFIG_VGA_SWITCHEROO) += amdgpu_atpx_handler.o amdgpu-$(CONFIG_ACPI) += amdgpu_acpi.o -amdgpu-$(CONFIG_MMU_NOTIFIER) += amdgpu_mn.o +amdgpu-$(CONFIG_HMM_MIRROR) += amdgpu_mn.o include $(FULL_AMD_PATH)/powerplay/Makefiledrivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +2
-3@@ -45,7 +45,7 @@#include <linux/firmware.h> #include <linux/module.h> -#include <linux/mmu_notifier.h> +#include <linux/hmm.h> #include <linux/interval_tree.h> #include <drm/drmP.h> #include <drm/drm.h> @@ -58,7 +58,6 @@* * @adev: amdgpu device pointer * @mm: process address space -* @mn: MMU notifier structure * @type: type of MMU notifier * @work: destruction work item * @node: hash table node to find structure by adev and mn @@ -66,6 +65,7 @@* @objects: interval tree containing amdgpu_mn_nodes * @read_lock: mutex for recursive locking of @lock * @recursion: depth of recursion +* @mirror: HMM mirror function support * * Data for each amdgpu device and process address space. */ @@ -73,7 +73,6 @@ struct amdgpu_mn {/* constant after initialisation */ struct amdgpu_device *adev;struct mm_struct *mm; -struct mmu_notifier mn;enum amdgpu_mn_type type;/* only used on destruction */
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c struct amdgpu_mn { · line 89
Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +0
-0@@ -87,6 +86,9 @@ struct amdgpu_mn {struct rb_root_cached objects;struct mutex read_lock;atomic_t recursion;
With HMM we don't need this any more. Please remove it and simplify amdgpu_mn_read_lock() and amdgpu_mn_read_unlock().
Apart from that looks good to me,
Christian.
Quoted 280 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +3
-0+ +/* HMM mirror */ +struct hmm_mirror mirror; };/**
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +57
-74@@ -103,7 +105,7 @@ struct amdgpu_mn_node { };/** -* amdgpu_mn_destroy - destroy the MMU notifier +* amdgpu_mn_destroy - destroy the HMM mirror * * @work: previously sheduled work item * @@ -129,28 +131,26 @@ static void amdgpu_mn_destroy(struct work_struct *work) }up_write (&amn->lock);mutex_unlock (&adev->mn_lock); -mmu_notifier_unregister_no_release (&amn->mn, amn->mm); + +hmm_mirror_unregister (&amn->mirror);kfree (amn); }/** -* amdgpu_mn_release - callback to notify about mm destruction +* amdgpu_hmm_mirror_release - callback to notify about mm destruction * -* @mn: our notifier -* @mm: the mm this callback is about +* @mirror: the HMM mirror (mm) this callback is about * -* Shedule a work item to lazy destroy our notifier. +* Shedule a work item to lazy destroy HMM mirror. */ -static void amdgpu_mn_release (struct mmu_notifier *mn, -struct mm_struct *mm) +static void amdgpu_hmm_mirror_release (struct hmm_mirror *mirror) { - struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn, mn); + struct amdgpu_mn *amn = container_of(mirror, struct amdgpu_mn, mirror);INIT_WORK (&amn->work, amdgpu_mn_destroy);schedule_work (&amn->work); } -/** * amdgpu_mn_lock - take the write side lock for this notifier * @@ -237,141 +237,126 @@ static void amdgpu_mn_invalidate_node(struct amdgpu_mn_node *node,/** * amdgpu_mn_invalidate_range_start_gfx - callback to notify about mm change * -* @mn: our notifier -* @range: mmu notifier context +* @mirror: the hmm_mirror (mm) is about to update +* @update: the update start, end address * * Block for operations on BOs to finish and mark pages as accessed and * potentially dirty. */ -static int amdgpu_mn_invalidate_range_start_gfx (struct mmu_notifier *mn, -const struct mmu_notifier_range *range) +static int amdgpu_mn_sync_pagetables_gfx (struct hmm_mirror *mirror, +const struct hmm_update *update) { - struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn, mn); + struct amdgpu_mn *amn = container_of(mirror, struct amdgpu_mn, mirror); +unsigned long start = update->start; +unsigned long end = update->end; +bool blockable = update->blockable;struct interval_tree_node *it; -unsigned long end;/* notification is exclusive, but interval is inclusive */ - end = range->end -1 ; + end -=1 ;/* TODO we should be able to split locking for interval tree and * amdgpu_mn_invalidate_node */ -if (amdgpu_mn_read_lock (amn, range->blockable)) +if (amdgpu_mn_read_lock (amn, blockable))return -EAGAIN ; - it =interval_tree_iter_first (&amn->objects, range->start, end); + it =interval_tree_iter_first (&amn->objects, start, end);while (it) {struct amdgpu_mn_node *node; -if (!range->blockable) { +if (!blockable) {amdgpu_mn_read_unlock (amn);return -EAGAIN ; } node = container_of(it, struct amdgpu_mn_node, it); - it =interval_tree_iter_next (it, range->start, end); + it =interval_tree_iter_next (it, start, end); -amdgpu_mn_invalidate_node (node, range->start, end); +amdgpu_mn_invalidate_node (node, start, end); } +amdgpu_mn_read_unlock (amn); +return 0 ; }/** * amdgpu_mn_invalidate_range_start_hsa - callback to notify about mm change * -* @mn: our notifier -* @mm: the mm this callback is about -* @start: start of updated range -* @end: end of updated range +* @mirror: the hmm_mirror (mm) is about to update +* @update: the update start, end address * * We temporarily evict all BOs between start and end. This * necessitates evicting all user-mode queues of the process. The BOs * are restorted in amdgpu_mn_invalidate_range_end_hsa. */ -static int amdgpu_mn_invalidate_range_start_hsa (struct mmu_notifier *mn, -const struct mmu_notifier_range *range) +static int amdgpu_mn_sync_pagetables_hsa (struct hmm_mirror *mirror, +const struct hmm_update *update) { - struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn, mn); + struct amdgpu_mn *amn = container_of(mirror, struct amdgpu_mn, mirror); +unsigned long start = update->start; +unsigned long end = update->end; +bool blockable = update->blockable;struct interval_tree_node *it; -unsigned long end;/* notification is exclusive, but interval is inclusive */ - end = range->end -1 ; + end -=1 ; -if (amdgpu_mn_read_lock (amn, range->blockable)) +if (amdgpu_mn_read_lock (amn, blockable))return -EAGAIN ; - it =interval_tree_iter_first (&amn->objects, range->start, end); + it =interval_tree_iter_first (&amn->objects, start, end);while (it) {struct amdgpu_mn_node *node;struct amdgpu_bo *bo; -if (!range->blockable) { +if (!blockable) {amdgpu_mn_read_unlock (amn);return -EAGAIN ; } node = container_of(it, struct amdgpu_mn_node, it); - it =interval_tree_iter_next (it, range->start, end); + it =interval_tree_iter_next (it, start, end);list_for_each_entry (bo, &node->bos, mn_list) {struct kgd_mem *mem = bo->kfd_bo;if (amdgpu_ttm_tt_affect_userptr (bo->tbo.ttm, - range->start, - end)) -amdgpu_amdkfd_evict_userptr (mem, range->mm); + start, end)) +amdgpu_amdkfd_evict_userptr (mem, amn->mm); } } +amdgpu_mn_read_unlock (amn); +return 0 ; } -/** -* amdgpu_mn_invalidate_range_end - callback to notify about mm change -* -* @mn: our notifier -* @mm: the mm this callback is about -* @start: start of updated range -* @end: end of updated range -* -* Release the lock again to allow new command submissions. +/* Low bits of any reasonable mm pointer will be unused due to struct +* alignment. Use these bits to make a unique key from the mm pointer +* and notifier type. */ -static void amdgpu_mn_invalidate_range_end (struct mmu_notifier *mn, -const struct mmu_notifier_range *range) -{ - struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn, mn); - -amdgpu_mn_read_unlock (amn); -} +#define AMDGPU_MN_KEY (mm, type) ((unsigned long)(mm) + (type)) -static const struct mmu_notifier_ops amdgpu_mn_ops[] = { +static struct hmm_mirror_ops amdgpu_hmm_mirror_ops[] = { [AMDGPU_MN_TYPE_GFX ] = { - .release = amdgpu_mn_release, - .invalidate_range_start = amdgpu_mn_invalidate_range_start_gfx, - .invalidate_range_end = amdgpu_mn_invalidate_range_end, + .sync_cpu_device_pagetables = amdgpu_mn_sync_pagetables_gfx, + .release = amdgpu_hmm_mirror_release }, [AMDGPU_MN_TYPE_HSA ] = { - .release = amdgpu_mn_release, - .invalidate_range_start = amdgpu_mn_invalidate_range_start_hsa, - .invalidate_range_end = amdgpu_mn_invalidate_range_end, + .sync_cpu_device_pagetables = amdgpu_mn_sync_pagetables_hsa, + .release = amdgpu_hmm_mirror_release }, }; -/* Low bits of any reasonable mm pointer will be unused due to struct -* alignment. Use these bits to make a unique key from the mm pointer -* and notifier type. -*/ -#define AMDGPU_MN_KEY (mm, type) ((unsigned long)(mm) + (type)) -/** -* amdgpu_mn_get - create notifier context +* amdgpu_mn_get - create HMM mirror context * * @adev: amdgpu device pointer * @type: type of MMU notifier context * -* Creates a notifier context for current->mm. +* Creates a HMM mirror context for current->mm. */ struct amdgpu_mn *amdgpu_mn_get (struct amdgpu_device *adev,enum amdgpu_mn_type type)@@ -401,12 +386,12 @@ struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev, amn->mm = mm;init_rwsem (&amn->lock); amn->type = type; - amn->mn.ops = &amdgpu_mn_ops[type]; amn->objects =RB_ROOT_CACHED ;mutex_init (&amn->read_lock);atomic_set (&amn->recursion,0 ); - r =__mmu_notifier_register (&amn->mn, mm); + amn->mirror.ops = &amdgpu_hmm_mirror_ops[type]; + r =hmm_mirror_register (&amn->mirror, mm);if (r) goto free_amn;@@ -432,7 +417,7 @@ struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev,* @bo: amdgpu buffer object * @addr: userptr addr we should monitor * -* Registers an MMU notifier for the given BO at the specified address. +* Registers an HMM mirror for the given BO at the specified address. * Returns 0 on success, -ERRNO if anything goes wrong. */ int amdgpu_mn_register (struct amdgpu_bo *bo,unsigned long addr)@@ -488,11 +473,11 @@ int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr) }/** -* amdgpu_mn_unregister - unregister a BO for notifier updates +* amdgpu_mn_unregister - unregister a BO for HMM mirror updates * * @bo: amdgpu buffer object * -* Remove any registration of MMU notifier updates from the buffer object. +* Remove any registration of HMM mirror updates from the buffer object. */ void amdgpu_mn_unregister (struct amdgpu_bo *bo) {drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h +0
-0
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h enum amdgpu_mn_type { · line 40
Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h +1
-1@@ -34,7 +34,7 @@ enum amdgpu_mn_type { AMDGPU_MN_TYPE_HSA, }; -#if defined(CONFIG_MMU_NOTIFIER ) +#if defined(CONFIG_HMM_MIRROR )void amdgpu_mn_lock (struct amdgpu_mn *mn);void amdgpu_mn_unlock (struct amdgpu_mn *mn); struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev,
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): 71e6094c-a257-7aa3-854c-9a9dff163b01-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org
Re: [PATCH 1/3] drm/amdgpu: use HMM mirror callback to replace mmu notifier v6
Quoted 89 lines · Christian König
Am 04.02.19 um 16:06 schrieb Yang, Philip:
Quoted 87 lines
Replace our MMU notifier with hmm_mirror_ops.sync_cpu_device_pagetables callback. Enable
CONFIG_HMMandCONFIG_HMM_MIRRORas a dependency inDRM_AMDGPU_USERPTRKconfig.It supports both KFD userptr and gfx userptr paths.
The depdent HMM patchset from Jérôme Glisse are all merged into 4.20.0 kernel now.
Change-Id Ie62c3c5e3c5b8521ab3b438d1eff2aa2a003835eSigned-off-by Philip Yang <[email protected]>---
drivers/gpu/drm/amd/amdgpu/Kconfig | 6 +- drivers/gpu/drm/amd/amdgpu/Makefile | 2 +-drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c| 139 +++++++++++--------------drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h| 2 +-4 files changed, 67 insertions(+), 82 deletions(-)drivers/gpu/drm/amd/amdgpu/Kconfig +0
-0@@ -26,10 +26,10 @@ config DRM_AMDGPU_CIKconfig
DRM_AMDGPU_USERPTR
bool "Always enable userptr write support" depends onDRM_AMDGPU
- selectMMU_NOTIFIER
+ selectHMM_MIRROR
help
- This option selectsCONFIG_MMU_NOTIFIERif it isn't already - selected to enabled full userptr support.
+ This option selectsCONFIG_HMMandCONFIG_HMM_MIRRORif it + isn't already selected to enabled full userptr support. configDRM_AMDGPU_GART_DEBUGFS
bool "Allow GART access through debugfs"drivers/gpu/drm/amd/amdgpu/Makefile +0
-0@@ -172,7 +172,7 @@ endifamdgpu-$(CONFIG_COMPAT) += amdgpu_ioc32.o amdgpu-$(CONFIG_VGA_SWITCHEROO) += amdgpu_atpx_handler.o amdgpu-$(CONFIG_ACPI) += amdgpu_acpi.o
drivers/gpu/drm/amd/amdgpu/Makefile +1
-1-amdgpu-$(CONFIG_MMU_NOTIFIER) += amdgpu_mn.o +amdgpu-$(CONFIG_HMM_MIRROR) += amdgpu_mn.oinclude $(FULL_AMD_PATH)/powerplay/Makefile
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +2
-3@@ -45,7 +45,7 @@ #include <linux/firmware.h> #include <linux/module.h> -#include <linux/mmu_notifier.h> +#include <linux/hmm.h> #include <linux/interval_tree.h> #include <drm/drmP.h> #include <drm/drm.h>@@ -58,7 +58,6 @@ * * @adev: amdgpu device pointer * @mm: process address space - * @mn: MMU notifier structure * @type: type of MMU notifier * @work: destruction work item * @node: hash table node to find structure by adev and mn@@ -66,6 +65,7 @@* @objects: interval tree containing amdgpu_mn_nodes * @read_lock: mutex for recursive locking of @lock * @recursion: depth of recursion +* @mirror: HMM mirror function support * * Data for each amdgpu device and process address space. */ @@ -73,7 +73,6 @@ struct amdgpu_mn { /* constant after initialisation */ struct amdgpu_device *adev; struct mm_struct *mm; - struct mmu_notifier mn; enum amdgpu_mn_type type; /* only used on destruction */@@ -87,6 +86,9 @@ struct amdgpu_mn { struct rb_root_cached objects; struct mutex read_lock;
Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c struct amdgpu_mn { · line 88
Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +0
-0atomic_t recursion;With HMM we don't need this any more. Please remove it and simplify
amdgpu_mn_read_lock()andamdgpu_mn_read_unlock().
Thanks, this makes sense because HMM uses hmm->mirror_sem to serialize invalidate range.
amn->read_lock is also not needed anymore, this was used to protect atomic operations for atomic_inc_return(amn->recursion) and down_read(amn->lock).
Just one amn->lock is needed to sync amdgpu_cs_submit and userptr update. I will submit another patch.
Philip
Quoted 267 lines
Apart from that looks good to me,
Christian.Quoted 264 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +11
-10+ + /* HMM mirror */ + struct hmm_mirror mirror; }; /**@@ -103,7 +105,7 @@ struct amdgpu_mn_node { }; /** -* amdgpu_mn_destroy - destroy the MMU notifier +* amdgpu_mn_destroy - destroy the HMM mirror * * @work: previously sheduled work item * @@ -129,28 +131,26 @@ static void amdgpu_mn_destroy(struct work_struct *work) } up_write(&amn->lock); mutex_unlock(&adev->mn_lock); - mmu_notifier_unregister_no_release(&amn->mn, amn->mm); + + hmm_mirror_unregister(&amn->mirror); kfree(amn); } /** -* amdgpu_mn_release - callback to notify about mm destruction +* amdgpu_hmm_mirror_release - callback to notify about mm destruction * -* @mn: our notifier -* @mm: the mm this callback is about +* @mirror: the HMM mirror (mm) this callback is about * -* Shedule a work item to lazy destroy our notifier. +* Shedule a work item to lazy destroy HMM mirror. */ -static void amdgpu_mn_release (struct mmu_notifier *mn, - struct mm_struct *mm) +static void amdgpu_hmm_mirror_release (struct hmm_mirror *mirror) { - struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn, mn); + struct amdgpu_mn *amn = container_of(mirror, struct amdgpu_mn, mirror); INIT_WORK(&amn->work, amdgpu_mn_destroy); schedule_work(&amn->work); } - /*** amdgpu_mn_lock - take the write side lock for this notifier * @@ -237,141 +237,126 @@ static void amdgpu_mn_invalidate_node(struct amdgpu_mn_node *node, /*** amdgpu_mn_invalidate_range_start_gfx - callback to notify about mm change
*
- * @mn: our notifier
- * @range: mmu notifier context
+ * @mirror: thehmm_mirror(mm) is about to update
+ * @update: the update start, end address
*
* Block for operations on BOs to finish and mark pages as accessed and
* potentially dirty.
*/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +2
-2-static int amdgpu_mn_invalidate_range_start_gfx (struct mmu_notifier *mn, - const struct mmu_notifier_range *range) +static int amdgpu_mn_sync_pagetables_gfx (struct hmm_mirror *mirror, + const struct hmm_update *update){
- structamdgpu_mn*amn =container_of(mn, struct amdgpu_mn, mn); + structamdgpu_mn*amn =container_of(mirror, struct amdgpu_mn, mirror);
+ unsigned long start =update->start;
+ unsigned long end =update->end;
+ bool blockable =update->blockable;
structinterval_tree_node*it;
- unsigned long end;
/* notification is exclusive, but interval is inclusive */ - end =range->end- 1;
+ end -= 1;
/* TODO we should be able to split locking for interval tree and
*amdgpu_mn_invalidate_node
*/
- if (amdgpu_mn_read_lock(amn, range->blockable))
+ if (amdgpu_mn_read_lock(amn, blockable))
return-EAGAIN;
- it =interval_tree_iter_first(&amn->objects, range->start, end); + it =interval_tree_iter_first(&amn->objects, start, end); while (it) {
structamdgpu_mn_node*node;
- if (!range->blockable) {
+ if (!blockable) {
amdgpu_mn_read_unlock(amn);
return-EAGAIN;
}
node =container_of(it, struct amdgpu_mn_node, it); - it =interval_tree_iter_next(it, range->start, end); + it =interval_tree_iter_next(it, start, end);
-amdgpu_mn_invalidate_node(node, range->start, end); +amdgpu_mn_invalidate_node(node, start, end);
}
+amdgpu_mn_read_unlock(amn);
+
return 0;
}
/**
*amdgpu_mn_invalidate_range_start_hsa- callback to notify about mm change
*
- * @mn: our notifier
- * @mm: the mm this callback is about
- * @start: start of updated range
- * @end: end of updated range
+ * @mirror: thehmm_mirror(mm) is about to update
+ * @update: the update start, end address
*
* We temporarily evict all BOs between start and end. This
* necessitates evicting all user-mode queues of the process. The BOs
* are restorted inamdgpu_mn_invalidate_range_end_hsa.
*/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +2
-2-static int amdgpu_mn_invalidate_range_start_hsa (struct mmu_notifier *mn, - const struct mmu_notifier_range *range) +static int amdgpu_mn_sync_pagetables_hsa (struct hmm_mirror *mirror, + const struct hmm_update *update){
- structamdgpu_mn*amn =container_of(mn, struct amdgpu_mn, mn); + structamdgpu_mn*amn =container_of(mirror, struct amdgpu_mn, mirror);
+ unsigned long start =update->start;
+ unsigned long end =update->end;
+ bool blockable =update->blockable;
structinterval_tree_node*it;
- unsigned long end;
/* notification is exclusive, but interval is inclusive */ - end =range->end- 1;
+ end -= 1;
- if (amdgpu_mn_read_lock(amn, range->blockable))
+ if (amdgpu_mn_read_lock(amn, blockable))
return-EAGAIN;
- it =interval_tree_iter_first(&amn->objects, range->start, end); + it =interval_tree_iter_first(&amn->objects, start, end); while (it) {
structamdgpu_mn_node*node;
structamdgpu_bo*bo;
- if (!range->blockable) {
+ if (!blockable) {
amdgpu_mn_read_unlock(amn);
return-EAGAIN;
}
node =container_of(it, struct amdgpu_mn_node, it); - it =interval_tree_iter_next(it, range->start, end); + it =interval_tree_iter_next(it, start, end);list_for_each_entry(bo, &node->bos, mn_list){ structkgd_mem*mem =bo->kfd_bo;
if (amdgpu_ttm_tt_affect_userptr(bo->tbo.ttm, -range->start,
- end))
-amdgpu_amdkfd_evict_userptr(mem, range->mm); + start, end))
+amdgpu_amdkfd_evict_userptr(mem, amn->mm); }
}
+amdgpu_mn_read_unlock(amn);
+
return 0;
}drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +3
-9-/** -* amdgpu_mn_invalidate_range_end - callback to notify about mm change -* -* @mn: our notifier -* @mm: the mm this callback is about -* @start: start of updated range -* @end: end of updated range -* -* Release the lock again to allow new command submissions. +/* Low bits of any reasonable mm pointer will be unused due to struct +* alignment. Use these bits to make a unique key from the mm pointer +* and notifier type. */
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +2
-8-static void amdgpu_mn_invalidate_range_end (struct mmu_notifier *mn, - const struct mmu_notifier_range *range) -{ - struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn, mn); - - amdgpu_mn_read_unlock(amn); -} +#define AMDGPU_MN_KEY (mm, type) ((unsigned long)(mm) + (type)) -static const struct mmu_notifier_ops amdgpu_mn_ops[] = { +static struct hmm_mirror_ops amdgpu_hmm_mirror_ops[] = {[
AMDGPU_MN_TYPE_GFX] = {
- .release =amdgpu_mn_release,
- .invalidate_range_start =amdgpu_mn_invalidate_range_start_gfx, - .invalidate_range_end =amdgpu_mn_invalidate_range_end, + .sync_cpu_device_pagetables =amdgpu_mn_sync_pagetables_gfx, + .release =amdgpu_hmm_mirror_release
},
[AMDGPU_MN_TYPE_HSA] = {
- .release =amdgpu_mn_release,
- .invalidate_range_start =amdgpu_mn_invalidate_range_start_hsa, - .invalidate_range_end =amdgpu_mn_invalidate_range_end, + .sync_cpu_device_pagetables =amdgpu_mn_sync_pagetables_hsa, + .release =amdgpu_hmm_mirror_release
},
};drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +0
-6-/* Low bits of any reasonable mm pointer will be unused due to struct -* alignment. Use these bits to make a unique key from the mm pointer -* and notifier type. -*/ -#define AMDGPU_MN_KEY (mm, type) ((unsigned long)(mm) + (type)) -/**
- *amdgpu_mn_get- create notifier context
+ *amdgpu_mn_get- create HMM mirror context
*
* @adev: amdgpu device pointer
* @type: type of MMU notifier context
*
- * Creates a notifier context forcurrent->mm.
+ * Creates a HMM mirror context forcurrent->mm. */
structamdgpu_mn*amdgpu_mn_get(structamdgpu_device*adev, enumamdgpu_mn_typetype)drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +3
-3@@ -401,12 +386,12 @@ struct amdgpu_mn *amdgpu_mn_get(structamdgpu_device *adev, amn->mm = mm; init_rwsem(&amn->lock); amn->type = type; - amn->mn.ops = &amdgpu_mn_ops[type]; amn->objects = RB_ROOT_CACHED; mutex_init(&amn->read_lock); atomic_set(&amn->recursion, 0); - r = __mmu_notifier_register(&amn->mn, mm); + amn->mirror.ops = &amdgpu_hmm_mirror_ops[type]; + r = hmm_mirror_register(&amn->mirror, mm); if (r) goto free_amn;@@ -432,7 +417,7 @@ struct amdgpu_mn *amdgpu_mn_get(structamdgpu_device *adev, * @bo: amdgpu buffer object * @addr: userptr addr we should monitor * -* Registers an MMU notifier for the given BO at the specified address. +* Registers an HMM mirror for the given BO at the specified address. * Returns 0 on success, -ERRNO if anything goes wrong. */ int
amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +1
-2@@ -488,11 +473,11 @@ int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr) } /** -* amdgpu_mn_unregister - unregister a BO for notifier updates +* amdgpu_mn_unregister - unregister a BO for HMM mirror updates * * @bo: amdgpu buffer object * -* Remove any registration of MMU notifier updates from the buffer object.
+ * Remove any registration of HMM mirror updates from the buffer object. */
voidamdgpu_mn_unregister(struct amdgpu_bo *bo)
{drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h +0
-0
Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h enum amdgpu_mn_type { · line 39
Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h +1
-1@@ -34,7 +34,7 @@ enum amdgpu_mn_type { AMDGPU_MN_TYPE_HSA, }; -#if defined(CONFIG_MMU_NOTIFIER ) +#if defined(CONFIG_HMM_MIRROR ) void amdgpu_mn_lock(struct amdgpu_mn *mn); void amdgpu_mn_unlock(struct amdgpu_mn *mn); struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev,
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): ee46d8e7-cf64-eb8c-cfd6-793db28c6c69-5C7GfCeVMHo@public.gmane.org
Re: [PATCH 1/3] drm/amdgpu: use HMM mirror callback to replace mmu notifier v6
Am 04.02.19 um 18:17 schrieb Yang, Philip:
Quoted 97 lines · Christian König
Quoted 83 lines
Am 04.02.19 um 16:06 schrieb Yang, Philip:
Quoted 82 lines
Replace our MMU notifier with hmm_mirror_ops.sync_cpu_device_pagetables callback. Enable
CONFIG_HMMandCONFIG_HMM_MIRRORas a dependency inDRM_AMDGPU_USERPTRKconfig.It supports both KFD userptr and gfx userptr paths.
The depdent HMM patchset from Jérôme Glisse are all merged into 4.20.0 kernel now.
Change-Id Ie62c3c5e3c5b8521ab3b438d1eff2aa2a003835eSigned-off-by Philip Yang <[email protected]>--- drivers/gpu/drm/amd/amdgpu/Kconfig | 6 +- drivers/gpu/drm/amd/amdgpu/Makefile | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c | 139 +++++++++++-------------- drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h | 2 +- 4 files changed, 67 insertions(+), 82 deletions(-)drivers/gpu/drm/amd/amdgpu/Kconfig +3
-3@@ -26,10 +26,10 @@ config DRM_AMDGPU_CIK config DRM_AMDGPU_USERPTR bool "Always enable userptr write support" depends on DRM_AMDGPU - select MMU_NOTIFIER + select HMM_MIRROR help - This option selects CONFIG_MMU_NOTIFIER if it isn't already - selected to enabled full userptr support. + This option selects CONFIG_HMM and CONFIG_HMM_MIRROR if it + isn't already selected to enabled full userptr support. config DRM_AMDGPU_GART_DEBUGFS bool "Allow GART access through debugfs"drivers/gpu/drm/amd/amdgpu/Makefile +1
-1@@ -172,7 +172,7 @@ endif amdgpu-$(CONFIG_COMPAT) += amdgpu_ioc32.o amdgpu-$(CONFIG_VGA_SWITCHEROO) += amdgpu_atpx_handler.o amdgpu-$(CONFIG_ACPI) += amdgpu_acpi.o -amdgpu-$(CONFIG_MMU_NOTIFIER) += amdgpu_mn.o +amdgpu-$(CONFIG_HMM_MIRROR) += amdgpu_mn.o include $(FULL_AMD_PATH)/powerplay/Makefiledrivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +2
-3@@ -45,7 +45,7 @@ #include <linux/firmware.h> #include <linux/module.h> -#include <linux/mmu_notifier.h> +#include <linux/hmm.h> #include <linux/interval_tree.h> #include <drm/drmP.h> #include <drm/drm.h>@@ -58,7 +58,6 @@ * * @adev: amdgpu device pointer * @mm: process address space - * @mn: MMU notifier structure * @type: type of MMU notifier * @work: destruction work item * @node: hash table node to find structure by adev and mn@@ -66,6 +65,7 @@* @objects: interval tree containing amdgpu_mn_nodes * @read_lock: mutex for recursive locking of @lock * @recursion: depth of recursion +* @mirror: HMM mirror function support * * Data for each amdgpu device and process address space. */ @@ -73,7 +73,6 @@ struct amdgpu_mn { /* constant after initialisation */ struct amdgpu_device *adev; struct mm_struct *mm; - struct mmu_notifier mn;Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c struct amdgpu_mn { · line 88
Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +0
-0enum amdgpu_mn_type type; /* only used on destruction */@@ -87,6 +86,9 @@ struct amdgpu_mn { struct rb_root_cached objects; struct mutex read_lock; atomic_t recursion;With HMM we don't need this any more. Please remove it and simplify
amdgpu_mn_read_lock()andamdgpu_mn_read_unlock().Thanks, this makes sense because HMM uses
hmm->mirror_semto serialize invalidate range.
Quoted 6 lines
amn->read_lockis also not needed anymore, this was used to protect atomic operations foratomic_inc_return(amn->recursion)anddown_read(amn->lock).Just one
amn->lockis needed to syncamdgpu_cs_submitand userptr update. I will submit another patch.
Sounds good. I will take a closer look at the other patches tomorrow.
Thanks,
Christian.
Quoted 274 lines
Philip
Quoted 271 lines
Apart from that looks good to me,
Christian.Quoted 268 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +11
-10+ + /* HMM mirror */ + struct hmm_mirror mirror; }; /**@@ -103,7 +105,7 @@ struct amdgpu_mn_node { }; /** -* amdgpu_mn_destroy - destroy the MMU notifier +* amdgpu_mn_destroy - destroy the HMM mirror * * @work: previously sheduled work item * @@ -129,28 +131,26 @@ static void amdgpu_mn_destroy(struct work_struct *work) } up_write(&amn->lock); mutex_unlock(&adev->mn_lock); - mmu_notifier_unregister_no_release(&amn->mn, amn->mm); + + hmm_mirror_unregister(&amn->mirror); kfree(amn); } /** -* amdgpu_mn_release - callback to notify about mm destruction +* amdgpu_hmm_mirror_release - callback to notify about mm destruction * -* @mn: our notifier -* @mm: the mm this callback is about +* @mirror: the HMM mirror (mm) this callback is about * -* Shedule a work item to lazy destroy our notifier. +* Shedule a work item to lazy destroy HMM mirror. */ -static void amdgpu_mn_release (struct mmu_notifier *mn, - struct mm_struct *mm) +static void amdgpu_hmm_mirror_release (struct hmm_mirror *mirror) { - struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn, mn); + struct amdgpu_mn *amn = container_of(mirror, struct amdgpu_mn, mirror); INIT_WORK(&amn->work, amdgpu_mn_destroy); schedule_work(&amn->work); } - /*** amdgpu_mn_lock - take the write side lock for this notifier * @@ -237,141 +237,126 @@ static void amdgpu_mn_invalidate_node(struct amdgpu_mn_node *node, /*** amdgpu_mn_invalidate_range_start_gfx - callback to notify about mm change
*
- * @mn: our notifier
- * @range: mmu notifier context
+ * @mirror: thehmm_mirror(mm) is about to update
+ * @update: the update start, end address
*
* Block for operations on BOs to finish and mark pages as accessed and
* potentially dirty.
*/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +3
-3-static int amdgpu_mn_invalidate_range_start_gfx (struct mmu_notifier *mn, - const struct mmu_notifier_range *range) +static int amdgpu_mn_sync_pagetables_gfx (struct hmm_mirror *mirror, + const struct hmm_update *update) { - struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn, mn); + struct amdgpu_mn *amn = container_of(mirror, struct amdgpu_mn,mirror);
+ unsigned long start =update->start;
+ unsigned long end =update->end;
+ bool blockable =update->blockable;
structinterval_tree_node*it;
- unsigned long end;
/* notification is exclusive, but interval is inclusive */
- end =range->end- 1;
+ end -= 1;
/* TODO we should be able to split locking for interval tree and
*amdgpu_mn_invalidate_node
*/
- if (amdgpu_mn_read_lock(amn, range->blockable))
+ if (amdgpu_mn_read_lock(amn, blockable))
return-EAGAIN;
- it =interval_tree_iter_first(&amn->objects, range->start, end); + it =interval_tree_iter_first(&amn->objects, start, end);
while (it) {
structamdgpu_mn_node*node;
- if (!range->blockable) {
+ if (!blockable) {
amdgpu_mn_read_unlock(amn);
return-EAGAIN;
}
node =container_of(it, struct amdgpu_mn_node, it);
- it =interval_tree_iter_next(it, range->start, end); + it =interval_tree_iter_next(it, start, end);
-amdgpu_mn_invalidate_node(node, range->start, end); +amdgpu_mn_invalidate_node(node, start, end);
}
+amdgpu_mn_read_unlock(amn);
+
return 0;
}
/**
*amdgpu_mn_invalidate_range_start_hsa- callback to notify about
mm change
*
- * @mn: our notifier
- * @mm: the mm this callback is about
- * @start: start of updated range
- * @end: end of updated range
+ * @mirror: thehmm_mirror(mm) is about to update
+ * @update: the update start, end address
*
* We temporarily evict all BOs between start and end. This
* necessitates evicting all user-mode queues of the process. The BOs
* are restorted inamdgpu_mn_invalidate_range_end_hsa. */drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +3
-3-static int amdgpu_mn_invalidate_range_start_hsa (struct mmu_notifier *mn, - const struct mmu_notifier_range *range) +static int amdgpu_mn_sync_pagetables_hsa (struct hmm_mirror *mirror, + const struct hmm_update *update) { - struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn, mn); + struct amdgpu_mn *amn = container_of(mirror, struct amdgpu_mn,mirror);
+ unsigned long start =update->start;
+ unsigned long end =update->end;
+ bool blockable =update->blockable;
structinterval_tree_node*it;
- unsigned long end;
/* notification is exclusive, but interval is inclusive */
- end =range->end- 1;
+ end -= 1;
- if (amdgpu_mn_read_lock(amn, range->blockable))
+ if (amdgpu_mn_read_lock(amn, blockable))
return-EAGAIN;
- it =interval_tree_iter_first(&amn->objects, range->start, end); + it =interval_tree_iter_first(&amn->objects, start, end);
while (it) {
structamdgpu_mn_node*node;
structamdgpu_bo*bo;
- if (!range->blockable) {
+ if (!blockable) {
amdgpu_mn_read_unlock(amn);
return-EAGAIN;
}
node =container_of(it, struct amdgpu_mn_node, it);
- it =interval_tree_iter_next(it, range->start, end); + it =interval_tree_iter_next(it, start, end);
list_for_each_entry(bo, &node->bos, mn_list){ structkgd_mem*mem =bo->kfd_bo;
if (amdgpu_ttm_tt_affect_userptr(bo->tbo.ttm,
-range->start,
- end))
-amdgpu_amdkfd_evict_userptr(mem, range->mm); + start, end))
+amdgpu_amdkfd_evict_userptr(mem, amn->mm);
}
}
+amdgpu_mn_read_unlock(amn);drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +15
-34+ return 0; } -/** -* amdgpu_mn_invalidate_range_end - callback to notify about mm change -* -* @mn: our notifier -* @mm: the mm this callback is about -* @start: start of updated range -* @end: end of updated range -* -* Release the lock again to allow new command submissions. +/* Low bits of any reasonable mm pointer will be unused due to struct +* alignment. Use these bits to make a unique key from the mm pointer +* and notifier type. */ -static void amdgpu_mn_invalidate_range_end (struct mmu_notifier *mn, - const struct mmu_notifier_range *range) -{ - struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn, mn); - - amdgpu_mn_read_unlock(amn); -} +#define AMDGPU_MN_KEY (mm, type) ((unsigned long)(mm) + (type)) -static const struct mmu_notifier_ops amdgpu_mn_ops[] = { +static struct hmm_mirror_ops amdgpu_hmm_mirror_ops[] = { [AMDGPU_MN_TYPE_GFX] = { - .release = amdgpu_mn_release, - .invalidate_range_start = amdgpu_mn_invalidate_range_start_gfx, - .invalidate_range_end = amdgpu_mn_invalidate_range_end, + .sync_cpu_device_pagetables = amdgpu_mn_sync_pagetables_gfx, + .release = amdgpu_hmm_mirror_release }, [AMDGPU_MN_TYPE_HSA] = { - .release = amdgpu_mn_release, - .invalidate_range_start = amdgpu_mn_invalidate_range_start_hsa, - .invalidate_range_end = amdgpu_mn_invalidate_range_end, + .sync_cpu_device_pagetables = amdgpu_mn_sync_pagetables_hsa, + .release = amdgpu_hmm_mirror_release }, }; -/* Low bits of any reasonable mm pointer will be unused due to struct -* alignment. Use these bits to make a unique key from the mm pointer -* and notifier type. -*/ -#define AMDGPU_MN_KEY (mm, type) ((unsigned long)(mm) + (type)) - /** - * amdgpu_mn_get - create notifier context + * amdgpu_mn_get - create HMM mirror context * * @adev: amdgpu device pointer * @type: type of MMU notifier context * - * Creates a notifier context for current->mm. + * Creates a HMM mirror context for current->mm. */ struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev, enum amdgpu_mn_type type)@@ -401,12 +386,12 @@ struct amdgpu_mn *amdgpu_mn_get(structamdgpu_device *adev, amn->mm = mm; init_rwsem(&amn->lock); amn->type = type; - amn->mn.ops = &amdgpu_mn_ops[type]; amn->objects = RB_ROOT_CACHED; mutex_init(&amn->read_lock); atomic_set(&amn->recursion, 0); - r = __mmu_notifier_register(&amn->mn, mm); + amn->mirror.ops = &amdgpu_hmm_mirror_ops[type]; + r = hmm_mirror_register(&amn->mirror, mm); if (r) goto free_amn;@@ -432,7 +417,7 @@ struct amdgpu_mn *amdgpu_mn_get(structamdgpu_device *adev, * @bo: amdgpu buffer object * @addr: userptr addr we should monitor * -* Registers an MMU notifier for the given BO at the specified address. +* Registers an HMM mirror for the given BO at the specified address. * Returns 0 on success, -ERRNO if anything goes wrong. */ int
amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +1
-2@@ -488,11 +473,11 @@ int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr) } /** -* amdgpu_mn_unregister - unregister a BO for notifier updates +* amdgpu_mn_unregister - unregister a BO for HMM mirror updates * * @bo: amdgpu buffer object * -* Remove any registration of MMU notifier updates from the buffer object.
+ * Remove any registration of HMM mirror updates from the buffer object. */
voidamdgpu_mn_unregister(struct amdgpu_bo *bo)
{drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h +0
-1@@ -34,7 +34,7 @@ enum amdgpu_mn_type { AMDGPU_MN_TYPE_HSA, }; -#if defined(CONFIG_MMU_NOTIFIER )
Quoted 8 lines
Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h enum amdgpu_mn_type { · line 40
Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h +1
-0+#if defined(CONFIG_HMM_MIRROR ) void amdgpu_mn_lock(struct amdgpu_mn *mn); void amdgpu_mn_unlock(struct amdgpu_mn *mn); struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev,_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
PATCH 2/3 drm/amdkfd: avoid HMM change cause circular lock dependency v2
In reply to a message not in the archiveThere is circular lock between gfx and kfd path with HMM change:
lock(dqm) -> bo::reserve -> amdgpu_mn_lock
To avoid this, move init/unint_mqd() out of lock(dqm), to remove nested locking between mmap_sem and bo::reserve. The locking order
is: bo::reserve -> amdgpu_mn_lock(p->mn)
--- .../drm/amd/amdkfd/kfd_device_queue_manager.c | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +17 -15
@@ -1156,21 +1156,17 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,int retval;struct mqd_manager *mqd_mgr; - retval =0 ; - -dqm_lock (dqm); -if (dqm->total_queue_count >= max_num_of_queues_per_device) {pr_warn ("Can't create new usermode queue because %d queues were already created\n" , dqm->total_queue_count); retval = -EPERM ; - goto out_unlock; + goto out; }if (q->properties.type ==KFD_QUEUE_TYPE_SDMA ) { retval =allocate_sdma_queue (dqm, &q->sdma_id);if (retval) - goto out_unlock; + goto out; q->properties.sdma_queue_id = q->sdma_id /get_num_sdma_engines (dqm); q->properties.sdma_engine_id =
@@ -1181,6 +1177,9 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,if (retval) goto out_deallocate_sdma_queue; +/* Do init_mqd before dqm_lock(dqm) to avoid circular locking order: +* lock(dqm) -> bo::reserve +*/ mqd_mgr = dqm->ops.get_mqd_manager (dqm,get_mqd_type_from_queue_type (q->properties.type));
@@ -1188,6 +1187,7 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, retval = -ENOMEM ; goto out_deallocate_doorbell; } +/* * Eviction state logic: we only mark active queues as evicted * to avoid the overhead of restoring inactive queues later
@@ -1196,9 +1196,7 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, q->properties.is_evicted = (q->properties.queue_size >0 && q->properties.queue_percent >0 && q->properties.queue_address !=0 ); - dqm->asic_ops.init_sdma_vm (dqm, q, qpd); - q->properties.tba_addr = qpd->tba_addr; q->properties.tma_addr = qpd->tma_addr; retval = mqd_mgr->init_mqd(mqd_mgr, &q->mqd, &q->mqd_mem_obj,
@@ -1206,6 +1204,8 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,if (retval) goto out_deallocate_doorbell; +dqm_lock (dqm); +list_add (&q->list, &qpd->queues_list); qpd->queue_count++;if (q->properties.is_active) {
@@ -1233,9 +1233,7 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, out_deallocate_sdma_queue:if (q->properties.type ==KFD_QUEUE_TYPE_SDMA )deallocate_sdma_queue (dqm, q->sdma_id); -out_unlock: -dqm_unlock (dqm); - +out:return retval; }
@@ -1398,8 +1396,6 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm, qpd->reset_wavefronts = true; } - mqd_mgr->uninit_mqd (mqd_mgr, q->mqd, q->mqd_mem_obj); -/* * Unconditionally decrement this counter, regardless of the queue's * type
@@ -1410,6 +1406,9 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm,dqm_unlock (dqm); +/* Do uninit_mqd after dqm_unlock(dqm) to avoid circular locking */ + mqd_mgr->uninit_mqd (mqd_mgr, q->mqd, q->mqd_mem_obj); +return retval; failed:
@@ -1631,7 +1630,11 @@ static int process_termination_cpsch(struct device_queue_manager *dqm, qpd->reset_wavefronts = false; } -/* lastly, free mqd resources */ +dqm_unlock (dqm); + +/* Lastly, free mqd resources. +* Do uninit_mqd() after dqm_unlock to avoid circular locking. +*/ list_for_each_entry_safe (q, next, &qpd->queues_list, list) { mqd_mgr = dqm->ops.get_mqd_manager (dqm,get_mqd_type_from_queue_type (q->properties.type));
@@ -1645,7 +1648,6 @@ static int process_termination_cpsch(struct device_queue_manager *dqm, } out: -dqm_unlock (dqm);return retval; }
Signature · 6 lines
2.17.1 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
PATCH 3/3 drm/amdgpu: replace get_user_pages with HMM address mirror helpers v6
In reply to a message not in the archiveUse HMM helper function hmm_vma_fault() to get physical pages backing userptr and start CPU page table update track of those pages. Then use hmm_vma_range_done() to check if those pages are updated before amdgpu_cs_submit for gfx or before user queues are resumed for kfd.
If userptr pages are updated, for gfx, amdgpu_cs_ioctl will restart from scratch, for kfd, restore worker is rescheduled to retry.
HMM simplify the CPU page table concurrent update check, so remove guptasklock, mmu_invalidations, last_set_pages fields from amdgpu_ttm_tt struct.
HMM does not pin the page (increase page ref count), so remove related operations like release_pages(), put_page(), mark_page_dirty().
--- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 1 - .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 95 +++------- drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.h | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c | 158 +++++++--------- drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c | 14 +- drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c | 25 ++- drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h | 4 +- drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 173 ++++++++---------- drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 3 +- 9 files changed, 198 insertions(+), 277 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +0 -1
@@ -61,7 +61,6 @@ struct kgd_mem {atomic_t invalid;struct amdkfd_process_info *process_info; -struct page **user_pages;struct amdgpu_sync sync;
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +25 -70
178 lines not on this page · raw message
drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.h +1 -1
@@ -36,7 +36,7 @@ struct amdgpu_bo_list_entry {struct amdgpu_bo_va *bo_va;uint32_t priority;struct page **user_pages; -int user_invalidated; +bool user_invalidated; };struct amdgpu_bo_list {
drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +66 -92
233 lines not on this page · raw message
drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +6 -8
@@ -329,26 +329,24 @@ int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data, r =amdgpu_bo_reserve (bo, true);if (r) - goto free_pages; + goto user_pages_done;amdgpu_bo_placement_from_domain (bo,AMDGPU_GEM_DOMAIN_GTT ); r =ttm_bo_validate (&bo->tbo, &bo->placement, &ctx);amdgpu_bo_unreserve (bo);if (r) - goto free_pages; + goto user_pages_done; } r =drm_gem_handle_create (filp, gobj, &handle); -/* drop reference from allocate - handle holds it now */ -drm_gem_object_put_unlocked (gobj);if (r) -return r; + goto user_pages_done; args->handle = handle; -return 0 ; -free_pages: -release_pages (bo->tbo.ttm->pages, bo->tbo.ttm->num_pages); +user_pages_done: +if (args->flags &AMDGPU_GEM_USERPTR_VALIDATE ) +amdgpu_ttm_tt_get_user_pages_done (bo->tbo.ttm); release_object:drm_gem_object_put_unlocked (gobj);
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +23 -2
@@ -229,8 +229,6 @@ static void amdgpu_mn_invalidate_node(struct amdgpu_mn_node *node, true, false, MAX_SCHEDULE_TIMEOUT);if (r <=0 )DRM_ERROR ("(%ld) failed to wait for user bo\n" , r); - -amdgpu_ttm_tt_mark_user_pages (bo->tbo.ttm); } }
@@ -513,3 +511,26 @@ void amdgpu_mn_unregister(struct amdgpu_bo *bo)mutex_unlock (&adev->mn_lock); } +/* flags used by HMM internal, not related to CPU/GPU PTE flags */ +static const uint64_t hmm_range_flags[HMM_PFN_FLAG_MAX ] = { + (1 <<0 ),/* HMM_PFN_VALID */ + (1 <<1 ),/* HMM_PFN_WRITE */ +0 /* HMM_PFN_DEVICE_PRIVATE */ +}; + +static const uint64_t hmm_range_values[HMM_PFN_VALUE_MAX ] = { +0xfffffffffffffffeUL ,/* HMM_PFN_ERROR */ +0 ,/* HMM_PFN_NONE */ +0xfffffffffffffffcUL /* HMM_PFN_SPECIAL */ +}; + +void amdgpu_hmm_init_range (struct hmm_range *range) +{ +if (range) { + range->flags = hmm_range_flags; + range->values = hmm_range_values; + range->pfn_shift =PAGE_SHIFT ; + range->pfns =NULL ; +INIT_LIST_HEAD (&range->list); + } +}
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h +3 -1
@@ -25,9 +25,10 @@#define __AMDGPU_MN_H__/* -* MMU Notifier +* HMM mirror */ struct amdgpu_mn ; +struct hmm_range ;enum amdgpu_mn_type {AMDGPU_MN_TYPE_GFX ,
@@ -41,6 +42,7 @@ struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev, enum amdgpu_mn_type type);int amdgpu_mn_register (struct amdgpu_bo *bo,unsigned long addr);void amdgpu_mn_unregister (struct amdgpu_bo *bo); +void amdgpu_hmm_init_range (struct hmm_range *range);#else static inline void amdgpu_mn_lock (struct amdgpu_mn *mn) {}static inline void amdgpu_mn_unlock (struct amdgpu_mn *mn) {}
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +72 -101
263 lines not on this page · raw message
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h +2 -1
@@ -102,6 +102,7 @@ int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo);int amdgpu_ttm_recover_gart (struct ttm_buffer_object *tbo);int amdgpu_ttm_tt_get_user_pages (struct ttm_tt *ttm,struct page **pages); +bool amdgpu_ttm_tt_get_user_pages_done (struct ttm_tt *ttm);void amdgpu_ttm_tt_set_user_pages (struct ttm_tt *ttm,struct page **pages);void amdgpu_ttm_tt_mark_user_pages (struct ttm_tt *ttm); int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,
@@ -112,7 +113,7 @@ bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start, unsigned long end);bool amdgpu_ttm_tt_userptr_invalidated (struct ttm_tt *ttm,int *last_invalidated); -bool amdgpu_ttm_tt_userptr_needs_pages (struct ttm_tt *ttm); +bool amdgpu_ttm_tt_is_userptr (struct ttm_tt *ttm);bool amdgpu_ttm_tt_is_readonly (struct ttm_tt *ttm);uint64_t amdgpu_ttm_tt_pde_flags (struct ttm_tt *ttm,struct ttm_mem_reg *mem); uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,
Signature · 6 lines
2.17.1 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx