Use HMM to replace get_user_pages
Subsystems (3)
cover Use HMM to replace get_user_pages
Hi Christian,
Resend patch 1/3, 2/3, added Reviewed-by in comments.
Change in patch 3/3, amdgpu_cs_submit, amdgpu_cs_ioctl return -EAGAIN to user space to retry cs_ioctl.
Regards,
Philip
Philip Yang (3):
drm/amdgpu: use HMM mirror callback to replace mmu notifier v7 drm/amdkfd: avoid HMM change cause circular lock dependency v2 drm/amdgpu: replace get_user_pages with HMM address mirror helpers v8
drivers/gpu/drm/amd/amdgpu/Kconfig | 6 +- drivers/gpu/drm/amd/amdgpu/Makefile | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 1 - .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 95 +++------- drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.h | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c | 138 +++++--------- drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c | 14 +- drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c | 179 +++++++++--------- drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h | 6 +- drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 178 +++++++---------- drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 3 +- .../drm/amd/amdkfd/kfd_device_queue_manager.c | 32 ++-- 12 files changed, 269 insertions(+), 387 deletions(-)
Signature · 6 lines
2.17.1 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): [email protected]
PATCH 1/3 drm/amdgpu: use HMM mirror callback to replace mmu notifier v7
Replace our MMU notifier with hmm_mirror_ops.sync_cpu_device_pagetables callback. Enable CONFIG_HMM and CONFIG_HMM_MIRROR as a dependency in DRM_AMDGPU_USERPTR Kconfig.
It supports both KFD userptr and gfx userptr paths.
--- drivers/gpu/drm/amd/amdgpu/Kconfig | 6 +- drivers/gpu/drm/amd/amdgpu/Makefile | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c | 154 +++++++++++-------------- drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h | 2 +- 4 files changed, 70 insertions(+), 94 deletions(-)
drivers/gpu/drm/amd/amdgpu/Kconfig +3 -3
@@ -26,10 +26,10 @@ config DRM_AMDGPU_CIK config DRM_AMDGPU_USERPTR bool "Always enable userptr write support" depends on DRM_AMDGPU - select MMU_NOTIFIER + select HMM_MIRROR help - This option selects CONFIG_MMU_NOTIFIER if it isn't already - selected to enabled full userptr support. + This option selects CONFIG_HMM and CONFIG_HMM_MIRROR if it + isn't already selected to enabled full userptr support. config DRM_AMDGPU_GART_DEBUGFS bool "Allow GART access through debugfs"
drivers/gpu/drm/amd/amdgpu/Makefile +1 -1
@@ -172,7 +172,7 @@ endif amdgpu-$(CONFIG_COMPAT) += amdgpu_ioc32.o amdgpu-$(CONFIG_VGA_SWITCHEROO) += amdgpu_atpx_handler.o amdgpu-$(CONFIG_ACPI) += amdgpu_acpi.o -amdgpu-$(CONFIG_MMU_NOTIFIER) += amdgpu_mn.o +amdgpu-$(CONFIG_HMM_MIRROR) += amdgpu_mn.o include $(FULL_AMD_PATH)/powerplay/Makefile
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +65 -89
328 lines not on this page · raw message
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h +1 -1
@@ -34,7 +34,7 @@ enum amdgpu_mn_type { AMDGPU_MN_TYPE_HSA, }; -#if defined(CONFIG_MMU_NOTIFIER ) +#if defined(CONFIG_HMM_MIRROR )void amdgpu_mn_lock (struct amdgpu_mn *mn);void amdgpu_mn_unlock (struct amdgpu_mn *mn); struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev,
Signature · 6 lines
2.17.1 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
PATCH 2/3 drm/amdkfd: avoid HMM change cause circular lock dependency v2
In reply to a message not in the archiveThere is circular lock between gfx and kfd path with HMM change:
lock(dqm) -> bo::reserve -> amdgpu_mn_lock
To avoid this, move init/unint_mqd() out of lock(dqm), to remove nested locking between mmap_sem and bo::reserve. The locking order
is: bo::reserve -> amdgpu_mn_lock(p->mn)
--- .../drm/amd/amdkfd/kfd_device_queue_manager.c | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +17 -15
@@ -1156,21 +1156,17 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,int retval;struct mqd_manager *mqd_mgr; - retval =0 ; - -dqm_lock (dqm); -if (dqm->total_queue_count >= max_num_of_queues_per_device) {pr_warn ("Can't create new usermode queue because %d queues were already created\n" , dqm->total_queue_count); retval = -EPERM ; - goto out_unlock; + goto out; }if (q->properties.type ==KFD_QUEUE_TYPE_SDMA ) { retval =allocate_sdma_queue (dqm, &q->sdma_id);if (retval) - goto out_unlock; + goto out; q->properties.sdma_queue_id = q->sdma_id /get_num_sdma_engines (dqm); q->properties.sdma_engine_id =
@@ -1181,6 +1177,9 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,if (retval) goto out_deallocate_sdma_queue; +/* Do init_mqd before dqm_lock(dqm) to avoid circular locking order: +* lock(dqm) -> bo::reserve +*/ mqd_mgr = dqm->ops.get_mqd_manager (dqm,get_mqd_type_from_queue_type (q->properties.type));
@@ -1188,6 +1187,7 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, retval = -ENOMEM ; goto out_deallocate_doorbell; } +/* * Eviction state logic: we only mark active queues as evicted * to avoid the overhead of restoring inactive queues later
@@ -1196,9 +1196,7 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, q->properties.is_evicted = (q->properties.queue_size >0 && q->properties.queue_percent >0 && q->properties.queue_address !=0 ); - dqm->asic_ops.init_sdma_vm (dqm, q, qpd); - q->properties.tba_addr = qpd->tba_addr; q->properties.tma_addr = qpd->tma_addr; retval = mqd_mgr->init_mqd(mqd_mgr, &q->mqd, &q->mqd_mem_obj,
@@ -1206,6 +1204,8 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,if (retval) goto out_deallocate_doorbell; +dqm_lock (dqm); +list_add (&q->list, &qpd->queues_list); qpd->queue_count++;if (q->properties.is_active) {
@@ -1233,9 +1233,7 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, out_deallocate_sdma_queue:if (q->properties.type ==KFD_QUEUE_TYPE_SDMA )deallocate_sdma_queue (dqm, q->sdma_id); -out_unlock: -dqm_unlock (dqm); - +out:return retval; }
@@ -1398,8 +1396,6 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm, qpd->reset_wavefronts = true; } - mqd_mgr->uninit_mqd (mqd_mgr, q->mqd, q->mqd_mem_obj); -/* * Unconditionally decrement this counter, regardless of the queue's * type
@@ -1410,6 +1406,9 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm,dqm_unlock (dqm); +/* Do uninit_mqd after dqm_unlock(dqm) to avoid circular locking */ + mqd_mgr->uninit_mqd (mqd_mgr, q->mqd, q->mqd_mem_obj); +return retval; failed:
@@ -1631,7 +1630,11 @@ static int process_termination_cpsch(struct device_queue_manager *dqm, qpd->reset_wavefronts = false; } -/* lastly, free mqd resources */ +dqm_unlock (dqm); + +/* Lastly, free mqd resources. +* Do uninit_mqd() after dqm_unlock to avoid circular locking. +*/ list_for_each_entry_safe (q, next, &qpd->queues_list, list) { mqd_mgr = dqm->ops.get_mqd_manager (dqm,get_mqd_type_from_queue_type (q->properties.type));
@@ -1645,7 +1648,6 @@ static int process_termination_cpsch(struct device_queue_manager *dqm, } out: -dqm_unlock (dqm);return retval; }
Signature · 6 lines
2.17.1 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
PATCH 3/3 drm/amdgpu: replace get_user_pages with HMM address mirror helpers v8
In reply to a message not in the archiveUse HMM helper function hmm_vma_fault() to get physical pages backing userptr and start CPU page table update track of those pages. Then use hmm_vma_range_done() to check if those pages are updated before amdgpu_cs_submit for gfx or before user queues are resumed for kfd.
If userptr pages are updated, for gfx, amdgpu_cs_ioctl will restart from scratch, for kfd, restore worker is rescheduled to retry.
HMM simplify the CPU page table concurrent update check, so remove guptasklock, mmu_invalidations, last_set_pages fields from amdgpu_ttm_tt struct.
HMM does not pin the page (increase page ref count), so remove related operations like release_pages(), put_page(), mark_page_dirty().
--- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 1 - .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 95 +++------- drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.h | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c | 138 +++++--------- drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c | 14 +- drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c | 25 ++- drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h | 4 +- drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 178 +++++++----------- drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 3 +- 9 files changed, 182 insertions(+), 278 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +0 -1
@@ -61,7 +61,6 @@ struct kgd_mem {atomic_t invalid;struct amdkfd_process_info *process_info; -struct page **user_pages;struct amdgpu_sync sync;
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +25 -70
178 lines not on this page · raw message
drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.h +1 -1
@@ -36,7 +36,7 @@ struct amdgpu_bo_list_entry {struct amdgpu_bo_va *bo_va;uint32_t priority;struct page **user_pages; -int user_invalidated; +bool user_invalidated; };struct amdgpu_bo_list {
drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +50 -88
202 lines not on this page · raw message
drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +6 -8
@@ -329,26 +329,24 @@ int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data, r =amdgpu_bo_reserve (bo, true);if (r) - goto free_pages; + goto user_pages_done;amdgpu_bo_placement_from_domain (bo,AMDGPU_GEM_DOMAIN_GTT ); r =ttm_bo_validate (&bo->tbo, &bo->placement, &ctx);amdgpu_bo_unreserve (bo);if (r) - goto free_pages; + goto user_pages_done; } r =drm_gem_handle_create (filp, gobj, &handle); -/* drop reference from allocate - handle holds it now */ -drm_gem_object_put_unlocked (gobj);if (r) -return r; + goto user_pages_done; args->handle = handle; -return 0 ; -free_pages: -release_pages (bo->tbo.ttm->pages, bo->tbo.ttm->num_pages); +user_pages_done: +if (args->flags &AMDGPU_GEM_USERPTR_VALIDATE ) +amdgpu_ttm_tt_get_user_pages_done (bo->tbo.ttm); release_object:drm_gem_object_put_unlocked (gobj);
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +23 -2
@@ -222,8 +222,6 @@ static void amdgpu_mn_invalidate_node(struct amdgpu_mn_node *node, true, false, MAX_SCHEDULE_TIMEOUT);if (r <=0 )DRM_ERROR ("(%ld) failed to wait for user bo\n" , r); - -amdgpu_ttm_tt_mark_user_pages (bo->tbo.ttm); } }
@@ -504,3 +502,26 @@ void amdgpu_mn_unregister(struct amdgpu_bo *bo)mutex_unlock (&adev->mn_lock); } +/* flags used by HMM internal, not related to CPU/GPU PTE flags */ +static const uint64_t hmm_range_flags[HMM_PFN_FLAG_MAX ] = { + (1 <<0 ),/* HMM_PFN_VALID */ + (1 <<1 ),/* HMM_PFN_WRITE */ +0 /* HMM_PFN_DEVICE_PRIVATE */ +}; + +static const uint64_t hmm_range_values[HMM_PFN_VALUE_MAX ] = { +0xfffffffffffffffeUL ,/* HMM_PFN_ERROR */ +0 ,/* HMM_PFN_NONE */ +0xfffffffffffffffcUL /* HMM_PFN_SPECIAL */ +}; + +void amdgpu_hmm_init_range (struct hmm_range *range) +{ +if (range) { + range->flags = hmm_range_flags; + range->values = hmm_range_values; + range->pfn_shift =PAGE_SHIFT ; + range->pfns =NULL ; +INIT_LIST_HEAD (&range->list); + } +}
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h +3 -1
@@ -25,9 +25,10 @@#define __AMDGPU_MN_H__/* -* MMU Notifier +* HMM mirror */ struct amdgpu_mn ; +struct hmm_range ;enum amdgpu_mn_type {AMDGPU_MN_TYPE_GFX ,
@@ -41,6 +42,7 @@ struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev, enum amdgpu_mn_type type);int amdgpu_mn_register (struct amdgpu_bo *bo,unsigned long addr);void amdgpu_mn_unregister (struct amdgpu_bo *bo); +void amdgpu_hmm_init_range (struct hmm_range *range);#else static inline void amdgpu_mn_lock (struct amdgpu_mn *mn) {}static inline void amdgpu_mn_unlock (struct amdgpu_mn *mn) {}
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +72 -106
268 lines not on this page · raw message
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h +2 -1
@@ -102,6 +102,7 @@ int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo);int amdgpu_ttm_recover_gart (struct ttm_buffer_object *tbo);int amdgpu_ttm_tt_get_user_pages (struct ttm_tt *ttm,struct page **pages); +bool amdgpu_ttm_tt_get_user_pages_done (struct ttm_tt *ttm);void amdgpu_ttm_tt_set_user_pages (struct ttm_tt *ttm,struct page **pages);void amdgpu_ttm_tt_mark_user_pages (struct ttm_tt *ttm); int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,
@@ -112,7 +113,7 @@ bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start, unsigned long end);bool amdgpu_ttm_tt_userptr_invalidated (struct ttm_tt *ttm,int *last_invalidated); -bool amdgpu_ttm_tt_userptr_needs_pages (struct ttm_tt *ttm); +bool amdgpu_ttm_tt_is_userptr (struct ttm_tt *ttm);bool amdgpu_ttm_tt_is_readonly (struct ttm_tt *ttm);uint64_t amdgpu_ttm_tt_pde_flags (struct ttm_tt *ttm,struct ttm_mem_reg *mem); uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,
Signature · 6 lines
2.17.1 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx