drm/xe/dgfx: Release mmap mappings on rpm suspend
Subsystems (2)
PATCH v4 1/1 drm/xe/dgfx: Release mmap mappings on rpm suspend
Release all mmap mappings for all vram objects which are associated with userfault such that, while pcie function in D3hot, any access to memory mappings will raise a userfault.
Upon userfault, in order to access memory mappings, if graphics function is in D3 then runtime resume of dgpu will be triggered to transition to D0.
v2:
- Avoid iomem check before bo migration check as bo can migrate to system memory (Matthew Auld)
v3:
- Delete bo userfault link during bo destroy
- Upon bo move (vram-smem), do bo userfault link deletion in xe_bo_move_notify instead of xe_bo_move (Thomas Hellström)
- Grab lock in rpm hook while deleting bo userfault link (Matthew Auld)
v4:
- Add kernel doc and wrap vram_userfault related
stuff in the structure (Matthew Auld)
- Mem access get for rpm before taking dma reserve lock (Matthew Auld)
- In suspend path apply lock for entire list op
including list iteration (Matthew Auld)
--- drivers/gpu/drm/xe/xe_bo.c | 63 ++++++++++++++++++++++++++-- drivers/gpu/drm/xe/xe_bo.h | 2 + drivers/gpu/drm/xe/xe_bo_types.h | 10 +++++ drivers/gpu/drm/xe/xe_device_types.h | 12 ++++++ drivers/gpu/drm/xe/xe_pm.c | 13 ++++++ 5 files changed, 96 insertions(+), 4 deletions(-)
drivers/gpu/drm/xe/xe_bo.c +59 -4
@@ -586,6 +586,8 @@ static int xe_bo_move_notify(struct xe_bo *bo, {struct ttm_buffer_object *ttm_bo = &bo->ttm;struct xe_device *xe =ttm_to_xe_device (ttm_bo->bdev); +struct ttm_resource *old_mem = ttm_bo->resource; +u32 old_mem_type = old_mem ? old_mem->mem_type :XE_PL_SYSTEM ;int ret;/*
@@ -597,6 +599,7 @@ static int xe_bo_move_notify(struct xe_bo *bo,return -EINVAL ;xe_bo_vunmap (bo); + ret =xe_bo_trigger_rebind (xe, bo, ctx);if (ret)return ret;
@@ -605,6 +608,18 @@ static int xe_bo_move_notify(struct xe_bo *bo,if (ttm_bo->base.dma_buf && !ttm_bo->base.import_attach)dma_buf_move_notify (ttm_bo->base.dma_buf); +/* +* TTM has already nuked the mmap for us (see ttm_bo_unmap_virtual), +* so if we moved from VRAM make sure to unlink this from the userfault +* tracking. +*/ +if (mem_type_is_vram (old_mem_type)) { +spin_lock (&xe->mem_access.vram_userfault.lock); +if (!list_empty (&bo->vram_userfault.link)) +list_del_init (&bo->vram_userfault.link); +spin_unlock (&xe->mem_access.vram_userfault.lock); + } +return 0 ; }
@@ -1063,6 +1078,11 @@ static void xe_ttm_bo_destroy(struct ttm_buffer_object *ttm_bo)if (bo->vm &&xe_bo_is_user (bo))xe_vm_put (bo->vm); +spin_lock (&xe->mem_access.vram_userfault.lock); +if (!list_empty (&bo->vram_userfault.link)) +list_del (&bo->vram_userfault.link); +spin_unlock (&xe->mem_access.vram_userfault.lock); +kfree (bo); }
@@ -1110,16 +1130,20 @@ static vm_fault_t xe_gem_fault(struct vm_fault *vmf) {struct ttm_buffer_object *tbo = vmf->vma->vm_private_data;struct drm_device *ddev = tbo->base.dev; +struct xe_device *xe =to_xe_device (ddev); +struct xe_bo *bo =ttm_to_xe_bo (tbo); +bool needs_rpm = bo->flags &XE_BO_CREATE_VRAM_MASK ;vm_fault_t ret;int idx, r =0 ; +if (needs_rpm) +xe_device_mem_access_get (xe); + ret =ttm_bo_vm_reserve (tbo, vmf);if (ret) -return ret; + goto out;if (drm_dev_enter (ddev, &idx)) { -struct xe_bo *bo =ttm_to_xe_bo (tbo); -trace_xe_bo_cpu_fault (bo);if (should_migrate_to_system (bo)) {
@@ -1137,10 +1161,28 @@ static vm_fault_t xe_gem_fault(struct vm_fault *vmf) }else { ret =ttm_bo_vm_dummy_page (vmf, vmf->vma->vm_page_prot); } +if (ret ==VM_FAULT_RETRY && !(vmf->flags &FAULT_FLAG_RETRY_NOWAIT )) -return ret; + goto out; +/* +* ttm_bo_vm_reserve() already has dma_resv_lock. +* vram_userfault.count is protected by dma_resv lock and rpm wakeref. +*/ +if (ret ==VM_FAULT_NOPAGE && !bo->vram_userfault.count) { +if (tbo->resource->bus.is_iomem) { + bo->vram_userfault.count =1 ; + +spin_lock (&xe->mem_access.vram_userfault.lock); +list_add (&bo->vram_userfault.link, &xe->mem_access.vram_userfault.list); +spin_unlock (&xe->mem_access.vram_userfault.lock); + } + }dma_resv_unlock (tbo->base.resv); +out: +if (needs_rpm) +xe_device_mem_access_put (xe); +return ret; }
@@ -1254,6 +1296,7 @@ struct xe_bo *___xe_bo_create_locked(struct xe_device *xe, struct xe_bo *bo,#ifdef CONFIG_PROC_FS INIT_LIST_HEAD (&bo->client_link);#endif +INIT_LIST_HEAD (&bo->vram_userfault.link);drm_gem_private_object_init (&xe->drm, &bo->ttm.base, size);
@@ -2264,6 +2307,18 @@ int xe_bo_dumb_create(struct drm_file *file_priv,return err; } +void xe_bo_runtime_pm_release_mmap_offset (struct xe_bo *bo) +{ +struct ttm_buffer_object *tbo = &bo->ttm; +struct ttm_device *bdev = tbo->bdev; + +drm_vma_node_unmap (&tbo->base.vma_node, bdev->dev_mapping); + +XE_WARN_ON (!bo->vram_userfault.count); +list_del (&bo->vram_userfault.link); + bo->vram_userfault.count =0 ; +} +#if IS_ENABLED (CONFIG_DRM_XE_KUNIT_TEST )#include "tests/xe_bo.c" #endif
drivers/gpu/drm/xe/xe_bo.h +2 -0
@@ -249,6 +249,8 @@ int xe_gem_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file);int xe_gem_mmap_offset_ioctl (struct drm_device *dev,void *data,struct drm_file *file); +void xe_bo_runtime_pm_release_mmap_offset (struct xe_bo *bo); +int xe_bo_dumb_create (struct drm_file *file_priv,struct drm_device *dev,struct drm_mode_create_dumb *args);
drivers/gpu/drm/xe/xe_bo_types.h +10 -0
@@ -88,6 +88,16 @@ struct xe_bo {* objects. */ u16 cpu_caching; + +/** +* @vram_userfault: Encapsulate vram_userfault related stuff +*/ +struct { +/** @link: Link into @mem_access.vram_userfault.list */ +struct list_head link; +/** @count: Whether bo is @mem_access.vram_userfault.list */ +unsigned int count; + } vram_userfault; };#define intel_bo_to_drm_bo (bo) (&(bo)->ttm.base)
drivers/gpu/drm/xe/xe_device_types.h +12 -0
@@ -385,6 +385,18 @@ struct xe_device {struct {/** @ref: ref count of memory accesses */ atomic_t ref; + +/** @vram_userfault: Encapsulate vram_userfault related stuff */ +struct { +/** @lock: Protects access to @vram_usefault.list */ +spinlock_t lock; + +/** +* @list: Keep list of userfaulted vram bo, which require to release their +* mmap mappings at runtime suspend path. +*/ +struct list_head list; + } vram_userfault; } mem_access;/**
drivers/gpu/drm/xe/xe_pm.c +13 -0
@@ -181,6 +181,8 @@ void xe_pm_init(struct xe_device *xe) }xe_pm_runtime_init (xe); +INIT_LIST_HEAD (&xe->mem_access.vram_userfault.list); +spin_lock_init (&xe->mem_access.vram_userfault.lock); }void xe_pm_runtime_fini (struct xe_device *xe)
@@ -214,6 +216,7 @@ struct task_struct *xe_pm_read_callback_task(struct xe_device *xe)int xe_pm_runtime_suspend (struct xe_device *xe) { +struct xe_bo *bo, *on;struct xe_gt *gt;u8 id;int err =0 ;
@@ -247,6 +250,16 @@ int xe_pm_runtime_suspend(struct xe_device *xe)*/ lock_map_acquire (&xe_device_mem_access_lockdep_map); +/* +* Applying lock for entire list op as xe_ttm_bo_destroy and xe_bo_move_notify +* also checks and delets bo entry from user fault list. +*/ +spin_lock (&xe->mem_access.vram_userfault.lock); +list_for_each_entry_safe (bo, on, + &xe->mem_access.vram_userfault.list, vram_userfault.link) +xe_bo_runtime_pm_release_mmap_offset (bo); +spin_unlock (&xe->mem_access.vram_userfault.lock); +if (xe->d3cold.allowed) { err =xe_bo_evict_all (xe);if (err)
Signature · 1 line
2.25.1