drm/ttm: remove ttm_bo_(un)lock_delayed_workqueue
Subsystems (5)
PATCH 1/3 drm/ttm: remove ttm_bo_(un)lock_delayed_workqueue
Those functions never worked correctly since it is still perfectly possible that a buffer object is released and the background worker restarted even after calling them.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c | 6 +----- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 4 +--- drivers/gpu/drm/radeon/radeon_device.c | 5 ----- drivers/gpu/drm/radeon/radeon_pm.c | 4 +--- drivers/gpu/drm/ttm/ttm_bo.c | 14 -------------- include/drm/ttm/ttm_bo_api.h | 16 ---------------- 6 files changed, 3 insertions(+), 46 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +1 -5
@@ -1717,7 +1717,7 @@ static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring)static int amdgpu_debugfs_ib_preempt (void *data,u64 val) { -int r, resched, length; +int r, length;struct amdgpu_ring *ring;struct dma_fence **fences =NULL ;struct amdgpu_device *adev = (struct amdgpu_device *)data;
@@ -1747,8 +1747,6 @@ static int amdgpu_debugfs_ib_preempt(void *data, u64 val)/* stop the scheduler */ kthread_park (ring->sched.thread); - resched =ttm_bo_lock_delayed_workqueue (&adev->mman.bdev); -/* preempt the IB */ r =amdgpu_ring_preempt_ib (ring);if (r) {
@@ -1785,8 +1783,6 @@ static int amdgpu_debugfs_ib_preempt(void *data, u64 val)up_read (&adev->reset_domain->sem); -ttm_bo_unlock_delayed_workqueue (&adev->mman.bdev, resched); - pro_end:kfree (fences);
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1 -3
@@ -3965,10 +3965,8 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev) }amdgpu_fence_driver_hw_fini (adev); -if (adev->mman.initialized) { +if (adev->mman.initialized)flush_delayed_work (&adev->mman.bdev.wq); -ttm_bo_lock_delayed_workqueue (&adev->mman.bdev); - }if (adev->pm_sysfs_en)amdgpu_pm_sysfs_fini (adev);
drivers/gpu/drm/radeon/radeon_device.c +0 -5
@@ -1768,7 +1768,6 @@ int radeon_gpu_reset(struct radeon_device *rdev)bool saved = false;int i, r; -int resched;down_write (&rdev->exclusive_lock);
@@ -1780,8 +1779,6 @@ int radeon_gpu_reset(struct radeon_device *rdev)atomic_inc (&rdev->gpu_reset_counter);radeon_save_bios_scratch_regs (rdev); -/* block TTM */ - resched =ttm_bo_lock_delayed_workqueue (&rdev->mman.bdev);radeon_suspend (rdev);radeon_hpd_fini (rdev);
@@ -1840,8 +1837,6 @@ int radeon_gpu_reset(struct radeon_device *rdev)/* reset hpd state */ radeon_hpd_init (rdev); -ttm_bo_unlock_delayed_workqueue (&rdev->mman.bdev, resched); - rdev->in_reset = true; rdev->needs_reset = false;
drivers/gpu/drm/radeon/radeon_pm.c +1 -3
@@ -1853,11 +1853,10 @@ static bool radeon_pm_debug_check_in_vbl(struct radeon_device *rdev, bool finishstatic void radeon_dynpm_idle_work_handler (struct work_struct *work) {struct radeon_device *rdev; -int resched; + rdev = container_of(work, struct radeon_device, pm.dynpm_idle_work.work); - resched =ttm_bo_lock_delayed_workqueue (&rdev->mman.bdev);mutex_lock (&rdev->pm.mutex);if (rdev->pm.dynpm_state ==DYNPM_STATE_ACTIVE ) {int not_processed =0 ;
@@ -1908,7 +1907,6 @@ static void radeon_dynpm_idle_work_handler(struct work_struct *work) msecs_to_jiffies(RADEON_IDLE_LOOP_MS)); }mutex_unlock (&rdev->pm.mutex); -ttm_bo_unlock_delayed_workqueue (&rdev->mman.bdev, resched); }/*
drivers/gpu/drm/ttm/ttm_bo.c +0 -14
@@ -418,20 +418,6 @@ void ttm_bo_put(struct ttm_buffer_object *bo) }EXPORT_SYMBOL (ttm_bo_put); -int ttm_bo_lock_delayed_workqueue (struct ttm_device *bdev) -{ -return cancel_delayed_work_sync (&bdev->wq); -} -EXPORT_SYMBOL (ttm_bo_lock_delayed_workqueue); - -void ttm_bo_unlock_delayed_workqueue (struct ttm_device *bdev,int resched) -{ -if (resched) -schedule_delayed_work (&bdev->wq, - ((HZ /100 ) <1 ) ?1 :HZ /100 ); -} -EXPORT_SYMBOL (ttm_bo_unlock_delayed_workqueue); - static int ttm_bo_bounce_temp_buffer(struct ttm_buffer_object *bo, struct ttm_resource **mem, struct ttm_operation_ctx *ctx,
include/drm/ttm/ttm_bo_api.h +0 -16
@@ -290,22 +290,6 @@ void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo);void ttm_bo_set_bulk_move (struct ttm_buffer_object *bo,struct ttm_lru_bulk_move *bulk); -/** -* ttm_bo_lock_delayed_workqueue -* -* Prevent the delayed workqueue from running. -* Returns -* True if the workqueue was queued at the time -*/ -int ttm_bo_lock_delayed_workqueue (struct ttm_device *bdev); - -/** -* ttm_bo_unlock_delayed_workqueue -* -* Allows the delayed workqueue to run. -*/ -void ttm_bo_unlock_delayed_workqueue (struct ttm_device *bdev,int resched); -/** * ttm_bo_eviction_valuable *
Signature · 1 line
2.34.1
PATCH 2/3 drm/ttm: use per BO cleanup workers
Instead of a single worker going over the list of delete BOs in regular intervals use a per BO worker which blocks for the resv object and locking of the BO.
This not only simplifies the handling massively, but also results in much better response time when cleaning up buffers.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 2 +- drivers/gpu/drm/i915/i915_gem.c | 2 +- drivers/gpu/drm/i915/intel_region_ttm.c | 2 +- drivers/gpu/drm/ttm/ttm_bo.c | 112 ++++++++------------- drivers/gpu/drm/ttm/ttm_bo_util.c | 1 - drivers/gpu/drm/ttm/ttm_device.c | 24 ++--- include/drm/ttm/ttm_bo_api.h | 18 +--- include/drm/ttm/ttm_device.h | 7 +- 8 files changed, 57 insertions(+), 111 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1 -1
@@ -3966,7 +3966,7 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev)amdgpu_fence_driver_hw_fini (adev);if (adev->mman.initialized) -flush_delayed_work (&adev->mman.bdev.wq); +drain_workqueue (adev->mman.bdev.wq);if (adev->pm_sysfs_en)amdgpu_pm_sysfs_fini (adev);
drivers/gpu/drm/i915/i915_gem.c +1 -1
@@ -1099,7 +1099,7 @@ void i915_gem_drain_freed_objects(struct drm_i915_private *i915) {while (atomic_read (&i915->mm.free_count)) {flush_work (&i915->mm.free_work); -flush_delayed_work (&i915->bdev.wq); +drain_workqueue (i915->bdev.wq);rcu_barrier (); } }
drivers/gpu/drm/i915/intel_region_ttm.c +1 -1
@@ -132,7 +132,7 @@ int intel_region_ttm_fini(struct intel_memory_region *mem)break ;msleep (20 ); -flush_delayed_work (&mem->i915->bdev.wq); +drain_workqueue (mem->i915->bdev.wq); }/* If we leaked objects, Don't free the region causing use after free */
drivers/gpu/drm/ttm/ttm_bo.c +40 -72
152 lines not on this page · raw message
drivers/gpu/drm/ttm/ttm_bo_util.c +0 -1
@@ -230,7 +230,6 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo,*/ atomic_inc (&ttm_glob.bo_count); -INIT_LIST_HEAD (&fbo->base.ddestroy);drm_vma_node_reset (&fbo->base.base.vma_node);kref_init (&fbo->base.kref);
drivers/gpu/drm/ttm/ttm_device.c +8 -16
@@ -175,16 +175,6 @@ int ttm_device_swapout(struct ttm_device *bdev, struct ttm_operation_ctx *ctx, }EXPORT_SYMBOL (ttm_device_swapout); -static void ttm_device_delayed_workqueue (struct work_struct *work) -{ -struct ttm_device *bdev = - container_of(work, struct ttm_device, wq.work); - -if (!ttm_bo_delayed_delete (bdev, false)) -schedule_delayed_work (&bdev->wq, - ((HZ /100 ) <1 ) ?1 :HZ /100 ); -} -/** * ttm_device_init *
@@ -215,15 +205,19 @@ int ttm_device_init(struct ttm_device *bdev, struct ttm_device_funcs *funcs,if (ret)return ret; + bdev->wq =alloc_workqueue ("ttm" ,WQ_MEM_RECLAIM |WQ_HIGHPRI ,16 ); +if (!bdev->wq) { +ttm_global_release (); +return -ENOMEM ; + } + bdev->funcs = funcs;ttm_sys_man_init (bdev);ttm_pool_init (&bdev->pool, dev, use_dma_alloc, use_dma32); bdev->vma_manager = vma_manager; -INIT_DELAYED_WORK (&bdev->wq, ttm_device_delayed_workqueue);spin_lock_init (&bdev->lru_lock); -INIT_LIST_HEAD (&bdev->ddestroy);INIT_LIST_HEAD (&bdev->pinned); bdev->dev_mapping = mapping;mutex_lock (&ttm_global_mutex);
@@ -247,10 +241,8 @@ void ttm_device_fini(struct ttm_device *bdev)list_del (&bdev->device_list);mutex_unlock (&ttm_global_mutex); -cancel_delayed_work_sync (&bdev->wq); - -if (ttm_bo_delayed_delete (bdev, true)) -pr_debug ("Delayed destroy list was clean\n" ); +drain_workqueue (bdev->wq); +destroy_workqueue (bdev->wq);spin_lock (&bdev->lru_lock);for (i =0 ; i <TTM_MAX_BO_PRIORITY ; ++i)
include/drm/ttm/ttm_bo_api.h +5 -13
@@ -92,7 +92,6 @@ struct ttm_tt;* @ttm: TTM structure holding system pages. * @evicted: Whether the object was evicted without user-space knowing. * @deleted: True if the object is only a zombie and already deleted. -* @ddestroy: List head for the delayed destroy list. * @swap: List head for swap LRU list. * @offset: The current GPU offset, which can have different meanings * depending on the memory type. For SYSTEM type memory, it should be 0.
@@ -135,19 +134,14 @@ struct ttm_buffer_object {struct ttm_tt *ttm;bool deleted;struct ttm_lru_bulk_move *bulk_move; +unsigned priority; +unsigned pin_count;/** -* Members protected by the bdev::lru_lock. -*/ - -struct list_head ddestroy; - -/** -* Members protected by a bo reservation. +* @delayed_delete: Work item used when we can't delete the BO +* immediately */ - -unsigned priority; -unsigned pin_count; +struct work_struct delayed_delete;/** * Special members that are protected by the reserve lock
@@ -448,8 +442,6 @@ void ttm_bo_vm_close(struct vm_area_struct *vma);int ttm_bo_vm_access (struct vm_area_struct *vma,unsigned long addr,void *buf,int len,int write); -bool ttm_bo_delayed_delete (struct ttm_device *bdev,bool remove_all); -vm_fault_t ttm_bo_vm_dummy_page (struct vm_fault *vmf,pgprot_t prot);#endif
include/drm/ttm/ttm_device.h +1 -6
@@ -251,11 +251,6 @@ struct ttm_device {*/ spinlock_t lru_lock; -/** -* @ddestroy: Destroyed but not yet cleaned up buffer objects. -*/ -struct list_head ddestroy; -/** * @pinned: Buffer objects which are pinned and so not on any LRU list. */
@@ -270,7 +265,7 @@ struct ttm_device {/** * @wq: Work queue structure for the delayed delete workqueue. */ -struct delayed_work wq; +struct workqueue_struct *wq; };int ttm_global_swapout (struct ttm_operation_ctx *ctx,gfp_t gfp_flags);
Signature · 1 line
2.34.1
PATCH 3/3 drm/amdgpu: generally allow over-commit during BO allocation
In reply to Christian KönigWe already fallback to a dummy BO with no backing store when we allocate GDS,GWS and OA resources and to GTT when we allocate VRAM.
Drop all those workarounds and generalize this for GTT as well. This fixes ENOMEM issues with runaway applications which try to allocate/free GTT in a loop and are otherwise only limited by the CPU speed.
The CS will wait for the cleanup of freed up BOs to satisfy the various domain specific limits and so effectively throttle those buggy applications down to a sane allocation behavior again.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c | 16 +++------------- drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 6 +----- 2 files changed, 4 insertions(+), 18 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +3 -13
@@ -112,7 +112,7 @@ int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size, bp.resv = resv; bp.preferred_domain = initial_domain; bp.flags = flags; - bp.domain = initial_domain; + bp.domain = initial_domain |AMDGPU_GEM_DOMAIN_CPU ; bp.bo_ptr_size =sizeof (struct amdgpu_bo ); r =amdgpu_bo_create_user (adev, &bp, &ubo);
@@ -331,20 +331,10 @@ int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data, } initial_domain = (u32)(0xffffffff & args->in.domains); -retry: r =amdgpu_gem_object_create (adev, size, args->in.alignment, - initial_domain, - flags, ttm_bo_type_device, resv, &gobj); + initial_domain, flags, ttm_bo_type_device, + resv, &gobj);if (r && r != -ERESTARTSYS ) { -if (flags &AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED ) { - flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED ; - goto retry; - } - -if (initial_domain ==AMDGPU_GEM_DOMAIN_VRAM ) { - initial_domain |=AMDGPU_GEM_DOMAIN_GTT ; - goto retry; - }DRM_DEBUG ("Failed to allocate GEM object (%llu, %d, %llu, %d)\n" , size, initial_domain, args->in.alignment, r); }
drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +1 -5
@@ -581,11 +581,7 @@ int amdgpu_bo_create(struct amdgpu_device *adev, bo->flags |=AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE ; bo->tbo.bdev = &adev->mman.bdev; -if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS |AMDGPU_GEM_DOMAIN_OA | -AMDGPU_GEM_DOMAIN_GDS )) -amdgpu_bo_placement_from_domain (bo,AMDGPU_GEM_DOMAIN_CPU ); -else -amdgpu_bo_placement_from_domain (bo, bp->domain); +amdgpu_bo_placement_from_domain (bo, bp->domain);if (bp->type == ttm_bo_type_kernel) bo->tbo.priority =1 ;
Signature · 1 line
2.34.1
Quoted 6 lines · Christian König
Those functions never worked correctly since it is still perfectly possible that a buffer object is released and the background worker restarted even after calling them.
Signed-off-by Christian König <[email protected]>
I know you usually do, but just a friendly reminder to Cc: intel-gfx on the next revision or before merging, just so our CI can give the series a quick test. Thanks.