drm+msm: Shrinker and LRU rework
Subsystems (5)
v3 cover drm+msm: Shrinker and LRU rework
From: Rob Clark <[email protected]>
Mostly a resend with switch to lockdep asserts for the GEM LRU helper and conversion of msm locking checks to lockdep asserts, since that seemed like a good idea.
original description below:
This is mostly motivated by getting drm/msm to pass an i-g-t shrinker test that I've been working on. In particular the test creates and cycles between more GEM buffers than what will fit in RAM to force eviction and re-pin. (There are sub-tests that cover this case both single threaded and with many child processes in parallel.)
Getting this test to pass necessitated a few improvements:
1. Re-ordering submit path to get rid of __GFP_NORETRY (in the common case, doing this for syncobjs is still TODO)
2. Decoupling locks needed in the retire path from locks that could be held while hitting reclaim in the submit path
3. If necessary, allow stalling on active BOs for reclaim.
The latter point is because we pin objects in the synchronous part of the submit path (before queuing on the drm gpu-scheduler), which means in the parallel variant of the i-g-t test, we need to be able to block in the reclaim path until some queued work has completed/retired.
In the process of re-working how drm/msm tracks buffer state in it's various LRU lists, I refactored out a drm_gem_lru helper which, in theory, should be usable by other drivers and drm shmem helpers for implementing LRU tracking and shrinker.
v2: rebase + small fix in 13/13
v3: use lockdep_assert_held in GEM LRU helper, and add a couple patches
at the end to convert MSM from WARN_ON(!is_locked()) to lockdep asserts
Rob Clark (15):
drm/msm: Reorder lock vs submit alloc drm/msm: Small submit cleanup
drm/msm: Split out idr_lock
drm/msm/gem: Check for active in shrinker path drm/msm/gem: Rename update_inactive drm/msm/gem: Rename to pin/unpin_pages drm/msm/gem: Consolidate pin/unpin paths drm/msm/gem: Remove active refcnt
drm/gem: Add LRU/shrinker helper drm/msm/gem: Convert to using drm_gem_lru drm/msm/gem: Unpin buffers earlier drm/msm/gem: Consolidate shrinker trace drm/msm/gem: Evict active GEM objects when necessary drm/msm/gem: Add msm_gem_assert_locked() drm/msm/gem: Convert to lockdep assert
drivers/gpu/drm/drm_gem.c | 183 +++++++++++++++++++++++++ drivers/gpu/drm/msm/msm_drv.c | 18 ++- drivers/gpu/drm/msm/msm_drv.h | 70 +++++++--- drivers/gpu/drm/msm/msm_gem.c | 179 +++++++++--------------- drivers/gpu/drm/msm/msm_gem.h | 123 ++--------------- drivers/gpu/drm/msm/msm_gem_prime.c | 4 +- drivers/gpu/drm/msm/msm_gem_shrinker.c | 164 ++++++++++------------ drivers/gpu/drm/msm/msm_gem_submit.c | 78 ++++------- drivers/gpu/drm/msm/msm_gpu.c | 3 - drivers/gpu/drm/msm/msm_gpu.h | 10 +- drivers/gpu/drm/msm/msm_gpu_trace.h | 36 +++-- drivers/gpu/drm/msm/msm_submitqueue.c | 1 + include/drm/drm_gem.h | 56 ++++++++ 13 files changed, 505 insertions(+), 420 deletions(-)
Signature · 1 line
2.36.1
PATCH v3 1/15 drm/msm: Reorder lock vs submit alloc
From: Rob Clark <[email protected]>
This lets us drop the NORETRY.
--- drivers/gpu/drm/msm/msm_gem_submit.c | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-)
drivers/gpu/drm/msm/msm_gem_submit.c +10 -14
@@ -36,7 +36,7 @@ static struct msm_gem_submit *submit_create(struct drm_device *dev,if (sz >SIZE_MAX )return ERR_PTR (-ENOMEM ); - submit =kzalloc (sz,GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY); + submit =kzalloc (sz,GFP_KERNEL );if (!submit)return ERR_PTR (-ENOMEM );
@@ -771,25 +771,21 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data,trace_msm_gpu_submit (pid_nr (pid), ring->id, submitid, args->nr_bos, args->nr_cmds); - ret =mutex_lock_interruptible (&queue->lock); -if (ret) - goto out_post_unlock; -if (args->flags &MSM_SUBMIT_FENCE_FD_OUT ) { out_fence_fd =get_unused_fd_flags (O_CLOEXEC );if (out_fence_fd <0 ) { ret = out_fence_fd; - goto out_unlock; +return ret; } } - submit =submit_create (dev, gpu, queue, args->nr_bos, - args->nr_cmds); -if (IS_ERR (submit)) { - ret =PTR_ERR (submit); - submit =NULL ; - goto out_unlock; - } + submit =submit_create (dev, gpu, queue, args->nr_bos, args->nr_cmds); +if (IS_ERR (submit)) +return PTR_ERR (submit); + + ret =mutex_lock_interruptible (&queue->lock); +if (ret) + goto out_post_unlock; submit->pid = pid; submit->ident = submitid;
@@ -965,9 +961,9 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data,if (ret && (out_fence_fd >=0 ))put_unused_fd (out_fence_fd);mutex_unlock (&queue->lock); +out_post_unlock:if (submit)msm_gem_submit_put (submit); -out_post_unlock:if (!IS_ERR_OR_NULL (post_deps)) {for (i =0 ; i < args->nr_out_syncobjs; ++i) {kfree (post_deps[i].chain);
Signature · 1 line
2.36.1
PATCH v3 2/15 drm/msm: Small submit cleanup
In reply to Rob ClarkFrom: Rob Clark <[email protected]>
Move more initialization into submit_create().
--- drivers/gpu/drm/msm/msm_gem_submit.c | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-)
drivers/gpu/drm/msm/msm_gem_submit.c +9 -11
@@ -26,6 +26,7 @@ static struct msm_gem_submit *submit_create(struct drm_device *dev, struct msm_gpu_submitqueue *queue, uint32_t nr_bos, uint32_t nr_cmds) { + static atomic_t ident = ATOMIC_INIT(0);struct msm_gem_submit *submit;uint64_t sz;int ret;
@@ -52,9 +53,13 @@ static struct msm_gem_submit *submit_create(struct drm_device *dev, submit->gpu = gpu; submit->cmd = (void *)&submit->bos[nr_bos]; submit->queue = queue; + submit->pid =get_pid (task_pid (current)); submit->ring = gpu->rb[queue->ring_nr]; submit->fault_dumped = false; +/* Get a unique identifier for the submission for logging purposes */ + submit->ident =atomic_inc_return (&ident) -1 ; +INIT_LIST_HEAD (&submit->node);return submit;
@@ -718,7 +723,6 @@ static void msm_process_post_deps(struct msm_submit_post_dep *post_deps,int msm_ioctl_gem_submit (struct drm_device *dev,void *data,struct drm_file *file) { -static atomic_t ident =ATOMIC_INIT (0 );struct msm_drm_private *priv = dev->dev_private;struct drm_msm_gem_submit *args = data;struct msm_file_private *ctx = file->driver_priv;
@@ -729,10 +733,9 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data,struct msm_submit_post_dep *post_deps =NULL ;struct drm_syncobj **syncobjs_to_reset =NULL ;int out_fence_fd =-1 ; -struct pid *pid =get_pid (task_pid (current));bool has_ww_ticket = false;unsigned i; -int ret, submitid; +int ret;if (!gpu)return -ENXIO ;
@@ -764,12 +767,7 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data,if (!queue)return -ENOENT ; -/* Get a unique identifier for the submission for logging purposes */ - submitid =atomic_inc_return (&ident) -1 ; - ring = gpu->rb[queue->ring_nr]; -trace_msm_gpu_submit (pid_nr (pid), ring->id, submitid, - args->nr_bos, args->nr_cmds);if (args->flags &MSM_SUBMIT_FENCE_FD_OUT ) { out_fence_fd =get_unused_fd_flags (O_CLOEXEC );
@@ -783,13 +781,13 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data,if (IS_ERR (submit))return PTR_ERR (submit); +trace_msm_gpu_submit (pid_nr (submit->pid), ring->id, submit->ident, + args->nr_bos, args->nr_cmds); + ret =mutex_lock_interruptible (&queue->lock);if (ret) goto out_post_unlock; - submit->pid = pid; - submit->ident = submitid; -if (args->flags &MSM_SUBMIT_SUDO ) submit->in_rb = true;
Signature · 1 line
2.36.1
PATCH v3 3/15 drm/msm: Split out idr_lock
In reply to Rob ClarkFrom: Rob Clark <[email protected]>
Otherwise if we hit reclaim pinning objects in the submit path, we'll be blocking retire_worker trying to free a submit.
--- drivers/gpu/drm/msm/msm_drv.c | 4 ++-- drivers/gpu/drm/msm/msm_gem_submit.c | 10 ++++++++-- drivers/gpu/drm/msm/msm_gpu.h | 4 +++- drivers/gpu/drm/msm/msm_submitqueue.c | 1 + 4 files changed, 14 insertions(+), 5 deletions(-)
drivers/gpu/drm/msm/msm_drv.c +2 -2
@@ -883,13 +883,13 @@ static int wait_fence(struct msm_gpu_submitqueue *queue, uint32_t fence_id,* retired, so if the fence is not found it means there is nothing * to wait for */ - ret =mutex_lock_interruptible (&queue->lock); + ret =mutex_lock_interruptible (&queue->idr_lock);if (ret)return ret; fence =idr_find (&queue->fence_idr, fence_id);if (fence) fence =dma_fence_get_rcu (fence); -mutex_unlock (&queue->lock); +mutex_unlock (&queue->idr_lock);if (!fence)return 0 ;
drivers/gpu/drm/msm/msm_gem_submit.c +8 -2
@@ -72,9 +72,9 @@ void __msm_gem_submit_destroy(struct kref *kref)unsigned i;if (submit->fence_id) { -mutex_lock (&submit->queue->lock); +mutex_lock (&submit->queue->idr_lock);idr_remove (&submit->queue->fence_idr, submit->fence_id); -mutex_unlock (&submit->queue->lock); +mutex_unlock (&submit->queue->idr_lock); }dma_fence_put (submit->user_fence);
@@ -881,6 +881,8 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data, submit->nr_cmds = i; +mutex_lock (&queue->idr_lock); +/* * If using userspace provided seqno fence, validate that the id * is available before arming sched job. Since access to fence_idr
@@ -889,6 +891,7 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data,*/ if ((args->flags &MSM_SUBMIT_FENCE_SN_IN ) &&idr_find (&queue->fence_idr, args->fence)) { +mutex_unlock (&queue->idr_lock); ret = -EINVAL ; goto out; }
@@ -921,6 +924,9 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data, submit->user_fence,1 , INT_MAX, GFP_KERNEL); } + +mutex_unlock (&queue->idr_lock); +if (submit->fence_id <0 ) { ret = submit->fence_id; submit->fence_id =0 ;
drivers/gpu/drm/msm/msm_gpu.h +3 -1
@@ -466,7 +466,8 @@ static inline int msm_gpu_convert_priority(struct msm_gpu *gpu, int prio,* @node: node in the context's list of submitqueues * @fence_idr: maps fence-id to dma_fence for userspace visible fence * seqno, protected by submitqueue lock -* @lock: submitqueue lock +* @idr_lock: for serializing access to fence_idr +* @lock: submitqueue lock for serializing submits on a queue * @ref: reference count * @entity: the submit job-queue */
@@ -479,6 +480,7 @@ struct msm_gpu_submitqueue {struct msm_file_private *ctx;struct list_head node;struct idr fence_idr; +struct mutex idr_lock;struct mutex lock;struct kref ref;struct drm_sched_entity *entity;
drivers/gpu/drm/msm/msm_submitqueue.c +1 -0
@@ -200,6 +200,7 @@ int msm_submitqueue_create(struct drm_device *drm, struct msm_file_private *ctx, *id = queue->id;idr_init (&queue->fence_idr); +mutex_init (&queue->idr_lock);mutex_init (&queue->lock);list_add_tail (&queue->node, &ctx->submitqueues);
Signature · 1 line
2.36.1
PATCH v3 4/15 drm/msm/gem: Check for active in shrinker path
In reply to Rob ClarkFrom: Rob Clark <[email protected]>
Currently in our shrinker path we shouldn't be encountering anything that is active, but this will change in subsequent patches. So check if there are unsignaled fences.
--- drivers/gpu/drm/msm/msm_gem.c | 10 ++++++++++ drivers/gpu/drm/msm/msm_gem.h | 1 + drivers/gpu/drm/msm/msm_gem_shrinker.c | 6 ++++++ 3 files changed, 17 insertions(+)
drivers/gpu/drm/msm/msm_gem.c +10 -0
@@ -870,6 +870,16 @@ static void update_inactive(struct msm_gem_object *msm_obj)mutex_unlock (&priv->mm_lock); } +bool msm_gem_active (struct drm_gem_object *obj) +{ +GEM_WARN_ON (!msm_gem_is_locked (obj)); + +if (to_msm_bo (obj)->pin_count) +return true; + +return !dma_resv_test_signaled (obj->resv,dma_resv_usage_rw (true)); +} +int msm_gem_cpu_prep (struct drm_gem_object *obj,uint32_t op,ktime_t *timeout) {bool write = !!(op &MSM_PREP_WRITE );
drivers/gpu/drm/msm/msm_gem.h +1 -0
@@ -173,6 +173,7 @@ void msm_gem_put_vaddr(struct drm_gem_object *obj);int msm_gem_madvise (struct drm_gem_object *obj,unsigned madv);void msm_gem_active_get (struct drm_gem_object *obj,struct msm_gpu *gpu);void msm_gem_active_put (struct drm_gem_object *obj); +bool msm_gem_active (struct drm_gem_object *obj);int msm_gem_cpu_prep (struct drm_gem_object *obj,uint32_t op,ktime_t *timeout);int msm_gem_cpu_fini (struct drm_gem_object *obj); int msm_gem_new_handle(struct drm_device *dev, struct drm_file *file,
drivers/gpu/drm/msm/msm_gem_shrinker.c +6 -0
@@ -43,6 +43,9 @@ purge(struct msm_gem_object *msm_obj)if (!is_purgeable (msm_obj))return false; +if (msm_gem_active (&msm_obj->base)) +return false; +/* * This will move the obj out of still_in_list to * the purged list
@@ -58,6 +61,9 @@ evict(struct msm_gem_object *msm_obj)if (is_unevictable (msm_obj))return false; +if (msm_gem_active (&msm_obj->base)) +return false; +msm_gem_evict (&msm_obj->base);return true;
Signature · 1 line
2.36.1
PATCH v3 5/15 drm/msm/gem: Rename update_inactive
In reply to Rob ClarkFrom: Rob Clark <[email protected]>
Really what this is doing is updating various LRU lists.
--- drivers/gpu/drm/msm/msm_gem.c | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-)
drivers/gpu/drm/msm/msm_gem.c +13 -12
@@ -19,7 +19,7 @@#include "msm_gpu.h" #include "msm_mmu.h" -static void update_inactive (struct msm_gem_object *msm_obj); +static void update_lru (struct drm_gem_object *obj);static dma_addr_t physaddr (struct drm_gem_object *obj) {
@@ -132,7 +132,7 @@ static struct page **get_pages(struct drm_gem_object *obj)if (msm_obj->flags &MSM_BO_WC )sync_for_device (msm_obj); -update_inactive (msm_obj); +update_lru (obj); }return msm_obj->pages;
@@ -193,7 +193,7 @@ struct page **msm_gem_get_pages(struct drm_gem_object *obj)if (!IS_ERR (p)) { msm_obj->pin_count++; -update_inactive (msm_obj); +update_lru (obj); }msm_gem_unlock (obj);
@@ -207,7 +207,7 @@ void msm_gem_put_pages(struct drm_gem_object *obj)msm_gem_lock (obj); msm_obj->pin_count--;GEM_WARN_ON (msm_obj->pin_count <0 ); -update_inactive (msm_obj); +update_lru (obj);msm_gem_unlock (obj); }
@@ -449,7 +449,7 @@ void msm_gem_unpin_locked(struct drm_gem_object *obj) msm_obj->pin_count--;GEM_WARN_ON (msm_obj->pin_count <0 ); -update_inactive (msm_obj); +update_lru (obj); } struct msm_gem_vma *msm_gem_get_vma_locked(struct drm_gem_object *obj,
@@ -658,7 +658,7 @@ static void *get_vaddr(struct drm_gem_object *obj, unsigned madv) goto fail; } -update_inactive (msm_obj); +update_lru (obj); }return msm_obj->vaddr;
@@ -730,7 +730,7 @@ int msm_gem_madvise(struct drm_gem_object *obj, unsigned madv)* between inactive lists */ if (msm_obj->active_count ==0 ) -update_inactive (msm_obj); +update_lru (obj);msm_gem_unlock (obj);
@@ -757,7 +757,7 @@ void msm_gem_purge(struct drm_gem_object *obj)put_iova_vmas (obj); msm_obj->madv = __MSM_MADV_PURGED; -update_inactive (msm_obj); +update_lru (obj);drm_gem_free_mmap_offset (obj);
@@ -792,7 +792,7 @@ void msm_gem_evict(struct drm_gem_object *obj)put_pages (obj); -update_inactive (msm_obj); +update_lru (obj); }void msm_gem_vunmap (struct drm_gem_object *obj)
@@ -835,13 +835,14 @@ void msm_gem_active_put(struct drm_gem_object *obj)GEM_WARN_ON (!msm_gem_is_locked (obj));if (--msm_obj->active_count ==0 ) { -update_inactive (msm_obj); +update_lru (obj); } } -static void update_inactive (struct msm_gem_object *msm_obj) +static void update_lru (struct drm_gem_object *obj) { -struct msm_drm_private *priv = msm_obj->base.dev->dev_private; +struct msm_drm_private *priv = obj->dev->dev_private; +struct msm_gem_object *msm_obj =to_msm_bo (obj);GEM_WARN_ON (!msm_gem_is_locked (&msm_obj->base));
Signature · 1 line
2.36.1
PATCH v3 6/15 drm/msm/gem: Rename to pin/unpin_pages
In reply to Rob ClarkFrom: Rob Clark <[email protected]>
Since that is what these fxns actually do.. they are getting *pinned* pages (as opposed to cases where we need pages, but don't need them pinned, like CPU mappings).
--- drivers/gpu/drm/msm/msm_gem.c | 18 +++++++++++++----- drivers/gpu/drm/msm/msm_gem.h | 4 ++-- drivers/gpu/drm/msm/msm_gem_prime.c | 4 ++-- 3 files changed, 17 insertions(+), 9 deletions(-)
drivers/gpu/drm/msm/msm_gem.c +13 -5
@@ -177,30 +177,38 @@ static void put_pages(struct drm_gem_object *obj) } } -struct page **msm_gem_get_pages (struct drm_gem_object *obj) +static struct page **msm_gem_pin_pages_locked (struct drm_gem_object *obj) {struct msm_gem_object *msm_obj =to_msm_bo (obj);struct page **p; -msm_gem_lock (obj); +GEM_WARN_ON (!msm_gem_is_locked (obj));if (GEM_WARN_ON (msm_obj->madv !=MSM_MADV_WILLNEED )) { -msm_gem_unlock (obj);return ERR_PTR (-EBUSY ); } p =get_pages (obj); -if (!IS_ERR (p)) { msm_obj->pin_count++;update_lru (obj); } +return p; +} + +struct page **msm_gem_pin_pages (struct drm_gem_object *obj) +{ +struct page **p; + +msm_gem_lock (obj); + p =msm_gem_pin_pages_locked (obj);msm_gem_unlock (obj); +return p; } -void msm_gem_put_pages (struct drm_gem_object *obj) +void msm_gem_unpin_pages (struct drm_gem_object *obj) {struct msm_gem_object *msm_obj =to_msm_bo (obj);
drivers/gpu/drm/msm/msm_gem.h +2 -2
@@ -159,8 +159,8 @@ int msm_gem_get_and_pin_iova(struct drm_gem_object *obj, struct msm_gem_address_space *aspace, uint64_t *iova);void msm_gem_unpin_iova (struct drm_gem_object *obj,struct msm_gem_address_space *aspace); -struct page **msm_gem_get_pages (struct drm_gem_object *obj); -void msm_gem_put_pages (struct drm_gem_object *obj); +struct page **msm_gem_pin_pages (struct drm_gem_object *obj); +void msm_gem_unpin_pages (struct drm_gem_object *obj);int msm_gem_dumb_create (struct drm_file *file,struct drm_device *dev,struct drm_mode_create_dumb *args); int msm_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev,
drivers/gpu/drm/msm/msm_gem_prime.c +2 -2
@@ -63,12 +63,12 @@ struct drm_gem_object *msm_gem_prime_import_sg_table(struct drm_device *dev,int msm_gem_prime_pin (struct drm_gem_object *obj) {if (!obj->import_attach) -msm_gem_get_pages (obj); +msm_gem_pin_pages (obj);return 0 ; }void msm_gem_prime_unpin (struct drm_gem_object *obj) {if (!obj->import_attach) -msm_gem_put_pages (obj); +msm_gem_unpin_pages (obj); }
Signature · 1 line
2.36.1
PATCH v3 7/15 drm/msm/gem: Consolidate pin/unpin paths
In reply to Rob ClarkFrom: Rob Clark <[email protected]>
Avoid having multiple spots where we increment/decrement pin_count (and associated LRU updating)
--- drivers/gpu/drm/msm/msm_gem.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-)
drivers/gpu/drm/msm/msm_gem.c +5 -8
@@ -190,7 +190,7 @@ static struct page **msm_gem_pin_pages_locked(struct drm_gem_object *obj) p =get_pages (obj);if (!IS_ERR (p)) { - msm_obj->pin_count++; +to_msm_bo (obj)->pin_count++;update_lru (obj); }
@@ -213,9 +213,7 @@ void msm_gem_unpin_pages(struct drm_gem_object *obj)struct msm_gem_object *msm_obj =to_msm_bo (obj);msm_gem_lock (obj); - msm_obj->pin_count--; -GEM_WARN_ON (msm_obj->pin_count <0 ); -update_lru (obj); +msm_gem_unpin_locked (obj);msm_gem_unlock (obj); }
@@ -436,14 +434,13 @@ int msm_gem_pin_vma_locked(struct drm_gem_object *obj, struct msm_gem_vma *vma)if (GEM_WARN_ON (msm_obj->madv !=MSM_MADV_WILLNEED ))return -EBUSY ; - pages =get_pages (obj); + pages =msm_gem_pin_pages_locked (obj);if (IS_ERR (pages))return PTR_ERR (pages); ret =msm_gem_map_vma (vma->aspace, vma, prot, msm_obj->sgt, obj->size); - -if (!ret) - msm_obj->pin_count++; +if (ret) +msm_gem_unpin_locked (obj);return ret; }
Signature · 1 line
2.36.1
PATCH v3 8/15 drm/msm/gem: Remove active refcnt
In reply to Rob ClarkFrom: Rob Clark <[email protected]>
At this point the pinned refcnt is sufficient, and the shrinker is already prepared to encounter objects which are still active according to fences attached to the resv.
--- drivers/gpu/drm/msm/msm_gem.c | 45 ++-------------------------- drivers/gpu/drm/msm/msm_gem.h | 14 ++------- drivers/gpu/drm/msm/msm_gem_submit.c | 22 ++------------ 3 files changed, 8 insertions(+), 73 deletions(-)
drivers/gpu/drm/msm/msm_gem.c +3 -42
@@ -734,8 +734,7 @@ int msm_gem_madvise(struct drm_gem_object *obj, unsigned madv)/* If the obj is inactive, we might need to move it * between inactive lists */ -if (msm_obj->active_count ==0 ) -update_lru (obj); +update_lru (obj);msm_gem_unlock (obj);
@@ -788,7 +787,6 @@ void msm_gem_evict(struct drm_gem_object *obj)GEM_WARN_ON (!msm_gem_is_locked (obj));GEM_WARN_ON (is_unevictable (msm_obj));GEM_WARN_ON (!msm_obj->evictable); -GEM_WARN_ON (msm_obj->active_count);/* Get rid of any iommu mapping(s): */ put_iova_spaces (obj, false);
@@ -813,37 +811,6 @@ void msm_gem_vunmap(struct drm_gem_object *obj) msm_obj->vaddr =NULL ; } -void msm_gem_active_get (struct drm_gem_object *obj,struct msm_gpu *gpu) -{ -struct msm_gem_object *msm_obj =to_msm_bo (obj); -struct msm_drm_private *priv = obj->dev->dev_private; - -might_sleep (); -GEM_WARN_ON (!msm_gem_is_locked (obj)); -GEM_WARN_ON (msm_obj->madv !=MSM_MADV_WILLNEED ); -GEM_WARN_ON (msm_obj->dontneed); - -if (msm_obj->active_count++ ==0 ) { -mutex_lock (&priv->mm_lock); -if (msm_obj->evictable) -mark_unevictable (msm_obj); -list_move_tail (&msm_obj->mm_list, &gpu->active_list); -mutex_unlock (&priv->mm_lock); - } -} - -void msm_gem_active_put (struct drm_gem_object *obj) -{ -struct msm_gem_object *msm_obj =to_msm_bo (obj); - -might_sleep (); -GEM_WARN_ON (!msm_gem_is_locked (obj)); - -if (--msm_obj->active_count ==0 ) { -update_lru (obj); - } -} -static void update_lru (struct drm_gem_object *obj) {struct msm_drm_private *priv = obj->dev->dev_private;
@@ -851,9 +818,6 @@ static void update_lru(struct drm_gem_object *obj)GEM_WARN_ON (!msm_gem_is_locked (&msm_obj->base)); -if (msm_obj->active_count !=0 ) -return ; -mutex_lock (&priv->mm_lock);if (msm_obj->dontneed)
@@ -926,7 +890,7 @@ void msm_gem_describe(struct drm_gem_object *obj, struct seq_file *m, stats->all.count++; stats->all.size += obj->size; -if (is_active (msm_obj)) { +if (msm_gem_active (obj)) { stats->active.count++; stats->active.size += obj->size; }
@@ -954,7 +918,7 @@ void msm_gem_describe(struct drm_gem_object *obj, struct seq_file *m, }seq_printf (m,"%08x: %c %2d (%2d) %08llx %p" , - msm_obj->flags,is_active (msm_obj) ?'A' :'I' , + msm_obj->flags,msm_gem_active (obj) ?'A' :'I' , obj->name,kref_read (&obj->refcount), off, msm_obj->vaddr);
@@ -1037,9 +1001,6 @@ static void msm_gem_free_object(struct drm_gem_object *obj)list_del (&msm_obj->mm_list);mutex_unlock (&priv->mm_lock); -/* object should not be on active list: */ -GEM_WARN_ON (is_active (msm_obj)); -put_iova_spaces (obj, true);if (obj->import_attach) {
drivers/gpu/drm/msm/msm_gem.h +2 -12
@@ -138,7 +138,6 @@ struct msm_gem_object {char name[32 ];/* Identifier to print for the debugfs files */ -int active_count;int pin_count; };#define to_msm_bo (x) container_of(x, struct msm_gem_object, base)
@@ -171,8 +170,6 @@ void *msm_gem_get_vaddr_active(struct drm_gem_object *obj);void msm_gem_put_vaddr_locked (struct drm_gem_object *obj);void msm_gem_put_vaddr (struct drm_gem_object *obj);int msm_gem_madvise (struct drm_gem_object *obj,unsigned madv); -void msm_gem_active_get (struct drm_gem_object *obj,struct msm_gpu *gpu); -void msm_gem_active_put (struct drm_gem_object *obj);bool msm_gem_active (struct drm_gem_object *obj);int msm_gem_cpu_prep (struct drm_gem_object *obj,uint32_t op,ktime_t *timeout);int msm_gem_cpu_fini (struct drm_gem_object *obj);
@@ -245,12 +242,6 @@ msm_gem_is_locked(struct drm_gem_object *obj)return dma_resv_is_locked (obj->resv) || (kref_read (&obj->refcount) ==0 ); } -static inline bool is_active (struct msm_gem_object *msm_obj) -{ -GEM_WARN_ON (!msm_gem_is_locked (&msm_obj->base)); -return msm_obj->active_count; -} -/* imported/exported objects are not purgeable: */ static inline bool is_unpurgeable (struct msm_gem_object *msm_obj) {
@@ -391,9 +382,8 @@ struct msm_gem_submit {/* make sure these don't conflict w/ MSM_SUBMIT_BO_x */ #define BO_VALID 0x8000/* is current addr in cmdstream correct/valid? */ #define BO_LOCKED 0x4000/* obj lock is held */ -#define BO_ACTIVE 0x2000/* active refcnt is held */ -#define BO_OBJ_PINNED 0x1000/* obj (pages) is pinned and on active list */ -#define BO_VMA_PINNED 0x0800/* vma (virtual address) is pinned */ +#define BO_OBJ_PINNED 0x2000/* obj (pages) is pinned and on active list */ +#define BO_VMA_PINNED 0x1000/* vma (virtual address) is pinned */ uint32_t flags;union {struct msm_gem_object *obj;
drivers/gpu/drm/msm/msm_gem_submit.c +3 -19
@@ -243,17 +243,13 @@ static void submit_cleanup_bo(struct msm_gem_submit *submit, int i,if (flags &BO_OBJ_PINNED )msm_gem_unpin_locked (obj); -if (flags &BO_ACTIVE ) -msm_gem_active_put (obj); -if (flags &BO_LOCKED )dma_resv_unlock (obj->resv); }static void submit_unlock_unpin_bo (struct msm_gem_submit *submit,int i) { -unsigned cleanup_flags =BO_VMA_PINNED |BO_OBJ_PINNED | -BO_ACTIVE |BO_LOCKED ; +unsigned cleanup_flags =BO_VMA_PINNED |BO_OBJ_PINNED |BO_LOCKED ;submit_cleanup_bo (submit, i, cleanup_flags);if (!(submit->bos[i].flags &BO_VALID ))
@@ -358,18 +354,6 @@ static int submit_pin_objects(struct msm_gem_submit *submit) submit->valid = true; -/* -* Increment active_count first, so if under memory pressure, we -* don't inadvertently evict a bo needed by the submit in order -* to pin an earlier bo in the same submit. -*/ -for (i =0 ; i < submit->nr_bos; i++) { -struct drm_gem_object *obj = &submit->bos[i].obj->base; - -msm_gem_active_get (obj, submit->gpu); - submit->bos[i].flags |=BO_ACTIVE ; - } -for (i =0 ; i < submit->nr_bos; i++) {struct drm_gem_object *obj = &submit->bos[i].obj->base;struct msm_gem_vma *vma;
@@ -521,7 +505,7 @@ static void submit_cleanup(struct msm_gem_submit *submit, bool error)unsigned i;if (error) - cleanup_flags |=BO_VMA_PINNED |BO_OBJ_PINNED |BO_ACTIVE ; + cleanup_flags |=BO_VMA_PINNED |BO_OBJ_PINNED ;for (i =0 ; i < submit->nr_bos; i++) {struct msm_gem_object *msm_obj = submit->bos[i].obj;
@@ -540,7 +524,7 @@ void msm_submit_retire(struct msm_gem_submit *submit)msm_gem_lock (obj);/* Note, VMA already fence-unpinned before submit: */ -submit_cleanup_bo (submit, i,BO_OBJ_PINNED |BO_ACTIVE ); +submit_cleanup_bo (submit, i,BO_OBJ_PINNED );msm_gem_unlock (obj);drm_gem_object_put (obj); }
Signature · 1 line
2.36.1
PATCH v3 9/15 drm/gem: Add LRU/shrinker helper
In reply to Rob ClarkFrom: Rob Clark <[email protected]>
Add a simple LRU helper to assist with driver's shrinker implementation. It handles tracking the number of backing pages associated with a given LRU, and provides a helper to implement shrinker_scan.
A driver can use multiple LRU instances to track objects in various states, for example a dontneed LRU for purgeable objects, a willneed LRU for evictable objects, and an unpinned LRU for objects without backing pages.
All LRUs that the object can be moved between must share a single lock.
v2: lockdep_assert_held() instead of WARN_ON(!mutex_is_locked())
--- drivers/gpu/drm/drm_gem.c | 183 ++++++++++++++++++++++++++++++++++++++ include/drm/drm_gem.h | 56 ++++++++++++ 2 files changed, 239 insertions(+)
drivers/gpu/drm/drm_gem.c +183 -0
@@ -165,6 +165,7 @@ void drm_gem_private_object_init(struct drm_device *dev, obj->resv = &obj->_resv;drm_vma_node_reset (&obj->vma_node); +INIT_LIST_HEAD (&obj->lru_node); }EXPORT_SYMBOL (drm_gem_private_object_init);
@@ -951,6 +952,7 @@ drm_gem_object_release(struct drm_gem_object *obj)dma_resv_fini (&obj->_resv);drm_gem_free_mmap_offset (obj); +drm_gem_lru_remove (obj); }EXPORT_SYMBOL (drm_gem_object_release);
@@ -1274,3 +1276,184 @@ drm_gem_unlock_reservations(struct drm_gem_object **objs, int count,ww_acquire_fini (acquire_ctx); }EXPORT_SYMBOL (drm_gem_unlock_reservations); + +/** +* drm_gem_lru_init - initialize a LRU +* +* @lru: The LRU to initialize +* @lock: The lock protecting the LRU +*/ +void +drm_gem_lru_init (struct drm_gem_lru *lru,struct mutex *lock) +{ + lru->lock = lock; + lru->count =0 ; +INIT_LIST_HEAD (&lru->list); +} +EXPORT_SYMBOL (drm_gem_lru_init); + +static void +lru_remove (struct drm_gem_object *obj) +{ + obj->lru->count -= obj->size >>PAGE_SHIFT ; +WARN_ON (obj->lru->count <0 ); +list_del (&obj->lru_node); + obj->lru =NULL ; +} + +/** +* drm_gem_lru_remove - remove object from whatever LRU it is in +* +* If the object is currently in any LRU, remove it. +* +* @obj: The GEM object to remove from current LRU +*/ +void +drm_gem_lru_remove (struct drm_gem_object *obj) +{ +struct drm_gem_lru *lru = obj->lru; + +if (!lru) +return ; + +mutex_lock (lru->lock); +lru_remove (obj); +mutex_unlock (lru->lock); +} +EXPORT_SYMBOL (drm_gem_lru_remove); + +/** +* drm_gem_lru_move_tail - move the object to the tail of the LRU +* +* If the object is already in this LRU it will be moved to the +* tail. Otherwise it will be removed from whichever other LRU +* it is in (if any) and moved into this LRU. +* +* @lru: The LRU to move the object into. +* @obj: The GEM object to move into this LRU +*/ +void +drm_gem_lru_move_tail (struct drm_gem_lru *lru,struct drm_gem_object *obj) +{ +mutex_lock (lru->lock); +drm_gem_lru_move_tail_locked (lru, obj); +mutex_unlock (lru->lock); +} +EXPORT_SYMBOL (drm_gem_lru_move_tail); + +/** +* drm_gem_lru_move_tail_locked - move the object to the tail of the LRU +* +* If the object is already in this LRU it will be moved to the +* tail. Otherwise it will be removed from whichever other LRU +* it is in (if any) and moved into this LRU. +* +* Call with LRU lock held. +* +* @lru: The LRU to move the object into. +* @obj: The GEM object to move into this LRU +*/ +void +drm_gem_lru_move_tail_locked (struct drm_gem_lru *lru,struct drm_gem_object *obj) +{ +lockdep_assert_held_once (lru->lock); + +if (obj->lru) +lru_remove (obj); + + lru->count += obj->size >>PAGE_SHIFT ; +list_add_tail (&obj->lru_node, &lru->list); + obj->lru = lru; +} +EXPORT_SYMBOL (drm_gem_lru_move_tail_locked); + +/** +* drm_gem_lru_scan - helper to implement shrinker.scan_objects +* +* If the shrink callback succeeds, it is expected that the driver +* move the object out of this LRU. +* +* If the LRU possibly contain active buffers, it is the responsibility +* of the shrink callback to check for this (ie. dma_resv_test_signaled()) +* or if necessary block until the buffer becomes idle. +* +* @lru: The LRU to scan +* @nr_to_scan: The number of pages to try to reclaim +* @shrink: Callback to try to shrink/reclaim the object. +*/ +unsigned long +drm_gem_lru_scan (struct drm_gem_lru *lru,unsigned nr_to_scan, +bool (*shrink)(struct drm_gem_object *obj)) +{ +struct drm_gem_lru still_in_lru; +struct drm_gem_object *obj; +unsigned freed =0 ; + +drm_gem_lru_init (&still_in_lru, lru->lock); + +mutex_lock (lru->lock); + +while (freed < nr_to_scan) { + obj =list_first_entry_or_null (&lru->list,typeof (*obj), lru_node); + +if (!obj) +break ; + +drm_gem_lru_move_tail_locked (&still_in_lru, obj); + +/* +* If it's in the process of being freed, gem_object->free() +* may be blocked on lock waiting to remove it. So just +* skip it. +*/ +if (!kref_get_unless_zero (&obj->refcount)) +continue ; + +/* +* Now that we own a reference, we can drop the lock for the +* rest of the loop body, to reduce contention with other +* code paths that need the LRU lock +*/ +mutex_unlock (lru->lock); + +/* +* Note that this still needs to be trylock, since we can +* hit shrinker in response to trying to get backing pages +* for this obj (ie. while it's lock is already held) +*/ +if (!dma_resv_trylock (obj->resv)) + goto tail; + +if (shrink (obj)) { + freed += obj->size >>PAGE_SHIFT ; + +/* +* If we succeeded in releasing the object's backing +* pages, we expect the driver to have moved the object +* out of this LRU +*/ +WARN_ON (obj->lru == &still_in_lru); +WARN_ON (obj->lru == lru); + } + +dma_resv_unlock (obj->resv); + +tail: +drm_gem_object_put (obj); +mutex_lock (lru->lock); + } + +/* +* Move objects we've skipped over out of the temporary still_in_lru +* back into this LRU +*/ +list_for_each_entry (obj, &still_in_lru.list, lru_node) + obj->lru = lru; +list_splice_tail (&still_in_lru.list, &lru->list); + lru->count += still_in_lru.count; + +mutex_unlock (lru->lock); + +return freed; +} +EXPORT_SYMBOL (drm_gem_lru_scan);
include/drm/drm_gem.h +56 -0
72 lines not on this page · raw message
Signature · 1 line
2.36.1
Re: [PATCH v3 09/15] drm/gem: Add LRU/shrinker helper
Quoted 9 lines · Rob Clark
drivers/gpu/drm/drm_gem.c +9
-0+/** +* drm_gem_lru_move_tail_locked - move the object to the tail of the LRU +* +* If the object is already in this LRU it will be moved to the +* tail. Otherwise it will be removed from whichever other LRU +* it is in (if any) and moved into this LRU. +* +* Call with LRU lock held. +*
drivers/gpu/drm/drm_gem.c drm_gem_unlock_reservations(struct drm_gem_object **objs, int count, · line 1362
Quoted 10 lines
drivers/gpu/drm/drm_gem.c +10
-0+* @lru: The LRU to move the object into. +* @obj: The GEM object to move into this LRU +*/ +void +drm_gem_lru_move_tail_locked (struct drm_gem_lru *lru,struct drm_gem_object *obj) +{ +lockdep_assert_held_once (lru->lock); + +if (obj->lru) +lru_remove (obj);
The obj->lru also needs to be locked if lru != obj->lru, isn't it? And then we should add lockdep_assert_held_once(obj->lru->lock).
Signature · 2 lines
Best regards, Dmitry
Re: [PATCH v3 09/15] drm/gem: Add LRU/shrinker helper
Quoted 16 lines · Dmitry Osipenko
Quoted 14 lines · Rob Clark
drivers/gpu/drm/drm_gem.c +14
-0+/** +* drm_gem_lru_move_tail_locked - move the object to the tail of the LRU +* +* If the object is already in this LRU it will be moved to the +* tail. Otherwise it will be removed from whichever other LRU +* it is in (if any) and moved into this LRU. +* +* Call with LRU lock held. +* +* @lru: The LRU to move the object into. +* @obj: The GEM object to move into this LRU +*/ +void +drm_gem_lru_move_tail_locked (struct drm_gem_lru *lru,struct drm_gem_object *obj)
Quoted 9 lines
drivers/gpu/drm/drm_gem.c drm_gem_unlock_reservations(struct drm_gem_object **objs, int count, · line 1362
Quoted 5 lines
drivers/gpu/drm/drm_gem.c +5
-0+{ +lockdep_assert_held_once (lru->lock); + +if (obj->lru) +lru_remove (obj);The
obj->lrualso needs to be locked if lru !=obj->lru, isn't it? And then we should addlockdep_assert_held_once(obj->lru->lock).
It is expected (mentioned in comment on drm_gem_lru::lock) that all lru's are sharing the same lock. Possibly that could be made more obvious? Having per-lru locks wouldn't really work for accessing the single drm_gem_object::lru_node.
BR,
-R
Re: [PATCH v3 09/15] drm/gem: Add LRU/shrinker helper
Quoted 28 lines · Rob Clark
Quoted 16 lines · Dmitry Osipenko
Quoted 14 lines · Rob Clark
drivers/gpu/drm/drm_gem.c +14
-0+/** +* drm_gem_lru_move_tail_locked - move the object to the tail of the LRU +* +* If the object is already in this LRU it will be moved to the +* tail. Otherwise it will be removed from whichever other LRU +* it is in (if any) and moved into this LRU. +* +* Call with LRU lock held. +* +* @lru: The LRU to move the object into. +* @obj: The GEM object to move into this LRU +*/ +void +drm_gem_lru_move_tail_locked (struct drm_gem_lru *lru,struct drm_gem_object *obj)Quoted 9 lines
drivers/gpu/drm/drm_gem.c drm_gem_unlock_reservations(struct drm_gem_object **objs, int count, · line 1362
Quoted 5 lines
drivers/gpu/drm/drm_gem.c +5
-0+{ +lockdep_assert_held_once (lru->lock); + +if (obj->lru) +lru_remove (obj);The
obj->lrualso needs to be locked if lru !=obj->lru, isn't it? And then we should addlockdep_assert_held_once(obj->lru->lock).
Quoted 4 lines
It is expected (mentioned in comment on
drm_gem_lru::lock) that all lru's are sharing the same lock. Possibly that could be made more obvious? Having per-lru locks wouldn't really work for accessing the singledrm_gem_object::lru_node.
Right, my bad. I began to update the DRM-SHMEM shrinker patches on top of the shrinker helper, but missed that the lock is shared when was looking at this patch again today.
Adding comment to the code about the shared lock may help a tad, but it's not really necessary. It was my fault that I forgot about it.
Thank you!
Signature · 2 lines
Best regards, Dmitry
Re: [PATCH v3 09/15] drm/gem: Add LRU/shrinker helper
In reply to Rob ClarkQuoted 10 lines · Rob Clark
drivers/gpu/drm/drm_gem.c +10
-0+/** +* drm_gem_lru_remove - remove object from whatever LRU it is in +* +* If the object is currently in any LRU, remove it. +* +* @obj: The GEM object to remove from current LRU +*/ +void +drm_gem_lru_remove (struct drm_gem_object *obj) +{
drivers/gpu/drm/drm_gem.c drm_gem_unlock_reservations(struct drm_gem_object **objs, int count, · line 1323
Quoted 10 lines
drivers/gpu/drm/drm_gem.c +10
-0+struct drm_gem_lru *lru = obj->lru; + +if (!lru) +return ; + +mutex_lock (lru->lock); +lru_remove (obj); +mutex_unlock (lru->lock); +} +EXPORT_SYMBOL (drm_gem_lru_remove);
I made a preliminary port of the DRM-SHMEM shrinker on top of the the latest version of dma-buf locking convention and yours LRU patches. It all works good, the only thing that is missing for the DRM-SHMEM shrinker is the drm_gem_lru_remove_locked().
What about to add a locked variant of drm_gem_lru_remove()?
Signature · 2 lines
Best regards, Dmitry
Re: [PATCH v3 09/15] drm/gem: Add LRU/shrinker helper
Quoted 28 lines · Dmitry Osipenko
Quoted 10 lines · Rob Clark
drivers/gpu/drm/drm_gem.c +10
-0+/** +* drm_gem_lru_remove - remove object from whatever LRU it is in +* +* If the object is currently in any LRU, remove it. +* +* @obj: The GEM object to remove from current LRU +*/ +void +drm_gem_lru_remove (struct drm_gem_object *obj) +{drivers/gpu/drm/drm_gem.c drm_gem_unlock_reservations(struct drm_gem_object **objs, int count, · line 1323
Quoted 10 lines
drivers/gpu/drm/drm_gem.c +10
-0+struct drm_gem_lru *lru = obj->lru; + +if (!lru) +return ; + +mutex_lock (lru->lock); +lru_remove (obj); +mutex_unlock (lru->lock); +} +EXPORT_SYMBOL (drm_gem_lru_remove);I made a preliminary port of the DRM-SHMEM shrinker on top of the the latest version of dma-buf locking convention and yours LRU patches. It all works good, the only thing that is missing for the DRM-SHMEM shrinker is the
drm_gem_lru_remove_locked().
Quoted 1 line
What about to add a locked variant of
drm_gem_lru_remove()?
Sounds fine to me.. the only reason it didn't exist yet was because it wasn't needed yet..
I can respin w/ an addition of a _locked() version, or you can add it on top in your patchset. Either is fine by me
BR,
-R
Re: [PATCH v3 09/15] drm/gem: Add LRU/shrinker helper
Quoted 30 lines · Rob Clark
Quoted 28 lines · Dmitry Osipenko
Quoted 10 lines · Rob Clark
drivers/gpu/drm/drm_gem.c +10
-0+/** +* drm_gem_lru_remove - remove object from whatever LRU it is in +* +* If the object is currently in any LRU, remove it. +* +* @obj: The GEM object to remove from current LRU +*/ +void +drm_gem_lru_remove (struct drm_gem_object *obj) +{drivers/gpu/drm/drm_gem.c drm_gem_unlock_reservations(struct drm_gem_object **objs, int count, · line 1323
Quoted 10 lines
drivers/gpu/drm/drm_gem.c +10
-0+struct drm_gem_lru *lru = obj->lru; + +if (!lru) +return ; + +mutex_lock (lru->lock); +lru_remove (obj); +mutex_unlock (lru->lock); +} +EXPORT_SYMBOL (drm_gem_lru_remove);I made a preliminary port of the DRM-SHMEM shrinker on top of the the latest version of dma-buf locking convention and yours LRU patches. It all works good, the only thing that is missing for the DRM-SHMEM shrinker is the
drm_gem_lru_remove_locked().
Quoted 4 lines
Quoted 1 line
What about to add a locked variant of
drm_gem_lru_remove()?Sounds fine to me.. the only reason it didn't exist yet was because it wasn't needed yet..
There is no use for the drm_gem_lru_move_tail_locked() as well, you're not using it in the MSM driver. Hence I thought it might be good to add the drm_gem_lru_remove_locked(), or maybe the drm_gem_lru_move_tail_locked() should be dropped then?
Quoted 2 lines
I can respin w/ an addition of a
_locked()version, or you can add it on top in your patchset. Either is fine by me
The either option is fine by me too. If you'll keep the unused drm_gem_lru_move_tail_locked(), then will be nice to add drm_gem_lru_remove_locked().
Signature · 2 lines
Best regards, Dmitry
Re: [PATCH v3 09/15] drm/gem: Add LRU/shrinker helper
Quoted 41 lines · Dmitry Osipenko
Quoted 30 lines · Rob Clark
Quoted 28 lines · Dmitry Osipenko
Quoted 10 lines · Rob Clark
drivers/gpu/drm/drm_gem.c +10
-0+/** +* drm_gem_lru_remove - remove object from whatever LRU it is in +* +* If the object is currently in any LRU, remove it. +* +* @obj: The GEM object to remove from current LRU +*/ +void +drm_gem_lru_remove (struct drm_gem_object *obj) +{drivers/gpu/drm/drm_gem.c drm_gem_unlock_reservations(struct drm_gem_object **objs, int count, · line 1323
Quoted 10 lines
drivers/gpu/drm/drm_gem.c +10
-0+struct drm_gem_lru *lru = obj->lru; + +if (!lru) +return ; + +mutex_lock (lru->lock); +lru_remove (obj); +mutex_unlock (lru->lock); +} +EXPORT_SYMBOL (drm_gem_lru_remove);I made a preliminary port of the DRM-SHMEM shrinker on top of the the latest version of dma-buf locking convention and yours LRU patches. It all works good, the only thing that is missing for the DRM-SHMEM shrinker is the
drm_gem_lru_remove_locked().Quoted 4 lines
Quoted 1 line
What about to add a locked variant of
drm_gem_lru_remove()?Sounds fine to me.. the only reason it didn't exist yet was because it wasn't needed yet..
There is no use for the
drm_gem_lru_move_tail_locked()as well, you're not using it in the MSM driver. Hence I thought it might be good to add thedrm_gem_lru_remove_locked(), or maybe thedrm_gem_lru_move_tail_locked()should be dropped then?
Quoted 7 lines
Quoted 2 lines
I can respin w/ an addition of a
_locked()version, or you can add it on top in your patchset. Either is fine by meThe either option is fine by me too. If you'll keep the unused
drm_gem_lru_move_tail_locked(), then will be nice to adddrm_gem_lru_remove_locked().
The drm_gem_lru_move_tail_locked() will be needed by DRM-SHMEM shrinker, BTW.
Signature · 2 lines
Best regards, Dmitry
Re: [PATCH v3 09/15] drm/gem: Add LRU/shrinker helper
Quoted 45 lines · Dmitry Osipenko
Quoted 44 lines · Dmitry Osipenko
Quoted 30 lines · Rob Clark
Quoted 28 lines · Dmitry Osipenko
Quoted 10 lines · Rob Clark
drivers/gpu/drm/drm_gem.c +10
-0+/** +* drm_gem_lru_remove - remove object from whatever LRU it is in +* +* If the object is currently in any LRU, remove it. +* +* @obj: The GEM object to remove from current LRU +*/ +void +drm_gem_lru_remove (struct drm_gem_object *obj) +{drivers/gpu/drm/drm_gem.c drm_gem_unlock_reservations(struct drm_gem_object **objs, int count, · line 1323
Quoted 10 lines
drivers/gpu/drm/drm_gem.c +10
-0+struct drm_gem_lru *lru = obj->lru; + +if (!lru) +return ; + +mutex_lock (lru->lock); +lru_remove (obj); +mutex_unlock (lru->lock); +} +EXPORT_SYMBOL (drm_gem_lru_remove);I made a preliminary port of the DRM-SHMEM shrinker on top of the the latest version of dma-buf locking convention and yours LRU patches. It all works good, the only thing that is missing for the DRM-SHMEM shrinker is the
drm_gem_lru_remove_locked().Quoted 4 lines
Quoted 1 line
What about to add a locked variant of
drm_gem_lru_remove()?Sounds fine to me.. the only reason it didn't exist yet was because it wasn't needed yet..
There is no use for the
drm_gem_lru_move_tail_locked()as well, you're not using it in the MSM driver. Hence I thought it might be good to add thedrm_gem_lru_remove_locked(), or maybe thedrm_gem_lru_move_tail_locked()should be dropped then?Quoted 2 lines
I can respin w/ an addition of a
_locked()version, or you can add it on top in your patchset. Either is fine by me
Quoted 7 lines
Quoted 4 lines
The either option is fine by me too. If you'll keep the unused
drm_gem_lru_move_tail_locked(), then will be nice to adddrm_gem_lru_remove_locked().The
drm_gem_lru_move_tail_locked()will be needed by DRM-SHMEM shrinker, BTW.
On the other hand, I see now that DRM-SHMEM shrinker can use the unlocked versions only. Hence both drm_gem_lru_move_tail_locked() and drm_gem_lru_remove_locked() aren't needed.
Signature · 2 lines
Best regards, Dmitry
Re: [PATCH v3 09/15] drm/gem: Add LRU/shrinker helper
Quoted 52 lines · Dmitry Osipenko
Quoted 50 lines · Dmitry Osipenko
Quoted 48 lines · Dmitry Osipenko
Quoted 30 lines · Rob Clark
Quoted 28 lines · Dmitry Osipenko
Quoted 10 lines · Rob Clark
drivers/gpu/drm/drm_gem.c +10
-0+/** +* drm_gem_lru_remove - remove object from whatever LRU it is in +* +* If the object is currently in any LRU, remove it. +* +* @obj: The GEM object to remove from current LRU +*/ +void +drm_gem_lru_remove (struct drm_gem_object *obj) +{drivers/gpu/drm/drm_gem.c drm_gem_unlock_reservations(struct drm_gem_object **objs, int count, · line 1323
Quoted 10 lines
drivers/gpu/drm/drm_gem.c +10
-0+struct drm_gem_lru *lru = obj->lru; + +if (!lru) +return ; + +mutex_lock (lru->lock); +lru_remove (obj); +mutex_unlock (lru->lock); +} +EXPORT_SYMBOL (drm_gem_lru_remove);I made a preliminary port of the DRM-SHMEM shrinker on top of the the latest version of dma-buf locking convention and yours LRU patches. It all works good, the only thing that is missing for the DRM-SHMEM shrinker is the
drm_gem_lru_remove_locked().Quoted 4 lines
Quoted 1 line
What about to add a locked variant of
drm_gem_lru_remove()?Sounds fine to me.. the only reason it didn't exist yet was because it wasn't needed yet..
There is no use for the
drm_gem_lru_move_tail_locked()as well, you're not using it in the MSM driver. Hence I thought it might be good to add thedrm_gem_lru_remove_locked(), or maybe thedrm_gem_lru_move_tail_locked()should be dropped then?Quoted 2 lines
I can respin w/ an addition of a
_locked()version, or you can add it on top in your patchset. Either is fine by meThe either option is fine by me too. If you'll keep the unused
drm_gem_lru_move_tail_locked(), then will be nice to adddrm_gem_lru_remove_locked().
Quoted 6 lines
Quoted 2 lines
The
drm_gem_lru_move_tail_locked()will be needed by DRM-SHMEM shrinker, BTW.On the other hand, I see now that DRM-SHMEM shrinker can use the unlocked versions only. Hence both
drm_gem_lru_move_tail_locked()anddrm_gem_lru_remove_locked()aren't needed.
drm_gem_lru_move_tail_locked() is used internally, but I guess it could be made static since there ended up not being external users (yet?)
I could see _move_tail_locked() being useful for a driver that wanted to bulk update a bunch of GEM objs, for ex. all the bo's associated with a submit/job.
BR,
-R
Re: [PATCH v3 09/15] drm/gem: Add LRU/shrinker helper
Quoted 57 lines · Rob Clark
Quoted 55 lines · Dmitry Osipenko
Quoted 52 lines · Dmitry Osipenko
Quoted 41 lines · Dmitry Osipenko
Quoted 30 lines · Rob Clark
Quoted 28 lines · Dmitry Osipenko
Quoted 10 lines · Rob Clark
(quoted diff) +10
-0+/** + * drm_gem_lru_remove - remove object from whatever LRU it is in + * + * If the object is currently in any LRU, remove it. + * + * @obj: The GEM object to remove from current LRU + */ +void +drm_gem_lru_remove(struct drm_gem_object *obj) +{Review
Quoted 10 lines
(quoted diff) +10
-0+ struct drm_gem_lru *lru = obj->lru; + + if (!lru) + return; + + mutex_lock(lru->lock); + lru_remove(obj); + mutex_unlock(lru->lock); +} +EXPORT_SYMBOL(drm_gem_lru_remove);I made a preliminary port of the DRM-SHMEM shrinker on top of the the latest version of dma-buf locking convention and yours LRU patches. It all works good, the only thing that is missing for the DRM-SHMEM shrinker is the
drm_gem_lru_remove_locked().Quoted 4 lines
Quoted 1 line
What about to add a locked variant of
drm_gem_lru_remove()?Sounds fine to me.. the only reason it didn't exist yet was because it wasn't needed yet..
There is no use for the
drm_gem_lru_move_tail_locked()as well, you're not using it in the MSM driver. Hence I thought it might be good to add thedrm_gem_lru_remove_locked(), or maybe thedrm_gem_lru_move_tail_locked()should be dropped then?Quoted 7 lines
Quoted 2 lines
I can respin w/ an addition of a
_locked()version, or you can add it on top in your patchset. Either is fine by meThe either option is fine by me too. If you'll keep the unused
drm_gem_lru_move_tail_locked(), then will be nice to adddrm_gem_lru_remove_locked().The
drm_gem_lru_move_tail_locked()will be needed by DRM-SHMEM shrinker, BTW.
Quoted 7 lines
Quoted 3 lines
On the other hand, I see now that DRM-SHMEM shrinker can use the unlocked versions only. Hence both
drm_gem_lru_move_tail_locked()anddrm_gem_lru_remove_locked()aren't needed.
drm_gem_lru_move_tail_locked()is used internally, but I guess it could be made static since there ended up not being external users (yet?)
Making it static will be good.
Quoted 3 lines
I could see
_move_tail_locked()being useful for a driver that wanted to bulk update a bunch of GEM objs, for ex. all the bo's associated with a submit/job.
At minimum we shouldn't expose the unused kernel symbols. But if you're planning to make use of this function later on, then it might be fine to add it.
Signature · 2 lines
Best regards, Dmitry
PATCH v3 10/15 drm/msm/gem: Convert to using drm_gem_lru
In reply to Rob ClarkFrom: Rob Clark <[email protected]>
This converts over to use the shared GEM LRU/shrinker helpers. Note that it means we are no longer tracking purgeable or willneed buffers that are active separately. But the most recently pinned buffers should be at the tail of the various LRUs, and the shrinker is already prepared to encounter objects which are still active.
--- drivers/gpu/drm/msm/msm_drv.c | 14 +-- drivers/gpu/drm/msm/msm_drv.h | 70 +++++++++++---- drivers/gpu/drm/msm/msm_gem.c | 58 ++++-------- drivers/gpu/drm/msm/msm_gem.h | 93 -------------------- drivers/gpu/drm/msm/msm_gem_shrinker.c | 117 ++++++------------------- drivers/gpu/drm/msm/msm_gpu.c | 3 - drivers/gpu/drm/msm/msm_gpu.h | 6 -- 7 files changed, 104 insertions(+), 257 deletions(-)
drivers/gpu/drm/msm/msm_drv.c +9 -5
@@ -418,14 +418,18 @@ static int msm_drm_init(struct device *dev, const struct drm_driver *drv)INIT_LIST_HEAD (&priv->objects);mutex_init (&priv->obj_lock); -INIT_LIST_HEAD (&priv->inactive_willneed); -INIT_LIST_HEAD (&priv->inactive_dontneed); -INIT_LIST_HEAD (&priv->inactive_unpinned); -mutex_init (&priv->mm_lock); +/* +* Initialize the LRUs: +*/ +mutex_init (&priv->lru.lock); +drm_gem_lru_init (&priv->lru.unbacked, &priv->lru.lock); +drm_gem_lru_init (&priv->lru.pinned, &priv->lru.lock); +drm_gem_lru_init (&priv->lru.willneed, &priv->lru.lock); +drm_gem_lru_init (&priv->lru.dontneed, &priv->lru.lock);/* Teach lockdep about lock ordering wrt. shrinker: */ fs_reclaim_acquire (GFP_KERNEL ); -might_lock (&priv->mm_lock); +might_lock (&priv->lru.lock);fs_reclaim_release (GFP_KERNEL );drm_mode_config_init (ddev);
drivers/gpu/drm/msm/msm_drv.h +51 -19
79 lines not on this page · raw message
drivers/gpu/drm/msm/msm_gem.c +15 -43
120 lines not on this page · raw message
drivers/gpu/drm/msm/msm_gem.h +0 -93
122 lines not on this page · raw message
drivers/gpu/drm/msm/msm_gem_shrinker.c +29 -88
179 lines not on this page · raw message
drivers/gpu/drm/msm/msm_gpu.c +0 -3
@@ -846,7 +846,6 @@ int msm_gpu_init(struct drm_device *drm, struct platform_device *pdev,sched_set_fifo_low (gpu->worker->task); -INIT_LIST_HEAD (&gpu->active_list);mutex_init (&gpu->active_lock);mutex_init (&gpu->lock);init_waitqueue_head (&gpu->retire_event);
@@ -974,8 +973,6 @@ void msm_gpu_cleanup(struct msm_gpu *gpu)DBG ("%s" , gpu->name); -WARN_ON (!list_empty (&gpu->active_list)); -for (i =0 ; i <ARRAY_SIZE (gpu->rb); i++) {msm_ringbuffer_destroy (gpu->rb[i]); gpu->rb[i] =NULL ;
drivers/gpu/drm/msm/msm_gpu.h +0 -6
@@ -187,12 +187,6 @@ struct msm_gpu {*/ int cur_ctx_seqno; -/* -* List of GEM active objects on this gpu. Protected by -* msm_drm_private::mm_lock -*/ -struct list_head active_list; -/** * lock: *
Signature · 1 line
2.36.1
PATCH v3 11/15 drm/msm/gem: Unpin buffers earlier
In reply to Rob ClarkFrom: Rob Clark <[email protected]>
We've already attached the fences, so obj->resv (which shrinker checks) tells us whether they are still active. So we can unpin sooner, before we drop the queue lock.
This also avoids the need to grab the obj lock in the retire path, avoiding potential for lock contention between submit and retire.
--- drivers/gpu/drm/msm/msm_gem_submit.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-)
drivers/gpu/drm/msm/msm_gem_submit.c +2 -6
@@ -501,11 +501,11 @@ static int submit_reloc(struct msm_gem_submit *submit, struct msm_gem_object *ob*/ static void submit_cleanup (struct msm_gem_submit *submit,bool error) { -unsigned cleanup_flags =BO_LOCKED ; +unsigned cleanup_flags =BO_LOCKED |BO_OBJ_PINNED ;unsigned i;if (error) - cleanup_flags |=BO_VMA_PINNED |BO_OBJ_PINNED ; + cleanup_flags |=BO_VMA_PINNED ;for (i =0 ; i < submit->nr_bos; i++) {struct msm_gem_object *msm_obj = submit->bos[i].obj;
@@ -522,10 +522,6 @@ void msm_submit_retire(struct msm_gem_submit *submit)for (i =0 ; i < submit->nr_bos; i++) {struct drm_gem_object *obj = &submit->bos[i].obj->base; -msm_gem_lock (obj); -/* Note, VMA already fence-unpinned before submit: */ -submit_cleanup_bo (submit, i,BO_OBJ_PINNED ); -msm_gem_unlock (obj);drm_gem_object_put (obj); } }
Signature · 1 line
2.36.1
PATCH v3 12/15 drm/msm/gem: Consolidate shrinker trace
In reply to Rob ClarkFrom: Rob Clark <[email protected]>
Combine separate trace events for purge vs evict into one. When we add support for purging/evicting active buffers we'll just add more info into this one trace event, rather than adding a bunch more events.
--- drivers/gpu/drm/msm/msm_gem_shrinker.c | 19 ++++++--------- drivers/gpu/drm/msm/msm_gpu_trace.h | 32 +++++++++++--------------- 2 files changed, 20 insertions(+), 31 deletions(-)
drivers/gpu/drm/msm/msm_gem_shrinker.c +7 -12
@@ -71,25 +71,20 @@ msm_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)struct msm_drm_private *priv = container_of(shrinker, struct msm_drm_private, shrinker);long nr = sc->nr_to_scan; -unsigned long freed; +unsigned long freed, purged, evicted =0 ; - freed =drm_gem_lru_scan (&priv->lru.dontneed, nr, purge); - nr -= freed; - -if (freed >0 ) -trace_msm_gem_purge (freed <<PAGE_SHIFT ); + purged =drm_gem_lru_scan (&priv->lru.dontneed, nr, purge); + nr -= purged;if (can_swap () && nr >0 ) { -unsigned long evicted; - evicted =drm_gem_lru_scan (&priv->lru.willneed, nr, evict); nr -= evicted; + } -if (evicted >0 ) -trace_msm_gem_evict (evicted <<PAGE_SHIFT ); + freed = purged + evicted; - freed += evicted; - } +if (freed) +trace_msm_gem_shrink (sc->nr_to_scan, purged, evicted);return (freed >0 ) ? freed :SHRINK_STOP ; }
drivers/gpu/drm/msm/msm_gpu_trace.h +13 -19
@@ -115,29 +115,23 @@ TRACE_EVENT(msm_gmu_freq_change, ); -TRACE_EVENT (msm_gem_purge, - TP_PROTO(u32 bytes), -TP_ARGS (bytes), +TRACE_EVENT (msm_gem_shrink, + TP_PROTO(u32 nr_to_scan, u32 purged, u32 evicted), +TP_ARGS (nr_to_scan, purged, evicted),TP_STRUCT__entry ( -__field (u32, bytes) + __field(u32, nr_to_scan) + __field(u32, purged) +__field (u32, evicted) ),TP_fast_assign ( - __entry->bytes = bytes; + __entry->nr_to_scan = nr_to_scan; + __entry->purged = purged; + __entry->evicted = evicted; ), -TP_printk ("Purging %u bytes" , __entry->bytes) -); - - -TRACE_EVENT (msm_gem_evict, - TP_PROTO(u32 bytes), -TP_ARGS (bytes), -TP_STRUCT__entry ( -__field (u32, bytes) - ), -TP_fast_assign ( - __entry->bytes = bytes; - ), -TP_printk ("Evicting %u bytes" , __entry->bytes) +TP_printk ("nr_to_scan=%u pages, purged=%u pages, evicted=%u pages" , + __entry->nr_to_scan, + __entry->purged, + __entry->evicted) );
Signature · 1 line
2.36.1
PATCH v3 13/15 drm/msm/gem: Evict active GEM objects when necessary
In reply to Rob ClarkFrom: Rob Clark <[email protected]>
If we are under enough memory pressure, we should stall waiting for active buffers to become idle in order to evict.
v2: Check for __GFP_ATOMIC before blocking
--- drivers/gpu/drm/msm/msm_gem_shrinker.c | 70 +++++++++++++++++++++----- drivers/gpu/drm/msm/msm_gpu_trace.h | 16 +++--- 2 files changed, 68 insertions(+), 18 deletions(-)
drivers/gpu/drm/msm/msm_gem_shrinker.c +58 -12
@@ -24,6 +24,13 @@ static bool can_swap(void)return enable_eviction &&get_nr_swap_pages () >0 ; } +static bool can_block (struct shrink_control *sc) +{ +if (sc->gfp_mask & __GFP_ATOMIC) +return false; +return current_is_kswapd () || (sc->gfp_mask & __GFP_RECLAIM); +} +static unsigned long msm_gem_shrinker_count (struct shrinker *shrinker,struct shrink_control *sc) {
@@ -65,26 +72,65 @@ evict(struct drm_gem_object *obj)return true; } +static bool +wait_for_idle (struct drm_gem_object *obj) +{ +enum dma_resv_usage usage =dma_resv_usage_rw (true); +return dma_resv_wait_timeout (obj->resv, usage, false,1000 ) >0 ; +} + +static bool +active_purge (struct drm_gem_object *obj) +{ +if (!wait_for_idle (obj)) +return false; + +return purge (obj); +} + +static bool +active_evict (struct drm_gem_object *obj) +{ +if (!wait_for_idle (obj)) +return false; + +return evict (obj); +} +static unsigned long msm_gem_shrinker_scan (struct shrinker *shrinker,struct shrink_control *sc) {struct msm_drm_private *priv = container_of(shrinker, struct msm_drm_private, shrinker); +struct { +struct drm_gem_lru *lru; +bool (*shrink)(struct drm_gem_object *obj); +bool cond; +unsigned long freed; + } stages[] = { +/* Stages of progressively more aggressive/expensive reclaim: */ + { &priv->lru.dontneed, purge, true }, + { &priv->lru.willneed, evict,can_swap () }, + { &priv->lru.dontneed, active_purge,can_block (sc) }, + { &priv->lru.willneed, active_evict,can_swap () &&can_block (sc) }, + };long nr = sc->nr_to_scan; -unsigned long freed, purged, evicted =0 ; - - purged =drm_gem_lru_scan (&priv->lru.dontneed, nr, purge); - nr -= purged; - -if (can_swap () && nr >0 ) { - evicted =drm_gem_lru_scan (&priv->lru.willneed, nr, evict); - nr -= evicted; +unsigned long freed =0 ; + +for (unsigned i =0 ; (nr >0 ) && (i <ARRAY_SIZE (stages)); i++) { +if (!stages[i].cond) +continue ; + stages[i].freed = +drm_gem_lru_scan (stages[i].lru, nr, stages[i].shrink); + nr -= stages[i].freed; + freed += stages[i].freed; } - freed = purged + evicted; - -if (freed) -trace_msm_gem_shrink (sc->nr_to_scan, purged, evicted); +if (freed) { +trace_msm_gem_shrink (sc->nr_to_scan, stages[0 ].freed, + stages[1 ].freed, stages[2 ].freed, + stages[3 ].freed); + }return (freed >0 ) ? freed :SHRINK_STOP ; }
drivers/gpu/drm/msm/msm_gpu_trace.h +10 -6
@@ -116,22 +116,26 @@ TRACE_EVENT(msm_gmu_freq_change,TRACE_EVENT (msm_gem_shrink, - TP_PROTO(u32 nr_to_scan, u32 purged, u32 evicted), -TP_ARGS (nr_to_scan, purged, evicted), + TP_PROTO(u32 nr_to_scan, u32 purged, u32 evicted, + u32 active_purged, u32 active_evicted), +TP_ARGS (nr_to_scan, purged, evicted, active_purged, active_evicted),TP_STRUCT__entry ( __field(u32, nr_to_scan) __field(u32, purged) __field(u32, evicted) + __field(u32, active_purged) +__field (u32, active_evicted) ),TP_fast_assign ( __entry->nr_to_scan = nr_to_scan; __entry->purged = purged; __entry->evicted = evicted; + __entry->active_purged = active_purged; + __entry->active_evicted = active_evicted; ), -TP_printk ("nr_to_scan=%u pages, purged=%u pages, evicted=%u pages" , - __entry->nr_to_scan, - __entry->purged, - __entry->evicted) +TP_printk ("nr_to_scan=%u pg, purged=%u pg, evicted=%u pg, active_purged=%u pg, active_evicted=%u pg" , + __entry->nr_to_scan, __entry->purged, __entry->evicted, + __entry->active_purged, __entry->active_evicted) );
Signature · 1 line
2.36.1
PATCH v3 14/15 drm/msm/gem: Add msm_gem_assert_locked()
In reply to Rob ClarkFrom: Rob Clark <[email protected]>
All use of msm_gem_is_locked() is just for WARN_ON()s, so extract out into an msm_gem_assert_locked() patch.
--- drivers/gpu/drm/msm/msm_gem.c | 36 +++++++++++++++++------------------ drivers/gpu/drm/msm/msm_gem.h | 8 +++++++- 2 files changed, 25 insertions(+), 19 deletions(-)
drivers/gpu/drm/msm/msm_gem.c +18 -18
@@ -97,7 +97,7 @@ static struct page **get_pages(struct drm_gem_object *obj) {struct msm_gem_object *msm_obj =to_msm_bo (obj); -GEM_WARN_ON (!msm_gem_is_locked (obj)); +msm_gem_assert_locked (obj);if (!msm_obj->pages) {struct drm_device *dev = obj->dev;
@@ -183,7 +183,7 @@ static struct page **msm_gem_pin_pages_locked(struct drm_gem_object *obj)struct msm_gem_object *msm_obj =to_msm_bo (obj);struct page **p; -GEM_WARN_ON (!msm_gem_is_locked (obj)); +msm_gem_assert_locked (obj);if (GEM_WARN_ON (msm_obj->madv !=MSM_MADV_WILLNEED )) {return ERR_PTR (-EBUSY );
@@ -278,7 +278,7 @@ static uint64_t mmap_offset(struct drm_gem_object *obj)struct drm_device *dev = obj->dev;int ret; -GEM_WARN_ON (!msm_gem_is_locked (obj)); +msm_gem_assert_locked (obj);/* Make it mmapable */ ret =drm_gem_create_mmap_offset (obj);
@@ -307,7 +307,7 @@ static struct msm_gem_vma *add_vma(struct drm_gem_object *obj,struct msm_gem_object *msm_obj =to_msm_bo (obj);struct msm_gem_vma *vma; -GEM_WARN_ON (!msm_gem_is_locked (obj)); +msm_gem_assert_locked (obj); vma =kzalloc (sizeof (*vma),GFP_KERNEL );if (!vma)
@@ -326,7 +326,7 @@ static struct msm_gem_vma *lookup_vma(struct drm_gem_object *obj,struct msm_gem_object *msm_obj =to_msm_bo (obj);struct msm_gem_vma *vma; -GEM_WARN_ON (!msm_gem_is_locked (obj)); +msm_gem_assert_locked (obj);list_for_each_entry (vma, &msm_obj->vmas, list) {if (vma->aspace == aspace)
@@ -357,7 +357,7 @@ put_iova_spaces(struct drm_gem_object *obj, bool close)struct msm_gem_object *msm_obj =to_msm_bo (obj);struct msm_gem_vma *vma; -GEM_WARN_ON (!msm_gem_is_locked (obj)); +msm_gem_assert_locked (obj);list_for_each_entry (vma, &msm_obj->vmas, list) {if (vma->aspace) {
@@ -375,7 +375,7 @@ put_iova_vmas(struct drm_gem_object *obj)struct msm_gem_object *msm_obj =to_msm_bo (obj);struct msm_gem_vma *vma, *tmp; -GEM_WARN_ON (!msm_gem_is_locked (obj)); +msm_gem_assert_locked (obj);list_for_each_entry_safe (vma, tmp, &msm_obj->vmas, list) {del_vma (vma);
@@ -388,7 +388,7 @@ static struct msm_gem_vma *get_vma_locked(struct drm_gem_object *obj, {struct msm_gem_vma *vma; -GEM_WARN_ON (!msm_gem_is_locked (obj)); +msm_gem_assert_locked (obj); vma =lookup_vma (obj, aspace);
@@ -428,7 +428,7 @@ int msm_gem_pin_vma_locked(struct drm_gem_object *obj, struct msm_gem_vma *vma)if (msm_obj->flags &MSM_BO_CACHED_COHERENT ) prot |=IOMMU_CACHE ; -GEM_WARN_ON (!msm_gem_is_locked (obj)); +msm_gem_assert_locked (obj);if (GEM_WARN_ON (msm_obj->madv !=MSM_MADV_WILLNEED ))return -EBUSY ;
@@ -448,7 +448,7 @@ void msm_gem_unpin_locked(struct drm_gem_object *obj) {struct msm_gem_object *msm_obj =to_msm_bo (obj); -GEM_WARN_ON (!msm_gem_is_locked (obj)); +msm_gem_assert_locked (obj); msm_obj->pin_count--;GEM_WARN_ON (msm_obj->pin_count <0 );
@@ -469,7 +469,7 @@ static int get_and_pin_iova_range_locked(struct drm_gem_object *obj,struct msm_gem_vma *vma;int ret; -GEM_WARN_ON (!msm_gem_is_locked (obj)); +msm_gem_assert_locked (obj); vma =get_vma_locked (obj, aspace, range_start, range_end);if (IS_ERR (vma))
@@ -630,7 +630,7 @@ static void *get_vaddr(struct drm_gem_object *obj, unsigned madv)struct msm_gem_object *msm_obj =to_msm_bo (obj);int ret =0 ; -GEM_WARN_ON (!msm_gem_is_locked (obj)); +msm_gem_assert_locked (obj);if (obj->import_attach)return ERR_PTR (-ENODEV );
@@ -703,7 +703,7 @@ void msm_gem_put_vaddr_locked(struct drm_gem_object *obj) {struct msm_gem_object *msm_obj =to_msm_bo (obj); -GEM_WARN_ON (!msm_gem_is_locked (obj)); +msm_gem_assert_locked (obj);GEM_WARN_ON (msm_obj->vmap_count <1 ); msm_obj->vmap_count--;
@@ -745,7 +745,7 @@ void msm_gem_purge(struct drm_gem_object *obj)struct drm_device *dev = obj->dev;struct msm_gem_object *msm_obj =to_msm_bo (obj); -GEM_WARN_ON (!msm_gem_is_locked (obj)); +msm_gem_assert_locked (obj);GEM_WARN_ON (!is_purgeable (msm_obj));/* Get rid of any iommu mapping(s): */
@@ -782,7 +782,7 @@ void msm_gem_evict(struct drm_gem_object *obj)struct drm_device *dev = obj->dev;struct msm_gem_object *msm_obj =to_msm_bo (obj); -GEM_WARN_ON (!msm_gem_is_locked (obj)); +msm_gem_assert_locked (obj);GEM_WARN_ON (is_unevictable (msm_obj));/* Get rid of any iommu mapping(s): */
@@ -797,7 +797,7 @@ void msm_gem_vunmap(struct drm_gem_object *obj) {struct msm_gem_object *msm_obj =to_msm_bo (obj); -GEM_WARN_ON (!msm_gem_is_locked (obj)); +msm_gem_assert_locked (obj);if (!msm_obj->vaddr ||GEM_WARN_ON (!is_vunmapable (msm_obj)))return ;
@@ -811,7 +811,7 @@ static void update_lru(struct drm_gem_object *obj)struct msm_drm_private *priv = obj->dev->dev_private;struct msm_gem_object *msm_obj =to_msm_bo (obj); -GEM_WARN_ON (!msm_gem_is_locked (&msm_obj->base)); +msm_gem_assert_locked (&msm_obj->base);if (!msm_obj->pages) {GEM_WARN_ON (msm_obj->pin_count);
@@ -831,7 +831,7 @@ static void update_lru(struct drm_gem_object *obj)bool msm_gem_active (struct drm_gem_object *obj) { -GEM_WARN_ON (!msm_gem_is_locked (obj)); +msm_gem_assert_locked (obj);if (to_msm_bo (obj)->pin_count)return true;
drivers/gpu/drm/msm/msm_gem.h +7 -1
@@ -215,6 +215,12 @@ msm_gem_is_locked(struct drm_gem_object *obj)return dma_resv_is_locked (obj->resv) || (kref_read (&obj->refcount) ==0 ); } +static inline void +msm_gem_assert_locked (struct drm_gem_object *obj) +{ +GEM_WARN_ON (!msm_gem_is_locked (obj)); +} +/* imported/exported objects are not purgeable: */ static inline bool is_unpurgeable (struct msm_gem_object *msm_obj) {
@@ -229,7 +235,7 @@ static inline bool is_purgeable(struct msm_gem_object *msm_obj)static inline bool is_vunmapable (struct msm_gem_object *msm_obj) { -GEM_WARN_ON (!msm_gem_is_locked (&msm_obj->base)); +msm_gem_assert_locked (&msm_obj->base);return (msm_obj->vmap_count ==0 ) && msm_obj->vaddr; }
Signature · 1 line
2.36.1
PATCH v3 15/15 drm/msm/gem: Convert to lockdep assert
In reply to Rob ClarkFrom: Rob Clark <[email protected]>
--- drivers/gpu/drm/msm/msm_gem.h | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-)
drivers/gpu/drm/msm/msm_gem.h +6 -9
@@ -197,8 +197,8 @@ msm_gem_unlock(struct drm_gem_object *obj)dma_resv_unlock (obj->resv); } -static inline bool -msm_gem_is_locked (struct drm_gem_object *obj) +static inline void +msm_gem_assert_locked (struct drm_gem_object *obj) {/* * Destroying the object is a special case.. msm_gem_free_object()
@@ -212,13 +212,10 @@ msm_gem_is_locked(struct drm_gem_object *obj)* Unfortunately lockdep is not aware of this detail. So when the * refcount drops to zero, we pretend it is already locked. */ -return dma_resv_is_locked (obj->resv) || (kref_read (&obj->refcount) ==0 ); -} - -static inline void -msm_gem_assert_locked (struct drm_gem_object *obj) -{ -GEM_WARN_ON (!msm_gem_is_locked (obj)); +lockdep_assert_once ( + (kref_read (&obj->refcount) ==0 ) || + (lockdep_is_held (&obj->resv->lock.base) !=LOCK_STATE_NOT_HELD ) + ); }/* imported/exported objects are not purgeable: */
Signature · 1 line
2.36.1