drm/msm: Submit overhead opts
Subsystems (4)
cover drm/msm: Submit overhead opts
From: Rob Clark <[email protected]>
I recently wrote myself a submitoverhead igt test[1] and spent a bit of time profiling. The end result ranges from 1.6x faster for NO_IMPLICIT_SYNC commits with 100 BOs to 2.5x faster for 1000 BOs.
[1] https://patchwork.freedesktop.org/series/121909/
Rob Clark (4):
drm/msm: Take lru lock once per job_run drm/msm: Use drm_gem_object in submit bos table drm/msm: Take lru lock once per submit_pin_objects() drm/msm: Remove vma use tracking
drivers/gpu/drm/msm/adreno/a5xx_gpu.c | 6 +-- drivers/gpu/drm/msm/msm_gem.c | 57 ++++++++++++----------- drivers/gpu/drm/msm/msm_gem.h | 15 ++---- drivers/gpu/drm/msm/msm_gem_submit.c | 62 +++++++++++++------------ drivers/gpu/drm/msm/msm_gem_vma.c | 67 +-------------------------- drivers/gpu/drm/msm/msm_gpu.c | 20 ++++---- drivers/gpu/drm/msm/msm_rd.c | 8 ++-- drivers/gpu/drm/msm/msm_ringbuffer.c | 10 ++-- 8 files changed, 91 insertions(+), 154 deletions(-)
Signature · 1 line
2.41.0
PATCH 1/4 drm/msm: Take lru lock once per job_run
From: Rob Clark <[email protected]>
Rather than acquiring it and dropping it for each individual obj.
--- drivers/gpu/drm/msm/msm_gem.c | 3 --- drivers/gpu/drm/msm/msm_ringbuffer.c | 5 +++++ 2 files changed, 5 insertions(+), 3 deletions(-)
drivers/gpu/drm/msm/msm_gem.c +0 -3
@@ -509,14 +509,11 @@ void msm_gem_unpin_locked(struct drm_gem_object *obj)*/ void msm_gem_unpin_active (struct drm_gem_object *obj) { -struct msm_drm_private *priv = obj->dev->dev_private;struct msm_gem_object *msm_obj =to_msm_bo (obj); -mutex_lock (&priv->lru.lock); msm_obj->pin_count--;GEM_WARN_ON (msm_obj->pin_count <0 );update_lru_active (obj); -mutex_unlock (&priv->lru.lock); } struct msm_gem_vma *msm_gem_get_vma_locked(struct drm_gem_object *obj,
drivers/gpu/drm/msm/msm_ringbuffer.c +5 -0
@@ -16,10 +16,13 @@ static struct dma_fence *msm_job_run(struct drm_sched_job *job)struct msm_gem_submit *submit =to_msm_submit (job);struct msm_fence_context *fctx = submit->ring->fctx;struct msm_gpu *gpu = submit->gpu; +struct msm_drm_private *priv = gpu->dev->dev_private;int i;msm_fence_init (submit->hw_fence, fctx); +mutex_lock (&priv->lru.lock); +for (i =0 ; i < submit->nr_bos; i++) {struct drm_gem_object *obj = &submit->bos[i].obj->base;
@@ -28,6 +31,8 @@ static struct dma_fence *msm_job_run(struct drm_sched_job *job) submit->bos[i].flags &= ~(BO_VMA_PINNED |BO_OBJ_PINNED ); } +mutex_unlock (&priv->lru.lock); +/* TODO move submit path over to using a per-ring lock.. */ mutex_lock (&gpu->lock);
Signature · 1 line
2.41.0
PATCH 2/4 drm/msm: Use drm_gem_object in submit bos table
In reply to Rob ClarkFrom: Rob Clark <[email protected]>
Basically everywhere wants the base ptr type. So store that instead of msm_gem_object.
--- drivers/gpu/drm/msm/adreno/a5xx_gpu.c | 6 ++-- drivers/gpu/drm/msm/msm_gem.h | 2 +- drivers/gpu/drm/msm/msm_gem_submit.c | 42 +++++++++++++-------------- drivers/gpu/drm/msm/msm_gpu.c | 20 ++++++------- drivers/gpu/drm/msm/msm_rd.c | 8 ++--- drivers/gpu/drm/msm/msm_ringbuffer.c | 2 +- 6 files changed, 40 insertions(+), 40 deletions(-)
drivers/gpu/drm/msm/adreno/a5xx_gpu.c +3 -3
@@ -66,7 +66,7 @@ void a5xx_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring,static void a5xx_submit_in_rb (struct msm_gpu *gpu,struct msm_gem_submit *submit) {struct msm_ringbuffer *ring = submit->ring; -struct msm_gem_object *obj; +struct drm_gem_object *obj;uint32_t *ptr, dwords;unsigned int i;
@@ -83,7 +83,7 @@ static void a5xx_submit_in_rb(struct msm_gpu *gpu, struct msm_gem_submit *submit obj = submit->bos[submit->cmd[i].idx].obj; dwords = submit->cmd[i].size; - ptr =msm_gem_get_vaddr (&obj->base); + ptr =msm_gem_get_vaddr (obj);/* _get_vaddr() shouldn't fail at this point, * since we've already mapped it once in
@@ -103,7 +103,7 @@ static void a5xx_submit_in_rb(struct msm_gpu *gpu, struct msm_gem_submit *submitOUT_RING (ring, ptr[i]); } -msm_gem_put_vaddr (&obj->base); +msm_gem_put_vaddr (obj);break ; }
drivers/gpu/drm/msm/msm_gem.h +1 -1
@@ -301,7 +301,7 @@ struct msm_gem_submit {#define BO_VMA_PINNED 0x1000/* vma (virtual address) is pinned */ uint32_t flags;union { -struct msm_gem_object *obj; +struct drm_gem_object *obj;uint32_t handle; };uint64_t iova;
drivers/gpu/drm/msm/msm_gem_submit.c +21 -21
155 lines not on this page · raw message
drivers/gpu/drm/msm/msm_gpu.c +10 -10
@@ -219,36 +219,36 @@ static void msm_gpu_devcoredump_free(void *data) }static void msm_gpu_crashstate_get_bo (struct msm_gpu_state *state, -struct msm_gem_object *obj,u64 iova,bool full) +struct drm_gem_object *obj,u64 iova,bool full) {struct msm_gpu_state_bo *state_bo = &state->bos[state->nr_bos];/* Don't record write only objects */ - state_bo->size = obj->base.size; + state_bo->size = obj->size; state_bo->iova = iova; -BUILD_BUG_ON (sizeof (state_bo->name) !=sizeof (obj->name)); +BUILD_BUG_ON (sizeof (state_bo->name) !=sizeof (to_msm_bo (obj)->name)); -memcpy (state_bo->name, obj->name,sizeof (state_bo->name)); +memcpy (state_bo->name,to_msm_bo (obj)->name,sizeof (state_bo->name));if (full) {void *ptr; - state_bo->data =kvmalloc (obj->base.size,GFP_KERNEL ); + state_bo->data =kvmalloc (obj->size,GFP_KERNEL );if (!state_bo->data) goto out; -msm_gem_lock (&obj->base); - ptr =msm_gem_get_vaddr_active (&obj->base); -msm_gem_unlock (&obj->base); +msm_gem_lock (obj); + ptr =msm_gem_get_vaddr_active (obj); +msm_gem_unlock (obj);if (IS_ERR (ptr)) {kvfree (state_bo->data); state_bo->data =NULL ; goto out; } -memcpy (state_bo->data, ptr, obj->base.size); -msm_gem_put_vaddr (&obj->base); +memcpy (state_bo->data, ptr, obj->size); +msm_gem_put_vaddr (obj); } out: state->nr_bos++;
drivers/gpu/drm/msm/msm_rd.c +4 -4
@@ -310,7 +310,7 @@ static void snapshot_buf(struct msm_rd_state *rd, struct msm_gem_submit *submit, int idx, uint64_t iova, uint32_t size, bool full) { - struct msm_gem_object *obj = submit->bos[idx].obj; + struct drm_gem_object *obj = submit->bos[idx].obj;unsigned offset =0 ;const char *buf;
@@ -318,7 +318,7 @@ static void snapshot_buf(struct msm_rd_state *rd, offset = iova - submit->bos[idx].iova; }else { iova = submit->bos[idx].iova; - size = obj->base.size; + size = obj->size; }/*
@@ -335,7 +335,7 @@ static void snapshot_buf(struct msm_rd_state *rd,if (!(submit->bos[idx].flags &MSM_SUBMIT_BO_READ ))return ; - buf =msm_gem_get_vaddr_active (&obj->base); + buf =msm_gem_get_vaddr_active (obj);if (IS_ERR (buf))return ;
@@ -343,7 +343,7 @@ static void snapshot_buf(struct msm_rd_state *rd,rd_write_section (rd,RD_BUFFER_CONTENTS , buf, size); -msm_gem_put_vaddr_locked (&obj->base); +msm_gem_put_vaddr_locked (obj); }/* called under gpu->lock */
drivers/gpu/drm/msm/msm_ringbuffer.c +1 -1
@@ -24,7 +24,7 @@ static struct dma_fence *msm_job_run(struct drm_sched_job *job)mutex_lock (&priv->lru.lock);for (i =0 ; i < submit->nr_bos; i++) { -struct drm_gem_object *obj = &submit->bos[i].obj->base; +struct drm_gem_object *obj = submit->bos[i].obj;msm_gem_vma_unpin_fenced (submit->bos[i].vma, fctx);msm_gem_unpin_active (obj);
Signature · 1 line
2.41.0
PATCH 3/4 drm/msm: Take lru lock once per submit_pin_objects()
In reply to Rob ClarkFrom: Rob Clark <[email protected]>
Split out pin_count incrementing and lru updating into a separate loop so we can take the lru lock only once for all objs. Since we are still holding the obj lock, it is safe to split this up.
--- drivers/gpu/drm/msm/msm_gem.c | 45 ++++++++++++++++++---------- drivers/gpu/drm/msm/msm_gem.h | 1 + drivers/gpu/drm/msm/msm_gem_submit.c | 10 ++++++- 3 files changed, 39 insertions(+), 17 deletions(-)
drivers/gpu/drm/msm/msm_gem.c +29 -16
@@ -222,9 +222,7 @@ static void put_pages(struct drm_gem_object *obj)static struct page **msm_gem_pin_pages_locked (struct drm_gem_object *obj,unsigned madv) { -struct msm_drm_private *priv = obj->dev->dev_private;struct msm_gem_object *msm_obj =to_msm_bo (obj); -struct page **p;msm_gem_assert_locked (obj);
@@ -234,16 +232,29 @@ static struct page **msm_gem_pin_pages_locked(struct drm_gem_object *obj,return ERR_PTR (-EBUSY ); } - p =get_pages (obj); -if (IS_ERR (p)) -return p; +return get_pages (obj); +} + +/* +* Update the pin count of the object, call under lru.lock +*/ +void msm_gem_pin_obj_locked (struct drm_gem_object *obj) +{ +struct msm_drm_private *priv = obj->dev->dev_private; + +msm_gem_assert_locked (obj); + +to_msm_bo (obj)->pin_count++; +drm_gem_lru_move_tail_locked (&priv->lru.pinned, obj); +} + +static void pin_obj_locked (struct drm_gem_object *obj) +{ +struct msm_drm_private *priv = obj->dev->dev_private;mutex_lock (&priv->lru.lock); - msm_obj->pin_count++; -update_lru_locked (obj); +msm_gem_pin_obj_locked (obj);mutex_unlock (&priv->lru.lock); - -return p; }struct page **msm_gem_pin_pages (struct drm_gem_object *obj)
@@ -252,6 +263,8 @@ struct page **msm_gem_pin_pages(struct drm_gem_object *obj)msm_gem_lock (obj); p =msm_gem_pin_pages_locked (obj,MSM_MADV_WILLNEED ); +if (!IS_ERR (p)) +pin_obj_locked (obj);msm_gem_unlock (obj);return p;
@@ -463,7 +476,7 @@ int msm_gem_pin_vma_locked(struct drm_gem_object *obj, struct msm_gem_vma *vma) {struct msm_gem_object *msm_obj =to_msm_bo (obj);struct page **pages; -int ret, prot =IOMMU_READ ; +int prot =IOMMU_READ ;if (!(msm_obj->flags &MSM_BO_GPU_READONLY )) prot |=IOMMU_WRITE ;
@@ -480,11 +493,7 @@ int msm_gem_pin_vma_locked(struct drm_gem_object *obj, struct msm_gem_vma *vma)if (IS_ERR (pages))return PTR_ERR (pages); - ret =msm_gem_vma_map (vma, prot, msm_obj->sgt, obj->size); -if (ret) -msm_gem_unpin_locked (obj); - -return ret; +return msm_gem_vma_map (vma, prot, msm_obj->sgt, obj->size); }void msm_gem_unpin_locked (struct drm_gem_object *obj)
@@ -536,8 +545,10 @@ static int get_and_pin_iova_range_locked(struct drm_gem_object *obj,return PTR_ERR (vma); ret =msm_gem_pin_vma_locked (obj, vma); -if (!ret) +if (!ret) { *iova = vma->iova; +pin_obj_locked (obj); + }return ret; }
@@ -700,6 +711,8 @@ static void *get_vaddr(struct drm_gem_object *obj, unsigned madv)if (IS_ERR (pages))return ERR_CAST (pages); +pin_obj_locked (obj); +/* increment vmap_count *before* vmap() call, so shrinker can * check vmap_count (is_vunmapable()) outside of msm_obj lock. * This guarantees that we won't try to msm_gem_vunmap() this
drivers/gpu/drm/msm/msm_gem.h +1 -0
@@ -142,6 +142,7 @@ int msm_gem_get_and_pin_iova(struct drm_gem_object *obj, struct msm_gem_address_space *aspace, uint64_t *iova);void msm_gem_unpin_iova (struct drm_gem_object *obj,struct msm_gem_address_space *aspace); +void msm_gem_pin_obj_locked (struct drm_gem_object *obj);struct page **msm_gem_pin_pages (struct drm_gem_object *obj);void msm_gem_unpin_pages (struct drm_gem_object *obj); int msm_gem_dumb_create(struct drm_file *file, struct drm_device *dev,
drivers/gpu/drm/msm/msm_gem_submit.c +9 -1
@@ -384,6 +384,7 @@ static int submit_fence_sync(struct msm_gem_submit *submit, bool no_implicit)static int submit_pin_objects (struct msm_gem_submit *submit) { +struct msm_drm_private *priv = submit->dev->dev_private;int i, ret =0 ; submit->valid = true;
@@ -403,7 +404,7 @@ static int submit_pin_objects(struct msm_gem_submit *submit)if (ret)break ; - submit->bos[i].flags |=BO_OBJ_PINNED |BO_VMA_PINNED ; + submit->bos[i].flags |=BO_VMA_PINNED ; submit->bos[i].vma = vma;if (vma->iova == submit->bos[i].iova) {
@@ -416,6 +417,13 @@ static int submit_pin_objects(struct msm_gem_submit *submit) } } +mutex_lock (&priv->lru.lock); +for (i =0 ; i < submit->nr_bos; i++) { +msm_gem_pin_obj_locked (submit->bos[i].obj); + submit->bos[i].flags |=BO_OBJ_PINNED ; + } +mutex_unlock (&priv->lru.lock); +return ret; }
Signature · 1 line
2.41.0
PATCH 4/4 drm/msm: Remove vma use tracking
In reply to Rob ClarkFrom: Rob Clark <[email protected]>
This was not strictly necessary, as page unpinning (ie. shrinker) only cares about the resv. It did give us some extra sanity checking for userspace controlled iova, and was useful to catch issues on kernel and userspace side when enabling userspace iova. But if userspace screws this up, it just corrupts it's own gpu buffers and/or gets iova faults. So we can just let userspace shoot it's own foot and drop the extra per- buffer SUBMIT overhead.
--- drivers/gpu/drm/msm/msm_gem.c | 9 +--- drivers/gpu/drm/msm/msm_gem.h | 12 +---- drivers/gpu/drm/msm/msm_gem_submit.c | 14 ++---- drivers/gpu/drm/msm/msm_gem_vma.c | 67 +--------------------------- drivers/gpu/drm/msm/msm_ringbuffer.c | 3 +- 5 files changed, 9 insertions(+), 96 deletions(-)
drivers/gpu/drm/msm/msm_gem.c +2 -7
@@ -607,9 +607,6 @@ static int clear_iova(struct drm_gem_object *obj,if (!vma)return 0 ; -if (msm_gem_vma_inuse (vma)) -return -EBUSY ; -msm_gem_vma_purge (vma);msm_gem_vma_close (vma);del_vma (vma);
@@ -660,7 +657,6 @@ void msm_gem_unpin_iova(struct drm_gem_object *obj,msm_gem_lock (obj); vma =lookup_vma (obj, aspace);if (!GEM_WARN_ON (!vma)) { -msm_gem_vma_unpin (vma);msm_gem_unpin_locked (obj); }msm_gem_unlock (obj);
@@ -991,11 +987,10 @@ void msm_gem_describe(struct drm_gem_object *obj, struct seq_file *m, }else { name = comm =NULL ; } -seq_printf (m," [%s%s%s: aspace=%p, %08llx,%s,inuse=%d]" , +seq_printf (m," [%s%s%s: aspace=%p, %08llx,%s]" , name, comm ?":" :"" , comm ? comm :"" , vma->aspace, vma->iova, - vma->mapped ?"mapped" :"unmapped" , -msm_gem_vma_inuse (vma)); + vma->mapped ?"mapped" :"unmapped" );kfree (comm); }
drivers/gpu/drm/msm/msm_gem.h +1 -11
@@ -59,24 +59,16 @@ struct msm_fence_context;struct msm_gem_vma {struct drm_mm_node node; -spinlock_t lock;uint64_t iova;struct msm_gem_address_space *aspace;struct list_head list;/* node in msm_gem_object::vmas */ bool mapped; -int inuse; -uint32_t fence_mask; -uint32_t fence[MSM_GPU_MAX_RINGS ]; -struct msm_fence_context *fctx[MSM_GPU_MAX_RINGS ]; };struct msm_gem_vma *msm_gem_vma_new (struct msm_gem_address_space *aspace);int msm_gem_vma_init (struct msm_gem_vma *vma,int size,u64 range_start,u64 range_end); -bool msm_gem_vma_inuse (struct msm_gem_vma *vma);void msm_gem_vma_purge (struct msm_gem_vma *vma); -void msm_gem_vma_unpin (struct msm_gem_vma *vma); -void msm_gem_vma_unpin_fenced (struct msm_gem_vma *vma,struct msm_fence_context *fctx);int msm_gem_vma_map (struct msm_gem_vma *vma,int prot,struct sg_table *sgt,int size);void msm_gem_vma_close (struct msm_gem_vma *vma);
@@ -298,15 +290,13 @@ struct msm_gem_submit {/* make sure these don't conflict w/ MSM_SUBMIT_BO_x */ #define BO_VALID 0x8000/* is current addr in cmdstream correct/valid? */ #define BO_LOCKED 0x4000/* obj lock is held */ -#define BO_OBJ_PINNED 0x2000/* obj (pages) is pinned and on active list */ -#define BO_VMA_PINNED 0x1000/* vma (virtual address) is pinned */ +#define BO_PINNED 0x2000/* obj (pages) is pinned and on active list */ uint32_t flags;union {struct drm_gem_object *obj;uint32_t handle; };uint64_t iova; -struct msm_gem_vma *vma; } bos[]; };
drivers/gpu/drm/msm/msm_gem_submit.c +4 -10
@@ -261,10 +261,7 @@ static void submit_cleanup_bo(struct msm_gem_submit *submit, int i,*/ submit->bos[i].flags &= ~cleanup_flags; -if (flags &BO_VMA_PINNED ) -msm_gem_vma_unpin (submit->bos[i].vma); - -if (flags &BO_OBJ_PINNED ) +if (flags &BO_PINNED )msm_gem_unpin_locked (obj);if (flags &BO_LOCKED )
@@ -273,7 +270,7 @@ static void submit_cleanup_bo(struct msm_gem_submit *submit, int i,static void submit_unlock_unpin_bo (struct msm_gem_submit *submit,int i) { -unsigned cleanup_flags =BO_VMA_PINNED |BO_OBJ_PINNED |BO_LOCKED ; +unsigned cleanup_flags =BO_PINNED |BO_LOCKED ;submit_cleanup_bo (submit, i, cleanup_flags);if (!(submit->bos[i].flags &BO_VALID ))
@@ -404,9 +401,6 @@ static int submit_pin_objects(struct msm_gem_submit *submit)if (ret)break ; - submit->bos[i].flags |=BO_VMA_PINNED ; - submit->bos[i].vma = vma; -if (vma->iova == submit->bos[i].iova) { submit->bos[i].flags |=BO_VALID ; }else {
@@ -420,7 +414,7 @@ static int submit_pin_objects(struct msm_gem_submit *submit)mutex_lock (&priv->lru.lock);for (i =0 ; i < submit->nr_bos; i++) {msm_gem_pin_obj_locked (submit->bos[i].obj); - submit->bos[i].flags |=BO_OBJ_PINNED ; + submit->bos[i].flags |=BO_PINNED ; }mutex_unlock (&priv->lru.lock);
@@ -547,7 +541,7 @@ static void submit_cleanup(struct msm_gem_submit *submit, bool error)unsigned i;if (error) - cleanup_flags |=BO_VMA_PINNED |BO_OBJ_PINNED ; + cleanup_flags |=BO_PINNED ;for (i =0 ; i < submit->nr_bos; i++) {struct drm_gem_object *obj = submit->bos[i].obj;
drivers/gpu/drm/msm/msm_gem_vma.c +1 -66
109 lines not on this page · raw message
drivers/gpu/drm/msm/msm_ringbuffer.c +1 -2
@@ -26,9 +26,8 @@ static struct dma_fence *msm_job_run(struct drm_sched_job *job)for (i =0 ; i < submit->nr_bos; i++) {struct drm_gem_object *obj = submit->bos[i].obj; -msm_gem_vma_unpin_fenced (submit->bos[i].vma, fctx);msm_gem_unpin_active (obj); - submit->bos[i].flags &= ~(BO_VMA_PINNED |BO_OBJ_PINNED ); + submit->bos[i].flags &= ~BO_PINNED ; }mutex_unlock (&priv->lru.lock);
Signature · 1 line
2.41.0
Re: [PATCH 4/4] drm/msm: Remove vma use tracking
Quoted 11 lines · Rob Clark
From: Rob Clark <[email protected]>
This was not strictly necessary, as page unpinning (ie. shrinker) only cares about the resv. It did give us some extra sanity checking for userspace controlled iova, and was useful to catch issues on kernel and userspace side when enabling userspace iova. But if userspace screws this up, it just corrupts it's own gpu buffers and/or gets iova faults. So we can just let userspace shoot it's own foot and drop the extra per- buffer SUBMIT overhead.
Signed-off-by Rob Clark <[email protected]>
I did check a few things (like that the gem lru helpers have all the needed lockdep_assert_held) and I think aside from the optimization this is a nice semantic cleanup. Since iirc we've had a locking inversion discussion and the vma tracking here came up as a culprit. On the series:
Quoted 280 lines
--- drivers/gpu/drm/msm/msm_gem.c | 9 +--- drivers/gpu/drm/msm/msm_gem.h | 12 +---- drivers/gpu/drm/msm/msm_gem_submit.c | 14 ++---- drivers/gpu/drm/msm/msm_gem_vma.c | 67 +--------------------------- drivers/gpu/drm/msm/msm_ringbuffer.c | 3 +- 5 files changed, 9 insertions(+), 96 deletions(-)drivers/gpu/drm/msm/msm_gem.c +2
-7@@ -607,9 +607,6 @@ static int clear_iova(struct drm_gem_object *obj,if (!vma)return 0 ; -if (msm_gem_vma_inuse (vma)) -return -EBUSY ; -msm_gem_vma_purge (vma);msm_gem_vma_close (vma);del_vma (vma);@@ -660,7 +657,6 @@ void msm_gem_unpin_iova(struct drm_gem_object *obj,msm_gem_lock (obj); vma =lookup_vma (obj, aspace);if (!GEM_WARN_ON (!vma)) { -msm_gem_vma_unpin (vma);msm_gem_unpin_locked (obj); }msm_gem_unlock (obj);@@ -991,11 +987,10 @@ void msm_gem_describe(struct drm_gem_object *obj, struct seq_file *m, }else { name = comm =NULL ; } -seq_printf (m," [%s%s%s: aspace=%p, %08llx,%s,inuse=%d]" , +seq_printf (m," [%s%s%s: aspace=%p, %08llx,%s]" , name, comm ?":" :"" , comm ? comm :"" , vma->aspace, vma->iova, - vma->mapped ?"mapped" :"unmapped" , -msm_gem_vma_inuse (vma)); + vma->mapped ?"mapped" :"unmapped" );kfree (comm); }drivers/gpu/drm/msm/msm_gem.h +1
-11@@ -59,24 +59,16 @@ struct msm_fence_context;struct msm_gem_vma {struct drm_mm_node node; -spinlock_t lock;uint64_t iova;struct msm_gem_address_space *aspace;struct list_head list;/* node in msm_gem_object::vmas */ bool mapped; -int inuse; -uint32_t fence_mask; -uint32_t fence[MSM_GPU_MAX_RINGS ]; -struct msm_fence_context *fctx[MSM_GPU_MAX_RINGS ]; };struct msm_gem_vma *msm_gem_vma_new (struct msm_gem_address_space *aspace);int msm_gem_vma_init (struct msm_gem_vma *vma,int size,u64 range_start,u64 range_end); -bool msm_gem_vma_inuse (struct msm_gem_vma *vma);void msm_gem_vma_purge (struct msm_gem_vma *vma); -void msm_gem_vma_unpin (struct msm_gem_vma *vma); -void msm_gem_vma_unpin_fenced (struct msm_gem_vma *vma,struct msm_fence_context *fctx);int msm_gem_vma_map (struct msm_gem_vma *vma,int prot,struct sg_table *sgt,int size);void msm_gem_vma_close (struct msm_gem_vma *vma);@@ -298,15 +290,13 @@ struct msm_gem_submit {/* make sure these don't conflict w/ MSM_SUBMIT_BO_x */ #define BO_VALID 0x8000/* is current addr in cmdstream correct/valid? */ #define BO_LOCKED 0x4000/* obj lock is held */ -#define BO_OBJ_PINNED 0x2000/* obj (pages) is pinned and on active list */ -#define BO_VMA_PINNED 0x1000/* vma (virtual address) is pinned */ +#define BO_PINNED 0x2000/* obj (pages) is pinned and on active list */ uint32_t flags;union {struct drm_gem_object *obj;uint32_t handle; };uint64_t iova; -struct msm_gem_vma *vma; } bos[]; };drivers/gpu/drm/msm/msm_gem_submit.c +4
-10@@ -261,10 +261,7 @@ static void submit_cleanup_bo(struct msm_gem_submit *submit, int i,*/ submit->bos[i].flags &= ~cleanup_flags; -if (flags &BO_VMA_PINNED ) -msm_gem_vma_unpin (submit->bos[i].vma); - -if (flags &BO_OBJ_PINNED ) +if (flags &BO_PINNED )msm_gem_unpin_locked (obj);if (flags &BO_LOCKED )@@ -273,7 +270,7 @@ static void submit_cleanup_bo(struct msm_gem_submit *submit, int i,static void submit_unlock_unpin_bo (struct msm_gem_submit *submit,int i) { -unsigned cleanup_flags =BO_VMA_PINNED |BO_OBJ_PINNED |BO_LOCKED ; +unsigned cleanup_flags =BO_PINNED |BO_LOCKED ;submit_cleanup_bo (submit, i, cleanup_flags);if (!(submit->bos[i].flags &BO_VALID ))@@ -404,9 +401,6 @@ static int submit_pin_objects(struct msm_gem_submit *submit)if (ret)break ; - submit->bos[i].flags |=BO_VMA_PINNED ; - submit->bos[i].vma = vma; -if (vma->iova == submit->bos[i].iova) { submit->bos[i].flags |=BO_VALID ; }else {@@ -420,7 +414,7 @@ static int submit_pin_objects(struct msm_gem_submit *submit)mutex_lock (&priv->lru.lock);for (i =0 ; i < submit->nr_bos; i++) {msm_gem_pin_obj_locked (submit->bos[i].obj); - submit->bos[i].flags |=BO_OBJ_PINNED ; + submit->bos[i].flags |=BO_PINNED ; }mutex_unlock (&priv->lru.lock);@@ -547,7 +541,7 @@ static void submit_cleanup(struct msm_gem_submit *submit, bool error)unsigned i;if (error) - cleanup_flags |=BO_VMA_PINNED |BO_OBJ_PINNED ; + cleanup_flags |=BO_PINNED ;for (i =0 ; i < submit->nr_bos; i++) {struct drm_gem_object *obj = submit->bos[i].obj;drivers/gpu/drm/msm/msm_gem_vma.c +1
-66@@ -38,41 +38,12 @@ msm_gem_address_space_get(struct msm_gem_address_space *aspace)return aspace; } -bool msm_gem_vma_inuse (struct msm_gem_vma *vma) -{ -bool ret = true; - -spin_lock (&vma->lock); - -if (vma->inuse >0 ) - goto out; - -while (vma->fence_mask) { -unsigned idx =ffs (vma->fence_mask) -1 ; - -if (!msm_fence_completed (vma->fctx[idx], vma->fence[idx])) - goto out; - - vma->fence_mask &= ~BIT (idx); - } - - ret = false; - -out: -spin_unlock (&vma->lock); - -return ret; -} -/* Actually unmap memory for the vma */ void msm_gem_vma_purge (struct msm_gem_vma *vma) {struct msm_gem_address_space *aspace = vma->aspace;unsigned size = vma->node.size; -/* Print a message if we try to purge a vma in use */ -GEM_WARN_ON (msm_gem_vma_inuse (vma)); -/* Don't do anything if the memory isn't mapped */ if (!vma->mapped)return ;@@ -82,33 +53,6 @@ void msm_gem_vma_purge(struct msm_gem_vma *vma) vma->mapped = false; } -static void vma_unpin_locked (struct msm_gem_vma *vma) -{ -if (GEM_WARN_ON (!vma->inuse)) -return ; -if (!GEM_WARN_ON (!vma->iova)) - vma->inuse--; -} - -/* Remove reference counts for the mapping */ -void msm_gem_vma_unpin (struct msm_gem_vma *vma) -{ -spin_lock (&vma->lock); -vma_unpin_locked (vma); -spin_unlock (&vma->lock); -} - -/* Replace pin reference with fence: */ -void msm_gem_vma_unpin_fenced (struct msm_gem_vma *vma,struct msm_fence_context *fctx) -{ -spin_lock (&vma->lock); - vma->fctx[fctx->index] = fctx; - vma->fence[fctx->index] = fctx->last_fence; - vma->fence_mask |=BIT (fctx->index); -vma_unpin_locked (vma); -spin_unlock (&vma->lock); -} -/* Map and pin vma: */ int msm_gem_vma_map(struct msm_gem_vma *vma, int prot,@@ -120,11 +64,6 @@ msm_gem_vma_map(struct msm_gem_vma *vma, int prot,if (GEM_WARN_ON (!vma->iova))return -EINVAL ; -/* Increase the usage counter */ -spin_lock (&vma->lock); - vma->inuse++; -spin_unlock (&vma->lock); -if (vma->mapped)return 0 ;@@ -146,9 +85,6 @@ msm_gem_vma_map(struct msm_gem_vma *vma, int prot,if (ret) { vma->mapped = false; -spin_lock (&vma->lock); - vma->inuse--; -spin_unlock (&vma->lock); }return ret;@@ -159,7 +95,7 @@ void msm_gem_vma_close(struct msm_gem_vma *vma) {struct msm_gem_address_space *aspace = vma->aspace; -GEM_WARN_ON (msm_gem_vma_inuse (vma) || vma->mapped); +GEM_WARN_ON (vma->mapped);spin_lock (&aspace->lock);if (vma->iova)@@ -179,7 +115,6 @@ struct msm_gem_vma *msm_gem_vma_new(struct msm_gem_address_space *aspace)if (!vma)return NULL ; -spin_lock_init (&vma->lock); vma->aspace = aspace;return vma;drivers/gpu/drm/msm/msm_ringbuffer.c +1
-2@@ -26,9 +26,8 @@ static struct dma_fence *msm_job_run(struct drm_sched_job *job)for (i =0 ; i < submit->nr_bos; i++) {struct drm_gem_object *obj = submit->bos[i].obj; -msm_gem_vma_unpin_fenced (submit->bos[i].vma, fctx);msm_gem_unpin_active (obj); - submit->bos[i].flags &= ~(BO_VMA_PINNED |BO_OBJ_PINNED ); + submit->bos[i].flags &= ~BO_PINNED ; }mutex_unlock (&priv->lru.lock);Signature · 1 line
2.41.0
Signature · 3 lines
Daniel Vetter Software Engineer, Intel Corporation http://blog.ffwll.ch
Re: [PATCH 4/4] drm/msm: Remove vma use tracking
Quoted 12 lines · Daniel Vetter
Quoted 10 lines · Rob Clark
From: Rob Clark <[email protected]>
This was not strictly necessary, as page unpinning (ie. shrinker) only cares about the resv. It did give us some extra sanity checking for userspace controlled iova, and was useful to catch issues on kernel and userspace side when enabling userspace iova. But if userspace screws this up, it just corrupts it's own gpu buffers and/or gets iova faults. So we can just let userspace shoot it's own foot and drop the extra per- buffer SUBMIT overhead.
Quoted 6 lines
Quoted 1 line
Signed-off-by Rob Clark <[email protected]>I did check a few things (like that the gem lru helpers have all the needed
lockdep_assert_held) and I think aside from the optimization this is a nice semantic cleanup. Since iirc we've had a locking inversion discussion and the vma tracking here came up as a culprit. On the series:
yup, I try to make sure there are sufficient locking asserts when it comes to gem stuff ;-)
There shouldn't have been any locking inversion with the vma lock (no other locks acquired under it) but it was hurting cpu overhead. The big remaining locking headache for me is run-pm / pm-qos
BR,
-R
Quoted 287 lines
Acked-by Daniel Vetter <[email protected]>Quoted 278 lines
--- drivers/gpu/drm/msm/msm_gem.c | 9 +--- drivers/gpu/drm/msm/msm_gem.h | 12 +---- drivers/gpu/drm/msm/msm_gem_submit.c | 14 ++---- drivers/gpu/drm/msm/msm_gem_vma.c | 67 +--------------------------- drivers/gpu/drm/msm/msm_ringbuffer.c | 3 +- 5 files changed, 9 insertions(+), 96 deletions(-)drivers/gpu/drm/msm/msm_gem.c +2
-7@@ -607,9 +607,6 @@ static int clear_iova(struct drm_gem_object *obj,if (!vma)return 0 ; -if (msm_gem_vma_inuse (vma)) -return -EBUSY ; -msm_gem_vma_purge (vma);msm_gem_vma_close (vma);del_vma (vma);@@ -660,7 +657,6 @@ void msm_gem_unpin_iova(struct drm_gem_object *obj,msm_gem_lock (obj); vma =lookup_vma (obj, aspace);if (!GEM_WARN_ON (!vma)) { -msm_gem_vma_unpin (vma);msm_gem_unpin_locked (obj); }msm_gem_unlock (obj);@@ -991,11 +987,10 @@ void msm_gem_describe(struct drm_gem_object *obj, struct seq_file *m, }else { name = comm =NULL ; } -seq_printf (m," [%s%s%s: aspace=%p, %08llx,%s,inuse=%d]" , +seq_printf (m," [%s%s%s: aspace=%p, %08llx,%s]" , name, comm ?":" :"" , comm ? comm :"" , vma->aspace, vma->iova, - vma->mapped ?"mapped" :"unmapped" , -msm_gem_vma_inuse (vma)); + vma->mapped ?"mapped" :"unmapped" );kfree (comm); }drivers/gpu/drm/msm/msm_gem.h +1
-11@@ -59,24 +59,16 @@ struct msm_fence_context;struct msm_gem_vma {struct drm_mm_node node; -spinlock_t lock;uint64_t iova;struct msm_gem_address_space *aspace;struct list_head list;/* node in msm_gem_object::vmas */ bool mapped; -int inuse; -uint32_t fence_mask; -uint32_t fence[MSM_GPU_MAX_RINGS ]; -struct msm_fence_context *fctx[MSM_GPU_MAX_RINGS ]; };struct msm_gem_vma *msm_gem_vma_new (struct msm_gem_address_space *aspace);int msm_gem_vma_init (struct msm_gem_vma *vma,int size,u64 range_start,u64 range_end); -bool msm_gem_vma_inuse (struct msm_gem_vma *vma);void msm_gem_vma_purge (struct msm_gem_vma *vma); -void msm_gem_vma_unpin (struct msm_gem_vma *vma); -void msm_gem_vma_unpin_fenced (struct msm_gem_vma *vma,struct msm_fence_context *fctx);int msm_gem_vma_map (struct msm_gem_vma *vma,int prot,struct sg_table *sgt,int size);void msm_gem_vma_close (struct msm_gem_vma *vma);@@ -298,15 +290,13 @@ struct msm_gem_submit {/* make sure these don't conflict w/ MSM_SUBMIT_BO_x */ #define BO_VALID 0x8000/* is current addr in cmdstream correct/valid? */ #define BO_LOCKED 0x4000/* obj lock is held */ -#define BO_OBJ_PINNED 0x2000/* obj (pages) is pinned and on active list */ -#define BO_VMA_PINNED 0x1000/* vma (virtual address) is pinned */ +#define BO_PINNED 0x2000/* obj (pages) is pinned and on active list */ uint32_t flags;union {struct drm_gem_object *obj;uint32_t handle; };uint64_t iova; -struct msm_gem_vma *vma; } bos[]; };drivers/gpu/drm/msm/msm_gem_submit.c +4
-10@@ -261,10 +261,7 @@ static void submit_cleanup_bo(struct msm_gem_submit *submit, int i,*/ submit->bos[i].flags &= ~cleanup_flags; -if (flags &BO_VMA_PINNED ) -msm_gem_vma_unpin (submit->bos[i].vma); - -if (flags &BO_OBJ_PINNED ) +if (flags &BO_PINNED )msm_gem_unpin_locked (obj);if (flags &BO_LOCKED )@@ -273,7 +270,7 @@ static void submit_cleanup_bo(struct msm_gem_submit *submit, int i,static void submit_unlock_unpin_bo (struct msm_gem_submit *submit,int i) { -unsigned cleanup_flags =BO_VMA_PINNED |BO_OBJ_PINNED |BO_LOCKED ; +unsigned cleanup_flags =BO_PINNED |BO_LOCKED ;submit_cleanup_bo (submit, i, cleanup_flags);if (!(submit->bos[i].flags &BO_VALID ))@@ -404,9 +401,6 @@ static int submit_pin_objects(struct msm_gem_submit *submit)if (ret)break ; - submit->bos[i].flags |=BO_VMA_PINNED ; - submit->bos[i].vma = vma; -if (vma->iova == submit->bos[i].iova) { submit->bos[i].flags |=BO_VALID ; }else {@@ -420,7 +414,7 @@ static int submit_pin_objects(struct msm_gem_submit *submit)mutex_lock (&priv->lru.lock);for (i =0 ; i < submit->nr_bos; i++) {msm_gem_pin_obj_locked (submit->bos[i].obj); - submit->bos[i].flags |=BO_OBJ_PINNED ; + submit->bos[i].flags |=BO_PINNED ; }mutex_unlock (&priv->lru.lock);@@ -547,7 +541,7 @@ static void submit_cleanup(struct msm_gem_submit *submit, bool error)unsigned i;if (error) - cleanup_flags |=BO_VMA_PINNED |BO_OBJ_PINNED ; + cleanup_flags |=BO_PINNED ;for (i =0 ; i < submit->nr_bos; i++) {struct drm_gem_object *obj = submit->bos[i].obj;drivers/gpu/drm/msm/msm_gem_vma.c +1
-66@@ -38,41 +38,12 @@ msm_gem_address_space_get(struct msm_gem_address_space *aspace)return aspace; } -bool msm_gem_vma_inuse (struct msm_gem_vma *vma) -{ -bool ret = true; - -spin_lock (&vma->lock); - -if (vma->inuse >0 ) - goto out; - -while (vma->fence_mask) { -unsigned idx =ffs (vma->fence_mask) -1 ; - -if (!msm_fence_completed (vma->fctx[idx], vma->fence[idx])) - goto out; - - vma->fence_mask &= ~BIT (idx); - } - - ret = false; - -out: -spin_unlock (&vma->lock); - -return ret; -} -/* Actually unmap memory for the vma */ void msm_gem_vma_purge (struct msm_gem_vma *vma) {struct msm_gem_address_space *aspace = vma->aspace;unsigned size = vma->node.size; -/* Print a message if we try to purge a vma in use */ -GEM_WARN_ON (msm_gem_vma_inuse (vma)); -/* Don't do anything if the memory isn't mapped */ if (!vma->mapped)return ;@@ -82,33 +53,6 @@ void msm_gem_vma_purge(struct msm_gem_vma *vma) vma->mapped = false; } -static void vma_unpin_locked (struct msm_gem_vma *vma) -{ -if (GEM_WARN_ON (!vma->inuse)) -return ; -if (!GEM_WARN_ON (!vma->iova)) - vma->inuse--; -} - -/* Remove reference counts for the mapping */ -void msm_gem_vma_unpin (struct msm_gem_vma *vma) -{ -spin_lock (&vma->lock); -vma_unpin_locked (vma); -spin_unlock (&vma->lock); -} - -/* Replace pin reference with fence: */ -void msm_gem_vma_unpin_fenced (struct msm_gem_vma *vma,struct msm_fence_context *fctx) -{ -spin_lock (&vma->lock); - vma->fctx[fctx->index] = fctx; - vma->fence[fctx->index] = fctx->last_fence; - vma->fence_mask |=BIT (fctx->index); -vma_unpin_locked (vma); -spin_unlock (&vma->lock); -} -/* Map and pin vma: */ int msm_gem_vma_map(struct msm_gem_vma *vma, int prot,@@ -120,11 +64,6 @@ msm_gem_vma_map(struct msm_gem_vma *vma, int prot,if (GEM_WARN_ON (!vma->iova))return -EINVAL ; -/* Increase the usage counter */ -spin_lock (&vma->lock); - vma->inuse++; -spin_unlock (&vma->lock); -if (vma->mapped)return 0 ;@@ -146,9 +85,6 @@ msm_gem_vma_map(struct msm_gem_vma *vma, int prot,if (ret) { vma->mapped = false; -spin_lock (&vma->lock); - vma->inuse--; -spin_unlock (&vma->lock); }return ret;@@ -159,7 +95,7 @@ void msm_gem_vma_close(struct msm_gem_vma *vma) {struct msm_gem_address_space *aspace = vma->aspace; -GEM_WARN_ON (msm_gem_vma_inuse (vma) || vma->mapped); +GEM_WARN_ON (vma->mapped);spin_lock (&aspace->lock);if (vma->iova)@@ -179,7 +115,6 @@ struct msm_gem_vma *msm_gem_vma_new(struct msm_gem_address_space *aspace)if (!vma)return NULL ; -spin_lock_init (&vma->lock); vma->aspace = aspace;return vma;drivers/gpu/drm/msm/msm_ringbuffer.c +1
-3@@ -26,9 +26,8 @@ static struct dma_fence *msm_job_run(struct drm_sched_job *job)for (i =0 ; i < submit->nr_bos; i++) {struct drm_gem_object *obj = submit->bos[i].obj; -msm_gem_vma_unpin_fenced (submit->bos[i].vma, fctx);msm_gem_unpin_active (obj); - submit->bos[i].flags &= ~(BO_VMA_PINNED |BO_OBJ_PINNED ); + submit->bos[i].flags &= ~BO_PINNED ; }mutex_unlock (&priv->lru.lock); --Quoted 2 lines
2.41.0
--
Daniel Vetter
Software Engineer, Intel Corporation
http://blog.ffwll.ch