Skip to contentSkip to conversation

drm/msm: Fix hw_fence error path cleanup

PATCH In review Rob Clark 2 participants 3 messages Jul 12, 2023 – Jul 13, 2023

Subsystems (3)
lore
Share & tools
lore

Rob Clark

author
#0 lore raw

PATCH 1/1 drm/msm: Fix hw_fence error path cleanup

From: Rob Clark <[email protected]>

In an error path where the submit is free'd without the job being run, the hw_fence pointer is simply a kzalloc'd block of memory. In this case we should just kfree() it, rather than trying to decrement it's reference count. Fortunately we can tell that this is the case by checking for a zero refcount, since if the job was run, the submit would be holding a reference to the hw_fence.

Fixes f94e6a51e17c ("drm/msm: Pre-allocate hw_fence")
---
 drivers/gpu/drm/msm/msm_fence.c      |  6 ++++++
 drivers/gpu/drm/msm/msm_gem_submit.c | 14 +++++++++++++-
 2 files changed, 19 insertions(+), 1 deletion(-)
drivers/gpu/drm/msm/msm_fence.c +6 -0
index 96599ec3eb78..1a5d4f1c8b42 100644
@@ -191,6 +191,12 @@ msm_fence_init(struct dma_fence *fence, struct msm_fence_context *fctx)
 
 	f->fctx = fctx;
 
+	/*
+	 * Until this point, the fence was just some pre-allocated memory,
+	 * no-one should have taken a reference to it yet.
+	 */
+	WARN_ON(kref_read(&fence->refcount));
+
 	dma_fence_init(&f->base, &msm_fence_ops, &fctx->spinlock,
 		       fctx->context, ++fctx->last_fence);
 }
drivers/gpu/drm/msm/msm_gem_submit.c +13 -1
index 3f1aa4de3b87..9d66498cdc04 100644
@@ -86,7 +86,19 @@ void __msm_gem_submit_destroy(struct kref *kref)
 	}
 
 	dma_fence_put(submit->user_fence);
-	dma_fence_put(submit->hw_fence);
+
+	/*
+	 * If the submit is freed before msm_job_run(), then hw_fence is
+	 * just some pre-allocated memory, not a reference counted fence.
+	 * Once the job runs and the hw_fence is initialized, it will
+	 * have a refcount of at least one, since the submit holds a ref
+	 * to the hw_fence.
+	 */
+	if (kref_read(&submit->hw_fence->refcount) == 0) {
+		kfree(submit->hw_fence);
+	} else {
+		dma_fence_put(submit->hw_fence);
+	}
 
 	put_pid(submit->pid);
 	msm_submitqueue_put(submit->queue);
Signature · 1 line
2.41.0
#1 lore raw
Quoted 21 lines · Rob Clark

From: Rob Clark <[email protected]>

In an error path where the submit is free'd without the job being run, the hw_fence pointer is simply a kzalloc'd block of memory. In this case we should just kfree() it, rather than trying to decrement it's reference count. Fortunately we can tell that this is the case by checking for a zero refcount, since if the job was run, the submit would be holding a reference to the hw_fence.

Fixes f94e6a51e17c ("drm/msm: Pre-allocate hw_fence")
---
  drivers/gpu/drm/msm/msm_fence.c      |  6 ++++++
  drivers/gpu/drm/msm/msm_gem_submit.c | 14 +++++++++++++-
  2 files changed, 19 insertions(+), 1 deletion(-)
drivers/gpu/drm/msm/msm_fence.c +0 -0
index 96599ec3eb78..1a5d4f1c8b42 100644

drivers/gpu/drm/msm/msm_fence.c msm_fence_init(struct dma_fence *fence, struct msm_fence_context *fctx) · line 198

Quoted 8 lines
drivers/gpu/drm/msm/msm_fence.c +5 -0
@@ -191,6 +191,12 @@ msm_fence_init(struct dma_fence *fence, struct msm_fence_context *fctx)
 
  	f->fctx = fctx;
 
+	/*
+	 * Until this point, the fence was just some pre-allocated memory,
+	 * no-one should have taken a reference to it yet.
+	 */
+	WARN_ON(kref_read(&fence->refcount));

It this really correct to return a refcounted object with 0 refcount (I'm looking at submit_create() / msm_fence_alloc() )? Maybe it would be better to move dma_fence_get() to msm_fence_alloc() ? But don't immediately see, which one should be moved.

Quoted 29 lines
drivers/gpu/drm/msm/msm_fence.c +1 -0
+
  	dma_fence_init(&f->base, &msm_fence_ops, &fctx->spinlock,
  		       fctx->context, ++fctx->last_fence);
  }
drivers/gpu/drm/msm/msm_gem_submit.c +13 -1
index 3f1aa4de3b87..9d66498cdc04 100644
@@ -86,7 +86,19 @@ void __msm_gem_submit_destroy(struct kref *kref)
  	}
 
  	dma_fence_put(submit->user_fence);
-	dma_fence_put(submit->hw_fence);
+
+	/*
+	 * If the submit is freed before msm_job_run(), then hw_fence is
+	 * just some pre-allocated memory, not a reference counted fence.
+	 * Once the job runs and the hw_fence is initialized, it will
+	 * have a refcount of at least one, since the submit holds a ref
+	 * to the hw_fence.
+	 */
+	if (kref_read(&submit->hw_fence->refcount) == 0) {
+		kfree(submit->hw_fence);
+	} else {
+		dma_fence_put(submit->hw_fence);
+	}
 
  	put_pid(submit->pid);
  	msm_submitqueue_put(submit->queue);
Signature · 2 lines
With best wishes
Dmitry
author
#2 lore raw
Quoted 26 lines · Dmitry Baryshkov
Quoted 24 lines · Rob Clark

From: Rob Clark <[email protected]>

In an error path where the submit is free'd without the job being run, the hw_fence pointer is simply a kzalloc'd block of memory. In this case we should just kfree() it, rather than trying to decrement it's reference count. Fortunately we can tell that this is the case by checking for a zero refcount, since if the job was run, the submit would be holding a reference to the hw_fence.

Fixes f94e6a51e17c ("drm/msm: Pre-allocate hw_fence")
---
  drivers/gpu/drm/msm/msm_fence.c      |  6 ++++++
  drivers/gpu/drm/msm/msm_gem_submit.c | 14 +++++++++++++-
  2 files changed, 19 insertions(+), 1 deletion(-)
drivers/gpu/drm/msm/msm_fence.c +0 -0
index 96599ec3eb78..1a5d4f1c8b42 100644
@@ -191,6 +191,12 @@ msm_fence_init(struct dma_fence *fence, struct msm_fence_context *fctx)
 
      f->fctx = fctx;
 
Quoted 10 lines

drivers/gpu/drm/msm/msm_fence.c msm_fence_init(struct dma_fence *fence, struct msm_fence_context *fctx) · line 198

Quoted 5 lines
drivers/gpu/drm/msm/msm_fence.c +5 -0
+     /*
+      * Until this point, the fence was just some pre-allocated memory,
+      * no-one should have taken a reference to it yet.
+      */
+     WARN_ON(kref_read(&fence->refcount));

It this really correct to return a refcounted object with 0 refcount (I'm looking at submit_create() / msm_fence_alloc() )? Maybe it would be better to move dma_fence_get() to msm_fence_alloc() ? But don't immediately see, which one should be moved.

The issue is that we can't really initialize the fence until msm_job_run(), when it is known what order the fence would be signaled. But we don't want to do any allocations in msm_job_run() because that could trigger the shrinker, which could need to wait until jobs complete to release memory, forming a deadlock.

BR,
-R

Quoted 34 lines
Quoted 19 lines
drivers/gpu/drm/msm/msm_fence.c +1 -0
+
      dma_fence_init(&f->base, &msm_fence_ops, &fctx->spinlock,
                     fctx->context, ++fctx->last_fence);
  }
drivers/gpu/drm/msm/msm_gem_submit.c +6 -1
index 3f1aa4de3b87..9d66498cdc04 100644
@@ -86,7 +86,19 @@ void __msm_gem_submit_destroy(struct kref *kref)
      }
 
      dma_fence_put(submit->user_fence);
-     dma_fence_put(submit->hw_fence);
+
+     /*
+      * If the submit is freed before msm_job_run(), then hw_fence is
+      * just some pre-allocated memory, not a reference counted fence.
+      * Once the job runs and the hw_fence is initialized, it will
+      * have a refcount of at least one, since the submit holds a ref

drivers/gpu/drm/msm/msm_gem_submit.c void __msm_gem_submit_destroy(struct kref *kref) · line 104

Quoted 10 lines
drivers/gpu/drm/msm/msm_gem_submit.c +7 -0
+      * to the hw_fence.
+      */
+     if (kref_read(&submit->hw_fence->refcount) == 0) {
+             kfree(submit->hw_fence);
+     } else {
+             dma_fence_put(submit->hw_fence);
+     }
 
      put_pid(submit->pid);
      msm_submitqueue_put(submit->queue);

--
With best wishes
Dmitry