drm/amd/amdgpu embed hw_fence into amdgpu_job
Subsystems (3)
PATCH v5 1/1 drm/amd/amdgpu embed hw_fence into amdgpu_job
From: Jack Zhang <[email protected]>
Why: Previously hw fence is alloced separately with job. It caused historical lifetime issues and corner cases. The ideal situation is to take fence to manage both job and fence's lifetime, and simplify the design of gpu-scheduler.
How:
We propose to embed hw_fence into amdgpu_job.
1. We cover the normal job submission by this method.
2. For ib_test, and submit without a parent job keep the legacy way to create a hw fence separately.
v2:
use AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT to show that the fence is embedded in a job.
v3:
remove redundant variable ring in amdgpu_job
v4:
add tdr sequence support for this feature. Add a job_run_counter to indicate whether this job is a resubmit job.
v5
add missing handling in amdgpu_fence_enable_signaling
--- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 1 - drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 13 +++- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 86 ++++++++++++++++----- drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 39 +++++++--- drivers/gpu/drm/amd/amdgpu/amdgpu_job.h | 6 +- drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h | 5 +- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 2 +- 9 files changed, 119 insertions(+), 37 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +0 -1
@@ -714,7 +714,6 @@ int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine, ret =dma_fence_wait (f, false); err_ib_sched: -dma_fence_put (f);amdgpu_job_free (job); err:return ret;
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +1 -1
@@ -1414,7 +1414,7 @@ static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring)continue ; } job =to_amdgpu_job (s_job); -if (preempted && job->fence == fence) +if (preempted && (&job->hw_fence) == fence)/* mark the job as preempted */ job->preemption_status |=AMDGPU_IB_PREEMPTED ; }
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +12 -1
@@ -4448,7 +4448,7 @@ int amdgpu_device_mode1_reset(struct amdgpu_device *adev)int amdgpu_device_pre_asic_reset (struct amdgpu_device *adev,struct amdgpu_reset_context *reset_context) { -int i, r =0 ; +int i, j, r =0 ;struct amdgpu_job *job =NULL ;bool need_full_reset =test_bit (AMDGPU_NEED_FULL_RESET , &reset_context->flags);
@@ -4472,6 +4472,17 @@ int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,if (!ring || !ring->sched.thread)continue ; +/*clear job fence from fence drv to avoid force_completion +*leave NULL and vm flush fence in fence drv */ +for (j =0 ; j <= ring->fence_drv.num_fences_mask; j++) { +struct dma_fence *old, **ptr; + + ptr = &ring->fence_drv.fences[j]; + old =rcu_dereference_protected (*ptr,1 ); +if (old &&test_bit (AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT , &old->flags)) { +RCU_INIT_POINTER (*ptr,NULL ); + } + }/* after all hw jobs are reset, hw fence is meaningless, so force_completion */ amdgpu_fence_driver_force_completion (ring); }
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +67 -19
141 lines not on this page · raw message
drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c +1 -1
@@ -262,7 +262,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs, fence_flags | AMDGPU_FENCE_FLAG_64BIT); } - r =amdgpu_fence_emit (ring, f, fence_flags); + r =amdgpu_fence_emit (ring, f, job, fence_flags);if (r) {dev_err (adev->dev,"failed to emit fence (%d)\n" , r);if (job && job->vmid)
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +28 -11
@@ -127,11 +127,16 @@ void amdgpu_job_free_resources(struct amdgpu_job *job) {struct amdgpu_ring *ring =to_amdgpu_ring (job->base.sched);struct dma_fence *f; +struct dma_fence *hw_fence;unsigned i; -/* use sched fence if available */ - f = job->base.s_fence ? &job->base.s_fence->finished : job->fence; +if (job->hw_fence.ops ==NULL ) + hw_fence = job->external_hw_fence; +else + hw_fence = &job->hw_fence; +/* use sched fence if available */ + f = job->base.s_fence ? &job->base.s_fence->finished : hw_fence;for (i =0 ; i < job->num_ibs; ++i)amdgpu_ib_free (ring->adev, &job->ibs[i], f); }
@@ -142,20 +147,27 @@ static void amdgpu_job_free_cb(struct drm_sched_job *s_job)drm_sched_job_cleanup (s_job); -dma_fence_put (job->fence);amdgpu_sync_free (&job->sync);amdgpu_sync_free (&job->sched_sync); -kfree (job); + +/* only put the hw fence if has embedded fence */ +if (job->hw_fence.ops !=NULL ) +dma_fence_put (&job->hw_fence); +else +kfree (job); }void amdgpu_job_free (struct amdgpu_job *job) {amdgpu_job_free_resources (job); - -dma_fence_put (job->fence);amdgpu_sync_free (&job->sync);amdgpu_sync_free (&job->sched_sync); -kfree (job); + +/* only put the hw fence if has embedded fence */ +if (job->hw_fence.ops !=NULL ) +dma_fence_put (&job->hw_fence); +else +kfree (job); } int amdgpu_job_submit(struct amdgpu_job *job, struct drm_sched_entity *entity,
@@ -184,11 +196,14 @@ int amdgpu_job_submit_direct(struct amdgpu_job *job, struct amdgpu_ring *ring, job->base.sched = &ring->sched; r =amdgpu_ib_schedule (ring, job->num_ibs, job->ibs,NULL , fence); - job->fence =dma_fence_get (*fence); +/* record external_hw_fence for direct submit */ + job->external_hw_fence =dma_fence_get (*fence);if (r)return r;amdgpu_job_free (job); +dma_fence_put (*fence); +return 0 ; }
@@ -246,10 +261,12 @@ static struct dma_fence *amdgpu_job_run(struct drm_sched_job *sched_job)if (r)DRM_ERROR ("Error scheduling IBs (%d)\n" , r); } -/* if gpu reset, hw fence will be replaced here */ -dma_fence_put (job->fence); - job->fence =dma_fence_get (fence); +if (!job->job_run_counter) +dma_fence_get (fence); +else if (finished->error <0 ) +dma_fence_put (&job->hw_fence); + job->job_run_counter++;amdgpu_job_free_resources (job); fence = r ?ERR_PTR (r) : fence;
drivers/gpu/drm/amd/amdgpu/amdgpu_job.h +5 -1
@@ -46,7 +46,8 @@ struct amdgpu_job {struct amdgpu_sync sync;struct amdgpu_sync sched_sync;struct amdgpu_ib *ibs; -struct dma_fence *fence;/* the hw fence */ +struct dma_fence hw_fence; +struct dma_fence *external_hw_fence;uint32_t preamble_status;uint32_t preemption_status;uint32_t num_ibs;
@@ -62,6 +63,9 @@ struct amdgpu_job {/* user fence handling */ uint64_t uf_addr;uint64_t uf_sequence; + +/* job_run_counter >= 1 means a resubmit job */ +uint32_t job_run_counter; }; int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs,
drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +4 -1
@@ -48,6 +48,9 @@#define AMDGPU_FENCE_FLAG_INT (1 << 1)#define AMDGPU_FENCE_FLAG_TC_WB_ONLY (1 << 2) +/* fence flag bit to indicate the face is embedded in job*/ +#define AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT (DMA_FENCE_FLAG_USER_BITS + 1) +#define to_amdgpu_ring (s) container_of((s), struct amdgpu_ring, sched)#define AMDGPU_IB_POOL_SIZE (1024 * 1024)
@@ -118,7 +121,7 @@ void amdgpu_fence_driver_hw_init(struct amdgpu_device *adev);void amdgpu_fence_driver_hw_fini (struct amdgpu_device *adev);int amdgpu_fence_driver_sw_init (struct amdgpu_device *adev);void amdgpu_fence_driver_sw_fini (struct amdgpu_device *adev); -int amdgpu_fence_emit (struct amdgpu_ring *ring,struct dma_fence **fence, +int amdgpu_fence_emit (struct amdgpu_ring *ring,struct dma_fence **fence,struct amdgpu_job *job,unsigned flags);int amdgpu_fence_emit_polling (struct amdgpu_ring *ring,uint32_t *s,uint32_t timeout);
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1 -1
@@ -1218,7 +1218,7 @@ int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job,amdgpu_gmc_emit_pasid_mapping (ring, job->vmid, job->pasid);if (vm_flush_needed || pasid_mapping_needed) { - r =amdgpu_fence_emit (ring, &fence,0 ); + r =amdgpu_fence_emit (ring, &fence,NULL ,0 );if (r)return r; }
Signature · 1 line
2.25.1
[AMD Official Use Only]
Reviewed by: Monk Liu <[email protected]>
Thanks
------------------------------------------
Monk Liu | Cloud-GPU Core team
------------------------------------------
-----Original Message-----
From: Jingwen Chen <[email protected]>
Sent: Wednesday, August 11, 2021 9:02 PM
To: [email protected]
Cc: Liu, Monk <[email protected]>; Koenig, Christian <[email protected]>; Grodzovsky, Andrey <[email protected]>; Jack Zhang <[email protected]>; Chen, JingWen <[email protected]>; Jack Zhang <[email protected]>; Grodzovsky, Andrey <[email protected]>
Subject: [PATCH v5] drm/amd/amdgpu embed hw_fence into amdgpu_job
From: Jack Zhang <[email protected]>
Why: Previously hw fence is alloced separately with job.
It caused historical lifetime issues and corner cases.
The ideal situation is to take fence to manage both job and fence's lifetime, and simplify the design of gpu-scheduler.
How:
We propose to embed hw_fence into amdgpu_job.
1. We cover the normal job submission by this method.
2. For ib_test, and submit without a parent job keep the legacy way to create a hw fence separately. v2:
use AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT to show that the fence is embedded in a job.
v3:
remove redundant variable ring in amdgpu_job
v4:
add tdr sequence support for this feature. Add a job_run_counter to indicate whether this job is a resubmit job. v5
add missing handling in amdgpu_fence_enable_signaling
--- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 1 - drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 13 +++- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 86 ++++++++++++++++----- drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 39 +++++++--- drivers/gpu/drm/amd/amdgpu/amdgpu_job.h | 6 +- drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h | 5 +- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 2 +- 9 files changed, 119 insertions(+), 37 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +0 -1
@@ -714,7 +714,6 @@ int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine, ret =dma_fence_wait (f, false); err_ib_sched: -dma_fence_put (f);amdgpu_job_free (job); err:return ret;
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +1 -1
@@ -1414,7 +1414,7 @@ static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring)continue ; } job =to_amdgpu_job (s_job); -if (preempted && job->fence == fence) +if (preempted && (&job->hw_fence) == fence)/* mark the job as preempted */ job->preemption_status |=AMDGPU_IB_PREEMPTED ; }
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1 -1
@@ -4448,7 +4448,7 @@ int amdgpu_device_mode1_reset(struct amdgpu_device *adev) int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev, struct amdgpu_reset_context *reset_context) { -int i, r =0 ; +int i, j, r =0 ;struct amdgpu_job *job =NULL ;bool need_full_reset = test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); @@ -4472,6 +4472,17 @@ int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev, if (!ring || !ring->sched.thread)continue ;
+ /*clear job fence from fence drv to avoid force_completion
+ *leave NULL and vm flush fence in fence drv */
+ for (j = 0; j <= ring->fence_drv.num_fences_mask; j++) {
+ struct dma_fence *old, **ptr;
+
+ ptr = &ring->fence_drv.fences[j];
+ old = rcu_dereference_protected(*ptr, 1);
+ if (old && test_bit(AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT, &old->flags)) {
+ RCU_INIT_POINTER(*ptr, NULL);
+ }
+ }
/* after all hw jobs are reset, hw fence is meaningless, so force_completion */ amdgpu_fence_driver_force_completion(ring);
}
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +31 -10
60 lines not on this page · raw message
pm_runtime_get_noresume(adev_to_drm(adev)->dev);
@@ -175,9 +195,9 @@ int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f,
/* This function can't be called concurrently anyway, otherwise
* emitting the fence would mess up the hardware ring buffer.
*/
- rcu_assign_pointer(*ptr, dma_fence_get(&fence->base));
+ rcu_assign_pointer(*ptr, dma_fence_get(fence));
- *f = &fence->base;
+ *f = fence;
return 0;
}
@@ -621,8 +641,16 @@ static const char *amdgpu_fence_get_driver_name(struct dma_fence *fence)
static const char *amdgpu_fence_get_timeline_name(struct dma_fence *f) {
- struct amdgpu_fence *fence = to_amdgpu_fence(f);
- return (const char *)fence->ring->name;
+ struct amdgpu_ring *ring;
+
+ if (test_bit(AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT, &f->flags)) {
+ struct amdgpu_job *job = container_of(f, struct amdgpu_job,
+hw_fence);
+
+ ring = to_amdgpu_ring(job->base.sched);
+ } else {
+ ring = to_amdgpu_fence(f)->ring;
+ }
+ return (const char *)ring->name;
}
/**
@@ -635,13 +663,20 @@ static const char *amdgpu_fence_get_timeline_name(struct dma_fence *f)
*/
static bool amdgpu_fence_enable_signaling(struct dma_fence *f) {
- struct amdgpu_fence *fence = to_amdgpu_fence(f);
- struct amdgpu_ring *ring = fence->ring;
+ struct amdgpu_ring *ring;
+
+ if (test_bit(AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT, &f->flags)) {
+ struct amdgpu_job *job = container_of(f, struct amdgpu_job,
+hw_fence);
+
+ ring = to_amdgpu_ring(job->base.sched);
+ } else {
+ ring = to_amdgpu_fence(f)->ring;
+ }
if (!timer_pending(&ring->fence_drv.fallback_timer))
amdgpu_fence_schedule_fallback(ring);
- DMA_FENCE_TRACE(&fence->base, "armed on ring %i!\n", ring->idx);
+ DMA_FENCE_TRACE(f, "armed on ring %i!\n", ring->idx);
return true;
}
@@ -656,8 +691,20 @@ static bool amdgpu_fence_enable_signaling(struct dma_fence *f) static void amdgpu_fence_free(struct rcu_head *rcu) {
struct dma_fence *f = container_of(rcu, struct dma_fence, rcu);
- struct amdgpu_fence *fence = to_amdgpu_fence(f);
- kmem_cache_free(amdgpu_fence_slab, fence);
+
+ if (test_bit(AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT, &f->flags)) {
+ /* free job if fence has a parent job */
+ struct amdgpu_job *job;
+
+ job = container_of(f, struct amdgpu_job, hw_fence);
+ kfree(job);
+ } else {
+ /* free fence_slab if it's separated fence*/
+ struct amdgpu_fence *fence;
+
+ fence = to_amdgpu_fence(f);
+ kmem_cache_free(amdgpu_fence_slab, fence);
+ }
}
/**
@@ -680,6 +727,7 @@ static const struct dma_fence_ops amdgpu_fence_ops = {
.release = amdgpu_fence_release,
};
+
/*
* Fence debugfs
*/
drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c +1 -1
@@ -262,7 +262,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs, fence_flags | AMDGPU_FENCE_FLAG_64BIT); } - r =amdgpu_fence_emit (ring, f, fence_flags); + r =amdgpu_fence_emit (ring, f, job, fence_flags);if (r) {dev_err (adev->dev,"failed to emit fence (%d)\n" , r);if (job && job->vmid)
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +7 -2
@@ -127,11 +127,16 @@ void amdgpu_job_free_resources(struct amdgpu_job *job) {struct amdgpu_ring *ring =to_amdgpu_ring (job->base.sched);struct dma_fence *f; +struct dma_fence *hw_fence;unsigned i; -/* use sched fence if available */ - f = job->base.s_fence ? &job->base.s_fence->finished : job->fence; +if (job->hw_fence.ops ==NULL ) + hw_fence = job->external_hw_fence; +else + hw_fence = &job->hw_fence; +/* use sched fence if available */ + f = job->base.s_fence ? &job->base.s_fence->finished : hw_fence;for (i =0 ; i < job->num_ibs; ++i) amdgpu_ib_free(ring->adev, &job->ibs[i], f); } @@ -142,20 +147,27 @@ static void amdgpu_job_free_cb(struct drm_sched_job *s_job) drm_sched_job_cleanup(s_job);
- dma_fence_put(job->fence);
amdgpu_sync_free(&job->sync);
amdgpu_sync_free(&job->sched_sync);
- kfree(job);
+
+ /* only put the hw fence if has embedded fence */
+ if (job->hw_fence.ops != NULL)
+ dma_fence_put(&job->hw_fence);
+ else
+ kfree(job);
}
void amdgpu_job_free(struct amdgpu_job *job) {
amdgpu_job_free_resources(job);
-
- dma_fence_put(job->fence);
amdgpu_sync_free(&job->sync);
amdgpu_sync_free(&job->sched_sync);
- kfree(job);
+
+ /* only put the hw fence if has embedded fence */
+ if (job->hw_fence.ops != NULL)
+ dma_fence_put(&job->hw_fence);
+ else
+ kfree(job);
}
int amdgpu_job_submit(struct amdgpu_job *job, struct drm_sched_entity *entity, @@ -184,11 +196,14 @@ int amdgpu_job_submit_direct(struct amdgpu_job *job, struct amdgpu_ring *ring,
job->base.sched = &ring->sched;
r = amdgpu_ib_schedule(ring, job->num_ibs, job->ibs, NULL, fence);
- job->fence = dma_fence_get(*fence);
+ /* record external_hw_fence for direct submit */
+ job->external_hw_fence = dma_fence_get(*fence);
if (r)
return r;
amdgpu_job_free(job);
+ dma_fence_put(*fence);
+
return 0;
}
@@ -246,10 +261,12 @@ static struct dma_fence *amdgpu_job_run(struct drm_sched_job *sched_job)
if (r)
DRM_ERROR("Error scheduling IBs (%d)\n", r);
}
- /* if gpu reset, hw fence will be replaced here */
- dma_fence_put(job->fence);
- job->fence = dma_fence_get(fence);
+ if (!job->job_run_counter)
+ dma_fence_get(fence);
+ else if (finished->error < 0)
+ dma_fence_put(&job->hw_fence);
+ job->job_run_counter++;
amdgpu_job_free_resources(job);
fence = r ? ERR_PTR(r) : fence;
drivers/gpu/drm/amd/amdgpu/amdgpu_job.h +5 -1
@@ -46,7 +46,8 @@ struct amdgpu_job {struct amdgpu_sync sync;struct amdgpu_sync sched_sync;struct amdgpu_ib *ibs; -struct dma_fence *fence;/* the hw fence */ +struct dma_fence hw_fence; +struct dma_fence *external_hw_fence;uint32_t preamble_status;uint32_t preemption_status;uint32_t num_ibs;
@@ -62,6 +63,9 @@ struct amdgpu_job {/* user fence handling */ uint64_t uf_addr;uint64_t uf_sequence; + +/* job_run_counter >= 1 means a resubmit job */ +uint32_t job_run_counter; }; int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
index 9c11ced4312c..e713d31619fe 100644
drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +5 -0
@@ -48,6 +48,9 @@#define AMDGPU_FENCE_FLAG_INT (1 << 1)#define AMDGPU_FENCE_FLAG_TC_WB_ONLY (1 << 2) +/* fence flag bit to indicate the face is embedded in job*/ +#define AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT (DMA_FENCE_FLAG_USER_BITS + 1) +#define to_amdgpu_ring (s) container_of((s), struct amdgpu_ring, sched)#define AMDGPU_IB_POOL_SIZE (1024 * 1024)
@@ -118,7 +121,7 @@ void amdgpu_fence_driver_hw_init(struct amdgpu_device *adev); void amdgpu_fence_driver_hw_fini(struct amdgpu_device *adev); int amdgpu_fence_driver_sw_init(struct amdgpu_device *adev); void amdgpu_fence_driver_sw_fini(struct amdgpu_device *adev); -int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **fence, +int amdgpu_fence_emit (struct amdgpu_ring *ring,struct dma_fence +**fence,struct amdgpu_job *job,unsigned flags);int amdgpu_fence_emit_polling (struct amdgpu_ring *ring,uint32_t *s,uint32_t timeout);
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1 -1
@@ -1218,7 +1218,7 @@ int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job,amdgpu_gmc_emit_pasid_mapping (ring, job->vmid, job->pasid);if (vm_flush_needed || pasid_mapping_needed) { - r =amdgpu_fence_emit (ring, &fence,0 ); + r =amdgpu_fence_emit (ring, &fence,NULL ,0 );if (r)return r; }
--
2.25.1