Skip to contentSkip to conversation

drm/amd/amdgpu embed hw_fence into amdgpu_job

PATCH Superseded Jingwen Chen 3 participants 10 messages Aug 5, 2021 – Aug 10, 2021

Subsystems (3)

v2 of 5 Superseded Latest is v5 →

lore
Share & tools
lore
#0 lore raw

PATCH v2 1/2 drm/amd/amdgpu embed hw_fence into amdgpu_job

From: Jack Zhang <[email protected]>

Why: Previously hw fence is alloced separately with job. It caused historical lifetime issues and corner cases. The ideal situation is to take fence to manage both job and fence's lifetime, and simplify the design of gpu-scheduler.

How:
We propose to embed hw_fence into amdgpu_job.
1. We cover the normal job submission by this method.
2. For ib_test, and submit without a parent job keep the legacy way to create a hw fence separately.
v2:
use AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT to show that the fence is embeded in a job.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c  |  1 -
 drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c |  2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c   | 63 ++++++++++++++++-----
 drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c      |  2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_job.c     | 35 ++++++++----
 drivers/gpu/drm/amd/amdgpu/amdgpu_job.h     |  4 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h    |  5 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c      |  2 +-
 8 files changed, 84 insertions(+), 30 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +0 -1
index 7b46ba551cb2..3003ee1c9487 100644
@@ -714,7 +714,6 @@ int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine,
 	ret = dma_fence_wait(f, false);
 
 err_ib_sched:
-	dma_fence_put(f);
 	amdgpu_job_free(job);
 err:
 	return ret;
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +1 -1
index 536005bff24a..277128846dd1 100644
@@ -1414,7 +1414,7 @@ static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring)
 			continue;
 		}
 		job = to_amdgpu_job(s_job);
-		if (preempted && job->fence == fence)
+		if (preempted && (&job->hw_fence) == fence)
 			/* mark the job as preempted */
 			job->preemption_status |= AMDGPU_IB_PREEMPTED;
 	}
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +50 -13

111 lines not on this page · raw message

drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c +1 -1
index ec65ab0ddf89..c076a6b9a5a2 100644
@@ -262,7 +262,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
 				       fence_flags | AMDGPU_FENCE_FLAG_64BIT);
 	}
 
-	r = amdgpu_fence_emit(ring, f, fence_flags);
+	r = amdgpu_fence_emit(ring, f, job, fence_flags);
 	if (r) {
 		dev_err(adev->dev, "failed to emit fence (%d)\n", r);
 		if (job && job->vmid)
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +24 -11
index d33e6d97cc89..65a395060de2 100644
@@ -127,11 +127,16 @@ void amdgpu_job_free_resources(struct amdgpu_job *job)
 {
 	struct amdgpu_ring *ring = to_amdgpu_ring(job->base.sched);
 	struct dma_fence *f;
+	struct dma_fence *hw_fence;
 	unsigned i;
 
-	/* use sched fence if available */
-	f = job->base.s_fence ? &job->base.s_fence->finished : job->fence;
+	if (job->hw_fence.ops == NULL)
+		hw_fence = job->external_hw_fence;
+	else
+		hw_fence = &job->hw_fence;
 
+	/* use sched fence if available */
+	f = job->base.s_fence ? &job->base.s_fence->finished : hw_fence;
 	for (i = 0; i < job->num_ibs; ++i)
 		amdgpu_ib_free(ring->adev, &job->ibs[i], f);
 }
@@ -142,20 +147,27 @@ static void amdgpu_job_free_cb(struct drm_sched_job *s_job)
 
 	drm_sched_job_cleanup(s_job);
 
-	dma_fence_put(job->fence);
 	amdgpu_sync_free(&job->sync);
 	amdgpu_sync_free(&job->sched_sync);
-	kfree(job);
+
+    /* only put the hw fence if has embedded fence */
+	if (job->hw_fence.ops != NULL)
+		dma_fence_put(&job->hw_fence);
+	else
+		kfree(job);
 }
 
 void amdgpu_job_free(struct amdgpu_job *job)
 {
 	amdgpu_job_free_resources(job);
-
-	dma_fence_put(job->fence);
 	amdgpu_sync_free(&job->sync);
 	amdgpu_sync_free(&job->sched_sync);
-	kfree(job);
+
+	/* only put the hw fence if has embedded fence */
+	if (job->hw_fence.ops != NULL)
+		dma_fence_put(&job->hw_fence);
+	else
+		kfree(job);
 }
 
 int amdgpu_job_submit(struct amdgpu_job *job, struct drm_sched_entity *entity,
@@ -184,11 +196,14 @@ int amdgpu_job_submit_direct(struct amdgpu_job *job, struct amdgpu_ring *ring,
 
 	job->base.sched = &ring->sched;
 	r = amdgpu_ib_schedule(ring, job->num_ibs, job->ibs, NULL, fence);
-	job->fence = dma_fence_get(*fence);
+	/* record external_hw_fence for direct submit */
+	job->external_hw_fence = dma_fence_get(*fence);
 	if (r)
 		return r;
 
 	amdgpu_job_free(job);
+	dma_fence_put(*fence);
+
 	return 0;
 }
 
@@ -246,10 +261,8 @@ static struct dma_fence *amdgpu_job_run(struct drm_sched_job *sched_job)
 		if (r)
 			DRM_ERROR("Error scheduling IBs (%d)\n", r);
 	}
-	/* if gpu reset, hw fence will be replaced here */
-	dma_fence_put(job->fence);
-	job->fence = dma_fence_get(fence);
 
+	dma_fence_get(fence);
 	amdgpu_job_free_resources(job);
 
 	fence = r ? ERR_PTR(r) : fence;
drivers/gpu/drm/amd/amdgpu/amdgpu_job.h +3 -1
index 81caac9b958a..92324c978534 100644
@@ -46,7 +46,9 @@ struct amdgpu_job {
 	struct amdgpu_sync	sync;
 	struct amdgpu_sync	sched_sync;
 	struct amdgpu_ib	*ibs;
-	struct dma_fence	*fence; /* the hw fence */
+	struct dma_fence	hw_fence;
+	struct amdgpu_ring *ring;
+	struct dma_fence	*external_hw_fence;
 	uint32_t		preamble_status;
 	uint32_t                preemption_status;
 	uint32_t		num_ibs;
drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +4 -1
index 9c11ced4312c..03d4b29a76d6 100644
@@ -48,6 +48,9 @@
 #define AMDGPU_FENCE_FLAG_INT           (1 << 1)
 #define AMDGPU_FENCE_FLAG_TC_WB_ONLY    (1 << 2)
 
+/* fence flag bit to indicate the face is embeded in job*/
+#define AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT		(DMA_FENCE_FLAG_USER_BITS + 1)
+
 #define to_amdgpu_ring(s) container_of((s), struct amdgpu_ring, sched)
 
 #define AMDGPU_IB_POOL_SIZE	(1024 * 1024)
@@ -118,7 +121,7 @@ void amdgpu_fence_driver_hw_init(struct amdgpu_device *adev);
 void amdgpu_fence_driver_hw_fini(struct amdgpu_device *adev);
 int amdgpu_fence_driver_sw_init(struct amdgpu_device *adev);
 void amdgpu_fence_driver_sw_fini(struct amdgpu_device *adev);
-int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **fence,
+int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **fence, struct amdgpu_job *job,
 		      unsigned flags);
 int amdgpu_fence_emit_polling(struct amdgpu_ring *ring, uint32_t *s,
 			      uint32_t timeout);
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1 -1
index 2a88ed5d983b..2af8860d74cc 100644
@@ -1218,7 +1218,7 @@ int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job,
 		amdgpu_gmc_emit_pasid_mapping(ring, job->vmid, job->pasid);
 
 	if (vm_flush_needed || pasid_mapping_needed) {
-		r = amdgpu_fence_emit(ring, &fence, 0);
+		r = amdgpu_fence_emit(ring, &fence, NULL, 0);
 		if (r)
 			return r;
 	}
Signature · 1 line
2.25.1
#1 lore raw

PATCH v2 2/2 drm/amd/amdgpu: add tdr support for embeded hw_fence

[Why]
After embeded hw_fence to amdgpu_job, we need to add tdr support for this feature.

[How]
1. Add a resubmit_flag for resubmit jobs.
2. Clear job fence from RCU and force complete vm flush fences in pre_asic_reset
3. skip dma_fence_get for resubmit jobs and add a dma_fence_put for guilty jobs.
v2:
use a job_run_counter in amdgpu_job to replace the resubmit_flag in drm_sched_job. When the job_run_counter >= 1, it means this job is a resubmit job.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 12 +++++++++++-
 drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c  | 13 +++++++++----
 drivers/gpu/drm/amd/amdgpu/amdgpu_job.c    |  5 ++++-
 drivers/gpu/drm/amd/amdgpu/amdgpu_job.h    |  3 +++
 4 files changed, 27 insertions(+), 6 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +11 -1
index 9e53ff851496..ade2fa07a50a 100644
@@ -4447,7 +4447,7 @@ int amdgpu_device_mode1_reset(struct amdgpu_device *adev)
 int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
 				 struct amdgpu_reset_context *reset_context)
 {
-	int i, r = 0;
+	int i, j, r = 0;
 	struct amdgpu_job *job = NULL;
 	bool need_full_reset =
 		test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
@@ -4471,6 +4471,16 @@ int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
 		if (!ring || !ring->sched.thread)
 			continue;
 
+		/*clear job fence from fence drv to avoid force_completion
+		 *leave NULL and vm flush fence in fence drv */
+		for (j = 0; j <= ring->fence_drv.num_fences_mask; j ++) {
+			struct dma_fence *old,**ptr;
+			ptr = &ring->fence_drv.fences[j];
+			old = rcu_dereference_protected(*ptr, 1);
+			if (old && test_bit(AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT, &old->flags)) {
+				RCU_INIT_POINTER(*ptr, NULL);
+			}
+		}
 		/* after all hw jobs are reset, hw fence is meaningless, so force_completion */
 		amdgpu_fence_driver_force_completion(ring);
 	}
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +9 -4
index 5e29d797a265..c9752cf794fb 100644
@@ -159,10 +159,15 @@ int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f, struct amd
 	}
 
 	seq = ++ring->fence_drv.sync_seq;
-	dma_fence_init(fence, &amdgpu_fence_ops,
-		       &ring->fence_drv.lock,
-		       adev->fence_context + ring->idx,
-		       seq);
+	if (job != NULL && job->job_run_counter) {
+		/* reinit seq for resubmitted jobs */
+		fence->seqno = seq;
+	} else {
+		dma_fence_init(fence, &amdgpu_fence_ops,
+				&ring->fence_drv.lock,
+				adev->fence_context + ring->idx,
+				seq);
+	}
 
 	if (job != NULL) {
 		/* mark this fence has a parent job */
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +4 -1
index 65a395060de2..19b13a65c73b 100644
@@ -254,6 +254,7 @@ static struct dma_fence *amdgpu_job_run(struct drm_sched_job *sched_job)
 		dma_fence_set_error(finished, -ECANCELED);/* skip IB as well if VRAM lost */
 
 	if (finished->error < 0) {
+		dma_fence_put(&job->hw_fence);
 		DRM_INFO("Skip scheduling IBs!\n");
 	} else {
 		r = amdgpu_ib_schedule(ring, job->num_ibs, job->ibs, job,
@@ -262,7 +263,9 @@ static struct dma_fence *amdgpu_job_run(struct drm_sched_job *sched_job)
 			DRM_ERROR("Error scheduling IBs (%d)\n", r);
 	}
 
-	dma_fence_get(fence);
+	if (!job->job_run_counter)
+		dma_fence_get(fence);
+	job->job_run_counter ++;
 	amdgpu_job_free_resources(job);
 
 	fence = r ? ERR_PTR(r) : fence;
drivers/gpu/drm/amd/amdgpu/amdgpu_job.h +3 -0
index 92324c978534..1fa667f245e1 100644
@@ -64,6 +64,9 @@ struct amdgpu_job {
 	/* user fence handling */
 	uint64_t		uf_addr;
 	uint64_t		uf_sequence;
+
+	/* job_run_counter >= 1 means a resubmit job */
+	uint32_t		job_run_counter;
 };
 
 int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs,
Signature · 1 line
2.25.1
#2 lore raw

Re: [PATCHv2 2/2] drm/amd/amdgpu: add tdr support for embeded hw_fence

Quoted 65 lines · Jingwen Chen

[Why]
After embeded hw_fence to amdgpu_job, we need to add tdr support for this feature.

[How]
1. Add a resubmit_flag for resubmit jobs.
2. Clear job fence from RCU and force complete vm flush fences in pre_asic_reset
3. skip dma_fence_get for resubmit jobs and add a dma_fence_put for guilty jobs.
v2:
use a job_run_counter in amdgpu_job to replace the resubmit_flag in drm_sched_job. When the job_run_counter >= 1, it means this job is a resubmit job.

---
  drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 12 +++++++++++-
  drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c  | 13 +++++++++----
  drivers/gpu/drm/amd/amdgpu/amdgpu_job.c    |  5 ++++-
  drivers/gpu/drm/amd/amdgpu/amdgpu_job.h    |  3 +++
  4 files changed, 27 insertions(+), 6 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +11 -1
index 9e53ff851496..ade2fa07a50a 100644
@@ -4447,7 +4447,7 @@ int amdgpu_device_mode1_reset(struct amdgpu_device *adev)
  int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
  				 struct amdgpu_reset_context *reset_context)
  {
-	int i, r = 0;
+	int i, j, r = 0;
  	struct amdgpu_job *job = NULL;
  	bool need_full_reset =
  		test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
@@ -4471,6 +4471,16 @@ int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
  		if (!ring || !ring->sched.thread)
  			continue;
 
+		/*clear job fence from fence drv to avoid force_completion
+		 *leave NULL and vm flush fence in fence drv */
+		for (j = 0; j <= ring->fence_drv.num_fences_mask; j ++) {
+			struct dma_fence *old,**ptr;
+			ptr = &ring->fence_drv.fences[j];
+			old = rcu_dereference_protected(*ptr, 1);
+			if (old && test_bit(AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT, &old->flags)) {
+				RCU_INIT_POINTER(*ptr, NULL);
+			}
+		}
  		/* after all hw jobs are reset, hw fence is meaningless, so force_completion */
  		amdgpu_fence_driver_force_completion(ring);
  	}
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0 -3
index 5e29d797a265..c9752cf794fb 100644
@@ -159,10 +159,15 @@ int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f, struct amd
  	}
 
  	seq = ++ring->fence_drv.sync_seq;
-	dma_fence_init(fence, &amdgpu_fence_ops,
-		       &ring->fence_drv.lock,
-		       adev->fence_context + ring->idx,

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f, struct amd · line 170

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +9 -1
-		       seq);
+	if (job != NULL && job->job_run_counter) {
+		/* reinit seq for resubmitted jobs */
+		fence->seqno = seq;
+	} else {
+		dma_fence_init(fence, &amdgpu_fence_ops,
+				&ring->fence_drv.lock,
+				adev->fence_context + ring->idx,
+				seq);
+	}

I think this should be in the first patch actually (and the counter too), without it the first patch is buggy.

drivers/gpu/drm/amd/amdgpu/amdgpu_job.c static struct dma_fence *amdgpu_job_run(struct drm_sched_job *sched_job) · line 257

Quoted 12 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0 -0
  	if (job != NULL) {
  		/* mark this fence has a parent job */
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +1 -0
index 65a395060de2..19b13a65c73b 100644
@@ -254,6 +254,7 @@ static struct dma_fence *amdgpu_job_run(struct drm_sched_job *sched_job)
  		dma_fence_set_error(finished, -ECANCELED);/* skip IB as well if VRAM lost */
 
  	if (finished->error < 0) {
+		dma_fence_put(&job->hw_fence);

Would put this check bellow with the job_run_counter check

drivers/gpu/drm/amd/amdgpu/amdgpu_job.c static struct dma_fence *amdgpu_job_run(struct drm_sched_job *sched_job) · line 269

Quoted 12 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +3 -1
  		DRM_INFO("Skip scheduling IBs!\n");
  	} else {
  		r = amdgpu_ib_schedule(ring, job->num_ibs, job->ibs, job,
@@ -262,7 +263,9 @@ static struct dma_fence *amdgpu_job_run(struct drm_sched_job *sched_job)
  			DRM_ERROR("Error scheduling IBs (%d)\n", r);
  	}
 
-	dma_fence_get(fence);
+	if (!job->job_run_counter)
+		dma_fence_get(fence);
+	job->job_run_counter ++;
  	amdgpu_job_free_resources(job);

Here you  modify code you already changed in patch 1, looks to me like those 2 patches should be squashed into one patch as the changes are directly dependent and it's hard to follow

Andrey

Quoted 16 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +0 -0
  	fence = r ? ERR_PTR(r) : fence;
drivers/gpu/drm/amd/amdgpu/amdgpu_job.h +3 -0
index 92324c978534..1fa667f245e1 100644
@@ -64,6 +64,9 @@ struct amdgpu_job {
  	/* user fence handling */
  	uint64_t		uf_addr;
  	uint64_t		uf_sequence;
+
+	/* job_run_counter >= 1 means a resubmit job */
+	uint32_t		job_run_counter;
  };
 
  int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs,
#3 lore raw

Re: [PATCHv2 2/2] drm/amd/amdgpu: add tdr support for embeded hw_fence

Quoted 71 lines · Andrey Grodzovsky
Quoted 69 lines · Jingwen Chen

[Why]
After embeded hw_fence to amdgpu_job, we need to add tdr support for this feature.

[How]
1. Add a resubmit_flag for resubmit jobs.
2. Clear job fence from RCU and force complete vm flush fences in pre_asic_reset
3. skip dma_fence_get for resubmit jobs and add a dma_fence_put for guilty jobs.
v2:
use a job_run_counter in amdgpu_job to replace the resubmit_flag in drm_sched_job. When the job_run_counter >= 1, it means this job is a resubmit job.

---
  drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 12 +++++++++++-
  drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c  | 13 +++++++++----
  drivers/gpu/drm/amd/amdgpu/amdgpu_job.c    |  5 ++++-
  drivers/gpu/drm/amd/amdgpu/amdgpu_job.h    |  3 +++
  4 files changed, 27 insertions(+), 6 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +11 -1
index 9e53ff851496..ade2fa07a50a 100644
@@ -4447,7 +4447,7 @@ int amdgpu_device_mode1_reset(struct amdgpu_device *adev)
  int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
  				 struct amdgpu_reset_context *reset_context)
  {
-	int i, r = 0;
+	int i, j, r = 0;
  	struct amdgpu_job *job = NULL;
  	bool need_full_reset =
  		test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
@@ -4471,6 +4471,16 @@ int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
  		if (!ring || !ring->sched.thread)
  			continue;
+		/*clear job fence from fence drv to avoid force_completion
+		 *leave NULL and vm flush fence in fence drv */
+		for (j = 0; j <= ring->fence_drv.num_fences_mask; j ++) {
+			struct dma_fence *old,**ptr;
+			ptr = &ring->fence_drv.fences[j];
+			old = rcu_dereference_protected(*ptr, 1);
+			if (old && test_bit(AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT, &old->flags)) {
+				RCU_INIT_POINTER(*ptr, NULL);
+			}
+		}
  		/* after all hw jobs are reset, hw fence is meaningless, so force_completion */
  		amdgpu_fence_driver_force_completion(ring);
  	}
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +5 -4
index 5e29d797a265..c9752cf794fb 100644
@@ -159,10 +159,15 @@ int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f, struct amd
  	}
  	seq = ++ring->fence_drv.sync_seq;
-	dma_fence_init(fence, &amdgpu_fence_ops,
-		       &ring->fence_drv.lock,
-		       adev->fence_context + ring->idx,
-		       seq);
+	if (job != NULL && job->job_run_counter) {
+		/* reinit seq for resubmitted jobs */
+		fence->seqno = seq;
+	} else {
+		dma_fence_init(fence, &amdgpu_fence_ops,
Quoted 9 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f, struct amd · line 170

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +4 -0
+				&ring->fence_drv.lock,
+				adev->fence_context + ring->idx,
+				seq);
+	}

I think this should be in the first patch actually (and the counter too), without it the first patch is buggy.

I was originally split these two patches by adding job submission seqence and adding tdr sequence, But yes, I should merge these two patches otherwise the tdr sequence will fail without second patch. Will send a merged version today.

Best Regards,
Jingwen

Quoted 50 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_job.c static struct dma_fence *amdgpu_job_run(struct drm_sched_job *sched_job) · line 256

Quoted 11 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0 -0
  	if (job != NULL) {
  		/* mark this fence has a parent job */
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +1 -0
index 65a395060de2..19b13a65c73b 100644
@@ -254,6 +254,7 @@ static struct dma_fence *amdgpu_job_run(struct drm_sched_job *sched_job)
  		dma_fence_set_error(finished, -ECANCELED);/* skip IB as well if VRAM lost */
  	if (finished->error < 0) {
+		dma_fence_put(&job->hw_fence);

Would put this check bellow with the job_run_counter check

drivers/gpu/drm/amd/amdgpu/amdgpu_job.c static struct dma_fence *amdgpu_job_run(struct drm_sched_job *sched_job) · line 268

Quoted 11 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +3 -1
  		DRM_INFO("Skip scheduling IBs!\n");
  	} else {
  		r = amdgpu_ib_schedule(ring, job->num_ibs, job->ibs, job,
@@ -262,7 +263,9 @@ static struct dma_fence *amdgpu_job_run(struct drm_sched_job *sched_job)
  			DRM_ERROR("Error scheduling IBs (%d)\n", r);
  	}
-	dma_fence_get(fence);
+	if (!job->job_run_counter)
+		dma_fence_get(fence);
+	job->job_run_counter ++;
  	amdgpu_job_free_resources(job);

Here you  modify code you already changed in patch 1, looks to me like those 2 patches should be squashed into one patch as the changes are directly dependent and it's hard to follow

Andrey

Quoted 14 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +0 -0
  	fence = r ? ERR_PTR(r) : fence;
drivers/gpu/drm/amd/amdgpu/amdgpu_job.h +3 -0
index 92324c978534..1fa667f245e1 100644
@@ -64,6 +64,9 @@ struct amdgpu_job {
  	/* user fence handling */
  	uint64_t		uf_addr;
  	uint64_t		uf_sequence;
+
+	/* job_run_counter >= 1 means a resubmit job */
+	uint32_t		job_run_counter;
  };
  int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs,
#4 lore raw

Re: [PATCHv2 2/2] drm/amd/amdgpu: add tdr support for embeded hw_fence

Hi Andrey,

The latest patch [PATCH v4] drm/amd/amdgpu embed hw_fence into amdgpu_job has been sent to amd-gfx. can you help review this patch?

Best Regards,
Jingwen

Quoted 138 lines · Jingwen Chen
Quoted 71 lines · Andrey Grodzovsky
Quoted 69 lines · Jingwen Chen

[Why]
After embeded hw_fence to amdgpu_job, we need to add tdr support for this feature.

[How]
1. Add a resubmit_flag for resubmit jobs.
2. Clear job fence from RCU and force complete vm flush fences in pre_asic_reset
3. skip dma_fence_get for resubmit jobs and add a dma_fence_put for guilty jobs.
v2:
use a job_run_counter in amdgpu_job to replace the resubmit_flag in drm_sched_job. When the job_run_counter >= 1, it means this job is a resubmit job.

---
  drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 12 +++++++++++-
  drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c  | 13 +++++++++----
  drivers/gpu/drm/amd/amdgpu/amdgpu_job.c    |  5 ++++-
  drivers/gpu/drm/amd/amdgpu/amdgpu_job.h    |  3 +++
  4 files changed, 27 insertions(+), 6 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +11 -1
index 9e53ff851496..ade2fa07a50a 100644
@@ -4447,7 +4447,7 @@ int amdgpu_device_mode1_reset(struct amdgpu_device *adev)
  int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
  				 struct amdgpu_reset_context *reset_context)
  {
-	int i, r = 0;
+	int i, j, r = 0;
  	struct amdgpu_job *job = NULL;
  	bool need_full_reset =
  		test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
@@ -4471,6 +4471,16 @@ int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
  		if (!ring || !ring->sched.thread)
  			continue;
+		/*clear job fence from fence drv to avoid force_completion
+		 *leave NULL and vm flush fence in fence drv */
+		for (j = 0; j <= ring->fence_drv.num_fences_mask; j ++) {
+			struct dma_fence *old,**ptr;
+			ptr = &ring->fence_drv.fences[j];
+			old = rcu_dereference_protected(*ptr, 1);
+			if (old && test_bit(AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT, &old->flags)) {
+				RCU_INIT_POINTER(*ptr, NULL);
+			}
+		}
  		/* after all hw jobs are reset, hw fence is meaningless, so force_completion */
  		amdgpu_fence_driver_force_completion(ring);
  	}
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +5 -4
index 5e29d797a265..c9752cf794fb 100644
@@ -159,10 +159,15 @@ int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f, struct amd
  	}
  	seq = ++ring->fence_drv.sync_seq;
-	dma_fence_init(fence, &amdgpu_fence_ops,
-		       &ring->fence_drv.lock,
-		       adev->fence_context + ring->idx,
-		       seq);
+	if (job != NULL && job->job_run_counter) {
+		/* reinit seq for resubmitted jobs */
+		fence->seqno = seq;
+	} else {
+		dma_fence_init(fence, &amdgpu_fence_ops,
Quoted 9 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f, struct amd · line 170

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +4 -0
+				&ring->fence_drv.lock,
+				adev->fence_context + ring->idx,
+				seq);
+	}

I think this should be in the first patch actually (and the counter too), without it the first patch is buggy.

I was originally split these two patches by adding job submission seqence and adding tdr sequence, But yes, I should merge these two patches otherwise the tdr sequence will fail without second patch. Will send a merged version today.

Best Regards,
Jingwen

Quoted 50 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_job.c static struct dma_fence *amdgpu_job_run(struct drm_sched_job *sched_job) · line 256

Quoted 11 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0 -0
  	if (job != NULL) {
  		/* mark this fence has a parent job */
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +1 -0
index 65a395060de2..19b13a65c73b 100644
@@ -254,6 +254,7 @@ static struct dma_fence *amdgpu_job_run(struct drm_sched_job *sched_job)
  		dma_fence_set_error(finished, -ECANCELED);/* skip IB as well if VRAM lost */
  	if (finished->error < 0) {
+		dma_fence_put(&job->hw_fence);

Would put this check bellow with the job_run_counter check

drivers/gpu/drm/amd/amdgpu/amdgpu_job.c static struct dma_fence *amdgpu_job_run(struct drm_sched_job *sched_job) · line 268

Quoted 11 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +3 -1
  		DRM_INFO("Skip scheduling IBs!\n");
  	} else {
  		r = amdgpu_ib_schedule(ring, job->num_ibs, job->ibs, job,
@@ -262,7 +263,9 @@ static struct dma_fence *amdgpu_job_run(struct drm_sched_job *sched_job)
  			DRM_ERROR("Error scheduling IBs (%d)\n", r);
  	}
-	dma_fence_get(fence);
+	if (!job->job_run_counter)
+		dma_fence_get(fence);
+	job->job_run_counter ++;
  	amdgpu_job_free_resources(job);

Here you  modify code you already changed in patch 1, looks to me like those 2 patches should be squashed into one patch as the changes are directly dependent and it's hard to follow

Andrey

Quoted 14 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +0 -0
  	fence = r ? ERR_PTR(r) : fence;
drivers/gpu/drm/amd/amdgpu/amdgpu_job.h +3 -0
index 92324c978534..1fa667f245e1 100644
@@ -64,6 +64,9 @@ struct amdgpu_job {
  	/* user fence handling */
  	uint64_t		uf_addr;
  	uint64_t		uf_sequence;
+
+	/* job_run_counter >= 1 means a resubmit job */
+	uint32_t		job_run_counter;
  };
  int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs,
#5 lore raw
In reply to Jingwen Chen
Quoted 84 lines · Jingwen Chen

From: Jack Zhang <[email protected]>

Why: Previously hw fence is alloced separately with job. It caused historical lifetime issues and corner cases. The ideal situation is to take fence to manage both job and fence's lifetime, and simplify the design of gpu-scheduler.

How:
We propose to embed hw_fence into amdgpu_job.
1. We cover the normal job submission by this method.
2. For ib_test, and submit without a parent job keep the legacy way to create a hw fence separately.
v2:
use AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT to show that the fence is embeded in a job.

---
  drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c  |  1 -
  drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c |  2 +-
  drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c   | 63 ++++++++++++++++-----
  drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c      |  2 +-
  drivers/gpu/drm/amd/amdgpu/amdgpu_job.c     | 35 ++++++++----
  drivers/gpu/drm/amd/amdgpu/amdgpu_job.h     |  4 +-
  drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h    |  5 +-
  drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c      |  2 +-
  8 files changed, 84 insertions(+), 30 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +0 -1
index 7b46ba551cb2..3003ee1c9487 100644
@@ -714,7 +714,6 @@ int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine,
  	ret = dma_fence_wait(f, false);
 
  err_ib_sched:
-	dma_fence_put(f);
  	amdgpu_job_free(job);
  err:
  	return ret;
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +1 -1
index 536005bff24a..277128846dd1 100644
@@ -1414,7 +1414,7 @@ static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring)
  			continue;
  		}
  		job = to_amdgpu_job(s_job);
-		if (preempted && job->fence == fence)
+		if (preempted && (&job->hw_fence) == fence)
  			/* mark the job as preempted */
  			job->preemption_status |= AMDGPU_IB_PREEMPTED;
  	}
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +5 -5
index 7495911516c2..5e29d797a265 100644
@@ -129,30 +129,46 @@ static u32 amdgpu_fence_read(struct amdgpu_ring *ring)
   *
   * @ring: ring the fence is associated with
   * @f: resulting fence object
+ * @job: job the fence is embeded in
   * @flags: flags to pass into the subordinate .emit_fence() call
   *
   * Emits a fence command on the requested ring (all asics).
   * Returns 0 on success, -ENOMEM on failure.
   */
-int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f,
+int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f, struct amdgpu_job *job,
  		      unsigned flags)
  {
  	struct amdgpu_device *adev = ring->adev;
-	struct amdgpu_fence *fence;
+	struct dma_fence *fence;
+	struct amdgpu_fence *am_fence;
  	struct dma_fence __rcu **ptr;
  	uint32_t seq;
  	int r;
 
-	fence = kmem_cache_alloc(amdgpu_fence_slab, GFP_KERNEL);
-	if (fence == NULL)
-		return -ENOMEM;
+	if (job == NULL) {

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static u32 amdgpu_fence_read(struct amdgpu_ring *ring) · line 158

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +10 -0
+		/* create a sperate hw fence */
+		am_fence = kmem_cache_alloc(amdgpu_fence_slab, GFP_ATOMIC);
+		if (am_fence == NULL)
+			return -ENOMEM;
+		fence = &am_fence->base;
+		am_fence->ring = ring;
+	} else {
+		/* take use of job-embedded fence */
+		fence = &job->hw_fence;
+		job->ring = ring;

If you would make hw_fence of type amdgpu_fence you could probably avoid the special job->ring = ring See more in related comment at the bottom

Quoted 35 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +10 -5
+	}
 
  	seq = ++ring->fence_drv.sync_seq;
-	fence->ring = ring;
-	dma_fence_init(&fence->base, &amdgpu_fence_ops,
+	dma_fence_init(fence, &amdgpu_fence_ops,
  		       &ring->fence_drv.lock,
  		       adev->fence_context + ring->idx,
  		       seq);
+
+	if (job != NULL) {
+		/* mark this fence has a parent job */
+		set_bit(AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT, &fence->flags);
+	}
+
  	amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,
  			       seq, flags | AMDGPU_FENCE_FLAG_INT);
  	pm_runtime_get_noresume(adev_to_drm(adev)->dev);
@@ -175,9 +191,9 @@ int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f,
  	/* This function can't be called concurrently anyway, otherwise
  	 * emitting the fence would mess up the hardware ring buffer.
  	 */
-	rcu_assign_pointer(*ptr, dma_fence_get(&fence->base));
+	rcu_assign_pointer(*ptr, dma_fence_get(fence));
 
-	*f = &fence->base;
+	*f = fence;
 
  	return 0;
  }
@@ -621,8 +637,16 @@ static const char *amdgpu_fence_get_driver_name(struct dma_fence *fence)
 
  static const char *amdgpu_fence_get_timeline_name(struct dma_fence *f)
  {
-	struct amdgpu_fence *fence = to_amdgpu_fence(f);

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static const char *amdgpu_fence_get_driver_name(struct dma_fence *fence) · line 648

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +9 -1
-	return (const char *)fence->ring->name;
+	struct amdgpu_ring *ring;
+
+	if (test_bit(AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT, &f->flags)) {
+		struct amdgpu_job *job = container_of(f, struct amdgpu_job, hw_fence);
+
+		ring = job->ring;
+	} else {
+		ring = to_amdgpu_fence(f)->ring;
+	}

Same as above

Quoted 137 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +16 -2
+	return (const char *)ring->name;
  }
 
  /**
@@ -656,8 +680,20 @@ static bool amdgpu_fence_enable_signaling(struct dma_fence *f)
  static void amdgpu_fence_free(struct rcu_head *rcu)
  {
  	struct dma_fence *f = container_of(rcu, struct dma_fence, rcu);
-	struct amdgpu_fence *fence = to_amdgpu_fence(f);
-	kmem_cache_free(amdgpu_fence_slab, fence);
+
+	if (test_bit(AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT, &f->flags)) {
+	/* free job if fence has a parent job */
+		struct amdgpu_job *job;
+
+		job = container_of(f, struct amdgpu_job, hw_fence);
+		kfree(job);
+	} else {
+	/* free fence_slab if it's separated fence*/
+		struct amdgpu_fence *fence;
+
+		fence = to_amdgpu_fence(f);
+		kmem_cache_free(amdgpu_fence_slab, fence);
+	}
  }
 
  /**
@@ -680,6 +716,7 @@ static const struct dma_fence_ops amdgpu_fence_ops = {
  	.release = amdgpu_fence_release,
  };
 
+
  /*
   * Fence debugfs
   */
drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c +1 -1
index ec65ab0ddf89..c076a6b9a5a2 100644
@@ -262,7 +262,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
  				       fence_flags | AMDGPU_FENCE_FLAG_64BIT);
  	}
 
-	r = amdgpu_fence_emit(ring, f, fence_flags);
+	r = amdgpu_fence_emit(ring, f, job, fence_flags);
  	if (r) {
  		dev_err(adev->dev, "failed to emit fence (%d)\n", r);
  		if (job && job->vmid)
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +24 -11
index d33e6d97cc89..65a395060de2 100644
@@ -127,11 +127,16 @@ void amdgpu_job_free_resources(struct amdgpu_job *job)
  {
  	struct amdgpu_ring *ring = to_amdgpu_ring(job->base.sched);
  	struct dma_fence *f;
+	struct dma_fence *hw_fence;
  	unsigned i;
 
-	/* use sched fence if available */
-	f = job->base.s_fence ? &job->base.s_fence->finished : job->fence;
+	if (job->hw_fence.ops == NULL)
+		hw_fence = job->external_hw_fence;
+	else
+		hw_fence = &job->hw_fence;
 
+	/* use sched fence if available */
+	f = job->base.s_fence ? &job->base.s_fence->finished : hw_fence;
  	for (i = 0; i < job->num_ibs; ++i)
  		amdgpu_ib_free(ring->adev, &job->ibs[i], f);
  }
@@ -142,20 +147,27 @@ static void amdgpu_job_free_cb(struct drm_sched_job *s_job)
 
  	drm_sched_job_cleanup(s_job);
 
-	dma_fence_put(job->fence);
  	amdgpu_sync_free(&job->sync);
  	amdgpu_sync_free(&job->sched_sync);
-	kfree(job);
+
+    /* only put the hw fence if has embedded fence */
+	if (job->hw_fence.ops != NULL)
+		dma_fence_put(&job->hw_fence);
+	else
+		kfree(job);
  }
 
  void amdgpu_job_free(struct amdgpu_job *job)
  {
  	amdgpu_job_free_resources(job);
-
-	dma_fence_put(job->fence);
  	amdgpu_sync_free(&job->sync);
  	amdgpu_sync_free(&job->sched_sync);
-	kfree(job);
+
+	/* only put the hw fence if has embedded fence */
+	if (job->hw_fence.ops != NULL)
+		dma_fence_put(&job->hw_fence);
+	else
+		kfree(job);
  }
 
  int amdgpu_job_submit(struct amdgpu_job *job, struct drm_sched_entity *entity,
@@ -184,11 +196,14 @@ int amdgpu_job_submit_direct(struct amdgpu_job *job, struct amdgpu_ring *ring,
 
  	job->base.sched = &ring->sched;
  	r = amdgpu_ib_schedule(ring, job->num_ibs, job->ibs, NULL, fence);
-	job->fence = dma_fence_get(*fence);
+	/* record external_hw_fence for direct submit */
+	job->external_hw_fence = dma_fence_get(*fence);
  	if (r)
  		return r;
 
  	amdgpu_job_free(job);
+	dma_fence_put(*fence);
+
  	return 0;
  }
 
@@ -246,10 +261,8 @@ static struct dma_fence *amdgpu_job_run(struct drm_sched_job *sched_job)
  		if (r)
  			DRM_ERROR("Error scheduling IBs (%d)\n", r);
  	}
-	/* if gpu reset, hw fence will be replaced here */
-	dma_fence_put(job->fence);
-	job->fence = dma_fence_get(fence);
 
+	dma_fence_get(fence);
  	amdgpu_job_free_resources(job);
 
  	fence = r ? ERR_PTR(r) : fence;
drivers/gpu/drm/amd/amdgpu/amdgpu_job.h +0 -0
index 81caac9b958a..92324c978534 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_job.h struct amdgpu_job { · line 50

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_job.h +2 -1
@@ -46,7 +46,9 @@ struct amdgpu_job {
  	struct amdgpu_sync	sync;
  	struct amdgpu_sync	sched_sync;
  	struct amdgpu_ib	*ibs;
-	struct dma_fence	*fence; /* the hw fence */
+	struct dma_fence	hw_fence;
+	struct amdgpu_ring *ring;

Why not instead of 2 fields above just embed   struct amdgpu_fence as hw_fence  and by this save the extra 'ring' field handling ?

Andrey

Quoted 40 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_job.h +1 -0
+	struct dma_fence	*external_hw_fence;
  	uint32_t		preamble_status;
  	uint32_t                preemption_status;
  	uint32_t		num_ibs;
drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +4 -1
index 9c11ced4312c..03d4b29a76d6 100644
@@ -48,6 +48,9 @@
  #define AMDGPU_FENCE_FLAG_INT           (1 << 1)
  #define AMDGPU_FENCE_FLAG_TC_WB_ONLY    (1 << 2)
 
+/* fence flag bit to indicate the face is embeded in job*/
+#define AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT		(DMA_FENCE_FLAG_USER_BITS + 1)
+
  #define to_amdgpu_ring(s) container_of((s), struct amdgpu_ring, sched)
 
  #define AMDGPU_IB_POOL_SIZE	(1024 * 1024)
@@ -118,7 +121,7 @@ void amdgpu_fence_driver_hw_init(struct amdgpu_device *adev);
  void amdgpu_fence_driver_hw_fini(struct amdgpu_device *adev);
  int amdgpu_fence_driver_sw_init(struct amdgpu_device *adev);
  void amdgpu_fence_driver_sw_fini(struct amdgpu_device *adev);
-int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **fence,
+int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **fence, struct amdgpu_job *job,
  		      unsigned flags);
  int amdgpu_fence_emit_polling(struct amdgpu_ring *ring, uint32_t *s,
  			      uint32_t timeout);
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1 -1
index 2a88ed5d983b..2af8860d74cc 100644
@@ -1218,7 +1218,7 @@ int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job,
  		amdgpu_gmc_emit_pasid_mapping(ring, job->vmid, job->pasid);
 
  	if (vm_flush_needed || pasid_mapping_needed) {
-		r = amdgpu_fence_emit(ring, &fence, 0);
+		r = amdgpu_fence_emit(ring, &fence, NULL, 0);
  		if (r)
  			return r;
  	}
#6 lore raw
Quoted 91 lines · Andrey Grodzovsky
Quoted 89 lines · Jingwen Chen

From: Jack Zhang <[email protected]>

Why: Previously hw fence is alloced separately with job. It caused historical lifetime issues and corner cases. The ideal situation is to take fence to manage both job and fence's lifetime, and simplify the design of gpu-scheduler.

How:
We propose to embed hw_fence into amdgpu_job.
1. We cover the normal job submission by this method.
2. For ib_test, and submit without a parent job keep the legacy way to create a hw fence separately.
v2:
use AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT to show that the fence is embeded in a job.

---
  drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c  |  1 -
  drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c |  2 +-
  drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c   | 63 ++++++++++++++++-----
  drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c      |  2 +-
  drivers/gpu/drm/amd/amdgpu/amdgpu_job.c     | 35 ++++++++----
  drivers/gpu/drm/amd/amdgpu/amdgpu_job.h     |  4 +-
  drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h    |  5 +-
  drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c      |  2 +-
  8 files changed, 84 insertions(+), 30 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +0 -1
index 7b46ba551cb2..3003ee1c9487 100644
@@ -714,7 +714,6 @@ int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine,
  	ret = dma_fence_wait(f, false);
  err_ib_sched:
-	dma_fence_put(f);
  	amdgpu_job_free(job);
  err:
  	return ret;
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +1 -1
index 536005bff24a..277128846dd1 100644
@@ -1414,7 +1414,7 @@ static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring)
  			continue;
  		}
  		job = to_amdgpu_job(s_job);
-		if (preempted && job->fence == fence)
+		if (preempted && (&job->hw_fence) == fence)
  			/* mark the job as preempted */
  			job->preemption_status |= AMDGPU_IB_PREEMPTED;
  	}
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +12 -5
index 7495911516c2..5e29d797a265 100644
@@ -129,30 +129,46 @@ static u32 amdgpu_fence_read(struct amdgpu_ring *ring)
   *
   * @ring: ring the fence is associated with
   * @f: resulting fence object
+ * @job: job the fence is embeded in
   * @flags: flags to pass into the subordinate .emit_fence() call
   *
   * Emits a fence command on the requested ring (all asics).
   * Returns 0 on success, -ENOMEM on failure.
   */
-int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f,
+int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f, struct amdgpu_job *job,
  		      unsigned flags)
  {
  	struct amdgpu_device *adev = ring->adev;
-	struct amdgpu_fence *fence;
+	struct dma_fence *fence;
+	struct amdgpu_fence *am_fence;
  	struct dma_fence __rcu **ptr;
  	uint32_t seq;
  	int r;
-	fence = kmem_cache_alloc(amdgpu_fence_slab, GFP_KERNEL);
-	if (fence == NULL)
-		return -ENOMEM;
+	if (job == NULL) {
+		/* create a sperate hw fence */
+		am_fence = kmem_cache_alloc(amdgpu_fence_slab, GFP_ATOMIC);
+		if (am_fence == NULL)
+			return -ENOMEM;
+		fence = &am_fence->base;
+		am_fence->ring = ring;
+	} else {
Quoted 9 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static u32 amdgpu_fence_read(struct amdgpu_ring *ring) · line 158

Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +3 -0
+		/* take use of job-embedded fence */
+		fence = &job->hw_fence;
+		job->ring = ring;

If you would make hw_fence of type amdgpu_fence you could probably avoid the special job->ring = ring See more in related comment at the bottom

Hi Andry,

I'm only make the amdgpu_fence for the fence without job parameter provided to amdgpu_fence_emit. For embeded fence which is the hw_fence in amdgpu_job, it will be allocated along with amdgpu_job as dma_fence.

Regards,
Jingwen Chen

Quoted 219 lines
Quoted 32 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +10 -5
+	}
  	seq = ++ring->fence_drv.sync_seq;
-	fence->ring = ring;
-	dma_fence_init(&fence->base, &amdgpu_fence_ops,
+	dma_fence_init(fence, &amdgpu_fence_ops,
  		       &ring->fence_drv.lock,
  		       adev->fence_context + ring->idx,
  		       seq);
+
+	if (job != NULL) {
+		/* mark this fence has a parent job */
+		set_bit(AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT, &fence->flags);
+	}
+
  	amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,
  			       seq, flags | AMDGPU_FENCE_FLAG_INT);
  	pm_runtime_get_noresume(adev_to_drm(adev)->dev);
@@ -175,9 +191,9 @@ int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f,
  	/* This function can't be called concurrently anyway, otherwise
  	 * emitting the fence would mess up the hardware ring buffer.
  	 */
-	rcu_assign_pointer(*ptr, dma_fence_get(&fence->base));
+	rcu_assign_pointer(*ptr, dma_fence_get(fence));
-	*f = &fence->base;
+	*f = fence;
  	return 0;
  }
@@ -621,8 +637,16 @@ static const char *amdgpu_fence_get_driver_name(struct dma_fence *fence)
  static const char *amdgpu_fence_get_timeline_name(struct dma_fence *f)
  {
-	struct amdgpu_fence *fence = to_amdgpu_fence(f);

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static const char *amdgpu_fence_get_driver_name(struct dma_fence *fence) · line 647

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +9 -1
-	return (const char *)fence->ring->name;
+	struct amdgpu_ring *ring;
+
+	if (test_bit(AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT, &f->flags)) {
+		struct amdgpu_job *job = container_of(f, struct amdgpu_job, hw_fence);
+
+		ring = job->ring;
+	} else {
+		ring = to_amdgpu_fence(f)->ring;
+	}

Same as above

Quoted 122 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +16 -2
+	return (const char *)ring->name;
  }
  /**
@@ -656,8 +680,20 @@ static bool amdgpu_fence_enable_signaling(struct dma_fence *f)
  static void amdgpu_fence_free(struct rcu_head *rcu)
  {
  	struct dma_fence *f = container_of(rcu, struct dma_fence, rcu);
-	struct amdgpu_fence *fence = to_amdgpu_fence(f);
-	kmem_cache_free(amdgpu_fence_slab, fence);
+
+	if (test_bit(AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT, &f->flags)) {
+	/* free job if fence has a parent job */
+		struct amdgpu_job *job;
+
+		job = container_of(f, struct amdgpu_job, hw_fence);
+		kfree(job);
+	} else {
+	/* free fence_slab if it's separated fence*/
+		struct amdgpu_fence *fence;
+
+		fence = to_amdgpu_fence(f);
+		kmem_cache_free(amdgpu_fence_slab, fence);
+	}
  }
  /**
@@ -680,6 +716,7 @@ static const struct dma_fence_ops amdgpu_fence_ops = {
  	.release = amdgpu_fence_release,
  };
+
  /*
   * Fence debugfs
   */
drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c +1 -1
index ec65ab0ddf89..c076a6b9a5a2 100644
@@ -262,7 +262,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
  				       fence_flags | AMDGPU_FENCE_FLAG_64BIT);
  	}
-	r = amdgpu_fence_emit(ring, f, fence_flags);
+	r = amdgpu_fence_emit(ring, f, job, fence_flags);
  	if (r) {
  		dev_err(adev->dev, "failed to emit fence (%d)\n", r);
  		if (job && job->vmid)
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +24 -11
index d33e6d97cc89..65a395060de2 100644
@@ -127,11 +127,16 @@ void amdgpu_job_free_resources(struct amdgpu_job *job)
  {
  	struct amdgpu_ring *ring = to_amdgpu_ring(job->base.sched);
  	struct dma_fence *f;
+	struct dma_fence *hw_fence;
  	unsigned i;
-	/* use sched fence if available */
-	f = job->base.s_fence ? &job->base.s_fence->finished : job->fence;
+	if (job->hw_fence.ops == NULL)
+		hw_fence = job->external_hw_fence;
+	else
+		hw_fence = &job->hw_fence;
+	/* use sched fence if available */
+	f = job->base.s_fence ? &job->base.s_fence->finished : hw_fence;
  	for (i = 0; i < job->num_ibs; ++i)
  		amdgpu_ib_free(ring->adev, &job->ibs[i], f);
  }
@@ -142,20 +147,27 @@ static void amdgpu_job_free_cb(struct drm_sched_job *s_job)
  	drm_sched_job_cleanup(s_job);
-	dma_fence_put(job->fence);
  	amdgpu_sync_free(&job->sync);
  	amdgpu_sync_free(&job->sched_sync);
-	kfree(job);
+
+    /* only put the hw fence if has embedded fence */
+	if (job->hw_fence.ops != NULL)
+		dma_fence_put(&job->hw_fence);
+	else
+		kfree(job);
  }
  void amdgpu_job_free(struct amdgpu_job *job)
  {
  	amdgpu_job_free_resources(job);
-
-	dma_fence_put(job->fence);
  	amdgpu_sync_free(&job->sync);
  	amdgpu_sync_free(&job->sched_sync);
-	kfree(job);
+
+	/* only put the hw fence if has embedded fence */
+	if (job->hw_fence.ops != NULL)
+		dma_fence_put(&job->hw_fence);
+	else
+		kfree(job);
  }
  int amdgpu_job_submit(struct amdgpu_job *job, struct drm_sched_entity *entity,
@@ -184,11 +196,14 @@ int amdgpu_job_submit_direct(struct amdgpu_job *job, struct amdgpu_ring *ring,
  	job->base.sched = &ring->sched;
  	r = amdgpu_ib_schedule(ring, job->num_ibs, job->ibs, NULL, fence);
-	job->fence = dma_fence_get(*fence);
+	/* record external_hw_fence for direct submit */
+	job->external_hw_fence = dma_fence_get(*fence);
  	if (r)
  		return r;
  	amdgpu_job_free(job);
+	dma_fence_put(*fence);
+
  	return 0;
  }
@@ -246,10 +261,8 @@ static struct dma_fence *amdgpu_job_run(struct drm_sched_job *sched_job)
  		if (r)
  			DRM_ERROR("Error scheduling IBs (%d)\n", r);
  	}
-	/* if gpu reset, hw fence will be replaced here */
-	dma_fence_put(job->fence);
-	job->fence = dma_fence_get(fence);
+	dma_fence_get(fence);
  	amdgpu_job_free_resources(job);
  	fence = r ? ERR_PTR(r) : fence;
drivers/gpu/drm/amd/amdgpu/amdgpu_job.h +0 -0
index 81caac9b958a..92324c978534 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_job.h struct amdgpu_job { · line 50

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_job.h +2 -1
@@ -46,7 +46,9 @@ struct amdgpu_job {
  	struct amdgpu_sync	sync;
  	struct amdgpu_sync	sched_sync;
  	struct amdgpu_ib	*ibs;
-	struct dma_fence	*fence; /* the hw fence */
+	struct dma_fence	hw_fence;
+	struct amdgpu_ring *ring;

Why not instead of 2 fields above just embed   struct amdgpu_fence as hw_fence  and by this save the extra 'ring' field handling ?

Andrey

Quoted 37 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_job.h +1 -0
+	struct dma_fence	*external_hw_fence;
  	uint32_t		preamble_status;
  	uint32_t                preemption_status;
  	uint32_t		num_ibs;
drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +4 -1
index 9c11ced4312c..03d4b29a76d6 100644
@@ -48,6 +48,9 @@
  #define AMDGPU_FENCE_FLAG_INT           (1 << 1)
  #define AMDGPU_FENCE_FLAG_TC_WB_ONLY    (1 << 2)
+/* fence flag bit to indicate the face is embeded in job*/
+#define AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT		(DMA_FENCE_FLAG_USER_BITS + 1)
+
  #define to_amdgpu_ring(s) container_of((s), struct amdgpu_ring, sched)
  #define AMDGPU_IB_POOL_SIZE	(1024 * 1024)
@@ -118,7 +121,7 @@ void amdgpu_fence_driver_hw_init(struct amdgpu_device *adev);
  void amdgpu_fence_driver_hw_fini(struct amdgpu_device *adev);
  int amdgpu_fence_driver_sw_init(struct amdgpu_device *adev);
  void amdgpu_fence_driver_sw_fini(struct amdgpu_device *adev);
-int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **fence,
+int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **fence, struct amdgpu_job *job,
  		      unsigned flags);
  int amdgpu_fence_emit_polling(struct amdgpu_ring *ring, uint32_t *s,
  			      uint32_t timeout);
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1 -1
index 2a88ed5d983b..2af8860d74cc 100644
@@ -1218,7 +1218,7 @@ int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job,
  		amdgpu_gmc_emit_pasid_mapping(ring, job->vmid, job->pasid);
  	if (vm_flush_needed || pasid_mapping_needed) {
-		r = amdgpu_fence_emit(ring, &fence, 0);
+		r = amdgpu_fence_emit(ring, &fence, NULL, 0);
  		if (r)
  			return r;
  	}
#7 lore raw

Am 06.08.21 um 07:52 schrieb Jingwen Chen:

Quoted 99 lines · Andrey Grodzovsky
Quoted 89 lines · Jingwen Chen
Quoted 88 lines

From: Jack Zhang <[email protected]>

Why: Previously hw fence is alloced separately with job. It caused historical lifetime issues and corner cases. The ideal situation is to take fence to manage both job and fence's lifetime, and simplify the design of gpu-scheduler.

How:
We propose to embed hw_fence into amdgpu_job.
1. We cover the normal job submission by this method.
2. For ib_test, and submit without a parent job keep the legacy way to create a hw fence separately.
v2:
use AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT to show that the fence is embeded in a job.

---
   drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c  |  1 -
   drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c |  2 +-
   drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c   | 63 ++++++++++++++++-----
   drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c      |  2 +-
   drivers/gpu/drm/amd/amdgpu/amdgpu_job.c     | 35 ++++++++----
   drivers/gpu/drm/amd/amdgpu/amdgpu_job.h     |  4 +-
   drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h    |  5 +-
   drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c      |  2 +-
   8 files changed, 84 insertions(+), 30 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +0 -1
index 7b46ba551cb2..3003ee1c9487 100644
@@ -714,7 +714,6 @@ int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine,
   	ret = dma_fence_wait(f, false);
   err_ib_sched:
-	dma_fence_put(f);
   	amdgpu_job_free(job);
   err:
   	return ret;
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +1 -1
index 536005bff24a..277128846dd1 100644
@@ -1414,7 +1414,7 @@ static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring)
   			continue;
   		}
   		job = to_amdgpu_job(s_job);
-		if (preempted && job->fence == fence)
+		if (preempted && (&job->hw_fence) == fence)
   			/* mark the job as preempted */
   			job->preemption_status |= AMDGPU_IB_PREEMPTED;
   	}
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +11 -5
index 7495911516c2..5e29d797a265 100644
@@ -129,30 +129,46 @@ static u32 amdgpu_fence_read(struct amdgpu_ring *ring)
    *
    * @ring: ring the fence is associated with
    * @f: resulting fence object
+ * @job: job the fence is embeded in
    * @flags: flags to pass into the subordinate .emit_fence() call
    *
    * Emits a fence command on the requested ring (all asics).
    * Returns 0 on success, -ENOMEM on failure.
    */
-int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f,
+int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f, struct amdgpu_job *job,
   		      unsigned flags)
   {
   	struct amdgpu_device *adev = ring->adev;
-	struct amdgpu_fence *fence;
+	struct dma_fence *fence;
+	struct amdgpu_fence *am_fence;
   	struct dma_fence __rcu **ptr;
   	uint32_t seq;
   	int r;
-	fence = kmem_cache_alloc(amdgpu_fence_slab, GFP_KERNEL);
-	if (fence == NULL)
-		return -ENOMEM;
+	if (job == NULL) {
+		/* create a sperate hw fence */
+		am_fence = kmem_cache_alloc(amdgpu_fence_slab, GFP_ATOMIC);
+		if (am_fence == NULL)
+			return -ENOMEM;
+		fence = &am_fence->base;
+		am_fence->ring = ring;
Quoted 9 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static u32 amdgpu_fence_read(struct amdgpu_ring *ring) · line 158

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +4 -0
+	} else {
+		/* take use of job-embedded fence */
+		fence = &job->hw_fence;
+		job->ring = ring;

If you would make hw_fence of type amdgpu_fence you could probably avoid the special job->ring = ring See more in related comment at the bottom

Quoted 5 lines

Hi Andry,

I'm only make the amdgpu_fence for the fence without job parameter provided to amdgpu_fence_emit. For embeded fence which is the hw_fence in amdgpu_job, it will be allocated along with amdgpu_job as dma_fence.

When you have the job and need the ring you can just do conatiner_of(job->sched, struct amdgpu_ring, sched).

No need for an extra ring pointer here.

Regards,
Christian.

Quoted 219 lines

Regards,
Jingwen Chen

Quoted 216 lines
Quoted 31 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +10 -5
+	}
   	seq = ++ring->fence_drv.sync_seq;
-	fence->ring = ring;
-	dma_fence_init(&fence->base, &amdgpu_fence_ops,
+	dma_fence_init(fence, &amdgpu_fence_ops,
   		       &ring->fence_drv.lock,
   		       adev->fence_context + ring->idx,
   		       seq);
+
+	if (job != NULL) {
+		/* mark this fence has a parent job */
+		set_bit(AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT, &fence->flags);
+	}
+
   	amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,
   			       seq, flags | AMDGPU_FENCE_FLAG_INT);
   	pm_runtime_get_noresume(adev_to_drm(adev)->dev);
@@ -175,9 +191,9 @@ int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f,
   	/* This function can't be called concurrently anyway, otherwise
   	 * emitting the fence would mess up the hardware ring buffer.
   	 */
-	rcu_assign_pointer(*ptr, dma_fence_get(&fence->base));
+	rcu_assign_pointer(*ptr, dma_fence_get(fence));
-	*f = &fence->base;
+	*f = fence;
   	return 0;
   }
@@ -621,8 +637,16 @@ static const char *amdgpu_fence_get_driver_name(struct dma_fence *fence)
   static const char *amdgpu_fence_get_timeline_name(struct dma_fence *f)
   {
-	struct amdgpu_fence *fence = to_amdgpu_fence(f);

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static const char *amdgpu_fence_get_driver_name(struct dma_fence *fence) · line 647

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +9 -1
-	return (const char *)fence->ring->name;
+	struct amdgpu_ring *ring;
+
+	if (test_bit(AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT, &f->flags)) {
+		struct amdgpu_job *job = container_of(f, struct amdgpu_job, hw_fence);
+
+		ring = job->ring;
+	} else {
+		ring = to_amdgpu_fence(f)->ring;
+	}

Same as above

Quoted 122 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +16 -2
+	return (const char *)ring->name;
   }
   /**
@@ -656,8 +680,20 @@ static bool amdgpu_fence_enable_signaling(struct dma_fence *f)
   static void amdgpu_fence_free(struct rcu_head *rcu)
   {
   	struct dma_fence *f = container_of(rcu, struct dma_fence, rcu);
-	struct amdgpu_fence *fence = to_amdgpu_fence(f);
-	kmem_cache_free(amdgpu_fence_slab, fence);
+
+	if (test_bit(AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT, &f->flags)) {
+	/* free job if fence has a parent job */
+		struct amdgpu_job *job;
+
+		job = container_of(f, struct amdgpu_job, hw_fence);
+		kfree(job);
+	} else {
+	/* free fence_slab if it's separated fence*/
+		struct amdgpu_fence *fence;
+
+		fence = to_amdgpu_fence(f);
+		kmem_cache_free(amdgpu_fence_slab, fence);
+	}
   }
   /**
@@ -680,6 +716,7 @@ static const struct dma_fence_ops amdgpu_fence_ops = {
   	.release = amdgpu_fence_release,
   };
+
   /*
    * Fence debugfs
    */
drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c +1 -1
index ec65ab0ddf89..c076a6b9a5a2 100644
@@ -262,7 +262,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
   				       fence_flags | AMDGPU_FENCE_FLAG_64BIT);
   	}
-	r = amdgpu_fence_emit(ring, f, fence_flags);
+	r = amdgpu_fence_emit(ring, f, job, fence_flags);
   	if (r) {
   		dev_err(adev->dev, "failed to emit fence (%d)\n", r);
   		if (job && job->vmid)
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +24 -11
index d33e6d97cc89..65a395060de2 100644
@@ -127,11 +127,16 @@ void amdgpu_job_free_resources(struct amdgpu_job *job)
   {
   	struct amdgpu_ring *ring = to_amdgpu_ring(job->base.sched);
   	struct dma_fence *f;
+	struct dma_fence *hw_fence;
   	unsigned i;
-	/* use sched fence if available */
-	f = job->base.s_fence ? &job->base.s_fence->finished : job->fence;
+	if (job->hw_fence.ops == NULL)
+		hw_fence = job->external_hw_fence;
+	else
+		hw_fence = &job->hw_fence;
+	/* use sched fence if available */
+	f = job->base.s_fence ? &job->base.s_fence->finished : hw_fence;
   	for (i = 0; i < job->num_ibs; ++i)
   		amdgpu_ib_free(ring->adev, &job->ibs[i], f);
   }
@@ -142,20 +147,27 @@ static void amdgpu_job_free_cb(struct drm_sched_job *s_job)
   	drm_sched_job_cleanup(s_job);
-	dma_fence_put(job->fence);
   	amdgpu_sync_free(&job->sync);
   	amdgpu_sync_free(&job->sched_sync);
-	kfree(job);
+
+    /* only put the hw fence if has embedded fence */
+	if (job->hw_fence.ops != NULL)
+		dma_fence_put(&job->hw_fence);
+	else
+		kfree(job);
   }
   void amdgpu_job_free(struct amdgpu_job *job)
   {
   	amdgpu_job_free_resources(job);
-
-	dma_fence_put(job->fence);
   	amdgpu_sync_free(&job->sync);
   	amdgpu_sync_free(&job->sched_sync);
-	kfree(job);
+
+	/* only put the hw fence if has embedded fence */
+	if (job->hw_fence.ops != NULL)
+		dma_fence_put(&job->hw_fence);
+	else
+		kfree(job);
   }
   int amdgpu_job_submit(struct amdgpu_job *job, struct drm_sched_entity *entity,
@@ -184,11 +196,14 @@ int amdgpu_job_submit_direct(struct amdgpu_job *job, struct amdgpu_ring *ring,
   	job->base.sched = &ring->sched;
   	r = amdgpu_ib_schedule(ring, job->num_ibs, job->ibs, NULL, fence);
-	job->fence = dma_fence_get(*fence);
+	/* record external_hw_fence for direct submit */
+	job->external_hw_fence = dma_fence_get(*fence);
   	if (r)
   		return r;
   	amdgpu_job_free(job);
+	dma_fence_put(*fence);
+
   	return 0;
   }
@@ -246,10 +261,8 @@ static struct dma_fence *amdgpu_job_run(struct drm_sched_job *sched_job)
   		if (r)
   			DRM_ERROR("Error scheduling IBs (%d)\n", r);
   	}
-	/* if gpu reset, hw fence will be replaced here */
-	dma_fence_put(job->fence);
-	job->fence = dma_fence_get(fence);
+	dma_fence_get(fence);
   	amdgpu_job_free_resources(job);
   	fence = r ? ERR_PTR(r) : fence;
drivers/gpu/drm/amd/amdgpu/amdgpu_job.h +0 -0
index 81caac9b958a..92324c978534 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_job.h struct amdgpu_job { · line 50

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_job.h +2 -1
@@ -46,7 +46,9 @@ struct amdgpu_job {
   	struct amdgpu_sync	sync;
   	struct amdgpu_sync	sched_sync;
   	struct amdgpu_ib	*ibs;
-	struct dma_fence	*fence; /* the hw fence */
+	struct dma_fence	hw_fence;
+	struct amdgpu_ring *ring;

Why not instead of 2 fields above just embed   struct amdgpu_fence as hw_fence  and by this save the extra 'ring' field handling ?

Andrey

Quoted 37 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_job.h +1 -0
+	struct dma_fence	*external_hw_fence;
   	uint32_t		preamble_status;
   	uint32_t                preemption_status;
   	uint32_t		num_ibs;
drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +4 -1
index 9c11ced4312c..03d4b29a76d6 100644
@@ -48,6 +48,9 @@
   #define AMDGPU_FENCE_FLAG_INT           (1 << 1)
   #define AMDGPU_FENCE_FLAG_TC_WB_ONLY    (1 << 2)
+/* fence flag bit to indicate the face is embeded in job*/
+#define AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT		(DMA_FENCE_FLAG_USER_BITS + 1)
+
   #define to_amdgpu_ring(s) container_of((s), struct amdgpu_ring, sched)
   #define AMDGPU_IB_POOL_SIZE	(1024 * 1024)
@@ -118,7 +121,7 @@ void amdgpu_fence_driver_hw_init(struct amdgpu_device *adev);
   void amdgpu_fence_driver_hw_fini(struct amdgpu_device *adev);
   int amdgpu_fence_driver_sw_init(struct amdgpu_device *adev);
   void amdgpu_fence_driver_sw_fini(struct amdgpu_device *adev);
-int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **fence,
+int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **fence, struct amdgpu_job *job,
   		      unsigned flags);
   int amdgpu_fence_emit_polling(struct amdgpu_ring *ring, uint32_t *s,
   			      uint32_t timeout);
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1 -1
index 2a88ed5d983b..2af8860d74cc 100644
@@ -1218,7 +1218,7 @@ int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job,
   		amdgpu_gmc_emit_pasid_mapping(ring, job->vmid, job->pasid);
   	if (vm_flush_needed || pasid_mapping_needed) {
-		r = amdgpu_fence_emit(ring, &fence, 0);
+		r = amdgpu_fence_emit(ring, &fence, NULL, 0);
   		if (r)
   			return r;
   	}
#8 lore raw
Quoted 108 lines · Christian König

Am 06.08.21 um 07:52 schrieb Jingwen Chen:

Quoted 99 lines · Andrey Grodzovsky
Quoted 89 lines · Jingwen Chen
Quoted 88 lines

From: Jack Zhang <[email protected]>

Why: Previously hw fence is alloced separately with job. It caused historical lifetime issues and corner cases. The ideal situation is to take fence to manage both job and fence's lifetime, and simplify the design of gpu-scheduler.

How:
We propose to embed hw_fence into amdgpu_job.
1. We cover the normal job submission by this method.
2. For ib_test, and submit without a parent job keep the legacy way to create a hw fence separately.
v2:
use AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT to show that the fence is embeded in a job.

---
   drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c  |  1 -
   drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c |  2 +-
   drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c   | 63 ++++++++++++++++-----
   drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c      |  2 +-
   drivers/gpu/drm/amd/amdgpu/amdgpu_job.c     | 35 ++++++++----
   drivers/gpu/drm/amd/amdgpu/amdgpu_job.h     |  4 +-
   drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h    |  5 +-
   drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c      |  2 +-
   8 files changed, 84 insertions(+), 30 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +0 -1
index 7b46ba551cb2..3003ee1c9487 100644
@@ -714,7 +714,6 @@ int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine,
   	ret = dma_fence_wait(f, false);
   err_ib_sched:
-	dma_fence_put(f);
   	amdgpu_job_free(job);
   err:
   	return ret;
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +1 -1
index 536005bff24a..277128846dd1 100644
@@ -1414,7 +1414,7 @@ static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring)
   			continue;
   		}
   		job = to_amdgpu_job(s_job);
-		if (preempted && job->fence == fence)
+		if (preempted && (&job->hw_fence) == fence)
   			/* mark the job as preempted */
   			job->preemption_status |= AMDGPU_IB_PREEMPTED;
   	}
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +11 -5
index 7495911516c2..5e29d797a265 100644
@@ -129,30 +129,46 @@ static u32 amdgpu_fence_read(struct amdgpu_ring *ring)
    *
    * @ring: ring the fence is associated with
    * @f: resulting fence object
+ * @job: job the fence is embeded in
    * @flags: flags to pass into the subordinate .emit_fence() call
    *
    * Emits a fence command on the requested ring (all asics).
    * Returns 0 on success, -ENOMEM on failure.
    */
-int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f,
+int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f, struct amdgpu_job *job,
   		      unsigned flags)
   {
   	struct amdgpu_device *adev = ring->adev;
-	struct amdgpu_fence *fence;
+	struct dma_fence *fence;
+	struct amdgpu_fence *am_fence;
   	struct dma_fence __rcu **ptr;
   	uint32_t seq;
   	int r;
-	fence = kmem_cache_alloc(amdgpu_fence_slab, GFP_KERNEL);
-	if (fence == NULL)
-		return -ENOMEM;
+	if (job == NULL) {
+		/* create a sperate hw fence */
+		am_fence = kmem_cache_alloc(amdgpu_fence_slab, GFP_ATOMIC);
+		if (am_fence == NULL)
+			return -ENOMEM;
+		fence = &am_fence->base;
+		am_fence->ring = ring;
Quoted 9 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static u32 amdgpu_fence_read(struct amdgpu_ring *ring) · line 158

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +4 -0
+	} else {
+		/* take use of job-embedded fence */
+		fence = &job->hw_fence;
+		job->ring = ring;

If you would make hw_fence of type amdgpu_fence you could probably avoid the special job->ring = ring See more in related comment at the bottom

Quoted 5 lines

Hi Andry,

I'm only make the amdgpu_fence for the fence without job parameter provided to amdgpu_fence_emit. For embeded fence which is the hw_fence in amdgpu_job, it will be allocated along with amdgpu_job as dma_fence.

Quoted 7 lines

When you have the job and need the ring you can just do conatiner_of(job->sched, struct amdgpu_ring, sched).

No need for an extra ring pointer here.

Regards,
Christian.

I see, I will change this.

Best Regards,
JingWen Chen

Quoted 221 lines
Quoted 220 lines

Regards,
Jingwen Chen

Quoted 216 lines
Quoted 31 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +10 -5
+	}
   	seq = ++ring->fence_drv.sync_seq;
-	fence->ring = ring;
-	dma_fence_init(&fence->base, &amdgpu_fence_ops,
+	dma_fence_init(fence, &amdgpu_fence_ops,
   		       &ring->fence_drv.lock,
   		       adev->fence_context + ring->idx,
   		       seq);
+
+	if (job != NULL) {
+		/* mark this fence has a parent job */
+		set_bit(AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT, &fence->flags);
+	}
+
   	amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,
   			       seq, flags | AMDGPU_FENCE_FLAG_INT);
   	pm_runtime_get_noresume(adev_to_drm(adev)->dev);
@@ -175,9 +191,9 @@ int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f,
   	/* This function can't be called concurrently anyway, otherwise
   	 * emitting the fence would mess up the hardware ring buffer.
   	 */
-	rcu_assign_pointer(*ptr, dma_fence_get(&fence->base));
+	rcu_assign_pointer(*ptr, dma_fence_get(fence));
-	*f = &fence->base;
+	*f = fence;
   	return 0;
   }
@@ -621,8 +637,16 @@ static const char *amdgpu_fence_get_driver_name(struct dma_fence *fence)
   static const char *amdgpu_fence_get_timeline_name(struct dma_fence *f)
   {
-	struct amdgpu_fence *fence = to_amdgpu_fence(f);

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static const char *amdgpu_fence_get_driver_name(struct dma_fence *fence) · line 647

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +9 -1
-	return (const char *)fence->ring->name;
+	struct amdgpu_ring *ring;
+
+	if (test_bit(AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT, &f->flags)) {
+		struct amdgpu_job *job = container_of(f, struct amdgpu_job, hw_fence);
+
+		ring = job->ring;
+	} else {
+		ring = to_amdgpu_fence(f)->ring;
+	}

Same as above

Quoted 122 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +16 -2
+	return (const char *)ring->name;
   }
   /**
@@ -656,8 +680,20 @@ static bool amdgpu_fence_enable_signaling(struct dma_fence *f)
   static void amdgpu_fence_free(struct rcu_head *rcu)
   {
   	struct dma_fence *f = container_of(rcu, struct dma_fence, rcu);
-	struct amdgpu_fence *fence = to_amdgpu_fence(f);
-	kmem_cache_free(amdgpu_fence_slab, fence);
+
+	if (test_bit(AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT, &f->flags)) {
+	/* free job if fence has a parent job */
+		struct amdgpu_job *job;
+
+		job = container_of(f, struct amdgpu_job, hw_fence);
+		kfree(job);
+	} else {
+	/* free fence_slab if it's separated fence*/
+		struct amdgpu_fence *fence;
+
+		fence = to_amdgpu_fence(f);
+		kmem_cache_free(amdgpu_fence_slab, fence);
+	}
   }
   /**
@@ -680,6 +716,7 @@ static const struct dma_fence_ops amdgpu_fence_ops = {
   	.release = amdgpu_fence_release,
   };
+
   /*
    * Fence debugfs
    */
drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c +1 -1
index ec65ab0ddf89..c076a6b9a5a2 100644
@@ -262,7 +262,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
   				       fence_flags | AMDGPU_FENCE_FLAG_64BIT);
   	}
-	r = amdgpu_fence_emit(ring, f, fence_flags);
+	r = amdgpu_fence_emit(ring, f, job, fence_flags);
   	if (r) {
   		dev_err(adev->dev, "failed to emit fence (%d)\n", r);
   		if (job && job->vmid)
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +24 -11
index d33e6d97cc89..65a395060de2 100644
@@ -127,11 +127,16 @@ void amdgpu_job_free_resources(struct amdgpu_job *job)
   {
   	struct amdgpu_ring *ring = to_amdgpu_ring(job->base.sched);
   	struct dma_fence *f;
+	struct dma_fence *hw_fence;
   	unsigned i;
-	/* use sched fence if available */
-	f = job->base.s_fence ? &job->base.s_fence->finished : job->fence;
+	if (job->hw_fence.ops == NULL)
+		hw_fence = job->external_hw_fence;
+	else
+		hw_fence = &job->hw_fence;
+	/* use sched fence if available */
+	f = job->base.s_fence ? &job->base.s_fence->finished : hw_fence;
   	for (i = 0; i < job->num_ibs; ++i)
   		amdgpu_ib_free(ring->adev, &job->ibs[i], f);
   }
@@ -142,20 +147,27 @@ static void amdgpu_job_free_cb(struct drm_sched_job *s_job)
   	drm_sched_job_cleanup(s_job);
-	dma_fence_put(job->fence);
   	amdgpu_sync_free(&job->sync);
   	amdgpu_sync_free(&job->sched_sync);
-	kfree(job);
+
+    /* only put the hw fence if has embedded fence */
+	if (job->hw_fence.ops != NULL)
+		dma_fence_put(&job->hw_fence);
+	else
+		kfree(job);
   }
   void amdgpu_job_free(struct amdgpu_job *job)
   {
   	amdgpu_job_free_resources(job);
-
-	dma_fence_put(job->fence);
   	amdgpu_sync_free(&job->sync);
   	amdgpu_sync_free(&job->sched_sync);
-	kfree(job);
+
+	/* only put the hw fence if has embedded fence */
+	if (job->hw_fence.ops != NULL)
+		dma_fence_put(&job->hw_fence);
+	else
+		kfree(job);
   }
   int amdgpu_job_submit(struct amdgpu_job *job, struct drm_sched_entity *entity,
@@ -184,11 +196,14 @@ int amdgpu_job_submit_direct(struct amdgpu_job *job, struct amdgpu_ring *ring,
   	job->base.sched = &ring->sched;
   	r = amdgpu_ib_schedule(ring, job->num_ibs, job->ibs, NULL, fence);
-	job->fence = dma_fence_get(*fence);
+	/* record external_hw_fence for direct submit */
+	job->external_hw_fence = dma_fence_get(*fence);
   	if (r)
   		return r;
   	amdgpu_job_free(job);
+	dma_fence_put(*fence);
+
   	return 0;
   }
@@ -246,10 +261,8 @@ static struct dma_fence *amdgpu_job_run(struct drm_sched_job *sched_job)
   		if (r)
   			DRM_ERROR("Error scheduling IBs (%d)\n", r);
   	}
-	/* if gpu reset, hw fence will be replaced here */
-	dma_fence_put(job->fence);
-	job->fence = dma_fence_get(fence);
+	dma_fence_get(fence);
   	amdgpu_job_free_resources(job);
   	fence = r ? ERR_PTR(r) : fence;
drivers/gpu/drm/amd/amdgpu/amdgpu_job.h +0 -0
index 81caac9b958a..92324c978534 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_job.h struct amdgpu_job { · line 50

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_job.h +2 -1
@@ -46,7 +46,9 @@ struct amdgpu_job {
   	struct amdgpu_sync	sync;
   	struct amdgpu_sync	sched_sync;
   	struct amdgpu_ib	*ibs;
-	struct dma_fence	*fence; /* the hw fence */
+	struct dma_fence	hw_fence;
+	struct amdgpu_ring *ring;

Why not instead of 2 fields above just embed   struct amdgpu_fence as hw_fence  and by this save the extra 'ring' field handling ?

Andrey

Quoted 37 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_job.h +1 -0
+	struct dma_fence	*external_hw_fence;
   	uint32_t		preamble_status;
   	uint32_t                preemption_status;
   	uint32_t		num_ibs;
drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +4 -1
index 9c11ced4312c..03d4b29a76d6 100644
@@ -48,6 +48,9 @@
   #define AMDGPU_FENCE_FLAG_INT           (1 << 1)
   #define AMDGPU_FENCE_FLAG_TC_WB_ONLY    (1 << 2)
+/* fence flag bit to indicate the face is embeded in job*/
+#define AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT		(DMA_FENCE_FLAG_USER_BITS + 1)
+
   #define to_amdgpu_ring(s) container_of((s), struct amdgpu_ring, sched)
   #define AMDGPU_IB_POOL_SIZE	(1024 * 1024)
@@ -118,7 +121,7 @@ void amdgpu_fence_driver_hw_init(struct amdgpu_device *adev);
   void amdgpu_fence_driver_hw_fini(struct amdgpu_device *adev);
   int amdgpu_fence_driver_sw_init(struct amdgpu_device *adev);
   void amdgpu_fence_driver_sw_fini(struct amdgpu_device *adev);
-int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **fence,
+int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **fence, struct amdgpu_job *job,
   		      unsigned flags);
   int amdgpu_fence_emit_polling(struct amdgpu_ring *ring, uint32_t *s,
   			      uint32_t timeout);
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1 -1
index 2a88ed5d983b..2af8860d74cc 100644
@@ -1218,7 +1218,7 @@ int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job,
   		amdgpu_gmc_emit_pasid_mapping(ring, job->vmid, job->pasid);
   	if (vm_flush_needed || pasid_mapping_needed) {
-		r = amdgpu_fence_emit(ring, &fence, 0);
+		r = amdgpu_fence_emit(ring, &fence, NULL, 0);
   		if (r)
   			return r;
   	}
#9 lore raw
Quoted 112 lines · Jingwen Chen
Quoted 111 lines · Christian König

Am 06.08.21 um 07:52 schrieb Jingwen Chen:

Quoted 99 lines · Andrey Grodzovsky
Quoted 89 lines · Jingwen Chen
Quoted 88 lines

From: Jack Zhang <[email protected]>

Why: Previously hw fence is alloced separately with job. It caused historical lifetime issues and corner cases. The ideal situation is to take fence to manage both job and fence's lifetime, and simplify the design of gpu-scheduler.

How:
We propose to embed hw_fence into amdgpu_job.
1. We cover the normal job submission by this method.
2. For ib_test, and submit without a parent job keep the legacy way to create a hw fence separately.
v2:
use AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT to show that the fence is embeded in a job.

---
   drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c  |  1 -
   drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c |  2 +-
   drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c   | 63 ++++++++++++++++-----
   drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c      |  2 +-
   drivers/gpu/drm/amd/amdgpu/amdgpu_job.c     | 35 ++++++++----
   drivers/gpu/drm/amd/amdgpu/amdgpu_job.h     |  4 +-
   drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h    |  5 +-
   drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c      |  2 +-
   8 files changed, 84 insertions(+), 30 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +0 -1
index 7b46ba551cb2..3003ee1c9487 100644
@@ -714,7 +714,6 @@ int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine,
   	ret = dma_fence_wait(f, false);
   err_ib_sched:
-	dma_fence_put(f);
   	amdgpu_job_free(job);
   err:
   	return ret;
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +1 -1
index 536005bff24a..277128846dd1 100644
@@ -1414,7 +1414,7 @@ static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring)
   			continue;
   		}
   		job = to_amdgpu_job(s_job);
-		if (preempted && job->fence == fence)
+		if (preempted && (&job->hw_fence) == fence)
   			/* mark the job as preempted */
   			job->preemption_status |= AMDGPU_IB_PREEMPTED;
   	}
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +11 -5
index 7495911516c2..5e29d797a265 100644
@@ -129,30 +129,46 @@ static u32 amdgpu_fence_read(struct amdgpu_ring *ring)
    *
    * @ring: ring the fence is associated with
    * @f: resulting fence object
+ * @job: job the fence is embeded in
    * @flags: flags to pass into the subordinate .emit_fence() call
    *
    * Emits a fence command on the requested ring (all asics).
    * Returns 0 on success, -ENOMEM on failure.
    */
-int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f,
+int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f, struct amdgpu_job *job,
   		      unsigned flags)
   {
   	struct amdgpu_device *adev = ring->adev;
-	struct amdgpu_fence *fence;
+	struct dma_fence *fence;
+	struct amdgpu_fence *am_fence;
   	struct dma_fence __rcu **ptr;
   	uint32_t seq;
   	int r;
-	fence = kmem_cache_alloc(amdgpu_fence_slab, GFP_KERNEL);
-	if (fence == NULL)
-		return -ENOMEM;
+	if (job == NULL) {
+		/* create a sperate hw fence */
+		am_fence = kmem_cache_alloc(amdgpu_fence_slab, GFP_ATOMIC);
+		if (am_fence == NULL)
+			return -ENOMEM;
+		fence = &am_fence->base;
+		am_fence->ring = ring;
Quoted 9 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static u32 amdgpu_fence_read(struct amdgpu_ring *ring) · line 158

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +4 -0
+	} else {
+		/* take use of job-embedded fence */
+		fence = &job->hw_fence;
+		job->ring = ring;

If you would make hw_fence of type amdgpu_fence you could probably avoid the special job->ring = ring See more in related comment at the bottom

Quoted 5 lines

Hi Andry,

I'm only make the amdgpu_fence for the fence without job parameter provided to amdgpu_fence_emit. For embeded fence which is the hw_fence in amdgpu_job, it will be allocated along with amdgpu_job as dma_fence.

When you have the job and need the ring you can just do conatiner_of(job->sched, struct amdgpu_ring, sched).

Quoted 8 lines
Quoted 4 lines

No need for an extra ring pointer here.

Regards,
Christian.

I see, I will change this.

Best Regards,
JingWen Chen

Hi Christian & Andrey,

A version 3 of this patch has been sent, and the other tdr related patch doesn't need change, so version 2 of tdr is OK to review.

Best Regards,
JingWen Chen

Quoted 221 lines
Quoted 221 lines
Quoted 220 lines

Regards,
Jingwen Chen

Quoted 216 lines
Quoted 31 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +10 -5
+	}
   	seq = ++ring->fence_drv.sync_seq;
-	fence->ring = ring;
-	dma_fence_init(&fence->base, &amdgpu_fence_ops,
+	dma_fence_init(fence, &amdgpu_fence_ops,
   		       &ring->fence_drv.lock,
   		       adev->fence_context + ring->idx,
   		       seq);
+
+	if (job != NULL) {
+		/* mark this fence has a parent job */
+		set_bit(AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT, &fence->flags);
+	}
+
   	amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,
   			       seq, flags | AMDGPU_FENCE_FLAG_INT);
   	pm_runtime_get_noresume(adev_to_drm(adev)->dev);
@@ -175,9 +191,9 @@ int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f,
   	/* This function can't be called concurrently anyway, otherwise
   	 * emitting the fence would mess up the hardware ring buffer.
   	 */
-	rcu_assign_pointer(*ptr, dma_fence_get(&fence->base));
+	rcu_assign_pointer(*ptr, dma_fence_get(fence));
-	*f = &fence->base;
+	*f = fence;
   	return 0;
   }
@@ -621,8 +637,16 @@ static const char *amdgpu_fence_get_driver_name(struct dma_fence *fence)
   static const char *amdgpu_fence_get_timeline_name(struct dma_fence *f)
   {
-	struct amdgpu_fence *fence = to_amdgpu_fence(f);

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static const char *amdgpu_fence_get_driver_name(struct dma_fence *fence) · line 647

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +9 -1
-	return (const char *)fence->ring->name;
+	struct amdgpu_ring *ring;
+
+	if (test_bit(AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT, &f->flags)) {
+		struct amdgpu_job *job = container_of(f, struct amdgpu_job, hw_fence);
+
+		ring = job->ring;
+	} else {
+		ring = to_amdgpu_fence(f)->ring;
+	}

Same as above

Quoted 122 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +16 -2
+	return (const char *)ring->name;
   }
   /**
@@ -656,8 +680,20 @@ static bool amdgpu_fence_enable_signaling(struct dma_fence *f)
   static void amdgpu_fence_free(struct rcu_head *rcu)
   {
   	struct dma_fence *f = container_of(rcu, struct dma_fence, rcu);
-	struct amdgpu_fence *fence = to_amdgpu_fence(f);
-	kmem_cache_free(amdgpu_fence_slab, fence);
+
+	if (test_bit(AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT, &f->flags)) {
+	/* free job if fence has a parent job */
+		struct amdgpu_job *job;
+
+		job = container_of(f, struct amdgpu_job, hw_fence);
+		kfree(job);
+	} else {
+	/* free fence_slab if it's separated fence*/
+		struct amdgpu_fence *fence;
+
+		fence = to_amdgpu_fence(f);
+		kmem_cache_free(amdgpu_fence_slab, fence);
+	}
   }
   /**
@@ -680,6 +716,7 @@ static const struct dma_fence_ops amdgpu_fence_ops = {
   	.release = amdgpu_fence_release,
   };
+
   /*
    * Fence debugfs
    */
drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c +1 -1
index ec65ab0ddf89..c076a6b9a5a2 100644
@@ -262,7 +262,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
   				       fence_flags | AMDGPU_FENCE_FLAG_64BIT);
   	}
-	r = amdgpu_fence_emit(ring, f, fence_flags);
+	r = amdgpu_fence_emit(ring, f, job, fence_flags);
   	if (r) {
   		dev_err(adev->dev, "failed to emit fence (%d)\n", r);
   		if (job && job->vmid)
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +24 -11
index d33e6d97cc89..65a395060de2 100644
@@ -127,11 +127,16 @@ void amdgpu_job_free_resources(struct amdgpu_job *job)
   {
   	struct amdgpu_ring *ring = to_amdgpu_ring(job->base.sched);
   	struct dma_fence *f;
+	struct dma_fence *hw_fence;
   	unsigned i;
-	/* use sched fence if available */
-	f = job->base.s_fence ? &job->base.s_fence->finished : job->fence;
+	if (job->hw_fence.ops == NULL)
+		hw_fence = job->external_hw_fence;
+	else
+		hw_fence = &job->hw_fence;
+	/* use sched fence if available */
+	f = job->base.s_fence ? &job->base.s_fence->finished : hw_fence;
   	for (i = 0; i < job->num_ibs; ++i)
   		amdgpu_ib_free(ring->adev, &job->ibs[i], f);
   }
@@ -142,20 +147,27 @@ static void amdgpu_job_free_cb(struct drm_sched_job *s_job)
   	drm_sched_job_cleanup(s_job);
-	dma_fence_put(job->fence);
   	amdgpu_sync_free(&job->sync);
   	amdgpu_sync_free(&job->sched_sync);
-	kfree(job);
+
+    /* only put the hw fence if has embedded fence */
+	if (job->hw_fence.ops != NULL)
+		dma_fence_put(&job->hw_fence);
+	else
+		kfree(job);
   }
   void amdgpu_job_free(struct amdgpu_job *job)
   {
   	amdgpu_job_free_resources(job);
-
-	dma_fence_put(job->fence);
   	amdgpu_sync_free(&job->sync);
   	amdgpu_sync_free(&job->sched_sync);
-	kfree(job);
+
+	/* only put the hw fence if has embedded fence */
+	if (job->hw_fence.ops != NULL)
+		dma_fence_put(&job->hw_fence);
+	else
+		kfree(job);
   }
   int amdgpu_job_submit(struct amdgpu_job *job, struct drm_sched_entity *entity,
@@ -184,11 +196,14 @@ int amdgpu_job_submit_direct(struct amdgpu_job *job, struct amdgpu_ring *ring,
   	job->base.sched = &ring->sched;
   	r = amdgpu_ib_schedule(ring, job->num_ibs, job->ibs, NULL, fence);
-	job->fence = dma_fence_get(*fence);
+	/* record external_hw_fence for direct submit */
+	job->external_hw_fence = dma_fence_get(*fence);
   	if (r)
   		return r;
   	amdgpu_job_free(job);
+	dma_fence_put(*fence);
+
   	return 0;
   }
@@ -246,10 +261,8 @@ static struct dma_fence *amdgpu_job_run(struct drm_sched_job *sched_job)
   		if (r)
   			DRM_ERROR("Error scheduling IBs (%d)\n", r);
   	}
-	/* if gpu reset, hw fence will be replaced here */
-	dma_fence_put(job->fence);
-	job->fence = dma_fence_get(fence);
+	dma_fence_get(fence);
   	amdgpu_job_free_resources(job);
   	fence = r ? ERR_PTR(r) : fence;
drivers/gpu/drm/amd/amdgpu/amdgpu_job.h +0 -0
index 81caac9b958a..92324c978534 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_job.h struct amdgpu_job { · line 50

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_job.h +2 -1
@@ -46,7 +46,9 @@ struct amdgpu_job {
   	struct amdgpu_sync	sync;
   	struct amdgpu_sync	sched_sync;
   	struct amdgpu_ib	*ibs;
-	struct dma_fence	*fence; /* the hw fence */
+	struct dma_fence	hw_fence;
+	struct amdgpu_ring *ring;

Why not instead of 2 fields above just embed   struct amdgpu_fence as hw_fence  and by this save the extra 'ring' field handling ?

Andrey

Quoted 37 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_job.h +1 -0
+	struct dma_fence	*external_hw_fence;
   	uint32_t		preamble_status;
   	uint32_t                preemption_status;
   	uint32_t		num_ibs;
drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +4 -1
index 9c11ced4312c..03d4b29a76d6 100644
@@ -48,6 +48,9 @@
   #define AMDGPU_FENCE_FLAG_INT           (1 << 1)
   #define AMDGPU_FENCE_FLAG_TC_WB_ONLY    (1 << 2)
+/* fence flag bit to indicate the face is embeded in job*/
+#define AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT		(DMA_FENCE_FLAG_USER_BITS + 1)
+
   #define to_amdgpu_ring(s) container_of((s), struct amdgpu_ring, sched)
   #define AMDGPU_IB_POOL_SIZE	(1024 * 1024)
@@ -118,7 +121,7 @@ void amdgpu_fence_driver_hw_init(struct amdgpu_device *adev);
   void amdgpu_fence_driver_hw_fini(struct amdgpu_device *adev);
   int amdgpu_fence_driver_sw_init(struct amdgpu_device *adev);
   void amdgpu_fence_driver_sw_fini(struct amdgpu_device *adev);
-int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **fence,
+int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **fence, struct amdgpu_job *job,
   		      unsigned flags);
   int amdgpu_fence_emit_polling(struct amdgpu_ring *ring, uint32_t *s,
   			      uint32_t timeout);
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1 -1
index 2a88ed5d983b..2af8860d74cc 100644
@@ -1218,7 +1218,7 @@ int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job,
   		amdgpu_gmc_emit_pasid_mapping(ring, job->vmid, job->pasid);
   	if (vm_flush_needed || pasid_mapping_needed) {
-		r = amdgpu_fence_emit(ring, &fence, 0);
+		r = amdgpu_fence_emit(ring, &fence, NULL, 0);
   		if (r)
   			return r;
   	}