Skip to contentSkip to conversation

drm/scheduler: add a list of run queues to the entity

PATCH In review Nayan Deshmukh 10 participants 15 messages Aug 1, 2018 – Aug 2, 2018 4 messages not in the archive

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

PATCH 1/4 drm/scheduler: add a list of run queues to the entity

These are the potential run queues on which the jobs from this entity can be scheduled. We will use this to do load balancing.

---
 drivers/gpu/drm/scheduler/gpu_scheduler.c | 8 ++++++++
 include/drm/gpu_scheduler.h               | 7 ++++++-
 2 files changed, 14 insertions(+), 1 deletion(-)
drivers/gpu/drm/scheduler/gpu_scheduler.c +8 -0
index 3f2fc5e8242a..a3eacc35cf98 100644
@@ -179,6 +179,8 @@ int drm_sched_entity_init(struct drm_sched_entity *entity,
 			  unsigned int num_rq_list,
 			  atomic_t *guilty)
 {
+	int i;
+
 	if (!(entity && rq_list && num_rq_list > 0 && rq_list[0]))
 		return -EINVAL;
 
@@ -186,6 +188,11 @@ int drm_sched_entity_init(struct drm_sched_entity *entity,
 	INIT_LIST_HEAD(&entity->list);
 	entity->rq = rq_list[0];
 	entity->guilty = guilty;
+	entity->num_rq_list = num_rq_list;
+	entity->rq_list = kcalloc(num_rq_list, sizeof(struct drm_sched_rq *),
+				GFP_KERNEL);
+	for (i = 0; i < num_rq_list; ++i)
+		entity->rq_list[i] = rq_list[i];
 	entity->last_scheduled = NULL;
 
 	spin_lock_init(&entity->rq_lock);
@@ -363,6 +370,7 @@ void drm_sched_entity_fini(struct drm_sched_entity *entity)
 
 	dma_fence_put(entity->last_scheduled);
 	entity->last_scheduled = NULL;
+	kfree(entity->rq_list);
 }
 EXPORT_SYMBOL(drm_sched_entity_fini);
 
include/drm/gpu_scheduler.h +6 -1
index 091b9afcd184..a60896222a3e 100644
@@ -50,7 +50,10 @@ enum drm_sched_priority {
  *
  * @list: used to append this struct to the list of entities in the
  *        runqueue.
- * @rq: runqueue to which this entity belongs.
+ * @rq: runqueue on which this entity is currently scheduled.
+ * @rq_list: a list of run queues on which jobs from this entity can
+ *           be scheduled
+ * @num_rq_list: number of run queues in the rq_list
  * @rq_lock: lock to modify the runqueue to which this entity belongs.
  * @job_queue: the list of jobs of this entity.
  * @fence_seq: a linearly increasing seqno incremented with each
@@ -74,6 +77,8 @@ enum drm_sched_priority {
 struct drm_sched_entity {
 	struct list_head		list;
 	struct drm_sched_rq		*rq;
+	struct drm_sched_rq		**rq_list;
+	unsigned int                    num_rq_list;
 	spinlock_t			rq_lock;
 
 	struct spsc_queue		job_queue;
Signature · 6 lines
2.14.3

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
#2 lore raw

PATCH 2/4 drm/scheduler: add counter for total jobs in scheduler

In reply to a message not in the archive
---
 drivers/gpu/drm/scheduler/gpu_scheduler.c | 3 +++
 include/drm/gpu_scheduler.h               | 2 ++
 2 files changed, 5 insertions(+)
drivers/gpu/drm/scheduler/gpu_scheduler.c +3 -0
index a3eacc35cf98..375f6f7f6a93 100644
@@ -549,6 +549,7 @@ void drm_sched_entity_push_job(struct drm_sched_job *sched_job,
 
 	trace_drm_sched_job(sched_job, entity);
 
+	atomic_inc(&entity->rq->sched->num_jobs);
 	first = spsc_queue_push(&entity->job_queue, &sched_job->queue_node);
 
 	/* first job wakes up scheduler */
@@ -836,6 +837,7 @@ static void drm_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb)
 
 	dma_fence_get(&s_fence->finished);
 	atomic_dec(&sched->hw_rq_count);
+	atomic_dec(&sched->num_jobs);
 	drm_sched_fence_finished(s_fence);
 
 	trace_drm_sched_process_job(s_fence);
@@ -953,6 +955,7 @@ int drm_sched_init(struct drm_gpu_scheduler *sched,
 	INIT_LIST_HEAD(&sched->ring_mirror_list);
 	spin_lock_init(&sched->job_list_lock);
 	atomic_set(&sched->hw_rq_count, 0);
+	atomic_set(&sched->num_jobs, 0);
 	atomic64_set(&sched->job_id_count, 0);
 
 	/* Each scheduler will run on a seperate kernel thread */
include/drm/gpu_scheduler.h +2 -0
index a60896222a3e..89881ce974a5 100644
@@ -260,6 +260,7 @@ struct drm_sched_backend_ops {
  * @job_list_lock: lock to protect the ring_mirror_list.
  * @hang_limit: once the hangs by a job crosses this limit then it is marked
  *              guilty and it will be considered for scheduling further.
+ * @num_jobs: the number of jobs in queue in the scheduler
  *
  * One scheduler is implemented for each hardware ring.
  */
@@ -277,6 +278,7 @@ struct drm_gpu_scheduler {
 	struct list_head		ring_mirror_list;
 	spinlock_t			job_list_lock;
 	int				hang_limit;
+	atomic_t                        num_jobs;
 };
 
 int drm_sched_init(struct drm_gpu_scheduler *sched,
Signature · 6 lines
2.14.3

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
#8 lore raw

PATCH 3/4 drm/scheduler: add new function to get least loaded sched v2

In reply to a message not in the archive

The function selects the run queue from the rq_list with the least load. The load is decided by the number of jobs in a scheduler.

v2: avoid using atomic read twice consecutively, instead store
it locally

---
 drivers/gpu/drm/scheduler/gpu_scheduler.c | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)
drivers/gpu/drm/scheduler/gpu_scheduler.c +25 -0
index 375f6f7f6a93..fb4e542660b0 100644
@@ -255,6 +255,31 @@ static bool drm_sched_entity_is_ready(struct drm_sched_entity *entity)
 	return true;
 }
 
+/**
+ * drm_sched_entity_get_free_sched - Get the rq from rq_list with least load
+ *
+ * @entity: scheduler entity
+ *
+ * Return the pointer to the rq with least load.
+ */
+static struct drm_sched_rq *
+drm_sched_entity_get_free_sched(struct drm_sched_entity *entity)
+{
+	struct drm_sched_rq *rq = NULL;
+	unsigned int min_jobs = UINT_MAX, num_jobs;
+	int i;
+
+	for (i = 0; i < entity->num_rq_list; ++i) {
+		num_jobs = atomic_read(&entity->rq_list[i]->sched->num_jobs);
+		if (num_jobs < min_jobs) {
+			min_jobs = num_jobs;
+			rq = entity->rq_list[i];
+		}
+	}
+
+	return rq;
+}
+
 static void drm_sched_entity_kill_jobs_cb(struct dma_fence *f,
 				    struct dma_fence_cb *cb)
 {
Signature · 6 lines
2.14.3

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
#18 lore raw

PATCH 4/4 drm/scheduler: move idle entities to scheduler with less load v2

In reply to Nayan Deshmukh

This is the first attempt to move entities between schedulers to have dynamic load balancing. We just move entities with no jobs for now as moving the ones with jobs will lead to other compilcations like ensuring that the other scheduler does not remove a job from the current entity while we are moving.

v2: remove unused variable and an unecessary check

---
 drivers/gpu/drm/scheduler/gpu_scheduler.c | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)
drivers/gpu/drm/scheduler/gpu_scheduler.c +18 -4
index fb4e542660b0..087fa479f7e0 100644
@@ -539,6 +539,8 @@ drm_sched_entity_pop_job(struct drm_sched_entity *entity)
 	if (!sched_job)
 		return NULL;
 
+	sched_job->sched = sched;
+	sched_job->s_fence->sched = sched;
 	while ((entity->dependency = sched->ops->dependency(sched_job, entity)))
 		if (drm_sched_entity_add_dependency_cb(entity))
 			return NULL;
@@ -569,11 +571,23 @@ drm_sched_entity_pop_job(struct drm_sched_entity *entity)
 void drm_sched_entity_push_job(struct drm_sched_job *sched_job,
 			       struct drm_sched_entity *entity)
 {
-	struct drm_gpu_scheduler *sched = sched_job->sched;
-	bool first = false;
+	struct drm_sched_rq *rq = entity->rq;
+	bool first = false, reschedule, idle;
 
-	trace_drm_sched_job(sched_job, entity);
+	idle = entity->last_scheduled == NULL ||
+		dma_fence_is_signaled(entity->last_scheduled);
+	first = spsc_queue_count(&entity->job_queue) == 0;
+	reschedule = idle && first && (entity->num_rq_list > 1);
 
+	if (reschedule) {
+		rq = drm_sched_entity_get_free_sched(entity);
+		spin_lock(&entity->rq_lock);
+		drm_sched_rq_remove_entity(entity->rq, entity);
+		entity->rq = rq;
+		spin_unlock(&entity->rq_lock);
+	}
+
+	trace_drm_sched_job(sched_job, entity);
 	atomic_inc(&entity->rq->sched->num_jobs);
 	first = spsc_queue_push(&entity->job_queue, &sched_job->queue_node);
 
@@ -588,7 +602,7 @@ void drm_sched_entity_push_job(struct drm_sched_job *sched_job,
 		}
 		drm_sched_rq_add_entity(entity->rq, entity);
 		spin_unlock(&entity->rq_lock);
-		drm_sched_wakeup(sched);
+		drm_sched_wakeup(entity->rq->sched);
 	}
 }
 EXPORT_SYMBOL(drm_sched_entity_push_job);
Signature · 6 lines
2.14.3

_______________________________________________
dri-devel mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/dri-devel
Reviewed-by
#17 lore raw
In reply to a message not in the archive

Am 01.08.2018 um 10:19 schrieb Nayan Deshmukh:

Quoted 4 lines

These are the potential run queues on which the jobs from this entity can be scheduled. We will use this to do load balancing.

I also just pushed them into our internal branch for upstreaming.

Thanks for all the work,
Christian.

Quoted 55 lines
---
  drivers/gpu/drm/scheduler/gpu_scheduler.c | 8 ++++++++
  include/drm/gpu_scheduler.h               | 7 ++++++-
  2 files changed, 14 insertions(+), 1 deletion(-)
drivers/gpu/drm/scheduler/gpu_scheduler.c +8 -0
index 3f2fc5e8242a..a3eacc35cf98 100644
@@ -179,6 +179,8 @@ int drm_sched_entity_init(struct drm_sched_entity *entity,
  			  unsigned int num_rq_list,
  			  atomic_t *guilty)
  {
+	int i;
+
  	if (!(entity && rq_list && num_rq_list > 0 && rq_list[0]))
  		return -EINVAL;
 
@@ -186,6 +188,11 @@ int drm_sched_entity_init(struct drm_sched_entity *entity,
  	INIT_LIST_HEAD(&entity->list);
  	entity->rq = rq_list[0];
  	entity->guilty = guilty;
+	entity->num_rq_list = num_rq_list;
+	entity->rq_list = kcalloc(num_rq_list, sizeof(struct drm_sched_rq *),
+				GFP_KERNEL);
+	for (i = 0; i < num_rq_list; ++i)
+		entity->rq_list[i] = rq_list[i];
  	entity->last_scheduled = NULL;
 
  	spin_lock_init(&entity->rq_lock);
@@ -363,6 +370,7 @@ void drm_sched_entity_fini(struct drm_sched_entity *entity)
 
  	dma_fence_put(entity->last_scheduled);
  	entity->last_scheduled = NULL;
+	kfree(entity->rq_list);
  }
  EXPORT_SYMBOL(drm_sched_entity_fini);
 
include/drm/gpu_scheduler.h +4 -1
index 091b9afcd184..a60896222a3e 100644
@@ -50,7 +50,10 @@ enum drm_sched_priority {
   *
   * @list: used to append this struct to the list of entities in the
   *        runqueue.
- * @rq: runqueue to which this entity belongs.
+ * @rq: runqueue on which this entity is currently scheduled.
+ * @rq_list: a list of run queues on which jobs from this entity can
+ *           be scheduled
+ * @num_rq_list: number of run queues in the rq_list
   * @rq_lock: lock to modify the runqueue to which this entity belongs.
   * @job_queue: the list of jobs of this entity.
   * @fence_seq: a linearly increasing seqno incremented with each

include/drm/gpu_scheduler.h enum drm_sched_priority { · line 84

Quoted 8 lines
include/drm/gpu_scheduler.h +2 -0
@@ -74,6 +77,8 @@ enum drm_sched_priority {
  struct drm_sched_entity {
  	struct list_head		list;
  	struct drm_sched_rq		*rq;
+	struct drm_sched_rq		**rq_list;
+	unsigned int                    num_rq_list;
  	spinlock_t			rq_lock;
 
  	struct spsc_queue		job_queue;

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#5 lore raw

Re: [PATCH 2/4] drm/scheduler: add counter for total jobs in scheduler

In reply to Huang Rui

Yeah, I've actually added one before pushing it to amd-staging-drm-next.

But thanks for the reminder, wanted to note that to Nayan as well :)

Christian.

Am 01.08.2018 um 15:15 schrieb Huang Rui:

Quoted 64 lines · Nayan Deshmukh

This should need a commmit message.

Thanks,
Ray

Quoted 57 lines
---
  drivers/gpu/drm/scheduler/gpu_scheduler.c | 3 +++
  include/drm/gpu_scheduler.h               | 2 ++
  2 files changed, 5 insertions(+)
drivers/gpu/drm/scheduler/gpu_scheduler.c +3 -0
index a3eacc35cf98..375f6f7f6a93 100644
@@ -549,6 +549,7 @@ void drm_sched_entity_push_job(struct drm_sched_job *sched_job,
 
  	trace_drm_sched_job(sched_job, entity);
 
+	atomic_inc(&entity->rq->sched->num_jobs);
  	first = spsc_queue_push(&entity->job_queue, &sched_job->queue_node);
 
  	/* first job wakes up scheduler */
@@ -836,6 +837,7 @@ static void drm_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb)
 
  	dma_fence_get(&s_fence->finished);
  	atomic_dec(&sched->hw_rq_count);
+	atomic_dec(&sched->num_jobs);
  	drm_sched_fence_finished(s_fence);
 
  	trace_drm_sched_process_job(s_fence);
@@ -953,6 +955,7 @@ int drm_sched_init(struct drm_gpu_scheduler *sched,
  	INIT_LIST_HEAD(&sched->ring_mirror_list);
  	spin_lock_init(&sched->job_list_lock);
  	atomic_set(&sched->hw_rq_count, 0);
+	atomic_set(&sched->num_jobs, 0);
  	atomic64_set(&sched->job_id_count, 0);
 
  	/* Each scheduler will run on a seperate kernel thread */
include/drm/gpu_scheduler.h +2 -0
index a60896222a3e..89881ce974a5 100644
@@ -260,6 +260,7 @@ struct drm_sched_backend_ops {
   * @job_list_lock: lock to protect the ring_mirror_list.
   * @hang_limit: once the hangs by a job crosses this limit then it is marked
   *              guilty and it will be considered for scheduling further.
+ * @num_jobs: the number of jobs in queue in the scheduler
   *
   * One scheduler is implemented for each hardware ring.
   */
@@ -277,6 +278,7 @@ struct drm_gpu_scheduler {
  	struct list_head		ring_mirror_list;
  	spinlock_t			job_list_lock;
  	int				hang_limit;
+	atomic_t                        num_jobs;
  };
 
  int drm_sched_init(struct drm_gpu_scheduler *sched,
Signature · 1 line
2.14.3
Quoted 8 lines
Quoted 4 lines

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
dri-devel mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/dri-devel

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#4 lore raw

Re: [PATCH 2/4] drm/scheduler: add counter for total jobs in scheduler

In reply to a message not in the archive

On Wed, Aug 01, 2018 at 01:50:00PM +0530, Nayan Deshmukh wrote:

This should need a commmit message.

Thanks,
Ray

Quoted 61 lines
---
 drivers/gpu/drm/scheduler/gpu_scheduler.c | 3 +++
 include/drm/gpu_scheduler.h               | 2 ++
 2 files changed, 5 insertions(+)
drivers/gpu/drm/scheduler/gpu_scheduler.c +3 -0
index a3eacc35cf98..375f6f7f6a93 100644
@@ -549,6 +549,7 @@ void drm_sched_entity_push_job(struct drm_sched_job *sched_job,
 
 	trace_drm_sched_job(sched_job, entity);
 
+	atomic_inc(&entity->rq->sched->num_jobs);
 	first = spsc_queue_push(&entity->job_queue, &sched_job->queue_node);
 
 	/* first job wakes up scheduler */
@@ -836,6 +837,7 @@ static void drm_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb)
 
 	dma_fence_get(&s_fence->finished);
 	atomic_dec(&sched->hw_rq_count);
+	atomic_dec(&sched->num_jobs);
 	drm_sched_fence_finished(s_fence);
 
 	trace_drm_sched_process_job(s_fence);
@@ -953,6 +955,7 @@ int drm_sched_init(struct drm_gpu_scheduler *sched,
 	INIT_LIST_HEAD(&sched->ring_mirror_list);
 	spin_lock_init(&sched->job_list_lock);
 	atomic_set(&sched->hw_rq_count, 0);
+	atomic_set(&sched->num_jobs, 0);
 	atomic64_set(&sched->job_id_count, 0);
 
 	/* Each scheduler will run on a seperate kernel thread */
include/drm/gpu_scheduler.h +2 -0
index a60896222a3e..89881ce974a5 100644
@@ -260,6 +260,7 @@ struct drm_sched_backend_ops {
  * @job_list_lock: lock to protect the ring_mirror_list.
  * @hang_limit: once the hangs by a job crosses this limit then it is marked
  *              guilty and it will be considered for scheduling further.
+ * @num_jobs: the number of jobs in queue in the scheduler
  *
  * One scheduler is implemented for each hardware ring.
  */
@@ -277,6 +278,7 @@ struct drm_gpu_scheduler {
 	struct list_head		ring_mirror_list;
 	spinlock_t			job_list_lock;
 	int				hang_limit;
+	atomic_t                        num_jobs;
 };
 
 int drm_sched_init(struct drm_gpu_scheduler *sched,
Signature · 6 lines
2.14.3

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#9 lore raw

Re: [PATCH 3/4] drm/scheduler: add new function to get least loaded sched v2

In reply to Nayan Deshmukh

Clarification question -  if the run queues belong to different schedulers they effectively point to different rings,

it means we allow to move (reschedule) a drm_sched_entity from one ring to another - i assume that the idea int the first place, that

you have a set of HW rings and you can utilize any of them for your jobs (like compute rings). Correct ?

Andrey

Quoted 38 lines · Nayan Deshmukh

The function selects the run queue from the rq_list with the least load. The load is decided by the number of jobs in a scheduler.

v2: avoid using atomic read twice consecutively, instead store
it locally

---
  drivers/gpu/drm/scheduler/gpu_scheduler.c | 25 +++++++++++++++++++++++++
  1 file changed, 25 insertions(+)
drivers/gpu/drm/scheduler/gpu_scheduler.c +18 -0
index 375f6f7f6a93..fb4e542660b0 100644
@@ -255,6 +255,31 @@ static bool drm_sched_entity_is_ready(struct drm_sched_entity *entity)
  	return true;
  }
 
+/**
+ * drm_sched_entity_get_free_sched - Get the rq from rq_list with least load
+ *
+ * @entity: scheduler entity
+ *
+ * Return the pointer to the rq with least load.
+ */
+static struct drm_sched_rq *
+drm_sched_entity_get_free_sched(struct drm_sched_entity *entity)
+{
+	struct drm_sched_rq *rq = NULL;
+	unsigned int min_jobs = UINT_MAX, num_jobs;
+	int i;
+
+	for (i = 0; i < entity->num_rq_list; ++i) {
+		num_jobs = atomic_read(&entity->rq_list[i]->sched->num_jobs);
+		if (num_jobs < min_jobs) {
+			min_jobs = num_jobs;

drivers/gpu/drm/scheduler/gpu_scheduler.c static bool drm_sched_entity_is_ready(struct drm_sched_entity *entity) · line 285

Quoted 10 lines
drivers/gpu/drm/scheduler/gpu_scheduler.c +7 -0
+			rq = entity->rq_list[i];
+		}
+	}
+
+	return rq;
+}
+
  static void drm_sched_entity_kill_jobs_cb(struct dma_fence *f,
  				    struct dma_fence_cb *cb)
  {

_______________________________________________
dri-devel mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/dri-devel

#10 lore raw

Re: [PATCH 3/4] drm/scheduler: add new function to get least loaded sched v2

Yes, that is correct.

Nayan

Quoted 54 lines · Andrey Grodzovsky

Clarification question - if the run queues belong to different schedulers they effectively point to different rings,

it means we allow to move (reschedule) a drm_sched_entity from one ring to another - i assume that the idea int the first place, that

you have a set of HW rings and you can utilize any of them for your jobs (like compute rings). Correct ?

Andrey

Quoted 10 lines · Nayan Deshmukh

The function selects the run queue from the rq_list with the least load. The load is decided by the number of jobs in a scheduler.

v2: avoid using atomic read twice consecutively, instead store
it locally

---
  drivers/gpu/drm/scheduler/gpu_scheduler.c | 25

+++++++++++++++++++++++++

Review

Quoted 3 lines
  1 file changed, 25 insertions(+)
(quoted diff) +0 -0

b/drivers/gpu/drm/scheduler/gpu_scheduler.c

drivers/gpu/drm/scheduler/gpu_scheduler.c static bool drm_sched_entity_is_ready(struct

Quoted 4 lines

index 375f6f7f6a93..fb4e542660b0 100644

drivers/gpu/drm/scheduler/gpu_scheduler.c +0 -0
@@ -255,6 +255,31 @@ static bool drm_sched_entity_is_ready(struct

drm_sched_entity *entity)

drivers/gpu/drm/scheduler/gpu_scheduler.c static bool drm_sched_entity_is_ready(struct drm_sched_entity *entity)

Quoted 5 lines
drivers/gpu/drm/scheduler/gpu_scheduler.c +2 -0
      return true;
  }
 
+/**
+ * drm_sched_entity_get_free_sched - Get the rq from rq_list with least

load

Quoted 4 lines
drivers/gpu/drm/scheduler/gpu_scheduler.c +4 -0
+ *
+ * @entity: scheduler entity
+ *
+ * Return the pointer to the rq with least load.

drivers/gpu/drm/scheduler/gpu_scheduler.c static bool drm_sched_entity_is_ready(struct drm_sched_entity *entity)

Quoted 10 lines
drivers/gpu/drm/scheduler/gpu_scheduler.c +10 -0
+ */
+static struct drm_sched_rq *
+drm_sched_entity_get_free_sched(struct drm_sched_entity *entity)
+{
+     struct drm_sched_rq *rq = NULL;
+     unsigned int min_jobs = UINT_MAX, num_jobs;
+     int i;
+
+     for (i = 0; i < entity->num_rq_list; ++i) {
+             num_jobs =

atomic_read(&entity->rq_list[i]->sched->num_jobs);

Quoted 14 lines

drivers/gpu/drm/scheduler/gpu_scheduler.c static bool drm_sched_entity_is_ready(struct drm_sched_entity *entity) · line 285

Quoted 12 lines
drivers/gpu/drm/scheduler/gpu_scheduler.c +9 -0
+             if (num_jobs < min_jobs) {
+                     min_jobs = num_jobs;
+                     rq = entity->rq_list[i];
+             }
+     }
+
+     return rq;
+}
+
  static void drm_sched_entity_kill_jobs_cb(struct dma_fence *f,
                                  struct dma_fence_cb *cb)
  {

_______________________________________________
dri-devel mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/dri-devel

#12 lore raw

Re: [PATCH 3/4] drm/scheduler: add new function to get least loaded sched v2

In reply to a message not in the archive

Series is Acked-by: Andrey Grodzovsky <[email protected]>

Andrey

Quoted 67 lines · Nayan Deshmukh

Yes, that is correct.

Nayan

On Wed, Aug 1, 2018, 9:05 PM Andrey Grodzovsky <[email protected] <mailto:[email protected]>> wrote:

    Clarification question -  if the run queues belong to different
    schedulers they effectively point to different rings,

    it means we allow to move (reschedule) a drm_sched_entity from one
    ring
    to another - i assume that the idea int the first place, that

    you have a set of HW rings and you can utilize any of them for
    your jobs
    (like compute rings). Correct ?

    Andrey


    On 08/01/2018 04:20 AM, Nayan Deshmukh wrote:
    > The function selects the run queue from the rq_list with the
    > least load. The load is decided by the number of jobs in a
    > scheduler.
    >
    > v2: avoid using atomic read twice consecutively, instead store
    >      it locally
    >
    > Signed-off-by: Nayan Deshmukh <[email protected]
    <mailto:[email protected]>>
    > ---
    >   drivers/gpu/drm/scheduler/gpu_scheduler.c | 25
    +++++++++++++++++++++++++
    >   1 file changed, 25 insertions(+)
    >
    > diff --git a/drivers/gpu/drm/scheduler/gpu_scheduler.c
    b/drivers/gpu/drm/scheduler/gpu_scheduler.c
    > index 375f6f7f6a93..fb4e542660b0 100644
    > --- a/drivers/gpu/drm/scheduler/gpu_scheduler.c
    > +++ b/drivers/gpu/drm/scheduler/gpu_scheduler.c
    > @@ -255,6 +255,31 @@ static bool
    drm_sched_entity_is_ready(struct drm_sched_entity *entity)
    >       return true;
    >   }
    >
    > +/**
    > + * drm_sched_entity_get_free_sched - Get the rq from rq_list
    with least load
    > + *
    > + * @entity: scheduler entity
    > + *
    > + * Return the pointer to the rq with least load.
    > + */
    > +static struct drm_sched_rq *
    > +drm_sched_entity_get_free_sched(struct drm_sched_entity *entity)
    > +{
    > +     struct drm_sched_rq *rq = NULL;
    > +     unsigned int min_jobs = UINT_MAX, num_jobs;
    > +     int i;
    > +
    > +     for (i = 0; i < entity->num_rq_list; ++i) {
    > +             num_jobs =
    atomic_read(&entity->rq_list[i]->sched->num_jobs);
    > +             if (num_jobs < min_jobs) {
    > +                     min_jobs = num_jobs;
    > +                     rq = entity->rq_list[i];
    > +             }
Quoted 9 lines
    > +     }
    > +
    > +     return rq;
    > +}
    > +
    >   static void drm_sched_entity_kill_jobs_cb(struct dma_fence *f,
    >                                   struct dma_fence_cb *cb)
    >   {

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#13 lore raw

RE: [PATCH 3/4] drm/scheduler: add new function to get least loaded sched v2

In reply to a message not in the archive

Another big question:
I agree the general idea is good to balance scheduler load for same ring family.
But, when same entity job run on different scheduler, that means the later job could be completed ahead of front, Right? That will break fence design, later fence must be signaled after front fence in same fence context.

Anything I missed?

Regards,
David Zhou

From: dri-devel <[email protected]> On Behalf Of Nayan Deshmukh
Sent: Thursday, August 02, 2018 12:07 AM
To: Grodzovsky, Andrey <[email protected]>
Cc: [email protected]; Maling list - DRI developers <[email protected]>; Koenig, Christian <[email protected]>
Subject: Re: [PATCH 3/4] drm/scheduler: add new function to get least loaded sched v2

Yes, that is correct.

Nayan

On Wed, Aug 1, 2018, 9:05 PM Andrey Grodzovsky <[email protected]<mailto:[email protected]>> wrote:
Clarification question - if the run queues belong to different
schedulers they effectively point to different rings,

it means we allow to move (reschedule) a drm_sched_entity from one ring to another - i assume that the idea int the first place, that

you have a set of HW rings and you can utilize any of them for your jobs (like compute rings). Correct ?

Andrey

Quoted 38 lines · Nayan Deshmukh

The function selects the run queue from the rq_list with the least load. The load is decided by the number of jobs in a scheduler.

v2: avoid using atomic read twice consecutively, instead store
it locally

---
  drivers/gpu/drm/scheduler/gpu_scheduler.c | 25 +++++++++++++++++++++++++
  1 file changed, 25 insertions(+)
drivers/gpu/drm/scheduler/gpu_scheduler.c +18 -0
index 375f6f7f6a93..fb4e542660b0 100644
@@ -255,6 +255,31 @@ static bool drm_sched_entity_is_ready(struct drm_sched_entity *entity)
      return true;
  }
 
+/**
+ * drm_sched_entity_get_free_sched - Get the rq from rq_list with least load
+ *
+ * @entity: scheduler entity
+ *
+ * Return the pointer to the rq with least load.
+ */
+static struct drm_sched_rq *
+drm_sched_entity_get_free_sched(struct drm_sched_entity *entity)
+{
+     struct drm_sched_rq *rq = NULL;
+     unsigned int min_jobs = UINT_MAX, num_jobs;
+     int i;
+
+     for (i = 0; i < entity->num_rq_list; ++i) {
+             num_jobs = atomic_read(&entity->rq_list[i]->sched->num_jobs);
+             if (num_jobs < min_jobs) {
+                     min_jobs = num_jobs;

drivers/gpu/drm/scheduler/gpu_scheduler.c static bool drm_sched_entity_is_ready(struct drm_sched_entity *entity) · line 285

Quoted 10 lines
drivers/gpu/drm/scheduler/gpu_scheduler.c +7 -0
+                     rq = entity->rq_list[i];
+             }
+     }
+
+     return rq;
+}
+
  static void drm_sched_entity_kill_jobs_cb(struct dma_fence *f,
                                  struct dma_fence_cb *cb)
  {

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#6 lore raw

Re: [PATCH 2/4] drm/scheduler: add counter for total jobs in scheduler

In reply to Christian König
Quoted 4 lines · Christian König

Yeah, I've actually added one before pushing it to amd-staging-drm-next.

But thanks for the reminder, wanted to note that to Nayan as well :)

Yes, a soft reminder to Nayan. Thanks Nayan for the contribution. :-)

Thanks,
Ray

Quoted 76 lines

Christian.

Am 01.08.2018 um 15:15 schrieb Huang Rui:

Quoted 72 lines · Nayan Deshmukh

This should need a commmit message.

Thanks,
Ray

Quoted 61 lines
---
  drivers/gpu/drm/scheduler/gpu_scheduler.c | 3 +++
  include/drm/gpu_scheduler.h               | 2 ++
  2 files changed, 5 insertions(+)
drivers/gpu/drm/scheduler/gpu_scheduler.c +3 -0
index a3eacc35cf98..375f6f7f6a93 100644
@@ -549,6 +549,7 @@ void drm_sched_entity_push_job(struct drm_sched_job *sched_job,
 
  	trace_drm_sched_job(sched_job, entity);
 
+	atomic_inc(&entity->rq->sched->num_jobs);
  	first = spsc_queue_push(&entity->job_queue, &sched_job->queue_node);
 
  	/* first job wakes up scheduler */
@@ -836,6 +837,7 @@ static void drm_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb)
 
  	dma_fence_get(&s_fence->finished);
  	atomic_dec(&sched->hw_rq_count);
+	atomic_dec(&sched->num_jobs);
  	drm_sched_fence_finished(s_fence);
 
  	trace_drm_sched_process_job(s_fence);
@@ -953,6 +955,7 @@ int drm_sched_init(struct drm_gpu_scheduler *sched,
  	INIT_LIST_HEAD(&sched->ring_mirror_list);
  	spin_lock_init(&sched->job_list_lock);
  	atomic_set(&sched->hw_rq_count, 0);
+	atomic_set(&sched->num_jobs, 0);
  	atomic64_set(&sched->job_id_count, 0);
 
  	/* Each scheduler will run on a seperate kernel thread */
include/drm/gpu_scheduler.h +2 -0
index a60896222a3e..89881ce974a5 100644
@@ -260,6 +260,7 @@ struct drm_sched_backend_ops {
   * @job_list_lock: lock to protect the ring_mirror_list.
   * @hang_limit: once the hangs by a job crosses this limit then it is marked
   *              guilty and it will be considered for scheduling further.
+ * @num_jobs: the number of jobs in queue in the scheduler
   *
   * One scheduler is implemented for each hardware ring.
   */
@@ -277,6 +278,7 @@ struct drm_gpu_scheduler {
  	struct list_head		ring_mirror_list;
  	spinlock_t			job_list_lock;
  	int				hang_limit;
+	atomic_t                        num_jobs;
  };
 
  int drm_sched_init(struct drm_gpu_scheduler *sched,
Signature · 6 lines
2.14.3

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
dri-devel mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/dri-devel

_______________________________________________
dri-devel mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/dri-devel

#14 lore raw

Re: [PATCH 3/4] drm/scheduler: add new function to get least loaded sched v2

In reply to Zhou, David(ChunMing)

Hi David,

Quoted 8 lines · David(ChunMing)

Another big question:

I agree the general idea is good to balance scheduler load for same ring family.

But, when same entity job run on different scheduler, that means the later job could be completed ahead of front, Right?

Really good question. To avoid this senario we do not move an entity which already has a job in the hardware queue. We only move entities whose last_scheduled fence has been signalled which means that the last submitted job of this entity has finished executing.

Moving an entity which already has a job in the hardware queue will hinder the dependency optimization that we are using and hence will not anyway lead to a better performance. I have talked about the issue in more detail here [1]. Please let me know if you have any more doubts regarding this.

Cheers,
Nayan

[1]
http://ndesh26.github.io/gsoc/2018/06/14/GSoC-Update-A-Curious-Case-of-Dependency-Handling/

That will break fence design, later fence must be signaled after front

Quoted 91 lines

fence in same fence context.

Anything I missed?

Regards,

David Zhou

*From:* dri-devel <[email protected]> *On Behalf Of *Nayan Deshmukh
*Sent:* Thursday, August 02, 2018 12:07 AM
*To:* Grodzovsky, Andrey <[email protected]>
*Cc:* [email protected]; Maling list - DRI developers < [email protected]>; Koenig, Christian < [email protected]>
*Subject:* Re: [PATCH 3/4] drm/scheduler: add new function to get least loaded sched v2

Yes, that is correct.

Nayan

On Wed, Aug 1, 2018, 9:05 PM Andrey Grodzovsky <[email protected]> wrote:

Clarification question - if the run queues belong to different schedulers they effectively point to different rings,

it means we allow to move (reschedule) a drm_sched_entity from one ring to another - i assume that the idea int the first place, that

you have a set of HW rings and you can utilize any of them for your jobs (like compute rings). Correct ?

Andrey

Quoted 10 lines · Nayan Deshmukh

The function selects the run queue from the rq_list with the least load. The load is decided by the number of jobs in a scheduler.

v2: avoid using atomic read twice consecutively, instead store
it locally

---
  drivers/gpu/drm/scheduler/gpu_scheduler.c | 25

+++++++++++++++++++++++++

Review

Quoted 3 lines
  1 file changed, 25 insertions(+)
(quoted diff) +0 -0

b/drivers/gpu/drm/scheduler/gpu_scheduler.c

drivers/gpu/drm/scheduler/gpu_scheduler.c static bool drm_sched_entity_is_ready(struct

Quoted 4 lines

index 375f6f7f6a93..fb4e542660b0 100644

drivers/gpu/drm/scheduler/gpu_scheduler.c +0 -0
@@ -255,6 +255,31 @@ static bool drm_sched_entity_is_ready(struct

drm_sched_entity *entity)

drivers/gpu/drm/scheduler/gpu_scheduler.c static bool drm_sched_entity_is_ready(struct drm_sched_entity *entity)

Quoted 5 lines
drivers/gpu/drm/scheduler/gpu_scheduler.c +2 -0
      return true;
  }
 
+/**
+ * drm_sched_entity_get_free_sched - Get the rq from rq_list with least

load

Quoted 4 lines
drivers/gpu/drm/scheduler/gpu_scheduler.c +4 -0
+ *
+ * @entity: scheduler entity
+ *
+ * Return the pointer to the rq with least load.

drivers/gpu/drm/scheduler/gpu_scheduler.c static bool drm_sched_entity_is_ready(struct drm_sched_entity *entity)

Quoted 10 lines
drivers/gpu/drm/scheduler/gpu_scheduler.c +10 -0
+ */
+static struct drm_sched_rq *
+drm_sched_entity_get_free_sched(struct drm_sched_entity *entity)
+{
+     struct drm_sched_rq *rq = NULL;
+     unsigned int min_jobs = UINT_MAX, num_jobs;
+     int i;
+
+     for (i = 0; i < entity->num_rq_list; ++i) {
+             num_jobs =

atomic_read(&entity->rq_list[i]->sched->num_jobs);

Quoted 14 lines

drivers/gpu/drm/scheduler/gpu_scheduler.c static bool drm_sched_entity_is_ready(struct drm_sched_entity *entity) · line 285

Quoted 12 lines
drivers/gpu/drm/scheduler/gpu_scheduler.c +9 -0
+             if (num_jobs < min_jobs) {
+                     min_jobs = num_jobs;
+                     rq = entity->rq_list[i];
+             }
+     }
+
+     return rq;
+}
+
  static void drm_sched_entity_kill_jobs_cb(struct dma_fence *f,
                                  struct dma_fence_cb *cb)
  {

_______________________________________________
dri-devel mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/dri-devel

#7 lore raw

Re: [PATCH 2/4] drm/scheduler: add counter for total jobs in scheduler

In reply to Huang Rui

Thanks for the reminders. I felt that the commit header was sufficient enough but I guess that didn't cover the motivation for the change.

Thanks Christian for adding the commit message.

Regards,
Nayan

Quoted 68 lines · Huang Rui
Quoted 4 lines · Christian König

Yeah, I've actually added one before pushing it to amd-staging-drm-next.

But thanks for the reminder, wanted to note that to Nayan as well :)

Yes, a soft reminder to Nayan. Thanks Nayan for the contribution. :-)

Thanks,
Ray

Quoted 10 lines

Christian.

Am 01.08.2018 um 15:15 schrieb Huang Rui:

Quoted 7 lines · Nayan Deshmukh

This should need a commmit message.

Thanks,
Ray

Quoted 7 lines
Quoted 7 lines

Review

Quoted 7 lines
---
  drivers/gpu/drm/scheduler/gpu_scheduler.c | 3 +++
  include/drm/gpu_scheduler.h               | 2 ++
  2 files changed, 5 insertions(+)
(quoted diff) +0 -0

b/drivers/gpu/drm/scheduler/gpu_scheduler.c

Quoted 4 lines
Quoted 4 lines

drivers/gpu/drm/scheduler/gpu_scheduler.c void drm_sched_entity_push_job(struct

Quoted 4 lines

index a3eacc35cf98..375f6f7f6a93 100644

drivers/gpu/drm/scheduler/gpu_scheduler.c +0 -0
@@ -549,6 +549,7 @@ void drm_sched_entity_push_job(struct

drm_sched_job *sched_job,

Quoted 5 lines
Quoted 5 lines

drivers/gpu/drm/scheduler/gpu_scheduler.c void drm_sched_entity_push_job(struct drm_sched_job *sched_job,

Quoted 5 lines
drivers/gpu/drm/scheduler/gpu_scheduler.c +1 -0
   trace_drm_sched_job(sched_job, entity);
 
+  atomic_inc(&entity->rq->sched->num_jobs);
   first = spsc_queue_push(&entity->job_queue,

&sched_job->queue_node);

Quoted 3 lines
Quoted 3 lines

drivers/gpu/drm/scheduler/gpu_scheduler.c static void drm_sched_process_job(struct

Quoted 3 lines
drivers/gpu/drm/scheduler/gpu_scheduler.c +0 -0
   /* first job wakes up scheduler */
@@ -836,6 +837,7 @@ static void drm_sched_process_job(struct

dma_fence *f, struct dma_fence_cb *cb)

Quoted 8 lines
Quoted 8 lines

drivers/gpu/drm/scheduler/gpu_scheduler.c int drm_sched_init(struct drm_gpu_scheduler

Quoted 8 lines
drivers/gpu/drm/scheduler/gpu_scheduler.c +1 -0
   dma_fence_get(&s_fence->finished);
   atomic_dec(&sched->hw_rq_count);
+  atomic_dec(&sched->num_jobs);
   drm_sched_fence_finished(s_fence);
 
   trace_drm_sched_process_job(s_fence);
@@ -953,6 +955,7 @@ int drm_sched_init(struct drm_gpu_scheduler

*sched,

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
drivers/gpu/drm/scheduler/gpu_scheduler.c +1 -0
   INIT_LIST_HEAD(&sched->ring_mirror_list);
   spin_lock_init(&sched->job_list_lock);
   atomic_set(&sched->hw_rq_count, 0);
+  atomic_set(&sched->num_jobs, 0);
   atomic64_set(&sched->job_id_count, 0);
 
Quoted 8 lines
Quoted 8 lines

include/drm/gpu_scheduler.h struct drm_sched_backend_ops { · line 261

Quoted 8 lines
drivers/gpu/drm/scheduler/gpu_scheduler.c +0 -0
   /* Each scheduler will run on a seperate kernel thread */
include/drm/gpu_scheduler.h +0 -0
index a60896222a3e..89881ce974a5 100644
@@ -260,6 +260,7 @@ struct drm_sched_backend_ops {
   * @job_list_lock: lock to protect the ring_mirror_list.
   * @hang_limit: once the hangs by a job crosses this limit then it

is marked

Quoted 27 lines
Quoted 1 line
Quoted 1 line

include/drm/gpu_scheduler.h struct drm_sched_backend_ops { · line 262

Quoted 1 line
include/drm/gpu_scheduler.h +0 -0
   *              guilty and it will be considered for scheduling

further.

Quoted 24 lines
Quoted 23 lines
Quoted 15 lines
include/drm/gpu_scheduler.h +2 -1
+ * @num_jobs: the number of jobs in queue in the scheduler
   *
   * One scheduler is implemented for each hardware ring.
   */
@@ -277,6 +278,7 @@ struct drm_gpu_scheduler {
   struct list_head                ring_mirror_list;
   spinlock_t                      job_list_lock;
   int                             hang_limit;
+  atomic_t                        num_jobs;
  };
 
  int drm_sched_init(struct drm_gpu_scheduler *sched,
--

2.14.3

Quoted 4 lines

_______________________________________________
amd-gfx mailing list [email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
dri-devel mailing list [email protected]
https://lists.freedesktop.org/mailman/listinfo/dri-devel

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#16 lore raw

Re: [PATCH 3/4] drm/scheduler: add new function to get least loaded sched v2

In reply to a message not in the archive
Quoted 6 lines · Nayan Deshmukh

Hi David,

On Thu, Aug 2, 2018 at 8:22 AM Zhou, David(ChunMing) <[email protected] <mailto:[email protected]>> wrote:

Another big question:

Quoted 7 lines

I agree the general idea is good to balance scheduler load for same ring family.

But, when same entity job run on different scheduler, that means the later job could be completed ahead of front, Right?

Really good question. To avoid this senario we do not move an entity which already has a job in the hardware queue. We only move entities whose last_scheduled fence has been signalled which means that the last submitted job of this entity has finished executing.

Good handling I missed when reviewing them.

Cheers,
David Zhou

Quoted 100 lines

Moving an entity which already has a job in the hardware queue will hinder the dependency optimization that we are using and hence will not anyway lead to a better performance. I have talked about the issue in more detail here [1]. Please let me know if you have any more doubts regarding this.

Cheers,
Nayan

[1] http://ndesh26.github.io/gsoc/2018/06/14/GSoC-Update-A-Curious-Case-of-Dependency-Handling/

That will break fence design, later fence must be signaled after front fence in same fence context.

Anything I missed?

Regards,

David Zhou

*From:* dri-devel <dri-devel-bounces-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org <mailto:dri-devel-bounces-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org>> *On Behalf Of *Nayan Deshmukh
*Sent:* Thursday, August 02, 2018 12:07 AM
*To:* Grodzovsky, Andrey <[email protected] <mailto:[email protected]>>
*Cc:* [email protected] <mailto:[email protected]>; Maling list - DRI developers <[email protected] <mailto:[email protected]>>; Koenig, Christian <[email protected] <mailto:[email protected]>> *Subject:* Re: [PATCH 3/4] drm/scheduler: add new function to get
least loaded sched v2

Yes, that is correct.

Nayan

On Wed, Aug 1, 2018, 9:05 PM Andrey Grodzovsky <[email protected] <mailto:[email protected]>> wrote:

Clarification question -  if the run queues belong to different schedulers they effectively point to different rings,

it means we allow to move (reschedule) a drm_sched_entity from one ring
to another - i assume that the idea int the first place, that

you have a set of HW rings and you can utilize any of them for your jobs
(like compute rings). Correct ?

Andrey

On 08/01/2018 04:20 AM, Nayan Deshmukh wrote:
> The function selects the run queue from the rq_list with the
> least load. The load is decided by the number of jobs in a
> scheduler.
>
> v2: avoid using atomic read twice consecutively, instead store
>      it locally
>
> Signed-off-by: Nayan Deshmukh <[email protected] <mailto:[email protected]>>
> ---
>   drivers/gpu/drm/scheduler/gpu_scheduler.c | 25
+++++++++++++++++++++++++
>   1 file changed, 25 insertions(+)
>
> diff --git a/drivers/gpu/drm/scheduler/gpu_scheduler.c b/drivers/gpu/drm/scheduler/gpu_scheduler.c
> index 375f6f7f6a93..fb4e542660b0 100644
> --- a/drivers/gpu/drm/scheduler/gpu_scheduler.c
> +++ b/drivers/gpu/drm/scheduler/gpu_scheduler.c
> @@ -255,6 +255,31 @@ static bool
drm_sched_entity_is_ready(struct drm_sched_entity *entity)
>       return true;
>   }
>
> +/**
> + * drm_sched_entity_get_free_sched - Get the rq from
rq_list with least load
> + *
> + * @entity: scheduler entity
> + *
> + * Return the pointer to the rq with least load.
> + */
> +static struct drm_sched_rq *
> +drm_sched_entity_get_free_sched(struct drm_sched_entity
*entity)
> +{
> +     struct drm_sched_rq *rq = NULL;
> +     unsigned int min_jobs = UINT_MAX, num_jobs;
> +     int i;
> +
> +     for (i = 0; i < entity->num_rq_list; ++i) {
> +             num_jobs = atomic_read(&entity->rq_list[i]->sched->num_jobs);
> +             if (num_jobs < min_jobs) {
> +                     min_jobs = num_jobs;
> +                     rq = entity->rq_list[i];
> +             }

Quoted 9 lines

> +     }
> +
> +     return rq;
> +}
> +
>   static void drm_sched_entity_kill_jobs_cb(struct dma_fence *f, >                                   struct dma_fence_cb *cb) >   {

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx