drm/scheduler: add a list of run queues to the entity
Subsystems (3)
PATCH 1/4 drm/scheduler: add a list of run queues to the entity
These are the potential run queues on which the jobs from this entity can be scheduled. We will use this to do load balancing.
--- drivers/gpu/drm/scheduler/gpu_scheduler.c | 8 ++++++++ include/drm/gpu_scheduler.h | 7 ++++++- 2 files changed, 14 insertions(+), 1 deletion(-)
drivers/gpu/drm/scheduler/gpu_scheduler.c +8 -0
@@ -179,6 +179,8 @@ int drm_sched_entity_init(struct drm_sched_entity *entity,unsigned int num_rq_list, atomic_t *guilty) { + int i; +if (!(entity && rq_list && num_rq_list >0 && rq_list[0 ]))return -EINVAL ;
@@ -186,6 +188,11 @@ int drm_sched_entity_init(struct drm_sched_entity *entity,INIT_LIST_HEAD (&entity->list); entity->rq = rq_list[0 ]; entity->guilty = guilty; + entity->num_rq_list = num_rq_list; + entity->rq_list =kcalloc (num_rq_list,sizeof (struct drm_sched_rq *), +GFP_KERNEL ); +for (i =0 ; i < num_rq_list; ++i) + entity->rq_list[i] = rq_list[i]; entity->last_scheduled =NULL ;spin_lock_init (&entity->rq_lock);
@@ -363,6 +370,7 @@ void drm_sched_entity_fini(struct drm_sched_entity *entity)dma_fence_put (entity->last_scheduled); entity->last_scheduled =NULL ; +kfree (entity->rq_list); }EXPORT_SYMBOL (drm_sched_entity_fini);
include/drm/gpu_scheduler.h +6 -1
@@ -50,7 +50,10 @@ enum drm_sched_priority {* * @list: used to append this struct to the list of entities in the * runqueue. -* @rq: runqueue to which this entity belongs. +* @rq: runqueue on which this entity is currently scheduled. +* @rq_list: a list of run queues on which jobs from this entity can +* be scheduled +* @num_rq_list: number of run queues in the rq_list * @rq_lock: lock to modify the runqueue to which this entity belongs. * @job_queue: the list of jobs of this entity. * @fence_seq: a linearly increasing seqno incremented with each
@@ -74,6 +77,8 @@ enum drm_sched_priority {struct drm_sched_entity {struct list_head list;struct drm_sched_rq *rq; +struct drm_sched_rq **rq_list; +unsigned int num_rq_list;spinlock_t rq_lock;struct spsc_queue job_queue;
Signature · 6 lines
2.14.3 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): 20180801082002.20696-1-nayan26deshmukh-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org
PATCH 2/4 drm/scheduler: add counter for total jobs in scheduler
--- drivers/gpu/drm/scheduler/gpu_scheduler.c | 3 +++ include/drm/gpu_scheduler.h | 2 ++ 2 files changed, 5 insertions(+)
drivers/gpu/drm/scheduler/gpu_scheduler.c +3 -0
@@ -549,6 +549,7 @@ void drm_sched_entity_push_job(struct drm_sched_job *sched_job,trace_drm_sched_job (sched_job, entity); +atomic_inc (&entity->rq->sched->num_jobs); first =spsc_queue_push (&entity->job_queue, &sched_job->queue_node);/* first job wakes up scheduler */
@@ -836,6 +837,7 @@ static void drm_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb)dma_fence_get (&s_fence->finished);atomic_dec (&sched->hw_rq_count); +atomic_dec (&sched->num_jobs);drm_sched_fence_finished (s_fence);trace_drm_sched_process_job (s_fence);
@@ -953,6 +955,7 @@ int drm_sched_init(struct drm_gpu_scheduler *sched,INIT_LIST_HEAD (&sched->ring_mirror_list);spin_lock_init (&sched->job_list_lock);atomic_set (&sched->hw_rq_count,0 ); +atomic_set (&sched->num_jobs,0 );atomic64_set (&sched->job_id_count,0 );/* Each scheduler will run on a seperate kernel thread */
include/drm/gpu_scheduler.h +2 -0
@@ -260,6 +260,7 @@ struct drm_sched_backend_ops {* @job_list_lock: lock to protect the ring_mirror_list. * @hang_limit: once the hangs by a job crosses this limit then it is marked * guilty and it will be considered for scheduling further. +* @num_jobs: the number of jobs in queue in the scheduler * * One scheduler is implemented for each hardware ring. */
@@ -277,6 +278,7 @@ struct drm_gpu_scheduler {struct list_head ring_mirror_list;spinlock_t job_list_lock;int hang_limit; +atomic_t num_jobs; }; int drm_sched_init(struct drm_gpu_scheduler *sched,
Signature · 6 lines
2.14.3 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): 20180801082002.20696-2-nayan26deshmukh-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org
Re: [PATCH 2/4] drm/scheduler: add counter for total jobs in scheduler
On Wed, Aug 01, 2018 at 01:50:00PM +0530, Nayan Deshmukh wrote:
This should need a commmit message.
Thanks,
Ray
Quoted 61 lines
Signed-off-by Nayan Deshmukh <[email protected]>--- drivers/gpu/drm/scheduler/gpu_scheduler.c | 3 +++ include/drm/gpu_scheduler.h | 2 ++ 2 files changed, 5 insertions(+)drivers/gpu/drm/scheduler/gpu_scheduler.c +3
-0@@ -549,6 +549,7 @@ void drm_sched_entity_push_job(struct drm_sched_job *sched_job,trace_drm_sched_job (sched_job, entity); +atomic_inc (&entity->rq->sched->num_jobs); first =spsc_queue_push (&entity->job_queue, &sched_job->queue_node);/* first job wakes up scheduler */ @@ -836,6 +837,7 @@ static void drm_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb)dma_fence_get (&s_fence->finished);atomic_dec (&sched->hw_rq_count); +atomic_dec (&sched->num_jobs);drm_sched_fence_finished (s_fence);trace_drm_sched_process_job (s_fence);@@ -953,6 +955,7 @@ int drm_sched_init(struct drm_gpu_scheduler *sched,INIT_LIST_HEAD (&sched->ring_mirror_list);spin_lock_init (&sched->job_list_lock);atomic_set (&sched->hw_rq_count,0 ); +atomic_set (&sched->num_jobs,0 );atomic64_set (&sched->job_id_count,0 );/* Each scheduler will run on a seperate kernel thread */ include/drm/gpu_scheduler.h +2
-0@@ -260,6 +260,7 @@ struct drm_sched_backend_ops {* @job_list_lock: lock to protect the ring_mirror_list. * @hang_limit: once the hangs by a job crosses this limit then it is marked * guilty and it will be considered for scheduling further. +* @num_jobs: the number of jobs in queue in the scheduler * * One scheduler is implemented for each hardware ring. */ @@ -277,6 +278,7 @@ struct drm_gpu_scheduler {struct list_head ring_mirror_list;spinlock_t job_list_lock;int hang_limit; +atomic_t num_jobs; }; int drm_sched_init(struct drm_gpu_scheduler *sched,Signature · 6 lines
2.14.3 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Re: [PATCH 2/4] drm/scheduler: add counter for total jobs in scheduler
Yeah, I've actually added one before pushing it to amd-staging-drm-next.
But thanks for the reminder, wanted to note that to Nayan as well :)
Christian.
Am 01.08.2018 um 15:15 schrieb Huang Rui:
Quoted 64 lines · Nayan Deshmukh
This should need a commmit message.
Thanks,
RayQuoted 57 lines
Signed-off-by Nayan Deshmukh <[email protected]>--- drivers/gpu/drm/scheduler/gpu_scheduler.c | 3 +++ include/drm/gpu_scheduler.h | 2 ++ 2 files changed, 5 insertions(+)drivers/gpu/drm/scheduler/gpu_scheduler.c +3
-0@@ -549,6 +549,7 @@ void drm_sched_entity_push_job(struct drm_sched_job *sched_job,trace_drm_sched_job (sched_job, entity); +atomic_inc (&entity->rq->sched->num_jobs); first =spsc_queue_push (&entity->job_queue, &sched_job->queue_node);/* first job wakes up scheduler */ @@ -836,6 +837,7 @@ static void drm_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb)dma_fence_get (&s_fence->finished);atomic_dec (&sched->hw_rq_count); +atomic_dec (&sched->num_jobs);drm_sched_fence_finished (s_fence);trace_drm_sched_process_job (s_fence);@@ -953,6 +955,7 @@ int drm_sched_init(struct drm_gpu_scheduler *sched,INIT_LIST_HEAD (&sched->ring_mirror_list);spin_lock_init (&sched->job_list_lock);atomic_set (&sched->hw_rq_count,0 ); +atomic_set (&sched->num_jobs,0 );atomic64_set (&sched->job_id_count,0 );/* Each scheduler will run on a seperate kernel thread */ include/drm/gpu_scheduler.h +2
-0@@ -260,6 +260,7 @@ struct drm_sched_backend_ops {* @job_list_lock: lock to protect the ring_mirror_list. * @hang_limit: once the hangs by a job crosses this limit then it is marked * guilty and it will be considered for scheduling further. +* @num_jobs: the number of jobs in queue in the scheduler * * One scheduler is implemented for each hardware ring. */ @@ -277,6 +278,7 @@ struct drm_gpu_scheduler {struct list_head ring_mirror_list;spinlock_t job_list_lock;int hang_limit; +atomic_t num_jobs; }; int drm_sched_init(struct drm_gpu_scheduler *sched,Signature · 1 line
2.14.3
Quoted 8 lines
Quoted 4 lines
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx_______________________________________________
dri-devel mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/dri-devel
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Re: [PATCH 2/4] drm/scheduler: add counter for total jobs in scheduler
Quoted 4 lines · Christian König
Yeah, I've actually added one before pushing it to amd-staging-drm-next.
But thanks for the reminder, wanted to note that to Nayan as well :)
Yes, a soft reminder to Nayan. Thanks Nayan for the contribution. :-)
Thanks,
Ray
Quoted 76 lines
Christian.
Am 01.08.2018 um 15:15 schrieb Huang Rui:
Quoted 72 lines · Nayan Deshmukh
This should need a commmit message.
Thanks,
RayQuoted 61 lines
Signed-off-by Nayan Deshmukh <[email protected]>--- drivers/gpu/drm/scheduler/gpu_scheduler.c | 3 +++ include/drm/gpu_scheduler.h | 2 ++ 2 files changed, 5 insertions(+)drivers/gpu/drm/scheduler/gpu_scheduler.c +3
-0@@ -549,6 +549,7 @@ void drm_sched_entity_push_job(struct drm_sched_job *sched_job,trace_drm_sched_job (sched_job, entity); +atomic_inc (&entity->rq->sched->num_jobs); first =spsc_queue_push (&entity->job_queue, &sched_job->queue_node);/* first job wakes up scheduler */ @@ -836,6 +837,7 @@ static void drm_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb)dma_fence_get (&s_fence->finished);atomic_dec (&sched->hw_rq_count); +atomic_dec (&sched->num_jobs);drm_sched_fence_finished (s_fence);trace_drm_sched_process_job (s_fence);@@ -953,6 +955,7 @@ int drm_sched_init(struct drm_gpu_scheduler *sched,INIT_LIST_HEAD (&sched->ring_mirror_list);spin_lock_init (&sched->job_list_lock);atomic_set (&sched->hw_rq_count,0 ); +atomic_set (&sched->num_jobs,0 );atomic64_set (&sched->job_id_count,0 );/* Each scheduler will run on a seperate kernel thread */ include/drm/gpu_scheduler.h +2
-0@@ -260,6 +260,7 @@ struct drm_sched_backend_ops {* @job_list_lock: lock to protect the ring_mirror_list. * @hang_limit: once the hangs by a job crosses this limit then it is marked * guilty and it will be considered for scheduling further. +* @num_jobs: the number of jobs in queue in the scheduler * * One scheduler is implemented for each hardware ring. */ @@ -277,6 +278,7 @@ struct drm_gpu_scheduler {struct list_head ring_mirror_list;spinlock_t job_list_lock;int hang_limit; +atomic_t num_jobs; }; int drm_sched_init(struct drm_gpu_scheduler *sched,Signature · 6 lines
2.14.3 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx_______________________________________________
dri-devel mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/dri-devel
_______________________________________________
dri-devel mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/dri-devel
Re: [PATCH 2/4] drm/scheduler: add counter for total jobs in scheduler
Thanks for the reminders. I felt that the commit header was sufficient enough but I guess that didn't cover the motivation for the change.
Thanks Christian for adding the commit message.
Regards,
Nayan
Quoted 68 lines · Huang Rui
Quoted 4 lines · Christian König
Yeah, I've actually added one before pushing it to amd-staging-drm-next.
But thanks for the reminder, wanted to note that to Nayan as well :)
Yes, a soft reminder to Nayan. Thanks Nayan for the contribution. :-)
Thanks,
RayQuoted 10 lines
Christian.
Am 01.08.2018 um 15:15 schrieb Huang Rui:
Quoted 7 lines · Nayan Deshmukh
This should need a commmit message.
Thanks,
RayQuoted 7 lines
Quoted 7 lines
Review
Quoted 7 lines
Signed-off-by Nayan Deshmukh <[email protected]>--- drivers/gpu/drm/scheduler/gpu_scheduler.c | 3 +++ include/drm/gpu_scheduler.h | 2 ++ 2 files changed, 5 insertions(+)(quoted diff) +0
-0
b/drivers/gpu/drm/scheduler/gpu_scheduler.cQuoted 4 lines
Quoted 4 lines
drivers/gpu/drm/scheduler/gpu_scheduler.c void drm_sched_entity_push_job(struct
Quoted 4 lines
index
a3eacc35cf98..375f6f7f6a93100644drivers/gpu/drm/scheduler/gpu_scheduler.c +0
-0@@ -549,6 +549,7 @@ void drm_sched_entity_push_job(struct
drm_sched_job*sched_job,Quoted 5 lines
Quoted 5 lines
drivers/gpu/drm/scheduler/gpu_scheduler.c void drm_sched_entity_push_job(struct drm_sched_job *sched_job,
Quoted 5 lines
drivers/gpu/drm/scheduler/gpu_scheduler.c +1
-0trace_drm_sched_job (sched_job, entity); +atomic_inc (&entity->rq->sched->num_jobs); first = spsc_queue_push(&entity->job_queue,
&sched_job->queue_node);Quoted 3 lines
Quoted 3 lines
drivers/gpu/drm/scheduler/gpu_scheduler.c static void drm_sched_process_job(struct
Quoted 3 lines
drivers/gpu/drm/scheduler/gpu_scheduler.c +0
-0/* first job wakes up scheduler */ @@ -836,6 +837,7 @@ static void drm_sched_process_job(struct
dma_fence*f, structdma_fence_cb*cb)Quoted 8 lines
Quoted 8 lines
drivers/gpu/drm/scheduler/gpu_scheduler.c int drm_sched_init(struct drm_gpu_scheduler
Quoted 8 lines
drivers/gpu/drm/scheduler/gpu_scheduler.c +1
-0dma_fence_get (&s_fence->finished);atomic_dec (&sched->hw_rq_count); +atomic_dec (&sched->num_jobs);drm_sched_fence_finished (s_fence);trace_drm_sched_process_job (s_fence);@@ -953,6 +955,7 @@ int drm_sched_init(struct drm_gpu_scheduler*sched,
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
drivers/gpu/drm/scheduler/gpu_scheduler.c +1
-0INIT_LIST_HEAD (&sched->ring_mirror_list);spin_lock_init (&sched->job_list_lock);atomic_set (&sched->hw_rq_count,0 ); +atomic_set (&sched->num_jobs,0 );atomic64_set (&sched->job_id_count,0 );Quoted 8 lines
Quoted 8 lines
include/drm/gpu_scheduler.h struct drm_sched_backend_ops { · line 261
Quoted 8 lines
drivers/gpu/drm/scheduler/gpu_scheduler.c +0
-0/* Each scheduler will run on a seperate kernel thread */ include/drm/gpu_scheduler.h +0
-0@@ -260,6 +260,7 @@ struct drm_sched_backend_ops {* @job_list_lock: lock to protect the ring_mirror_list. * @hang_limit: once the hangs by a job crosses this limit then it is marked
Quoted 27 lines
Quoted 1 line
Quoted 1 line
include/drm/gpu_scheduler.h struct drm_sched_backend_ops { · line 262
Quoted 1 line
include/drm/gpu_scheduler.h +0
-0* guilty and it will be considered for scheduling further.
Quoted 24 lines
Quoted 23 lines
Quoted 15 lines
include/drm/gpu_scheduler.h +2
-1+* @num_jobs: the number of jobs in queue in the scheduler * * One scheduler is implemented for each hardware ring. */ @@ -277,6 +278,7 @@ struct drm_gpu_scheduler {struct list_head ring_mirror_list;spinlock_t job_list_lock;int hang_limit; +atomic_t num_jobs; }; int drm_sched_init(struct drm_gpu_scheduler *sched, --2.14.3
Quoted 4 lines
_______________________________________________
amd-gfx mailing list [email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx_______________________________________________
dri-devel mailing list [email protected]
https://lists.freedesktop.org/mailman/listinfo/dri-devel
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
PATCH 3/4 drm/scheduler: add new function to get least loaded sched v2
In reply to a message not in the archiveThe function selects the run queue from the rq_list with the least load. The load is decided by the number of jobs in a scheduler.
v2: avoid using atomic read twice consecutively, instead store
it locally
--- drivers/gpu/drm/scheduler/gpu_scheduler.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+)
drivers/gpu/drm/scheduler/gpu_scheduler.c +25 -0
@@ -255,6 +255,31 @@ static bool drm_sched_entity_is_ready(struct drm_sched_entity *entity)return true; } +/** +* drm_sched_entity_get_free_sched - Get the rq from rq_list with least load +* +* @entity: scheduler entity +* +* Return the pointer to the rq with least load. +*/ +static struct drm_sched_rq * +drm_sched_entity_get_free_sched (struct drm_sched_entity *entity) +{ +struct drm_sched_rq *rq =NULL ; +unsigned int min_jobs =UINT_MAX , num_jobs; +int i; + +for (i =0 ; i < entity->num_rq_list; ++i) { + num_jobs =atomic_read (&entity->rq_list[i]->sched->num_jobs); +if (num_jobs < min_jobs) { + min_jobs = num_jobs; + rq = entity->rq_list[i]; + } + } + +return rq; +} +static void drm_sched_entity_kill_jobs_cb (struct dma_fence *f,struct dma_fence_cb *cb) {
Signature · 6 lines
2.14.3 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Re: [PATCH 3/4] drm/scheduler: add new function to get least loaded sched v2
Clarification question - if the run queues belong to different schedulers they effectively point to different rings,
it means we allow to move (reschedule) a drm_sched_entity from one ring to another - i assume that the idea int the first place, that
you have a set of HW rings and you can utilize any of them for your jobs (like compute rings). Correct ?
Andrey
Quoted 38 lines · Nayan Deshmukh
The function selects the run queue from the
rq_listwith the least load. The load is decided by the number of jobs in a scheduler.v2: avoid using atomic read twice consecutively, instead store
it locallySigned-off-by Nayan Deshmukh <[email protected]>--- drivers/gpu/drm/scheduler/gpu_scheduler.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+)drivers/gpu/drm/scheduler/gpu_scheduler.c +18
-0@@ -255,6 +255,31 @@ static bool drm_sched_entity_is_ready(struct drm_sched_entity *entity)return true; } +/** +* drm_sched_entity_get_free_sched - Get the rq from rq_list with least load +* +* @entity: scheduler entity +* +* Return the pointer to the rq with least load. +*/ +static struct drm_sched_rq * +drm_sched_entity_get_free_sched (struct drm_sched_entity *entity) +{ +struct drm_sched_rq *rq =NULL ; +unsigned int min_jobs =UINT_MAX , num_jobs; +int i; + +for (i =0 ; i < entity->num_rq_list; ++i) { + num_jobs =atomic_read (&entity->rq_list[i]->sched->num_jobs); +if (num_jobs < min_jobs) { + min_jobs = num_jobs;
drivers/gpu/drm/scheduler/gpu_scheduler.c static bool drm_sched_entity_is_ready(struct drm_sched_entity *entity) · line 285
Quoted 10 lines
drivers/gpu/drm/scheduler/gpu_scheduler.c +7
-0+ rq = entity->rq_list[i]; + } + } + +return rq; +} +static void drm_sched_entity_kill_jobs_cb (struct dma_fence *f,struct dma_fence_cb *cb) {
_______________________________________________
dri-devel mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/dri-devel
Re: [PATCH 3/4] drm/scheduler: add new function to get least loaded sched v2
Yes, that is correct.
Nayan
Quoted 54 lines · Andrey Grodzovsky
Clarification question - if the run queues belong to different schedulers they effectively point to different rings,
it means we allow to move (reschedule) a
drm_sched_entityfrom one ring to another - i assume that the idea int the first place, thatyou have a set of HW rings and you can utilize any of them for your jobs (like compute rings). Correct ?
Andrey
Quoted 10 lines · Nayan Deshmukh
The function selects the run queue from the
rq_listwith the least load. The load is decided by the number of jobs in a scheduler.v2: avoid using atomic read twice consecutively, instead store
it locallySigned-off-by Nayan Deshmukh <[email protected]>--- drivers/gpu/drm/scheduler/gpu_scheduler.c | 25+++++++++++++++++++++++++
Review
Quoted 3 lines
1 file changed, 25 insertions(+)(quoted diff) +0
-0
b/drivers/gpu/drm/scheduler/gpu_scheduler.cdrivers/gpu/drm/scheduler/gpu_scheduler.c static bool drm_sched_entity_is_ready(struct
Quoted 4 lines
index
375f6f7f6a93..fb4e542660b0100644drivers/gpu/drm/scheduler/gpu_scheduler.c +0
-0@@ -255,6 +255,31 @@ static bool drm_sched_entity_is_ready(struct
drm_sched_entity*entity)drivers/gpu/drm/scheduler/gpu_scheduler.c static bool drm_sched_entity_is_ready(struct drm_sched_entity *entity)
Quoted 5 lines
drivers/gpu/drm/scheduler/gpu_scheduler.c +2
-0return true; } +/** +* drm_sched_entity_get_free_sched - Get the rq from rq_list with least load
Quoted 4 lines
drivers/gpu/drm/scheduler/gpu_scheduler.c +4
-0+* +* @entity: scheduler entity +* +* Return the pointer to the rq with least load. drivers/gpu/drm/scheduler/gpu_scheduler.c static bool drm_sched_entity_is_ready(struct drm_sched_entity *entity)
Quoted 10 lines
drivers/gpu/drm/scheduler/gpu_scheduler.c +10
-0+*/ +static struct drm_sched_rq * +drm_sched_entity_get_free_sched (struct drm_sched_entity *entity) +{ +struct drm_sched_rq *rq =NULL ; +unsigned int min_jobs =UINT_MAX , num_jobs; +int i; + +for (i =0 ; i < entity->num_rq_list; ++i) { + num_jobs =
atomic_read(&entity->rq_list[i]->sched->num_jobs);
Quoted 14 lines
drivers/gpu/drm/scheduler/gpu_scheduler.c static bool drm_sched_entity_is_ready(struct drm_sched_entity *entity) · line 285
Quoted 12 lines
drivers/gpu/drm/scheduler/gpu_scheduler.c +9
-0+if (num_jobs < min_jobs) { + min_jobs = num_jobs; + rq = entity->rq_list[i]; + } + } + +return rq; +} +static void drm_sched_entity_kill_jobs_cb (struct dma_fence *f,struct dma_fence_cb *cb) {
_______________________________________________
dri-devel mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/dri-devel
A message that is not in the archive (sent off-list or not yet received): CAFd4ddx8D2iKquRu4YVh1gnRMpLFgWf3CBPdk5SD-nJ8dNXEPQ-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org
Re: [PATCH 3/4] drm/scheduler: add new function to get least loaded sched v2
Series is Acked-by: Andrey Grodzovsky <[email protected]>
Andrey
Quoted 67 lines · Nayan Deshmukh
Yes, that is correct.
Nayan
On Wed, Aug 1, 2018, 9:05 PM Andrey Grodzovsky <[email protected] <mailto:[email protected]>> wrote:
Clarification question - if the run queues belong to different schedulers they effectively point to different rings, it means we allow to move (reschedule) a drm_sched_entity from one ring to another - i assume that the idea int the first place, that you have a set of HW rings and you can utilize any of them for your jobs (like compute rings). Correct ? Andrey On 08/01/2018 04:20 AM, Nayan Deshmukh wrote: > The function selects the run queue from the rq_list with the > least load. The load is decided by the number of jobs in a > scheduler. > > v2: avoid using atomic read twice consecutively, instead store > it locally > > Signed-off-by: Nayan Deshmukh <[email protected] <mailto:[email protected]>> > --- > drivers/gpu/drm/scheduler/gpu_scheduler.c | 25 +++++++++++++++++++++++++ > 1 file changed, 25 insertions(+) > > diff --git a/drivers/gpu/drm/scheduler/gpu_scheduler.c b/drivers/gpu/drm/scheduler/gpu_scheduler.c > index 375f6f7f6a93..fb4e542660b0 100644 > --- a/drivers/gpu/drm/scheduler/gpu_scheduler.c > +++ b/drivers/gpu/drm/scheduler/gpu_scheduler.c > @@ -255,6 +255,31 @@ static bool drm_sched_entity_is_ready(struct drm_sched_entity *entity) > return true; > } > > +/** > + * drm_sched_entity_get_free_sched - Get the rq from rq_list with least load > + * > + * @entity: scheduler entity > + * > + * Return the pointer to the rq with least load. > + */ > +static struct drm_sched_rq * > +drm_sched_entity_get_free_sched(struct drm_sched_entity *entity) > +{ > + struct drm_sched_rq *rq = NULL; > + unsigned int min_jobs = UINT_MAX, num_jobs; > + int i; > + > + for (i = 0; i < entity->num_rq_list; ++i) { > + num_jobs = atomic_read(&entity->rq_list[i]->sched->num_jobs); > + if (num_jobs < min_jobs) { > + min_jobs = num_jobs; > + rq = entity->rq_list[i]; > + }
Quoted 9 lines
> + }
> +
> + return rq;
> +}
> +
> static void drm_sched_entity_kill_jobs_cb(struct dma_fence *f,
> struct dma_fence_cb *cb)
> {_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
RE: [PATCH 3/4] drm/scheduler: add new function to get least loaded sched v2
In reply to a message not in the archiveAnother big question:
I agree the general idea is good to balance scheduler load for same ring family.
But, when same entity job run on different scheduler, that means the later job could be completed ahead of front, Right? That will break fence design, later fence must be signaled after front fence in same fence context.
Anything I missed?
Regards,
David Zhou
From: dri-devel <[email protected]> On Behalf Of Nayan Deshmukh
Sent: Thursday, August 02, 2018 12:07 AM
To: Grodzovsky, Andrey <[email protected]>
Cc: [email protected]; Maling list - DRI developers <[email protected]>; Koenig, Christian <[email protected]>
Subject: Re: [PATCH 3/4] drm/scheduler: add new function to get least loaded sched v2
Yes, that is correct.
Nayan
On Wed, Aug 1, 2018, 9:05 PM Andrey Grodzovsky <[email protected]<mailto:[email protected]>> wrote:
Clarification question - if the run queues belong to different
schedulers they effectively point to different rings,
it means we allow to move (reschedule) a drm_sched_entity from one ring to another - i assume that the idea int the first place, that
you have a set of HW rings and you can utilize any of them for your jobs (like compute rings). Correct ?
Andrey
Quoted 38 lines · Nayan Deshmukh
The function selects the run queue from the
rq_listwith the least load. The load is decided by the number of jobs in a scheduler.v2: avoid using atomic read twice consecutively, instead store
it locally--- drivers/gpu/drm/scheduler/gpu_scheduler.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+)drivers/gpu/drm/scheduler/gpu_scheduler.c +18
-0@@ -255,6 +255,31 @@ static bool drm_sched_entity_is_ready(struct drm_sched_entity *entity)return true; } +/** +* drm_sched_entity_get_free_sched - Get the rq from rq_list with least load +* +* @entity: scheduler entity +* +* Return the pointer to the rq with least load. +*/ +static struct drm_sched_rq * +drm_sched_entity_get_free_sched (struct drm_sched_entity *entity) +{ +struct drm_sched_rq *rq =NULL ; +unsigned int min_jobs =UINT_MAX , num_jobs; +int i; + +for (i =0 ; i < entity->num_rq_list; ++i) { + num_jobs =atomic_read (&entity->rq_list[i]->sched->num_jobs); +if (num_jobs < min_jobs) { + min_jobs = num_jobs;
drivers/gpu/drm/scheduler/gpu_scheduler.c static bool drm_sched_entity_is_ready(struct drm_sched_entity *entity) · line 285
Quoted 10 lines
drivers/gpu/drm/scheduler/gpu_scheduler.c +7
-0+ rq = entity->rq_list[i]; + } + } + +return rq; +} +static void drm_sched_entity_kill_jobs_cb (struct dma_fence *f,struct dma_fence_cb *cb) {
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Re: [PATCH 3/4] drm/scheduler: add new function to get least loaded sched v2
Hi David,
Quoted 8 lines · David(ChunMing)
Another big question:
I agree the general idea is good to balance scheduler load for same ring family.
But, when same entity job run on different scheduler, that means the later job could be completed ahead of front, Right?
Really good question. To avoid this senario we do not move an entity which already has a job in the hardware queue. We only move entities whose last_scheduled fence has been signalled which means that the last submitted job of this entity has finished executing.
Moving an entity which already has a job in the hardware queue will hinder the dependency optimization that we are using and hence will not anyway lead to a better performance. I have talked about the issue in more detail here [1]. Please let me know if you have any more doubts regarding this.
Cheers,
Nayan
[1]
http://ndesh26.github.io/gsoc/2018/06/14/GSoC-Update-A-Curious-Case-of-Dependency-Handling/
That will break fence design, later fence must be signaled after front
Quoted 91 lines
fence in same fence context.
Anything I missed?
Regards,
David Zhou
*From:* dri-devel <[email protected]> *On Behalf Of *Nayan Deshmukh
*Sent:* Thursday, August 02, 2018 12:07 AM
*To:* Grodzovsky, Andrey <[email protected]>
*Cc:* [email protected]; Maling list - DRI developers < [email protected]>; Koenig, Christian < [email protected]>
*Subject:* Re: [PATCH 3/4] drm/scheduler: add new function to get least loaded sched v2Yes, that is correct.
Nayan
On Wed, Aug 1, 2018, 9:05 PM Andrey Grodzovsky <[email protected]> wrote:
Clarification question - if the run queues belong to different schedulers they effectively point to different rings,
it means we allow to move (reschedule) a
drm_sched_entityfrom one ring to another - i assume that the idea int the first place, thatyou have a set of HW rings and you can utilize any of them for your jobs (like compute rings). Correct ?
Andrey
Quoted 10 lines · Nayan Deshmukh
The function selects the run queue from the
rq_listwith the least load. The load is decided by the number of jobs in a scheduler.v2: avoid using atomic read twice consecutively, instead store
it locallySigned-off-by Nayan Deshmukh <[email protected]>--- drivers/gpu/drm/scheduler/gpu_scheduler.c | 25+++++++++++++++++++++++++
Review
Quoted 3 lines
1 file changed, 25 insertions(+)(quoted diff) +0
-0
b/drivers/gpu/drm/scheduler/gpu_scheduler.cdrivers/gpu/drm/scheduler/gpu_scheduler.c static bool drm_sched_entity_is_ready(struct
Quoted 4 lines
index
375f6f7f6a93..fb4e542660b0100644drivers/gpu/drm/scheduler/gpu_scheduler.c +0
-0@@ -255,6 +255,31 @@ static bool drm_sched_entity_is_ready(struct
drm_sched_entity*entity)drivers/gpu/drm/scheduler/gpu_scheduler.c static bool drm_sched_entity_is_ready(struct drm_sched_entity *entity)
Quoted 5 lines
drivers/gpu/drm/scheduler/gpu_scheduler.c +2
-0return true; } +/** +* drm_sched_entity_get_free_sched - Get the rq from rq_list with least load
Quoted 4 lines
drivers/gpu/drm/scheduler/gpu_scheduler.c +4
-0+* +* @entity: scheduler entity +* +* Return the pointer to the rq with least load. drivers/gpu/drm/scheduler/gpu_scheduler.c static bool drm_sched_entity_is_ready(struct drm_sched_entity *entity)
Quoted 10 lines
drivers/gpu/drm/scheduler/gpu_scheduler.c +10
-0+*/ +static struct drm_sched_rq * +drm_sched_entity_get_free_sched (struct drm_sched_entity *entity) +{ +struct drm_sched_rq *rq =NULL ; +unsigned int min_jobs =UINT_MAX , num_jobs; +int i; + +for (i =0 ; i < entity->num_rq_list; ++i) { + num_jobs =
atomic_read(&entity->rq_list[i]->sched->num_jobs);
Quoted 14 lines
drivers/gpu/drm/scheduler/gpu_scheduler.c static bool drm_sched_entity_is_ready(struct drm_sched_entity *entity) · line 285
Quoted 12 lines
drivers/gpu/drm/scheduler/gpu_scheduler.c +9
-0+if (num_jobs < min_jobs) { + min_jobs = num_jobs; + rq = entity->rq_list[i]; + } + } + +return rq; +} +static void drm_sched_entity_kill_jobs_cb (struct dma_fence *f,struct dma_fence_cb *cb) {
_______________________________________________
dri-devel mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/dri-devel
A message that is not in the archive (sent off-list or not yet received): CAFd4ddyf=EhJ7pmzq3sEGa6U1sQ7Ga7fUH+sW+VKeBxJABjnKQ-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org
Re: [PATCH 3/4] drm/scheduler: add new function to get least loaded sched v2
Quoted 6 lines · Nayan Deshmukh
Hi David,
On Thu, Aug 2, 2018 at 8:22 AM Zhou, David(ChunMing) <[email protected] <mailto:[email protected]>> wrote:
Another big question:
Quoted 7 lines
I agree the general idea is good to balance scheduler load for same ring family.
But, when same entity job run on different scheduler, that means the later job could be completed ahead of front, Right?
Really good question. To avoid this senario we do not move an entity which already has a job in the hardware queue. We only move entities whose
last_scheduledfence has been signalled which means that the last submitted job of this entity has finished executing.
Good handling I missed when reviewing them.
Cheers,
David Zhou
Quoted 100 lines
Moving an entity which already has a job in the hardware queue will hinder the dependency optimization that we are using and hence will not anyway lead to a better performance. I have talked about the issue in more detail here [1]. Please let me know if you have any more doubts regarding this.
Cheers,
Nayan[1] http://ndesh26.github.io/gsoc/2018/06/14/GSoC-Update-A-Curious-Case-of-Dependency-Handling/
That will break fence design, later fence must be signaled after front fence in same fence context.
Anything I missed?
Regards,
David Zhou
*From:* dri-devel <dri-devel-bounces-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org <mailto:dri-devel-bounces-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org>> *On Behalf Of *Nayan Deshmukh
*Sent:* Thursday, August 02, 2018 12:07 AM
*To:* Grodzovsky, Andrey <[email protected] <mailto:[email protected]>>
*Cc:* [email protected] <mailto:[email protected]>; Maling list - DRI developers <[email protected] <mailto:[email protected]>>; Koenig, Christian <[email protected] <mailto:[email protected]>> *Subject:* Re: [PATCH 3/4] drm/scheduler: add new function to get
least loaded sched v2Yes, that is correct.
Nayan
On Wed, Aug 1, 2018, 9:05 PM Andrey Grodzovsky <[email protected] <mailto:[email protected]>> wrote:
Clarification question - if the run queues belong to different schedulers they effectively point to different rings,
it means we allow to move (reschedule) a
drm_sched_entityfrom one ring
to another - i assume that the idea int the first place, thatyou have a set of HW rings and you can utilize any of them for your jobs
(like compute rings). Correct ?Andrey
On 08/01/2018 04:20 AM, Nayan Deshmukh wrote:
> The function selects the run queue from therq_listwith the
> least load. The load is decided by the number of jobs in a
> scheduler.
>
> v2: avoid using atomic read twice consecutively, instead store
> it locally
>
> Signed-off-by: Nayan Deshmukh <[email protected] <mailto:[email protected]>>
> ---
> drivers/gpu/drm/scheduler/gpu_scheduler.c | 25
+++++++++++++++++++++++++
> 1 file changed, 25 insertions(+)
>
> diff --gita/drivers/gpu/drm/scheduler/gpu_scheduler.cb/drivers/gpu/drm/scheduler/gpu_scheduler.c
> index375f6f7f6a93..fb4e542660b0100644
> ---a/drivers/gpu/drm/scheduler/gpu_scheduler.c
> +++b/drivers/gpu/drm/scheduler/gpu_scheduler.c
> @@ -255,6 +255,31 @@ static bool
drm_sched_entity_is_ready(struct drm_sched_entity *entity)
> return true;
> }
>
> +/**
> + *drm_sched_entity_get_free_sched- Get the rq from
rq_listwith least load
> + *
> + * @entity: scheduler entity
> + *
> + * Return the pointer to the rq with least load.
> + */
> +static structdrm_sched_rq*
> +drm_sched_entity_get_free_sched(structdrm_sched_entity
*entity)
> +{
> + structdrm_sched_rq*rq = NULL;
> + unsigned intmin_jobs=UINT_MAX,num_jobs;
> + int i;
> +
> + for (i = 0; i <entity->num_rq_list; ++i) {
> + num_jobs =atomic_read(&entity->rq_list[i]->sched->num_jobs);
> + if (num_jobs<min_jobs) {
> + min_jobs =num_jobs;
> + rq = entity->rq_list[i];
> + }
Quoted 9 lines
> + }
> +
> + return rq;
> +}
> +
> static voiddrm_sched_entity_kill_jobs_cb(struct dma_fence *f, > struct dma_fence_cb *cb)> {
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Christian König
Reviewed-byAm 01.08.2018 um 10:19 schrieb Nayan Deshmukh:
Quoted 4 lines
These are the potential run queues on which the jobs from this entity can be scheduled. We will use this to do load balancing.
Signed-off-by Nayan Deshmukh <[email protected]>
I also just pushed them into our internal branch for upstreaming.
Thanks for all the work,
Christian.
Quoted 55 lines
--- drivers/gpu/drm/scheduler/gpu_scheduler.c | 8 ++++++++ include/drm/gpu_scheduler.h | 7 ++++++- 2 files changed, 14 insertions(+), 1 deletion(-)drivers/gpu/drm/scheduler/gpu_scheduler.c +8
-0@@ -179,6 +179,8 @@ int drm_sched_entity_init(struct drm_sched_entity *entity,unsigned int num_rq_list, atomic_t *guilty) { + int i; +if (!(entity && rq_list && num_rq_list >0 && rq_list[0 ]))return -EINVAL ;@@ -186,6 +188,11 @@ int drm_sched_entity_init(struct drm_sched_entity *entity,INIT_LIST_HEAD (&entity->list); entity->rq = rq_list[0 ]; entity->guilty = guilty; + entity->num_rq_list = num_rq_list; + entity->rq_list =kcalloc (num_rq_list,sizeof (struct drm_sched_rq *), +GFP_KERNEL ); +for (i =0 ; i < num_rq_list; ++i) + entity->rq_list[i] = rq_list[i]; entity->last_scheduled =NULL ;spin_lock_init (&entity->rq_lock);@@ -363,6 +370,7 @@ void drm_sched_entity_fini(struct drm_sched_entity *entity)dma_fence_put (entity->last_scheduled); entity->last_scheduled =NULL ; +kfree (entity->rq_list); }EXPORT_SYMBOL (drm_sched_entity_fini);include/drm/gpu_scheduler.h +4
-1@@ -50,7 +50,10 @@ enum drm_sched_priority {* * @list: used to append this struct to the list of entities in the * runqueue. -* @rq: runqueue to which this entity belongs. +* @rq: runqueue on which this entity is currently scheduled. +* @rq_list: a list of run queues on which jobs from this entity can +* be scheduled +* @num_rq_list: number of run queues in the rq_list * @rq_lock: lock to modify the runqueue to which this entity belongs. * @job_queue: the list of jobs of this entity. * @fence_seq: a linearly increasing seqno incremented with each
include/drm/gpu_scheduler.h enum drm_sched_priority { · line 84
Quoted 8 lines
include/drm/gpu_scheduler.h +2
-0@@ -74,6 +77,8 @@ enum drm_sched_priority {struct drm_sched_entity {struct list_head list;struct drm_sched_rq *rq; +struct drm_sched_rq **rq_list; +unsigned int num_rq_list;spinlock_t rq_lock;struct spsc_queue job_queue;
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
PATCH 4/4 drm/scheduler: move idle entities to scheduler with less load v2
In reply to Nayan DeshmukhThis is the first attempt to move entities between schedulers to have dynamic load balancing. We just move entities with no jobs for now as moving the ones with jobs will lead to other compilcations like ensuring that the other scheduler does not remove a job from the current entity while we are moving.
v2: remove unused variable and an unecessary check
--- drivers/gpu/drm/scheduler/gpu_scheduler.c | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-)
drivers/gpu/drm/scheduler/gpu_scheduler.c +18 -4
@@ -539,6 +539,8 @@ drm_sched_entity_pop_job(struct drm_sched_entity *entity)if (!sched_job)return NULL ; + sched_job->sched = sched; + sched_job->s_fence->sched = sched;while ((entity->dependency = sched->ops->dependency (sched_job, entity)))if (drm_sched_entity_add_dependency_cb (entity))return NULL ;
@@ -569,11 +571,23 @@ drm_sched_entity_pop_job(struct drm_sched_entity *entity)void drm_sched_entity_push_job (struct drm_sched_job *sched_job,struct drm_sched_entity *entity) { -struct drm_gpu_scheduler *sched = sched_job->sched; -bool first = false; +struct drm_sched_rq *rq = entity->rq; +bool first = false, reschedule, idle; -trace_drm_sched_job (sched_job, entity); + idle = entity->last_scheduled ==NULL || +dma_fence_is_signaled (entity->last_scheduled); + first =spsc_queue_count (&entity->job_queue) ==0 ; + reschedule = idle && first && (entity->num_rq_list >1 ); +if (reschedule) { + rq =drm_sched_entity_get_free_sched (entity); +spin_lock (&entity->rq_lock); +drm_sched_rq_remove_entity (entity->rq, entity); + entity->rq = rq; +spin_unlock (&entity->rq_lock); + } + +trace_drm_sched_job (sched_job, entity);atomic_inc (&entity->rq->sched->num_jobs); first =spsc_queue_push (&entity->job_queue, &sched_job->queue_node);
@@ -588,7 +602,7 @@ void drm_sched_entity_push_job(struct drm_sched_job *sched_job, }drm_sched_rq_add_entity (entity->rq, entity);spin_unlock (&entity->rq_lock); -drm_sched_wakeup (sched); +drm_sched_wakeup (entity->rq->sched); } }EXPORT_SYMBOL (drm_sched_entity_push_job);
Signature · 6 lines
2.14.3 _______________________________________________ dri-devel mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/dri-devel