Skip to contentSkip to conversation

drm/scheduler: add a pointer to scheduler in the rq

PATCH In review Nayan Deshmukh 3 participants 4 messages Jul 13, 2018 2 messages not in the archive

Subsystems (6)
lore
Share & tools
lore
#0 lore raw

PATCH v2 1/2 drm/scheduler: add a pointer to scheduler in the rq

This patch is in preparation for a better load balancing in scheduler. It allows us to associate entities with the run queues instead of binding them to a scheduler.

---
 drivers/gpu/drm/scheduler/gpu_scheduler.c | 6 ++++--
 include/drm/gpu_scheduler.h               | 2 ++
 2 files changed, 6 insertions(+), 2 deletions(-)
drivers/gpu/drm/scheduler/gpu_scheduler.c +4 -2
index 7d2560699b84..429b1328653a 100644
@@ -69,11 +69,13 @@ static void drm_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb);
  *
  * Initializes a scheduler runqueue.
  */
-static void drm_sched_rq_init(struct drm_sched_rq *rq)
+static void drm_sched_rq_init(struct drm_gpu_scheduler *sched,
+			      struct drm_sched_rq *rq)
 {
 	spin_lock_init(&rq->lock);
 	INIT_LIST_HEAD(&rq->entities);
 	rq->current_entity = NULL;
+	rq->sched = sched;
 }
 
 /**
@@ -926,7 +928,7 @@ int drm_sched_init(struct drm_gpu_scheduler *sched,
 	sched->timeout = timeout;
 	sched->hang_limit = hang_limit;
 	for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_MAX; i++)
-		drm_sched_rq_init(&sched->sched_rq[i]);
+		drm_sched_rq_init(sched, &sched->sched_rq[i]);
 
 	init_waitqueue_head(&sched->wake_up_worker);
 	init_waitqueue_head(&sched->job_scheduled);
include/drm/gpu_scheduler.h +2 -0
index 4214ceb71c05..43e93d6077cf 100644
@@ -93,6 +93,7 @@ struct drm_sched_entity {
  * struct drm_sched_rq - queue of entities to be scheduled.
  *
  * @lock: to modify the entities list.
+ * @sched: the scheduler to which this rq belongs to.
  * @entities: list of the entities to be scheduled.
  * @current_entity: the entity which is to be scheduled.
  *
@@ -102,6 +103,7 @@ struct drm_sched_entity {
  */
 struct drm_sched_rq {
 	spinlock_t			lock;
+	struct drm_gpu_scheduler	*sched;
 	struct list_head		entities;
 	struct drm_sched_entity		*current_entity;
 };
Signature · 6 lines
2.14.3

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
#1 lore raw

PATCH v2 2/2 drm/scheduler: modify args of drm_sched_entity_init

replace run queue by a list of run queues and remove the sched arg as that is part of run queue itself

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c   |  4 ++--
 drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c   |  3 +--
 drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c   |  4 ++--
 drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c   |  3 +--
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c    |  3 +--
 drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c     |  4 ++--
 drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c     |  4 ++--
 drivers/gpu/drm/etnaviv/etnaviv_drv.c     |  8 ++++----
 drivers/gpu/drm/scheduler/gpu_scheduler.c | 20 ++++++++++++--------
 drivers/gpu/drm/v3d/v3d_drv.c             |  7 +++----
 include/drm/gpu_scheduler.h               |  6 +++---
 11 files changed, 33 insertions(+), 33 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c +2 -2
index 0120b24fae1b..83e3b320a793 100644
@@ -90,8 +90,8 @@ static int amdgpu_ctx_init(struct amdgpu_device *adev,
 		if (ring == &adev->gfx.kiq.ring)
 			continue;
 
-		r = drm_sched_entity_init(&ring->sched, &ctx->rings[i].entity,
-					  rq, &ctx->guilty);
+		r = drm_sched_entity_init(&ctx->rings[i].entity,
+					  &rq, 1, &ctx->guilty);
 		if (r)
 			goto failed;
 	}
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +1 -2
index 0246cb87d9e4..360d6b82c3c7 100644
@@ -140,8 +140,7 @@ static int amdgpu_ttm_global_init(struct amdgpu_device *adev)
 
 	ring = adev->mman.buffer_funcs_ring;
 	rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_KERNEL];
-	r = drm_sched_entity_init(&ring->sched, &adev->mman.entity,
-				  rq, NULL);
+	r = drm_sched_entity_init(&adev->mman.entity, &rq, 1, NULL);
 	if (r) {
 		DRM_ERROR("Failed setting up TTM BO move run queue.\n");
 		goto error_entity;
drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c +2 -2
index 0b46ea1c6290..0d8c57fc3d09 100644
@@ -266,8 +266,8 @@ int amdgpu_uvd_sw_init(struct amdgpu_device *adev)
 
 		ring = &adev->uvd.inst[j].ring;
 		rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL];
-		r = drm_sched_entity_init(&ring->sched, &adev->uvd.inst[j].entity,
-					  rq, NULL);
+		r = drm_sched_entity_init(&adev->uvd.inst[j].entity, &rq,
+					  1, NULL);
 		if (r != 0) {
 			DRM_ERROR("Failed setting up UVD(%d) run queue.\n", j);
 			return r;
drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c +1 -2
index b0dcdfd85f5b..c0790e4c5454 100644
@@ -190,8 +190,7 @@ int amdgpu_vce_sw_init(struct amdgpu_device *adev, unsigned long size)
 
 	ring = &adev->vce.ring[0];
 	rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL];
-	r = drm_sched_entity_init(&ring->sched, &adev->vce.entity,
-				  rq, NULL);
+	r = drm_sched_entity_init(&adev->vce.entity, &rq, 1, NULL);
 	if (r != 0) {
 		DRM_ERROR("Failed setting up VCE run queue.\n");
 		return r;
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1 -2
index 712af5c1a5d6..fb7da0740ad8 100644
@@ -2562,8 +2562,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	ring_instance %= adev->vm_manager.vm_pte_num_rings;
 	ring = adev->vm_manager.vm_pte_rings[ring_instance];
 	rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_KERNEL];
-	r = drm_sched_entity_init(&ring->sched, &vm->entity,
-				  rq, NULL);
+	r = drm_sched_entity_init(&vm->entity, &rq, 1, NULL);
 	if (r)
 		return r;
 
drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c +2 -2
index 8ee1c2eaaa14..da38aa2f3d13 100644
@@ -429,8 +429,8 @@ static int uvd_v6_0_sw_init(void *handle)
 		struct drm_sched_rq *rq;
 		ring = &adev->uvd.inst->ring_enc[0];
 		rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL];
-		r = drm_sched_entity_init(&ring->sched, &adev->uvd.inst->entity_enc,
-					  rq, NULL);
+		r = drm_sched_entity_init(&adev->uvd.inst->entity_enc,
+					  &rq, 1, NULL);
 		if (r) {
 			DRM_ERROR("Failed setting up UVD ENC run queue.\n");
 			return r;
drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c +2 -2
index ba244d3b74db..69221430aa38 100644
@@ -431,8 +431,8 @@ static int uvd_v7_0_sw_init(void *handle)
 	for (j = 0; j < adev->uvd.num_uvd_inst; j++) {
 		ring = &adev->uvd.inst[j].ring_enc[0];
 		rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL];
-		r = drm_sched_entity_init(&ring->sched, &adev->uvd.inst[j].entity_enc,
-					  rq, NULL);
+		r = drm_sched_entity_init(&adev->uvd.inst[j].entity_enc,
+					  &rq, 1, NULL);
 		if (r) {
 			DRM_ERROR("(%d)Failed setting up UVD ENC run queue.\n", j);
 			return r;
drivers/gpu/drm/etnaviv/etnaviv_drv.c +4 -4
index 45bfdf4cc107..36414ba56b22 100644
@@ -49,12 +49,12 @@ static int etnaviv_open(struct drm_device *dev, struct drm_file *file)
 
 	for (i = 0; i < ETNA_MAX_PIPES; i++) {
 		struct etnaviv_gpu *gpu = priv->gpu[i];
+		struct drm_sched_rq *rq;
 
 		if (gpu) {
-			drm_sched_entity_init(&gpu->sched,
-				&ctx->sched_entity[i],
-				&gpu->sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL],
-				NULL);
+			rq = &gpu->sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL];
+			drm_sched_entity_init(&ctx->sched_entity[i],
+					      &rq, 1, NULL);
 			}
 	}
 
drivers/gpu/drm/scheduler/gpu_scheduler.c +12 -8
index 429b1328653a..16bf446aa6b3 100644
@@ -162,26 +162,30 @@ drm_sched_rq_select_entity(struct drm_sched_rq *rq)
  * drm_sched_entity_init - Init a context entity used by scheduler when
  * submit to HW ring.
  *
- * @sched: scheduler instance
  * @entity: scheduler entity to init
- * @rq: the run queue this entity belongs
+ * @rq_list: the list of run queue on which jobs from this
+ *           entity can be submitted
+ * @num_rq_list: number of run queue in rq_list
  * @guilty: atomic_t set to 1 when a job on this queue
  *          is found to be guilty causing a timeout
  *
+ * Note: the rq_list should have atleast one element to schedule
+ *       the entity
+ *
  * Returns 0 on success or a negative error code on failure.
 */
-int drm_sched_entity_init(struct drm_gpu_scheduler *sched,
-			  struct drm_sched_entity *entity,
-			  struct drm_sched_rq *rq,
+int drm_sched_entity_init(struct drm_sched_entity *entity,
+			  struct drm_sched_rq **rq_list,
+			  unsigned int num_rq_list,
 			  atomic_t *guilty)
 {
-	if (!(sched && entity && rq))
+	if (!(entity && rq_list && num_rq_list > 0 && rq_list[0]))
 		return -EINVAL;
 
 	memset(entity, 0, sizeof(struct drm_sched_entity));
 	INIT_LIST_HEAD(&entity->list);
-	entity->rq = rq;
-	entity->sched = sched;
+	entity->rq = rq_list[0];
+	entity->sched = rq_list[0]->sched;
 	entity->guilty = guilty;
 	entity->last_scheduled = NULL;
 
drivers/gpu/drm/v3d/v3d_drv.c +3 -4
index 567f7d46d912..1dceba2b42fd 100644
@@ -123,6 +123,7 @@ v3d_open(struct drm_device *dev, struct drm_file *file)
 {
 	struct v3d_dev *v3d = to_v3d_dev(dev);
 	struct v3d_file_priv *v3d_priv;
+	struct drm_sched_rq *rq;
 	int i;
 
 	v3d_priv = kzalloc(sizeof(*v3d_priv), GFP_KERNEL);
@@ -132,10 +133,8 @@ v3d_open(struct drm_device *dev, struct drm_file *file)
 	v3d_priv->v3d = v3d;
 
 	for (i = 0; i < V3D_MAX_QUEUES; i++) {
-		drm_sched_entity_init(&v3d->queue[i].sched,
-				      &v3d_priv->sched_entity[i],
-				      &v3d->queue[i].sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL],
-				      NULL);
+		rq = &v3d->queue[i].sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL];
+		drm_sched_entity_init(&v3d_priv->sched_entity[i], &rq, 1, NULL);
 	}
 
 	file->driver_priv = v3d_priv;
include/drm/gpu_scheduler.h +3 -3
index 43e93d6077cf..2205e89722f6 100644
@@ -282,9 +282,9 @@ int drm_sched_init(struct drm_gpu_scheduler *sched,
 		   const char *name);
 void drm_sched_fini(struct drm_gpu_scheduler *sched);
 
-int drm_sched_entity_init(struct drm_gpu_scheduler *sched,
-			  struct drm_sched_entity *entity,
-			  struct drm_sched_rq *rq,
+int drm_sched_entity_init(struct drm_sched_entity *entity,
+			  struct drm_sched_rq **rq_list,
+			  unsigned int num_rq_list,
 			  atomic_t *guilty);
 long drm_sched_entity_flush(struct drm_gpu_scheduler *sched,
 			   struct drm_sched_entity *entity, long timeout);
Signature · 6 lines
2.14.3

_______________________________________________
dri-devel mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/dri-devel

A message that is not in the archive (sent off-list or not yet received): 20180713095114.29211-1-nayan26deshmukh-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org

author Reviewed-by Christian König Acked-by Eric Anholt
#3 lore raw

I forgot to add these:

Quoted 6 lines · Nayan Deshmukh

This patch is in preparation for a better load balancing in scheduler. It allows us to associate entities with the run queues instead of binding them to a scheduler.

Quoted 54 lines
---
 drivers/gpu/drm/scheduler/gpu_scheduler.c | 6 ++++--
 include/drm/gpu_scheduler.h               | 2 ++
 2 files changed, 6 insertions(+), 2 deletions(-)
drivers/gpu/drm/scheduler/gpu_scheduler.c +4 -2
index 7d2560699b84..429b1328653a 100644
@@ -69,11 +69,13 @@ static void drm_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb);
  *
  * Initializes a scheduler runqueue.
  */
-static void drm_sched_rq_init(struct drm_sched_rq *rq)
+static void drm_sched_rq_init(struct drm_gpu_scheduler *sched,
+                             struct drm_sched_rq *rq)
 {
        spin_lock_init(&rq->lock);
        INIT_LIST_HEAD(&rq->entities);
        rq->current_entity = NULL;
+       rq->sched = sched;
 }
 
 /**
@@ -926,7 +928,7 @@ int drm_sched_init(struct drm_gpu_scheduler *sched,
        sched->timeout = timeout;
        sched->hang_limit = hang_limit;
        for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_MAX; i++)
-               drm_sched_rq_init(&sched->sched_rq[i]);
+               drm_sched_rq_init(sched, &sched->sched_rq[i]);
 
        init_waitqueue_head(&sched->wake_up_worker);
        init_waitqueue_head(&sched->job_scheduled);
include/drm/gpu_scheduler.h +2 -1
index 4214ceb71c05..43e93d6077cf 100644
@@ -93,6 +93,7 @@ struct drm_sched_entity {
  * struct drm_sched_rq - queue of entities to be scheduled.
  *
  * @lock: to modify the entities list.
+ * @sched: the scheduler to which this rq belongs to.
  * @entities: list of the entities to be scheduled.
  * @current_entity: the entity which is to be scheduled.
  *
@@ -102,6 +103,7 @@ struct drm_sched_entity {
  */
 struct drm_sched_rq {
        spinlock_t                      lock;
+       struct drm_gpu_scheduler        *sched;
        struct list_head                entities;
        struct drm_sched_entity         *current_entity;
 };
--
Quoted 2 lines

2.14.3

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): CAFd4ddwQuSTdRz-ya145kNyXA84JGWjydRo11v=yvzTc6VW58w-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org

#5 lore raw

Am 13.07.2018 um 15:40 schrieb Nayan Deshmukh:

Quoted 1 line

I forgot to add these:

Not a problem at all. It is actually my job to do this while committing the patches.

BTW: I've pushed you patches to amd-staging-drm-next just a few minutes ago. Thanks for the help.

I'm preparing quite a cleanup for amdgpu which should make your live easier, but still no idea why the scheduler hangs when we try to load balance jobs.

Regards,
Christian.

Quoted 58 lines
Quoted 5 lines · Nayan Deshmukh

This patch is in preparation for a better load balancing in scheduler. It allows us to associate entities with the run queues instead of binding them to a scheduler.

Quoted 48 lines
---
  drivers/gpu/drm/scheduler/gpu_scheduler.c | 6 ++++--
  include/drm/gpu_scheduler.h               | 2 ++
  2 files changed, 6 insertions(+), 2 deletions(-)
drivers/gpu/drm/scheduler/gpu_scheduler.c +4 -2
index 7d2560699b84..429b1328653a 100644
@@ -69,11 +69,13 @@ static void drm_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb);
   *
   * Initializes a scheduler runqueue.
   */
-static void drm_sched_rq_init(struct drm_sched_rq *rq)
+static void drm_sched_rq_init(struct drm_gpu_scheduler *sched,
+                             struct drm_sched_rq *rq)
  {
         spin_lock_init(&rq->lock);
         INIT_LIST_HEAD(&rq->entities);
         rq->current_entity = NULL;
+       rq->sched = sched;
  }
 
  /**
@@ -926,7 +928,7 @@ int drm_sched_init(struct drm_gpu_scheduler *sched,
         sched->timeout = timeout;
         sched->hang_limit = hang_limit;
         for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_MAX; i++)
-               drm_sched_rq_init(&sched->sched_rq[i]);
+               drm_sched_rq_init(sched, &sched->sched_rq[i]);
 
         init_waitqueue_head(&sched->wake_up_worker);
         init_waitqueue_head(&sched->job_scheduled);
include/drm/gpu_scheduler.h +1 -0
index 4214ceb71c05..43e93d6077cf 100644
@@ -93,6 +93,7 @@ struct drm_sched_entity {
   * struct drm_sched_rq - queue of entities to be scheduled.
   *
   * @lock: to modify the entities list.
+ * @sched: the scheduler to which this rq belongs to.
   * @entities: list of the entities to be scheduled.
   * @current_entity: the entity which is to be scheduled.
   *
@@ -102,6 +103,7 @@ struct drm_sched_entity {
   */
  struct drm_sched_rq {
Quoted 8 lines
Quoted 8 lines
include/drm/gpu_scheduler.h +1 -1
         spinlock_t                      lock;
+       struct drm_gpu_scheduler        *sched;
         struct list_head                entities;
         struct drm_sched_entity         *current_entity;
  };
--

2.14.3

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx