Skip to contentSkip to conversation

drm/sched: implement and export drm_sched_pick_best

PATCH Superseded Nirmoy Das 3 participants 6 messages Mar 13, 2020

Subsystems (4)

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore

Nirmoy Das

author
#0 lore raw

PATCH v2 1/2 drm/sched: implement and export drm_sched_pick_best

Remove drm_sched_entity_get_free_sched() and use the logic of picking the least loaded drm scheduler from a drm scheduler list to implement drm_sched_pick_best(). This patch also exports drm_sched_pick_best() so that it can be utilized by other drm drivers.

---
 drivers/gpu/drm/scheduler/sched_entity.c | 40 ++++--------------------
 drivers/gpu/drm/scheduler/sched_main.c   | 35 +++++++++++++++++++++
 include/drm/gpu_scheduler.h              |  3 ++
 3 files changed, 44 insertions(+), 34 deletions(-)
drivers/gpu/drm/scheduler/sched_entity.c +6 -34
index d631521a9679..ffc8adb6ff25 100644
@@ -137,38 +137,6 @@ bool drm_sched_entity_is_ready(struct drm_sched_entity *entity)
 	return true;
 }
 
-/**
- * drm_sched_entity_get_free_sched - Get the rq from rq_list with least load
- *
- * @entity: scheduler entity
- *
- * Return the pointer to the rq with least load.
- */
-static struct drm_sched_rq *
-drm_sched_entity_get_free_sched(struct drm_sched_entity *entity)
-{
-	struct drm_sched_rq *rq = NULL;
-	unsigned int min_jobs = UINT_MAX, num_jobs;
-	int i;
-
-	for (i = 0; i < entity->num_sched_list; ++i) {
-		struct drm_gpu_scheduler *sched = entity->sched_list[i];
-
-		if (!entity->sched_list[i]->ready) {
-			DRM_WARN("sched%s is not ready, skipping", sched->name);
-			continue;
-		}
-
-		num_jobs = atomic_read(&sched->num_jobs);
-		if (num_jobs < min_jobs) {
-			min_jobs = num_jobs;
-			rq = &entity->sched_list[i]->sched_rq[entity->priority];
-		}
-	}
-
-	return rq;
-}
-
 /**
  * drm_sched_entity_flush - Flush a context entity
  *
@@ -479,7 +447,8 @@ struct drm_sched_job *drm_sched_entity_pop_job(struct drm_sched_entity *entity)
 void drm_sched_entity_select_rq(struct drm_sched_entity *entity)
 {
 	struct dma_fence *fence;
-	struct drm_sched_rq *rq;
+	struct drm_gpu_scheduler *sched;
+	struct drm_sched_rq *rq = NULL;
 
 	if (spsc_queue_count(&entity->job_queue) || entity->num_sched_list <= 1)
 		return;
@@ -489,7 +458,10 @@ void drm_sched_entity_select_rq(struct drm_sched_entity *entity)
 		return;
 
 	spin_lock(&entity->rq_lock);
-	rq = drm_sched_entity_get_free_sched(entity);
+	sched = drm_sched_pick_best(entity->sched_list, entity->num_sched_list);
+	if (sched)
+		rq = &sched->sched_rq[entity->priority];
+
 	if (rq != entity->rq) {
 		drm_sched_rq_remove_entity(entity->rq, entity);
 		entity->rq = rq;
drivers/gpu/drm/scheduler/sched_main.c +35 -0
index 3fad5876a13f..d640f4087795 100644
@@ -705,6 +705,41 @@ drm_sched_get_cleanup_job(struct drm_gpu_scheduler *sched)
 	return job;
 }
 
+/**
+ * drm_sched_pick_best - Get a drm sched from a sched_list with the least load
+ * @sched_list: list of drm_gpu_schedulers
+ * @num_sched_list: number of drm_gpu_schedulers in the sched_list
+ *
+ * Returns pointer of the sched with the least load or NULL if none of the
+ * drm_gpu_schedulers are ready
+ */
+struct drm_gpu_scheduler *
+drm_sched_pick_best(struct drm_gpu_scheduler **sched_list,
+		     unsigned int num_sched_list)
+{
+	struct drm_gpu_scheduler *sched, *picked_sched = NULL;
+	int i;
+	unsigned int min_jobs = UINT_MAX, num_jobs;
+
+	for (i = 0; i < num_sched_list; ++i) {
+		sched = sched_list[i];
+
+		if (!sched->ready) {
+			DRM_WARN("sched%s is not ready, skipping", sched->name);
+			continue;
+		}
+
+		num_jobs = atomic_read(&sched->num_jobs);
+		if (num_jobs < min_jobs) {
+			min_jobs = num_jobs;
+			picked_sched = sched;
+		}
+	}
+
+	return picked_sched;
+}
+EXPORT_SYMBOL(drm_sched_pick_best);
+
 /**
  * drm_sched_blocked - check if the scheduler is blocked
  *
include/drm/gpu_scheduler.h +3 -0
index ae39eacee250..ca6b8b01fac9 100644
@@ -341,5 +341,8 @@ void drm_sched_fence_finished(struct drm_sched_fence *fence);
 unsigned long drm_sched_suspend_timeout(struct drm_gpu_scheduler *sched);
 void drm_sched_resume_timeout(struct drm_gpu_scheduler *sched,
 		                unsigned long remaining);
+struct drm_gpu_scheduler *
+drm_sched_pick_best(struct drm_gpu_scheduler **sched_list,
+		     unsigned int num_sched_list);
 
 #endif

--
2.25.0

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

author
#1 lore raw

PATCH 2/2 drm/amdgpu: disable gpu_sched load balancer for vcn jobs

VCN HW doesn't support dynamic load balance on multiple instances for a context. This patch initializes VNC entities with only one drm_gpu_scheduler picked by drm_sched_pick_best(). Picking a drm_gpu_scheduler using drm_sched_pick_best() ensures that we do load balance among multiple contexts but not among multiple jobs in a context.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c +6 -1
index fa575bdc03c8..8f1e3c05812e 100644
@@ -121,12 +121,17 @@ static int amdgpu_ctx_init_entity(struct amdgpu_ctx *ctx, const u32 hw_ip, const
 			num_scheds = 1;
 			break;
 		case AMDGPU_HW_IP_VCN_DEC:
-			scheds = adev->vcn.vcn_dec_sched;
 			num_scheds =  adev->vcn.num_vcn_dec_sched;
+			sched = drm_sched_pick_best(adev->vcn.vcn_dec_sched,
+						    num_scheds);
+			scheds = &sched;
 			break;
 		case AMDGPU_HW_IP_VCN_ENC:
 			scheds = adev->vcn.vcn_enc_sched;
 			num_scheds =  adev->vcn.num_vcn_enc_sched;
+			sched = drm_sched_pick_best(adev->vcn.vcn_enc_sched,
+						    num_scheds);
+			scheds = &sched;
 			break;
 		case AMDGPU_HW_IP_VCN_JPEG:
 			scheds = adev->jpeg.jpeg_sched;
Signature · 6 lines
2.25.0

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
maintainer
#2 lore raw

Re: [PATCH 2/2] drm/amdgpu: disable gpu_sched load balancer for vcn jobs

Am 13.03.20 um 14:59 schrieb Nirmoy Das:

Quoted 22 lines

VCN HW doesn't support dynamic load balance on multiple instances for a context. This patch initializes VNC entities with only one drm_gpu_scheduler picked by drm_sched_pick_best(). Picking a drm_gpu_scheduler using drm_sched_pick_best() ensures that we do load balance among multiple contexts but not among multiple jobs in a context.

---
  drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c | 7 ++++++-
  1 file changed, 6 insertions(+), 1 deletion(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c +0 -1
index fa575bdc03c8..8f1e3c05812e 100644
@@ -121,12 +121,17 @@ static int amdgpu_ctx_init_entity(struct amdgpu_ctx *ctx, const u32 hw_ip, const
  			num_scheds = 1;
  			break;
  		case AMDGPU_HW_IP_VCN_DEC:
-			scheds = adev->vcn.vcn_dec_sched;
  			num_scheds =  adev->vcn.num_vcn_dec_sched;

drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c static int amdgpu_ctx_init_entity(struct amdgpu_ctx *ctx, const u32 hw_ip, const · line 134

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c +6 -0
+			sched = drm_sched_pick_best(adev->vcn.vcn_dec_sched,
+						    num_scheds);
+			scheds = &sched;
  			break;
  		case AMDGPU_HW_IP_VCN_ENC:
  			scheds = adev->vcn.vcn_enc_sched;
  			num_scheds =  adev->vcn.num_vcn_enc_sched;
+			sched = drm_sched_pick_best(adev->vcn.vcn_enc_sched,
+						    num_scheds);
+			scheds = &sched;

num_scheds need to be set to 1 here and above.

And maybe add a separate patch to fix the indentation of the case labels.

Christian.

drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c static int amdgpu_ctx_init_entity(struct amdgpu_ctx *ctx, const u32 hw_ip, const · line 137

Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c +0 -0
  			break;
  		case AMDGPU_HW_IP_VCN_JPEG:
  			scheds = adev->jpeg.jpeg_sched;

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#3 lore raw

Re: [PATCH 2/2] drm/amdgpu: disable gpu_sched load balancer for vcn jobs

Quoted 28 lines · Christian König

Am 13.03.20 um 14:59 schrieb Nirmoy Das:

Quoted 27 lines

VCN HW doesn't support dynamic load balance on multiple instances for a context. This patch initializes VNC entities with only one drm_gpu_scheduler picked by drm_sched_pick_best(). Picking a drm_gpu_scheduler using drm_sched_pick_best() ensures that we do load balance among multiple contexts but not among multiple jobs in a context.

---
  drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c | 7 ++++++-

  1 file changed, 6 insertions(+), 1 deletion(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c +0 -0
index fa575bdc03c8..8f1e3c05812e 100644
@@ -121,12 +121,17 @@ static int amdgpu_ctx_init_entity(struct 

amdgpu_ctx *ctx, const u32 hw_ip, const
              num_scheds = 1;
              break;
          case AMDGPU_HW_IP_VCN_DEC:
-            scheds = adev->vcn.vcn_dec_sched; num_scheds =  adev->vcn.num_vcn_dec_sched; +            sched = drm_sched_pick_best(adev->vcn.vcn_dec_sched, +                            num_scheds);
+            scheds = &sched;
              break;

Quoted 8 lines
Quoted 6 lines

          case AMDGPU_HW_IP_VCN_ENC:
              scheds = adev->vcn.vcn_enc_sched; num_scheds =  adev->vcn.num_vcn_enc_sched; +            sched = drm_sched_pick_best(adev->vcn.vcn_enc_sched, +                            num_scheds);
+            scheds = &sched;

num_scheds need to be set to 1 here and above.

Right or else it would've caused NULL exception.

Thanks,

Nirmoy

Quoted 9 lines

And maybe add a separate patch to fix the indentation of the case labels.

Christian.

Quoted 3 lines

              break;
          case AMDGPU_HW_IP_VCN_JPEG:
              scheds = adev->jpeg.jpeg_sched;

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#4 lore raw
In reply to Nirmoy Das

ah please ignore this I forgot to change the "sched" -> "scheduler"

Quoted 137 lines · Nirmoy Das

Remove drm_sched_entity_get_free_sched() and use the logic of picking the least loaded drm scheduler from a drm scheduler list to implement drm_sched_pick_best(). This patch also exports drm_sched_pick_best() so that it can be utilized by other drm drivers.

---
  drivers/gpu/drm/scheduler/sched_entity.c | 40 ++++--------------------
  drivers/gpu/drm/scheduler/sched_main.c   | 35 +++++++++++++++++++++
  include/drm/gpu_scheduler.h              |  3 ++
  3 files changed, 44 insertions(+), 34 deletions(-)
drivers/gpu/drm/scheduler/sched_entity.c +6 -34
index d631521a9679..ffc8adb6ff25 100644
@@ -137,38 +137,6 @@ bool drm_sched_entity_is_ready(struct drm_sched_entity *entity)
  	return true;
  }
 
-/**
- * drm_sched_entity_get_free_sched - Get the rq from rq_list with least load
- *
- * @entity: scheduler entity
- *
- * Return the pointer to the rq with least load.
- */
-static struct drm_sched_rq *
-drm_sched_entity_get_free_sched(struct drm_sched_entity *entity)
-{
-	struct drm_sched_rq *rq = NULL;
-	unsigned int min_jobs = UINT_MAX, num_jobs;
-	int i;
-
-	for (i = 0; i < entity->num_sched_list; ++i) {
-		struct drm_gpu_scheduler *sched = entity->sched_list[i];
-
-		if (!entity->sched_list[i]->ready) {
-			DRM_WARN("sched%s is not ready, skipping", sched->name);
-			continue;
-		}
-
-		num_jobs = atomic_read(&sched->num_jobs);
-		if (num_jobs < min_jobs) {
-			min_jobs = num_jobs;
-			rq = &entity->sched_list[i]->sched_rq[entity->priority];
-		}
-	}
-
-	return rq;
-}
-
  /**
   * drm_sched_entity_flush - Flush a context entity
   *
@@ -479,7 +447,8 @@ struct drm_sched_job *drm_sched_entity_pop_job(struct drm_sched_entity *entity)
  void drm_sched_entity_select_rq(struct drm_sched_entity *entity)
  {
  	struct dma_fence *fence;
-	struct drm_sched_rq *rq;
+	struct drm_gpu_scheduler *sched;
+	struct drm_sched_rq *rq = NULL;
 
  	if (spsc_queue_count(&entity->job_queue) || entity->num_sched_list <= 1)
  		return;
@@ -489,7 +458,10 @@ void drm_sched_entity_select_rq(struct drm_sched_entity *entity)
  		return;
 
  	spin_lock(&entity->rq_lock);
-	rq = drm_sched_entity_get_free_sched(entity);
+	sched = drm_sched_pick_best(entity->sched_list, entity->num_sched_list);
+	if (sched)
+		rq = &sched->sched_rq[entity->priority];
+
  	if (rq != entity->rq) {
  		drm_sched_rq_remove_entity(entity->rq, entity);
  		entity->rq = rq;
drivers/gpu/drm/scheduler/sched_main.c +35 -0
index 3fad5876a13f..d640f4087795 100644
@@ -705,6 +705,41 @@ drm_sched_get_cleanup_job(struct drm_gpu_scheduler *sched)
  	return job;
  }
 
+/**
+ * drm_sched_pick_best - Get a drm sched from a sched_list with the least load
+ * @sched_list: list of drm_gpu_schedulers
+ * @num_sched_list: number of drm_gpu_schedulers in the sched_list
+ *
+ * Returns pointer of the sched with the least load or NULL if none of the
+ * drm_gpu_schedulers are ready
+ */
+struct drm_gpu_scheduler *
+drm_sched_pick_best(struct drm_gpu_scheduler **sched_list,
+		     unsigned int num_sched_list)
+{
+	struct drm_gpu_scheduler *sched, *picked_sched = NULL;
+	int i;
+	unsigned int min_jobs = UINT_MAX, num_jobs;
+
+	for (i = 0; i < num_sched_list; ++i) {
+		sched = sched_list[i];
+
+		if (!sched->ready) {
+			DRM_WARN("sched%s is not ready, skipping", sched->name);
+			continue;
+		}
+
+		num_jobs = atomic_read(&sched->num_jobs);
+		if (num_jobs < min_jobs) {
+			min_jobs = num_jobs;
+			picked_sched = sched;
+		}
+	}
+
+	return picked_sched;
+}
+EXPORT_SYMBOL(drm_sched_pick_best);
+
  /**
   * drm_sched_blocked - check if the scheduler is blocked
   *
include/drm/gpu_scheduler.h +3 -1
index ae39eacee250..ca6b8b01fac9 100644
@@ -341,5 +341,8 @@ void drm_sched_fence_finished(struct drm_sched_fence *fence);
  unsigned long drm_sched_suspend_timeout(struct drm_gpu_scheduler *sched);
  void drm_sched_resume_timeout(struct drm_gpu_scheduler *sched,
  		                unsigned long remaining);
+struct drm_gpu_scheduler *
+drm_sched_pick_best(struct drm_gpu_scheduler **sched_list,
+		     unsigned int num_sched_list);
 
  #endif
--
Quoted 2 lines

2.25.0

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

maintainer
#5 lore raw
In reply to Nirmoy Das

Am 13.03.20 um 14:59 schrieb Nirmoy Das:

Quoted 66 lines

Remove drm_sched_entity_get_free_sched() and use the logic of picking the least loaded drm scheduler from a drm scheduler list to implement drm_sched_pick_best(). This patch also exports drm_sched_pick_best() so that it can be utilized by other drm drivers.

---
  drivers/gpu/drm/scheduler/sched_entity.c | 40 ++++--------------------
  drivers/gpu/drm/scheduler/sched_main.c   | 35 +++++++++++++++++++++
  include/drm/gpu_scheduler.h              |  3 ++
  3 files changed, 44 insertions(+), 34 deletions(-)
drivers/gpu/drm/scheduler/sched_entity.c +2 -33
index d631521a9679..ffc8adb6ff25 100644
@@ -137,38 +137,6 @@ bool drm_sched_entity_is_ready(struct drm_sched_entity *entity)
  	return true;
  }
 
-/**
- * drm_sched_entity_get_free_sched - Get the rq from rq_list with least load
- *
- * @entity: scheduler entity
- *
- * Return the pointer to the rq with least load.
- */
-static struct drm_sched_rq *
-drm_sched_entity_get_free_sched(struct drm_sched_entity *entity)
-{
-	struct drm_sched_rq *rq = NULL;
-	unsigned int min_jobs = UINT_MAX, num_jobs;
-	int i;
-
-	for (i = 0; i < entity->num_sched_list; ++i) {
-		struct drm_gpu_scheduler *sched = entity->sched_list[i];
-
-		if (!entity->sched_list[i]->ready) {
-			DRM_WARN("sched%s is not ready, skipping", sched->name);
-			continue;
-		}
-
-		num_jobs = atomic_read(&sched->num_jobs);
-		if (num_jobs < min_jobs) {
-			min_jobs = num_jobs;
-			rq = &entity->sched_list[i]->sched_rq[entity->priority];
-		}
-	}
-
-	return rq;
-}
-
  /**
   * drm_sched_entity_flush - Flush a context entity
   *
@@ -479,7 +447,8 @@ struct drm_sched_job *drm_sched_entity_pop_job(struct drm_sched_entity *entity)
  void drm_sched_entity_select_rq(struct drm_sched_entity *entity)
  {
  	struct dma_fence *fence;
-	struct drm_sched_rq *rq;
+	struct drm_gpu_scheduler *sched;
+	struct drm_sched_rq *rq = NULL;
 
  	if (spsc_queue_count(&entity->job_queue) || entity->num_sched_list <= 1)
  		return;

drivers/gpu/drm/scheduler/sched_entity.c void drm_sched_entity_select_rq(struct drm_sched_entity *entity) · line 463

Quoted 7 lines
drivers/gpu/drm/scheduler/sched_entity.c +3 -1
@@ -489,7 +458,10 @@ void drm_sched_entity_select_rq(struct drm_sched_entity *entity)
  		return;
 
  	spin_lock(&entity->rq_lock);
-	rq = drm_sched_entity_get_free_sched(entity);
+	sched = drm_sched_pick_best(entity->sched_list, entity->num_sched_list);
+	if (sched)
+		rq = &sched->sched_rq[entity->priority];

Maybe write this as "rq = sched ? &sched->sched_rq[entity->priority] : NULL" and drop the NULL initialization above.

Apart from that looks good to me,
Christian.

Quoted 64 lines
drivers/gpu/drm/scheduler/sched_entity.c +1 -0
+
  	if (rq != entity->rq) {
  		drm_sched_rq_remove_entity(entity->rq, entity);
  		entity->rq = rq;
drivers/gpu/drm/scheduler/sched_main.c +35 -0
index 3fad5876a13f..d640f4087795 100644
@@ -705,6 +705,41 @@ drm_sched_get_cleanup_job(struct drm_gpu_scheduler *sched)
  	return job;
  }
 
+/**
+ * drm_sched_pick_best - Get a drm sched from a sched_list with the least load
+ * @sched_list: list of drm_gpu_schedulers
+ * @num_sched_list: number of drm_gpu_schedulers in the sched_list
+ *
+ * Returns pointer of the sched with the least load or NULL if none of the
+ * drm_gpu_schedulers are ready
+ */
+struct drm_gpu_scheduler *
+drm_sched_pick_best(struct drm_gpu_scheduler **sched_list,
+		     unsigned int num_sched_list)
+{
+	struct drm_gpu_scheduler *sched, *picked_sched = NULL;
+	int i;
+	unsigned int min_jobs = UINT_MAX, num_jobs;
+
+	for (i = 0; i < num_sched_list; ++i) {
+		sched = sched_list[i];
+
+		if (!sched->ready) {
+			DRM_WARN("sched%s is not ready, skipping", sched->name);
+			continue;
+		}
+
+		num_jobs = atomic_read(&sched->num_jobs);
+		if (num_jobs < min_jobs) {
+			min_jobs = num_jobs;
+			picked_sched = sched;
+		}
+	}
+
+	return picked_sched;
+}
+EXPORT_SYMBOL(drm_sched_pick_best);
+
  /**
   * drm_sched_blocked - check if the scheduler is blocked
   *
include/drm/gpu_scheduler.h +3 -1
index ae39eacee250..ca6b8b01fac9 100644
@@ -341,5 +341,8 @@ void drm_sched_fence_finished(struct drm_sched_fence *fence);
  unsigned long drm_sched_suspend_timeout(struct drm_gpu_scheduler *sched);
  void drm_sched_resume_timeout(struct drm_gpu_scheduler *sched,
  		                unsigned long remaining);
+struct drm_gpu_scheduler *
+drm_sched_pick_best(struct drm_gpu_scheduler **sched_list,
+		     unsigned int num_sched_list);
 
  #endif
--
Quoted 2 lines

2.25.0

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx