Skip to contentSkip to conversation

drm/amdgpu: Fix KFD oversubscription by tracking queues correctly

PATCH Superseded Jay Cornwall 3 participants 3 messages Jul 13, 2017 2 messages not in the archive

Subsystems (3)

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 drm/amdgpu: Fix KFD oversubscription by tracking queues correctly

The number of compute queues available to the KFD was erroneously calculated as 64. Only the first MEC can execute compute queues and it has 32 queue slots.

This caused the oversubscription limit to be calculated incorrectly, leading to a missing chained runlist command at the end of an oversubscribed runlist.

v2: Remove unused num_mec field to avoid duplicate logic

Change-Id Ic4a139c04b8a6d025fbb831a0a67e98728bfe461
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c            | 3 +--
 drivers/gpu/drm/amd/amdkfd/kfd_device.c               | 4 ----
 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c | 7 -------
 drivers/gpu/drm/amd/include/kgd_kfd_interface.h       | 3 ---
 drivers/gpu/drm/radeon/radeon_kfd.c                   | 1 -
 5 files changed, 1 insertion(+), 17 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +1 -2
index 7060daf..8c710f7 100644
@@ -116,7 +116,6 @@ void amdgpu_amdkfd_device_init(struct amdgpu_device *adev)
 	if (adev->kfd) {
 		struct kgd2kfd_shared_resources gpu_resources = {
 			.compute_vmid_bitmap = global_compute_vmid_bitmap,
-			.num_mec = adev->gfx.mec.num_mec,
 			.num_pipe_per_mec = adev->gfx.mec.num_pipe_per_mec,
 			.num_queue_per_pipe = adev->gfx.mec.num_queue_per_pipe,
 			.gpuvm_size = (uint64_t)amdgpu_vm_size << 30
@@ -140,7 +139,7 @@ void amdgpu_amdkfd_device_init(struct amdgpu_device *adev)
 		/* According to linux/bitmap.h we shouldn't use bitmap_clear if
 		 * nbits is not compile time constant
 		 */
-		last_valid_bit = adev->gfx.mec.num_mec
+		last_valid_bit = 1 /* only first MEC can have compute queues */
 				* adev->gfx.mec.num_pipe_per_mec
 				* adev->gfx.mec.num_queue_per_pipe;
 		for (i = last_valid_bit; i < KGD_MAX_QUEUES; ++i)
drivers/gpu/drm/amd/amdkfd/kfd_device.c +0 -4
index 1cf00d4..95f9396 100644
@@ -494,10 +494,6 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
 	} else
 		kfd->max_proc_per_quantum = hws_max_conc_proc;
 
-	/* We only use the first MEC */
-	if (kfd->shared_resources.num_mec > 1)
-		kfd->shared_resources.num_mec = 1;
-
 	/* calculate max size of mqds needed for queues */
 	size = max_num_of_queues_per_device *
 			kfd->device_info->mqd_size_aligned;
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +0 -7
index 7607989..306144f 100644
@@ -82,13 +82,6 @@ static bool is_pipe_enabled(struct device_queue_manager *dqm, int mec, int pipe)
 	return false;
 }
 
-unsigned int get_mec_num(struct device_queue_manager *dqm)
-{
-	BUG_ON(!dqm || !dqm->dev);
-
-	return dqm->dev->shared_resources.num_mec;
-}
-
 unsigned int get_queues_num(struct device_queue_manager *dqm)
 {
 	BUG_ON(!dqm || !dqm->dev);
drivers/gpu/drm/amd/include/kgd_kfd_interface.h +0 -3
index a4d2fee..10794b3 100644
@@ -107,9 +107,6 @@ struct kgd2kfd_shared_resources {
 	/* Bit n == 1 means VMID n is available for KFD. */
 	unsigned int compute_vmid_bitmap;
 
-	/* number of mec available from the hardware */
-	uint32_t num_mec;
-
 	/* number of pipes per mec */
 	uint32_t num_pipe_per_mec;
 
drivers/gpu/drm/radeon/radeon_kfd.c +0 -1
index 719ea51..8f8c7c1 100644
@@ -251,7 +251,6 @@ void radeon_kfd_device_init(struct radeon_device *rdev)
 	if (rdev->kfd) {
 		struct kgd2kfd_shared_resources gpu_resources = {
 			.compute_vmid_bitmap = 0xFF00,
-			.num_mec = 1,
 			.num_pipe_per_mec = 4,
 			.num_queue_per_pipe = 8,
 			.gpuvm_size = (uint64_t)radeon_vm_size << 30
Signature · 6 lines
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 1499977419-19065-1-git-send-email-Jay.Cornwall-5C7GfCeVMHo@public.gmane.org

Reviewed-by
#2 lore raw
Quoted 94 lines · Jay Cornwall

The number of compute queues available to the KFD was erroneously calculated as 64. Only the first MEC can execute compute queues and it has 32 queue slots.

This caused the oversubscription limit to be calculated incorrectly, leading to a missing chained runlist command at the end of an oversubscribed runlist.

v2: Remove unused num_mec field to avoid duplicate logic

Change-Id Ic4a139c04b8a6d025fbb831a0a67e98728bfe461
---
  drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c            | 3 +--
  drivers/gpu/drm/amd/amdkfd/kfd_device.c               | 4 ----
  drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c | 7 -------
  drivers/gpu/drm/amd/include/kgd_kfd_interface.h       | 3 ---
  drivers/gpu/drm/radeon/radeon_kfd.c                   | 1 -
  5 files changed, 1 insertion(+), 17 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +1 -2
index 7060daf..8c710f7 100644
@@ -116,7 +116,6 @@ void amdgpu_amdkfd_device_init(struct amdgpu_device *adev)
  	if (adev->kfd) {
  		struct kgd2kfd_shared_resources gpu_resources = {
  			.compute_vmid_bitmap = global_compute_vmid_bitmap,
-			.num_mec = adev->gfx.mec.num_mec,
  			.num_pipe_per_mec = adev->gfx.mec.num_pipe_per_mec,
  			.num_queue_per_pipe = adev->gfx.mec.num_queue_per_pipe,
  			.gpuvm_size = (uint64_t)amdgpu_vm_size << 30
@@ -140,7 +139,7 @@ void amdgpu_amdkfd_device_init(struct amdgpu_device *adev)
  		/* According to linux/bitmap.h we shouldn't use bitmap_clear if
  		 * nbits is not compile time constant
  		 */
-		last_valid_bit = adev->gfx.mec.num_mec
+		last_valid_bit = 1 /* only first MEC can have compute queues */
  				* adev->gfx.mec.num_pipe_per_mec
  				* adev->gfx.mec.num_queue_per_pipe;
  		for (i = last_valid_bit; i < KGD_MAX_QUEUES; ++i)
drivers/gpu/drm/amd/amdkfd/kfd_device.c +0 -4
index 1cf00d4..95f9396 100644
@@ -494,10 +494,6 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
  	} else
  		kfd->max_proc_per_quantum = hws_max_conc_proc;
 
-	/* We only use the first MEC */
-	if (kfd->shared_resources.num_mec > 1)
-		kfd->shared_resources.num_mec = 1;
-
  	/* calculate max size of mqds needed for queues */
  	size = max_num_of_queues_per_device *
  			kfd->device_info->mqd_size_aligned;
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +0 -7
index 7607989..306144f 100644
@@ -82,13 +82,6 @@ static bool is_pipe_enabled(struct device_queue_manager *dqm, int mec, int pipe)
  	return false;
  }
 
-unsigned int get_mec_num(struct device_queue_manager *dqm)
-{
-	BUG_ON(!dqm || !dqm->dev);
-
-	return dqm->dev->shared_resources.num_mec;
-}
-
  unsigned int get_queues_num(struct device_queue_manager *dqm)
  {
  	BUG_ON(!dqm || !dqm->dev);
drivers/gpu/drm/amd/include/kgd_kfd_interface.h +0 -3
index a4d2fee..10794b3 100644
@@ -107,9 +107,6 @@ struct kgd2kfd_shared_resources {
  	/* Bit n == 1 means VMID n is available for KFD. */
  	unsigned int compute_vmid_bitmap;
 
-	/* number of mec available from the hardware */
-	uint32_t num_mec;
-
  	/* number of pipes per mec */
  	uint32_t num_pipe_per_mec;
 
drivers/gpu/drm/radeon/radeon_kfd.c +0 -0
index 719ea51..8f8c7c1 100644

drivers/gpu/drm/radeon/radeon_kfd.c void radeon_kfd_device_init(struct radeon_device *rdev) · line 256

Quoted 7 lines
drivers/gpu/drm/radeon/radeon_kfd.c +0 -1
@@ -251,7 +251,6 @@ void radeon_kfd_device_init(struct radeon_device *rdev)
  	if (rdev->kfd) {
  		struct kgd2kfd_shared_resources gpu_resources = {
  			.compute_vmid_bitmap = 0xFF00,
-			.num_mec = 1,
  			.num_pipe_per_mec = 4,
  			.num_queue_per_pipe = 8,
  			.gpuvm_size = (uint64_t)radeon_vm_size << 30

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 65ff9929-33db-2691-78c5-0924ec7bcf3a-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org

#4 lore raw

People don't seem to like cross-component changes if they can be avoided. I'd prefer separate commits for KFD, radeon, amdgpu, and finally the kgd2kfd interface.

With that fixed, Reviewed-by: Felix Kuehling <[email protected]>

Quoted 107 lines · Andres Rodriguez
Quoted 103 lines · Jay Cornwall

The number of compute queues available to the KFD was erroneously calculated as 64. Only the first MEC can execute compute queues and it has 32 queue slots.

This caused the oversubscription limit to be calculated incorrectly, leading to a missing chained runlist command at the end of an oversubscribed runlist.

v2: Remove unused num_mec field to avoid duplicate logic

Change-Id Ic4a139c04b8a6d025fbb831a0a67e98728bfe461
---
  drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c            | 3 +--
  drivers/gpu/drm/amd/amdkfd/kfd_device.c               | 4 ----
  drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c | 7 -------
  drivers/gpu/drm/amd/include/kgd_kfd_interface.h       | 3 ---
  drivers/gpu/drm/radeon/radeon_kfd.c                   | 1 -
  5 files changed, 1 insertion(+), 17 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +0 -1
index 7060daf..8c710f7 100644
@@ -116,7 +116,6 @@ void amdgpu_amdkfd_device_init(struct
 amdgpu_device *adev)
      if (adev->kfd) {
          struct kgd2kfd_shared_resources gpu_resources = {
              .compute_vmid_bitmap = global_compute_vmid_bitmap,
-            .num_mec = adev->gfx.mec.num_mec,
              .num_pipe_per_mec = adev->gfx.mec.num_pipe_per_mec,
              .num_queue_per_pipe = adev->gfx.mec.num_queue_per_pipe,
              .gpuvm_size = (uint64_t)amdgpu_vm_size << 30
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +0 -0
@@ -140,7 +139,7 @@ void amdgpu_amdkfd_device_init(struct
 amdgpu_device *adev)
          /* According to linux/bitmap.h we shouldn't use

bitmap_clear if
* nbits is not compile time constant
*/

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +1 -1
-        last_valid_bit = adev->gfx.mec.num_mec
+        last_valid_bit = 1 /* only first MEC can have compute queues */
                  * adev->gfx.mec.num_pipe_per_mec
                  * adev->gfx.mec.num_queue_per_pipe;
          for (i = last_valid_bit; i < KGD_MAX_QUEUES; ++i)
drivers/gpu/drm/amd/amdkfd/kfd_device.c +0 -3
index 1cf00d4..95f9396 100644
@@ -494,10 +494,6 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
      } else
          kfd->max_proc_per_quantum = hws_max_conc_proc;
  -    /* We only use the first MEC */
-    if (kfd->shared_resources.num_mec > 1)
-        kfd->shared_resources.num_mec = 1;
-
      /* calculate max size of mqds needed for queues */
      size = max_num_of_queues_per_device *
              kfd->device_info->mqd_size_aligned;
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +0 -6
index 7607989..306144f 100644
@@ -82,13 +82,6 @@ static bool is_pipe_enabled(struct
 device_queue_manager *dqm, int mec, int pipe)
      return false;
  }
  -unsigned int get_mec_num(struct device_queue_manager *dqm)
-{
-    BUG_ON(!dqm || !dqm->dev);
-
-    return dqm->dev->shared_resources.num_mec;
-}
-
  unsigned int get_queues_num(struct device_queue_manager *dqm)
  {
      BUG_ON(!dqm || !dqm->dev);
drivers/gpu/drm/amd/include/kgd_kfd_interface.h +0 -2
index a4d2fee..10794b3 100644
@@ -107,9 +107,6 @@ struct kgd2kfd_shared_resources {
      /* Bit n == 1 means VMID n is available for KFD. */
      unsigned int compute_vmid_bitmap;
  -    /* number of mec available from the hardware */
-    uint32_t num_mec;
-
      /* number of pipes per mec */
      uint32_t num_pipe_per_mec;
  diff --git a/drivers/gpu/drm/radeon/radeon_kfd.c

b/drivers/gpu/drm/radeon/radeon_kfd.c
index 719ea51..8f8c7c1 100644

drivers/gpu/drm/radeon/radeon_kfd.c +0 -1
@@ -251,7 +251,6 @@ void radeon_kfd_device_init(struct radeon_device
 *rdev)
      if (rdev->kfd) {
          struct kgd2kfd_shared_resources gpu_resources = {
              .compute_vmid_bitmap = 0xFF00,
-            .num_mec = 1,
Quoted 8 lines
Quoted 4 lines
drivers/gpu/drm/radeon/radeon_kfd.c +0 -0
              .num_pipe_per_mec = 4,
              .num_queue_per_pipe = 8,
              .gpuvm_size = (uint64_t)radeon_vm_size << 30

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx