Skip to contentSkip to conversation

drm/amdgpu: Remove rounding from vram allocation path

PATCH In review Daniel Phillips 2 participants 2 messages Jul 25, 2022

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 drm/amdgpu: Remove rounding from vram allocation path

Rounding up allocations in the allocation path caused test regressions, so now just round in the availability path.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +5 -6
index 043a808c88a3..014a594899fb 100644
@@ -40,10 +40,10 @@
 #define AMDGPU_USERPTR_RESTORE_DELAY_MS 1
 
 /*
- * Align VRAM allocations to 2MB to avoid fragmentation caused by 4K allocations in the tail 2MB
+ * Align VRAM availability to 2MB to avoid fragmentation caused by 4K allocations in the tail 2MB
  * BO chunk
  */
-#define VRAM_ALLOCATION_ALIGN (1 << 21)
+#define VRAM_AVAILABLITY_ALIGN (1 << 21)
 
 /* Impose limit on how much memory KFD can use */
 static struct {
@@ -149,7 +149,7 @@ static int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,
 		 * to avoid fragmentation caused by 4K allocations in the tail
 		 * 2M BO chunk.
 		 */
-		vram_needed = ALIGN(size, VRAM_ALLOCATION_ALIGN);
+		vram_needed = size;
 	} else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
 		system_mem_needed = size;
 	} else if (!(alloc_flag &
@@ -198,7 +198,7 @@ static void unreserve_mem_limit(struct amdgpu_device *adev,
 		kfd_mem_limit.system_mem_used -= size;
 		kfd_mem_limit.ttm_mem_used -= size;
 	} else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
-		adev->kfd.vram_used -= ALIGN(size, VRAM_ALLOCATION_ALIGN);
+		adev->kfd.vram_used -= size;
 	} else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
 		kfd_mem_limit.system_mem_used -= size;
 	} else if (!(alloc_flag &
@@ -1642,7 +1642,6 @@ size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev)
 	uint64_t reserved_for_pt =
 		ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size);
 	size_t available;
-
 	spin_lock(&kfd_mem_limit.mem_limit_lock);
 	available = adev->gmc.real_vram_size
 		- adev->kfd.vram_used
@@ -1650,7 +1649,7 @@ size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev)
 		- reserved_for_pt;
 	spin_unlock(&kfd_mem_limit.mem_limit_lock);
 
-	return ALIGN_DOWN(available, VRAM_ALLOCATION_ALIGN);
+	return ALIGN_DOWN(available, VRAM_AVAILABLITY_ALIGN);
 }
 
 int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
Signature · 1 line
2.34.1
maintainer
#1 lore raw

Am 2022-07-25 um 13:14 schrieb Daniel Phillips:

Quoted 33 lines

Rounding up allocations in the allocation path caused test regressions, so now just round in the availability path.

---
  drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 11 +++++------
  1 file changed, 5 insertions(+), 6 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +3 -3
index 043a808c88a3..014a594899fb 100644
@@ -40,10 +40,10 @@
  #define AMDGPU_USERPTR_RESTORE_DELAY_MS 1
 
  /*
- * Align VRAM allocations to 2MB to avoid fragmentation caused by 4K allocations in the tail 2MB
+ * Align VRAM availability to 2MB to avoid fragmentation caused by 4K allocations in the tail 2MB
   * BO chunk
   */
-#define VRAM_ALLOCATION_ALIGN (1 << 21)
+#define VRAM_AVAILABLITY_ALIGN (1 << 21)
 
  /* Impose limit on how much memory KFD can use */
  static struct {
@@ -149,7 +149,7 @@ static int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,
  		 * to avoid fragmentation caused by 4K allocations in the tail
  		 * 2M BO chunk.
  		 */
-		vram_needed = ALIGN(size, VRAM_ALLOCATION_ALIGN);
+		vram_needed = size;
  	} else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
  		system_mem_needed = size;
  	} else if (!(alloc_flag &

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c static void unreserve_mem_limit(struct amdgpu_device *adev, · line 201

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +1 -1
@@ -198,7 +198,7 @@ static void unreserve_mem_limit(struct amdgpu_device *adev,
  		kfd_mem_limit.system_mem_used -= size;
  		kfd_mem_limit.ttm_mem_used -= size;
  	} else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
-		adev->kfd.vram_used -= ALIGN(size, VRAM_ALLOCATION_ALIGN);
+		adev->kfd.vram_used -= size;

To calculate the available size taking potential fragmentation into account, we still need to track the aligned VRAM usage somewhere (say adev->kfd.vram_used_aligned), and use that in amdgpu_amdkfd_get_available_memory instead of adev->kfd.vram_used.

Regards,
  Felix

Quoted 20 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +1 -2
  	} else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
  		kfd_mem_limit.system_mem_used -= size;
  	} else if (!(alloc_flag &
@@ -1642,7 +1642,6 @@ size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev)
  	uint64_t reserved_for_pt =
  		ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size);
  	size_t available;
-
  	spin_lock(&kfd_mem_limit.mem_limit_lock);
  	available = adev->gmc.real_vram_size
  		- adev->kfd.vram_used
@@ -1650,7 +1649,7 @@ size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev)
  		- reserved_for_pt;
  	spin_unlock(&kfd_mem_limit.mem_limit_lock);
 
-	return ALIGN_DOWN(available, VRAM_ALLOCATION_ALIGN);
+	return ALIGN_DOWN(available, VRAM_AVAILABLITY_ALIGN);
  }
 
  int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(