Best effort contiguous VRAM allocation
Subsystems (4)
cover Best effort contiguous VRAM allocation
This patch series implement new KFD memory alloc flag for best effort contiguous VRAM allocation, to support peer direct access RDMA device with limited scatter-gather dma capability.
Philip Yang (6):
drm/amdgpu: Support contiguous VRAM allocation drm/amdgpu: Evict BOs from same process for contiguous allocation drm/amdkfd: Evict BO itself for contiguous allocation drm/amdkfd: Increase KFD bo restore wait time drm/amdgpu: Skip dma map resource for null RDMA device drm/amdkfd: Bump kfd version for contiguous VRAM allocation
.../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 20 +++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 3 +- drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c | 42 ++++++++++++------- drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 2 +- include/uapi/drm/amdgpu_drm.h | 5 +++ include/uapi/linux/kfd_ioctl.h | 4 +- 6 files changed, 57 insertions(+), 19 deletions(-)
Signature · 1 line
2.43.2
PATCH 1/6 drm/amdgpu: Support contiguous VRAM allocation
RDMA device with limited scatter-gather capability requires physical address contiguous VRAM buffer for RDMA peer direct access.
Add a new KFD alloc memory flag and store as new GEM bo alloc flag. When pin this buffer object to export for RDMA peerdirect access, set AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS flag, and then vram_mgr will set TTM_PL_FLAG_CONTIFUOUS flag to ask VRAM buddy allocator to get contiguous VRAM.
Remove the 2GB max memory block size limit for contiguous allocation.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 7 +++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c | 9 +++++++-- include/uapi/drm/amdgpu_drm.h | 5 +++++ include/uapi/linux/kfd_ioctl.h | 1 + 4 files changed, 20 insertions(+), 2 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +7 -0
@@ -1470,6 +1470,9 @@ static int amdgpu_amdkfd_gpuvm_pin_bo(struct amdgpu_bo *bo, u32 domain)if (unlikely (ret))return ret; +if (bo->flags &AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS_BEST_EFFORT ) + bo->flags |=AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS ; + ret =amdgpu_bo_pin_restricted (bo, domain,0 ,0 );if (ret)pr_err ("Error in Pinning BO to domain: %d\n" , domain);
@@ -1712,6 +1715,10 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu( alloc_flags =AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE ; alloc_flags |= (flags &KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC ) ?AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED :0 ; + +/* For contiguous VRAM allocation */ +if (flags &KFD_IOC_ALLOC_MEM_FLAGS_CONTIGUOUS_BEST_EFFORT ) + alloc_flags |=AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS_BEST_EFFORT ; } xcp_id = fpriv->xcp_id ==AMDGPU_XCP_NO_PARTITION ?0 : fpriv->xcp_id;
drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c +7 -2
@@ -516,8 +516,13 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man,BUG_ON (min_block_size < mm->chunk_size); -/* Limit maximum size to 2GiB due to SG table limitations */ - size =min (remaining_size,2ULL <<30 ); +if (place->flags &TTM_PL_FLAG_CONTIGUOUS ) + size = remaining_size; +else +/* Limit maximum size to 2GiB due to SG table limitations +* for no contiguous allocation. +*/ + size =min (remaining_size,2ULL <<30 );if ((size >= (u64 )pages_per_block <<PAGE_SHIFT ) && !(size & (((u64 )pages_per_block <<PAGE_SHIFT ) -1 )))
include/uapi/drm/amdgpu_drm.h +5 -0
@@ -171,6 +171,11 @@ extern "C" {* may override the MTYPE selected in AMDGPU_VA_OP_MAP. */ #define AMDGPU_GEM_CREATE_EXT_COHERENT (1 << 15) +/* Flag that allocating the BO with best effort for contiguous VRAM. +* If no contiguous VRAM, fallback to scattered allocation. +* Pin the BO for peerdirect RDMA trigger VRAM defragmentation. +*/ +#define AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS_BEST_EFFORT (1 << 16)struct drm_amdgpu_gem_create_in {/** the requested memory size */
include/uapi/linux/kfd_ioctl.h +1 -0
@@ -407,6 +407,7 @@ struct kfd_ioctl_acquire_vm_args {#define KFD_IOC_ALLOC_MEM_FLAGS_COHERENT (1 << 26)#define KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED (1 << 25)#define KFD_IOC_ALLOC_MEM_FLAGS_EXT_COHERENT (1 << 24) +#define KFD_IOC_ALLOC_MEM_FLAGS_CONTIGUOUS_BEST_EFFORT (1 << 23)/* Allocate memory for later SVM (shared virtual memory) mapping. *
Signature · 1 line
2.43.2
Christian König
maintainerRe: [PATCH 1/6] drm/amdgpu: Support contiguous VRAM allocation
Am 12.04.24 um 22:12 schrieb Philip Yang:
Quoted 10 lines
RDMA device with limited scatter-gather capability requires physical address contiguous VRAM buffer for RDMA peer direct access.
Add a new KFD alloc memory flag and store as new GEM bo alloc flag. When pin this buffer object to export for RDMA peerdirect access, set
AMDGPU_GEM_CREATE_VRAM_CONTIGUOUSflag, and thenvram_mgrwill setTTM_PL_FLAG_CONTIFUOUSflag to ask VRAM buddy allocator to get contiguous VRAM.Remove the 2GB max memory block size limit for contiguous allocation.
I'm going to sync up with Arun on this once more, but I think we won't even need the new flag.
We will just downgrade the existing flag to be a best effort allocation for contiguous buffers and only use the TTM flag internally to signal that we need to alter it while pinning.
Regards,
Christian.
Quoted 82 lines
Signed-off-by Philip Yang <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 7 +++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c | 9 +++++++-- include/uapi/drm/amdgpu_drm.h | 5 +++++ include/uapi/linux/kfd_ioctl.h | 1 + 4 files changed, 20 insertions(+), 2 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +7
-0@@ -1470,6 +1470,9 @@ static int amdgpu_amdkfd_gpuvm_pin_bo(struct amdgpu_bo *bo, u32 domain)if (unlikely (ret))return ret; +if (bo->flags &AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS_BEST_EFFORT ) + bo->flags |=AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS ; + ret =amdgpu_bo_pin_restricted (bo, domain,0 ,0 );if (ret)pr_err ("Error in Pinning BO to domain: %d\n" , domain);@@ -1712,6 +1715,10 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu( alloc_flags =AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE ; alloc_flags |= (flags &KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC ) ?AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED :0 ; + +/* For contiguous VRAM allocation */ +if (flags &KFD_IOC_ALLOC_MEM_FLAGS_CONTIGUOUS_BEST_EFFORT ) + alloc_flags |=AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS_BEST_EFFORT ; } xcp_id = fpriv->xcp_id ==AMDGPU_XCP_NO_PARTITION ?0 : fpriv->xcp_id;drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c +7
-2@@ -516,8 +516,13 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man,BUG_ON (min_block_size < mm->chunk_size); -/* Limit maximum size to 2GiB due to SG table limitations */ - size =min (remaining_size,2ULL <<30 ); +if (place->flags &TTM_PL_FLAG_CONTIGUOUS ) + size = remaining_size; +else +/* Limit maximum size to 2GiB due to SG table limitations +* for no contiguous allocation. +*/ + size =min (remaining_size,2ULL <<30 );if ((size >= (u64 )pages_per_block <<PAGE_SHIFT ) && !(size & (((u64 )pages_per_block <<PAGE_SHIFT ) -1 )))include/uapi/drm/amdgpu_drm.h +5
-0@@ -171,6 +171,11 @@ extern "C" {* may override the MTYPE selected in AMDGPU_VA_OP_MAP. */ #define AMDGPU_GEM_CREATE_EXT_COHERENT (1 << 15) +/* Flag that allocating the BO with best effort for contiguous VRAM. +* If no contiguous VRAM, fallback to scattered allocation. +* Pin the BO for peerdirect RDMA trigger VRAM defragmentation. +*/ +#define AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS_BEST_EFFORT (1 << 16)struct drm_amdgpu_gem_create_in {/** the requested memory size */ include/uapi/linux/kfd_ioctl.h +1
-0@@ -407,6 +407,7 @@ struct kfd_ioctl_acquire_vm_args {#define KFD_IOC_ALLOC_MEM_FLAGS_COHERENT (1 << 26)#define KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED (1 << 25)#define KFD_IOC_ALLOC_MEM_FLAGS_EXT_COHERENT (1 << 24) +#define KFD_IOC_ALLOC_MEM_FLAGS_CONTIGUOUS_BEST_EFFORT (1 << 23)/* Allocate memory for later SVM (shared virtual memory) mapping. *
Re: [PATCH 1/6] drm/amdgpu: Support contiguous VRAM allocation
On 2024-04-15 08:02, Christian König wrote:
Am 12.04.24 um 22:12 schrieb Philip Yang:
RDMA device with limited scatter-gather capability requires physical address contiguous VRAM buffer for RDMA peer direct access.
Add a new KFD alloc memory flag and store as new GEM bo alloc flag. When pin this buffer object to export for RDMA peerdirect access, set AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS flag, and then vram_mgr will set TTM_PL_FLAG_CONTIFUOUS flag to ask VRAM buddy allocator to get contiguous VRAM.
Remove the 2GB max memory block size limit for contiguous allocation.
I'm going to sync up with Arun on this once more, but I think we won't even need the new flag.
We will just downgrade the existing flag to be a best effort allocation for contiguous buffers and only use the TTM flag internally to signal that we need to alter it while pinning. sure, I will rebase this patch series to "[PATCH] drm/amdgpu: Modify the contiguous flags behaviour", this will remove the new flag.
Will send v2 patch series after Arun's v2 patch.
Regards,
Philip
Regards,
Christian.
---
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 7 +++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c | 9 +++++++-- include/uapi/drm/amdgpu_drm.h | 5 +++++ include/uapi/linux/kfd_ioctl.h | 1 +
4 files changed, 20 insertions(+), 2 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +0 -0
@@ -1470,6 +1470,9 @@ static int amdgpu_amdkfd_gpuvm_pin_bo(struct amdgpu_bo *bo, u32 domain)
if (unlikely(ret))
return ret;
+ if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS_BEST_EFFORT) + bo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
+
ret = amdgpu_bo_pin_restricted(bo, domain, 0, 0);
if (ret)
pr_err("Error in Pinning BO to domain: %d\n", domain); @@ -1712,6 +1715,10 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu( alloc_flags = AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE; alloc_flags |= (flags & KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC) ? AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED : 0;
+
+ /* For contiguous VRAM allocation */
+ if (flags & KFD_IOC_ALLOC_MEM_FLAGS_CONTIGUOUS_BEST_EFFORT) + alloc_flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS_BEST_EFFORT; }
xcp_id = fpriv->xcp_id == AMDGPU_XCP_NO_PARTITION ?
0 : fpriv->xcp_id;
drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c +0 -0
@@ -516,8 +516,13 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man,
BUG_ON(min_block_size < mm->chunk_size);
- /* Limit maximum size to 2GiB due to SG table limitations */ - size = min(remaining_size, 2ULL << 30);
+ if (place->flags & TTM_PL_FLAG_CONTIGUOUS)
+ size = remaining_size;
+ else
+ /* Limit maximum size to 2GiB due to SG table limitations + * for no contiguous allocation.
+ */
+ size = min(remaining_size, 2ULL << 30);
if ((size >= (u64)pages_per_block << PAGE_SHIFT) && !(size & (((u64)pages_per_block << PAGE_SHIFT) - 1)))
include/uapi/drm/amdgpu_drm.h +0 -0
@@ -171,6 +171,11 @@ extern "C" {
* may override the MTYPE selected in AMDGPU_VA_OP_MAP. */
#define AMDGPU_GEM_CREATE_EXT_COHERENT (1 << 15) +/* Flag that allocating the BO with best effort for contiguous VRAM.
+ * If no contiguous VRAM, fallback to scattered allocation.
+ * Pin the BO for peerdirect RDMA trigger VRAM defragmentation.
+ */
+#define AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS_BEST_EFFORT (1 << 16) struct drm_amdgpu_gem_create_in {
/** the requested memory size */
include/uapi/linux/kfd_ioctl.h +0 -0
@@ -407,6 +407,7 @@ struct kfd_ioctl_acquire_vm_args {
#define KFD_IOC_ALLOC_MEM_FLAGS_COHERENT (1 << 26) #define KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED (1 << 25) #define KFD_IOC_ALLOC_MEM_FLAGS_EXT_COHERENT (1 << 24) +#define KFD_IOC_ALLOC_MEM_FLAGS_CONTIGUOUS_BEST_EFFORT (1 << 23) /* Allocate memory for later SVM (shared virtual memory) mapping. *
PATCH 2/6 drm/amdgpu: Evict BOs from same process for contiguous allocation
In reply to Philip YangWhen TTM failed to alloc VRAM, TTM evict BOs from VRAM to system memory then retry the allocation, this currently skips the KFD BOs from the same process because KFD requires all BOs are resident for user queues.
If TTM BO with TTM_PL_FLAG_CONTIGUOUS flag to alloc contiguous VRAM, allow TTM evict KFD BOs from the same process, this will evict the user queues first, and restore the queues later after contiguous VRAM allocation.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +2 -1
@@ -1383,7 +1383,8 @@ static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,*/ dma_resv_for_each_fence (&resv_cursor, bo->base.resv,DMA_RESV_USAGE_BOOKKEEP , f) { -if (amdkfd_fence_check_mm (f, current->mm)) +if (amdkfd_fence_check_mm (f, current->mm) && + !(place->flags &TTM_PL_FLAG_CONTIGUOUS ))return false; }
Signature · 1 line
2.43.2
PATCH 3/6 drm/amdkfd: Evict BO itself for contiguous allocation
In reply to Philip YangIf the BO pages pinned for RDMA is not contiguous on VRAM, evict it to system memory first to free the VRAM space, then allocate contiguous VRAM and then move it from system memory back to VRAM.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +14 -1
@@ -1470,8 +1470,21 @@ static int amdgpu_amdkfd_gpuvm_pin_bo(struct amdgpu_bo *bo, u32 domain)if (unlikely (ret))return ret; -if (bo->flags &AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS_BEST_EFFORT ) +if (bo->flags &AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS_BEST_EFFORT ) { +/* +* If bo is not contiguous on VRAM, move to system memory first to ensure +* we can get contiguous VRAM space after evicting other BOs. +*/ +if (!(bo->tbo.resource->placement &TTM_PL_FLAG_CONTIGUOUS )) { + ret =amdgpu_amdkfd_bo_validate (bo,AMDGPU_GEM_DOMAIN_GTT , false); +if (unlikely (ret)) { +pr_debug ("validate bo 0x%p to GTT failed %d\n" , &bo->tbo, ret); +return ret; + } + } + bo->flags |=AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS ; + } ret =amdgpu_bo_pin_restricted (bo, domain,0 ,0 );if (ret)
Signature · 1 line
2.43.2
PATCH 4/6 drm/amdkfd: Increase KFD bo restore wait time
In reply to Philip YangTTM allocate contiguous VRAM may takes more than 1 second to evict BOs for larger size RDMA buffer. Because KFD restore bo worker reserves all KFD BOs, then TTM cannot hold the remainning KFD BOs lock to evict them, this may causes TTM failed to alloc contiguous VRAM.
Increase the KFD restore BO wait time to 2 seconds, long enough for RDMA pin BO to finish the contiguous VRAM allocation.
--- drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/gpu/drm/amd/amdkfd/kfd_priv.h +1 -1
@@ -698,7 +698,7 @@ struct qcm_process_device {/* KFD Memory Eviction */ /* Approx. wait time before attempting to restore evicted BOs */ -#define PROCESS_RESTORE_TIME_MS 100 +#define PROCESS_RESTORE_TIME_MS 2000/* Approx. back off time if restore fails due to lack of memory */ #define PROCESS_BACK_OFF_TIME_MS 100/* Approx. time before evicting the process again */
Signature · 1 line
2.43.2
PATCH 5/6 drm/amdgpu: Skip dma map resource for null RDMA device
In reply to Philip YangTo test RDMA using dummy driver on the system without NIC/RDMA device, the get dma pages pass in null device pointer, skip the dma map resource to avoid null device pointer access.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c | 33 +++++++++++--------- 1 file changed, 19 insertions(+), 14 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c +19 -14
@@ -674,12 +674,15 @@ int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev,size_t size = cursor.size;dma_addr_t addr; - addr =dma_map_resource (dev, phys, size, dir, -DMA_ATTR_SKIP_CPU_SYNC ); - r =dma_mapping_error (dev, addr); -if (r) - goto error_unmap; - +if (dev) { + addr =dma_map_resource (dev, phys, size, dir, +DMA_ATTR_SKIP_CPU_SYNC ); + r =dma_mapping_error (dev, addr); +if (r) + goto error_unmap; + }else { + addr = phys; + }sg_set_page (sg,NULL , size,0 );sg_dma_address (sg) = addr;sg_dma_len (sg) = size;
@@ -693,10 +696,10 @@ int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev,for_each_sgtable_sg ((*sgt), sg, i) {if (!sg->length)continue ; - -dma_unmap_resource (dev, sg->dma_address, - sg->length, dir, -DMA_ATTR_SKIP_CPU_SYNC ); +if (dev) +dma_unmap_resource (dev, sg->dma_address, + sg->length, dir, +DMA_ATTR_SKIP_CPU_SYNC ); }sg_free_table (*sgt);
@@ -721,10 +724,12 @@ void amdgpu_vram_mgr_free_sgt(struct device *dev,struct scatterlist *sg;int i; -for_each_sgtable_sg (sgt, sg, i) -dma_unmap_resource (dev, sg->dma_address, - sg->length, dir, -DMA_ATTR_SKIP_CPU_SYNC ); +if (dev) { +for_each_sgtable_sg (sgt, sg, i) +dma_unmap_resource (dev, sg->dma_address, + sg->length, dir, +DMA_ATTR_SKIP_CPU_SYNC ); + }sg_free_table (sgt);kfree (sgt); }
Signature · 1 line
2.43.2
PATCH 6/6 drm/amdkfd: Bump kfd version for contiguous VRAM allocation
In reply to Philip YangBump the kfd ioctl minor version to delcare the contiguous VRAM allocation flag support.
--- include/uapi/linux/kfd_ioctl.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
include/uapi/linux/kfd_ioctl.h +2 -1
@@ -41,9 +41,10 @@* - 1.13 - Add debugger API * - 1.14 - Update kfd_event_data * - 1.15 - Enable managing mappings in compute VMs with GEM_VA ioctl +* - 1.16 - Add contiguous VRAM allocation flag for RDMA */ #define KFD_IOCTL_MAJOR_VERSION 1 -#define KFD_IOCTL_MINOR_VERSION 15 +#define KFD_IOCTL_MINOR_VERSION 16struct kfd_ioctl_get_version_args {__u32 major_version;/* from KFD */
Signature · 1 line
2.43.2