Skip to contentSkip to conversation

drm/amdkfd: allocate MMIO/DOORBELL BOs with AMDGPU_GEM_CREATE_PREEMPTIBLE

PATCH In review Lang Yu 2 participants 2 messages May 11, 2022 – May 12, 2022

Subsystems (3)
lore
Share & tools
lore

Lang Yu

author
#0 lore raw

PATCH 1/1 drm/amdkfd: allocate MMIO/DOORBELL BOs with AMDGPU_GEM_CREATE_PREEMPTIBLE

MMIO/DOORBELL BOs' backing resources(bus address resources that are used to talk to the GPU) are not managed by GTT manager, but they are counted by GTT manager. That makes no sense.

With AMDGPU_GEM_CREATE_PREEMPTIBLE flag, such BOs will be managed by PREEMPT manager(for preemptible contexts, e.g., KFD). Then they won't be evicted and don't need to be pinned as well.

But we still leave these BOs pinned to indicate that the underlying resource never moves.

---
 .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c  | 32 +++++++++----------
 1 file changed, 16 insertions(+), 16 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +16 -16
index 8b14c55a0793..fada3b149361 100644
@@ -1518,26 +1518,26 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
 	} else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_GTT) {
 		domain = alloc_domain = AMDGPU_GEM_DOMAIN_GTT;
 		alloc_flags = 0;
-	} else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
+	} else {
 		domain = AMDGPU_GEM_DOMAIN_GTT;
 		alloc_domain = AMDGPU_GEM_DOMAIN_CPU;
 		alloc_flags = AMDGPU_GEM_CREATE_PREEMPTIBLE;
-		if (!offset || !*offset)
-			return -EINVAL;
-		user_addr = untagged_addr(*offset);
-	} else if (flags & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
-			KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
-		domain = AMDGPU_GEM_DOMAIN_GTT;
-		alloc_domain = AMDGPU_GEM_DOMAIN_CPU;
-		bo_type = ttm_bo_type_sg;
-		alloc_flags = 0;
-		if (size > UINT_MAX)
+
+		if (flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
+			if (!offset || !*offset)
+				return -EINVAL;
+			user_addr = untagged_addr(*offset);
+		} else if (flags & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
+				    KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
+			bo_type = ttm_bo_type_sg;
+			if (size > UINT_MAX)
+				return -EINVAL;
+			sg = create_doorbell_sg(*offset, size);
+			if (!sg)
+				return -ENOMEM;
+		} else {
 			return -EINVAL;
-		sg = create_doorbell_sg(*offset, size);
-		if (!sg)
-			return -ENOMEM;
-	} else {
-		return -EINVAL;
+		}
 	}
 
 	*mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL);
Signature · 1 line
2.25.1
maintainer Reviewed-by on v1 1/1
#1 lore raw
Quoted 4 lines · Lang Yu

MMIO/DOORBELL BOs' backing resources(bus address resources that are used to talk to the GPU) are not managed by GTT manager, but they are counted by GTT manager. That makes no sense.

Quoted 8 lines

With AMDGPU_GEM_CREATE_PREEMPTIBLE flag, such BOs will be managed by PREEMPT manager(for preemptible contexts, e.g., KFD). Then they won't be evicted and don't need to be pinned as well.

But we still leave these BOs pinned to indicate that the underlying resource never moves.

Quoted 51 lines
---
  .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c  | 32 +++++++++----------
  1 file changed, 16 insertions(+), 16 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +16 -16
index 8b14c55a0793..fada3b149361 100644
@@ -1518,26 +1518,26 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
  	} else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_GTT) {
  		domain = alloc_domain = AMDGPU_GEM_DOMAIN_GTT;
  		alloc_flags = 0;
-	} else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
+	} else {
  		domain = AMDGPU_GEM_DOMAIN_GTT;
  		alloc_domain = AMDGPU_GEM_DOMAIN_CPU;
  		alloc_flags = AMDGPU_GEM_CREATE_PREEMPTIBLE;
-		if (!offset || !*offset)
-			return -EINVAL;
-		user_addr = untagged_addr(*offset);
-	} else if (flags & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
-			KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
-		domain = AMDGPU_GEM_DOMAIN_GTT;
-		alloc_domain = AMDGPU_GEM_DOMAIN_CPU;
-		bo_type = ttm_bo_type_sg;
-		alloc_flags = 0;
-		if (size > UINT_MAX)
+
+		if (flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
+			if (!offset || !*offset)
+				return -EINVAL;
+			user_addr = untagged_addr(*offset);
+		} else if (flags & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
+				    KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
+			bo_type = ttm_bo_type_sg;
+			if (size > UINT_MAX)
+				return -EINVAL;
+			sg = create_doorbell_sg(*offset, size);
+			if (!sg)
+				return -ENOMEM;
+		} else {
  			return -EINVAL;
-		sg = create_doorbell_sg(*offset, size);
-		if (!sg)
-			return -ENOMEM;
-	} else {
-		return -EINVAL;
+		}
  	}
 
  	*mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL);