Skip to contentSkip to conversation

drm/amdgpu: Add a new flag for SR-IOV to share memory between PF & VF

PATCH In review Horace Chen 2 participants 6 messages Sep 27, 2017 3 messages not in the archive

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/3 drm/amdgpu: Add a new flag for SR-IOV to share memory between PF & VF

Add ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION to identify whether driver need to reserve VRAM for SR-IOV shared memory.

---
 drivers/gpu/drm/amd/include/atombios.h | 1 +
 1 file changed, 1 insertion(+)
drivers/gpu/drm/amd/include/atombios.h +1 -0
index 181a2c3..f696bbb 100644
@@ -4292,6 +4292,7 @@ typedef struct _ATOM_DPCD_INFO
 #define ATOM_VRAM_OPERATION_FLAGS_SHIFT        30
 #define   ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION   0x1
 #define   ATOM_VRAM_BLOCK_NEEDS_RESERVATION      0x0
+#define   ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION 0x2
 
 /***********************************************************************************/
 // Structure used in VRAM_UsageByFirmwareTable
Signature · 6 lines
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 1506504231-24507-1-git-send-email-horace.chen-5C7GfCeVMHo@public.gmane.org

#2 lore raw

PATCH 2/3 drm/amdgpu: add functions to create bo at the specified position

Add two function to create and free bo at the specified position
on the VRAM.
Add a new parameter to amdgpu_bo_do_create to tell the start address
of the special bo. If the start address is located in the GPU MC address space, the placement will be set to the exact place according to the size and start address. Otherwise the start address will be ingored.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 118 ++++++++++++++++++++++++++++-
 drivers/gpu/drm/amd/amdgpu/amdgpu_object.h |   6 ++
 2 files changed, 122 insertions(+), 2 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +116 -2
index 6982bae..8ea7fec 100644
@@ -284,6 +284,7 @@ void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr,
 }
 
 static int amdgpu_bo_do_create(struct amdgpu_device *adev,
+			       u64 start_addr,
 			       unsigned long size, int byte_align,
 			       bool kernel, u32 domain, u64 flags,
 			       struct sg_table *sg,
@@ -297,9 +298,11 @@ static int amdgpu_bo_do_create(struct amdgpu_device *adev,
 	u64 initial_bytes_moved, bytes_moved;
 	size_t acc_size;
 	int r;
+	int i;
 
 	page_align = roundup(byte_align, PAGE_SIZE) >> PAGE_SHIFT;
 	size = ALIGN(size, PAGE_SIZE);
+	start_addr = ALIGN(start_addr, PAGE_SIZE);
 
 	if (kernel) {
 		type = ttm_bo_type_kernel;
@@ -366,6 +369,23 @@ static int amdgpu_bo_do_create(struct amdgpu_device *adev,
 	bo->tbo.bdev = &adev->mman.bdev;
 	amdgpu_ttm_placement_from_domain(bo, domain);
 
+	/*
+	 * if domain is VRAM only && start_addr in GPU address space,
+	 * set the place to the specified place
+	 */
+	if (domain == AMDGPU_GEM_DOMAIN_VRAM &&
+	    start_addr >= adev->mc.vram_start &&
+	    (start_addr + size) < adev->mc.vram_end) {
+		/*change it to offset */
+		start_addr -= adev->mc.vram_start;
+		for (i = 0; i < bo->placement.num_placement; ++i) {
+			bo->placements[i].fpfn =
+				start_addr >> PAGE_SHIFT;
+			bo->placements[i].lpfn =
+				(start_addr + size) >> PAGE_SHIFT;
+		}
+	}
+
 	initial_bytes_moved = atomic64_read(&adev->num_bytes_moved);
 	/* Kernel allocation are uninterruptible */
 	r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, size, type,
@@ -418,6 +438,100 @@ static int amdgpu_bo_do_create(struct amdgpu_device *adev,
 	return r;
 }
 
+/**
+ * amdgpu_bo_create_vram_specified -
+ *    create BO at the specified place on the VRAM.
+ *
+ * @adev: amdgpu device object
+ * @offset: start offset of the new BO
+ * @size: size for the new BO
+ * @byte_align: alignment for the new BO
+ * @flags: addition flags for the BO
+ * @bo_ptr: resulting BO
+ * @gpu_addr: GPU addr of the pinned BO
+ * @cpu_addr: optional CPU address mapping
+ *
+ * Allocates and pins a BO at the specified place on the VRAM.
+ *
+ * Returns 0 on success, negative error code otherwise.
+ */
+int amdgpu_bo_create_vram_specified(struct amdgpu_device *adev,
+		     u64 offset, unsigned long size, int byte_align,
+		     u64 flags, struct amdgpu_bo **bo_ptr,
+		     u64 *gpu_addr, void **cpu_addr)
+{
+	int r;
+	u64 start_addr = offset + adev->mc.vram_start;
+	/* specified memory must be in contiguous*/
+	flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
+
+	r = amdgpu_bo_do_create(adev, start_addr, size, byte_align, true,
+		AMDGPU_GEM_DOMAIN_VRAM, flags, NULL, NULL, 0, bo_ptr);
+	if (r)
+		return r;
+
+	r = amdgpu_bo_reserve(*bo_ptr, false);
+	if (r)
+		goto error_reserve;
+	r = amdgpu_bo_pin_restricted(*bo_ptr,
+		AMDGPU_GEM_DOMAIN_VRAM, offset, (offset + size),
+		gpu_addr);
+	if (r)
+		goto error_pin;
+	if (cpu_addr) {
+		r = amdgpu_bo_kmap(*bo_ptr,
+					cpu_addr);
+		if (r)
+			goto error_kmap;
+	}
+
+	amdgpu_bo_unreserve(*bo_ptr);
+
+	return r;
+error_kmap:
+	amdgpu_bo_unpin(*bo_ptr);
+error_pin:
+	amdgpu_bo_unreserve(*bo_ptr);
+error_reserve:
+	amdgpu_bo_unref(bo_ptr);
+	if (cpu_addr)
+		*cpu_addr = NULL;
+	if (gpu_addr)
+		*gpu_addr = 0;
+	return r;
+}
+
+/**
+ * amdgpu_bo_free_vram_specified - free BO for specified vram
+ *
+ * @bo: amdgpu BO to free
+ * @gpu_addr : gpu address
+ * @cpu_addr : cpu address if mapped
+ *
+ * unmaps and unpin a BO for specified vram.
+ */
+void amdgpu_bo_free_vram_specified(struct amdgpu_bo **bo, u64 *gpu_addr,
+			   void **cpu_addr)
+{
+	if (*bo == NULL)
+		return;
+
+	if (likely(amdgpu_bo_reserve(*bo, true) == 0)) {
+		if (cpu_addr)
+			amdgpu_bo_kunmap(*bo);
+		amdgpu_bo_unpin(*bo);
+		amdgpu_bo_unreserve(*bo);
+	}
+	amdgpu_bo_unref(bo);
+
+	if (gpu_addr)
+		*gpu_addr = 0;
+
+	if (cpu_addr)
+		*cpu_addr = NULL;
+}
+
+
 static int amdgpu_bo_create_shadow(struct amdgpu_device *adev,
 				   unsigned long size, int byte_align,
 				   struct amdgpu_bo *bo)
@@ -427,7 +541,7 @@ static int amdgpu_bo_create_shadow(struct amdgpu_device *adev,
 	if (bo->shadow)
 		return 0;
 
-	r = amdgpu_bo_do_create(adev, size, byte_align, true,
+	r = amdgpu_bo_do_create(adev, 0, size, byte_align, true,
 				AMDGPU_GEM_DOMAIN_GTT,
 				AMDGPU_GEM_CREATE_CPU_GTT_USWC |
 				AMDGPU_GEM_CREATE_SHADOW,
@@ -457,7 +571,7 @@ int amdgpu_bo_create(struct amdgpu_device *adev,
 	uint64_t parent_flags = flags & ~AMDGPU_GEM_CREATE_SHADOW;
 	int r;
 
-	r = amdgpu_bo_do_create(adev, size, byte_align, kernel, domain,
+	r = amdgpu_bo_do_create(adev, 0, size, byte_align, kernel, domain,
 				parent_flags, sg, resv, init_value, bo_ptr);
 	if (r)
 		return r;
drivers/gpu/drm/amd/amdgpu/amdgpu_object.h +6 -0
index 39b6bf6..8742583 100644
@@ -205,6 +205,12 @@ int amdgpu_bo_create_kernel(struct amdgpu_device *adev,
 			    u64 *gpu_addr, void **cpu_addr);
 void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr,
 			   void **cpu_addr);
+int amdgpu_bo_create_vram_specified(struct amdgpu_device *adev,
+		     u64 offset, unsigned long size, int byte_align,
+		     u64 flags, struct amdgpu_bo **bo_ptr,
+		     u64 *gpu_addr, void **cpu_addr);
+void amdgpu_bo_free_vram_specified(struct amdgpu_bo **bo, u64 *gpu_addr,
+			   void **cpu_addr);
 int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr);
 void *amdgpu_bo_kptr(struct amdgpu_bo *bo);
 void amdgpu_bo_kunmap(struct amdgpu_bo *bo);
Signature · 6 lines
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 1506504231-24507-2-git-send-email-horace.chen-5C7GfCeVMHo@public.gmane.org

Acked-by Alex Deucher
#4 lore raw

RE: [PATCH 2/3] drm/amdgpu: add functions to create bo at the specified position

Quoted 61 lines

-----Original Message-----
From: Horace Chen [mailto:[email protected]]
Sent: Wednesday, September 27, 2017 5:24 AM
To: [email protected]
Cc: Deucher, Alexander; Chen, Horace
Subject: [PATCH 2/3] drm/amdgpu: add functions to create bo at the specified position

Add two function to create and free bo at the specified position
on the VRAM.
Add a new parameter to amdgpu_bo_do_create to tell the start address
of the special bo. If the start address is located in the GPU MC address space, the placement will be set to the exact place according to the size and start address. Otherwise the start address will be ingored.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 118
++++++++++++++++++++++++++++-
 drivers/gpu/drm/amd/amdgpu/amdgpu_object.h |   6 ++
 2 files changed, 122 insertions(+), 2 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +1 -0
index 6982bae..8ea7fec 100644
@@ -284,6 +284,7 @@ void amdgpu_bo_free_kernel(struct amdgpu_bo
 **bo, u64 *gpu_addr,
 }
 
 static int amdgpu_bo_do_create(struct amdgpu_device *adev,
+			       u64 start_addr,
 			       unsigned long size, int byte_align,
 			       bool kernel, u32 domain, u64 flags,
drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +2 -0
 			       struct sg_table *sg,
@@ -297,9 +298,11 @@ static int amdgpu_bo_do_create(struct
 amdgpu_device *adev,
 	u64 initial_bytes_moved, bytes_moved;
 	size_t acc_size;
 	int r;
+	int i;
 
 	page_align = roundup(byte_align, PAGE_SIZE) >> PAGE_SHIFT;
 	size = ALIGN(size, PAGE_SIZE);
+	start_addr = ALIGN(start_addr, PAGE_SIZE);
 
 	if (kernel) {
drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +6 -0
 		type = ttm_bo_type_kernel;
@@ -366,6 +369,23 @@ static int amdgpu_bo_do_create(struct
 amdgpu_device *adev,
 	bo->tbo.bdev = &adev->mman.bdev;
 	amdgpu_ttm_placement_from_domain(bo, domain);
 
+	/*
+	 * if domain is VRAM only && start_addr in GPU address space,
+	 * set the place to the specified place
+	 */
+	if (domain == AMDGPU_GEM_DOMAIN_VRAM &&
+	    start_addr >= adev->mc.vram_start &&

drivers/gpu/drm/amd/amdgpu/amdgpu_object.c static int amdgpu_bo_do_create(struct · line 388

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +10 -0
+	    (start_addr + size) < adev->mc.vram_end) {
+		/*change it to offset */
+		start_addr -= adev->mc.vram_start;
+		for (i = 0; i < bo->placement.num_placement; ++i) {
+			bo->placements[i].fpfn =
+				start_addr >> PAGE_SHIFT;
+			bo->placements[i].lpfn =
+				(start_addr + size) >> PAGE_SHIFT;
+		}
+	}

Any reason to limit this to only VRAM? Should be possible to handle gart as well, although I don't know that we have a particular need for that. I guess we can always extend it if we need it.

Quoted 18 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +1 -0
+
 	initial_bytes_moved = atomic64_read(&adev->num_bytes_moved);
 	/* Kernel allocation are uninterruptible */

r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, size,
type,

drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +8 -0
@@ -418,6 +438,100 @@ static int amdgpu_bo_do_create(struct
 amdgpu_device *adev,
 	return r;
 }
 
+/**
+ * amdgpu_bo_create_vram_specified -
+ *    create BO at the specified place on the VRAM.
+ *
+ * @adev: amdgpu device object
+ * @offset: start offset of the new BO
+ * @size: size for the new BO
+ * @byte_align: alignment for the new BO

drivers/gpu/drm/amd/amdgpu/amdgpu_object.c static int amdgpu_bo_do_create(struct · line 459

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +10 -0
+ * @flags: addition flags for the BO
+ * @bo_ptr: resulting BO
+ * @gpu_addr: GPU addr of the pinned BO
+ * @cpu_addr: optional CPU address mapping
+ *
+ * Allocates and pins a BO at the specified place on the VRAM.
+ *
+ * Returns 0 on success, negative error code otherwise.
+ */
+int amdgpu_bo_create_vram_specified(struct amdgpu_device *adev,

Maybe make the name a bit more consistent? Use restricted to mirror the pin naming and kernel so we know it's just for kernel allocations. How about? amdgpu_bo_create_vram_restricted_kernel() and
amdgpu_bo_free_vram_restricted_kernel()?

With that fixed:

Quoted 118 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +76 -0
+		     u64 offset, unsigned long size, int byte_align,
+		     u64 flags, struct amdgpu_bo **bo_ptr,
+		     u64 *gpu_addr, void **cpu_addr)
+{
+	int r;
+	u64 start_addr = offset + adev->mc.vram_start;
+	/* specified memory must be in contiguous*/
+	flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
+
+	r = amdgpu_bo_do_create(adev, start_addr, size, byte_align, true,
+		AMDGPU_GEM_DOMAIN_VRAM, flags, NULL, NULL, 0,
 bo_ptr);
+	if (r)
+		return r;
+
+	r = amdgpu_bo_reserve(*bo_ptr, false);
+	if (r)
+		goto error_reserve;
+	r = amdgpu_bo_pin_restricted(*bo_ptr,
+		AMDGPU_GEM_DOMAIN_VRAM, offset, (offset + size),
+		gpu_addr);
+	if (r)
+		goto error_pin;
+	if (cpu_addr) {
+		r = amdgpu_bo_kmap(*bo_ptr,
+					cpu_addr);
+		if (r)
+			goto error_kmap;
+	}
+
+	amdgpu_bo_unreserve(*bo_ptr);
+
+	return r;
+error_kmap:
+	amdgpu_bo_unpin(*bo_ptr);
+error_pin:
+	amdgpu_bo_unreserve(*bo_ptr);
+error_reserve:
+	amdgpu_bo_unref(bo_ptr);
+	if (cpu_addr)
+		*cpu_addr = NULL;
+	if (gpu_addr)
+		*gpu_addr = 0;
+	return r;
+}
+
+/**
+ * amdgpu_bo_free_vram_specified - free BO for specified vram
+ *
+ * @bo: amdgpu BO to free
+ * @gpu_addr : gpu address
+ * @cpu_addr : cpu address if mapped
+ *
+ * unmaps and unpin a BO for specified vram.
+ */
+void amdgpu_bo_free_vram_specified(struct amdgpu_bo **bo, u64
 *gpu_addr,
+			   void **cpu_addr)
+{
+	if (*bo == NULL)
+		return;
+
+	if (likely(amdgpu_bo_reserve(*bo, true) == 0)) {
+		if (cpu_addr)
+			amdgpu_bo_kunmap(*bo);
+		amdgpu_bo_unpin(*bo);
+		amdgpu_bo_unreserve(*bo);
+	}
+	amdgpu_bo_unref(bo);
+
+	if (gpu_addr)
+		*gpu_addr = 0;
+
+	if (cpu_addr)
+		*cpu_addr = NULL;
+}
+
+

static int amdgpu_bo_create_shadow(struct amdgpu_device *adev,

drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +1 -1
 				   unsigned long size, int byte_align,
 				   struct amdgpu_bo *bo)
@@ -427,7 +541,7 @@ static int amdgpu_bo_create_shadow(struct
 amdgpu_device *adev,
 	if (bo->shadow)
 		return 0;
 
-	r = amdgpu_bo_do_create(adev, size, byte_align, true,
+	r = amdgpu_bo_do_create(adev, 0, size, byte_align, true,
 				AMDGPU_GEM_DOMAIN_GTT,
 				AMDGPU_GEM_CREATE_CPU_GTT_USWC |
drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +1 -1
 				AMDGPU_GEM_CREATE_SHADOW,
@@ -457,7 +571,7 @@ int amdgpu_bo_create(struct amdgpu_device *adev,
 	uint64_t parent_flags = flags & ~AMDGPU_GEM_CREATE_SHADOW;
 	int r;
 
-	r = amdgpu_bo_do_create(adev, size, byte_align, kernel, domain,
+	r = amdgpu_bo_do_create(adev, 0, size, byte_align, kernel, domain,
 				parent_flags, sg, resv, init_value, bo_ptr);
 	if (r)
 		return r;
drivers/gpu/drm/amd/amdgpu/amdgpu_object.h +6 -0
index 39b6bf6..8742583 100644
@@ -205,6 +205,12 @@ int amdgpu_bo_create_kernel(struct
 amdgpu_device *adev,
 			    u64 *gpu_addr, void **cpu_addr);
 void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr,
 			   void **cpu_addr);
+int amdgpu_bo_create_vram_specified(struct amdgpu_device *adev,
+		     u64 offset, unsigned long size, int byte_align,
+		     u64 flags, struct amdgpu_bo **bo_ptr,
+		     u64 *gpu_addr, void **cpu_addr);
+void amdgpu_bo_free_vram_specified(struct amdgpu_bo **bo, u64
 *gpu_addr,
+			   void **cpu_addr);
 int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr);
Quoted 4 lines

void *amdgpu_bo_kptr(struct amdgpu_bo *bo);
void amdgpu_bo_kunmap(struct amdgpu_bo *bo);
--
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#5 lore raw

PATCH 3/3 drm/amdgpu: Reserve shared memory on VRAM for SR-IOV

In reply to a message not in the archive

SR-IOV need to reserve a piece of shared VRAM to exchange data
betweem PF and VF. The start address and size of the shared mem are passed to guest through VBIOS structure VRAM_UsageByFirmware.

VRAM_UsageByFirmware is a general feature in VBIOS, it indicates that VBIOS need to reserve a piece of memory on the VRAM.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu.h          | 17 ++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c | 18 +++++++-
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c   | 62 ++++++++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c      |  9 ++++
 4 files changed, 105 insertions(+), 1 deletion(-)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +17 -0
index a5b0b67..ac53dba 100644
@@ -1380,6 +1380,21 @@ struct amdgpu_atcs {
 };
 
 /*
+ * Firmware VRAM reservation
+ */
+#define KB_TO_BYTE(x)                           ((x) << 10)
+#define BYTE_TO_KB(x)                           ((x) >> 10)
+
+struct amdgpu_fw_vram_usage {
+	uint32_t start_address_in_kb;
+	uint16_t size_in_kb;
+	struct amdgpu_bo *reserved_bo;
+	volatile uint32_t *va;
+};
+
+int amdgpu_fw_reserve_vram_init(struct amdgpu_device *adev);
+
+/*
  * CGS
  */
 struct cgs_device *amdgpu_cgs_create_device(struct amdgpu_device *adev);
@@ -1588,6 +1603,8 @@ struct amdgpu_device {
 	struct delayed_work     late_init_work;
 
 	struct amdgpu_virt	virt;
+	/* firmware VRAM reservation */
+	struct amdgpu_fw_vram_usage fw_vram_usage;
 
 	/* link all shadow bo */
 	struct list_head                shadow_list;
drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c +17 -1
index ce44358..56bfddf 100644
@@ -1807,6 +1807,8 @@ int amdgpu_atombios_allocate_fb_scratch(struct amdgpu_device *adev)
 	uint16_t data_offset;
 	int usage_bytes = 0;
 	struct _ATOM_VRAM_USAGE_BY_FIRMWARE *firmware_usage;
+	uint32_t start_addr;
+	uint16_t size;
 
 	if (amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) {
 		firmware_usage = (struct _ATOM_VRAM_USAGE_BY_FIRMWARE *)(ctx->bios + data_offset);
@@ -1815,7 +1817,21 @@ int amdgpu_atombios_allocate_fb_scratch(struct amdgpu_device *adev)
 			  le32_to_cpu(firmware_usage->asFirmwareVramReserveInfo[0].ulStartAddrUsedByFirmware),
 			  le16_to_cpu(firmware_usage->asFirmwareVramReserveInfo[0].usFirmwareUseInKb));
 
-		usage_bytes = le16_to_cpu(firmware_usage->asFirmwareVramReserveInfo[0].usFirmwareUseInKb) * 1024;
+		start_addr = firmware_usage->asFirmwareVramReserveInfo[0].ulStartAddrUsedByFirmware;
+		size = firmware_usage->asFirmwareVramReserveInfo[0].usFirmwareUseInKb;
+
+		if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
+			(uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
+			ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
+			/* Firmware request VRAM reservation for SR-IOV */
+			adev->fw_vram_usage.start_address_in_kb =
+			    start_addr & (~ATOM_VRAM_OPERATION_FLAGS_MASK);
+			adev->fw_vram_usage.size_in_kb = size;
+			/* Use the default scratch size */
+			usage_bytes = 0;
+		} else {
+			usage_bytes = le16_to_cpu(firmware_usage->asFirmwareVramReserveInfo[0].usFirmwareUseInKb) * 1024;
+		}
 	}
 	ctx->scratch_size_bytes = 0;
 	if (usage_bytes == 0)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +62 -0

80 lines not on this page · raw message

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +9 -0
index 1086f03..a5253cf 100644
@@ -1256,6 +1256,15 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
 	/* Change the size here instead of the init above so only lpfn is affected */
 	amdgpu_ttm_set_active_vram_size(adev, adev->mc.visible_vram_size);
 
+	/*
+	 *The reserved vram for firmware must be pinned to the specified
+	 *place on the VRAM, so reserve it early.
+	 */
+	r = amdgpu_fw_reserve_vram_init(adev);
+	if (r) {
+		return r;
+	}
+
 	r = amdgpu_bo_create_kernel(adev, adev->mc.stolen_size, PAGE_SIZE,
 				    AMDGPU_GEM_DOMAIN_VRAM,
 				    &adev->stolen_vga_memory,
Signature · 6 lines
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 1506504231-24507-3-git-send-email-horace.chen-5C7GfCeVMHo@public.gmane.org

#7 lore raw

RE: [PATCH 3/3] drm/amdgpu: Reserve shared memory on VRAM for SR-IOV

Quoted 32 lines

-----Original Message-----
From: amd-gfx [mailto:[email protected]] On Behalf Of Horace Chen
Sent: Wednesday, September 27, 2017 5:24 AM
To: [email protected]
Cc: Deucher, Alexander; Chen, Horace
Subject: [PATCH 3/3] drm/amdgpu: Reserve shared memory on VRAM for SR-IOV

SR-IOV need to reserve a piece of shared VRAM to exchange data
betweem PF and VF. The start address and size of the shared mem are passed to guest through VBIOS structure VRAM_UsageByFirmware.

VRAM_UsageByFirmware is a general feature in VBIOS, it indicates that VBIOS need to reserve a piece of memory on the VRAM.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu.h          | 17 ++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c | 18 +++++++-
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c   | 62
++++++++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c      |  9 ++++
 4 files changed, 105 insertions(+), 1 deletion(-)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +0 -0
index a5b0b67..ac53dba 100644
@@ -1380,6 +1380,21 @@ struct amdgpu_atcs {
 };

drivers/gpu/drm/amd/amdgpu/amdgpu.h struct amdgpu_atcs { · line 1390

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu.h +8 -0
 
 /*
+ * Firmware VRAM reservation
+ */
+#define KB_TO_BYTE(x)                           ((x) << 10)
+#define BYTE_TO_KB(x)                           ((x) >> 10)
+
+struct amdgpu_fw_vram_usage {
+	uint32_t start_address_in_kb;
+	uint16_t size_in_kb;

Just store these in bytes. Do the conversion when you read them out of the vbios table. That's more consistent with how we start addresses elsewhere in the driver.

Alex

Quoted 150 lines
drivers/gpu/drm/amd/amdgpu/amdgpu.h +7 -0
+	struct amdgpu_bo *reserved_bo;
+	volatile uint32_t *va;
+};
+
+int amdgpu_fw_reserve_vram_init(struct amdgpu_device *adev);
+
+/*
  * CGS
  */
 struct cgs_device *amdgpu_cgs_create_device(struct amdgpu_device

*adev);

drivers/gpu/drm/amd/amdgpu/amdgpu.h +2 -0
@@ -1588,6 +1603,8 @@ struct amdgpu_device {
 	struct delayed_work     late_init_work;
 
 	struct amdgpu_virt	virt;
+	/* firmware VRAM reservation */
+	struct amdgpu_fw_vram_usage fw_vram_usage;
 
 	/* link all shadow bo */
 	struct list_head                shadow_list;
drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c +2 -0
index ce44358..56bfddf 100644
@@ -1807,6 +1807,8 @@ int amdgpu_atombios_allocate_fb_scratch(struct
 amdgpu_device *adev)
 	uint16_t data_offset;
 	int usage_bytes = 0;
 	struct _ATOM_VRAM_USAGE_BY_FIRMWARE *firmware_usage;
+	uint32_t start_addr;
+	uint16_t size;
 
 	if (amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL,

&data_offset)) {
firmware_usage = (struct
_ATOM_VRAM_USAGE_BY_FIRMWARE *)(ctx->bios + data_offset);

drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c +0 -1
@@ -1815,7 +1817,21 @@ int amdgpu_atombios_allocate_fb_scratch(struct
 amdgpu_device *adev)
 			  le32_to_cpu(firmware_usage-
 >asFirmwareVramReserveInfo[0].ulStartAddrUsedByFirmware),
 			  le16_to_cpu(firmware_usage-
 >asFirmwareVramReserveInfo[0].usFirmwareUseInKb));
 
-		usage_bytes = le16_to_cpu(firmware_usage-
Quoted 1 line

asFirmwareVramReserveInfo[0].usFirmwareUseInKb) * 1024;

drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c +1 -0
+		start_addr = firmware_usage-
Quoted 1 line

asFirmwareVramReserveInfo[0].ulStartAddrUsedByFirmware;

drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c +1 -0
+		size = firmware_usage-
Quoted 1 line

asFirmwareVramReserveInfo[0].usFirmwareUseInKb;

drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c +2 -0
+
+		if ((uint32_t)(start_addr &

ATOM_VRAM_OPERATION_FLAGS_MASK) ==
+
(uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATIO
N <<

drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c +4 -0
+			ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
+			/* Firmware request VRAM reservation for SR-IOV */
+			adev->fw_vram_usage.start_address_in_kb =
+			    start_addr &

(~ATOM_VRAM_OPERATION_FLAGS_MASK);

drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c +5 -0
+			adev->fw_vram_usage.size_in_kb = size;
+			/* Use the default scratch size */
+			usage_bytes = 0;
+		} else {
+			usage_bytes = le16_to_cpu(firmware_usage-
Quoted 1 line

asFirmwareVramReserveInfo[0].usFirmwareUseInKb) * 1024;

drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c +1 -0
+		}
 	}
 	ctx->scratch_size_bytes = 0;
 	if (usage_bytes == 0)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +43 -0
index a86d856..3da87cc 100644
@@ -658,6 +658,66 @@ void amdgpu_gart_location(struct amdgpu_device
 *adev, struct amdgpu_mc *mc)
 }
 
 /*
+ * Firmware Reservation function
+ */
+/**
+ * amdgpu_fw_reserve_vram_fini - free fw reserved vram
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * free fw reserved vram if it is reserved.
+ */
+void amdgpu_fw_reserve_vram_fini(struct amdgpu_device *adev)
+{
+
+	if (adev->fw_vram_usage.reserved_bo == NULL)
+		return;
+
+	amdgpu_bo_free_vram_specified(&adev-
 >fw_vram_usage.reserved_bo, NULL,
+		(void **)&adev->fw_vram_usage.va);
+	adev->fw_vram_usage.reserved_bo = NULL;
+}
+
+/**
+ * amdgpu_fw_reserve_vram_fini - create bo vram reservation from fw
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * create bo vram reservation from fw.
+ */
+int amdgpu_fw_reserve_vram_init(struct amdgpu_device *adev)
+{
+	int r = 0;
+	u64 gpu_addr;
+	u64 vram_size = adev->mc.visible_vram_size;
+	u64 start_addr;
+	u32 size;
+
+	adev->fw_vram_usage.va = NULL;
+	adev->fw_vram_usage.reserved_bo = NULL;
+
+	if (adev->fw_vram_usage.size_in_kb > 0 &&
+		KB_TO_BYTE((u64)adev->fw_vram_usage.size_in_kb) <=
 vram_size) {
+
+		start_addr =
+		    KB_TO_BYTE((u64)adev-
Quoted 1 line

fw_vram_usage.start_address_in_kb);

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +5 -0
+		size = KB_TO_BYTE((u64)adev->fw_vram_usage.size_in_kb);
+
+		r = amdgpu_bo_create_vram_specified(adev,
+			    start_addr, size, PAGE_SIZE,
+			    AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED

|

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +5 -0
+			    AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
+			    &adev->fw_vram_usage.reserved_bo,
+			    &gpu_addr, (void **)&adev->fw_vram_usage.va);
+		if (r)
+			dev_info(adev->dev, "Reserve VRAM for firmware

failed");

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +7 -0
+	}
+
+	return r;
+}
+
+
+/*
  * GPU helpers function.
  */
 /**

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c int amdgpu_device_init(struct amdgpu_device · line 2120

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1 -0
@@ -2055,6 +2115,7 @@ int amdgpu_device_init(struct amdgpu_device
 *adev,
 	adev->vm_manager.vm_pte_num_rings = 0;
 	adev->gart.gart_funcs = NULL;
 	adev->fence_context =
 dma_fence_context_alloc(AMDGPU_MAX_RINGS);
+	adev->fw_vram_usage.size_in_kb = 0;

No need to set this explicitly. The device structure is already initialized to 0 at allocation time.

Quoted 40 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
 	adev->smc_rreg = &amdgpu_invalid_rreg;
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1 -0
 	adev->smc_wreg = &amdgpu_invalid_wreg;
@@ -2331,6 +2392,7 @@ void amdgpu_device_fini(struct amdgpu_device
 *adev)
 	/* evict vram memory */
 	amdgpu_bo_evict_vram(adev);
 	amdgpu_ib_pool_fini(adev);
+	amdgpu_fw_reserve_vram_fini(adev);
 	amdgpu_fence_driver_fini(adev);
 	amdgpu_fbdev_fini(adev);
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
 	r = amdgpu_fini(adev);
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +0 -0
index 1086f03..a5253cf 100644
@@ -1256,6 +1256,15 @@ int amdgpu_ttm_init(struct amdgpu_device
 *adev)
 	/* Change the size here instead of the init above so only lpfn is

affected */
amdgpu_ttm_set_active_vram_size(adev, adev-

Quoted 1 line

mc.visible_vram_size);

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +9 -0
+	/*
+	 *The reserved vram for firmware must be pinned to the specified
+	 *place on the VRAM, so reserve it early.
+	 */
+	r = amdgpu_fw_reserve_vram_init(adev);
+	if (r) {
+		return r;
+	}
+
 	r = amdgpu_bo_create_kernel(adev, adev->mc.stolen_size,

PAGE_SIZE,

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +0 -1
 				    AMDGPU_GEM_DOMAIN_VRAM,
 				    &adev->stolen_vga_memory,
--

2.7.4

Quoted 4 lines

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Reviewed-by Alex Deucher
#8 lore raw
In reply to a message not in the archive
Quoted 9 lines

-----Original Message-----
From: amd-gfx [mailto:[email protected]] On Behalf Of Horace Chen
Sent: Wednesday, September 27, 2017 5:24 AM
To: [email protected]
Cc: Deucher, Alexander; Chen, Horace
Subject: [PATCH 1/3] drm/amdgpu: Add a new flag for SR-IOV to share memory between PF & VF

Quoted 5 lines

Add ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION to identify whether
driver need to reserve VRAM for SR-IOV shared memory.

Quoted 22 lines
---
 drivers/gpu/drm/amd/include/atombios.h | 1 +
 1 file changed, 1 insertion(+)
drivers/gpu/drm/amd/include/atombios.h +1 -0
index 181a2c3..f696bbb 100644
@@ -4292,6 +4292,7 @@ typedef struct _ATOM_DPCD_INFO
 #define ATOM_VRAM_OPERATION_FLAGS_SHIFT        30
 #define   ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION   0x1
 #define   ATOM_VRAM_BLOCK_NEEDS_RESERVATION      0x0
+#define   ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION 0x2
 
 
 /**********************************************************

*************************/
// Structure used in VRAM_UsageByFirmwareTable
--
2.7.4

Quoted 4 lines

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx