Skip to contentSkip to conversation

drm/amdgpu: extend mtype override to non-contiguous pages

PATCH In review Philip Yang 3 participants 3 messages Apr 9 – Apr 13

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 drm/amdgpu: extend mtype override to non-contiguous pages

On multi-socket MI300A APU systems, system memory pages mapped to the closest GPU must use MTYPE_RW instead of MTYPE_NC to maintain correct cache coherence. The existing mtype override in amdgpu_vm_pte_update_flags() excluded non-contiguous page mappings from the override. This caused incorrect MTYPE_NC for scattered local pages, leading to cache coherence issues.

The override applies to both contiguous and non-contiguous mappings. When pages_addr is set, resolve the physical address via pages_addr[addr >> PAGE_SHIFT] before passing it to the override callback for NUMA node lookup.

Introduce amdgpu_vm_addr_contiguous() helper that, on MI300A, treats pages on different NUMA nodes as non-contiguous even if their DMA addresses are adjacent. This ensures amdgpu_vm_update_range() splits page table updates at NUMA node boundaries so each batch gets the correct mtype override.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c    | 51 +++++++++++++++++++----
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 14 +++++--
 2 files changed, 53 insertions(+), 12 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +43 -8
index 63156289ae7f..7b7cbe054d73 100644
@@ -1099,6 +1099,32 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params,
 	}
 }
 
+/**
+ * amdgpu_vm_addr_same_group - check if two DMA addresses are same contiguity state
+ *
+ * @same_nid: true to check if two address on same NUMA node
+ * @addr: current DMA address
+ * @addr_next: next DMA address to check against
+ * @contiguous: current contiguity state of the range being built
+ *
+ * Check whether @addr and @addr_next are physically contiguous. On APU
+ * platforms with multiple NUMA nodes (e.g. MI300A), a NUMA node boundary
+ * also breaks contiguity so that each contiguous batch stays within a
+ * single NUMA node for correct MTYPE override selection.
+ *
+ * Returns:
+ * true if @addr_next continues the current contiguous range, false otherwise.
+ */
+static inline bool amdgpu_vm_addr_same_group(bool same_nid, dma_addr_t addr,
+					     dma_addr_t addr_next, bool contiguous)
+{
+	if (same_nid && page_is_ram(addr >> PAGE_SHIFT) &&
+	    pfn_to_nid(addr >> PAGE_SHIFT) != pfn_to_nid(addr_next >> PAGE_SHIFT))
+		return !contiguous;
+
+	return (addr + PAGE_SIZE) == addr_next;
+}
+
 /**
  * amdgpu_vm_update_range - update a range in the vm page table
  *
@@ -1134,6 +1160,7 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	struct amdgpu_vm_tlb_seq_struct *tlb_cb;
 	struct amdgpu_vm_update_params params;
 	struct amdgpu_res_cursor cursor;
+	bool same_nid;
 	int r, idx;
 
 	if (!drm_dev_enter(adev_to_drm(adev), &idx))
@@ -1166,6 +1193,10 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	params.allow_override = allow_override;
 	INIT_LIST_HEAD(&params.tlb_flush_waitlist);
 
+	same_nid = adev->gmc.is_app_apu && adev->ram_is_direct_mapped &&
+		   adev->gmc.gmc_funcs->override_vm_pte_flags &&
+		   num_possible_nodes() > 1 && params.allow_override;
+
 	amdgpu_vm_eviction_lock(vm);
 	if (vm->evicting) {
 		r = -EBUSY;
@@ -1198,22 +1229,26 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 				uint64_t pfn = cursor.start >> PAGE_SHIFT;
 				uint64_t count;
 
-				contiguous = pages_addr[pfn + 1] ==
-					pages_addr[pfn] + PAGE_SIZE;
+				contiguous = amdgpu_vm_addr_same_group(same_nid,
+								       pages_addr[pfn],
+								       pages_addr[pfn + 1],
+								       contiguous);
 
-				tmp = num_entries /
-					AMDGPU_GPU_PAGES_IN_CPU_PAGE;
+				tmp = num_entries / AMDGPU_GPU_PAGES_IN_CPU_PAGE;
 				for (count = 2; count < tmp; ++count) {
 					uint64_t idx = pfn + count;
 
-					if (contiguous != (pages_addr[idx] ==
-					    pages_addr[idx - 1] + PAGE_SIZE))
+					if (contiguous != amdgpu_vm_addr_same_group(same_nid,
+									pages_addr[idx - 1],
+									pages_addr[idx],
+									contiguous))
 						break;
 				}
+
 				if (!contiguous)
 					count--;
-				num_entries = count *
-					AMDGPU_GPU_PAGES_IN_CPU_PAGE;
+
+				num_entries = count * AMDGPU_GPU_PAGES_IN_CPU_PAGE;
 			}
 
 			if (!contiguous) {
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +10 -4
index 31a437ce9570..9e1607fb3b2e 100644
@@ -708,13 +708,19 @@ static void amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params *params,
 		amdgpu_vm_pte_update_noretry_flags(adev, &flags);
 
 	/* APUs mapping system memory may need different MTYPEs on different
-	 * NUMA nodes. Only do this for contiguous ranges that can be assumed
-	 * to be on the same NUMA node.
+	 * NUMA nodes. Both contiguous and non-contiguous ranges are handled
+	 * since amdgpu_vm_update_range ensures updates don't span NUMA
+	 * node boundaries.
 	 */
 	if ((flags & AMDGPU_PTE_SYSTEM) && (adev->flags & AMD_IS_APU) &&
 	    adev->gmc.gmc_funcs->override_vm_pte_flags &&
-	    num_possible_nodes() > 1 && !params->pages_addr && params->allow_override)
-		amdgpu_gmc_override_vm_pte_flags(adev, params->vm, addr, &flags);
+	    num_possible_nodes() > 1 && params->allow_override) {
+		if (params->pages_addr)
+			amdgpu_gmc_override_vm_pte_flags(adev, params->vm,
+					params->pages_addr[addr >> PAGE_SHIFT], &flags);
+		else
+			amdgpu_gmc_override_vm_pte_flags(adev, params->vm, addr, &flags);
+	}
 
 	params->vm->update_funcs->update(params, pt, pe, addr, count, incr,
 					 flags);
Signature · 1 line
2.50.1
maintainer
#1 lore raw
Quoted 44 lines · Philip Yang

On multi-socket MI300A APU systems, system memory pages mapped to the closest GPU must use MTYPE_RW instead of MTYPE_NC to maintain correct cache coherence. The existing mtype override in amdgpu_vm_pte_update_flags() excluded non-contiguous page mappings from the override. This caused incorrect MTYPE_NC for scattered local pages, leading to cache coherence issues.

The override applies to both contiguous and non-contiguous mappings. When pages_addr is set, resolve the physical address via pages_addr[addr >> PAGE_SHIFT] before passing it to the override callback for NUMA node lookup.

Introduce amdgpu_vm_addr_contiguous() helper that, on MI300A, treats pages on different NUMA nodes as non-contiguous even if their DMA addresses are adjacent. This ensures amdgpu_vm_update_range() splits page table updates at NUMA node boundaries so each batch gets the correct mtype override.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c    | 51 +++++++++++++++++++----
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 14 +++++--
 2 files changed, 53 insertions(+), 12 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +12 -0
index 63156289ae7f..7b7cbe054d73 100644
@@ -1099,6 +1099,32 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params,
 	}
 }
 
+/**
+ * amdgpu_vm_addr_same_group - check if two DMA addresses are same contiguity state
+ *
+ * @same_nid: true to check if two address on same NUMA node
+ * @addr: current DMA address
+ * @addr_next: next DMA address to check against
+ * @contiguous: current contiguity state of the range being built
+ *
+ * Check whether @addr and @addr_next are physically contiguous. On APU
+ * platforms with multiple NUMA nodes (e.g. MI300A), a NUMA node boundary
+ * also breaks contiguity so that each contiguous batch stays within a
+ * single NUMA node for correct MTYPE override selection.

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params, · line 1123

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +10 -0
+ *
+ * Returns:
+ * true if @addr_next continues the current contiguous range, false otherwise.
+ */
+static inline bool amdgpu_vm_addr_same_group(bool same_nid, dma_addr_t addr,
+					     dma_addr_t addr_next, bool contiguous)
+{
+	if (same_nid && page_is_ram(addr >> PAGE_SHIFT) &&
+	    pfn_to_nid(addr >> PAGE_SHIFT) != pfn_to_nid(addr_next >> PAGE_SHIFT))
+		return !contiguous;

This relys on that the dma_addr is equal to the PFN and is a clear NO-GO from my side to that approach.

Question is why exactly would we need that in the first place?

Regards,
Christian.

Quoted 89 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +21 -8
+
+	return (addr + PAGE_SIZE) == addr_next;
+}
+
 /**
  * amdgpu_vm_update_range - update a range in the vm page table
  *
@@ -1134,6 +1160,7 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	struct amdgpu_vm_tlb_seq_struct *tlb_cb;
 	struct amdgpu_vm_update_params params;
 	struct amdgpu_res_cursor cursor;
+	bool same_nid;
 	int r, idx;
 
 	if (!drm_dev_enter(adev_to_drm(adev), &idx))
@@ -1166,6 +1193,10 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	params.allow_override = allow_override;
 	INIT_LIST_HEAD(&params.tlb_flush_waitlist);
 
+	same_nid = adev->gmc.is_app_apu && adev->ram_is_direct_mapped &&
+		   adev->gmc.gmc_funcs->override_vm_pte_flags &&
+		   num_possible_nodes() > 1 && params.allow_override;
+
 	amdgpu_vm_eviction_lock(vm);
 	if (vm->evicting) {
 		r = -EBUSY;
@@ -1198,22 +1229,26 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 				uint64_t pfn = cursor.start >> PAGE_SHIFT;
 				uint64_t count;
 
-				contiguous = pages_addr[pfn + 1] ==
-					pages_addr[pfn] + PAGE_SIZE;
+				contiguous = amdgpu_vm_addr_same_group(same_nid,
+								       pages_addr[pfn],
+								       pages_addr[pfn + 1],
+								       contiguous);
 
-				tmp = num_entries /
-					AMDGPU_GPU_PAGES_IN_CPU_PAGE;
+				tmp = num_entries / AMDGPU_GPU_PAGES_IN_CPU_PAGE;
 				for (count = 2; count < tmp; ++count) {
 					uint64_t idx = pfn + count;
 
-					if (contiguous != (pages_addr[idx] ==
-					    pages_addr[idx - 1] + PAGE_SIZE))
+					if (contiguous != amdgpu_vm_addr_same_group(same_nid,
+									pages_addr[idx - 1],
+									pages_addr[idx],
+									contiguous))
 						break;
 				}
+
 				if (!contiguous)
 					count--;
-				num_entries = count *
-					AMDGPU_GPU_PAGES_IN_CPU_PAGE;
+
+				num_entries = count * AMDGPU_GPU_PAGES_IN_CPU_PAGE;
 			}
 
 			if (!contiguous) {
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +10 -4
index 31a437ce9570..9e1607fb3b2e 100644
@@ -708,13 +708,19 @@ static void amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params *params,
 		amdgpu_vm_pte_update_noretry_flags(adev, &flags);
 
 	/* APUs mapping system memory may need different MTYPEs on different
-	 * NUMA nodes. Only do this for contiguous ranges that can be assumed
-	 * to be on the same NUMA node.
+	 * NUMA nodes. Both contiguous and non-contiguous ranges are handled
+	 * since amdgpu_vm_update_range ensures updates don't span NUMA
+	 * node boundaries.
 	 */
 	if ((flags & AMDGPU_PTE_SYSTEM) && (adev->flags & AMD_IS_APU) &&
 	    adev->gmc.gmc_funcs->override_vm_pte_flags &&
-	    num_possible_nodes() > 1 && !params->pages_addr && params->allow_override)
-		amdgpu_gmc_override_vm_pte_flags(adev, params->vm, addr, &flags);
+	    num_possible_nodes() > 1 && params->allow_override) {
+		if (params->pages_addr)
+			amdgpu_gmc_override_vm_pte_flags(adev, params->vm,
+					params->pages_addr[addr >> PAGE_SHIFT], &flags);
+		else
+			amdgpu_gmc_override_vm_pte_flags(adev, params->vm, addr, &flags);
+	}
 
 	params->vm->update_funcs->update(params, pt, pe, addr, count, incr,
 					 flags);
#2 lore raw
Quoted 49 lines · Christian König
Quoted 48 lines · Philip Yang

On multi-socket MI300A APU systems, system memory pages mapped to the closest GPU must use MTYPE_RW instead of MTYPE_NC to maintain correct cache coherence. The existing mtype override in amdgpu_vm_pte_update_flags() excluded non-contiguous page mappings from the override. This caused incorrect MTYPE_NC for scattered local pages, leading to cache coherence issues.

The override applies to both contiguous and non-contiguous mappings. When pages_addr is set, resolve the physical address via pages_addr[addr >> PAGE_SHIFT] before passing it to the override callback for NUMA node lookup.

Introduce amdgpu_vm_addr_contiguous() helper that, on MI300A, treats pages on different NUMA nodes as non-contiguous even if their DMA addresses are adjacent. This ensures amdgpu_vm_update_range() splits page table updates at NUMA node boundaries so each batch gets the correct mtype override.

---
  drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c    | 51 +++++++++++++++++++----
  drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 14 +++++--
  2 files changed, 53 insertions(+), 12 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +16 -0
index 63156289ae7f..7b7cbe054d73 100644
@@ -1099,6 +1099,32 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params,
  	}
  }
 
+/**
+ * amdgpu_vm_addr_same_group - check if two DMA addresses are same contiguity state
+ *
+ * @same_nid: true to check if two address on same NUMA node
+ * @addr: current DMA address
+ * @addr_next: next DMA address to check against
+ * @contiguous: current contiguity state of the range being built
+ *
+ * Check whether @addr and @addr_next are physically contiguous. On APU
+ * platforms with multiple NUMA nodes (e.g. MI300A), a NUMA node boundary
+ * also breaks contiguity so that each contiguous batch stays within a
+ * single NUMA node for correct MTYPE override selection.
+ *
+ * Returns:
+ * true if @addr_next continues the current contiguous range, false otherwise.
+ */
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params, · line 1123

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +6 -0
+static inline bool amdgpu_vm_addr_same_group(bool same_nid, dma_addr_t addr,
+					     dma_addr_t addr_next, bool contiguous)
+{
+	if (same_nid && page_is_ram(addr >> PAGE_SHIFT) &&
+	    pfn_to_nid(addr >> PAGE_SHIFT) != pfn_to_nid(addr_next >> PAGE_SHIFT))
+		return !contiguous;

This relys on that the dma_addr is equal to the PFN and is a clear NO-GO from my side to that approach.

To add pages array for this seems too much overhead, to support mtype override if adev->ram_is_direct_mapped is enough.

Quoted 2 lines

Question is why exactly would we need that in the first place?

MI300A with NPS2/4 or MI300A 4P, should use mtype RW to map on the near compute partition or GPU, to maintain hw cache coherence.

As discussed offline, app may alloc with uncached flag to switch mtype, the test don't show issue when switching mtype b/w RW and CC.

Regards,
Philip

Quoted 93 lines

Regards,
Christian.

Quoted 89 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +21 -8
+
+	return (addr + PAGE_SIZE) == addr_next;
+}
+
  /**
   * amdgpu_vm_update_range - update a range in the vm page table
   *
@@ -1134,6 +1160,7 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
  	struct amdgpu_vm_tlb_seq_struct *tlb_cb;
  	struct amdgpu_vm_update_params params;
  	struct amdgpu_res_cursor cursor;
+	bool same_nid;
  	int r, idx;
 
  	if (!drm_dev_enter(adev_to_drm(adev), &idx))
@@ -1166,6 +1193,10 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
  	params.allow_override = allow_override;
  	INIT_LIST_HEAD(&params.tlb_flush_waitlist);
 
+	same_nid = adev->gmc.is_app_apu && adev->ram_is_direct_mapped &&
+		   adev->gmc.gmc_funcs->override_vm_pte_flags &&
+		   num_possible_nodes() > 1 && params.allow_override;
+
  	amdgpu_vm_eviction_lock(vm);
  	if (vm->evicting) {
  		r = -EBUSY;
@@ -1198,22 +1229,26 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
  				uint64_t pfn = cursor.start >> PAGE_SHIFT;
  				uint64_t count;
 
-				contiguous = pages_addr[pfn + 1] ==
-					pages_addr[pfn] + PAGE_SIZE;
+				contiguous = amdgpu_vm_addr_same_group(same_nid,
+								       pages_addr[pfn],
+								       pages_addr[pfn + 1],
+								       contiguous);
 
-				tmp = num_entries /
-					AMDGPU_GPU_PAGES_IN_CPU_PAGE;
+				tmp = num_entries / AMDGPU_GPU_PAGES_IN_CPU_PAGE;
  				for (count = 2; count < tmp; ++count) {
  					uint64_t idx = pfn + count;
 
-					if (contiguous != (pages_addr[idx] ==
-					    pages_addr[idx - 1] + PAGE_SIZE))
+					if (contiguous != amdgpu_vm_addr_same_group(same_nid,
+									pages_addr[idx - 1],
+									pages_addr[idx],
+									contiguous))
  						break;
  				}
+
  				if (!contiguous)
  					count--;
-				num_entries = count *
-					AMDGPU_GPU_PAGES_IN_CPU_PAGE;
+
+				num_entries = count * AMDGPU_GPU_PAGES_IN_CPU_PAGE;
  			}
 
  			if (!contiguous) {
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +10 -4
index 31a437ce9570..9e1607fb3b2e 100644
@@ -708,13 +708,19 @@ static void amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params *params,
  		amdgpu_vm_pte_update_noretry_flags(adev, &flags);
 
  	/* APUs mapping system memory may need different MTYPEs on different
-	 * NUMA nodes. Only do this for contiguous ranges that can be assumed
-	 * to be on the same NUMA node.
+	 * NUMA nodes. Both contiguous and non-contiguous ranges are handled
+	 * since amdgpu_vm_update_range ensures updates don't span NUMA
+	 * node boundaries.
  	 */
  	if ((flags & AMDGPU_PTE_SYSTEM) && (adev->flags & AMD_IS_APU) &&
  	    adev->gmc.gmc_funcs->override_vm_pte_flags &&
-	    num_possible_nodes() > 1 && !params->pages_addr && params->allow_override)
-		amdgpu_gmc_override_vm_pte_flags(adev, params->vm, addr, &flags);
+	    num_possible_nodes() > 1 && params->allow_override) {
+		if (params->pages_addr)
+			amdgpu_gmc_override_vm_pte_flags(adev, params->vm,
+					params->pages_addr[addr >> PAGE_SHIFT], &flags);
+		else
+			amdgpu_gmc_override_vm_pte_flags(adev, params->vm, addr, &flags);
+	}
 
  	params->vm->update_funcs->update(params, pt, pe, addr, count, incr,
  					 flags);