Skip to contentSkip to conversation

drm/amdgpu: extend mtype override to non-contiguous pages

PATCH Superseded Philip Yang 5 participants 8 messages Apr 7 – Apr 10

Subsystems (2)

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 drm/amdgpu: extend mtype override to non-contiguous pages

On multi-socket MI300A APU systems, system memory pages mapped to the closest GPU must use MTYPE_RW instead of MTYPE_NC to maintain correct cache coherence. The existing mtype override in amdgpu_vm_pte_update_flags() excluded non-contiguous page mappings from the override. This caused incorrect MTYPE_NC for scattered local pages, leading to cache coherence issues.

The override applies to both contiguous and non-contiguous mappings. When pages_addr is set, resolve the physical address via pages_addr[addr >> PAGE_SHIFT] before passing it to the override callback for NUMA node lookup.

Introduce amdgpu_vm_addr_contiguous() helper that, on MI300A, treats pages on different NUMA nodes as non-contiguous even if their DMA addresses are adjacent. This ensures amdgpu_vm_update_range() splits page table updates at NUMA node boundaries so each batch gets the correct mtype override.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c    | 48 +++++++++++++++++++----
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 14 +++++--
 2 files changed, 50 insertions(+), 12 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +40 -8
index 63156289ae7f..f8fcbf079bf4 100644
@@ -1099,6 +1099,34 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params,
 	}
 }
 
+/**
+ * amdgpu_vm_addr_contiguous - check if two DMA addresses are contiguous
+ *
+ * @adev: amdgpu_device pointer
+ * @addr: current DMA address
+ * @addr_next: next DMA address to check against
+ * @contiguous: current contiguity state of the range being built
+ *
+ * Check whether @addr and @addr_next are physically contiguous. On APU
+ * platforms with multiple NUMA nodes (e.g. MI300A), a NUMA node boundary
+ * also breaks contiguity so that each contiguous batch stays within a
+ * single NUMA node for correct MTYPE override selection.
+ *
+ * Returns:
+ * true if @addr_next continues the current contiguous range, false otherwise.
+ */
+static inline bool amdgpu_vm_addr_contiguous(struct amdgpu_device *adev, dma_addr_t addr,
+					     dma_addr_t addr_next, bool contiguous)
+{
+	if (!adev->gmc.is_app_apu || !page_is_ram(addr >> PAGE_SHIFT))
+		return (addr + PAGE_SIZE) == addr_next;
+
+	if (pfn_to_nid(addr >> PAGE_SHIFT) != pfn_to_nid(addr_next >> PAGE_SHIFT))
+		return !contiguous;
+
+	return (addr + PAGE_SIZE) == addr_next;
+}
+
 /**
  * amdgpu_vm_update_range - update a range in the vm page table
  *
@@ -1198,22 +1226,26 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 				uint64_t pfn = cursor.start >> PAGE_SHIFT;
 				uint64_t count;
 
-				contiguous = pages_addr[pfn + 1] ==
-					pages_addr[pfn] + PAGE_SIZE;
+				contiguous = amdgpu_vm_addr_contiguous(adev,
+								       pages_addr[pfn],
+								       pages_addr[pfn + 1],
+								       contiguous);
 
-				tmp = num_entries /
-					AMDGPU_GPU_PAGES_IN_CPU_PAGE;
+				tmp = num_entries / AMDGPU_GPU_PAGES_IN_CPU_PAGE;
 				for (count = 2; count < tmp; ++count) {
 					uint64_t idx = pfn + count;
 
-					if (contiguous != (pages_addr[idx] ==
-					    pages_addr[idx - 1] + PAGE_SIZE))
+					if (contiguous != amdgpu_vm_addr_contiguous(adev,
+									pages_addr[idx - 1],
+									pages_addr[idx],
+									contiguous))
 						break;
 				}
+
 				if (!contiguous)
 					count--;
-				num_entries = count *
-					AMDGPU_GPU_PAGES_IN_CPU_PAGE;
+
+				num_entries = count * AMDGPU_GPU_PAGES_IN_CPU_PAGE;
 			}
 
 			if (!contiguous) {
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +10 -4
index 31a437ce9570..9e1607fb3b2e 100644
@@ -708,13 +708,19 @@ static void amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params *params,
 		amdgpu_vm_pte_update_noretry_flags(adev, &flags);
 
 	/* APUs mapping system memory may need different MTYPEs on different
-	 * NUMA nodes. Only do this for contiguous ranges that can be assumed
-	 * to be on the same NUMA node.
+	 * NUMA nodes. Both contiguous and non-contiguous ranges are handled
+	 * since amdgpu_vm_update_range ensures updates don't span NUMA
+	 * node boundaries.
 	 */
 	if ((flags & AMDGPU_PTE_SYSTEM) && (adev->flags & AMD_IS_APU) &&
 	    adev->gmc.gmc_funcs->override_vm_pte_flags &&
-	    num_possible_nodes() > 1 && !params->pages_addr && params->allow_override)
-		amdgpu_gmc_override_vm_pte_flags(adev, params->vm, addr, &flags);
+	    num_possible_nodes() > 1 && params->allow_override) {
+		if (params->pages_addr)
+			amdgpu_gmc_override_vm_pte_flags(adev, params->vm,
+					params->pages_addr[addr >> PAGE_SHIFT], &flags);
+		else
+			amdgpu_gmc_override_vm_pte_flags(adev, params->vm, addr, &flags);
+	}
 
 	params->vm->update_funcs->update(params, pt, pe, addr, count, incr,
 					 flags);
Signature · 1 line
2.50.1
#1 lore raw
Quoted 38 lines · Philip Yang

On multi-socket MI300A APU systems, system memory pages mapped to the closest GPU must use MTYPE_RW instead of MTYPE_NC to maintain correct cache coherence. The existing mtype override in amdgpu_vm_pte_update_flags() excluded non-contiguous page mappings from the override. This caused incorrect MTYPE_NC for scattered local pages, leading to cache coherence issues.

The override applies to both contiguous and non-contiguous mappings. When pages_addr is set, resolve the physical address via pages_addr[addr >> PAGE_SHIFT] before passing it to the override callback for NUMA node lookup.

Introduce amdgpu_vm_addr_contiguous() helper that, on MI300A, treats pages on different NUMA nodes as non-contiguous even if their DMA addresses are adjacent. This ensures amdgpu_vm_update_range() splits page table updates at NUMA node boundaries so each batch gets the correct mtype override.

---
  drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c    | 48 +++++++++++++++++++----
  drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 14 +++++--
  2 files changed, 50 insertions(+), 12 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +6 -0
index 63156289ae7f..f8fcbf079bf4 100644
@@ -1099,6 +1099,34 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params,
  	}
  }
 
+/**
+ * amdgpu_vm_addr_contiguous - check if two DMA addresses are contiguous
+ *
+ * @adev: amdgpu_device pointer
+ * @addr: current DMA address
+ * @addr_next: next DMA address to check against

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params, · line 1117

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +10 -0
+ * @contiguous: current contiguity state of the range being built
+ *
+ * Check whether @addr and @addr_next are physically contiguous. On APU
+ * platforms with multiple NUMA nodes (e.g. MI300A), a NUMA node boundary
+ * also breaks contiguity so that each contiguous batch stays within a
+ * single NUMA node for correct MTYPE override selection.
+ *
+ * Returns:
+ * true if @addr_next continues the current contiguous range, false otherwise.
+ */

We can use pfn_to_nid or page_to_nid to get which noma(id) the backing memory is at. pfn_to_nid uses pfn from physical address. You use dma_addr_t that is device dependent. It is not always same as physical address of RAM.

ttm_tt also has

/** @pages: Array of pages backing the data. */ struct page **pages;

I think using the pages to get numa id by page_to_nid is more appropriate.

Regards

Xiaogang

Quoted 78 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +24 -8
+static inline bool amdgpu_vm_addr_contiguous(struct amdgpu_device *adev, dma_addr_t addr,
+					     dma_addr_t addr_next, bool contiguous)
+{
+	if (!adev->gmc.is_app_apu || !page_is_ram(addr >> PAGE_SHIFT))
+		return (addr + PAGE_SIZE) == addr_next;
+
+	if (pfn_to_nid(addr >> PAGE_SHIFT) != pfn_to_nid(addr_next >> PAGE_SHIFT))
+		return !contiguous;
+
+	return (addr + PAGE_SIZE) == addr_next;
+}
+
  /**
   * amdgpu_vm_update_range - update a range in the vm page table
   *
@@ -1198,22 +1226,26 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
  				uint64_t pfn = cursor.start >> PAGE_SHIFT;
  				uint64_t count;
 
-				contiguous = pages_addr[pfn + 1] ==
-					pages_addr[pfn] + PAGE_SIZE;
+				contiguous = amdgpu_vm_addr_contiguous(adev,
+								       pages_addr[pfn],
+								       pages_addr[pfn + 1],
+								       contiguous);
 
-				tmp = num_entries /
-					AMDGPU_GPU_PAGES_IN_CPU_PAGE;
+				tmp = num_entries / AMDGPU_GPU_PAGES_IN_CPU_PAGE;
  				for (count = 2; count < tmp; ++count) {
  					uint64_t idx = pfn + count;
 
-					if (contiguous != (pages_addr[idx] ==
-					    pages_addr[idx - 1] + PAGE_SIZE))
+					if (contiguous != amdgpu_vm_addr_contiguous(adev,
+									pages_addr[idx - 1],
+									pages_addr[idx],
+									contiguous))
  						break;
  				}
+
  				if (!contiguous)
  					count--;
-				num_entries = count *
-					AMDGPU_GPU_PAGES_IN_CPU_PAGE;
+
+				num_entries = count * AMDGPU_GPU_PAGES_IN_CPU_PAGE;
  			}
 
  			if (!contiguous) {
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +10 -4
index 31a437ce9570..9e1607fb3b2e 100644
@@ -708,13 +708,19 @@ static void amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params *params,
  		amdgpu_vm_pte_update_noretry_flags(adev, &flags);
 
  	/* APUs mapping system memory may need different MTYPEs on different
-	 * NUMA nodes. Only do this for contiguous ranges that can be assumed
-	 * to be on the same NUMA node.
+	 * NUMA nodes. Both contiguous and non-contiguous ranges are handled
+	 * since amdgpu_vm_update_range ensures updates don't span NUMA
+	 * node boundaries.
  	 */
  	if ((flags & AMDGPU_PTE_SYSTEM) && (adev->flags & AMD_IS_APU) &&
  	    adev->gmc.gmc_funcs->override_vm_pte_flags &&
-	    num_possible_nodes() > 1 && !params->pages_addr && params->allow_override)
-		amdgpu_gmc_override_vm_pte_flags(adev, params->vm, addr, &flags);
+	    num_possible_nodes() > 1 && params->allow_override) {
+		if (params->pages_addr)
+			amdgpu_gmc_override_vm_pte_flags(adev, params->vm,
+					params->pages_addr[addr >> PAGE_SHIFT], &flags);
+		else
+			amdgpu_gmc_override_vm_pte_flags(adev, params->vm, addr, &flags);
+	}
 
  	params->vm->update_funcs->update(params, pt, pe, addr, count, incr,
  					 flags);
#2 lore raw
Quoted 54 lines · Xiaogang
Quoted 38 lines · Philip Yang

On multi-socket MI300A APU systems, system memory pages mapped to the closest GPU must use MTYPE_RW instead of MTYPE_NC to maintain correct cache coherence. The existing mtype override in amdgpu_vm_pte_update_flags() excluded non-contiguous page mappings from the override. This caused incorrect MTYPE_NC for scattered local pages, leading to cache coherence issues.

The override applies to both contiguous and non-contiguous mappings. When pages_addr is set, resolve the physical address via pages_addr[addr >> PAGE_SHIFT] before passing it to the override callback for NUMA node lookup.

Introduce amdgpu_vm_addr_contiguous() helper that, on MI300A, treats pages on different NUMA nodes as non-contiguous even if their DMA addresses are adjacent. This ensures amdgpu_vm_update_range() splits page table updates at NUMA node boundaries so each batch gets the correct mtype override.

---
  drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c    | 48 +++++++++++++++++++----
  drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 14 +++++--
  2 files changed, 50 insertions(+), 12 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +6 -0
index 63156289ae7f..f8fcbf079bf4 100644
@@ -1099,6 +1099,34 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params,
  	}
  }
 
+/**
+ * amdgpu_vm_addr_contiguous - check if two DMA addresses are contiguous
+ *
+ * @adev: amdgpu_device pointer
+ * @addr: current DMA address
+ * @addr_next: next DMA address to check against

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params, · line 1117

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +10 -0
+ * @contiguous: current contiguity state of the range being built
+ *
+ * Check whether @addr and @addr_next are physically contiguous. On APU
+ * platforms with multiple NUMA nodes (e.g. MI300A), a NUMA node boundary
+ * also breaks contiguity so that each contiguous batch stays within a
+ * single NUMA node for correct MTYPE override selection.
+ *
+ * Returns:
+ * true if @addr_next continues the current contiguous range, false otherwise.
+ */

We can use pfn_to_nid or page_to_nid to get which noma(id) the backing memory is at. pfn_to_nid uses pfn from physical address. You use dma_addr_t that is device dependent. It is not always same as physical address of RAM.

Quoted 6 lines

ttm_tt also has

/** @pages: Array of pages backing the data. */ struct page **pages;

I think using the pages to get numa id by page_to_nid is more appropriate.

For MI300A APU, iommu is not enabled, dma_address (dma_map_page) is same as pfn (page_to_pfn), otherwise the current override path will not work correctly with pfn_to_nid.

Regards,
Philip

Quoted 83 lines

Regards

Xiaogang

Quoted 78 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +24 -8
+static inline bool amdgpu_vm_addr_contiguous(struct amdgpu_device *adev, dma_addr_t addr,
+					     dma_addr_t addr_next, bool contiguous)
+{
+	if (!adev->gmc.is_app_apu || !page_is_ram(addr >> PAGE_SHIFT))
+		return (addr + PAGE_SIZE) == addr_next;
+
+	if (pfn_to_nid(addr >> PAGE_SHIFT) != pfn_to_nid(addr_next >> PAGE_SHIFT))
+		return !contiguous;
+
+	return (addr + PAGE_SIZE) == addr_next;
+}
+
  /**
   * amdgpu_vm_update_range - update a range in the vm page table
   *
@@ -1198,22 +1226,26 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
  				uint64_t pfn = cursor.start >> PAGE_SHIFT;
  				uint64_t count;
 
-				contiguous = pages_addr[pfn + 1] ==
-					pages_addr[pfn] + PAGE_SIZE;
+				contiguous = amdgpu_vm_addr_contiguous(adev,
+								       pages_addr[pfn],
+								       pages_addr[pfn + 1],
+								       contiguous);
 
-				tmp = num_entries /
-					AMDGPU_GPU_PAGES_IN_CPU_PAGE;
+				tmp = num_entries / AMDGPU_GPU_PAGES_IN_CPU_PAGE;
  				for (count = 2; count < tmp; ++count) {
  					uint64_t idx = pfn + count;
 
-					if (contiguous != (pages_addr[idx] ==
-					    pages_addr[idx - 1] + PAGE_SIZE))
+					if (contiguous != amdgpu_vm_addr_contiguous(adev,
+									pages_addr[idx - 1],
+									pages_addr[idx],
+									contiguous))
  						break;
  				}
+
  				if (!contiguous)
  					count--;
-				num_entries = count *
-					AMDGPU_GPU_PAGES_IN_CPU_PAGE;
+
+				num_entries = count * AMDGPU_GPU_PAGES_IN_CPU_PAGE;
  			}
 
  			if (!contiguous) {
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +10 -4
index 31a437ce9570..9e1607fb3b2e 100644
@@ -708,13 +708,19 @@ static void amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params *params,
  		amdgpu_vm_pte_update_noretry_flags(adev, &flags);
 
  	/* APUs mapping system memory may need different MTYPEs on different
-	 * NUMA nodes. Only do this for contiguous ranges that can be assumed
-	 * to be on the same NUMA node.
+	 * NUMA nodes. Both contiguous and non-contiguous ranges are handled
+	 * since amdgpu_vm_update_range ensures updates don't span NUMA
+	 * node boundaries.
  	 */
  	if ((flags & AMDGPU_PTE_SYSTEM) && (adev->flags & AMD_IS_APU) &&
  	    adev->gmc.gmc_funcs->override_vm_pte_flags &&
-	    num_possible_nodes() > 1 && !params->pages_addr && params->allow_override)
-		amdgpu_gmc_override_vm_pte_flags(adev, params->vm, addr, &flags);
+	    num_possible_nodes() > 1 && params->allow_override) {
+		if (params->pages_addr)
+			amdgpu_gmc_override_vm_pte_flags(adev, params->vm,
+					params->pages_addr[addr >> PAGE_SHIFT], &flags);
+		else
+			amdgpu_gmc_override_vm_pte_flags(adev, params->vm, addr, &flags);
+	}
 
  	params->vm->update_funcs->update(params, pt, pe, addr, count, incr,
  					 flags);
maintainer
#3 lore raw
In reply to Chen, Xiaogang
Quoted 44 lines · Xiaogang
Quoted 42 lines · Philip Yang

On multi-socket MI300A APU systems, system memory pages mapped to the closest GPU must use MTYPE_RW instead of MTYPE_NC to maintain correct cache coherence. The existing mtype override in amdgpu_vm_pte_update_flags() excluded non-contiguous page mappings from the override. This caused incorrect MTYPE_NC for scattered local pages, leading to cache coherence issues.

The override applies to both contiguous and non-contiguous mappings. When pages_addr is set, resolve the physical address via pages_addr[addr >> PAGE_SHIFT] before passing it to the override callback for NUMA node lookup.

Introduce amdgpu_vm_addr_contiguous() helper that, on MI300A, treats pages on different NUMA nodes as non-contiguous even if their DMA addresses are adjacent. This ensures amdgpu_vm_update_range() splits page table updates at NUMA node boundaries so each batch gets the correct mtype override.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c    | 48 +++++++++++++++++++----
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 14 +++++--
 2 files changed, 50 insertions(+), 12 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +10 -0
index 63156289ae7f..f8fcbf079bf4 100644
@@ -1099,6 +1099,34 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params,
 	}
 }
 
+/**
+ * amdgpu_vm_addr_contiguous - check if two DMA addresses are contiguous
+ *
+ * @adev: amdgpu_device pointer
+ * @addr: current DMA address
+ * @addr_next: next DMA address to check against
+ * @contiguous: current contiguity state of the range being built
+ *
+ * Check whether @addr and @addr_next are physically contiguous. On APU
+ * platforms with multiple NUMA nodes (e.g. MI300A), a NUMA node boundary
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params, · line 1117

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +6 -0
+ * also breaks contiguity so that each contiguous batch stays within a
+ * single NUMA node for correct MTYPE override selection.
+ *
+ * Returns:
+ * true if @addr_next continues the current contiguous range, false otherwise.
+ */

We can use pfn_to_nid or page_to_nid to get which noma(id) the backing memory is at. pfn_to_nid uses pfn from physical address. You use dma_addr_t that is device dependent. It is not always same as physical address of RAM.

Yeah that here won't work at all.

Quoted 6 lines

ttm_tt also has

/** @pages: Array of pages backing the data. */ struct page **pages;

I think using the pages to get numa id by page_to_nid is more appropriate.

That array isn't filled in for imported pages.

As far as I can see the whole approach won't work reliable. For imports we only know the dma_addr and not the struct page nor the pfn.

I think we need to re-iterate the whole idea of MTYPE override.

Regards,
Christian.

Quoted 83 lines

Regards

Xiaogang

Quoted 78 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +24 -8
+static inline bool amdgpu_vm_addr_contiguous(struct amdgpu_device *adev, dma_addr_t addr,
+					     dma_addr_t addr_next, bool contiguous)
+{
+	if (!adev->gmc.is_app_apu || !page_is_ram(addr >> PAGE_SHIFT))
+		return (addr + PAGE_SIZE) == addr_next;
+
+	if (pfn_to_nid(addr >> PAGE_SHIFT) != pfn_to_nid(addr_next >> PAGE_SHIFT))
+		return !contiguous;
+
+	return (addr + PAGE_SIZE) == addr_next;
+}
+
 /**
  * amdgpu_vm_update_range - update a range in the vm page table
  *
@@ -1198,22 +1226,26 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 				uint64_t pfn = cursor.start >> PAGE_SHIFT;
 				uint64_t count;
 
-				contiguous = pages_addr[pfn + 1] ==
-					pages_addr[pfn] + PAGE_SIZE;
+				contiguous = amdgpu_vm_addr_contiguous(adev,
+								       pages_addr[pfn],
+								       pages_addr[pfn + 1],
+								       contiguous);
 
-				tmp = num_entries /
-					AMDGPU_GPU_PAGES_IN_CPU_PAGE;
+				tmp = num_entries / AMDGPU_GPU_PAGES_IN_CPU_PAGE;
 				for (count = 2; count < tmp; ++count) {
 					uint64_t idx = pfn + count;
 
-					if (contiguous != (pages_addr[idx] ==
-					    pages_addr[idx - 1] + PAGE_SIZE))
+					if (contiguous != amdgpu_vm_addr_contiguous(adev,
+									pages_addr[idx - 1],
+									pages_addr[idx],
+									contiguous))
 						break;
 				}
+
 				if (!contiguous)
 					count--;
-				num_entries = count *
-					AMDGPU_GPU_PAGES_IN_CPU_PAGE;
+
+				num_entries = count * AMDGPU_GPU_PAGES_IN_CPU_PAGE;
 			}
 
 			if (!contiguous) {
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +10 -4
index 31a437ce9570..9e1607fb3b2e 100644
@@ -708,13 +708,19 @@ static void amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params *params,
 		amdgpu_vm_pte_update_noretry_flags(adev, &flags);
 
 	/* APUs mapping system memory may need different MTYPEs on different
-	 * NUMA nodes. Only do this for contiguous ranges that can be assumed
-	 * to be on the same NUMA node.
+	 * NUMA nodes. Both contiguous and non-contiguous ranges are handled
+	 * since amdgpu_vm_update_range ensures updates don't span NUMA
+	 * node boundaries.
 	 */
 	if ((flags & AMDGPU_PTE_SYSTEM) && (adev->flags & AMD_IS_APU) &&
 	    adev->gmc.gmc_funcs->override_vm_pte_flags &&
-	    num_possible_nodes() > 1 && !params->pages_addr && params->allow_override)
-		amdgpu_gmc_override_vm_pte_flags(adev, params->vm, addr, &flags);
+	    num_possible_nodes() > 1 && params->allow_override) {
+		if (params->pages_addr)
+			amdgpu_gmc_override_vm_pte_flags(adev, params->vm,
+					params->pages_addr[addr >> PAGE_SHIFT], &flags);
+		else
+			amdgpu_gmc_override_vm_pte_flags(adev, params->vm, addr, &flags);
+	}
 
 	params->vm->update_funcs->update(params, pt, pe, addr, count, incr,
 					 flags);
#4 lore raw
Quoted 53 lines · Christian König
Quoted 42 lines · Xiaogang
Quoted 41 lines · Philip Yang

On multi-socket MI300A APU systems, system memory pages mapped to the closest GPU must use MTYPE_RW instead of MTYPE_NC to maintain correct cache coherence. The existing mtype override in amdgpu_vm_pte_update_flags() excluded non-contiguous page mappings from the override. This caused incorrect MTYPE_NC for scattered local pages, leading to cache coherence issues.

The override applies to both contiguous and non-contiguous mappings. When pages_addr is set, resolve the physical address via pages_addr[addr >> PAGE_SHIFT] before passing it to the override callback for NUMA node lookup.

Introduce amdgpu_vm_addr_contiguous() helper that, on MI300A, treats pages on different NUMA nodes as non-contiguous even if their DMA addresses are adjacent. This ensures amdgpu_vm_update_range() splits page table updates at NUMA node boundaries so each batch gets the correct mtype override.

---
  drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c    | 48 +++++++++++++++++++----
  drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 14 +++++--
  2 files changed, 50 insertions(+), 12 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +9 -0
index 63156289ae7f..f8fcbf079bf4 100644
@@ -1099,6 +1099,34 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params,
  	}
  }
 
+/**
+ * amdgpu_vm_addr_contiguous - check if two DMA addresses are contiguous
+ *
+ * @adev: amdgpu_device pointer
+ * @addr: current DMA address
+ * @addr_next: next DMA address to check against
+ * @contiguous: current contiguity state of the range being built
+ *
+ * Check whether @addr and @addr_next are physically contiguous. On APU
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params, · line 1117

Quoted 7 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +7 -0
+ * platforms with multiple NUMA nodes (e.g. MI300A), a NUMA node boundary
+ * also breaks contiguity so that each contiguous batch stays within a
+ * single NUMA node for correct MTYPE override selection.
+ *
+ * Returns:
+ * true if @addr_next continues the current contiguous range, false otherwise.
+ */

We can use pfn_to_nid or page_to_nid to get which noma(id) the backing memory is at. pfn_to_nid uses pfn from physical address. You use dma_addr_t that is device dependent. It is not always same as physical address of RAM.

Yeah that here won't work at all.

Quoted 8 lines
Quoted 5 lines

ttm_tt also has

/** @pages: Array of pages backing the data. */ struct page **pages;

I think using the pages to get numa id by page_to_nid is more appropriate.

That array isn't filled in for imported pages.

As far as I can see the whole approach won't work reliable. For imports we only know the dma_addr and not the struct page nor the pfn.

if adev->ram_is_direct_mapped, then dma_addr equals to pfn, we can use dma_addr to call pfn_to_nid.
I will add ram_is_direct_mapped condition, this is currently inside override function, and add fast path change suggested by Felix in next version.

Regards,
Philip

Quoted 88 lines

I think we need to re-iterate the whole idea of MTYPE override.

Regards,
Christian.

Quoted 82 lines

Regards

Xiaogang

Quoted 78 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +24 -8
+static inline bool amdgpu_vm_addr_contiguous(struct amdgpu_device *adev, dma_addr_t addr,
+					     dma_addr_t addr_next, bool contiguous)
+{
+	if (!adev->gmc.is_app_apu || !page_is_ram(addr >> PAGE_SHIFT))
+		return (addr + PAGE_SIZE) == addr_next;
+
+	if (pfn_to_nid(addr >> PAGE_SHIFT) != pfn_to_nid(addr_next >> PAGE_SHIFT))
+		return !contiguous;
+
+	return (addr + PAGE_SIZE) == addr_next;
+}
+
  /**
   * amdgpu_vm_update_range - update a range in the vm page table
   *
@@ -1198,22 +1226,26 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
  				uint64_t pfn = cursor.start >> PAGE_SHIFT;
  				uint64_t count;
 
-				contiguous = pages_addr[pfn + 1] ==
-					pages_addr[pfn] + PAGE_SIZE;
+				contiguous = amdgpu_vm_addr_contiguous(adev,
+								       pages_addr[pfn],
+								       pages_addr[pfn + 1],
+								       contiguous);
 
-				tmp = num_entries /
-					AMDGPU_GPU_PAGES_IN_CPU_PAGE;
+				tmp = num_entries / AMDGPU_GPU_PAGES_IN_CPU_PAGE;
  				for (count = 2; count < tmp; ++count) {
  					uint64_t idx = pfn + count;
 
-					if (contiguous != (pages_addr[idx] ==
-					    pages_addr[idx - 1] + PAGE_SIZE))
+					if (contiguous != amdgpu_vm_addr_contiguous(adev,
+									pages_addr[idx - 1],
+									pages_addr[idx],
+									contiguous))
  						break;
  				}
+
  				if (!contiguous)
  					count--;
-				num_entries = count *
-					AMDGPU_GPU_PAGES_IN_CPU_PAGE;
+
+				num_entries = count * AMDGPU_GPU_PAGES_IN_CPU_PAGE;
  			}
 
  			if (!contiguous) {
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +10 -4
index 31a437ce9570..9e1607fb3b2e 100644
@@ -708,13 +708,19 @@ static void amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params *params,
  		amdgpu_vm_pte_update_noretry_flags(adev, &flags);
 
  	/* APUs mapping system memory may need different MTYPEs on different
-	 * NUMA nodes. Only do this for contiguous ranges that can be assumed
-	 * to be on the same NUMA node.
+	 * NUMA nodes. Both contiguous and non-contiguous ranges are handled
+	 * since amdgpu_vm_update_range ensures updates don't span NUMA
+	 * node boundaries.
  	 */
  	if ((flags & AMDGPU_PTE_SYSTEM) && (adev->flags & AMD_IS_APU) &&
  	    adev->gmc.gmc_funcs->override_vm_pte_flags &&
-	    num_possible_nodes() > 1 && !params->pages_addr && params->allow_override)
-		amdgpu_gmc_override_vm_pte_flags(adev, params->vm, addr, &flags);
+	    num_possible_nodes() > 1 && params->allow_override) {
+		if (params->pages_addr)
+			amdgpu_gmc_override_vm_pte_flags(adev, params->vm,
+					params->pages_addr[addr >> PAGE_SHIFT], &flags);
+		else
+			amdgpu_gmc_override_vm_pte_flags(adev, params->vm, addr, &flags);
+	}
 
  	params->vm->update_funcs->update(params, pt, pe, addr, count, incr,
  					 flags);
maintainer
#5 lore raw
Quoted 56 lines · Philip Yang
Quoted 55 lines · Christian König
Quoted 42 lines · Xiaogang
Quoted 41 lines · Philip Yang

On multi-socket MI300A APU systems, system memory pages mapped to the closest GPU must use MTYPE_RW instead of MTYPE_NC to maintain correct cache coherence. The existing mtype override in amdgpu_vm_pte_update_flags() excluded non-contiguous page mappings from the override. This caused incorrect MTYPE_NC for scattered local pages, leading to cache coherence issues.

The override applies to both contiguous and non-contiguous mappings. When pages_addr is set, resolve the physical address via pages_addr[addr >> PAGE_SHIFT] before passing it to the override callback for NUMA node lookup.

Introduce amdgpu_vm_addr_contiguous() helper that, on MI300A, treats pages on different NUMA nodes as non-contiguous even if their DMA addresses are adjacent. This ensures amdgpu_vm_update_range() splits page table updates at NUMA node boundaries so each batch gets the correct mtype override.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c    | 48 +++++++++++++++++++----
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 14 +++++--
 2 files changed, 50 insertions(+), 12 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +9 -0
index 63156289ae7f..f8fcbf079bf4 100644
@@ -1099,6 +1099,34 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params,
 	}
 }
 
+/**
+ * amdgpu_vm_addr_contiguous - check if two DMA addresses are contiguous
+ *
+ * @adev: amdgpu_device pointer
+ * @addr: current DMA address
+ * @addr_next: next DMA address to check against
+ * @contiguous: current contiguity state of the range being built
+ *
+ * Check whether @addr and @addr_next are physically contiguous. On APU
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params, · line 1117

Quoted 7 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +7 -0
+ * platforms with multiple NUMA nodes (e.g. MI300A), a NUMA node boundary
+ * also breaks contiguity so that each contiguous batch stays within a
+ * single NUMA node for correct MTYPE override selection.
+ *
+ * Returns:
+ * true if @addr_next continues the current contiguous range, false otherwise.
+ */

We can use pfn_to_nid or page_to_nid to get which noma(id) the backing memory is at. pfn_to_nid uses pfn from physical address. You use dma_addr_t that is device dependent. It is not always same as physical address of RAM.

Yeah that here won't work at all.

Quoted 2 lines

ttm_tt also has

Quoted 8 lines
Quoted 6 lines
Quoted 3 lines

/** @pages: Array of pages backing the data. */ struct page **pages;

I think using the pages to get numa id by page_to_nid is more appropriate.

That array isn't filled in for imported pages.

As far as I can see the whole approach won't work reliable. For imports we only know the dma_addr and not the struct page nor the pfn.

if adev->ram_is_direct_mapped, then dma_addr equals to pfn, we can use dma_addr to call pfn_to_nid.
I will add ram_is_direct_mapped condition, this is currently inside override function, and add fast path change suggested by Felix in next version.

That is seriously not something we can do. This relies an specific HW behavior in common code and is clearly a no-go from my side.

Regards,
Christian.

Quoted 91 lines

Regards,
Philip

Quoted 87 lines

I think we need to re-iterate the whole idea of MTYPE override.

Regards,
Christian.

Quoted 82 lines

Regards

Xiaogang

Quoted 78 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +24 -8
+static inline bool amdgpu_vm_addr_contiguous(struct amdgpu_device *adev, dma_addr_t addr,
+					     dma_addr_t addr_next, bool contiguous)
+{
+	if (!adev->gmc.is_app_apu || !page_is_ram(addr >> PAGE_SHIFT))
+		return (addr + PAGE_SIZE) == addr_next;
+
+	if (pfn_to_nid(addr >> PAGE_SHIFT) != pfn_to_nid(addr_next >> PAGE_SHIFT))
+		return !contiguous;
+
+	return (addr + PAGE_SIZE) == addr_next;
+}
+
 /**
  * amdgpu_vm_update_range - update a range in the vm page table
  *
@@ -1198,22 +1226,26 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 				uint64_t pfn = cursor.start >> PAGE_SHIFT;
 				uint64_t count;
 
-				contiguous = pages_addr[pfn + 1] ==
-					pages_addr[pfn] + PAGE_SIZE;
+				contiguous = amdgpu_vm_addr_contiguous(adev,
+								       pages_addr[pfn],
+								       pages_addr[pfn + 1],
+								       contiguous);
 
-				tmp = num_entries /
-					AMDGPU_GPU_PAGES_IN_CPU_PAGE;
+				tmp = num_entries / AMDGPU_GPU_PAGES_IN_CPU_PAGE;
 				for (count = 2; count < tmp; ++count) {
 					uint64_t idx = pfn + count;
 
-					if (contiguous != (pages_addr[idx] ==
-					    pages_addr[idx - 1] + PAGE_SIZE))
+					if (contiguous != amdgpu_vm_addr_contiguous(adev,
+									pages_addr[idx - 1],
+									pages_addr[idx],
+									contiguous))
 						break;
 				}
+
 				if (!contiguous)
 					count--;
-				num_entries = count *
-					AMDGPU_GPU_PAGES_IN_CPU_PAGE;
+
+				num_entries = count * AMDGPU_GPU_PAGES_IN_CPU_PAGE;
 			}
 
 			if (!contiguous) {
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +10 -4
index 31a437ce9570..9e1607fb3b2e 100644
@@ -708,13 +708,19 @@ static void amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params *params,
 		amdgpu_vm_pte_update_noretry_flags(adev, &flags);
 
 	/* APUs mapping system memory may need different MTYPEs on different
-	 * NUMA nodes. Only do this for contiguous ranges that can be assumed
-	 * to be on the same NUMA node.
+	 * NUMA nodes. Both contiguous and non-contiguous ranges are handled
+	 * since amdgpu_vm_update_range ensures updates don't span NUMA
+	 * node boundaries.
 	 */
 	if ((flags & AMDGPU_PTE_SYSTEM) && (adev->flags & AMD_IS_APU) &&
 	    adev->gmc.gmc_funcs->override_vm_pte_flags &&
-	    num_possible_nodes() > 1 && !params->pages_addr && params->allow_override)
-		amdgpu_gmc_override_vm_pte_flags(adev, params->vm, addr, &flags);
+	    num_possible_nodes() > 1 && params->allow_override) {
+		if (params->pages_addr)
+			amdgpu_gmc_override_vm_pte_flags(adev, params->vm,
+					params->pages_addr[addr >> PAGE_SHIFT], &flags);
+		else
+			amdgpu_gmc_override_vm_pte_flags(adev, params->vm, addr, &flags);
+	}
 
 	params->vm->update_funcs->update(params, pt, pe, addr, count, incr,
 					 flags);
#6 lore raw
In reply to Philip Yang
Quoted 29 lines · Philip Yang

On multi-socket MI300A APU systems, system memory pages mapped to the closest GPU must use MTYPE_RW instead of MTYPE_NC to maintain correct cache coherence. The existing mtype override in amdgpu_vm_pte_update_flags() excluded non-contiguous page mappings from the override. This caused incorrect MTYPE_NC for scattered local pages, leading to cache coherence issues.

The override applies to both contiguous and non-contiguous mappings. When pages_addr is set, resolve the physical address via pages_addr[addr >> PAGE_SHIFT] before passing it to the override callback for NUMA node lookup.

Introduce amdgpu_vm_addr_contiguous() helper that, on MI300A, treats pages on different NUMA nodes as non-contiguous even if their DMA addresses are adjacent. This ensures amdgpu_vm_update_range() splits page table updates at NUMA node boundaries so each batch gets the correct mtype override.

---
  drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c    | 48 +++++++++++++++++++----
  drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 14 +++++--
  2 files changed, 50 insertions(+), 12 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +0 -0
index 63156289ae7f..f8fcbf079bf4 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params, · line 1103

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +2 -0
@@ -1099,6 +1099,34 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params,
  	}
  }
 
+/**
+ * amdgpu_vm_addr_contiguous - check if two DMA addresses are contiguous

The function name is a bit misleading. It doesn't return whether the pages are contiguous, but whether they are in the same "group" (same amdgpu_vm_ptes_update call) by some pretty arbitrary criteria. Maybe call it amdgpu_vm_addr_same_group.

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +8 -0
+ *
+ * @adev: amdgpu_device pointer
+ * @addr: current DMA address
+ * @addr_next: next DMA address to check against
+ * @contiguous: current contiguity state of the range being built
+ *
+ * Check whether @addr and @addr_next are physically contiguous. On APU
+ * platforms with multiple NUMA nodes (e.g. MI300A), a NUMA node boundary

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params, · line 1121

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +10 -0
+ * also breaks contiguity so that each contiguous batch stays within a
+ * single NUMA node for correct MTYPE override selection.
+ *
+ * Returns:
+ * true if @addr_next continues the current contiguous range, false otherwise.
+ */
+static inline bool amdgpu_vm_addr_contiguous(struct amdgpu_device *adev, dma_addr_t addr,
+					     dma_addr_t addr_next, bool contiguous)
+{
+	if (!adev->gmc.is_app_apu || !page_is_ram(addr >> PAGE_SHIFT))

Isn't page_is_ram always true on APP APUs?

On the other hand, I think you need a check here that IOMMU device isolation is off (adev->ram_is_direct_mapped). Otherwise you cannot infer the NUMA node from the DMA address. I'd put the condition for that in the caller where you only need to check it once. Then replace the adev parameter with a bool same_nid. And while you're at it, you can add some more conditions to make the fast-path more likely:

	same_nid = adev->gmc.is_app_apu && adev->ram_is_direct_mapped &&
		   adev->gmc.gmc_funcs->override_vm_pte_flags &&
		   num_possible_nodes() > 1 && params->allow_override;

Regards,
  Felix

Quoted 75 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +20 -8
+		return (addr + PAGE_SIZE) == addr_next;
+
+	if (pfn_to_nid(addr >> PAGE_SHIFT) != pfn_to_nid(addr_next >> PAGE_SHIFT))
+		return !contiguous;
+
+	return (addr + PAGE_SIZE) == addr_next;
+}
+
  /**
   * amdgpu_vm_update_range - update a range in the vm page table
   *
@@ -1198,22 +1226,26 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
  				uint64_t pfn = cursor.start >> PAGE_SHIFT;
  				uint64_t count;
 
-				contiguous = pages_addr[pfn + 1] ==
-					pages_addr[pfn] + PAGE_SIZE;
+				contiguous = amdgpu_vm_addr_contiguous(adev,
+								       pages_addr[pfn],
+								       pages_addr[pfn + 1],
+								       contiguous);
 
-				tmp = num_entries /
-					AMDGPU_GPU_PAGES_IN_CPU_PAGE;
+				tmp = num_entries / AMDGPU_GPU_PAGES_IN_CPU_PAGE;
  				for (count = 2; count < tmp; ++count) {
  					uint64_t idx = pfn + count;
 
-					if (contiguous != (pages_addr[idx] ==
-					    pages_addr[idx - 1] + PAGE_SIZE))
+					if (contiguous != amdgpu_vm_addr_contiguous(adev,
+									pages_addr[idx - 1],
+									pages_addr[idx],
+									contiguous))
  						break;
  				}
+
  				if (!contiguous)
  					count--;
-				num_entries = count *
-					AMDGPU_GPU_PAGES_IN_CPU_PAGE;
+
+				num_entries = count * AMDGPU_GPU_PAGES_IN_CPU_PAGE;
  			}
 
  			if (!contiguous) {
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +10 -4
index 31a437ce9570..9e1607fb3b2e 100644
@@ -708,13 +708,19 @@ static void amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params *params,
  		amdgpu_vm_pte_update_noretry_flags(adev, &flags);
 
  	/* APUs mapping system memory may need different MTYPEs on different
-	 * NUMA nodes. Only do this for contiguous ranges that can be assumed
-	 * to be on the same NUMA node.
+	 * NUMA nodes. Both contiguous and non-contiguous ranges are handled
+	 * since amdgpu_vm_update_range ensures updates don't span NUMA
+	 * node boundaries.
  	 */
  	if ((flags & AMDGPU_PTE_SYSTEM) && (adev->flags & AMD_IS_APU) &&
  	    adev->gmc.gmc_funcs->override_vm_pte_flags &&
-	    num_possible_nodes() > 1 && !params->pages_addr && params->allow_override)
-		amdgpu_gmc_override_vm_pte_flags(adev, params->vm, addr, &flags);
+	    num_possible_nodes() > 1 && params->allow_override) {
+		if (params->pages_addr)
+			amdgpu_gmc_override_vm_pte_flags(adev, params->vm,
+					params->pages_addr[addr >> PAGE_SHIFT], &flags);
+		else
+			amdgpu_gmc_override_vm_pte_flags(adev, params->vm, addr, &flags);
+	}
 
  	params->vm->update_funcs->update(params, pt, pe, addr, count, incr,
  					 flags);
#7 lore raw
Quoted 33 lines · Felix Kuehling
Quoted 32 lines · Philip Yang

On multi-socket MI300A APU systems, system memory pages mapped to the closest GPU must use MTYPE_RW instead of MTYPE_NC to maintain correct cache coherence. The existing mtype override in amdgpu_vm_pte_update_flags() excluded non-contiguous page mappings from the override. This caused incorrect MTYPE_NC for scattered local pages, leading to cache coherence issues.

The override applies to both contiguous and non-contiguous mappings. When pages_addr is set, resolve the physical address via pages_addr[addr >> PAGE_SHIFT] before passing it to the override callback for NUMA node lookup.

Introduce amdgpu_vm_addr_contiguous() helper that, on MI300A, treats pages on different NUMA nodes as non-contiguous even if their DMA addresses are adjacent. This ensures amdgpu_vm_update_range() splits page table updates at NUMA node boundaries so each batch gets the correct mtype override.

---
  drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c    | 48 +++++++++++++++++++----
  drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 14 +++++--
  2 files changed, 50 insertions(+), 12 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +0 -0
index 63156289ae7f..f8fcbf079bf4 100644
@@ -1099,6 +1099,34 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params,
  	}
  }
 
Quoted 6 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params, · line 1103

Quoted 2 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +2 -0
+/**
+ * amdgpu_vm_addr_contiguous - check if two DMA addresses are contiguous

The function name is a bit misleading. It doesn't return whether the pages are contiguous, but whether they are in the same "group" (same amdgpu_vm_ptes_update call) by some pretty arbitrary criteria. Maybe call it amdgpu_vm_addr_same_group.

ok

Quoted 12 lines
Quoted 12 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +12 -0
+ *
+ * @adev: amdgpu_device pointer
+ * @addr: current DMA address
+ * @addr_next: next DMA address to check against
+ * @contiguous: current contiguity state of the range being built
+ *
+ * Check whether @addr and @addr_next are physically contiguous. On APU
+ * platforms with multiple NUMA nodes (e.g. MI300A), a NUMA node boundary
+ * also breaks contiguity so that each contiguous batch stays within a
+ * single NUMA node for correct MTYPE override selection.
+ *
+ * Returns:
Quoted 9 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params, · line 1121

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +6 -0
+ * true if @addr_next continues the current contiguous range, false otherwise.
+ */
+static inline bool amdgpu_vm_addr_contiguous(struct amdgpu_device *adev, dma_addr_t addr,
+					     dma_addr_t addr_next, bool contiguous)
+{
+	if (!adev->gmc.is_app_apu || !page_is_ram(addr >> PAGE_SHIFT))

Isn't page_is_ram always true on APP APUs?

There is address (probably doorbell) outside ram space mapping to GPU, without structure page and cause crash inside pfn_to_nid. gmc_v9_0_override_vm_pte_flags has the same checking too.

Quoted 3 lines

On the other hand, I think you need a check here that IOMMU device isolation is off (adev->ram_is_direct_mapped). Otherwise you cannot infer the NUMA node from the DMA address.

yes, should add !adev->ram_is_direct_mapped

Quoted 6 lines

I'd put the condition for that in the caller where you only need to check it once. Then replace the adev parameter with a bool same_nid. And while you're at it, you can add some more conditions to make the fast-path more likely:

	same_nid = adev->gmc.is_app_apu && adev->ram_is_direct_mapped &&
		   adev->gmc.gmc_funcs->override_vm_pte_flags &&
		   num_possible_nodes() > 1 && params->allow_override;

done in next version.

Regards,
Philip

Quoted 80 lines

Regards,
  Felix

Quoted 75 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +20 -8
+		return (addr + PAGE_SIZE) == addr_next;
+
+	if (pfn_to_nid(addr >> PAGE_SHIFT) != pfn_to_nid(addr_next >> PAGE_SHIFT))
+		return !contiguous;
+
+	return (addr + PAGE_SIZE) == addr_next;
+}
+
  /**
   * amdgpu_vm_update_range - update a range in the vm page table
   *
@@ -1198,22 +1226,26 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
  				uint64_t pfn = cursor.start >> PAGE_SHIFT;
  				uint64_t count;
 
-				contiguous = pages_addr[pfn + 1] ==
-					pages_addr[pfn] + PAGE_SIZE;
+				contiguous = amdgpu_vm_addr_contiguous(adev,
+								       pages_addr[pfn],
+								       pages_addr[pfn + 1],
+								       contiguous);
 
-				tmp = num_entries /
-					AMDGPU_GPU_PAGES_IN_CPU_PAGE;
+				tmp = num_entries / AMDGPU_GPU_PAGES_IN_CPU_PAGE;
  				for (count = 2; count < tmp; ++count) {
  					uint64_t idx = pfn + count;
 
-					if (contiguous != (pages_addr[idx] ==
-					    pages_addr[idx - 1] + PAGE_SIZE))
+					if (contiguous != amdgpu_vm_addr_contiguous(adev,
+									pages_addr[idx - 1],
+									pages_addr[idx],
+									contiguous))
  						break;
  				}
+
  				if (!contiguous)
  					count--;
-				num_entries = count *
-					AMDGPU_GPU_PAGES_IN_CPU_PAGE;
+
+				num_entries = count * AMDGPU_GPU_PAGES_IN_CPU_PAGE;
  			}
 
  			if (!contiguous) {
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +10 -4
index 31a437ce9570..9e1607fb3b2e 100644
@@ -708,13 +708,19 @@ static void amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params *params,
  		amdgpu_vm_pte_update_noretry_flags(adev, &flags);
 
  	/* APUs mapping system memory may need different MTYPEs on different
-	 * NUMA nodes. Only do this for contiguous ranges that can be assumed
-	 * to be on the same NUMA node.
+	 * NUMA nodes. Both contiguous and non-contiguous ranges are handled
+	 * since amdgpu_vm_update_range ensures updates don't span NUMA
+	 * node boundaries.
  	 */
  	if ((flags & AMDGPU_PTE_SYSTEM) && (adev->flags & AMD_IS_APU) &&
  	    adev->gmc.gmc_funcs->override_vm_pte_flags &&
-	    num_possible_nodes() > 1 && !params->pages_addr && params->allow_override)
-		amdgpu_gmc_override_vm_pte_flags(adev, params->vm, addr, &flags);
+	    num_possible_nodes() > 1 && params->allow_override) {
+		if (params->pages_addr)
+			amdgpu_gmc_override_vm_pte_flags(adev, params->vm,
+					params->pages_addr[addr >> PAGE_SHIFT], &flags);
+		else
+			amdgpu_gmc_override_vm_pte_flags(adev, params->vm, addr, &flags);
+	}
 
  	params->vm->update_funcs->update(params, pt, pe, addr, count, incr,
  					 flags);