drm/amdgpu: extend mtype override to non-contiguous pages
Subsystems (2)
PATCH v2 1/1 drm/amdgpu: extend mtype override to non-contiguous pages
On multi-socket MI300A APU systems, system memory pages mapped to the closest GPU must use MTYPE_RW instead of MTYPE_NC to maintain correct cache coherence. The existing mtype override in amdgpu_vm_pte_update_flags() excluded non-contiguous page mappings from the override. This caused incorrect MTYPE_NC for scattered local pages, leading to cache coherence issues.
The override applies to both contiguous and non-contiguous mappings. When pages_addr is set, resolve the physical address via pages_addr[addr >> PAGE_SHIFT] before passing it to the override callback for NUMA node lookup.
Introduce amdgpu_vm_addr_contiguous() helper that, on MI300A, treats pages on different NUMA nodes as non-contiguous even if their DMA addresses are adjacent. This ensures amdgpu_vm_update_range() splits page table updates at NUMA node boundaries so each batch gets the correct mtype override.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 51 +++++++++++++++++++---- drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 14 +++++-- 2 files changed, 53 insertions(+), 12 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +43 -8
@@ -1099,6 +1099,32 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params, } } +/** +* amdgpu_vm_addr_same_group - check if two DMA addresses are same contiguity state +* +* @same_nid: true to check if two address on same NUMA node +* @addr: current DMA address +* @addr_next: next DMA address to check against +* @contiguous: current contiguity state of the range being built +* +* Check whether @addr and @addr_next are physically contiguous. On APU +* platforms with multiple NUMA nodes (e.g. MI300A), a NUMA node boundary +* also breaks contiguity so that each contiguous batch stays within a +* single NUMA node for correct MTYPE override selection. +* +* Returns: +* true if @addr_next continues the current contiguous range, false otherwise. +*/ +static inline bool amdgpu_vm_addr_same_group (bool same_nid,dma_addr_t addr, +dma_addr_t addr_next,bool contiguous) +{ +if (same_nid &&page_is_ram (addr >>PAGE_SHIFT ) && +pfn_to_nid (addr >>PAGE_SHIFT ) !=pfn_to_nid (addr_next >>PAGE_SHIFT )) +return !contiguous; + +return (addr +PAGE_SIZE ) == addr_next; +} +/** * amdgpu_vm_update_range - update a range in the vm page table *
@@ -1134,6 +1160,7 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,struct amdgpu_vm_tlb_seq_struct *tlb_cb;struct amdgpu_vm_update_params params;struct amdgpu_res_cursor cursor; +bool same_nid;int r, idx;if (!drm_dev_enter (adev_to_drm (adev), &idx))
@@ -1166,6 +1193,10 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm, params.allow_override = allow_override;INIT_LIST_HEAD (¶ms.tlb_flush_waitlist); + same_nid = adev->gmc.is_app_apu && adev->ram_is_direct_mapped && + adev->gmc.gmc_funcs->override_vm_pte_flags && +num_possible_nodes () >1 && params.allow_override; +amdgpu_vm_eviction_lock (vm);if (vm->evicting) { r = -EBUSY ;
@@ -1198,22 +1229,26 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,uint64_t pfn = cursor.start >>PAGE_SHIFT ;uint64_t count; - contiguous = pages_addr[pfn +1 ] == - pages_addr[pfn] +PAGE_SIZE ; + contiguous =amdgpu_vm_addr_same_group (same_nid, + pages_addr[pfn], + pages_addr[pfn +1 ], + contiguous); - tmp = num_entries / -AMDGPU_GPU_PAGES_IN_CPU_PAGE ; + tmp = num_entries /AMDGPU_GPU_PAGES_IN_CPU_PAGE ;for (count =2 ; count < tmp; ++count) {uint64_t idx = pfn + count; -if (contiguous != (pages_addr[idx] == - pages_addr[idx -1 ] +PAGE_SIZE )) +if (contiguous !=amdgpu_vm_addr_same_group (same_nid, + pages_addr[idx -1 ], + pages_addr[idx], + contiguous))break ; } +if (!contiguous) count--; - num_entries = count * -AMDGPU_GPU_PAGES_IN_CPU_PAGE ; + + num_entries = count *AMDGPU_GPU_PAGES_IN_CPU_PAGE ; }if (!contiguous) {
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +10 -4
@@ -708,13 +708,19 @@ static void amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params *params,amdgpu_vm_pte_update_noretry_flags (adev, &flags);/* APUs mapping system memory may need different MTYPEs on different -* NUMA nodes. Only do this for contiguous ranges that can be assumed -* to be on the same NUMA node. +* NUMA nodes. Both contiguous and non-contiguous ranges are handled +* since amdgpu_vm_update_range ensures updates don't span NUMA +* node boundaries. */ if ((flags &AMDGPU_PTE_SYSTEM ) && (adev->flags &AMD_IS_APU ) && adev->gmc.gmc_funcs->override_vm_pte_flags && -num_possible_nodes () >1 && !params->pages_addr && params->allow_override) -amdgpu_gmc_override_vm_pte_flags (adev, params->vm, addr, &flags); +num_possible_nodes () >1 && params->allow_override) { +if (params->pages_addr) +amdgpu_gmc_override_vm_pte_flags (adev, params->vm, + params->pages_addr[addr >>PAGE_SHIFT ], &flags); +else +amdgpu_gmc_override_vm_pte_flags (adev, params->vm, addr, &flags); + } params->vm->update_funcs->update (params, pt, pe, addr, count, incr, flags);
Signature · 1 line
2.50.1
Christian König
maintainerQuoted 44 lines · Philip Yang
On multi-socket MI300A APU systems, system memory pages mapped to the closest GPU must use
MTYPE_RWinstead ofMTYPE_NCto maintain correct cache coherence. The existing mtype override inamdgpu_vm_pte_update_flags()excluded non-contiguous page mappings from the override. This caused incorrectMTYPE_NCfor scattered local pages, leading to cache coherence issues.The override applies to both contiguous and non-contiguous mappings. When
pages_addris set, resolve the physical address via pages_addr[addr >>PAGE_SHIFT] before passing it to the override callback for NUMA node lookup.Introduce
amdgpu_vm_addr_contiguous()helper that, on MI300A, treats pages on different NUMA nodes as non-contiguous even if their DMA addresses are adjacent. This ensuresamdgpu_vm_update_range()splits page table updates at NUMA node boundaries so each batch gets the correct mtype override.Signed-off-by Philip Yang <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 51 +++++++++++++++++++---- drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 14 +++++-- 2 files changed, 53 insertions(+), 12 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +12
-0@@ -1099,6 +1099,32 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params, } } +/** +* amdgpu_vm_addr_same_group - check if two DMA addresses are same contiguity state +* +* @same_nid: true to check if two address on same NUMA node +* @addr: current DMA address +* @addr_next: next DMA address to check against +* @contiguous: current contiguity state of the range being built +* +* Check whether @addr and @addr_next are physically contiguous. On APU +* platforms with multiple NUMA nodes (e.g. MI300A), a NUMA node boundary +* also breaks contiguity so that each contiguous batch stays within a +* single NUMA node for correct MTYPE override selection.
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params, · line 1123
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +10
-0+* +* Returns: +* true if @addr_next continues the current contiguous range, false otherwise. +*/ +static inline bool amdgpu_vm_addr_same_group (bool same_nid,dma_addr_t addr, +dma_addr_t addr_next,bool contiguous) +{ +if (same_nid &&page_is_ram (addr >>PAGE_SHIFT ) && +pfn_to_nid (addr >>PAGE_SHIFT ) !=pfn_to_nid (addr_next >>PAGE_SHIFT )) +return !contiguous;
This relys on that the dma_addr is equal to the PFN and is a clear NO-GO from my side to that approach.
Question is why exactly would we need that in the first place?
Regards,
Christian.
Quoted 89 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +21
-8+ +return (addr +PAGE_SIZE ) == addr_next; +} +/** * amdgpu_vm_update_range - update a range in the vm page table * @@ -1134,6 +1160,7 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,struct amdgpu_vm_tlb_seq_struct *tlb_cb;struct amdgpu_vm_update_params params;struct amdgpu_res_cursor cursor; +bool same_nid;int r, idx;if (!drm_dev_enter (adev_to_drm (adev), &idx))@@ -1166,6 +1193,10 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm, params.allow_override = allow_override;INIT_LIST_HEAD (¶ms.tlb_flush_waitlist); + same_nid = adev->gmc.is_app_apu && adev->ram_is_direct_mapped && + adev->gmc.gmc_funcs->override_vm_pte_flags && +num_possible_nodes () >1 && params.allow_override; +amdgpu_vm_eviction_lock (vm);if (vm->evicting) { r = -EBUSY ;@@ -1198,22 +1229,26 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,uint64_t pfn = cursor.start >>PAGE_SHIFT ;uint64_t count; - contiguous = pages_addr[pfn +1 ] == - pages_addr[pfn] +PAGE_SIZE ; + contiguous =amdgpu_vm_addr_same_group (same_nid, + pages_addr[pfn], + pages_addr[pfn +1 ], + contiguous); - tmp = num_entries / -AMDGPU_GPU_PAGES_IN_CPU_PAGE ; + tmp = num_entries /AMDGPU_GPU_PAGES_IN_CPU_PAGE ;for (count =2 ; count < tmp; ++count) {uint64_t idx = pfn + count; -if (contiguous != (pages_addr[idx] == - pages_addr[idx -1 ] +PAGE_SIZE )) +if (contiguous !=amdgpu_vm_addr_same_group (same_nid, + pages_addr[idx -1 ], + pages_addr[idx], + contiguous))break ; } +if (!contiguous) count--; - num_entries = count * -AMDGPU_GPU_PAGES_IN_CPU_PAGE ; + + num_entries = count *AMDGPU_GPU_PAGES_IN_CPU_PAGE ; }if (!contiguous) {drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +10
-4@@ -708,13 +708,19 @@ static void amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params *params,amdgpu_vm_pte_update_noretry_flags (adev, &flags);/* APUs mapping system memory may need different MTYPEs on different -* NUMA nodes. Only do this for contiguous ranges that can be assumed -* to be on the same NUMA node. +* NUMA nodes. Both contiguous and non-contiguous ranges are handled +* since amdgpu_vm_update_range ensures updates don't span NUMA +* node boundaries. */ if ((flags &AMDGPU_PTE_SYSTEM ) && (adev->flags &AMD_IS_APU ) && adev->gmc.gmc_funcs->override_vm_pte_flags && -num_possible_nodes () >1 && !params->pages_addr && params->allow_override) -amdgpu_gmc_override_vm_pte_flags (adev, params->vm, addr, &flags); +num_possible_nodes () >1 && params->allow_override) { +if (params->pages_addr) +amdgpu_gmc_override_vm_pte_flags (adev, params->vm, + params->pages_addr[addr >>PAGE_SHIFT ], &flags); +else +amdgpu_gmc_override_vm_pte_flags (adev, params->vm, addr, &flags); + } params->vm->update_funcs->update (params, pt, pe, addr, count, incr, flags);
Quoted 49 lines · Christian König
Quoted 48 lines · Philip Yang
On multi-socket MI300A APU systems, system memory pages mapped to the closest GPU must use
MTYPE_RWinstead ofMTYPE_NCto maintain correct cache coherence. The existing mtype override inamdgpu_vm_pte_update_flags()excluded non-contiguous page mappings from the override. This caused incorrectMTYPE_NCfor scattered local pages, leading to cache coherence issues.The override applies to both contiguous and non-contiguous mappings. When
pages_addris set, resolve the physical address via pages_addr[addr >>PAGE_SHIFT] before passing it to the override callback for NUMA node lookup.Introduce
amdgpu_vm_addr_contiguous()helper that, on MI300A, treats pages on different NUMA nodes as non-contiguous even if their DMA addresses are adjacent. This ensuresamdgpu_vm_update_range()splits page table updates at NUMA node boundaries so each batch gets the correct mtype override.Signed-off-by Philip Yang<[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 51 +++++++++++++++++++---- drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 14 +++++-- 2 files changed, 53 insertions(+), 12 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +16
-0@@ -1099,6 +1099,32 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params, } } +/** +* amdgpu_vm_addr_same_group - check if two DMA addresses are same contiguity state +* +* @same_nid: true to check if two address on same NUMA node +* @addr: current DMA address +* @addr_next: next DMA address to check against +* @contiguous: current contiguity state of the range being built +* +* Check whether @addr and @addr_next are physically contiguous. On APU +* platforms with multiple NUMA nodes (e.g. MI300A), a NUMA node boundary +* also breaks contiguity so that each contiguous batch stays within a +* single NUMA node for correct MTYPE override selection. +* +* Returns: +* true if @addr_next continues the current contiguous range, false otherwise. +*/
Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params, · line 1123
Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +6
-0+static inline bool amdgpu_vm_addr_same_group (bool same_nid,dma_addr_t addr, +dma_addr_t addr_next,bool contiguous) +{ +if (same_nid &&page_is_ram (addr >>PAGE_SHIFT ) && +pfn_to_nid (addr >>PAGE_SHIFT ) !=pfn_to_nid (addr_next >>PAGE_SHIFT )) +return !contiguous;This relys on that the
dma_addris equal to the PFN and is a clear NO-GO from my side to that approach.
To add pages array for this seems too much overhead, to support mtype override if adev->ram_is_direct_mapped is enough.
Quoted 2 lines
Question is why exactly would we need that in the first place?
MI300A with NPS2/4 or MI300A 4P, should use mtype RW to map on the near compute partition or GPU, to maintain hw cache coherence.
As discussed offline, app may alloc with uncached flag to switch mtype, the test don't show issue when switching mtype b/w RW and CC.
Regards,
Philip
Quoted 93 lines
Regards,
Christian.Quoted 89 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +21
-8+ +return (addr +PAGE_SIZE ) == addr_next; +} +/** * amdgpu_vm_update_range - update a range in the vm page table * @@ -1134,6 +1160,7 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,struct amdgpu_vm_tlb_seq_struct *tlb_cb;struct amdgpu_vm_update_params params;struct amdgpu_res_cursor cursor; +bool same_nid;int r, idx;if (!drm_dev_enter (adev_to_drm (adev), &idx))@@ -1166,6 +1193,10 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm, params.allow_override = allow_override;INIT_LIST_HEAD (¶ms.tlb_flush_waitlist); + same_nid = adev->gmc.is_app_apu && adev->ram_is_direct_mapped && + adev->gmc.gmc_funcs->override_vm_pte_flags && +num_possible_nodes () >1 && params.allow_override; +amdgpu_vm_eviction_lock (vm);if (vm->evicting) { r = -EBUSY ;@@ -1198,22 +1229,26 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,uint64_t pfn = cursor.start >>PAGE_SHIFT ;uint64_t count; - contiguous = pages_addr[pfn +1 ] == - pages_addr[pfn] +PAGE_SIZE ; + contiguous =amdgpu_vm_addr_same_group (same_nid, + pages_addr[pfn], + pages_addr[pfn +1 ], + contiguous); - tmp = num_entries / -AMDGPU_GPU_PAGES_IN_CPU_PAGE ; + tmp = num_entries /AMDGPU_GPU_PAGES_IN_CPU_PAGE ;for (count =2 ; count < tmp; ++count) {uint64_t idx = pfn + count; -if (contiguous != (pages_addr[idx] == - pages_addr[idx -1 ] +PAGE_SIZE )) +if (contiguous !=amdgpu_vm_addr_same_group (same_nid, + pages_addr[idx -1 ], + pages_addr[idx], + contiguous))break ; } +if (!contiguous) count--; - num_entries = count * -AMDGPU_GPU_PAGES_IN_CPU_PAGE ; + + num_entries = count *AMDGPU_GPU_PAGES_IN_CPU_PAGE ; }if (!contiguous) {drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +10
-4@@ -708,13 +708,19 @@ static void amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params *params,amdgpu_vm_pte_update_noretry_flags (adev, &flags);/* APUs mapping system memory may need different MTYPEs on different -* NUMA nodes. Only do this for contiguous ranges that can be assumed -* to be on the same NUMA node. +* NUMA nodes. Both contiguous and non-contiguous ranges are handled +* since amdgpu_vm_update_range ensures updates don't span NUMA +* node boundaries. */ if ((flags &AMDGPU_PTE_SYSTEM ) && (adev->flags &AMD_IS_APU ) && adev->gmc.gmc_funcs->override_vm_pte_flags && -num_possible_nodes () >1 && !params->pages_addr && params->allow_override) -amdgpu_gmc_override_vm_pte_flags (adev, params->vm, addr, &flags); +num_possible_nodes () >1 && params->allow_override) { +if (params->pages_addr) +amdgpu_gmc_override_vm_pte_flags (adev, params->vm, + params->pages_addr[addr >>PAGE_SHIFT ], &flags); +else +amdgpu_gmc_override_vm_pte_flags (adev, params->vm, addr, &flags); + } params->vm->update_funcs->update (params, pt, pe, addr, count, incr, flags);