drm/amdgpu: extend mtype override to non-contiguous pages
Subsystems (2)
PATCH 1/1 drm/amdgpu: extend mtype override to non-contiguous pages
On multi-socket MI300A APU systems, system memory pages mapped to the closest GPU must use MTYPE_RW instead of MTYPE_NC to maintain correct cache coherence. The existing mtype override in amdgpu_vm_pte_update_flags() excluded non-contiguous page mappings from the override. This caused incorrect MTYPE_NC for scattered local pages, leading to cache coherence issues.
The override applies to both contiguous and non-contiguous mappings. When pages_addr is set, resolve the physical address via pages_addr[addr >> PAGE_SHIFT] before passing it to the override callback for NUMA node lookup.
Introduce amdgpu_vm_addr_contiguous() helper that, on MI300A, treats pages on different NUMA nodes as non-contiguous even if their DMA addresses are adjacent. This ensures amdgpu_vm_update_range() splits page table updates at NUMA node boundaries so each batch gets the correct mtype override.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 48 +++++++++++++++++++---- drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 14 +++++-- 2 files changed, 50 insertions(+), 12 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +40 -8
@@ -1099,6 +1099,34 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params, } } +/** +* amdgpu_vm_addr_contiguous - check if two DMA addresses are contiguous +* +* @adev: amdgpu_device pointer +* @addr: current DMA address +* @addr_next: next DMA address to check against +* @contiguous: current contiguity state of the range being built +* +* Check whether @addr and @addr_next are physically contiguous. On APU +* platforms with multiple NUMA nodes (e.g. MI300A), a NUMA node boundary +* also breaks contiguity so that each contiguous batch stays within a +* single NUMA node for correct MTYPE override selection. +* +* Returns: +* true if @addr_next continues the current contiguous range, false otherwise. +*/ +static inline bool amdgpu_vm_addr_contiguous (struct amdgpu_device *adev,dma_addr_t addr, +dma_addr_t addr_next,bool contiguous) +{ +if (!adev->gmc.is_app_apu || !page_is_ram (addr >>PAGE_SHIFT )) +return (addr +PAGE_SIZE ) == addr_next; + +if (pfn_to_nid (addr >>PAGE_SHIFT ) !=pfn_to_nid (addr_next >>PAGE_SHIFT )) +return !contiguous; + +return (addr +PAGE_SIZE ) == addr_next; +} +/** * amdgpu_vm_update_range - update a range in the vm page table *
@@ -1198,22 +1226,26 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,uint64_t pfn = cursor.start >>PAGE_SHIFT ;uint64_t count; - contiguous = pages_addr[pfn +1 ] == - pages_addr[pfn] +PAGE_SIZE ; + contiguous =amdgpu_vm_addr_contiguous (adev, + pages_addr[pfn], + pages_addr[pfn +1 ], + contiguous); - tmp = num_entries / -AMDGPU_GPU_PAGES_IN_CPU_PAGE ; + tmp = num_entries /AMDGPU_GPU_PAGES_IN_CPU_PAGE ;for (count =2 ; count < tmp; ++count) {uint64_t idx = pfn + count; -if (contiguous != (pages_addr[idx] == - pages_addr[idx -1 ] +PAGE_SIZE )) +if (contiguous !=amdgpu_vm_addr_contiguous (adev, + pages_addr[idx -1 ], + pages_addr[idx], + contiguous))break ; } +if (!contiguous) count--; - num_entries = count * -AMDGPU_GPU_PAGES_IN_CPU_PAGE ; + + num_entries = count *AMDGPU_GPU_PAGES_IN_CPU_PAGE ; }if (!contiguous) {
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +10 -4
@@ -708,13 +708,19 @@ static void amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params *params,amdgpu_vm_pte_update_noretry_flags (adev, &flags);/* APUs mapping system memory may need different MTYPEs on different -* NUMA nodes. Only do this for contiguous ranges that can be assumed -* to be on the same NUMA node. +* NUMA nodes. Both contiguous and non-contiguous ranges are handled +* since amdgpu_vm_update_range ensures updates don't span NUMA +* node boundaries. */ if ((flags &AMDGPU_PTE_SYSTEM ) && (adev->flags &AMD_IS_APU ) && adev->gmc.gmc_funcs->override_vm_pte_flags && -num_possible_nodes () >1 && !params->pages_addr && params->allow_override) -amdgpu_gmc_override_vm_pte_flags (adev, params->vm, addr, &flags); +num_possible_nodes () >1 && params->allow_override) { +if (params->pages_addr) +amdgpu_gmc_override_vm_pte_flags (adev, params->vm, + params->pages_addr[addr >>PAGE_SHIFT ], &flags); +else +amdgpu_gmc_override_vm_pte_flags (adev, params->vm, addr, &flags); + } params->vm->update_funcs->update (params, pt, pe, addr, count, incr, flags);
Signature · 1 line
2.50.1
Quoted 38 lines · Philip Yang
On multi-socket MI300A APU systems, system memory pages mapped to the closest GPU must use
MTYPE_RWinstead ofMTYPE_NCto maintain correct cache coherence. The existing mtype override inamdgpu_vm_pte_update_flags()excluded non-contiguous page mappings from the override. This caused incorrectMTYPE_NCfor scattered local pages, leading to cache coherence issues.The override applies to both contiguous and non-contiguous mappings. When
pages_addris set, resolve the physical address via pages_addr[addr >>PAGE_SHIFT] before passing it to the override callback for NUMA node lookup.Introduce
amdgpu_vm_addr_contiguous()helper that, on MI300A, treats pages on different NUMA nodes as non-contiguous even if their DMA addresses are adjacent. This ensuresamdgpu_vm_update_range()splits page table updates at NUMA node boundaries so each batch gets the correct mtype override.Signed-off-by Philip Yang<[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 48 +++++++++++++++++++---- drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 14 +++++-- 2 files changed, 50 insertions(+), 12 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +6
-0@@ -1099,6 +1099,34 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params, } } +/** +* amdgpu_vm_addr_contiguous - check if two DMA addresses are contiguous +* +* @adev: amdgpu_device pointer +* @addr: current DMA address +* @addr_next: next DMA address to check against
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params, · line 1117
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +10
-0+* @contiguous: current contiguity state of the range being built +* +* Check whether @addr and @addr_next are physically contiguous. On APU +* platforms with multiple NUMA nodes (e.g. MI300A), a NUMA node boundary +* also breaks contiguity so that each contiguous batch stays within a +* single NUMA node for correct MTYPE override selection. +* +* Returns: +* true if @addr_next continues the current contiguous range, false otherwise. +*/
We can use pfn_to_nid or page_to_nid to get which noma(id) the backing memory is at. pfn_to_nid uses pfn from physical address. You use dma_addr_t that is device dependent. It is not always same as physical address of RAM.
ttm_tt also has
/** @pages: Array of pages backing the data. */ struct page **pages;
I think using the pages to get numa id by page_to_nid is more appropriate.
Regards
Xiaogang
Quoted 78 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +24
-8+static inline bool amdgpu_vm_addr_contiguous (struct amdgpu_device *adev,dma_addr_t addr, +dma_addr_t addr_next,bool contiguous) +{ +if (!adev->gmc.is_app_apu || !page_is_ram (addr >>PAGE_SHIFT )) +return (addr +PAGE_SIZE ) == addr_next; + +if (pfn_to_nid (addr >>PAGE_SHIFT ) !=pfn_to_nid (addr_next >>PAGE_SHIFT )) +return !contiguous; + +return (addr +PAGE_SIZE ) == addr_next; +} +/** * amdgpu_vm_update_range - update a range in the vm page table * @@ -1198,22 +1226,26 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,uint64_t pfn = cursor.start >>PAGE_SHIFT ;uint64_t count; - contiguous = pages_addr[pfn +1 ] == - pages_addr[pfn] +PAGE_SIZE ; + contiguous =amdgpu_vm_addr_contiguous (adev, + pages_addr[pfn], + pages_addr[pfn +1 ], + contiguous); - tmp = num_entries / -AMDGPU_GPU_PAGES_IN_CPU_PAGE ; + tmp = num_entries /AMDGPU_GPU_PAGES_IN_CPU_PAGE ;for (count =2 ; count < tmp; ++count) {uint64_t idx = pfn + count; -if (contiguous != (pages_addr[idx] == - pages_addr[idx -1 ] +PAGE_SIZE )) +if (contiguous !=amdgpu_vm_addr_contiguous (adev, + pages_addr[idx -1 ], + pages_addr[idx], + contiguous))break ; } +if (!contiguous) count--; - num_entries = count * -AMDGPU_GPU_PAGES_IN_CPU_PAGE ; + + num_entries = count *AMDGPU_GPU_PAGES_IN_CPU_PAGE ; }if (!contiguous) {drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +10
-4@@ -708,13 +708,19 @@ static void amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params *params,amdgpu_vm_pte_update_noretry_flags (adev, &flags);/* APUs mapping system memory may need different MTYPEs on different -* NUMA nodes. Only do this for contiguous ranges that can be assumed -* to be on the same NUMA node. +* NUMA nodes. Both contiguous and non-contiguous ranges are handled +* since amdgpu_vm_update_range ensures updates don't span NUMA +* node boundaries. */ if ((flags &AMDGPU_PTE_SYSTEM ) && (adev->flags &AMD_IS_APU ) && adev->gmc.gmc_funcs->override_vm_pte_flags && -num_possible_nodes () >1 && !params->pages_addr && params->allow_override) -amdgpu_gmc_override_vm_pte_flags (adev, params->vm, addr, &flags); +num_possible_nodes () >1 && params->allow_override) { +if (params->pages_addr) +amdgpu_gmc_override_vm_pte_flags (adev, params->vm, + params->pages_addr[addr >>PAGE_SHIFT ], &flags); +else +amdgpu_gmc_override_vm_pte_flags (adev, params->vm, addr, &flags); + } params->vm->update_funcs->update (params, pt, pe, addr, count, incr, flags);
Quoted 54 lines · Xiaogang
Quoted 38 lines · Philip Yang
On multi-socket MI300A APU systems, system memory pages mapped to the closest GPU must use
MTYPE_RWinstead ofMTYPE_NCto maintain correct cache coherence. The existing mtype override inamdgpu_vm_pte_update_flags()excluded non-contiguous page mappings from the override. This caused incorrectMTYPE_NCfor scattered local pages, leading to cache coherence issues.The override applies to both contiguous and non-contiguous mappings. When
pages_addris set, resolve the physical address via pages_addr[addr >>PAGE_SHIFT] before passing it to the override callback for NUMA node lookup.Introduce
amdgpu_vm_addr_contiguous()helper that, on MI300A, treats pages on different NUMA nodes as non-contiguous even if their DMA addresses are adjacent. This ensuresamdgpu_vm_update_range()splits page table updates at NUMA node boundaries so each batch gets the correct mtype override.Signed-off-by Philip Yang<[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 48 +++++++++++++++++++---- drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 14 +++++-- 2 files changed, 50 insertions(+), 12 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +6
-0@@ -1099,6 +1099,34 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params, } } +/** +* amdgpu_vm_addr_contiguous - check if two DMA addresses are contiguous +* +* @adev: amdgpu_device pointer +* @addr: current DMA address +* @addr_next: next DMA address to check against drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params, · line 1117
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +10
-0+* @contiguous: current contiguity state of the range being built +* +* Check whether @addr and @addr_next are physically contiguous. On APU +* platforms with multiple NUMA nodes (e.g. MI300A), a NUMA node boundary +* also breaks contiguity so that each contiguous batch stays within a +* single NUMA node for correct MTYPE override selection. +* +* Returns: +* true if @addr_next continues the current contiguous range, false otherwise. +*/ We can use
pfn_to_nidorpage_to_nidto get which noma(id) the backing memory is at.pfn_to_niduses pfn from physical address. You usedma_addr_tthat is device dependent. It is not always same as physical address of RAM.
Quoted 6 lines
ttm_ttalso has/** @pages: Array of pages backing the data. */ struct page **pages;
I think using the pages to get numa id by
page_to_nidis more appropriate.
For MI300A APU, iommu is not enabled, dma_address (dma_map_page) is same as pfn (page_to_pfn), otherwise the current override path will not work correctly with pfn_to_nid.
Regards,
Philip
Quoted 83 lines
Regards
Xiaogang
Quoted 78 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +24
-8+static inline bool amdgpu_vm_addr_contiguous (struct amdgpu_device *adev,dma_addr_t addr, +dma_addr_t addr_next,bool contiguous) +{ +if (!adev->gmc.is_app_apu || !page_is_ram (addr >>PAGE_SHIFT )) +return (addr +PAGE_SIZE ) == addr_next; + +if (pfn_to_nid (addr >>PAGE_SHIFT ) !=pfn_to_nid (addr_next >>PAGE_SHIFT )) +return !contiguous; + +return (addr +PAGE_SIZE ) == addr_next; +} +/** * amdgpu_vm_update_range - update a range in the vm page table * @@ -1198,22 +1226,26 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,uint64_t pfn = cursor.start >>PAGE_SHIFT ;uint64_t count; - contiguous = pages_addr[pfn +1 ] == - pages_addr[pfn] +PAGE_SIZE ; + contiguous =amdgpu_vm_addr_contiguous (adev, + pages_addr[pfn], + pages_addr[pfn +1 ], + contiguous); - tmp = num_entries / -AMDGPU_GPU_PAGES_IN_CPU_PAGE ; + tmp = num_entries /AMDGPU_GPU_PAGES_IN_CPU_PAGE ;for (count =2 ; count < tmp; ++count) {uint64_t idx = pfn + count; -if (contiguous != (pages_addr[idx] == - pages_addr[idx -1 ] +PAGE_SIZE )) +if (contiguous !=amdgpu_vm_addr_contiguous (adev, + pages_addr[idx -1 ], + pages_addr[idx], + contiguous))break ; } +if (!contiguous) count--; - num_entries = count * -AMDGPU_GPU_PAGES_IN_CPU_PAGE ; + + num_entries = count *AMDGPU_GPU_PAGES_IN_CPU_PAGE ; }if (!contiguous) {drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +10
-4@@ -708,13 +708,19 @@ static void amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params *params,amdgpu_vm_pte_update_noretry_flags (adev, &flags);/* APUs mapping system memory may need different MTYPEs on different -* NUMA nodes. Only do this for contiguous ranges that can be assumed -* to be on the same NUMA node. +* NUMA nodes. Both contiguous and non-contiguous ranges are handled +* since amdgpu_vm_update_range ensures updates don't span NUMA +* node boundaries. */ if ((flags &AMDGPU_PTE_SYSTEM ) && (adev->flags &AMD_IS_APU ) && adev->gmc.gmc_funcs->override_vm_pte_flags && -num_possible_nodes () >1 && !params->pages_addr && params->allow_override) -amdgpu_gmc_override_vm_pte_flags (adev, params->vm, addr, &flags); +num_possible_nodes () >1 && params->allow_override) { +if (params->pages_addr) +amdgpu_gmc_override_vm_pte_flags (adev, params->vm, + params->pages_addr[addr >>PAGE_SHIFT ], &flags); +else +amdgpu_gmc_override_vm_pte_flags (adev, params->vm, addr, &flags); + } params->vm->update_funcs->update (params, pt, pe, addr, count, incr, flags);
Christian König
maintainerQuoted 44 lines · Xiaogang
Quoted 42 lines · Philip Yang
On multi-socket MI300A APU systems, system memory pages mapped to the closest GPU must use
MTYPE_RWinstead ofMTYPE_NCto maintain correct cache coherence. The existing mtype override inamdgpu_vm_pte_update_flags()excluded non-contiguous page mappings from the override. This caused incorrectMTYPE_NCfor scattered local pages, leading to cache coherence issues.The override applies to both contiguous and non-contiguous mappings. When
pages_addris set, resolve the physical address via pages_addr[addr >>PAGE_SHIFT] before passing it to the override callback for NUMA node lookup.Introduce
amdgpu_vm_addr_contiguous()helper that, on MI300A, treats pages on different NUMA nodes as non-contiguous even if their DMA addresses are adjacent. This ensuresamdgpu_vm_update_range()splits page table updates at NUMA node boundaries so each batch gets the correct mtype override.Signed-off-by Philip Yang <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 48 +++++++++++++++++++---- drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 14 +++++-- 2 files changed, 50 insertions(+), 12 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +10
-0@@ -1099,6 +1099,34 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params, } } +/** +* amdgpu_vm_addr_contiguous - check if two DMA addresses are contiguous +* +* @adev: amdgpu_device pointer +* @addr: current DMA address +* @addr_next: next DMA address to check against +* @contiguous: current contiguity state of the range being built +* +* Check whether @addr and @addr_next are physically contiguous. On APU +* platforms with multiple NUMA nodes (e.g. MI300A), a NUMA node boundary
Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params, · line 1117
Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +6
-0+* also breaks contiguity so that each contiguous batch stays within a +* single NUMA node for correct MTYPE override selection. +* +* Returns: +* true if @addr_next continues the current contiguous range, false otherwise. +*/ We can use
pfn_to_nidorpage_to_nidto get which noma(id) the backing memory is at. pfn_to_nid uses pfn from physical address. You usedma_addr_tthat is device dependent. It is not always same as physical address of RAM.
Yeah that here won't work at all.
Quoted 6 lines
ttm_ttalso has/** @pages: Array of pages backing the data. */ struct page **pages;
I think using the pages to get numa id by page_to_nid is more appropriate.
That array isn't filled in for imported pages.
As far as I can see the whole approach won't work reliable. For imports we only know the dma_addr and not the struct page nor the pfn.
I think we need to re-iterate the whole idea of MTYPE override.
Regards,
Christian.
Quoted 83 lines
Regards
Xiaogang
Quoted 78 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +24
-8+static inline bool amdgpu_vm_addr_contiguous (struct amdgpu_device *adev,dma_addr_t addr, +dma_addr_t addr_next,bool contiguous) +{ +if (!adev->gmc.is_app_apu || !page_is_ram (addr >>PAGE_SHIFT )) +return (addr +PAGE_SIZE ) == addr_next; + +if (pfn_to_nid (addr >>PAGE_SHIFT ) !=pfn_to_nid (addr_next >>PAGE_SHIFT )) +return !contiguous; + +return (addr +PAGE_SIZE ) == addr_next; +} +/** * amdgpu_vm_update_range - update a range in the vm page table * @@ -1198,22 +1226,26 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,uint64_t pfn = cursor.start >>PAGE_SHIFT ;uint64_t count; - contiguous = pages_addr[pfn +1 ] == - pages_addr[pfn] +PAGE_SIZE ; + contiguous =amdgpu_vm_addr_contiguous (adev, + pages_addr[pfn], + pages_addr[pfn +1 ], + contiguous); - tmp = num_entries / -AMDGPU_GPU_PAGES_IN_CPU_PAGE ; + tmp = num_entries /AMDGPU_GPU_PAGES_IN_CPU_PAGE ;for (count =2 ; count < tmp; ++count) {uint64_t idx = pfn + count; -if (contiguous != (pages_addr[idx] == - pages_addr[idx -1 ] +PAGE_SIZE )) +if (contiguous !=amdgpu_vm_addr_contiguous (adev, + pages_addr[idx -1 ], + pages_addr[idx], + contiguous))break ; } +if (!contiguous) count--; - num_entries = count * -AMDGPU_GPU_PAGES_IN_CPU_PAGE ; + + num_entries = count *AMDGPU_GPU_PAGES_IN_CPU_PAGE ; }if (!contiguous) {drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +10
-4@@ -708,13 +708,19 @@ static void amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params *params,amdgpu_vm_pte_update_noretry_flags (adev, &flags);/* APUs mapping system memory may need different MTYPEs on different -* NUMA nodes. Only do this for contiguous ranges that can be assumed -* to be on the same NUMA node. +* NUMA nodes. Both contiguous and non-contiguous ranges are handled +* since amdgpu_vm_update_range ensures updates don't span NUMA +* node boundaries. */ if ((flags &AMDGPU_PTE_SYSTEM ) && (adev->flags &AMD_IS_APU ) && adev->gmc.gmc_funcs->override_vm_pte_flags && -num_possible_nodes () >1 && !params->pages_addr && params->allow_override) -amdgpu_gmc_override_vm_pte_flags (adev, params->vm, addr, &flags); +num_possible_nodes () >1 && params->allow_override) { +if (params->pages_addr) +amdgpu_gmc_override_vm_pte_flags (adev, params->vm, + params->pages_addr[addr >>PAGE_SHIFT ], &flags); +else +amdgpu_gmc_override_vm_pte_flags (adev, params->vm, addr, &flags); + } params->vm->update_funcs->update (params, pt, pe, addr, count, incr, flags);
Quoted 53 lines · Christian König
Quoted 42 lines · Xiaogang
Quoted 41 lines · Philip Yang
On multi-socket MI300A APU systems, system memory pages mapped to the closest GPU must use
MTYPE_RWinstead ofMTYPE_NCto maintain correct cache coherence. The existing mtype override inamdgpu_vm_pte_update_flags()excluded non-contiguous page mappings from the override. This caused incorrectMTYPE_NCfor scattered local pages, leading to cache coherence issues.The override applies to both contiguous and non-contiguous mappings. When
pages_addris set, resolve the physical address via pages_addr[addr >>PAGE_SHIFT] before passing it to the override callback for NUMA node lookup.Introduce
amdgpu_vm_addr_contiguous()helper that, on MI300A, treats pages on different NUMA nodes as non-contiguous even if their DMA addresses are adjacent. This ensuresamdgpu_vm_update_range()splits page table updates at NUMA node boundaries so each batch gets the correct mtype override.Signed-off-by Philip Yang<[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 48 +++++++++++++++++++---- drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 14 +++++-- 2 files changed, 50 insertions(+), 12 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +9
-0@@ -1099,6 +1099,34 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params, } } +/** +* amdgpu_vm_addr_contiguous - check if two DMA addresses are contiguous +* +* @adev: amdgpu_device pointer +* @addr: current DMA address +* @addr_next: next DMA address to check against +* @contiguous: current contiguity state of the range being built +* +* Check whether @addr and @addr_next are physically contiguous. On APU Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params, · line 1117
Quoted 7 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +7
-0+* platforms with multiple NUMA nodes (e.g. MI300A), a NUMA node boundary +* also breaks contiguity so that each contiguous batch stays within a +* single NUMA node for correct MTYPE override selection. +* +* Returns: +* true if @addr_next continues the current contiguous range, false otherwise. +*/ We can use
pfn_to_nidorpage_to_nidto get which noma(id) the backing memory is at. pfn_to_nid uses pfn from physical address. You usedma_addr_tthat is device dependent. It is not always same as physical address of RAM.Yeah that here won't work at all.
Quoted 8 lines
Quoted 5 lines
ttm_ttalso has/** @pages: Array of pages backing the data. */ struct page **pages;
I think using the pages to get numa id by page_to_nid is more appropriate.
That array isn't filled in for imported pages.
As far as I can see the whole approach won't work reliable. For imports we only know the
dma_addrand not the struct page nor the pfn.
if adev->ram_is_direct_mapped, then dma_addr equals to pfn, we can use dma_addr to call pfn_to_nid.
I will add ram_is_direct_mapped condition, this is currently inside override function, and add fast path change suggested by Felix in next version.
Regards,
Philip
Quoted 88 lines
I think we need to re-iterate the whole idea of MTYPE override.
Regards,
Christian.Quoted 82 lines
Regards
Xiaogang
Quoted 78 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +24
-8+static inline bool amdgpu_vm_addr_contiguous (struct amdgpu_device *adev,dma_addr_t addr, +dma_addr_t addr_next,bool contiguous) +{ +if (!adev->gmc.is_app_apu || !page_is_ram (addr >>PAGE_SHIFT )) +return (addr +PAGE_SIZE ) == addr_next; + +if (pfn_to_nid (addr >>PAGE_SHIFT ) !=pfn_to_nid (addr_next >>PAGE_SHIFT )) +return !contiguous; + +return (addr +PAGE_SIZE ) == addr_next; +} +/** * amdgpu_vm_update_range - update a range in the vm page table * @@ -1198,22 +1226,26 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,uint64_t pfn = cursor.start >>PAGE_SHIFT ;uint64_t count; - contiguous = pages_addr[pfn +1 ] == - pages_addr[pfn] +PAGE_SIZE ; + contiguous =amdgpu_vm_addr_contiguous (adev, + pages_addr[pfn], + pages_addr[pfn +1 ], + contiguous); - tmp = num_entries / -AMDGPU_GPU_PAGES_IN_CPU_PAGE ; + tmp = num_entries /AMDGPU_GPU_PAGES_IN_CPU_PAGE ;for (count =2 ; count < tmp; ++count) {uint64_t idx = pfn + count; -if (contiguous != (pages_addr[idx] == - pages_addr[idx -1 ] +PAGE_SIZE )) +if (contiguous !=amdgpu_vm_addr_contiguous (adev, + pages_addr[idx -1 ], + pages_addr[idx], + contiguous))break ; } +if (!contiguous) count--; - num_entries = count * -AMDGPU_GPU_PAGES_IN_CPU_PAGE ; + + num_entries = count *AMDGPU_GPU_PAGES_IN_CPU_PAGE ; }if (!contiguous) {drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +10
-4@@ -708,13 +708,19 @@ static void amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params *params,amdgpu_vm_pte_update_noretry_flags (adev, &flags);/* APUs mapping system memory may need different MTYPEs on different -* NUMA nodes. Only do this for contiguous ranges that can be assumed -* to be on the same NUMA node. +* NUMA nodes. Both contiguous and non-contiguous ranges are handled +* since amdgpu_vm_update_range ensures updates don't span NUMA +* node boundaries. */ if ((flags &AMDGPU_PTE_SYSTEM ) && (adev->flags &AMD_IS_APU ) && adev->gmc.gmc_funcs->override_vm_pte_flags && -num_possible_nodes () >1 && !params->pages_addr && params->allow_override) -amdgpu_gmc_override_vm_pte_flags (adev, params->vm, addr, &flags); +num_possible_nodes () >1 && params->allow_override) { +if (params->pages_addr) +amdgpu_gmc_override_vm_pte_flags (adev, params->vm, + params->pages_addr[addr >>PAGE_SHIFT ], &flags); +else +amdgpu_gmc_override_vm_pte_flags (adev, params->vm, addr, &flags); + } params->vm->update_funcs->update (params, pt, pe, addr, count, incr, flags);
Christian König
maintainerQuoted 56 lines · Philip Yang
Quoted 55 lines · Christian König
Quoted 42 lines · Xiaogang
Quoted 41 lines · Philip Yang
On multi-socket MI300A APU systems, system memory pages mapped to the closest GPU must use
MTYPE_RWinstead ofMTYPE_NCto maintain correct cache coherence. The existing mtype override inamdgpu_vm_pte_update_flags()excluded non-contiguous page mappings from the override. This caused incorrectMTYPE_NCfor scattered local pages, leading to cache coherence issues.The override applies to both contiguous and non-contiguous mappings. When
pages_addris set, resolve the physical address via pages_addr[addr >>PAGE_SHIFT] before passing it to the override callback for NUMA node lookup.Introduce
amdgpu_vm_addr_contiguous()helper that, on MI300A, treats pages on different NUMA nodes as non-contiguous even if their DMA addresses are adjacent. This ensuresamdgpu_vm_update_range()splits page table updates at NUMA node boundaries so each batch gets the correct mtype override.Signed-off-by Philip Yang <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 48 +++++++++++++++++++---- drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 14 +++++-- 2 files changed, 50 insertions(+), 12 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +9
-0@@ -1099,6 +1099,34 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params, } } +/** +* amdgpu_vm_addr_contiguous - check if two DMA addresses are contiguous +* +* @adev: amdgpu_device pointer +* @addr: current DMA address +* @addr_next: next DMA address to check against +* @contiguous: current contiguity state of the range being built +* +* Check whether @addr and @addr_next are physically contiguous. On APU Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params, · line 1117
Quoted 7 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +7
-0+* platforms with multiple NUMA nodes (e.g. MI300A), a NUMA node boundary +* also breaks contiguity so that each contiguous batch stays within a +* single NUMA node for correct MTYPE override selection. +* +* Returns: +* true if @addr_next continues the current contiguous range, false otherwise. +*/ We can use
pfn_to_nidorpage_to_nidto get which noma(id) the backing memory is at. pfn_to_nid uses pfn from physical address. You usedma_addr_tthat is device dependent. It is not always same as physical address of RAM.Yeah that here won't work at all.
Quoted 2 lines
ttm_ttalso has
Quoted 8 lines
Quoted 6 lines
Quoted 3 lines
/** @pages: Array of pages backing the data. */ struct page **pages;
I think using the pages to get numa id by page_to_nid is more appropriate.
That array isn't filled in for imported pages.
As far as I can see the whole approach won't work reliable. For imports we only know the
dma_addrand not the struct page nor the pfn.if
adev->ram_is_direct_mapped, thendma_addrequals to pfn, we can usedma_addrto callpfn_to_nid.
I will addram_is_direct_mappedcondition, this is currently inside override function, and add fast path change suggested by Felix in next version.
That is seriously not something we can do. This relies an specific HW behavior in common code and is clearly a no-go from my side.
Regards,
Christian.
Quoted 91 lines
Regards,
PhilipQuoted 87 lines
I think we need to re-iterate the whole idea of MTYPE override.
Regards,
Christian.Quoted 82 lines
Regards
Xiaogang
Quoted 78 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +24
-8+static inline bool amdgpu_vm_addr_contiguous (struct amdgpu_device *adev,dma_addr_t addr, +dma_addr_t addr_next,bool contiguous) +{ +if (!adev->gmc.is_app_apu || !page_is_ram (addr >>PAGE_SHIFT )) +return (addr +PAGE_SIZE ) == addr_next; + +if (pfn_to_nid (addr >>PAGE_SHIFT ) !=pfn_to_nid (addr_next >>PAGE_SHIFT )) +return !contiguous; + +return (addr +PAGE_SIZE ) == addr_next; +} +/** * amdgpu_vm_update_range - update a range in the vm page table * @@ -1198,22 +1226,26 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,uint64_t pfn = cursor.start >>PAGE_SHIFT ;uint64_t count; - contiguous = pages_addr[pfn +1 ] == - pages_addr[pfn] +PAGE_SIZE ; + contiguous =amdgpu_vm_addr_contiguous (adev, + pages_addr[pfn], + pages_addr[pfn +1 ], + contiguous); - tmp = num_entries / -AMDGPU_GPU_PAGES_IN_CPU_PAGE ; + tmp = num_entries /AMDGPU_GPU_PAGES_IN_CPU_PAGE ;for (count =2 ; count < tmp; ++count) {uint64_t idx = pfn + count; -if (contiguous != (pages_addr[idx] == - pages_addr[idx -1 ] +PAGE_SIZE )) +if (contiguous !=amdgpu_vm_addr_contiguous (adev, + pages_addr[idx -1 ], + pages_addr[idx], + contiguous))break ; } +if (!contiguous) count--; - num_entries = count * -AMDGPU_GPU_PAGES_IN_CPU_PAGE ; + + num_entries = count *AMDGPU_GPU_PAGES_IN_CPU_PAGE ; }if (!contiguous) {drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +10
-4@@ -708,13 +708,19 @@ static void amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params *params,amdgpu_vm_pte_update_noretry_flags (adev, &flags);/* APUs mapping system memory may need different MTYPEs on different -* NUMA nodes. Only do this for contiguous ranges that can be assumed -* to be on the same NUMA node. +* NUMA nodes. Both contiguous and non-contiguous ranges are handled +* since amdgpu_vm_update_range ensures updates don't span NUMA +* node boundaries. */ if ((flags &AMDGPU_PTE_SYSTEM ) && (adev->flags &AMD_IS_APU ) && adev->gmc.gmc_funcs->override_vm_pte_flags && -num_possible_nodes () >1 && !params->pages_addr && params->allow_override) -amdgpu_gmc_override_vm_pte_flags (adev, params->vm, addr, &flags); +num_possible_nodes () >1 && params->allow_override) { +if (params->pages_addr) +amdgpu_gmc_override_vm_pte_flags (adev, params->vm, + params->pages_addr[addr >>PAGE_SHIFT ], &flags); +else +amdgpu_gmc_override_vm_pte_flags (adev, params->vm, addr, &flags); + } params->vm->update_funcs->update (params, pt, pe, addr, count, incr, flags);
Quoted 29 lines · Philip Yang
On multi-socket MI300A APU systems, system memory pages mapped to the closest GPU must use
MTYPE_RWinstead ofMTYPE_NCto maintain correct cache coherence. The existing mtype override inamdgpu_vm_pte_update_flags()excluded non-contiguous page mappings from the override. This caused incorrectMTYPE_NCfor scattered local pages, leading to cache coherence issues.The override applies to both contiguous and non-contiguous mappings. When
pages_addris set, resolve the physical address via pages_addr[addr >>PAGE_SHIFT] before passing it to the override callback for NUMA node lookup.Introduce
amdgpu_vm_addr_contiguous()helper that, on MI300A, treats pages on different NUMA nodes as non-contiguous even if their DMA addresses are adjacent. This ensuresamdgpu_vm_update_range()splits page table updates at NUMA node boundaries so each batch gets the correct mtype override.Signed-off-by Philip Yang<[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 48 +++++++++++++++++++---- drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 14 +++++-- 2 files changed, 50 insertions(+), 12 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +0
-0
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params, · line 1103
Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +2
-0@@ -1099,6 +1099,34 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params, } } +/** +* amdgpu_vm_addr_contiguous - check if two DMA addresses are contiguous
The function name is a bit misleading. It doesn't return whether the pages are contiguous, but whether they are in the same "group" (same amdgpu_vm_ptes_update call) by some pretty arbitrary criteria. Maybe call it amdgpu_vm_addr_same_group.
Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +8
-0+* +* @adev: amdgpu_device pointer +* @addr: current DMA address +* @addr_next: next DMA address to check against +* @contiguous: current contiguity state of the range being built +* +* Check whether @addr and @addr_next are physically contiguous. On APU +* platforms with multiple NUMA nodes (e.g. MI300A), a NUMA node boundary
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params, · line 1121
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +10
-0+* also breaks contiguity so that each contiguous batch stays within a +* single NUMA node for correct MTYPE override selection. +* +* Returns: +* true if @addr_next continues the current contiguous range, false otherwise. +*/ +static inline bool amdgpu_vm_addr_contiguous (struct amdgpu_device *adev,dma_addr_t addr, +dma_addr_t addr_next,bool contiguous) +{ +if (!adev->gmc.is_app_apu || !page_is_ram (addr >>PAGE_SHIFT ))
Isn't page_is_ram always true on APP APUs?
On the other hand, I think you need a check here that IOMMU device isolation is off (adev->ram_is_direct_mapped). Otherwise you cannot infer the NUMA node from the DMA address. I'd put the condition for that in the caller where you only need to check it once. Then replace the adev parameter with a bool same_nid. And while you're at it, you can add some more conditions to make the fast-path more likely:
same_nid = adev->gmc.is_app_apu && adev->ram_is_direct_mapped && adev->gmc.gmc_funcs->override_vm_pte_flags && num_possible_nodes() > 1 && params->allow_override;
Regards,
Felix
Quoted 75 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +20
-8+return (addr +PAGE_SIZE ) == addr_next; + +if (pfn_to_nid (addr >>PAGE_SHIFT ) !=pfn_to_nid (addr_next >>PAGE_SHIFT )) +return !contiguous; + +return (addr +PAGE_SIZE ) == addr_next; +} +/** * amdgpu_vm_update_range - update a range in the vm page table * @@ -1198,22 +1226,26 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,uint64_t pfn = cursor.start >>PAGE_SHIFT ;uint64_t count; - contiguous = pages_addr[pfn +1 ] == - pages_addr[pfn] +PAGE_SIZE ; + contiguous =amdgpu_vm_addr_contiguous (adev, + pages_addr[pfn], + pages_addr[pfn +1 ], + contiguous); - tmp = num_entries / -AMDGPU_GPU_PAGES_IN_CPU_PAGE ; + tmp = num_entries /AMDGPU_GPU_PAGES_IN_CPU_PAGE ;for (count =2 ; count < tmp; ++count) {uint64_t idx = pfn + count; -if (contiguous != (pages_addr[idx] == - pages_addr[idx -1 ] +PAGE_SIZE )) +if (contiguous !=amdgpu_vm_addr_contiguous (adev, + pages_addr[idx -1 ], + pages_addr[idx], + contiguous))break ; } +if (!contiguous) count--; - num_entries = count * -AMDGPU_GPU_PAGES_IN_CPU_PAGE ; + + num_entries = count *AMDGPU_GPU_PAGES_IN_CPU_PAGE ; }if (!contiguous) {drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +10
-4@@ -708,13 +708,19 @@ static void amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params *params,amdgpu_vm_pte_update_noretry_flags (adev, &flags);/* APUs mapping system memory may need different MTYPEs on different -* NUMA nodes. Only do this for contiguous ranges that can be assumed -* to be on the same NUMA node. +* NUMA nodes. Both contiguous and non-contiguous ranges are handled +* since amdgpu_vm_update_range ensures updates don't span NUMA +* node boundaries. */ if ((flags &AMDGPU_PTE_SYSTEM ) && (adev->flags &AMD_IS_APU ) && adev->gmc.gmc_funcs->override_vm_pte_flags && -num_possible_nodes () >1 && !params->pages_addr && params->allow_override) -amdgpu_gmc_override_vm_pte_flags (adev, params->vm, addr, &flags); +num_possible_nodes () >1 && params->allow_override) { +if (params->pages_addr) +amdgpu_gmc_override_vm_pte_flags (adev, params->vm, + params->pages_addr[addr >>PAGE_SHIFT ], &flags); +else +amdgpu_gmc_override_vm_pte_flags (adev, params->vm, addr, &flags); + } params->vm->update_funcs->update (params, pt, pe, addr, count, incr, flags);
Quoted 33 lines · Felix Kuehling
Quoted 32 lines · Philip Yang
On multi-socket MI300A APU systems, system memory pages mapped to the closest GPU must use
MTYPE_RWinstead ofMTYPE_NCto maintain correct cache coherence. The existing mtype override inamdgpu_vm_pte_update_flags()excluded non-contiguous page mappings from the override. This caused incorrectMTYPE_NCfor scattered local pages, leading to cache coherence issues.The override applies to both contiguous and non-contiguous mappings. When
pages_addris set, resolve the physical address via pages_addr[addr >>PAGE_SHIFT] before passing it to the override callback for NUMA node lookup.Introduce
amdgpu_vm_addr_contiguous()helper that, on MI300A, treats pages on different NUMA nodes as non-contiguous even if their DMA addresses are adjacent. This ensuresamdgpu_vm_update_range()splits page table updates at NUMA node boundaries so each batch gets the correct mtype override.Signed-off-by Philip Yang<[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 48 +++++++++++++++++++---- drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 14 +++++-- 2 files changed, 50 insertions(+), 12 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +0
-0@@ -1099,6 +1099,34 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params, } }
Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params, · line 1103
Quoted 2 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +2
-0+/** +* amdgpu_vm_addr_contiguous - check if two DMA addresses are contiguous The function name is a bit misleading. It doesn't return whether the pages are contiguous, but whether they are in the same "group" (same
amdgpu_vm_ptes_updatecall) by some pretty arbitrary criteria. Maybe call itamdgpu_vm_addr_same_group.
ok
Quoted 12 lines
Quoted 12 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +12
-0+* +* @adev: amdgpu_device pointer +* @addr: current DMA address +* @addr_next: next DMA address to check against +* @contiguous: current contiguity state of the range being built +* +* Check whether @addr and @addr_next are physically contiguous. On APU +* platforms with multiple NUMA nodes (e.g. MI300A), a NUMA node boundary +* also breaks contiguity so that each contiguous batch stays within a +* single NUMA node for correct MTYPE override selection. +* +* Returns:
Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params, · line 1121
Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +6
-0+* true if @addr_next continues the current contiguous range, false otherwise. +*/ +static inline bool amdgpu_vm_addr_contiguous (struct amdgpu_device *adev,dma_addr_t addr, +dma_addr_t addr_next,bool contiguous) +{ +if (!adev->gmc.is_app_apu || !page_is_ram (addr >>PAGE_SHIFT ))Isn't
page_is_ramalways true on APP APUs?
There is address (probably doorbell) outside ram space mapping to GPU, without structure page and cause crash inside pfn_to_nid. gmc_v9_0_override_vm_pte_flags has the same checking too.
Quoted 3 lines
On the other hand, I think you need a check here that IOMMU device isolation is off (
adev->ram_is_direct_mapped). Otherwise you cannot infer the NUMA node from the DMA address.
yes, should add !adev->ram_is_direct_mapped
Quoted 6 lines
I'd put the condition for that in the caller where you only need to check it once. Then replace the adev parameter with a bool
same_nid. And while you're at it, you can add some more conditions to make the fast-path more likely:same_nid = adev->gmc.is_app_apu && adev->ram_is_direct_mapped && adev->gmc.gmc_funcs->override_vm_pte_flags && num_possible_nodes() > 1 && params->allow_override;
done in next version.
Regards,
Philip
Quoted 80 lines
Regards,
FelixQuoted 75 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +20
-8+return (addr +PAGE_SIZE ) == addr_next; + +if (pfn_to_nid (addr >>PAGE_SHIFT ) !=pfn_to_nid (addr_next >>PAGE_SHIFT )) +return !contiguous; + +return (addr +PAGE_SIZE ) == addr_next; +} +/** * amdgpu_vm_update_range - update a range in the vm page table * @@ -1198,22 +1226,26 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,uint64_t pfn = cursor.start >>PAGE_SHIFT ;uint64_t count; - contiguous = pages_addr[pfn +1 ] == - pages_addr[pfn] +PAGE_SIZE ; + contiguous =amdgpu_vm_addr_contiguous (adev, + pages_addr[pfn], + pages_addr[pfn +1 ], + contiguous); - tmp = num_entries / -AMDGPU_GPU_PAGES_IN_CPU_PAGE ; + tmp = num_entries /AMDGPU_GPU_PAGES_IN_CPU_PAGE ;for (count =2 ; count < tmp; ++count) {uint64_t idx = pfn + count; -if (contiguous != (pages_addr[idx] == - pages_addr[idx -1 ] +PAGE_SIZE )) +if (contiguous !=amdgpu_vm_addr_contiguous (adev, + pages_addr[idx -1 ], + pages_addr[idx], + contiguous))break ; } +if (!contiguous) count--; - num_entries = count * -AMDGPU_GPU_PAGES_IN_CPU_PAGE ; + + num_entries = count *AMDGPU_GPU_PAGES_IN_CPU_PAGE ; }if (!contiguous) {drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +10
-4@@ -708,13 +708,19 @@ static void amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params *params,amdgpu_vm_pte_update_noretry_flags (adev, &flags);/* APUs mapping system memory may need different MTYPEs on different -* NUMA nodes. Only do this for contiguous ranges that can be assumed -* to be on the same NUMA node. +* NUMA nodes. Both contiguous and non-contiguous ranges are handled +* since amdgpu_vm_update_range ensures updates don't span NUMA +* node boundaries. */ if ((flags &AMDGPU_PTE_SYSTEM ) && (adev->flags &AMD_IS_APU ) && adev->gmc.gmc_funcs->override_vm_pte_flags && -num_possible_nodes () >1 && !params->pages_addr && params->allow_override) -amdgpu_gmc_override_vm_pte_flags (adev, params->vm, addr, &flags); +num_possible_nodes () >1 && params->allow_override) { +if (params->pages_addr) +amdgpu_gmc_override_vm_pte_flags (adev, params->vm, + params->pages_addr[addr >>PAGE_SHIFT ], &flags); +else +amdgpu_gmc_override_vm_pte_flags (adev, params->vm, addr, &flags); + } params->vm->update_funcs->update (params, pt, pe, addr, count, incr, flags);