Speedup mremap on ppc64
v8 cover Speedup mremap on ppc64
This patchset enables MOVE_PMD/MOVE_PUD support on power. This requires the platform to support updating higher-level page tables without updating page table entries. This also needs to invalidate the Page Walk Cache on architecture supporting the same.
This patchset is dependent on
https://lore.kernel.org/linux-mm/[email protected]/
Changes from v7:
* Split mremap fixes to a separate series
Changes from v6:
* Update ppc64 flush_tlb_range to invalidate page walk cache.
* Add patches to fix race between mremap and page out
* Add patch to fix build error with page table levels 2
Changes from v5:
* Drop patch mm/mremap: Move TLB flush outside page table lock
* Add fixes for race between optimized mremap and page out
Changes from v4:
* Change function name and arguments based on review feedback.
Changes from v3:
* Fix build error reported by kernel test robot
* Address review feedback.
Changes from v2:
* switch from using mmu_gather to flush_pte_tlb_pwc_range()
Changes from v1:
* Rebase to recent upstream
* Fix build issues with tlb_gather_mmu changes
Aneesh Kumar K.V (3):
mm/mremap: Allow arch runtime override powerpc/book3s64/mm: Update flush_tlb_range to flush page walk cache powerpc/mm: Enable HAVE_MOVE_PMD support
.../include/asm/book3s/64/tlbflush-radix.h | 2 + arch/powerpc/include/asm/tlb.h | 6 +++ arch/powerpc/mm/book3s64/radix_hugetlbpage.c | 8 +++- arch/powerpc/mm/book3s64/radix_tlb.c | 44 ++++++++++++------- arch/powerpc/platforms/Kconfig.cputype | 2 + mm/mremap.c | 15 ++++++- 6 files changed, 58 insertions(+), 19 deletions(-)
Signature · 1 line
2.31.1
PATCH v8 1/3 mm/mremap: Allow arch runtime override
Architectures like ppc64 support faster mremap only with radix translation. Hence allow a runtime check w.r.t support for fast mremap.
--- arch/powerpc/include/asm/tlb.h | 6 ++++++ mm/mremap.c | 15 ++++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-)
arch/powerpc/include/asm/tlb.h +6 -0
@@ -83,5 +83,11 @@ static inline int mm_is_thread_local(struct mm_struct *mm) }#endif +#define arch_supports_page_table_move arch_supports_page_table_move +static inline bool arch_supports_page_table_move (void ) +{ +return radix_enabled (); +} +#endif /* __KERNEL__ */ #endif /* __ASM_POWERPC_TLB_H */
mm/mremap.c +14 -1
@@ -25,7 +25,7 @@#include <linux/userfaultfd_k.h> #include <asm/cacheflush.h> -#include <asm/tlbflush.h> +#include <asm/tlb.h> #include <asm/pgalloc.h> #include "internal.h"
@@ -210,6 +210,15 @@ static void move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd,drop_rmap_locks (vma); } +#ifndef arch_supports_page_table_move +#define arch_supports_page_table_move arch_supports_page_table_move +static inline bool arch_supports_page_table_move (void ) +{ +return IS_ENABLED (CONFIG_HAVE_MOVE_PMD ) || +IS_ENABLED (CONFIG_HAVE_MOVE_PUD ); +} +#endif +#ifdef CONFIG_HAVE_MOVE_PMD static bool move_normal_pmd (struct vm_area_struct *vma,unsigned long old_addr,unsigned long new_addr,pmd_t *old_pmd,pmd_t *new_pmd)
@@ -218,6 +227,8 @@ static bool move_normal_pmd(struct vm_area_struct *vma, unsigned long old_addr,struct mm_struct *mm = vma->vm_mm;pmd_t pmd; +if (!arch_supports_page_table_move ()) +return false;/* * The destination pmd shouldn't be established, free_pgtables() * should have released it.
@@ -284,6 +295,8 @@ static bool move_normal_pud(struct vm_area_struct *vma, unsigned long old_addr,struct mm_struct *mm = vma->vm_mm;pud_t pud; +if (!arch_supports_page_table_move ()) +return false;/* * The destination pud shouldn't be established, free_pgtables() * should have released it.
Signature · 1 line
2.31.1
PATCH v8 2/3 powerpc/book3s64/mm: Update flush_tlb_range to flush page walk cache
In reply to Aneesh Kumar K.Vflush_tlb_range is special in that we don't specify the page size used for the translation. Hence when flushing TLB we flush the translation cache for all possible page sizes. The kernel also uses the same interface when moving page tables around. Such a move requires us to flush the page walk cache.
Instead of adding another interface to force page walk cache flush, update flush_tlb_range to flush page walk cache if the range flushed is more than the PMD range. A page table move will always involve an invalidate range more than PMD_SIZE.
Running microbenchmark with mprotect and parallel memory access didn't show any observable performance impact.
--- .../include/asm/book3s/64/tlbflush-radix.h | 2 + arch/powerpc/mm/book3s64/radix_hugetlbpage.c | 8 +++- arch/powerpc/mm/book3s64/radix_tlb.c | 44 ++++++++++++------- 3 files changed, 36 insertions(+), 18 deletions(-)
arch/powerpc/include/asm/book3s/64/tlbflush-radix.h +2 -0
@@ -60,6 +60,8 @@ extern void radix__flush_hugetlb_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);extern void radix__flush_tlb_range_psize (struct mm_struct *mm,unsigned long start,unsigned long end,int psize); +void radix__flush_tlb_pwc_range_psize (struct mm_struct *mm,unsigned long start, +unsigned long end,int psize);extern void radix__flush_pmd_tlb_range (struct vm_area_struct *vma,unsigned long start,unsigned long end); extern void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
arch/powerpc/mm/book3s64/radix_hugetlbpage.c +7 -1
@@ -32,7 +32,13 @@ void radix__flush_hugetlb_tlb_range(struct vm_area_struct *vma, unsigned long ststruct hstate *hstate =hstate_file (vma->vm_file); psize =hstate_get_psize (hstate); -radix__flush_tlb_range_psize (vma->vm_mm, start, end, psize); +/* +* Flush PWC even if we get PUD_SIZE hugetlb invalidate to keep this simpler. +*/ +if (end - start >=PUD_SIZE ) +radix__flush_tlb_pwc_range_psize (vma->vm_mm, start, end, psize); +else +radix__flush_tlb_range_psize (vma->vm_mm, start, end, psize); }/*
arch/powerpc/mm/book3s64/radix_tlb.c +27 -17
107 lines not on this page · raw message
Signature · 1 line
2.31.1
PATCH v8 3/3 powerpc/mm: Enable HAVE_MOVE_PMD support
In reply to Aneesh Kumar K.Vmremap HAVE_MOVE_PMD/PUD optimization time comparison for 1GB region:
1GB mremap - Source PTE-aligned, Destination PTE-aligned
mremap time: 2292772ns
1GB mremap - Source PMD-aligned, Destination PMD-aligned
mremap time: 1158928ns
1GB mremap - Source PUD-aligned, Destination PUD-aligned
mremap time: 63886ns
--- arch/powerpc/platforms/Kconfig.cputype | 2 ++ 1 file changed, 2 insertions(+)
arch/powerpc/platforms/Kconfig.cputype +2 -0
@@ -101,6 +101,8 @@ config PPC_BOOK3S_64 select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE select ARCH_SUPPORTS_HUGETLBFS select ARCH_SUPPORTS_NUMA_BALANCING + select HAVE_MOVE_PMD + select HAVE_MOVE_PUD select IRQ_WORK select PPC_MM_SLICES select PPC_HAVE_KUEP
Signature · 1 line
2.31.1