Add support for DAX vmemmap optimization for ppc64
Subsystems (16)
PATCH v4 1/13 mm/hugepage pud: Allow arch-specific helper function to check huge page pud support
In reply to Aneesh Kumar K.VArchitectures like powerpc would like to enable transparent huge page pud support only with radix translation. To support that add has_transparent_pud_hugepage() helper that architectures can override.
--- drivers/nvdimm/pfn_devs.c | 2 +- include/linux/pgtable.h | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-)
drivers/nvdimm/pfn_devs.c +1 -1
@@ -100,7 +100,7 @@ static unsigned long *nd_pfn_supported_alignments(unsigned long *alignments)if (has_transparent_hugepage ()) { alignments[1 ] =HPAGE_PMD_SIZE ; -if (IS_ENABLED (CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD )) +if (has_transparent_pud_hugepage ()) alignments[2 ] =HPAGE_PUD_SIZE ; }
include/linux/pgtable.h +3 -0
@@ -1499,6 +1499,9 @@ typedef unsigned int pgtbl_mod_mask;#define has_transparent_hugepage () IS_BUILTIN(CONFIG_TRANSPARENT_HUGEPAGE)#endif +#ifndef has_transparent_pud_hugepage +#define has_transparent_pud_hugepage () IS_BUILTIN(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD) +#endif /* * On some architectures it depends on the mm if the p4d/pud or pmd * layer of the page table hierarchy is folded or not.
Signature · 1 line
2.41.0
PATCH v4 2/13 mm: Change pudp_huge_get_and_clear_full take vm_area_struct as arg
In reply to Aneesh Kumar K.VWe will use this in a later patch to do tlb flush when clearing pud entries on powerpc. This is similar to commit 93a98695f2f9 ("mm: change pmdp_huge_get_and_clear_full take vm_area_struct as arg")
--- include/linux/pgtable.h | 4 ++-- mm/debug_vm_pgtable.c | 2 +- mm/huge_memory.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-)
include/linux/pgtable.h +2 -2
@@ -450,11 +450,11 @@ static inline pmd_t pmdp_huge_get_and_clear_full(struct vm_area_struct *vma,#endif #ifndef __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR_FULL -static inline pud_t pudp_huge_get_and_clear_full (struct mm_struct *mm, +static inline pud_t pudp_huge_get_and_clear_full (struct vm_area_struct *vma,unsigned long address,pud_t *pudp,int full) { -return pudp_huge_get_and_clear (mm, address, pudp); +return pudp_huge_get_and_clear (vma->vm_mm, address, pudp); }#endif #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
mm/debug_vm_pgtable.c +1 -1
@@ -385,7 +385,7 @@ static void __init pud_advanced_tests(struct pgtable_debug_args *args)WARN_ON (!(pud_write (pud) &&pud_dirty (pud)));#ifndef __PAGETABLE_PMD_FOLDED -pudp_huge_get_and_clear_full (args->mm, vaddr, args->pudp,1 ); +pudp_huge_get_and_clear_full (args->vma, vaddr, args->pudp,1 ); pud =READ_ONCE (*args->pudp);WARN_ON (!pud_none (pud));#endif /* __PAGETABLE_PMD_FOLDED */
mm/huge_memory.c +1 -1
@@ -1981,7 +1981,7 @@ int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma,if (!ptl)return 0 ; -pudp_huge_get_and_clear_full (tlb->mm, addr, pud, tlb->fullmm); +pudp_huge_get_and_clear_full (vma, addr, pud, tlb->fullmm);tlb_remove_pud_tlb_entry (tlb, pud, addr);if (vma_is_special_huge (vma)) {spin_unlock (ptl);
Signature · 1 line
2.41.0
PATCH v4 3/13 mm/vmemmap: Improve vmemmap_can_optimize and allow architectures to override
In reply to Aneesh Kumar K.Vdax vmemmap optimization requires a minimum of 2 PAGE_SIZE area within vmemmap such that tail page mapping can point to the second PAGE_SIZE area. Enforce that in vmemmap_can_optimize() function.
Architectures like powerpc also want to enable vmemmap optimization conditionally (only with radix MMU translation). Hence allow architecture override.
--- include/linux/mm.h | 27 +++++++++++++++++++++++---- mm/mm_init.c | 2 +- 2 files changed, 24 insertions(+), 5 deletions(-)
include/linux/mm.h +23 -4
@@ -3639,13 +3639,32 @@ void vmemmap_free(unsigned long start, unsigned long end, struct vmem_altmap *altmap);#endif +#define VMEMMAP_RESERVE_NR 2#ifdef CONFIG_ARCH_WANT_OPTIMIZE_VMEMMAP -static inline bool vmemmap_can_optimize (struct vmem_altmap *altmap, -struct dev_pagemap *pgmap) +static inline bool __vmemmap_can_optimize (struct vmem_altmap *altmap, +struct dev_pagemap *pgmap) { -return is_power_of_2 (sizeof (struct page )) && - pgmap && (pgmap_vmemmap_nr (pgmap) >1 ) && !altmap; +unsigned long nr_pages; +unsigned long nr_vmemmap_pages; + +if (!pgmap || !is_power_of_2 (sizeof (struct page ))) +return false; + + nr_pages =pgmap_vmemmap_nr (pgmap); + nr_vmemmap_pages = ((nr_pages *sizeof (struct page )) >>PAGE_SHIFT ); +/* +* For vmemmap optimization with DAX we need minimum 2 vmemmap +* pages. See layout diagram in Documentation/mm/vmemmap_dedup.rst +*/ +return !altmap && (nr_vmemmap_pages >VMEMMAP_RESERVE_NR ); } +/* +* If we don't have an architecture override, use the generic rule +*/ +#ifndef vmemmap_can_optimize +#define vmemmap_can_optimize __vmemmap_can_optimize +#endif +#else static inline bool vmemmap_can_optimize (struct vmem_altmap *altmap,struct dev_pagemap *pgmap)
mm/mm_init.c +1 -1
@@ -1020,7 +1020,7 @@ static inline unsigned long compound_nr_pages(struct vmem_altmap *altmap,if (!vmemmap_can_optimize (altmap, pgmap))return pgmap_vmemmap_nr (pgmap); -return 2 * (PAGE_SIZE /sizeof (struct page )); +return VMEMMAP_RESERVE_NR * (PAGE_SIZE /sizeof (struct page )); } static void __ref memmap_init_compound(struct page *head,
Signature · 1 line
2.41.0
PATCH v4 4/13 mm/vmemmap: Allow architectures to override how vmemmap optimization works
In reply to Aneesh Kumar K.VArchitectures like powerpc will like to use different page table allocators and mapping mechanisms to implement vmemmap optimization. Similar to vmemmap_populate allow architectures to implement vmemap_populate_compound_pages
--- mm/sparse-vmemmap.c | 3 +++ 1 file changed, 3 insertions(+)
mm/sparse-vmemmap.c +3 -0
@@ -141,6 +141,7 @@ void __meminit vmemmap_verify(pte_t *pte, int node, start, end - 1); } +#ifndef vmemmap_populate_compound_pagespte_t *__meminit vmemmap_pte_populate (pmd_t *pmd,unsigned long addr,int node,struct vmem_altmap *altmap,struct page *reuse)
@@ -446,6 +447,8 @@ static int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, return 0; } +#endif +struct page *__meminit __populate_section_memmap (unsigned long pfn,unsigned long nr_pages,int nid,struct vmem_altmap *altmap,struct dev_pagemap *pgmap)
Signature · 1 line
2.41.0
PATCH v4 5/13 mm: Add __HAVE_ARCH_PUD_SAME similar to __HAVE_ARCH_P4D_SAME
In reply to Aneesh Kumar K.VThis helps architectures to override pmd_same and pud_same independently.
--- include/linux/pgtable.h | 2 ++ 1 file changed, 2 insertions(+)
include/linux/pgtable.h +2 -0
@@ -693,7 +693,9 @@ static inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b) {return pmd_val (pmd_a) ==pmd_val (pmd_b); } +#endif +#ifndef __HAVE_ARCH_PUD_SAMEstatic inline int pud_same (pud_t pud_a,pud_t pud_b) {return pud_val (pud_a) ==pud_val (pud_b);
Signature · 1 line
2.41.0
v4 cover Add support for DAX vmemmap optimization for ppc64
This patch series implements changes required to support DAX vmemmap optimization for ppc64. The vmemmap optimization is only enabled with radix MMU translation and 1GB PUD mapping with 64K page size. The patch series also split hugetlb vmemmap optimization as a separate Kconfig variable so that architectures can enable DAX vmemmap optimization without enabling hugetlb vmemmap optimization. This should enable architectures like arm64 to enable DAX vmemmap optimization while they can't enable hugetlb vmemmap optimization. More details of the same are in patch "mm/vmemmap optimization: Split hugetlb and devdax vmemmap optimization"
Changes from v3:
* Rebase to latest linus tree
* Build fix with SPARSEMEM_VMEMMP disabled
* Add hash_pud_same outisde THP Kconfig
Changes from v2:
* Rebase to latest linus tree
* Address review feedback
Changes from V1:
* Fix make htmldocs warning
* Fix vmemmap allocation bugs with different alignment values.
* Correctly check for section validity to before we free vmemmap area
Aneesh Kumar K.V (13):
mm/hugepage pud: Allow arch-specific helper function to check huge
page pud support
mm: Change pudp_huge_get_and_clear_full take vm_area_struct as arg mm/vmemmap: Improve vmemmap_can_optimize and allow architectures to
override
mm/vmemmap: Allow architectures to override how vmemmap optimization
works
mm: Add __HAVE_ARCH_PUD_SAME similar to __HAVE_ARCH_P4D_SAME mm/huge pud: Use transparent huge pud helpers only with
CONFIG_TRANSPARENT_HUGEPAGE
mm/vmemmap optimization: Split hugetlb and devdax vmemmap optimization powerpc/mm/trace: Convert trace event to trace event class powerpc/book3s64/mm: Enable transparent pud hugepage powerpc/book3s64/vmemmap: Switch radix to use a different vmemmap
handling function
powerpc/book3s64/radix: Add support for vmemmap optimization for radix powerpc/book3s64/radix: Remove mmu_vmemmap_psize powerpc/book3s64/radix: Add debug message to give more details of
vmemmap allocation
Documentation/mm/vmemmap_dedup.rst | 1 + Documentation/powerpc/index.rst | 1 + Documentation/powerpc/vmemmap_dedup.rst | 101 +++ arch/loongarch/Kconfig | 2 +- arch/powerpc/Kconfig | 1 + arch/powerpc/include/asm/book3s/64/hash.h | 9 + arch/powerpc/include/asm/book3s/64/pgtable.h | 155 ++++- arch/powerpc/include/asm/book3s/64/radix.h | 47 ++ .../include/asm/book3s/64/tlbflush-radix.h | 2 + arch/powerpc/include/asm/book3s/64/tlbflush.h | 8 + arch/powerpc/include/asm/pgtable.h | 4 + arch/powerpc/mm/book3s64/hash_pgtable.c | 2 +- arch/powerpc/mm/book3s64/pgtable.c | 78 +++ arch/powerpc/mm/book3s64/radix_pgtable.c | 573 ++++++++++++++++-- arch/powerpc/mm/book3s64/radix_tlb.c | 7 + arch/powerpc/mm/init_64.c | 37 +- arch/powerpc/platforms/Kconfig.cputype | 1 + arch/riscv/Kconfig | 2 +- arch/s390/Kconfig | 2 +- arch/x86/Kconfig | 3 +- drivers/nvdimm/pfn_devs.c | 2 +- fs/Kconfig | 2 +- include/linux/mm.h | 29 +- include/linux/pgtable.h | 11 +- include/trace/events/thp.h | 33 +- mm/Kconfig | 5 +- mm/debug_vm_pgtable.c | 2 +- mm/huge_memory.c | 2 +- mm/mm_init.c | 2 +- mm/mremap.c | 2 +- mm/sparse-vmemmap.c | 3 + 31 files changed, 1047 insertions(+), 82 deletions(-) create mode 100644 Documentation/powerpc/vmemmap_dedup.rst
Signature · 1 line
2.41.0
PATCH v4 6/13 mm/huge pud: Use transparent huge pud helpers only with CONFIG_TRANSPARENT_HUGEPAGE
pudp_set_wrprotect and move_huge_pud helpers are only used when CONFIG_TRANSPARENT_HUGEPAGE is enabled. Similar to pmdp_set_wrprotect and move_huge_pmd_helpers use architecture override only if CONFIG_TRANSPARENT_HUGEPAGE is set
--- include/linux/pgtable.h | 2 ++ mm/mremap.c | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-)
include/linux/pgtable.h +2 -0
@@ -558,6 +558,7 @@ static inline void pmdp_set_wrprotect(struct mm_struct *mm,#endif #ifndef __HAVE_ARCH_PUDP_SET_WRPROTECT #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD +#ifdef CONFIG_TRANSPARENT_HUGEPAGE static inline void pudp_set_wrprotect(struct mm_struct *mm, unsigned long address, pud_t *pudp) {
@@ -571,6 +572,7 @@ static inline void pudp_set_wrprotect(struct mm_struct *mm, {BUILD_BUG (); } +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ #endif
mm/mremap.c +1 -1
@@ -349,7 +349,7 @@ static inline bool move_normal_pud(struct vm_area_struct *vma, }#endif -#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD +#if defined(CONFIG_TRANSPARENT_HUGEPAGE ) && defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD )static bool move_huge_pud (struct vm_area_struct *vma,unsigned long old_addr,unsigned long new_addr,pud_t *old_pud,pud_t *new_pud) {
Signature · 1 line
2.41.0
PATCH v4 7/13 mm/vmemmap optimization: Split hugetlb and devdax vmemmap optimization
In reply to Aneesh Kumar K.VArm disabled hugetlb vmemmap optimization [1] because hugetlb vmemmap optimization includes an update of both the permissions (writeable to read-only) and the output address (pfn) of the vmemmap ptes. That is not supported without unmapping of pte(marking it invalid) by some architectures.
With DAX vmemmap optimization we don't require such pte updates and architectures can enable DAX vmemmap optimization while having hugetlb vmemmap optimization disabled. Hence split DAX optimization support into a different config.
s390, loongarch and riscv don't have devdax support. So the DAX config is not enabled for them. With this change, arm64 should be able to select DAX optimization
[1] commit 060a2c92d1b6 ("arm64: mm: hugetlb: Disable HUGETLB_PAGE_OPTIMIZE_VMEMMAP")
--- arch/loongarch/Kconfig | 2 +- arch/riscv/Kconfig | 2 +- arch/s390/Kconfig | 2 +- arch/x86/Kconfig | 3 ++- fs/Kconfig | 2 +- include/linux/mm.h | 2 +- mm/Kconfig | 5 ++++- 7 files changed, 11 insertions(+), 7 deletions(-)
arch/loongarch/Kconfig +1 -1
@@ -59,7 +59,7 @@ config LOONGARCH select ARCH_USE_QUEUED_SPINLOCKS select ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT select ARCH_WANT_LD_ORPHAN_WARN - select ARCH_WANT_OPTIMIZE_VMEMMAP + select ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP select ARCH_WANTS_NO_INSTR select BUILDTIME_TABLE_SORT select COMMON_CLK
arch/riscv/Kconfig +1 -1
@@ -53,7 +53,7 @@ config RISCV select ARCH_WANT_GENERAL_HUGETLB if !RISCV_ISA_SVNAPOT select ARCH_WANT_HUGE_PMD_SHARE if 64BIT select ARCH_WANT_LD_ORPHAN_WARN if !XIP_KERNEL - select ARCH_WANT_OPTIMIZE_VMEMMAP + select ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP select ARCH_WANTS_THP_SWAP if HAVE_ARCH_TRANSPARENT_HUGEPAGE select BINFMT_FLAT_NO_DATA_START_OFFSET if !MMU select BUILDTIME_TABLE_SORT if MMU
arch/s390/Kconfig +1 -1
@@ -127,7 +127,7 @@ config S390 select ARCH_WANTS_NO_INSTR select ARCH_WANT_DEFAULT_BPF_JIT select ARCH_WANT_IPC_PARSE_VERSION - select ARCH_WANT_OPTIMIZE_VMEMMAP + select ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP select BUILDTIME_TABLE_SORT select CLONE_BACKWARDS2 select DMA_OPS if PCI
arch/x86/Kconfig +2 -1
@@ -128,7 +128,8 @@ config X86 select ARCH_WANT_GENERAL_HUGETLB select ARCH_WANT_HUGE_PMD_SHARE select ARCH_WANT_LD_ORPHAN_WARN - select ARCH_WANT_OPTIMIZE_VMEMMAP if X86_64 + select ARCH_WANT_OPTIMIZE_DAX_VMEMMAP if X86_64 + select ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP if X86_64 select ARCH_WANTS_THP_SWAP if X86_64 select ARCH_HAS_PARANOID_L1D_FLUSH select BUILDTIME_TABLE_SORT
fs/Kconfig +1 -1
@@ -252,7 +252,7 @@ config HUGETLB_PAGE config HUGETLB_PAGE_OPTIMIZE_VMEMMAP def_bool HUGETLB_PAGE - depends on ARCH_WANT_OPTIMIZE_VMEMMAP + depends on ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP depends on SPARSEMEM_VMEMMAP config HUGETLB_PAGE_OPTIMIZE_VMEMMAP_DEFAULT_ON
include/linux/mm.h +1 -1
@@ -3640,7 +3640,7 @@ void vmemmap_free(unsigned long start, unsigned long end,#endif #define VMEMMAP_RESERVE_NR 2 -#ifdef CONFIG_ARCH_WANT_OPTIMIZE_VMEMMAP +#ifdef CONFIG_ARCH_WANT_OPTIMIZE_DAX_VMEMMAP static inline bool __vmemmap_can_optimize (struct vmem_altmap *altmap,struct dev_pagemap *pgmap) {
mm/Kconfig +4 -1
@@ -487,7 +487,10 @@ config SPARSEMEM_VMEMMAP # Select this config option from the architecture Kconfig, if it is preferred # to enable the feature of HugeTLB/dev_dax vmemmap optimization. # -config ARCH_WANT_OPTIMIZE_VMEMMAP +config ARCH_WANT_OPTIMIZE_DAX_VMEMMAP + bool + +config ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP bool config HAVE_MEMBLOCK_PHYS_MAP
Signature · 1 line
2.41.0
PATCH v4 8/13 powerpc/mm/trace: Convert trace event to trace event class
In reply to Aneesh Kumar K.VA follow-up patch will add a pud variant for this same event. Using event class makes that addition simpler.
No functional change in this patch.
--- arch/powerpc/mm/book3s64/hash_pgtable.c | 2 +- arch/powerpc/mm/book3s64/radix_pgtable.c | 2 +- include/trace/events/thp.h | 23 ++++++++++++++++------- 3 files changed, 18 insertions(+), 9 deletions(-)
arch/powerpc/mm/book3s64/hash_pgtable.c +1 -1
@@ -214,7 +214,7 @@ unsigned long hash__pmd_hugepage_update(struct mm_struct *mm, unsigned long addr old =be64_to_cpu (old_be); -trace_hugepage_update (addr, old, clr, set); +trace_hugepage_update_pmd (addr, old, clr, set);if (old &H_PAGE_HASHPTE )hpte_do_hugepage_flush (mm, addr, pmdp, old);return old;
arch/powerpc/mm/book3s64/radix_pgtable.c +1 -1
@@ -962,7 +962,7 @@ unsigned long radix__pmd_hugepage_update(struct mm_struct *mm, unsigned long add#endif old =radix__pte_update (mm, addr,pmdp_ptep (pmdp), clr, set,1 ); -trace_hugepage_update (addr, old, clr, set); +trace_hugepage_update_pmd (addr, old, clr, set);return old; }
include/trace/events/thp.h +16 -7
@@ -8,25 +8,29 @@#include <linux/types.h> #include <linux/tracepoint.h> -TRACE_EVENT (hugepage_set_pmd, +DECLARE_EVENT_CLASS (hugepage_set, - TP_PROTO(unsigned long addr, unsigned long pmd), -TP_ARGS (addr, pmd), + TP_PROTO(unsigned long addr, unsigned long pte), +TP_ARGS (addr, pte),TP_STRUCT__entry ( __field(unsigned long, addr) - __field(unsigned long, pmd) + __field(unsigned long, pte) ),TP_fast_assign ( __entry->addr = addr; - __entry->pmd = pmd; + __entry->pte = pte; ), -TP_printk ("Set pmd with 0x%lx with 0x%lx" , __entry->addr, __entry->pmd) +TP_printk ("Set page table entry with 0x%lx with 0x%lx" , __entry->addr, __entry->pte) ); +DEFINE_EVENT (hugepage_set, hugepage_set_pmd, + TP_PROTO(unsigned long addr, unsigned long pmd), +TP_ARGS (addr, pmd) +); -TRACE_EVENT(hugepage_update, +DECLARE_EVENT_CLASS(hugepage_update, TP_PROTO(unsigned long addr, unsigned long pte, unsigned long clr, unsigned long set), TP_ARGS(addr, pte, clr, set),
@@ -48,6 +52,11 @@ TRACE_EVENT(hugepage_update,TP_printk ("hugepage update at addr 0x%lx and pte = 0x%lx clr = 0x%lx, set = 0x%lx" , __entry->addr, __entry->pte, __entry->clr, __entry->set) ); +DEFINE_EVENT (hugepage_update, hugepage_update_pmd, + TP_PROTO(unsigned long addr, unsigned long pmd, unsigned long clr, unsigned long set), +TP_ARGS (addr, pmd, clr, set) +); + DECLARE_EVENT_CLASS(migration_pmd, TP_PROTO(unsigned long addr, unsigned long pmd),
Signature · 1 line
2.41.0
PATCH v4 9/13 powerpc/book3s64/mm: Enable transparent pud hugepage
In reply to Aneesh Kumar K.VThis is enabled only with radix translation and 1G hugepage size. This will be used with devdax device memory with a namespace alignment of 1G.
Anon transparent hugepage is not supported even though we do have helpers checking pud_trans_huge(). We should never find that return true. The only expected pte bit combination is _PAGE_PTE | _PAGE_DEVMAP.
Some of the helpers are never expected to get called on hash translation and hence is marked to call BUG() in such a case.
--- arch/powerpc/include/asm/book3s/64/hash.h | 9 + arch/powerpc/include/asm/book3s/64/pgtable.h | 155 ++++++++++++++++-- arch/powerpc/include/asm/book3s/64/radix.h | 36 ++++ .../include/asm/book3s/64/tlbflush-radix.h | 2 + arch/powerpc/include/asm/book3s/64/tlbflush.h | 8 + arch/powerpc/mm/book3s64/pgtable.c | 78 +++++++++ arch/powerpc/mm/book3s64/radix_pgtable.c | 28 ++++ arch/powerpc/mm/book3s64/radix_tlb.c | 7 + arch/powerpc/platforms/Kconfig.cputype | 1 + include/trace/events/thp.h | 10 ++ 10 files changed, 323 insertions(+), 11 deletions(-)
arch/powerpc/include/asm/book3s/64/hash.h +9 -0
@@ -138,7 +138,16 @@ static inline int hash__pmd_same(pmd_t pmd_a, pmd_t pmd_b) }#define hash__pmd_bad (pmd) (pmd_val(pmd) & H_PMD_BAD_BITS) + +/* +* pud comparison that will work with both pte and page table pointer. +*/ +static inline int hash__pud_same (pud_t pud_a,pud_t pud_b) +{ +return (((pud_raw (pud_a) ^pud_raw (pud_b)) & ~cpu_to_be64 (_PAGE_HPTEFLAGS)) ==0 ); +}#define hash__pud_bad (pud) (pud_val(pud) & H_PUD_BAD_BITS) +static inline int hash__p4d_bad (p4d_t p4d) {return (p4d_val (p4d) ==0 );
arch/powerpc/include/asm/book3s/64/pgtable.h +144 -11
270 lines not on this page · raw message
arch/powerpc/include/asm/book3s/64/radix.h +36 -0
@@ -250,6 +250,10 @@ static inline int radix__pud_bad(pud_t pud)return !!(pud_val (pud) &RADIX_PUD_BAD_BITS ); } +static inline int radix__pud_same (pud_t pud_a,pud_t pud_b) +{ +return ((pud_raw (pud_a) ^pud_raw (pud_b)) ==0 ); +}static inline int radix__p4d_bad (p4d_t p4d) {
@@ -268,9 +272,22 @@ static inline pmd_t radix__pmd_mkhuge(pmd_t pmd)return __pmd (pmd_val (pmd) | _PAGE_PTE); } +static inline int radix__pud_trans_huge (pud_t pud) +{ +return (pud_val (pud) & (_PAGE_PTE | _PAGE_DEVMAP)) == _PAGE_PTE; +} + +static inline pud_t radix__pud_mkhuge (pud_t pud) +{ +return __pud (pud_val (pud) | _PAGE_PTE); +} +extern unsigned long radix__pmd_hugepage_update (struct mm_struct *mm,unsigned long addr,pmd_t *pmdp,unsigned long clr,unsigned long set); +extern unsigned long radix__pud_hugepage_update (struct mm_struct *mm,unsigned long addr, +pud_t *pudp,unsigned long clr, +unsigned long set);extern pmd_t radix__pmdp_collapse_flush (struct vm_area_struct *vma,unsigned long address,pmd_t *pmdp); extern void radix__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
@@ -278,6 +295,9 @@ extern void radix__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,extern pgtable_t radix__pgtable_trans_huge_withdraw (struct mm_struct *mm,pmd_t *pmdp);extern pmd_t radix__pmdp_huge_get_and_clear (struct mm_struct *mm,unsigned long addr,pmd_t *pmdp); +pud_t radix__pudp_huge_get_and_clear (struct mm_struct *mm, +unsigned long addr,pud_t *pudp); +static inline int radix__has_transparent_hugepage (void ) {/* For radix 2M at PMD level means thp */
@@ -285,6 +305,14 @@ static inline int radix__has_transparent_hugepage(void)return 1 ; return 0; } + +static inline int radix__has_transparent_pud_hugepage (void ) +{ +/* For radix 1G at PUD level means pud hugepage support */ +if (mmu_psize_defs[MMU_PAGE_1G ].shift ==PUD_SHIFT ) +return 1 ; +return 0 ; +}#endif static inline pmd_t radix__pmd_mkdevmap (pmd_t pmd)
@@ -292,9 +320,17 @@ static inline pmd_t radix__pmd_mkdevmap(pmd_t pmd)return __pmd (pmd_val (pmd) | (_PAGE_PTE | _PAGE_DEVMAP)); } +static inline pud_t radix__pud_mkdevmap (pud_t pud) +{ +return __pud (pud_val (pud) | (_PAGE_PTE | _PAGE_DEVMAP)); +} + +struct vmem_altmap ;extern int __meminit radix__vmemmap_create_mapping (unsigned long start,unsigned long page_size,unsigned long phys); +int __meminit radix__vmemmap_populate (unsigned long start,unsigned long end, +int node,struct vmem_altmap *altmap);extern void radix__vmemmap_remove_mapping (unsigned long start,unsigned long page_size);
arch/powerpc/include/asm/book3s/64/tlbflush-radix.h +2 -0
@@ -68,6 +68,8 @@ void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start, unsigned long end, int psize);extern void radix__flush_pmd_tlb_range (struct vm_area_struct *vma,unsigned long start,unsigned long end); +extern void radix__flush_pud_tlb_range (struct vm_area_struct *vma, +unsigned long start,unsigned long end);extern void radix__flush_tlb_range (struct vm_area_struct *vma,unsigned long start,unsigned long end);extern void radix__flush_tlb_kernel_range (unsigned long start,unsigned long end);
arch/powerpc/include/asm/book3s/64/tlbflush.h +8 -0
@@ -50,6 +50,14 @@ static inline void flush_pmd_tlb_range(struct vm_area_struct *vma,radix__flush_pmd_tlb_range (vma, start, end); } +#define __HAVE_ARCH_FLUSH_PUD_TLB_RANGE +static inline void flush_pud_tlb_range (struct vm_area_struct *vma, +unsigned long start,unsigned long end) +{ +if (radix_enabled ()) +radix__flush_pud_tlb_range (vma, start, end); +} +#define __HAVE_ARCH_FLUSH_HUGETLB_TLB_RANGE static inline void flush_hugetlb_tlb_range(struct vm_area_struct *vma, unsigned long start,
arch/powerpc/mm/book3s64/pgtable.c +78 -0
112 lines not on this page · raw message
arch/powerpc/mm/book3s64/radix_pgtable.c +28 -0
@@ -967,6 +967,23 @@ unsigned long radix__pmd_hugepage_update(struct mm_struct *mm, unsigned long addreturn old; } +unsigned long radix__pud_hugepage_update (struct mm_struct *mm,unsigned long addr, +pud_t *pudp,unsigned long clr, +unsigned long set) +{ +unsigned long old; + +#ifdef CONFIG_DEBUG_VM +WARN_ON (!pud_devmap (*pudp)); +assert_spin_locked (pud_lockptr (mm, pudp)); +#endif + + old =radix__pte_update (mm, addr,pudp_ptep (pudp), clr, set,1 ); +trace_hugepage_update_pud (addr, old, clr, set); + +return old; +} +pmd_t radix__pmdp_collapse_flush (struct vm_area_struct *vma,unsigned long address,pmd_t *pmdp)
@@ -1043,6 +1060,17 @@ pmd_t radix__pmdp_huge_get_and_clear(struct mm_struct *mm,return old_pmd; } +pud_t radix__pudp_huge_get_and_clear (struct mm_struct *mm, +unsigned long addr,pud_t *pudp) +{ +pud_t old_pud; +unsigned long old; + + old =radix__pud_hugepage_update (mm, addr, pudp, ~0UL ,0 ); + old_pud =__pud (old); +return old_pud; +} +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ void radix__ptep_set_access_flags(struct vm_area_struct *vma, pte_t *ptep,
arch/powerpc/mm/book3s64/radix_tlb.c +7 -0
@@ -1461,6 +1461,13 @@ void radix__flush_pmd_tlb_range(struct vm_area_struct *vma, }EXPORT_SYMBOL (radix__flush_pmd_tlb_range); +void radix__flush_pud_tlb_range (struct vm_area_struct *vma, +unsigned long start,unsigned long end) +{ +radix__flush_tlb_range_psize (vma->vm_mm, start, end,MMU_PAGE_1G ); +} +EXPORT_SYMBOL (radix__flush_pud_tlb_range); +void radix__flush_tlb_all (void ) {unsigned long rb,prs,r,rs;
arch/powerpc/platforms/Kconfig.cputype +1 -0
@@ -94,6 +94,7 @@ config PPC_BOOK3S_64 select PPC_FPU select PPC_HAVE_PMU_SUPPORT select HAVE_ARCH_TRANSPARENT_HUGEPAGE + select HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD select ARCH_ENABLE_HUGEPAGE_MIGRATION if HUGETLB_PAGE && MIGRATION select ARCH_ENABLE_SPLIT_PMD_PTLOCK select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE
include/trace/events/thp.h +10 -0
@@ -30,6 +30,11 @@ DEFINE_EVENT(hugepage_set, hugepage_set_pmd,TP_ARGS (addr, pmd) ); +DEFINE_EVENT (hugepage_set, hugepage_set_pud, + TP_PROTO(unsigned long addr, unsigned long pud), +TP_ARGS (addr, pud) +); + DECLARE_EVENT_CLASS(hugepage_update, TP_PROTO(unsigned long addr, unsigned long pte, unsigned long clr, unsigned long set),
@@ -57,6 +62,11 @@ DEFINE_EVENT(hugepage_update, hugepage_update_pmd,TP_ARGS (addr, pmd, clr, set) ); +DEFINE_EVENT (hugepage_update, hugepage_update_pud, + TP_PROTO(unsigned long addr, unsigned long pud, unsigned long clr, unsigned long set), +TP_ARGS (addr, pud, clr, set) +); + DECLARE_EVENT_CLASS(migration_pmd, TP_PROTO(unsigned long addr, unsigned long pmd),
Signature · 1 line
2.41.0
PATCH v4 10/13 powerpc/book3s64/vmemmap: Switch radix to use a different vmemmap handling function
In reply to Aneesh Kumar K.VThis is in preparation to update radix to implement vmemmap optimization for devdax. Below are the rules w.r.t radix vmemmap mapping
1. First try to map things using PMD (2M)
2. With altmap if altmap cross-boundary check returns true, fall back to PAGE_SIZE
3. If we can't allocate PMD_SIZE backing memory for vmemmap, fallback to PAGE_SIZE
On removing vmemmap mapping, check if every subsection that is using the vmemmap area is invalid. If found to be invalid, that implies we can safely free the vmemmap area. We don't use the PAGE_UNUSED pattern used by x86 because with 64K page size, we need to do the above check even at the PAGE_SIZE granularity.
--- arch/powerpc/include/asm/book3s/64/radix.h | 2 + arch/powerpc/include/asm/pgtable.h | 4 + arch/powerpc/mm/book3s64/radix_pgtable.c | 326 +++++++++++++++++++-- arch/powerpc/mm/init_64.c | 26 +- 4 files changed, 327 insertions(+), 31 deletions(-)
arch/powerpc/include/asm/book3s/64/radix.h +2 -0
@@ -331,6 +331,8 @@ extern int __meminit radix__vmemmap_create_mapping(unsigned long start, unsigned long phys);int __meminit radix__vmemmap_populate (unsigned long start,unsigned long end,int node,struct vmem_altmap *altmap); +void __ref radix__vmemmap_free (unsigned long start,unsigned long end, +struct vmem_altmap *altmap);extern void radix__vmemmap_remove_mapping (unsigned long start,unsigned long page_size);
arch/powerpc/include/asm/pgtable.h +4 -0
@@ -165,6 +165,10 @@ static inline bool is_ioremap_addr(const void *x)return addr >=IOREMAP_BASE && addr <IOREMAP_END ; } + +int __meminit vmemmap_populated (unsigned long vmemmap_addr,int vmemmap_map_size); +bool altmap_cross_boundary (struct vmem_altmap *altmap,unsigned long start, +unsigned long page_size);#endif /* CONFIG_PPC64 */ #endif /* __ASSEMBLY__ */
arch/powerpc/mm/book3s64/radix_pgtable.c +300 -26
396 lines not on this page · raw message
arch/powerpc/mm/init_64.c +21 -5
@@ -92,7 +92,7 @@ static struct page * __meminit vmemmap_subsection_start(unsigned long vmemmap_ad* a page table lookup here because with the hash translation we don't keep * vmemmap details in linux page table. */ -static int __meminit vmemmap_populated (unsigned long vmemmap_addr,int vmemmap_map_size) +int __meminit vmemmap_populated (unsigned long vmemmap_addr,int vmemmap_map_size) {struct page *start;unsigned long vmemmap_end = vmemmap_addr + vmemmap_map_size;
@@ -183,8 +183,8 @@ static __meminit int vmemmap_list_populate(unsigned long phys,return 0 ; } -static bool altmap_cross_boundary (struct vmem_altmap *altmap,unsigned long start, -unsigned long page_size) +bool altmap_cross_boundary (struct vmem_altmap *altmap,unsigned long start, +unsigned long page_size) {unsigned long nr_pfn = page_size /sizeof (struct page );unsigned long start_pfn =page_to_pfn ((struct page *)start);
@@ -204,6 +204,11 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,bool altmap_alloc;unsigned long page_size =1 << mmu_psize_defs[mmu_vmemmap_psize].shift; +#ifdef CONFIG_PPC_BOOK3S_64 +if (radix_enabled ()) +return radix__vmemmap_populate (start, end, node, altmap); +#endif +/* Align to the page size of the linear mapping. */ start =ALIGN_DOWN (start, page_size);
@@ -303,8 +308,8 @@ static unsigned long vmemmap_list_free(unsigned long start)return vmem_back->phys; } -void __ref vmemmap_free (unsigned long start,unsigned long end, -struct vmem_altmap *altmap) +void __ref __vmemmap_free (unsigned long start,unsigned long end, +struct vmem_altmap *altmap) {unsigned long page_size =1 << mmu_psize_defs[mmu_vmemmap_psize].shift;unsigned long page_order =get_order (page_size);
@@ -362,6 +367,17 @@ void __ref vmemmap_free(unsigned long start, unsigned long end, vmemmap_remove_mapping(start, page_size); } } + +void __ref vmemmap_free (unsigned long start,unsigned long end, +struct vmem_altmap *altmap) +{ +#ifdef CONFIG_PPC_BOOK3S_64 +if (radix_enabled ()) +return radix__vmemmap_free (start, end, altmap); +#endif +return __vmemmap_free (start, end, altmap); +} +#endif void register_page_bootmem_memmap (unsigned long section_nr,struct page *start_page,unsigned long size)
Signature · 1 line
2.41.0
PATCH v4 11/13 powerpc/book3s64/radix: Add support for vmemmap optimization for radix
In reply to Aneesh Kumar K.VWith 2M PMD-level mapping, we require 32 struct pages and a single vmemmap page can contain 1024 struct pages (PAGE_SIZE/sizeof(struct page)). Hence with 64K page size, we don't use vmemmap deduplication for PMD-level mapping.
--- Documentation/mm/vmemmap_dedup.rst | 1 + Documentation/powerpc/index.rst | 1 + Documentation/powerpc/vmemmap_dedup.rst | 101 ++++++++++ arch/powerpc/Kconfig | 1 + arch/powerpc/include/asm/book3s/64/radix.h | 9 + arch/powerpc/mm/book3s64/radix_pgtable.c | 203 +++++++++++++++++++++ 6 files changed, 316 insertions(+) create mode 100644 Documentation/powerpc/vmemmap_dedup.rst
Documentation/mm/vmemmap_dedup.rst +1 -0
@@ -210,6 +210,7 @@ the device (altmap). The following page sizes are supported in DAX: PAGE_SIZE (4K on x86_64), PMD_SIZE (2M on x86_64) and PUD_SIZE (1G on x86_64). +For powerpc equivalent details see Documentation/powerpc/vmemmap_dedup.rst The differences with HugeTLB are relatively minor.
Documentation/powerpc/index.rst +1 -0
@@ -36,6 +36,7 @@ powerpc ultravisor vas-api vcpudispatch_stats + vmemmap_dedup features
Documentation/powerpc/vmemmap_dedup.rst new file +101 -0
101 lines not on this page · raw message
arch/powerpc/Kconfig +1 -0
@@ -174,6 +174,7 @@ config PPC select ARCH_WANT_IPC_PARSE_VERSION select ARCH_WANT_IRQS_OFF_ACTIVATE_MM select ARCH_WANT_LD_ORPHAN_WARN + select ARCH_WANT_OPTIMIZE_DAX_VMEMMAP if PPC_RADIX_MMU select ARCH_WANTS_MODULES_DATA_IN_VMALLOC if PPC_BOOK3S_32 || PPC_8xx select ARCH_WEAK_RELEASE_ACQUIRE select BINFMT_ELF
arch/powerpc/include/asm/book3s/64/radix.h +9 -0
@@ -326,6 +326,7 @@ static inline pud_t radix__pud_mkdevmap(pud_t pud) }struct vmem_altmap ; +struct dev_pagemap ;extern int __meminit radix__vmemmap_create_mapping (unsigned long start,unsigned long page_size,unsigned long phys);
@@ -363,5 +364,13 @@ int radix__remove_section_mapping(unsigned long start, unsigned long end);void radix__kernel_map_pages (struct page *page,int numpages,int enable); +#define vmemmap_can_optimize vmemmap_can_optimize +bool vmemmap_can_optimize (struct vmem_altmap *altmap,struct dev_pagemap *pgmap); + +#define vmemmap_populate_compound_pages vmemmap_populate_compound_pages +int __meminit vmemmap_populate_compound_pages (unsigned long start_pfn, +unsigned long start, +unsigned long end,int node, +struct dev_pagemap *pgmap);#endif /* __ASSEMBLY__ */ #endif
arch/powerpc/mm/book3s64/radix_pgtable.c +203 -0
215 lines not on this page · raw message
Signature · 1 line
2.41.0
PATCH v4 12/13 powerpc/book3s64/radix: Remove mmu_vmemmap_psize
In reply to Aneesh Kumar K.VThis is not used by radix anymore.
--- arch/powerpc/mm/book3s64/radix_pgtable.c | 11 ----------- arch/powerpc/mm/init_64.c | 21 ++++++++++++++------- 2 files changed, 14 insertions(+), 18 deletions(-)
arch/powerpc/mm/book3s64/radix_pgtable.c +0 -11
@@ -601,17 +601,6 @@ void __init radix__early_init_mmu(void)#else mmu_virtual_psize =MMU_PAGE_4K ;#endif - -#ifdef CONFIG_SPARSEMEM_VMEMMAP -/* vmemmap mapping */ -if (mmu_psize_defs[MMU_PAGE_2M ].shift) { -/* -* map vmemmap using 2M if available -*/ - mmu_vmemmap_psize =MMU_PAGE_2M ; - }else - mmu_vmemmap_psize = mmu_virtual_psize; -#endif #endif /* * initialize page table size
arch/powerpc/mm/init_64.c +14 -7
@@ -198,17 +198,12 @@ bool altmap_cross_boundary(struct vmem_altmap *altmap, unsigned long start,return false; } -int __meminit vmemmap_populate (unsigned long start,unsigned long end,int node, -struct vmem_altmap *altmap) +int __meminit __vmemmap_populate (unsigned long start,unsigned long end,int node, +struct vmem_altmap *altmap) {bool altmap_alloc;unsigned long page_size =1 << mmu_psize_defs[mmu_vmemmap_psize].shift; -#ifdef CONFIG_PPC_BOOK3S_64 -if (radix_enabled ()) -return radix__vmemmap_populate (start, end, node, altmap); -#endif -/* Align to the page size of the linear mapping. */ start =ALIGN_DOWN (start, page_size);
@@ -277,6 +272,18 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,return 0 ; } +int __meminit vmemmap_populate (unsigned long start,unsigned long end,int node, +struct vmem_altmap *altmap) +{ + +#ifdef CONFIG_PPC_BOOK3S_64 +if (radix_enabled ()) +return radix__vmemmap_populate (start, end, node, altmap); +#endif + +return __vmemmap_populate (start, end, node, altmap); +} +#ifdef CONFIG_MEMORY_HOTPLUG static unsigned long vmemmap_list_free (unsigned long start) {
Signature · 1 line
2.41.0
PATCH v4 13/13 powerpc/book3s64/radix: Add debug message to give more details of vmemmap allocation
In reply to Aneesh Kumar K.VAdd some extra vmemmap pr_debug message that will indicate the type of vmemmap allocations.
For ex: with DAX vmemmap optimization we can find the below details:
[ 187.166580] radix-mmu: PAGE_SIZE vmemmap mapping [ 187.166587] radix-mmu: PAGE_SIZE vmemmap mapping [ 187.166591] radix-mmu: Tail page reuse vmemmap mapping [ 187.166594] radix-mmu: Tail page reuse vmemmap mapping [ 187.166598] radix-mmu: Tail page reuse vmemmap mapping [ 187.166601] radix-mmu: Tail page reuse vmemmap mapping [ 187.166604] radix-mmu: Tail page reuse vmemmap mapping [ 187.166608] radix-mmu: Tail page reuse vmemmap mapping [ 187.166611] radix-mmu: Tail page reuse vmemmap mapping [ 187.166614] radix-mmu: Tail page reuse vmemmap mapping [ 187.166617] radix-mmu: Tail page reuse vmemmap mapping [ 187.166620] radix-mmu: Tail page reuse vmemmap mapping [ 187.166623] radix-mmu: Tail page reuse vmemmap mapping [ 187.166626] radix-mmu: Tail page reuse vmemmap mapping [ 187.166629] radix-mmu: Tail page reuse vmemmap mapping [ 187.166632] radix-mmu: Tail page reuse vmemmap mapping
And without vmemmap optimization
[ 293.549931] radix-mmu: PMD_SIZE vmemmap mapping [ 293.549984] radix-mmu: PMD_SIZE vmemmap mapping [ 293.550032] radix-mmu: PMD_SIZE vmemmap mapping [ 293.550076] radix-mmu: PMD_SIZE vmemmap mapping [ 293.550117] radix-mmu: PMD_SIZE vmemmap mapping
--- arch/powerpc/mm/book3s64/radix_pgtable.c | 3 +++ 1 file changed, 3 insertions(+)
arch/powerpc/mm/book3s64/radix_pgtable.c +3 -0
@@ -1034,6 +1034,7 @@ static pte_t * __meminit radix__vmemmap_pte_populate(pmd_t *pmdp, unsigned long p =vmemmap_alloc_block_buf (PAGE_SIZE , node,NULL );if (!p)return NULL ; +pr_debug ("PAGE_SIZE vmemmap mapping\n" ); }else {/* * When a PTE/PMD entry is freed from the init_mm
@@ -1046,6 +1047,7 @@ static pte_t * __meminit radix__vmemmap_pte_populate(pmd_t *pmdp, unsigned long*/ get_page (reuse); p =page_to_virt (reuse); +pr_debug ("Tail page reuse vmemmap mapping\n" ); }VM_BUG_ON (!PAGE_ALIGNED (addr));
@@ -1155,6 +1157,7 @@ int __meminit radix__vmemmap_populate(unsigned long start, unsigned long end, in p =vmemmap_alloc_block_buf (PMD_SIZE , node, altmap);if (p) {vmemmap_set_pmd (pmd, p, node, addr, next); +pr_debug ("PMD_SIZE vmemmap mapping\n" );continue ; }else if (altmap) {/*
Signature · 1 line
2.41.0
Re: [PATCH v4 01/13] mm/hugepage pud: Allow arch-specific helper function to check huge page pud support
In reply to Aneesh Kumar K.VLe 10/07/2023 à 18:08, Aneesh Kumar K.V a écrit :
Quoted 5 lines
Architectures like powerpc would like to enable transparent huge page pud support only with radix translation. To support that add
has_transparent_pud_hugepage()helper that architectures can override.Signed-off-by Aneesh Kumar K.V <[email protected]>
Quoted 32 lines
--- drivers/nvdimm/pfn_devs.c | 2 +- include/linux/pgtable.h | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-)drivers/nvdimm/pfn_devs.c +1
-1@@ -100,7 +100,7 @@ static unsigned long *nd_pfn_supported_alignments(unsigned long *alignments)if (has_transparent_hugepage ()) { alignments[1 ] =HPAGE_PMD_SIZE ; -if (IS_ENABLED (CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD )) +if (has_transparent_pud_hugepage ()) alignments[2 ] =HPAGE_PUD_SIZE ; }include/linux/pgtable.h +3
-0@@ -1499,6 +1499,9 @@ typedef unsigned int pgtbl_mod_mask;#define has_transparent_hugepage () IS_BUILTIN(CONFIG_TRANSPARENT_HUGEPAGE)#endif +#ifndef has_transparent_pud_hugepage +#define has_transparent_pud_hugepage () IS_BUILTIN(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD) +#endif /* * On some architectures it depends on the mm if the p4d/pud or pmd * layer of the page table hierarchy is folded or not.
Re: [PATCH v4 02/13] mm: Change pudp_huge_get_and_clear_full take vm_area_struct as arg
In reply to Aneesh Kumar K.VLe 10/07/2023 à 18:08, Aneesh Kumar K.V a écrit :
Quoted 5 lines
We will use this in a later patch to do tlb flush when clearing pud entries on powerpc. This is similar to commit
93a98695f2f9("mm: changepmdp_huge_get_and_clear_fulltakevm_area_structas arg")Signed-off-by Aneesh Kumar K.V <[email protected]>
Quoted 50 lines
--- include/linux/pgtable.h | 4 ++-- mm/debug_vm_pgtable.c | 2 +- mm/huge_memory.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-)include/linux/pgtable.h +2
-2@@ -450,11 +450,11 @@ static inline pmd_t pmdp_huge_get_and_clear_full(struct vm_area_struct *vma,#endif #ifndef __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR_FULL -static inline pud_t pudp_huge_get_and_clear_full (struct mm_struct *mm, +static inline pud_t pudp_huge_get_and_clear_full (struct vm_area_struct *vma,unsigned long address,pud_t *pudp,int full) { -return pudp_huge_get_and_clear (mm, address, pudp); +return pudp_huge_get_and_clear (vma->vm_mm, address, pudp); }#endif #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ mm/debug_vm_pgtable.c +1
-1@@ -385,7 +385,7 @@ static void __init pud_advanced_tests(struct pgtable_debug_args *args)WARN_ON (!(pud_write (pud) &&pud_dirty (pud)));#ifndef __PAGETABLE_PMD_FOLDED -pudp_huge_get_and_clear_full (args->mm, vaddr, args->pudp,1 ); +pudp_huge_get_and_clear_full (args->vma, vaddr, args->pudp,1 ); pud =READ_ONCE (*args->pudp);WARN_ON (!pud_none (pud));#endif /* __PAGETABLE_PMD_FOLDED */ mm/huge_memory.c +1
-1@@ -1981,7 +1981,7 @@ int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma,if (!ptl)return 0 ; -pudp_huge_get_and_clear_full (tlb->mm, addr, pud, tlb->fullmm); +pudp_huge_get_and_clear_full (vma, addr, pud, tlb->fullmm);tlb_remove_pud_tlb_entry (tlb, pud, addr);if (vma_is_special_huge (vma)) {spin_unlock (ptl);
Re: [PATCH v4 03/13] mm/vmemmap: Improve vmemmap_can_optimize and allow architectures to override
In reply to Aneesh Kumar K.VLe 10/07/2023 à 18:08, Aneesh Kumar K.V a écrit :
Quoted 9 lines
dax vmemmap optimization requires a minimum of 2
PAGE_SIZEarea within vmemmap such that tail page mapping can point to the secondPAGE_SIZEarea. Enforce that invmemmap_can_optimize()function.Architectures like powerpc also want to enable vmemmap optimization conditionally (only with radix MMU translation). Hence allow architecture override.
Signed-off-by Aneesh Kumar K.V <[email protected]>
Why renaming vmemmap_can_optimize() to __vmemmap_can_optimize() and keep it when vmemmap_can_optimize() has been override ? Is that because you expect overriding version of vmemmap_can_optimize() to call __vmemmap_can_optimize() ?
Quoted 59 lines
--- include/linux/mm.h | 27 +++++++++++++++++++++++---- mm/mm_init.c | 2 +- 2 files changed, 24 insertions(+), 5 deletions(-)include/linux/mm.h +23
-4@@ -3639,13 +3639,32 @@ void vmemmap_free(unsigned long start, unsigned long end, struct vmem_altmap *altmap);#endif +#define VMEMMAP_RESERVE_NR 2#ifdef CONFIG_ARCH_WANT_OPTIMIZE_VMEMMAP -static inline bool vmemmap_can_optimize (struct vmem_altmap *altmap, -struct dev_pagemap *pgmap) +static inline bool __vmemmap_can_optimize (struct vmem_altmap *altmap, +struct dev_pagemap *pgmap) { -return is_power_of_2 (sizeof (struct page )) && - pgmap && (pgmap_vmemmap_nr (pgmap) >1 ) && !altmap; +unsigned long nr_pages; +unsigned long nr_vmemmap_pages; + +if (!pgmap || !is_power_of_2 (sizeof (struct page ))) +return false; + + nr_pages =pgmap_vmemmap_nr (pgmap); + nr_vmemmap_pages = ((nr_pages *sizeof (struct page )) >>PAGE_SHIFT ); +/* +* For vmemmap optimization with DAX we need minimum 2 vmemmap +* pages. See layout diagram in Documentation/mm/vmemmap_dedup.rst +*/ +return !altmap && (nr_vmemmap_pages >VMEMMAP_RESERVE_NR ); } +/* +* If we don't have an architecture override, use the generic rule +*/ +#ifndef vmemmap_can_optimize +#define vmemmap_can_optimize __vmemmap_can_optimize +#endif +#else static inline bool vmemmap_can_optimize (struct vmem_altmap *altmap,struct dev_pagemap *pgmap)mm/mm_init.c +1
-1@@ -1020,7 +1020,7 @@ static inline unsigned long compound_nr_pages(struct vmem_altmap *altmap,if (!vmemmap_can_optimize (altmap, pgmap))return pgmap_vmemmap_nr (pgmap); -return 2 * (PAGE_SIZE /sizeof (struct page )); +return VMEMMAP_RESERVE_NR * (PAGE_SIZE /sizeof (struct page )); } static void __ref memmap_init_compound(struct page *head,
Re: [PATCH v4 04/13] mm/vmemmap: Allow architectures to override how vmemmap optimization works
In reply to Aneesh Kumar K.VLe 10/07/2023 à 18:08, Aneesh Kumar K.V a écrit :
Quoted 15 lines
Architectures like powerpc will like to use different page table allocators and mapping mechanisms to implement vmemmap optimization. Similar to
vmemmap_populateallow architectures to implementvmemap_populate_compound_pagesSigned-off-by Aneesh Kumar K.V <[email protected]>--- mm/sparse-vmemmap.c | 3 +++ 1 file changed, 3 insertions(+)mm/sparse-vmemmap.c +0
-0
mm/sparse-vmemmap.c void __meminit vmemmap_verify(pte_t *pte, int node, · line 147
Quoted 7 lines
mm/sparse-vmemmap.c +1
-0@@ -141,6 +141,7 @@ void __meminit vmemmap_verify(pte_t *pte, int node, start, end - 1); } +#ifndef vmemmap_populate_compound_pagespte_t *__meminit vmemmap_pte_populate (pmd_t *pmd,unsigned long addr,int node,struct vmem_altmap *altmap,struct page *reuse)
Should vmemmap_pte_populate() be static ?
It looks odd to exclude a non-static function based on a non related macro.
There are several such function in the block being excluded here. Can you explain why it is correct to do that ?
Quoted 9 lines
mm/sparse-vmemmap.c +2
-0@@ -446,6 +447,8 @@ static int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, return 0; } +#endif +struct page *__meminit __populate_section_memmap (unsigned long pfn,unsigned long nr_pages,int nid,struct vmem_altmap *altmap,struct dev_pagemap *pgmap)
Re: [PATCH v4 05/13] mm: Add __HAVE_ARCH_PUD_SAME similar to __HAVE_ARCH_P4D_SAME
In reply to Aneesh Kumar K.VLe 10/07/2023 à 18:08, Aneesh Kumar K.V a écrit :
Quoted 3 lines
This helps architectures to override
pmd_sameandpud_sameindependently.Signed-off-by Aneesh Kumar K.V <[email protected]>
Shouldn't you do it the modern way and use #ifndef pud_same instead of a new __HAVE_ARCH_PUD_SAME like in the old days ?
Quoted 18 lines
--- include/linux/pgtable.h | 2 ++ 1 file changed, 2 insertions(+)include/linux/pgtable.h +2
-0@@ -693,7 +693,9 @@ static inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b) {return pmd_val (pmd_a) ==pmd_val (pmd_b); } +#endif +#ifndef __HAVE_ARCH_PUD_SAMEstatic inline int pud_same (pud_t pud_a,pud_t pud_b) {return pud_val (pud_a) ==pud_val (pud_b);
Re: [PATCH v4 06/13] mm/huge pud: Use transparent huge pud helpers only with CONFIG_TRANSPARENT_HUGEPAGE
In reply to Aneesh Kumar K.VLe 10/07/2023 à 18:08, Aneesh Kumar K.V a écrit :
Quoted 6 lines
pudp_set_wrprotectandmove_huge_pudhelpers are only used whenCONFIG_TRANSPARENT_HUGEPAGEis enabled. Similar topmdp_set_wrprotectandmove_huge_pmd_helpersuse architecture override only ifCONFIG_TRANSPARENT_HUGEPAGEis setSigned-off-by Aneesh Kumar K.V <[email protected]>
Quoted 38 lines
--- include/linux/pgtable.h | 2 ++ mm/mremap.c | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-)include/linux/pgtable.h +2
-0@@ -558,6 +558,7 @@ static inline void pmdp_set_wrprotect(struct mm_struct *mm,#endif #ifndef __HAVE_ARCH_PUDP_SET_WRPROTECT #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD +#ifdef CONFIG_TRANSPARENT_HUGEPAGE static inline void pudp_set_wrprotect(struct mm_struct *mm, unsigned long address, pud_t *pudp) {@@ -571,6 +572,7 @@ static inline void pudp_set_wrprotect(struct mm_struct *mm, {BUILD_BUG (); } +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ #endif mm/mremap.c +1
-1@@ -349,7 +349,7 @@ static inline bool move_normal_pud(struct vm_area_struct *vma, }#endif -#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD +#if defined(CONFIG_TRANSPARENT_HUGEPAGE ) && defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD )static bool move_huge_pud (struct vm_area_struct *vma,unsigned long old_addr,unsigned long new_addr,pud_t *old_pud,pud_t *new_pud) {
Re: [PATCH v4 08/13] powerpc/mm/trace: Convert trace event to trace event class
In reply to Aneesh Kumar K.VLe 10/07/2023 à 18:08, Aneesh Kumar K.V a écrit :
Quoted 6 lines
A follow-up patch will add a pud variant for this same event. Using event class makes that addition simpler.
No functional change in this patch.
Signed-off-by Aneesh Kumar K.V <[email protected]>
Quoted 85 lines
--- arch/powerpc/mm/book3s64/hash_pgtable.c | 2 +- arch/powerpc/mm/book3s64/radix_pgtable.c | 2 +- include/trace/events/thp.h | 23 ++++++++++++++++------- 3 files changed, 18 insertions(+), 9 deletions(-)arch/powerpc/mm/book3s64/hash_pgtable.c +1
-1@@ -214,7 +214,7 @@ unsigned long hash__pmd_hugepage_update(struct mm_struct *mm, unsigned long addr old =be64_to_cpu (old_be); -trace_hugepage_update (addr, old, clr, set); +trace_hugepage_update_pmd (addr, old, clr, set);if (old &H_PAGE_HASHPTE )hpte_do_hugepage_flush (mm, addr, pmdp, old);return old;arch/powerpc/mm/book3s64/radix_pgtable.c +1
-1@@ -962,7 +962,7 @@ unsigned long radix__pmd_hugepage_update(struct mm_struct *mm, unsigned long add#endif old =radix__pte_update (mm, addr,pmdp_ptep (pmdp), clr, set,1 ); -trace_hugepage_update (addr, old, clr, set); +trace_hugepage_update_pmd (addr, old, clr, set);return old; }include/trace/events/thp.h +16
-7@@ -8,25 +8,29 @@#include <linux/types.h> #include <linux/tracepoint.h> -TRACE_EVENT (hugepage_set_pmd, +DECLARE_EVENT_CLASS (hugepage_set, - TP_PROTO(unsigned long addr, unsigned long pmd), -TP_ARGS (addr, pmd), + TP_PROTO(unsigned long addr, unsigned long pte), +TP_ARGS (addr, pte),TP_STRUCT__entry ( __field(unsigned long, addr) - __field(unsigned long, pmd) + __field(unsigned long, pte) ),TP_fast_assign ( __entry->addr = addr; - __entry->pmd = pmd; + __entry->pte = pte; ), -TP_printk ("Set pmd with 0x%lx with 0x%lx" , __entry->addr, __entry->pmd) +TP_printk ("Set page table entry with 0x%lx with 0x%lx" , __entry->addr, __entry->pte) ); +DEFINE_EVENT (hugepage_set, hugepage_set_pmd, + TP_PROTO(unsigned long addr, unsigned long pmd), +TP_ARGS (addr, pmd) +); -TRACE_EVENT(hugepage_update, +DECLARE_EVENT_CLASS(hugepage_update, TP_PROTO(unsigned long addr, unsigned long pte, unsigned long clr, unsigned long set), TP_ARGS(addr, pte, clr, set),@@ -48,6 +52,11 @@ TRACE_EVENT(hugepage_update,TP_printk ("hugepage update at addr 0x%lx and pte = 0x%lx clr = 0x%lx, set = 0x%lx" , __entry->addr, __entry->pte, __entry->clr, __entry->set) ); +DEFINE_EVENT (hugepage_update, hugepage_update_pmd, + TP_PROTO(unsigned long addr, unsigned long pmd, unsigned long clr, unsigned long set), +TP_ARGS (addr, pmd, clr, set) +); + DECLARE_EVENT_CLASS(migration_pmd, TP_PROTO(unsigned long addr, unsigned long pmd),
Re: [PATCH v4 05/13] mm: Add __HAVE_ARCH_PUD_SAME similar to __HAVE_ARCH_P4D_SAME
In reply to Christophe LeroyQuoted 10 lines · Christophe Leroy
Le 10/07/2023 à 18:08, Aneesh Kumar K.V a écrit :
Quoted 3 lines
This helps architectures to override
pmd_sameandpud_sameindependently.Signed-off-by Aneesh Kumar K.V <[email protected]>Reviewed-by Christophe Leroy <[email protected]>Shouldn't you do it the modern way and use #ifndef
pud_sameinstead of a new__HAVE_ARCH_PUD_SAMElike in the old days ?
Sure will update. I was following existing pmd_same override. But I also agree #ifndef pud_same is better.
Quoted 15 lines
Quoted 15 lines
--- include/linux/pgtable.h | 2 ++ 1 file changed, 2 insertions(+)include/linux/pgtable.h +1
-0@@ -693,7 +693,9 @@ static inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b) {return pmd_val (pmd_a) ==pmd_val (pmd_b); } +#endif
Quoted 4 lines
include/linux/pgtable.h static inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b) · line 701
Quoted 4 lines
include/linux/pgtable.h +1
-0+#ifndef __HAVE_ARCH_PUD_SAMEstatic inline int pud_same (pud_t pud_a,pud_t pud_b) {return pud_val (pud_a) ==pud_val (pud_b);
-aneesh
Re: [PATCH v4 03/13] mm/vmemmap: Improve vmemmap_can_optimize and allow architectures to override
In reply to Christophe LeroyQuoted 9 lines · Christophe Leroy
Le 10/07/2023 à 18:08, Aneesh Kumar K.V a écrit :
Quoted 8 lines
dax vmemmap optimization requires a minimum of 2
PAGE_SIZEarea within vmemmap such that tail page mapping can point to the secondPAGE_SIZEarea. Enforce that invmemmap_can_optimize()function.Architectures like powerpc also want to enable vmemmap optimization conditionally (only with radix MMU translation). Hence allow architecture override.
Quoted 9 lines
Quoted 1 line
Signed-off-by Aneesh Kumar K.V <[email protected]>Reviewed-by Christophe Leroy <[email protected]>Why renaming
vmemmap_can_optimize()to__vmemmap_can_optimize()and keep it whenvmemmap_can_optimize()has been override ? Is that because you expect overriding version ofvmemmap_can_optimize()to call__vmemmap_can_optimize()?
Yes, __vmemap_can_optimize will be used in patch 11
https://lore.kernel.org/linuxppc-dev/[email protected]
-aneesh
Re: [PATCH v4 04/13] mm/vmemmap: Allow architectures to override how vmemmap optimization works
In reply to Christophe LeroyQuoted 26 lines · Christophe Leroy
Le 10/07/2023 à 18:08, Aneesh Kumar K.V a écrit :
Quoted 15 lines
Architectures like powerpc will like to use different page table allocators and mapping mechanisms to implement vmemmap optimization. Similar to
vmemmap_populateallow architectures to implementvmemap_populate_compound_pagesSigned-off-by Aneesh Kumar K.V <[email protected]>--- mm/sparse-vmemmap.c | 3 +++ 1 file changed, 3 insertions(+)mm/sparse-vmemmap.c +0
-0mm/sparse-vmemmap.c void __meminit vmemmap_verify(pte_t *pte, int node, · line 147
Quoted 7 lines
mm/sparse-vmemmap.c +1
-0@@ -141,6 +141,7 @@ void __meminit vmemmap_verify(pte_t *pte, int node, start, end - 1); } +#ifndef vmemmap_populate_compound_pagespte_t *__meminit vmemmap_pte_populate (pmd_t *pmd,unsigned long addr,int node,struct vmem_altmap *altmap,struct page *reuse)Should
vmemmap_pte_populate()be static ?
Quoted 5 lines
It looks odd to exclude a non-static function based on a non related macro.
There are several such function in the block being excluded here. Can you explain why it is correct to do that ?
Those functions can actually be made static. But I will do that as a part of different patch. I will update this patch and make sure the
#ifdef will only override the vmemmap_populate_compound_pages.
modified mm/sparse-vmemmap.c
@@ -141,7 +141,6 @@ void __meminit vmemmap_verify(pte_t *pte, int node,
start, end - 1);
}
-#ifndef vmemmap_populate_compound_pages
pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node,
struct vmem_altmap *altmap,
struct page *reuse)
@@ -359,6 +358,7 @@ int __meminit vmemmap_populate_hugepages(unsigned long start, unsigned long end,
return 0;
}
+#ifndef vmemmap_populate_compound_pages
/*
* For compound pages bigger than section size (e.g. x86 1G compound
* pages with 2M subsection size) fill the rest of sections as tail
-aneesh