mm: move vma_is_anonymous check within pmd_move_must_withdraw
Subsystems (5)
PATCH v3 1/2 mm: move vma_is_anonymous check within pmd_move_must_withdraw
Architectures like ppc64 want to use page table deposit/withraw even with huge pmd dax entries. Allow arch to override the vma_is_anonymous check by moving that to pmd_move_must_withdraw function
--- arch/powerpc/include/asm/book3s/64/pgtable.h | 3 ++- include/asm-generic/pgtable.h | 12 ------------ mm/huge_memory.c | 18 ++++++++++++++++-- 3 files changed, 18 insertions(+), 15 deletions(-)
arch/powerpc/include/asm/book3s/64/pgtable.h +2 -1
@@ -1009,7 +1009,8 @@ static inline void pmdp_huge_split_prepare(struct vm_area_struct *vma,#define pmd_move_must_withdraw pmd_move_must_withdrawstruct spinlock ;static inline int pmd_move_must_withdraw (struct spinlock *new_pmd_ptl, -struct spinlock *old_pmd_ptl) +struct spinlock *old_pmd_ptl, +struct vm_area_struct *vma) {if (radix_enabled ())return false;
include/asm-generic/pgtable.h +0 -12
@@ -653,18 +653,6 @@ static inline pmd_t pmd_read_atomic(pmd_t *pmdp) }#endif -#ifndef pmd_move_must_withdraw -static inline int pmd_move_must_withdraw (spinlock_t *new_pmd_ptl, -spinlock_t *old_pmd_ptl) -{ -/* -* With split pmd lock we also need to move preallocated -* PTE page table if new_pmd is on different PMD page table. -*/ -return new_pmd_ptl != old_pmd_ptl; -} -#endif -/* * This function is meant to be used by sites walking pagetables with * the mmap_sem hold in read mode to protect against MADV_DONTNEED and
mm/huge_memory.c +16 -2
@@ -1424,6 +1424,21 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,return 1 ; } +#ifndef pmd_move_must_withdraw +static inline int pmd_move_must_withdraw (spinlock_t *new_pmd_ptl, +spinlock_t *old_pmd_ptl, +struct vm_area_struct *vma) +{ +/* +* With split pmd lock we also need to move preallocated +* PTE page table if new_pmd is on different PMD page table. +* +* We also don't deposit and withdraw tables for file pages. +*/ +return (new_pmd_ptl != old_pmd_ptl) &&vma_is_anonymous (vma); +} +#endif +bool move_huge_pmd (struct vm_area_struct *vma,unsigned long old_addr,unsigned long new_addr,unsigned long old_end,pmd_t *old_pmd,pmd_t *new_pmd)
@@ -1458,8 +1473,7 @@ bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr, pmd =pmdp_huge_get_and_clear (mm, old_addr, old_pmd);VM_BUG_ON (!pmd_none (*new_pmd)); -if (pmd_move_must_withdraw (new_ptl, old_ptl) && -vma_is_anonymous (vma)) { +if (pmd_move_must_withdraw (new_ptl, old_ptl, vma)) {pgtable_t pgtable; pgtable =pgtable_trans_huge_withdraw (mm, old_pmd);pgtable_trans_huge_deposit (mm, new_pmd, pgtable);
Signature · 7 lines
2.10.2 -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH v3 2/2 mm: THP page cache support for ppc64
Add arch specific callback in the generic THP page cache code that will deposit and withdarw preallocated page table. Archs like ppc64 use this preallocated table to store the hash pte slot information.
Testing:
kernel build of the patch series on tmpfs mounted with option huge=always
The related thp stat:thp_fault_alloc 72939thp_fault_fallback 60547thp_collapse_alloc 603thp_collapse_alloc_failed 0thp_file_alloc 253763thp_file_mapped 4251thp_split_page 51518thp_split_page_failed 1thp_deferred_split_page 73566thp_split_pmd 665thp_zero_page_alloc 3thp_zero_page_alloc_failed 0
---
Changes from V2:
* Handle page table allocation failures.
arch/powerpc/include/asm/book3s/64/pgtable.h | 10 +++++ include/asm-generic/pgtable.h | 3 ++ mm/Kconfig | 6 +-- mm/huge_memory.c | 17 ++++++++ mm/khugepaged.c | 21 +++++++++- mm/memory.c | 60 +++++++++++++++++++++++----- 6 files changed, 100 insertions(+), 17 deletions(-)
arch/powerpc/include/asm/book3s/64/pgtable.h +10 -0
@@ -1021,6 +1021,16 @@ static inline int pmd_move_must_withdraw(struct spinlock *new_pmd_ptl,*/ return true; } + + +#define arch_needs_pgtable_deposit arch_needs_pgtable_deposit +static inline bool arch_needs_pgtable_deposit (void ) +{ +if (radix_enabled ()) +return false; +return true; +} +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ #endif /* __ASSEMBLY__ */ #endif /* _ASM_POWERPC_BOOK3S_64_PGTABLE_H_ */
include/asm-generic/pgtable.h +3 -0
@@ -653,6 +653,9 @@ static inline pmd_t pmd_read_atomic(pmd_t *pmdp) }#endif +#ifndef arch_needs_pgtable_deposit +#define arch_needs_pgtable_deposit () (false) +#endif /* * This function is meant to be used by sites walking pagetables with * the mmap_sem hold in read mode to protect against MADV_DONTNEED and
mm/Kconfig +1 -5
@@ -447,13 +447,9 @@ choice benefit. endchoice -# -# We don't deposit page tables on file THP mapping, -# but Power makes use of them to address MMU quirk. -# config TRANSPARENT_HUGE_PAGECACHE def_bool y - depends on TRANSPARENT_HUGEPAGE && !PPC + depends on TRANSPARENT_HUGEPAGE # # UP and nommu archs use km based percpu allocator
mm/huge_memory.c +17 -0
@@ -1377,6 +1377,15 @@ bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,return ret; } +static inline void zap_deposited_table (struct mm_struct *mm,pmd_t *pmd) +{ +pgtable_t pgtable; + + pgtable =pgtable_trans_huge_withdraw (mm, pmd); +pte_free (mm, pgtable); +atomic_long_dec (&mm->nr_ptes); +} +int zap_huge_pmd (struct mmu_gather *tlb,struct vm_area_struct *vma,pmd_t *pmd,unsigned long addr) {
@@ -1416,6 +1425,8 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,atomic_long_dec (&tlb->mm->nr_ptes);add_mm_counter (tlb->mm,MM_ANONPAGES , -HPAGE_PMD_NR ); }else { +if (arch_needs_pgtable_deposit ()) +zap_deposited_table (tlb->mm, pmd);add_mm_counter (tlb->mm,MM_FILEPAGES , -HPAGE_PMD_NR ); }spin_unlock (ptl);
@@ -1595,6 +1606,12 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,if (!vma_is_anonymous (vma)) { _pmd =pmdp_huge_clear_flush_notify (vma, haddr, pmd); +/* +* We are going to unmap this huge page. So +* just go ahead and zap it +*/ +if (arch_needs_pgtable_deposit ()) +zap_deposited_table (mm, pmd);if (vma_is_dax (vma))return ; page =pmd_page (_pmd);
mm/khugepaged.c +19 -2
@@ -1240,6 +1240,7 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff)struct vm_area_struct *vma;unsigned long addr;pmd_t *pmd, _pmd; +bool deposited = false;i_mmap_lock_write (mapping); vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {
@@ -1264,10 +1265,26 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff)spinlock_t *ptl =pmd_lock (vma->vm_mm, pmd);/* assume page table is clear */ _pmd =pmdp_collapse_flush (vma, addr, pmd); +/* +* now deposit the pgtable for arch that need it +* otherwise free it. +*/ +if (arch_needs_pgtable_deposit ()) { +/* +* The deposit should be visibile only after +* collapse is seen by others. +*/ +smp_wmb (); +pgtable_trans_huge_deposit (vma->vm_mm, pmd, +pmd_pgtable (_pmd)); + deposited = true; + }spin_unlock (ptl);up_write (&vma->vm_mm->mmap_sem); -atomic_long_dec (&vma->vm_mm->nr_ptes); -pte_free (vma->vm_mm,pmd_pgtable (_pmd)); +if (!deposited) { +atomic_long_dec (&vma->vm_mm->nr_ptes); +pte_free (vma->vm_mm,pmd_pgtable (_pmd)); + } } }i_mmap_unlock_write (mapping);
mm/memory.c +50 -10
111 lines not on this page · raw message
Signature · 7 lines
2.10.2 -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH V3 2/2] mm: THP page cache support for ppc64
Quoted 21 lines · Aneesh Kumar K.V
Add arch specific callback in the generic THP page cache code that will deposit and withdarw preallocated page table. Archs like ppc64 use this preallocated table to store the hash pte slot information.
Testing:
kernel build of the patch series on tmpfs mounted with option huge=alwaysThe related thp stat:
thp_fault_alloc72939thp_fault_fallback60547thp_collapse_alloc603thp_collapse_alloc_failed0thp_file_alloc253763thp_file_mapped4251thp_split_page51518thp_split_page_failed1thp_deferred_split_page73566thp_split_pmd665thp_zero_page_alloc3thp_zero_page_alloc_failed0
Quoted 1 line
Signed-off-by Aneesh Kumar K.V <[email protected]>
One nit-pick below, but otherwise
mm/memory.c static int do_set_pmd(struct fault_env *fe, struct page *page) · line 3011
Quoted 9 lines
mm/memory.c +5
-0@@ -2975,6 +3004,13 @@ static int do_set_pmd(struct fault_env *fe, struct page *page) ret =0 ;count_vm_event (THP_FILE_MAPPED ); out: +/* +* If we are going to fallback to pte mapping, do a +* withdraw with pmd lock held. +*/ +if (arch_needs_pgtable_deposit () && (ret ==VM_FAULT_FALLBACK ))
Parenthesis are redundant around ret check.
Quoted 5 lines
mm/memory.c +2
-0+ fe->prealloc_pte =pgtable_trans_huge_withdraw (vma->vm_mm, + fe->pmd);spin_unlock (fe->ptl);return ret; }
Signature · 7 lines
Kirill A. Shutemov -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 5 lines · Aneesh Kumar K.V
Architectures like ppc64 want to use page table deposit/withraw even with huge pmd dax entries. Allow arch to override the
vma_is_anonymouscheck by moving that topmd_move_must_withdrawfunction
I think the changelog can be reworded a bit
Independent of whether the vma is for anonymous memory, some arches like ppc64 would like to override pmd_move_must_withdraw(). One option is to encapsulate the vma_is_anonymous() check for general architectures inside pmd_move_must_withdraw() so that is always called and architectures that need unconditional overriding can override this function. ppc64 needs to override the function when the MMU is configured to use hash PTE's.
What do you think?
Balbir Singh.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 5 lines · Aneesh Kumar K.V
Architectures like ppc64 want to use page table deposit/withraw even with huge pmd dax entries. Allow arch to override the
vma_is_anonymouscheck by moving that topmd_move_must_withdrawfunction
Quoted 7 lines
Signed-off-by Aneesh Kumar K.V <[email protected]>--- arch/powerpc/include/asm/book3s/64/pgtable.h | 3 ++- include/asm-generic/pgtable.h | 12 ------------ mm/huge_memory.c | 18 ++++++++++++++++-- 3 files changed, 18 insertions(+), 15 deletions(-)
cheers