NestMMU pte upgrade workaround for mprotect and autonuma
Subsystems (11)
cover NestMMU pte upgrade workaround for mprotect and autonuma
We can upgrade pte access (R -> RW transition) via mprotect or autonuma. We need to make sure we follow the recommended pte update sequence as outlined in
commit: bd5050e38aec ("powerpc/mm/radix: Change pte relax sequence to handle nest MMU hang") for such updates. This patch series do that.
-aneesh
Aneesh Kumar K.V (5):
mm: Update ptep_modify_prot_start/commit to take vm_area_struct as arg
mm: update ptep_modify_prot_commit to take old pte value as arg arch/powerpc/mm: Nest MMU workaround for mprotect/autonuma RW upgrade. mm/hugetlb: Add prot_modify_start/commit sequence for hugetlb update arch/powerpc/mm/hugetlb: NestMMU workaround for hugetlb mprotect RW
upgrade
arch/powerpc/include/asm/book3s/64/hugetlb.h | 8 +++++ arch/powerpc/include/asm/book3s/64/pgtable.h | 18 ++++++++++ arch/powerpc/include/asm/hugetlb.h | 2 +- arch/powerpc/mm/hugetlbpage.c | 35 ++++++++++++++++++++ arch/powerpc/mm/pgtable-book3s64.c | 34 +++++++++++++++++++ arch/s390/include/asm/pgtable.h | 5 +-- arch/s390/mm/pgtable.c | 8 +++-- arch/x86/include/asm/paravirt.h | 9 +++-- fs/proc/task_mmu.c | 8 +++-- include/asm-generic/pgtable.h | 10 +++--- include/linux/hugetlb.h | 18 ++++++++++ mm/hugetlb.c | 8 +++-- mm/memory.c | 8 ++--- mm/mprotect.c | 6 ++-- 14 files changed, 150 insertions(+), 27 deletions(-)
Signature · 1 line
2.17.1
RFC PATCH 1/5 mm: Update ptep_modify_prot_start/commit to take vm_area_struct as arg
Some architecture may want to call flush_tlb_range from these helpers.
--- arch/s390/include/asm/pgtable.h | 4 ++-- arch/s390/mm/pgtable.c | 6 ++++-- arch/x86/include/asm/paravirt.h | 7 +++++-- fs/proc/task_mmu.c | 4 ++-- include/asm-generic/pgtable.h | 8 ++++---- mm/memory.c | 4 ++-- mm/mprotect.c | 4 ++-- 7 files changed, 21 insertions(+), 16 deletions(-)
arch/s390/include/asm/pgtable.h +2 -2
@@ -1035,8 +1035,8 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm, }#define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION -pte_t ptep_modify_prot_start (struct mm_struct *,unsigned long ,pte_t *); -void ptep_modify_prot_commit (struct mm_struct *,unsigned long ,pte_t *,pte_t ); +pte_t ptep_modify_prot_start (struct vm_area_struct *,unsigned long ,pte_t *); +void ptep_modify_prot_commit (struct vm_area_struct *,unsigned long ,pte_t *,pte_t );#define __HAVE_ARCH_PTEP_CLEAR_FLUSH static inline pte_t ptep_clear_flush(struct vm_area_struct *vma,
arch/s390/mm/pgtable.c +4 -2
@@ -301,12 +301,13 @@ pte_t ptep_xchg_lazy(struct mm_struct *mm, unsigned long addr, }EXPORT_SYMBOL (ptep_xchg_lazy); -pte_t ptep_modify_prot_start (struct mm_struct *mm,unsigned long addr, +pte_t ptep_modify_prot_start (struct vm_area_struct *vma,unsigned long addr,pte_t *ptep) {pgste_t pgste;pte_t old;int nodat; +struct mm_struct *mm = vma->vm_mm;preempt_disable (); pgste =ptep_xchg_start (mm, addr, ptep);
@@ -320,10 +321,11 @@ pte_t ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr, }EXPORT_SYMBOL (ptep_modify_prot_start); -void ptep_modify_prot_commit (struct mm_struct *mm,unsigned long addr, +void ptep_modify_prot_commit (struct vm_area_struct *vma,unsigned long addr,pte_t *ptep,pte_t pte) {pgste_t pgste; +struct mm_struct *mm = vma->vm_mm;if (!MACHINE_HAS_NX )pte_val (pte) &= ~_PAGE_NOEXEC;
arch/x86/include/asm/paravirt.h +5 -2
@@ -421,10 +421,11 @@ static inline pgdval_t pgd_val(pgd_t pgd) }#define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION -static inline pte_t ptep_modify_prot_start (struct mm_struct *mm,unsigned long addr, +static inline pte_t ptep_modify_prot_start (struct vm_area_struct *vma,unsigned long addr,pte_t *ptep) {pteval_t ret; +struct mm_struct *mm = vma->vm_mm; ret =PVOP_CALL3 (pteval_t, pv_mmu_ops.ptep_modify_prot_start, mm, addr, ptep);
@@ -432,9 +433,11 @@ static inline pte_t ptep_modify_prot_start(struct mm_struct *mm, unsigned long areturn (pte_t ) { .pte = ret }; } -static inline void ptep_modify_prot_commit (struct mm_struct *mm,unsigned long addr, +static inline void ptep_modify_prot_commit (struct vm_area_struct *vma,unsigned long addr,pte_t *ptep,pte_t pte) { +struct mm_struct *mm = vma->vm_mm; +if (sizeof (pteval_t) >sizeof (long ))/* 5 arg words */ pv_mmu_ops.ptep_modify_prot_commit (mm, addr, ptep, pte);
fs/proc/task_mmu.c +2 -2
@@ -938,10 +938,10 @@ static inline void clear_soft_dirty(struct vm_area_struct *vma,pte_t ptent = *pte;if (pte_present (ptent)) { - ptent =ptep_modify_prot_start (vma->vm_mm, addr, pte); + ptent =ptep_modify_prot_start (vma, addr, pte); ptent =pte_wrprotect (ptent); ptent =pte_clear_soft_dirty (ptent); -ptep_modify_prot_commit (vma->vm_mm, addr, pte, ptent); +ptep_modify_prot_commit (vma, addr, pte, ptent); }else if (is_swap_pte (ptent)) { ptent =pte_swp_clear_soft_dirty (ptent);set_pte_at (vma->vm_mm, addr, pte, ptent);
include/asm-generic/pgtable.h +4 -4
@@ -606,22 +606,22 @@ static inline void __ptep_modify_prot_commit(struct mm_struct *mm,* queue the update to be done at some later time. The update must be * actually committed before the pte lock is released, however. */ -static inline pte_t ptep_modify_prot_start (struct mm_struct *mm, +static inline pte_t ptep_modify_prot_start (struct vm_area_struct *vma,unsigned long addr,pte_t *ptep) { -return __ptep_modify_prot_start (mm, addr, ptep); +return __ptep_modify_prot_start (vma->vm_mm, addr, ptep); }/* * Commit an update to a pte, leaving any hardware-controlled bits in * the PTE unmodified. */ -static inline void ptep_modify_prot_commit (struct mm_struct *mm, +static inline void ptep_modify_prot_commit (struct vm_area_struct *vma,unsigned long addr,pte_t *ptep,pte_t pte) { -__ptep_modify_prot_commit (mm, addr, ptep, pte); +__ptep_modify_prot_commit (vma->vm_mm, addr, ptep, pte); }#endif /* __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION */ #endif /* CONFIG_MMU */
mm/memory.c +2 -2
@@ -3806,12 +3806,12 @@ static vm_fault_t do_numa_page(struct vm_fault *vmf)* Make it present again, Depending on how arch implementes non * accessible ptes, some can allow access by kernel mode. */ - pte =ptep_modify_prot_start (vma->vm_mm, vmf->address, vmf->pte); + pte =ptep_modify_prot_start (vma, vmf->address, vmf->pte); pte =pte_modify (pte, vma->vm_page_prot); pte =pte_mkyoung (pte);if (was_writable) pte =pte_mkwrite (pte); -ptep_modify_prot_commit (vma->vm_mm, vmf->address, vmf->pte, pte); +ptep_modify_prot_commit (vma, vmf->address, vmf->pte, pte);update_mmu_cache (vma, vmf->address, vmf->pte); page =vm_normal_page (vma, vmf->address, pte);
mm/mprotect.c +2 -2
@@ -110,7 +110,7 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd,continue ; } - ptent =ptep_modify_prot_start (mm, addr, pte); + ptent =ptep_modify_prot_start (vma, addr, pte); ptent =pte_modify (ptent, newprot);if (preserve_write) ptent =pte_mk_savedwrite (ptent);
@@ -121,7 +121,7 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd, !(vma->vm_flags & VM_SOFTDIRTY))) { ptent =pte_mkwrite (ptent); } -ptep_modify_prot_commit (mm, addr, pte, ptent); +ptep_modify_prot_commit (vma, addr, pte, ptent); pages++; }else if (IS_ENABLED (CONFIG_MIGRATION )) {swp_entry_t entry =pte_to_swp_entry (oldpte);
Signature · 1 line
2.17.1
RFC PATCH 2/5 mm: update ptep_modify_prot_commit to take old pte value as arg
In reply to Aneesh Kumar K.VArchitectures like ppc64 requires to do a conditional tlb flush based on the old and new value of pte. Enable that by passing old pte value as the arg.
--- arch/s390/include/asm/pgtable.h | 3 ++- arch/s390/mm/pgtable.c | 2 +- arch/x86/include/asm/paravirt.h | 2 +- fs/proc/task_mmu.c | 8 +++++--- include/asm-generic/pgtable.h | 2 +- mm/memory.c | 8 ++++---- mm/mprotect.c | 6 +++--- 7 files changed, 17 insertions(+), 14 deletions(-)
arch/s390/include/asm/pgtable.h +2 -1
@@ -1036,7 +1036,8 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm,#define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTIONpte_t ptep_modify_prot_start (struct vm_area_struct *,unsigned long ,pte_t *); -void ptep_modify_prot_commit (struct vm_area_struct *,unsigned long ,pte_t *,pte_t ); +void ptep_modify_prot_commit (struct vm_area_struct *,unsigned long , +pte_t *,pte_t ,pte_t );#define __HAVE_ARCH_PTEP_CLEAR_FLUSH static inline pte_t ptep_clear_flush(struct vm_area_struct *vma,
arch/s390/mm/pgtable.c +1 -1
@@ -322,7 +322,7 @@ pte_t ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr,EXPORT_SYMBOL (ptep_modify_prot_start);void ptep_modify_prot_commit (struct vm_area_struct *vma,unsigned long addr, -pte_t *ptep,pte_t pte) +pte_t *ptep,pte_t old_pte,pte_t pte) {pgste_t pgste;struct mm_struct *mm = vma->vm_mm;
arch/x86/include/asm/paravirt.h +1 -1
@@ -434,7 +434,7 @@ static inline pte_t ptep_modify_prot_start(struct vm_area_struct *vma, unsigned }static inline void ptep_modify_prot_commit (struct vm_area_struct *vma,unsigned long addr, -pte_t *ptep,pte_t pte) +pte_t *ptep,pte_t old_pte,pte_t pte) {struct mm_struct *mm = vma->vm_mm;
fs/proc/task_mmu.c +5 -3
@@ -938,10 +938,12 @@ static inline void clear_soft_dirty(struct vm_area_struct *vma,pte_t ptent = *pte;if (pte_present (ptent)) { - ptent =ptep_modify_prot_start (vma, addr, pte); - ptent =pte_wrprotect (ptent); +pte_t old_pte; + + old_pte =ptep_modify_prot_start (vma, addr, pte); + ptent =pte_wrprotect (old_pte); ptent =pte_clear_soft_dirty (ptent); -ptep_modify_prot_commit (vma, addr, pte, ptent); +ptep_modify_prot_commit (vma, addr, pte, old_pte, ptent); }else if (is_swap_pte (ptent)) { ptent =pte_swp_clear_soft_dirty (ptent);set_pte_at (vma->vm_mm, addr, pte, ptent);
include/asm-generic/pgtable.h +1 -1
@@ -619,7 +619,7 @@ static inline pte_t ptep_modify_prot_start(struct vm_area_struct *vma,*/ static inline void ptep_modify_prot_commit (struct vm_area_struct *vma,unsigned long addr, -pte_t *ptep,pte_t pte) +pte_t *ptep,pte_t old_pte,pte_t pte) {__ptep_modify_prot_commit (vma->vm_mm, addr, ptep, pte); }
mm/memory.c +4 -4
@@ -3786,7 +3786,7 @@ static vm_fault_t do_numa_page(struct vm_fault *vmf)int last_cpupid;int target_nid;bool migrated = false; -pte_t pte; +pte_t pte, old_pte;bool was_writable =pte_savedwrite (vmf->orig_pte);int flags =0 ;
@@ -3806,12 +3806,12 @@ static vm_fault_t do_numa_page(struct vm_fault *vmf)* Make it present again, Depending on how arch implementes non * accessible ptes, some can allow access by kernel mode. */ - pte =ptep_modify_prot_start (vma, vmf->address, vmf->pte); - pte =pte_modify (pte, vma->vm_page_prot); + old_pte =ptep_modify_prot_start (vma, vmf->address, vmf->pte); + pte =pte_modify (old_pte, vma->vm_page_prot); pte =pte_mkyoung (pte);if (was_writable) pte =pte_mkwrite (pte); -ptep_modify_prot_commit (vma, vmf->address, vmf->pte, pte); +ptep_modify_prot_commit (vma, vmf->address, vmf->pte, old_pte, pte);update_mmu_cache (vma, vmf->address, vmf->pte); page =vm_normal_page (vma, vmf->address, pte);
mm/mprotect.c +3 -3
@@ -110,8 +110,8 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd,continue ; } - ptent =ptep_modify_prot_start (vma, addr, pte); - ptent =pte_modify (ptent, newprot); + oldpte =ptep_modify_prot_start (vma, addr, pte); + ptent =pte_modify (oldpte, newprot);if (preserve_write) ptent =pte_mk_savedwrite (ptent);
@@ -121,7 +121,7 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd, !(vma->vm_flags & VM_SOFTDIRTY))) { ptent =pte_mkwrite (ptent); } -ptep_modify_prot_commit (vma, addr, pte, ptent); +ptep_modify_prot_commit (vma, addr, pte, oldpte, ptent); pages++; }else if (IS_ENABLED (CONFIG_MIGRATION )) {swp_entry_t entry =pte_to_swp_entry (oldpte);
Signature · 1 line
2.17.1
RFC PATCH 3/5 arch/powerpc/mm: Nest MMU workaround for mprotect/autonuma RW upgrade.
In reply to Aneesh Kumar K.VNestMMU requires us to mark the pte invalid and flush the tlb when we do a RW upgrade of pte. We fixed a variant of this in the fault path in commit
Do the same for mprotect and autonuma upgrades.
Hugetlb is handled in the next patch.
--- arch/powerpc/include/asm/book3s/64/pgtable.h | 18 +++++++++++ arch/powerpc/mm/pgtable-book3s64.c | 34 ++++++++++++++++++++ 2 files changed, 52 insertions(+)
arch/powerpc/include/asm/book3s/64/pgtable.h +18 -0
@@ -1291,6 +1291,24 @@ static inline const int pud_pfn(pud_t pud)BUILD_BUG ();return 0 ; } +#define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION +pte_t ptep_modify_prot_start (struct vm_area_struct *,unsigned long ,pte_t *); +void ptep_modify_prot_commit (struct vm_area_struct *,unsigned long , +pte_t *,pte_t ,pte_t ); + +/* +* Returns true for a Read or Write upgrade of pte. +*/ +static inline bool is_pte_upgrade (unsigned long old_val,unsigned long new_val) +{ +if ((!(old_val & _PAGE_READ)) && (new_val & _PAGE_READ)) +return true; + +if ((!(old_val & _PAGE_WRITE)) && (new_val & _PAGE_WRITE)) +return true; + +return false; +}#endif /* __ASSEMBLY__ */ #endif /* _ASM_POWERPC_BOOK3S_64_PGTABLE_H_ */
arch/powerpc/mm/pgtable-book3s64.c +34 -0
@@ -477,3 +477,37 @@ void arch_report_meminfo(struct seq_file *m) atomic_long_read(&direct_pages_count[MMU_PAGE_1G]) << 20); }#endif /* CONFIG_PROC_FS */ + +pte_t ptep_modify_prot_start (struct vm_area_struct *vma,unsigned long addr, +pte_t *ptep) +{ +unsigned long pte_val; + +/* +* Clear the _PAGE_PRESENT so that no hardware parallel update is +* possible. Also keep the pte_present true so that we don't take +* wrong fault. +*/ + pte_val =pte_update (vma->vm_mm, addr, ptep, _PAGE_PRESENT, _PAGE_INVALID,0 ); + +return __pte (pte_val); + +} +EXPORT_SYMBOL (ptep_modify_prot_start); + +void ptep_modify_prot_commit (struct vm_area_struct *vma,unsigned long addr, +pte_t *ptep,pte_t old_pte,pte_t pte) +{ +struct mm_struct *mm = vma->vm_mm; + +/* +* To avoid NMMU hang while relaxing access we need to flush the tlb before +* we set the new value. +*/ +if (is_pte_upgrade (pte_val (old_pte),pte_val (pte)) && + (atomic_read (&mm->context.copros) >0 )) +flush_tlb_page (vma, addr); + +set_pte_at (mm, addr, ptep, pte); +} +EXPORT_SYMBOL (ptep_modify_prot_commit);
Signature · 1 line
2.17.1
RFC PATCH 4/5 mm/hugetlb: Add prot_modify_start/commit sequence for hugetlb update
In reply to Aneesh Kumar K.V--- include/linux/hugetlb.h | 18 ++++++++++++++++++ mm/hugetlb.c | 8 +++++--- 2 files changed, 23 insertions(+), 3 deletions(-)
include/linux/hugetlb.h +18 -0
@@ -529,6 +529,24 @@ static inline void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr set_huge_pte_at(mm, addr, ptep, pte); }#endif + +#ifndef huge_ptep_modify_prot_start +static inline pte_t huge_ptep_modify_prot_start (struct vm_area_struct *vma, +unsigned long addr,pte_t *ptep) +{ +return huge_ptep_get_and_clear (mm, address, ptep); +} +#endif + +#ifndef huge_ptep_modify_prot_commit +static inline void huge_ptep_modify_prot_commit (struct vm_area_struct *vma, +unsigned long addr,pte_t *ptep, +pte_t old_pte,pte_t pte) +{ +set_huge_pte_at (vma->vm_mm, addr, ptep, pte); +} +#endif +#else /* CONFIG_HUGETLB_PAGE */ struct hstate {};#define alloc_huge_page (v, a, r) NULL
mm/hugetlb.c +5 -3
@@ -4336,10 +4336,12 @@ unsigned long hugetlb_change_protection(struct vm_area_struct *vma,continue ; }if (!huge_pte_none (pte)) { - pte =huge_ptep_get_and_clear (mm, address, ptep); - pte =pte_mkhuge (huge_pte_modify (pte, newprot)); +pte_t old_pte; + + old_pte =huge_ptep_modify_prot_start (vma, address, ptep); + pte =pte_mkhuge (huge_pte_modify (old_pte, newprot)); pte =arch_make_huge_pte (pte, vma,NULL ,0 ); -set_huge_pte_at (mm, address, ptep, pte); +huge_ptep_modify_prot_commit (vma, address, ptep, old_pte, pte); pages++; }spin_unlock (ptl);
Signature · 1 line
2.17.1
RFC PATCH 5/5 arch/powerpc/mm/hugetlb: NestMMU workaround for hugetlb mprotect RW upgrade
In reply to Aneesh Kumar K.VNestMMU requires us to mark the pte invalid and flush the tlb when we do a RW upgrade of pte. We fixed a variant of this in the fault path in commit
--- arch/powerpc/include/asm/book3s/64/hugetlb.h | 8 +++++ arch/powerpc/include/asm/hugetlb.h | 2 +- arch/powerpc/mm/hugetlbpage.c | 35 ++++++++++++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-)
arch/powerpc/include/asm/book3s/64/hugetlb.h +8 -0
@@ -39,4 +39,12 @@ static inline bool gigantic_page_supported(void) }#endif +#define huge_ptep_modify_prot_start huge_ptep_modify_prot_start +extern pte_t huge_ptep_modify_prot_start (struct vm_area_struct *vma, +unsigned long addr,pte_t *ptep); + +#define huge_ptep_modify_prot_commit huge_ptep_modify_prot_commit +extern void huge_ptep_modify_prot_commit (struct vm_area_struct *vma, +unsigned long addr,pte_t *ptep, +pte_t old_pte,pte_t new_pte);#endif
arch/powerpc/include/asm/hugetlb.h +1 -1
@@ -4,7 +4,6 @@#ifdef CONFIG_HUGETLB_PAGE #include <asm/page.h> -#include <asm-generic/hugetlb.h> extern struct kmem_cache *hugepte_cache;
@@ -176,6 +175,7 @@ static inline void arch_clear_hugepage_flags(struct page *page) { } +#include <asm-generic/hugetlb.h> #else /* ! CONFIG_HUGETLB_PAGE */ static inline void flush_hugetlb_page (struct vm_area_struct *vma,unsigned long vmaddr)
arch/powerpc/mm/hugetlbpage.c +35 -0
@@ -906,3 +906,38 @@ int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr,return 1 ; } + +#ifdef CONFIG_PPC_BOOK3S_64 +pte_t huge_ptep_modify_prot_start (struct vm_area_struct *vma, +unsigned long addr,pte_t *ptep) +{ +unsigned long pte_val; +/* +* Clear the _PAGE_PRESENT so that no hardware parallel update is +* possible. Also keep the pte_present true so that we don't take +* wrong fault. +*/ + pte_val =pte_update (vma->vm_mm, addr, ptep, + _PAGE_PRESENT, _PAGE_INVALID,1 ); + +return __pte (pte_val); +} +EXPORT_SYMBOL (huge_ptep_modify_prot_start); + +void huge_ptep_modify_prot_commit (struct vm_area_struct *vma,unsigned long addr, +pte_t *ptep,pte_t old_pte,pte_t pte) +{ +struct mm_struct *mm = vma->vm_mm; + +/* +* To avoid NMMU hang while relaxing access we need to flush the tlb before +* we set the new value. +*/ +if (is_pte_upgrade (pte_val (old_pte),pte_val (pte)) && + (atomic_read (&mm->context.copros) >0 )) +flush_hugetlb_page (vma, addr); + +set_huge_pte_at (vma->vm_mm, addr, ptep, pte); +} +EXPORT_SYMBOL (huge_ptep_modify_prot_commit); +#endif
Signature · 1 line
2.17.1