mm/powerpc/book3s64/radix: Remove unused code.
RFC PATCH 1/3 mm/powerpc/book3s64/radix: Remove unused code.
mm_tlb_flush_nested change was added in the mmu gather tlb flush to handle the case of parallel pte invalidate happening with mmap_sem held in read mode. This fix was done by commit: 02390f66bd23 ("powerpc/64s/radix: Fix MADV_[FREE|DONTNEED] TLB flush miss problem with THP") and the problem is explained in detail in commit: 99baac21e458 ("mm: fix MADV_[FREE|DONTNEED] TLB flush miss problem")
This was later updated by commit: 7a30df49f63a ("mm: mmu_gather: remove __tlb_reset_range() for force flush") to do a full mm flush rather than a range flush. By commit: dd2283f2605e ("mm: mmap: zap pages with read mmap_sem in munmap") we are also now allowing a page table free in mmap_sem read mode which means we should do a PWC flush too. Our current full mm flush imply a PWC flush.
With all the above change the mm_tlb_flush_nested(mm) branch in radix__tlb_flush will never be taken because for the nested case we would have taken the if (tlb->fullmm) branch. This patch removes the unused code. Also, remove the gflush change in __radix__flush_tlb_range that was added to handle the range tlb flush code. We only check for THP there because hugetlb is flushed via a different code path where page size is explicitly specified
This is a partial revert of commit: 02390f66bd23 ("powerpc/64s/radix: Fix MADV_[FREE|DONTNEED] TLB flush miss problem with THP")
--- arch/powerpc/mm/book3s64/radix_tlb.c | 62 +++------------------------- 1 file changed, 6 insertions(+), 56 deletions(-)
arch/powerpc/mm/book3s64/radix_tlb.c +6 -56
@@ -692,8 +692,7 @@ static unsigned long tlb_single_page_flush_ceiling __read_mostly = 33;static unsigned long tlb_local_single_page_flush_ceiling__read_mostly =POWER9_TLB_SETS_RADIX *2 ;static inline void __radix__flush_tlb_range (struct mm_struct *mm, -unsigned long start,unsigned long end, -bool flush_all_sizes) +unsigned long start,unsigned long end) {unsigned long pid;
@@ -735,26 +734,16 @@ static inline void __radix__flush_tlb_range(struct mm_struct *mm,_tlbie_pid (pid,RIC_FLUSH_TLB ); } }else { -bool hflush = flush_all_sizes; -bool gflush = flush_all_sizes; +bool hflush = false;unsigned long hstart, hend; -unsigned long gstart, gend; -if (IS_ENABLED (CONFIG_TRANSPARENT_HUGEPAGE )) - hflush = true; - -if (hflush) { +if (IS_ENABLED (CONFIG_TRANSPARENT_HUGEPAGE )) { hstart = (start +PMD_SIZE -1 ) &PMD_MASK ; hend = end &PMD_MASK ;if (hstart == hend) hflush = false; - } - -if (gflush) { - gstart = (start +PUD_SIZE -1 ) &PUD_MASK ; - gend = end &PUD_MASK ; -if (gstart == gend) - gflush = false; +else + hflush = true; } asmvolatile ("ptesync" : : :"memory" );
@@ -763,18 +752,12 @@ static inline void __radix__flush_tlb_range(struct mm_struct *mm,if (hflush)__tlbiel_va_range (hstart, hend, pid,PMD_SIZE ,MMU_PAGE_2M ); -if (gflush) -__tlbiel_va_range (gstart, gend, pid, -PUD_SIZE ,MMU_PAGE_1G ); asmvolatile ("ptesync" : : :"memory" ); }else {__tlbie_va_range (start, end, pid, page_size, mmu_virtual_psize);if (hflush)__tlbie_va_range (hstart, hend, pid,PMD_SIZE ,MMU_PAGE_2M ); -if (gflush) -__tlbie_va_range (gstart, gend, pid, -PUD_SIZE ,MMU_PAGE_1G );fixup_tlbie (); asmvolatile ("eieio; tlbsync; ptesync" : : :"memory" ); }
@@ -791,7 +774,7 @@ void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,return radix__flush_hugetlb_tlb_range (vma, start, end);#endif -__radix__flush_tlb_range (vma->vm_mm, start, end, false); +__radix__flush_tlb_range (vma->vm_mm, start, end); }EXPORT_SYMBOL (radix__flush_tlb_range);
@@ -882,39 +865,6 @@ void radix__tlb_flush(struct mmu_gather *tlb)*/ if (tlb->fullmm) {__flush_all_mm (mm, true); -#if defined(CONFIG_TRANSPARENT_HUGEPAGE ) || defined(CONFIG_HUGETLB_PAGE ) - } else if (mm_tlb_flush_nested(mm)) { -/* -* If there is a concurrent invalidation that is clearing ptes, -* then it's possible this invalidation will miss one of those -* cleared ptes and miss flushing the TLB. If this invalidate -* returns before the other one flushes TLBs, that can result -* in it returning while there are still valid TLBs inside the -* range to be invalidated. -* -* See mm/memory.c:tlb_finish_mmu() for more details. -* -* The solution to this is ensure the entire range is always -* flushed here. The problem for powerpc is that the flushes -* are page size specific, so this "forced flush" would not -* do the right thing if there are a mix of page sizes in -* the range to be invalidated. So use __flush_tlb_range -* which invalidates all possible page sizes in the range. -* -* PWC flush probably is not be required because the core code -* shouldn't free page tables in this path, but accounting -* for the possibility makes us a bit more robust. -* -* need_flush_all is an uncommon case because page table -* teardown should be done with exclusive locks held (but -* after locks are dropped another invalidate could come -* in), it could be optimized further if necessary. -*/ -if (!tlb->need_flush_all) -__radix__flush_tlb_range (mm, start, end, true); -else -radix__flush_all_mm (mm); -#endif }else if ( (psize =radix_get_mmu_psize (page_size)) ==-1 ) {if (!tlb->need_flush_all)radix__flush_tlb_mm (mm);
Signature · 1 line
2.21.0
RFC PATCH 2/3 powerpc/mm/book3s64/radix: Use freed_tables instead of need_flush_all
With commit: 22a61c3c4f13 ("asm-generic/tlb: Track freeing of page-table directories in struct mmu_gather") we now track whether we freed page table in mmu_gather. Use that to decide whether to flush Page Walk Cache.
--- arch/powerpc/include/asm/book3s/64/pgalloc.h | 15 --------------- arch/powerpc/include/asm/book3s/64/tlbflush.h | 15 --------------- arch/powerpc/mm/book3s64/radix_tlb.c | 11 +++-------- 3 files changed, 3 insertions(+), 38 deletions(-)
arch/powerpc/include/asm/book3s/64/pgalloc.h +0 -15
@@ -122,11 +122,6 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)static inline void __pud_free_tlb (struct mmu_gather *tlb,pud_t *pud,unsigned long address) { -/* -* By now all the pud entries should be none entries. So go -* ahead and flush the page walk cache -*/ -flush_tlb_pgtable (tlb, address);pgtable_free_tlb (tlb, pud,PUD_INDEX ); }
@@ -143,11 +138,6 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)static inline void __pmd_free_tlb (struct mmu_gather *tlb,pmd_t *pmd,unsigned long address) { -/* -* By now all the pud entries should be none entries. So go -* ahead and flush the page walk cache -*/ -flush_tlb_pgtable (tlb, address);return pgtable_free_tlb (tlb, pmd,PMD_INDEX ); }
@@ -166,11 +156,6 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd,static inline void __pte_free_tlb (struct mmu_gather *tlb,pgtable_t table,unsigned long address) { -/* -* By now all the pud entries should be none entries. So go -* ahead and flush the page walk cache -*/ -flush_tlb_pgtable (tlb, address);pgtable_free_tlb (tlb, table,PTE_INDEX ); }
arch/powerpc/include/asm/book3s/64/tlbflush.h +0 -15
@@ -147,19 +147,4 @@ static inline void flush_tlb_fix_spurious_fault(struct vm_area_struct *vma, flush_tlb_page(vma, address); } -/* -* flush the page walk cache for the address -*/ -static inline void flush_tlb_pgtable (struct mmu_gather *tlb,unsigned long address) -{ -/* -* Flush the page table walk cache on freeing a page table. We already -* have marked the upper/higher level page table entry none by now. -* So it is safe to flush PWC here. -*/ -if (!radix_enabled ()) -return ; - -radix__flush_tlb_pwc (tlb, address); -}#endif /* _ASM_POWERPC_BOOK3S_64_TLBFLUSH_H */
arch/powerpc/mm/book3s64/radix_tlb.c +3 -8
@@ -616,18 +616,13 @@ static void __flush_all_mm(struct mm_struct *mm, bool fullmm) }preempt_enable (); } +void radix__flush_all_mm (struct mm_struct *mm) {__flush_all_mm (mm, false); }EXPORT_SYMBOL (radix__flush_all_mm); -void radix__flush_tlb_pwc (struct mmu_gather *tlb,unsigned long addr) -{ - tlb->need_flush_all =1 ; -} -EXPORT_SYMBOL (radix__flush_tlb_pwc); -void radix__flush_tlb_page_psize (struct mm_struct *mm,unsigned long vmaddr,int psize) {
@@ -866,12 +861,12 @@ void radix__tlb_flush(struct mmu_gather *tlb)if (tlb->fullmm) {__flush_all_mm (mm, true); }else if ( (psize =radix_get_mmu_psize (page_size)) ==-1 ) { -if (!tlb->need_flush_all) +if (!tlb->freed_tables)radix__flush_tlb_mm (mm);else radix__flush_all_mm (mm); }else { -if (!tlb->need_flush_all) +if (!tlb->freed_tables)radix__flush_tlb_range_psize (mm, start, end, psize);else radix__flush_tlb_pwc_range_psize (mm, start, end, psize);
Signature · 1 line
2.21.0
RFC PATCH 3/3 powerpc/mm/book3s64/radix: Flush the full mm even when need_flush_all is set
In reply to Aneesh Kumar K.VWith the previous patch, we should now not be using need_flush_all for powerpc. But then make sure we force a PID tlbie flush with RIC=2 if we ever find need_flush_all set. Also don't reset it after a mmu gather flush
--- arch/powerpc/mm/book3s64/radix_tlb.c | 3 +-- include/asm-generic/tlb.h | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-)
arch/powerpc/mm/book3s64/radix_tlb.c +1 -2
@@ -858,7 +858,7 @@ void radix__tlb_flush(struct mmu_gather *tlb)* that flushes the process table entry cache upon process teardown. * See the comment for radix in arch_exit_mmap(). */ -if (tlb->fullmm) { +if (tlb->fullmm || tlb->need_flush_all) {__flush_all_mm (mm, true); }else if ( (psize =radix_get_mmu_psize (page_size)) ==-1 ) {if (!tlb->freed_tables)
@@ -871,7 +871,6 @@ void radix__tlb_flush(struct mmu_gather *tlb)else radix__flush_tlb_pwc_range_psize (mm, start, end, psize); } - tlb->need_flush_all =0 ; } static __always_inline void __radix__flush_tlb_range_psize(struct mm_struct *mm,
include/asm-generic/tlb.h +1 -1
@@ -428,7 +428,7 @@ static inline void tlb_change_page_size(struct mmu_gather *tlb, {#ifdef CONFIG_HAVE_MMU_GATHER_PAGE_SIZE if (tlb->page_size && tlb->page_size != page_size) { -if (!tlb->fullmm) +if (!tlb->fullmm && !tlb->need_flush_all)tlb_flush_mmu (tlb); }
Signature · 1 line
2.21.0