mm/powerpc/book3s64/radix: Remove unused code.
PATCH 1/3 mm/powerpc/book3s64/radix: Remove unused code.
mm_tlb_flush_nested change was added in the mmu gather tlb flush to handle the case of parallel pte invalidate happening with mmap_sem held in read mode. This fix was done by commit: 02390f66bd23 ("powerpc/64s/radix: Fix MADV_[FREE|DONTNEED] TLB flush miss problem with THP") and the problem is explained in detail in commit: 99baac21e458 ("mm: fix MADV_[FREE|DONTNEED] TLB flush miss problem")
This was later updated by commit: 7a30df49f63a ("mm: mmu_gather: remove __tlb_reset_range() for force flush") to do a full mm flush rather than a range flush. By commit: dd2283f2605e ("mm: mmap: zap pages with read mmap_sem in munmap") we are also now allowing a page table free in mmap_sem read mode which means we should do a PWC flush too. Our current full mm flush imply a PWC flush.
With all the above change the mm_tlb_flush_nested(mm) branch in radix__tlb_flush will never be taken because for the nested case we would have taken the if (tlb->fullmm) branch. This patch removes the unused code. Also, remove the gflush change in __radix__flush_tlb_range that was added to handle the range tlb flush code. We only check for THP there because hugetlb is flushed via a different code path where page size is explicitly specified
This is a partial revert of commit: 02390f66bd23 ("powerpc/64s/radix: Fix MADV_[FREE|DONTNEED] TLB flush miss problem with THP")
--- arch/powerpc/mm/book3s64/radix_tlb.c | 66 +++------------------------- 1 file changed, 6 insertions(+), 60 deletions(-)
arch/powerpc/mm/book3s64/radix_tlb.c +6 -60
@@ -832,8 +832,7 @@ static unsigned long tlb_single_page_flush_ceiling __read_mostly = 33;static unsigned long tlb_local_single_page_flush_ceiling__read_mostly =POWER9_TLB_SETS_RADIX *2 ;static inline void __radix__flush_tlb_range (struct mm_struct *mm, -unsigned long start,unsigned long end, -bool flush_all_sizes) +unsigned long start,unsigned long end) {unsigned long pid;
@@ -879,26 +878,16 @@ static inline void __radix__flush_tlb_range(struct mm_struct *mm, } } }else { -bool hflush = flush_all_sizes; -bool gflush = flush_all_sizes; +bool hflush = false;unsigned long hstart, hend; -unsigned long gstart, gend; -if (IS_ENABLED (CONFIG_TRANSPARENT_HUGEPAGE )) - hflush = true; - -if (hflush) { +if (IS_ENABLED (CONFIG_TRANSPARENT_HUGEPAGE )) { hstart = (start +PMD_SIZE -1 ) &PMD_MASK ; hend = end &PMD_MASK ;if (hstart == hend) hflush = false; - } - -if (gflush) { - gstart = (start +PUD_SIZE -1 ) &PUD_MASK ; - gend = end &PUD_MASK ; -if (gstart == gend) - gflush = false; +else + hflush = true; }if (local) {
@@ -907,9 +896,6 @@ static inline void __radix__flush_tlb_range(struct mm_struct *mm,if (hflush)__tlbiel_va_range (hstart, hend, pid,PMD_SIZE ,MMU_PAGE_2M ); -if (gflush) -__tlbiel_va_range (gstart, gend, pid, -PUD_SIZE ,MMU_PAGE_1G ); asmvolatile ("ptesync" : : :"memory" ); }else if (cputlb_use_tlbie ()) { asmvolatile ("ptesync" : : :"memory" );
@@ -917,10 +903,6 @@ static inline void __radix__flush_tlb_range(struct mm_struct *mm,if (hflush)__tlbie_va_range (hstart, hend, pid,PMD_SIZE ,MMU_PAGE_2M ); -if (gflush) -__tlbie_va_range (gstart, gend, pid, -PUD_SIZE ,MMU_PAGE_1G ); - asmvolatile ("eieio; tlbsync; ptesync" : : :"memory" ); }else { _tlbiel_va_range_multicast(mm,
@@ -928,9 +910,6 @@ static inline void __radix__flush_tlb_range(struct mm_struct *mm,if (hflush)_tlbiel_va_range_multicast (mm, hstart, hend, pid,PMD_SIZE ,MMU_PAGE_2M , false); -if (gflush) -_tlbiel_va_range_multicast (mm, - gstart, gend, pid,PUD_SIZE ,MMU_PAGE_1G , false); } }preempt_enable ();
@@ -945,7 +924,7 @@ void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,return radix__flush_hugetlb_tlb_range (vma, start, end);#endif -__radix__flush_tlb_range (vma->vm_mm, start, end, false); +__radix__flush_tlb_range (vma->vm_mm, start, end); }EXPORT_SYMBOL (radix__flush_tlb_range);
@@ -1023,39 +1002,6 @@ void radix__tlb_flush(struct mmu_gather *tlb)*/ if (tlb->fullmm) {__flush_all_mm (mm, true); -#if defined(CONFIG_TRANSPARENT_HUGEPAGE ) || defined(CONFIG_HUGETLB_PAGE ) - } else if (mm_tlb_flush_nested(mm)) { -/* -* If there is a concurrent invalidation that is clearing ptes, -* then it's possible this invalidation will miss one of those -* cleared ptes and miss flushing the TLB. If this invalidate -* returns before the other one flushes TLBs, that can result -* in it returning while there are still valid TLBs inside the -* range to be invalidated. -* -* See mm/memory.c:tlb_finish_mmu() for more details. -* -* The solution to this is ensure the entire range is always -* flushed here. The problem for powerpc is that the flushes -* are page size specific, so this "forced flush" would not -* do the right thing if there are a mix of page sizes in -* the range to be invalidated. So use __flush_tlb_range -* which invalidates all possible page sizes in the range. -* -* PWC flush probably is not be required because the core code -* shouldn't free page tables in this path, but accounting -* for the possibility makes us a bit more robust. -* -* need_flush_all is an uncommon case because page table -* teardown should be done with exclusive locks held (but -* after locks are dropped another invalidate could come -* in), it could be optimized further if necessary. -*/ -if (!tlb->need_flush_all) -__radix__flush_tlb_range (mm, start, end, true); -else -radix__flush_all_mm (mm); -#endif }else if ( (psize =radix_get_mmu_psize (page_size)) ==-1 ) {if (!tlb->need_flush_all)radix__flush_tlb_mm (mm);
Signature · 1 line
2.21.0
PATCH 2/3 powerpc/mm/book3s64/radix: Use freed_tables instead of need_flush_all
With commit: 22a61c3c4f13 ("asm-generic/tlb: Track freeing of page-table directories in struct mmu_gather") we now track whether we freed page table in mmu_gather. Use that to decide whether to flush Page Walk Cache.
--- arch/powerpc/include/asm/book3s/64/pgalloc.h | 15 --------------- arch/powerpc/include/asm/book3s/64/tlbflush.h | 16 ---------------- arch/powerpc/mm/book3s64/radix_tlb.c | 11 +++-------- 3 files changed, 3 insertions(+), 39 deletions(-)
arch/powerpc/include/asm/book3s/64/pgalloc.h +0 -15
@@ -122,11 +122,6 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)static inline void __pud_free_tlb (struct mmu_gather *tlb,pud_t *pud,unsigned long address) { -/* -* By now all the pud entries should be none entries. So go -* ahead and flush the page walk cache -*/ -flush_tlb_pgtable (tlb, address);pgtable_free_tlb (tlb, pud,PUD_INDEX ); }
@@ -143,11 +138,6 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)static inline void __pmd_free_tlb (struct mmu_gather *tlb,pmd_t *pmd,unsigned long address) { -/* -* By now all the pud entries should be none entries. So go -* ahead and flush the page walk cache -*/ -flush_tlb_pgtable (tlb, address);return pgtable_free_tlb (tlb, pmd,PMD_INDEX ); }
@@ -166,11 +156,6 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd,static inline void __pte_free_tlb (struct mmu_gather *tlb,pgtable_t table,unsigned long address) { -/* -* By now all the pud entries should be none entries. So go -* ahead and flush the page walk cache -*/ -flush_tlb_pgtable (tlb, address);pgtable_free_tlb (tlb, table,PTE_INDEX ); }
arch/powerpc/include/asm/book3s/64/tlbflush.h +0 -16
@@ -147,22 +147,6 @@ static inline void flush_tlb_fix_spurious_fault(struct vm_area_struct *vma,flush_tlb_page (vma, address); } -/* -* flush the page walk cache for the address -*/ -static inline void flush_tlb_pgtable (struct mmu_gather *tlb,unsigned long address) -{ -/* -* Flush the page table walk cache on freeing a page table. We already -* have marked the upper/higher level page table entry none by now. -* So it is safe to flush PWC here. -*/ -if (!radix_enabled ()) -return ; - -radix__flush_tlb_pwc (tlb, address); -} -extern bool tlbie_capable;extern bool tlbie_enabled;
arch/powerpc/mm/book3s64/radix_tlb.c +3 -8
@@ -732,18 +732,13 @@ static void __flush_all_mm(struct mm_struct *mm, bool fullmm) }preempt_enable (); } +void radix__flush_all_mm (struct mm_struct *mm) {__flush_all_mm (mm, false); }EXPORT_SYMBOL (radix__flush_all_mm); -void radix__flush_tlb_pwc (struct mmu_gather *tlb,unsigned long addr) -{ - tlb->need_flush_all =1 ; -} -EXPORT_SYMBOL (radix__flush_tlb_pwc); -void radix__flush_tlb_page_psize (struct mm_struct *mm,unsigned long vmaddr,int psize) {
@@ -1003,12 +998,12 @@ void radix__tlb_flush(struct mmu_gather *tlb)if (tlb->fullmm) {__flush_all_mm (mm, true); }else if ( (psize =radix_get_mmu_psize (page_size)) ==-1 ) { -if (!tlb->need_flush_all) +if (!tlb->freed_tables)radix__flush_tlb_mm (mm);else radix__flush_all_mm (mm); }else { -if (!tlb->need_flush_all) +if (!tlb->freed_tables)radix__flush_tlb_range_psize (mm, start, end, psize);else radix__flush_tlb_pwc_range_psize (mm, start, end, psize);
Signature · 1 line
2.21.0
PATCH 3/3 powerpc/mm/book3s64/radix: Flush the full mm even when need_flush_all is set
In reply to Aneesh Kumar K.VWith the previous patch, we should now not be using need_flush_all for powerpc. But then make sure we force a PID tlbie flush with RIC=2 if we ever find need_flush_all set. Also don't reset it after a mmu gather flush
--- arch/powerpc/mm/book3s64/radix_tlb.c | 3 +-- include/asm-generic/tlb.h | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-)
arch/powerpc/mm/book3s64/radix_tlb.c +1 -2
@@ -995,7 +995,7 @@ void radix__tlb_flush(struct mmu_gather *tlb)* that flushes the process table entry cache upon process teardown. * See the comment for radix in arch_exit_mmap(). */ -if (tlb->fullmm) { +if (tlb->fullmm || tlb->need_flush_all) {__flush_all_mm (mm, true); }else if ( (psize =radix_get_mmu_psize (page_size)) ==-1 ) {if (!tlb->freed_tables)
@@ -1008,7 +1008,6 @@ void radix__tlb_flush(struct mmu_gather *tlb)else radix__flush_tlb_pwc_range_psize (mm, start, end, psize); } - tlb->need_flush_all =0 ; } static __always_inline void __radix__flush_tlb_range_psize(struct mm_struct *mm,
include/asm-generic/tlb.h +1 -1
@@ -428,7 +428,7 @@ static inline void tlb_change_page_size(struct mmu_gather *tlb, {#ifdef CONFIG_HAVE_MMU_GATHER_PAGE_SIZE if (tlb->page_size && tlb->page_size != page_size) { -if (!tlb->fullmm) +if (!tlb->fullmm && !tlb->need_flush_all)tlb_flush_mmu (tlb); }
Signature · 1 line
2.21.0
Quoted 21 lines · Aneesh Kumar K.V
mm_tlb_flush_nestedchange was added in the mmu gather tlb flush to handle the case of parallel pte invalidate happening withmmap_semheld in read mode. This fix was done by commit:02390f66bd23("powerpc/64s/radix: Fix MADV_[FREE|DONTNEED] TLB flush miss problem with THP") and the problem is explained in detail in commit:99baac21e458("mm: fix MADV_[FREE|DONTNEED] TLB flush miss problem")This was later updated by commit:
7a30df49f63a("mm:mmu_gather: remove__tlb_reset_range()for force flush") to do a full mm flush rather than a range flush. By commit:dd2283f2605e("mm: mmap: zap pages with readmmap_semin munmap") we are also now allowing a page table free inmmap_semread mode which means we should do a PWC flush too. Our current full mm flush imply a PWC flush.With all the above change the
mm_tlb_flush_nested(mm)branch inradix__tlb_flushwill never be taken because for the nested case we would have taken the if (tlb->fullmm) branch. This patch removes the unused code. Also, remove the gflush change in__radix__flush_tlb_rangethat was added to handle the range tlb flush code. We only check for THP there because hugetlb is flushed via a different code path where page size is explicitly specified
Quoted 4 lines
This is a partial revert of commit:
02390f66bd23("powerpc/64s/radix: Fix MADV_[FREE|DONTNEED] TLB flush miss problem with THP")Signed-off-by Aneesh Kumar K.V <[email protected]>
Series applied to powerpc next, thanks.
https://git.kernel.org/powerpc/c/a42d6ba8c5be5aa597d25dbc15e336a2eca40260
cheers