tlb: Fix (soft-)dirty bit management clean up API
Subsystems (13)
Will Deacon
maintainerv2 cover tlb: Fix (soft-)dirty bit management clean up API
Hi all,
This is version two of the patches I previously posted here:
https://lore.kernel.org/r/[email protected]
Since then, the arm64 fixes have been merged upstream, so this series is now mainly just cleanup, with the exception of the first patch which no longer appears to be theoretical after a bug report came up independently:
https://lore.kernel.org/linux-mm/CA+32v5zzFYJQ7eHfJP-2OHeR+6p5PZsX=RDJNU6vGF3hLO+j-g@mail.gmail.com/
(albeit in a test case, but I think we should restore the old behaviour)
Changes since v1 include:
* Remove the MMU gather code from the soft-dirty path altogether, making it consistent with the mprotect() path in terms of handling the TLB invalidation explicitly.
* Remove start/end arguments from tlb_gather_mmu()
* Improve commit messages and fix typos
Cheers,
Will
--->8
Will Deacon (6):
mm: proc: Invalidate TLB after clearing soft-dirty page state
tlb: mmu_gather: Remove unused start/end arguments from
tlb_finish_mmu()
tlb: mmu_gather: Introduce tlb_gather_mmu_fullmm()
tlb: mmu_gather: Remove start/end arguments from tlb_gather_mmu()
tlb: arch: Remove empty __tlb_remove_tlb_entry() stubs x86/ldt: Use tlb_gather_mmu_fullmm() when freeing LDT page-tables
arch/ia64/include/asm/tlb.h | 4 ++-- arch/sparc/include/asm/tlb_64.h | 1 - arch/x86/include/asm/tlb.h | 1 - arch/x86/kernel/ldt.c | 10 ++++++++-- fs/exec.c | 4 ++-- fs/proc/task_mmu.c | 9 +++++---- include/asm-generic/tlb.h | 6 ++++-- include/linux/mm_types.h | 7 +++---- mm/hugetlb.c | 18 ++---------------- mm/madvise.c | 12 ++++++------ mm/memory.c | 8 ++++---- mm/mmap.c | 8 ++++---- mm/mmu_gather.c | 31 +++++++++++++++++-------------- mm/oom_kill.c | 6 +++--- 14 files changed, 60 insertions(+), 65 deletions(-)
Signature · 1 line
2.29.2.576.ga3fc446d84-goog
Will Deacon
maintainerPATCH v2 1/6 mm: proc: Invalidate TLB after clearing soft-dirty page state
Since commit 0758cd830494 ("asm-generic/tlb: avoid potential double flush"), TLB invalidation is elided in tlb_finish_mmu() if no entries were batched via the tlb_remove_*() functions. Consequently, the page-table modifications performed by clear_refs_write() in response to a write to /proc/<pid>/clear_refs do not perform TLB invalidation. Although this is fine when simply aging the ptes, in the case of clearing the "soft-dirty" state we can end up with entries where pte_write() is false, yet a writable mapping remains in the TLB.
Fix this by avoiding the mmu_gather API altogether: managing both the 'tlb_flush_pending' flag on the 'mm_struct' and explicit TLB invalidation for the sort-dirty path, much like mprotect() does already.
--- fs/proc/task_mmu.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-)
fs/proc/task_mmu.c +5 -4
@@ -1189,7 +1189,6 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf,struct mm_struct *mm;struct vm_area_struct *vma;enum clear_refs_types type; -struct mmu_gather tlb;int itype;int rv;
@@ -1234,7 +1233,6 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, count = -EINTR ; goto out_mm; } -tlb_gather_mmu (&tlb, mm,0 ,-1 );if (type ==CLEAR_REFS_SOFT_DIRTY ) {for (vma = mm->mmap; vma; vma = vma->vm_next) {if (!(vma->vm_flags &VM_SOFTDIRTY ))
@@ -1252,15 +1250,18 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf,break ; } +inc_tlb_flush_pending (mm);mmu_notifier_range_init (&range,MMU_NOTIFY_SOFT_DIRTY ,0 ,NULL , mm,0 ,-1UL );mmu_notifier_invalidate_range_start (&range); }walk_page_range (mm,0 , mm->highest_vm_end, &clear_refs_walk_ops, &cp); -if (type ==CLEAR_REFS_SOFT_DIRTY ) +if (type ==CLEAR_REFS_SOFT_DIRTY ) {mmu_notifier_invalidate_range_end (&range); -tlb_finish_mmu (&tlb,0 ,-1 ); +flush_tlb_mm (mm); +dec_tlb_flush_pending (mm); + }mmap_read_unlock (mm); out_mm:mmput (mm);
Signature · 1 line
2.29.2.576.ga3fc446d84-goog
Will Deacon
maintainerPATCH v2 2/6 tlb: mmu_gather: Remove unused start/end arguments from tlb_finish_mmu()
In reply to Will DeaconSince commit 7a30df49f63a ("mm: mmu_gather: remove __tlb_reset_range() for force flush"), the 'start' and 'end' arguments to tlb_finish_mmu() are no longer used, since we flush the whole mm in case of a nested invalidation.
Remove the unused arguments and update all callers.
--- arch/ia64/include/asm/tlb.h | 2 +- arch/x86/kernel/ldt.c | 2 +- fs/exec.c | 2 +- include/linux/mm_types.h | 3 +-- mm/hugetlb.c | 2 +- mm/madvise.c | 6 +++--- mm/memory.c | 4 ++-- mm/mmap.c | 4 ++-- mm/mmu_gather.c | 5 +---- mm/oom_kill.c | 4 ++-- 10 files changed, 15 insertions(+), 19 deletions(-)
arch/ia64/include/asm/tlb.h +1 -1
@@ -36,7 +36,7 @@* tlb_end_vma(tlb, vma); * } * } -* tlb_finish_mmu(tlb, start, end); // finish unmap for address space MM +* tlb_finish_mmu(tlb); // finish unmap for address space MM */ #include <linux/mm.h> #include <linux/pagemap.h>
arch/x86/kernel/ldt.c +1 -1
@@ -400,7 +400,7 @@ static void free_ldt_pgtables(struct mm_struct *mm)tlb_gather_mmu (&tlb, mm, start, end);free_pgd_range (&tlb, start, end, start, end); -tlb_finish_mmu (&tlb, start, end); +tlb_finish_mmu (&tlb);#endif }
fs/exec.c +1 -1
@@ -724,7 +724,7 @@ static int shift_arg_pages(struct vm_area_struct *vma, unsigned long shift)free_pgd_range (&tlb, old_start, old_end, new_end, vma->vm_next ? vma->vm_next->vm_start :USER_PGTABLES_CEILING ); } -tlb_finish_mmu (&tlb, old_start, old_end); +tlb_finish_mmu (&tlb);/* * Shrink the vma to just the new range. Always succeeds.
include/linux/mm_types.h +1 -2
@@ -585,8 +585,7 @@ static inline cpumask_t *mm_cpumask(struct mm_struct *mm)struct mmu_gather ;extern void tlb_gather_mmu (struct mmu_gather *tlb,struct mm_struct *mm,unsigned long start,unsigned long end); -extern void tlb_finish_mmu (struct mmu_gather *tlb, -unsigned long start,unsigned long end); +extern void tlb_finish_mmu (struct mmu_gather *tlb);static inline void init_tlb_flush_pending (struct mm_struct *mm) {
mm/hugetlb.c +1 -1
@@ -3985,7 +3985,7 @@ void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,tlb_gather_mmu (&tlb, mm, tlb_start, tlb_end);__unmap_hugepage_range (&tlb, vma, start, end, ref_page); -tlb_finish_mmu (&tlb, tlb_start, tlb_end); +tlb_finish_mmu (&tlb); }/*
mm/madvise.c +3 -3
@@ -508,7 +508,7 @@ static long madvise_cold(struct vm_area_struct *vma,lru_add_drain ();tlb_gather_mmu (&tlb, mm, start_addr, end_addr);madvise_cold_page_range (&tlb, vma, start_addr, end_addr); -tlb_finish_mmu (&tlb, start_addr, end_addr); +tlb_finish_mmu (&tlb);return 0 ; }
@@ -560,7 +560,7 @@ static long madvise_pageout(struct vm_area_struct *vma,lru_add_drain ();tlb_gather_mmu (&tlb, mm, start_addr, end_addr);madvise_pageout_page_range (&tlb, vma, start_addr, end_addr); -tlb_finish_mmu (&tlb, start_addr, end_addr); +tlb_finish_mmu (&tlb);return 0 ; }
@@ -732,7 +732,7 @@ static int madvise_free_single_vma(struct vm_area_struct *vma, &madvise_free_walk_ops, &tlb);tlb_end_vma (&tlb, vma);mmu_notifier_invalidate_range_end (&range); -tlb_finish_mmu (&tlb, range.start, range.end); +tlb_finish_mmu (&tlb);return 0 ; }
mm/memory.c +2 -2
@@ -1529,7 +1529,7 @@ void zap_page_range(struct vm_area_struct *vma, unsigned long start,for ( ; vma && vma->vm_start < range.end; vma = vma->vm_next)unmap_single_vma (&tlb, vma, start, range.end,NULL );mmu_notifier_invalidate_range_end (&range); -tlb_finish_mmu (&tlb, start, range.end); +tlb_finish_mmu (&tlb); }/**
@@ -1555,7 +1555,7 @@ static void zap_page_range_single(struct vm_area_struct *vma, unsigned long addrmmu_notifier_invalidate_range_start (&range);unmap_single_vma (&tlb, vma, address, range.end, details);mmu_notifier_invalidate_range_end (&range); -tlb_finish_mmu (&tlb, address, range.end); +tlb_finish_mmu (&tlb); }/**
mm/mmap.c +2 -2
@@ -2676,7 +2676,7 @@ static void unmap_region(struct mm_struct *mm,unmap_vmas (&tlb, vma, start, end);free_pgtables (&tlb, vma, prev ? prev->vm_end :FIRST_USER_ADDRESS , next ? next->vm_start :USER_PGTABLES_CEILING ); -tlb_finish_mmu (&tlb, start, end); +tlb_finish_mmu (&tlb); }/*
@@ -3219,7 +3219,7 @@ void exit_mmap(struct mm_struct *mm)/* Use -1 here to ensure all VMAs in the mm are unmapped */ unmap_vmas (&tlb, vma,0 ,-1 );free_pgtables (&tlb, vma,FIRST_USER_ADDRESS ,USER_PGTABLES_CEILING ); -tlb_finish_mmu (&tlb,0 ,-1 ); +tlb_finish_mmu (&tlb);/* * Walk the list again, actually closing and freeing it,
mm/mmu_gather.c +1 -4
@@ -290,14 +290,11 @@ void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,/** * tlb_finish_mmu - finish an mmu_gather structure * @tlb: the mmu_gather structure to finish -* @start: start of the region that will be removed from the page-table -* @end: end of the region that will be removed from the page-table * * Called at the end of the shootdown operation to free up any resources that * were required. */ -void tlb_finish_mmu (struct mmu_gather *tlb, -unsigned long start,unsigned long end) +void tlb_finish_mmu (struct mmu_gather *tlb) {/* * If there are parallel threads are doing PTE changes on same range
mm/oom_kill.c +2 -2
@@ -546,13 +546,13 @@ bool __oom_reap_task_mm(struct mm_struct *mm) vma->vm_end);tlb_gather_mmu (&tlb, mm, range.start, range.end);if (mmu_notifier_invalidate_range_start_nonblock (&range)) { -tlb_finish_mmu (&tlb, range.start, range.end); +tlb_finish_mmu (&tlb); ret = false;continue ; }unmap_page_range (&tlb, vma, range.start, range.end,NULL );mmu_notifier_invalidate_range_end (&range); -tlb_finish_mmu (&tlb, range.start, range.end); +tlb_finish_mmu (&tlb); } }
Signature · 1 line
2.29.2.576.ga3fc446d84-goog
Will Deacon
maintainerPATCH v2 3/6 tlb: mmu_gather: Introduce tlb_gather_mmu_fullmm()
In reply to Will DeaconPassing the range '0, -1' to tlb_gather_mmu() sets the 'fullmm' flag, which indicates that the mm_struct being operated on is going away. In this case, some architectures (such as arm64) can elide TLB invalidation by ensuring that the TLB tag (ASID) associated with this mm is not immediately reclaimed. Although this behaviour is documented in asm-generic/tlb.h, it's subtle and easily missed.
Introduce tlb_gather_mmu_fullmm() to make it clearer that this is for the entire mm and WARN() if tlb_gather_mmu() is called with the 'fullmm' address range.
--- include/asm-generic/tlb.h | 6 ++++-- include/linux/mm_types.h | 1 + mm/mmap.c | 2 +- mm/mmu_gather.c | 16 ++++++++++++++-- 4 files changed, 20 insertions(+), 5 deletions(-)
include/asm-generic/tlb.h +4 -2
@@ -46,7 +46,9 @@* * The mmu_gather API consists of: * -* - tlb_gather_mmu() / tlb_finish_mmu(); start and finish a mmu_gather +* - tlb_gather_mmu() / tlb_gather_mmu_fullmm() / tlb_finish_mmu() +* +* start and finish a mmu_gather * * Finish in particular will issue a (final) TLB invalidate and free * all (remaining) queued pages.
@@ -91,7 +93,7 @@* * - mmu_gather::fullmm * -* A flag set by tlb_gather_mmu() to indicate we're going to free +* A flag set by tlb_gather_mmu_fullmm() to indicate we're going to free * the entire mm; this allows a number of optimizations. * * - We can ignore tlb_{start,end}_vma(); because we don't
include/linux/mm_types.h +1 -0
@@ -585,6 +585,7 @@ static inline cpumask_t *mm_cpumask(struct mm_struct *mm)struct mmu_gather ;extern void tlb_gather_mmu (struct mmu_gather *tlb,struct mm_struct *mm,unsigned long start,unsigned long end); +extern void tlb_gather_mmu_fullmm (struct mmu_gather *tlb,struct mm_struct *mm);extern void tlb_finish_mmu (struct mmu_gather *tlb);static inline void init_tlb_flush_pending (struct mm_struct *mm)
mm/mmap.c +1 -1
@@ -3214,7 +3214,7 @@ void exit_mmap(struct mm_struct *mm)lru_add_drain ();flush_cache_mm (mm); -tlb_gather_mmu (&tlb, mm,0 ,-1 ); +tlb_gather_mmu_fullmm (&tlb, mm);/* update_hiwater_rss(mm) here? but nobody should be looking */ /* Use -1 here to ensure all VMAs in the mm are unmapped */ unmap_vmas (&tlb, vma,0 ,-1 );
mm/mmu_gather.c +14 -2
@@ -261,8 +261,8 @@ void tlb_flush_mmu(struct mmu_gather *tlb)* respectively when @mm is without users and we're going to destroy * the full address space (exit/execve). */ -void tlb_gather_mmu (struct mmu_gather *tlb,struct mm_struct *mm, -unsigned long start,unsigned long end) +static void __tlb_gather_mmu (struct mmu_gather *tlb,struct mm_struct *mm, +unsigned long start,unsigned long end) { tlb->mm = mm;
@@ -287,6 +287,18 @@ void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,inc_tlb_flush_pending (tlb->mm); } +void tlb_gather_mmu (struct mmu_gather *tlb,struct mm_struct *mm, +unsigned long start,unsigned long end) +{ +WARN_ON (!(start | (end +1 )));/* Use _fullmm() instead */ +__tlb_gather_mmu (tlb, mm, start, end); +} + +void tlb_gather_mmu_fullmm (struct mmu_gather *tlb,struct mm_struct *mm) +{ +__tlb_gather_mmu (tlb, mm,0 ,-1 ); +} +/** * tlb_finish_mmu - finish an mmu_gather structure * @tlb: the mmu_gather structure to finish
Signature · 1 line
2.29.2.576.ga3fc446d84-goog
Will Deacon
maintainerPATCH v2 4/6 tlb: mmu_gather: Remove start/end arguments from tlb_gather_mmu()
In reply to Will DeaconThe 'start' and 'end' arguments to tlb_gather_mmu() are no longer needed now that there is a separate function for 'fullmm' flushing.
Remove the unused arguments and update all callers.
--- arch/ia64/include/asm/tlb.h | 2 +- arch/x86/kernel/ldt.c | 2 +- fs/exec.c | 2 +- include/linux/mm_types.h | 3 +-- mm/hugetlb.c | 16 +--------------- mm/madvise.c | 6 +++--- mm/memory.c | 4 ++-- mm/mmap.c | 2 +- mm/mmu_gather.c | 22 ++++++++-------------- mm/oom_kill.c | 2 +- 10 files changed, 20 insertions(+), 41 deletions(-)
arch/ia64/include/asm/tlb.h +1 -1
@@ -23,7 +23,7 @@* unmapping a portion of the virtual address space, these hooks are called according to * the following template: * -* tlb <- tlb_gather_mmu(mm, start, end); // start unmap for address space MM +* tlb <- tlb_gather_mmu(mm); // start unmap for address space MM * { * for each vma that needs a shootdown do { * tlb_start_vma(tlb, vma);
arch/x86/kernel/ldt.c +1 -1
@@ -398,7 +398,7 @@ static void free_ldt_pgtables(struct mm_struct *mm)if (!boot_cpu_has (X86_FEATURE_PTI ))return ; -tlb_gather_mmu (&tlb, mm, start, end); +tlb_gather_mmu (&tlb, mm);free_pgd_range (&tlb, start, end, start, end);tlb_finish_mmu (&tlb);#endif
fs/exec.c +1 -1
@@ -707,7 +707,7 @@ static int shift_arg_pages(struct vm_area_struct *vma, unsigned long shift)return -ENOMEM ;lru_add_drain (); -tlb_gather_mmu (&tlb, mm, old_start, old_end); +tlb_gather_mmu (&tlb, mm);if (new_end > old_start) {/* * when the old and new regions overlap clear from new_end.
include/linux/mm_types.h +1 -2
@@ -583,8 +583,7 @@ static inline cpumask_t *mm_cpumask(struct mm_struct *mm) }struct mmu_gather ; -extern void tlb_gather_mmu (struct mmu_gather *tlb,struct mm_struct *mm, -unsigned long start,unsigned long end); +extern void tlb_gather_mmu (struct mmu_gather *tlb,struct mm_struct *mm);extern void tlb_gather_mmu_fullmm (struct mmu_gather *tlb,struct mm_struct *mm);extern void tlb_finish_mmu (struct mmu_gather *tlb);
mm/hugetlb.c +1 -15
@@ -3967,23 +3967,9 @@ void __unmap_hugepage_range_final(struct mmu_gather *tlb,void unmap_hugepage_range (struct vm_area_struct *vma,unsigned long start,unsigned long end,struct page *ref_page) { -struct mm_struct *mm;struct mmu_gather tlb; -unsigned long tlb_start = start; -unsigned long tlb_end = end; -/* -* If shared PMDs were possibly used within this vma range, adjust -* start/end for worst case tlb flushing. -* Note that we can not be sure if PMDs are shared until we try to -* unmap pages. However, we want to make sure TLB flushing covers -* the largest possible range. -*/ -adjust_range_if_pmd_sharing_possible (vma, &tlb_start, &tlb_end); - - mm = vma->vm_mm; - -tlb_gather_mmu (&tlb, mm, tlb_start, tlb_end); +tlb_gather_mmu (&tlb, vma->vm_mm);__unmap_hugepage_range (&tlb, vma, start, end, ref_page);tlb_finish_mmu (&tlb); }
mm/madvise.c +3 -3
@@ -506,7 +506,7 @@ static long madvise_cold(struct vm_area_struct *vma,return -EINVAL ;lru_add_drain (); -tlb_gather_mmu (&tlb, mm, start_addr, end_addr); +tlb_gather_mmu (&tlb, mm);madvise_cold_page_range (&tlb, vma, start_addr, end_addr);tlb_finish_mmu (&tlb);
@@ -558,7 +558,7 @@ static long madvise_pageout(struct vm_area_struct *vma,return 0 ;lru_add_drain (); -tlb_gather_mmu (&tlb, mm, start_addr, end_addr); +tlb_gather_mmu (&tlb, mm);madvise_pageout_page_range (&tlb, vma, start_addr, end_addr);tlb_finish_mmu (&tlb);
@@ -723,7 +723,7 @@ static int madvise_free_single_vma(struct vm_area_struct *vma, range.start, range.end);lru_add_drain (); -tlb_gather_mmu (&tlb, mm, range.start, range.end); +tlb_gather_mmu (&tlb, mm);update_hiwater_rss (mm);mmu_notifier_invalidate_range_start (&range);
mm/memory.c +2 -2
@@ -1523,7 +1523,7 @@ void zap_page_range(struct vm_area_struct *vma, unsigned long start,lru_add_drain ();mmu_notifier_range_init (&range,MMU_NOTIFY_CLEAR ,0 , vma, vma->vm_mm, start, start + size); -tlb_gather_mmu (&tlb, vma->vm_mm, start, range.end); +tlb_gather_mmu (&tlb, vma->vm_mm);update_hiwater_rss (vma->vm_mm);mmu_notifier_invalidate_range_start (&range);for ( ; vma && vma->vm_start < range.end; vma = vma->vm_next)
@@ -1550,7 +1550,7 @@ static void zap_page_range_single(struct vm_area_struct *vma, unsigned long addrlru_add_drain ();mmu_notifier_range_init (&range,MMU_NOTIFY_CLEAR ,0 , vma, vma->vm_mm, address, address + size); -tlb_gather_mmu (&tlb, vma->vm_mm, address, range.end); +tlb_gather_mmu (&tlb, vma->vm_mm);update_hiwater_rss (vma->vm_mm);mmu_notifier_invalidate_range_start (&range);unmap_single_vma (&tlb, vma, address, range.end, details);
mm/mmap.c +1 -1
@@ -2671,7 +2671,7 @@ static void unmap_region(struct mm_struct *mm,struct mmu_gather tlb;lru_add_drain (); -tlb_gather_mmu (&tlb, mm, start, end); +tlb_gather_mmu (&tlb, mm);update_hiwater_rss (mm);unmap_vmas (&tlb, vma, start, end); free_pgtables(&tlb, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS,
mm/mmu_gather.c +8 -14
@@ -253,21 +253,17 @@ void tlb_flush_mmu(struct mmu_gather *tlb)* tlb_gather_mmu - initialize an mmu_gather structure for page-table tear-down * @tlb: the mmu_gather structure to initialize * @mm: the mm_struct of the target address space -* @start: start of the region that will be removed from the page-table -* @end: end of the region that will be removed from the page-table +* @fullmm: @mm is without users and we're going to destroy the full address +* space (exit/execve) * * Called to initialize an (on-stack) mmu_gather structure for page-table -* tear-down from @mm. The @start and @end are set to 0 and -1 -* respectively when @mm is without users and we're going to destroy -* the full address space (exit/execve). +* tear-down from @mm. */ static void __tlb_gather_mmu (struct mmu_gather *tlb,struct mm_struct *mm, -unsigned long start,unsigned long end) +bool fullmm) { tlb->mm = mm; - -/* Is it from 0 to ~0? */ - tlb->fullmm = !(start | (end+1 )); + tlb->fullmm = fullmm;#ifndef CONFIG_MMU_GATHER_NO_GATHER tlb->need_flush_all =0 ;
@@ -287,16 +283,14 @@ static void __tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,inc_tlb_flush_pending (tlb->mm); } -void tlb_gather_mmu (struct mmu_gather *tlb,struct mm_struct *mm, -unsigned long start,unsigned long end) +void tlb_gather_mmu (struct mmu_gather *tlb,struct mm_struct *mm) { -WARN_ON (!(start | (end +1 )));/* Use _fullmm() instead */ -__tlb_gather_mmu (tlb, mm, start, end); +__tlb_gather_mmu (tlb, mm, false); }void tlb_gather_mmu_fullmm (struct mmu_gather *tlb,struct mm_struct *mm) { -__tlb_gather_mmu (tlb, mm,0 ,-1 ); +__tlb_gather_mmu (tlb, mm, true); }/**
mm/oom_kill.c +1 -1
@@ -544,7 +544,7 @@ bool __oom_reap_task_mm(struct mm_struct *mm)mmu_notifier_range_init (&range,MMU_NOTIFY_UNMAP ,0 , vma, mm, vma->vm_start, vma->vm_end); -tlb_gather_mmu (&tlb, mm, range.start, range.end); +tlb_gather_mmu (&tlb, mm);if (mmu_notifier_invalidate_range_start_nonblock (&range)) {tlb_finish_mmu (&tlb); ret = false;
Signature · 1 line
2.29.2.576.ga3fc446d84-goog
Will Deacon
maintainerPATCH v2 5/6 tlb: arch: Remove empty __tlb_remove_tlb_entry() stubs
In reply to Will DeaconIf __tlb_remove_tlb_entry() is not defined by the architecture then we provide an empty definition in asm-generic/tlb.h.
Remove the redundant empty definitions for sparc64 and x86.
--- arch/sparc/include/asm/tlb_64.h | 1 - arch/x86/include/asm/tlb.h | 1 - 2 files changed, 2 deletions(-)
arch/sparc/include/asm/tlb_64.h +0 -1
@@ -24,7 +24,6 @@ void flush_tlb_pending(void);#define tlb_start_vma (tlb, vma) do { } while (0)#define tlb_end_vma (tlb, vma) do { } while (0) -#define __tlb_remove_tlb_entry (tlb, ptep, address) do { } while (0)#define tlb_flush (tlb) flush_tlb_pending()/*
arch/x86/include/asm/tlb.h +0 -1
@@ -4,7 +4,6 @@#define tlb_start_vma (tlb, vma) do { } while (0)#define tlb_end_vma (tlb, vma) do { } while (0) -#define __tlb_remove_tlb_entry (tlb, ptep, address) do { } while (0)#define tlb_flush tlb_flushstatic inline void tlb_flush (struct mmu_gather *tlb);
Signature · 1 line
2.29.2.576.ga3fc446d84-goog
Will Deacon
maintainerPATCH v2 6/6 x86/ldt: Use tlb_gather_mmu_fullmm() when freeing LDT page-tables
In reply to Will Deaconfree_ldt_pgtables() uses the MMU gather API for batching TLB flushes over the call to free_pgd_range(). However, tlb_gather_mmu() expects to operate on user addresses and so passing LDT_{BASE,END}_ADDR will confuse the range setting logic in __tlb_adjust_range(), causing the gather to identify a range starting at TASK_SIZE. Such a large range will be converted into a 'fullmm' flush by the low-level invalidation code, so change the caller to invoke tlb_gather_mmu_fullmm() directly.
--- arch/x86/kernel/ldt.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-)
arch/x86/kernel/ldt.c +7 -1
@@ -398,7 +398,13 @@ static void free_ldt_pgtables(struct mm_struct *mm)if (!boot_cpu_has (X86_FEATURE_PTI ))return ; -tlb_gather_mmu (&tlb, mm); +/* +* Although free_pgd_range() is intended for freeing user +* page-tables, it also works out for kernel mappings on x86. +* We use tlb_gather_mmu_fullmm() to avoid confusing the +* range-tracking logic in __tlb_adjust_range(). +*/ +tlb_gather_mmu_fullmm (&tlb, mm);free_pgd_range (&tlb, start, end, start, end);tlb_finish_mmu (&tlb);#endif
Signature · 1 line
2.29.2.576.ga3fc446d84-goog
Quoted 8 lines · Will Deacon
Will Deacon (6):
mm: proc: Invalidate TLB after clearing soft-dirty page state
tlb:mmu_gather: Remove unused start/end arguments from
tlb_finish_mmu()
tlb:mmu_gather: Introducetlb_gather_mmu_fullmm()
tlb:mmu_gather: Remove start/end arguments fromtlb_gather_mmu()
tlb: arch: Remove empty__tlb_remove_tlb_entry()stubs x86/ldt: Usetlb_gather_mmu_fullmm()when freeing LDT page-tables
Re: [PATCH v2 3/6] tlb: mmu_gather: Introduce tlb_gather_mmu_fullmm()
In reply to Will DeaconQuoted 89 lines · Will Deacon
Passing the range '0, -1' to
tlb_gather_mmu()sets the 'fullmm' flag, which indicates that themm_structbeing operated on is going away. In this case, some architectures (such as arm64) can elide TLB invalidation by ensuring that the TLB tag (ASID) associated with this mm is not immediately reclaimed. Although this behaviour is documented inasm-generic/tlb.h, it's subtle and easily missed.Introduce
tlb_gather_mmu_fullmm()to make it clearer that this is for the entire mm andWARN()iftlb_gather_mmu()is called with the 'fullmm' address range.Signed-off-by Will Deacon <[email protected]>--- include/asm-generic/tlb.h | 6 ++++-- include/linux/mm_types.h | 1 + mm/mmap.c | 2 +- mm/mmu_gather.c | 16 ++++++++++++++-- 4 files changed, 20 insertions(+), 5 deletions(-)include/asm-generic/tlb.h +4
-2@@ -46,7 +46,9 @@* * The mmu_gather API consists of: * -* - tlb_gather_mmu() / tlb_finish_mmu(); start and finish a mmu_gather +* - tlb_gather_mmu() / tlb_gather_mmu_fullmm() / tlb_finish_mmu() +* +* start and finish a mmu_gather * * Finish in particular will issue a (final) TLB invalidate and free * all (remaining) queued pages. @@ -91,7 +93,7 @@* * - mmu_gather::fullmm * -* A flag set by tlb_gather_mmu() to indicate we're going to free +* A flag set by tlb_gather_mmu_fullmm() to indicate we're going to free * the entire mm; this allows a number of optimizations. * * - We can ignore tlb_{start,end}_vma(); because we don't include/linux/mm_types.h +1
-0@@ -585,6 +585,7 @@ static inline cpumask_t *mm_cpumask(struct mm_struct *mm)struct mmu_gather ;extern void tlb_gather_mmu (struct mmu_gather *tlb,struct mm_struct *mm,unsigned long start,unsigned long end); +extern void tlb_gather_mmu_fullmm (struct mmu_gather *tlb,struct mm_struct *mm);extern void tlb_finish_mmu (struct mmu_gather *tlb);static inline void init_tlb_flush_pending (struct mm_struct *mm)mm/mmap.c +1
-1@@ -3214,7 +3214,7 @@ void exit_mmap(struct mm_struct *mm)lru_add_drain ();flush_cache_mm (mm); -tlb_gather_mmu (&tlb, mm,0 ,-1 ); +tlb_gather_mmu_fullmm (&tlb, mm);/* update_hiwater_rss(mm) here? but nobody should be looking */ /* Use -1 here to ensure all VMAs in the mm are unmapped */ unmap_vmas (&tlb, vma,0 ,-1 );mm/mmu_gather.c +4
-2@@ -261,8 +261,8 @@ void tlb_flush_mmu(struct mmu_gather *tlb)* respectively when @mm is without users and we're going to destroy * the full address space (exit/execve). */ -void tlb_gather_mmu (struct mmu_gather *tlb,struct mm_struct *mm, -unsigned long start,unsigned long end) +static void __tlb_gather_mmu (struct mmu_gather *tlb,struct mm_struct *mm, +unsigned long start,unsigned long end) { tlb->mm = mm;@@ -287,6 +287,18 @@ void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,inc_tlb_flush_pending (tlb->mm); } +void tlb_gather_mmu (struct mmu_gather *tlb,struct mm_struct *mm, +unsigned long start,unsigned long end)
mm/mmu_gather.c void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, · line 301
Quoted 10 lines
mm/mmu_gather.c +10
-0+{ +WARN_ON (!(start | (end +1 )));/* Use _fullmm() instead */ +__tlb_gather_mmu (tlb, mm, start, end); +} + +void tlb_gather_mmu_fullmm (struct mmu_gather *tlb,struct mm_struct *mm) +{ +__tlb_gather_mmu (tlb, mm,0 ,-1 ); +} +
IMO, there is no point adding the wrappers given you will remove the WARN_ON in the next patch.
But if you prefer, they should be at least moved to the header for inlining.
Consider this whole series
Will Deacon
maintainerQuoted 11 lines · Peter Zijlstra
Quoted 8 lines · Will Deacon
Will Deacon (6):
mm: proc: Invalidate TLB after clearing soft-dirty page state
tlb:mmu_gather: Remove unused start/end arguments from
tlb_finish_mmu()
tlb:mmu_gather: Introducetlb_gather_mmu_fullmm()
tlb:mmu_gather: Remove start/end arguments fromtlb_gather_mmu()
tlb: arch: Remove empty__tlb_remove_tlb_entry()stubs x86/ldt: Usetlb_gather_mmu_fullmm()when freeing LDT page-tables
Cheers. I don't think this series is urgent, so I'll post it again in the New Year.
Will