riscv: mm: Introduce lazy tlb flush
Subsystems (5)
v2 cover riscv: mm: Introduce lazy tlb flush
This patch series introduces a lazy tlb flush mechanism for riscv. This mechanism is based on two insights:
1) Since each CPU has limited TLB entries, there exist limited active ASIDs in each CPU's TLB at the same time. When a mm has not been used for enough long time (or, after enough switch_mm times), we can assume its TLB entries are all evicted out. Then we can clear current CPU in its mm_cpumask so that next time when the memory mapping of this mm is modified, no IPI will be sent to current CPU.
2) When memory mapping of a mm is modified, instead of sending IPI to all CPUs recorded in its mm_cpumask, we check whether each target CPU is using this mm right now. If not, we just store the TLB Flush information in target CPU's percpu buffer, avoiding the IPI. Next time when the target CPU switch_mm to this mm, it can check the percpu buffer and perform TLB Flush itself, without IPI involvement either.
Using this mechanism, we significantly reduced the number of IPI due to TLB Flush:
* ltp - mmapstress01
Before: ~108k
After: ~17k
* ltp - hackbench
Before: ~385k
After: ~2k
Thanks Guo Ren for his advice on memory access latency test via lmbench. We are unable to test it now due to lack of real machines. We will supply this test and adjust our mechanism according to it as soon as possible.
Xu Lu (9):
riscv: Introduce RISCV_LAZY_TLB_FLUSH config
riscv: mm: Apply a threshold to the number of active ASIDs on each CPU
riscv: mm: Grab mm_count to avoid mm getting released
fork: Add arch override for do_shoot_lazy_tlb()
riscv: mm: Introduce arch_do_shoot_lazy_tlb
riscv: mm: Introduce percpu TLB Flush queue
riscv: mm: Defer the TLB Flush to switch_mm
riscv: mm: Clear mm_cpumask during local_flush_tlb_all_asid()
riscv: mm: Clear mm_cpumask during local_flush_tlb_all()
arch/riscv/Kconfig | 12 ++ arch/riscv/include/asm/mmu.h | 4 + arch/riscv/include/asm/mmu_context.h | 5 + arch/riscv/include/asm/tlbflush.h | 63 ++++++ arch/riscv/mm/context.c | 24 ++- arch/riscv/mm/tlbflush.c | 302 +++++++++++++++++++++++++-- kernel/fork.c | 6 +- 7 files changed, 394 insertions(+), 22 deletions(-)
Signature · 1 line
2.20.1
RFC PATCH v2 1/9 riscv: Introduce RISCV_LAZY_TLB_FLUSH config
This feature avoids unnecessary TLB Flush IPIs. After memory mapping modifications on certain mm_struct, instead of sending IPIs, this feature records the TLB Flush information on percpu buffer, defer the TLB Flush to the moment when target CPUs really load this mm_struct.
--- arch/riscv/Kconfig | 12 ++++++++++++ 1 file changed, 12 insertions(+)
arch/riscv/Kconfig +12 -0
@@ -925,6 +925,18 @@ config RISCV_VECTOR_MISALIGNED help Enable detecting support for vector misaligned loads and stores. +config RISCV_LAZY_TLB_FLUSH + bool "Defer TLB Flush to context switch to avoid IPIs" + depends on MMU && SMP + def_bool n + help + This feature avoids unnecessary TLB Flush IPIs. After memory mapping + modifications on certain mm_struct, instead of sending IPIs, this feature + records the TLB Flush information on percpu buffer, defer the TLB Flush + to the moment when target CPUs really load this mm_struct. + + If unsure what to do here, say N. + choice prompt "Unaligned Accesses Support" default RISCV_PROBE_UNALIGNED_ACCESS
Signature · 1 line
2.20.1
RFC PATCH v2 2/9 riscv: mm: Apply a threshold to the number of active ASIDs on each CPU
In reply to Xu LuSince each CPU has limited TLB entries, there exist limited active ASIDs in each CPU's TLB at the same time. Thus we apply a threshold here. When a mm_struct is loaded, we mark its ASID as active. If the number of active ASIDs exceeds the threshold, we evict the mm_struct that has not been used for the longest time, flush its TLB entries, mark its ASID inactive, and clear current CPU in its mm_cpumask.
--- arch/riscv/include/asm/tlbflush.h | 27 +++++++++++++ arch/riscv/mm/context.c | 1 + arch/riscv/mm/tlbflush.c | 66 +++++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+)
arch/riscv/include/asm/tlbflush.h +27 -0
@@ -66,6 +66,33 @@ void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *batch,void arch_tlbbatch_flush (struct arch_tlbflush_unmap_batch *batch);extern unsigned long tlb_flush_all_threshold; + +#ifdef CONFIG_RISCV_LAZY_TLB_FLUSH + +#define MAX_LOADED_MM 6 + +struct tlb_context { +struct mm_struct *mm; +unsigned int gen; +}; + +struct tlb_info { +rwlock_t rwlock; +struct mm_struct *active_mm; +unsigned int next_gen; +struct tlb_context contexts[MAX_LOADED_MM ]; +}; + +DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_info, tlbinfo); + +void local_load_tlb_mm (struct mm_struct *mm); + +#else /* CONFIG_RISCV_LAZY_TLB_FLUSH */ + +static inline void local_load_tlb_mm (struct mm_struct *mm) {} + +#endif /* CONFIG_RISCV_LAZY_TLB_FLUSH */ +#else /* CONFIG_MMU */ #define local_flush_tlb_all () do { } while (0)#endif /* CONFIG_MMU */
arch/riscv/mm/context.c +1 -0
@@ -217,6 +217,7 @@ static inline void set_mm(struct mm_struct *prev,*/ cpumask_set_cpu (cpu,mm_cpumask (next));if (static_branch_unlikely (&use_asid_allocator)) { +local_load_tlb_mm (next);set_mm_asid (next, cpu); }else {cpumask_clear_cpu (cpu,mm_cpumask (prev));
arch/riscv/mm/tlbflush.c +66 -0
75 lines not on this page · raw message
Signature · 1 line
2.20.1
RFC PATCH v2 3/9 riscv: mm: Grab mm_count to avoid mm getting released
In reply to Xu LuWe maintain an array of mm_structs whose ASIDs are active on the current CPU. To avoid these mm_structs getting released, we grab their mm_count before loaded them into the array. And drop their mm_count via tasklet when they are evicted out of the array.
--- arch/riscv/include/asm/mmu.h | 4 +++ arch/riscv/mm/tlbflush.c | 47 ++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+)
arch/riscv/include/asm/mmu.h +4 -0
@@ -30,6 +30,10 @@ typedef struct {#ifdef CONFIG_RISCV_ISA_SUPM u8 pmlen;#endif +#ifdef CONFIG_RISCV_LAZY_TLB_FLUSH +atomic_t lazy_tlb_cnt; +void *next; +#endif } mm_context_t;/* Lock the pointer masking mode because this mm is multithreaded */
arch/riscv/mm/tlbflush.c +47 -0
71 lines not on this page · raw message
Signature · 1 line
2.20.1
RFC PATCH v2 4/9 fork: Add arch override for do_shoot_lazy_tlb()
In reply to Xu LuWhen both CONFIG_RISCV_LAZY_TLB_FLUSH and CONFIG_MMU_LSYZ_TLB_SHOOTDOWN is enabled, riscv needs an arch special method to free the mm that needs to be shot down. Thus we add arch override for do_shoot_lazy_tlb().
--- kernel/fork.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-)
kernel/fork.c +4 -2
@@ -612,7 +612,8 @@ static void do_check_lazy_tlb(void *arg)WARN_ON_ONCE (current->active_mm == mm); } -static void do_shoot_lazy_tlb (void *arg) +#ifndef arch_do_shoot_lazy_tlb +static void arch_do_shoot_lazy_tlb (void *arg) {struct mm_struct *mm = arg;
@@ -622,6 +623,7 @@ static void do_shoot_lazy_tlb(void *arg) switch_mm(mm, &init_mm, current); } } +#endif static void cleanup_lazy_tlbs (struct mm_struct *mm) {
@@ -661,7 +663,7 @@ static void cleanup_lazy_tlbs(struct mm_struct *mm)* - A delayed freeing and RCU-like quiescing sequence based on mm * switching to avoid IPIs completely. */ -on_each_cpu_mask (mm_cpumask (mm), do_shoot_lazy_tlb, (void *)mm,1 ); +on_each_cpu_mask (mm_cpumask (mm), arch_do_shoot_lazy_tlb, (void *)mm,1 );if (IS_ENABLED (CONFIG_DEBUG_VM_SHOOT_LAZIES ))on_each_cpu (do_check_lazy_tlb, (void *)mm,1 ); }
Signature · 1 line
2.20.1
RFC PATCH v2 5/9 riscv: mm: Introduce arch_do_shoot_lazy_tlb
In reply to Xu LuWhen an active_mm is shot down, we switch it to the init_mm, evict it out of percpu active mm array.
--- arch/riscv/include/asm/mmu_context.h | 5 ++++ arch/riscv/include/asm/tlbflush.h | 11 +++++++++ arch/riscv/mm/context.c | 19 ++++++++++++++++ arch/riscv/mm/tlbflush.c | 34 ++++++++++++++++++++++++---- 4 files changed, 64 insertions(+), 5 deletions(-)
arch/riscv/include/asm/mmu_context.h +5 -0
@@ -16,6 +16,11 @@void switch_mm (struct mm_struct *prev,struct mm_struct *next,struct task_struct *task); +#ifdef CONFIG_RISCV_LAZY_TLB_FLUSH +#define arch_do_shoot_lazy_tlb arch_do_shoot_lazy_tlb +void arch_do_shoot_lazy_tlb (void *arg); +#endif +#define activate_mm activate_mmstatic inline void activate_mm (struct mm_struct *prev,struct mm_struct *next)
arch/riscv/include/asm/tlbflush.h +11 -0
@@ -15,6 +15,11 @@#define FLUSH_TLB_NO_ASID ((unsigned long)-1)#ifdef CONFIG_MMU +static inline unsigned long get_mm_asid (struct mm_struct *mm) +{ +return mm ?cntx2asid (atomic_long_read (&mm->context.id)) :FLUSH_TLB_NO_ASID ; +} +static inline void local_flush_tlb_all (void ) { __asm__ __volatile__ ("sfence.vma" : : :"memory" );
@@ -86,11 +91,17 @@ struct tlb_info { DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_info, tlbinfo);void local_load_tlb_mm (struct mm_struct *mm); +void local_flush_tlb_mm (struct mm_struct *mm);#else /* CONFIG_RISCV_LAZY_TLB_FLUSH */ static inline void local_load_tlb_mm (struct mm_struct *mm) {} +static inline void local_flush_tlb_mm (struct mm_struct *mm) +{ +local_flush_tlb_all_asid (get_mm_asid (mm)); +} +#endif /* CONFIG_RISCV_LAZY_TLB_FLUSH */ #else /* CONFIG_MMU */
arch/riscv/mm/context.c +19 -0
@@ -274,6 +274,25 @@ static int __init asids_init(void) return 0; }early_initcall (asids_init); + +#ifdef CONFIG_RISCV_LAZY_TLB_FLUSH +void arch_do_shoot_lazy_tlb (void *arg) +{ +struct mm_struct *mm = arg; + +if (current->active_mm == mm) { +WARN_ON_ONCE (current->mm); + current->active_mm = &init_mm; +switch_mm (mm, &init_mm, current); + } + +if (!static_branch_unlikely (&use_asid_allocator) || !mm) +return ; + +local_flush_tlb_mm (mm); +} +#endif /* CONFIG_RISCV_LAZY_TLB_FLUSH */ +#else static inline void set_mm (struct mm_struct *prev,struct mm_struct *next,unsigned int cpu)
arch/riscv/mm/tlbflush.c +29 -5
@@ -164,11 +164,6 @@ static void __ipi_flush_tlb_range_asid(void *info)local_flush_tlb_range_asid (d->start, d->size, d->stride, d->asid); } -static inline unsigned long get_mm_asid (struct mm_struct *mm) -{ -return mm ?cntx2asid (atomic_long_read (&mm->context.id)) :FLUSH_TLB_NO_ASID ; -} - static void __flush_tlb_range(struct mm_struct *mm, const struct cpumask *cmask, unsigned long start, unsigned long size,
@@ -352,4 +347,33 @@ void local_load_tlb_mm(struct mm_struct *mm) } } +void local_flush_tlb_mm (struct mm_struct *mm) +{ +struct tlb_info *info =this_cpu_ptr (&tlbinfo); +struct tlb_context *contexts = info->contexts; +unsigned long asid =get_mm_asid (mm); +unsigned int i; + +if (!mm || mm == info->active_mm) { +local_flush_tlb_all_asid (asid); +return ; + } + +for (i =0 ; i <MAX_LOADED_MM ; i++) { +if (contexts[i].mm != mm) +continue ; + +write_lock (&info->rwlock); + contexts[i].mm =NULL ; + contexts[i].gen =0 ; +write_unlock (&info->rwlock); + +cpumask_clear_cpu (raw_smp_processor_id (),mm_cpumask (mm)); +mmdrop_lazy_mm (mm); +break ; + } + +local_flush_tlb_all_asid (asid); +} +#endif /* CONFIG_RISCV_LAZY_TLB_FLUSH */
Signature · 1 line
2.20.1
RFC PATCH v2 6/9 riscv: mm: Introduce percpu TLB Flush queue
In reply to Xu LuWhen memory mapping of a mm is modified, instead of sending IPI to all CPUs recorded in its mm_cpumask, we check whether each target CPU is using this mm right now. If not, we just store the TLB Flush information in target CPU's percpu TLB Flush queue, avoiding the IPI.
--- arch/riscv/include/asm/tlbflush.h | 19 +++++++++ arch/riscv/mm/context.c | 2 + arch/riscv/mm/tlbflush.c | 71 ++++++++++++++++++++++++++++++- 3 files changed, 90 insertions(+), 2 deletions(-)
arch/riscv/include/asm/tlbflush.h +19 -0
@@ -75,23 +75,40 @@ extern unsigned long tlb_flush_all_threshold;#ifdef CONFIG_RISCV_LAZY_TLB_FLUSH #define MAX_LOADED_MM 6 +#define MAX_TLB_FLUSH_TASK 32 +#define FLUSH_TLB_ALL_ASID 0x1struct tlb_context {struct mm_struct *mm;unsigned int gen; +bool need_flush; }; +struct tlb_flush_task { +unsigned long start; +unsigned long size; +unsigned long stride; +}; + +struct tlb_flush_queue { +atomic_t len; +unsigned int flag; +struct tlb_flush_task tasks[MAX_TLB_FLUSH_TASK ]; +}____cacheline_aligned_in_smp ; +struct tlb_info {rwlock_t rwlock;struct mm_struct *active_mm;unsigned int next_gen;struct tlb_context contexts[MAX_LOADED_MM ]; +struct tlb_flush_queue *flush_queues; }; DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_info, tlbinfo);void local_load_tlb_mm (struct mm_struct *mm);void local_flush_tlb_mm (struct mm_struct *mm); +void __init lazy_tlb_flush_init (void );#else /* CONFIG_RISCV_LAZY_TLB_FLUSH */
@@ -102,6 +119,8 @@ static inline void local_flush_tlb_mm(struct mm_struct *mm) local_flush_tlb_all_asid(get_mm_asid(mm)); } +static inline void lazy_tlb_flush_init (void ) {} +#endif /* CONFIG_RISCV_LAZY_TLB_FLUSH */ #else /* CONFIG_MMU */
arch/riscv/mm/context.c +2 -0
@@ -263,6 +263,8 @@ static int __init asids_init(void)__set_bit (0 , context_asid_map); +lazy_tlb_flush_init (); +static_branch_enable (&use_asid_allocator); pr_info("ASID allocator using %lu bits (%lu entries)\n",
arch/riscv/mm/tlbflush.c +69 -2
103 lines not on this page · raw message
Signature · 1 line
2.20.1
RFC PATCH v2 7/9 riscv: mm: Defer the TLB Flush to switch_mm
In reply to Xu LuSince the TLB Flush IPI is avoided when the modified mm is not running on the target CPU, next time when target CPU switches to the modified mm, it has to check percpu TLB Flush queue and perform TLB FLush for the midified mm.
--- arch/riscv/mm/tlbflush.c | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-)
arch/riscv/mm/tlbflush.c +28 -1
@@ -365,8 +365,10 @@ void local_load_tlb_mm(struct mm_struct *mm) {struct tlb_info *info =this_cpu_ptr (&tlbinfo);struct tlb_context *contexts = info->contexts; +struct tlb_flush_queue *queue =NULL ;struct mm_struct *victim =NULL ; -unsigned int i, pos =0 , min =UINT_MAX ; +unsigned int i, len, pos =0 , min =UINT_MAX ; +unsigned long asid, start, size, stride;for (i =0 ; i <MAX_LOADED_MM ; i++) {if (contexts[i].mm == mm) {
@@ -387,11 +389,36 @@ void local_load_tlb_mm(struct mm_struct *mm)mmgrab_lazy_mm (mm); victim = contexts[pos].mm; contexts[pos].mm = mm; + contexts[pos].need_flush = false; + + queue = &info->flush_queues[pos]; +atomic_set (&queue->len,0 ); + queue->flag =0 ; } contexts[pos].gen =new_tlb_gen (info);write_unlock (&info->rwlock); +if (contexts[pos].need_flush) { + queue = &info->flush_queues[pos]; + asid =get_mm_asid (mm); +if (queue->flag &FLUSH_TLB_ALL_ASID ) { +local_flush_tlb_all_asid (asid); + }else { + len =atomic_read (&queue->len); +for (i =0 ; i < len; i++) { + start = queue->tasks[i].start; + size = queue->tasks[i].size; + stride = queue->tasks[i].stride; +local_flush_tlb_range_asid (start, size, + stride, asid); + } + } + contexts[pos].need_flush = false; +atomic_set (&queue->len,0 ); + queue->flag =0 ; + } +if (victim) {cpumask_clear_cpu (raw_smp_processor_id (),mm_cpumask (victim));local_flush_tlb_all_asid (get_mm_asid (victim));
Signature · 1 line
2.20.1
RFC PATCH v2 8/9 riscv: mm: Clear mm_cpumask during local_flush_tlb_all_asid()
In reply to Xu LuWhen local_flush_tlb_all_asid() is called, the target mm's TLB entries are all flushed out, then we can clear current CPU in its mm_cpumask so that next time the mm's memory mapping is modified, no IPI will be sent to current CPU.
--- arch/riscv/mm/tlbflush.c | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-)
arch/riscv/mm/tlbflush.c +19 -13
@@ -35,7 +35,8 @@ static inline void local_sinval_vma(unsigned long vma, unsigned long asid)*/ unsigned long tlb_flush_all_threshold__read_mostly =64 ; -static void local_flush_tlb_range_threshold_asid (unsigned long start, +static void local_flush_tlb_range_threshold_asid (struct mm_struct *mm, +unsigned long start,unsigned long size,unsigned long stride,unsigned long asid)
@@ -44,7 +45,7 @@ static void local_flush_tlb_range_threshold_asid(unsigned long start,int i;if (nr_ptes_in_range > tlb_flush_all_threshold) { -local_flush_tlb_all_asid (asid); +local_flush_tlb_mm (mm);return ; }
@@ -64,21 +65,26 @@ static void local_flush_tlb_range_threshold_asid(unsigned long start, } } -static inline void local_flush_tlb_range_asid (unsigned long start, -unsigned long size,unsigned long stride,unsigned long asid) +static inline void local_flush_tlb_range_asid (struct mm_struct *mm, +unsigned long start, +unsigned long size, +unsigned long stride, +unsigned long asid) { -if (size <= stride) +if (size <= stride) {local_flush_tlb_page_asid (start, asid); -else if (size ==FLUSH_TLB_MAX_SIZE ) -local_flush_tlb_all_asid (asid); -else -local_flush_tlb_range_threshold_asid (start, size, stride, asid); + }else if (size ==FLUSH_TLB_MAX_SIZE ) { +local_flush_tlb_mm (mm); + }else { +local_flush_tlb_range_threshold_asid (mm, start, size, stride, + asid); + } }/* Flush a range of kernel pages without broadcasting */ void local_flush_tlb_kernel_range (unsigned long start,unsigned long end) { -local_flush_tlb_range_asid (start, end - start,PAGE_SIZE ,FLUSH_TLB_NO_ASID ); +local_flush_tlb_range_asid (NULL , start, end - start,PAGE_SIZE ,FLUSH_TLB_NO_ASID ); }static void __ipi_flush_tlb_all (void *info)
@@ -204,7 +210,7 @@ static void __ipi_flush_tlb_range_asid(void *info) {struct flush_tlb_range_data *d = info; -local_flush_tlb_range_asid (d->start, d->size, d->stride, d->asid); +local_flush_tlb_range_asid (d->mm, d->start, d->size, d->stride, d->asid); } static void __flush_tlb_range(struct mm_struct *mm,
@@ -222,7 +228,7 @@ static void __flush_tlb_range(struct mm_struct *mm,/* Check if the TLB flush needs to be sent to other CPUs. */ if (cpumask_any_but (cmask, cpu) >= nr_cpu_ids) { -local_flush_tlb_range_asid (start, size, stride, asid); +local_flush_tlb_range_asid (mm, start, size, stride, asid); }else if (riscv_use_sbi_for_rfence ()) {sbi_remote_sfence_vma_asid (cmask, start, size, asid); }else {
@@ -410,7 +416,7 @@ void local_load_tlb_mm(struct mm_struct *mm) start = queue->tasks[i].start; size = queue->tasks[i].size; stride = queue->tasks[i].stride; -local_flush_tlb_range_asid (start, size, +local_flush_tlb_range_asid (mm, start, size, stride, asid); } }
Signature · 1 line
2.20.1
RFC PATCH v2 9/9 riscv: mm: Clear mm_cpumask during local_flush_tlb_all()
In reply to Xu LuNow that we maintain an array of active mms on each CPU, when local_flush_tlb_all() is called, we can clear current CPU in the mm_cpumask of all active mms on current CPU.
--- arch/riscv/include/asm/tlbflush.h | 6 ++++++ arch/riscv/mm/context.c | 2 +- arch/riscv/mm/tlbflush.c | 31 +++++++++++++++++++++++++++++-- 3 files changed, 36 insertions(+), 3 deletions(-)
arch/riscv/include/asm/tlbflush.h +6 -0
@@ -108,6 +108,7 @@ DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_info, tlbinfo);void local_load_tlb_mm (struct mm_struct *mm);void local_flush_tlb_mm (struct mm_struct *mm); +void local_flush_tlb_all_mm (void );void __init lazy_tlb_flush_init (void );#else /* CONFIG_RISCV_LAZY_TLB_FLUSH */
@@ -119,6 +120,11 @@ static inline void local_flush_tlb_mm(struct mm_struct *mm) local_flush_tlb_all_asid(get_mm_asid(mm)); } +static inline void local_flush_tlb_all_mm (void ) +{ +local_flush_tlb_all (); +} +static inline void lazy_tlb_flush_init (void ) {}#endif /* CONFIG_RISCV_LAZY_TLB_FLUSH */
arch/riscv/mm/context.c +1 -1
@@ -194,7 +194,7 @@ static void set_mm_asid(struct mm_struct *mm, unsigned int cpu) satp_mode);if (need_flush_tlb) -local_flush_tlb_all (); +local_flush_tlb_all_mm (); }static void set_mm_noasid (struct mm_struct *mm)
arch/riscv/mm/tlbflush.c +29 -2
@@ -89,13 +89,13 @@ void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)static void __ipi_flush_tlb_all (void *info) { -local_flush_tlb_all (); +local_flush_tlb_all_mm (); }void flush_tlb_all (void ) {if (num_online_cpus () <2 ) -local_flush_tlb_all (); +local_flush_tlb_all_mm ();else if (riscv_use_sbi_for_rfence ())sbi_remote_sfence_vma_asid (NULL ,0 ,FLUSH_TLB_MAX_SIZE ,FLUSH_TLB_NO_ASID );else
@@ -461,6 +461,33 @@ void local_flush_tlb_mm(struct mm_struct *mm)local_flush_tlb_all_asid (asid); } +void local_flush_tlb_all_mm (void ) +{ +struct tlb_info *info =this_cpu_ptr (&tlbinfo); +struct tlb_context *contexts = info->contexts; +struct mm_struct *mms[MAX_LOADED_MM ]; +unsigned int cpu =raw_smp_processor_id (); +unsigned int i, num =0 ; + +write_lock (&info->rwlock); +for (i =0 ; i <MAX_LOADED_MM ; i++) { +if (!contexts[i].mm || contexts[i].mm == info->active_mm) +continue ; + + mms[num++] = contexts[i].mm; + contexts[i].mm =NULL ; + contexts[i].gen =0 ; + } +write_unlock (&info->rwlock); + +for (i =0 ; i < num; i++) { +cpumask_clear_cpu (cpu,mm_cpumask (mms[i])); +mmdrop_lazy_mm (mms[i]); + } + +local_flush_tlb_all (); +} +void __init lazy_tlb_flush_init (void ) {struct tlb_flush_queue *queue;
Signature · 1 line
2.20.1