Skip to contentSkip to conversation

powerpc/64s: TLB flushing improvements

PATCH Applied Nicholas Piggin 3 participants 10 messages Dec 17, 2020 – Feb 10, 2021 PowerPC

lore
Share & tools
lore

Nicholas Piggin

author reviewer
#0 lore raw

cover powerpc/64s: TLB flushing improvements

Another round of reducing TLB flushing (mostly on radix).

Thanks,
Nick

Nicholas Piggin (7):
powerpc/64s/radix: add warning and comments in mm_cpumask trim powerpc/64s/radix: refactor TLB flush type selection powerpc/64s/radix: Check for no TLB flush required powerpc/64s/radix: Allow mm_cpumask trimming from external sources powerpc/64s/radix: occasionally attempt to trim mm_cpumask powerpc/64s/radix: serialize_against_pte_lookup IPIs trim mm_cpumask powerpc/64s: Implement ptep_clear_flush_young that does not flush TLBs

 arch/powerpc/include/asm/book3s/64/pgtable.h |  23 +-
 arch/powerpc/mm/book3s64/pgtable.c           |  13 +-
 arch/powerpc/mm/book3s64/radix_tlb.c         | 297 ++++++++++++-------
 3 files changed, 225 insertions(+), 108 deletions(-)
Signature · 1 line
2.23.0
author reviewer
#1 lore raw

PATCH 1/7 powerpc/64s/radix: add warning and comments in mm_cpumask trim

Add a comment explaining part of the logic for mm_cpumask trimming, and add a (hopefully graceful) check and warning in case something gets it wrong.

---
 arch/powerpc/mm/book3s64/radix_tlb.c | 27 +++++++++++++++++++++------
 1 file changed, 21 insertions(+), 6 deletions(-)
arch/powerpc/mm/book3s64/radix_tlb.c +21 -6
index b487b489d4b6..5f09e0cd0016 100644
@@ -644,13 +644,14 @@ static void do_exit_flush_lazy_tlb(void *arg)
 {
 	struct mm_struct *mm = arg;
 	unsigned long pid = mm->context.id;
+	int cpu = smp_processor_id();
 
 	/*
 	 * A kthread could have done a mmget_not_zero() after the flushing CPU
-	 * checked mm_is_singlethreaded, and be in the process of
-	 * kthread_use_mm when interrupted here. In that case, current->mm will
-	 * be set to mm, because kthread_use_mm() setting ->mm and switching to
-	 * the mm is done with interrupts off.
+	 * checked mm_cpumask, and be in the process of kthread_use_mm when
+	 * interrupted here. In that case, current->mm will be set to mm,
+	 * because kthread_use_mm() setting ->mm and switching to the mm is
+	 * done with interrupts off.
 	 */
 	if (current->mm == mm)
 		goto out_flush;
@@ -664,8 +665,22 @@ static void do_exit_flush_lazy_tlb(void *arg)
 		mmdrop(mm);
 	}
 
-	atomic_dec(&mm->context.active_cpus);
-	cpumask_clear_cpu(smp_processor_id(), mm_cpumask(mm));
+	/*
+	 * This IPI is only initiated from a CPU which is running mm which
+	 * is a single-threaded process, so there will not be another racing
+	 * IPI coming in where we would find our cpumask already clear.
+	 *
+	 * Nothing else clears our bit in the cpumask except CPU offlining,
+	 * in which case we should not be taking IPIs here. However check
+	 * this just in case the logic is wrong somewhere, and don't underflow
+	 * the active_cpus count.
+	 */
+	if (cpumask_test_cpu(cpu, mm_cpumask(mm))) {
+		atomic_dec(&mm->context.active_cpus);
+		cpumask_clear_cpu(cpu, mm_cpumask(mm));
+	} else {
+		WARN_ON_ONCE(1);
+	}
 
 out_flush:
 	_tlbiel_pid(pid, RIC_FLUSH_ALL);
Signature · 1 line
2.23.0
author reviewer
#2 lore raw

PATCH 2/7 powerpc/64s/radix: refactor TLB flush type selection

In reply to Nicholas Piggin

The logic to decide what kind of TLB flush is required (local, global, or IPI) is spread multiple times over the several kinds of TLB flushes.

Move it all into a single function which may issue IPIs if necessary, and also returns a flush type that is to be used.

---
 arch/powerpc/mm/book3s64/radix_tlb.c | 176 ++++++++++++++-------------
 1 file changed, 94 insertions(+), 82 deletions(-)
arch/powerpc/mm/book3s64/radix_tlb.c +94 -82
index 5f09e0cd0016..12481c864ab6 100644
@@ -618,15 +618,6 @@ void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmadd
 }
 EXPORT_SYMBOL(radix__local_flush_tlb_page);
 
-static bool mm_is_singlethreaded(struct mm_struct *mm)
-{
-	if (atomic_read(&mm->context.copros) > 0)
-		return false;
-	if (atomic_read(&mm->mm_users) <= 1 && current->mm == mm)
-		return true;
-	return false;
-}
-
 static bool mm_needs_flush_escalation(struct mm_struct *mm)
 {
 	/*
@@ -698,10 +689,58 @@ static void exit_flush_lazy_tlbs(struct mm_struct *mm)
 	smp_call_function_many(mm_cpumask(mm), do_exit_flush_lazy_tlb,
 				(void *)mm, 1);
 }
+#else /* CONFIG_SMP */
+static inline void exit_flush_lazy_tlbs(struct mm_struct *mm) { }
+#endif /* CONFIG_SMP */
+
+enum tlb_flush_type {
+	FLUSH_TYPE_LOCAL,
+	FLUSH_TYPE_GLOBAL,
+};
+
+static enum tlb_flush_type flush_type_needed(struct mm_struct *mm, bool fullmm)
+{
+	if (mm_is_thread_local(mm))
+		return FLUSH_TYPE_LOCAL;
+
+	/* Coprocessors require TLBIE to invalidate nMMU. */
+	if (atomic_read(&mm->context.copros) > 0)
+		return FLUSH_TYPE_GLOBAL;
+
+	/*
+	 * In the fullmm case there's no point doing the exit_flush_lazy_tlbs
+	 * because the mm is being taken down anyway, and a TLBIE tends to
+	 * be faster than an IPI+TLBIEL.
+	 */
+	if (fullmm)
+		return FLUSH_TYPE_GLOBAL;
+
+	/*
+	 * If we are running the only thread of a single-threaded process,
+	 * then we should almost always be able to trim off the rest of the
+	 * CPU mask (except in the case of use_mm() races), so always try
+	 * trimming the mask.
+	 */
+	if (atomic_read(&mm->mm_users) <= 1 && current->mm == mm) {
+		exit_flush_lazy_tlbs(mm);
+		/*
+		 * use_mm() race could prevent IPIs from being able to clear
+		 * the cpumask here, however those users are established
+		 * after our first check (and so after the PTEs are removed),
+		 * and the TLB still gets flushed by the IPI, so this CPU
+		 * will only require a local flush.
+		 */
+		return FLUSH_TYPE_LOCAL;
+	}
+
+	return FLUSH_TYPE_GLOBAL;
+}
 
+#ifdef CONFIG_SMP
 void radix__flush_tlb_mm(struct mm_struct *mm)
 {
 	unsigned long pid;
+	enum tlb_flush_type type;
 
 	pid = mm->context.id;
 	if (unlikely(pid == MMU_NO_CONTEXT))
@@ -709,16 +748,13 @@ void radix__flush_tlb_mm(struct mm_struct *mm)
 
 	preempt_disable();
 	/*
-	 * Order loads of mm_cpumask vs previous stores to clear ptes before
-	 * the invalidate. See barrier in switch_mm_irqs_off
+	 * Order loads of mm_cpumask (in flush_type_needed) vs previous
+	 * stores to clear ptes before the invalidate. See barrier in
+	 * switch_mm_irqs_off
 	 */
 	smp_mb();
-	if (!mm_is_thread_local(mm)) {
-		if (unlikely(mm_is_singlethreaded(mm))) {
-			exit_flush_lazy_tlbs(mm);
-			goto local;
-		}
-
+	type = flush_type_needed(mm, false);
+	if (type == FLUSH_TYPE_GLOBAL) {
 		if (!mmu_has_feature(MMU_FTR_GTSE)) {
 			unsigned long tgt = H_RPTI_TARGET_CMMU;
 
@@ -735,7 +771,6 @@ void radix__flush_tlb_mm(struct mm_struct *mm)
 			_tlbiel_pid_multicast(mm, pid, RIC_FLUSH_TLB);
 		}
 	} else {
-local:
 		_tlbiel_pid(pid, RIC_FLUSH_TLB);
 	}
 	preempt_enable();
@@ -745,6 +780,7 @@ EXPORT_SYMBOL(radix__flush_tlb_mm);
 static void __flush_all_mm(struct mm_struct *mm, bool fullmm)
 {
 	unsigned long pid;
+	enum tlb_flush_type type;
 
 	pid = mm->context.id;
 	if (unlikely(pid == MMU_NO_CONTEXT))
@@ -752,13 +788,8 @@ static void __flush_all_mm(struct mm_struct *mm, bool fullmm)
 
 	preempt_disable();
 	smp_mb(); /* see radix__flush_tlb_mm */
-	if (!mm_is_thread_local(mm)) {
-		if (unlikely(mm_is_singlethreaded(mm))) {
-			if (!fullmm) {
-				exit_flush_lazy_tlbs(mm);
-				goto local;
-			}
-		}
+	type = flush_type_needed(mm, fullmm);
+	if (type == FLUSH_TYPE_GLOBAL) {
 		if (!mmu_has_feature(MMU_FTR_GTSE)) {
 			unsigned long tgt = H_RPTI_TARGET_CMMU;
 			unsigned long type = H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC |
@@ -773,7 +804,6 @@ static void __flush_all_mm(struct mm_struct *mm, bool fullmm)
 		else
 			_tlbiel_pid_multicast(mm, pid, RIC_FLUSH_ALL);
 	} else {
-local:
 		_tlbiel_pid(pid, RIC_FLUSH_ALL);
 	}
 	preempt_enable();
@@ -789,6 +819,7 @@ void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
 				 int psize)
 {
 	unsigned long pid;
+	enum tlb_flush_type type;
 
 	pid = mm->context.id;
 	if (unlikely(pid == MMU_NO_CONTEXT))
@@ -796,11 +827,8 @@ void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
 
 	preempt_disable();
 	smp_mb(); /* see radix__flush_tlb_mm */
-	if (!mm_is_thread_local(mm)) {
-		if (unlikely(mm_is_singlethreaded(mm))) {
-			exit_flush_lazy_tlbs(mm);
-			goto local;
-		}
+	type = flush_type_needed(mm, false);
+	if (type == FLUSH_TYPE_GLOBAL) {
 		if (!mmu_has_feature(MMU_FTR_GTSE)) {
 			unsigned long tgt, pg_sizes, size;
 
@@ -818,7 +846,6 @@ void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
 		else
 			_tlbiel_va_multicast(mm, vmaddr, pid, psize, RIC_FLUSH_TLB);
 	} else {
-local:
 		_tlbiel_va(vmaddr, pid, psize, RIC_FLUSH_TLB);
 	}
 	preempt_enable();
@@ -834,8 +861,6 @@ void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
 }
 EXPORT_SYMBOL(radix__flush_tlb_page);
 
-#else /* CONFIG_SMP */
-static inline void exit_flush_lazy_tlbs(struct mm_struct *mm) { }
 #endif /* CONFIG_SMP */
 
 static void do_tlbiel_kernel(void *info)
@@ -899,7 +924,9 @@ static inline void __radix__flush_tlb_range(struct mm_struct *mm,
 	unsigned int page_shift = mmu_psize_defs[mmu_virtual_psize].shift;
 	unsigned long page_size = 1UL << page_shift;
 	unsigned long nr_pages = (end - start) >> page_shift;
-	bool local, full;
+	bool fullmm = (end == TLB_FLUSH_ALL);
+	bool flush_pid;
+	enum tlb_flush_type type;
 
 	pid = mm->context.id;
 	if (unlikely(pid == MMU_NO_CONTEXT))
@@ -907,24 +934,16 @@ static inline void __radix__flush_tlb_range(struct mm_struct *mm,
 
 	preempt_disable();
 	smp_mb(); /* see radix__flush_tlb_mm */
-	if (!mm_is_thread_local(mm)) {
-		if (unlikely(mm_is_singlethreaded(mm))) {
-			if (end != TLB_FLUSH_ALL) {
-				exit_flush_lazy_tlbs(mm);
-				goto is_local;
-			}
-		}
-		local = false;
-		full = (end == TLB_FLUSH_ALL ||
-				nr_pages > tlb_single_page_flush_ceiling);
-	} else {
-is_local:
-		local = true;
-		full = (end == TLB_FLUSH_ALL ||
-				nr_pages > tlb_local_single_page_flush_ceiling);
-	}
+	type = flush_type_needed(mm, fullmm);
+
+	if (fullmm)
+		flush_pid = true;
+	else if (type == FLUSH_TYPE_GLOBAL)
+		flush_pid = nr_pages > tlb_single_page_flush_ceiling;
+	else
+		flush_pid = nr_pages > tlb_local_single_page_flush_ceiling;
 
-	if (!mmu_has_feature(MMU_FTR_GTSE) && !local) {
+	if (!mmu_has_feature(MMU_FTR_GTSE) && type == FLUSH_TYPE_GLOBAL) {
 		unsigned long tgt = H_RPTI_TARGET_CMMU;
 		unsigned long pg_sizes = psize_to_rpti_pgsize(mmu_virtual_psize);
 
@@ -934,8 +953,8 @@ static inline void __radix__flush_tlb_range(struct mm_struct *mm,
 			tgt |= H_RPTI_TARGET_NMMU;
 		pseries_rpt_invalidate(pid, tgt, H_RPTI_TYPE_TLB, pg_sizes,
 				       start, end);
-	} else if (full) {
-		if (local) {
+	} else if (flush_pid) {
+		if (type == FLUSH_TYPE_LOCAL) {
 			_tlbiel_pid(pid, RIC_FLUSH_TLB);
 		} else {
 			if (cputlb_use_tlbie()) {
@@ -958,7 +977,7 @@ static inline void __radix__flush_tlb_range(struct mm_struct *mm,
 				hflush = true;
 		}
 
-		if (local) {
+		if (type == FLUSH_TYPE_LOCAL) {
 			asm volatile("ptesync": : :"memory");
 			__tlbiel_va_range(start, end, pid, page_size, mmu_virtual_psize);
 			if (hflush)
@@ -1091,32 +1110,28 @@ static __always_inline void __radix__flush_tlb_range_psize(struct mm_struct *mm,
 	unsigned int page_shift = mmu_psize_defs[psize].shift;
 	unsigned long page_size = 1UL << page_shift;
 	unsigned long nr_pages = (end - start) >> page_shift;
-	bool local, full;
+	bool fullmm = (end == TLB_FLUSH_ALL);
+	bool flush_pid;
+	enum tlb_flush_type type;
 
 	pid = mm->context.id;
 	if (unlikely(pid == MMU_NO_CONTEXT))
 		return;
 
+	fullmm = (end == TLB_FLUSH_ALL);
+
 	preempt_disable();
 	smp_mb(); /* see radix__flush_tlb_mm */
-	if (!mm_is_thread_local(mm)) {
-		if (unlikely(mm_is_singlethreaded(mm))) {
-			if (end != TLB_FLUSH_ALL) {
-				exit_flush_lazy_tlbs(mm);
-				goto is_local;
-			}
-		}
-		local = false;
-		full = (end == TLB_FLUSH_ALL ||
-				nr_pages > tlb_single_page_flush_ceiling);
-	} else {
-is_local:
-		local = true;
-		full = (end == TLB_FLUSH_ALL ||
-				nr_pages > tlb_local_single_page_flush_ceiling);
-	}
+	type = flush_type_needed(mm, fullmm);
 
-	if (!mmu_has_feature(MMU_FTR_GTSE) && !local) {
+	if (fullmm)
+		flush_pid = true;
+	else if (type == FLUSH_TYPE_GLOBAL)
+		flush_pid = nr_pages > tlb_single_page_flush_ceiling;
+	else
+		flush_pid = nr_pages > tlb_local_single_page_flush_ceiling;
+
+	if (!mmu_has_feature(MMU_FTR_GTSE) && type == FLUSH_TYPE_GLOBAL) {
 		unsigned long tgt = H_RPTI_TARGET_CMMU;
 		unsigned long type = H_RPTI_TYPE_TLB;
 		unsigned long pg_sizes = psize_to_rpti_pgsize(psize);
@@ -1126,8 +1141,8 @@ static __always_inline void __radix__flush_tlb_range_psize(struct mm_struct *mm,
 		if (atomic_read(&mm->context.copros) > 0)
 			tgt |= H_RPTI_TARGET_NMMU;
 		pseries_rpt_invalidate(pid, tgt, type, pg_sizes, start, end);
-	} else if (full) {
-		if (local) {
+	} else if (flush_pid) {
+		if (type == FLUSH_TYPE_LOCAL) {
 			_tlbiel_pid(pid, also_pwc ? RIC_FLUSH_ALL : RIC_FLUSH_TLB);
 		} else {
 			if (cputlb_use_tlbie()) {
@@ -1143,7 +1158,7 @@ static __always_inline void __radix__flush_tlb_range_psize(struct mm_struct *mm,
 
 		}
 	} else {
-		if (local)
+		if (type == FLUSH_TYPE_LOCAL)
 			_tlbiel_va_range(start, end, pid, page_size, psize, also_pwc);
 		else if (cputlb_use_tlbie())
 			_tlbie_va_range(start, end, pid, page_size, psize, also_pwc);
@@ -1170,6 +1185,7 @@ static void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long
 void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr)
 {
 	unsigned long pid, end;
+	enum tlb_flush_type type;
 
 	pid = mm->context.id;
 	if (unlikely(pid == MMU_NO_CONTEXT))
@@ -1186,11 +1202,8 @@ void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr)
 	/* Otherwise first do the PWC, then iterate the pages. */
 	preempt_disable();
 	smp_mb(); /* see radix__flush_tlb_mm */
-	if (!mm_is_thread_local(mm)) {
-		if (unlikely(mm_is_singlethreaded(mm))) {
-			exit_flush_lazy_tlbs(mm);
-			goto local;
-		}
+	type = flush_type_needed(mm, false);
+	if (type == FLUSH_TYPE_GLOBAL) {
 		if (!mmu_has_feature(MMU_FTR_GTSE)) {
 			unsigned long tgt, type, pg_sizes;
 
@@ -1209,7 +1222,6 @@ void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr)
 			_tlbiel_va_range_multicast(mm,
 					addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true);
 	} else {
-local:
 		_tlbiel_va_range(addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true);
 	}
 
Signature · 1 line
2.23.0
author reviewer
#3 lore raw

PATCH 3/7 powerpc/64s/radix: Check for no TLB flush required

In reply to Nicholas Piggin

If there are no CPUs in mm_cpumask, no TLB flush is required at all. This patch adds a check for this case.

Currently it's not tested for, in fact mm_is_thread_local() returns false if the current CPU is not in mm_cpumask, so it's treated as a global flush.

This can come up in some cases like exec failure before the new mm has ever been switched to. This patch reduces TLBIE instructions required to build a kernel from about 120,000 to 45,000. Another situation it could help is page reclaim, KSM, THP, etc., (i.e., asynch operations external to the process) where the process is sleeping and has all TLBs flushed out of all CPUs.

---
 arch/powerpc/mm/book3s64/radix_tlb.c | 38 ++++++++++++++++++----------
 1 file changed, 25 insertions(+), 13 deletions(-)
arch/powerpc/mm/book3s64/radix_tlb.c +25 -13
index 12481c864ab6..5b62e2e7371c 100644
@@ -694,13 +694,19 @@ static inline void exit_flush_lazy_tlbs(struct mm_struct *mm) { }
 #endif /* CONFIG_SMP */
 
 enum tlb_flush_type {
+	FLUSH_TYPE_NONE,
 	FLUSH_TYPE_LOCAL,
 	FLUSH_TYPE_GLOBAL,
 };
 
 static enum tlb_flush_type flush_type_needed(struct mm_struct *mm, bool fullmm)
 {
-	if (mm_is_thread_local(mm))
+	int active_cpus = atomic_read(&mm->context.active_cpus);
+	int cpu = smp_processor_id();
+
+	if (active_cpus == 0)
+		return FLUSH_TYPE_NONE;
+	if (active_cpus == 1 && cpumask_test_cpu(cpu, mm_cpumask(mm)))
 		return FLUSH_TYPE_LOCAL;
 
 	/* Coprocessors require TLBIE to invalidate nMMU. */
@@ -754,7 +760,9 @@ void radix__flush_tlb_mm(struct mm_struct *mm)
 	 */
 	smp_mb();
 	type = flush_type_needed(mm, false);
-	if (type == FLUSH_TYPE_GLOBAL) {
+	if (type == FLUSH_TYPE_LOCAL) {
+		_tlbiel_pid(pid, RIC_FLUSH_TLB);
+	} else if (type == FLUSH_TYPE_GLOBAL) {
 		if (!mmu_has_feature(MMU_FTR_GTSE)) {
 			unsigned long tgt = H_RPTI_TARGET_CMMU;
 
@@ -770,8 +778,6 @@ void radix__flush_tlb_mm(struct mm_struct *mm)
 		} else {
 			_tlbiel_pid_multicast(mm, pid, RIC_FLUSH_TLB);
 		}
-	} else {
-		_tlbiel_pid(pid, RIC_FLUSH_TLB);
 	}
 	preempt_enable();
 }
@@ -789,7 +795,9 @@ static void __flush_all_mm(struct mm_struct *mm, bool fullmm)
 	preempt_disable();
 	smp_mb(); /* see radix__flush_tlb_mm */
 	type = flush_type_needed(mm, fullmm);
-	if (type == FLUSH_TYPE_GLOBAL) {
+	if (type == FLUSH_TYPE_LOCAL) {
+		_tlbiel_pid(pid, RIC_FLUSH_ALL);
+	} else if (type == FLUSH_TYPE_GLOBAL) {
 		if (!mmu_has_feature(MMU_FTR_GTSE)) {
 			unsigned long tgt = H_RPTI_TARGET_CMMU;
 			unsigned long type = H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC |
@@ -803,8 +811,6 @@ static void __flush_all_mm(struct mm_struct *mm, bool fullmm)
 			_tlbie_pid(pid, RIC_FLUSH_ALL);
 		else
 			_tlbiel_pid_multicast(mm, pid, RIC_FLUSH_ALL);
-	} else {
-		_tlbiel_pid(pid, RIC_FLUSH_ALL);
 	}
 	preempt_enable();
 }
@@ -828,7 +834,9 @@ void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
 	preempt_disable();
 	smp_mb(); /* see radix__flush_tlb_mm */
 	type = flush_type_needed(mm, false);
-	if (type == FLUSH_TYPE_GLOBAL) {
+	if (type == FLUSH_TYPE_LOCAL) {
+		_tlbiel_va(vmaddr, pid, psize, RIC_FLUSH_TLB);
+	} else if (type == FLUSH_TYPE_GLOBAL) {
 		if (!mmu_has_feature(MMU_FTR_GTSE)) {
 			unsigned long tgt, pg_sizes, size;
 
@@ -845,8 +853,6 @@ void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
 			_tlbie_va(vmaddr, pid, psize, RIC_FLUSH_TLB);
 		else
 			_tlbiel_va_multicast(mm, vmaddr, pid, psize, RIC_FLUSH_TLB);
-	} else {
-		_tlbiel_va(vmaddr, pid, psize, RIC_FLUSH_TLB);
 	}
 	preempt_enable();
 }
@@ -935,6 +941,8 @@ static inline void __radix__flush_tlb_range(struct mm_struct *mm,
 	preempt_disable();
 	smp_mb(); /* see radix__flush_tlb_mm */
 	type = flush_type_needed(mm, fullmm);
+	if (type == FLUSH_TYPE_NONE)
+		goto out;
 
 	if (fullmm)
 		flush_pid = true;
@@ -999,6 +1007,7 @@ static inline void __radix__flush_tlb_range(struct mm_struct *mm,
 					hstart, hend, pid, PMD_SIZE, MMU_PAGE_2M, false);
 		}
 	}
+out:
 	preempt_enable();
 }
 
@@ -1123,6 +1132,8 @@ static __always_inline void __radix__flush_tlb_range_psize(struct mm_struct *mm,
 	preempt_disable();
 	smp_mb(); /* see radix__flush_tlb_mm */
 	type = flush_type_needed(mm, fullmm);
+	if (type == FLUSH_TYPE_NONE)
+		goto out;
 
 	if (fullmm)
 		flush_pid = true;
@@ -1166,6 +1177,7 @@ static __always_inline void __radix__flush_tlb_range_psize(struct mm_struct *mm,
 			_tlbiel_va_range_multicast(mm,
 					start, end, pid, page_size, psize, also_pwc);
 	}
+out:
 	preempt_enable();
 }
 
@@ -1203,7 +1215,9 @@ void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr)
 	preempt_disable();
 	smp_mb(); /* see radix__flush_tlb_mm */
 	type = flush_type_needed(mm, false);
-	if (type == FLUSH_TYPE_GLOBAL) {
+	if (type == FLUSH_TYPE_LOCAL) {
+		_tlbiel_va_range(addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true);
+	} else if (type == FLUSH_TYPE_GLOBAL) {
 		if (!mmu_has_feature(MMU_FTR_GTSE)) {
 			unsigned long tgt, type, pg_sizes;
 
@@ -1221,8 +1235,6 @@ void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr)
 		else
 			_tlbiel_va_range_multicast(mm,
 					addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true);
-	} else {
-		_tlbiel_va_range(addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true);
 	}
 
 	preempt_enable();
Signature · 1 line
2.23.0
author reviewer
#4 lore raw

PATCH 4/7 powerpc/64s/radix: Allow mm_cpumask trimming from external sources

In reply to Nicholas Piggin

mm_cpumask trimming is currently restricted to be issued by the current thread of a single-threaded mm. This patch relaxes that and allows the mask to be trimmed from any context.

---
 arch/powerpc/mm/book3s64/radix_tlb.c | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)
arch/powerpc/mm/book3s64/radix_tlb.c +6 -10
index 5b62e2e7371c..7b199bee4baa 100644
@@ -657,20 +657,16 @@ static void do_exit_flush_lazy_tlb(void *arg)
 	}
 
 	/*
-	 * This IPI is only initiated from a CPU which is running mm which
-	 * is a single-threaded process, so there will not be another racing
-	 * IPI coming in where we would find our cpumask already clear.
-	 *
-	 * Nothing else clears our bit in the cpumask except CPU offlining,
-	 * in which case we should not be taking IPIs here. However check
-	 * this just in case the logic is wrong somewhere, and don't underflow
-	 * the active_cpus count.
+	 * This IPI may be initiated from any source including those not
+	 * running the mm, so there may be a racing IPI that comes after
+	 * this one which finds the cpumask already clear. Check and avoid
+	 * underflowing the active_cpus count in that case. The race should
+	 * not otherwise be a problem, but the TLB must be flushed because
+	 * that's what the caller expects.
 	 */
 	if (cpumask_test_cpu(cpu, mm_cpumask(mm))) {
 		atomic_dec(&mm->context.active_cpus);
 		cpumask_clear_cpu(cpu, mm_cpumask(mm));
-	} else {
-		WARN_ON_ONCE(1);
 	}
 
 out_flush:
Signature · 1 line
2.23.0
author reviewer
#5 lore raw

PATCH 5/7 powerpc/64s/radix: occasionally attempt to trim mm_cpumask

In reply to Nicholas Piggin

A single-threaded process that is flushing its own address space is so far the only case where the mm_cpumask is attempted to be trimmed. This patch expands that to flush in other situations, multi-threaded processes and external sources. For now it's a relatively simple occasional trim attempt. The main aim is to add the mechanism, tweaking and tuning can come with more data.

---
 arch/powerpc/mm/book3s64/radix_tlb.c | 60 ++++++++++++++++++++++++++--
 1 file changed, 56 insertions(+), 4 deletions(-)
arch/powerpc/mm/book3s64/radix_tlb.c +56 -4
index 7b199bee4baa..4dca7cbf07e9 100644
@@ -630,10 +630,8 @@ static bool mm_needs_flush_escalation(struct mm_struct *mm)
 	return false;
 }
 
-#ifdef CONFIG_SMP
-static void do_exit_flush_lazy_tlb(void *arg)
+static void exit_lazy_flush_tlb(struct mm_struct *mm)
 {
-	struct mm_struct *mm = arg;
 	unsigned long pid = mm->context.id;
 	int cpu = smp_processor_id();
 
@@ -673,6 +671,13 @@ static void do_exit_flush_lazy_tlb(void *arg)
 	_tlbiel_pid(pid, RIC_FLUSH_ALL);
 }
 
+#ifdef CONFIG_SMP
+static void do_exit_flush_lazy_tlb(void *arg)
+{
+	struct mm_struct *mm = arg;
+	exit_lazy_flush_tlb(mm);
+}
+
 static void exit_flush_lazy_tlbs(struct mm_struct *mm)
 {
 	/*
@@ -685,10 +690,32 @@ static void exit_flush_lazy_tlbs(struct mm_struct *mm)
 	smp_call_function_many(mm_cpumask(mm), do_exit_flush_lazy_tlb,
 				(void *)mm, 1);
 }
+
 #else /* CONFIG_SMP */
 static inline void exit_flush_lazy_tlbs(struct mm_struct *mm) { }
 #endif /* CONFIG_SMP */
 
+static DEFINE_PER_CPU(unsigned int, mm_cpumask_trim_clock);
+
+/*
+ * Interval between flushes at which we send out IPIs to check whether the
+ * mm_cpumask can be trimmed for the case where it's not a single-threaded
+ * process flushing its own mm. The intent is to reduce the cost of later
+ * flushes. Don't want this to be so low that it adds noticable cost to TLB
+ * flushing, or so high that it doesn't help reduce global TLBIEs.
+ */
+static unsigned long tlb_mm_cpumask_trim_timer = 1073;
+
+static bool tick_and_test_trim_clock(void)
+{
+	if (__this_cpu_inc_return(mm_cpumask_trim_clock) ==
+			tlb_mm_cpumask_trim_timer) {
+		__this_cpu_write(mm_cpumask_trim_clock, 0);
+		return true;
+	}
+	return false;
+}
+
 enum tlb_flush_type {
 	FLUSH_TYPE_NONE,
 	FLUSH_TYPE_LOCAL,
@@ -702,8 +729,20 @@ static enum tlb_flush_type flush_type_needed(struct mm_struct *mm, bool fullmm)
 
 	if (active_cpus == 0)
 		return FLUSH_TYPE_NONE;
-	if (active_cpus == 1 && cpumask_test_cpu(cpu, mm_cpumask(mm)))
+	if (active_cpus == 1 && cpumask_test_cpu(cpu, mm_cpumask(mm))) {
+		if (current->mm != mm) {
+			/*
+			 * Asynchronous flush sources may trim down to nothing
+			 * if the process is not running, so occasionally try
+			 * to trim.
+			 */
+			if (tick_and_test_trim_clock()) {
+				exit_lazy_flush_tlb(mm);
+				return FLUSH_TYPE_NONE;
+			}
+		}
 		return FLUSH_TYPE_LOCAL;
+	}
 
 	/* Coprocessors require TLBIE to invalidate nMMU. */
 	if (atomic_read(&mm->context.copros) > 0)
@@ -735,6 +774,19 @@ static enum tlb_flush_type flush_type_needed(struct mm_struct *mm, bool fullmm)
 		return FLUSH_TYPE_LOCAL;
 	}
 
+	/*
+	 * Occasionally try to trim down the cpumask. It's possible this can
+	 * bring the mask to zero, which results in no flush.
+	 */
+	if (tick_and_test_trim_clock()) {
+		exit_flush_lazy_tlbs(mm);
+		if (current->mm == mm)
+			return FLUSH_TYPE_LOCAL;
+		if (cpumask_test_cpu(cpu, mm_cpumask(mm)))
+			exit_lazy_flush_tlb(mm);
+		return FLUSH_TYPE_NONE;
+	}
+
 	return FLUSH_TYPE_GLOBAL;
 }
 
Signature · 1 line
2.23.0
author reviewer
#6 lore raw

PATCH 6/7 powerpc/64s/radix: serialize_against_pte_lookup IPIs trim mm_cpumask

In reply to Nicholas Piggin

serialize_against_pte_lookup() performs IPIs to all CPUs in mm_cpumask. Take this opportunity to try trim the CPU out of mm_cpumask. This can reduce the cost of future serialize_against_pte_lookup() and/or the cost of future TLB flushes.

---
 arch/powerpc/mm/book3s64/pgtable.c   | 13 ++++++++++---
 arch/powerpc/mm/book3s64/radix_tlb.c | 20 +++++++++++++-------
 2 files changed, 23 insertions(+), 10 deletions(-)
arch/powerpc/mm/book3s64/pgtable.c +10 -3
index e18ae50a275c..ec23faf102b2 100644
@@ -79,10 +79,17 @@ void set_pmd_at(struct mm_struct *mm, unsigned long addr,
 	return set_pte_at(mm, addr, pmdp_ptep(pmdp), pmd_pte(pmd));
 }
 
-static void do_nothing(void *unused)
-{
+void exit_lazy_flush_tlb(struct mm_struct *mm, bool always_flush);
 
+static void do_serialize(void *arg)
+{
+	/* We've taken the IPI, so try to trim the mask while here */
+	if (radix_enabled()) {
+		struct mm_struct *mm = arg;
+		exit_lazy_flush_tlb(mm, false);
+	}
 }
+
 /*
  * Serialize against find_current_mm_pte which does lock-less
  * lookup in page tables with local interrupts disabled. For huge pages
@@ -96,7 +103,7 @@ static void do_nothing(void *unused)
 void serialize_against_pte_lookup(struct mm_struct *mm)
 {
 	smp_mb();
-	smp_call_function_many(mm_cpumask(mm), do_nothing, NULL, 1);
+	smp_call_function_many(mm_cpumask(mm), do_serialize, mm, 1);
 }
 
 /*
arch/powerpc/mm/book3s64/radix_tlb.c +13 -7
index 4dca7cbf07e9..d04c80d6f52c 100644
@@ -630,7 +630,11 @@ static bool mm_needs_flush_escalation(struct mm_struct *mm)
 	return false;
 }
 
-static void exit_lazy_flush_tlb(struct mm_struct *mm)
+/*
+ * If always_flush is true, then flush even if this CPU can't be removed
+ * from mm_cpumask.
+ */
+void exit_lazy_flush_tlb(struct mm_struct *mm, bool always_flush)
 {
 	unsigned long pid = mm->context.id;
 	int cpu = smp_processor_id();
@@ -643,7 +647,7 @@ static void exit_lazy_flush_tlb(struct mm_struct *mm)
 	 * done with interrupts off.
 	 */
 	if (current->mm == mm)
-		goto out_flush;
+		goto out;
 
 	if (current->active_mm == mm) {
 		WARN_ON_ONCE(current->mm != NULL);
@@ -665,17 +669,19 @@ static void exit_lazy_flush_tlb(struct mm_struct *mm)
 	if (cpumask_test_cpu(cpu, mm_cpumask(mm))) {
 		atomic_dec(&mm->context.active_cpus);
 		cpumask_clear_cpu(cpu, mm_cpumask(mm));
+		always_flush = true;
 	}
 
-out_flush:
-	_tlbiel_pid(pid, RIC_FLUSH_ALL);
+out:
+	if (always_flush)
+		_tlbiel_pid(pid, RIC_FLUSH_ALL);
 }
 
 #ifdef CONFIG_SMP
 static void do_exit_flush_lazy_tlb(void *arg)
 {
 	struct mm_struct *mm = arg;
-	exit_lazy_flush_tlb(mm);
+	exit_lazy_flush_tlb(mm, true);
 }
 
 static void exit_flush_lazy_tlbs(struct mm_struct *mm)
@@ -737,7 +743,7 @@ static enum tlb_flush_type flush_type_needed(struct mm_struct *mm, bool fullmm)
 			 * to trim.
 			 */
 			if (tick_and_test_trim_clock()) {
-				exit_lazy_flush_tlb(mm);
+				exit_lazy_flush_tlb(mm, true);
 				return FLUSH_TYPE_NONE;
 			}
 		}
@@ -783,7 +789,7 @@ static enum tlb_flush_type flush_type_needed(struct mm_struct *mm, bool fullmm)
 		if (current->mm == mm)
 			return FLUSH_TYPE_LOCAL;
 		if (cpumask_test_cpu(cpu, mm_cpumask(mm)))
-			exit_lazy_flush_tlb(mm);
+			exit_lazy_flush_tlb(mm, true);
 		return FLUSH_TYPE_NONE;
 	}
 
Signature · 1 line
2.23.0
#7 lore raw

Re: [PATCH 6/7] powerpc/64s/radix: serialize_against_pte_lookup IPIs trim mm_cpumask

I love your patch! Yet something to improve:

Hi Nicholas,

I love your patch! Yet something to improve:

[auto build test ERROR on powerpc/next]
[also build test ERROR on v5.10 next-20201217]
[If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]

url:    https://github.com/0day-ci/linux/commits/Nicholas-Piggin/powerpc-64s-TLB-flushing-improvements/20201217-220230
base:   https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git next
config: powerpc-randconfig-m031-20201217 (attached as .config)
compiler: powerpc64le-linux-gcc (GCC) 9.3.0

reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# https://github.com/0day-ci/linux/commit/5bd02bea3da4e21fa00729e7687fc93a5c653e2b
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Nicholas-Piggin/powerpc-64s-TLB-flushing-improvements/20201217-220230 git checkout 5bd02bea3da4e21fa00729e7687fc93a5c653e2b
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=powerpc

If you fix the issue, kindly add following tag as appropriate

All errors (new ones prefixed by >>):

>> arch/powerpc/mm/book3s64/radix_tlb.c:646:6: error: no previous prototype for 'exit_lazy_flush_tlb' [-Werror=missing-prototypes]

646 | void exit_lazy_flush_tlb(struct mm_struct *mm, bool always_flush)
| ^~~~~~~~~~~~~~~~~~~
cc1: all warnings being treated as errors

vim +/exit_lazy_flush_tlb +646 arch/powerpc/mm/book3s64/radix_tlb.c

641
642 /*
643 * If always_flush is true, then flush even if this CPU can't be removed
644 * from mm_cpumask.
645 */

 > 646	void exit_lazy_flush_tlb(struct mm_struct *mm, bool always_flush)

647 {
648 unsigned long pid = mm->context.id;
649 int cpu = smp_processor_id();
650
651 /*
652 * A kthread could have done a mmget_not_zero() after the flushing CPU
653 * checked mm_cpumask, and be in the process of kthread_use_mm when
654 * interrupted here. In that case, current->mm will be set to mm,
655 * because kthread_use_mm() setting ->mm and switching to the mm is
656 * done with interrupts off.
657 */
658 if (current->mm == mm)
659 goto out;
660
661 if (current->active_mm == mm) {
662 WARN_ON_ONCE(current->mm != NULL);
663 /* Is a kernel thread and is using mm as the lazy tlb */
664 mmgrab(&init_mm);
665 current->active_mm = &init_mm;
666 switch_mm_irqs_off(mm, &init_mm, current);
667 mmdrop(mm);
668 }
669
670 /*
671 * This IPI may be initiated from any source including those not
672 * running the mm, so there may be a racing IPI that comes after
673 * this one which finds the cpumask already clear. Check and avoid
674 * underflowing the active_cpus count in that case. The race should
675 * not otherwise be a problem, but the TLB must be flushed because
676 * that's what the caller expects.
677 */
678 if (cpumask_test_cpu(cpu, mm_cpumask(mm))) {
679 atomic_dec(&mm->context.active_cpus);
680 cpumask_clear_cpu(cpu, mm_cpumask(mm));
681 always_flush = true;
682 }
683
684 out:
685 if (always_flush)
686 _tlbiel_pid(pid, RIC_FLUSH_ALL);
687 }
688

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org

author reviewer
#8 lore raw

PATCH 7/7 powerpc/64s: Implement ptep_clear_flush_young that does not flush TLBs

In reply to Nicholas Piggin

Similarly to the x86 commit b13b1d2d8692 ("x86/mm: In the PTE swapout page reclaim case clear the accessed bit instead of flushing the TLB"), implement ptep_clear_flush_young that does not actually flush the TLB in the case the referenced bit is cleared.

---
 arch/powerpc/include/asm/book3s/64/pgtable.h | 23 +++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)
arch/powerpc/include/asm/book3s/64/pgtable.h +20 -3
index cd3feeac6e87..751f98a40aca 100644
@@ -388,11 +388,28 @@ static inline int __ptep_test_and_clear_young(struct mm_struct *mm,
 #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
 #define ptep_test_and_clear_young(__vma, __addr, __ptep)	\
 ({								\
-	int __r;						\
-	__r = __ptep_test_and_clear_young((__vma)->vm_mm, __addr, __ptep); \
-	__r;							\
+	__ptep_test_and_clear_young((__vma)->vm_mm, __addr, __ptep); \
 })
 
+/*
+ * On Book3S CPUs, clearing the accessed bit without a TLB flush
+ * doesn't cause data corruption. [ It could cause incorrect
+ * page aging and the (mistaken) reclaim of hot pages, but the
+ * chance of that should be relatively low. ]
+ *
+ * So as a performance optimization don't flush the TLB when
+ * clearing the accessed bit, it will eventually be flushed by
+ * a context switch or a VM operation anyway. [ In the rare
+ * event of it not getting flushed for a long time the delay
+ * shouldn't really matter because there's no real memory
+ * pressure for swapout to react to. ]
+ */
+#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
+#define ptep_clear_flush_young ptep_test_and_clear_young
+
+#define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
+#define pmdp_clear_flush_young pmdp_test_and_clear_young
+
 static inline int __pte_write(pte_t pte)
 {
 	return !!(pte_raw(pte) & cpu_to_be64(_PAGE_WRITE));
Signature · 1 line
2.23.0
#9 lore raw
In reply to Nicholas Piggin
Quoted 14 lines · Nicholas Piggin

Another round of reducing TLB flushing (mostly on radix).

Thanks,
Nick

Nicholas Piggin (7):
powerpc/64s/radix: add warning and comments in mm_cpumask trim powerpc/64s/radix: refactor TLB flush type selection powerpc/64s/radix: Check for no TLB flush required powerpc/64s/radix: Allow mm_cpumask trimming from external sources powerpc/64s/radix: occasionally attempt to trim mm_cpumask powerpc/64s/radix: serialize_against_pte_lookup IPIs trim mm_cpumask powerpc/64s: Implement ptep_clear_flush_young that does not flush TLBs

Quoted 1 line

[...]

Applied to powerpc/next.

[1/7] powerpc/64s/radix: add warning and comments in mm_cpumask trim

      https://git.kernel.org/powerpc/c/a2496049f1f1006178d0db706a8451dd03bd3ec6
[2/7] powerpc/64s/radix: refactor TLB flush type selection
      https://git.kernel.org/powerpc/c/26418b36a11f2eaf2556aa8cefe86132907e311f
[3/7] powerpc/64s/radix: Check for no TLB flush required
      https://git.kernel.org/powerpc/c/54bb503345b81399575e2b7a3a6497ae212ad827
[4/7] powerpc/64s/radix: Allow mm_cpumask trimming from external sources
      https://git.kernel.org/powerpc/c/780de40601aabeca41bc9aa717a329a77aa85e1a
[5/7] powerpc/64s/radix: occasionally attempt to trim mm_cpumask
      https://git.kernel.org/powerpc/c/9393544842d6c85ebfc387c43a5059f8171d598f
[6/7] powerpc/64s/radix: serialize_against_pte_lookup IPIs trim mm_cpumask
      https://git.kernel.org/powerpc/c/032b7f08932c9b212952d6d585e45b2941b3e8be
[7/7] powerpc/64s: Implement ptep_clear_flush_young that does not flush TLBs
      https://git.kernel.org/powerpc/c/3cb1aa7aa39402f4f2cb847b1f16ade3bce43a97

cheers