Skip to contentSkip to conversation

generic mmu_gather patches

PATCH In review Peter Zijlstra 9 participants 46 messages Feb 19, 2019 – Jul 15, 2020

Subsystems (29)
lore
Share & tools
lore

Peter Zijlstra

author maintainer
#0 lore raw

v6 cover generic mmu_gather patches

Hi all,

Sorry I haven't posted these in a while, I sorta forgot about them for a little.

Not much changed since last time; one change to the ARM patch as suggested by Will and a fresh Changelog for patch 12 as requested by Vineet. And some trivial rebasing of the s390 bits.

They've sat in my queue.git for a while and 0-day hasn't reported anything funny with them.

git://git.kernel.org/pub/scm/linux/kernel/git/peterz/queue.git mm/tlb

I'm thinking this is about ready to go.

---
 arch/Kconfig                      |   8 +-
 arch/alpha/Kconfig                |   1 +
 arch/alpha/include/asm/tlb.h      |   6 -
 arch/arc/include/asm/tlb.h        |  32 -----
 arch/arm/include/asm/tlb.h        | 255 ++-------------------------------
 arch/arm64/Kconfig                |   1 -
 arch/arm64/include/asm/tlb.h      |   1 +
 arch/c6x/Kconfig                  |   1 +
 arch/c6x/include/asm/tlb.h        |   2 -
 arch/h8300/include/asm/tlb.h      |   2 -
 arch/hexagon/include/asm/tlb.h    |  12 --
 arch/ia64/include/asm/tlb.h       | 257 +---------------------------------
 arch/ia64/include/asm/tlbflush.h  |  25 ++++
 arch/ia64/mm/tlb.c                |  23 ++-
 arch/m68k/Kconfig                 |   1 +
 arch/m68k/include/asm/tlb.h       |  14 --
 arch/microblaze/Kconfig           |   1 +
 arch/microblaze/include/asm/tlb.h |   9 --
 arch/mips/include/asm/tlb.h       |  17 ---
 arch/nds32/include/asm/tlb.h      |  16 ---
 arch/nios2/Kconfig                |   1 +
 arch/nios2/include/asm/tlb.h      |  14 +-
 arch/openrisc/Kconfig             |   1 +
 arch/openrisc/include/asm/tlb.h   |   8 +-
 arch/parisc/include/asm/tlb.h     |  18 ---
 arch/powerpc/Kconfig              |   2 +
 arch/powerpc/include/asm/tlb.h    |  18 +--
 arch/riscv/include/asm/tlb.h      |   1 +
 arch/s390/Kconfig                 |   2 +
 arch/s390/include/asm/tlb.h       | 130 ++++++-----------
 arch/s390/mm/pgalloc.c            |  63 +--------
 arch/sh/include/asm/pgalloc.h     |   9 ++
 arch/sh/include/asm/tlb.h         | 132 +----------------
 arch/sparc/Kconfig                |   1 +
 arch/sparc/include/asm/tlb_32.h   |  18 ---
 arch/um/include/asm/tlb.h         | 158 +--------------------
 arch/unicore32/Kconfig            |   1 +
 arch/unicore32/include/asm/tlb.h  |   7 +-
 arch/x86/Kconfig                  |   1 -
 arch/x86/include/asm/tlb.h        |   1 +
 arch/xtensa/include/asm/tlb.h     |  26 ----
 include/asm-generic/tlb.h         | 288 ++++++++++++++++++++++++++++++++++----
 mm/huge_memory.c                  |   4 +-
 mm/hugetlb.c                      |   2 +-
 mm/madvise.c                      |   2 +-
 mm/memory.c                       |   6 +-
 mm/mmu_gather.c                   | 129 +++++++++--------
 47 files changed, 477 insertions(+), 1250 deletions(-)
author maintainer
#1 lore raw

PATCH v6 1/18 asm-generic/tlb: Provide a comment

Write a comment explaining some of this..

---
 include/asm-generic/tlb.h |  119 ++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 116 insertions(+), 3 deletions(-)
include/asm-generic/tlb.h +116 -3
@@ -22,6 +22,118 @@
 
 #ifdef CONFIG_MMU
 
+/*
+ * Generic MMU-gather implementation.
+ *
+ * The mmu_gather data structure is used by the mm code to implement the
+ * correct and efficient ordering of freeing pages and TLB invalidations.
+ *
+ * This correct ordering is:
+ *
+ *  1) unhook page
+ *  2) TLB invalidate page
+ *  3) free page
+ *
+ * That is, we must never free a page before we have ensured there are no live
+ * translations left to it. Otherwise it might be possible to observe (or
+ * worse, change) the page content after it has been reused.
+ *
+ * The mmu_gather API consists of:
+ *
+ *  - tlb_gather_mmu() / tlb_finish_mmu(); start and finish a mmu_gather
+ *
+ *    Finish in particular will issue a (final) TLB invalidate and free
+ *    all (remaining) queued pages.
+ *
+ *  - tlb_start_vma() / tlb_end_vma(); marks the start / end of a VMA
+ *
+ *    Defaults to flushing at tlb_end_vma() to reset the range; helps when
+ *    there's large holes between the VMAs.
+ *
+ *  - tlb_remove_page() / __tlb_remove_page()
+ *  - tlb_remove_page_size() / __tlb_remove_page_size()
+ *
+ *    __tlb_remove_page_size() is the basic primitive that queues a page for
+ *    freeing. __tlb_remove_page() assumes PAGE_SIZE. Both will return a
+ *    boolean indicating if the queue is (now) full and a call to
+ *    tlb_flush_mmu() is required.
+ *
+ *    tlb_remove_page() and tlb_remove_page_size() imply the call to
+ *    tlb_flush_mmu() when required and has no return value.
+ *
+ *  - tlb_remove_check_page_size_change()
+ *
+ *    call before __tlb_remove_page*() to set the current page-size; implies a
+ *    possible tlb_flush_mmu() call.
+ *
+ *  - tlb_flush_mmu() / tlb_flush_mmu_tlbonly() / tlb_flush_mmu_free()
+ *
+ *    tlb_flush_mmu_tlbonly() - does the TLB invalidate (and resets
+ *                              related state, like the range)
+ *
+ *    tlb_flush_mmu_free() - frees the queued pages; make absolutely
+ *			     sure no additional tlb_remove_page()
+ *			     calls happen between _tlbonly() and this.
+ *
+ *    tlb_flush_mmu() - the above two calls.
+ *
+ *  - mmu_gather::fullmm
+ *
+ *    A flag set by tlb_gather_mmu() to indicate we're going to free
+ *    the entire mm; this allows a number of optimizations.
+ *
+ *    - We can ignore tlb_{start,end}_vma(); because we don't
+ *      care about ranges. Everything will be shot down.
+ *
+ *    - (RISC) architectures that use ASIDs can cycle to a new ASID
+ *      and delay the invalidation until ASID space runs out.
+ *
+ *  - mmu_gather::need_flush_all
+ *
+ *    A flag that can be set by the arch code if it wants to force
+ *    flush the entire TLB irrespective of the range. For instance
+ *    x86-PAE needs this when changing top-level entries.
+ *
+ * And requires the architecture to provide and implement tlb_flush().
+ *
+ * tlb_flush() may, in addition to the above mentioned mmu_gather fields, make
+ * use of:
+ *
+ *  - mmu_gather::start / mmu_gather::end
+ *
+ *    which provides the range that needs to be flushed to cover the pages to
+ *    be freed.
+ *
+ *  - mmu_gather::freed_tables
+ *
+ *    set when we freed page table pages
+ *
+ *  - tlb_get_unmap_shift() / tlb_get_unmap_size()
+ *
+ *    returns the smallest TLB entry size unmapped in this range
+ *
+ * Additionally there are a few opt-in features:
+ *
+ *  HAVE_RCU_TABLE_FREE
+ *
+ *  This provides tlb_remove_table(), to be used instead of tlb_remove_page()
+ *  for page directores (__p*_free_tlb()). This provides separate freeing of
+ *  the page-table pages themselves in a semi-RCU fashion (see comment below).
+ *  Useful if your architecture doesn't use IPIs for remote TLB invalidates
+ *  and therefore doesn't naturally serialize with software page-table walkers.
+ *
+ *  When used, an architecture is expected to provide __tlb_remove_table()
+ *  which does the actual freeing of these pages.
+ *
+ *  HAVE_RCU_TABLE_INVALIDATE
+ *
+ *  This makes HAVE_RCU_TABLE_FREE call tlb_flush_mmu_tlbonly() before freeing
+ *  the page-table pages. Required if you use HAVE_RCU_TABLE_FREE and your
+ *  architecture uses the Linux page-tables natively.
+ *
+ */
+#define HAVE_GENERIC_MMU_GATHER
+
 #ifdef CONFIG_HAVE_RCU_TABLE_FREE
 /*
  * Semi RCU freeing of the page directories.
@@ -89,14 +201,17 @@ struct mmu_gather_batch {
  */
 #define MAX_GATHER_BATCH_COUNT	(10000UL/MAX_GATHER_BATCH)
 
-/* struct mmu_gather is an opaque type used by the mm code for passing around
+/*
+ * struct mmu_gather is an opaque type used by the mm code for passing around
  * any data needed by arch specific code for tlb_remove_page.
  */
 struct mmu_gather {
 	struct mm_struct	*mm;
+
 #ifdef CONFIG_HAVE_RCU_TABLE_FREE
 	struct mmu_table_batch	*batch;
 #endif
+
 	unsigned long		start;
 	unsigned long		end;
 	/*
@@ -131,8 +246,6 @@ struct mmu_gather {
 	int page_size;
 };
 
-#define HAVE_GENERIC_MMU_GATHER
-
 void arch_tlb_gather_mmu(struct mmu_gather *tlb,
 	struct mm_struct *mm, unsigned long start, unsigned long end);
 void tlb_flush_mmu(struct mmu_gather *tlb);
author maintainer
#2 lore raw

PATCH v6 2/18 asm-generic/tlb: Provide HAVE_MMU_GATHER_PAGE_SIZE

In reply to Peter Zijlstra

Move the mmu_gather::page_size things into the generic code instead of powerpc specific bits.

---
 arch/Kconfig                   |    3 +++
 arch/arm/include/asm/tlb.h     |    3 +--
 arch/ia64/include/asm/tlb.h    |    3 +--
 arch/powerpc/Kconfig           |    1 +
 arch/powerpc/include/asm/tlb.h |   17 -----------------
 arch/s390/include/asm/tlb.h    |    4 +---
 arch/sh/include/asm/tlb.h      |    4 +---
 arch/um/include/asm/tlb.h      |    4 +---
 include/asm-generic/tlb.h      |   32 +++++++++++++++++++-------------
 mm/huge_memory.c               |    4 ++--
 mm/hugetlb.c                   |    2 +-
 mm/madvise.c                   |    2 +-
 mm/memory.c                    |    4 ++--
 mm/mmu_gather.c                |    5 +++++
 14 files changed, 39 insertions(+), 49 deletions(-)
arch/Kconfig +3 -0
@@ -368,6 +368,9 @@ config HAVE_RCU_TABLE_FREE
 config HAVE_RCU_TABLE_INVALIDATE
 	bool
 
+config HAVE_MMU_GATHER_PAGE_SIZE
+	bool
+
 config ARCH_HAVE_NMI_SAFE_CMPXCHG
 	bool
 
arch/arm/include/asm/tlb.h +1 -2
@@ -286,8 +286,7 @@ tlb_remove_pmd_tlb_entry(struct mmu_gath
 
 #define tlb_migrate_finish(mm)		do { } while (0)
 
-#define tlb_remove_check_page_size_change tlb_remove_check_page_size_change
-static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb,
+static inline void tlb_change_page_size(struct mmu_gather *tlb,
 						     unsigned int page_size)
 {
 }
arch/ia64/include/asm/tlb.h +1 -2
@@ -282,8 +282,7 @@ do {							\
 #define tlb_remove_huge_tlb_entry(h, tlb, ptep, address)	\
 	tlb_remove_tlb_entry(tlb, ptep, address)
 
-#define tlb_remove_check_page_size_change tlb_remove_check_page_size_change
-static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb,
+static inline void tlb_change_page_size(struct mmu_gather *tlb,
 						     unsigned int page_size)
 {
 }
arch/powerpc/Kconfig +1 -0
@@ -216,6 +216,7 @@ config PPC
 	select HAVE_PERF_REGS
 	select HAVE_PERF_USER_STACK_DUMP
 	select HAVE_RCU_TABLE_FREE		if SMP
+	select HAVE_MMU_GATHER_PAGE_SIZE
 	select HAVE_REGS_AND_STACK_ACCESS_API
 	select HAVE_RELIABLE_STACKTRACE		if PPC64 && CPU_LITTLE_ENDIAN
 	select HAVE_SYSCALL_TRACEPOINTS
arch/powerpc/include/asm/tlb.h +0 -17
@@ -27,7 +27,6 @@
 #define tlb_start_vma(tlb, vma)	do { } while (0)
 #define tlb_end_vma(tlb, vma)	do { } while (0)
 #define __tlb_remove_tlb_entry	__tlb_remove_tlb_entry
-#define tlb_remove_check_page_size_change tlb_remove_check_page_size_change
 
 extern void tlb_flush(struct mmu_gather *tlb);
 
@@ -46,22 +45,6 @@ static inline void __tlb_remove_tlb_entr
 #endif
 }
 
-static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb,
-						     unsigned int page_size)
-{
-	if (!tlb->page_size)
-		tlb->page_size = page_size;
-	else if (tlb->page_size != page_size) {
-		if (!tlb->fullmm)
-			tlb_flush_mmu(tlb);
-		/*
-		 * update the page size after flush for the new
-		 * mmu_gather.
-		 */
-		tlb->page_size = page_size;
-	}
-}
-
 #ifdef CONFIG_SMP
 static inline int mm_is_core_local(struct mm_struct *mm)
 {
arch/s390/include/asm/tlb.h +1 -3
@@ -180,9 +180,7 @@ static inline void pud_free_tlb(struct m
 #define tlb_remove_huge_tlb_entry(h, tlb, ptep, address)	\
 	tlb_remove_tlb_entry(tlb, ptep, address)
 
-#define tlb_remove_check_page_size_change tlb_remove_check_page_size_change
-static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb,
-						     unsigned int page_size)
+static inline void tlb_change_page_size(struct mmu_gather *tlb, unsigned int page_size)
 {
 }
 
arch/sh/include/asm/tlb.h +1 -3
@@ -127,9 +127,7 @@ static inline void tlb_remove_page_size(
 	return tlb_remove_page(tlb, page);
 }
 
-#define tlb_remove_check_page_size_change tlb_remove_check_page_size_change
-static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb,
-						     unsigned int page_size)
+static inline void tlb_change_page_size(struct mmu_gather *tlb, unsigned int page_size)
 {
 }
 
arch/um/include/asm/tlb.h +1 -3
@@ -146,9 +146,7 @@ static inline void tlb_remove_page_size(
 #define tlb_remove_huge_tlb_entry(h, tlb, ptep, address)	\
 	tlb_remove_tlb_entry(tlb, ptep, address)
 
-#define tlb_remove_check_page_size_change tlb_remove_check_page_size_change
-static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb,
-						     unsigned int page_size)
+static inline void tlb_change_page_size(struct mmu_gather *tlb, unsigned int page_size)
 {
 }
 
include/asm-generic/tlb.h +19 -13
@@ -61,7 +61,7 @@
  *    tlb_remove_page() and tlb_remove_page_size() imply the call to
  *    tlb_flush_mmu() when required and has no return value.
  *
- *  - tlb_remove_check_page_size_change()
+ *  - tlb_change_page_size()
  *
  *    call before __tlb_remove_page*() to set the current page-size; implies a
  *    possible tlb_flush_mmu() call.
@@ -114,6 +114,11 @@
  *
  * Additionally there are a few opt-in features:
  *
+ *  HAVE_MMU_GATHER_PAGE_SIZE
+ *
+ *  This ensures we call tlb_flush() every time tlb_change_page_size() actually
+ *  changes the size and provides mmu_gather::page_size to tlb_flush().
+ *
  *  HAVE_RCU_TABLE_FREE
  *
  *  This provides tlb_remove_table(), to be used instead of tlb_remove_page()
@@ -239,11 +244,15 @@ struct mmu_gather {
 	unsigned int		cleared_puds : 1;
 	unsigned int		cleared_p4ds : 1;
 
+	unsigned int		batch_count;
+
 	struct mmu_gather_batch *active;
 	struct mmu_gather_batch	local;
 	struct page		*__pages[MMU_GATHER_BUNDLE];
-	unsigned int		batch_count;
-	int page_size;
+
+#ifdef CONFIG_HAVE_MMU_GATHER_PAGE_SIZE
+	unsigned int page_size;
+#endif
 };
 
 void arch_tlb_gather_mmu(struct mmu_gather *tlb,
@@ -309,21 +318,18 @@ static inline void tlb_remove_page(struc
 	return tlb_remove_page_size(tlb, page, PAGE_SIZE);
 }
 
-#ifndef tlb_remove_check_page_size_change
-#define tlb_remove_check_page_size_change tlb_remove_check_page_size_change
-static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb,
+static inline void tlb_change_page_size(struct mmu_gather *tlb,
 						     unsigned int page_size)
 {
-	/*
-	 * We don't care about page size change, just update
-	 * mmu_gather page size here so that debug checks
-	 * doesn't throw false warning.
-	 */
-#ifdef CONFIG_DEBUG_VM
+#ifdef CONFIG_HAVE_MMU_GATHER_PAGE_SIZE
+	if (tlb->page_size && tlb->page_size != page_size) {
+		if (!tlb->fullmm)
+			tlb_flush_mmu(tlb);
+	}
+
 	tlb->page_size = page_size;
 #endif
 }
-#endif
 
 static inline unsigned long tlb_get_unmap_shift(struct mmu_gather *tlb)
 {
mm/huge_memory.c +2 -2
@@ -1617,7 +1617,7 @@ bool madvise_free_huge_pmd(struct mmu_ga
 	struct mm_struct *mm = tlb->mm;
 	bool ret = false;
 
-	tlb_remove_check_page_size_change(tlb, HPAGE_PMD_SIZE);
+	tlb_change_page_size(tlb, HPAGE_PMD_SIZE);
 
 	ptl = pmd_trans_huge_lock(pmd, vma);
 	if (!ptl)
@@ -1693,7 +1693,7 @@ int zap_huge_pmd(struct mmu_gather *tlb,
 	pmd_t orig_pmd;
 	spinlock_t *ptl;
 
-	tlb_remove_check_page_size_change(tlb, HPAGE_PMD_SIZE);
+	tlb_change_page_size(tlb, HPAGE_PMD_SIZE);
 
 	ptl = __pmd_trans_huge_lock(pmd, vma);
 	if (!ptl)
mm/hugetlb.c +1 -1
@@ -3337,7 +3337,7 @@ void __unmap_hugepage_range(struct mmu_g
 	 * This is a hugetlb vma, all the pte entries should point
 	 * to huge page.
 	 */
-	tlb_remove_check_page_size_change(tlb, sz);
+	tlb_change_page_size(tlb, sz);
 	tlb_start_vma(tlb, vma);
 
 	/*
mm/madvise.c +1 -1
@@ -328,7 +328,7 @@ static int madvise_free_pte_range(pmd_t
 	if (pmd_trans_unstable(pmd))
 		return 0;
 
-	tlb_remove_check_page_size_change(tlb, PAGE_SIZE);
+	tlb_change_page_size(tlb, PAGE_SIZE);
 	orig_pte = pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
 	flush_tlb_batched_pending(mm);
 	arch_enter_lazy_mmu_mode();
mm/memory.c +2 -2
@@ -355,7 +355,7 @@ void free_pgd_range(struct mmu_gather *t
 	 * We add page table cache pages with PAGE_SIZE,
 	 * (see pte_free_tlb()), flush the tlb if we need
 	 */
-	tlb_remove_check_page_size_change(tlb, PAGE_SIZE);
+	tlb_change_page_size(tlb, PAGE_SIZE);
 	pgd = pgd_offset(tlb->mm, addr);
 	do {
 		next = pgd_addr_end(addr, end);
@@ -1046,7 +1046,7 @@ static unsigned long zap_pte_range(struc
 	pte_t *pte;
 	swp_entry_t entry;
 
-	tlb_remove_check_page_size_change(tlb, PAGE_SIZE);
+	tlb_change_page_size(tlb, PAGE_SIZE);
 again:
 	init_rss_vec(rss);
 	start_pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
mm/mmu_gather.c +5 -0
@@ -58,7 +58,9 @@ void arch_tlb_gather_mmu(struct mmu_gath
 #ifdef CONFIG_HAVE_RCU_TABLE_FREE
 	tlb->batch = NULL;
 #endif
+#ifdef CONFIG_HAVE_MMU_GATHER_PAGE_SIZE
 	tlb->page_size = 0;
+#endif
 
 	__tlb_reset_range(tlb);
 }
@@ -121,7 +123,10 @@ bool __tlb_remove_page_size(struct mmu_g
 	struct mmu_gather_batch *batch;
 
 	VM_BUG_ON(!tlb->end);
+
+#ifdef CONFIG_HAVE_MMU_GATHER_PAGE_SIZE
 	VM_WARN_ON(tlb->page_size != page_size);
+#endif
 
 	batch = tlb->active;
 	/*
author maintainer
#3 lore raw

PATCH v6 3/18 asm-generic/tlb: Provide generic VIPT cache flush

In reply to Peter Zijlstra

The one obvious thing SH and ARM want is a sensible default for tlb_start_vma(). (also: https://lkml.org/lkml/2004/1/15/6 )

Avoid all VIPT architectures providing their own tlb_start_vma() implementation and rely on architectures to provide a no-op flush_cache_range() when it is not relevant.

The below makes tlb_start_vma() default to flush_cache_range(), which should be right and sufficient. The only exceptions that I found where (oddly):

- m68k-mmu
- sparc64
- unicore

Those architectures appear to have flush_cache_range(), but their current tlb_start_vma() does not call it.

Cc "Aneesh Kumar K.V" <[email protected]>
Cc Andrew Morton <[email protected]>
Cc Nick Piggin <[email protected]>
Cc David Miller <[email protected]>
Cc Guan Xuetao <[email protected]>
---
 arch/arc/include/asm/tlb.h      |    9 ---------
 arch/mips/include/asm/tlb.h     |    9 ---------
 arch/nds32/include/asm/tlb.h    |    6 ------
 arch/nios2/include/asm/tlb.h    |   10 ----------
 arch/parisc/include/asm/tlb.h   |    5 -----
 arch/sparc/include/asm/tlb_32.h |    5 -----
 arch/xtensa/include/asm/tlb.h   |    9 ---------
 include/asm-generic/tlb.h       |   19 +++++++++++--------
 8 files changed, 11 insertions(+), 61 deletions(-)
arch/arc/include/asm/tlb.h +0 -9
@@ -23,15 +23,6 @@ do {						\
  *
  * Note, read http://lkml.org/lkml/2004/1/15/6
  */
-#ifndef CONFIG_ARC_CACHE_VIPT_ALIASING
-#define tlb_start_vma(tlb, vma)
-#else
-#define tlb_start_vma(tlb, vma)						\
-do {									\
-	if (!tlb->fullmm)						\
-		flush_cache_range(vma, vma->vm_start, vma->vm_end);	\
-} while(0)
-#endif
 
 #define tlb_end_vma(tlb, vma)						\
 do {									\
arch/mips/include/asm/tlb.h +0 -9
@@ -5,15 +5,6 @@
 #include <asm/cpu-features.h>
 #include <asm/mipsregs.h>
 
-/*
- * MIPS doesn't need any special per-pte or per-vma handling, except
- * we need to flush cache for area to be unmapped.
- */
-#define tlb_start_vma(tlb, vma)					\
-	do {							\
-		if (!tlb->fullmm)				\
-			flush_cache_range(vma, vma->vm_start, vma->vm_end); \
-	}  while (0)
 #define tlb_end_vma(tlb, vma) do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
 
arch/nds32/include/asm/tlb.h +0 -6
@@ -4,12 +4,6 @@
 #ifndef __ASMNDS32_TLB_H
 #define __ASMNDS32_TLB_H
 
-#define tlb_start_vma(tlb,vma)						\
-	do {								\
-		if (!tlb->fullmm)					\
-			flush_cache_range(vma, vma->vm_start, vma->vm_end); \
-	} while (0)
-
 #define tlb_end_vma(tlb,vma)				\
 	do { 						\
 		if(!tlb->fullmm)			\
arch/nios2/include/asm/tlb.h +0 -10
@@ -15,16 +15,6 @@
 
 extern void set_mmu_pid(unsigned long pid);
 
-/*
- * NiosII doesn't need any special per-pte or per-vma handling, except
- * we need to flush cache for the area to be unmapped.
- */
-#define tlb_start_vma(tlb, vma)					\
-	do {							\
-		if (!tlb->fullmm)				\
-			flush_cache_range(vma, vma->vm_start, vma->vm_end); \
-	}  while (0)
-
 #define tlb_end_vma(tlb, vma)	do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address)	do { } while (0)
 
arch/parisc/include/asm/tlb.h +0 -5
@@ -7,11 +7,6 @@ do {	if ((tlb)->fullmm)		\
 		flush_tlb_mm((tlb)->mm);\
 } while (0)
 
-#define tlb_start_vma(tlb, vma) \
-do {	if (!(tlb)->fullmm)	\
-		flush_cache_range(vma, vma->vm_start, vma->vm_end); \
-} while (0)
-
 #define tlb_end_vma(tlb, vma)	\
 do {	if (!(tlb)->fullmm)	\
 		flush_tlb_range(vma, vma->vm_start, vma->vm_end); \
arch/sparc/include/asm/tlb_32.h +0 -5
@@ -2,11 +2,6 @@
 #ifndef _SPARC_TLB_H
 #define _SPARC_TLB_H
 
-#define tlb_start_vma(tlb, vma) \
-do {								\
-	flush_cache_range(vma, vma->vm_start, vma->vm_end);	\
-} while (0)
-
 #define tlb_end_vma(tlb, vma) \
 do {								\
 	flush_tlb_range(vma, vma->vm_start, vma->vm_end);	\
arch/xtensa/include/asm/tlb.h +0 -9
@@ -16,19 +16,10 @@
 
 #if (DCACHE_WAY_SIZE <= PAGE_SIZE)
 
-/* Note, read http://lkml.org/lkml/2004/1/15/6 */
-
-# define tlb_start_vma(tlb,vma)			do { } while (0)
 # define tlb_end_vma(tlb,vma)			do { } while (0)
 
 #else
 
-# define tlb_start_vma(tlb, vma)					      \
-	do {								      \
-		if (!tlb->fullmm)					      \
-			flush_cache_range(vma, vma->vm_start, vma->vm_end);   \
-	} while(0)
-
 # define tlb_end_vma(tlb, vma)						      \
 	do {								      \
 		if (!tlb->fullmm)					      \
include/asm-generic/tlb.h +11 -8
@@ -19,6 +19,7 @@
 #include <linux/swap.h>
 #include <asm/pgalloc.h>
 #include <asm/tlbflush.h>
+#include <asm/cacheflush.h>
 
 #ifdef CONFIG_MMU
 
@@ -351,17 +352,19 @@ static inline unsigned long tlb_get_unma
  * the vmas are adjusted to only cover the region to be torn down.
  */
 #ifndef tlb_start_vma
-#define tlb_start_vma(tlb, vma) do { } while (0)
+#define tlb_start_vma(tlb, vma)						\
+do {									\
+	if (!tlb->fullmm)						\
+		flush_cache_range(vma, vma->vm_start, vma->vm_end);	\
+} while (0)
 #endif
 
-#define __tlb_end_vma(tlb, vma)					\
-	do {							\
-		if (!tlb->fullmm)				\
-			tlb_flush_mmu_tlbonly(tlb);		\
-	} while (0)
-
 #ifndef tlb_end_vma
-#define tlb_end_vma	__tlb_end_vma
+#define tlb_end_vma(tlb, vma)						\
+do {									\
+	if (!tlb->fullmm)						\
+		tlb_flush_mmu_tlbonly(tlb);				\
+} while (0)
 #endif
 
 #ifndef __tlb_remove_tlb_entry
author maintainer
#4 lore raw

PATCH v6 4/18 asm-generic/tlb: Provide generic tlb_flush() based on flush_tlb_range()

In reply to Peter Zijlstra

Provide a generic tlb_flush() implementation that relies on flush_tlb_range(). This is a little awkward because flush_tlb_range() assumes a VMA for range invalidation, but we no longer have one.

Audit of all flush_tlb_range() implementations shows only vma->vm_mm and vma->vm_flags are used, and of the latter only VM_EXEC (I-TLB invalidates) and VM_HUGETLB (large TLB invalidate) are used.

Therefore, track VM_EXEC and VM_HUGETLB in two more bits, and create a 'fake' VMA.

This allows architectures that have a reasonably efficient flush_tlb_range() to not require any additional effort.

---
 arch/arm64/include/asm/tlb.h   |    1 
 arch/powerpc/include/asm/tlb.h |    1 
 arch/riscv/include/asm/tlb.h   |    1 
 arch/x86/include/asm/tlb.h     |    1 
 include/asm-generic/tlb.h      |   95 +++++++++++++++++++++++++++++++++++------
 5 files changed, 87 insertions(+), 12 deletions(-)
arch/arm64/include/asm/tlb.h +1 -0
@@ -27,6 +27,7 @@ static inline void __tlb_remove_table(vo
 	free_page_and_swap_cache((struct page *)_table);
 }
 
+#define tlb_flush tlb_flush
 static void tlb_flush(struct mmu_gather *tlb);
 
 #include <asm-generic/tlb.h>
arch/powerpc/include/asm/tlb.h +1 -0
@@ -28,6 +28,7 @@
 #define tlb_end_vma(tlb, vma)	do { } while (0)
 #define __tlb_remove_tlb_entry	__tlb_remove_tlb_entry
 
+#define tlb_flush tlb_flush
 extern void tlb_flush(struct mmu_gather *tlb);
 
 /* Get the generic bits... */
arch/riscv/include/asm/tlb.h +1 -0
@@ -18,6 +18,7 @@ struct mmu_gather;
 
 static void tlb_flush(struct mmu_gather *tlb);
 
+#define tlb_flush tlb_flush
 #include <asm-generic/tlb.h>
 
 static inline void tlb_flush(struct mmu_gather *tlb)
arch/x86/include/asm/tlb.h +1 -0
@@ -6,6 +6,7 @@
 #define tlb_end_vma(tlb, vma) do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
 
+#define tlb_flush tlb_flush
 static inline void tlb_flush(struct mmu_gather *tlb);
 
 #include <asm-generic/tlb.h>
include/asm-generic/tlb.h +83 -12

130 lines not on this page · raw message

author maintainer
#5 lore raw

PATCH v6 5/18 asm-generic/tlb: Provide generic tlb_flush() based on flush_tlb_mm()

In reply to Peter Zijlstra

When an architecture does not have (an efficient) flush_tlb_range(), but instead always uses full TLB invalidates, the current generic tlb_flush() is sub-optimal, for it will generate extra flushes in order to keep the range small.

But if we cannot do range flushes, that is a moot concern. Optionally provide this simplified default.

---
 include/asm-generic/tlb.h |   41 ++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 40 insertions(+), 1 deletion(-)
include/asm-generic/tlb.h +40 -1
@@ -114,7 +114,8 @@
  *    returns the smallest TLB entry size unmapped in this range.
  *
  * If an architecture does not provide tlb_flush() a default implementation
- * based on flush_tlb_range() will be used.
+ * based on flush_tlb_range() will be used, unless MMU_GATHER_NO_RANGE is
+ * specified, in which case we'll default to flush_tlb_mm().
  *
  * Additionally there are a few opt-in features:
  *
@@ -140,6 +141,9 @@
  *  the page-table pages. Required if you use HAVE_RCU_TABLE_FREE and your
  *  architecture uses the Linux page-tables natively.
  *
+ *  MMU_GATHER_NO_RANGE
+ *
+ *  Use this if your architecture lacks an efficient flush_tlb_range().
  */
 #define HAVE_GENERIC_MMU_GATHER
 
@@ -302,12 +306,45 @@ static inline void __tlb_reset_range(str
 	 */
 }
 
+#ifdef CONFIG_MMU_GATHER_NO_RANGE
+
+#if defined(tlb_flush) || defined(tlb_start_vma) || defined(tlb_end_vma)
+#error MMU_GATHER_NO_RANGE relies on default tlb_flush(), tlb_start_vma() and tlb_end_vma()
+#endif
+
+/*
+ * When an architecture does not have efficient means of range flushing TLBs
+ * there is no point in doing intermediate flushes on tlb_end_vma() to keep the
+ * range small. We equally don't have to worry about page granularity or other
+ * things.
+ *
+ * All we need to do is issue a full flush for any !0 range.
+ */
+static inline void tlb_flush(struct mmu_gather *tlb)
+{
+	if (tlb->end)
+		flush_tlb_mm(tlb->mm);
+}
+
+static inline void
+tlb_update_vma_flags(struct mmu_gather *tlb, struct vm_area_struct *vma) { }
+
+#define tlb_end_vma tlb_end_vma
+static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma) { }
+
+#else /* CONFIG_MMU_GATHER_NO_RANGE */
+
 #ifndef tlb_flush
 
 #if defined(tlb_start_vma) || defined(tlb_end_vma)
 #error Default tlb_flush() relies on default tlb_start_vma() and tlb_end_vma()
 #endif
 
+/*
+ * When an architecture does not provide its own tlb_flush() implementation
+ * but does have a reasonably efficient flush_vma_range() implementation
+ * use that.
+ */
 static inline void tlb_flush(struct mmu_gather *tlb)
 {
 	if (tlb->fullmm || tlb->need_flush_all) {
@@ -348,6 +385,8 @@ tlb_update_vma_flags(struct mmu_gather *
 
 #endif
 
+#endif /* CONFIG_MMU_GATHER_NO_RANGE */
+
 static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
 {
 	if (!tlb->end)
Acked-by on v6 5/18
#6 lore raw

Re: [PATCH v6 05/18] asm-generic/tlb: Provide generic tlb_flush() based on flush_tlb_mm()

Quoted 48 lines · Peter Zijlstra

When an architecture does not have (an efficient) flush_tlb_range(), but instead always uses full TLB invalidates, the current generic tlb_flush() is sub-optimal, for it will generate extra flushes in order to keep the range small.

But if we cannot do range flushes, that is a moot concern. Optionally provide this simplified default.

---
 include/asm-generic/tlb.h |   41 ++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 40 insertions(+), 1 deletion(-)
include/asm-generic/tlb.h +14 -1
@@ -114,7 +114,8 @@
  *    returns the smallest TLB entry size unmapped in this range.
  *
  * If an architecture does not provide tlb_flush() a default implementation
- * based on flush_tlb_range() will be used.
+ * based on flush_tlb_range() will be used, unless MMU_GATHER_NO_RANGE is
+ * specified, in which case we'll default to flush_tlb_mm().
  *
  * Additionally there are a few opt-in features:
  *
@@ -140,6 +141,9 @@
  *  the page-table pages. Required if you use HAVE_RCU_TABLE_FREE and your
  *  architecture uses the Linux page-tables natively.
  *
+ *  MMU_GATHER_NO_RANGE
+ *
+ *  Use this if your architecture lacks an efficient flush_tlb_range().
  */
 #define HAVE_GENERIC_MMU_GATHER
 
@@ -302,12 +306,45 @@ static inline void __tlb_reset_range(str
 	 */
 }
 
+#ifdef CONFIG_MMU_GATHER_NO_RANGE
+
+#if defined(tlb_flush) || defined(tlb_start_vma) || defined(tlb_end_vma)
+#error MMU_GATHER_NO_RANGE relies on default tlb_flush(), tlb_start_vma() and tlb_end_vma()
+#endif
+
+/*
+ * When an architecture does not have efficient means of range flushing TLBs
+ * there is no point in doing intermediate flushes on tlb_end_vma() to keep the

include/asm-generic/tlb.h static inline void __tlb_reset_range(str · line 327

Quoted 10 lines
include/asm-generic/tlb.h +10 -0
+ * range small. We equally don't have to worry about page granularity or other
+ * things.
+ *
+ * All we need to do is issue a full flush for any !0 range.
+ */
+static inline void tlb_flush(struct mmu_gather *tlb)
+{
+	if (tlb->end)
+		flush_tlb_mm(tlb->mm);
+}

I guess another way we could handle these architectures is by unconditionally resetting tlb->fullmm to 1, but this works too.

Will

author maintainer
#7 lore raw

PATCH v6 6/18 asm-generic/tlb: Conditionally provide tlb_migrate_finish()

In reply to Peter Zijlstra

Needed for ia64 -- alternatively we drop the entire hook.

Cc Will Deacon <[email protected]>
Cc "Aneesh Kumar K.V" <[email protected]>
Cc Andrew Morton <[email protected]>
Cc Nick Piggin <[email protected]>
---
 include/asm-generic/tlb.h |    2 ++
 1 file changed, 2 insertions(+)
include/asm-generic/tlb.h +2 -0
@@ -539,6 +539,8 @@ static inline void tlb_end_vma(struct mm
 
 #endif /* CONFIG_MMU */
 
+#ifndef tlb_migrate_finish
 #define tlb_migrate_finish(mm) do {} while (0)
+#endif
 
 #endif /* _ASM_GENERIC__TLB_H */
Acked-by on v6 6/18
#8 lore raw

Re: [PATCH v6 06/18] asm-generic/tlb: Conditionally provide tlb_migrate_finish()

Quoted 14 lines · Peter Zijlstra

Needed for ia64 -- alternatively we drop the entire hook.

Cc Will Deacon <[email protected]>
Cc "Aneesh Kumar K.V" <[email protected]>
Cc Andrew Morton <[email protected]>
Cc Nick Piggin <[email protected]>
---
 include/asm-generic/tlb.h |    2 ++
 1 file changed, 2 insertions(+)

include/asm-generic/tlb.h static inline void tlb_end_vma(struct mm · line 544

Quoted 6 lines
include/asm-generic/tlb.h +2 -0
@@ -539,6 +539,8 @@ static inline void tlb_end_vma(struct mm
 
 #endif /* CONFIG_MMU */
 
+#ifndef tlb_migrate_finish
 #define tlb_migrate_finish(mm) do {} while (0)
+#endif

Fine for now, but I agree that we should drop the hook altogether. AFAICT, this only exists to help an ia64 optimisation which looks suspicious to me since it uses:

    mm == current->active_mm && atomic_read(&mm->mm_users) == 1

to identify a "single-threaded fork()" and therefore perform only local TLB invalidation. Even if this was the right thing to do, it's not clear to me that tlb_migrate_finish() is called on the right CPU anyway.

So I'd be keen to remove this hook before it spreads, but in the meantime:

Will

author maintainer
#9 lore raw

Re: [PATCH v6 06/18] asm-generic/tlb: Conditionally provide tlb_migrate_finish()

Quoted 11 lines · Will Deacon

Fine for now, but I agree that we should drop the hook altogether. AFAICT, this only exists to help an ia64 optimisation which looks suspicious to me since it uses:

    mm == current->active_mm && atomic_read(&mm->mm_users) == 1

to identify a "single-threaded fork()" and therefore perform only local TLB invalidation. Even if this was the right thing to do, it's not clear to me that tlb_migrate_finish() is called on the right CPU anyway.

So I'd be keen to remove this hook before it spreads, but in the meantime:

Agreed :-)

The obvious slash and kill patch ... untested

---
 Documentation/core-api/cachetlb.rst |   10 ----------
 arch/ia64/include/asm/machvec.h     |    8 --------
 arch/ia64/include/asm/machvec_sn2.h |    2 --
 arch/ia64/include/asm/tlb.h         |    2 --
 arch/ia64/sn/kernel/sn2/sn2_smp.c   |    7 -------
 arch/nds32/include/asm/tlbflush.h   |    1 -
 include/asm-generic/tlb.h           |    4 ----
 kernel/sched/core.c                 |    1 -
 8 files changed, 35 deletions(-)
Documentation/core-api/cachetlb.rst +0 -10
@@ -101,16 +101,6 @@ invoke one of the following flush method
 	translations for software managed TLB configurations.
 	The sparc64 port currently does this.
 
-6) ``void tlb_migrate_finish(struct mm_struct *mm)``
-
-	This interface is called at the end of an explicit
-	process migration. This interface provides a hook
-	to allow a platform to update TLB or context-specific
-	information for the address space.
-
-	The ia64 sn2 platform is one example of a platform
-	that uses this interface.
-
 Next, we have the cache flushing interfaces.  In general, when Linux
 is changing an existing virtual-->physical mapping to a new value,
 the sequence will be in one of the following forms::
arch/ia64/include/asm/machvec.h +0 -8
@@ -30,7 +30,6 @@ typedef void ia64_mv_irq_init_t (void);
 typedef void ia64_mv_send_ipi_t (int, int, int, int);
 typedef void ia64_mv_timer_interrupt_t (int, void *);
 typedef void ia64_mv_global_tlb_purge_t (struct mm_struct *, unsigned long, unsigned long, unsigned long);
-typedef void ia64_mv_tlb_migrate_finish_t (struct mm_struct *);
 typedef u8 ia64_mv_irq_to_vector (int);
 typedef unsigned int ia64_mv_local_vector_to_irq (u8);
 typedef char *ia64_mv_pci_get_legacy_mem_t (struct pci_bus *);
@@ -96,7 +95,6 @@ machvec_noop_bus (struct pci_bus *bus)
 
 extern void machvec_setup (char **);
 extern void machvec_timer_interrupt (int, void *);
-extern void machvec_tlb_migrate_finish (struct mm_struct *);
 
 # if defined (CONFIG_IA64_HP_SIM)
 #  include <asm/machvec_hpsim.h>
@@ -124,7 +122,6 @@ extern void machvec_tlb_migrate_finish (
 #  define platform_send_ipi	ia64_mv.send_ipi
 #  define platform_timer_interrupt	ia64_mv.timer_interrupt
 #  define platform_global_tlb_purge	ia64_mv.global_tlb_purge
-#  define platform_tlb_migrate_finish	ia64_mv.tlb_migrate_finish
 #  define platform_dma_init		ia64_mv.dma_init
 #  define platform_dma_get_ops		ia64_mv.dma_get_ops
 #  define platform_irq_to_vector	ia64_mv.irq_to_vector
@@ -167,7 +164,6 @@ struct ia64_machine_vector {
 	ia64_mv_send_ipi_t *send_ipi;
 	ia64_mv_timer_interrupt_t *timer_interrupt;
 	ia64_mv_global_tlb_purge_t *global_tlb_purge;
-	ia64_mv_tlb_migrate_finish_t *tlb_migrate_finish;
 	ia64_mv_dma_init *dma_init;
 	ia64_mv_dma_get_ops *dma_get_ops;
 	ia64_mv_irq_to_vector *irq_to_vector;
@@ -206,7 +202,6 @@ struct ia64_machine_vector {
 	platform_send_ipi,			\
 	platform_timer_interrupt,		\
 	platform_global_tlb_purge,		\
-	platform_tlb_migrate_finish,		\
 	platform_dma_init,			\
 	platform_dma_get_ops,			\
 	platform_irq_to_vector,			\
@@ -270,9 +265,6 @@ extern const struct dma_map_ops *dma_get
 #ifndef platform_global_tlb_purge
 # define platform_global_tlb_purge	ia64_global_tlb_purge /* default to architected version */
 #endif
-#ifndef platform_tlb_migrate_finish
-# define platform_tlb_migrate_finish	machvec_noop_mm
-#endif
 #ifndef platform_kernel_launch_event
 # define platform_kernel_launch_event	machvec_noop
 #endif
arch/ia64/include/asm/machvec_sn2.h +0 -2
@@ -34,7 +34,6 @@ extern ia64_mv_irq_init_t sn_irq_init;
 extern ia64_mv_send_ipi_t sn2_send_IPI;
 extern ia64_mv_timer_interrupt_t sn_timer_interrupt;
 extern ia64_mv_global_tlb_purge_t sn2_global_tlb_purge;
-extern ia64_mv_tlb_migrate_finish_t	sn_tlb_migrate_finish;
 extern ia64_mv_irq_to_vector sn_irq_to_vector;
 extern ia64_mv_local_vector_to_irq sn_local_vector_to_irq;
 extern ia64_mv_pci_get_legacy_mem_t sn_pci_get_legacy_mem;
@@ -77,7 +76,6 @@ extern ia64_mv_pci_fixup_bus_t		sn_pci_f
 #define platform_send_ipi		sn2_send_IPI
 #define platform_timer_interrupt	sn_timer_interrupt
 #define platform_global_tlb_purge       sn2_global_tlb_purge
-#define platform_tlb_migrate_finish	sn_tlb_migrate_finish
 #define platform_pci_fixup		sn_pci_fixup
 #define platform_inb			__sn_inb
 #define platform_inw			__sn_inw
arch/ia64/include/asm/tlb.h +0 -2
@@ -47,8 +47,6 @@
 #include <asm/tlbflush.h>
 #include <asm/machvec.h>
 
-#define tlb_migrate_finish(mm)	platform_tlb_migrate_finish(mm)
-
 #include <asm-generic/tlb.h>
 
 #endif /* _ASM_IA64_TLB_H */
arch/ia64/sn/kernel/sn2/sn2_smp.c +0 -7
@@ -120,13 +120,6 @@ void sn_migrate(struct task_struct *task
 		cpu_relax();
 }
 
-void sn_tlb_migrate_finish(struct mm_struct *mm)
-{
-	/* flush_tlb_mm is inefficient if more than 1 users of mm */
-	if (mm == current->mm && mm && atomic_read(&mm->mm_users) == 1)
-		flush_tlb_mm(mm);
-}
-
 static void
 sn2_ipi_flush_all_tlb(struct mm_struct *mm)
 {
arch/nds32/include/asm/tlbflush.h +0 -1
@@ -42,6 +42,5 @@ void local_flush_tlb_page(struct vm_area
 
 void update_mmu_cache(struct vm_area_struct *vma,
 		      unsigned long address, pte_t * pte);
-void tlb_migrate_finish(struct mm_struct *mm);
 
 #endif
include/asm-generic/tlb.h +0 -4
@@ -604,8 +604,4 @@ static inline void tlb_end_vma(struct mm
 
 #endif /* CONFIG_MMU */
 
-#ifndef tlb_migrate_finish
-#define tlb_migrate_finish(mm) do {} while (0)
-#endif
-
 #endif /* _ASM_GENERIC__TLB_H */
kernel/sched/core.c +0 -1
@@ -1151,7 +1151,6 @@ static int __set_cpus_allowed_ptr(struct
 		/* Need help from migration thread: drop lock and wait. */
 		task_rq_unlock(rq, p, &rf);
 		stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);
-		tlb_migrate_finish(p->mm);
 		return 0;
 	} else if (task_on_rq_queued(p)) {
 		/*
#10 lore raw

Re: [PATCH v6 06/18] asm-generic/tlb: Conditionally provide tlb_migrate_finish()

Quoted 11 lines · Peter Zijlstra
Quoted 10 lines · Will Deacon

Fine for now, but I agree that we should drop the hook altogether. AFAICT, this only exists to help an ia64 optimisation which looks suspicious to me since it uses:

    mm == current->active_mm && atomic_read(&mm->mm_users) == 1

to identify a "single-threaded fork()" and therefore perform only local TLB invalidation. Even if this was the right thing to do, it's not clear to me that tlb_migrate_finish() is called on the right CPU anyway.

Quoted 5 lines
Quoted 1 line

So I'd be keen to remove this hook before it spreads, but in the meantime:

Agreed :-)

The obvious slash and kill patch ... untested

I'm also unable to test this, unfortunately. Can we get it into next after the merge window and see if anybody reports issues?

Will

#11 lore raw

Re: [PATCH v6 06/18] asm-generic/tlb: Conditionally provide tlb_migrate_finish()

Quoted 12 lines · Will Deacon
Quoted 11 lines · Peter Zijlstra
Quoted 10 lines · Will Deacon

Fine for now, but I agree that we should drop the hook altogether. AFAICT, this only exists to help an ia64 optimisation which looks suspicious to me since it uses:

    mm == current->active_mm && atomic_read(&mm->mm_users) == 1

to identify a "single-threaded fork()" and therefore perform only local TLB invalidation. Even if this was the right thing to do, it's not clear to me that tlb_migrate_finish() is called on the right CPU anyway.

Quoted 8 lines
Quoted 5 lines
Quoted 1 line

So I'd be keen to remove this hook before it spreads, but in the meantime:

Agreed :-)

The obvious slash and kill patch ... untested

I'm also unable to test this, unfortunately. Can we get it into next after the merge window and see if anybody reports issues?

While I do have a pair of Itanium systems in my basement, neither are sn2 machines, which was the only sub-architecture that implemented tlb_migrate_finish(). I see NASA decomissioned Columbia in 2013, and I imagine most sn2 machines have been similarly scrapped.

author maintainer
#12 lore raw

PATCH v6 7/18 asm-generic/tlb: Invert HAVE_RCU_TABLE_INVALIDATE

In reply to Peter Zijlstra

Make issuing a TLB invalidate for page-table pages the normal case.

The reason is twofold:

- too many invalidates is safer than too few,
- most architectures use the linux page-tables natively and would thus require this.

Make it an opt-out, instead of an opt-in.

---
 arch/Kconfig              |    2 +-
 arch/arm64/Kconfig        |    1 -
 arch/powerpc/Kconfig      |    1 +
 arch/sparc/Kconfig        |    1 +
 arch/x86/Kconfig          |    1 -
 include/asm-generic/tlb.h |    9 +++++----
 mm/mmu_gather.c           |    2 +-
 7 files changed, 9 insertions(+), 8 deletions(-)
arch/Kconfig +1 -1
@@ -372,7 +372,7 @@ config HAVE_ARCH_JUMP_LABEL_RELATIVE
 config HAVE_RCU_TABLE_FREE
 	bool
 
-config HAVE_RCU_TABLE_INVALIDATE
+config HAVE_RCU_TABLE_NO_INVALIDATE
 	bool
 
 config HAVE_MMU_GATHER_PAGE_SIZE
arch/arm64/Kconfig +0 -1
@@ -147,7 +147,6 @@ config ARM64
 	select HAVE_PERF_USER_STACK_DUMP
 	select HAVE_REGS_AND_STACK_ACCESS_API
 	select HAVE_RCU_TABLE_FREE
-	select HAVE_RCU_TABLE_INVALIDATE
 	select HAVE_RSEQ
 	select HAVE_STACKPROTECTOR
 	select HAVE_SYSCALL_TRACEPOINTS
arch/powerpc/Kconfig +1 -0
@@ -218,6 +218,7 @@ config PPC
 	select HAVE_PERF_REGS
 	select HAVE_PERF_USER_STACK_DUMP
 	select HAVE_RCU_TABLE_FREE		if SMP
+	select HAVE_RCU_TABLE_NO_INVALIDATE	if HAVE_RCU_TABLE_FREE
 	select HAVE_MMU_GATHER_PAGE_SIZE
 	select HAVE_REGS_AND_STACK_ACCESS_API
 	select HAVE_RELIABLE_STACKTRACE		if PPC64 && CPU_LITTLE_ENDIAN
arch/sparc/Kconfig +1 -0
@@ -62,6 +62,7 @@ config SPARC64
 	select HAVE_KRETPROBES
 	select HAVE_KPROBES
 	select HAVE_RCU_TABLE_FREE if SMP
+	select HAVE_RCU_TABLE_NO_INVALIDATE if HAVE_RCU_TABLE_FREE
 	select HAVE_MEMBLOCK_NODE_MAP
 	select HAVE_ARCH_TRANSPARENT_HUGEPAGE
 	select HAVE_DYNAMIC_FTRACE
arch/x86/Kconfig +0 -1
@@ -183,7 +183,6 @@ config X86
 	select HAVE_PERF_REGS
 	select HAVE_PERF_USER_STACK_DUMP
 	select HAVE_RCU_TABLE_FREE		if PARAVIRT
-	select HAVE_RCU_TABLE_INVALIDATE	if HAVE_RCU_TABLE_FREE
 	select HAVE_REGS_AND_STACK_ACCESS_API
 	select HAVE_RELIABLE_STACKTRACE		if X86_64 && (UNWINDER_FRAME_POINTER || UNWINDER_ORC) && STACK_VALIDATION
 	select HAVE_FUNCTION_ARG_ACCESS_API
include/asm-generic/tlb.h +5 -4
@@ -135,11 +135,12 @@
  *  When used, an architecture is expected to provide __tlb_remove_table()
  *  which does the actual freeing of these pages.
  *
- *  HAVE_RCU_TABLE_INVALIDATE
+ *  HAVE_RCU_TABLE_NO_INVALIDATE
  *
- *  This makes HAVE_RCU_TABLE_FREE call tlb_flush_mmu_tlbonly() before freeing
- *  the page-table pages. Required if you use HAVE_RCU_TABLE_FREE and your
- *  architecture uses the Linux page-tables natively.
+ *  This makes HAVE_RCU_TABLE_FREE avoid calling tlb_flush_mmu_tlbonly() before
+ *  freeing the page-table pages. This can be avoided if you use
+ *  HAVE_RCU_TABLE_FREE and your architecture does _NOT_ use the Linux
+ *  page-tables natively.
  *
  *  MMU_GATHER_NO_RANGE
  *
mm/mmu_gather.c +1 -1
@@ -157,7 +157,7 @@ bool __tlb_remove_page_size(struct mmu_g
  */
 static inline void tlb_table_invalidate(struct mmu_gather *tlb)
 {
-#ifdef CONFIG_HAVE_RCU_TABLE_INVALIDATE
+#ifndef CONFIG_HAVE_RCU_TABLE_NO_INVALIDATE
 	/*
 	 * Invalidate page-table caches used by hardware walkers. Then we still
 	 * need to RCU-sched wait while freeing the pages because software
author maintainer
#13 lore raw

PATCH v6 8/18 arm/tlb: Convert to generic mmu_gather

In reply to Peter Zijlstra

Generic mmu_gather provides everything that ARM needs:

- range tracking
- RCU table free
- VM_EXEC tracking
- VIPT cache flushing

The one notable curiosity is the 'funny' range tracking for classical ARM in __pte_free_tlb().

Cc Nick Piggin <[email protected]>
Cc "Aneesh Kumar K.V" <[email protected]>
Cc Andrew Morton <[email protected]>
Cc Russell King <[email protected]>
---
 arch/arm/include/asm/tlb.h |  254 ++-------------------------------------------
 1 file changed, 13 insertions(+), 241 deletions(-)
arch/arm/include/asm/tlb.h +13 -241
@@ -33,270 +33,42 @@
 #include <asm/pgalloc.h>
 #include <asm/tlbflush.h>
 
-#define MMU_GATHER_BUNDLE	8
-
-#ifdef CONFIG_HAVE_RCU_TABLE_FREE
 static inline void __tlb_remove_table(void *_table)
 {
 	free_page_and_swap_cache((struct page *)_table);
 }
 
-struct mmu_table_batch {
-	struct rcu_head		rcu;
-	unsigned int		nr;
-	void			*tables[0];
-};
-
-#define MAX_TABLE_BATCH		\
-	((PAGE_SIZE - sizeof(struct mmu_table_batch)) / sizeof(void *))
-
-extern void tlb_table_flush(struct mmu_gather *tlb);
-extern void tlb_remove_table(struct mmu_gather *tlb, void *table);
-
-#define tlb_remove_entry(tlb, entry)	tlb_remove_table(tlb, entry)
-#else
-#define tlb_remove_entry(tlb, entry)	tlb_remove_page(tlb, entry)
-#endif /* CONFIG_HAVE_RCU_TABLE_FREE */
-
-/*
- * TLB handling.  This allows us to remove pages from the page
- * tables, and efficiently handle the TLB issues.
- */
-struct mmu_gather {
-	struct mm_struct	*mm;
-#ifdef CONFIG_HAVE_RCU_TABLE_FREE
-	struct mmu_table_batch	*batch;
-	unsigned int		need_flush;
-#endif
-	unsigned int		fullmm;
-	struct vm_area_struct	*vma;
-	unsigned long		start, end;
-	unsigned long		range_start;
-	unsigned long		range_end;
-	unsigned int		nr;
-	unsigned int		max;
-	struct page		**pages;
-	struct page		*local[MMU_GATHER_BUNDLE];
-};
-
-DECLARE_PER_CPU(struct mmu_gather, mmu_gathers);
-
-/*
- * This is unnecessarily complex.  There's three ways the TLB shootdown
- * code is used:
- *  1. Unmapping a range of vmas.  See zap_page_range(), unmap_region().
- *     tlb->fullmm = 0, and tlb_start_vma/tlb_end_vma will be called.
- *     tlb->vma will be non-NULL.
- *  2. Unmapping all vmas.  See exit_mmap().
- *     tlb->fullmm = 1, and tlb_start_vma/tlb_end_vma will be called.
- *     tlb->vma will be non-NULL.  Additionally, page tables will be freed.
- *  3. Unmapping argument pages.  See shift_arg_pages().
- *     tlb->fullmm = 0, but tlb_start_vma/tlb_end_vma will not be called.
- *     tlb->vma will be NULL.
- */
-static inline void tlb_flush(struct mmu_gather *tlb)
-{
-	if (tlb->fullmm || !tlb->vma)
-		flush_tlb_mm(tlb->mm);
-	else if (tlb->range_end > 0) {
-		flush_tlb_range(tlb->vma, tlb->range_start, tlb->range_end);
-		tlb->range_start = TASK_SIZE;
-		tlb->range_end = 0;
-	}
-}
-
-static inline void tlb_add_flush(struct mmu_gather *tlb, unsigned long addr)
-{
-	if (!tlb->fullmm) {
-		if (addr < tlb->range_start)
-			tlb->range_start = addr;
-		if (addr + PAGE_SIZE > tlb->range_end)
-			tlb->range_end = addr + PAGE_SIZE;
-	}
-}
-
-static inline void __tlb_alloc_page(struct mmu_gather *tlb)
-{
-	unsigned long addr = __get_free_pages(GFP_NOWAIT | __GFP_NOWARN, 0);
-
-	if (addr) {
-		tlb->pages = (void *)addr;
-		tlb->max = PAGE_SIZE / sizeof(struct page *);
-	}
-}
-
-static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
-{
-	tlb_flush(tlb);
-#ifdef CONFIG_HAVE_RCU_TABLE_FREE
-	tlb_table_flush(tlb);
-#endif
-}
-
-static inline void tlb_flush_mmu_free(struct mmu_gather *tlb)
-{
-	free_pages_and_swap_cache(tlb->pages, tlb->nr);
-	tlb->nr = 0;
-	if (tlb->pages == tlb->local)
-		__tlb_alloc_page(tlb);
-}
-
-static inline void tlb_flush_mmu(struct mmu_gather *tlb)
-{
-	tlb_flush_mmu_tlbonly(tlb);
-	tlb_flush_mmu_free(tlb);
-}
-
-static inline void
-arch_tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,
-			unsigned long start, unsigned long end)
-{
-	tlb->mm = mm;
-	tlb->fullmm = !(start | (end+1));
-	tlb->start = start;
-	tlb->end = end;
-	tlb->vma = NULL;
-	tlb->max = ARRAY_SIZE(tlb->local);
-	tlb->pages = tlb->local;
-	tlb->nr = 0;
-	__tlb_alloc_page(tlb);
+#include <asm-generic/tlb.h>
 
-#ifdef CONFIG_HAVE_RCU_TABLE_FREE
-	tlb->batch = NULL;
+#ifndef CONFIG_HAVE_RCU_TABLE_FREE
+#define tlb_remove_table(tlb, entry) tlb_remove_page(tlb, entry)
 #endif
-}
-
-static inline void
-arch_tlb_finish_mmu(struct mmu_gather *tlb,
-			unsigned long start, unsigned long end, bool force)
-{
-	if (force) {
-		tlb->range_start = start;
-		tlb->range_end = end;
-	}
-
-	tlb_flush_mmu(tlb);
 
-	/* keep the page table cache within bounds */
-	check_pgt_cache();
-
-	if (tlb->pages != tlb->local)
-		free_pages((unsigned long)tlb->pages, 0);
-}
-
-/*
- * Memorize the range for the TLB flush.
- */
 static inline void
-tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, unsigned long addr)
-{
-	tlb_add_flush(tlb, addr);
-}
-
-#define tlb_remove_huge_tlb_entry(h, tlb, ptep, address)	\
-	tlb_remove_tlb_entry(tlb, ptep, address)
-/*
- * In the case of tlb vma handling, we can optimise these away in the
- * case where we're doing a full MM flush.  When we're doing a munmap,
- * the vmas are adjusted to only cover the region to be torn down.
- */
-static inline void
-tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
-{
-	if (!tlb->fullmm) {
-		flush_cache_range(vma, vma->vm_start, vma->vm_end);
-		tlb->vma = vma;
-		tlb->range_start = TASK_SIZE;
-		tlb->range_end = 0;
-	}
-}
-
-static inline void
-tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
-{
-	if (!tlb->fullmm)
-		tlb_flush(tlb);
-}
-
-static inline bool __tlb_remove_page(struct mmu_gather *tlb, struct page *page)
-{
-	tlb->pages[tlb->nr++] = page;
-	VM_WARN_ON(tlb->nr > tlb->max);
-	if (tlb->nr == tlb->max)
-		return true;
-	return false;
-}
-
-static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
-{
-	if (__tlb_remove_page(tlb, page))
-		tlb_flush_mmu(tlb);
-}
-
-static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
-					  struct page *page, int page_size)
-{
-	return __tlb_remove_page(tlb, page);
-}
-
-static inline void tlb_remove_page_size(struct mmu_gather *tlb,
-					struct page *page, int page_size)
-{
-	return tlb_remove_page(tlb, page);
-}
-
-static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
-	unsigned long addr)
+__pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, unsigned long addr)
 {
 	pgtable_page_dtor(pte);
 
-#ifdef CONFIG_ARM_LPAE
-	tlb_add_flush(tlb, addr);
-#else
+#ifndef CONFIG_ARM_LPAE
 	/*
 	 * With the classic ARM MMU, a pte page has two corresponding pmd
 	 * entries, each covering 1MB.
 	 */
-	addr &= PMD_MASK;
-	tlb_add_flush(tlb, addr + SZ_1M - PAGE_SIZE);
-	tlb_add_flush(tlb, addr + SZ_1M);
+	addr = (addr & PMD_MASK) + SZ_1M;
+	__tlb_adjust_range(tlb, addr - PAGE_SIZE, 2 * PAGE_SIZE);
 #endif
 
-	tlb_remove_entry(tlb, pte);
-}
-
-static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp,
-				  unsigned long addr)
-{
-#ifdef CONFIG_ARM_LPAE
-	tlb_add_flush(tlb, addr);
-	tlb_remove_entry(tlb, virt_to_page(pmdp));
-#endif
+	tlb_remove_table(tlb, pte);
 }
 
 static inline void
-tlb_remove_pmd_tlb_entry(struct mmu_gather *tlb, pmd_t *pmdp, unsigned long addr)
+__pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp, unsigned long addr)
 {
-	tlb_add_flush(tlb, addr);
-}
-
-#define pte_free_tlb(tlb, ptep, addr)	__pte_free_tlb(tlb, ptep, addr)
-#define pmd_free_tlb(tlb, pmdp, addr)	__pmd_free_tlb(tlb, pmdp, addr)
-#define pud_free_tlb(tlb, pudp, addr)	pud_free((tlb)->mm, pudp)
-
-#define tlb_migrate_finish(mm)		do { } while (0)
-
-static inline void tlb_change_page_size(struct mmu_gather *tlb,
-						     unsigned int page_size)
-{
-}
-
-static inline void tlb_flush_remove_tables(struct mm_struct *mm)
-{
-}
+#ifdef CONFIG_ARM_LPAE
+	struct page *page = virt_to_page(pmdp);
 
-static inline void tlb_flush_remove_tables_local(void *arg)
-{
+	tlb_remove_table(tlb, page);
+#endif
 }
 
 #endif /* CONFIG_MMU */
author maintainer
#14 lore raw

PATCH v6 9/18 ia64/tlb: Conver to generic mmu_gather

In reply to Peter Zijlstra

Generic mmu_gather provides everything ia64 needs (range tracking).

Cc Will Deacon <[email protected]>
Cc "Aneesh Kumar K.V" <[email protected]>
Cc Andrew Morton <[email protected]>
Cc Nick Piggin <[email protected]>
Cc Tony Luck <[email protected]>
---
 arch/ia64/include/asm/tlb.h      |  256 ---------------------------------------
 arch/ia64/include/asm/tlbflush.h |   25 +++
 arch/ia64/mm/tlb.c               |   23 +++
 3 files changed, 47 insertions(+), 257 deletions(-)
arch/ia64/include/asm/tlb.h +1 -255
@@ -47,262 +47,8 @@
 #include <asm/tlbflush.h>
 #include <asm/machvec.h>
 
-/*
- * If we can't allocate a page to make a big batch of page pointers
- * to work on, then just handle a few from the on-stack structure.
- */
-#define	IA64_GATHER_BUNDLE	8
-
-struct mmu_gather {
-	struct mm_struct	*mm;
-	unsigned int		nr;
-	unsigned int		max;
-	unsigned char		fullmm;		/* non-zero means full mm flush */
-	unsigned char		need_flush;	/* really unmapped some PTEs? */
-	unsigned long		start, end;
-	unsigned long		start_addr;
-	unsigned long		end_addr;
-	struct page		**pages;
-	struct page		*local[IA64_GATHER_BUNDLE];
-};
-
-struct ia64_tr_entry {
-	u64 ifa;
-	u64 itir;
-	u64 pte;
-	u64 rr;
-}; /*Record for tr entry!*/
-
-extern int ia64_itr_entry(u64 target_mask, u64 va, u64 pte, u64 log_size);
-extern void ia64_ptr_entry(u64 target_mask, int slot);
-
-extern struct ia64_tr_entry *ia64_idtrs[NR_CPUS];
-
-/*
- region register macros
-*/
-#define RR_TO_VE(val)   (((val) >> 0) & 0x0000000000000001)
-#define RR_VE(val)	(((val) & 0x0000000000000001) << 0)
-#define RR_VE_MASK	0x0000000000000001L
-#define RR_VE_SHIFT	0
-#define RR_TO_PS(val)	(((val) >> 2) & 0x000000000000003f)
-#define RR_PS(val)	(((val) & 0x000000000000003f) << 2)
-#define RR_PS_MASK	0x00000000000000fcL
-#define RR_PS_SHIFT	2
-#define RR_RID_MASK	0x00000000ffffff00L
-#define RR_TO_RID(val) 	((val >> 8) & 0xffffff)
-
-static inline void
-ia64_tlb_flush_mmu_tlbonly(struct mmu_gather *tlb, unsigned long start, unsigned long end)
-{
-	tlb->need_flush = 0;
-
-	if (tlb->fullmm) {
-		/*
-		 * Tearing down the entire address space.  This happens both as a result
-		 * of exit() and execve().  The latter case necessitates the call to
-		 * flush_tlb_mm() here.
-		 */
-		flush_tlb_mm(tlb->mm);
-	} else if (unlikely (end - start >= 1024*1024*1024*1024UL
-			     || REGION_NUMBER(start) != REGION_NUMBER(end - 1)))
-	{
-		/*
-		 * If we flush more than a tera-byte or across regions, we're probably
-		 * better off just flushing the entire TLB(s).  This should be very rare
-		 * and is not worth optimizing for.
-		 */
-		flush_tlb_all();
-	} else {
-		/*
-		 * flush_tlb_range() takes a vma instead of a mm pointer because
-		 * some architectures want the vm_flags for ITLB/DTLB flush.
-		 */
-		struct vm_area_struct vma = TLB_FLUSH_VMA(tlb->mm, 0);
-
-		/* flush the address range from the tlb: */
-		flush_tlb_range(&vma, start, end);
-		/* now flush the virt. page-table area mapping the address range: */
-		flush_tlb_range(&vma, ia64_thash(start), ia64_thash(end));
-	}
-
-}
-
-static inline void
-ia64_tlb_flush_mmu_free(struct mmu_gather *tlb)
-{
-	unsigned long i;
-	unsigned int nr;
-
-	/* lastly, release the freed pages */
-	nr = tlb->nr;
-
-	tlb->nr = 0;
-	tlb->start_addr = ~0UL;
-	for (i = 0; i < nr; ++i)
-		free_page_and_swap_cache(tlb->pages[i]);
-}
-
-/*
- * Flush the TLB for address range START to END and, if not in fast mode, release the
- * freed pages that where gathered up to this point.
- */
-static inline void
-ia64_tlb_flush_mmu (struct mmu_gather *tlb, unsigned long start, unsigned long end)
-{
-	if (!tlb->need_flush)
-		return;
-	ia64_tlb_flush_mmu_tlbonly(tlb, start, end);
-	ia64_tlb_flush_mmu_free(tlb);
-}
-
-static inline void __tlb_alloc_page(struct mmu_gather *tlb)
-{
-	unsigned long addr = __get_free_pages(GFP_NOWAIT | __GFP_NOWARN, 0);
-
-	if (addr) {
-		tlb->pages = (void *)addr;
-		tlb->max = PAGE_SIZE / sizeof(void *);
-	}
-}
-
-
-static inline void
-arch_tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,
-			unsigned long start, unsigned long end)
-{
-	tlb->mm = mm;
-	tlb->max = ARRAY_SIZE(tlb->local);
-	tlb->pages = tlb->local;
-	tlb->nr = 0;
-	tlb->fullmm = !(start | (end+1));
-	tlb->start = start;
-	tlb->end = end;
-	tlb->start_addr = ~0UL;
-}
-
-/*
- * Called at the end of the shootdown operation to free up any resources that were
- * collected.
- */
-static inline void
-arch_tlb_finish_mmu(struct mmu_gather *tlb,
-			unsigned long start, unsigned long end, bool force)
-{
-	if (force)
-		tlb->need_flush = 1;
-	/*
-	 * Note: tlb->nr may be 0 at this point, so we can't rely on tlb->start_addr and
-	 * tlb->end_addr.
-	 */
-	ia64_tlb_flush_mmu(tlb, start, end);
-
-	/* keep the page table cache within bounds */
-	check_pgt_cache();
-
-	if (tlb->pages != tlb->local)
-		free_pages((unsigned long)tlb->pages, 0);
-}
-
-/*
- * Logically, this routine frees PAGE.  On MP machines, the actual freeing of the page
- * must be delayed until after the TLB has been flushed (see comments at the beginning of
- * this file).
- */
-static inline bool __tlb_remove_page(struct mmu_gather *tlb, struct page *page)
-{
-	tlb->need_flush = 1;
-
-	if (!tlb->nr && tlb->pages == tlb->local)
-		__tlb_alloc_page(tlb);
-
-	tlb->pages[tlb->nr++] = page;
-	VM_WARN_ON(tlb->nr > tlb->max);
-	if (tlb->nr == tlb->max)
-		return true;
-	return false;
-}
-
-static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
-{
-	ia64_tlb_flush_mmu_tlbonly(tlb, tlb->start_addr, tlb->end_addr);
-}
-
-static inline void tlb_flush_mmu_free(struct mmu_gather *tlb)
-{
-	ia64_tlb_flush_mmu_free(tlb);
-}
-
-static inline void tlb_flush_mmu(struct mmu_gather *tlb)
-{
-	ia64_tlb_flush_mmu(tlb, tlb->start_addr, tlb->end_addr);
-}
-
-static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
-{
-	if (__tlb_remove_page(tlb, page))
-		tlb_flush_mmu(tlb);
-}
-
-static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
-					  struct page *page, int page_size)
-{
-	return __tlb_remove_page(tlb, page);
-}
-
-static inline void tlb_remove_page_size(struct mmu_gather *tlb,
-					struct page *page, int page_size)
-{
-	return tlb_remove_page(tlb, page);
-}
-
-/*
- * Remove TLB entry for PTE mapped at virtual address ADDRESS.  This is called for any
- * PTE, not just those pointing to (normal) physical memory.
- */
-static inline void
-__tlb_remove_tlb_entry (struct mmu_gather *tlb, pte_t *ptep, unsigned long address)
-{
-	if (tlb->start_addr == ~0UL)
-		tlb->start_addr = address;
-	tlb->end_addr = address + PAGE_SIZE;
-}
-
 #define tlb_migrate_finish(mm)	platform_tlb_migrate_finish(mm)
 
-#define tlb_start_vma(tlb, vma)			do { } while (0)
-#define tlb_end_vma(tlb, vma)			do { } while (0)
-
-#define tlb_remove_tlb_entry(tlb, ptep, addr)		\
-do {							\
-	tlb->need_flush = 1;				\
-	__tlb_remove_tlb_entry(tlb, ptep, addr);	\
-} while (0)
-
-#define tlb_remove_huge_tlb_entry(h, tlb, ptep, address)	\
-	tlb_remove_tlb_entry(tlb, ptep, address)
-
-static inline void tlb_change_page_size(struct mmu_gather *tlb,
-						     unsigned int page_size)
-{
-}
-
-#define pte_free_tlb(tlb, ptep, address)		\
-do {							\
-	tlb->need_flush = 1;				\
-	__pte_free_tlb(tlb, ptep, address);		\
-} while (0)
-
-#define pmd_free_tlb(tlb, ptep, address)		\
-do {							\
-	tlb->need_flush = 1;				\
-	__pmd_free_tlb(tlb, ptep, address);		\
-} while (0)
-
-#define pud_free_tlb(tlb, pudp, address)		\
-do {							\
-	tlb->need_flush = 1;				\
-	__pud_free_tlb(tlb, pudp, address);		\
-} while (0)
+#include <asm-generic/tlb.h>
 
 #endif /* _ASM_IA64_TLB_H */
arch/ia64/include/asm/tlbflush.h +25 -0
@@ -14,6 +14,31 @@
 #include <asm/mmu_context.h>
 #include <asm/page.h>
 
+struct ia64_tr_entry {
+	u64 ifa;
+	u64 itir;
+	u64 pte;
+	u64 rr;
+}; /*Record for tr entry!*/
+
+extern int ia64_itr_entry(u64 target_mask, u64 va, u64 pte, u64 log_size);
+extern void ia64_ptr_entry(u64 target_mask, int slot);
+extern struct ia64_tr_entry *ia64_idtrs[NR_CPUS];
+
+/*
+ region register macros
+*/
+#define RR_TO_VE(val)   (((val) >> 0) & 0x0000000000000001)
+#define RR_VE(val)     (((val) & 0x0000000000000001) << 0)
+#define RR_VE_MASK     0x0000000000000001L
+#define RR_VE_SHIFT    0
+#define RR_TO_PS(val)  (((val) >> 2) & 0x000000000000003f)
+#define RR_PS(val)     (((val) & 0x000000000000003f) << 2)
+#define RR_PS_MASK     0x00000000000000fcL
+#define RR_PS_SHIFT    2
+#define RR_RID_MASK    0x00000000ffffff00L
+#define RR_TO_RID(val)         ((val >> 8) & 0xffffff)
+
 /*
  * Now for some TLB flushing routines.  This is the kind of stuff that
  * can be very expensive, so try to avoid them whenever possible.
arch/ia64/mm/tlb.c +21 -2
@@ -297,8 +297,8 @@ local_flush_tlb_all (void)
 	ia64_srlz_i();			/* srlz.i implies srlz.d */
 }
 
-void
-flush_tlb_range (struct vm_area_struct *vma, unsigned long start,
+static void
+__flush_tlb_range (struct vm_area_struct *vma, unsigned long start,
 		 unsigned long end)
 {
 	struct mm_struct *mm = vma->vm_mm;
@@ -335,6 +335,25 @@ flush_tlb_range (struct vm_area_struct *
 	preempt_enable();
 	ia64_srlz_i();			/* srlz.i implies srlz.d */
 }
+
+void flush_tlb_range(struct vm_area_struct *vma,
+		unsigned long start, unsigned long end)
+{
+	if (unlikely(end - start >= 1024*1024*1024*1024UL
+			|| REGION_NUMBER(start) != REGION_NUMBER(end - 1))) {
+		/*
+		 * If we flush more than a tera-byte or across regions, we're
+		 * probably better off just flushing the entire TLB(s).  This
+		 * should be very rare and is not worth optimizing for.
+		 */
+		flush_tlb_all();
+	} else {
+		/* flush the address range from the tlb */
+		__flush_tlb_range(vma, start, end);
+		/* flush the virt. page-table area mapping the addr range */
+		__flush_tlb_range(vma, ia64_thash(start), ia64_thash(end));
+	}
+}
 EXPORT_SYMBOL(flush_tlb_range);
 
 void ia64_tlb_init(void)
Acked-by on v6 9/18
#15 lore raw

Re: [PATCH v6 09/18] ia64/tlb: Conver to generic mmu_gather

Quoted 8 lines · Peter Zijlstra

Generic mmu_gather provides everything ia64 needs (range tracking).

Cc Will Deacon <[email protected]>
Cc "Aneesh Kumar K.V" <[email protected]>
Cc Andrew Morton <[email protected]>
Cc Nick Piggin <[email protected]>
Cc Tony Luck <[email protected]>
Quoted 5 lines
---
 arch/ia64/include/asm/tlb.h      |  256 ---------------------------------------
 arch/ia64/include/asm/tlbflush.h |   25 +++
 arch/ia64/mm/tlb.c               |   23 +++
 3 files changed, 47 insertions(+), 257 deletions(-)

Typo in subject (s/Conver/Convert) but other than that this looks sensible:

Will

#16 lore raw

Re: [PATCH v6 09/18] ia64/tlb: Conver to generic mmu_gather

In reply to Peter Zijlstra
Quoted 342 lines · Peter Zijlstra

Generic mmu_gather provides everything ia64 needs (range tracking).

Cc Will Deacon <[email protected]>
Cc "Aneesh Kumar K.V" <[email protected]>
Cc Andrew Morton <[email protected]>
Cc Nick Piggin <[email protected]>
Cc Tony Luck <[email protected]>
---
 arch/ia64/include/asm/tlb.h      |  256 ---------------------------------------
 arch/ia64/include/asm/tlbflush.h |   25 +++
 arch/ia64/mm/tlb.c               |   23 +++
 3 files changed, 47 insertions(+), 257 deletions(-)
arch/ia64/include/asm/tlb.h +1 -255
@@ -47,262 +47,8 @@
 #include <asm/tlbflush.h>
 #include <asm/machvec.h>
 
-/*
- * If we can't allocate a page to make a big batch of page pointers
- * to work on, then just handle a few from the on-stack structure.
- */
-#define        IA64_GATHER_BUNDLE      8
-
-struct mmu_gather {
-       struct mm_struct        *mm;
-       unsigned int            nr;
-       unsigned int            max;
-       unsigned char           fullmm;         /* non-zero means full mm flush */
-       unsigned char           need_flush;     /* really unmapped some PTEs? */
-       unsigned long           start, end;
-       unsigned long           start_addr;
-       unsigned long           end_addr;
-       struct page             **pages;
-       struct page             *local[IA64_GATHER_BUNDLE];
-};
-
-struct ia64_tr_entry {
-       u64 ifa;
-       u64 itir;
-       u64 pte;
-       u64 rr;
-}; /*Record for tr entry!*/
-
-extern int ia64_itr_entry(u64 target_mask, u64 va, u64 pte, u64 log_size);
-extern void ia64_ptr_entry(u64 target_mask, int slot);
-
-extern struct ia64_tr_entry *ia64_idtrs[NR_CPUS];
-
-/*
- region register macros
-*/
-#define RR_TO_VE(val)   (((val) >> 0) & 0x0000000000000001)
-#define RR_VE(val)     (((val) & 0x0000000000000001) << 0)
-#define RR_VE_MASK     0x0000000000000001L
-#define RR_VE_SHIFT    0
-#define RR_TO_PS(val)  (((val) >> 2) & 0x000000000000003f)
-#define RR_PS(val)     (((val) & 0x000000000000003f) << 2)
-#define RR_PS_MASK     0x00000000000000fcL
-#define RR_PS_SHIFT    2
-#define RR_RID_MASK    0x00000000ffffff00L
-#define RR_TO_RID(val)         ((val >> 8) & 0xffffff)
-
-static inline void
-ia64_tlb_flush_mmu_tlbonly(struct mmu_gather *tlb, unsigned long start, unsigned long end)
-{
-       tlb->need_flush = 0;
-
-       if (tlb->fullmm) {
-               /*
-                * Tearing down the entire address space.  This happens both as a result
-                * of exit() and execve().  The latter case necessitates the call to
-                * flush_tlb_mm() here.
-                */
-               flush_tlb_mm(tlb->mm);
-       } else if (unlikely (end - start >= 1024*1024*1024*1024UL
-                            || REGION_NUMBER(start) != REGION_NUMBER(end - 1)))
-       {
-               /*
-                * If we flush more than a tera-byte or across regions, we're probably
-                * better off just flushing the entire TLB(s).  This should be very rare
-                * and is not worth optimizing for.
-                */
-               flush_tlb_all();
-       } else {
-               /*
-                * flush_tlb_range() takes a vma instead of a mm pointer because
-                * some architectures want the vm_flags for ITLB/DTLB flush.
-                */
-               struct vm_area_struct vma = TLB_FLUSH_VMA(tlb->mm, 0);
-
-               /* flush the address range from the tlb: */
-               flush_tlb_range(&vma, start, end);
-               /* now flush the virt. page-table area mapping the address range: */
-               flush_tlb_range(&vma, ia64_thash(start), ia64_thash(end));
-       }
-
-}
-
-static inline void
-ia64_tlb_flush_mmu_free(struct mmu_gather *tlb)
-{
-       unsigned long i;
-       unsigned int nr;
-
-       /* lastly, release the freed pages */
-       nr = tlb->nr;
-
-       tlb->nr = 0;
-       tlb->start_addr = ~0UL;
-       for (i = 0; i < nr; ++i)
-               free_page_and_swap_cache(tlb->pages[i]);
-}
-
-/*
- * Flush the TLB for address range START to END and, if not in fast mode, release the
- * freed pages that where gathered up to this point.
- */
-static inline void
-ia64_tlb_flush_mmu (struct mmu_gather *tlb, unsigned long start, unsigned long end)
-{
-       if (!tlb->need_flush)
-               return;
-       ia64_tlb_flush_mmu_tlbonly(tlb, start, end);
-       ia64_tlb_flush_mmu_free(tlb);
-}
-
-static inline void __tlb_alloc_page(struct mmu_gather *tlb)
-{
-       unsigned long addr = __get_free_pages(GFP_NOWAIT | __GFP_NOWARN, 0);
-
-       if (addr) {
-               tlb->pages = (void *)addr;
-               tlb->max = PAGE_SIZE / sizeof(void *);
-       }
-}
-
-
-static inline void
-arch_tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,
-                       unsigned long start, unsigned long end)
-{
-       tlb->mm = mm;
-       tlb->max = ARRAY_SIZE(tlb->local);
-       tlb->pages = tlb->local;
-       tlb->nr = 0;
-       tlb->fullmm = !(start | (end+1));
-       tlb->start = start;
-       tlb->end = end;
-       tlb->start_addr = ~0UL;
-}
-
-/*
- * Called at the end of the shootdown operation to free up any resources that were
- * collected.
- */
-static inline void
-arch_tlb_finish_mmu(struct mmu_gather *tlb,
-                       unsigned long start, unsigned long end, bool force)
-{
-       if (force)
-               tlb->need_flush = 1;
-       /*
-        * Note: tlb->nr may be 0 at this point, so we can't rely on tlb->start_addr and
-        * tlb->end_addr.
-        */
-       ia64_tlb_flush_mmu(tlb, start, end);
-
-       /* keep the page table cache within bounds */
-       check_pgt_cache();
-
-       if (tlb->pages != tlb->local)
-               free_pages((unsigned long)tlb->pages, 0);
-}
-
-/*
- * Logically, this routine frees PAGE.  On MP machines, the actual freeing of the page
- * must be delayed until after the TLB has been flushed (see comments at the beginning of
- * this file).
- */
-static inline bool __tlb_remove_page(struct mmu_gather *tlb, struct page *page)
-{
-       tlb->need_flush = 1;
-
-       if (!tlb->nr && tlb->pages == tlb->local)
-               __tlb_alloc_page(tlb);
-
-       tlb->pages[tlb->nr++] = page;
-       VM_WARN_ON(tlb->nr > tlb->max);
-       if (tlb->nr == tlb->max)
-               return true;
-       return false;
-}
-
-static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
-{
-       ia64_tlb_flush_mmu_tlbonly(tlb, tlb->start_addr, tlb->end_addr);
-}
-
-static inline void tlb_flush_mmu_free(struct mmu_gather *tlb)
-{
-       ia64_tlb_flush_mmu_free(tlb);
-}
-
-static inline void tlb_flush_mmu(struct mmu_gather *tlb)
-{
-       ia64_tlb_flush_mmu(tlb, tlb->start_addr, tlb->end_addr);
-}
-
-static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
-{
-       if (__tlb_remove_page(tlb, page))
-               tlb_flush_mmu(tlb);
-}
-
-static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
-                                         struct page *page, int page_size)
-{
-       return __tlb_remove_page(tlb, page);
-}
-
-static inline void tlb_remove_page_size(struct mmu_gather *tlb,
-                                       struct page *page, int page_size)
-{
-       return tlb_remove_page(tlb, page);
-}
-
-/*
- * Remove TLB entry for PTE mapped at virtual address ADDRESS.  This is called for any
- * PTE, not just those pointing to (normal) physical memory.
- */
-static inline void
-__tlb_remove_tlb_entry (struct mmu_gather *tlb, pte_t *ptep, unsigned long address)
-{
-       if (tlb->start_addr == ~0UL)
-               tlb->start_addr = address;
-       tlb->end_addr = address + PAGE_SIZE;
-}
-
 #define tlb_migrate_finish(mm) platform_tlb_migrate_finish(mm)
 
-#define tlb_start_vma(tlb, vma)                        do { } while (0)
-#define tlb_end_vma(tlb, vma)                  do { } while (0)
-
-#define tlb_remove_tlb_entry(tlb, ptep, addr)          \
-do {                                                   \
-       tlb->need_flush = 1;                            \
-       __tlb_remove_tlb_entry(tlb, ptep, addr);        \
-} while (0)
-
-#define tlb_remove_huge_tlb_entry(h, tlb, ptep, address)       \
-       tlb_remove_tlb_entry(tlb, ptep, address)
-
-static inline void tlb_change_page_size(struct mmu_gather *tlb,
-                                                    unsigned int page_size)
-{
-}
-
-#define pte_free_tlb(tlb, ptep, address)               \
-do {                                                   \
-       tlb->need_flush = 1;                            \
-       __pte_free_tlb(tlb, ptep, address);             \
-} while (0)
-
-#define pmd_free_tlb(tlb, ptep, address)               \
-do {                                                   \
-       tlb->need_flush = 1;                            \
-       __pmd_free_tlb(tlb, ptep, address);             \
-} while (0)
-
-#define pud_free_tlb(tlb, pudp, address)               \
-do {                                                   \
-       tlb->need_flush = 1;                            \
-       __pud_free_tlb(tlb, pudp, address);             \
-} while (0)
+#include <asm-generic/tlb.h>
 
 #endif /* _ASM_IA64_TLB_H */
arch/ia64/include/asm/tlbflush.h +25 -0
@@ -14,6 +14,31 @@
 #include <asm/mmu_context.h>
 #include <asm/page.h>
 
+struct ia64_tr_entry {
+       u64 ifa;
+       u64 itir;
+       u64 pte;
+       u64 rr;
+}; /*Record for tr entry!*/
+
+extern int ia64_itr_entry(u64 target_mask, u64 va, u64 pte, u64 log_size);
+extern void ia64_ptr_entry(u64 target_mask, int slot);
+extern struct ia64_tr_entry *ia64_idtrs[NR_CPUS];
+
+/*
+ region register macros
+*/
+#define RR_TO_VE(val)   (((val) >> 0) & 0x0000000000000001)
+#define RR_VE(val)     (((val) & 0x0000000000000001) << 0)
+#define RR_VE_MASK     0x0000000000000001L
+#define RR_VE_SHIFT    0
+#define RR_TO_PS(val)  (((val) >> 2) & 0x000000000000003f)
+#define RR_PS(val)     (((val) & 0x000000000000003f) << 2)
+#define RR_PS_MASK     0x00000000000000fcL
+#define RR_PS_SHIFT    2
+#define RR_RID_MASK    0x00000000ffffff00L
+#define RR_TO_RID(val)         ((val >> 8) & 0xffffff)
+
 /*
  * Now for some TLB flushing routines.  This is the kind of stuff that
  * can be very expensive, so try to avoid them whenever possible.
arch/ia64/mm/tlb.c +12 -2
@@ -297,8 +297,8 @@ local_flush_tlb_all (void)
        ia64_srlz_i();                  /* srlz.i implies srlz.d */
 }
 
-void
-flush_tlb_range (struct vm_area_struct *vma, unsigned long start,
+static void
+__flush_tlb_range (struct vm_area_struct *vma, unsigned long start,
                 unsigned long end)
 {
        struct mm_struct *mm = vma->vm_mm;
@@ -335,6 +335,25 @@ flush_tlb_range (struct vm_area_struct *
        preempt_enable();
        ia64_srlz_i();                  /* srlz.i implies srlz.d */
 }
+
+void flush_tlb_range(struct vm_area_struct *vma,
+               unsigned long start, unsigned long end)
+{
+       if (unlikely(end - start >= 1024*1024*1024*1024UL
+                       || REGION_NUMBER(start) != REGION_NUMBER(end - 1))) {
+               /*
+                * If we flush more than a tera-byte or across regions, we're
+                * probably better off just flushing the entire TLB(s).  This
+                * should be very rare and is not worth optimizing for.

arch/ia64/mm/tlb.c flush_tlb_range (struct vm_area_struct * · line 357

Quoted 10 lines
arch/ia64/mm/tlb.c +9 -0
+                */
+               flush_tlb_all();
+       } else {
+               /* flush the address range from the tlb */
+               __flush_tlb_range(vma, start, end);
+               /* flush the virt. page-table area mapping the addr range */
+               __flush_tlb_range(vma, ia64_thash(start), ia64_thash(end));
+       }
+}
 EXPORT_SYMBOL(flush_tlb_range);

Just a minor one,
As this is a public API, I think adding docs might be helpful.

author maintainer
#17 lore raw

PATCH v6 10/18 sh/tlb: Convert SH to generic mmu_gather

In reply to Peter Zijlstra

Generic mmu_gather provides everything SH needs (range tracking and cache coherency).

Cc Will Deacon <[email protected]>
Cc "Aneesh Kumar K.V" <[email protected]>
Cc Andrew Morton <[email protected]>
Cc Nick Piggin <[email protected]>
Cc Yoshinori Sato <[email protected]>
Cc Rich Felker <[email protected]>
---
 arch/sh/include/asm/pgalloc.h |    7 ++
 arch/sh/include/asm/tlb.h     |  130 ------------------------------------------
 2 files changed, 8 insertions(+), 129 deletions(-)
arch/sh/include/asm/pgalloc.h +9 -0
@@ -72,6 +72,15 @@ do {							\
 	tlb_remove_page((tlb), (pte));			\
 } while (0)
 
+#if CONFIG_PGTABLE_LEVELS > 2
+#define __pmd_free_tlb(tlb, pmdp, addr)			\
+do {							\
+	struct page *page = virt_to_page(pmdp);		\
+	pgtable_pmd_page_dtor(page);			\
+	tlb_remove_page((tlb), page);			\
+} while (0);
+#endif
+
 static inline void check_pgt_cache(void)
 {
 	quicklist_trim(QUICK_PT, NULL, 25, 16);
arch/sh/include/asm/tlb.h +1 -129

143 lines not on this page · raw message

#18 lore raw

Re: [PATCH v6 10/18] sh/tlb: Convert SH to generic mmu_gather

Hoi Peter,

Quoted 10 lines · Peter Zijlstra

Generic mmu_gather provides everything SH needs (range tracking and cache coherency).

Cc Will Deacon <[email protected]>
Cc "Aneesh Kumar K.V" <[email protected]>
Cc Andrew Morton <[email protected]>
Cc Nick Piggin <[email protected]>
Cc Yoshinori Sato <[email protected]>
Cc Rich Felker <[email protected]>

I got remote access to an SH7722-based Migo-R again, which spews a long sequence of BUGs during userspace startup. I've bisected this to commit c5b27a889da92f4a ("sh/tlb: Convert SH to generic mmu_gather"). Do you have a clue?
Thanks!

mount: mounting none on /dev failed: No such device

BUG: Bad page state in process grep  pfn:0e8c1
page:8fb50820 count:0 mapcount:2 mapping:8f403480 index:0x0
  (null)
flags: 0xc000000()
raw: 0c000000 00000100 00000200 8f403480 00000000 00000000 00000001 00000000
page dumped because: non-NULL mapping
CPU: 0 PID: 765 Comm: grep Tainted: G        W
5.1.0-rc3-migor-00024-gc5b27a889da92f4a #31
Stack: (0x8e8a7d4c to 0x8e8a8000)

7d40: 8c072e90 8fb50820 8fbf400c 00000000 8c0734bc
7d60: 00000200 8fbf4010 00000001 00000002 8fbf4000 00000002 8fbf400c 8e8a7d9c 7d80: 0030f231 00000100 8e8a7da0 00000000 8c2a8a38 8c2a8c0c 8c2a8a94 8c339fa0 7da0: 8fb052a4 8fb052a4 9165dc07 8c0744ea 00000010 00000000 8c0027ac 00000000 7dc0: 8c073ad0 8e8a7df8 8fb054a0 8fb50980 8c07ab54 000000aa 00000011 8e8b52b4 7de0: 00010000 000000aa 8c2a8a38 8fb50820 8c0027ac 00000000 8fb50824 8fb054a4 7e00: 9165dc07 8c0928e0 8c210e48 8c28900c 8e8a7ecc 8e8a7ee4 8c07a958 00000000 7e20: 8e8b5000 8c09296e 8e8a7ed4 8c08f414 8c28900c 8e8a7e60 8e8a7ecc 00000000 7e40: 9165dc07 8c099d08 8ef16178 8c339f60 00000001 00000000 8c0950bc 8c0027ac 7e60: 8c08b00e 7ba1bfff 40000000 7ba1c000 00000000 7ba1bfff 00000000 ffffffff 7e80: 8c08b1ec 8ef16140 8c08ae58 00000000 8ef16140 7b9fb000 8c094fe4 9165dc07 7ea0: 8c0929e6 8e8a7ecc 00000001 8e8a7ecc 8c092b3e 00100000 00000001 8e8a7ecc 7ec0: 8c090cfe 00000000 8ef16020 8f4b78e0 ffffffff ffffffff 8fb50501 00000001 7ee0: 8e8b5000 8e8b5000 00000000 00000008 8fb74f80 8fb74fa0 8fb74f00 8fb74f20 7f00: 8fb74f40 8fb74f60 8fb76c00 8fb76c20 9165dc07 8c0128a2 00000000 00000000 7f20: 8f4b7918 8f4b78e0 00000000 8f4b78e0 8c016326 8ea34f00 8ea3503c 8ea34f00 7f40: 8ea351bc 00000000 8ea3503c 8ea3507c 8c297c1c 8f4a717c 8f4f37a0 9165dc07 7f60: 8c016a58 7ba1bbe4 00000000 00000000 00000000 8eedd720 8eedd6e0 00000000 7f80: 8c016ac6 00000000 00000071 00000100 8c016abc 8c006242 00000000 00427b38 7fa0: 004c025c 00000000 00427b38 004c025c 000000fc 00000000 000000fc fffff000 7fc0: 00000000 00098d9c 00000000 004ac6f4 7ba1be94 00000000 00000000 7ba1bbe4 7fe0: 7ba1bbe4 00427b4e 0040a978 00000101 004c5450 00000022 ffffd000 00000044

Call trace:
 [<(ptrval)>] free_pcppages_bulk+0x114/0x33c
 [<(ptrval)>] free_unref_page_list+0xae/0x10c
 [<(ptrval)>] arch_local_irq_restore+0x0/0x24
 [<(ptrval)>] free_unref_page_commit.isra.24+0x0/0x74
 [<(ptrval)>] release_pages+0x1fc/0x2d0
 [<(ptrval)>] arch_local_irq_restore+0x0/0x24
 [<(ptrval)>] tlb_flush_mmu_free+0x30/0x4c
 [<(ptrval)>] down_read+0x0/0xc
 [<(ptrval)>] release_pages+0x0/0x2d0
 [<(ptrval)>] tlb_flush_mmu+0x72/0xc0
 [<(ptrval)>] remove_vma+0x0/0x48
 [<(ptrval)>] kmem_cache_free+0x34/0x90
 [<(ptrval)>] unlink_anon_vmas+0xd8/0x150
 [<(ptrval)>] arch_local_irq_restore+0x0/0x24
 [<(ptrval)>] free_pgd_range+0x1b6/0x324
 [<(ptrval)>] free_pgtables+0x70/0xb4
 [<(ptrval)>] free_pgd_range+0x0/0x324
 [<(ptrval)>] unlink_anon_vmas+0x0/0x150
 [<(ptrval)>] arch_tlb_finish_mmu+0x2a/0x74
 [<(ptrval)>] tlb_finish_mmu+0x1a/0x38
 [<(ptrval)>] exit_mmap+0xa2/0x16c
 [<(ptrval)>] mmput+0x2a/0x80
 [<(ptrval)>] do_exit+0x186/0x85c
 [<(ptrval)>] do_group_exit+0x2c/0x90
 [<(ptrval)>] sys_exit_group+0xa/0x10
 [<(ptrval)>] sys_exit_group+0x0/0x10
 [<(ptrval)>] syscall_call+0x18/0x1e

Disabling lock debugging due to kernel taint

BUG: Bad page state in process rcS  pfn:0ee78
page:8fb5bf00 count:0 mapcount:2 mapping:8f403480 index:0x0
  (null)
flags: 0xc000000()
raw: 0c000000 00000100 00000200 8f403480 00000000 00000000 00000001 00000000
page dumped because: non-NULL mapping
CPU: 0 PID: 763 Comm: rcS Tainted: G    B   W
5.1.0-rc3-migor-00024-gc5b27a889da92f4a #31
Stack: (0x8ef55e54 to 0x8ef56000)

5e40: 8c072e90 8fb5bf00 8fbf400c
5e60: 00000000 8c0734bc 00000200 8fbf4010 00000001 00000002 8fbf4000 00000002 5e80: 8fbf400c 00989680 0030f231 00000100 8ef55ea8 00000000 8c2a8a38 00000028 5ea0: 00000000 8c29d340 8fb05344 8fb05344 b362ad67 8c074416 8f4f38e0 8c0027ac 5ec0: 8e9125e0 00000000 00000000 8c0027ac 8fb50760 8c0a59e4 8e9125a0 00000180 5ee0: 00000010 8c0a5a9a 8f4a69e0 8f01ad00 8f4f38e0 8f01b21c 8e9125a0 8c09f526 5f00: 8f414cf0 8f01b120 8f4f38e0 8c0281e0 8c20f7dc 8f4a6cdc 00000000 8f4a6cec 5f20: 8c003c18 00401f80 004c6aa0 00000000 8ef55fa4 8ef55fa4 09000080 8c28900c 5f40: 00000000 b362ad67 8c00bbde 8ef55fe4 8f4b7aa0 00000001 09000000 8f4a6c1c 5f60: 004c6ab4 00000055 8c215f8c ffff8bce 00000004 00000009 00003f10 8f4f38e0 5f80: b362ad67 8c0060f8 00401f80 004c6aa0 00000000 00000000 40000000 00000100 5fa0: 8ef54000 00000000 00000450 00000002 00000006 00000003 00000000 004c6ab4 5fc0: 00000000 7bdf069c 004c61d4 00000003 004c61bc 7bdf0670 004c6aa0 00401f80 5fe0: 7bdf069c 00401f94 0047d1cc 00000001 004c5450 00000043 0000000c 00000044

Call trace:
 [<(ptrval)>] free_pcppages_bulk+0x114/0x33c
 [<(ptrval)>] free_unref_page+0x4a/0x70
 [<(ptrval)>] arch_local_irq_restore+0x0/0x24
 [<(ptrval)>] arch_local_irq_restore+0x0/0x24
 [<(ptrval)>] free_pipe_info+0x60/0x84
 [<(ptrval)>] pipe_release+0x92/0xb8
 [<(ptrval)>] __fput+0x3e/0x120
 [<(ptrval)>] task_work_run+0x78/0xa8
 [<(ptrval)>] _cond_resched+0x0/0x54
 [<(ptrval)>] do_notify_resume+0xd4/0x4c8
 [<(ptrval)>] do_page_fault+0xda/0x2a8
 [<(ptrval)>] resume_userspace+0x0/0x10

BUG: Bad page state in process rcS  pfn:0e8b2
page:8fb50640 count:0 mapcount:1026 mapping:8f403480 index:0x0
  (null)
flags: 0xc000000()
raw: 0c000000 00000100 00000200 8f403480 00000000 00000000 00000401 00000000
page dumped because: non-NULL mapping
CPU: 0 PID: 763 Comm: rcS Tainted: G    B   W
5.1.0-rc3-migor-00024-gc5b27a889da92f4a #31
Stack: (0x8ef55e54 to 0x8ef56000)

5e40: 8c072e90 8fb50640 8fbf400c
5e60: 00000000 8c0734bc 00000200 8fbf4010 00000001 00000001 8fbf4000 00000001 5e80: 8fbf400c 00989680 0030f231 00000100 8ef55ea8 00000000 8c2a8a38 00000028 5ea0: 00000000 8c29d340 8fb05344 8fb05344 b362ad67 8c074416 8f4f38e0 8c0027ac 5ec0: 8e9125e0 00000000 00000000 8c0027ac 8fb50760 8c0a59e4 8e9125a0 00000180 5ee0: 00000010 8c0a5a9a 8f4a69e0 8f01ad00 8f4f38e0 8f01b21c 8e9125a0 8c09f526 5f00: 8f414cf0 8f01b120 8f4f38e0 8c0281e0 8c20f7dc 8f4a6cdc 00000000 8f4a6cec 5f20: 8c003c18 00401f80 004c6aa0 00000000 8ef55fa4 8ef55fa4 09000080 8c28900c 5f40: 00000000 b362ad67 8c00bbde 8ef55fe4 8f4b7aa0 00000001 09000000 8f4a6c1c 5f60: 004c6ab4 00000055 8c215f8c ffff8bce 00000004 00000009 00003f10 8f4f38e0 5f80: b362ad67 8c0060f8 00401f80 004c6aa0 00000000 00000000 40000000 00000100 5fa0: 8ef54000 00000000 00000450 00000002 00000006 00000003 00000000 004c6ab4 5fc0: 00000000 7bdf069c 004c61d4 00000003 004c61bc 7bdf0670 004c6aa0 00401f80 5fe0: 7bdf069c 00401f94 0047d1cc 00000001 004c5450 00000043 0000000c 00000044

Call trace:
 [<(ptrval)>] free_pcppages_bulk+0x114/0x33c
 [<(ptrval)>] free_unref_page+0x4a/0x70
 [<(ptrval)>] arch_local_irq_restore+0x0/0x24
 [<(ptrval)>] arch_local_irq_restore+0x0/0x24
 [<(ptrval)>] free_pipe_info+0x60/0x84
 [<(ptrval)>] pipe_release+0x92/0xb8
 [<(ptrval)>] __fput+0x3e/0x120
 [<(ptrval)>] task_work_run+0x78/0xa8
 [<(ptrval)>] _cond_resched+0x0/0x54
 [<(ptrval)>] do_notify_resume+0xd4/0x4c8
 [<(ptrval)>] do_page_fault+0xda/0x2a8
 [<(ptrval)>] resume_userspace+0x0/0x10

...

Quoted 160 lines
---
 arch/sh/include/asm/pgalloc.h |    7 ++
 arch/sh/include/asm/tlb.h     |  130 ------------------------------------------
 2 files changed, 8 insertions(+), 129 deletions(-)
arch/sh/include/asm/pgalloc.h +9 -0
@@ -72,6 +72,15 @@ do {                                                 \
        tlb_remove_page((tlb), (pte));                  \
 } while (0)
 
+#if CONFIG_PGTABLE_LEVELS > 2
+#define __pmd_free_tlb(tlb, pmdp, addr)                        \
+do {                                                   \
+       struct page *page = virt_to_page(pmdp);         \
+       pgtable_pmd_page_dtor(page);                    \
+       tlb_remove_page((tlb), page);                   \
+} while (0);
+#endif
+
 static inline void check_pgt_cache(void)
 {
        quicklist_trim(QUICK_PT, NULL, 25, 16);
arch/sh/include/asm/tlb.h +1 -124
@@ -11,131 +11,8 @@
 
 #ifdef CONFIG_MMU
 #include <linux/swap.h>
-#include <asm/pgalloc.h>
-#include <asm/tlbflush.h>
-#include <asm/mmu_context.h>
-
-/*
- * TLB handling.  This allows us to remove pages from the page
- * tables, and efficiently handle the TLB issues.
- */
-struct mmu_gather {
-       struct mm_struct        *mm;
-       unsigned int            fullmm;
-       unsigned long           start, end;
-};
 
-static inline void init_tlb_gather(struct mmu_gather *tlb)
-{
-       tlb->start = TASK_SIZE;
-       tlb->end = 0;
-
-       if (tlb->fullmm) {
-               tlb->start = 0;
-               tlb->end = TASK_SIZE;
-       }
-}
-
-static inline void
-arch_tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,
-               unsigned long start, unsigned long end)
-{
-       tlb->mm = mm;
-       tlb->start = start;
-       tlb->end = end;
-       tlb->fullmm = !(start | (end+1));
-
-       init_tlb_gather(tlb);
-}
-
-static inline void
-arch_tlb_finish_mmu(struct mmu_gather *tlb,
-               unsigned long start, unsigned long end, bool force)
-{
-       if (tlb->fullmm || force)
-               flush_tlb_mm(tlb->mm);
-
-       /* keep the page table cache within bounds */
-       check_pgt_cache();
-}
-
-static inline void
-tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, unsigned long address)
-{
-       if (tlb->start > address)
-               tlb->start = address;
-       if (tlb->end < address + PAGE_SIZE)
-               tlb->end = address + PAGE_SIZE;
-}
-
-#define tlb_remove_huge_tlb_entry(h, tlb, ptep, address)       \
-       tlb_remove_tlb_entry(tlb, ptep, address)
-
-/*
- * In the case of tlb vma handling, we can optimise these away in the
- * case where we're doing a full MM flush.  When we're doing a munmap,
- * the vmas are adjusted to only cover the region to be torn down.
- */
-static inline void
-tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
-{
-       if (!tlb->fullmm)
-               flush_cache_range(vma, vma->vm_start, vma->vm_end);
-}
-
-static inline void
-tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
-{
-       if (!tlb->fullmm && tlb->end) {
-               flush_tlb_range(vma, tlb->start, tlb->end);
-               init_tlb_gather(tlb);
-       }
-}
-
-static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
-{
-}
-
-static inline void tlb_flush_mmu_free(struct mmu_gather *tlb)
-{
-}
-
-static inline void tlb_flush_mmu(struct mmu_gather *tlb)
-{
-}
-
-static inline int __tlb_remove_page(struct mmu_gather *tlb, struct page *page)
-{
-       free_page_and_swap_cache(page);
-       return false; /* avoid calling tlb_flush_mmu */
-}
-
-static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
-{
-       __tlb_remove_page(tlb, page);
-}
-
-static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
-                                         struct page *page, int page_size)
-{
-       return __tlb_remove_page(tlb, page);
-}
-
-static inline void tlb_remove_page_size(struct mmu_gather *tlb,
-                                       struct page *page, int page_size)
-{
-       return tlb_remove_page(tlb, page);
-}
-
-static inline void tlb_change_page_size(struct mmu_gather *tlb, unsigned int page_size)
-{
-}
-
-#define pte_free_tlb(tlb, ptep, addr)  pte_free((tlb)->mm, ptep)
-#define pmd_free_tlb(tlb, pmdp, addr)  pmd_free((tlb)->mm, pmdp)
-#define pud_free_tlb(tlb, pudp, addr)  pud_free((tlb)->mm, pudp)
-
-#define tlb_migrate_finish(mm)         do { } while (0)
+#include <asm-generic/tlb.h>
 
 #if defined(CONFIG_CPU_SH4) || defined(CONFIG_SUPERH64)
 extern void tlb_wire_entry(struct vm_area_struct *, unsigned long, pte_t);
@@ -155,11 +32,6 @@ static inline void tlb_unwire_entry(void
 

arch/sh/include/asm/tlb.h static inline void tlb_unwire_entry(void · line 37

Quoted 10 lines
arch/sh/include/asm/tlb.h +0 -5
 #else /* CONFIG_MMU */
 
-#define tlb_start_vma(tlb, vma)                                do { } while (0)
-#define tlb_end_vma(tlb, vma)                          do { } while (0)
-#define __tlb_remove_tlb_entry(tlb, pte, address)      do { } while (0)
-#define tlb_flush(tlb)                                 do { } while (0)
-
 #include <asm-generic/tlb.h>
 
 #endif /* CONFIG_MMU */

Gr{oetje,eeting}s,

Geert

--
Geert Uytterhoeven -- There's lots of Linux beyond ia32 -- [email protected]

In personal conversations with technical people, I call myself a hacker. But when I'm talking to journalists I just say "programmer" or something like that.
-- Linus Torvalds

author maintainer
#19 lore raw

Re: [PATCH v6 10/18] sh/tlb: Convert SH to generic mmu_gather

Quoted 14 lines · Geert Uytterhoeven

Hoi Peter,

Quoted 10 lines · Peter Zijlstra

Generic mmu_gather provides everything SH needs (range tracking and cache coherency).

Cc Will Deacon <[email protected]>
Cc "Aneesh Kumar K.V" <[email protected]>
Cc Andrew Morton <[email protected]>
Cc Nick Piggin <[email protected]>
Cc Yoshinori Sato <[email protected]>
Cc Rich Felker <[email protected]>
Quoted 3 lines

I got remote access to an SH7722-based Migo-R again, which spews a long sequence of BUGs during userspace startup. I've bisected this to commit c5b27a889da92f4a ("sh/tlb: Convert SH to generic mmu_gather").

Whoopsy.. also, is this really the first time anybody booted an SH kernel in over a year ?!?

Quoted 1 line

Do you have a clue?

Does the below help?

arch/sh/include/asm/pgalloc.h +2 -3
index 22d968bfe9bb..73a2c00de6c5 100644
@@ -36,9 +36,8 @@ do {							\
 #if CONFIG_PGTABLE_LEVELS > 2
 #define __pmd_free_tlb(tlb, pmdp, addr)			\
 do {							\
-	struct page *page = virt_to_page(pmdp);		\
-	pgtable_pmd_page_dtor(page);			\
-	tlb_remove_page((tlb), page);			\
+	pgtable_pmd_page_dtor(pmdp);			\
+	tlb_remove_page((tlb), (pmdp));			\
 } while (0);
 #endif
 
#20 lore raw

Re: [PATCH v6 10/18] sh/tlb: Convert SH to generic mmu_gather

Hoi Peter,

Quoted 13 lines · Peter Zijlstra
Quoted 12 lines · Geert Uytterhoeven
Quoted 10 lines · Peter Zijlstra

Generic mmu_gather provides everything SH needs (range tracking and cache coherency).

Cc Will Deacon <[email protected]>
Cc "Aneesh Kumar K.V" <[email protected]>
Cc Andrew Morton <[email protected]>
Cc Nick Piggin <[email protected]>
Cc Yoshinori Sato <[email protected]>
Cc Rich Felker <[email protected]>
Quoted 6 lines
Quoted 3 lines

I got remote access to an SH7722-based Migo-R again, which spews a long sequence of BUGs during userspace startup. I've bisected this to commit c5b27a889da92f4a ("sh/tlb: Convert SH to generic mmu_gather").

Whoopsy.. also, is this really the first time anybody booted an SH kernel in over a year ?!?

Nah, but the v5.4-rc3 I booted recently on qemu -M r2d had CONFIG_PGTABLE_LEVELS=2, so it didn't show the problem.

Quoted 3 lines
Quoted 1 line

Do you have a clue?

Does the below help?

Unfortunately not.

arch/sh/include/asm/pgalloc.h do { \ · line 39

Quoted 12 lines
arch/sh/include/asm/pgalloc.h +1 -3
index 22d968bfe9bb..73a2c00de6c5 100644
@@ -36,9 +36,8 @@ do {                                                  \
 #if CONFIG_PGTABLE_LEVELS > 2
 #define __pmd_free_tlb(tlb, pmdp, addr)                        \
 do {                                                   \
-       struct page *page = virt_to_page(pmdp);         \
-       pgtable_pmd_page_dtor(page);                    \
-       tlb_remove_page((tlb), page);                   \
+       pgtable_pmd_page_dtor(pmdp);                    \

expected ‘struct page *’ but argument is of type ‘pmd_t * {aka struct <anonymous> *}’

arch/sh/include/asm/pgalloc.h do { \ · line 40

Quoted 1 line
arch/sh/include/asm/pgalloc.h +1 -0
+       tlb_remove_page((tlb), (pmdp));                 \

likewise

arch/sh/include/asm/pgalloc.h do { \ · line 42

Quoted 2 lines
arch/sh/include/asm/pgalloc.h +0 -0
 } while (0);
 #endif

Gr{oetje,eeting}s,

Geert

Signature · 5 lines
Geert Uytterhoeven -- There's lots of Linux beyond ia32 -- [email protected]

In personal conversations with technical people, I call myself a hacker. But
when I'm talking to journalists I just say "programmer" or something like that.
                                -- Linus Torvalds
#21 lore raw

Re: [PATCH v6 10/18] sh/tlb: Convert SH to generic mmu_gather

Quoted 20 lines · Geert Uytterhoeven

Hoi Peter,

Quoted 17 lines · Peter Zijlstra
Quoted 15 lines · Geert Uytterhoeven
Quoted 10 lines · Peter Zijlstra

Generic mmu_gather provides everything SH needs (range tracking and cache coherency).

Cc Will Deacon <[email protected]>
Cc "Aneesh Kumar K.V" <[email protected]>
Cc Andrew Morton <[email protected]>
Cc Nick Piggin <[email protected]>
Cc Yoshinori Sato <[email protected]>
Cc Rich Felker <[email protected]>

I got remote access to an SH7722-based Migo-R again, which spews a long sequence of BUGs during userspace startup. I've bisected this to commit c5b27a889da92f4a ("sh/tlb: Convert SH to generic mmu_gather").

Quoted 6 lines
Quoted 2 lines

Whoopsy.. also, is this really the first time anybody booted an SH kernel in over a year ?!?

Nah, but the v5.4-rc3 I booted recently on qemu -M r2d had CONFIG_PGTABLE_LEVELS=2, so it didn't show the problem.

Guess that explains why I do not see the problem with my qemu boots. I use rts7751r2dplus_defconfig. Is it possible to reproduce the problem with qemu ? I don't think so, but maybe I am missing something.

Guenter

Quoted 33 lines
Quoted 3 lines
Quoted 1 line

Do you have a clue?

Does the below help?

Unfortunately not.

arch/sh/include/asm/pgalloc.h do { \ · line 39

Quoted 12 lines
arch/sh/include/asm/pgalloc.h +1 -3
index 22d968bfe9bb..73a2c00de6c5 100644
@@ -36,9 +36,8 @@ do {                                                  \
  #if CONFIG_PGTABLE_LEVELS > 2
  #define __pmd_free_tlb(tlb, pmdp, addr)                        \
  do {                                                   \
-       struct page *page = virt_to_page(pmdp);         \
-       pgtable_pmd_page_dtor(page);                    \
-       tlb_remove_page((tlb), page);                   \
+       pgtable_pmd_page_dtor(pmdp);                    \

expected ‘struct page *’ but argument is of type ‘pmd_t * {aka struct <anonymous> *}’

arch/sh/include/asm/pgalloc.h do { \ · line 40

Quoted 1 line
arch/sh/include/asm/pgalloc.h +1 -0
+       tlb_remove_page((tlb), (pmdp));                 \

likewise

arch/sh/include/asm/pgalloc.h do { \ · line 42

Quoted 2 lines
arch/sh/include/asm/pgalloc.h +0 -0
  } while (0);
  #endif

Gr{oetje,eeting}s,

Geert

#22 lore raw

Re: [PATCH v6 10/18] sh/tlb: Convert SH to generic mmu_gather

Hi Günter,

Quoted 14 lines · Guenter Roeck
Quoted 13 lines · Geert Uytterhoeven
Quoted 11 lines · Peter Zijlstra
Quoted 7 lines · Geert Uytterhoeven
Quoted 2 lines · Peter Zijlstra

Generic mmu_gather provides everything SH needs (range tracking and cache coherency).

I got remote access to an SH7722-based Migo-R again, which spews a long sequence of BUGs during userspace startup. I've bisected this to commit c5b27a889da92f4a ("sh/tlb: Convert SH to generic mmu_gather").

Whoopsy.. also, is this really the first time anybody booted an SH kernel in over a year ?!?

Quoted 7 lines
Quoted 3 lines

Nah, but the v5.4-rc3 I booted recently on qemu -M r2d had CONFIG_PGTABLE_LEVELS=2, so it didn't show the problem.

Guess that explains why I do not see the problem with my qemu boots. I use rts7751r2dplus_defconfig. Is it possible to reproduce the problem with qemu ? I don't think so, but maybe I am missing something.

Qemu seems to support r2d and shix only.
For the latter, the website pointed to by the qemu sources no longer exists. But according to those sources, it's also sh7750-based, so no luck.

Gr{oetje,eeting}s,

Geert

Signature · 5 lines
Geert Uytterhoeven -- There's lots of Linux beyond ia32 -- [email protected]

In personal conversations with technical people, I call myself a hacker. But
when I'm talking to journalists I just say "programmer" or something like that.
                                -- Linus Torvalds
#23 lore raw

Re: [PATCH v6 10/18] sh/tlb: Convert SH to generic mmu_gather

Quoted 6 lines · Geert Uytterhoeven

Hi Günter,

Quoted 4 lines
Quoted 3 lines

Nah, but the v5.4-rc3 I booted recently on qemu -M r2d had CONFIG_PGTABLE_LEVELS=2, so it didn't show the problem.

Quoted 8 lines
Quoted 3 lines

Guess that explains why I do not see the problem with my qemu boots. I use rts7751r2dplus_defconfig. Is it possible to reproduce the problem with qemu ? I don't think so, but maybe I am missing something.

Qemu seems to support r2d and shix only.
For the latter, the website pointed to by the qemu sources no longer exists. But according to those sources, it's also sh7750-based, so no luck.

Oh, well, worth asking. Thanks for the feedback.

Guenter

author maintainer
#24 lore raw

Re: [PATCH v6 10/18] sh/tlb: Convert SH to generic mmu_gather

In reply to Geert Uytterhoeven
Quoted 20 lines · Geert Uytterhoeven
Quoted 1 line

Does the below help?

Unfortunately not.

arch/sh/include/asm/pgalloc.h do { \ · line 39

Quoted 12 lines
arch/sh/include/asm/pgalloc.h +1 -3
index 22d968bfe9bb..73a2c00de6c5 100644
@@ -36,9 +36,8 @@ do {                                                  \
 #if CONFIG_PGTABLE_LEVELS > 2
 #define __pmd_free_tlb(tlb, pmdp, addr)                        \
 do {                                                   \
-       struct page *page = virt_to_page(pmdp);         \
-       pgtable_pmd_page_dtor(page);                    \
-       tlb_remove_page((tlb), page);                   \
+       pgtable_pmd_page_dtor(pmdp);                    \

expected ‘struct page *’ but argument is of type ‘pmd_t * {aka struct <anonymous> *}’

Quoted 3 lines

arch/sh/include/asm/pgalloc.h do { \ · line 40

Quoted 1 line
arch/sh/include/asm/pgalloc.h +1 -0
+       tlb_remove_page((tlb), (pmdp));                 \

likewise

Duh.. clearly I misplaced my SH cross compiler. Let me go find it.

Also, looking at pgtable.c the pmd_t* actually comes from a kmemcach() and should probably use pmd_free() (which is what the old code did too).

Also, since SH doesn't have ARCH_ENABLE_SPLIT_PMD_PTLOCK, it will never need pgtable_pmd_page_dtor().

The below seems to build se7722_defconfig using sh4-linux-. That is, the build fails, on 'node_reclaim_distance', not pgtable stuff.

Does this fare better?

---
 arch/sh/include/asm/pgalloc.h | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)
arch/sh/include/asm/pgalloc.h +1 -3
index 22d968bfe9bb..c910e5bcde62 100644
@@ -36,9 +36,7 @@ do {							\
 #if CONFIG_PGTABLE_LEVELS > 2
 #define __pmd_free_tlb(tlb, pmdp, addr)			\
 do {							\
-	struct page *page = virt_to_page(pmdp);		\
-	pgtable_pmd_page_dtor(page);			\
-	tlb_remove_page((tlb), page);			\
+	pmd_free((tlb)->mm, (pmdp));			\
 } while (0);
 #endif
 
maintainer Tested-by
#25 lore raw

Re: [PATCH v6 10/18] sh/tlb: Convert SH to generic mmu_gather

Hoi Peter,

Quoted 30 lines · Peter Zijlstra
Quoted 20 lines · Geert Uytterhoeven
Quoted 1 line

Does the below help?

Unfortunately not.

arch/sh/include/asm/pgalloc.h do { \ · line 39

Quoted 12 lines
arch/sh/include/asm/pgalloc.h +1 -3
index 22d968bfe9bb..73a2c00de6c5 100644
@@ -36,9 +36,8 @@ do {                                                  \
 #if CONFIG_PGTABLE_LEVELS > 2
 #define __pmd_free_tlb(tlb, pmdp, addr)                        \
 do {                                                   \
-       struct page *page = virt_to_page(pmdp);         \
-       pgtable_pmd_page_dtor(page);                    \
-       tlb_remove_page((tlb), page);                   \
+       pgtable_pmd_page_dtor(pmdp);                    \

expected ‘struct page *’ but argument is of type ‘pmd_t * {aka struct <anonymous> *}’

Quoted 3 lines

arch/sh/include/asm/pgalloc.h do { \ · line 40

Quoted 1 line
arch/sh/include/asm/pgalloc.h +1 -0
+       tlb_remove_page((tlb), (pmdp));                 \

likewise

Duh.. clearly I misplaced my SH cross compiler. Let me go find it.

Also, looking at pgtable.c the pmd_t* actually comes from a kmemcach() and should probably use pmd_free() (which is what the old code did too).

Quoted 7 lines

Also, since SH doesn't have ARCH_ENABLE_SPLIT_PMD_PTLOCK, it will never need pgtable_pmd_page_dtor().

The below seems to build se7722_defconfig using sh4-linux-. That is, the build fails, on 'node_reclaim_distance', not pgtable stuff.

Does this fare better?

Yes. Migo-R is happy again.

arch/sh/include/asm/pgalloc.h do { \ · line 41

Quoted 12 lines
arch/sh/include/asm/pgalloc.h +1 -3
@@ -36,9 +36,7 @@ do {                                                  \
 #if CONFIG_PGTABLE_LEVELS > 2
 #define __pmd_free_tlb(tlb, pmdp, addr)                        \
 do {                                                   \
-       struct page *page = virt_to_page(pmdp);         \
-       pgtable_pmd_page_dtor(page);                    \
-       tlb_remove_page((tlb), page);                   \
+       pmd_free((tlb)->mm, (pmdp));                    \
 } while (0);
 #endif

Gr{oetje,eeting}s,

Geert

Signature · 5 lines
Geert Uytterhoeven -- There's lots of Linux beyond ia32 -- [email protected]

In personal conversations with technical people, I call myself a hacker. But
when I'm talking to journalists I just say "programmer" or something like that.
                                -- Linus Torvalds
author maintainer
#26 lore raw

Re: [PATCH v6 10/18] sh/tlb: Convert SH to generic mmu_gather

Quoted 7 lines · Geert Uytterhoeven
Quoted 1 line · Peter Zijlstra

Does this fare better?

Yes. Migo-R is happy again.

Quoted 12 lines

arch/sh/include/asm/pgalloc.h do { \ · line 41

Quoted 12 lines
arch/sh/include/asm/pgalloc.h +1 -3
@@ -36,9 +36,7 @@ do {                                                  \
 #if CONFIG_PGTABLE_LEVELS > 2
 #define __pmd_free_tlb(tlb, pmdp, addr)                        \
 do {                                                   \
-       struct page *page = virt_to_page(pmdp);         \
-       pgtable_pmd_page_dtor(page);                    \
-       tlb_remove_page((tlb), page);                   \
+       pmd_free((tlb)->mm, (pmdp));                    \
 } while (0);
 #endif

OK, so I was going to write a Changelog to go with that, but then I realized that while this works and is similar to before the patch, I'm not sure this is in fact correct.

With this on (and also before) we're freeing the PMD before we've done the TLB invalidate, that seems wrong!

Looking at the size of that pmd_cache, that looks to be 30-(12+12-3)+3 == 12, which is exactly 1 page, for PAGE_SIZE_4K, less for the larger pages.

I'm thinking perhaps we should do something like the below instead?

---
 arch/sh/mm/pgtable.c | 16 ++--------------
 1 file changed, 2 insertions(+), 14 deletions(-)
arch/sh/mm/pgtable.c +2 -14
index 5c8f9247c3c2..fac7e822fd0c 100644
@@ -5,9 +5,6 @@
 #define PGALLOC_GFP GFP_KERNEL | __GFP_ZERO
 
 static struct kmem_cache *pgd_cachep;
-#if PAGETABLE_LEVELS > 2
-static struct kmem_cache *pmd_cachep;
-#endif
 
 void pgd_ctor(void *x)
 {
@@ -23,11 +20,6 @@ void pgtable_cache_init(void)
 	pgd_cachep = kmem_cache_create("pgd_cache",
 				       PTRS_PER_PGD * (1<<PTE_MAGNITUDE),
 				       PAGE_SIZE, SLAB_PANIC, pgd_ctor);
-#if PAGETABLE_LEVELS > 2
-	pmd_cachep = kmem_cache_create("pmd_cache",
-				       PTRS_PER_PMD * (1<<PTE_MAGNITUDE),
-				       PAGE_SIZE, SLAB_PANIC, NULL);
-#endif
 }
 
 pgd_t *pgd_alloc(struct mm_struct *mm)
@@ -48,11 +40,7 @@ void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
 
 pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address)
 {
-	return kmem_cache_alloc(pmd_cachep, PGALLOC_GFP);
-}
-
-void pmd_free(struct mm_struct *mm, pmd_t *pmd)
-{
-	kmem_cache_free(pmd_cachep, pmd);
+	BUILD_BUG_ON(PTRS_PER_PMD * (1<<PTE_MAGNITUDE) <= PAGE_SIZE);
+	return (pmd_t *)__get_free_page(PGALLOC_GFP);
 }
 #endif /* PAGETABLE_LEVELS > 2 */
#27 lore raw

Re: [PATCH v6 10/18] sh/tlb: Convert SH to generic mmu_gather

Hoi Peter,

Quoted 24 lines · Peter Zijlstra
Quoted 6 lines · Geert Uytterhoeven
Quoted 1 line · Peter Zijlstra

Does this fare better?

Yes. Migo-R is happy again.

Quoted 12 lines

arch/sh/include/asm/pgalloc.h do { \ · line 41

Quoted 12 lines
arch/sh/include/asm/pgalloc.h +1 -3
@@ -36,9 +36,7 @@ do {                                                  \
 #if CONFIG_PGTABLE_LEVELS > 2
 #define __pmd_free_tlb(tlb, pmdp, addr)                        \
 do {                                                   \
-       struct page *page = virt_to_page(pmdp);         \
-       pgtable_pmd_page_dtor(page);                    \
-       tlb_remove_page((tlb), page);                   \
+       pmd_free((tlb)->mm, (pmdp));                    \
 } while (0);
 #endif

OK, so I was going to write a Changelog to go with that, but then I realized that while this works and is similar to before the patch, I'm not sure this is in fact correct.

Quoted 8 lines

With this on (and also before) we're freeing the PMD before we've done the TLB invalidate, that seems wrong!

Looking at the size of that pmd_cache, that looks to be 30-(12+12-3)+3 == 12, which is exactly 1 page, for PAGE_SIZE_4K, less for the larger pages.

I'm thinking perhaps we should do something like the below instead?

Your advice is better when in close vicinity of an SH cross compiler, though ;-)

Quoted 25 lines
arch/sh/mm/pgtable.c +0 -8
@@ -5,9 +5,6 @@
 #define PGALLOC_GFP GFP_KERNEL | __GFP_ZERO
 
 static struct kmem_cache *pgd_cachep;
-#if PAGETABLE_LEVELS > 2
-static struct kmem_cache *pmd_cachep;
-#endif
 
 void pgd_ctor(void *x)
 {
@@ -23,11 +20,6 @@ void pgtable_cache_init(void)
        pgd_cachep = kmem_cache_create("pgd_cache",
                                       PTRS_PER_PGD * (1<<PTE_MAGNITUDE),
                                       PAGE_SIZE, SLAB_PANIC, pgd_ctor);
-#if PAGETABLE_LEVELS > 2
-       pmd_cachep = kmem_cache_create("pmd_cache",
-                                      PTRS_PER_PMD * (1<<PTE_MAGNITUDE),
-                                      PAGE_SIZE, SLAB_PANIC, NULL);
-#endif
 }
 
 pgd_t *pgd_alloc(struct mm_struct *mm)

arch/sh/mm/pgtable.c void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) · line 54

Quoted 7 lines
arch/sh/mm/pgtable.c +0 -4
@@ -48,11 +40,7 @@ void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
 
 pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address)
 {
-       return kmem_cache_alloc(pmd_cachep, PGALLOC_GFP);
-}
-
-void pmd_free(struct mm_struct *mm, pmd_t *pmd)

mm/memory.o: In function `__pmd_alloc':
memory.c:(.text+0x1d74): undefined reference to `pmd_free'

arch/sh/mm/pgtable.c void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) · line 43

Quoted 3 lines
arch/sh/mm/pgtable.c +1 -2
-{
-       kmem_cache_free(pmd_cachep, pmd);
+       BUILD_BUG_ON(PTRS_PER_PMD * (1<<PTE_MAGNITUDE) <= PAGE_SIZE);

... > PAGE_SIZE ?

Else it triggers all the time.

arch/sh/mm/pgtable.c void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) · line 46

Quoted 3 lines
arch/sh/mm/pgtable.c +1 -0
+       return (pmd_t *)__get_free_page(PGALLOC_GFP);
 }
 #endif /* PAGETABLE_LEVELS > 2 */

BTW, I'm still running Willy's fix that never made it upstream to kill an ugly boot warning, which also touches this code:
https://patchwork.kernel.org/patch/10549883/#22166333

Gr{oetje,eeting}s,

Geert

Signature · 5 lines
Geert Uytterhoeven -- There's lots of Linux beyond ia32 -- [email protected]

In personal conversations with technical people, I call myself a hacker. But
when I'm talking to journalists I just say "programmer" or something like that.
                                -- Linus Torvalds
#28 lore raw

Re: [PATCH v6 10/18] sh/tlb: Convert SH to generic mmu_gather

In reply to Geert Uytterhoeven

Hi!

Quoted 40 lines · Geert Uytterhoeven
Quoted 8 lines
Quoted 8 lines
Quoted 7 lines
Cc Will Deacon <[email protected]>
Cc "Aneesh Kumar K.V" <[email protected]>
Cc Andrew Morton <[email protected]>
Cc Nick Piggin <[email protected]>
Cc Yoshinori Sato <[email protected]>
Cc Rich Felker <[email protected]>
Quoted 6 lines
Quoted 3 lines

I got remote access to an SH7722-based Migo-R again, which spews a long sequence of BUGs during userspace startup. I've bisected this to commit c5b27a889da92f4a ("sh/tlb: Convert SH to generic mmu_gather").

Whoopsy.. also, is this really the first time anybody booted an SH kernel in over a year ?!?

Nah, but the v5.4-rc3 I booted recently on qemu -M r2d had CONFIG_PGTABLE_LEVELS=2, so it didn't show the problem.

Quoted 3 lines
Quoted 1 line

Do you have a clue?

Does the below help?

Unfortunately not.

arch/sh/include/asm/pgalloc.h do { \ · line 39

Quoted 12 lines
arch/sh/include/asm/pgalloc.h +1 -3
index 22d968bfe9bb..73a2c00de6c5 100644
@@ -36,9 +36,8 @@ do {                                                  \
 #if CONFIG_PGTABLE_LEVELS > 2
 #define __pmd_free_tlb(tlb, pmdp, addr)                        \
 do {                                                   \
-       struct page *page = virt_to_page(pmdp);         \
-       pgtable_pmd_page_dtor(page);                    \
-       tlb_remove_page((tlb), page);                   \
+       pgtable_pmd_page_dtor(pmdp);                    \

expected ‘struct page *’ but argument is of type ‘pmd_t * {aka struct <anonymous> *}’

Quoted 6 lines

arch/sh/include/asm/pgalloc.h do { \ · line 40

Quoted 1 line
arch/sh/include/asm/pgalloc.h +1 -0
+       tlb_remove_page((tlb), (pmdp));                 \

likewise

Quoted 2 lines
arch/sh/include/asm/pgalloc.h +0 -0
 } while (0);
 #endif

Any chance we can have another go at this? The original change

commit c5b27a889da92f4a969d61df77bd4f79ffce57c9 (refs/bisect/bad)

Author: Peter Zijlstra <[email protected]>
Date:   Tue Sep 4 14:45:04 2018 +0200

sh/tlb: Convert SH to generic mmu_gather

Generic mmu_gather provides everything SH needs (range tracking and cache coherency).

breaks systemd for me on my SH-7785LCR [1].

Adrian

Quoted 1 line

[1] https://marc.info/?l=linux-kernel&m=159479951822677&w=2

Adrian

Signature · 4 lines
 .''`.  John Paul Adrian Glaubitz
: :' :  Debian Developer - [email protected]
`. `'   Freie Universitaet Berlin - [email protected]
  `-    GPG: 62FF 8A75 84E0 2956 9546  0006 7426 3B37 F5B5 F913
#29 lore raw

Re: [PATCH v6 10/18] sh/tlb: Convert SH to generic mmu_gather

In reply to Peter Zijlstra
Quoted 15 lines · Peter Zijlstra
Quoted 14 lines · Geert Uytterhoeven

Hoi Peter,

Quoted 10 lines · Peter Zijlstra

Generic mmu_gather provides everything SH needs (range tracking and cache coherency).

Cc Will Deacon <[email protected]>
Cc "Aneesh Kumar K.V" <[email protected]>
Cc Andrew Morton <[email protected]>
Cc Nick Piggin <[email protected]>
Cc Yoshinori Sato <[email protected]>
Cc Rich Felker <[email protected]>
Quoted 6 lines
Quoted 3 lines

I got remote access to an SH7722-based Migo-R again, which spews a long sequence of BUGs during userspace startup. I've bisected this to commit c5b27a889da92f4a ("sh/tlb: Convert SH to generic mmu_gather").

Whoopsy.. also, is this really the first time anybody booted an SH kernel in over a year ?!?

No, but most people running this kind of hardware tend not to upgrade to current kernels on a regular basis.

The j-core guys tested the 5.3 release. I can't find an email about 5.4 so I dunno if that's been tested yet?

I just tested yesterday's git and it works fine with
http://lkml.iu.edu/hypermail/linux/kernel/1912.0/01554.html installed, modulo it _still_ has the suprious stack dump shortly before calling init, which I've complained about on linux-sh and off for a year now?

------------[ cut here ]------------
WARNING: CPU: 0 PID: 1 at mm/slub.c:2451 kmem_cache_free_bulk+0x2c2/0x37c

CPU: 0 PID: 1 Comm: swapper Not tainted 5.4.0 #1
PC is at kmem_cache_free_bulk+0x2c2/0x37c
PR is at kmem_cache_alloc_bulk+0x36/0x1a0

PC : 8c0a6fae SP : 8f829e9c SR : 400080f0
TEA : c0001240
R0 : 8c0a6de4 R1 : 00000100 R2 : 00000100 R3 : 00000000 R4 : 8f8020a0 R5 : 00000dc0 R6 : 8c01d66c R7 : 8fff5180 R8 : 8c011a00 R9 : 8fff5180 R10 : 8c01d66c R11 : 80000000 R12 : 00007fff R13 : 00000dc0 R14 : 8f8020a0
MACH: 0000017a MACL: 0ae4849d GBR : 00000000 PR : 8c0a709e

Call trace:
 [<(ptrval)>] copy_process+0x218/0x1094
 [<(ptrval)>] copy_process+0x7ba/0x1094
 [<(ptrval)>] kmem_cache_alloc_bulk+0x36/0x1a0
 [<(ptrval)>] restore_sigcontext+0x94/0x1b0
 [<(ptrval)>] restore_sigcontext+0x70/0x1b0
 [<(ptrval)>] copy_process+0x218/0x1094
 [<(ptrval)>] sysfs_slab_add+0x106/0x354
 [<(ptrval)>] restore_sigcontext+0x70/0x1b0
 [<(ptrval)>] copy_process+0x218/0x1094
 [<(ptrval)>] copy_process+0x218/0x1094
 [<(ptrval)>] fprop_fraction_single+0x38/0xa4
 [<(ptrval)>] pipe_read+0x7a/0x23c
 [<(ptrval)>] restore_sigcontext+0x70/0x1b0
 [<(ptrval)>] restore_sigcontext+0x94/0x1b0
 [<(ptrval)>] alloc_pipe_info+0x162/0x1c8
 [<(ptrval)>] restore_sigcontext+0x94/0x1b0
 [<(ptrval)>] restore_sigcontext+0x70/0x1b0
 [<(ptrval)>] handle_bad_irq+0x154/0x188
 [<(ptrval)>] raw6_exit_net+0x0/0x14
 [<(ptrval)>] prepare_stack+0xe4/0x2fc
 [<(ptrval)>] sys_sched_get_priority_min+0x18/0x28
 [<(ptrval)>] ndisc_net_exit+0x4/0x24

---[ end trace 6ce4eefeb577b078 ]---

But it's cosmetic...

I haven't got one of the new Turtle boards yet (next time I visit Japan...) and the USB connector broke off my old one, so I haven't got test hardware in my bag to boot it on with me at this coffee shop. So just qemu testing at the moment. The actual j-core deployment environment I'm working on this month is a deeply embedded thing with 128k sram so isn't running Linux. :)

Rob

maintainer
#30 lore raw

Re: [PATCH v6 10/18] sh/tlb: Convert SH to generic mmu_gather

Quoted 20 lines · Rob Landley
Quoted 19 lines · Peter Zijlstra
Quoted 17 lines · Geert Uytterhoeven

Hoi Peter,

Quoted 10 lines · Peter Zijlstra

Generic mmu_gather provides everything SH needs (range tracking and cache coherency).

Cc Will Deacon <[email protected]>
Cc "Aneesh Kumar K.V" <[email protected]>
Cc Andrew Morton <[email protected]>
Cc Nick Piggin <[email protected]>
Cc Yoshinori Sato <[email protected]>
Cc Rich Felker <[email protected]>

I got remote access to an SH7722-based Migo-R again, which spews a long sequence of BUGs during userspace startup. I've bisected this to commit c5b27a889da92f4a ("sh/tlb: Convert SH to generic mmu_gather").

Quoted 8 lines
Quoted 2 lines

Whoopsy.. also, is this really the first time anybody booted an SH kernel in over a year ?!?

No, but most people running this kind of hardware tend not to upgrade to current kernels on a regular basis.

The j-core guys tested the 5.3 release. I can't find an email about 5.4 so I dunno if that's been tested yet?

Being that this code is about mmu, does it affect nommu machines? That's what we've got at present on the J-core side.

Rich

#31 lore raw

Re: [PATCH v6 10/18] sh/tlb: Convert SH to generic mmu_gather

In reply to Rob Landley

Hi Rob!

Quoted 51 lines · Rob Landley

No, but most people running this kind of hardware tend not to upgrade to current kernels on a regular basis.

The j-core guys tested the 5.3 release. I can't find an email about 5.4 so I dunno if that's been tested yet?

I just tested yesterday's git and it works fine with
http://lkml.iu.edu/hypermail/linux/kernel/1912.0/01554.html installed, modulo it _still_ has the suprious stack dump shortly before calling init, which I've complained about on linux-sh and off for a year now?

------------[ cut here ]------------
WARNING: CPU: 0 PID: 1 at mm/slub.c:2451 kmem_cache_free_bulk+0x2c2/0x37c

CPU: 0 PID: 1 Comm: swapper Not tainted 5.4.0 #1
PC is at kmem_cache_free_bulk+0x2c2/0x37c
PR is at kmem_cache_alloc_bulk+0x36/0x1a0

PC : 8c0a6fae SP : 8f829e9c SR : 400080f0
TEA : c0001240
R0 : 8c0a6de4 R1 : 00000100 R2 : 00000100 R3 : 00000000 R4 : 8f8020a0 R5 : 00000dc0 R6 : 8c01d66c R7 : 8fff5180 R8 : 8c011a00 R9 : 8fff5180 R10 : 8c01d66c R11 : 80000000 R12 : 00007fff R13 : 00000dc0 R14 : 8f8020a0
MACH: 0000017a MACL: 0ae4849d GBR : 00000000 PR : 8c0a709e

Call trace:
 [<(ptrval)>] copy_process+0x218/0x1094
 [<(ptrval)>] copy_process+0x7ba/0x1094
 [<(ptrval)>] kmem_cache_alloc_bulk+0x36/0x1a0
 [<(ptrval)>] restore_sigcontext+0x94/0x1b0
 [<(ptrval)>] restore_sigcontext+0x70/0x1b0
 [<(ptrval)>] copy_process+0x218/0x1094
 [<(ptrval)>] sysfs_slab_add+0x106/0x354
 [<(ptrval)>] restore_sigcontext+0x70/0x1b0
 [<(ptrval)>] copy_process+0x218/0x1094
 [<(ptrval)>] copy_process+0x218/0x1094
 [<(ptrval)>] fprop_fraction_single+0x38/0xa4
 [<(ptrval)>] pipe_read+0x7a/0x23c
 [<(ptrval)>] restore_sigcontext+0x70/0x1b0
 [<(ptrval)>] restore_sigcontext+0x94/0x1b0
 [<(ptrval)>] alloc_pipe_info+0x162/0x1c8
 [<(ptrval)>] restore_sigcontext+0x94/0x1b0
 [<(ptrval)>] restore_sigcontext+0x70/0x1b0
 [<(ptrval)>] handle_bad_irq+0x154/0x188
 [<(ptrval)>] raw6_exit_net+0x0/0x14
 [<(ptrval)>] prepare_stack+0xe4/0x2fc
 [<(ptrval)>] sys_sched_get_priority_min+0x18/0x28
 [<(ptrval)>] ndisc_net_exit+0x4/0x24

---[ end trace 6ce4eefeb577b078 ]---
Quoted 1 line

But it's cosmetic...

This is fixed by the following patch [1]:

sh: Fix unneeded constructor in page table allocation

The pgd kmem_cache allocation both specified __GFP_ZERO and had a constructor which makes no sense. Remove __GFP_ZERO and zero the user parts of the pgd explicitly.

Adrian

Quoted 1 line

[1] https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git/commit/?idsc348f31b63d28d176ed290eb1aa2a648f3e51e

Signature · 4 lines
 .''`.  John Paul Adrian Glaubitz
: :' :  Debian Developer - [email protected]
`. `'   Freie Universitaet Berlin - [email protected]
  `-    GPG: 62FF 8A75 84E0 2956 9546  0006 7426 3B37 F5B5 F913
#32 lore raw

Re: [PATCH v6 10/18] sh/tlb: Convert SH to generic mmu_gather

In reply to Peter Zijlstra

Hi!

Quoted 6 lines · Peter Zijlstra
Quoted 3 lines

I got remote access to an SH7722-based Migo-R again, which spews a long sequence of BUGs during userspace startup. I've bisected this to commit c5b27a889da92f4a ("sh/tlb: Convert SH to generic mmu_gather").

Whoopsy.. also, is this really the first time anybody booted an SH kernel in over a year ?!?

I have to admit, I have been very lazy with kernel updates. I have been planning to upgrade to a much more recent release on my boards for a while now, I have just been postponing it since the machines run very stable with the current kernel I am using.

Adrian

Signature · 4 lines
 .''`.  John Paul Adrian Glaubitz
: :' :  Debian Developer - [email protected]
`. `'   Freie Universitaet Berlin - [email protected]
  `-    GPG: 62FF 8A75 84E0 2956 9546  0006 7426 3B37 F5B5 F913
#33 lore raw

Re: [PATCH v6 10/18] sh/tlb: Convert SH to generic mmu_gather

Quoted 7 lines · John Paul Adrian Glaubitz

Hi!

Quoted 4 lines · Peter Zijlstra
Quoted 3 lines

I got remote access to an SH7722-based Migo-R again, which spews a long sequence of BUGs during userspace startup. I've bisected this to commit c5b27a889da92f4a ("sh/tlb: Convert SH to generic mmu_gather").

Quoted 8 lines
Quoted 2 lines

Whoopsy.. also, is this really the first time anybody booted an SH kernel in over a year ?!?

I have to admit, I have been very lazy with kernel updates. I have been planning to upgrade to a much more recent release on my boards for a while now, I have just been postponing it since the machines run very stable with the current kernel I am using.

Hey, if you write a qemu emulation, I'll be happy to run it on a regular basis :-)

Problem is really that the architecture doesn't get as much attention as it needs. The backtrace pointed to by Rob has been seen for a long time, but either there is no one with the knowledge to fix it, or they are all busy with other stuff.

Guenter

Quoted 7 lines

Adrian

Signature · 4 lines
 .''`.  John Paul Adrian Glaubitz
: :' :  Debian Developer - [email protected]
`. `'   Freie Universitaet Berlin - [email protected]
  `-    GPG: 62FF 8A75 84E0 2956 9546  0006 7426 3B37 F5B5 F913
#34 lore raw

Re: [PATCH v6 10/18] sh/tlb: Convert SH to generic mmu_gather

Hi!

Quoted 8 lines · Guenter Roeck
Quoted 5 lines

I have to admit, I have been very lazy with kernel updates. I have been planning to upgrade to a much more recent release on my boards for a while now, I have just been postponing it since the machines run very stable with the current kernel I am using.

Hey, if you write a qemu emulation, I'll be happy to run it on a regular basis :-)

There is working SH emulation in QEMU. I have been wanting to prepare a current Debian/sh4 test image for that purpose for a while now.

For the time being, you can use the image by Aurelien Jarno:

Quoted 6 lines

https://people.debian.org/~aurel32/qemu/sh4/

Problem is really that the architecture doesn't get as much attention as it needs. The backtrace pointed to by Rob has been seen for a long time, but either there is no one with the knowledge to fix it, or they are all busy with other stuff.

I fully agree. I'm mostly busy with the userland stuff, i.e. Debian packages for sh4 and not so much with the kernel.

Adrian

Signature · 4 lines
 .''`.  John Paul Adrian Glaubitz
: :' :  Debian Developer - [email protected]
`. `'   Freie Universitaet Berlin - [email protected]
  `-    GPG: 62FF 8A75 84E0 2956 9546  0006 7426 3B37 F5B5 F913
#35 lore raw

Re: [PATCH v6 10/18] sh/tlb: Convert SH to generic mmu_gather

Quoted 9 lines · John Paul Adrian Glaubitz

Hi!

Quoted 6 lines · Guenter Roeck
Quoted 5 lines

I have to admit, I have been very lazy with kernel updates. I have been planning to upgrade to a much more recent release on my boards for a while now, I have just been postponing it since the machines run very stable with the current kernel I am using.

Quoted 6 lines
Quoted 2 lines

Hey, if you write a qemu emulation, I'll be happy to run it on a regular basis :-)

There is working SH emulation in QEMU. I have been wanting to prepare a current Debian/sh4 test image for that purpose for a while now.

I do run this emulation in my tests. It does not support the affected CPU, unfortunately.

Guenter

Quoted 14 lines

For the time being, you can use the image by Aurelien Jarno:

Quoted 6 lines

https://people.debian.org/~aurel32/qemu/sh4/

Problem is really that the architecture doesn't get as much attention as it needs. The backtrace pointed to by Rob has been seen for a long time, but either there is no one with the knowledge to fix it, or they are all busy with other stuff.

I fully agree. I'm mostly busy with the userland stuff, i.e. Debian packages for sh4 and not so much with the kernel.

Adrian

author maintainer
#36 lore raw

PATCH v6 11/18 um/tlb: Convert to generic mmu_gather

In reply to Peter Zijlstra

Generic mmu_gather provides the simple flush_tlb_range() based range tracking mmu_gather UM needs.

Cc Will Deacon <[email protected]>
Cc "Aneesh Kumar K.V" <[email protected]>
Cc Andrew Morton <[email protected]>
Cc Nick Piggin <[email protected]>
Cc Richard Weinberger <[email protected]>
---
 arch/um/include/asm/tlb.h |  156 ----------------------------------------------
 1 file changed, 2 insertions(+), 154 deletions(-)
arch/um/include/asm/tlb.h +2 -154
@@ -2,160 +2,8 @@
 #ifndef __UM_TLB_H
 #define __UM_TLB_H
 
-#include <linux/pagemap.h>
-#include <linux/swap.h>
-#include <asm/percpu.h>
-#include <asm/pgalloc.h>
 #include <asm/tlbflush.h>
-
-#define tlb_start_vma(tlb, vma) do { } while (0)
-#define tlb_end_vma(tlb, vma) do { } while (0)
-#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
-
-/* struct mmu_gather is an opaque type used by the mm code for passing around
- * any data needed by arch specific code for tlb_remove_page.
- */
-struct mmu_gather {
-	struct mm_struct	*mm;
-	unsigned int		need_flush; /* Really unmapped some ptes? */
-	unsigned long		start;
-	unsigned long		end;
-	unsigned int		fullmm; /* non-zero means full mm flush */
-};
-
-static inline void __tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep,
-					  unsigned long address)
-{
-	if (tlb->start > address)
-		tlb->start = address;
-	if (tlb->end < address + PAGE_SIZE)
-		tlb->end = address + PAGE_SIZE;
-}
-
-static inline void init_tlb_gather(struct mmu_gather *tlb)
-{
-	tlb->need_flush = 0;
-
-	tlb->start = TASK_SIZE;
-	tlb->end = 0;
-
-	if (tlb->fullmm) {
-		tlb->start = 0;
-		tlb->end = TASK_SIZE;
-	}
-}
-
-static inline void
-arch_tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,
-		unsigned long start, unsigned long end)
-{
-	tlb->mm = mm;
-	tlb->start = start;
-	tlb->end = end;
-	tlb->fullmm = !(start | (end+1));
-
-	init_tlb_gather(tlb);
-}
-
-extern void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
-			       unsigned long end);
-
-static inline void
-tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
-{
-	flush_tlb_mm_range(tlb->mm, tlb->start, tlb->end);
-}
-
-static inline void
-tlb_flush_mmu_free(struct mmu_gather *tlb)
-{
-	init_tlb_gather(tlb);
-}
-
-static inline void
-tlb_flush_mmu(struct mmu_gather *tlb)
-{
-	if (!tlb->need_flush)
-		return;
-
-	tlb_flush_mmu_tlbonly(tlb);
-	tlb_flush_mmu_free(tlb);
-}
-
-/* arch_tlb_finish_mmu
- *	Called at the end of the shootdown operation to free up any resources
- *	that were required.
- */
-static inline void
-arch_tlb_finish_mmu(struct mmu_gather *tlb,
-		unsigned long start, unsigned long end, bool force)
-{
-	if (force) {
-		tlb->start = start;
-		tlb->end = end;
-		tlb->need_flush = 1;
-	}
-	tlb_flush_mmu(tlb);
-
-	/* keep the page table cache within bounds */
-	check_pgt_cache();
-}
-
-/* tlb_remove_page
- *	Must perform the equivalent to __free_pte(pte_get_and_clear(ptep)),
- *	while handling the additional races in SMP caused by other CPUs
- *	caching valid mappings in their TLBs.
- */
-static inline int __tlb_remove_page(struct mmu_gather *tlb, struct page *page)
-{
-	tlb->need_flush = 1;
-	free_page_and_swap_cache(page);
-	return false; /* avoid calling tlb_flush_mmu */
-}
-
-static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
-{
-	__tlb_remove_page(tlb, page);
-}
-
-static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
-					  struct page *page, int page_size)
-{
-	return __tlb_remove_page(tlb, page);
-}
-
-static inline void tlb_remove_page_size(struct mmu_gather *tlb,
-					struct page *page, int page_size)
-{
-	return tlb_remove_page(tlb, page);
-}
-
-/**
- * tlb_remove_tlb_entry - remember a pte unmapping for later tlb invalidation.
- *
- * Record the fact that pte's were really umapped in ->need_flush, so we can
- * later optimise away the tlb invalidate.   This helps when userspace is
- * unmapping already-unmapped pages, which happens quite a lot.
- */
-#define tlb_remove_tlb_entry(tlb, ptep, address)		\
-	do {							\
-		tlb->need_flush = 1;				\
-		__tlb_remove_tlb_entry(tlb, ptep, address);	\
-	} while (0)
-
-#define tlb_remove_huge_tlb_entry(h, tlb, ptep, address)	\
-	tlb_remove_tlb_entry(tlb, ptep, address)
-
-static inline void tlb_change_page_size(struct mmu_gather *tlb, unsigned int page_size)
-{
-}
-
-#define pte_free_tlb(tlb, ptep, addr) __pte_free_tlb(tlb, ptep, addr)
-
-#define pud_free_tlb(tlb, pudp, addr) __pud_free_tlb(tlb, pudp, addr)
-
-#define pmd_free_tlb(tlb, pmdp, addr) __pmd_free_tlb(tlb, pmdp, addr)
-
-#define tlb_migrate_finish(mm) do {} while (0)
+#include <asm-generic/cacheflush.h>
+#include <asm-generic/tlb.h>
 
 #endif
author maintainer
#37 lore raw

PATCH v6 12/18 arch/tlb: Clean up simple architectures

In reply to Peter Zijlstra

For the architectures that do not implement their own tlb_flush() but do already use the generic mmu_gather, there are two options:

1) the platform has an efficient flush_tlb_range() and asm-generic/tlb.h doesn't need any overrides at all.

2) the platform lacks an efficient flush_tlb_range() and we select MMU_GATHER_NO_RANGE to minimize full invalidates.

Convert all 'simple' architectures to one of these two forms.

alpha: has no range invalidate -> 2
arc: already used flush_tlb_range() -> 1
c6x: has no range invalidate -> 2
hexagon: has an efficient flush_tlb_range() -> 1
(flush_tlb_mm() is in fact a full range invalidate,
so no need to shoot down everything)
m68k: has inefficient flush_tlb_range() -> 2
microblaze: has no flush_tlb_range() -> 2
mips: has efficient flush_tlb_range() -> 1
(even though it currently seems to use flush_tlb_mm())
nds32: already uses flush_tlb_range() -> 1
nios2: has inefficient flush_tlb_range() -> 2
(no limit on range iteration)
openrisc: has inefficient flush_tlb_range() -> 2
(no limit on range iteration)

parisc:	    already uses flush_tlb_range() -> 1
sparc32:    already uses flush_tlb_range() -> 1
unicore32:  has inefficient flush_tlb_range() -> 2

(no limit on range iteration)
xtensa: has efficient flush_tlb_range() -> 1

Note this also fixes a bug in the existing code for a number platforms. Those platforms that did:

tlb_end_vma() -> if (!full_mm) flush_tlb_*()
tlb_flush -> if (full_mm) flush_tlb_mm()

missed the case of shift_arg_pages(), which doesn't have @fullmm set, nor calls into tlb_*vma(), but still frees page-tables and thus needs an invalidate. The new code handles this by detecting a non-empty range, and either issuing the matching range invalidate or a full invalidate, depending on the capabilities.

Cc Nick Piggin <[email protected]>
Cc "David S. Miller" <[email protected]>
Cc Michal Simek <[email protected]>
Cc Helge Deller <[email protected]>
Cc Greentime Hu <[email protected]>
Cc Richard Henderson <[email protected]>
Cc Andrew Morton <[email protected]>
Cc "Aneesh Kumar K.V" <[email protected]>
Cc Will Deacon <[email protected]>
Cc Ley Foon Tan <[email protected]>
Cc Jonas Bonn <[email protected]>
Cc Mark Salter <[email protected]>
Cc Richard Kuo <[email protected]>
Cc Vineet Gupta <[email protected]>
Cc Paul Burton <[email protected]>
Cc Max Filippov <[email protected]>
Cc Guan Xuetao <[email protected]>
---
 arch/alpha/Kconfig                |    1 +
 arch/alpha/include/asm/tlb.h      |    6 ------
 arch/arc/include/asm/tlb.h        |   23 -----------------------
 arch/c6x/Kconfig                  |    1 +
 arch/c6x/include/asm/tlb.h        |    2 --
 arch/h8300/include/asm/tlb.h      |    2 --
 arch/hexagon/include/asm/tlb.h    |   12 ------------
 arch/m68k/Kconfig                 |    1 +
 arch/m68k/include/asm/tlb.h       |   14 --------------
 arch/microblaze/Kconfig           |    1 +
 arch/microblaze/include/asm/tlb.h |    9 ---------
 arch/mips/include/asm/tlb.h       |    8 --------
 arch/nds32/include/asm/tlb.h      |   10 ----------
 arch/nios2/Kconfig                |    1 +
 arch/nios2/include/asm/tlb.h      |    8 ++++----
 arch/openrisc/Kconfig             |    1 +
 arch/openrisc/include/asm/tlb.h   |    8 ++------
 arch/parisc/include/asm/tlb.h     |   13 -------------
 arch/sparc/include/asm/tlb_32.h   |   13 -------------
 arch/unicore32/Kconfig            |    1 +
 arch/unicore32/include/asm/tlb.h  |    7 +++----
 arch/xtensa/include/asm/tlb.h     |   17 -----------------
 22 files changed, 16 insertions(+), 143 deletions(-)
arch/alpha/Kconfig +1 -0
@@ -36,6 +36,7 @@ config ALPHA
 	select ODD_RT_SIGACTION
 	select OLD_SIGSUSPEND
 	select CPU_NO_EFFICIENT_FFS if !ALPHA_EV67
+	select MMU_GATHER_NO_RANGE
 	help
 	  The Alpha is a 64-bit general-purpose processor designed and
 	  marketed by the Digital Equipment Corporation of blessed memory,
arch/alpha/include/asm/tlb.h +0 -6
@@ -2,12 +2,6 @@
 #ifndef _ALPHA_TLB_H
 #define _ALPHA_TLB_H
 
-#define tlb_start_vma(tlb, vma)			do { } while (0)
-#define tlb_end_vma(tlb, vma)			do { } while (0)
-#define __tlb_remove_tlb_entry(tlb, pte, addr)	do { } while (0)
-
-#define tlb_flush(tlb)				flush_tlb_mm((tlb)->mm)
-
 #include <asm-generic/tlb.h>
 
 #define __pte_free_tlb(tlb, pte, address)		pte_free((tlb)->mm, pte)
arch/arc/include/asm/tlb.h +0 -23
@@ -9,29 +9,6 @@
 #ifndef _ASM_ARC_TLB_H
 #define _ASM_ARC_TLB_H
 
-#define tlb_flush(tlb)				\
-do {						\
-	if (tlb->fullmm)			\
-		flush_tlb_mm((tlb)->mm);	\
-} while (0)
-
-/*
- * This pair is called at time of munmap/exit to flush cache and TLB entries
- * for mappings being torn down.
- * 1) cache-flush part -implemented via tlb_start_vma( ) for VIPT aliasing D$
- * 2) tlb-flush part - implemted via tlb_end_vma( ) flushes the TLB range
- *
- * Note, read http://lkml.org/lkml/2004/1/15/6
- */
-
-#define tlb_end_vma(tlb, vma)						\
-do {									\
-	if (!tlb->fullmm)						\
-		flush_tlb_range(vma, vma->vm_start, vma->vm_end);	\
-} while (0)
-
-#define __tlb_remove_tlb_entry(tlb, ptep, address)
-
 #include <linux/pagemap.h>
 #include <asm-generic/tlb.h>
 
arch/c6x/Kconfig +1 -0
@@ -19,6 +19,7 @@ config C6X
 	select GENERIC_CLOCKEVENTS
 	select MODULES_USE_ELF_RELA
 	select ARCH_NO_COHERENT_DMA_MMAP
+	select MMU_GATHER_NO_RANGE if MMU
 
 config MMU
 	def_bool n
arch/c6x/include/asm/tlb.h +0 -2
@@ -2,8 +2,6 @@
 #ifndef _ASM_C6X_TLB_H
 #define _ASM_C6X_TLB_H
 
-#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
-
 #include <asm-generic/tlb.h>
 
 #endif /* _ASM_C6X_TLB_H */
arch/h8300/include/asm/tlb.h +0 -2
@@ -2,8 +2,6 @@
 #ifndef __H8300_TLB_H__
 #define __H8300_TLB_H__
 
-#define tlb_flush(tlb)	do { } while (0)
-
 #include <asm-generic/tlb.h>
 
 #endif
arch/hexagon/include/asm/tlb.h +0 -12
@@ -22,18 +22,6 @@
 #include <linux/pagemap.h>
 #include <asm/tlbflush.h>
 
-/*
- * We don't need any special per-pte or per-vma handling...
- */
-#define tlb_start_vma(tlb, vma)				do { } while (0)
-#define tlb_end_vma(tlb, vma)				do { } while (0)
-#define __tlb_remove_tlb_entry(tlb, ptep, address)	do { } while (0)
-
-/*
- * .. because we flush the whole mm when it fills up
- */
-#define tlb_flush(tlb)		flush_tlb_mm((tlb)->mm)
-
 #include <asm-generic/tlb.h>
 
 #endif
arch/m68k/Kconfig +1 -0
@@ -27,6 +27,7 @@ config M68K
 	select OLD_SIGSUSPEND3
 	select OLD_SIGACTION
 	select ARCH_DISCARD_MEMBLOCK
+	select MMU_GATHER_NO_RANGE if MMU
 
 config CPU_BIG_ENDIAN
 	def_bool y
arch/m68k/include/asm/tlb.h +0 -14
@@ -2,20 +2,6 @@
 #ifndef _M68K_TLB_H
 #define _M68K_TLB_H
 
-/*
- * m68k doesn't need any special per-pte or
- * per-vma handling..
- */
-#define tlb_start_vma(tlb, vma)	do { } while (0)
-#define tlb_end_vma(tlb, vma)	do { } while (0)
-#define __tlb_remove_tlb_entry(tlb, ptep, address)	do { } while (0)
-
-/*
- * .. because we flush the whole mm when it
- * fills up.
- */
-#define tlb_flush(tlb)		flush_tlb_mm((tlb)->mm)
-
 #include <asm-generic/tlb.h>
 
 #endif /* _M68K_TLB_H */
arch/microblaze/Kconfig +1 -0
@@ -40,6 +40,7 @@ config MICROBLAZE
 	select TRACING_SUPPORT
 	select VIRT_TO_BUS
 	select CPU_NO_EFFICIENT_FFS
+	select MMU_GATHER_NO_RANGE if MMU
 
 # Endianness selection
 choice
arch/microblaze/include/asm/tlb.h +0 -9
@@ -11,16 +11,7 @@
 #ifndef _ASM_MICROBLAZE_TLB_H
 #define _ASM_MICROBLAZE_TLB_H
 
-#define tlb_flush(tlb)	flush_tlb_mm((tlb)->mm)
-
 #include <linux/pagemap.h>
-
-#ifdef CONFIG_MMU
-#define tlb_start_vma(tlb, vma)		do { } while (0)
-#define tlb_end_vma(tlb, vma)		do { } while (0)
-#define __tlb_remove_tlb_entry(tlb, pte, address) do { } while (0)
-#endif
-
 #include <asm-generic/tlb.h>
 
 #endif /* _ASM_MICROBLAZE_TLB_H */
arch/mips/include/asm/tlb.h +0 -8
@@ -5,14 +5,6 @@
 #include <asm/cpu-features.h>
 #include <asm/mipsregs.h>
 
-#define tlb_end_vma(tlb, vma) do { } while (0)
-#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
-
-/*
- * .. because we flush the whole mm when it fills up.
- */
-#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
-
 #define _UNIQUE_ENTRYHI(base, idx)					\
 		(((base) + ((idx) << (PAGE_SHIFT + 1))) |		\
 		 (cpu_has_tlbinv ? MIPS_ENTRYHI_EHINV : 0))
arch/nds32/include/asm/tlb.h +0 -10
@@ -4,16 +4,6 @@
 #ifndef __ASMNDS32_TLB_H
 #define __ASMNDS32_TLB_H
 
-#define tlb_end_vma(tlb,vma)				\
-	do { 						\
-		if(!tlb->fullmm)			\
-			flush_tlb_range(vma, vma->vm_start, vma->vm_end); \
-	} while (0)
-
-#define __tlb_remove_tlb_entry(tlb, pte, addr) do { } while (0)
-
-#define tlb_flush(tlb)	flush_tlb_mm((tlb)->mm)
-
 #include <asm-generic/tlb.h>
 
 #define __pte_free_tlb(tlb, pte, addr)	pte_free((tlb)->mm, pte)
arch/nios2/Kconfig +1 -0
@@ -23,6 +23,7 @@ config NIOS2
 	select USB_ARCH_HAS_HCD if USB_SUPPORT
 	select CPU_NO_EFFICIENT_FFS
 	select ARCH_DISCARD_MEMBLOCK
+	select MMU_GATHER_NO_RANGE if MMU
 
 config GENERIC_CSUM
 	def_bool y
arch/nios2/include/asm/tlb.h +4 -4
@@ -11,12 +11,12 @@
 #ifndef _ASM_NIOS2_TLB_H
 #define _ASM_NIOS2_TLB_H
 
-#define tlb_flush(tlb)	flush_tlb_mm((tlb)->mm)
-
 extern void set_mmu_pid(unsigned long pid);
 
-#define tlb_end_vma(tlb, vma)	do { } while (0)
-#define __tlb_remove_tlb_entry(tlb, ptep, address)	do { } while (0)
+/*
+ * NIOS32 does have flush_tlb_range(), but it lacks a limit and fallback to
+ * full mm invalidation. So use flush_tlb_mm() for everything.
+ */
 
 #include <linux/pagemap.h>
 #include <asm-generic/tlb.h>
arch/openrisc/Kconfig +1 -0
@@ -35,6 +35,7 @@ config OPENRISC
 	select OMPIC if SMP
 	select ARCH_WANT_FRAME_POINTERS
 	select GENERIC_IRQ_MULTI_HANDLER
+	select MMU_GATHER_NO_RANGE if MMU
 
 config CPU_BIG_ENDIAN
 	def_bool y
arch/openrisc/include/asm/tlb.h +2 -6
@@ -20,14 +20,10 @@
 #define __ASM_OPENRISC_TLB_H__
 
 /*
- * or32 doesn't need any special per-pte or
- * per-vma handling..
+ * OpenRISC doesn't have an efficient flush_tlb_range() so use flush_tlb_mm()
+ * for everything.
  */
-#define tlb_start_vma(tlb, vma) do { } while (0)
-#define tlb_end_vma(tlb, vma) do { } while (0)
-#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
 
-#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
 #include <linux/pagemap.h>
 #include <asm-generic/tlb.h>
 
arch/parisc/include/asm/tlb.h +0 -13
@@ -2,19 +2,6 @@
 #ifndef _PARISC_TLB_H
 #define _PARISC_TLB_H
 
-#define tlb_flush(tlb)			\
-do {	if ((tlb)->fullmm)		\
-		flush_tlb_mm((tlb)->mm);\
-} while (0)
-
-#define tlb_end_vma(tlb, vma)	\
-do {	if (!(tlb)->fullmm)	\
-		flush_tlb_range(vma, vma->vm_start, vma->vm_end); \
-} while (0)
-
-#define __tlb_remove_tlb_entry(tlb, pte, address) \
-	do { } while (0)
-
 #include <asm-generic/tlb.h>
 
 #define __pmd_free_tlb(tlb, pmd, addr)	pmd_free((tlb)->mm, pmd)
arch/sparc/include/asm/tlb_32.h +0 -13
@@ -2,19 +2,6 @@
 #ifndef _SPARC_TLB_H
 #define _SPARC_TLB_H
 
-#define tlb_end_vma(tlb, vma) \
-do {								\
-	flush_tlb_range(vma, vma->vm_start, vma->vm_end);	\
-} while (0)
-
-#define __tlb_remove_tlb_entry(tlb, pte, address) \
-	do { } while (0)
-
-#define tlb_flush(tlb) \
-do {								\
-	flush_tlb_mm((tlb)->mm);				\
-} while (0)
-
 #include <asm-generic/tlb.h>
 
 #endif /* _SPARC_TLB_H */
arch/unicore32/Kconfig +1 -0
@@ -20,6 +20,7 @@ config UNICORE32
 	select GENERIC_IOMAP
 	select MODULES_USE_ELF_REL
 	select NEED_DMA_MAP_STATE
+	select MMU_GATHER_NO_RANGE if MMU
 	help
 	  UniCore-32 is 32-bit Instruction Set Architecture,
 	  including a series of low-power-consumption RISC chip
arch/unicore32/include/asm/tlb.h +3 -4
@@ -12,10 +12,9 @@
 #ifndef __UNICORE_TLB_H__
 #define __UNICORE_TLB_H__
 
-#define tlb_start_vma(tlb, vma)				do { } while (0)
-#define tlb_end_vma(tlb, vma)				do { } while (0)
-#define __tlb_remove_tlb_entry(tlb, ptep, address)	do { } while (0)
-#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
+/*
+ * unicore32 lacks an efficient flush_tlb_range(), use flush_tlb_mm().
+ */
 
 #define __pte_free_tlb(tlb, pte, addr)				\
 	do {							\
arch/xtensa/include/asm/tlb.h +0 -17
@@ -14,23 +14,6 @@
 #include <asm/cache.h>
 #include <asm/page.h>
 
-#if (DCACHE_WAY_SIZE <= PAGE_SIZE)
-
-# define tlb_end_vma(tlb,vma)			do { } while (0)
-
-#else
-
-# define tlb_end_vma(tlb, vma)						      \
-	do {								      \
-		if (!tlb->fullmm)					      \
-			flush_tlb_range(vma, vma->vm_start, vma->vm_end);     \
-	} while(0)
-
-#endif
-
-#define __tlb_remove_tlb_entry(tlb,pte,addr)	do { } while (0)
-#define tlb_flush(tlb)				flush_tlb_mm((tlb)->mm)
-
 #include <asm-generic/tlb.h>
 
 #define __pte_free_tlb(tlb, pte, address)	pte_free((tlb)->mm, pte)
author maintainer
#38 lore raw

PATCH v6 13/18 asm-generic/tlb: Introduce HAVE_MMU_GATHER_NO_GATHER

In reply to Peter Zijlstra

Add the Kconfig option HAVE_MMU_GATHER_NO_GATHER to the generic mmu_gather code. If the option is set the mmu_gather will not track individual pages for delayed page free anymore. A platform that enables the option needs to provide its own implementation of the __tlb_remove_page_size function to free pages.

---
 arch/Kconfig              |    3 +
 include/asm-generic/tlb.h |    9 +++
 mm/mmu_gather.c           |  107 +++++++++++++++++++++++++---------------------
 3 files changed, 70 insertions(+), 49 deletions(-)
arch/Kconfig +3 -0
@@ -368,6 +368,9 @@ config HAVE_RCU_TABLE_NO_INVALIDATE
 config HAVE_MMU_GATHER_PAGE_SIZE
 	bool
 
+config HAVE_MMU_GATHER_NO_GATHER
+	bool
+
 config ARCH_HAVE_NMI_SAFE_CMPXCHG
 	bool
 
include/asm-generic/tlb.h +7 -2
@@ -184,6 +184,7 @@ extern void tlb_remove_table(struct mmu_
 
 #endif
 
+#ifndef CONFIG_HAVE_MMU_GATHER_NO_GATHER
 /*
  * If we can't allocate a page to make a big batch of page pointers
  * to work on, then just handle a few from the on-stack structure.
@@ -208,6 +209,10 @@ struct mmu_gather_batch {
  */
 #define MAX_GATHER_BATCH_COUNT	(10000UL/MAX_GATHER_BATCH)
 
+extern bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page,
+				   int page_size);
+#endif
+
 /*
  * struct mmu_gather is an opaque type used by the mm code for passing around
  * any data needed by arch specific code for tlb_remove_page.
@@ -254,6 +259,7 @@ struct mmu_gather {
 
 	unsigned int		batch_count;
 
+#ifndef CONFIG_HAVE_MMU_GATHER_NO_GATHER
 	struct mmu_gather_batch *active;
 	struct mmu_gather_batch	local;
 	struct page		*__pages[MMU_GATHER_BUNDLE];
@@ -261,6 +267,7 @@ struct mmu_gather {
 #ifdef CONFIG_HAVE_MMU_GATHER_PAGE_SIZE
 	unsigned int page_size;
 #endif
+#endif
 };
 
 void arch_tlb_gather_mmu(struct mmu_gather *tlb,
@@ -269,8 +276,6 @@ void tlb_flush_mmu(struct mmu_gather *tl
 void arch_tlb_finish_mmu(struct mmu_gather *tlb,
 			 unsigned long start, unsigned long end, bool force);
 void tlb_flush_mmu_free(struct mmu_gather *tlb);
-extern bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page,
-				   int page_size);
 
 static inline void __tlb_adjust_range(struct mmu_gather *tlb,
 				      unsigned long address,
mm/mmu_gather.c +60 -47

153 lines not on this page · raw message

Acked-by on v6 13/18
#39 lore raw

Re: [PATCH v6 13/18] asm-generic/tlb: Introduce HAVE_MMU_GATHER_NO_GATHER

Quoted 16 lines · Peter Zijlstra

Add the Kconfig option HAVE_MMU_GATHER_NO_GATHER to the generic mmu_gather code. If the option is set the mmu_gather will not track individual pages for delayed page free anymore. A platform that enables the option needs to provide its own implementation of the __tlb_remove_page_size function to free pages.

Quoted 5 lines
---
 arch/Kconfig              |    3 +
 include/asm-generic/tlb.h |    9 +++
 mm/mmu_gather.c           |  107 +++++++++++++++++++++++++---------------------
 3 files changed, 70 insertions(+), 49 deletions(-)

Will

author maintainer
#40 lore raw

PATCH v6 14/18 s390/tlb: convert to generic mmu_gather

In reply to Peter Zijlstra
---
 arch/s390/Kconfig           |    2 
 arch/s390/include/asm/tlb.h |  128 +++++++++++++-------------------------------
 arch/s390/mm/pgalloc.c      |   63 ---------------------
 3 files changed, 42 insertions(+), 151 deletions(-)
arch/s390/Kconfig +2 -0
@@ -163,11 +163,13 @@ config S390
 	select HAVE_PERF_USER_STACK_DUMP
 	select HAVE_MEMBLOCK_NODE_MAP
 	select HAVE_MEMBLOCK_PHYS_MAP
+	select HAVE_MMU_GATHER_NO_GATHER
 	select HAVE_MOD_ARCH_SPECIFIC
 	select HAVE_NOP_MCOUNT
 	select HAVE_OPROFILE
 	select HAVE_PCI
 	select HAVE_PERF_EVENTS
+	select HAVE_RCU_TABLE_FREE
 	select HAVE_REGS_AND_STACK_ACCESS_API
 	select HAVE_RSEQ
 	select HAVE_SYSCALL_TRACEPOINTS
arch/s390/include/asm/tlb.h +39 -89

176 lines not on this page · raw message

arch/s390/mm/pgalloc.c +1 -62

75 lines not on this page · raw message

#41 lore raw

Re: [PATCH v6 14/18] s390/tlb: convert to generic mmu_gather

Quoted 11 lines · Peter Zijlstra
Quoted 5 lines
---
 arch/s390/Kconfig           |    2 
 arch/s390/include/asm/tlb.h |  128 +++++++++++++-------------------------------
 arch/s390/mm/pgalloc.c      |   63 ---------------------
 3 files changed, 42 insertions(+), 151 deletions(-)

-ENOCOMMITMESSAGE ?

Will

author maintainer
#42 lore raw

PATCH v6 15/18 asm-generic/tlb: Remove arch_tlb*_mmu()

In reply to Peter Zijlstra

Now that all architectures are converted to the generic code, remove the arch hooks.

---
 mm/mmu_gather.c |   93 +++++++++++++++++++++++++-------------------------------
 1 file changed, 42 insertions(+), 51 deletions(-)
mm/mmu_gather.c +42 -51
@@ -93,33 +93,6 @@ bool __tlb_remove_page_size(struct mmu_g
 
 #endif /* HAVE_MMU_GATHER_NO_GATHER */
 
-void arch_tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,
-				unsigned long start, unsigned long end)
-{
-	tlb->mm = mm;
-
-	/* Is it from 0 to ~0? */
-	tlb->fullmm     = !(start | (end+1));
-
-#ifndef CONFIG_HAVE_MMU_GATHER_NO_GATHER
-	tlb->need_flush_all = 0;
-	tlb->local.next = NULL;
-	tlb->local.nr   = 0;
-	tlb->local.max  = ARRAY_SIZE(tlb->__pages);
-	tlb->active     = &tlb->local;
-	tlb->batch_count = 0;
-#endif
-
-#ifdef CONFIG_HAVE_RCU_TABLE_FREE
-	tlb->batch = NULL;
-#endif
-#ifdef CONFIG_HAVE_MMU_GATHER_PAGE_SIZE
-	tlb->page_size = 0;
-#endif
-
-	__tlb_reset_range(tlb);
-}
-
 void tlb_flush_mmu_free(struct mmu_gather *tlb)
 {
 #ifdef CONFIG_HAVE_RCU_TABLE_FREE
@@ -136,27 +109,6 @@ void tlb_flush_mmu(struct mmu_gather *tl
 	tlb_flush_mmu_free(tlb);
 }
 
-/* tlb_finish_mmu
- *	Called at the end of the shootdown operation to free up any resources
- *	that were required.
- */
-void arch_tlb_finish_mmu(struct mmu_gather *tlb,
-		unsigned long start, unsigned long end, bool force)
-{
-	if (force) {
-		__tlb_reset_range(tlb);
-		__tlb_adjust_range(tlb, start, end - start);
-	}
-
-	tlb_flush_mmu(tlb);
-
-	/* keep the page table cache within bounds */
-	check_pgt_cache();
-#ifndef CONFIG_HAVE_MMU_GATHER_NO_GATHER
-	tlb_batch_list_free(tlb);
-#endif
-}
-
 #endif /* HAVE_GENERIC_MMU_GATHER */
 
 #ifdef CONFIG_HAVE_RCU_TABLE_FREE
@@ -258,10 +210,40 @@ void tlb_remove_table(struct mmu_gather
 void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,
 			unsigned long start, unsigned long end)
 {
-	arch_tlb_gather_mmu(tlb, mm, start, end);
+	tlb->mm = mm;
+
+	/* Is it from 0 to ~0? */
+	tlb->fullmm     = !(start | (end+1));
+
+#ifndef CONFIG_HAVE_MMU_GATHER_NO_GATHER
+	tlb->need_flush_all = 0;
+	tlb->local.next = NULL;
+	tlb->local.nr   = 0;
+	tlb->local.max  = ARRAY_SIZE(tlb->__pages);
+	tlb->active     = &tlb->local;
+	tlb->batch_count = 0;
+#endif
+
+#ifdef CONFIG_HAVE_RCU_TABLE_FREE
+	tlb->batch = NULL;
+#endif
+#ifdef CONFIG_HAVE_MMU_GATHER_PAGE_SIZE
+	tlb->page_size = 0;
+#endif
+
+	__tlb_reset_range(tlb);
 	inc_tlb_flush_pending(tlb->mm);
 }
 
+/**
+ * tlb_finish_mmu - finish an mmu_gather structure
+ * @tlb: the mmu_gather structure to finish
+ * @start: start of the region that will be removed from the page-table
+ * @end: end of the region that will be removed from the page-table
+ *
+ * Called at the end of the shootdown operation to free up any resources that
+ * were required.
+ */
 void tlb_finish_mmu(struct mmu_gather *tlb,
 		unsigned long start, unsigned long end)
 {
@@ -272,8 +254,17 @@ void tlb_finish_mmu(struct mmu_gather *t
 	 * the TLB by observing pte_none|!pte_dirty, for example so flush TLB
 	 * forcefully if we detect parallel PTE batching threads.
 	 */
-	bool force = mm_tlb_flush_nested(tlb->mm);
+	if (mm_tlb_flush_nested(tlb->mm)) {
+		__tlb_reset_range(tlb);
+		__tlb_adjust_range(tlb, start, end - start);
+	}
 
-	arch_tlb_finish_mmu(tlb, start, end, force);
+	tlb_flush_mmu(tlb);
+
+	/* keep the page table cache within bounds */
+	check_pgt_cache();
+#ifndef CONFIG_HAVE_MMU_GATHER_NO_GATHER
+	tlb_batch_list_free(tlb);
+#endif
 	dec_tlb_flush_pending(tlb->mm);
 }
author maintainer
#43 lore raw

PATCH v6 16/18 asm-generic/tlb: Remove HAVE_GENERIC_MMU_GATHER

In reply to Peter Zijlstra

Since all architectures are now using it, it is redundant.

---
 include/asm-generic/tlb.h |    1 -
 mm/mmu_gather.c           |    4 ----
 2 files changed, 5 deletions(-)
include/asm-generic/tlb.h +0 -1
@@ -142,7 +142,6 @@
  *
  *  Use this if your architecture lacks an efficient flush_tlb_range().
  */
-#define HAVE_GENERIC_MMU_GATHER
 
 #ifdef CONFIG_HAVE_RCU_TABLE_FREE
 /*
mm/mmu_gather.c +0 -4
@@ -11,8 +11,6 @@
 #include <asm/pgalloc.h>
 #include <asm/tlb.h>
 
-#ifdef HAVE_GENERIC_MMU_GATHER
-
 #ifndef CONFIG_HAVE_MMU_GATHER_NO_GATHER
 
 static bool tlb_next_batch(struct mmu_gather *tlb)
@@ -109,8 +107,6 @@ void tlb_flush_mmu(struct mmu_gather *tl
 	tlb_flush_mmu_free(tlb);
 }
 
-#endif /* HAVE_GENERIC_MMU_GATHER */
-
 #ifdef CONFIG_HAVE_RCU_TABLE_FREE
 
 /*
author maintainer
#44 lore raw

PATCH v6 17/18 asm-generic/tlb: Remove tlb_flush_mmu_free()

In reply to Peter Zijlstra

As the comment notes; it is a potentially dangerous operation. Just use tlb_flush_mmu(), that will skip the (double) TLB invalidate if it really isn't needed anyway.

---
 include/asm-generic/tlb.h |   10 +++-------
 mm/memory.c               |    2 +-
 mm/mmu_gather.c           |    2 +-
 3 files changed, 5 insertions(+), 9 deletions(-)
include/asm-generic/tlb.h +3 -7
@@ -67,16 +67,13 @@
  *    call before __tlb_remove_page*() to set the current page-size; implies a
  *    possible tlb_flush_mmu() call.
  *
- *  - tlb_flush_mmu() / tlb_flush_mmu_tlbonly() / tlb_flush_mmu_free()
+ *  - tlb_flush_mmu() / tlb_flush_mmu_tlbonly()
  *
  *    tlb_flush_mmu_tlbonly() - does the TLB invalidate (and resets
  *                              related state, like the range)
  *
- *    tlb_flush_mmu_free() - frees the queued pages; make absolutely
- *			     sure no additional tlb_remove_page()
- *			     calls happen between _tlbonly() and this.
- *
- *    tlb_flush_mmu() - the above two calls.
+ *    tlb_flush_mmu() - in addition to the above TLB invalidate, also frees
+ *			whatever pages are still batched.
  *
  *  - mmu_gather::fullmm
  *
@@ -274,7 +271,6 @@ void arch_tlb_gather_mmu(struct mmu_gath
 void tlb_flush_mmu(struct mmu_gather *tlb);
 void arch_tlb_finish_mmu(struct mmu_gather *tlb,
 			 unsigned long start, unsigned long end, bool force);
-void tlb_flush_mmu_free(struct mmu_gather *tlb);
 
 static inline void __tlb_adjust_range(struct mmu_gather *tlb,
 				      unsigned long address,
mm/memory.c +1 -1
@@ -1155,7 +1155,7 @@ static unsigned long zap_pte_range(struc
 	 */
 	if (force_flush) {
 		force_flush = 0;
-		tlb_flush_mmu_free(tlb);
+		tlb_flush_mmu(tlb);
 		if (addr != end)
 			goto again;
 	}
mm/mmu_gather.c +1 -1
@@ -91,7 +91,7 @@ bool __tlb_remove_page_size(struct mmu_g
 
 #endif /* HAVE_MMU_GATHER_NO_GATHER */
 
-void tlb_flush_mmu_free(struct mmu_gather *tlb)
+static void tlb_flush_mmu_free(struct mmu_gather *tlb)
 {
 #ifdef CONFIG_HAVE_RCU_TABLE_FREE
 	tlb_table_flush(tlb);
author maintainer
#45 lore raw

PATCH v6 18/18 asm-generic/tlb: Remove tlb_table_flush()

In reply to Peter Zijlstra

There are no external users of this API (nor should there be); remove it.

---
 include/asm-generic/tlb.h |    1 -
 mm/mmu_gather.c           |   34 +++++++++++++++++-----------------
 2 files changed, 17 insertions(+), 18 deletions(-)
include/asm-generic/tlb.h +0 -1
@@ -174,7 +174,6 @@ struct mmu_table_batch {
 #define MAX_TABLE_BATCH		\
 	((PAGE_SIZE - sizeof(struct mmu_table_batch)) / sizeof(void *))
 
-extern void tlb_table_flush(struct mmu_gather *tlb);
 extern void tlb_remove_table(struct mmu_gather *tlb, void *table);
 
 #endif
mm/mmu_gather.c +17 -17
@@ -91,22 +91,6 @@ bool __tlb_remove_page_size(struct mmu_g
 
 #endif /* HAVE_MMU_GATHER_NO_GATHER */
 
-static void tlb_flush_mmu_free(struct mmu_gather *tlb)
-{
-#ifdef CONFIG_HAVE_RCU_TABLE_FREE
-	tlb_table_flush(tlb);
-#endif
-#ifndef CONFIG_HAVE_MMU_GATHER_NO_GATHER
-	tlb_batch_pages_flush(tlb);
-#endif
-}
-
-void tlb_flush_mmu(struct mmu_gather *tlb)
-{
-	tlb_flush_mmu_tlbonly(tlb);
-	tlb_flush_mmu_free(tlb);
-}
-
 #ifdef CONFIG_HAVE_RCU_TABLE_FREE
 
 /*
@@ -159,7 +143,7 @@ static void tlb_remove_table_rcu(struct
 	free_page((unsigned long)batch);
 }
 
-void tlb_table_flush(struct mmu_gather *tlb)
+static void tlb_table_flush(struct mmu_gather *tlb)
 {
 	struct mmu_table_batch **batch = &tlb->batch;
 
@@ -191,6 +175,22 @@ void tlb_remove_table(struct mmu_gather
 
 #endif /* CONFIG_HAVE_RCU_TABLE_FREE */
 
+static void tlb_flush_mmu_free(struct mmu_gather *tlb)
+{
+#ifdef CONFIG_HAVE_RCU_TABLE_FREE
+	tlb_table_flush(tlb);
+#endif
+#ifndef CONFIG_HAVE_MMU_GATHER_NO_GATHER
+	tlb_batch_pages_flush(tlb);
+#endif
+}
+
+void tlb_flush_mmu(struct mmu_gather *tlb)
+{
+	tlb_flush_mmu_tlbonly(tlb);
+	tlb_flush_mmu_free(tlb);
+}
+
 /**
  * tlb_gather_mmu - initialize an mmu_gather structure for page-table tear-down
  * @tlb: the mmu_gather structure to initialize