Skip to contentSkip to conversation

Unify TLB gather implementations -v3

PATCH NAK Peter Zijlstra 9 participants 54 messages Jun 27, 2012 – Jul 24, 2012

Subsystems (24)
lore
Share & tools
lore
#0 lore raw

cover Unify TLB gather implementations -v3

Its been a while since I last send this out, but here goes..

There's no arch left over, I finally got s390 converted too. The series is compile tested on:

arm, powerpc64, sparc64, sparc32, s390x, arm, ia64, xtensa

I lack a working toolchain for: sh, avr32
Simply wouldn't build: mips, parisc

---
 arch/Kconfig                         |   16 ++
 arch/alpha/include/asm/tlb.h         |    2 -
 arch/arm/Kconfig                     |    1 +
 arch/arm/include/asm/tlb.h           |  183 ++--------------------
 arch/avr32/Kconfig                   |    1 +
 arch/avr32/include/asm/tlb.h         |   11 --
 arch/blackfin/include/asm/tlb.h      |    6 -
 arch/c6x/include/asm/tlb.h           |    2 -
 arch/cris/include/asm/tlb.h          |    1 -
 arch/frv/include/asm/tlb.h           |    5 -
 arch/h8300/include/asm/tlb.h         |   13 --
 arch/hexagon/include/asm/tlb.h       |    5 -
 arch/ia64/Kconfig                    |    1 +
 arch/ia64/include/asm/tlb.h          |  233 +---------------------------
 arch/ia64/include/asm/tlbflush.h     |   25 +++
 arch/ia64/mm/tlb.c                   |   24 +++-
 arch/m32r/include/asm/tlb.h          |    6 -
 arch/m68k/include/asm/tlb.h          |    6 -
 arch/microblaze/include/asm/tlb.h    |    2 -
 arch/mips/Kconfig                    |    1 +
 arch/mips/include/asm/tlb.h          |   15 --
 arch/mn10300/include/asm/tlb.h       |    5 -
 arch/openrisc/include/asm/tlb.h      |    1 -
 arch/parisc/Kconfig                  |    1 +
 arch/parisc/include/asm/tlb.h        |   15 --
 arch/powerpc/include/asm/tlb.h       |    2 -
 arch/powerpc/mm/hugetlbpage.c        |    4 +-
 arch/powerpc/mm/tlb_hash32.c         |   15 --
 arch/powerpc/mm/tlb_hash64.c         |   14 --
 arch/powerpc/mm/tlb_nohash.c         |    5 -
 arch/s390/Kconfig                    |    1 +
 arch/s390/include/asm/pgalloc.h      |    3 +
 arch/s390/include/asm/pgtable.h      |    1 +
 arch/s390/include/asm/tlb.h          |   71 ++-------
 arch/s390/mm/pgtable.c               |   63 +-------
 arch/score/include/asm/tlb.h         |    1 -
 arch/sh/Kconfig                      |    1 +
 arch/sh/include/asm/tlb.h            |   99 +-----------
 arch/sparc/Kconfig                   |    1 +
 arch/sparc/Makefile                  |    1 +
 arch/sparc/include/asm/tlb_32.h      |   15 --
 arch/sparc/include/asm/tlb_64.h      |    1 -
 arch/sparc/include/asm/tlbflush_64.h |   11 ++
 arch/tile/include/asm/tlb.h          |    1 -
 arch/um/Kconfig.common               |    1 +
 arch/um/include/asm/tlb.h            |  111 +-------------
 arch/um/kernel/tlb.c                 |   13 --
 arch/unicore32/include/asm/tlb.h     |    1 -
 arch/x86/include/asm/tlb.h           |    2 +-
 arch/x86/mm/pgtable.c                |    6 +-
 arch/xtensa/Kconfig                  |    1 +
 arch/xtensa/include/asm/tlb.h        |   24 ---
 arch/xtensa/mm/tlb.c                 |    2 +-
 include/asm-generic/4level-fixup.h   |    2 +-
 include/asm-generic/tlb.h            |  284 +++++++++++++++++++++++++++++-----
 mm/memory.c                          |   54 +++++--
 56 files changed, 415 insertions(+), 977 deletions(-)

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#1 lore raw

PATCH 1/20 mm, x86: Add HAVE_RCU_TABLE_FREE support

Implements optional HAVE_RCU_TABLE_FREE support for x86.

This is useful for things like Xen and KVM where paravirt tlb flush means the software page table walkers like GUP-fast cannot rely on IRQs disabling like regular x86 can.

Cc Nikunj A Dadhania <[email protected]>
Cc Jeremy Fitzhardinge <[email protected]>
Cc Avi Kivity <[email protected]>
---
 arch/x86/include/asm/tlb.h |    1 +
 arch/x86/mm/pgtable.c      |    6 +++---
 include/asm-generic/tlb.h  |    9 +++++++++
 3 files changed, 13 insertions(+), 3 deletions(-)
arch/x86/include/asm/tlb.h +1 -0
@@ -1,6 +1,7 @@
 #ifndef _ASM_X86_TLB_H
 #define _ASM_X86_TLB_H
 
+#define __tlb_remove_table(table) free_page_and_swap_cache(table)
 #define tlb_start_vma(tlb, vma) do { } while (0)
 #define tlb_end_vma(tlb, vma) do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
arch/x86/mm/pgtable.c +3 -3
@@ -51,21 +51,21 @@ void ___pte_free_tlb(struct mmu_gather *
 {
 	pgtable_page_dtor(pte);
 	paravirt_release_pte(page_to_pfn(pte));
-	tlb_remove_page(tlb, pte);
+	tlb_remove_table(tlb, pte);
 }
 
 #if PAGETABLE_LEVELS > 2
 void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd)
 {
 	paravirt_release_pmd(__pa(pmd) >> PAGE_SHIFT);
-	tlb_remove_page(tlb, virt_to_page(pmd));
+	tlb_remove_table(tlb, virt_to_page(pmd));
 }
 
 #if PAGETABLE_LEVELS > 3
 void ___pud_free_tlb(struct mmu_gather *tlb, pud_t *pud)
 {
 	paravirt_release_pud(__pa(pud) >> PAGE_SHIFT);
-	tlb_remove_page(tlb, virt_to_page(pud));
+	tlb_remove_table(tlb, virt_to_page(pud));
 }
 #endif	/* PAGETABLE_LEVELS > 3 */
 #endif	/* PAGETABLE_LEVELS > 2 */
include/asm-generic/tlb.h +9 -0
@@ -19,6 +19,8 @@
 #include <asm/pgalloc.h>
 #include <asm/tlbflush.h>
 
+static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page);
+
 #ifdef CONFIG_HAVE_RCU_TABLE_FREE
 /*
  * Semi RCU freeing of the page directories.
@@ -60,6 +62,13 @@ struct mmu_table_batch {
 extern void tlb_table_flush(struct mmu_gather *tlb);
 extern void tlb_remove_table(struct mmu_gather *tlb, void *table);
 
+#else
+
+static inline void tlb_remove_table(struct mmu_gather *tlb, void *table)
+{
+	tlb_remove_page(tlb, table);
+}
+
 #endif
 
 /*

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#2 lore raw

PATCH 2/20 mm: Add optional TLB flush to generic RCU page-table freeing

In reply to Peter Zijlstra

From: Nikunj A. Dadhania <[email protected]>

Certain architectures (viz. x86, arm, s390) have hardware page-table walkers (#PF). So during the RCU page-table teardown process make sure we do a tlb flush of page-table pages on all relevant CPUs to synchronize against hardware walkers, and then free the pages.

Moreover, the (mm_users < 2) condition does not hold good for the above architectures, as the hardware engine is one of the user.

This patch should also make the generic RCU page-table freeing code suitable for s390 again since it fixes the issues raised in cd94154cc6a ("[S390] fix tlb flushing for page table pages").

Cc Martin Schwidefsky <[email protected]>
[ Edited Kconfig bit ]
---
 arch/Kconfig |   13 +++++++++++++
 mm/memory.c  |   23 +++++++++++++++++++++--
 2 files changed, 34 insertions(+), 2 deletions(-)
arch/Kconfig +13 -0
@@ -231,6 +231,19 @@ config HAVE_ARCH_MUTEX_CPU_RELAX
 config HAVE_RCU_TABLE_FREE
 	bool
 
+config HAVE_HW_PAGE_TABLE_WALKS
+	def_bool y
+	depends on HAVE_RCU_TABLE_FREE && !(SPARC64 || PPC)
+	help
+	  An arch should be excluded if it doesn't have hardware page-table
+	  walkers that can (re)populate TLB caches concurrently with us
+	  tearing down page-tables.
+
+	  Both SPARC and PPC are excluded because they have 'external'
+	  hash-table based MMUs which are cleared before we take down the
+	  linux page-table structure. Therefore we don't need to emit
+	  hardware TLB flush instructions before freeing page-table pages.
+
 config ARCH_HAVE_NMI_SAFE_CMPXCHG
 	bool
 
mm/memory.c +21 -2
@@ -329,11 +329,26 @@ static void tlb_remove_table_rcu(struct 
 	free_page((unsigned long)batch);
 }
 
+#ifdef CONFIG_HAVE_HW_PAGE_TABLE_WALKS
+/*
+ * Some architectures (x86, arm, s390) can walk the page tables when
+ * the page-table tear down might be happening. So make sure we flush
+ * the TLBs before freeing the page-table pages.
+ */
+static inline void tlb_table_flush_mmu(struct mmu_gather *tlb)
+{
+	tlb_flush_mmu(tlb);
+}
+#else
+static inline void tlb_table_flush_mmu(struct mmu_gather *tlb) { }
+#endif /* CONFIG_HAVE_HW_PAGE_TABLE_WALKS */
+
 void tlb_table_flush(struct mmu_gather *tlb)
 {
 	struct mmu_table_batch **batch = &tlb->batch;
 
 	if (*batch) {
+		tlb_table_flush_mmu(tlb);
 		call_rcu_sched(&(*batch)->rcu, tlb_remove_table_rcu);
 		*batch = NULL;
 	}
@@ -345,18 +360,22 @@ void tlb_remove_table(struct mmu_gather 
 
 	tlb->need_flush = 1;
 
+#ifndef CONFIG_HAVE_HW_PAGE_TABLE_WALKS
 	/*
-	 * When there's less then two users of this mm there cannot be a
-	 * concurrent page-table walk.
+	 * When there's less then two users of this mm there cannot be
+	 * a concurrent page-table walk for architectures that do not
+	 * have hardware page-table walkers.
 	 */
 	if (atomic_read(&tlb->mm->mm_users) < 2) {
 		__tlb_remove_table(table);
 		return;
 	}
+#endif
 
 	if (*batch == NULL) {
 		*batch = (struct mmu_table_batch *)__get_free_page(GFP_NOWAIT | __GFP_NOWARN);
 		if (*batch == NULL) {
+			tlb_table_flush_mmu(tlb);
 			tlb_remove_table_one(table);
 			return;
 		}

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#10 lore raw

PATCH 3/20 mm, tlb: Remove a few #ifdefs

In reply to Peter Zijlstra
---
 include/asm-generic/tlb.h |   85 ++++++++++++++++++++++++++--------------------
 mm/memory.c               |    6 ---
 2 files changed, 50 insertions(+), 41 deletions(-)
include/asm-generic/tlb.h +49 -36
@@ -21,6 +21,40 @@
 
 static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page);
 
+/*
+ * If we can't allocate a page to make a big batch of page pointers
+ * to work on, then just handle a few from the on-stack structure.
+ */
+#define MMU_GATHER_BUNDLE	8
+
+struct mmu_gather_batch {
+	struct mmu_gather_batch	*next;
+	unsigned int		nr;
+	unsigned int		max;
+	struct page		*pages[0];
+};
+
+#define MAX_GATHER_BATCH	\
+	((PAGE_SIZE - sizeof(struct mmu_gather_batch)) / sizeof(void *))
+
+/* struct mmu_gather is an opaque type used by the mm code for passing around
+ * any data needed by arch specific code for tlb_remove_page.
+ */
+struct mmu_gather {
+	struct mm_struct	*mm;
+#ifdef CONFIG_HAVE_RCU_TABLE_FREE
+	struct mmu_table_batch	*batch;
+#endif
+	unsigned int		need_flush : 1,	/* Did free PTEs */
+				fast_mode  : 1; /* No batching   */
+
+	unsigned int		fullmm;
+
+	struct mmu_gather_batch *active;
+	struct mmu_gather_batch	local;
+	struct page		*__pages[MMU_GATHER_BUNDLE];
+};
+
 #ifdef CONFIG_HAVE_RCU_TABLE_FREE
 /*
  * Semi RCU freeing of the page directories.
@@ -59,51 +93,30 @@ struct mmu_table_batch {
 #define MAX_TABLE_BATCH		\
 	((PAGE_SIZE - sizeof(struct mmu_table_batch)) / sizeof(void *))
 
+static inline void tlb_table_init(struct mmu_gather *tlb)
+{
+	tlb->batch = NULL;
+}
+
 extern void tlb_table_flush(struct mmu_gather *tlb);
 extern void tlb_remove_table(struct mmu_gather *tlb, void *table);
 
-#else
+#else /* CONFIG_HAVE_RCU_TABLE_FREE */
 
-static inline void tlb_remove_table(struct mmu_gather *tlb, void *table)
+static inline void tlb_table_init(struct mmu_gather *tlb)
 {
-	tlb_remove_page(tlb, table);
 }
 
-#endif
-
-/*
- * If we can't allocate a page to make a big batch of page pointers
- * to work on, then just handle a few from the on-stack structure.
- */
-#define MMU_GATHER_BUNDLE	8
-
-struct mmu_gather_batch {
-	struct mmu_gather_batch	*next;
-	unsigned int		nr;
-	unsigned int		max;
-	struct page		*pages[0];
-};
-
-#define MAX_GATHER_BATCH	\
-	((PAGE_SIZE - sizeof(struct mmu_gather_batch)) / sizeof(void *))
-
-/* struct mmu_gather is an opaque type used by the mm code for passing around
- * any data needed by arch specific code for tlb_remove_page.
- */
-struct mmu_gather {
-	struct mm_struct	*mm;
-#ifdef CONFIG_HAVE_RCU_TABLE_FREE
-	struct mmu_table_batch	*batch;
-#endif
-	unsigned int		need_flush : 1,	/* Did free PTEs */
-				fast_mode  : 1; /* No batching   */
+static inline void tlb_table_flush(struct mmu_gather *tlb)
+{
+}
 
-	unsigned int		fullmm;
+static inline void tlb_remove_table(struct mmu_gather *tlb, void *table)
+{
+	tlb_remove_page(tlb, table);
+}
 
-	struct mmu_gather_batch *active;
-	struct mmu_gather_batch	local;
-	struct page		*__pages[MMU_GATHER_BUNDLE];
-};
+#endif /* CONFIG_HAVE_RCU_TABLE_FREE */
 
 #define HAVE_GENERIC_MMU_GATHER
 
mm/memory.c +1 -5
@@ -214,9 +214,7 @@ void tlb_gather_mmu(struct mmu_gather *t
 	tlb->local.max  = ARRAY_SIZE(tlb->__pages);
 	tlb->active     = &tlb->local;
 
-#ifdef CONFIG_HAVE_RCU_TABLE_FREE
-	tlb->batch = NULL;
-#endif
+	tlb_table_init(tlb);
 }
 
 void tlb_flush_mmu(struct mmu_gather *tlb)
@@ -227,9 +225,7 @@ void tlb_flush_mmu(struct mmu_gather *tl
 		return;
 	tlb->need_flush = 0;
 	tlb_flush(tlb);
-#ifdef CONFIG_HAVE_RCU_TABLE_FREE
 	tlb_table_flush(tlb);
-#endif
 
 	if (tlb_fast_mode(tlb))
 		return;

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#11 lore raw

PATCH 4/20 mm, s390: use generic RCU page-table freeing code

In reply to Peter Zijlstra

Now that we fixed the problem that caused the revert cd94154cc6a ("[S390] fix tlb flushing for page table pages") of the original 36409f6353fc2 ("[S390] use generic RCU page-table freeing code"), we can revert the revert.

---
 arch/s390/Kconfig               |    1 
 arch/s390/include/asm/pgalloc.h |    3 +
 arch/s390/include/asm/tlb.h     |   22 +++++++++++++
 arch/s390/mm/pgtable.c          |   63 +---------------------------------------
 4 files changed, 28 insertions(+), 61 deletions(-)
arch/s390/Kconfig +1 -0
@@ -84,6 +84,7 @@ config S390
 	select HAVE_KERNEL_XZ
 	select HAVE_ARCH_MUTEX_CPU_RELAX
 	select HAVE_ARCH_JUMP_LABEL if !MARCH_G5
+	select HAVE_RCU_TABLE_FREE if SMP
 	select ARCH_SAVE_PAGE_KEYS if HIBERNATION
 	select HAVE_MEMBLOCK
 	select HAVE_MEMBLOCK_NODE_MAP
arch/s390/include/asm/pgalloc.h +3 -0
@@ -22,7 +22,10 @@ void crst_table_free(struct mm_struct *,
 
 unsigned long *page_table_alloc(struct mm_struct *, unsigned long);
 void page_table_free(struct mm_struct *, unsigned long *);
+#ifdef CONFIG_HAVE_RCU_TABLE_FREE
 void page_table_free_rcu(struct mmu_gather *, unsigned long *);
+void __tlb_remove_table(void *_table);
+#endif
 
 static inline void clear_table(unsigned long *s, unsigned long val, size_t n)
 {
arch/s390/include/asm/tlb.h +21 -1
@@ -30,10 +30,14 @@
 
 struct mmu_gather {
 	struct mm_struct *mm;
+#ifdef CONFIG_HAVE_RCU_TABLE_FREE
 	struct mmu_table_batch *batch;
+#endif
 	unsigned int fullmm;
+	unsigned int need_flush;
 };
 
+#ifdef CONFIG_HAVE_RCU_TABLE_FREE
 struct mmu_table_batch {
 	struct rcu_head		rcu;
 	unsigned int		nr;
@@ -45,6 +49,7 @@ struct mmu_table_batch {
 
 extern void tlb_table_flush(struct mmu_gather *tlb);
 extern void tlb_remove_table(struct mmu_gather *tlb, void *table);
+#endif
 
 static inline void tlb_gather_mmu(struct mmu_gather *tlb,
 				  struct mm_struct *mm,
@@ -52,20 +57,29 @@ static inline void tlb_gather_mmu(struct
 {
 	tlb->mm = mm;
 	tlb->fullmm = full_mm_flush;
+	tlb->need_flush = 0;
+#ifdef CONFIG_HAVE_RCU_TABLE_FREE
 	tlb->batch = NULL;
+#endif
 	if (tlb->fullmm)
 		__tlb_flush_mm(mm);
 }
 
 static inline void tlb_flush_mmu(struct mmu_gather *tlb)
 {
+	if (!tlb->need_flush)
+		return;
+	tlb->need_flush = 0;
+	__tlb_flush_mm(tlb->mm);
+#ifdef CONFIG_HAVE_RCU_TABLE_FREE
 	tlb_table_flush(tlb);
+#endif
 }
 
 static inline void tlb_finish_mmu(struct mmu_gather *tlb,
 				  unsigned long start, unsigned long end)
 {
-	tlb_table_flush(tlb);
+	tlb_flush_mmu(tlb);
 }
 
 /*
@@ -91,8 +105,10 @@ static inline void tlb_remove_page(struc
 static inline void pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
 				unsigned long address)
 {
+#ifdef CONFIG_HAVE_RCU_TABLE_FREE
 	if (!tlb->fullmm)
 		return page_table_free_rcu(tlb, (unsigned long *) pte);
+#endif
 	page_table_free(tlb->mm, (unsigned long *) pte);
 }
 
@@ -109,8 +125,10 @@ static inline void pmd_free_tlb(struct m
 #ifdef CONFIG_64BIT
 	if (tlb->mm->context.asce_limit <= (1UL << 31))
 		return;
+#ifdef CONFIG_HAVE_RCU_TABLE_FREE
 	if (!tlb->fullmm)
 		return tlb_remove_table(tlb, pmd);
+#endif
 	crst_table_free(tlb->mm, (unsigned long *) pmd);
 #endif
 }
@@ -128,8 +146,10 @@ static inline void pud_free_tlb(struct m
 #ifdef CONFIG_64BIT
 	if (tlb->mm->context.asce_limit <= (1UL << 42))
 		return;
+#ifdef CONFIG_HAVE_RCU_TABLE_FREE
 	if (!tlb->fullmm)
 		return tlb_remove_table(tlb, pud);
+#endif
 	crst_table_free(tlb->mm, (unsigned long *) pud);
 #endif
 }
arch/s390/mm/pgtable.c +3 -60

75 lines not on this page · raw message

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#12 lore raw

PATCH 5/20 mm, powerpc: Dont use tlb_flush for external tlb flushes

In reply to Peter Zijlstra

Both sparc64 and powerpc64 use tlb_flush() to flush their respective hash-tables which is entirely different from what flush_tlb_range()/flush_tlb_mm() would do.

Powerpc64 already uses arch_*_lazy_mmu_mode() to batch and flush these so any tlb_flush() caller should already find an empty batch. So remove this functionality from tlb_flush().

Cc Benjamin Herrenschmidt <[email protected]>
---
 arch/powerpc/mm/tlb_hash64.c |   10 ----------
 1 file changed, 10 deletions(-)
arch/powerpc/mm/tlb_hash64.c +0 -10
@@ -155,16 +155,6 @@ void __flush_tlb_pending(struct ppc64_tl
 
 void tlb_flush(struct mmu_gather *tlb)
 {
-	struct ppc64_tlb_batch *tlbbatch = &get_cpu_var(ppc64_tlb_batch);
-
-	/* If there's a TLB batch pending, then we must flush it because the
-	 * pages are going to be freed and we really don't want to have a CPU
-	 * access a freed page because it has a stale TLB
-	 */
-	if (tlbbatch->index)
-		__flush_tlb_pending(tlbbatch);
-
-	put_cpu_var(ppc64_tlb_batch);
 }
 
 /**

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#13 lore raw

PATCH 6/20 mm, sparc64: Dont use tlb_flush for external tlb flushes

In reply to Peter Zijlstra

Both sparc64 and powerpc64 use tlb_flush() to flush their respective hash-tables which is entirely different from what flush_tlb_range()/flush_tlb_mm() would do.

Powerpc64 already uses arch_*_lazy_mmu_mode() to batch and flush these so any tlb_flush() caller should already find an empty batch, make sparc64 do the same.

This ensures all platforms now have a tlb_flush() implementation that is either flush_tlb_mm() or flush_tlb_range().

---
 arch/sparc/Makefile                  |    1 +
 arch/sparc/include/asm/tlb_64.h      |    2 +-
 arch/sparc/include/asm/tlbflush_64.h |   11 +++++++++++
 3 files changed, 13 insertions(+), 1 deletion(-)
arch/sparc/Makefile +1 -0
@@ -37,6 +37,7 @@ LDFLAGS       := -m elf64_sparc
 export BITS   := 64
 UTS_MACHINE   := sparc64
 
+KBUILD_CPPFLAGS += -D__HAVE_ARCH_ENTER_LAZY_MMU_MODE
 KBUILD_CFLAGS += -m64 -pipe -mno-fpu -mcpu=ultrasparc -mcmodel=medlow
 KBUILD_CFLAGS += -ffixed-g4 -ffixed-g5 -fcall-used-g7 -Wno-sign-compare
 KBUILD_CFLAGS += -Wa,--undeclared-regs
arch/sparc/include/asm/tlb_64.h +1 -1
@@ -25,7 +25,7 @@ extern void flush_tlb_pending(void);
 #define tlb_start_vma(tlb, vma) do { } while (0)
 #define tlb_end_vma(tlb, vma)	do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
-#define tlb_flush(tlb)	flush_tlb_pending()
+#define tlb_flush(tlb)	do { } while (0)
 
 #include <asm-generic/tlb.h>
 
arch/sparc/include/asm/tlbflush_64.h +11 -0
@@ -26,6 +26,17 @@ extern void flush_tlb_pending(void);
 #define flush_tlb_page(vma,addr)	flush_tlb_pending()
 #define flush_tlb_mm(mm)		flush_tlb_pending()
 
+static inline void arch_enter_lazy_mmu_mode(void)
+{
+}
+
+static inline void arch_leave_lazy_mmu_mode(void)
+{
+	flush_tlb_pending();
+}
+
+#define arch_flush_lazy_mmu_mode()      do {} while (0)
+
 /* Local cpu only.  */
 extern void __flush_tlb_all(void);
 

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#14 lore raw

PATCH 7/20 mm, arch: Remove tlb_flush()

In reply to Peter Zijlstra

Since all asm-generic/tlb.h users their tlb_flush() implementation is now either a nop or flush_tlb_mm(), remove it and make the generic code use flush_tlb_mm() directly.

---
 arch/alpha/include/asm/tlb.h      |    2 --
 arch/arm/include/asm/tlb.h        |    2 --
 arch/avr32/include/asm/tlb.h      |    5 -----
 arch/blackfin/include/asm/tlb.h   |    6 ------
 arch/c6x/include/asm/tlb.h        |    2 --
 arch/cris/include/asm/tlb.h       |    1 -
 arch/frv/include/asm/tlb.h        |    5 -----
 arch/h8300/include/asm/tlb.h      |   13 -------------
 arch/hexagon/include/asm/tlb.h    |    5 -----
 arch/m32r/include/asm/tlb.h       |    6 ------
 arch/m68k/include/asm/tlb.h       |    6 ------
 arch/microblaze/include/asm/tlb.h |    2 --
 arch/mips/include/asm/tlb.h       |    5 -----
 arch/mn10300/include/asm/tlb.h    |    5 -----
 arch/openrisc/include/asm/tlb.h   |    1 -
 arch/parisc/include/asm/tlb.h     |    5 -----
 arch/powerpc/include/asm/tlb.h    |    2 --
 arch/powerpc/mm/tlb_hash32.c      |   15 ---------------
 arch/powerpc/mm/tlb_hash64.c      |    4 ----
 arch/powerpc/mm/tlb_nohash.c      |    5 -----
 arch/score/include/asm/tlb.h      |    1 -
 arch/sh/include/asm/tlb.h         |    1 -
 arch/sparc/include/asm/tlb_32.h   |    5 -----
 arch/sparc/include/asm/tlb_64.h   |    1 -
 arch/tile/include/asm/tlb.h       |    1 -
 arch/unicore32/include/asm/tlb.h  |    1 -
 arch/x86/include/asm/tlb.h        |    1 -
 arch/xtensa/include/asm/tlb.h     |    1 -
 mm/memory.c                       |    2 +-
 29 files changed, 1 insertion(+), 110 deletions(-)
arch/alpha/include/asm/tlb.h +0 -2
@@ -5,8 +5,6 @@
 #define tlb_end_vma(tlb, vma)			do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, pte, addr)	do { } while (0)
 
-#define tlb_flush(tlb)				flush_tlb_mm((tlb)->mm)
-
 #include <asm-generic/tlb.h>
 
 #define __pte_free_tlb(tlb, pte, address)		pte_free((tlb)->mm, pte)
arch/arm/include/asm/tlb.h +0 -2
@@ -23,8 +23,6 @@
 
 #include <linux/pagemap.h>
 
-#define tlb_flush(tlb)	((void) tlb)
-
 #include <asm-generic/tlb.h>
 
 #else /* !CONFIG_MMU */
arch/avr32/include/asm/tlb.h +0 -5
@@ -16,11 +16,6 @@
 
 #define __tlb_remove_tlb_entry(tlb, pte, address) do { } while(0)
 
-/*
- * Flush whole TLB for MM
- */
-#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
-
 #include <asm-generic/tlb.h>
 
 /*
arch/blackfin/include/asm/tlb.h +0 -6
@@ -11,12 +11,6 @@
 #define tlb_end_vma(tlb, vma)	do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address)	do { } while (0)
 
-/*
- * .. because we flush the whole mm when it
- * fills up.
- */
-#define tlb_flush(tlb)		flush_tlb_mm((tlb)->mm)
-
 #include <asm-generic/tlb.h>
 
 #endif				/* _BLACKFIN_TLB_H */
arch/c6x/include/asm/tlb.h +0 -2
@@ -1,8 +1,6 @@
 #ifndef _ASM_C6X_TLB_H
 #define _ASM_C6X_TLB_H
 
-#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
-
 #include <asm-generic/tlb.h>
 
 #endif /* _ASM_C6X_TLB_H */
arch/cris/include/asm/tlb.h +0 -1
@@ -13,7 +13,6 @@
 #define tlb_end_vma(tlb, vma) do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
 
-#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
 #include <asm-generic/tlb.h>
 
 #endif
arch/frv/include/asm/tlb.h +0 -5
@@ -16,11 +16,6 @@ extern void check_pgt_cache(void);
 #define tlb_end_vma(tlb, vma)				do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address)	do { } while (0)
 
-/*
- * .. because we flush the whole mm when it fills up
- */
-#define tlb_flush(tlb)		flush_tlb_mm((tlb)->mm)
-
 #include <asm-generic/tlb.h>
 
 #endif /* _ASM_TLB_H */
arch/h8300/include/asm/tlb.h +0 -13
@@ -5,19 +5,6 @@
 #ifndef __H8300_TLB_H__
 #define __H8300_TLB_H__
 
-#define tlb_flush(tlb)	do { } while(0)
-
-/* 
-  include/asm-h8300/tlb.h 
-*/
-
-#ifndef __H8300_TLB_H__
-#define __H8300_TLB_H__
-
-#define tlb_flush(tlb)	do { } while(0)
-
 #include <asm-generic/tlb.h>
 
 #endif
-
-#endif
arch/hexagon/include/asm/tlb.h +0 -5
@@ -29,11 +29,6 @@
 #define tlb_end_vma(tlb, vma)				do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address)	do { } while (0)
 
-/*
- * .. because we flush the whole mm when it fills up
- */
-#define tlb_flush(tlb)		flush_tlb_mm((tlb)->mm)
-
 #include <asm-generic/tlb.h>
 
 #endif
arch/m32r/include/asm/tlb.h +0 -6
@@ -9,12 +9,6 @@
 #define tlb_end_vma(tlb, vma) do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, pte, address) do { } while (0)
 
-/*
- * .. because we flush the whole mm when it
- * fills up.
- */
-#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
-
 #include <asm-generic/tlb.h>
 
 #endif /* _M32R_TLB_H */
arch/m68k/include/asm/tlb.h +0 -6
@@ -9,12 +9,6 @@
 #define tlb_end_vma(tlb, vma)	do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address)	do { } while (0)
 
-/*
- * .. because we flush the whole mm when it
- * fills up.
- */
-#define tlb_flush(tlb)		flush_tlb_mm((tlb)->mm)
-
 #include <asm-generic/tlb.h>
 
 #endif /* _M68K_TLB_H */
arch/microblaze/include/asm/tlb.h +0 -2
@@ -11,8 +11,6 @@
 #ifndef _ASM_MICROBLAZE_TLB_H
 #define _ASM_MICROBLAZE_TLB_H
 
-#define tlb_flush(tlb)	flush_tlb_mm((tlb)->mm)
-
 #include <linux/pagemap.h>
 #include <asm-generic/tlb.h>
 
arch/mips/include/asm/tlb.h +0 -5
@@ -13,11 +13,6 @@
 #define tlb_end_vma(tlb, vma) do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
 
-/*
- * .. because we flush the whole mm when it fills up.
- */
-#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
-
 #include <asm-generic/tlb.h>
 
 #endif /* __ASM_TLB_H */
arch/mn10300/include/asm/tlb.h +0 -5
@@ -23,11 +23,6 @@ extern void check_pgt_cache(void);
 #define tlb_end_vma(tlb, vma)				do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address)	do { } while (0)
 
-/*
- * .. because we flush the whole mm when it fills up
- */
-#define tlb_flush(tlb)	flush_tlb_mm((tlb)->mm)
-
 /* for now, just use the generic stuff */
 #include <asm-generic/tlb.h>
 
arch/openrisc/include/asm/tlb.h +0 -1
@@ -27,7 +27,6 @@
 #define tlb_end_vma(tlb, vma) do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
 
-#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
 #include <linux/pagemap.h>
 #include <asm-generic/tlb.h>
 
arch/parisc/include/asm/tlb.h +0 -5
@@ -1,11 +1,6 @@
 #ifndef _PARISC_TLB_H
 #define _PARISC_TLB_H
 
-#define tlb_flush(tlb)			\
-do {	if ((tlb)->fullmm)		\
-		flush_tlb_mm((tlb)->mm);\
-} while (0)
-
 #define tlb_start_vma(tlb, vma) \
 do {	if (!(tlb)->fullmm)	\
 		flush_cache_range(vma, vma->vm_start, vma->vm_end); \
arch/powerpc/include/asm/tlb.h +0 -2
@@ -28,8 +28,6 @@
 #define tlb_start_vma(tlb, vma)	do { } while (0)
 #define tlb_end_vma(tlb, vma)	do { } while (0)
 
-extern void tlb_flush(struct mmu_gather *tlb);
-
 /* Get the generic bits... */
 #include <asm-generic/tlb.h>
 
arch/powerpc/mm/tlb_hash32.c +0 -15
@@ -60,21 +60,6 @@ void flush_tlb_page_nohash(struct vm_are
 }
 
 /*
- * Called at the end of a mmu_gather operation to make sure the
- * TLB flush is completely done.
- */
-void tlb_flush(struct mmu_gather *tlb)
-{
-	if (Hash == 0) {
-		/*
-		 * 603 needs to flush the whole TLB here since
-		 * it doesn't use a hash table.
-		 */
-		_tlbia();
-	}
-}
-
-/*
  * TLB flushing:
  *
  *  - flush_tlb_mm(mm) flushes the specified mm context TLB's
arch/powerpc/mm/tlb_hash64.c +0 -4
@@ -153,10 +153,6 @@ void __flush_tlb_pending(struct ppc64_tl
 	batch->index = 0;
 }
 
-void tlb_flush(struct mmu_gather *tlb)
-{
-}
-
 /**
  * __flush_hash_table_range - Flush all HPTEs for a given address range
  *                            from the hash table (and the TLB). But keeps
arch/powerpc/mm/tlb_nohash.c +0 -5
@@ -357,11 +357,6 @@ void flush_tlb_range(struct vm_area_stru
 }
 EXPORT_SYMBOL(flush_tlb_range);
 
-void tlb_flush(struct mmu_gather *tlb)
-{
-	flush_tlb_mm(tlb->mm);
-}
-
 /*
  * Below are functions specific to the 64-bit variant of Book3E though that
  * may change in the future
arch/score/include/asm/tlb.h +0 -1
@@ -8,7 +8,6 @@
 #define tlb_start_vma(tlb, vma)		do {} while (0)
 #define tlb_end_vma(tlb, vma)		do {} while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address) do {} while (0)
-#define tlb_flush(tlb)			flush_tlb_mm((tlb)->mm)
 
 extern void score7_FTLB_refill_Handler(void);
 
arch/sh/include/asm/tlb.h +0 -1
@@ -126,7 +126,6 @@ static inline void tlb_unwire_entry(void
 #define tlb_start_vma(tlb, vma)				do { } while (0)
 #define tlb_end_vma(tlb, vma)				do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, pte, address)	do { } while (0)
-#define tlb_flush(tlb)					do { } while (0)
 
 #include <asm-generic/tlb.h>
 
arch/sparc/include/asm/tlb_32.h +0 -5
@@ -14,11 +14,6 @@ do {								\
 #define __tlb_remove_tlb_entry(tlb, pte, address) \
 	do { } while (0)
 
-#define tlb_flush(tlb) \
-do {								\
-	flush_tlb_mm((tlb)->mm);				\
-} while (0)
-
 #include <asm-generic/tlb.h>
 
 #endif /* _SPARC_TLB_H */
arch/sparc/include/asm/tlb_64.h +0 -1
@@ -25,7 +25,6 @@ extern void flush_tlb_pending(void);
 #define tlb_start_vma(tlb, vma) do { } while (0)
 #define tlb_end_vma(tlb, vma)	do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
-#define tlb_flush(tlb)	do { } while (0)
 
 #include <asm-generic/tlb.h>
 
arch/tile/include/asm/tlb.h +0 -1
@@ -18,7 +18,6 @@
 #define tlb_start_vma(tlb, vma) do { } while (0)
 #define tlb_end_vma(tlb, vma) do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
-#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
 
 #include <asm-generic/tlb.h>
 
arch/unicore32/include/asm/tlb.h +0 -1
@@ -15,7 +15,6 @@
 #define tlb_start_vma(tlb, vma)				do { } while (0)
 #define tlb_end_vma(tlb, vma)				do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address)	do { } while (0)
-#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
 
 #define __pte_free_tlb(tlb, pte, addr)				\
 	do {							\
arch/x86/include/asm/tlb.h +0 -1
@@ -5,7 +5,6 @@
 #define tlb_start_vma(tlb, vma) do { } while (0)
 #define tlb_end_vma(tlb, vma) do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
-#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
 
 #include <asm-generic/tlb.h>
 
arch/xtensa/include/asm/tlb.h +0 -1
@@ -38,7 +38,6 @@
 #endif
 
 #define __tlb_remove_tlb_entry(tlb,pte,addr)	do { } while (0)
-#define tlb_flush(tlb)				flush_tlb_mm((tlb)->mm)
 
 #include <asm-generic/tlb.h>
 
mm/memory.c +1 -1
@@ -226,7 +226,7 @@ void tlb_flush_mmu(struct mmu_gather *tl
 	if (!tlb->need_flush)
 		return;
 	tlb->need_flush = 0;
-	tlb_flush(tlb);
+	flush_tlb_mm(tlb->mm);
 	tlb_table_flush(tlb);
 
 	if (tlb_fast_mode(tlb))

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#15 lore raw

PATCH 8/20 mm: Optimize fullmm TLB flushing

In reply to Peter Zijlstra

This originated from s390 which does something similar and would allow s390 to use the generic TLB flushing code.

The idea is to flush the mm wide cache and tlb a priory and not bother with multiple flushes if the batching isn't large enough.

This can be safely done since there cannot be any concurrency on this mm, its either after the process died (exit) or in the middle of execve where the thread switched to the new mm.

Cc Martin Schwidefsky <[email protected]>
---
 mm/memory.c |   14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)
mm/memory.c +10 -4
@@ -215,16 +215,22 @@ void tlb_gather_mmu(struct mmu_gather *t
 	tlb->active     = &tlb->local;
 
 	tlb_table_init(tlb);
+
+	if (fullmm) {
+		flush_cache_mm(mm);
+		flush_tlb_mm(mm);
+	}
 }
 
 void tlb_flush_mmu(struct mmu_gather *tlb)
 {
 	struct mmu_gather_batch *batch;
 
-	if (!tlb->need_flush)
-		return;
-	tlb->need_flush = 0;
-	flush_tlb_mm(tlb->mm);
+	if (!tlb->fullmm && tlb->need_flush) {
+		tlb->need_flush = 0;
+		flush_tlb_mm(tlb->mm);
+	}
+
 	tlb_table_flush(tlb);
 
 	if (tlb_fast_mode(tlb))

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#38 lore raw

PATCH 9/20 mm, arch: Add end argument to p??_free_tlb()

In reply to Peter Zijlstra

In order to facilitate range tracking we need the end address of the object we're freeing. The callsites already compute this address so change things to simply pass it along.

---
 arch/arm/include/asm/tlb.h         |    6 +++---
 arch/ia64/include/asm/tlb.h        |    6 +++---
 arch/powerpc/mm/hugetlbpage.c      |    4 ++--
 arch/s390/include/asm/tlb.h        |    6 +++---
 arch/sh/include/asm/tlb.h          |    6 +++---
 arch/um/include/asm/tlb.h          |    6 +++---
 include/asm-generic/4level-fixup.h |    2 +-
 include/asm-generic/tlb.h          |    6 +++---
 mm/memory.c                        |   10 +++++-----
 9 files changed, 26 insertions(+), 26 deletions(-)
arch/arm/include/asm/tlb.h +3 -3
@@ -217,9 +217,9 @@ static inline void __pmd_free_tlb(struct
 #endif
 }
 
-#define pte_free_tlb(tlb, ptep, addr)	__pte_free_tlb(tlb, ptep, addr)
-#define pmd_free_tlb(tlb, pmdp, addr)	__pmd_free_tlb(tlb, pmdp, addr)
-#define pud_free_tlb(tlb, pudp, addr)	pud_free((tlb)->mm, pudp)
+#define pte_free_tlb(tlb, ptep, addr, end)	__pte_free_tlb(tlb, ptep, addr)
+#define pmd_free_tlb(tlb, pmdp, addr, end)	__pmd_free_tlb(tlb, pmdp, addr)
+#define pud_free_tlb(tlb, pudp, addr, end)	pud_free((tlb)->mm, pudp)
 
 #define tlb_migrate_finish(mm)		do { } while (0)
 
arch/ia64/include/asm/tlb.h +3 -3
@@ -262,19 +262,19 @@ do {							\
 	__tlb_remove_tlb_entry(tlb, ptep, addr);	\
 } while (0)
 
-#define pte_free_tlb(tlb, ptep, address)		\
+#define pte_free_tlb(tlb, ptep, address, end)		\
 do {							\
 	tlb->need_flush = 1;				\
 	__pte_free_tlb(tlb, ptep, address);		\
 } while (0)
 
-#define pmd_free_tlb(tlb, ptep, address)		\
+#define pmd_free_tlb(tlb, ptep, address, end)		\
 do {							\
 	tlb->need_flush = 1;				\
 	__pmd_free_tlb(tlb, ptep, address);		\
 } while (0)
 
-#define pud_free_tlb(tlb, pudp, address)		\
+#define pud_free_tlb(tlb, pudp, address, end)		\
 do {							\
 	tlb->need_flush = 1;				\
 	__pud_free_tlb(tlb, pudp, address);		\
arch/powerpc/mm/hugetlbpage.c +2 -2
@@ -503,7 +503,7 @@ static void hugetlb_free_pmd_range(struc
 
 	pmd = pmd_offset(pud, start);
 	pud_clear(pud);
-	pmd_free_tlb(tlb, pmd, start);
+	pmd_free_tlb(tlb, pmd, start, end);
 }
 
 static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
@@ -551,7 +551,7 @@ static void hugetlb_free_pud_range(struc
 
 	pud = pud_offset(pgd, start);
 	pgd_clear(pgd);
-	pud_free_tlb(tlb, pud, start);
+	pud_free_tlb(tlb, pud, start, end);
 }
 
 /*
arch/s390/include/asm/tlb.h +3 -3
@@ -103,7 +103,7 @@ static inline void tlb_remove_page(struc
  * page table from the tlb.
  */
 static inline void pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
-				unsigned long address)
+				unsigned long address, unsigned long end)
 {
 #ifdef CONFIG_HAVE_RCU_TABLE_FREE
 	if (!tlb->fullmm)
@@ -120,7 +120,7 @@ static inline void pte_free_tlb(struct m
  * to avoid the double free of the pmd in this case.
  */
 static inline void pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,
-				unsigned long address)
+				unsigned long address, unsigned long end)
 {
 #ifdef CONFIG_64BIT
 	if (tlb->mm->context.asce_limit <= (1UL << 31))
@@ -141,7 +141,7 @@ static inline void pmd_free_tlb(struct m
  * to avoid the double free of the pud in this case.
  */
 static inline void pud_free_tlb(struct mmu_gather *tlb, pud_t *pud,
-				unsigned long address)
+				unsigned long address, unsigned long end)
 {
 #ifdef CONFIG_64BIT
 	if (tlb->mm->context.asce_limit <= (1UL << 42))
arch/sh/include/asm/tlb.h +3 -3
@@ -99,9 +99,9 @@ static inline void tlb_remove_page(struc
 	__tlb_remove_page(tlb, page);
 }
 
-#define pte_free_tlb(tlb, ptep, addr)	pte_free((tlb)->mm, ptep)
-#define pmd_free_tlb(tlb, pmdp, addr)	pmd_free((tlb)->mm, pmdp)
-#define pud_free_tlb(tlb, pudp, addr)	pud_free((tlb)->mm, pudp)
+#define pte_free_tlb(tlb, ptep, addr, end)	pte_free((tlb)->mm, ptep)
+#define pmd_free_tlb(tlb, pmdp, addr, end)	pmd_free((tlb)->mm, pmdp)
+#define pud_free_tlb(tlb, pudp, addr, end)	pud_free((tlb)->mm, pudp)
 
 #define tlb_migrate_finish(mm)		do { } while (0)
 
arch/um/include/asm/tlb.h +3 -3
@@ -109,11 +109,11 @@ static inline void tlb_remove_page(struc
 		__tlb_remove_tlb_entry(tlb, ptep, address);	\
 	} while (0)
 
-#define pte_free_tlb(tlb, ptep, addr) __pte_free_tlb(tlb, ptep, addr)
+#define pte_free_tlb(tlb, ptep, addr, end) __pte_free_tlb(tlb, ptep, addr)
 
-#define pud_free_tlb(tlb, pudp, addr) __pud_free_tlb(tlb, pudp, addr)
+#define pud_free_tlb(tlb, pudp, addr, end) __pud_free_tlb(tlb, pudp, addr)
 
-#define pmd_free_tlb(tlb, pmdp, addr) __pmd_free_tlb(tlb, pmdp, addr)
+#define pmd_free_tlb(tlb, pmdp, addr, end) __pmd_free_tlb(tlb, pmdp, addr)
 
 #define tlb_migrate_finish(mm) do {} while (0)
 
include/asm-generic/4level-fixup.h +1 -1
@@ -27,7 +27,7 @@
 #define pud_page_vaddr(pud)		pgd_page_vaddr(pud)
 
 #undef pud_free_tlb
-#define pud_free_tlb(tlb, x, addr)	do { } while (0)
+#define pud_free_tlb(tlb, x, addr, end)	do { } while (0)
 #define pud_free(mm, x)			do { } while (0)
 #define __pud_free_tlb(tlb, x, addr)	do { } while (0)
 
include/asm-generic/tlb.h +3 -3
@@ -166,21 +166,21 @@ static inline void tlb_remove_page(struc
 		__tlb_remove_pmd_tlb_entry(tlb, pmdp, address);	\
 	} while (0)
 
-#define pte_free_tlb(tlb, ptep, address)			\
+#define pte_free_tlb(tlb, ptep, address, end)			\
 	do {							\
 		tlb->need_flush = 1;				\
 		__pte_free_tlb(tlb, ptep, address);		\
 	} while (0)
 
 #ifndef __ARCH_HAS_4LEVEL_HACK
-#define pud_free_tlb(tlb, pudp, address)			\
+#define pud_free_tlb(tlb, pudp, address, end)			\
 	do {							\
 		tlb->need_flush = 1;				\
 		__pud_free_tlb(tlb, pudp, address);		\
 	} while (0)
 #endif
 
-#define pmd_free_tlb(tlb, pmdp, address)			\
+#define pmd_free_tlb(tlb, pmdp, address, end)			\
 	do {							\
 		tlb->need_flush = 1;				\
 		__pmd_free_tlb(tlb, pmdp, address);		\
mm/memory.c +5 -5
@@ -421,11 +421,11 @@ void pmd_clear_bad(pmd_t *pmd)
  * has been handled earlier when unmapping all the memory regions.
  */
 static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd,
-			   unsigned long addr)
+			   unsigned long addr, unsigned long end)
 {
 	pgtable_t token = pmd_pgtable(*pmd);
 	pmd_clear(pmd);
-	pte_free_tlb(tlb, token, addr);
+	pte_free_tlb(tlb, token, addr, end);
 	tlb->mm->nr_ptes--;
 }
 
@@ -443,7 +443,7 @@ static inline void free_pmd_range(struct
 		next = pmd_addr_end(addr, end);
 		if (pmd_none_or_clear_bad(pmd))
 			continue;
-		free_pte_range(tlb, pmd, addr);
+		free_pte_range(tlb, pmd, addr, next);
 	} while (pmd++, addr = next, addr != end);
 
 	start &= PUD_MASK;
@@ -459,7 +459,7 @@ static inline void free_pmd_range(struct
 
 	pmd = pmd_offset(pud, start);
 	pud_clear(pud);
-	pmd_free_tlb(tlb, pmd, start);
+	pmd_free_tlb(tlb, pmd, start, end);
 }
 
 static inline void free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
@@ -492,7 +492,7 @@ static inline void free_pud_range(struct
 
 	pud = pud_offset(pgd, start);
 	pgd_clear(pgd);
-	pud_free_tlb(tlb, pud, start);
+	pud_free_tlb(tlb, pud, start, end);
 }
 
 /*

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#39 lore raw

PATCH 10/20 mm: Provide generic range tracking and flushing

In reply to Peter Zijlstra

In order to convert various architectures to generic tlb we need to provide some extra infrastructure to track the range of the flushed page tables.

There are two mmu_gather cases to consider:

unmap_region()

    tlb_gather_mmu()
    unmap_vmas()
      for (; vma; vma = vma->vm_next)
        unmap_page_range()
          tlb_start_vma() -> flush cache range/track vm_flags
          zap_*_range()
            arch_enter_lazy_mmu_mode()
            ptep_get_and_clear_full() -> batch/track external tlbs
            tlb_remove_tlb_entry() -> track range/external tlbs
            tlb_remove_page() -> batch page
            arch_lazy_leave_mmu_mode() -> flush external tlbs
          tlb_end_vma()
    free_pgtables()
      while (vma)
        unlink_*_vma()
        free_*_range()
          *_free_tlb() -> track range/batch page
    tlb_finish_mmu() -> flush TLBs and flush everything

free vmas

and:

shift_arg_pages()

    tlb_gather_mmu()
    free_*_range()
      *_free_tlb() -> track tlb range
    tlb_finish_mmu() -> flush things

There are various reasons that we need to flush TLBs _after_ tearing down the page-tables themselves. For some architectures (x86 among others) this serializes against (both hardware and software) page table walkers like gup_fast().

For others (ARM) this is (also) needed to evict stale page-table caches - ARM LPAE mode apparently caches page tables and concurrent hardware walkers could re-populate these caches if the final tlb flush were to be from tlb_end_vma() since an concurrent walk could still be in progress.

So implement generic range tracking over both clearing the PTEs and tearing down the page-tables.

Cc Russell King <[email protected]>
Cc Tony Luck <[email protected]>
Cc Paul Mundt <[email protected]>
Cc Jeff Dike <[email protected]>
Cc Richard Weinberger <[email protected]>
Cc Hans-Christian Egtvedt <[email protected]>
Cc Ralf Baechle <[email protected]>
Cc Kyle McMartin <[email protected]>
Cc James Bottomley <[email protected]>
Cc David Miller <[email protected]>
Cc Chris Zankel <[email protected]>
---
 arch/Kconfig              |    3 
 include/asm-generic/tlb.h |  193 ++++++++++++++++++++++++++++++++++++++++++----
 mm/memory.c               |    3 
 3 files changed, 185 insertions(+), 14 deletions(-)
arch/Kconfig +3 -0
@@ -244,6 +244,9 @@ config HAVE_HW_PAGE_TABLE_WALKS
 	  linux page-table structure. Therefore we don't need to emit
 	  hardware TLB flush instructions before freeing page-table pages.
 
+config HAVE_MMU_GATHER_RANGE
+	bool
+
 config ARCH_HAVE_NMI_SAFE_CMPXCHG
 	bool
 
include/asm-generic/tlb.h +180 -13

262 lines not on this page · raw message

mm/memory.c +2 -1
@@ -214,6 +214,7 @@ void tlb_gather_mmu(struct mmu_gather *t
 	tlb->local.max  = ARRAY_SIZE(tlb->__pages);
 	tlb->active     = &tlb->local;
 
+	tlb_range_init(tlb);
 	tlb_table_init(tlb);
 
 	if (fullmm) {
@@ -228,7 +229,7 @@ void tlb_flush_mmu(struct mmu_gather *tl
 
 	if (!tlb->fullmm && tlb->need_flush) {
 		tlb->need_flush = 0;
-		flush_tlb_mm(tlb->mm);
+		tlb_flush(tlb);
 	}
 
 	tlb_table_flush(tlb);

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#40 lore raw

PATCH 11/20 mm, s390: Convert to use generic mmu_gather

In reply to Peter Zijlstra

Now that s390 is using the generic RCU freeing of page-table pages, all that remains different wrt the generic mmu_gather code is the lack of mmu_gather based TLB flushing for regular entries.

S390 doesn't need a TLB flush after ptep_get_and_clear_full() and before __tlb_remove_page() because its ptep_get_and_clear*() family already does a full TLB invalidate. Therefore force it to use tlb_fast_mode.

Cc Martin Schwidefsky <[email protected]>
---
 arch/s390/include/asm/pgtable.h |    1 
 arch/s390/include/asm/tlb.h     |   85 ++++------------------------------------
 include/asm-generic/tlb.h       |    7 +++
 3 files changed, 17 insertions(+), 76 deletions(-)
arch/s390/include/asm/pgtable.h +1 -0
@@ -1242,6 +1242,7 @@ extern int s390_enable_sie(void);
  * No page table caches to initialise
  */
 #define pgtable_cache_init()	do { } while (0)
+#define check_pgt_cache()	do { } while (0)
 
 #include <asm-generic/pgtable.h>
 
arch/s390/include/asm/tlb.h +9 -76

114 lines not on this page · raw message

include/asm-generic/tlb.h +7 -0
@@ -207,6 +207,12 @@ static inline void tlb_flush(struct mmu_
 
 #endif /* CONFIG_HAVE_MMU_GATHER_RANGE */
 
+/*
+ * Some architectures (s390) do a TLB flush from their ptep_get_and_clear*()
+ * functions, these archs don't need another TLB invalidate and can free their
+ * pages immediately. They'll over-ride tlb_fast_mode with a constant enable.
+ */
+#ifndef tlb_fast_mode
 static inline int tlb_fast_mode(struct mmu_gather *tlb)
 {
 #ifdef CONFIG_SMP
@@ -219,6 +225,7 @@ static inline int tlb_fast_mode(struct m
 	return 1;
 #endif
 }
+#endif
 
 void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, bool fullmm);
 void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end);

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#43 lore raw

PATCH 12/20 mm, arm: Convert arm to generic tlb

In reply to Peter Zijlstra

Might want to optimize the tlb_flush() function to do a full mm flush when the range is 'large', IA64 does this too.

---
 arch/arm/Kconfig           |    1 
 arch/arm/include/asm/tlb.h |  181 +++------------------------------------------
 include/asm-generic/tlb.h  |    4 
 3 files changed, 19 insertions(+), 167 deletions(-)
arch/arm/Kconfig +1 -0
@@ -45,6 +45,7 @@ config ARM
 	select GENERIC_SMP_IDLE_THREAD
 	select KTIME_SCALAR
 	select GENERIC_CLOCKEVENTS_BROADCAST if SMP
+	select HAVE_MMU_GATHER_RANGE if MMU
 	help
 	  The ARM series is a line of low-power-consumption RISC chip designs
 	  licensed by ARM Ltd and targeted at embedded applications and
arch/arm/include/asm/tlb.h +14 -167

213 lines not on this page · raw message

include/asm-generic/tlb.h +4 -0
@@ -286,6 +286,10 @@ static inline void tlb_flush(struct mmu_
  * Some architectures (s390) do a TLB flush from their ptep_get_and_clear*()
  * functions, these archs don't need another TLB invalidate and can free their
  * pages immediately. They'll over-ride tlb_fast_mode with a constant enable.
+ *
+ * Other archs (ARMv7) can have speculative TLB loaders such that we have
+ * concurrency, even on UP, and have to over-ride tlb_fast_mode with a constant
+ * disable.
  */
 #ifndef tlb_fast_mode
 static inline int tlb_fast_mode(struct mmu_gather *tlb)

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#44 lore raw

PATCH 13/20 mm, ia64: Convert ia64 to generic tlb

In reply to Peter Zijlstra
---
 arch/ia64/Kconfig                |    1 
 arch/ia64/include/asm/tlb.h      |  233 ---------------------------------------
 arch/ia64/include/asm/tlbflush.h |   25 ++++
 arch/ia64/mm/tlb.c               |   24 +++-
 4 files changed, 49 insertions(+), 234 deletions(-)
arch/ia64/Kconfig +1 -0
@@ -28,6 +28,7 @@ config IA64
 	select ARCH_DISCARD_MEMBLOCK
 	select GENERIC_IRQ_PROBE
 	select GENERIC_PENDING_IRQ if SMP
+	select HAVE_MMU_GATHER_RANGE
 	select IRQ_PER_CPU
 	select GENERIC_IRQ_SHOW
 	select ARCH_WANT_OPTIONAL_GPIOLIB
arch/ia64/include/asm/tlb.h +2 -231

240 lines not on this page · raw message

arch/ia64/include/asm/tlbflush.h +25 -0
@@ -13,6 +13,31 @@
 #include <asm/mmu_context.h>
 #include <asm/page.h>
 
+struct ia64_tr_entry {
+	u64 ifa;
+	u64 itir;
+	u64 pte;
+	u64 rr;
+}; /*Record for tr entry!*/
+
+extern int ia64_itr_entry(u64 target_mask, u64 va, u64 pte, u64 log_size);
+extern void ia64_ptr_entry(u64 target_mask, int slot);
+extern struct ia64_tr_entry *ia64_idtrs[NR_CPUS];
+
+/*
+ region register macros
+*/
+#define RR_TO_VE(val)   (((val) >> 0) & 0x0000000000000001)
+#define RR_VE(val)     (((val) & 0x0000000000000001) << 0)
+#define RR_VE_MASK     0x0000000000000001L
+#define RR_VE_SHIFT    0
+#define RR_TO_PS(val)  (((val) >> 2) & 0x000000000000003f)
+#define RR_PS(val)     (((val) & 0x000000000000003f) << 2)
+#define RR_PS_MASK     0x00000000000000fcL
+#define RR_PS_SHIFT    2
+#define RR_RID_MASK    0x00000000ffffff00L
+#define RR_TO_RID(val)         ((val >> 8) & 0xffffff)
+
 /*
  * Now for some TLB flushing routines.  This is the kind of stuff that
  * can be very expensive, so try to avoid them whenever possible.
arch/ia64/mm/tlb.c +21 -3
@@ -297,9 +297,8 @@ local_flush_tlb_all (void)
 	ia64_srlz_i();			/* srlz.i implies srlz.d */
 }
 
-void
-flush_tlb_range (struct vm_area_struct *vma, unsigned long start,
-		 unsigned long end)
+void __flush_tlb_range(struct vm_area_struct *vma,
+		  unsigned long start, unsigned long end)
 {
 	struct mm_struct *mm = vma->vm_mm;
 	unsigned long size = end - start;
@@ -335,6 +334,25 @@ flush_tlb_range (struct vm_area_struct *
 	preempt_enable();
 	ia64_srlz_i();			/* srlz.i implies srlz.d */
 }
+
+void flush_tlb_range(struct vm_area_struct *vma,
+		     unsigned long start, unsigned long end)
+{
+	if (unlikely(end - start >= 1024*1024*1024*1024UL
+			|| REGION_NUMBER(start) != REGION_NUMBER(end - 1))) {
+		/*
+		 * If we flush more than a tera-byte or across regions, we're
+		 * probably better off just flushing the entire TLB(s).  This
+		 * should be very rare and is not worth optimizing for.
+		 */
+		flush_tlb_all();
+	} else {
+		/* flush the address range from the tlb */
+		__flush_tlb_range(vma, start, end);
+		/* flush the virt. page-table area mapping the addr range */
+		__flush_tlb_range(vma, ia64_thash(start), ia64_thash(end));
+	}
+}
 EXPORT_SYMBOL(flush_tlb_range);
 
 void __devinit

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#45 lore raw

PATCH 14/20 mm, sh: Convert sh to generic tlb

In reply to Peter Zijlstra
---
 arch/sh/Kconfig           |    1 
 arch/sh/include/asm/tlb.h |   98 ++--------------------------------------------
 2 files changed, 6 insertions(+), 93 deletions(-)
arch/sh/Kconfig +1 -0
@@ -28,6 +28,7 @@ config SUPERH
 	select IRQ_FORCED_THREADING
 	select RTC_LIB
 	select GENERIC_ATOMIC64
+	select HAVE_MMU_GATHER_RANGE if MMU
 	select GENERIC_IRQ_SHOW
 	select GENERIC_SMP_IDLE_THREAD
 	select GENERIC_CLOCKEVENTS
arch/sh/include/asm/tlb.h +5 -93

113 lines not on this page · raw message

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#48 lore raw

PATCH 15/20 mm, um: Convert um to generic tlb

In reply to Peter Zijlstra
Cc Jeff Dike <[email protected]>
Cc Richard Weinberger <[email protected]>
---
 arch/um/Kconfig.common    |    1 
 arch/um/include/asm/tlb.h |  111 +---------------------------------------------
 arch/um/kernel/tlb.c      |   13 -----
 3 files changed, 4 insertions(+), 121 deletions(-)
arch/um/Kconfig.common +1 -0
@@ -11,6 +11,7 @@ config UML
 	select GENERIC_CPU_DEVICES
 	select GENERIC_IO
 	select GENERIC_CLOCKEVENTS
+	select HAVE_MMU_GATHER_RANGE
 
 config MMU
 	bool
arch/um/include/asm/tlb.h +3 -108

117 lines not on this page · raw message

arch/um/kernel/tlb.c +0 -13
@@ -502,19 +502,6 @@ void flush_tlb_range(struct vm_area_stru
 }
 EXPORT_SYMBOL(flush_tlb_range);
 
-void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
-			unsigned long end)
-{
-	/*
-	 * Don't bother flushing if this address space is about to be
-	 * destroyed.
-	 */
-	if (atomic_read(&mm->mm_users) == 0)
-		return;
-
-	fix_range(mm, start, end, 0);
-}
-
 void flush_tlb_mm(struct mm_struct *mm)
 {
 	struct vm_area_struct *vma = mm->mmap;

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#49 lore raw

PATCH 16/20 mm, avr32: Convert avr32 to generic tlb

In reply to Peter Zijlstra
Cc Hans-Christian Egtvedt <[email protected]>
---
 arch/avr32/Kconfig           |    1 +
 arch/avr32/include/asm/tlb.h |    6 ------
 2 files changed, 1 insertion(+), 6 deletions(-)
arch/avr32/Kconfig +1 -0
@@ -14,6 +14,7 @@ config AVR32
 	select ARCH_HAVE_CUSTOM_GPIO_H
 	select ARCH_HAVE_NMI_SAFE_CMPXCHG
 	select GENERIC_CLOCKEVENTS
+	select HAVE_MMU_GATHER_RANGE
 	help
 	  AVR32 is a high-performance 32-bit RISC microprocessor core,
 	  designed for cost-sensitive embedded applications, with particular
arch/avr32/include/asm/tlb.h +0 -6
@@ -8,12 +8,6 @@
 #ifndef __ASM_AVR32_TLB_H
 #define __ASM_AVR32_TLB_H
 
-#define tlb_start_vma(tlb, vma) \
-	flush_cache_range(vma, vma->vm_start, vma->vm_end)
-
-#define tlb_end_vma(tlb, vma) \
-	flush_tlb_range(vma, vma->vm_start, vma->vm_end)
-
 #define __tlb_remove_tlb_entry(tlb, pte, address) do { } while(0)
 
 #include <asm-generic/tlb.h>

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#50 lore raw

PATCH 17/20 mm, mips: Convert mips to generic tlb

In reply to Peter Zijlstra
---
 arch/mips/Kconfig           |    1 +
 arch/mips/include/asm/tlb.h |   10 ----------
 2 files changed, 1 insertion(+), 10 deletions(-)
arch/mips/Kconfig +1 -0
@@ -34,6 +34,7 @@ config MIPS
 	select BUILDTIME_EXTABLE_SORT
 	select GENERIC_CLOCKEVENTS
 	select GENERIC_CMOS_UPDATE
+	select HAVE_MMU_GATHER_RANGE
 
 menu "Machine selection"
 
arch/mips/include/asm/tlb.h +0 -10
@@ -1,16 +1,6 @@
 #ifndef __ASM_TLB_H
 #define __ASM_TLB_H
 
-/*
- * MIPS doesn't need any special per-pte or per-vma handling, except
- * we need to flush cache for area to be unmapped.
- */
-#define tlb_start_vma(tlb, vma) 				\
-	do {							\
-		if (!tlb->fullmm)				\
-			flush_cache_range(vma, vma->vm_start, vma->vm_end); \
-	}  while (0)
-#define tlb_end_vma(tlb, vma) do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
 
 #include <asm-generic/tlb.h>

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#51 lore raw

PATCH 18/20 mm, parisc: Convert parisc to generic tlb

In reply to Peter Zijlstra
Cc Kyle McMartin <[email protected]>
Cc James Bottomley <[email protected]>
---
 arch/parisc/Kconfig           |    1 +
 arch/parisc/include/asm/tlb.h |   10 ----------
 2 files changed, 1 insertion(+), 10 deletions(-)
arch/parisc/Kconfig +1 -0
@@ -19,6 +19,7 @@ config PARISC
 	select ARCH_HAVE_NMI_SAFE_CMPXCHG
 	select GENERIC_SMP_IDLE_THREAD
 	select GENERIC_STRNCPY_FROM_USER
+	select HAVE_MMU_GATHER_RANGE
 
 	help
 	  The PA-RISC microprocessor is designed by Hewlett-Packard and used
arch/parisc/include/asm/tlb.h +0 -10
@@ -1,16 +1,6 @@
 #ifndef _PARISC_TLB_H
 #define _PARISC_TLB_H
 
-#define tlb_start_vma(tlb, vma) \
-do {	if (!(tlb)->fullmm)	\
-		flush_cache_range(vma, vma->vm_start, vma->vm_end); \
-} while (0)
-
-#define tlb_end_vma(tlb, vma)	\
-do {	if (!(tlb)->fullmm)	\
-		flush_tlb_range(vma, vma->vm_start, vma->vm_end); \
-} while (0)
-
 #define __tlb_remove_tlb_entry(tlb, pte, address) \
 	do { } while (0)
 

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#52 lore raw

PATCH 19/20 mm, sparc32: Convert sparc32 to generic tlb

In reply to Peter Zijlstra
---
 arch/sparc/Kconfig              |    1 +
 arch/sparc/include/asm/tlb_32.h |   10 ----------
 2 files changed, 1 insertion(+), 10 deletions(-)
arch/sparc/Kconfig +1 -0
@@ -41,6 +41,7 @@ config SPARC32
 	def_bool !64BIT
 	select GENERIC_ATOMIC64
 	select CLZ_TAB
+	select HAVE_MMU_GATHER_RANGE
 
 config SPARC64
 	def_bool 64BIT
arch/sparc/include/asm/tlb_32.h +0 -10
@@ -1,16 +1,6 @@
 #ifndef _SPARC_TLB_H
 #define _SPARC_TLB_H
 
-#define tlb_start_vma(tlb, vma) \
-do {								\
-	flush_cache_range(vma, vma->vm_start, vma->vm_end);	\
-} while (0)
-
-#define tlb_end_vma(tlb, vma) \
-do {								\
-	flush_tlb_range(vma, vma->vm_start, vma->vm_end);	\
-} while (0)
-
 #define __tlb_remove_tlb_entry(tlb, pte, address) \
 	do { } while (0)
 

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#53 lore raw

PATCH 20/20 mm, xtensa: Convert xtensa to generic tlb

In reply to Peter Zijlstra
---
 arch/xtensa/Kconfig           |    1 +
 arch/xtensa/include/asm/tlb.h |   23 -----------------------
 arch/xtensa/mm/tlb.c          |    2 +-
 3 files changed, 2 insertions(+), 24 deletions(-)
arch/xtensa/Kconfig +1 -0
@@ -10,6 +10,7 @@ config XTENSA
 	select HAVE_GENERIC_HARDIRQS
 	select GENERIC_IRQ_SHOW
 	select GENERIC_CPU_DEVICES
+	select HAVE_MMU_GATHER_RANGE
 	help
 	  Xtensa processors are 32-bit RISC machines designed by Tensilica
 	  primarily for embedded systems.  These processors are both
arch/xtensa/include/asm/tlb.h +0 -23
@@ -14,29 +14,6 @@
 #include <asm/cache.h>
 #include <asm/page.h>
 
-#if (DCACHE_WAY_SIZE <= PAGE_SIZE)
-
-/* Note, read http://lkml.org/lkml/2004/1/15/6 */
-
-# define tlb_start_vma(tlb,vma)			do { } while (0)
-# define tlb_end_vma(tlb,vma)			do { } while (0)
-
-#else
-
-# define tlb_start_vma(tlb, vma)					      \
-	do {								      \
-		if (!tlb->fullmm)					      \
-			flush_cache_range(vma, vma->vm_start, vma->vm_end);   \
-	} while(0)
-
-# define tlb_end_vma(tlb, vma)						      \
-	do {								      \
-		if (!tlb->fullmm)					      \
-			flush_tlb_range(vma, vma->vm_start, vma->vm_end);     \
-	} while(0)
-
-#endif
-
 #define __tlb_remove_tlb_entry(tlb,pte,addr)	do { } while (0)
 
 #include <asm-generic/tlb.h>
arch/xtensa/mm/tlb.c +1 -1
@@ -63,7 +63,7 @@ void flush_tlb_all (void)
 void flush_tlb_mm(struct mm_struct *mm)
 {
 	if (mm == current->active_mm) {
-		int flags;
+		unsigned long flags;
 		local_save_flags(flags);
 		__get_new_mmu_context(mm);
 		__load_mmu_context(mm);

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#41 lore raw

Re: [PATCH 11/20] mm, s390: Convert to use generic mmu_gather

In reply to Peter Zijlstra
Quoted 5 lines · Peter Zijlstra

S390 doesn't need a TLB flush after ptep_get_and_clear_full() and before __tlb_remove_page() because its ptep_get_and_clear*() family already does a full TLB invalidate. Therefore force it to use tlb_fast_mode.

On that.. ptep_get_and_clear() says:

/*
* This is hard to understand. ptep_get_and_clear and ptep_clear_flush
* both clear the TLB for the unmapped pte. The reason is that
* ptep_get_and_clear is used in common code (e.g. change_pte_range)
* to modify an active pte. The sequence is

 *   1) ptep_get_and_clear                                                                     
 *   2) set_pte_at                                                                             
 *   3) flush_tlb_range                                                                        

* On s390 the tlb needs to get flushed with the modification of the pte
* if the pte is active. The only way how this can be implemented is to
* have ptep_get_and_clear do the tlb flush. In exchange flush_tlb_range
* is a nop.
*/

I think there is another way, arch_{enter,leave}_lazy_mmu_mode() seems to wrap these sites so you can do as SPARC64 and PPC do and batch through there.

That should save a number of TLB invalidates..

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#3 lore raw

Re: [PATCH 02/20] mm: Add optional TLB flush to generic RCU page-table freeing

In reply to Peter Zijlstra
Quoted 5 lines · Peter Zijlstra

Certain architectures (viz. x86, arm, s390) have hardware page-table walkers (#PF). So during the RCU page-table teardown process make sure we do a tlb flush of page-table pages on all relevant CPUs to synchronize against hardware walkers, and then free the pages.

NACK.

Why would hw page table walkers be that special? Plus your config option is horribly done anyway, where you do it as some kind of "default y" and then have complex conditionals on it.

Plus it really isn't about hardware page table walkers at all. It's more about the possibility of speculative TLB fils, it has nothing to do with *how* they are done. Sure, it's likely that a software pagetable walker wouldn't be something that gets called speculatively, but it's not out of the question.

So I think your config option is totally mis-designed and actively misleading. It's also horrible from a design standpoint, since it's entirely possible that some day POWERPC will actually see the light and do speculative TLB fills etc.

So *if* this needs to be done, it needs to be done right. That means:

- don't talk about HW walking, since it's not about that

- don't say "if you have speculative walkers", and use an ifndef. Say
"If you can *guarantee* that nothing else walks page tables speculatively, and we have only one thread that owns the mmu, and that one thread is us, *then* we can do this optimization". So switch the config option around.

- make it a per-architecture thing to say "I guarantee that I never
fill the TLB speculatively". Don't do that "default y" with complex conditionals crap.

IOW, if Sparc/PPC really want to guarantee that they never fill TLB entries speculatively, and that if we are in a kernel thread they will *never* fill the TLB with anything else, then make them enable CONFIG_STRICT_TLB_FILL or something in their architecture Kconfig files.

Not like this patch. And not with the misleading names and comments.

Linus

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#16 lore raw

Re: [PATCH 08/20] mm: Optimize fullmm TLB flushing

In reply to Peter Zijlstra
Quoted 9 lines · Peter Zijlstra

This originated from s390 which does something similar and would allow s390 to use the generic TLB flushing code.

The idea is to flush the mm wide cache and tlb a priory and not bother with multiple flushes if the batching isn't large enough.

This can be safely done since there cannot be any concurrency on this mm, its either after the process died (exit) or in the middle of execve where the thread switched to the new mm.

I think we actually *used* to do the final TLB flush from within the context of the process that died. That doesn't seem to ever be the case any more, but it does worry me a bit. Maybe a

   VM_BUG_ON(current->active_mm == mm);

or something for the fullmm case?

Linus

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#4 lore raw

Re: [PATCH 02/20] mm: Add optional TLB flush to generic RCU page-table freeing

In reply to Linus Torvalds
Quoted 6 lines · Linus Torvalds

Plus it really isn't about hardware page table walkers at all. It's more about the possibility of speculative TLB fils, it has nothing to do with *how* they are done. Sure, it's likely that a software pagetable walker wouldn't be something that gets called speculatively, but it's not out of the question.

Hmm, I would call gup_fast() as speculative as we can get in software. It does a lock-less walk of the page-tables. That's what the RCU free'd page-table stuff is for to begin with.

Quoted 6 lines

IOW, if Sparc/PPC really want to guarantee that they never fill TLB entries speculatively, and that if we are in a kernel thread they will *never* fill the TLB with anything else, then make them enable CONFIG_STRICT_TLB_FILL or something in their architecture Kconfig files.

Since we've dealt with the speculative software side by using RCU-ish stuff, the only thing that's left is hardware, now neither sparc64 nor ppc actually know about the linux page-tables from what I understood, they only look at their hash-table thing.

So even if the hardware did do speculative tlb fills, it would do them from the hash-table, but that's already cleared out.

How about something like this

---
Subject: mm: Add missing TLB invalidate to RCU page-table freeing
From: Peter Zijlstra <[email protected]>
Date: Thu Jun 28 00:49:33 CEST 2012

For normal systems we need a TLB invalidate before freeing the page-tables, the generic RCU based page-table freeing code lacked this.

This is because this code originally came from ppc where the hardware never walks the linux page-tables and thus this invalidate is not required.

Others, notably s390 which ran into this problem in cd94154cc6a ("[S390] fix tlb flushing for page table pages"), do very much need this TLB invalidation.

Therefore add it, with a Kconfig option to disable it so as to not unduly slow down PPC and SPARC64 which neither of them need it.

---
 arch/Kconfig         |    3 +++
 arch/powerpc/Kconfig |    1 +
 arch/sparc/Kconfig   |    1 +
 mm/memory.c          |   18 ++++++++++++++++++
 4 files changed, 23 insertions(+)
arch/Kconfig +3 -0
@@ -231,6 +231,9 @@ config HAVE_ARCH_MUTEX_CPU_RELAX
 config HAVE_RCU_TABLE_FREE
 	bool
 
+config STRICT_TLB_FILL
+	bool
+
 config ARCH_HAVE_NMI_SAFE_CMPXCHG
 	bool
 
arch/powerpc/Kconfig +1 -0
@@ -127,6 +127,7 @@ config PPC
 	select GENERIC_IRQ_SHOW_LEVEL
 	select IRQ_FORCED_THREADING
 	select HAVE_RCU_TABLE_FREE if SMP
+	select STRICT_TLB_FILL
 	select HAVE_SYSCALL_TRACEPOINTS
 	select HAVE_BPF_JIT if PPC64
 	select HAVE_ARCH_JUMP_LABEL
arch/sparc/Kconfig +1 -0
@@ -52,6 +52,7 @@ config SPARC64
 	select HAVE_KRETPROBES
 	select HAVE_KPROBES
 	select HAVE_RCU_TABLE_FREE if SMP
+	select STRICT_TLB_FILL
 	select HAVE_MEMBLOCK
 	select HAVE_MEMBLOCK_NODE_MAP
 	select HAVE_SYSCALL_WRAPPERS
mm/memory.c +18 -0
@@ -329,11 +329,27 @@ static void tlb_remove_table_rcu(struct 
 	free_page((unsigned long)batch);
 }
 
+#ifdef CONFIG_STRICT_TLB_FILL
+/*
+ * Some archictures (sparc64, ppc) cannot refill TLBs after the they've removed
+ * the PTE entries from their hash-table. Their hardware never looks at the
+ * linux page-table structures, so they don't need a hardware TLB invalidate
+ * when tearing down the page-table structure itself.
+ */
+static inline void tlb_table_flush_mmu(struct mmu_gather *tlb) { }
+#else
+static inline void tlb_table_flush_mmu(struct mmu_gather *tlb)
+{
+	tlb_flush_mmu(tlb);
+}
+#endif
+
 void tlb_table_flush(struct mmu_gather *tlb)
 {
 	struct mmu_table_batch **batch = &tlb->batch;
 
 	if (*batch) {
+		tlb_table_flush_mmu(tlb);
 		call_rcu_sched(&(*batch)->rcu, tlb_remove_table_rcu);
 		*batch = NULL;
 	}
@@ -345,6 +361,7 @@ void tlb_remove_table(struct mmu_gather 
 
 	tlb->need_flush = 1;
 
+#ifdef CONFIG_STRICT_TLB_FILL
 	/*
 	 * When there's less then two users of this mm there cannot be a
 	 * concurrent page-table walk.
@@ -353,6 +370,7 @@ void tlb_remove_table(struct mmu_gather 
 		__tlb_remove_table(table);
 		return;
 	}
+#endif
 
 	if (*batch == NULL) {
 		*batch = (struct mmu_table_batch *)__get_free_page(GFP_NOWAIT | __GFP_NOWARN);

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#17 lore raw

Re: [PATCH 08/20] mm: Optimize fullmm TLB flushing

In reply to Linus Torvalds
Quoted 11 lines · Linus Torvalds
Quoted 9 lines · Peter Zijlstra

This originated from s390 which does something similar and would allow s390 to use the generic TLB flushing code.

The idea is to flush the mm wide cache and tlb a priory and not bother with multiple flushes if the batching isn't large enough.

This can be safely done since there cannot be any concurrency on this mm, its either after the process died (exit) or in the middle of execve where the thread switched to the new mm.

Quoted 7 lines

I think we actually *used* to do the final TLB flush from within the context of the process that died. That doesn't seem to ever be the case any more, but it does worry me a bit. Maybe a

   VM_BUG_ON(current->active_mm == mm);

or something for the fullmm case?

OK, added it and am rebooting the test box..

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

maintainer
#18 lore raw

Re: [PATCH 08/20] mm: Optimize fullmm TLB flushing

Quoted 16 lines · Peter Zijlstra
Quoted 15 lines · Linus Torvalds
Quoted 9 lines · Peter Zijlstra

This originated from s390 which does something similar and would allow s390 to use the generic TLB flushing code.

The idea is to flush the mm wide cache and tlb a priory and not bother with multiple flushes if the batching isn't large enough.

This can be safely done since there cannot be any concurrency on this mm, its either after the process died (exit) or in the middle of execve where the thread switched to the new mm.

I think we actually *used* to do the final TLB flush from within the context of the process that died. That doesn't seem to ever be the case any more, but it does worry me a bit. Maybe a

Quoted 5 lines
Quoted 3 lines
   VM_BUG_ON(current->active_mm == mm);

or something for the fullmm case?

OK, added it and am rebooting the test box..

That triggered.. is this a problem though, at this point userspace is very dead so it shouldn't matter, right?

Will have to properly think about it tomorrow, its been 1am, brain is mostly sleeping already.

------------[ cut here ]------------
kernel BUG at /home/root/src/linux-2.6/mm/memory.c:221!
invalid opcode: 0000 [#1] PREEMPT SMP DEBUG_PAGEALLOC
Modules linked in:
CPU 13 
Pid: 132, comm: modprobe Not tainted 3.5.0-rc4-01507-g912ca15-dirty #180 Supermicro X8DTN/X8DTN
RIP: 0010:[<ffffffff811511bf>]  [<ffffffff811511bf>] tlb_gather_mmu+0x9f/0xb0
RSP: 0018:ffff880235b2bd78  EFLAGS: 00010246
RAX: ffff880235b18000 RBX: ffff880235b2bdc0 RCX: ffff880235b18000
RDX: 0000000000000000 RSI: 0000000000000100 RDI: 0000000000000000
RBP: ffff880235b2bd98 R08: 0000000000000018 R09: 0000000000000004
R10: ffffffff81eedfc0 R11: 0000000000000084 R12: ffff8804356b8000
R13: 0000000000000001 R14: ffff880235b185f0 R15: ffff880235b18000
FS:  0000000000000000(0000) GS:ffff880237ce0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000038ce8ae150 CR3: 0000000436ad6000 CR4: 00000000000007e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Process modprobe (pid: 132, threadinfo ffff880235b2a000, task ffff880235b18000)
Stack:
 ffff880235b2bd98 0000000000000000 ffff8804356b8000 ffff8804356b8060
 ffff880235b2be38 ffffffff8115ad38 ffff880235b2be38 ffff880235b4e000
 ffff880235b4e630 ffff8804356b8000 0000000100000000 ffff880235b2bdd8
Call Trace:
 [<ffffffff8115ad38>] exit_mmap+0x98/0x150
 [<ffffffff810bf98e>] ? exit_numa+0xae/0xe0
 [<ffffffff81078b74>] mmput+0x84/0x120
 [<ffffffff81080ce8>] exit_mm+0x108/0x130
 [<ffffffff81081388>] do_exit+0x678/0x950
 [<ffffffff811a3ad6>] ? alloc_fd+0xd6/0x120
 [<ffffffff811791c0>] ? kmem_cache_free+0x20/0x130
 [<ffffffff810819af>] do_group_exit+0x3f/0xa0
 [<ffffffff81081a27>] sys_exit_group+0x17/0x20
 [<ffffffff81980ed2>] system_call_fastpath+0x16/0x1b
Code: 10 74 1a 65 48 8b 04 25 80 ba 00 00 4c 3b a0 90 02 00 00 74 16 4c 89 e7 e8 5f 39 f2 ff 48 8b 5d e8 4c 8b 65 f0 4c 8b 6d f8 c9 c3 <0f> 0b 66 66 66 66 66 66 2e 0f 1f 84 00 00 00 00 00 55 48 89 e5 
RIP  [<ffffffff811511bf>] tlb_gather_mmu+0x9f/0xb0
 RSP <ffff880235b2bd78>
---[ end trace f99f121b09c974f8 ]---

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#19 lore raw

Re: [PATCH 08/20] mm: Optimize fullmm TLB flushing

Quoted 3 lines · Peter Zijlstra

That triggered.. is this a problem though, at this point userspace is very dead so it shouldn't matter, right?

It still matters. Even if user space is dead, kernel space accesses can result in TLB fills in user space. Exactly because of things like speculative fills etc.

So what can happen - for example - is that the kernel does a indirect jump, and the CPU predicts the destination of the jump that using the branch prediction tables.

But the branch prediction tables are obviously just predictions, and they easily contain user addresses etc in them. So the kernel may well end up speculatively doing a TLB fill on a user access.

And your whole optimization depends on this not happening, unless I read the logic wrong. The whole "invalidate the TLB just once up-front" approach is *only* valid if you know that nothing is going to ever fill that TLB again. But see above - if we're still running within that TLB context, we have no idea what speculative execution may or may not end up filling.

That said, maybe I misread your patch?

Linus

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#20 lore raw

Re: [PATCH 08/20] mm: Optimize fullmm TLB flushing

Quoted 4 lines · Linus Torvalds

But the branch prediction tables are obviously just predictions, and they easily contain user addresses etc in them. So the kernel may well end up speculatively doing a TLB fill on a user access.

That should be ".. on a user *address*", hopefully that was clear from the context, if not from the text.

IOW, the point I'm trying to make is that even if there are zero *actual* accesses of user space (because user space is dead, and the kernel hopefully does no "get_user()/put_user()" stuff at this point any more), the CPU may speculatively use user addresses for the bog-standard kernel addresses that happen.

Taking a user address from the BTB is just one example. Speculative memory accesses might happen after a mis-predicted branch, where we test a pointer against NULL, and after the branch we access it. So doing a speculative TLB walk of the NULL address would not necessarily even be unusual. Obviously normally nothing is actually mapped there, but these kinds of things can *easily* result in the page tables themselves being cached, even if the final page doesn't exist.

Also, all of this obviously depends on how aggressive the speculation is. It's entirely possible that effects like these are really hard to see in practice, and you'll almost never hit it. But stale TLB contents (or stale page directory caches) are *really* nasty when they do happen, and almost impossible to debug. So we want to be insanely anal in this area.

Linus

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#5 lore raw

Re: [PATCH 02/20] mm: Add optional TLB flush to generic RCU page-table freeing

In reply to Peter Zijlstra
Quoted 2 lines · Peter Zijlstra

How about something like this

Looks better.

I'd be even happier if you made the whole

"When there's less then two users.."

(There's a misspelling there, btw, I didn't notice until I cut-and-pasted that) logic be a helper function, and have that helper function be inside that same #ifdef CONFIG_STRICT_TLB_FILL block together witht he tlb_table_flush_mmu() function.

IOW, something like

  static int tlb_remove_table_quick( struct mmu_gather *tlb, void *table)
  {
        if (atomic_read(&tlb->mm->mm_users) < 2) {
            __tlb_remove_table(table);
            return 1;
        }
        return 0;
  }

for the CONFIG_STRICT_TLB_FILL case, and then the default case just does an unconditional "return 0".

So that the actual code can avoid having #ifdef's in the middle of a function, and could just do

if (tlb_remove_table_quick(tlb, table))
return;

instead.

Maybe it's just me, but I detest seeing #ifdef's in the middle of code. I'd much rather have the #ifdef's *outside* the code and have these kinds of helper functions that sometimes end up becoming empty.

Linus

#6 lore raw

Re: [PATCH 02/20] mm: Add optional TLB flush to generic RCU page-table freeing

In reply to Peter Zijlstra
Quoted 11 lines · Peter Zijlstra
Quoted 6 lines · Linus Torvalds

Plus it really isn't about hardware page table walkers at all. It's more about the possibility of speculative TLB fils, it has nothing to do with *how* they are done. Sure, it's likely that a software pagetable walker wouldn't be something that gets called speculatively, but it's not out of the question.

Hmm, I would call gup_fast() as speculative as we can get in software. It does a lock-less walk of the page-tables. That's what the RCU free'd page-table stuff is for to begin with.

Strictly speaking it's not :-) To *begin with* (as in the origin of that code) it comes from powerpc hash table code which walks the linux page tables locklessly :-) It then came in handy with gup_fast :-)

Quoted 10 lines
Quoted 5 lines

IOW, if Sparc/PPC really want to guarantee that they never fill TLB entries speculatively, and that if we are in a kernel thread they will *never* fill the TLB with anything else, then make them enable CONFIG_STRICT_TLB_FILL or something in their architecture Kconfig files.

Since we've dealt with the speculative software side by using RCU-ish stuff, the only thing that's left is hardware, now neither sparc64 nor ppc actually know about the linux page-tables from what I understood, they only look at their hash-table thing.

Some embedded ppc's know about the lowest level (SW loaded PMD) but that's not an issue here. We flush these special TLB entries specifically and synchronously in __pte_free_tlb().

Quoted 2 lines

So even if the hardware did do speculative tlb fills, it would do them from the hash-table, but that's already cleared out.

Right,

Cheers,
Ben.

Quoted 111 lines

How about something like this

---
Subject: mm: Add missing TLB invalidate to RCU page-table freeing
From: Peter Zijlstra <[email protected]>
Date: Thu Jun 28 00:49:33 CEST 2012

For normal systems we need a TLB invalidate before freeing the page-tables, the generic RCU based page-table freeing code lacked this.

This is because this code originally came from ppc where the hardware never walks the linux page-tables and thus this invalidate is not required.

Others, notably s390 which ran into this problem in cd94154cc6a ("[S390] fix tlb flushing for page table pages"), do very much need this TLB invalidation.

Therefore add it, with a Kconfig option to disable it so as to not unduly slow down PPC and SPARC64 which neither of them need it.

---
 arch/Kconfig         |    3 +++
 arch/powerpc/Kconfig |    1 +
 arch/sparc/Kconfig   |    1 +
 mm/memory.c          |   18 ++++++++++++++++++
 4 files changed, 23 insertions(+)
arch/Kconfig +3 -0
@@ -231,6 +231,9 @@ config HAVE_ARCH_MUTEX_CPU_RELAX
 config HAVE_RCU_TABLE_FREE
 	bool
 
+config STRICT_TLB_FILL
+	bool
+
 config ARCH_HAVE_NMI_SAFE_CMPXCHG
 	bool
 
arch/powerpc/Kconfig +1 -0
@@ -127,6 +127,7 @@ config PPC
 	select GENERIC_IRQ_SHOW_LEVEL
 	select IRQ_FORCED_THREADING
 	select HAVE_RCU_TABLE_FREE if SMP
+	select STRICT_TLB_FILL
 	select HAVE_SYSCALL_TRACEPOINTS
 	select HAVE_BPF_JIT if PPC64
 	select HAVE_ARCH_JUMP_LABEL
arch/sparc/Kconfig +1 -0
@@ -52,6 +52,7 @@ config SPARC64
 	select HAVE_KRETPROBES
 	select HAVE_KPROBES
 	select HAVE_RCU_TABLE_FREE if SMP
+	select STRICT_TLB_FILL
 	select HAVE_MEMBLOCK
 	select HAVE_MEMBLOCK_NODE_MAP
 	select HAVE_SYSCALL_WRAPPERS
mm/memory.c +18 -1
@@ -329,11 +329,27 @@ static void tlb_remove_table_rcu(struct 
 	free_page((unsigned long)batch);
 }
 
+#ifdef CONFIG_STRICT_TLB_FILL
+/*
+ * Some archictures (sparc64, ppc) cannot refill TLBs after the they've removed
+ * the PTE entries from their hash-table. Their hardware never looks at the
+ * linux page-table structures, so they don't need a hardware TLB invalidate
+ * when tearing down the page-table structure itself.
+ */
+static inline void tlb_table_flush_mmu(struct mmu_gather *tlb) { }
+#else
+static inline void tlb_table_flush_mmu(struct mmu_gather *tlb)
+{
+	tlb_flush_mmu(tlb);
+}
+#endif
+
 void tlb_table_flush(struct mmu_gather *tlb)
 {
 	struct mmu_table_batch **batch = &tlb->batch;
 
 	if (*batch) {
+		tlb_table_flush_mmu(tlb);
 		call_rcu_sched(&(*batch)->rcu, tlb_remove_table_rcu);
 		*batch = NULL;
 	}
@@ -345,6 +361,7 @@ void tlb_remove_table(struct mmu_gather 
 
 	tlb->need_flush = 1;
 
+#ifdef CONFIG_STRICT_TLB_FILL
 	/*
 	 * When there's less then two users of this mm there cannot be a
 	 * concurrent page-table walk.
@@ -353,6 +370,7 @@ void tlb_remove_table(struct mmu_gather 
 		__tlb_remove_table(table);
 		return;
 	}
+#endif
 
 	if (*batch == NULL) {
 		*batch = (struct mmu_table_batch *)__get_free_page(GFP_NOWAIT | __GFP_NOWARN);
 
--
Quoted 4 lines

To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#42 lore raw

Re: [PATCH 11/20] mm, s390: Convert to use generic mmu_gather

In reply to Peter Zijlstra
Quoted 23 lines · On Thu, 28 Jun 2012 00:13:19 +0200 Peter Zijlstra <[email protected]>…
Quoted 5 lines · Peter Zijlstra

S390 doesn't need a TLB flush after ptep_get_and_clear_full() and before __tlb_remove_page() because its ptep_get_and_clear*() family already does a full TLB invalidate. Therefore force it to use tlb_fast_mode.

On that.. ptep_get_and_clear() says:

/*
* This is hard to understand. ptep_get_and_clear and ptep_clear_flush
* both clear the TLB for the unmapped pte. The reason is that
* ptep_get_and_clear is used in common code (e.g. change_pte_range)
* to modify an active pte. The sequence is

 *   1) ptep_get_and_clear                                                                     
 *   2) set_pte_at                                                                             
 *   3) flush_tlb_range                                                                        

* On s390 the tlb needs to get flushed with the modification of the pte
* if the pte is active. The only way how this can be implemented is to
* have ptep_get_and_clear do the tlb flush. In exchange flush_tlb_range
* is a nop.
*/

Quoted 5 lines

I think there is another way, arch_{enter,leave}_lazy_mmu_mode() seems to wrap these sites so you can do as SPARC64 and PPC do and batch through there.

That should save a number of TLB invalidates..

Unfortunately that is not good enough. The point is that a pte that can be referenced by another cpu may not be modified without using one of the special instructions that flushes the TLBs on all cpu at the same time. It really is one pte at a time if more than one cpu attached a particular mm.

Signature · 10 lines
blue skies,
   Martin.

"Reality continues to ruin my life." - Calvin.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
#21 lore raw

Re: [PATCH 08/20] mm: Optimize fullmm TLB flushing

In reply to Linus Torvalds
Quoted 6 lines · Linus Torvalds
Quoted 3 lines · Linus Torvalds

But the branch prediction tables are obviously just predictions, and they easily contain user addresses etc in them. So the kernel may well end up speculatively doing a TLB fill on a user access.

Quoted 8 lines

That should be ".. on a user *address*", hopefully that was clear from the context, if not from the text.

IOW, the point I'm trying to make is that even if there are zero *actual* accesses of user space (because user space is dead, and the kernel hopefully does no "get_user()/put_user()" stuff at this point any more), the CPU may speculatively use user addresses for the bog-standard kernel addresses that happen.

That's definitely an issue on ARM and it was hit on older kernels. Basically ARM processors can cache any page translation level in the TLB. We need to make sure that no page entry at any level (either cached in the TLB or not) points to an invalid next level table (hence the TLB shootdown). For example, in cases like free_pgd_range(), if the cached pgd entry points to an already freed pud/pmd table (pgd_clear is not enough) it may walk the page tables speculatively cache another entry in the TLB. Depending on the random data it reads from an old table page, it may find a global entry (it's just a bit in the pte) which is not tagged with an ASID (application specific id). A latter flush_tlb_mm() only flushes the current ASID and doesn't touch global entries (used only by kernel mappings). So we end up with global TLB entry in user space that overrides any other application mapping.

Signature · 7 lines
Catalin

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
#22 lore raw

Re: [PATCH 08/20] mm: Optimize fullmm TLB flushing

Quoted 5 lines · Catalin Marinas

That's definitely an issue on ARM and it was hit on older kernels. Basically ARM processors can cache any page translation level in the TLB. We need to make sure that no page entry at any level (either cached in the TLB or not) points to an invalid next level table (hence the TLB shootdown). For example, in cases like free_pgd_range(), if the cached

Quoted 8 lines

pgd entry points to an already freed pud/pmd table (pgd_clear is not enough) it may walk the page tables speculatively cache another entry in the TLB. Depending on the random data it reads from an old table page, it may find a global entry (it's just a bit in the pte) which is not tagged with an ASID (application specific id). A latter flush_tlb_mm() only flushes the current ASID and doesn't touch global entries (used only by kernel mappings). So we end up with global TLB entry in user space that overrides any other application mapping.

Right, that's the typical scenario. I haven't looked at your flush implementation though, but surely you can defer the actual freeing so you can batch them & limit the number of TLB flushes right ?

Cheers,
Ben.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

maintainer
#34 lore raw

Re: [PATCH 08/20] mm: Optimize fullmm TLB flushing

In reply to Linus Torvalds
Quoted 5 lines · Linus Torvalds

IOW, the point I'm trying to make is that even if there are zero *actual* accesses of user space (because user space is dead, and the kernel hopefully does no "get_user()/put_user()" stuff at this point any more), the CPU may speculatively use user addresses for the bog-standard kernel addresses that happen.

Right.. and s390 having done this only says that s390 appears to be ok with it. Martin, does s390 hardware guarantee no speculative stuff like Linus explained, or might there even be a latent issue on s390?

But it looks like we cannot do this in general, and esp. ARM (as already noted by Catalin) has very aggressive speculative behaviour.

The alternative is that we do a switch_mm() to init_mm instead of the TLB flush. On x86 that should be about the same cost, but I've not looked at other architectures yet.

The second and least favourite alternative is of course special casing this for s390 if it turns out its a safe thing to do for them.

/me goes look through arch code.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

maintainer
#23 lore raw

Re: [PATCH 08/20] mm: Optimize fullmm TLB flushing

In reply to Benjamin Herrenschmidt
Quoted 15 lines · Benjamin Herrenschmidt
Quoted 5 lines · Catalin Marinas

That's definitely an issue on ARM and it was hit on older kernels. Basically ARM processors can cache any page translation level in the TLB. We need to make sure that no page entry at any level (either cached in the TLB or not) points to an invalid next level table (hence the TLB shootdown). For example, in cases like free_pgd_range(), if the cached

Quoted 8 lines

pgd entry points to an already freed pud/pmd table (pgd_clear is not enough) it may walk the page tables speculatively cache another entry in the TLB. Depending on the random data it reads from an old table page, it may find a global entry (it's just a bit in the pte) which is not tagged with an ASID (application specific id). A latter flush_tlb_mm() only flushes the current ASID and doesn't touch global entries (used only by kernel mappings). So we end up with global TLB entry in user space that overrides any other application mapping.

Quoted 3 lines

Right, that's the typical scenario. I haven't looked at your flush implementation though, but surely you can defer the actual freeing so you can batch them & limit the number of TLB flushes right ?

Yes they do.. its just the up-front TLB invalidate for fullmm that's a problem.

s390 really wants this so it can avoid the per pte invalidate otherwise required by ptep_get_and_clear_full().

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#7 lore raw

Re: [PATCH 02/20] mm: Add optional TLB flush to generic RCU page-table freeing

In reply to Benjamin Herrenschmidt
Quoted 9 lines · Benjamin Herrenschmidt
Quoted 8 lines · Peter Zijlstra
Quoted 6 lines · Linus Torvalds

Plus it really isn't about hardware page table walkers at all. It's more about the possibility of speculative TLB fils, it has nothing to do with *how* they are done. Sure, it's likely that a software pagetable walker wouldn't be something that gets called speculatively, but it's not out of the question.

Quoted 7 lines
Quoted 3 lines

Hmm, I would call gup_fast() as speculative as we can get in software. It does a lock-less walk of the page-tables. That's what the RCU free'd page-table stuff is for to begin with.

Strictly speaking it's not :-) To *begin with* (as in the origin of that code) it comes from powerpc hash table code which walks the linux page tables locklessly :-) It then came in handy with gup_fast :-)

Ah, ok my bad.

Quoted 6 lines
Quoted 6 lines
Quoted 5 lines

IOW, if Sparc/PPC really want to guarantee that they never fill TLB entries speculatively, and that if we are in a kernel thread they will *never* fill the TLB with anything else, then make them enable CONFIG_STRICT_TLB_FILL or something in their architecture Kconfig files.

Quoted 8 lines
Quoted 4 lines

Since we've dealt with the speculative software side by using RCU-ish stuff, the only thing that's left is hardware, now neither sparc64 nor ppc actually know about the linux page-tables from what I understood, they only look at their hash-table thing.

Some embedded ppc's know about the lowest level (SW loaded PMD) but that's not an issue here. We flush these special TLB entries specifically and synchronously in __pte_free_tlb().

OK, I missed that.. is that arch/powerpc/mm/tlb_nohash.c:tlb_flush_pgtable() ?

Quoted 4 lines
Quoted 2 lines

So even if the hardware did do speculative tlb fills, it would do them from the hash-table, but that's already cleared out.

Right,

Phew at least I got the important thing right ;-)

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#35 lore raw

Re: [PATCH 08/20] mm: Optimize fullmm TLB flushing

In reply to Peter Zijlstra
Quoted 10 lines · On Thu, 28 Jun 2012 12:55:04 +0200 Peter Zijlstra <[email protected]> wrote:
Quoted 5 lines · Linus Torvalds

IOW, the point I'm trying to make is that even if there are zero *actual* accesses of user space (because user space is dead, and the kernel hopefully does no "get_user()/put_user()" stuff at this point any more), the CPU may speculatively use user addresses for the bog-standard kernel addresses that happen.

Right.. and s390 having done this only says that s390 appears to be ok with it. Martin, does s390 hardware guarantee no speculative stuff like Linus explained, or might there even be a latent issue on s390?

The cpu can create speculative TLB entries, but only if it runs in the mode that uses the respective mm. We have two mm's active at the same time, the kernel mm (init_mm) and the user mm. While the cpu runs only in kernel mode it is not allowed to create TLBs for the user mm. While running in user mode it is allowed to speculatively create TLBs.

Quoted 11 lines

But it looks like we cannot do this in general, and esp. ARM (as already noted by Catalin) has very aggressive speculative behaviour.

The alternative is that we do a switch_mm() to init_mm instead of the TLB flush. On x86 that should be about the same cost, but I've not looked at other architectures yet.

The second and least favourite alternative is of course special casing this for s390 if it turns out its a safe thing to do for them.

/me goes look through arch code.

Basically we have two special requirements on s390:
1) do not modify ptes while attached to another cpu except with the special IPTE / IDTE instructions
2) do a TLB flush before freeing any kind of page table page, s390 needs a flush for pud, pmd & pte tables.

Signature · 10 lines
blue skies,
   Martin.

"Reality continues to ruin my life." - Calvin.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
maintainer
#36 lore raw

Re: [PATCH 08/20] mm: Optimize fullmm TLB flushing

Quoted 5 lines · Martin Schwidefsky

The cpu can create speculative TLB entries, but only if it runs in the mode that uses the respective mm. We have two mm's active at the same time, the kernel mm (init_mm) and the user mm. While the cpu runs only in kernel mode it is not allowed to create TLBs for the user mm. While running in user mode it is allowed to speculatively create TLBs.

OK, that's neat.

Quoted 3 lines

Basically we have two special requirements on s390:
1) do not modify ptes while attached to another cpu except with the special IPTE / IDTE instructions

Right, and your fullmm case works by doing a global invalidate after all threads have ceased userspace execution, this allows you to do away with the IPTE/IDTE instructions since there's no other active cpus on the userspace mm anymore.

Quoted 2 lines

2) do a TLB flush before freeing any kind of page table page, s390 needs a flush for pud, pmd & pte tables.

Right, we do that (now)..

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#8 lore raw

Re: [PATCH 02/20] mm: Add optional TLB flush to generic RCU page-table freeing

In reply to Peter Zijlstra
Quoted 7 lines · Peter Zijlstra
Quoted 4 lines

Some embedded ppc's know about the lowest level (SW loaded PMD) but that's not an issue here. We flush these special TLB entries specifically and synchronously in __pte_free_tlb().

OK, I missed that.. is that arch/powerpc/mm/tlb_nohash.c:tlb_flush_pgtable() ?

Yup.

Quoted 7 lines
Quoted 1 line
Quoted 1 line

So even if the hardware did do speculative tlb fills, it would do

them

Quoted 3 lines
Quoted 1 line

from the hash-table, but that's already cleared out.

Right,

Phew at least I got the important thing right ;-)

Yeah as long as we have that hash :-) The day we move on (if ever) it will be as bad as ARM :-)

Cheers,
Ben.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#24 lore raw

Re: [PATCH 08/20] mm: Optimize fullmm TLB flushing

In reply to Peter Zijlstra
Quoted 16 lines · Peter Zijlstra
Quoted 15 lines · Benjamin Herrenschmidt
Quoted 13 lines · Catalin Marinas

That's definitely an issue on ARM and it was hit on older kernels. Basically ARM processors can cache any page translation level in the TLB. We need to make sure that no page entry at any level (either cached in the TLB or not) points to an invalid next level table (hence the TLB shootdown). For example, in cases like free_pgd_range(), if the cached pgd entry points to an already freed pud/pmd table (pgd_clear is not enough) it may walk the page tables speculatively cache another entry in the TLB. Depending on the random data it reads from an old table page, it may find a global entry (it's just a bit in the pte) which is not tagged with an ASID (application specific id). A latter flush_tlb_mm() only flushes the current ASID and doesn't touch global entries (used only by kernel mappings). So we end up with global TLB entry in user space that overrides any other application mapping.

Quoted 6 lines
Quoted 3 lines

Right, that's the typical scenario. I haven't looked at your flush implementation though, but surely you can defer the actual freeing so you can batch them & limit the number of TLB flushes right ?

Yes they do.. its just the up-front TLB invalidate for fullmm that's a problem.

The upfront invalidate is fine (i.e. harmless), it's the tlb_flush_mmu() change to check for !tlb->fullmm that's not helpful on ARM.

Signature · 7 lines
Catalin

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
#37 lore raw

Re: [PATCH 08/20] mm: Optimize fullmm TLB flushing

In reply to Peter Zijlstra
Quoted 9 lines · Peter Zijlstra
Quoted 5 lines · Martin Schwidefsky

The cpu can create speculative TLB entries, but only if it runs in the mode that uses the respective mm. We have two mm's active at the same time, the kernel mm (init_mm) and the user mm. While the cpu runs only in kernel mode it is not allowed to create TLBs for the user mm. While running in user mode it is allowed to speculatively create TLBs.

OK, that's neat.

Note that we can do that for x86 now using the new PCID feature. Basically you get a tagged TLB, so you can switch between the kernel-only address space and the kernel+user address space quickly.

It's still going to be slower than what we do now, but it might please some security people if the kernel can't accidentally access user data.

Signature · 8 lines
error compiling committee.c: too many arguments to function


--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
maintainer
#25 lore raw

Re: [PATCH 08/20] mm: Optimize fullmm TLB flushing

In reply to Catalin Marinas
Quoted 5 lines · Catalin Marinas
Quoted 2 lines

Yes they do.. its just the up-front TLB invalidate for fullmm that's a problem.

The upfront invalidate is fine (i.e. harmless), it's the tlb_flush_mmu() change to check for !tlb->fullmm that's not helpful on ARM.

I think we're saying the same but differently. The point is that the flush up front isn't sufficient for most of us.

Also, we'd very much want to avoid superfluous flushes since they are somewhat expensive.

How horrid is something like the below. It detaches the mm so that hardware speculation simply doesn't matter.

Now the switch_mm should imply the same cache+TBL flush we'd otherwise do, and I'd think that that would be the majority of the cost. Am I wrong there?

Also, the below seems to leak mm_structs so I did mess up the ref-counting, its too bloody hot here.

---
 mm/memory.c |   51 +++++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 47 insertions(+), 4 deletions(-)
mm/memory.c +47 -4
@@ -65,6 +65,7 @@
 #include <asm/tlb.h>
 #include <asm/tlbflush.h>
 #include <asm/pgtable.h>
+#include <asm/mmu_context.h>
 
 #include "internal.h"
 
@@ -197,6 +198,33 @@ static int tlb_next_batch(struct mmu_gat
 	return 1;
 }
 
+/*
+ * Anonymize the task by detaching the mm and attaching it
+ * to the init_mm.
+ */
+static void detach_mm(struct mm_struct *mm, struct task_struct *tsk)
+{
+	/*
+	 * We should only be called when there's no users left and we're
+	 * destroying the mm.
+	 */
+	VM_BUG_ON(atomic_read(&mm->mm_users));
+	VM_BUG_ON(tsk->mm != mm);
+	VM_BUG_ON(mm == &init_mm);
+
+	task_lock(tsk);
+	tsk->mm = NULL;
+	tsk->active_mm = &init_mm;
+	switch_mm(mm, &init_mm, tsk);
+	/*
+	 * We have to take an extra ref on init_mm for TASK_DEAD in
+	 * finish_task_switch(), we don't drop our mm->mm_count reference
+	 * since mmput() will do this.
+	 */
+	atomic_inc(&init_mm.mm_count);
+	task_unlock(tsk);
+}
+
 /* tlb_gather_mmu
  *	Called to initialize an (on-stack) mmu_gather structure for page-table
  *	tear-down from @mm. The @fullmm argument is used when @mm is without
@@ -215,16 +243,31 @@ void tlb_gather_mmu(struct mmu_gather *t
 	tlb->active     = &tlb->local;
 
 	tlb_table_init(tlb);
+
+	if (fullmm && current->mm == mm) {
+		/*
+		 * Instead of doing:
+		 *
+		 *  flush_cache_mm(mm);
+		 *  flush_tlb_mm(mm);
+		 *
+		 * We switch to init_mm, this context switch should imply both
+		 * the cache and TLB flush as well as guarantee that hardware
+		 * speculation cannot load TLBs on this mm anymore.
+		 */
+		detach_mm(mm, current);
+	}
 }
 
 void tlb_flush_mmu(struct mmu_gather *tlb)
 {
 	struct mmu_gather_batch *batch;
 
-	if (!tlb->need_flush)
-		return;
-	tlb->need_flush = 0;
-	flush_tlb_mm(tlb->mm);
+	if (!tlb->fullmm && tlb->need_flush) {
+		tlb->need_flush = 0;
+		flush_tlb_mm(tlb->mm);
+	}
+
 	tlb_table_flush(tlb);
 
 	if (tlb_fast_mode(tlb))

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

maintainer
#26 lore raw

Re: [PATCH 08/20] mm: Optimize fullmm TLB flushing

Quoted 3 lines · Peter Zijlstra

Now the switch_mm should imply the same cache+TBL flush we'd otherwise do, and I'd think that that would be the majority of the cost. Am I wrong there?

The advantage of doing this is that you don't need any of the batching and possibly multiple invalidate nonsense you otherwise need. So it might still be an over-all win, even if the switch is slightly more expensive than a regular flush. Simply because you can avoid most (if not all) the usual complexities.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#27 lore raw

Re: [PATCH 08/20] mm: Optimize fullmm TLB flushing

In reply to Peter Zijlstra
Quoted 3 lines · Peter Zijlstra

How horrid is something like the below. It detaches the mm so that hardware speculation simply doesn't matter.

Actually, that's wrong. Even when detached, kernel threads may still use that mm lazily. Now, that only happens on other CPU's (if any scheduling happens on *this* CPU, they will lazily take the mm of the thread it scheduled away from), but even if you detach the VM that doesn't mean that hardware speculation wouldn't matter. Kernel threads on other CPU's may still be doing TLB accesses.

Of course, I *think* that if we do an IPI on the thing, we also kick those kernel threads out of using that mm. So it may actually work if you also do that explicit TLB flush to make sure other CPU's don't have this MM. I don't think switch_mm() does that for you, it only does a local-cpu invalidate.

I didn't look at the code, though. Maybe I'm wrong in thinking that you are wrong.

Linus

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

maintainer
#28 lore raw

Re: [PATCH 08/20] mm: Optimize fullmm TLB flushing

Quoted 12 lines · Linus Torvalds
Quoted 3 lines · Peter Zijlstra

How horrid is something like the below. It detaches the mm so that hardware speculation simply doesn't matter.

Actually, that's wrong. Even when detached, kernel threads may still use that mm lazily. Now, that only happens on other CPU's (if any scheduling happens on *this* CPU, they will lazily take the mm of the thread it scheduled away from), but even if you detach the VM that doesn't mean that hardware speculation wouldn't matter. Kernel threads on other CPU's may still be doing TLB accesses.

Quoted 8 lines

Of course, I *think* that if we do an IPI on the thing, we also kick those kernel threads out of using that mm. So it may actually work if you also do that explicit TLB flush to make sure other CPU's don't have this MM. I don't think switch_mm() does that for you, it only does a local-cpu invalidate.

I didn't look at the code, though. Maybe I'm wrong in thinking that you are wrong.

No I think you're right (as always).. also an IPI will not force schedule the thread that might be running on the receiving cpu, also we'd have to wait for any such schedule to complete in order to guarantee the mm isn't lazily used anymore.

Bugger.. it would've been nice to do this. I guess I'd better go special case s390 for now until we can come up with something that would work.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#46 lore raw

Re: [PATCH 14/20] mm, sh: Convert sh to generic tlb

In reply to Peter Zijlstra
Quoted 6 lines · Peter Zijlstra
---
 arch/sh/Kconfig           |    1 
 arch/sh/include/asm/tlb.h |   98 ++--------------------------------------------
 2 files changed, 6 insertions(+), 93 deletions(-)

This blows up in the same way as last time.

I direct you to the same bug report and patch as before:

http://marc.info/?l=linux-kernel&m=133722116507075&w=2

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#47 lore raw

Re: [PATCH 14/20] mm, sh: Convert sh to generic tlb

Quoted 8 lines · Paul Mundt
Quoted 6 lines · Peter Zijlstra
---
 arch/sh/Kconfig           |    1 
 arch/sh/include/asm/tlb.h |   98 ++--------------------------------------------
 2 files changed, 6 insertions(+), 93 deletions(-)
Quoted 5 lines

This blows up in the same way as last time.

I direct you to the same bug report and patch as before:

http://marc.info/?l=linux-kernel&m=133722116507075&w=2

Sorry about that.. /me goes amend.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#29 lore raw

Re: [PATCH 08/20] mm: Optimize fullmm TLB flushing

In reply to Peter Zijlstra
Quoted 6 lines · Peter Zijlstra

No I think you're right (as always).. also an IPI will not force schedule the thread that might be running on the receiving cpu, also we'd have to wait for any such schedule to complete in order to guarantee the mm isn't lazily used anymore.

Bugger..

You can still do it if the mm count is 1 no ? Ie, current is the last holder of a reference to the mm struct... which will probably be the common case for short lived programs.

Cheers,
Ben.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#30 lore raw

Re: [PATCH 08/20] mm: Optimize fullmm TLB flushing

Quoted 11 lines · Benjamin Herrenschmidt
Quoted 6 lines · Peter Zijlstra

No I think you're right (as always).. also an IPI will not force schedule the thread that might be running on the receiving cpu, also we'd have to wait for any such schedule to complete in order to guarantee the mm isn't lazily used anymore.

Bugger..

You can still do it if the mm count is 1 no ? Ie, current is the last holder of a reference to the mm struct... which will probably be the common case for short lived programs.

Also I just remembered... x86 flushes in SMP via IPIs right ? So maybe you can invent a "detach and flush" variant of it ?

Cheers,
Ben.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

maintainer
#31 lore raw

Re: [PATCH 08/20] mm: Optimize fullmm TLB flushing

Quoted 7 lines · Benjamin Herrenschmidt
Quoted 6 lines · Benjamin Herrenschmidt
Quoted 5 lines · Peter Zijlstra

No I think you're right (as always).. also an IPI will not force schedule the thread that might be running on the receiving cpu, also we'd have to wait for any such schedule to complete in order to guarantee the mm isn't lazily used anymore.

Quoted 8 lines
Quoted 5 lines
Quoted 1 line

Bugger..

You can still do it if the mm count is 1 no ? Ie, current is the last holder of a reference to the mm struct... which will probably be the common case for short lived programs.

Also I just remembered... x86 flushes in SMP via IPIs right ? So maybe you can invent a "detach and flush" variant of it ?

Its not just x86 I worry about.. I want to share as much as possible between all our architectures.

But yeah, I could do it for mm_count == 1, but I'd still need to special case s390 because they always want it.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#32 lore raw

Re: [PATCH 08/20] mm: Optimize fullmm TLB flushing

In reply to Benjamin Herrenschmidt
Quoted 11 lines · Benjamin Herrenschmidt
Quoted 6 lines · Peter Zijlstra

No I think you're right (as always).. also an IPI will not force schedule the thread that might be running on the receiving cpu, also we'd have to wait for any such schedule to complete in order to guarantee the mm isn't lazily used anymore.

Bugger..

You can still do it if the mm count is 1 no ? Ie, current is the last holder of a reference to the mm struct... which will probably be the common case for short lived programs.

BTW, can we not move the free_pgtables() call in exit_mmap() to __mmdrop()? Something like below but I'm not entirely sure about its implications:

include/linux/mm.h +1 -0
index b36d08c..507ee9f 100644
@@ -1372,6 +1372,7 @@ extern void unlink_file_vma(struct vm_area_struct *);
 extern struct vm_area_struct *copy_vma(struct vm_area_struct **,
 	unsigned long addr, unsigned long len, pgoff_t pgoff);
 extern void exit_mmap(struct mm_struct *);
+extern void exit_pgtables(struct mm_struct *mm);
 
 extern int mm_take_all_locks(struct mm_struct *mm);
 extern void mm_drop_all_locks(struct mm_struct *mm);
kernel/fork.c +1 -0
index ab5211b..3412b1a 100644
@@ -588,6 +588,7 @@ struct mm_struct *mm_alloc(void)
 void __mmdrop(struct mm_struct *mm)
 {
 	BUG_ON(mm == &init_mm);
+	exit_pgtables(mm);
 	mm_free_pgd(mm);
 	destroy_context(mm);
 	mmu_notifier_mm_destroy(mm);
mm/mmap.c +13 -2
index 074b487..d9ebfdb 100644
@@ -2269,7 +2269,6 @@ void exit_mmap(struct mm_struct *mm)
 {
 	struct mmu_gather tlb;
 	struct vm_area_struct *vma;
-	unsigned long nr_accounted = 0;
 
 	/* mm's last user has gone, and its about to be pulled down */
 	mmu_notifier_release(mm);
@@ -2291,11 +2290,23 @@ void exit_mmap(struct mm_struct *mm)
 
 	lru_add_drain();
 	flush_cache_mm(mm);
-	tlb_gather_mmu(&tlb, mm, 1);
+	tlb_gather_mmu(&tlb, mm, 0);
 	/* update_hiwater_rss(mm) here? but nobody should be looking */
 	/* Use -1 here to ensure all VMAs in the mm are unmapped */
 	unmap_vmas(&tlb, vma, 0, -1);
+	tlb_finish_mmu(&tlb, 0, -1);
+}
+
+void exit_pgtables(struct mm_struct *mm)
+{
+	struct mmu_gather tlb;
+	struct vm_area_struct *vma;
+	unsigned long nr_accounted = 0;
 
+	vma = mm->mmap;
+	if (!vma)
+		return;
+	tlb_gather_mmu(&tlb, mm, 1);
 	free_pgtables(&tlb, vma, FIRST_USER_ADDRESS, TASK_SIZE);
 	tlb_finish_mmu(&tlb, 0, -1);
 
Signature · 7 lines
Catalin

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
#33 lore raw

Re: [PATCH 08/20] mm: Optimize fullmm TLB flushing

Quoted 9 lines · Catalin Marinas
Quoted 8 lines · Benjamin Herrenschmidt
Quoted 6 lines · Peter Zijlstra

No I think you're right (as always).. also an IPI will not force schedule the thread that might be running on the receiving cpu, also we'd have to wait for any such schedule to complete in order to guarantee the mm isn't lazily used anymore.

Bugger..

Quoted 7 lines
Quoted 3 lines

You can still do it if the mm count is 1 no ? Ie, current is the last holder of a reference to the mm struct... which will probably be the common case for short lived programs.

BTW, can we not move the free_pgtables() call in exit_mmap() to __mmdrop()? Something like below but I'm not entirely sure about its implications:

The main one is that it might remain active on another core for a -loooong- time if that cores is only running kernel threads or otherwise idle, thus wasting memory etc...

Also, mm_count being 1 is probably the common case for many short lived processes, so it should be fine, I don't think the count can every increase back at that point can it ? (we could make sure it doesn't, mark the mm as dead and WARN loudly if somebody tries to increase the count).

The advantage of doing a "detach & flush" IPI if the count is larger is that you already do the IPI for flushing anyway, so you just add a detach to the path.

That avoids the problem of the mm staying around for too long as well.

Cheers,
Ben.

Quoted 53 lines
include/linux/mm.h +1 -0
index b36d08c..507ee9f 100644
@@ -1372,6 +1372,7 @@ extern void unlink_file_vma(struct vm_area_struct *);
 extern struct vm_area_struct *copy_vma(struct vm_area_struct **,
 	unsigned long addr, unsigned long len, pgoff_t pgoff);
 extern void exit_mmap(struct mm_struct *);
+extern void exit_pgtables(struct mm_struct *mm);
 
 extern int mm_take_all_locks(struct mm_struct *mm);
 extern void mm_drop_all_locks(struct mm_struct *mm);
kernel/fork.c +1 -0
index ab5211b..3412b1a 100644
@@ -588,6 +588,7 @@ struct mm_struct *mm_alloc(void)
 void __mmdrop(struct mm_struct *mm)
 {
 	BUG_ON(mm == &init_mm);
+	exit_pgtables(mm);
 	mm_free_pgd(mm);
 	destroy_context(mm);
 	mmu_notifier_mm_destroy(mm);
mm/mmap.c +7 -2
index 074b487..d9ebfdb 100644
@@ -2269,7 +2269,6 @@ void exit_mmap(struct mm_struct *mm)
 {
 	struct mmu_gather tlb;
 	struct vm_area_struct *vma;
-	unsigned long nr_accounted = 0;
 
 	/* mm's last user has gone, and its about to be pulled down */
 	mmu_notifier_release(mm);
@@ -2291,11 +2290,23 @@ void exit_mmap(struct mm_struct *mm)
 
 	lru_add_drain();
 	flush_cache_mm(mm);
-	tlb_gather_mmu(&tlb, mm, 1);
+	tlb_gather_mmu(&tlb, mm, 0);
 	/* update_hiwater_rss(mm) here? but nobody should be looking */
 	/* Use -1 here to ensure all VMAs in the mm are unmapped */
 	unmap_vmas(&tlb, vma, 0, -1);
+	tlb_finish_mmu(&tlb, 0, -1);
+}
+
+void exit_pgtables(struct mm_struct *mm)
+{
+	struct mmu_gather tlb;

mm/mmap.c void exit_mmap(struct mm_struct *mm) · line 2312

Quoted 10 lines
mm/mmap.c +6 -0
+	struct vm_area_struct *vma;
+	unsigned long nr_accounted = 0;
 
+	vma = mm->mmap;
+	if (!vma)
+		return;
+	tlb_gather_mmu(&tlb, mm, 1);
 	free_pgtables(&tlb, vma, FIRST_USER_ADDRESS, TASK_SIZE);
 	tlb_finish_mmu(&tlb, 0, -1);
 

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#9 lore raw

Re: [PATCH 02/20] mm: Add optional TLB flush to generic RCU page-table freeing

In reply to Peter Zijlstra
Quoted 14 lines · Peter Zijlstra
(quoted diff) +14 -0
+#ifdef CONFIG_STRICT_TLB_FILL
+/*
+ * Some archictures (sparc64, ppc) cannot refill TLBs after the they've removed
+ * the PTE entries from their hash-table. Their hardware never looks at the
+ * linux page-table structures, so they don't need a hardware TLB invalidate
+ * when tearing down the page-table structure itself.
+ */
+static inline void tlb_table_flush_mmu(struct mmu_gather *tlb) { }
+#else
+static inline void tlb_table_flush_mmu(struct mmu_gather *tlb)
+{
+	tlb_flush_mmu(tlb);
+}
+#endif

Review

Quoted 10 lines
(quoted diff) +2 -0
+
 void tlb_table_flush(struct mmu_gather *tlb)
 {
 	struct mmu_table_batch **batch = &tlb->batch;
 
 	if (*batch) {
+		tlb_table_flush_mmu(tlb);
 		call_rcu_sched(&(*batch)->rcu, tlb_remove_table_rcu);
 		*batch = NULL;
 	}

Hi Peter,

When running munmap(https://lkml.org/lkml/2012/5/17/59) test with KVM and pvflush patches I got a crash. I have verified that the crash happens on the base(non virt) as well when I have CONFIG_HAVE_RCU_TABLE_FREE defined. Here is the crash details and my analysis below:

-----------------------------------------------------------------------

BUG: unable to handle kernel NULL pointer dereference at 0000000000000008
IP: [<ffffffff810d31d9>] __call_rcu+0x29/0x1c0
PGD 0 
Oops: 0002 [#1] SMP 
CPU 24 
Modules linked in: kvm_intel kvm [last unloaded: scsi_wait_scan]

Pid: 32643, comm: munmap Not tainted 3.5.0-rc7+ #46 IBM System x3850 X5 -[7042CR6]-[root@mx3850x5 ~/Node 1, Processor Card]#

RIP: 0010:[<ffffffff810d31d9>]  [<ffffffff810d31d9>] __call_rcu+0x29/0x1c0
RSP: 0018:ffff88203164fc28  EFLAGS: 00010246
RAX: ffff88203164fba8 RBX: 0000000000000000 RCX: 0000000000000000
RDX: ffffffff81e34280 RSI: ffffffff81130330 RDI: 0000000000000000
RBP: ffff88203164fc58 R08: ffffea00d2680340 R09: 0000000000000000
R10: ffff883c7fbd4ef8 R11: 0000000000000078 R12: ffffffff81130330
R13: 00007f09ee803000 R14: ffff883c2fa5bab0 R15: ffff88203164fe08
FS:  00007f09ee7ee700(0000) GS:ffff883c7fc00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 0000000000000008 CR3: 0000000001e0b000 CR4: 00000000000007e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Process munmap (pid: 32643, threadinfo ffff88203164e000, task ffff882030458a70)
Stack:
 ffff883c2fa5bab0 ffff88203164fe08 ffff88203164fc68 ffff88203164fe08
 ffff88203164fe08 00007f09ee803000 ffff88203164fc68 ffffffff810d33c7
 ffff88203164fc88 ffffffff81130e0d ffff88203164fc88 ffffea00d28e54f8
Call Trace:
 [<ffffffff810d33c7>] call_rcu_sched+0x17/0x20
 [<ffffffff81130e0d>] tlb_table_flush+0x2d/0x40
 [<ffffffff81130e80>] tlb_remove_table+0x60/0xc0
 [<ffffffff8103a5e3>] ___pte_free_tlb+0x63/0x70
 [<ffffffff81131b38>] free_pgd_range+0x298/0x4b0
 [<ffffffff81131e1e>] free_pgtables+0xce/0x120
 [<ffffffff81137247>] exit_mmap+0xa7/0x160
 [<ffffffff81043fdf>] mmput+0x6f/0xf0
 [<ffffffff8104c3f5>] exit_mm+0x105/0x130
 [<ffffffff810d6c7d>] ? taskstats_exit+0x17d/0x240
 [<ffffffff8104c596>] do_exit+0x176/0x480
 [<ffffffff8104c8f5>] do_group_exit+0x55/0xd0
 [<ffffffff8104c987>] sys_exit_group+0x17/0x20
 [<ffffffff818a3829>] system_call_fastpath+0x16/0x1b
Code: ff ff 55 48 89 e5 48 83 ec 30 48 89 5d e8 4c 89 65 f0 4c 89 6d f8 66 66 66 66 90 40 f6 c7 03 48 89 fb 49 89 f4 0f 85 19 01 00 00 <4c> 89 63 08 48 c7 03 00 00 00 00 0f ae f0 9c 58 66 66 90 66 90 
RIP  [<ffffffff810d31d9>] __call_rcu+0x29/0x1c0
 RSP <ffff88203164fc28>
CR2: 0000000000000008
---[ end trace 3ed30a91ea7cb375 ]---

----------------------------------------------------------------------------

I think this is what is happening:

___pte_free_tlb
tlb_remove_table
tlb_table_flush
tlb_table_flush_mmu
tlb_flush_mmu
Sets need_flush = 0
tlb_table_flush (if CONFIG_HAVE_RCU_TABLE_FREE)
[Gets called twice with same *tlb!]

tlb_table_flush_mmu
tlb_flush_mmu(nop as need_flush is 0)
call_rcu_sched(&(*batch)->rcu,...); *batch = NULL;
call_rcu_sched(&(*batch)->rcu,...); <---- *batch would be NULL

I verified this by putting following fix and do not see the crash anymore:

mm/memory.c +2 -1
index 1797bc1..329fcb9 100644
@@ -367,7 +367,8 @@ void tlb_table_flush(struct mmu_gather *tlb)
 
 	if (*batch) {
 		tlb_table_flush_mmu(tlb);
-		call_rcu_sched(&(*batch)->rcu, tlb_remove_table_rcu);
+		if(*batch)
+			call_rcu_sched(&(*batch)->rcu, tlb_remove_table_rcu);
 		*batch = NULL;
 	}
 }

Thanks
Nikunj

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>