Skip to contentSkip to conversation

Fixup page directory freeing

PATCH In review Peter Zijlstra 2 participants 14 messages Jul 17, 2020 – Sep 21, 2020

Subsystems (10)
Memory management MIPS RISC-V SuperH +6 more subsystems
lore
Share & tools
lore

Peter Zijlstra

author maintainer
#0 lore raw

v2 cover Fixup page directory freeing

Hi All,

While fixing a silly bug on SH (patch #1), I realized that even with the trivial patch to restore prior behaviour, page directory freeing was still broken.

The thing is, on anything SMP, freeing page directories should observe the exact same order as normal page freeing:

1) unhook page/directory
2) TLB invalidate
3) free page/directory

Without this any concurrent page-table walk could end up with a Use-after-Free. This is esp. trivial for anything that has software page-table walkers (HAVE_FAST_GUP / software TLB fill) or the hardware caches partial page-walks (ie. caches page directories).

Even on UP this might give issues, since mmu_gather is preemptible these days. An interrupt or preempted task accessing user pages might stumble into the free page if the hardware caches page directories.

So I've converted everything to always observe the above order, simply so we don't have to worry about it.

If however I've been over zealous and your arch/mmu really doesn't need this and you're offended by this potentially superfluous code, please let me know and I'll replace the patch with one that adds a comment describing your rationale for why it is not needed.

v1: https://lkml.kernel.org/r/[email protected]

author maintainer
#1 lore raw

PATCH v2 1/11 asm-generic/tlb: Fix MMU_GATHER_TABLE_FREE

The first MMU_GATHER_TABLE_FREE user showed a logic error in the tlb_needs_table_invalidate() definition. Make sure any TABLE_FREE has it defined.

Fixes 0d6e24d430ef ("asm-generic/tlb: provide MMU_GATHER_TABLE_FREE")
---
 include/asm-generic/tlb.h |   35 +++++++++++++++++++----------------
 1 file changed, 19 insertions(+), 16 deletions(-)
include/asm-generic/tlb.h +19 -16
@@ -172,6 +172,18 @@
  *  various ptep_get_and_clear() functions.
  */
 
+#ifndef CONFIG_MMU_GATHER_RCU_TABLE_FREE
+
+/*
+ * Only RCU table free can override this; otherwise the TLBI is needed to
+ * provide existence guarantees for software walkers.
+ */
+#ifdef tlb_needs_table_invalidate
+#error tlb_needs_table_invalidate() requires MMU_GATHER_RCU_TABLE_FREE
+#endif
+
+#endif /* CONFIG_MMU_GATHER_RCU_TABLE_FREE */
+
 #ifdef CONFIG_MMU_GATHER_TABLE_FREE
 
 struct mmu_table_batch {
@@ -187,17 +199,6 @@ struct mmu_table_batch {
 
 extern void tlb_remove_table(struct mmu_gather *tlb, void *table);
 
-#else /* !CONFIG_MMU_GATHER_HAVE_TABLE_FREE */
-
-/*
- * Without MMU_GATHER_TABLE_FREE the architecture is assumed to have page based
- * page directories and we can use the normal page batching to free them.
- */
-#define tlb_remove_table(tlb, page) tlb_remove_page((tlb), (page))
-
-#endif /* CONFIG_MMU_GATHER_TABLE_FREE */
-
-#ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE
 /*
  * This allows an architecture that does not use the linux page-tables for
  * hardware to skip the TLBI when freeing page tables.
@@ -206,13 +207,15 @@ extern void tlb_remove_table(struct mmu_
 #define tlb_needs_table_invalidate() (true)
 #endif
 
-#else
+#else /* !CONFIG_MMU_GATHER_HAVE_TABLE_FREE */
 
-#ifdef tlb_needs_table_invalidate
-#error tlb_needs_table_invalidate() requires MMU_GATHER_RCU_TABLE_FREE
-#endif
+/*
+ * Without MMU_GATHER_TABLE_FREE the architecture is assumed to have page based
+ * page directories and we can use the normal page batching to free them.
+ */
+#define tlb_remove_table(tlb, page) tlb_remove_page((tlb), (page))
 
-#endif /* CONFIG_MMU_GATHER_RCU_TABLE_FREE */
+#endif /* CONFIG_MMU_GATHER_TABLE_FREE */
 
 
 #ifndef CONFIG_MMU_GATHER_NO_GATHER
maintainer
#2 lore raw

Re: [PATCH v2 01/11] asm-generic/tlb: Fix MMU_GATHER_TABLE_FREE

Quoted 3 lines · Peter Zijlstra

The first MMU_GATHER_TABLE_FREE user showed a logic error in the tlb_needs_table_invalidate() definition. Make sure any TABLE_FREE has it defined.

Could you elaborate on the logic error, please? It's difficult to see through all of the #ifdefs, but afaict we #error if tlb_needs_table_invalidate() is defined but not CONFIG_MMU_GATHER_RCU_TABLE_FREE with and without this patch. In other words, I'm failing to see what this patch changes!

Quoted 46 lines
Fixes 0d6e24d430ef ("asm-generic/tlb: provide MMU_GATHER_TABLE_FREE")
---
 include/asm-generic/tlb.h |   35 +++++++++++++++++++----------------
 1 file changed, 19 insertions(+), 16 deletions(-)
include/asm-generic/tlb.h +12 -11
@@ -172,6 +172,18 @@
  *  various ptep_get_and_clear() functions.
  */
 
+#ifndef CONFIG_MMU_GATHER_RCU_TABLE_FREE
+
+/*
+ * Only RCU table free can override this; otherwise the TLBI is needed to
+ * provide existence guarantees for software walkers.
+ */
+#ifdef tlb_needs_table_invalidate
+#error tlb_needs_table_invalidate() requires MMU_GATHER_RCU_TABLE_FREE
+#endif
+
+#endif /* CONFIG_MMU_GATHER_RCU_TABLE_FREE */
+
 #ifdef CONFIG_MMU_GATHER_TABLE_FREE
 
 struct mmu_table_batch {
@@ -187,17 +199,6 @@ struct mmu_table_batch {
 
 extern void tlb_remove_table(struct mmu_gather *tlb, void *table);
 
-#else /* !CONFIG_MMU_GATHER_HAVE_TABLE_FREE */
-
-/*
- * Without MMU_GATHER_TABLE_FREE the architecture is assumed to have page based
- * page directories and we can use the normal page batching to free them.
- */
-#define tlb_remove_table(tlb, page) tlb_remove_page((tlb), (page))
-
-#endif /* CONFIG_MMU_GATHER_TABLE_FREE */
-
-#ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE
 /*
  * This allows an architecture that does not use the linux page-tables for
  * hardware to skip the TLBI when freeing page tables.

include/asm-generic/tlb.h extern void tlb_remove_table(struct mmu_ · line 210

Quoted 5 lines
include/asm-generic/tlb.h +1 -1
@@ -206,13 +207,15 @@ extern void tlb_remove_table(struct mmu_
 #define tlb_needs_table_invalidate() (true)
 #endif
 
-#else
+#else /* !CONFIG_MMU_GATHER_HAVE_TABLE_FREE */

While you're at it, this comment can be fixed to refer to CONFIG_MMU_GATHER_RCU_TABLE_FREE.

Will

author maintainer
#3 lore raw

PATCH v2 2/11 sh/tlb: Fix PGTABLE_LEVELS > 2

In reply to Peter Zijlstra

Geert reported that his SH7722-based Migo-R board failed to boot after commit:

c5b27a889da9 ("sh/tlb: Convert SH to generic mmu_gather")

That commit fell victim to copying the wrong pattern -- __pmd_free_tlb() used to be implemented with pmd_free().

Fixes c5b27a889da9 ("sh/tlb: Convert SH to generic mmu_gather")

---

arch/sh/include/asm/pgalloc.h +1 -9
@@ -12,6 +12,7 @@ extern void pgd_free(struct mm_struct *m
 extern void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd);
 extern pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address);
 extern void pmd_free(struct mm_struct *mm, pmd_t *pmd);
+#define __pmd_free_tlb(tlb, pmdp, addr)		pmd_free((tlb)->mm, (pmdp))
 #endif
 
 static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd,
@@ -33,13 +34,4 @@ do {							\
 	tlb_remove_page((tlb), (pte));			\
 } while (0)
 
-#if CONFIG_PGTABLE_LEVELS > 2
-#define __pmd_free_tlb(tlb, pmdp, addr)			\
-do {							\
-	struct page *page = virt_to_page(pmdp);		\
-	pgtable_pmd_page_dtor(page);			\
-	tlb_remove_page((tlb), page);			\
-} while (0);
-#endif
-
 #endif /* __ASM_SH_PGALLOC_H */
author maintainer
#4 lore raw

PATCH v2 3/11 sh/tlb: Fix __pmd_free_tlb()

In reply to Peter Zijlstra

SH violates the freeing order for it's PMD page directories. It's __pmd_free_tlb() does not ensure there is a TLB invalidation between itself and the eventualy freeing of the page.

Further complicating the situation is that SH uses non page based allocation for it's PMDs.

Use the shiny new HAVE_TABLE_FREE option to enable a custom page table freeer.

(SuperH uses IPI based TLB invalidation and therefore doesn't need HAVE_RCU_TABLE_FREE for its HAVE_FAST_GUP).

---
 arch/sh/Kconfig               |    1 +
 arch/sh/include/asm/pgalloc.h |    3 ++-
 arch/sh/mm/pgtable.c          |    6 ++++++
 3 files changed, 9 insertions(+), 1 deletion(-)
arch/sh/Kconfig +1 -0
@@ -14,6 +14,7 @@ config SUPERH
 	select HAVE_PERF_EVENTS
 	select HAVE_DEBUG_BUGVERBOSE
 	select HAVE_FAST_GUP if MMU
+	select MMU_GATHER_TABLE_FREE if X2TLB
 	select ARCH_HAVE_CUSTOM_GPIO_H
 	select ARCH_HAVE_NMI_SAFE_CMPXCHG if (GUSA_RB || CPU_SH4A)
 	select ARCH_HAS_GCOV_PROFILE_ALL
arch/sh/include/asm/pgalloc.h +2 -1
@@ -12,7 +12,8 @@ extern void pgd_free(struct mm_struct *m
 extern void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd);
 extern pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address);
 extern void pmd_free(struct mm_struct *mm, pmd_t *pmd);
-#define __pmd_free_tlb(tlb, pmdp, addr)		pmd_free((tlb)->mm, (pmdp))
+extern void __tlb_remove_table(void *table);
+#define __pmd_free_tlb(tlb, pmdp, addr)	tlb_remove_table((tlb), (pmdp))
 #endif
 
 static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd,
arch/sh/mm/pgtable.c +6 -0
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 #include <linux/mm.h>
 #include <linux/slab.h>
+#include <asm/pgalloc.h>
 
 #define PGALLOC_GFP GFP_KERNEL | __GFP_ZERO
 
@@ -55,4 +56,9 @@ void pmd_free(struct mm_struct *mm, pmd_
 {
 	kmem_cache_free(pmd_cachep, pmd);
 }
+
+void __tlb_remove_table(void *table)
+{
+	pmd_free(NULL, table);
+}
 #endif /* PAGETABLE_LEVELS > 2 */
author maintainer
#5 lore raw

PATCH v2 4/11 sparc32/tlb: Fix __p*_free_tlb()

In reply to Peter Zijlstra

Just like regular pages, page directories need to observe the following order:

1) unhook
2) TLB invalidate
3) free

to ensure it is safe against concurrent accesses.

Because Sparc32 has non-page based page directories, use a custom table freeer.

---
 arch/sparc/Kconfig                  |    1 +
 arch/sparc/include/asm/pgalloc_32.h |    7 +++++--
 arch/sparc/mm/srmmu.c               |   28 ++++++++++++++++++++++++++++
 3 files changed, 34 insertions(+), 2 deletions(-)
arch/sparc/Kconfig +1 -0
@@ -57,6 +57,7 @@ config SPARC32
 	select CLZ_TAB
 	select HAVE_UID16
 	select OLD_SIGACTION
+	select MMU_GATHER_TABLE_FREE
 
 config SPARC64
 	def_bool 64BIT
arch/sparc/include/asm/pgalloc_32.h +5 -2
@@ -12,6 +12,9 @@
 
 struct page;
 
+extern void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int idx);
+extern void __tlb_remove_table(void *table);
+
 void *srmmu_get_nocache(int size, int align);
 void srmmu_free_nocache(void *addr, int size);
 
@@ -48,7 +51,7 @@ static inline void free_pmd_fast(pmd_t *
 }
 
 #define pmd_free(mm, pmd)		free_pmd_fast(pmd)
-#define __pmd_free_tlb(tlb, pmd, addr)	pmd_free((tlb)->mm, pmd)
+#define __pmd_free_tlb(tlb, pmd, addr)	pgtable_free_tlb((tlb), (pmd), 1)
 
 #define pmd_populate(mm, pmd, pte)	pmd_set(pmd, pte)
 #define pmd_pgtable(pmd)		(pgtable_t)__pmd_page(pmd)
@@ -73,6 +76,6 @@ static inline void free_pte_fast(pte_t *
 #define pte_free_kernel(mm, pte)	free_pte_fast(pte)
 
 void pte_free(struct mm_struct * mm, pgtable_t pte);
-#define __pte_free_tlb(tlb, pte, addr)	pte_free((tlb)->mm, pte)
+#define __pte_free_tlb(tlb, pte, addr)	pgtable_free_tlb((tlb), (pte), 0)
 
 #endif /* _SPARC_PGALLOC_H */
arch/sparc/mm/srmmu.c +28 -0
@@ -38,6 +38,7 @@
 #include <asm/page.h>
 #include <asm/asi.h>
 #include <asm/smp.h>
+#include <asm/tlb.h>
 #include <asm/io.h>
 
 /* Now the cpu specific definitions. */
@@ -1831,3 +1832,30 @@ void __init load_mmu(void)
 		sun4m_init_smp();
 #endif
 }
+
+#define TLB_IDX_MASK	1UL
+
+void __tlb_remove_table(void *table)
+{
+	void *dir = (void *)((unsigned long)table & ~TLB_IDX_MASK);
+	int idx = (unsigned long)table & TLB_IDX_MASK;
+
+	switch (idx) {
+	case 1: /* PMD */
+		pmd_free(NULL, dir);
+		break;
+	case 0: /* PTE */
+		pte_free(NULL, dir);
+		break;
+	}
+}
+
+void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int idx)
+{
+	unsigned long pgf = (unsigned long)table;
+	BUG_ON(idx > TLB_IDX_MASK);
+	BUG_ON(pgf & TLB_IDX_MASK);
+	pgf |= idx;
+	tlb_remove_table(tlb, (void *)pgf);
+}
+
author maintainer
#6 lore raw

PATCH v2 5/11 parisc/tlb: Fix __p*_free_tlb()

In reply to Peter Zijlstra

Just like regular pages, page directories need to observe the following order:

1) unhook
2) TLB invalidate
3) free

to ensure it is safe against concurrent accesses.

Because PARISC has non-page-size PMDs, use a custom table freeer.

---
 arch/parisc/Kconfig               |    1 +
 arch/parisc/include/asm/pgalloc.h |   23 ++++++++++++++++++++---
 arch/parisc/include/asm/tlb.h     |    9 +++++----
 3 files changed, 26 insertions(+), 7 deletions(-)
arch/parisc/Kconfig +1 -0
@@ -63,6 +63,7 @@ config PARISC
 	select HAVE_KPROBES_ON_FTRACE
 	select HAVE_DYNAMIC_FTRACE_WITH_REGS
 	select HAVE_COPY_THREAD_TLS
+	select MMU_GATHER_TABLE_FREE if PGTABLE_LEVELS >= 3
 
 	help
 	  The PA-RISC microprocessor is designed by Hewlett-Packard and used
arch/parisc/include/asm/pgalloc.h +20 -3
@@ -73,7 +73,7 @@ static inline pmd_t *pmd_alloc_one(struc
 	return pmd;
 }
 
-static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
+static inline bool __pmd_free(struct mm_struct *mm, pmd_t *pmd)
 {
 	if (pmd_flag(*pmd) & PxD_FLAG_ATTACHED) {
 		/*
@@ -83,11 +83,28 @@ static inline void pmd_free(struct mm_st
 		 * done by generic mm code.
 		 */
 		mm_inc_nr_pmds(mm);
-		return;
+		return false;
 	}
-	free_pages((unsigned long)pmd, PMD_ORDER);
+	return true;
+}
+
+static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
+{
+	if (__pmd_free(mm, pmd))
+		free_pages((unsigned long)pmd, PMD_ORDER);
 }
 
+static inline void __tlb_remove_table(void *table)
+{
+	free_pages((unsigned long)table, PMD_ORDER);
+}
+
+#define __pmd_free_tlb(tlb, pmd, addr)		\
+do {						\
+	if (__pmd_free((tlb)->mm, (pmd)))	\
+		tlb_remove_table((tlb), (pmd));	\
+} while (0)
+
 #endif
 
 static inline void
arch/parisc/include/asm/tlb.h +5 -4
@@ -4,9 +4,10 @@
 
 #include <asm-generic/tlb.h>
 
-#if CONFIG_PGTABLE_LEVELS == 3
-#define __pmd_free_tlb(tlb, pmd, addr)	pmd_free((tlb)->mm, pmd)
-#endif
-#define __pte_free_tlb(tlb, pte, addr)	pte_free((tlb)->mm, pte)
+#define __pte_free_tlb(tlb,pte,addr)			\
+do {							\
+	pgtable_pte_page_dtor(pte);			\
+	tlb_remove_page((tlb), (pte));			\
+} while (0)
 
 #endif
author maintainer
#7 lore raw

PATCH v2 6/11 mips/tlb: Fix __p*_free_tlb()

In reply to Peter Zijlstra

Just like regular pages, page directories need to observe the following order:

1) unhook
2) TLB invalidate
3) free

to ensure it is safe against concurrent accesses.

MIPS has (thanks to Paul Burton for setting me straight):

- HAVE_FAST_GUP, which means we need to consider software walkers (many MIPS also have software TLB-fill, which is a similar software walker).

- non-page, page directories, which requires a custom table free.

- Mostly IPI-based TLB invalidate, but it wouldn't be MIPS if it didn't also have broadcast TLB invalidate (I6500, GINVT).

This means that in generic MIPS needs HAVE_RCU_TABLE_FREE.

---
 arch/mips/Kconfig               |    1 +
 arch/mips/include/asm/pgalloc.h |   13 ++++++-------
 arch/mips/mm/pgtable.c          |   34 ++++++++++++++++++++++++++++++++++
 3 files changed, 41 insertions(+), 7 deletions(-)
arch/mips/Kconfig +1 -0
@@ -84,6 +84,7 @@ config MIPS
 	select HAVE_VIRT_CPU_ACCOUNTING_GEN if 64BIT || !SMP
 	select IRQ_FORCED_THREADING
 	select ISA if EISA
+	select MMU_GATHER_RCU_TABLE_FREE
 	select MODULES_USE_ELF_REL if MODULES
 	select MODULES_USE_ELF_RELA if MODULES && 64BIT
 	select PERF_USE_VMALLOC
arch/mips/include/asm/pgalloc.h +6 -7
@@ -41,6 +41,9 @@ static inline void pud_populate(struct m
 }
 #endif
 
+extern void __tlb_remove_table(void *table);
+extern void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int idx);
+
 /*
  * Initialize a new pgd / pmd table with invalid pointers.
  */
@@ -52,11 +55,7 @@ static inline void pgd_free(struct mm_st
 	free_pages((unsigned long)pgd, PGD_ORDER);
 }
 
-#define __pte_free_tlb(tlb,pte,address)			\
-do {							\
-	pgtable_pte_page_dtor(pte);			\
-	tlb_remove_page((tlb), pte);			\
-} while (0)
+#define __pte_free_tlb(tlb,pte,address)	pgtable_free_tlb((tlb), (pte), 0)
 
 #ifndef __PAGETABLE_PMD_FOLDED
 
@@ -75,7 +74,7 @@ static inline void pmd_free(struct mm_st
 	free_pages((unsigned long)pmd, PMD_ORDER);
 }
 
-#define __pmd_free_tlb(tlb, x, addr)	pmd_free((tlb)->mm, x)
+#define __pmd_free_tlb(tlb, x, addr)	pgtable_free_tlb((tlb), (x), 1)
 
 #endif
 
@@ -101,7 +100,7 @@ static inline void p4d_populate(struct m
 	set_p4d(p4d, __p4d((unsigned long)pud));
 }
 
-#define __pud_free_tlb(tlb, x, addr)	pud_free((tlb)->mm, x)
+#define __pud_free_tlb(tlb, x, addr)	pgtable_free_tlb((tlb), (x), 2)
 
 #endif /* __PAGETABLE_PUD_FOLDED */
 
arch/mips/mm/pgtable.c +34 -0
@@ -7,6 +7,7 @@
 #include <linux/mm.h>
 #include <linux/string.h>
 #include <asm/pgalloc.h>
+#include <asm-generic/tlb.h>
 
 pgd_t *pgd_alloc(struct mm_struct *mm)
 {
@@ -23,3 +24,36 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
 	return ret;
 }
 EXPORT_SYMBOL_GPL(pgd_alloc);
+
+#define TLB_IDX_MASK	0x3
+
+void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int idx)
+{
+	unsigned long pgf = (unsigned long)table;
+	BUG_ON(idx > TLB_IDX_MASK);
+	BUG_ON(pgf & TLB_IDX_MASK);
+	pgf |= idx;
+	tlb_remove_table(tlb, (void *)pgf);
+}
+
+void __tlb_remove_table(void *table)
+{
+	void *dir = (void *)((unsigned long)table & ~TLB_IDX_MASK);
+	int idx = (unsigned long)table & TLB_IDX_MASK;
+
+	switch (idx) {
+#ifndef __PAGETABLE_PUD_FOLDED
+	case 2: /* PUD */
+		pud_free(NULL, dir);
+		break;
+#endif
+#ifndef __PAGETABLE_PMD_FOLDED
+	case 1: /* PMD */
+		pmd_free(NULL, dir);
+		break;
+#endif
+	case 0: /* PTE */
+		pte_free(NULL, dir);
+		break;
+	}
+}
author maintainer
#8 lore raw

PATCH v2 7/11 ia64/tlb: Fix __p*_free_tlb()

In reply to Peter Zijlstra

Just like regular pages, page directories need to observe the following order:

1) unhook
2) TLB invalidate
3) free

to ensure it is safe against concurrent accesses.

Since IA64 has page based P[UM]Ds, no software walkers and IPI based TLB invalidation, it can use the simple tlb_remove_page() based freeer.

---
 arch/ia64/include/asm/pgalloc.h |   13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)
arch/ia64/include/asm/pgalloc.h +10 -3
@@ -50,7 +50,9 @@ static inline void pud_free(struct mm_st
 {
 	free_page((unsigned long)pud);
 }
-#define __pud_free_tlb(tlb, pud, address)	pud_free((tlb)->mm, pud)
+
+#define __pud_free_tlb(tlb, pud, address)	\
+	tlb_remove_table((tlb), virt_to_page(pud))
 #endif /* CONFIG_PGTABLE_LEVELS == 4 */
 
 static inline void
@@ -69,7 +71,8 @@ static inline void pmd_free(struct mm_st
 	free_page((unsigned long)pmd);
 }
 
-#define __pmd_free_tlb(tlb, pmd, address)	pmd_free((tlb)->mm, pmd)
+#define __pmd_free_tlb(tlb, pmd, address)	\
+	tlb_remove_table((tlb), virt_to_page(pmd)
 
 static inline void
 pmd_populate(struct mm_struct *mm, pmd_t * pmd_entry, pgtable_t pte)
@@ -84,6 +87,10 @@ pmd_populate_kernel(struct mm_struct *mm
 	pmd_val(*pmd_entry) = __pa(pte);
 }
 
-#define __pte_free_tlb(tlb, pte, address)	pte_free((tlb)->mm, pte)
+#define __pte_free_tlb(tlb, pte, address)	\
+do {						\
+	pgtable_pte_page_dtor(pte);		\
+	tlb_remove_table((tlb), (pte));		\
+} while (0)
 
 #endif				/* _ASM_IA64_PGALLOC_H */
author maintainer
#9 lore raw

PATCH v2 8/11 alpha/tlb: Fix __p*_free_tlb()

In reply to Peter Zijlstra

Just like regular pages, page directories need to observe the following order:

1) unhook
2) TLB invalidate
3) free

to ensure it is safe against concurrent accesses.

Since Alpha has page based PMDs, no software walkers and IPI based TLB invalidation, it can use the simple tlb_remove_page() based freeer.

---
 arch/alpha/include/asm/tlb.h |   13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)
arch/alpha/include/asm/tlb.h +11 -2
@@ -4,7 +4,16 @@
 
 #include <asm-generic/tlb.h>
 
-#define __pte_free_tlb(tlb, pte, address)		pte_free((tlb)->mm, pte)
-#define __pmd_free_tlb(tlb, pmd, address)		pmd_free((tlb)->mm, pmd)
+extern void ___pte_free_tlb(struct mmu_gather *tlb, pte_t *pte);
+extern void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd);
+
+#define __pte_free_tlb(tlb, pte, address)	\
+do {						\
+	pgtable_pte_page_dtor(pte);		\
+	tlb_remove_table((tlb), (pte));		\
+} while (0)
+
+#define __pmd_free_tlb(tlb, pmd, address)	\
+	tlb_remove_table((tlb), virt_to_page(pmd))
  
 #endif
author maintainer
#10 lore raw

PATCH v2 9/11 nds32/tlb: Fix __p*_free_tlb()

In reply to Peter Zijlstra

Just like regular pages, page directories need to observe the following order:

1) unhook
2) TLB invalidate
3) free

to ensure it is safe against concurrent accesses.

Even though NDS32 is UP only, we still need to observe this order because mmu_gather is preemptible.

NDS32 does not in fact have PMDs.

---
 arch/nds32/include/asm/tlb.h |    7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)
arch/nds32/include/asm/tlb.h +6 -1
@@ -6,6 +6,11 @@
 
 #include <asm-generic/tlb.h>
 
-#define __pte_free_tlb(tlb, pte, addr)	pte_free((tlb)->mm, pte)
+#define __pte_free_tlb(tlb, pte, address)	\
+do {						\
+	pgtable_pte_page_dtor(pte);		\
+	tlb_remove_table((tlb), (pte));		\
+} while (0)
+
 
 #endif
author maintainer
#11 lore raw

PATCH v2 10/11 riscv/tlb: Fix __p*_free_tlb()

In reply to Peter Zijlstra

Just like regular pages, page directories need to observe the following order:

1) unhook
2) TLB invalidate
3) free

to ensure it is safe against concurrent accesses.

Since RISC-V has page based PMDs, no software walkers and IPI based TLB invalidation, it can use the simple tlb_remove_page() based freeer.

---
 arch/riscv/include/asm/pgalloc.h |   11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)
arch/riscv/include/asm/pgalloc.h +6 -5
@@ -73,14 +73,15 @@ static inline void pmd_free(struct mm_st
 	free_page((unsigned long)pmd);
 }
 
-#define __pmd_free_tlb(tlb, pmd, addr)  pmd_free((tlb)->mm, pmd)
+#define __pmd_free_tlb(tlb, pmd, addr)	\
+	tlb_remove_table((tlb), virt_to_page(pmd))
 
 #endif /* __PAGETABLE_PMD_FOLDED */
 
-#define __pte_free_tlb(tlb, pte, buf)   \
-do {                                    \
-	pgtable_pte_page_dtor(pte);     \
-	tlb_remove_page((tlb), pte);    \
+#define __pte_free_tlb(tlb, pte, buf)	\
+do {					\
+	pgtable_pte_page_dtor(pte);	\
+	tlb_remove_table((tlb), (pte));	\
 } while (0)
 #endif /* CONFIG_MMU */
 
author maintainer
#12 lore raw

PATCH v2 11/11 m68k/tlb: Fix __p*_free_tlb()

In reply to Peter Zijlstra

Just like regular pages, page directories need to observe the following order:

1) unhook
2) TLB invalidate
3) free

to ensure it is safe against concurrent accesses.

The Motorola MMU has funny PMD stuff, so use a custom table freeer.

---
 arch/m68k/Kconfig                        |    1 +
 arch/m68k/include/asm/mcf_pgalloc.h      |   14 ++++++--------
 arch/m68k/include/asm/motorola_pgalloc.h |   10 ++++++----
 arch/m68k/mm/motorola.c                  |   17 +++++++++++++++++
 4 files changed, 30 insertions(+), 12 deletions(-)
arch/m68k/Kconfig +1 -0
@@ -32,6 +32,7 @@ config M68K
 	select OLD_SIGSUSPEND3
 	select OLD_SIGACTION
 	select MMU_GATHER_NO_RANGE if MMU
+	select MMU_GATHER_TABLE_FREE if MMU_MOTOROLA
 
 config CPU_BIG_ENDIAN
 	def_bool y
arch/m68k/include/asm/mcf_pgalloc.h +6 -8
@@ -34,14 +34,12 @@ extern inline pmd_t *pmd_alloc_kernel(pg
 
 #define pmd_pgtable(pmd) pfn_to_virt(pmd_val(pmd) >> PAGE_SHIFT)
 
-static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pgtable,
-				  unsigned long address)
-{
-	struct page *page = virt_to_page(pgtable);
-
-	pgtable_pte_page_dtor(page);
-	__free_page(page);
-}
+#define __pte_free_tlb(tlb, pgtable, addr)		\
+do {							\
+	struct page *page = virt_to_page(pgtable);	\
+	pgtable_pte_page_dtor(page);			\
+	tlb_remove_page((tlb), page);			\
+} while (0)
 
 static inline pgtable_t pte_alloc_one(struct mm_struct *mm)
 {
arch/m68k/include/asm/motorola_pgalloc.h +6 -4
@@ -14,6 +14,8 @@ enum m68k_table_types {
 	TABLE_PTE = 1,
 };
 
+extern void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int type);
+extern void __tlb_remove_table(void *table);
 extern void init_pointer_table(void *table, int type);
 extern void *get_pointer_table(int type);
 extern int free_pointer_table(void *table, int type);
@@ -47,7 +49,7 @@ static inline void pte_free(struct mm_st
 static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pgtable,
 				  unsigned long address)
 {
-	free_pointer_table(pgtable, TABLE_PTE);
+	pgtable_free_tlb(tlb, pgtable, TABLE_PTE);
 }
 
 
@@ -61,10 +63,10 @@ static inline int pmd_free(struct mm_str
 	return free_pointer_table(pmd, TABLE_PMD);
 }
 
-static inline int __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,
-				 unsigned long address)
+static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,
+				  unsigned long address)
 {
-	return free_pointer_table(pmd, TABLE_PMD);
+	pgtable_free_tlb(tlb, pmd, TABLE_PMD);
 }
 
 
arch/m68k/mm/motorola.c +17 -0
@@ -215,6 +215,23 @@ int free_pointer_table(void *table, int
 	return 0;
 }
 
+void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int type)
+{
+	unsigned long ptr = (unsigned long)table;
+	BUG_ON(type > 1);
+	BUG_ON(ptr  & 1);
+	ptr |= type;
+	tlb_remove_table(tlb, (void *)ptr);
+}
+
+void __tlb_remove_table(void *table)
+{
+	int type = (unsigned long)table & 1;
+	table = (void *)((unsigned long)table & ~1);
+
+	free_pointer_table(table, type);
+}
+
 /* size of memory already mapped in head.S */
 extern __initdata unsigned long m68k_init_mapped_size;
 
author maintainer
#13 lore raw
In reply to Peter Zijlstra
Quoted 5 lines · Peter Zijlstra

Hi All,

While fixing a silly bug on SH (patch #1), I realized that even with the trivial patch to restore prior behaviour, page directory freeing was still broken.

*sigh*, I got patches 1 and 2 in the 'wrong' order.