Skip to contentSkip to conversation

x86/mm: Clean up and use temportary_mm more

PATCH New Peter Zijlstra 1 participant 8 messages Nov 19, 2024

Subsystems (4)
lore
Share & tools
lore

Peter Zijlstra

author maintainer
#0 lore raw

cover x86/mm: Clean up and use temportary_mm more

Hi,

These are most of the patches I had in x86/lazy and are the preparatory patches for x86 to eventually run witn MMU_LAZY_TLB_REFCOUNT=n.

The rebase and submission prompted by Rik's recent context switch optimization which touched unuse_temporary_mm() in a way incompatible with these patches.

I'll see if I can find time this merge window to also finish that last patch so that x86 can indeed switch to MMU_LAZY_TLB_REFCOUNT=n, that too should help with context switch times.

author maintainer
#1 lore raw

PATCH 1/7 x86/mm: Add mm argument to unuse_temporary_mm()

In commit 209954cbc7d0 ("x86/mm/tlb: Update mm_cpumask lazily") unuse_temporary_mm() grew the assumption that it gets used on poking_nn exclusively. While this is currently true, lets not hard code this assumption.

---
 arch/x86/kernel/alternative.c |    6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
arch/x86/kernel/alternative.c +3 -3
@@ -1828,14 +1828,14 @@ static inline temp_mm_state_t use_tempor
 __ro_after_init struct mm_struct *poking_mm;
 __ro_after_init unsigned long poking_addr;
 
-static inline void unuse_temporary_mm(temp_mm_state_t prev_state)
+static inline void unuse_temporary_mm(struct mm_struct *mm, temp_mm_state_t prev_state)
 {
 	lockdep_assert_irqs_disabled();
 
 	switch_mm_irqs_off(NULL, prev_state.mm, current);
 
 	/* Clear the cpumask, to indicate no TLB flushing is needed anywhere */
-	cpumask_clear_cpu(raw_smp_processor_id(), mm_cpumask(poking_mm));
+	cpumask_clear_cpu(raw_smp_processor_id(), mm_cpumask(mm));
 
 	/*
 	 * Restore the breakpoints if they were disabled before the temporary mm
@@ -1942,7 +1942,7 @@ static void *__text_poke(text_poke_f fun
 	 * instruction that already allows the core to see the updated version.
 	 * Xen-PV is assumed to serialize execution in a similar manner.
 	 */
-	unuse_temporary_mm(prev);
+	unuse_temporary_mm(poking_mm, prev);
 
 	/*
 	 * Flushing the TLB might involve IPIs, which would require enabled
author maintainer
#2 lore raw

PATCH 2/7 x86/events, x86/insn-eval: Remove incorrect active_mm references

In reply to Peter Zijlstra

From: Andy Lutomirski <[email protected]>

When decoding an instruction or handling a perf event that references an LDT segment, if we don't have a valid user context, trying to access the LDT by any means other than SLDT is racy. Certainly, using current->active_mm is wrong, as active_mm can point to a real user mm when CR3 and LDTR no longer reference that mm.

Clean up the code. If nmi_uaccess_okay() says we don't have a valid context, just fail. Otherwise use current->mm.

---
 arch/x86/events/core.c   |    9 ++++++++-
 arch/x86/lib/insn-eval.c |   13 ++++++++++---
 2 files changed, 18 insertions(+), 4 deletions(-)
arch/x86/events/core.c +8 -1
@@ -2798,8 +2798,15 @@ static unsigned long get_segment_base(un
 #ifdef CONFIG_MODIFY_LDT_SYSCALL
 		struct ldt_struct *ldt;
 
+		/*
+		 * If we're not in a valid context with a real (not just lazy)
+		 * user mm, then don't even try.
+		 */
+		if (!nmi_uaccess_okay())
+			return 0;
+
 		/* IRQs are off, so this synchronizes with smp_store_release */
-		ldt = READ_ONCE(current->active_mm->context.ldt);
+		ldt = smp_load_acquire(&current->mm->context.ldt);
 		if (!ldt || idx >= ldt->nr_entries)
 			return 0;
 
arch/x86/lib/insn-eval.c +10 -3
@@ -631,14 +631,21 @@ static bool get_desc(struct desc_struct
 		/* Bits [15:3] contain the index of the desired entry. */
 		sel >>= 3;
 
-		mutex_lock(&current->active_mm->context.lock);
-		ldt = current->active_mm->context.ldt;
+		/*
+		 * If we're not in a valid context with a real (not just lazy)
+		 * user mm, then don't even try.
+		 */
+		if (!nmi_uaccess_okay())
+			return false;
+
+		mutex_lock(&current->mm->context.lock);
+		ldt = current->mm->context.ldt;
 		if (ldt && sel < ldt->nr_entries) {
 			*out = ldt->entries[sel];
 			success = true;
 		}
 
-		mutex_unlock(&current->active_mm->context.lock);
+		mutex_unlock(&current->mm->context.lock);
 
 		return success;
 	}
author maintainer
#3 lore raw

PATCH 3/7 x86/mm: Make use/unuse_temporary_mm() non-static

In reply to Peter Zijlstra

From: Andy Lutomirski <[email protected]>

This prepares them for use outside of the alternative machinery. The code is unchanged.

---
 arch/x86/include/asm/mmu_context.h |    7 +++
 arch/x86/kernel/alternative.c      |   68 -------------------------------------
 arch/x86/mm/tlb.c                  |   63 ++++++++++++++++++++++++++++++++++
 3 files changed, 70 insertions(+), 68 deletions(-)
arch/x86/include/asm/mmu_context.h +7 -0
@@ -263,4 +263,11 @@ unsigned long __get_current_cr3_fast(voi
 
 #include <asm-generic/mmu_context.h>
 
+typedef struct {
+	struct mm_struct *mm;
+} temp_mm_state_t;
+
+extern temp_mm_state_t use_temporary_mm(struct mm_struct *mm);
+extern void unuse_temporary_mm(struct mm_struct *mm, temp_mm_state_t prev_state);
+
 #endif /* _ASM_X86_MMU_CONTEXT_H */
arch/x86/kernel/alternative.c +0 -68

77 lines not on this page · raw message

arch/x86/mm/tlb.c +63 -0

69 lines not on this page · raw message

author maintainer
#4 lore raw

PATCH 4/7 x86/mm: Remove mm argument from unuse_temporary_mm() again

In reply to Peter Zijlstra

Now that unuse_temporary_mm() lives in tlb.c it can access loaded_mm.

---
 arch/x86/include/asm/mmu_context.h |    2 +-
 arch/x86/kernel/alternative.c      |    2 +-
 arch/x86/mm/tlb.c                  |    8 +++++---
 3 files changed, 7 insertions(+), 5 deletions(-)
arch/x86/include/asm/mmu_context.h +1 -1
@@ -268,6 +268,6 @@ typedef struct {
 } temp_mm_state_t;
 
 extern temp_mm_state_t use_temporary_mm(struct mm_struct *mm);
-extern void unuse_temporary_mm(struct mm_struct *mm, temp_mm_state_t prev_state);
+extern void unuse_temporary_mm(temp_mm_state_t prev_state);
 
 #endif /* _ASM_X86_MMU_CONTEXT_H */
arch/x86/kernel/alternative.c +1 -1
@@ -1874,7 +1874,7 @@ static void *__text_poke(text_poke_f fun
 	 * instruction that already allows the core to see the updated version.
 	 * Xen-PV is assumed to serialize execution in a similar manner.
 	 */
-	unuse_temporary_mm(poking_mm, prev);
+	unuse_temporary_mm(prev);
 
 	/*
 	 * Flushing the TLB might involve IPIs, which would require enabled
arch/x86/mm/tlb.c +5 -3
@@ -720,13 +720,15 @@ temp_mm_state_t use_temporary_mm(struct
 	return temp_state;
 }
 
-void unuse_temporary_mm(struct mm_struct *mm, temp_mm_state_t prev_state)
+void unuse_temporary_mm(temp_mm_state_t prev_state)
 {
 	lockdep_assert_irqs_disabled();
-	switch_mm_irqs_off(NULL, prev_state.mm, current);
 
 	/* Clear the cpumask, to indicate no TLB flushing is needed anywhere */
-	cpumask_clear_cpu(raw_smp_processor_id(), mm_cpumask(mm));
+	cpumask_clear_cpu(smp_processor_id(),
+			  mm_cpumask(this_cpu_read(cpu_tlbstate.loaded_mm)));
+
+	switch_mm_irqs_off(NULL, prev_state.mm, current);
 
 	/*
 	 * Restore the breakpoints if they were disabled before the temporary mm
author maintainer
#5 lore raw

PATCH 5/7 x86/mm: Allow temporary mms when IRQs are on

In reply to Peter Zijlstra

From: Andy Lutomirski <[email protected]>

EFI runtime services should use temporary mms, but EFI runtime services want IRQs on. Preemption must still be disabled in a temporary mm context.

At some point, the entirely temporary mm mechanism should be moved out of arch code.

---
 arch/x86/mm/tlb.c |   19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)
arch/x86/mm/tlb.c +12 -7
@@ -679,18 +679,23 @@ void enter_lazy_tlb(struct mm_struct *mm
  * that override the kernel memory protections (e.g., W^X), without exposing the
  * temporary page-table mappings that are required for these write operations to
  * other CPUs. Using a temporary mm also allows to avoid TLB shootdowns when the
- * mapping is torn down.
+ * mapping is torn down.  Temporary mms can also be used for EFI runtime service
+ * calls or similar functionality.
  *
- * Context: The temporary mm needs to be used exclusively by a single core. To
- *          harden security IRQs must be disabled while the temporary mm is
- *          loaded, thereby preventing interrupt handler bugs from overriding
- *          the kernel memory protection.
+ * It is illegal to schedule while using a temporary mm -- the context switch
+ * code is unaware of the temporary mm and does not know how to context switch.
+ * Use a real (non-temporary) mm in a kernel thread if you need to sleep.
+ *
+ * Note: For sensitive memory writes, the temporary mm needs to be used
+ *       exclusively by a single core, and IRQs should be disabled while the
+ *       temporary mm is loaded, thereby preventing interrupt handler bugs from
+ *       overriding the kernel memory protection.
  */
 temp_mm_state_t use_temporary_mm(struct mm_struct *mm)
 {
 	temp_mm_state_t temp_state;
 
-	lockdep_assert_irqs_disabled();
+	lockdep_assert_preemption_disabled();
 
 	/*
 	 * Make sure not to be in TLB lazy mode, as otherwise we'll end up
@@ -722,7 +727,7 @@ temp_mm_state_t use_temporary_mm(struct
 
 void unuse_temporary_mm(temp_mm_state_t prev_state)
 {
-	lockdep_assert_irqs_disabled();
+	lockdep_assert_preemption_disabled();
 
 	/* Clear the cpumask, to indicate no TLB flushing is needed anywhere */
 	cpumask_clear_cpu(smp_processor_id(),
author maintainer
#6 lore raw

PATCH 6/7 x86/efi: Make efi_enter/leave_mm use the temporary_mm machinery

In reply to Peter Zijlstra

From: Andy Lutomirski <[email protected]>

This should be considerably more robust. It's also necessary for optimized for_each_possible_lazymm_cpu() on x86 -- without this patch, EFI calls in lazy context would remove the lazy mm from mm_cpumask().

---
 arch/x86/platform/efi/efi_64.c |    9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)
arch/x86/platform/efi/efi_64.c +3 -6
@@ -54,7 +54,7 @@
  * 0xffff_ffff_0000_0000 and limit EFI VA mapping space to 64G.
  */
 static u64 efi_va = EFI_VA_START;
-static struct mm_struct *efi_prev_mm;
+static temp_mm_state_t efi_temp_mm_state;
 
 /*
  * We need our own copy of the higher levels of the page tables
@@ -476,15 +476,12 @@ void __init efi_dump_pagetable(void)
  */
 static void efi_enter_mm(void)
 {
-	efi_prev_mm = current->active_mm;
-	current->active_mm = &efi_mm;
-	switch_mm(efi_prev_mm, &efi_mm, NULL);
+	efi_temp_mm_state = use_temporary_mm(&efi_mm);
 }
 
 static void efi_leave_mm(void)
 {
-	current->active_mm = efi_prev_mm;
-	switch_mm(&efi_mm, efi_prev_mm, NULL);
+	unuse_temporary_mm(efi_temp_mm_state);
 }
 
 void arch_efi_call_virt_setup(void)
author maintainer
#7 lore raw

PATCH 7/7 x86/mm: Opt in to IRQs-off activate_mm()

In reply to Peter Zijlstra

From: Andy Lutomirski <[email protected]>

We gain nothing by having the core code enable IRQs right before calling activate_mm() only for us to turn them right back off again in switch_mm().

This will save a few cycles, so execve() should be blazingly fast with this patch applied!

---
 arch/x86/Kconfig                   |    1 +
 arch/x86/include/asm/mmu_context.h |    2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)
arch/x86/Kconfig +1 -0
Index: linux-2.6/arch/x86/Kconfig
===================================================================
@@ -133,6 +133,7 @@ config X86
 	select ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP	if X86_64
 	select ARCH_WANTS_THP_SWAP		if X86_64
 	select ARCH_HAS_PARANOID_L1D_FLUSH
+	select ARCH_WANT_IRQS_OFF_ACTIVATE_MM
 	select BUILDTIME_TABLE_SORT
 	select CLKEVT_I8253
 	select CLOCKSOURCE_VALIDATE_LAST_CYCLE
arch/x86/include/asm/mmu_context.h +1 -1
Index: linux-2.6/arch/x86/include/asm/mmu_context.h
===================================================================
@@ -175,7 +175,7 @@ extern void switch_mm_irqs_off(struct mm
 #define activate_mm(prev, next)			\
 do {						\
 	paravirt_enter_mmap(next);		\
-	switch_mm((prev), (next), NULL);	\
+	switch_mm_irqs_off((prev), (next), NULL);	\
 } while (0);
 
 #ifdef CONFIG_X86_32