x86/mm: Clean up and use temportary_mm more
Subsystems (4)
Peter Zijlstra
maintainercover x86/mm: Clean up and use temportary_mm more
Hi,
These are most of the patches I had in x86/lazy and are the preparatory patches for x86 to eventually run witn MMU_LAZY_TLB_REFCOUNT=n.
The rebase and submission prompted by Rik's recent context switch optimization which touched unuse_temporary_mm() in a way incompatible with these patches.
I'll see if I can find time this merge window to also finish that last patch so that x86 can indeed switch to MMU_LAZY_TLB_REFCOUNT=n, that too should help with context switch times.
Peter Zijlstra
maintainerPATCH 1/7 x86/mm: Add mm argument to unuse_temporary_mm()
In commit 209954cbc7d0 ("x86/mm/tlb: Update mm_cpumask lazily") unuse_temporary_mm() grew the assumption that it gets used on poking_nn exclusively. While this is currently true, lets not hard code this assumption.
--- arch/x86/kernel/alternative.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
arch/x86/kernel/alternative.c +3 -3
@@ -1828,14 +1828,14 @@ static inline temp_mm_state_t use_tempor__ro_after_init struct mm_struct *poking_mm;__ro_after_init unsigned long poking_addr; -static inline void unuse_temporary_mm (temp_mm_state_t prev_state) +static inline void unuse_temporary_mm (struct mm_struct *mm,temp_mm_state_t prev_state) {lockdep_assert_irqs_disabled ();switch_mm_irqs_off (NULL , prev_state.mm, current);/* Clear the cpumask, to indicate no TLB flushing is needed anywhere */ -cpumask_clear_cpu (raw_smp_processor_id (),mm_cpumask (poking_mm)); +cpumask_clear_cpu (raw_smp_processor_id (),mm_cpumask (mm));/* * Restore the breakpoints if they were disabled before the temporary mm
@@ -1942,7 +1942,7 @@ static void *__text_poke(text_poke_f fun* instruction that already allows the core to see the updated version. * Xen-PV is assumed to serialize execution in a similar manner. */ -unuse_temporary_mm (prev); +unuse_temporary_mm (poking_mm, prev);/* * Flushing the TLB might involve IPIs, which would require enabled
Peter Zijlstra
maintainerPATCH 2/7 x86/events, x86/insn-eval: Remove incorrect active_mm references
In reply to Peter ZijlstraFrom: Andy Lutomirski <[email protected]>
When decoding an instruction or handling a perf event that references an LDT segment, if we don't have a valid user context, trying to access the LDT by any means other than SLDT is racy. Certainly, using current->active_mm is wrong, as active_mm can point to a real user mm when CR3 and LDTR no longer reference that mm.
Clean up the code. If nmi_uaccess_okay() says we don't have a valid context, just fail. Otherwise use current->mm.
--- arch/x86/events/core.c | 9 ++++++++- arch/x86/lib/insn-eval.c | 13 ++++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-)
arch/x86/events/core.c +8 -1
@@ -2798,8 +2798,15 @@ static unsigned long get_segment_base(un#ifdef CONFIG_MODIFY_LDT_SYSCALL struct ldt_struct *ldt; +/* +* If we're not in a valid context with a real (not just lazy) +* user mm, then don't even try. +*/ +if (!nmi_uaccess_okay ()) +return 0 ; +/* IRQs are off, so this synchronizes with smp_store_release */ - ldt =READ_ONCE (current->active_mm->context.ldt); + ldt =smp_load_acquire (¤t->mm->context.ldt);if (!ldt || idx >= ldt->nr_entries)return 0 ;
arch/x86/lib/insn-eval.c +10 -3
@@ -631,14 +631,21 @@ static bool get_desc(struct desc_struct/* Bits [15:3] contain the index of the desired entry. */ sel >>=3 ; -mutex_lock (¤t->active_mm->context.lock); - ldt = current->active_mm->context.ldt; +/* +* If we're not in a valid context with a real (not just lazy) +* user mm, then don't even try. +*/ +if (!nmi_uaccess_okay ()) +return false; + +mutex_lock (¤t->mm->context.lock); + ldt = current->mm->context.ldt;if (ldt && sel < ldt->nr_entries) { *out = ldt->entries[sel]; success = true; } -mutex_unlock (¤t->active_mm->context.lock); +mutex_unlock (¤t->mm->context.lock);return success; }
Peter Zijlstra
maintainerPATCH 3/7 x86/mm: Make use/unuse_temporary_mm() non-static
In reply to Peter ZijlstraFrom: Andy Lutomirski <[email protected]>
This prepares them for use outside of the alternative machinery. The code is unchanged.
--- arch/x86/include/asm/mmu_context.h | 7 +++ arch/x86/kernel/alternative.c | 68 ------------------------------------- arch/x86/mm/tlb.c | 63 ++++++++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 68 deletions(-)
arch/x86/include/asm/mmu_context.h +7 -0
@@ -263,4 +263,11 @@ unsigned long __get_current_cr3_fast(voi#include <asm-generic/mmu_context.h> +typedef struct { +struct mm_struct *mm; +}temp_mm_state_t ; + +extern temp_mm_state_t use_temporary_mm (struct mm_struct *mm); +extern void unuse_temporary_mm (struct mm_struct *mm,temp_mm_state_t prev_state); +#endif /* _ASM_X86_MMU_CONTEXT_H */
arch/x86/kernel/alternative.c +0 -68
77 lines not on this page · raw message
arch/x86/mm/tlb.c +63 -0
69 lines not on this page · raw message
Peter Zijlstra
maintainerPATCH 4/7 x86/mm: Remove mm argument from unuse_temporary_mm() again
In reply to Peter ZijlstraNow that unuse_temporary_mm() lives in tlb.c it can access loaded_mm.
--- arch/x86/include/asm/mmu_context.h | 2 +- arch/x86/kernel/alternative.c | 2 +- arch/x86/mm/tlb.c | 8 +++++--- 3 files changed, 7 insertions(+), 5 deletions(-)
arch/x86/include/asm/mmu_context.h +1 -1
@@ -268,6 +268,6 @@ typedef struct { } temp_mm_state_t;extern temp_mm_state_t use_temporary_mm (struct mm_struct *mm); -extern void unuse_temporary_mm (struct mm_struct *mm,temp_mm_state_t prev_state); +extern void unuse_temporary_mm (temp_mm_state_t prev_state);#endif /* _ASM_X86_MMU_CONTEXT_H */
arch/x86/kernel/alternative.c +1 -1
@@ -1874,7 +1874,7 @@ static void *__text_poke(text_poke_f fun* instruction that already allows the core to see the updated version. * Xen-PV is assumed to serialize execution in a similar manner. */ -unuse_temporary_mm (poking_mm, prev); +unuse_temporary_mm (prev);/* * Flushing the TLB might involve IPIs, which would require enabled
arch/x86/mm/tlb.c +5 -3
@@ -720,13 +720,15 @@ temp_mm_state_t use_temporary_mm(structreturn temp_state; } -void unuse_temporary_mm (struct mm_struct *mm,temp_mm_state_t prev_state) +void unuse_temporary_mm (temp_mm_state_t prev_state) {lockdep_assert_irqs_disabled (); -switch_mm_irqs_off (NULL , prev_state.mm, current);/* Clear the cpumask, to indicate no TLB flushing is needed anywhere */ -cpumask_clear_cpu (raw_smp_processor_id (),mm_cpumask (mm)); +cpumask_clear_cpu (smp_processor_id (), +mm_cpumask (this_cpu_read (cpu_tlbstate.loaded_mm))); + +switch_mm_irqs_off (NULL , prev_state.mm, current);/* * Restore the breakpoints if they were disabled before the temporary mm
Peter Zijlstra
maintainerPATCH 5/7 x86/mm: Allow temporary mms when IRQs are on
In reply to Peter ZijlstraFrom: Andy Lutomirski <[email protected]>
EFI runtime services should use temporary mms, but EFI runtime services want IRQs on. Preemption must still be disabled in a temporary mm context.
At some point, the entirely temporary mm mechanism should be moved out of arch code.
--- arch/x86/mm/tlb.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-)
arch/x86/mm/tlb.c +12 -7
@@ -679,18 +679,23 @@ void enter_lazy_tlb(struct mm_struct *mm* that override the kernel memory protections (e.g., W^X), without exposing the * temporary page-table mappings that are required for these write operations to * other CPUs. Using a temporary mm also allows to avoid TLB shootdowns when the -* mapping is torn down. +* mapping is torn down. Temporary mms can also be used for EFI runtime service +* calls or similar functionality. * -* Context: The temporary mm needs to be used exclusively by a single core. To -* harden security IRQs must be disabled while the temporary mm is -* loaded, thereby preventing interrupt handler bugs from overriding -* the kernel memory protection. +* It is illegal to schedule while using a temporary mm -- the context switch +* code is unaware of the temporary mm and does not know how to context switch. +* Use a real (non-temporary) mm in a kernel thread if you need to sleep. +* +* Note: For sensitive memory writes, the temporary mm needs to be used +* exclusively by a single core, and IRQs should be disabled while the +* temporary mm is loaded, thereby preventing interrupt handler bugs from +* overriding the kernel memory protection. */ temp_mm_state_t use_temporary_mm (struct mm_struct *mm) {temp_mm_state_t temp_state; -lockdep_assert_irqs_disabled (); +lockdep_assert_preemption_disabled ();/* * Make sure not to be in TLB lazy mode, as otherwise we'll end up
@@ -722,7 +727,7 @@ temp_mm_state_t use_temporary_mm(structvoid unuse_temporary_mm (temp_mm_state_t prev_state) { -lockdep_assert_irqs_disabled (); +lockdep_assert_preemption_disabled ();/* Clear the cpumask, to indicate no TLB flushing is needed anywhere */ cpumask_clear_cpu(smp_processor_id(),
Peter Zijlstra
maintainerPATCH 6/7 x86/efi: Make efi_enter/leave_mm use the temporary_mm machinery
In reply to Peter ZijlstraFrom: Andy Lutomirski <[email protected]>
This should be considerably more robust. It's also necessary for optimized for_each_possible_lazymm_cpu() on x86 -- without this patch, EFI calls in lazy context would remove the lazy mm from mm_cpumask().
--- arch/x86/platform/efi/efi_64.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-)
arch/x86/platform/efi/efi_64.c +3 -6
@@ -54,7 +54,7 @@* 0xffff_ffff_0000_0000 and limit EFI VA mapping space to 64G. */ static u64 efi_va =EFI_VA_START ; -static struct mm_struct *efi_prev_mm; +static temp_mm_state_t efi_temp_mm_state;/* * We need our own copy of the higher levels of the page tables
@@ -476,15 +476,12 @@ void __init efi_dump_pagetable(void)*/ static void efi_enter_mm (void ) { - efi_prev_mm = current->active_mm; - current->active_mm = &efi_mm; -switch_mm (efi_prev_mm, &efi_mm,NULL ); + efi_temp_mm_state =use_temporary_mm (&efi_mm); }static void efi_leave_mm (void ) { - current->active_mm = efi_prev_mm; -switch_mm (&efi_mm, efi_prev_mm,NULL ); +unuse_temporary_mm (efi_temp_mm_state); }void arch_efi_call_virt_setup (void )
Peter Zijlstra
maintainerPATCH 7/7 x86/mm: Opt in to IRQs-off activate_mm()
In reply to Peter ZijlstraFrom: Andy Lutomirski <[email protected]>
We gain nothing by having the core code enable IRQs right before calling activate_mm() only for us to turn them right back off again in switch_mm().
This will save a few cycles, so execve() should be blazingly fast with this patch applied!
--- arch/x86/Kconfig | 1 + arch/x86/include/asm/mmu_context.h | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-)
arch/x86/Kconfig +1 -0
@@ -133,6 +133,7 @@ config X86 select ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP if X86_64 select ARCH_WANTS_THP_SWAP if X86_64 select ARCH_HAS_PARANOID_L1D_FLUSH + select ARCH_WANT_IRQS_OFF_ACTIVATE_MM select BUILDTIME_TABLE_SORT select CLKEVT_I8253 select CLOCKSOURCE_VALIDATE_LAST_CYCLE
arch/x86/include/asm/mmu_context.h +1 -1
@@ -175,7 +175,7 @@ extern void switch_mm_irqs_off(struct mm#define activate_mm (prev, next) \ do { \ paravirt_enter_mmap(next); \ - switch_mm((prev), (next), NULL); \ + switch_mm_irqs_off((prev), (next), NULL); \ } while (0);#ifdef CONFIG_X86_32