x86/pti: Get rid of entry trampolines and add some docs
Subsystems (4)
Andy Lutomirski
maintainerv2 cover x86/pti: Get rid of entry trampolines and add some docs
This gets rid of entry trampolines. It's more or less the same as the RFC version, except that I rebased it to v4.19-rc1 due to massive conflicts with some perf changes. I have *not* reverted all of the perf support for entry trampolines -- I leave that to the perf crew, if needed.
Changes from v1: Get rid of the rsp_scratch macro
Andy Lutomirski (3):
x86/entry/64: Document idtentry
x86/entry/64: Use the TSS sp2 slot for SYSCALL/SYSRET scratch space x86/pti/64: Remove the SYSCALL64 entry trampoline
arch/x86/entry/entry_64.S | 116 ++++++++++---------------- arch/x86/include/asm/cpu_entry_area.h | 2 - arch/x86/include/asm/processor.h | 6 ++ arch/x86/include/asm/sections.h | 1 - arch/x86/kernel/asm-offsets.c | 5 +- arch/x86/kernel/cpu/common.c | 11 +-- arch/x86/kernel/kprobes/core.c | 10 +-- arch/x86/kernel/process_64.c | 2 - arch/x86/kernel/traps.c | 4 + arch/x86/kernel/vmlinux.lds.S | 10 --- arch/x86/mm/cpu_entry_area.c | 36 -------- arch/x86/mm/pti.c | 33 +++++++- arch/x86/xen/xen-asm_64.S | 8 +- 13 files changed, 95 insertions(+), 149 deletions(-)
Signature · 1 line
2.17.1
Andy Lutomirski
maintainerPATCH v2 1/3 x86/entry/64: Document idtentry
The idtentry macro is complicated and magical. Document what it does to help future readers and to allow future patches to adjust the code and docs at the same time.
--- arch/x86/entry/entry_64.S | 35 +++++++++++++++++++++++++++++++++++ arch/x86/kernel/traps.c | 4 ++++ 2 files changed, 39 insertions(+)
arch/x86/entry/entry_64.S +35 -0
@@ -900,6 +900,41 @@ apicinterrupt IRQ_WORK_VECTOR irq_work_interrupt smp_irq_work_interrupt */ #define CPU_TSS_IST(x) PER_CPU_VAR(cpu_tss_rw) + (TSS_ist + ((x) - 1) * 8) +/** + * idtentry - Generate an IDT entry stub + * @sym: Name of the generated entry point + * @do_sym: C function to be called + * @has_error_code: True if this IDT vector has an error code on the stack + * @paranoid: non-zero means that this vector may be invoked from kernel + * mode with user GSBASE and/or user CR3. 2 is special -- see below. + * @shift_ist: Set to an IST index if entries from kernel mode should + * decrement the IST stack so that nested entries get a fresh + * stack. (This is for #DB, which has a nasty habit of + * recursing.) + * + * idtentry generates an IDT stub that sets up a usable kernel context, + * creates struct pt_regs, and calls @do_sym. The stub has the following + * special behaviors: + * + * On an entry from user mode, the stub switches from the trampoline or + * IST stack to the normal thread stack. On an exit to user mode, the + * normal exit-to-usermode path is invoked. + * + * On an exit to kernel mode, if paranoid == 0, we check for preemption, + * whereas we omit the preemption check if @paranoid != 0. This is purely + * because the implementation is simpler this way. The kernel only needs + * to check for asynchronous kernel preemption when IRQ handlers return. + * + * If @paranoid == 0, then the stub will handle IRET faults by pretending + * that the fault came from user mode. It will handle gs_change faults by + * pretending that the fault happened with kernel GSBASE. Since this handling + * is omitted for @paranoid != 0, the #GP, #SS, and #NP stubs must have + * @paranoid == 0. This special handling will do the wrong thing for + * espfix-induced #DF on IRET, so #DF must not use @paranoid == 0. + * + * @paranoid == 2 is special: the stub will never switch stacks. This is for + * #DF: if the thread stack is somehow unusable, we'll still get a useful OOPS. + */ .macro idtentry sym do_sym has_error_code:req paranoid=0 shift_ist=-1 ENTRY(\sym) UNWIND_HINT_IRET_REGS offset=\has_error_code*8
arch/x86/kernel/traps.c +4 -0
@@ -383,6 +383,10 @@ dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code)* we won't enable interupts or schedule before we invoke * general_protection, so nothing will clobber the stack * frame we just set up. +* +* We will enter general_protection with kernel GSBASE, +* which is what the stub expects, given that the faulting +* RIP will be the IRET instruction. */ regs->ip = (unsigned long )general_protection; regs->sp = (unsigned long )&gpregs->orig_ax;
Signature · 1 line
2.17.1
Andy Lutomirski
maintainerPATCH v2 2/3 x86/entry/64: Use the TSS sp2 slot for SYSCALL/SYSRET scratch space
In reply to Andy LutomirskiIn the non-trampoline SYSCALL64 path, we use a percpu variable to temporarily store the user RSP value. Instead of a separate variable, use the otherwise unused sp2 slot in the TSS. This will improve cache locality, as the sp1 slot is already used in the same code to find the kernel stack. It will also simplify a future change to make the non-trampoline path work in PTI mode.
--- arch/x86/entry/entry_64.S | 16 +++++++++------- arch/x86/include/asm/processor.h | 6 ++++++ arch/x86/kernel/asm-offsets.c | 3 ++- arch/x86/kernel/process_64.c | 2 -- arch/x86/xen/xen-asm_64.S | 8 +++++--- 5 files changed, 22 insertions(+), 13 deletions(-)
arch/x86/entry/entry_64.S +9 -7
@@ -215,18 +215,20 @@ ENTRY(entry_SYSCALL_64) /* * This path is only taken when PAGE_TABLE_ISOLATION is disabled so it * is not required to switch CR3. + * + * tss.sp2 is scratch space. */ - movq %rsp, PER_CPU_VAR(rsp_scratch) + movq %rsp, PER_CPU_VAR(cpu_tss_rw + TSS_sp2) movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp /* Construct struct pt_regs on stack */ - pushq $__USER_DS /* pt_regs->ss */ - pushq PER_CPU_VAR(rsp_scratch) /* pt_regs->sp */ - pushq %r11 /* pt_regs->flags */ - pushq $__USER_CS /* pt_regs->cs */ - pushq %rcx /* pt_regs->ip */ + pushq $__USER_DS /* pt_regs->ss */ + pushq PER_CPU_VAR(cpu_tss_rw + TSS_sp2) /* pt_regs->sp */ + pushq %r11 /* pt_regs->flags */ + pushq $__USER_CS /* pt_regs->cs */ + pushq %rcx /* pt_regs->ip */ GLOBAL(entry_SYSCALL_64_after_hwframe) - pushq %rax /* pt_regs->orig_ax */ + pushq %rax /* pt_regs->orig_ax */ PUSH_AND_CLEAR_REGS rax=$-ENOSYS
arch/x86/include/asm/processor.h +6 -0
@@ -313,7 +313,13 @@ struct x86_hw_tss {*/ u64 sp1; +/* +* Since Linux does not use ring 2, the 'sp2' slot is unused by +* hardware. entry_SYSCALL_64 uses it as scratch space to stash +* the user RSP value. +*/ u64 sp2; +u64 reserved2;u64 ist[7 ];u32 reserved3;
arch/x86/kernel/asm-offsets.c +2 -1
@@ -105,7 +105,8 @@ void common(void) {DEFINE (SIZEOF_entry_stack,sizeof (struct entry_stack ));DEFINE (MASK_entry_stack, (~(sizeof (struct entry_stack ) -1 ))); -/* Offset for sp0 and sp1 into the tss_struct */ +/* Offset for fields in tss_struct */ OFFSET (TSS_sp0, tss_struct, x86_tss.sp0);OFFSET (TSS_sp1, tss_struct, x86_tss.sp1); +OFFSET (TSS_sp2, tss_struct, x86_tss.sp2); }
arch/x86/kernel/process_64.c +0 -2
@@ -59,8 +59,6 @@#include <asm/unistd_32_ia32.h> #endif -__visible DEFINE_PER_CPU (unsigned long ,rsp_scratch ); -/* Prints also some state that isn't saved in the pt_regs */ void __show_regs (struct pt_regs *regs,int all) {
arch/x86/xen/xen-asm_64.S +5 -3
@@ -91,13 +91,15 @@ ENTRY(xen_iret) ENTRY(xen_sysret64) /* * We're already on the usermode stack at this point, but - * still with the kernel gs, so we can easily switch back + * still with the kernel gs, so we can easily switch back. + * + * tss.sp2 is scratch space. */ - movq %rsp, PER_CPU_VAR(rsp_scratch) + movq %rsp, PER_CPU_VAR(cpu_tss_rw + TSS_sp2) movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp pushq $__USER_DS - pushq PER_CPU_VAR(rsp_scratch) + pushq PER_CPU_VAR(cpu_tss_rw + TSS_sp2) pushq %r11 pushq $__USER_CS pushq %rcx
Signature · 1 line
2.17.1
Andy Lutomirski
maintainerPATCH v2 3/3 x86/pti/64: Remove the SYSCALL64 entry trampoline
In reply to Andy LutomirskiThe SYSCALL64 trampoline has a couple of nice properties:
- The usual sequence of SWAPGS followed by two GS-relative accesses to set up RSP is somewhat slow because the GS-relative accesses need to wait for SWAPGS to finish. The trampoline approach allows RIP-relative accesses to set up RSP, which avoids the stall.
- The trampoline avoids any percpu access before CR3 is set up, which means that no percpu memory needs to be mapped in the user page tables. This prevents using Meltdown to read any percpu memory outside the cpu_entry_area and prevents using timing leaks to directly locate the percpu areas.
The downsides of using a trampoline may outweigh the upsides, however. It adds an extra non-contiguous I$ cache line to system calls, and it forces an indirect jump to transfer control back to the normal kernel text after CR3 is set up. The latter is because x86 lacks a 64-bit direct jump instruction that could jump from the trampoline to the entry text. With retpolines enabled, the indirect jump is extremely slow.
This patch changes the code to map the percpu TSS into the user page tables to allow the non-trampoline SYSCALL64 path to work under PTI. This does not add a new direct information leak, since the TSS is readable by Meltdown from the cpu_entry_area alias regardless. It does allow a timing attack to locate the percpu area, but KASLR is more or less a lost cause against local attack on CPUs vulnerable to Meltdown regardless. As far as I'm concerned, on current hardware, KASLR is only useful to mitigate remote attacks that try to attack the kernel without first gaining RCE against a vulnerable user process.
On Skylake, with CONFIG_RETPOLINE=y and KPTI on, this reduces syscall overhead from ~237ns to ~228ns.
There is a possible alternative approach: we could instead move the trampoline within 2G of the entry text and make a separate copy for each CPU. Then we could use a direct jump to rejoin the normal entry path.
--- arch/x86/entry/entry_64.S | 69 +-------------------------- arch/x86/include/asm/cpu_entry_area.h | 2 - arch/x86/include/asm/sections.h | 1 - arch/x86/kernel/asm-offsets.c | 2 - arch/x86/kernel/cpu/common.c | 11 +---- arch/x86/kernel/kprobes/core.c | 10 +--- arch/x86/kernel/vmlinux.lds.S | 10 ---- arch/x86/mm/cpu_entry_area.c | 36 -------------- arch/x86/mm/pti.c | 33 ++++++++++++- 9 files changed, 36 insertions(+), 138 deletions(-)
arch/x86/entry/entry_64.S +2 -67
@@ -142,67 +142,6 @@ END(native_usergs_sysret64) * with them due to bugs in both AMD and Intel CPUs. */ - .pushsection .entry_trampoline, "ax" - -/* - * The code in here gets remapped into cpu_entry_area's trampoline. This means - * that the assembler and linker have the wrong idea as to where this code - * lives (and, in fact, it's mapped more than once, so it's not even at a - * fixed address). So we can't reference any symbols outside the entry - * trampoline and expect it to work. - * - * Instead, we carefully abuse %rip-relative addressing. - * _entry_trampoline(%rip) refers to the start of the remapped) entry - * trampoline. We can thus find cpu_entry_area with this macro: - */ - -#define CPU_ENTRY_AREA \ - _entry_trampoline - CPU_ENTRY_AREA_entry_trampoline(%rip) - -/* The top word of the SYSENTER stack is hot and is usable as scratch space. */ -#define RSP_SCRATCH CPU_ENTRY_AREA_entry_stack + \ - SIZEOF_entry_stack - 8 + CPU_ENTRY_AREA - -ENTRY(entry_SYSCALL_64_trampoline) - UNWIND_HINT_EMPTY - swapgs - - /* Stash the user RSP. */ - movq %rsp, RSP_SCRATCH - - /* Note: using %rsp as a scratch reg. */ - SWITCH_TO_KERNEL_CR3 scratch_reg=%rsp - - /* Load the top of the task stack into RSP */ - movq CPU_ENTRY_AREA_tss + TSS_sp1 + CPU_ENTRY_AREA, %rsp - - /* Start building the simulated IRET frame. */ - pushq $__USER_DS /* pt_regs->ss */ - pushq RSP_SCRATCH /* pt_regs->sp */ - pushq %r11 /* pt_regs->flags */ - pushq $__USER_CS /* pt_regs->cs */ - pushq %rcx /* pt_regs->ip */ - - /* - * x86 lacks a near absolute jump, and we can't jump to the real - * entry text with a relative jump. We could push the target - * address and then use retq, but this destroys the pipeline on - * many CPUs (wasting over 20 cycles on Sandy Bridge). Instead, - * spill RDI and restore it in a second-stage trampoline. - */ - pushq %rdi - movq $entry_SYSCALL_64_stage2, %rdi - JMP_NOSPEC %rdi -END(entry_SYSCALL_64_trampoline) - - .popsection - -ENTRY(entry_SYSCALL_64_stage2) - UNWIND_HINT_EMPTY - popq %rdi - jmp entry_SYSCALL_64_after_hwframe -END(entry_SYSCALL_64_stage2) - ENTRY(entry_SYSCALL_64) UNWIND_HINT_EMPTY /*
@@ -212,13 +151,9 @@ ENTRY(entry_SYSCALL_64) */ swapgs - /* - * This path is only taken when PAGE_TABLE_ISOLATION is disabled so it - * is not required to switch CR3. - * - * tss.sp2 is scratch space. - */ + /* tss.sp2 is scratch space. */ movq %rsp, PER_CPU_VAR(cpu_tss_rw + TSS_sp2) + SWITCH_TO_KERNEL_CR3 scratch_reg=%rsp movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp /* Construct struct pt_regs on stack */
arch/x86/include/asm/cpu_entry_area.h +0 -2
@@ -30,8 +30,6 @@ struct cpu_entry_area {*/ struct tss_struct tss; -char entry_trampoline[PAGE_SIZE ]; -#ifdef CONFIG_X86_64 /* * Exception stacks used for IST entries.
arch/x86/include/asm/sections.h +0 -1
@@ -11,7 +11,6 @@ extern char __end_rodata_aligned[];#if defined(CONFIG_X86_64 )extern char __end_rodata_hpage_align[]; -extern char __entry_trampoline_start[], __entry_trampoline_end[];#endif #endif /* _ASM_X86_SECTIONS_H */
arch/x86/kernel/asm-offsets.c +0 -2
@@ -99,8 +99,6 @@ void common(void) {OFFSET (TLB_STATE_user_pcid_flush_mask, tlb_state, user_pcid_flush_mask);/* Layout info for cpu_entry_area */ -OFFSET (CPU_ENTRY_AREA_tss, cpu_entry_area, tss); -OFFSET (CPU_ENTRY_AREA_entry_trampoline, cpu_entry_area, entry_trampoline);OFFSET (CPU_ENTRY_AREA_entry_stack, cpu_entry_area, entry_stack_page);DEFINE (SIZEOF_entry_stack,sizeof (struct entry_stack ));DEFINE (MASK_entry_stack, (~(sizeof (struct entry_stack ) -1 )));
arch/x86/kernel/cpu/common.c +1 -10
@@ -1530,19 +1530,10 @@ EXPORT_PER_CPU_SYMBOL(__preempt_count);/* May not be marked __init: used by software suspend */ void syscall_init (void ) { -extern char _entry_trampoline[]; -extern char entry_SYSCALL_64_trampoline[]; -int cpu =smp_processor_id (); -unsigned long SYSCALL64_entry_trampoline = - (unsigned long )get_cpu_entry_area (cpu)->entry_trampoline + - (entry_SYSCALL_64_trampoline - _entry_trampoline);wrmsr (MSR_STAR ,0 , (__USER32_CS <<16 ) | __KERNEL_CS); -if (static_cpu_has (X86_FEATURE_PTI )) -wrmsrl (MSR_LSTAR , SYSCALL64_entry_trampoline); -else -wrmsrl (MSR_LSTAR , (unsigned long )entry_SYSCALL_64); +wrmsrl (MSR_LSTAR , (unsigned long )entry_SYSCALL_64);#ifdef CONFIG_IA32_EMULATION wrmsrl (MSR_CSTAR , (unsigned long )entry_SYSCALL_compat);
arch/x86/kernel/kprobes/core.c +1 -9
@@ -1066,18 +1066,10 @@ NOKPROBE_SYMBOL(kprobe_exceptions_notify);bool arch_within_kprobe_blacklist (unsigned long addr) { -bool is_in_entry_trampoline_section = false; - -#ifdef CONFIG_X86_64 - is_in_entry_trampoline_section = - (addr >= (unsigned long )__entry_trampoline_start && - addr < (unsigned long )__entry_trampoline_end); -#endif return (addr >= (unsigned long )__kprobes_text_start && addr < (unsigned long )__kprobes_text_end) || (addr >= (unsigned long )__entry_text_start && - addr < (unsigned long )__entry_text_end) || - is_in_entry_trampoline_section; + addr < (unsigned long )__entry_text_end); }int __init arch_init_kprobes (void )
arch/x86/kernel/vmlinux.lds.S +0 -10
@@ -118,16 +118,6 @@ SECTIONS *(.fixup) *(.gnu.warning) -#ifdef CONFIG_X86_64 - . = ALIGN(PAGE_SIZE); - __entry_trampoline_start = .; - _entry_trampoline = .; - *(.entry_trampoline) - . = ALIGN(PAGE_SIZE); - __entry_trampoline_end = .; - ASSERT(. - _entry_trampoline == PAGE_SIZE, "entry trampoline is too big"); -#endif - #ifdef CONFIG_RETPOLINE __indirect_thunk_start = .; *(.text.__x86.indirect_thunk)
arch/x86/mm/cpu_entry_area.c +0 -36
@@ -15,7 +15,6 @@ static DEFINE_PER_CPU_PAGE_ALIGNED(struct entry_stack_page, entry_stack_storage)#ifdef CONFIG_X86_64 static DEFINE_PER_CPU_PAGE_ALIGNED(char, exception_stacks [(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + DEBUG_STKSZ]); -static DEFINE_PER_CPU(struct kcore_list, kcore_entry_trampoline);#endif struct cpu_entry_area *get_cpu_entry_area (int cpu)
@@ -83,8 +82,6 @@ static void percpu_setup_debug_store(int cpu)static void __init setup_cpu_entry_area (int cpu) {#ifdef CONFIG_X86_64 -extern char _entry_trampoline[]; -/* On 64-bit systems, we use a read-only fixmap GDT and TSS. */ pgprot_t gdt_prot =PAGE_KERNEL_RO ;pgprot_t tss_prot =PAGE_KERNEL_RO ;
@@ -146,43 +143,10 @@ static void __init setup_cpu_entry_area(int cpu)cea_map_percpu_pages (&get_cpu_entry_area (cpu)->exception_stacks, &per_cpu (exception_stacks, cpu),sizeof (exception_stacks) /PAGE_SIZE ,PAGE_KERNEL ); - -cea_set_pte (&get_cpu_entry_area (cpu)->entry_trampoline, -__pa_symbol (_entry_trampoline),PAGE_KERNEL_RX ); -/* -* The cpu_entry_area alias addresses are not in the kernel binary -* so they do not show up in /proc/kcore normally. This adds entries -* for them manually. -*/ -kclist_add_remap (&per_cpu (kcore_entry_trampoline, cpu), - _entry_trampoline, - &get_cpu_entry_area (cpu)->entry_trampoline,PAGE_SIZE );#endif percpu_setup_debug_store (cpu); } -#ifdef CONFIG_X86_64 -int arch_get_kallsym (unsigned int symnum,unsigned long *value,char *type, -char *name) -{ -unsigned int cpu, ncpu =0 ; - -if (symnum >=num_possible_cpus ()) -return -EINVAL ; - -for_each_possible_cpu (cpu) { -if (ncpu++ >= symnum) -break ; - } - - *value = (unsigned long )&get_cpu_entry_area (cpu)->entry_trampoline; - *type ='t' ; -strlcpy (name,"__entry_SYSCALL_64_trampoline" ,KSYM_NAME_LEN ); - -return 0 ; -} -#endif -static __init void setup_cpu_entry_area_ptes (void ) {#ifdef CONFIG_X86_32
arch/x86/mm/pti.c +32 -1
@@ -434,11 +434,42 @@ static void __init pti_clone_p4d(unsigned long addr) }/* -* Clone the CPU_ENTRY_AREA into the user space visible page table. +* Clone the CPU_ENTRY_AREA and associated data into the user space visible +* page table. */ static void __init pti_clone_user_shared (void ) { +unsigned cpu; +pti_clone_p4d (CPU_ENTRY_AREA_BASE ); + +for_each_possible_cpu (cpu) { +/* +* The SYSCALL64 entry code needs to be able to find the +* thread stack and needs one word of scratch space in which +* to spill a register. All of this lives in the TSS, in +* the sp1 and sp2 slots. +* +* This is done for all possible CPUs during boot to ensure +* that it's propagated to all mms. If we were to add one of +* these mappings during CPU hotplug, we would need to take +* some measure to make sure that every mm that subsequently +* ran on that CPU would have the relevant PGD entry in its +* pagetables. The usual vmalloc_fault() mechanism would not +* work for page faults taken in entry_SYSCALL_64 before RSP +* is set up. +*/ + +unsigned long va = (unsigned long )&per_cpu (cpu_tss_rw, cpu); +phys_addr_t pa =per_cpu_ptr_to_phys ((void *)va); +pte_t *target_pte; + + target_pte =pti_user_pagetable_walk_pte (va); +if (WARN_ON (!target_pte)) +return ; + + *target_pte =pfn_pte (pa >>PAGE_SHIFT ,PAGE_KERNEL ); + } }#else /* CONFIG_X86_64 */
Signature · 1 line
2.17.1
Quoted 2 lines · Andy Lutomirski
Changes from v1: Get rid of the
rsp_scratchmacro
I can't see anything wrong with this.
Linus
Peter Zijlstra
maintainerRe: [PATCH v2 3/3] x86/pti/64: Remove the SYSCALL64 entry trampoline
In reply to Andy LutomirskiQuoted 31 lines · Andy Lutomirski
The SYSCALL64 trampoline has a couple of nice properties:
- The usual sequence of SWAPGS followed by two GS-relative accesses to set up RSP is somewhat slow because the GS-relative accesses need to wait for SWAPGS to finish. The trampoline approach allows RIP-relative accesses to set up RSP, which avoids the stall.
- The trampoline avoids any percpu access before CR3 is set up, which means that no percpu memory needs to be mapped in the user page tables. This prevents using Meltdown to read any percpu memory outside the
cpu_entry_areaand prevents using timing leaks to directly locate the percpu areas.The downsides of using a trampoline may outweigh the upsides, however. It adds an extra non-contiguous I$ cache line to system calls, and it forces an indirect jump to transfer control back to the normal kernel text after CR3 is set up. The latter is because x86 lacks a 64-bit direct jump instruction that could jump from the trampoline to the entry text. With retpolines enabled, the indirect jump is extremely slow.
This patch changes the code to map the percpu TSS into the user page tables to allow the non-trampoline SYSCALL64 path to work under PTI. This does not add a new direct information leak, since the TSS is readable by Meltdown from the
cpu_entry_areaalias regardless. It does allow a timing attack to locate the percpu area, but KASLR is more or less a lost cause against local attack on CPUs vulnerable to Meltdown regardless. As far as I'm concerned, on current hardware, KASLR is only useful to mitigate remote attacks that try to attack the kernel without first gaining RCE against a vulnerable user process.
Quoted 7 lines
On Skylake, with
CONFIG_RETPOLINE=yand KPTI on, this reduces syscall overhead from ~237ns to ~228ns.There is a possible alternative approach: we could instead move the trampoline within 2G of the entry text and make a separate copy for each CPU. Then we could use a direct jump to rejoin the normal entry path.
Can we have a few words on why this solution and not this alternative? I mean, you raise the possibility, but then surely you chose not to implement that. Might as well share that with us.
Andy Lutomirski
maintainerRe: [PATCH v2 3/3] x86/pti/64: Remove the SYSCALL64 entry trampoline
Quoted 35 lines · Peter Zijlstra
Quoted 34 lines · Andy Lutomirski
The SYSCALL64 trampoline has a couple of nice properties:
- The usual sequence of SWAPGS followed by two GS-relative accesses to set up RSP is somewhat slow because the GS-relative accesses need to wait for SWAPGS to finish. The trampoline approach allows RIP-relative accesses to set up RSP, which avoids the stall.
- The trampoline avoids any percpu access before CR3 is set up, which means that no percpu memory needs to be mapped in the user page tables. This prevents using Meltdown to read any percpu memory outside the
cpu_entry_areaand prevents using timing leaks to directly locate the percpu areas.The downsides of using a trampoline may outweigh the upsides, however. It adds an extra non-contiguous I$ cache line to system calls, and it forces an indirect jump to transfer control back to the normal kernel text after CR3 is set up. The latter is because x86 lacks a 64-bit direct jump instruction that could jump from the trampoline to the entry text. With retpolines enabled, the indirect jump is extremely slow.
This patch changes the code to map the percpu TSS into the user page tables to allow the non-trampoline SYSCALL64 path to work under PTI. This does not add a new direct information leak, since the TSS is readable by Meltdown from the
cpu_entry_areaalias regardless. It does allow a timing attack to locate the percpu area, but KASLR is more or less a lost cause against local attack on CPUs vulnerable to Meltdown regardless. As far as I'm concerned, on current hardware, KASLR is only useful to mitigate remote attacks that try to attack the kernel without first gaining RCE against a vulnerable user process.On Skylake, with
CONFIG_RETPOLINE=yand KPTI on, this reduces syscall overhead from ~237ns to ~228ns.
Quoted 8 lines
Quoted 4 lines
There is a possible alternative approach: we could instead move the trampoline within 2G of the entry text and make a separate copy for each CPU. Then we could use a direct jump to rejoin the normal entry path.
Can we have a few words on why this solution and not this alternative? I mean, you raise the possibility, but then surely you chose not to implement that. Might as well share that with us.
I can give some pros and cons. With the other approach:
- We avoid a pipeline stall.
- We execute from an extra page and read from another extra page
during the syscall. (The latter is because we need to use a relative addressing mode to find sp1 -- it's the same *cacheline* we'd use anyway, but we're accessing it using an alias, so it's an extra TLB entry.)
- We use more memory. This would be one page per CPU for a simple
implementation and 64-ish bytes per CPU or one page per node for a more complex implementation.
- More code complexity.
I'm not convinced this is a good tradeoff.
Re: [PATCH v2 1/3] x86/entry/64: Document idtentry
In reply to Andy LutomirskiQuoted 29 lines · Andy Lutomirski
The idtentry macro is complicated and magical. Document what it does to help future readers and to allow future patches to adjust the code and docs at the same time.
Signed-off-by Andy Lutomirski <[email protected]>--- arch/x86/entry/entry_64.S | 35 +++++++++++++++++++++++++++++++++++ arch/x86/kernel/traps.c | 4 ++++ 2 files changed, 39 insertions(+)arch/x86/entry/entry_64.S +11
-0@@ -900,6 +900,41 @@ apicinterrupt IRQ_WORK_VECTOR irq_work_interrupt smp_irq_work_interrupt */ #define CPU_TSS_IST(x) PER_CPU_VAR(cpu_tss_rw) + (TSS_ist + ((x) - 1) * 8) +/** + * idtentry - Generate an IDT entry stub + * @sym: Name of the generated entry point + * @do_sym: C function to be called + * @has_error_code: True if this IDT vector has an error code on the stack + * @paranoid: non-zero means that this vector may be invoked from kernel + * mode with user GSBASE and/or user CR3. 2 is special -- see below. + * @shift_ist: Set to an IST index if entries from kernel mode should + * decrement the IST stack so that nested entries get a fresh + * stack. (This is for #DB, which has a nasty habit of + * recursing.)
arch/x86/entry/entry_64.S apicinterrupt IRQ_WORK_VECTOR irq_work_interrupt smp_irq_work_interrupt · line 923
Quoted 10 lines
arch/x86/entry/entry_64.S +10
-0+ * + * idtentry generates an IDT stub that sets up a usable kernel context, + * creates struct pt_regs, and calls @do_sym. The stub has the following + * special behaviors: + * + * On an entry from user mode, the stub switches from the trampoline or + * IST stack to the normal thread stack. On an exit to user mode, the + * normal exit-to-usermode path is invoked. + * + * On an exit to kernel mode, if paranoid == 0, we check for preemption,
@paranoid
Otherwise, documenting our entry maze is always a good idea!
Signature · 4 lines
Regards/Gruss,
Boris.
Good mailing practices for 400: avoid top-posting and trim the reply.Re: [PATCH v2 2/3] x86/entry/64: Use the TSS sp2 slot for SYSCALL/SYSRET scratch space
In reply to Andy LutomirskiQuoted 7 lines · Andy Lutomirski
In the non-trampoline SYSCALL64 path, we use a percpu variable to temporarily store the user RSP value. Instead of a separate variable, use the otherwise unused sp2 slot in the TSS. This will improve cache locality, as the sp1 slot is already used in the same code to find the kernel stack. It will also simplify a future change to make the non-trampoline path work in PTI mode.
Quoted 8 lines
Signed-off-by Andy Lutomirski <[email protected]>--- arch/x86/entry/entry_64.S | 16 +++++++++------- arch/x86/include/asm/processor.h | 6 ++++++ arch/x86/kernel/asm-offsets.c | 3 ++- arch/x86/kernel/process_64.c | 2 -- arch/x86/xen/xen-asm_64.S | 8 +++++--- 5 files changed, 22 insertions(+), 13 deletions(-)
Signature · 4 lines
Regards/Gruss,
Boris.
Good mailing practices for 400: avoid top-posting and trim the reply.Re: [PATCH v2 3/3] x86/pti/64: Remove the SYSCALL64 entry trampoline
In reply to Andy LutomirskiQuoted 6 lines · Andy Lutomirski
The SYSCALL64 trampoline has a couple of nice properties:
- The usual sequence of SWAPGS followed by two GS-relative accesses to set up RSP is somewhat slow because the GS-relative accesses need to wait for SWAPGS to finish. The trampoline approach allows RIP-relative accesses to set up RSP, which avoids the stall.
...
Quoted 5 lines
arch/x86/mm/pti.c +0
-0
arch/x86/mm/pti.c static void __init pti_clone_p4d(unsigned long addr) · line 442
Quoted 10 lines
arch/x86/mm/pti.c +3
-1@@ -434,11 +434,42 @@ static void __init pti_clone_p4d(unsigned long addr) }/* -* Clone the CPU_ENTRY_AREA into the user space visible page table. +* Clone the CPU_ENTRY_AREA and associated data into the user space visible +* page table. */ static void __init pti_clone_user_shared (void ) { +unsigned cpu;
Make that
unsigned int cpu;
Otherwise, patches removing complex code are always good!
Signature · 4 lines
Regards/Gruss,
Boris.
Good mailing practices for 400: avoid top-posting and trim the reply.Peter Zijlstra
maintainerRe: [PATCH v2 3/3] x86/pti/64: Remove the SYSCALL64 entry trampoline
In reply to Andy LutomirskiQuoted 25 lines · Andy Lutomirski
Quoted 10 lines · Peter Zijlstra
Quoted 4 lines · Andy Lutomirski
There is a possible alternative approach: we could instead move the trampoline within 2G of the entry text and make a separate copy for each CPU. Then we could use a direct jump to rejoin the normal entry path.
Can we have a few words on why this solution and not this alternative? I mean, you raise the possibility, but then surely you chose not to implement that. Might as well share that with us.
I can give some pros and cons. With the other approach:
- We avoid a pipeline stall.
- We execute from an extra page and read from another extra page
during the syscall. (The latter is because we need to use a relative addressing mode to find sp1 -- it's the same *cacheline* we'd use anyway, but we're accessing it using an alias, so it's an extra TLB entry.)
- We use more memory. This would be one page per CPU for a simple
implementation and 64-ish bytes per CPU or one page per node for a more complex implementation.
- More code complexity.
Quoted 1 line
I'm not convinced this is a good tradeoff.
Fair enough, thanks!
Re: [PATCH v2 3/3] x86/pti/64: Remove the SYSCALL64 entry trampoline
In reply to Andy LutomirskiQuoted 34 lines · Andy Lutomirski
The SYSCALL64 trampoline has a couple of nice properties:
- The usual sequence of SWAPGS followed by two GS-relative accesses to set up RSP is somewhat slow because the GS-relative accesses need to wait for SWAPGS to finish. The trampoline approach allows RIP-relative accesses to set up RSP, which avoids the stall.
- The trampoline avoids any percpu access before CR3 is set up, which means that no percpu memory needs to be mapped in the user page tables. This prevents using Meltdown to read any percpu memory outside the
cpu_entry_areaand prevents using timing leaks to directly locate the percpu areas.The downsides of using a trampoline may outweigh the upsides, however. It adds an extra non-contiguous I$ cache line to system calls, and it forces an indirect jump to transfer control back to the normal kernel text after CR3 is set up. The latter is because x86 lacks a 64-bit direct jump instruction that could jump from the trampoline to the entry text. With retpolines enabled, the indirect jump is extremely slow.
This patch changes the code to map the percpu TSS into the user page tables to allow the non-trampoline SYSCALL64 path to work under PTI. This does not add a new direct information leak, since the TSS is readable by Meltdown from the
cpu_entry_areaalias regardless. It does allow a timing attack to locate the percpu area, but KASLR is more or less a lost cause against local attack on CPUs vulnerable to Meltdown regardless. As far as I'm concerned, on current hardware, KASLR is only useful to mitigate remote attacks that try to attack the kernel without first gaining RCE against a vulnerable user process.On Skylake, with
CONFIG_RETPOLINE=yand KPTI on, this reduces syscall overhead from ~237ns to ~228ns.
Quoted 6 lines
There is a possible alternative approach: we could instead move the trampoline within 2G of the entry text and make a separate copy for each CPU. Then we could use a direct jump to rejoin the normal entry path.
Signed-off-by Andy Lutomirski <[email protected]>
The following commit should also be reverted:
4d99e4136580 ("perf machine: Workaround missing maps for x86 PTI entry trampolines")
Signature · 1 line
Josh
Re: [PATCH v2 3/3] x86/pti/64: Remove the SYSCALL64 entry trampoline
In reply to Andy LutomirskiQuoted 8 lines · Andy Lutomirski
Quoted 3 lines · Peter Zijlstra
Can we have a few words on why this solution and not this alternative? I mean, you raise the possibility, but then surely you chose not to implement that. Might as well share that with us.
I can give some pros and cons. With the other approach:
- We avoid a pipeline stall.
Which is good.
Quoted 5 lines
- We execute from an extra page and read from another extra page
during the syscall. (The latter is because we need to use a relative addressing mode to find sp1 -- it's the same *cacheline* we'd use anyway, but we're accessing it using an alias, so it's an extra TLB entry.)
Ok, but is this really an issue with PTI?
Quoted 3 lines
- We use more memory. This would be one page per CPU for a simple
implementation and 64-ish bytes per CPU or one page per node for a more complex implementation.
That's the least interesting argument really.
Quoted 1 line
- More code complexity.
Ok, but how much complex code is that?
Quoted 1 line
I'm not convinced this is a good tradeoff.
Well, the real question here is whether this has any advantage vs. the percpu area exposure?
Thanks,
tglx
Re: [PATCH v2 3/3] x86/pti/64: Remove the SYSCALL64 entry trampoline
Quoted 8 lines · Thomas Gleixner
Quoted 6 lines
- We execute from an extra page and read from another extra page
during the syscall. (The latter is because we need to use a relative addressing mode to find sp1 -- it's the same *cacheline* we'd use anyway, but we're accessing it using an alias, so it's an extra TLB entry.)Ok, but is this really an issue with PTI?
I'd expect it to be *more* of an issue with PTI, since you're already wasting TLB entries due to the whole "two different page tables".
Sure, address space ID's save you from reloading them all the time, but don't help with capacity.
But yeah, in the sense of "with PTI, all kernel entries are slow anyway, so none of this matters" is probably correct in a very real sense.
That said, the real reason I like Andy's patch series is that I think it's simpler than the alternatives (including the current setup). No subtle mappings, no nothing. It removes a lot more lines than it adds, and half the lines that it *does* add are comments.
Virtual mapping tricks may be cool, but in the end, not having to use them is better still, I think.
Linus
Andy Lutomirski
maintainerRe: [PATCH v2 3/3] x86/pti/64: Remove the SYSCALL64 entry trampoline
Quoted 20 lines · Linus Torvalds
Quoted 8 lines · Thomas Gleixner
Quoted 6 lines
- We execute from an extra page and read from another extra page
during the syscall. (The latter is because we need to use a relative addressing mode to find sp1 -- it's the same *cacheline* we'd use anyway, but we're accessing it using an alias, so it's an extra TLB entry.)Ok, but is this really an issue with PTI?
I'd expect it to be *more* of an issue with PTI, since you're already wasting TLB entries due to the whole "two different page tables".
Sure, address space ID's save you from reloading them all the time, but don't help with capacity.
But yeah, in the sense of "with PTI, all kernel entries are slow anyway, so none of this matters" is probably correct in a very real sense.
Quoted 8 lines
That said, the real reason I like Andy's patch series is that I think it's simpler than the alternatives (including the current setup). No subtle mappings, no nothing. It removes a lot more lines than it adds, and half the lines that it *does* add are comments.
Virtual mapping tricks may be cool, but in the end, not having to use them is better still, I think.
If (and this is a *big* if) all the percpu data is within 2GB of the entry text, then we could avoid this extra TLB reference by accessing it directly instead of using an alias.
I suppose the summary is that the retpoline-free trampoline variant is even more complicated than the code I'm removing in this series, and that it would be at best a teeny tiny win. Once all the Spectre dust settles and we get fixed CPUs, we could consider re-optimizing this.
Andy Lutomirski
maintainerRe: [PATCH v2 3/3] x86/pti/64: Remove the SYSCALL64 entry trampoline
In reply to Josh PoimboeufQuoted 40 lines · Josh Poimboeuf
Quoted 39 lines · Andy Lutomirski
The SYSCALL64 trampoline has a couple of nice properties:
- The usual sequence of SWAPGS followed by two GS-relative accesses to set up RSP is somewhat slow because the GS-relative accesses need to wait for SWAPGS to finish. The trampoline approach allows RIP-relative accesses to set up RSP, which avoids the stall.
- The trampoline avoids any percpu access before CR3 is set up, which means that no percpu memory needs to be mapped in the user page tables. This prevents using Meltdown to read any percpu memory outside the
cpu_entry_areaand prevents using timing leaks to directly locate the percpu areas.The downsides of using a trampoline may outweigh the upsides, however. It adds an extra non-contiguous I$ cache line to system calls, and it forces an indirect jump to transfer control back to the normal kernel text after CR3 is set up. The latter is because x86 lacks a 64-bit direct jump instruction that could jump from the trampoline to the entry text. With retpolines enabled, the indirect jump is extremely slow.
This patch changes the code to map the percpu TSS into the user page tables to allow the non-trampoline SYSCALL64 path to work under PTI. This does not add a new direct information leak, since the TSS is readable by Meltdown from the
cpu_entry_areaalias regardless. It does allow a timing attack to locate the percpu area, but KASLR is more or less a lost cause against local attack on CPUs vulnerable to Meltdown regardless. As far as I'm concerned, on current hardware, KASLR is only useful to mitigate remote attacks that try to attack the kernel without first gaining RCE against a vulnerable user process.On Skylake, with
CONFIG_RETPOLINE=yand KPTI on, this reduces syscall overhead from ~237ns to ~228ns.There is a possible alternative approach: we could instead move the trampoline within 2G of the entry text and make a separate copy for each CPU. Then we could use a direct jump to rejoin the normal entry path.
Quoted 5 lines
Quoted 1 line
Signed-off-by Andy Lutomirski <[email protected]>The following commit should also be reverted:
4d99e4136580("perf machine: Workaround missing maps for x86 PTI entry trampolines")
I think we shouldn't, since the perf folks want new perf versions to be fully functional on older kernels. My plan was to let the perf crew do whatever reversions they felt appropriate.
Re: [PATCH v2 3/3] x86/pti/64: Remove the SYSCALL64 entry trampoline
In reply to Linus TorvaldsQuoted 13 lines · Linus Torvalds
Quoted 8 lines · Thomas Gleixner
Quoted 6 lines
- We execute from an extra page and read from another extra page
during the syscall. (The latter is because we need to use a relative addressing mode to find sp1 -- it's the same *cacheline* we'd use anyway, but we're accessing it using an alias, so it's an extra TLB entry.)Ok, but is this really an issue with PTI?
I'd expect it to be *more* of an issue with PTI, since you're already wasting TLB entries due to the whole "two different page tables".
Quoted 6 lines
Sure, address space ID's save you from reloading them all the time, but don't help with capacity.
But yeah, in the sense of "with PTI, all kernel entries are slow anyway, so none of this matters" is probably correct in a very real sense.
Unfortunately yes. As Andy pointed out the alternative approach would trade an avoided pipeline stall with an extra TLB.
Quoted 4 lines
That said, the real reason I like Andy's patch series is that I think it's simpler than the alternatives (including the current setup). No subtle mappings, no nothing. It removes a lot more lines than it adds, and half the lines that it *does* add are comments.
I agree and in fact I had the series already queued for merging, I just wanted to get the full picture and add some of that information to the changelog.
Quoted 2 lines
Virtual mapping tricks may be cool, but in the end, not having to use them is better still, I think.
Sure, unless they provide a real value. The maybe smaller leak of address information is probably not worth it.
Thanks,
tglx
Re: [PATCH v2 3/3] x86/pti/64: Remove the SYSCALL64 entry trampoline
In reply to Andy LutomirskiQuoted 6 lines · Andy Lutomirski
Quoted 3 lines · Linus Torvalds
Virtual mapping tricks may be cool, but in the end, not having to use them is better still, I think.
Quoted 8 lines
If (and this is a *big* if) all the percpu data is within 2GB of the entry text, then we could avoid this extra TLB reference by accessing it directly instead of using an alias.
I suppose the summary is that the retpoline-free trampoline variant is even more complicated than the code I'm removing in this series, and that it would be at best a teeny tiny win. Once all the Spectre dust settles and we get fixed CPUs, we could consider re-optimizing this.
That's going to be after my retirement ...
tip-bot for Andy Lutomirski
merge bot[tip:x86/pti] x86/entry/64: Document idtentry
In reply to Andy Lutomirskix86/entry/64: Document idtentry
Commit-ID: bd7b1f7cbf9cb35dab8e1b99145d07afc5b7a132 Gitweb: https://git.kernel.org/tip/bd7b1f7cbf9cb35dab8e1b99145d07afc5b7a132 Author: Andy Lutomirski <[email protected]> AuthorDate: Mon, 3 Sep 2018 15:59:42 -0700 Committer: Thomas Gleixner <[email protected]> CommitDate: Sat, 8 Sep 2018 11:20:11 +0200
x86/entry/64: Document idtentry
The idtentry macro is complicated and magical. Document what it does to help future readers and to allow future patches to adjust the code and docs at the same time.
--- arch/x86/entry/entry_64.S | 36 ++++++++++++++++++++++++++++++++++++ arch/x86/kernel/traps.c | 4 ++++ 2 files changed, 40 insertions(+)
arch/x86/entry/entry_64.S +36 -0
@@ -900,6 +900,42 @@ apicinterrupt IRQ_WORK_VECTOR irq_work_interrupt smp_irq_work_interrupt */ #define CPU_TSS_IST(x) PER_CPU_VAR(cpu_tss_rw) + (TSS_ist + ((x) - 1) * 8) +/** + * idtentry - Generate an IDT entry stub + * @sym: Name of the generated entry point + * @do_sym: C function to be called + * @has_error_code: True if this IDT vector has an error code on the stack + * @paranoid: non-zero means that this vector may be invoked from + * kernel mode with user GSBASE and/or user CR3. + * 2 is special -- see below. + * @shift_ist: Set to an IST index if entries from kernel mode should + * decrement the IST stack so that nested entries get a + * fresh stack. (This is for #DB, which has a nasty habit + * of recursing.) + * + * idtentry generates an IDT stub that sets up a usable kernel context, + * creates struct pt_regs, and calls @do_sym. The stub has the following + * special behaviors: + * + * On an entry from user mode, the stub switches from the trampoline or + * IST stack to the normal thread stack. On an exit to user mode, the + * normal exit-to-usermode path is invoked. + * + * On an exit to kernel mode, if @paranoid == 0, we check for preemption, + * whereas we omit the preemption check if @paranoid != 0. This is purely + * because the implementation is simpler this way. The kernel only needs + * to check for asynchronous kernel preemption when IRQ handlers return. + * + * If @paranoid == 0, then the stub will handle IRET faults by pretending + * that the fault came from user mode. It will handle gs_change faults by + * pretending that the fault happened with kernel GSBASE. Since this handling + * is omitted for @paranoid != 0, the #GP, #SS, and #NP stubs must have + * @paranoid == 0. This special handling will do the wrong thing for + * espfix-induced #DF on IRET, so #DF must not use @paranoid == 0. + * + * @paranoid == 2 is special: the stub will never switch stacks. This is for + * #DF: if the thread stack is somehow unusable, we'll still get a useful OOPS. + */ .macro idtentry sym do_sym has_error_code:req paranoid=0 shift_ist=-1 ENTRY(\sym) UNWIND_HINT_IRET_REGS offset=\has_error_code*8
arch/x86/kernel/traps.c +4 -0
@@ -383,6 +383,10 @@ dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code)* we won't enable interupts or schedule before we invoke * general_protection, so nothing will clobber the stack * frame we just set up. +* +* We will enter general_protection with kernel GSBASE, +* which is what the stub expects, given that the faulting +* RIP will be the IRET instruction. */ regs->ip = (unsigned long )general_protection; regs->sp = (unsigned long )&gpregs->orig_ax;
tip-bot for Andy Lutomirski
merge bot[tip:x86/pti] x86/entry/64: Use the TSS sp2 slot for SYSCALL/SYSRET scratch space
In reply to Andy Lutomirskix86/entry/64: Use the TSS sp2 slot for SYSCALL/SYSRET scratch space
Commit-ID: 98f05b5138f0a9b56022295cc1387e635b25635d Gitweb: https://git.kernel.org/tip/98f05b5138f0a9b56022295cc1387e635b25635d Author: Andy Lutomirski <[email protected]> AuthorDate: Mon, 3 Sep 2018 15:59:43 -0700 Committer: Thomas Gleixner <[email protected]> CommitDate: Sat, 8 Sep 2018 11:20:11 +0200
x86/entry/64: Use the TSS sp2 slot for SYSCALL/SYSRET scratch space
In the non-trampoline SYSCALL64 path, a percpu variable is used to temporarily store the user RSP value.
Instead of a separate variable, use the otherwise unused sp2 slot in the TSS. This will improve cache locality, as the sp1 slot is already used in the same code to find the kernel stack. It will also simplify a future change to make the non-trampoline path work in PTI mode.
--- arch/x86/entry/entry_64.S | 16 +++++++++------- arch/x86/include/asm/processor.h | 6 ++++++ arch/x86/kernel/asm-offsets.c | 3 ++- arch/x86/kernel/process_64.c | 2 -- arch/x86/xen/xen-asm_64.S | 8 +++++--- 5 files changed, 22 insertions(+), 13 deletions(-)
arch/x86/entry/entry_64.S +9 -7
@@ -215,18 +215,20 @@ ENTRY(entry_SYSCALL_64) /* * This path is only taken when PAGE_TABLE_ISOLATION is disabled so it * is not required to switch CR3. + * + * tss.sp2 is scratch space. */ - movq %rsp, PER_CPU_VAR(rsp_scratch) + movq %rsp, PER_CPU_VAR(cpu_tss_rw + TSS_sp2) movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp /* Construct struct pt_regs on stack */ - pushq $__USER_DS /* pt_regs->ss */ - pushq PER_CPU_VAR(rsp_scratch) /* pt_regs->sp */ - pushq %r11 /* pt_regs->flags */ - pushq $__USER_CS /* pt_regs->cs */ - pushq %rcx /* pt_regs->ip */ + pushq $__USER_DS /* pt_regs->ss */ + pushq PER_CPU_VAR(cpu_tss_rw + TSS_sp2) /* pt_regs->sp */ + pushq %r11 /* pt_regs->flags */ + pushq $__USER_CS /* pt_regs->cs */ + pushq %rcx /* pt_regs->ip */ GLOBAL(entry_SYSCALL_64_after_hwframe) - pushq %rax /* pt_regs->orig_ax */ + pushq %rax /* pt_regs->orig_ax */ PUSH_AND_CLEAR_REGS rax=$-ENOSYS
arch/x86/include/asm/processor.h +6 -0
@@ -315,7 +315,13 @@ struct x86_hw_tss {*/ u64 sp1; +/* +* Since Linux does not use ring 2, the 'sp2' slot is unused by +* hardware. entry_SYSCALL_64 uses it as scratch space to stash +* the user RSP value. +*/ u64 sp2; +u64 reserved2;u64 ist[7 ];u32 reserved3;
arch/x86/kernel/asm-offsets.c +2 -1
@@ -105,7 +105,8 @@ void common(void) {DEFINE (SIZEOF_entry_stack,sizeof (struct entry_stack ));DEFINE (MASK_entry_stack, (~(sizeof (struct entry_stack ) -1 ))); -/* Offset for sp0 and sp1 into the tss_struct */ +/* Offset for fields in tss_struct */ OFFSET (TSS_sp0, tss_struct, x86_tss.sp0);OFFSET (TSS_sp1, tss_struct, x86_tss.sp1); +OFFSET (TSS_sp2, tss_struct, x86_tss.sp2); }
arch/x86/kernel/process_64.c +0 -2
@@ -59,8 +59,6 @@#include <asm/unistd_32_ia32.h> #endif -__visible DEFINE_PER_CPU (unsigned long ,rsp_scratch ); -/* Prints also some state that isn't saved in the pt_regs */ void __show_regs (struct pt_regs *regs,int all) {
arch/x86/xen/xen-asm_64.S +5 -3
@@ -91,13 +91,15 @@ ENTRY(xen_iret) ENTRY(xen_sysret64) /* * We're already on the usermode stack at this point, but - * still with the kernel gs, so we can easily switch back + * still with the kernel gs, so we can easily switch back. + * + * tss.sp2 is scratch space. */ - movq %rsp, PER_CPU_VAR(rsp_scratch) + movq %rsp, PER_CPU_VAR(cpu_tss_rw + TSS_sp2) movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp pushq $__USER_DS - pushq PER_CPU_VAR(rsp_scratch) + pushq PER_CPU_VAR(cpu_tss_rw + TSS_sp2) pushq %r11 pushq $__USER_CS pushq %rcx
tip-bot for Andy Lutomirski
merge bot[tip:x86/pti] x86/pti/64: Remove the SYSCALL64 entry trampoline
In reply to Andy Lutomirskix86/pti/64: Remove the SYSCALL64 entry trampoline
Commit-ID: 344347941aba1ec906ff50b4cdb8178c906746f2 Gitweb: https://git.kernel.org/tip/344347941aba1ec906ff50b4cdb8178c906746f2 Author: Andy Lutomirski <[email protected]> AuthorDate: Mon, 3 Sep 2018 15:59:44 -0700 Committer: Thomas Gleixner <[email protected]> CommitDate: Sat, 8 Sep 2018 11:20:12 +0200
x86/pti/64: Remove the SYSCALL64 entry trampoline
The SYSCALL64 trampoline has a couple of nice properties:
- The usual sequence of SWAPGS followed by two GS-relative accesses to set up RSP is somewhat slow because the GS-relative accesses need to wait for SWAPGS to finish. The trampoline approach allows RIP-relative accesses to set up RSP, which avoids the stall.
- The trampoline avoids any percpu access before CR3 is set up, which means that no percpu memory needs to be mapped in the user page tables. This prevents using Meltdown to read any percpu memory outside the cpu_entry_area and prevents using timing leaks to directly locate the percpu areas.
The downsides of using a trampoline may outweigh the upsides, however. It adds an extra non-contiguous I$ cache line to system calls, and it forces an indirect jump to transfer control back to the normal kernel text after CR3 is set up. The latter is because x86 lacks a 64-bit direct jump instruction that could jump from the trampoline to the entry text. With retpolines enabled, the indirect jump is extremely slow.
Change the code to map the percpu TSS into the user page tables to allow the non-trampoline SYSCALL64 path to work under PTI. This does not add a new direct information leak, since the TSS is readable by Meltdown from the cpu_entry_area alias regardless. It does allow a timing attack to locate the percpu area, but KASLR is more or less a lost cause against local attack on CPUs vulnerable to Meltdown regardless. As far as I'm concerned, on current hardware, KASLR is only useful to mitigate remote attacks that try to attack the kernel without first gaining RCE against a vulnerable user process.
On Skylake, with CONFIG_RETPOLINE=y and KPTI on, this reduces syscall overhead from ~237ns to ~228ns.
There is a possible alternative approach: Move the trampoline within 2G of the entry text and make a separate copy for each CPU. This would allow a direct jump to rejoin the normal entry path. There are pro's and con's for this approach:
+ It avoids a pipeline stall
- It executes from an extra page and read from another extra page during the syscall. The latter is because it needs to use a relative addressing mode to find sp1 -- it's the same *cacheline*, but accessed using an alias, so it's an extra TLB entry.
- Slightly more memory. This would be one page per CPU for a simple implementation and 64-ish bytes per CPU or one page per node for a more complex implementation.
- More code complexity.
The current approach is chosen for simplicity and because the alternative does not provide a significant benefit, which makes it worth.
[ tglx: Added the alternative discussion to the changelog ]
--- arch/x86/entry/entry_64.S | 69 +---------------------------------- arch/x86/include/asm/cpu_entry_area.h | 2 - arch/x86/include/asm/sections.h | 1 - arch/x86/kernel/asm-offsets.c | 2 - arch/x86/kernel/cpu/common.c | 11 +----- arch/x86/kernel/kprobes/core.c | 10 +---- arch/x86/kernel/vmlinux.lds.S | 10 ----- arch/x86/mm/cpu_entry_area.c | 36 ------------------ arch/x86/mm/pti.c | 33 ++++++++++++++++- 9 files changed, 36 insertions(+), 138 deletions(-)
arch/x86/entry/entry_64.S +2 -67
@@ -142,67 +142,6 @@ END(native_usergs_sysret64) * with them due to bugs in both AMD and Intel CPUs. */ - .pushsection .entry_trampoline, "ax" - -/* - * The code in here gets remapped into cpu_entry_area's trampoline. This means - * that the assembler and linker have the wrong idea as to where this code - * lives (and, in fact, it's mapped more than once, so it's not even at a - * fixed address). So we can't reference any symbols outside the entry - * trampoline and expect it to work. - * - * Instead, we carefully abuse %rip-relative addressing. - * _entry_trampoline(%rip) refers to the start of the remapped) entry - * trampoline. We can thus find cpu_entry_area with this macro: - */ - -#define CPU_ENTRY_AREA \ - _entry_trampoline - CPU_ENTRY_AREA_entry_trampoline(%rip) - -/* The top word of the SYSENTER stack is hot and is usable as scratch space. */ -#define RSP_SCRATCH CPU_ENTRY_AREA_entry_stack + \ - SIZEOF_entry_stack - 8 + CPU_ENTRY_AREA - -ENTRY(entry_SYSCALL_64_trampoline) - UNWIND_HINT_EMPTY - swapgs - - /* Stash the user RSP. */ - movq %rsp, RSP_SCRATCH - - /* Note: using %rsp as a scratch reg. */ - SWITCH_TO_KERNEL_CR3 scratch_reg=%rsp - - /* Load the top of the task stack into RSP */ - movq CPU_ENTRY_AREA_tss + TSS_sp1 + CPU_ENTRY_AREA, %rsp - - /* Start building the simulated IRET frame. */ - pushq $__USER_DS /* pt_regs->ss */ - pushq RSP_SCRATCH /* pt_regs->sp */ - pushq %r11 /* pt_regs->flags */ - pushq $__USER_CS /* pt_regs->cs */ - pushq %rcx /* pt_regs->ip */ - - /* - * x86 lacks a near absolute jump, and we can't jump to the real - * entry text with a relative jump. We could push the target - * address and then use retq, but this destroys the pipeline on - * many CPUs (wasting over 20 cycles on Sandy Bridge). Instead, - * spill RDI and restore it in a second-stage trampoline. - */ - pushq %rdi - movq $entry_SYSCALL_64_stage2, %rdi - JMP_NOSPEC %rdi -END(entry_SYSCALL_64_trampoline) - - .popsection - -ENTRY(entry_SYSCALL_64_stage2) - UNWIND_HINT_EMPTY - popq %rdi - jmp entry_SYSCALL_64_after_hwframe -END(entry_SYSCALL_64_stage2) - ENTRY(entry_SYSCALL_64) UNWIND_HINT_EMPTY /*
@@ -212,13 +151,9 @@ ENTRY(entry_SYSCALL_64) */ swapgs - /* - * This path is only taken when PAGE_TABLE_ISOLATION is disabled so it - * is not required to switch CR3. - * - * tss.sp2 is scratch space. - */ + /* tss.sp2 is scratch space. */ movq %rsp, PER_CPU_VAR(cpu_tss_rw + TSS_sp2) + SWITCH_TO_KERNEL_CR3 scratch_reg=%rsp movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp /* Construct struct pt_regs on stack */
arch/x86/include/asm/cpu_entry_area.h +0 -2
@@ -30,8 +30,6 @@ struct cpu_entry_area {*/ struct tss_struct tss; -char entry_trampoline[PAGE_SIZE ]; -#ifdef CONFIG_X86_64 /* * Exception stacks used for IST entries.
arch/x86/include/asm/sections.h +0 -1
@@ -11,7 +11,6 @@ extern char __end_rodata_aligned[];#if defined(CONFIG_X86_64 )extern char __end_rodata_hpage_align[]; -extern char __entry_trampoline_start[], __entry_trampoline_end[];#endif #endif /* _ASM_X86_SECTIONS_H */
arch/x86/kernel/asm-offsets.c +0 -2
@@ -99,8 +99,6 @@ void common(void) {OFFSET (TLB_STATE_user_pcid_flush_mask, tlb_state, user_pcid_flush_mask);/* Layout info for cpu_entry_area */ -OFFSET (CPU_ENTRY_AREA_tss, cpu_entry_area, tss); -OFFSET (CPU_ENTRY_AREA_entry_trampoline, cpu_entry_area, entry_trampoline);OFFSET (CPU_ENTRY_AREA_entry_stack, cpu_entry_area, entry_stack_page);DEFINE (SIZEOF_entry_stack,sizeof (struct entry_stack ));DEFINE (MASK_entry_stack, (~(sizeof (struct entry_stack ) -1 )));
arch/x86/kernel/cpu/common.c +1 -10
@@ -1531,19 +1531,10 @@ EXPORT_PER_CPU_SYMBOL(__preempt_count);/* May not be marked __init: used by software suspend */ void syscall_init (void ) { -extern char _entry_trampoline[]; -extern char entry_SYSCALL_64_trampoline[]; -int cpu =smp_processor_id (); -unsigned long SYSCALL64_entry_trampoline = - (unsigned long )get_cpu_entry_area (cpu)->entry_trampoline + - (entry_SYSCALL_64_trampoline - _entry_trampoline);wrmsr (MSR_STAR ,0 , (__USER32_CS <<16 ) | __KERNEL_CS); -if (static_cpu_has (X86_FEATURE_PTI )) -wrmsrl (MSR_LSTAR , SYSCALL64_entry_trampoline); -else -wrmsrl (MSR_LSTAR , (unsigned long )entry_SYSCALL_64); +wrmsrl (MSR_LSTAR , (unsigned long )entry_SYSCALL_64);#ifdef CONFIG_IA32_EMULATION wrmsrl (MSR_CSTAR , (unsigned long )entry_SYSCALL_compat);
arch/x86/kernel/kprobes/core.c +1 -9
@@ -1066,18 +1066,10 @@ NOKPROBE_SYMBOL(kprobe_exceptions_notify);bool arch_within_kprobe_blacklist (unsigned long addr) { -bool is_in_entry_trampoline_section = false; - -#ifdef CONFIG_X86_64 - is_in_entry_trampoline_section = - (addr >= (unsigned long )__entry_trampoline_start && - addr < (unsigned long )__entry_trampoline_end); -#endif return (addr >= (unsigned long )__kprobes_text_start && addr < (unsigned long )__kprobes_text_end) || (addr >= (unsigned long )__entry_text_start && - addr < (unsigned long )__entry_text_end) || - is_in_entry_trampoline_section; + addr < (unsigned long )__entry_text_end); }int __init arch_init_kprobes (void )
arch/x86/kernel/vmlinux.lds.S +0 -10
@@ -118,16 +118,6 @@ SECTIONS *(.fixup) *(.gnu.warning) -#ifdef CONFIG_X86_64 - . = ALIGN(PAGE_SIZE); - __entry_trampoline_start = .; - _entry_trampoline = .; - *(.entry_trampoline) - . = ALIGN(PAGE_SIZE); - __entry_trampoline_end = .; - ASSERT(. - _entry_trampoline == PAGE_SIZE, "entry trampoline is too big"); -#endif - #ifdef CONFIG_RETPOLINE __indirect_thunk_start = .; *(.text.__x86.indirect_thunk)
arch/x86/mm/cpu_entry_area.c +0 -36
@@ -15,7 +15,6 @@ static DEFINE_PER_CPU_PAGE_ALIGNED(struct entry_stack_page, entry_stack_storage)#ifdef CONFIG_X86_64 static DEFINE_PER_CPU_PAGE_ALIGNED(char, exception_stacks [(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + DEBUG_STKSZ]); -static DEFINE_PER_CPU(struct kcore_list, kcore_entry_trampoline);#endif struct cpu_entry_area *get_cpu_entry_area (int cpu)
@@ -83,8 +82,6 @@ static void percpu_setup_debug_store(int cpu)static void __init setup_cpu_entry_area (int cpu) {#ifdef CONFIG_X86_64 -extern char _entry_trampoline[]; -/* On 64-bit systems, we use a read-only fixmap GDT and TSS. */ pgprot_t gdt_prot =PAGE_KERNEL_RO ;pgprot_t tss_prot =PAGE_KERNEL_RO ;
@@ -146,43 +143,10 @@ static void __init setup_cpu_entry_area(int cpu)cea_map_percpu_pages (&get_cpu_entry_area (cpu)->exception_stacks, &per_cpu (exception_stacks, cpu),sizeof (exception_stacks) /PAGE_SIZE ,PAGE_KERNEL ); - -cea_set_pte (&get_cpu_entry_area (cpu)->entry_trampoline, -__pa_symbol (_entry_trampoline),PAGE_KERNEL_RX ); -/* -* The cpu_entry_area alias addresses are not in the kernel binary -* so they do not show up in /proc/kcore normally. This adds entries -* for them manually. -*/ -kclist_add_remap (&per_cpu (kcore_entry_trampoline, cpu), - _entry_trampoline, - &get_cpu_entry_area (cpu)->entry_trampoline,PAGE_SIZE );#endif percpu_setup_debug_store (cpu); } -#ifdef CONFIG_X86_64 -int arch_get_kallsym (unsigned int symnum,unsigned long *value,char *type, -char *name) -{ -unsigned int cpu, ncpu =0 ; - -if (symnum >=num_possible_cpus ()) -return -EINVAL ; - -for_each_possible_cpu (cpu) { -if (ncpu++ >= symnum) -break ; - } - - *value = (unsigned long )&get_cpu_entry_area (cpu)->entry_trampoline; - *type ='t' ; -strlcpy (name,"__entry_SYSCALL_64_trampoline" ,KSYM_NAME_LEN ); - -return 0 ; -} -#endif -static __init void setup_cpu_entry_area_ptes (void ) {#ifdef CONFIG_X86_32
arch/x86/mm/pti.c +32 -1
@@ -434,11 +434,42 @@ static void __init pti_clone_p4d(unsigned long addr) }/* -* Clone the CPU_ENTRY_AREA into the user space visible page table. +* Clone the CPU_ENTRY_AREA and associated data into the user space visible +* page table. */ static void __init pti_clone_user_shared (void ) { +unsigned cpu; +pti_clone_p4d (CPU_ENTRY_AREA_BASE ); + +for_each_possible_cpu (cpu) { +/* +* The SYSCALL64 entry code needs to be able to find the +* thread stack and needs one word of scratch space in which +* to spill a register. All of this lives in the TSS, in +* the sp1 and sp2 slots. +* +* This is done for all possible CPUs during boot to ensure +* that it's propagated to all mms. If we were to add one of +* these mappings during CPU hotplug, we would need to take +* some measure to make sure that every mm that subsequently +* ran on that CPU would have the relevant PGD entry in its +* pagetables. The usual vmalloc_fault() mechanism would not +* work for page faults taken in entry_SYSCALL_64 before RSP +* is set up. +*/ + +unsigned long va = (unsigned long )&per_cpu (cpu_tss_rw, cpu); +phys_addr_t pa =per_cpu_ptr_to_phys ((void *)va); +pte_t *target_pte; + + target_pte =pti_user_pagetable_walk_pte (va); +if (WARN_ON (!target_pte)) +return ; + + *target_pte =pfn_pte (pa >>PAGE_SHIFT ,PAGE_KERNEL ); + } }#else /* CONFIG_X86_64 */
tip-bot for Andy Lutomirski
merge bot[tip:x86/pti] x86/pti/64: Remove the SYSCALL64 entry trampoline
In reply to Andy Lutomirskix86/pti/64: Remove the SYSCALL64 entry trampoline
Commit-ID: 86635715ee4228ded59f662dab36e9732b9c978f Gitweb: https://git.kernel.org/tip/86635715ee4228ded59f662dab36e9732b9c978f Author: Andy Lutomirski <[email protected]> AuthorDate: Mon, 3 Sep 2018 15:59:44 -0700 Committer: Thomas Gleixner <[email protected]> CommitDate: Sat, 8 Sep 2018 11:53:16 +0200
x86/pti/64: Remove the SYSCALL64 entry trampoline
The SYSCALL64 trampoline has a couple of nice properties:
- The usual sequence of SWAPGS followed by two GS-relative accesses to set up RSP is somewhat slow because the GS-relative accesses need to wait for SWAPGS to finish. The trampoline approach allows RIP-relative accesses to set up RSP, which avoids the stall.
- The trampoline avoids any percpu access before CR3 is set up, which means that no percpu memory needs to be mapped in the user page tables. This prevents using Meltdown to read any percpu memory outside the cpu_entry_area and prevents using timing leaks to directly locate the percpu areas.
The downsides of using a trampoline may outweigh the upsides, however. It adds an extra non-contiguous I$ cache line to system calls, and it forces an indirect jump to transfer control back to the normal kernel text after CR3 is set up. The latter is because x86 lacks a 64-bit direct jump instruction that could jump from the trampoline to the entry text. With retpolines enabled, the indirect jump is extremely slow.
Change the code to map the percpu TSS into the user page tables to allow the non-trampoline SYSCALL64 path to work under PTI. This does not add a new direct information leak, since the TSS is readable by Meltdown from the cpu_entry_area alias regardless. It does allow a timing attack to locate the percpu area, but KASLR is more or less a lost cause against local attack on CPUs vulnerable to Meltdown regardless. As far as I'm concerned, on current hardware, KASLR is only useful to mitigate remote attacks that try to attack the kernel without first gaining RCE against a vulnerable user process.
On Skylake, with CONFIG_RETPOLINE=y and KPTI on, this reduces syscall overhead from ~237ns to ~228ns.
There is a possible alternative approach: Move the trampoline within 2G of the entry text and make a separate copy for each CPU. This would allow a direct jump to rejoin the normal entry path. There are pro's and con's for this approach:
+ It avoids a pipeline stall
- It executes from an extra page and read from another extra page during the syscall. The latter is because it needs to use a relative addressing mode to find sp1 -- it's the same *cacheline*, but accessed using an alias, so it's an extra TLB entry.
- Slightly more memory. This would be one page per CPU for a simple implementation and 64-ish bytes per CPU or one page per node for a more complex implementation.
- More code complexity.
The current approach is chosen for simplicity and because the alternative does not provide a significant benefit, which makes it worth.
[ tglx: Added the alternative discussion to the changelog ]
--- arch/x86/entry/entry_64.S | 69 +---------------------------------- arch/x86/include/asm/cpu_entry_area.h | 2 - arch/x86/include/asm/sections.h | 1 - arch/x86/kernel/asm-offsets.c | 2 - arch/x86/kernel/cpu/common.c | 11 +----- arch/x86/kernel/kprobes/core.c | 10 +---- arch/x86/kernel/vmlinux.lds.S | 10 ----- arch/x86/mm/cpu_entry_area.c | 36 ------------------ arch/x86/mm/pti.c | 33 ++++++++++++++++- 9 files changed, 36 insertions(+), 138 deletions(-)
arch/x86/entry/entry_64.S +2 -67
@@ -142,67 +142,6 @@ END(native_usergs_sysret64) * with them due to bugs in both AMD and Intel CPUs. */ - .pushsection .entry_trampoline, "ax" - -/* - * The code in here gets remapped into cpu_entry_area's trampoline. This means - * that the assembler and linker have the wrong idea as to where this code - * lives (and, in fact, it's mapped more than once, so it's not even at a - * fixed address). So we can't reference any symbols outside the entry - * trampoline and expect it to work. - * - * Instead, we carefully abuse %rip-relative addressing. - * _entry_trampoline(%rip) refers to the start of the remapped) entry - * trampoline. We can thus find cpu_entry_area with this macro: - */ - -#define CPU_ENTRY_AREA \ - _entry_trampoline - CPU_ENTRY_AREA_entry_trampoline(%rip) - -/* The top word of the SYSENTER stack is hot and is usable as scratch space. */ -#define RSP_SCRATCH CPU_ENTRY_AREA_entry_stack + \ - SIZEOF_entry_stack - 8 + CPU_ENTRY_AREA - -ENTRY(entry_SYSCALL_64_trampoline) - UNWIND_HINT_EMPTY - swapgs - - /* Stash the user RSP. */ - movq %rsp, RSP_SCRATCH - - /* Note: using %rsp as a scratch reg. */ - SWITCH_TO_KERNEL_CR3 scratch_reg=%rsp - - /* Load the top of the task stack into RSP */ - movq CPU_ENTRY_AREA_tss + TSS_sp1 + CPU_ENTRY_AREA, %rsp - - /* Start building the simulated IRET frame. */ - pushq $__USER_DS /* pt_regs->ss */ - pushq RSP_SCRATCH /* pt_regs->sp */ - pushq %r11 /* pt_regs->flags */ - pushq $__USER_CS /* pt_regs->cs */ - pushq %rcx /* pt_regs->ip */ - - /* - * x86 lacks a near absolute jump, and we can't jump to the real - * entry text with a relative jump. We could push the target - * address and then use retq, but this destroys the pipeline on - * many CPUs (wasting over 20 cycles on Sandy Bridge). Instead, - * spill RDI and restore it in a second-stage trampoline. - */ - pushq %rdi - movq $entry_SYSCALL_64_stage2, %rdi - JMP_NOSPEC %rdi -END(entry_SYSCALL_64_trampoline) - - .popsection - -ENTRY(entry_SYSCALL_64_stage2) - UNWIND_HINT_EMPTY - popq %rdi - jmp entry_SYSCALL_64_after_hwframe -END(entry_SYSCALL_64_stage2) - ENTRY(entry_SYSCALL_64) UNWIND_HINT_EMPTY /*
@@ -212,13 +151,9 @@ ENTRY(entry_SYSCALL_64) */ swapgs - /* - * This path is only taken when PAGE_TABLE_ISOLATION is disabled so it - * is not required to switch CR3. - * - * tss.sp2 is scratch space. - */ + /* tss.sp2 is scratch space. */ movq %rsp, PER_CPU_VAR(cpu_tss_rw + TSS_sp2) + SWITCH_TO_KERNEL_CR3 scratch_reg=%rsp movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp /* Construct struct pt_regs on stack */
arch/x86/include/asm/cpu_entry_area.h +0 -2
@@ -30,8 +30,6 @@ struct cpu_entry_area {*/ struct tss_struct tss; -char entry_trampoline[PAGE_SIZE ]; -#ifdef CONFIG_X86_64 /* * Exception stacks used for IST entries.
arch/x86/include/asm/sections.h +0 -1
@@ -11,7 +11,6 @@ extern char __end_rodata_aligned[];#if defined(CONFIG_X86_64 )extern char __end_rodata_hpage_align[]; -extern char __entry_trampoline_start[], __entry_trampoline_end[];#endif #endif /* _ASM_X86_SECTIONS_H */
arch/x86/kernel/asm-offsets.c +0 -2
@@ -99,8 +99,6 @@ void common(void) {OFFSET (TLB_STATE_user_pcid_flush_mask, tlb_state, user_pcid_flush_mask);/* Layout info for cpu_entry_area */ -OFFSET (CPU_ENTRY_AREA_tss, cpu_entry_area, tss); -OFFSET (CPU_ENTRY_AREA_entry_trampoline, cpu_entry_area, entry_trampoline);OFFSET (CPU_ENTRY_AREA_entry_stack, cpu_entry_area, entry_stack_page);DEFINE (SIZEOF_entry_stack,sizeof (struct entry_stack ));DEFINE (MASK_entry_stack, (~(sizeof (struct entry_stack ) -1 )));
arch/x86/kernel/cpu/common.c +1 -10
@@ -1531,19 +1531,10 @@ EXPORT_PER_CPU_SYMBOL(__preempt_count);/* May not be marked __init: used by software suspend */ void syscall_init (void ) { -extern char _entry_trampoline[]; -extern char entry_SYSCALL_64_trampoline[]; -int cpu =smp_processor_id (); -unsigned long SYSCALL64_entry_trampoline = - (unsigned long )get_cpu_entry_area (cpu)->entry_trampoline + - (entry_SYSCALL_64_trampoline - _entry_trampoline);wrmsr (MSR_STAR ,0 , (__USER32_CS <<16 ) | __KERNEL_CS); -if (static_cpu_has (X86_FEATURE_PTI )) -wrmsrl (MSR_LSTAR , SYSCALL64_entry_trampoline); -else -wrmsrl (MSR_LSTAR , (unsigned long )entry_SYSCALL_64); +wrmsrl (MSR_LSTAR , (unsigned long )entry_SYSCALL_64);#ifdef CONFIG_IA32_EMULATION wrmsrl (MSR_CSTAR , (unsigned long )entry_SYSCALL_compat);
arch/x86/kernel/kprobes/core.c +1 -9
@@ -1066,18 +1066,10 @@ NOKPROBE_SYMBOL(kprobe_exceptions_notify);bool arch_within_kprobe_blacklist (unsigned long addr) { -bool is_in_entry_trampoline_section = false; - -#ifdef CONFIG_X86_64 - is_in_entry_trampoline_section = - (addr >= (unsigned long )__entry_trampoline_start && - addr < (unsigned long )__entry_trampoline_end); -#endif return (addr >= (unsigned long )__kprobes_text_start && addr < (unsigned long )__kprobes_text_end) || (addr >= (unsigned long )__entry_text_start && - addr < (unsigned long )__entry_text_end) || - is_in_entry_trampoline_section; + addr < (unsigned long )__entry_text_end); }int __init arch_init_kprobes (void )
arch/x86/kernel/vmlinux.lds.S +0 -10
@@ -118,16 +118,6 @@ SECTIONS *(.fixup) *(.gnu.warning) -#ifdef CONFIG_X86_64 - . = ALIGN(PAGE_SIZE); - __entry_trampoline_start = .; - _entry_trampoline = .; - *(.entry_trampoline) - . = ALIGN(PAGE_SIZE); - __entry_trampoline_end = .; - ASSERT(. - _entry_trampoline == PAGE_SIZE, "entry trampoline is too big"); -#endif - #ifdef CONFIG_RETPOLINE __indirect_thunk_start = .; *(.text.__x86.indirect_thunk)
arch/x86/mm/cpu_entry_area.c +0 -36
@@ -15,7 +15,6 @@ static DEFINE_PER_CPU_PAGE_ALIGNED(struct entry_stack_page, entry_stack_storage)#ifdef CONFIG_X86_64 static DEFINE_PER_CPU_PAGE_ALIGNED(char, exception_stacks [(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + DEBUG_STKSZ]); -static DEFINE_PER_CPU(struct kcore_list, kcore_entry_trampoline);#endif struct cpu_entry_area *get_cpu_entry_area (int cpu)
@@ -83,8 +82,6 @@ static void percpu_setup_debug_store(int cpu)static void __init setup_cpu_entry_area (int cpu) {#ifdef CONFIG_X86_64 -extern char _entry_trampoline[]; -/* On 64-bit systems, we use a read-only fixmap GDT and TSS. */ pgprot_t gdt_prot =PAGE_KERNEL_RO ;pgprot_t tss_prot =PAGE_KERNEL_RO ;
@@ -146,43 +143,10 @@ static void __init setup_cpu_entry_area(int cpu)cea_map_percpu_pages (&get_cpu_entry_area (cpu)->exception_stacks, &per_cpu (exception_stacks, cpu),sizeof (exception_stacks) /PAGE_SIZE ,PAGE_KERNEL ); - -cea_set_pte (&get_cpu_entry_area (cpu)->entry_trampoline, -__pa_symbol (_entry_trampoline),PAGE_KERNEL_RX ); -/* -* The cpu_entry_area alias addresses are not in the kernel binary -* so they do not show up in /proc/kcore normally. This adds entries -* for them manually. -*/ -kclist_add_remap (&per_cpu (kcore_entry_trampoline, cpu), - _entry_trampoline, - &get_cpu_entry_area (cpu)->entry_trampoline,PAGE_SIZE );#endif percpu_setup_debug_store (cpu); } -#ifdef CONFIG_X86_64 -int arch_get_kallsym (unsigned int symnum,unsigned long *value,char *type, -char *name) -{ -unsigned int cpu, ncpu =0 ; - -if (symnum >=num_possible_cpus ()) -return -EINVAL ; - -for_each_possible_cpu (cpu) { -if (ncpu++ >= symnum) -break ; - } - - *value = (unsigned long )&get_cpu_entry_area (cpu)->entry_trampoline; - *type ='t' ; -strlcpy (name,"__entry_SYSCALL_64_trampoline" ,KSYM_NAME_LEN ); - -return 0 ; -} -#endif -static __init void setup_cpu_entry_area_ptes (void ) {#ifdef CONFIG_X86_32
arch/x86/mm/pti.c +32 -1
@@ -434,11 +434,42 @@ static void __init pti_clone_p4d(unsigned long addr) }/* -* Clone the CPU_ENTRY_AREA into the user space visible page table. +* Clone the CPU_ENTRY_AREA and associated data into the user space visible +* page table. */ static void __init pti_clone_user_shared (void ) { +unsigned int cpu; +pti_clone_p4d (CPU_ENTRY_AREA_BASE ); + +for_each_possible_cpu (cpu) { +/* +* The SYSCALL64 entry code needs to be able to find the +* thread stack and needs one word of scratch space in which +* to spill a register. All of this lives in the TSS, in +* the sp1 and sp2 slots. +* +* This is done for all possible CPUs during boot to ensure +* that it's propagated to all mms. If we were to add one of +* these mappings during CPU hotplug, we would need to take +* some measure to make sure that every mm that subsequently +* ran on that CPU would have the relevant PGD entry in its +* pagetables. The usual vmalloc_fault() mechanism would not +* work for page faults taken in entry_SYSCALL_64 before RSP +* is set up. +*/ + +unsigned long va = (unsigned long )&per_cpu (cpu_tss_rw, cpu); +phys_addr_t pa =per_cpu_ptr_to_phys ((void *)va); +pte_t *target_pte; + + target_pte =pti_user_pagetable_walk_pte (va); +if (WARN_ON (!target_pte)) +return ; + + *target_pte =pfn_pte (pa >>PAGE_SHIFT ,PAGE_KERNEL ); + } }#else /* CONFIG_X86_64 */
tip-bot for Andy Lutomirski
merge bot[tip:x86/pti] x86/pti/64: Remove the SYSCALL64 entry trampoline
In reply to Andy Lutomirskix86/pti/64: Remove the SYSCALL64 entry trampoline
Commit-ID: e536a56190d412b0f98dbd4dde608f9f7081bb6d Gitweb: https://git.kernel.org/tip/e536a56190d412b0f98dbd4dde608f9f7081bb6d Author: Andy Lutomirski <[email protected]> AuthorDate: Mon, 3 Sep 2018 15:59:44 -0700 Committer: Thomas Gleixner <[email protected]> CommitDate: Wed, 12 Sep 2018 21:29:09 +0200
x86/pti/64: Remove the SYSCALL64 entry trampoline
The SYSCALL64 trampoline has a couple of nice properties:
- The usual sequence of SWAPGS followed by two GS-relative accesses to set up RSP is somewhat slow because the GS-relative accesses need to wait for SWAPGS to finish. The trampoline approach allows RIP-relative accesses to set up RSP, which avoids the stall.
- The trampoline avoids any percpu access before CR3 is set up, which means that no percpu memory needs to be mapped in the user page tables. This prevents using Meltdown to read any percpu memory outside the cpu_entry_area and prevents using timing leaks to directly locate the percpu areas.
The downsides of using a trampoline may outweigh the upsides, however. It adds an extra non-contiguous I$ cache line to system calls, and it forces an indirect jump to transfer control back to the normal kernel text after CR3 is set up. The latter is because x86 lacks a 64-bit direct jump instruction that could jump from the trampoline to the entry text. With retpolines enabled, the indirect jump is extremely slow.
Change the code to map the percpu TSS into the user page tables to allow the non-trampoline SYSCALL64 path to work under PTI. This does not add a new direct information leak, since the TSS is readable by Meltdown from the cpu_entry_area alias regardless. It does allow a timing attack to locate the percpu area, but KASLR is more or less a lost cause against local attack on CPUs vulnerable to Meltdown regardless. As far as I'm concerned, on current hardware, KASLR is only useful to mitigate remote attacks that try to attack the kernel without first gaining RCE against a vulnerable user process.
On Skylake, with CONFIG_RETPOLINE=y and KPTI on, this reduces syscall overhead from ~237ns to ~228ns.
There is a possible alternative approach: Move the trampoline within 2G of the entry text and make a separate copy for each CPU. This would allow a direct jump to rejoin the normal entry path. There are pro's and con's for this approach:
+ It avoids a pipeline stall
- It executes from an extra page and read from another extra page during the syscall. The latter is because it needs to use a relative addressing mode to find sp1 -- it's the same *cacheline*, but accessed using an alias, so it's an extra TLB entry.
- Slightly more memory. This would be one page per CPU for a simple implementation and 64-ish bytes per CPU or one page per node for a more complex implementation.
- More code complexity.
The current approach is chosen for simplicity and because the alternative does not provide a significant benefit, which makes it worth.
[ tglx: Added the alternative discussion to the changelog ]
--- arch/x86/entry/entry_64.S | 69 +---------------------------------- arch/x86/include/asm/cpu_entry_area.h | 2 - arch/x86/include/asm/sections.h | 1 - arch/x86/kernel/asm-offsets.c | 2 - arch/x86/kernel/cpu/common.c | 13 +------ arch/x86/kernel/kprobes/core.c | 10 +---- arch/x86/kernel/vmlinux.lds.S | 10 ----- arch/x86/mm/cpu_entry_area.c | 36 ------------------ arch/x86/mm/pti.c | 33 ++++++++++++++++- 9 files changed, 36 insertions(+), 140 deletions(-)
arch/x86/entry/entry_64.S +2 -67
@@ -142,67 +142,6 @@ END(native_usergs_sysret64) * with them due to bugs in both AMD and Intel CPUs. */ - .pushsection .entry_trampoline, "ax" - -/* - * The code in here gets remapped into cpu_entry_area's trampoline. This means - * that the assembler and linker have the wrong idea as to where this code - * lives (and, in fact, it's mapped more than once, so it's not even at a - * fixed address). So we can't reference any symbols outside the entry - * trampoline and expect it to work. - * - * Instead, we carefully abuse %rip-relative addressing. - * _entry_trampoline(%rip) refers to the start of the remapped) entry - * trampoline. We can thus find cpu_entry_area with this macro: - */ - -#define CPU_ENTRY_AREA \ - _entry_trampoline - CPU_ENTRY_AREA_entry_trampoline(%rip) - -/* The top word of the SYSENTER stack is hot and is usable as scratch space. */ -#define RSP_SCRATCH CPU_ENTRY_AREA_entry_stack + \ - SIZEOF_entry_stack - 8 + CPU_ENTRY_AREA - -ENTRY(entry_SYSCALL_64_trampoline) - UNWIND_HINT_EMPTY - swapgs - - /* Stash the user RSP. */ - movq %rsp, RSP_SCRATCH - - /* Note: using %rsp as a scratch reg. */ - SWITCH_TO_KERNEL_CR3 scratch_reg=%rsp - - /* Load the top of the task stack into RSP */ - movq CPU_ENTRY_AREA_tss + TSS_sp1 + CPU_ENTRY_AREA, %rsp - - /* Start building the simulated IRET frame. */ - pushq $__USER_DS /* pt_regs->ss */ - pushq RSP_SCRATCH /* pt_regs->sp */ - pushq %r11 /* pt_regs->flags */ - pushq $__USER_CS /* pt_regs->cs */ - pushq %rcx /* pt_regs->ip */ - - /* - * x86 lacks a near absolute jump, and we can't jump to the real - * entry text with a relative jump. We could push the target - * address and then use retq, but this destroys the pipeline on - * many CPUs (wasting over 20 cycles on Sandy Bridge). Instead, - * spill RDI and restore it in a second-stage trampoline. - */ - pushq %rdi - movq $entry_SYSCALL_64_stage2, %rdi - JMP_NOSPEC %rdi -END(entry_SYSCALL_64_trampoline) - - .popsection - -ENTRY(entry_SYSCALL_64_stage2) - UNWIND_HINT_EMPTY - popq %rdi - jmp entry_SYSCALL_64_after_hwframe -END(entry_SYSCALL_64_stage2) - ENTRY(entry_SYSCALL_64) UNWIND_HINT_EMPTY /*
@@ -212,13 +151,9 @@ ENTRY(entry_SYSCALL_64) */ swapgs - /* - * This path is only taken when PAGE_TABLE_ISOLATION is disabled so it - * is not required to switch CR3. - * - * tss.sp2 is scratch space. - */ + /* tss.sp2 is scratch space. */ movq %rsp, PER_CPU_VAR(cpu_tss_rw + TSS_sp2) + SWITCH_TO_KERNEL_CR3 scratch_reg=%rsp movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp /* Construct struct pt_regs on stack */
arch/x86/include/asm/cpu_entry_area.h +0 -2
@@ -30,8 +30,6 @@ struct cpu_entry_area {*/ struct tss_struct tss; -char entry_trampoline[PAGE_SIZE ]; -#ifdef CONFIG_X86_64 /* * Exception stacks used for IST entries.
arch/x86/include/asm/sections.h +0 -1
@@ -11,7 +11,6 @@ extern char __end_rodata_aligned[];#if defined(CONFIG_X86_64 )extern char __end_rodata_hpage_align[]; -extern char __entry_trampoline_start[], __entry_trampoline_end[];#endif #endif /* _ASM_X86_SECTIONS_H */
arch/x86/kernel/asm-offsets.c +0 -2
@@ -99,8 +99,6 @@ void common(void) {OFFSET (TLB_STATE_user_pcid_flush_mask, tlb_state, user_pcid_flush_mask);/* Layout info for cpu_entry_area */ -OFFSET (CPU_ENTRY_AREA_tss, cpu_entry_area, tss); -OFFSET (CPU_ENTRY_AREA_entry_trampoline, cpu_entry_area, entry_trampoline);OFFSET (CPU_ENTRY_AREA_entry_stack, cpu_entry_area, entry_stack_page);DEFINE (SIZEOF_entry_stack,sizeof (struct entry_stack ));DEFINE (MASK_entry_stack, (~(sizeof (struct entry_stack ) -1 )));
arch/x86/kernel/cpu/common.c +1 -12
@@ -1531,19 +1531,8 @@ EXPORT_PER_CPU_SYMBOL(__preempt_count);/* May not be marked __init: used by software suspend */ void syscall_init (void ) { -extern char _entry_trampoline[]; -extern char entry_SYSCALL_64_trampoline[]; - -int cpu =smp_processor_id (); -unsigned long SYSCALL64_entry_trampoline = - (unsigned long )get_cpu_entry_area (cpu)->entry_trampoline + - (entry_SYSCALL_64_trampoline - _entry_trampoline); -wrmsr (MSR_STAR ,0 , (__USER32_CS <<16 ) | __KERNEL_CS); -if (static_cpu_has (X86_FEATURE_PTI )) -wrmsrl (MSR_LSTAR , SYSCALL64_entry_trampoline); -else -wrmsrl (MSR_LSTAR , (unsigned long )entry_SYSCALL_64); +wrmsrl (MSR_LSTAR , (unsigned long )entry_SYSCALL_64);#ifdef CONFIG_IA32_EMULATION wrmsrl (MSR_CSTAR , (unsigned long )entry_SYSCALL_compat);
arch/x86/kernel/kprobes/core.c +1 -9
@@ -1066,18 +1066,10 @@ NOKPROBE_SYMBOL(kprobe_exceptions_notify);bool arch_within_kprobe_blacklist (unsigned long addr) { -bool is_in_entry_trampoline_section = false; - -#ifdef CONFIG_X86_64 - is_in_entry_trampoline_section = - (addr >= (unsigned long )__entry_trampoline_start && - addr < (unsigned long )__entry_trampoline_end); -#endif return (addr >= (unsigned long )__kprobes_text_start && addr < (unsigned long )__kprobes_text_end) || (addr >= (unsigned long )__entry_text_start && - addr < (unsigned long )__entry_text_end) || - is_in_entry_trampoline_section; + addr < (unsigned long )__entry_text_end); }int __init arch_init_kprobes (void )
arch/x86/kernel/vmlinux.lds.S +0 -10
@@ -118,16 +118,6 @@ SECTIONS *(.fixup) *(.gnu.warning) -#ifdef CONFIG_X86_64 - . = ALIGN(PAGE_SIZE); - __entry_trampoline_start = .; - _entry_trampoline = .; - *(.entry_trampoline) - . = ALIGN(PAGE_SIZE); - __entry_trampoline_end = .; - ASSERT(. - _entry_trampoline == PAGE_SIZE, "entry trampoline is too big"); -#endif - #ifdef CONFIG_RETPOLINE __indirect_thunk_start = .; *(.text.__x86.indirect_thunk)
arch/x86/mm/cpu_entry_area.c +0 -36
@@ -15,7 +15,6 @@ static DEFINE_PER_CPU_PAGE_ALIGNED(struct entry_stack_page, entry_stack_storage)#ifdef CONFIG_X86_64 static DEFINE_PER_CPU_PAGE_ALIGNED(char, exception_stacks [(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + DEBUG_STKSZ]); -static DEFINE_PER_CPU(struct kcore_list, kcore_entry_trampoline);#endif struct cpu_entry_area *get_cpu_entry_area (int cpu)
@@ -83,8 +82,6 @@ static void percpu_setup_debug_store(int cpu)static void __init setup_cpu_entry_area (int cpu) {#ifdef CONFIG_X86_64 -extern char _entry_trampoline[]; -/* On 64-bit systems, we use a read-only fixmap GDT and TSS. */ pgprot_t gdt_prot =PAGE_KERNEL_RO ;pgprot_t tss_prot =PAGE_KERNEL_RO ;
@@ -146,43 +143,10 @@ static void __init setup_cpu_entry_area(int cpu)cea_map_percpu_pages (&get_cpu_entry_area (cpu)->exception_stacks, &per_cpu (exception_stacks, cpu),sizeof (exception_stacks) /PAGE_SIZE ,PAGE_KERNEL ); - -cea_set_pte (&get_cpu_entry_area (cpu)->entry_trampoline, -__pa_symbol (_entry_trampoline),PAGE_KERNEL_RX ); -/* -* The cpu_entry_area alias addresses are not in the kernel binary -* so they do not show up in /proc/kcore normally. This adds entries -* for them manually. -*/ -kclist_add_remap (&per_cpu (kcore_entry_trampoline, cpu), - _entry_trampoline, - &get_cpu_entry_area (cpu)->entry_trampoline,PAGE_SIZE );#endif percpu_setup_debug_store (cpu); } -#ifdef CONFIG_X86_64 -int arch_get_kallsym (unsigned int symnum,unsigned long *value,char *type, -char *name) -{ -unsigned int cpu, ncpu =0 ; - -if (symnum >=num_possible_cpus ()) -return -EINVAL ; - -for_each_possible_cpu (cpu) { -if (ncpu++ >= symnum) -break ; - } - - *value = (unsigned long )&get_cpu_entry_area (cpu)->entry_trampoline; - *type ='t' ; -strlcpy (name,"__entry_SYSCALL_64_trampoline" ,KSYM_NAME_LEN ); - -return 0 ; -} -#endif -static __init void setup_cpu_entry_area_ptes (void ) {#ifdef CONFIG_X86_32
arch/x86/mm/pti.c +32 -1
@@ -434,11 +434,42 @@ static void __init pti_clone_p4d(unsigned long addr) }/* -* Clone the CPU_ENTRY_AREA into the user space visible page table. +* Clone the CPU_ENTRY_AREA and associated data into the user space visible +* page table. */ static void __init pti_clone_user_shared (void ) { +unsigned int cpu; +pti_clone_p4d (CPU_ENTRY_AREA_BASE ); + +for_each_possible_cpu (cpu) { +/* +* The SYSCALL64 entry code needs to be able to find the +* thread stack and needs one word of scratch space in which +* to spill a register. All of this lives in the TSS, in +* the sp1 and sp2 slots. +* +* This is done for all possible CPUs during boot to ensure +* that it's propagated to all mms. If we were to add one of +* these mappings during CPU hotplug, we would need to take +* some measure to make sure that every mm that subsequently +* ran on that CPU would have the relevant PGD entry in its +* pagetables. The usual vmalloc_fault() mechanism would not +* work for page faults taken in entry_SYSCALL_64 before RSP +* is set up. +*/ + +unsigned long va = (unsigned long )&per_cpu (cpu_tss_rw, cpu); +phys_addr_t pa =per_cpu_ptr_to_phys ((void *)va); +pte_t *target_pte; + + target_pte =pti_user_pagetable_walk_pte (va); +if (WARN_ON (!target_pte)) +return ; + + *target_pte =pfn_pte (pa >>PAGE_SHIFT ,PAGE_KERNEL ); + } }#else /* CONFIG_X86_64 */
tip-bot for Andy Lutomirski
merge bot[tip:x86/pti] x86/pti/64: Remove the SYSCALL64 entry trampoline
In reply to Andy Lutomirskix86/pti/64: Remove the SYSCALL64 entry trampoline
Commit-ID: bf904d2762ee6fc1e4acfcb0772bbfb4a27ad8a6 Gitweb: https://git.kernel.org/tip/bf904d2762ee6fc1e4acfcb0772bbfb4a27ad8a6 Author: Andy Lutomirski <[email protected]> AuthorDate: Mon, 3 Sep 2018 15:59:44 -0700 Committer: Thomas Gleixner <[email protected]> CommitDate: Wed, 12 Sep 2018 21:33:53 +0200
x86/pti/64: Remove the SYSCALL64 entry trampoline
The SYSCALL64 trampoline has a couple of nice properties:
- The usual sequence of SWAPGS followed by two GS-relative accesses to set up RSP is somewhat slow because the GS-relative accesses need to wait for SWAPGS to finish. The trampoline approach allows RIP-relative accesses to set up RSP, which avoids the stall.
- The trampoline avoids any percpu access before CR3 is set up, which means that no percpu memory needs to be mapped in the user page tables. This prevents using Meltdown to read any percpu memory outside the cpu_entry_area and prevents using timing leaks to directly locate the percpu areas.
The downsides of using a trampoline may outweigh the upsides, however. It adds an extra non-contiguous I$ cache line to system calls, and it forces an indirect jump to transfer control back to the normal kernel text after CR3 is set up. The latter is because x86 lacks a 64-bit direct jump instruction that could jump from the trampoline to the entry text. With retpolines enabled, the indirect jump is extremely slow.
Change the code to map the percpu TSS into the user page tables to allow the non-trampoline SYSCALL64 path to work under PTI. This does not add a new direct information leak, since the TSS is readable by Meltdown from the cpu_entry_area alias regardless. It does allow a timing attack to locate the percpu area, but KASLR is more or less a lost cause against local attack on CPUs vulnerable to Meltdown regardless. As far as I'm concerned, on current hardware, KASLR is only useful to mitigate remote attacks that try to attack the kernel without first gaining RCE against a vulnerable user process.
On Skylake, with CONFIG_RETPOLINE=y and KPTI on, this reduces syscall overhead from ~237ns to ~228ns.
There is a possible alternative approach: Move the trampoline within 2G of the entry text and make a separate copy for each CPU. This would allow a direct jump to rejoin the normal entry path. There are pro's and con's for this approach:
+ It avoids a pipeline stall
- It executes from an extra page and read from another extra page during the syscall. The latter is because it needs to use a relative addressing mode to find sp1 -- it's the same *cacheline*, but accessed using an alias, so it's an extra TLB entry.
- Slightly more memory. This would be one page per CPU for a simple implementation and 64-ish bytes per CPU or one page per node for a more complex implementation.
- More code complexity.
The current approach is chosen for simplicity and because the alternative does not provide a significant benefit, which makes it worth.
[ tglx: Added the alternative discussion to the changelog ]
--- arch/x86/entry/entry_64.S | 69 +---------------------------------- arch/x86/include/asm/cpu_entry_area.h | 2 - arch/x86/include/asm/sections.h | 1 - arch/x86/kernel/asm-offsets.c | 2 - arch/x86/kernel/cpu/common.c | 13 +------ arch/x86/kernel/kprobes/core.c | 10 +---- arch/x86/kernel/vmlinux.lds.S | 10 ----- arch/x86/mm/cpu_entry_area.c | 36 ------------------ arch/x86/mm/pti.c | 33 ++++++++++++++++- 9 files changed, 37 insertions(+), 139 deletions(-)
arch/x86/entry/entry_64.S +2 -67
@@ -142,67 +142,6 @@ END(native_usergs_sysret64) * with them due to bugs in both AMD and Intel CPUs. */ - .pushsection .entry_trampoline, "ax" - -/* - * The code in here gets remapped into cpu_entry_area's trampoline. This means - * that the assembler and linker have the wrong idea as to where this code - * lives (and, in fact, it's mapped more than once, so it's not even at a - * fixed address). So we can't reference any symbols outside the entry - * trampoline and expect it to work. - * - * Instead, we carefully abuse %rip-relative addressing. - * _entry_trampoline(%rip) refers to the start of the remapped) entry - * trampoline. We can thus find cpu_entry_area with this macro: - */ - -#define CPU_ENTRY_AREA \ - _entry_trampoline - CPU_ENTRY_AREA_entry_trampoline(%rip) - -/* The top word of the SYSENTER stack is hot and is usable as scratch space. */ -#define RSP_SCRATCH CPU_ENTRY_AREA_entry_stack + \ - SIZEOF_entry_stack - 8 + CPU_ENTRY_AREA - -ENTRY(entry_SYSCALL_64_trampoline) - UNWIND_HINT_EMPTY - swapgs - - /* Stash the user RSP. */ - movq %rsp, RSP_SCRATCH - - /* Note: using %rsp as a scratch reg. */ - SWITCH_TO_KERNEL_CR3 scratch_reg=%rsp - - /* Load the top of the task stack into RSP */ - movq CPU_ENTRY_AREA_tss + TSS_sp1 + CPU_ENTRY_AREA, %rsp - - /* Start building the simulated IRET frame. */ - pushq $__USER_DS /* pt_regs->ss */ - pushq RSP_SCRATCH /* pt_regs->sp */ - pushq %r11 /* pt_regs->flags */ - pushq $__USER_CS /* pt_regs->cs */ - pushq %rcx /* pt_regs->ip */ - - /* - * x86 lacks a near absolute jump, and we can't jump to the real - * entry text with a relative jump. We could push the target - * address and then use retq, but this destroys the pipeline on - * many CPUs (wasting over 20 cycles on Sandy Bridge). Instead, - * spill RDI and restore it in a second-stage trampoline. - */ - pushq %rdi - movq $entry_SYSCALL_64_stage2, %rdi - JMP_NOSPEC %rdi -END(entry_SYSCALL_64_trampoline) - - .popsection - -ENTRY(entry_SYSCALL_64_stage2) - UNWIND_HINT_EMPTY - popq %rdi - jmp entry_SYSCALL_64_after_hwframe -END(entry_SYSCALL_64_stage2) - ENTRY(entry_SYSCALL_64) UNWIND_HINT_EMPTY /*
@@ -212,13 +151,9 @@ ENTRY(entry_SYSCALL_64) */ swapgs - /* - * This path is only taken when PAGE_TABLE_ISOLATION is disabled so it - * is not required to switch CR3. - * - * tss.sp2 is scratch space. - */ + /* tss.sp2 is scratch space. */ movq %rsp, PER_CPU_VAR(cpu_tss_rw + TSS_sp2) + SWITCH_TO_KERNEL_CR3 scratch_reg=%rsp movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp /* Construct struct pt_regs on stack */
arch/x86/include/asm/cpu_entry_area.h +0 -2
@@ -30,8 +30,6 @@ struct cpu_entry_area {*/ struct tss_struct tss; -char entry_trampoline[PAGE_SIZE ]; -#ifdef CONFIG_X86_64 /* * Exception stacks used for IST entries.
arch/x86/include/asm/sections.h +0 -1
@@ -11,7 +11,6 @@ extern char __end_rodata_aligned[];#if defined(CONFIG_X86_64 )extern char __end_rodata_hpage_align[]; -extern char __entry_trampoline_start[], __entry_trampoline_end[];#endif #endif /* _ASM_X86_SECTIONS_H */
arch/x86/kernel/asm-offsets.c +0 -2
@@ -99,8 +99,6 @@ void common(void) {OFFSET (TLB_STATE_user_pcid_flush_mask, tlb_state, user_pcid_flush_mask);/* Layout info for cpu_entry_area */ -OFFSET (CPU_ENTRY_AREA_tss, cpu_entry_area, tss); -OFFSET (CPU_ENTRY_AREA_entry_trampoline, cpu_entry_area, entry_trampoline);OFFSET (CPU_ENTRY_AREA_entry_stack, cpu_entry_area, entry_stack_page);DEFINE (SIZEOF_entry_stack,sizeof (struct entry_stack ));DEFINE (MASK_entry_stack, (~(sizeof (struct entry_stack ) -1 )));
arch/x86/kernel/cpu/common.c +2 -11
@@ -1531,19 +1531,10 @@ EXPORT_PER_CPU_SYMBOL(__preempt_count);/* May not be marked __init: used by software suspend */ void syscall_init (void ) { -extern char _entry_trampoline[]; -extern char entry_SYSCALL_64_trampoline[]; - -int cpu =smp_processor_id (); -unsigned long SYSCALL64_entry_trampoline = - (unsigned long )get_cpu_entry_area (cpu)->entry_trampoline + - (entry_SYSCALL_64_trampoline - _entry_trampoline); +int __maybe_unused cpu =smp_processor_id ();wrmsr (MSR_STAR ,0 , (__USER32_CS <<16 ) | __KERNEL_CS); -if (static_cpu_has (X86_FEATURE_PTI )) -wrmsrl (MSR_LSTAR , SYSCALL64_entry_trampoline); -else -wrmsrl (MSR_LSTAR , (unsigned long )entry_SYSCALL_64); +wrmsrl (MSR_LSTAR , (unsigned long )entry_SYSCALL_64);#ifdef CONFIG_IA32_EMULATION wrmsrl (MSR_CSTAR , (unsigned long )entry_SYSCALL_compat);
arch/x86/kernel/kprobes/core.c +1 -9
@@ -1066,18 +1066,10 @@ NOKPROBE_SYMBOL(kprobe_exceptions_notify);bool arch_within_kprobe_blacklist (unsigned long addr) { -bool is_in_entry_trampoline_section = false; - -#ifdef CONFIG_X86_64 - is_in_entry_trampoline_section = - (addr >= (unsigned long )__entry_trampoline_start && - addr < (unsigned long )__entry_trampoline_end); -#endif return (addr >= (unsigned long )__kprobes_text_start && addr < (unsigned long )__kprobes_text_end) || (addr >= (unsigned long )__entry_text_start && - addr < (unsigned long )__entry_text_end) || - is_in_entry_trampoline_section; + addr < (unsigned long )__entry_text_end); }int __init arch_init_kprobes (void )
arch/x86/kernel/vmlinux.lds.S +0 -10
@@ -118,16 +118,6 @@ SECTIONS *(.fixup) *(.gnu.warning) -#ifdef CONFIG_X86_64 - . = ALIGN(PAGE_SIZE); - __entry_trampoline_start = .; - _entry_trampoline = .; - *(.entry_trampoline) - . = ALIGN(PAGE_SIZE); - __entry_trampoline_end = .; - ASSERT(. - _entry_trampoline == PAGE_SIZE, "entry trampoline is too big"); -#endif - #ifdef CONFIG_RETPOLINE __indirect_thunk_start = .; *(.text.__x86.indirect_thunk)
arch/x86/mm/cpu_entry_area.c +0 -36
@@ -15,7 +15,6 @@ static DEFINE_PER_CPU_PAGE_ALIGNED(struct entry_stack_page, entry_stack_storage)#ifdef CONFIG_X86_64 static DEFINE_PER_CPU_PAGE_ALIGNED(char, exception_stacks [(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + DEBUG_STKSZ]); -static DEFINE_PER_CPU(struct kcore_list, kcore_entry_trampoline);#endif struct cpu_entry_area *get_cpu_entry_area (int cpu)
@@ -83,8 +82,6 @@ static void percpu_setup_debug_store(int cpu)static void __init setup_cpu_entry_area (int cpu) {#ifdef CONFIG_X86_64 -extern char _entry_trampoline[]; -/* On 64-bit systems, we use a read-only fixmap GDT and TSS. */ pgprot_t gdt_prot =PAGE_KERNEL_RO ;pgprot_t tss_prot =PAGE_KERNEL_RO ;
@@ -146,43 +143,10 @@ static void __init setup_cpu_entry_area(int cpu)cea_map_percpu_pages (&get_cpu_entry_area (cpu)->exception_stacks, &per_cpu (exception_stacks, cpu),sizeof (exception_stacks) /PAGE_SIZE ,PAGE_KERNEL ); - -cea_set_pte (&get_cpu_entry_area (cpu)->entry_trampoline, -__pa_symbol (_entry_trampoline),PAGE_KERNEL_RX ); -/* -* The cpu_entry_area alias addresses are not in the kernel binary -* so they do not show up in /proc/kcore normally. This adds entries -* for them manually. -*/ -kclist_add_remap (&per_cpu (kcore_entry_trampoline, cpu), - _entry_trampoline, - &get_cpu_entry_area (cpu)->entry_trampoline,PAGE_SIZE );#endif percpu_setup_debug_store (cpu); } -#ifdef CONFIG_X86_64 -int arch_get_kallsym (unsigned int symnum,unsigned long *value,char *type, -char *name) -{ -unsigned int cpu, ncpu =0 ; - -if (symnum >=num_possible_cpus ()) -return -EINVAL ; - -for_each_possible_cpu (cpu) { -if (ncpu++ >= symnum) -break ; - } - - *value = (unsigned long )&get_cpu_entry_area (cpu)->entry_trampoline; - *type ='t' ; -strlcpy (name,"__entry_SYSCALL_64_trampoline" ,KSYM_NAME_LEN ); - -return 0 ; -} -#endif -static __init void setup_cpu_entry_area_ptes (void ) {#ifdef CONFIG_X86_32
arch/x86/mm/pti.c +32 -1
@@ -434,11 +434,42 @@ static void __init pti_clone_p4d(unsigned long addr) }/* -* Clone the CPU_ENTRY_AREA into the user space visible page table. +* Clone the CPU_ENTRY_AREA and associated data into the user space visible +* page table. */ static void __init pti_clone_user_shared (void ) { +unsigned int cpu; +pti_clone_p4d (CPU_ENTRY_AREA_BASE ); + +for_each_possible_cpu (cpu) { +/* +* The SYSCALL64 entry code needs to be able to find the +* thread stack and needs one word of scratch space in which +* to spill a register. All of this lives in the TSS, in +* the sp1 and sp2 slots. +* +* This is done for all possible CPUs during boot to ensure +* that it's propagated to all mms. If we were to add one of +* these mappings during CPU hotplug, we would need to take +* some measure to make sure that every mm that subsequently +* ran on that CPU would have the relevant PGD entry in its +* pagetables. The usual vmalloc_fault() mechanism would not +* work for page faults taken in entry_SYSCALL_64 before RSP +* is set up. +*/ + +unsigned long va = (unsigned long )&per_cpu (cpu_tss_rw, cpu); +phys_addr_t pa =per_cpu_ptr_to_phys ((void *)va); +pte_t *target_pte; + + target_pte =pti_user_pagetable_walk_pte (va); +if (WARN_ON (!target_pte)) +return ; + + *target_pte =pfn_pte (pa >>PAGE_SHIFT ,PAGE_KERNEL ); + } }#else /* CONFIG_X86_64 */