Pile o' entry stack changes
Subsystems (4)
Andy Lutomirski
maintainercover Pile o' entry stack changes
Hi all-
Here's a pile of entry changes. In brief summary:
- Lots of people (Linus included) have asked to convert the entry code to pop registers on exit instead of movqing them off the stack. This makes a bunch of progress in that direction.
- Linux's sp0 handling has annoyed me for a while. We have thread_struct::sp0, which never made much sense to me. This series removes it on x86_64 and removes most references on x86_32.
- Xen PV's cpuinit code did incomprehensible things with stack pointers. This makes it comprehensible.
Juergen, this needs a bit of help on Xen -- see the NMI patch for details.
Reviews would be appreciated :)
Andy Lutomirski (17):
x86/asm/64: Remove the restore_c_regs_and_iret label x86/asm/64: Split the iret-to-user and iret-to-kernel paths x86/asm/64: Move SWAPGS into the common iret-to-usermode path x86/asm/64: Simplify reg restore code in the standard IRET paths x86/asm/64: Shrink paranoid_exit_restore and make labels local x86/asm/64: Use pop instead of movq in syscall_return_via_sysret x86/asm/64: Merge the fast and slow SYSRET paths x86/asm/64: De-Xen-ify our NMI code
x86/asm/32: Pull MSR_IA32_SYSENTER_CS update code out of
native_load_sp0()
x86/asm/64: Pass sp0 directly to load_sp0()
x86/asm: Add task_top_of_stack() to find the top of a task's stack x86/xen/64: Clean up SP code in cpu_initialize_context() x86/boot/64: Stop initializing TSS.sp0 at boot
x86/asm/64: Remove all remaining direct thread_struct::sp0 reads x86/boot/32: Fix cpu_current_top_of_stack initialization at boot x86/asm/64: Remove thread_struct::sp0
x86/traps: Use a new on_thread_stack() helper to clean up an assertion
arch/x86/entry/calling.h | 9 +++ arch/x86/entry/entry_64.S | 133 ++++++++++++++++++---------------- arch/x86/entry/entry_64_compat.S | 3 +- arch/x86/include/asm/compat.h | 1 + arch/x86/include/asm/paravirt.h | 5 +- arch/x86/include/asm/paravirt_types.h | 2 +- arch/x86/include/asm/processor.h | 68 +++++++++-------- arch/x86/include/asm/switch_to.h | 23 ++++++ arch/x86/include/asm/thread_info.h | 11 --- arch/x86/kernel/cpu/common.c | 12 ++- arch/x86/kernel/head_64.S | 2 +- arch/x86/kernel/process.c | 3 +- arch/x86/kernel/process_32.c | 3 +- arch/x86/kernel/process_64.c | 5 +- arch/x86/kernel/smpboot.c | 3 +- arch/x86/kernel/traps.c | 3 +- arch/x86/kernel/vm86_32.c | 14 ++-- arch/x86/lguest/boot.c | 7 +- arch/x86/xen/enlighten_pv.c | 7 +- arch/x86/xen/smp_pv.c | 17 ++++- 20 files changed, 192 insertions(+), 139 deletions(-)
Signature · 1 line
2.13.5
Andy Lutomirski
maintainerRFC 2/17 x86/asm/64: Split the iret-to-user and iret-to-kernel paths
These code paths will diverge soon.
--- arch/x86/entry/entry_64.S | 20 +++++++++++--------- arch/x86/entry/entry_64_compat.S | 2 +- arch/x86/kernel/head_64.S | 2 +- 3 files changed, 13 insertions(+), 11 deletions(-)
arch/x86/entry/entry_64.S +11 -9
@@ -315,7 +315,7 @@ syscall_return_via_sysret: opportunistic_sysret_failed: SWAPGS - jmp restore_regs_and_iret + jmp restore_regs_and_return_to_usermode END(entry_SYSCALL_64) ENTRY(stub_ptregs_64)
@@ -412,7 +412,7 @@ ENTRY(ret_from_fork) call syscall_return_slowpath /* returns with IRQs disabled */ TRACE_IRQS_ON /* user mode is traced as IRQS on */ SWAPGS - jmp restore_regs_and_iret + jmp restore_regs_and_return_to_usermode 1: /* kernel thread */
@@ -524,7 +524,13 @@ GLOBAL(retint_user) call prepare_exit_to_usermode TRACE_IRQS_IRETQ SWAPGS - jmp restore_regs_and_iret + +GLOBAL(restore_regs_and_return_to_usermode) + RESTORE_EXTRA_REGS + RESTORE_C_REGS + REMOVE_PT_GPREGS_FROM_STACK 8 + INTERRUPT_RETURN + /* Returning to kernel space */ retint_kernel:
@@ -544,11 +550,7 @@ retint_kernel: */ TRACE_IRQS_IRETQ -/* - * At this label, code paths which return to kernel and to user, - * which come from interrupts/exception and from syscalls, merge. - */ -GLOBAL(restore_regs_and_iret) +GLOBAL(restore_regs_and_return_to_kernel) RESTORE_EXTRA_REGS RESTORE_C_REGS REMOVE_PT_GPREGS_FROM_STACK 8
@@ -1264,7 +1266,7 @@ ENTRY(nmi) * work, because we don't want to enable interrupts. */ SWAPGS - jmp restore_regs_and_iret + jmp restore_regs_and_return_to_usermode .Lnmi_from_kernel: /*
arch/x86/entry/entry_64_compat.S +1 -1
@@ -338,7 +338,7 @@ ENTRY(entry_INT80_compat) /* Go back to user mode. */ TRACE_IRQS_ON SWAPGS - jmp restore_regs_and_iret + jmp restore_regs_and_return_to_usermode END(entry_INT80_compat) ALIGN
arch/x86/kernel/head_64.S +1 -1
@@ -310,7 +310,7 @@ early_idt_handler_common: 20: decl early_recursion_flag(%rip) - jmp restore_regs_and_iret + jmp restore_regs_and_return_to_kernel ENDPROC(early_idt_handler_common) __INITDATA
Signature · 1 line
2.13.5
Andy Lutomirski
maintainerRFC 5/17 x86/asm/64: Shrink paranoid_exit_restore and make labels local
In reply to Andy Lutomirskiparanoid_exit_restore was a copy of restore_regs_and_return_to_kernel. Merge them and make the paranoid_exit internal labels local.
Keeping .Lparanoid_exit makes the code a bit shorter because it allows a 2-byte jnz instead of a 5-byte jnz.
Saves 96 bytes of text.
(This is still a bit suboptimal in a non-CONFIG_TRACE_IRQFLAGS
kernel, but fixing that would make the code rather messy.)
--- arch/x86/entry/entry_64.S | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-)
arch/x86/entry/entry_64.S +5 -8
@@ -1057,17 +1057,14 @@ ENTRY(paranoid_exit) DISABLE_INTERRUPTS(CLBR_ANY) TRACE_IRQS_OFF_DEBUG testl %ebx, %ebx /* swapgs needed? */ - jnz paranoid_exit_no_swapgs + jnz .Lparanoid_exit_no_swapgs TRACE_IRQS_IRETQ SWAPGS_UNSAFE_STACK - jmp paranoid_exit_restore -paranoid_exit_no_swapgs: + jmp .Lparanoid_exit_restore +.Lparanoid_exit_no_swapgs: TRACE_IRQS_IRETQ_DEBUG -paranoid_exit_restore: - RESTORE_EXTRA_REGS - RESTORE_C_REGS - REMOVE_PT_GPREGS_FROM_STACK 8 - INTERRUPT_RETURN +.Lparanoid_exit_restore: + jmp restore_regs_and_return_to_kernel END(paranoid_exit) /*
Signature · 1 line
2.13.5
Andy Lutomirski
maintainerRFC 9/17 x86/asm/32: Pull MSR_IA32_SYSENTER_CS update code out of native_load_sp0()
In reply to Andy LutomirskiThis causees the MSR_IA32_SYSENTER_CS write to move out of the paravirt hook. This shouldn't affect Xen or lgeust: Xen already ignores MSR_IA32_SYSENTER_ESP writes and lguest doesn't support SYSENTER at all. In any event, Xen doesn't support vm86() in a useful way.
--- arch/x86/include/asm/processor.h | 7 ------- arch/x86/include/asm/switch_to.h | 11 +++++++++++ arch/x86/kernel/process_32.c | 1 + arch/x86/kernel/process_64.c | 2 +- arch/x86/kernel/vm86_32.c | 6 +++++- 5 files changed, 18 insertions(+), 9 deletions(-)
arch/x86/include/asm/processor.h +0 -7
@@ -513,13 +513,6 @@ static inline void native_load_sp0(struct tss_struct *tss, struct thread_struct *thread) { tss->x86_tss.sp0 = thread->sp0; -#ifdef CONFIG_X86_32 -/* Only happens when SEP is enabled, no need to test "SEP"arately: */ -if (unlikely (tss->x86_tss.ss1 != thread->sysenter_cs)) { - tss->x86_tss.ss1 = thread->sysenter_cs; -wrmsr (MSR_IA32_SYSENTER_CS , thread->sysenter_cs,0 ); - } -#endif }static inline void native_swapgs (void )
arch/x86/include/asm/switch_to.h +11 -0
@@ -72,4 +72,15 @@ do { \ ((last) =__switch_to_asm ((prev), (next))); \ } while (0) +#ifdef CONFIG_X86_32 +static inline void refresh_sysenter_cs (struct thread_struct *thread) +{ +/* Only happens when SEP is enabled, no need to test "SEP"arately: */ +if (unlikely (this_cpu_read (cpu_tss.x86_tss.ss1) == thread->sysenter_cs)) +return ; + +this_cpu_write (cpu_tss.x86_tss.ss1, thread->sysenter_cs); +} +#endif +#endif /* _ASM_X86_SWITCH_TO_H */
arch/x86/kernel/process_32.c +1 -0
@@ -287,6 +287,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)* current_thread_info(). */ load_sp0 (tss, next); +refresh_sysenter_cs ();/* in case prev or next is vm86 */ this_cpu_write (cpu_current_top_of_stack, (unsigned long )task_stack_page (next_p) +THREAD_SIZE );
arch/x86/kernel/process_64.c +1 -1
@@ -436,7 +436,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)*/ this_cpu_write (current_task, next_p); -/* Reload esp0 and ss1. This changes current_thread_info(). */ +/* Reload sp0. */ load_sp0 (tss, next);/*
arch/x86/kernel/vm86_32.c +5 -1
@@ -54,6 +54,7 @@#include <asm/irq.h> #include <asm/traps.h> #include <asm/vm86.h> +#include <asm/switch_to.h> /* * Known problems:
@@ -149,6 +150,7 @@ void save_v86_state(struct kernel_vm86_regs *regs, int retval) tsk->thread.sp0 = vm86->saved_sp0; tsk->thread.sysenter_cs = __KERNEL_CS;load_sp0 (tss, &tsk->thread); +refresh_sysenter_cs (); vm86->saved_sp0 =0 ;put_cpu ();
@@ -368,8 +370,10 @@ static long do_sys_vm86(struct vm86plus_struct __user *user_vm86, bool plus)/* make room for real-mode segments */ tsk->thread.sp0 +=16 ; -if (static_cpu_has (X86_FEATURE_SEP )) +if (static_cpu_has (X86_FEATURE_SEP )) { tsk->thread.sysenter_cs =0 ; +refresh_sysenter_cs (); + }load_sp0 (tss, &tsk->thread);put_cpu ();
Signature · 1 line
2.13.5
Andy Lutomirski
maintainerRFC 8/17 x86/asm/64: De-Xen-ify our NMI code
In reply to Andy LutomirskiXen PV is fundamentally incompatible with our fancy NMI code: it doesn't use IST at all, and Xen entries clobber two stack slots below the hardware frame.
Drop Xen PV support from our NMI code entirely.
XXX: Juergen: could you write and test the tiny patch needed to make Xen PV have a xen_nmi entry that handles NMIs? I don't know how to test it.
--- arch/x86/entry/entry_64.S | 41 +++++++++++++++++------------------------ 1 file changed, 17 insertions(+), 24 deletions(-)
arch/x86/entry/entry_64.S +17 -24
@@ -1171,21 +1171,12 @@ ENTRY(error_exit) jmp retint_user END(error_exit) -/* Runs on exception stack */ +/* + * Runs on exception stack. Xen PV does not go through this path at all, + * so we can use real assembly here. + */ ENTRY(nmi) /* - * Fix up the exception frame if we're on Xen. - * PARAVIRT_ADJUST_EXCEPTION_FRAME is guaranteed to push at most - * one value to the stack on native, so it may clobber the rdx - * scratch slot, but it won't clobber any of the important - * slots past it. - * - * Xen is a different story, because the Xen frame itself overlaps - * the "NMI executing" variable. - */ - PARAVIRT_ADJUST_EXCEPTION_FRAME - - /* * We allow breakpoints in NMIs. If a breakpoint occurs, then * the iretq it performs will take us out of NMI context. * This means that we can have nested NMIs where the next
@@ -1240,7 +1231,7 @@ ENTRY(nmi) * stacks lest we corrupt the "NMI executing" variable. */ - SWAPGS_UNSAFE_STACK + swapgs cld movq %rsp, %rdx movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
@@ -1402,7 +1393,7 @@ nested_nmi_out: popq %rdx /* We are returning to kernel mode, so this cannot result in a fault. */ - INTERRUPT_RETURN + iretq first_nmi: /* Restore rdx. */
@@ -1432,7 +1423,7 @@ first_nmi: pushfq /* RFLAGS */ pushq $__KERNEL_CS /* CS */ pushq $1f /* RIP */ - INTERRUPT_RETURN /* continues at repeat_nmi below */ + iretq /* continues at repeat_nmi below */ 1: #endif
@@ -1502,20 +1493,22 @@ nmi_restore: /* * Clear "NMI executing". Set DF first so that we can easily * distinguish the remaining code between here and IRET from - * the SYSCALL entry and exit paths. On a native kernel, we - * could just inspect RIP, but, on paravirt kernels, - * INTERRUPT_RETURN can translate into a jump into a - * hypercall page. + * the SYSCALL entry and exit paths. + * + * We arguably should just inspect RIP instead, but I (Andy) wrote + * this code when I had the misapprehension that Xen PV supported + * NMIs, and Xen PV would break that approach. */ std movq $0, 5*8(%rsp) /* clear "NMI executing" */ /* - * INTERRUPT_RETURN reads the "iret" frame and exits the NMI - * stack in a single instruction. We are returning to kernel - * mode, so this cannot result in a fault. + * iretq reads the "iret" frame and exits the NMI stack in a + * single instruction. We are returning to kernel mode, so this + * cannot result in a fault. Similarly, we don't need to worry + * about espfix64 on the way back to kernel mode. */ - INTERRUPT_RETURN + iretq END(nmi) ENTRY(ignore_sysret)
Signature · 1 line
2.13.5
Andy Lutomirski
maintainerRFC 13/17 x86/boot/64: Stop initializing TSS.sp0 at boot
In reply to Andy LutomirskiIn my quest to get rid of thread_struct::sp0, I want to clean up or remove all of its readers. Two of them are in cpu_init() (32-bit and 64-bit), and they aren't needed. This is because we never enter userspace at all on the threads that CPUs are initialized in.
Poison the initial TSS.sp0 and stop initializing it on CPU init.
--- arch/x86/kernel/cpu/common.c | 12 ++++++++++-- arch/x86/kernel/process.c | 3 ++- 2 files changed, 12 insertions(+), 3 deletions(-)
arch/x86/kernel/cpu/common.c +10 -2
@@ -1554,9 +1554,13 @@ void cpu_init(void)BUG_ON (me->mm);enter_lazy_tlb (&init_mm, me); -load_sp0 (current->thread.sp0); +/* +* Initialize the TSS. Don't bother initializing sp0, as the initial +* task never enters user mode. +*/ set_tss_desc (cpu, t);load_TR_desc (); +load_mm_ldt (&init_mm);clear_all_debug_regs ();
@@ -1608,9 +1612,13 @@ void cpu_init(void)BUG_ON (curr->mm);enter_lazy_tlb (&init_mm, curr); -load_sp0 (thread->sp0); +/* +* Initialize the TSS. Don't bother initializing sp0, as the initial +* task never enters user mode. +*/ set_tss_desc (cpu, t);load_TR_desc (); +load_mm_ldt (&init_mm); t->x86_tss.io_bitmap_base = offsetof(struct tss_struct , io_bitmap);
arch/x86/kernel/process.c +2 -1
@@ -48,7 +48,8 @@*/ __visible DEFINE_PER_CPU_SHARED_ALIGNED (struct tss_struct ,cpu_tss ) = { .x86_tss = { - .sp0 =TOP_OF_INIT_STACK , +/* Initialize sp0 to a value that is definitely invalid. */ + .sp0 =0x8000000000000001 , #ifdef CONFIG_X86_32 .ss0 = __KERNEL_DS, .ss1 = __KERNEL_CS,
Signature · 1 line
2.13.5
Andy Lutomirski
maintainerRFC 16/17 x86/asm/64: Remove thread_struct::sp0
In reply to Andy LutomirskiOn x86_64, we can easily calculate sp0 when needed instead of storing it in thread_struct.
On x86_32, a similar cleanup would be possible, but it would require cleaning up the vm86 code first, and that can wait for a later cleanup series.
--- arch/x86/include/asm/compat.h | 1 + arch/x86/include/asm/processor.h | 33 +++++++++++++++------------------ arch/x86/include/asm/switch_to.h | 6 ++++++ arch/x86/kernel/process_64.c | 1 - 4 files changed, 22 insertions(+), 19 deletions(-)
arch/x86/include/asm/compat.h +1 -0
@@ -6,6 +6,7 @@*/ #include <linux/types.h> #include <linux/sched.h> +#include <linux/sched/task_stack.h> #include <asm/processor.h> #include <asm/user32.h> #include <asm/unistd.h>
arch/x86/include/asm/processor.h +15 -18
@@ -423,7 +423,9 @@ typedef struct {struct thread_struct {/* Cached TLS descriptors: */ struct desc_struct tls_array[GDT_ENTRY_TLS_ENTRIES ]; +#ifdef CONFIG_X86_32 unsigned long sp0; +#endif unsigned long sp;#ifdef CONFIG_X86_32 unsigned long sysenter_cs;
@@ -790,6 +792,13 @@ static inline void spin_lock_prefetch(const void *x)#define task_top_of_stack (task) ((unsigned long)(task_pt_regs(task) + 1)) +#define task_pt_regs (task) \ +({ \ + unsigned long __ptr = (unsigned long)task_stack_page(task); \ + __ptr += THREAD_SIZE - TOP_OF_KERNEL_STACK_PADDING; \ + ((struct pt_regs *)__ptr) - 1; \ +}) +#ifdef CONFIG_X86_32 /* * User space process size: 3GB (default).
@@ -807,23 +816,6 @@ static inline void spin_lock_prefetch(const void *x) .addr_limit = KERNEL_DS, \ } -/* -* TOP_OF_KERNEL_STACK_PADDING reserves 8 bytes on top of the ring0 stack. -* This is necessary to guarantee that the entire "struct pt_regs" -* is accessible even if the CPU haven't stored the SS/ESP registers -* on the stack (interrupt gate does not save these registers -* when switching to the same priv ring). -* Therefore beware: accessing the ss/esp fields of the -* "struct pt_regs" is possible, but they may contain the -* completely wrong values. -*/ -#define task_pt_regs (task) \ -({ \ - unsigned long __ptr = (unsigned long)task_stack_page(task); \ - __ptr += THREAD_SIZE - TOP_OF_KERNEL_STACK_PADDING; \ - ((struct pt_regs *)__ptr) - 1; \ -}) -#define KSTK_ESP (task) (task_pt_regs(task)->sp)#else
@@ -852,12 +844,17 @@ static inline void spin_lock_prefetch(const void *x)#define STACK_TOP TASK_SIZE#define STACK_TOP_MAX TASK_SIZE_MAX +#ifdef CONFIG_X86_32 #define INIT_THREAD { \ .sp0 = TOP_OF_INIT_STACK, \ .addr_limit = KERNEL_DS, \ } +#else +#define INIT_THREAD { \ + .addr_limit = KERNEL_DS, \ +} +#endif -#define task_pt_regs (tsk) ((struct pt_regs *)(tsk)->thread.sp0 - 1)extern unsigned long KSTK_ESP (struct task_struct *task);#endif /* CONFIG_X86_64 */
arch/x86/include/asm/switch_to.h +6 -0
@@ -1,6 +1,8 @@#ifndef _ASM_X86_SWITCH_TO_H#define _ASM_X86_SWITCH_TO_H +#include <linux/sched/task_stack.h> +struct task_struct ;/* one of the stranger aspects of C forward declarations */ struct task_struct *__switch_to_asm(struct task_struct *prev,
@@ -86,7 +88,11 @@ static inline void refresh_sysenter_cs(struct thread_struct *thread)/* This is used when switching tasks or entering/exiting vm86 mode. */ static inline void update_sp0 (struct task_struct *task) { +#ifdef CONFIG_X86_32 load_sp0 (task->thread.sp0); +#else +load_sp0 (task_top_of_stack (task)); +#endif }#endif /* _ASM_X86_SWITCH_TO_H */
arch/x86/kernel/process_64.c +0 -1
@@ -158,7 +158,6 @@ int copy_thread_tls(unsigned long clone_flags, unsigned long sp,struct inactive_task_frame *frame;struct task_struct *me = current; - p->thread.sp0 = (unsigned long )task_stack_page (p) +THREAD_SIZE ; childregs =task_pt_regs (p); fork_frame = container_of(childregs, struct fork_frame, regs); frame = &fork_frame->frame;
Signature · 1 line
2.13.5
Andy Lutomirski
maintainerRFC 17/17 x86/traps: Use a new on_thread_stack() helper to clean up an assertion
In reply to Andy Lutomirski--- arch/x86/include/asm/processor.h | 17 +++++++++++++++++ arch/x86/include/asm/thread_info.h | 11 ----------- arch/x86/kernel/traps.c | 3 +-- 3 files changed, 18 insertions(+), 13 deletions(-)
arch/x86/include/asm/processor.h +17 -0
@@ -534,6 +534,23 @@ static inline unsigned long current_top_of_stack(void)#endif } +static inline unsigned long current_stack_pointer (void ) +{ +unsigned long sp; +#ifdef CONFIG_X86_64 + asm("mov %%rsp,%0" :"=g" (sp)); +#else + asm("mov %%esp,%0" :"=g" (sp)); +#endif +return sp; +} + +static inline bool on_thread_stack (void ) +{ +return (unsigned long )(current_top_of_stack () - +current_stack_pointer ()) <THREAD_SIZE ; +} +#ifdef CONFIG_PARAVIRT #include <asm/paravirt.h> #else
arch/x86/include/asm/thread_info.h +0 -11
@@ -155,17 +155,6 @@ struct thread_info {*/ #ifndef __ASSEMBLY__ -static inline unsigned long current_stack_pointer (void ) -{ -unsigned long sp; -#ifdef CONFIG_X86_64 - asm("mov %%rsp,%0" :"=g" (sp)); -#else - asm("mov %%esp,%0" :"=g" (sp)); -#endif -return sp; -} -/* * Walks up the stack frames to make sure that the specified object is * entirely contained by a single stack frame.
arch/x86/kernel/traps.c +1 -2
@@ -153,8 +153,7 @@ void ist_begin_non_atomic(struct pt_regs *regs)* will catch asm bugs and any attempt to use ist_preempt_enable * from double_fault. */ -BUG_ON ((unsigned long )(current_top_of_stack () - -current_stack_pointer ()) >=THREAD_SIZE ); +BUG_ON (!on_thread_stack ());preempt_enable_no_resched (); }
Signature · 1 line
2.13.5
Andy Lutomirski
maintainerRFC 15/17 x86/boot/32: Fix cpu_current_top_of_stack initialization at boot
In reply to Andy Lutomirskicpu_current_top_of_stack's initialization forgot about TOP_OF_KERNEL_STACK_PADDING. This bug didn't matter because the idle threads never enter user mode.
--- arch/x86/kernel/smpboot.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)
arch/x86/kernel/smpboot.c +1 -2
@@ -958,8 +958,7 @@ void common_cpu_up(unsigned int cpu, struct task_struct *idle)#ifdef CONFIG_X86_32 /* Stack for startup_32 can be just as for start_secondary onwards */ irq_ctx_init (cpu); -per_cpu (cpu_current_top_of_stack, cpu) = - (unsigned long )task_stack_page (idle) +THREAD_SIZE ; +per_cpu (cpu_current_top_of_stack, cpu) =task_top_of_stack (idle);#else initial_gs =per_cpu_offset (cpu);#endif
Signature · 1 line
2.13.5
Andy Lutomirski
maintainerRFC 14/17 x86/asm/64: Remove all remaining direct thread_struct::sp0 reads
In reply to Andy LutomirskiThe only remaining readers in context switch code or vm86(), and they all just want to update TSS.sp0 to match the current task. Replace them all with a new helper update_sp0().
--- arch/x86/include/asm/switch_to.h | 6 ++++++ arch/x86/kernel/process_32.c | 2 +- arch/x86/kernel/process_64.c | 2 +- arch/x86/kernel/vm86_32.c | 4 ++-- 4 files changed, 10 insertions(+), 4 deletions(-)
arch/x86/include/asm/switch_to.h +6 -0
@@ -83,4 +83,10 @@ static inline void refresh_sysenter_cs(struct thread_struct *thread) }#endif +/* This is used when switching tasks or entering/exiting vm86 mode. */ +static inline void update_sp0 (struct task_struct *task) +{ +load_sp0 (task->thread.sp0); +} +#endif /* _ASM_X86_SWITCH_TO_H */
arch/x86/kernel/process_32.c +1 -1
@@ -286,7 +286,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)* Reload esp0 and cpu_current_top_of_stack. This changes * current_thread_info(). */ -load_sp0 (next->sp0); +update_sp0 (next);refresh_sysenter_cs (next);/* in case prev or next is vm86 */ this_cpu_write(cpu_current_top_of_stack, (unsigned long)task_stack_page(next_p) +
arch/x86/kernel/process_64.c +1 -1
@@ -437,7 +437,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)this_cpu_write (current_task, next_p);/* Reload sp0. */ -load_sp0 (next->sp0); +update_sp0 (next_p);/* * Now maybe reload the debug registers and handle I/O bitmaps
arch/x86/kernel/vm86_32.c +2 -2
@@ -147,7 +147,7 @@ void save_v86_state(struct kernel_vm86_regs *regs, int retval) tsk->thread.sp0 = vm86->saved_sp0; tsk->thread.sysenter_cs = __KERNEL_CS; -load_sp0 (tsk->thread.sp0); +update_sp0 (tsk);refresh_sysenter_cs (&tsk->thread); vm86->saved_sp0 =0 ;put_cpu ();
@@ -371,7 +371,7 @@ static long do_sys_vm86(struct vm86plus_struct __user *user_vm86, bool plus)refresh_sysenter_cs (&tsk->thread); } -load_sp0 (tsk->thread.sp0); +update_sp0 (tsk);put_cpu ();if (vm86->flags &VM86_SCREEN_BITMAP )
Signature · 1 line
2.13.5
Andy Lutomirski
maintainerRFC 12/17 x86/xen/64: Clean up SP code in cpu_initialize_context()
In reply to Andy LutomirskiI'm removing thread_struct::sp0, and Xen's usage of it is slightly dubious and unnecessary. Use task_top_of_stack() instead.
While we're at at, reorder the code slightly to make it more obvious what's going on.
--- arch/x86/xen/smp_pv.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-)
arch/x86/xen/smp_pv.c +14 -3
@@ -13,6 +13,7 @@* single-threaded. */ #include <linux/sched.h> +#include <linux/sched/task_stack.h> #include <linux/err.h> #include <linux/slab.h> #include <linux/smp.h>
@@ -293,12 +294,19 @@ cpu_initialize_context(unsigned int cpu, struct task_struct *idle)#endif memset (&ctxt->fpu_ctxt,0 ,sizeof (ctxt->fpu_ctxt)); +/* +* Bring up the CPU in cpu_bringup_and_idle() with the stack +* pointing just below where pt_regs would be if it were a normal +* kernel entry. +*/ ctxt->user_regs.eip = (unsigned long )cpu_bringup_and_idle; ctxt->flags =VGCF_IN_KERNEL ; ctxt->user_regs.eflags =0x1000 ;/* IOPL_RING1 */ ctxt->user_regs.ds = __USER_DS; ctxt->user_regs.es = __USER_DS; ctxt->user_regs.ss = __KERNEL_DS; + ctxt->user_regs.cs = __KERNEL_CS; + ctxt->user_regs.esp =task_top_of_stack (idle) -sizeof (struct pt_regs );xen_copy_trap_info (ctxt->trap_ctxt);
@@ -313,8 +321,13 @@ cpu_initialize_context(unsigned int cpu, struct task_struct *idle) ctxt->gdt_frames[0 ] = gdt_mfn; ctxt->gdt_ents =GDT_ENTRIES ; +/* +* Set SS:SP that Xen will use when entering guest kernel mode +* from guest user mode. Subsequent calls to load_sp0() can +* change this value. +*/ ctxt->kernel_ss = __KERNEL_DS; - ctxt->kernel_sp = idle->thread.sp0; + ctxt->kernel_sp =task_top_of_stack (idle);#ifdef CONFIG_X86_32 ctxt->event_callback_cs = __KERNEL_CS;
@@ -326,10 +339,8 @@ cpu_initialize_context(unsigned int cpu, struct task_struct *idle) (unsigned long )xen_hypervisor_callback; ctxt->failsafe_callback_eip = (unsigned long )xen_failsafe_callback; - ctxt->user_regs.cs = __KERNEL_CS;per_cpu (xen_cr3, cpu) =__pa (swapper_pg_dir); - ctxt->user_regs.esp = idle->thread.sp0 -sizeof (struct pt_regs ); ctxt->ctrlreg[3 ] =xen_pfn_to_cr3 (virt_to_gfn (swapper_pg_dir));if (HYPERVISOR_vcpu_op (VCPUOP_initialise,xen_vcpu_nr (cpu), ctxt))BUG ();
Signature · 1 line
2.13.5
Andy Lutomirski
maintainerRFC 11/17 x86/asm: Add task_top_of_stack() to find the top of a task's stack
In reply to Andy LutomirskiThis will let us get rid of a few places that hardcode accesses to thread.sp0.
--- arch/x86/include/asm/processor.h | 2 ++ 1 file changed, 2 insertions(+)
arch/x86/include/asm/processor.h +2 -0
@@ -788,6 +788,8 @@ static inline void spin_lock_prefetch(const void *x)#define TOP_OF_INIT_STACK ((unsigned long)&init_stack + sizeof(init_stack) - \ TOP_OF_KERNEL_STACK_PADDING) +#define task_top_of_stack (task) ((unsigned long)(task_pt_regs(task) + 1)) +#ifdef CONFIG_X86_32 /* * User space process size: 3GB (default).
Signature · 1 line
2.13.5
Andy Lutomirski
maintainerRFC 10/17 x86/asm/64: Pass sp0 directly to load_sp0()
In reply to Andy Lutomirskiload_sp0() had an odd signature:
void load_sp0(struct tss_struct *tss, struct thread_struct *thread);
Simplify it to:
void load_sp0(unsigned long sp0);
--- arch/x86/include/asm/paravirt.h | 5 ++--- arch/x86/include/asm/paravirt_types.h | 2 +- arch/x86/include/asm/processor.h | 9 ++++----- arch/x86/kernel/cpu/common.c | 4 ++-- arch/x86/kernel/process_32.c | 4 ++-- arch/x86/kernel/process_64.c | 2 +- arch/x86/kernel/vm86_32.c | 12 ++++-------- arch/x86/lguest/boot.c | 7 +++---- arch/x86/xen/enlighten_pv.c | 7 +++---- 9 files changed, 22 insertions(+), 30 deletions(-)
arch/x86/include/asm/paravirt.h +2 -3
@@ -15,10 +15,9 @@#include <linux/cpumask.h> #include <asm/frame.h> -static inline void load_sp0 (struct tss_struct *tss, -struct thread_struct *thread) +static inline void load_sp0 (unsigned long sp0) { -PVOP_VCALL2 (pv_cpu_ops.load_sp0, tss, thread); +PVOP_VCALL1 (pv_cpu_ops.load_sp0, sp0); }/* The paravirtualized CPUID instruction. */
arch/x86/include/asm/paravirt_types.h +1 -1
@@ -136,7 +136,7 @@ struct pv_cpu_ops {void (*alloc_ldt)(struct desc_struct *ldt,unsigned entries);void (*free_ldt)(struct desc_struct *ldt,unsigned entries); -void (*load_sp0)(struct tss_struct *tss,struct thread_struct *t); +void (*load_sp0)(unsigned long sp0);void (*set_iopl_mask)(unsigned mask);
arch/x86/include/asm/processor.h +4 -5
@@ -510,9 +510,9 @@ static inline void native_set_iopl_mask(unsigned mask) }static inline void -native_load_sp0 (struct tss_struct *tss,struct thread_struct *thread) +native_load_sp0 (unsigned long sp0) { - tss->x86_tss.sp0 = thread->sp0; +this_cpu_write (cpu_tss.x86_tss.sp0, sp0); }static inline void native_swapgs (void )
@@ -537,10 +537,9 @@ static inline unsigned long current_top_of_stack(void)#else #define __cpuid native_cpuid -static inline void load_sp0 (struct tss_struct *tss, -struct thread_struct *thread) +static inline void load_sp0 (unsigned long sp0) { -native_load_sp0 (tss, thread); +native_load_sp0 (sp0); }#define set_iopl_mask native_set_iopl_mask
arch/x86/kernel/cpu/common.c +2 -2
@@ -1554,7 +1554,7 @@ void cpu_init(void)BUG_ON (me->mm);enter_lazy_tlb (&init_mm, me); -load_sp0 (t, ¤t->thread); +load_sp0 (current->thread.sp0);set_tss_desc (cpu, t);load_TR_desc ();load_mm_ldt (&init_mm);
@@ -1608,7 +1608,7 @@ void cpu_init(void)BUG_ON (curr->mm);enter_lazy_tlb (&init_mm, curr); -load_sp0 (t, thread); +load_sp0 (thread->sp0);set_tss_desc (cpu, t);load_TR_desc ();load_mm_ldt (&init_mm);
arch/x86/kernel/process_32.c +2 -2
@@ -286,8 +286,8 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)* Reload esp0 and cpu_current_top_of_stack. This changes * current_thread_info(). */ -load_sp0 (tss, next); -refresh_sysenter_cs ();/* in case prev or next is vm86 */ +load_sp0 (next->sp0); +refresh_sysenter_cs (next);/* in case prev or next is vm86 */ this_cpu_write (cpu_current_top_of_stack, (unsigned long )task_stack_page (next_p) +THREAD_SIZE );
arch/x86/kernel/process_64.c +1 -1
@@ -437,7 +437,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)this_cpu_write (current_task, next_p);/* Reload sp0. */ -load_sp0 (tss, next); +load_sp0 (next->sp0);/* * Now maybe reload the debug registers and handle I/O bitmaps
arch/x86/kernel/vm86_32.c +4 -8
@@ -94,7 +94,6 @@void save_v86_state (struct kernel_vm86_regs *regs,int retval) { -struct tss_struct *tss;struct task_struct *tsk = current;struct vm86plus_struct __user *user;struct vm86 *vm86 = current->thread.vm86;
@@ -146,11 +145,10 @@ void save_v86_state(struct kernel_vm86_regs *regs, int retval)do_exit (SIGSEGV ); } - tss = &per_cpu (cpu_tss,get_cpu ()); tsk->thread.sp0 = vm86->saved_sp0; tsk->thread.sysenter_cs = __KERNEL_CS; -load_sp0 (tss, &tsk->thread); -refresh_sysenter_cs (); +load_sp0 (tsk->thread.sp0); +refresh_sysenter_cs (&tsk->thread); vm86->saved_sp0 =0 ;put_cpu ();
@@ -238,7 +236,6 @@ SYSCALL_DEFINE2(vm86, unsigned long, cmd, unsigned long, arg)static long do_sys_vm86 (struct vm86plus_struct __user *user_vm86,bool plus) { -struct tss_struct *tss;struct task_struct *tsk = current;struct vm86 *vm86 = tsk->thread.vm86;struct kernel_vm86_regs vm86regs;
@@ -366,16 +363,15 @@ static long do_sys_vm86(struct vm86plus_struct __user *user_vm86, bool plus) vm86->saved_sp0 = tsk->thread.sp0;lazy_save_gs (vm86->regs32.gs); - tss = &per_cpu (cpu_tss,get_cpu ());/* make room for real-mode segments */ tsk->thread.sp0 +=16 ;if (static_cpu_has (X86_FEATURE_SEP )) { tsk->thread.sysenter_cs =0 ; -refresh_sysenter_cs (); +refresh_sysenter_cs (&tsk->thread); } -load_sp0 (tss, &tsk->thread); +load_sp0 (tsk->thread.sp0);put_cpu ();if (vm86->flags &VM86_SCREEN_BITMAP )
arch/x86/lguest/boot.c +3 -4
@@ -1054,12 +1054,11 @@ static void lguest_time_init(void)* will not tolerate us trying to use that), the stack pointer, and the number * of pages in the stack. */ -static void lguest_load_sp0 (struct tss_struct *tss, -struct thread_struct *thread) +static void lguest_load_sp0 (unsigned long sp0) { -lazy_hcall3 (LHCALL_SET_STACK , __KERNEL_DS |0x1 , thread->sp0, +lazy_hcall3 (LHCALL_SET_STACK , __KERNEL_DS |0x1 , sp0,THREAD_SIZE /PAGE_SIZE ); - tss->x86_tss.sp0 = thread->sp0; +this_cpu_write (cpu_tss.x86_tss.sp0, sp0); }/* Let's just say, I wouldn't do debugging under a Guest. */
arch/x86/xen/enlighten_pv.c +3 -4
@@ -771,15 +771,14 @@ static void __init xen_write_gdt_entry_boot(struct desc_struct *dt, int entry, } } -static void xen_load_sp0 (struct tss_struct *tss, -struct thread_struct *thread) +static void xen_load_sp0 (unsigned long sp0) {struct multicall_space mcs; mcs =xen_mc_entry (0 ); -MULTI_stack_switch (mcs.mc, __KERNEL_DS, thread->sp0); +MULTI_stack_switch (mcs.mc, __KERNEL_DS, sp0);xen_mc_issue (PARAVIRT_LAZY_CPU ); - tss->x86_tss.sp0 = thread->sp0; +this_cpu_write (cpu_tss.x86_tss.sp0, sp0); }void xen_set_iopl_mask (unsigned mask)
Signature · 1 line
2.13.5
Andy Lutomirski
maintainerRFC 7/17 x86/asm/64: Merge the fast and slow SYSRET paths
In reply to Andy Lutomirski--- arch/x86/entry/entry_64.S | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
arch/x86/entry/entry_64.S +3 -3
@@ -216,9 +216,8 @@ entry_SYSCALL_64_fastpath: TRACE_IRQS_ON /* user mode is traced as IRQs on */ movq RIP(%rsp), %rcx movq EFLAGS(%rsp), %r11 - RESTORE_C_REGS_EXCEPT_RCX_R11 - movq RSP(%rsp), %rsp - USERGS_SYSRET64 + addq $6*8, %rsp + jmp .Lpop_c_regs_and_sysret 1: /*
@@ -309,6 +308,7 @@ return_from_SYSCALL_64: syscall_return_via_sysret: /* rcx and r11 are already restored (see code above) */ POP_EXTRA_REGS +.Lpop_c_regs_and_sysret: popq %rsi /* skip r11 */ popq %r10 popq %r9
Signature · 1 line
2.13.5
Andy Lutomirski
maintainerRFC 6/17 x86/asm/64: Use pop instead of movq in syscall_return_via_sysret
In reply to Andy LutomirskiSaves 64 bytes.
--- arch/x86/entry/entry_64.S | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-)
arch/x86/entry/entry_64.S +11 -3
@@ -308,9 +308,17 @@ return_from_SYSCALL_64: */ syscall_return_via_sysret: /* rcx and r11 are already restored (see code above) */ - RESTORE_EXTRA_REGS - RESTORE_C_REGS_EXCEPT_RCX_R11 - movq RSP(%rsp), %rsp + POP_EXTRA_REGS + popq %rsi /* skip r11 */ + popq %r10 + popq %r9 + popq %r8 + popq %rax + popq %rsi /* skip rcx */ + popq %rdx + popq %rsi + popq %rdi + movq RSP-ORIG_RAX(%rsp), %rsp USERGS_SYSRET64 END(entry_SYSCALL_64)
Signature · 1 line
2.13.5
Andy Lutomirski
maintainerRFC 4/17 x86/asm/64: Simplify reg restore code in the standard IRET paths
In reply to Andy LutomirskiThe old code restored all the registers with movq instead of pop. In theory, this was done because some CPUs have higher movq throughput, but any gain there would be tiny and is almost certainly outweighed by the higher text size.
This saves 96 bytes of text.
--- arch/x86/entry/calling.h | 9 +++++++++ arch/x86/entry/entry_64.S | 28 ++++++++++++++++++++++------ 2 files changed, 31 insertions(+), 6 deletions(-)
arch/x86/entry/calling.h +9 -0
@@ -147,6 +147,15 @@ For 32-bit we have the following conventions - kernel is built with movq 5*8+\offset(%rsp), %rbx .endm + .macro POP_EXTRA_REGS + popq %r15 + popq %r14 + popq %r13 + popq %r12 + popq %rbp + popq %rbx + .endm + .macro RESTORE_C_REGS_HELPER rstor_rax=1, rstor_rcx=1, rstor_r11=1, rstor_r8910=1, rstor_rdx=1 .if \rstor_r11 movq 6*8(%rsp), %r11
arch/x86/entry/entry_64.S +22 -6
@@ -521,9 +521,17 @@ GLOBAL(retint_user) GLOBAL(swapgs_restore_regs_and_return_to_usermode) SWAPGS - RESTORE_EXTRA_REGS - RESTORE_C_REGS - REMOVE_PT_GPREGS_FROM_STACK 8 + POP_EXTRA_REGS + popq %r11 + popq %r10 + popq %r9 + popq %r8 + popq %rax + popq %rcx + popq %rdx + popq %rsi + popq %rdi + addq $8, %rsp INTERRUPT_RETURN
@@ -546,9 +554,17 @@ retint_kernel: TRACE_IRQS_IRETQ GLOBAL(restore_regs_and_return_to_kernel) - RESTORE_EXTRA_REGS - RESTORE_C_REGS - REMOVE_PT_GPREGS_FROM_STACK 8 + POP_EXTRA_REGS + popq %r11 + popq %r10 + popq %r9 + popq %r8 + popq %rax + popq %rcx + popq %rdx + popq %rsi + popq %rdi + addq $8, %rsp INTERRUPT_RETURN ENTRY(native_iret)
Signature · 1 line
2.13.5
Andy Lutomirski
maintainerRFC 3/17 x86/asm/64: Move SWAPGS into the common iret-to-usermode path
In reply to Andy LutomirskiAll of the code paths that ended up doing IRET to usermode did SWAPGS immediately beforehand. Move the SWAPGS into the common code.
--- arch/x86/entry/entry_64.S | 26 ++++++++++---------------- arch/x86/entry/entry_64_compat.S | 3 +-- 2 files changed, 11 insertions(+), 18 deletions(-)
arch/x86/entry/entry_64.S +10 -16
@@ -249,7 +249,7 @@ return_from_SYSCALL_64: movq RCX(%rsp), %rcx movq RIP(%rsp), %r11 cmpq %rcx, %r11 /* RCX == RIP */ - jne opportunistic_sysret_failed + jne swapgs_restore_regs_and_return_to_usermode /* * On Intel CPUs, SYSRET with non-canonical RCX/RIP will #GP
@@ -267,14 +267,14 @@ return_from_SYSCALL_64: /* If this changed %rcx, it was not canonical */ cmpq %rcx, %r11 - jne opportunistic_sysret_failed + jne swapgs_restore_regs_and_return_to_usermode cmpq $__USER_CS, CS(%rsp) /* CS must match SYSRET */ - jne opportunistic_sysret_failed + jne swapgs_restore_regs_and_return_to_usermode movq R11(%rsp), %r11 cmpq %r11, EFLAGS(%rsp) /* R11 == RFLAGS */ - jne opportunistic_sysret_failed + jne swapgs_restore_regs_and_return_to_usermode /* * SYSCALL clears RF when it saves RFLAGS in R11 and SYSRET cannot
@@ -295,12 +295,12 @@ return_from_SYSCALL_64: * would never get past 'stuck_here'. */ testq $(X86_EFLAGS_RF|X86_EFLAGS_TF), %r11 - jnz opportunistic_sysret_failed + jnz swapgs_restore_regs_and_return_to_usermode /* nothing to check for RSP */ cmpq $__USER_DS, SS(%rsp) /* SS must match SYSRET */ - jne opportunistic_sysret_failed + jne swapgs_restore_regs_and_return_to_usermode /* * We win! This label is here just for ease of understanding
@@ -312,10 +312,6 @@ syscall_return_via_sysret: RESTORE_C_REGS_EXCEPT_RCX_R11 movq RSP(%rsp), %rsp USERGS_SYSRET64 - -opportunistic_sysret_failed: - SWAPGS - jmp restore_regs_and_return_to_usermode END(entry_SYSCALL_64) ENTRY(stub_ptregs_64)
@@ -411,8 +407,7 @@ ENTRY(ret_from_fork) movq %rsp, %rdi call syscall_return_slowpath /* returns with IRQs disabled */ TRACE_IRQS_ON /* user mode is traced as IRQS on */ - SWAPGS - jmp restore_regs_and_return_to_usermode + jmp swapgs_restore_regs_and_return_to_usermode 1: /* kernel thread */
@@ -523,9 +518,9 @@ GLOBAL(retint_user) mov %rsp,%rdi call prepare_exit_to_usermode TRACE_IRQS_IRETQ - SWAPGS -GLOBAL(restore_regs_and_return_to_usermode) +GLOBAL(swapgs_restore_regs_and_return_to_usermode) + SWAPGS RESTORE_EXTRA_REGS RESTORE_C_REGS REMOVE_PT_GPREGS_FROM_STACK 8
@@ -1265,8 +1260,7 @@ ENTRY(nmi) * Return back to user mode. We must *not* do the normal exit * work, because we don't want to enable interrupts. */ - SWAPGS - jmp restore_regs_and_return_to_usermode + jmp swapgs_restore_regs_and_return_to_usermode .Lnmi_from_kernel: /*
arch/x86/entry/entry_64_compat.S +1 -2
@@ -337,8 +337,7 @@ ENTRY(entry_INT80_compat) /* Go back to user mode. */ TRACE_IRQS_ON - SWAPGS - jmp restore_regs_and_return_to_usermode + jmp swapgs_restore_regs_and_return_to_usermode END(entry_INT80_compat) ALIGN
Signature · 1 line
2.13.5
Andy Lutomirski
maintainerRFC 1/17 x86/asm/64: Remove the restore_c_regs_and_iret label
In reply to Andy LutomirskiThe only user was the 64-bit opportunistic SYSRET failure path, and that path didn't really need it. This change makes the opportunistic SYSRET code a bit more straightforward and gets rid of the label.
--- arch/x86/entry/entry_64.S | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-)
arch/x86/entry/entry_64.S +2 -3
@@ -240,7 +240,6 @@ entry_SYSCALL64_slow_path: call do_syscall_64 /* returns with IRQs disabled */ return_from_SYSCALL_64: - RESTORE_EXTRA_REGS TRACE_IRQS_IRETQ /* we're about to change IF */ /*
@@ -309,13 +308,14 @@ return_from_SYSCALL_64: */ syscall_return_via_sysret: /* rcx and r11 are already restored (see code above) */ + RESTORE_EXTRA_REGS RESTORE_C_REGS_EXCEPT_RCX_R11 movq RSP(%rsp), %rsp USERGS_SYSRET64 opportunistic_sysret_failed: SWAPGS - jmp restore_c_regs_and_iret + jmp restore_regs_and_iret END(entry_SYSCALL_64) ENTRY(stub_ptregs_64)
@@ -550,7 +550,6 @@ retint_kernel: */ GLOBAL(restore_regs_and_iret) RESTORE_EXTRA_REGS -restore_c_regs_and_iret: RESTORE_C_REGS REMOVE_PT_GPREGS_FROM_STACK 8 INTERRUPT_RETURN
Signature · 1 line
2.13.5
Quoted 4 lines · Andy Lutomirski
- Lots of people (Linus included) have asked to convert the entry code to pop registers on exit instead of movqing them off the stack. This makes a bunch of progress in that direction.
You should benchmark it on Atoms. Likely it's a regression there because they don't have the special PUSH/POP acceleration.
-Andi
Andy Lutomirski
maintainerQuoted 8 lines · Andi Kleen
Quoted 4 lines · Andy Lutomirski
- Lots of people (Linus included) have asked to convert the entry code to pop registers on exit instead of movqing them off the stack. This makes a bunch of progress in that direction.
You should benchmark it on Atoms. Likely it's a regression there because they don't have the special PUSH/POP acceleration.
I'm not entirely sure this is a worthwhile reason. Atom will lose a few cycles due to POP throughput, but there's a lot less decode bandwidth needed and we save a cache line or two.
--Andy
Quoted 7 lines · Andy Lutomirski
Quoted 6 lines · Andi Kleen
Quoted 4 lines · Andy Lutomirski
- Lots of people (Linus included) have asked to convert the entry code to pop registers on exit instead of movqing them off the stack. This makes a bunch of progress in that direction.
Quoted 6 lines
Quoted 2 lines
You should benchmark it on Atoms. Likely it's a regression there because they don't have the special PUSH/POP acceleration.
I'm not entirely sure this is a worthwhile reason. Atom will lose a few cycles due to POP throughput, but there's a lot less decode bandwidth needed and we save a cache line or two.
I think we can also safely assume that Atom will eventually either join the 21st century or die out - mild Atom micro-costs are not a good reason to complicate the entry code...
Thanks,
Ingo
Juergen Gross
maintainerRe: [RFC 08/17] x86/asm/64: De-Xen-ify our NMI code
In reply to Andy LutomirskiQuoted 9 lines · Andy Lutomirski
Xen PV is fundamentally incompatible with our fancy NMI code: it doesn't use IST at all, and Xen entries clobber two stack slots below the hardware frame.
Drop Xen PV support from our NMI code entirely.
XXX: Juergen: could you write and test the tiny patch needed to make Xen PV have a
xen_nmientry that handles NMIs? I don't know how to test it.
You mean something like the attached one?
Seems to work at least for the "normal" case of a NMI coming in at a random point in time.
Regarding testing: in case you have a Xen setup you can easily send a NMI to a domain from dom0:
xl trigger <domain> nmi
Juergen
Quoted 98 lines
Cc Juergen Gross <[email protected]>Cc Boris Ostrovsky <[email protected]>Signed-off-by Andy Lutomirski <[email protected]>--- arch/x86/entry/entry_64.S | 41 +++++++++++++++++------------------------ 1 file changed, 17 insertions(+), 24 deletions(-)arch/x86/entry/entry_64.S +17
-24@@ -1171,21 +1171,12 @@ ENTRY(error_exit) jmp retint_user END(error_exit) -/* Runs on exception stack */ +/* + * Runs on exception stack. Xen PV does not go through this path at all, + * so we can use real assembly here. + */ ENTRY(nmi) /* - * Fix up the exception frame if we're on Xen. - * PARAVIRT_ADJUST_EXCEPTION_FRAME is guaranteed to push at most - * one value to the stack on native, so it may clobber the rdx - * scratch slot, but it won't clobber any of the important - * slots past it. - * - * Xen is a different story, because the Xen frame itself overlaps - * the "NMI executing" variable. - */ - PARAVIRT_ADJUST_EXCEPTION_FRAME - - /* * We allow breakpoints in NMIs. If a breakpoint occurs, then * the iretq it performs will take us out of NMI context. * This means that we can have nested NMIs where the next@@ -1240,7 +1231,7 @@ ENTRY(nmi) * stacks lest we corrupt the "NMI executing" variable. */ - SWAPGS_UNSAFE_STACK + swapgs cld movq %rsp, %rdx movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp@@ -1402,7 +1393,7 @@ nested_nmi_out: popq %rdx /* We are returning to kernel mode, so this cannot result in a fault. */ - INTERRUPT_RETURN + iretq first_nmi: /* Restore rdx. */@@ -1432,7 +1423,7 @@ first_nmi: pushfq /* RFLAGS */ pushq $__KERNEL_CS /* CS */ pushq $1f /* RIP */ - INTERRUPT_RETURN /* continues at repeat_nmi below */ + iretq /* continues at repeat_nmi below */ 1: #endif@@ -1502,20 +1493,22 @@ nmi_restore: /* * Clear "NMI executing". Set DF first so that we can easily * distinguish the remaining code between here and IRET from - * the SYSCALL entry and exit paths. On a native kernel, we - * could just inspect RIP, but, on paravirt kernels, - * INTERRUPT_RETURN can translate into a jump into a - * hypercall page. + * the SYSCALL entry and exit paths. + * + * We arguably should just inspect RIP instead, but I (Andy) wrote + * this code when I had the misapprehension that Xen PV supported + * NMIs, and Xen PV would break that approach. */ std movq $0, 5*8(%rsp) /* clear "NMI executing" */ /* - * INTERRUPT_RETURN reads the "iret" frame and exits the NMI - * stack in a single instruction. We are returning to kernel - * mode, so this cannot result in a fault. + * iretq reads the "iret" frame and exits the NMI stack in a + * single instruction. We are returning to kernel mode, so this + * cannot result in a fault. Similarly, we don't need to worry + * about espfix64 on the way back to kernel mode. */ - INTERRUPT_RETURN + iretq END(nmi) ENTRY(ignore_sysret)
Quoted 1 line
From
544e6cf531efaafbeed7d4c15690ee360ae1e45cMon Sep 17 00:00:00 2001
From: Juergen Gross <[email protected]>
Date: Thu, 7 Sep 2017 10:23:19 +0200
Subject: [PATCH] xen: add xen nmi trap entry
Instead of trying to execute any NMI via the bare metal's NMI trap handler use a Xen specific one for pv domains, like we do for e.g. debug traps. As in a pv domain the NMI is handled via the normal kernel stack this is the correct thing to do.
This will enable us to get rid of the very fragile and questionable dependencies between the bare metal NMI handler and Xen assumptions believed to be broken anyway.
--- arch/x86/entry/entry_64.S | 2 +- arch/x86/include/asm/traps.h | 2 +- arch/x86/xen/enlighten_pv.c | 2 +- arch/x86/xen/xen-asm_64.S | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-)
arch/x86/entry/entry_64.S +1 -1
@@ -1058,6 +1058,7 @@ idtentry int3 do_int3 has_error_code=0 paranoid=1 shift_ist=DEBUG_STACK idtentry stack_segment do_stack_segment has_error_code=1 #ifdef CONFIG_XEN +idtentry xennmi do_nmi has_error_code=0 idtentry xendebug do_debug has_error_code=0 idtentry xenint3 do_int3 has_error_code=0 #endif
@@ -1223,7 +1224,6 @@ ENTRY(error_exit) END(error_exit) /* Runs on exception stack */ -/* XXX: broken on Xen PV */ ENTRY(nmi) UNWIND_HINT_IRET_REGS /*
arch/x86/include/asm/traps.h +1 -1
@@ -37,9 +37,9 @@ asmlinkage void simd_coprocessor_error(void);#if defined(CONFIG_X86_64 ) && defined(CONFIG_XEN_PV ) asmlinkage void xen_divide_error(void); +asmlinkage void xen_xennmi(void); asmlinkage void xen_xendebug(void); asmlinkage void xen_xenint3(void); -asmlinkage void xen_nmi(void); asmlinkage void xen_overflow(void); asmlinkage void xen_bounds(void); asmlinkage void xen_invalid_op(void);
arch/x86/xen/enlighten_pv.c +1 -1
@@ -600,7 +600,7 @@ static struct trap_array_entry trap_array[] = { #ifdef CONFIG_X86_MCE { machine_check, xen_machine_check, true }, #endif - { nmi, xen_nmi, true }, + { nmi, xen_xennmi, true }, { overflow, xen_overflow, false }, #ifdef CONFIG_IA32_EMULATION { entry_INT80_compat, xen_entry_INT80_compat, false },
arch/x86/xen/xen-asm_64.S +1 -1
@@ -29,7 +29,7 @@ xen_pv_trap debug xen_pv_trap xendebug xen_pv_trap int3 xen_pv_trap xenint3 -xen_pv_trap nmi +xen_pv_trap xennmi xen_pv_trap overflow xen_pv_trap bounds xen_pv_trap invalid_op
Signature · 1 line
2.12.3
Re: [RFC 01/17] x86/asm/64: Remove the restore_c_regs_and_iret label
In reply to Andy LutomirskiQuoted 9 lines · Andy Lutomirski
The only user was the 64-bit opportunistic SYSRET failure path, and that path didn't really need it. This change makes the opportunistic SYSRET code a bit more straightforward and gets rid of the label.
Signed-off-by Andy Lutomirski <[email protected]>--- arch/x86/entry/entry_64.S | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-)
Btw, you need to refresh your stuff because of those UNWIND_HINT_EMPTY things. I get:
checking file arch/x86/entry/entry_64.S Hunk #1 succeeded at 245 (offset 5 lines). Hunk #2 FAILED at 308.
Hunk #3 succeeded at 637 (offset 88 lines). 1 out of 3 hunks FAILED
Otherwise
Signature · 4 lines
Regards/Gruss,
Boris.
Good mailing practices for 400: avoid top-posting and trim the reply.Re: [RFC 01/17] x86/asm/64: Remove the restore_c_regs_and_iret label
Quoted 20 lines · Borislav Petkov
Quoted 9 lines · Andy Lutomirski
The only user was the 64-bit opportunistic SYSRET failure path, and that path didn't really need it. This change makes the opportunistic SYSRET code a bit more straightforward and gets rid of the label.
Signed-off-by Andy Lutomirski <[email protected]>--- arch/x86/entry/entry_64.S | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-)Btw, you need to refresh your stuff because of those
UNWIND_HINT_EMPTYthings. I get:checking file
arch/x86/entry/entry_64.SHunk #1 succeeded at 245 (offset 5 lines). Hunk #2 FAILED at 308.
Hunk #3 succeeded at 637 (offset 88 lines). 1 out of 3 hunks FAILED
Quoted 3 lines
Otherwise
Reviewed-by Borislav Petkov <[email protected]>
I'd suggest tip:master or upstream 24e700e291d5 as a post-merge-window base for x86 bits.
Thanks,
Ingo
Re: [RFC 01/17] x86/asm/64: Remove the restore_c_regs_and_iret label
Quoted 23 lines · Ingo Molnar
Quoted 20 lines · Borislav Petkov
Quoted 9 lines · Andy Lutomirski
The only user was the 64-bit opportunistic SYSRET failure path, and that path didn't really need it. This change makes the opportunistic SYSRET code a bit more straightforward and gets rid of the label.
Signed-off-by Andy Lutomirski <[email protected]>--- arch/x86/entry/entry_64.S | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-)Btw, you need to refresh your stuff because of those
UNWIND_HINT_EMPTYthings. I get:checking file
arch/x86/entry/entry_64.SHunk #1 succeeded at 245 (offset 5 lines). Hunk #2 FAILED at 308.
Hunk #3 succeeded at 637 (offset 88 lines). 1 out of 3 hunks FAILED
Quoted 6 lines
Quoted 3 lines
Otherwise
Reviewed-by Borislav Petkov <[email protected]>I'd suggest tip:master or upstream
24e700e291d5as a post-merge-window base for x86 bits.
Actually, scratch that, 1c9fe4409ce3 is probably the best base, it includes the PCID fixes. Haven't had much time to test that base though.
Thanks,
Ingo
Borislav Petkov
maintainerRe: [RFC 01/17] x86/asm/64: Remove the restore_c_regs_and_iret label
Quoted 2 lines · Ingo Molnar
Actually, scratch that,
1c9fe4409ce3is probably the best base, it includes the PCID fixes. Haven't had much time to test that base though.
Ok, I'll use that and scream if something's sh*tting in its pants.
Signature · 4 lines
Regards/Gruss,
Boris.
Good mailing practices for 400: avoid top-posting and trim the reply.Re: [RFC 01/17] x86/asm/64: Remove the restore_c_regs_and_iret label
Quoted 5 lines · Borislav Petkov
Quoted 2 lines · Ingo Molnar
Actually, scratch that,
1c9fe4409ce3is probably the best base, it includes the PCID fixes. Haven't had much time to test that base though.Ok, I'll use that and scream if something's sh*tting in its pants.
not the best of kernels, 32-bit allyesconfig doesn't even appear to build:
net/netfilter/xt_hashlimit.o: In function `hashlimit_mt_common.isra.6':
xt_hashlimit.c:(.text+0x1146): undefined reference to `__udivdi3'
:-/
Thanks,
Ingo
Andy Lutomirski
maintainerRe: [RFC 08/17] x86/asm/64: De-Xen-ify our NMI code
In reply to Juergen GrossQuoted 5 lines · Juergen Gross
Quoted 4 lines · Andy Lutomirski
Xen PV is fundamentally incompatible with our fancy NMI code: it doesn't use IST at all, and Xen entries clobber two stack slots below the hardware frame.
Quoted 7 lines
Quoted 5 lines
Drop Xen PV support from our NMI code entirely.
XXX: Juergen: could you write and test the tiny patch needed to make Xen PV have a
xen_nmientry that handles NMIs? I don't know how to test it.You mean something like the attached one?
Yes. Mind if I add it to my series?
Quoted 8 lines
Seems to work at least for the "normal" case of a NMI coming in at a random point in time.
Regarding testing: in case you have a Xen setup you can easily send a NMI to a domain from dom0:
xl trigger <domain> nmi
Thanks!
Quoted 102 lines
Juergen
Quoted 97 lines
Cc Juergen Gross <[email protected]>Cc Boris Ostrovsky <[email protected]>Signed-off-by Andy Lutomirski <[email protected]>--- arch/x86/entry/entry_64.S | 41 +++++++++++++++++------------------------ 1 file changed, 17 insertions(+), 24 deletions(-)arch/x86/entry/entry_64.S +17
-24@@ -1171,21 +1171,12 @@ ENTRY(error_exit) jmp retint_user END(error_exit) -/* Runs on exception stack */ +/* + * Runs on exception stack. Xen PV does not go through this path at all, + * so we can use real assembly here. + */ ENTRY(nmi) /* - * Fix up the exception frame if we're on Xen. - * PARAVIRT_ADJUST_EXCEPTION_FRAME is guaranteed to push at most - * one value to the stack on native, so it may clobber the rdx - * scratch slot, but it won't clobber any of the important - * slots past it. - * - * Xen is a different story, because the Xen frame itself overlaps - * the "NMI executing" variable. - */ - PARAVIRT_ADJUST_EXCEPTION_FRAME - - /* * We allow breakpoints in NMIs. If a breakpoint occurs, then * the iretq it performs will take us out of NMI context. * This means that we can have nested NMIs where the next@@ -1240,7 +1231,7 @@ ENTRY(nmi) * stacks lest we corrupt the "NMI executing" variable. */ - SWAPGS_UNSAFE_STACK + swapgs cld movq %rsp, %rdx movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp@@ -1402,7 +1393,7 @@ nested_nmi_out: popq %rdx /* We are returning to kernel mode, so this cannot result in a fault. */ - INTERRUPT_RETURN + iretq first_nmi: /* Restore rdx. */@@ -1432,7 +1423,7 @@ first_nmi: pushfq /* RFLAGS */ pushq $__KERNEL_CS /* CS */ pushq $1f /* RIP */ - INTERRUPT_RETURN /* continues at repeat_nmi below */ + iretq /* continues at repeat_nmi below */ 1: #endif@@ -1502,20 +1493,22 @@ nmi_restore: /* * Clear "NMI executing". Set DF first so that we can easily * distinguish the remaining code between here and IRET from - * the SYSCALL entry and exit paths. On a native kernel, we - * could just inspect RIP, but, on paravirt kernels, - * INTERRUPT_RETURN can translate into a jump into a - * hypercall page. + * the SYSCALL entry and exit paths. + * + * We arguably should just inspect RIP instead, but I (Andy) wrote + * this code when I had the misapprehension that Xen PV supported + * NMIs, and Xen PV would break that approach. */ std movq $0, 5*8(%rsp) /* clear "NMI executing" */ /* - * INTERRUPT_RETURN reads the "iret" frame and exits the NMI - * stack in a single instruction. We are returning to kernel - * mode, so this cannot result in a fault. + * iretq reads the "iret" frame and exits the NMI stack in a + * single instruction. We are returning to kernel mode, so this + * cannot result in a fault. Similarly, we don't need to worry + * about espfix64 on the way back to kernel mode. */ - INTERRUPT_RETURN + iretq END(nmi) ENTRY(ignore_sysret)
Juergen Gross
maintainerRe: [RFC 08/17] x86/asm/64: De-Xen-ify our NMI code
Quoted 8 lines · Andy Lutomirski
Quoted 7 lines · Juergen Gross
Quoted 6 lines · Andy Lutomirski
Xen PV is fundamentally incompatible with our fancy NMI code: it doesn't use IST at all, and Xen entries clobber two stack slots below the hardware frame.
Drop Xen PV support from our NMI code entirely.
Quoted 7 lines
Quoted 5 lines
Quoted 3 lines
XXX: Juergen: could you write and test the tiny patch needed to make Xen PV have a
xen_nmientry that handles NMIs? I don't know how to test it.You mean something like the attached one?
Yes. Mind if I add it to my series?
Go ahead!
Juergen
Re: [RFC 04/17] x86/asm/64: Simplify reg restore code in the standard IRET paths
In reply to Andy LutomirskiQuoted 67 lines · Andy Lutomirski
The old code restored all the registers with movq instead of pop. In theory, this was done because some CPUs have higher movq throughput, but any gain there would be tiny and is almost certainly outweighed by the higher text size.
This saves 96 bytes of text.
Signed-off-by Andy Lutomirski <[email protected]>--- arch/x86/entry/calling.h | 9 +++++++++ arch/x86/entry/entry_64.S | 28 ++++++++++++++++++++++------ 2 files changed, 31 insertions(+), 6 deletions(-)arch/x86/entry/calling.h +9
-0@@ -147,6 +147,15 @@ For 32-bit we have the following conventions - kernel is built with movq 5*8+\offset(%rsp), %rbx .endm + .macro POP_EXTRA_REGS + popq %r15 + popq %r14 + popq %r13 + popq %r12 + popq %rbp + popq %rbx + .endm + .macro RESTORE_C_REGS_HELPER rstor_rax=1, rstor_rcx=1, rstor_r11=1, rstor_r8910=1, rstor_rdx=1 .if \rstor_r11 movq 6*8(%rsp), %r11arch/x86/entry/entry_64.S +13
-6@@ -521,9 +521,17 @@ GLOBAL(retint_user) GLOBAL(swapgs_restore_regs_and_return_to_usermode) SWAPGS - RESTORE_EXTRA_REGS - RESTORE_C_REGS - REMOVE_PT_GPREGS_FROM_STACK 8 + POP_EXTRA_REGS + popq %r11 + popq %r10 + popq %r9 + popq %r8 + popq %rax + popq %rcx + popq %rdx + popq %rsi + popq %rdi + addq $8, %rsp INTERRUPT_RETURN@@ -546,9 +554,17 @@ retint_kernel: TRACE_IRQS_IRETQ GLOBAL(restore_regs_and_return_to_kernel) - RESTORE_EXTRA_REGS - RESTORE_C_REGS - REMOVE_PT_GPREGS_FROM_STACK 8 + POP_EXTRA_REGS + popq %r11
arch/x86/entry/entry_64.S retint_kernel: · line 568
Quoted 10 lines
arch/x86/entry/entry_64.S +9
-0+ popq %r10 + popq %r9 + popq %r8 + popq %rax + popq %rcx + popq %rdx + popq %rsi + popq %rdi + addq $8, %rsp INTERRUPT_RETURN
Any reason why these aren't in a POP_C_REGS macro? I think that would make it easier to verify correctness when reading the code.
Signature · 1 line
Josh
Re: [RFC 09/17] x86/asm/32: Pull MSR_IA32_SYSENTER_CS update code out of native_load_sp0()
In reply to Andy LutomirskiQuoted 5 lines · Andy Lutomirski
This causees the
MSR_IA32_SYSENTER_CSwrite to move out of the paravirt hook. This shouldn't affect Xen or lgeust: Xen already ignoresMSR_IA32_SYSENTER_ESPwrites and lguest doesn't support SYSENTER at all. In any event, Xen doesn't supportvm86()in a useful way.
Thanks to Juergen, lguest is no more :-)
Signature · 1 line
Josh
Re: [RFC 12/17] x86/xen/64: Clean up SP code in cpu_initialize_context()
In reply to Andy LutomirskiQuoted 7 lines · Andy Lutomirski
arch/x86/xen/smp_pv.c +3
-0@@ -293,12 +294,19 @@ cpu_initialize_context(unsigned int cpu, struct task_struct *idle)#endif memset (&ctxt->fpu_ctxt,0 ,sizeof (ctxt->fpu_ctxt)); +/* +* Bring up the CPU in cpu_bringup_and_idle() with the stack +* pointing just below where pt_regs would be if it were a normal
arch/x86/xen/smp_pv.c cpu_initialize_context(unsigned int cpu, struct task_struct *idle) · line 309
Quoted 10 lines
arch/x86/xen/smp_pv.c +4
-0+* kernel entry. +*/ ctxt->user_regs.eip = (unsigned long )cpu_bringup_and_idle; ctxt->flags =VGCF_IN_KERNEL ; ctxt->user_regs.eflags =0x1000 ;/* IOPL_RING1 */ ctxt->user_regs.ds = __USER_DS; ctxt->user_regs.es = __USER_DS; ctxt->user_regs.ss = __KERNEL_DS; + ctxt->user_regs.cs = __KERNEL_CS; + ctxt->user_regs.esp =task_top_of_stack (idle) -sizeof (struct pt_regs );
Isn't this the same as task_pt_regs(idle)?
Signature · 1 line
Josh
Re: [RFC 17/17] x86/traps: Use a new on_thread_stack() helper to clean up an assertion
In reply to Andy LutomirskiQuoted 15 lines · Andy Lutomirski
Signed-off-by Andy Lutomirski <[email protected]>--- arch/x86/include/asm/processor.h | 17 +++++++++++++++++ arch/x86/include/asm/thread_info.h | 11 ----------- arch/x86/kernel/traps.c | 3 +-- 3 files changed, 18 insertions(+), 13 deletions(-)arch/x86/include/asm/processor.h +0
-0@@ -534,6 +534,23 @@ static inline unsigned long current_top_of_stack(void)#endif }
arch/x86/include/asm/processor.h static inline unsigned long current_top_of_stack(void) · line 546
Quoted 10 lines
arch/x86/include/asm/processor.h +10
-0+static inline unsigned long current_stack_pointer (void ) +{ +unsigned long sp; +#ifdef CONFIG_X86_64 + asm("mov %%rsp,%0" :"=g" (sp)); +#else + asm("mov %%esp,%0" :"=g" (sp)); +#endif +return sp; +}
I know you're just moving the function, but this function could also be cleaned up by using _ASM_SP and getting rid of the ifdef.
Signature · 1 line
Josh
Re: [RFC 17/17] x86/traps: Use a new on_thread_stack() helper to clean up an assertion
Quoted 16 lines · Josh Poimboeuf
Quoted 15 lines · Andy Lutomirski
Signed-off-by Andy Lutomirski <[email protected]>--- arch/x86/include/asm/processor.h | 17 +++++++++++++++++ arch/x86/include/asm/thread_info.h | 11 ----------- arch/x86/kernel/traps.c | 3 +-- 3 files changed, 18 insertions(+), 13 deletions(-)arch/x86/include/asm/processor.h +0
-0@@ -534,6 +534,23 @@ static inline unsigned long current_top_of_stack(void)#endif }
Quoted 13 lines
arch/x86/include/asm/processor.h static inline unsigned long current_top_of_stack(void) · line 546
Quoted 10 lines
arch/x86/include/asm/processor.h +10
-0+static inline unsigned long current_stack_pointer (void ) +{ +unsigned long sp; +#ifdef CONFIG_X86_64 + asm("mov %%rsp,%0" :"=g" (sp)); +#else + asm("mov %%esp,%0" :"=g" (sp)); +#endif +return sp; +}I know you're just moving the function, but this function could also be cleaned up by using
_ASM_SPand getting rid of the ifdef.
For GCC, there is a rather more efficient way of doing this, which allows the compiler much more flexibility than forcing the use of a mov instruction.
I don't know how much you care in Linux, but in Xen it makes a massive code-volume difference, as that construct is the core of current (because we can't use %fs/%gs).
~Andrew