x86: Rewrite switch_to()
Subsystems (2)
v2 cover x86: Rewrite switch_to()
This patch set simplifies the switch_to() code, by moving the stack switch code out of line into an asm stub before calling __switch_to(). This ends up being more readable, and using the C calling convention instead of clobbering all registers improves code generation. It also allows newly forked processes to construct a special stack frame to seamlessly flow to ret_from_fork, instead of using a test and branch, or an unbalanced call/ret.
Changes from v1:
- Added struct inactive_task_frame
- Added comments about kernel threads returning to userspace
- Cleaned up some incorrect uses of thread.sp
Brian Gerst (6):
x86-32, kgdb: Don't use thread.ip in sleeping_thread_to_gdb_regs() x86-64, kgdb: clear GDB_PS on 64-bit
x86: Add struct inactive_task_frame
x86: Rewrite switch_to() code
x86: Pass kernel thread parameters in fork_frame
x86: Fix thread_saved_pc()
arch/x86/entry/entry_32.S | 68 +++++++++++++----- arch/x86/entry/entry_64.S | 76 ++++++++++++++------ arch/x86/include/asm/processor.h | 13 +--- arch/x86/include/asm/stacktrace.h | 4 +- arch/x86/include/asm/switch_to.h | 142 +++++++------------------------------ arch/x86/include/asm/thread_info.h | 2 - arch/x86/kernel/asm-offsets.c | 6 ++ arch/x86/kernel/asm-offsets_32.c | 5 ++ arch/x86/kernel/asm-offsets_64.c | 5 ++ arch/x86/kernel/kgdb.c | 8 +-- arch/x86/kernel/process.c | 13 +++- arch/x86/kernel/process_32.c | 29 +++----- arch/x86/kernel/process_64.c | 19 ++--- arch/x86/kernel/smpboot.c | 1 - 14 files changed, 187 insertions(+), 204 deletions(-)
PATCH v2 1/6 x86-32, kgdb: Don't use thread.ip in sleeping_thread_to_gdb_regs()
Match 64-bit and set gdb_regs[GDB_PC] to zero. thread.ip is always the same point in the scheduler (except for newly forked processes), and will be removed in a future patch.
--- arch/x86/kernel/kgdb.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)
arch/x86/kernel/kgdb.c +1 -2
@@ -172,7 +172,6 @@ void sleeping_thread_to_gdb_regs(unsigned long *gdb_regs, struct task_struct *p) gdb_regs[GDB_ES ] = __KERNEL_DS; gdb_regs[GDB_PS ] =0 ; gdb_regs[GDB_CS ] = __KERNEL_CS; - gdb_regs[GDB_PC ] = p->thread.ip; gdb_regs[GDB_SS ] = __KERNEL_DS; gdb_regs[GDB_FS ] =0xFFFF ; gdb_regs[GDB_GS ] =0xFFFF ;
@@ -180,7 +179,6 @@ void sleeping_thread_to_gdb_regs(unsigned long *gdb_regs, struct task_struct *p) gdb_regs32[GDB_PS ] = *(unsigned long *)(p->thread.sp +8 ); gdb_regs32[GDB_CS ] = __KERNEL_CS; gdb_regs32[GDB_SS ] = __KERNEL_DS; - gdb_regs[GDB_PC ] =0 ; gdb_regs[GDB_R8 ] =0 ; gdb_regs[GDB_R9 ] =0 ; gdb_regs[GDB_R10 ] =0 ;
@@ -190,6 +188,7 @@ void sleeping_thread_to_gdb_regs(unsigned long *gdb_regs, struct task_struct *p) gdb_regs[GDB_R14 ] =0 ; gdb_regs[GDB_R15 ] =0 ;#endif + gdb_regs[GDB_PC ] =0 ; gdb_regs[GDB_SP ] = p->thread.sp; }
Signature · 1 line
2.5.5
PATCH v2 2/6 x86-64, kgdb: clear GDB_PS on 64-bit
In reply to Brian Gerstswitch_to() no longer saves EFLAGS, so it's bogus to look for it on the stack. Set it to zero like 32-bit.
--- arch/x86/kernel/kgdb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/x86/kernel/kgdb.c +1 -1
@@ -176,7 +176,7 @@ void sleeping_thread_to_gdb_regs(unsigned long *gdb_regs, struct task_struct *p) gdb_regs[GDB_FS ] =0xFFFF ; gdb_regs[GDB_GS ] =0xFFFF ;#else - gdb_regs32[GDB_PS ] = *(unsigned long *)(p->thread.sp +8 ); + gdb_regs32[GDB_PS ] =0 ; gdb_regs32[GDB_CS ] = __KERNEL_CS; gdb_regs32[GDB_SS ] = __KERNEL_DS; gdb_regs[GDB_R8 ] =0 ;
Signature · 1 line
2.5.5
PATCH v2 3/6 x86: Add struct inactive_task_frame
In reply to Brian GerstAdd struct inactive_task_frame, which defines the layout of the stack for a sleeping process. For now, the only defined field is the BP register (frame pointer).
--- arch/x86/include/asm/stacktrace.h | 4 ++-- arch/x86/include/asm/switch_to.h | 5 +++++ arch/x86/kernel/kgdb.c | 3 ++- arch/x86/kernel/process.c | 3 ++- 4 files changed, 11 insertions(+), 4 deletions(-)
arch/x86/include/asm/stacktrace.h +2 -2
@@ -8,6 +8,7 @@#include <linux/uaccess.h> #include <linux/ptrace.h> +#include <asm/switch_to.h> extern int kstack_depth_to_print;
@@ -70,8 +71,7 @@ stack_frame(struct task_struct *task, struct pt_regs *regs) return bp; } -/* bp is the last reg pushed by switch_to */ -return *(unsigned long *)task->thread.sp; +return ((struct inactive_task_frame *)task->thread.sp)->bp; }#else static inline unsigned long
arch/x86/include/asm/switch_to.h +5 -0
@@ -8,6 +8,11 @@ struct tss_struct;void __switch_to_xtra (struct task_struct *prev_p,struct task_struct *next_p,struct tss_struct *tss); +/* data that is pointed to by thread.sp */ +struct inactive_task_frame { +unsigned long bp; +}; +#ifdef CONFIG_X86_32 #ifdef CONFIG_CC_STACKPROTECTOR
arch/x86/kernel/kgdb.c +2 -1
@@ -50,6 +50,7 @@#include <asm/apicdef.h> #include <asm/apic.h> #include <asm/nmi.h> +#include <asm/switch_to.h> struct dbg_reg_def_t dbg_reg_def[DBG_MAX_REG_NUM ] = {
@@ -166,7 +167,7 @@ void sleeping_thread_to_gdb_regs(unsigned long *gdb_regs, struct task_struct *p) gdb_regs[GDB_DX ] =0 ; gdb_regs[GDB_SI ] =0 ; gdb_regs[GDB_DI ] =0 ; - gdb_regs[GDB_BP ] = *(unsigned long *)p->thread.sp; + gdb_regs[GDB_BP ] = ((struct inactive_task_frame *)p->thread.sp)->bp;#ifdef CONFIG_X86_32 gdb_regs[GDB_DS ] = __KERNEL_DS; gdb_regs[GDB_ES ] = __KERNEL_DS;
arch/x86/kernel/process.c +2 -1
@@ -31,6 +31,7 @@#include <asm/tlbflush.h> #include <asm/mce.h> #include <asm/vm86.h> +#include <asm/switch_to.h> /* * per-CPU TSS segments. Threads are completely 'soft' on Linux,
@@ -555,7 +556,7 @@ unsigned long get_wchan(struct task_struct *p)if (sp < bottom || sp > top)return 0 ; - fp =READ_ONCE_NOCHECK (*(unsigned long *)sp); + fp =READ_ONCE_NOCHECK (((struct inactive_task_frame *)sp)->bp); do { if (fp < bottom || fp > top) return 0;
Signature · 1 line
2.5.5
Re: [PATCH v2 3/6] x86: Add struct inactive_task_frame
Quoted 3 lines · Brian Gerst
Add struct
inactive_task_frame, which defines the layout of the stack for a sleeping process. For now, the only defined field is the BP register (frame pointer).
I like this version.
Re: [PATCH v2 3/6] x86: Add struct inactive_task_frame
In reply to Brian GerstQuoted 5 lines · Brian Gerst
Add struct
inactive_task_frame, which defines the layout of the stack for a sleeping process. For now, the only defined field is the BP register (frame pointer).Signed-off-by Brian Gerst <[email protected]>
Signature · 1 line
Josh
PATCH v2 4/6 x86: Rewrite switch_to() code
In reply to Brian GerstMove the low-level context switch code to an out-of-line asm stub instead of using complex inline asm. This allows constructing a new stack frame for the child process to make it seamlessly flow to ret_from_fork without an extra test and branch in __switch_to(). It also improves code generation for __schedule() by using the C calling convention instead of clobbering all registers.
--- arch/x86/entry/entry_32.S | 37 ++++++++++ arch/x86/entry/entry_64.S | 41 ++++++++++- arch/x86/include/asm/processor.h | 3 - arch/x86/include/asm/switch_to.h | 137 ++++++------------------------------- arch/x86/include/asm/thread_info.h | 2 - arch/x86/kernel/asm-offsets.c | 6 ++ arch/x86/kernel/asm-offsets_32.c | 5 ++ arch/x86/kernel/asm-offsets_64.c | 5 ++ arch/x86/kernel/process_32.c | 9 ++- arch/x86/kernel/process_64.c | 9 ++- arch/x86/kernel/smpboot.c | 1 - 11 files changed, 125 insertions(+), 130 deletions(-)
arch/x86/entry/entry_32.S +37 -0
@@ -204,6 +204,43 @@ POP_GS_EX .endm +/* + * %eax: prev task + * %edx: next task + */ +ENTRY(__switch_to_asm) + /* + * Save callee-saved registers + * This must match the order in struct inactive_task_frame + */ + pushl %ebp + pushl %ebx + pushl %edi + pushl %esi + + /* switch stack */ + movl %esp, TASK_threadsp(%eax) + movl TASK_threadsp(%edx), %esp + +#ifdef CONFIG_CC_STACKPROTECTOR + movl TASK_stack_canary(%edx), %ebx + movl %ebx, PER_CPU_VAR(stack_canary)+stack_canary_offset +#endif + + /* restore callee-saved registers */ + popl %esi + popl %edi + popl %ebx + popl %ebp + + jmp __switch_to +END(__switch_to_asm) + +/* + * A newly forked process directly context switches into this address. + * + * eax: prev task we switched from + */ ENTRY(ret_from_fork) pushl %eax call schedule_tail
arch/x86/entry/entry_64.S +38 -3
51 lines not on this page · raw message
arch/x86/include/asm/processor.h +0 -3
@@ -385,9 +385,6 @@ struct thread_struct {unsigned short fsindex;unsigned short gsindex;#endif -#ifdef CONFIG_X86_32 -unsigned long ip; -#endif #ifdef CONFIG_X86_64 unsigned long fsbase;unsigned long gsbase;
arch/x86/include/asm/switch_to.h +21 -116
156 lines not on this page · raw message
arch/x86/include/asm/thread_info.h +0 -2
@@ -99,7 +99,6 @@ struct thread_info {#define TIF_UPROBE 12/* breakpointed or singlestepping */ #define TIF_NOTSC 16/* TSC is not accessible in userland */ #define TIF_IA32 17/* IA32 compatibility process */ -#define TIF_FORK 18/* ret_from_fork */ #define TIF_NOHZ 19/* in adaptive nohz mode */ #define TIF_MEMDIE 20/* is terminating due to OOM killer */ #define TIF_POLLING_NRFLAG 21/* idle is polling for TIF_NEED_RESCHED */
@@ -123,7 +122,6 @@ struct thread_info {#define _TIF_UPROBE (1 << TIF_UPROBE)#define _TIF_NOTSC (1 << TIF_NOTSC)#define _TIF_IA32 (1 << TIF_IA32) -#define _TIF_FORK (1 << TIF_FORK)#define _TIF_NOHZ (1 << TIF_NOHZ)#define _TIF_POLLING_NRFLAG (1 << TIF_POLLING_NRFLAG)#define _TIF_IO_BITMAP (1 << TIF_IO_BITMAP)
arch/x86/kernel/asm-offsets.c +6 -0
@@ -29,6 +29,12 @@void common (void ) {BLANK (); +OFFSET (TASK_threadsp, task_struct, thread.sp); +#ifdef CONFIG_CC_STACKPROTECTOR +OFFSET (TASK_stack_canary, task_struct, stack_canary); +#endif + +BLANK ();OFFSET (TI_flags, thread_info, flags);OFFSET (TI_status, thread_info, status);OFFSET (TI_addr_limit, thread_info, addr_limit);
arch/x86/kernel/asm-offsets_32.c +5 -0
@@ -57,6 +57,11 @@ void foo(void)/* Size of SYSENTER_stack */ DEFINE (SIZEOF_SYSENTER_stack,sizeof (((struct tss_struct *)0 )->SYSENTER_stack)); +#ifdef CONFIG_CC_STACKPROTECTOR +BLANK (); +OFFSET (stack_canary_offset, stack_canary, canary); +#endif +#if defined(CONFIG_LGUEST ) || defined(CONFIG_LGUEST_GUEST ) || defined(CONFIG_LGUEST_MODULE )BLANK ();OFFSET (LGUEST_DATA_irq_enabled, lguest_data, irq_enabled);
arch/x86/kernel/asm-offsets_64.c +5 -0
@@ -56,6 +56,11 @@ int main(void)OFFSET (TSS_sp0, tss_struct, x86_tss.sp0);BLANK (); +#ifdef CONFIG_CC_STACKPROTECTOR +DEFINE (stack_canary_offset, offsetof(union irq_stack_union , stack_canary)); +BLANK (); +#endif +DEFINE (__NR_syscall_max,sizeof (syscalls_64) -1 );DEFINE (NR_syscalls,sizeof (syscalls_64));
arch/x86/kernel/process_32.c +6 -3
@@ -133,17 +133,20 @@ int copy_thread_tls(unsigned long clone_flags, unsigned long sp, unsigned long arg, struct task_struct *p, unsigned long tls) { struct pt_regs *childregs = task_pt_regs(p); + struct fork_frame *fork_frame = container_of(childregs, struct fork_frame, regs); +struct inactive_task_frame *frame = &fork_frame->frame;struct task_struct *tsk;int err; - p->thread.sp = (unsigned long ) childregs; + frame->bp =0 ; + p->thread.sp = (unsigned long ) fork_frame; p->thread.sp0 = (unsigned long ) (childregs+1 );memset (p->thread.ptrace_bps,0 ,sizeof (p->thread.ptrace_bps));if (unlikely (p->flags &PF_KTHREAD )) {/* kernel thread */ memset (childregs,0 ,sizeof (struct pt_regs )); - p->thread.ip = (unsigned long ) ret_from_kernel_thread; + frame->ret_addr = (unsigned long ) ret_from_kernel_thread;task_user_gs (p) = __KERNEL_STACK_CANARY; childregs->ds = __USER_DS; childregs->es = __USER_DS;
@@ -161,7 +164,7 @@ int copy_thread_tls(unsigned long clone_flags, unsigned long sp,if (sp) childregs->sp = sp; - p->thread.ip = (unsigned long ) ret_from_fork; + frame->ret_addr = (unsigned long ) ret_from_fork;task_user_gs (p) =get_user_gs (current_pt_regs ()); p->thread.io_bitmap_ptr =NULL ;
arch/x86/kernel/process_64.c +7 -2
@@ -141,12 +141,17 @@ int copy_thread_tls(unsigned long clone_flags, unsigned long sp, {int err;struct pt_regs *childregs; +struct fork_frame *fork_frame; +struct inactive_task_frame *frame;struct task_struct *me = current; p->thread.sp0 = (unsigned long )task_stack_page (p) +THREAD_SIZE ; childregs =task_pt_regs (p); - p->thread.sp = (unsigned long ) childregs; -set_tsk_thread_flag (p,TIF_FORK ); + fork_frame = container_of(childregs, struct fork_frame, regs); + frame = &fork_frame->frame; + frame->bp =0 ; + frame->ret_addr = (unsigned long ) ret_from_fork; + p->thread.sp = (unsigned long ) fork_frame; p->thread.io_bitmap_ptr =NULL ;savesegment (gs, p->thread.gsindex);
arch/x86/kernel/smpboot.c +0 -1
@@ -935,7 +935,6 @@ void common_cpu_up(unsigned int cpu, struct task_struct *idle)per_cpu (cpu_current_top_of_stack, cpu) = (unsigned long )task_stack_page (idle) +THREAD_SIZE ;#else -clear_tsk_thread_flag (idle,TIF_FORK ); initial_gs =per_cpu_offset (cpu);#endif }
Signature · 1 line
2.5.5
Re: [PATCH v2 4/6] x86: Rewrite switch_to() code
Quoted 6 lines · Brian Gerst
Move the low-level context switch code to an out-of-line asm stub instead of using complex inline asm. This allows constructing a new stack frame for the child process to make it seamlessly flow to
ret_from_forkwithout an extra test and branch in__switch_to(). It also improves code generation for__schedule()by using the C calling convention instead of clobbering all registers.
This is great. The conflict with my stack code is minimal enough that I won't bother rebasing unless someone specifically wants me to.
--Andy
Re: [PATCH v2 4/6] x86: Rewrite switch_to() code
In reply to Brian GerstQuoted 8 lines · Brian Gerst
Move the low-level context switch code to an out-of-line asm stub instead of using complex inline asm. This allows constructing a new stack frame for the child process to make it seamlessly flow to
ret_from_forkwithout an extra test and branch in__switch_to(). It also improves code generation for__schedule()by using the C calling convention instead of clobbering all registers.Signed-off-by Brian Gerst <[email protected]>
You can also remove the STACK_FRAME_NON_STANDARD annotation for __schedule() in kernel/sched/core.c since it no longer produces an objtool warning.
Otherwise it looks great:
Signature · 1 line
Josh
PATCH v2 5/6 x86: Pass kernel thread parameters in fork_frame
In reply to Brian GerstInstead of setting up a fake pt_regs context, put the kernel thread function pointer and arg into the unused callee-restored registers of struct fork_frame.
--- arch/x86/entry/entry_32.S | 31 +++++++++++++++---------------- arch/x86/entry/entry_64.S | 35 ++++++++++++++++------------------- arch/x86/kernel/process_32.c | 14 ++++---------- arch/x86/kernel/process_64.c | 10 +++------- 4 files changed, 38 insertions(+), 52 deletions(-)
arch/x86/entry/entry_32.S +15 -16
@@ -240,35 +240,34 @@ END(__switch_to_asm) * A newly forked process directly context switches into this address. * * eax: prev task we switched from + * ebx: kernel thread func + * edi: kernel thread arg */ ENTRY(ret_from_fork) pushl %eax call schedule_tail popl %eax + testl %ebx, %ebx + jnz 1f + +2: /* When we fork, we trace the syscall return in the child, too. */ movl %esp, %eax call syscall_return_slowpath jmp restore_all -END(ret_from_fork) - -ENTRY(ret_from_kernel_thread) - pushl %eax - call schedule_tail - popl %eax - movl PT_EBP(%esp), %eax - call *PT_EBX(%esp) - movl $0, PT_EAX(%esp) + /* kernel thread */ +1: movl %edi, %eax + call *%ebx /* - * Kernel threads return to userspace as if returning from a syscall. - * We should check whether anything actually uses this path and, if so, - * consider switching it over to ret_from_fork. + * A kernel thread is allowed to return here after successfully + * calling do_execve(). Exit to userspace to complete the execve() + * syscall. */ - movl %esp, %eax - call syscall_return_slowpath - jmp restore_all -ENDPROC(ret_from_kernel_thread) + movl $0, PT_EAX(%esp) + jmp 2b +END(ret_from_fork) /* * Return to user mode is not as complex as all this looks,
arch/x86/entry/entry_64.S +16 -19
@@ -404,37 +404,34 @@ END(__switch_to_asm) * A newly forked process directly context switches into this address. * * rax: prev task we switched from + * rbx: kernel thread func + * r12: kernel thread arg */ ENTRY(ret_from_fork) movq %rax, %rdi call schedule_tail /* rdi: 'prev' task parameter */ - testb $3, CS(%rsp) /* from kernel_thread? */ + testq %rbx, %rbx /* from kernel_thread? */ jnz 1f - /* - * We came from kernel_thread. This code path is quite twisted, and - * someone should clean it up. - * - * copy_thread_tls stashes the function pointer in RBX and the - * parameter to be passed in RBP. The called function is permitted - * to call do_execve and thereby jump to user mode. - */ - movq RBP(%rsp), %rdi - call *RBX(%rsp) - movl $0, RAX(%rsp) - - /* - * Fall through as though we're exiting a syscall. This makes a - * twisted sort of sense if we just called do_execve. - */ - -1: +2: movq %rsp, %rdi call syscall_return_slowpath /* returns with IRQs disabled */ TRACE_IRQS_ON /* user mode is traced as IRQS on */ SWAPGS jmp restore_regs_and_iret + +1: + /* kernel thread */ + movq %r12, %rdi + call *%rbx + /* + * A kernel thread is allowed to return here after successfully + * calling do_execve(). Exit to userspace to complete the execve() + * syscall. + */ + movq $0, RAX(%rsp) + jmp 2b END(ret_from_fork) /*
arch/x86/kernel/process_32.c +4 -10
@@ -139,6 +139,7 @@ int copy_thread_tls(unsigned long clone_flags, unsigned long sp,int err; frame->bp =0 ; + frame->ret_addr = (unsigned long ) ret_from_fork; p->thread.sp = (unsigned long ) fork_frame; p->thread.sp0 = (unsigned long ) (childregs+1 );memset (p->thread.ptrace_bps,0 ,sizeof (p->thread.ptrace_bps));
@@ -146,19 +147,12 @@ int copy_thread_tls(unsigned long clone_flags, unsigned long sp,if (unlikely (p->flags &PF_KTHREAD )) {/* kernel thread */ memset (childregs,0 ,sizeof (struct pt_regs )); - frame->ret_addr = (unsigned long ) ret_from_kernel_thread; -task_user_gs (p) = __KERNEL_STACK_CANARY; - childregs->ds = __USER_DS; - childregs->es = __USER_DS; - childregs->fs = __KERNEL_PERCPU; - childregs->bx = sp;/* function */ - childregs->bp = arg; - childregs->orig_ax =-1 ; - childregs->cs = __KERNEL_CS |get_kernel_rpl (); - childregs->flags =X86_EFLAGS_IF |X86_EFLAGS_FIXED ; + frame->bx = sp;/* function */ + frame->di = arg; p->thread.io_bitmap_ptr =NULL ;return 0 ; } + frame->bx =0 ; *childregs = *current_pt_regs (); childregs->ax =0 ;if (sp)
arch/x86/kernel/process_64.c +3 -7
@@ -165,15 +165,11 @@ int copy_thread_tls(unsigned long clone_flags, unsigned long sp,if (unlikely (p->flags &PF_KTHREAD )) {/* kernel thread */ memset (childregs,0 ,sizeof (struct pt_regs )); - childregs->sp = (unsigned long )childregs; - childregs->ss = __KERNEL_DS; - childregs->bx = sp;/* function */ - childregs->bp = arg; - childregs->orig_ax =-1 ; - childregs->cs = __KERNEL_CS |get_kernel_rpl (); - childregs->flags =X86_EFLAGS_IF |X86_EFLAGS_FIXED ; + frame->bx = sp;/* function */ + frame->r12 = arg;return 0 ; } + frame->bx =0 ; *childregs = *current_pt_regs (); childregs->ax =0 ;
Signature · 1 line
2.5.5
Re: [PATCH v2 5/6] x86: Pass kernel thread parameters in fork_frame
Quoted 3 lines · Brian Gerst
Instead of setting up a fake
pt_regscontext, put the kernel thread function pointer and arg into the unused callee-restored registers of structfork_frame.
This seems generally okay.
Quoted 4 lines
Signed-off-by Brian Gerst <[email protected]>
arch/x86/kernel/process_32.c int copy_thread_tls(unsigned long clone_flags, unsigned long sp, · line 155
Quoted 10 lines
arch/x86/kernel/process_32.c +0
-5@@ -146,19 +147,12 @@ int copy_thread_tls(unsigned long clone_flags, unsigned long sp,if (unlikely (p->flags &PF_KTHREAD )) {/* kernel thread */ memset (childregs,0 ,sizeof (struct pt_regs )); - frame->ret_addr = (unsigned long ) ret_from_kernel_thread; -task_user_gs (p) = __KERNEL_STACK_CANARY; - childregs->ds = __USER_DS; - childregs->es = __USER_DS; - childregs->fs = __KERNEL_PERCPU;
Is the idea that do_execve promises to initialize all these fields to something sensible if the kernel thread in question tries to return to user mode?
--Andy
Re: [PATCH v2 5/6] x86: Pass kernel thread parameters in fork_frame
Quoted 22 lines · Andy Lutomirski
Quoted 3 lines · Brian Gerst
Instead of setting up a fake
pt_regscontext, put the kernel thread function pointer and arg into the unused callee-restored registers of structfork_frame.This seems generally okay.
Quoted 4 lines
Signed-off-by Brian Gerst <[email protected]>arch/x86/kernel/process_32.c int copy_thread_tls(unsigned long clone_flags, unsigned long sp, · line 155
Quoted 10 lines
arch/x86/kernel/process_32.c +0
-5@@ -146,19 +147,12 @@ int copy_thread_tls(unsigned long clone_flags, unsigned long sp,if (unlikely (p->flags &PF_KTHREAD )) {/* kernel thread */ memset (childregs,0 ,sizeof (struct pt_regs )); - frame->ret_addr = (unsigned long ) ret_from_kernel_thread; -task_user_gs (p) = __KERNEL_STACK_CANARY; - childregs->ds = __USER_DS; - childregs->es = __USER_DS; - childregs->fs = __KERNEL_PERCPU;
Quoted 5 lines
Is the idea that
do_execvepromises to initialize all these fields to something sensible if the kernel thread in question tries to return to user mode?--Andy
Yes, do_execve() should be setting the full pt_regs.
--
Brian Gerst
Re: [PATCH v2 5/6] x86: Pass kernel thread parameters in fork_frame
In reply to Brian GerstQuoted 12 lines · Brian Gerst
Instead of setting up a fake
pt_regscontext, put the kernel thread function pointer and arg into the unused callee-restored registers of structfork_frame.Signed-off-by Brian Gerst <[email protected]>--- arch/x86/entry/entry_32.S | 31 +++++++++++++++---------------- arch/x86/entry/entry_64.S | 35 ++++++++++++++++------------------- arch/x86/kernel/process_32.c | 14 ++++---------- arch/x86/kernel/process_64.c | 10 +++------- 4 files changed, 38 insertions(+), 52 deletions(-)
...
Quoted 46 lines
arch/x86/entry/entry_64.S +7
-19@@ -404,37 +404,34 @@ END(__switch_to_asm) * A newly forked process directly context switches into this address. * * rax: prev task we switched from + * rbx: kernel thread func + * r12: kernel thread arg */ ENTRY(ret_from_fork) movq %rax, %rdi call schedule_tail /* rdi: 'prev' task parameter */ - testb $3, CS(%rsp) /* from kernel_thread? */ + testq %rbx, %rbx /* from kernel_thread? */ jnz 1f - /* - * We came from kernel_thread. This code path is quite twisted, and - * someone should clean it up. - * - * copy_thread_tls stashes the function pointer in RBX and the - * parameter to be passed in RBP. The called function is permitted - * to call do_execve and thereby jump to user mode. - */ - movq RBP(%rsp), %rdi - call *RBX(%rsp) - movl $0, RAX(%rsp) - - /* - * Fall through as though we're exiting a syscall. This makes a - * twisted sort of sense if we just called do_execve. - */ - -1: +2: movq %rsp, %rdi call syscall_return_slowpath /* returns with IRQs disabled */ TRACE_IRQS_ON /* user mode is traced as IRQS on */ SWAPGS jmp restore_regs_and_iret + +1: + /* kernel thread */
arch/x86/entry/entry_64.S END(__switch_to_asm) · line 435
Quoted 10 lines
arch/x86/entry/entry_64.S +9
-0+ movq %r12, %rdi + call *%rbx + /* + * A kernel thread is allowed to return here after successfully + * calling do_execve(). Exit to userspace to complete the execve() + * syscall. + */ + movq $0, RAX(%rsp) + jmp 2b END(ret_from_fork)
Wouldn't it be simpler to reverse the JNZ after the TESTB above and thus have a single label and simpler code flow (pasting the whole thing here because a diff is unreadable):
ENTRY(ret_from_fork)
movq %rax, %rdi
call schedule_tail /* rdi: 'prev' task parameter */
testq %rbx, %rbx /* from kernel_thread? */
jz 1f
/* kernel thread */
movq %r12, %rdi
call *%rbx
/*
* A kernel thread is allowed to return here after successfully
* calling do_execve(). Exit to userspace to complete the execve()
* syscall.
*/
movq $0, RAX(%rsp)
1:
movq %rsp, %rdi
call syscall_return_slowpath /* returns with IRQs disabled */
TRACE_IRQS_ON /* user mode is traced as IRQS on */
SWAPGS
jmp restore_regs_and_iretEND(ret_from_fork)
It boots fine in my guest this way too.
Ditto for 32-bit.
Signature · 7 lines
Regards/Gruss,
Boris.
ECO tip #101: Trim your mails when you reply.
SUSE Linux GmbH, GF: Felix Imendörffer, Jane Smithard, Graham Norton, HRB 21284 (AG Nürnberg)
--Re: [PATCH v2 5/6] x86: Pass kernel thread parameters in fork_frame
Quoted 102 lines · Borislav Petkov
Quoted 12 lines · Brian Gerst
Instead of setting up a fake
pt_regscontext, put the kernel thread function pointer and arg into the unused callee-restored registers of structfork_frame.Signed-off-by Brian Gerst <[email protected]>--- arch/x86/entry/entry_32.S | 31 +++++++++++++++---------------- arch/x86/entry/entry_64.S | 35 ++++++++++++++++------------------- arch/x86/kernel/process_32.c | 14 ++++---------- arch/x86/kernel/process_64.c | 10 +++------- 4 files changed, 38 insertions(+), 52 deletions(-)...
Quoted 46 lines
arch/x86/entry/entry_64.S +7
-19@@ -404,37 +404,34 @@ END(__switch_to_asm) * A newly forked process directly context switches into this address. * * rax: prev task we switched from + * rbx: kernel thread func + * r12: kernel thread arg */ ENTRY(ret_from_fork) movq %rax, %rdi call schedule_tail /* rdi: 'prev' task parameter */ - testb $3, CS(%rsp) /* from kernel_thread? */ + testq %rbx, %rbx /* from kernel_thread? */ jnz 1f - /* - * We came from kernel_thread. This code path is quite twisted, and - * someone should clean it up. - * - * copy_thread_tls stashes the function pointer in RBX and the - * parameter to be passed in RBP. The called function is permitted - * to call do_execve and thereby jump to user mode. - */ - movq RBP(%rsp), %rdi - call *RBX(%rsp) - movl $0, RAX(%rsp) - - /* - * Fall through as though we're exiting a syscall. This makes a - * twisted sort of sense if we just called do_execve. - */ - -1: +2: movq %rsp, %rdi call syscall_return_slowpath /* returns with IRQs disabled */ TRACE_IRQS_ON /* user mode is traced as IRQS on */ SWAPGS jmp restore_regs_and_iret + +1: + /* kernel thread */arch/x86/entry/entry_64.S END(__switch_to_asm) · line 435
Quoted 10 lines
arch/x86/entry/entry_64.S +9
-0+ movq %r12, %rdi + call *%rbx + /* + * A kernel thread is allowed to return here after successfully + * calling do_execve(). Exit to userspace to complete the execve() + * syscall. + */ + movq $0, RAX(%rsp) + jmp 2b END(ret_from_fork)Wouldn't it be simpler to reverse the JNZ after the TESTB above and thus have a single label and simpler code flow (pasting the whole thing here because a diff is unreadable):
ENTRY(ret_from_fork)
movq %rax, %rdi
callschedule_tail/* rdi: 'prev' task parameter */testq %rbx, %rbx /* from
kernel_thread? */
jz 1f/* kernel thread */
movq %r12, %rdi
call *%rbx
/*
* A kernel thread is allowed to return here after successfully
* callingdo_execve(). Exit to userspace to complete theexecve()
* syscall.
*/
movq $0, RAX(%rsp)1:
movq %rsp, %rdi
callsyscall_return_slowpath/* returns with IRQs disabled */
TRACE_IRQS_ON/* user mode is traced as IRQS on */
SWAPGS
jmprestore_regs_and_iretEND(ret_from_fork)
Quoted 1 line
It boots fine in my guest this way too.
The idea was to put the uncommon case (kernel thread) out of line for performance reasons.
--
Brian Gerst
Borislav Petkov
maintainerRe: [PATCH v2 5/6] x86: Pass kernel thread parameters in fork_frame
Quoted 2 lines · Brian Gerst
The idea was to put the uncommon case (kernel thread) out of line for performance reasons.
A comment saying so wouldn't hurt...
Signature · 5 lines
Regards/Gruss,
Boris.
ECO tip #101: Trim your mails when you reply.
--Re: [PATCH v2 5/6] x86: Pass kernel thread parameters in fork_frame
Quoted 5 lines · Borislav Petkov
Quoted 2 lines · Brian Gerst
The idea was to put the uncommon case (kernel thread) out of line for performance reasons.
A comment saying so wouldn't hurt...
This is a fairly common pattern. Do we have to document every case of it?
--
Brian Gerst
Re: [PATCH v2 5/6] x86: Pass kernel thread parameters in fork_frame
Quoted 8 lines · Brian Gerst
Quoted 5 lines · Borislav Petkov
Quoted 2 lines · Brian Gerst
The idea was to put the uncommon case (kernel thread) out of line for performance reasons.
A comment saying so wouldn't hurt...
This is a fairly common pattern. Do we have to document every case of it?
Yeah, would be good to do that in general, so that people can re-evaluate whether 'this is rare' is still true years down the line. For newly touched code it makes sense to add a minimal comment that explains what is rare about the branch and so.
Thanks,
Ingo
PATCH v2 6/6 x86: Fix thread_saved_pc()
In reply to Brian Gerstthread_saved_pc() was using a completely bogus method to get the return address. Since switch_to() was previously inlined, there was no sane way to know where on the stack the return address was stored. Now with the frame of a sleeping thread well defined, this can be implemented correctly.
--- arch/x86/include/asm/processor.h | 10 ++-------- arch/x86/kernel/process.c | 10 ++++++++++ arch/x86/kernel/process_32.c | 8 -------- 3 files changed, 12 insertions(+), 16 deletions(-)
arch/x86/include/asm/processor.h +2 -8
@@ -716,8 +716,6 @@ static inline void spin_lock_prefetch(const void *x) .io_bitmap_ptr = NULL, \ } -extern unsigned long thread_saved_pc (struct task_struct *tsk); -/* * TOP_OF_KERNEL_STACK_PADDING reserves 8 bytes on top of the ring0 stack. * This is necessary to guarantee that the entire "struct pt_regs"
@@ -767,17 +765,13 @@ extern unsigned long thread_saved_pc(struct task_struct *tsk); .sp0 = TOP_OF_INIT_STACK \ } -/* -* Return saved PC of a blocked thread. -* What is this good for? it will be always the scheduler or ret_from_fork. -*/ -#define thread_saved_pc (t) READ_ONCE_NOCHECK(*(unsigned long *)((t)->thread.sp - 8)) -#define task_pt_regs (tsk) ((struct pt_regs *)(tsk)->thread.sp0 - 1)extern unsigned long KSTK_ESP (struct task_struct *task);#endif /* CONFIG_X86_64 */ +extern unsigned long thread_saved_pc (struct task_struct *tsk); +extern void start_thread (struct pt_regs *regs,unsigned long new_ip,unsigned long new_sp);
arch/x86/kernel/process.c +10 -0
@@ -513,6 +513,16 @@ unsigned long arch_randomize_brk(struct mm_struct *mm) }/* +* Return saved PC of a blocked thread. +*/ +unsigned long thread_saved_pc (struct task_struct *tsk) +{ +struct inactive_task_frame *frame = + (struct inactive_task_frame *)READ_ONCE (tsk->thread.sp); +return READ_ONCE_NOCHECK (frame->ret_addr); +} + +/* * Called from fs/proc with a reference on @p to find the function * which called into schedule(). This needs to be done carefully * because the task might wake up and we might look at a stack
arch/x86/kernel/process_32.c +0 -8
@@ -58,14 +58,6 @@ asmlinkage void ret_from_fork(void) __asm__("ret_from_fork"); asmlinkage void ret_from_kernel_thread(void) __asm__("ret_from_kernel_thread"); -/* -* Return saved PC of a blocked thread. -*/ -unsigned long thread_saved_pc (struct task_struct *tsk) -{ -return ((unsigned long *)tsk->thread.sp)[3 ]; -} -void __show_regs (struct pt_regs *regs,int all) {unsigned long cr0 =0L , cr2 =0L , cr3 =0L , cr4 =0L ;
Signature · 1 line
2.5.5
Re: [PATCH v2 6/6] x86: Fix thread_saved_pc()
Quoted 53 lines · Brian Gerst
thread_saved_pc()was using a completely bogus method to get the return address. Sinceswitch_to()was previously inlined, there was no sane way to know where on the stack the return address was stored. Now with the frame of a sleeping thread well defined, this can be implemented correctly.Signed-off-by Brian Gerst <[email protected]>--- arch/x86/include/asm/processor.h | 10 ++-------- arch/x86/kernel/process.c | 10 ++++++++++ arch/x86/kernel/process_32.c | 8 -------- 3 files changed, 12 insertions(+), 16 deletions(-)arch/x86/include/asm/processor.h +2
-8@@ -716,8 +716,6 @@ static inline void spin_lock_prefetch(const void *x) .io_bitmap_ptr = NULL, \ } -extern unsigned long thread_saved_pc (struct task_struct *tsk); -/* * TOP_OF_KERNEL_STACK_PADDING reserves 8 bytes on top of the ring0 stack. * This is necessary to guarantee that the entire "struct pt_regs" @@ -767,17 +765,13 @@ extern unsigned long thread_saved_pc(struct task_struct *tsk); .sp0 = TOP_OF_INIT_STACK \ } -/* -* Return saved PC of a blocked thread. -* What is this good for? it will be always the scheduler or ret_from_fork. -*/ -#define thread_saved_pc (t) READ_ONCE_NOCHECK(*(unsigned long *)((t)->thread.sp - 8)) -#define task_pt_regs (tsk) ((struct pt_regs *)(tsk)->thread.sp0 - 1)extern unsigned long KSTK_ESP (struct task_struct *task);#endif /* CONFIG_X86_64 */ +extern unsigned long thread_saved_pc (struct task_struct *tsk); +extern void start_thread (struct pt_regs *regs,unsigned long new_ip,unsigned long new_sp);arch/x86/kernel/process.c +0
-0@@ -513,6 +513,16 @@ unsigned long arch_randomize_brk(struct mm_struct *mm) }/*
arch/x86/kernel/process.c unsigned long arch_randomize_brk(struct mm_struct *mm) · line 525
Quoted 10 lines
arch/x86/kernel/process.c +10
-0+* Return saved PC of a blocked thread. +*/ +unsigned long thread_saved_pc (struct task_struct *tsk) +{ +struct inactive_task_frame *frame = + (struct inactive_task_frame *)READ_ONCE (tsk->thread.sp); +return READ_ONCE_NOCHECK (frame->ret_addr); +} + +/*
I would agree with the above (removed) comment:
"What is this good for? it will be always the scheduler or ret_from_fork."
And I'd guess the same is true for all the arches which have to implement it. Maybe this function (and its single call site in sched_show_task()) should just be removed altogether?
Signature · 1 line
Josh
Re: [PATCH v2 6/6] x86: Fix thread_saved_pc()
Quoted 67 lines · Josh Poimboeuf
Quoted 53 lines · Brian Gerst
thread_saved_pc()was using a completely bogus method to get the return address. Sinceswitch_to()was previously inlined, there was no sane way to know where on the stack the return address was stored. Now with the frame of a sleeping thread well defined, this can be implemented correctly.Signed-off-by Brian Gerst <[email protected]>--- arch/x86/include/asm/processor.h | 10 ++-------- arch/x86/kernel/process.c | 10 ++++++++++ arch/x86/kernel/process_32.c | 8 -------- 3 files changed, 12 insertions(+), 16 deletions(-)arch/x86/include/asm/processor.h +2
-8@@ -716,8 +716,6 @@ static inline void spin_lock_prefetch(const void *x) .io_bitmap_ptr = NULL, \ } -extern unsigned long thread_saved_pc (struct task_struct *tsk); -/* * TOP_OF_KERNEL_STACK_PADDING reserves 8 bytes on top of the ring0 stack. * This is necessary to guarantee that the entire "struct pt_regs" @@ -767,17 +765,13 @@ extern unsigned long thread_saved_pc(struct task_struct *tsk); .sp0 = TOP_OF_INIT_STACK \ } -/* -* Return saved PC of a blocked thread. -* What is this good for? it will be always the scheduler or ret_from_fork. -*/ -#define thread_saved_pc (t) READ_ONCE_NOCHECK(*(unsigned long *)((t)->thread.sp - 8)) -#define task_pt_regs (tsk) ((struct pt_regs *)(tsk)->thread.sp0 - 1)extern unsigned long KSTK_ESP (struct task_struct *task);#endif /* CONFIG_X86_64 */ +extern unsigned long thread_saved_pc (struct task_struct *tsk); +extern void start_thread (struct pt_regs *regs,unsigned long new_ip,unsigned long new_sp);arch/x86/kernel/process.c +0
-0@@ -513,6 +513,16 @@ unsigned long arch_randomize_brk(struct mm_struct *mm) }/* arch/x86/kernel/process.c unsigned long arch_randomize_brk(struct mm_struct *mm) · line 525
Quoted 10 lines
arch/x86/kernel/process.c +10
-0+* Return saved PC of a blocked thread. +*/ +unsigned long thread_saved_pc (struct task_struct *tsk) +{ +struct inactive_task_frame *frame = + (struct inactive_task_frame *)READ_ONCE (tsk->thread.sp); +return READ_ONCE_NOCHECK (frame->ret_addr); +} + +/* I would agree with the above (removed) comment:
Quoted 5 lines
"What is this good for? it will be always the scheduler or
ret_from_fork."And I'd guess the same is true for all the arches which have to implement it. Maybe this function (and its single call site in
sched_show_task()) should just be removed altogether?
I didn't really want to stray down that path with this series. This just makes it functional again. the usefulness is still open for debate.
--
Brian Gerst
Josh Poimboeuf
Reviewed-byRe: [PATCH v2 6/6] x86: Fix thread_saved_pc()
Quoted 70 lines · Brian Gerst
Quoted 69 lines · Josh Poimboeuf
Quoted 53 lines · Brian Gerst
thread_saved_pc()was using a completely bogus method to get the return address. Sinceswitch_to()was previously inlined, there was no sane way to know where on the stack the return address was stored. Now with the frame of a sleeping thread well defined, this can be implemented correctly.Signed-off-by Brian Gerst <[email protected]>--- arch/x86/include/asm/processor.h | 10 ++-------- arch/x86/kernel/process.c | 10 ++++++++++ arch/x86/kernel/process_32.c | 8 -------- 3 files changed, 12 insertions(+), 16 deletions(-)arch/x86/include/asm/processor.h +2
-8@@ -716,8 +716,6 @@ static inline void spin_lock_prefetch(const void *x) .io_bitmap_ptr = NULL, \ } -extern unsigned long thread_saved_pc (struct task_struct *tsk); -/* * TOP_OF_KERNEL_STACK_PADDING reserves 8 bytes on top of the ring0 stack. * This is necessary to guarantee that the entire "struct pt_regs" @@ -767,17 +765,13 @@ extern unsigned long thread_saved_pc(struct task_struct *tsk); .sp0 = TOP_OF_INIT_STACK \ } -/* -* Return saved PC of a blocked thread. -* What is this good for? it will be always the scheduler or ret_from_fork. -*/ -#define thread_saved_pc (t) READ_ONCE_NOCHECK(*(unsigned long *)((t)->thread.sp - 8)) -#define task_pt_regs (tsk) ((struct pt_regs *)(tsk)->thread.sp0 - 1)extern unsigned long KSTK_ESP (struct task_struct *task);#endif /* CONFIG_X86_64 */ +extern unsigned long thread_saved_pc (struct task_struct *tsk); +extern void start_thread (struct pt_regs *regs,unsigned long new_ip,unsigned long new_sp);arch/x86/kernel/process.c +0
-0@@ -513,6 +513,16 @@ unsigned long arch_randomize_brk(struct mm_struct *mm) }/* arch/x86/kernel/process.c unsigned long arch_randomize_brk(struct mm_struct *mm) · line 525
Quoted 10 lines
arch/x86/kernel/process.c +10
-0+* Return saved PC of a blocked thread. +*/ +unsigned long thread_saved_pc (struct task_struct *tsk) +{ +struct inactive_task_frame *frame = + (struct inactive_task_frame *)READ_ONCE (tsk->thread.sp); +return READ_ONCE_NOCHECK (frame->ret_addr); +} + +/* I would agree with the above (removed) comment:
"What is this good for? it will be always the scheduler or
ret_from_fork."
Quoted 7 lines
Quoted 3 lines
And I'd guess the same is true for all the arches which have to implement it. Maybe this function (and its single call site in
sched_show_task()) should just be removed altogether?I didn't really want to stray down that path with this series. This just makes it functional again. the usefulness is still open for debate.
Fair enough.
Signature · 1 line
Josh
Quoted 8 lines · Brian Gerst
This patch set simplifies the
switch_to()code, by moving the stack switch code out of line into an asm stub before calling__switch_to(). This ends up being more readable, and using the C calling convention instead of clobbering all registers improves code generation. It also allows newly forked processes to construct a special stack frame to seamlessly flow toret_from_fork, instead of using a test and branch, or an unbalanced call/ret.
Quoted 4 lines
Changes from v1:
- Added structinactive_task_frame
- Added comments about kernel threads returning to userspace
- Cleaned up some incorrect uses of thread.sp
I forgot to also add:
- Rearranged inactive stack frame so that BP (frame pointer) is in the natural position right below the return address. This should take care of unwinding issues Josh raised.
--
Brian Gerst