x86/fpu changes for v4.1
Linus,
Please pull the latest x86-fpu-for-linus git tree from:
git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git x86-fpu-for-linus
# HEAD: 7fc253e277ecf1ea57c2d670bdbcda3dffd19453 x86/fpu: Kill eager_fpu_init_bp()
Various x86 FPU handling cleanups, refactorings and fixes. (Borislav Petkov, Oleg Nesterov, Rik van Riel)
Thanks,
Ingo
------------------>
Borislav Petkov (2):
x86/fpu: Fold __drop_fpu() into its sole user x86/fpu: Rename drop_init_fpu() to fpu_reset_state()
Oleg Nesterov (14):
x86/fpu: Don't reset thread.fpu_counter
x86/fpu: Don't do __thread_fpu_end() if use_eager_fpu()
x86/fpu: Change math_error() to use unlazy_fpu(), kill (now) unused save_init_fpu() x86/fpu: __kernel_fpu_begin() should clear fpu_owner_task even if use_eager_fpu() x86/fpu: Always allow FPU in interrupt if use_eager_fpu()
x86/fpu: Don't abuse FPU in kernel threads if use_eager_fpu() x86/fpu: Change xstateregs_get()/set() to use ->xsave.i387 rather than ->fxsave x86/fpu: Factor out memset(xstate, 0) in fpu_finit() paths
x86/fpu: Document user_fpu_begin()
x86/fpu: Introduce restore_init_xstate()
x86/fpu: Use restore_init_xstate() instead of math_state_restore() on kthread exec x86/fpu: Don't abuse drop_init_fpu() in flush_thread()
x86/fpu: Don't allocate fpu->state for swapper/0
x86/fpu: Kill eager_fpu_init_bp()
Rik van Riel (5):
x86/fpu: Move lazy restore functions up a few lines x86/fpu: Introduce task_disable_lazy_fpu_restore() helper x86/fpu: Use an explicit if/else in switch_fpu_prepare() x86/fpu: Use task_disable_lazy_fpu_restore() helper x86/fpu: Also check fpu_lazy_restore() when use_eager_fpu()
arch/x86/include/asm/fpu-internal.h | 130 ++++++++++++++++++------------------ arch/x86/kernel/i387.c | 54 ++++++++------- arch/x86/kernel/process.c | 19 ++++-- arch/x86/kernel/signal.c | 2 +- arch/x86/kernel/traps.c | 4 +- arch/x86/kernel/xsave.c | 39 +++-------- 6 files changed, 117 insertions(+), 131 deletions(-)
arch/x86/include/asm/fpu-internal.h +66 -64
@@ -67,6 +67,34 @@ extern void finit_soft_fpu(struct i387_soft_struct *soft);static inline void finit_soft_fpu (struct i387_soft_struct *soft) {}#endif +/* +* Must be run with preemption disabled: this clears the fpu_owner_task, +* on this CPU. +* +* This will disable any lazy FPU state restore of the current FPU state, +* but if the current thread owns the FPU, it will still be saved by. +*/ +static inline void __cpu_disable_lazy_restore (unsigned int cpu) +{ +per_cpu (fpu_owner_task, cpu) =NULL ; +} + +/* +* Used to indicate that the FPU state in memory is newer than the FPU +* state in registers, and the FPU state should be reloaded next time the +* task is run. Only safe on the current task, or non-running tasks. +*/ +static inline void task_disable_lazy_fpu_restore (struct task_struct *tsk) +{ + tsk->thread.fpu.last_cpu = ~0 ; +} + +static inline int fpu_lazy_restore (struct task_struct *new,unsigned int cpu) +{ +return new ==this_cpu_read_stable (fpu_owner_task) && + cpu == new->thread.fpu.last_cpu; +} +static inline int is_ia32_compat_frame (void ) { return config_enabled(CONFIG_IA32_EMULATION) &&
@@ -107,7 +135,6 @@ static __always_inline __pure bool use_fxsr(void)static inline void fx_finit (struct i387_fxsave_struct *fx) { -memset (fx,0 , xstate_size); fx->cwd =0x37f ; fx->mxcsr =MXCSR_DEFAULT ; }
@@ -351,8 +378,14 @@ static inline void __thread_fpu_begin(struct task_struct *tsk)__thread_set_has_fpu (tsk); } -static inline void __drop_fpu (struct task_struct *tsk) +static inline void drop_fpu (struct task_struct *tsk) { +/* +* Forget coprocessor state.. +*/ +preempt_disable (); + tsk->thread.fpu_counter =0 ; +if (__thread_has_fpu (tsk)) {/* Ignore delayed exceptions from user space */ asm volatile("1: fwait\n"
@@ -360,30 +393,29 @@ static inline void __drop_fpu(struct task_struct *tsk) _ASM_EXTABLE(1b, 2b));__thread_fpu_end (tsk); } -} -static inline void drop_fpu (struct task_struct *tsk) -{ -/* -* Forget coprocessor state.. -*/ -preempt_disable (); - tsk->thread.fpu_counter =0 ; -__drop_fpu (tsk);clear_stopped_child_used_math (tsk);preempt_enable (); } -static inline void drop_init_fpu (struct task_struct *tsk) +static inline void restore_init_xstate (void ) +{ +if (use_xsave ()) +xrstor_state (init_xstate_buf,-1 ); +else +fxrstor_checking (&init_xstate_buf->i387); +} + +/* +* Reset the FPU state in the eager case and drop it in the lazy case (later use +* will reinit it). +*/ +static inline void fpu_reset_state (struct task_struct *tsk) {if (!use_eager_fpu ())drop_fpu (tsk); -else { -if (use_xsave ()) -xrstor_state (init_xstate_buf,-1 ); -else -fxrstor_checking (&init_xstate_buf->i387); - } +else +restore_init_xstate (); }/*
@@ -400,24 +432,6 @@ static inline void drop_init_fpu(struct task_struct *tsk)*/ typedef struct {int preload; }fpu_switch_t ; -/* -* Must be run with preemption disabled: this clears the fpu_owner_task, -* on this CPU. -* -* This will disable any lazy FPU state restore of the current FPU state, -* but if the current thread owns the FPU, it will still be saved by. -*/ -static inline void __cpu_disable_lazy_restore (unsigned int cpu) -{ -per_cpu (fpu_owner_task, cpu) =NULL ; -} - -static inline int fpu_lazy_restore (struct task_struct *new,unsigned int cpu) -{ -return new ==this_cpu_read_stable (fpu_owner_task) && - cpu == new->thread.fpu.last_cpu; -} -static inline fpu_switch_t switch_fpu_prepare (struct task_struct *old,struct task_struct *new,int cpu) {fpu_switch_t fpu;
@@ -426,13 +440,17 @@ static inline fpu_switch_t switch_fpu_prepare(struct task_struct *old, struct ta* If the task has used the math, pre-load the FPU on xsave processors * or if the past 5 consecutive context-switches used math. */ - fpu.preload =tsk_used_math (new) && (use_eager_fpu () || - new->thread.fpu_counter >5 ); + fpu.preload =tsk_used_math (new) && + (use_eager_fpu () || new->thread.fpu_counter >5 ); +if (__thread_has_fpu (old)) {if (!__save_init_fpu (old)) - cpu = ~0 ; - old->thread.fpu.last_cpu = cpu; - old->thread.fpu.has_fpu =0 ;/* But leave fpu_owner_task! */ +task_disable_lazy_fpu_restore (old); +else + old->thread.fpu.last_cpu = cpu; + +/* But leave fpu_owner_task! */ + old->thread.fpu.has_fpu =0 ;/* Don't change CR0.TS if we just switch! */ if (fpu.preload) {
@@ -443,10 +461,10 @@ static inline fpu_switch_t switch_fpu_prepare(struct task_struct *old, struct tastts (); }else { old->thread.fpu_counter =0 ; - old->thread.fpu.last_cpu = ~0 ; +task_disable_lazy_fpu_restore (old);if (fpu.preload) { new->thread.fpu_counter++; -if (!use_eager_fpu () &&fpu_lazy_restore (new, cpu)) +if (fpu_lazy_restore (new, cpu)) fpu.preload =0 ;else prefetch (new->thread.fpu.state);
@@ -466,7 +484,7 @@ static inline void switch_fpu_finish(struct task_struct *new, fpu_switch_t fpu) {if (fpu.preload) {if (unlikely (restore_fpu_checking (new))) -drop_init_fpu (new); +fpu_reset_state (new); } }
@@ -495,10 +513,12 @@ static inline int restore_xstate_sig(void __user *buf, int ia32_frame) }/* -* Need to be preemption-safe. +* Needs to be preemption-safe. * * NOTE! user_fpu_begin() must be used only immediately before restoring -* it. This function does not do any save/restore on their own. +* the save state. It does not do any saving/restoring on its own. In +* lazy FPU mode, it is just an optimization to avoid a #NM exception, +* the task can lose the FPU right after preempt_enable(). */ static inline void user_fpu_begin (void ) {
@@ -520,24 +540,6 @@ static inline void __save_fpu(struct task_struct *tsk) }/* -* These disable preemption on their own and are safe -*/ -static inline void save_init_fpu (struct task_struct *tsk) -{ -WARN_ON_ONCE (!__thread_has_fpu (tsk)); - -if (use_eager_fpu ()) { -__save_fpu (tsk); -return ; - } - -preempt_disable (); -__save_init_fpu (tsk); -__thread_fpu_end (tsk); -preempt_enable (); -} - -/* * i387 state interaction */ static inline unsigned short get_fpu_cwd (struct task_struct *tsk)
arch/x86/kernel/i387.c +26 -28
135 lines not on this page · raw message
arch/x86/kernel/process.c +12 -7
@@ -69,8 +69,8 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) dst->thread.fpu_counter =0 ; dst->thread.fpu.has_fpu =0 ; - dst->thread.fpu.last_cpu = ~0 ; dst->thread.fpu.state =NULL ; +task_disable_lazy_fpu_restore (dst);if (tsk_used_math (src)) {int err =fpu_alloc (&dst->thread.fpu);if (err)
@@ -131,13 +131,18 @@ void flush_thread(void)flush_ptrace_hw_breakpoint (tsk);memset (tsk->thread.tls_array,0 ,sizeof (tsk->thread.tls_array)); -drop_init_fpu (tsk); -/* -* Free the FPU state for non xsave platforms. They get reallocated -* lazily at the first use. -*/ -if (!use_eager_fpu ()) + +if (!use_eager_fpu ()) { +/* FPU state will be reallocated lazily at the first use. */ +drop_fpu (tsk);free_thread_xstate (tsk); + }else if (!used_math ()) { +/* kthread execs. TODO: cleanup this horror. */ +if (WARN_ON (init_fpu (tsk))) +force_sig (SIGKILL , tsk); +user_fpu_begin (); +restore_init_xstate (); + } }static void hard_disable_TSC (void )
arch/x86/kernel/signal.c +1 -1
@@ -679,7 +679,7 @@ handle_signal(struct ksignal *ksig, struct pt_regs *regs)* Ensure the signal handler starts with the new fpu state. */ if (used_math ()) -drop_init_fpu (current); +fpu_reset_state (current); }signal_setup_done (failed, ksig,test_thread_flag (TIF_SINGLESTEP )); }
arch/x86/kernel/traps.c +2 -2
@@ -734,7 +734,7 @@ static void math_error(struct pt_regs *regs, int error_code, int trapnr)/* * Save the info for the exception handler and clear the error. */ -save_init_fpu (task); +unlazy_fpu (task); task->thread.trap_nr = trapnr; task->thread.error_code = error_code; info.si_signo =SIGFPE ;
@@ -863,7 +863,7 @@ void math_state_restore(void)kernel_fpu_disable ();__thread_fpu_begin (tsk);if (unlikely (restore_fpu_checking (tsk))) { -drop_init_fpu (tsk); +fpu_reset_state (tsk);force_sig_info (SIGSEGV ,SEND_SIG_PRIV , tsk); }else { tsk->thread.fpu_counter++;
arch/x86/kernel/xsave.c +10 -29
@@ -342,7 +342,7 @@ int __restore_xstate_sig(void __user *buf, void __user *buf_fx, int size) config_enabled(CONFIG_IA32_EMULATION));if (!buf) { -drop_init_fpu (tsk); +fpu_reset_state (tsk);return 0 ; }
@@ -416,7 +416,7 @@ int __restore_xstate_sig(void __user *buf, void __user *buf_fx, int size)*/ user_fpu_begin ();if (restore_user_xstate (buf_fx, xstate_bv, fx_only)) { -drop_init_fpu (tsk); +fpu_reset_state (tsk);return -1 ; } }
@@ -678,19 +678,13 @@ void xsave_init(void)this_func (); } -static inline void __init eager_fpu_init_bp (void ) -{ - current->thread.fpu.state = -alloc_bootmem_align (xstate_size, __alignof__(struct xsave_struct )); -if (!init_xstate_buf) -setup_init_fpu_buf (); -} - -void eager_fpu_init (void ) +/* +* setup_init_fpu_buf() is __init and it is OK to call it here because +* init_xstate_buf will be unset only once during boot. +*/ +void __init_refok eager_fpu_init(void) { -static __refdata void (*boot_func)(void ) = eager_fpu_init_bp; - -clear_used_math (); +WARN_ON (used_math ());current_thread_info ()->status =0 ;if (eagerfpu ==ENABLE )
@@ -701,21 +695,8 @@ void eager_fpu_init(void)return ; } -if (boot_func) { -boot_func (); - boot_func =NULL ; - } - -/* -* This is same as math_state_restore(). But use_xsave() is -* not yet patched to use math_state_restore(). -*/ -init_fpu (current); -__thread_fpu_begin (current); -if (cpu_has_xsave) -xrstor_state (init_xstate_buf,-1 ); -else -fxrstor_checking (&init_xstate_buf->i387); +if (!init_xstate_buf) +setup_init_fpu_buf (); }/*