big x86 FPU code rewrite

index 21d0b845c8c4..5dbba7224221 100644
@@ -36,8 +36,7 @@
 #include <crypto/ctr.h>
 #include <crypto/lrw.h>
 #include <crypto/xts.h>
-#include <asm/xcr.h>
-#include <asm/fpu/xstate.h>
+#include <asm/fpu/api.h>
 #include <asm/crypto/glue_helper.h>
 
 #define CAST6_PARALLEL_BLOCKS 8

@@ -590,16 +589,10 @@ static struct crypto_alg cast6_algs[10] = { {
 
 static int __init cast6_init(void)
 {
-	u64 xcr0;
+	const char *feature_name;
 
-	if (!cpu_has_avx || !cpu_has_osxsave) {
-		pr_info("AVX instructions are not detected.\n");
-		return -ENODEV;
-	}
-
-	xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
-	if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
-		pr_info("AVX detected but unusable.\n");
+	if (!cpu_has_xfeatures(XSTATE_SSE | XSTATE_YMM, &feature_name)) {
+		pr_info("CPU feature '%s' is not supported.\n", feature_name);
 		return -ENODEV;
 	}
 

index 84db12f052e8..7c48e8b20848 100644
@@ -30,8 +30,6 @@
 #include <crypto/sha.h>
 #include <crypto/sha1_base.h>
 #include <asm/fpu/api.h>
-#include <asm/xcr.h>
-#include <asm/fpu/xstate.h>
 
 
 asmlinkage void sha1_transform_ssse3(u32 *digest, const char *data,

@@ -123,15 +121,9 @@ static struct shash_alg alg = {
 #ifdef CONFIG_AS_AVX
 static bool __init avx_usable(void)
 {
-	u64 xcr0;
-
-	if (!cpu_has_avx || !cpu_has_osxsave)
-		return false;
-
-	xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
-	if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
-		pr_info("AVX detected but unusable.\n");
-
+	if (!cpu_has_xfeatures(XSTATE_SSE | XSTATE_YMM, NULL)) {
+		if (cpu_has_avx)
+			pr_info("AVX detected but unusable.\n");
 		return false;
 	}
 

index aa325fa5c7a6..f226ad41fde1 100644
@@ -20,8 +20,7 @@
 #include <crypto/lrw.h>
 #include <crypto/xts.h>
 #include <crypto/serpent.h>
-#include <asm/xcr.h>
-#include <asm/fpu/xstate.h>
+#include <asm/fpu/api.h>
 #include <asm/crypto/serpent-avx.h>
 #include <asm/crypto/glue_helper.h>
 

@@ -537,16 +536,10 @@ static struct crypto_alg srp_algs[10] = { {
 
 static int __init init(void)
 {
-	u64 xcr0;
+	const char *feature_name;
 
-	if (!cpu_has_avx2 || !cpu_has_osxsave) {
-		pr_info("AVX2 instructions are not detected.\n");
-		return -ENODEV;
-	}
-
-	xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
-	if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
-		pr_info("AVX detected but unusable.\n");
+	if (!cpu_has_xfeatures(XSTATE_SSE | XSTATE_YMM, &feature_name)) {
+		pr_info("CPU feature '%s' is not supported.\n", feature_name);
 		return -ENODEV;
 	}
 

index 6f3f76568bd5..f53ed1dc88ea 100644
@@ -65,10 +65,8 @@
 #include <crypto/mcryptd.h>
 #include <crypto/crypto_wq.h>
 #include <asm/byteorder.h>
-#include <asm/xcr.h>
-#include <asm/fpu/xstate.h>
 #include <linux/hardirq.h>
-#include <asm/fpu/internal.h>
+#include <asm/fpu/api.h>
 #include "sha_mb_ctx.h"
 
 #define FLUSH_INTERVAL 1000 /* in usec */

index 8ec785ecce81..161b51bf267e 100644
@@ -431,6 +431,31 @@ static inline void fpu_reset_state(struct fpu *fpu)
 }
 
 /*
+ * Definitions for the eXtended Control Register instructions
+ */
+
+#define XCR_XFEATURE_ENABLED_MASK	0x00000000
+
+static inline u64 xgetbv(u32 index)
+{
+	u32 eax, edx;
+
+	asm volatile(".byte 0x0f,0x01,0xd0" /* xgetbv */
+		     : "=a" (eax), "=d" (edx)
+		     : "c" (index));
+	return eax + ((u64)edx << 32);
+}
+
+static inline void xsetbv(u32 index, u64 value)
+{
+	u32 eax = value;
+	u32 edx = value >> 32;
+
+	asm volatile(".byte 0x0f,0x01,0xd1" /* xsetbv */
+		     : : "a" (eax), "d" (edx), "c" (index));
+}
+
+/*
  * FPU state switching for scheduling.
  *
  * This is a two-stage process:

49 lines not on this page · raw message

index 0f849229c93b..c087f2d0f2d1 100644
@@ -9,7 +9,6 @@
 #include <asm/fpu/internal.h>
 #include <asm/sigframe.h>
 #include <asm/tlbflush.h>
-#include <asm/xcr.h>
 
 static const char *xfeature_names[] =
 {

index f93ae71416e4..2de55e953842 100644
@@ -41,7 +41,6 @@
 #include <asm/virtext.h>
 #include <asm/mce.h>
 #include <asm/fpu/internal.h>
-#include <asm/xcr.h>
 #include <asm/perf_event.h>
 #include <asm/debugreg.h>
 #include <asm/kexec.h>

index 9ff4df77e069..5c61aae277f9 100644
@@ -61,7 +61,6 @@
 #include <asm/mce.h>
 #include <linux/kernel_stat.h>
 #include <asm/fpu/internal.h> /* Ugh! */
-#include <asm/xcr.h>
 #include <asm/pvclock.h>
 #include <asm/div64.h>
 

index 161b51bf267e..6b6fa46037f8 100644
@@ -139,13 +139,13 @@ static inline void fx_finit(struct i387_fxsave_struct *fx)
 	fx->mxcsr = MXCSR_DEFAULT;
 }
 
-extern void __sanitize_i387_state(struct task_struct *);
+extern void __fpstate_sanitize_xstate(struct task_struct *);
 
-static inline void sanitize_i387_state(struct task_struct *tsk)
+static inline void fpstate_sanitize_xstate(struct task_struct *tsk)
 {
 	if (!use_xsaveopt())
 		return;
-	__sanitize_i387_state(tsk);
+	__fpstate_sanitize_xstate(tsk);
 }
 
 #define user_insn(insn, output, input...)				\

index 8ae4c2450c2b..9ccf2b838de0 100644
@@ -395,7 +395,7 @@ int xfpregs_get(struct task_struct *target, const struct user_regset *regset,
 		return -ENODEV;
 
 	fpu__activate_stopped(fpu);
-	sanitize_i387_state(target);
+	fpstate_sanitize_xstate(target);
 
 	return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
 				   &fpu->state.fxsave, 0, -1);

@@ -412,7 +412,7 @@ int xfpregs_set(struct task_struct *target, const struct user_regset *regset,
 		return -ENODEV;
 
 	fpu__activate_stopped(fpu);
-	sanitize_i387_state(target);
+	fpstate_sanitize_xstate(target);
 
 	ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
 				 &fpu->state.fxsave, 0, -1);

@@ -644,7 +644,7 @@ int fpregs_get(struct task_struct *target, const struct user_regset *regset,
 					   &fpu->state.fsave, 0,
 					   -1);
 
-	sanitize_i387_state(target);
+	fpstate_sanitize_xstate(target);
 
 	if (kbuf && pos == 0 && count == sizeof(env)) {
 		convert_from_fxsr(kbuf, target);

@@ -666,7 +666,7 @@ int fpregs_set(struct task_struct *target, const struct user_regset *regset,
 
 	fpu__activate_stopped(fpu);
 
-	sanitize_i387_state(target);
+	fpstate_sanitize_xstate(target);
 
 	if (!static_cpu_has(X86_FEATURE_FPU))
 		return fpregs_soft_set(target, regset, pos, count, kbuf, ubuf);

index c087f2d0f2d1..fc2ff1239fea 100644
@@ -92,7 +92,7 @@ EXPORT_SYMBOL_GPL(cpu_has_xfeatures);
  * if the corresponding header bit is zero. This is to ensure that user-space doesn't
  * see some stale state in the memory layout during signal handling, debugging etc.
  */
-void __sanitize_i387_state(struct task_struct *tsk)
+void __fpstate_sanitize_xstate(struct task_struct *tsk)
 {
 	struct i387_fxsave_struct *fx = &tsk->thread.fpu.state.fxsave;
 	int feature_bit;

@@ -318,7 +318,7 @@ int save_xstate_sig(void __user *buf, void __user *buf_fx, int size)
 		if (ia32_fxstate)
 			fpu_fxsave(&tsk->thread.fpu);
 	} else {
-		sanitize_i387_state(tsk);
+		fpstate_sanitize_xstate(tsk);
 		if (__copy_to_user(buf_fx, xsave, xstate_size))
 			return -1;
 	}

index 6b6fa46037f8..88fec3f108de 100644
@@ -139,14 +139,7 @@ static inline void fx_finit(struct i387_fxsave_struct *fx)
 	fx->mxcsr = MXCSR_DEFAULT;
 }
 
-extern void __fpstate_sanitize_xstate(struct task_struct *);
-
-static inline void fpstate_sanitize_xstate(struct task_struct *tsk)
-{
-	if (!use_xsaveopt())
-		return;
-	__fpstate_sanitize_xstate(tsk);
-}
+extern void fpstate_sanitize_xstate(struct task_struct *);
 
 #define user_insn(insn, output, input...)				\
 ({									\

index fc2ff1239fea..47b9591947e1 100644
@@ -92,13 +92,13 @@ EXPORT_SYMBOL_GPL(cpu_has_xfeatures);
  * if the corresponding header bit is zero. This is to ensure that user-space doesn't
  * see some stale state in the memory layout during signal handling, debugging etc.
  */
-void __fpstate_sanitize_xstate(struct task_struct *tsk)
+void fpstate_sanitize_xstate(struct task_struct *tsk)
 {
 	struct i387_fxsave_struct *fx = &tsk->thread.fpu.state.fxsave;
 	int feature_bit;
 	u64 xfeatures;
 
-	if (!fx)
+	if (!use_xsaveopt())
 		return;
 
 	xfeatures = tsk->thread.fpu.state.xsave.header.xfeatures;

index 88fec3f108de..da96b0cbfcb3 100644
@@ -139,7 +139,7 @@ static inline void fx_finit(struct i387_fxsave_struct *fx)
 	fx->mxcsr = MXCSR_DEFAULT;
 }
 
-extern void fpstate_sanitize_xstate(struct task_struct *);
+extern void fpstate_sanitize_xstate(struct fpu *fpu);
 
 #define user_insn(insn, output, input...)				\
 ({									\

index 9ccf2b838de0..7e91a6f7564a 100644
@@ -395,7 +395,7 @@ int xfpregs_get(struct task_struct *target, const struct user_regset *regset,
 		return -ENODEV;
 
 	fpu__activate_stopped(fpu);
-	fpstate_sanitize_xstate(target);
+	fpstate_sanitize_xstate(fpu);
 
 	return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
 				   &fpu->state.fxsave, 0, -1);

@@ -412,7 +412,7 @@ int xfpregs_set(struct task_struct *target, const struct user_regset *regset,
 		return -ENODEV;
 
 	fpu__activate_stopped(fpu);
-	fpstate_sanitize_xstate(target);
+	fpstate_sanitize_xstate(fpu);
 
 	ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
 				 &fpu->state.fxsave, 0, -1);

@@ -644,7 +644,7 @@ int fpregs_get(struct task_struct *target, const struct user_regset *regset,
 					   &fpu->state.fsave, 0,
 					   -1);
 
-	fpstate_sanitize_xstate(target);
+	fpstate_sanitize_xstate(fpu);
 
 	if (kbuf && pos == 0 && count == sizeof(env)) {
 		convert_from_fxsr(kbuf, target);

@@ -665,8 +665,7 @@ int fpregs_set(struct task_struct *target, const struct user_regset *regset,
 	int ret;
 
 	fpu__activate_stopped(fpu);
-
-	fpstate_sanitize_xstate(target);
+	fpstate_sanitize_xstate(fpu);
 
 	if (!static_cpu_has(X86_FEATURE_FPU))
 		return fpregs_soft_set(target, regset, pos, count, kbuf, ubuf);

index 47b9591947e1..a8ce38a9d70b 100644
@@ -92,16 +92,16 @@ EXPORT_SYMBOL_GPL(cpu_has_xfeatures);
  * if the corresponding header bit is zero. This is to ensure that user-space doesn't
  * see some stale state in the memory layout during signal handling, debugging etc.
  */
-void fpstate_sanitize_xstate(struct task_struct *tsk)
+void fpstate_sanitize_xstate(struct fpu *fpu)
 {
-	struct i387_fxsave_struct *fx = &tsk->thread.fpu.state.fxsave;
+	struct i387_fxsave_struct *fx = &fpu->state.fxsave;
 	int feature_bit;
 	u64 xfeatures;
 
 	if (!use_xsaveopt())
 		return;
 
-	xfeatures = tsk->thread.fpu.state.xsave.header.xfeatures;
+	xfeatures = fpu->state.xsave.header.xfeatures;
 
 	/*
 	 * None of the feature bits are in init state. So nothing else

@@ -318,7 +318,7 @@ int save_xstate_sig(void __user *buf, void __user *buf_fx, int size)
 		if (ia32_fxstate)
 			fpu_fxsave(&tsk->thread.fpu);
 	} else {
-		fpstate_sanitize_xstate(tsk);
+		fpstate_sanitize_xstate(&tsk->thread.fpu);
 		if (__copy_to_user(buf_fx, xsave, xstate_size))
 			return -1;
 	}

index d6d8f4ca5136..2e0b1b7842ae 100644
@@ -327,7 +327,7 @@ static void __user *get_sigframe(struct ksignal *ksig, struct pt_regs *regs,
 
 		sp = alloc_mathframe(sp, 1, &fx_aligned, &math_size);
 		*fpstate = (struct _fpstate_ia32 __user *) sp;
-		if (save_xstate_sig(*fpstate, (void __user *)fx_aligned,
+		if (copy_fpstate_to_sigframe(*fpstate, (void __user *)fx_aligned,
 				    math_size) < 0)
 			return (void __user *) -1L;
 	}

index da96b0cbfcb3..58c274dfcb62 100644
@@ -523,7 +523,7 @@ static inline void switch_fpu_finish(struct fpu *new_fpu, fpu_switch_t fpu_switc
 /*
  * Signal frame handlers...
  */
-extern int save_xstate_sig(void __user *buf, void __user *fx, int size);
+extern int copy_fpstate_to_sigframe(void __user *buf, void __user *fx, int size);
 extern int __restore_xstate_sig(void __user *buf, void __user *fx, int size);
 
 static inline int xstate_sigframe_size(void)

index a8ce38a9d70b..5178158a03ff 100644
@@ -293,7 +293,7 @@ static inline int save_user_xstate(struct xsave_struct __user *buf)
  * For [f]xsave state, update the SW reserved fields in the [f]xsave frame
  * indicating the absence/presence of the extended state to the user.
  */
-int save_xstate_sig(void __user *buf, void __user *buf_fx, int size)
+int copy_fpstate_to_sigframe(void __user *buf, void __user *buf_fx, int size)
 {
 	struct xsave_struct *xsave = &current->thread.fpu.state.xsave;
 	struct task_struct *tsk = current;

index c67f96c87938..59cfc9c97491 100644
@@ -235,7 +235,7 @@ get_sigframe(struct k_sigaction *ka, struct pt_regs *regs, size_t frame_size,
 
 	/* save i387 and extended state */
 	if (fpu->fpstate_active &&
-	    save_xstate_sig(*fpstate, (void __user *)buf_fx, math_size) < 0)
+	    copy_fpstate_to_sigframe(*fpstate, (void __user *)buf_fx, math_size) < 0)
 		return (void __user *)-1L;
 
 	return (void __user *)sp;

index 5178158a03ff..28638820ed0e 100644
@@ -257,7 +257,7 @@ static inline int save_xstate_epilog(void __user *buf, int ia32_frame)
 	return err;
 }
 
-static inline int save_user_xstate(struct xsave_struct __user *buf)
+static inline int copy_fpregs_to_sigframe(struct xsave_struct __user *buf)
 {
 	int err;
 

@@ -312,7 +312,7 @@ int copy_fpstate_to_sigframe(void __user *buf, void __user *buf_fx, int size)
 
 	if (user_has_fpu()) {
 		/* Save the live register state to the user directly. */
-		if (save_user_xstate(buf_fx))
+		if (copy_fpregs_to_sigframe(buf_fx))
 			return -1;
 		/* Update the thread's fxstate to save the fsave header. */
 		if (ia32_fxstate)

index 7e91a6f7564a..45ef4e51928b 100644
@@ -319,14 +319,14 @@ static void fpu__activate_stopped(struct fpu *child_fpu)
 }
 
 /*
- * 'fpu__restore()' saves the current math information in the
- * old math state array, and gets the new ones from the current task
+ * 'fpu__restore()' is called to copy FPU registers from
+ * the FPU fpstate to the live hw registers and to activate
+ * access to the hardware registers, so that FPU instructions
+ * can be used afterwards.
  *
- * Careful.. There are problems with IBM-designed IRQ13 behaviour.
- * Don't touch unless you *really* know how it works.
- *
- * Must be called with kernel preemption disabled (eg with local
- * local interrupts as in the case of do_device_not_available).
+ * Must be called with kernel preemption disabled (for example
+ * with local interrupts disabled, as it is in the case of
+ * do_device_not_available()).
  */
 void fpu__restore(void)
 {

index 58c274dfcb62..0f17cd4e4e58 100644
@@ -358,7 +358,7 @@ static inline void __fpregs_activate(struct fpu *fpu)
  * to save the FP state - we'll just take a #NM
  * fault and get the FPU access back.
  */
-static inline int user_has_fpu(void)
+static inline int fpregs_active(void)
 {
 	return current->thread.fpu.fpregs_active;
 }

@@ -557,7 +557,7 @@ static inline void user_fpu_begin(void)
 	struct fpu *fpu = &current->thread.fpu;
 
 	preempt_disable();
-	if (!user_has_fpu())
+	if (!fpregs_active())
 		fpregs_activate(fpu);
 	preempt_enable();
 }

index 28638820ed0e..b8e5fee2aef3 100644
@@ -310,7 +310,7 @@ int copy_fpstate_to_sigframe(void __user *buf, void __user *buf_fx, int size)
 			sizeof(struct user_i387_ia32_struct), NULL,
 			(struct _fpstate_ia32 __user *) buf) ? -1 : 1;
 
-	if (user_has_fpu()) {
+	if (fpregs_active()) {
 		/* Save the live register state to the user directly. */
 		if (copy_fpregs_to_sigframe(buf_fx))
 			return -1;

index 2de55e953842..1c384bf856e5 100644
@@ -1882,7 +1882,7 @@ static void __vmx_load_host_state(struct vcpu_vmx *vmx)
 	 * If the FPU is not active (through the host task or
 	 * the guest vcpu), then restore the cr0.TS bit.
 	 */
-	if (!user_has_fpu() && !vmx->vcpu.guest_fpu_loaded)
+	if (!fpregs_active() && !vmx->vcpu.guest_fpu_loaded)
 		stts();
 	load_gdt(this_cpu_ptr(&host_gdt));
 }

index b80e4b8c9b6e..99bb3009e2d5 100644
@@ -251,7 +251,7 @@ void lguest_arch_run_guest(struct lg_cpu *cpu)
 	 * we set it now, so we can trap and pass that trap to the Guest if it
 	 * uses the FPU.
 	 */
-	if (cpu->ts && user_has_fpu())
+	if (cpu->ts && fpregs_active())
 		stts();
 
 	/*

@@ -283,7 +283,7 @@ void lguest_arch_run_guest(struct lg_cpu *cpu)
 		wrmsr(MSR_IA32_SYSENTER_CS, __KERNEL_CS, 0);
 
 	/* Clear the host TS bit if it was set above. */
-	if (cpu->ts && user_has_fpu())
+	if (cpu->ts && fpregs_active())
 		clts();
 
 	/*

@@ -301,7 +301,7 @@ void lguest_arch_run_guest(struct lg_cpu *cpu)
 	 * a different CPU. So all the critical stuff should be done
 	 * before this.
 	 */
-	else if (cpu->regs->trapnum == 7 && !user_has_fpu())
+	else if (cpu->regs->trapnum == 7 && !fpregs_active())
 		fpu__restore();
 }
 

index 460e7e2c6186..72219ce2385a 100644
@@ -36,6 +36,9 @@ static void fpu__init_cpu_generic(void)
 	if (!cpu_has_fpu)
 		cr0 |= X86_CR0_EM;
 	write_cr0(cr0);
+
+	/* Flush out any pending x87 state: */
+	asm volatile ("fninit");
 }
 
 /*

index 45ef4e51928b..33c9a43b000e 100644
@@ -348,6 +348,10 @@ void fpu__restore(void)
 }
 EXPORT_SYMBOL_GPL(fpu__restore);
 
+/*
+ * Called by sys_execve() to clear the FPU fpregs, so that FPU state
+ * of the previous binary does not leak over into the exec()ed binary:
+ */
 void fpu__clear(struct task_struct *tsk)
 {
 	struct fpu *fpu = &tsk->thread.fpu;

@@ -361,8 +365,8 @@ void fpu__clear(struct task_struct *tsk)
 		 if (!fpu->fpstate_active) {
 			fpu__activate_curr(fpu);
 			user_fpu_begin();
-			restore_init_xstate();
 		}
+		restore_init_xstate();
 	}
 }
 

index ba32af062f61..7bfc85bbb8ff 100644
@@ -52,6 +52,12 @@ struct alt_instr {
 	u8  padlen;		/* length of build-time padding */
 } __packed;
 
+/*
+ * Debug flag that can be tested to see whether alternative
+ * instructions were patched in already:
+ */
+extern int alternatives_patched;
+
 extern void alternative_instructions(void);
 extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end);
 

index 31a002ad5aeb..ab2c507b58b6 100644
@@ -119,7 +119,7 @@ static inline int xsave_state(struct xsave_struct *fx)
 	u32 hmask = mask >> 32;
 	int err = 0;
 
-	WARN_ON(system_state == SYSTEM_BOOTING);
+	WARN_ON(!alternatives_patched);
 
 	/*
 	 * If xsaves is enabled, xsaves replaces xsaveopt because

index aef653193160..7fe097235376 100644
@@ -21,6 +21,10 @@
 #include <asm/io.h>
 #include <asm/fixmap.h>
 
+int __read_mostly alternatives_patched;
+
+EXPORT_SYMBOL_GPL(alternatives_patched);
+
 #define MAX_PATCH_LEN (255-1)
 
 static int __initdata_or_module debug_alternative;

@@ -627,6 +631,7 @@ void __init alternative_instructions(void)
 	apply_paravirt(__parainstructions, __parainstructions_end);
 
 	restart_nmi();
+	alternatives_patched = 1;
 }
 
 /**

index 0f17cd4e4e58..31bfda818f30 100644
@@ -382,11 +382,17 @@ static inline void fpregs_deactivate(struct fpu *fpu)
 	__fpregs_deactivate_hw();
 }
 
-static inline void drop_fpu(struct fpu *fpu)
+/*
+ * Drops current FPU state: deactivates the fpregs and
+ * the fpstate. NOTE: it still leaves previous contents
+ * in the fpregs in the eager-FPU case.
+ *
+ * This function can be used in cases where we know that
+ * a state-restore is coming: either an explicit one,
+ * or a reschedule.
+ */
+static inline void fpu__drop(struct fpu *fpu)
 {
-	/*
-	 * Forget coprocessor state..
-	 */
 	preempt_disable();
 	fpu->counter = 0;
 

@@ -412,13 +418,12 @@ static inline void restore_init_xstate(void)
 }
 
 /*
- * Reset the FPU state in the eager case and drop it in the lazy case (later use
- * will reinit it).
+ * Reset the FPU state back to init state.
  */
-static inline void fpu_reset_state(struct fpu *fpu)
+static inline void fpu__reset(struct fpu *fpu)
 {
 	if (!use_eager_fpu())
-		drop_fpu(fpu);
+		fpu__drop(fpu);
 	else
 		restore_init_xstate();
 }

@@ -516,7 +521,7 @@ static inline void switch_fpu_finish(struct fpu *new_fpu, fpu_switch_t fpu_switc
 {
 	if (fpu_switch.preload) {
 		if (unlikely(restore_fpu_checking(new_fpu)))
-			fpu_reset_state(new_fpu);
+			fpu__reset(new_fpu);
 	}
 }
 

index 33c9a43b000e..11ec1b736172 100644
@@ -116,7 +116,7 @@ void __kernel_fpu_end(void)
 
 	if (fpu->fpregs_active) {
 		if (WARN_ON(restore_fpu_checking(fpu)))
-			fpu_reset_state(fpu);
+			fpu__reset(fpu);
 	} else {
 		__fpregs_deactivate_hw();
 	}

@@ -339,7 +339,7 @@ void fpu__restore(void)
 	kernel_fpu_disable();
 	fpregs_activate(fpu);
 	if (unlikely(restore_fpu_checking(fpu))) {
-		fpu_reset_state(fpu);
+		fpu__reset(fpu);
 		force_sig_info(SIGSEGV, SEND_SIG_PRIV, tsk);
 	} else {
 		tsk->thread.fpu.counter++;

@@ -360,7 +360,7 @@ void fpu__clear(struct task_struct *tsk)
 
 	if (!use_eager_fpu()) {
 		/* FPU state will be reallocated lazily at the first use. */
-		drop_fpu(fpu);
+		fpu__drop(fpu);
 	} else {
 		 if (!fpu->fpstate_active) {
 			fpu__activate_curr(fpu);

index b8e5fee2aef3..5e3d9242bb95 100644
@@ -401,7 +401,7 @@ int __restore_xstate_sig(void __user *buf, void __user *buf_fx, int size)
 			 config_enabled(CONFIG_IA32_EMULATION));
 
 	if (!buf) {
-		fpu_reset_state(fpu);
+		fpu__reset(fpu);
 		return 0;
 	}
 

@@ -449,7 +449,7 @@ int __restore_xstate_sig(void __user *buf, void __user *buf_fx, int size)
 		 * We will be ready to restore/save the state only after
 		 * fpu->fpstate_active is again set.
 		 */
-		drop_fpu(fpu);
+		fpu__drop(fpu);
 
 		if (__copy_from_user(&fpu->state.xsave, buf_fx, state_size) ||
 		    __copy_from_user(&env, buf, sizeof(env))) {

@@ -474,7 +474,7 @@ int __restore_xstate_sig(void __user *buf, void __user *buf_fx, int size)
 		 */
 		user_fpu_begin();
 		if (restore_user_xstate(buf_fx, xfeatures, fx_only)) {
-			fpu_reset_state(fpu);
+			fpu__reset(fpu);
 			return -1;
 		}
 	}

index 4b4b16c8e6ee..099e7a889ab9 100644
@@ -110,7 +110,7 @@ void exit_thread(void)
 		kfree(bp);
 	}
 
-	drop_fpu(fpu);
+	fpu__drop(fpu);
 }
 
 void flush_thread(void)

index 59cfc9c97491..6bf512390536 100644
@@ -667,7 +667,7 @@ handle_signal(struct ksignal *ksig, struct pt_regs *regs)
 		 * Ensure the signal handler starts with the new fpu state.
 		 */
 		if (fpu->fpstate_active)
-			fpu_reset_state(fpu);
+			fpu__reset(fpu);
 	}
 	signal_setup_done(failed, ksig, stepping);
 }

index 31bfda818f30..c09aea145e09 100644
@@ -289,7 +289,7 @@ static inline int copy_fpregs_to_fpstate(struct fpu *fpu)
 
 extern void fpu__save(struct fpu *fpu);
 
-static inline int fpu_restore_checking(struct fpu *fpu)
+static inline int __copy_fpstate_to_fpregs(struct fpu *fpu)
 {
 	if (use_xsave())
 		return fpu_xrstor_checking(&fpu->state.xsave);

@@ -299,7 +299,7 @@ static inline int fpu_restore_checking(struct fpu *fpu)
 		return frstor_checking(&fpu->state.fsave);
 }
 
-static inline int restore_fpu_checking(struct fpu *fpu)
+static inline int copy_fpstate_to_fpregs(struct fpu *fpu)
 {
 	/*
 	 * AMD K7/K8 CPUs don't save/restore FDP/FIP/FOP unless an exception is

@@ -314,7 +314,7 @@ static inline int restore_fpu_checking(struct fpu *fpu)
 			: : [addr] "m" (fpu->fpregs_active));
 	}
 
-	return fpu_restore_checking(fpu);
+	return __copy_fpstate_to_fpregs(fpu);
 }
 
 /*

@@ -520,7 +520,7 @@ switch_fpu_prepare(struct fpu *old_fpu, struct fpu *new_fpu, int cpu)
 static inline void switch_fpu_finish(struct fpu *new_fpu, fpu_switch_t fpu_switch)
 {
 	if (fpu_switch.preload) {
-		if (unlikely(restore_fpu_checking(new_fpu)))
+		if (unlikely(copy_fpstate_to_fpregs(new_fpu)))
 			fpu__reset(new_fpu);
 	}
 }

index 11ec1b736172..72e3f02db40d 100644
@@ -115,7 +115,7 @@ void __kernel_fpu_end(void)
 	struct fpu *fpu = &current->thread.fpu;
 
 	if (fpu->fpregs_active) {
-		if (WARN_ON(restore_fpu_checking(fpu)))
+		if (WARN_ON(copy_fpstate_to_fpregs(fpu)))
 			fpu__reset(fpu);
 	} else {
 		__fpregs_deactivate_hw();

@@ -338,7 +338,7 @@ void fpu__restore(void)
 	/* Avoid __kernel_fpu_begin() right after fpregs_activate() */
 	kernel_fpu_disable();
 	fpregs_activate(fpu);
-	if (unlikely(restore_fpu_checking(fpu))) {
+	if (unlikely(copy_fpstate_to_fpregs(fpu))) {
 		fpu__reset(fpu);
 		force_sig_info(SIGSEGV, SEND_SIG_PRIV, tsk);
 	} else {

index 5c61aae277f9..f4438179398b 100644
@@ -7030,7 +7030,7 @@ void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
 	kvm_put_guest_xcr0(vcpu);
 	vcpu->guest_fpu_loaded = 1;
 	__kernel_fpu_begin();
-	fpu_restore_checking(&vcpu->arch.guest_fpu);
+	__copy_fpstate_to_fpregs(&vcpu->arch.guest_fpu);
 	trace_kvm_fpu(1);
 }
 

index c09aea145e09..f20a0030f6a1 100644
@@ -46,10 +46,19 @@ extern void fpu__init_system(struct cpuinfo_x86 *c);
 
 extern void fpu__activate_curr(struct fpu *fpu);
 extern void fpstate_init(struct fpu *fpu);
-extern void fpu__clear(struct task_struct *tsk);
 
 extern int dump_fpu(struct pt_regs *, struct user_i387_struct *);
+
+/*
+ * High level FPU state handling functions:
+ */
+extern void fpu__save(struct fpu *fpu);
 extern void fpu__restore(void);
+extern void fpu__drop(struct fpu *fpu);
+extern int  fpu__copy(struct fpu *dst_fpu, struct fpu *src_fpu);
+extern void fpu__reset(struct fpu *fpu);
+extern void fpu__clear(struct task_struct *tsk);
+
 extern void fpu__init_check_bugs(void);
 extern void fpu__resume_cpu(void);
 

@@ -287,8 +296,6 @@ static inline int copy_fpregs_to_fpstate(struct fpu *fpu)
 	return 0;
 }
 
-extern void fpu__save(struct fpu *fpu);
-
 static inline int __copy_fpstate_to_fpregs(struct fpu *fpu)
 {
 	if (use_xsave())

@@ -382,33 +389,6 @@ static inline void fpregs_deactivate(struct fpu *fpu)
 	__fpregs_deactivate_hw();
 }
 
-/*
- * Drops current FPU state: deactivates the fpregs and
- * the fpstate. NOTE: it still leaves previous contents
- * in the fpregs in the eager-FPU case.
- *
- * This function can be used in cases where we know that
- * a state-restore is coming: either an explicit one,
- * or a reschedule.
- */
-static inline void fpu__drop(struct fpu *fpu)
-{
-	preempt_disable();
-	fpu->counter = 0;
-
-	if (fpu->fpregs_active) {
-		/* Ignore delayed exceptions from user space */
-		asm volatile("1: fwait\n"
-			     "2:\n"
-			     _ASM_EXTABLE(1b, 2b));
-		fpregs_deactivate(fpu);
-	}
-
-	fpu->fpstate_active = 0;
-
-	preempt_enable();
-}
-
 static inline void restore_init_xstate(void)
 {
 	if (use_xsave())

@@ -418,17 +398,6 @@ static inline void restore_init_xstate(void)
 }
 
 /*
- * Reset the FPU state back to init state.
- */
-static inline void fpu__reset(struct fpu *fpu)
-{
-	if (!use_eager_fpu())
-		fpu__drop(fpu);
-	else
-		restore_init_xstate();
-}
-
-/*
  * Definitions for the eXtended Control Register instructions
  */
 

@@ -597,8 +566,6 @@ static inline unsigned short get_fpu_mxcsr(struct task_struct *tsk)
 	}
 }
 
-extern int fpu__copy(struct fpu *dst_fpu, struct fpu *src_fpu);
-
 static inline unsigned long
 alloc_mathframe(unsigned long sp, int ia32_frame, unsigned long *buf_fx,
 		unsigned long *size)

index 72e3f02db40d..cba02f7e337b 100644
@@ -349,6 +349,44 @@ void fpu__restore(void)
 EXPORT_SYMBOL_GPL(fpu__restore);
 
 /*
+ * Drops current FPU state: deactivates the fpregs and
+ * the fpstate. NOTE: it still leaves previous contents
+ * in the fpregs in the eager-FPU case.
+ *
+ * This function can be used in cases where we know that
+ * a state-restore is coming: either an explicit one,
+ * or a reschedule.
+ */
+void fpu__drop(struct fpu *fpu)
+{
+	preempt_disable();
+	fpu->counter = 0;
+
+	if (fpu->fpregs_active) {
+		/* Ignore delayed exceptions from user space */
+		asm volatile("1: fwait\n"
+			     "2:\n"
+			     _ASM_EXTABLE(1b, 2b));
+		fpregs_deactivate(fpu);
+	}
+
+	fpu->fpstate_active = 0;
+
+	preempt_enable();
+}
+
+/*
+ * Reset the FPU state back to init state:
+ */
+void fpu__reset(struct fpu *fpu)
+{
+	if (!use_eager_fpu())
+		fpu__drop(fpu);
+	else
+		restore_init_xstate();
+}
+
+/*
  * Called by sys_execve() to clear the FPU fpregs, so that FPU state
  * of the previous binary does not leak over into the exec()ed binary:
  */

index f20a0030f6a1..d6ac4611f05e 100644
@@ -37,9 +37,8 @@ int ia32_setup_frame(int sig, struct ksignal *ksig,
 #define	MXCSR_DEFAULT		0x1f80
 
 extern unsigned int mxcsr_feature_mask;
-extern void fpu__init_cpu(void);
-extern void eager_fpu_init(void);
 
+extern void fpu__init_cpu(void);
 extern void fpu__init_system_xstate(void);
 extern void fpu__init_cpu_xstate(void);
 extern void fpu__init_system(struct cpuinfo_x86 *c);

@@ -57,7 +56,7 @@ extern void fpu__restore(void);
 extern void fpu__drop(struct fpu *fpu);
 extern int  fpu__copy(struct fpu *dst_fpu, struct fpu *src_fpu);
 extern void fpu__reset(struct fpu *fpu);
-extern void fpu__clear(struct task_struct *tsk);
+extern void fpu__clear(struct fpu *fpu);
 
 extern void fpu__init_check_bugs(void);
 extern void fpu__resume_cpu(void);

index cba02f7e337b..51afe4466ae3 100644
@@ -390,11 +390,9 @@ void fpu__reset(struct fpu *fpu)
  * Called by sys_execve() to clear the FPU fpregs, so that FPU state
  * of the previous binary does not leak over into the exec()ed binary:
  */
-void fpu__clear(struct task_struct *tsk)
+void fpu__clear(struct fpu *fpu)
 {
-	struct fpu *fpu = &tsk->thread.fpu;
-
-	WARN_ON_ONCE(tsk != current); /* Almost certainly an anomaly */
+	WARN_ON_ONCE(fpu != &current->thread.fpu); /* Almost certainly an anomaly */
 
 	if (!use_eager_fpu()) {
 		/* FPU state will be reallocated lazily at the first use. */

index 099e7a889ab9..a478e8ba16f7 100644
@@ -120,7 +120,7 @@ void flush_thread(void)
 	flush_ptrace_hw_breakpoint(tsk);
 	memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array));
 
-	fpu__clear(tsk);
+	fpu__clear(&tsk->thread.fpu);
 }
 
 static void hard_disable_TSC(void)

index 2e0b1b7842ae..de81ddf3b0b2 100644
@@ -197,7 +197,7 @@ static int ia32_restore_sigcontext(struct pt_regs *regs,
 		buf = compat_ptr(tmp);
 	} get_user_catch(err);
 
-	err |= restore_xstate_sig(buf, 1);
+	err |= fpu__restore_sig(buf, 1);
 
 	force_iret();
 

index d6ac4611f05e..d6cfbdafbab2 100644
@@ -497,14 +497,14 @@ static inline void switch_fpu_finish(struct fpu *new_fpu, fpu_switch_t fpu_switc
  * Signal frame handlers...
  */
 extern int copy_fpstate_to_sigframe(void __user *buf, void __user *fx, int size);
-extern int __restore_xstate_sig(void __user *buf, void __user *fx, int size);
+extern int __fpu__restore_sig(void __user *buf, void __user *fx, int size);
 
 static inline int xstate_sigframe_size(void)
 {
 	return use_xsave() ? xstate_size + FP_XSTATE_MAGIC2_SIZE : xstate_size;
 }
 
-static inline int restore_xstate_sig(void __user *buf, int ia32_frame)
+static inline int fpu__restore_sig(void __user *buf, int ia32_frame)
 {
 	void __user *buf_fx = buf;
 	int size = xstate_sigframe_size();

@@ -514,7 +514,7 @@ static inline int restore_xstate_sig(void __user *buf, int ia32_frame)
 		size += sizeof(struct i387_fsave_struct);
 	}
 
-	return __restore_xstate_sig(buf, buf_fx, size);
+	return __fpu__restore_sig(buf, buf_fx, size);
 }
 
 /*

index 5e3d9242bb95..ea514d6a34e8 100644
@@ -388,7 +388,7 @@ static inline int restore_user_xstate(void __user *buf, u64 xbv, int fx_only)
 		return frstor_user(buf);
 }
 
-int __restore_xstate_sig(void __user *buf, void __user *buf_fx, int size)
+int __fpu__restore_sig(void __user *buf, void __user *buf_fx, int size)
 {
 	int ia32_fxstate = (buf != buf_fx);
 	struct task_struct *tsk = current;

index 6bf512390536..7416fa86f3c7 100644
@@ -102,7 +102,7 @@ int restore_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc)
 		get_user_ex(buf, &sc->fpstate);
 	} get_user_catch(err);
 
-	err |= restore_xstate_sig(buf, config_enabled(CONFIG_X86_32));
+	err |= fpu__restore_sig(buf, config_enabled(CONFIG_X86_32));
 
 	force_iret();
 

index de81ddf3b0b2..54605eb1631f 100644
@@ -325,7 +325,7 @@ static void __user *get_sigframe(struct ksignal *ksig, struct pt_regs *regs,
 	if (fpu->fpstate_active) {
 		unsigned long fx_aligned, math_size;
 
-		sp = alloc_mathframe(sp, 1, &fx_aligned, &math_size);
+		sp = fpu__alloc_mathframe(sp, 1, &fx_aligned, &math_size);
 		*fpstate = (struct _fpstate_ia32 __user *) sp;
 		if (copy_fpstate_to_sigframe(*fpstate, (void __user *)fx_aligned,
 				    math_size) < 0)

index d6cfbdafbab2..34fbf95bbe14 100644
@@ -53,6 +53,7 @@ extern int dump_fpu(struct pt_regs *, struct user_i387_struct *);
  */
 extern void fpu__save(struct fpu *fpu);
 extern void fpu__restore(void);
+extern int  fpu__restore_sig(void __user *buf, int ia32_frame);
 extern void fpu__drop(struct fpu *fpu);
 extern int  fpu__copy(struct fpu *dst_fpu, struct fpu *src_fpu);
 extern void fpu__reset(struct fpu *fpu);

@@ -497,25 +498,6 @@ static inline void switch_fpu_finish(struct fpu *new_fpu, fpu_switch_t fpu_switc
  * Signal frame handlers...
  */
 extern int copy_fpstate_to_sigframe(void __user *buf, void __user *fx, int size);
-extern int __fpu__restore_sig(void __user *buf, void __user *fx, int size);
-
-static inline int xstate_sigframe_size(void)
-{
-	return use_xsave() ? xstate_size + FP_XSTATE_MAGIC2_SIZE : xstate_size;
-}
-
-static inline int fpu__restore_sig(void __user *buf, int ia32_frame)
-{
-	void __user *buf_fx = buf;
-	int size = xstate_sigframe_size();
-
-	if (ia32_frame && use_fxsr()) {
-		buf_fx = buf + sizeof(struct i387_fsave_struct);
-		size += sizeof(struct i387_fsave_struct);
-	}
-
-	return __fpu__restore_sig(buf, buf_fx, size);
-}
 
 /*
  * Needs to be preemption-safe.

@@ -565,20 +547,8 @@ static inline unsigned short get_fpu_mxcsr(struct task_struct *tsk)
 	}
 }
 
-static inline unsigned long
-alloc_mathframe(unsigned long sp, int ia32_frame, unsigned long *buf_fx,
-		unsigned long *size)
-{
-	unsigned long frame_size = xstate_sigframe_size();
-
-	*buf_fx = sp = round_down(sp - frame_size, 64);
-	if (ia32_frame && use_fxsr()) {
-		frame_size += sizeof(struct i387_fsave_struct);
-		sp -= sizeof(struct i387_fsave_struct);
-	}
-
-	*size = frame_size;
-	return sp;
-}
+unsigned long
+fpu__alloc_mathframe(unsigned long sp, int ia32_frame,
+		     unsigned long *buf_fx, unsigned long *size);
 
 #endif /* _ASM_X86_FPU_INTERNAL_H */

index ea514d6a34e8..810f080fadf3 100644
@@ -388,7 +388,7 @@ static inline int restore_user_xstate(void __user *buf, u64 xbv, int fx_only)
 		return frstor_user(buf);
 }
 
-int __fpu__restore_sig(void __user *buf, void __user *buf_fx, int size)
+static int __fpu__restore_sig(void __user *buf, void __user *buf_fx, int size)
 {
 	int ia32_fxstate = (buf != buf_fx);
 	struct task_struct *tsk = current;

@@ -482,6 +482,43 @@ int __fpu__restore_sig(void __user *buf, void __user *buf_fx, int size)
 	return 0;
 }
 
+static inline int xstate_sigframe_size(void)
+{
+	return use_xsave() ? xstate_size + FP_XSTATE_MAGIC2_SIZE : xstate_size;
+}
+
+/*
+ * Restore FPU state from a sigframe:
+ */
+int fpu__restore_sig(void __user *buf, int ia32_frame)
+{
+	void __user *buf_fx = buf;
+	int size = xstate_sigframe_size();
+
+	if (ia32_frame && use_fxsr()) {
+		buf_fx = buf + sizeof(struct i387_fsave_struct);
+		size += sizeof(struct i387_fsave_struct);
+	}
+
+	return __fpu__restore_sig(buf, buf_fx, size);
+}
+
+unsigned long
+fpu__alloc_mathframe(unsigned long sp, int ia32_frame,
+		     unsigned long *buf_fx, unsigned long *size)
+{
+	unsigned long frame_size = xstate_sigframe_size();
+
+	*buf_fx = sp = round_down(sp - frame_size, 64);
+	if (ia32_frame && use_fxsr()) {
+		frame_size += sizeof(struct i387_fsave_struct);
+		sp -= sizeof(struct i387_fsave_struct);
+	}
+
+	*size = frame_size;
+
+	return sp;
+}
 /*
  * Prepare the SW reserved portion of the fxsave memory layout, indicating
  * the presence of the extended state information in the memory layout

index 7416fa86f3c7..9554ca69a84e 100644
@@ -219,8 +219,8 @@ get_sigframe(struct k_sigaction *ka, struct pt_regs *regs, size_t frame_size,
 	}
 
 	if (fpu->fpstate_active) {
-		sp = alloc_mathframe(sp, config_enabled(CONFIG_X86_32),
-				     &buf_fx, &math_size);
+		sp = fpu__alloc_mathframe(sp, config_enabled(CONFIG_X86_32),
+					  &buf_fx, &math_size);
 		*fpstate = (void __user *)sp;
 	}
 

index 34fbf95bbe14..a55d63efab0f 100644
@@ -56,7 +56,6 @@ extern void fpu__restore(void);
 extern int  fpu__restore_sig(void __user *buf, int ia32_frame);
 extern void fpu__drop(struct fpu *fpu);
 extern int  fpu__copy(struct fpu *dst_fpu, struct fpu *src_fpu);
-extern void fpu__reset(struct fpu *fpu);
 extern void fpu__clear(struct fpu *fpu);
 
 extern void fpu__init_check_bugs(void);

@@ -490,7 +489,7 @@ static inline void switch_fpu_finish(struct fpu *new_fpu, fpu_switch_t fpu_switc
 {
 	if (fpu_switch.preload) {
 		if (unlikely(copy_fpstate_to_fpregs(new_fpu)))
-			fpu__reset(new_fpu);
+			fpu__clear(new_fpu);
 	}
 }
 

index 51afe4466ae3..c8eb76a41ce8 100644
@@ -116,7 +116,7 @@ void __kernel_fpu_end(void)
 
 	if (fpu->fpregs_active) {
 		if (WARN_ON(copy_fpstate_to_fpregs(fpu)))
-			fpu__reset(fpu);
+			fpu__clear(fpu);
 	} else {
 		__fpregs_deactivate_hw();
 	}

@@ -339,7 +339,7 @@ void fpu__restore(void)
 	kernel_fpu_disable();
 	fpregs_activate(fpu);
 	if (unlikely(copy_fpstate_to_fpregs(fpu))) {
-		fpu__reset(fpu);
+		fpu__clear(fpu);
 		force_sig_info(SIGSEGV, SEND_SIG_PRIV, tsk);
 	} else {
 		tsk->thread.fpu.counter++;

@@ -376,19 +376,10 @@ void fpu__drop(struct fpu *fpu)
 }
 
 /*
- * Reset the FPU state back to init state:
- */
-void fpu__reset(struct fpu *fpu)
-{
-	if (!use_eager_fpu())
-		fpu__drop(fpu);
-	else
-		restore_init_xstate();
-}
-
-/*
- * Called by sys_execve() to clear the FPU fpregs, so that FPU state
- * of the previous binary does not leak over into the exec()ed binary:
+ * Clear the FPU state back to init state.
+ *
+ * Called by sys_execve(), by the signal handler code and by various
+ * error paths.
  */
 void fpu__clear(struct fpu *fpu)
 {

index 810f080fadf3..9bc3734acc16 100644
@@ -401,7 +401,7 @@ static int __fpu__restore_sig(void __user *buf, void __user *buf_fx, int size)
 			 config_enabled(CONFIG_IA32_EMULATION));
 
 	if (!buf) {
-		fpu__reset(fpu);
+		fpu__clear(fpu);
 		return 0;
 	}
 

@@ -474,7 +474,7 @@ static int __fpu__restore_sig(void __user *buf, void __user *buf_fx, int size)
 		 */
 		user_fpu_begin();
 		if (restore_user_xstate(buf_fx, xfeatures, fx_only)) {
-			fpu__reset(fpu);
+			fpu__clear(fpu);
 			return -1;
 		}
 	}

index 9554ca69a84e..7c08795073d2 100644
@@ -667,7 +667,7 @@ handle_signal(struct ksignal *ksig, struct pt_regs *regs)
 		 * Ensure the signal handler starts with the new fpu state.
 		 */
 		if (fpu->fpstate_active)
-			fpu__reset(fpu);
+			fpu__clear(fpu);
 	}
 	signal_setup_done(failed, ksig, stepping);
 }

index a55d63efab0f..dc4842b0831b 100644
@@ -103,22 +103,6 @@ static inline int fpu_want_lazy_restore(struct fpu *fpu, unsigned int cpu)
 	return fpu == this_cpu_read_stable(fpu_fpregs_owner_ctx) && cpu == fpu->last_cpu;
 }
 
-static inline int is_ia32_compat_frame(void)
-{
-	return config_enabled(CONFIG_IA32_EMULATION) &&
-	       test_thread_flag(TIF_IA32);
-}
-
-static inline int is_ia32_frame(void)
-{
-	return config_enabled(CONFIG_X86_32) || is_ia32_compat_frame();
-}
-
-static inline int is_x32_frame(void)
-{
-	return config_enabled(CONFIG_X86_X32_ABI) && test_thread_flag(TIF_X32);
-}
-
 #define X87_FSW_ES (1 << 7)	/* Exception Summary */
 
 static __always_inline __pure bool use_eager_fpu(void)

index 7c08795073d2..f4b205686527 100644
@@ -593,6 +593,22 @@ asmlinkage long sys_rt_sigreturn(void)
 	return 0;
 }
 
+static inline int is_ia32_compat_frame(void)
+{
+	return config_enabled(CONFIG_IA32_EMULATION) &&
+	       test_thread_flag(TIF_IA32);
+}
+
+static inline int is_ia32_frame(void)
+{
+	return config_enabled(CONFIG_X86_32) || is_ia32_compat_frame();
+}
+
+static inline int is_x32_frame(void)
+{
+	return config_enabled(CONFIG_X86_X32_ABI) && test_thread_flag(TIF_X32);
+}
+
 static int
 setup_rt_frame(struct ksignal *ksig, struct pt_regs *regs)
 {

index 54605eb1631f..ae3a29ae875b 100644
@@ -22,6 +22,7 @@
 #include <asm/ucontext.h>
 #include <asm/uaccess.h>
 #include <asm/fpu/internal.h>
+#include <asm/fpu/signal.h>
 #include <asm/ptrace.h>
 #include <asm/ia32_unistd.h>
 #include <asm/user32.h>

index dc4842b0831b..e2ceb49d310d 100644
@@ -19,21 +19,6 @@
 #include <asm/fpu/api.h>
 #include <asm/fpu/xstate.h>
 
-#ifdef CONFIG_X86_64
-# include <asm/sigcontext32.h>
-# include <asm/user32.h>
-struct ksignal;
-int ia32_setup_rt_frame(int sig, struct ksignal *ksig,
-			compat_sigset_t *set, struct pt_regs *regs);
-int ia32_setup_frame(int sig, struct ksignal *ksig,
-		     compat_sigset_t *set, struct pt_regs *regs);
-#else
-# define user_i387_ia32_struct	user_i387_struct
-# define user32_fxsr_struct	user_fxsr_struct
-# define ia32_setup_frame	__setup_frame
-# define ia32_setup_rt_frame	__setup_rt_frame
-#endif
-
 #define	MXCSR_DEFAULT		0x1f80
 
 extern unsigned int mxcsr_feature_mask;

@@ -63,11 +48,6 @@ extern void fpu__resume_cpu(void);
 
 DECLARE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx);
 
-extern void convert_from_fxsr(struct user_i387_ia32_struct *env,
-			      struct task_struct *tsk);
-extern void convert_to_fxsr(struct task_struct *tsk,
-			    const struct user_i387_ia32_struct *env);
-
 extern user_regset_active_fn regset_fpregs_active, regset_xregset_fpregs_active;
 extern user_regset_get_fn fpregs_get, xfpregs_get, fpregs_soft_get,
 				xstateregs_get;

@@ -530,8 +510,4 @@ static inline unsigned short get_fpu_mxcsr(struct task_struct *tsk)
 	}
 }
 
-unsigned long
-fpu__alloc_mathframe(unsigned long sp, int ia32_frame,
-		     unsigned long *buf_fx, unsigned long *size);
-
 #endif /* _ASM_X86_FPU_INTERNAL_H */

new file mode 100644
index 000000000000..0803dc2aba80
@@ -0,0 +1,31 @@
+/*
+ * x86 FPU signal frame handling methods:
+ */
+#ifndef _ASM_X86_FPU_SIGNAL_H
+#define _ASM_X86_FPU_SIGNAL_H
+
+#ifdef CONFIG_X86_64
+# include <asm/sigcontext32.h>
+# include <asm/user32.h>
+struct ksignal;
+int ia32_setup_rt_frame(int sig, struct ksignal *ksig,
+			compat_sigset_t *set, struct pt_regs *regs);
+int ia32_setup_frame(int sig, struct ksignal *ksig,
+		     compat_sigset_t *set, struct pt_regs *regs);
+#else
+# define user_i387_ia32_struct	user_i387_struct
+# define user32_fxsr_struct	user_fxsr_struct
+# define ia32_setup_frame	__setup_frame
+# define ia32_setup_rt_frame	__setup_rt_frame
+#endif
+
+extern void convert_from_fxsr(struct user_i387_ia32_struct *env,
+			      struct task_struct *tsk);
+extern void convert_to_fxsr(struct task_struct *tsk,
+			    const struct user_i387_ia32_struct *env);
+
+unsigned long
+fpu__alloc_mathframe(unsigned long sp, int ia32_frame,
+		     unsigned long *buf_fx, unsigned long *size);
+
+#endif /* _ASM_X86_FPU_SIGNAL_H */

index c8eb76a41ce8..45a2ca26a36f 100644
@@ -6,6 +6,8 @@
  *	Gareth Hughes <[email protected]>, May 2000
  */
 #include <asm/fpu/internal.h>
+#include <asm/fpu/signal.h>
+
 #include <linux/hardirq.h>
 
 /*

index 9bc3734acc16..78710740e9a0 100644
@@ -7,6 +7,7 @@
 #include <linux/cpu.h>
 #include <asm/fpu/api.h>
 #include <asm/fpu/internal.h>
+#include <asm/fpu/signal.h>
 #include <asm/sigframe.h>
 #include <asm/tlbflush.h>
 

index 4c615661ec72..51e73a685ce4 100644
@@ -29,6 +29,7 @@
 #include <asm/pgtable.h>
 #include <asm/processor.h>
 #include <asm/fpu/internal.h>
+#include <asm/fpu/signal.h>
 #include <asm/debugreg.h>
 #include <asm/ldt.h>
 #include <asm/desc.h>

index f4b205686527..206996c1669d 100644
@@ -27,6 +27,7 @@
 #include <asm/processor.h>
 #include <asm/ucontext.h>
 #include <asm/fpu/internal.h>
+#include <asm/fpu/signal.h>
 #include <asm/vdso.h>
 #include <asm/mce.h>
 #include <asm/sighandling.h>

index e2ceb49d310d..7b62d9032623 100644
@@ -10,7 +10,6 @@
 #ifndef _ASM_X86_FPU_INTERNAL_H
 #define _ASM_X86_FPU_INTERNAL_H
 
-#include <linux/regset.h>
 #include <linux/compat.h>
 #include <linux/sched.h>
 #include <linux/slab.h>

@@ -48,18 +47,6 @@ extern void fpu__resume_cpu(void);
 
 DECLARE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx);
 
-extern user_regset_active_fn regset_fpregs_active, regset_xregset_fpregs_active;
-extern user_regset_get_fn fpregs_get, xfpregs_get, fpregs_soft_get,
-				xstateregs_get;
-extern user_regset_set_fn fpregs_set, xfpregs_set, fpregs_soft_set,
-				 xstateregs_set;
-
-/*
- * xstateregs_active == regset_fpregs_active. Please refer to the comment
- * at the definition of regset_fpregs_active.
- */
-#define xstateregs_active	regset_fpregs_active
-
 #ifdef CONFIG_MATH_EMULATION
 extern void finit_soft_fpu(struct i387_soft_struct *soft);
 #else

new file mode 100644
index 000000000000..39d3107ac6c7
@@ -0,0 +1,21 @@
+/*
+ * FPU regset handling methods:
+ */
+#ifndef _ASM_X86_FPU_REGSET_H
+#define _ASM_X86_FPU_REGSET_H
+
+#include <linux/regset.h>
+
+extern user_regset_active_fn regset_fpregs_active, regset_xregset_fpregs_active;
+extern user_regset_get_fn fpregs_get, xfpregs_get, fpregs_soft_get,
+				xstateregs_get;
+extern user_regset_set_fn fpregs_set, xfpregs_set, fpregs_soft_set,
+				 xstateregs_set;
+
+/*
+ * xstateregs_active == regset_fpregs_active. Please refer to the comment
+ * at the definition of regset_fpregs_active.
+ */
+#define xstateregs_active	regset_fpregs_active
+
+#endif /* _ASM_X86_FPU_REGSET_H */

index ab2c507b58b6..afd21329c585 100644
@@ -3,6 +3,7 @@
 
 #include <linux/types.h>
 #include <asm/processor.h>
+#include <linux/uaccess.h>
 
 /* Bit 63 of XCR0 is reserved for future expansion */
 #define XSTATE_EXTEND_MASK	(~(XSTATE_FPSSE | (1ULL << 63)))

index 45a2ca26a36f..b34c1a72d838 100644
@@ -6,6 +6,7 @@
  *	Gareth Hughes <[email protected]>, May 2000
  */
 #include <asm/fpu/internal.h>
+#include <asm/fpu/regset.h>
 #include <asm/fpu/signal.h>
 
 #include <linux/hardirq.h>

index 78710740e9a0..59bd35a57afc 100644
@@ -5,9 +5,11 @@
  */
 #include <linux/compat.h>
 #include <linux/cpu.h>
+
 #include <asm/fpu/api.h>
 #include <asm/fpu/internal.h>
 #include <asm/fpu/signal.h>
+#include <asm/fpu/regset.h>
 #include <asm/sigframe.h>
 #include <asm/tlbflush.h>
 

index 51e73a685ce4..9be72bc3613f 100644
@@ -11,7 +11,6 @@
 #include <linux/errno.h>
 #include <linux/slab.h>
 #include <linux/ptrace.h>
-#include <linux/regset.h>
 #include <linux/tracehook.h>
 #include <linux/user.h>
 #include <linux/elf.h>

@@ -30,6 +29,7 @@
 #include <asm/processor.h>
 #include <asm/fpu/internal.h>
 #include <asm/fpu/signal.h>
+#include <asm/fpu/regset.h>
 #include <asm/debugreg.h>
 #include <asm/ldt.h>
 #include <asm/desc.h>

index 72219ce2385a..7b6265df6082 100644
@@ -143,12 +143,6 @@ EXPORT_SYMBOL_GPL(xstate_size);
  */
 static void fpu__init_system_xstate_size_legacy(void)
 {
-	static bool on_boot_cpu = 1;
-
-	if (!on_boot_cpu)
-		return;
-	on_boot_cpu = 0;
-
 	/*
 	 * Note that xstate_size might be overwriten later during
 	 * fpu__init_system_xstate().

index 59bd35a57afc..8285d4b40763 100644
@@ -658,12 +658,6 @@ void setup_xstate_comp(void)
  */
 static void setup_init_fpu_buf(void)
 {
-	static int on_boot_cpu = 1;
-
-	if (!on_boot_cpu)
-		return;
-	on_boot_cpu = 0;
-
 	if (!cpu_has_xsave)
 		return;
 

@@ -719,11 +713,6 @@ static void __init init_xstate_size(void)
 void fpu__init_system_xstate(void)
 {
 	unsigned int eax, ebx, ecx, edx;
-	static bool on_boot_cpu = 1;
-
-	if (!on_boot_cpu)
-		return;
-	on_boot_cpu = 0;
 
 	if (!cpu_has_xsave) {
 		pr_info("x86/fpu: Legacy x87 FPU detected.\n");

index 7b62d9032623..dfdafea6e56f 100644
@@ -30,7 +30,8 @@ extern void fpu__init_system(struct cpuinfo_x86 *c);
 extern void fpu__activate_curr(struct fpu *fpu);
 extern void fpstate_init(struct fpu *fpu);
 
-extern int dump_fpu(struct pt_regs *, struct user_i387_struct *);
+extern int  dump_fpu(struct pt_regs *, struct user_i387_struct *);
+extern int  fpu__exception_code(struct fpu *fpu, int trap_nr);
 
 /*
  * High level FPU state handling functions:

@@ -467,34 +468,4 @@ static inline void user_fpu_begin(void)
 	preempt_enable();
 }
 
-/*
- * i387 state interaction
- */
-static inline unsigned short get_fpu_cwd(struct task_struct *tsk)
-{
-	if (cpu_has_fxsr) {
-		return tsk->thread.fpu.state.fxsave.cwd;
-	} else {
-		return (unsigned short)tsk->thread.fpu.state.fsave.cwd;
-	}
-}
-
-static inline unsigned short get_fpu_swd(struct task_struct *tsk)
-{
-	if (cpu_has_fxsr) {
-		return tsk->thread.fpu.state.fxsave.swd;
-	} else {
-		return (unsigned short)tsk->thread.fpu.state.fsave.swd;
-	}
-}
-
-static inline unsigned short get_fpu_mxcsr(struct task_struct *tsk)
-{
-	if (cpu_has_xmm) {
-		return tsk->thread.fpu.state.fxsave.mxcsr;
-	} else {
-		return MXCSR_DEFAULT;
-	}
-}
-
 #endif /* _ASM_X86_FPU_INTERNAL_H */

97 lines not on this page · raw message

90 lines not on this page · raw message

index dfdafea6e56f..0236ae6ffc26 100644
@@ -28,7 +28,18 @@ extern void fpu__init_cpu_xstate(void);
 extern void fpu__init_system(struct cpuinfo_x86 *c);
 
 extern void fpu__activate_curr(struct fpu *fpu);
+
 extern void fpstate_init(struct fpu *fpu);
+#ifdef CONFIG_MATH_EMULATION
+extern void fpstate_init_soft(struct i387_soft_struct *soft);
+#else
+static inline void fpstate_init_soft(struct i387_soft_struct *soft) {}
+#endif
+static inline void fpstate_init_fxstate(struct i387_fxsave_struct *fx)
+{
+	fx->cwd = 0x37f;
+	fx->mxcsr = MXCSR_DEFAULT;
+}
 
 extern int  dump_fpu(struct pt_regs *, struct user_i387_struct *);
 extern int  fpu__exception_code(struct fpu *fpu, int trap_nr);

@@ -48,12 +59,6 @@ extern void fpu__resume_cpu(void);
 
 DECLARE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx);
 
-#ifdef CONFIG_MATH_EMULATION
-extern void finit_soft_fpu(struct i387_soft_struct *soft);
-#else
-static inline void finit_soft_fpu(struct i387_soft_struct *soft) {}
-#endif
-
 /*
  * Must be run with preemption disabled: this clears the fpu_fpregs_owner_ctx,
  * on this CPU.

@@ -93,12 +98,6 @@ static __always_inline __pure bool use_fxsr(void)
 	return static_cpu_has_safe(X86_FEATURE_FXSR);
 }
 
-static inline void fx_finit(struct i387_fxsave_struct *fx)
-{
-	fx->cwd = 0x37f;
-	fx->mxcsr = MXCSR_DEFAULT;
-}
-
 extern void fpstate_sanitize_xstate(struct fpu *fpu);
 
 #define user_insn(insn, output, input...)				\

index 38274909e193..8eeba60bf78b 100644
@@ -191,24 +191,30 @@ void fpu__save(struct fpu *fpu)
 }
 EXPORT_SYMBOL_GPL(fpu__save);
 
+/*
+ * Legacy x87 fpstate state init:
+ */
+static inline void fpstate_init_fstate(struct i387_fsave_struct *fp)
+{
+	fp->cwd = 0xffff037fu;
+	fp->swd = 0xffff0000u;
+	fp->twd = 0xffffffffu;
+	fp->fos = 0xffff0000u;
+}
+
 void fpstate_init(struct fpu *fpu)
 {
 	if (!cpu_has_fpu) {
-		finit_soft_fpu(&fpu->state.soft);
+		fpstate_init_soft(&fpu->state.soft);
 		return;
 	}
 
 	memset(&fpu->state, 0, xstate_size);
 
-	if (cpu_has_fxsr) {
-		fx_finit(&fpu->state.fxsave);
-	} else {
-		struct i387_fsave_struct *fp = &fpu->state.fsave;
-		fp->cwd = 0xffff037fu;
-		fp->swd = 0xffff0000u;
-		fp->twd = 0xffffffffu;
-		fp->fos = 0xffff0000u;
-	}
+	if (cpu_has_fxsr)
+		fpstate_init_fxstate(&fpu->state.fxsave);
+	else
+		fpstate_init_fstate(&fpu->state.fsave);
 }
 EXPORT_SYMBOL_GPL(fpstate_init);
 

index 7b6265df6082..5a7e57078935 100644
@@ -121,7 +121,7 @@ static void fpu__init_system_generic(void)
 	 * Set up the legacy init FPU context. (xstate init might overwrite this
 	 * with a more modern format, if the CPU supports it.)
 	 */
-	fx_finit(&init_xstate_ctx.i387);
+	fpstate_init_fxstate(&init_xstate_ctx.i387);
 
 	fpu__init_system_mxcsr();
 }

index 7562341ce299..768b2b8271d6 100644
@@ -30,7 +30,7 @@ static void fclex(void)
 }
 
 /* Needs to be externally visible */
-void finit_soft_fpu(struct i387_soft_struct *soft)
+void fpstate_init_soft(struct i387_soft_struct *soft)
 {
 	struct address *oaddr, *iaddr;
 	memset(soft, 0, sizeof(*soft));

@@ -52,7 +52,7 @@ void finit_soft_fpu(struct i387_soft_struct *soft)
 
 void finit(void)
 {
-	finit_soft_fpu(&current->thread.fpu.state.soft);
+	fpstate_init_soft(&current->thread.fpu.state.soft);
 }
 
 /*

index 0236ae6ffc26..b74aa4329aeb 100644
@@ -29,7 +29,7 @@ extern void fpu__init_system(struct cpuinfo_x86 *c);
 
 extern void fpu__activate_curr(struct fpu *fpu);
 
-extern void fpstate_init(struct fpu *fpu);
+extern void fpstate_init(union thread_xstate *state);
 #ifdef CONFIG_MATH_EMULATION
 extern void fpstate_init_soft(struct i387_soft_struct *soft);
 #else

index 8eeba60bf78b..2642a1ebed2a 100644
@@ -202,19 +202,19 @@ static inline void fpstate_init_fstate(struct i387_fsave_struct *fp)
 	fp->fos = 0xffff0000u;
 }
 
-void fpstate_init(struct fpu *fpu)
+void fpstate_init(union thread_xstate *state)
 {
 	if (!cpu_has_fpu) {
-		fpstate_init_soft(&fpu->state.soft);
+		fpstate_init_soft(&state->soft);
 		return;
 	}
 
-	memset(&fpu->state, 0, xstate_size);
+	memset(state, 0, xstate_size);
 
 	if (cpu_has_fxsr)
-		fpstate_init_fxstate(&fpu->state.fxsave);
+		fpstate_init_fxstate(&state->fxsave);
 	else
-		fpstate_init_fstate(&fpu->state.fsave);
+		fpstate_init_fstate(&state->fsave);
 }
 EXPORT_SYMBOL_GPL(fpstate_init);
 

@@ -282,7 +282,7 @@ void fpu__activate_curr(struct fpu *fpu)
 	WARN_ON_ONCE(fpu != &current->thread.fpu);
 
 	if (!fpu->fpstate_active) {
-		fpstate_init(fpu);
+		fpstate_init(&fpu->state);
 
 		/* Safe to do for the current task: */
 		fpu->fpstate_active = 1;

@@ -321,7 +321,7 @@ static void fpu__activate_stopped(struct fpu *child_fpu)
 	if (child_fpu->fpstate_active) {
 		child_fpu->last_cpu = -1;
 	} else {
-		fpstate_init(child_fpu);
+		fpstate_init(&child_fpu->state);
 
 		/* Safe to do for stopped child tasks: */
 		child_fpu->fpstate_active = 1;

index 8285d4b40763..afbd58277430 100644
@@ -456,7 +456,7 @@ static int __fpu__restore_sig(void __user *buf, void __user *buf_fx, int size)
 
 		if (__copy_from_user(&fpu->state.xsave, buf_fx, state_size) ||
 		    __copy_from_user(&env, buf, sizeof(env))) {
-			fpstate_init(fpu);
+			fpstate_init(&fpu->state);
 			err = -1;
 		} else {
 			sanitize_restored_xstate(tsk, &env, xfeatures, fx_only);

index f4438179398b..3d811bb2728f 100644
@@ -7004,7 +7004,7 @@ int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 
 static void fx_init(struct kvm_vcpu *vcpu)
 {
-	fpstate_init(&vcpu->arch.guest_fpu);
+	fpstate_init(&vcpu->arch.guest_fpu.state);
 	if (cpu_has_xsaves)
 		vcpu->arch.guest_fpu.state.xsave.header.xcomp_bv =
 			host_xcr0 | XSTATE_COMPACTION_ENABLED;

index b74aa4329aeb..792fdbe64179 100644
@@ -22,6 +22,8 @@
 
 extern unsigned int mxcsr_feature_mask;
 
+extern union thread_xstate init_fpstate;
+
 extern void fpu__init_cpu(void);
 extern void fpu__init_system_xstate(void);
 extern void fpu__init_cpu_xstate(void);

@@ -342,9 +344,9 @@ static inline void fpregs_deactivate(struct fpu *fpu)
 static inline void restore_init_xstate(void)
 {
 	if (use_xsave())
-		xrstor_state(&init_xstate_ctx, -1);
+		xrstor_state(&init_fpstate.xsave, -1);
 	else
-		fxrstor_checking(&init_xstate_ctx.i387);
+		fxrstor_checking(&init_fpstate.fxsave);
 }
 
 /*

index afd21329c585..3051280887b8 100644
@@ -37,7 +37,6 @@
 extern unsigned int xstate_size;
 extern u64 xfeatures_mask;
 extern u64 xstate_fx_sw_bytes[USER_XSTATE_FX_SW_WORDS];
-extern struct xsave_struct init_xstate_ctx;
 
 extern void update_regset_xstate_info(unsigned int size, u64 xstate_mask);
 

index 2642a1ebed2a..43c3f40aa447 100644
@@ -13,6 +13,12 @@
 #include <linux/hardirq.h>
 
 /*
+ * Represents the initial FPU state. It's mostly (but not completely) zeroes,
+ * depending on the FPU hardware format:
+ */
+union thread_xstate init_fpstate __read_mostly;
+
+/*
  * Track whether the kernel is using the FPU state
  * currently.
  *

index 5a7e57078935..93bc11a5812c 100644
@@ -121,7 +121,7 @@ static void fpu__init_system_generic(void)
 	 * Set up the legacy init FPU context. (xstate init might overwrite this
 	 * with a more modern format, if the CPU supports it.)
 	 */
-	fpstate_init_fxstate(&init_xstate_ctx.i387);
+	fpstate_init_fxstate(&init_fpstate.fxsave);
 
 	fpu__init_system_mxcsr();
 }

index afbd58277430..527d4bf4f304 100644
@@ -31,11 +31,6 @@ static const char *xfeature_names[] =
  */
 u64 xfeatures_mask __read_mostly;
 
-/*
- * Represents init state for the supported extended state.
- */
-struct xsave_struct init_xstate_ctx;
-
 static struct _fpx_sw_bytes fx_sw_reserved, fx_sw_reserved_ia32;
 static unsigned int xstate_offsets[XFEATURES_NR_MAX], xstate_sizes[XFEATURES_NR_MAX];
 static unsigned int xstate_comp_offsets[sizeof(xfeatures_mask)*8];

@@ -150,7 +145,7 @@ void fpstate_sanitize_xstate(struct fpu *fpu)
 			int size = xstate_sizes[feature_bit];
 
 			memcpy((void *)fx + offset,
-			       (void *)&init_xstate_ctx + offset,
+			       (void *)&init_fpstate.xsave + offset,
 			       size);
 		}
 

@@ -377,12 +372,12 @@ static inline int restore_user_xstate(void __user *buf, u64 xbv, int fx_only)
 	if (use_xsave()) {
 		if ((unsigned long)buf % 64 || fx_only) {
 			u64 init_bv = xfeatures_mask & ~XSTATE_FPSSE;
-			xrstor_state(&init_xstate_ctx, init_bv);
+			xrstor_state(&init_fpstate.xsave, init_bv);
 			return fxrstor_user(buf);
 		} else {
 			u64 init_bv = xfeatures_mask & ~xbv;
 			if (unlikely(init_bv))
-				xrstor_state(&init_xstate_ctx, init_bv);
+				xrstor_state(&init_fpstate.xsave, init_bv);
 			return xrestore_user(buf, xbv);
 		}
 	} else if (use_fxsr()) {

@@ -665,20 +660,20 @@ static void setup_init_fpu_buf(void)
 	print_xstate_features();
 
 	if (cpu_has_xsaves) {
-		init_xstate_ctx.header.xcomp_bv = (u64)1 << 63 | xfeatures_mask;
-		init_xstate_ctx.header.xfeatures = xfeatures_mask;
+		init_fpstate.xsave.header.xcomp_bv = (u64)1 << 63 | xfeatures_mask;
+		init_fpstate.xsave.header.xfeatures = xfeatures_mask;
 	}
 
 	/*
 	 * Init all the features state with header_bv being 0x0
 	 */
-	xrstor_state_booting(&init_xstate_ctx, -1);
+	xrstor_state_booting(&init_fpstate.xsave, -1);
 
 	/*
 	 * Dump the init state again. This is to identify the init state
 	 * of any feature which is not represented by all zero's.
 	 */
-	xsave_state_booting(&init_xstate_ctx);
+	xsave_state_booting(&init_fpstate.xsave);
 }
 
 /*