Skip to contentSkip to conversation

x86/fpu: Spring cleaning and PKRU sanitizing

PATCH Applied Thomas Gleixner 6 participants 142 messages Jun 23, 2021 – Aug 5, 2022 Page 2 of 2

Subsystems (4)
lore
Share & tools
lore
#135 lore raw

PATCH v4 63/65 x86/fpu: Return proper error codes from user access functions

In reply to Thomas Gleixner

When *RSTOR from user memory raises an exception there is no way to differentiate them. That's bad because it forces the slow path even when the failure was not a fault. If the operation raised eg. #GP then going through the slow path is pointless.

Use _ASM_EXTABLE_FAULT() which stores the trap number and let the exception fixup return the negated trap number as error.

This allows to separate the fast path and let it handle faults directly and avoid the slow path for all other exceptions.

---
V4: Fixup XSTATE_OP() as well (Boris)

---
 arch/x86/include/asm/fpu/internal.h |   19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)
arch/x86/include/asm/fpu/internal.h +12 -7
@@ -88,6 +88,7 @@ static inline void fpstate_init_soft(str
 #endif
 extern void save_fpregs_to_fpstate(struct fpu *fpu);
 
+/* Returns 0 or the negated trap number, which results in -EFAULT for #PF */
 #define user_insn(insn, output, input...)				\
 ({									\
 	int err;							\
@@ -95,14 +96,14 @@ extern void save_fpregs_to_fpstate(struc
 	might_fault();							\
 									\
 	asm volatile(ASM_STAC "\n"					\
-		     "1:" #insn "\n\t"					\
+		     "1: " #insn "\n"					\
 		     "2: " ASM_CLAC "\n"				\
 		     ".section .fixup,\"ax\"\n"				\
-		     "3:  movl $-1,%[err]\n"				\
+		     "3:  negl %%eax\n"					\
 		     "    jmp  2b\n"					\
 		     ".previous\n"					\
-		     _ASM_EXTABLE(1b, 3b)				\
-		     : [err] "=r" (err), output				\
+		     _ASM_EXTABLE_FAULT(1b, 3b)				\
+		     : [err] "=a" (err), output				\
 		     : "0"(0), input);					\
 	err;								\
 })
@@ -196,16 +197,20 @@ static inline void fxsave(struct fxregs_
 #define XRSTOR		".byte " REX_PREFIX "0x0f,0xae,0x2f"
 #define XRSTORS		".byte " REX_PREFIX "0x0f,0xc7,0x1f"
 
+/*
+ * After this @err contains 0 on success or the negated trap number when
+ * the operation raises an exception. For faults this results in -EFAULT.
+ */
 #define XSTATE_OP(op, st, lmask, hmask, err)				\
 	asm volatile("1:" op "\n\t"					\
 		     "xor %[err], %[err]\n"				\
 		     "2:\n\t"						\
 		     ".pushsection .fixup,\"ax\"\n\t"			\
-		     "3: movl $-2,%[err]\n\t"				\
+		     "3: negl %%eax\n\t"				\
 		     "jmp 2b\n\t"					\
 		     ".popsection\n\t"					\
-		     _ASM_EXTABLE(1b, 3b)				\
-		     : [err] "=r" (err)					\
+		     _ASM_EXTABLE_FAULT(1b, 3b)				\
+		     : [err] "=a" (err)					\
 		     : "D" (st), "m" (*st), "a" (lmask), "d" (hmask)	\
 		     : "memory")
 
#136 lore raw

[tip: x86/fpu] x86/fpu: Return proper error codes from user access functions

The following commit has been merged into the x86/fpu branch of tip:

The following commit has been merged into the x86/fpu branch of tip:

Commit-ID:     aee8c67a4faa40a8df4e79316dbfc92d123989c1
Gitweb:        https://git.kernel.org/tip/aee8c67a4faa40a8df4e79316dbfc92d123989c1
Author:        Thomas Gleixner <[email protected]>
AuthorDate:    Wed, 23 Jun 2021 14:02:30 +02:00
Committer:     Borislav Petkov <[email protected]>
CommitterDate: Wed, 23 Jun 2021 20:04:58 +02:00

x86/fpu: Return proper error codes from user access functions

When *RSTOR from user memory raises an exception, there is no way to differentiate them. That's bad because it forces the slow path even when the failure was not a fault. If the operation raised eg. #GP then going through the slow path is pointless.

Use _ASM_EXTABLE_FAULT() which stores the trap number and let the exception fixup return the negated trap number as error.

This allows to separate the fast path and let it handle faults directly and avoid the slow path for all other exceptions.

---
 arch/x86/include/asm/fpu/internal.h | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)
arch/x86/include/asm/fpu/internal.h +12 -7
index 528a868..5a18694 100644
@@ -88,6 +88,7 @@ static inline void fpstate_init_soft(struct swregs_state *soft) {}
 #endif
 extern void save_fpregs_to_fpstate(struct fpu *fpu);
 
+/* Returns 0 or the negated trap number, which results in -EFAULT for #PF */
 #define user_insn(insn, output, input...)				\
 ({									\
 	int err;							\
@@ -95,14 +96,14 @@ extern void save_fpregs_to_fpstate(struct fpu *fpu);
 	might_fault();							\
 									\
 	asm volatile(ASM_STAC "\n"					\
-		     "1:" #insn "\n\t"					\
+		     "1: " #insn "\n"					\
 		     "2: " ASM_CLAC "\n"				\
 		     ".section .fixup,\"ax\"\n"				\
-		     "3:  movl $-1,%[err]\n"				\
+		     "3:  negl %%eax\n"					\
 		     "    jmp  2b\n"					\
 		     ".previous\n"					\
-		     _ASM_EXTABLE(1b, 3b)				\
-		     : [err] "=r" (err), output				\
+		     _ASM_EXTABLE_FAULT(1b, 3b)				\
+		     : [err] "=a" (err), output				\
 		     : "0"(0), input);					\
 	err;								\
 })
@@ -196,16 +197,20 @@ static inline void fxsave(struct fxregs_state *fx)
 #define XRSTOR		".byte " REX_PREFIX "0x0f,0xae,0x2f"
 #define XRSTORS		".byte " REX_PREFIX "0x0f,0xc7,0x1f"
 
+/*
+ * After this @err contains 0 on success or the negated trap number when
+ * the operation raises an exception. For faults this results in -EFAULT.
+ */
 #define XSTATE_OP(op, st, lmask, hmask, err)				\
 	asm volatile("1:" op "\n\t"					\
 		     "xor %[err], %[err]\n"				\
 		     "2:\n\t"						\
 		     ".pushsection .fixup,\"ax\"\n\t"			\
-		     "3: movl $-2,%[err]\n\t"				\
+		     "3: negl %%eax\n\t"				\
 		     "jmp 2b\n\t"					\
 		     ".popsection\n\t"					\
-		     _ASM_EXTABLE(1b, 3b)				\
-		     : [err] "=r" (err)					\
+		     _ASM_EXTABLE_FAULT(1b, 3b)				\
+		     : [err] "=a" (err)					\
 		     : "D" (st), "m" (*st), "a" (lmask), "d" (hmask)	\
 		     : "memory")
 
#137 lore raw

PATCH v4 64/65 x86/fpu/signal: Handle #PF in the direct restore path

In reply to Thomas Gleixner

If *RSTOR raises an exception, then the slow path is taken. That's wrong because if the reason was not #PF then going through the slow path is waste of time because that will end up with the same conclusion that the data is invalid.

Now that the wrapper around *RSTOR return an negative error code, which is the negated trap number, it's possible to differentiate.

If the *RSTOR raised #PF then handle it directly in the fast path and if it was some other exception, e.g. #GP, then give up and do not try the fast path.

This removes the legacy frame FRSTOR code from the slow path because FRSTOR is not a ia32_fxstate frame and is therefore handled in the fast path.

---
V4: Made the #PF retry path a bit more readable.

---
 arch/x86/kernel/fpu/signal.c |   67 +++++++++++++++++++++----------------------
 1 file changed, 33 insertions(+), 34 deletions(-)
arch/x86/kernel/fpu/signal.c +33 -34
@@ -272,11 +272,17 @@ static int __restore_fpregs_from_user(vo
 	}
 }
 
-static int restore_fpregs_from_user(void __user *buf, u64 xrestore, bool fx_only)
+/*
+ * Attempt to restore the FPU registers directly from user memory.
+ * Pagefaults are handled and any errors returned are fatal.
+ */
+static int restore_fpregs_from_user(void __user *buf, u64 xrestore,
+				    bool fx_only, unsigned int size)
 {
 	struct fpu *fpu = &current->thread.fpu;
 	int ret;
 
+retry:
 	fpregs_lock();
 	pagefault_disable();
 	ret = __restore_fpregs_from_user(buf, xrestore, fx_only);
@@ -293,14 +299,18 @@ static int restore_fpregs_from_user(void
 		 * invalidate the FPU register state otherwise the task
 		 * might preempt current and return to user space with
 		 * corrupted FPU registers.
-		 *
-		 * In case current owns the FPU registers then no further
-		 * action is required. The fixup in the slow path will
-		 * handle it correctly.
 		 */
 		if (test_thread_flag(TIF_NEED_FPU_LOAD))
 			__cpu_invalidate_fpregs_state();
 		fpregs_unlock();
+
+		/* Try to handle #PF, but anything else is fatal. */
+		if (ret != -EFAULT)
+			return -EINVAL;
+
+		ret = fault_in_pages_readable(buf, size);
+		if (!ret)
+			goto retry;
 		return ret;
 	}
 
@@ -311,9 +321,7 @@ static int restore_fpregs_from_user(void
 	 *
 	 * It would be optimal to handle this with a single XRSTORS, but
 	 * this does not work because the rest of the FPU registers have
-	 * been restored from a user buffer directly. The single XRSTORS
-	 * happens below, when the user buffer has been copied to the
-	 * kernel one.
+	 * been restored from a user buffer directly.
 	 */
 	if (test_thread_flag(TIF_NEED_FPU_LOAD) && xfeatures_mask_supervisor())
 		os_xrstor(&fpu->state.xsave, xfeatures_mask_supervisor());
@@ -326,14 +334,13 @@ static int restore_fpregs_from_user(void
 static int __fpu_restore_sig(void __user *buf, void __user *buf_fx,
 			     bool ia32_fxstate)
 {
-	struct user_i387_ia32_struct *envp = NULL;
 	int state_size = fpu_kernel_xstate_size;
 	struct task_struct *tsk = current;
 	struct fpu *fpu = &tsk->thread.fpu;
 	struct user_i387_ia32_struct env;
 	u64 user_xfeatures = 0;
 	bool fx_only = false;
-	int ret = 0;
+	int ret;
 
 	if (use_xsave()) {
 		struct _fpx_sw_bytes fx_sw_user;
@@ -354,21 +361,20 @@ static int __fpu_restore_sig(void __user
 		 * faults. If it does, fall back to the slow path below, going
 		 * through the kernel buffer with the enabled pagefault handler.
 		 */
-		ret = restore_fpregs_from_user(buf_fx, user_xfeatures, fx_only);
-		if (likely(!ret))
-			return 0;
-	} else {
-		/*
-		 * For 32-bit frames with fxstate, copy the fxstate so it can
-		 * be reconstructed later.
-		 */
-		ret = __copy_from_user(&env, buf, sizeof(env));
-		if (ret)
-			return ret;
-		envp = &env;
+		return restore_fpregs_from_user(buf_fx, user_xfeatures, fx_only,
+						state_size);
 	}
 
 	/*
+	 * Copy the legacy state because the FP portion of the FX frame has
+	 * to be ignored for histerical raisins. The legacy state is folded
+	 * in once the larger state has been copied.
+	 */
+	ret = __copy_from_user(&env, buf, sizeof(env));
+	if (ret)
+		return ret;
+
+	/*
 	 * By setting TIF_NEED_FPU_LOAD it is ensured that our xstate is
 	 * not modified on context switch and that the xstate is considered
 	 * to be loaded again on return to userland (overriding last_cpu avoids
@@ -382,8 +388,7 @@ static int __fpu_restore_sig(void __user
 		 * supervisor state is preserved. Save the full state for
 		 * simplicity. There is no point in optimizing this by only
 		 * saving the supervisor states and then shuffle them to
-		 * the right place in memory. This is the slow path and the
-		 * above XRSTOR failed or ia32_fxstate is true. Shrug.
+		 * the right place in memory. It's ia32 mode. Shrug.
 		 */
 		if (xfeatures_mask_supervisor())
 			os_xsave(&fpu->state.xsave);
@@ -399,7 +404,7 @@ static int __fpu_restore_sig(void __user
 		if (ret)
 			return ret;
 
-		sanitize_restored_user_xstate(&fpu->state, envp, user_xfeatures);
+		sanitize_restored_user_xstate(&fpu->state, &env, user_xfeatures);
 
 		fpregs_lock();
 		if (unlikely(init_bv))
@@ -412,12 +417,12 @@ static int __fpu_restore_sig(void __user
 		ret = os_xrstor_safe(&fpu->state.xsave,
 				     user_xfeatures | xfeatures_mask_supervisor());
 
-	} else if (use_fxsr()) {
+	} else {
 		ret = __copy_from_user(&fpu->state.fxsave, buf_fx, state_size);
 		if (ret)
 			return -EFAULT;
 
-		sanitize_restored_user_xstate(&fpu->state, envp, user_xfeatures);
+		sanitize_restored_user_xstate(&fpu->state, &env, user_xfeatures);
 
 		fpregs_lock();
 		if (use_xsave()) {
@@ -428,14 +433,8 @@ static int __fpu_restore_sig(void __user
 		}
 
 		ret = fxrstor_safe(&fpu->state.fxsave);
-	} else {
-		ret = __copy_from_user(&fpu->state.fsave, buf_fx, state_size);
-		if (ret)
-			return ret;
-
-		fpregs_lock();
-		ret = frstor_safe(&fpu->state.fsave);
 	}
+
 	if (!ret)
 		fpregs_mark_activate();
 	else
#138 lore raw

[tip: x86/fpu] x86/fpu/signal: Handle #PF in the direct restore path

The following commit has been merged into the x86/fpu branch of tip:

The following commit has been merged into the x86/fpu branch of tip:

Commit-ID:     fcb3635f5018e53024c6be3c3213737f469f74ff
Gitweb:        https://git.kernel.org/tip/fcb3635f5018e53024c6be3c3213737f469f74ff
Author:        Thomas Gleixner <[email protected]>
AuthorDate:    Wed, 23 Jun 2021 14:02:31 +02:00
Committer:     Borislav Petkov <[email protected]>
CommitterDate: Wed, 23 Jun 2021 20:05:33 +02:00

x86/fpu/signal: Handle #PF in the direct restore path

If *RSTOR raises an exception, then the slow path is taken. That's wrong because if the reason was not #PF then going through the slow path is waste of time because that will end up with the same conclusion that the data is invalid.

Now that the wrapper around *RSTOR return an negative error code, which is the negated trap number, it's possible to differentiate.

If the *RSTOR raised #PF then handle it directly in the fast path and if it was some other exception, e.g. #GP, then give up and do not try the fast path.

This removes the legacy frame FRSTOR code from the slow path because FRSTOR is not a ia32_fxstate frame and is therefore handled in the fast path.

---
 arch/x86/kernel/fpu/signal.c | 67 +++++++++++++++++------------------
 1 file changed, 33 insertions(+), 34 deletions(-)
arch/x86/kernel/fpu/signal.c +33 -34
index aa268d9..4c252d0 100644
@@ -272,11 +272,17 @@ static int __restore_fpregs_from_user(void __user *buf, u64 xrestore,
 	}
 }
 
-static int restore_fpregs_from_user(void __user *buf, u64 xrestore, bool fx_only)
+/*
+ * Attempt to restore the FPU registers directly from user memory.
+ * Pagefaults are handled and any errors returned are fatal.
+ */
+static int restore_fpregs_from_user(void __user *buf, u64 xrestore,
+				    bool fx_only, unsigned int size)
 {
 	struct fpu *fpu = &current->thread.fpu;
 	int ret;
 
+retry:
 	fpregs_lock();
 	pagefault_disable();
 	ret = __restore_fpregs_from_user(buf, xrestore, fx_only);
@@ -293,14 +299,18 @@ static int restore_fpregs_from_user(void __user *buf, u64 xrestore, bool fx_only
 		 * invalidate the FPU register state otherwise the task
 		 * might preempt current and return to user space with
 		 * corrupted FPU registers.
-		 *
-		 * In case current owns the FPU registers then no further
-		 * action is required. The fixup in the slow path will
-		 * handle it correctly.
 		 */
 		if (test_thread_flag(TIF_NEED_FPU_LOAD))
 			__cpu_invalidate_fpregs_state();
 		fpregs_unlock();
+
+		/* Try to handle #PF, but anything else is fatal. */
+		if (ret != -EFAULT)
+			return -EINVAL;
+
+		ret = fault_in_pages_readable(buf, size);
+		if (!ret)
+			goto retry;
 		return ret;
 	}
 
@@ -311,9 +321,7 @@ static int restore_fpregs_from_user(void __user *buf, u64 xrestore, bool fx_only
 	 *
 	 * It would be optimal to handle this with a single XRSTORS, but
 	 * this does not work because the rest of the FPU registers have
-	 * been restored from a user buffer directly. The single XRSTORS
-	 * happens below, when the user buffer has been copied to the
-	 * kernel one.
+	 * been restored from a user buffer directly.
 	 */
 	if (test_thread_flag(TIF_NEED_FPU_LOAD) && xfeatures_mask_supervisor())
 		os_xrstor(&fpu->state.xsave, xfeatures_mask_supervisor());
@@ -326,14 +334,13 @@ static int restore_fpregs_from_user(void __user *buf, u64 xrestore, bool fx_only
 static int __fpu_restore_sig(void __user *buf, void __user *buf_fx,
 			     bool ia32_fxstate)
 {
-	struct user_i387_ia32_struct *envp = NULL;
 	int state_size = fpu_kernel_xstate_size;
 	struct task_struct *tsk = current;
 	struct fpu *fpu = &tsk->thread.fpu;
 	struct user_i387_ia32_struct env;
 	u64 user_xfeatures = 0;
 	bool fx_only = false;
-	int ret = 0;
+	int ret;
 
 	if (use_xsave()) {
 		struct _fpx_sw_bytes fx_sw_user;
@@ -354,21 +361,20 @@ static int __fpu_restore_sig(void __user *buf, void __user *buf_fx,
 		 * faults. If it does, fall back to the slow path below, going
 		 * through the kernel buffer with the enabled pagefault handler.
 		 */
-		ret = restore_fpregs_from_user(buf_fx, user_xfeatures, fx_only);
-		if (likely(!ret))
-			return 0;
-	} else {
-		/*
-		 * For 32-bit frames with fxstate, copy the fxstate so it can
-		 * be reconstructed later.
-		 */
-		ret = __copy_from_user(&env, buf, sizeof(env));
-		if (ret)
-			return ret;
-		envp = &env;
+		return restore_fpregs_from_user(buf_fx, user_xfeatures, fx_only,
+						state_size);
 	}
 
 	/*
+	 * Copy the legacy state because the FP portion of the FX frame has
+	 * to be ignored for histerical raisins. The legacy state is folded
+	 * in once the larger state has been copied.
+	 */
+	ret = __copy_from_user(&env, buf, sizeof(env));
+	if (ret)
+		return ret;
+
+	/*
 	 * By setting TIF_NEED_FPU_LOAD it is ensured that our xstate is
 	 * not modified on context switch and that the xstate is considered
 	 * to be loaded again on return to userland (overriding last_cpu avoids
@@ -382,8 +388,7 @@ static int __fpu_restore_sig(void __user *buf, void __user *buf_fx,
 		 * supervisor state is preserved. Save the full state for
 		 * simplicity. There is no point in optimizing this by only
 		 * saving the supervisor states and then shuffle them to
-		 * the right place in memory. This is the slow path and the
-		 * above XRSTOR failed or ia32_fxstate is true. Shrug.
+		 * the right place in memory. It's ia32 mode. Shrug.
 		 */
 		if (xfeatures_mask_supervisor())
 			os_xsave(&fpu->state.xsave);
@@ -399,7 +404,7 @@ static int __fpu_restore_sig(void __user *buf, void __user *buf_fx,
 		if (ret)
 			return ret;
 
-		sanitize_restored_user_xstate(&fpu->state, envp, user_xfeatures);
+		sanitize_restored_user_xstate(&fpu->state, &env, user_xfeatures);
 
 		fpregs_lock();
 		if (unlikely(init_bv))
@@ -412,12 +417,12 @@ static int __fpu_restore_sig(void __user *buf, void __user *buf_fx,
 		ret = os_xrstor_safe(&fpu->state.xsave,
 				     user_xfeatures | xfeatures_mask_supervisor());
 
-	} else if (use_fxsr()) {
+	} else {
 		ret = __copy_from_user(&fpu->state.fxsave, buf_fx, state_size);
 		if (ret)
 			return -EFAULT;
 
-		sanitize_restored_user_xstate(&fpu->state, envp, user_xfeatures);
+		sanitize_restored_user_xstate(&fpu->state, &env, user_xfeatures);
 
 		fpregs_lock();
 		if (use_xsave()) {
@@ -428,14 +433,8 @@ static int __fpu_restore_sig(void __user *buf, void __user *buf_fx,
 		}
 
 		ret = fxrstor_safe(&fpu->state.fxsave);
-	} else {
-		ret = __copy_from_user(&fpu->state.fsave, buf_fx, state_size);
-		if (ret)
-			return ret;
-
-		fpregs_lock();
-		ret = frstor_safe(&fpu->state.fsave);
 	}
+
 	if (!ret)
 		fpregs_mark_activate();
 	else
#139 lore raw

PATCH v4 65/65 x86/fpu/signal: Let xrstor handle the features to init

In reply to Thomas Gleixner

There is no reason to do an extra XRSTOR from initfp_state for feature bits which have been cleared by user space in the FX magic xfeatures storage.

Just clear them in the task's XSTATE header and do a full restore which will put these cleared features into init state.

There is no real difference in performance because the current code already does a full restore when the xfeatures bits are preserved as the signal frame setup has stored them, which is the full UABI feature set.

---
 arch/x86/kernel/fpu/signal.c |   89 ++++++++++++++-----------------------------
 1 file changed, 31 insertions(+), 58 deletions(-)
arch/x86/kernel/fpu/signal.c +31 -58
@@ -220,36 +220,6 @@ int copy_fpstate_to_sigframe(void __user
 	return 0;
 }
 
-static inline void
-sanitize_restored_user_xstate(union fpregs_state *state,
-			      struct user_i387_ia32_struct *ia32_env, u64 mask)
-{
-	struct xregs_state *xsave = &state->xsave;
-	struct xstate_header *header = &xsave->header;
-
-	if (use_xsave()) {
-		/*
-		 * Clear all feature bits which are not set in mask.
-		 *
-		 * Supervisor state has to be preserved. The sigframe
-		 * restore can only modify user features, i.e. @mask
-		 * cannot contain them.
-		 */
-		header->xfeatures &= mask | xfeatures_mask_supervisor();
-	}
-
-	if (use_fxsr()) {
-		/*
-		 * mscsr reserved bits must be masked to zero for security
-		 * reasons.
-		 */
-		xsave->i387.mxcsr &= mxcsr_feature_mask;
-
-		if (ia32_env)
-			convert_to_fxsr(&state->fxsave, ia32_env);
-	}
-}
-
 static int __restore_fpregs_from_user(void __user *buf, u64 xrestore,
 				      bool fx_only)
 {
@@ -352,6 +322,8 @@ static int __fpu_restore_sig(void __user
 		fx_only = !fx_sw_user.magic1;
 		state_size = fx_sw_user.xstate_size;
 		user_xfeatures = fx_sw_user.xfeatures;
+	} else {
+		user_xfeatures = XFEATURE_MASK_FPSSE;
 	}
 
 	if (likely(!ia32_fxstate)) {
@@ -395,54 +367,55 @@ static int __fpu_restore_sig(void __user
 		set_thread_flag(TIF_NEED_FPU_LOAD);
 	}
 	__fpu_invalidate_fpregs_state(fpu);
+	__cpu_invalidate_fpregs_state();
 	fpregs_unlock();
 
 	if (use_xsave() && !fx_only) {
-		u64 init_bv = xfeatures_mask_uabi() & ~user_xfeatures;
-
 		ret = copy_sigframe_from_user_to_xstate(&fpu->state.xsave, buf_fx);
 		if (ret)
 			return ret;
+	} else {
+		if (__copy_from_user(&fpu->state.fxsave, buf_fx,
+				     sizeof(fpu->state.fxsave)))
+			return -EFAULT;
 
-		sanitize_restored_user_xstate(&fpu->state, &env, user_xfeatures);
+		/* Reject invalid MXCSR values. */
+		if (fpu->state.fxsave.mxcsr & mxcsr_feature_mask)
+			return -EINVAL;
 
-		fpregs_lock();
-		if (unlikely(init_bv))
-			os_xrstor(&init_fpstate.xsave, init_bv);
+		/* Enforce XFEATURE_MASK_FPSSE when XSAVE is enabled */
+		if (use_xsave())
+			fpu->state.xsave.header.xfeatures |= XFEATURE_MASK_FPSSE;
+	}
+
+	/* Fold the legacy FP storage */
+	convert_to_fxsr(&fpu->state.fxsave, &env);
 
+	fpregs_lock();
+	if (use_xsave()) {
 		/*
-		 * Restore previously saved supervisor xstates along with
-		 * copied-in user xstates.
+		 * Remove all UABI feature bits not set in user_xfeatures
+		 * from the memory xstate header which makes the full
+		 * restore below bring them into init state. This works for
+		 * fx_only mode as well because that has only FP and SSE
+		 * set in user_xfeatures.
+		 *
+		 * Preserve supervisor states!
 		 */
-		ret = os_xrstor_safe(&fpu->state.xsave,
-				     user_xfeatures | xfeatures_mask_supervisor());
+		u64 mask = user_xfeatures | xfeatures_mask_supervisor();
 
+		fpu->state.xsave.header.xfeatures &= mask;
+		ret = os_xrstor_safe(&fpu->state.xsave, xfeatures_mask_all);
 	} else {
-		ret = __copy_from_user(&fpu->state.fxsave, buf_fx, state_size);
-		if (ret)
-			return -EFAULT;
-
-		sanitize_restored_user_xstate(&fpu->state, &env, user_xfeatures);
-
-		fpregs_lock();
-		if (use_xsave()) {
-			u64 init_bv;
-
-			init_bv = xfeatures_mask_uabi() & ~XFEATURE_MASK_FPSSE;
-			os_xrstor(&init_fpstate.xsave, init_bv);
-		}
-
 		ret = fxrstor_safe(&fpu->state.fxsave);
 	}
 
-	if (!ret)
+	if (likely(!ret))
 		fpregs_mark_activate();
-	else
-		fpregs_deactivate(fpu);
+
 	fpregs_unlock();
 	return ret;
 }
-
 static inline int xstate_sigframe_size(void)
 {
 	return use_xsave() ? fpu_user_xstate_size + FP_XSTATE_MAGIC2_SIZE :
#140 lore raw

[tip: x86/fpu] x86/fpu/signal: Let xrstor handle the features to init

The following commit has been merged into the x86/fpu branch of tip:

The following commit has been merged into the x86/fpu branch of tip:

Commit-ID:     6f9866a166cd1ad3ebb2dcdb3874aa8fee8dea2f
Gitweb:        https://git.kernel.org/tip/6f9866a166cd1ad3ebb2dcdb3874aa8fee8dea2f
Author:        Thomas Gleixner <[email protected]>
AuthorDate:    Wed, 23 Jun 2021 14:02:32 +02:00
Committer:     Borislav Petkov <[email protected]>
CommitterDate: Wed, 23 Jun 2021 23:45:31 +02:00

x86/fpu/signal: Let xrstor handle the features to init

There is no reason to do an extra XRSTOR from init_fpstate for feature bits which have been cleared by user space in the FX magic xfeatures storage.

Just clear them in the task's XSTATE header and do a full restore which will put these cleared features into init state.

There is no real difference in performance because the current code already does a full restore when the xfeatures bits are preserved as the signal frame setup has stored them, which is the full UABI feature set.

[ bp: Use the negated mxcsr_feature_mask in the MXCSR check. ]

---
 arch/x86/kernel/fpu/signal.c | 89 ++++++++++++-----------------------
 1 file changed, 31 insertions(+), 58 deletions(-)
arch/x86/kernel/fpu/signal.c +31 -58
index 4c252d0..445c57c 100644
@@ -220,36 +220,6 @@ retry:
 	return 0;
 }
 
-static inline void
-sanitize_restored_user_xstate(union fpregs_state *state,
-			      struct user_i387_ia32_struct *ia32_env, u64 mask)
-{
-	struct xregs_state *xsave = &state->xsave;
-	struct xstate_header *header = &xsave->header;
-
-	if (use_xsave()) {
-		/*
-		 * Clear all feature bits which are not set in mask.
-		 *
-		 * Supervisor state has to be preserved. The sigframe
-		 * restore can only modify user features, i.e. @mask
-		 * cannot contain them.
-		 */
-		header->xfeatures &= mask | xfeatures_mask_supervisor();
-	}
-
-	if (use_fxsr()) {
-		/*
-		 * mscsr reserved bits must be masked to zero for security
-		 * reasons.
-		 */
-		xsave->i387.mxcsr &= mxcsr_feature_mask;
-
-		if (ia32_env)
-			convert_to_fxsr(&state->fxsave, ia32_env);
-	}
-}
-
 static int __restore_fpregs_from_user(void __user *buf, u64 xrestore,
 				      bool fx_only)
 {
@@ -352,6 +322,8 @@ static int __fpu_restore_sig(void __user *buf, void __user *buf_fx,
 		fx_only = !fx_sw_user.magic1;
 		state_size = fx_sw_user.xstate_size;
 		user_xfeatures = fx_sw_user.xfeatures;
+	} else {
+		user_xfeatures = XFEATURE_MASK_FPSSE;
 	}
 
 	if (likely(!ia32_fxstate)) {
@@ -395,54 +367,55 @@ static int __fpu_restore_sig(void __user *buf, void __user *buf_fx,
 		set_thread_flag(TIF_NEED_FPU_LOAD);
 	}
 	__fpu_invalidate_fpregs_state(fpu);
+	__cpu_invalidate_fpregs_state();
 	fpregs_unlock();
 
 	if (use_xsave() && !fx_only) {
-		u64 init_bv = xfeatures_mask_uabi() & ~user_xfeatures;
-
 		ret = copy_sigframe_from_user_to_xstate(&fpu->state.xsave, buf_fx);
 		if (ret)
 			return ret;
+	} else {
+		if (__copy_from_user(&fpu->state.fxsave, buf_fx,
+				     sizeof(fpu->state.fxsave)))
+			return -EFAULT;
 
-		sanitize_restored_user_xstate(&fpu->state, &env, user_xfeatures);
+		/* Reject invalid MXCSR values. */
+		if (fpu->state.fxsave.mxcsr & ~mxcsr_feature_mask)
+			return -EINVAL;
 
-		fpregs_lock();
-		if (unlikely(init_bv))
-			os_xrstor(&init_fpstate.xsave, init_bv);
+		/* Enforce XFEATURE_MASK_FPSSE when XSAVE is enabled */
+		if (use_xsave())
+			fpu->state.xsave.header.xfeatures |= XFEATURE_MASK_FPSSE;
+	}
+
+	/* Fold the legacy FP storage */
+	convert_to_fxsr(&fpu->state.fxsave, &env);
 
+	fpregs_lock();
+	if (use_xsave()) {
 		/*
-		 * Restore previously saved supervisor xstates along with
-		 * copied-in user xstates.
+		 * Remove all UABI feature bits not set in user_xfeatures
+		 * from the memory xstate header which makes the full
+		 * restore below bring them into init state. This works for
+		 * fx_only mode as well because that has only FP and SSE
+		 * set in user_xfeatures.
+		 *
+		 * Preserve supervisor states!
 		 */
-		ret = os_xrstor_safe(&fpu->state.xsave,
-				     user_xfeatures | xfeatures_mask_supervisor());
+		u64 mask = user_xfeatures | xfeatures_mask_supervisor();
 
+		fpu->state.xsave.header.xfeatures &= mask;
+		ret = os_xrstor_safe(&fpu->state.xsave, xfeatures_mask_all);
 	} else {
-		ret = __copy_from_user(&fpu->state.fxsave, buf_fx, state_size);
-		if (ret)
-			return -EFAULT;
-
-		sanitize_restored_user_xstate(&fpu->state, &env, user_xfeatures);
-
-		fpregs_lock();
-		if (use_xsave()) {
-			u64 init_bv;
-
-			init_bv = xfeatures_mask_uabi() & ~XFEATURE_MASK_FPSSE;
-			os_xrstor(&init_fpstate.xsave, init_bv);
-		}
-
 		ret = fxrstor_safe(&fpu->state.fxsave);
 	}
 
-	if (!ret)
+	if (likely(!ret))
 		fpregs_mark_activate();
-	else
-		fpregs_deactivate(fpu);
+
 	fpregs_unlock();
 	return ret;
 }
-
 static inline int xstate_sigframe_size(void)
 {
 	return use_xsave() ? fpu_user_xstate_size + FP_XSTATE_MAGIC2_SIZE :
#141 lore raw
In reply to Thomas Gleixner

Hi Thomas,

Quoted 33 lines · Thomas Gleixner

The main parts of this series are:

- Simplification and removal/replacement of redundant and/or overengineered code.

- Name space cleanup as the existing names were just a permanent source of confusion.

- Clear seperation of user ABI and kernel internal state handling.

- Removal of PKRU from being XSTATE managed in the kernel because PKRU has to be eagerly restored on context switch and keeping it in sync in the xstate buffer is just pointless overhead and fragile.

The kernel still XSAVEs PKRU on context switch but the value in the buffer is not longer used and never restored from the buffer.

This still needs to be cleaned up, but the series is already 40+ patches large and the cleanup of this is not a functional problem.

The functional issues of PKRU management are fully addressed with the series as is.

- Cleanup of fpu signal restore

- Make the fast path self contained. Handle #PF directly and skip the slow path on any other exception as that will just end up with the same result that the frame is invalid. This allows the compiler to optimize the slow path out for 64bit kernels w/o ia32 emulation.

- Reduce code duplication and unnecessary operations

Quoted 7 lines

It applies on top of

git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git master

and is also available via git:

git://git.kernel.org/pub/scm/linux/kernel/git/tglx/devel.git x86/fpu

0-Day kernel CI tested this branch from performance view, choosing some sub-tests from will-it-scale and stress-ng (detail as below)

Test Summary
============
no obvious performance changes found vs v5.13-rc7 so far

Test Environment
================
https://git.kernel.org/pub/scm/linux/kernel/git/tglx/devel.git/log/?h=x86/fpu
* a263e30a4f111 (tglx-devel/x86/fpu) x86/fpu/signal: Let xrstor handle the features to init
* cf917e53a97e9 x86/fpu/signal: Handle #PF in the direct restore path
* 7b269dff0be66 x86/fpu: Return proper error codes from user access functions
* d2d6be9e16386 x86/fpu/signal: Split out the direct restore code
...
* d3ca29fb9911f x86/fpu: Fix copy_xstate_to_kernel() gap handling
* 58a664af246e6 Merge branch 'x86/fpu' of ../tip into x86/fpu
|\
| * b7c11876d24bd (tip/x86/fpu, peterz-queue/x86/fpu) selftests/x86: Test signal frame XSTATE header corruption handling
...
* | 13311e74253fe (tag: v5.13-rc7

64bit kernel testing, upon below platforms:
(1)
model: Cascade Lake
Intel(R) Xeon(R) Gold 6238M CPU @ 2.10GHz
nr_node: 2
nr_cpu: 88
memory: 128G
(2)
model: Ice Lake
nr_node: 2
nr_cpu: 96
memory: 256G

32bit kernel testing, upon below platform:
Intel(R) Core(TM) i7-3770K CPU @ 3.50GHz
model: Ivy Bridge
nr_node: 1
nr_cpu: 8
memory: 16G

tested below test suites:
will-it-scale-performance-context_switch1
will-it-scale-performance-page_fault1
will-it-scale-performance-poll1
will-it-scale-performance-pthread_mutex1
will-it-scale-performance-writeseek1

regarding stress-ng, either no gap or data is unstable on 64bit (1) platform

commit:
v5.13-rc7
a263e30a4f111e13a6c859102d1b319a79b072c0

       v5.13-rc7 a263e30a4f111e13a6c859102d1                                                                     
---------------- ---------------------------                                                                     
         %stddev     %change         %stddev
             \          |                \
 2.786e+08 ±  2%      -1.3%  2.751e+08        stress-ng.switch.ops
   4643776 ±  2%      -1.3%    4585061        stress-ng.switch.ops_per_sec
--
  25884344 ± 15%      +5.3%   27252860 ± 12%  stress-ng.nanosleep.ops
    431397 ± 15%      +5.3%     454204 ± 12%  stress-ng.nanosleep.ops_per_sec
--
  5.48e+09 ±  2%      +4.3%  5.713e+09 ±  3%  stress-ng.yield.ops
  91326562 ±  2%      +4.3%   95215138 ±  3%  stress-ng.yield.ops_per_sec
--
   6723205 ±  3%      -2.0%    6585690 ±  9%  stress-ng.tee.ops
    112049 ±  3%      -2.0%     109755 ±  9%  stress-ng.tee.ops_per_sec
--
   3078530            -0.2%    3072610        stress-ng.zombie.ops
     49390            -0.2%      49295        stress-ng.zombie.ops_per_sec
--
   2261601            -0.1%    2259339        stress-ng.cyclic.ops
     37693            -0.1%      37655        stress-ng.cyclic.ops_per_sec
--
 1.543e+09 ± 16%      +0.4%   1.55e+09 ± 19%  stress-ng.fifo.ops
  25721633 ± 16%      +0.4%   25831339 ± 19%  stress-ng.fifo.ops_per_sec
--
    192171            +0.6%     193383        stress-ng.clone.ops
      2998            +0.6%       3016        stress-ng.clone.ops_per_sec
--
  88703410            -0.5%   88257268        stress-ng.sleep.ops
   1470841            -0.5%    1463509        stress-ng.sleep.ops_per_sec
--
    401598            +0.1%     401990        stress-ng.netlink-proc.ops
      6692            +0.1%       6699        stress-ng.netlink-proc.ops_per_sec
--
 1.182e+08            +2.5%  1.212e+08        stress-ng.softlockup.ops
   1969636            +2.5%    2019730        stress-ng.softlockup.ops_per_sec

--
1.866e+08 -0.2% 1.863e+08 stress-ng.fanotify.ops
4754116 ± 12% -2.8% 4620280 ± 5% stress-ng.fanotify.ops_per_sec
--

  17926280            -0.3%   17866738        stress-ng.vforkmany.ops
    286256            +0.4%     287262        stress-ng.vforkmany.ops_per_sec
--
 1.381e+09 ± 13%      +8.7%  1.501e+09 ± 19%  stress-ng.fifo.ops
  23013090 ± 13%      +8.7%   25020209 ± 19%  stress-ng.fifo.ops_per_sec
--
 1.165e+08 ±  2%      -2.0%  1.141e+08 ±  3%  stress-ng.splice.ops
   1941398 ±  2%      -2.0%    1901755 ±  3%  stress-ng.splice.ops_per_sec
--
   5476091            -0.2%    5466962        stress-ng.vfork.ops
     91262            -0.2%      91116        stress-ng.vfork.ops_per_sec
--
  21878786            -1.4%   21574984        stress-ng.netlink-task.ops
    364646            -1.4%     359582        stress-ng.netlink-task.ops_per_sec
--
   5023286            +1.6%    5101458        stress-ng.pthread.ops
     83554            +1.6%      84869        stress-ng.pthread.ops_per_sec
--
  92688395 ±  2%      -8.3%   84967931 ±  4%  stress-ng.futex.ops
   1544792 ±  2%      -8.3%    1416117 ±  4%  stress-ng.futex.ops_per_sec
--
   5416582 ±  2%      +1.6%    5504734 ±  5%  stress-ng.wait.ops
     90275 ±  2%      +1.6%      91744 ±  5%  stress-ng.wait.ops_per_sec
--
 7.939e+08 ±  4%      +0.7%  7.998e+08        stress-ng.mq.ops
  13231931 ±  4%      +0.7%   13329018        stress-ng.mq.ops_per_sec
--
 3.936e+08 ± 53%     +51.5%  5.965e+08 ± 37%  stress-ng.hrtimers.ops
   6416615 ± 52%     +49.8%    9609730 ± 38%  stress-ng.hrtimers.ops_per_sec
--
  35268682            -1.1%   34872339        stress-ng.sendfile.ops
    587805            -1.1%     581200        stress-ng.sendfile.ops_per_sec
--
  18126587            -0.7%   18000304 ±  3%  stress-ng.sem-sysv.ops
    302108            -0.7%     300004 ±  3%  stress-ng.sem-sysv.ops_per_sec
--
 9.953e+08 ±  2%      -0.4%  9.912e+08        stress-ng.pipe.ops
  16589030 ±  2%      -0.4%   16519202        stress-ng.pipe.ops_per_sec
--
   1056000            +0.0%    1056000        stress-ng.schedpolicy.ops
     17599            +0.0%      17599        stress-ng.schedpolicy.ops_per_sec
--
 2.481e+09            +1.2%   2.51e+09        stress-ng.vm-splice.ops
  41349053            +1.2%   41828864        stress-ng.vm-splice.ops_per_sec
--
  56805932 ±  2%      +1.1%   57451245 ±  2%  stress-ng.eventfd.ops
    946751 ±  2%      +1.1%     957505 ±  2%  stress-ng.eventfd.ops_per_sec
--
      1605            -1.2%       1585 ±  2%  stress-ng.mmapfork.ops
     25.18 ±  4%      -0.8%      24.98 ±  4%  stress-ng.mmapfork.ops_per_sec
--
   2549139            +1.2%    2578537        stress-ng.fork.ops
     42485            +1.2%      42975        stress-ng.fork.ops_per_sec
--
  16582541 ±  3%     -10.8%   14792067 ± 21%  stress-ng.close.ops
    276357 ±  3%     -10.8%     246515 ± 21%  stress-ng.close.ops_per_sec
--
 1.847e+09            -1.3%  1.822e+09        stress-ng.pipeherd.ops
  30743202            -1.3%   30330538        stress-ng.pipeherd.ops_per_sec
--
   6780757 ±  2%      +4.3%    7070424        stress-ng.tee.ops
    113008 ±  2%      +4.3%     117835        stress-ng.tee.ops_per_sec
--
   3421466            +0.0%    3422853        stress-ng.dnotify.ops
     57024            +0.0%      57047        stress-ng.dnotify.ops_per_sec
--
    749895            -0.9%     742801        stress-ng.kill.ops
     12498            -0.9%      12379        stress-ng.kill.ops_per_sec
--
 3.761e+08            -2.5%  3.668e+08        stress-ng.msg.ops
   6266833            -2.5%    6112949        stress-ng.msg.ops_per_sec
--
   2258479            +0.1%    2260255        stress-ng.daemon.ops
     37641            +0.1%      37670        stress-ng.daemon.ops_per_sec
--
  40560445            -0.8%   40239283        stress-ng.affinity.ops
    668138            -1.6%     657142        stress-ng.affinity.ops_per_sec
--
     21230 ±  4%      -0.9%      21047 ±  2%  stress-ng.inotify.ops
    353.14 ±  4%      -0.9%     349.88 ±  2%  stress-ng.inotify.ops_per_sec
--
  23250444 ±  2%      +1.6%   23627954        stress-ng.fault.ops
    387495 ±  2%      +1.6%     393795        stress-ng.fault.ops_per_sec
--
  3.94e+08            -0.2%  3.933e+08        stress-ng.sem.ops
   6567438            -0.2%    6555371        stress-ng.sem.ops_per_sec
--
    698544            -0.0%     698480        stress-ng.session.ops
     11494            +0.0%      11495        stress-ng.session.ops_per_sec
Quoted 59 lines

This is a follow up to V3 which can be found here:

     https://lore.kernel.org/r/[email protected]

Changes vs. V3:

- Dropped the two bugfixes which are applied already and rebased on top

- Addressed review comments (Andy, Boris)

    Patches: 13, 35, 36, 37, 46, 58, 62, 63

- Fixed the math-emu fallout which I had stashed safely on the 32bit

    testbox (Boris)

    Patch: 28

- Picked up tags

Thanks to everyone for review, feedback and testing (various teams @Intel).

Note: I've not picked up any tested-by tags. It would be nice to have them on this hopefully final version.

Thanks,

tglx

---
 arch/x86/events/intel/lbr.c          |    6 
 arch/x86/include/asm/fpu/internal.h  |  202 ++++------
 arch/x86/include/asm/fpu/xstate.h    |   78 +++-
 arch/x86/include/asm/pgtable.h       |   57 ---
 arch/x86/include/asm/pkeys.h         |    9 
 arch/x86/include/asm/pkru.h          |   62 +++
 arch/x86/include/asm/processor.h     |    9 
 arch/x86/include/asm/special_insns.h |   14 
 arch/x86/kernel/cpu/common.c         |   34 -
 arch/x86/kernel/fpu/core.c           |  282 +++++++--------
 arch/x86/kernel/fpu/init.c           |   15 
 arch/x86/kernel/fpu/regset.c         |  223 ++++++------
 arch/x86/kernel/fpu/signal.c         |  419 ++++++++++------------
 arch/x86/kernel/fpu/xstate.c         |  645 +++++++++++++----------------------
 arch/x86/kernel/process.c            |   22 +
 arch/x86/kernel/process_64.c         |   28 +
 arch/x86/kernel/traps.c              |    5 
 arch/x86/kvm/svm/sev.c               |    1 
 arch/x86/kvm/x86.c                   |   56 +--
 arch/x86/math-emu/fpu_proto.h        |    2 
 arch/x86/math-emu/load_store.c       |    2 
 arch/x86/math-emu/reg_ld_str.c       |    2 
 arch/x86/mm/extable.c                |    2 
 arch/x86/mm/fault.c                  |    2 
 arch/x86/mm/pkeys.c                  |   22 -
 include/linux/pkeys.h                |    4 
 26 files changed, 1037 insertions(+), 1166 deletions(-)