x86/fpu: Remove xstate_fault()
Borislav Petkov
maintainercover x86/fpu: Remove xstate_fault()
From: Borislav Petkov <[email protected]>
Hi,
so that xstate_fault() thing did cause trouble in the past, see
06c8173eb92b ("x86/fpu/xsaves: Fix improper uses of __ex_table")
So lemme kill it.
I had an earlier version of that patch, here's an updated one and split into two for better digestion.
The guest boots fine so far, I need to run it on the rest of the hw to check everything's fine.
Borislav Petkov (2):
x86/fpu: Add an XSTATE_OP macro
x86/fpu: Get rid of xstate_fault()
arch/x86/include/asm/fpu/internal.h | 173 +++++++++++++++++------------------- 1 file changed, 83 insertions(+), 90 deletions(-)
Signature · 1 line
2.3.5
Borislav Petkov
maintainerRFC PATCH 1/2 x86/fpu: Add an XSTATE_OP macro
From: Borislav Petkov <[email protected]>
Add an XSTATE_OP macro which contains the XSAVE* fault handling and replace all non-alternatives users of xstate_fault() with it.
This fixes also the buglet in copy_xregs_to_user() and copy_user_to_xregs() where the inline asm didn't have @xstate as memory reference and thus potentially causing unwanted reordering of accesses to the extended state.
--- arch/x86/include/asm/fpu/internal.h | 68 +++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 37 deletions(-)
arch/x86/include/asm/fpu/internal.h +31 -37
@@ -237,6 +237,20 @@ static inline void copy_fxregs_to_kernel(struct fpu *fpu) _ASM_EXTABLE(1b, 3b) \ : [_err] "=r" (__err) +#define XSTATE_OP (op, st, lmask, hmask, err) \ + asm volatile("1:" op "\n\t" \ + "xor %[err], %[err]\n" \ + "2:\n\t" \ + ".pushsection .fixup,\"ax\"\n\t" \ + "3: movl $-2,%[err]\n\t" \ + "jmp 2b\n\t" \ + ".popsection\n\t" \ + _ASM_EXTABLE(1b, 3b) \ + : [err] "=r" (err) \ + : "D" (st), "m" (*st), "a" (lmask), "d" (hmask) \ + : "memory") + +/* * This function is called only during boot time when x86 caps are not set * up and alternative can not be used yet.
@@ -246,22 +260,14 @@ static inline void copy_xregs_to_kernel_booting(struct xregs_state *xstate)u64 mask =-1 ;u32 lmask = mask;u32 hmask = mask >>32 ; -int err =0 ; +int err;WARN_ON (system_state !=SYSTEM_BOOTING ); -if (boot_cpu_has (X86_FEATURE_XSAVES )) - asm volatile("1:"XSAVES"\n\t" - "2:\n\t" - xstate_fault(err) - : "D" (xstate), "m" (*xstate), "a" (lmask), "d" (hmask), "0" (err) - : "memory"); +if (static_cpu_has_safe (X86_FEATURE_XSAVES )) +XSTATE_OP (XSAVES , xstate, lmask, hmask, err);else - asm volatile("1:"XSAVE"\n\t" - "2:\n\t" - xstate_fault(err) - : "D" (xstate), "m" (*xstate), "a" (lmask), "d" (hmask), "0" (err) - : "memory"); +XSTATE_OP (XSAVE , xstate, lmask, hmask, err);/* We should never fault when copying to a kernel buffer: */ WARN_ON_FPU (err);
@@ -276,22 +282,14 @@ static inline void copy_kernel_to_xregs_booting(struct xregs_state *xstate)u64 mask =-1 ;u32 lmask = mask;u32 hmask = mask >>32 ; -int err =0 ; +int err;WARN_ON (system_state !=SYSTEM_BOOTING ); -if (boot_cpu_has (X86_FEATURE_XSAVES )) - asm volatile("1:"XRSTORS"\n\t" - "2:\n\t" - xstate_fault(err) - : "D" (xstate), "m" (*xstate), "a" (lmask), "d" (hmask), "0" (err) - : "memory"); +if (static_cpu_has_safe (X86_FEATURE_XSAVES )) +XSTATE_OP (XRSTORS , xstate, lmask, hmask, err);else - asm volatile("1:"XRSTOR"\n\t" - "2:\n\t" - xstate_fault(err) - : "D" (xstate), "m" (*xstate), "a" (lmask), "d" (hmask), "0" (err) - : "memory"); +XSTATE_OP (XRSTOR , xstate, lmask, hmask, err);/* We should never fault when copying from a kernel buffer: */ WARN_ON_FPU (err);
@@ -388,12 +386,10 @@ static inline int copy_xregs_to_user(struct xregs_state __user *buf)if (unlikely (err))return -EFAULT ; - __asm__ __volatile__(ASM_STAC "\n" -"1:" XSAVE "\n" -"2: " ASM_CLAC "\n" - xstate_fault(err) - : "D" (buf), "a" (-1), "d" (-1), "0" (err) - : "memory"); +stac (); +XSTATE_OP (XSAVE , buf,-1 ,-1 , err); +clac (); +return err; }
@@ -405,14 +401,12 @@ static inline int copy_user_to_xregs(struct xregs_state __user *buf, u64 mask)struct xregs_state *xstate = ((__force struct xregs_state *)buf);u32 lmask = mask;u32 hmask = mask >>32 ; -int err =0 ; +int err; + +stac (); +XSTATE_OP (XRSTOR , xstate, lmask, hmask, err); +clac (); - __asm__ __volatile__(ASM_STAC "\n" -"1:" XRSTOR "\n" -"2: " ASM_CLAC "\n" - xstate_fault(err) - : "D" (xstate), "a" (lmask), "d" (hmask), "0" (err) - : "memory"); /* memory required? */return err; }
Signature · 1 line
2.3.5
tip-bot for Borislav Petkov
merge bot[tip:x86/fpu] x86/fpu: Add an XSTATE_OP() macro
x86/fpu: Add an XSTATE_OP() macro
Commit-ID: b74a0cf1b3db30173eefa00c411775d2b1697700 Gitweb: http://git.kernel.org/tip/b74a0cf1b3db30173eefa00c411775d2b1697700 Author: Borislav Petkov <[email protected]> AuthorDate: Thu, 19 Nov 2015 12:25:25 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Tue, 24 Nov 2015 09:52:52 +0100
x86/fpu: Add an XSTATE_OP() macro
Add an XSTATE_OP() macro which contains the XSAVE* fault handling and replace all non-alternatives users of xstate_fault() with it.
This fixes also the buglet in copy_xregs_to_user() and copy_user_to_xregs() where the inline asm didn't have @xstate as memory reference and thus potentially causing unwanted reordering of accesses to the extended state.
--- arch/x86/include/asm/fpu/internal.h | 68 +++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 37 deletions(-)
arch/x86/include/asm/fpu/internal.h +31 -37
@@ -237,6 +237,20 @@ static inline void copy_fxregs_to_kernel(struct fpu *fpu) _ASM_EXTABLE(1b, 3b) \ : [_err] "=r" (__err) +#define XSTATE_OP (op, st, lmask, hmask, err) \ + asm volatile("1:" op "\n\t" \ + "xor %[err], %[err]\n" \ + "2:\n\t" \ + ".pushsection .fixup,\"ax\"\n\t" \ + "3: movl $-2,%[err]\n\t" \ + "jmp 2b\n\t" \ + ".popsection\n\t" \ + _ASM_EXTABLE(1b, 3b) \ + : [err] "=r" (err) \ + : "D" (st), "m" (*st), "a" (lmask), "d" (hmask) \ + : "memory") + +/* * This function is called only during boot time when x86 caps are not set * up and alternative can not be used yet.
@@ -246,22 +260,14 @@ static inline void copy_xregs_to_kernel_booting(struct xregs_state *xstate)u64 mask =-1 ;u32 lmask = mask;u32 hmask = mask >>32 ; -int err =0 ; +int err;WARN_ON (system_state !=SYSTEM_BOOTING ); -if (boot_cpu_has (X86_FEATURE_XSAVES )) - asm volatile("1:"XSAVES"\n\t" - "2:\n\t" - xstate_fault(err) - : "D" (xstate), "m" (*xstate), "a" (lmask), "d" (hmask), "0" (err) - : "memory"); +if (static_cpu_has_safe (X86_FEATURE_XSAVES )) +XSTATE_OP (XSAVES , xstate, lmask, hmask, err);else - asm volatile("1:"XSAVE"\n\t" - "2:\n\t" - xstate_fault(err) - : "D" (xstate), "m" (*xstate), "a" (lmask), "d" (hmask), "0" (err) - : "memory"); +XSTATE_OP (XSAVE , xstate, lmask, hmask, err);/* We should never fault when copying to a kernel buffer: */ WARN_ON_FPU (err);
@@ -276,22 +282,14 @@ static inline void copy_kernel_to_xregs_booting(struct xregs_state *xstate)u64 mask =-1 ;u32 lmask = mask;u32 hmask = mask >>32 ; -int err =0 ; +int err;WARN_ON (system_state !=SYSTEM_BOOTING ); -if (boot_cpu_has (X86_FEATURE_XSAVES )) - asm volatile("1:"XRSTORS"\n\t" - "2:\n\t" - xstate_fault(err) - : "D" (xstate), "m" (*xstate), "a" (lmask), "d" (hmask), "0" (err) - : "memory"); +if (static_cpu_has_safe (X86_FEATURE_XSAVES )) +XSTATE_OP (XRSTORS , xstate, lmask, hmask, err);else - asm volatile("1:"XRSTOR"\n\t" - "2:\n\t" - xstate_fault(err) - : "D" (xstate), "m" (*xstate), "a" (lmask), "d" (hmask), "0" (err) - : "memory"); +XSTATE_OP (XRSTOR , xstate, lmask, hmask, err);/* We should never fault when copying from a kernel buffer: */ WARN_ON_FPU (err);
@@ -388,12 +386,10 @@ static inline int copy_xregs_to_user(struct xregs_state __user *buf)if (unlikely (err))return -EFAULT ; - __asm__ __volatile__(ASM_STAC "\n" -"1:" XSAVE "\n" -"2: " ASM_CLAC "\n" - xstate_fault(err) - : "D" (buf), "a" (-1), "d" (-1), "0" (err) - : "memory"); +stac (); +XSTATE_OP (XSAVE , buf,-1 ,-1 , err); +clac (); +return err; }
@@ -405,14 +401,12 @@ static inline int copy_user_to_xregs(struct xregs_state __user *buf, u64 mask)struct xregs_state *xstate = ((__force struct xregs_state *)buf);u32 lmask = mask;u32 hmask = mask >>32 ; -int err =0 ; +int err; + +stac (); +XSTATE_OP (XRSTOR , xstate, lmask, hmask, err); +clac (); - __asm__ __volatile__(ASM_STAC "\n" -"1:" XRSTOR "\n" -"2: " ASM_CLAC "\n" - xstate_fault(err) - : "D" (xstate), "a" (lmask), "d" (hmask), "0" (err) - : "memory"); /* memory required? */return err; }
Borislav Petkov
maintainerRFC PATCH 2/2 x86/fpu: Get rid of xstate_fault()
In reply to Borislav PetkovFrom: Borislav Petkov <[email protected]>
Add macros for the alternative XSAVE*/XRSTOR* operations which contain the fault handling and use them. Kill xstate_fault().
Also, copy_xregs_to_kernel() didn't have the extended state as memory reference in the asm.
--- arch/x86/include/asm/fpu/internal.h | 105 ++++++++++++++++++------------------ 1 file changed, 52 insertions(+), 53 deletions(-)
arch/x86/include/asm/fpu/internal.h +52 -53
@@ -224,19 +224,6 @@ static inline void copy_fxregs_to_kernel(struct fpu *fpu)#define XRSTOR ".byte " REX_PREFIX "0x0f,0xae,0x2f"#define XRSTORS ".byte " REX_PREFIX "0x0f,0xc7,0x1f" -/* xstate instruction fault handler: */ -#define xstate_fault (__err) \ - \ - ".section .fixup,\"ax\"\n" \ - \ - "3: movl $-2,%[_err]\n" \ - " jmp 2b\n" \ - \ - ".previous\n" \ - \ - _ASM_EXTABLE(1b, 3b) \ - : [_err] "=r" (__err) -#define XSTATE_OP (op, st, lmask, hmask, err) \ asm volatile("1:" op "\n\t" \ "xor %[err], %[err]\n" \
@@ -250,6 +237,54 @@ static inline void copy_fxregs_to_kernel(struct fpu *fpu) : "D" (st), "m" (*st), "a" (lmask), "d" (hmask) \ : "memory") +/* +* If XSAVES is enabled, it replaces XSAVEOPT because it supports a compact +* format and supervisor states in addition to modified optimization in +* XSAVEOPT. +* +* Otherwise, if XSAVEOPT is enabled, XSAVEOPT replaces XSAVE because XSAVEOPT +* supports modified optimization which is not supported by XSAVE. +* +* We use XSAVE as a fallback. +* +* The 661 label is defined in the ALTERNATIVE* macros as the address of the +* original instruction which gets replaced. We need to use it here as the +* address of the instruction where we might get an exception at. +*/ +#define XSTATE_XSAVE (st, lmask, hmask, err) \ + asm volatile(ALTERNATIVE_2(XSAVE, \ + XSAVEOPT, X86_FEATURE_XSAVEOPT, \ + XSAVES, X86_FEATURE_XSAVES) \ + "\n" \ + "xor %[err], %[err]\n" \ + "3:\n" \ + ".pushsection .fixup,\"ax\"\n" \ + "4: movl $-2, %[err]\n" \ + "jmp 3b\n" \ + ".popsection\n" \ + _ASM_EXTABLE(661b, 4b) \ + : [err] "=r" (err) \ + : "D" (st), "m" (*st), "a" (lmask), "d" (hmask) \ + : "memory") + +/* +* Use XRSTORS to restore context if it is enabled. XRSTORS supports compact +* XSAVE area format. +*/ +#define XSTATE_XRESTORE (st, lmask, hmask, err) \ + asm volatile(ALTERNATIVE(XRSTOR, \ + XRSTORS, X86_FEATURE_XSAVES) \ + "\n" \ + "xor %[err], %[err]\n" \ + "3:\n" \ + ".pushsection .fixup,\"ax\"\n" \ + "4: movl $-2, %[err]\n" \ + "jmp 3b\n" \ + ".popsection\n" \ + _ASM_EXTABLE(661b, 4b) \ + : [err] "=r" (err) \ + : "D" (st), "m" (*st), "a" (lmask), "d" (hmask) \ + : "memory")/* * This function is called only during boot time when x86 caps are not set
@@ -303,33 +338,11 @@ static inline void copy_xregs_to_kernel(struct xregs_state *xstate)u64 mask =-1 ;u32 lmask = mask;u32 hmask = mask >>32 ; -int err =0 ; +int err;WARN_ON (!alternatives_patched); -/* -* If xsaves is enabled, xsaves replaces xsaveopt because -* it supports compact format and supervisor states in addition to -* modified optimization in xsaveopt. -* -* Otherwise, if xsaveopt is enabled, xsaveopt replaces xsave -* because xsaveopt supports modified optimization which is not -* supported by xsave. -* -* If none of xsaves and xsaveopt is enabled, use xsave. -*/ -alternative_input_2 ( -"1:" XSAVE , -XSAVEOPT , -X86_FEATURE_XSAVEOPT , -XSAVES , - X86_FEATURE_XSAVES, - [xstate] "D" (xstate), "a" (lmask), "d" (hmask) : -"memory" ); - asm volatile("2:\n\t" - xstate_fault(err) - : "0" (err) - : "memory"); +XSTATE_XSAVE (xstate, lmask, hmask, err);/* We should never fault when copying to a kernel buffer: */ WARN_ON_FPU (err);
@@ -342,23 +355,9 @@ static inline void copy_kernel_to_xregs(struct xregs_state *xstate, u64 mask) {u32 lmask = mask;u32 hmask = mask >>32 ; -int err =0 ; +int err; -/* -* Use xrstors to restore context if it is enabled. xrstors supports -* compacted format of xsave area which is not supported by xrstor. -*/ -alternative_input ( -"1: " XRSTOR , -XRSTORS , -X86_FEATURE_XSAVES , - "D" (xstate), "m" (*xstate), "a" (lmask), "d" (hmask) - : "memory"); - - asm volatile("2:\n" - xstate_fault(err) - : "0" (err) - : "memory"); +XSTATE_XRESTORE (xstate, lmask, hmask, err);/* We should never fault when copying from a kernel buffer: */ WARN_ON_FPU (err);
Signature · 1 line
2.3.5
tip-bot for Borislav Petkov
merge bot[tip:x86/fpu] x86/fpu: Get rid of xstate_fault()
x86/fpu: Get rid of xstate_fault()
Commit-ID: b7106fa0f29f9fd83d2d1905ab690d334ef855c1 Gitweb: http://git.kernel.org/tip/b7106fa0f29f9fd83d2d1905ab690d334ef855c1 Author: Borislav Petkov <[email protected]> AuthorDate: Thu, 19 Nov 2015 12:25:26 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Tue, 24 Nov 2015 09:52:52 +0100
x86/fpu: Get rid of xstate_fault()
Add macros for the alternative XSAVE*/XRSTOR* operations which contain the fault handling and use them. Kill xstate_fault().
Also, copy_xregs_to_kernel() didn't have the extended state as memory reference in the asm.
--- arch/x86/include/asm/fpu/internal.h | 105 ++++++++++++++++++------------------ 1 file changed, 52 insertions(+), 53 deletions(-)
arch/x86/include/asm/fpu/internal.h +52 -53
@@ -224,19 +224,6 @@ static inline void copy_fxregs_to_kernel(struct fpu *fpu)#define XRSTOR ".byte " REX_PREFIX "0x0f,0xae,0x2f"#define XRSTORS ".byte " REX_PREFIX "0x0f,0xc7,0x1f" -/* xstate instruction fault handler: */ -#define xstate_fault (__err) \ - \ - ".section .fixup,\"ax\"\n" \ - \ - "3: movl $-2,%[_err]\n" \ - " jmp 2b\n" \ - \ - ".previous\n" \ - \ - _ASM_EXTABLE(1b, 3b) \ - : [_err] "=r" (__err) -#define XSTATE_OP (op, st, lmask, hmask, err) \ asm volatile("1:" op "\n\t" \ "xor %[err], %[err]\n" \
@@ -250,6 +237,54 @@ static inline void copy_fxregs_to_kernel(struct fpu *fpu) : "D" (st), "m" (*st), "a" (lmask), "d" (hmask) \ : "memory") +/* +* If XSAVES is enabled, it replaces XSAVEOPT because it supports a compact +* format and supervisor states in addition to modified optimization in +* XSAVEOPT. +* +* Otherwise, if XSAVEOPT is enabled, XSAVEOPT replaces XSAVE because XSAVEOPT +* supports modified optimization which is not supported by XSAVE. +* +* We use XSAVE as a fallback. +* +* The 661 label is defined in the ALTERNATIVE* macros as the address of the +* original instruction which gets replaced. We need to use it here as the +* address of the instruction where we might get an exception at. +*/ +#define XSTATE_XSAVE (st, lmask, hmask, err) \ + asm volatile(ALTERNATIVE_2(XSAVE, \ + XSAVEOPT, X86_FEATURE_XSAVEOPT, \ + XSAVES, X86_FEATURE_XSAVES) \ + "\n" \ + "xor %[err], %[err]\n" \ + "3:\n" \ + ".pushsection .fixup,\"ax\"\n" \ + "4: movl $-2, %[err]\n" \ + "jmp 3b\n" \ + ".popsection\n" \ + _ASM_EXTABLE(661b, 4b) \ + : [err] "=r" (err) \ + : "D" (st), "m" (*st), "a" (lmask), "d" (hmask) \ + : "memory") + +/* +* Use XRSTORS to restore context if it is enabled. XRSTORS supports compact +* XSAVE area format. +*/ +#define XSTATE_XRESTORE (st, lmask, hmask, err) \ + asm volatile(ALTERNATIVE(XRSTOR, \ + XRSTORS, X86_FEATURE_XSAVES) \ + "\n" \ + "xor %[err], %[err]\n" \ + "3:\n" \ + ".pushsection .fixup,\"ax\"\n" \ + "4: movl $-2, %[err]\n" \ + "jmp 3b\n" \ + ".popsection\n" \ + _ASM_EXTABLE(661b, 4b) \ + : [err] "=r" (err) \ + : "D" (st), "m" (*st), "a" (lmask), "d" (hmask) \ + : "memory")/* * This function is called only during boot time when x86 caps are not set
@@ -303,33 +338,11 @@ static inline void copy_xregs_to_kernel(struct xregs_state *xstate)u64 mask =-1 ;u32 lmask = mask;u32 hmask = mask >>32 ; -int err =0 ; +int err;WARN_ON (!alternatives_patched); -/* -* If xsaves is enabled, xsaves replaces xsaveopt because -* it supports compact format and supervisor states in addition to -* modified optimization in xsaveopt. -* -* Otherwise, if xsaveopt is enabled, xsaveopt replaces xsave -* because xsaveopt supports modified optimization which is not -* supported by xsave. -* -* If none of xsaves and xsaveopt is enabled, use xsave. -*/ -alternative_input_2 ( -"1:" XSAVE , -XSAVEOPT , -X86_FEATURE_XSAVEOPT , -XSAVES , - X86_FEATURE_XSAVES, - [xstate] "D" (xstate), "a" (lmask), "d" (hmask) : -"memory" ); - asm volatile("2:\n\t" - xstate_fault(err) - : "0" (err) - : "memory"); +XSTATE_XSAVE (xstate, lmask, hmask, err);/* We should never fault when copying to a kernel buffer: */ WARN_ON_FPU (err);
@@ -342,23 +355,9 @@ static inline void copy_kernel_to_xregs(struct xregs_state *xstate, u64 mask) {u32 lmask = mask;u32 hmask = mask >>32 ; -int err =0 ; +int err; -/* -* Use xrstors to restore context if it is enabled. xrstors supports -* compacted format of xsave area which is not supported by xrstor. -*/ -alternative_input ( -"1: " XRSTOR , -XRSTORS , -X86_FEATURE_XSAVES , - "D" (xstate), "m" (*xstate), "a" (lmask), "d" (hmask) - : "memory"); - - asm volatile("2:\n" - xstate_fault(err) - : "0" (err) - : "memory"); +XSTATE_XRESTORE (xstate, lmask, hmask, err);/* We should never fault when copying from a kernel buffer: */ WARN_ON_FPU (err);