x86/fpu: Preparatory cleanups for AMX support (part 1)
Subsystems (8)
v2 cover x86/fpu: Preparatory cleanups for AMX support (part 1)
This is the rework of the V1 part-1 series of the effort to support the new AMX feature:
https://lore.kernel.org/r/[email protected]
This part cleans up existing mess. Historical leftovers, shortcomings and especially the universal kitchen sink asm/fpu/internal.h which is included all over the place for the wrong reasons.
This series has a value independent of AMX, but allows to make the integration and conversion to the new world order of dynamically enabled feature bits simpler.
The serries is based on:
git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git x86/fpu
and also available from git:
git://git.kernel.org/pub/scm/linux/kernel/git/tglx/devel.git x86/fpu-1
The full series which has the full AMX support included can be found at:
git://git.kernel.org/pub/scm/linux/kernel/git/tglx/devel.git x86/fpu
Changes vs. V1:
- Fixed the issue in the !XSAVE code path of the kvm UABI copy function - Paolo
- Renamed the KVM functions - Boris, Paolo, Sean
- Fixed the comments and changelog issues - Boris
Thanks,
tglx
--- arch/x86/events/perf_event.h | 1 arch/x86/ia32/ia32_signal.c | 1 arch/x86/include/asm/fpu/api.h | 31 ++ arch/x86/include/asm/fpu/internal.h | 530 ------------------------------------ arch/x86/include/asm/fpu/signal.h | 13 arch/x86/include/asm/fpu/xcr.h | 11 arch/x86/include/asm/fpu/xstate.h | 6 arch/x86/include/asm/pkru.h | 2 arch/x86/kernel/cpu/bugs.c | 2 arch/x86/kernel/cpu/common.c | 2 arch/x86/kernel/fpu/bugs.c | 2 arch/x86/kernel/fpu/core.c | 161 ++++++++-- arch/x86/kernel/fpu/init.c | 29 - arch/x86/kernel/fpu/regset.c | 6 arch/x86/kernel/fpu/signal.c | 21 - arch/x86/kernel/fpu/xstate.c | 172 ++++++----- arch/x86/kernel/process.c | 6 arch/x86/kernel/process_32.c | 5 arch/x86/kernel/process_64.c | 5 arch/x86/kernel/ptrace.c | 1 arch/x86/kernel/sev.c | 2 arch/x86/kernel/signal.c | 1 arch/x86/kernel/smpboot.c | 2 arch/x86/kernel/traps.c | 2 arch/x86/kvm/svm/sev.c | 2 arch/x86/kvm/vmx/vmx.c | 2 arch/x86/kvm/x86.c | 192 +------------ arch/x86/math-emu/fpu_entry.c | 2 arch/x86/mm/extable.c | 4 arch/x86/power/cpu.c | 2 b/arch/x86/include/asm/fpu/sched.h | 68 ++++ b/arch/x86/kernel/fpu/context.h | 85 +++++ b/arch/x86/kernel/fpu/internal.h | 30 ++ b/arch/x86/kernel/fpu/legacy.h | 115 +++++++ b/arch/x86/kernel/fpu/xstate.h | 198 +++++++++++++ 35 files changed, 824 insertions(+), 890 deletions(-)
PATCH v2 1/30 x86/fpu: Remove pointless argument from switch_fpu_finish()
Unused since the FPU switching rework.
--- arch/x86/include/asm/fpu/internal.h | 2 +- arch/x86/kernel/process_32.c | 3 +-- arch/x86/kernel/process_64.c | 3 +-- 3 files changed, 3 insertions(+), 5 deletions(-)
---
arch/x86/include/asm/fpu/internal.h +1 -1
@@ -521,7 +521,7 @@ static inline void switch_fpu_prepare(struct fpu *old_fpu, int cpu)* Delay loading of the complete FPU state until the return to userland. * PKRU is handled separately. */ -static inline void switch_fpu_finish (struct fpu *new_fpu) +static inline void switch_fpu_finish (void ) {if (cpu_feature_enabled (X86_FEATURE_FPU ))set_thread_flag (TIF_NEED_FPU_LOAD );
arch/x86/kernel/process_32.c +1 -2
@@ -160,7 +160,6 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)struct thread_struct *prev = &prev_p->thread, *next = &next_p->thread;struct fpu *prev_fpu = &prev->fpu; -struct fpu *next_fpu = &next->fpu;int cpu =smp_processor_id ();/* never put a printk in __switch_to... printk() calls wake_up*() indirectly */
@@ -213,7 +212,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)this_cpu_write (current_task, next_p); -switch_fpu_finish (next_fpu); +switch_fpu_finish ();/* Load the Intel cache allocation PQR MSR. */ resctrl_sched_in ();
arch/x86/kernel/process_64.c +1 -2
@@ -559,7 +559,6 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)struct thread_struct *prev = &prev_p->thread;struct thread_struct *next = &next_p->thread;struct fpu *prev_fpu = &prev->fpu; -struct fpu *next_fpu = &next->fpu;int cpu =smp_processor_id (); WARN_ON_ONCE(IS_ENABLED(CONFIG_DEBUG_ENTRY) &&
@@ -620,7 +619,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)this_cpu_write (current_task, next_p);this_cpu_write (cpu_current_top_of_stack,task_top_of_stack (next_p)); -switch_fpu_finish (next_fpu); +switch_fpu_finish ();/* Reload sp0. */ update_task_stack (next_p);
[tip: x86/fpu] x86/fpu: Remove pointless argument from switch_fpu_finish()
The following commit has been merged into the x86/fpu branch of tip:
The following commit has been merged into the x86/fpu branch of tip:
Commit-ID: 9568bfb4f04bd9a280c592879ccd7a26a77c1390 Gitweb: https://git.kernel.org/tip/9568bfb4f04bd9a280c592879ccd7a26a77c1390 Author: Thomas Gleixner <[email protected]> AuthorDate: Fri, 15 Oct 2021 03:15:54 +02:00 Committer: Borislav Petkov <[email protected]> CommitterDate: Wed, 20 Oct 2021 15:27:25 +02:00
x86/fpu: Remove pointless argument from switch_fpu_finish()
Unused since the FPU switching rework.
--- arch/x86/include/asm/fpu/internal.h | 2 +- arch/x86/kernel/process_32.c | 3 +-- arch/x86/kernel/process_64.c | 3 +-- 3 files changed, 3 insertions(+), 5 deletions(-)
arch/x86/include/asm/fpu/internal.h +1 -1
@@ -521,7 +521,7 @@ static inline void switch_fpu_prepare(struct fpu *old_fpu, int cpu)* Delay loading of the complete FPU state until the return to userland. * PKRU is handled separately. */ -static inline void switch_fpu_finish (struct fpu *new_fpu) +static inline void switch_fpu_finish (void ) {if (cpu_feature_enabled (X86_FEATURE_FPU ))set_thread_flag (TIF_NEED_FPU_LOAD );
arch/x86/kernel/process_32.c +1 -2
@@ -160,7 +160,6 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)struct thread_struct *prev = &prev_p->thread, *next = &next_p->thread;struct fpu *prev_fpu = &prev->fpu; -struct fpu *next_fpu = &next->fpu;int cpu =smp_processor_id ();/* never put a printk in __switch_to... printk() calls wake_up*() indirectly */
@@ -213,7 +212,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)this_cpu_write (current_task, next_p); -switch_fpu_finish (next_fpu); +switch_fpu_finish ();/* Load the Intel cache allocation PQR MSR. */ resctrl_sched_in ();
arch/x86/kernel/process_64.c +1 -2
@@ -559,7 +559,6 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)struct thread_struct *prev = &prev_p->thread;struct thread_struct *next = &next_p->thread;struct fpu *prev_fpu = &prev->fpu; -struct fpu *next_fpu = &next->fpu;int cpu =smp_processor_id (); WARN_ON_ONCE(IS_ENABLED(CONFIG_DEBUG_ENTRY) &&
@@ -620,7 +619,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)this_cpu_write (current_task, next_p);this_cpu_write (cpu_current_top_of_stack,task_top_of_stack (next_p)); -switch_fpu_finish (next_fpu); +switch_fpu_finish ();/* Reload sp0. */ update_task_stack (next_p);
PATCH v2 2/30 x86/fpu: Update stale comments
In reply to Thomas Gleixnercopy_fpstate_to_sigframe() does not have a slow path anymore. Neither does the !ia32 restore in __fpu_restore_sig().
Update the comments accordingly.
--- arch/x86/kernel/fpu/signal.c | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-)
---
arch/x86/kernel/fpu/signal.c +3 -10
@@ -155,10 +155,8 @@ static inline int copy_fpregs_to_sigframe(struct xregs_state __user *buf)* buf == buf_fx for 64-bit frames and 32-bit fsave frame. * buf != buf_fx for 32-bit frames with fxstate. * -* Try to save it directly to the user frame with disabled page fault handler. -* If this fails then do the slow path where the FPU state is first saved to -* task's fpu->state and then copy it to the user frame pointed to by the -* aligned pointer 'buf_fx'. +* Save it directly to the user frame with disabled page fault handler. If +* that faults, try to clear the frame which handles the page fault. * * If this is a 32-bit frame with fxstate, put a fsave header before * the aligned state at 'buf_fx'.
@@ -334,12 +332,7 @@ static bool __fpu_restore_sig(void __user *buf, void __user *buf_fx, }if (likely (!ia32_fxstate)) { -/* -* Attempt to restore the FPU registers directly from user -* memory. For that to succeed, the user access cannot cause page -* faults. If it does, fall back to the slow path below, going -* through the kernel buffer with the enabled pagefault handler. -*/ +/* Restore the FPU registers directly from user memory. */ return restore_fpregs_from_user (buf_fx, user_xfeatures, fx_only, state_size); }
[tip: x86/fpu] x86/fpu: Update stale comments
The following commit has been merged into the x86/fpu branch of tip:
The following commit has been merged into the x86/fpu branch of tip:
Commit-ID: d2d926482cdfbd5517826eca4e39dcd8757f04d3 Gitweb: https://git.kernel.org/tip/d2d926482cdfbd5517826eca4e39dcd8757f04d3 Author: Thomas Gleixner <[email protected]> AuthorDate: Fri, 15 Oct 2021 03:15:56 +02:00 Committer: Borislav Petkov <[email protected]> CommitterDate: Wed, 20 Oct 2021 15:27:25 +02:00
x86/fpu: Update stale comments
copy_fpstate_to_sigframe() does not have a slow path anymore. Neither does the !ia32 restore in __fpu_restore_sig().
Update the comments accordingly.
--- arch/x86/kernel/fpu/signal.c | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-)
arch/x86/kernel/fpu/signal.c +3 -10
@@ -155,10 +155,8 @@ static inline int copy_fpregs_to_sigframe(struct xregs_state __user *buf)* buf == buf_fx for 64-bit frames and 32-bit fsave frame. * buf != buf_fx for 32-bit frames with fxstate. * -* Try to save it directly to the user frame with disabled page fault handler. -* If this fails then do the slow path where the FPU state is first saved to -* task's fpu->state and then copy it to the user frame pointed to by the -* aligned pointer 'buf_fx'. +* Save it directly to the user frame with disabled page fault handler. If +* that faults, try to clear the frame which handles the page fault. * * If this is a 32-bit frame with fxstate, put a fsave header before * the aligned state at 'buf_fx'.
@@ -334,12 +332,7 @@ static bool __fpu_restore_sig(void __user *buf, void __user *buf_fx, }if (likely (!ia32_fxstate)) { -/* -* Attempt to restore the FPU registers directly from user -* memory. For that to succeed, the user access cannot cause page -* faults. If it does, fall back to the slow path below, going -* through the kernel buffer with the enabled pagefault handler. -*/ +/* Restore the FPU registers directly from user memory. */ return restore_fpregs_from_user (buf_fx, user_xfeatures, fx_only, state_size); }
PATCH v2 3/30 x86/pkru: Remove useless include
In reply to Thomas GleixnerPKRU code does not need anything from fpu headers. Include cpufeature.h instead and fixup the resulting fallout in perf.
This is a preparation for FPU changes in order to prevent recursive include hell.
--- arch/x86/events/perf_event.h | 1 + arch/x86/include/asm/pkru.h | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-)
---
arch/x86/events/perf_event.h +1 -0
@@ -14,6 +14,7 @@#include <linux/perf_event.h> +#include <asm/fpu/xstate.h> #include <asm/intel_ds.h> #include <asm/cpu.h>
arch/x86/include/asm/pkru.h +1 -1
@@ -2,7 +2,7 @@#ifndef _ASM_X86_PKRU_H#define _ASM_X86_PKRU_H -#include <asm/fpu/xstate.h> +#include <asm/cpufeature.h> #define PKRU_AD_BIT 0x1#define PKRU_WD_BIT 0x2
[tip: x86/fpu] x86/pkru: Remove useless include
The following commit has been merged into the x86/fpu branch of tip:
The following commit has been merged into the x86/fpu branch of tip:
Commit-ID: b50854eca0e014c2d3738073b387ab8ec85118ab Gitweb: https://git.kernel.org/tip/b50854eca0e014c2d3738073b387ab8ec85118ab Author: Thomas Gleixner <[email protected]> AuthorDate: Fri, 15 Oct 2021 03:15:57 +02:00 Committer: Borislav Petkov <[email protected]> CommitterDate: Wed, 20 Oct 2021 15:27:25 +02:00
x86/pkru: Remove useless include
PKRU code does not need anything from FPU headers. Include cpufeature.h instead and fixup the resulting fallout in perf.
This is a preparation for FPU changes in order to prevent recursive include hell.
--- arch/x86/events/perf_event.h | 1 + arch/x86/include/asm/pkru.h | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-)
arch/x86/events/perf_event.h +1 -0
@@ -14,6 +14,7 @@#include <linux/perf_event.h> +#include <asm/fpu/xstate.h> #include <asm/intel_ds.h> #include <asm/cpu.h>
arch/x86/include/asm/pkru.h +1 -1
@@ -2,7 +2,7 @@#ifndef _ASM_X86_PKRU_H#define _ASM_X86_PKRU_H -#include <asm/fpu/xstate.h> +#include <asm/cpufeature.h> #define PKRU_AD_BIT 0x1#define PKRU_WD_BIT 0x2
PATCH v2 4/30 x86/fpu: Restrict xsaves()/xrstors() to independent states
In reply to Thomas GleixnerThese interfaces are really only valid for features which are independently managed and not part of the task context state for various reasons.
Tighten the checks and adjust the misleading comments.
---
V2: Rename functions and address the comment related review - Boris
--- arch/x86/kernel/fpu/xstate.c | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-)
---
arch/x86/kernel/fpu/xstate.c +7 -15
@@ -1175,20 +1175,14 @@ int copy_sigframe_from_user_to_xstate(struct xregs_state *xsave,return copy_uabi_to_xstate (xsave,NULL , ubuf); } -static bool validate_xsaves_xrstors (u64 mask) +static bool validate_independent_components (u64 mask) {u64 xchk;if (WARN_ON_FPU (!cpu_feature_enabled (X86_FEATURE_XSAVES )))return false; -/* -* Validate that this is either a task->fpstate related component -* subset or an independent one. -*/ -if (mask &xfeatures_mask_independent ()) - xchk = ~xfeatures_mask_independent (); -else - xchk = ~xfeatures_mask_all; + + xchk = ~xfeatures_mask_independent ();if (WARN_ON_ONCE (!mask || mask & xchk))return false;
@@ -1206,14 +1200,13 @@ static bool validate_xsaves_xrstors(u64 mask)* buffer should be zeroed otherwise a consecutive XRSTORS from that buffer * can #GP. * -* The feature mask must either be a subset of the independent features or -* a subset of the task->fpstate related features. +* The feature mask must be a subset of the independent features. */ void xsaves (struct xregs_state *xstate,u64 mask) {int err; -if (!validate_xsaves_xrstors (mask)) +if (!validate_independent_components (mask))return ;XSTATE_OP (XSAVES , xstate, (u32 )mask, (u32)(mask >>32 ), err);
@@ -1231,14 +1224,13 @@ void xsaves(struct xregs_state *xstate, u64 mask)* Proper usage is to restore the state which was saved with * xsaves() into @xstate. * -* The feature mask must either be a subset of the independent features or -* a subset of the task->fpstate related features. +* The feature mask must be a subset of the independent features. */ void xrstors (struct xregs_state *xstate,u64 mask) {int err; -if (!validate_xsaves_xrstors (mask)) +if (!validate_independent_components (mask))return ;XSTATE_OP (XRSTORS , xstate, (u32 )mask, (u32)(mask >>32 ), err);
[tip: x86/fpu] x86/fpu: Restrict xsaves()/xrstors() to independent states
The following commit has been merged into the x86/fpu branch of tip:
The following commit has been merged into the x86/fpu branch of tip:
Commit-ID: f5daf836f292f795f9cf8f36e036bf47adcbc3a3 Gitweb: https://git.kernel.org/tip/f5daf836f292f795f9cf8f36e036bf47adcbc3a3 Author: Thomas Gleixner <[email protected]> AuthorDate: Fri, 15 Oct 2021 03:15:59 +02:00 Committer: Borislav Petkov <[email protected]> CommitterDate: Wed, 20 Oct 2021 15:27:26 +02:00
x86/fpu: Restrict xsaves()/xrstors() to independent states
These interfaces are really only valid for features which are independently managed and not part of the task context state for various reasons.
Tighten the checks and adjust the misleading comments.
--- arch/x86/kernel/fpu/xstate.c | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-)
arch/x86/kernel/fpu/xstate.c +7 -15
@@ -1175,20 +1175,14 @@ int copy_sigframe_from_user_to_xstate(struct xregs_state *xsave,return copy_uabi_to_xstate (xsave,NULL , ubuf); } -static bool validate_xsaves_xrstors (u64 mask) +static bool validate_independent_components (u64 mask) {u64 xchk;if (WARN_ON_FPU (!cpu_feature_enabled (X86_FEATURE_XSAVES )))return false; -/* -* Validate that this is either a task->fpstate related component -* subset or an independent one. -*/ -if (mask &xfeatures_mask_independent ()) - xchk = ~xfeatures_mask_independent (); -else - xchk = ~xfeatures_mask_all; + + xchk = ~xfeatures_mask_independent ();if (WARN_ON_ONCE (!mask || mask & xchk))return false;
@@ -1206,14 +1200,13 @@ static bool validate_xsaves_xrstors(u64 mask)* buffer should be zeroed otherwise a consecutive XRSTORS from that buffer * can #GP. * -* The feature mask must either be a subset of the independent features or -* a subset of the task->fpstate related features. +* The feature mask must be a subset of the independent features. */ void xsaves (struct xregs_state *xstate,u64 mask) {int err; -if (!validate_xsaves_xrstors (mask)) +if (!validate_independent_components (mask))return ;XSTATE_OP (XSAVES , xstate, (u32 )mask, (u32)(mask >>32 ), err);
@@ -1231,14 +1224,13 @@ void xsaves(struct xregs_state *xstate, u64 mask)* Proper usage is to restore the state which was saved with * xsaves() into @xstate. * -* The feature mask must either be a subset of the independent features or -* a subset of the task->fpstate related features. +* The feature mask must be a subset of the independent features. */ void xrstors (struct xregs_state *xstate,u64 mask) {int err; -if (!validate_xsaves_xrstors (mask)) +if (!validate_independent_components (mask))return ;XSTATE_OP (XRSTORS , xstate, (u32 )mask, (u32)(mask >>32 ), err);
PATCH v2 5/30 x86/fpu: Cleanup the on_boot_cpu clutter
In reply to Thomas GleixnerDefensive programming is useful, but this on_boot_cpu debug is really silly.
--- arch/x86/kernel/fpu/init.c | 16 ---------------- arch/x86/kernel/fpu/xstate.c | 9 --------- 2 files changed, 25 deletions(-)
---
arch/x86/kernel/fpu/init.c +0 -16
@@ -192,11 +192,6 @@ static void __init fpu__init_task_struct_size(void)*/ static void __init fpu__init_system_xstate_size_legacy (void ) { -static int on_boot_cpu__initdata =1 ; - -WARN_ON_FPU (!on_boot_cpu); - on_boot_cpu =0 ; -/* * Note that xstate sizes might be overwritten later during * fpu__init_system_xstate().
@@ -216,15 +211,6 @@ static void __init fpu__init_system_xstate_size_legacy(void) fpu_user_xstate_size = fpu_kernel_xstate_size; } -/* Legacy code to initialize eager fpu mode. */ -static void __init fpu__init_system_ctx_switch (void ) -{ -static bool on_boot_cpu__initdata =1 ; - -WARN_ON_FPU (!on_boot_cpu); - on_boot_cpu =0 ; -} -/* * Called on the boot CPU once per system bootup, to set up the initial * FPU state that is later cloned into all processes:
@@ -243,6 +229,4 @@ void __init fpu__init_system(struct cpuinfo_x86 *c)fpu__init_system_xstate_size_legacy ();fpu__init_system_xstate ();fpu__init_task_struct_size (); - -fpu__init_system_ctx_switch (); }
arch/x86/kernel/fpu/xstate.c +0 -9
@@ -379,15 +379,10 @@ static void __init print_xstate_offset_size(void)*/ static void __init setup_init_fpu_buf (void ) { -static int on_boot_cpu__initdata =1 ; -BUILD_BUG_ON ((XFEATURE_MASK_USER_SUPPORTED |XFEATURE_MASK_SUPERVISOR_SUPPORTED ) !=XFEATURES_INIT_FPSTATE_HANDLED ); -WARN_ON_FPU (!on_boot_cpu); - on_boot_cpu =0 ; -if (!boot_cpu_has (X86_FEATURE_XSAVE ))return ;
@@ -721,14 +716,10 @@ static void fpu__init_disable_system_xstate(void)void __init fpu__init_system_xstate (void ) {unsigned int eax, ebx, ecx, edx; -static int on_boot_cpu__initdata =1 ;u64 xfeatures;int err;int i; -WARN_ON_FPU (!on_boot_cpu); - on_boot_cpu =0 ; -if (!boot_cpu_has (X86_FEATURE_FPU )) {pr_info ("x86/fpu: No FPU detected\n" );return ;
[tip: x86/fpu] x86/fpu: Cleanup the on_boot_cpu clutter
The following commit has been merged into the x86/fpu branch of tip:
The following commit has been merged into the x86/fpu branch of tip:
Commit-ID: dc2f39fd1bf23eee644d409b84e8e435606997bf Gitweb: https://git.kernel.org/tip/dc2f39fd1bf23eee644d409b84e8e435606997bf Author: Thomas Gleixner <[email protected]> AuthorDate: Fri, 15 Oct 2021 03:16:01 +02:00 Committer: Borislav Petkov <[email protected]> CommitterDate: Wed, 20 Oct 2021 15:27:26 +02:00
x86/fpu: Cleanup the on_boot_cpu clutter
Defensive programming is useful, but this on_boot_cpu debug is really silly.
--- arch/x86/kernel/fpu/init.c | 16 ---------------- arch/x86/kernel/fpu/xstate.c | 9 --------- 2 files changed, 25 deletions(-)
arch/x86/kernel/fpu/init.c +0 -16
@@ -192,11 +192,6 @@ static void __init fpu__init_task_struct_size(void)*/ static void __init fpu__init_system_xstate_size_legacy (void ) { -static int on_boot_cpu__initdata =1 ; - -WARN_ON_FPU (!on_boot_cpu); - on_boot_cpu =0 ; -/* * Note that xstate sizes might be overwritten later during * fpu__init_system_xstate().
@@ -216,15 +211,6 @@ static void __init fpu__init_system_xstate_size_legacy(void) fpu_user_xstate_size = fpu_kernel_xstate_size; } -/* Legacy code to initialize eager fpu mode. */ -static void __init fpu__init_system_ctx_switch (void ) -{ -static bool on_boot_cpu__initdata =1 ; - -WARN_ON_FPU (!on_boot_cpu); - on_boot_cpu =0 ; -} -/* * Called on the boot CPU once per system bootup, to set up the initial * FPU state that is later cloned into all processes:
@@ -243,6 +229,4 @@ void __init fpu__init_system(struct cpuinfo_x86 *c)fpu__init_system_xstate_size_legacy ();fpu__init_system_xstate ();fpu__init_task_struct_size (); - -fpu__init_system_ctx_switch (); }
arch/x86/kernel/fpu/xstate.c +0 -9
@@ -379,15 +379,10 @@ static void __init print_xstate_offset_size(void)*/ static void __init setup_init_fpu_buf (void ) { -static int on_boot_cpu__initdata =1 ; -BUILD_BUG_ON ((XFEATURE_MASK_USER_SUPPORTED |XFEATURE_MASK_SUPERVISOR_SUPPORTED ) !=XFEATURES_INIT_FPSTATE_HANDLED ); -WARN_ON_FPU (!on_boot_cpu); - on_boot_cpu =0 ; -if (!boot_cpu_has (X86_FEATURE_XSAVE ))return ;
@@ -721,14 +716,10 @@ static void fpu__init_disable_system_xstate(void)void __init fpu__init_system_xstate (void ) {unsigned int eax, ebx, ecx, edx; -static int on_boot_cpu__initdata =1 ;u64 xfeatures;int err;int i; -WARN_ON_FPU (!on_boot_cpu); - on_boot_cpu =0 ; -if (!boot_cpu_has (X86_FEATURE_FPU )) {pr_info ("x86/fpu: No FPU detected\n" );return ;
PATCH v2 6/30 x86/fpu: Remove pointless memset in fpu_clone()
In reply to Thomas GleixnerZeroing the forked task's FPU register buffer to avoid leaking init optimized stale data into the clone is a pointless exercise for the case where the current task has TIF_NEED_FPU_LOAD set. In that case the FPU register state is copied from current's FPU register buffer which can contain stale init optimized data as well.
The alledged information leak is non-existant because this stale init optimized data is nowhere used and cannot leak anywhere.
--- arch/x86/kernel/fpu/core.c | 6 ------ 1 file changed, 6 deletions(-)
---
arch/x86/kernel/fpu/core.c +0 -6
@@ -260,12 +260,6 @@ int fpu_clone(struct task_struct *dst)return 0 ;/* -* Don't let 'init optimized' areas of the XSAVE area -* leak into the child task: -*/ -memset (&dst_fpu->state.xsave,0 , fpu_kernel_xstate_size); - -/* * If the FPU registers are not owned by current just memcpy() the * state. Otherwise save the FPU registers directly into the * child's FPU context, without any memory-to-memory copying.
[tip: x86/fpu] x86/fpu: Remove pointless memset in fpu_clone()
The following commit has been merged into the x86/fpu branch of tip:
The following commit has been merged into the x86/fpu branch of tip:
Commit-ID: 01f9f62d3ae75077a54a11d2777082f1e58e2d9f Gitweb: https://git.kernel.org/tip/01f9f62d3ae75077a54a11d2777082f1e58e2d9f Author: Thomas Gleixner <[email protected]> AuthorDate: Fri, 15 Oct 2021 03:16:02 +02:00 Committer: Borislav Petkov <[email protected]> CommitterDate: Wed, 20 Oct 2021 15:27:26 +02:00
x86/fpu: Remove pointless memset in fpu_clone()
Zeroing the forked task's FPU registers buffer to avoid leaking init optimized stale data into the clone is a pointless exercise for the case where the current task has TIF_NEED_FPU_LOAD set. In that case, the FPU registers state is copied from current's FPU register buffer which can contain stale init optimized data as well.
The alledged information leak is non-existant because this stale init optimized data is used nowhere and cannot leak anywhere.
--- arch/x86/kernel/fpu/core.c | 6 ------ 1 file changed, 6 deletions(-)
arch/x86/kernel/fpu/core.c +0 -6
@@ -260,12 +260,6 @@ int fpu_clone(struct task_struct *dst)return 0 ;/* -* Don't let 'init optimized' areas of the XSAVE area -* leak into the child task: -*/ -memset (&dst_fpu->state.xsave,0 , fpu_kernel_xstate_size); - -/* * If the FPU registers are not owned by current just memcpy() the * state. Otherwise save the FPU registers directly into the * child's FPU context, without any memory-to-memory copying.
PATCH v2 7/30 x86/process: Clone FPU in copy_thread()
In reply to Thomas GleixnerThere is no reason to clone FPU in arch_dup_task_struct(). Quite the contrary it prevents optimizations. Move it to copy_thread()
--- arch/x86/kernel/process.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
---
arch/x86/kernel/process.c +3 -1
@@ -87,7 +87,7 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)#ifdef CONFIG_VM86 dst->thread.vm86 =NULL ;#endif -return fpu_clone (dst); +return 0 ; }/*
@@ -154,6 +154,8 @@ int copy_thread(unsigned long clone_flags, unsigned long sp, unsigned long arg, frame->flags =X86_EFLAGS_FIXED ;#endif +fpu_clone (p); +/* Kernel thread ? */ if (unlikely (p->flags &PF_KTHREAD )) { p->thread.pkru =pkru_get_init_value ();
[tip: x86/fpu] x86/process: Clone FPU in copy_thread()
The following commit has been merged into the x86/fpu branch of tip:
The following commit has been merged into the x86/fpu branch of tip:
Commit-ID: 2d16a1876f20218f8970ea4b7f679cead1cdb510 Gitweb: https://git.kernel.org/tip/2d16a1876f20218f8970ea4b7f679cead1cdb510 Author: Thomas Gleixner <[email protected]> AuthorDate: Fri, 15 Oct 2021 03:16:04 +02:00 Committer: Borislav Petkov <[email protected]> CommitterDate: Wed, 20 Oct 2021 15:27:26 +02:00
x86/process: Clone FPU in copy_thread()
There is no reason to clone FPU in arch_dup_task_struct(). Quite the contrary - it prevents optimizations. Move it to copy_thread().
--- arch/x86/kernel/process.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
arch/x86/kernel/process.c +3 -1
@@ -87,7 +87,7 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)#ifdef CONFIG_VM86 dst->thread.vm86 =NULL ;#endif -return fpu_clone (dst); +return 0 ; }/*
@@ -154,6 +154,8 @@ int copy_thread(unsigned long clone_flags, unsigned long sp, unsigned long arg, frame->flags =X86_EFLAGS_FIXED ;#endif +fpu_clone (p); +/* Kernel thread ? */ if (unlikely (p->flags &PF_KTHREAD )) { p->thread.pkru =pkru_get_init_value ();
PATCH v2 8/30 x86/fpu: Do not inherit FPU context for kernel and IO worker threads
In reply to Thomas GleixnerThere is no reason why kernel and IO worker threads need a full clone of the parent's FPU state. Both are kernel threads which are not supposed to use FPU. So copying a large state or doing XSAVE() is pointless. Just clean out the minimaly required state for those tasks.
--- arch/x86/kernel/fpu/core.c | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-)
---
arch/x86/kernel/fpu/core.c +26 -11
@@ -212,6 +212,15 @@ static inline void fpstate_init_xstate(struct xregs_state *xsave) xsave->header.xcomp_bv =XCOMP_BV_COMPACTED_FORMAT | xfeatures_mask_all; } +static inline unsigned int init_fpstate_copy_size (void ) +{ +if (!use_xsave ()) +return fpu_kernel_xstate_size; + +/* XSAVE(S) just needs the legacy and the xstate header part */ +return sizeof (init_fpstate.xsave); +} +static inline void fpstate_init_fxstate (struct fxregs_state *fx) { fx->cwd =0x37f ;
@@ -260,6 +269,23 @@ int fpu_clone(struct task_struct *dst)return 0 ;/* +* Enforce reload for user space tasks and prevent kernel threads +* from trying to save the FPU registers on context switch. +*/ +set_tsk_thread_flag (dst,TIF_NEED_FPU_LOAD ); + +/* +* No FPU state inheritance for kernel threads and IO +* worker threads. +*/ +if (dst->flags & (PF_KTHREAD |PF_IO_WORKER )) { +/* Clear out the minimal state */ +memcpy (&dst_fpu->state, &init_fpstate, +init_fpstate_copy_size ()); +return 0 ; + } + +/* * If the FPU registers are not owned by current just memcpy() the * state. Otherwise save the FPU registers directly into the * child's FPU context, without any memory-to-memory copying.
@@ -272,8 +298,6 @@ int fpu_clone(struct task_struct *dst)save_fpregs_to_fpstate (dst_fpu);fpregs_unlock (); -set_tsk_thread_flag (dst,TIF_NEED_FPU_LOAD ); -trace_x86_fpu_copy_src (src_fpu);trace_x86_fpu_copy_dst (dst_fpu);
@@ -322,15 +346,6 @@ static inline void restore_fpregs_from_init_fpstate(u64 features_mask)pkru_write_default (); } -static inline unsigned int init_fpstate_copy_size (void ) -{ -if (!use_xsave ()) -return fpu_kernel_xstate_size; - -/* XSAVE(S) just needs the legacy and the xstate header part */ -return sizeof (init_fpstate.xsave); -} -/* * Reset current->fpu memory state to the init values. */
[tip: x86/fpu] x86/fpu: Do not inherit FPU context for kernel and IO worker threads
The following commit has been merged into the x86/fpu branch of tip:
The following commit has been merged into the x86/fpu branch of tip:
Commit-ID: 509e7a30cd0a9f38abac4114832d9f69ff0d73b4 Gitweb: https://git.kernel.org/tip/509e7a30cd0a9f38abac4114832d9f69ff0d73b4 Author: Thomas Gleixner <[email protected]> AuthorDate: Fri, 15 Oct 2021 03:16:06 +02:00 Committer: Borislav Petkov <[email protected]> CommitterDate: Wed, 20 Oct 2021 15:27:26 +02:00
x86/fpu: Do not inherit FPU context for kernel and IO worker threads
There is no reason why kernel and IO worker threads need a full clone of the parent's FPU state. Both are kernel threads which are not supposed to use FPU. So copying a large state or doing XSAVE() is pointless. Just clean out the minimally required state for those tasks.
--- arch/x86/kernel/fpu/core.c | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-)
arch/x86/kernel/fpu/core.c +26 -11
@@ -212,6 +212,15 @@ static inline void fpstate_init_xstate(struct xregs_state *xsave) xsave->header.xcomp_bv =XCOMP_BV_COMPACTED_FORMAT | xfeatures_mask_all; } +static inline unsigned int init_fpstate_copy_size (void ) +{ +if (!use_xsave ()) +return fpu_kernel_xstate_size; + +/* XSAVE(S) just needs the legacy and the xstate header part */ +return sizeof (init_fpstate.xsave); +} +static inline void fpstate_init_fxstate (struct fxregs_state *fx) { fx->cwd =0x37f ;
@@ -260,6 +269,23 @@ int fpu_clone(struct task_struct *dst)return 0 ;/* +* Enforce reload for user space tasks and prevent kernel threads +* from trying to save the FPU registers on context switch. +*/ +set_tsk_thread_flag (dst,TIF_NEED_FPU_LOAD ); + +/* +* No FPU state inheritance for kernel threads and IO +* worker threads. +*/ +if (dst->flags & (PF_KTHREAD |PF_IO_WORKER )) { +/* Clear out the minimal state */ +memcpy (&dst_fpu->state, &init_fpstate, +init_fpstate_copy_size ()); +return 0 ; + } + +/* * If the FPU registers are not owned by current just memcpy() the * state. Otherwise save the FPU registers directly into the * child's FPU context, without any memory-to-memory copying.
@@ -272,8 +298,6 @@ int fpu_clone(struct task_struct *dst)save_fpregs_to_fpstate (dst_fpu);fpregs_unlock (); -set_tsk_thread_flag (dst,TIF_NEED_FPU_LOAD ); -trace_x86_fpu_copy_src (src_fpu);trace_x86_fpu_copy_dst (dst_fpu);
@@ -322,15 +346,6 @@ static inline void restore_fpregs_from_init_fpstate(u64 features_mask)pkru_write_default (); } -static inline unsigned int init_fpstate_copy_size (void ) -{ -if (!use_xsave ()) -return fpu_kernel_xstate_size; - -/* XSAVE(S) just needs the legacy and the xstate header part */ -return sizeof (init_fpstate.xsave); -} -/* * Reset current->fpu memory state to the init values. */
PATCH v2 9/30 x86/fpu: Cleanup xstate xcomp_bv initialization
In reply to Thomas GleixnerNo point in having this duplicated all over the place with needlessly different defines.
Provide a proper initialization function which initializes user buffers properly and make KVM use it.
--- arch/x86/include/asm/fpu/internal.h | 4 +++- arch/x86/kernel/fpu/core.c | 35 +++++++++++++++++++---------------- arch/x86/kernel/fpu/init.c | 6 +++--- arch/x86/kernel/fpu/xstate.c | 8 +++----- arch/x86/kernel/fpu/xstate.h | 18 ++++++++++++++++++ arch/x86/kvm/x86.c | 11 +++-------- 6 files changed, 49 insertions(+), 33 deletions(-) create mode 100644 arch/x86/kernel/fpu/xstate.h
---
arch/x86/include/asm/fpu/internal.h +3 -1
@@ -80,7 +80,9 @@ static __always_inline __pure bool use_fxsr(void)extern union fpregs_state init_fpstate; -extern void fpstate_init (union fpregs_state *state); +extern void fpstate_init_user (union fpregs_state *state); +extern void fpu_init_fpstate_user (struct fpu *fpu); +#ifdef CONFIG_MATH_EMULATION extern void fpstate_init_soft (struct swregs_state *soft);#else
arch/x86/kernel/fpu/core.c +19 -16
@@ -16,6 +16,8 @@#include <linux/hardirq.h> #include <linux/pkeys.h> +#include "xstate.h" +#define CREATE_TRACE_POINTS #include <asm/trace/fpu.h>
@@ -203,15 +205,6 @@ void fpu_sync_fpstate(struct fpu *fpu)fpregs_unlock (); } -static inline void fpstate_init_xstate (struct xregs_state *xsave) -{ -/* -* XRSTORS requires these bits set in xcomp_bv, or it will -* trigger #GP: -*/ - xsave->header.xcomp_bv =XCOMP_BV_COMPACTED_FORMAT | xfeatures_mask_all; -} -static inline unsigned int init_fpstate_copy_size (void ) {if (!use_xsave ())
@@ -238,23 +231,33 @@ static inline void fpstate_init_fstate(struct fregs_state *fp) fp->fos =0xffff0000u ; } -void fpstate_init (union fpregs_state *state) +/* +* Used in two places: +* 1) Early boot to setup init_fpstate for non XSAVE systems +* 2) fpu_init_fpstate_user() which is invoked from KVM +*/ +void fpstate_init_user (union fpregs_state *state) { -if (!static_cpu_has (X86_FEATURE_FPU )) { +if (!cpu_feature_enabled (X86_FEATURE_FPU )) {fpstate_init_soft (&state->soft);return ; } -memset (state,0 , fpu_kernel_xstate_size); +xstate_init_xcomp_bv (&state->xsave,xfeatures_mask_uabi ()); -if (static_cpu_has (X86_FEATURE_XSAVES )) -fpstate_init_xstate (&state->xsave); -if (static_cpu_has (X86_FEATURE_FXSR )) +if (cpu_feature_enabled (X86_FEATURE_FXSR ))fpstate_init_fxstate (&state->fxsave);else fpstate_init_fstate (&state->fsave); } -EXPORT_SYMBOL_GPL (fpstate_init); + +#if IS_ENABLED (CONFIG_KVM ) +void fpu_init_fpstate_user (struct fpu *fpu) +{ +fpstate_init_user (&fpu->state); +} +EXPORT_SYMBOL_GPL (fpu_init_fpstate_user); +#endif /* Clone current's FPU state on fork */ int fpu_clone (struct task_struct *dst)
arch/x86/kernel/fpu/init.c +3 -3
@@ -121,10 +121,10 @@ static void __init fpu__init_system_mxcsr(void)static void __init fpu__init_system_generic (void ) {/* -* Set up the legacy init FPU context. (xstate init might overwrite this -* with a more modern format, if the CPU supports it.) +* Set up the legacy init FPU context. Will be updated when the +* CPU supports XSAVE[S]. */ -fpstate_init (&init_fpstate); +fpstate_init_user (&init_fpstate);fpu__init_system_mxcsr (); }
arch/x86/kernel/fpu/xstate.c +3 -5
@@ -15,10 +15,10 @@#include <asm/fpu/internal.h> #include <asm/fpu/signal.h> #include <asm/fpu/regset.h> -#include <asm/fpu/xstate.h> #include <asm/tlbflush.h> -#include <asm/cpufeature.h> + +#include "xstate.h" /* * Although we spell it out in here, the Processor Trace
@@ -389,9 +389,7 @@ static void __init setup_init_fpu_buf(void)setup_xstate_features ();print_xstate_features (); -if (boot_cpu_has (X86_FEATURE_XSAVES )) - init_fpstate.xsave.header.xcomp_bv =XCOMP_BV_COMPACTED_FORMAT | - xfeatures_mask_all; +xstate_init_xcomp_bv (&init_fpstate.xsave, xfeatures_mask_all);/* * Init all the features state with header.xfeatures being 0x0
arch/x86/kernel/fpu/xstate.h new file +18 -0
@@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __X86_KERNEL_FPU_XSTATE_H +#define __X86_KERNEL_FPU_XSTATE_H + +#include <asm/cpufeature.h> +#include <asm/fpu/xstate.h> + +static inline void xstate_init_xcomp_bv (struct xregs_state *xsave,u64 mask) +{ +/* +* XRSTORS requires these bits set in xcomp_bv, or it will +* trigger #GP: +*/ +if (cpu_feature_enabled (X86_FEATURE_XSAVES )) + xsave->header.xcomp_bv = mask |XCOMP_BV_COMPACTED_FORMAT ; +} + +#endif
arch/x86/kvm/x86.c +3 -8
@@ -10612,14 +10612,6 @@ static int sync_regs(struct kvm_vcpu *vcpu)static void fx_init (struct kvm_vcpu *vcpu) { -if (!vcpu->arch.guest_fpu) -return ; - -fpstate_init (&vcpu->arch.guest_fpu->state); -if (boot_cpu_has (X86_FEATURE_XSAVES )) - vcpu->arch.guest_fpu->state.xsave.header.xcomp_bv = - host_xcr0 |XSTATE_COMPACTION_ENABLED ; -/* * Ensure guest xcr0 is valid for loading */
@@ -10704,6 +10696,9 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)pr_err ("kvm: failed to allocate vcpu's fpu\n" ); goto free_user_fpu; } + +fpu_init_fpstate_user (vcpu->arch.user_fpu); +fpu_init_fpstate_user (vcpu->arch.guest_fpu);fx_init (vcpu); vcpu->arch.maxphyaddr =cpuid_query_maxphyaddr (vcpu);
[tip: x86/fpu] x86/fpu: Cleanup xstate xcomp_bv initialization
The following commit has been merged into the x86/fpu branch of tip:
The following commit has been merged into the x86/fpu branch of tip:
Commit-ID: 126fe0401883598b45b34dbbd5e0d7d8a0aefa21 Gitweb: https://git.kernel.org/tip/126fe0401883598b45b34dbbd5e0d7d8a0aefa21 Author: Thomas Gleixner <[email protected]> AuthorDate: Fri, 15 Oct 2021 03:16:07 +02:00 Committer: Borislav Petkov <[email protected]> CommitterDate: Wed, 20 Oct 2021 15:27:26 +02:00
x86/fpu: Cleanup xstate xcomp_bv initialization
No point in having this duplicated all over the place with needlessly different defines.
Provide a proper initialization function which initializes user buffers properly and make KVM use it.
--- arch/x86/include/asm/fpu/internal.h | 4 ++- arch/x86/kernel/fpu/core.c | 35 +++++++++++++++------------- arch/x86/kernel/fpu/init.c | 6 ++--- arch/x86/kernel/fpu/xstate.c | 8 ++---- arch/x86/kernel/fpu/xstate.h | 18 ++++++++++++++- arch/x86/kvm/x86.c | 11 ++------- 6 files changed, 49 insertions(+), 33 deletions(-) create mode 100644 arch/x86/kernel/fpu/xstate.h
arch/x86/include/asm/fpu/internal.h +3 -1
@@ -80,7 +80,9 @@ static __always_inline __pure bool use_fxsr(void)extern union fpregs_state init_fpstate; -extern void fpstate_init (union fpregs_state *state); +extern void fpstate_init_user (union fpregs_state *state); +extern void fpu_init_fpstate_user (struct fpu *fpu); +#ifdef CONFIG_MATH_EMULATION extern void fpstate_init_soft (struct swregs_state *soft);#else
arch/x86/kernel/fpu/core.c +19 -16
@@ -16,6 +16,8 @@#include <linux/hardirq.h> #include <linux/pkeys.h> +#include "xstate.h" +#define CREATE_TRACE_POINTS #include <asm/trace/fpu.h>
@@ -203,15 +205,6 @@ void fpu_sync_fpstate(struct fpu *fpu)fpregs_unlock (); } -static inline void fpstate_init_xstate (struct xregs_state *xsave) -{ -/* -* XRSTORS requires these bits set in xcomp_bv, or it will -* trigger #GP: -*/ - xsave->header.xcomp_bv =XCOMP_BV_COMPACTED_FORMAT | xfeatures_mask_all; -} -static inline unsigned int init_fpstate_copy_size (void ) {if (!use_xsave ())
@@ -238,23 +231,33 @@ static inline void fpstate_init_fstate(struct fregs_state *fp) fp->fos =0xffff0000u ; } -void fpstate_init (union fpregs_state *state) +/* +* Used in two places: +* 1) Early boot to setup init_fpstate for non XSAVE systems +* 2) fpu_init_fpstate_user() which is invoked from KVM +*/ +void fpstate_init_user (union fpregs_state *state) { -if (!static_cpu_has (X86_FEATURE_FPU )) { +if (!cpu_feature_enabled (X86_FEATURE_FPU )) {fpstate_init_soft (&state->soft);return ; } -memset (state,0 , fpu_kernel_xstate_size); +xstate_init_xcomp_bv (&state->xsave,xfeatures_mask_uabi ()); -if (static_cpu_has (X86_FEATURE_XSAVES )) -fpstate_init_xstate (&state->xsave); -if (static_cpu_has (X86_FEATURE_FXSR )) +if (cpu_feature_enabled (X86_FEATURE_FXSR ))fpstate_init_fxstate (&state->fxsave);else fpstate_init_fstate (&state->fsave); } -EXPORT_SYMBOL_GPL (fpstate_init); + +#if IS_ENABLED (CONFIG_KVM ) +void fpu_init_fpstate_user (struct fpu *fpu) +{ +fpstate_init_user (&fpu->state); +} +EXPORT_SYMBOL_GPL (fpu_init_fpstate_user); +#endif /* Clone current's FPU state on fork */ int fpu_clone (struct task_struct *dst)
arch/x86/kernel/fpu/init.c +3 -3
@@ -121,10 +121,10 @@ static void __init fpu__init_system_mxcsr(void)static void __init fpu__init_system_generic (void ) {/* -* Set up the legacy init FPU context. (xstate init might overwrite this -* with a more modern format, if the CPU supports it.) +* Set up the legacy init FPU context. Will be updated when the +* CPU supports XSAVE[S]. */ -fpstate_init (&init_fpstate); +fpstate_init_user (&init_fpstate);fpu__init_system_mxcsr (); }
arch/x86/kernel/fpu/xstate.c +3 -5
@@ -15,10 +15,10 @@#include <asm/fpu/internal.h> #include <asm/fpu/signal.h> #include <asm/fpu/regset.h> -#include <asm/fpu/xstate.h> #include <asm/tlbflush.h> -#include <asm/cpufeature.h> + +#include "xstate.h" /* * Although we spell it out in here, the Processor Trace
@@ -389,9 +389,7 @@ static void __init setup_init_fpu_buf(void)setup_xstate_features ();print_xstate_features (); -if (boot_cpu_has (X86_FEATURE_XSAVES )) - init_fpstate.xsave.header.xcomp_bv =XCOMP_BV_COMPACTED_FORMAT | - xfeatures_mask_all; +xstate_init_xcomp_bv (&init_fpstate.xsave, xfeatures_mask_all);/* * Init all the features state with header.xfeatures being 0x0
arch/x86/kernel/fpu/xstate.h new file +18 -0
@@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __X86_KERNEL_FPU_XSTATE_H +#define __X86_KERNEL_FPU_XSTATE_H + +#include <asm/cpufeature.h> +#include <asm/fpu/xstate.h> + +static inline void xstate_init_xcomp_bv (struct xregs_state *xsave,u64 mask) +{ +/* +* XRSTORS requires these bits set in xcomp_bv, or it will +* trigger #GP: +*/ +if (cpu_feature_enabled (X86_FEATURE_XSAVES )) + xsave->header.xcomp_bv = mask |XCOMP_BV_COMPACTED_FORMAT ; +} + +#endif
arch/x86/kvm/x86.c +3 -8
@@ -10626,14 +10626,6 @@ static int sync_regs(struct kvm_vcpu *vcpu)static void fx_init (struct kvm_vcpu *vcpu) { -if (!vcpu->arch.guest_fpu) -return ; - -fpstate_init (&vcpu->arch.guest_fpu->state); -if (boot_cpu_has (X86_FEATURE_XSAVES )) - vcpu->arch.guest_fpu->state.xsave.header.xcomp_bv = - host_xcr0 |XSTATE_COMPACTION_ENABLED ; -/* * Ensure guest xcr0 is valid for loading */
@@ -10720,6 +10712,9 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)pr_err ("kvm: failed to allocate vcpu's fpu\n" ); goto free_user_fpu; } + +fpu_init_fpstate_user (vcpu->arch.user_fpu); +fpu_init_fpstate_user (vcpu->arch.guest_fpu);fx_init (vcpu); vcpu->arch.maxphyaddr =cpuid_query_maxphyaddr (vcpu);
PATCH v2 10/30 x86/fpu/xstate: Provide and use for_each_xfeature()
In reply to Thomas GleixnerThese loops evaluating xfeature bits are really hard to read. Create an iterator and use for_each_set_bit_from() inside which already does the right thing.
No functional changes.
---
V2: Update changelog - Boris
--- arch/x86/kernel/fpu/xstate.c | 56 ++++++++++++++++++--------------------------- 1 file changed, 23 insertions(+), 33 deletions(-)
---
arch/x86/kernel/fpu/xstate.c +23 -33
@@ -4,6 +4,7 @@* * Author: Suresh Siddha <[email protected]> */ +#include <linux/bitops.h> #include <linux/compat.h> #include <linux/cpu.h> #include <linux/mman.h>
@@ -20,6 +21,10 @@#include "xstate.h" +#define for_each_extended_xfeature (bit, mask) \ + (bit) = FIRST_EXTENDED_XFEATURE; \ + for_each_set_bit_from(bit, (unsigned long *)&(mask), 8 * sizeof(mask)) +/* * Although we spell it out in here, the Processor Trace * xfeature is completely unused. We use other mechanisms
@@ -184,10 +189,7 @@ static void __init setup_xstate_features(void) xstate_sizes[XFEATURE_SSE] = sizeof_field(struct fxregs_state, xmm_space); -for (i =FIRST_EXTENDED_XFEATURE ; i <XFEATURE_MAX ; i++) { -if (!xfeature_enabled (i)) -continue ; - +for_each_extended_xfeature (i, xfeatures_mask_all) {cpuid_count (XSTATE_CPUID , i, &eax, &ebx, &ecx, &edx); xstate_sizes[i] = eax;
@@ -291,20 +293,15 @@ static void __init setup_xstate_comp_offsets(void) xstate_comp_offsets[XFEATURE_SSE ] = offsetof(struct fxregs_state , xmm_space); -if (!boot_cpu_has (X86_FEATURE_XSAVES )) { -for (i =FIRST_EXTENDED_XFEATURE ; i <XFEATURE_MAX ; i++) { -if (xfeature_enabled (i)) - xstate_comp_offsets[i] = xstate_offsets[i]; - } +if (!cpu_feature_enabled (X86_FEATURE_XSAVES )) { +for_each_extended_xfeature (i, xfeatures_mask_all) + xstate_comp_offsets[i] = xstate_offsets[i];return ; } next_offset =FXSAVE_SIZE +XSAVE_HDR_SIZE ; -for (i =FIRST_EXTENDED_XFEATURE ; i <XFEATURE_MAX ; i++) { -if (!xfeature_enabled (i)) -continue ; - +for_each_extended_xfeature (i, xfeatures_mask_all) {if (xfeature_is_aligned (i)) next_offset =ALIGN (next_offset,64 );
@@ -328,8 +325,8 @@ static void __init setup_supervisor_only_offsets(void) next_offset =FXSAVE_SIZE +XSAVE_HDR_SIZE ; -for (i =FIRST_EXTENDED_XFEATURE ; i <XFEATURE_MAX ; i++) { -if (!xfeature_enabled (i) || !xfeature_is_supervisor (i)) +for_each_extended_xfeature (i, xfeatures_mask_all) { +if (!xfeature_is_supervisor (i))continue ;if (xfeature_is_aligned (i))
@@ -347,9 +344,7 @@ static void __init print_xstate_offset_size(void) {int i; -for (i =FIRST_EXTENDED_XFEATURE ; i <XFEATURE_MAX ; i++) { -if (!xfeature_enabled (i)) -continue ; +for_each_extended_xfeature (i, xfeatures_mask_all) {pr_info ("x86/fpu: xstate_offset[%d]: %4d, xstate_sizes[%d]: %4d\n" , i, xstate_comp_offsets[i], i, xstate_sizes[i]); }
@@ -554,10 +549,7 @@ static void do_extra_xstate_size_checks(void)int paranoid_xstate_size =FXSAVE_SIZE +XSAVE_HDR_SIZE ;int i; -for (i =FIRST_EXTENDED_XFEATURE ; i <XFEATURE_MAX ; i++) { -if (!xfeature_enabled (i)) -continue ; - +for_each_extended_xfeature (i, xfeatures_mask_all) {check_xstate_against_struct (i);/* * Supervisor state components can be managed only by
@@ -586,7 +578,6 @@ static void do_extra_xstate_size_checks(void)XSTATE_WARN_ON (paranoid_xstate_size != fpu_kernel_xstate_size); } -/* * Get total size of enabled xstates in XCR0 | IA32_XSS. *
@@ -969,6 +960,7 @@ void copy_xstate_to_uabi_buf(struct membuf to, struct task_struct *tsk,struct xregs_state *xinit = &init_fpstate.xsave;struct xstate_header header;unsigned int zerofrom; +u64 mask;int i;memset (&header,0 ,sizeof (header));
@@ -1022,17 +1014,15 @@ void copy_xstate_to_uabi_buf(struct membuf to, struct task_struct *tsk, zerofrom = offsetof(struct xregs_state , extended_state_area); -for (i =FIRST_EXTENDED_XFEATURE ; i <XFEATURE_MAX ; i++) { -/* -* The ptrace buffer is in non-compacted XSAVE format. -* In non-compacted format disabled features still occupy -* state space, but there is no state to copy from in the -* compacted init_fpstate. The gap tracking will zero this -* later. -*/ -if (!(xfeatures_mask_uabi () &BIT_ULL (i))) -continue ; +/* +* The ptrace buffer is in non-compacted XSAVE format. In +* non-compacted format disabled features still occupy state space, +* but there is no state to copy from in the compacted +* init_fpstate. The gap tracking will zero these states. +*/ + mask =xfeatures_mask_uabi (); +for_each_extended_xfeature (i, mask) {/* * If there was a feature or alignment gap, zero the space * in the destination buffer.
[tip: x86/fpu] x86/fpu/xstate: Provide and use for_each_xfeature()
The following commit has been merged into the x86/fpu branch of tip:
The following commit has been merged into the x86/fpu branch of tip:
Commit-ID: ffd3e504c9e0de8b85755f3c7eabbbdd984cfeed Gitweb: https://git.kernel.org/tip/ffd3e504c9e0de8b85755f3c7eabbbdd984cfeed Author: Thomas Gleixner <[email protected]> AuthorDate: Fri, 15 Oct 2021 03:16:09 +02:00 Committer: Borislav Petkov <[email protected]> CommitterDate: Wed, 20 Oct 2021 15:27:26 +02:00
x86/fpu/xstate: Provide and use for_each_xfeature()
These loops evaluating xfeature bits are really hard to read. Create an iterator and use for_each_set_bit_from() inside which already does the right thing.
No functional changes.
--- arch/x86/kernel/fpu/xstate.c | 56 ++++++++++++++--------------------- 1 file changed, 23 insertions(+), 33 deletions(-)
arch/x86/kernel/fpu/xstate.c +23 -33
@@ -4,6 +4,7 @@* * Author: Suresh Siddha <[email protected]> */ +#include <linux/bitops.h> #include <linux/compat.h> #include <linux/cpu.h> #include <linux/mman.h>
@@ -20,6 +21,10 @@#include "xstate.h" +#define for_each_extended_xfeature (bit, mask) \ + (bit) = FIRST_EXTENDED_XFEATURE; \ + for_each_set_bit_from(bit, (unsigned long *)&(mask), 8 * sizeof(mask)) +/* * Although we spell it out in here, the Processor Trace * xfeature is completely unused. We use other mechanisms
@@ -184,10 +189,7 @@ static void __init setup_xstate_features(void) xstate_sizes[XFEATURE_SSE] = sizeof_field(struct fxregs_state, xmm_space); -for (i =FIRST_EXTENDED_XFEATURE ; i <XFEATURE_MAX ; i++) { -if (!xfeature_enabled (i)) -continue ; - +for_each_extended_xfeature (i, xfeatures_mask_all) {cpuid_count (XSTATE_CPUID , i, &eax, &ebx, &ecx, &edx); xstate_sizes[i] = eax;
@@ -291,20 +293,15 @@ static void __init setup_xstate_comp_offsets(void) xstate_comp_offsets[XFEATURE_SSE ] = offsetof(struct fxregs_state , xmm_space); -if (!boot_cpu_has (X86_FEATURE_XSAVES )) { -for (i =FIRST_EXTENDED_XFEATURE ; i <XFEATURE_MAX ; i++) { -if (xfeature_enabled (i)) - xstate_comp_offsets[i] = xstate_offsets[i]; - } +if (!cpu_feature_enabled (X86_FEATURE_XSAVES )) { +for_each_extended_xfeature (i, xfeatures_mask_all) + xstate_comp_offsets[i] = xstate_offsets[i];return ; } next_offset =FXSAVE_SIZE +XSAVE_HDR_SIZE ; -for (i =FIRST_EXTENDED_XFEATURE ; i <XFEATURE_MAX ; i++) { -if (!xfeature_enabled (i)) -continue ; - +for_each_extended_xfeature (i, xfeatures_mask_all) {if (xfeature_is_aligned (i)) next_offset =ALIGN (next_offset,64 );
@@ -328,8 +325,8 @@ static void __init setup_supervisor_only_offsets(void) next_offset =FXSAVE_SIZE +XSAVE_HDR_SIZE ; -for (i =FIRST_EXTENDED_XFEATURE ; i <XFEATURE_MAX ; i++) { -if (!xfeature_enabled (i) || !xfeature_is_supervisor (i)) +for_each_extended_xfeature (i, xfeatures_mask_all) { +if (!xfeature_is_supervisor (i))continue ;if (xfeature_is_aligned (i))
@@ -347,9 +344,7 @@ static void __init print_xstate_offset_size(void) {int i; -for (i =FIRST_EXTENDED_XFEATURE ; i <XFEATURE_MAX ; i++) { -if (!xfeature_enabled (i)) -continue ; +for_each_extended_xfeature (i, xfeatures_mask_all) {pr_info ("x86/fpu: xstate_offset[%d]: %4d, xstate_sizes[%d]: %4d\n" , i, xstate_comp_offsets[i], i, xstate_sizes[i]); }
@@ -554,10 +549,7 @@ static void do_extra_xstate_size_checks(void)int paranoid_xstate_size =FXSAVE_SIZE +XSAVE_HDR_SIZE ;int i; -for (i =FIRST_EXTENDED_XFEATURE ; i <XFEATURE_MAX ; i++) { -if (!xfeature_enabled (i)) -continue ; - +for_each_extended_xfeature (i, xfeatures_mask_all) {check_xstate_against_struct (i);/* * Supervisor state components can be managed only by
@@ -586,7 +578,6 @@ static void do_extra_xstate_size_checks(void)XSTATE_WARN_ON (paranoid_xstate_size != fpu_kernel_xstate_size); } -/* * Get total size of enabled xstates in XCR0 | IA32_XSS. *
@@ -969,6 +960,7 @@ void copy_xstate_to_uabi_buf(struct membuf to, struct task_struct *tsk,struct xregs_state *xinit = &init_fpstate.xsave;struct xstate_header header;unsigned int zerofrom; +u64 mask;int i;memset (&header,0 ,sizeof (header));
@@ -1022,17 +1014,15 @@ void copy_xstate_to_uabi_buf(struct membuf to, struct task_struct *tsk, zerofrom = offsetof(struct xregs_state , extended_state_area); -for (i =FIRST_EXTENDED_XFEATURE ; i <XFEATURE_MAX ; i++) { -/* -* The ptrace buffer is in non-compacted XSAVE format. -* In non-compacted format disabled features still occupy -* state space, but there is no state to copy from in the -* compacted init_fpstate. The gap tracking will zero this -* later. -*/ -if (!(xfeatures_mask_uabi () &BIT_ULL (i))) -continue ; +/* +* The ptrace buffer is in non-compacted XSAVE format. In +* non-compacted format disabled features still occupy state space, +* but there is no state to copy from in the compacted +* init_fpstate. The gap tracking will zero these states. +*/ + mask =xfeatures_mask_uabi (); +for_each_extended_xfeature (i, mask) {/* * If there was a feature or alignment gap, zero the space * in the destination buffer.
PATCH v2 11/30 x86/fpu/xstate: Mark all init only functions __init
In reply to Thomas GleixnerNo point to keep them around after boot.
--- arch/x86/kernel/fpu/xstate.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-)
---
arch/x86/kernel/fpu/xstate.c +5 -5
@@ -462,7 +462,7 @@ static int validate_user_xstate_header(const struct xstate_header *hdr)return 0 ; } -static void __xstate_dump_leaves (void ) +static void __init __xstate_dump_leaves (void ) {int i;u32 eax, ebx, ecx, edx;
@@ -502,7 +502,7 @@ static void __xstate_dump_leaves(void)* that our software representation matches what the CPU * tells us about the state's size. */ -static void check_xstate_against_struct (int nr) +static void __init check_xstate_against_struct (int nr) {/* * Ask the CPU for the size of the state.
@@ -544,7 +544,7 @@ static void check_xstate_against_struct(int nr)* covered by these checks. Only the size of the buffer for task->fpu * is checked here. */ -static void do_extra_xstate_size_checks (void ) +static void __init do_extra_xstate_size_checks (void ) {int paranoid_xstate_size =FXSAVE_SIZE +XSAVE_HDR_SIZE ;int i;
@@ -646,7 +646,7 @@ static unsigned int __init get_xsave_size(void)* Will the runtime-enumerated 'xstate_size' fit in the init * task's statically-allocated buffer? */ -static bool is_supported_xstate_size (unsigned int test_xstate_size) +static bool __init is_supported_xstate_size (unsigned int test_xstate_size) {if (test_xstate_size <=sizeof (union fpregs_state ))return true;
@@ -691,7 +691,7 @@ static int __init init_xstate_size(void)* We enabled the XSAVE hardware, but something went wrong and * we can not use it. Disable it. */ -static void fpu__init_disable_system_xstate (void ) +static void __init fpu__init_disable_system_xstate (void ) { xfeatures_mask_all =0 ;cr4_clear_bits (X86_CR4_OSXSAVE );
[tip: x86/fpu] x86/fpu/xstate: Mark all init only functions __init
The following commit has been merged into the x86/fpu branch of tip:
The following commit has been merged into the x86/fpu branch of tip:
Commit-ID: 63cf05a19a5d3fb6e66b5f7ceb76e77dfc2695f2 Gitweb: https://git.kernel.org/tip/63cf05a19a5d3fb6e66b5f7ceb76e77dfc2695f2 Author: Thomas Gleixner <[email protected]> AuthorDate: Fri, 15 Oct 2021 03:16:10 +02:00 Committer: Borislav Petkov <[email protected]> CommitterDate: Wed, 20 Oct 2021 15:27:27 +02:00
x86/fpu/xstate: Mark all init only functions __init
No point to keep them around after boot.
--- arch/x86/kernel/fpu/xstate.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-)
arch/x86/kernel/fpu/xstate.c +5 -5
@@ -462,7 +462,7 @@ static int validate_user_xstate_header(const struct xstate_header *hdr)return 0 ; } -static void __xstate_dump_leaves (void ) +static void __init __xstate_dump_leaves (void ) {int i;u32 eax, ebx, ecx, edx;
@@ -502,7 +502,7 @@ static void __xstate_dump_leaves(void)* that our software representation matches what the CPU * tells us about the state's size. */ -static void check_xstate_against_struct (int nr) +static void __init check_xstate_against_struct (int nr) {/* * Ask the CPU for the size of the state.
@@ -544,7 +544,7 @@ static void check_xstate_against_struct(int nr)* covered by these checks. Only the size of the buffer for task->fpu * is checked here. */ -static void do_extra_xstate_size_checks (void ) +static void __init do_extra_xstate_size_checks (void ) {int paranoid_xstate_size =FXSAVE_SIZE +XSAVE_HDR_SIZE ;int i;
@@ -646,7 +646,7 @@ static unsigned int __init get_xsave_size(void)* Will the runtime-enumerated 'xstate_size' fit in the init * task's statically-allocated buffer? */ -static bool is_supported_xstate_size (unsigned int test_xstate_size) +static bool __init is_supported_xstate_size (unsigned int test_xstate_size) {if (test_xstate_size <=sizeof (union fpregs_state ))return true;
@@ -691,7 +691,7 @@ static int __init init_xstate_size(void)* We enabled the XSAVE hardware, but something went wrong and * we can not use it. Disable it. */ -static void fpu__init_disable_system_xstate (void ) +static void __init fpu__init_disable_system_xstate (void ) { xfeatures_mask_all =0 ;cr4_clear_bits (X86_CR4_OSXSAVE );
PATCH v2 12/30 x86/fpu: Move KVMs FPU swapping to FPU core
In reply to Thomas GleixnerSwapping the host/guest FPU is directly fiddling with FPU internals which requires 5 exports. The upcoming support of dynamically enabled states would even need more.
Implement a swap function in the FPU core code and export that instead.
---
V2: Fix changelog and comments - Boris
--- arch/x86/include/asm/fpu/api.h | 8 ++++++- arch/x86/include/asm/fpu/internal.h | 15 +---------- arch/x86/kernel/fpu/core.c | 30 +++++++++++++++++++--- arch/x86/kernel/fpu/init.c | 1 +- arch/x86/kernel/fpu/xstate.c | 1 +- arch/x86/kvm/x86.c | 51 ++++++++------------------------------ arch/x86/mm/extable.c | 2 +- 7 files changed, 48 insertions(+), 60 deletions(-)
---
arch/x86/include/asm/fpu/api.h +8 -0
@@ -12,6 +12,8 @@#define _ASM_X86_FPU_API_H#include <linux/bottom_half.h> +#include <asm/fpu/types.h> +/* * Use kernel_fpu_begin/end() if you intend to use FPU in kernel context. It * disables preemption so be careful if you intend to use it for long periods
@@ -108,4 +110,10 @@ extern int cpu_has_xfeatures(u64 xfeatures_mask, const char **feature_name);static inline void update_pasid (void ) { } +/* fpstate-related functions which are exported to KVM */ +extern void fpu_init_fpstate_user (struct fpu *fpu); + +/* KVM specific functions */ +extern void fpu_swap_kvm_fpu (struct fpu *save,struct fpu *rstor,u64 restore_mask); +#endif /* _ASM_X86_FPU_API_H */
arch/x86/include/asm/fpu/internal.h +2 -13
@@ -74,14 +74,8 @@ static __always_inline __pure bool use_fxsr(void)return static_cpu_has (X86_FEATURE_FXSR ); } -/* -* fpstate handling functions: -*/ -extern union fpregs_state init_fpstate; -extern void fpstate_init_user (union fpregs_state *state); -extern void fpu_init_fpstate_user (struct fpu *fpu);#ifdef CONFIG_MATH_EMULATION extern void fpstate_init_soft (struct swregs_state *soft);
@@ -381,12 +375,7 @@ static inline int os_xrstor_safe(struct xregs_state *xstate, u64 mask)return err; } -extern void __restore_fpregs_from_fpstate (union fpregs_state *fpstate,u64 mask); - -static inline void restore_fpregs_from_fpstate (union fpregs_state *fpstate) -{ -__restore_fpregs_from_fpstate (fpstate,xfeatures_mask_fpstate ()); -} +extern void restore_fpregs_from_fpstate (union fpregs_state *fpstate,u64 mask);extern bool copy_fpstate_to_sigframe (void __user *buf,void __user *fp,int size);
@@ -467,7 +456,7 @@ static inline void fpregs_restore_userregs(void)*/ mask =xfeatures_mask_restore_user () |xfeatures_mask_supervisor (); -__restore_fpregs_from_fpstate (&fpu->state, mask); +restore_fpregs_from_fpstate (&fpu->state, mask);fpregs_activate (fpu); fpu->last_cpu = cpu;
arch/x86/kernel/fpu/core.c +26 -4
@@ -124,9 +124,8 @@ void save_fpregs_to_fpstate(struct fpu *fpu) asmvolatile ("fnsave %[fp]; fwait" : [fp]"=m" (fpu->state.fsave));frstor (&fpu->state.fsave); } -EXPORT_SYMBOL (save_fpregs_to_fpstate); -void __restore_fpregs_from_fpstate (union fpregs_state *fpstate,u64 mask) +void restore_fpregs_from_fpstate (union fpregs_state *fpstate,u64 mask) {/* * AMD K7/K8 and later CPUs up to Zen don't save/restore
@@ -151,7 +150,31 @@ void __restore_fpregs_from_fpstate(union fpregs_state *fpstate, u64 mask)frstor (&fpstate->fsave); } } -EXPORT_SYMBOL_GPL (__restore_fpregs_from_fpstate); + +#if IS_ENABLED (CONFIG_KVM ) +void fpu_swap_kvm_fpu (struct fpu *save,struct fpu *rstor,u64 restore_mask) +{ +fpregs_lock (); + +if (save) { +if (test_thread_flag (TIF_NEED_FPU_LOAD )) { +memcpy (&save->state, ¤t->thread.fpu.state, + fpu_kernel_xstate_size); + }else { +save_fpregs_to_fpstate (save); + } + } + +if (rstor) { + restore_mask &=xfeatures_mask_fpstate (); +restore_fpregs_from_fpstate (&rstor->state, restore_mask); + } + +fpregs_mark_activate (); +fpregs_unlock (); +} +EXPORT_SYMBOL_GPL (fpu_swap_kvm_fpu); +#endif void kernel_fpu_begin_mask (unsigned int kfpu_mask) {
@@ -457,7 +480,6 @@ void fpregs_mark_activate(void) fpu->last_cpu =smp_processor_id ();clear_thread_flag (TIF_NEED_FPU_LOAD ); } -EXPORT_SYMBOL_GPL (fpregs_mark_activate);/* * x87 math exception handling:
arch/x86/kernel/fpu/init.c +0 -1
@@ -136,7 +136,6 @@ static void __init fpu__init_system_generic(void)* components into a single, continuous memory block: */ unsigned int fpu_kernel_xstate_size__ro_after_init ; -EXPORT_SYMBOL_GPL (fpu_kernel_xstate_size);/* Get alignment of the TYPE. */ #define TYPE_ALIGN (TYPE ) offsetof(struct { char x; TYPE test; }, test)
arch/x86/kernel/fpu/xstate.c +0 -1
@@ -65,7 +65,6 @@ static short xsave_cpuid_features[] __initdata = {* XSAVE buffer, both supervisor and user xstates. */ u64 xfeatures_mask_all__ro_after_init ; -EXPORT_SYMBOL_GPL (xfeatures_mask_all);static unsigned int xstate_offsets[XFEATURE_MAX ]__ro_after_init = { [0 ...XFEATURE_MAX -1 ] =-1 };
arch/x86/kvm/x86.c +11 -40
76 lines not on this page · raw message
arch/x86/mm/extable.c +1 -1
@@ -47,7 +47,7 @@ static bool ex_handler_fprestore(const struct exception_table_entry *fixup,WARN_ONCE (1 ,"Bad FPU state detected at %pB, reinitializing FPU registers." , (void *)instruction_pointer (regs)); -__restore_fpregs_from_fpstate (&init_fpstate,xfeatures_mask_fpstate ()); +restore_fpregs_from_fpstate (&init_fpstate,xfeatures_mask_fpstate ());return true; }
[tip: x86/fpu] x86/fpu: Move KVMs FPU swapping to FPU core
The following commit has been merged into the x86/fpu branch of tip:
The following commit has been merged into the x86/fpu branch of tip:
Commit-ID: a0ff0611c2fbde94f6c9db8351939b08f2cb6797 Gitweb: https://git.kernel.org/tip/a0ff0611c2fbde94f6c9db8351939b08f2cb6797 Author: Thomas Gleixner <[email protected]> AuthorDate: Fri, 15 Oct 2021 03:16:12 +02:00 Committer: Borislav Petkov <[email protected]> CommitterDate: Wed, 20 Oct 2021 15:27:27 +02:00
x86/fpu: Move KVMs FPU swapping to FPU core
Swapping the host/guest FPU is directly fiddling with FPU internals which requires 5 exports. The upcoming support of dynamically enabled states would even need more.
Implement a swap function in the FPU core code and export that instead.
--- arch/x86/include/asm/fpu/api.h | 8 ++++- arch/x86/include/asm/fpu/internal.h | 15 +------- arch/x86/kernel/fpu/core.c | 30 +++++++++++++--- arch/x86/kernel/fpu/init.c | 1 +- arch/x86/kernel/fpu/xstate.c | 1 +- arch/x86/kvm/x86.c | 51 ++++++---------------------- arch/x86/mm/extable.c | 2 +- 7 files changed, 48 insertions(+), 60 deletions(-)
arch/x86/include/asm/fpu/api.h +8 -0
@@ -12,6 +12,8 @@#define _ASM_X86_FPU_API_H#include <linux/bottom_half.h> +#include <asm/fpu/types.h> +/* * Use kernel_fpu_begin/end() if you intend to use FPU in kernel context. It * disables preemption so be careful if you intend to use it for long periods
@@ -108,4 +110,10 @@ extern int cpu_has_xfeatures(u64 xfeatures_mask, const char **feature_name);static inline void update_pasid (void ) { } +/* fpstate-related functions which are exported to KVM */ +extern void fpu_init_fpstate_user (struct fpu *fpu); + +/* KVM specific functions */ +extern void fpu_swap_kvm_fpu (struct fpu *save,struct fpu *rstor,u64 restore_mask); +#endif /* _ASM_X86_FPU_API_H */
arch/x86/include/asm/fpu/internal.h +2 -13
@@ -74,14 +74,8 @@ static __always_inline __pure bool use_fxsr(void)return static_cpu_has (X86_FEATURE_FXSR ); } -/* -* fpstate handling functions: -*/ -extern union fpregs_state init_fpstate; -extern void fpstate_init_user (union fpregs_state *state); -extern void fpu_init_fpstate_user (struct fpu *fpu);#ifdef CONFIG_MATH_EMULATION extern void fpstate_init_soft (struct swregs_state *soft);
@@ -381,12 +375,7 @@ static inline int os_xrstor_safe(struct xregs_state *xstate, u64 mask)return err; } -extern void __restore_fpregs_from_fpstate (union fpregs_state *fpstate,u64 mask); - -static inline void restore_fpregs_from_fpstate (union fpregs_state *fpstate) -{ -__restore_fpregs_from_fpstate (fpstate,xfeatures_mask_fpstate ()); -} +extern void restore_fpregs_from_fpstate (union fpregs_state *fpstate,u64 mask);extern bool copy_fpstate_to_sigframe (void __user *buf,void __user *fp,int size);
@@ -467,7 +456,7 @@ static inline void fpregs_restore_userregs(void)*/ mask =xfeatures_mask_restore_user () |xfeatures_mask_supervisor (); -__restore_fpregs_from_fpstate (&fpu->state, mask); +restore_fpregs_from_fpstate (&fpu->state, mask);fpregs_activate (fpu); fpu->last_cpu = cpu;
arch/x86/kernel/fpu/core.c +26 -4
@@ -124,9 +124,8 @@ void save_fpregs_to_fpstate(struct fpu *fpu) asmvolatile ("fnsave %[fp]; fwait" : [fp]"=m" (fpu->state.fsave));frstor (&fpu->state.fsave); } -EXPORT_SYMBOL (save_fpregs_to_fpstate); -void __restore_fpregs_from_fpstate (union fpregs_state *fpstate,u64 mask) +void restore_fpregs_from_fpstate (union fpregs_state *fpstate,u64 mask) {/* * AMD K7/K8 and later CPUs up to Zen don't save/restore
@@ -151,7 +150,31 @@ void __restore_fpregs_from_fpstate(union fpregs_state *fpstate, u64 mask)frstor (&fpstate->fsave); } } -EXPORT_SYMBOL_GPL (__restore_fpregs_from_fpstate); + +#if IS_ENABLED (CONFIG_KVM ) +void fpu_swap_kvm_fpu (struct fpu *save,struct fpu *rstor,u64 restore_mask) +{ +fpregs_lock (); + +if (save) { +if (test_thread_flag (TIF_NEED_FPU_LOAD )) { +memcpy (&save->state, ¤t->thread.fpu.state, + fpu_kernel_xstate_size); + }else { +save_fpregs_to_fpstate (save); + } + } + +if (rstor) { + restore_mask &=xfeatures_mask_fpstate (); +restore_fpregs_from_fpstate (&rstor->state, restore_mask); + } + +fpregs_mark_activate (); +fpregs_unlock (); +} +EXPORT_SYMBOL_GPL (fpu_swap_kvm_fpu); +#endif void kernel_fpu_begin_mask (unsigned int kfpu_mask) {
@@ -457,7 +480,6 @@ void fpregs_mark_activate(void) fpu->last_cpu =smp_processor_id ();clear_thread_flag (TIF_NEED_FPU_LOAD ); } -EXPORT_SYMBOL_GPL (fpregs_mark_activate);/* * x87 math exception handling:
arch/x86/kernel/fpu/init.c +0 -1
@@ -136,7 +136,6 @@ static void __init fpu__init_system_generic(void)* components into a single, continuous memory block: */ unsigned int fpu_kernel_xstate_size__ro_after_init ; -EXPORT_SYMBOL_GPL (fpu_kernel_xstate_size);/* Get alignment of the TYPE. */ #define TYPE_ALIGN (TYPE ) offsetof(struct { char x; TYPE test; }, test)
arch/x86/kernel/fpu/xstate.c +0 -1
@@ -65,7 +65,6 @@ static short xsave_cpuid_features[] __initdata = {* XSAVE buffer, both supervisor and user xstates. */ u64 xfeatures_mask_all__ro_after_init ; -EXPORT_SYMBOL_GPL (xfeatures_mask_all);static unsigned int xstate_offsets[XFEATURE_MAX ]__ro_after_init = { [0 ...XFEATURE_MAX -1 ] =-1 };
arch/x86/kvm/x86.c +11 -40
76 lines not on this page · raw message
arch/x86/mm/extable.c +1 -1
@@ -47,7 +47,7 @@ static bool ex_handler_fprestore(const struct exception_table_entry *fixup,WARN_ONCE (1 ,"Bad FPU state detected at %pB, reinitializing FPU registers." , (void *)instruction_pointer (regs)); -__restore_fpregs_from_fpstate (&init_fpstate,xfeatures_mask_fpstate ()); +restore_fpregs_from_fpstate (&init_fpstate,xfeatures_mask_fpstate ());return true; }
PATCH v2 13/30 x86/fpu: Replace KVMs home brewed FPU copy from user
In reply to Thomas GleixnerCopying a user space buffer to the memory buffer is already available in the FPU core. The copy mechanism in KVM lacks sanity checks and needs to use cpuid() to lookup the offset of each component, while the FPU core has this information cached.
Make the FPU core variant accessible for KVM and replace the home brewed mechanism.
--- arch/x86/include/asm/fpu/api.h | 2 +- arch/x86/kernel/fpu/core.c | 38 +++++++++++++++++++++- arch/x86/kernel/fpu/xstate.c | 3 +-- arch/x86/kvm/x86.c | 74 +------------------------------------------ 4 files changed, 43 insertions(+), 74 deletions(-)
---
arch/x86/include/asm/fpu/api.h +2 -0
@@ -116,4 +116,6 @@ extern void fpu_init_fpstate_user(struct fpu *fpu);/* KVM specific functions */ extern void fpu_swap_kvm_fpu (struct fpu *save,struct fpu *rstor,u64 restore_mask); +extern int fpu_copy_kvm_uabi_to_fpstate (struct fpu *fpu,const void *buf,u64 xcr0,u32 *pkru); +#endif /* _ASM_X86_FPU_API_H */
arch/x86/kernel/fpu/core.c +37 -1
@@ -174,7 +174,43 @@ void fpu_swap_kvm_fpu(struct fpu *save, struct fpu *rstor, u64 restore_mask)fpregs_unlock (); }EXPORT_SYMBOL_GPL (fpu_swap_kvm_fpu); -#endif + +int fpu_copy_kvm_uabi_to_fpstate (struct fpu *fpu,const void *buf,u64 xcr0, +u32 *vpkru) +{ +union fpregs_state *kstate = &fpu->state; +const union fpregs_state *ustate = buf; +struct pkru_state *xpkru; +int ret; + +if (!cpu_feature_enabled (X86_FEATURE_XSAVE )) { +if (ustate->xsave.header.xfeatures & ~XFEATURE_MASK_FPSSE ) +return -EINVAL ; +if (ustate->fxsave.mxcsr & ~mxcsr_feature_mask) +return -EINVAL ; +memcpy (&kstate->fxsave, &ustate->fxsave,sizeof (ustate->fxsave)); +return 0 ; + } + +if (ustate->xsave.header.xfeatures & ~xcr0) +return -EINVAL ; + + ret =copy_uabi_from_kernel_to_xstate (&kstate->xsave, ustate); +if (ret) +return ret; + +/* Retrieve PKRU if not in init state */ +if (kstate->xsave.header.xfeatures &XFEATURE_MASK_PKRU ) { + xpkru =get_xsave_addr (&kstate->xsave,XFEATURE_PKRU ); + *vpkru = xpkru->pkru; + } + +/* Ensure that XCOMP_BV is set up for XSAVES */ +xstate_init_xcomp_bv (&kstate->xsave,xfeatures_mask_uabi ()); +return 0 ; +} +EXPORT_SYMBOL_GPL (fpu_copy_kvm_uabi_to_fpstate); +#endif /* CONFIG_KVM */ void kernel_fpu_begin_mask (unsigned int kfpu_mask) {
arch/x86/kernel/fpu/xstate.c +1 -2
@@ -1134,8 +1134,7 @@ static int copy_uabi_to_xstate(struct xregs_state *xsave, const void *kbuf,/* * Convert from a ptrace standard-format kernel buffer to kernel XSAVE[S] -* format and copy to the target thread. This is called from -* xstateregs_set(). +* format and copy to the target thread. Used by ptrace and KVM. */ int copy_uabi_from_kernel_to_xstate (struct xregs_state *xsave,const void *kbuf) {
arch/x86/kvm/x86.c +3 -71
98 lines not on this page · raw message
[tip: x86/fpu] x86/fpu: Replace KVMs home brewed FPU copy from user
The following commit has been merged into the x86/fpu branch of tip:
The following commit has been merged into the x86/fpu branch of tip:
Commit-ID: ea4d6938d4c0761672ff6237964a20db3cb95cc1 Gitweb: https://git.kernel.org/tip/ea4d6938d4c0761672ff6237964a20db3cb95cc1 Author: Thomas Gleixner <[email protected]> AuthorDate: Fri, 15 Oct 2021 03:16:13 +02:00 Committer: Borislav Petkov <[email protected]> CommitterDate: Wed, 20 Oct 2021 15:27:27 +02:00
x86/fpu: Replace KVMs home brewed FPU copy from user
Copying a user space buffer to the memory buffer is already available in the FPU core. The copy mechanism in KVM lacks sanity checks and needs to use cpuid() to lookup the offset of each component, while the FPU core has this information cached.
Make the FPU core variant accessible for KVM and replace the home brewed mechanism.
--- arch/x86/include/asm/fpu/api.h | 2 +- arch/x86/kernel/fpu/core.c | 38 ++++++++++++++++- arch/x86/kernel/fpu/xstate.c | 3 +- arch/x86/kvm/x86.c | 74 +-------------------------------- 4 files changed, 43 insertions(+), 74 deletions(-)
arch/x86/include/asm/fpu/api.h +2 -0
@@ -116,4 +116,6 @@ extern void fpu_init_fpstate_user(struct fpu *fpu);/* KVM specific functions */ extern void fpu_swap_kvm_fpu (struct fpu *save,struct fpu *rstor,u64 restore_mask); +extern int fpu_copy_kvm_uabi_to_fpstate (struct fpu *fpu,const void *buf,u64 xcr0,u32 *pkru); +#endif /* _ASM_X86_FPU_API_H */
arch/x86/kernel/fpu/core.c +37 -1
@@ -174,7 +174,43 @@ void fpu_swap_kvm_fpu(struct fpu *save, struct fpu *rstor, u64 restore_mask)fpregs_unlock (); }EXPORT_SYMBOL_GPL (fpu_swap_kvm_fpu); -#endif + +int fpu_copy_kvm_uabi_to_fpstate (struct fpu *fpu,const void *buf,u64 xcr0, +u32 *vpkru) +{ +union fpregs_state *kstate = &fpu->state; +const union fpregs_state *ustate = buf; +struct pkru_state *xpkru; +int ret; + +if (!cpu_feature_enabled (X86_FEATURE_XSAVE )) { +if (ustate->xsave.header.xfeatures & ~XFEATURE_MASK_FPSSE ) +return -EINVAL ; +if (ustate->fxsave.mxcsr & ~mxcsr_feature_mask) +return -EINVAL ; +memcpy (&kstate->fxsave, &ustate->fxsave,sizeof (ustate->fxsave)); +return 0 ; + } + +if (ustate->xsave.header.xfeatures & ~xcr0) +return -EINVAL ; + + ret =copy_uabi_from_kernel_to_xstate (&kstate->xsave, ustate); +if (ret) +return ret; + +/* Retrieve PKRU if not in init state */ +if (kstate->xsave.header.xfeatures &XFEATURE_MASK_PKRU ) { + xpkru =get_xsave_addr (&kstate->xsave,XFEATURE_PKRU ); + *vpkru = xpkru->pkru; + } + +/* Ensure that XCOMP_BV is set up for XSAVES */ +xstate_init_xcomp_bv (&kstate->xsave,xfeatures_mask_uabi ()); +return 0 ; +} +EXPORT_SYMBOL_GPL (fpu_copy_kvm_uabi_to_fpstate); +#endif /* CONFIG_KVM */ void kernel_fpu_begin_mask (unsigned int kfpu_mask) {
arch/x86/kernel/fpu/xstate.c +1 -2
@@ -1134,8 +1134,7 @@ static int copy_uabi_to_xstate(struct xregs_state *xsave, const void *kbuf,/* * Convert from a ptrace standard-format kernel buffer to kernel XSAVE[S] -* format and copy to the target thread. This is called from -* xstateregs_set(). +* format and copy to the target thread. Used by ptrace and KVM. */ int copy_uabi_from_kernel_to_xstate (struct xregs_state *xsave,const void *kbuf) {
arch/x86/kvm/x86.c +3 -71
98 lines not on this page · raw message
PATCH v2 14/30 x86/fpu: Rework copy_xstate_to_uabi_buf()
In reply to Thomas GleixnerPrepare for replacing the KVM copy xstate to user function by extending copy_xstate_to_uabi_buf() with a pkru argument which allows the caller to hand in the pkru value, which is required for KVM because the guest PKRU is not accessible via current. Fixup all callsites accordingly.
--- arch/x86/kernel/fpu/xstate.c | 34 ++++++++++++++++++++++++++-------- arch/x86/kernel/fpu/xstate.h | 3 +++ 2 files changed, 29 insertions(+), 8 deletions(-)
---
arch/x86/kernel/fpu/xstate.c +26 -8
@@ -940,9 +940,10 @@ static void copy_feature(bool from_xstate, struct membuf *to, void *xstate, }/** -* copy_xstate_to_uabi_buf - Copy kernel saved xstate to a UABI buffer +* __copy_xstate_to_uabi_buf - Copy kernel saved xstate to a UABI buffer * @to: membuf descriptor -* @tsk: The task from which to copy the saved xstate +* @xsave: The xsave from which to copy +* @pkru_val: The PKRU value to store in the PKRU component * @copy_mode: The requested copy mode * * Converts from kernel XSAVE or XSAVES compacted format to UABI conforming
@@ -951,11 +952,10 @@ static void copy_feature(bool from_xstate, struct membuf *to, void *xstate,* * It supports partial copy but @to.pos always starts from zero. */ -void copy_xstate_to_uabi_buf (struct membuf to,struct task_struct *tsk, -enum xstate_copy_mode copy_mode) +void __copy_xstate_to_uabi_buf (struct membuf to,struct xregs_state *xsave, +u32 pkru_val,enum xstate_copy_mode copy_mode) {const unsigned int off_mxcsr = offsetof(struct fxregs_state , mxcsr); -struct xregs_state *xsave = &tsk->thread.fpu.state.xsave;struct xregs_state *xinit = &init_fpstate.xsave;struct xstate_header header;unsigned int zerofrom;
@@ -1033,10 +1033,9 @@ void copy_xstate_to_uabi_buf(struct membuf to, struct task_struct *tsk,struct pkru_state pkru = {0 };/* * PKRU is not necessarily up to date in the -* thread's XSAVE buffer. Fill this part from the -* per-thread storage. +* XSAVE buffer. Use the provided value. */ - pkru.pkru = tsk->thread.pkru; + pkru.pkru = pkru_val;membuf_write (&to, &pkru,sizeof (pkru)); }else { copy_feature(header.xfeatures & BIT_ULL(i), &to,
@@ -1056,6 +1055,25 @@ void copy_xstate_to_uabi_buf(struct membuf to, struct task_struct *tsk,membuf_zero (&to, to.left); } +/** +* copy_xstate_to_uabi_buf - Copy kernel saved xstate to a UABI buffer +* @to: membuf descriptor +* @tsk: The task from which to copy the saved xstate +* @copy_mode: The requested copy mode +* +* Converts from kernel XSAVE or XSAVES compacted format to UABI conforming +* format, i.e. from the kernel internal hardware dependent storage format +* to the requested @mode. UABI XSTATE is always uncompacted! +* +* It supports partial copy but @to.pos always starts from zero. +*/ +void copy_xstate_to_uabi_buf (struct membuf to,struct task_struct *tsk, +enum xstate_copy_mode copy_mode) +{ +__copy_xstate_to_uabi_buf (to, &tsk->thread.fpu.state.xsave, + tsk->thread.pkru, copy_mode); +} +static int copy_from_buffer (void *dst,unsigned int offset,unsigned int size,const void *kbuf,const void __user *ubuf) {
arch/x86/kernel/fpu/xstate.h +3 -0
@@ -15,4 +15,7 @@ static inline void xstate_init_xcomp_bv(struct xregs_state *xsave, u64 mask) xsave->header.xcomp_bv = mask | XCOMP_BV_COMPACTED_FORMAT; } +extern void __copy_xstate_to_uabi_buf (struct membuf to,struct xregs_state *xsave, +u32 pkru_val,enum xstate_copy_mode copy_mode); +#endif
[tip: x86/fpu] x86/fpu: Rework copy_xstate_to_uabi_buf()
The following commit has been merged into the x86/fpu branch of tip:
The following commit has been merged into the x86/fpu branch of tip:
Commit-ID: ca834defd33bae9cf9542ff92b15635a84e91946 Gitweb: https://git.kernel.org/tip/ca834defd33bae9cf9542ff92b15635a84e91946 Author: Thomas Gleixner <[email protected]> AuthorDate: Fri, 15 Oct 2021 03:16:15 +02:00 Committer: Borislav Petkov <[email protected]> CommitterDate: Wed, 20 Oct 2021 15:27:27 +02:00
x86/fpu: Rework copy_xstate_to_uabi_buf()
Prepare for replacing the KVM copy xstate to user function by extending copy_xstate_to_uabi_buf() with a pkru argument which allows the caller to hand in the pkru value, which is required for KVM because the guest PKRU is not accessible via current. Fixup all callsites accordingly.
--- arch/x86/kernel/fpu/xstate.c | 34 ++++++++++++++++++++++++++-------- arch/x86/kernel/fpu/xstate.h | 3 +++ 2 files changed, 29 insertions(+), 8 deletions(-)
arch/x86/kernel/fpu/xstate.c +26 -8
@@ -940,9 +940,10 @@ static void copy_feature(bool from_xstate, struct membuf *to, void *xstate, }/** -* copy_xstate_to_uabi_buf - Copy kernel saved xstate to a UABI buffer +* __copy_xstate_to_uabi_buf - Copy kernel saved xstate to a UABI buffer * @to: membuf descriptor -* @tsk: The task from which to copy the saved xstate +* @xsave: The xsave from which to copy +* @pkru_val: The PKRU value to store in the PKRU component * @copy_mode: The requested copy mode * * Converts from kernel XSAVE or XSAVES compacted format to UABI conforming
@@ -951,11 +952,10 @@ static void copy_feature(bool from_xstate, struct membuf *to, void *xstate,* * It supports partial copy but @to.pos always starts from zero. */ -void copy_xstate_to_uabi_buf (struct membuf to,struct task_struct *tsk, -enum xstate_copy_mode copy_mode) +void __copy_xstate_to_uabi_buf (struct membuf to,struct xregs_state *xsave, +u32 pkru_val,enum xstate_copy_mode copy_mode) {const unsigned int off_mxcsr = offsetof(struct fxregs_state , mxcsr); -struct xregs_state *xsave = &tsk->thread.fpu.state.xsave;struct xregs_state *xinit = &init_fpstate.xsave;struct xstate_header header;unsigned int zerofrom;
@@ -1033,10 +1033,9 @@ void copy_xstate_to_uabi_buf(struct membuf to, struct task_struct *tsk,struct pkru_state pkru = {0 };/* * PKRU is not necessarily up to date in the -* thread's XSAVE buffer. Fill this part from the -* per-thread storage. +* XSAVE buffer. Use the provided value. */ - pkru.pkru = tsk->thread.pkru; + pkru.pkru = pkru_val;membuf_write (&to, &pkru,sizeof (pkru)); }else { copy_feature(header.xfeatures & BIT_ULL(i), &to,
@@ -1056,6 +1055,25 @@ out:membuf_zero (&to, to.left); } +/** +* copy_xstate_to_uabi_buf - Copy kernel saved xstate to a UABI buffer +* @to: membuf descriptor +* @tsk: The task from which to copy the saved xstate +* @copy_mode: The requested copy mode +* +* Converts from kernel XSAVE or XSAVES compacted format to UABI conforming +* format, i.e. from the kernel internal hardware dependent storage format +* to the requested @mode. UABI XSTATE is always uncompacted! +* +* It supports partial copy but @to.pos always starts from zero. +*/ +void copy_xstate_to_uabi_buf (struct membuf to,struct task_struct *tsk, +enum xstate_copy_mode copy_mode) +{ +__copy_xstate_to_uabi_buf (to, &tsk->thread.fpu.state.xsave, + tsk->thread.pkru, copy_mode); +} +static int copy_from_buffer (void *dst,unsigned int offset,unsigned int size,const void *kbuf,const void __user *ubuf) {
arch/x86/kernel/fpu/xstate.h +3 -0
@@ -15,4 +15,7 @@ static inline void xstate_init_xcomp_bv(struct xregs_state *xsave, u64 mask) xsave->header.xcomp_bv = mask | XCOMP_BV_COMPACTED_FORMAT; } +extern void __copy_xstate_to_uabi_buf (struct membuf to,struct xregs_state *xsave, +u32 pkru_val,enum xstate_copy_mode copy_mode); +#endif
PATCH v2 15/30 x86/fpu: Replace KVMs home brewed FPU copy to user
In reply to Thomas GleixnerSimilar to the copy from user function the FPU core has this already implemented with all bells and whistles.
Get rid of the duplicated code and use the core functionality.
---
V2: Fix the non xsave path - Paolo
Fix changelog, comments and function name - Boris, Paolo, Sean
--- arch/x86/include/asm/fpu/api.h | 1 arch/x86/kernel/fpu/core.c | 18 +++++++++++++ arch/x86/kvm/x86.c | 56 ++--------------------------------------- 3 files changed, 22 insertions(+), 53 deletions(-)
---
arch/x86/include/asm/fpu/api.h +1 -0
@@ -117,5 +117,6 @@ extern void fpu_init_fpstate_user(structextern void fpu_swap_kvm_fpu (struct fpu *save,struct fpu *rstor,u64 restore_mask);extern int fpu_copy_kvm_uabi_to_fpstate (struct fpu *fpu,const void *buf,u64 xcr0,u32 *pkru); +extern void fpu_copy_fpstate_to_kvm_uabi (struct fpu *fpu,void *buf,unsigned int size,u32 pkru);#endif /* _ASM_X86_FPU_API_H */
arch/x86/kernel/fpu/core.c +18 -0
@@ -175,6 +175,24 @@ void fpu_swap_kvm_fpu(struct fpu *save, }EXPORT_SYMBOL_GPL (fpu_swap_kvm_fpu); +void fpu_copy_fpstate_to_kvm_uabi (struct fpu *fpu,void *buf, +unsigned int size,u32 pkru) +{ +union fpregs_state *kstate = &fpu->state; +union fpregs_state *ustate = buf; +struct membuf mb = { .p = buf, .left = size }; + +if (cpu_feature_enabled (X86_FEATURE_XSAVE )) { +__copy_xstate_to_uabi_buf (mb, &kstate->xsave, pkru, +XSTATE_COPY_XSAVE ); + }else { +memcpy (&ustate->fxsave, &kstate->fxsave,sizeof (ustate->fxsave)); +/* Make it restorable on a XSAVE enabled host */ + ustate->xsave.header.xfeatures =XFEATURE_MASK_FPSSE ; + } +} +EXPORT_SYMBOL_GPL (fpu_copy_fpstate_to_kvm_uabi); +int fpu_copy_kvm_uabi_to_fpstate (struct fpu *fpu,const void *buf,u64 xcr0,u32 *vpkru) {
arch/x86/kvm/x86.c +3 -53
68 lines not on this page · raw message
[tip: x86/fpu] x86/fpu: Replace KVMs home brewed FPU copy to user
The following commit has been merged into the x86/fpu branch of tip:
The following commit has been merged into the x86/fpu branch of tip:
Commit-ID: bf5d00470787067ff27593c6a097b5eb6e01168e Gitweb: https://git.kernel.org/tip/bf5d00470787067ff27593c6a097b5eb6e01168e Author: Thomas Gleixner <[email protected]> AuthorDate: Fri, 15 Oct 2021 03:16:17 +02:00 Committer: Borislav Petkov <[email protected]> CommitterDate: Wed, 20 Oct 2021 22:17:17 +02:00
x86/fpu: Replace KVMs home brewed FPU copy to user
Similar to the copy from user function the FPU core has this already implemented with all bells and whistles.
Get rid of the duplicated code and use the core functionality.
--- arch/x86/include/asm/fpu/api.h | 1 +- arch/x86/kernel/fpu/core.c | 18 +++++++++++- arch/x86/kvm/x86.c | 56 +-------------------------------- 3 files changed, 22 insertions(+), 53 deletions(-)
arch/x86/include/asm/fpu/api.h +1 -0
@@ -137,5 +137,6 @@ extern void fpu_init_fpstate_user(struct fpu *fpu);extern void fpu_swap_kvm_fpu (struct fpu *save,struct fpu *rstor,u64 restore_mask);extern int fpu_copy_kvm_uabi_to_fpstate (struct fpu *fpu,const void *buf,u64 xcr0,u32 *pkru); +extern void fpu_copy_fpstate_to_kvm_uabi (struct fpu *fpu,void *buf,unsigned int size,u32 pkru);#endif /* _ASM_X86_FPU_API_H */
arch/x86/kernel/fpu/core.c +18 -0
@@ -184,6 +184,24 @@ void fpu_swap_kvm_fpu(struct fpu *save, struct fpu *rstor, u64 restore_mask) }EXPORT_SYMBOL_GPL (fpu_swap_kvm_fpu); +void fpu_copy_fpstate_to_kvm_uabi (struct fpu *fpu,void *buf, +unsigned int size,u32 pkru) +{ +union fpregs_state *kstate = &fpu->state; +union fpregs_state *ustate = buf; +struct membuf mb = { .p = buf, .left = size }; + +if (cpu_feature_enabled (X86_FEATURE_XSAVE )) { +__copy_xstate_to_uabi_buf (mb, &kstate->xsave, pkru, +XSTATE_COPY_XSAVE ); + }else { +memcpy (&ustate->fxsave, &kstate->fxsave,sizeof (ustate->fxsave)); +/* Make it restorable on a XSAVE enabled host */ + ustate->xsave.header.xfeatures =XFEATURE_MASK_FPSSE ; + } +} +EXPORT_SYMBOL_GPL (fpu_copy_fpstate_to_kvm_uabi); +int fpu_copy_kvm_uabi_to_fpstate (struct fpu *fpu,const void *buf,u64 xcr0,u32 *vpkru) {
arch/x86/kvm/x86.c +3 -53
68 lines not on this page · raw message
PATCH v2 16/30 x86/fpu: Mark fpu__init_prepare_fx_sw_frame() as __init
In reply to Thomas GleixnerNo need to keep it around.
--- arch/x86/include/asm/fpu/signal.h | 2 -- arch/x86/kernel/fpu/internal.h | 8 ++++++++ arch/x86/kernel/fpu/signal.c | 4 +++- arch/x86/kernel/fpu/xstate.c | 1 + 4 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 arch/x86/kernel/fpu/internal.h
---
arch/x86/include/asm/fpu/signal.h +0 -2
@@ -31,6 +31,4 @@ fpu__alloc_mathframe(unsigned long sp, int ia32_frame,unsigned long fpu__get_fpstate_size (void ); -extern void fpu__init_prepare_fx_sw_frame (void ); -#endif /* _ASM_X86_FPU_SIGNAL_H */
arch/x86/kernel/fpu/internal.h new file +8 -0
@@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __X86_KERNEL_FPU_INTERNAL_H +#define __X86_KERNEL_FPU_INTERNAL_H + +/* Init functions */ +extern void fpu__init_prepare_fx_sw_frame (void ); + +#endif
arch/x86/kernel/fpu/signal.c +3 -1
@@ -16,6 +16,8 @@#include <asm/trapnr.h> #include <asm/trace/fpu.h> +#include "internal.h" +static struct _fpx_sw_bytes fx_sw_reserved__ro_after_init ;static struct _fpx_sw_bytes fx_sw_reserved_ia32__ro_after_init ;
@@ -514,7 +516,7 @@ unsigned long fpu__get_fpstate_size(void)* This will be saved when ever the FP and extended state context is * saved on the user stack during the signal handler delivery to the user. */ -void fpu__init_prepare_fx_sw_frame (void ) +void __init fpu__init_prepare_fx_sw_frame (void ) {int size = fpu_user_xstate_size +FP_XSTATE_MAGIC2_SIZE ;
arch/x86/kernel/fpu/xstate.c +1 -0
@@ -19,6 +19,7 @@#include <asm/tlbflush.h> +#include "internal.h" #include "xstate.h" #define for_each_extended_xfeature (bit, mask) \
[tip: x86/fpu] x86/fpu: Mark fpu__init_prepare_fx_sw_frame() as __init
The following commit has been merged into the x86/fpu branch of tip:
The following commit has been merged into the x86/fpu branch of tip:
Commit-ID: 9603445549dacd7688532a4076c377e43a3ecfce Gitweb: https://git.kernel.org/tip/9603445549dacd7688532a4076c377e43a3ecfce Author: Thomas Gleixner <[email protected]> AuthorDate: Fri, 15 Oct 2021 03:16:18 +02:00 Committer: Borislav Petkov <[email protected]> CommitterDate: Wed, 20 Oct 2021 15:27:27 +02:00
x86/fpu: Mark fpu__init_prepare_fx_sw_frame() as __init
No need to keep it around.
--- arch/x86/include/asm/fpu/signal.h | 2 -- arch/x86/kernel/fpu/internal.h | 8 ++++++++ arch/x86/kernel/fpu/signal.c | 4 +++- arch/x86/kernel/fpu/xstate.c | 1 + 4 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 arch/x86/kernel/fpu/internal.h
arch/x86/include/asm/fpu/signal.h +0 -2
@@ -31,6 +31,4 @@ fpu__alloc_mathframe(unsigned long sp, int ia32_frame,unsigned long fpu__get_fpstate_size (void ); -extern void fpu__init_prepare_fx_sw_frame (void ); -#endif /* _ASM_X86_FPU_SIGNAL_H */
arch/x86/kernel/fpu/internal.h new file +8 -0
@@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __X86_KERNEL_FPU_INTERNAL_H +#define __X86_KERNEL_FPU_INTERNAL_H + +/* Init functions */ +extern void fpu__init_prepare_fx_sw_frame (void ); + +#endif
arch/x86/kernel/fpu/signal.c +3 -1
@@ -16,6 +16,8 @@#include <asm/trapnr.h> #include <asm/trace/fpu.h> +#include "internal.h" +static struct _fpx_sw_bytes fx_sw_reserved__ro_after_init ;static struct _fpx_sw_bytes fx_sw_reserved_ia32__ro_after_init ;
@@ -514,7 +516,7 @@ unsigned long fpu__get_fpstate_size(void)* This will be saved when ever the FP and extended state context is * saved on the user stack during the signal handler delivery to the user. */ -void fpu__init_prepare_fx_sw_frame (void ) +void __init fpu__init_prepare_fx_sw_frame (void ) {int size = fpu_user_xstate_size +FP_XSTATE_MAGIC2_SIZE ;
arch/x86/kernel/fpu/xstate.c +1 -0
@@ -19,6 +19,7 @@#include <asm/tlbflush.h> +#include "internal.h" #include "xstate.h" #define for_each_extended_xfeature (bit, mask) \
PATCH v2 17/30 x86/fpu: Move context switch and exit to user inlines into sched.h
In reply to Thomas Gleixnerinternal.h is a kitchen sink which needs to get out of the way to prepare for the upcoming changes.
Move the context switch and exit to user inlines into a separate header, which is all that code needs.
--- arch/x86/include/asm/fpu/internal.h | 60 +---------------------------------- arch/x86/include/asm/fpu/sched.h | 68 ++++++++++++++++++++++++++++++++++++++- arch/x86/kernel/fpu/core.c | 1 +- arch/x86/kernel/process.c | 2 +- arch/x86/kernel/process_32.c | 2 +- arch/x86/kernel/process_64.c | 2 +- 6 files changed, 72 insertions(+), 63 deletions(-) create mode 100644 arch/x86/include/asm/fpu/sched.h
---
arch/x86/include/asm/fpu/internal.h +0 -60
@@ -27,16 +27,11 @@* High level FPU state handling functions: */ extern bool fpu__restore_sig (void __user *buf,int ia32_frame); -extern void fpu__drop (struct fpu *fpu);extern void fpu__clear_user_states (struct fpu *fpu);extern int fpu__exception_code (struct fpu *fpu,int trap_nr);extern void fpu_sync_fpstate (struct fpu *fpu); -/* Clone and exit operations */ -extern int fpu_clone (struct task_struct *dst); -extern void fpu_flush_thread (void ); -/* * Boot time FPU initialization functions: */
@@ -82,7 +77,6 @@ extern void fpstate_init_soft(struct swregs_state *soft);#else static inline void fpstate_init_soft (struct swregs_state *soft) {}#endif -extern void save_fpregs_to_fpstate (struct fpu *fpu);/* * Returns 0 on success or the trap number when the operation raises an
@@ -464,58 +458,4 @@ static inline void fpregs_restore_userregs(void) clear_thread_flag(TIF_NEED_FPU_LOAD); } -/* -* FPU state switching for scheduling. -* -* This is a two-stage process: -* -* - switch_fpu_prepare() saves the old state. -* This is done within the context of the old process. -* -* - switch_fpu_finish() sets TIF_NEED_FPU_LOAD; the floating point state -* will get loaded on return to userspace, or when the kernel needs it. -* -* If TIF_NEED_FPU_LOAD is cleared then the CPU's FPU registers -* are saved in the current thread's FPU register state. -* -* If TIF_NEED_FPU_LOAD is set then CPU's FPU registers may not -* hold current()'s FPU registers. It is required to load the -* registers before returning to userland or using the content -* otherwise. -* -* The FPU context is only stored/restored for a user task and -* PF_KTHREAD is used to distinguish between kernel and user threads. -*/ -static inline void switch_fpu_prepare (struct fpu *old_fpu,int cpu) -{ -if (static_cpu_has (X86_FEATURE_FPU ) && !(current->flags &PF_KTHREAD )) { -save_fpregs_to_fpstate (old_fpu); -/* -* The save operation preserved register state, so the -* fpu_fpregs_owner_ctx is still @old_fpu. Store the -* current CPU number in @old_fpu, so the next return -* to user space can avoid the FPU register restore -* when is returns on the same CPU and still owns the -* context. -*/ - old_fpu->last_cpu = cpu; - -trace_x86_fpu_regs_deactivated (old_fpu); - } -} - -/* -* Misc helper functions: -*/ - -/* -* Delay loading of the complete FPU state until the return to userland. -* PKRU is handled separately. -*/ -static inline void switch_fpu_finish (void ) -{ -if (cpu_feature_enabled (X86_FEATURE_FPU )) -set_thread_flag (TIF_NEED_FPU_LOAD ); -} -#endif /* _ASM_X86_FPU_INTERNAL_H */
arch/x86/include/asm/fpu/sched.h new file +68 -0
68 lines not on this page · raw message
arch/x86/kernel/fpu/core.c +1 -0
@@ -8,6 +8,7 @@*/ #include <asm/fpu/internal.h> #include <asm/fpu/regset.h> +#include <asm/fpu/sched.h> #include <asm/fpu/signal.h> #include <asm/fpu/types.h> #include <asm/traps.h>
arch/x86/kernel/process.c +1 -1
@@ -30,7 +30,7 @@#include <asm/apic.h> #include <linux/uaccess.h> #include <asm/mwait.h> -#include <asm/fpu/internal.h> +#include <asm/fpu/sched.h> #include <asm/debugreg.h> #include <asm/nmi.h> #include <asm/tlbflush.h>
arch/x86/kernel/process_32.c +1 -1
@@ -41,7 +41,7 @@#include <asm/ldt.h> #include <asm/processor.h> -#include <asm/fpu/internal.h> +#include <asm/fpu/sched.h> #include <asm/desc.h> #include <linux/err.h>
arch/x86/kernel/process_64.c +1 -1
@@ -42,7 +42,7 @@#include <asm/processor.h> #include <asm/pkru.h> -#include <asm/fpu/internal.h> +#include <asm/fpu/sched.h> #include <asm/mmu_context.h> #include <asm/prctl.h> #include <asm/desc.h>
[tip: x86/fpu] x86/fpu: Move context switch and exit to user inlines into sched.h
The following commit has been merged into the x86/fpu branch of tip:
The following commit has been merged into the x86/fpu branch of tip:
Commit-ID: 63e81807c1f94e91b9d71c536112a40cd74bab85 Gitweb: https://git.kernel.org/tip/63e81807c1f94e91b9d71c536112a40cd74bab85 Author: Thomas Gleixner <[email protected]> AuthorDate: Fri, 15 Oct 2021 03:16:20 +02:00 Committer: Borislav Petkov <[email protected]> CommitterDate: Wed, 20 Oct 2021 15:27:27 +02:00
x86/fpu: Move context switch and exit to user inlines into sched.h
internal.h is a kitchen sink which needs to get out of the way to prepare for the upcoming changes.
Move the context switch and exit to user inlines into a separate header, which is all that code needs.
--- arch/x86/include/asm/fpu/internal.h | 60 +------------------------- arch/x86/include/asm/fpu/sched.h | 68 ++++++++++++++++++++++++++++- arch/x86/kernel/fpu/core.c | 1 +- arch/x86/kernel/process.c | 2 +- arch/x86/kernel/process_32.c | 2 +- arch/x86/kernel/process_64.c | 2 +- 6 files changed, 72 insertions(+), 63 deletions(-) create mode 100644 arch/x86/include/asm/fpu/sched.h
arch/x86/include/asm/fpu/internal.h +0 -60
@@ -27,16 +27,11 @@* High level FPU state handling functions: */ extern bool fpu__restore_sig (void __user *buf,int ia32_frame); -extern void fpu__drop (struct fpu *fpu);extern void fpu__clear_user_states (struct fpu *fpu);extern int fpu__exception_code (struct fpu *fpu,int trap_nr);extern void fpu_sync_fpstate (struct fpu *fpu); -/* Clone and exit operations */ -extern int fpu_clone (struct task_struct *dst); -extern void fpu_flush_thread (void ); -/* * Boot time FPU initialization functions: */
@@ -82,7 +77,6 @@ extern void fpstate_init_soft(struct swregs_state *soft);#else static inline void fpstate_init_soft (struct swregs_state *soft) {}#endif -extern void save_fpregs_to_fpstate (struct fpu *fpu);/* * Returns 0 on success or the trap number when the operation raises an
@@ -464,58 +458,4 @@ static inline void fpregs_restore_userregs(void) clear_thread_flag(TIF_NEED_FPU_LOAD); } -/* -* FPU state switching for scheduling. -* -* This is a two-stage process: -* -* - switch_fpu_prepare() saves the old state. -* This is done within the context of the old process. -* -* - switch_fpu_finish() sets TIF_NEED_FPU_LOAD; the floating point state -* will get loaded on return to userspace, or when the kernel needs it. -* -* If TIF_NEED_FPU_LOAD is cleared then the CPU's FPU registers -* are saved in the current thread's FPU register state. -* -* If TIF_NEED_FPU_LOAD is set then CPU's FPU registers may not -* hold current()'s FPU registers. It is required to load the -* registers before returning to userland or using the content -* otherwise. -* -* The FPU context is only stored/restored for a user task and -* PF_KTHREAD is used to distinguish between kernel and user threads. -*/ -static inline void switch_fpu_prepare (struct fpu *old_fpu,int cpu) -{ -if (static_cpu_has (X86_FEATURE_FPU ) && !(current->flags &PF_KTHREAD )) { -save_fpregs_to_fpstate (old_fpu); -/* -* The save operation preserved register state, so the -* fpu_fpregs_owner_ctx is still @old_fpu. Store the -* current CPU number in @old_fpu, so the next return -* to user space can avoid the FPU register restore -* when is returns on the same CPU and still owns the -* context. -*/ - old_fpu->last_cpu = cpu; - -trace_x86_fpu_regs_deactivated (old_fpu); - } -} - -/* -* Misc helper functions: -*/ - -/* -* Delay loading of the complete FPU state until the return to userland. -* PKRU is handled separately. -*/ -static inline void switch_fpu_finish (void ) -{ -if (cpu_feature_enabled (X86_FEATURE_FPU )) -set_thread_flag (TIF_NEED_FPU_LOAD ); -} -#endif /* _ASM_X86_FPU_INTERNAL_H */
arch/x86/include/asm/fpu/sched.h new file +68 -0
68 lines not on this page · raw message
arch/x86/kernel/fpu/core.c +1 -0
@@ -8,6 +8,7 @@*/ #include <asm/fpu/internal.h> #include <asm/fpu/regset.h> +#include <asm/fpu/sched.h> #include <asm/fpu/signal.h> #include <asm/fpu/types.h> #include <asm/traps.h>
arch/x86/kernel/process.c +1 -1
@@ -30,7 +30,7 @@#include <asm/apic.h> #include <linux/uaccess.h> #include <asm/mwait.h> -#include <asm/fpu/internal.h> +#include <asm/fpu/sched.h> #include <asm/debugreg.h> #include <asm/nmi.h> #include <asm/tlbflush.h>
arch/x86/kernel/process_32.c +1 -1
@@ -41,7 +41,7 @@#include <asm/ldt.h> #include <asm/processor.h> -#include <asm/fpu/internal.h> +#include <asm/fpu/sched.h> #include <asm/desc.h> #include <linux/err.h>
arch/x86/kernel/process_64.c +1 -1
@@ -42,7 +42,7 @@#include <asm/processor.h> #include <asm/pkru.h> -#include <asm/fpu/internal.h> +#include <asm/fpu/sched.h> #include <asm/mmu_context.h> #include <asm/prctl.h> #include <asm/desc.h>
PATCH v2 18/30 x86/fpu: Clean up cpu feature tests
In reply to Thomas GleixnerFurther disintegration of internal.h:
Move the cpu feature tests to a core header and remove the unused one.
--- arch/x86/include/asm/fpu/internal.h | 18 ------------------ arch/x86/kernel/fpu/core.c | 1 + arch/x86/kernel/fpu/internal.h | 11 +++++++++++ arch/x86/kernel/fpu/regset.c | 2 ++ 4 files changed, 14 insertions(+), 18 deletions(-)
---
arch/x86/include/asm/fpu/internal.h +0 -18
@@ -51,24 +51,6 @@ extern void fpu__resume_cpu(void);# define WARN_ON_FPU (x) ({ (void)(x); 0; })#endif -/* -* FPU related CPU feature flag helper routines: -*/ -static __always_inline __pure bool use_xsaveopt(void) -{ -return static_cpu_has (X86_FEATURE_XSAVEOPT ); -} - -static __always_inline __pure bool use_xsave(void) -{ -return static_cpu_has (X86_FEATURE_XSAVE ); -} - -static __always_inline __pure bool use_fxsr(void) -{ -return static_cpu_has (X86_FEATURE_FXSR ); -} -extern union fpregs_state init_fpstate;extern void fpstate_init_user (union fpregs_state *state);
arch/x86/kernel/fpu/core.c +1 -0
@@ -17,6 +17,7 @@#include <linux/hardirq.h> #include <linux/pkeys.h> +#include "internal.h" #include "xstate.h" #define CREATE_TRACE_POINTS
arch/x86/kernel/fpu/internal.h +11 -0
@@ -2,6 +2,17 @@#ifndef __X86_KERNEL_FPU_INTERNAL_H#define __X86_KERNEL_FPU_INTERNAL_H +/* CPU feature check wrappers */ +static __always_inline __pure bool use_xsave(void) +{ +return cpu_feature_enabled (X86_FEATURE_XSAVE ); +} + +static __always_inline __pure bool use_fxsr(void) +{ +return cpu_feature_enabled (X86_FEATURE_FXSR ); +} +/* Init functions */ extern void fpu__init_prepare_fx_sw_frame (void );
arch/x86/kernel/fpu/regset.c +2 -0
@@ -10,6 +10,8 @@#include <asm/fpu/regset.h> #include <asm/fpu/xstate.h> +#include "internal.h" +/* * The xstateregs_active() routine is the same as the regset_fpregs_active() routine, * as the "regset->n" for the xstate regset will be updated based on the feature
[tip: x86/fpu] x86/fpu: Clean up CPU feature tests
The following commit has been merged into the x86/fpu branch of tip:
The following commit has been merged into the x86/fpu branch of tip:
Commit-ID: d06241f52cfe4a0580856ef2cfac90dc7f752cae Gitweb: https://git.kernel.org/tip/d06241f52cfe4a0580856ef2cfac90dc7f752cae Author: Thomas Gleixner <[email protected]> AuthorDate: Fri, 15 Oct 2021 03:16:21 +02:00 Committer: Borislav Petkov <[email protected]> CommitterDate: Wed, 20 Oct 2021 15:27:27 +02:00
x86/fpu: Clean up CPU feature tests
Further disintegration of internal.h:
Move the CPU feature tests to a core header and remove the unused one.
--- arch/x86/include/asm/fpu/internal.h | 18 ------------------ arch/x86/kernel/fpu/core.c | 1 + arch/x86/kernel/fpu/internal.h | 11 +++++++++++ arch/x86/kernel/fpu/regset.c | 2 ++ 4 files changed, 14 insertions(+), 18 deletions(-)
arch/x86/include/asm/fpu/internal.h +0 -18
@@ -51,24 +51,6 @@ extern void fpu__resume_cpu(void);# define WARN_ON_FPU (x) ({ (void)(x); 0; })#endif -/* -* FPU related CPU feature flag helper routines: -*/ -static __always_inline __pure bool use_xsaveopt(void) -{ -return static_cpu_has (X86_FEATURE_XSAVEOPT ); -} - -static __always_inline __pure bool use_xsave(void) -{ -return static_cpu_has (X86_FEATURE_XSAVE ); -} - -static __always_inline __pure bool use_fxsr(void) -{ -return static_cpu_has (X86_FEATURE_FXSR ); -} -extern union fpregs_state init_fpstate;extern void fpstate_init_user (union fpregs_state *state);
arch/x86/kernel/fpu/core.c +1 -0
@@ -17,6 +17,7 @@#include <linux/hardirq.h> #include <linux/pkeys.h> +#include "internal.h" #include "xstate.h" #define CREATE_TRACE_POINTS
arch/x86/kernel/fpu/internal.h +11 -0
@@ -2,6 +2,17 @@#ifndef __X86_KERNEL_FPU_INTERNAL_H#define __X86_KERNEL_FPU_INTERNAL_H +/* CPU feature check wrappers */ +static __always_inline __pure bool use_xsave(void) +{ +return cpu_feature_enabled (X86_FEATURE_XSAVE ); +} + +static __always_inline __pure bool use_fxsr(void) +{ +return cpu_feature_enabled (X86_FEATURE_FXSR ); +} +/* Init functions */ extern void fpu__init_prepare_fx_sw_frame (void );
arch/x86/kernel/fpu/regset.c +2 -0
@@ -10,6 +10,8 @@#include <asm/fpu/regset.h> #include <asm/fpu/xstate.h> +#include "internal.h" +/* * The xstateregs_active() routine is the same as the regset_fpregs_active() routine, * as the "regset->n" for the xstate regset will be updated based on the feature
PATCH v2 19/30 x86/fpu: Make os_xrstor_booting() private
In reply to Thomas GleixnerIt's only required in the xstate init code.
--- arch/x86/include/asm/fpu/internal.h | 25 ------------------------- arch/x86/kernel/fpu/xstate.c | 23 +++++++++++++++++++++++ 2 files changed, 23 insertions(+), 25 deletions(-)
---
arch/x86/include/asm/fpu/internal.h +0 -25
@@ -226,31 +226,6 @@ static inline void fxsave(struct fxregs_state *fx) : "memory")/* -* This function is called only during boot time when x86 caps are not set -* up and alternative can not be used yet. -*/ -static inline void os_xrstor_booting (struct xregs_state *xstate) -{ -u64 mask =xfeatures_mask_fpstate (); -u32 lmask = mask; -u32 hmask = mask >>32 ; -int err; - -WARN_ON (system_state !=SYSTEM_BOOTING ); - -if (boot_cpu_has (X86_FEATURE_XSAVES )) -XSTATE_OP (XRSTORS , xstate, lmask, hmask, err); -else -XSTATE_OP (XRSTOR , xstate, lmask, hmask, err); - -/* -* We should never fault when copying from a kernel buffer, and the FPU -* state we set at boot time should be valid. -*/ -WARN_ON_FPU (err); -} - -/* * Save processor xstate to xsave area. * * Uses either XSAVE or XSAVEOPT or XSAVES depending on the CPU features
arch/x86/kernel/fpu/xstate.c +23 -0
@@ -351,6 +351,29 @@ static void __init print_xstate_offset_size(void) }/* +* This function is called only during boot time when x86 caps are not set +* up and alternative can not be used yet. +*/ +static __init void os_xrstor_booting (struct xregs_state *xstate) +{ +u64 mask =xfeatures_mask_fpstate (); +u32 lmask = mask; +u32 hmask = mask >>32 ; +int err; + +if (cpu_feature_enabled (X86_FEATURE_XSAVES )) +XSTATE_OP (XRSTORS , xstate, lmask, hmask, err); +else +XSTATE_OP (XRSTOR , xstate, lmask, hmask, err); + +/* +* We should never fault when copying from a kernel buffer, and the FPU +* state we set at boot time should be valid. +*/ +WARN_ON_FPU (err); +} + +/* * All supported features have either init state all zeros or are * handled in setup_init_fpu() individually. This is an explicit * feature list and does not use XFEATURE_MASK*SUPPORTED to catch
[tip: x86/fpu] x86/fpu: Make os_xrstor_booting() private
The following commit has been merged into the x86/fpu branch of tip:
The following commit has been merged into the x86/fpu branch of tip:
Commit-ID: b579d0c3750eedc0dee433edaba88206a8e4348a Gitweb: https://git.kernel.org/tip/b579d0c3750eedc0dee433edaba88206a8e4348a Author: Thomas Gleixner <[email protected]> AuthorDate: Fri, 15 Oct 2021 03:16:23 +02:00 Committer: Borislav Petkov <[email protected]> CommitterDate: Wed, 20 Oct 2021 15:27:28 +02:00
x86/fpu: Make os_xrstor_booting() private
It's only required in the xstate init code.
--- arch/x86/include/asm/fpu/internal.h | 25 ------------------------- arch/x86/kernel/fpu/xstate.c | 23 +++++++++++++++++++++++ 2 files changed, 23 insertions(+), 25 deletions(-)
arch/x86/include/asm/fpu/internal.h +0 -25
@@ -226,31 +226,6 @@ static inline void fxsave(struct fxregs_state *fx) : "memory")/* -* This function is called only during boot time when x86 caps are not set -* up and alternative can not be used yet. -*/ -static inline void os_xrstor_booting (struct xregs_state *xstate) -{ -u64 mask =xfeatures_mask_fpstate (); -u32 lmask = mask; -u32 hmask = mask >>32 ; -int err; - -WARN_ON (system_state !=SYSTEM_BOOTING ); - -if (boot_cpu_has (X86_FEATURE_XSAVES )) -XSTATE_OP (XRSTORS , xstate, lmask, hmask, err); -else -XSTATE_OP (XRSTOR , xstate, lmask, hmask, err); - -/* -* We should never fault when copying from a kernel buffer, and the FPU -* state we set at boot time should be valid. -*/ -WARN_ON_FPU (err); -} - -/* * Save processor xstate to xsave area. * * Uses either XSAVE or XSAVEOPT or XSAVES depending on the CPU features
arch/x86/kernel/fpu/xstate.c +23 -0
@@ -351,6 +351,29 @@ static void __init print_xstate_offset_size(void) }/* +* This function is called only during boot time when x86 caps are not set +* up and alternative can not be used yet. +*/ +static __init void os_xrstor_booting (struct xregs_state *xstate) +{ +u64 mask =xfeatures_mask_fpstate (); +u32 lmask = mask; +u32 hmask = mask >>32 ; +int err; + +if (cpu_feature_enabled (X86_FEATURE_XSAVES )) +XSTATE_OP (XRSTORS , xstate, lmask, hmask, err); +else +XSTATE_OP (XRSTOR , xstate, lmask, hmask, err); + +/* +* We should never fault when copying from a kernel buffer, and the FPU +* state we set at boot time should be valid. +*/ +WARN_ON_FPU (err); +} + +/* * All supported features have either init state all zeros or are * handled in setup_init_fpu() individually. This is an explicit * feature list and does not use XFEATURE_MASK*SUPPORTED to catch
PATCH v2 20/30 x86/fpu: Move os_xsave() and os_xrstor() to core
In reply to Thomas GleixnerNothing outside the core code needs these.
--- arch/x86/include/asm/fpu/internal.h | 165 +----------------------------------- arch/x86/include/asm/fpu/xstate.h | 6 +- arch/x86/kernel/fpu/signal.c | 1 +- arch/x86/kernel/fpu/xstate.h | 174 +++++++++++++++++++++++++++++++++++++- 4 files changed, 175 insertions(+), 171 deletions(-)
---
arch/x86/include/asm/fpu/internal.h +0 -165
@@ -161,171 +161,6 @@ static inline void fxsave(struct fxregs_state *fx) asmvolatile ("fxsaveq %[fx]" : [fx]"=m" (*fx)); } -/* These macros all use (%edi)/(%rdi) as the single memory argument. */ -#define XSAVE ".byte " REX_PREFIX "0x0f,0xae,0x27" -#define XSAVEOPT ".byte " REX_PREFIX "0x0f,0xae,0x37" -#define XSAVES ".byte " REX_PREFIX "0x0f,0xc7,0x2f" -#define XRSTOR ".byte " REX_PREFIX "0x0f,0xae,0x2f" -#define XRSTORS ".byte " REX_PREFIX "0x0f,0xc7,0x1f" - -/* -* After this @err contains 0 on success or the trap number when the -* operation raises an exception. -*/ -#define XSTATE_OP (op, st, lmask, hmask, err) \ - asm volatile("1:" op "\n\t" \ - "xor %[err], %[err]\n" \ - "2:\n\t" \ - _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_FAULT_MCE_SAFE) \ - : [err] "=a" (err) \ - : "D" (st), "m" (*st), "a" (lmask), "d" (hmask) \ - : "memory") - -/* -* If XSAVES is enabled, it replaces XSAVEOPT because it supports a compact -* format and supervisor states in addition to modified optimization in -* XSAVEOPT. -* -* Otherwise, if XSAVEOPT is enabled, XSAVEOPT replaces XSAVE because XSAVEOPT -* supports modified optimization which is not supported by XSAVE. -* -* We use XSAVE as a fallback. -* -* The 661 label is defined in the ALTERNATIVE* macros as the address of the -* original instruction which gets replaced. We need to use it here as the -* address of the instruction where we might get an exception at. -*/ -#define XSTATE_XSAVE (st, lmask, hmask, err) \ - asm volatile(ALTERNATIVE_2(XSAVE, \ - XSAVEOPT, X86_FEATURE_XSAVEOPT, \ - XSAVES, X86_FEATURE_XSAVES) \ - "\n" \ - "xor %[err], %[err]\n" \ - "3:\n" \ - ".pushsection .fixup,\"ax\"\n" \ - "4: movl $-2, %[err]\n" \ - "jmp 3b\n" \ - ".popsection\n" \ - _ASM_EXTABLE(661b, 4b) \ - : [err] "=r" (err) \ - : "D" (st), "m" (*st), "a" (lmask), "d" (hmask) \ - : "memory") - -/* -* Use XRSTORS to restore context if it is enabled. XRSTORS supports compact -* XSAVE area format. -*/ -#define XSTATE_XRESTORE (st, lmask, hmask) \ - asm volatile(ALTERNATIVE(XRSTOR, \ - XRSTORS, X86_FEATURE_XSAVES) \ - "\n" \ - "3:\n" \ - _ASM_EXTABLE_TYPE(661b, 3b, EX_TYPE_FPU_RESTORE) \ - : \ - : "D" (st), "m" (*st), "a" (lmask), "d" (hmask) \ - : "memory") - -/* -* Save processor xstate to xsave area. -* -* Uses either XSAVE or XSAVEOPT or XSAVES depending on the CPU features -* and command line options. The choice is permanent until the next reboot. -*/ -static inline void os_xsave (struct xregs_state *xstate) -{ -u64 mask = xfeatures_mask_all; -u32 lmask = mask; -u32 hmask = mask >>32 ; -int err; - -WARN_ON_FPU (!alternatives_patched); - -XSTATE_XSAVE (xstate, lmask, hmask, err); - -/* We should never fault when copying to a kernel buffer: */ -WARN_ON_FPU (err); -} - -/* -* Restore processor xstate from xsave area. -* -* Uses XRSTORS when XSAVES is used, XRSTOR otherwise. -*/ -static inline void os_xrstor (struct xregs_state *xstate,u64 mask) -{ -u32 lmask = mask; -u32 hmask = mask >>32 ; - -XSTATE_XRESTORE (xstate, lmask, hmask); -} - -/* -* Save xstate to user space xsave area. -* -* We don't use modified optimization because xrstor/xrstors might track -* a different application. -* -* We don't use compacted format xsave area for backward compatibility for -* old applications which don't understand the compacted format of the -* xsave area. -* -* The caller has to zero buf::header before calling this because XSAVE* -* does not touch the reserved fields in the header. -*/ -static inline int xsave_to_user_sigframe (struct xregs_state __user *buf) -{ -/* -* Include the features which are not xsaved/rstored by the kernel -* internally, e.g. PKRU. That's user space ABI and also required -* to allow the signal handler to modify PKRU. -*/ -u64 mask =xfeatures_mask_uabi (); -u32 lmask = mask; -u32 hmask = mask >>32 ; -int err; - -stac (); -XSTATE_OP (XSAVE , buf, lmask, hmask, err); -clac (); - -return err; -} - -/* -* Restore xstate from user space xsave area. -*/ -static inline int xrstor_from_user_sigframe (struct xregs_state __user *buf,u64 mask) -{ -struct xregs_state *xstate = ((__force struct xregs_state *)buf); -u32 lmask = mask; -u32 hmask = mask >>32 ; -int err; - -stac (); -XSTATE_OP (XRSTOR , xstate, lmask, hmask, err); -clac (); - -return err; -} - -/* -* Restore xstate from kernel space xsave area, return an error code instead of -* an exception. -*/ -static inline int os_xrstor_safe (struct xregs_state *xstate,u64 mask) -{ -u32 lmask = mask; -u32 hmask = mask >>32 ; -int err; - -if (cpu_feature_enabled (X86_FEATURE_XSAVES )) -XSTATE_OP (XRSTORS , xstate, lmask, hmask, err); -else -XSTATE_OP (XRSTOR , xstate, lmask, hmask, err); - -return err; -} -extern void restore_fpregs_from_fpstate (union fpregs_state *fpstate,u64 mask);extern bool copy_fpstate_to_sigframe (void __user *buf,void __user *fp,int size);
arch/x86/include/asm/fpu/xstate.h +0 -6
@@ -78,12 +78,6 @@ XFEATURE_MASK_INDEPENDENT | \ XFEATURE_MASK_SUPERVISOR_UNSUPPORTED) -#ifdef CONFIG_X86_64 -#define REX_PREFIX "0x48, " -#else -#define REX_PREFIX -#endif -extern u64 xfeatures_mask_all;static inline u64 xfeatures_mask_supervisor (void )
arch/x86/kernel/fpu/signal.c +1 -0
@@ -17,6 +17,7 @@#include <asm/trace/fpu.h> #include "internal.h" +#include "xstate.h" static struct _fpx_sw_bytes fx_sw_reserved__ro_after_init ;static struct _fpx_sw_bytes fx_sw_reserved_ia32__ro_after_init ;
arch/x86/kernel/fpu/xstate.h +174 -0
178 lines not on this page · raw message
[tip: x86/fpu] x86/fpu: Move os_xsave() and os_xrstor() to core
The following commit has been merged into the x86/fpu branch of tip:
The following commit has been merged into the x86/fpu branch of tip:
Commit-ID: df95b0f1aa56dfa71a0ef657e3e62294ee6d9034 Gitweb: https://git.kernel.org/tip/df95b0f1aa56dfa71a0ef657e3e62294ee6d9034 Author: Thomas Gleixner <[email protected]> AuthorDate: Fri, 15 Oct 2021 03:16:24 +02:00 Committer: Borislav Petkov <[email protected]> CommitterDate: Wed, 20 Oct 2021 15:27:28 +02:00
x86/fpu: Move os_xsave() and os_xrstor() to core
Nothing outside the core code needs these.
--- arch/x86/include/asm/fpu/internal.h | 165 +-------------------------- arch/x86/include/asm/fpu/xstate.h | 6 +- arch/x86/kernel/fpu/signal.c | 1 +- arch/x86/kernel/fpu/xstate.h | 174 +++++++++++++++++++++++++++- 4 files changed, 175 insertions(+), 171 deletions(-)
arch/x86/include/asm/fpu/internal.h +0 -165
@@ -161,171 +161,6 @@ static inline void fxsave(struct fxregs_state *fx) asmvolatile ("fxsaveq %[fx]" : [fx]"=m" (*fx)); } -/* These macros all use (%edi)/(%rdi) as the single memory argument. */ -#define XSAVE ".byte " REX_PREFIX "0x0f,0xae,0x27" -#define XSAVEOPT ".byte " REX_PREFIX "0x0f,0xae,0x37" -#define XSAVES ".byte " REX_PREFIX "0x0f,0xc7,0x2f" -#define XRSTOR ".byte " REX_PREFIX "0x0f,0xae,0x2f" -#define XRSTORS ".byte " REX_PREFIX "0x0f,0xc7,0x1f" - -/* -* After this @err contains 0 on success or the trap number when the -* operation raises an exception. -*/ -#define XSTATE_OP (op, st, lmask, hmask, err) \ - asm volatile("1:" op "\n\t" \ - "xor %[err], %[err]\n" \ - "2:\n\t" \ - _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_FAULT_MCE_SAFE) \ - : [err] "=a" (err) \ - : "D" (st), "m" (*st), "a" (lmask), "d" (hmask) \ - : "memory") - -/* -* If XSAVES is enabled, it replaces XSAVEOPT because it supports a compact -* format and supervisor states in addition to modified optimization in -* XSAVEOPT. -* -* Otherwise, if XSAVEOPT is enabled, XSAVEOPT replaces XSAVE because XSAVEOPT -* supports modified optimization which is not supported by XSAVE. -* -* We use XSAVE as a fallback. -* -* The 661 label is defined in the ALTERNATIVE* macros as the address of the -* original instruction which gets replaced. We need to use it here as the -* address of the instruction where we might get an exception at. -*/ -#define XSTATE_XSAVE (st, lmask, hmask, err) \ - asm volatile(ALTERNATIVE_2(XSAVE, \ - XSAVEOPT, X86_FEATURE_XSAVEOPT, \ - XSAVES, X86_FEATURE_XSAVES) \ - "\n" \ - "xor %[err], %[err]\n" \ - "3:\n" \ - ".pushsection .fixup,\"ax\"\n" \ - "4: movl $-2, %[err]\n" \ - "jmp 3b\n" \ - ".popsection\n" \ - _ASM_EXTABLE(661b, 4b) \ - : [err] "=r" (err) \ - : "D" (st), "m" (*st), "a" (lmask), "d" (hmask) \ - : "memory") - -/* -* Use XRSTORS to restore context if it is enabled. XRSTORS supports compact -* XSAVE area format. -*/ -#define XSTATE_XRESTORE (st, lmask, hmask) \ - asm volatile(ALTERNATIVE(XRSTOR, \ - XRSTORS, X86_FEATURE_XSAVES) \ - "\n" \ - "3:\n" \ - _ASM_EXTABLE_TYPE(661b, 3b, EX_TYPE_FPU_RESTORE) \ - : \ - : "D" (st), "m" (*st), "a" (lmask), "d" (hmask) \ - : "memory") - -/* -* Save processor xstate to xsave area. -* -* Uses either XSAVE or XSAVEOPT or XSAVES depending on the CPU features -* and command line options. The choice is permanent until the next reboot. -*/ -static inline void os_xsave (struct xregs_state *xstate) -{ -u64 mask = xfeatures_mask_all; -u32 lmask = mask; -u32 hmask = mask >>32 ; -int err; - -WARN_ON_FPU (!alternatives_patched); - -XSTATE_XSAVE (xstate, lmask, hmask, err); - -/* We should never fault when copying to a kernel buffer: */ -WARN_ON_FPU (err); -} - -/* -* Restore processor xstate from xsave area. -* -* Uses XRSTORS when XSAVES is used, XRSTOR otherwise. -*/ -static inline void os_xrstor (struct xregs_state *xstate,u64 mask) -{ -u32 lmask = mask; -u32 hmask = mask >>32 ; - -XSTATE_XRESTORE (xstate, lmask, hmask); -} - -/* -* Save xstate to user space xsave area. -* -* We don't use modified optimization because xrstor/xrstors might track -* a different application. -* -* We don't use compacted format xsave area for backward compatibility for -* old applications which don't understand the compacted format of the -* xsave area. -* -* The caller has to zero buf::header before calling this because XSAVE* -* does not touch the reserved fields in the header. -*/ -static inline int xsave_to_user_sigframe (struct xregs_state __user *buf) -{ -/* -* Include the features which are not xsaved/rstored by the kernel -* internally, e.g. PKRU. That's user space ABI and also required -* to allow the signal handler to modify PKRU. -*/ -u64 mask =xfeatures_mask_uabi (); -u32 lmask = mask; -u32 hmask = mask >>32 ; -int err; - -stac (); -XSTATE_OP (XSAVE , buf, lmask, hmask, err); -clac (); - -return err; -} - -/* -* Restore xstate from user space xsave area. -*/ -static inline int xrstor_from_user_sigframe (struct xregs_state __user *buf,u64 mask) -{ -struct xregs_state *xstate = ((__force struct xregs_state *)buf); -u32 lmask = mask; -u32 hmask = mask >>32 ; -int err; - -stac (); -XSTATE_OP (XRSTOR , xstate, lmask, hmask, err); -clac (); - -return err; -} - -/* -* Restore xstate from kernel space xsave area, return an error code instead of -* an exception. -*/ -static inline int os_xrstor_safe (struct xregs_state *xstate,u64 mask) -{ -u32 lmask = mask; -u32 hmask = mask >>32 ; -int err; - -if (cpu_feature_enabled (X86_FEATURE_XSAVES )) -XSTATE_OP (XRSTORS , xstate, lmask, hmask, err); -else -XSTATE_OP (XRSTOR , xstate, lmask, hmask, err); - -return err; -} -extern void restore_fpregs_from_fpstate (union fpregs_state *fpstate,u64 mask);extern bool copy_fpstate_to_sigframe (void __user *buf,void __user *fp,int size);
arch/x86/include/asm/fpu/xstate.h +0 -6
@@ -78,12 +78,6 @@ XFEATURE_MASK_INDEPENDENT | \ XFEATURE_MASK_SUPERVISOR_UNSUPPORTED) -#ifdef CONFIG_X86_64 -#define REX_PREFIX "0x48, " -#else -#define REX_PREFIX -#endif -extern u64 xfeatures_mask_all;static inline u64 xfeatures_mask_supervisor (void )
arch/x86/kernel/fpu/signal.c +1 -0
@@ -17,6 +17,7 @@#include <asm/trace/fpu.h> #include "internal.h" +#include "xstate.h" static struct _fpx_sw_bytes fx_sw_reserved__ro_after_init ;static struct _fpx_sw_bytes fx_sw_reserved_ia32__ro_after_init ;
arch/x86/kernel/fpu/xstate.h +174 -0
178 lines not on this page · raw message
PATCH v2 21/30 x86/fpu: Move legacy ASM wrappers to core
In reply to Thomas GleixnerNothing outside the core code requires them.
--- arch/x86/include/asm/fpu/internal.h | 101 +----------------------------------- arch/x86/kernel/fpu/core.c | 1 +- arch/x86/kernel/fpu/legacy.h | 108 +++++++++++++++++++++++++++++++++++++- arch/x86/kernel/fpu/signal.c | 1 +- arch/x86/kernel/fpu/xstate.c | 1 +- 5 files changed, 111 insertions(+), 101 deletions(-) create mode 100644 arch/x86/kernel/fpu/legacy.h
---
arch/x86/include/asm/fpu/internal.h +0 -101
@@ -60,107 +60,6 @@ extern void fpstate_init_soft(struct swregs_state *soft);static inline void fpstate_init_soft (struct swregs_state *soft) {}#endif -/* -* Returns 0 on success or the trap number when the operation raises an -* exception. -*/ -#define user_insn(insn, output, input...) \ -({ \ - int err; \ - \ - might_fault(); \ - \ - asm volatile(ASM_STAC "\n" \ - "1: " #insn "\n" \ - "2: " ASM_CLAC "\n" \ - _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_FAULT_MCE_SAFE) \ - : [err] "=a" (err), output \ - : "0"(0), input); \ - err; \ -}) - -#define kernel_insn_err(insn, output, input...) \ -({ \ - int err; \ - asm volatile("1:" #insn "\n\t" \ - "2:\n" \ - ".section .fixup,\"ax\"\n" \ - "3: movl $-1,%[err]\n" \ - " jmp 2b\n" \ - ".previous\n" \ - _ASM_EXTABLE(1b, 3b) \ - : [err] "=r" (err), output \ - : "0"(0), input); \ - err; \ -}) - -#define kernel_insn(insn, output, input...) \ - asm volatile("1:" #insn "\n\t" \ - "2:\n" \ - _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_FPU_RESTORE) \ - : output : input) - -static inline int fnsave_to_user_sigframe (struct fregs_state __user *fx) -{ - return user_insn(fnsave %[fx]; fwait, [fx] "=m" (*fx), "m" (*fx)); -} - -static inline int fxsave_to_user_sigframe (struct fxregs_state __user *fx) -{ -if (IS_ENABLED (CONFIG_X86_32 )) - return user_insn(fxsave %[fx], [fx] "=m" (*fx), "m" (*fx)); -else - return user_insn(fxsaveq %[fx], [fx] "=m" (*fx), "m" (*fx)); - -} - -static inline void fxrstor (struct fxregs_state *fx) -{ -if (IS_ENABLED (CONFIG_X86_32 )) - kernel_insn(fxrstor %[fx], "=m" (*fx), [fx] "m" (*fx)); -else - kernel_insn(fxrstorq %[fx], "=m" (*fx), [fx] "m" (*fx)); -} - -static inline int fxrstor_safe (struct fxregs_state *fx) -{ -if (IS_ENABLED (CONFIG_X86_32 )) - return kernel_insn_err(fxrstor %[fx], "=m" (*fx), [fx] "m" (*fx)); -else - return kernel_insn_err(fxrstorq %[fx], "=m" (*fx), [fx] "m" (*fx)); -} - -static inline int fxrstor_from_user_sigframe (struct fxregs_state __user *fx) -{ -if (IS_ENABLED (CONFIG_X86_32 )) - return user_insn(fxrstor %[fx], "=m" (*fx), [fx] "m" (*fx)); -else - return user_insn(fxrstorq %[fx], "=m" (*fx), [fx] "m" (*fx)); -} - -static inline void frstor (struct fregs_state *fx) -{ - kernel_insn(frstor %[fx], "=m" (*fx), [fx] "m" (*fx)); -} - -static inline int frstor_safe (struct fregs_state *fx) -{ - return kernel_insn_err(frstor %[fx], "=m" (*fx), [fx] "m" (*fx)); -} - -static inline int frstor_from_user_sigframe (struct fregs_state __user *fx) -{ - return user_insn(frstor %[fx], "=m" (*fx), [fx] "m" (*fx)); -} - -static inline void fxsave (struct fxregs_state *fx) -{ -if (IS_ENABLED (CONFIG_X86_32 )) - asmvolatile ("fxsave %[fx]" : [fx]"=m" (*fx)); -else - asmvolatile ("fxsaveq %[fx]" : [fx]"=m" (*fx)); -} -extern void restore_fpregs_from_fpstate (union fpregs_state *fpstate,u64 mask);extern bool copy_fpstate_to_sigframe (void __user *buf,void __user *fp,int size);
arch/x86/kernel/fpu/core.c +1 -0
@@ -18,6 +18,7 @@#include <linux/pkeys.h> #include "internal.h" +#include "legacy.h" #include "xstate.h" #define CREATE_TRACE_POINTS
arch/x86/kernel/fpu/legacy.h new file +108 -0
108 lines not on this page · raw message
arch/x86/kernel/fpu/signal.c +1 -0
@@ -17,6 +17,7 @@#include <asm/trace/fpu.h> #include "internal.h" +#include "legacy.h" #include "xstate.h" static struct _fpx_sw_bytes fx_sw_reserved__ro_after_init ;
arch/x86/kernel/fpu/xstate.c +1 -0
@@ -20,6 +20,7 @@#include <asm/tlbflush.h> #include "internal.h" +#include "legacy.h" #include "xstate.h" #define for_each_extended_xfeature (bit, mask) \
[tip: x86/fpu] x86/fpu: Move legacy ASM wrappers to core
The following commit has been merged into the x86/fpu branch of tip:
The following commit has been merged into the x86/fpu branch of tip:
Commit-ID: 34002571cb4199a446f7582704424d20a01c276e Gitweb: https://git.kernel.org/tip/34002571cb4199a446f7582704424d20a01c276e Author: Thomas Gleixner <[email protected]> AuthorDate: Fri, 15 Oct 2021 03:16:26 +02:00 Committer: Borislav Petkov <[email protected]> CommitterDate: Wed, 20 Oct 2021 15:27:28 +02:00
x86/fpu: Move legacy ASM wrappers to core
Nothing outside the core code requires them.
--- arch/x86/include/asm/fpu/internal.h | 101 +------------------------- arch/x86/kernel/fpu/core.c | 1 +- arch/x86/kernel/fpu/legacy.h | 108 +++++++++++++++++++++++++++- arch/x86/kernel/fpu/signal.c | 1 +- arch/x86/kernel/fpu/xstate.c | 1 +- 5 files changed, 111 insertions(+), 101 deletions(-) create mode 100644 arch/x86/kernel/fpu/legacy.h
arch/x86/include/asm/fpu/internal.h +0 -101
@@ -60,107 +60,6 @@ extern void fpstate_init_soft(struct swregs_state *soft);static inline void fpstate_init_soft (struct swregs_state *soft) {}#endif -/* -* Returns 0 on success or the trap number when the operation raises an -* exception. -*/ -#define user_insn(insn, output, input...) \ -({ \ - int err; \ - \ - might_fault(); \ - \ - asm volatile(ASM_STAC "\n" \ - "1: " #insn "\n" \ - "2: " ASM_CLAC "\n" \ - _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_FAULT_MCE_SAFE) \ - : [err] "=a" (err), output \ - : "0"(0), input); \ - err; \ -}) - -#define kernel_insn_err(insn, output, input...) \ -({ \ - int err; \ - asm volatile("1:" #insn "\n\t" \ - "2:\n" \ - ".section .fixup,\"ax\"\n" \ - "3: movl $-1,%[err]\n" \ - " jmp 2b\n" \ - ".previous\n" \ - _ASM_EXTABLE(1b, 3b) \ - : [err] "=r" (err), output \ - : "0"(0), input); \ - err; \ -}) - -#define kernel_insn(insn, output, input...) \ - asm volatile("1:" #insn "\n\t" \ - "2:\n" \ - _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_FPU_RESTORE) \ - : output : input) - -static inline int fnsave_to_user_sigframe (struct fregs_state __user *fx) -{ - return user_insn(fnsave %[fx]; fwait, [fx] "=m" (*fx), "m" (*fx)); -} - -static inline int fxsave_to_user_sigframe (struct fxregs_state __user *fx) -{ -if (IS_ENABLED (CONFIG_X86_32 )) - return user_insn(fxsave %[fx], [fx] "=m" (*fx), "m" (*fx)); -else - return user_insn(fxsaveq %[fx], [fx] "=m" (*fx), "m" (*fx)); - -} - -static inline void fxrstor (struct fxregs_state *fx) -{ -if (IS_ENABLED (CONFIG_X86_32 )) - kernel_insn(fxrstor %[fx], "=m" (*fx), [fx] "m" (*fx)); -else - kernel_insn(fxrstorq %[fx], "=m" (*fx), [fx] "m" (*fx)); -} - -static inline int fxrstor_safe (struct fxregs_state *fx) -{ -if (IS_ENABLED (CONFIG_X86_32 )) - return kernel_insn_err(fxrstor %[fx], "=m" (*fx), [fx] "m" (*fx)); -else - return kernel_insn_err(fxrstorq %[fx], "=m" (*fx), [fx] "m" (*fx)); -} - -static inline int fxrstor_from_user_sigframe (struct fxregs_state __user *fx) -{ -if (IS_ENABLED (CONFIG_X86_32 )) - return user_insn(fxrstor %[fx], "=m" (*fx), [fx] "m" (*fx)); -else - return user_insn(fxrstorq %[fx], "=m" (*fx), [fx] "m" (*fx)); -} - -static inline void frstor (struct fregs_state *fx) -{ - kernel_insn(frstor %[fx], "=m" (*fx), [fx] "m" (*fx)); -} - -static inline int frstor_safe (struct fregs_state *fx) -{ - return kernel_insn_err(frstor %[fx], "=m" (*fx), [fx] "m" (*fx)); -} - -static inline int frstor_from_user_sigframe (struct fregs_state __user *fx) -{ - return user_insn(frstor %[fx], "=m" (*fx), [fx] "m" (*fx)); -} - -static inline void fxsave (struct fxregs_state *fx) -{ -if (IS_ENABLED (CONFIG_X86_32 )) - asmvolatile ("fxsave %[fx]" : [fx]"=m" (*fx)); -else - asmvolatile ("fxsaveq %[fx]" : [fx]"=m" (*fx)); -} -extern void restore_fpregs_from_fpstate (union fpregs_state *fpstate,u64 mask);extern bool copy_fpstate_to_sigframe (void __user *buf,void __user *fp,int size);
arch/x86/kernel/fpu/core.c +1 -0
@@ -18,6 +18,7 @@#include <linux/pkeys.h> #include "internal.h" +#include "legacy.h" #include "xstate.h" #define CREATE_TRACE_POINTS
arch/x86/kernel/fpu/legacy.h new file +108 -0
108 lines not on this page · raw message
arch/x86/kernel/fpu/signal.c +1 -0
@@ -17,6 +17,7 @@#include <asm/trace/fpu.h> #include "internal.h" +#include "legacy.h" #include "xstate.h" static struct _fpx_sw_bytes fx_sw_reserved__ro_after_init ;
arch/x86/kernel/fpu/xstate.c +1 -0
@@ -20,6 +20,7 @@#include <asm/tlbflush.h> #include "internal.h" +#include "legacy.h" #include "xstate.h" #define for_each_extended_xfeature (bit, mask) \
PATCH v2 22/30 x86/fpu: Make WARN_ON_FPU() private
In reply to Thomas GleixnerNo point in being in global headers.
--- arch/x86/include/asm/fpu/internal.h | 9 --------- arch/x86/kernel/fpu/init.c | 2 ++ arch/x86/kernel/fpu/internal.h | 6 ++++++ 3 files changed, 8 insertions(+), 9 deletions(-)
---
arch/x86/include/asm/fpu/internal.h +0 -9
@@ -42,15 +42,6 @@ extern void fpu__init_system(struct cpuinfo_x86 *c);extern void fpu__init_check_bugs (void );extern void fpu__resume_cpu (void ); -/* -* Debugging facility: -*/ -#ifdef CONFIG_X86_DEBUG_FPU -# define WARN_ON_FPU (x) WARN_ON_ONCE(x) -#else -# define WARN_ON_FPU (x) ({ (void)(x); 0; }) -#endif -extern union fpregs_state init_fpstate;extern void fpstate_init_user (union fpregs_state *state);
arch/x86/kernel/fpu/init.c +2 -0
@@ -10,6 +10,8 @@#include <linux/sched/task.h> #include <linux/init.h> +#include "internal.h" +/* * Initialize the registers found in all CPUs, CR0 and CR4: */
arch/x86/kernel/fpu/internal.h +6 -0
@@ -13,6 +13,12 @@ static __always_inline __pure bool use_fxsr(void)return cpu_feature_enabled (X86_FEATURE_FXSR ); } +#ifdef CONFIG_X86_DEBUG_FPU +# define WARN_ON_FPU (x) WARN_ON_ONCE(x) +#else +# define WARN_ON_FPU (x) ({ (void)(x); 0; }) +#endif +/* Init functions */ extern void fpu__init_prepare_fx_sw_frame (void );
[tip: x86/fpu] x86/fpu: Make WARN_ON_FPU() private
The following commit has been merged into the x86/fpu branch of tip:
The following commit has been merged into the x86/fpu branch of tip:
Commit-ID: cdcb6fa14e1499ff2b2a3f3e0938c7b3b7ef2cd6 Gitweb: https://git.kernel.org/tip/cdcb6fa14e1499ff2b2a3f3e0938c7b3b7ef2cd6 Author: Thomas Gleixner <[email protected]> AuthorDate: Fri, 15 Oct 2021 03:16:28 +02:00 Committer: Borislav Petkov <[email protected]> CommitterDate: Wed, 20 Oct 2021 15:27:28 +02:00
x86/fpu: Make WARN_ON_FPU() private
No point in being in global headers.
--- arch/x86/include/asm/fpu/internal.h | 9 --------- arch/x86/kernel/fpu/init.c | 2 ++ arch/x86/kernel/fpu/internal.h | 6 ++++++ 3 files changed, 8 insertions(+), 9 deletions(-)
arch/x86/include/asm/fpu/internal.h +0 -9
@@ -42,15 +42,6 @@ extern void fpu__init_system(struct cpuinfo_x86 *c);extern void fpu__init_check_bugs (void );extern void fpu__resume_cpu (void ); -/* -* Debugging facility: -*/ -#ifdef CONFIG_X86_DEBUG_FPU -# define WARN_ON_FPU (x) WARN_ON_ONCE(x) -#else -# define WARN_ON_FPU (x) ({ (void)(x); 0; }) -#endif -extern union fpregs_state init_fpstate;extern void fpstate_init_user (union fpregs_state *state);
arch/x86/kernel/fpu/init.c +2 -0
@@ -10,6 +10,8 @@#include <linux/sched/task.h> #include <linux/init.h> +#include "internal.h" +/* * Initialize the registers found in all CPUs, CR0 and CR4: */
arch/x86/kernel/fpu/internal.h +6 -0
@@ -13,6 +13,12 @@ static __always_inline __pure bool use_fxsr(void)return cpu_feature_enabled (X86_FEATURE_FXSR ); } +#ifdef CONFIG_X86_DEBUG_FPU +# define WARN_ON_FPU (x) WARN_ON_ONCE(x) +#else +# define WARN_ON_FPU (x) ({ (void)(x); 0; }) +#endif +/* Init functions */ extern void fpu__init_prepare_fx_sw_frame (void );
PATCH v2 23/30 x86/fpu: Move fpregs_restore_userregs() to core
In reply to Thomas GleixnerOnly used internally in the FPU core code.
---
V2: Fixed changelog - Boris
--- arch/x86/include/asm/fpu/internal.h | 83 +------------------------------------- arch/x86/kernel/fpu/context.h | 85 ++++++++++++++++++++++++++++++++++++++- arch/x86/kernel/fpu/core.c | 1 +- arch/x86/kernel/fpu/regset.c | 1 +- arch/x86/kernel/fpu/signal.c | 1 +- 5 files changed, 88 insertions(+), 83 deletions(-) create mode 100644 arch/x86/kernel/fpu/context.h
---
arch/x86/include/asm/fpu/internal.h +0 -83
@@ -55,89 +55,6 @@ extern void restore_fpregs_from_fpstate(union fpregs_state *fpstate, u64 mask);extern bool copy_fpstate_to_sigframe (void __user *buf,void __user *fp,int size); -/* -* FPU context switch related helper methods: -*/ - DECLARE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx); -/* -* The in-register FPU state for an FPU context on a CPU is assumed to be -* valid if the fpu->last_cpu matches the CPU, and the fpu_fpregs_owner_ctx -* matches the FPU. -* -* If the FPU register state is valid, the kernel can skip restoring the -* FPU state from memory. -* -* Any code that clobbers the FPU registers or updates the in-memory -* FPU state for a task MUST let the rest of the kernel know that the -* FPU registers are no longer valid for this task. -* -* Either one of these invalidation functions is enough. Invalidate -* a resource you control: CPU if using the CPU for something else -* (with preemption disabled), FPU for the current task, or a task that -* is prevented from running by the current task. -*/ -static inline void __cpu_invalidate_fpregs_state (void ) -{ -__this_cpu_write (fpu_fpregs_owner_ctx,NULL ); -} - -static inline void __fpu_invalidate_fpregs_state (struct fpu *fpu) -{ - fpu->last_cpu =-1 ; -} - -static inline int fpregs_state_valid (struct fpu *fpu,unsigned int cpu) -{ -return fpu ==this_cpu_read (fpu_fpregs_owner_ctx) && cpu == fpu->last_cpu; -} - -/* -* These generally need preemption protection to work, -* do try to avoid using these on their own: -*/ -static inline void fpregs_deactivate (struct fpu *fpu) -{ -this_cpu_write (fpu_fpregs_owner_ctx,NULL ); -trace_x86_fpu_regs_deactivated (fpu); -} - -static inline void fpregs_activate (struct fpu *fpu) -{ -this_cpu_write (fpu_fpregs_owner_ctx, fpu); -trace_x86_fpu_regs_activated (fpu); -} - -/* Internal helper for switch_fpu_return() and signal frame setup */ -static inline void fpregs_restore_userregs (void ) -{ -struct fpu *fpu = ¤t->thread.fpu; -int cpu =smp_processor_id (); - -if (WARN_ON_ONCE (current->flags &PF_KTHREAD )) -return ; - -if (!fpregs_state_valid (fpu, cpu)) { -u64 mask; - -/* -* This restores _all_ xstate which has not been -* established yet. -* -* If PKRU is enabled, then the PKRU value is already -* correct because it was either set in switch_to() or in -* flush_thread(). So it is excluded because it might be -* not up to date in current->thread.fpu.xsave state. -*/ - mask =xfeatures_mask_restore_user () | -xfeatures_mask_supervisor (); -restore_fpregs_from_fpstate (&fpu->state, mask); - -fpregs_activate (fpu); - fpu->last_cpu = cpu; - } -clear_thread_flag (TIF_NEED_FPU_LOAD ); -} -#endif /* _ASM_X86_FPU_INTERNAL_H */
arch/x86/kernel/fpu/context.h new file +85 -0
85 lines not on this page · raw message
arch/x86/kernel/fpu/core.c +1 -0
@@ -17,6 +17,7 @@#include <linux/hardirq.h> #include <linux/pkeys.h> +#include "context.h" #include "internal.h" #include "legacy.h" #include "xstate.h"
arch/x86/kernel/fpu/regset.c +1 -0
@@ -10,6 +10,7 @@#include <asm/fpu/regset.h> #include <asm/fpu/xstate.h> +#include "context.h" #include "internal.h" /*
arch/x86/kernel/fpu/signal.c +1 -0
@@ -16,6 +16,7 @@#include <asm/trapnr.h> #include <asm/trace/fpu.h> +#include "context.h" #include "internal.h" #include "legacy.h" #include "xstate.h"
Borislav Petkov
maintainerRe: [patch V2 23/30] x86/fpu: Move fpregs_restore_userregs() to core
arch/x86/kernel/fpu/context.h line 51
Quoted 10 lines · Thomas Gleixner
arch/x86/kernel/fpu/context.h new file +10
-0+static inline void fpregs_deactivate (struct fpu *fpu) +{ +__this_cpu_write (fpu_fpregs_owner_ctx,NULL ); +trace_x86_fpu_regs_deactivated (fpu); +} + +static inline void fpregs_activate (struct fpu *fpu) +{ +__this_cpu_write (fpu_fpregs_owner_ctx, fpu); +trace_x86_fpu_regs_activated (fpu);
You're silently changing here the percpu writes to the __ variants and AFAICT, there's no difference on x86:
# arch/x86/kernel/fpu/context.h:50: this_cpu_write(fpu_fpregs_owner_ctx, fpu);
#APP
# 50 "arch/x86/kernel/fpu/context.h" 1
movq %rsi, %gs:fpu_fpregs_owner_ctx(%rip) # fpu, fpu_fpregs_owner_ctx
# 0 "" 2
VS
# arch/x86/kernel/fpu/context.h:50: __this_cpu_write(fpu_fpregs_owner_ctx, fpu);
#APP
# 50 "arch/x86/kernel/fpu/context.h" 1
movq %rsi, %gs:fpu_fpregs_owner_ctx(%rip) # fpu, fpu_fpregs_owner_ctx
# 0 "" 2
except maybe the __ variant doesn't use the "volatile" inline asm qualifier in the lower-level raw_cpu_write_8() vs this_cpu_write_8(). And there's the preemption check, ofc.
Or maybe this could have something to do with RT...?
Commit message could mention this change, though.
Thx.
Signature · 4 lines
Regards/Gruss,
Boris.
https://people.kernel.org/tglx/notes-about-netiquette[tip: x86/fpu] x86/fpu: Move fpregs_restore_userregs() to core
In reply to Thomas GleixnerThe following commit has been merged into the x86/fpu branch of tip:
The following commit has been merged into the x86/fpu branch of tip:
Commit-ID: 9848fb96839bfd6ad4c00748842ccfd5bd3b0346 Gitweb: https://git.kernel.org/tip/9848fb96839bfd6ad4c00748842ccfd5bd3b0346 Author: Thomas Gleixner <[email protected]> AuthorDate: Fri, 15 Oct 2021 03:16:30 +02:00 Committer: Borislav Petkov <[email protected]> CommitterDate: Wed, 20 Oct 2021 15:27:28 +02:00
x86/fpu: Move fpregs_restore_userregs() to core
Only used internally in the FPU core code.
While at it, convert to the percpu accessors which verify preemption is disabled.
--- arch/x86/include/asm/fpu/internal.h | 83 +--------------------------- arch/x86/kernel/fpu/context.h | 85 ++++++++++++++++++++++++++++- arch/x86/kernel/fpu/core.c | 1 +- arch/x86/kernel/fpu/regset.c | 1 +- arch/x86/kernel/fpu/signal.c | 1 +- 5 files changed, 88 insertions(+), 83 deletions(-) create mode 100644 arch/x86/kernel/fpu/context.h
arch/x86/include/asm/fpu/internal.h +0 -83
@@ -55,89 +55,6 @@ extern void restore_fpregs_from_fpstate(union fpregs_state *fpstate, u64 mask);extern bool copy_fpstate_to_sigframe (void __user *buf,void __user *fp,int size); -/* -* FPU context switch related helper methods: -*/ - DECLARE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx); -/* -* The in-register FPU state for an FPU context on a CPU is assumed to be -* valid if the fpu->last_cpu matches the CPU, and the fpu_fpregs_owner_ctx -* matches the FPU. -* -* If the FPU register state is valid, the kernel can skip restoring the -* FPU state from memory. -* -* Any code that clobbers the FPU registers or updates the in-memory -* FPU state for a task MUST let the rest of the kernel know that the -* FPU registers are no longer valid for this task. -* -* Either one of these invalidation functions is enough. Invalidate -* a resource you control: CPU if using the CPU for something else -* (with preemption disabled), FPU for the current task, or a task that -* is prevented from running by the current task. -*/ -static inline void __cpu_invalidate_fpregs_state (void ) -{ -__this_cpu_write (fpu_fpregs_owner_ctx,NULL ); -} - -static inline void __fpu_invalidate_fpregs_state (struct fpu *fpu) -{ - fpu->last_cpu =-1 ; -} - -static inline int fpregs_state_valid (struct fpu *fpu,unsigned int cpu) -{ -return fpu ==this_cpu_read (fpu_fpregs_owner_ctx) && cpu == fpu->last_cpu; -} - -/* -* These generally need preemption protection to work, -* do try to avoid using these on their own: -*/ -static inline void fpregs_deactivate (struct fpu *fpu) -{ -this_cpu_write (fpu_fpregs_owner_ctx,NULL ); -trace_x86_fpu_regs_deactivated (fpu); -} - -static inline void fpregs_activate (struct fpu *fpu) -{ -this_cpu_write (fpu_fpregs_owner_ctx, fpu); -trace_x86_fpu_regs_activated (fpu); -} - -/* Internal helper for switch_fpu_return() and signal frame setup */ -static inline void fpregs_restore_userregs (void ) -{ -struct fpu *fpu = ¤t->thread.fpu; -int cpu =smp_processor_id (); - -if (WARN_ON_ONCE (current->flags &PF_KTHREAD )) -return ; - -if (!fpregs_state_valid (fpu, cpu)) { -u64 mask; - -/* -* This restores _all_ xstate which has not been -* established yet. -* -* If PKRU is enabled, then the PKRU value is already -* correct because it was either set in switch_to() or in -* flush_thread(). So it is excluded because it might be -* not up to date in current->thread.fpu.xsave state. -*/ - mask =xfeatures_mask_restore_user () | -xfeatures_mask_supervisor (); -restore_fpregs_from_fpstate (&fpu->state, mask); - -fpregs_activate (fpu); - fpu->last_cpu = cpu; - } -clear_thread_flag (TIF_NEED_FPU_LOAD ); -} -#endif /* _ASM_X86_FPU_INTERNAL_H */
arch/x86/kernel/fpu/context.h new file +85 -0
85 lines not on this page · raw message
arch/x86/kernel/fpu/core.c +1 -0
@@ -17,6 +17,7 @@#include <linux/hardirq.h> #include <linux/pkeys.h> +#include "context.h" #include "internal.h" #include "legacy.h" #include "xstate.h"
arch/x86/kernel/fpu/regset.c +1 -0
@@ -10,6 +10,7 @@#include <asm/fpu/regset.h> #include <asm/fpu/xstate.h> +#include "context.h" #include "internal.h" /*
arch/x86/kernel/fpu/signal.c +1 -0
@@ -16,6 +16,7 @@#include <asm/trapnr.h> #include <asm/trace/fpu.h> +#include "context.h" #include "internal.h" #include "legacy.h" #include "xstate.h"
PATCH v2 24/30 x86/fpu: Move mxcsr related code to core
In reply to Thomas GleixnerNo need to expose that to code which only needs the XCR0 accessors.
--- arch/x86/include/asm/fpu/xcr.h | 11 ----------- arch/x86/kernel/fpu/init.c | 1 + arch/x86/kernel/fpu/legacy.h | 7 +++++++ arch/x86/kernel/fpu/regset.c | 1 + arch/x86/kernel/fpu/xstate.c | 3 ++- arch/x86/kvm/svm/sev.c | 2 +- 6 files changed, 12 insertions(+), 13 deletions(-)
---
arch/x86/include/asm/fpu/xcr.h +0 -11
@@ -2,17 +2,6 @@#ifndef _ASM_X86_FPU_XCR_H#define _ASM_X86_FPU_XCR_H -/* -* MXCSR and XCR definitions: -*/ - -static inline void ldmxcsr (u32 mxcsr) -{ - asmvolatile ("ldmxcsr %0" ::"m" (mxcsr)); -} - -extern unsigned int mxcsr_feature_mask; -#define XCR_XFEATURE_ENABLED_MASK 0x00000000static inline u64 xgetbv (u32 index)
arch/x86/kernel/fpu/init.c +1 -0
@@ -11,6 +11,7 @@#include <linux/init.h> #include "internal.h" +#include "legacy.h" /* * Initialize the registers found in all CPUs, CR0 and CR4:
arch/x86/kernel/fpu/legacy.h +7 -0
@@ -4,6 +4,13 @@#include <asm/fpu/types.h> +extern unsigned int mxcsr_feature_mask; + +static inline void ldmxcsr (u32 mxcsr) +{ + asmvolatile ("ldmxcsr %0" ::"m" (mxcsr)); +} +/* * Returns 0 on success or the trap number when the operation raises an * exception.
arch/x86/kernel/fpu/regset.c +1 -0
@@ -12,6 +12,7 @@#include "context.h" #include "internal.h" +#include "legacy.h" /* * The xstateregs_active() routine is the same as the regset_fpregs_active() routine,
arch/x86/kernel/fpu/xstate.c +2 -1
@@ -14,8 +14,9 @@#include <asm/fpu/api.h> #include <asm/fpu/internal.h> -#include <asm/fpu/signal.h> #include <asm/fpu/regset.h> +#include <asm/fpu/signal.h> +#include <asm/fpu/xcr.h> #include <asm/tlbflush.h>
arch/x86/kvm/svm/sev.c +1 -1
@@ -17,10 +17,10 @@#include <linux/misc_cgroup.h> #include <linux/processor.h> #include <linux/trace_events.h> -#include <asm/fpu/internal.h> #include <asm/pkru.h> #include <asm/trapnr.h> +#include <asm/fpu/xcr.h> #include "x86.h" #include "svm.h"
[tip: x86/fpu] x86/fpu: Move mxcsr related code to core
The following commit has been merged into the x86/fpu branch of tip:
The following commit has been merged into the x86/fpu branch of tip:
Commit-ID: d9d005f32aac7362a1998f4b7fdf8874e91546bd Gitweb: https://git.kernel.org/tip/d9d005f32aac7362a1998f4b7fdf8874e91546bd Author: Thomas Gleixner <[email protected]> AuthorDate: Fri, 15 Oct 2021 03:16:31 +02:00 Committer: Borislav Petkov <[email protected]> CommitterDate: Wed, 20 Oct 2021 15:27:28 +02:00
x86/fpu: Move mxcsr related code to core
No need to expose that to code which only needs the XCR0 accessors.
--- arch/x86/include/asm/fpu/xcr.h | 11 ----------- arch/x86/kernel/fpu/init.c | 1 + arch/x86/kernel/fpu/legacy.h | 7 +++++++ arch/x86/kernel/fpu/regset.c | 1 + arch/x86/kernel/fpu/xstate.c | 3 ++- arch/x86/kvm/svm/sev.c | 2 +- 6 files changed, 12 insertions(+), 13 deletions(-)
arch/x86/include/asm/fpu/xcr.h +0 -11
@@ -2,17 +2,6 @@#ifndef _ASM_X86_FPU_XCR_H#define _ASM_X86_FPU_XCR_H -/* -* MXCSR and XCR definitions: -*/ - -static inline void ldmxcsr (u32 mxcsr) -{ - asmvolatile ("ldmxcsr %0" ::"m" (mxcsr)); -} - -extern unsigned int mxcsr_feature_mask; -#define XCR_XFEATURE_ENABLED_MASK 0x00000000static inline u64 xgetbv (u32 index)
arch/x86/kernel/fpu/init.c +1 -0
@@ -11,6 +11,7 @@#include <linux/init.h> #include "internal.h" +#include "legacy.h" /* * Initialize the registers found in all CPUs, CR0 and CR4:
arch/x86/kernel/fpu/legacy.h +7 -0
@@ -4,6 +4,13 @@#include <asm/fpu/types.h> +extern unsigned int mxcsr_feature_mask; + +static inline void ldmxcsr (u32 mxcsr) +{ + asmvolatile ("ldmxcsr %0" ::"m" (mxcsr)); +} +/* * Returns 0 on success or the trap number when the operation raises an * exception.
arch/x86/kernel/fpu/regset.c +1 -0
@@ -12,6 +12,7 @@#include "context.h" #include "internal.h" +#include "legacy.h" /* * The xstateregs_active() routine is the same as the regset_fpregs_active() routine,
arch/x86/kernel/fpu/xstate.c +2 -1
@@ -14,8 +14,9 @@#include <asm/fpu/api.h> #include <asm/fpu/internal.h> -#include <asm/fpu/signal.h> #include <asm/fpu/regset.h> +#include <asm/fpu/signal.h> +#include <asm/fpu/xcr.h> #include <asm/tlbflush.h>
arch/x86/kvm/svm/sev.c +1 -1
@@ -17,10 +17,10 @@#include <linux/misc_cgroup.h> #include <linux/processor.h> #include <linux/trace_events.h> -#include <asm/fpu/internal.h> #include <asm/pkru.h> #include <asm/trapnr.h> +#include <asm/fpu/xcr.h> #include "x86.h" #include "svm.h"
PATCH v2 25/30 x86/fpu: Move fpstate functions to api.h
In reply to Thomas GleixnerMove function declarations which need to be globally available to api.h where they belong.
---
V2: Fix changelog typo - Boris
--- arch/x86/include/asm/fpu/api.h | 9 +++++++++ arch/x86/include/asm/fpu/internal.h | 9 --------- arch/x86/kernel/fpu/internal.h | 3 +++ arch/x86/math-emu/fpu_entry.c | 2 +- 4 files changed, 13 insertions(+), 10 deletions(-)
---
arch/x86/include/asm/fpu/api.h +9 -0
@@ -110,6 +110,15 @@ extern int cpu_has_xfeatures(u64 xfeatures_mask, const char **feature_name);static inline void update_pasid (void ) { } +#ifdef CONFIG_MATH_EMULATION +extern void fpstate_init_soft (struct swregs_state *soft); +#else +static inline void fpstate_init_soft (struct swregs_state *soft) {} +#endif + +/* fpstate */ +extern union fpregs_state init_fpstate; +/* fpstate-related functions which are exported to KVM */ extern void fpu_init_fpstate_user (struct fpu *fpu);
arch/x86/include/asm/fpu/internal.h +0 -9
@@ -42,15 +42,6 @@ extern void fpu__init_system(struct cpuinfo_x86 *c);extern void fpu__init_check_bugs (void );extern void fpu__resume_cpu (void ); -extern union fpregs_state init_fpstate; -extern void fpstate_init_user (union fpregs_state *state); - -#ifdef CONFIG_MATH_EMULATION -extern void fpstate_init_soft (struct swregs_state *soft); -#else -static inline void fpstate_init_soft (struct swregs_state *soft) {} -#endif -extern void restore_fpregs_from_fpstate (union fpregs_state *fpstate,u64 mask);extern bool copy_fpstate_to_sigframe (void __user *buf,void __user *fp,int size);
arch/x86/kernel/fpu/internal.h +3 -0
@@ -22,4 +22,7 @@ static __always_inline __pure bool use_fxsr(void)/* Init functions */ extern void fpu__init_prepare_fx_sw_frame (void ); +/* Used in init.c */ +extern void fpstate_init_user (union fpregs_state *state); +#endif
arch/x86/math-emu/fpu_entry.c +1 -1
@@ -31,7 +31,7 @@#include <linux/uaccess.h> #include <asm/traps.h> #include <asm/user.h> -#include <asm/fpu/internal.h> +#include <asm/fpu/api.h> #include "fpu_system.h" #include "fpu_emu.h"
[tip: x86/fpu] x86/fpu: Move fpstate functions to api.h
The following commit has been merged into the x86/fpu branch of tip:
The following commit has been merged into the x86/fpu branch of tip:
Commit-ID: 90489f1dee8b703a3301857917c0aba0b22b5d83 Gitweb: https://git.kernel.org/tip/90489f1dee8b703a3301857917c0aba0b22b5d83 Author: Thomas Gleixner <[email protected]> AuthorDate: Fri, 15 Oct 2021 03:16:33 +02:00 Committer: Borislav Petkov <[email protected]> CommitterDate: Wed, 20 Oct 2021 15:27:28 +02:00
x86/fpu: Move fpstate functions to api.h
Move function declarations which need to be globally available to api.h where they belong.
--- arch/x86/include/asm/fpu/api.h | 9 +++++++++ arch/x86/include/asm/fpu/internal.h | 9 --------- arch/x86/kernel/fpu/internal.h | 3 +++ arch/x86/math-emu/fpu_entry.c | 2 +- 4 files changed, 13 insertions(+), 10 deletions(-)
arch/x86/include/asm/fpu/api.h +9 -0
@@ -110,6 +110,15 @@ extern int cpu_has_xfeatures(u64 xfeatures_mask, const char **feature_name);static inline void update_pasid (void ) { } +#ifdef CONFIG_MATH_EMULATION +extern void fpstate_init_soft (struct swregs_state *soft); +#else +static inline void fpstate_init_soft (struct swregs_state *soft) {} +#endif + +/* fpstate */ +extern union fpregs_state init_fpstate; +/* fpstate-related functions which are exported to KVM */ extern void fpu_init_fpstate_user (struct fpu *fpu);
arch/x86/include/asm/fpu/internal.h +0 -9
@@ -42,15 +42,6 @@ extern void fpu__init_system(struct cpuinfo_x86 *c);extern void fpu__init_check_bugs (void );extern void fpu__resume_cpu (void ); -extern union fpregs_state init_fpstate; -extern void fpstate_init_user (union fpregs_state *state); - -#ifdef CONFIG_MATH_EMULATION -extern void fpstate_init_soft (struct swregs_state *soft); -#else -static inline void fpstate_init_soft (struct swregs_state *soft) {} -#endif -extern void restore_fpregs_from_fpstate (union fpregs_state *fpstate,u64 mask);extern bool copy_fpstate_to_sigframe (void __user *buf,void __user *fp,int size);
arch/x86/kernel/fpu/internal.h +3 -0
@@ -22,4 +22,7 @@ static __always_inline __pure bool use_fxsr(void)/* Init functions */ extern void fpu__init_prepare_fx_sw_frame (void ); +/* Used in init.c */ +extern void fpstate_init_user (union fpregs_state *state); +#endif
arch/x86/math-emu/fpu_entry.c +1 -1
@@ -31,7 +31,7 @@#include <linux/uaccess.h> #include <asm/traps.h> #include <asm/user.h> -#include <asm/fpu/internal.h> +#include <asm/fpu/api.h> #include "fpu_system.h" #include "fpu_emu.h"
PATCH v2 26/30 x86/fpu: Remove internal.h dependency from fpu/signal.h
In reply to Thomas GleixnerIn order to remove internal.h make signal.h independent of it.
--- arch/x86/ia32/ia32_signal.c | 1 - arch/x86/include/asm/fpu/api.h | 3 +++ arch/x86/include/asm/fpu/internal.h | 7 ------- arch/x86/include/asm/fpu/signal.h | 13 +++++++++++++ arch/x86/kernel/fpu/signal.c | 1 - arch/x86/kernel/ptrace.c | 1 - arch/x86/kernel/signal.c | 1 - arch/x86/mm/extable.c | 3 ++- 8 files changed, 18 insertions(+), 12 deletions(-)
---
arch/x86/ia32/ia32_signal.c +0 -1
@@ -24,7 +24,6 @@#include <linux/syscalls.h> #include <asm/ucontext.h> #include <linux/uaccess.h> -#include <asm/fpu/internal.h> #include <asm/fpu/signal.h> #include <asm/ptrace.h> #include <asm/ia32_unistd.h>
arch/x86/include/asm/fpu/api.h +3 -0
@@ -116,6 +116,9 @@ extern void fpstate_init_soft(struct swregs_state *soft);static inline void fpstate_init_soft (struct swregs_state *soft) {}#endif +/* State tracking */ +DECLARE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx); +/* fpstate */ extern union fpregs_state init_fpstate;
arch/x86/include/asm/fpu/internal.h +0 -7
@@ -26,7 +26,6 @@/* * High level FPU state handling functions: */ -extern bool fpu__restore_sig (void __user *buf,int ia32_frame);extern void fpu__clear_user_states (struct fpu *fpu);extern int fpu__exception_code (struct fpu *fpu,int trap_nr);
@@ -42,10 +41,4 @@ extern void fpu__init_system(struct cpuinfo_x86 *c);extern void fpu__init_check_bugs (void );extern void fpu__resume_cpu (void ); -extern void restore_fpregs_from_fpstate (union fpregs_state *fpstate,u64 mask); - -extern bool copy_fpstate_to_sigframe (void __user *buf,void __user *fp,int size); - -DECLARE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx); -#endif /* _ASM_X86_FPU_INTERNAL_H */
arch/x86/include/asm/fpu/signal.h +13 -0
@@ -5,6 +5,11 @@#ifndef _ASM_X86_FPU_SIGNAL_H#define _ASM_X86_FPU_SIGNAL_H +#include <linux/compat.h> +#include <linux/user.h> + +#include <asm/fpu/types.h> +#ifdef CONFIG_X86_64 # include <uapi/asm/sigcontext.h> # include <asm/user32.h>
@@ -31,4 +36,12 @@ fpu__alloc_mathframe(unsigned long sp, int ia32_frame,unsigned long fpu__get_fpstate_size (void ); +extern bool copy_fpstate_to_sigframe (void __user *buf,void __user *fp,int size); +extern void fpu__clear_user_states (struct fpu *fpu); +extern bool fpu__restore_sig (void __user *buf,int ia32_frame); + +extern void restore_fpregs_from_fpstate (union fpregs_state *fpstate,u64 mask); + +extern bool copy_fpstate_to_sigframe (void __user *buf,void __user *fp,int size); +#endif /* _ASM_X86_FPU_SIGNAL_H */
arch/x86/kernel/fpu/signal.c +0 -1
@@ -7,7 +7,6 @@#include <linux/cpu.h> #include <linux/pagemap.h> -#include <asm/fpu/internal.h> #include <asm/fpu/signal.h> #include <asm/fpu/regset.h> #include <asm/fpu/xstate.h>
arch/x86/kernel/ptrace.c +0 -1
@@ -29,7 +29,6 @@#include <linux/uaccess.h> #include <asm/processor.h> -#include <asm/fpu/internal.h> #include <asm/fpu/signal.h> #include <asm/fpu/regset.h> #include <asm/debugreg.h>
arch/x86/kernel/signal.c +0 -1
@@ -30,7 +30,6 @@#include <asm/processor.h> #include <asm/ucontext.h> -#include <asm/fpu/internal.h> #include <asm/fpu/signal.h> #include <asm/vdso.h> #include <asm/mce.h>
arch/x86/mm/extable.c +2 -1
@@ -4,7 +4,8 @@#include <linux/sched/debug.h> #include <xen/xen.h> -#include <asm/fpu/internal.h> +#include <asm/fpu/signal.h> +#include <asm/fpu/xstate.h> #include <asm/sev.h> #include <asm/traps.h> #include <asm/kdebug.h>
[tip: x86/fpu] x86/fpu: Remove internal.h dependency from fpu/signal.h
The following commit has been merged into the x86/fpu branch of tip:
The following commit has been merged into the x86/fpu branch of tip:
Commit-ID: 0ae67cc34f765078a63137120e4567ad2f050b75 Gitweb: https://git.kernel.org/tip/0ae67cc34f765078a63137120e4567ad2f050b75 Author: Thomas Gleixner <[email protected]> AuthorDate: Fri, 15 Oct 2021 03:16:35 +02:00 Committer: Borislav Petkov <[email protected]> CommitterDate: Wed, 20 Oct 2021 15:27:29 +02:00
x86/fpu: Remove internal.h dependency from fpu/signal.h
In order to remove internal.h make signal.h independent of it.
Include asm/fpu/xstate.h to fix a missing update_regset_xstate_info() prototype, which is
--- arch/x86/ia32/ia32_signal.c | 1 - arch/x86/include/asm/fpu/api.h | 3 +++ arch/x86/include/asm/fpu/internal.h | 7 ------- arch/x86/include/asm/fpu/signal.h | 13 +++++++++++++ arch/x86/kernel/fpu/signal.c | 1 - arch/x86/kernel/ptrace.c | 2 +- arch/x86/kernel/signal.c | 1 - arch/x86/mm/extable.c | 3 ++- 8 files changed, 19 insertions(+), 12 deletions(-)
arch/x86/ia32/ia32_signal.c +0 -1
@@ -24,7 +24,6 @@#include <linux/syscalls.h> #include <asm/ucontext.h> #include <linux/uaccess.h> -#include <asm/fpu/internal.h> #include <asm/fpu/signal.h> #include <asm/ptrace.h> #include <asm/ia32_unistd.h>
arch/x86/include/asm/fpu/api.h +3 -0
@@ -116,6 +116,9 @@ extern void fpstate_init_soft(struct swregs_state *soft);static inline void fpstate_init_soft (struct swregs_state *soft) {}#endif +/* State tracking */ +DECLARE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx); +/* fpstate */ extern union fpregs_state init_fpstate;
arch/x86/include/asm/fpu/internal.h +0 -7
@@ -26,7 +26,6 @@/* * High level FPU state handling functions: */ -extern bool fpu__restore_sig (void __user *buf,int ia32_frame);extern void fpu__clear_user_states (struct fpu *fpu);extern int fpu__exception_code (struct fpu *fpu,int trap_nr);
@@ -42,10 +41,4 @@ extern void fpu__init_system(struct cpuinfo_x86 *c);extern void fpu__init_check_bugs (void );extern void fpu__resume_cpu (void ); -extern void restore_fpregs_from_fpstate (union fpregs_state *fpstate,u64 mask); - -extern bool copy_fpstate_to_sigframe (void __user *buf,void __user *fp,int size); - -DECLARE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx); -#endif /* _ASM_X86_FPU_INTERNAL_H */
arch/x86/include/asm/fpu/signal.h +13 -0
@@ -5,6 +5,11 @@#ifndef _ASM_X86_FPU_SIGNAL_H#define _ASM_X86_FPU_SIGNAL_H +#include <linux/compat.h> +#include <linux/user.h> + +#include <asm/fpu/types.h> +#ifdef CONFIG_X86_64 # include <uapi/asm/sigcontext.h> # include <asm/user32.h>
@@ -31,4 +36,12 @@ fpu__alloc_mathframe(unsigned long sp, int ia32_frame,unsigned long fpu__get_fpstate_size (void ); +extern bool copy_fpstate_to_sigframe (void __user *buf,void __user *fp,int size); +extern void fpu__clear_user_states (struct fpu *fpu); +extern bool fpu__restore_sig (void __user *buf,int ia32_frame); + +extern void restore_fpregs_from_fpstate (union fpregs_state *fpstate,u64 mask); + +extern bool copy_fpstate_to_sigframe (void __user *buf,void __user *fp,int size); +#endif /* _ASM_X86_FPU_SIGNAL_H */
arch/x86/kernel/fpu/signal.c +0 -1
@@ -7,7 +7,6 @@#include <linux/cpu.h> #include <linux/pagemap.h> -#include <asm/fpu/internal.h> #include <asm/fpu/signal.h> #include <asm/fpu/regset.h> #include <asm/fpu/xstate.h>
arch/x86/kernel/ptrace.c +1 -1
@@ -29,9 +29,9 @@#include <linux/uaccess.h> #include <asm/processor.h> -#include <asm/fpu/internal.h> #include <asm/fpu/signal.h> #include <asm/fpu/regset.h> +#include <asm/fpu/xstate.h> #include <asm/debugreg.h> #include <asm/ldt.h> #include <asm/desc.h>
arch/x86/kernel/signal.c +0 -1
@@ -30,7 +30,6 @@#include <asm/processor.h> #include <asm/ucontext.h> -#include <asm/fpu/internal.h> #include <asm/fpu/signal.h> #include <asm/vdso.h> #include <asm/mce.h>
arch/x86/mm/extable.c +2 -1
@@ -4,7 +4,8 @@#include <linux/sched/debug.h> #include <xen/xen.h> -#include <asm/fpu/internal.h> +#include <asm/fpu/signal.h> +#include <asm/fpu/xstate.h> #include <asm/sev.h> #include <asm/traps.h> #include <asm/kdebug.h>
PATCH v2 27/30 x86/sev: Include fpu/xcr.h
In reply to Thomas GleixnerInclude the header which only provides the XCR accessors. That's all what is needed here.
---
V2: s/XRC/XCR/ - Xiaoyao Li
--- arch/x86/kernel/sev.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
---
arch/x86/kernel/sev.c +1 -1
@@ -23,7 +23,7 @@#include <asm/stacktrace.h> #include <asm/sev.h> #include <asm/insn-eval.h> -#include <asm/fpu/internal.h> +#include <asm/fpu/xcr.h> #include <asm/processor.h> #include <asm/realmode.h> #include <asm/traps.h>
[tip: x86/fpu] x86/sev: Include fpu/xcr.h
The following commit has been merged into the x86/fpu branch of tip:
The following commit has been merged into the x86/fpu branch of tip:
Commit-ID: ff0c37e191f2629bf2776dbd95db5d06f704ab93 Gitweb: https://git.kernel.org/tip/ff0c37e191f2629bf2776dbd95db5d06f704ab93 Author: Thomas Gleixner <[email protected]> AuthorDate: Fri, 15 Oct 2021 03:16:36 +02:00 Committer: Borislav Petkov <[email protected]> CommitterDate: Wed, 20 Oct 2021 15:27:29 +02:00
x86/sev: Include fpu/xcr.h
Include the header which only provides the XCR accessors. That's all what is needed here.
--- arch/x86/kernel/sev.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/x86/kernel/sev.c +1 -1
@@ -23,7 +23,7 @@#include <asm/stacktrace.h> #include <asm/sev.h> #include <asm/insn-eval.h> -#include <asm/fpu/internal.h> +#include <asm/fpu/xcr.h> #include <asm/processor.h> #include <asm/realmode.h> #include <asm/traps.h>
PATCH v2 28/30 x86/fpu: Mop up the internal.h leftovers
In reply to Thomas GleixnerMove the global interfaces to api.h and the rest into the core.
--- arch/x86/include/asm/fpu/api.h | 10 ++++++++++ arch/x86/include/asm/fpu/internal.h | 18 ------------------ arch/x86/kernel/fpu/init.c | 1 + arch/x86/kernel/fpu/xstate.h | 3 +++ 4 files changed, 14 insertions(+), 18 deletions(-)
---
arch/x86/include/asm/fpu/api.h +10 -0
@@ -110,6 +110,16 @@ extern int cpu_has_xfeatures(u64 xfeatures_mask, const char **feature_name);static inline void update_pasid (void ) { } +/* Trap handling */ +extern int fpu__exception_code (struct fpu *fpu,int trap_nr); +extern void fpu_sync_fpstate (struct fpu *fpu); + +/* Boot, hotplug and resume */ +extern void fpu__init_cpu (void ); +extern void fpu__init_system (struct cpuinfo_x86 *c); +extern void fpu__init_check_bugs (void ); +extern void fpu__resume_cpu (void ); +#ifdef CONFIG_MATH_EMULATION extern void fpstate_init_soft (struct swregs_state *soft);#else
arch/x86/include/asm/fpu/internal.h +0 -18
@@ -23,22 +23,4 @@#include <asm/cpufeature.h> #include <asm/trace/fpu.h> -/* -* High level FPU state handling functions: -*/ -extern void fpu__clear_user_states (struct fpu *fpu); -extern int fpu__exception_code (struct fpu *fpu,int trap_nr); - -extern void fpu_sync_fpstate (struct fpu *fpu); - -/* -* Boot time FPU initialization functions: -*/ -extern void fpu__init_cpu (void ); -extern void fpu__init_system_xstate (void ); -extern void fpu__init_cpu_xstate (void ); -extern void fpu__init_system (struct cpuinfo_x86 *c); -extern void fpu__init_check_bugs (void ); -extern void fpu__resume_cpu (void ); -#endif /* _ASM_X86_FPU_INTERNAL_H */
arch/x86/kernel/fpu/init.c +1 -0
@@ -12,6 +12,7 @@#include "internal.h" #include "legacy.h" +#include "xstate.h" /* * Initialize the registers found in all CPUs, CR0 and CR4:
arch/x86/kernel/fpu/xstate.h +3 -0
@@ -18,6 +18,9 @@ static inline void xstate_init_xcomp_bv(struct xregs_state *xsave, u64 mask)extern void __copy_xstate_to_uabi_buf (struct membuf to,struct xregs_state *xsave,u32 pkru_val,enum xstate_copy_mode copy_mode); +extern void fpu__init_cpu_xstate (void ); +extern void fpu__init_system_xstate (void ); +/* XSAVE/XRSTOR wrapper functions */ #ifdef CONFIG_X86_64
[tip: x86/fpu] x86/fpu: Mop up the internal.h leftovers
The following commit has been merged into the x86/fpu branch of tip:
The following commit has been merged into the x86/fpu branch of tip:
Commit-ID: 6415bb80926379310afd74800415f6ebf4bb5c31 Gitweb: https://git.kernel.org/tip/6415bb80926379310afd74800415f6ebf4bb5c31 Author: Thomas Gleixner <[email protected]> AuthorDate: Fri, 15 Oct 2021 03:16:38 +02:00 Committer: Borislav Petkov <[email protected]> CommitterDate: Wed, 20 Oct 2021 15:27:29 +02:00
x86/fpu: Mop up the internal.h leftovers
Move the global interfaces to api.h and the rest into the core.
--- arch/x86/include/asm/fpu/api.h | 10 ++++++++++ arch/x86/include/asm/fpu/internal.h | 18 ------------------ arch/x86/kernel/fpu/init.c | 1 + arch/x86/kernel/fpu/xstate.h | 3 +++ 4 files changed, 14 insertions(+), 18 deletions(-)
arch/x86/include/asm/fpu/api.h +10 -0
@@ -110,6 +110,16 @@ extern int cpu_has_xfeatures(u64 xfeatures_mask, const char **feature_name);static inline void update_pasid (void ) { } +/* Trap handling */ +extern int fpu__exception_code (struct fpu *fpu,int trap_nr); +extern void fpu_sync_fpstate (struct fpu *fpu); + +/* Boot, hotplug and resume */ +extern void fpu__init_cpu (void ); +extern void fpu__init_system (struct cpuinfo_x86 *c); +extern void fpu__init_check_bugs (void ); +extern void fpu__resume_cpu (void ); +#ifdef CONFIG_MATH_EMULATION extern void fpstate_init_soft (struct swregs_state *soft);#else
arch/x86/include/asm/fpu/internal.h +0 -18
@@ -23,22 +23,4 @@#include <asm/cpufeature.h> #include <asm/trace/fpu.h> -/* -* High level FPU state handling functions: -*/ -extern void fpu__clear_user_states (struct fpu *fpu); -extern int fpu__exception_code (struct fpu *fpu,int trap_nr); - -extern void fpu_sync_fpstate (struct fpu *fpu); - -/* -* Boot time FPU initialization functions: -*/ -extern void fpu__init_cpu (void ); -extern void fpu__init_system_xstate (void ); -extern void fpu__init_cpu_xstate (void ); -extern void fpu__init_system (struct cpuinfo_x86 *c); -extern void fpu__init_check_bugs (void ); -extern void fpu__resume_cpu (void ); -#endif /* _ASM_X86_FPU_INTERNAL_H */
arch/x86/kernel/fpu/init.c +1 -0
@@ -12,6 +12,7 @@#include "internal.h" #include "legacy.h" +#include "xstate.h" /* * Initialize the registers found in all CPUs, CR0 and CR4:
arch/x86/kernel/fpu/xstate.h +3 -0
@@ -18,6 +18,9 @@ static inline void xstate_init_xcomp_bv(struct xregs_state *xsave, u64 mask)extern void __copy_xstate_to_uabi_buf (struct membuf to,struct xregs_state *xsave,u32 pkru_val,enum xstate_copy_mode copy_mode); +extern void fpu__init_cpu_xstate (void ); +extern void fpu__init_system_xstate (void ); +/* XSAVE/XRSTOR wrapper functions */ #ifdef CONFIG_X86_64
PATCH v2 29/30 x86/fpu: Replace the includes of fpu/internal.h
In reply to Thomas GleixnerNow that the file is empty, fixup all references with the proper includes and delete the former kitchen sink.
--- arch/x86/include/asm/fpu/internal.h | 26 -------------------------- arch/x86/kernel/cpu/bugs.c | 2 +- arch/x86/kernel/cpu/common.c | 2 +- arch/x86/kernel/fpu/bugs.c | 2 +- arch/x86/kernel/fpu/core.c | 2 +- arch/x86/kernel/fpu/init.c | 2 +- arch/x86/kernel/fpu/regset.c | 2 +- arch/x86/kernel/fpu/xstate.c | 1 - arch/x86/kernel/smpboot.c | 2 +- arch/x86/kernel/traps.c | 2 +- arch/x86/kvm/vmx/vmx.c | 2 +- arch/x86/power/cpu.c | 2 +- 12 files changed, 10 insertions(+), 37 deletions(-)
---
arch/x86/include/asm/fpu/internal.h +0 -26
@@ -1,26 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* -* Copyright (C) 1994 Linus Torvalds -* -* Pentium III FXSR, SSE support -* General FPU state handling cleanups -* Gareth Hughes <[email protected]>, May 2000 -* x86-64 work by Andi Kleen 2002 -*/ - -#ifndef _ASM_X86_FPU_INTERNAL_H -#define _ASM_X86_FPU_INTERNAL_H - -#include <linux/compat.h> -#include <linux/sched.h> -#include <linux/slab.h> -#include <linux/mm.h> - -#include <asm/user.h> -#include <asm/fpu/api.h> -#include <asm/fpu/xstate.h> -#include <asm/fpu/xcr.h> -#include <asm/cpufeature.h> -#include <asm/trace/fpu.h> - -#endif /* _ASM_X86_FPU_INTERNAL_H */
arch/x86/kernel/cpu/bugs.c +1 -1
@@ -22,7 +22,7 @@#include <asm/bugs.h> #include <asm/processor.h> #include <asm/processor-flags.h> -#include <asm/fpu/internal.h> +#include <asm/fpu/api.h> #include <asm/msr.h> #include <asm/vmx.h> #include <asm/paravirt.h>
arch/x86/kernel/cpu/common.c +1 -1
@@ -42,7 +42,7 @@#include <asm/setup.h> #include <asm/apic.h> #include <asm/desc.h> -#include <asm/fpu/internal.h> +#include <asm/fpu/api.h> #include <asm/mtrr.h> #include <asm/hwcap2.h> #include <linux/numa.h>
arch/x86/kernel/fpu/bugs.c +1 -1
@@ -2,7 +2,7 @@/* * x86 FPU bug checks: */ -#include <asm/fpu/internal.h> +#include <asm/fpu/api.h> /* * Boot time CPU/FPU FDIV bug detection code:
arch/x86/kernel/fpu/core.c +1 -1
@@ -6,7 +6,7 @@* General FPU state handling cleanups * Gareth Hughes <[email protected]>, May 2000 */ -#include <asm/fpu/internal.h> +#include <asm/fpu/api.h> #include <asm/fpu/regset.h> #include <asm/fpu/sched.h> #include <asm/fpu/signal.h>
arch/x86/kernel/fpu/init.c +1 -1
@@ -2,7 +2,7 @@/* * x86 FPU boot time init code: */ -#include <asm/fpu/internal.h> +#include <asm/fpu/api.h> #include <asm/tlbflush.h> #include <asm/setup.h>
arch/x86/kernel/fpu/regset.c +1 -1
@@ -5,7 +5,7 @@#include <linux/sched/task_stack.h> #include <linux/vmalloc.h> -#include <asm/fpu/internal.h> +#include <asm/fpu/api.h> #include <asm/fpu/signal.h> #include <asm/fpu/regset.h> #include <asm/fpu/xstate.h>
arch/x86/kernel/fpu/xstate.c +0 -1
@@ -13,7 +13,6 @@#include <linux/proc_fs.h> #include <asm/fpu/api.h> -#include <asm/fpu/internal.h> #include <asm/fpu/regset.h> #include <asm/fpu/signal.h> #include <asm/fpu/xcr.h>
arch/x86/kernel/smpboot.c +1 -1
@@ -70,7 +70,7 @@#include <asm/mwait.h> #include <asm/apic.h> #include <asm/io_apic.h> -#include <asm/fpu/internal.h> +#include <asm/fpu/api.h> #include <asm/setup.h> #include <asm/uv/uv.h> #include <linux/mc146818rtc.h>
arch/x86/kernel/traps.c +1 -1
@@ -48,7 +48,7 @@#include <asm/ftrace.h> #include <asm/traps.h> #include <asm/desc.h> -#include <asm/fpu/internal.h> +#include <asm/fpu/api.h> #include <asm/cpu.h> #include <asm/cpu_entry_area.h> #include <asm/mce.h>
arch/x86/kvm/vmx/vmx.c +1 -1
@@ -35,7 +35,7 @@#include <asm/cpu_device_id.h> #include <asm/debugreg.h> #include <asm/desc.h> -#include <asm/fpu/internal.h> +#include <asm/fpu/api.h> #include <asm/idtentry.h> #include <asm/io.h> #include <asm/irq_remapping.h>
arch/x86/power/cpu.c +1 -1
@@ -20,7 +20,7 @@#include <asm/page.h> #include <asm/mce.h> #include <asm/suspend.h> -#include <asm/fpu/internal.h> +#include <asm/fpu/api.h> #include <asm/debugreg.h> #include <asm/cpu.h> #include <asm/mmu_context.h>
[tip: x86/fpu] x86/fpu: Replace the includes of fpu/internal.h
The following commit has been merged into the x86/fpu branch of tip:
The following commit has been merged into the x86/fpu branch of tip:
Commit-ID: b56d2795b29792c465cc8ef036abad5127a003fb Gitweb: https://git.kernel.org/tip/b56d2795b29792c465cc8ef036abad5127a003fb Author: Thomas Gleixner <[email protected]> AuthorDate: Fri, 15 Oct 2021 03:16:39 +02:00 Committer: Borislav Petkov <[email protected]> CommitterDate: Wed, 20 Oct 2021 15:27:29 +02:00
x86/fpu: Replace the includes of fpu/internal.h
Now that the file is empty, fixup all references with the proper includes and delete the former kitchen sink.
--- arch/x86/include/asm/fpu/internal.h | 26 -------------------------- arch/x86/kernel/cpu/bugs.c | 2 +- arch/x86/kernel/cpu/common.c | 2 +- arch/x86/kernel/fpu/bugs.c | 2 +- arch/x86/kernel/fpu/core.c | 2 +- arch/x86/kernel/fpu/init.c | 2 +- arch/x86/kernel/fpu/regset.c | 2 +- arch/x86/kernel/fpu/xstate.c | 1 - arch/x86/kernel/smpboot.c | 2 +- arch/x86/kernel/traps.c | 2 +- arch/x86/kvm/vmx/vmx.c | 2 +- arch/x86/power/cpu.c | 2 +- 12 files changed, 10 insertions(+), 37 deletions(-)
arch/x86/include/asm/fpu/internal.h +0 -26
@@ -1,26 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* -* Copyright (C) 1994 Linus Torvalds -* -* Pentium III FXSR, SSE support -* General FPU state handling cleanups -* Gareth Hughes <[email protected]>, May 2000 -* x86-64 work by Andi Kleen 2002 -*/ - -#ifndef _ASM_X86_FPU_INTERNAL_H -#define _ASM_X86_FPU_INTERNAL_H - -#include <linux/compat.h> -#include <linux/sched.h> -#include <linux/slab.h> -#include <linux/mm.h> - -#include <asm/user.h> -#include <asm/fpu/api.h> -#include <asm/fpu/xstate.h> -#include <asm/fpu/xcr.h> -#include <asm/cpufeature.h> -#include <asm/trace/fpu.h> - -#endif /* _ASM_X86_FPU_INTERNAL_H */
arch/x86/kernel/cpu/bugs.c +1 -1
@@ -22,7 +22,7 @@#include <asm/bugs.h> #include <asm/processor.h> #include <asm/processor-flags.h> -#include <asm/fpu/internal.h> +#include <asm/fpu/api.h> #include <asm/msr.h> #include <asm/vmx.h> #include <asm/paravirt.h>
arch/x86/kernel/cpu/common.c +1 -1
@@ -42,7 +42,7 @@#include <asm/setup.h> #include <asm/apic.h> #include <asm/desc.h> -#include <asm/fpu/internal.h> +#include <asm/fpu/api.h> #include <asm/mtrr.h> #include <asm/hwcap2.h> #include <linux/numa.h>
arch/x86/kernel/fpu/bugs.c +1 -1
@@ -2,7 +2,7 @@/* * x86 FPU bug checks: */ -#include <asm/fpu/internal.h> +#include <asm/fpu/api.h> /* * Boot time CPU/FPU FDIV bug detection code:
arch/x86/kernel/fpu/core.c +1 -1
@@ -6,7 +6,7 @@* General FPU state handling cleanups * Gareth Hughes <[email protected]>, May 2000 */ -#include <asm/fpu/internal.h> +#include <asm/fpu/api.h> #include <asm/fpu/regset.h> #include <asm/fpu/sched.h> #include <asm/fpu/signal.h>
arch/x86/kernel/fpu/init.c +1 -1
@@ -2,7 +2,7 @@/* * x86 FPU boot time init code: */ -#include <asm/fpu/internal.h> +#include <asm/fpu/api.h> #include <asm/tlbflush.h> #include <asm/setup.h>
arch/x86/kernel/fpu/regset.c +1 -1
@@ -5,7 +5,7 @@#include <linux/sched/task_stack.h> #include <linux/vmalloc.h> -#include <asm/fpu/internal.h> +#include <asm/fpu/api.h> #include <asm/fpu/signal.h> #include <asm/fpu/regset.h> #include <asm/fpu/xstate.h>
arch/x86/kernel/fpu/xstate.c +0 -1
@@ -13,7 +13,6 @@#include <linux/proc_fs.h> #include <asm/fpu/api.h> -#include <asm/fpu/internal.h> #include <asm/fpu/regset.h> #include <asm/fpu/signal.h> #include <asm/fpu/xcr.h>
arch/x86/kernel/smpboot.c +1 -1
@@ -70,7 +70,7 @@#include <asm/mwait.h> #include <asm/apic.h> #include <asm/io_apic.h> -#include <asm/fpu/internal.h> +#include <asm/fpu/api.h> #include <asm/setup.h> #include <asm/uv/uv.h> #include <linux/mc146818rtc.h>
arch/x86/kernel/traps.c +1 -1
@@ -48,7 +48,7 @@#include <asm/ftrace.h> #include <asm/traps.h> #include <asm/desc.h> -#include <asm/fpu/internal.h> +#include <asm/fpu/api.h> #include <asm/cpu.h> #include <asm/cpu_entry_area.h> #include <asm/mce.h>
arch/x86/kvm/vmx/vmx.c +1 -1
@@ -35,7 +35,7 @@#include <asm/cpu_device_id.h> #include <asm/debugreg.h> #include <asm/desc.h> -#include <asm/fpu/internal.h> +#include <asm/fpu/api.h> #include <asm/idtentry.h> #include <asm/io.h> #include <asm/irq_remapping.h>
arch/x86/power/cpu.c +1 -1
@@ -20,7 +20,7 @@#include <asm/page.h> #include <asm/mce.h> #include <asm/suspend.h> -#include <asm/fpu/internal.h> +#include <asm/fpu/api.h> #include <asm/debugreg.h> #include <asm/cpu.h> #include <asm/mmu_context.h>
PATCH v2 30/30 x86/fpu: Provide a proper function for ex_handler_fprestore()
In reply to Thomas GleixnerTo make upcoming changes for support of dynamically enabled features simpler, provide a proper function for the exception handler which removes exposure of FPU internals.
--- arch/x86/include/asm/fpu/api.h | 4 +--- arch/x86/kernel/fpu/core.c | 5 +++++ arch/x86/kernel/fpu/internal.h | 2 ++ arch/x86/mm/extable.c | 5 ++--- 4 files changed, 10 insertions(+), 6 deletions(-)
---
arch/x86/include/asm/fpu/api.h +1 -3
@@ -113,6 +113,7 @@ static inline void update_pasid(void) { }/* Trap handling */ extern int fpu__exception_code (struct fpu *fpu,int trap_nr);extern void fpu_sync_fpstate (struct fpu *fpu); +extern void fpu_reset_from_exception_fixup (void );/* Boot, hotplug and resume */ extern void fpu__init_cpu (void );
@@ -129,9 +130,6 @@ static inline void fpstate_init_soft(struct swregs_state *soft) {}/* State tracking */ DECLARE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx); -/* fpstate */ -extern union fpregs_state init_fpstate; -/* fpstate-related functions which are exported to KVM */ extern void fpu_init_fpstate_user (struct fpu *fpu);
arch/x86/kernel/fpu/core.c +5 -0
@@ -155,6 +155,11 @@ void restore_fpregs_from_fpstate(union fpregs_state *fpstate, u64 mask) } } +void fpu_reset_from_exception_fixup (void ) +{ +restore_fpregs_from_fpstate (&init_fpstate,xfeatures_mask_fpstate ()); +} +#if IS_ENABLED (CONFIG_KVM )void fpu_swap_kvm_fpu (struct fpu *save,struct fpu *rstor,u64 restore_mask) {
arch/x86/kernel/fpu/internal.h +2 -0
@@ -2,6 +2,8 @@#ifndef __X86_KERNEL_FPU_INTERNAL_H#define __X86_KERNEL_FPU_INTERNAL_H +extern union fpregs_state init_fpstate; +/* CPU feature check wrappers */ static __always_inline __pure bool use_xsave(void) {
arch/x86/mm/extable.c +2 -3
@@ -4,8 +4,7 @@#include <linux/sched/debug.h> #include <xen/xen.h> -#include <asm/fpu/signal.h> -#include <asm/fpu/xstate.h> +#include <asm/fpu/api.h> #include <asm/sev.h> #include <asm/traps.h> #include <asm/kdebug.h>
@@ -48,7 +47,7 @@ static bool ex_handler_fprestore(const struct exception_table_entry *fixup,WARN_ONCE (1 ,"Bad FPU state detected at %pB, reinitializing FPU registers." , (void *)instruction_pointer (regs)); -restore_fpregs_from_fpstate (&init_fpstate,xfeatures_mask_fpstate ()); +fpu_reset_from_exception_fixup ();return true; }
[tip: x86/fpu] x86/fpu: Provide a proper function for ex_handler_fprestore()
The following commit has been merged into the x86/fpu branch of tip:
The following commit has been merged into the x86/fpu branch of tip:
Commit-ID: 079ec41b22b952cdf3126527d735e373c9125f6d Gitweb: https://git.kernel.org/tip/079ec41b22b952cdf3126527d735e373c9125f6d Author: Thomas Gleixner <[email protected]> AuthorDate: Fri, 15 Oct 2021 03:16:41 +02:00 Committer: Borislav Petkov <[email protected]> CommitterDate: Wed, 20 Oct 2021 15:27:29 +02:00
x86/fpu: Provide a proper function for ex_handler_fprestore()
To make upcoming changes for support of dynamically enabled features simpler, provide a proper function for the exception handler which removes exposure of FPU internals.
--- arch/x86/include/asm/fpu/api.h | 4 +--- arch/x86/kernel/fpu/core.c | 5 +++++ arch/x86/kernel/fpu/internal.h | 2 ++ arch/x86/mm/extable.c | 5 ++--- 4 files changed, 10 insertions(+), 6 deletions(-)
arch/x86/include/asm/fpu/api.h +1 -3
@@ -113,6 +113,7 @@ static inline void update_pasid(void) { }/* Trap handling */ extern int fpu__exception_code (struct fpu *fpu,int trap_nr);extern void fpu_sync_fpstate (struct fpu *fpu); +extern void fpu_reset_from_exception_fixup (void );/* Boot, hotplug and resume */ extern void fpu__init_cpu (void );
@@ -129,9 +130,6 @@ static inline void fpstate_init_soft(struct swregs_state *soft) {}/* State tracking */ DECLARE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx); -/* fpstate */ -extern union fpregs_state init_fpstate; -/* fpstate-related functions which are exported to KVM */ extern void fpu_init_fpstate_user (struct fpu *fpu);
arch/x86/kernel/fpu/core.c +5 -0
@@ -155,6 +155,11 @@ void restore_fpregs_from_fpstate(union fpregs_state *fpstate, u64 mask) } } +void fpu_reset_from_exception_fixup (void ) +{ +restore_fpregs_from_fpstate (&init_fpstate,xfeatures_mask_fpstate ()); +} +#if IS_ENABLED (CONFIG_KVM )void fpu_swap_kvm_fpu (struct fpu *save,struct fpu *rstor,u64 restore_mask) {
arch/x86/kernel/fpu/internal.h +2 -0
@@ -2,6 +2,8 @@#ifndef __X86_KERNEL_FPU_INTERNAL_H#define __X86_KERNEL_FPU_INTERNAL_H +extern union fpregs_state init_fpstate; +/* CPU feature check wrappers */ static __always_inline __pure bool use_xsave(void) {
arch/x86/mm/extable.c +2 -3
@@ -4,8 +4,7 @@#include <linux/sched/debug.h> #include <xen/xen.h> -#include <asm/fpu/signal.h> -#include <asm/fpu/xstate.h> +#include <asm/fpu/api.h> #include <asm/sev.h> #include <asm/traps.h> #include <asm/kdebug.h>
@@ -48,7 +47,7 @@ static bool ex_handler_fprestore(const struct exception_table_entry *fixup,WARN_ONCE (1 ,"Bad FPU state detected at %pB, reinitializing FPU registers." , (void *)instruction_pointer (regs)); -restore_fpregs_from_fpstate (&init_fpstate,xfeatures_mask_fpstate ()); +fpu_reset_from_exception_fixup ();return true; }