x86: enable FRED for x86-64
Subsystems (15)
[tip: x86/fred] x86/fred: FRED entry/exit and dispatch code
In reply to Xin LiThe following commit has been merged into the x86/fred branch of tip:
The following commit has been merged into the x86/fred branch of tip:
Commit-ID: 14619d912b658ecd9573fb88400d3830a29cadcb Gitweb: https://git.kernel.org/tip/14619d912b658ecd9573fb88400d3830a29cadcb Author: H. Peter Anvin (Intel) <[email protected]> AuthorDate: Sat, 09 Dec 2023 13:42:14 -08:00 Committer: Borislav Petkov (AMD) <[email protected]> CommitterDate: Wed, 31 Jan 2024 22:02:31 +01:00
x86/fred: FRED entry/exit and dispatch code
The code to actually handle kernel and event entry/exit using FRED. It is split up into two files thus:
- entry_64_fred.S contains the actual entrypoints and exit code, and saves and restores registers.
- entry_fred.c contains the two-level event dispatch code for FRED. The first-level dispatch is on the event type, and the second-level is on the event vector.
[ bp: Fold in an allmodconfig clang build fix:
https://lore.kernel.org/r/[email protected] and a CONFIG_IA32_EMULATION=n build fix: https://lore.kernel.org/r/[email protected]]
--- arch/x86/entry/Makefile | 5 +- arch/x86/entry/entry_64_fred.S | 50 +++++- arch/x86/entry/entry_fred.c | 245 +++++++++++++++++++++++++- arch/x86/include/asm/asm-prototypes.h | 1 +- arch/x86/include/asm/fred.h | 6 +- arch/x86/include/asm/ia32.h | 4 +- 6 files changed, 308 insertions(+), 3 deletions(-) create mode 100644 arch/x86/entry/entry_64_fred.S create mode 100644 arch/x86/entry/entry_fred.c
arch/x86/entry/Makefile +4 -1
@@ -18,6 +18,9 @@ obj-y += vdso/ obj-y += vsyscall/ obj-$(CONFIG_PREEMPTION) += thunk_$(BITS).o +CFLAGS_entry_fred.o += -fno-stack-protector +CFLAGS_REMOVE_entry_fred.o += -pg $(CC_FLAGS_FTRACE) +obj-$(CONFIG_X86_FRED) += entry_64_fred.o entry_fred.o + obj-$(CONFIG_IA32_EMULATION) += entry_64_compat.o syscall_32.o obj-$(CONFIG_X86_X32_ABI) += syscall_x32.o -
arch/x86/entry/entry_64_fred.S new file +50 -0
50 lines not on this page · raw message
arch/x86/entry/entry_fred.c new file +245 -0
245 lines not on this page · raw message
arch/x86/include/asm/asm-prototypes.h +1 -0
@@ -12,6 +12,7 @@#include <asm/special_insns.h> #include <asm/preempt.h> #include <asm/asm.h> +#include <asm/fred.h> #include <asm/gsseg.h> #ifndef CONFIG_X86_CMPXCHG64
arch/x86/include/asm/fred.h +6 -0
@@ -60,6 +60,12 @@ static __always_inline unsigned long fred_event_data(struct pt_regs *regs)return fred_info (regs)->edata; } +void asm_fred_entrypoint_user (void ); +void asm_fred_entrypoint_kernel (void ); + +__visible void fred_entry_from_user(struct pt_regs *regs); +__visible void fred_entry_from_kernel(struct pt_regs *regs); +#else /* CONFIG_X86_FRED */ static __always_inline unsigned long fred_event_data (struct pt_regs *regs) {return 0 ; }#endif /* CONFIG_X86_FRED */
arch/x86/include/asm/ia32.h +2 -2
@@ -69,7 +69,7 @@ extern void ia32_pick_mmap_layout(struct mm_struct *mm);extern bool __ia32_enabled; -static inline bool ia32_enabled (void ) +static __always_inline bool ia32_enabled(void) {return __ia32_enabled; }
@@ -81,7 +81,7 @@ static inline void ia32_disable(void)#else /* !CONFIG_IA32_EMULATION */ -static inline bool ia32_enabled (void ) +static __always_inline bool ia32_enabled(void) {return IS_ENABLED (CONFIG_X86_32 ); }
PATCH v13 27/35 x86/traps: Add sysvec_install() to install a system interrupt handler
In reply to Xin LiAdd sysvec_install() to install a system interrupt handler into the IDT or the FRED system interrupt handler table.
---
Changes since v8:
* Introduce a macro sysvec_install() to derive the asm handler name from a C handler, which simplifies the code and avoids an ugly typecast (Thomas Gleixner).
--- arch/x86/entry/entry_fred.c | 14 ++++++++++++++ arch/x86/include/asm/desc.h | 2 -- arch/x86/include/asm/idtentry.h | 15 +++++++++++++++ arch/x86/kernel/cpu/acrn.c | 4 ++-- arch/x86/kernel/cpu/mshyperv.c | 15 +++++++-------- arch/x86/kernel/idt.c | 4 ++-- arch/x86/kernel/kvm.c | 2 +- drivers/xen/events/events_base.c | 2 +- 8 files changed, 42 insertions(+), 16 deletions(-)
arch/x86/entry/entry_fred.c +14 -0
@@ -126,6 +126,20 @@ static idtentry_t sysvec_table[NR_SYSTEM_VECTORS] __ro_after_init = { SYSVEC(POSTED_INTR_NESTED_VECTOR, kvm_posted_intr_nested_ipi), }; +static bool fred_setup_done__initdata ; + +void __init fred_install_sysvec (unsigned int sysvec,idtentry_t handler) +{ +if (WARN_ON_ONCE (sysvec <FIRST_SYSTEM_VECTOR )) +return ; + +if (WARN_ON_ONCE (fred_setup_done)) +return ; + +if (!WARN_ON_ONCE (sysvec_table[sysvec -FIRST_SYSTEM_VECTOR ])) + sysvec_table[sysvec -FIRST_SYSTEM_VECTOR ] = handler; +} + static noinstr void fred_extint(struct pt_regs *regs) {unsigned int vector = regs->fred_ss.vector;
arch/x86/include/asm/desc.h +0 -2
@@ -402,8 +402,6 @@ static inline void set_desc_limit(struct desc_struct *desc, unsigned long limit) desc->limit1 = (limit >>16 ) &0xf ; } -void alloc_intr_gate (unsigned int n,const void *addr); -static inline void init_idt_data (struct idt_data *data,unsigned int n,const void *addr) {
arch/x86/include/asm/idtentry.h +15 -0
@@ -459,6 +459,21 @@ __visible noinstr void func(struct pt_regs *regs, \#define DEFINE_FREDENTRY_DEBUG DEFINE_FREDENTRY_RAW#endif +void idt_install_sysvec (unsigned int n,const void *function); + +#ifdef CONFIG_X86_FRED +void fred_install_sysvec (unsigned int vector,const idtentry_t function); +#else +static inline void fred_install_sysvec (unsigned int vector,const idtentry_t function) { } +#endif + +#define sysvec_install (vector, function) { \ + if (cpu_feature_enabled(X86_FEATURE_FRED)) \ + fred_install_sysvec(vector, function); \ + else \ + idt_install_sysvec(vector, asm_##function); \ +} +#else /* !__ASSEMBLY__ */ /*
arch/x86/kernel/cpu/acrn.c +2 -2
@@ -26,8 +26,8 @@ static u32 __init acrn_detect(void)static void __init acrn_init_platform (void ) { -/* Setup the IDT for ACRN hypervisor callback */ -alloc_intr_gate (HYPERVISOR_CALLBACK_VECTOR , asm_sysvec_acrn_hv_callback); +/* Install system interrupt handler for ACRN hypervisor callback */ +sysvec_install (HYPERVISOR_CALLBACK_VECTOR , sysvec_acrn_hv_callback); x86_platform.calibrate_tsc = acrn_get_tsc_khz; x86_platform.calibrate_cpu = acrn_get_tsc_khz;
arch/x86/kernel/cpu/mshyperv.c +7 -8
@@ -539,19 +539,18 @@ static void __init ms_hyperv_init_platform(void)*/ x86_platform.apic_post_init = hyperv_init;hyperv_setup_mmu_ops (); -/* Setup the IDT for hypervisor callback */ -alloc_intr_gate (HYPERVISOR_CALLBACK_VECTOR , asm_sysvec_hyperv_callback); -/* Setup the IDT for reenlightenment notifications */ +/* Install system interrupt handler for hypervisor callback */ +sysvec_install (HYPERVISOR_CALLBACK_VECTOR , sysvec_hyperv_callback); + +/* Install system interrupt handler for reenlightenment notifications */ if (ms_hyperv.features &HV_ACCESS_REENLIGHTENMENT ) { -alloc_intr_gate (HYPERV_REENLIGHTENMENT_VECTOR , - asm_sysvec_hyperv_reenlightenment); +sysvec_install (HYPERV_REENLIGHTENMENT_VECTOR , sysvec_hyperv_reenlightenment); } -/* Setup the IDT for stimer0 */ +/* Install system interrupt handler for stimer0 */ if (ms_hyperv.misc_features &HV_STIMER_DIRECT_MODE_AVAILABLE ) { -alloc_intr_gate (HYPERV_STIMER0_VECTOR , - asm_sysvec_hyperv_stimer0); +sysvec_install (HYPERV_STIMER0_VECTOR , sysvec_hyperv_stimer0); }# ifdef CONFIG_SMP
arch/x86/kernel/idt.c +2 -2
@@ -337,7 +337,7 @@ void idt_invalidate(void)load_idt (&idt); } -void __init alloc_intr_gate (unsigned int n,const void *addr) +void __init idt_install_sysvec (unsigned int n,const void *function) {if (WARN_ON (n <FIRST_SYSTEM_VECTOR ))return ;
@@ -346,5 +346,5 @@ void __init alloc_intr_gate(unsigned int n, const void *addr)return ;if (!WARN_ON (test_and_set_bit (n, system_vectors))) -set_intr_gate (n, addr); +set_intr_gate (n, function); }
arch/x86/kernel/kvm.c +1 -1
@@ -829,7 +829,7 @@ static void __init kvm_guest_init(void)if (kvm_para_has_feature (KVM_FEATURE_ASYNC_PF_INT ) && kvmapf) {static_branch_enable (&kvm_async_pf_enabled); -alloc_intr_gate (HYPERVISOR_CALLBACK_VECTOR , asm_sysvec_kvm_asyncpf_interrupt); +sysvec_install (HYPERVISOR_CALLBACK_VECTOR , sysvec_kvm_asyncpf_interrupt); }#ifdef CONFIG_SMP
drivers/xen/events/events_base.c +1 -1
@@ -2216,7 +2216,7 @@ static __init void xen_alloc_callback_vector(void)return ;pr_info ("Xen HVM callback vector for event delivery is enabled\n" ); - alloc_intr_gate(HYPERVISOR_CALLBACK_VECTOR, asm_sysvec_xen_hvm_callback); + sysvec_install(HYPERVISOR_CALLBACK_VECTOR, sysvec_xen_hvm_callback); }#else void xen_setup_callback_vector (void ) {}
Signature · 1 line
2.43.0
[tip: x86/fred] x86/traps: Add sysvec_install() to install a system interrupt handler
The following commit has been merged into the x86/fred branch of tip:
The following commit has been merged into the x86/fred branch of tip:
Commit-ID: d1d736f1abf6996d7277eebd3cbf9173c8e430f7 Gitweb: https://git.kernel.org/tip/d1d736f1abf6996d7277eebd3cbf9173c8e430f7 Author: Xin Li <[email protected]> AuthorDate: Tue, 05 Dec 2023 02:50:16 -08:00 Committer: Thomas Gleixner <[email protected]> CommitterDate: Thu, 25 Jan 2024 19:10:32 +01:00
x86/traps: Add sysvec_install() to install a system interrupt handler
Add sysvec_install() to install a system interrupt handler into the IDT or the FRED system interrupt handler table.
--- arch/x86/entry/entry_fred.c | 14 ++++++++++++++ arch/x86/include/asm/desc.h | 2 -- arch/x86/include/asm/idtentry.h | 15 +++++++++++++++ arch/x86/kernel/cpu/acrn.c | 4 ++-- arch/x86/kernel/cpu/mshyperv.c | 15 +++++++-------- arch/x86/kernel/idt.c | 4 ++-- arch/x86/kernel/kvm.c | 2 +- drivers/xen/events/events_base.c | 2 +- 8 files changed, 42 insertions(+), 16 deletions(-)
arch/x86/entry/entry_fred.c +14 -0
@@ -117,6 +117,20 @@ static idtentry_t sysvec_table[NR_SYSTEM_VECTORS] __ro_after_init = { SYSVEC(POSTED_INTR_NESTED_VECTOR, kvm_posted_intr_nested_ipi), }; +static bool fred_setup_done__initdata ; + +void __init fred_install_sysvec (unsigned int sysvec,idtentry_t handler) +{ +if (WARN_ON_ONCE (sysvec <FIRST_SYSTEM_VECTOR )) +return ; + +if (WARN_ON_ONCE (fred_setup_done)) +return ; + +if (!WARN_ON_ONCE (sysvec_table[sysvec -FIRST_SYSTEM_VECTOR ])) + sysvec_table[sysvec -FIRST_SYSTEM_VECTOR ] = handler; +} + static noinstr void fred_extint(struct pt_regs *regs) {unsigned int vector = regs->fred_ss.vector;
arch/x86/include/asm/desc.h +0 -2
@@ -402,8 +402,6 @@ static inline void set_desc_limit(struct desc_struct *desc, unsigned long limit) desc->limit1 = (limit >>16 ) &0xf ; } -void alloc_intr_gate (unsigned int n,const void *addr); -static inline void init_idt_data (struct idt_data *data,unsigned int n,const void *addr) {
arch/x86/include/asm/idtentry.h +15 -0
@@ -459,6 +459,21 @@ __visible noinstr void func(struct pt_regs *regs, \#define DEFINE_FREDENTRY_DEBUG DEFINE_FREDENTRY_RAW#endif +void idt_install_sysvec (unsigned int n,const void *function); + +#ifdef CONFIG_X86_FRED +void fred_install_sysvec (unsigned int vector,const idtentry_t function); +#else +static inline void fred_install_sysvec (unsigned int vector,const idtentry_t function) { } +#endif + +#define sysvec_install (vector, function) { \ + if (cpu_feature_enabled(X86_FEATURE_FRED)) \ + fred_install_sysvec(vector, function); \ + else \ + idt_install_sysvec(vector, asm_##function); \ +} +#else /* !__ASSEMBLY__ */ /*
arch/x86/kernel/cpu/acrn.c +2 -2
@@ -26,8 +26,8 @@ static u32 __init acrn_detect(void)static void __init acrn_init_platform (void ) { -/* Setup the IDT for ACRN hypervisor callback */ -alloc_intr_gate (HYPERVISOR_CALLBACK_VECTOR , asm_sysvec_acrn_hv_callback); +/* Install system interrupt handler for ACRN hypervisor callback */ +sysvec_install (HYPERVISOR_CALLBACK_VECTOR , sysvec_acrn_hv_callback); x86_platform.calibrate_tsc = acrn_get_tsc_khz; x86_platform.calibrate_cpu = acrn_get_tsc_khz;
arch/x86/kernel/cpu/mshyperv.c +7 -8
@@ -539,19 +539,18 @@ static void __init ms_hyperv_init_platform(void)*/ x86_platform.apic_post_init = hyperv_init;hyperv_setup_mmu_ops (); -/* Setup the IDT for hypervisor callback */ -alloc_intr_gate (HYPERVISOR_CALLBACK_VECTOR , asm_sysvec_hyperv_callback); -/* Setup the IDT for reenlightenment notifications */ +/* Install system interrupt handler for hypervisor callback */ +sysvec_install (HYPERVISOR_CALLBACK_VECTOR , sysvec_hyperv_callback); + +/* Install system interrupt handler for reenlightenment notifications */ if (ms_hyperv.features &HV_ACCESS_REENLIGHTENMENT ) { -alloc_intr_gate (HYPERV_REENLIGHTENMENT_VECTOR , - asm_sysvec_hyperv_reenlightenment); +sysvec_install (HYPERV_REENLIGHTENMENT_VECTOR , sysvec_hyperv_reenlightenment); } -/* Setup the IDT for stimer0 */ +/* Install system interrupt handler for stimer0 */ if (ms_hyperv.misc_features &HV_STIMER_DIRECT_MODE_AVAILABLE ) { -alloc_intr_gate (HYPERV_STIMER0_VECTOR , - asm_sysvec_hyperv_stimer0); +sysvec_install (HYPERV_STIMER0_VECTOR , sysvec_hyperv_stimer0); }# ifdef CONFIG_SMP
arch/x86/kernel/idt.c +2 -2
@@ -337,7 +337,7 @@ void idt_invalidate(void)load_idt (&idt); } -void __init alloc_intr_gate (unsigned int n,const void *addr) +void __init idt_install_sysvec (unsigned int n,const void *function) {if (WARN_ON (n <FIRST_SYSTEM_VECTOR ))return ;
@@ -346,5 +346,5 @@ void __init alloc_intr_gate(unsigned int n, const void *addr)return ;if (!WARN_ON (test_and_set_bit (n, system_vectors))) -set_intr_gate (n, addr); +set_intr_gate (n, function); }
arch/x86/kernel/kvm.c +1 -1
@@ -829,7 +829,7 @@ static void __init kvm_guest_init(void)if (kvm_para_has_feature (KVM_FEATURE_ASYNC_PF_INT ) && kvmapf) {static_branch_enable (&kvm_async_pf_enabled); -alloc_intr_gate (HYPERVISOR_CALLBACK_VECTOR , asm_sysvec_kvm_asyncpf_interrupt); +sysvec_install (HYPERVISOR_CALLBACK_VECTOR , sysvec_kvm_asyncpf_interrupt); }#ifdef CONFIG_SMP
drivers/xen/events/events_base.c +1 -1
@@ -2216,7 +2216,7 @@ static __init void xen_alloc_callback_vector(void)return ;pr_info ("Xen HVM callback vector for event delivery is enabled\n" ); - alloc_intr_gate(HYPERVISOR_CALLBACK_VECTOR, asm_sysvec_xen_hvm_callback); + sysvec_install(HYPERVISOR_CALLBACK_VECTOR, sysvec_xen_hvm_callback); }#else void xen_setup_callback_vector (void ) {}
[tip: x86/fred] x86/traps: Add sysvec_install() to install a system interrupt handler
In reply to Xin LiThe following commit has been merged into the x86/fred branch of tip:
The following commit has been merged into the x86/fred branch of tip:
Commit-ID: db7c787d8ba268a8d8beabb0027715246375c6e0 Gitweb: https://git.kernel.org/tip/db7c787d8ba268a8d8beabb0027715246375c6e0 Author: Xin Li <[email protected]> AuthorDate: Tue, 05 Dec 2023 02:50:16 -08:00 Committer: Borislav Petkov (AMD) <[email protected]> CommitterDate: Tue, 30 Jan 2024 18:20:35 +01:00
x86/traps: Add sysvec_install() to install a system interrupt handler
Add sysvec_install() to install a system interrupt handler into the IDT or the FRED system interrupt handler table.
--- arch/x86/entry/entry_fred.c | 14 ++++++++++++++ arch/x86/include/asm/desc.h | 2 -- arch/x86/include/asm/idtentry.h | 15 +++++++++++++++ arch/x86/kernel/cpu/acrn.c | 4 ++-- arch/x86/kernel/cpu/mshyperv.c | 15 +++++++-------- arch/x86/kernel/idt.c | 4 ++-- arch/x86/kernel/kvm.c | 2 +- drivers/xen/events/events_base.c | 2 +- 8 files changed, 42 insertions(+), 16 deletions(-)
arch/x86/entry/entry_fred.c +14 -0
@@ -119,6 +119,20 @@ static idtentry_t sysvec_table[NR_SYSTEM_VECTORS] __ro_after_init = { SYSVEC(POSTED_INTR_NESTED_VECTOR, kvm_posted_intr_nested_ipi), }; +static bool fred_setup_done__initdata ; + +void __init fred_install_sysvec (unsigned int sysvec,idtentry_t handler) +{ +if (WARN_ON_ONCE (sysvec <FIRST_SYSTEM_VECTOR )) +return ; + +if (WARN_ON_ONCE (fred_setup_done)) +return ; + +if (!WARN_ON_ONCE (sysvec_table[sysvec -FIRST_SYSTEM_VECTOR ])) + sysvec_table[sysvec -FIRST_SYSTEM_VECTOR ] = handler; +} + static noinstr void fred_extint(struct pt_regs *regs) {unsigned int vector = regs->fred_ss.vector;
arch/x86/include/asm/desc.h +0 -2
@@ -402,8 +402,6 @@ static inline void set_desc_limit(struct desc_struct *desc, unsigned long limit) desc->limit1 = (limit >>16 ) &0xf ; } -void alloc_intr_gate (unsigned int n,const void *addr); -static inline void init_idt_data (struct idt_data *data,unsigned int n,const void *addr) {
arch/x86/include/asm/idtentry.h +15 -0
@@ -459,6 +459,21 @@ __visible noinstr void func(struct pt_regs *regs, \#define DEFINE_FREDENTRY_DEBUG DEFINE_FREDENTRY_RAW#endif +void idt_install_sysvec (unsigned int n,const void *function); + +#ifdef CONFIG_X86_FRED +void fred_install_sysvec (unsigned int vector,const idtentry_t function); +#else +static inline void fred_install_sysvec (unsigned int vector,const idtentry_t function) { } +#endif + +#define sysvec_install (vector, function) { \ + if (cpu_feature_enabled(X86_FEATURE_FRED)) \ + fred_install_sysvec(vector, function); \ + else \ + idt_install_sysvec(vector, asm_##function); \ +} +#else /* !__ASSEMBLY__ */ /*
arch/x86/kernel/cpu/acrn.c +2 -2
@@ -26,8 +26,8 @@ static u32 __init acrn_detect(void)static void __init acrn_init_platform (void ) { -/* Setup the IDT for ACRN hypervisor callback */ -alloc_intr_gate (HYPERVISOR_CALLBACK_VECTOR , asm_sysvec_acrn_hv_callback); +/* Install system interrupt handler for ACRN hypervisor callback */ +sysvec_install (HYPERVISOR_CALLBACK_VECTOR , sysvec_acrn_hv_callback); x86_platform.calibrate_tsc = acrn_get_tsc_khz; x86_platform.calibrate_cpu = acrn_get_tsc_khz;
arch/x86/kernel/cpu/mshyperv.c +7 -8
@@ -539,19 +539,18 @@ static void __init ms_hyperv_init_platform(void)*/ x86_platform.apic_post_init = hyperv_init;hyperv_setup_mmu_ops (); -/* Setup the IDT for hypervisor callback */ -alloc_intr_gate (HYPERVISOR_CALLBACK_VECTOR , asm_sysvec_hyperv_callback); -/* Setup the IDT for reenlightenment notifications */ +/* Install system interrupt handler for hypervisor callback */ +sysvec_install (HYPERVISOR_CALLBACK_VECTOR , sysvec_hyperv_callback); + +/* Install system interrupt handler for reenlightenment notifications */ if (ms_hyperv.features &HV_ACCESS_REENLIGHTENMENT ) { -alloc_intr_gate (HYPERV_REENLIGHTENMENT_VECTOR , - asm_sysvec_hyperv_reenlightenment); +sysvec_install (HYPERV_REENLIGHTENMENT_VECTOR , sysvec_hyperv_reenlightenment); } -/* Setup the IDT for stimer0 */ +/* Install system interrupt handler for stimer0 */ if (ms_hyperv.misc_features &HV_STIMER_DIRECT_MODE_AVAILABLE ) { -alloc_intr_gate (HYPERV_STIMER0_VECTOR , - asm_sysvec_hyperv_stimer0); +sysvec_install (HYPERV_STIMER0_VECTOR , sysvec_hyperv_stimer0); }# ifdef CONFIG_SMP
arch/x86/kernel/idt.c +2 -2
@@ -337,7 +337,7 @@ void idt_invalidate(void)load_idt (&idt); } -void __init alloc_intr_gate (unsigned int n,const void *addr) +void __init idt_install_sysvec (unsigned int n,const void *function) {if (WARN_ON (n <FIRST_SYSTEM_VECTOR ))return ;
@@ -346,5 +346,5 @@ void __init alloc_intr_gate(unsigned int n, const void *addr)return ;if (!WARN_ON (test_and_set_bit (n, system_vectors))) -set_intr_gate (n, addr); +set_intr_gate (n, function); }
arch/x86/kernel/kvm.c +1 -1
@@ -829,7 +829,7 @@ static void __init kvm_guest_init(void)if (kvm_para_has_feature (KVM_FEATURE_ASYNC_PF_INT ) && kvmapf) {static_branch_enable (&kvm_async_pf_enabled); -alloc_intr_gate (HYPERVISOR_CALLBACK_VECTOR , asm_sysvec_kvm_asyncpf_interrupt); +sysvec_install (HYPERVISOR_CALLBACK_VECTOR , sysvec_kvm_asyncpf_interrupt); }#ifdef CONFIG_SMP
drivers/xen/events/events_base.c +1 -1
@@ -2216,7 +2216,7 @@ static __init void xen_alloc_callback_vector(void)return ;pr_info ("Xen HVM callback vector for event delivery is enabled\n" ); - alloc_intr_gate(HYPERVISOR_CALLBACK_VECTOR, asm_sysvec_xen_hvm_callback); + sysvec_install(HYPERVISOR_CALLBACK_VECTOR, sysvec_xen_hvm_callback); }#else void xen_setup_callback_vector (void ) {}
[tip: x86/fred] x86/traps: Add sysvec_install() to install a system interrupt handler
In reply to Xin LiThe following commit has been merged into the x86/fred branch of tip:
The following commit has been merged into the x86/fred branch of tip:
Commit-ID: 8f4a29b0e8a40d865040800684d7ff4141c1394f Gitweb: https://git.kernel.org/tip/8f4a29b0e8a40d865040800684d7ff4141c1394f Author: Xin Li <[email protected]> AuthorDate: Tue, 05 Dec 2023 02:50:16 -08:00 Committer: Borislav Petkov (AMD) <[email protected]> CommitterDate: Wed, 31 Jan 2024 22:02:36 +01:00
x86/traps: Add sysvec_install() to install a system interrupt handler
Add sysvec_install() to install a system interrupt handler into the IDT or the FRED system interrupt handler table.
--- arch/x86/entry/entry_fred.c | 14 ++++++++++++++ arch/x86/include/asm/desc.h | 2 -- arch/x86/include/asm/idtentry.h | 15 +++++++++++++++ arch/x86/kernel/cpu/acrn.c | 4 ++-- arch/x86/kernel/cpu/mshyperv.c | 15 +++++++-------- arch/x86/kernel/idt.c | 4 ++-- arch/x86/kernel/kvm.c | 2 +- drivers/xen/events/events_base.c | 2 +- 8 files changed, 42 insertions(+), 16 deletions(-)
arch/x86/entry/entry_fred.c +14 -0
@@ -119,6 +119,20 @@ static idtentry_t sysvec_table[NR_SYSTEM_VECTORS] __ro_after_init = { SYSVEC(POSTED_INTR_NESTED_VECTOR, kvm_posted_intr_nested_ipi), }; +static bool fred_setup_done__initdata ; + +void __init fred_install_sysvec (unsigned int sysvec,idtentry_t handler) +{ +if (WARN_ON_ONCE (sysvec <FIRST_SYSTEM_VECTOR )) +return ; + +if (WARN_ON_ONCE (fred_setup_done)) +return ; + +if (!WARN_ON_ONCE (sysvec_table[sysvec -FIRST_SYSTEM_VECTOR ])) + sysvec_table[sysvec -FIRST_SYSTEM_VECTOR ] = handler; +} + static noinstr void fred_extint(struct pt_regs *regs) {unsigned int vector = regs->fred_ss.vector;
arch/x86/include/asm/desc.h +0 -2
@@ -402,8 +402,6 @@ static inline void set_desc_limit(struct desc_struct *desc, unsigned long limit) desc->limit1 = (limit >>16 ) &0xf ; } -void alloc_intr_gate (unsigned int n,const void *addr); -static inline void init_idt_data (struct idt_data *data,unsigned int n,const void *addr) {
arch/x86/include/asm/idtentry.h +15 -0
@@ -459,6 +459,21 @@ __visible noinstr void func(struct pt_regs *regs, \#define DEFINE_FREDENTRY_DEBUG DEFINE_FREDENTRY_RAW#endif +void idt_install_sysvec (unsigned int n,const void *function); + +#ifdef CONFIG_X86_FRED +void fred_install_sysvec (unsigned int vector,const idtentry_t function); +#else +static inline void fred_install_sysvec (unsigned int vector,const idtentry_t function) { } +#endif + +#define sysvec_install (vector, function) { \ + if (cpu_feature_enabled(X86_FEATURE_FRED)) \ + fred_install_sysvec(vector, function); \ + else \ + idt_install_sysvec(vector, asm_##function); \ +} +#else /* !__ASSEMBLY__ */ /*
arch/x86/kernel/cpu/acrn.c +2 -2
@@ -26,8 +26,8 @@ static u32 __init acrn_detect(void)static void __init acrn_init_platform (void ) { -/* Setup the IDT for ACRN hypervisor callback */ -alloc_intr_gate (HYPERVISOR_CALLBACK_VECTOR , asm_sysvec_acrn_hv_callback); +/* Install system interrupt handler for ACRN hypervisor callback */ +sysvec_install (HYPERVISOR_CALLBACK_VECTOR , sysvec_acrn_hv_callback); x86_platform.calibrate_tsc = acrn_get_tsc_khz; x86_platform.calibrate_cpu = acrn_get_tsc_khz;
arch/x86/kernel/cpu/mshyperv.c +7 -8
@@ -539,19 +539,18 @@ static void __init ms_hyperv_init_platform(void)*/ x86_platform.apic_post_init = hyperv_init;hyperv_setup_mmu_ops (); -/* Setup the IDT for hypervisor callback */ -alloc_intr_gate (HYPERVISOR_CALLBACK_VECTOR , asm_sysvec_hyperv_callback); -/* Setup the IDT for reenlightenment notifications */ +/* Install system interrupt handler for hypervisor callback */ +sysvec_install (HYPERVISOR_CALLBACK_VECTOR , sysvec_hyperv_callback); + +/* Install system interrupt handler for reenlightenment notifications */ if (ms_hyperv.features &HV_ACCESS_REENLIGHTENMENT ) { -alloc_intr_gate (HYPERV_REENLIGHTENMENT_VECTOR , - asm_sysvec_hyperv_reenlightenment); +sysvec_install (HYPERV_REENLIGHTENMENT_VECTOR , sysvec_hyperv_reenlightenment); } -/* Setup the IDT for stimer0 */ +/* Install system interrupt handler for stimer0 */ if (ms_hyperv.misc_features &HV_STIMER_DIRECT_MODE_AVAILABLE ) { -alloc_intr_gate (HYPERV_STIMER0_VECTOR , - asm_sysvec_hyperv_stimer0); +sysvec_install (HYPERV_STIMER0_VECTOR , sysvec_hyperv_stimer0); }# ifdef CONFIG_SMP
arch/x86/kernel/idt.c +2 -2
@@ -337,7 +337,7 @@ void idt_invalidate(void)load_idt (&idt); } -void __init alloc_intr_gate (unsigned int n,const void *addr) +void __init idt_install_sysvec (unsigned int n,const void *function) {if (WARN_ON (n <FIRST_SYSTEM_VECTOR ))return ;
@@ -346,5 +346,5 @@ void __init alloc_intr_gate(unsigned int n, const void *addr)return ;if (!WARN_ON (test_and_set_bit (n, system_vectors))) -set_intr_gate (n, addr); +set_intr_gate (n, function); }
arch/x86/kernel/kvm.c +1 -1
@@ -829,7 +829,7 @@ static void __init kvm_guest_init(void)if (kvm_para_has_feature (KVM_FEATURE_ASYNC_PF_INT ) && kvmapf) {static_branch_enable (&kvm_async_pf_enabled); -alloc_intr_gate (HYPERVISOR_CALLBACK_VECTOR , asm_sysvec_kvm_asyncpf_interrupt); +sysvec_install (HYPERVISOR_CALLBACK_VECTOR , sysvec_kvm_asyncpf_interrupt); }#ifdef CONFIG_SMP
drivers/xen/events/events_base.c +1 -1
@@ -2216,7 +2216,7 @@ static __init void xen_alloc_callback_vector(void)return ;pr_info ("Xen HVM callback vector for event delivery is enabled\n" ); - alloc_intr_gate(HYPERVISOR_CALLBACK_VECTOR, asm_sysvec_xen_hvm_callback); + sysvec_install(HYPERVISOR_CALLBACK_VECTOR, sysvec_xen_hvm_callback); }#else void xen_setup_callback_vector (void ) {}
PATCH v13 28/35 x86/fred: Let ret_from_fork_asm() jmp to asm_fred_exit_user when FRED is enabled
In reply to Xin LiFrom: "H. Peter Anvin (Intel)" <[email protected]>
Let ret_from_fork_asm() jmp to asm_fred_exit_user when FRED is enabled, otherwise the existing IDT code is chosen.
--- arch/x86/entry/entry_64.S | 6 ++++++ arch/x86/entry/entry_64_fred.S | 1 + 2 files changed, 7 insertions(+)
arch/x86/entry/entry_64.S +6 -0
@@ -247,7 +247,13 @@ SYM_CODE_START(ret_from_fork_asm) * and unwind should work normally. */ UNWIND_HINT_REGS + +#ifdef CONFIG_X86_FRED + ALTERNATIVE "jmp swapgs_restore_regs_and_return_to_usermode", \ + "jmp asm_fred_exit_user", X86_FEATURE_FRED +#else jmp swapgs_restore_regs_and_return_to_usermode +#endif SYM_CODE_END(ret_from_fork_asm) .popsection
arch/x86/entry/entry_64_fred.S +1 -0
@@ -32,6 +32,7 @@ SYM_CODE_START_NOALIGN(asm_fred_entrypoint_user) FRED_ENTER call fred_entry_from_user +SYM_INNER_LABEL(asm_fred_exit_user, SYM_L_GLOBAL) FRED_EXIT ERETU SYM_CODE_END(asm_fred_entrypoint_user)
Signature · 1 line
2.43.0
[tip: x86/fred] x86/fred: Let ret_from_fork_asm() jmp to asm_fred_exit_user when FRED is enabled
The following commit has been merged into the x86/fred branch of tip:
The following commit has been merged into the x86/fred branch of tip:
Commit-ID: 81e71a71266e2fbe42632f725147ed5b8f75b11e Gitweb: https://git.kernel.org/tip/81e71a71266e2fbe42632f725147ed5b8f75b11e Author: H. Peter Anvin (Intel) <[email protected]> AuthorDate: Tue, 05 Dec 2023 02:50:17 -08:00 Committer: Thomas Gleixner <[email protected]> CommitterDate: Thu, 25 Jan 2024 19:10:33 +01:00
x86/fred: Let ret_from_fork_asm() jmp to asm_fred_exit_user when FRED is enabled
Let ret_from_fork_asm() jmp to asm_fred_exit_user when FRED is enabled, otherwise the existing IDT code is chosen.
--- arch/x86/entry/entry_64.S | 6 ++++++ arch/x86/entry/entry_64_fred.S | 1 + 2 files changed, 7 insertions(+)
arch/x86/entry/entry_64.S +6 -0
@@ -247,7 +247,13 @@ SYM_CODE_START(ret_from_fork_asm) * and unwind should work normally. */ UNWIND_HINT_REGS + +#ifdef CONFIG_X86_FRED + ALTERNATIVE "jmp swapgs_restore_regs_and_return_to_usermode", \ + "jmp asm_fred_exit_user", X86_FEATURE_FRED +#else jmp swapgs_restore_regs_and_return_to_usermode +#endif SYM_CODE_END(ret_from_fork_asm) .popsection
arch/x86/entry/entry_64_fred.S +1 -0
@@ -32,6 +32,7 @@ SYM_CODE_START_NOALIGN(asm_fred_entrypoint_user) FRED_ENTER call fred_entry_from_user +SYM_INNER_LABEL(asm_fred_exit_user, SYM_L_GLOBAL) FRED_EXIT ERETU SYM_CODE_END(asm_fred_entrypoint_user)
[tip: x86/fred] x86/fred: Let ret_from_fork_asm() jmp to asm_fred_exit_user when FRED is enabled
In reply to Xin LiThe following commit has been merged into the x86/fred branch of tip:
The following commit has been merged into the x86/fred branch of tip:
Commit-ID: 531ff17a705a0f0ecafb8823956f69d5fbfda6fd Gitweb: https://git.kernel.org/tip/531ff17a705a0f0ecafb8823956f69d5fbfda6fd Author: H. Peter Anvin (Intel) <[email protected]> AuthorDate: Tue, 05 Dec 2023 02:50:17 -08:00 Committer: Borislav Petkov (AMD) <[email protected]> CommitterDate: Tue, 30 Jan 2024 18:20:36 +01:00
x86/fred: Let ret_from_fork_asm() jmp to asm_fred_exit_user when FRED is enabled
Let ret_from_fork_asm() jmp to asm_fred_exit_user when FRED is enabled, otherwise the existing IDT code is chosen.
--- arch/x86/entry/entry_64.S | 6 ++++++ arch/x86/entry/entry_64_fred.S | 1 + 2 files changed, 7 insertions(+)
arch/x86/entry/entry_64.S +6 -0
@@ -247,7 +247,13 @@ SYM_CODE_START(ret_from_fork_asm) * and unwind should work normally. */ UNWIND_HINT_REGS + +#ifdef CONFIG_X86_FRED + ALTERNATIVE "jmp swapgs_restore_regs_and_return_to_usermode", \ + "jmp asm_fred_exit_user", X86_FEATURE_FRED +#else jmp swapgs_restore_regs_and_return_to_usermode +#endif SYM_CODE_END(ret_from_fork_asm) .popsection
arch/x86/entry/entry_64_fred.S +1 -0
@@ -32,6 +32,7 @@ SYM_CODE_START_NOALIGN(asm_fred_entrypoint_user) FRED_ENTER call fred_entry_from_user +SYM_INNER_LABEL(asm_fred_exit_user, SYM_L_GLOBAL) FRED_EXIT ERETU SYM_CODE_END(asm_fred_entrypoint_user)
[tip: x86/fred] x86/fred: Let ret_from_fork_asm() jmp to asm_fred_exit_user when FRED is enabled
In reply to Xin LiThe following commit has been merged into the x86/fred branch of tip:
The following commit has been merged into the x86/fred branch of tip:
Commit-ID: 51ef2a4da7ec347e3315af69a426ac36fab98a6c Gitweb: https://git.kernel.org/tip/51ef2a4da7ec347e3315af69a426ac36fab98a6c Author: H. Peter Anvin (Intel) <[email protected]> AuthorDate: Tue, 05 Dec 2023 02:50:17 -08:00 Committer: Borislav Petkov (AMD) <[email protected]> CommitterDate: Wed, 31 Jan 2024 22:02:59 +01:00
x86/fred: Let ret_from_fork_asm() jmp to asm_fred_exit_user when FRED is enabled
Let ret_from_fork_asm() jmp to asm_fred_exit_user when FRED is enabled, otherwise the existing IDT code is chosen.
--- arch/x86/entry/entry_64.S | 6 ++++++ arch/x86/entry/entry_64_fred.S | 1 + 2 files changed, 7 insertions(+)
arch/x86/entry/entry_64.S +6 -0
@@ -247,7 +247,13 @@ SYM_CODE_START(ret_from_fork_asm) * and unwind should work normally. */ UNWIND_HINT_REGS + +#ifdef CONFIG_X86_FRED + ALTERNATIVE "jmp swapgs_restore_regs_and_return_to_usermode", \ + "jmp asm_fred_exit_user", X86_FEATURE_FRED +#else jmp swapgs_restore_regs_and_return_to_usermode +#endif SYM_CODE_END(ret_from_fork_asm) .popsection
arch/x86/entry/entry_64_fred.S +1 -0
@@ -32,6 +32,7 @@ SYM_CODE_START_NOALIGN(asm_fred_entrypoint_user) FRED_ENTER call fred_entry_from_user +SYM_INNER_LABEL(asm_fred_exit_user, SYM_L_GLOBAL) FRED_EXIT ERETU SYM_CODE_END(asm_fred_entrypoint_user)
PATCH v13 29/35 x86/fred: Fixup fault on ERETU by jumping to fred_entrypoint_user
In reply to Xin LiIf the stack frame contains an invalid user context (e.g. due to invalid SS, a non-canonical RIP, etc.) the ERETU instruction will trap (#SS or #GP).
From a Linux point of view, this really should be considered a user space failure, so use the standard fault fixup mechanism to intercept the fault, fix up the exception frame, and redirect execution to fred_entrypoint_user. The end result is that it appears just as if the hardware had taken the exception immediately after completing the transition to user space.
---
Changes since v8:
* Reflect the FRED spec 5.0 change that ERETS and ERETU add 8 to %rsp before popping the return context from the stack.
Changes since v6:
* Add a comment to explain why it is safe to write to the previous FRED stack frame. (Lai Jiangshan).
Changes since v5:
* Move the NMI bit from an invalid stack frame, which caused ERETU to fault, to the fault handler's stack frame, thus to unblock NMI ASAP if NMI is blocked (Lai Jiangshan).
--- arch/x86/entry/entry_64_fred.S | 5 +- arch/x86/include/asm/extable_fixup_types.h | 4 +- arch/x86/mm/extable.c | 79 ++++++++++++++++++++++ 3 files changed, 86 insertions(+), 2 deletions(-)
arch/x86/entry/entry_64_fred.S +4 -1
@@ -3,6 +3,7 @@ * The actual FRED entry points. */ +#include <asm/asm.h> #include <asm/fred.h> #include "calling.h"
@@ -34,7 +35,9 @@ SYM_CODE_START_NOALIGN(asm_fred_entrypoint_user) call fred_entry_from_user SYM_INNER_LABEL(asm_fred_exit_user, SYM_L_GLOBAL) FRED_EXIT - ERETU +1: ERETU + + _ASM_EXTABLE_TYPE(1b, asm_fred_entrypoint_user, EX_TYPE_ERETU) SYM_CODE_END(asm_fred_entrypoint_user) .fill asm_fred_entrypoint_kernel - ., 1, 0xcc
arch/x86/include/asm/extable_fixup_types.h +3 -1
@@ -64,6 +64,8 @@#define EX_TYPE_UCOPY_LEN4 (EX_TYPE_UCOPY_LEN | EX_DATA_IMM(4))#define EX_TYPE_UCOPY_LEN8 (EX_TYPE_UCOPY_LEN | EX_DATA_IMM(8)) -#define EX_TYPE_ZEROPAD 20/* longword load with zeropad on fault */ +#define EX_TYPE_ZEROPAD 20/* longword load with zeropad on fault */ + +#define EX_TYPE_ERETU 21#endif
arch/x86/mm/extable.c +79 -0
97 lines not on this page · raw message
Signature · 1 line
2.43.0
[tip: x86/fred] x86/fred: Fixup fault on ERETU by jumping to fred_entrypoint_user
The following commit has been merged into the x86/fred branch of tip:
The following commit has been merged into the x86/fred branch of tip:
Commit-ID: ea8fd5563541733a309fa00d17fe33912cef2f63 Gitweb: https://git.kernel.org/tip/ea8fd5563541733a309fa00d17fe33912cef2f63 Author: Xin Li <[email protected]> AuthorDate: Tue, 05 Dec 2023 02:50:18 -08:00 Committer: Thomas Gleixner <[email protected]> CommitterDate: Thu, 25 Jan 2024 19:10:33 +01:00
x86/fred: Fixup fault on ERETU by jumping to fred_entrypoint_user
If the stack frame contains an invalid user context (e.g. due to invalid SS, a non-canonical RIP, etc.) the ERETU instruction will trap (#SS or #GP).
Quoted 1 line
From a Linux point of view, this really should be considered a user space
failure, so use the standard fault fixup mechanism to intercept the fault, fix up the exception frame, and redirect execution to fred_entrypoint_user. The end result is that it appears just as if the hardware had taken the exception immediately after completing the transition to user space.
--- arch/x86/entry/entry_64_fred.S | 5 +- arch/x86/include/asm/extable_fixup_types.h | 4 +- arch/x86/mm/extable.c | 78 +++++++++++++++++++++- 3 files changed, 85 insertions(+), 2 deletions(-)
arch/x86/entry/entry_64_fred.S +4 -1
@@ -3,6 +3,7 @@ * The actual FRED entry points. */ +#include <asm/asm.h> #include <asm/fred.h> #include "calling.h"
@@ -34,7 +35,9 @@ SYM_CODE_START_NOALIGN(asm_fred_entrypoint_user) call fred_entry_from_user SYM_INNER_LABEL(asm_fred_exit_user, SYM_L_GLOBAL) FRED_EXIT - ERETU +1: ERETU + + _ASM_EXTABLE_TYPE(1b, asm_fred_entrypoint_user, EX_TYPE_ERETU) SYM_CODE_END(asm_fred_entrypoint_user) .fill asm_fred_entrypoint_kernel - ., 1, 0xcc
arch/x86/include/asm/extable_fixup_types.h +3 -1
@@ -64,6 +64,8 @@#define EX_TYPE_UCOPY_LEN4 (EX_TYPE_UCOPY_LEN | EX_DATA_IMM(4))#define EX_TYPE_UCOPY_LEN8 (EX_TYPE_UCOPY_LEN | EX_DATA_IMM(8)) -#define EX_TYPE_ZEROPAD 20/* longword load with zeropad on fault */ +#define EX_TYPE_ZEROPAD 20/* longword load with zeropad on fault */ + +#define EX_TYPE_ERETU 21#endif
arch/x86/mm/extable.c +78 -0
96 lines not on this page · raw message
[tip: x86/fred] x86/fred: Fixup fault on ERETU by jumping to fred_entrypoint_user
In reply to Xin LiThe following commit has been merged into the x86/fred branch of tip:
The following commit has been merged into the x86/fred branch of tip:
Commit-ID: ed63bc7d4953bd5fe93a5c3acef7f485fb216208 Gitweb: https://git.kernel.org/tip/ed63bc7d4953bd5fe93a5c3acef7f485fb216208 Author: Xin Li <[email protected]> AuthorDate: Tue, 05 Dec 2023 02:50:18 -08:00 Committer: Borislav Petkov (AMD) <[email protected]> CommitterDate: Tue, 30 Jan 2024 18:20:36 +01:00
x86/fred: Fixup fault on ERETU by jumping to fred_entrypoint_user
If the stack frame contains an invalid user context (e.g. due to invalid SS, a non-canonical RIP, etc.) the ERETU instruction will trap (#SS or #GP).
Quoted 1 line
From a Linux point of view, this really should be considered a user space
failure, so use the standard fault fixup mechanism to intercept the fault, fix up the exception frame, and redirect execution to fred_entrypoint_user. The end result is that it appears just as if the hardware had taken the exception immediately after completing the transition to user space.
--- arch/x86/entry/entry_64_fred.S | 5 +- arch/x86/include/asm/extable_fixup_types.h | 4 +- arch/x86/mm/extable.c | 78 +++++++++++++++++++++- 3 files changed, 85 insertions(+), 2 deletions(-)
arch/x86/entry/entry_64_fred.S +4 -1
@@ -3,6 +3,7 @@ * The actual FRED entry points. */ +#include <asm/asm.h> #include <asm/fred.h> #include "calling.h"
@@ -34,7 +35,9 @@ SYM_CODE_START_NOALIGN(asm_fred_entrypoint_user) call fred_entry_from_user SYM_INNER_LABEL(asm_fred_exit_user, SYM_L_GLOBAL) FRED_EXIT - ERETU +1: ERETU + + _ASM_EXTABLE_TYPE(1b, asm_fred_entrypoint_user, EX_TYPE_ERETU) SYM_CODE_END(asm_fred_entrypoint_user) /*
arch/x86/include/asm/extable_fixup_types.h +3 -1
@@ -64,6 +64,8 @@#define EX_TYPE_UCOPY_LEN4 (EX_TYPE_UCOPY_LEN | EX_DATA_IMM(4))#define EX_TYPE_UCOPY_LEN8 (EX_TYPE_UCOPY_LEN | EX_DATA_IMM(8)) -#define EX_TYPE_ZEROPAD 20/* longword load with zeropad on fault */ +#define EX_TYPE_ZEROPAD 20/* longword load with zeropad on fault */ + +#define EX_TYPE_ERETU 21#endif
arch/x86/mm/extable.c +78 -0
96 lines not on this page · raw message
[tip: x86/fred] x86/fred: Fixup fault on ERETU by jumping to fred_entrypoint_user
In reply to Xin LiThe following commit has been merged into the x86/fred branch of tip:
The following commit has been merged into the x86/fred branch of tip:
Commit-ID: 5105e7687ad3dffde77f6e4393b5530e83d672dc Gitweb: https://git.kernel.org/tip/5105e7687ad3dffde77f6e4393b5530e83d672dc Author: Xin Li <[email protected]> AuthorDate: Tue, 05 Dec 2023 02:50:18 -08:00 Committer: Borislav Petkov (AMD) <[email protected]> CommitterDate: Wed, 31 Jan 2024 22:03:04 +01:00
x86/fred: Fixup fault on ERETU by jumping to fred_entrypoint_user
If the stack frame contains an invalid user context (e.g. due to invalid SS, a non-canonical RIP, etc.) the ERETU instruction will trap (#SS or #GP).
Quoted 1 line
From a Linux point of view, this really should be considered a user space
failure, so use the standard fault fixup mechanism to intercept the fault, fix up the exception frame, and redirect execution to fred_entrypoint_user. The end result is that it appears just as if the hardware had taken the exception immediately after completing the transition to user space.
--- arch/x86/entry/entry_64_fred.S | 5 +- arch/x86/include/asm/extable_fixup_types.h | 4 +- arch/x86/mm/extable.c | 78 +++++++++++++++++++++- 3 files changed, 85 insertions(+), 2 deletions(-)
arch/x86/entry/entry_64_fred.S +4 -1
@@ -3,6 +3,7 @@ * The actual FRED entry points. */ +#include <asm/asm.h> #include <asm/fred.h> #include "calling.h"
@@ -34,7 +35,9 @@ SYM_CODE_START_NOALIGN(asm_fred_entrypoint_user) call fred_entry_from_user SYM_INNER_LABEL(asm_fred_exit_user, SYM_L_GLOBAL) FRED_EXIT - ERETU +1: ERETU + + _ASM_EXTABLE_TYPE(1b, asm_fred_entrypoint_user, EX_TYPE_ERETU) SYM_CODE_END(asm_fred_entrypoint_user) /*
arch/x86/include/asm/extable_fixup_types.h +3 -1
@@ -64,6 +64,8 @@#define EX_TYPE_UCOPY_LEN4 (EX_TYPE_UCOPY_LEN | EX_DATA_IMM(4))#define EX_TYPE_UCOPY_LEN8 (EX_TYPE_UCOPY_LEN | EX_DATA_IMM(8)) -#define EX_TYPE_ZEROPAD 20/* longword load with zeropad on fault */ +#define EX_TYPE_ZEROPAD 20/* longword load with zeropad on fault */ + +#define EX_TYPE_ERETU 21#endif
arch/x86/mm/extable.c +78 -0
96 lines not on this page · raw message
PATCH v13 30/35 x86/entry/calling: Allow PUSH_AND_CLEAR_REGS being used beyond actual entry code
In reply to Xin LiFrom: "Peter Zijlstra (Intel)" <[email protected]>
PUSH_AND_CLEAR_REGS could be used besides actual entry code; in that case %rbp shouldn't be cleared (otherwise the frame pointer is destroyed) and UNWIND_HINT shouldn't be added.
--- arch/x86/entry/calling.h | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-)
arch/x86/entry/calling.h +10 -5
@@ -65,7 +65,7 @@ For 32-bit we have the following conventions - kernel is built with* for assembly code: */ -.macro PUSH_REGS rdx=%rdx rcx=%rcx rax=%rax save_ret=0 +.macro PUSH_REGS rdx=%rdx rcx=%rcx rax=%rax save_ret=0 unwind_hint=1 .if \save_ret pushq %rsi /* pt_regs->si */ movq 8(%rsp), %rsi /* temporarily store the return address in %rsi */
@@ -87,14 +87,17 @@ For 32-bit we have the following conventions - kernel is built with pushq %r13 /* pt_regs->r13 */ pushq %r14 /* pt_regs->r14 */ pushq %r15/* pt_regs->r15 */ + + .if \unwind_hint UNWIND_HINT_REGS + .endif .if \save_ret pushq %rsi /* return address on top of stack */ .endif .endm -.macro CLEAR_REGS +.macro CLEAR_REGS clear_bp=1/* * Sanitize registers of values that a speculation attack might * otherwise want to exploit. The lower registers are likely clobbered
@@ -109,7 +112,9 @@ For 32-bit we have the following conventions - kernel is built with xorl %r10d, %r10d /* nospec r10 */ xorl %r11d, %r11d /* nospec r11 */ xorl %ebx, %ebx /* nospec rbx */ + .if \clear_bp xorl %ebp, %ebp /* nospec rbp */ + .endif xorl %r12d, %r12d /* nospec r12 */ xorl %r13d, %r13d /* nospec r13 */ xorl %r14d, %r14d /* nospec r14 */
@@ -117,9 +122,9 @@ For 32-bit we have the following conventions - kernel is built with .endm -.macro PUSH_AND_CLEAR_REGS rdx=%rdx rcx=%rcx rax=%rax save_ret=0 - PUSH_REGS rdx=\rdx, rcx=\rcx, rax=\rax, save_ret=\save_ret - CLEAR_REGS +.macro PUSH_AND_CLEAR_REGS rdx=%rdx rcx=%rcx rax=%rax save_ret=0 clear_bp=1 unwind_hint=1 + PUSH_REGS rdx=\rdx, rcx=\rcx, rax=\rax, save_ret=\save_ret unwind_hint=\unwind_hint + CLEAR_REGS clear_bp=\clear_bp .endm .macro POP_REGS pop_rdi=1
Signature · 1 line
2.43.0
[tip: x86/fred] x86/entry/calling: Allow PUSH_AND_CLEAR_REGS being used beyond actual entry code
The following commit has been merged into the x86/fred branch of tip:
The following commit has been merged into the x86/fred branch of tip:
Commit-ID: eea48ae4c9c766ba24fda42a0b3387c942240bba Gitweb: https://git.kernel.org/tip/eea48ae4c9c766ba24fda42a0b3387c942240bba Author: Peter Zijlstra (Intel) <[email protected]> AuthorDate: Tue, 05 Dec 2023 02:50:19 -08:00 Committer: Thomas Gleixner <[email protected]> CommitterDate: Thu, 25 Jan 2024 19:10:33 +01:00
x86/entry/calling: Allow PUSH_AND_CLEAR_REGS being used beyond actual entry code
PUSH_AND_CLEAR_REGS could be used besides actual entry code; in that case %rbp shouldn't be cleared (otherwise the frame pointer is destroyed) and UNWIND_HINT shouldn't be added.
--- arch/x86/entry/calling.h | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-)
arch/x86/entry/calling.h +10 -5
@@ -65,7 +65,7 @@ For 32-bit we have the following conventions - kernel is built with* for assembly code: */ -.macro PUSH_REGS rdx=%rdx rcx=%rcx rax=%rax save_ret=0 +.macro PUSH_REGS rdx=%rdx rcx=%rcx rax=%rax save_ret=0 unwind_hint=1 .if \save_ret pushq %rsi /* pt_regs->si */ movq 8(%rsp), %rsi /* temporarily store the return address in %rsi */
@@ -87,14 +87,17 @@ For 32-bit we have the following conventions - kernel is built with pushq %r13 /* pt_regs->r13 */ pushq %r14 /* pt_regs->r14 */ pushq %r15/* pt_regs->r15 */ + + .if \unwind_hint UNWIND_HINT_REGS + .endif .if \save_ret pushq %rsi /* return address on top of stack */ .endif .endm -.macro CLEAR_REGS +.macro CLEAR_REGS clear_bp=1/* * Sanitize registers of values that a speculation attack might * otherwise want to exploit. The lower registers are likely clobbered
@@ -109,7 +112,9 @@ For 32-bit we have the following conventions - kernel is built with xorl %r10d, %r10d /* nospec r10 */ xorl %r11d, %r11d /* nospec r11 */ xorl %ebx, %ebx /* nospec rbx */ + .if \clear_bp xorl %ebp, %ebp /* nospec rbp */ + .endif xorl %r12d, %r12d /* nospec r12 */ xorl %r13d, %r13d /* nospec r13 */ xorl %r14d, %r14d /* nospec r14 */
@@ -117,9 +122,9 @@ For 32-bit we have the following conventions - kernel is built with .endm -.macro PUSH_AND_CLEAR_REGS rdx=%rdx rcx=%rcx rax=%rax save_ret=0 - PUSH_REGS rdx=\rdx, rcx=\rcx, rax=\rax, save_ret=\save_ret - CLEAR_REGS +.macro PUSH_AND_CLEAR_REGS rdx=%rdx rcx=%rcx rax=%rax save_ret=0 clear_bp=1 unwind_hint=1 + PUSH_REGS rdx=\rdx, rcx=\rcx, rax=\rax, save_ret=\save_ret unwind_hint=\unwind_hint + CLEAR_REGS clear_bp=\clear_bp .endm .macro POP_REGS pop_rdi=1
[tip: x86/fred] x86/entry/calling: Allow PUSH_AND_CLEAR_REGS being used beyond actual entry code
In reply to Xin LiThe following commit has been merged into the x86/fred branch of tip:
The following commit has been merged into the x86/fred branch of tip:
Commit-ID: 8c968f4df73c62be94229c7dbbb330ba9fadbd50 Gitweb: https://git.kernel.org/tip/8c968f4df73c62be94229c7dbbb330ba9fadbd50 Author: Peter Zijlstra (Intel) <[email protected]> AuthorDate: Tue, 05 Dec 2023 02:50:19 -08:00 Committer: Borislav Petkov (AMD) <[email protected]> CommitterDate: Tue, 30 Jan 2024 18:20:36 +01:00
x86/entry/calling: Allow PUSH_AND_CLEAR_REGS being used beyond actual entry code
PUSH_AND_CLEAR_REGS could be used besides actual entry code; in that case %rbp shouldn't be cleared (otherwise the frame pointer is destroyed) and UNWIND_HINT shouldn't be added.
--- arch/x86/entry/calling.h | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-)
arch/x86/entry/calling.h +10 -5
@@ -65,7 +65,7 @@ For 32-bit we have the following conventions - kernel is built with* for assembly code: */ -.macro PUSH_REGS rdx=%rdx rcx=%rcx rax=%rax save_ret=0 +.macro PUSH_REGS rdx=%rdx rcx=%rcx rax=%rax save_ret=0 unwind_hint=1 .if \save_ret pushq %rsi /* pt_regs->si */ movq 8(%rsp), %rsi /* temporarily store the return address in %rsi */
@@ -87,14 +87,17 @@ For 32-bit we have the following conventions - kernel is built with pushq %r13 /* pt_regs->r13 */ pushq %r14 /* pt_regs->r14 */ pushq %r15/* pt_regs->r15 */ + + .if \unwind_hint UNWIND_HINT_REGS + .endif .if \save_ret pushq %rsi /* return address on top of stack */ .endif .endm -.macro CLEAR_REGS +.macro CLEAR_REGS clear_bp=1/* * Sanitize registers of values that a speculation attack might * otherwise want to exploit. The lower registers are likely clobbered
@@ -109,7 +112,9 @@ For 32-bit we have the following conventions - kernel is built with xorl %r10d, %r10d /* nospec r10 */ xorl %r11d, %r11d /* nospec r11 */ xorl %ebx, %ebx /* nospec rbx */ + .if \clear_bp xorl %ebp, %ebp /* nospec rbp */ + .endif xorl %r12d, %r12d /* nospec r12 */ xorl %r13d, %r13d /* nospec r13 */ xorl %r14d, %r14d /* nospec r14 */
@@ -117,9 +122,9 @@ For 32-bit we have the following conventions - kernel is built with .endm -.macro PUSH_AND_CLEAR_REGS rdx=%rdx rcx=%rcx rax=%rax save_ret=0 - PUSH_REGS rdx=\rdx, rcx=\rcx, rax=\rax, save_ret=\save_ret - CLEAR_REGS +.macro PUSH_AND_CLEAR_REGS rdx=%rdx rcx=%rcx rax=%rax save_ret=0 clear_bp=1 unwind_hint=1 + PUSH_REGS rdx=\rdx, rcx=\rcx, rax=\rax, save_ret=\save_ret unwind_hint=\unwind_hint + CLEAR_REGS clear_bp=\clear_bp .endm .macro POP_REGS pop_rdi=1
[tip: x86/fred] x86/entry/calling: Allow PUSH_AND_CLEAR_REGS being used beyond actual entry code
In reply to Xin LiThe following commit has been merged into the x86/fred branch of tip:
The following commit has been merged into the x86/fred branch of tip:
Commit-ID: 2333f3c473c1562633cd17ac2eb743c29c3b2d9d Gitweb: https://git.kernel.org/tip/2333f3c473c1562633cd17ac2eb743c29c3b2d9d Author: Peter Zijlstra (Intel) <[email protected]> AuthorDate: Tue, 05 Dec 2023 02:50:19 -08:00 Committer: Borislav Petkov (AMD) <[email protected]> CommitterDate: Wed, 31 Jan 2024 22:03:09 +01:00
x86/entry/calling: Allow PUSH_AND_CLEAR_REGS being used beyond actual entry code
PUSH_AND_CLEAR_REGS could be used besides actual entry code; in that case %rbp shouldn't be cleared (otherwise the frame pointer is destroyed) and UNWIND_HINT shouldn't be added.
--- arch/x86/entry/calling.h | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-)
arch/x86/entry/calling.h +10 -5
@@ -65,7 +65,7 @@ For 32-bit we have the following conventions - kernel is built with* for assembly code: */ -.macro PUSH_REGS rdx=%rdx rcx=%rcx rax=%rax save_ret=0 +.macro PUSH_REGS rdx=%rdx rcx=%rcx rax=%rax save_ret=0 unwind_hint=1 .if \save_ret pushq %rsi /* pt_regs->si */ movq 8(%rsp), %rsi /* temporarily store the return address in %rsi */
@@ -87,14 +87,17 @@ For 32-bit we have the following conventions - kernel is built with pushq %r13 /* pt_regs->r13 */ pushq %r14 /* pt_regs->r14 */ pushq %r15/* pt_regs->r15 */ + + .if \unwind_hint UNWIND_HINT_REGS + .endif .if \save_ret pushq %rsi /* return address on top of stack */ .endif .endm -.macro CLEAR_REGS +.macro CLEAR_REGS clear_bp=1/* * Sanitize registers of values that a speculation attack might * otherwise want to exploit. The lower registers are likely clobbered
@@ -109,7 +112,9 @@ For 32-bit we have the following conventions - kernel is built with xorl %r10d, %r10d /* nospec r10 */ xorl %r11d, %r11d /* nospec r11 */ xorl %ebx, %ebx /* nospec rbx */ + .if \clear_bp xorl %ebp, %ebp /* nospec rbp */ + .endif xorl %r12d, %r12d /* nospec r12 */ xorl %r13d, %r13d /* nospec r13 */ xorl %r14d, %r14d /* nospec r14 */
@@ -117,9 +122,9 @@ For 32-bit we have the following conventions - kernel is built with .endm -.macro PUSH_AND_CLEAR_REGS rdx=%rdx rcx=%rcx rax=%rax save_ret=0 - PUSH_REGS rdx=\rdx, rcx=\rcx, rax=\rax, save_ret=\save_ret - CLEAR_REGS +.macro PUSH_AND_CLEAR_REGS rdx=%rdx rcx=%rcx rax=%rax save_ret=0 clear_bp=1 unwind_hint=1 + PUSH_REGS rdx=\rdx, rcx=\rcx, rax=\rax, save_ret=\save_ret unwind_hint=\unwind_hint + CLEAR_REGS clear_bp=\clear_bp .endm .macro POP_REGS pop_rdi=1
PATCH v13 31/35 x86/entry: Add fred_entry_from_kvm() for VMX to handle IRQ/NMI
In reply to Xin LiIn IRQ/NMI induced VM exits, KVM VMX needs to execute the respective handlers, which requires the software to create a FRED stack frame, and use it to invoke the handlers. Add fred_irq_entry_from_kvm() for this job.
Export fred_entry_from_kvm() because VMX can be compiled as a module.
---
Changes since v10:
* Better explain the reason why no need to check current stack level (Paolo Bonzini).
Changes since v9:
* Shove the whole thing into arch/x86/entry/entry_64_fred.S for invoking external_interrupt() and fred_exc_nmi() (Sean Christopherson).
* Correct and improve a few comments (Sean Christopherson).
* Merge the two IRQ/NMI asm entries into one as it's fine to invoke noinstr code from regular code (Thomas Gleixner).
* Setup the long mode and NMI flags in the augmented SS field of FRED stack frame in C instead of asm (Thomas Gleixner).
* Add UNWIND_HINT_{SAVE,RESTORE} to get rid of the warning: "objtool:
asm_fred_entry_from_kvm+0x0: unreachable instruction" (Peter Zijlstra).
Changes since v8:
* Add a new macro VMX_DO_FRED_EVENT_IRQOFF for FRED instead of refactoring VMX_DO_EVENT_IRQOFF (Sean Christopherson).
* Do NOT use a trampoline, just LEA+PUSH the return RIP, PUSH the error code, and jump to the FRED kernel entry point for NMI or call external_interrupt() for IRQs (Sean Christopherson).
* Call external_interrupt() only when FRED is enabled, and convert the non-FRED handling to external_interrupt() after FRED lands (Sean Christopherson).
--- arch/x86/entry/entry_64_fred.S | 77 ++++++++++++++++++++++++++++++++++ arch/x86/entry/entry_fred.c | 14 +++++++ arch/x86/include/asm/fred.h | 18 ++++++++ 3 files changed, 109 insertions(+)
arch/x86/entry/entry_64_fred.S +77 -0
@@ -3,8 +3,11 @@ * The actual FRED entry points. */ +#include <linux/export.h> + #include <asm/asm.h> #include <asm/fred.h> +#include <asm/segment.h> #include "calling.h"
@@ -54,3 +57,77 @@ SYM_CODE_START_NOALIGN(asm_fred_entrypoint_kernel) FRED_EXIT ERETS SYM_CODE_END(asm_fred_entrypoint_kernel) + +#if IS_ENABLED(CONFIG_KVM_INTEL) +SYM_FUNC_START(asm_fred_entry_from_kvm) + push %rbp + mov %rsp, %rbp + + UNWIND_HINT_SAVE + + /* + * Both IRQ and NMI from VMX can be handled on current task stack + * because there is no need to protect from reentrancy and the call + * stack leading to this helper is effectively constant and shallow + * (relatively speaking). Do the same when FRED is active, i.e., no + * need to check current stack level for a stack switch. + * + * Emulate the FRED-defined redzone and stack alignment. + */ + sub $(FRED_CONFIG_REDZONE_AMOUNT << 6), %rsp + and $FRED_STACK_FRAME_RSP_MASK, %rsp + + /* + * Start to push a FRED stack frame, which is always 64 bytes: + * + * +--------+-----------------+ + * | Bytes | Usage | + * +--------+-----------------+ + * | 63:56 | Reserved | + * | 55:48 | Event Data | + * | 47:40 | SS + Event Info | + * | 39:32 | RSP | + * | 31:24 | RFLAGS | + * | 23:16 | CS + Aux Info | + * | 15:8 | RIP | + * | 7:0 | Error Code | + * +--------+-----------------+ + */ + push $0 /* Reserved, must be 0 */ + push $0 /* Event data, 0 for IRQ/NMI */ + push %rdi /* fred_ss handed in by the caller */ + push %rbp + pushf + mov $__KERNEL_CS, %rax + push %rax + + /* + * Unlike the IDT event delivery, FRED _always_ pushes an error code + * after pushing the return RIP, thus the CALL instruction CANNOT be + * used here to push the return RIP, otherwise there is no chance to + * push an error code before invoking the IRQ/NMI handler. + * + * Use LEA to get the return RIP and push it, then push an error code. + */ + lea 1f(%rip), %rax + push %rax /* Return RIP */ + push $0 /* Error code, 0 for IRQ/NMI */ + + PUSH_AND_CLEAR_REGS clear_bp=0 unwind_hint=0 + movq %rsp, %rdi /* %rdi -> pt_regs */ + call __fred_entry_from_kvm /* Call the C entry point */ + POP_REGS + ERETS +1: + /* + * Objtool doesn't understand what ERETS does, this hint tells it that + * yes, we'll reach here and with what stack state. A save/restore pair + * isn't strictly needed, but it's the simplest form. + */ + UNWIND_HINT_RESTORE + pop %rbp + RET + +SYM_FUNC_END(asm_fred_entry_from_kvm) +EXPORT_SYMBOL_GPL(asm_fred_entry_from_kvm); +#endif
arch/x86/entry/entry_fred.c +14 -0
@@ -242,3 +242,17 @@ __visible noinstr void fred_entry_from_kernel(struct pt_regs *regs)return fred_bad_type (regs, error_code); } } + +#if IS_ENABLED (CONFIG_KVM_INTEL ) +__visible noinstr void __fred_entry_from_kvm(struct pt_regs *regs) +{ +switch (regs->fred_ss.type) { +case EVENT_TYPE_EXTINT : +return fred_extint (regs); +case EVENT_TYPE_NMI : +return fred_exc_nmi (regs); +default : +WARN_ON_ONCE (1 ); + } +} +#endif
arch/x86/include/asm/fred.h +18 -0
@@ -9,6 +9,7 @@#include <linux/const.h> #include <asm/asm.h> +#include <asm/trapnr.h> /* * FRED event return instruction opcodes for ERET{S,U}; supported in
@@ -62,12 +63,29 @@ static __always_inline unsigned long fred_event_data(struct pt_regs *regs)void asm_fred_entrypoint_user (void );void asm_fred_entrypoint_kernel (void ); +void asm_fred_entry_from_kvm (struct fred_ss ); __visible void fred_entry_from_user(struct pt_regs *regs); __visible void fred_entry_from_kernel(struct pt_regs *regs); +__visible void __fred_entry_from_kvm(struct pt_regs *regs); + +/* Can be called from noinstr code, thus __always_inline */ +static __always_inline void fred_entry_from_kvm(unsigned int type, unsigned int vector) +{ +struct fred_ss ss = { + .ss =__KERNEL_DS, + .type = type, + .vector = vector, + .nmi = type ==EVENT_TYPE_NMI , + .lm =1 , + }; + +asm_fred_entry_from_kvm (ss); +}#else /* CONFIG_X86_FRED */ static __always_inline unsigned long fred_event_data (struct pt_regs *regs) {return 0 ; } +static __always_inline void fred_entry_from_kvm(unsigned int type, unsigned int vector) { }#endif /* CONFIG_X86_FRED */ #endif /* !__ASSEMBLY__ */
Signature · 1 line
2.43.0
[tip: x86/fred] x86/entry: Add fred_entry_from_kvm() for VMX to handle IRQ/NMI
The following commit has been merged into the x86/fred branch of tip:
The following commit has been merged into the x86/fred branch of tip:
Commit-ID: 9a823c61c212a9f076abc4b95ca2cbc328fbed01 Gitweb: https://git.kernel.org/tip/9a823c61c212a9f076abc4b95ca2cbc328fbed01 Author: Xin Li <[email protected]> AuthorDate: Tue, 05 Dec 2023 02:50:20 -08:00 Committer: Thomas Gleixner <[email protected]> CommitterDate: Thu, 25 Jan 2024 19:10:33 +01:00
x86/entry: Add fred_entry_from_kvm() for VMX to handle IRQ/NMI
In IRQ/NMI induced VM exits, KVM VMX needs to execute the respective handlers, which requires the software to create a FRED stack frame, and use it to invoke the handlers. Add fred_irq_entry_from_kvm() for this job.
Export fred_entry_from_kvm() because VMX can be compiled as a module.
--- arch/x86/entry/entry_64_fred.S | 77 +++++++++++++++++++++++++++++++++- arch/x86/entry/entry_fred.c | 14 ++++++- arch/x86/include/asm/fred.h | 18 ++++++++- 3 files changed, 109 insertions(+)
arch/x86/entry/entry_64_fred.S +77 -0
@@ -3,8 +3,11 @@ * The actual FRED entry points. */ +#include <linux/export.h> + #include <asm/asm.h> #include <asm/fred.h> +#include <asm/segment.h> #include "calling.h"
@@ -54,3 +57,77 @@ SYM_CODE_START_NOALIGN(asm_fred_entrypoint_kernel) FRED_EXIT ERETS SYM_CODE_END(asm_fred_entrypoint_kernel) + +#if IS_ENABLED(CONFIG_KVM_INTEL) +SYM_FUNC_START(asm_fred_entry_from_kvm) + push %rbp + mov %rsp, %rbp + + UNWIND_HINT_SAVE + + /* + * Both IRQ and NMI from VMX can be handled on current task stack + * because there is no need to protect from reentrancy and the call + * stack leading to this helper is effectively constant and shallow + * (relatively speaking). Do the same when FRED is active, i.e., no + * need to check current stack level for a stack switch. + * + * Emulate the FRED-defined redzone and stack alignment. + */ + sub $(FRED_CONFIG_REDZONE_AMOUNT << 6), %rsp + and $FRED_STACK_FRAME_RSP_MASK, %rsp + + /* + * Start to push a FRED stack frame, which is always 64 bytes: + * + * +--------+-----------------+ + * | Bytes | Usage | + * +--------+-----------------+ + * | 63:56 | Reserved | + * | 55:48 | Event Data | + * | 47:40 | SS + Event Info | + * | 39:32 | RSP | + * | 31:24 | RFLAGS | + * | 23:16 | CS + Aux Info | + * | 15:8 | RIP | + * | 7:0 | Error Code | + * +--------+-----------------+ + */ + push $0 /* Reserved, must be 0 */ + push $0 /* Event data, 0 for IRQ/NMI */ + push %rdi /* fred_ss handed in by the caller */ + push %rbp + pushf + mov $__KERNEL_CS, %rax + push %rax + + /* + * Unlike the IDT event delivery, FRED _always_ pushes an error code + * after pushing the return RIP, thus the CALL instruction CANNOT be + * used here to push the return RIP, otherwise there is no chance to + * push an error code before invoking the IRQ/NMI handler. + * + * Use LEA to get the return RIP and push it, then push an error code. + */ + lea 1f(%rip), %rax + push %rax /* Return RIP */ + push $0 /* Error code, 0 for IRQ/NMI */ + + PUSH_AND_CLEAR_REGS clear_bp=0 unwind_hint=0 + movq %rsp, %rdi /* %rdi -> pt_regs */ + call __fred_entry_from_kvm /* Call the C entry point */ + POP_REGS + ERETS +1: + /* + * Objtool doesn't understand what ERETS does, this hint tells it that + * yes, we'll reach here and with what stack state. A save/restore pair + * isn't strictly needed, but it's the simplest form. + */ + UNWIND_HINT_RESTORE + pop %rbp + RET + +SYM_FUNC_END(asm_fred_entry_from_kvm) +EXPORT_SYMBOL_GPL(asm_fred_entry_from_kvm); +#endif
arch/x86/entry/entry_fred.c +14 -0
@@ -255,3 +255,17 @@ __visible noinstr void fred_entry_from_kernel(struct pt_regs *regs)return fred_bad_type (regs, error_code); } + +#if IS_ENABLED (CONFIG_KVM_INTEL ) +__visible noinstr void __fred_entry_from_kvm(struct pt_regs *regs) +{ +switch (regs->fred_ss.type) { +case EVENT_TYPE_EXTINT : +return fred_extint (regs); +case EVENT_TYPE_NMI : +return fred_exc_nmi (regs); +default : +WARN_ON_ONCE (1 ); + } +} +#endif
arch/x86/include/asm/fred.h +18 -0
@@ -9,6 +9,7 @@#include <linux/const.h> #include <asm/asm.h> +#include <asm/trapnr.h> /* * FRED event return instruction opcodes for ERET{S,U}; supported in
@@ -62,12 +63,29 @@ static __always_inline unsigned long fred_event_data(struct pt_regs *regs)void asm_fred_entrypoint_user (void );void asm_fred_entrypoint_kernel (void ); +void asm_fred_entry_from_kvm (struct fred_ss ); __visible void fred_entry_from_user(struct pt_regs *regs); __visible void fred_entry_from_kernel(struct pt_regs *regs); +__visible void __fred_entry_from_kvm(struct pt_regs *regs); + +/* Can be called from noinstr code, thus __always_inline */ +static __always_inline void fred_entry_from_kvm(unsigned int type, unsigned int vector) +{ +struct fred_ss ss = { + .ss =__KERNEL_DS, + .type = type, + .vector = vector, + .nmi = type ==EVENT_TYPE_NMI , + .lm =1 , + }; + +asm_fred_entry_from_kvm (ss); +}#else /* CONFIG_X86_FRED */ static __always_inline unsigned long fred_event_data (struct pt_regs *regs) {return 0 ; } +static __always_inline void fred_entry_from_kvm(unsigned int type, unsigned int vector) { }#endif /* CONFIG_X86_FRED */ #endif /* !__ASSEMBLY__ */
[tip: x86/fred] x86/entry: Add fred_entry_from_kvm() for VMX to handle IRQ/NMI
In reply to Xin LiThe following commit has been merged into the x86/fred branch of tip:
The following commit has been merged into the x86/fred branch of tip:
Commit-ID: d8fbd04962865730bb67106e862bfbe363a9c284 Gitweb: https://git.kernel.org/tip/d8fbd04962865730bb67106e862bfbe363a9c284 Author: Xin Li <[email protected]> AuthorDate: Tue, 05 Dec 2023 02:50:20 -08:00 Committer: Borislav Petkov (AMD) <[email protected]> CommitterDate: Tue, 30 Jan 2024 18:20:36 +01:00
x86/entry: Add fred_entry_from_kvm() for VMX to handle IRQ/NMI
In IRQ/NMI induced VM exits, KVM VMX needs to execute the respective handlers, which requires the software to create a FRED stack frame, and use it to invoke the handlers. Add fred_irq_entry_from_kvm() for this job.
Export fred_entry_from_kvm() because VMX can be compiled as a module.
--- arch/x86/entry/entry_64_fred.S | 77 +++++++++++++++++++++++++++++++++- arch/x86/entry/entry_fred.c | 14 ++++++- arch/x86/include/asm/fred.h | 18 ++++++++- 3 files changed, 109 insertions(+)
arch/x86/entry/entry_64_fred.S +77 -0
@@ -3,8 +3,11 @@ * The actual FRED entry points. */ +#include <linux/export.h> + #include <asm/asm.h> #include <asm/fred.h> +#include <asm/segment.h> #include "calling.h"
@@ -52,3 +55,77 @@ SYM_CODE_START_NOALIGN(asm_fred_entrypoint_kernel) FRED_EXIT ERETS SYM_CODE_END(asm_fred_entrypoint_kernel) + +#if IS_ENABLED(CONFIG_KVM_INTEL) +SYM_FUNC_START(asm_fred_entry_from_kvm) + push %rbp + mov %rsp, %rbp + + UNWIND_HINT_SAVE + + /* + * Both IRQ and NMI from VMX can be handled on current task stack + * because there is no need to protect from reentrancy and the call + * stack leading to this helper is effectively constant and shallow + * (relatively speaking). Do the same when FRED is active, i.e., no + * need to check current stack level for a stack switch. + * + * Emulate the FRED-defined redzone and stack alignment. + */ + sub $(FRED_CONFIG_REDZONE_AMOUNT << 6), %rsp + and $FRED_STACK_FRAME_RSP_MASK, %rsp + + /* + * Start to push a FRED stack frame, which is always 64 bytes: + * + * +--------+-----------------+ + * | Bytes | Usage | + * +--------+-----------------+ + * | 63:56 | Reserved | + * | 55:48 | Event Data | + * | 47:40 | SS + Event Info | + * | 39:32 | RSP | + * | 31:24 | RFLAGS | + * | 23:16 | CS + Aux Info | + * | 15:8 | RIP | + * | 7:0 | Error Code | + * +--------+-----------------+ + */ + push $0 /* Reserved, must be 0 */ + push $0 /* Event data, 0 for IRQ/NMI */ + push %rdi /* fred_ss handed in by the caller */ + push %rbp + pushf + mov $__KERNEL_CS, %rax + push %rax + + /* + * Unlike the IDT event delivery, FRED _always_ pushes an error code + * after pushing the return RIP, thus the CALL instruction CANNOT be + * used here to push the return RIP, otherwise there is no chance to + * push an error code before invoking the IRQ/NMI handler. + * + * Use LEA to get the return RIP and push it, then push an error code. + */ + lea 1f(%rip), %rax + push %rax /* Return RIP */ + push $0 /* Error code, 0 for IRQ/NMI */ + + PUSH_AND_CLEAR_REGS clear_bp=0 unwind_hint=0 + movq %rsp, %rdi /* %rdi -> pt_regs */ + call __fred_entry_from_kvm /* Call the C entry point */ + POP_REGS + ERETS +1: + /* + * Objtool doesn't understand what ERETS does, this hint tells it that + * yes, we'll reach here and with what stack state. A save/restore pair + * isn't strictly needed, but it's the simplest form. + */ + UNWIND_HINT_RESTORE + pop %rbp + RET + +SYM_FUNC_END(asm_fred_entry_from_kvm) +EXPORT_SYMBOL_GPL(asm_fred_entry_from_kvm); +#endif
arch/x86/entry/entry_fred.c +14 -0
@@ -257,3 +257,17 @@ __visible noinstr void fred_entry_from_kernel(struct pt_regs *regs)return fred_bad_type (regs, error_code); } + +#if IS_ENABLED (CONFIG_KVM_INTEL ) +__visible noinstr void __fred_entry_from_kvm(struct pt_regs *regs) +{ +switch (regs->fred_ss.type) { +case EVENT_TYPE_EXTINT : +return fred_extint (regs); +case EVENT_TYPE_NMI : +return fred_exc_nmi (regs); +default : +WARN_ON_ONCE (1 ); + } +} +#endif
arch/x86/include/asm/fred.h +18 -0
@@ -9,6 +9,7 @@#include <linux/const.h> #include <asm/asm.h> +#include <asm/trapnr.h> /* * FRED event return instruction opcodes for ERET{S,U}; supported in
@@ -62,12 +63,29 @@ static __always_inline unsigned long fred_event_data(struct pt_regs *regs)void asm_fred_entrypoint_user (void );void asm_fred_entrypoint_kernel (void ); +void asm_fred_entry_from_kvm (struct fred_ss ); __visible void fred_entry_from_user(struct pt_regs *regs); __visible void fred_entry_from_kernel(struct pt_regs *regs); +__visible void __fred_entry_from_kvm(struct pt_regs *regs); + +/* Can be called from noinstr code, thus __always_inline */ +static __always_inline void fred_entry_from_kvm(unsigned int type, unsigned int vector) +{ +struct fred_ss ss = { + .ss =__KERNEL_DS, + .type = type, + .vector = vector, + .nmi = type ==EVENT_TYPE_NMI , + .lm =1 , + }; + +asm_fred_entry_from_kvm (ss); +}#else /* CONFIG_X86_FRED */ static __always_inline unsigned long fred_event_data (struct pt_regs *regs) {return 0 ; } +static __always_inline void fred_entry_from_kvm(unsigned int type, unsigned int vector) { }#endif /* CONFIG_X86_FRED */ #endif /* !__ASSEMBLY__ */
[tip: x86/fred] x86/entry: Add fred_entry_from_kvm() for VMX to handle IRQ/NMI
In reply to Xin LiThe following commit has been merged into the x86/fred branch of tip:
The following commit has been merged into the x86/fred branch of tip:
Commit-ID: 2e670358ec1829238c99fbff178e285d3eb43ef1 Gitweb: https://git.kernel.org/tip/2e670358ec1829238c99fbff178e285d3eb43ef1 Author: Xin Li <[email protected]> AuthorDate: Tue, 05 Dec 2023 02:50:20 -08:00 Committer: Borislav Petkov (AMD) <[email protected]> CommitterDate: Wed, 31 Jan 2024 22:03:16 +01:00
x86/entry: Add fred_entry_from_kvm() for VMX to handle IRQ/NMI
In IRQ/NMI induced VM exits, KVM VMX needs to execute the respective handlers, which requires the software to create a FRED stack frame, and use it to invoke the handlers. Add fred_irq_entry_from_kvm() for this job.
Export fred_entry_from_kvm() because VMX can be compiled as a module.
--- arch/x86/entry/entry_64_fred.S | 77 +++++++++++++++++++++++++++++++++- arch/x86/entry/entry_fred.c | 14 ++++++- arch/x86/include/asm/fred.h | 18 ++++++++- 3 files changed, 109 insertions(+)
arch/x86/entry/entry_64_fred.S +77 -0
@@ -3,8 +3,11 @@ * The actual FRED entry points. */ +#include <linux/export.h> + #include <asm/asm.h> #include <asm/fred.h> +#include <asm/segment.h> #include "calling.h"
@@ -52,3 +55,77 @@ SYM_CODE_START_NOALIGN(asm_fred_entrypoint_kernel) FRED_EXIT ERETS SYM_CODE_END(asm_fred_entrypoint_kernel) + +#if IS_ENABLED(CONFIG_KVM_INTEL) +SYM_FUNC_START(asm_fred_entry_from_kvm) + push %rbp + mov %rsp, %rbp + + UNWIND_HINT_SAVE + + /* + * Both IRQ and NMI from VMX can be handled on current task stack + * because there is no need to protect from reentrancy and the call + * stack leading to this helper is effectively constant and shallow + * (relatively speaking). Do the same when FRED is active, i.e., no + * need to check current stack level for a stack switch. + * + * Emulate the FRED-defined redzone and stack alignment. + */ + sub $(FRED_CONFIG_REDZONE_AMOUNT << 6), %rsp + and $FRED_STACK_FRAME_RSP_MASK, %rsp + + /* + * Start to push a FRED stack frame, which is always 64 bytes: + * + * +--------+-----------------+ + * | Bytes | Usage | + * +--------+-----------------+ + * | 63:56 | Reserved | + * | 55:48 | Event Data | + * | 47:40 | SS + Event Info | + * | 39:32 | RSP | + * | 31:24 | RFLAGS | + * | 23:16 | CS + Aux Info | + * | 15:8 | RIP | + * | 7:0 | Error Code | + * +--------+-----------------+ + */ + push $0 /* Reserved, must be 0 */ + push $0 /* Event data, 0 for IRQ/NMI */ + push %rdi /* fred_ss handed in by the caller */ + push %rbp + pushf + mov $__KERNEL_CS, %rax + push %rax + + /* + * Unlike the IDT event delivery, FRED _always_ pushes an error code + * after pushing the return RIP, thus the CALL instruction CANNOT be + * used here to push the return RIP, otherwise there is no chance to + * push an error code before invoking the IRQ/NMI handler. + * + * Use LEA to get the return RIP and push it, then push an error code. + */ + lea 1f(%rip), %rax + push %rax /* Return RIP */ + push $0 /* Error code, 0 for IRQ/NMI */ + + PUSH_AND_CLEAR_REGS clear_bp=0 unwind_hint=0 + movq %rsp, %rdi /* %rdi -> pt_regs */ + call __fred_entry_from_kvm /* Call the C entry point */ + POP_REGS + ERETS +1: + /* + * Objtool doesn't understand what ERETS does, this hint tells it that + * yes, we'll reach here and with what stack state. A save/restore pair + * isn't strictly needed, but it's the simplest form. + */ + UNWIND_HINT_RESTORE + pop %rbp + RET + +SYM_FUNC_END(asm_fred_entry_from_kvm) +EXPORT_SYMBOL_GPL(asm_fred_entry_from_kvm); +#endif
arch/x86/entry/entry_fred.c +14 -0
@@ -257,3 +257,17 @@ __visible noinstr void fred_entry_from_kernel(struct pt_regs *regs)return fred_bad_type (regs, error_code); } + +#if IS_ENABLED (CONFIG_KVM_INTEL ) +__visible noinstr void __fred_entry_from_kvm(struct pt_regs *regs) +{ +switch (regs->fred_ss.type) { +case EVENT_TYPE_EXTINT : +return fred_extint (regs); +case EVENT_TYPE_NMI : +return fred_exc_nmi (regs); +default : +WARN_ON_ONCE (1 ); + } +} +#endif
arch/x86/include/asm/fred.h +18 -0
@@ -9,6 +9,7 @@#include <linux/const.h> #include <asm/asm.h> +#include <asm/trapnr.h> /* * FRED event return instruction opcodes for ERET{S,U}; supported in
@@ -62,12 +63,29 @@ static __always_inline unsigned long fred_event_data(struct pt_regs *regs)void asm_fred_entrypoint_user (void );void asm_fred_entrypoint_kernel (void ); +void asm_fred_entry_from_kvm (struct fred_ss ); __visible void fred_entry_from_user(struct pt_regs *regs); __visible void fred_entry_from_kernel(struct pt_regs *regs); +__visible void __fred_entry_from_kvm(struct pt_regs *regs); + +/* Can be called from noinstr code, thus __always_inline */ +static __always_inline void fred_entry_from_kvm(unsigned int type, unsigned int vector) +{ +struct fred_ss ss = { + .ss =__KERNEL_DS, + .type = type, + .vector = vector, + .nmi = type ==EVENT_TYPE_NMI , + .lm =1 , + }; + +asm_fred_entry_from_kvm (ss); +}#else /* CONFIG_X86_FRED */ static __always_inline unsigned long fred_event_data (struct pt_regs *regs) {return 0 ; } +static __always_inline void fred_entry_from_kvm(unsigned int type, unsigned int vector) { }#endif /* CONFIG_X86_FRED */ #endif /* !__ASSEMBLY__ */
PATCH v13 32/35 KVM: VMX: Call fred_entry_from_kvm() for IRQ/NMI handling
In reply to Xin LiWhen FRED is enabled, call fred_entry_from_kvm() to handle IRQ/NMI in IRQ/NMI induced VM exits.
--- arch/x86/kvm/vmx/vmx.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-)
arch/x86/kvm/vmx/vmx.c +9 -3
@@ -38,6 +38,7 @@#include <asm/desc.h> #include <asm/fpu/api.h> #include <asm/fpu/xstate.h> +#include <asm/fred.h> #include <asm/idtentry.h> #include <asm/io.h> #include <asm/irq_remapping.h>
@@ -6962,14 +6963,16 @@ static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu) {u32 intr_info =vmx_get_intr_info (vcpu);unsigned int vector = intr_info &INTR_INFO_VECTOR_MASK ; -gate_desc *desc = (gate_desc *)host_idt_base + vector;if (KVM_BUG (!is_external_intr (intr_info), vcpu->kvm,"unexpected VM-Exit interrupt info: 0x%x" , intr_info))return ;kvm_before_interrupt (vcpu,KVM_HANDLING_IRQ ); -vmx_do_interrupt_irqoff (gate_offset (desc)); +if (cpu_feature_enabled (X86_FEATURE_FRED )) +fred_entry_from_kvm (EVENT_TYPE_EXTINT , vector); +else +vmx_do_interrupt_irqoff (gate_offset ((gate_desc *)host_idt_base + vector));kvm_after_interrupt (vcpu); vcpu->arch.at_instruction_boundary = true;
@@ -7262,7 +7265,10 @@ static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu,if ((u16 )vmx->exit_reason.basic ==EXIT_REASON_EXCEPTION_NMI &&is_nmi (vmx_get_intr_info (vcpu))) {kvm_before_interrupt (vcpu,KVM_HANDLING_NMI ); -vmx_do_nmi_irqoff (); +if (cpu_feature_enabled (X86_FEATURE_FRED )) +fred_entry_from_kvm (EVENT_TYPE_NMI ,NMI_VECTOR ); +else +vmx_do_nmi_irqoff ();kvm_after_interrupt (vcpu); }
Signature · 1 line
2.43.0
[tip: x86/fred] KVM: VMX: Call fred_entry_from_kvm() for IRQ/NMI handling
The following commit has been merged into the x86/fred branch of tip:
The following commit has been merged into the x86/fred branch of tip:
Commit-ID: e03c54464b62a1e043df045cc428c30cf08ec236 Gitweb: https://git.kernel.org/tip/e03c54464b62a1e043df045cc428c30cf08ec236 Author: Xin Li <[email protected]> AuthorDate: Tue, 05 Dec 2023 02:50:21 -08:00 Committer: Thomas Gleixner <[email protected]> CommitterDate: Thu, 25 Jan 2024 19:10:33 +01:00
KVM: VMX: Call fred_entry_from_kvm() for IRQ/NMI handling
When FRED is enabled, call fred_entry_from_kvm() to handle IRQ/NMI in IRQ/NMI induced VM exits.
--- arch/x86/kvm/vmx/vmx.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-)
arch/x86/kvm/vmx/vmx.c +9 -3
@@ -38,6 +38,7 @@#include <asm/desc.h> #include <asm/fpu/api.h> #include <asm/fpu/xstate.h> +#include <asm/fred.h> #include <asm/idtentry.h> #include <asm/io.h> #include <asm/irq_remapping.h>
@@ -6960,14 +6961,16 @@ static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu) {u32 intr_info =vmx_get_intr_info (vcpu);unsigned int vector = intr_info &INTR_INFO_VECTOR_MASK ; -gate_desc *desc = (gate_desc *)host_idt_base + vector;if (KVM_BUG (!is_external_intr (intr_info), vcpu->kvm,"unexpected VM-Exit interrupt info: 0x%x" , intr_info))return ;kvm_before_interrupt (vcpu,KVM_HANDLING_IRQ ); -vmx_do_interrupt_irqoff (gate_offset (desc)); +if (cpu_feature_enabled (X86_FEATURE_FRED )) +fred_entry_from_kvm (EVENT_TYPE_EXTINT , vector); +else +vmx_do_interrupt_irqoff (gate_offset ((gate_desc *)host_idt_base + vector));kvm_after_interrupt (vcpu); vcpu->arch.at_instruction_boundary = true;
@@ -7260,7 +7263,10 @@ static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu,if ((u16 )vmx->exit_reason.basic ==EXIT_REASON_EXCEPTION_NMI &&is_nmi (vmx_get_intr_info (vcpu))) {kvm_before_interrupt (vcpu,KVM_HANDLING_NMI ); -vmx_do_nmi_irqoff (); +if (cpu_feature_enabled (X86_FEATURE_FRED )) +fred_entry_from_kvm (EVENT_TYPE_NMI ,NMI_VECTOR ); +else +vmx_do_nmi_irqoff ();kvm_after_interrupt (vcpu); }
[tip: x86/fred] KVM: VMX: Call fred_entry_from_kvm() for IRQ/NMI handling
In reply to Xin LiThe following commit has been merged into the x86/fred branch of tip:
The following commit has been merged into the x86/fred branch of tip:
Commit-ID: cb5429aaa0c53d60414a08fb40f8d15d748c4cda Gitweb: https://git.kernel.org/tip/cb5429aaa0c53d60414a08fb40f8d15d748c4cda Author: Xin Li <[email protected]> AuthorDate: Tue, 05 Dec 2023 02:50:21 -08:00 Committer: Borislav Petkov (AMD) <[email protected]> CommitterDate: Tue, 30 Jan 2024 18:20:36 +01:00
KVM: VMX: Call fred_entry_from_kvm() for IRQ/NMI handling
When FRED is enabled, call fred_entry_from_kvm() to handle IRQ/NMI in IRQ/NMI induced VM exits.
--- arch/x86/kvm/vmx/vmx.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-)
arch/x86/kvm/vmx/vmx.c +9 -3
@@ -38,6 +38,7 @@#include <asm/desc.h> #include <asm/fpu/api.h> #include <asm/fpu/xstate.h> +#include <asm/fred.h> #include <asm/idtentry.h> #include <asm/io.h> #include <asm/irq_remapping.h>
@@ -6960,14 +6961,16 @@ static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu) {u32 intr_info =vmx_get_intr_info (vcpu);unsigned int vector = intr_info &INTR_INFO_VECTOR_MASK ; -gate_desc *desc = (gate_desc *)host_idt_base + vector;if (KVM_BUG (!is_external_intr (intr_info), vcpu->kvm,"unexpected VM-Exit interrupt info: 0x%x" , intr_info))return ;kvm_before_interrupt (vcpu,KVM_HANDLING_IRQ ); -vmx_do_interrupt_irqoff (gate_offset (desc)); +if (cpu_feature_enabled (X86_FEATURE_FRED )) +fred_entry_from_kvm (EVENT_TYPE_EXTINT , vector); +else +vmx_do_interrupt_irqoff (gate_offset ((gate_desc *)host_idt_base + vector));kvm_after_interrupt (vcpu); vcpu->arch.at_instruction_boundary = true;
@@ -7260,7 +7263,10 @@ static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu,if ((u16 )vmx->exit_reason.basic ==EXIT_REASON_EXCEPTION_NMI &&is_nmi (vmx_get_intr_info (vcpu))) {kvm_before_interrupt (vcpu,KVM_HANDLING_NMI ); -vmx_do_nmi_irqoff (); +if (cpu_feature_enabled (X86_FEATURE_FRED )) +fred_entry_from_kvm (EVENT_TYPE_NMI ,NMI_VECTOR ); +else +vmx_do_nmi_irqoff ();kvm_after_interrupt (vcpu); }
[tip: x86/fred] KVM: VMX: Call fred_entry_from_kvm() for IRQ/NMI handling
In reply to Xin LiThe following commit has been merged into the x86/fred branch of tip:
The following commit has been merged into the x86/fred branch of tip:
Commit-ID: 70d0fe5d0923abfb28c26e71171944f4801f9f38 Gitweb: https://git.kernel.org/tip/70d0fe5d0923abfb28c26e71171944f4801f9f38 Author: Xin Li <[email protected]> AuthorDate: Tue, 05 Dec 2023 02:50:21 -08:00 Committer: Borislav Petkov (AMD) <[email protected]> CommitterDate: Wed, 31 Jan 2024 22:03:20 +01:00
KVM: VMX: Call fred_entry_from_kvm() for IRQ/NMI handling
When FRED is enabled, call fred_entry_from_kvm() to handle IRQ/NMI in IRQ/NMI induced VM exits.
--- arch/x86/kvm/vmx/vmx.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-)
arch/x86/kvm/vmx/vmx.c +9 -3
@@ -38,6 +38,7 @@#include <asm/desc.h> #include <asm/fpu/api.h> #include <asm/fpu/xstate.h> +#include <asm/fred.h> #include <asm/idtentry.h> #include <asm/io.h> #include <asm/irq_remapping.h>
@@ -6960,14 +6961,16 @@ static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu) {u32 intr_info =vmx_get_intr_info (vcpu);unsigned int vector = intr_info &INTR_INFO_VECTOR_MASK ; -gate_desc *desc = (gate_desc *)host_idt_base + vector;if (KVM_BUG (!is_external_intr (intr_info), vcpu->kvm,"unexpected VM-Exit interrupt info: 0x%x" , intr_info))return ;kvm_before_interrupt (vcpu,KVM_HANDLING_IRQ ); -vmx_do_interrupt_irqoff (gate_offset (desc)); +if (cpu_feature_enabled (X86_FEATURE_FRED )) +fred_entry_from_kvm (EVENT_TYPE_EXTINT , vector); +else +vmx_do_interrupt_irqoff (gate_offset ((gate_desc *)host_idt_base + vector));kvm_after_interrupt (vcpu); vcpu->arch.at_instruction_boundary = true;
@@ -7260,7 +7263,10 @@ static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu,if ((u16 )vmx->exit_reason.basic ==EXIT_REASON_EXCEPTION_NMI &&is_nmi (vmx_get_intr_info (vcpu))) {kvm_before_interrupt (vcpu,KVM_HANDLING_NMI ); -vmx_do_nmi_irqoff (); +if (cpu_feature_enabled (X86_FEATURE_FRED )) +fred_entry_from_kvm (EVENT_TYPE_NMI ,NMI_VECTOR ); +else +vmx_do_nmi_irqoff ();kvm_after_interrupt (vcpu); }
PATCH v13 33/35 x86/syscall: Split IDT syscall setup code into idt_syscall_init()
In reply to Xin LiBecause FRED uses the ring 3 FRED entrypoint for SYSCALL and SYSENTER and ERETU is the only legit instruction to return to ring 3, there is NO need to setup SYSCALL and SYSENTER MSRs for FRED, except the IA32_STAR MSR.
Split IDT syscall setup code into idt_syscall_init() to make it easy to skip syscall setup code when FRED is enabled.
--- arch/x86/kernel/cpu/common.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-)
arch/x86/kernel/cpu/common.c +10 -3
@@ -2076,10 +2076,8 @@ static void wrmsrl_cstar(unsigned long val)wrmsrl (MSR_CSTAR , val); } -/* May not be marked __init: used by software suspend */ -void syscall_init (void ) +static inline void idt_syscall_init (void ) { -wrmsr (MSR_STAR ,0 , (__USER32_CS <<16 ) | __KERNEL_CS);wrmsrl (MSR_LSTAR , (unsigned long )entry_SYSCALL_64);if (ia32_enabled ()) {
@@ -2113,6 +2111,15 @@ void syscall_init(void) X86_EFLAGS_AC|X86_EFLAGS_ID); } +/* May not be marked __init: used by software suspend */ +void syscall_init (void ) +{ +/* The default user and kernel segments */ +wrmsr (MSR_STAR ,0 , (__USER32_CS <<16 ) | __KERNEL_CS); + +idt_syscall_init (); +} +#else /* CONFIG_X86_64 */ #ifdef CONFIG_STACKPROTECTOR
Signature · 1 line
2.43.0
[tip: x86/fred] x86/syscall: Split IDT syscall setup code into idt_syscall_init()
The following commit has been merged into the x86/fred branch of tip:
The following commit has been merged into the x86/fred branch of tip:
Commit-ID: 10e6d5d2a44789e1a4e57b9fd45c511b4187749b Gitweb: https://git.kernel.org/tip/10e6d5d2a44789e1a4e57b9fd45c511b4187749b Author: Xin Li <[email protected]> AuthorDate: Tue, 05 Dec 2023 02:50:22 -08:00 Committer: Thomas Gleixner <[email protected]> CommitterDate: Thu, 25 Jan 2024 19:10:33 +01:00
x86/syscall: Split IDT syscall setup code into idt_syscall_init()
Because FRED uses the ring 3 FRED entrypoint for SYSCALL and SYSENTER and ERETU is the only legit instruction to return to ring 3, there is NO need to setup SYSCALL and SYSENTER MSRs for FRED, except the IA32_STAR MSR.
Split IDT syscall setup code into idt_syscall_init() to make it easy to skip syscall setup code when FRED is enabled.
--- arch/x86/kernel/cpu/common.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-)
arch/x86/kernel/cpu/common.c +10 -3
@@ -2069,10 +2069,8 @@ static void wrmsrl_cstar(unsigned long val)wrmsrl (MSR_CSTAR , val); } -/* May not be marked __init: used by software suspend */ -void syscall_init (void ) +static inline void idt_syscall_init (void ) { -wrmsr (MSR_STAR ,0 , (__USER32_CS <<16 ) | __KERNEL_CS);wrmsrl (MSR_LSTAR , (unsigned long )entry_SYSCALL_64);if (ia32_enabled ()) {
@@ -2106,6 +2104,15 @@ void syscall_init(void) X86_EFLAGS_AC|X86_EFLAGS_ID); } +/* May not be marked __init: used by software suspend */ +void syscall_init (void ) +{ +/* The default user and kernel segments */ +wrmsr (MSR_STAR ,0 , (__USER32_CS <<16 ) | __KERNEL_CS); + +idt_syscall_init (); +} +#else /* CONFIG_X86_64 */ #ifdef CONFIG_STACKPROTECTOR
[tip: x86/fred] x86/syscall: Split IDT syscall setup code into idt_syscall_init()
In reply to Xin LiThe following commit has been merged into the x86/fred branch of tip:
The following commit has been merged into the x86/fred branch of tip:
Commit-ID: ae46f3978ae4eb9da013ff9963f105de2db2f8ec Gitweb: https://git.kernel.org/tip/ae46f3978ae4eb9da013ff9963f105de2db2f8ec Author: Xin Li <[email protected]> AuthorDate: Tue, 05 Dec 2023 02:50:22 -08:00 Committer: Borislav Petkov (AMD) <[email protected]> CommitterDate: Tue, 30 Jan 2024 18:20:36 +01:00
x86/syscall: Split IDT syscall setup code into idt_syscall_init()
Because FRED uses the ring 3 FRED entrypoint for SYSCALL and SYSENTER and ERETU is the only legit instruction to return to ring 3, there is NO need to setup SYSCALL and SYSENTER MSRs for FRED, except the IA32_STAR MSR.
Split IDT syscall setup code into idt_syscall_init() to make it easy to skip syscall setup code when FRED is enabled.
--- arch/x86/kernel/cpu/common.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-)
arch/x86/kernel/cpu/common.c +10 -3
@@ -2066,10 +2066,8 @@ static void wrmsrl_cstar(unsigned long val)wrmsrl (MSR_CSTAR , val); } -/* May not be marked __init: used by software suspend */ -void syscall_init (void ) +static inline void idt_syscall_init (void ) { -wrmsr (MSR_STAR ,0 , (__USER32_CS <<16 ) | __KERNEL_CS);wrmsrl (MSR_LSTAR , (unsigned long )entry_SYSCALL_64);if (ia32_enabled ()) {
@@ -2103,6 +2101,15 @@ void syscall_init(void) X86_EFLAGS_AC|X86_EFLAGS_ID); } +/* May not be marked __init: used by software suspend */ +void syscall_init (void ) +{ +/* The default user and kernel segments */ +wrmsr (MSR_STAR ,0 , (__USER32_CS <<16 ) | __KERNEL_CS); + +idt_syscall_init (); +} +#else /* CONFIG_X86_64 */ #ifdef CONFIG_STACKPROTECTOR
[tip: x86/fred] x86/syscall: Split IDT syscall setup code into idt_syscall_init()
In reply to Xin LiThe following commit has been merged into the x86/fred branch of tip:
The following commit has been merged into the x86/fred branch of tip:
Commit-ID: 530dce278afffd8084af9a23493532912cdbe98a Gitweb: https://git.kernel.org/tip/530dce278afffd8084af9a23493532912cdbe98a Author: Xin Li <[email protected]> AuthorDate: Tue, 05 Dec 2023 02:50:22 -08:00 Committer: Borislav Petkov (AMD) <[email protected]> CommitterDate: Wed, 31 Jan 2024 22:03:27 +01:00
x86/syscall: Split IDT syscall setup code into idt_syscall_init()
Because FRED uses the ring 3 FRED entrypoint for SYSCALL and SYSENTER and ERETU is the only legit instruction to return to ring 3, there is NO need to setup SYSCALL and SYSENTER MSRs for FRED, except the IA32_STAR MSR.
Split IDT syscall setup code into idt_syscall_init() to make it easy to skip syscall setup code when FRED is enabled.
--- arch/x86/kernel/cpu/common.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-)
arch/x86/kernel/cpu/common.c +10 -3
@@ -2066,10 +2066,8 @@ static void wrmsrl_cstar(unsigned long val)wrmsrl (MSR_CSTAR , val); } -/* May not be marked __init: used by software suspend */ -void syscall_init (void ) +static inline void idt_syscall_init (void ) { -wrmsr (MSR_STAR ,0 , (__USER32_CS <<16 ) | __KERNEL_CS);wrmsrl (MSR_LSTAR , (unsigned long )entry_SYSCALL_64);if (ia32_enabled ()) {
@@ -2103,6 +2101,15 @@ void syscall_init(void) X86_EFLAGS_AC|X86_EFLAGS_ID); } +/* May not be marked __init: used by software suspend */ +void syscall_init (void ) +{ +/* The default user and kernel segments */ +wrmsr (MSR_STAR ,0 , (__USER32_CS <<16 ) | __KERNEL_CS); + +idt_syscall_init (); +} +#else /* CONFIG_X86_64 */ #ifdef CONFIG_STACKPROTECTOR
PATCH v13 34/35 x86/fred: Add FRED initialization functions
In reply to Xin LiFrom: "H. Peter Anvin (Intel)" <[email protected]>
Add cpu_init_fred_exceptions() to:
- Set FRED entrypoints for events happening in ring 0 and 3.
- Specify the stack level for IRQs occurred ring 0.
- Specify dedicated event stacks for #DB/NMI/#MCE/#DF.
- Enable FRED and invalidtes IDT.
- Force 32-bit system calls to use "int $0x80" only.
Add fred_complete_exception_setup() to:
- Initialize system_vectors as done for IDT systems.
- Set unused sysvec_table entries to fred_handle_spurious_interrupt().
---
Changes since v9:
* Set unused sysvec table entries to fred_handle_spurious_interrupt() in fred_complete_exception_setup() (Thomas Gleixner).
Changes since v5:
* Add a comment for FRED stack level settings (Lai Jiangshan).
* Define NMI/#DB/#MCE/#DF stack levels using macros.
--- arch/x86/entry/entry_fred.c | 21 +++++++++++++ arch/x86/include/asm/fred.h | 5 ++++ arch/x86/kernel/Makefile | 1 + arch/x86/kernel/fred.c | 59 +++++++++++++++++++++++++++++++++++++ 4 files changed, 86 insertions(+) create mode 100644 arch/x86/kernel/fred.c
arch/x86/entry/entry_fred.c +21 -0
@@ -140,6 +140,27 @@ void __init fred_install_sysvec(unsigned int sysvec, idtentry_t handler) sysvec_table[sysvec -FIRST_SYSTEM_VECTOR ] = handler; } +static noinstr void fred_handle_spurious_interrupt(struct pt_regs *regs) +{ +spurious_interrupt (regs, regs->fred_ss.vector); +} + +void __init fred_complete_exception_setup (void ) +{ +unsigned int vector; + +for (vector =0 ; vector <FIRST_EXTERNAL_VECTOR ; vector++) +set_bit (vector, system_vectors); + +for (vector =0 ; vector <NR_SYSTEM_VECTORS ; vector++) { +if (sysvec_table[vector]) +set_bit (vector +FIRST_SYSTEM_VECTOR , system_vectors); +else + sysvec_table[vector] = fred_handle_spurious_interrupt; + } + fred_setup_done = true; +} + static noinstr void fred_extint(struct pt_regs *regs) {unsigned int vector = regs->fred_ss.vector;
arch/x86/include/asm/fred.h +5 -0
@@ -83,8 +83,13 @@ static __always_inline void fred_entry_from_kvm(unsigned int type, unsigned intasm_fred_entry_from_kvm (ss); } +void cpu_init_fred_exceptions (void ); +void fred_complete_exception_setup (void ); +#else /* CONFIG_X86_FRED */ static __always_inline unsigned long fred_event_data (struct pt_regs *regs) {return 0 ; } +static inline void cpu_init_fred_exceptions (void ) { } +static inline void fred_complete_exception_setup (void ) { } static __always_inline void fred_entry_from_kvm(unsigned int type, unsigned int vector) { }#endif /* CONFIG_X86_FRED */ #endif /* !__ASSEMBLY__ */
arch/x86/kernel/Makefile +1 -0
@@ -48,6 +48,7 @@ obj-y += platform-quirks.o obj-y += process_$(BITS).o signal.o signal_$(BITS).o obj-y += traps.o idt.o irq.o irq_$(BITS).o dumpstack_$(BITS).o obj-y += time.o ioport.o dumpstack.o nmi.o +obj-$(CONFIG_X86_FRED) += fred.o obj-$(CONFIG_MODIFY_LDT_SYSCALL) += ldt.o obj-$(CONFIG_X86_KERNEL_IBT) += ibt_selftest.o obj-y += setup.o x86_init.o i8259.o irqinit.o
arch/x86/kernel/fred.c new file +59 -0
59 lines not on this page · raw message
Signature · 1 line
2.43.0
[tip: x86/fred] x86/fred: Add FRED initialization functions
The following commit has been merged into the x86/fred branch of tip:
The following commit has been merged into the x86/fred branch of tip:
Commit-ID: 705dc2f2c8eba23abf24aff49ae87ee059a74744 Gitweb: https://git.kernel.org/tip/705dc2f2c8eba23abf24aff49ae87ee059a74744 Author: H. Peter Anvin (Intel) <[email protected]> AuthorDate: Tue, 05 Dec 2023 02:50:23 -08:00 Committer: Thomas Gleixner <[email protected]> CommitterDate: Thu, 25 Jan 2024 19:10:33 +01:00
x86/fred: Add FRED initialization functions
Add cpu_init_fred_exceptions() to:
- Set FRED entrypoints for events happening in ring 0 and 3.
- Specify the stack level for IRQs occurred ring 0.
- Specify dedicated event stacks for #DB/NMI/#MCE/#DF.
- Enable FRED and invalidtes IDT.
- Force 32-bit system calls to use "int $0x80" only.
Add fred_complete_exception_setup() to:
- Initialize system_vectors as done for IDT systems.
- Set unused sysvec_table entries to fred_handle_spurious_interrupt().
--- arch/x86/entry/entry_fred.c | 21 +++++++++++++- arch/x86/include/asm/fred.h | 5 +++- arch/x86/kernel/Makefile | 1 +- arch/x86/kernel/fred.c | 59 ++++++++++++++++++++++++++++++++++++- 4 files changed, 86 insertions(+) create mode 100644 arch/x86/kernel/fred.c
arch/x86/entry/entry_fred.c +21 -0
@@ -131,6 +131,27 @@ void __init fred_install_sysvec(unsigned int sysvec, idtentry_t handler) sysvec_table[sysvec -FIRST_SYSTEM_VECTOR ] = handler; } +static noinstr void fred_handle_spurious_interrupt(struct pt_regs *regs) +{ +spurious_interrupt (regs, regs->fred_ss.vector); +} + +void __init fred_complete_exception_setup (void ) +{ +unsigned int vector; + +for (vector =0 ; vector <FIRST_EXTERNAL_VECTOR ; vector++) +set_bit (vector, system_vectors); + +for (vector =0 ; vector <NR_SYSTEM_VECTORS ; vector++) { +if (sysvec_table[vector]) +set_bit (vector +FIRST_SYSTEM_VECTOR , system_vectors); +else + sysvec_table[vector] = fred_handle_spurious_interrupt; + } + fred_setup_done = true; +} + static noinstr void fred_extint(struct pt_regs *regs) {unsigned int vector = regs->fred_ss.vector;
arch/x86/include/asm/fred.h +5 -0
@@ -83,8 +83,13 @@ static __always_inline void fred_entry_from_kvm(unsigned int type, unsigned intasm_fred_entry_from_kvm (ss); } +void cpu_init_fred_exceptions (void ); +void fred_complete_exception_setup (void ); +#else /* CONFIG_X86_FRED */ static __always_inline unsigned long fred_event_data (struct pt_regs *regs) {return 0 ; } +static inline void cpu_init_fred_exceptions (void ) { } +static inline void fred_complete_exception_setup (void ) { } static __always_inline void fred_entry_from_kvm(unsigned int type, unsigned int vector) { }#endif /* CONFIG_X86_FRED */ #endif /* !__ASSEMBLY__ */
arch/x86/kernel/Makefile +1 -0
@@ -48,6 +48,7 @@ obj-y += platform-quirks.o obj-y += process_$(BITS).o signal.o signal_$(BITS).o obj-y += traps.o idt.o irq.o irq_$(BITS).o dumpstack_$(BITS).o obj-y += time.o ioport.o dumpstack.o nmi.o +obj-$(CONFIG_X86_FRED) += fred.o obj-$(CONFIG_MODIFY_LDT_SYSCALL) += ldt.o obj-$(CONFIG_X86_KERNEL_IBT) += ibt_selftest.o obj-y += setup.o x86_init.o i8259.o irqinit.o
arch/x86/kernel/fred.c new file +59 -0
59 lines not on this page · raw message
[tip: x86/fred] x86/fred: Add FRED initialization functions
In reply to Xin LiThe following commit has been merged into the x86/fred branch of tip:
The following commit has been merged into the x86/fred branch of tip:
Commit-ID: 43ca697baecf3c90fe108a61cf444de20bbfa5b9 Gitweb: https://git.kernel.org/tip/43ca697baecf3c90fe108a61cf444de20bbfa5b9 Author: H. Peter Anvin (Intel) <[email protected]> AuthorDate: Tue, 05 Dec 2023 02:50:23 -08:00 Committer: Borislav Petkov (AMD) <[email protected]> CommitterDate: Tue, 30 Jan 2024 18:20:36 +01:00
x86/fred: Add FRED initialization functions
Add cpu_init_fred_exceptions() to:
- Set FRED entrypoints for events happening in ring 0 and 3.
- Specify the stack level for IRQs occurred ring 0.
- Specify dedicated event stacks for #DB/NMI/#MCE/#DF.
- Enable FRED and invalidtes IDT.
- Force 32-bit system calls to use "int $0x80" only.
Add fred_complete_exception_setup() to:
- Initialize system_vectors as done for IDT systems.
- Set unused sysvec_table entries to fred_handle_spurious_interrupt().
--- arch/x86/entry/entry_fred.c | 21 +++++++++++++- arch/x86/include/asm/fred.h | 5 +++- arch/x86/kernel/Makefile | 1 +- arch/x86/kernel/fred.c | 59 ++++++++++++++++++++++++++++++++++++- 4 files changed, 86 insertions(+) create mode 100644 arch/x86/kernel/fred.c
arch/x86/entry/entry_fred.c +21 -0
@@ -133,6 +133,27 @@ void __init fred_install_sysvec(unsigned int sysvec, idtentry_t handler) sysvec_table[sysvec -FIRST_SYSTEM_VECTOR ] = handler; } +static noinstr void fred_handle_spurious_interrupt(struct pt_regs *regs) +{ +spurious_interrupt (regs, regs->fred_ss.vector); +} + +void __init fred_complete_exception_setup (void ) +{ +unsigned int vector; + +for (vector =0 ; vector <FIRST_EXTERNAL_VECTOR ; vector++) +set_bit (vector, system_vectors); + +for (vector =0 ; vector <NR_SYSTEM_VECTORS ; vector++) { +if (sysvec_table[vector]) +set_bit (vector +FIRST_SYSTEM_VECTOR , system_vectors); +else + sysvec_table[vector] = fred_handle_spurious_interrupt; + } + fred_setup_done = true; +} + static noinstr void fred_extint(struct pt_regs *regs) {unsigned int vector = regs->fred_ss.vector;
arch/x86/include/asm/fred.h +5 -0
@@ -83,8 +83,13 @@ static __always_inline void fred_entry_from_kvm(unsigned int type, unsigned intasm_fred_entry_from_kvm (ss); } +void cpu_init_fred_exceptions (void ); +void fred_complete_exception_setup (void ); +#else /* CONFIG_X86_FRED */ static __always_inline unsigned long fred_event_data (struct pt_regs *regs) {return 0 ; } +static inline void cpu_init_fred_exceptions (void ) { } +static inline void fred_complete_exception_setup (void ) { } static __always_inline void fred_entry_from_kvm(unsigned int type, unsigned int vector) { }#endif /* CONFIG_X86_FRED */ #endif /* !__ASSEMBLY__ */
arch/x86/kernel/Makefile +1 -0
@@ -48,6 +48,7 @@ obj-y += platform-quirks.o obj-y += process_$(BITS).o signal.o signal_$(BITS).o obj-y += traps.o idt.o irq.o irq_$(BITS).o dumpstack_$(BITS).o obj-y += time.o ioport.o dumpstack.o nmi.o +obj-$(CONFIG_X86_FRED) += fred.o obj-$(CONFIG_MODIFY_LDT_SYSCALL) += ldt.o obj-$(CONFIG_X86_KERNEL_IBT) += ibt_selftest.o obj-y += setup.o x86_init.o i8259.o irqinit.o
arch/x86/kernel/fred.c new file +59 -0
59 lines not on this page · raw message
[tip: x86/fred] x86/fred: Add FRED initialization functions
In reply to Xin LiThe following commit has been merged into the x86/fred branch of tip:
The following commit has been merged into the x86/fred branch of tip:
Commit-ID: cdd99dd873cb11c40adf1ef70693f72c622ac8f3 Gitweb: https://git.kernel.org/tip/cdd99dd873cb11c40adf1ef70693f72c622ac8f3 Author: H. Peter Anvin (Intel) <[email protected]> AuthorDate: Tue, 05 Dec 2023 02:50:23 -08:00 Committer: Borislav Petkov (AMD) <[email protected]> CommitterDate: Wed, 31 Jan 2024 22:03:32 +01:00
x86/fred: Add FRED initialization functions
Add cpu_init_fred_exceptions() to:
- Set FRED entrypoints for events happening in ring 0 and 3.
- Specify the stack level for IRQs occurred ring 0.
- Specify dedicated event stacks for #DB/NMI/#MCE/#DF.
- Enable FRED and invalidtes IDT.
- Force 32-bit system calls to use "int $0x80" only.
Add fred_complete_exception_setup() to:
- Initialize system_vectors as done for IDT systems.
- Set unused sysvec_table entries to fred_handle_spurious_interrupt().
--- arch/x86/entry/entry_fred.c | 21 +++++++++++++- arch/x86/include/asm/fred.h | 5 +++- arch/x86/kernel/Makefile | 1 +- arch/x86/kernel/fred.c | 59 ++++++++++++++++++++++++++++++++++++- 4 files changed, 86 insertions(+) create mode 100644 arch/x86/kernel/fred.c
arch/x86/entry/entry_fred.c +21 -0
@@ -133,6 +133,27 @@ void __init fred_install_sysvec(unsigned int sysvec, idtentry_t handler) sysvec_table[sysvec -FIRST_SYSTEM_VECTOR ] = handler; } +static noinstr void fred_handle_spurious_interrupt(struct pt_regs *regs) +{ +spurious_interrupt (regs, regs->fred_ss.vector); +} + +void __init fred_complete_exception_setup (void ) +{ +unsigned int vector; + +for (vector =0 ; vector <FIRST_EXTERNAL_VECTOR ; vector++) +set_bit (vector, system_vectors); + +for (vector =0 ; vector <NR_SYSTEM_VECTORS ; vector++) { +if (sysvec_table[vector]) +set_bit (vector +FIRST_SYSTEM_VECTOR , system_vectors); +else + sysvec_table[vector] = fred_handle_spurious_interrupt; + } + fred_setup_done = true; +} + static noinstr void fred_extint(struct pt_regs *regs) {unsigned int vector = regs->fred_ss.vector;
arch/x86/include/asm/fred.h +5 -0
@@ -83,8 +83,13 @@ static __always_inline void fred_entry_from_kvm(unsigned int type, unsigned intasm_fred_entry_from_kvm (ss); } +void cpu_init_fred_exceptions (void ); +void fred_complete_exception_setup (void ); +#else /* CONFIG_X86_FRED */ static __always_inline unsigned long fred_event_data (struct pt_regs *regs) {return 0 ; } +static inline void cpu_init_fred_exceptions (void ) { } +static inline void fred_complete_exception_setup (void ) { } static __always_inline void fred_entry_from_kvm(unsigned int type, unsigned int vector) { }#endif /* CONFIG_X86_FRED */ #endif /* !__ASSEMBLY__ */
arch/x86/kernel/Makefile +1 -0
@@ -48,6 +48,7 @@ obj-y += platform-quirks.o obj-y += process_$(BITS).o signal.o signal_$(BITS).o obj-y += traps.o idt.o irq.o irq_$(BITS).o dumpstack_$(BITS).o obj-y += time.o ioport.o dumpstack.o nmi.o +obj-$(CONFIG_X86_FRED) += fred.o obj-$(CONFIG_MODIFY_LDT_SYSCALL) += ldt.o obj-$(CONFIG_X86_KERNEL_IBT) += ibt_selftest.o obj-y += setup.o x86_init.o i8259.o irqinit.o
arch/x86/kernel/fred.c new file +59 -0
59 lines not on this page · raw message
PATCH v13 35/35 x86/fred: Invoke FRED initialization code to enable FRED
In reply to Xin LiFrom: "H. Peter Anvin (Intel)" <[email protected]>
Let cpu_init_exception_handling() call cpu_init_fred_exceptions() to initialize FRED. However if FRED is unavailable or disabled, it falls back to set up TSS IST and initialize IDT.
---
Changes since v10:
* No need to invalidate SYSCALL and SYSENTER MSRs (Thomas Gleixner).
Changes since v8:
* Move this patch after all required changes are in place (Thomas Gleixner).
--- arch/x86/kernel/cpu/common.c | 22 +++++++++++++++++----- arch/x86/kernel/irqinit.c | 7 ++++++- arch/x86/kernel/traps.c | 5 ++++- 3 files changed, 27 insertions(+), 7 deletions(-)
arch/x86/kernel/cpu/common.c +17 -5
@@ -61,6 +61,7 @@#include <asm/microcode.h> #include <asm/intel-family.h> #include <asm/cpu_device_id.h> +#include <asm/fred.h> #include <asm/uv/uv.h> #include <asm/ia32.h> #include <asm/set_memory.h>
@@ -2117,7 +2118,15 @@ void syscall_init(void)/* The default user and kernel segments */ wrmsr (MSR_STAR ,0 , (__USER32_CS <<16 ) | __KERNEL_CS); - idt_syscall_init(); +/* +* Except the IA32_STAR MSR, there is NO need to setup SYSCALL and +* SYSENTER MSRs for FRED, because FRED uses the ring 3 FRED +* entrypoint for SYSCALL and SYSENTER, and ERETU is the only legit +* instruction to return to ring 3 (both sysexit and sysret cause +* #UD when FRED is enabled). +*/ +if (!cpu_feature_enabled (X86_FEATURE_FRED )) + idt_syscall_init(); }#else /* CONFIG_X86_64 */
@@ -2223,8 +2232,9 @@ void cpu_init_exception_handling(void)/* paranoid_entry() gets the CPU number from the GDT */ setup_getcpu (cpu); -/* IST vectors need TSS to be set up. */ -tss_setup_ist (tss); +/* For IDT mode, IST vectors need to be set in TSS. */ +if (!cpu_feature_enabled (X86_FEATURE_FRED )) +tss_setup_ist (tss);tss_setup_io_bitmap (tss);set_tss_desc (cpu, &get_cpu_entry_area (cpu)->tss.x86_tss);
@@ -2233,8 +2243,10 @@ void cpu_init_exception_handling(void)/* GHCB needs to be setup to handle #VC. */ setup_ghcb (); -/* Finally load the IDT */ -load_current_idt (); +if (cpu_feature_enabled (X86_FEATURE_FRED )) +cpu_init_fred_exceptions (); +else +load_current_idt (); }/*
arch/x86/kernel/irqinit.c +6 -1
@@ -28,6 +28,7 @@#include <asm/setup.h> #include <asm/i8259.h> #include <asm/traps.h> +#include <asm/fred.h> #include <asm/prom.h> /*
@@ -96,7 +97,11 @@ void __init native_init_IRQ(void)/* Execute any quirks before the call gates are initialised: */ x86_init.irqs.pre_vector_init (); -idt_setup_apic_and_irq_gates (); +if (cpu_feature_enabled (X86_FEATURE_FRED )) +fred_complete_exception_setup (); +else +idt_setup_apic_and_irq_gates (); +lapic_assign_system_vectors ();if (!acpi_ioapic && !of_ioapic &&nr_legacy_irqs ()) {
arch/x86/kernel/traps.c +4 -1
@@ -1411,7 +1411,10 @@ void __init trap_init(void)/* Initialize TSS before setting up traps so ISTs work */ cpu_init_exception_handling (); +/* Setup traps as cpu_init() might #GP */ -idt_setup_traps (); +if (!cpu_feature_enabled (X86_FEATURE_FRED )) +idt_setup_traps (); +cpu_init (); }
Signature · 1 line
2.43.0
[tip: x86/fred] x86/fred: Invoke FRED initialization code to enable FRED
The following commit has been merged into the x86/fred branch of tip:
The following commit has been merged into the x86/fred branch of tip:
Commit-ID: a9f26154bf5478fc155309fc69128415f3a1be08 Gitweb: https://git.kernel.org/tip/a9f26154bf5478fc155309fc69128415f3a1be08 Author: H. Peter Anvin (Intel) <[email protected]> AuthorDate: Tue, 05 Dec 2023 02:50:24 -08:00 Committer: Thomas Gleixner <[email protected]> CommitterDate: Thu, 25 Jan 2024 19:10:33 +01:00
x86/fred: Invoke FRED initialization code to enable FRED
Let cpu_init_exception_handling() call cpu_init_fred_exceptions() to initialize FRED. However if FRED is unavailable or disabled, it falls back to set up TSS IST and initialize IDT.
--- arch/x86/kernel/cpu/common.c | 22 +++++++++++++++++----- arch/x86/kernel/irqinit.c | 7 ++++++- arch/x86/kernel/traps.c | 5 ++++- 3 files changed, 27 insertions(+), 7 deletions(-)
arch/x86/kernel/cpu/common.c +17 -5
@@ -61,6 +61,7 @@#include <asm/microcode.h> #include <asm/intel-family.h> #include <asm/cpu_device_id.h> +#include <asm/fred.h> #include <asm/uv/uv.h> #include <asm/ia32.h> #include <asm/set_memory.h>
@@ -2110,7 +2111,15 @@ void syscall_init(void)/* The default user and kernel segments */ wrmsr (MSR_STAR ,0 , (__USER32_CS <<16 ) | __KERNEL_CS); - idt_syscall_init(); +/* +* Except the IA32_STAR MSR, there is NO need to setup SYSCALL and +* SYSENTER MSRs for FRED, because FRED uses the ring 3 FRED +* entrypoint for SYSCALL and SYSENTER, and ERETU is the only legit +* instruction to return to ring 3 (both sysexit and sysret cause +* #UD when FRED is enabled). +*/ +if (!cpu_feature_enabled (X86_FEATURE_FRED )) + idt_syscall_init(); }#else /* CONFIG_X86_64 */
@@ -2216,8 +2225,9 @@ void cpu_init_exception_handling(void)/* paranoid_entry() gets the CPU number from the GDT */ setup_getcpu (cpu); -/* IST vectors need TSS to be set up. */ -tss_setup_ist (tss); +/* For IDT mode, IST vectors need to be set in TSS. */ +if (!cpu_feature_enabled (X86_FEATURE_FRED )) +tss_setup_ist (tss);tss_setup_io_bitmap (tss);set_tss_desc (cpu, &get_cpu_entry_area (cpu)->tss.x86_tss);
@@ -2226,8 +2236,10 @@ void cpu_init_exception_handling(void)/* GHCB needs to be setup to handle #VC. */ setup_ghcb (); -/* Finally load the IDT */ -load_current_idt (); +if (cpu_feature_enabled (X86_FEATURE_FRED )) +cpu_init_fred_exceptions (); +else +load_current_idt (); }/*
arch/x86/kernel/irqinit.c +6 -1
@@ -28,6 +28,7 @@#include <asm/setup.h> #include <asm/i8259.h> #include <asm/traps.h> +#include <asm/fred.h> #include <asm/prom.h> /*
@@ -96,7 +97,11 @@ void __init native_init_IRQ(void)/* Execute any quirks before the call gates are initialised: */ x86_init.irqs.pre_vector_init (); -idt_setup_apic_and_irq_gates (); +if (cpu_feature_enabled (X86_FEATURE_FRED )) +fred_complete_exception_setup (); +else +idt_setup_apic_and_irq_gates (); +lapic_assign_system_vectors ();if (!acpi_ioapic && !of_ioapic &&nr_legacy_irqs ()) {
arch/x86/kernel/traps.c +4 -1
@@ -1412,7 +1412,10 @@ void __init trap_init(void)/* Initialize TSS before setting up traps so ISTs work */ cpu_init_exception_handling (); +/* Setup traps as cpu_init() might #GP */ -idt_setup_traps (); +if (!cpu_feature_enabled (X86_FEATURE_FRED )) +idt_setup_traps (); +cpu_init (); }
[tip: x86/fred] x86/fred: Invoke FRED initialization code to enable FRED
In reply to Xin LiThe following commit has been merged into the x86/fred branch of tip:
The following commit has been merged into the x86/fred branch of tip:
Commit-ID: b564b0111a3f03d1a92ba87c4b0f054ad1845963 Gitweb: https://git.kernel.org/tip/b564b0111a3f03d1a92ba87c4b0f054ad1845963 Author: H. Peter Anvin (Intel) <[email protected]> AuthorDate: Tue, 05 Dec 2023 02:50:24 -08:00 Committer: Borislav Petkov (AMD) <[email protected]> CommitterDate: Tue, 30 Jan 2024 18:20:36 +01:00
x86/fred: Invoke FRED initialization code to enable FRED
Let cpu_init_exception_handling() call cpu_init_fred_exceptions() to initialize FRED. However if FRED is unavailable or disabled, it falls back to set up TSS IST and initialize IDT.
--- arch/x86/kernel/cpu/common.c | 22 +++++++++++++++++----- arch/x86/kernel/irqinit.c | 7 ++++++- arch/x86/kernel/traps.c | 5 ++++- 3 files changed, 27 insertions(+), 7 deletions(-)
arch/x86/kernel/cpu/common.c +17 -5
@@ -61,6 +61,7 @@#include <asm/microcode.h> #include <asm/intel-family.h> #include <asm/cpu_device_id.h> +#include <asm/fred.h> #include <asm/uv/uv.h> #include <asm/ia32.h> #include <asm/set_memory.h>
@@ -2107,7 +2108,15 @@ void syscall_init(void)/* The default user and kernel segments */ wrmsr (MSR_STAR ,0 , (__USER32_CS <<16 ) | __KERNEL_CS); - idt_syscall_init(); +/* +* Except the IA32_STAR MSR, there is NO need to setup SYSCALL and +* SYSENTER MSRs for FRED, because FRED uses the ring 3 FRED +* entrypoint for SYSCALL and SYSENTER, and ERETU is the only legit +* instruction to return to ring 3 (both sysexit and sysret cause +* #UD when FRED is enabled). +*/ +if (!cpu_feature_enabled (X86_FEATURE_FRED )) + idt_syscall_init(); }#else /* CONFIG_X86_64 */
@@ -2213,8 +2222,9 @@ void cpu_init_exception_handling(void)/* paranoid_entry() gets the CPU number from the GDT */ setup_getcpu (cpu); -/* IST vectors need TSS to be set up. */ -tss_setup_ist (tss); +/* For IDT mode, IST vectors need to be set in TSS. */ +if (!cpu_feature_enabled (X86_FEATURE_FRED )) +tss_setup_ist (tss);tss_setup_io_bitmap (tss);set_tss_desc (cpu, &get_cpu_entry_area (cpu)->tss.x86_tss);
@@ -2223,8 +2233,10 @@ void cpu_init_exception_handling(void)/* GHCB needs to be setup to handle #VC. */ setup_ghcb (); -/* Finally load the IDT */ -load_current_idt (); +if (cpu_feature_enabled (X86_FEATURE_FRED )) +cpu_init_fred_exceptions (); +else +load_current_idt (); }/*
arch/x86/kernel/irqinit.c +6 -1
@@ -28,6 +28,7 @@#include <asm/setup.h> #include <asm/i8259.h> #include <asm/traps.h> +#include <asm/fred.h> #include <asm/prom.h> /*
@@ -96,7 +97,11 @@ void __init native_init_IRQ(void)/* Execute any quirks before the call gates are initialised: */ x86_init.irqs.pre_vector_init (); -idt_setup_apic_and_irq_gates (); +if (cpu_feature_enabled (X86_FEATURE_FRED )) +fred_complete_exception_setup (); +else +idt_setup_apic_and_irq_gates (); +lapic_assign_system_vectors ();if (!acpi_ioapic && !of_ioapic &&nr_legacy_irqs ()) {
arch/x86/kernel/traps.c +4 -1
@@ -1438,7 +1438,10 @@ void __init trap_init(void)/* Initialize TSS before setting up traps so ISTs work */ cpu_init_exception_handling (); +/* Setup traps as cpu_init() might #GP */ -idt_setup_traps (); +if (!cpu_feature_enabled (X86_FEATURE_FRED )) +idt_setup_traps (); +cpu_init (); }
[tip: x86/fred] x86/fred: Invoke FRED initialization code to enable FRED
In reply to Xin LiThe following commit has been merged into the x86/fred branch of tip:
The following commit has been merged into the x86/fred branch of tip:
Commit-ID: 208d8c79fd0f155bce1b23d8d78926653f7603b7 Gitweb: https://git.kernel.org/tip/208d8c79fd0f155bce1b23d8d78926653f7603b7 Author: H. Peter Anvin (Intel) <[email protected]> AuthorDate: Tue, 05 Dec 2023 02:50:24 -08:00 Committer: Borislav Petkov (AMD) <[email protected]> CommitterDate: Wed, 31 Jan 2024 22:03:36 +01:00
x86/fred: Invoke FRED initialization code to enable FRED
Let cpu_init_exception_handling() call cpu_init_fred_exceptions() to initialize FRED. However if FRED is unavailable or disabled, it falls back to set up TSS IST and initialize IDT.
--- arch/x86/kernel/cpu/common.c | 22 +++++++++++++++++----- arch/x86/kernel/irqinit.c | 7 ++++++- arch/x86/kernel/traps.c | 5 ++++- 3 files changed, 27 insertions(+), 7 deletions(-)
arch/x86/kernel/cpu/common.c +17 -5
@@ -61,6 +61,7 @@#include <asm/microcode.h> #include <asm/intel-family.h> #include <asm/cpu_device_id.h> +#include <asm/fred.h> #include <asm/uv/uv.h> #include <asm/ia32.h> #include <asm/set_memory.h>
@@ -2107,7 +2108,15 @@ void syscall_init(void)/* The default user and kernel segments */ wrmsr (MSR_STAR ,0 , (__USER32_CS <<16 ) | __KERNEL_CS); - idt_syscall_init(); +/* +* Except the IA32_STAR MSR, there is NO need to setup SYSCALL and +* SYSENTER MSRs for FRED, because FRED uses the ring 3 FRED +* entrypoint for SYSCALL and SYSENTER, and ERETU is the only legit +* instruction to return to ring 3 (both sysexit and sysret cause +* #UD when FRED is enabled). +*/ +if (!cpu_feature_enabled (X86_FEATURE_FRED )) + idt_syscall_init(); }#else /* CONFIG_X86_64 */
@@ -2213,8 +2222,9 @@ void cpu_init_exception_handling(void)/* paranoid_entry() gets the CPU number from the GDT */ setup_getcpu (cpu); -/* IST vectors need TSS to be set up. */ -tss_setup_ist (tss); +/* For IDT mode, IST vectors need to be set in TSS. */ +if (!cpu_feature_enabled (X86_FEATURE_FRED )) +tss_setup_ist (tss);tss_setup_io_bitmap (tss);set_tss_desc (cpu, &get_cpu_entry_area (cpu)->tss.x86_tss);
@@ -2223,8 +2233,10 @@ void cpu_init_exception_handling(void)/* GHCB needs to be setup to handle #VC. */ setup_ghcb (); -/* Finally load the IDT */ -load_current_idt (); +if (cpu_feature_enabled (X86_FEATURE_FRED )) +cpu_init_fred_exceptions (); +else +load_current_idt (); }/*
arch/x86/kernel/irqinit.c +6 -1
@@ -28,6 +28,7 @@#include <asm/setup.h> #include <asm/i8259.h> #include <asm/traps.h> +#include <asm/fred.h> #include <asm/prom.h> /*
@@ -96,7 +97,11 @@ void __init native_init_IRQ(void)/* Execute any quirks before the call gates are initialised: */ x86_init.irqs.pre_vector_init (); -idt_setup_apic_and_irq_gates (); +if (cpu_feature_enabled (X86_FEATURE_FRED )) +fred_complete_exception_setup (); +else +idt_setup_apic_and_irq_gates (); +lapic_assign_system_vectors ();if (!acpi_ioapic && !of_ioapic &&nr_legacy_irqs ()) {
arch/x86/kernel/traps.c +4 -1
@@ -1438,7 +1438,10 @@ void __init trap_init(void)/* Initialize TSS before setting up traps so ISTs work */ cpu_init_exception_handling (); +/* Setup traps as cpu_init() might #GP */ -idt_setup_traps (); +if (!cpu_feature_enabled (X86_FEATURE_FRED )) +idt_setup_traps (); +cpu_init (); }