x86/kvm/vmx: Fix VMX interrupt injection vs hrtimer_rearm_deferred()
Subsystems (5)
cover x86/kvm/vmx: Fix VMX interrupt injection vs hrtimer_rearm_deferred()
Hi,
Nicely split out in two patches, but otherwise unchanged (except for a small comment).
PATCH 1/2 x86/kvm/vmx: Move IRQ/NMI dispatch from KVM into x86 core
Move the VMX interrupt dispatch magic into the x86 core code. This isolates KVM from the FRED/IDT decisions and reduces the amount of EXPORT_SYMBOL_FOR_KVM().
--- arch/x86/entry/Makefile | 2 - arch/x86/entry/common.c | 48 ++++++++++++++++++++++++++++++++++++ arch/x86/entry/entry.S | 46 ++++++++++++++++++++++++++++++++++ arch/x86/entry/entry_64_fred.S | 1 arch/x86/include/asm/desc.h | 4 +++ arch/x86/include/asm/desc_defs.h | 2 - arch/x86/include/asm/entry-common.h | 2 + arch/x86/include/asm/fred.h | 1 arch/x86/include/asm/idtentry.h | 11 -------- arch/x86/kernel/idt.c | 13 +++++++++ arch/x86/kernel/nmi.c | 8 ------ arch/x86/kvm/vmx/vmenter.S | 46 ---------------------------------- arch/x86/kvm/vmx/vmx.c | 20 ++------------- 13 files changed, 118 insertions(+), 86 deletions(-)
arch/x86/entry/Makefile +1 -1
@@ -13,7 +13,7 @@ CFLAGS_REMOVE_syscall_64.o = $(CC_FLAGS_ CFLAGS_syscall_32.o += -fno-stack-protector CFLAGS_syscall_64.o += -fno-stack-protector -obj-y := entry.o entry_$(BITS).o syscall_$(BITS).o +obj-y := entry.o entry_$(BITS).o syscall_$(BITS).o common.o obj-y += vdso/ obj-y += vsyscall/
arch/x86/entry/common.c new file +48 -0
48 lines not on this page · raw message
arch/x86/entry/entry.S +46 -0
49 lines not on this page · raw message
arch/x86/entry/entry_64_fred.S +0 -1
@@ -147,5 +147,4 @@ SYM_FUNC_START(asm_fred_entry_from_kvm) RET SYM_FUNC_END(asm_fred_entry_from_kvm) -EXPORT_SYMBOL_FOR_KVM(asm_fred_entry_from_kvm); #endif
arch/x86/include/asm/desc.h +4 -0
@@ -438,6 +438,10 @@ extern void idt_setup_traps(void);extern void idt_setup_apic_and_irq_gates (void );extern bool idt_is_f00f_address (unsigned long address); +extern void idt_do_interrupt_irqoff (unsigned int vector); +extern void idt_do_nmi_irqoff (void ); +extern void idt_entry_from_kvm (unsigned int vector); +#ifdef CONFIG_X86_64 extern void idt_setup_early_pf (void );#else
arch/x86/include/asm/desc_defs.h +1 -1
@@ -145,7 +145,7 @@ struct gate_struct {typedef struct gate_struct gate_desc ;#ifndef _SETUP -static inline unsigned long gate_offset (const gate_desc *g) +static __always_inline unsigned long gate_offset (const gate_desc *g) {#ifdef CONFIG_X86_64 return g->offset_low | ((unsigned long)g->offset_middle << 16) |
arch/x86/include/asm/entry-common.h +2 -0
@@ -97,4 +97,6 @@ static __always_inline void arch_exit_to }#define arch_exit_to_user_mode arch_exit_to_user_mode +extern void x86_entry_from_kvm (unsigned int entry_type,unsigned int vector); +#endif
arch/x86/include/asm/fred.h +0 -1
@@ -110,7 +110,6 @@ static __always_inline unsigned long frestatic inline void cpu_init_fred_exceptions (void ) { }static inline void cpu_init_fred_rsps (void ) { }static inline void fred_complete_exception_setup (void ) { } -static inline void fred_entry_from_kvm (unsigned int type,unsigned int vector) { }static inline void fred_sync_rsp0 (unsigned long rsp0) { }static inline void fred_update_rsp0 (void ) { }#endif /* CONFIG_X86_FRED */
arch/x86/include/asm/idtentry.h +0 -11
@@ -633,17 +633,6 @@ DECLARE_IDTENTRY_RAW(X86_TRAP_MC, xenpv_#endif /* NMI */ - -#if IS_ENABLED (CONFIG_KVM_INTEL ) -/* -* Special entry point for VMX which invokes this on the kernel stack, even for -* 64-bit, i.e. without using an IST. asm_exc_nmi() requires an IST to work -* correctly vs. the NMI 'executing' marker. Used for 32-bit kernels as well -* to avoid more ifdeffery. -*/ -DECLARE_IDTENTRY (X86_TRAP_NMI , exc_nmi_kvm_vmx); -#endif -DECLARE_IDTENTRY_NMI (X86_TRAP_NMI , exc_nmi);#ifdef CONFIG_XEN_PV DECLARE_IDTENTRY_RAW (X86_TRAP_NMI , xenpv_exc_nmi);
arch/x86/kernel/idt.c +13 -0
@@ -268,6 +268,19 @@ void __init idt_setup_early_pf(void) }#endif +noinstr void idt_entry_from_kvm(unsigned int vector) +{ +if (vector ==NMI_VECTOR ) +return idt_do_nmi_irqoff (); + +/* +* Only the NMI path requires noinstr. +*/ +instrumentation_begin (); +idt_do_interrupt_irqoff (gate_offset (idt_table + vector)); +instrumentation_end (); +} +static void __init idt_map_in_cea (void ) {/*
arch/x86/kernel/nmi.c +0 -8
@@ -609,14 +609,6 @@ DEFINE_IDTENTRY_RAW(exc_nmi) goto nmi_restart; } -#if IS_ENABLED (CONFIG_KVM_INTEL ) -DEFINE_IDTENTRY_RAW (exc_nmi_kvm_vmx) -{ -exc_nmi (regs); -} -EXPORT_SYMBOL_FOR_KVM (asm_exc_nmi_kvm_vmx); -#endif -#ifdef CONFIG_NMI_CHECK_CPU static char *nmi_check_stall_msg[] = {
arch/x86/kvm/vmx/vmenter.S +0 -46
61 lines not on this page · raw message
arch/x86/kvm/vmx/vmx.c +3 -17
@@ -7083,9 +7083,6 @@ void vmx_load_eoi_exitmap(struct kvm_vcpvmcs_write64 (EOI_EXIT_BITMAP3 , eoi_exit_bitmap[3 ]); } -void vmx_do_interrupt_irqoff (unsigned long entry); -void vmx_do_nmi_irqoff (void ); -static void handle_nm_fault_irqoff (struct kvm_vcpu *vcpu) {/*
@@ -7127,17 +7124,9 @@ static void handle_external_interrupt_ir "unexpected VM-Exit interrupt info: 0x%x", intr_info)) return; -/* -* Invoke the kernel's IRQ handler for the vector. Use the FRED path -* when it's available even if FRED isn't fully enabled, e.g. even if -* FRED isn't supported in hardware, in order to avoid the indirect -* CALL in the non-FRED path. -*/ +/* For the IRQ to the core kernel for processing. */ kvm_before_interrupt (vcpu,KVM_HANDLING_IRQ ); -if (IS_ENABLED (CONFIG_X86_FRED )) -fred_entry_from_kvm (EVENT_TYPE_EXTINT , vector); -else -vmx_do_interrupt_irqoff (gate_offset ((gate_desc *)host_idt_base + vector)); +x86_entry_from_kvm (EVENT_TYPE_EXTINT , vector);kvm_after_interrupt (vcpu); vcpu->arch.at_instruction_boundary = true;
@@ -7447,10 +7436,7 @@ noinstr void vmx_handle_nmi(struct kvm_vreturn ;kvm_before_interrupt (vcpu,KVM_HANDLING_NMI ); -if (cpu_feature_enabled (X86_FEATURE_FRED )) -fred_entry_from_kvm (EVENT_TYPE_NMI ,NMI_VECTOR ); -else -vmx_do_nmi_irqoff (); +x86_entry_from_kvm (EVENT_TYPE_NMI ,NMI_VECTOR );kvm_after_interrupt (vcpu); }
Re: [PATCH 1/2] x86/kvm/vmx: Move IRQ/NMI dispatch from KVM into x86 core
Few nits below, but
Quoted 12 lines · Peter Zijlstra
Move the VMX interrupt dispatch magic into the x86 core code. This isolates KVM from the FRED/IDT decisions and reduces the amount of
EXPORT_SYMBOL_FOR_KVM().Suggested-by Sean Christopherson <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>Tested-by "Verma, Vishal L" <[email protected]>
arch/x86/entry/common.c line 6
Quoted 6 lines
arch/x86/entry/common.c new file +6
-0@@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#include <linux/entry-common.h> +#include <linux/kvm_types.h> +#include <asm/fred.h> +#include <asm/desc.h>
Swap the two header inclusion order?
arch/x86/entry/common.c line 17
Quoted 11 lines
arch/x86/entry/common.c new file +11
-0+ +#if IS_ENABLED (CONFIG_KVM_INTEL ) +/* +* On VMX, NMIs and IRQs (as configured by KVM) are acknowledge by hardware as +* part of the VM-Exit, i.e. the event itself is consumed as part the VM-Exit. +* x86_entry_from_kvm() is invoked by KVM to effectively forward NMIs and IRQs +* to the kernel for servicing. On SVM, a.k.a. AMD, the NMI/IRQ VM-Exit is +* purely a signal that an NMI/IRQ is pending, i.e. the event that triggered +* the VM-Exit is held pending until it's unblocked in the host. +*/ +noinstr void x86_entry_from_kvm(unsigned int event_type, unsigned int vector)
Is u32 better than unsigned int as is close to hardware definition?
Currently, the event type and event vector are defined as 4-bit and 8-bit fields, respectively; and they can be readily expanded to 8 and 16 bits by utilizing adjacent reserved bits.
arch/x86/entry/common.c line 26
Quoted 9 lines
arch/x86/entry/common.c new file +9
-0+{ +if (event_type ==EVENT_TYPE_EXTINT ) { +#ifdef CONFIG_X86_64 +/* +* Use FRED dispatch, even when running IDT. The dispatch +* tables are kept in sync between FRED and IDT, and the FRED +* dispatch works well with CFI. +*/ +fred_entry_from_kvm (event_type, vector);
Then should we rename the “fred_” prefix, say “x86_64_"?
Sorry naming is hard to me ;)
Re: [PATCH 1/2] x86/kvm/vmx: Move IRQ/NMI dispatch from KVM into x86 core
In reply to Peter ZijlstraQuoted 37 lines · Peter Zijlstra
Move the VMX interrupt dispatch magic into the x86 core code. This isolates KVM from the FRED/IDT decisions and reduces the amount of
EXPORT_SYMBOL_FOR_KVM().Suggested-by Sean Christopherson <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>Tested-by "Verma, Vishal L" <[email protected]>--- arch/x86/entry/Makefile | 2 - arch/x86/entry/common.c | 48 ++++++++++++++++++++++++++++++++++++ arch/x86/entry/entry.S | 46 ++++++++++++++++++++++++++++++++++ arch/x86/entry/entry_64_fred.S | 1 arch/x86/include/asm/desc.h | 4 +++ arch/x86/include/asm/desc_defs.h | 2 - arch/x86/include/asm/entry-common.h | 2 + arch/x86/include/asm/fred.h | 1 arch/x86/include/asm/idtentry.h | 11 -------- arch/x86/kernel/idt.c | 13 +++++++++ arch/x86/kernel/nmi.c | 8 ------ arch/x86/kvm/vmx/vmenter.S | 46 ---------------------------------- arch/x86/kvm/vmx/vmx.c | 20 ++------------- 13 files changed, 118 insertions(+), 86 deletions(-)arch/x86/entry/Makefile +1
-1@@ -13,7 +13,7 @@ CFLAGS_REMOVE_syscall_64.o = $(CC_FLAGS_ CFLAGS_syscall_32.o += -fno-stack-protector CFLAGS_syscall_64.o += -fno-stack-protector -obj-y := entry.o entry_$(BITS).o syscall_$(BITS).o +obj-y := entry.o entry_$(BITS).o syscall_$(BITS).o common.o obj-y += vdso/ obj-y += vsyscall/
arch/x86/entry/common.c line 10
Quoted 10 lines
arch/x86/entry/common.c new file +10
-0@@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#include <linux/entry-common.h> +#include <linux/kvm_types.h> +#include <asm/fred.h> +#include <asm/desc.h> + +#if IS_ENABLED (CONFIG_KVM_INTEL ) +/* +* On VMX, NMIs and IRQs (as configured by KVM) are acknowledge by hardware as
s/acknowledge/acknowledged
Quoted 6 lines
arch/x86/entry/common.c new file +6
-0+* part of the VM-Exit, i.e. the event itself is consumed as part the VM-Exit. +* x86_entry_from_kvm() is invoked by KVM to effectively forward NMIs and IRQs +* to the kernel for servicing. On SVM, a.k.a. AMD, the NMI/IRQ VM-Exit is +* purely a signal that an NMI/IRQ is pending, i.e. the event that triggered +* the VM-Exit is held pending until it's unblocked in the host. +*/
[...]
arch/x86/entry/common.c
Quoted 5 lines
arch/x86/entry/common.c new file +0
-5- -#if IS_ENABLED (CONFIG_KVM_INTEL ) -/* -* Special entry point for VMX which invokes this on the kernel stack, even for -* 64-bit, i.e. without using an IST. asm_exc_nmi() requires an IST to work
Although it's being removed, I guess what it says is still true?
It says asm_exc_nmi() requires an IST to work correctly, and the new path for handling NMI when FRED is disabled.
idt_entry_from_kvm
idt_do_nmi_irqoff
IDT_DO_EVENT_IRQOFF call asm_exc_nmi
...
call asm_exc_nmi
It seems the stack before calling asm_exc_nmi is not an IST? Does it matter?
Quoted 9 lines
- * correctly vs. the NMI 'executing' marker. Used for 32-bit kernels as well
- * to avoid more ifdeffery.
- */arch/x86/entry/common.c new file +0
-3-DECLARE_IDTENTRY (X86_TRAP_NMI , exc_nmi_kvm_vmx); -#endif -DECLARE_IDTENTRY_NMI (X86_TRAP_NMI , exc_nmi);#ifdef CONFIG_XEN_PV DECLARE_IDTENTRY_RAW (X86_TRAP_NMI , xenpv_exc_nmi);
[...]
arch/x86/entry/common.c static void handle_external_interrupt_ir · line 7127
Quoted 11 lines
arch/x86/entry/common.c new file +1
-6@@ -7127,17 +7124,9 @@ static void handle_external_interrupt_ir "unexpected VM-Exit interrupt info: 0x%x", intr_info)) return; -/* -* Invoke the kernel's IRQ handler for the vector. Use the FRED path -* when it's available even if FRED isn't fully enabled, e.g. even if -* FRED isn't supported in hardware, in order to avoid the indirect -* CALL in the non-FRED path. -*/ +/* For the IRQ to the core kernel for processing. */
For -> Forward?
Quoted 24 lines
arch/x86/entry/common.c new file +2
-8kvm_before_interrupt (vcpu,KVM_HANDLING_IRQ ); -if (IS_ENABLED (CONFIG_X86_FRED )) -fred_entry_from_kvm (EVENT_TYPE_EXTINT , vector); -else -vmx_do_interrupt_irqoff (gate_offset ((gate_desc *)host_idt_base + vector)); +x86_entry_from_kvm (EVENT_TYPE_EXTINT , vector);kvm_after_interrupt (vcpu); vcpu->arch.at_instruction_boundary = true;@@ -7447,10 +7436,7 @@ noinstr void vmx_handle_nmi(struct kvm_vreturn ;kvm_before_interrupt (vcpu,KVM_HANDLING_NMI ); -if (cpu_feature_enabled (X86_FEATURE_FRED )) -fred_entry_from_kvm (EVENT_TYPE_NMI ,NMI_VECTOR ); -else -vmx_do_nmi_irqoff (); +x86_entry_from_kvm (EVENT_TYPE_NMI ,NMI_VECTOR );kvm_after_interrupt (vcpu); }
Paolo Bonzini
maintainerRe: [PATCH 1/2] x86/kvm/vmx: Move IRQ/NMI dispatch from KVM into x86 core
Quoted 73 lines · Binbin Wu
Quoted 37 lines · Peter Zijlstra
Move the VMX interrupt dispatch magic into the x86 core code. This isolates KVM from the FRED/IDT decisions and reduces the amount of
EXPORT_SYMBOL_FOR_KVM().Suggested-by Sean Christopherson <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>Tested-by "Verma, Vishal L" <[email protected]>--- arch/x86/entry/Makefile | 2 - arch/x86/entry/common.c | 48 ++++++++++++++++++++++++++++++++++++ arch/x86/entry/entry.S | 46 ++++++++++++++++++++++++++++++++++ arch/x86/entry/entry_64_fred.S | 1 arch/x86/include/asm/desc.h | 4 +++ arch/x86/include/asm/desc_defs.h | 2 - arch/x86/include/asm/entry-common.h | 2 + arch/x86/include/asm/fred.h | 1 arch/x86/include/asm/idtentry.h | 11 -------- arch/x86/kernel/idt.c | 13 +++++++++ arch/x86/kernel/nmi.c | 8 ------ arch/x86/kvm/vmx/vmenter.S | 46 ---------------------------------- arch/x86/kvm/vmx/vmx.c | 20 ++------------- 13 files changed, 118 insertions(+), 86 deletions(-)arch/x86/entry/Makefile +1
-1@@ -13,7 +13,7 @@ CFLAGS_REMOVE_syscall_64.o = $(CC_FLAGS_ CFLAGS_syscall_32.o += -fno-stack-protector CFLAGS_syscall_64.o += -fno-stack-protector -obj-y := entry.o entry_$(BITS).o syscall_$(BITS).o +obj-y := entry.o entry_$(BITS).o syscall_$(BITS).o common.o obj-y += vdso/ obj-y += vsyscall/arch/x86/entry/common.c line 10
Quoted 10 lines
arch/x86/entry/common.c new file +10
-0@@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#include <linux/entry-common.h> +#include <linux/kvm_types.h> +#include <asm/fred.h> +#include <asm/desc.h> + +#if IS_ENABLED (CONFIG_KVM_INTEL ) +/* +* On VMX, NMIs and IRQs (as configured by KVM) are acknowledge by hardware as s/acknowledge/acknowledged
Quoted 6 lines
arch/x86/entry/common.c new file +6
-0+* part of the VM-Exit, i.e. the event itself is consumed as part the VM-Exit. +* x86_entry_from_kvm() is invoked by KVM to effectively forward NMIs and IRQs +* to the kernel for servicing. On SVM, a.k.a. AMD, the NMI/IRQ VM-Exit is +* purely a signal that an NMI/IRQ is pending, i.e. the event that triggered +* the VM-Exit is held pending until it's unblocked in the host. +*/ [...]
arch/x86/entry/common.c
Quoted 5 lines
arch/x86/entry/common.c new file +0
-5- -#if IS_ENABLED (CONFIG_KVM_INTEL ) -/* -* Special entry point for VMX which invokes this on the kernel stack, even for -* 64-bit, i.e. without using an IST. asm_exc_nmi() requires an IST to work Although it's being removed, I guess what it says is still true?
It says
asm_exc_nmi()requires an IST to work correctly, and the new path for handling NMI when FRED is disabled.
Quoted 8 lines
idt_entry_from_kvm
idt_do_nmi_irqoff
IDT_DO_EVENT_IRQOFFcallasm_exc_nmi
...
callasm_exc_nmiIt seems the stack before calling
asm_exc_nmiis not an IST? Does it matter?
I think it does, the IST is needed because of all the stuff to detect recursive NMIs. So asm_exc_nmi_kvm_vmx needs to remain.
By the way, here:
arch/x86/entry/common.c line 44
Quoted 5 lines
arch/x86/entry/common.c new file +5
-0+/* +* Notably, we must use IDT dispatch for NMI when running in IDT mode. +* The FRED NMI context is significantly different and will not work +* right (speficially FRED fixed the NMI recursion issue). +*/
It's even more important to note that NMIs need an IRET in order to unblock further NMIs. This is even more important than the recursion issue, which does not affect KVM's non-IST entry into the NMI handler, and is the real reason to use IDT_DO_EVENT_IRQOFF to build the interrupt stack frame for NMIs.
Paolo
Re: [PATCH 1/2] x86/kvm/vmx: Move IRQ/NMI dispatch from KVM into x86 core
Quoted 39 lines · Paolo Bonzini
Quoted 25 lines
arch/x86/entry/common.c line 10
Quoted 3 lines
arch/x86/entry/common.c new file +3
-0+#if IS_ENABLED (CONFIG_KVM_INTEL ) +/* +* On VMX, NMIs and IRQs (as configured by KVM) are acknowledge by hardware as s/acknowledge/acknowledged
Quoted 6 lines
+ * part of the VM-Exit, i.e. the event itself is consumed as part the VM-Exit.
+ *x86_entry_from_kvm()is invoked by KVM to effectively forward NMIs and IRQs
+ * to the kernel for servicing. On SVM, a.k.a. AMD, the NMI/IRQ VM-Exit is
+ * purely a signal that an NMI/IRQ is pending, i.e. the event that triggered
+ * the VM-Exit is held pending until it's unblocked in the host.
+ */[...]
arch/x86/include/asm/idtentry.h DECLARE_IDTENTRY_RAW(X86_TRAP_MC, xenpv_ · line 640
Quoted 5 lines
arch/x86/include/asm/idtentry.h +0
-5- -#if IS_ENABLED (CONFIG_KVM_INTEL ) -/* -* Special entry point for VMX which invokes this on the kernel stack, even for -* 64-bit, i.e. without using an IST. asm_exc_nmi() requires an IST to work Although it's being removed, I guess what it says is still true?
It says
asm_exc_nmi()requires an IST to work correctly, and the new path for handling NMI when FRED is disabled.Quoted 8 lines
idt_entry_from_kvm
idt_do_nmi_irqoff
IDT_DO_EVENT_IRQOFFcallasm_exc_nmi
...
callasm_exc_nmiIt seems the stack before calling
asm_exc_nmiis not an IST? Does it matter?I think it does, the IST is needed because of all the stuff to detect recursive NMIs. So
asm_exc_nmi_kvm_vmxneeds to remain.By the way, here:
Quoted 11 lines
arch/x86/entry/common.c line 44
Quoted 5 lines
arch/x86/entry/common.c new file +5
-0+/* +* Notably, we must use IDT dispatch for NMI when running in IDT mode. +* The FRED NMI context is significantly different and will not work +* right (speficially FRED fixed the NMI recursion issue). +*/ It's even more important to note that NMIs need an IRET in order to unblock further NMIs. This is even more important than the recursion issue, which does not affect KVM's non-IST entry into the NMI handler, and is the real reason to use
IDT_DO_EVENT_IRQOFFto build the interrupt stack frame for NMIs.
Durr, I missed that: DECLARE_IDTENTRY_NMI != DECLARE_IDTENTRY_RAW
Let me go rectify that.
PATCH v2 1/2 x86/kvm/vmx: Move IRQ/NMI dispatch from KVM into x86 core
In reply to Peter ZijlstraMove the VMX interrupt dispatch magic into the x86 core code. This isolates KVM from the FRED/IDT decisions and reduces the amount of EXPORT_SYMBOL_FOR_KVM().
--- arch/x86/entry/Makefile | 2 - arch/x86/entry/common.c | 48 ++++++++++++++++++++++++++++++++++++ arch/x86/entry/entry.S | 46 ++++++++++++++++++++++++++++++++++ arch/x86/entry/entry_64_fred.S | 1 arch/x86/include/asm/desc.h | 4 +++ arch/x86/include/asm/desc_defs.h | 2 - arch/x86/include/asm/entry-common.h | 2 + arch/x86/include/asm/fred.h | 1 arch/x86/kernel/idt.c | 13 +++++++++ arch/x86/kernel/nmi.c | 1 arch/x86/kvm/vmx/vmenter.S | 46 ---------------------------------- arch/x86/kvm/vmx/vmx.c | 20 ++------------- 12 files changed, 118 insertions(+), 68 deletions(-)
arch/x86/entry/Makefile +1 -1
@@ -13,7 +13,7 @@ CFLAGS_REMOVE_syscall_64.o = $(CC_FLAGS_ CFLAGS_syscall_32.o += -fno-stack-protector CFLAGS_syscall_64.o += -fno-stack-protector -obj-y := entry.o entry_$(BITS).o syscall_$(BITS).o +obj-y := entry.o entry_$(BITS).o syscall_$(BITS).o common.o obj-y += vdso/ obj-y += vsyscall/
arch/x86/entry/common.c new file +48 -0
48 lines not on this page · raw message
arch/x86/entry/entry.S +46 -0
49 lines not on this page · raw message
arch/x86/entry/entry_64_fred.S +0 -1
@@ -147,5 +147,4 @@ SYM_FUNC_START(asm_fred_entry_from_kvm) RET SYM_FUNC_END(asm_fred_entry_from_kvm) -EXPORT_SYMBOL_FOR_KVM(asm_fred_entry_from_kvm); #endif
arch/x86/include/asm/desc.h +4 -0
@@ -438,6 +438,10 @@ extern void idt_setup_traps(void);extern void idt_setup_apic_and_irq_gates (void );extern bool idt_is_f00f_address (unsigned long address); +extern void idt_do_interrupt_irqoff (unsigned int vector); +extern void idt_do_nmi_irqoff (void ); +extern void idt_entry_from_kvm (unsigned int vector); +#ifdef CONFIG_X86_64 extern void idt_setup_early_pf (void );#else
arch/x86/include/asm/desc_defs.h +1 -1
@@ -145,7 +145,7 @@ struct gate_struct {typedef struct gate_struct gate_desc ;#ifndef _SETUP -static inline unsigned long gate_offset (const gate_desc *g) +static __always_inline unsigned long gate_offset (const gate_desc *g) {#ifdef CONFIG_X86_64 return g->offset_low | ((unsigned long)g->offset_middle << 16) |
arch/x86/include/asm/entry-common.h +2 -0
@@ -97,4 +97,6 @@ static __always_inline void arch_exit_to }#define arch_exit_to_user_mode arch_exit_to_user_mode +extern void x86_entry_from_kvm (unsigned int entry_type,unsigned int vector); +#endif
arch/x86/include/asm/fred.h +0 -1
@@ -110,7 +110,6 @@ static __always_inline unsigned long frestatic inline void cpu_init_fred_exceptions (void ) { }static inline void cpu_init_fred_rsps (void ) { }static inline void fred_complete_exception_setup (void ) { } -static inline void fred_entry_from_kvm (unsigned int type,unsigned int vector) { }static inline void fred_sync_rsp0 (unsigned long rsp0) { }static inline void fred_update_rsp0 (void ) { }#endif /* CONFIG_X86_FRED */
arch/x86/kernel/idt.c +13 -0
@@ -268,6 +268,19 @@ void __init idt_setup_early_pf(void) }#endif +noinstr void idt_entry_from_kvm(unsigned int vector) +{ +if (vector ==NMI_VECTOR ) +return idt_do_nmi_irqoff (); + +/* +* Only the NMI path requires noinstr. +*/ +instrumentation_begin (); +idt_do_interrupt_irqoff (gate_offset (idt_table + vector)); +instrumentation_end (); +} +static void __init idt_map_in_cea (void ) {/*
arch/x86/kernel/nmi.c +0 -1
@@ -614,7 +614,6 @@ DEFINE_IDTENTRY_RAW(exc_nmi_kvm_vmx) {exc_nmi (regs); } -EXPORT_SYMBOL_FOR_KVM (asm_exc_nmi_kvm_vmx);#endif #ifdef CONFIG_NMI_CHECK_CPU
arch/x86/kvm/vmx/vmenter.S +0 -46
61 lines not on this page · raw message
arch/x86/kvm/vmx/vmx.c +3 -17
@@ -7083,9 +7083,6 @@ void vmx_load_eoi_exitmap(struct kvm_vcpvmcs_write64 (EOI_EXIT_BITMAP3 , eoi_exit_bitmap[3 ]); } -void vmx_do_interrupt_irqoff (unsigned long entry); -void vmx_do_nmi_irqoff (void ); -static void handle_nm_fault_irqoff (struct kvm_vcpu *vcpu) {/*
@@ -7127,17 +7124,9 @@ static void handle_external_interrupt_ir "unexpected VM-Exit interrupt info: 0x%x", intr_info)) return; -/* -* Invoke the kernel's IRQ handler for the vector. Use the FRED path -* when it's available even if FRED isn't fully enabled, e.g. even if -* FRED isn't supported in hardware, in order to avoid the indirect -* CALL in the non-FRED path. -*/ +/* For the IRQ to the core kernel for processing. */ kvm_before_interrupt (vcpu,KVM_HANDLING_IRQ ); -if (IS_ENABLED (CONFIG_X86_FRED )) -fred_entry_from_kvm (EVENT_TYPE_EXTINT , vector); -else -vmx_do_interrupt_irqoff (gate_offset ((gate_desc *)host_idt_base + vector)); +x86_entry_from_kvm (EVENT_TYPE_EXTINT , vector);kvm_after_interrupt (vcpu); vcpu->arch.at_instruction_boundary = true;
@@ -7447,10 +7436,7 @@ noinstr void vmx_handle_nmi(struct kvm_vreturn ;kvm_before_interrupt (vcpu,KVM_HANDLING_NMI ); -if (cpu_feature_enabled (X86_FEATURE_FRED )) -fred_entry_from_kvm (EVENT_TYPE_NMI ,NMI_VECTOR ); -else -vmx_do_nmi_irqoff (); +x86_entry_from_kvm (EVENT_TYPE_NMI ,NMI_VECTOR );kvm_after_interrupt (vcpu); }
Re: [PATCH v2 1/2] x86/kvm/vmx: Move IRQ/NMI dispatch from KVM into x86 core
Quoted 21 lines · Peter Zijlstra
arch/x86/entry/common.c new file +18
-0@@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#include <linux/entry-common.h> +#include <linux/kvm_types.h> +#include <asm/fred.h> +#include <asm/desc.h> + +#if IS_ENABLED (CONFIG_KVM_INTEL ) +/* +* On VMX, NMIs and IRQs (as configured by KVM) are acknowledged by hardware as +* part of the VM-Exit, i.e. the event itself is consumed as part the VM-Exit. +* x86_entry_from_kvm() is invoked by KVM to effectively forward NMIs and IRQs +* to the kernel for servicing. On SVM, a.k.a. AMD, the NMI/IRQ VM-Exit is +* purely a signal that an NMI/IRQ is pending, i.e. the event that triggered +* the VM-Exit is held pending until it's unblocked in the host. +*/ +noinstr void x86_entry_from_kvm(unsigned int event_type, unsigned int vector) +{
arch/x86/entry/common.c line 28
Quoted 10 lines
arch/x86/entry/common.c new file +10
-0+if (event_type ==EVENT_TYPE_EXTINT ) { +#ifdef CONFIG_X86_64 +/* +* Use FRED dispatch, even when running IDT. The dispatch +* tables are kept in sync between FRED and IDT, and the FRED +* dispatch works well with CFI. +*/ +fred_entry_from_kvm (event_type, vector); +#else +idt_entry_from_kvm (vector);
FYI:
I can met below error with "make ARCH=i386 allnoconfig" or "make ARCH=x86_64 allnoconfig".
ld: vmlinux.o: in function `idt_entry_from_kvm':
(.noinstr.text+0x1618): undefined reference to `idt_do_interrupt_irqoff'
ld: (.noinstr.text+0x161f): undefined reference to `idt_do_nmi_irqoff' make[2]: *** [scripts/Makefile.vmlinux:72: vmlinux.unstripped] Error 1 make[1]: *** [/home/yan/kernel/tdx-upstream/tdx-kvm-upstream/Makefile:1335: vmlinux] Error 2
make: *** [Makefile:248: __sub-make] Error 2
So, I added below fix:
arch/x86/kernel/idt.c +2 -0
@@ -268,6 +268,7 @@ void __init idt_setup_early_pf(void) }#endif +#if IS_ENABLED (CONFIG_KVM_INTEL ) noinstr void idt_entry_from_kvm(unsigned int vector) {if (vector ==NMI_VECTOR )
@@ -280,6 +281,7 @@ noinstr void idt_entry_from_kvm(unsigned int vector)idt_do_interrupt_irqoff (gate_offset (idt_table + vector)); instrumentation_end(); } +#endif static void __init idt_map_in_cea (void ) {
Quoted 20 lines
arch/x86/entry/common.c new file +20
-0+#endif + return; + } + +WARN_ON_ONCE (event_type !=EVENT_TYPE_NMI ); + +#ifdef CONFIG_X86_64 +if (cpu_feature_enabled (X86_FEATURE_FRED )) +return fred_entry_from_kvm (event_type, vector); +#endif + +/* +* Notably, we must use IDT dispatch for NMI when running in IDT mode. +* The FRED NMI context is significantly different and will not work +* right (speficially FRED fixed the NMI recursion issue). +*/ +idt_entry_from_kvm (vector); +} +EXPORT_SYMBOL_FOR_KVM (x86_entry_from_kvm); +#endif
Re: [PATCH v2 1/2] x86/kvm/vmx: Move IRQ/NMI dispatch from KVM into x86 core
Quoted 15 lines · Yan Zhao
So, I added below fix:
arch/x86/kernel/idt.c +1
-0@@ -268,6 +268,7 @@ void __init idt_setup_early_pf(void) }#endif +#if IS_ENABLED (CONFIG_KVM_INTEL ) noinstr void idt_entry_from_kvm(unsigned int vector) {if (vector ==NMI_VECTOR )
arch/x86/kernel/idt.c noinstr void idt_entry_from_kvm(unsigned int vector) · line 287
Quoted 7 lines
arch/x86/kernel/idt.c +1
-0@@ -280,6 +281,7 @@ noinstr void idt_entry_from_kvm(unsigned int vector)idt_do_interrupt_irqoff (gate_offset (idt_table + vector)); instrumentation_end(); } +#endif static void __init idt_map_in_cea (void ) {
Thanks!
Re: [PATCH v2 1/2] x86/kvm/vmx: Move IRQ/NMI dispatch from KVM into x86 core
In reply to Peter ZijlstraOn 5/2/2026 4:37 AM, Peter Zijlstra wrote:
[...]
Quoted 26 lines
arch/x86/entry/common.c new file +23
-0@@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#include <linux/entry-common.h> +#include <linux/kvm_types.h> +#include <asm/fred.h> +#include <asm/desc.h> + +#if IS_ENABLED (CONFIG_KVM_INTEL ) +/* +* On VMX, NMIs and IRQs (as configured by KVM) are acknowledged by hardware as +* part of the VM-Exit, i.e. the event itself is consumed as part the VM-Exit. +* x86_entry_from_kvm() is invoked by KVM to effectively forward NMIs and IRQs +* to the kernel for servicing. On SVM, a.k.a. AMD, the NMI/IRQ VM-Exit is +* purely a signal that an NMI/IRQ is pending, i.e. the event that triggered +* the VM-Exit is held pending until it's unblocked in the host. +*/ +noinstr void x86_entry_from_kvm(unsigned int event_type, unsigned int vector) +{ +if (event_type ==EVENT_TYPE_EXTINT ) { +#ifdef CONFIG_X86_64 +/* +* Use FRED dispatch, even when running IDT. The dispatch +* tables are kept in sync between FRED and IDT, and the FRED
arch/x86/entry/common.c line 33
Quoted 10 lines
arch/x86/entry/common.c new file +10
-0+* dispatch works well with CFI. +*/ +fred_entry_from_kvm (event_type, vector); +#else +idt_entry_from_kvm (vector); +#endif +return ; + } + +WARN_ON_ONCE (event_type !=EVENT_TYPE_NMI );
Not sure if it's OK to use WARN_ON_ONCE() here.
If the warning is triggered, it could unblock NMI due to handling of #UD.
Quoted 15 lines
arch/x86/entry/common.c new file +15
-0+ +#ifdef CONFIG_X86_64 +if (cpu_feature_enabled (X86_FEATURE_FRED )) +return fred_entry_from_kvm (event_type, vector); +#endif + +/* +* Notably, we must use IDT dispatch for NMI when running in IDT mode. +* The FRED NMI context is significantly different and will not work +* right (speficially FRED fixed the NMI recursion issue). +*/ + idt_entry_from_kvm(vector); +} +EXPORT_SYMBOL_FOR_KVM (x86_entry_from_kvm); +#endif
[...]
arch/x86/include/asm/desc.h extern void idt_setup_traps(void); · line 441
Quoted 7 lines
arch/x86/include/asm/desc.h +1
-0@@ -438,6 +438,10 @@ extern void idt_setup_traps(void);extern void idt_setup_apic_and_irq_gates (void );extern bool idt_is_f00f_address (unsigned long address); +extern void idt_do_interrupt_irqoff (unsigned int vector);
In idt_entry_from_kvm() below, gate_offset() returns 'unsigned long', but here it uses 'unsigned int'. It's not safe since there is no guarantee that the address is within 32 bits for x86_64.
Also, the argument is not a vector.
[...]
Quoted 13 lines
arch/x86/include/asm/desc.h +13
-0+noinstr void idt_entry_from_kvm(unsigned int vector) +{ +if (vector ==NMI_VECTOR ) +return idt_do_nmi_irqoff (); + +/* +* Only the NMI path requires noinstr. +*/ +instrumentation_begin (); +idt_do_interrupt_irqoff (gate_offset (idt_table + vector)); +instrumentation_end (); +} +
Re: [PATCH v2 1/2] x86/kvm/vmx: Move IRQ/NMI dispatch from KVM into x86 core
Quoted 14 lines · Binbin Wu
Quoted 12 lines · Peter Zijlstra
arch/x86/entry/common.c new file +12
-0+noinstr void x86_entry_from_kvm(unsigned int event_type, unsigned int vector) +{ +if (event_type ==EVENT_TYPE_EXTINT ) { +#ifdef CONFIG_X86_64 +/* +* Use FRED dispatch, even when running IDT. The dispatch +* tables are kept in sync between FRED and IDT, and the FRED +* dispatch works well with CFI. +*/ +fred_entry_from_kvm (event_type, vector); +#else +idt_entry_from_kvm (vector);
Quoted 8 lines
arch/x86/entry/common.c line 33
Quoted 5 lines
arch/x86/entry/common.c new file +5
-0+#endif +return ; + } + +WARN_ON_ONCE (event_type !=EVENT_TYPE_NMI );Not sure if it's OK to use
WARN_ON_ONCE()here.
If the warning is triggered, it could unblock NMI due to handling of #UD.
If that ever triggers you've got bigger problems.
Quoted 6 lines
Quoted 6 lines
arch/x86/include/asm/desc.h +0
-0@@ -438,6 +438,10 @@ extern void idt_setup_traps(void);extern void idt_setup_apic_and_irq_gates (void );extern bool idt_is_f00f_address (unsigned long address);
Quoted 6 lines
arch/x86/include/asm/desc.h extern void idt_setup_traps(void); · line 441
Quoted 1 line
arch/x86/include/asm/desc.h +1
-0+extern void idt_do_interrupt_irqoff (unsigned int vector);In
idt_entry_from_kvm()below,gate_offset()returns 'unsigned long', but here it uses 'unsigned int'. It's not safe since there is no guarantee that the address is within 32 bits forx86_64.
Right you are, 'unsigned long address' it is.
Re: [PATCH v2 1/2] x86/kvm/vmx: Move IRQ/NMI dispatch from KVM into x86 core
Quoted 24 lines · Peter Zijlstra
Quoted 14 lines · Binbin Wu
Quoted 12 lines · Peter Zijlstra
arch/x86/entry/common.c new file +12
-0+noinstr void x86_entry_from_kvm(unsigned int event_type, unsigned int vector) +{ +if (event_type ==EVENT_TYPE_EXTINT ) { +#ifdef CONFIG_X86_64 +/* +* Use FRED dispatch, even when running IDT. The dispatch +* tables are kept in sync between FRED and IDT, and the FRED +* dispatch works well with CFI. +*/ +fred_entry_from_kvm (event_type, vector); +#else +idt_entry_from_kvm (vector);Quoted 8 lines
arch/x86/entry/common.c line 33
Quoted 5 lines
arch/x86/entry/common.c new file +5
-0+#endif +return ; + } + +WARN_ON_ONCE (event_type !=EVENT_TYPE_NMI );Not sure if it's OK to use
WARN_ON_ONCE()here.
If the warning is triggered, it could unblock NMI due to handling of #UD.
Quoted 1 line
If that ever triggers you've got bigger problems.
Agree. :)
Quoted 15 lines
Quoted 7 lines
Quoted 7 lines
arch/x86/include/asm/desc.h +0
-0@@ -438,6 +438,10 @@ extern void idt_setup_traps(void);extern void idt_setup_apic_and_irq_gates (void );extern bool idt_is_f00f_address (unsigned long address);Quoted 6 lines
arch/x86/include/asm/desc.h extern void idt_setup_traps(void); · line 441
Quoted 1 line
arch/x86/include/asm/desc.h +1
-0+extern void idt_do_interrupt_irqoff (unsigned int vector);In
idt_entry_from_kvm()below,gate_offset()returns 'unsigned long', but here it uses 'unsigned int'. It's not safe since there is no guarantee that the address is within 32 bits forx86_64.Right you are, 'unsigned long address' it is.
PATCH v3 1/2 x86/kvm/vmx: Move IRQ/NMI dispatch from KVM into x86 core
In reply to Peter ZijlstraMove the VMX interrupt dispatch magic into the x86 core code. This isolates KVM from the FRED/IDT decisions and reduces the amount of EXPORT_SYMBOL_FOR_KVM().
---
Changes since v2:
- one more IS_ENABLED(CONFIG_KVM_INTEL) (Yan Zhao)
- fixed idt_do_interrupt_irqoff() prototype (Binbin Wu)
arch/x86/entry/Makefile | 2 - arch/x86/entry/common.c | 48 ++++++++++++++++++++++++++++++++++++ arch/x86/entry/entry.S | 46 ++++++++++++++++++++++++++++++++++ arch/x86/entry/entry_64_fred.S | 1 arch/x86/include/asm/desc.h | 4 +++ arch/x86/include/asm/desc_defs.h | 2 - arch/x86/include/asm/entry-common.h | 2 + arch/x86/include/asm/fred.h | 1 arch/x86/kernel/idt.c | 15 +++++++++++ arch/x86/kernel/nmi.c | 1 arch/x86/kvm/vmx/vmenter.S | 46 ---------------------------------- arch/x86/kvm/vmx/vmx.c | 20 ++------------- 12 files changed, 120 insertions(+), 68 deletions(-)
arch/x86/entry/Makefile +1 -1
@@ -13,7 +13,7 @@ CFLAGS_REMOVE_syscall_64.o = $(CC_FLAGS_ CFLAGS_syscall_32.o += -fno-stack-protector CFLAGS_syscall_64.o += -fno-stack-protector -obj-y := entry.o entry_$(BITS).o syscall_$(BITS).o +obj-y := entry.o entry_$(BITS).o syscall_$(BITS).o common.o obj-y += vdso/ obj-y += vsyscall/
arch/x86/entry/common.c new file +48 -0
48 lines not on this page · raw message
arch/x86/entry/entry.S +46 -0
49 lines not on this page · raw message
arch/x86/entry/entry_64_fred.S +0 -1
@@ -147,5 +147,4 @@ SYM_FUNC_START(asm_fred_entry_from_kvm) RET SYM_FUNC_END(asm_fred_entry_from_kvm) -EXPORT_SYMBOL_FOR_KVM(asm_fred_entry_from_kvm); #endif
arch/x86/include/asm/desc.h +4 -0
@@ -438,6 +438,10 @@ extern void idt_setup_traps(void);extern void idt_setup_apic_and_irq_gates (void );extern bool idt_is_f00f_address (unsigned long address); +extern void idt_do_interrupt_irqoff (unsigned long address); +extern void idt_do_nmi_irqoff (void ); +extern void idt_entry_from_kvm (unsigned int vector); +#ifdef CONFIG_X86_64 extern void idt_setup_early_pf (void );#else
arch/x86/include/asm/desc_defs.h +1 -1
@@ -145,7 +145,7 @@ struct gate_struct {typedef struct gate_struct gate_desc ;#ifndef _SETUP -static inline unsigned long gate_offset (const gate_desc *g) +static __always_inline unsigned long gate_offset (const gate_desc *g) {#ifdef CONFIG_X86_64 return g->offset_low | ((unsigned long)g->offset_middle << 16) |
arch/x86/include/asm/entry-common.h +2 -0
@@ -97,4 +97,6 @@ static __always_inline void arch_exit_to }#define arch_exit_to_user_mode arch_exit_to_user_mode +extern void x86_entry_from_kvm (unsigned int entry_type,unsigned int vector); +#endif
arch/x86/include/asm/fred.h +0 -1
@@ -110,7 +110,6 @@ static __always_inline unsigned long frestatic inline void cpu_init_fred_exceptions (void ) { }static inline void cpu_init_fred_rsps (void ) { }static inline void fred_complete_exception_setup (void ) { } -static inline void fred_entry_from_kvm (unsigned int type,unsigned int vector) { }static inline void fred_sync_rsp0 (unsigned long rsp0) { }static inline void fred_update_rsp0 (void ) { }#endif /* CONFIG_X86_FRED */
arch/x86/kernel/idt.c +15 -0
@@ -268,6 +268,21 @@ void __init idt_setup_early_pf(void) }#endif +#if IS_ENABLED (CONFIG_KVM_INTEL ) +noinstr void idt_entry_from_kvm(unsigned int vector) +{ +if (vector ==NMI_VECTOR ) +return idt_do_nmi_irqoff (); + +/* +* Only the NMI path requires noinstr. +*/ +instrumentation_begin (); +idt_do_interrupt_irqoff (gate_offset (idt_table + vector)); +instrumentation_end (); +} +#endif +static void __init idt_map_in_cea (void ) {/*
arch/x86/kernel/nmi.c +0 -1
@@ -614,7 +614,6 @@ DEFINE_IDTENTRY_RAW(exc_nmi_kvm_vmx) {exc_nmi (regs); } -EXPORT_SYMBOL_FOR_KVM (asm_exc_nmi_kvm_vmx);#endif #ifdef CONFIG_NMI_CHECK_CPU
arch/x86/kvm/vmx/vmenter.S +0 -46
61 lines not on this page · raw message
arch/x86/kvm/vmx/vmx.c +3 -17
@@ -7108,9 +7108,6 @@ void vmx_load_eoi_exitmap(struct kvm_vcpvmcs_write64 (EOI_EXIT_BITMAP3 , eoi_exit_bitmap[3 ]); } -void vmx_do_interrupt_irqoff (unsigned long entry); -void vmx_do_nmi_irqoff (void ); -static void handle_nm_fault_irqoff (struct kvm_vcpu *vcpu) {/*
@@ -7152,17 +7149,9 @@ static void handle_external_interrupt_ir "unexpected VM-Exit interrupt info: 0x%x", intr_info)) return; -/* -* Invoke the kernel's IRQ handler for the vector. Use the FRED path -* when it's available even if FRED isn't fully enabled, e.g. even if -* FRED isn't supported in hardware, in order to avoid the indirect -* CALL in the non-FRED path. -*/ +/* For the IRQ to the core kernel for processing. */ kvm_before_interrupt (vcpu,KVM_HANDLING_IRQ ); -if (IS_ENABLED (CONFIG_X86_FRED )) -fred_entry_from_kvm (EVENT_TYPE_EXTINT , vector); -else -vmx_do_interrupt_irqoff (gate_offset ((gate_desc *)host_idt_base + vector)); +x86_entry_from_kvm (EVENT_TYPE_EXTINT , vector);kvm_after_interrupt (vcpu); vcpu->arch.at_instruction_boundary = true;
@@ -7472,10 +7461,7 @@ noinstr void vmx_handle_nmi(struct kvm_vreturn ;kvm_before_interrupt (vcpu,KVM_HANDLING_NMI ); -if (cpu_feature_enabled (X86_FEATURE_FRED )) -fred_entry_from_kvm (EVENT_TYPE_NMI ,NMI_VECTOR ); -else -vmx_do_nmi_irqoff (); +x86_entry_from_kvm (EVENT_TYPE_NMI ,NMI_VECTOR );kvm_after_interrupt (vcpu); }
Re: [PATCH v3 1/2] x86/kvm/vmx: Move IRQ/NMI dispatch from KVM into x86 core
Quoted 8 lines · Peter Zijlstra
Move the VMX interrupt dispatch magic into the x86 core code. This isolates KVM from the FRED/IDT decisions and reduces the amount of
EXPORT_SYMBOL_FOR_KVM().Suggested-by Sean Christopherson <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>Tested-by "Verma, Vishal L" <[email protected]>
One typo below.
[...]
arch/x86/entry/common.c line 43
Quoted 5 lines
arch/x86/entry/common.c new file +5
-0+ +/* +* Notably, we must use IDT dispatch for NMI when running in IDT mode. +* The FRED NMI context is significantly different and will not work +* right (speficially FRED fixed the NMI recursion issue).
speficially -> specifically
Quoted 3 lines
arch/x86/entry/common.c new file +3
-0+*/ +idt_entry_from_kvm (vector); +}
Re: [PATCH v3 1/2] x86/kvm/vmx: Move IRQ/NMI dispatch from KVM into x86 core
In reply to Peter ZijlstraQuoted 9 lines · Peter Zijlstra
Move the VMX interrupt dispatch magic into the x86 core code. This isolates KVM from the FRED/IDT decisions and reduces the amount of
EXPORT_SYMBOL_FOR_KVM().Suggested-by Sean Christopherson <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>Tested-by Vishal L Verma <[email protected]>---
arch/x86/kvm/vmx/vmx.c static void handle_external_interrupt_ir · line 7152
Quoted 11 lines
arch/x86/kvm/vmx/vmx.c +1
-6@@ -7152,17 +7149,9 @@ static void handle_external_interrupt_ir "unexpected VM-Exit interrupt info: 0x%x", intr_info)) return; -/* -* Invoke the kernel's IRQ handler for the vector. Use the FRED path -* when it's available even if FRED isn't fully enabled, e.g. even if -* FRED isn't supported in hardware, in order to avoid the indirect -* CALL in the non-FRED path. -*/ +/* For the IRQ to the core kernel for processing. */
Forward? Or just delete this comment entirely, either way works for me.
Quoted 9 lines
arch/x86/kvm/vmx/vmx.c +1
-4kvm_before_interrupt (vcpu,KVM_HANDLING_IRQ ); -if (IS_ENABLED (CONFIG_X86_FRED )) -fred_entry_from_kvm (EVENT_TYPE_EXTINT , vector); -else -vmx_do_interrupt_irqoff (gate_offset ((gate_desc *)host_idt_base + vector)); +x86_entry_from_kvm (EVENT_TYPE_EXTINT , vector);kvm_after_interrupt (vcpu); vcpu->arch.at_instruction_boundary = true;
Re: [PATCH v3 1/2] x86/kvm/vmx: Move IRQ/NMI dispatch from KVM into x86 core
In reply to Peter ZijlstraQuoted 10 lines · Peter Zijlstra
Date: Fri, 8 May 2026 11:18:29 +0200
From: Peter Zijlstra <[email protected]>
Subject: [PATCH v3 1/2] x86/kvm/vmx: Move IRQ/NMI dispatch from KVM into
x86 coreMove the VMX interrupt dispatch magic into the x86 core code. This isolates KVM from the FRED/IDT decisions and reduces the amount of
EXPORT_SYMBOL_FOR_KVM().
Quoted 7 lines
Suggested-by Sean Christopherson <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>Tested-by "Verma, Vishal L" <[email protected]>---
Changes since v2:
- one moreIS_ENABLED(CONFIG_KVM_INTEL)(Yan Zhao)
- fixedidt_do_interrupt_irqoff()prototype (Binbin Wu)
My Guest is running well and there's no interrupt error in system, so,
[tip: x86/urgent] x86/kvm/vmx: Move IRQ/NMI dispatch from KVM into x86 core
In reply to Peter ZijlstraThe following commit has been merged into the x86/urgent branch of tip:
The following commit has been merged into the x86/urgent branch of tip:
Commit-ID: 0701c9e17bd903d95b2ddf7dd2e1d8be5027f331 Gitweb: https://git.kernel.org/tip/0701c9e17bd903d95b2ddf7dd2e1d8be5027f331 Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 08 May 2026 11:18:29 +02:00 Committer: Thomas Gleixner <[email protected]> CommitterDate: Tue, 19 May 2026 20:25:51 +02:00
x86/kvm/vmx: Move IRQ/NMI dispatch from KVM into x86 core
Move the VMX interrupt dispatch magic into the x86 core code. This isolates KVM from the FRED/IDT decisions and reduces the amount of EXPORT_SYMBOL_FOR_KVM().
--- arch/x86/entry/Makefile | 2 +- arch/x86/entry/common.c | 48 ++++++++++++++++++++++++++++- arch/x86/entry/entry.S | 46 +++++++++++++++++++++++++++- arch/x86/entry/entry_64_fred.S | 1 +- arch/x86/include/asm/desc.h | 4 ++- arch/x86/include/asm/desc_defs.h | 2 +- arch/x86/include/asm/entry-common.h | 2 +- arch/x86/include/asm/fred.h | 1 +- arch/x86/kernel/idt.c | 15 +++++++++- arch/x86/kernel/nmi.c | 1 +- arch/x86/kvm/vmx/vmenter.S | 46 +--------------------------- arch/x86/kvm/vmx/vmx.c | 19 +---------- 12 files changed, 119 insertions(+), 68 deletions(-) create mode 100644 arch/x86/entry/common.c
arch/x86/entry/Makefile +1 -1
@@ -13,7 +13,7 @@ CFLAGS_REMOVE_syscall_64.o = $(CC_FLAGS_FTRACE) CFLAGS_syscall_32.o += -fno-stack-protector CFLAGS_syscall_64.o += -fno-stack-protector -obj-y := entry.o entry_$(BITS).o syscall_$(BITS).o +obj-y := entry.o entry_$(BITS).o syscall_$(BITS).o common.o obj-y += vdso/ obj-y += vsyscall/
arch/x86/entry/common.c new file +48 -0
48 lines not on this page · raw message
arch/x86/entry/entry.S +46 -0
49 lines not on this page · raw message
arch/x86/entry/entry_64_fred.S +0 -1
@@ -147,5 +147,4 @@ SYM_FUNC_START(asm_fred_entry_from_kvm) RET SYM_FUNC_END(asm_fred_entry_from_kvm) -EXPORT_SYMBOL_FOR_KVM(asm_fred_entry_from_kvm); #endif
arch/x86/include/asm/desc.h +4 -0
@@ -438,6 +438,10 @@ extern void idt_setup_traps(void);extern void idt_setup_apic_and_irq_gates (void );extern bool idt_is_f00f_address (unsigned long address); +extern void idt_do_interrupt_irqoff (unsigned long address); +extern void idt_do_nmi_irqoff (void ); +extern void idt_entry_from_kvm (unsigned int vector); +#ifdef CONFIG_X86_64 extern void idt_setup_early_pf (void );#else
arch/x86/include/asm/desc_defs.h +1 -1
@@ -145,7 +145,7 @@ struct gate_struct {typedef struct gate_struct gate_desc ;#ifndef _SETUP -static inline unsigned long gate_offset (const gate_desc *g) +static __always_inline unsigned long gate_offset (const gate_desc *g) {#ifdef CONFIG_X86_64 return g->offset_low | ((unsigned long)g->offset_middle << 16) |
arch/x86/include/asm/entry-common.h +2 -0
@@ -97,4 +97,6 @@ static __always_inline void arch_exit_to_user_mode(void) }#define arch_exit_to_user_mode arch_exit_to_user_mode +extern void x86_entry_from_kvm (unsigned int entry_type,unsigned int vector); +#endif
arch/x86/include/asm/fred.h +0 -1
@@ -110,7 +110,6 @@ static __always_inline unsigned long fred_event_data(struct pt_regs *regs) { retstatic inline void cpu_init_fred_exceptions (void ) { }static inline void cpu_init_fred_rsps (void ) { }static inline void fred_complete_exception_setup (void ) { } -static inline void fred_entry_from_kvm (unsigned int type,unsigned int vector) { }static inline void fred_sync_rsp0 (unsigned long rsp0) { }static inline void fred_update_rsp0 (void ) { }#endif /* CONFIG_X86_FRED */
arch/x86/kernel/idt.c +15 -0
@@ -268,6 +268,21 @@ void __init idt_setup_early_pf(void) }#endif +#if IS_ENABLED (CONFIG_KVM_INTEL ) +noinstr void idt_entry_from_kvm(unsigned int vector) +{ +if (vector ==NMI_VECTOR ) +return idt_do_nmi_irqoff (); + +/* +* Only the NMI path requires noinstr. +*/ +instrumentation_begin (); +idt_do_interrupt_irqoff (gate_offset (idt_table + vector)); +instrumentation_end (); +} +#endif +static void __init idt_map_in_cea (void ) {/*
arch/x86/kernel/nmi.c +0 -1
@@ -614,7 +614,6 @@ DEFINE_IDTENTRY_RAW(exc_nmi_kvm_vmx) {exc_nmi (regs); } -EXPORT_SYMBOL_FOR_KVM (asm_exc_nmi_kvm_vmx);#endif #ifdef CONFIG_NMI_CHECK_CPU
arch/x86/kvm/vmx/vmenter.S +0 -46
61 lines not on this page · raw message
arch/x86/kvm/vmx/vmx.c +2 -17
@@ -7117,9 +7117,6 @@ void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap)vmcs_write64 (EOI_EXIT_BITMAP3 , eoi_exit_bitmap[3 ]); } -void vmx_do_interrupt_irqoff (unsigned long entry); -void vmx_do_nmi_irqoff (void ); -static void handle_nm_fault_irqoff (struct kvm_vcpu *vcpu) {/*
@@ -7161,17 +7158,8 @@ static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu, "unexpected VM-Exit interrupt info: 0x%x", intr_info)) return; -/* -* Invoke the kernel's IRQ handler for the vector. Use the FRED path -* when it's available even if FRED isn't fully enabled, e.g. even if -* FRED isn't supported in hardware, in order to avoid the indirect -* CALL in the non-FRED path. -*/ kvm_before_interrupt (vcpu,KVM_HANDLING_IRQ ); -if (IS_ENABLED (CONFIG_X86_FRED )) -fred_entry_from_kvm (EVENT_TYPE_EXTINT , vector); -else -vmx_do_interrupt_irqoff (gate_offset ((gate_desc *)host_idt_base + vector)); +x86_entry_from_kvm (EVENT_TYPE_EXTINT , vector);kvm_after_interrupt (vcpu); vcpu->arch.at_instruction_boundary = true;
@@ -7481,10 +7469,7 @@ noinstr void vmx_handle_nmi(struct kvm_vcpu *vcpu)return ;kvm_before_interrupt (vcpu,KVM_HANDLING_NMI ); -if (cpu_feature_enabled (X86_FEATURE_FRED )) -fred_entry_from_kvm (EVENT_TYPE_NMI ,NMI_VECTOR ); -else -vmx_do_nmi_irqoff (); +x86_entry_from_kvm (EVENT_TYPE_NMI ,NMI_VECTOR );kvm_after_interrupt (vcpu); }
Re: [PATCH v3 1/2] x86/kvm/vmx: Move IRQ/NMI dispatch from KVM into x86 core
In reply to Peter ZijlstraQuoted 8 lines · Peter Zijlstra
Move the VMX interrupt dispatch magic into the x86 core code. This isolates KVM from the FRED/IDT decisions and reduces the amount of
EXPORT_SYMBOL_FOR_KVM().Suggested-by Sean Christopherson <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>Tested-by "Verma, Vishal L" <[email protected]>
I am seeing
vmlinux.o: warning: objtool: idt_do_interrupt_irqoff+0xe: no-cfi indirect call!
after this landed in -next.
$ cat arch/x86/configs/repro.config CONFIG_CFI=y CONFIG_KVM=y CONFIG_KVM_INTEL=y
$ make -skj"$(nproc)" ARCH=x86_64 LLVM=1 mrproper defconfig repro.config vmlinux vmlinux.o: warning: objtool: idt_do_interrupt_irqoff+0xe: no-cfi indirect call!
$ llvm-objdump -drS --disassemble-symbols=idt_do_interrupt_irqoff vmlinux.o
vmlinux.o: file format elf64-x86-64
Disassembly of section .text:
0000000000001540 <idt_do_interrupt_irqoff>:
1540: 55 pushq %rbp
1541: 48 89 e5 movq %rsp, %rbp
1544: 48 83 e4 f0 andq $-0x10, %rsp
1548: 6a 18 pushq $0x18
154a: 55 pushq %rbp
154b: 9c pushfq
154c: 6a 10 pushq $0x10
154e: e8 00 00 00 00 callq 0x1553 <idt_do_interrupt_irqoff+0x13>
000000000000154f: R_X86_64_PLT32 __x86_indirect_thunk_rdi-0x4
1553: c9 leave
1554: e9 00 00 00 00 jmp 0x1559 <idt_do_interrupt_irqoff+0x19>
0000000000001555: R_X86_64_PLT32 __x86_return_thunk-0x4
1559: cc int3
155a: cc int3
155b: cc int3
155c: cc int3
155d: cc int3
155e: cc int3
155f: cc int3
1560: 90 nop
1561: 90 nop
1562: 90 nop
1563: 90 nop
1564: 90 nop
1565: 90 nop
1566: 90 nop
1567: 90 nop
1568: 90 nop
1569: 90 nop
156a: 90 nop
156b: 90 nop
156c: 90 nop
156d: 90 nop
156e: 90 nop
156f: 90 nopCheers,
Nathan
Re: [PATCH v3 1/2] x86/kvm/vmx: Move IRQ/NMI dispatch from KVM into x86 core
Sorry, I missed this :/
Quoted 16 lines · Nathan Chancellor
Quoted 8 lines · Peter Zijlstra
Move the VMX interrupt dispatch magic into the x86 core code. This isolates KVM from the FRED/IDT decisions and reduces the amount of
EXPORT_SYMBOL_FOR_KVM().Suggested-by Sean Christopherson <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>Tested-by "Verma, Vishal L" <[email protected]>I am seeing
vmlinux.o: warning: objtool:
idt_do_interrupt_irqoff+0xe: no-cfi indirect call!after this landed in -next.
Quoted 8 lines
$ cat arch/x86/configs/repro.config CONFIG_CFI=y CONFIG_KVM=y CONFIG_KVM_INTEL=y$ make -skj"$(nproc)" ARCH=x86_64 LLVM=1 mrproper defconfig repro.config vmlinux vmlinux.o: warning: objtool: idt_do_interrupt_irqoff+0xe: no-cfi indirect call!
Durr.
---
Subject: x86/kvm/vmx: Fix x86_64 CFI build
I missed that idt_do_interrupt_irqoff() gets compiled on x84_64; this is a problem for CFI builds because it includes an unadorned indirect call. It is however completely dead code.
Rework things to not emit this function at all.
---
arch/x86/entry/common.c +1 -1
@@ -55,7 +55,7 @@ noinstr void x86_entry_from_kvm(unsigned int event_type, unsigned int vector)* The FRED NMI context is significantly different and will not work * right (specifically FRED fixed the NMI recursion issue). */ - idt_entry_from_kvm(vector); + idt_do_nmi_irqoff(); }EXPORT_SYMBOL_FOR_KVM (x86_entry_from_kvm);#endif
arch/x86/entry/entry.S +2 -0
@@ -109,11 +109,13 @@ EXPORT_SYMBOL(__ref_stack_chk_guard); RET .endm +#ifndef CONFIG_X86_64 .pushsection .text, "ax" SYM_FUNC_START(idt_do_interrupt_irqoff) IDT_DO_EVENT_IRQOFF CALL_NOSPEC _ASM_ARG1 SYM_FUNC_END(idt_do_interrupt_irqoff) .popsection +#endif .pushsection .noinstr.text, "ax" SYM_FUNC_START(idt_do_nmi_irqoff)
arch/x86/kernel/idt.c +2 -10
@@ -268,18 +268,10 @@ void __init idt_setup_early_pf(void) }#endif -#if IS_ENABLED (CONFIG_KVM_INTEL ) -noinstr void idt_entry_from_kvm(unsigned int vector) +#if IS_ENABLED (CONFIG_KVM_INTEL ) && !defined(CONFIG_X86_64 ) +void idt_entry_from_kvm (unsigned int vector) { -if (vector ==NMI_VECTOR ) -return idt_do_nmi_irqoff (); - -/* -* Only the NMI path requires noinstr. -*/ -instrumentation_begin ();idt_do_interrupt_irqoff (gate_offset (idt_table + vector)); -instrumentation_end (); }#endif
Nathan Chancellor
Tested-byRe: [PATCH v3 1/2] x86/kvm/vmx: Move IRQ/NMI dispatch from KVM into x86 core
Quoted 2 lines · Peter Zijlstra
Sorry, I missed this :/
No worries, I should have pinged about it elsewhere sooner.
Quoted 6 lines
Subject: x86/kvm/vmx: Fix
x86_64CFI buildI missed that
idt_do_interrupt_irqoff()gets compiled onx84_64; this is a problem for CFI builds because it includes an unadorned indirect call. It is however completely dead code.
Quoted 6 lines
Rework things to not emit this function at all.
Fixes 0701c9e17bd9 ("x86/kvm/vmx: Move IRQ/NMI dispatch from KVM into x86 core")Reported-by Nathan Chancellor <[email protected]>Reported-by Calvin Owens <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>
Quoted 57 lines
---
arch/x86/entry/common.c +1
-1@@ -55,7 +55,7 @@ noinstr void x86_entry_from_kvm(unsigned int event_type, unsigned int vector)* The FRED NMI context is significantly different and will not work * right (specifically FRED fixed the NMI recursion issue). */ - idt_entry_from_kvm(vector); + idt_do_nmi_irqoff(); }EXPORT_SYMBOL_FOR_KVM (x86_entry_from_kvm);#endif arch/x86/entry/entry.S +2
-0@@ -109,11 +109,13 @@ EXPORT_SYMBOL(__ref_stack_chk_guard); RET .endm +#ifndef CONFIG_X86_64 .pushsection .text, "ax" SYM_FUNC_START(idt_do_interrupt_irqoff) IDT_DO_EVENT_IRQOFF CALL_NOSPEC _ASM_ARG1 SYM_FUNC_END(idt_do_interrupt_irqoff) .popsection +#endif .pushsection .noinstr.text, "ax" SYM_FUNC_START(idt_do_nmi_irqoff)arch/x86/kernel/idt.c +2
-10@@ -268,18 +268,10 @@ void __init idt_setup_early_pf(void) }#endif -#if IS_ENABLED (CONFIG_KVM_INTEL ) -noinstr void idt_entry_from_kvm(unsigned int vector) +#if IS_ENABLED (CONFIG_KVM_INTEL ) && !defined(CONFIG_X86_64 ) +void idt_entry_from_kvm (unsigned int vector) { -if (vector ==NMI_VECTOR ) -return idt_do_nmi_irqoff (); - -/* -* Only the NMI path requires noinstr. -*/ -instrumentation_begin ();idt_do_interrupt_irqoff (gate_offset (idt_table + vector)); -instrumentation_end (); }#endif
Signature · 2 lines
Cheers, Nathan
[tip: x86/urgent] x86/kvm/vmx: Fix x86_64 CFI build
In reply to Peter ZijlstraThe following commit has been merged into the x86/urgent branch of tip:
The following commit has been merged into the x86/urgent branch of tip:
Commit-ID: 8aeb879baf12fe64889f019da9a4f8347c604e91 Gitweb: https://git.kernel.org/tip/8aeb879baf12fe64889f019da9a4f8347c604e91 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 26 May 2026 11:06:31 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Thu, 28 May 2026 11:31:50 +02:00
x86/kvm/vmx: Fix x86_64 CFI build
It was missed that idt_do_interrupt_irqoff() gets compiled on x84_64; this is a problem for CFI builds because it includes an unadorned indirect call. It is however completely dead code.
Rework things to not emit this function at all.
--- arch/x86/entry/common.c | 2 +- arch/x86/entry/entry.S | 2 ++ arch/x86/kernel/idt.c | 12 ++---------- 3 files changed, 5 insertions(+), 11 deletions(-)
arch/x86/entry/common.c +1 -1
@@ -55,7 +55,7 @@ noinstr void x86_entry_from_kvm(unsigned int event_type, unsigned int vector)* The FRED NMI context is significantly different and will not work * right (specifically FRED fixed the NMI recursion issue). */ - idt_entry_from_kvm(vector); + idt_do_nmi_irqoff(); }EXPORT_SYMBOL_FOR_KVM (x86_entry_from_kvm);#endif
arch/x86/entry/entry.S +2 -0
@@ -109,11 +109,13 @@ EXPORT_SYMBOL(__ref_stack_chk_guard); RET .endm +#ifndef CONFIG_X86_64 .pushsection .text, "ax" SYM_FUNC_START(idt_do_interrupt_irqoff) IDT_DO_EVENT_IRQOFF CALL_NOSPEC _ASM_ARG1 SYM_FUNC_END(idt_do_interrupt_irqoff) .popsection +#endif .pushsection .noinstr.text, "ax" SYM_FUNC_START(idt_do_nmi_irqoff)
arch/x86/kernel/idt.c +2 -10
@@ -268,18 +268,10 @@ void __init idt_setup_early_pf(void) }#endif -#if IS_ENABLED (CONFIG_KVM_INTEL ) -noinstr void idt_entry_from_kvm(unsigned int vector) +#if IS_ENABLED (CONFIG_KVM_INTEL ) && !defined(CONFIG_X86_64 ) +void idt_entry_from_kvm (unsigned int vector) { -if (vector ==NMI_VECTOR ) -return idt_do_nmi_irqoff (); - -/* -* Only the NMI path requires noinstr. -*/ -instrumentation_begin ();idt_do_interrupt_irqoff (gate_offset (idt_table + vector)); -instrumentation_end (); }#endif
Re: [PATCH v3 1/2] x86/kvm/vmx: Move IRQ/NMI dispatch from KVM into x86 core
In reply to Peter ZijlstraHi Peter,
Quoted 56 lines · Peter Zijlstra
Move the VMX interrupt dispatch magic into the x86 core code. This isolates KVM from the FRED/IDT decisions and reduces the amount of
EXPORT_SYMBOL_FOR_KVM().Suggested-by Sean Christopherson <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>Tested-by "Verma, Vishal L" <[email protected]>---
Changes since v2:
- one moreIS_ENABLED(CONFIG_KVM_INTEL)(Yan Zhao)
- fixedidt_do_interrupt_irqoff()prototype (Binbin Wu)arch/x86/entry/Makefile | 2 - arch/x86/entry/common.c | 48 ++++++++++++++++++++++++++++++++++++ arch/x86/entry/entry.S | 46 ++++++++++++++++++++++++++++++++++ arch/x86/entry/entry_64_fred.S | 1 arch/x86/include/asm/desc.h | 4 +++ arch/x86/include/asm/desc_defs.h | 2 - arch/x86/include/asm/entry-common.h | 2 + arch/x86/include/asm/fred.h | 1 arch/x86/kernel/idt.c | 15 +++++++++++ arch/x86/kernel/nmi.c | 1 arch/x86/kvm/vmx/vmenter.S | 46 ---------------------------------- arch/x86/kvm/vmx/vmx.c | 20 ++------------- 12 files changed, 120 insertions(+), 68 deletions(-)arch/x86/entry/Makefile +1
-1@@ -13,7 +13,7 @@ CFLAGS_REMOVE_syscall_64.o = $(CC_FLAGS_ CFLAGS_syscall_32.o += -fno-stack-protector CFLAGS_syscall_64.o += -fno-stack-protector -obj-y := entry.o entry_$(BITS).o syscall_$(BITS).o +obj-y := entry.o entry_$(BITS).o syscall_$(BITS).o common.o obj-y += vdso/ obj-y += vsyscall/arch/x86/entry/common.c new file +16
-0@@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#include <linux/entry-common.h> +#include <linux/kvm_types.h> +#include <asm/fred.h> +#include <asm/desc.h> + +#if IS_ENABLED (CONFIG_KVM_INTEL ) +/* +* On VMX, NMIs and IRQs (as configured by KVM) are acknowledged by hardware as +* part of the VM-Exit, i.e. the event itself is consumed as part the VM-Exit. +* x86_entry_from_kvm() is invoked by KVM to effectively forward NMIs and IRQs +* to the kernel for servicing. On SVM, a.k.a. AMD, the NMI/IRQ VM-Exit is +* purely a signal that an NMI/IRQ is pending, i.e. the event that triggered +* the VM-Exit is held pending until it's unblocked in the host. +*/
arch/x86/entry/common.c line 26
Quoted 10 lines
arch/x86/entry/common.c new file +10
-0+noinstr void x86_entry_from_kvm(unsigned int event_type, unsigned int vector) +{ +if (event_type ==EVENT_TYPE_EXTINT ) { +#ifdef CONFIG_X86_64 +/* +* Use FRED dispatch, even when running IDT. The dispatch +* tables are kept in sync between FRED and IDT, and the FRED +* dispatch works well with CFI. +*/ +fred_entry_from_kvm (event_type, vector);
Seems this landed in 7.1-rc5.
I'm seeing a build failure here:
arch/x86/entry/common.c: In function ‘x86_entry_from_kvm’:arch/x86/entry/common.c:27:17: error: implicit declaration of function ‘fred_entry_from_kvm’; did you mean ‘idt_entry_from_kvm’? [-Wimplicit-function-declaration]
27 | fred_entry_from_kvm(event_type, vector);
| ^~~~~~~~~~~~~~~~~~~
| idt_entry_from_kvm
arch/x86/entry/common.c:50:24: error: ‘return’ with a value, in function returning void [-Wreturn-mismatch]
50 | return fred_entry_from_kvm(event_type, vector);
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
arch/x86/entry/common.c:18:14: note: declared here
18 | noinstr void x86_entry_from_kvm(unsigned int event_type, unsigned int vector)
| ^~~~~~~~~~~~~~~~~~Quoted 23 lines
arch/x86/entry/common.c new file +22
-0+#else +idt_entry_from_kvm (vector); +#endif + return; + } + +WARN_ON_ONCE (event_type !=EVENT_TYPE_NMI ); + +#ifdef CONFIG_X86_64 +if (cpu_feature_enabled (X86_FEATURE_FRED )) +return fred_entry_from_kvm (event_type, vector); +#endif + +/* +* Notably, we must use IDT dispatch for NMI when running in IDT mode. +* The FRED NMI context is significantly different and will not work +* right (speficially FRED fixed the NMI recursion issue). +*/ +idt_entry_from_kvm (vector); +} +EXPORT_SYMBOL_FOR_KVM (x86_entry_from_kvm); +#endif
[snip]
Quoted 12 lines
arch/x86/include/asm/entry-common.h +2
-0@@ -97,4 +97,6 @@ static __always_inline void arch_exit_to }#define arch_exit_to_user_mode arch_exit_to_user_mode +extern void x86_entry_from_kvm (unsigned int entry_type,unsigned int vector); +#endif
arch/x86/include/asm/fred.h static __always_inline unsigned long fre · line 113
Quoted 4 lines
arch/x86/include/asm/fred.h +0
-1@@ -110,7 +110,6 @@ static __always_inline unsigned long frestatic inline void cpu_init_fred_exceptions (void ) { }static inline void cpu_init_fred_rsps (void ) { }static inline void fred_complete_exception_setup (void ) { } -static inline void fred_entry_from_kvm (unsigned int type,unsigned int vector) { }
That seems still necessary for the !CONFIG_X86_FRED case.
Quoted 3 lines
arch/x86/include/asm/fred.h +0
-0static inline void fred_sync_rsp0 (unsigned long rsp0) { }static inline void fred_update_rsp0 (void ) { }#endif /* CONFIG_X86_FRED */
[snip]
Re: [PATCH v3 1/2] x86/kvm/vmx: Move IRQ/NMI dispatch from KVM into x86 core
Quoted 70 lines · Florian
Quoted 9 lines
arch/x86/entry/common.c new file +9
-0+#if IS_ENABLED (CONFIG_KVM_INTEL ) +/* +* On VMX, NMIs and IRQs (as configured by KVM) are acknowledged by hardware as +* part of the VM-Exit, i.e. the event itself is consumed as part the VM-Exit. +* x86_entry_from_kvm() is invoked by KVM to effectively forward NMIs and IRQs +* to the kernel for servicing. On SVM, a.k.a. AMD, the NMI/IRQ VM-Exit is +* purely a signal that an NMI/IRQ is pending, i.e. the event that triggered +* the VM-Exit is held pending until it's unblocked in the host. +*/ arch/x86/entry/common.c line 26
Quoted 10 lines
arch/x86/entry/common.c new file +10
-0+noinstr void x86_entry_from_kvm(unsigned int event_type, unsigned int vector) +{ +if (event_type ==EVENT_TYPE_EXTINT ) { +#ifdef CONFIG_X86_64 +/* +* Use FRED dispatch, even when running IDT. The dispatch +* tables are kept in sync between FRED and IDT, and the FRED +* dispatch works well with CFI. +*/ +fred_entry_from_kvm (event_type, vector);Seems this landed in 7.1-rc5.
I'm seeing a build failure here:
arch/x86/entry/common.c: In function ‘x86_entry_from_kvm’:arch/x86/entry/common.c:27:17: error: implicit declaration of function ‘fred_entry_from_kvm’; did you mean ‘idt_entry_from_kvm’? [-Wimplicit-function-declaration]27 | fred_entry_from_kvm(event_type, vector); | ^~~~~~~~~~~~~~~~~~~ | idt_entry_from_kvm arch/x86/entry/common.c:50:24: error: ‘return’ with a value, in function returning void [-Wreturn-mismatch] 50 | return fred_entry_from_kvm(event_type, vector); | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ arch/x86/entry/common.c:18:14: note: declared here 18 | noinstr void x86_entry_from_kvm(unsigned int event_type, unsigned int vector) | ^~~~~~~~~~~~~~~~~~Quoted 23 lines
arch/x86/entry/common.c new file +22
-0+#else +idt_entry_from_kvm (vector); +#endif + return; + } + +WARN_ON_ONCE (event_type !=EVENT_TYPE_NMI ); + +#ifdef CONFIG_X86_64 +if (cpu_feature_enabled (X86_FEATURE_FRED )) +return fred_entry_from_kvm (event_type, vector); +#endif + +/* +* Notably, we must use IDT dispatch for NMI when running in IDT mode. +* The FRED NMI context is significantly different and will not work +* right (speficially FRED fixed the NMI recursion issue). +*/ +idt_entry_from_kvm (vector); +} +EXPORT_SYMBOL_FOR_KVM (x86_entry_from_kvm); +#endif [snip]
Quoted 6 lines
arch/x86/include/asm/entry-common.h +0
-0@@ -97,4 +97,6 @@ static __always_inline void arch_exit_to }#define arch_exit_to_user_mode arch_exit_to_user_mode
Quoted 12 lines
arch/x86/include/asm/fred.h static __always_inline unsigned long fre · line 113
Quoted 10 lines
arch/x86/include/asm/entry-common.h +2
-0+extern void x86_entry_from_kvm (unsigned int entry_type,unsigned int vector); +#endif arch/x86/include/asm/fred.h +0
-1@@ -110,7 +110,6 @@ static __always_inline unsigned long frestatic inline void cpu_init_fred_exceptions (void ) { }static inline void cpu_init_fred_rsps (void ) { }static inline void fred_complete_exception_setup (void ) { } -static inline void fred_entry_from_kvm (unsigned int type,unsigned int vector) { }That seems still necessary for the !CONFIG_X86_FRED case.
The thing is, KVM_INTEL should force X86_FRED, I'm not sure how you can have both KVM_INTEL and !X86_FRED, that should be an invalid config.
Re: [PATCH v3 1/2] x86/kvm/vmx: Move IRQ/NMI dispatch from KVM into x86 core
Quoted 85 lines · Peter Zijlstra
Quoted 70 lines · Florian
Quoted 9 lines
arch/x86/entry/common.c new file +9
-0+#if IS_ENABLED (CONFIG_KVM_INTEL ) +/* +* On VMX, NMIs and IRQs (as configured by KVM) are acknowledged by hardware as +* part of the VM-Exit, i.e. the event itself is consumed as part the VM-Exit. +* x86_entry_from_kvm() is invoked by KVM to effectively forward NMIs and IRQs +* to the kernel for servicing. On SVM, a.k.a. AMD, the NMI/IRQ VM-Exit is +* purely a signal that an NMI/IRQ is pending, i.e. the event that triggered +* the VM-Exit is held pending until it's unblocked in the host. +*/ arch/x86/entry/common.c line 26
Quoted 10 lines
arch/x86/entry/common.c new file +10
-0+noinstr void x86_entry_from_kvm(unsigned int event_type, unsigned int vector) +{ +if (event_type ==EVENT_TYPE_EXTINT ) { +#ifdef CONFIG_X86_64 +/* +* Use FRED dispatch, even when running IDT. The dispatch +* tables are kept in sync between FRED and IDT, and the FRED +* dispatch works well with CFI. +*/ +fred_entry_from_kvm (event_type, vector);Seems this landed in 7.1-rc5.
I'm seeing a build failure here:
arch/x86/entry/common.c: In function ‘x86_entry_from_kvm’:arch/x86/entry/common.c:27:17: error: implicit declaration of function ‘fred_entry_from_kvm’; did you mean ‘idt_entry_from_kvm’? [-Wimplicit-function-declaration]27 | fred_entry_from_kvm(event_type, vector); | ^~~~~~~~~~~~~~~~~~~ | idt_entry_from_kvm arch/x86/entry/common.c:50:24: error: ‘return’ with a value, in function returning void [-Wreturn-mismatch] 50 | return fred_entry_from_kvm(event_type, vector); | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ arch/x86/entry/common.c:18:14: note: declared here 18 | noinstr void x86_entry_from_kvm(unsigned int event_type, unsigned int vector) | ^~~~~~~~~~~~~~~~~~Quoted 23 lines
arch/x86/entry/common.c new file +22
-0+#else +idt_entry_from_kvm (vector); +#endif + return; + } + +WARN_ON_ONCE (event_type !=EVENT_TYPE_NMI ); + +#ifdef CONFIG_X86_64 +if (cpu_feature_enabled (X86_FEATURE_FRED )) +return fred_entry_from_kvm (event_type, vector); +#endif + +/* +* Notably, we must use IDT dispatch for NMI when running in IDT mode. +* The FRED NMI context is significantly different and will not work +* right (speficially FRED fixed the NMI recursion issue). +*/ +idt_entry_from_kvm (vector); +} +EXPORT_SYMBOL_FOR_KVM (x86_entry_from_kvm); +#endif [snip]
Quoted 6 lines
arch/x86/include/asm/entry-common.h +0
-0@@ -97,4 +97,6 @@ static __always_inline void arch_exit_to }#define arch_exit_to_user_mode arch_exit_to_user_modeQuoted 12 lines
arch/x86/include/asm/fred.h static __always_inline unsigned long fre · line 113
Quoted 10 lines
arch/x86/include/asm/entry-common.h +2
-0+extern void x86_entry_from_kvm (unsigned int entry_type,unsigned int vector); +#endif arch/x86/include/asm/fred.h +0
-1@@ -110,7 +110,6 @@ static __always_inline unsigned long frestatic inline void cpu_init_fred_exceptions (void ) { }static inline void cpu_init_fred_rsps (void ) { }static inline void fred_complete_exception_setup (void ) { } -static inline void fred_entry_from_kvm (unsigned int type,unsigned int vector) { }That seems still necessary for the !CONFIG_X86_FRED case.
Quoted 2 lines
The thing is,
KVM_INTELshould forceX86_FRED, I'm not sure how you can have bothKVM_INTELand !X86_FRED, that should be an invalid config.
Right. Thanks for the super fast response. Let me double check what is going on here...
PATCH 2/2 x86/kvm/vmx: Fix VMX vs hrtimer_rearm_deferred()
In reply to Peter ZijlstraVishal reported that KVM unit test 'x2apic' started failing after commit 0e98eb14814e ("entry: Prepare for deferred hrtimer rearming").
The reason is that KVM/VMX is injecting interrupts while it has IRQs disabled, for a context that will enable IRQs, this means that regs->flags.X86_EFLAGS_IF == 0 and the irqentry_exit() will not DTRT.
Notably, irqentry_exit() must not call hrtimer_rearm_deferred() when the return context does not have IF set, because this will cause problems vs NMIs.
Therefore, fix up the state after the injection.
--- arch/x86/entry/common.c | 13 +++++++++++++ 1 file changed, 13 insertions(+)
arch/x86/entry/common.c +13 -0
@@ -2,6 +2,7 @@#include <linux/entry-common.h> #include <linux/kvm_types.h> +#include <linux/hrtimer_rearm.h> #include <asm/fred.h> #include <asm/desc.h>
@@ -27,6 +28,18 @@ noinstr void x86_entry_from_kvm(unsigned#else idt_entry_from_kvm (vector);#endif +/* +* Strictly speaking, only the NMI path requires noinstr. +*/ +instrumentation_begin (); +/* +* KVM/VMX will dispatch from IRQ-disabled but for a context +* that will have IRQs-enabled. This confuses the entry code +* and it will not have reprogrammed the timer. Do so now. +*/ +hrtimer_rearm_deferred (); +instrumentation_end (); +return ; }
Re: [PATCH 2/2] x86/kvm/vmx: Fix VMX vs hrtimer_rearm_deferred()
Quoted 10 lines · Peter Zijlstra
Vishal reported that KVM unit test 'x2apic' started failing after commit
0e98eb14814e("entry: Prepare for deferred hrtimer rearming").The reason is that KVM/VMX is injecting interrupts while it has IRQs disabled, for a context that will enable IRQs, this means that
regs->flags.X86_EFLAGS_IF== 0 and theirqentry_exit()will not DTRT.Notably,
irqentry_exit()must not callhrtimer_rearm_deferred()when the return context does not have IF set, because this will cause problems vs NMIs.
Quoted 8 lines
Therefore, fix up the state after the injection.
Fixes 0e98eb14814e ("entry: Prepare for deferred hrtimer rearming")Reported-by "Verma, Vishal L" <[email protected]>Closes https://lore.kernel.org/r/70cd3e97fbb796e2eb2ff8cd4b7614ada05a5f24.camel%40intel.com open hereSuggested-by Thomas Gleixner <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>Tested-by "Verma, Vishal L" <[email protected]>
I don't see this being merged yet?
Without this, Xen timer delivery (tested by the xen_shinfo_test KVM selftest) is failing. I think the Fixes: tag is wrong though; it actually broke with commit 15dd3a948855 ("hrtimer: Push reprogramming timers into the interrupt return path"), didn't it?
Re: [PATCH 2/2] x86/kvm/vmx: Fix VMX vs hrtimer_rearm_deferred()
Quoted 13 lines · David Woodhouse
Quoted 12 lines · Peter Zijlstra
Vishal reported that KVM unit test 'x2apic' started failing after commit
0e98eb14814e("entry: Prepare for deferred hrtimer rearming").The reason is that KVM/VMX is injecting interrupts while it has IRQs disabled, for a context that will enable IRQs, this means that
regs->flags.X86_EFLAGS_IF== 0 and theirqentry_exit()will not DTRT.Notably,
irqentry_exit()must not callhrtimer_rearm_deferred()when the return context does not have IF set, because this will cause problems vs NMIs.Therefore, fix up the state after the injection.
Quoted 8 lines
Quoted 6 lines
Fixes 0e98eb14814e ("entry: Prepare for deferred hrtimer rearming")Reported-by Vishal L Verma <[email protected]>Closes https://lore.kernel.org/r/70cd3e97fbb796e2eb2ff8cd4b7614ada05a5f24.camel%40intel.com open hereSuggested-by Thomas Gleixner <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>Tested-by Vishal L Verma <[email protected]>Tested-by David Woodhouse <[email protected]>
Quoted 4 lines
I don't see this being merged yet?
Without this, Xen timer delivery (tested by the
xen_shinfo_testKVM selftest) is failing.
And many, many other issues with KVM. I.e. we need this or something like it in 7.1.
Re: [PATCH 2/2] x86/kvm/vmx: Fix VMX vs hrtimer_rearm_deferred()
In reply to David WoodhouseQuoted 21 lines · David Woodhouse
Quoted 10 lines · Peter Zijlstra
Vishal reported that KVM unit test 'x2apic' started failing after commit
0e98eb14814e("entry: Prepare for deferred hrtimer rearming").The reason is that KVM/VMX is injecting interrupts while it has IRQs disabled, for a context that will enable IRQs, this means that
regs->flags.X86_EFLAGS_IF== 0 and theirqentry_exit()will not DTRT.Notably,
irqentry_exit()must not callhrtimer_rearm_deferred()when the return context does not have IF set, because this will cause problems vs NMIs.Quoted 8 lines
Therefore, fix up the state after the injection.
Fixes 0e98eb14814e ("entry: Prepare for deferred hrtimer rearming")Reported-by "Verma, Vishal L" <[email protected]>Closes https://lore.kernel.org/r/70cd3e97fbb796e2eb2ff8cd4b7614ada05a5f24.camel%40intel.com open hereSuggested-by Thomas Gleixner <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>Tested-by "Verma, Vishal L" <[email protected]>
Quoted 8 lines
Tested-by David Woodhouse <[email protected]>I don't see this being merged yet?
Without this, Xen timer delivery (tested by the
xen_shinfo_testKVM selftest) is failing. I think the Fixes: tag is wrong though; it actually broke with commit15dd3a948855("hrtimer: Push reprogramming timers into the interrupt return path"), didn't it?
Just to add another voice, without these patches I have a host (current 7.1-rc mainline) running VM workloads that regularly gets "watchdog: CPUx: Watchdog detected hard LOCKUP on cpu x" messages and related stack traces.
A few weeks ago I had bisected the issue down to the same commit that David suggests, 15dd3a948855 ("hrtimer: Push reprogramming timers into the interrupt return path"), and verified that applying those patches made the oopses go away.
Thanks,
Marc
Re: [PATCH 2/2] x86/kvm/vmx: Fix VMX vs hrtimer_rearm_deferred()
In reply to Peter ZijlstraQuoted 22 lines · Peter Zijlstra
Date: Thu, 23 Apr 2026 17:56:13 +0200
From: Peter Zijlstra <[email protected]>
Subject: [PATCH 2/2] x86/kvm/vmx: Fix VMX vshrtimer_rearm_deferred()Vishal reported that KVM unit test 'x2apic' started failing after commit
0e98eb14814e("entry: Prepare for deferred hrtimer rearming").The reason is that KVM/VMX is injecting interrupts while it has IRQs disabled, for a context that will enable IRQs, this means that
regs->flags.X86_EFLAGS_IF== 0 and theirqentry_exit()will not DTRT.Notably,
irqentry_exit()must not callhrtimer_rearm_deferred()when the return context does not have IF set, because this will cause problems vs NMIs.Therefore, fix up the state after the injection.
Fixes 0e98eb14814e ("entry: Prepare for deferred hrtimer rearming")Reported-by "Verma, Vishal L" <[email protected]>Closes https://lore.kernel.org/r/70cd3e97fbb796e2eb2ff8cd4b7614ada05a5f24.camel%40intel.com open hereSuggested-by Thomas Gleixner <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>Tested-by "Verma, Vishal L" <[email protected]>
Quoted 3 lines
--- arch/x86/entry/common.c | 13 +++++++++++++ 1 file changed, 13 insertions(+)
I also found this issue for a normal VM on DMR (with 192 vCPUs). As the previous threads analysised, this issue is not platform-specific, nor is it specific to TD.
I applied this series and confirmed this could resolve the hard lockup in my case, so,
Re: [PATCH 2/2] x86/kvm/vmx: Fix VMX vs hrtimer_rearm_deferred()
In reply to Peter ZijlstraQuoted 10 lines · Peter Zijlstra
Vishal reported that KVM unit test 'x2apic' started failing after commit
0e98eb14814e("entry: Prepare for deferred hrtimer rearming").The reason is that KVM/VMX is injecting interrupts while it has IRQs disabled, for a context that will enable IRQs, this means that
regs->flags.X86_EFLAGS_IF== 0 and theirqentry_exit()will not DTRT.Notably,
irqentry_exit()must not callhrtimer_rearm_deferred()when the return context does not have IF set, because this will cause problems vs NMIs.
Quoted 8 lines
Therefore, fix up the state after the injection.
Fixes 0e98eb14814e ("entry: Prepare for deferred hrtimer rearming")Reported-by "Verma, Vishal L" <[email protected]>Closes https://lore.kernel.org/r/70cd3e97fbb796e2eb2ff8cd4b7614ada05a5f24.camel%40intel.com open hereSuggested-by Thomas Gleixner <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>Tested-by "Verma, Vishal L" <[email protected]>
Quoted 36 lines
--- arch/x86/entry/common.c | 13 +++++++++++++ 1 file changed, 13 insertions(+)arch/x86/entry/common.c +13
-0@@ -2,6 +2,7 @@#include <linux/entry-common.h> #include <linux/kvm_types.h> +#include <linux/hrtimer_rearm.h> #include <asm/fred.h> #include <asm/desc.h> @@ -27,6 +28,18 @@ noinstr void x86_entry_from_kvm(unsigned#else idt_entry_from_kvm (vector);#endif +/* +* Strictly speaking, only the NMI path requires noinstr. +*/ +instrumentation_begin (); +/* +* KVM/VMX will dispatch from IRQ-disabled but for a context +* that will have IRQs-enabled. This confuses the entry code +* and it will not have reprogrammed the timer. Do so now. +*/ +hrtimer_rearm_deferred (); +instrumentation_end (); +return ; }
[tip: x86/urgent] x86/kvm/vmx: Fix VMX vs hrtimer_rearm_deferred()
In reply to Peter ZijlstraThe following commit has been merged into the x86/urgent branch of tip:
The following commit has been merged into the x86/urgent branch of tip:
Commit-ID: 5fcc48d521877c5d83828d715c81f4d169ef97f3 Gitweb: https://git.kernel.org/tip/5fcc48d521877c5d83828d715c81f4d169ef97f3 Author: Peter Zijlstra <[email protected]> AuthorDate: Thu, 23 Apr 2026 17:56:13 +02:00 Committer: Thomas Gleixner <[email protected]> CommitterDate: Tue, 19 May 2026 20:25:51 +02:00
x86/kvm/vmx: Fix VMX vs hrtimer_rearm_deferred()
Vishal reported that KVM unit test 'x2apic' started failing after commit 0e98eb14814e ("entry: Prepare for deferred hrtimer rearming").
The reason is that KVM/VMX is injecting interrupts while it has interrupts disabled, for a context that will enable interrupts, this means that regs->flags.X86_EFLAGS_IF == 0 and irqentry_exit() will not do the right thing.
Notably, irqentry_exit() must not call hrtimer_rearm_deferred() when the return context does not have IF set, because this will cause problems vs NMIs.
Therefore, fix up the state after the injection.
--- arch/x86/entry/common.c | 13 +++++++++++++ 1 file changed, 13 insertions(+)
arch/x86/entry/common.c +13 -0
@@ -2,6 +2,7 @@#include <linux/entry-common.h> #include <linux/kvm_types.h> +#include <linux/hrtimer_rearm.h> #include <asm/fred.h> #include <asm/desc.h>
@@ -27,6 +28,18 @@ noinstr void x86_entry_from_kvm(unsigned int event_type, unsigned int vector)#else idt_entry_from_kvm (vector);#endif +/* +* Strictly speaking, only the NMI path requires noinstr. +*/ +instrumentation_begin (); +/* +* KVM/VMX will dispatch from IRQ-disabled but for a context +* that will have IRQs-enabled. This confuses the entry code +* and it will not have reprogrammed the timer. Do so now. +*/ +hrtimer_rearm_deferred (); +instrumentation_end (); +return ; }