x86/xen: call xen_set_upcall_vector() only after local apic setup
Subsystems (6)
Juergen Gross
maintainerPATCH 1/1 x86/xen: call xen_set_upcall_vector() only after local apic setup
Calling xen_set_upcall_vector() before the local APIC has been initialized results in warnings when setting up the local APIC:
[ 0.037322] APIC: Stale IRR: 00080000, ...
Avoid this problem by moving the call of xen_set_upcall_vector() for APs from xen_cpu_up_prepare_hvm() to a new hotplug callback called on the AP before interrupts are enabled and after the local APIC has been initialized. There is already a Xen specific hotplug state defined at a matching position, but right now it is specific to Arm. So rename the CPUHP_AP_ARM_XEN_STARTING to CPUHP_AP_XEN_STARTING and use it for x86, too.
On the boot processor use the x86_init.irqs.intr_mode_init hook to call xen_set_upcall_vector() only after the local APIC initialization. As interrupts are on already at the time when the hook is being used, modify xen_init_setup_upcall_vector() to disable interrupts. This is needed as the xen_percpu_upcall flag is set only after the callback has been registered, and the flag is needed for correctly acknowledging an interrupt.
--- arch/arm/xen/enlighten.c | 4 ++-- arch/x86/xen/enlighten_hvm.c | 9 --------- drivers/xen/events/events_base.c | 22 +++++++++++++++++----- include/linux/cpuhotplug.h | 2 +- 4 files changed, 20 insertions(+), 17 deletions(-)
arch/arm/xen/enlighten.c +2 -2
@@ -529,8 +529,8 @@ static int __init xen_guest_init(void)if (xen_initial_domain ())pvclock_gtod_register_notifier (&xen_pvclock_gtod_notifier); -return cpuhp_setup_state (CPUHP_AP_ARM_XEN_STARTING , -"arm/xen:starting" , xen_starting_cpu, +return cpuhp_setup_state (CPUHP_AP_XEN_STARTING , +"xen:starting" , xen_starting_cpu, xen_dying_cpu); }early_initcall (xen_guest_init);
arch/x86/xen/enlighten_hvm.c +0 -9
@@ -177,15 +177,6 @@ static int xen_cpu_up_prepare_hvm(unsigned int cpu)if (!xen_have_vector_callback)return 0 ; -if (xen_percpu_upcall) { - rc =xen_set_upcall_vector (cpu); -if (rc) { -WARN (1 ,"HVMOP_set_evtchn_upcall_vector" -" for CPU %d failed: %d\n" , cpu, rc); -return rc; - } - } -if (xen_feature (XENFEAT_hvm_safe_pvclock))xen_setup_timer (cpu);
drivers/xen/events/events_base.c +17 -5
@@ -39,6 +39,7 @@#include <linux/ktime.h> #ifdef CONFIG_X86 +#include <asm/apic.h> #include <asm/desc.h> #include <asm/ptrace.h> #include <asm/idtentry.h>
@@ -2209,16 +2210,27 @@ void xen_setup_callback_vector(void)*/ static __init void xen_init_setup_upcall_vector (void ) { +unsigned long flags; + +apic_intr_mode_init (); +if (!xen_have_vector_callback)return ; +local_irq_save (flags); +if ((cpuid_eax (xen_cpuid_base () +4 ) &XEN_HVM_CPUID_UPCALL_VECTOR ) && - !xen_set_upcall_vector (0 )) + !xen_set_upcall_vector (0 )) { xen_percpu_upcall = true; +WARN_ON (cpuhp_setup_state (CPUHP_AP_XEN_STARTING ,"xen:starting" , + xen_set_upcall_vector,NULL )); + }else if (xen_feature (XENFEAT_hvm_callback_vector))xen_setup_callback_vector ();else xen_have_vector_callback = false; + +local_irq_restore (flags); }int xen_set_upcall_vector (unsigned int cpu)
@@ -2250,9 +2262,7 @@ static __init void xen_alloc_callback_vector(void) }#else void xen_setup_callback_vector (void ) {} -static inline void xen_init_setup_upcall_vector (void ) {}int xen_set_upcall_vector (unsigned int cpu) {} -static inline void xen_alloc_callback_vector (void ) {}#endif /* CONFIG_XEN_PVHVM */ #endif /* CONFIG_X86 */
@@ -2314,9 +2324,11 @@ void __init xen_init_IRQ(void)if (xen_initial_domain ())pci_xen_initial_domain (); } -xen_init_setup_upcall_vector (); -xen_alloc_callback_vector (); +#ifdef CONFIG_XEN_PVHVM + x86_init.irqs.intr_mode_init = xen_init_setup_upcall_vector; +xen_alloc_callback_vector (); +#endif if (xen_hvm_domain ()) {native_init_IRQ ();
include/linux/cpuhotplug.h +1 -1
@@ -191,7 +191,7 @@ enum cpuhp_state {CPUHP_AP_KVM_ARM_TIMER_STARTING ,/* Must be the last timer callback */ CPUHP_AP_DUMMY_TIMER_STARTING , -CPUHP_AP_ARM_XEN_STARTING , +CPUHP_AP_XEN_STARTING ,CPUHP_AP_ARM_CORESIGHT_STARTING ,CPUHP_AP_ARM_CORESIGHT_CTI_STARTING , CPUHP_AP_ARM64_ISNDEP_STARTING,
Signature · 1 line
2.35.3
Quoted 21 lines · Juergen Gross
Calling
xen_set_upcall_vector()before the local APIC has been initialized results in warnings when setting up the local APIC:[ 0.037322] APIC: Stale IRR: 00080000, ...Avoid this problem by moving the call of
xen_set_upcall_vector()for APs fromxen_cpu_up_prepare_hvm()to a new hotplug callback called on the AP before interrupts are enabled and after the local APIC has been initialized. There is already a Xen specific hotplug state defined at a matching position, but right now it is specific to Arm. So rename theCPUHP_AP_ARM_XEN_STARTINGtoCPUHP_AP_XEN_STARTINGand use it for x86, too.On the boot processor use the x86_init.irqs.intr_mode_init hook to call
xen_set_upcall_vector()only after the local APIC initialization. As interrupts are on already at the time when the hook is being used, modifyxen_init_setup_upcall_vector()to disable interrupts. This is needed as thexen_percpu_upcallflag is set only after the callback has been registered, and the flag is needed for correctly acknowledging an interrupt.
Quoted 3 lines
Fixes b1c3497e604d ("x86/xen: Add support for HVMOP_set_evtchn_upcall_vector")Reported-by Jan Beulich <[email protected]>Signed-off-by Juergen Gross <[email protected]>
Just to mention it here - as per [1] the issue may instead result from hypervisor misbehavior, so I guess we want to investigate that route first.
Jan
[1] https://lists.xen.org/archives/html/xen-devel/2022-11/msg00783.html