x86: kvm: Initialize static calls before SMP boot
Subsystems (4)
PATCH 1/1 x86: kvm: Initialize static calls before SMP boot
Updating static calls is expensive on wide SMP systems because all online CPUs need to act in a coordinated manner for code patching to work as expected.
Static are initialized only after SMP boot, where the code patching overhead is noticeable. Pre-initialize the majority of these static calls before SMP boot to get rid of that overhead.
The pre-initialization most likely already sets the correct value. To still handle any potential differences, make the static call initialization re-entrant so that the differences are corrected during post-SMP-boot initialization. Values that are already set correctly are skipped.
This patch was tested on a 6th Gen Intel Xeon Platinum 8375C CPU with 128 SMT cores. When comparing before and after we see that 85% less time is spent in kvm_ops_update (29.5ms -> 4.2ms).
--- arch/x86/include/asm/kvm_host.h | 1 + arch/x86/kvm/svm/svm.c | 11 +++++++++++ arch/x86/kvm/vmx/vmx.c | 18 ++++++++++++++++++ arch/x86/kvm/x86.c | 19 ++++++++++++++++++- kernel/events/core.c | 6 +++++- 5 files changed, 53 insertions(+), 2 deletions(-)
arch/x86/include/asm/kvm_host.h +1 -0
@@ -2000,6 +2000,7 @@ extern struct kvm_x86_ops kvm_x86_ops;#define KVM_X86_OP_OPTIONAL_RET0 KVM_X86_OP#include <asm/kvm-x86-ops.h> +int kvm_x86_vendor_init_early (struct kvm_x86_init_ops *ops);int kvm_x86_vendor_init (struct kvm_x86_init_ops *ops);void kvm_x86_vendor_exit (void );
arch/x86/kvm/svm/svm.c +11 -0
@@ -5452,6 +5452,17 @@ static void __svm_exit(void)kvm_x86_vendor_exit (); } +#ifndef MODULE +static int __init svm_ops_early_init (void ) +{ +if (!kvm_is_svm_supported ()) +return -EOPNOTSUPP ; + +return kvm_x86_vendor_init_early (&svm_init_ops); +} +early_initcall (svm_ops_early_init); +#endif +static int __init svm_init (void ) {int r;
arch/x86/kvm/vmx/vmx.c +18 -0
@@ -8617,6 +8617,8 @@ __init int vmx_hardware_setup(void) vt_x86_ops.set_hv_timer =NULL ; vt_x86_ops.cancel_hv_timer =NULL ; } +WARN_ON (enable_preemption_timer && vt_x86_ops.set_hv_timer ==NULL ); +WARN_ON (enable_preemption_timer && vt_x86_ops.cancel_hv_timer ==NULL ); kvm_caps.supported_mce_cap |=MCG_LMCE_P ; kvm_caps.supported_mce_cap |=MCG_CMCI_P ;
@@ -8698,6 +8700,22 @@ void vmx_exit(void)kvm_x86_vendor_exit (); } +#ifndef MODULE +static int __init vmx_ops_early_init (void ) +{ +if (!kvm_is_vmx_supported ()) +return -EOPNOTSUPP ; + +if (!enable_preemption_timer) { + vt_x86_ops.set_hv_timer =NULL ; + vt_x86_ops.cancel_hv_timer =NULL ; + } + +return kvm_x86_vendor_init_early (&vt_init_ops); +} +early_initcall (vmx_ops_early_init); +#endif +int __init vmx_init (void ) {int r, cpu;
arch/x86/kvm/x86.c +18 -1
@@ -9971,6 +9971,22 @@ static void kvm_x86_check_cpu_compat(void *ret) *(int *)ret =kvm_x86_check_processor_compatibility (); } +int kvm_x86_vendor_init_early (struct kvm_x86_init_ops *ops) +{ +guard (mutex)(&vendor_module_lock); + +if (kvm_x86_ops.enable_virtualization_cpu) { +pr_err ("already loaded vendor module '%s'\n" , kvm_x86_ops.name); +return -EEXIST ; + } + +kvm_ops_update (ops); +kvm_register_perf_callbacks (ops->handle_intel_pt_intr); + +return 0 ; +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL (kvm_x86_vendor_init_early); +int kvm_x86_vendor_init (struct kvm_x86_init_ops *ops) {u64 host_pat;
@@ -9978,7 +9994,8 @@ int kvm_x86_vendor_init(struct kvm_x86_init_ops *ops)guard (mutex)(&vendor_module_lock); -if (kvm_x86_ops.enable_virtualization_cpu) { +if (kvm_x86_ops.enable_virtualization_cpu && + kvm_x86_ops.enable_virtualization_cpu != ops->runtime_ops->enable_virtualization_cpu) {pr_err ("already loaded vendor module '%s'\n" , kvm_x86_ops.name);return -EEXIST ; }
kernel/events/core.c +5 -1
@@ -7378,7 +7378,8 @@ DEFINE_STATIC_CALL_RET0(__perf_guest_handle_intel_pt_intr, *perf_guest_cbs->handvoid perf_register_guest_info_callbacks (struct perf_guest_info_callbacks *cbs) { -if (WARN_ON_ONCE (rcu_access_pointer (perf_guest_cbs))) +if (WARN_ON_ONCE (rcu_access_pointer (perf_guest_cbs) && +rcu_access_pointer (perf_guest_cbs) != cbs))return ;rcu_assign_pointer (perf_guest_cbs, cbs);
@@ -7389,6 +7390,9 @@ void perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)if (cbs->handle_intel_pt_intr)static_call_update (__perf_guest_handle_intel_pt_intr, cbs->handle_intel_pt_intr); +else +static_call_update (__perf_guest_handle_intel_pt_intr, + (void *)__static_call_return0); }EXPORT_SYMBOL_GPL (perf_register_guest_info_callbacks);
Signature · 12 lines
2.47.3 Amazon Web Services Development Center Germany GmbH Tamara-Danz-Str. 13 10243 Berlin Geschaeftsfuehrung: Christof Hellmis, Andreas Stieger Eingetragen am Amtsgericht Charlottenburg unter HRB 257764 B Sitz: Berlin Ust-ID: DE 365 538 597
Sean Christopherson
maintainerQuoted 3 lines · David Sauerwein
Updating static calls is expensive on wide SMP systems because all online CPUs need to act in a coordinated manner for code patching to work as expected.
Eww. I am very, very against an early_initcall() in KVM, even if we pinky swear we'll never use it for anything except prefetching static_call() targets. The initcall framework lacks the ability to express dependencies, and KVM most definitely has dependencies on arch and subsys code.
I also don't like hacking KVM to workaround what is effectively a generic infrastructure issue/limitation.
Have y'all looked at Valentin's series to defer IPIs? I assume/hope deferring IPIs would reduce the delay observed when doing the patching.
Paolo Bonzini
maintainerIl mer 4 mar 2026, 20:19 Sean Christopherson <[email protected]> ha scritto:
Quoted 11 lines · David Sauerwein
Quoted 3 lines
Updating static calls is expensive on wide SMP systems because all online CPUs need to act in a coordinated manner for code patching to work as expected.
Eww. I am very, very against an
early_initcall()in KVM, even if we pinky swear we'll never use it for anything except prefetchingstatic_call()targets. The initcall framework lacks the ability to express dependencies, and KVM most definitely has dependencies on arch and subsys code.
Quoted 5 lines
I also don't like hacking KVM to workaround what is effectively a generic infrastructure issue/limitation.
Have y'all looked at Valentin's series to defer IPIs? I assume/hope deferring IPIs would reduce the delay observed when doing the patching.
x86 has a smp_text_poke_batch_finish() function, maybe it could be exposed as arch_static_call_batch_finish() too?
Then you add a __static_call_update_batch_add() function, and __static_call_update() becomes
__static_call_update_batch_add(key, tramp, func);
arch_static_call_batch_finish();to limit the amount of synchronization.
The next step would be to expose the batch functionality and allow patching multiple static calls at once. I'm not sure where things stop being useful, though.
Paolo