Use static_call for kvm_x86_ops
Subsystems (4)
cover Use static_call for kvm_x86_ops
Hi,
Convert kvm_x86_ops to use static_call. Shows good performance gains for cpuid loop micro-benchmark (resulsts included in patch 2).
Thanks,
-Jason
Jason Baron (2):
KVM: x86: introduce definitions to support static calls for kvm_x86_ops
KVM: x86: use static calls to reduce kvm_x86_ops overhead
arch/x86/include/asm/kvm_host.h | 71 +++++++++- arch/x86/kvm/cpuid.c | 2 +- arch/x86/kvm/hyperv.c | 4 +- arch/x86/kvm/irq.c | 2 +- arch/x86/kvm/kvm_cache_regs.h | 10 +- arch/x86/kvm/lapic.c | 28 ++-- arch/x86/kvm/mmu.h | 6 +- arch/x86/kvm/mmu/mmu.c | 12 +- arch/x86/kvm/mmu/spte.c | 2 +- arch/x86/kvm/pmu.c | 2 +- arch/x86/kvm/trace.h | 4 +- arch/x86/kvm/x86.c | 299 ++++++++++++++++++++-------------------- arch/x86/kvm/x86.h | 6 +- 13 files changed, 259 insertions(+), 189 deletions(-)
Signature · 1 line
2.7.4
PATCH 1/2 KVM: x86: introduce definitions to support static calls for kvm_x86_ops
Use static calls to improve kvm_x86_ops performance. Introduce the definitions that will be used by a subsequent patch to actualize the savings.
Note that all kvm_x86_ops are covered here except for 'pmu_ops' and 'nested ops'. I think they can be covered by static calls in a simlilar manner, but were omitted from this series to reduce scope and because I don't think they have as large of a performance impact.
--- arch/x86/include/asm/kvm_host.h | 65 +++++++++++++++++++++++++++++++++++++++++ arch/x86/kvm/x86.c | 5 ++++ 2 files changed, 70 insertions(+)
arch/x86/include/asm/kvm_host.h +65 -0
@@ -1087,6 +1087,65 @@ static inline u16 kvm_lapic_irq_dest_mode(bool dest_mode_logical)return dest_mode_logical ?APIC_DEST_LOGICAL :APIC_DEST_PHYSICAL ; } +/* +* static calls cover all kvm_x86_ops except for functions under pmu_ops and +* nested_ops. +*/ +#define FOREACH_KVM_X86_OPS (F ) \ + F(hardware_enable); F(hardware_disable); F(hardware_unsetup); \ + F(cpu_has_accelerated_tpr); F(has_emulated_msr); \ + F(vcpu_after_set_cpuid); F(vm_init); F(vm_destroy); F(vcpu_create); \ + F(vcpu_free); F(vcpu_reset); F(prepare_guest_switch); F(vcpu_load); \ + F(vcpu_put); F(update_exception_bitmap); F(get_msr); F(set_msr); \ + F(get_segment_base); F(get_segment); F(get_cpl); F(set_segment); \ + F(get_cs_db_l_bits); F(set_cr0); F(is_valid_cr4); F(set_cr4); \ + F(set_efer); F(get_idt); F(set_idt); F(get_gdt); F(set_gdt); \ + F(sync_dirty_debug_regs); F(set_dr7); F(cache_reg); F(get_rflags); \ + F(set_rflags); F(tlb_flush_all); F(tlb_flush_current); \ + F(tlb_remote_flush); F(tlb_remote_flush_with_range); F(tlb_flush_gva); \ + F(tlb_flush_guest); F(run); F(handle_exit); \ + F(skip_emulated_instruction); F(update_emulated_instruction); \ + F(set_interrupt_shadow); F(get_interrupt_shadow); F(patch_hypercall); \ + F(set_irq); F(set_nmi); F(queue_exception); F(cancel_injection); \ + F(interrupt_allowed); F(nmi_allowed); F(get_nmi_mask); F(set_nmi_mask);\ + F(enable_nmi_window); F(enable_irq_window); F(update_cr8_intercept); \ + F(check_apicv_inhibit_reasons); F(pre_update_apicv_exec_ctrl); \ + F(refresh_apicv_exec_ctrl); F(hwapic_irr_update); F(hwapic_isr_update);\ + F(guest_apic_has_interrupt); F(load_eoi_exitmap); \ + F(set_virtual_apic_mode); F(set_apic_access_page_addr); \ + F(deliver_posted_interrupt); F(sync_pir_to_irr); F(set_tss_addr); \ + F(set_identity_map_addr); F(get_mt_mask); F(load_mmu_pgd); \ + F(has_wbinvd_exit); F(write_l1_tsc_offset); F(get_exit_info); \ + F(check_intercept); F(handle_exit_irqoff); F(request_immediate_exit); \ + F(sched_in); F(slot_enable_log_dirty); F(slot_disable_log_dirty); \ + F(flush_log_dirty); F(enable_log_dirty_pt_masked); \ + F(cpu_dirty_log_size); F(pre_block); F(post_block); F(vcpu_blocking); \ + F(vcpu_unblocking); F(update_pi_irte); F(apicv_post_state_restore); \ + F(dy_apicv_has_pending_interrupt); F(set_hv_timer); F(cancel_hv_timer);\ + F(setup_mce); F(smi_allowed); F(pre_enter_smm); F(pre_leave_smm); \ + F(enable_smi_window); F(mem_enc_op); F(mem_enc_reg_region); \ + F(mem_enc_unreg_region); F(get_msr_feature); \ + F(can_emulate_instruction); F(apic_init_signal_blocked); \ + F(enable_direct_tlbflush); F(migrate_timers); F(msr_filter_changed); \ + F(complete_emulated_msr) + +#define DEFINE_KVM_OPS_STATIC_CALL (func) \ + DEFINE_STATIC_CALL_NULL(kvm_x86_##func, \ + *(((struct kvm_x86_ops *)0)->func)) +#define DEFINE_KVM_OPS_STATIC_CALLS () \ + FOREACH_KVM_X86_OPS(DEFINE_KVM_OPS_STATIC_CALL) + +#define DECLARE_KVM_OPS_STATIC_CALL (func) \ + DECLARE_STATIC_CALL(kvm_x86_##func, \ + *(((struct kvm_x86_ops *)0)->func)) +#define DECLARE_KVM_OPS_STATIC_CALLS () \ + FOREACH_KVM_X86_OPS(DECLARE_KVM_OPS_STATIC_CALL) + +#define KVM_OPS_STATIC_CALL_UPDATE (func) \ + static_call_update(kvm_x86_##func, kvm_x86_ops.func) +#define KVM_OPS_STATIC_CALL_UPDATES () \ + FOREACH_KVM_X86_OPS(KVM_OPS_STATIC_CALL_UPDATE) +struct kvm_x86_ops {int (*hardware_enable)(void );void (*hardware_disable)(void );
@@ -1326,6 +1385,12 @@ extern u64 __read_mostly host_efer;extern bool __read_mostly allow_smaller_maxphyaddr;extern struct kvm_x86_ops kvm_x86_ops; +DECLARE_KVM_OPS_STATIC_CALLS (); +static inline void kvm_ops_static_call_update (void ) +{ +KVM_OPS_STATIC_CALL_UPDATES (); +} +#define __KVM_HAVE_ARCH_VM_ALLOCstatic inline struct kvm *kvm_arch_alloc_vm (void ) {
arch/x86/kvm/x86.c +5 -0
@@ -113,6 +113,11 @@ static int sync_regs(struct kvm_vcpu *vcpu);struct kvm_x86_ops kvm_x86_ops__read_mostly ;EXPORT_SYMBOL_GPL (kvm_x86_ops); +DEFINE_KVM_OPS_STATIC_CALLS (); +EXPORT_STATIC_CALL_GPL (kvm_x86_get_cs_db_l_bits); +EXPORT_STATIC_CALL_GPL (kvm_x86_cache_reg); +EXPORT_STATIC_CALL_GPL (kvm_x86_tlb_flush_current); +static bool __read_mostly ignore_msrs =0 ;module_param (ignore_msrs, bool,S_IRUGO |S_IWUSR );
Signature · 1 line
2.7.4
Sean Christopherson
maintainerRe: [PATCH 1/2] KVM: x86: introduce definitions to support static calls for kvm_x86_ops
Quoted 60 lines · Jason Baron
Use static calls to improve
kvm_x86_opsperformance. Introduce the definitions that will be used by a subsequent patch to actualize the savings.Note that all
kvm_x86_opsare covered here except for 'pmu_ops' and 'nested ops'. I think they can be covered by static calls in a simlilar manner, but were omitted from this series to reduce scope and because I don't think they have as large of a performance impact.Cc Paolo Bonzini <[email protected]>Cc Thomas Gleixner <[email protected]>Cc Ingo Molnar <[email protected]>Cc Borislav Petkov <[email protected]>Cc Peter Zijlstra <[email protected]>Cc Andrea Arcangeli <[email protected]>Signed-off-by Jason Baron <[email protected]>--- arch/x86/include/asm/kvm_host.h | 65 +++++++++++++++++++++++++++++++++++++++++ arch/x86/kvm/x86.c | 5 ++++ 2 files changed, 70 insertions(+)arch/x86/include/asm/kvm_host.h +31
-0@@ -1087,6 +1087,65 @@ static inline u16 kvm_lapic_irq_dest_mode(bool dest_mode_logical)return dest_mode_logical ?APIC_DEST_LOGICAL :APIC_DEST_PHYSICAL ; } +/* +* static calls cover all kvm_x86_ops except for functions under pmu_ops and +* nested_ops. +*/ +#define FOREACH_KVM_X86_OPS (F ) \ + F(hardware_enable); F(hardware_disable); F(hardware_unsetup); \ + F(cpu_has_accelerated_tpr); F(has_emulated_msr); \ + F(vcpu_after_set_cpuid); F(vm_init); F(vm_destroy); F(vcpu_create); \ + F(vcpu_free); F(vcpu_reset); F(prepare_guest_switch); F(vcpu_load); \ + F(vcpu_put); F(update_exception_bitmap); F(get_msr); F(set_msr); \ + F(get_segment_base); F(get_segment); F(get_cpl); F(set_segment); \ + F(get_cs_db_l_bits); F(set_cr0); F(is_valid_cr4); F(set_cr4); \ + F(set_efer); F(get_idt); F(set_idt); F(get_gdt); F(set_gdt); \ + F(sync_dirty_debug_regs); F(set_dr7); F(cache_reg); F(get_rflags); \ + F(set_rflags); F(tlb_flush_all); F(tlb_flush_current); \ + F(tlb_remote_flush); F(tlb_remote_flush_with_range); F(tlb_flush_gva); \ + F(tlb_flush_guest); F(run); F(handle_exit); \ + F(skip_emulated_instruction); F(update_emulated_instruction); \ + F(set_interrupt_shadow); F(get_interrupt_shadow); F(patch_hypercall); \ + F(set_irq); F(set_nmi); F(queue_exception); F(cancel_injection); \ + F(interrupt_allowed); F(nmi_allowed); F(get_nmi_mask); F(set_nmi_mask);\ + F(enable_nmi_window); F(enable_irq_window); F(update_cr8_intercept); \ + F(check_apicv_inhibit_reasons); F(pre_update_apicv_exec_ctrl); \ + F(refresh_apicv_exec_ctrl); F(hwapic_irr_update); F(hwapic_isr_update);\ + F(guest_apic_has_interrupt); F(load_eoi_exitmap); \ + F(set_virtual_apic_mode); F(set_apic_access_page_addr); \ + F(deliver_posted_interrupt); F(sync_pir_to_irr); F(set_tss_addr); \ + F(set_identity_map_addr); F(get_mt_mask); F(load_mmu_pgd); \ + F(has_wbinvd_exit); F(write_l1_tsc_offset); F(get_exit_info); \ + F(check_intercept); F(handle_exit_irqoff); F(request_immediate_exit); \ + F(sched_in); F(slot_enable_log_dirty); F(slot_disable_log_dirty); \
arch/x86/include/asm/kvm_host.h static inline u16 kvm_lapic_irq_dest_mode(bool dest_mode_logical) · line 1130
Quoted 10 lines
arch/x86/include/asm/kvm_host.h +10
-0+F (flush_log_dirty);F (enable_log_dirty_pt_masked); \ +F (cpu_dirty_log_size);F (pre_block);F (post_block);F (vcpu_blocking); \ +F (vcpu_unblocking);F (update_pi_irte);F (apicv_post_state_restore); \ +F (dy_apicv_has_pending_interrupt);F (set_hv_timer);F (cancel_hv_timer);\ +F (setup_mce);F (smi_allowed);F (pre_enter_smm);F (pre_leave_smm); \ +F (enable_smi_window);F (mem_enc_op);F (mem_enc_reg_region); \ +F (mem_enc_unreg_region);F (get_msr_feature); \ +F (can_emulate_instruction);F (apic_init_signal_blocked); \ +F (enable_direct_tlbflush);F (migrate_timers);F (msr_filter_changed); \ +F (complete_emulated_msr)
What about adding a dedicated .h file for this beast? Then it won't be so painful to do one function per line. As is, updates to kvm_x86_ops will be messy.
And add yet another macro layer (or maybe just tweak this one?) so that the caller controls the line ending? I suppose you could also just use a comma, but that's a bit dirty...
That would also allow using this to declare vmx_x86_ops and svm_x86_ops, which would need a comma insteat of a semi-colon. There have a been a few attempts to add a bit of automation to {vmx,svm}_x86_ops, this seems like it would be good motivation to go in a different direction and declare/define all ops, e.g. the VMX/SVM code could simply do something like:
#define DECLARE_VMX_X86_OP(func) \ .func = vmx_##func
static struct kvm_x86_ops vmx_x86_ops __initdata = {
.vm_size = sizeof(struct kvm_vmx),
.vm_init = vmx_vm_init,
.pmu_ops = &intel_pmu_ops,
.nested_ops = &vmx_nested_ops,
FOREACH_KVM_X86_OPS(DECLARE_VMX_X86_OP)
};Re: [PATCH 1/2] KVM: x86: introduce definitions to support static calls for kvm_x86_ops
Quoted 66 lines · Sean Christopherson
Quoted 65 lines · Jason Baron
Use static calls to improve
kvm_x86_opsperformance. Introduce the definitions that will be used by a subsequent patch to actualize the savings.Note that all
kvm_x86_opsare covered here except for 'pmu_ops' and 'nested ops'. I think they can be covered by static calls in a simlilar manner, but were omitted from this series to reduce scope and because I don't think they have as large of a performance impact.Cc Paolo Bonzini <[email protected]>Cc Thomas Gleixner <[email protected]>Cc Ingo Molnar <[email protected]>Cc Borislav Petkov <[email protected]>Cc Peter Zijlstra <[email protected]>Cc Andrea Arcangeli <[email protected]>Signed-off-by Jason Baron <[email protected]>--- arch/x86/include/asm/kvm_host.h | 65 +++++++++++++++++++++++++++++++++++++++++ arch/x86/kvm/x86.c | 5 ++++ 2 files changed, 70 insertions(+)arch/x86/include/asm/kvm_host.h +36
-0@@ -1087,6 +1087,65 @@ static inline u16 kvm_lapic_irq_dest_mode(bool dest_mode_logical)return dest_mode_logical ?APIC_DEST_LOGICAL :APIC_DEST_PHYSICAL ; } +/* +* static calls cover all kvm_x86_ops except for functions under pmu_ops and +* nested_ops. +*/ +#define FOREACH_KVM_X86_OPS (F ) \ + F(hardware_enable); F(hardware_disable); F(hardware_unsetup); \ + F(cpu_has_accelerated_tpr); F(has_emulated_msr); \ + F(vcpu_after_set_cpuid); F(vm_init); F(vm_destroy); F(vcpu_create); \ + F(vcpu_free); F(vcpu_reset); F(prepare_guest_switch); F(vcpu_load); \ + F(vcpu_put); F(update_exception_bitmap); F(get_msr); F(set_msr); \ + F(get_segment_base); F(get_segment); F(get_cpl); F(set_segment); \ + F(get_cs_db_l_bits); F(set_cr0); F(is_valid_cr4); F(set_cr4); \ + F(set_efer); F(get_idt); F(set_idt); F(get_gdt); F(set_gdt); \ + F(sync_dirty_debug_regs); F(set_dr7); F(cache_reg); F(get_rflags); \ + F(set_rflags); F(tlb_flush_all); F(tlb_flush_current); \ + F(tlb_remote_flush); F(tlb_remote_flush_with_range); F(tlb_flush_gva); \ + F(tlb_flush_guest); F(run); F(handle_exit); \ + F(skip_emulated_instruction); F(update_emulated_instruction); \ + F(set_interrupt_shadow); F(get_interrupt_shadow); F(patch_hypercall); \ + F(set_irq); F(set_nmi); F(queue_exception); F(cancel_injection); \ + F(interrupt_allowed); F(nmi_allowed); F(get_nmi_mask); F(set_nmi_mask);\ + F(enable_nmi_window); F(enable_irq_window); F(update_cr8_intercept); \ + F(check_apicv_inhibit_reasons); F(pre_update_apicv_exec_ctrl); \ + F(refresh_apicv_exec_ctrl); F(hwapic_irr_update); F(hwapic_isr_update);\ + F(guest_apic_has_interrupt); F(load_eoi_exitmap); \ + F(set_virtual_apic_mode); F(set_apic_access_page_addr); \ + F(deliver_posted_interrupt); F(sync_pir_to_irr); F(set_tss_addr); \ + F(set_identity_map_addr); F(get_mt_mask); F(load_mmu_pgd); \ + F(has_wbinvd_exit); F(write_l1_tsc_offset); F(get_exit_info); \ + F(check_intercept); F(handle_exit_irqoff); F(request_immediate_exit); \ + F(sched_in); F(slot_enable_log_dirty); F(slot_disable_log_dirty); \ + F(flush_log_dirty); F(enable_log_dirty_pt_masked); \ + F(cpu_dirty_log_size); F(pre_block); F(post_block); F(vcpu_blocking); \ + F(vcpu_unblocking); F(update_pi_irte); F(apicv_post_state_restore); \ + F(dy_apicv_has_pending_interrupt); F(set_hv_timer); F(cancel_hv_timer);\ + F(setup_mce); F(smi_allowed); F(pre_enter_smm); F(pre_leave_smm); \
Quoted 8 lines
arch/x86/include/asm/kvm_host.h static inline u16 kvm_lapic_irq_dest_mode(bool dest_mode_logical) · line 1130
Quoted 5 lines
arch/x86/include/asm/kvm_host.h +5
-0+F (enable_smi_window);F (mem_enc_op);F (mem_enc_reg_region); \ +F (mem_enc_unreg_region);F (get_msr_feature); \ +F (can_emulate_instruction);F (apic_init_signal_blocked); \ +F (enable_direct_tlbflush);F (migrate_timers);F (msr_filter_changed); \ +F (complete_emulated_msr)What about adding a dedicated .h file for this beast? Then it won't be so painful to do one function per line. As is, updates to
kvm_x86_opswill be messy.
Hi,
I'm fine moving it to a new header. I had put it right above 'struct kvm_x86_ops' in arch/x86/include/asm/kvm_host.h, such that it would be updated along with kvm_x86_ops changes. But I can put a big comment there to update the new header.
Quoted 14 lines
And add yet another macro layer (or maybe just tweak this one?) so that the caller controls the line ending? I suppose you could also just use a comma, but that's a bit dirty...
That would also allow using this to declare
vmx_x86_opsandsvm_x86_ops, which would need a comma insteat of a semi-colon. There have a been a few attempts to add a bit of automation to {vmx,svm}_x86_ops, this seems like it would be good motivation to go in a different direction and declare/define all ops, e.g. the VMX/SVM code could simply do something like:#define DECLARE_VMX_X86_OP(func) \ .func = vmx_##func
Quoted 10 lines
static struct kvm_x86_ops vmx_x86_ops __initdata = {
.vm_size = sizeof(struct kvm_vmx),
.vm_init = vmx_vm_init,
.pmu_ops = &intel_pmu_ops,
.nested_ops = &vmx_nested_ops,
FOREACH_KVM_X86_OPS(DECLARE_VMX_X86_OP)
};Looking at the vmx definitions I see quite a few that don't match that naming. For example:
hardware_unsetup,hardware_enable,hardware_disable,report_flexpriority,update_exception_bitmap,enable_nmi_window,enable_irq_window,update_cr8_intercept,pi_has_pending_interrupt,cpu_has_vmx_wbinvd_exit,pi_update_irte,kvm_complete_insn_gp,
So I'm not sure if we want to extend these macros to vmx/svm.
Thanks,
-Jason
Paolo Bonzini
maintainerRe: [PATCH 1/2] KVM: x86: introduce definitions to support static calls for kvm_x86_ops
Quoted 17 lines · Jason Baron
Quoted 1 line
Looking at the vmx definitions I see quite a few that don't match that naming. For example:
hardware_unsetup,hardware_enable,hardware_disable,report_flexpriority,update_exception_bitmap,enable_nmi_window,enable_irq_window,update_cr8_intercept,pi_has_pending_interrupt,cpu_has_vmx_wbinvd_exit,pi_update_irte,kvm_complete_insn_gp,
Quoted 2 lines
So I'm not sure if we want to extend these macros to vmx/svm.
Don't do it yourself, but once you introduce the new header it becomes a no-brainer to switch the declarations to use it. So let's plan the new header to make that switch easy.
Using trailing commas unconditionally would be okay, i.e.
#define X86_OP(func) .func = vmx_##func,
#include "kvm-x86-ops.h"
and leave out the terminator/delimiter in kvm-x86-ops.h. This is similar to how we use vmx/vmcs_shadow_fields.h:
#define SHADOW_FIELD_RO(x, y) { x, offsetof(struct vmcs12, y) }, #include "vmcs_shadow_fields.h"
#define SHADOW_FIELD_RW(x, y) case x:
#include "vmcs_shadow_fields.h"
Thanks,
Paolo
Sean Christopherson
maintainerRe: [PATCH 1/2] KVM: x86: introduce definitions to support static calls for kvm_x86_ops
Quoted 18 lines · Paolo Bonzini
Quoted 17 lines · Jason Baron
Quoted 1 line
Looking at the vmx definitions I see quite a few that don't match that naming. For example:
hardware_unsetup,hardware_enable,hardware_disable,report_flexpriority,update_exception_bitmap,enable_nmi_window,enable_irq_window,update_cr8_intercept,pi_has_pending_interrupt,cpu_has_vmx_wbinvd_exit,pi_update_irte,kvm_complete_insn_gp,
Quoted 6 lines
Quoted 2 lines
So I'm not sure if we want to extend these macros to vmx/svm.
Don't do it yourself, but once you introduce the new header it becomes a no-brainer to switch the declarations to use it. So let's plan the new header to make that switch easy.
Ya, sorry if I didn't make it clear that the vmx/svm conversion is firmly out of scope for this series.
Paolo Bonzini
maintainerRe: [PATCH 1/2] KVM: x86: introduce definitions to support static calls for kvm_x86_ops
In reply to Jason Baronarch/x86/include/asm/kvm_host.h static inline u16 kvm_lapic_irq_dest_mode(bool dest_mode_logical) · line 1136
Quoted 5 lines · Jason Baron
arch/x86/include/asm/kvm_host.h +5
-0+#define DEFINE_KVM_OPS_STATIC_CALL (func) \ + DEFINE_STATIC_CALL_NULL(kvm_x86_##func, \ + *(((struct kvm_x86_ops *)0)->func)) +#define DEFINE_KVM_OPS_STATIC_CALLS () \ + FOREACH_KVM_X86_OPS(DEFINE_KVM_OPS_STATIC_CALL)
Something wrong here?
Quoted 15 lines
arch/x86/include/asm/kvm_host.h +11
-0+#define DECLARE_KVM_OPS_STATIC_CALL (func) \ + DECLARE_STATIC_CALL(kvm_x86_##func, \ + *(((struct kvm_x86_ops *)0)->func)) +#define DECLARE_KVM_OPS_STATIC_CALLS () \ + FOREACH_KVM_X86_OPS(DECLARE_KVM_OPS_STATIC_CALL) + +#define KVM_OPS_STATIC_CALL_UPDATE (func) \ + static_call_update(kvm_x86_##func, kvm_x86_ops.func) +#define KVM_OPS_STATIC_CALL_UPDATES () \ + FOREACH_KVM_X86_OPS(KVM_OPS_STATIC_CALL_UPDATE) +struct kvm_x86_ops {int (*hardware_enable)(void );void (*hardware_disable)(void );
arch/x86/include/asm/kvm_host.h extern u64 __read_mostly host_efer; · line 1392
Quoted 8 lines
arch/x86/include/asm/kvm_host.h +5
-0@@ -1326,6 +1385,12 @@ extern u64 __read_mostly host_efer;extern bool __read_mostly allow_smaller_maxphyaddr;extern struct kvm_x86_ops kvm_x86_ops; +DECLARE_KVM_OPS_STATIC_CALLS (); +static inline void kvm_ops_static_call_update (void ) +{ +KVM_OPS_STATIC_CALL_UPDATES (); +}
This would become
#define KVM_X86_OP(func) \ DECLARE_STATIC_CALL(kvm_x86_##func, \ *(((struct kvm_x86_ops *)0)->func));
#include <asm/kvm-x86-ops.h>
static inline void kvm_ops_static_call_update(void) {
#define KVM_X86_OP(func) \
static_call_update(kvm_x86_##func, kvm_x86_ops.func)
#include <asm/kvm-x86-ops.h>
}
If you need to choose between DECLARE_STATIC_CALL_NULL and DECLARE_STATIC_CALL, you can have kvm-x86-ops.h use one of two macros KVM_X86_OP_NULL and KVM_X86_OP.
#define KVM_X86_OP(func) \ DECLARE_STATIC_CALL(kvm_x86_##func, \ *(((struct kvm_x86_ops *)0)->func));
#define KVM_X86_OP_NULL(func) \ DECLARE_STATIC_CALL_NULL(kvm_x86_##func, \ *(((struct kvm_x86_ops *)0)->func));
#include <asm/kvm-x86-ops.h>
...
#define KVM_X86_OP(func) \
static_call_update(kvm_x86_##func, kvm_x86_ops.func)
#define KVM_X86_OP_NULL(func) \
static_call_update(kvm_x86_##func, kvm_x86_ops.func)
#include <asm/kvm-x86-ops.h>
In that case vmx.c and svm.c could define KVM_X86_OP_NULL to an empty string and list the optional callbacks manually.
Paolo
Re: [PATCH 1/2] KVM: x86: introduce definitions to support static calls for kvm_x86_ops
Quoted 8 lines · Paolo Bonzini
arch/x86/include/asm/kvm_host.h static inline u16 kvm_lapic_irq_dest_mode(bool dest_mode_logical) · line 1136
Quoted 5 lines · Jason Baron
arch/x86/include/asm/kvm_host.h +5
-0+#define DEFINE_KVM_OPS_STATIC_CALL (func) \ + DEFINE_STATIC_CALL_NULL(kvm_x86_##func, \ + *(((struct kvm_x86_ops *)0)->func)) +#define DEFINE_KVM_OPS_STATIC_CALLS () \ + FOREACH_KVM_X86_OPS(DEFINE_KVM_OPS_STATIC_CALL)Something wrong here?
Hmmm...not sure what you are getting at here.
Quoted 54 lines
Quoted 16 lines
arch/x86/include/asm/kvm_host.h +11
-0+#define DECLARE_KVM_OPS_STATIC_CALL (func) \ + DECLARE_STATIC_CALL(kvm_x86_##func, \ + *(((struct kvm_x86_ops *)0)->func)) +#define DECLARE_KVM_OPS_STATIC_CALLS () \ + FOREACH_KVM_X86_OPS(DECLARE_KVM_OPS_STATIC_CALL) + +#define KVM_OPS_STATIC_CALL_UPDATE (func) \ + static_call_update(kvm_x86_##func, kvm_x86_ops.func) +#define KVM_OPS_STATIC_CALL_UPDATES () \ + FOREACH_KVM_X86_OPS(KVM_OPS_STATIC_CALL_UPDATE) +struct
kvm_x86_ops{
int (*hardware_enable)(void);
void (*hardware_disable)(void);arch/x86/include/asm/kvm_host.h extern u64 __read_mostly host_efer; · line 1391
Quoted 7 lines
arch/x86/include/asm/kvm_host.h +4
-0@@ -1326,6 +1385,12 @@ extern u64 __read_mostly host_efer; extern bool __read_mostly allow_smaller_maxphyaddr; extern struct kvm_x86_ops kvm_x86_ops; +DECLARE_KVM_OPS_STATIC_CALLS(); +static inline void kvm_ops_static_call_update (void ) +{ + KVM_OPS_STATIC_CALL_UPDATES(); +}This would become
#define
KVM_X86_OP(func)\
DECLARE_STATIC_CALL(kvm_x86_##func, \
*(((structkvm_x86_ops*)0)->func));#include <
asm/kvm-x86-ops.h>static inline void
kvm_ops_static_call_update(void){
#defineKVM_X86_OP(func)\
static_call_update(kvm_x86_##func, kvm_x86_ops.func)#include <asm/kvm-x86-ops.h>
}If you need to choose between
DECLARE_STATIC_CALL_NULLandDECLARE_STATIC_CALL, you can havekvm-x86-ops.huse one of two macrosKVM_X86_OP_NULLandKVM_X86_OP.#define
KVM_X86_OP(func)\
DECLARE_STATIC_CALL(kvm_x86_##func, \
*(((structkvm_x86_ops*)0)->func));#define
KVM_X86_OP_NULL(func)\
DECLARE_STATIC_CALL_NULL(kvm_x86_##func, \
*(((structkvm_x86_ops*)0)->func));#include <
asm/kvm-x86-ops.h>...
Quoted 8 lines
#define
KVM_X86_OP(func)\
static_call_update(kvm_x86_##func, kvm_x86_ops.func)#defineKVM_X86_OP_NULL(func)\
static_call_update(kvm_x86_##func, kvm_x86_ops.func)#include <asm/kvm-x86-ops.h>In that case
vmx.candsvm.ccould defineKVM_X86_OP_NULLto an empty string and list the optional callbacks manually.
Ok, yes, this all makes sense. So I looked at vmx/svm definitions and I see that there are 5 definitions that are common that don't use the vmx or svm prefix:
.update_exception_bitmap = update_exception_bitmap, .enable_nmi_window = enable_nmi_window, .enable_irq_window = enable_irq_window, .update_cr8_intercept = update_cr8_intercept, .enable_smi_window = enable_smi_window,
8 are specific to vmx that don't use the vmx prefix:
.hardware_unsetup = hardware_unsetup, .hardware_enable = hardware_enable, .hardware_disable = hardware_disable, .cpu_has_accelerated_tpr = report_flexpriority, .dy_apicv_has_pending_interrupt = pi_has_pending_interrupt, .has_wbinvd_exit = cpu_has_vmx_wbinvd_exit, .update_pi_irte = pi_update_irte, .complete_emulated_msr = kvm_complete_insn_gp,
and finally 7 specific to svm that don't use the svm prefix:
.get_cs_db_l_bits = kvm_get_cs_db_l_bits, .handle_exit = handle_exit, .skip_emulated_instruction = skip_emulated_instruction, .update_emulated_instruction = NULL, .sync_pir_to_irr = kvm_lapic_find_highest_irr, .apicv_post_state_restore = avic_post_state_restore, .request_immediate_exit = __kvm_request_immediate_exit,
So we could set all of these to empty definitions and specifically add the callbacks manually as you suggested. Or we could have 4 categories of macros:
1) KVM_X86_OP() - for the usual case where we have the 'svm' or 'vmx' prefix.
2) KVM_X86_OP_NULL() - for the case where both svm and vmx want to over-ride.
For the remaining cases I was thinking of having the VMX code and svm code, add a define for 'VMX' or 'SVM' respectively and then we could mark the vmx specific ones by adding something like this to asm/kvm-x86-ops.h:
#ifndef VMX
#define KVM_X86_OP_UNLESS_VMX(func) \ KVM_X86_OP(func)
#else
#define KVM_X86_OP_UNLESS_VMX(func)
#endif
and similarly for the svm specific ones:
#ifndef SVM
#define KVM_X86_OP_UNLESS_SVM(func) \ KVM_X86_OP(func)
#else
#define KVM_X86_OP_UNLESS_SVM(func)
#endif
So in vmx.c you would have:
#define X86_OP(func) .func = vmx_##func,
#define VMX 1
#include "kvm-x86-ops.h"
Or we could just use the KVM_X86_OP_NULL() macro for anything that doesn't have a 'svm' or 'vmx' prefix as I think you were suggesting? In that case I think there would then be 13 manual additions for vmx and 12 for svm.
Thanks,
-Jason
Re: [PATCH 1/2] KVM: x86: introduce definitions to support static calls for kvm_x86_ops
Quoted 143 lines · Jason Baron
Quoted 8 lines · Paolo Bonzini
arch/x86/include/asm/kvm_host.h static inline u16 kvm_lapic_irq_dest_mode(bool dest_mode_logical) · line 1136
Quoted 5 lines · Jason Baron
arch/x86/include/asm/kvm_host.h +5
-0+#define DEFINE_KVM_OPS_STATIC_CALL (func) \ + DEFINE_STATIC_CALL_NULL(kvm_x86_##func, \ + *(((struct kvm_x86_ops *)0)->func)) +#define DEFINE_KVM_OPS_STATIC_CALLS () \ + FOREACH_KVM_X86_OPS(DEFINE_KVM_OPS_STATIC_CALL)Something wrong here?
Hmmm...not sure what you are getting at here.
Quoted 54 lines
Quoted 16 lines
arch/x86/include/asm/kvm_host.h +11
-0+#define DECLARE_KVM_OPS_STATIC_CALL (func) \ + DECLARE_STATIC_CALL(kvm_x86_##func, \ + *(((struct kvm_x86_ops *)0)->func)) +#define DECLARE_KVM_OPS_STATIC_CALLS () \ + FOREACH_KVM_X86_OPS(DECLARE_KVM_OPS_STATIC_CALL) + +#define KVM_OPS_STATIC_CALL_UPDATE (func) \ + static_call_update(kvm_x86_##func, kvm_x86_ops.func) +#define KVM_OPS_STATIC_CALL_UPDATES () \ + FOREACH_KVM_X86_OPS(KVM_OPS_STATIC_CALL_UPDATE) +struct
kvm_x86_ops{
int (*hardware_enable)(void);
void (*hardware_disable)(void);arch/x86/include/asm/kvm_host.h extern u64 __read_mostly host_efer; · line 1391
Quoted 7 lines
arch/x86/include/asm/kvm_host.h +4
-0@@ -1326,6 +1385,12 @@ extern u64 __read_mostly host_efer; extern bool __read_mostly allow_smaller_maxphyaddr; extern struct kvm_x86_ops kvm_x86_ops; +DECLARE_KVM_OPS_STATIC_CALLS(); +static inline void kvm_ops_static_call_update (void ) +{ + KVM_OPS_STATIC_CALL_UPDATES(); +}This would become
#define
KVM_X86_OP(func)\
DECLARE_STATIC_CALL(kvm_x86_##func, \
*(((structkvm_x86_ops*)0)->func));#include <
asm/kvm-x86-ops.h>static inline void
kvm_ops_static_call_update(void){
#defineKVM_X86_OP(func)\
static_call_update(kvm_x86_##func, kvm_x86_ops.func)#include <asm/kvm-x86-ops.h>
}If you need to choose between
DECLARE_STATIC_CALL_NULLandDECLARE_STATIC_CALL, you can havekvm-x86-ops.huse one of two macrosKVM_X86_OP_NULLandKVM_X86_OP.#define
KVM_X86_OP(func)\
DECLARE_STATIC_CALL(kvm_x86_##func, \
*(((structkvm_x86_ops*)0)->func));#define
KVM_X86_OP_NULL(func)\
DECLARE_STATIC_CALL_NULL(kvm_x86_##func, \
*(((structkvm_x86_ops*)0)->func));#include <
asm/kvm-x86-ops.h>...
Quoted 8 lines
#define
KVM_X86_OP(func)\
static_call_update(kvm_x86_##func, kvm_x86_ops.func)#defineKVM_X86_OP_NULL(func)\
static_call_update(kvm_x86_##func, kvm_x86_ops.func)#include <asm/kvm-x86-ops.h>In that case
vmx.candsvm.ccould defineKVM_X86_OP_NULLto an empty string and list the optional callbacks manually.Ok, yes, this all makes sense. So I looked at vmx/svm definitions and I see that there are 5 definitions that are common that don't use the vmx or svm prefix:
.update_exception_bitmap =
update_exception_bitmap, .enable_nmi_window =enable_nmi_window, .enable_irq_window =enable_irq_window, .update_cr8_intercept =update_cr8_intercept, .enable_smi_window =enable_smi_window,8 are specific to vmx that don't use the vmx prefix:
.hardware_unsetup =
hardware_unsetup, .hardware_enable =hardware_enable, .hardware_disable =hardware_disable, .cpu_has_accelerated_tpr =report_flexpriority, .dy_apicv_has_pending_interrupt =pi_has_pending_interrupt, .has_wbinvd_exit =cpu_has_vmx_wbinvd_exit, .update_pi_irte =pi_update_irte, .complete_emulated_msr =kvm_complete_insn_gp,and finally 7 specific to svm that don't use the svm prefix:
.get_cs_db_l_bits =
kvm_get_cs_db_l_bits, .handle_exit =handle_exit, .skip_emulated_instruction =skip_emulated_instruction, .update_emulated_instruction = NULL, .sync_pir_to_irr =kvm_lapic_find_highest_irr, .apicv_post_state_restore =avic_post_state_restore, .request_immediate_exit =__kvm_request_immediate_exit,So we could set all of these to empty definitions and specifically add the callbacks manually as you suggested. Or we could have 4 categories of macros:
1)
KVM_X86_OP()- for the usual case where we have the 'svm' or 'vmx' prefix.2)
KVM_X86_OP_NULL()- for the case where both svm and vmx want to over-ride.For the remaining cases I was thinking of having the VMX code and svm code, add a define for 'VMX' or 'SVM' respectively and then we could mark the vmx specific ones by adding something like this to
asm/kvm-x86-ops.h:#ifndef VMX
#define KVM_X86_OP_UNLESS_VMX(func) \ KVM_X86_OP(func)#else
#defineKVM_X86_OP_UNLESS_VMX(func)
#endifand similarly for the svm specific ones:
#ifndef SVM
#define KVM_X86_OP_UNLESS_SVM(func) \ KVM_X86_OP(func)#else
#defineKVM_X86_OP_UNLESS_SVM(func)
#endifSo in
vmx.cyou would have:
Quoted 9 lines
#define
X86_OP(func).func = vmx_##func,
#define VMX 1
#include "kvm-x86-ops.h"Or we could just use the
KVM_X86_OP_NULL()macro for anything that doesn't have a 'svm' or 'vmx' prefix as I think you were suggesting? In that case I think there would then be 13 manual additions for vmx and 12 for svm.
Sorry this last bit is wrong. Using the KVM_X86_OP_NULL() for all definitions that don't use 'svm' and 'vmx', would mean manually defining all 20 in vmx and svm. Whereas if we we mark some of them specific to vmx or svm, we can get away with less - 13 manual for vmx and 12 manual for svm.
Thanks,
-Jason
Paolo Bonzini
maintainerRe: [PATCH 1/2] KVM: x86: introduce definitions to support static calls for kvm_x86_ops
In reply to Jason BaronQuoted 7 lines · Jason Baron
Quoted 6 lines
arch/x86/include/asm/kvm_host.h static inline u16 kvm_lapic_irq_dest_mode(bool dest_mode_logical) · line 1136
Quoted 5 lines
arch/x86/include/asm/kvm_host.h +5
-0+#define DEFINE_KVM_OPS_STATIC_CALL (func) \ + DEFINE_STATIC_CALL_NULL(kvm_x86_##func, \ + *(((struct kvm_x86_ops *)0)->func)) +#define DEFINE_KVM_OPS_STATIC_CALLS () \ + FOREACH_KVM_X86_OPS(DEFINE_KVM_OPS_STATIC_CALL)Something wrong here?
Hmmm...not sure what you are getting at here.
I just misread define vs. declare.
Quoted 7 lines
Or we could just use the
KVM_X86_OP_NULL()macro for anything that doesn't have a 'svm' or 'vmx' prefix as I think you were suggesting?Using the
KVM_X86_OP_NULL()for
all definitions that don't use 'svm' and 'vmx', would mean manually defining all 20 in vmx and svm
Yes, that's the simplest thing to do. Then we clean them up as we rename the functions. If you want, go ahead and rename the five easy ones yourself.
Paolo
Quoted 5 lines
.update_exception_bitmap =
update_exception_bitmap, .enable_nmi_window =enable_nmi_window, .enable_irq_window =enable_irq_window, .update_cr8_intercept =update_cr8_intercept, .enable_smi_window =enable_smi_window,
Sean Christopherson
maintainerRe: [PATCH 1/2] KVM: x86: introduce definitions to support static calls for kvm_x86_ops
In reply to Jason BaronQuoted 8 lines · Jason Baron
Quoted 6 lines · Paolo Bonzini
#define
KVM_X86_OP(func)\
static_call_update(kvm_x86_##func, kvm_x86_ops.func)#defineKVM_X86_OP_NULL(func)\
static_call_update(kvm_x86_##func, kvm_x86_ops.func)#include <asm/kvm-x86-ops.h>
Quoted 7 lines
Quoted 3 lines
In that case
vmx.candsvm.ccould defineKVM_X86_OP_NULLto an empty string and list the optional callbacks manually.Ok, yes, this all makes sense. So I looked at vmx/svm definitions and I see that there are 5 definitions that are common that don't use the vmx or svm prefix:
We'll rename the helpers when doing the conversion, i.e. you can safely assume that all VMX/SVM functions will use the pattern {vmx,svm}_##func. I did all the renaming a few months back, but it got tossed away when svm.c was broken up.
Quoted 5 lines
.update_exception_bitmap =
update_exception_bitmap, .enable_nmi_window =enable_nmi_window, .enable_irq_window =enable_irq_window, .update_cr8_intercept =update_cr8_intercept, .enable_smi_window =enable_smi_window,
Sean Christopherson
maintainerRe: [PATCH 1/2] KVM: x86: introduce definitions to support static calls for kvm_x86_ops
In reply to Paolo BonziniQuoted 10 lines · Paolo Bonzini
If you need to choose between
DECLARE_STATIC_CALL_NULLandDECLARE_STATIC_CALL, you can havekvm-x86-ops.huse one of two macrosKVM_X86_OP_NULLandKVM_X86_OP.#define KVM_X86_OP(func) \ DECLARE_STATIC_CALL(kvm_x86_##func, \ *(((struct kvm_x86_ops *)0)->func));#define KVM_X86_OP_NULL(func) \ DECLARE_STATIC_CALL_NULL(kvm_x86_##func, \
Gah, DECLARE_STATIC_CALL_NULL doesn't exist, though it's referenced in a comment. I assume these should be s/DECLARE/DEFINE? I haven't fully grokked the static call code yet...
Quoted 17 lines
*(((struct kvm_x86_ops *)0)->func));#include <
asm/kvm-x86-ops.h>...
#define
KVM_X86_OP(func)\
static_call_update(kvm_x86_##func, kvm_x86_ops.func)
#defineKVM_X86_OP_NULL(func)\
static_call_update(kvm_x86_##func, kvm_x86_ops.func)
#include <asm/kvm-x86-ops.h>In that case
vmx.candsvm.ccould defineKVM_X86_OP_NULLto an empty string and list the optional callbacks manually.Paolo
PATCH 2/2 KVM: x86: use static calls to reduce kvm_x86_ops overhead
In reply to Jason BaronConvert kvm_x86_ops to use static calls. Note that all kvm_x86_ops are covered here except for 'pmu_ops and 'nested ops'.
Here are some numbers running cpuid in a loop and measured timing in the vm (lower is better).
cpuid loop (1 million calls(avg. of 5 runs)):
|default |mitigations=off
---------------------------------------
vanilla |.671s |.486s static call|.573s(-15%)|.458s(-6%)
--- arch/x86/include/asm/kvm_host.h | 6 +- arch/x86/kvm/cpuid.c | 2 +- arch/x86/kvm/hyperv.c | 4 +- arch/x86/kvm/irq.c | 2 +- arch/x86/kvm/kvm_cache_regs.h | 10 +- arch/x86/kvm/lapic.c | 28 ++-- arch/x86/kvm/mmu.h | 6 +- arch/x86/kvm/mmu/mmu.c | 12 +- arch/x86/kvm/mmu/spte.c | 2 +- arch/x86/kvm/pmu.c | 2 +- arch/x86/kvm/trace.h | 4 +- arch/x86/kvm/x86.c | 294 ++++++++++++++++++++-------------------- arch/x86/kvm/x86.h | 6 +- 13 files changed, 189 insertions(+), 189 deletions(-)
arch/x86/include/asm/kvm_host.h +3 -3
@@ -1402,7 +1402,7 @@ void kvm_arch_free_vm(struct kvm *kvm);static inline int kvm_arch_flush_remote_tlb (struct kvm *kvm) {if (kvm_x86_ops.tlb_remote_flush && - !kvm_x86_ops.tlb_remote_flush (kvm)) + !static_call (kvm_x86_tlb_remote_flush)(kvm))return 0 ;else return -ENOTSUPP ;
@@ -1793,13 +1793,13 @@ static inline bool kvm_irq_is_postable(struct kvm_lapic_irq *irq)static inline void kvm_arch_vcpu_blocking (struct kvm_vcpu *vcpu) {if (kvm_x86_ops.vcpu_blocking) - kvm_x86_ops.vcpu_blocking (vcpu); +static_call (kvm_x86_vcpu_blocking)(vcpu); }static inline void kvm_arch_vcpu_unblocking (struct kvm_vcpu *vcpu) {if (kvm_x86_ops.vcpu_unblocking) - kvm_x86_ops.vcpu_unblocking (vcpu); +static_call (kvm_x86_vcpu_unblocking)(vcpu); }static inline void kvm_arch_vcpu_block_finish (struct kvm_vcpu *vcpu) {}
arch/x86/kvm/cpuid.c +1 -1
@@ -182,7 +182,7 @@ static void kvm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) vcpu->arch.cr3_lm_rsvd_bits =rsvd_bits (cpuid_maxphyaddr (vcpu),63 );/* Invoke the vendor callback only after the above state is updated. */ - kvm_x86_ops.vcpu_after_set_cpuid (vcpu); +static_call (kvm_x86_vcpu_after_set_cpuid)(vcpu); }static int is_efer_nx (void )
arch/x86/kvm/hyperv.c +2 -2
@@ -1154,7 +1154,7 @@ static int kvm_hv_set_msr_pw(struct kvm_vcpu *vcpu, u32 msr, u64 data, addr =gfn_to_hva (kvm, gfn);if (kvm_is_error_hva (addr))return 1 ; - kvm_x86_ops.patch_hypercall (vcpu, instructions); +static_call (kvm_x86_patch_hypercall)(vcpu, instructions); ((unsigned char *)instructions)[3 ] =0xc3 ;/* ret */ if (__copy_to_user ((void __user *)addr, instructions,4 ))return 1 ;
@@ -1745,7 +1745,7 @@ int kvm_hv_hypercall(struct kvm_vcpu *vcpu)* hypercall generates UD from non zero cpl and real mode * per HYPER-V spec */ -if (kvm_x86_ops.get_cpl (vcpu) !=0 || !is_protmode (vcpu)) { +if (static_call (kvm_x86_get_cpl)(vcpu) !=0 || !is_protmode (vcpu)) {kvm_queue_exception (vcpu,UD_VECTOR );return 1 ; }
arch/x86/kvm/irq.c +1 -1
@@ -144,7 +144,7 @@ void __kvm_migrate_timers(struct kvm_vcpu *vcpu)__kvm_migrate_apic_timer (vcpu);__kvm_migrate_pit_timer (vcpu);if (kvm_x86_ops.migrate_timers) - kvm_x86_ops.migrate_timers (vcpu); +static_call (kvm_x86_migrate_timers)(vcpu); }bool kvm_arch_irqfd_allowed (struct kvm *kvm,struct kvm_irqfd *args)
arch/x86/kvm/kvm_cache_regs.h +5 -5
@@ -69,7 +69,7 @@ static inline unsigned long kvm_register_read(struct kvm_vcpu *vcpu, int reg)return 0 ;if (!kvm_register_is_available (vcpu, reg)) - kvm_x86_ops.cache_reg (vcpu, reg); +static_call (kvm_x86_cache_reg)(vcpu, reg);return vcpu->arch.regs[reg]; }
@@ -109,7 +109,7 @@ static inline u64 kvm_pdptr_read(struct kvm_vcpu *vcpu, int index)might_sleep ();/* on svm */ if (!kvm_register_is_available (vcpu,VCPU_EXREG_PDPTR )) - kvm_x86_ops.cache_reg (vcpu,VCPU_EXREG_PDPTR ); +static_call (kvm_x86_cache_reg)(vcpu,VCPU_EXREG_PDPTR );return vcpu->arch.walk_mmu->pdptrs[index]; }
@@ -119,7 +119,7 @@ static inline ulong kvm_read_cr0_bits(struct kvm_vcpu *vcpu, ulong mask)ulong tmask = mask &KVM_POSSIBLE_CR0_GUEST_BITS ;if ((tmask & vcpu->arch.cr0_guest_owned_bits) && !kvm_register_is_available (vcpu,VCPU_EXREG_CR0 )) - kvm_x86_ops.cache_reg (vcpu,VCPU_EXREG_CR0 ); +static_call (kvm_x86_cache_reg)(vcpu,VCPU_EXREG_CR0 );return vcpu->arch.cr0 & mask; }
@@ -133,14 +133,14 @@ static inline ulong kvm_read_cr4_bits(struct kvm_vcpu *vcpu, ulong mask)ulong tmask = mask &KVM_POSSIBLE_CR4_GUEST_BITS ;if ((tmask & vcpu->arch.cr4_guest_owned_bits) && !kvm_register_is_available (vcpu,VCPU_EXREG_CR4 )) - kvm_x86_ops.cache_reg (vcpu,VCPU_EXREG_CR4 ); +static_call (kvm_x86_cache_reg)(vcpu,VCPU_EXREG_CR4 );return vcpu->arch.cr4 & mask; }static inline ulong kvm_read_cr3 (struct kvm_vcpu *vcpu) {if (!kvm_register_is_available (vcpu,VCPU_EXREG_CR3 )) - kvm_x86_ops.cache_reg (vcpu,VCPU_EXREG_CR3 ); +static_call (kvm_x86_cache_reg)(vcpu,VCPU_EXREG_CR3 );return vcpu->arch.cr3; }
arch/x86/kvm/lapic.c +14 -14
@@ -484,7 +484,7 @@ static inline void apic_clear_irr(int vec, struct kvm_lapic *apic)if (unlikely (vcpu->arch.apicv_active)) {/* need to update RVI */ kvm_lapic_clear_vector (vec, apic->regs +APIC_IRR ); - kvm_x86_ops.hwapic_irr_update (vcpu, +static_call (kvm_x86_hwapic_irr_update)(vcpu,apic_find_highest_irr (apic)); }else { apic->irr_pending = false;
@@ -515,7 +515,7 @@ static inline void apic_set_isr(int vec, struct kvm_lapic *apic)* just set SVI. */ if (unlikely (vcpu->arch.apicv_active)) - kvm_x86_ops.hwapic_isr_update (vcpu, vec); +static_call (kvm_x86_hwapic_isr_update)(vcpu, vec);else { ++apic->isr_count;BUG_ON (apic->isr_count >MAX_APIC_VECTOR );
@@ -563,7 +563,7 @@ static inline void apic_clear_isr(int vec, struct kvm_lapic *apic)* and must be left alone. */ if (unlikely (vcpu->arch.apicv_active)) - kvm_x86_ops.hwapic_isr_update (vcpu, +static_call (kvm_x86_hwapic_isr_update)(vcpu,apic_find_highest_isr (apic));else { --apic->isr_count;
@@ -701,7 +701,7 @@ static int apic_has_interrupt_for_ppr(struct kvm_lapic *apic, u32 ppr) {int highest_irr;if (apic->vcpu->arch.apicv_active) - highest_irr = kvm_x86_ops.sync_pir_to_irr (apic->vcpu); + highest_irr =static_call (kvm_x86_sync_pir_to_irr)(apic->vcpu);else highest_irr =apic_find_highest_irr (apic);if (highest_irr ==-1 || (highest_irr &0xF0 ) <= ppr)
@@ -1090,7 +1090,7 @@ static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, apic->regs + APIC_TMR); } -if (kvm_x86_ops.deliver_posted_interrupt (vcpu, vector)) { +if (static_call (kvm_x86_deliver_posted_interrupt)(vcpu, vector)) {kvm_lapic_set_irr (vector, apic);kvm_make_request (KVM_REQ_EVENT , vcpu);kvm_vcpu_kick (vcpu);
@@ -1814,7 +1814,7 @@ static void cancel_hv_timer(struct kvm_lapic *apic) {WARN_ON (preemptible ());WARN_ON (!apic->lapic_timer.hv_timer_in_use); - kvm_x86_ops.cancel_hv_timer (apic->vcpu); +static_call (kvm_x86_cancel_hv_timer)(apic->vcpu); apic->lapic_timer.hv_timer_in_use = false; }
@@ -1831,7 +1831,7 @@ static bool start_hv_timer(struct kvm_lapic *apic)if (!ktimer->tscdeadline)return false; -if (kvm_x86_ops.set_hv_timer (vcpu, ktimer->tscdeadline, &expired)) +if (static_call (kvm_x86_set_hv_timer)(vcpu, ktimer->tscdeadline, &expired))return false; ktimer->hv_timer_in_use = true;
@@ -2261,7 +2261,7 @@ void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value)kvm_apic_set_x2apic_id (apic, vcpu->vcpu_id);if ((old_value ^ value) & (MSR_IA32_APICBASE_ENABLE |X2APIC_ENABLE )) - kvm_x86_ops.set_virtual_apic_mode (vcpu); +static_call (kvm_x86_set_virtual_apic_mode)(vcpu); apic->base_address = apic->vcpu->arch.apic_base &MSR_IA32_APICBASE_BASE ;
@@ -2338,9 +2338,9 @@ void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event) vcpu->arch.pv_eoi.msr_val =0 ;apic_update_ppr (apic);if (vcpu->arch.apicv_active) { - kvm_x86_ops.apicv_post_state_restore (vcpu); - kvm_x86_ops.hwapic_irr_update (vcpu,-1 ); - kvm_x86_ops.hwapic_isr_update (vcpu,-1 ); +static_call (kvm_x86_apicv_post_state_restore)(vcpu); +static_call (kvm_x86_hwapic_irr_update)(vcpu,-1 ); +static_call (kvm_x86_hwapic_isr_update)(vcpu,-1 ); } vcpu->arch.apic_arb_prio =0 ;
@@ -2601,10 +2601,10 @@ int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s)kvm_apic_update_apicv (vcpu); apic->highest_isr_cache =-1 ;if (vcpu->arch.apicv_active) { - kvm_x86_ops.apicv_post_state_restore (vcpu); - kvm_x86_ops.hwapic_irr_update (vcpu, +static_call (kvm_x86_apicv_post_state_restore)(vcpu); +static_call (kvm_x86_hwapic_irr_update)(vcpu,apic_find_highest_irr (apic)); - kvm_x86_ops.hwapic_isr_update (vcpu, +static_call (kvm_x86_hwapic_isr_update)(vcpu,apic_find_highest_isr (apic)); }kvm_make_request (KVM_REQ_EVENT , vcpu);
arch/x86/kvm/mmu.h +3 -3
@@ -95,7 +95,7 @@ static inline void kvm_mmu_load_pgd(struct kvm_vcpu *vcpu)if (!VALID_PAGE (root_hpa))return ; - kvm_x86_ops.load_mmu_pgd (vcpu, root_hpa |kvm_get_active_pcid (vcpu), +static_call (kvm_x86_load_mmu_pgd)(vcpu, root_hpa |kvm_get_active_pcid (vcpu), vcpu->arch.mmu->shadow_root_level); }
@@ -167,8 +167,8 @@ static inline u8 permission_fault(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, unsigned pte_access, unsigned pte_pkey, unsigned pfec) { - int cpl = kvm_x86_ops.get_cpl(vcpu); -unsigned long rflags = kvm_x86_ops.get_rflags (vcpu); + int cpl = static_call(kvm_x86_get_cpl)(vcpu); +unsigned long rflags =static_call (kvm_x86_get_rflags)(vcpu);/* * If CPL < 3, SMAP prevention are disabled if EFLAGS.AC = 1.
arch/x86/kvm/mmu/mmu.c +6 -6
@@ -190,7 +190,7 @@ static void kvm_flush_remote_tlbs_with_range(struct kvm *kvm,int ret = -ENOTSUPP ;if (range && kvm_x86_ops.tlb_remote_flush_with_range) - ret = kvm_x86_ops.tlb_remote_flush_with_range (kvm, range); + ret =static_call (kvm_x86_tlb_remote_flush_with_range)(kvm, range);if (ret)kvm_flush_remote_tlbs (kvm);
@@ -1283,7 +1283,7 @@ void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, gfn_t gfn_offset, unsigned long mask) {if (kvm_x86_ops.enable_log_dirty_pt_masked) - kvm_x86_ops.enable_log_dirty_pt_masked (kvm, slot, gfn_offset, +static_call (kvm_x86_enable_log_dirty_pt_masked)(kvm, slot, gfn_offset, mask);else kvm_mmu_write_protect_pt_masked (kvm, slot, gfn_offset, mask);
@@ -1292,7 +1292,7 @@ void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,int kvm_cpu_dirty_log_size (void ) {if (kvm_x86_ops.cpu_dirty_log_size) -return kvm_x86_ops.cpu_dirty_log_size (); +return static_call (kvm_x86_cpu_dirty_log_size)();return 0 ; }
@@ -4804,7 +4804,7 @@ int kvm_mmu_load(struct kvm_vcpu *vcpu)if (r) goto out;kvm_mmu_load_pgd (vcpu); - kvm_x86_ops.tlb_flush_current (vcpu); +static_call (kvm_x86_tlb_flush_current)(vcpu); out:return r; }
@@ -5118,7 +5118,7 @@ void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,if (is_noncanonical_address (gva, vcpu))return ; - kvm_x86_ops.tlb_flush_gva (vcpu, gva); +static_call (kvm_x86_tlb_flush_gva)(vcpu, gva); }if (!mmu->invlpg)
@@ -5175,7 +5175,7 @@ void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid) }if (tlb_flush) - kvm_x86_ops.tlb_flush_gva (vcpu, gva); +static_call (kvm_x86_tlb_flush_gva)(vcpu, gva); ++vcpu->stat.invlpg;
arch/x86/kvm/mmu/spte.c +1 -1
@@ -120,7 +120,7 @@ int make_spte(struct kvm_vcpu *vcpu, unsigned int pte_access, int level,if (level >PG_LEVEL_4K ) spte |=PT_PAGE_SIZE_MASK ;if (tdp_enabled) - spte |= kvm_x86_ops.get_mt_mask (vcpu, gfn, + spte |=static_call (kvm_x86_get_mt_mask)(vcpu, gfn,kvm_is_mmio_pfn (pfn));if (host_writable)
arch/x86/kvm/pmu.c +1 -1
@@ -373,7 +373,7 @@ int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)return 1 ;if (!(kvm_read_cr4 (vcpu) &X86_CR4_PCE ) && - (kvm_x86_ops.get_cpl (vcpu) !=0 ) && + (static_call (kvm_x86_get_cpl)(vcpu) !=0 ) && (kvm_read_cr0 (vcpu) &X86_CR0_PE ))return 1 ;
arch/x86/kvm/trace.h +2 -2
@@ -256,7 +256,7 @@ TRACE_EVENT(name, \ __entry->guest_rip =kvm_rip_read (vcpu); \ __entry->isa = isa; \ __entry->vcpu_id = vcpu->vcpu_id; \ - kvm_x86_ops.get_exit_info (vcpu, &__entry->info1, \ +static_call (kvm_x86_get_exit_info)(vcpu, &__entry->info1, \ &__entry->info2, \ &__entry->intr_info, \ &__entry->error_code); \
@@ -738,7 +738,7 @@ TRACE_EVENT(kvm_emulate_insn, ),TP_fast_assign ( - __entry->csbase = kvm_x86_ops.get_segment_base(vcpu, VCPU_SREG_CS); + __entry->csbase = static_call(kvm_x86_get_segment_base)(vcpu, VCPU_SREG_CS); __entry->len = vcpu->arch.emulate_ctxt->fetch.ptr - vcpu->arch.emulate_ctxt->fetch.data; __entry->rip = vcpu->arch.emulate_ctxt->_eip - __entry->len;
arch/x86/kvm/x86.c +147 -147
1,084 lines not on this page · raw message
arch/x86/kvm/x86.h +3 -3
@@ -98,7 +98,7 @@ static inline bool is_64_bit_mode(struct kvm_vcpu *vcpu)if (!is_long_mode (vcpu))return false; - kvm_x86_ops.get_cs_db_l_bits (vcpu, &cs_db, &cs_l); +static_call (kvm_x86_get_cs_db_l_bits)(vcpu, &cs_db, &cs_l);return cs_l; }
@@ -129,7 +129,7 @@ static inline bool mmu_is_nested(struct kvm_vcpu *vcpu)static inline void kvm_vcpu_flush_tlb_current (struct kvm_vcpu *vcpu) { ++vcpu->stat.tlb_flush; - kvm_x86_ops.tlb_flush_current (vcpu); +static_call (kvm_x86_tlb_flush_current)(vcpu); }static inline int is_pae (struct kvm_vcpu *vcpu)
@@ -244,7 +244,7 @@ static inline bool kvm_check_has_quirk(struct kvm *kvm, u64 quirk)static inline bool kvm_vcpu_latch_init (struct kvm_vcpu *vcpu) { -return is_smm (vcpu) || kvm_x86_ops.apic_init_signal_blocked (vcpu); +return is_smm (vcpu) ||static_call (kvm_x86_apic_init_signal_blocked)(vcpu); }void kvm_inject_realmode_interrupt (struct kvm_vcpu *vcpu,int irq,int inc_eip);
Signature · 1 line
2.7.4
Paolo Bonzini
maintainerQuoted 13 lines · Jason Baron
Hi,
Convert
kvm_x86_opsto usestatic_call. Shows good performance gains for cpuid loop micro-benchmark (resulsts included in patch 2).Thanks,
-Jason
Jason Baron (2):
KVM: x86: introduce definitions to support static calls forkvm_x86_ops
KVM: x86: use static calls to reducekvm_x86_opsoverhead
Quoted 15 lines
arch/x86/include/asm/kvm_host.h | 71 +++++++++- arch/x86/kvm/cpuid.c | 2 +- arch/x86/kvm/hyperv.c | 4 +- arch/x86/kvm/irq.c | 2 +- arch/x86/kvm/kvm_cache_regs.h | 10 +- arch/x86/kvm/lapic.c | 28 ++-- arch/x86/kvm/mmu.h | 6 +- arch/x86/kvm/mmu/mmu.c | 12 +- arch/x86/kvm/mmu/spte.c | 2 +- arch/x86/kvm/pmu.c | 2 +- arch/x86/kvm/trace.h | 4 +- arch/x86/kvm/x86.c | 299 ++++++++++++++++++++-------------------- arch/x86/kvm/x86.h | 6 +- 13 files changed, 259 insertions(+), 189 deletions(-)
Thank you thank you thank you! :)
(Except that now I have to find another task for a new KVM developer, but that's not a big deal).
Paolo