Use static_call for kvm_x86_ops
Subsystems (5)
Jason Baron
maintainerv2 cover Use static_call for kvm_x86_ops
Hi,
Convert kvm_x86_ops to use static_call. Shows good performance gains for cpuid loop micro-benchmark (results in patch 3/3).
Thanks,
-Jason
Changes from v1:
-Introduce kvm-x86-ops header with eye towards using this to define
svm_x86_ops and vmx_x86_ops in follow on patches (Paolo, Sean)
-add new patch (1/3), that adds a vmx/svm prefix to help facilitate
svm_x86_ops and vmx_x86_ops future conversions.
-added amd perf numbres to description of patch 3/3
Jason Baron (3):
KVM: X86: append vmx/svm prefix to additional kvm_x86_ops functions
KVM: x86: introduce definitions to support static calls for kvm_x86_ops
KVM: x86: use static calls to reduce kvm_x86_ops overhead
arch/x86/include/asm/kvm-x86-ops.h | 127 +++++++++++++++ arch/x86/include/asm/kvm_host.h | 21 ++- arch/x86/kvm/cpuid.c | 2 +- arch/x86/kvm/hyperv.c | 4 +- arch/x86/kvm/irq.c | 3 +- arch/x86/kvm/kvm_cache_regs.h | 10 +- arch/x86/kvm/lapic.c | 30 ++-- arch/x86/kvm/mmu.h | 6 +- arch/x86/kvm/mmu/mmu.c | 15 +- arch/x86/kvm/mmu/spte.c | 2 +- arch/x86/kvm/pmu.c | 2 +- arch/x86/kvm/svm/svm.c | 20 +-- arch/x86/kvm/trace.h | 4 +- arch/x86/kvm/vmx/nested.c | 2 +- arch/x86/kvm/vmx/vmx.c | 30 ++-- arch/x86/kvm/vmx/vmx.h | 2 +- arch/x86/kvm/x86.c | 307 +++++++++++++++++++------------------ arch/x86/kvm/x86.h | 6 +- 18 files changed, 369 insertions(+), 224 deletions(-) create mode 100644 arch/x86/include/asm/kvm-x86-ops.h
Signature · 1 line
2.7.4
Jason Baron
maintainerPATCH v2 1/3 KVM: X86: append vmx/svm prefix to additional kvm_x86_ops functions
A subsequent patch introduces macros in preparation for simplifying the definition for vmx_x86_ops and svm_x86_ops. Making the naming more uniform expands the coverage of the macros. Add vmx/svm prefix to the following
functions: update_exception_bitmap(), enable_nmi_window(), enable_irq_window(), update_cr8_intercept and enable_smi_window().
--- arch/x86/kvm/svm/svm.c | 20 ++++++++++---------- arch/x86/kvm/vmx/nested.c | 2 +- arch/x86/kvm/vmx/vmx.c | 30 +++++++++++++++--------------- arch/x86/kvm/vmx/vmx.h | 2 +- 4 files changed, 27 insertions(+), 27 deletions(-)
arch/x86/kvm/svm/svm.c +10 -10
@@ -1810,7 +1810,7 @@ static void svm_set_segment(struct kvm_vcpu *vcpu,vmcb_mark_dirty (svm->vmcb,VMCB_SEG ); } -static void update_exception_bitmap (struct kvm_vcpu *vcpu) +static void svm_update_exception_bitmap (struct kvm_vcpu *vcpu) {struct vcpu_svm *svm =to_svm (vcpu);
@@ -3349,7 +3349,7 @@ static void svm_set_irq(struct kvm_vcpu *vcpu)SVM_EVTINJ_VALID |SVM_EVTINJ_TYPE_INTR ; } -static void update_cr8_intercept (struct kvm_vcpu *vcpu,int tpr,int irr) +static void svm_update_cr8_intercept (struct kvm_vcpu *vcpu,int tpr,int irr) {struct vcpu_svm *svm =to_svm (vcpu);
@@ -3474,7 +3474,7 @@ static int svm_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection)return !svm_interrupt_blocked (vcpu); } -static void enable_irq_window (struct kvm_vcpu *vcpu) +static void svm_enable_irq_window (struct kvm_vcpu *vcpu) {struct vcpu_svm *svm =to_svm (vcpu);
@@ -3498,7 +3498,7 @@ static void enable_irq_window(struct kvm_vcpu *vcpu) } } -static void enable_nmi_window (struct kvm_vcpu *vcpu) +static void svm_enable_nmi_window (struct kvm_vcpu *vcpu) {struct vcpu_svm *svm =to_svm (vcpu);
@@ -4280,7 +4280,7 @@ static int svm_pre_leave_smm(struct kvm_vcpu *vcpu, const char *smstate)return ret; } -static void enable_smi_window (struct kvm_vcpu *vcpu) +static void svm_enable_smi_window (struct kvm_vcpu *vcpu) {struct vcpu_svm *svm =to_svm (vcpu);
@@ -4426,7 +4426,7 @@ static struct kvm_x86_ops svm_x86_ops __initdata = { .vcpu_blocking = svm_vcpu_blocking, .vcpu_unblocking = svm_vcpu_unblocking, - .update_exception_bitmap = update_exception_bitmap, + .update_exception_bitmap = svm_update_exception_bitmap, .get_msr_feature = svm_get_msr_feature, .get_msr = svm_get_msr, .set_msr = svm_set_msr,
@@ -4469,9 +4469,9 @@ static struct kvm_x86_ops svm_x86_ops __initdata = { .nmi_allowed = svm_nmi_allowed, .get_nmi_mask = svm_get_nmi_mask, .set_nmi_mask = svm_set_nmi_mask, - .enable_nmi_window = enable_nmi_window, - .enable_irq_window = enable_irq_window, - .update_cr8_intercept = update_cr8_intercept, + .enable_nmi_window = svm_enable_nmi_window, + .enable_irq_window = svm_enable_irq_window, + .update_cr8_intercept = svm_update_cr8_intercept, .set_virtual_apic_mode = svm_set_virtual_apic_mode, .refresh_apicv_exec_ctrl = svm_refresh_apicv_exec_ctrl, .check_apicv_inhibit_reasons = svm_check_apicv_inhibit_reasons,
@@ -4514,7 +4514,7 @@ static struct kvm_x86_ops svm_x86_ops __initdata = { .smi_allowed = svm_smi_allowed, .pre_enter_smm = svm_pre_enter_smm, .pre_leave_smm = svm_pre_leave_smm, - .enable_smi_window = enable_smi_window, + .enable_smi_window = svm_enable_smi_window, .mem_enc_op = svm_mem_enc_op, .mem_enc_reg_region = svm_register_enc_region,
arch/x86/kvm/vmx/nested.c +1 -1
@@ -2532,7 +2532,7 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,* bitwise-or of what L1 wants to trap for L2, and what we want to * trap. Note that CR0.TS also needs updating - we do this later. */ -update_exception_bitmap (vcpu); +vmx_update_exception_bitmap (vcpu); vcpu->arch.cr0_guest_owned_bits &= ~vmcs12->cr0_guest_host_mask;vmcs_writel (CR0_GUEST_HOST_MASK , ~vcpu->arch.cr0_guest_owned_bits);
arch/x86/kvm/vmx/vmx.c +15 -15
@@ -806,7 +806,7 @@ static u32 vmx_read_guest_seg_ar(struct vcpu_vmx *vmx, unsigned seg)return *p; } -void update_exception_bitmap (struct kvm_vcpu *vcpu) +void vmx_update_exception_bitmap (struct kvm_vcpu *vcpu) {u32 eb;
@@ -2739,7 +2739,7 @@ static void enter_pmode(struct kvm_vcpu *vcpu)vmcs_writel (GUEST_CR4 , (vmcs_readl (GUEST_CR4 ) & ~X86_CR4_VME ) | (vmcs_readl (CR4_READ_SHADOW ) &X86_CR4_VME )); -update_exception_bitmap (vcpu); +vmx_update_exception_bitmap (vcpu);fix_pmode_seg (vcpu,VCPU_SREG_CS , &vmx->rmode.segs[VCPU_SREG_CS ]);fix_pmode_seg (vcpu,VCPU_SREG_SS , &vmx->rmode.segs[VCPU_SREG_SS ]);
@@ -2819,7 +2819,7 @@ static void enter_rmode(struct kvm_vcpu *vcpu)vmcs_writel (GUEST_RFLAGS , flags);vmcs_writel (GUEST_CR4 ,vmcs_readl (GUEST_CR4 ) |X86_CR4_VME ); -update_exception_bitmap (vcpu); +vmx_update_exception_bitmap (vcpu);fix_rmode_seg (VCPU_SREG_SS , &vmx->rmode.segs[VCPU_SREG_SS ]);fix_rmode_seg (VCPU_SREG_CS , &vmx->rmode.segs[VCPU_SREG_CS ]);
@@ -4467,23 +4467,23 @@ static void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)vmx_set_cr4 (vcpu,0 );vmx_set_efer (vcpu,0 ); -update_exception_bitmap (vcpu); +vmx_update_exception_bitmap (vcpu);vpid_sync_context (vmx->vpid);if (init_event)vmx_clear_hlt (vcpu); } -static void enable_irq_window (struct kvm_vcpu *vcpu) +static void vmx_enable_irq_window (struct kvm_vcpu *vcpu) {exec_controls_setbit (to_vmx (vcpu),CPU_BASED_INTR_WINDOW_EXITING ); } -static void enable_nmi_window (struct kvm_vcpu *vcpu) +static void vmx_enable_nmi_window (struct kvm_vcpu *vcpu) {if (!enable_vnmi ||vmcs_read32 (GUEST_INTERRUPTIBILITY_INFO ) &GUEST_INTR_STATE_STI ) { -enable_irq_window (vcpu); +vmx_enable_irq_window (vcpu);return ; }
@@ -6129,7 +6129,7 @@ static noinstr void vmx_l1d_flush(struct kvm_vcpu *vcpu) : "eax", "ebx", "ecx", "edx"); } -static void update_cr8_intercept (struct kvm_vcpu *vcpu,int tpr,int irr) +static void vmx_update_cr8_intercept (struct kvm_vcpu *vcpu,int tpr,int irr) {struct vmcs12 *vmcs12 =get_vmcs12 (vcpu);int tpr_threshold;
@@ -7245,7 +7245,7 @@ static void vmx_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu)set_cr4_guest_host_mask (vmx);/* Refresh #PF interception to account for MAXPHYADDR changes. */ -update_exception_bitmap (vcpu); +vmx_update_exception_bitmap (vcpu); }static __init void vmx_set_cpu_caps (void )
@@ -7535,7 +7535,7 @@ static int vmx_pre_leave_smm(struct kvm_vcpu *vcpu, const char *smstate)return 0 ; } -static void enable_smi_window (struct kvm_vcpu *vcpu) +static void vmx_enable_smi_window (struct kvm_vcpu *vcpu) {/* RSM will cause a vmexit anyway. */ }
@@ -7595,7 +7595,7 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = { .vcpu_load = vmx_vcpu_load, .vcpu_put = vmx_vcpu_put, - .update_exception_bitmap = update_exception_bitmap, + .update_exception_bitmap = vmx_update_exception_bitmap, .get_msr_feature = vmx_get_msr_feature, .get_msr = vmx_get_msr, .set_msr = vmx_set_msr,
@@ -7638,9 +7638,9 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = { .nmi_allowed = vmx_nmi_allowed, .get_nmi_mask = vmx_get_nmi_mask, .set_nmi_mask = vmx_set_nmi_mask, - .enable_nmi_window = enable_nmi_window, - .enable_irq_window = enable_irq_window, - .update_cr8_intercept = update_cr8_intercept, + .enable_nmi_window = vmx_enable_nmi_window, + .enable_irq_window = vmx_enable_irq_window, + .update_cr8_intercept = vmx_update_cr8_intercept, .set_virtual_apic_mode = vmx_set_virtual_apic_mode, .set_apic_access_page_addr = vmx_set_apic_access_page_addr, .refresh_apicv_exec_ctrl = vmx_refresh_apicv_exec_ctrl,
@@ -7698,7 +7698,7 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = { .smi_allowed = vmx_smi_allowed, .pre_enter_smm = vmx_pre_enter_smm, .pre_leave_smm = vmx_pre_leave_smm, - .enable_smi_window = enable_smi_window, + .enable_smi_window = vmx_enable_smi_window, .can_emulate_instruction = vmx_can_emulate_instruction, .apic_init_signal_blocked = vmx_apic_init_signal_blocked,
arch/x86/kvm/vmx/vmx.h +1 -1
@@ -329,7 +329,7 @@ void vmx_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg);u64 construct_eptp (struct kvm_vcpu *vcpu,unsigned long root_hpa,int root_level); -void update_exception_bitmap (struct kvm_vcpu *vcpu); +void vmx_update_exception_bitmap (struct kvm_vcpu *vcpu);void vmx_update_msr_bitmap (struct kvm_vcpu *vcpu);bool vmx_nmi_blocked (struct kvm_vcpu *vcpu);bool vmx_interrupt_blocked (struct kvm_vcpu *vcpu);
Signature · 1 line
2.7.4
Peter Zijlstra
maintainerRe: [PATCH v2 1/3] KVM: X86: append vmx/svm prefix to additional kvm_x86_ops functions
arch/x86/kvm/svm/svm.c static void svm_set_segment(struct kvm_vcpu *vcpu, · line 1813
Quoted 2 lines · Jason Baron
arch/x86/kvm/svm/svm.c +1
-1-static void update_exception_bitmap (struct kvm_vcpu *vcpu) +static void svm_update_exception_bitmap (struct kvm_vcpu *vcpu)
Just to be a total pendant: s/append/Prepend/ on $Subject
Jason Baron
maintainerRe: [PATCH v2 1/3] KVM: X86: append vmx/svm prefix to additional kvm_x86_ops functions
Quoted 8 lines · Peter Zijlstra
arch/x86/kvm/svm/svm.c static void svm_set_segment(struct kvm_vcpu *vcpu, · line 1813
Quoted 2 lines · Jason Baron
arch/x86/kvm/svm/svm.c +1
-1-static void update_exception_bitmap (struct kvm_vcpu *vcpu) +static void svm_update_exception_bitmap (struct kvm_vcpu *vcpu)Just to be a total pendant: s/append/Prepend/ on $Subject
Ha - I actually switched $subject to prepend and then switched it back because I thought I was being pedantic. But maybe not :)
Jason Baron
maintainerPATCH v2 2/3 KVM: x86: introduce definitions to support static calls for kvm_x86_ops
In reply to Jason BaronUse static calls to improve kvm_x86_ops performance. Introduce the definitions that will be used by a subsequent patch to actualize the savings. Add a new kvm-x86-ops.h header that can be used for the definition of static calls. This header is also intended to be used to simplify the defition of svm_kvm_ops and vmx_x86_ops.
Note that all functions in kvm_x86_ops are covered here except for 'pmu_ops' and 'nested ops'. I think they can be covered by static calls in a simlilar manner, but were omitted from this series to reduce scope and because I don't think they have as large of a performance impact.
--- arch/x86/include/asm/kvm-x86-ops.h | 127 +++++++++++++++++++++++++++++++++++++ arch/x86/include/asm/kvm_host.h | 13 ++++ arch/x86/kvm/x86.c | 9 +++ 3 files changed, 149 insertions(+) create mode 100644 arch/x86/include/asm/kvm-x86-ops.h
arch/x86/include/asm/kvm-x86-ops.h new file +127 -0
@@ -0,0 +1,127 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#if !defined(KVM_X86_OP ) || !defined(KVM_X86_OP_NULL ) +BUILD_BUG_ON(1) +#endif + +/* +* KVM_X86_OP() and KVM_X86_OP_NULL() are used to help generate +* "static_call()"s. They are also intended for use when defining +* the vmx/svm kvm_x86_ops. KVM_X86_OP() can be used for those +* functions that follow the [svm|vmx]_func_name convention. +* KVM_X86_OP_NULL() can leave a NULL definition for the +* case where there is no definition or a function name that +* doesn't match the typical naming convention is supplied. +*/ +KVM_X86_OP_NULL(hardware_enable) +KVM_X86_OP_NULL(hardware_disable) +KVM_X86_OP_NULL(hardware_unsetup) +KVM_X86_OP_NULL(cpu_has_accelerated_tpr) +KVM_X86_OP(has_emulated_msr) +KVM_X86_OP(vcpu_after_set_cpuid) +KVM_X86_OP(vm_init) +KVM_X86_OP_NULL(vm_destroy) +KVM_X86_OP(vcpu_create) +KVM_X86_OP(vcpu_free) +KVM_X86_OP(vcpu_reset) +KVM_X86_OP(prepare_guest_switch) +KVM_X86_OP(vcpu_load) +KVM_X86_OP(vcpu_put) +KVM_X86_OP(update_exception_bitmap) +KVM_X86_OP(get_msr) +KVM_X86_OP(set_msr) +KVM_X86_OP(get_segment_base) +KVM_X86_OP(get_segment) +KVM_X86_OP(get_cpl) +KVM_X86_OP(set_segment) +KVM_X86_OP_NULL(get_cs_db_l_bits) +KVM_X86_OP(set_cr0) +KVM_X86_OP(is_valid_cr4) +KVM_X86_OP(set_cr4) +KVM_X86_OP(set_efer) +KVM_X86_OP(get_idt) +KVM_X86_OP(set_idt) +KVM_X86_OP(get_gdt) +KVM_X86_OP(set_gdt) +KVM_X86_OP(sync_dirty_debug_regs) +KVM_X86_OP(set_dr7) +KVM_X86_OP(cache_reg) +KVM_X86_OP(get_rflags) +KVM_X86_OP(set_rflags) +KVM_X86_OP(tlb_flush_all) +KVM_X86_OP(tlb_flush_current) +KVM_X86_OP_NULL(tlb_remote_flush) +KVM_X86_OP_NULL(tlb_remote_flush_with_range) +KVM_X86_OP(tlb_flush_gva) +KVM_X86_OP(tlb_flush_guest) +KVM_X86_OP(run) +KVM_X86_OP_NULL(handle_exit) +KVM_X86_OP_NULL(skip_emulated_instruction) +KVM_X86_OP_NULL(update_emulated_instruction) +KVM_X86_OP(set_interrupt_shadow) +KVM_X86_OP(get_interrupt_shadow) +KVM_X86_OP(patch_hypercall) +KVM_X86_OP(set_irq) +KVM_X86_OP(set_nmi) +KVM_X86_OP(queue_exception) +KVM_X86_OP(cancel_injection) +KVM_X86_OP(interrupt_allowed) +KVM_X86_OP(nmi_allowed) +KVM_X86_OP(get_nmi_mask) +KVM_X86_OP(set_nmi_mask) +KVM_X86_OP(enable_nmi_window) +KVM_X86_OP(enable_irq_window) +KVM_X86_OP(update_cr8_intercept) +KVM_X86_OP(check_apicv_inhibit_reasons) +KVM_X86_OP_NULL(pre_update_apicv_exec_ctrl) +KVM_X86_OP(refresh_apicv_exec_ctrl) +KVM_X86_OP(hwapic_irr_update) +KVM_X86_OP(hwapic_isr_update) +KVM_X86_OP_NULL(guest_apic_has_interrupt) +KVM_X86_OP(load_eoi_exitmap) +KVM_X86_OP(set_virtual_apic_mode) +KVM_X86_OP_NULL(set_apic_access_page_addr) +KVM_X86_OP(deliver_posted_interrupt) +KVM_X86_OP_NULL(sync_pir_to_irr) +KVM_X86_OP(set_tss_addr) +KVM_X86_OP(set_identity_map_addr) +KVM_X86_OP(get_mt_mask) +KVM_X86_OP(load_mmu_pgd) +KVM_X86_OP_NULL(has_wbinvd_exit) +KVM_X86_OP(write_l1_tsc_offset) +KVM_X86_OP(get_exit_info) +KVM_X86_OP(check_intercept) +KVM_X86_OP(handle_exit_irqoff) +KVM_X86_OP_NULL(request_immediate_exit) +KVM_X86_OP(sched_in) +KVM_X86_OP_NULL(slot_enable_log_dirty) +KVM_X86_OP_NULL(slot_disable_log_dirty) +KVM_X86_OP_NULL(flush_log_dirty) +KVM_X86_OP_NULL(enable_log_dirty_pt_masked) +KVM_X86_OP_NULL(cpu_dirty_log_size) +KVM_X86_OP_NULL(pre_block) +KVM_X86_OP_NULL(post_block) +KVM_X86_OP_NULL(vcpu_blocking) +KVM_X86_OP_NULL(vcpu_unblocking) +KVM_X86_OP_NULL(update_pi_irte) +KVM_X86_OP_NULL(apicv_post_state_restore) +KVM_X86_OP_NULL(dy_apicv_has_pending_interrupt) +KVM_X86_OP_NULL(set_hv_timer) +KVM_X86_OP_NULL(cancel_hv_timer) +KVM_X86_OP(setup_mce) +KVM_X86_OP(smi_allowed) +KVM_X86_OP(pre_enter_smm) +KVM_X86_OP(pre_leave_smm) +KVM_X86_OP(enable_smi_window) +KVM_X86_OP_NULL(mem_enc_op) +KVM_X86_OP_NULL(mem_enc_reg_region) +KVM_X86_OP_NULL(mem_enc_unreg_region) +KVM_X86_OP(get_msr_feature) +KVM_X86_OP(can_emulate_instruction) +KVM_X86_OP(apic_init_signal_blocked) +KVM_X86_OP_NULL(enable_direct_tlbflush) +KVM_X86_OP_NULL(migrate_timers) +KVM_X86_OP(msr_filter_changed) +KVM_X86_OP_NULL(complete_emulated_msr) + +#undef KVM_X86_OP +#undef KVM_X86_OP_NULL
arch/x86/include/asm/kvm_host.h +13 -0
@@ -1326,6 +1326,19 @@ extern u64 __read_mostly host_efer;extern bool __read_mostly allow_smaller_maxphyaddr;extern struct kvm_x86_ops kvm_x86_ops; +#define KVM_X86_OP (func) \ + DECLARE_STATIC_CALL(kvm_x86_##func, *(((struct kvm_x86_ops *)0)->func)); +#define KVM_X86_OP_NULL KVM_X86_OP +#include <asm/kvm-x86-ops.h> + +static inline void kvm_ops_static_call_update (void ) +{ +#define KVM_X86_OP (func) \ + static_call_update(kvm_x86_##func, kvm_x86_ops.func); +#define KVM_X86_OP_NULL KVM_X86_OP +#include <asm/kvm-x86-ops.h> +} +#define __KVM_HAVE_ARCH_VM_ALLOCstatic inline struct kvm *kvm_arch_alloc_vm (void ) {
arch/x86/kvm/x86.c +9 -0
@@ -113,6 +113,15 @@ static int sync_regs(struct kvm_vcpu *vcpu);struct kvm_x86_ops kvm_x86_ops__read_mostly ;EXPORT_SYMBOL_GPL (kvm_x86_ops); +#define KVM_X86_OP (func) \ + DEFINE_STATIC_CALL_NULL(kvm_x86_##func, \ + *(((struct kvm_x86_ops *)0)->func)); +#define KVM_X86_OP_NULL KVM_X86_OP +#include <asm/kvm-x86-ops.h> +EXPORT_STATIC_CALL_GPL (kvm_x86_get_cs_db_l_bits); +EXPORT_STATIC_CALL_GPL (kvm_x86_cache_reg); +EXPORT_STATIC_CALL_GPL (kvm_x86_tlb_flush_current); +static bool __read_mostly ignore_msrs =0 ;module_param (ignore_msrs, bool,S_IRUGO |S_IWUSR );
Signature · 1 line
2.7.4
Peter Zijlstra
maintainerRe: [PATCH v2 2/3] KVM: x86: introduce definitions to support static calls for kvm_x86_ops
Quoted 6 lines · Jason Baron
arch/x86/kvm/x86.c +0
-0@@ -113,6 +113,15 @@ static int sync_regs(struct kvm_vcpu *vcpu);struct kvm_x86_ops kvm_x86_ops__read_mostly ;
arch/x86/kvm/x86.c static int sync_regs(struct kvm_vcpu *vcpu); · line 123
Quoted 10 lines
arch/x86/kvm/x86.c +8
-0EXPORT_SYMBOL_GPL (kvm_x86_ops); +#define KVM_X86_OP (func) \ + DEFINE_STATIC_CALL_NULL(kvm_x86_##func, \ + *(((struct kvm_x86_ops *)0)->func)); +#define KVM_X86_OP_NULL KVM_X86_OP +#include <asm/kvm-x86-ops.h> +EXPORT_STATIC_CALL_GPL (kvm_x86_get_cs_db_l_bits); +EXPORT_STATIC_CALL_GPL (kvm_x86_cache_reg); +EXPORT_STATIC_CALL_GPL (kvm_x86_tlb_flush_current);
Would something like:
https://lkml.kernel.org/r/[email protected]
Be useful? That way modules can call the static_call() but not change it.
Paolo Bonzini
maintainerRe: [PATCH v2 2/3] KVM: x86: introduce definitions to support static calls for kvm_x86_ops
Quoted 10 lines · Peter Zijlstra
arch/x86/kvm/x86.c static int sync_regs(struct kvm_vcpu *vcpu); · line 123
Quoted 8 lines
arch/x86/kvm/x86.c +8
-0+#define KVM_X86_OP (func) \ + DEFINE_STATIC_CALL_NULL(kvm_x86_##func, \ + *(((struct kvm_x86_ops *)0)->func)); +#define KVM_X86_OP_NULL KVM_X86_OP +#include <asm/kvm-x86-ops.h> +EXPORT_STATIC_CALL_GPL (kvm_x86_get_cs_db_l_bits); +EXPORT_STATIC_CALL_GPL (kvm_x86_cache_reg); +EXPORT_STATIC_CALL_GPL (kvm_x86_tlb_flush_current);Would something like:
Quoted 5 lines
https://lkml.kernel.org/r/[email protected]Be useful? That way modules can call the
static_call()but not change it.
Maybe not in these cases, but in general there may be cases where we later want to change the static_call (for example replacing jump labels with static_calls).
Paolo
Jason Baron
maintainerRe: [PATCH v2 2/3] KVM: x86: introduce definitions to support static calls for kvm_x86_ops
Quoted 14 lines · Paolo Bonzini
Quoted 13 lines · Peter Zijlstra
arch/x86/kvm/x86.c static int sync_regs(struct kvm_vcpu *vcpu); · line 123
Quoted 8 lines
arch/x86/kvm/x86.c +8
-0+#define KVM_X86_OP (func) \ + DEFINE_STATIC_CALL_NULL(kvm_x86_##func, \ + *(((struct kvm_x86_ops *)0)->func)); +#define KVM_X86_OP_NULL KVM_X86_OP +#include <asm/kvm-x86-ops.h> +EXPORT_STATIC_CALL_GPL (kvm_x86_get_cs_db_l_bits); +EXPORT_STATIC_CALL_GPL (kvm_x86_cache_reg); +EXPORT_STATIC_CALL_GPL (kvm_x86_tlb_flush_current);Would something like:
Quoted 9 lines
Quoted 3 lines
Be useful? That way modules can call the
static_call()but not change it.Maybe not in these cases, but in general there may be cases where we later want to change the
static_call(for example replacing jump labels withstatic_calls).Paolo
I tried this out but got:
ERROR: modpost: "__SCK__kvm_x86_cache_reg" [arch/x86/kvm/kvm-amd.ko] undefined!
ERROR: modpost: "__SCK__kvm_x86_tlb_flush_current" [arch/x86/kvm/kvm-intel.ko] undefined!
ERROR: modpost: "__SCK__kvm_x86_get_cs_db_l_bits" [arch/x86/kvm/kvm-intel.ko] undefined!
ERROR: modpost: "__SCK__kvm_x86_cache_reg" [arch/x86/kvm/kvm-intel.ko] undefined!
I'm a bit confused because we have:
#define __static_call(name) \
({ \
__ADDRESSABLE(STATIC_CALL_KEY(name)); \
&STATIC_CALL_TRAMP(name); \})
And so it looks to me like we need to still reference the key from the module code.
Thanks,
-Jason
Jason Baron
maintainerPATCH v2 3/3 KVM: x86: use static calls to reduce kvm_x86_ops overhead
In reply to Jason BaronConvert kvm_x86_ops to use static calls. Note that all kvm_x86_ops are covered here except for 'pmu_ops and 'nested ops'.
Here are some numbers running cpuid in a loop of 1 million calls averaged over 5 runs, measured in the vm (lower is better).
Intel Xeon 3000MHz:
|default |mitigations=off
-------------------------------------
vanilla |.671s |.486s static call|.573s(-15%)|.458s(-6%)
AMD EPYC 2500MHz:
|default |mitigations=off
-------------------------------------
vanilla |.710s |.609s static call|.664s(-6%) |.609s(0%)
--- arch/x86/include/asm/kvm_host.h | 8 +- arch/x86/kvm/cpuid.c | 2 +- arch/x86/kvm/hyperv.c | 4 +- arch/x86/kvm/irq.c | 3 +- arch/x86/kvm/kvm_cache_regs.h | 10 +- arch/x86/kvm/lapic.c | 30 ++-- arch/x86/kvm/mmu.h | 6 +- arch/x86/kvm/mmu/mmu.c | 15 +- arch/x86/kvm/mmu/spte.c | 2 +- arch/x86/kvm/pmu.c | 2 +- arch/x86/kvm/trace.h | 4 +- arch/x86/kvm/x86.c | 298 ++++++++++++++++++++-------------------- arch/x86/kvm/x86.h | 6 +- 13 files changed, 193 insertions(+), 197 deletions(-)
arch/x86/include/asm/kvm_host.h +3 -5
@@ -1350,7 +1350,7 @@ void kvm_arch_free_vm(struct kvm *kvm);static inline int kvm_arch_flush_remote_tlb (struct kvm *kvm) {if (kvm_x86_ops.tlb_remote_flush && - !kvm_x86_ops.tlb_remote_flush (kvm)) + !static_call (kvm_x86_tlb_remote_flush)(kvm))return 0 ;else return -ENOTSUPP ;
@@ -1740,14 +1740,12 @@ static inline bool kvm_irq_is_postable(struct kvm_lapic_irq *irq)static inline void kvm_arch_vcpu_blocking (struct kvm_vcpu *vcpu) { -if (kvm_x86_ops.vcpu_blocking) - kvm_x86_ops.vcpu_blocking (vcpu); +static_call_cond (kvm_x86_vcpu_blocking)(vcpu); }static inline void kvm_arch_vcpu_unblocking (struct kvm_vcpu *vcpu) { -if (kvm_x86_ops.vcpu_unblocking) - kvm_x86_ops.vcpu_unblocking (vcpu); +static_call_cond (kvm_x86_vcpu_unblocking)(vcpu); }static inline void kvm_arch_vcpu_block_finish (struct kvm_vcpu *vcpu) {}
arch/x86/kvm/cpuid.c +1 -1
@@ -182,7 +182,7 @@ static void kvm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) vcpu->arch.cr3_lm_rsvd_bits =rsvd_bits (cpuid_maxphyaddr (vcpu),63 );/* Invoke the vendor callback only after the above state is updated. */ - kvm_x86_ops.vcpu_after_set_cpuid (vcpu); +static_call (kvm_x86_vcpu_after_set_cpuid)(vcpu); }static int is_efer_nx (void )
arch/x86/kvm/hyperv.c +2 -2
@@ -1154,7 +1154,7 @@ static int kvm_hv_set_msr_pw(struct kvm_vcpu *vcpu, u32 msr, u64 data, addr =gfn_to_hva (kvm, gfn);if (kvm_is_error_hva (addr))return 1 ; - kvm_x86_ops.patch_hypercall (vcpu, instructions); +static_call (kvm_x86_patch_hypercall)(vcpu, instructions); ((unsigned char *)instructions)[3 ] =0xc3 ;/* ret */ if (__copy_to_user ((void __user *)addr, instructions,4 ))return 1 ;
@@ -1745,7 +1745,7 @@ int kvm_hv_hypercall(struct kvm_vcpu *vcpu)* hypercall generates UD from non zero cpl and real mode * per HYPER-V spec */ -if (kvm_x86_ops.get_cpl (vcpu) !=0 || !is_protmode (vcpu)) { +if (static_call (kvm_x86_get_cpl)(vcpu) !=0 || !is_protmode (vcpu)) {kvm_queue_exception (vcpu,UD_VECTOR );return 1 ; }
arch/x86/kvm/irq.c +1 -2
@@ -143,8 +143,7 @@ void __kvm_migrate_timers(struct kvm_vcpu *vcpu) {__kvm_migrate_apic_timer (vcpu);__kvm_migrate_pit_timer (vcpu); -if (kvm_x86_ops.migrate_timers) - kvm_x86_ops.migrate_timers (vcpu); +static_call_cond (kvm_x86_migrate_timers)(vcpu); }bool kvm_arch_irqfd_allowed (struct kvm *kvm,struct kvm_irqfd *args)
arch/x86/kvm/kvm_cache_regs.h +5 -5
@@ -69,7 +69,7 @@ static inline unsigned long kvm_register_read(struct kvm_vcpu *vcpu, int reg)return 0 ;if (!kvm_register_is_available (vcpu, reg)) - kvm_x86_ops.cache_reg (vcpu, reg); +static_call (kvm_x86_cache_reg)(vcpu, reg);return vcpu->arch.regs[reg]; }
@@ -109,7 +109,7 @@ static inline u64 kvm_pdptr_read(struct kvm_vcpu *vcpu, int index)might_sleep ();/* on svm */ if (!kvm_register_is_available (vcpu,VCPU_EXREG_PDPTR )) - kvm_x86_ops.cache_reg (vcpu,VCPU_EXREG_PDPTR ); +static_call (kvm_x86_cache_reg)(vcpu,VCPU_EXREG_PDPTR );return vcpu->arch.walk_mmu->pdptrs[index]; }
@@ -119,7 +119,7 @@ static inline ulong kvm_read_cr0_bits(struct kvm_vcpu *vcpu, ulong mask)ulong tmask = mask &KVM_POSSIBLE_CR0_GUEST_BITS ;if ((tmask & vcpu->arch.cr0_guest_owned_bits) && !kvm_register_is_available (vcpu,VCPU_EXREG_CR0 )) - kvm_x86_ops.cache_reg (vcpu,VCPU_EXREG_CR0 ); +static_call (kvm_x86_cache_reg)(vcpu,VCPU_EXREG_CR0 );return vcpu->arch.cr0 & mask; }
@@ -133,14 +133,14 @@ static inline ulong kvm_read_cr4_bits(struct kvm_vcpu *vcpu, ulong mask)ulong tmask = mask &KVM_POSSIBLE_CR4_GUEST_BITS ;if ((tmask & vcpu->arch.cr4_guest_owned_bits) && !kvm_register_is_available (vcpu,VCPU_EXREG_CR4 )) - kvm_x86_ops.cache_reg (vcpu,VCPU_EXREG_CR4 ); +static_call (kvm_x86_cache_reg)(vcpu,VCPU_EXREG_CR4 );return vcpu->arch.cr4 & mask; }static inline ulong kvm_read_cr3 (struct kvm_vcpu *vcpu) {if (!kvm_register_is_available (vcpu,VCPU_EXREG_CR3 )) - kvm_x86_ops.cache_reg (vcpu,VCPU_EXREG_CR3 ); +static_call (kvm_x86_cache_reg)(vcpu,VCPU_EXREG_CR3 );return vcpu->arch.cr3; }
arch/x86/kvm/lapic.c +15 -15
@@ -484,7 +484,7 @@ static inline void apic_clear_irr(int vec, struct kvm_lapic *apic)if (unlikely (vcpu->arch.apicv_active)) {/* need to update RVI */ kvm_lapic_clear_vector (vec, apic->regs +APIC_IRR ); - kvm_x86_ops.hwapic_irr_update (vcpu, +static_call (kvm_x86_hwapic_irr_update)(vcpu,apic_find_highest_irr (apic)); }else { apic->irr_pending = false;
@@ -515,7 +515,7 @@ static inline void apic_set_isr(int vec, struct kvm_lapic *apic)* just set SVI. */ if (unlikely (vcpu->arch.apicv_active)) - kvm_x86_ops.hwapic_isr_update (vcpu, vec); +static_call (kvm_x86_hwapic_isr_update)(vcpu, vec);else { ++apic->isr_count;BUG_ON (apic->isr_count >MAX_APIC_VECTOR );
@@ -563,8 +563,8 @@ static inline void apic_clear_isr(int vec, struct kvm_lapic *apic)* and must be left alone. */ if (unlikely (vcpu->arch.apicv_active)) - kvm_x86_ops.hwapic_isr_update (vcpu, -apic_find_highest_isr (apic)); +static_call (kvm_x86_hwapic_isr_update)(vcpu, +apic_find_highest_isr (apic));else { --apic->isr_count;BUG_ON (apic->isr_count <0 );
@@ -701,7 +701,7 @@ static int apic_has_interrupt_for_ppr(struct kvm_lapic *apic, u32 ppr) {int highest_irr;if (apic->vcpu->arch.apicv_active) - highest_irr = kvm_x86_ops.sync_pir_to_irr (apic->vcpu); + highest_irr =static_call (kvm_x86_sync_pir_to_irr)(apic->vcpu);else highest_irr =apic_find_highest_irr (apic);if (highest_irr ==-1 || (highest_irr &0xF0 ) <= ppr)
@@ -1090,7 +1090,7 @@ static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, apic->regs + APIC_TMR); } -if (kvm_x86_ops.deliver_posted_interrupt (vcpu, vector)) { +if (static_call (kvm_x86_deliver_posted_interrupt)(vcpu, vector)) {kvm_lapic_set_irr (vector, apic);kvm_make_request (KVM_REQ_EVENT , vcpu);kvm_vcpu_kick (vcpu);
@@ -1814,7 +1814,7 @@ static void cancel_hv_timer(struct kvm_lapic *apic) {WARN_ON (preemptible ());WARN_ON (!apic->lapic_timer.hv_timer_in_use); - kvm_x86_ops.cancel_hv_timer (apic->vcpu); +static_call (kvm_x86_cancel_hv_timer)(apic->vcpu); apic->lapic_timer.hv_timer_in_use = false; }
@@ -1831,7 +1831,7 @@ static bool start_hv_timer(struct kvm_lapic *apic)if (!ktimer->tscdeadline)return false; -if (kvm_x86_ops.set_hv_timer (vcpu, ktimer->tscdeadline, &expired)) +if (static_call (kvm_x86_set_hv_timer)(vcpu, ktimer->tscdeadline, &expired))return false; ktimer->hv_timer_in_use = true;
@@ -2261,7 +2261,7 @@ void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value)kvm_apic_set_x2apic_id (apic, vcpu->vcpu_id);if ((old_value ^ value) & (MSR_IA32_APICBASE_ENABLE |X2APIC_ENABLE )) - kvm_x86_ops.set_virtual_apic_mode (vcpu); +static_call (kvm_x86_set_virtual_apic_mode)(vcpu); apic->base_address = apic->vcpu->arch.apic_base &MSR_IA32_APICBASE_BASE ;
@@ -2338,9 +2338,9 @@ void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event) vcpu->arch.pv_eoi.msr_val =0 ;apic_update_ppr (apic);if (vcpu->arch.apicv_active) { - kvm_x86_ops.apicv_post_state_restore (vcpu); - kvm_x86_ops.hwapic_irr_update (vcpu,-1 ); - kvm_x86_ops.hwapic_isr_update (vcpu,-1 ); +static_call (kvm_x86_apicv_post_state_restore)(vcpu); +static_call (kvm_x86_hwapic_irr_update)(vcpu,-1 ); +static_call (kvm_x86_hwapic_isr_update)(vcpu,-1 ); } vcpu->arch.apic_arb_prio =0 ;
@@ -2601,10 +2601,10 @@ int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s)kvm_apic_update_apicv (vcpu); apic->highest_isr_cache =-1 ;if (vcpu->arch.apicv_active) { - kvm_x86_ops.apicv_post_state_restore (vcpu); - kvm_x86_ops.hwapic_irr_update (vcpu, +static_call (kvm_x86_apicv_post_state_restore)(vcpu); +static_call (kvm_x86_hwapic_irr_update)(vcpu,apic_find_highest_irr (apic)); - kvm_x86_ops.hwapic_isr_update (vcpu, +static_call (kvm_x86_hwapic_isr_update)(vcpu,apic_find_highest_isr (apic)); }kvm_make_request (KVM_REQ_EVENT , vcpu);
arch/x86/kvm/mmu.h +3 -3
@@ -95,7 +95,7 @@ static inline void kvm_mmu_load_pgd(struct kvm_vcpu *vcpu)if (!VALID_PAGE (root_hpa))return ; - kvm_x86_ops.load_mmu_pgd (vcpu, root_hpa |kvm_get_active_pcid (vcpu), +static_call (kvm_x86_load_mmu_pgd)(vcpu, root_hpa |kvm_get_active_pcid (vcpu), vcpu->arch.mmu->shadow_root_level); }
@@ -167,8 +167,8 @@ static inline u8 permission_fault(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, unsigned pte_access, unsigned pte_pkey, unsigned pfec) { - int cpl = kvm_x86_ops.get_cpl(vcpu); -unsigned long rflags = kvm_x86_ops.get_rflags (vcpu); + int cpl = static_call(kvm_x86_get_cpl)(vcpu); +unsigned long rflags =static_call (kvm_x86_get_rflags)(vcpu);/* * If CPL < 3, SMAP prevention are disabled if EFLAGS.AC = 1.
arch/x86/kvm/mmu/mmu.c +8 -7
@@ -190,7 +190,7 @@ static void kvm_flush_remote_tlbs_with_range(struct kvm *kvm,int ret = -ENOTSUPP ;if (range && kvm_x86_ops.tlb_remote_flush_with_range) - ret = kvm_x86_ops.tlb_remote_flush_with_range (kvm, range); + ret =static_call (kvm_x86_tlb_remote_flush_with_range)(kvm, range);if (ret)kvm_flush_remote_tlbs (kvm);
@@ -1283,8 +1283,9 @@ void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, gfn_t gfn_offset, unsigned long mask) {if (kvm_x86_ops.enable_log_dirty_pt_masked) - kvm_x86_ops.enable_log_dirty_pt_masked (kvm, slot, gfn_offset, - mask); +static_call (kvm_x86_enable_log_dirty_pt_masked)(kvm, slot, + gfn_offset, + mask);else kvm_mmu_write_protect_pt_masked (kvm, slot, gfn_offset, mask); }
@@ -1292,7 +1293,7 @@ void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,int kvm_cpu_dirty_log_size (void ) {if (kvm_x86_ops.cpu_dirty_log_size) -return kvm_x86_ops.cpu_dirty_log_size (); +return static_call (kvm_x86_cpu_dirty_log_size)();return 0 ; }
@@ -4804,7 +4805,7 @@ int kvm_mmu_load(struct kvm_vcpu *vcpu)if (r) goto out;kvm_mmu_load_pgd (vcpu); - kvm_x86_ops.tlb_flush_current (vcpu); +static_call (kvm_x86_tlb_flush_current)(vcpu); out:return r; }
@@ -5118,7 +5119,7 @@ void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,if (is_noncanonical_address (gva, vcpu))return ; - kvm_x86_ops.tlb_flush_gva (vcpu, gva); +static_call (kvm_x86_tlb_flush_gva)(vcpu, gva); }if (!mmu->invlpg)
@@ -5175,7 +5176,7 @@ void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid) }if (tlb_flush) - kvm_x86_ops.tlb_flush_gva (vcpu, gva); +static_call (kvm_x86_tlb_flush_gva)(vcpu, gva); ++vcpu->stat.invlpg;
arch/x86/kvm/mmu/spte.c +1 -1
@@ -120,7 +120,7 @@ int make_spte(struct kvm_vcpu *vcpu, unsigned int pte_access, int level,if (level >PG_LEVEL_4K ) spte |=PT_PAGE_SIZE_MASK ;if (tdp_enabled) - spte |= kvm_x86_ops.get_mt_mask (vcpu, gfn, + spte |=static_call (kvm_x86_get_mt_mask)(vcpu, gfn,kvm_is_mmio_pfn (pfn));if (host_writable)
arch/x86/kvm/pmu.c +1 -1
@@ -373,7 +373,7 @@ int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)return 1 ;if (!(kvm_read_cr4 (vcpu) &X86_CR4_PCE ) && - (kvm_x86_ops.get_cpl (vcpu) !=0 ) && + (static_call (kvm_x86_get_cpl)(vcpu) !=0 ) && (kvm_read_cr0 (vcpu) &X86_CR0_PE ))return 1 ;
arch/x86/kvm/trace.h +2 -2
@@ -256,7 +256,7 @@ TRACE_EVENT(name, \ __entry->guest_rip =kvm_rip_read (vcpu); \ __entry->isa = isa; \ __entry->vcpu_id = vcpu->vcpu_id; \ - kvm_x86_ops.get_exit_info (vcpu, &__entry->info1, \ +static_call (kvm_x86_get_exit_info)(vcpu, &__entry->info1, \ &__entry->info2, \ &__entry->intr_info, \ &__entry->error_code); \
@@ -738,7 +738,7 @@ TRACE_EVENT(kvm_emulate_insn, ),TP_fast_assign ( - __entry->csbase = kvm_x86_ops.get_segment_base(vcpu, VCPU_SREG_CS); + __entry->csbase = static_call(kvm_x86_get_segment_base)(vcpu, VCPU_SREG_CS); __entry->len = vcpu->arch.emulate_ctxt->fetch.ptr - vcpu->arch.emulate_ctxt->fetch.data; __entry->rip = vcpu->arch.emulate_ctxt->_eip - __entry->len;
arch/x86/kvm/x86.c +148 -150
1,087 lines not on this page · raw message
arch/x86/kvm/x86.h +3 -3
@@ -98,7 +98,7 @@ static inline bool is_64_bit_mode(struct kvm_vcpu *vcpu)if (!is_long_mode (vcpu))return false; - kvm_x86_ops.get_cs_db_l_bits (vcpu, &cs_db, &cs_l); +static_call (kvm_x86_get_cs_db_l_bits)(vcpu, &cs_db, &cs_l);return cs_l; }
@@ -129,7 +129,7 @@ static inline bool mmu_is_nested(struct kvm_vcpu *vcpu)static inline void kvm_vcpu_flush_tlb_current (struct kvm_vcpu *vcpu) { ++vcpu->stat.tlb_flush; - kvm_x86_ops.tlb_flush_current (vcpu); +static_call (kvm_x86_tlb_flush_current)(vcpu); }static inline int is_pae (struct kvm_vcpu *vcpu)
@@ -244,7 +244,7 @@ static inline bool kvm_check_has_quirk(struct kvm *kvm, u64 quirk)static inline bool kvm_vcpu_latch_init (struct kvm_vcpu *vcpu) { -return is_smm (vcpu) || kvm_x86_ops.apic_init_signal_blocked (vcpu); +return is_smm (vcpu) ||static_call (kvm_x86_apic_init_signal_blocked)(vcpu); }void kvm_inject_realmode_interrupt (struct kvm_vcpu *vcpu,int irq,int inc_eip);
Signature · 1 line
2.7.4
Peter Zijlstra
maintainerRe: [PATCH v2 3/3] KVM: x86: use static calls to reduce kvm_x86_ops overhead
arch/x86/include/asm/kvm_host.h void kvm_arch_free_vm(struct kvm *kvm); · line 1356
Quoted 13 lines · Jason Baron
arch/x86/include/asm/kvm_host.h +1
-1@@ -1350,7 +1350,7 @@ void kvm_arch_free_vm(struct kvm *kvm);static inline int kvm_arch_flush_remote_tlb (struct kvm *kvm) {if (kvm_x86_ops.tlb_remote_flush && - !kvm_x86_ops.tlb_remote_flush (kvm)) + !static_call (kvm_x86_tlb_remote_flush)(kvm))return 0 ;else return -ENOTSUPP ;
Would you be able to use something like this?
https://lkml.kernel.org/r/[email protected]
we could also add __static_call_return1(), if that would help.
Peter Zijlstra
maintainerRe: [PATCH v2 3/3] KVM: x86: use static calls to reduce kvm_x86_ops overhead
Quoted 17 lines · Peter Zijlstra
arch/x86/include/asm/kvm_host.h void kvm_arch_free_vm(struct kvm *kvm); · line 1356
Quoted 13 lines · Jason Baron
arch/x86/include/asm/kvm_host.h +1
-1@@ -1350,7 +1350,7 @@ void kvm_arch_free_vm(struct kvm *kvm);static inline int kvm_arch_flush_remote_tlb (struct kvm *kvm) {if (kvm_x86_ops.tlb_remote_flush && - !kvm_x86_ops.tlb_remote_flush (kvm)) + !static_call (kvm_x86_tlb_remote_flush)(kvm))return 0 ;else return -ENOTSUPP ;Would you be able to use something like this?
Quoted 3 lines
https://lkml.kernel.org/r/[email protected]
we could also add
__static_call_return1(), if that would help.
Something like so on top of the above.
arch/x86/kernel/static_call.c +5 -0
@@ -17,6 +17,7 @@ static void __ref __static_call_transfor* disp16 disp16 xorq %rax, %rax - a single 5 byte instruction that clears %rax */ static const u8 ret0[5 ] = {0x66 ,0x66 ,0x48 ,0x31 ,0xc0 }; +static const u8 ret1[5 ] = {0xb8 ,0x01 ,0x00 ,0x00 ,0x00 };/* mov $0x1,%eax */ int size =CALL_INSN_SIZE ;const void *emulate =NULL ;const void *code;
@@ -27,7 +28,11 @@ static void __ref __static_call_transforif (func == &__static_call_return0) { emulate = code; code = ret0; + }else if (func == &__static_call_return1) { + emulate = code; + code = ret1; } +break ;case NOP :
include/linux/static_call.h +3 -0
@@ -137,6 +137,7 @@ extern void arch_static_call_transform(v#ifdef CONFIG_HAVE_STATIC_CALL_INLINE extern long __static_call_return0 (void ); +extern long __static_call_return1 (void );extern int __init static_call_init (void );
@@ -190,6 +191,7 @@ extern int static_call_text_reserved(voi#elif defined(CONFIG_HAVE_STATIC_CALL )static inline long __static_call_return0 (void ) {return 0 ; } +static inline long __static_call_return0 (void ) {return 1 ; }static inline int static_call_init (void ) {return 0 ; }
@@ -239,6 +241,7 @@ static inline int static_call_text_reser#else /* Generic implementation */ static inline long __static_call_return0 (void ) {return 0 ; } +static inline long __static_call_return0 (void ) {return 1 ; }static inline int static_call_init (void ) {return 0 ; }
kernel/static_call.c +5 -0
@@ -443,6 +443,11 @@ long __static_call_return0(void)return 0 ; } +long __static_call_return1 (void ) +{ +return 1 ; +} +#ifdef CONFIG_STATIC_CALL_SELFTEST static int func_a (int x)
Peter Zijlstra
maintainerRe: [PATCH v2 3/3] KVM: x86: use static calls to reduce kvm_x86_ops overhead
include/linux/static_call.h static inline int static_call_text_reser · line 244
Quoted 13 lines · Peter Zijlstra
include/linux/static_call.h +2
-0@@ -190,6 +191,7 @@ extern int static_call_text_reserved(voi#elif defined(CONFIG_HAVE_STATIC_CALL )static inline long __static_call_return0 (void ) {return 0 ; } +static inline long __static_call_return0 (void ) {return 1 ; }static inline int static_call_init (void ) {return 0 ; }@@ -239,6 +241,7 @@ static inline int static_call_text_reser#else /* Generic implementation */ static inline long __static_call_return0 (void ) {return 0 ; } +static inline long __static_call_return0 (void ) {return 1 ; }
Too much copy-fail... Lemme go make more tea :-)
Paolo Bonzini
maintainerRe: [PATCH v2 3/3] KVM: x86: use static calls to reduce kvm_x86_ops overhead
In reply to Peter ZijlstraQuoted 17 lines · Peter Zijlstra
arch/x86/include/asm/kvm_host.h void kvm_arch_free_vm(struct kvm *kvm); · line 1356
Quoted 13 lines · Jason Baron
arch/x86/include/asm/kvm_host.h +1
-1@@ -1350,7 +1350,7 @@ void kvm_arch_free_vm(struct kvm *kvm);static inline int kvm_arch_flush_remote_tlb (struct kvm *kvm) {if (kvm_x86_ops.tlb_remote_flush && - !kvm_x86_ops.tlb_remote_flush (kvm)) + !static_call (kvm_x86_tlb_remote_flush)(kvm))return 0 ;else return -ENOTSUPP ;Would you be able to use something like this?
Quoted 4 lines
https://lkml.kernel.org/r/[email protected]we could also add
__static_call_return1(), if that would help.
I think I'd rather make the default callee return -ENOTSUPP directly and remove the "if" completely. So __static_call_return1() is not particularly useful here.
Paolo
Quoted 4 lines · Jason Baron
Hi,
Convert
kvm_x86_opsto usestatic_call. Shows good performance gains for cpuid loop micro-benchmark (results in patch 3/3).
Queued, thanks.
Paolo
Quoted 38 lines
Thanks,
-Jason
Changes from v1:
(quoted diff) +0
-3-Introduce kvm-x86-ops header with eye towards using this to define svm_x86_ops and vmx_x86_ops in follow on patches (Paolo, Sean) -add new patch (1/3), that adds a vmx/svm prefix to help facilitate svm_x86_ops and vmx_x86_ops future conversions. -added amd perf numbres to description of patch 3/3Jason Baron (3):
KVM: X86: append vmx/svm prefix to additionalkvm_x86_opsfunctions
KVM: x86: introduce definitions to support static calls forkvm_x86_ops
KVM: x86: use static calls to reducekvm_x86_opsoverheadarch/x86/include/asm/kvm-x86-ops.h | 127 +++++++++++++++ arch/x86/include/asm/kvm_host.h | 21 ++- arch/x86/kvm/cpuid.c | 2 +- arch/x86/kvm/hyperv.c | 4 +- arch/x86/kvm/irq.c | 3 +- arch/x86/kvm/kvm_cache_regs.h | 10 +- arch/x86/kvm/lapic.c | 30 ++-- arch/x86/kvm/mmu.h | 6 +- arch/x86/kvm/mmu/mmu.c | 15 +- arch/x86/kvm/mmu/spte.c | 2 +- arch/x86/kvm/pmu.c | 2 +- arch/x86/kvm/svm/svm.c | 20 +-- arch/x86/kvm/trace.h | 4 +- arch/x86/kvm/vmx/nested.c | 2 +- arch/x86/kvm/vmx/vmx.c | 30 ++-- arch/x86/kvm/vmx/vmx.h | 2 +- arch/x86/kvm/x86.c | 307 +++++++++++++++++++------------------ arch/x86/kvm/x86.h | 6 +- 18 files changed, 369 insertions(+), 224 deletions(-)create mode 100644
arch/x86/include/asm/kvm-x86-ops.h