Skip to contentSkip to conversation

Use static_call for kvm_x86_ops

PATCH Applied Jason Baron 3 participants 14 messages Jan 15, 2021 – Jan 26, 2021

Subsystems (5)
x86 KVM KVM x86 Static branch/call +1 more subsystems
lore
Share & tools
lore

Jason Baron

author maintainer
#0 lore raw

v2 cover Use static_call for kvm_x86_ops

Hi,

Convert kvm_x86_ops to use static_call. Shows good performance gains for cpuid loop micro-benchmark (results in patch 3/3).

Thanks,

-Jason

Changes from v1:
-Introduce kvm-x86-ops header with eye towards using this to define
svm_x86_ops and vmx_x86_ops in follow on patches (Paolo, Sean)
-add new patch (1/3), that adds a vmx/svm prefix to help facilitate
svm_x86_ops and vmx_x86_ops future conversions.
-added amd perf numbres to description of patch 3/3

Jason Baron (3):
KVM: X86: append vmx/svm prefix to additional kvm_x86_ops functions
KVM: x86: introduce definitions to support static calls for kvm_x86_ops
KVM: x86: use static calls to reduce kvm_x86_ops overhead

 arch/x86/include/asm/kvm-x86-ops.h | 127 +++++++++++++++
 arch/x86/include/asm/kvm_host.h    |  21 ++-
 arch/x86/kvm/cpuid.c               |   2 +-
 arch/x86/kvm/hyperv.c              |   4 +-
 arch/x86/kvm/irq.c                 |   3 +-
 arch/x86/kvm/kvm_cache_regs.h      |  10 +-
 arch/x86/kvm/lapic.c               |  30 ++--
 arch/x86/kvm/mmu.h                 |   6 +-
 arch/x86/kvm/mmu/mmu.c             |  15 +-
 arch/x86/kvm/mmu/spte.c            |   2 +-
 arch/x86/kvm/pmu.c                 |   2 +-
 arch/x86/kvm/svm/svm.c             |  20 +--
 arch/x86/kvm/trace.h               |   4 +-
 arch/x86/kvm/vmx/nested.c          |   2 +-
 arch/x86/kvm/vmx/vmx.c             |  30 ++--
 arch/x86/kvm/vmx/vmx.h             |   2 +-
 arch/x86/kvm/x86.c                 | 307 +++++++++++++++++++------------------
 arch/x86/kvm/x86.h                 |   6 +-
 18 files changed, 369 insertions(+), 224 deletions(-)
 create mode 100644 arch/x86/include/asm/kvm-x86-ops.h
Signature · 1 line
2.7.4
author maintainer
#1 lore raw

PATCH v2 1/3 KVM: X86: append vmx/svm prefix to additional kvm_x86_ops functions

A subsequent patch introduces macros in preparation for simplifying the definition for vmx_x86_ops and svm_x86_ops. Making the naming more uniform expands the coverage of the macros. Add vmx/svm prefix to the following
functions: update_exception_bitmap(), enable_nmi_window(), enable_irq_window(), update_cr8_intercept and enable_smi_window().

Cc Paolo Bonzini <[email protected]>
Cc Sean Christopherson <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Borislav Petkov <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Andrea Arcangeli <[email protected]>
---
 arch/x86/kvm/svm/svm.c    | 20 ++++++++++----------
 arch/x86/kvm/vmx/nested.c |  2 +-
 arch/x86/kvm/vmx/vmx.c    | 30 +++++++++++++++---------------
 arch/x86/kvm/vmx/vmx.h    |  2 +-
 4 files changed, 27 insertions(+), 27 deletions(-)
arch/x86/kvm/svm/svm.c +10 -10
index cce0143..04d3126 100644
@@ -1810,7 +1810,7 @@ static void svm_set_segment(struct kvm_vcpu *vcpu,
 	vmcb_mark_dirty(svm->vmcb, VMCB_SEG);
 }
 
-static void update_exception_bitmap(struct kvm_vcpu *vcpu)
+static void svm_update_exception_bitmap(struct kvm_vcpu *vcpu)
 {
 	struct vcpu_svm *svm = to_svm(vcpu);
 
@@ -3349,7 +3349,7 @@ static void svm_set_irq(struct kvm_vcpu *vcpu)
 		SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR;
 }
 
-static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
+static void svm_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
 {
 	struct vcpu_svm *svm = to_svm(vcpu);
 
@@ -3474,7 +3474,7 @@ static int svm_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection)
 	return !svm_interrupt_blocked(vcpu);
 }
 
-static void enable_irq_window(struct kvm_vcpu *vcpu)
+static void svm_enable_irq_window(struct kvm_vcpu *vcpu)
 {
 	struct vcpu_svm *svm = to_svm(vcpu);
 
@@ -3498,7 +3498,7 @@ static void enable_irq_window(struct kvm_vcpu *vcpu)
 	}
 }
 
-static void enable_nmi_window(struct kvm_vcpu *vcpu)
+static void svm_enable_nmi_window(struct kvm_vcpu *vcpu)
 {
 	struct vcpu_svm *svm = to_svm(vcpu);
 
@@ -4280,7 +4280,7 @@ static int svm_pre_leave_smm(struct kvm_vcpu *vcpu, const char *smstate)
 	return ret;
 }
 
-static void enable_smi_window(struct kvm_vcpu *vcpu)
+static void svm_enable_smi_window(struct kvm_vcpu *vcpu)
 {
 	struct vcpu_svm *svm = to_svm(vcpu);
 
@@ -4426,7 +4426,7 @@ static struct kvm_x86_ops svm_x86_ops __initdata = {
 	.vcpu_blocking = svm_vcpu_blocking,
 	.vcpu_unblocking = svm_vcpu_unblocking,
 
-	.update_exception_bitmap = update_exception_bitmap,
+	.update_exception_bitmap = svm_update_exception_bitmap,
 	.get_msr_feature = svm_get_msr_feature,
 	.get_msr = svm_get_msr,
 	.set_msr = svm_set_msr,
@@ -4469,9 +4469,9 @@ static struct kvm_x86_ops svm_x86_ops __initdata = {
 	.nmi_allowed = svm_nmi_allowed,
 	.get_nmi_mask = svm_get_nmi_mask,
 	.set_nmi_mask = svm_set_nmi_mask,
-	.enable_nmi_window = enable_nmi_window,
-	.enable_irq_window = enable_irq_window,
-	.update_cr8_intercept = update_cr8_intercept,
+	.enable_nmi_window = svm_enable_nmi_window,
+	.enable_irq_window = svm_enable_irq_window,
+	.update_cr8_intercept = svm_update_cr8_intercept,
 	.set_virtual_apic_mode = svm_set_virtual_apic_mode,
 	.refresh_apicv_exec_ctrl = svm_refresh_apicv_exec_ctrl,
 	.check_apicv_inhibit_reasons = svm_check_apicv_inhibit_reasons,
@@ -4514,7 +4514,7 @@ static struct kvm_x86_ops svm_x86_ops __initdata = {
 	.smi_allowed = svm_smi_allowed,
 	.pre_enter_smm = svm_pre_enter_smm,
 	.pre_leave_smm = svm_pre_leave_smm,
-	.enable_smi_window = enable_smi_window,
+	.enable_smi_window = svm_enable_smi_window,
 
 	.mem_enc_op = svm_mem_enc_op,
 	.mem_enc_reg_region = svm_register_enc_region,
arch/x86/kvm/vmx/nested.c +1 -1
index e2f2656..8a0b817 100644
@@ -2532,7 +2532,7 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
 	 * bitwise-or of what L1 wants to trap for L2, and what we want to
 	 * trap. Note that CR0.TS also needs updating - we do this later.
 	 */
-	update_exception_bitmap(vcpu);
+	vmx_update_exception_bitmap(vcpu);
 	vcpu->arch.cr0_guest_owned_bits &= ~vmcs12->cr0_guest_host_mask;
 	vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);
 
arch/x86/kvm/vmx/vmx.c +15 -15
index 75c9c6a..3c7ca71 100644
@@ -806,7 +806,7 @@ static u32 vmx_read_guest_seg_ar(struct vcpu_vmx *vmx, unsigned seg)
 	return *p;
 }
 
-void update_exception_bitmap(struct kvm_vcpu *vcpu)
+void vmx_update_exception_bitmap(struct kvm_vcpu *vcpu)
 {
 	u32 eb;
 
@@ -2739,7 +2739,7 @@ static void enter_pmode(struct kvm_vcpu *vcpu)
 	vmcs_writel(GUEST_CR4, (vmcs_readl(GUEST_CR4) & ~X86_CR4_VME) |
 			(vmcs_readl(CR4_READ_SHADOW) & X86_CR4_VME));
 
-	update_exception_bitmap(vcpu);
+	vmx_update_exception_bitmap(vcpu);
 
 	fix_pmode_seg(vcpu, VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]);
 	fix_pmode_seg(vcpu, VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]);
@@ -2819,7 +2819,7 @@ static void enter_rmode(struct kvm_vcpu *vcpu)
 
 	vmcs_writel(GUEST_RFLAGS, flags);
 	vmcs_writel(GUEST_CR4, vmcs_readl(GUEST_CR4) | X86_CR4_VME);
-	update_exception_bitmap(vcpu);
+	vmx_update_exception_bitmap(vcpu);
 
 	fix_rmode_seg(VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]);
 	fix_rmode_seg(VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]);
@@ -4467,23 +4467,23 @@ static void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
 	vmx_set_cr4(vcpu, 0);
 	vmx_set_efer(vcpu, 0);
 
-	update_exception_bitmap(vcpu);
+	vmx_update_exception_bitmap(vcpu);
 
 	vpid_sync_context(vmx->vpid);
 	if (init_event)
 		vmx_clear_hlt(vcpu);
 }
 
-static void enable_irq_window(struct kvm_vcpu *vcpu)
+static void vmx_enable_irq_window(struct kvm_vcpu *vcpu)
 {
 	exec_controls_setbit(to_vmx(vcpu), CPU_BASED_INTR_WINDOW_EXITING);
 }
 
-static void enable_nmi_window(struct kvm_vcpu *vcpu)
+static void vmx_enable_nmi_window(struct kvm_vcpu *vcpu)
 {
 	if (!enable_vnmi ||
 	    vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_STI) {
-		enable_irq_window(vcpu);
+		vmx_enable_irq_window(vcpu);
 		return;
 	}
 
@@ -6129,7 +6129,7 @@ static noinstr void vmx_l1d_flush(struct kvm_vcpu *vcpu)
 		: "eax", "ebx", "ecx", "edx");
 }
 
-static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
+static void vmx_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
 {
 	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
 	int tpr_threshold;
@@ -7245,7 +7245,7 @@ static void vmx_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu)
 	set_cr4_guest_host_mask(vmx);
 
 	/* Refresh #PF interception to account for MAXPHYADDR changes. */
-	update_exception_bitmap(vcpu);
+	vmx_update_exception_bitmap(vcpu);
 }
 
 static __init void vmx_set_cpu_caps(void)
@@ -7535,7 +7535,7 @@ static int vmx_pre_leave_smm(struct kvm_vcpu *vcpu, const char *smstate)
 	return 0;
 }
 
-static void enable_smi_window(struct kvm_vcpu *vcpu)
+static void vmx_enable_smi_window(struct kvm_vcpu *vcpu)
 {
 	/* RSM will cause a vmexit anyway.  */
 }
@@ -7595,7 +7595,7 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = {
 	.vcpu_load = vmx_vcpu_load,
 	.vcpu_put = vmx_vcpu_put,
 
-	.update_exception_bitmap = update_exception_bitmap,
+	.update_exception_bitmap = vmx_update_exception_bitmap,
 	.get_msr_feature = vmx_get_msr_feature,
 	.get_msr = vmx_get_msr,
 	.set_msr = vmx_set_msr,
@@ -7638,9 +7638,9 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = {
 	.nmi_allowed = vmx_nmi_allowed,
 	.get_nmi_mask = vmx_get_nmi_mask,
 	.set_nmi_mask = vmx_set_nmi_mask,
-	.enable_nmi_window = enable_nmi_window,
-	.enable_irq_window = enable_irq_window,
-	.update_cr8_intercept = update_cr8_intercept,
+	.enable_nmi_window = vmx_enable_nmi_window,
+	.enable_irq_window = vmx_enable_irq_window,
+	.update_cr8_intercept = vmx_update_cr8_intercept,
 	.set_virtual_apic_mode = vmx_set_virtual_apic_mode,
 	.set_apic_access_page_addr = vmx_set_apic_access_page_addr,
 	.refresh_apicv_exec_ctrl = vmx_refresh_apicv_exec_ctrl,
@@ -7698,7 +7698,7 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = {
 	.smi_allowed = vmx_smi_allowed,
 	.pre_enter_smm = vmx_pre_enter_smm,
 	.pre_leave_smm = vmx_pre_leave_smm,
-	.enable_smi_window = enable_smi_window,
+	.enable_smi_window = vmx_enable_smi_window,
 
 	.can_emulate_instruction = vmx_can_emulate_instruction,
 	.apic_init_signal_blocked = vmx_apic_init_signal_blocked,
arch/x86/kvm/vmx/vmx.h +1 -1
index 9d3a557..8f70c25 100644
@@ -329,7 +329,7 @@ void vmx_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg);
 u64 construct_eptp(struct kvm_vcpu *vcpu, unsigned long root_hpa,
 		   int root_level);
 
-void update_exception_bitmap(struct kvm_vcpu *vcpu);
+void vmx_update_exception_bitmap(struct kvm_vcpu *vcpu);
 void vmx_update_msr_bitmap(struct kvm_vcpu *vcpu);
 bool vmx_nmi_blocked(struct kvm_vcpu *vcpu);
 bool vmx_interrupt_blocked(struct kvm_vcpu *vcpu);
Signature · 1 line
2.7.4
maintainer
#2 lore raw

Re: [PATCH v2 1/3] KVM: X86: append vmx/svm prefix to additional kvm_x86_ops functions

arch/x86/kvm/svm/svm.c static void svm_set_segment(struct kvm_vcpu *vcpu, · line 1813

Quoted 2 lines · Jason Baron
arch/x86/kvm/svm/svm.c +1 -1
-static void update_exception_bitmap(struct kvm_vcpu *vcpu)
+static void svm_update_exception_bitmap(struct kvm_vcpu *vcpu)

Just to be a total pendant: s/append/Prepend/ on $Subject

author maintainer
#3 lore raw

Re: [PATCH v2 1/3] KVM: X86: append vmx/svm prefix to additional kvm_x86_ops functions

Quoted 8 lines · Peter Zijlstra

arch/x86/kvm/svm/svm.c static void svm_set_segment(struct kvm_vcpu *vcpu, · line 1813

Quoted 2 lines · Jason Baron
arch/x86/kvm/svm/svm.c +1 -1
-static void update_exception_bitmap(struct kvm_vcpu *vcpu)
+static void svm_update_exception_bitmap(struct kvm_vcpu *vcpu)

Just to be a total pendant: s/append/Prepend/ on $Subject

Ha - I actually switched $subject to prepend and then switched it back because I thought I was being pedantic. But maybe not :)

author maintainer
#4 lore raw

PATCH v2 2/3 KVM: x86: introduce definitions to support static calls for kvm_x86_ops

In reply to Jason Baron

Use static calls to improve kvm_x86_ops performance. Introduce the definitions that will be used by a subsequent patch to actualize the savings. Add a new kvm-x86-ops.h header that can be used for the definition of static calls. This header is also intended to be used to simplify the defition of svm_kvm_ops and vmx_x86_ops.

Note that all functions in kvm_x86_ops are covered here except for 'pmu_ops' and 'nested ops'. I think they can be covered by static calls in a simlilar manner, but were omitted from this series to reduce scope and because I don't think they have as large of a performance impact.

Cc Paolo Bonzini <[email protected]>
Cc Sean Christopherson <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Borislav Petkov <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Andrea Arcangeli <[email protected]>
---
 arch/x86/include/asm/kvm-x86-ops.h | 127 +++++++++++++++++++++++++++++++++++++
 arch/x86/include/asm/kvm_host.h    |  13 ++++
 arch/x86/kvm/x86.c                 |   9 +++
 3 files changed, 149 insertions(+)
 create mode 100644 arch/x86/include/asm/kvm-x86-ops.h
arch/x86/include/asm/kvm-x86-ops.h new file +127 -0
new file mode 100644
index 0000000..355a2ab
@@ -0,0 +1,127 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#if !defined(KVM_X86_OP) || !defined(KVM_X86_OP_NULL)
+BUILD_BUG_ON(1)
+#endif
+
+/*
+ * KVM_X86_OP() and KVM_X86_OP_NULL() are used to help generate
+ * "static_call()"s. They are also intended for use when defining
+ * the vmx/svm kvm_x86_ops. KVM_X86_OP() can be used for those
+ * functions that follow the [svm|vmx]_func_name convention.
+ * KVM_X86_OP_NULL() can leave a NULL definition for the
+ * case where there is no definition or a function name that
+ * doesn't match the typical naming convention is supplied.
+ */
+KVM_X86_OP_NULL(hardware_enable)
+KVM_X86_OP_NULL(hardware_disable)
+KVM_X86_OP_NULL(hardware_unsetup)
+KVM_X86_OP_NULL(cpu_has_accelerated_tpr)
+KVM_X86_OP(has_emulated_msr)
+KVM_X86_OP(vcpu_after_set_cpuid)
+KVM_X86_OP(vm_init)
+KVM_X86_OP_NULL(vm_destroy)
+KVM_X86_OP(vcpu_create)
+KVM_X86_OP(vcpu_free)
+KVM_X86_OP(vcpu_reset)
+KVM_X86_OP(prepare_guest_switch)
+KVM_X86_OP(vcpu_load)
+KVM_X86_OP(vcpu_put)
+KVM_X86_OP(update_exception_bitmap)
+KVM_X86_OP(get_msr)
+KVM_X86_OP(set_msr)
+KVM_X86_OP(get_segment_base)
+KVM_X86_OP(get_segment)
+KVM_X86_OP(get_cpl)
+KVM_X86_OP(set_segment)
+KVM_X86_OP_NULL(get_cs_db_l_bits)
+KVM_X86_OP(set_cr0)
+KVM_X86_OP(is_valid_cr4)
+KVM_X86_OP(set_cr4)
+KVM_X86_OP(set_efer)
+KVM_X86_OP(get_idt)
+KVM_X86_OP(set_idt)
+KVM_X86_OP(get_gdt)
+KVM_X86_OP(set_gdt)
+KVM_X86_OP(sync_dirty_debug_regs)
+KVM_X86_OP(set_dr7)
+KVM_X86_OP(cache_reg)
+KVM_X86_OP(get_rflags)
+KVM_X86_OP(set_rflags)
+KVM_X86_OP(tlb_flush_all)
+KVM_X86_OP(tlb_flush_current)
+KVM_X86_OP_NULL(tlb_remote_flush)
+KVM_X86_OP_NULL(tlb_remote_flush_with_range)
+KVM_X86_OP(tlb_flush_gva)
+KVM_X86_OP(tlb_flush_guest)
+KVM_X86_OP(run)
+KVM_X86_OP_NULL(handle_exit)
+KVM_X86_OP_NULL(skip_emulated_instruction)
+KVM_X86_OP_NULL(update_emulated_instruction)
+KVM_X86_OP(set_interrupt_shadow)
+KVM_X86_OP(get_interrupt_shadow)
+KVM_X86_OP(patch_hypercall)
+KVM_X86_OP(set_irq)
+KVM_X86_OP(set_nmi)
+KVM_X86_OP(queue_exception)
+KVM_X86_OP(cancel_injection)
+KVM_X86_OP(interrupt_allowed)
+KVM_X86_OP(nmi_allowed)
+KVM_X86_OP(get_nmi_mask)
+KVM_X86_OP(set_nmi_mask)
+KVM_X86_OP(enable_nmi_window)
+KVM_X86_OP(enable_irq_window)
+KVM_X86_OP(update_cr8_intercept)
+KVM_X86_OP(check_apicv_inhibit_reasons)
+KVM_X86_OP_NULL(pre_update_apicv_exec_ctrl)
+KVM_X86_OP(refresh_apicv_exec_ctrl)
+KVM_X86_OP(hwapic_irr_update)
+KVM_X86_OP(hwapic_isr_update)
+KVM_X86_OP_NULL(guest_apic_has_interrupt)
+KVM_X86_OP(load_eoi_exitmap)
+KVM_X86_OP(set_virtual_apic_mode)
+KVM_X86_OP_NULL(set_apic_access_page_addr)
+KVM_X86_OP(deliver_posted_interrupt)
+KVM_X86_OP_NULL(sync_pir_to_irr)
+KVM_X86_OP(set_tss_addr)
+KVM_X86_OP(set_identity_map_addr)
+KVM_X86_OP(get_mt_mask)
+KVM_X86_OP(load_mmu_pgd)
+KVM_X86_OP_NULL(has_wbinvd_exit)
+KVM_X86_OP(write_l1_tsc_offset)
+KVM_X86_OP(get_exit_info)
+KVM_X86_OP(check_intercept)
+KVM_X86_OP(handle_exit_irqoff)
+KVM_X86_OP_NULL(request_immediate_exit)
+KVM_X86_OP(sched_in)
+KVM_X86_OP_NULL(slot_enable_log_dirty)
+KVM_X86_OP_NULL(slot_disable_log_dirty)
+KVM_X86_OP_NULL(flush_log_dirty)
+KVM_X86_OP_NULL(enable_log_dirty_pt_masked)
+KVM_X86_OP_NULL(cpu_dirty_log_size)
+KVM_X86_OP_NULL(pre_block)
+KVM_X86_OP_NULL(post_block)
+KVM_X86_OP_NULL(vcpu_blocking)
+KVM_X86_OP_NULL(vcpu_unblocking)
+KVM_X86_OP_NULL(update_pi_irte)
+KVM_X86_OP_NULL(apicv_post_state_restore)
+KVM_X86_OP_NULL(dy_apicv_has_pending_interrupt)
+KVM_X86_OP_NULL(set_hv_timer)
+KVM_X86_OP_NULL(cancel_hv_timer)
+KVM_X86_OP(setup_mce)
+KVM_X86_OP(smi_allowed)
+KVM_X86_OP(pre_enter_smm)
+KVM_X86_OP(pre_leave_smm)
+KVM_X86_OP(enable_smi_window)
+KVM_X86_OP_NULL(mem_enc_op)
+KVM_X86_OP_NULL(mem_enc_reg_region)
+KVM_X86_OP_NULL(mem_enc_unreg_region)
+KVM_X86_OP(get_msr_feature)
+KVM_X86_OP(can_emulate_instruction)
+KVM_X86_OP(apic_init_signal_blocked)
+KVM_X86_OP_NULL(enable_direct_tlbflush)
+KVM_X86_OP_NULL(migrate_timers)
+KVM_X86_OP(msr_filter_changed)
+KVM_X86_OP_NULL(complete_emulated_msr)
+
+#undef KVM_X86_OP
+#undef KVM_X86_OP_NULL
arch/x86/include/asm/kvm_host.h +13 -0
index 3ab7b46..5060922 100644
@@ -1326,6 +1326,19 @@ extern u64 __read_mostly host_efer;
 extern bool __read_mostly allow_smaller_maxphyaddr;
 extern struct kvm_x86_ops kvm_x86_ops;
 
+#define KVM_X86_OP(func) \
+	DECLARE_STATIC_CALL(kvm_x86_##func, *(((struct kvm_x86_ops *)0)->func));
+#define KVM_X86_OP_NULL KVM_X86_OP
+#include <asm/kvm-x86-ops.h>
+
+static inline void kvm_ops_static_call_update(void)
+{
+#define KVM_X86_OP(func) \
+	static_call_update(kvm_x86_##func, kvm_x86_ops.func);
+#define KVM_X86_OP_NULL KVM_X86_OP
+#include <asm/kvm-x86-ops.h>
+}
+
 #define __KVM_HAVE_ARCH_VM_ALLOC
 static inline struct kvm *kvm_arch_alloc_vm(void)
 {
arch/x86/kvm/x86.c +9 -0
index 3f7c1fc..c21927f 100644
@@ -113,6 +113,15 @@ static int sync_regs(struct kvm_vcpu *vcpu);
 struct kvm_x86_ops kvm_x86_ops __read_mostly;
 EXPORT_SYMBOL_GPL(kvm_x86_ops);
 
+#define KVM_X86_OP(func)					     \
+	DEFINE_STATIC_CALL_NULL(kvm_x86_##func,			     \
+				*(((struct kvm_x86_ops *)0)->func));
+#define KVM_X86_OP_NULL KVM_X86_OP
+#include <asm/kvm-x86-ops.h>
+EXPORT_STATIC_CALL_GPL(kvm_x86_get_cs_db_l_bits);
+EXPORT_STATIC_CALL_GPL(kvm_x86_cache_reg);
+EXPORT_STATIC_CALL_GPL(kvm_x86_tlb_flush_current);
+
 static bool __read_mostly ignore_msrs = 0;
 module_param(ignore_msrs, bool, S_IRUGO | S_IWUSR);
 
Signature · 1 line
2.7.4
maintainer
#5 lore raw

Re: [PATCH v2 2/3] KVM: x86: introduce definitions to support static calls for kvm_x86_ops

Quoted 6 lines · Jason Baron
arch/x86/kvm/x86.c +0 -0
index 3f7c1fc..c21927f 100644
@@ -113,6 +113,15 @@ static int sync_regs(struct kvm_vcpu *vcpu);
 struct kvm_x86_ops kvm_x86_ops __read_mostly;

arch/x86/kvm/x86.c static int sync_regs(struct kvm_vcpu *vcpu); · line 123

Quoted 10 lines
arch/x86/kvm/x86.c +8 -0
 EXPORT_SYMBOL_GPL(kvm_x86_ops);
 
+#define KVM_X86_OP(func)					     \
+	DEFINE_STATIC_CALL_NULL(kvm_x86_##func,			     \
+				*(((struct kvm_x86_ops *)0)->func));
+#define KVM_X86_OP_NULL KVM_X86_OP
+#include <asm/kvm-x86-ops.h>
+EXPORT_STATIC_CALL_GPL(kvm_x86_get_cs_db_l_bits);
+EXPORT_STATIC_CALL_GPL(kvm_x86_cache_reg);
+EXPORT_STATIC_CALL_GPL(kvm_x86_tlb_flush_current);

Would something like:

https://lkml.kernel.org/r/[email protected]

Be useful? That way modules can call the static_call() but not change it.

maintainer
#6 lore raw

Re: [PATCH v2 2/3] KVM: x86: introduce definitions to support static calls for kvm_x86_ops

Quoted 10 lines · Peter Zijlstra

arch/x86/kvm/x86.c static int sync_regs(struct kvm_vcpu *vcpu); · line 123

Quoted 8 lines
arch/x86/kvm/x86.c +8 -0
+#define KVM_X86_OP(func)					     \
+	DEFINE_STATIC_CALL_NULL(kvm_x86_##func,			     \
+				*(((struct kvm_x86_ops *)0)->func));
+#define KVM_X86_OP_NULL KVM_X86_OP
+#include <asm/kvm-x86-ops.h>
+EXPORT_STATIC_CALL_GPL(kvm_x86_get_cs_db_l_bits);
+EXPORT_STATIC_CALL_GPL(kvm_x86_cache_reg);
+EXPORT_STATIC_CALL_GPL(kvm_x86_tlb_flush_current);

Would something like:

Quoted 5 lines
   https://lkml.kernel.org/r/[email protected]

Be useful? That way modules can call the static_call() but not change it.

Maybe not in these cases, but in general there may be cases where we later want to change the static_call (for example replacing jump labels with static_calls).

Paolo

author maintainer
#7 lore raw

Re: [PATCH v2 2/3] KVM: x86: introduce definitions to support static calls for kvm_x86_ops

Quoted 14 lines · Paolo Bonzini
Quoted 13 lines · Peter Zijlstra

arch/x86/kvm/x86.c static int sync_regs(struct kvm_vcpu *vcpu); · line 123

Quoted 8 lines
arch/x86/kvm/x86.c +8 -0
+#define KVM_X86_OP(func)                         \
+    DEFINE_STATIC_CALL_NULL(kvm_x86_##func,                 \
+                *(((struct kvm_x86_ops *)0)->func));
+#define KVM_X86_OP_NULL KVM_X86_OP
+#include <asm/kvm-x86-ops.h>
+EXPORT_STATIC_CALL_GPL(kvm_x86_get_cs_db_l_bits);
+EXPORT_STATIC_CALL_GPL(kvm_x86_cache_reg);
+EXPORT_STATIC_CALL_GPL(kvm_x86_tlb_flush_current);

Would something like:


https://urldefense.com/v3/__https://lkml.kernel.org/r/[email protected]__;!!GjvTz_vk!GbAPurpdyP1TaDRZN0NvvBkOLJhmRHzNtv0ZVIwZqNrJpMYze75mJzpUNJMRAg$

Quoted 9 lines
Quoted 3 lines

Be useful? That way modules can call the static_call() but not change it.

Maybe not in these cases, but in general there may be cases where we later want to change the static_call (for example replacing jump labels with static_calls).

Paolo

I tried this out but got:

ERROR: modpost: "__SCK__kvm_x86_cache_reg" [arch/x86/kvm/kvm-amd.ko] undefined!
ERROR: modpost: "__SCK__kvm_x86_tlb_flush_current" [arch/x86/kvm/kvm-intel.ko] undefined!
ERROR: modpost: "__SCK__kvm_x86_get_cs_db_l_bits" [arch/x86/kvm/kvm-intel.ko] undefined!
ERROR: modpost: "__SCK__kvm_x86_cache_reg" [arch/x86/kvm/kvm-intel.ko] undefined!

I'm a bit confused because we have:

#define __static_call(name)                                             \
({                                                                      \
        __ADDRESSABLE(STATIC_CALL_KEY(name));                           \
        &STATIC_CALL_TRAMP(name);                                       \

})

And so it looks to me like we need to still reference the key from the module code.

Thanks,

-Jason

author maintainer
#8 lore raw

PATCH v2 3/3 KVM: x86: use static calls to reduce kvm_x86_ops overhead

In reply to Jason Baron

Convert kvm_x86_ops to use static calls. Note that all kvm_x86_ops are covered here except for 'pmu_ops and 'nested ops'.

Here are some numbers running cpuid in a loop of 1 million calls averaged over 5 runs, measured in the vm (lower is better).

Intel Xeon 3000MHz:

|default |mitigations=off
-------------------------------------

vanilla    |.671s      |.486s
static call|.573s(-15%)|.458s(-6%)

AMD EPYC 2500MHz:

|default |mitigations=off
-------------------------------------

vanilla    |.710s      |.609s
static call|.664s(-6%) |.609s(0%)
Cc Paolo Bonzini <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Borislav Petkov <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Andrea Arcangeli <[email protected]>
Cc Sean Christopherson <[email protected]>
---
 arch/x86/include/asm/kvm_host.h |   8 +-
 arch/x86/kvm/cpuid.c            |   2 +-
 arch/x86/kvm/hyperv.c           |   4 +-
 arch/x86/kvm/irq.c              |   3 +-
 arch/x86/kvm/kvm_cache_regs.h   |  10 +-
 arch/x86/kvm/lapic.c            |  30 ++--
 arch/x86/kvm/mmu.h              |   6 +-
 arch/x86/kvm/mmu/mmu.c          |  15 +-
 arch/x86/kvm/mmu/spte.c         |   2 +-
 arch/x86/kvm/pmu.c              |   2 +-
 arch/x86/kvm/trace.h            |   4 +-
 arch/x86/kvm/x86.c              | 298 ++++++++++++++++++++--------------------
 arch/x86/kvm/x86.h              |   6 +-
 13 files changed, 193 insertions(+), 197 deletions(-)
arch/x86/include/asm/kvm_host.h +3 -5
index 5060922..9d4492b 100644
@@ -1350,7 +1350,7 @@ void kvm_arch_free_vm(struct kvm *kvm);
 static inline int kvm_arch_flush_remote_tlb(struct kvm *kvm)
 {
 	if (kvm_x86_ops.tlb_remote_flush &&
-	    !kvm_x86_ops.tlb_remote_flush(kvm))
+	    !static_call(kvm_x86_tlb_remote_flush)(kvm))
 		return 0;
 	else
 		return -ENOTSUPP;
@@ -1740,14 +1740,12 @@ static inline bool kvm_irq_is_postable(struct kvm_lapic_irq *irq)
 
 static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu)
 {
-	if (kvm_x86_ops.vcpu_blocking)
-		kvm_x86_ops.vcpu_blocking(vcpu);
+	static_call_cond(kvm_x86_vcpu_blocking)(vcpu);
 }
 
 static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu)
 {
-	if (kvm_x86_ops.vcpu_unblocking)
-		kvm_x86_ops.vcpu_unblocking(vcpu);
+	static_call_cond(kvm_x86_vcpu_unblocking)(vcpu);
 }
 
 static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) {}
arch/x86/kvm/cpuid.c +1 -1
index 13036cf..f951d18 100644
@@ -182,7 +182,7 @@ static void kvm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu)
 	vcpu->arch.cr3_lm_rsvd_bits = rsvd_bits(cpuid_maxphyaddr(vcpu), 63);
 
 	/* Invoke the vendor callback only after the above state is updated. */
-	kvm_x86_ops.vcpu_after_set_cpuid(vcpu);
+	static_call(kvm_x86_vcpu_after_set_cpuid)(vcpu);
 }
 
 static int is_efer_nx(void)
arch/x86/kvm/hyperv.c +2 -2
index 922c69d..5c45d80 100644
@@ -1154,7 +1154,7 @@ static int kvm_hv_set_msr_pw(struct kvm_vcpu *vcpu, u32 msr, u64 data,
 		addr = gfn_to_hva(kvm, gfn);
 		if (kvm_is_error_hva(addr))
 			return 1;
-		kvm_x86_ops.patch_hypercall(vcpu, instructions);
+		static_call(kvm_x86_patch_hypercall)(vcpu, instructions);
 		((unsigned char *)instructions)[3] = 0xc3; /* ret */
 		if (__copy_to_user((void __user *)addr, instructions, 4))
 			return 1;
@@ -1745,7 +1745,7 @@ int kvm_hv_hypercall(struct kvm_vcpu *vcpu)
 	 * hypercall generates UD from non zero cpl and real mode
 	 * per HYPER-V spec
 	 */
-	if (kvm_x86_ops.get_cpl(vcpu) != 0 || !is_protmode(vcpu)) {
+	if (static_call(kvm_x86_get_cpl)(vcpu) != 0 || !is_protmode(vcpu)) {
 		kvm_queue_exception(vcpu, UD_VECTOR);
 		return 1;
 	}
arch/x86/kvm/irq.c +1 -2
index 814698e..a035cca 100644
@@ -143,8 +143,7 @@ void __kvm_migrate_timers(struct kvm_vcpu *vcpu)
 {
 	__kvm_migrate_apic_timer(vcpu);
 	__kvm_migrate_pit_timer(vcpu);
-	if (kvm_x86_ops.migrate_timers)
-		kvm_x86_ops.migrate_timers(vcpu);
+	static_call_cond(kvm_x86_migrate_timers)(vcpu);
 }
 
 bool kvm_arch_irqfd_allowed(struct kvm *kvm, struct kvm_irqfd *args)
arch/x86/kvm/kvm_cache_regs.h +5 -5
index f15bc16..61d2965 100644
@@ -69,7 +69,7 @@ static inline unsigned long kvm_register_read(struct kvm_vcpu *vcpu, int reg)
 		return 0;
 
 	if (!kvm_register_is_available(vcpu, reg))
-		kvm_x86_ops.cache_reg(vcpu, reg);
+		static_call(kvm_x86_cache_reg)(vcpu, reg);
 
 	return vcpu->arch.regs[reg];
 }
@@ -109,7 +109,7 @@ static inline u64 kvm_pdptr_read(struct kvm_vcpu *vcpu, int index)
 	might_sleep();  /* on svm */
 
 	if (!kvm_register_is_available(vcpu, VCPU_EXREG_PDPTR))
-		kvm_x86_ops.cache_reg(vcpu, VCPU_EXREG_PDPTR);
+		static_call(kvm_x86_cache_reg)(vcpu, VCPU_EXREG_PDPTR);
 
 	return vcpu->arch.walk_mmu->pdptrs[index];
 }
@@ -119,7 +119,7 @@ static inline ulong kvm_read_cr0_bits(struct kvm_vcpu *vcpu, ulong mask)
 	ulong tmask = mask & KVM_POSSIBLE_CR0_GUEST_BITS;
 	if ((tmask & vcpu->arch.cr0_guest_owned_bits) &&
 	    !kvm_register_is_available(vcpu, VCPU_EXREG_CR0))
-		kvm_x86_ops.cache_reg(vcpu, VCPU_EXREG_CR0);
+		static_call(kvm_x86_cache_reg)(vcpu, VCPU_EXREG_CR0);
 	return vcpu->arch.cr0 & mask;
 }
 
@@ -133,14 +133,14 @@ static inline ulong kvm_read_cr4_bits(struct kvm_vcpu *vcpu, ulong mask)
 	ulong tmask = mask & KVM_POSSIBLE_CR4_GUEST_BITS;
 	if ((tmask & vcpu->arch.cr4_guest_owned_bits) &&
 	    !kvm_register_is_available(vcpu, VCPU_EXREG_CR4))
-		kvm_x86_ops.cache_reg(vcpu, VCPU_EXREG_CR4);
+		static_call(kvm_x86_cache_reg)(vcpu, VCPU_EXREG_CR4);
 	return vcpu->arch.cr4 & mask;
 }
 
 static inline ulong kvm_read_cr3(struct kvm_vcpu *vcpu)
 {
 	if (!kvm_register_is_available(vcpu, VCPU_EXREG_CR3))
-		kvm_x86_ops.cache_reg(vcpu, VCPU_EXREG_CR3);
+		static_call(kvm_x86_cache_reg)(vcpu, VCPU_EXREG_CR3);
 	return vcpu->arch.cr3;
 }
 
arch/x86/kvm/lapic.c +15 -15
index 3136e05..41fdb85 100644
@@ -484,7 +484,7 @@ static inline void apic_clear_irr(int vec, struct kvm_lapic *apic)
 	if (unlikely(vcpu->arch.apicv_active)) {
 		/* need to update RVI */
 		kvm_lapic_clear_vector(vec, apic->regs + APIC_IRR);
-		kvm_x86_ops.hwapic_irr_update(vcpu,
+		static_call(kvm_x86_hwapic_irr_update)(vcpu,
 				apic_find_highest_irr(apic));
 	} else {
 		apic->irr_pending = false;
@@ -515,7 +515,7 @@ static inline void apic_set_isr(int vec, struct kvm_lapic *apic)
 	 * just set SVI.
 	 */
 	if (unlikely(vcpu->arch.apicv_active))
-		kvm_x86_ops.hwapic_isr_update(vcpu, vec);
+		static_call(kvm_x86_hwapic_isr_update)(vcpu, vec);
 	else {
 		++apic->isr_count;
 		BUG_ON(apic->isr_count > MAX_APIC_VECTOR);
@@ -563,8 +563,8 @@ static inline void apic_clear_isr(int vec, struct kvm_lapic *apic)
 	 * and must be left alone.
 	 */
 	if (unlikely(vcpu->arch.apicv_active))
-		kvm_x86_ops.hwapic_isr_update(vcpu,
-					       apic_find_highest_isr(apic));
+		static_call(kvm_x86_hwapic_isr_update)(vcpu,
+						apic_find_highest_isr(apic));
 	else {
 		--apic->isr_count;
 		BUG_ON(apic->isr_count < 0);
@@ -701,7 +701,7 @@ static int apic_has_interrupt_for_ppr(struct kvm_lapic *apic, u32 ppr)
 {
 	int highest_irr;
 	if (apic->vcpu->arch.apicv_active)
-		highest_irr = kvm_x86_ops.sync_pir_to_irr(apic->vcpu);
+		highest_irr = static_call(kvm_x86_sync_pir_to_irr)(apic->vcpu);
 	else
 		highest_irr = apic_find_highest_irr(apic);
 	if (highest_irr == -1 || (highest_irr & 0xF0) <= ppr)
@@ -1090,7 +1090,7 @@ static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
 						       apic->regs + APIC_TMR);
 		}
 
-		if (kvm_x86_ops.deliver_posted_interrupt(vcpu, vector)) {
+		if (static_call(kvm_x86_deliver_posted_interrupt)(vcpu, vector)) {
 			kvm_lapic_set_irr(vector, apic);
 			kvm_make_request(KVM_REQ_EVENT, vcpu);
 			kvm_vcpu_kick(vcpu);
@@ -1814,7 +1814,7 @@ static void cancel_hv_timer(struct kvm_lapic *apic)
 {
 	WARN_ON(preemptible());
 	WARN_ON(!apic->lapic_timer.hv_timer_in_use);
-	kvm_x86_ops.cancel_hv_timer(apic->vcpu);
+	static_call(kvm_x86_cancel_hv_timer)(apic->vcpu);
 	apic->lapic_timer.hv_timer_in_use = false;
 }
 
@@ -1831,7 +1831,7 @@ static bool start_hv_timer(struct kvm_lapic *apic)
 	if (!ktimer->tscdeadline)
 		return false;
 
-	if (kvm_x86_ops.set_hv_timer(vcpu, ktimer->tscdeadline, &expired))
+	if (static_call(kvm_x86_set_hv_timer)(vcpu, ktimer->tscdeadline, &expired))
 		return false;
 
 	ktimer->hv_timer_in_use = true;
@@ -2261,7 +2261,7 @@ void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value)
 		kvm_apic_set_x2apic_id(apic, vcpu->vcpu_id);
 
 	if ((old_value ^ value) & (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE))
-		kvm_x86_ops.set_virtual_apic_mode(vcpu);
+		static_call(kvm_x86_set_virtual_apic_mode)(vcpu);
 
 	apic->base_address = apic->vcpu->arch.apic_base &
 			     MSR_IA32_APICBASE_BASE;
@@ -2338,9 +2338,9 @@ void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event)
 	vcpu->arch.pv_eoi.msr_val = 0;
 	apic_update_ppr(apic);
 	if (vcpu->arch.apicv_active) {
-		kvm_x86_ops.apicv_post_state_restore(vcpu);
-		kvm_x86_ops.hwapic_irr_update(vcpu, -1);
-		kvm_x86_ops.hwapic_isr_update(vcpu, -1);
+		static_call(kvm_x86_apicv_post_state_restore)(vcpu);
+		static_call(kvm_x86_hwapic_irr_update)(vcpu, -1);
+		static_call(kvm_x86_hwapic_isr_update)(vcpu, -1);
 	}
 
 	vcpu->arch.apic_arb_prio = 0;
@@ -2601,10 +2601,10 @@ int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s)
 	kvm_apic_update_apicv(vcpu);
 	apic->highest_isr_cache = -1;
 	if (vcpu->arch.apicv_active) {
-		kvm_x86_ops.apicv_post_state_restore(vcpu);
-		kvm_x86_ops.hwapic_irr_update(vcpu,
+		static_call(kvm_x86_apicv_post_state_restore)(vcpu);
+		static_call(kvm_x86_hwapic_irr_update)(vcpu,
 				apic_find_highest_irr(apic));
-		kvm_x86_ops.hwapic_isr_update(vcpu,
+		static_call(kvm_x86_hwapic_isr_update)(vcpu,
 				apic_find_highest_isr(apic));
 	}
 	kvm_make_request(KVM_REQ_EVENT, vcpu);
arch/x86/kvm/mmu.h +3 -3
index 9c4a9c8..6ec46b5 100644
@@ -95,7 +95,7 @@ static inline void kvm_mmu_load_pgd(struct kvm_vcpu *vcpu)
 	if (!VALID_PAGE(root_hpa))
 		return;
 
-	kvm_x86_ops.load_mmu_pgd(vcpu, root_hpa | kvm_get_active_pcid(vcpu),
+	static_call(kvm_x86_load_mmu_pgd)(vcpu, root_hpa | kvm_get_active_pcid(vcpu),
 				 vcpu->arch.mmu->shadow_root_level);
 }
 
@@ -167,8 +167,8 @@ static inline u8 permission_fault(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
 				  unsigned pte_access, unsigned pte_pkey,
 				  unsigned pfec)
 {
-	int cpl = kvm_x86_ops.get_cpl(vcpu);
-	unsigned long rflags = kvm_x86_ops.get_rflags(vcpu);
+	int cpl = static_call(kvm_x86_get_cpl)(vcpu);
+	unsigned long rflags = static_call(kvm_x86_get_rflags)(vcpu);
 
 	/*
 	 * If CPL < 3, SMAP prevention are disabled if EFLAGS.AC = 1.
arch/x86/kvm/mmu/mmu.c +8 -7
index c478904..e9a6e5c 100644
@@ -190,7 +190,7 @@ static void kvm_flush_remote_tlbs_with_range(struct kvm *kvm,
 	int ret = -ENOTSUPP;
 
 	if (range && kvm_x86_ops.tlb_remote_flush_with_range)
-		ret = kvm_x86_ops.tlb_remote_flush_with_range(kvm, range);
+		ret = static_call(kvm_x86_tlb_remote_flush_with_range)(kvm, range);
 
 	if (ret)
 		kvm_flush_remote_tlbs(kvm);
@@ -1283,8 +1283,9 @@ void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
 				gfn_t gfn_offset, unsigned long mask)
 {
 	if (kvm_x86_ops.enable_log_dirty_pt_masked)
-		kvm_x86_ops.enable_log_dirty_pt_masked(kvm, slot, gfn_offset,
-				mask);
+		static_call(kvm_x86_enable_log_dirty_pt_masked)(kvm, slot,
+								gfn_offset,
+								mask);
 	else
 		kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask);
 }
@@ -1292,7 +1293,7 @@ void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
 int kvm_cpu_dirty_log_size(void)
 {
 	if (kvm_x86_ops.cpu_dirty_log_size)
-		return kvm_x86_ops.cpu_dirty_log_size();
+		return static_call(kvm_x86_cpu_dirty_log_size)();
 
 	return 0;
 }
@@ -4804,7 +4805,7 @@ int kvm_mmu_load(struct kvm_vcpu *vcpu)
 	if (r)
 		goto out;
 	kvm_mmu_load_pgd(vcpu);
-	kvm_x86_ops.tlb_flush_current(vcpu);
+	static_call(kvm_x86_tlb_flush_current)(vcpu);
 out:
 	return r;
 }
@@ -5118,7 +5119,7 @@ void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
 		if (is_noncanonical_address(gva, vcpu))
 			return;
 
-		kvm_x86_ops.tlb_flush_gva(vcpu, gva);
+		static_call(kvm_x86_tlb_flush_gva)(vcpu, gva);
 	}
 
 	if (!mmu->invlpg)
@@ -5175,7 +5176,7 @@ void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid)
 	}
 
 	if (tlb_flush)
-		kvm_x86_ops.tlb_flush_gva(vcpu, gva);
+		static_call(kvm_x86_tlb_flush_gva)(vcpu, gva);
 
 	++vcpu->stat.invlpg;
 
arch/x86/kvm/mmu/spte.c +1 -1
index c51ad54..ef55f0b 100644
@@ -120,7 +120,7 @@ int make_spte(struct kvm_vcpu *vcpu, unsigned int pte_access, int level,
 	if (level > PG_LEVEL_4K)
 		spte |= PT_PAGE_SIZE_MASK;
 	if (tdp_enabled)
-		spte |= kvm_x86_ops.get_mt_mask(vcpu, gfn,
+		spte |= static_call(kvm_x86_get_mt_mask)(vcpu, gfn,
 			kvm_is_mmio_pfn(pfn));
 
 	if (host_writable)
arch/x86/kvm/pmu.c +1 -1
index 67741d2..326c740 100644
@@ -373,7 +373,7 @@ int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)
 		return 1;
 
 	if (!(kvm_read_cr4(vcpu) & X86_CR4_PCE) &&
-	    (kvm_x86_ops.get_cpl(vcpu) != 0) &&
+	    (static_call(kvm_x86_get_cpl)(vcpu) != 0) &&
 	    (kvm_read_cr0(vcpu) & X86_CR0_PE))
 		return 1;
 
arch/x86/kvm/trace.h +2 -2
index 2de30c2..5ef2386 100644
@@ -256,7 +256,7 @@ TRACE_EVENT(name,							     \
 		__entry->guest_rip	= kvm_rip_read(vcpu);		     \
 		__entry->isa            = isa;				     \
 		__entry->vcpu_id        = vcpu->vcpu_id;		     \
-		kvm_x86_ops.get_exit_info(vcpu, &__entry->info1,	     \
+		static_call(kvm_x86_get_exit_info)(vcpu, &__entry->info1,    \
 					  &__entry->info2,		     \
 					  &__entry->intr_info,		     \
 					  &__entry->error_code);	     \
@@ -738,7 +738,7 @@ TRACE_EVENT(kvm_emulate_insn,
 		),
 
 	TP_fast_assign(
-		__entry->csbase = kvm_x86_ops.get_segment_base(vcpu, VCPU_SREG_CS);
+		__entry->csbase = static_call(kvm_x86_get_segment_base)(vcpu, VCPU_SREG_CS);
 		__entry->len = vcpu->arch.emulate_ctxt->fetch.ptr
 			       - vcpu->arch.emulate_ctxt->fetch.data;
 		__entry->rip = vcpu->arch.emulate_ctxt->_eip - __entry->len;
arch/x86/kvm/x86.c +148 -150

1,087 lines not on this page · raw message

arch/x86/kvm/x86.h +3 -3
index c5ee0f5..62f4f64 100644
@@ -98,7 +98,7 @@ static inline bool is_64_bit_mode(struct kvm_vcpu *vcpu)
 
 	if (!is_long_mode(vcpu))
 		return false;
-	kvm_x86_ops.get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
+	static_call(kvm_x86_get_cs_db_l_bits)(vcpu, &cs_db, &cs_l);
 	return cs_l;
 }
 
@@ -129,7 +129,7 @@ static inline bool mmu_is_nested(struct kvm_vcpu *vcpu)
 static inline void kvm_vcpu_flush_tlb_current(struct kvm_vcpu *vcpu)
 {
 	++vcpu->stat.tlb_flush;
-	kvm_x86_ops.tlb_flush_current(vcpu);
+	static_call(kvm_x86_tlb_flush_current)(vcpu);
 }
 
 static inline int is_pae(struct kvm_vcpu *vcpu)
@@ -244,7 +244,7 @@ static inline bool kvm_check_has_quirk(struct kvm *kvm, u64 quirk)
 
 static inline bool kvm_vcpu_latch_init(struct kvm_vcpu *vcpu)
 {
-	return is_smm(vcpu) || kvm_x86_ops.apic_init_signal_blocked(vcpu);
+	return is_smm(vcpu) || static_call(kvm_x86_apic_init_signal_blocked)(vcpu);
 }
 
 void kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip);
Signature · 1 line
2.7.4
maintainer
#9 lore raw

Re: [PATCH v2 3/3] KVM: x86: use static calls to reduce kvm_x86_ops overhead

arch/x86/include/asm/kvm_host.h void kvm_arch_free_vm(struct kvm *kvm); · line 1356

Quoted 13 lines · Jason Baron
arch/x86/include/asm/kvm_host.h +1 -1
index 5060922..9d4492b 100644
@@ -1350,7 +1350,7 @@ void kvm_arch_free_vm(struct kvm *kvm);
 static inline int kvm_arch_flush_remote_tlb(struct kvm *kvm)
 {
 	if (kvm_x86_ops.tlb_remote_flush &&
-	    !kvm_x86_ops.tlb_remote_flush(kvm))
+	    !static_call(kvm_x86_tlb_remote_flush)(kvm))
 		return 0;
 	else
 		return -ENOTSUPP;

Would you be able to use something like this?

https://lkml.kernel.org/r/[email protected]

we could also add __static_call_return1(), if that would help.

maintainer
#10 lore raw

Re: [PATCH v2 3/3] KVM: x86: use static calls to reduce kvm_x86_ops overhead

Quoted 17 lines · Peter Zijlstra

arch/x86/include/asm/kvm_host.h void kvm_arch_free_vm(struct kvm *kvm); · line 1356

Quoted 13 lines · Jason Baron
arch/x86/include/asm/kvm_host.h +1 -1
index 5060922..9d4492b 100644
@@ -1350,7 +1350,7 @@ void kvm_arch_free_vm(struct kvm *kvm);
 static inline int kvm_arch_flush_remote_tlb(struct kvm *kvm)
 {
 	if (kvm_x86_ops.tlb_remote_flush &&
-	    !kvm_x86_ops.tlb_remote_flush(kvm))
+	    !static_call(kvm_x86_tlb_remote_flush)(kvm))
 		return 0;
 	else
 		return -ENOTSUPP;

Would you be able to use something like this?

Quoted 3 lines

https://lkml.kernel.org/r/[email protected]

we could also add __static_call_return1(), if that would help.

Something like so on top of the above.

arch/x86/kernel/static_call.c +5 -0
@@ -17,6 +17,7 @@ static void __ref __static_call_transfor
 	 * disp16 disp16 xorq %rax, %rax - a single 5 byte instruction that clears %rax
 	 */
 	static const u8 ret0[5] = { 0x66, 0x66, 0x48, 0x31, 0xc0 };
+	static const u8 ret1[5] = { 0xb8, 0x01, 0x00, 0x00, 0x00 }; /* mov $0x1,%eax */
 	int size = CALL_INSN_SIZE;
 	const void *emulate = NULL;
 	const void *code;
@@ -27,7 +28,11 @@ static void __ref __static_call_transfor
 		if (func == &__static_call_return0) {
 			emulate = code;
 			code = ret0;
+		} else if (func == &__static_call_return1) {
+			emulate = code;
+			code = ret1;
 		}
+
 		break;
 
 	case NOP:
include/linux/static_call.h +3 -0
@@ -137,6 +137,7 @@ extern void arch_static_call_transform(v
 #ifdef CONFIG_HAVE_STATIC_CALL_INLINE
 
 extern long __static_call_return0(void);
+extern long __static_call_return1(void);
 
 extern int __init static_call_init(void);
 
@@ -190,6 +191,7 @@ extern int static_call_text_reserved(voi
 #elif defined(CONFIG_HAVE_STATIC_CALL)
 
 static inline long __static_call_return0(void) { return 0; }
+static inline long __static_call_return0(void) { return 1; }
 
 static inline int static_call_init(void) { return 0; }
 
@@ -239,6 +241,7 @@ static inline int static_call_text_reser
 #else /* Generic implementation */
 
 static inline long __static_call_return0(void) { return 0; }
+static inline long __static_call_return0(void) { return 1; }
 
 static inline int static_call_init(void) { return 0; }
 
kernel/static_call.c +5 -0
@@ -443,6 +443,11 @@ long __static_call_return0(void)
 	return 0;
 }
 
+long __static_call_return1(void)
+{
+	return 1;
+}
+
 #ifdef CONFIG_STATIC_CALL_SELFTEST
 
 static int func_a(int x)
maintainer
#11 lore raw

Re: [PATCH v2 3/3] KVM: x86: use static calls to reduce kvm_x86_ops overhead

include/linux/static_call.h static inline int static_call_text_reser · line 244

Quoted 13 lines · Peter Zijlstra
include/linux/static_call.h +2 -0
@@ -190,6 +191,7 @@ extern int static_call_text_reserved(voi
 #elif defined(CONFIG_HAVE_STATIC_CALL)
 
 static inline long __static_call_return0(void) { return 0; }
+static inline long __static_call_return0(void) { return 1; }
 
 static inline int static_call_init(void) { return 0; }
 
@@ -239,6 +241,7 @@ static inline int static_call_text_reser
 #else /* Generic implementation */
 
 static inline long __static_call_return0(void) { return 0; }
+static inline long __static_call_return0(void) { return 1; }

Too much copy-fail... Lemme go make more tea :-)

maintainer
#12 lore raw

Re: [PATCH v2 3/3] KVM: x86: use static calls to reduce kvm_x86_ops overhead

In reply to Peter Zijlstra
Quoted 17 lines · Peter Zijlstra

arch/x86/include/asm/kvm_host.h void kvm_arch_free_vm(struct kvm *kvm); · line 1356

Quoted 13 lines · Jason Baron
arch/x86/include/asm/kvm_host.h +1 -1
index 5060922..9d4492b 100644
@@ -1350,7 +1350,7 @@ void kvm_arch_free_vm(struct kvm *kvm);
  static inline int kvm_arch_flush_remote_tlb(struct kvm *kvm)
  {
  	if (kvm_x86_ops.tlb_remote_flush &&
-	    !kvm_x86_ops.tlb_remote_flush(kvm))
+	    !static_call(kvm_x86_tlb_remote_flush)(kvm))
  		return 0;
  	else
  		return -ENOTSUPP;

Would you be able to use something like this?

Quoted 4 lines
   https://lkml.kernel.org/r/[email protected]

we could also add __static_call_return1(), if that would help.

I think I'd rather make the default callee return -ENOTSUPP directly and remove the "if" completely. So __static_call_return1() is not particularly useful here.

Paolo

maintainer Applied
#13 lore raw
In reply to Jason Baron
Quoted 4 lines · Jason Baron

Hi,

Convert kvm_x86_ops to use static_call. Shows good performance gains for cpuid loop micro-benchmark (results in patch 3/3).

Queued, thanks.

Paolo

Quoted 38 lines

Thanks,

-Jason

Changes from v1:

(quoted diff) +0 -3
-Introduce kvm-x86-ops header with eye towards using this to define
  svm_x86_ops and vmx_x86_ops in follow on patches (Paolo, Sean)
-add new patch (1/3), that adds a vmx/svm prefix to help facilitate
  svm_x86_ops and vmx_x86_ops future conversions.
-added amd perf numbres to description of patch 3/3

Jason Baron (3):
KVM: X86: append vmx/svm prefix to additional kvm_x86_ops functions
KVM: x86: introduce definitions to support static calls for kvm_x86_ops
KVM: x86: use static calls to reduce kvm_x86_ops overhead

  arch/x86/include/asm/kvm-x86-ops.h | 127 +++++++++++++++
  arch/x86/include/asm/kvm_host.h    |  21 ++-
  arch/x86/kvm/cpuid.c               |   2 +-
  arch/x86/kvm/hyperv.c              |   4 +-
  arch/x86/kvm/irq.c                 |   3 +-
  arch/x86/kvm/kvm_cache_regs.h      |  10 +-
  arch/x86/kvm/lapic.c               |  30 ++--
  arch/x86/kvm/mmu.h                 |   6 +-
  arch/x86/kvm/mmu/mmu.c             |  15 +-
  arch/x86/kvm/mmu/spte.c            |   2 +-
  arch/x86/kvm/pmu.c                 |   2 +-
  arch/x86/kvm/svm/svm.c             |  20 +--
  arch/x86/kvm/trace.h               |   4 +-
  arch/x86/kvm/vmx/nested.c          |   2 +-
  arch/x86/kvm/vmx/vmx.c             |  30 ++--
  arch/x86/kvm/vmx/vmx.h             |   2 +-
  arch/x86/kvm/x86.c                 | 307 +++++++++++++++++++------------------
  arch/x86/kvm/x86.h                 |   6 +-
  18 files changed, 369 insertions(+), 224 deletions(-)

create mode 100644 arch/x86/include/asm/kvm-x86-ops.h