Skip to contentSkip to conversation

Use static_call for kvm_x86_ops

PATCH Superseded Jason Baron 3 participants 14 messages Jan 11, 2021 – Jan 13, 2021

Subsystems (4)

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

cover Use static_call for kvm_x86_ops

Hi,

Convert kvm_x86_ops to use static_call. Shows good performance gains for cpuid loop micro-benchmark (resulsts included in patch 2).

Thanks,

-Jason

Jason Baron (2):
KVM: x86: introduce definitions to support static calls for kvm_x86_ops
KVM: x86: use static calls to reduce kvm_x86_ops overhead

 arch/x86/include/asm/kvm_host.h |  71 +++++++++-
 arch/x86/kvm/cpuid.c            |   2 +-
 arch/x86/kvm/hyperv.c           |   4 +-
 arch/x86/kvm/irq.c              |   2 +-
 arch/x86/kvm/kvm_cache_regs.h   |  10 +-
 arch/x86/kvm/lapic.c            |  28 ++--
 arch/x86/kvm/mmu.h              |   6 +-
 arch/x86/kvm/mmu/mmu.c          |  12 +-
 arch/x86/kvm/mmu/spte.c         |   2 +-
 arch/x86/kvm/pmu.c              |   2 +-
 arch/x86/kvm/trace.h            |   4 +-
 arch/x86/kvm/x86.c              | 299 ++++++++++++++++++++--------------------
 arch/x86/kvm/x86.h              |   6 +-
 13 files changed, 259 insertions(+), 189 deletions(-)
Signature · 1 line
2.7.4
#1 lore raw

PATCH 1/2 KVM: x86: introduce definitions to support static calls for kvm_x86_ops

Use static calls to improve kvm_x86_ops performance. Introduce the definitions that will be used by a subsequent patch to actualize the savings.

Note that all kvm_x86_ops are covered here except for 'pmu_ops' and 'nested ops'. I think they can be covered by static calls in a simlilar manner, but were omitted from this series to reduce scope and because I don't think they have as large of a performance impact.

Cc Paolo Bonzini <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Borislav Petkov <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Andrea Arcangeli <[email protected]>
---
 arch/x86/include/asm/kvm_host.h | 65 +++++++++++++++++++++++++++++++++++++++++
 arch/x86/kvm/x86.c              |  5 ++++
 2 files changed, 70 insertions(+)
arch/x86/include/asm/kvm_host.h +65 -0
index 3ab7b46..e947522 100644
@@ -1087,6 +1087,65 @@ static inline u16 kvm_lapic_irq_dest_mode(bool dest_mode_logical)
 	return dest_mode_logical ? APIC_DEST_LOGICAL : APIC_DEST_PHYSICAL;
 }
 
+/*
+ * static calls cover all kvm_x86_ops except for functions under pmu_ops and
+ * nested_ops.
+ */
+#define FOREACH_KVM_X86_OPS(F) \
+	F(hardware_enable); F(hardware_disable); F(hardware_unsetup);	       \
+	F(cpu_has_accelerated_tpr); F(has_emulated_msr);		       \
+	F(vcpu_after_set_cpuid); F(vm_init); F(vm_destroy); F(vcpu_create);    \
+	F(vcpu_free); F(vcpu_reset); F(prepare_guest_switch); F(vcpu_load);    \
+	F(vcpu_put); F(update_exception_bitmap); F(get_msr); F(set_msr);       \
+	F(get_segment_base); F(get_segment); F(get_cpl); F(set_segment);       \
+	F(get_cs_db_l_bits); F(set_cr0); F(is_valid_cr4); F(set_cr4);	       \
+	F(set_efer); F(get_idt); F(set_idt); F(get_gdt); F(set_gdt);	       \
+	F(sync_dirty_debug_regs); F(set_dr7); F(cache_reg); F(get_rflags);     \
+	F(set_rflags); F(tlb_flush_all); F(tlb_flush_current);		       \
+	F(tlb_remote_flush); F(tlb_remote_flush_with_range); F(tlb_flush_gva); \
+	F(tlb_flush_guest); F(run); F(handle_exit);			       \
+	F(skip_emulated_instruction); F(update_emulated_instruction);	       \
+	F(set_interrupt_shadow); F(get_interrupt_shadow); F(patch_hypercall);  \
+	F(set_irq); F(set_nmi); F(queue_exception); F(cancel_injection);       \
+	F(interrupt_allowed); F(nmi_allowed); F(get_nmi_mask); F(set_nmi_mask);\
+	F(enable_nmi_window); F(enable_irq_window); F(update_cr8_intercept);   \
+	F(check_apicv_inhibit_reasons); F(pre_update_apicv_exec_ctrl);	       \
+	F(refresh_apicv_exec_ctrl); F(hwapic_irr_update); F(hwapic_isr_update);\
+	F(guest_apic_has_interrupt); F(load_eoi_exitmap);		       \
+	F(set_virtual_apic_mode); F(set_apic_access_page_addr);		       \
+	F(deliver_posted_interrupt); F(sync_pir_to_irr); F(set_tss_addr);      \
+	F(set_identity_map_addr); F(get_mt_mask); F(load_mmu_pgd);	       \
+	F(has_wbinvd_exit); F(write_l1_tsc_offset); F(get_exit_info);	       \
+	F(check_intercept); F(handle_exit_irqoff); F(request_immediate_exit);  \
+	F(sched_in); F(slot_enable_log_dirty); F(slot_disable_log_dirty);      \
+	F(flush_log_dirty); F(enable_log_dirty_pt_masked);		       \
+	F(cpu_dirty_log_size); F(pre_block); F(post_block); F(vcpu_blocking);  \
+	F(vcpu_unblocking); F(update_pi_irte); F(apicv_post_state_restore);    \
+	F(dy_apicv_has_pending_interrupt); F(set_hv_timer); F(cancel_hv_timer);\
+	F(setup_mce); F(smi_allowed); F(pre_enter_smm); F(pre_leave_smm);      \
+	F(enable_smi_window); F(mem_enc_op); F(mem_enc_reg_region);	       \
+	F(mem_enc_unreg_region); F(get_msr_feature);			       \
+	F(can_emulate_instruction); F(apic_init_signal_blocked);	       \
+	F(enable_direct_tlbflush); F(migrate_timers); F(msr_filter_changed);   \
+	F(complete_emulated_msr)
+
+#define DEFINE_KVM_OPS_STATIC_CALL(func)	\
+	DEFINE_STATIC_CALL_NULL(kvm_x86_##func,	\
+				*(((struct kvm_x86_ops *)0)->func))
+#define DEFINE_KVM_OPS_STATIC_CALLS() \
+	FOREACH_KVM_X86_OPS(DEFINE_KVM_OPS_STATIC_CALL)
+
+#define DECLARE_KVM_OPS_STATIC_CALL(func)	\
+	DECLARE_STATIC_CALL(kvm_x86_##func,	\
+			    *(((struct kvm_x86_ops *)0)->func))
+#define DECLARE_KVM_OPS_STATIC_CALLS()		\
+	FOREACH_KVM_X86_OPS(DECLARE_KVM_OPS_STATIC_CALL)
+
+#define KVM_OPS_STATIC_CALL_UPDATE(func)	\
+	static_call_update(kvm_x86_##func, kvm_x86_ops.func)
+#define KVM_OPS_STATIC_CALL_UPDATES()		\
+	FOREACH_KVM_X86_OPS(KVM_OPS_STATIC_CALL_UPDATE)
+
 struct kvm_x86_ops {
 	int (*hardware_enable)(void);
 	void (*hardware_disable)(void);
@@ -1326,6 +1385,12 @@ extern u64 __read_mostly host_efer;
 extern bool __read_mostly allow_smaller_maxphyaddr;
 extern struct kvm_x86_ops kvm_x86_ops;
 
+DECLARE_KVM_OPS_STATIC_CALLS();
+static inline void kvm_ops_static_call_update(void)
+{
+	KVM_OPS_STATIC_CALL_UPDATES();
+}
+
 #define __KVM_HAVE_ARCH_VM_ALLOC
 static inline struct kvm *kvm_arch_alloc_vm(void)
 {
arch/x86/kvm/x86.c +5 -0
index 3f7c1fc..6ae32ab 100644
@@ -113,6 +113,11 @@ static int sync_regs(struct kvm_vcpu *vcpu);
 struct kvm_x86_ops kvm_x86_ops __read_mostly;
 EXPORT_SYMBOL_GPL(kvm_x86_ops);
 
+DEFINE_KVM_OPS_STATIC_CALLS();
+EXPORT_STATIC_CALL_GPL(kvm_x86_get_cs_db_l_bits);
+EXPORT_STATIC_CALL_GPL(kvm_x86_cache_reg);
+EXPORT_STATIC_CALL_GPL(kvm_x86_tlb_flush_current);
+
 static bool __read_mostly ignore_msrs = 0;
 module_param(ignore_msrs, bool, S_IRUGO | S_IWUSR);
 
Signature · 1 line
2.7.4
#12 lore raw

PATCH 2/2 KVM: x86: use static calls to reduce kvm_x86_ops overhead

In reply to Jason Baron

Convert kvm_x86_ops to use static calls. Note that all kvm_x86_ops are covered here except for 'pmu_ops and 'nested ops'.

Here are some numbers running cpuid in a loop and measured timing in the vm (lower is better).

cpuid loop (1 million calls(avg. of 5 runs)):

|default |mitigations=off
---------------------------------------

vanilla    |.671s      |.486s
static call|.573s(-15%)|.458s(-6%)
Cc Paolo Bonzini <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Borislav Petkov <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Andrea Arcangeli <[email protected]>
---
 arch/x86/include/asm/kvm_host.h |   6 +-
 arch/x86/kvm/cpuid.c            |   2 +-
 arch/x86/kvm/hyperv.c           |   4 +-
 arch/x86/kvm/irq.c              |   2 +-
 arch/x86/kvm/kvm_cache_regs.h   |  10 +-
 arch/x86/kvm/lapic.c            |  28 ++--
 arch/x86/kvm/mmu.h              |   6 +-
 arch/x86/kvm/mmu/mmu.c          |  12 +-
 arch/x86/kvm/mmu/spte.c         |   2 +-
 arch/x86/kvm/pmu.c              |   2 +-
 arch/x86/kvm/trace.h            |   4 +-
 arch/x86/kvm/x86.c              | 294 ++++++++++++++++++++--------------------
 arch/x86/kvm/x86.h              |   6 +-
 13 files changed, 189 insertions(+), 189 deletions(-)
arch/x86/include/asm/kvm_host.h +3 -3
index e947522..2d47cff 100644
@@ -1402,7 +1402,7 @@ void kvm_arch_free_vm(struct kvm *kvm);
 static inline int kvm_arch_flush_remote_tlb(struct kvm *kvm)
 {
 	if (kvm_x86_ops.tlb_remote_flush &&
-	    !kvm_x86_ops.tlb_remote_flush(kvm))
+	    !static_call(kvm_x86_tlb_remote_flush)(kvm))
 		return 0;
 	else
 		return -ENOTSUPP;
@@ -1793,13 +1793,13 @@ static inline bool kvm_irq_is_postable(struct kvm_lapic_irq *irq)
 static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu)
 {
 	if (kvm_x86_ops.vcpu_blocking)
-		kvm_x86_ops.vcpu_blocking(vcpu);
+		static_call(kvm_x86_vcpu_blocking)(vcpu);
 }
 
 static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu)
 {
 	if (kvm_x86_ops.vcpu_unblocking)
-		kvm_x86_ops.vcpu_unblocking(vcpu);
+		static_call(kvm_x86_vcpu_unblocking)(vcpu);
 }
 
 static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) {}
arch/x86/kvm/cpuid.c +1 -1
index 13036cf..f951d18 100644
@@ -182,7 +182,7 @@ static void kvm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu)
 	vcpu->arch.cr3_lm_rsvd_bits = rsvd_bits(cpuid_maxphyaddr(vcpu), 63);
 
 	/* Invoke the vendor callback only after the above state is updated. */
-	kvm_x86_ops.vcpu_after_set_cpuid(vcpu);
+	static_call(kvm_x86_vcpu_after_set_cpuid)(vcpu);
 }
 
 static int is_efer_nx(void)
arch/x86/kvm/hyperv.c +2 -2
index 922c69d..5c45d80 100644
@@ -1154,7 +1154,7 @@ static int kvm_hv_set_msr_pw(struct kvm_vcpu *vcpu, u32 msr, u64 data,
 		addr = gfn_to_hva(kvm, gfn);
 		if (kvm_is_error_hva(addr))
 			return 1;
-		kvm_x86_ops.patch_hypercall(vcpu, instructions);
+		static_call(kvm_x86_patch_hypercall)(vcpu, instructions);
 		((unsigned char *)instructions)[3] = 0xc3; /* ret */
 		if (__copy_to_user((void __user *)addr, instructions, 4))
 			return 1;
@@ -1745,7 +1745,7 @@ int kvm_hv_hypercall(struct kvm_vcpu *vcpu)
 	 * hypercall generates UD from non zero cpl and real mode
 	 * per HYPER-V spec
 	 */
-	if (kvm_x86_ops.get_cpl(vcpu) != 0 || !is_protmode(vcpu)) {
+	if (static_call(kvm_x86_get_cpl)(vcpu) != 0 || !is_protmode(vcpu)) {
 		kvm_queue_exception(vcpu, UD_VECTOR);
 		return 1;
 	}
arch/x86/kvm/irq.c +1 -1
index 814698e..67c634a 100644
@@ -144,7 +144,7 @@ void __kvm_migrate_timers(struct kvm_vcpu *vcpu)
 	__kvm_migrate_apic_timer(vcpu);
 	__kvm_migrate_pit_timer(vcpu);
 	if (kvm_x86_ops.migrate_timers)
-		kvm_x86_ops.migrate_timers(vcpu);
+		static_call(kvm_x86_migrate_timers)(vcpu);
 }
 
 bool kvm_arch_irqfd_allowed(struct kvm *kvm, struct kvm_irqfd *args)
arch/x86/kvm/kvm_cache_regs.h +5 -5
index f15bc16..61d2965 100644
@@ -69,7 +69,7 @@ static inline unsigned long kvm_register_read(struct kvm_vcpu *vcpu, int reg)
 		return 0;
 
 	if (!kvm_register_is_available(vcpu, reg))
-		kvm_x86_ops.cache_reg(vcpu, reg);
+		static_call(kvm_x86_cache_reg)(vcpu, reg);
 
 	return vcpu->arch.regs[reg];
 }
@@ -109,7 +109,7 @@ static inline u64 kvm_pdptr_read(struct kvm_vcpu *vcpu, int index)
 	might_sleep();  /* on svm */
 
 	if (!kvm_register_is_available(vcpu, VCPU_EXREG_PDPTR))
-		kvm_x86_ops.cache_reg(vcpu, VCPU_EXREG_PDPTR);
+		static_call(kvm_x86_cache_reg)(vcpu, VCPU_EXREG_PDPTR);
 
 	return vcpu->arch.walk_mmu->pdptrs[index];
 }
@@ -119,7 +119,7 @@ static inline ulong kvm_read_cr0_bits(struct kvm_vcpu *vcpu, ulong mask)
 	ulong tmask = mask & KVM_POSSIBLE_CR0_GUEST_BITS;
 	if ((tmask & vcpu->arch.cr0_guest_owned_bits) &&
 	    !kvm_register_is_available(vcpu, VCPU_EXREG_CR0))
-		kvm_x86_ops.cache_reg(vcpu, VCPU_EXREG_CR0);
+		static_call(kvm_x86_cache_reg)(vcpu, VCPU_EXREG_CR0);
 	return vcpu->arch.cr0 & mask;
 }
 
@@ -133,14 +133,14 @@ static inline ulong kvm_read_cr4_bits(struct kvm_vcpu *vcpu, ulong mask)
 	ulong tmask = mask & KVM_POSSIBLE_CR4_GUEST_BITS;
 	if ((tmask & vcpu->arch.cr4_guest_owned_bits) &&
 	    !kvm_register_is_available(vcpu, VCPU_EXREG_CR4))
-		kvm_x86_ops.cache_reg(vcpu, VCPU_EXREG_CR4);
+		static_call(kvm_x86_cache_reg)(vcpu, VCPU_EXREG_CR4);
 	return vcpu->arch.cr4 & mask;
 }
 
 static inline ulong kvm_read_cr3(struct kvm_vcpu *vcpu)
 {
 	if (!kvm_register_is_available(vcpu, VCPU_EXREG_CR3))
-		kvm_x86_ops.cache_reg(vcpu, VCPU_EXREG_CR3);
+		static_call(kvm_x86_cache_reg)(vcpu, VCPU_EXREG_CR3);
 	return vcpu->arch.cr3;
 }
 
arch/x86/kvm/lapic.c +14 -14
index 3136e05..4c56389 100644
@@ -484,7 +484,7 @@ static inline void apic_clear_irr(int vec, struct kvm_lapic *apic)
 	if (unlikely(vcpu->arch.apicv_active)) {
 		/* need to update RVI */
 		kvm_lapic_clear_vector(vec, apic->regs + APIC_IRR);
-		kvm_x86_ops.hwapic_irr_update(vcpu,
+		static_call(kvm_x86_hwapic_irr_update)(vcpu,
 				apic_find_highest_irr(apic));
 	} else {
 		apic->irr_pending = false;
@@ -515,7 +515,7 @@ static inline void apic_set_isr(int vec, struct kvm_lapic *apic)
 	 * just set SVI.
 	 */
 	if (unlikely(vcpu->arch.apicv_active))
-		kvm_x86_ops.hwapic_isr_update(vcpu, vec);
+		static_call(kvm_x86_hwapic_isr_update)(vcpu, vec);
 	else {
 		++apic->isr_count;
 		BUG_ON(apic->isr_count > MAX_APIC_VECTOR);
@@ -563,7 +563,7 @@ static inline void apic_clear_isr(int vec, struct kvm_lapic *apic)
 	 * and must be left alone.
 	 */
 	if (unlikely(vcpu->arch.apicv_active))
-		kvm_x86_ops.hwapic_isr_update(vcpu,
+		static_call(kvm_x86_hwapic_isr_update)(vcpu,
 					       apic_find_highest_isr(apic));
 	else {
 		--apic->isr_count;
@@ -701,7 +701,7 @@ static int apic_has_interrupt_for_ppr(struct kvm_lapic *apic, u32 ppr)
 {
 	int highest_irr;
 	if (apic->vcpu->arch.apicv_active)
-		highest_irr = kvm_x86_ops.sync_pir_to_irr(apic->vcpu);
+		highest_irr = static_call(kvm_x86_sync_pir_to_irr)(apic->vcpu);
 	else
 		highest_irr = apic_find_highest_irr(apic);
 	if (highest_irr == -1 || (highest_irr & 0xF0) <= ppr)
@@ -1090,7 +1090,7 @@ static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
 						       apic->regs + APIC_TMR);
 		}
 
-		if (kvm_x86_ops.deliver_posted_interrupt(vcpu, vector)) {
+		if (static_call(kvm_x86_deliver_posted_interrupt)(vcpu, vector)) {
 			kvm_lapic_set_irr(vector, apic);
 			kvm_make_request(KVM_REQ_EVENT, vcpu);
 			kvm_vcpu_kick(vcpu);
@@ -1814,7 +1814,7 @@ static void cancel_hv_timer(struct kvm_lapic *apic)
 {
 	WARN_ON(preemptible());
 	WARN_ON(!apic->lapic_timer.hv_timer_in_use);
-	kvm_x86_ops.cancel_hv_timer(apic->vcpu);
+	static_call(kvm_x86_cancel_hv_timer)(apic->vcpu);
 	apic->lapic_timer.hv_timer_in_use = false;
 }
 
@@ -1831,7 +1831,7 @@ static bool start_hv_timer(struct kvm_lapic *apic)
 	if (!ktimer->tscdeadline)
 		return false;
 
-	if (kvm_x86_ops.set_hv_timer(vcpu, ktimer->tscdeadline, &expired))
+	if (static_call(kvm_x86_set_hv_timer)(vcpu, ktimer->tscdeadline, &expired))
 		return false;
 
 	ktimer->hv_timer_in_use = true;
@@ -2261,7 +2261,7 @@ void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value)
 		kvm_apic_set_x2apic_id(apic, vcpu->vcpu_id);
 
 	if ((old_value ^ value) & (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE))
-		kvm_x86_ops.set_virtual_apic_mode(vcpu);
+		static_call(kvm_x86_set_virtual_apic_mode)(vcpu);
 
 	apic->base_address = apic->vcpu->arch.apic_base &
 			     MSR_IA32_APICBASE_BASE;
@@ -2338,9 +2338,9 @@ void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event)
 	vcpu->arch.pv_eoi.msr_val = 0;
 	apic_update_ppr(apic);
 	if (vcpu->arch.apicv_active) {
-		kvm_x86_ops.apicv_post_state_restore(vcpu);
-		kvm_x86_ops.hwapic_irr_update(vcpu, -1);
-		kvm_x86_ops.hwapic_isr_update(vcpu, -1);
+		static_call(kvm_x86_apicv_post_state_restore)(vcpu);
+		static_call(kvm_x86_hwapic_irr_update)(vcpu, -1);
+		static_call(kvm_x86_hwapic_isr_update)(vcpu, -1);
 	}
 
 	vcpu->arch.apic_arb_prio = 0;
@@ -2601,10 +2601,10 @@ int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s)
 	kvm_apic_update_apicv(vcpu);
 	apic->highest_isr_cache = -1;
 	if (vcpu->arch.apicv_active) {
-		kvm_x86_ops.apicv_post_state_restore(vcpu);
-		kvm_x86_ops.hwapic_irr_update(vcpu,
+		static_call(kvm_x86_apicv_post_state_restore)(vcpu);
+		static_call(kvm_x86_hwapic_irr_update)(vcpu,
 				apic_find_highest_irr(apic));
-		kvm_x86_ops.hwapic_isr_update(vcpu,
+		static_call(kvm_x86_hwapic_isr_update)(vcpu,
 				apic_find_highest_isr(apic));
 	}
 	kvm_make_request(KVM_REQ_EVENT, vcpu);
arch/x86/kvm/mmu.h +3 -3
index 9c4a9c8..6ec46b5 100644
@@ -95,7 +95,7 @@ static inline void kvm_mmu_load_pgd(struct kvm_vcpu *vcpu)
 	if (!VALID_PAGE(root_hpa))
 		return;
 
-	kvm_x86_ops.load_mmu_pgd(vcpu, root_hpa | kvm_get_active_pcid(vcpu),
+	static_call(kvm_x86_load_mmu_pgd)(vcpu, root_hpa | kvm_get_active_pcid(vcpu),
 				 vcpu->arch.mmu->shadow_root_level);
 }
 
@@ -167,8 +167,8 @@ static inline u8 permission_fault(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
 				  unsigned pte_access, unsigned pte_pkey,
 				  unsigned pfec)
 {
-	int cpl = kvm_x86_ops.get_cpl(vcpu);
-	unsigned long rflags = kvm_x86_ops.get_rflags(vcpu);
+	int cpl = static_call(kvm_x86_get_cpl)(vcpu);
+	unsigned long rflags = static_call(kvm_x86_get_rflags)(vcpu);
 
 	/*
 	 * If CPL < 3, SMAP prevention are disabled if EFLAGS.AC = 1.
arch/x86/kvm/mmu/mmu.c +6 -6
index c478904..96bb1b1 100644
@@ -190,7 +190,7 @@ static void kvm_flush_remote_tlbs_with_range(struct kvm *kvm,
 	int ret = -ENOTSUPP;
 
 	if (range && kvm_x86_ops.tlb_remote_flush_with_range)
-		ret = kvm_x86_ops.tlb_remote_flush_with_range(kvm, range);
+		ret = static_call(kvm_x86_tlb_remote_flush_with_range)(kvm, range);
 
 	if (ret)
 		kvm_flush_remote_tlbs(kvm);
@@ -1283,7 +1283,7 @@ void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
 				gfn_t gfn_offset, unsigned long mask)
 {
 	if (kvm_x86_ops.enable_log_dirty_pt_masked)
-		kvm_x86_ops.enable_log_dirty_pt_masked(kvm, slot, gfn_offset,
+		static_call(kvm_x86_enable_log_dirty_pt_masked)(kvm, slot, gfn_offset,
 				mask);
 	else
 		kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask);
@@ -1292,7 +1292,7 @@ void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
 int kvm_cpu_dirty_log_size(void)
 {
 	if (kvm_x86_ops.cpu_dirty_log_size)
-		return kvm_x86_ops.cpu_dirty_log_size();
+		return static_call(kvm_x86_cpu_dirty_log_size)();
 
 	return 0;
 }
@@ -4804,7 +4804,7 @@ int kvm_mmu_load(struct kvm_vcpu *vcpu)
 	if (r)
 		goto out;
 	kvm_mmu_load_pgd(vcpu);
-	kvm_x86_ops.tlb_flush_current(vcpu);
+	static_call(kvm_x86_tlb_flush_current)(vcpu);
 out:
 	return r;
 }
@@ -5118,7 +5118,7 @@ void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
 		if (is_noncanonical_address(gva, vcpu))
 			return;
 
-		kvm_x86_ops.tlb_flush_gva(vcpu, gva);
+		static_call(kvm_x86_tlb_flush_gva)(vcpu, gva);
 	}
 
 	if (!mmu->invlpg)
@@ -5175,7 +5175,7 @@ void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid)
 	}
 
 	if (tlb_flush)
-		kvm_x86_ops.tlb_flush_gva(vcpu, gva);
+		static_call(kvm_x86_tlb_flush_gva)(vcpu, gva);
 
 	++vcpu->stat.invlpg;
 
arch/x86/kvm/mmu/spte.c +1 -1
index c51ad54..ef55f0b 100644
@@ -120,7 +120,7 @@ int make_spte(struct kvm_vcpu *vcpu, unsigned int pte_access, int level,
 	if (level > PG_LEVEL_4K)
 		spte |= PT_PAGE_SIZE_MASK;
 	if (tdp_enabled)
-		spte |= kvm_x86_ops.get_mt_mask(vcpu, gfn,
+		spte |= static_call(kvm_x86_get_mt_mask)(vcpu, gfn,
 			kvm_is_mmio_pfn(pfn));
 
 	if (host_writable)
arch/x86/kvm/pmu.c +1 -1
index 67741d2..326c740 100644
@@ -373,7 +373,7 @@ int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)
 		return 1;
 
 	if (!(kvm_read_cr4(vcpu) & X86_CR4_PCE) &&
-	    (kvm_x86_ops.get_cpl(vcpu) != 0) &&
+	    (static_call(kvm_x86_get_cpl)(vcpu) != 0) &&
 	    (kvm_read_cr0(vcpu) & X86_CR0_PE))
 		return 1;
 
arch/x86/kvm/trace.h +2 -2
index 2de30c2..5ef2386 100644
@@ -256,7 +256,7 @@ TRACE_EVENT(name,							     \
 		__entry->guest_rip	= kvm_rip_read(vcpu);		     \
 		__entry->isa            = isa;				     \
 		__entry->vcpu_id        = vcpu->vcpu_id;		     \
-		kvm_x86_ops.get_exit_info(vcpu, &__entry->info1,	     \
+		static_call(kvm_x86_get_exit_info)(vcpu, &__entry->info1,    \
 					  &__entry->info2,		     \
 					  &__entry->intr_info,		     \
 					  &__entry->error_code);	     \
@@ -738,7 +738,7 @@ TRACE_EVENT(kvm_emulate_insn,
 		),
 
 	TP_fast_assign(
-		__entry->csbase = kvm_x86_ops.get_segment_base(vcpu, VCPU_SREG_CS);
+		__entry->csbase = static_call(kvm_x86_get_segment_base)(vcpu, VCPU_SREG_CS);
 		__entry->len = vcpu->arch.emulate_ctxt->fetch.ptr
 			       - vcpu->arch.emulate_ctxt->fetch.data;
 		__entry->rip = vcpu->arch.emulate_ctxt->_eip - __entry->len;
arch/x86/kvm/x86.c +147 -147

1,084 lines not on this page · raw message

arch/x86/kvm/x86.h +3 -3
index c5ee0f5..62f4f64 100644
@@ -98,7 +98,7 @@ static inline bool is_64_bit_mode(struct kvm_vcpu *vcpu)
 
 	if (!is_long_mode(vcpu))
 		return false;
-	kvm_x86_ops.get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
+	static_call(kvm_x86_get_cs_db_l_bits)(vcpu, &cs_db, &cs_l);
 	return cs_l;
 }
 
@@ -129,7 +129,7 @@ static inline bool mmu_is_nested(struct kvm_vcpu *vcpu)
 static inline void kvm_vcpu_flush_tlb_current(struct kvm_vcpu *vcpu)
 {
 	++vcpu->stat.tlb_flush;
-	kvm_x86_ops.tlb_flush_current(vcpu);
+	static_call(kvm_x86_tlb_flush_current)(vcpu);
 }
 
 static inline int is_pae(struct kvm_vcpu *vcpu)
@@ -244,7 +244,7 @@ static inline bool kvm_check_has_quirk(struct kvm *kvm, u64 quirk)
 
 static inline bool kvm_vcpu_latch_init(struct kvm_vcpu *vcpu)
 {
-	return is_smm(vcpu) || kvm_x86_ops.apic_init_signal_blocked(vcpu);
+	return is_smm(vcpu) || static_call(kvm_x86_apic_init_signal_blocked)(vcpu);
 }
 
 void kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip);
Signature · 1 line
2.7.4
maintainer
#13 lore raw
In reply to Jason Baron
Quoted 13 lines · Jason Baron

Hi,

Convert kvm_x86_ops to use static_call. Shows good performance gains for cpuid loop micro-benchmark (resulsts included in patch 2).

Thanks,

-Jason

Jason Baron (2):
KVM: x86: introduce definitions to support static calls for kvm_x86_ops
KVM: x86: use static calls to reduce kvm_x86_ops overhead

Quoted 15 lines
  arch/x86/include/asm/kvm_host.h |  71 +++++++++-
  arch/x86/kvm/cpuid.c            |   2 +-
  arch/x86/kvm/hyperv.c           |   4 +-
  arch/x86/kvm/irq.c              |   2 +-
  arch/x86/kvm/kvm_cache_regs.h   |  10 +-
  arch/x86/kvm/lapic.c            |  28 ++--
  arch/x86/kvm/mmu.h              |   6 +-
  arch/x86/kvm/mmu/mmu.c          |  12 +-
  arch/x86/kvm/mmu/spte.c         |   2 +-
  arch/x86/kvm/pmu.c              |   2 +-
  arch/x86/kvm/trace.h            |   4 +-
  arch/x86/kvm/x86.c              | 299 ++++++++++++++++++++--------------------
  arch/x86/kvm/x86.h              |   6 +-
  13 files changed, 259 insertions(+), 189 deletions(-)

Thank you thank you thank you! :)

(Except that now I have to find another task for a new KVM developer, but that's not a big deal).

Paolo

#2 lore raw

Re: [PATCH 1/2] KVM: x86: introduce definitions to support static calls for kvm_x86_ops

In reply to Jason Baron
Quoted 60 lines · Jason Baron

Use static calls to improve kvm_x86_ops performance. Introduce the definitions that will be used by a subsequent patch to actualize the savings.

Note that all kvm_x86_ops are covered here except for 'pmu_ops' and 'nested ops'. I think they can be covered by static calls in a simlilar manner, but were omitted from this series to reduce scope and because I don't think they have as large of a performance impact.

Cc Paolo Bonzini <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Borislav Petkov <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Andrea Arcangeli <[email protected]>
---
 arch/x86/include/asm/kvm_host.h | 65 +++++++++++++++++++++++++++++++++++++++++
 arch/x86/kvm/x86.c              |  5 ++++
 2 files changed, 70 insertions(+)
arch/x86/include/asm/kvm_host.h +31 -0
index 3ab7b46..e947522 100644
@@ -1087,6 +1087,65 @@ static inline u16 kvm_lapic_irq_dest_mode(bool dest_mode_logical)
 	return dest_mode_logical ? APIC_DEST_LOGICAL : APIC_DEST_PHYSICAL;
 }
 
+/*
+ * static calls cover all kvm_x86_ops except for functions under pmu_ops and
+ * nested_ops.
+ */
+#define FOREACH_KVM_X86_OPS(F) \
+	F(hardware_enable); F(hardware_disable); F(hardware_unsetup);	       \
+	F(cpu_has_accelerated_tpr); F(has_emulated_msr);		       \
+	F(vcpu_after_set_cpuid); F(vm_init); F(vm_destroy); F(vcpu_create);    \
+	F(vcpu_free); F(vcpu_reset); F(prepare_guest_switch); F(vcpu_load);    \
+	F(vcpu_put); F(update_exception_bitmap); F(get_msr); F(set_msr);       \
+	F(get_segment_base); F(get_segment); F(get_cpl); F(set_segment);       \
+	F(get_cs_db_l_bits); F(set_cr0); F(is_valid_cr4); F(set_cr4);	       \
+	F(set_efer); F(get_idt); F(set_idt); F(get_gdt); F(set_gdt);	       \
+	F(sync_dirty_debug_regs); F(set_dr7); F(cache_reg); F(get_rflags);     \
+	F(set_rflags); F(tlb_flush_all); F(tlb_flush_current);		       \
+	F(tlb_remote_flush); F(tlb_remote_flush_with_range); F(tlb_flush_gva); \
+	F(tlb_flush_guest); F(run); F(handle_exit);			       \
+	F(skip_emulated_instruction); F(update_emulated_instruction);	       \
+	F(set_interrupt_shadow); F(get_interrupt_shadow); F(patch_hypercall);  \
+	F(set_irq); F(set_nmi); F(queue_exception); F(cancel_injection);       \
+	F(interrupt_allowed); F(nmi_allowed); F(get_nmi_mask); F(set_nmi_mask);\
+	F(enable_nmi_window); F(enable_irq_window); F(update_cr8_intercept);   \
+	F(check_apicv_inhibit_reasons); F(pre_update_apicv_exec_ctrl);	       \
+	F(refresh_apicv_exec_ctrl); F(hwapic_irr_update); F(hwapic_isr_update);\
+	F(guest_apic_has_interrupt); F(load_eoi_exitmap);		       \
+	F(set_virtual_apic_mode); F(set_apic_access_page_addr);		       \
+	F(deliver_posted_interrupt); F(sync_pir_to_irr); F(set_tss_addr);      \
+	F(set_identity_map_addr); F(get_mt_mask); F(load_mmu_pgd);	       \
+	F(has_wbinvd_exit); F(write_l1_tsc_offset); F(get_exit_info);	       \
+	F(check_intercept); F(handle_exit_irqoff); F(request_immediate_exit);  \
+	F(sched_in); F(slot_enable_log_dirty); F(slot_disable_log_dirty);      \

arch/x86/include/asm/kvm_host.h static inline u16 kvm_lapic_irq_dest_mode(bool dest_mode_logical) · line 1130

Quoted 10 lines
arch/x86/include/asm/kvm_host.h +10 -0
+	F(flush_log_dirty); F(enable_log_dirty_pt_masked);		       \
+	F(cpu_dirty_log_size); F(pre_block); F(post_block); F(vcpu_blocking);  \
+	F(vcpu_unblocking); F(update_pi_irte); F(apicv_post_state_restore);    \
+	F(dy_apicv_has_pending_interrupt); F(set_hv_timer); F(cancel_hv_timer);\
+	F(setup_mce); F(smi_allowed); F(pre_enter_smm); F(pre_leave_smm);      \
+	F(enable_smi_window); F(mem_enc_op); F(mem_enc_reg_region);	       \
+	F(mem_enc_unreg_region); F(get_msr_feature);			       \
+	F(can_emulate_instruction); F(apic_init_signal_blocked);	       \
+	F(enable_direct_tlbflush); F(migrate_timers); F(msr_filter_changed);   \
+	F(complete_emulated_msr)

What about adding a dedicated .h file for this beast? Then it won't be so painful to do one function per line. As is, updates to kvm_x86_ops will be messy.

And add yet another macro layer (or maybe just tweak this one?) so that the caller controls the line ending? I suppose you could also just use a comma, but that's a bit dirty...

That would also allow using this to declare vmx_x86_ops and svm_x86_ops, which would need a comma insteat of a semi-colon. There have a been a few attempts to add a bit of automation to {vmx,svm}_x86_ops, this seems like it would be good motivation to go in a different direction and declare/define all ops, e.g. the VMX/SVM code could simply do something like:

#define DECLARE_VMX_X86_OP(func) \
	.func = vmx_##func
static struct kvm_x86_ops vmx_x86_ops __initdata = {
	.vm_size = sizeof(struct kvm_vmx),
	.vm_init = vmx_vm_init,

	.pmu_ops = &intel_pmu_ops,
	.nested_ops = &vmx_nested_ops,

	FOREACH_KVM_X86_OPS(DECLARE_VMX_X86_OP)
};
#3 lore raw

Re: [PATCH 1/2] KVM: x86: introduce definitions to support static calls for kvm_x86_ops

Quoted 66 lines · Sean Christopherson
Quoted 65 lines · Jason Baron

Use static calls to improve kvm_x86_ops performance. Introduce the definitions that will be used by a subsequent patch to actualize the savings.

Note that all kvm_x86_ops are covered here except for 'pmu_ops' and 'nested ops'. I think they can be covered by static calls in a simlilar manner, but were omitted from this series to reduce scope and because I don't think they have as large of a performance impact.

Cc Paolo Bonzini <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Borislav Petkov <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Andrea Arcangeli <[email protected]>
---
  arch/x86/include/asm/kvm_host.h | 65 +++++++++++++++++++++++++++++++++++++++++
  arch/x86/kvm/x86.c              |  5 ++++
  2 files changed, 70 insertions(+)
arch/x86/include/asm/kvm_host.h +36 -0
index 3ab7b46..e947522 100644
@@ -1087,6 +1087,65 @@ static inline u16 kvm_lapic_irq_dest_mode(bool dest_mode_logical)
  	return dest_mode_logical ? APIC_DEST_LOGICAL : APIC_DEST_PHYSICAL;
  }
 
+/*
+ * static calls cover all kvm_x86_ops except for functions under pmu_ops and
+ * nested_ops.
+ */
+#define FOREACH_KVM_X86_OPS(F) \
+	F(hardware_enable); F(hardware_disable); F(hardware_unsetup);	       \
+	F(cpu_has_accelerated_tpr); F(has_emulated_msr);		       \
+	F(vcpu_after_set_cpuid); F(vm_init); F(vm_destroy); F(vcpu_create);    \
+	F(vcpu_free); F(vcpu_reset); F(prepare_guest_switch); F(vcpu_load);    \
+	F(vcpu_put); F(update_exception_bitmap); F(get_msr); F(set_msr);       \
+	F(get_segment_base); F(get_segment); F(get_cpl); F(set_segment);       \
+	F(get_cs_db_l_bits); F(set_cr0); F(is_valid_cr4); F(set_cr4);	       \
+	F(set_efer); F(get_idt); F(set_idt); F(get_gdt); F(set_gdt);	       \
+	F(sync_dirty_debug_regs); F(set_dr7); F(cache_reg); F(get_rflags);     \
+	F(set_rflags); F(tlb_flush_all); F(tlb_flush_current);		       \
+	F(tlb_remote_flush); F(tlb_remote_flush_with_range); F(tlb_flush_gva); \
+	F(tlb_flush_guest); F(run); F(handle_exit);			       \
+	F(skip_emulated_instruction); F(update_emulated_instruction);	       \
+	F(set_interrupt_shadow); F(get_interrupt_shadow); F(patch_hypercall);  \
+	F(set_irq); F(set_nmi); F(queue_exception); F(cancel_injection);       \
+	F(interrupt_allowed); F(nmi_allowed); F(get_nmi_mask); F(set_nmi_mask);\
+	F(enable_nmi_window); F(enable_irq_window); F(update_cr8_intercept);   \
+	F(check_apicv_inhibit_reasons); F(pre_update_apicv_exec_ctrl);	       \
+	F(refresh_apicv_exec_ctrl); F(hwapic_irr_update); F(hwapic_isr_update);\
+	F(guest_apic_has_interrupt); F(load_eoi_exitmap);		       \
+	F(set_virtual_apic_mode); F(set_apic_access_page_addr);		       \
+	F(deliver_posted_interrupt); F(sync_pir_to_irr); F(set_tss_addr);      \
+	F(set_identity_map_addr); F(get_mt_mask); F(load_mmu_pgd);	       \
+	F(has_wbinvd_exit); F(write_l1_tsc_offset); F(get_exit_info);	       \
+	F(check_intercept); F(handle_exit_irqoff); F(request_immediate_exit);  \
+	F(sched_in); F(slot_enable_log_dirty); F(slot_disable_log_dirty);      \
+	F(flush_log_dirty); F(enable_log_dirty_pt_masked);		       \
+	F(cpu_dirty_log_size); F(pre_block); F(post_block); F(vcpu_blocking);  \
+	F(vcpu_unblocking); F(update_pi_irte); F(apicv_post_state_restore);    \
+	F(dy_apicv_has_pending_interrupt); F(set_hv_timer); F(cancel_hv_timer);\
+	F(setup_mce); F(smi_allowed); F(pre_enter_smm); F(pre_leave_smm);      \
Quoted 8 lines

arch/x86/include/asm/kvm_host.h static inline u16 kvm_lapic_irq_dest_mode(bool dest_mode_logical) · line 1130

Quoted 5 lines
arch/x86/include/asm/kvm_host.h +5 -0
+	F(enable_smi_window); F(mem_enc_op); F(mem_enc_reg_region);	       \
+	F(mem_enc_unreg_region); F(get_msr_feature);			       \
+	F(can_emulate_instruction); F(apic_init_signal_blocked);	       \
+	F(enable_direct_tlbflush); F(migrate_timers); F(msr_filter_changed);   \
+	F(complete_emulated_msr)

What about adding a dedicated .h file for this beast? Then it won't be so painful to do one function per line. As is, updates to kvm_x86_ops will be messy.

Hi,

I'm fine moving it to a new header. I had put it right above 'struct kvm_x86_ops' in arch/x86/include/asm/kvm_host.h, such that it would be updated along with kvm_x86_ops changes. But I can put a big comment there to update the new header.

Quoted 14 lines

And add yet another macro layer (or maybe just tweak this one?) so that the caller controls the line ending? I suppose you could also just use a comma, but that's a bit dirty...

That would also allow using this to declare vmx_x86_ops and svm_x86_ops, which would need a comma insteat of a semi-colon. There have a been a few attempts to add a bit of automation to {vmx,svm}_x86_ops, this seems like it would be good motivation to go in a different direction and declare/define all ops, e.g. the VMX/SVM code could simply do something like:

#define DECLARE_VMX_X86_OP(func) \
	.func = vmx_##func
Quoted 10 lines
static struct kvm_x86_ops vmx_x86_ops __initdata = {
	.vm_size = sizeof(struct kvm_vmx),
	.vm_init = vmx_vm_init,

	.pmu_ops = &intel_pmu_ops,
	.nested_ops = &vmx_nested_ops,

	FOREACH_KVM_X86_OPS(DECLARE_VMX_X86_OP)
};

Looking at the vmx definitions I see quite a few that don't match that naming. For example:

hardware_unsetup,
hardware_enable,
hardware_disable,
report_flexpriority,
update_exception_bitmap,
enable_nmi_window,
enable_irq_window,
update_cr8_intercept,
pi_has_pending_interrupt,
cpu_has_vmx_wbinvd_exit,
pi_update_irte,
kvm_complete_insn_gp,

So I'm not sure if we want to extend these macros to vmx/svm.

Thanks,

-Jason

maintainer
#4 lore raw

Re: [PATCH 1/2] KVM: x86: introduce definitions to support static calls for kvm_x86_ops

Quoted 17 lines · Jason Baron
Quoted 1 line

Looking at the vmx definitions I see quite a few that don't match that naming. For example:

hardware_unsetup,
hardware_enable,
hardware_disable,
report_flexpriority,
update_exception_bitmap,
enable_nmi_window,
enable_irq_window,
update_cr8_intercept,
pi_has_pending_interrupt,
cpu_has_vmx_wbinvd_exit,
pi_update_irte,
kvm_complete_insn_gp,

Quoted 2 lines

So I'm not sure if we want to extend these macros to vmx/svm.

Don't do it yourself, but once you introduce the new header it becomes a no-brainer to switch the declarations to use it. So let's plan the new header to make that switch easy.

Using trailing commas unconditionally would be okay, i.e.

#define X86_OP(func) .func = vmx_##func,
#include "kvm-x86-ops.h"

and leave out the terminator/delimiter in kvm-x86-ops.h. This is similar to how we use vmx/vmcs_shadow_fields.h:

#define SHADOW_FIELD_RO(x, y) { x, offsetof(struct vmcs12, y) }, #include "vmcs_shadow_fields.h"

#define SHADOW_FIELD_RW(x, y) case x:
#include "vmcs_shadow_fields.h"

Thanks,

Paolo

maintainer
#6 lore raw

Re: [PATCH 1/2] KVM: x86: introduce definitions to support static calls for kvm_x86_ops

In reply to Jason Baron

arch/x86/include/asm/kvm_host.h static inline u16 kvm_lapic_irq_dest_mode(bool dest_mode_logical) · line 1136

Quoted 5 lines · Jason Baron
arch/x86/include/asm/kvm_host.h +5 -0
+#define DEFINE_KVM_OPS_STATIC_CALL(func)	\
+	DEFINE_STATIC_CALL_NULL(kvm_x86_##func,	\
+				*(((struct kvm_x86_ops *)0)->func))
+#define DEFINE_KVM_OPS_STATIC_CALLS() \
+	FOREACH_KVM_X86_OPS(DEFINE_KVM_OPS_STATIC_CALL)

Something wrong here?

Quoted 15 lines
arch/x86/include/asm/kvm_host.h +11 -0
+#define DECLARE_KVM_OPS_STATIC_CALL(func)	\
+	DECLARE_STATIC_CALL(kvm_x86_##func,	\
+			    *(((struct kvm_x86_ops *)0)->func))
+#define DECLARE_KVM_OPS_STATIC_CALLS()		\
+	FOREACH_KVM_X86_OPS(DECLARE_KVM_OPS_STATIC_CALL)
+
+#define KVM_OPS_STATIC_CALL_UPDATE(func)	\
+	static_call_update(kvm_x86_##func, kvm_x86_ops.func)
+#define KVM_OPS_STATIC_CALL_UPDATES()		\
+	FOREACH_KVM_X86_OPS(KVM_OPS_STATIC_CALL_UPDATE)
+
  struct kvm_x86_ops {
  	int (*hardware_enable)(void);
  	void (*hardware_disable)(void);

arch/x86/include/asm/kvm_host.h extern u64 __read_mostly host_efer; · line 1392

Quoted 8 lines
arch/x86/include/asm/kvm_host.h +5 -0
@@ -1326,6 +1385,12 @@ extern u64 __read_mostly host_efer;
  extern bool __read_mostly allow_smaller_maxphyaddr;
  extern struct kvm_x86_ops kvm_x86_ops;
 
+DECLARE_KVM_OPS_STATIC_CALLS();
+static inline void kvm_ops_static_call_update(void)
+{
+	KVM_OPS_STATIC_CALL_UPDATES();
+}

This would become

#define KVM_X86_OP(func) \
	DECLARE_STATIC_CALL(kvm_x86_##func,	\
			    *(((struct kvm_x86_ops *)0)->func));

#include <asm/kvm-x86-ops.h>

static inline void kvm_ops_static_call_update(void) {
#define KVM_X86_OP(func) \
static_call_update(kvm_x86_##func, kvm_x86_ops.func)
#include <asm/kvm-x86-ops.h>
}

If you need to choose between DECLARE_STATIC_CALL_NULL and DECLARE_STATIC_CALL, you can have kvm-x86-ops.h use one of two macros KVM_X86_OP_NULL and KVM_X86_OP.

#define KVM_X86_OP(func) \
	DECLARE_STATIC_CALL(kvm_x86_##func,	\
			    *(((struct kvm_x86_ops *)0)->func));
#define KVM_X86_OP_NULL(func) \
	DECLARE_STATIC_CALL_NULL(kvm_x86_##func,	\
			    *(((struct kvm_x86_ops *)0)->func));

#include <asm/kvm-x86-ops.h>

...

#define KVM_X86_OP(func) \
static_call_update(kvm_x86_##func, kvm_x86_ops.func)
#define KVM_X86_OP_NULL(func) \
static_call_update(kvm_x86_##func, kvm_x86_ops.func)
#include <asm/kvm-x86-ops.h>

In that case vmx.c and svm.c could define KVM_X86_OP_NULL to an empty string and list the optional callbacks manually.

Paolo

#7 lore raw

Re: [PATCH 1/2] KVM: x86: introduce definitions to support static calls for kvm_x86_ops

Quoted 8 lines · Paolo Bonzini

arch/x86/include/asm/kvm_host.h static inline u16 kvm_lapic_irq_dest_mode(bool dest_mode_logical) · line 1136

Quoted 5 lines · Jason Baron
arch/x86/include/asm/kvm_host.h +5 -0
+#define DEFINE_KVM_OPS_STATIC_CALL(func)    \
+    DEFINE_STATIC_CALL_NULL(kvm_x86_##func,    \
+                *(((struct kvm_x86_ops *)0)->func))
+#define DEFINE_KVM_OPS_STATIC_CALLS() \
+    FOREACH_KVM_X86_OPS(DEFINE_KVM_OPS_STATIC_CALL)

Something wrong here?

Hmmm...not sure what you are getting at here.

Quoted 54 lines
Quoted 16 lines
arch/x86/include/asm/kvm_host.h +11 -0
+#define DECLARE_KVM_OPS_STATIC_CALL(func)    \
+    DECLARE_STATIC_CALL(kvm_x86_##func,    \
+                *(((struct kvm_x86_ops *)0)->func))
+#define DECLARE_KVM_OPS_STATIC_CALLS()        \
+    FOREACH_KVM_X86_OPS(DECLARE_KVM_OPS_STATIC_CALL)
+
+#define KVM_OPS_STATIC_CALL_UPDATE(func)    \
+    static_call_update(kvm_x86_##func, kvm_x86_ops.func)
+#define KVM_OPS_STATIC_CALL_UPDATES()        \
+    FOREACH_KVM_X86_OPS(KVM_OPS_STATIC_CALL_UPDATE)
+

  struct kvm_x86_ops {
      int (*hardware_enable)(void);
      void (*hardware_disable)(void);

arch/x86/include/asm/kvm_host.h extern u64 __read_mostly host_efer; · line 1391

Quoted 7 lines
arch/x86/include/asm/kvm_host.h +4 -0
@@ -1326,6 +1385,12 @@ extern u64 __read_mostly host_efer;
   extern bool __read_mostly allow_smaller_maxphyaddr;
   extern struct kvm_x86_ops kvm_x86_ops;
   +DECLARE_KVM_OPS_STATIC_CALLS();
+static inline void kvm_ops_static_call_update(void)
+{
+    KVM_OPS_STATIC_CALL_UPDATES();
+}

This would become

#define KVM_X86_OP(func) \
    DECLARE_STATIC_CALL(kvm_x86_##func,    \
                *(((struct kvm_x86_ops *)0)->func));

#include <asm/kvm-x86-ops.h>

static inline void kvm_ops_static_call_update(void) {
#define KVM_X86_OP(func) \
  static_call_update(kvm_x86_##func, kvm_x86_ops.func) #include <asm/kvm-x86-ops.h>
}

If you need to choose between DECLARE_STATIC_CALL_NULL and DECLARE_STATIC_CALL, you can have kvm-x86-ops.h use one of two macros KVM_X86_OP_NULL and KVM_X86_OP.

#define KVM_X86_OP(func) \
    DECLARE_STATIC_CALL(kvm_x86_##func,    \
                *(((struct kvm_x86_ops *)0)->func));

#define KVM_X86_OP_NULL(func) \
    DECLARE_STATIC_CALL_NULL(kvm_x86_##func,    \
                *(((struct kvm_x86_ops *)0)->func));

#include <asm/kvm-x86-ops.h>

...

Quoted 8 lines

#define KVM_X86_OP(func) \
  static_call_update(kvm_x86_##func, kvm_x86_ops.func) #define KVM_X86_OP_NULL(func) \
  static_call_update(kvm_x86_##func, kvm_x86_ops.func) #include <asm/kvm-x86-ops.h>

In that case vmx.c and svm.c could define KVM_X86_OP_NULL to an empty string and list the optional callbacks manually.

Ok, yes, this all makes sense. So I looked at vmx/svm definitions and I see that there are 5 definitions that are common that don't use the vmx or svm prefix:

.update_exception_bitmap = update_exception_bitmap, .enable_nmi_window = enable_nmi_window, .enable_irq_window = enable_irq_window, .update_cr8_intercept = update_cr8_intercept, .enable_smi_window = enable_smi_window,

8 are specific to vmx that don't use the vmx prefix:

.hardware_unsetup = hardware_unsetup, .hardware_enable = hardware_enable, .hardware_disable = hardware_disable, .cpu_has_accelerated_tpr = report_flexpriority, .dy_apicv_has_pending_interrupt = pi_has_pending_interrupt, .has_wbinvd_exit = cpu_has_vmx_wbinvd_exit, .update_pi_irte = pi_update_irte, .complete_emulated_msr = kvm_complete_insn_gp,

and finally 7 specific to svm that don't use the svm prefix:

.get_cs_db_l_bits = kvm_get_cs_db_l_bits, .handle_exit = handle_exit, .skip_emulated_instruction = skip_emulated_instruction, .update_emulated_instruction = NULL, .sync_pir_to_irr = kvm_lapic_find_highest_irr, .apicv_post_state_restore = avic_post_state_restore, .request_immediate_exit = __kvm_request_immediate_exit,

So we could set all of these to empty definitions and specifically add the callbacks manually as you suggested. Or we could have 4 categories of macros:

1) KVM_X86_OP() - for the usual case where we have the 'svm' or 'vmx' prefix.

2) KVM_X86_OP_NULL() - for the case where both svm and vmx want to over-ride.

For the remaining cases I was thinking of having the VMX code and svm code, add a define for 'VMX' or 'SVM' respectively and then we could mark the vmx specific ones by adding something like this to asm/kvm-x86-ops.h:

#ifndef VMX

 #define KVM_X86_OP_UNLESS_VMX(func) \
	KVM_X86_OP(func)

#else
#define KVM_X86_OP_UNLESS_VMX(func)
#endif

and similarly for the svm specific ones:

#ifndef SVM

 #define KVM_X86_OP_UNLESS_SVM(func) \
	KVM_X86_OP(func)

#else
#define KVM_X86_OP_UNLESS_SVM(func)
#endif

So in vmx.c you would have:

#define X86_OP(func) .func = vmx_##func,
#define VMX 1
#include "kvm-x86-ops.h"

Or we could just use the KVM_X86_OP_NULL() macro for anything that doesn't have a 'svm' or 'vmx' prefix as I think you were suggesting? In that case I think there would then be 13 manual additions for vmx and 12 for svm.

Thanks,

-Jason

#5 lore raw

Re: [PATCH 1/2] KVM: x86: introduce definitions to support static calls for kvm_x86_ops

In reply to Paolo Bonzini
Quoted 18 lines · Paolo Bonzini
Quoted 17 lines · Jason Baron
Quoted 1 line

Looking at the vmx definitions I see quite a few that don't match that naming. For example:

hardware_unsetup,
hardware_enable,
hardware_disable,
report_flexpriority,
update_exception_bitmap,
enable_nmi_window,
enable_irq_window,
update_cr8_intercept,
pi_has_pending_interrupt,
cpu_has_vmx_wbinvd_exit,
pi_update_irte,
kvm_complete_insn_gp,

Quoted 6 lines
Quoted 2 lines

So I'm not sure if we want to extend these macros to vmx/svm.

Don't do it yourself, but once you introduce the new header it becomes a no-brainer to switch the declarations to use it. So let's plan the new header to make that switch easy.

Ya, sorry if I didn't make it clear that the vmx/svm conversion is firmly out of scope for this series.

#8 lore raw

Re: [PATCH 1/2] KVM: x86: introduce definitions to support static calls for kvm_x86_ops

In reply to Jason Baron
Quoted 143 lines · Jason Baron
Quoted 8 lines · Paolo Bonzini

arch/x86/include/asm/kvm_host.h static inline u16 kvm_lapic_irq_dest_mode(bool dest_mode_logical) · line 1136

Quoted 5 lines · Jason Baron
arch/x86/include/asm/kvm_host.h +5 -0
+#define DEFINE_KVM_OPS_STATIC_CALL(func)    \
+    DEFINE_STATIC_CALL_NULL(kvm_x86_##func,    \
+                *(((struct kvm_x86_ops *)0)->func))
+#define DEFINE_KVM_OPS_STATIC_CALLS() \
+    FOREACH_KVM_X86_OPS(DEFINE_KVM_OPS_STATIC_CALL)

Something wrong here?

Hmmm...not sure what you are getting at here.

Quoted 54 lines
Quoted 16 lines
arch/x86/include/asm/kvm_host.h +11 -0
+#define DECLARE_KVM_OPS_STATIC_CALL(func)    \
+    DECLARE_STATIC_CALL(kvm_x86_##func,    \
+                *(((struct kvm_x86_ops *)0)->func))
+#define DECLARE_KVM_OPS_STATIC_CALLS()        \
+    FOREACH_KVM_X86_OPS(DECLARE_KVM_OPS_STATIC_CALL)
+
+#define KVM_OPS_STATIC_CALL_UPDATE(func)    \
+    static_call_update(kvm_x86_##func, kvm_x86_ops.func)
+#define KVM_OPS_STATIC_CALL_UPDATES()        \
+    FOREACH_KVM_X86_OPS(KVM_OPS_STATIC_CALL_UPDATE)
+

  struct kvm_x86_ops {
      int (*hardware_enable)(void);
      void (*hardware_disable)(void);

arch/x86/include/asm/kvm_host.h extern u64 __read_mostly host_efer; · line 1391

Quoted 7 lines
arch/x86/include/asm/kvm_host.h +4 -0
@@ -1326,6 +1385,12 @@ extern u64 __read_mostly host_efer;
   extern bool __read_mostly allow_smaller_maxphyaddr;
   extern struct kvm_x86_ops kvm_x86_ops;
   +DECLARE_KVM_OPS_STATIC_CALLS();
+static inline void kvm_ops_static_call_update(void)
+{
+    KVM_OPS_STATIC_CALL_UPDATES();
+}

This would become

#define KVM_X86_OP(func) \
    DECLARE_STATIC_CALL(kvm_x86_##func,    \
                *(((struct kvm_x86_ops *)0)->func));

#include <asm/kvm-x86-ops.h>

static inline void kvm_ops_static_call_update(void) {
#define KVM_X86_OP(func) \
  static_call_update(kvm_x86_##func, kvm_x86_ops.func) #include <asm/kvm-x86-ops.h>
}

If you need to choose between DECLARE_STATIC_CALL_NULL and DECLARE_STATIC_CALL, you can have kvm-x86-ops.h use one of two macros KVM_X86_OP_NULL and KVM_X86_OP.

#define KVM_X86_OP(func) \
    DECLARE_STATIC_CALL(kvm_x86_##func,    \
                *(((struct kvm_x86_ops *)0)->func));

#define KVM_X86_OP_NULL(func) \
    DECLARE_STATIC_CALL_NULL(kvm_x86_##func,    \
                *(((struct kvm_x86_ops *)0)->func));

#include <asm/kvm-x86-ops.h>

...

Quoted 8 lines

#define KVM_X86_OP(func) \
  static_call_update(kvm_x86_##func, kvm_x86_ops.func) #define KVM_X86_OP_NULL(func) \
  static_call_update(kvm_x86_##func, kvm_x86_ops.func) #include <asm/kvm-x86-ops.h>

In that case vmx.c and svm.c could define KVM_X86_OP_NULL to an empty string and list the optional callbacks manually.

Ok, yes, this all makes sense. So I looked at vmx/svm definitions and I see that there are 5 definitions that are common that don't use the vmx or svm prefix:

.update_exception_bitmap = update_exception_bitmap, .enable_nmi_window = enable_nmi_window, .enable_irq_window = enable_irq_window, .update_cr8_intercept = update_cr8_intercept, .enable_smi_window = enable_smi_window,

8 are specific to vmx that don't use the vmx prefix:

.hardware_unsetup = hardware_unsetup, .hardware_enable = hardware_enable, .hardware_disable = hardware_disable, .cpu_has_accelerated_tpr = report_flexpriority, .dy_apicv_has_pending_interrupt = pi_has_pending_interrupt, .has_wbinvd_exit = cpu_has_vmx_wbinvd_exit, .update_pi_irte = pi_update_irte, .complete_emulated_msr = kvm_complete_insn_gp,

and finally 7 specific to svm that don't use the svm prefix:

.get_cs_db_l_bits = kvm_get_cs_db_l_bits, .handle_exit = handle_exit, .skip_emulated_instruction = skip_emulated_instruction, .update_emulated_instruction = NULL, .sync_pir_to_irr = kvm_lapic_find_highest_irr, .apicv_post_state_restore = avic_post_state_restore, .request_immediate_exit = __kvm_request_immediate_exit,

So we could set all of these to empty definitions and specifically add the callbacks manually as you suggested. Or we could have 4 categories of macros:

1) KVM_X86_OP() - for the usual case where we have the 'svm' or 'vmx' prefix.

2) KVM_X86_OP_NULL() - for the case where both svm and vmx want to over-ride.

For the remaining cases I was thinking of having the VMX code and svm code, add a define for 'VMX' or 'SVM' respectively and then we could mark the vmx specific ones by adding something like this to asm/kvm-x86-ops.h:

#ifndef VMX

 #define KVM_X86_OP_UNLESS_VMX(func) \
	KVM_X86_OP(func)

#else
#define KVM_X86_OP_UNLESS_VMX(func)
#endif

and similarly for the svm specific ones:

#ifndef SVM

 #define KVM_X86_OP_UNLESS_SVM(func) \
	KVM_X86_OP(func)

#else
#define KVM_X86_OP_UNLESS_SVM(func)
#endif

So in vmx.c you would have:

Quoted 9 lines

#define X86_OP(func) .func = vmx_##func,
#define VMX 1
#include "kvm-x86-ops.h"

Or we could just use the KVM_X86_OP_NULL() macro for anything that doesn't have a 'svm' or 'vmx' prefix as I think you were suggesting? In that case I think there would then be 13 manual additions for vmx and 12 for svm.

Sorry this last bit is wrong. Using the KVM_X86_OP_NULL() for all definitions that don't use 'svm' and 'vmx', would mean manually defining all 20 in vmx and svm. Whereas if we we mark some of them specific to vmx or svm, we can get away with less - 13 manual for vmx and 12 manual for svm.

Thanks,

-Jason

maintainer
#9 lore raw

Re: [PATCH 1/2] KVM: x86: introduce definitions to support static calls for kvm_x86_ops

In reply to Jason Baron
Quoted 7 lines · Jason Baron
Quoted 6 lines

arch/x86/include/asm/kvm_host.h static inline u16 kvm_lapic_irq_dest_mode(bool dest_mode_logical) · line 1136

Quoted 5 lines
arch/x86/include/asm/kvm_host.h +5 -0
+#define DEFINE_KVM_OPS_STATIC_CALL(func)    \
+    DEFINE_STATIC_CALL_NULL(kvm_x86_##func,    \
+                *(((struct kvm_x86_ops *)0)->func))
+#define DEFINE_KVM_OPS_STATIC_CALLS() \
+    FOREACH_KVM_X86_OPS(DEFINE_KVM_OPS_STATIC_CALL)

Something wrong here?

Hmmm...not sure what you are getting at here.

I just misread define vs. declare.

Quoted 7 lines

Or we could just use the KVM_X86_OP_NULL() macro for anything that doesn't have a 'svm' or 'vmx' prefix as I think you were suggesting?

Using the KVM_X86_OP_NULL() for
all definitions that don't use 'svm' and 'vmx', would mean manually defining all 20 in vmx and svm

Yes, that's the simplest thing to do. Then we clean them up as we rename the functions. If you want, go ahead and rename the five easy ones yourself.

Paolo

Quoted 5 lines

.update_exception_bitmap = update_exception_bitmap, .enable_nmi_window = enable_nmi_window, .enable_irq_window = enable_irq_window, .update_cr8_intercept = update_cr8_intercept, .enable_smi_window = enable_smi_window,

#10 lore raw

Re: [PATCH 1/2] KVM: x86: introduce definitions to support static calls for kvm_x86_ops

In reply to Jason Baron
Quoted 8 lines · Jason Baron
Quoted 6 lines · Paolo Bonzini

#define KVM_X86_OP(func) \
  static_call_update(kvm_x86_##func, kvm_x86_ops.func) #define KVM_X86_OP_NULL(func) \
  static_call_update(kvm_x86_##func, kvm_x86_ops.func) #include <asm/kvm-x86-ops.h>

Quoted 7 lines
Quoted 3 lines

In that case vmx.c and svm.c could define KVM_X86_OP_NULL to an empty string and list the optional callbacks manually.

Ok, yes, this all makes sense. So I looked at vmx/svm definitions and I see that there are 5 definitions that are common that don't use the vmx or svm prefix:

We'll rename the helpers when doing the conversion, i.e. you can safely assume that all VMX/SVM functions will use the pattern {vmx,svm}_##func. I did all the renaming a few months back, but it got tossed away when svm.c was broken up.

Quoted 5 lines

.update_exception_bitmap = update_exception_bitmap, .enable_nmi_window = enable_nmi_window, .enable_irq_window = enable_irq_window, .update_cr8_intercept = update_cr8_intercept, .enable_smi_window = enable_smi_window,

#11 lore raw

Re: [PATCH 1/2] KVM: x86: introduce definitions to support static calls for kvm_x86_ops

In reply to Paolo Bonzini
Quoted 10 lines · Paolo Bonzini

If you need to choose between DECLARE_STATIC_CALL_NULL and DECLARE_STATIC_CALL, you can have kvm-x86-ops.h use one of two macros KVM_X86_OP_NULL and KVM_X86_OP.

#define KVM_X86_OP(func) \
	DECLARE_STATIC_CALL(kvm_x86_##func,	\
			    *(((struct kvm_x86_ops *)0)->func));
#define KVM_X86_OP_NULL(func) \
	DECLARE_STATIC_CALL_NULL(kvm_x86_##func,	\

Gah, DECLARE_STATIC_CALL_NULL doesn't exist, though it's referenced in a comment. I assume these should be s/DECLARE/DEFINE? I haven't fully grokked the static call code yet...

Quoted 17 lines
			    *(((struct kvm_x86_ops *)0)->func));

#include <asm/kvm-x86-ops.h>

...

#define KVM_X86_OP(func) \
static_call_update(kvm_x86_##func, kvm_x86_ops.func)
#define KVM_X86_OP_NULL(func) \
static_call_update(kvm_x86_##func, kvm_x86_ops.func)
#include <asm/kvm-x86-ops.h>

In that case vmx.c and svm.c could define KVM_X86_OP_NULL to an empty string and list the optional callbacks manually.

Paolo