Skip to contentSkip to conversation

Use static_call for kvm_pmu_ops

PATCH Superseded Like Xu 3 participants 13 messages Nov 3, 2021 – Nov 8, 2021

Subsystems (3)

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore

Like Xu

author
#0 lore raw

cover Use static_call for kvm_pmu_ops

Hi,

This is a successor to a previous patch set from Jason Baron. Let's convert kvm_pmu_ops to use static_call. Shows good performance gains for a typical perf use case [2] in the guest (results in patch 3/3).

[1] https://lore.kernel.org/lkml/[email protected]/
[2] perf record -e branch-instructions -e branch-misses \
-e cache-misses -e cache-references -e cpu-cycles \
-e instructions ./workload

Thanks,

Like Xu (3):
KVM: x86: Copy kvm_pmu_ops by value to eliminate layer of indirection
KVM: x86: Introduce definitions to support static calls for
kvm_pmu_ops
KVM: x86: Use static calls to reduce kvm_pmu_ops overhead

 arch/x86/include/asm/kvm-x86-pmu-ops.h | 32 ++++++++++++++++++
 arch/x86/kvm/pmu.c                     | 46 +++++++++++++++-----------
 arch/x86/kvm/pmu.h                     | 19 ++++++++++-
 arch/x86/kvm/vmx/nested.c              |  2 +-
 arch/x86/kvm/x86.c                     |  5 +++
 5 files changed, 83 insertions(+), 21 deletions(-)
 create mode 100644 arch/x86/include/asm/kvm-x86-pmu-ops.h
Signature · 1 line
2.33.0
author
#1 lore raw

PATCH 1/3 KVM: x86: Copy kvm_pmu_ops by value to eliminate layer of indirection

Replace the kvm_pmu_ops pointer in common x86 with an instance of the struct to save one pointer dereference when invoking functions. Copy the struct by value to set the ops during kvm_init().

Using kvm_x86_ops.hardware_enable to track whether or not the ops have been initialized, i.e. a vendor KVM module has been loaded.

---
 arch/x86/kvm/pmu.c        | 41 +++++++++++++++++++++------------------
 arch/x86/kvm/pmu.h        |  4 +++-
 arch/x86/kvm/vmx/nested.c |  2 +-
 arch/x86/kvm/x86.c        |  3 +++
 4 files changed, 29 insertions(+), 21 deletions(-)
arch/x86/kvm/pmu.c +22 -19
index 0772bad9165c..0db1887137d9 100644
@@ -47,6 +47,9 @@
  *        * AMD:   [0 .. AMD64_NUM_COUNTERS-1] <=> gp counters
  */
 
+struct kvm_pmu_ops kvm_pmu_ops __read_mostly;
+EXPORT_SYMBOL_GPL(kvm_pmu_ops);
+
 static void kvm_pmi_trigger_fn(struct irq_work *irq_work)
 {
 	struct kvm_pmu *pmu = container_of(irq_work, struct kvm_pmu, irq_work);
@@ -214,7 +217,7 @@ void reprogram_gp_counter(struct kvm_pmc *pmc, u64 eventsel)
 			  ARCH_PERFMON_EVENTSEL_CMASK |
 			  HSW_IN_TX |
 			  HSW_IN_TX_CHECKPOINTED))) {
-		config = kvm_x86_ops.pmu_ops->find_arch_event(pmc_to_pmu(pmc),
+		config = kvm_pmu_ops.find_arch_event(pmc_to_pmu(pmc),
 						      event_select,
 						      unit_mask);
 		if (config != PERF_COUNT_HW_MAX)
@@ -268,7 +271,7 @@ void reprogram_fixed_counter(struct kvm_pmc *pmc, u8 ctrl, int idx)
 
 	pmc->current_config = (u64)ctrl;
 	pmc_reprogram_counter(pmc, PERF_TYPE_HARDWARE,
-			      kvm_x86_ops.pmu_ops->find_fixed_event(idx),
+			      kvm_pmu_ops.find_fixed_event(idx),
 			      !(en_field & 0x2), /* exclude user */
 			      !(en_field & 0x1), /* exclude kernel */
 			      pmi, false, false);
@@ -277,7 +280,7 @@ EXPORT_SYMBOL_GPL(reprogram_fixed_counter);
 
 void reprogram_counter(struct kvm_pmu *pmu, int pmc_idx)
 {
-	struct kvm_pmc *pmc = kvm_x86_ops.pmu_ops->pmc_idx_to_pmc(pmu, pmc_idx);
+	struct kvm_pmc *pmc = kvm_pmu_ops.pmc_idx_to_pmc(pmu, pmc_idx);
 
 	if (!pmc)
 		return;
@@ -299,7 +302,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu)
 	int bit;
 
 	for_each_set_bit(bit, pmu->reprogram_pmi, X86_PMC_IDX_MAX) {
-		struct kvm_pmc *pmc = kvm_x86_ops.pmu_ops->pmc_idx_to_pmc(pmu, bit);
+		struct kvm_pmc *pmc = kvm_pmu_ops.pmc_idx_to_pmc(pmu, bit);
 
 		if (unlikely(!pmc || !pmc->perf_event)) {
 			clear_bit(bit, pmu->reprogram_pmi);
@@ -321,7 +324,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu)
 /* check if idx is a valid index to access PMU */
 int kvm_pmu_is_valid_rdpmc_ecx(struct kvm_vcpu *vcpu, unsigned int idx)
 {
-	return kvm_x86_ops.pmu_ops->is_valid_rdpmc_ecx(vcpu, idx);
+	return kvm_pmu_ops.is_valid_rdpmc_ecx(vcpu, idx);
 }
 
 bool is_vmware_backdoor_pmc(u32 pmc_idx)
@@ -371,7 +374,7 @@ int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)
 	if (is_vmware_backdoor_pmc(idx))
 		return kvm_pmu_rdpmc_vmware(vcpu, idx, data);
 
-	pmc = kvm_x86_ops.pmu_ops->rdpmc_ecx_to_pmc(vcpu, idx, &mask);
+	pmc = kvm_pmu_ops.rdpmc_ecx_to_pmc(vcpu, idx, &mask);
 	if (!pmc)
 		return 1;
 
@@ -387,22 +390,22 @@ int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)
 void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu)
 {
 	if (lapic_in_kernel(vcpu)) {
-		if (kvm_x86_ops.pmu_ops->deliver_pmi)
-			kvm_x86_ops.pmu_ops->deliver_pmi(vcpu);
+		if (kvm_pmu_ops.deliver_pmi)
+			kvm_pmu_ops.deliver_pmi(vcpu);
 		kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTPC);
 	}
 }
 
 bool kvm_pmu_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr)
 {
-	return kvm_x86_ops.pmu_ops->msr_idx_to_pmc(vcpu, msr) ||
-		kvm_x86_ops.pmu_ops->is_valid_msr(vcpu, msr);
+	return kvm_pmu_ops.msr_idx_to_pmc(vcpu, msr) ||
+		kvm_pmu_ops.is_valid_msr(vcpu, msr);
 }
 
 static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr)
 {
 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
-	struct kvm_pmc *pmc = kvm_x86_ops.pmu_ops->msr_idx_to_pmc(vcpu, msr);
+	struct kvm_pmc *pmc = kvm_pmu_ops.msr_idx_to_pmc(vcpu, msr);
 
 	if (pmc)
 		__set_bit(pmc->idx, pmu->pmc_in_use);
@@ -410,13 +413,13 @@ static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr)
 
 int kvm_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 {
-	return kvm_x86_ops.pmu_ops->get_msr(vcpu, msr_info);
+	return kvm_pmu_ops.get_msr(vcpu, msr_info);
 }
 
 int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 {
 	kvm_pmu_mark_pmc_in_use(vcpu, msr_info->index);
-	return kvm_x86_ops.pmu_ops->set_msr(vcpu, msr_info);
+	return kvm_pmu_ops.set_msr(vcpu, msr_info);
 }
 
 /* refresh PMU settings. This function generally is called when underlying
@@ -425,7 +428,7 @@ int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
  */
 void kvm_pmu_refresh(struct kvm_vcpu *vcpu)
 {
-	kvm_x86_ops.pmu_ops->refresh(vcpu);
+	kvm_pmu_ops.refresh(vcpu);
 }
 
 void kvm_pmu_reset(struct kvm_vcpu *vcpu)
@@ -433,7 +436,7 @@ void kvm_pmu_reset(struct kvm_vcpu *vcpu)
 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
 
 	irq_work_sync(&pmu->irq_work);
-	kvm_x86_ops.pmu_ops->reset(vcpu);
+	kvm_pmu_ops.reset(vcpu);
 }
 
 void kvm_pmu_init(struct kvm_vcpu *vcpu)
@@ -441,7 +444,7 @@ void kvm_pmu_init(struct kvm_vcpu *vcpu)
 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
 
 	memset(pmu, 0, sizeof(*pmu));
-	kvm_x86_ops.pmu_ops->init(vcpu);
+	kvm_pmu_ops.init(vcpu);
 	init_irq_work(&pmu->irq_work, kvm_pmi_trigger_fn);
 	pmu->event_count = 0;
 	pmu->need_cleanup = false;
@@ -473,14 +476,14 @@ void kvm_pmu_cleanup(struct kvm_vcpu *vcpu)
 		      pmu->pmc_in_use, X86_PMC_IDX_MAX);
 
 	for_each_set_bit(i, bitmask, X86_PMC_IDX_MAX) {
-		pmc = kvm_x86_ops.pmu_ops->pmc_idx_to_pmc(pmu, i);
+		pmc = kvm_pmu_ops.pmc_idx_to_pmc(pmu, i);
 
 		if (pmc && pmc->perf_event && !pmc_speculative_in_use(pmc))
 			pmc_stop_counter(pmc);
 	}
 
-	if (kvm_x86_ops.pmu_ops->cleanup)
-		kvm_x86_ops.pmu_ops->cleanup(vcpu);
+	if (kvm_pmu_ops.cleanup)
+		kvm_pmu_ops.cleanup(vcpu);
 
 	bitmap_zero(pmu->pmc_in_use, X86_PMC_IDX_MAX);
 }
arch/x86/kvm/pmu.h +3 -1
index 0e4f2b1fa9fb..b2fe135d395a 100644
@@ -17,6 +17,8 @@
 
 #define MAX_FIXED_COUNTERS	3
 
+extern struct kvm_pmu_ops kvm_pmu_ops;
+
 struct kvm_event_hw_type_mapping {
 	u8 eventsel;
 	u8 unit_mask;
@@ -92,7 +94,7 @@ static inline bool pmc_is_fixed(struct kvm_pmc *pmc)
 
 static inline bool pmc_is_enabled(struct kvm_pmc *pmc)
 {
-	return kvm_x86_ops.pmu_ops->pmc_is_enabled(pmc);
+	return kvm_pmu_ops.pmc_is_enabled(pmc);
 }
 
 static inline bool kvm_valid_perf_global_ctrl(struct kvm_pmu *pmu,
arch/x86/kvm/vmx/nested.c +1 -1
index b4ee5e9f9e20..1e793e44b5ff 100644
@@ -4796,7 +4796,7 @@ void nested_vmx_pmu_entry_exit_ctls_update(struct kvm_vcpu *vcpu)
 		return;
 
 	vmx = to_vmx(vcpu);
-	if (kvm_x86_ops.pmu_ops->is_valid_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL)) {
+	if (kvm_pmu_ops.is_valid_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL)) {
 		vmx->nested.msrs.entry_ctls_high |=
 				VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL;
 		vmx->nested.msrs.exit_ctls_high |=
arch/x86/kvm/x86.c +3 -0
index ac83d873d65b..72d286595012 100644
@@ -11317,6 +11317,9 @@ int kvm_arch_hardware_setup(void *opaque)
 	memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops));
 	kvm_ops_static_call_update();
 
+	if (kvm_x86_ops.hardware_enable)
+		memcpy(&kvm_pmu_ops, kvm_x86_ops.pmu_ops, sizeof(kvm_pmu_ops));
+
 	if (!kvm_cpu_cap_has(X86_FEATURE_XSAVES))
 		supported_xss = 0;
 
Signature · 1 line
2.33.0
#2 lore raw

Re: [PATCH 1/3] KVM: x86: Copy kvm_pmu_ops by value to eliminate layer of indirection

Quoted 26 lines · Like Xu

Replace the kvm_pmu_ops pointer in common x86 with an instance of the struct to save one pointer dereference when invoking functions. Copy the struct by value to set the ops during kvm_init().

Using kvm_x86_ops.hardware_enable to track whether or not the ops have been initialized, i.e. a vendor KVM module has been loaded.

---
 arch/x86/kvm/pmu.c        | 41 +++++++++++++++++++++------------------
 arch/x86/kvm/pmu.h        |  4 +++-
 arch/x86/kvm/vmx/nested.c |  2 +-
 arch/x86/kvm/x86.c        |  3 +++
 4 files changed, 29 insertions(+), 21 deletions(-)
arch/x86/kvm/pmu.c +3 -0
index 0772bad9165c..0db1887137d9 100644
@@ -47,6 +47,9 @@
  *        * AMD:   [0 .. AMD64_NUM_COUNTERS-1] <=> gp counters
  */
 
+struct kvm_pmu_ops kvm_pmu_ops __read_mostly;
+EXPORT_SYMBOL_GPL(kvm_pmu_ops);
+

...

arch/x86/kvm/vmx/nested.c void nested_vmx_pmu_entry_exit_ctls_update(struct kvm_vcpu *vcpu) · line 4799

Quoted 10 lines
arch/x86/kvm/vmx/nested.c +1 -1
index b4ee5e9f9e20..1e793e44b5ff 100644
@@ -4796,7 +4796,7 @@ void nested_vmx_pmu_entry_exit_ctls_update(struct kvm_vcpu *vcpu)
 		return;
 
 	vmx = to_vmx(vcpu);
-	if (kvm_x86_ops.pmu_ops->is_valid_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL)) {
+	if (kvm_pmu_ops.is_valid_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL)) {

I would much prefer we export kvm_pmu_is_valid_msr() and go through that for nVMX than export all of kvm_pmu_ops for this one case.

arch/x86/kvm/x86.c int kvm_arch_hardware_setup(void *opaque) · line 11320

Quoted 12 lines
arch/x86/kvm/vmx/nested.c +0 -0
 		vmx->nested.msrs.entry_ctls_high |=
 				VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL;
 		vmx->nested.msrs.exit_ctls_high |=
arch/x86/kvm/x86.c +1 -0
index ac83d873d65b..72d286595012 100644
@@ -11317,6 +11317,9 @@ int kvm_arch_hardware_setup(void *opaque)
 	memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops));
 	kvm_ops_static_call_update();
 
+	if (kvm_x86_ops.hardware_enable)

Huh? Did you intend this to be?

	if (kvm_x86_ops.pmu_ops)

Either way, I don't see the point, VMX and SVM unconditionally provide the ops.

I would also say land this memcpy() above kvm_ops_static_call_update(), then the enabling patch can do the static call updates in kvm_ops_static_call_update() instead of adding another helper.

arch/x86/kvm/x86.c int kvm_arch_hardware_setup(void *opaque) · line 11321

Quoted 1 line
arch/x86/kvm/x86.c +1 -0
+		memcpy(&kvm_pmu_ops, kvm_x86_ops.pmu_ops, sizeof(kvm_pmu_ops));

As part of this change, the pmu_ops should be moved to kvm_x86_init_ops and tagged as __initdata. That'll save those precious few bytes, and more importantly make the original ops unreachable, i.e. make it harder to sneak in post-init modification bugs.

Quoted 7 lines
arch/x86/kvm/x86.c +1 -0
+
 	if (!kvm_cpu_cap_has(X86_FEATURE_XSAVES))
 		supported_xss = 0;
 
Signature · 1 line
2.33.0
#3 lore raw

Re: [PATCH 1/3] KVM: x86: Copy kvm_pmu_ops by value to eliminate layer of indirection

Quoted 4 lines · Sean Christopherson

On Wed, Nov 03, 2021, Like Xu wrote:
I would also say land this memcpy() above kvm_ops_static_call_update(), then the enabling patch can do the static call updates in kvm_ops_static_call_update() instead of adding another helper.

Ugh, kvm_ops_static_call_update() is defined in kvm_host.h. That's completely unnecessary, it should have exactly one caller, kvm_arch_hardware_setup(). As a prep match, move kvm_ops_static_call_update() to x86.c, then it can reference the pmu ops.

author
#4 lore raw

Re: [PATCH 1/3] KVM: x86: Copy kvm_pmu_ops by value to eliminate layer of indirection

Quoted 10 lines · Sean Christopherson
Quoted 4 lines · Sean Christopherson

On Wed, Nov 03, 2021, Like Xu wrote:
I would also say land this memcpy() above kvm_ops_static_call_update(), then the enabling patch can do the static call updates in kvm_ops_static_call_update() instead of adding another helper.

Ugh, kvm_ops_static_call_update() is defined in kvm_host.h. That's completely unnecessary, it should have exactly one caller, kvm_arch_hardware_setup(). As a prep match, move kvm_ops_static_call_update() to x86.c, then it can reference the pmu ops.

Quite good and thank you.

author Applied
#5 lore raw

Re: [PATCH 1/3] KVM: x86: Copy kvm_pmu_ops by value to eliminate layer of indirection

In reply to Sean Christopherson
Quoted 37 lines · Sean Christopherson
Quoted 26 lines · Like Xu

Replace the kvm_pmu_ops pointer in common x86 with an instance of the struct to save one pointer dereference when invoking functions. Copy the struct by value to set the ops during kvm_init().

Using kvm_x86_ops.hardware_enable to track whether or not the ops have been initialized, i.e. a vendor KVM module has been loaded.

---
  arch/x86/kvm/pmu.c        | 41 +++++++++++++++++++++------------------
  arch/x86/kvm/pmu.h        |  4 +++-
  arch/x86/kvm/vmx/nested.c |  2 +-
  arch/x86/kvm/x86.c        |  3 +++
  4 files changed, 29 insertions(+), 21 deletions(-)
arch/x86/kvm/pmu.c +3 -0
index 0772bad9165c..0db1887137d9 100644
@@ -47,6 +47,9 @@
   *        * AMD:   [0 .. AMD64_NUM_COUNTERS-1] <=> gp counters
   */
 
+struct kvm_pmu_ops kvm_pmu_ops __read_mostly;
+EXPORT_SYMBOL_GPL(kvm_pmu_ops);
+

...

Quoted 7 lines
arch/x86/kvm/vmx/nested.c +0 -0
index b4ee5e9f9e20..1e793e44b5ff 100644
@@ -4796,7 +4796,7 @@ void nested_vmx_pmu_entry_exit_ctls_update(struct kvm_vcpu *vcpu)
  		return;
 
Quoted 6 lines

arch/x86/kvm/vmx/nested.c void nested_vmx_pmu_entry_exit_ctls_update(struct kvm_vcpu *vcpu) · line 4799

Quoted 3 lines
arch/x86/kvm/vmx/nested.c +1 -1
  	vmx = to_vmx(vcpu);
-	if (kvm_x86_ops.pmu_ops->is_valid_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL)) {
+	if (kvm_pmu_ops.is_valid_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL)) {

I would much prefer we export kvm_pmu_is_valid_msr() and go through that for nVMX than export all of kvm_pmu_ops for this one case.

Applied. Is it an abuse to export a function for only one case ?

Quoted 12 lines
Quoted 12 lines
arch/x86/kvm/vmx/nested.c +0 -0
  		vmx->nested.msrs.entry_ctls_high |=
  				VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL;
  		vmx->nested.msrs.exit_ctls_high |=
arch/x86/kvm/x86.c +0 -0
index ac83d873d65b..72d286595012 100644
@@ -11317,6 +11317,9 @@ int kvm_arch_hardware_setup(void *opaque)
  	memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops));
  	kvm_ops_static_call_update();
 
Quoted 7 lines

arch/x86/kvm/x86.c int kvm_arch_hardware_setup(void *opaque) · line 11320

Quoted 1 line
arch/x86/kvm/x86.c +1 -0
+	if (kvm_x86_ops.hardware_enable)

Huh? Did you intend this to be?

	if (kvm_x86_ops.pmu_ops)

Either way, I don't see the point, VMX and SVM unconditionally provide the ops.

Let me drop it.

Quoted 4 lines

I would also say land this memcpy() above kvm_ops_static_call_update(), then the enabling patch can do the static call updates in kvm_ops_static_call_update() instead of adding another helper.

Both applied.

Quoted 7 lines

arch/x86/kvm/x86.c int kvm_arch_hardware_setup(void *opaque) · line 11321

Quoted 2 lines
arch/x86/kvm/x86.c +1 -0
+		memcpy(&kvm_pmu_ops, kvm_x86_ops.pmu_ops, sizeof(kvm_pmu_ops));

As part of this change, the pmu_ops should be moved to kvm_x86_init_ops and tagged as __initdata. That'll save those precious few bytes, and more importantly make the original ops unreachable, i.e. make it harder to sneak in post-init modification bugs.

Applied. Opportunistically,

-struct kvm_pmu_ops [amd|intel]_pmu_ops = { +struct kvm_pmu_ops [amd|intel]_pmu_ops __initdata = {

Quoted 9 lines
Quoted 8 lines
arch/x86/kvm/x86.c +1 -0
+
  	if (!kvm_cpu_cap_has(X86_FEATURE_XSAVES))
  		supported_xss = 0;
 
Signature · 1 line
2.33.0
author
#6 lore raw

PATCH 2/3 KVM: x86: Introduce definitions to support static calls for kvm_pmu_ops

In reply to Like Xu

Use static calls to improve kvm_pmu_ops performance. Introduce the definitions that will be used by a subsequent patch to actualize the savings. Add a new kvm-x86-pmu-ops.h header that can be used for the definition of static calls. This header is also intended to be used to simplify the defition of amd_pmu_ops and intel_pmu_ops.

Like what we did for kvm_x86_ops, 'pmu_ops' can be covered by static calls in a simlilar manner for insignificant but not negligible performance impact, especially on older models.

---
 arch/x86/include/asm/kvm-x86-pmu-ops.h | 32 ++++++++++++++++++++++++++
 arch/x86/kvm/pmu.c                     |  7 ++++++
 arch/x86/kvm/pmu.h                     | 15 ++++++++++++
 3 files changed, 54 insertions(+)
 create mode 100644 arch/x86/include/asm/kvm-x86-pmu-ops.h
arch/x86/include/asm/kvm-x86-pmu-ops.h new file +32 -0
new file mode 100644
index 000000000000..b7713b16d21d
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#if !defined(KVM_X86_PMU_OP) || !defined(KVM_X86_PMU_OP_NULL)
+BUILD_BUG_ON(1)
+#endif
+
+/*
+ * KVM_X86_PMU_OP() and KVM_X86_PMU_OP_NULL() are used to
+ * help generate "static_call()"s. They are also intended for use when defining
+ * the amd/intel KVM_X86_PMU_OPs. KVM_X86_PMU_OP() can be used
+ * for those functions that follow the [amd|intel]_func_name convention.
+ * KVM_X86_PMU_OP_NULL() can leave a NULL definition for the
+ * case where there is no definition or a function name that
+ * doesn't match the typical naming convention is supplied.
+ */
+KVM_X86_PMU_OP(find_arch_event);
+KVM_X86_PMU_OP(find_fixed_event);
+KVM_X86_PMU_OP(pmc_is_enabled);
+KVM_X86_PMU_OP(pmc_idx_to_pmc);
+KVM_X86_PMU_OP(rdpmc_ecx_to_pmc);
+KVM_X86_PMU_OP(msr_idx_to_pmc);
+KVM_X86_PMU_OP(is_valid_rdpmc_ecx);
+KVM_X86_PMU_OP(is_valid_msr);
+KVM_X86_PMU_OP(get_msr);
+KVM_X86_PMU_OP(set_msr);
+KVM_X86_PMU_OP(refresh);
+KVM_X86_PMU_OP(init);
+KVM_X86_PMU_OP(reset);
+KVM_X86_PMU_OP_NULL(deliver_pmi);
+KVM_X86_PMU_OP_NULL(cleanup);
+
+#undef KVM_X86_PMU_OP
+#undef KVM_X86_PMU_OP_NULL
arch/x86/kvm/pmu.c +7 -0
index 0db1887137d9..b6f08c719125 100644
@@ -50,6 +50,13 @@
 struct kvm_pmu_ops kvm_pmu_ops __read_mostly;
 EXPORT_SYMBOL_GPL(kvm_pmu_ops);
 
+#define	KVM_X86_PMU_OP(func)	\
+	DEFINE_STATIC_CALL_NULL(kvm_x86_pmu_##func,	\
+				*(((struct kvm_pmu_ops *)0)->func))
+#define	KVM_X86_PMU_OP_NULL	KVM_X86_PMU_OP
+#include <asm/kvm-x86-pmu-ops.h>
+EXPORT_STATIC_CALL_GPL(kvm_x86_pmu_is_valid_msr);
+
 static void kvm_pmi_trigger_fn(struct irq_work *irq_work)
 {
 	struct kvm_pmu *pmu = container_of(irq_work, struct kvm_pmu, irq_work);
arch/x86/kvm/pmu.h +15 -0
index b2fe135d395a..e5550d4acf14 100644
@@ -3,6 +3,8 @@
 #define __KVM_X86_PMU_H
 
 #include <linux/nospec.h>
+#include <linux/static_call_types.h>
+#include <linux/static_call.h>
 
 #define vcpu_to_pmu(vcpu) (&(vcpu)->arch.pmu)
 #define pmu_to_vcpu(pmu)  (container_of((pmu), struct kvm_vcpu, arch.pmu))
@@ -45,6 +47,19 @@ struct kvm_pmu_ops {
 	void (*cleanup)(struct kvm_vcpu *vcpu);
 };
 
+#define	KVM_X86_PMU_OP(func)	\
+	DECLARE_STATIC_CALL(kvm_x86_pmu_##func, *(((struct kvm_pmu_ops *)0)->func))
+#define	KVM_X86_PMU_OP_NULL	KVM_X86_PMU_OP
+#include <asm/kvm-x86-pmu-ops.h>
+
+static inline void kvm_pmu_ops_static_call_update(void)
+{
+#define	KVM_X86_PMU_OP(func)	\
+	static_call_update(kvm_x86_pmu_##func, kvm_pmu_ops.func)
+#define	KVM_X86_PMU_OP_NULL	KVM_X86_PMU_OP
+#include <asm/kvm-x86-pmu-ops.h>
+}
+
 static inline u64 pmc_bitmask(struct kvm_pmc *pmc)
 {
 	struct kvm_pmu *pmu = pmc_to_pmu(pmc);
Signature · 1 line
2.33.0
#7 lore raw

Re: [PATCH 2/3] KVM: x86: Introduce definitions to support static calls for kvm_pmu_ops

arch/x86/kvm/pmu.c line 56

Quoted 12 lines · Like Xu
arch/x86/kvm/pmu.c +4 -0
index 0db1887137d9..b6f08c719125 100644
@@ -50,6 +50,13 @@
 struct kvm_pmu_ops kvm_pmu_ops __read_mostly;
 EXPORT_SYMBOL_GPL(kvm_pmu_ops);
 
+#define	KVM_X86_PMU_OP(func)	\
+	DEFINE_STATIC_CALL_NULL(kvm_x86_pmu_##func,	\
+				*(((struct kvm_pmu_ops *)0)->func))
+#define	KVM_X86_PMU_OP_NULL	KVM_X86_PMU_OP

More of a question for the existing code, what's the point of KVM_X86_OP_NULL? AFAICT, it always resolves to KVM_X86_OP. Unless there's some magic I'm missing, I vote we remove KVM_X86_OP_NULL and then not introduce KVM_X86_PMU_OP_NULL. And I'm pretty sure it's useless, e.g. get_cs_db_l_bits is defined with the NULL variant, but it's never NULL and its calls aren't guarded with anything. And if KVM_X86_OP_NULL is intended to aid in documenting behavior, it's doing a pretty miserable job of that :-)

arch/x86/kvm/pmu.c line 58

Quoted 2 lines
arch/x86/kvm/pmu.c +2 -0
+#include <asm/kvm-x86-pmu-ops.h>
+EXPORT_STATIC_CALL_GPL(kvm_x86_pmu_is_valid_msr);

I'll double down on my nVMX suggestion so that this export can be avoided.

Quoted 22 lines
arch/x86/kvm/pmu.c +0 -0
 static void kvm_pmi_trigger_fn(struct irq_work *irq_work)
 {
 	struct kvm_pmu *pmu = container_of(irq_work, struct kvm_pmu, irq_work);
arch/x86/kvm/pmu.h +4 -0
index b2fe135d395a..e5550d4acf14 100644
@@ -3,6 +3,8 @@
 #define __KVM_X86_PMU_H
 
 #include <linux/nospec.h>
+#include <linux/static_call_types.h>
+#include <linux/static_call.h>
 
 #define vcpu_to_pmu(vcpu) (&(vcpu)->arch.pmu)
 #define pmu_to_vcpu(pmu)  (container_of((pmu), struct kvm_vcpu, arch.pmu))
@@ -45,6 +47,19 @@ struct kvm_pmu_ops {
 	void (*cleanup)(struct kvm_vcpu *vcpu);
 };
 
+#define	KVM_X86_PMU_OP(func)	\
+	DECLARE_STATIC_CALL(kvm_x86_pmu_##func, *(((struct kvm_pmu_ops *)0)->func))

arch/x86/kvm/pmu.h struct kvm_pmu_ops { · line 61

Quoted 10 lines
arch/x86/kvm/pmu.h +10 -0
+#define	KVM_X86_PMU_OP_NULL	KVM_X86_PMU_OP
+#include <asm/kvm-x86-pmu-ops.h>
+
+static inline void kvm_pmu_ops_static_call_update(void)
+{
+#define	KVM_X86_PMU_OP(func)	\
+	static_call_update(kvm_x86_pmu_##func, kvm_pmu_ops.func)
+#define	KVM_X86_PMU_OP_NULL	KVM_X86_PMU_OP
+#include <asm/kvm-x86-pmu-ops.h>
+}

As alluded to in patch 01, I'd prefer these go in kvm_ops_static_call_update() to keep the static call magic somewhat contained.

Quoted 7 lines
arch/x86/kvm/pmu.h +1 -0
+
 static inline u64 pmc_bitmask(struct kvm_pmc *pmc)
 {
 	struct kvm_pmu *pmu = pmc_to_pmu(pmc);
Signature · 1 line
2.33.0
author
#8 lore raw

Re: [PATCH 2/3] KVM: x86: Introduce definitions to support static calls for kvm_pmu_ops

Quoted 9 lines · Sean Christopherson
Quoted 8 lines · Like Xu
arch/x86/kvm/pmu.c +0 -0
index 0db1887137d9..b6f08c719125 100644
@@ -50,6 +50,13 @@
  struct kvm_pmu_ops kvm_pmu_ops __read_mostly;
  EXPORT_SYMBOL_GPL(kvm_pmu_ops);
 
Quoted 6 lines

arch/x86/kvm/pmu.c line 56

Quoted 4 lines
arch/x86/kvm/pmu.c +4 -0
+#define	KVM_X86_PMU_OP(func)	\
+	DEFINE_STATIC_CALL_NULL(kvm_x86_pmu_##func,	\
+				*(((struct kvm_pmu_ops *)0)->func))
+#define	KVM_X86_PMU_OP_NULL	KVM_X86_PMU_OP

More of a question for the existing code, what's the point of KVM_X86_OP_NULL?

The comment says:

* KVM_X86_OP_NULL() can leave a NULL definition for the
* case where there is no definition or a function name that
* doesn't match the typical naming convention is supplied.

Does it help ?

Quoted 3 lines

AFAICT, it always resolves to KVM_X86_OP. Unless there's some magic I'm missing, I vote we remove KVM_X86_OP_NULL and then not introduce KVM_X86_PMU_OP_NULL. And I'm pretty sure it's useless, e.g. get_cs_db_l_bits is defined with the NULL

This transitions will not be included in the next version. Open to you.

Quoted 8 lines

variant, but it's never NULL and its calls aren't guarded with anything. And if KVM_X86_OP_NULL is intended to aid in documenting behavior, it's doing a pretty miserable job of that :-)

arch/x86/kvm/pmu.c line 58

Quoted 2 lines
arch/x86/kvm/pmu.c +2 -0
+#include <asm/kvm-x86-pmu-ops.h>
+EXPORT_STATIC_CALL_GPL(kvm_x86_pmu_is_valid_msr);

I'll double down on my nVMX suggestion so that this export can be avoided.

Fine to me.

Quoted 21 lines
Quoted 21 lines
arch/x86/kvm/pmu.c +0 -0
  static void kvm_pmi_trigger_fn(struct irq_work *irq_work)
  {
  	struct kvm_pmu *pmu = container_of(irq_work, struct kvm_pmu, irq_work);
arch/x86/kvm/pmu.h +2 -0
index b2fe135d395a..e5550d4acf14 100644
@@ -3,6 +3,8 @@
  #define __KVM_X86_PMU_H
 
  #include <linux/nospec.h>
+#include <linux/static_call_types.h>
+#include <linux/static_call.h>
 
  #define vcpu_to_pmu(vcpu) (&(vcpu)->arch.pmu)
  #define pmu_to_vcpu(pmu)  (container_of((pmu), struct kvm_vcpu, arch.pmu))
@@ -45,6 +47,19 @@ struct kvm_pmu_ops {
  	void (*cleanup)(struct kvm_vcpu *vcpu);
  };
 
Quoted 15 lines

arch/x86/kvm/pmu.h struct kvm_pmu_ops { · line 61

Quoted 12 lines
arch/x86/kvm/pmu.h +12 -0
+#define	KVM_X86_PMU_OP(func)	\
+	DECLARE_STATIC_CALL(kvm_x86_pmu_##func, *(((struct kvm_pmu_ops *)0)->func))
+#define	KVM_X86_PMU_OP_NULL	KVM_X86_PMU_OP
+#include <asm/kvm-x86-pmu-ops.h>
+
+static inline void kvm_pmu_ops_static_call_update(void)
+{
+#define	KVM_X86_PMU_OP(func)	\
+	static_call_update(kvm_x86_pmu_##func, kvm_pmu_ops.func)
+#define	KVM_X86_PMU_OP_NULL	KVM_X86_PMU_OP
+#include <asm/kvm-x86-pmu-ops.h>
+}

As alluded to in patch 01, I'd prefer these go in kvm_ops_static_call_update() to keep the static call magic somewhat contained.

Thank and applied.

Quoted 9 lines
Quoted 8 lines
arch/x86/kvm/pmu.h +1 -0
+
  static inline u64 pmc_bitmask(struct kvm_pmc *pmc)
  {
  	struct kvm_pmu *pmu = pmc_to_pmu(pmc);
Signature · 1 line
2.33.0
#9 lore raw

Re: [PATCH 2/3] KVM: x86: Introduce definitions to support static calls for kvm_pmu_ops

Quoted 16 lines · Like Xu
Quoted 14 lines · Sean Christopherson

arch/x86/kvm/pmu.c line 55

Quoted 11 lines · Like Xu
arch/x86/kvm/pmu.c +4 -0
index 0db1887137d9..b6f08c719125 100644
@@ -50,6 +50,13 @@
  struct kvm_pmu_ops kvm_pmu_ops __read_mostly;
  EXPORT_SYMBOL_GPL(kvm_pmu_ops);
+#define	KVM_X86_PMU_OP(func)	\
+	DEFINE_STATIC_CALL_NULL(kvm_x86_pmu_##func,	\
+				*(((struct kvm_pmu_ops *)0)->func))
+#define	KVM_X86_PMU_OP_NULL	KVM_X86_PMU_OP

More of a question for the existing code, what's the point of KVM_X86_OP_NULL?

Quoted 7 lines

The comment says:

* KVM_X86_OP_NULL() can leave a NULL definition for the
* case where there is no definition or a function name that
* doesn't match the typical naming convention is supplied.

Does it help ?

No. I understand the original intent of KVM_X86_OP_NULL, but unless there's some form of enforcement, it does more harm than good because it can very easily become stale, e.g. see get_cs_db_l_bits(). I guess "what's the point of KVM_X86_OP_NULL?" was somewhat of a rhetorical question.

Quoted 3 lines
Quoted 3 lines

AFAICT, it always resolves to KVM_X86_OP. Unless there's some magic I'm missing, I vote we remove KVM_X86_OP_NULL and then not introduce KVM_X86_PMU_OP_NULL. And I'm pretty sure it's useless, e.g. get_cs_db_l_bits is defined with the NULL

author
#10 lore raw

PATCH 3/3 KVM: x86: Use static calls to reduce kvm_pmu_ops overhead

In reply to Like Xu

Convert kvm_pmu_ops to use static calls.

Here are the worst sched_clock() nanosecond numbers for the kvm_pmu_ops functions that is most often called (up to 7 digits of calls) when running a single perf test case in a guest on an ICX 2.70GHz host (mitigations=on):

| legacy | static call
------------------------------------------------------------

.pmc_idx_to_pmc	|	10946	|	10047 (8%)
.pmc_is_enabled	|	11291	|	11175 (1%)
.msr_idx_to_pmc	|	13526	|	12346 (8%)

.is_valid_msr | 10895 | 10484 (3%)

---
 arch/x86/kvm/pmu.c        | 36 +++++++++++++++++-------------------
 arch/x86/kvm/pmu.h        |  2 +-
 arch/x86/kvm/vmx/nested.c |  2 +-
 arch/x86/kvm/x86.c        |  4 +++-
 4 files changed, 22 insertions(+), 22 deletions(-)
arch/x86/kvm/pmu.c +17 -19
index b6f08c719125..193f925e2064 100644
@@ -224,7 +224,7 @@ void reprogram_gp_counter(struct kvm_pmc *pmc, u64 eventsel)
 			  ARCH_PERFMON_EVENTSEL_CMASK |
 			  HSW_IN_TX |
 			  HSW_IN_TX_CHECKPOINTED))) {
-		config = kvm_pmu_ops.find_arch_event(pmc_to_pmu(pmc),
+		config = static_call(kvm_x86_pmu_find_arch_event)(pmc_to_pmu(pmc),
 						      event_select,
 						      unit_mask);
 		if (config != PERF_COUNT_HW_MAX)
@@ -278,7 +278,7 @@ void reprogram_fixed_counter(struct kvm_pmc *pmc, u8 ctrl, int idx)
 
 	pmc->current_config = (u64)ctrl;
 	pmc_reprogram_counter(pmc, PERF_TYPE_HARDWARE,
-			      kvm_pmu_ops.find_fixed_event(idx),
+			      static_call(kvm_x86_pmu_find_fixed_event)(idx),
 			      !(en_field & 0x2), /* exclude user */
 			      !(en_field & 0x1), /* exclude kernel */
 			      pmi, false, false);
@@ -287,7 +287,7 @@ EXPORT_SYMBOL_GPL(reprogram_fixed_counter);
 
 void reprogram_counter(struct kvm_pmu *pmu, int pmc_idx)
 {
-	struct kvm_pmc *pmc = kvm_pmu_ops.pmc_idx_to_pmc(pmu, pmc_idx);
+	struct kvm_pmc *pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, pmc_idx);
 
 	if (!pmc)
 		return;
@@ -309,7 +309,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu)
 	int bit;
 
 	for_each_set_bit(bit, pmu->reprogram_pmi, X86_PMC_IDX_MAX) {
-		struct kvm_pmc *pmc = kvm_pmu_ops.pmc_idx_to_pmc(pmu, bit);
+		struct kvm_pmc *pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, bit);
 
 		if (unlikely(!pmc || !pmc->perf_event)) {
 			clear_bit(bit, pmu->reprogram_pmi);
@@ -331,7 +331,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu)
 /* check if idx is a valid index to access PMU */
 int kvm_pmu_is_valid_rdpmc_ecx(struct kvm_vcpu *vcpu, unsigned int idx)
 {
-	return kvm_pmu_ops.is_valid_rdpmc_ecx(vcpu, idx);
+	return static_call(kvm_x86_pmu_is_valid_rdpmc_ecx)(vcpu, idx);
 }
 
 bool is_vmware_backdoor_pmc(u32 pmc_idx)
@@ -381,7 +381,7 @@ int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)
 	if (is_vmware_backdoor_pmc(idx))
 		return kvm_pmu_rdpmc_vmware(vcpu, idx, data);
 
-	pmc = kvm_pmu_ops.rdpmc_ecx_to_pmc(vcpu, idx, &mask);
+	pmc = static_call(kvm_x86_pmu_rdpmc_ecx_to_pmc)(vcpu, idx, &mask);
 	if (!pmc)
 		return 1;
 
@@ -397,22 +397,21 @@ int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)
 void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu)
 {
 	if (lapic_in_kernel(vcpu)) {
-		if (kvm_pmu_ops.deliver_pmi)
-			kvm_pmu_ops.deliver_pmi(vcpu);
+		static_call_cond(kvm_x86_pmu_deliver_pmi)(vcpu);
 		kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTPC);
 	}
 }
 
 bool kvm_pmu_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr)
 {
-	return kvm_pmu_ops.msr_idx_to_pmc(vcpu, msr) ||
-		kvm_pmu_ops.is_valid_msr(vcpu, msr);
+	return static_call(kvm_x86_pmu_msr_idx_to_pmc)(vcpu, msr) ||
+		static_call(kvm_x86_pmu_is_valid_msr)(vcpu, msr);
 }
 
 static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr)
 {
 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
-	struct kvm_pmc *pmc = kvm_pmu_ops.msr_idx_to_pmc(vcpu, msr);
+	struct kvm_pmc *pmc = static_call(kvm_x86_pmu_msr_idx_to_pmc)(vcpu, msr);
 
 	if (pmc)
 		__set_bit(pmc->idx, pmu->pmc_in_use);
@@ -420,13 +419,13 @@ static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr)
 
 int kvm_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 {
-	return kvm_pmu_ops.get_msr(vcpu, msr_info);
+	return static_call(kvm_x86_pmu_get_msr)(vcpu, msr_info);
 }
 
 int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 {
 	kvm_pmu_mark_pmc_in_use(vcpu, msr_info->index);
-	return kvm_pmu_ops.set_msr(vcpu, msr_info);
+	return static_call(kvm_x86_pmu_set_msr)(vcpu, msr_info);
 }
 
 /* refresh PMU settings. This function generally is called when underlying
@@ -435,7 +434,7 @@ int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
  */
 void kvm_pmu_refresh(struct kvm_vcpu *vcpu)
 {
-	kvm_pmu_ops.refresh(vcpu);
+	static_call(kvm_x86_pmu_refresh)(vcpu);
 }
 
 void kvm_pmu_reset(struct kvm_vcpu *vcpu)
@@ -443,7 +442,7 @@ void kvm_pmu_reset(struct kvm_vcpu *vcpu)
 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
 
 	irq_work_sync(&pmu->irq_work);
-	kvm_pmu_ops.reset(vcpu);
+	static_call(kvm_x86_pmu_reset)(vcpu);
 }
 
 void kvm_pmu_init(struct kvm_vcpu *vcpu)
@@ -451,7 +450,7 @@ void kvm_pmu_init(struct kvm_vcpu *vcpu)
 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
 
 	memset(pmu, 0, sizeof(*pmu));
-	kvm_pmu_ops.init(vcpu);
+	static_call(kvm_x86_pmu_init)(vcpu);
 	init_irq_work(&pmu->irq_work, kvm_pmi_trigger_fn);
 	pmu->event_count = 0;
 	pmu->need_cleanup = false;
@@ -483,14 +482,13 @@ void kvm_pmu_cleanup(struct kvm_vcpu *vcpu)
 		      pmu->pmc_in_use, X86_PMC_IDX_MAX);
 
 	for_each_set_bit(i, bitmask, X86_PMC_IDX_MAX) {
-		pmc = kvm_pmu_ops.pmc_idx_to_pmc(pmu, i);
+		pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, i);
 
 		if (pmc && pmc->perf_event && !pmc_speculative_in_use(pmc))
 			pmc_stop_counter(pmc);
 	}
 
-	if (kvm_pmu_ops.cleanup)
-		kvm_pmu_ops.cleanup(vcpu);
+	static_call_cond(kvm_x86_pmu_cleanup)(vcpu);
 
 	bitmap_zero(pmu->pmc_in_use, X86_PMC_IDX_MAX);
 }
arch/x86/kvm/pmu.h +1 -1
index e5550d4acf14..1818d1371ece 100644
@@ -109,7 +109,7 @@ static inline bool pmc_is_fixed(struct kvm_pmc *pmc)
 
 static inline bool pmc_is_enabled(struct kvm_pmc *pmc)
 {
-	return kvm_pmu_ops.pmc_is_enabled(pmc);
+	return static_call(kvm_x86_pmu_pmc_is_enabled)(pmc);
 }
 
 static inline bool kvm_valid_perf_global_ctrl(struct kvm_pmu *pmu,
arch/x86/kvm/vmx/nested.c +1 -1
index 1e793e44b5ff..a61661de1f39 100644
@@ -4796,7 +4796,7 @@ void nested_vmx_pmu_entry_exit_ctls_update(struct kvm_vcpu *vcpu)
 		return;
 
 	vmx = to_vmx(vcpu);
-	if (kvm_pmu_ops.is_valid_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL)) {
+	if (static_call(kvm_x86_pmu_is_valid_msr)(vcpu, MSR_CORE_PERF_GLOBAL_CTRL)) {
 		vmx->nested.msrs.entry_ctls_high |=
 				VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL;
 		vmx->nested.msrs.exit_ctls_high |=
arch/x86/kvm/x86.c +3 -1
index 72d286595012..88a3ef809c98 100644
@@ -11317,8 +11317,10 @@ int kvm_arch_hardware_setup(void *opaque)
 	memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops));
 	kvm_ops_static_call_update();
 
-	if (kvm_x86_ops.hardware_enable)
+	if (kvm_x86_ops.hardware_enable) {
 		memcpy(&kvm_pmu_ops, kvm_x86_ops.pmu_ops, sizeof(kvm_pmu_ops));
+		kvm_pmu_ops_static_call_update();
+	}
 
 	if (!kvm_cpu_cap_has(X86_FEATURE_XSAVES))
 		supported_xss = 0;
Signature · 1 line
2.33.0
#11 lore raw

Re: [PATCH 3/3] KVM: x86: Use static calls to reduce kvm_pmu_ops overhead

Quoted 26 lines · Like Xu

Convert kvm_pmu_ops to use static calls.

Here are the worst sched_clock() nanosecond numbers for the kvm_pmu_ops functions that is most often called (up to 7 digits of calls) when running a single perf test case in a guest on an ICX 2.70GHz host (mitigations=on):

| legacy | static call
------------------------------------------------------------

.pmc_idx_to_pmc	|	10946	|	10047 (8%)
.pmc_is_enabled	|	11291	|	11175 (1%)
.msr_idx_to_pmc	|	13526	|	12346 (8%)

.is_valid_msr | 10895 | 10484 (3%)

---
 arch/x86/kvm/pmu.c        | 36 +++++++++++++++++-------------------
 arch/x86/kvm/pmu.h        |  2 +-
 arch/x86/kvm/vmx/nested.c |  2 +-
 arch/x86/kvm/x86.c        |  4 +++-
 4 files changed, 22 insertions(+), 22 deletions(-)
arch/x86/kvm/pmu.c +0 -0
index b6f08c719125..193f925e2064 100644

arch/x86/kvm/pmu.c void reprogram_gp_counter(struct kvm_pmc *pmc, u64 eventsel) · line 227

Quoted 5 lines
arch/x86/kvm/pmu.c +1 -1
@@ -224,7 +224,7 @@ void reprogram_gp_counter(struct kvm_pmc *pmc, u64 eventsel)
            ARCH_PERFMON_EVENTSEL_CMASK |
            HSW_IN_TX |
            HSW_IN_TX_CHECKPOINTED))) {
-		config = kvm_pmu_ops.find_arch_event(pmc_to_pmu(pmc),
+		config = static_call(kvm_x86_pmu_find_arch_event)(pmc_to_pmu(pmc),

Why you need change them into kvm_pmu_ops.XXX then convert them into static call ? Move the instance definition of kvm_pmu_ops from patch 1 into patch 3 and then drop patch 1, will this work ?

Quoted 180 lines
arch/x86/kvm/pmu.c +16 -18
                            event_select,
                            unit_mask);
      if (config != PERF_COUNT_HW_MAX)
@@ -278,7 +278,7 @@ void reprogram_fixed_counter(struct kvm_pmc *pmc, u8 ctrl, int idx)
 
  pmc->current_config = (u64)ctrl;
  pmc_reprogram_counter(pmc, PERF_TYPE_HARDWARE,
-               kvm_pmu_ops.find_fixed_event(idx),
+               static_call(kvm_x86_pmu_find_fixed_event)(idx),
                !(en_field & 0x2), /* exclude user */
                !(en_field & 0x1), /* exclude kernel */
                pmi, false, false);
@@ -287,7 +287,7 @@ EXPORT_SYMBOL_GPL(reprogram_fixed_counter);
 
 void reprogram_counter(struct kvm_pmu *pmu, int pmc_idx)
 {
-	struct kvm_pmc *pmc = kvm_pmu_ops.pmc_idx_to_pmc(pmu, pmc_idx);
+	struct kvm_pmc *pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, pmc_idx);
 
  if (!pmc)
      return;
@@ -309,7 +309,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu)
  int bit;
 
  for_each_set_bit(bit, pmu->reprogram_pmi, X86_PMC_IDX_MAX) {
-		struct kvm_pmc *pmc = kvm_pmu_ops.pmc_idx_to_pmc(pmu, bit);
+		struct kvm_pmc *pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, bit);
 
      if (unlikely(!pmc || !pmc->perf_event)) {
          clear_bit(bit, pmu->reprogram_pmi);
@@ -331,7 +331,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu)
 /* check if idx is a valid index to access PMU */
 int kvm_pmu_is_valid_rdpmc_ecx(struct kvm_vcpu *vcpu, unsigned int idx)
 {
-	return kvm_pmu_ops.is_valid_rdpmc_ecx(vcpu, idx);
+	return static_call(kvm_x86_pmu_is_valid_rdpmc_ecx)(vcpu, idx);
 }
 
 bool is_vmware_backdoor_pmc(u32 pmc_idx)
@@ -381,7 +381,7 @@ int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)
  if (is_vmware_backdoor_pmc(idx))
      return kvm_pmu_rdpmc_vmware(vcpu, idx, data);
 
-	pmc = kvm_pmu_ops.rdpmc_ecx_to_pmc(vcpu, idx, &mask);
+	pmc = static_call(kvm_x86_pmu_rdpmc_ecx_to_pmc)(vcpu, idx, &mask);
  if (!pmc)
      return 1;
 
@@ -397,22 +397,21 @@ int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)
 void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu)
 {
  if (lapic_in_kernel(vcpu)) {
-		if (kvm_pmu_ops.deliver_pmi)
-			kvm_pmu_ops.deliver_pmi(vcpu);
+		static_call_cond(kvm_x86_pmu_deliver_pmi)(vcpu);
      kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTPC);
  }
 }
 
 bool kvm_pmu_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr)
 {
-	return kvm_pmu_ops.msr_idx_to_pmc(vcpu, msr) ||
-		kvm_pmu_ops.is_valid_msr(vcpu, msr);
+	return static_call(kvm_x86_pmu_msr_idx_to_pmc)(vcpu, msr) ||
+		static_call(kvm_x86_pmu_is_valid_msr)(vcpu, msr);
 }
 
 static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr)
 {
  struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
-	struct kvm_pmc *pmc = kvm_pmu_ops.msr_idx_to_pmc(vcpu, msr);
+	struct kvm_pmc *pmc = static_call(kvm_x86_pmu_msr_idx_to_pmc)(vcpu, msr);
 
  if (pmc)
      __set_bit(pmc->idx, pmu->pmc_in_use);
@@ -420,13 +419,13 @@ static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr)
 
 int kvm_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 {
-	return kvm_pmu_ops.get_msr(vcpu, msr_info);
+	return static_call(kvm_x86_pmu_get_msr)(vcpu, msr_info);
 }
 
 int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 {
  kvm_pmu_mark_pmc_in_use(vcpu, msr_info->index);
-	return kvm_pmu_ops.set_msr(vcpu, msr_info);
+	return static_call(kvm_x86_pmu_set_msr)(vcpu, msr_info);
 }
 
 /* refresh PMU settings. This function generally is called when underlying
@@ -435,7 +434,7 @@ int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
  */
 void kvm_pmu_refresh(struct kvm_vcpu *vcpu)
 {
-	kvm_pmu_ops.refresh(vcpu);
+	static_call(kvm_x86_pmu_refresh)(vcpu);
 }
 
 void kvm_pmu_reset(struct kvm_vcpu *vcpu)
@@ -443,7 +442,7 @@ void kvm_pmu_reset(struct kvm_vcpu *vcpu)
  struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
 
  irq_work_sync(&pmu->irq_work);
-	kvm_pmu_ops.reset(vcpu);
+	static_call(kvm_x86_pmu_reset)(vcpu);
 }
 
 void kvm_pmu_init(struct kvm_vcpu *vcpu)
@@ -451,7 +450,7 @@ void kvm_pmu_init(struct kvm_vcpu *vcpu)
  struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
 
  memset(pmu, 0, sizeof(*pmu));
-	kvm_pmu_ops.init(vcpu);
+	static_call(kvm_x86_pmu_init)(vcpu);
  init_irq_work(&pmu->irq_work, kvm_pmi_trigger_fn);
  pmu->event_count = 0;
  pmu->need_cleanup = false;
@@ -483,14 +482,13 @@ void kvm_pmu_cleanup(struct kvm_vcpu *vcpu)
            pmu->pmc_in_use, X86_PMC_IDX_MAX);
 
  for_each_set_bit(i, bitmask, X86_PMC_IDX_MAX) {
-		pmc = kvm_pmu_ops.pmc_idx_to_pmc(pmu, i);
+		pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, i);
 
      if (pmc && pmc->perf_event && !pmc_speculative_in_use(pmc))
          pmc_stop_counter(pmc);
  }
 
-	if (kvm_pmu_ops.cleanup)
-		kvm_pmu_ops.cleanup(vcpu);
+	static_call_cond(kvm_x86_pmu_cleanup)(vcpu);
 
  bitmap_zero(pmu->pmc_in_use, X86_PMC_IDX_MAX);
 }
arch/x86/kvm/pmu.h +1 -1
index e5550d4acf14..1818d1371ece 100644
@@ -109,7 +109,7 @@ static inline bool pmc_is_fixed(struct kvm_pmc *pmc)
 
 static inline bool pmc_is_enabled(struct kvm_pmc *pmc)
 {
-	return kvm_pmu_ops.pmc_is_enabled(pmc);
+	return static_call(kvm_x86_pmu_pmc_is_enabled)(pmc);
 }
 
 static inline bool kvm_valid_perf_global_ctrl(struct kvm_pmu *pmu,
arch/x86/kvm/vmx/nested.c +1 -1
index 1e793e44b5ff..a61661de1f39 100644
@@ -4796,7 +4796,7 @@ void nested_vmx_pmu_entry_exit_ctls_update(struct kvm_vcpu *vcpu)
      return;
 
  vmx = to_vmx(vcpu);
-	if (kvm_pmu_ops.is_valid_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL)) {
+	if (static_call(kvm_x86_pmu_is_valid_msr)(vcpu, MSR_CORE_PERF_GLOBAL_CTRL)) {
      vmx->nested.msrs.entry_ctls_high |=
              VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL;
      vmx->nested.msrs.exit_ctls_high |=
arch/x86/kvm/x86.c +3 -2
index 72d286595012..88a3ef809c98 100644
@@ -11317,8 +11317,10 @@ int kvm_arch_hardware_setup(void *opaque)
  memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops));
  kvm_ops_static_call_update();
 
-	if (kvm_x86_ops.hardware_enable)
+	if (kvm_x86_ops.hardware_enable) {
      memcpy(&kvm_pmu_ops, kvm_x86_ops.pmu_ops, sizeof(kvm_pmu_ops));
+		kvm_pmu_ops_static_call_update();
+	}
 
  if (!kvm_cpu_cap_has(X86_FEATURE_XSAVES))
      supported_xss = 0;
--

2.33.0

author
#12 lore raw

Re: [PATCH 3/3] KVM: x86: Use static calls to reduce kvm_pmu_ops overhead

Quoted 22 lines · Yao Yuan
Quoted 21 lines · Like Xu

Convert kvm_pmu_ops to use static calls.

Here are the worst sched_clock() nanosecond numbers for the kvm_pmu_ops functions that is most often called (up to 7 digits of calls) when running a single perf test case in a guest on an ICX 2.70GHz host (mitigations=on):

| legacy | static call
------------------------------------------------------------

.pmc_idx_to_pmc	|	10946	|	10047 (8%)
.pmc_is_enabled	|	11291	|	11175 (1%)
.msr_idx_to_pmc	|	13526	|	12346 (8%)

.is_valid_msr | 10895 | 10484 (3%)

---
  arch/x86/kvm/pmu.c        | 36 +++++++++++++++++-------------------
  arch/x86/kvm/pmu.h        |  2 +-
  arch/x86/kvm/vmx/nested.c |  2 +-
  arch/x86/kvm/x86.c        |  4 +++-
  4 files changed, 22 insertions(+), 22 deletions(-)
Quoted 15 lines

arch/x86/kvm/pmu.c void reprogram_gp_counter(struct kvm_pmc *pmc, u64 eventsel) · line 227

Quoted 10 lines
arch/x86/kvm/pmu.c +1 -1
index b6f08c719125..193f925e2064 100644
@@ -224,7 +224,7 @@ void reprogram_gp_counter(struct kvm_pmc *pmc, u64 eventsel)
             ARCH_PERFMON_EVENTSEL_CMASK |
             HSW_IN_TX |
             HSW_IN_TX_CHECKPOINTED))) {
-		config = kvm_pmu_ops.find_arch_event(pmc_to_pmu(pmc),
+		config = static_call(kvm_x86_pmu_find_arch_event)(pmc_to_pmu(pmc),

Why you need change them into kvm_pmu_ops.XXX then convert them into static call ? Move the instance definition of kvm_pmu_ops from patch 1 into patch 3 and then drop patch 1, will this work ?

You may git squash all commits to get it *work*.

With reference to afaf0b2f9b801c6eb2278b52d49e6a7d7b659cf1[1], doing one thing at a time will make things go smoother.

[1] https://lore.kernel.org/lkml/[email protected]/

Quoted 182 lines
Quoted 181 lines
arch/x86/kvm/pmu.c +16 -18
                             event_select,
                             unit_mask);
       if (config != PERF_COUNT_HW_MAX)
@@ -278,7 +278,7 @@ void reprogram_fixed_counter(struct kvm_pmc *pmc, u8 ctrl, int idx)
 
   pmc->current_config = (u64)ctrl;
   pmc_reprogram_counter(pmc, PERF_TYPE_HARDWARE,
-               kvm_pmu_ops.find_fixed_event(idx),
+               static_call(kvm_x86_pmu_find_fixed_event)(idx),
                 !(en_field & 0x2), /* exclude user */
                 !(en_field & 0x1), /* exclude kernel */
                 pmi, false, false);
@@ -287,7 +287,7 @@ EXPORT_SYMBOL_GPL(reprogram_fixed_counter);
 
  void reprogram_counter(struct kvm_pmu *pmu, int pmc_idx)
  {
-	struct kvm_pmc *pmc = kvm_pmu_ops.pmc_idx_to_pmc(pmu, pmc_idx);
+	struct kvm_pmc *pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, pmc_idx);
 
   if (!pmc)
       return;
@@ -309,7 +309,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu)
   int bit;
 
   for_each_set_bit(bit, pmu->reprogram_pmi, X86_PMC_IDX_MAX) {
-		struct kvm_pmc *pmc = kvm_pmu_ops.pmc_idx_to_pmc(pmu, bit);
+		struct kvm_pmc *pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, bit);
 
       if (unlikely(!pmc || !pmc->perf_event)) {
           clear_bit(bit, pmu->reprogram_pmi);
@@ -331,7 +331,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu)
  /* check if idx is a valid index to access PMU */
  int kvm_pmu_is_valid_rdpmc_ecx(struct kvm_vcpu *vcpu, unsigned int idx)
  {
-	return kvm_pmu_ops.is_valid_rdpmc_ecx(vcpu, idx);
+	return static_call(kvm_x86_pmu_is_valid_rdpmc_ecx)(vcpu, idx);
  }
 
  bool is_vmware_backdoor_pmc(u32 pmc_idx)
@@ -381,7 +381,7 @@ int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)
   if (is_vmware_backdoor_pmc(idx))
       return kvm_pmu_rdpmc_vmware(vcpu, idx, data);
 
-	pmc = kvm_pmu_ops.rdpmc_ecx_to_pmc(vcpu, idx, &mask);
+	pmc = static_call(kvm_x86_pmu_rdpmc_ecx_to_pmc)(vcpu, idx, &mask);
   if (!pmc)
       return 1;
 
@@ -397,22 +397,21 @@ int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)
  void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu)
  {
   if (lapic_in_kernel(vcpu)) {
-		if (kvm_pmu_ops.deliver_pmi)
-			kvm_pmu_ops.deliver_pmi(vcpu);
+		static_call_cond(kvm_x86_pmu_deliver_pmi)(vcpu);
       kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTPC);
   }
  }
 
  bool kvm_pmu_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr)
  {
-	return kvm_pmu_ops.msr_idx_to_pmc(vcpu, msr) ||
-		kvm_pmu_ops.is_valid_msr(vcpu, msr);
+	return static_call(kvm_x86_pmu_msr_idx_to_pmc)(vcpu, msr) ||
+		static_call(kvm_x86_pmu_is_valid_msr)(vcpu, msr);
  }
 
  static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr)
  {
   struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
-	struct kvm_pmc *pmc = kvm_pmu_ops.msr_idx_to_pmc(vcpu, msr);
+	struct kvm_pmc *pmc = static_call(kvm_x86_pmu_msr_idx_to_pmc)(vcpu, msr);
 
   if (pmc)
       __set_bit(pmc->idx, pmu->pmc_in_use);
@@ -420,13 +419,13 @@ static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr)
 
  int kvm_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
  {
-	return kvm_pmu_ops.get_msr(vcpu, msr_info);
+	return static_call(kvm_x86_pmu_get_msr)(vcpu, msr_info);
  }
 
  int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
  {
   kvm_pmu_mark_pmc_in_use(vcpu, msr_info->index);
-	return kvm_pmu_ops.set_msr(vcpu, msr_info);
+	return static_call(kvm_x86_pmu_set_msr)(vcpu, msr_info);
  }
 
  /* refresh PMU settings. This function generally is called when underlying
@@ -435,7 +434,7 @@ int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
   */
  void kvm_pmu_refresh(struct kvm_vcpu *vcpu)
  {
-	kvm_pmu_ops.refresh(vcpu);
+	static_call(kvm_x86_pmu_refresh)(vcpu);
  }
 
  void kvm_pmu_reset(struct kvm_vcpu *vcpu)
@@ -443,7 +442,7 @@ void kvm_pmu_reset(struct kvm_vcpu *vcpu)
   struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
 
   irq_work_sync(&pmu->irq_work);
-	kvm_pmu_ops.reset(vcpu);
+	static_call(kvm_x86_pmu_reset)(vcpu);
  }
 
  void kvm_pmu_init(struct kvm_vcpu *vcpu)
@@ -451,7 +450,7 @@ void kvm_pmu_init(struct kvm_vcpu *vcpu)
   struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
 
   memset(pmu, 0, sizeof(*pmu));
-	kvm_pmu_ops.init(vcpu);
+	static_call(kvm_x86_pmu_init)(vcpu);
   init_irq_work(&pmu->irq_work, kvm_pmi_trigger_fn);
   pmu->event_count = 0;
   pmu->need_cleanup = false;
@@ -483,14 +482,13 @@ void kvm_pmu_cleanup(struct kvm_vcpu *vcpu)
             pmu->pmc_in_use, X86_PMC_IDX_MAX);
 
   for_each_set_bit(i, bitmask, X86_PMC_IDX_MAX) {
-		pmc = kvm_pmu_ops.pmc_idx_to_pmc(pmu, i);
+		pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, i);
 
       if (pmc && pmc->perf_event && !pmc_speculative_in_use(pmc))
           pmc_stop_counter(pmc);
   }
 
-	if (kvm_pmu_ops.cleanup)
-		kvm_pmu_ops.cleanup(vcpu);
+	static_call_cond(kvm_x86_pmu_cleanup)(vcpu);
 
   bitmap_zero(pmu->pmc_in_use, X86_PMC_IDX_MAX);
  }
arch/x86/kvm/pmu.h +1 -1
index e5550d4acf14..1818d1371ece 100644
@@ -109,7 +109,7 @@ static inline bool pmc_is_fixed(struct kvm_pmc *pmc)
 
  static inline bool pmc_is_enabled(struct kvm_pmc *pmc)
  {
-	return kvm_pmu_ops.pmc_is_enabled(pmc);
+	return static_call(kvm_x86_pmu_pmc_is_enabled)(pmc);
  }
 
  static inline bool kvm_valid_perf_global_ctrl(struct kvm_pmu *pmu,
arch/x86/kvm/vmx/nested.c +1 -1
index 1e793e44b5ff..a61661de1f39 100644
@@ -4796,7 +4796,7 @@ void nested_vmx_pmu_entry_exit_ctls_update(struct kvm_vcpu *vcpu)
       return;
 
   vmx = to_vmx(vcpu);
-	if (kvm_pmu_ops.is_valid_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL)) {
+	if (static_call(kvm_x86_pmu_is_valid_msr)(vcpu, MSR_CORE_PERF_GLOBAL_CTRL)) {
       vmx->nested.msrs.entry_ctls_high |=
               VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL;
       vmx->nested.msrs.exit_ctls_high |=
arch/x86/kvm/x86.c +3 -2
index 72d286595012..88a3ef809c98 100644
@@ -11317,8 +11317,10 @@ int kvm_arch_hardware_setup(void *opaque)
   memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops));
   kvm_ops_static_call_update();
 
-	if (kvm_x86_ops.hardware_enable)
+	if (kvm_x86_ops.hardware_enable) {
       memcpy(&kvm_pmu_ops, kvm_x86_ops.pmu_ops, sizeof(kvm_pmu_ops));
+		kvm_pmu_ops_static_call_update();
+	}
 
   if (!kvm_cpu_cap_has(X86_FEATURE_XSAVES))
       supported_xss = 0;
--

2.33.0