Use static_call for kvm_pmu_ops
cover Use static_call for kvm_pmu_ops
Hi,
This is a successor to a previous patch set from Jason Baron. Let's convert kvm_pmu_ops to use static_call. Shows good performance gains for a typical perf use case [2] in the guest (results in patch 3/3).
[1] https://lore.kernel.org/lkml/[email protected]/
[2] perf record -e branch-instructions -e branch-misses \
-e cache-misses -e cache-references -e cpu-cycles \
-e instructions ./workload
Thanks,
Like Xu (3):
KVM: x86: Copy kvm_pmu_ops by value to eliminate layer of indirection
KVM: x86: Introduce definitions to support static calls for
kvm_pmu_ops
KVM: x86: Use static calls to reduce kvm_pmu_ops overhead
arch/x86/include/asm/kvm-x86-pmu-ops.h | 32 ++++++++++++++++++ arch/x86/kvm/pmu.c | 46 +++++++++++++++----------- arch/x86/kvm/pmu.h | 19 ++++++++++- arch/x86/kvm/vmx/nested.c | 2 +- arch/x86/kvm/x86.c | 5 +++ 5 files changed, 83 insertions(+), 21 deletions(-) create mode 100644 arch/x86/include/asm/kvm-x86-pmu-ops.h
Signature · 1 line
2.33.0
PATCH 1/3 KVM: x86: Copy kvm_pmu_ops by value to eliminate layer of indirection
Replace the kvm_pmu_ops pointer in common x86 with an instance of the struct to save one pointer dereference when invoking functions. Copy the struct by value to set the ops during kvm_init().
Using kvm_x86_ops.hardware_enable to track whether or not the ops have been initialized, i.e. a vendor KVM module has been loaded.
--- arch/x86/kvm/pmu.c | 41 +++++++++++++++++++++------------------ arch/x86/kvm/pmu.h | 4 +++- arch/x86/kvm/vmx/nested.c | 2 +- arch/x86/kvm/x86.c | 3 +++ 4 files changed, 29 insertions(+), 21 deletions(-)
arch/x86/kvm/pmu.c +22 -19
@@ -47,6 +47,9 @@* * AMD: [0 .. AMD64_NUM_COUNTERS-1] <=> gp counters */ +struct kvm_pmu_ops kvm_pmu_ops__read_mostly ; +EXPORT_SYMBOL_GPL (kvm_pmu_ops); +static void kvm_pmi_trigger_fn (struct irq_work *irq_work) { struct kvm_pmu *pmu = container_of(irq_work, struct kvm_pmu, irq_work);
@@ -214,7 +217,7 @@ void reprogram_gp_counter(struct kvm_pmc *pmc, u64 eventsel) ARCH_PERFMON_EVENTSEL_CMASK | HSW_IN_TX | HSW_IN_TX_CHECKPOINTED))) { - config = kvm_x86_ops.pmu_ops->find_arch_event (pmc_to_pmu (pmc), + config = kvm_pmu_ops.find_arch_event (pmc_to_pmu (pmc), event_select, unit_mask);if (config !=PERF_COUNT_HW_MAX )
@@ -268,7 +271,7 @@ void reprogram_fixed_counter(struct kvm_pmc *pmc, u8 ctrl, int idx) pmc->current_config = (u64 )ctrl;pmc_reprogram_counter (pmc,PERF_TYPE_HARDWARE , - kvm_x86_ops.pmu_ops->find_fixed_event (idx), + kvm_pmu_ops.find_fixed_event (idx), !(en_field &0x2 ),/* exclude user */ !(en_field &0x1 ),/* exclude kernel */ pmi, false, false);
@@ -277,7 +280,7 @@ EXPORT_SYMBOL_GPL(reprogram_fixed_counter);void reprogram_counter (struct kvm_pmu *pmu,int pmc_idx) { -struct kvm_pmc *pmc = kvm_x86_ops.pmu_ops->pmc_idx_to_pmc (pmu, pmc_idx); +struct kvm_pmc *pmc = kvm_pmu_ops.pmc_idx_to_pmc (pmu, pmc_idx);if (!pmc)return ;
@@ -299,7 +302,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu)int bit;for_each_set_bit (bit, pmu->reprogram_pmi,X86_PMC_IDX_MAX ) { -struct kvm_pmc *pmc = kvm_x86_ops.pmu_ops->pmc_idx_to_pmc (pmu, bit); +struct kvm_pmc *pmc = kvm_pmu_ops.pmc_idx_to_pmc (pmu, bit);if (unlikely (!pmc || !pmc->perf_event)) {clear_bit (bit, pmu->reprogram_pmi);
@@ -321,7 +324,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu)/* check if idx is a valid index to access PMU */ int kvm_pmu_is_valid_rdpmc_ecx (struct kvm_vcpu *vcpu,unsigned int idx) { -return kvm_x86_ops.pmu_ops->is_valid_rdpmc_ecx (vcpu, idx); +return kvm_pmu_ops.is_valid_rdpmc_ecx (vcpu, idx); }bool is_vmware_backdoor_pmc (u32 pmc_idx)
@@ -371,7 +374,7 @@ int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)if (is_vmware_backdoor_pmc (idx))return kvm_pmu_rdpmc_vmware (vcpu, idx, data); - pmc = kvm_x86_ops.pmu_ops->rdpmc_ecx_to_pmc (vcpu, idx, &mask); + pmc = kvm_pmu_ops.rdpmc_ecx_to_pmc (vcpu, idx, &mask);if (!pmc)return 1 ;
@@ -387,22 +390,22 @@ int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)void kvm_pmu_deliver_pmi (struct kvm_vcpu *vcpu) {if (lapic_in_kernel (vcpu)) { -if (kvm_x86_ops.pmu_ops->deliver_pmi) - kvm_x86_ops.pmu_ops->deliver_pmi (vcpu); +if (kvm_pmu_ops.deliver_pmi) + kvm_pmu_ops.deliver_pmi (vcpu);kvm_apic_local_deliver (vcpu->arch.apic,APIC_LVTPC ); } }bool kvm_pmu_is_valid_msr (struct kvm_vcpu *vcpu,u32 msr) { -return kvm_x86_ops.pmu_ops->msr_idx_to_pmc (vcpu, msr) || - kvm_x86_ops.pmu_ops->is_valid_msr (vcpu, msr); +return kvm_pmu_ops.msr_idx_to_pmc (vcpu, msr) || + kvm_pmu_ops.is_valid_msr (vcpu, msr); }static void kvm_pmu_mark_pmc_in_use (struct kvm_vcpu *vcpu,u32 msr) {struct kvm_pmu *pmu =vcpu_to_pmu (vcpu); -struct kvm_pmc *pmc = kvm_x86_ops.pmu_ops->msr_idx_to_pmc (vcpu, msr); +struct kvm_pmc *pmc = kvm_pmu_ops.msr_idx_to_pmc (vcpu, msr);if (pmc)__set_bit (pmc->idx, pmu->pmc_in_use);
@@ -410,13 +413,13 @@ static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr)int kvm_pmu_get_msr (struct kvm_vcpu *vcpu,struct msr_data *msr_info) { -return kvm_x86_ops.pmu_ops->get_msr (vcpu, msr_info); +return kvm_pmu_ops.get_msr (vcpu, msr_info); }int kvm_pmu_set_msr (struct kvm_vcpu *vcpu,struct msr_data *msr_info) {kvm_pmu_mark_pmc_in_use (vcpu, msr_info->index); -return kvm_x86_ops.pmu_ops->set_msr (vcpu, msr_info); +return kvm_pmu_ops.set_msr (vcpu, msr_info); }/* refresh PMU settings. This function generally is called when underlying
@@ -425,7 +428,7 @@ int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)*/ void kvm_pmu_refresh (struct kvm_vcpu *vcpu) { - kvm_x86_ops.pmu_ops->refresh (vcpu); + kvm_pmu_ops.refresh (vcpu); }void kvm_pmu_reset (struct kvm_vcpu *vcpu)
@@ -433,7 +436,7 @@ void kvm_pmu_reset(struct kvm_vcpu *vcpu)struct kvm_pmu *pmu =vcpu_to_pmu (vcpu);irq_work_sync (&pmu->irq_work); - kvm_x86_ops.pmu_ops->reset (vcpu); + kvm_pmu_ops.reset (vcpu); }void kvm_pmu_init (struct kvm_vcpu *vcpu)
@@ -441,7 +444,7 @@ void kvm_pmu_init(struct kvm_vcpu *vcpu)struct kvm_pmu *pmu =vcpu_to_pmu (vcpu);memset (pmu,0 ,sizeof (*pmu)); - kvm_x86_ops.pmu_ops->init (vcpu); + kvm_pmu_ops.init (vcpu);init_irq_work (&pmu->irq_work, kvm_pmi_trigger_fn); pmu->event_count =0 ; pmu->need_cleanup = false;
@@ -473,14 +476,14 @@ void kvm_pmu_cleanup(struct kvm_vcpu *vcpu) pmu->pmc_in_use, X86_PMC_IDX_MAX);for_each_set_bit (i, bitmask,X86_PMC_IDX_MAX ) { - pmc = kvm_x86_ops.pmu_ops->pmc_idx_to_pmc (pmu, i); + pmc = kvm_pmu_ops.pmc_idx_to_pmc (pmu, i);if (pmc && pmc->perf_event && !pmc_speculative_in_use (pmc))pmc_stop_counter (pmc); } -if (kvm_x86_ops.pmu_ops->cleanup) - kvm_x86_ops.pmu_ops->cleanup (vcpu); +if (kvm_pmu_ops.cleanup) + kvm_pmu_ops.cleanup (vcpu);bitmap_zero (pmu->pmc_in_use,X86_PMC_IDX_MAX ); }
arch/x86/kvm/pmu.h +3 -1
@@ -17,6 +17,8 @@#define MAX_FIXED_COUNTERS 3 +extern struct kvm_pmu_ops kvm_pmu_ops; +struct kvm_event_hw_type_mapping {u8 eventsel;u8 unit_mask;
@@ -92,7 +94,7 @@ static inline bool pmc_is_fixed(struct kvm_pmc *pmc)static inline bool pmc_is_enabled (struct kvm_pmc *pmc) { -return kvm_x86_ops.pmu_ops->pmc_is_enabled (pmc); +return kvm_pmu_ops.pmc_is_enabled (pmc); } static inline bool kvm_valid_perf_global_ctrl(struct kvm_pmu *pmu,
arch/x86/kvm/vmx/nested.c +1 -1
@@ -4796,7 +4796,7 @@ void nested_vmx_pmu_entry_exit_ctls_update(struct kvm_vcpu *vcpu)return ; vmx =to_vmx (vcpu); -if (kvm_x86_ops.pmu_ops->is_valid_msr (vcpu,MSR_CORE_PERF_GLOBAL_CTRL )) { +if (kvm_pmu_ops.is_valid_msr (vcpu,MSR_CORE_PERF_GLOBAL_CTRL )) { vmx->nested.msrs.entry_ctls_high |=VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL ; vmx->nested.msrs.exit_ctls_high |=
arch/x86/kvm/x86.c +3 -0
@@ -11317,6 +11317,9 @@ int kvm_arch_hardware_setup(void *opaque)memcpy (&kvm_x86_ops, ops->runtime_ops,sizeof (kvm_x86_ops));kvm_ops_static_call_update (); +if (kvm_x86_ops.hardware_enable) +memcpy (&kvm_pmu_ops, kvm_x86_ops.pmu_ops,sizeof (kvm_pmu_ops)); +if (!kvm_cpu_cap_has (X86_FEATURE_XSAVES )) supported_xss =0 ;
Signature · 1 line
2.33.0
Sean Christopherson
maintainerRe: [PATCH 1/3] KVM: x86: Copy kvm_pmu_ops by value to eliminate layer of indirection
Quoted 26 lines · Like Xu
Replace the
kvm_pmu_opspointer in common x86 with an instance of the struct to save one pointer dereference when invoking functions. Copy the struct by value to set the ops duringkvm_init().Using kvm_x86_ops.hardware_enable to track whether or not the ops have been initialized, i.e. a vendor KVM module has been loaded.
Signed-off-by Like Xu <[email protected]>--- arch/x86/kvm/pmu.c | 41 +++++++++++++++++++++------------------ arch/x86/kvm/pmu.h | 4 +++- arch/x86/kvm/vmx/nested.c | 2 +- arch/x86/kvm/x86.c | 3 +++ 4 files changed, 29 insertions(+), 21 deletions(-)arch/x86/kvm/pmu.c +3
-0@@ -47,6 +47,9 @@* * AMD: [0 .. AMD64_NUM_COUNTERS-1] <=> gp counters */ +struct kvm_pmu_ops kvm_pmu_ops__read_mostly ; +EXPORT_SYMBOL_GPL (kvm_pmu_ops); +
...
arch/x86/kvm/vmx/nested.c void nested_vmx_pmu_entry_exit_ctls_update(struct kvm_vcpu *vcpu) · line 4799
Quoted 10 lines
arch/x86/kvm/vmx/nested.c +1
-1@@ -4796,7 +4796,7 @@ void nested_vmx_pmu_entry_exit_ctls_update(struct kvm_vcpu *vcpu)return ; vmx =to_vmx (vcpu); -if (kvm_x86_ops.pmu_ops->is_valid_msr (vcpu,MSR_CORE_PERF_GLOBAL_CTRL )) { +if (kvm_pmu_ops.is_valid_msr (vcpu,MSR_CORE_PERF_GLOBAL_CTRL )) {
I would much prefer we export kvm_pmu_is_valid_msr() and go through that for nVMX than export all of kvm_pmu_ops for this one case.
arch/x86/kvm/x86.c int kvm_arch_hardware_setup(void *opaque) · line 11320
Quoted 12 lines
arch/x86/kvm/vmx/nested.c +0
-0vmx->nested.msrs.entry_ctls_high |=VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL ; vmx->nested.msrs.exit_ctls_high |=arch/x86/kvm/x86.c +1
-0@@ -11317,6 +11317,9 @@ int kvm_arch_hardware_setup(void *opaque)memcpy (&kvm_x86_ops, ops->runtime_ops,sizeof (kvm_x86_ops));kvm_ops_static_call_update (); +if (kvm_x86_ops.hardware_enable)
Huh? Did you intend this to be?
if (kvm_x86_ops.pmu_ops)
Either way, I don't see the point, VMX and SVM unconditionally provide the ops.
I would also say land this memcpy() above kvm_ops_static_call_update(), then the enabling patch can do the static call updates in kvm_ops_static_call_update() instead of adding another helper.
arch/x86/kvm/x86.c int kvm_arch_hardware_setup(void *opaque) · line 11321
Quoted 1 line
arch/x86/kvm/x86.c +1
-0+memcpy (&kvm_pmu_ops, kvm_x86_ops.pmu_ops,sizeof (kvm_pmu_ops));
As part of this change, the pmu_ops should be moved to kvm_x86_init_ops and tagged as __initdata. That'll save those precious few bytes, and more importantly make the original ops unreachable, i.e. make it harder to sneak in post-init modification bugs.
Quoted 7 lines
arch/x86/kvm/x86.c +1
-0+if (!kvm_cpu_cap_has (X86_FEATURE_XSAVES )) supported_xss =0 ;Signature · 1 line
2.33.0
Sean Christopherson
maintainerRe: [PATCH 1/3] KVM: x86: Copy kvm_pmu_ops by value to eliminate layer of indirection
Quoted 4 lines · Sean Christopherson
On Wed, Nov 03, 2021, Like Xu wrote:
I would also say land thismemcpy()abovekvm_ops_static_call_update(), then the enabling patch can do the static call updates inkvm_ops_static_call_update()instead of adding another helper.
Ugh, kvm_ops_static_call_update() is defined in kvm_host.h. That's completely unnecessary, it should have exactly one caller, kvm_arch_hardware_setup(). As a prep match, move kvm_ops_static_call_update() to x86.c, then it can reference the pmu ops.
Re: [PATCH 1/3] KVM: x86: Copy kvm_pmu_ops by value to eliminate layer of indirection
Quoted 10 lines · Sean Christopherson
Quoted 4 lines · Sean Christopherson
On Wed, Nov 03, 2021, Like Xu wrote:
I would also say land thismemcpy()abovekvm_ops_static_call_update(), then the enabling patch can do the static call updates inkvm_ops_static_call_update()instead of adding another helper.Ugh,
kvm_ops_static_call_update()is defined inkvm_host.h. That's completely unnecessary, it should have exactly one caller,kvm_arch_hardware_setup(). As a prep match, movekvm_ops_static_call_update()tox86.c, then it can reference the pmu ops.
Quite good and thank you.
Like Xu
AppliedRe: [PATCH 1/3] KVM: x86: Copy kvm_pmu_ops by value to eliminate layer of indirection
In reply to Sean ChristophersonQuoted 37 lines · Sean Christopherson
Quoted 26 lines · Like Xu
Replace the
kvm_pmu_opspointer in common x86 with an instance of the struct to save one pointer dereference when invoking functions. Copy the struct by value to set the ops duringkvm_init().Using kvm_x86_ops.hardware_enable to track whether or not the ops have been initialized, i.e. a vendor KVM module has been loaded.
Signed-off-by Like Xu <[email protected]>--- arch/x86/kvm/pmu.c | 41 +++++++++++++++++++++------------------ arch/x86/kvm/pmu.h | 4 +++- arch/x86/kvm/vmx/nested.c | 2 +- arch/x86/kvm/x86.c | 3 +++ 4 files changed, 29 insertions(+), 21 deletions(-)arch/x86/kvm/pmu.c +3
-0@@ -47,6 +47,9 @@* * AMD: [0 .. AMD64_NUM_COUNTERS-1] <=> gp counters */ +struct kvm_pmu_ops kvm_pmu_ops__read_mostly ; +EXPORT_SYMBOL_GPL (kvm_pmu_ops); +...
Quoted 7 lines
arch/x86/kvm/vmx/nested.c +0
-0@@ -4796,7 +4796,7 @@ void nested_vmx_pmu_entry_exit_ctls_update(struct kvm_vcpu *vcpu)return ;
Quoted 6 lines
arch/x86/kvm/vmx/nested.c void nested_vmx_pmu_entry_exit_ctls_update(struct kvm_vcpu *vcpu) · line 4799
Quoted 3 lines
arch/x86/kvm/vmx/nested.c +1
-1vmx =to_vmx (vcpu); -if (kvm_x86_ops.pmu_ops->is_valid_msr (vcpu,MSR_CORE_PERF_GLOBAL_CTRL )) { +if (kvm_pmu_ops.is_valid_msr (vcpu,MSR_CORE_PERF_GLOBAL_CTRL )) {I would much prefer we export
kvm_pmu_is_valid_msr()and go through that for nVMX than export all ofkvm_pmu_opsfor this one case.
Applied. Is it an abuse to export a function for only one case ?
Quoted 12 lines
Quoted 12 lines
arch/x86/kvm/vmx/nested.c +0
-0vmx->nested.msrs.entry_ctls_high |=VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL ; vmx->nested.msrs.exit_ctls_high |=arch/x86/kvm/x86.c +0
-0@@ -11317,6 +11317,9 @@ int kvm_arch_hardware_setup(void *opaque)memcpy (&kvm_x86_ops, ops->runtime_ops,sizeof (kvm_x86_ops));kvm_ops_static_call_update ();
Quoted 7 lines
arch/x86/kvm/x86.c int kvm_arch_hardware_setup(void *opaque) · line 11320
Quoted 1 line
arch/x86/kvm/x86.c +1
-0+if (kvm_x86_ops.hardware_enable)Huh? Did you intend this to be?
if (kvm_x86_ops.pmu_ops)Either way, I don't see the point, VMX and SVM unconditionally provide the ops.
Let me drop it.
Quoted 4 lines
I would also say land this
memcpy()abovekvm_ops_static_call_update(), then the enabling patch can do the static call updates inkvm_ops_static_call_update()instead of adding another helper.
Both applied.
Quoted 7 lines
arch/x86/kvm/x86.c int kvm_arch_hardware_setup(void *opaque) · line 11321
Quoted 2 lines
arch/x86/kvm/x86.c +1
-0+memcpy (&kvm_pmu_ops, kvm_x86_ops.pmu_ops,sizeof (kvm_pmu_ops));As part of this change, the
pmu_opsshould be moved tokvm_x86_init_opsand tagged as__initdata. That'll save those precious few bytes, and more importantly make the original ops unreachable, i.e. make it harder to sneak in post-init modification bugs.
Applied. Opportunistically,
-struct kvm_pmu_ops [amd|intel]_pmu_ops = { +struct kvm_pmu_ops [amd|intel]_pmu_ops __initdata = {
Quoted 9 lines
Quoted 8 lines
arch/x86/kvm/x86.c +1
-0+if (!kvm_cpu_cap_has (X86_FEATURE_XSAVES )) supported_xss =0 ;Signature · 1 line
2.33.0
PATCH 2/3 KVM: x86: Introduce definitions to support static calls for kvm_pmu_ops
In reply to Like XuUse static calls to improve kvm_pmu_ops performance. Introduce the definitions that will be used by a subsequent patch to actualize the savings. Add a new kvm-x86-pmu-ops.h header that can be used for the definition of static calls. This header is also intended to be used to simplify the defition of amd_pmu_ops and intel_pmu_ops.
Like what we did for kvm_x86_ops, 'pmu_ops' can be covered by static calls in a simlilar manner for insignificant but not negligible performance impact, especially on older models.
--- arch/x86/include/asm/kvm-x86-pmu-ops.h | 32 ++++++++++++++++++++++++++ arch/x86/kvm/pmu.c | 7 ++++++ arch/x86/kvm/pmu.h | 15 ++++++++++++ 3 files changed, 54 insertions(+) create mode 100644 arch/x86/include/asm/kvm-x86-pmu-ops.h
arch/x86/include/asm/kvm-x86-pmu-ops.h new file +32 -0
@@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#if !defined(KVM_X86_PMU_OP ) || !defined(KVM_X86_PMU_OP_NULL ) +BUILD_BUG_ON(1) +#endif + +/* +* KVM_X86_PMU_OP() and KVM_X86_PMU_OP_NULL() are used to +* help generate "static_call()"s. They are also intended for use when defining +* the amd/intel KVM_X86_PMU_OPs. KVM_X86_PMU_OP() can be used +* for those functions that follow the [amd|intel]_func_name convention. +* KVM_X86_PMU_OP_NULL() can leave a NULL definition for the +* case where there is no definition or a function name that +* doesn't match the typical naming convention is supplied. +*/ +KVM_X86_PMU_OP (find_arch_event); +KVM_X86_PMU_OP (find_fixed_event); +KVM_X86_PMU_OP (pmc_is_enabled); +KVM_X86_PMU_OP (pmc_idx_to_pmc); +KVM_X86_PMU_OP (rdpmc_ecx_to_pmc); +KVM_X86_PMU_OP (msr_idx_to_pmc); +KVM_X86_PMU_OP (is_valid_rdpmc_ecx); +KVM_X86_PMU_OP (is_valid_msr); +KVM_X86_PMU_OP (get_msr); +KVM_X86_PMU_OP (set_msr); +KVM_X86_PMU_OP (refresh); +KVM_X86_PMU_OP (init); +KVM_X86_PMU_OP (reset); +KVM_X86_PMU_OP_NULL (deliver_pmi); +KVM_X86_PMU_OP_NULL (cleanup); + +#undef KVM_X86_PMU_OP +#undef KVM_X86_PMU_OP_NULL
arch/x86/kvm/pmu.c +7 -0
@@ -50,6 +50,13 @@struct kvm_pmu_ops kvm_pmu_ops__read_mostly ;EXPORT_SYMBOL_GPL (kvm_pmu_ops); +#define KVM_X86_PMU_OP (func) \ + DEFINE_STATIC_CALL_NULL(kvm_x86_pmu_##func, \ + *(((struct kvm_pmu_ops *)0)->func)) +#define KVM_X86_PMU_OP_NULL KVM_X86_PMU_OP +#include <asm/kvm-x86-pmu-ops.h> +EXPORT_STATIC_CALL_GPL (kvm_x86_pmu_is_valid_msr); +static void kvm_pmi_trigger_fn (struct irq_work *irq_work) { struct kvm_pmu *pmu = container_of(irq_work, struct kvm_pmu, irq_work);
arch/x86/kvm/pmu.h +15 -0
@@ -3,6 +3,8 @@#define __KVM_X86_PMU_H#include <linux/nospec.h> +#include <linux/static_call_types.h> +#include <linux/static_call.h> #define vcpu_to_pmu (vcpu) (&(vcpu)->arch.pmu)#define pmu_to_vcpu (pmu) (container_of((pmu), struct kvm_vcpu, arch.pmu))
@@ -45,6 +47,19 @@ struct kvm_pmu_ops {void (*cleanup)(struct kvm_vcpu *vcpu); }; +#define KVM_X86_PMU_OP (func) \ + DECLARE_STATIC_CALL(kvm_x86_pmu_##func, *(((struct kvm_pmu_ops *)0)->func)) +#define KVM_X86_PMU_OP_NULL KVM_X86_PMU_OP +#include <asm/kvm-x86-pmu-ops.h> + +static inline void kvm_pmu_ops_static_call_update (void ) +{ +#define KVM_X86_PMU_OP (func) \ + static_call_update(kvm_x86_pmu_##func, kvm_pmu_ops.func) +#define KVM_X86_PMU_OP_NULL KVM_X86_PMU_OP +#include <asm/kvm-x86-pmu-ops.h> +} +static inline u64 pmc_bitmask (struct kvm_pmc *pmc) {struct kvm_pmu *pmu =pmc_to_pmu (pmc);
Signature · 1 line
2.33.0
Sean Christopherson
maintainerRe: [PATCH 2/3] KVM: x86: Introduce definitions to support static calls for kvm_pmu_ops
arch/x86/kvm/pmu.c line 56
Quoted 12 lines · Like Xu
arch/x86/kvm/pmu.c +4
-0@@ -50,6 +50,13 @@struct kvm_pmu_ops kvm_pmu_ops__read_mostly ;EXPORT_SYMBOL_GPL (kvm_pmu_ops); +#define KVM_X86_PMU_OP (func) \ + DEFINE_STATIC_CALL_NULL(kvm_x86_pmu_##func, \ + *(((struct kvm_pmu_ops *)0)->func)) +#define KVM_X86_PMU_OP_NULL KVM_X86_PMU_OP
More of a question for the existing code, what's the point of KVM_X86_OP_NULL? AFAICT, it always resolves to KVM_X86_OP. Unless there's some magic I'm missing, I vote we remove KVM_X86_OP_NULL and then not introduce KVM_X86_PMU_OP_NULL. And I'm pretty sure it's useless, e.g. get_cs_db_l_bits is defined with the NULL variant, but it's never NULL and its calls aren't guarded with anything. And if KVM_X86_OP_NULL is intended to aid in documenting behavior, it's doing a pretty miserable job of that :-)
arch/x86/kvm/pmu.c line 58
Quoted 2 lines
arch/x86/kvm/pmu.c +2
-0+#include <asm/kvm-x86-pmu-ops.h> +EXPORT_STATIC_CALL_GPL (kvm_x86_pmu_is_valid_msr);
I'll double down on my nVMX suggestion so that this export can be avoided.
Quoted 22 lines
arch/x86/kvm/pmu.c +0
-0static void kvm_pmi_trigger_fn (struct irq_work *irq_work) { struct kvm_pmu *pmu = container_of(irq_work, struct kvm_pmu, irq_work);arch/x86/kvm/pmu.h +4
-0@@ -3,6 +3,8 @@#define __KVM_X86_PMU_H#include <linux/nospec.h> +#include <linux/static_call_types.h> +#include <linux/static_call.h> #define vcpu_to_pmu (vcpu) (&(vcpu)->arch.pmu)#define pmu_to_vcpu (pmu) (container_of((pmu), struct kvm_vcpu, arch.pmu))@@ -45,6 +47,19 @@ struct kvm_pmu_ops {void (*cleanup)(struct kvm_vcpu *vcpu); }; +#define KVM_X86_PMU_OP (func) \ + DECLARE_STATIC_CALL(kvm_x86_pmu_##func, *(((struct kvm_pmu_ops *)0)->func))
arch/x86/kvm/pmu.h struct kvm_pmu_ops { · line 61
Quoted 10 lines
arch/x86/kvm/pmu.h +10
-0+#define KVM_X86_PMU_OP_NULL KVM_X86_PMU_OP +#include <asm/kvm-x86-pmu-ops.h> + +static inline void kvm_pmu_ops_static_call_update (void ) +{ +#define KVM_X86_PMU_OP (func) \ + static_call_update(kvm_x86_pmu_##func, kvm_pmu_ops.func) +#define KVM_X86_PMU_OP_NULL KVM_X86_PMU_OP +#include <asm/kvm-x86-pmu-ops.h> +}
As alluded to in patch 01, I'd prefer these go in kvm_ops_static_call_update() to keep the static call magic somewhat contained.
Quoted 7 lines
arch/x86/kvm/pmu.h +1
-0+static inline u64 pmc_bitmask (struct kvm_pmc *pmc) {struct kvm_pmu *pmu =pmc_to_pmu (pmc);Signature · 1 line
2.33.0
Re: [PATCH 2/3] KVM: x86: Introduce definitions to support static calls for kvm_pmu_ops
Quoted 9 lines · Sean Christopherson
Quoted 8 lines · Like Xu
arch/x86/kvm/pmu.c +0
-0@@ -50,6 +50,13 @@struct kvm_pmu_ops kvm_pmu_ops__read_mostly ;EXPORT_SYMBOL_GPL (kvm_pmu_ops);
Quoted 6 lines
arch/x86/kvm/pmu.c line 56
Quoted 4 lines
arch/x86/kvm/pmu.c +4
-0+#define KVM_X86_PMU_OP (func) \ + DEFINE_STATIC_CALL_NULL(kvm_x86_pmu_##func, \ + *(((struct kvm_pmu_ops *)0)->func)) +#define KVM_X86_PMU_OP_NULL KVM_X86_PMU_OPMore of a question for the existing code, what's the point of
KVM_X86_OP_NULL?
The comment says:
* KVM_X86_OP_NULL() can leave a NULL definition for the
* case where there is no definition or a function name that
* doesn't match the typical naming convention is supplied.
Does it help ?
Quoted 3 lines
AFAICT, it always resolves to
KVM_X86_OP. Unless there's some magic I'm missing, I vote we removeKVM_X86_OP_NULLand then not introduceKVM_X86_PMU_OP_NULL. And I'm pretty sure it's useless, e.g.get_cs_db_l_bitsis defined with the NULL
This transitions will not be included in the next version. Open to you.
Quoted 8 lines
variant, but it's never NULL and its calls aren't guarded with anything. And if
KVM_X86_OP_NULLis intended to aid in documenting behavior, it's doing a pretty miserable job of that :-)arch/x86/kvm/pmu.c line 58
Quoted 2 lines
arch/x86/kvm/pmu.c +2
-0+#include <asm/kvm-x86-pmu-ops.h> +EXPORT_STATIC_CALL_GPL (kvm_x86_pmu_is_valid_msr);I'll double down on my nVMX suggestion so that this export can be avoided.
Fine to me.
Quoted 21 lines
Quoted 21 lines
arch/x86/kvm/pmu.c +0
-0static void kvm_pmi_trigger_fn (struct irq_work *irq_work) { struct kvm_pmu *pmu = container_of(irq_work, struct kvm_pmu, irq_work);arch/x86/kvm/pmu.h +2
-0@@ -3,6 +3,8 @@#define __KVM_X86_PMU_H#include <linux/nospec.h> +#include <linux/static_call_types.h> +#include <linux/static_call.h> #define vcpu_to_pmu (vcpu) (&(vcpu)->arch.pmu)#define pmu_to_vcpu (pmu) (container_of((pmu), struct kvm_vcpu, arch.pmu))@@ -45,6 +47,19 @@ struct kvm_pmu_ops {void (*cleanup)(struct kvm_vcpu *vcpu); };
Quoted 15 lines
arch/x86/kvm/pmu.h struct kvm_pmu_ops { · line 61
Quoted 12 lines
arch/x86/kvm/pmu.h +12
-0+#define KVM_X86_PMU_OP (func) \ + DECLARE_STATIC_CALL(kvm_x86_pmu_##func, *(((struct kvm_pmu_ops *)0)->func)) +#define KVM_X86_PMU_OP_NULL KVM_X86_PMU_OP +#include <asm/kvm-x86-pmu-ops.h> + +static inline void kvm_pmu_ops_static_call_update (void ) +{ +#define KVM_X86_PMU_OP (func) \ + static_call_update(kvm_x86_pmu_##func, kvm_pmu_ops.func) +#define KVM_X86_PMU_OP_NULL KVM_X86_PMU_OP +#include <asm/kvm-x86-pmu-ops.h> +}As alluded to in patch 01, I'd prefer these go in
kvm_ops_static_call_update()to keep the static call magic somewhat contained.
Thank and applied.
Quoted 9 lines
Quoted 8 lines
arch/x86/kvm/pmu.h +1
-0+static inline u64 pmc_bitmask (struct kvm_pmc *pmc) {struct kvm_pmu *pmu =pmc_to_pmu (pmc);Signature · 1 line
2.33.0
Sean Christopherson
maintainerRe: [PATCH 2/3] KVM: x86: Introduce definitions to support static calls for kvm_pmu_ops
Quoted 16 lines · Like Xu
Quoted 14 lines · Sean Christopherson
arch/x86/kvm/pmu.c line 55
Quoted 11 lines · Like Xu
arch/x86/kvm/pmu.c +4
-0@@ -50,6 +50,13 @@struct kvm_pmu_ops kvm_pmu_ops__read_mostly ;EXPORT_SYMBOL_GPL (kvm_pmu_ops); +#define KVM_X86_PMU_OP (func) \ + DEFINE_STATIC_CALL_NULL(kvm_x86_pmu_##func, \ + *(((struct kvm_pmu_ops *)0)->func)) +#define KVM_X86_PMU_OP_NULL KVM_X86_PMU_OPMore of a question for the existing code, what's the point of
KVM_X86_OP_NULL?
Quoted 7 lines
The comment says:
*
KVM_X86_OP_NULL()can leave a NULL definition for the
* case where there is no definition or a function name that
* doesn't match the typical naming convention is supplied.Does it help ?
No. I understand the original intent of KVM_X86_OP_NULL, but unless there's some form of enforcement, it does more harm than good because it can very easily become stale, e.g. see get_cs_db_l_bits(). I guess "what's the point of KVM_X86_OP_NULL?" was somewhat of a rhetorical question.
Quoted 3 lines
Quoted 3 lines
AFAICT, it always resolves to
KVM_X86_OP. Unless there's some magic I'm missing, I vote we removeKVM_X86_OP_NULLand then not introduceKVM_X86_PMU_OP_NULL. And I'm pretty sure it's useless, e.g.get_cs_db_l_bitsis defined with the NULL
PATCH 3/3 KVM: x86: Use static calls to reduce kvm_pmu_ops overhead
In reply to Like XuConvert kvm_pmu_ops to use static calls.
Here are the worst sched_clock() nanosecond numbers for the kvm_pmu_ops functions that is most often called (up to 7 digits of calls) when running a single perf test case in a guest on an ICX 2.70GHz host (mitigations=on):
| legacy | static call
------------------------------------------------------------
.pmc_idx_to_pmc | 10946 | 10047 (8%) .pmc_is_enabled | 11291 | 11175 (1%) .msr_idx_to_pmc | 13526 | 12346 (8%)
.is_valid_msr | 10895 | 10484 (3%)
--- arch/x86/kvm/pmu.c | 36 +++++++++++++++++------------------- arch/x86/kvm/pmu.h | 2 +- arch/x86/kvm/vmx/nested.c | 2 +- arch/x86/kvm/x86.c | 4 +++- 4 files changed, 22 insertions(+), 22 deletions(-)
arch/x86/kvm/pmu.c +17 -19
@@ -224,7 +224,7 @@ void reprogram_gp_counter(struct kvm_pmc *pmc, u64 eventsel) ARCH_PERFMON_EVENTSEL_CMASK | HSW_IN_TX | HSW_IN_TX_CHECKPOINTED))) { - config = kvm_pmu_ops.find_arch_event (pmc_to_pmu (pmc), + config =static_call (kvm_x86_pmu_find_arch_event)(pmc_to_pmu (pmc), event_select, unit_mask);if (config !=PERF_COUNT_HW_MAX )
@@ -278,7 +278,7 @@ void reprogram_fixed_counter(struct kvm_pmc *pmc, u8 ctrl, int idx) pmc->current_config = (u64 )ctrl;pmc_reprogram_counter (pmc,PERF_TYPE_HARDWARE , - kvm_pmu_ops.find_fixed_event (idx), +static_call (kvm_x86_pmu_find_fixed_event)(idx), !(en_field &0x2 ),/* exclude user */ !(en_field &0x1 ),/* exclude kernel */ pmi, false, false);
@@ -287,7 +287,7 @@ EXPORT_SYMBOL_GPL(reprogram_fixed_counter);void reprogram_counter (struct kvm_pmu *pmu,int pmc_idx) { -struct kvm_pmc *pmc = kvm_pmu_ops.pmc_idx_to_pmc (pmu, pmc_idx); +struct kvm_pmc *pmc =static_call (kvm_x86_pmu_pmc_idx_to_pmc)(pmu, pmc_idx);if (!pmc)return ;
@@ -309,7 +309,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu)int bit;for_each_set_bit (bit, pmu->reprogram_pmi,X86_PMC_IDX_MAX ) { -struct kvm_pmc *pmc = kvm_pmu_ops.pmc_idx_to_pmc (pmu, bit); +struct kvm_pmc *pmc =static_call (kvm_x86_pmu_pmc_idx_to_pmc)(pmu, bit);if (unlikely (!pmc || !pmc->perf_event)) {clear_bit (bit, pmu->reprogram_pmi);
@@ -331,7 +331,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu)/* check if idx is a valid index to access PMU */ int kvm_pmu_is_valid_rdpmc_ecx (struct kvm_vcpu *vcpu,unsigned int idx) { -return kvm_pmu_ops.is_valid_rdpmc_ecx (vcpu, idx); +return static_call (kvm_x86_pmu_is_valid_rdpmc_ecx)(vcpu, idx); }bool is_vmware_backdoor_pmc (u32 pmc_idx)
@@ -381,7 +381,7 @@ int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)if (is_vmware_backdoor_pmc (idx))return kvm_pmu_rdpmc_vmware (vcpu, idx, data); - pmc = kvm_pmu_ops.rdpmc_ecx_to_pmc (vcpu, idx, &mask); + pmc =static_call (kvm_x86_pmu_rdpmc_ecx_to_pmc)(vcpu, idx, &mask);if (!pmc)return 1 ;
@@ -397,22 +397,21 @@ int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)void kvm_pmu_deliver_pmi (struct kvm_vcpu *vcpu) {if (lapic_in_kernel (vcpu)) { -if (kvm_pmu_ops.deliver_pmi) - kvm_pmu_ops.deliver_pmi (vcpu); +static_call_cond (kvm_x86_pmu_deliver_pmi)(vcpu);kvm_apic_local_deliver (vcpu->arch.apic,APIC_LVTPC ); } }bool kvm_pmu_is_valid_msr (struct kvm_vcpu *vcpu,u32 msr) { -return kvm_pmu_ops.msr_idx_to_pmc (vcpu, msr) || - kvm_pmu_ops.is_valid_msr (vcpu, msr); +return static_call (kvm_x86_pmu_msr_idx_to_pmc)(vcpu, msr) || +static_call (kvm_x86_pmu_is_valid_msr)(vcpu, msr); }static void kvm_pmu_mark_pmc_in_use (struct kvm_vcpu *vcpu,u32 msr) {struct kvm_pmu *pmu =vcpu_to_pmu (vcpu); -struct kvm_pmc *pmc = kvm_pmu_ops.msr_idx_to_pmc (vcpu, msr); +struct kvm_pmc *pmc =static_call (kvm_x86_pmu_msr_idx_to_pmc)(vcpu, msr);if (pmc)__set_bit (pmc->idx, pmu->pmc_in_use);
@@ -420,13 +419,13 @@ static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr)int kvm_pmu_get_msr (struct kvm_vcpu *vcpu,struct msr_data *msr_info) { -return kvm_pmu_ops.get_msr (vcpu, msr_info); +return static_call (kvm_x86_pmu_get_msr)(vcpu, msr_info); }int kvm_pmu_set_msr (struct kvm_vcpu *vcpu,struct msr_data *msr_info) {kvm_pmu_mark_pmc_in_use (vcpu, msr_info->index); -return kvm_pmu_ops.set_msr (vcpu, msr_info); +return static_call (kvm_x86_pmu_set_msr)(vcpu, msr_info); }/* refresh PMU settings. This function generally is called when underlying
@@ -435,7 +434,7 @@ int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)*/ void kvm_pmu_refresh (struct kvm_vcpu *vcpu) { - kvm_pmu_ops.refresh (vcpu); +static_call (kvm_x86_pmu_refresh)(vcpu); }void kvm_pmu_reset (struct kvm_vcpu *vcpu)
@@ -443,7 +442,7 @@ void kvm_pmu_reset(struct kvm_vcpu *vcpu)struct kvm_pmu *pmu =vcpu_to_pmu (vcpu);irq_work_sync (&pmu->irq_work); - kvm_pmu_ops.reset (vcpu); +static_call (kvm_x86_pmu_reset)(vcpu); }void kvm_pmu_init (struct kvm_vcpu *vcpu)
@@ -451,7 +450,7 @@ void kvm_pmu_init(struct kvm_vcpu *vcpu)struct kvm_pmu *pmu =vcpu_to_pmu (vcpu);memset (pmu,0 ,sizeof (*pmu)); - kvm_pmu_ops.init (vcpu); +static_call (kvm_x86_pmu_init)(vcpu);init_irq_work (&pmu->irq_work, kvm_pmi_trigger_fn); pmu->event_count =0 ; pmu->need_cleanup = false;
@@ -483,14 +482,13 @@ void kvm_pmu_cleanup(struct kvm_vcpu *vcpu) pmu->pmc_in_use, X86_PMC_IDX_MAX);for_each_set_bit (i, bitmask,X86_PMC_IDX_MAX ) { - pmc = kvm_pmu_ops.pmc_idx_to_pmc (pmu, i); + pmc =static_call (kvm_x86_pmu_pmc_idx_to_pmc)(pmu, i);if (pmc && pmc->perf_event && !pmc_speculative_in_use (pmc))pmc_stop_counter (pmc); } -if (kvm_pmu_ops.cleanup) - kvm_pmu_ops.cleanup (vcpu); +static_call_cond (kvm_x86_pmu_cleanup)(vcpu);bitmap_zero (pmu->pmc_in_use,X86_PMC_IDX_MAX ); }
arch/x86/kvm/pmu.h +1 -1
@@ -109,7 +109,7 @@ static inline bool pmc_is_fixed(struct kvm_pmc *pmc)static inline bool pmc_is_enabled (struct kvm_pmc *pmc) { -return kvm_pmu_ops.pmc_is_enabled (pmc); +return static_call (kvm_x86_pmu_pmc_is_enabled)(pmc); } static inline bool kvm_valid_perf_global_ctrl(struct kvm_pmu *pmu,
arch/x86/kvm/vmx/nested.c +1 -1
@@ -4796,7 +4796,7 @@ void nested_vmx_pmu_entry_exit_ctls_update(struct kvm_vcpu *vcpu)return ; vmx =to_vmx (vcpu); -if (kvm_pmu_ops.is_valid_msr (vcpu,MSR_CORE_PERF_GLOBAL_CTRL )) { +if (static_call (kvm_x86_pmu_is_valid_msr)(vcpu,MSR_CORE_PERF_GLOBAL_CTRL )) { vmx->nested.msrs.entry_ctls_high |=VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL ; vmx->nested.msrs.exit_ctls_high |=
arch/x86/kvm/x86.c +3 -1
@@ -11317,8 +11317,10 @@ int kvm_arch_hardware_setup(void *opaque)memcpy (&kvm_x86_ops, ops->runtime_ops,sizeof (kvm_x86_ops));kvm_ops_static_call_update (); -if (kvm_x86_ops.hardware_enable) +if (kvm_x86_ops.hardware_enable) {memcpy (&kvm_pmu_ops, kvm_x86_ops.pmu_ops,sizeof (kvm_pmu_ops)); +kvm_pmu_ops_static_call_update (); + }if (!kvm_cpu_cap_has (X86_FEATURE_XSAVES )) supported_xss =0 ;
Signature · 1 line
2.33.0
Re: [PATCH 3/3] KVM: x86: Use static calls to reduce kvm_pmu_ops overhead
Quoted 26 lines · Like Xu
Convert
kvm_pmu_opsto use static calls.Here are the worst
sched_clock()nanosecond numbers for thekvm_pmu_opsfunctions that is most often called (up to 7 digits of calls) when running a single perf test case in a guest on an ICX 2.70GHz host (mitigations=on):| legacy | static call
------------------------------------------------------------.pmc_idx_to_pmc | 10946 | 10047 (8%) .pmc_is_enabled | 11291 | 11175 (1%) .msr_idx_to_pmc | 13526 | 12346 (8%).is_valid_msr | 10895 | 10484 (3%)
Signed-off-by Like Xu <[email protected]>--- arch/x86/kvm/pmu.c | 36 +++++++++++++++++------------------- arch/x86/kvm/pmu.h | 2 +- arch/x86/kvm/vmx/nested.c | 2 +- arch/x86/kvm/x86.c | 4 +++- 4 files changed, 22 insertions(+), 22 deletions(-)arch/x86/kvm/pmu.c +0
-0
arch/x86/kvm/pmu.c void reprogram_gp_counter(struct kvm_pmc *pmc, u64 eventsel) · line 227
Quoted 5 lines
arch/x86/kvm/pmu.c +1
-1@@ -224,7 +224,7 @@ void reprogram_gp_counter(struct kvm_pmc *pmc, u64 eventsel) ARCH_PERFMON_EVENTSEL_CMASK | HSW_IN_TX | HSW_IN_TX_CHECKPOINTED))) { - config = kvm_pmu_ops.find_arch_event(pmc_to_pmu(pmc), + config = static_call(kvm_x86_pmu_find_arch_event)(pmc_to_pmu(pmc),
Why you need change them into kvm_pmu_ops.XXX then convert them into static call ? Move the instance definition of kvm_pmu_ops from patch 1 into patch 3 and then drop patch 1, will this work ?
Quoted 180 lines
arch/x86/kvm/pmu.c +16
-18event_select, unit_mask);if (config !=PERF_COUNT_HW_MAX )@@ -278,7 +278,7 @@ void reprogram_fixed_counter(struct kvm_pmc *pmc, u8 ctrl, int idx) pmc->current_config = (u64 )ctrl;pmc_reprogram_counter (pmc,PERF_TYPE_HARDWARE , - kvm_pmu_ops.find_fixed_event (idx), +static_call (kvm_x86_pmu_find_fixed_event)(idx), !(en_field &0x2 ),/* exclude user */ !(en_field &0x1 ),/* exclude kernel */ pmi, false, false);@@ -287,7 +287,7 @@ EXPORT_SYMBOL_GPL(reprogram_fixed_counter);void reprogram_counter (struct kvm_pmu *pmu,int pmc_idx) { -struct kvm_pmc *pmc = kvm_pmu_ops.pmc_idx_to_pmc (pmu, pmc_idx); +struct kvm_pmc *pmc =static_call (kvm_x86_pmu_pmc_idx_to_pmc)(pmu, pmc_idx);if (!pmc)return ;@@ -309,7 +309,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu)int bit;for_each_set_bit (bit, pmu->reprogram_pmi,X86_PMC_IDX_MAX ) { -struct kvm_pmc *pmc = kvm_pmu_ops.pmc_idx_to_pmc (pmu, bit); +struct kvm_pmc *pmc =static_call (kvm_x86_pmu_pmc_idx_to_pmc)(pmu, bit);if (unlikely (!pmc || !pmc->perf_event)) {clear_bit (bit, pmu->reprogram_pmi);@@ -331,7 +331,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu)/* check if idx is a valid index to access PMU */ int kvm_pmu_is_valid_rdpmc_ecx (struct kvm_vcpu *vcpu,unsigned int idx) { -return kvm_pmu_ops.is_valid_rdpmc_ecx (vcpu, idx); +return static_call (kvm_x86_pmu_is_valid_rdpmc_ecx)(vcpu, idx); }bool is_vmware_backdoor_pmc (u32 pmc_idx)@@ -381,7 +381,7 @@ int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)if (is_vmware_backdoor_pmc (idx))return kvm_pmu_rdpmc_vmware (vcpu, idx, data); - pmc = kvm_pmu_ops.rdpmc_ecx_to_pmc (vcpu, idx, &mask); + pmc =static_call (kvm_x86_pmu_rdpmc_ecx_to_pmc)(vcpu, idx, &mask);if (!pmc)return 1 ;@@ -397,22 +397,21 @@ int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)void kvm_pmu_deliver_pmi (struct kvm_vcpu *vcpu) {if (lapic_in_kernel (vcpu)) { -if (kvm_pmu_ops.deliver_pmi) - kvm_pmu_ops.deliver_pmi (vcpu); +static_call_cond (kvm_x86_pmu_deliver_pmi)(vcpu);kvm_apic_local_deliver (vcpu->arch.apic,APIC_LVTPC ); } }bool kvm_pmu_is_valid_msr (struct kvm_vcpu *vcpu,u32 msr) { -return kvm_pmu_ops.msr_idx_to_pmc (vcpu, msr) || - kvm_pmu_ops.is_valid_msr (vcpu, msr); +return static_call (kvm_x86_pmu_msr_idx_to_pmc)(vcpu, msr) || +static_call (kvm_x86_pmu_is_valid_msr)(vcpu, msr); }static void kvm_pmu_mark_pmc_in_use (struct kvm_vcpu *vcpu,u32 msr) {struct kvm_pmu *pmu =vcpu_to_pmu (vcpu); -struct kvm_pmc *pmc = kvm_pmu_ops.msr_idx_to_pmc (vcpu, msr); +struct kvm_pmc *pmc =static_call (kvm_x86_pmu_msr_idx_to_pmc)(vcpu, msr);if (pmc)__set_bit (pmc->idx, pmu->pmc_in_use);@@ -420,13 +419,13 @@ static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr)int kvm_pmu_get_msr (struct kvm_vcpu *vcpu,struct msr_data *msr_info) { -return kvm_pmu_ops.get_msr (vcpu, msr_info); +return static_call (kvm_x86_pmu_get_msr)(vcpu, msr_info); }int kvm_pmu_set_msr (struct kvm_vcpu *vcpu,struct msr_data *msr_info) {kvm_pmu_mark_pmc_in_use (vcpu, msr_info->index); -return kvm_pmu_ops.set_msr (vcpu, msr_info); +return static_call (kvm_x86_pmu_set_msr)(vcpu, msr_info); }/* refresh PMU settings. This function generally is called when underlying @@ -435,7 +434,7 @@ int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)*/ void kvm_pmu_refresh (struct kvm_vcpu *vcpu) { - kvm_pmu_ops.refresh (vcpu); +static_call (kvm_x86_pmu_refresh)(vcpu); }void kvm_pmu_reset (struct kvm_vcpu *vcpu)@@ -443,7 +442,7 @@ void kvm_pmu_reset(struct kvm_vcpu *vcpu)struct kvm_pmu *pmu =vcpu_to_pmu (vcpu);irq_work_sync (&pmu->irq_work); - kvm_pmu_ops.reset (vcpu); +static_call (kvm_x86_pmu_reset)(vcpu); }void kvm_pmu_init (struct kvm_vcpu *vcpu)@@ -451,7 +450,7 @@ void kvm_pmu_init(struct kvm_vcpu *vcpu)struct kvm_pmu *pmu =vcpu_to_pmu (vcpu);memset (pmu,0 ,sizeof (*pmu)); - kvm_pmu_ops.init (vcpu); +static_call (kvm_x86_pmu_init)(vcpu);init_irq_work (&pmu->irq_work, kvm_pmi_trigger_fn); pmu->event_count =0 ; pmu->need_cleanup = false;@@ -483,14 +482,13 @@ void kvm_pmu_cleanup(struct kvm_vcpu *vcpu) pmu->pmc_in_use, X86_PMC_IDX_MAX);for_each_set_bit (i, bitmask,X86_PMC_IDX_MAX ) { - pmc = kvm_pmu_ops.pmc_idx_to_pmc (pmu, i); + pmc =static_call (kvm_x86_pmu_pmc_idx_to_pmc)(pmu, i);if (pmc && pmc->perf_event && !pmc_speculative_in_use (pmc))pmc_stop_counter (pmc); } -if (kvm_pmu_ops.cleanup) - kvm_pmu_ops.cleanup (vcpu); +static_call_cond (kvm_x86_pmu_cleanup)(vcpu);bitmap_zero (pmu->pmc_in_use,X86_PMC_IDX_MAX ); }arch/x86/kvm/pmu.h +1
-1@@ -109,7 +109,7 @@ static inline bool pmc_is_fixed(struct kvm_pmc *pmc)static inline bool pmc_is_enabled (struct kvm_pmc *pmc) { -return kvm_pmu_ops.pmc_is_enabled (pmc); +return static_call (kvm_x86_pmu_pmc_is_enabled)(pmc); } static inline bool kvm_valid_perf_global_ctrl(struct kvm_pmu *pmu,arch/x86/kvm/vmx/nested.c +1
-1@@ -4796,7 +4796,7 @@ void nested_vmx_pmu_entry_exit_ctls_update(struct kvm_vcpu *vcpu)return ; vmx =to_vmx (vcpu); -if (kvm_pmu_ops.is_valid_msr (vcpu,MSR_CORE_PERF_GLOBAL_CTRL )) { +if (static_call (kvm_x86_pmu_is_valid_msr)(vcpu,MSR_CORE_PERF_GLOBAL_CTRL )) { vmx->nested.msrs.entry_ctls_high |=VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL ; vmx->nested.msrs.exit_ctls_high |=arch/x86/kvm/x86.c +3
-2@@ -11317,8 +11317,10 @@ int kvm_arch_hardware_setup(void *opaque)memcpy (&kvm_x86_ops, ops->runtime_ops,sizeof (kvm_x86_ops));kvm_ops_static_call_update (); -if (kvm_x86_ops.hardware_enable) +if (kvm_x86_ops.hardware_enable) {memcpy (&kvm_pmu_ops, kvm_x86_ops.pmu_ops,sizeof (kvm_pmu_ops)); +kvm_pmu_ops_static_call_update (); + }if (!kvm_cpu_cap_has (X86_FEATURE_XSAVES )) supported_xss =0 ; --2.33.0
Re: [PATCH 3/3] KVM: x86: Use static calls to reduce kvm_pmu_ops overhead
Quoted 22 lines · Yao Yuan
Quoted 21 lines · Like Xu
Convert
kvm_pmu_opsto use static calls.Here are the worst
sched_clock()nanosecond numbers for thekvm_pmu_opsfunctions that is most often called (up to 7 digits of calls) when running a single perf test case in a guest on an ICX 2.70GHz host (mitigations=on):| legacy | static call
------------------------------------------------------------.pmc_idx_to_pmc | 10946 | 10047 (8%) .pmc_is_enabled | 11291 | 11175 (1%) .msr_idx_to_pmc | 13526 | 12346 (8%).is_valid_msr | 10895 | 10484 (3%)
Signed-off-by Like Xu <[email protected]>--- arch/x86/kvm/pmu.c | 36 +++++++++++++++++------------------- arch/x86/kvm/pmu.h | 2 +- arch/x86/kvm/vmx/nested.c | 2 +- arch/x86/kvm/x86.c | 4 +++- 4 files changed, 22 insertions(+), 22 deletions(-)
Quoted 15 lines
arch/x86/kvm/pmu.c void reprogram_gp_counter(struct kvm_pmc *pmc, u64 eventsel) · line 227
Quoted 10 lines
arch/x86/kvm/pmu.c +1
-1@@ -224,7 +224,7 @@ void reprogram_gp_counter(struct kvm_pmc *pmc, u64 eventsel) ARCH_PERFMON_EVENTSEL_CMASK | HSW_IN_TX | HSW_IN_TX_CHECKPOINTED))) { - config = kvm_pmu_ops.find_arch_event(pmc_to_pmu(pmc), + config = static_call(kvm_x86_pmu_find_arch_event)(pmc_to_pmu(pmc),Why you need change them into kvm_pmu_ops.XXX then convert them into static call ? Move the instance definition of
kvm_pmu_opsfrom patch 1 into patch 3 and then drop patch 1, will this work ?
You may git squash all commits to get it *work*.
With reference to afaf0b2f9b801c6eb2278b52d49e6a7d7b659cf1[1], doing one thing at a time will make things go smoother.
[1] https://lore.kernel.org/lkml/[email protected]/
Quoted 182 lines
Quoted 181 lines
arch/x86/kvm/pmu.c +16
-18event_select, unit_mask);if (config !=PERF_COUNT_HW_MAX )@@ -278,7 +278,7 @@ void reprogram_fixed_counter(struct kvm_pmc *pmc, u8 ctrl, int idx) pmc->current_config = (u64 )ctrl;pmc_reprogram_counter (pmc,PERF_TYPE_HARDWARE , - kvm_pmu_ops.find_fixed_event (idx), +static_call (kvm_x86_pmu_find_fixed_event)(idx), !(en_field &0x2 ),/* exclude user */ !(en_field &0x1 ),/* exclude kernel */ pmi, false, false);@@ -287,7 +287,7 @@ EXPORT_SYMBOL_GPL(reprogram_fixed_counter);void reprogram_counter (struct kvm_pmu *pmu,int pmc_idx) { -struct kvm_pmc *pmc = kvm_pmu_ops.pmc_idx_to_pmc (pmu, pmc_idx); +struct kvm_pmc *pmc =static_call (kvm_x86_pmu_pmc_idx_to_pmc)(pmu, pmc_idx);if (!pmc)return ;@@ -309,7 +309,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu)int bit;for_each_set_bit (bit, pmu->reprogram_pmi,X86_PMC_IDX_MAX ) { -struct kvm_pmc *pmc = kvm_pmu_ops.pmc_idx_to_pmc (pmu, bit); +struct kvm_pmc *pmc =static_call (kvm_x86_pmu_pmc_idx_to_pmc)(pmu, bit);if (unlikely (!pmc || !pmc->perf_event)) {clear_bit (bit, pmu->reprogram_pmi);@@ -331,7 +331,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu)/* check if idx is a valid index to access PMU */ int kvm_pmu_is_valid_rdpmc_ecx (struct kvm_vcpu *vcpu,unsigned int idx) { -return kvm_pmu_ops.is_valid_rdpmc_ecx (vcpu, idx); +return static_call (kvm_x86_pmu_is_valid_rdpmc_ecx)(vcpu, idx); }bool is_vmware_backdoor_pmc (u32 pmc_idx)@@ -381,7 +381,7 @@ int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)if (is_vmware_backdoor_pmc (idx))return kvm_pmu_rdpmc_vmware (vcpu, idx, data); - pmc = kvm_pmu_ops.rdpmc_ecx_to_pmc (vcpu, idx, &mask); + pmc =static_call (kvm_x86_pmu_rdpmc_ecx_to_pmc)(vcpu, idx, &mask);if (!pmc)return 1 ;@@ -397,22 +397,21 @@ int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)void kvm_pmu_deliver_pmi (struct kvm_vcpu *vcpu) {if (lapic_in_kernel (vcpu)) { -if (kvm_pmu_ops.deliver_pmi) - kvm_pmu_ops.deliver_pmi (vcpu); +static_call_cond (kvm_x86_pmu_deliver_pmi)(vcpu);kvm_apic_local_deliver (vcpu->arch.apic,APIC_LVTPC ); } }bool kvm_pmu_is_valid_msr (struct kvm_vcpu *vcpu,u32 msr) { -return kvm_pmu_ops.msr_idx_to_pmc (vcpu, msr) || - kvm_pmu_ops.is_valid_msr (vcpu, msr); +return static_call (kvm_x86_pmu_msr_idx_to_pmc)(vcpu, msr) || +static_call (kvm_x86_pmu_is_valid_msr)(vcpu, msr); }static void kvm_pmu_mark_pmc_in_use (struct kvm_vcpu *vcpu,u32 msr) {struct kvm_pmu *pmu =vcpu_to_pmu (vcpu); -struct kvm_pmc *pmc = kvm_pmu_ops.msr_idx_to_pmc (vcpu, msr); +struct kvm_pmc *pmc =static_call (kvm_x86_pmu_msr_idx_to_pmc)(vcpu, msr);if (pmc)__set_bit (pmc->idx, pmu->pmc_in_use);@@ -420,13 +419,13 @@ static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr)int kvm_pmu_get_msr (struct kvm_vcpu *vcpu,struct msr_data *msr_info) { -return kvm_pmu_ops.get_msr (vcpu, msr_info); +return static_call (kvm_x86_pmu_get_msr)(vcpu, msr_info); }int kvm_pmu_set_msr (struct kvm_vcpu *vcpu,struct msr_data *msr_info) {kvm_pmu_mark_pmc_in_use (vcpu, msr_info->index); -return kvm_pmu_ops.set_msr (vcpu, msr_info); +return static_call (kvm_x86_pmu_set_msr)(vcpu, msr_info); }/* refresh PMU settings. This function generally is called when underlying @@ -435,7 +434,7 @@ int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)*/ void kvm_pmu_refresh (struct kvm_vcpu *vcpu) { - kvm_pmu_ops.refresh (vcpu); +static_call (kvm_x86_pmu_refresh)(vcpu); }void kvm_pmu_reset (struct kvm_vcpu *vcpu)@@ -443,7 +442,7 @@ void kvm_pmu_reset(struct kvm_vcpu *vcpu)struct kvm_pmu *pmu =vcpu_to_pmu (vcpu);irq_work_sync (&pmu->irq_work); - kvm_pmu_ops.reset (vcpu); +static_call (kvm_x86_pmu_reset)(vcpu); }void kvm_pmu_init (struct kvm_vcpu *vcpu)@@ -451,7 +450,7 @@ void kvm_pmu_init(struct kvm_vcpu *vcpu)struct kvm_pmu *pmu =vcpu_to_pmu (vcpu);memset (pmu,0 ,sizeof (*pmu)); - kvm_pmu_ops.init (vcpu); +static_call (kvm_x86_pmu_init)(vcpu);init_irq_work (&pmu->irq_work, kvm_pmi_trigger_fn); pmu->event_count =0 ; pmu->need_cleanup = false;@@ -483,14 +482,13 @@ void kvm_pmu_cleanup(struct kvm_vcpu *vcpu) pmu->pmc_in_use, X86_PMC_IDX_MAX);for_each_set_bit (i, bitmask,X86_PMC_IDX_MAX ) { - pmc = kvm_pmu_ops.pmc_idx_to_pmc (pmu, i); + pmc =static_call (kvm_x86_pmu_pmc_idx_to_pmc)(pmu, i);if (pmc && pmc->perf_event && !pmc_speculative_in_use (pmc))pmc_stop_counter (pmc); } -if (kvm_pmu_ops.cleanup) - kvm_pmu_ops.cleanup (vcpu); +static_call_cond (kvm_x86_pmu_cleanup)(vcpu);bitmap_zero (pmu->pmc_in_use,X86_PMC_IDX_MAX ); }arch/x86/kvm/pmu.h +1
-1@@ -109,7 +109,7 @@ static inline bool pmc_is_fixed(struct kvm_pmc *pmc)static inline bool pmc_is_enabled (struct kvm_pmc *pmc) { -return kvm_pmu_ops.pmc_is_enabled (pmc); +return static_call (kvm_x86_pmu_pmc_is_enabled)(pmc); } static inline bool kvm_valid_perf_global_ctrl(struct kvm_pmu *pmu,arch/x86/kvm/vmx/nested.c +1
-1@@ -4796,7 +4796,7 @@ void nested_vmx_pmu_entry_exit_ctls_update(struct kvm_vcpu *vcpu)return ; vmx =to_vmx (vcpu); -if (kvm_pmu_ops.is_valid_msr (vcpu,MSR_CORE_PERF_GLOBAL_CTRL )) { +if (static_call (kvm_x86_pmu_is_valid_msr)(vcpu,MSR_CORE_PERF_GLOBAL_CTRL )) { vmx->nested.msrs.entry_ctls_high |=VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL ; vmx->nested.msrs.exit_ctls_high |=arch/x86/kvm/x86.c +3
-2@@ -11317,8 +11317,10 @@ int kvm_arch_hardware_setup(void *opaque)memcpy (&kvm_x86_ops, ops->runtime_ops,sizeof (kvm_x86_ops));kvm_ops_static_call_update (); -if (kvm_x86_ops.hardware_enable) +if (kvm_x86_ops.hardware_enable) {memcpy (&kvm_pmu_ops, kvm_x86_ops.pmu_ops,sizeof (kvm_pmu_ops)); +kvm_pmu_ops_static_call_update (); + }if (!kvm_cpu_cap_has (X86_FEATURE_XSAVES )) supported_xss =0 ; --2.33.0