KVM: x86: Provide per VM capability for disabling PMU virtualization
Subsystems (5)
v7 cover KVM: x86: Provide per VM capability for disabling PMU virtualization
Allow usermode to disable PMU virtualization on individual x86 VMs. When disabled, the PMU is not accessible from the guest.
v6: https://lore.kernel.org/all/[email protected]/
v6 -> v7
* Incorporating feedback from Sean.
* Rephrased commit messages (borrowed from Sean).
* Added locking around kvm->created_vcpus access.
* Moved KVM_CAP_PMU_VALID_MASK out of user ABI headers.
* Changed comments on vm_create_without_vcpus.
* Directly check results in selftest TEST_ASSERT statements.
v5 -> v6
* resolve minor conflicts that were queued after v5 was reviewed.
v4 -> v5
* Remove automatic CPUID adjustment when PMU disabled.
* Update documentation and changelog to reflect above.
* Update documentation to document arg[0] and return values.
David Dunn (3):
KVM: x86: Provide per VM capability for disabling PMU virtualization
KVM: selftests: Carve out helper to create "default" VM without vCPUs
KVM: selftests: Verify disabling PMU virtualization via
KVM_CAP_CONFIG_PMU
Documentation/virt/kvm/api.rst | 22 ++++++++++++ arch/x86/include/asm/kvm_host.h | 1 + arch/x86/kvm/svm/pmu.c | 2 +- arch/x86/kvm/vmx/pmu_intel.c | 2 +- arch/x86/kvm/x86.c | 17 +++++++++ include/uapi/linux/kvm.h | 3 ++ tools/include/uapi/linux/kvm.h | 3 ++ .../selftests/kvm/include/kvm_util_base.h | 3 ++ tools/testing/selftests/kvm/lib/kvm_util.c | 35 +++++++++++++++---- .../kvm/x86_64/pmu_event_filter_test.c | 33 +++++++++++++++++ 10 files changed, 113 insertions(+), 8 deletions(-)
Signature · 1 line
2.35.1.265.g69c8d7142f-goog
PATCH v7 1/3 KVM: x86: Provide per VM capability for disabling PMU virtualization
Add a new capability, KVM_CAP_PMU_CAPABILITY, that takes a bitmask of settings/features to allow userspace to configure PMU virtualization on a per-VM basis. For now, support a single flag, KVM_CAP_PMU_DISABLE, to allow disabling PMU virtualization for a VM even when KVM is configured with enable_pmu=true a module level.
To keep KVM simple, disallow changing VM's PMU configuration after vCPUs have been created.
--- Documentation/virt/kvm/api.rst | 22 ++++++++++++++++++++++ arch/x86/include/asm/kvm_host.h | 1 + arch/x86/kvm/svm/pmu.c | 2 +- arch/x86/kvm/vmx/pmu_intel.c | 2 +- arch/x86/kvm/x86.c | 17 +++++++++++++++++ include/uapi/linux/kvm.h | 3 +++ tools/include/uapi/linux/kvm.h | 3 +++ 7 files changed, 48 insertions(+), 2 deletions(-)
Documentation/virt/kvm/api.rst +22 -0
@@ -7561,3 +7561,25 @@ The argument to KVM_ENABLE_CAP is also a bitmask, and must be a subset of the result of KVM_CHECK_EXTENSION. KVM will forward to userspace the hypercalls whose corresponding bit is in the argument, and return ENOSYS for the others. + +8.35 KVM_CAP_PMU_CAPABILITY +--------------------------- + +:Capability KVM_CAP_PMU_CAPABILITY +:Architectures: x86 +:Type: vm +:Parameters: arg[0] is bitmask of PMU virtualization capabilities. +:Returns 0 on success, -EINVAL when arg[0] contains invalid bits + +This capability alters PMU virtualization in KVM. + +Calling KVM_CHECK_EXTENSION for this capability returns a bitmask of +PMU virtualization capabilities that can be adjusted on a VM. + +The argument to KVM_ENABLE_CAP is also a bitmask and selects specific +PMU virtualization capabilities to be applied to the VM. This can +only be invoked on a VM prior to the creation of VCPUs. + +At this time, KVM_CAP_PMU_DISABLE is the only capability. Setting +this capability will disable PMU virtualization for that VM. Usermode +should adjust CPUID leaf 0xA to reflect that the PMU is disabled.
arch/x86/include/asm/kvm_host.h +1 -0
@@ -1233,6 +1233,7 @@ struct kvm_arch {hpa_t hv_root_tdp;spinlock_t hv_root_tdp_lock;#endif +bool enable_pmu; };struct kvm_vm_stat {
arch/x86/kvm/svm/pmu.c +1 -1
@@ -101,7 +101,7 @@ static inline struct kvm_pmc *get_gp_pmc_amd(struct kvm_pmu *pmu, u32 msr, {struct kvm_vcpu *vcpu =pmu_to_vcpu (pmu); -if (!enable_pmu) +if (!vcpu->kvm->arch.enable_pmu)return NULL ;switch (msr) {
arch/x86/kvm/vmx/pmu_intel.c +1 -1
@@ -487,7 +487,7 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu) pmu->reserved_bits =0xffffffff00200000ull ; entry =kvm_find_cpuid_entry (vcpu,0xa ,0 ); -if (!entry || !enable_pmu) +if (!entry || !vcpu->kvm->arch.enable_pmu)return ; eax.full = entry->eax; edx.full = entry->edx;
arch/x86/kvm/x86.c +17 -0
@@ -110,6 +110,8 @@ static u64 __read_mostly cr4_reserved_bits = CR4_RESERVED_BITS;#define KVM_EXIT_HYPERCALL_VALID_MASK (1 << KVM_HC_MAP_GPA_RANGE) +#define KVM_CAP_PMU_VALID_MASK KVM_CAP_PMU_DISABLE +#define KVM_X2APIC_API_VALID_FLAGS (KVM_X2APIC_API_USE_32BIT_IDS | \ KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK)
@@ -4331,6 +4333,9 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)if (r <sizeof (struct kvm_xsave )) r =sizeof (struct kvm_xsave );break ; +case KVM_CAP_PMU_CAPABILITY : + r = enable_pmu ?KVM_CAP_PMU_VALID_MASK :0 ; +break ; }default :break ;
@@ -6005,6 +6010,17 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm, kvm->arch.exit_on_emulation_error = cap->args[0 ]; r =0 ;break ; +case KVM_CAP_PMU_CAPABILITY : + r = -EINVAL ; +if (!enable_pmu || (cap->args[0 ] & ~KVM_CAP_PMU_VALID_MASK )) +break ; +mutex_lock (&kvm->lock); +if (!kvm->created_vcpus) { + kvm->arch.enable_pmu = !(cap->args[0 ] &KVM_CAP_PMU_DISABLE ); + r =0 ; + } +mutex_unlock (&kvm->lock); +break ;default : r = -EINVAL ;break ;
@@ -11591,6 +11607,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)raw_spin_unlock_irqrestore (&kvm->arch.tsc_write_lock, flags); kvm->arch.guest_can_read_msr_platform_info = true; + kvm->arch.enable_pmu = enable_pmu;#if IS_ENABLED (CONFIG_HYPERV )spin_lock_init (&kvm->arch.hv_root_tdp_lock);
include/uapi/linux/kvm.h +3 -0
@@ -1134,6 +1134,7 @@ struct kvm_ppc_resize_hpt {#define KVM_CAP_VM_GPA_BITS 207#define KVM_CAP_XSAVE2 208#define KVM_CAP_SYS_ATTRIBUTES 209 +#define KVM_CAP_PMU_CAPABILITY 210#ifdef KVM_CAP_IRQ_ROUTING
@@ -1970,6 +1971,8 @@ struct kvm_dirty_gfn {#define KVM_BUS_LOCK_DETECTION_OFF (1 << 0)#define KVM_BUS_LOCK_DETECTION_EXIT (1 << 1) +#define KVM_CAP_PMU_DISABLE (1 << 0) +/** * struct kvm_stats_header - Header of per vm/vcpu binary statistics data. * @flags: Some extra information for header, always 0 for now.
tools/include/uapi/linux/kvm.h +3 -0
@@ -1134,6 +1134,7 @@ struct kvm_ppc_resize_hpt {#define KVM_CAP_VM_GPA_BITS 207#define KVM_CAP_XSAVE2 208#define KVM_CAP_SYS_ATTRIBUTES 209 +#define KVM_CAP_PMU_CAPABILITY 210#ifdef KVM_CAP_IRQ_ROUTING
@@ -1970,6 +1971,8 @@ struct kvm_dirty_gfn {#define KVM_BUS_LOCK_DETECTION_OFF (1 << 0)#define KVM_BUS_LOCK_DETECTION_EXIT (1 << 1) +#define KVM_CAP_PMU_DISABLE (1 << 0) +/** * struct kvm_stats_header - Header of per vm/vcpu binary statistics data. * @flags: Some extra information for header, always 0 for now.
Signature · 1 line
2.35.1.265.g69c8d7142f-goog
Quoted 11 lines · David Dunn
Add a new capability,
KVM_CAP_PMU_CAPABILITY, that takes a bitmask of settings/features to allow userspace to configure PMU virtualization on a per-VM basis. For now, support a single flag,KVM_CAP_PMU_DISABLE, to allow disabling PMU virtualization for a VM even when KVM is configured with enable_pmu=true a module level.To keep KVM simple, disallow changing VM's PMU configuration after vCPUs have been created.
Signed-off-by David Dunn <[email protected]>---
A few nits, otherwise
Somewhat off-topic, looking at this got me looking at vmx_get_perf_capabilities(). We really should cache the result of its RDMSR.
And I believe emulation of WRMSR from the guest to MSR_IA32_PERF_CAPABILITIES is missing a guest_cpuid_has() check.
Quoted 15 lines
Documentation/virt/kvm/api.rst | 22 ++++++++++++++++++++++ arch/x86/include/asm/kvm_host.h | 1 + arch/x86/kvm/svm/pmu.c | 2 +- arch/x86/kvm/vmx/pmu_intel.c | 2 +- arch/x86/kvm/x86.c | 17 +++++++++++++++++ include/uapi/linux/kvm.h | 3 +++ tools/include/uapi/linux/kvm.h | 3 +++ 7 files changed, 48 insertions(+), 2 deletions(-)Documentation/virt/kvm/api.rst +0
-0@@ -7561,3 +7561,25 @@ The argument to KVM_ENABLE_CAP is also a bitmask, and must be a subset of the result of KVM_CHECK_EXTENSION. KVM will forward to userspace
Documentation/virt/kvm/api.rst The argument to KVM_ENABLE_CAP is also a bitmask, and must be a subset · line 7571
Quoted 10 lines
Documentation/virt/kvm/api.rst +8
-0the hypercalls whose corresponding bit is in the argument, and return ENOSYS for the others. + +8.35 KVM_CAP_PMU_CAPABILITY +--------------------------- + +:Capability KVM_CAP_PMU_CAPABILITY +:Architectures: x86 +:Type: vm +:Parameters: arg[0] is bitmask of PMU virtualization capabilities.
Referring to "DISABLE" as a capabilitiy is rather odd, but I don't have a better suggestion.
Quoted 19 lines
Documentation/virt/kvm/api.rst +14
-0+:Returns 0 on success, -EINVAL when arg[0] contains invalid bits + +This capability alters PMU virtualization in KVM. + +Calling KVM_CHECK_EXTENSION for this capability returns a bitmask of +PMU virtualization capabilities that can be adjusted on a VM. + +The argument to KVM_ENABLE_CAP is also a bitmask and selects specific +PMU virtualization capabilities to be applied to the VM. This can +only be invoked on a VM prior to the creation of VCPUs. + +At this time, KVM_CAP_PMU_DISABLE is the only capability. Setting +this capability will disable PMU virtualization for that VM. Usermode +should adjust CPUID leaf 0xA to reflect that the PMU is disabled.arch/x86/include/asm/kvm_host.h +0
-0
arch/x86/include/asm/kvm_host.h struct kvm_arch { · line 1236
Quoted 4 lines
arch/x86/include/asm/kvm_host.h +1
-0@@ -1233,6 +1233,7 @@ struct kvm_arch {hpa_t hv_root_tdp;spinlock_t hv_root_tdp_lock;#endif +bool enable_pmu;
Doesn''t really matter, but I'd prefer we put this up among the many other bools, e.g.
arch/x86/include/asm/kvm_host.h +1 -1
@@ -1150,6 +1150,7 @@ struct kvm_arch {bool guest_can_read_msr_platform_info;bool exception_payload_enabled; +bool enable_pmu;bool bus_lock_detection_enabled;/* * If exit_on_emulation_error is set, and the in-kernel instruction
@@ -1233,7 +1234,6 @@ struct kvm_arch {hpa_t hv_root_tdp;spinlock_t hv_root_tdp_lock;#endif -bool enable_pmu; };struct kvm_vm_stat {
Quoted 53 lines
arch/x86/include/asm/kvm_host.h +0
-0};struct kvm_vm_stat {arch/x86/kvm/svm/pmu.c +1
-1@@ -101,7 +101,7 @@ static inline struct kvm_pmc *get_gp_pmc_amd(struct kvm_pmu *pmu, u32 msr, {struct kvm_vcpu *vcpu =pmu_to_vcpu (pmu); -if (!enable_pmu) +if (!vcpu->kvm->arch.enable_pmu)return NULL ;switch (msr) {arch/x86/kvm/vmx/pmu_intel.c +1
-1@@ -487,7 +487,7 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu) pmu->reserved_bits =0xffffffff00200000ull ; entry =kvm_find_cpuid_entry (vcpu,0xa ,0 ); -if (!entry || !enable_pmu) +if (!entry || !vcpu->kvm->arch.enable_pmu)return ; eax.full = entry->eax; edx.full = entry->edx;arch/x86/kvm/x86.c +5
-0@@ -110,6 +110,8 @@ static u64 __read_mostly cr4_reserved_bits = CR4_RESERVED_BITS;#define KVM_EXIT_HYPERCALL_VALID_MASK (1 << KVM_HC_MAP_GPA_RANGE) +#define KVM_CAP_PMU_VALID_MASK KVM_CAP_PMU_DISABLE +#define KVM_X2APIC_API_VALID_FLAGS (KVM_X2APIC_API_USE_32BIT_IDS | \ KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK)@@ -4331,6 +4333,9 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)if (r <sizeof (struct kvm_xsave )) r =sizeof (struct kvm_xsave );break ; +case KVM_CAP_PMU_CAPABILITY : + r = enable_pmu ?KVM_CAP_PMU_VALID_MASK :0 ; +break ; }default :break ;
arch/x86/kvm/x86.c int kvm_vm_ioctl_enable_cap(struct kvm *kvm, · line 6016
Quoted 7 lines
arch/x86/kvm/x86.c +4
-0@@ -6005,6 +6010,17 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm, kvm->arch.exit_on_emulation_error = cap->args[0 ]; r =0 ;break ; +case KVM_CAP_PMU_CAPABILITY : + r = -EINVAL ; +if (!enable_pmu || (cap->args[0 ] & ~KVM_CAP_PMU_VALID_MASK )) +break ;
I'd prefer another newline here, the other case statements are just bad influences :-)
Quoted 10 lines
arch/x86/kvm/x86.c +7
-0+mutex_lock (&kvm->lock); +if (!kvm->created_vcpus) { + kvm->arch.enable_pmu = !(cap->args[0 ] &KVM_CAP_PMU_DISABLE ); + r =0 ; + } +mutex_unlock (&kvm->lock); +break ;default : r = -EINVAL ;break ;
PATCH v7 2/3 KVM: selftests: Carve out helper to create "default" VM without vCPUs
In reply to David DunnCarve out portion of vm_create_default so that selftests can modify a "default" VM prior to creating vcpus.
--- .../selftests/kvm/include/kvm_util_base.h | 3 ++ tools/testing/selftests/kvm/lib/kvm_util.c | 35 +++++++++++++++---- 2 files changed, 32 insertions(+), 6 deletions(-)
tools/testing/selftests/kvm/include/kvm_util_base.h +3 -0
@@ -336,6 +336,9 @@ struct kvm_vm *vm_create_with_vcpus(enum vm_guest_mode mode, uint32_t nr_vcpus, uint32_t num_percpu_pages, void *guest_code, uint32_t vcpuids[]); +/* Create a default VM without any vcpus. */ +struct kvm_vm *vm_create_without_vcpus (enum vm_guest_mode mode,uint64_t pages); +/* * Adds a vCPU with reasonable defaults (e.g. a stack) *
tools/testing/selftests/kvm/lib/kvm_util.c +29 -6
@@ -362,6 +362,34 @@ struct kvm_vm *vm_create(enum vm_guest_mode mode, uint64_t phy_pages, int perm)return vm; } +/* +* Default VM creation without creating VCPUs +* +* Input Args: +* mode - VM Mode (e.g. VM_MODE_P52V48_4K) +* pages - pages of memory required for VM +* +* Output Args: None +* +* Return: +* Pointer to opaque structure that describes the created VM. +* +* Creates a VM with the mode specified by mode (e.g. VM_MODE_P52V48_4K). +*/ +struct kvm_vm *vm_create_without_vcpus (enum vm_guest_mode mode,uint64_t pages) +{ +struct kvm_vm *vm; + + vm =vm_create (mode, pages,O_RDWR ); + +kvm_vm_elf_load (vm, program_invocation_name); + +#ifdef __x86_64__ +vm_create_irqchip (vm); +#endif +return vm; +} +/* * VM Create with customized parameters *
@@ -412,13 +440,8 @@ struct kvm_vm *vm_create_with_vcpus(enum vm_guest_mode mode, uint32_t nr_vcpus, nr_vcpus, kvm_check_cap(KVM_CAP_MAX_VCPUS)); pages =vm_adjust_num_guest_pages (mode, pages); - vm =vm_create (mode, pages,O_RDWR ); - -kvm_vm_elf_load (vm, program_invocation_name); -#ifdef __x86_64__ -vm_create_irqchip (vm); -#endif + vm =vm_create_without_vcpus (mode, pages);for (i =0 ; i < nr_vcpus; ++i) {uint32_t vcpuid = vcpuids ? vcpuids[i] : i;
Signature · 1 line
2.35.1.265.g69c8d7142f-goog
Sean Christopherson
maintainerRe: [PATCH v7 2/3] KVM: selftests: Carve out helper to create "default" VM without vCPUs
Quoted 35 lines · David Dunn
Carve out portion of
vm_create_defaultso that selftests can modify a "default" VM prior to creating vcpus.Signed-off-by David Dunn <[email protected]>--- .../selftests/kvm/include/kvm_util_base.h | 3 ++ tools/testing/selftests/kvm/lib/kvm_util.c | 35 +++++++++++++++---- 2 files changed, 32 insertions(+), 6 deletions(-)tools/testing/selftests/kvm/include/kvm_util_base.h +3
-0@@ -336,6 +336,9 @@ struct kvm_vm *vm_create_with_vcpus(enum vm_guest_mode mode, uint32_t nr_vcpus, uint32_t num_percpu_pages, void *guest_code, uint32_t vcpuids[]); +/* Create a default VM without any vcpus. */ +struct kvm_vm *vm_create_without_vcpus (enum vm_guest_mode mode,uint64_t pages); +/* * Adds a vCPU with reasonable defaults (e.g. a stack) * tools/testing/selftests/kvm/lib/kvm_util.c +4
-0@@ -362,6 +362,34 @@ struct kvm_vm *vm_create(enum vm_guest_mode mode, uint64_t phy_pages, int perm)return vm; } +/* +* Default VM creation without creating VCPUs +* +* Input Args:
tools/testing/selftests/kvm/lib/kvm_util.c struct kvm_vm *vm_create(enum vm_guest_mode mode, uint64_t phy_pages, int perm) · line 378
Quoted 10 lines
tools/testing/selftests/kvm/lib/kvm_util.c +10
-0+* mode - VM Mode (e.g. VM_MODE_P52V48_4K) +* pages - pages of memory required for VM +* +* Output Args: None +* +* Return: +* Pointer to opaque structure that describes the created VM. +* +* Creates a VM with the mode specified by mode (e.g. VM_MODE_P52V48_4K). +*/
I vote to omit this function comment. Most of the existing comments in kvm_util.c are a waste of space, and (no offence) this is no different. And I'm planning on deleting most of the existing boilerplate comments that don't provide any insight :-)
Quoted 35 lines
tools/testing/selftests/kvm/lib/kvm_util.c +15
-6+struct kvm_vm *vm_create_without_vcpus (enum vm_guest_mode mode,uint64_t pages) +{ +struct kvm_vm *vm; + + vm =vm_create (mode, pages,O_RDWR ); + +kvm_vm_elf_load (vm, program_invocation_name); + +#ifdef __x86_64__ +vm_create_irqchip (vm); +#endif +return vm; +} +/* * VM Create with customized parameters * @@ -412,13 +440,8 @@ struct kvm_vm *vm_create_with_vcpus(enum vm_guest_mode mode, uint32_t nr_vcpus, nr_vcpus, kvm_check_cap(KVM_CAP_MAX_VCPUS)); pages =vm_adjust_num_guest_pages (mode, pages); - vm =vm_create (mode, pages,O_RDWR ); - -kvm_vm_elf_load (vm, program_invocation_name); -#ifdef __x86_64__ -vm_create_irqchip (vm); -#endif + vm =vm_create_without_vcpus (mode, pages);for (i =0 ; i < nr_vcpus; ++i) {uint32_t vcpuid = vcpuids ? vcpuids[i] : i;Signature · 1 line
2.35.1.265.g69c8d7142f-goog
PATCH v7 3/3 KVM: selftests: Verify disabling PMU virtualization via KVM_CAP_CONFIG_PMU
In reply to David DunnOn a VM with PMU disabled via KVM_CAP_PMU_CONFIG, the PMU should not be usable by the guest.
--- .../kvm/x86_64/pmu_event_filter_test.c | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+)
tools/testing/selftests/kvm/x86_64/pmu_event_filter_test.c +33 -0
@@ -325,6 +325,37 @@ static void test_not_member_allow_list(struct kvm_vm *vm)TEST_ASSERT (!count,"Disallowed PMU Event is counting" ); } +/* +* Verify KVM_CAP_PMU_DISABLE prevents the use of the PMU. +* +* Note that KVM_CAP_PMU_CAPABILITY must be invoked prior to creating VCPUs. +*/ +static void test_pmu_config_disable (void (*guest_code)(void )) +{ +int r; +struct kvm_vm *vm; +struct kvm_enable_cap cap = {0 }; + + r =kvm_check_cap (KVM_CAP_PMU_CAPABILITY ); +if (!(r &KVM_CAP_PMU_DISABLE )) +return ; + + vm =vm_create_without_vcpus (VM_MODE_DEFAULT ,DEFAULT_GUEST_PHY_PAGES ); + + cap.cap =KVM_CAP_PMU_CAPABILITY ; + cap.args[0 ] =KVM_CAP_PMU_DISABLE ; +TEST_ASSERT (!vm_enable_cap (vm, &cap),"Failed KVM_CAP_PMU_DISABLE." ); + +vm_vcpu_add_default (vm,VCPU_ID , guest_code); +vm_init_descriptor_tables (vm); +vcpu_init_descriptor_tables (vm,VCPU_ID ); + +TEST_ASSERT (!sanity_check_pmu (vm), +"Guest should not be able to use disabled PMU." ); + +kvm_vm_free (vm); +} +/* * Check for a non-zero PMU version, at least one general-purpose * counter per logical processor, an EBX bit vector of length greater
@@ -430,5 +461,7 @@ int main(int argc, char *argv[])kvm_vm_free (vm); +test_pmu_config_disable (guest_code); +return 0 ; }
Signature · 1 line
2.35.1.265.g69c8d7142f-goog