Skip to contentSkip to conversation

Provide VM capability to disable PMU virtualization for individual VMs

PATCH In review David Dunn 2 participants 5 messages Jan 21, 2022

Subsystems (4)
lore
Share & tools
lore

David Dunn

author
#0 lore raw

v2 cover Provide VM capability to disable PMU virtualization for individual VMs

This patchset adds a new per VM capability for configuring the x86 PMU. Right now the only configuration is to disable the PMU for an entire VM.

Thanks to Sean and Like for great feedback on v1.

I've incorporated most of the comments. Unfortunately I wasn't able to get the AMD version to work by using pmu->version == 0 as a flag. And it makes the Intel pmu->refresh() logic complicated. So this version stays with the kvm->arch.enable_pmu boolean on each VM.

David Dunn (3):
Provide VM capability to disable PMU virtualization for individual VMs
selftests: introduce function to create test VM without vcpus.
selftests: Verify disabling PMU via KVM_CAP_CONFIG_PMU

 arch/x86/include/asm/kvm_host.h               |  1 +
 arch/x86/kvm/svm/pmu.c                        |  2 +-
 arch/x86/kvm/vmx/pmu_intel.c                  |  2 +-
 arch/x86/kvm/x86.c                            | 12 +++++
 include/uapi/linux/kvm.h                      |  4 ++
 tools/include/uapi/linux/kvm.h                |  4 ++
 .../selftests/kvm/include/kvm_util_base.h     |  3 ++
 tools/testing/selftests/kvm/lib/kvm_util.c    | 48 ++++++++++++++-----
 .../kvm/x86_64/pmu_event_filter_test.c        | 35 ++++++++++++++
 9 files changed, 96 insertions(+), 15 deletions(-)
Signature · 1 line
2.35.0.rc0.227.g00780c9af4-goog
author
#1 lore raw

PATCH v2 1/3 Provide VM capability to disable PMU virtualization for individual VMs

KVM_PMU_CONFIG_DISABLE can be used to disable PMU virtualization on individual x86 VMs. PMU configuration must be done prior to creating VCPUs.

To enable future extension, KVM_CAP_PMU_CONFIG reports available settings when queried via check_extension.

Since KVM_GET_SUPPORTED_CPUID reports the maximal CPUID information based on module parameters, usermode will need to adjust CPUID when disabling PMU virtualization on individual VMs.

---
 arch/x86/include/asm/kvm_host.h |  1 +
 arch/x86/kvm/svm/pmu.c          |  2 +-
 arch/x86/kvm/vmx/pmu_intel.c    |  2 +-
 arch/x86/kvm/x86.c              | 12 ++++++++++++
 include/uapi/linux/kvm.h        |  4 ++++
 tools/include/uapi/linux/kvm.h  |  4 ++++
 6 files changed, 23 insertions(+), 2 deletions(-)
arch/x86/include/asm/kvm_host.h +1 -0
index 682ad02a4e58..5cdcd4a7671b 100644
@@ -1232,6 +1232,7 @@ struct kvm_arch {
 	hpa_t	hv_root_tdp;
 	spinlock_t hv_root_tdp_lock;
 #endif
+	bool enable_pmu;
 };
 
 struct kvm_vm_stat {
arch/x86/kvm/svm/pmu.c +1 -1
index 5aa45f13b16d..d4de52409335 100644
@@ -101,7 +101,7 @@ static inline struct kvm_pmc *get_gp_pmc_amd(struct kvm_pmu *pmu, u32 msr,
 {
 	struct kvm_vcpu *vcpu = pmu_to_vcpu(pmu);
 
-	if (!enable_pmu)
+	if (!vcpu->kvm->arch.enable_pmu)
 		return NULL;
 
 	switch (msr) {
arch/x86/kvm/vmx/pmu_intel.c +1 -1
index 466d18fc0c5d..2c5868d77268 100644
@@ -487,7 +487,7 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu)
 	pmu->reserved_bits = 0xffffffff00200000ull;
 
 	entry = kvm_find_cpuid_entry(vcpu, 0xa, 0);
-	if (!entry || !enable_pmu)
+	if (!entry || !vcpu->kvm->arch.enable_pmu)
 		return;
 	eax.full = entry->eax;
 	edx.full = entry->edx;
arch/x86/kvm/x86.c +12 -0
index 55518b7d3b96..42a98635bea5 100644
@@ -4326,6 +4326,9 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 		if (r < sizeof(struct kvm_xsave))
 			r = sizeof(struct kvm_xsave);
 		break;
+	case KVM_CAP_PMU_CONFIG:
+		r = enable_pmu ? KVM_PMU_CONFIG_VALID : 0;
+		break;
 	}
 	default:
 		break;
@@ -5937,6 +5940,14 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
 		kvm->arch.exit_on_emulation_error = cap->args[0];
 		r = 0;
 		break;
+	case KVM_CAP_PMU_CONFIG:
+		r = -EINVAL;
+		if (!enable_pmu || kvm->created_vcpus > 0 ||
+		    cap->args[0] & ~KVM_PMU_CONFIG_VALID)
+			break;
+		kvm->arch.enable_pmu = !(cap->args[0] & KVM_PMU_CONFIG_DISABLE);
+		r = 0;
+		break;
 	default:
 		r = -EINVAL;
 		break;
@@ -11562,6 +11573,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 	raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
 
 	kvm->arch.guest_can_read_msr_platform_info = true;
+	kvm->arch.enable_pmu = enable_pmu;
 
 #if IS_ENABLED(CONFIG_HYPERV)
 	spin_lock_init(&kvm->arch.hv_root_tdp_lock);
include/uapi/linux/kvm.h +4 -0
index 9563d294f181..57a1280fa43b 100644
@@ -1133,6 +1133,7 @@ struct kvm_ppc_resize_hpt {
 #define KVM_CAP_VM_MOVE_ENC_CONTEXT_FROM 206
 #define KVM_CAP_VM_GPA_BITS 207
 #define KVM_CAP_XSAVE2 208
+#define KVM_CAP_PMU_CONFIG 209
 
 #ifdef KVM_CAP_IRQ_ROUTING
 
@@ -1972,6 +1973,9 @@ struct kvm_dirty_gfn {
 #define KVM_BUS_LOCK_DETECTION_OFF             (1 << 0)
 #define KVM_BUS_LOCK_DETECTION_EXIT            (1 << 1)
 
+#define KVM_PMU_CONFIG_DISABLE                 (1 << 0)
+#define KVM_PMU_CONFIG_VALID                   (KVM_PMU_CONFIG_DISABLE)
+
 /**
  * struct kvm_stats_header - Header of per vm/vcpu binary statistics data.
  * @flags: Some extra information for header, always 0 for now.
tools/include/uapi/linux/kvm.h +4 -0
index 9563d294f181..57a1280fa43b 100644
@@ -1133,6 +1133,7 @@ struct kvm_ppc_resize_hpt {
 #define KVM_CAP_VM_MOVE_ENC_CONTEXT_FROM 206
 #define KVM_CAP_VM_GPA_BITS 207
 #define KVM_CAP_XSAVE2 208
+#define KVM_CAP_PMU_CONFIG 209
 
 #ifdef KVM_CAP_IRQ_ROUTING
 
@@ -1972,6 +1973,9 @@ struct kvm_dirty_gfn {
 #define KVM_BUS_LOCK_DETECTION_OFF             (1 << 0)
 #define KVM_BUS_LOCK_DETECTION_EXIT            (1 << 1)
 
+#define KVM_PMU_CONFIG_DISABLE                 (1 << 0)
+#define KVM_PMU_CONFIG_VALID                   (KVM_PMU_CONFIG_DISABLE)
+
 /**
  * struct kvm_stats_header - Header of per vm/vcpu binary statistics data.
  * @flags: Some extra information for header, always 0 for now.
Signature · 1 line
2.35.0.rc0.227.g00780c9af4-goog
#2 lore raw

Hi David,

Please modify the subject of this patch to follow the convention.

Quoted 1 line · David Dunn

KVM_PMU_CONFIG_DISABLE can be used to disable PMU virtualization on

How about using:

case KVM_CAP_PMU_CAPABILITY
#define KVM_CAP_PMU_DISABLE (1 << 0)
#define KVM_CAP_PMU_MASK (KVM_CAP_PMU_DISABLE)

Quoted 5 lines

individual x86 VMs. PMU configuration must be done prior to creating VCPUs.

To enable future extension, KVM_CAP_PMU_CONFIG reports available settings when queried via check_extension.

Please help update the document in the Documentation/virt/kvm/api.rst

Quoted 4 lines

Since KVM_GET_SUPPORTED_CPUID reports the maximal CPUID information based on module parameters, usermode will need to adjust CPUID when disabling PMU virtualization on individual VMs.

For an irrational user space (set both pmu cpuid bits and KVM_CAP_PMU_DISABLE), how about we remove the relevant PMU cpuid bits in kvm_pmu_refresh() silently?

Quoted 28 lines
---
  arch/x86/include/asm/kvm_host.h |  1 +
  arch/x86/kvm/svm/pmu.c          |  2 +-
  arch/x86/kvm/vmx/pmu_intel.c    |  2 +-
  arch/x86/kvm/x86.c              | 12 ++++++++++++
  include/uapi/linux/kvm.h        |  4 ++++
  tools/include/uapi/linux/kvm.h  |  4 ++++
  6 files changed, 23 insertions(+), 2 deletions(-)
arch/x86/include/asm/kvm_host.h +1 -0
index 682ad02a4e58..5cdcd4a7671b 100644
@@ -1232,6 +1232,7 @@ struct kvm_arch {
  	hpa_t	hv_root_tdp;
  	spinlock_t hv_root_tdp_lock;
  #endif
+	bool enable_pmu;
  };
 
  struct kvm_vm_stat {
arch/x86/kvm/svm/pmu.c +0 -0
index 5aa45f13b16d..d4de52409335 100644

arch/x86/kvm/svm/pmu.c static inline struct kvm_pmc *get_gp_pmc_amd(struct kvm_pmu *pmu, u32 msr, · line 104

Quoted 5 lines
arch/x86/kvm/svm/pmu.c +1 -1
@@ -101,7 +101,7 @@ static inline struct kvm_pmc *get_gp_pmc_amd(struct kvm_pmu *pmu, u32 msr,
  {
  	struct kvm_vcpu *vcpu = pmu_to_vcpu(pmu);
 
-	if (!enable_pmu)
+	if (!vcpu->kvm->arch.enable_pmu)

Compared to diff in the v1:

- if (!enable_pmu)
+ if (!enable_pmu || !vcpu->kvm->arch.enable_pmu)

Now I have come to appreciate the ingenious name.

Quoted 21 lines
arch/x86/kvm/svm/pmu.c +0 -0
  		return NULL;
 
  	switch (msr) {
arch/x86/kvm/vmx/pmu_intel.c +1 -1
index 466d18fc0c5d..2c5868d77268 100644
@@ -487,7 +487,7 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu)
  	pmu->reserved_bits = 0xffffffff00200000ull;
 
  	entry = kvm_find_cpuid_entry(vcpu, 0xa, 0);
-	if (!entry || !enable_pmu)
+	if (!entry || !vcpu->kvm->arch.enable_pmu)
  		return;
  	eax.full = entry->eax;
  	edx.full = entry->edx;
arch/x86/kvm/x86.c +0 -0
index 55518b7d3b96..42a98635bea5 100644

arch/x86/kvm/x86.c int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) · line 4331

Quoted 6 lines
arch/x86/kvm/x86.c +3 -0
@@ -4326,6 +4326,9 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
  		if (r < sizeof(struct kvm_xsave))
  			r = sizeof(struct kvm_xsave);
  		break;
+	case KVM_CAP_PMU_CONFIG:
+		r = enable_pmu ? KVM_PMU_CONFIG_VALID : 0;
+		break;
arch/x86/kvm/x86.c +1 -1
index 42a98635bea5..94481f92355d 100644
@@ -4326,10 +4326,10 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 		if (r < sizeof(struct kvm_xsave))
 			r = sizeof(struct kvm_xsave);
 		break;
+	}
 	case KVM_CAP_PMU_CONFIG:
 		r = enable_pmu ? KVM_CAP_PMU_MASK : 0;
 		break;
-	}
 	default:
 		break;
 	}
Quoted 70 lines
arch/x86/kvm/x86.c +9 -0
  	}
  	default:
  		break;
@@ -5937,6 +5940,14 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
  		kvm->arch.exit_on_emulation_error = cap->args[0];
  		r = 0;
  		break;
+	case KVM_CAP_PMU_CONFIG:
+		r = -EINVAL;
+		if (!enable_pmu || kvm->created_vcpus > 0 ||
+		    cap->args[0] & ~KVM_PMU_CONFIG_VALID)
+			break;
+		kvm->arch.enable_pmu = !(cap->args[0] & KVM_PMU_CONFIG_DISABLE);
+		r = 0;
+		break;
  	default:
  		r = -EINVAL;
  		break;
@@ -11562,6 +11573,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
  	raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
 
  	kvm->arch.guest_can_read_msr_platform_info = true;
+	kvm->arch.enable_pmu = enable_pmu;
 
  #if IS_ENABLED(CONFIG_HYPERV)
  	spin_lock_init(&kvm->arch.hv_root_tdp_lock);
include/uapi/linux/kvm.h +4 -0
index 9563d294f181..57a1280fa43b 100644
@@ -1133,6 +1133,7 @@ struct kvm_ppc_resize_hpt {
  #define KVM_CAP_VM_MOVE_ENC_CONTEXT_FROM 206
  #define KVM_CAP_VM_GPA_BITS 207
  #define KVM_CAP_XSAVE2 208
+#define KVM_CAP_PMU_CONFIG 209
 
  #ifdef KVM_CAP_IRQ_ROUTING
 
@@ -1972,6 +1973,9 @@ struct kvm_dirty_gfn {
  #define KVM_BUS_LOCK_DETECTION_OFF             (1 << 0)
  #define KVM_BUS_LOCK_DETECTION_EXIT            (1 << 1)
 
+#define KVM_PMU_CONFIG_DISABLE                 (1 << 0)
+#define KVM_PMU_CONFIG_VALID                   (KVM_PMU_CONFIG_DISABLE)
+
  /**
   * struct kvm_stats_header - Header of per vm/vcpu binary statistics data.
   * @flags: Some extra information for header, always 0 for now.
tools/include/uapi/linux/kvm.h +4 -0
index 9563d294f181..57a1280fa43b 100644
@@ -1133,6 +1133,7 @@ struct kvm_ppc_resize_hpt {
  #define KVM_CAP_VM_MOVE_ENC_CONTEXT_FROM 206
  #define KVM_CAP_VM_GPA_BITS 207
  #define KVM_CAP_XSAVE2 208
+#define KVM_CAP_PMU_CONFIG 209
 
  #ifdef KVM_CAP_IRQ_ROUTING
 
@@ -1972,6 +1973,9 @@ struct kvm_dirty_gfn {
  #define KVM_BUS_LOCK_DETECTION_OFF             (1 << 0)
  #define KVM_BUS_LOCK_DETECTION_EXIT            (1 << 1)
 
+#define KVM_PMU_CONFIG_DISABLE                 (1 << 0)
+#define KVM_PMU_CONFIG_VALID                   (KVM_PMU_CONFIG_DISABLE)
+
  /**
   * struct kvm_stats_header - Header of per vm/vcpu binary statistics data.
   * @flags: Some extra information for header, always 0 for now.
author
#3 lore raw

PATCH v2 2/3 selftests: introduce function to create test VM without vcpus.

In reply to David Dunn

Break out portion of vm_create_with_vcpus so that selftests can modify the VM prior to creating vcpus.

---
 .../selftests/kvm/include/kvm_util_base.h     |  3 ++
 tools/testing/selftests/kvm/lib/kvm_util.c    | 48 ++++++++++++++-----
 2 files changed, 38 insertions(+), 13 deletions(-)
tools/testing/selftests/kvm/include/kvm_util_base.h +3 -0
index 66775de26952..0454027d588d 100644
@@ -336,6 +336,9 @@ struct kvm_vm *vm_create_with_vcpus(enum vm_guest_mode mode, uint32_t nr_vcpus,
 				    uint32_t num_percpu_pages, void *guest_code,
 				    uint32_t vcpuids[]);
 
+/* First phase of vm_create_with_vcpus, allows customization before vcpu add */
+struct kvm_vm *vm_create_without_vcpus(enum vm_guest_mode mode, uint64_t pages);
+
 /*
  * Adds a vCPU with reasonable defaults (e.g. a stack)
  *
tools/testing/selftests/kvm/lib/kvm_util.c +35 -13

67 lines not on this page · raw message

Signature · 1 line
2.35.0.rc0.227.g00780c9af4-goog
author
#4 lore raw

PATCH v2 3/3 selftests: Verify disabling PMU via KVM_CAP_CONFIG_PMU

In reply to David Dunn

On a VM with PMU disabled via KVM_CAP_PMU_CONFIG, the PMU will not be usable by the guest. On Intel, this causes a #GP. And on AMD, the counters no longer increment.

KVM_CAP_PMU_CONFIG must be invoked on a VM prior to creating VCPUs.

---
 .../kvm/x86_64/pmu_event_filter_test.c        | 35 +++++++++++++++++++
 1 file changed, 35 insertions(+)
tools/testing/selftests/kvm/x86_64/pmu_event_filter_test.c +35 -0
index c715adcbd487..40fecaa4ea9f 100644
@@ -325,6 +325,39 @@ static void test_not_member_allow_list(struct kvm_vm *vm)
 	TEST_ASSERT(!count, "Disallowed PMU Event is counting");
 }
 
+/*
+ * Verify KVM_PMU_CONFIG_DISABLE prevents the use of the PMU.
+ *
+ * Note that KVM_CAP_PMU_CONFIG must be invoked prior to creating VCPUs.
+ */
+static void test_pmu_config_disable(void (*guest_code)(void))
+{
+	int r;
+	struct kvm_vm *vm;
+	struct kvm_enable_cap cap = { 0 };
+	bool sane;
+
+	r = kvm_check_cap(KVM_CAP_PMU_CONFIG);
+	if ((r & KVM_PMU_CONFIG_DISABLE) == 0)
+		return;
+
+	vm = vm_create_without_vcpus(VM_MODE_DEFAULT, DEFAULT_GUEST_PHY_PAGES);
+
+	cap.cap = KVM_CAP_PMU_CONFIG;
+	cap.args[0] = KVM_PMU_CONFIG_DISABLE;
+	r = vm_enable_cap(vm, &cap);
+	TEST_ASSERT(r == 0, "Failed KVM_PMU_CONFIG_DISABLE.");
+
+	vm_vcpu_add_default(vm, VCPU_ID, guest_code);
+	vm_init_descriptor_tables(vm);
+	vcpu_init_descriptor_tables(vm, VCPU_ID);
+
+	sane = sanity_check_pmu(vm);
+	TEST_ASSERT(!sane, "Guest should not see PMU when disabled.");
+
+	kvm_vm_free(vm);
+}
+
 /*
  * Check for a non-zero PMU version, at least one general-purpose
  * counter per logical processor, an EBX bit vector of length greater
@@ -430,5 +463,7 @@ int main(int argc, char *argv[])
 
 	kvm_vm_free(vm);
 
+	test_pmu_config_disable(guest_code);
+
 	return 0;
 }
Signature · 1 line
2.35.0.rc0.227.g00780c9af4-goog