Skip to contentSkip to conversation

KVM: x86/svm: Add module param to control PMU virtualization

PATCH Applied Like Xu 3 participants 15 messages Nov 17, 2021 – Jan 17, 2022

Subsystems (3)
lore
Share & tools
lore

Like Xu

author
#0 lore raw

PATCH 1/1 KVM: x86/svm: Add module param to control PMU virtualization

From: Like Xu <[email protected]>

For Intel, the guest PMU can be disabled via clearing the PMU CPUID. For AMD, all hw implementations support the base set of four performance counters, with current mainstream hardware indicating the presence of two additional counters via X86_FEATURE_PERFCTR_CORE.

In the virtualized world, the AMD guest driver may detect the presence of at least one counter MSR. Most hypervisor vendors would introduce a module param (like lbrv for svm) to disable PMU for all guests.

Another control proposal per-VM is to pass PMU disable information via MSR_IA32_PERF_CAPABILITIES or one bit in CPUID Fn4000_00[FF:00]. Both of methods require some guest-side changes, so a module parameter may not be sufficiently granular, but practical enough.

---
 arch/x86/kvm/cpuid.c   |  2 +-
 arch/x86/kvm/svm/pmu.c |  4 ++++
 arch/x86/kvm/svm/svm.c | 11 +++++++++++
 arch/x86/kvm/svm/svm.h |  1 +
 4 files changed, 17 insertions(+), 1 deletion(-)
arch/x86/kvm/cpuid.c +1 -1
index 2d70edb0f323..647af2a184ad 100644
@@ -487,7 +487,7 @@ void kvm_set_cpu_caps(void)
 		F(CR8_LEGACY) | F(ABM) | F(SSE4A) | F(MISALIGNSSE) |
 		F(3DNOWPREFETCH) | F(OSVW) | 0 /* IBS */ | F(XOP) |
 		0 /* SKINIT, WDT, LWP */ | F(FMA4) | F(TBM) |
-		F(TOPOEXT) | F(PERFCTR_CORE)
+		F(TOPOEXT) | 0 /* PERFCTR_CORE */
 	);
 
 	kvm_cpu_cap_mask(CPUID_8000_0001_EDX,
arch/x86/kvm/svm/pmu.c +4 -0
index fdf587f19c5f..a0bcf0144664 100644
@@ -16,6 +16,7 @@
 #include "cpuid.h"
 #include "lapic.h"
 #include "pmu.h"
+#include "svm.h"
 
 enum pmu_type {
 	PMU_TYPE_COUNTER = 0,
@@ -100,6 +101,9 @@ static inline struct kvm_pmc *get_gp_pmc_amd(struct kvm_pmu *pmu, u32 msr,
 {
 	struct kvm_vcpu *vcpu = pmu_to_vcpu(pmu);
 
+	if (!pmuv)
+		return NULL;
+
 	switch (msr) {
 	case MSR_F15H_PERF_CTL0:
 	case MSR_F15H_PERF_CTL1:
arch/x86/kvm/svm/svm.c +11 -0
index 21bb81710e0f..062e48c191ee 100644
@@ -190,6 +190,10 @@ module_param(vgif, int, 0444);
 static int lbrv = true;
 module_param(lbrv, int, 0444);
 
+/* enable/disable PMU virtualization */
+bool pmuv = true;
+module_param(pmuv, bool, 0444);
+
 static int tsc_scaling = true;
 module_param(tsc_scaling, int, 0444);
 
@@ -952,6 +956,10 @@ static __init void svm_set_cpu_caps(void)
 	    boot_cpu_has(X86_FEATURE_AMD_SSBD))
 		kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD);
 
+	/* AMD PMU PERFCTR_CORE CPUID */
+	if (pmuv && boot_cpu_has(X86_FEATURE_PERFCTR_CORE))
+		kvm_cpu_cap_set(X86_FEATURE_PERFCTR_CORE);
+
 	/* CPUID 0x8000001F (SME/SEV features) */
 	sev_set_cpu_caps();
 }
@@ -1085,6 +1093,9 @@ static __init int svm_hardware_setup(void)
 			pr_info("LBR virtualization supported\n");
 	}
 
+	if (!pmuv)
+		pr_info("PMU virtualization is disabled\n");
+
 	svm_set_cpu_caps();
 
 	/*
arch/x86/kvm/svm/svm.h +1 -0
index 0d7bbe548ac3..08e1c19ffbdf 100644
@@ -32,6 +32,7 @@
 extern u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly;
 extern bool npt_enabled;
 extern bool intercept_smi;
+extern bool pmuv;
 
 /*
  * Clean bits in VMCB.
Signature · 1 line
2.33.1
maintainer Applied
#1 lore raw
Quoted 102 lines · Like Xu

From: Like Xu <[email protected]>

For Intel, the guest PMU can be disabled via clearing the PMU CPUID. For AMD, all hw implementations support the base set of four performance counters, with current mainstream hardware indicating the presence of two additional counters via X86_FEATURE_PERFCTR_CORE.

In the virtualized world, the AMD guest driver may detect the presence of at least one counter MSR. Most hypervisor vendors would introduce a module param (like lbrv for svm) to disable PMU for all guests.

Another control proposal per-VM is to pass PMU disable information via MSR_IA32_PERF_CAPABILITIES or one bit in CPUID Fn4000_00[FF:00]. Both of methods require some guest-side changes, so a module parameter may not be sufficiently granular, but practical enough.

---
  arch/x86/kvm/cpuid.c   |  2 +-
  arch/x86/kvm/svm/pmu.c |  4 ++++
  arch/x86/kvm/svm/svm.c | 11 +++++++++++
  arch/x86/kvm/svm/svm.h |  1 +
  4 files changed, 17 insertions(+), 1 deletion(-)
arch/x86/kvm/cpuid.c +1 -1
index 2d70edb0f323..647af2a184ad 100644
@@ -487,7 +487,7 @@ void kvm_set_cpu_caps(void)
  		F(CR8_LEGACY) | F(ABM) | F(SSE4A) | F(MISALIGNSSE) |
  		F(3DNOWPREFETCH) | F(OSVW) | 0 /* IBS */ | F(XOP) |
  		0 /* SKINIT, WDT, LWP */ | F(FMA4) | F(TBM) |
-		F(TOPOEXT) | F(PERFCTR_CORE)
+		F(TOPOEXT) | 0 /* PERFCTR_CORE */
  	);
 
  	kvm_cpu_cap_mask(CPUID_8000_0001_EDX,
arch/x86/kvm/svm/pmu.c +4 -0
index fdf587f19c5f..a0bcf0144664 100644
@@ -16,6 +16,7 @@
  #include "cpuid.h"
  #include "lapic.h"
  #include "pmu.h"
+#include "svm.h"
 
  enum pmu_type {
  	PMU_TYPE_COUNTER = 0,
@@ -100,6 +101,9 @@ static inline struct kvm_pmc *get_gp_pmc_amd(struct kvm_pmu *pmu, u32 msr,
  {
  	struct kvm_vcpu *vcpu = pmu_to_vcpu(pmu);
 
+	if (!pmuv)
+		return NULL;
+
  	switch (msr) {
  	case MSR_F15H_PERF_CTL0:
  	case MSR_F15H_PERF_CTL1:
arch/x86/kvm/svm/svm.c +11 -0
index 21bb81710e0f..062e48c191ee 100644
@@ -190,6 +190,10 @@ module_param(vgif, int, 0444);
  static int lbrv = true;
  module_param(lbrv, int, 0444);
 
+/* enable/disable PMU virtualization */
+bool pmuv = true;
+module_param(pmuv, bool, 0444);
+
  static int tsc_scaling = true;
  module_param(tsc_scaling, int, 0444);
 
@@ -952,6 +956,10 @@ static __init void svm_set_cpu_caps(void)
  	    boot_cpu_has(X86_FEATURE_AMD_SSBD))
  		kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD);
 
+	/* AMD PMU PERFCTR_CORE CPUID */
+	if (pmuv && boot_cpu_has(X86_FEATURE_PERFCTR_CORE))
+		kvm_cpu_cap_set(X86_FEATURE_PERFCTR_CORE);
+
  	/* CPUID 0x8000001F (SME/SEV features) */
  	sev_set_cpu_caps();
  }
@@ -1085,6 +1093,9 @@ static __init int svm_hardware_setup(void)
  			pr_info("LBR virtualization supported\n");
  	}
 
+	if (!pmuv)
+		pr_info("PMU virtualization is disabled\n");
+
  	svm_set_cpu_caps();
 
  	/*
arch/x86/kvm/svm/svm.h +0 -0
index 0d7bbe548ac3..08e1c19ffbdf 100644

arch/x86/kvm/svm/svm.h line 38

Quoted 7 lines
arch/x86/kvm/svm/svm.h +1 -0
@@ -32,6 +32,7 @@
  extern u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly;
  extern bool npt_enabled;
  extern bool intercept_smi;
+extern bool pmuv;
 
  /*
   * Clean bits in VMCB.

Queued, thanks - just changed the parameter name to "pmu".

Paolo

#2 lore raw
Quoted 14 lines · Paolo Bonzini
Quoted 12 lines · Like Xu

From: Like Xu <[email protected]>

For Intel, the guest PMU can be disabled via clearing the PMU CPUID. For AMD, all hw implementations support the base set of four performance counters, with current mainstream hardware indicating the presence of two additional counters via X86_FEATURE_PERFCTR_CORE.

In the virtualized world, the AMD guest driver may detect the presence of at least one counter MSR. Most hypervisor vendors would introduce a module param (like lbrv for svm) to disable PMU for all guests.

Quoted 7 lines
Quoted 7 lines

Another control proposal per-VM is to pass PMU disable information via MSR_IA32_PERF_CAPABILITIES or one bit in CPUID Fn4000_00[FF:00]. Both of methods require some guest-side changes, so a module parameter may not be sufficiently granular, but practical enough.

---

Thanks for this patch. It saves us from upstreaming our equivalent patch.

In the long run, I'd like to be able to override this system-wide setting on a per-VM basis, for VMs that I trust. (Of course, this implies that I trust the userspace process as well.)

How would you feel if we were to add a kvm ioctl to override this setting, for a particular VM, guarded by an appropriate permissions check, like capable(CAP_SYS_ADMIN) or capable(CAP_SYS_MODULE)?

maintainer
#3 lore raw
Quoted 7 lines · Jim Mattson

In the long run, I'd like to be able to override this system-wide setting on a per-VM basis, for VMs that I trust. (Of course, this implies that I trust the userspace process as well.)

How would you feel if we were to add a kvm ioctl to override this setting, for a particular VM, guarded by an appropriate permissions check, like capable(CAP_SYS_ADMIN) or capable(CAP_SYS_MODULE)?

What's the rationale for guarding this with a capability check? IIRC you don't have such checks for perf_event_open (apart for getting kernel addresses, which is not a problem for virtualization).

Paolo

#4 lore raw
Quoted 6 lines · Paolo Bonzini
Quoted 4 lines · Jim Mattson

In the long run, I'd like to be able to override this system-wide setting on a per-VM basis, for VMs that I trust. (Of course, this implies that I trust the userspace process as well.)

Quoted 7 lines
Quoted 3 lines

How would you feel if we were to add a kvm ioctl to override this setting, for a particular VM, guarded by an appropriate permissions check, like capable(CAP_SYS_ADMIN) or capable(CAP_SYS_MODULE)?

What's the rationale for guarding this with a capability check? IIRC you don't have such checks for perf_event_open (apart for getting kernel addresses, which is not a problem for virtualization).

My reasoning was simply that for userspace to override a mode 0444 kernel module parameter, it should have the rights to reload the module with the parameter override. I wasn't thinking specifically about PMU capabilities.

#5 lore raw
Quoted 12 lines · Jim Mattson
Quoted 10 lines · Paolo Bonzini
Quoted 7 lines · Jim Mattson

In the long run, I'd like to be able to override this system-wide setting on a per-VM basis, for VMs that I trust. (Of course, this implies that I trust the userspace process as well.)

How would you feel if we were to add a kvm ioctl to override this setting, for a particular VM, guarded by an appropriate permissions check, like capable(CAP_SYS_ADMIN) or capable(CAP_SYS_MODULE)?

Quoted 8 lines
Quoted 3 lines

What's the rationale for guarding this with a capability check? IIRC you don't have such checks for perf_event_open (apart for getting kernel addresses, which is not a problem for virtualization).

My reasoning was simply that for userspace to override a mode 0444 kernel module parameter, it should have the rights to reload the module with the parameter override. I wasn't thinking specifically about PMU capabilities.

Assuming that we trust userspace to decide whether or not to expose a virtual PMU to a guest (as we do on the Intel side), perhaps we could make use of the existing PMU_EVENT_FILTER to give us per-VM control, rather than adding a new module parameter for per-host control. If userspace calls KVM_SET_PMU_EVENT_FILTER with an action of KVM_PMU_EVENT_ALLOW and an empty list of allowed events, KVM could just disable the virtual PMU for that VM.

Today, the semantics of an empty allow list are quite different from the proposed pmuv module parameter being false. However, it should be an easy conversion. Would anyone be concerned about changing the current semantics of an empty allow list? Is there a need for disabling PMU virtualization for legacy userspace implementations that can't be modified to ask for an empty allow list?

author
#6 lore raw
Quoted 13 lines · Jim Mattson
Quoted 12 lines · Jim Mattson
Quoted 10 lines · Paolo Bonzini
Quoted 7 lines · Jim Mattson

In the long run, I'd like to be able to override this system-wide setting on a per-VM basis, for VMs that I trust. (Of course, this implies that I trust the userspace process as well.)

How would you feel if we were to add a kvm ioctl to override this setting, for a particular VM, guarded by an appropriate permissions check, like capable(CAP_SYS_ADMIN) or capable(CAP_SYS_MODULE)?

Quoted 8 lines
Quoted 8 lines
Quoted 3 lines

What's the rationale for guarding this with a capability check? IIRC you don't have such checks for perf_event_open (apart for getting kernel addresses, which is not a problem for virtualization).

My reasoning was simply that for userspace to override a mode 0444 kernel module parameter, it should have the rights to reload the module with the parameter override. I wasn't thinking specifically about PMU capabilities.

Do we have a precedent on any module parameter rewriting for privileger ?

A further requirement is whether we can dynamically change this part of the behaviour when the guest is already booted up.

Quoted 5 lines

Assuming that we trust userspace to decide whether or not to expose a virtual PMU to a guest (as we do on the Intel side), perhaps we could make use of the existing PMU_EVENT_FILTER to give us per-VM control, rather than adding a new module parameter for per-host control. If

Various granularities of control are required to support vPMU production scenarios, including per-host, per-VM, and dynamic-guest-alive control.

Quoted 3 lines

userspace calls KVM_SET_PMU_EVENT_FILTER with an action of KVM_PMU_EVENT_ALLOW and an empty list of allowed events, KVM could just disable the virtual PMU for that VM.

AMD will also have "CPUID Fn8000_0022_EBX[NumCorePmc, 3:0]".

Quoted 8 lines

Today, the semantics of an empty allow list are quite different from the proposed pmuv module parameter being false. However, it should be an easy conversion. Would anyone be concerned about changing the current semantics of an empty allow list? Is there a need for disabling PMU virtualization for legacy userspace implementations that can't be modified to ask for an empty allow list?

AFAI, at least one user-space agent has integrated with it plus additional "action"s.

Once the API that the kernel presents to user space has been defined, it's best not to change it and instead fall into remorse.

"But I am not a decision maker. " :D

Thanks,
Like Xu

#7 lore raw
Quoted 35 lines · Like Xu
Quoted 13 lines · Jim Mattson
Quoted 12 lines · Jim Mattson
Quoted 10 lines · Paolo Bonzini
Quoted 7 lines · Jim Mattson

In the long run, I'd like to be able to override this system-wide setting on a per-VM basis, for VMs that I trust. (Of course, this implies that I trust the userspace process as well.)

How would you feel if we were to add a kvm ioctl to override this setting, for a particular VM, guarded by an appropriate permissions check, like capable(CAP_SYS_ADMIN) or capable(CAP_SYS_MODULE)?

Quoted 8 lines
Quoted 8 lines
Quoted 3 lines

What's the rationale for guarding this with a capability check? IIRC you don't have such checks for perf_event_open (apart for getting kernel addresses, which is not a problem for virtualization).

My reasoning was simply that for userspace to override a mode 0444 kernel module parameter, it should have the rights to reload the module with the parameter override. I wasn't thinking specifically about PMU capabilities.

Do we have a precedent on any module parameter rewriting for privileger ?

A further requirement is whether we can dynamically change this part of the behaviour when the guest is already booted up.

Quoted 5 lines

Assuming that we trust userspace to decide whether or not to expose a virtual PMU to a guest (as we do on the Intel side), perhaps we could make use of the existing PMU_EVENT_FILTER to give us per-VM control, rather than adding a new module parameter for per-host control. If

Quoted 8 lines

Various granularities of control are required to support vPMU production scenarios, including per-host, per-VM, and dynamic-guest-alive control.

Quoted 3 lines

userspace calls KVM_SET_PMU_EVENT_FILTER with an action of KVM_PMU_EVENT_ALLOW and an empty list of allowed events, KVM could just disable the virtual PMU for that VM.

AMD will also have "CPUID Fn8000_0022_EBX[NumCorePmc, 3:0]".

Where do you see this? Revision 3.33 (November 2021) of the AMD APM, volume 3, only goes as high as CPUID Fn8000_0021.

Quoted 9 lines
Quoted 8 lines

Today, the semantics of an empty allow list are quite different from the proposed pmuv module parameter being false. However, it should be an easy conversion. Would anyone be concerned about changing the current semantics of an empty allow list? Is there a need for disabling PMU virtualization for legacy userspace implementations that can't be modified to ask for an empty allow list?

Quoted 5 lines

AFAI, at least one user-space agent has integrated with it plus additional "action"s.

Once the API that the kernel presents to user space has been defined, it's best not to change it and instead fall into remorse.

Okay.

I propose the following:
1) The new module parameter should apply to Intel as well as AMD, for situations where userspace is not trusted.
2) If the module parameter allows PMU virtualization, there should be a new KVM_CAP whereby userspace can enable/disable PMU virtualization. (Since you require a dynamic toggle, and there is a move afoot to refuse guest CPUID changes once a guest is running, this new KVM_CAP is needed on Intel as well as AMD).
3) If the module parameter does not allow PMU virtualization, there should be no userspace override, since we have no precedent for authorizing that kind of override.

Quoted 5 lines

"But I am not a decision maker. " :D

Thanks,
Like Xu

author
#8 lore raw
Quoted 39 lines · Jim Mattson
Quoted 38 lines · Like Xu
Quoted 13 lines · Jim Mattson
Quoted 12 lines · Jim Mattson
Quoted 10 lines · Paolo Bonzini
Quoted 7 lines · Jim Mattson

In the long run, I'd like to be able to override this system-wide setting on a per-VM basis, for VMs that I trust. (Of course, this implies that I trust the userspace process as well.)

How would you feel if we were to add a kvm ioctl to override this setting, for a particular VM, guarded by an appropriate permissions check, like capable(CAP_SYS_ADMIN) or capable(CAP_SYS_MODULE)?

Quoted 8 lines
Quoted 8 lines
Quoted 3 lines

What's the rationale for guarding this with a capability check? IIRC you don't have such checks for perf_event_open (apart for getting kernel addresses, which is not a problem for virtualization).

My reasoning was simply that for userspace to override a mode 0444 kernel module parameter, it should have the rights to reload the module with the parameter override. I wasn't thinking specifically about PMU capabilities.

Do we have a precedent on any module parameter rewriting for privileger ?

A further requirement is whether we can dynamically change this part of the behaviour when the guest is already booted up.

Quoted 5 lines

Assuming that we trust userspace to decide whether or not to expose a virtual PMU to a guest (as we do on the Intel side), perhaps we could make use of the existing PMU_EVENT_FILTER to give us per-VM control, rather than adding a new module parameter for per-host control. If

Various granularities of control are required to support vPMU production scenarios, including per-host, per-VM, and dynamic-guest-alive control.

Quoted 8 lines
Quoted 5 lines
Quoted 3 lines

userspace calls KVM_SET_PMU_EVENT_FILTER with an action of KVM_PMU_EVENT_ALLOW and an empty list of allowed events, KVM could just disable the virtual PMU for that VM.

AMD will also have "CPUID Fn8000_0022_EBX[NumCorePmc, 3:0]".

Where do you see this? Revision 3.33 (November 2021) of the AMD APM, volume 3, only goes as high as CPUID Fn8000_0021.

Try APM Revision: 4.04 (November 2021), page 1849/3273, "CPUID Fn8000_0022_EBX Extended Performance Monitoring and Debug".

Given the current ambiguity in this revision, the AMD folks will reveal more details bout this field in the next revision.

Quoted 18 lines
Quoted 10 lines
Quoted 9 lines

Today, the semantics of an empty allow list are quite different from the proposed pmuv module parameter being false. However, it should be an easy conversion. Would anyone be concerned about changing the current semantics of an empty allow list? Is there a need for disabling PMU virtualization for legacy userspace implementations that can't be modified to ask for an empty allow list?

Quoted 5 lines

AFAI, at least one user-space agent has integrated with it plus additional "action"s.

Once the API that the kernel presents to user space has been defined, it's best not to change it and instead fall into remorse.

Okay.

Quoted 8 lines

I propose the following:
1) The new module parameter should apply to Intel as well as AMD, for situations where userspace is not trusted.
2) If the module parameter allows PMU virtualization, there should be a new KVM_CAP whereby userspace can enable/disable PMU virtualization. (Since you require a dynamic toggle, and there is a move afoot to refuse guest CPUID changes once a guest is running, this new KVM_CAP is needed on Intel as well as AMD).

Both hands in favour. Do you need me as a labourer, or you have a ready-made one ?

Quoted 3 lines

3) If the module parameter does not allow PMU virtualization, there should be no userspace override, since we have no precedent for authorizing that kind of override.

Uh, I thought you (Google) had a lot of these (interesting) use cases internally.

Quoted 6 lines
Quoted 6 lines

"But I am not a decision maker. " :D

Thanks,
Like Xu

#9 lore raw
Quoted 45 lines · Like Xu
Quoted 43 lines · Jim Mattson
Quoted 42 lines · Like Xu
Quoted 13 lines · Jim Mattson
Quoted 12 lines · Jim Mattson
Quoted 10 lines · Paolo Bonzini
Quoted 7 lines · Jim Mattson

In the long run, I'd like to be able to override this system-wide setting on a per-VM basis, for VMs that I trust. (Of course, this implies that I trust the userspace process as well.)

How would you feel if we were to add a kvm ioctl to override this setting, for a particular VM, guarded by an appropriate permissions check, like capable(CAP_SYS_ADMIN) or capable(CAP_SYS_MODULE)?

Quoted 8 lines
Quoted 8 lines
Quoted 3 lines

What's the rationale for guarding this with a capability check? IIRC you don't have such checks for perf_event_open (apart for getting kernel addresses, which is not a problem for virtualization).

My reasoning was simply that for userspace to override a mode 0444 kernel module parameter, it should have the rights to reload the module with the parameter override. I wasn't thinking specifically about PMU capabilities.

Do we have a precedent on any module parameter rewriting for privileger ?

A further requirement is whether we can dynamically change this part of the behaviour when the guest is already booted up.

Quoted 5 lines

Assuming that we trust userspace to decide whether or not to expose a virtual PMU to a guest (as we do on the Intel side), perhaps we could make use of the existing PMU_EVENT_FILTER to give us per-VM control, rather than adding a new module parameter for per-host control. If

Various granularities of control are required to support vPMU production scenarios, including per-host, per-VM, and dynamic-guest-alive control.

Quoted 3 lines

userspace calls KVM_SET_PMU_EVENT_FILTER with an action of KVM_PMU_EVENT_ALLOW and an empty list of allowed events, KVM could just disable the virtual PMU for that VM.

Quoted 7 lines
Quoted 4 lines
Quoted 1 line

AMD will also have "CPUID Fn8000_0022_EBX[NumCorePmc, 3:0]".

Where do you see this? Revision 3.33 (November 2021) of the AMD APM, volume 3, only goes as high as CPUID Fn8000_0021.

Try APM Revision: 4.04 (November 2021), page 1849/3273, "CPUID Fn8000_0022_EBX Extended Performance Monitoring and Debug".

Is this a public document?

Quoted 30 lines

Given the current ambiguity in this revision, the AMD folks will reveal more details bout this field in the next revision.

Quoted 18 lines
Quoted 10 lines
Quoted 9 lines

Today, the semantics of an empty allow list are quite different from the proposed pmuv module parameter being false. However, it should be an easy conversion. Would anyone be concerned about changing the current semantics of an empty allow list? Is there a need for disabling PMU virtualization for legacy userspace implementations that can't be modified to ask for an empty allow list?

Quoted 5 lines

AFAI, at least one user-space agent has integrated with it plus additional "action"s.

Once the API that the kernel presents to user space has been defined, it's best not to change it and instead fall into remorse.

Okay.

Quoted 8 lines

I propose the following:
1) The new module parameter should apply to Intel as well as AMD, for situations where userspace is not trusted.
2) If the module parameter allows PMU virtualization, there should be a new KVM_CAP whereby userspace can enable/disable PMU virtualization. (Since you require a dynamic toggle, and there is a move afoot to refuse guest CPUID changes once a guest is running, this new KVM_CAP is needed on Intel as well as AMD).

Quoted 1 line

Both hands in favour. Do you need me as a labourer, or you have a ready-made one ?

We could split the work. Since (1) is a modification of the change you proposed in this thread, perhaps you could apply it to both AMD and Intel in v2? We can find someone for (2).

Quoted 5 lines
Quoted 3 lines

3) If the module parameter does not allow PMU virtualization, there should be no userspace override, since we have no precedent for authorizing that kind of override.

Uh, I thought you (Google) had a lot of these (interesting) use cases internally.

We have modified some module parameters so that they can be changed at runtime, but we don't have any concept of a privileged userspace overriding a module parameter restriction.

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

"But I am not a decision maker. " :D

Thanks,
Like Xu

author
#10 lore raw
Quoted 48 lines · Jim Mattson
Quoted 47 lines · Like Xu
Quoted 45 lines · Jim Mattson
Quoted 43 lines · Like Xu
Quoted 13 lines · Jim Mattson
Quoted 12 lines · Jim Mattson
Quoted 10 lines · Paolo Bonzini
Quoted 7 lines · Jim Mattson

In the long run, I'd like to be able to override this system-wide setting on a per-VM basis, for VMs that I trust. (Of course, this implies that I trust the userspace process as well.)

How would you feel if we were to add a kvm ioctl to override this setting, for a particular VM, guarded by an appropriate permissions check, like capable(CAP_SYS_ADMIN) or capable(CAP_SYS_MODULE)?

Quoted 8 lines
Quoted 8 lines
Quoted 3 lines

What's the rationale for guarding this with a capability check? IIRC you don't have such checks for perf_event_open (apart for getting kernel addresses, which is not a problem for virtualization).

My reasoning was simply that for userspace to override a mode 0444 kernel module parameter, it should have the rights to reload the module with the parameter override. I wasn't thinking specifically about PMU capabilities.

Do we have a precedent on any module parameter rewriting for privileger ?

A further requirement is whether we can dynamically change this part of the behaviour when the guest is already booted up.

Quoted 5 lines

Assuming that we trust userspace to decide whether or not to expose a virtual PMU to a guest (as we do on the Intel side), perhaps we could make use of the existing PMU_EVENT_FILTER to give us per-VM control, rather than adding a new module parameter for per-host control. If

Various granularities of control are required to support vPMU production scenarios, including per-host, per-VM, and dynamic-guest-alive control.

Quoted 3 lines

userspace calls KVM_SET_PMU_EVENT_FILTER with an action of KVM_PMU_EVENT_ALLOW and an empty list of allowed events, KVM could just disable the virtual PMU for that VM.

AMD will also have "CPUID Fn8000_0022_EBX[NumCorePmc, 3:0]".

Quoted 7 lines
Quoted 5 lines
Quoted 2 lines

Where do you see this? Revision 3.33 (November 2021) of the AMD APM, volume 3, only goes as high as CPUID Fn8000_0021.

Try APM Revision: 4.04 (November 2021), page 1849/3273, "CPUID Fn8000_0022_EBX Extended Performance Monitoring and Debug".

Is this a public document?

The latest version of APM (40332) is revision v4.04, released on 12/1/2021.

Quoted 31 lines
Quoted 31 lines

Given the current ambiguity in this revision, the AMD folks will reveal more details bout this field in the next revision.

Quoted 26 lines
Quoted 10 lines
Quoted 9 lines

Today, the semantics of an empty allow list are quite different from the proposed pmuv module parameter being false. However, it should be an easy conversion. Would anyone be concerned about changing the current semantics of an empty allow list? Is there a need for disabling PMU virtualization for legacy userspace implementations that can't be modified to ask for an empty allow list?

Quoted 5 lines

AFAI, at least one user-space agent has integrated with it plus additional "action"s.

Once the API that the kernel presents to user space has been defined, it's best not to change it and instead fall into remorse.

Okay.

I propose the following:
1) The new module parameter should apply to Intel as well as AMD, for situations where userspace is not trusted.
2) If the module parameter allows PMU virtualization, there should be a new KVM_CAP whereby userspace can enable/disable PMU virtualization. (Since you require a dynamic toggle, and there is a move afoot to refuse guest CPUID changes once a guest is running, this new KVM_CAP is needed on Intel as well as AMD).

Quoted 4 lines
Quoted 1 line

Both hands in favour. Do you need me as a labourer, or you have a ready-made one ?

We could split the work. Since (1) is a modification of the change you proposed in this thread, perhaps you could apply it to both AMD and

We obviously need extra code to make the module parameters suitable for Intel since it affects other features (such as LBR and PEBS), we may not rush to draw this line clearly.

Quoted 1 line

Intel in v2? We can find someone for (2).

The ioctl_set_pmu_event_filter() interface is already practical for dynamic toggle, as not being able to program any events is the same as having none vPMU, w/o considering performance impact of traversing the list.

I am not sure if the maintainer will buy in another KVM_CAP for only per-VM, considering "CPUID Fn8000_0022_EBX[NumCorePmc, 3:0]" is a feature that will be available soon.

Quoted 10 lines
Quoted 6 lines
Quoted 4 lines

3) If the module parameter does not allow PMU virtualization, there should be no userspace override, since we have no precedent for authorizing that kind of override.

Uh, I thought you (Google) had a lot of these (interesting) use cases internally.

We have modified some module parameters so that they can be changed at runtime, but we don't have any concept of a privileged userspace overriding a module parameter restriction.

Considering that the semantics of the different module parameters are different, allowing one of them to be overridden does not mean that such a generic framework is promising, but it makes sense for the community to see another case for it.

Quoted 8 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

"But I am not a decision maker. " :D

Thanks,
Like Xu

#11 lore raw
Quoted 53 lines · Like Xu
Quoted 51 lines · Jim Mattson
Quoted 50 lines · Like Xu
Quoted 47 lines · Jim Mattson
Quoted 35 lines · Like Xu
Quoted 13 lines · Jim Mattson
Quoted 12 lines · Jim Mattson
Quoted 10 lines · Paolo Bonzini
Quoted 7 lines · Jim Mattson

In the long run, I'd like to be able to override this system-wide setting on a per-VM basis, for VMs that I trust. (Of course, this implies that I trust the userspace process as well.)

How would you feel if we were to add a kvm ioctl to override this setting, for a particular VM, guarded by an appropriate permissions check, like capable(CAP_SYS_ADMIN) or capable(CAP_SYS_MODULE)?

Quoted 8 lines
Quoted 8 lines
Quoted 3 lines

What's the rationale for guarding this with a capability check? IIRC you don't have such checks for perf_event_open (apart for getting kernel addresses, which is not a problem for virtualization).

My reasoning was simply that for userspace to override a mode 0444 kernel module parameter, it should have the rights to reload the module with the parameter override. I wasn't thinking specifically about PMU capabilities.

Do we have a precedent on any module parameter rewriting for privileger ?

A further requirement is whether we can dynamically change this part of the behaviour when the guest is already booted up.

Quoted 5 lines

Assuming that we trust userspace to decide whether or not to expose a virtual PMU to a guest (as we do on the Intel side), perhaps we could make use of the existing PMU_EVENT_FILTER to give us per-VM control, rather than adding a new module parameter for per-host control. If

Quoted 8 lines

Various granularities of control are required to support vPMU production scenarios, including per-host, per-VM, and dynamic-guest-alive control.

Quoted 3 lines

userspace calls KVM_SET_PMU_EVENT_FILTER with an action of KVM_PMU_EVENT_ALLOW and an empty list of allowed events, KVM could just disable the virtual PMU for that VM.

AMD will also have "CPUID Fn8000_0022_EBX[NumCorePmc, 3:0]".

Where do you see this? Revision 3.33 (November 2021) of the AMD APM, volume 3, only goes as high as CPUID Fn8000_0021.

Quoted 6 lines
Quoted 4 lines
Quoted 2 lines

Try APM Revision: 4.04 (November 2021), page 1849/3273, "CPUID Fn8000_0022_EBX Extended Performance Monitoring and Debug".

Is this a public document?

The latest version of APM (40332) is revision v4.04, released on 12/1/2021.

LOL. I was misled by the table of contents for Appendix E.4, which stops at E.4.19 Function 8000_0021—Extended Feature Identification 2.

Quoted 41 lines
Quoted 31 lines
Quoted 31 lines

Given the current ambiguity in this revision, the AMD folks will reveal more details bout this field in the next revision.

Quoted 26 lines
Quoted 10 lines
Quoted 9 lines

Today, the semantics of an empty allow list are quite different from the proposed pmuv module parameter being false. However, it should be an easy conversion. Would anyone be concerned about changing the current semantics of an empty allow list? Is there a need for disabling PMU virtualization for legacy userspace implementations that can't be modified to ask for an empty allow list?

Quoted 5 lines

AFAI, at least one user-space agent has integrated with it plus additional "action"s.

Once the API that the kernel presents to user space has been defined, it's best not to change it and instead fall into remorse.

Okay.

I propose the following:
1) The new module parameter should apply to Intel as well as AMD, for situations where userspace is not trusted.
2) If the module parameter allows PMU virtualization, there should be a new KVM_CAP whereby userspace can enable/disable PMU virtualization. (Since you require a dynamic toggle, and there is a move afoot to refuse guest CPUID changes once a guest is running, this new KVM_CAP is needed on Intel as well as AMD).

Quoted 4 lines
Quoted 1 line

Both hands in favour. Do you need me as a labourer, or you have a ready-made one ?

We could split the work. Since (1) is a modification of the change you proposed in this thread, perhaps you could apply it to both AMD and

We obviously need extra code to make the module parameters suitable for Intel since it
affects other features (such as LBR and PEBS), we may not rush to draw this line clearly.

Quoted 5 lines
Quoted 1 line

Intel in v2? We can find someone for (2).

The ioctl_set_pmu_event_filter() interface is already practical for dynamic toggle, as not being able to program any events is the same as having none vPMU, w/o considering performance impact of traversing the list.

Not being able to program any events is actually not the same as setting the pmuv module parameter to false. In the former case, prohibited events simply don't advance the counters. In the latter case, all accesses to the PMU MSRs raise #GP. That was what I meant earlier, when I said that we would have to change the semantics of an empty allow list if we want to match the behavior of pmuv=0.

Quoted 3 lines

I am not sure if the maintainer will buy in another KVM_CAP for only per-VM, considering
"CPUID Fn8000_0022_EBX[NumCorePmc, 3:0]" is a feature that will be available soon.

For existing guests, the future availability of that feature is irrelevant. We're going to need another solution that doesn't involve recompiling every guest kernel.

Quoted 23 lines
Quoted 10 lines
Quoted 6 lines
Quoted 4 lines

3) If the module parameter does not allow PMU virtualization, there should be no userspace override, since we have no precedent for authorizing that kind of override.

Uh, I thought you (Google) had a lot of these (interesting) use cases internally.

We have modified some module parameters so that they can be changed at runtime, but we don't have any concept of a privileged userspace overriding a module parameter restriction.

Considering that the semantics of the different module parameters are different, allowing one of them to be overridden does not mean that such a generic framework is promising, but it makes sense for the community to see another case for it.

Quoted 8 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

"But I am not a decision maker. " :D

Thanks,
Like Xu

#12 lore raw
In reply to Paolo Bonzini
Quoted 108 lines · Paolo Bonzini
Quoted 106 lines · Like Xu

From: Like Xu <[email protected]>

For Intel, the guest PMU can be disabled via clearing the PMU CPUID. For AMD, all hw implementations support the base set of four performance counters, with current mainstream hardware indicating the presence of two additional counters via X86_FEATURE_PERFCTR_CORE.

In the virtualized world, the AMD guest driver may detect the presence of at least one counter MSR. Most hypervisor vendors would introduce a module param (like lbrv for svm) to disable PMU for all guests.

Another control proposal per-VM is to pass PMU disable information via MSR_IA32_PERF_CAPABILITIES or one bit in CPUID Fn4000_00[FF:00]. Both of methods require some guest-side changes, so a module parameter may not be sufficiently granular, but practical enough.

---
  arch/x86/kvm/cpuid.c   |  2 +-
  arch/x86/kvm/svm/pmu.c |  4 ++++
  arch/x86/kvm/svm/svm.c | 11 +++++++++++
  arch/x86/kvm/svm/svm.h |  1 +
  4 files changed, 17 insertions(+), 1 deletion(-)
arch/x86/kvm/cpuid.c +1 -1
index 2d70edb0f323..647af2a184ad 100644
@@ -487,7 +487,7 @@ void kvm_set_cpu_caps(void)
              F(CR8_LEGACY) | F(ABM) | F(SSE4A) | F(MISALIGNSSE) |
              F(3DNOWPREFETCH) | F(OSVW) | 0 /* IBS */ | F(XOP) |
              0 /* SKINIT, WDT, LWP */ | F(FMA4) | F(TBM) |
-             F(TOPOEXT) | F(PERFCTR_CORE)
+             F(TOPOEXT) | 0 /* PERFCTR_CORE */
      );
 
      kvm_cpu_cap_mask(CPUID_8000_0001_EDX,
arch/x86/kvm/svm/pmu.c +4 -0
index fdf587f19c5f..a0bcf0144664 100644
@@ -16,6 +16,7 @@
  #include "cpuid.h"
  #include "lapic.h"
  #include "pmu.h"
+#include "svm.h"
 
  enum pmu_type {
      PMU_TYPE_COUNTER = 0,
@@ -100,6 +101,9 @@ static inline struct kvm_pmc *get_gp_pmc_amd(struct kvm_pmu *pmu, u32 msr,
  {
      struct kvm_vcpu *vcpu = pmu_to_vcpu(pmu);
 
+     if (!pmuv)
+             return NULL;
+
      switch (msr) {
      case MSR_F15H_PERF_CTL0:
      case MSR_F15H_PERF_CTL1:
arch/x86/kvm/svm/svm.c +11 -0
index 21bb81710e0f..062e48c191ee 100644
@@ -190,6 +190,10 @@ module_param(vgif, int, 0444);
  static int lbrv = true;
  module_param(lbrv, int, 0444);
 
+/* enable/disable PMU virtualization */
+bool pmuv = true;
+module_param(pmuv, bool, 0444);
+
  static int tsc_scaling = true;
  module_param(tsc_scaling, int, 0444);
 
@@ -952,6 +956,10 @@ static __init void svm_set_cpu_caps(void)
          boot_cpu_has(X86_FEATURE_AMD_SSBD))
              kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD);
 
+     /* AMD PMU PERFCTR_CORE CPUID */
+     if (pmuv && boot_cpu_has(X86_FEATURE_PERFCTR_CORE))
+             kvm_cpu_cap_set(X86_FEATURE_PERFCTR_CORE);
+
      /* CPUID 0x8000001F (SME/SEV features) */
      sev_set_cpu_caps();
  }
@@ -1085,6 +1093,9 @@ static __init int svm_hardware_setup(void)
                      pr_info("LBR virtualization supported\n");
      }
 
+     if (!pmuv)
+             pr_info("PMU virtualization is disabled\n");
+
      svm_set_cpu_caps();
 
      /*
arch/x86/kvm/svm/svm.h +1 -0
index 0d7bbe548ac3..08e1c19ffbdf 100644
@@ -32,6 +32,7 @@
  extern u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly;
  extern bool npt_enabled;
  extern bool intercept_smi;
+extern bool pmuv;
 
Quoted 5 lines

arch/x86/kvm/svm/svm.h line 38

Quoted 3 lines
arch/x86/kvm/svm/svm.h +0 -0
  /*
   * Clean bits in VMCB.

Queued, thanks - just changed the parameter name to "pmu".

Whoops! The global 'pmu' is hidden by a local 'pmu' in get_gp_pmc_amd().

author
#13 lore raw
Quoted 109 lines · Jim Mattson
Quoted 108 lines · Paolo Bonzini
Quoted 106 lines · Like Xu

From: Like Xu <[email protected]>

For Intel, the guest PMU can be disabled via clearing the PMU CPUID. For AMD, all hw implementations support the base set of four performance counters, with current mainstream hardware indicating the presence of two additional counters via X86_FEATURE_PERFCTR_CORE.

In the virtualized world, the AMD guest driver may detect the presence of at least one counter MSR. Most hypervisor vendors would introduce a module param (like lbrv for svm) to disable PMU for all guests.

Another control proposal per-VM is to pass PMU disable information via MSR_IA32_PERF_CAPABILITIES or one bit in CPUID Fn4000_00[FF:00]. Both of methods require some guest-side changes, so a module parameter may not be sufficiently granular, but practical enough.

---
   arch/x86/kvm/cpuid.c   |  2 +-
   arch/x86/kvm/svm/pmu.c |  4 ++++
   arch/x86/kvm/svm/svm.c | 11 +++++++++++
   arch/x86/kvm/svm/svm.h |  1 +
   4 files changed, 17 insertions(+), 1 deletion(-)
arch/x86/kvm/cpuid.c +1 -1
index 2d70edb0f323..647af2a184ad 100644
@@ -487,7 +487,7 @@ void kvm_set_cpu_caps(void)
               F(CR8_LEGACY) | F(ABM) | F(SSE4A) | F(MISALIGNSSE) |
               F(3DNOWPREFETCH) | F(OSVW) | 0 /* IBS */ | F(XOP) |
               0 /* SKINIT, WDT, LWP */ | F(FMA4) | F(TBM) |
-             F(TOPOEXT) | F(PERFCTR_CORE)
+             F(TOPOEXT) | 0 /* PERFCTR_CORE */
       );
 
       kvm_cpu_cap_mask(CPUID_8000_0001_EDX,
arch/x86/kvm/svm/pmu.c +4 -0
index fdf587f19c5f..a0bcf0144664 100644
@@ -16,6 +16,7 @@
   #include "cpuid.h"
   #include "lapic.h"
   #include "pmu.h"
+#include "svm.h"
 
   enum pmu_type {
       PMU_TYPE_COUNTER = 0,
@@ -100,6 +101,9 @@ static inline struct kvm_pmc *get_gp_pmc_amd(struct kvm_pmu *pmu, u32 msr,
   {
       struct kvm_vcpu *vcpu = pmu_to_vcpu(pmu);
 
+     if (!pmuv)
+             return NULL;
+
       switch (msr) {
       case MSR_F15H_PERF_CTL0:
       case MSR_F15H_PERF_CTL1:
arch/x86/kvm/svm/svm.c +11 -0
index 21bb81710e0f..062e48c191ee 100644
@@ -190,6 +190,10 @@ module_param(vgif, int, 0444);
   static int lbrv = true;
   module_param(lbrv, int, 0444);
 
+/* enable/disable PMU virtualization */
+bool pmuv = true;
+module_param(pmuv, bool, 0444);
+
   static int tsc_scaling = true;
   module_param(tsc_scaling, int, 0444);
 
@@ -952,6 +956,10 @@ static __init void svm_set_cpu_caps(void)
           boot_cpu_has(X86_FEATURE_AMD_SSBD))
               kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD);
 
+     /* AMD PMU PERFCTR_CORE CPUID */
+     if (pmuv && boot_cpu_has(X86_FEATURE_PERFCTR_CORE))
+             kvm_cpu_cap_set(X86_FEATURE_PERFCTR_CORE);
+
       /* CPUID 0x8000001F (SME/SEV features) */
       sev_set_cpu_caps();
   }
@@ -1085,6 +1093,9 @@ static __init int svm_hardware_setup(void)
                       pr_info("LBR virtualization supported\n");
       }
 
+     if (!pmuv)
+             pr_info("PMU virtualization is disabled\n");
+
       svm_set_cpu_caps();
 
       /*
arch/x86/kvm/svm/svm.h +1 -0
index 0d7bbe548ac3..08e1c19ffbdf 100644
@@ -32,6 +32,7 @@
   extern u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly;
   extern bool npt_enabled;
   extern bool intercept_smi;
+extern bool pmuv;
 
Quoted 7 lines
Quoted 5 lines

arch/x86/kvm/svm/svm.h line 38

Quoted 3 lines
arch/x86/kvm/svm/svm.h +0 -0
   /*
    * Clean bits in VMCB.

Queued, thanks - just changed the parameter name to "pmu".

Whoops! The global 'pmu' is hidden by a local 'pmu' in get_gp_pmc_amd().

Indeed, I wonder if Poalo would like to take a look at this fix:
https://lore.kernel.org/kvm/[email protected]/