Skip to contentSkip to conversation

KVM: arm64: Expose PMMIR_EL1.SLOTS to guests

PATCH Superseded Congkai Tan 3 participants 7 messages Jul 2 – Jul 10

Subsystems (5)
Documentation KVM Kselftest arm64 +1 more subsystems

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

v2 cover KVM: arm64: Expose PMMIR_EL1.SLOTS to guests

Today when the perf tool runs in a guest on cores with PMUv3p4, it fails to parse the default metrics with "Failure to read '#slots'", since perf can only read 0 from sysfs caps/slots, which is backed by PMMIR_EL1.SLOTS that KVM traps as RAZ/WI.

Taking into account backward compatibility and heterogeneous systems, the exposure of PMMIR_EL1.SLOTS is gated behind a new vCPU feature flag:

- Patch 1 adds the new flag KVM_ARM_VCPU_PMU_V3_STRICT. When set, KVM does not create a default PMU during vCPU init, and the VMM must select one explicitly via KVM_ARM_VCPU_PMU_V3_SET_PMU before the first KVM_RUN.
- Patch 2 exposes PMMIR_EL1.SLOTS of the selected PMU under the flag, and adds userspace get/set for PMMIR_EL1 so that SLOTS can be reset to 0 for backward compatibility.
- Patch 3 stops masking STALL_SLOT* in PMCEID1 under the flag.

When the flag is not set, behaviors are unchanged.

v1: https://lore.kernel.org/r/[email protected]

v1 -> v2 changes:
- Gate the whole feature behind a new KVM_ARM_VCPU_PMU_V3_STRICT vCPU feature flag, instead of unconditionally exposing PMMIR_EL1.SLOTS.
- When the flag is set, skip creating a default PMU during vCPU init.
- Split the PMCEID1 unmask into its own patch, also gated by the flag.
- Snapshot SLOTS into a new field pmmir_slots in kvm_arch during the handling of KVM_ARM_VCPU_PMU_V3_SET_PMU when the flag is set; access_pmmir()/get_pmmir() return it and set_pmmir() only accepts the SLOTS field (rejecting other bits with -EINVAL).
- Add get_user and set_user for PMMIR_EL1 to support setting the SLOTS back to 0, and add PMMIR_EL1 to the get-reg-list selftest.

Congkai Tan (3):
KVM: arm64: Add KVM_ARM_VCPU_PMU_V3_STRICT vCPU feature
KVM: arm64: Expose PMMIR_EL1.SLOTS under strict PMUv3 UAPI
KVM: arm64: Advertise STALL_SLOT* in PMCEID1 under strict PMUv3 UAPI

 Documentation/virt/kvm/api.rst                   |  5 ++
 arch/arm64/include/asm/kvm_host.h                |  5 +-
 arch/arm64/include/uapi/asm/kvm.h                |  1 +
 arch/arm64/kvm/arm.c                             | 18 +++++--
 arch/arm64/kvm/pmu-emul.c                        | 50 ++++++++++++++-----
 arch/arm64/kvm/sys_regs.c                        | 63 +++++++++++++++++++++++-
 include/kvm/arm_pmu.h                            |  4 ++
 tools/arch/arm64/include/uapi/asm/kvm.h          |  1 +
 tools/testing/selftests/kvm/arm64/get-reg-list.c |  1 +
 9 files changed, 128 insertions(+), 20 deletions(-)

base-commit: 1702da76e017ae0fbe1a92b07bc332972c293e89 --
2.50.1

#1 lore raw

PATCH v2 1/3 KVM: arm64: Add KVM_ARM_VCPU_PMU_V3_STRICT vCPU feature

Introduce a new vCPU feature KVM_ARM_VCPU_PMU_V3_STRICT. When set, KVM does not create a default PMU when initializing the vCPU, and userspace must select one explicitly via KVM_ARM_VCPU_PMU_V3_SET_PMU before the first KVM_RUN.

The flag forces the VMM to be aware of the PMU implementation of the guest to be created, so that certain information about the PMU becomes deterministic (if on a heterogeneous system) and becomes safe to be exposed to the guest. It can be used as an umbrella flag to gate future PMUv3 UAPI changes.

When no default PMU is created, kvm->arch.arm_pmu stays NULL until SET_PMU runs, so kvm_arm_pmu_v3_enable() now refuses to run if kvm->arch.arm_pmu is NULL.

---
 Documentation/virt/kvm/api.rst          |  5 +++++
 arch/arm64/include/asm/kvm_host.h       |  2 +-
 arch/arm64/include/uapi/asm/kvm.h       |  1 +
 arch/arm64/kvm/arm.c                    | 18 ++++++++++++++----
 arch/arm64/kvm/pmu-emul.c               | 14 +++++++++++++-
 include/kvm/arm_pmu.h                   |  4 ++++
 tools/arch/arm64/include/uapi/asm/kvm.h |  1 +
 7 files changed, 39 insertions(+), 6 deletions(-)
Documentation/virt/kvm/api.rst +5 -0
index 52bbbb553ce1..79b024a7ba16 100644
@@ -3515,6 +3515,11 @@ Possible features:
 	  Depends on KVM_CAP_ARM_PSCI_0_2.
 	- KVM_ARM_VCPU_PMU_V3: Emulate PMUv3 for the CPU.
 	  Depends on KVM_CAP_ARM_PMU_V3.
+	- KVM_ARM_VCPU_PMU_V3_STRICT: Enable strict PMUv3 UAPI.
+	  Requires KVM_ARM_VCPU_PMU_V3. If set, KVM does not create a default
+	  PMU; userspace must select a PMU implementation with
+	  KVM_ARM_VCPU_PMU_V3_SET_PMU before the first KVM_RUN. The selected
+	  PMU exposes the SLOTS field of its PMMIR_EL1 register to the guest.
 
 	- KVM_ARM_VCPU_PTRAUTH_ADDRESS: Enables Address Pointer authentication
 	  for arm64 only.
arch/arm64/include/asm/kvm_host.h +1 -1
index 65eead8362e0..a6e33aaf400d 100644
@@ -39,7 +39,7 @@
 
 #define KVM_MAX_VCPUS VGIC_V3_MAX_CPUS
 
-#define KVM_VCPU_MAX_FEATURES 9
+#define KVM_VCPU_MAX_FEATURES 10
 #define KVM_VCPU_VALID_FEATURES	(BIT(KVM_VCPU_MAX_FEATURES) - 1)
 
 #define KVM_REQ_SLEEP \
arch/arm64/include/uapi/asm/kvm.h +1 -0
index 1c13bfa2d38a..019e5e3d892e 100644
@@ -106,6 +106,7 @@ struct kvm_regs {
 #define KVM_ARM_VCPU_PTRAUTH_GENERIC	6 /* VCPU uses generic authentication */
 #define KVM_ARM_VCPU_HAS_EL2		7 /* Support nested virtualization */
 #define KVM_ARM_VCPU_HAS_EL2_E2H0	8 /* Limit NV support to E2H RES0 */
+#define KVM_ARM_VCPU_PMU_V3_STRICT	9 /* No default PMU creation */
 
 struct kvm_vcpu_init {
 	__u32 target;
arch/arm64/kvm/arm.c +14 -4
index 9453321ef8c6..d1914bee1e76 100644
@@ -1548,8 +1548,10 @@ static unsigned long system_supported_vcpu_features(void)
 	if (!cpus_have_final_cap(ARM64_HAS_32BIT_EL1))
 		clear_bit(KVM_ARM_VCPU_EL1_32BIT, &features);
 
-	if (!kvm_supports_guest_pmuv3())
+	if (!kvm_supports_guest_pmuv3()) {
 		clear_bit(KVM_ARM_VCPU_PMU_V3, &features);
+		clear_bit(KVM_ARM_VCPU_PMU_V3_STRICT, &features);
+	}
 
 	if (!system_supports_sve())
 		clear_bit(KVM_ARM_VCPU_SVE, &features);
@@ -1590,6 +1592,11 @@ static int kvm_vcpu_init_check_features(struct kvm_vcpu *vcpu,
 	    test_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, &features))
 		return -EINVAL;
 
+	/* Strict PMUv3 UAPI requires PMUv3. */
+	if (test_bit(KVM_ARM_VCPU_PMU_V3_STRICT, &features) &&
+	    !test_bit(KVM_ARM_VCPU_PMU_V3, &features))
+		return -EINVAL;
+
 	if (!test_bit(KVM_ARM_VCPU_EL1_32BIT, &features))
 		return 0;
 
@@ -1619,10 +1626,13 @@ static int kvm_setup_vcpu(struct kvm_vcpu *vcpu)
 	int ret = 0;
 
 	/*
-	 * When the vCPU has a PMU, but no PMU is set for the guest
-	 * yet, set the default one.
+	 * When the vCPU has a PMU, but no PMU is set for the guest yet, set
+	 * the default one. If KVM_ARM_VCPU_PMU_V3_STRICT is set, no default
+	 * PMU is created, and userspace must select a PMU via
+	 * KVM_ARM_VCPU_PMU_V3_SET_PMU.
 	 */
-	if (kvm_vcpu_has_pmu(vcpu) && !kvm->arch.arm_pmu)
+	if (kvm_vcpu_has_pmu(vcpu) && !kvm->arch.arm_pmu &&
+	    !kvm_vcpu_has_pmuv3_strict(vcpu))
 		ret = kvm_arm_set_default_pmu(kvm);
 
 	/* Prepare for nested if required */
arch/arm64/kvm/pmu-emul.c +13 -1
index e1860acae641..1f24169505a9 100644
@@ -923,6 +923,9 @@ void kvm_vcpu_reload_pmu(struct kvm_vcpu *vcpu)
 
 int kvm_arm_pmu_v3_enable(struct kvm_vcpu *vcpu)
 {
+	if (!vcpu->kvm->arch.arm_pmu)
+		return -EINVAL;
+
 	if (!vcpu->arch.pmu.created)
 		return -EINVAL;
 
@@ -1021,6 +1024,14 @@ u8 kvm_arm_pmu_get_max_counters(struct kvm *kvm)
 {
 	struct arm_pmu *arm_pmu = kvm->arch.arm_pmu;
 
+	/*
+	 * Under KVM_ARM_VCPU_PMU_V3_STRICT no PMU exists until userspace sets
+	 * one, so this can be reached before arm_pmu is set. Report no
+	 * counters in that case.
+	 */
+	if (!arm_pmu)
+		return 0;
+
 	/*
 	 * PMUv3 requires that all event counters are capable of counting any
 	 * event, though the same may not be true of non-PMUv3 hardware.
@@ -1062,7 +1073,8 @@ static void kvm_arm_set_pmu(struct kvm *kvm, struct arm_pmu *arm_pmu)
 }
 
 /**
- * kvm_arm_set_default_pmu - No PMU set, get the default one.
+ * kvm_arm_set_default_pmu - No PMU set and KVM_ARM_VCPU_PMU_V3_STRICT not
+ * set, get the default one.
  * @kvm: The kvm pointer
  *
  * The observant among you will notice that the supported_cpus
include/kvm/arm_pmu.h +4 -0
index 0a36a3d5c894..13468bd5bbf2 100644
@@ -76,6 +76,9 @@ void kvm_vcpu_pmu_resync_el0(void);
 #define kvm_vcpu_has_pmu(vcpu)					\
 	(vcpu_has_feature(vcpu, KVM_ARM_VCPU_PMU_V3))
 
+#define kvm_vcpu_has_pmuv3_strict(vcpu)				\
+	(vcpu_has_feature(vcpu, KVM_ARM_VCPU_PMU_V3_STRICT))
+
 /*
  * Updates the vcpu's view of the pmu events for this cpu.
  * Must be called before every vcpu run after disabling interrupts, to ensure
@@ -161,6 +164,7 @@ static inline u64 kvm_pmu_get_pmceid(struct kvm_vcpu *vcpu, bool pmceid1)
 }
 
 #define kvm_vcpu_has_pmu(vcpu)		({ false; })
+#define kvm_vcpu_has_pmuv3_strict(vcpu)	({ false; })
 static inline void kvm_pmu_update_vcpu_events(struct kvm_vcpu *vcpu) {}
 static inline void kvm_vcpu_pmu_restore_guest(struct kvm_vcpu *vcpu) {}
 static inline void kvm_vcpu_pmu_restore_host(struct kvm_vcpu *vcpu) {}
tools/arch/arm64/include/uapi/asm/kvm.h +1 -0
index 1c13bfa2d38a..019e5e3d892e 100644
@@ -106,6 +106,7 @@ struct kvm_regs {
 #define KVM_ARM_VCPU_PTRAUTH_GENERIC	6 /* VCPU uses generic authentication */
 #define KVM_ARM_VCPU_HAS_EL2		7 /* Support nested virtualization */
 #define KVM_ARM_VCPU_HAS_EL2_E2H0	8 /* Limit NV support to E2H RES0 */
+#define KVM_ARM_VCPU_PMU_V3_STRICT	9 /* No default PMU creation */
 
 struct kvm_vcpu_init {
 	__u32 target;
Signature · 1 line
2.50.1
#2 lore raw

Re: [PATCH v2 1/3] KVM: arm64: Add KVM_ARM_VCPU_PMU_V3_STRICT vCPU feature

Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider: - [High] Silent truncation of PMU event counter states and PMCR_EL0.N during VCPU sysreg restoration before…

Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
- [High] Silent truncation of PMU event counter states and PMCR_EL0.N during VCPU sysreg restoration before PMU initialization. --

commit 7651e4c8e75621127e5fba611a9aa7f3257fa4b2
Author: Congkai Tan <[email protected]>

KVM: arm64: Add KVM_ARM_VCPU_PMU_V3_STRICT vCPU feature

This patch introduces the KVM_ARM_VCPU_PMU_V3_STRICT vCPU feature, preventing KVM from creating a default PMU on initialization. Userspace must instead explicitly select a PMU via KVM_ARM_VCPU_PMU_V3_SET_PMU before the first KVM_RUN.

Quoted 4 lines
arch/arm64/kvm/pmu-emul.c +0 -0
index e1860acae641f..1f24169505a91 100644

[ ... ]

arch/arm64/kvm/pmu-emul.c u8 kvm_arm_pmu_get_max_counters(struct kvm *kvm) · line 1034

Quoted 12 lines
arch/arm64/kvm/pmu-emul.c +8 -0
@@ -1021,6 +1024,14 @@ u8 kvm_arm_pmu_get_max_counters(struct kvm *kvm)
 {
 	struct arm_pmu *arm_pmu = kvm->arch.arm_pmu;
 
+	/*
+	 * Under KVM_ARM_VCPU_PMU_V3_STRICT no PMU exists until userspace sets
+	 * one, so this can be reached before arm_pmu is set. Report no
+	 * counters in that case.
+	 */
+	if (!arm_pmu)
+		return 0;
+

[Severity: High]
Could returning 0 here before the PMU is explicitly selected cause silent truncation of PMU event counter states during live migration?

If a VMM restores PMU sysregs via KVM_SET_ONE_REG before selecting the PMU via KVM_ARM_VCPU_PMU_V3_SET_PMU, set_pmcr() in arch/arm64/kvm/sys_regs.c evaluates:

arch/arm64/kvm/sys_regs.c:set_pmcr() {
    if (!kvm_vm_has_ran_once(kvm) &&
        !vcpu_has_nv(vcpu)        &&
        new_n <= kvm_arm_pmu_get_max_counters(kvm))
        kvm->arch.nr_pmu_counters = new_n;
    ...
}

Because kvm_arm_pmu_get_max_counters() now returns 0, the bounds check fails. set_pmcr() then leaves nr_pmu_counters at 0 without returning an error to userspace.

Consequently, when set_pmreg() executes:

arch/arm64/kvm/sys_regs.c:set_pmreg() {
    ...
    u64 mask = kvm_pmu_accessible_counter_mask(vcpu);
    __vcpu_assign_sys_reg(vcpu, r->reg, val & mask);
}

The mask evaluates to only the cycle counter since nr_pmu_counters is 0. This silently zeroes out the restored event counter bits for registers like PMOVSSET_EL0, PMINTENSET_EL1, and PMCNTENSET_EL0.

Should the KVM API reject this out-of-order restoration sequence with an error like -EINVAL, rather than silently discarding the VMM's restored PMCR_EL0.N and event counter states?

Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=1
#3 lore raw

PATCH v2 2/3 KVM: arm64: Expose PMMIR_EL1.SLOTS under strict PMUv3 UAPI

In reply to Congkai Tan

Introduce a new field pmmir_slots in struct kvm_arch to store PMMIR_EL1.SLOTS. It only saves the actual hardware PMU value when the VMM explicitly selects a PMU under KVM_ARM_VCPU_PMU_V3_STRICT. Otherwise, it stays 0 after allocation.

Use this field to implement guest access, userspace get, and userspace set for PMMIR_EL1:
- access_pmmir(): uses the value in kvm->arch.pmmir_slots directly. If the VMM selected a PMU and KVM_ARM_VCPU_PMU_V3_STRICT is set, the guest can correctly read the underlying core's SLOTS. Otherwise, it continues to read 0 since the true SLOTS value can be nondeterministic.
- get_pmmir(): same as access_pmmir().
- set_pmmir(): only the SLOTS field is writable; a value setting any other bit is rejected with -EINVAL, since get_pmmir() returns SLOTS zero-extended. A value of 0 resets kvm->arch.pmmir_slots to 0 for backward compatibility, as the register is RAZ in older KVM, a value matching the current SLOTS is accepted as a no-op, and anything else is rejected with -EINVAL. Once the VM has run PMMIR_EL1 is immutable, so a mismatching write then returns -EBUSY.

The register is now exposed via KVM_GET_REG_LIST for PMUv3 vCPUs, so add it to the get-reg-list selftest's PMU register list.

---
 arch/arm64/include/asm/kvm_host.h             |  3 +
 arch/arm64/kvm/pmu-emul.c                     | 11 ++++
 arch/arm64/kvm/sys_regs.c                     | 63 ++++++++++++++++++-
 .../selftests/kvm/arm64/get-reg-list.c        |  1 +
 4 files changed, 76 insertions(+), 2 deletions(-)
arch/arm64/include/asm/kvm_host.h +3 -0
index a6e33aaf400d..b896d6eef822 100644
@@ -387,6 +387,9 @@ struct kvm_arch {
 	/* Maximum number of counters for the guest */
 	u8 nr_pmu_counters;
 
+	/* PMMIR_EL1.SLOTS value exposed to the guest. */
+	u8 pmmir_slots;
+
 	/* Hypercall features firmware registers' descriptor */
 	struct kvm_smccc_features smccc_feat;
 	struct maple_tree smccc_filter;
arch/arm64/kvm/pmu-emul.c +11 -0
index 1f24169505a9..9595bce6519f 100644
@@ -1117,6 +1117,17 @@ static int kvm_arm_pmu_v3_set_pmu(struct kvm_vcpu *vcpu, int pmu_id)
 
 			kvm_arm_set_pmu(kvm, arm_pmu);
 			cpumask_copy(kvm->arch.supported_cpus, &arm_pmu->supported_cpus);
+
+			/*
+			 * Since a specific PMU is explicitly selected,
+			 * PMMIR_EL1.SLOTS is deterministic to the guest.
+			 * If KVM_ARM_VCPU_PMU_V3_STRICT is set, snapshot
+			 * the value to allow the guest to read it.
+			 */
+			if (kvm_vcpu_has_pmuv3_strict(vcpu))
+				kvm->arch.pmmir_slots =
+					FIELD_GET(ARMV8_PMU_SLOTS,
+						  arm_pmu->reg_pmmir);
 			ret = 0;
 			break;
 		}
arch/arm64/kvm/sys_regs.c +61 -2

81 lines not on this page · raw message

tools/testing/selftests/kvm/arm64/get-reg-list.c +1 -0
index 0a3a94c4cca1..cfa99979d57c 100644
@@ -532,6 +532,7 @@ static __u64 base_regs[] = {
 static __u64 pmu_regs[] = {
 	ARM64_SYS_REG(3, 0, 9, 14, 1),	/* PMINTENSET_EL1 */
 	ARM64_SYS_REG(3, 0, 9, 14, 2),	/* PMINTENCLR_EL1 */
+	ARM64_SYS_REG(3, 0, 9, 14, 6),	/* PMMIR_EL1 */
 	ARM64_SYS_REG(3, 3, 9, 12, 0),	/* PMCR_EL0 */
 	ARM64_SYS_REG(3, 3, 9, 12, 1),	/* PMCNTENSET_EL0 */
 	ARM64_SYS_REG(3, 3, 9, 12, 2),	/* PMCNTENCLR_EL0 */
Signature · 1 line
2.50.1
#4 lore raw

PATCH v2 3/3 KVM: arm64: Advertise STALL_SLOT* in PMCEID1 under strict PMUv3 UAPI

In reply to Congkai Tan

Skip masking STALL_SLOT, STALL_SLOT_FRONTEND and STALL_SLOT_BACKEND out of PMCEID1 when KVM_ARM_VCPU_PMU_V3_STRICT is set, because this is when PMMIR_EL1.SLOTS is exposed to guests, making these events meaningful for collection.

Change the parameter of compute_pmceid1() from arm_pmu to kvm_vcpu, to check if KVM_ARM_VCPU_PMU_V3_STRICT is set. Also updated the signature of compute_pmceid0() for consistency.

---
 arch/arm64/kvm/pmu-emul.c | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)
arch/arm64/kvm/pmu-emul.c +13 -12
index 9595bce6519f..89eec5a1a499 100644
@@ -851,9 +851,9 @@ static u64 __compute_pmceid(struct arm_pmu *pmu, bool pmceid1)
 	return ((u64)hi[pmceid1] << 32) | lo[pmceid1];
 }
 
-static u64 compute_pmceid0(struct arm_pmu *pmu)
+static u64 compute_pmceid0(struct kvm_vcpu *vcpu)
 {
-	u64 val = __compute_pmceid(pmu, 0);
+	u64 val = __compute_pmceid(vcpu->kvm->arch.arm_pmu, 0);
 
 	/* always support SW_INCR */
 	val |= BIT(ARMV8_PMUV3_PERFCTR_SW_INCR);
@@ -862,32 +862,33 @@ static u64 compute_pmceid0(struct arm_pmu *pmu)
 	return val;
 }
 
-static u64 compute_pmceid1(struct arm_pmu *pmu)
+static u64 compute_pmceid1(struct kvm_vcpu *vcpu)
 {
-	u64 val = __compute_pmceid(pmu, 1);
+	u64 val = __compute_pmceid(vcpu->kvm->arch.arm_pmu, 1);
 
 	/*
-	 * Don't advertise STALL_SLOT*, as PMMIR_EL0 is handled
-	 * as RAZ
+	 * If KVM_ARM_VCPU_PMU_V3_STRICT is not set, PMMIR_EL1 is
+	 * unconditionally RAZ, so don't advertise STALL_SLOT* events.
 	 */
-	val &= ~(BIT_ULL(ARMV8_PMUV3_PERFCTR_STALL_SLOT - 32) |
-		 BIT_ULL(ARMV8_PMUV3_PERFCTR_STALL_SLOT_FRONTEND - 32) |
-		 BIT_ULL(ARMV8_PMUV3_PERFCTR_STALL_SLOT_BACKEND - 32));
+	if (!kvm_vcpu_has_pmuv3_strict(vcpu))
+		val &= ~(BIT_ULL(ARMV8_PMUV3_PERFCTR_STALL_SLOT - 32) |
+			 BIT_ULL(ARMV8_PMUV3_PERFCTR_STALL_SLOT_FRONTEND - 32) |
+			 BIT_ULL(ARMV8_PMUV3_PERFCTR_STALL_SLOT_BACKEND - 32));
+
 	return val;
 }
 
 u64 kvm_pmu_get_pmceid(struct kvm_vcpu *vcpu, bool pmceid1)
 {
-	struct arm_pmu *cpu_pmu = vcpu->kvm->arch.arm_pmu;
 	unsigned long *bmap = vcpu->kvm->arch.pmu_filter;
 	u64 val, mask = 0;
 	int base, i, nr_events;
 
 	if (!pmceid1) {
-		val = compute_pmceid0(cpu_pmu);
+		val = compute_pmceid0(vcpu);
 		base = 0;
 	} else {
-		val = compute_pmceid1(cpu_pmu);
+		val = compute_pmceid1(vcpu);
 		base = 32;
 	}
 
Signature · 1 line
2.50.1
maintainer
#5 lore raw
In reply to Congkai Tan

Hi Congkai,

Thanks for respinning.

Quoted 8 lines · Congkai Tan

Today when the perf tool runs in a guest on cores with PMUv3p4, it fails to parse the default metrics with "Failure to read '#slots'", since perf can only read 0 from sysfs caps/slots, which is backed by PMMIR_EL1.SLOTS that KVM traps as RAZ/WI.

Taking into account backward compatibility and heterogeneous systems, the exposure of PMMIR_EL1.SLOTS is gated behind a new vCPU feature flag:

Quoted 7 lines

- Patch 1 adds the new flag KVM_ARM_VCPU_PMU_V3_STRICT. When set, KVM does not create a default PMU during vCPU init, and the VMM must select one explicitly via KVM_ARM_VCPU_PMU_V3_SET_PMU before the first KVM_RUN.
- Patch 2 exposes PMMIR_EL1.SLOTS of the selected PMU under the flag, and adds userspace get/set for PMMIR_EL1 so that SLOTS can be reset to 0 for backward compatibility.
- Patch 3 stops masking STALL_SLOT* in PMCEID1 under the flag.

I think the series is starting to shape up. Few more things to address:

- Didn't mention it in v1, but writes to PMCR_EL0.N should be ignored when V3_STRICT is set. We now have a vCPU attribute for configuring event counters and the register-based thing is just broken :)

- The vCPU feature flag needs a corresponding KVM_CAP so userspace can detect it

- In terms of patch ordering, the vCPU flag / KVM_CAP exposure should come last after all the behavior changes are implemented (and flag-guarded)

- Move enforcement of a non-NULL arm_pmu to kvm_arm_pmu_v3_init() since userspace must call KVM_ARM_VCPU_PMU_V3_INIT before KVM_RUN

- Prevent the PMU event filter from being configured until a hardware PMU has been selected

I've addressed all of this locally and pushed to my tree [*]. Untested, as always :) Would you be able to give it a spin?

Also, do you have VMM patches for using the new feature flag?

[*]: https://git.kernel.org/pub/scm/linux/kernel/git/oupton/linux.git/log/?h=kvm-arm64/pmu-7.3

Thanks,
Oliver

#6 lore raw
Quoted 2 lines · Oliver Upton

I've addressed all of this locally and pushed to my tree [*]. Untested, as always :) Would you be able to give it a spin?

Thank you so much for the review and code suggestions! I integrated them and created v3 (with one extra change of wiring KVM_CAP_ARM_PMU_V3_STRICT into kvm_vm_ioctl_check_extension() so userspace can detect it). I tested the patches with kernel 7.2 and patched QEMU 9.2.3, and the behaviors were as expected.

I'll send out the v3 patches shortly.

Quoted 1 line

Also, do you have VMM patches for using the new feature flag?

Below is the patch against QEMU 9.2.3 that I used for testing. The tests were performed using different combinations of the four flags (host PMMIR_EL1.SLOTS = 0xa):

- pmu-strict=on,pmu-set=on,pmmir-force=off,pmcr-n-force=off:

    new happy path where guest read the correct SLOTS (0xa).

- pmu-strict=off,pmu-set=off,pmmir-force=off,pmcr-n-force=off:

    old happy path where guest read SLOTS as 0x0.

- pmu-strict=off,pmu-set=on,pmmir-force=off,pmcr-n-force=off:
non-strict with a PMU set, guest read SLOTS as 0x0 (SLOTS stays gated on the flag).

- pmu-strict=on,pmu-set=off,pmmir-force=off,pmcr-n-force=off:
strict with no PMU, so KVM_ARM_VCPU_PMU_V3_INIT failed with -ENXIO and the guest did not boot.

- pmu-strict=on,pmu-set=on,pmmir-force=0,pmcr-n-force=off:
strict with PMMIR forced to 0 (backward compatibility), where the KVM_SET_ONE_REG succeeded and guest read SLOTS as 0x0.

- pmu-strict=on,pmu-set=on,pmmir-force=7,pmcr-n-force=off:
strict with PMMIR forced to a mismatching value, so KVM_SET_ONE_REG rejected with -EINVAL and guest still read SLOTS as 0xa.

- pmu-strict=off,pmu-set=on,pmmir-force=0,pmcr-n-force=off:
non-strict with PMMIR forced to 0, KVM_SET_ONE_REG returned no error and guest read SLOTS as 0x0.

- pmu-strict=off,pmu-set=off,pmmir-force=10,pmcr-n-force=off:
non-strict with PMMIR forced to a non-zero value, KVM_SET_ONE_REG rejected with -EINVAL and guest read SLOTS as 0x0.

- pmu-strict=off,pmu-set=off,pmmir-force=off,pmcr-n-force=3:
non-strict with a PMCR_EL0.N write (original behavior), so the write was honored and N read back as 3.

- pmu-strict=on,pmu-set=on,pmmir-force=off,pmcr-n-force=3:
strict with a PMCR_EL0.N write, so the write was ignored and N read back 6 (the PMU maximum).

---
Local test patch for the kernel "strict PMUv3 UAPI / PMMIR_EL1.SLOTS" series.

Adds four 'virt' machine properties (all default off):

  pmu-strict=on   request KVM_ARM_VCPU_PMU_V3_STRICT at VCPU_INIT
  pmu-set=off     skip KVM_ARM_VCPU_PMU_V3_SET_PMU (exercise the no-PMU path)
  pmmir-force=N   write PMMIR_EL1=N via KVM_SET_ONE_REG (0 = set_pmmir reset)
  pmcr-n-force=N  write then read PMCR_EL0.N; strict ignores the write

SET_PMU is issued before PMU_V3_INIT (the kernel rejects it afterwards) and is followed by a cpreg-cache refresh so the stale PMMIR_EL1=0 cached at init is not written back.

---

target/arm/kvm.c +85 -0
@@ -18,11 +18,16 @@
 #include "qemu/timer.h"
 #include "qemu/error-report.h"
 #include "qemu/main-loop.h"
+#include "qemu/cutils.h"
 #include "qom/object.h"
 #include "qapi/error.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/runstate.h"
 #include "sysemu/kvm.h"
+
+#ifndef KVM_ARM_VCPU_PMU_V3_STRICT
+#define KVM_ARM_VCPU_PMU_V3_STRICT 9
+#endif
 #include "sysemu/kvm_int.h"
 #include "kvm_arm.h"
 #include "cpu.h"
@@ -1728,6 +1733,82 @@
     }
 }
 
+/* Test knob: 'pmu-strict' machine property (request KVM_ARM_VCPU_PMU_V3_STRICT). */
+bool kvm_arm_pmu_strict_enabled(void)
+{
+    Object *m = qdev_get_machine();
+
+    if (m && object_property_find(m, "pmu-strict")) {
+        return object_property_get_bool(m, "pmu-strict", NULL);
+    }
+    return false;
+}
+
+/* Test knob: 'pmu-set' machine property (issue SET_PMU; off = no-PMU path). */
+static bool kvm_arm_pmu_set_enabled(void)
+{
+    Object *m = qdev_get_machine();
+
+    if (m && object_property_find(m, "pmu-set")) {
+        return object_property_get_bool(m, "pmu-set", NULL);
+    }
+    return false;
+}
+
+/* Read the host PMUv3 perf "type" id from sysfs; -1 on failure. */
+static int kvm_arm_pmu_read_host_pmu_type(void)
+{
+    const char *path = "/sys/bus/event_source/devices/armv8_pmuv3_0/type";
+    g_autofree char *contents = NULL;
+    const char *ep;
+    gsize len;
+    int type;
+
+    if (!g_file_get_contents(path, &contents, &len, NULL)) {
+        return -1;
+    }
+    if (qemu_strtoi(contents, &ep, 10, &type) < 0 || (*ep && *ep != '\n')) {
+        return -1;
+    }
+    return type;
+}
+
+/*
+ * Select the host PMU (SET_PMU). Under the strict UAPI this is mandatory and
+ * makes the kernel snapshot PMMIR_EL1.SLOTS. Call after INIT, before PMU_V3_INIT.
+ */
+void kvm_arm_pmu_set_pmu(ARMCPU *cpu)
+{
+    int pmu_type;
+    struct kvm_device_attr attr = {
+        .group = KVM_ARM_VCPU_PMU_V3_CTRL,
+        .attr = KVM_ARM_VCPU_PMU_V3_SET_PMU,
+    };
+
+    if (!cpu->has_pmu || !kvm_arm_pmu_set_enabled()) {
+        return;
+    }
+
+    pmu_type = kvm_arm_pmu_read_host_pmu_type();
+    if (pmu_type < 0) {
+        warn_report("PMU: could not read host PMU type; "
+                    "PMMIR_EL1.SLOTS will read 0 in the guest");
+        return;
+    }
+    attr.addr = (intptr_t)&pmu_type;
+
+    if (!kvm_arm_set_device_attr(cpu, &attr, "PMU SET_PMU")) {
+        warn_report("PMU: KVM_ARM_VCPU_PMU_V3_SET_PMU unsupported or failed; "
+                    "PMMIR_EL1.SLOTS will read 0 in the guest");
+        return;
+    }
+
+    /* Refresh the cpreg cache so the stale PMMIR_EL1=0 isn't written back. */
+    if (!write_kvmstate_to_list(cpu)) {
+        warn_report("PMU: failed to refresh cpreg cache after SET_PMU");
+    }
+}
+
 void kvm_arm_pmu_set_irq(ARMCPU *cpu, int irq)
 {
     struct kvm_device_attr attr = {
@@ -1904,6 +1985,10 @@
     }
     if (cpu->has_pmu) {
         cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_PMU_V3;
+        /* Test knob: opt in to the strict PMUv3 UAPI. */
+        if (kvm_arm_pmu_strict_enabled()) {
+            cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_PMU_V3_STRICT;
+        }
     }
     if (cpu_isar_feature(aa64_sve, cpu)) {
         assert(kvm_arm_sve_supported());
target/arm/kvm_arm.h +14 -0
@@ -207,6 +207,10 @@
 
 void kvm_arm_pmu_init(ARMCPU *cpu);
 void kvm_arm_pmu_set_irq(ARMCPU *cpu, int irq);
+/* Test knob: select the host PMU (SET_PMU). Call after INIT, before PMU_V3_INIT. */
+void kvm_arm_pmu_set_pmu(ARMCPU *cpu);
+/* Test knob: true if the 'pmu-strict' machine property is set. */
+bool kvm_arm_pmu_strict_enabled(void);
 
 /**
  * kvm_arm_pvtime_init:
@@ -280,6 +284,16 @@
     g_assert_not_reached();
 }
 
+static inline void kvm_arm_pmu_set_pmu(ARMCPU *cpu)
+{
+    g_assert_not_reached();
+}
+
+static inline bool kvm_arm_pmu_strict_enabled(void)
+{
+    return false;
+}
+
 static inline void kvm_arm_pvtime_init(ARMCPU *cpu, uint64_t ipa)
 {
     g_assert_not_reached();
hw/arm/virt.c +181 -0

212 lines not on this page · raw message

include/hw/arm/virt.h +5 -0
@@ -180,6 +180,11 @@
     char *oem_id;
     char *oem_table_id;
     bool ns_el2_virt_timer_irq;
+    /* Test knobs for the strict PMUv3 series (-1 / false = off). */
+    bool pmu_strict;      /* request KVM_ARM_VCPU_PMU_V3_STRICT */
+    bool pmu_set;         /* issue SET_PMU (default on) */
+    int64_t pmmir_force;  /* force PMMIR_EL1 via KVM_SET_ONE_REG */
+    int64_t pmcr_n_force; /* write+read PMCR_EL0.N via KVM_SET_ONE_REG */
 };
 
 #define VIRT_ECAM_ID(high) (high ? VIRT_HIGH_PCIE_ECAM : VIRT_PCIE_ECAM)