Skip to contentSkip to conversation

Fix up test failures due to recent KVM changes

PATCH In review Yang Weijiang 2 participants 8 messages Jun 24, 2022 – Jun 25, 2022 KVM

lore
Share & tools
lore
#0 lore raw

v3 cover Fix up test failures due to recent KVM changes

Recently a few pmu KVM patches have been queued, and resulted into some test failures based on the queue branch. Fix them in this series.

Patches were tested with below config:

kernel:
kvm/queue, commit 4284f0063c48

qemu:
master, commit 3a821c52e1a3

platform:
Skylake/Sapphire Rapids

v3:
1. Replaced msr test fixup with Paolo's patch at below link per
Sean's comments, change RO bits to bit 11 and 12 due to Sean's
commit 9fc222967a39 ("KVM: x86: Give host userspace full control of MSR_IA32_MISC_ENABLES")
https://lore.kernel.org/all/[email protected]/

2. Added helpers to check pmu verison and perf_global_ctrl MSR. [Sean]
3. Refactored pmu_lbr code with new helper.

Paolo Bonzini (1):
x86: Don't overwrite bits 11 and 12 of MSR_IA32_MISC_ENABLE

Yang Weijiang (2):
x86: Skip perf related tests when platform cannot support
x86: Check platform vPMU capabilities before run lbr tests

 lib/x86/processor.h | 12 +++++++++++-
 x86/msr.c           |  8 +++++++-
 x86/pmu_lbr.c       | 32 +++++++++++++-------------------
 x86/vmx_tests.c     | 18 ++++++++++++++++++
 4 files changed, 49 insertions(+), 21 deletions(-)

base-commit: 610c15284a537484682adfb4b6d6313991ab954f

Signature · 1 line
2.27.0
#1 lore raw

PATCH v3 1/3 x86: Don't overwrite bits 11 and 12 of MSR_IA32_MISC_ENABLE

From: Paolo Bonzini <[email protected]>

Bits 11 and 12 of MSR_IA32_MISC_ENABLE represent the configuration of the vPMU, and latest KVM does not allow the guest to modify them. Adjust kvm-unit-tests to avoid failures.

---
 x86/msr.c | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)
x86/msr.c +7 -1
index 44fbb3b..8bf38ef 100644
@@ -19,6 +19,7 @@ struct msr_info {
 	bool is_64bit_only;
 	const char *name;
 	unsigned long long value;
+	unsigned long long keep;
 };
 
 
@@ -27,6 +28,8 @@ struct msr_info {
 
 #define MSR_TEST(msr, val, only64)	\
 	{ .index = msr, .name = #msr, .value = val, .is_64bit_only = only64 }
+#define MSR_TEST_RO_BITS(msr, val, only64, ro)	\
+	{ .index = msr, .name = #msr, .value = val, .is_64bit_only = only64, .keep = ro }
 
 struct msr_info msr_info[] =
 {
@@ -34,7 +37,8 @@ struct msr_info msr_info[] =
 	MSR_TEST(MSR_IA32_SYSENTER_ESP, addr_ul, false),
 	MSR_TEST(MSR_IA32_SYSENTER_EIP, addr_ul, false),
 	// reserved: 1:2, 4:6, 8:10, 13:15, 17, 19:21, 24:33, 35:63
-	MSR_TEST(MSR_IA32_MISC_ENABLE, 0x400c51889, false),
+	// read-only: 11, 12
+	MSR_TEST_RO_BITS(MSR_IA32_MISC_ENABLE, 0x400c50009, false, 0x1800),
 	MSR_TEST(MSR_IA32_CR_PAT, 0x07070707, false),
 	MSR_TEST(MSR_FS_BASE, addr_64, true),
 	MSR_TEST(MSR_GS_BASE, addr_64, true),
@@ -59,6 +63,8 @@ static void test_msr_rw(struct msr_info *msr, unsigned long long val)
 	 */
 	if (msr->index == MSR_EFER)
 		val |= orig;
+	else
+		val = (val & ~msr->keep) | (orig & msr->keep);
 	wrmsr(msr->index, val);
 	r = rdmsr(msr->index);
 	wrmsr(msr->index, orig);
Signature · 1 line
2.27.0
#2 lore raw

PATCH v3 2/3 x86: Skip perf related tests when platform cannot support

In reply to Yang Weijiang

Add helpers to check whether MSR_CORE_PERF_GLOBAL_CTRL and rdpmc are supported in KVM. When pmu is disabled with enable_pmu=0, reading MSR_CORE_PERF_GLOBAL_CTRL or executing rdpmc leads to #GP, so skip related tests in this case to avoid test failure.

---
 lib/x86/processor.h | 10 ++++++++++
 x86/vmx_tests.c     | 18 ++++++++++++++++++
 2 files changed, 28 insertions(+)
lib/x86/processor.h +10 -0
index 9a0dad6..70b9193 100644
@@ -690,4 +690,14 @@ static inline bool cpuid_osxsave(void)
 	return cpuid(1).c & (1 << (X86_FEATURE_OSXSAVE % 32));
 }
 
+static inline u8 pmu_version(void)
+{
+	return cpuid(10).a & 0xff;
+}
+
+static inline bool has_perf_global_ctrl(void)
+{
+	return pmu_version() > 1;
+}
+
 #endif
x86/vmx_tests.c +18 -0
index 4d581e7..3cf0776 100644
@@ -944,6 +944,14 @@ static void insn_intercept_main(void)
 			continue;
 		}
 
+		if (insn_table[cur_insn].flag == CPU_RDPMC) {
+			if (!!pmu_version()) {
+				printf("\tFeature required for %s is not supported.\n",
+				       insn_table[cur_insn].name);
+				continue;
+			}
+		}
+
 		if (insn_table[cur_insn].disabled) {
 			printf("\tFeature required for %s is not supported.\n",
 			       insn_table[cur_insn].name);
@@ -7490,6 +7498,11 @@ static void test_perf_global_ctrl(u32 nr, const char *name, u32 ctrl_nr,
 
 static void test_load_host_perf_global_ctrl(void)
 {
+	if (!has_perf_global_ctrl()) {
+		report_skip("test_load_host_perf_global_ctrl");
+		return;
+	}
+
 	if (!(ctrl_exit_rev.clr & EXI_LOAD_PERF)) {
 		printf("\"load IA32_PERF_GLOBAL_CTRL\" exit control not supported\n");
 		return;
@@ -7502,6 +7515,11 @@ static void test_load_host_perf_global_ctrl(void)
 
 static void test_load_guest_perf_global_ctrl(void)
 {
+	if (!has_perf_global_ctrl()) {
+		report_skip("test_load_guest_perf_global_ctrl");
+		return;
+	}
+
 	if (!(ctrl_enter_rev.clr & ENT_LOAD_PERF)) {
 		printf("\"load IA32_PERF_GLOBAL_CTRL\" entry control not supported\n");
 		return;
Signature · 1 line
2.27.0
#3 lore raw

Re: [PATCH v3 2/3] x86: Skip perf related tests when platform cannot support

Quoted 16 lines · Yang Weijiang

Add helpers to check whether MSR_CORE_PERF_GLOBAL_CTRL and rdpmc are supported in KVM. When pmu is disabled with enable_pmu=0, reading MSR_CORE_PERF_GLOBAL_CTRL or executing rdpmc leads to #GP, so skip related tests in this case to avoid test failure.

---
 lib/x86/processor.h | 10 ++++++++++
 x86/vmx_tests.c     | 18 ++++++++++++++++++
 2 files changed, 28 insertions(+)
lib/x86/processor.h +0 -0
index 9a0dad6..70b9193 100644

lib/x86/processor.h static inline bool cpuid_osxsave(void) · line 698

Quoted 9 lines
lib/x86/processor.h +6 -0
@@ -690,4 +690,14 @@ static inline bool cpuid_osxsave(void)
 	return cpuid(1).c & (1 << (X86_FEATURE_OSXSAVE % 32));
 }
 
+static inline u8 pmu_version(void)
+{
+	return cpuid(10).a & 0xff;
+}
+
+static inline bool has_perf_global_ctrl(void)

Slight preference for this_cpu_has_perf_global_ctrl() or cpu_has_perf_global_ctrl().

Quoted 10 lines
lib/x86/processor.h +4 -0
+{
+	return pmu_version() > 1;
+}
+
 #endif
x86/vmx_tests.c +0 -0
index 4d581e7..3cf0776 100644

x86/vmx_tests.c static void insn_intercept_main(void) · line 953

Quoted 10 lines
x86/vmx_tests.c +7 -0
@@ -944,6 +944,14 @@ static void insn_intercept_main(void)
 			continue;
 		}
 
+		if (insn_table[cur_insn].flag == CPU_RDPMC) {
+			if (!!pmu_version()) {
+				printf("\tFeature required for %s is not supported.\n",
+				       insn_table[cur_insn].name);
+				continue;
+			}
+		}

There's no need to copy+paste a bunch of code plus a one-off check, just add another helper that plays nice with supported_fn().

static inline bool this_cpu_has_pmu(void)

{
	return !!pmu_version();
}

x86/vmx_tests.c static void test_perf_global_ctrl(u32 nr, const char *name, u32 ctrl_nr, · line 7502

Quoted 10 lines
x86/vmx_tests.c +3 -0
+
 		if (insn_table[cur_insn].disabled) {
 			printf("\tFeature required for %s is not supported.\n",
 			       insn_table[cur_insn].name);
@@ -7490,6 +7498,11 @@ static void test_perf_global_ctrl(u32 nr, const char *name, u32 ctrl_nr,
 
 static void test_load_host_perf_global_ctrl(void)
 {
+	if (!has_perf_global_ctrl()) {
+		report_skip("test_load_host_perf_global_ctrl");

If you're going to print just the function name, then

		report_skip(__func__);

will suffice. I'd still prefer a more helpful message, especially since there's another "skip" in this function.

x86/vmx_tests.c static void test_perf_global_ctrl(u32 nr, const char *name, u32 ctrl_nr, · line 7508

Quoted 6 lines
x86/vmx_tests.c +3 -0
+		return;
+	}
+
 	if (!(ctrl_exit_rev.clr & EXI_LOAD_PERF)) {
 		printf("\"load IA32_PERF_GLOBAL_CTRL\" exit control not supported\n");
 		return;

Speaking of said skip, can you clean up the existing code to use report_skip()?

Quoted 15 lines
x86/vmx_tests.c +5 -0
@@ -7502,6 +7515,11 @@ static void test_load_host_perf_global_ctrl(void)
 
 static void test_load_guest_perf_global_ctrl(void)
 {
+	if (!has_perf_global_ctrl()) {
+		report_skip("test_load_guest_perf_global_ctrl");
+		return;
+	}
+
 	if (!(ctrl_enter_rev.clr & ENT_LOAD_PERF)) {
 		printf("\"load IA32_PERF_GLOBAL_CTRL\" entry control not supported\n");
 		return;
Signature · 1 line
2.27.0
#4 lore raw

Re: [PATCH v3 2/3] x86: Skip perf related tests when platform cannot support

Quoted 20 lines · Sean Christopherson
Quoted 19 lines · Yang Weijiang

Add helpers to check whether MSR_CORE_PERF_GLOBAL_CTRL and rdpmc are supported in KVM. When pmu is disabled with enable_pmu=0, reading MSR_CORE_PERF_GLOBAL_CTRL or executing rdpmc leads to #GP, so skip related tests in this case to avoid test failure.

---
  lib/x86/processor.h | 10 ++++++++++
  x86/vmx_tests.c     | 18 ++++++++++++++++++
  2 files changed, 28 insertions(+)
lib/x86/processor.h +0 -0
index 9a0dad6..70b9193 100644
@@ -690,4 +690,14 @@ static inline bool cpuid_osxsave(void)
  	return cpuid(1).c & (1 << (X86_FEATURE_OSXSAVE % 32));
  }
 
Quoted 7 lines

lib/x86/processor.h static inline bool cpuid_osxsave(void) · line 698

Quoted 6 lines
lib/x86/processor.h +6 -0
+static inline u8 pmu_version(void)
+{
+	return cpuid(10).a & 0xff;
+}
+
+static inline bool has_perf_global_ctrl(void)

Slight preference for this_cpu_has_perf_global_ctrl() or cpu_has_perf_global_ctrl().

OK, will change it. Thanks.

Quoted 14 lines
Quoted 14 lines
lib/x86/processor.h +4 -0
+{
+	return pmu_version() > 1;
+}
+
  #endif
x86/vmx_tests.c +0 -0
index 4d581e7..3cf0776 100644
@@ -944,6 +944,14 @@ static void insn_intercept_main(void)
  			continue;
  		}
 
Quoted 9 lines

x86/vmx_tests.c static void insn_intercept_main(void) · line 953

Quoted 7 lines
x86/vmx_tests.c +7 -0
+		if (insn_table[cur_insn].flag == CPU_RDPMC) {
+			if (!!pmu_version()) {
+				printf("\tFeature required for %s is not supported.\n",
+				       insn_table[cur_insn].name);
+				continue;
+			}
+		}

There's no need to copy+paste a bunch of code plus a one-off check, just add another helper that plays nice with supported_fn().

Good, I'll add a supported_fn().

Quoted 18 lines

static inline bool this_cpu_has_pmu(void)

{
	return !!pmu_version();
}

x86/vmx_tests.c static void test_perf_global_ctrl(u32 nr, const char *name, u32 ctrl_nr, · line 7502

Quoted 10 lines
x86/vmx_tests.c +3 -0
+
  		if (insn_table[cur_insn].disabled) {
  			printf("\tFeature required for %s is not supported.\n",
  			       insn_table[cur_insn].name);
@@ -7490,6 +7498,11 @@ static void test_perf_global_ctrl(u32 nr, const char *name, u32 ctrl_nr,
 
  static void test_load_host_perf_global_ctrl(void)
  {
+	if (!has_perf_global_ctrl()) {
+		report_skip("test_load_host_perf_global_ctrl");

If you're going to print just the function name, then

Quoted 4 lines
		report_skip(__func__);

will suffice. I'd still prefer a more helpful message, especially since there's another "skip" in this function.

Will do it.

Quoted 8 lines

x86/vmx_tests.c static void test_perf_global_ctrl(u32 nr, const char *name, u32 ctrl_nr, · line 7508

Quoted 7 lines
x86/vmx_tests.c +3 -0
+		return;
+	}
+
  	if (!(ctrl_exit_rev.clr & EXI_LOAD_PERF)) {
  		printf("\"load IA32_PERF_GLOBAL_CTRL\" exit control not supported\n");
  		return;

Speaking of said skip, can you clean up the existing code to use report_skip()?

Will do it.

Quoted 16 lines
Quoted 16 lines
x86/vmx_tests.c +5 -0
@@ -7502,6 +7515,11 @@ static void test_load_host_perf_global_ctrl(void)
 
  static void test_load_guest_perf_global_ctrl(void)
  {
+	if (!has_perf_global_ctrl()) {
+		report_skip("test_load_guest_perf_global_ctrl");
+		return;
+	}
+
  	if (!(ctrl_enter_rev.clr & ENT_LOAD_PERF)) {
  		printf("\"load IA32_PERF_GLOBAL_CTRL\" entry control not supported\n");
  		return;
Signature · 1 line
2.27.0
#5 lore raw

PATCH v3 3/3 x86: Check platform vPMU capabilities before run lbr tests

In reply to Yang Weijiang

Use new helper to check whether pmu is available and Perfmon/Debug capbilities are supported before read MSR_IA32_PERF_CAPABILITIES to avoid test failure. The issue can be captured when enable_pmu=0.

---
 lib/x86/processor.h |  2 +-
 x86/pmu_lbr.c       | 32 +++++++++++++-------------------
 2 files changed, 14 insertions(+), 20 deletions(-)
lib/x86/processor.h +1 -1
index 70b9193..bb917b0 100644
@@ -193,7 +193,7 @@ static inline bool is_intel(void)
 #define X86_FEATURE_PAUSEFILTER     (CPUID(0x8000000A, 0, EDX, 10))
 #define X86_FEATURE_PFTHRESHOLD     (CPUID(0x8000000A, 0, EDX, 12))
 #define	X86_FEATURE_VGIF		(CPUID(0x8000000A, 0, EDX, 16))
-
+#define	X86_FEATURE_PDCM		(CPUID(0x1, 0, ECX, 15))
 
 static inline bool this_cpu_has(u64 feature)
 {
x86/pmu_lbr.c +13 -19
index 688634d..497df1e 100644
@@ -15,6 +15,7 @@
 #define MSR_LBR_SELECT		0x000001c8
 
 volatile int count;
+u32 lbr_from, lbr_to;
 
 static noinline int compute_flag(int i)
 {
@@ -38,18 +39,6 @@ static noinline int lbr_test(void)
 	return 0;
 }
 
-union cpuid10_eax {
-	struct {
-		unsigned int version_id:8;
-		unsigned int num_counters:8;
-		unsigned int bit_width:8;
-		unsigned int mask_length:8;
-	} split;
-	unsigned int full;
-} eax;
-
-u32 lbr_from, lbr_to;
-
 static void init_lbr(void *index)
 {
 	wrmsr(lbr_from + *(int *) index, 0);
@@ -63,7 +52,7 @@ static bool test_init_lbr_from_exception(u64 index)
 
 int main(int ac, char **av)
 {
-	struct cpuid id = cpuid(10);
+	u8 version = pmu_version();
 	u64 perf_cap;
 	int max, i;
 
@@ -74,19 +63,24 @@ int main(int ac, char **av)
 		return 0;
 	}
 
-	perf_cap = rdmsr(MSR_IA32_PERF_CAPABILITIES);
-	eax.full = id.a;
-
-	if (!eax.split.version_id) {
+	if (!version) {
 		printf("No pmu is detected!\n");
 		return report_summary();
 	}
+
+	if (!this_cpu_has(X86_FEATURE_PDCM)) {
+		printf("Perfmon/Debug Capabilities MSR isn't supported\n");
+		return report_summary();
+	}
+
+	perf_cap = rdmsr(MSR_IA32_PERF_CAPABILITIES);
+
 	if (!(perf_cap & PMU_CAP_LBR_FMT)) {
-		printf("No LBR is detected!\n");
+		printf("(Architectural) LBR is not supported.\n");
 		return report_summary();
 	}
 
-	printf("PMU version:		 %d\n", eax.split.version_id);
+	printf("PMU version:		 %d\n", version);
 	printf("LBR version:		 %ld\n", perf_cap & PMU_CAP_LBR_FMT);
 
 	/* Look for LBR from and to MSRs */
Signature · 1 line
2.27.0
#6 lore raw

Re: [PATCH v3 3/3] x86: Check platform vPMU capabilities before run lbr tests

Quoted 15 lines · Yang Weijiang

Use new helper to check whether pmu is available and Perfmon/Debug capbilities are supported before read MSR_IA32_PERF_CAPABILITIES to avoid test failure. The issue can be captured when enable_pmu=0.

---
 lib/x86/processor.h |  2 +-
 x86/pmu_lbr.c       | 32 +++++++++++++-------------------
 2 files changed, 14 insertions(+), 20 deletions(-)
lib/x86/processor.h +0 -0
index 70b9193..bb917b0 100644

lib/x86/processor.h static inline bool is_intel(void) · line 196

Quoted 5 lines
lib/x86/processor.h +1 -1
@@ -193,7 +193,7 @@ static inline bool is_intel(void)
 #define X86_FEATURE_PAUSEFILTER     (CPUID(0x8000000A, 0, EDX, 10))
 #define X86_FEATURE_PFTHRESHOLD     (CPUID(0x8000000A, 0, EDX, 12))
 #define	X86_FEATURE_VGIF		(CPUID(0x8000000A, 0, EDX, 16))
-
+#define	X86_FEATURE_PDCM		(CPUID(0x1, 0, ECX, 15))

Please try to think critically about the code you're writing. All of the existing X86_FEATURE_* definitions are organized by leaf, sub-leaf, register _and_ bit position. And now there's X86_FEATURE_PDCM...

#7 lore raw

Re: [PATCH v3 3/3] x86: Check platform vPMU capabilities before run lbr tests

Quoted 11 lines · Sean Christopherson
Quoted 10 lines · Yang Weijiang

Use new helper to check whether pmu is available and Perfmon/Debug capbilities are supported before read MSR_IA32_PERF_CAPABILITIES to avoid test failure. The issue can be captured when enable_pmu=0.

---
  lib/x86/processor.h |  2 +-
  x86/pmu_lbr.c       | 32 +++++++++++++-------------------
  2 files changed, 14 insertions(+), 20 deletions(-)
Quoted 13 lines

lib/x86/processor.h static inline bool is_intel(void) · line 196

Quoted 10 lines
lib/x86/processor.h +1 -1
index 70b9193..bb917b0 100644
@@ -193,7 +193,7 @@ static inline bool is_intel(void)
  #define X86_FEATURE_PAUSEFILTER     (CPUID(0x8000000A, 0, EDX, 10))
  #define X86_FEATURE_PFTHRESHOLD     (CPUID(0x8000000A, 0, EDX, 12))
  #define	X86_FEATURE_VGIF		(CPUID(0x8000000A, 0, EDX, 16))
-
+#define	X86_FEATURE_PDCM		(CPUID(0x1, 0, ECX, 15))

Please try to think critically about the code you're writing. All of the existing X86_FEATURE_* definitions are organized by leaf, sub-leaf, register _and_ bit position. And now there's X86_FEATURE_PDCM...

My fault, will put it at the right place. thanks!