Skip to contentSkip to conversation

AMX KVM selftest

PATCH Superseded Yang Zhong 2 participants 6 messages Dec 21, 2021 – Dec 22, 2021

Subsystems (2)

v1 of 3 Superseded Latest is v3 →

lore
Share & tools
lore

Yang Zhong

author
#0 lore raw

cover AMX KVM selftest

Hi Paolo,

Please help review this patchset, which is based on Jing's AMX v2.
https://lore.kernel.org/all/[email protected]/

Hope this patchset will be merged into Jing's v3 for further review.

About this selftest requirement, please check below link:
https://lore.kernel.org/all/[email protected]/

By the way, this amx_test.c file referenced Chang's older test code:
https://lore.kernel.org/lkml/[email protected]/

Thanks!

Yang

Paolo Bonzini (1):
selftest: kvm: Reorder vcpu_load_state steps for AMX

Yang Zhong (2):
selftest: Move struct kvm_x86_state to header
selftest: Support amx selftest

 tools/testing/selftests/kvm/Makefile          |   1 +
 .../selftests/kvm/include/x86_64/processor.h  |  16 +-
 .../selftests/kvm/lib/x86_64/processor.c      |  44 +--
 tools/testing/selftests/kvm/x86_64/amx_test.c | 370 ++++++++++++++++++
 4 files changed, 401 insertions(+), 30 deletions(-)
 create mode 100644 tools/testing/selftests/kvm/x86_64/amx_test.c
author
#1 lore raw

PATCH 1/3 selftest: kvm: Reorder vcpu_load_state steps for AMX

From: Paolo Bonzini <[email protected]>

For AMX support it is recommended to load XCR0 after XFD, so that KVM does not see XFD=0, XCR=1 for a save state that will eventually be disabled (which would lead to premature allocation of the space required for that save state).

It is also required to load XSAVE data after XCR0 and XFD, so that KVM can trigger allocation of the extra space required to store AMX state.

Adjust vcpu_load_state to obey these new requirements.

---
 .../selftests/kvm/lib/x86_64/processor.c      | 29 ++++++++++---------
 1 file changed, 15 insertions(+), 14 deletions(-)
tools/testing/selftests/kvm/lib/x86_64/processor.c +15 -14
index 00324d73c687..9b5abf488211 100644
@@ -1192,9 +1192,14 @@ void vcpu_load_state(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_x86_state *s
 	struct vcpu *vcpu = vcpu_find(vm, vcpuid);
 	int r;
 
-	r = ioctl(vcpu->fd, KVM_SET_XSAVE, &state->xsave);
-        TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_XSAVE, r: %i",
-                r);
+	r = ioctl(vcpu->fd, KVM_SET_SREGS, &state->sregs);
+	TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_SREGS, r: %i",
+		r);
+
+	r = ioctl(vcpu->fd, KVM_SET_MSRS, &state->msrs);
+	TEST_ASSERT(r == state->msrs.nmsrs,
+		"Unexpected result from KVM_SET_MSRS,r: %i (failed at %x)",
+		r, r == state->msrs.nmsrs ? -1 : state->msrs.entries[r].index);
 
 	if (kvm_check_cap(KVM_CAP_XCRS)) {
 		r = ioctl(vcpu->fd, KVM_SET_XCRS, &state->xcrs);
@@ -1202,17 +1207,13 @@ void vcpu_load_state(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_x86_state *s
 			    r);
 	}
 
-	r = ioctl(vcpu->fd, KVM_SET_SREGS, &state->sregs);
-        TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_SREGS, r: %i",
-                r);
-
-	r = ioctl(vcpu->fd, KVM_SET_MSRS, &state->msrs);
-        TEST_ASSERT(r == state->msrs.nmsrs, "Unexpected result from KVM_SET_MSRS, r: %i (failed at %x)",
-                r, r == state->msrs.nmsrs ? -1 : state->msrs.entries[r].index);
+	r = ioctl(vcpu->fd, KVM_SET_XSAVE, &state->xsave);
+	TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_XSAVE, r: %i",
+		r);
 
 	r = ioctl(vcpu->fd, KVM_SET_VCPU_EVENTS, &state->events);
-        TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_VCPU_EVENTS, r: %i",
-                r);
+	TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_VCPU_EVENTS, r: %i",
+		r);
 
 	r = ioctl(vcpu->fd, KVM_SET_MP_STATE, &state->mp_state);
         TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_MP_STATE, r: %i",
@@ -1223,8 +1224,8 @@ void vcpu_load_state(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_x86_state *s
                 r);
 
 	r = ioctl(vcpu->fd, KVM_SET_REGS, &state->regs);
-        TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_REGS, r: %i",
-                r);
+	TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_REGS, r: %i",
+		r);
 
 	if (state->nested.size) {
 		r = ioctl(vcpu->fd, KVM_SET_NESTED_STATE, &state->nested);
author
#2 lore raw

PATCH 2/3 selftest: Move struct kvm_x86_state to header

In reply to Yang Zhong

Those changes can avoid dereferencing pointer compile issue when amx_test.c reference state->xsave.

Move struct kvm_x86_state definition to processor.h.

---
 .../selftests/kvm/include/x86_64/processor.h     | 16 +++++++++++++++-
 .../testing/selftests/kvm/lib/x86_64/processor.c | 15 ---------------
 2 files changed, 15 insertions(+), 16 deletions(-)
tools/testing/selftests/kvm/include/x86_64/processor.h +15 -1
index 0546173ab628..28f8fa78a47b 100644
@@ -94,6 +94,21 @@ struct desc_ptr {
 	uint64_t address;
 } __attribute__((packed));
 
+struct kvm_x86_state {
+	struct kvm_xsave *xsave;
+	struct kvm_vcpu_events events;
+	struct kvm_mp_state mp_state;
+	struct kvm_regs regs;
+	struct kvm_xcrs xcrs;
+	struct kvm_sregs sregs;
+	struct kvm_debugregs debugregs;
+	union {
+		struct kvm_nested_state nested;
+		char nested_[16384];
+	};
+	struct kvm_msrs msrs;
+};
+
 static inline uint64_t get_desc64_base(const struct desc64 *desc)
 {
 	return ((uint64_t)desc->base3 << 32) |
@@ -350,7 +365,6 @@ static inline unsigned long get_xmm(int n)
 
 bool is_intel_cpu(void);
 
-struct kvm_x86_state;
 struct kvm_x86_state *vcpu_save_state(struct kvm_vm *vm, uint32_t vcpuid);
 void vcpu_load_state(struct kvm_vm *vm, uint32_t vcpuid,
 		     struct kvm_x86_state *state);
tools/testing/selftests/kvm/lib/x86_64/processor.c +0 -15
index 9b5abf488211..126f8e743bc2 100644
@@ -1036,21 +1036,6 @@ void vcpu_dump(FILE *stream, struct kvm_vm *vm, uint32_t vcpuid, uint8_t indent)
 	sregs_dump(stream, &sregs, indent + 4);
 }
 
-struct kvm_x86_state {
-	struct kvm_xsave *xsave;
-	struct kvm_vcpu_events events;
-	struct kvm_mp_state mp_state;
-	struct kvm_regs regs;
-	struct kvm_xcrs xcrs;
-	struct kvm_sregs sregs;
-	struct kvm_debugregs debugregs;
-	union {
-		struct kvm_nested_state nested;
-		char nested_[16384];
-	};
-	struct kvm_msrs msrs;
-};
-
 static int kvm_get_num_msrs_fd(int kvm_fd)
 {
 	struct kvm_msr_list nmsrs;
author
#3 lore raw

PATCH 3/3 selftest: Support amx selftest

In reply to Yang Zhong

This selftest do two test cases, one is to trigger #NM exception and check MSR XFD_ERR value. Another case is guest load tile data into tmm0 registers and trap to host side to check memory data after save/restore.

---
 tools/testing/selftests/kvm/Makefile          |   1 +
 tools/testing/selftests/kvm/x86_64/amx_test.c | 370 ++++++++++++++++++
 2 files changed, 371 insertions(+)
 create mode 100644 tools/testing/selftests/kvm/x86_64/amx_test.c
tools/testing/selftests/kvm/Makefile +1 -0
index c4e34717826a..ecb1fe022111 100644
@@ -75,6 +75,7 @@ TEST_GEN_PROGS_x86_64 += x86_64/xen_shinfo_test
 TEST_GEN_PROGS_x86_64 += x86_64/xen_vmcall_test
 TEST_GEN_PROGS_x86_64 += x86_64/vmx_pi_mmio_test
 TEST_GEN_PROGS_x86_64 += x86_64/sev_migrate_tests
+TEST_GEN_PROGS_x86_64 += x86_64/amx_test
 TEST_GEN_PROGS_x86_64 += demand_paging_test
 TEST_GEN_PROGS_x86_64 += dirty_log_test
 TEST_GEN_PROGS_x86_64 += dirty_log_perf_test
tools/testing/selftests/kvm/x86_64/amx_test.c new file +370 -0

370 lines not on this page · raw message

maintainer
#4 lore raw

Re: [PATCH 3/3] selftest: Support amx selftest

Quoted 6 lines · Yang Zhong

This selftest do two test cases, one is to trigger #NM exception and check MSR XFD_ERR value. Another case is guest load tile data into tmm0 registers and trap to host side to check memory data after save/restore.

This is a great start, mainly I'd add a lot more GUEST_SYNCs.

Basically any instruction after the initial GUEST_ASSERTs are a potential point for GUEST_SYNC, except right after the call to set_tilecfg:

GUEST_SYNC(1)

tools/testing/selftests/kvm/x86_64/amx_test.c line 247

Quoted 2 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +2 -0
+	/* xfd=0, enable amx */
+	wrmsr(MSR_IA32_XFD, 0);

GUEST_SYNC(2)

tools/testing/selftests/kvm/x86_64/amx_test.c line 250

Quoted 3 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +3 -0
+	GUEST_ASSERT(rdmsr(MSR_IA32_XFD) == 0);
+	set_tilecfg(amx_cfg);
+	__ldtilecfg(amx_cfg);

GUEST_SYNC(3)

tools/testing/selftests/kvm/x86_64/amx_test.c line 253

Quoted 3 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +3 -0
+	/* Check save/restore when trap to userspace */
+	__tileloadd(tiledata);
+	GUEST_SYNC(1);

This would become 4; here add tilerelease+GUEST_SYNC(5)+XSAVEC, and check that state 18 is not included in XCOMP_BV.

tools/testing/selftests/kvm/x86_64/amx_test.c line 256

Quoted 2 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +2 -0
+	/* xfd=0x40000, disable amx tiledata */
+	wrmsr(MSR_IA32_XFD, XFEATURE_MASK_XTILEDATA);

GUEST_SYNC(6)

tools/testing/selftests/kvm/x86_64/amx_test.c line 261

Quoted 5 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +5 -0
+	GUEST_ASSERT(rdmsr(MSR_IA32_XFD) == XFEATURE_MASK_XTILEDATA);
+	set_tilecfg(amx_cfg);
+	__ldtilecfg(amx_cfg);
+	/* Trigger #NM exception */
+	__tileloadd(tiledata);

GUEST_SYNC(10); this final GUEST_SYNC should also check TMM0 in the host.

tools/testing/selftests/kvm/x86_64/amx_test.c line 268

Quoted 6 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +6 -0
+	GUEST_DONE();
+}
+
+void guest_nm_handler(struct ex_regs *regs)
+{
+	/* Check if #NM is triggered by XFEATURE_MASK_XTILEDATA */

GUEST_SYNC(7)

tools/testing/selftests/kvm/x86_64/amx_test.c line 270

Quoted 2 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +2 -0
+	GUEST_ASSERT(rdmsr(MSR_IA32_XFD_ERR) == XFEATURE_MASK_XTILEDATA);
+	/* Clear xfd_err */

Same here, I'd do a GUEST_SYNC(8) and re-read MSR_IA32_XFD_ERR.

tools/testing/selftests/kvm/x86_64/amx_test.c line 272

Quoted 2 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +2 -0
+	wrmsr(MSR_IA32_XFD_ERR, 0);
+	GUEST_SYNC(2);

This becomes GUEST_SYNC(9).

Quoted 11 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +9 -0
+}
 
 
+		case UCALL_SYNC:
+			switch (uc.args[1]) {
+			case 1:
+				fprintf(stderr,
+					"Exit VM by GUEST_SYNC(1), check save/restore.\n");
+
+				/* Compacted mode, get amx offset by xsave area
+				 * size subtract 8K amx size.

tools/testing/selftests/kvm/x86_64/amx_test.c line 355

Quoted 10 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +10 -0
+				 */
+				amx_offset = xsave_restore_size - NUM_TILES*TILE_SIZE;
+				state = vcpu_save_state(vm, VCPU_ID);
+				void *amx_start = (void *)state->xsave + amx_offset;
+				void *tiles_data = (void *)addr_gva2hva(vm, tiledata);
+				/* Only check TMM0 register, 1 tile */
+				ret = memcmp(amx_start, tiles_data, TILE_SIZE);
+				TEST_ASSERT(ret == 0, "memcmp failed, ret=%d\n", ret);
+				kvm_x86_state_cleanup(state);
+				break;

All GUEST_SYNCs should do save_state/load_state like state_test.c. Then of course you can *also* check TMM0 after __tileloadd, which would be cases 4 and 10.

Thanks,

Paolo

Quoted 16 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +15 -0
+			case 2:
+				fprintf(stderr,
+					"Exit VM by GUEST_SYNC(2), generate #NM exception.\n");
+				goto done;
+			}
+			break;
+		case UCALL_DONE:
+			goto done;
+		default:
+			TEST_FAIL("Unknown ucall %lu", uc.cmd);
+		}
+	}
+done:
+	kvm_vm_free(vm);
+}
author
#5 lore raw

Re: [PATCH 3/3] selftest: Support amx selftest

Quoted 21 lines · Paolo Bonzini
Quoted 6 lines · Yang Zhong

This selftest do two test cases, one is to trigger #NM exception and check MSR XFD_ERR value. Another case is guest load tile data into tmm0 registers and trap to host side to check memory data after save/restore.

This is a great start, mainly I'd add a lot more GUEST_SYNCs.

Basically any instruction after the initial GUEST_ASSERTs are a potential point for GUEST_SYNC, except right after the call to set_tilecfg:

GUEST_SYNC(1)

tools/testing/selftests/kvm/x86_64/amx_test.c line 247

Quoted 2 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +2 -0
+	/* xfd=0, enable amx */
+	wrmsr(MSR_IA32_XFD, 0);

GUEST_SYNC(2)

Quoted 5 lines

tools/testing/selftests/kvm/x86_64/amx_test.c line 250

Quoted 3 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +3 -0
+	GUEST_ASSERT(rdmsr(MSR_IA32_XFD) == 0);
+	set_tilecfg(amx_cfg);
+	__ldtilecfg(amx_cfg);

GUEST_SYNC(3)

Paolo, with this GUEST_SYNC(3) between __ldtilecfg and __tileloadd instructions, the guest code generate (vector:0x6, Invalid Opcode) exception, which is a strange issue. thanks!

Yang

Quoted 8 lines

tools/testing/selftests/kvm/x86_64/amx_test.c line 253

Quoted 4 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +3 -0
+	/* Check save/restore when trap to userspace */
+	__tileloadd(tiledata);
+	GUEST_SYNC(1);

This would become 4; here add tilerelease+GUEST_SYNC(5)+XSAVEC, and check that state 18 is not included in XCOMP_BV.

Thanks Paolo, the new code like below:

    GUEST_SYNC(5);
    /* bit 18 not in the XCOMP_BV after xsavec() */
    set_xstatebv(xsave_data, XFEATURE_MASK_XTILEDATA);
     __xsavec(xsave_data, XFEATURE_MASK_XTILEDATA);
    GUEST_ASSERT((get_xstatebv(xsave_data) & XFEATURE_MASK_XTILEDATA) == 0)


    Yang
Quoted 22 lines

tools/testing/selftests/kvm/x86_64/amx_test.c line 256

Quoted 2 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +2 -0
+	/* xfd=0x40000, disable amx tiledata */
+	wrmsr(MSR_IA32_XFD, XFEATURE_MASK_XTILEDATA);

GUEST_SYNC(6)

tools/testing/selftests/kvm/x86_64/amx_test.c line 261

Quoted 5 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +5 -0
+	GUEST_ASSERT(rdmsr(MSR_IA32_XFD) == XFEATURE_MASK_XTILEDATA);
+	set_tilecfg(amx_cfg);
+	__ldtilecfg(amx_cfg);
+	/* Trigger #NM exception */
+	__tileloadd(tiledata);

GUEST_SYNC(10); this final GUEST_SYNC should also check TMM0 in the host.

tools/testing/selftests/kvm/x86_64/amx_test.c line 268

Quoted 6 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +6 -0
+	GUEST_DONE();
+}
+
+void guest_nm_handler(struct ex_regs *regs)
+{
+	/* Check if #NM is triggered by XFEATURE_MASK_XTILEDATA */

GUEST_SYNC(7)

Quoted 8 lines

tools/testing/selftests/kvm/x86_64/amx_test.c line 270

Quoted 2 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +2 -0
+	GUEST_ASSERT(rdmsr(MSR_IA32_XFD_ERR) == XFEATURE_MASK_XTILEDATA);
+	/* Clear xfd_err */

Same here, I'd do a GUEST_SYNC(8) and re-read MSR_IA32_XFD_ERR.

tools/testing/selftests/kvm/x86_64/amx_test.c line 272

Quoted 2 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +2 -0
+	wrmsr(MSR_IA32_XFD_ERR, 0);
+	GUEST_SYNC(2);

This need add regs->rip +=3 to skip __tileloadd() instruction(generate #NM) when VM resume from GUEST_SYNC(9).

Thanks!

Yang

Quoted 19 lines

This becomes GUEST_SYNC(9).

Quoted 17 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +15 -0
+}
 
 
+		case UCALL_SYNC:
+			switch (uc.args[1]) {
+			case 1:
+				fprintf(stderr,
+					"Exit VM by GUEST_SYNC(1), check save/restore.\n");
+
+				/* Compacted mode, get amx offset by xsave area
+				 * size subtract 8K amx size.
+				 */
+				amx_offset = xsave_restore_size - NUM_TILES*TILE_SIZE;
+				state = vcpu_save_state(vm, VCPU_ID);
+				void *amx_start = (void *)state->xsave + amx_offset;
+				void *tiles_data = (void *)addr_gva2hva(vm, tiledata);
+				/* Only check TMM0 register, 1 tile */
Quoted 9 lines

tools/testing/selftests/kvm/x86_64/amx_test.c line 355

Quoted 4 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +4 -0
+				ret = memcmp(amx_start, tiles_data, TILE_SIZE);
+				TEST_ASSERT(ret == 0, "memcmp failed, ret=%d\n", ret);
+				kvm_x86_state_cleanup(state);
+				break;

All GUEST_SYNCs should do save_state/load_state like state_test.c. Then of course you can *also* check TMM0 after __tileloadd, which would be cases 4 and 10.

Yes, the new code will add save_state/load_state as in the state_test.c file.

Thanks!

Yang

Quoted 20 lines

Thanks,

Paolo

Quoted 16 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +15 -0
+			case 2:
+				fprintf(stderr,
+					"Exit VM by GUEST_SYNC(2), generate #NM exception.\n");
+				goto done;
+			}
+			break;
+		case UCALL_DONE:
+			goto done;
+		default:
+			TEST_FAIL("Unknown ucall %lu", uc.cmd);
+		}
+	}
+done:
+	kvm_vm_free(vm);
+}