AMX KVM selftest
cover AMX KVM selftest
Hi Paolo,
Please help review this patchset, which is based on Jing's AMX v2.
https://lore.kernel.org/all/[email protected]/
Hope this patchset will be merged into Jing's v3 for further review.
About this selftest requirement, please check below link:
https://lore.kernel.org/all/[email protected]/
By the way, this amx_test.c file referenced Chang's older test code:
https://lore.kernel.org/lkml/[email protected]/
Thanks!
Yang
Paolo Bonzini (1):
selftest: kvm: Reorder vcpu_load_state steps for AMX
Yang Zhong (2):
selftest: Move struct kvm_x86_state to header
selftest: Support amx selftest
tools/testing/selftests/kvm/Makefile | 1 + .../selftests/kvm/include/x86_64/processor.h | 16 +- .../selftests/kvm/lib/x86_64/processor.c | 44 +-- tools/testing/selftests/kvm/x86_64/amx_test.c | 370 ++++++++++++++++++ 4 files changed, 401 insertions(+), 30 deletions(-) create mode 100644 tools/testing/selftests/kvm/x86_64/amx_test.c
PATCH 1/3 selftest: kvm: Reorder vcpu_load_state steps for AMX
From: Paolo Bonzini <[email protected]>
For AMX support it is recommended to load XCR0 after XFD, so that KVM does not see XFD=0, XCR=1 for a save state that will eventually be disabled (which would lead to premature allocation of the space required for that save state).
It is also required to load XSAVE data after XCR0 and XFD, so that KVM can trigger allocation of the extra space required to store AMX state.
Adjust vcpu_load_state to obey these new requirements.
--- .../selftests/kvm/lib/x86_64/processor.c | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-)
tools/testing/selftests/kvm/lib/x86_64/processor.c +15 -14
@@ -1192,9 +1192,14 @@ void vcpu_load_state(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_x86_state *sstruct vcpu *vcpu =vcpu_find (vm, vcpuid);int r; - r =ioctl (vcpu->fd,KVM_SET_XSAVE , &state->xsave); -TEST_ASSERT (r ==0 ,"Unexpected result from KVM_SET_XSAVE, r: %i" , - r); + r =ioctl (vcpu->fd,KVM_SET_SREGS , &state->sregs); +TEST_ASSERT (r ==0 ,"Unexpected result from KVM_SET_SREGS, r: %i" , + r); + + r =ioctl (vcpu->fd,KVM_SET_MSRS , &state->msrs); +TEST_ASSERT (r == state->msrs.nmsrs, +"Unexpected result from KVM_SET_MSRS,r: %i (failed at %x)" , + r, r == state->msrs.nmsrs ?-1 : state->msrs.entries[r].index);if (kvm_check_cap (KVM_CAP_XCRS )) { r =ioctl (vcpu->fd,KVM_SET_XCRS , &state->xcrs);
@@ -1202,17 +1207,13 @@ void vcpu_load_state(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_x86_state *s r); } - r =ioctl (vcpu->fd,KVM_SET_SREGS , &state->sregs); -TEST_ASSERT (r ==0 ,"Unexpected result from KVM_SET_SREGS, r: %i" , - r); - - r =ioctl (vcpu->fd,KVM_SET_MSRS , &state->msrs); -TEST_ASSERT (r == state->msrs.nmsrs,"Unexpected result from KVM_SET_MSRS, r: %i (failed at %x)" , - r, r == state->msrs.nmsrs ?-1 : state->msrs.entries[r].index); + r =ioctl (vcpu->fd,KVM_SET_XSAVE , &state->xsave); +TEST_ASSERT (r ==0 ,"Unexpected result from KVM_SET_XSAVE, r: %i" , + r); r =ioctl (vcpu->fd,KVM_SET_VCPU_EVENTS , &state->events); -TEST_ASSERT (r ==0 ,"Unexpected result from KVM_SET_VCPU_EVENTS, r: %i" , - r); +TEST_ASSERT (r ==0 ,"Unexpected result from KVM_SET_VCPU_EVENTS, r: %i" , + r); r =ioctl (vcpu->fd,KVM_SET_MP_STATE , &state->mp_state); TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_MP_STATE, r: %i",
@@ -1223,8 +1224,8 @@ void vcpu_load_state(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_x86_state *s r); r =ioctl (vcpu->fd,KVM_SET_REGS , &state->regs); -TEST_ASSERT (r ==0 ,"Unexpected result from KVM_SET_REGS, r: %i" , - r); +TEST_ASSERT (r ==0 ,"Unexpected result from KVM_SET_REGS, r: %i" , + r);if (state->nested.size) { r =ioctl (vcpu->fd,KVM_SET_NESTED_STATE , &state->nested);
PATCH 2/3 selftest: Move struct kvm_x86_state to header
In reply to Yang ZhongThose changes can avoid dereferencing pointer compile issue when amx_test.c reference state->xsave.
Move struct kvm_x86_state definition to processor.h.
--- .../selftests/kvm/include/x86_64/processor.h | 16 +++++++++++++++- .../testing/selftests/kvm/lib/x86_64/processor.c | 15 --------------- 2 files changed, 15 insertions(+), 16 deletions(-)
tools/testing/selftests/kvm/include/x86_64/processor.h +15 -1
@@ -94,6 +94,21 @@ struct desc_ptr {uint64_t address; } __attribute__((packed)); +struct kvm_x86_state { +struct kvm_xsave *xsave; +struct kvm_vcpu_events events; +struct kvm_mp_state mp_state; +struct kvm_regs regs; +struct kvm_xcrs xcrs; +struct kvm_sregs sregs; +struct kvm_debugregs debugregs; +union { +struct kvm_nested_state nested; +char nested_[16384 ]; + }; +struct kvm_msrs msrs; +}; +static inline uint64_t get_desc64_base (const struct desc64 *desc) { return ((uint64_t)desc->base3 << 32) |
@@ -350,7 +365,6 @@ static inline unsigned long get_xmm(int n)bool is_intel_cpu (void ); -struct kvm_x86_state ;struct kvm_x86_state *vcpu_save_state (struct kvm_vm *vm,uint32_t vcpuid);void vcpu_load_state (struct kvm_vm *vm,uint32_t vcpuid,struct kvm_x86_state *state);
tools/testing/selftests/kvm/lib/x86_64/processor.c +0 -15
@@ -1036,21 +1036,6 @@ void vcpu_dump(FILE *stream, struct kvm_vm *vm, uint32_t vcpuid, uint8_t indent)sregs_dump (stream, &sregs, indent +4 ); } -struct kvm_x86_state { -struct kvm_xsave *xsave; -struct kvm_vcpu_events events; -struct kvm_mp_state mp_state; -struct kvm_regs regs; -struct kvm_xcrs xcrs; -struct kvm_sregs sregs; -struct kvm_debugregs debugregs; -union { -struct kvm_nested_state nested; -char nested_[16384 ]; - }; -struct kvm_msrs msrs; -}; -static int kvm_get_num_msrs_fd (int kvm_fd) {struct kvm_msr_list nmsrs;
PATCH 3/3 selftest: Support amx selftest
In reply to Yang ZhongThis selftest do two test cases, one is to trigger #NM exception and check MSR XFD_ERR value. Another case is guest load tile data into tmm0 registers and trap to host side to check memory data after save/restore.
--- tools/testing/selftests/kvm/Makefile | 1 + tools/testing/selftests/kvm/x86_64/amx_test.c | 370 ++++++++++++++++++ 2 files changed, 371 insertions(+) create mode 100644 tools/testing/selftests/kvm/x86_64/amx_test.c
tools/testing/selftests/kvm/Makefile +1 -0
@@ -75,6 +75,7 @@ TEST_GEN_PROGS_x86_64 += x86_64/xen_shinfo_test TEST_GEN_PROGS_x86_64 += x86_64/xen_vmcall_test TEST_GEN_PROGS_x86_64 += x86_64/vmx_pi_mmio_test TEST_GEN_PROGS_x86_64 += x86_64/sev_migrate_tests +TEST_GEN_PROGS_x86_64 += x86_64/amx_test TEST_GEN_PROGS_x86_64 += demand_paging_test TEST_GEN_PROGS_x86_64 += dirty_log_test TEST_GEN_PROGS_x86_64 += dirty_log_perf_test
tools/testing/selftests/kvm/x86_64/amx_test.c new file +370 -0
370 lines not on this page · raw message
Paolo Bonzini
maintainerRe: [PATCH 3/3] selftest: Support amx selftest
Quoted 6 lines · Yang Zhong
This selftest do two test cases, one is to trigger #NM exception and check MSR
XFD_ERRvalue. Another case is guest load tile data into tmm0 registers and trap to host side to check memory data after save/restore.Signed-off-by Yang Zhong <[email protected]>
This is a great start, mainly I'd add a lot more GUEST_SYNCs.
Basically any instruction after the initial GUEST_ASSERTs are a potential point for GUEST_SYNC, except right after the call to set_tilecfg:
GUEST_SYNC(1)
tools/testing/selftests/kvm/x86_64/amx_test.c line 247
Quoted 2 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +2
-0+/* xfd=0, enable amx */ +wrmsr (MSR_IA32_XFD ,0 );
GUEST_SYNC(2)
tools/testing/selftests/kvm/x86_64/amx_test.c line 250
Quoted 3 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +3
-0+GUEST_ASSERT (rdmsr (MSR_IA32_XFD ) ==0 ); +set_tilecfg (amx_cfg); +__ldtilecfg (amx_cfg);
GUEST_SYNC(3)
tools/testing/selftests/kvm/x86_64/amx_test.c line 253
Quoted 3 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +3
-0+/* Check save/restore when trap to userspace */ +__tileloadd (tiledata); +GUEST_SYNC (1 );
This would become 4; here add tilerelease+GUEST_SYNC(5)+XSAVEC, and check that state 18 is not included in XCOMP_BV.
tools/testing/selftests/kvm/x86_64/amx_test.c line 256
Quoted 2 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +2
-0+/* xfd=0x40000, disable amx tiledata */ +wrmsr (MSR_IA32_XFD ,XFEATURE_MASK_XTILEDATA );
GUEST_SYNC(6)
tools/testing/selftests/kvm/x86_64/amx_test.c line 261
Quoted 5 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +5
-0+GUEST_ASSERT (rdmsr (MSR_IA32_XFD ) ==XFEATURE_MASK_XTILEDATA ); +set_tilecfg (amx_cfg); +__ldtilecfg (amx_cfg); +/* Trigger #NM exception */ +__tileloadd (tiledata);
GUEST_SYNC(10); this final GUEST_SYNC should also check TMM0 in the host.
tools/testing/selftests/kvm/x86_64/amx_test.c line 268
Quoted 6 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +6
-0+GUEST_DONE (); +} + +void guest_nm_handler (struct ex_regs *regs) +{ +/* Check if #NM is triggered by XFEATURE_MASK_XTILEDATA */
GUEST_SYNC(7)
tools/testing/selftests/kvm/x86_64/amx_test.c line 270
Quoted 2 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +2
-0+GUEST_ASSERT (rdmsr (MSR_IA32_XFD_ERR ) ==XFEATURE_MASK_XTILEDATA ); +/* Clear xfd_err */
Same here, I'd do a GUEST_SYNC(8) and re-read MSR_IA32_XFD_ERR.
tools/testing/selftests/kvm/x86_64/amx_test.c line 272
Quoted 2 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +2
-0+wrmsr (MSR_IA32_XFD_ERR ,0 ); +GUEST_SYNC (2 );
This becomes GUEST_SYNC(9).
Quoted 11 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +9
-0+} +case UCALL_SYNC : +switch (uc.args[1 ]) { +case 1 : +fprintf (stderr, +"Exit VM by GUEST_SYNC(1), check save/restore.\n" ); + +/* Compacted mode, get amx offset by xsave area +* size subtract 8K amx size.
tools/testing/selftests/kvm/x86_64/amx_test.c line 355
Quoted 10 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +10
-0+*/ + amx_offset = xsave_restore_size -NUM_TILES *TILE_SIZE ; + state =vcpu_save_state (vm,VCPU_ID ); +void *amx_start = (void *)state->xsave + amx_offset; +void *tiles_data = (void *)addr_gva2hva (vm, tiledata); +/* Only check TMM0 register, 1 tile */ + ret =memcmp (amx_start, tiles_data,TILE_SIZE ); +TEST_ASSERT (ret ==0 ,"memcmp failed, ret=%d\n" , ret); +kvm_x86_state_cleanup (state); +break ;
All GUEST_SYNCs should do save_state/load_state like state_test.c. Then of course you can *also* check TMM0 after __tileloadd, which would be cases 4 and 10.
Thanks,
Paolo
Quoted 16 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +15
-0+case 2 : +fprintf (stderr, +"Exit VM by GUEST_SYNC(2), generate #NM exception.\n" ); + goto done; + } +break ; +case UCALL_DONE : + goto done; +default : +TEST_FAIL ("Unknown ucall %lu" , uc.cmd); + } + } +done: +kvm_vm_free (vm); +}
Re: [PATCH 3/3] selftest: Support amx selftest
Quoted 21 lines · Paolo Bonzini
Quoted 6 lines · Yang Zhong
This selftest do two test cases, one is to trigger #NM exception and check MSR
XFD_ERRvalue. Another case is guest load tile data into tmm0 registers and trap to host side to check memory data after save/restore.Signed-off-by Yang Zhong <[email protected]>This is a great start, mainly I'd add a lot more
GUEST_SYNCs.Basically any instruction after the initial
GUEST_ASSERTsare a potential point forGUEST_SYNC, except right after the call toset_tilecfg:
GUEST_SYNC(1)tools/testing/selftests/kvm/x86_64/amx_test.c line 247
Quoted 2 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +2
-0+/* xfd=0, enable amx */ +wrmsr (MSR_IA32_XFD ,0 );
GUEST_SYNC(2)
Quoted 5 lines
tools/testing/selftests/kvm/x86_64/amx_test.c line 250
Quoted 3 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +3
-0+GUEST_ASSERT (rdmsr (MSR_IA32_XFD ) ==0 ); +set_tilecfg (amx_cfg); +__ldtilecfg (amx_cfg);
GUEST_SYNC(3)
Paolo, with this GUEST_SYNC(3) between __ldtilecfg and __tileloadd instructions, the guest code generate (vector:0x6, Invalid Opcode) exception, which is a strange issue. thanks!
Yang
Quoted 8 lines
tools/testing/selftests/kvm/x86_64/amx_test.c line 253
Quoted 4 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +3
-0+/* Check save/restore when trap to userspace */ +__tileloadd (tiledata); +GUEST_SYNC (1 );This would become 4; here add tilerelease+GUEST_SYNC(5)+XSAVEC, and check that state 18 is not included in
XCOMP_BV.
Thanks Paolo, the new code like below:
GUEST_SYNC(5);
/* bit 18 not in the XCOMP_BV after xsavec() */
set_xstatebv(xsave_data, XFEATURE_MASK_XTILEDATA);
__xsavec(xsave_data, XFEATURE_MASK_XTILEDATA);
GUEST_ASSERT((get_xstatebv(xsave_data) & XFEATURE_MASK_XTILEDATA) == 0)
YangQuoted 22 lines
tools/testing/selftests/kvm/x86_64/amx_test.c line 256
Quoted 2 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +2
-0+/* xfd=0x40000, disable amx tiledata */ +wrmsr (MSR_IA32_XFD ,XFEATURE_MASK_XTILEDATA );
GUEST_SYNC(6)tools/testing/selftests/kvm/x86_64/amx_test.c line 261
Quoted 5 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +5
-0+GUEST_ASSERT (rdmsr (MSR_IA32_XFD ) ==XFEATURE_MASK_XTILEDATA ); +set_tilecfg (amx_cfg); +__ldtilecfg (amx_cfg); +/* Trigger #NM exception */ +__tileloadd (tiledata);
GUEST_SYNC(10); this finalGUEST_SYNCshould also check TMM0 in the host.tools/testing/selftests/kvm/x86_64/amx_test.c line 268
Quoted 6 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +6
-0+GUEST_DONE (); +} + +void guest_nm_handler (struct ex_regs *regs) +{ +/* Check if #NM is triggered by XFEATURE_MASK_XTILEDATA */
GUEST_SYNC(7)
Quoted 8 lines
tools/testing/selftests/kvm/x86_64/amx_test.c line 270
Quoted 2 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +2
-0+GUEST_ASSERT (rdmsr (MSR_IA32_XFD_ERR ) ==XFEATURE_MASK_XTILEDATA ); +/* Clear xfd_err */ Same here, I'd do a
GUEST_SYNC(8)and re-readMSR_IA32_XFD_ERR.tools/testing/selftests/kvm/x86_64/amx_test.c line 272
Quoted 2 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +2
-0+wrmsr (MSR_IA32_XFD_ERR ,0 ); +GUEST_SYNC (2 );
This need add regs->rip +=3 to skip __tileloadd() instruction(generate #NM) when VM resume from GUEST_SYNC(9).
Thanks!
Yang
Quoted 19 lines
This becomes
GUEST_SYNC(9).Quoted 17 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +15
-0+} +case UCALL_SYNC : +switch (uc.args[1 ]) { +case 1 : +fprintf (stderr, +"Exit VM by GUEST_SYNC(1), check save/restore.\n" ); + +/* Compacted mode, get amx offset by xsave area +* size subtract 8K amx size. +*/ + amx_offset = xsave_restore_size -NUM_TILES *TILE_SIZE ; + state =vcpu_save_state (vm,VCPU_ID ); +void *amx_start = (void *)state->xsave + amx_offset; +void *tiles_data = (void *)addr_gva2hva (vm, tiledata); +/* Only check TMM0 register, 1 tile */
Quoted 9 lines
tools/testing/selftests/kvm/x86_64/amx_test.c line 355
Quoted 4 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +4
-0+ ret =memcmp (amx_start, tiles_data,TILE_SIZE ); +TEST_ASSERT (ret ==0 ,"memcmp failed, ret=%d\n" , ret); +kvm_x86_state_cleanup (state); +break ;All
GUEST_SYNCsshould do save_state/load_state likestate_test.c. Then of course you can *also* check TMM0 after__tileloadd, which would be cases 4 and 10.
Yes, the new code will add save_state/load_state as in the state_test.c file.
Thanks!
Yang
Quoted 20 lines
Thanks,
Paolo
Quoted 16 lines
tools/testing/selftests/kvm/x86_64/amx_test.c new file +15
-0+case 2 : +fprintf (stderr, +"Exit VM by GUEST_SYNC(2), generate #NM exception.\n" ); + goto done; + } +break ; +case UCALL_DONE : + goto done; +default : +TEST_FAIL ("Unknown ucall %lu" , uc.cmd); + } + } +done: +kvm_vm_free (vm); +}