x86: nvmx: test max atomic switch MSRs
PATCH 1/1 x86: nvmx: test max atomic switch MSRs
Excerise nested VMX's atomic MSR switch code (e.g., VM-entry MSR-load list) at the maximum number of MSRs supported, as described in the SDM, in the appendix chapter titled "MISCELLANEOUS DATA".
--- x86/vmx_tests.c | 139 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 139 insertions(+)
x86/vmx_tests.c +139 -0
@@ -2718,6 +2718,11 @@ static void ept_reserved_bit(int bit)#define PAGE_2M_ORDER 9#define PAGE_1G_ORDER 18 +static void *alloc_2m_page (void ) +{ +return alloc_pages (PAGE_2M_ORDER ); +} +static void *get_1g_page (void ) {static void *alloc;
@@ -8570,6 +8575,138 @@ static int invalid_msr_entry_failure(struct vmentry_failure *failure)return VMX_TEST_VMEXIT ; } +enum atomic_switch_msr_scenario { +VM_ENTER_LOAD , +VM_EXIT_LOAD , +VM_EXIT_STORE , +ATOMIC_SWITCH_MSR_SCENARIO_END , +}; + +static void atomic_switch_msr_limit_test_guest (void ) +{ +vmcall (); +} + +static void populate_msr_list (struct vmx_msr_entry *msr_list,int count) +{ +int i; + +for (i =0 ; i < count; i++) { + msr_list[i].index =MSR_IA32_TSC ; + msr_list[i].reserved =0 ; + msr_list[i].value =0x1234567890abcdef ; + } +} + +static void configure_atomic_switch_msr_limit_test ( +struct vmx_msr_entry *test_msr_list,int count) +{ +struct vmx_msr_entry *msr_list; +const u32 two_mb =1 <<21 ; +enum atomic_switch_msr_scenario s; +enum Encoding addr_field; +enum Encoding cnt_field; + +for (s =0 ; s <ATOMIC_SWITCH_MSR_SCENARIO_END ; s++) { +switch (s) { +case VM_ENTER_LOAD : + addr_field =ENTER_MSR_LD_ADDR ; + cnt_field =ENT_MSR_LD_CNT ; +break ; +case VM_EXIT_LOAD : + addr_field =EXIT_MSR_LD_ADDR ; + cnt_field =EXI_MSR_LD_CNT ; +break ; +case VM_EXIT_STORE : + addr_field =EXIT_MSR_ST_ADDR ; + cnt_field =EXI_MSR_ST_CNT ; +break ; +default : +TEST_ASSERT (false); + } + + msr_list = (struct vmx_msr_entry *)vmcs_read (addr_field); +memset (msr_list,0xff , two_mb); +if (msr_list == test_msr_list) { +populate_msr_list (msr_list, count); +vmcs_write (cnt_field, count); + }else { +vmcs_write (cnt_field,0 ); + } + } +} + +static int max_msr_list_size (void ) +{ +u32 vmx_misc =rdmsr (MSR_IA32_VMX_MISC ); +u32 factor = ((vmx_misc &GENMASK (27 ,25 )) >>25 ) +1 ; + +return factor *512 ; +} + +static void atomic_switch_msr_limit_test (void ) +{ +struct vmx_msr_entry *msr_list; +enum atomic_switch_msr_scenario s; + +/* +* Check for the IA32_TSC MSR, +* available with the "TSC flag" and used to populate the MSR lists. +*/ +if (!(cpuid (1 ).d & (1 <<4 ))) { +report_skip (__func__); +return ; + } + +/* Set L2 guest. */ +test_set_guest (atomic_switch_msr_limit_test_guest); + +/* Setup atomic MSR switch lists. */ + msr_list =alloc_2m_page (); +vmcs_write (ENTER_MSR_LD_ADDR ,virt_to_phys (msr_list)); + msr_list =alloc_2m_page (); +vmcs_write (EXIT_MSR_LD_ADDR ,virt_to_phys (msr_list)); + msr_list =alloc_2m_page (); +vmcs_write (EXIT_MSR_ST_ADDR ,virt_to_phys (msr_list)); + +/* Execute each test case. */ +for (s =0 ; s <ATOMIC_SWITCH_MSR_SCENARIO_END ; s++) { +struct vmx_msr_entry *msr_list; +int count =max_msr_list_size (); + +switch (s) { +case VM_ENTER_LOAD : + msr_list = (struct vmx_msr_entry *)vmcs_read ( +ENTER_MSR_LD_ADDR ); +break ; +case VM_EXIT_LOAD : + msr_list = (struct vmx_msr_entry *)vmcs_read ( +EXIT_MSR_LD_ADDR ); +break ; +case VM_EXIT_STORE : + msr_list = (struct vmx_msr_entry *)vmcs_read ( +EXIT_MSR_ST_ADDR ); +break ; +default : +report ("Bad test scenario, %d." , false, s); +continue ; + } + +configure_atomic_switch_msr_limit_test (msr_list, count); +enter_guest (); +assert_exit_reason (VMX_VMCALL ); + } + +/* Reset the atomic MSR switch count to 0 for all three lists. */ +configure_atomic_switch_msr_limit_test (0 ,0 ); +/* Proceed past guest's single vmcall instruction. */ +enter_guest (); +skip_exit_vmcall (); +/* Terminate the guest. */ +enter_guest (); +skip_exit_vmcall (); +} +#define TEST (name) { #name, .v2 = name }
@@ -8660,5 +8797,7 @@ struct vmx_test vmx_tests[] = {TEST (ept_access_test_paddr_read_execute_ad_enabled),TEST (ept_access_test_paddr_not_present_page_fault),TEST (ept_access_test_force_2m_page), +/* Atomic MSR switch tests. */ +TEST (atomic_switch_msr_limit_test), { NULL, NULL, NULL, NULL, NULL, {0} }, };
Signature · 1 line
2.23.0.237.gc6a4ce50a0-goog
Quoted 16 lines · Marc Orr
Excerise nested VMX's atomic MSR switch code (e.g., VM-entry MSR-load list) at the maximum number of MSRs supported, as described in the SDM, in the appendix chapter titled "MISCELLANEOUS DATA".
Suggested-by Jim Mattson <[email protected]>Reviewed-by Jim Mattson <[email protected]>Signed-off-by Marc Orr <[email protected]>--- x86/vmx_tests.c | 139 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 139 insertions(+)x86/vmx_tests.c +0
-0
x86/vmx_tests.c static void ept_reserved_bit(int bit) · line 2724
Quoted 7 lines
x86/vmx_tests.c +4
-0@@ -2718,6 +2718,11 @@ static void ept_reserved_bit(int bit)#define PAGE_2M_ORDER 9#define PAGE_1G_ORDER 18 +static void *alloc_2m_page (void ) +{ +return alloc_pages (PAGE_2M_ORDER ); +}
Allocating 2mb pages is complete overkill. The absolute theoretical max for the number of MSRs is (8 * 512) = 4096, for a total of 32kb per list. We can even show the math so that it's obvious how the size is calculated. Plus one order so we can test overrun.
/*
* The max number of MSRs is specified in 3 bits bits, plus 1. I.e. 7+1==8.
* Allocate 64k bytes of data to cover max_msr_list_size and then some. */
static const u32 msr_list_page_order = 4;
Quoted 5 lines
x86/vmx_tests.c +1
-0+static void *get_1g_page (void ) {static void *alloc;
x86/vmx_tests.c static int invalid_msr_entry_failure(struct vmentry_failure *failure) · line 8583
Quoted 9 lines
x86/vmx_tests.c +6
-0@@ -8570,6 +8575,138 @@ static int invalid_msr_entry_failure(struct vmentry_failure *failure)return VMX_TEST_VMEXIT ; } +enum atomic_switch_msr_scenario { +VM_ENTER_LOAD , +VM_EXIT_LOAD , +VM_EXIT_STORE , +ATOMIC_SWITCH_MSR_SCENARIO_END , +};
How about:
enum atomic_switch_msr_lists {
VM_ENTER_LOAD,
VM_EXIT_LOAD,
VM_EXIT_STORE,
NR_ATOMIC_SWITCH_MSR_LISTS,
};
IMO that yields a much more intuitive test loop:
for (i = 0; i < NR_ATOMIC_SWITCH_MSR_LISTS; i++) {
}But we probably don't even need a loop...
Quoted 4 lines
x86/vmx_tests.c +4
-0+ +static void atomic_switch_msr_limit_test_guest (void ) +{ +vmcall ();
x86/vmx_tests.c static int invalid_msr_entry_failure(struct vmentry_failure *failure) · line 8597
Quoted 10 lines
x86/vmx_tests.c +10
-0+} + +static void populate_msr_list (struct vmx_msr_entry *msr_list,int count) +{ +int i; + +for (i =0 ; i < count; i++) { + msr_list[i].index =MSR_IA32_TSC ; + msr_list[i].reserved =0 ; + msr_list[i].value =0x1234567890abcdef ;
Maybe overkill, but we can use a fast string op for this. I think I got the union right?
static void populate_msr_list(struct vmx_msr_entry *msr_list, int count)
{
union {
struct vmx_msr_entry msr;
u64 val;
} tmp;
tmp.msr.index = MSR_IA32_TSC;
tmp.msr.reserved = 0;
tmp.msr.value = 0x1234567890abcdef;
asm volatile (
"rep stosq\n\t"
: "=c"(count), "=D"(msr_list)
: "a"(tmp.val), "c"(count), "D"(msr_list)
: "memory"
);
}Quoted 22 lines
x86/vmx_tests.c +22
-0+ } +} + +static void configure_atomic_switch_msr_limit_test ( +struct vmx_msr_entry *test_msr_list,int count) +{ +struct vmx_msr_entry *msr_list; +const u32 two_mb =1 <<21 ; +enum atomic_switch_msr_scenario s; +enum Encoding addr_field; +enum Encoding cnt_field; + +for (s =0 ; s <ATOMIC_SWITCH_MSR_SCENARIO_END ; s++) { +switch (s) { +case VM_ENTER_LOAD : + addr_field =ENTER_MSR_LD_ADDR ; + cnt_field =ENT_MSR_LD_CNT ; +break ; +case VM_EXIT_LOAD : + addr_field =EXIT_MSR_LD_ADDR ; + cnt_field =EXI_MSR_LD_CNT ; +break ;
x86/vmx_tests.c static int invalid_msr_entry_failure(struct vmentry_failure *failure) · line 8629
Quoted 10 lines
x86/vmx_tests.c +10
-0+case VM_EXIT_STORE : + addr_field =EXIT_MSR_ST_ADDR ; + cnt_field =EXI_MSR_ST_CNT ; +break ; +default : +TEST_ASSERT (false); + } + + msr_list = (struct vmx_msr_entry *)vmcs_read (addr_field); +memset (msr_list,0xff , two_mb);
Writing 8mb of data for each test is a waste of time, i.e. 6mb to reset each list, and another 2mb to populate the target list.
The for-loop in the helper is also confusing and superfluous.
Quoted 31 lines
x86/vmx_tests.c +31
-0+if (msr_list == test_msr_list) { +populate_msr_list (msr_list, count); +vmcs_write (cnt_field, count); + }else { +vmcs_write (cnt_field,0 ); + } + } +} + +static int max_msr_list_size (void ) +{ +u32 vmx_misc =rdmsr (MSR_IA32_VMX_MISC ); +u32 factor = ((vmx_misc &GENMASK (27 ,25 )) >>25 ) +1 ; + +return factor *512 ; +} + +static void atomic_switch_msr_limit_test (void ) +{ +struct vmx_msr_entry *msr_list; +enum atomic_switch_msr_scenario s; + +/* +* Check for the IA32_TSC MSR, +* available with the "TSC flag" and used to populate the MSR lists. +*/ +if (!(cpuid (1 ).d & (1 <<4 ))) { +report_skip (__func__); +return ; + } +
x86/vmx_tests.c static int invalid_msr_entry_failure(struct vmentry_failure *failure) · line 8670
Quoted 10 lines
x86/vmx_tests.c +10
-0+/* Set L2 guest. */ +test_set_guest (atomic_switch_msr_limit_test_guest); + +/* Setup atomic MSR switch lists. */ + msr_list =alloc_2m_page (); +vmcs_write (ENTER_MSR_LD_ADDR ,virt_to_phys (msr_list)); + msr_list =alloc_2m_page (); +vmcs_write (EXIT_MSR_LD_ADDR ,virt_to_phys (msr_list)); + msr_list =alloc_2m_page (); +vmcs_write (EXIT_MSR_ST_ADDR ,virt_to_phys (msr_list));
This memory should really be freed. Not holding pointers for each list also seems silly, e.g. requires a VMREAD just to get a pointer.
x86/vmx_tests.c static int invalid_msr_entry_failure(struct vmentry_failure *failure) · line 8673
Quoted 3 lines
x86/vmx_tests.c +3
-0+ +/* Execute each test case. */ +for (s =0 ; s <ATOMIC_SWITCH_MSR_SCENARIO_END ; s++) {
Since you're testing the passing case, why not test all three at once? I.e. hammer KVM while also consuming less test cycles. The "MSR switch" test already verifies the correctness of each list.
x86/vmx_tests.c static int invalid_msr_entry_failure(struct vmentry_failure *failure) · line 8680
Quoted 7 lines
x86/vmx_tests.c +7
-0+struct vmx_msr_entry *msr_list; +int count =max_msr_list_size (); + +switch (s) { +case VM_ENTER_LOAD : + msr_list = (struct vmx_msr_entry *)vmcs_read ( +ENTER_MSR_LD_ADDR );
These should use phys_to_virt() since virt_to_phys() is used to write them.
Quoted 5 lines
x86/vmx_tests.c +5
-0+break ; +case VM_EXIT_LOAD : + msr_list = (struct vmx_msr_entry *)vmcs_read ( +EXIT_MSR_LD_ADDR ); +break ;
x86/vmx_tests.c static int invalid_msr_entry_failure(struct vmentry_failure *failure) · line 8695
Quoted 10 lines
x86/vmx_tests.c +10
-0+case VM_EXIT_STORE : + msr_list = (struct vmx_msr_entry *)vmcs_read ( +EXIT_MSR_ST_ADDR ); +break ; +default : +report ("Bad test scenario, %d." , false, s); +continue ; + } + +configure_atomic_switch_msr_limit_test (msr_list, count);
Again, feeding the list into a helper that also iterates over the lists is not intuitive in terms of understanding what is being tested.
Quoted 18 lines
x86/vmx_tests.c +14
-0+enter_guest (); +assert_exit_reason (VMX_VMCALL ); + } + +/* Reset the atomic MSR switch count to 0 for all three lists. */ +configure_atomic_switch_msr_limit_test (0 ,0 ); +/* Proceed past guest's single vmcall instruction. */ +enter_guest (); +skip_exit_vmcall (); +/* Terminate the guest. */ +enter_guest (); +skip_exit_vmcall (); +} +#define TEST (name) { #name, .v2 = name }
x86/vmx_tests.c struct vmx_test vmx_tests[] = { · line 8801
Quoted 5 lines
x86/vmx_tests.c +2
-0@@ -8660,5 +8797,7 @@ struct vmx_test vmx_tests[] = {TEST (ept_access_test_paddr_read_execute_ad_enabled),TEST (ept_access_test_paddr_not_present_page_fault),TEST (ept_access_test_force_2m_page), +/* Atomic MSR switch tests. */ + TEST(atomic_switch_msr_limit_test),
This is a misleading name, e.g. it took me quite a while to realize this is testing only the passing scenario. For me, "limit test" implies that it'd be deliberately exceeding the limit, or at least testing both the passing and failing cases. I suppose we can't easily test the VMX abort cases, but we can at least test VM_ENTER_LOAD.
Distilling things down to the bare minimum yields something like the following.
struct vmx_msr_entry *vm_enter_load; struct vmx_msr_entry *vm_exit_load; struct vmx_msr_entry *vm_exit_store; u32 i, max_allowed; max_allowed = max_msr_list_size(); /* Setup atomic MSR switch lists. */ vm_enter_load = alloc_pages(msr_list_page_order); vm_exit_load = alloc_pages(msr_list_page_order); vm_exit_store = alloc_pages(msr_list_page_order); populate_msr_list(vm_enter_load, max_allowed + 1); populate_msr_list(vm_exit_load, max_allowed + 1); populate_msr_list(vm_exit_store, max_allowed + 1); vmcs_write(ENTER_MSR_LD_ADDR, virt_to_phys(vm_enter_load)); vmcs_write(EXIT_MSR_LD_ADDR, virt_to_phys(vm_exit_load)); vmcs_write(EXIT_MSR_ST_ADDR, virt_to_phys(vm_exit_store)); /* * VM-Enter should fail when exceeing max number of MSRs. The VM-Exit * lists cause VMX abort, so those are currently not tested. */ vmcs_write(ENT_MSR_LD_CNT, max_allowed + 1); vmcs_write(EXI_MSR_LD_CNT, max_allowed); vmcs_write(EXI_MSR_ST_CNT, max_allowed); helper_function_to_verify_vm_fail(); /* * VM-Enter should succeed up to the max number of MSRs per list, and * should not consume junk beyond the last entry. */ vmcs_write(ENT_MSR_LD_CNT, max_allowed); /* This pointer arithmetic is probably wrong. */ memset(vm_enter_load + max_allowed + 1, 0xff, sizeof(*vm_enter_load); memset(vm_exit_load + max_allowed + 1, 0xff, sizeof(*vm_exit_load); memset(vm_exit_store + max_allowed + 1, 0xff, sizeof(*vm_exit_store); enter_guest(); assert_exit_reason(VMX_VMCALL); <clean up>
Quoted 5 lines
x86/vmx_tests.c +0
-0{ NULL, NULL, NULL, NULL, NULL, {0} }, };Signature · 1 line
2.23.0.237.gc6a4ce50a0-goog
Quoted 5 lines · Sean Christopherson
This is a misleading name, e.g. it took me quite a while to realize this is testing only the passing scenario. For me, "limit test" implies that it'd be deliberately exceeding the limit, or at least testing both the passing and failing cases. I suppose we can't easily test the VMX abort cases, but we can at least test
VM_ENTER_LOAD.
It's hard to test for "undefined behavior may result." :-)
One could check to see if the test is running under KVM, and then check for the behavior that Marc's other patch introduces, but even that is implementation-dependent.
Quoted 10 lines · Jim Mattson
Quoted 5 lines · Sean Christopherson
This is a misleading name, e.g. it took me quite a while to realize this is testing only the passing scenario. For me, "limit test" implies that it'd be deliberately exceeding the limit, or at least testing both the passing and failing cases. I suppose we can't easily test the VMX abort cases, but we can at least test
VM_ENTER_LOAD.It's hard to test for "undefined behavior may result." :-)
Fortune favors the bold?
Quoted 3 lines
One could check to see if the test is running under KVM, and then check for the behavior that Marc's other patch introduces, but even that is implementation-dependent.
Hmm, what if kvm-unit-tests accepts both VM-Enter success and fail as passing conditions? We can at least verify KVM doesn't explode, and if VM-Enter fails, that the exit qual is correct.
The SDM state that the max is a recommendation, which leads me to believe that bare metal will work just fine if the software exceeds the recommended max by an entry or two, but can run into trouble if the list is ludicrously big. There's no way the CPU is tuned so finely that it works at N but fails at N+1.
Quoted 23 lines · Sean Christopherson
Quoted 10 lines · Jim Mattson
Quoted 5 lines · Sean Christopherson
This is a misleading name, e.g. it took me quite a while to realize this is testing only the passing scenario. For me, "limit test" implies that it'd be deliberately exceeding the limit, or at least testing both the passing and failing cases. I suppose we can't easily test the VMX abort cases, but we can at least test
VM_ENTER_LOAD.It's hard to test for "undefined behavior may result." :-)
Fortune favors the bold?
Quoted 3 lines
One could check to see if the test is running under KVM, and then check for the behavior that Marc's other patch introduces, but even that is implementation-dependent.
Hmm, what if kvm-unit-tests accepts both VM-Enter success and fail as passing conditions? We can at least verify KVM doesn't explode, and if VM-Enter fails, that the exit qual is correct.
Quoted 5 lines
The SDM state that the max is a recommendation, which leads me to believe that bare metal will work just fine if the software exceeds the recommended max by an entry or two, but can run into trouble if the list is ludicrously big. There's no way the CPU is tuned so finely that it works at N but fails at N+1.
It would have been nice if the hardware designers had seen fit to architect a hard failure at N+1, but I guess that ship sailed long ago. :-(
Quoted 6 lines
x86/vmx_tests.c static void ept_reserved_bit(int bit) · line 2724
Quoted 4 lines
x86/vmx_tests.c +4
-0+static void *alloc_2m_page (void ) +{ +return alloc_pages (PAGE_2M_ORDER ); +}Allocating 2mb pages is complete overkill. The absolute theoretical max
Quoted 8 lines
for the number of MSRs is (8 * 512) = 4096, for a total of 32kb per list. We can even show the math so that it's obvious how the size is calculated. Plus one order so we can test overrun.
/*
* The max number of MSRs is specified in 3 bits bits, plus 1. I.e. 7+1==8.
* Allocate 64k bytes of data to covermax_msr_list_sizeand then some. */
static const u32msr_list_page_order= 4;
8 * 512 should be 16 * 512, right [1]? Therefore, the maximum theoretical is 64 kB.
I'll happily apply what you've suggested in v2. But I don't see why it's so terrible to over-allocate here. Leveraging a generic 2 MB page allocator can be reused going forward, and encourages uniformity across tests.
[1]
struct vmx_msr_entry {
u32 index;
u32 reserved;
u64 value;
} __aligned(16);
Quoted 23 lines
x86/vmx_tests.c static int invalid_msr_entry_failure(struct vmentry_failure *failure) · line 8597
Quoted 8 lines
x86/vmx_tests.c +8
-0+static void populate_msr_list (struct vmx_msr_entry *msr_list,int count) +{ +int i; + +for (i =0 ; i < count; i++) { + msr_list[i].index =MSR_IA32_TSC ; + msr_list[i].reserved =0 ; + msr_list[i].value =0x1234567890abcdef ;Maybe overkill, but we can use a fast string op for this. I think I got the union right?
static void
populate_msr_list(struct vmx_msr_entry *msr_list, int count){ union { struct vmx_msr_entry msr; u64 val; } tmp; tmp.msr.index = MSR_IA32_TSC; tmp.msr.reserved = 0; tmp.msr.value = 0x1234567890abcdef;
Quoted 7 lines
asm volatile (
"rep stosq\n\t"
: "=c"(count), "=D"(msr_list)
: "a"(tmp.val), "c"(count), "D"(msr_list)
: "memory"
);
}:-). I do think it's overkill. However, I'm OK to apply this suggestion in v2 if everyone is OK with me adding a comment that explains it in terms of the original code, so that x86 noobs, like myself, can understand what's going on.
Quoted 8 lines
This is a misleading name, e.g. it took me quite a while to realize this is testing only the passing scenario. For me, "limit test" implies that it'd be deliberately exceeding the limit, or at least testing both the passing and failing cases. I suppose we can't easily test the VMX abort cases, but we can at least test
VM_ENTER_LOAD.Distilling things down to the bare minimum yields something like the following.
Looks excellent overall. Still not clear what the consensus is on whether or not to test the VM-entry failure. I think a flag seems like a reasonable compromise. I've never added a flag to a kvm-unit-test, so I'll see if I can figure that out.
Quoted 15 lines · Marc Orr
Quoted 6 lines
x86/vmx_tests.c static void ept_reserved_bit(int bit) · line 2724
Quoted 4 lines
x86/vmx_tests.c +4
-0+static void *alloc_2m_page (void ) +{ +return alloc_pages (PAGE_2M_ORDER ); +}Allocating 2mb pages is complete overkill. The absolute theoretical max
Quoted 8 lines
for the number of MSRs is (8 * 512) = 4096, for a total of 32kb per list. We can even show the math so that it's obvious how the size is calculated. Plus one order so we can test overrun.
/*
* The max number of MSRs is specified in 3 bits bits, plus 1. I.e. 7+1==8.
* Allocate 64k bytes of data to covermax_msr_list_sizeand then some. */
static const u32msr_list_page_order= 4;
Quoted 2 lines
8 * 512 should be 16 * 512, right [1]? Therefore, the maximum theoretical is 64 kB.
*sigh* I stand by my assertion that "Math is hard". :-)
Quoted 4 lines
I'll happily apply what you've suggested in v2. But I don't see why it's so terrible to over-allocate here. Leveraging a generic 2 MB page allocator can be reused going forward, and encourages uniformity across tests.
My main concern is avoiding setting 6mb+ of memory. I like to run the tests in L1 and L2 and would prefer to keep overhead at a minimum.
As for the allocation itself, being precise in the allocation size is a form of documentation, e.g. it conveys that the size/order was chosen to ensure enough space for the maximum theoretical list size. A purely arbitrary size, especially one that corresponds with a large page size, can lead to people looking for things that don't exist, e.g. the 2mb size is partially what led me to believe that this test was deliberately exceeding the limit, otherwise why allocate such a large amount of memory? I also didn't know if 2mb was sufficient to handle the maximum theoretical list size.
Quoted 38 lines
[1]
structvmx_msr_entry{
u32 index;
u32 reserved;
u64 value;
}__aligned(16);Quoted 23 lines
x86/vmx_tests.c static int invalid_msr_entry_failure(struct vmentry_failure *failure) · line 8597
Quoted 8 lines
x86/vmx_tests.c +8
-0+static void populate_msr_list (struct vmx_msr_entry *msr_list,int count) +{ +int i; + +for (i =0 ; i < count; i++) { + msr_list[i].index =MSR_IA32_TSC ; + msr_list[i].reserved =0 ; + msr_list[i].value =0x1234567890abcdef ;Maybe overkill, but we can use a fast string op for this. I think I got the union right?
static void
populate_msr_list(struct vmx_msr_entry *msr_list, int count){ union { struct vmx_msr_entry msr; u64 val; } tmp; tmp.msr.index = MSR_IA32_TSC; tmp.msr.reserved = 0; tmp.msr.value = 0x1234567890abcdef;Quoted 7 lines
asm volatile ( "rep stosq\n\t" : "=c"(count), "=D"(msr_list) : "a"(tmp.val), "c"(count), "D"(msr_list) : "memory" ); }
Quoted 4 lines
:-). I do think it's overkill. However, I'm OK to apply this suggestion in v2 if everyone is OK with me adding a comment that explains it in terms of the original code, so that x86 noobs, like myself, can understand what's going on.
Never mind, I can't count. This only works if the size of the entry is 8 bytes.
Quoted 6 lines
Quoted 6 lines
This is a misleading name, e.g. it took me quite a while to realize this is testing only the passing scenario. For me, "limit test" implies that it'd be deliberately exceeding the limit, or at least testing both the passing and failing cases. I suppose we can't easily test the VMX abort cases, but we can at least test
VM_ENTER_LOAD.
Quoted 7 lines
Quoted 2 lines
Distilling things down to the bare minimum yields something like the following.
Looks excellent overall. Still not clear what the consensus is on whether or not to test the VM-entry failure. I think a flag seems like a reasonable compromise. I've never added a flag to a kvm-unit-test, so I'll see if I can figure that out.
No need for a flag if you want to go that route, just put it in a separate VMX subtest and exclude said test from the [vmx] config, i.e. make the test opt-in.
Thanks for the review! I'll get to work on v2 now.
Quoted 8 lines
Quoted 4 lines
I'll happily apply what you've suggested in v2. But I don't see why it's so terrible to over-allocate here. Leveraging a generic 2 MB page allocator can be reused going forward, and encourages uniformity across tests.
My main concern is avoiding setting 6mb+ of memory. I like to run the tests in L1 and L2 and would prefer to keep overhead at a minimum.
Quoted 9 lines
As for the allocation itself, being precise in the allocation size is a form of documentation, e.g. it conveys that the size/order was chosen to ensure enough space for the maximum theoretical list size. A purely arbitrary size, especially one that corresponds with a large page size, can lead to people looking for things that don't exist, e.g. the 2mb size is partially what led me to believe that this test was deliberately exceeding the limit, otherwise why allocate such a large amount of memory? I also didn't know if 2mb was sufficient to handle the maximum theoretical list size.
SGTM. I'll make this change in v2.
Quoted 11 lines
Quoted 7 lines
Quoted 2 lines
Distilling things down to the bare minimum yields something like the following.
Looks excellent overall. Still not clear what the consensus is on whether or not to test the VM-entry failure. I think a flag seems like a reasonable compromise. I've never added a flag to a kvm-unit-test, so I'll see if I can figure that out.
No need for a flag if you want to go that route, just put it in a separate VMX subtest and exclude said test from the [vmx] config, i.e. make the test opt-in.
SGTM, thanks!
Quoted 10 lines
x86/vmx_tests.c static void ept_reserved_bit(int bit) · line 2724
Quoted 4 lines
x86/vmx_tests.c +4
-0+static void *alloc_2m_page (void ) +{ +return alloc_pages (PAGE_2M_ORDER ); +}Allocating 2mb pages is complete overkill. The absolute theoretical max for the number of MSRs is (8 * 512) = 4096, for a total of 32kb per list. We can even show the math so that it's obvious how the size is calculated. Plus one order so we can test overrun.
Quoted 6 lines
/*
* The max number of MSRs is specified in 3 bits bits, plus 1. I.e. 7+1==8.
* Allocate 64k bytes of data to covermax_msr_list_sizeand then some. */
static const u32msr_list_page_order= 4;
Done. Changed msr_list_page_order to 5, per our previous discussion that you meant 16 * 512.
Quoted 16 lines
x86/vmx_tests.c static int invalid_msr_entry_failure(struct vmentry_failure *failure) · line 8583
Quoted 6 lines
x86/vmx_tests.c +6
-0+enum atomic_switch_msr_scenario { +VM_ENTER_LOAD , +VM_EXIT_LOAD , +VM_EXIT_STORE , +ATOMIC_SWITCH_MSR_SCENARIO_END , +};How about:
enum
atomic_switch_msr_lists{
VM_ENTER_LOAD,
VM_EXIT_LOAD,
VM_EXIT_STORE,
NR_ATOMIC_SWITCH_MSR_LISTS,
};
Quoted 6 lines
IMO that yields a much more intuitive test loop:
for (i = 0; i < NR_ATOMIC_SWITCH_MSR_LISTS; i++) { }But we probably don't even need a loop...
Ack. Got rid of the loop.
Quoted 28 lines
x86/vmx_tests.c static int invalid_msr_entry_failure(struct vmentry_failure *failure) · line 8597
Quoted 13 lines
x86/vmx_tests.c +13
-0+static void atomic_switch_msr_limit_test_guest (void ) +{ +vmcall (); +} + +static void populate_msr_list (struct vmx_msr_entry *msr_list,int count) +{ +int i; + +for (i =0 ; i < count; i++) { + msr_list[i].index =MSR_IA32_TSC ; + msr_list[i].reserved =0 ; + msr_list[i].value =0x1234567890abcdef ;Maybe overkill, but we can use a fast string op for this. I think I got the union right?
static void
populate_msr_list(struct vmx_msr_entry *msr_list, int count){ union { struct vmx_msr_entry msr; u64 val; } tmp; tmp.msr.index = MSR_IA32_TSC; tmp.msr.reserved = 0; tmp.msr.value = 0x1234567890abcdef;
Quoted 7 lines
asm volatile (
"rep stosq\n\t"
: "=c"(count), "=D"(msr_list)
: "a"(tmp.val), "c"(count), "D"(msr_list)
: "memory"
);
}Skipped per our previous conversation that this doesn't work due to the string being 16 bytes.
Quoted 21 lines
Quoted 10 lines
x86/vmx_tests.c +10
-0+for (s =0 ; s <ATOMIC_SWITCH_MSR_SCENARIO_END ; s++) { +switch (s) { +case VM_ENTER_LOAD : + addr_field =ENTER_MSR_LD_ADDR ; + cnt_field =ENT_MSR_LD_CNT ; +break ; +case VM_EXIT_LOAD : + addr_field =EXIT_MSR_LD_ADDR ; + cnt_field =EXI_MSR_LD_CNT ; +break ;x86/vmx_tests.c static int invalid_msr_entry_failure(struct vmentry_failure *failure) · line 8629
Quoted 10 lines
x86/vmx_tests.c +10
-0+case VM_EXIT_STORE : + addr_field =EXIT_MSR_ST_ADDR ; + cnt_field =EXI_MSR_ST_CNT ; +break ; +default : +TEST_ASSERT (false); + } + + msr_list = (struct vmx_msr_entry *)vmcs_read (addr_field); +memset (msr_list,0xff , two_mb);
Quoted 4 lines
Writing 8mb of data for each test is a waste of time, i.e. 6mb to reset each list, and another 2mb to populate the target list.
The for-loop in the helper is also confusing and superfluous.
Ack. Got rid of the helper.
Quoted 10 lines
x86/vmx_tests.c static int invalid_msr_entry_failure(struct vmentry_failure *failure) · line 8670
Quoted 7 lines
x86/vmx_tests.c +7
-0+/* Setup atomic MSR switch lists. */ + msr_list =alloc_2m_page (); +vmcs_write (ENTER_MSR_LD_ADDR ,virt_to_phys (msr_list)); + msr_list =alloc_2m_page (); +vmcs_write (EXIT_MSR_LD_ADDR ,virt_to_phys (msr_list)); + msr_list =alloc_2m_page (); +vmcs_write (EXIT_MSR_ST_ADDR ,virt_to_phys (msr_list));This memory should really be freed. Not holding pointers for each list also seems silly, e.g. requires a VMREAD just to get a pointer.
Done.
Quoted 7 lines
x86/vmx_tests.c static int invalid_msr_entry_failure(struct vmentry_failure *failure) · line 8673
Quoted 3 lines
x86/vmx_tests.c +3
-0+ +/* Execute each test case. */ +for (s =0 ; s <ATOMIC_SWITCH_MSR_SCENARIO_END ; s++) {Since you're testing the passing case, why not test all three at once? I.e. hammer KVM while also consuming less test cycles. The "MSR switch" test already verifies the correctness of each list.
Done.
Quoted 9 lines
x86/vmx_tests.c static int invalid_msr_entry_failure(struct vmentry_failure *failure) · line 8680
Quoted 7 lines
x86/vmx_tests.c +7
-0+struct vmx_msr_entry *msr_list; +int count =max_msr_list_size (); + +switch (s) { +case VM_ENTER_LOAD : + msr_list = (struct vmx_msr_entry *)vmcs_read ( +ENTER_MSR_LD_ADDR );These should use
phys_to_virt()sincevirt_to_phys()is used to write them.
Hmm. Actually, why don't we just use an explicit (u64) cast. That's what I originally had, but then when Jim pre-reviewed the patch before I posted it on the list he suggested virt_to_phys() with a ?. I didn't really understand why that was better than the explicit cast. And your suggestion to use phys_to_virt() doesn't work at all because it returns a pointer rather than a u64.
Quoted 16 lines
Quoted 5 lines
x86/vmx_tests.c +5
-0+break ; +case VM_EXIT_LOAD : + msr_list = (struct vmx_msr_entry *)vmcs_read ( +EXIT_MSR_LD_ADDR ); +break ;x86/vmx_tests.c static int invalid_msr_entry_failure(struct vmentry_failure *failure) · line 8695
Quoted 10 lines
x86/vmx_tests.c +10
-0+case VM_EXIT_STORE : + msr_list = (struct vmx_msr_entry *)vmcs_read ( +EXIT_MSR_ST_ADDR ); +break ; +default : +report ("Bad test scenario, %d." , false, s); +continue ; + } + +configure_atomic_switch_msr_limit_test (msr_list, count);
Quoted 2 lines
Again, feeding the list into a helper that also iterates over the lists is not intuitive in terms of understanding what is being tested.
Done. Got rid of the loop.
Quoted 8 lines
x86/vmx_tests.c struct vmx_test vmx_tests[] = { · line 8801
Quoted 2 lines
x86/vmx_tests.c +2
-0+/* Atomic MSR switch tests. */ + TEST(atomic_switch_msr_limit_test),This is a misleading name, e.g. it took me quite a while to realize this is testing only the passing scenario. For me, "limit test" implies that it'd be deliberately exceeding the limit, or at least testing both the passing and failing cases. I suppose we can't easily test the VMX abort cases, but we can at least test
VM_ENTER_LOAD.
Done. Renamed to atomic_switch_max_msrs_test.
Quoted 2 lines
Distilling things down to the bare minimum yields something like the following.
Thanks!