s390x: test storage keys during migration
v3 cover s390x: test storage keys during migration
v2->v3:
---
* remove some now useless arguments to get shorter function signatures (thanks Claudio)
* fix barriers (thanks Nina)
* improve command line arg parser (thanks Claudio)
* use posix-style arguments (thanks Claudio)
* factor out argument parsing into own function (thanks Claudio)
* cleanup includes a bit (thanks Nina)
v1->v2:
---
* remove the skey library and move both versions of the skey migration test to a single file
* rename skey_set_keys/verify_keys to set_test_pattern/verify_test_pattern
* add a few comments
Add a test which changes storage keys while VM is being migrated.
This series bases on Claudio's new PSW macros ("[PATCH v3 0/2] lib:
s390x: add PSW and PSW_WITH_CUR_MASK macros").
Nico Boehr (1):
s390x: add parallel skey migration test
s390x/migration-skey.c | 218 +++++++++++++++++++++++++++++++++++++---- s390x/unittests.cfg | 15 ++- 2 files changed, 210 insertions(+), 23 deletions(-)
Signature · 1 line
2.36.1
PATCH v3 1/1 s390x: add parallel skey migration test
Right now, we have a test which sets storage keys, then migrates the VM and - after migration finished - verifies the skeys are still there.
Add a new version of the test which changes storage keys while the migration is in progress. This is achieved by adding a command line argument to the existing migration-skey test.
--- s390x/migration-skey.c | 218 +++++++++++++++++++++++++++++++++++++---- s390x/unittests.cfg | 15 ++- 2 files changed, 210 insertions(+), 23 deletions(-)
s390x/migration-skey.c +199 -19
@@ -2,6 +2,12 @@/* * Storage Key migration tests * +* There are two variants of this test: +* - sequential: sets some storage keys on pages, migrates the VM and then +* verifies the storage keys are still as we expect. +* - parallel: start migration of a VM and set and check storage keys on some +* pages while migration is in process. +* * Copyright IBM Corp. 2022 * * Authors:
@@ -13,16 +19,45 @@#include <asm/facility.h> #include <asm/page.h> #include <asm/mem.h> -#include <asm/interrupt.h> +#include <asm/barrier.h> #include <hardware.h> +#include <smp.h> + +struct verify_result { +bool verify_failed; +union skey expected_key; +union skey actual_key; +unsigned long page_mismatch_idx; +unsigned long page_mismatch_addr; +};#define NUM_PAGES 128 -static uint8_t pagebuf[NUM_PAGES][PAGE_SIZE] __attribute__((aligned(PAGE_SIZE))); +static uint8_t pagebuf[NUM_PAGES * PAGE_SIZE] __attribute__((aligned(PAGE_SIZE))); + +static struct verify_result result; + +static unsigned int thread_iters; +static bool thread_should_exit; +static bool thread_exited; + +static enum { +TEST_INVALID , +TEST_SEQUENTIAL , +TEST_PARALLEL +} arg_test_to_run; -static void test_migration (void ) +/* +* Set storage key test pattern on pagebuf with a seed for the storage keys. +* +* Each page's storage key is generated by taking the page's index in pagebuf, +* XOR-ing that with the given seed and then multipling the result with two. +* +* Only the lower seven bits of the seed are considered. +*/ +static void set_test_pattern (unsigned char seed) { -union skey expected_key, actual_key; -int i, key_to_set, key_mismatches =0 ; +unsigned char key_to_set; +unsigned long i;for (i =0 ; i <NUM_PAGES ; i++) {/*
@@ -32,15 +67,34 @@ static void test_migration(void)* protection as well as reference and change indication for * some keys. */ - key_to_set = i *2 ; -set_storage_key (pagebuf[i], key_to_set,1 ); + key_to_set = (i ^ seed) *2 ; +set_storage_key (pagebuf + i *PAGE_SIZE , key_to_set,1 ); } +} -migrate_once (); +/* +* Verify storage keys on pagebuf. +* Storage keys must have been set by set_test_pattern on pagebuf before. +* set_test_pattern must have been called with the same seed value. +* +* If storage keys match the expected result, will return a verify_result +* with verify_failed false. All other fields are then invalid. +* If there is a mismatch, returned struct will have verify_failed true and will +* be filled with the details on the first mismatch encountered. +*/ +static struct verify_result verify_test_pattern (unsigned char seed) +{ +union skey expected_key, actual_key; +struct verify_result result = { + .verify_failed = true + }; +uint8_t *cur_page; +unsigned long i;for (i =0 ; i <NUM_PAGES ; i++) { - actual_key.val =get_storage_key (pagebuf[i]); - expected_key.val = i *2 ; + cur_page = pagebuf + i *PAGE_SIZE ; + actual_key.val =get_storage_key (cur_page); + expected_key.val = (i ^ seed) *2 ;/* * The PoP neither gives a guarantee that the reference bit is
@@ -51,27 +105,153 @@ static void test_migration(void) actual_key.str.rf =0 ; expected_key.str.rf =0 ; -/* don't log anything when key matches to avoid spamming the log */ if (actual_key.val != expected_key.val) { - key_mismatches++; -report_fail ("page %d expected_key=0x%x actual_key=0x%x" , i, expected_key.val, actual_key.val); + result.expected_key.val = expected_key.val; + result.actual_key.val = actual_key.val; + result.page_mismatch_idx = i; + result.page_mismatch_addr = (unsigned long )cur_page; +return result; } } -report (!key_mismatches,"skeys after migration match" ); + result.verify_failed = false; +return result; +} + +static void report_verify_result (struct verify_result *const result) +{ +if (result->verify_failed) +report_fail ("page skey mismatch: first page idx = %lu, addr = 0x%lx, " +"expected_key = 0x%x, actual_key = 0x%x" , + result->page_mismatch_idx, result->page_mismatch_addr, + result->expected_key.val, result->actual_key.val); +else +report_pass ("skeys match" ); +} + +static void test_skey_migration_sequential (void ) +{ +report_prefix_push ("sequential" ); + +set_test_pattern (0 ); + +migrate_once (); + + result =verify_test_pattern (0 ); +report_verify_result (&result); + +report_prefix_pop (); +} + +static void set_skeys_thread (void ) +{ +while (!READ_ONCE (thread_should_exit)) { +set_test_pattern (thread_iters); + + result =verify_test_pattern (thread_iters); + +/* +* Always increment even if the verify fails. This ensures primary CPU knows where +* we left off and can do an additional verify round after migration finished. +*/ + thread_iters++; + +if (result.verify_failed) +break ; + } + +WRITE_ONCE (thread_exited,1 ); +} + +static void test_skey_migration_parallel (void ) +{ +report_prefix_push ("parallel" ); + +if (smp_query_num_cpus () ==1 ) { +report_skip ("need at least 2 cpus for this test" ); + goto error; + } + +smp_cpu_setup (1 ,PSW_WITH_CUR_MASK (set_skeys_thread)); + +migrate_once (); + +WRITE_ONCE (thread_should_exit,1 ); + +while (!READ_ONCE (thread_exited)) + ; + +/* Ensure we read result and thread_iters below from memory after thread exited */ +mb (); +report_info ("thread completed %u iterations" , thread_iters); + +report_prefix_push ("during migration" ); +report_verify_result (&result); +report_prefix_pop (); + +/* +* Verification of skeys occurs on the thread. We don't know if we +* were still migrating during the verification. +* To be sure, make another verification round after the migration +* finished to catch skeys which might not have been migrated +* correctly. +*/ +report_prefix_push ("after migration" ); +assert (thread_iters >0 ); + result =verify_test_pattern (thread_iters -1 ); +report_verify_result (&result); +report_prefix_pop (); + +error: +report_prefix_pop (); +} + +static void print_usage (void ) +{ +report_info ("Usage: migration-skey [parallel|sequential]" ); +} + +static void parse_args (int argc,char **argv) +{ +if (argc <2 ) { +/* default to sequential since it only needs one cpu */ + arg_test_to_run =TEST_SEQUENTIAL ; +return ; + } + +if (!strcmp ("--parallel" , argv[1 ])) + arg_test_to_run =TEST_PARALLEL ; +else if (!strcmp ("--sequential" , argv[1 ])) + arg_test_to_run =TEST_SEQUENTIAL ; +else + arg_test_to_run =TEST_INVALID ; } -int main (void ) +int main (int argc,char **argv) {report_prefix_push ("migration-skey" ); -if (test_facility (169 )) +if (test_facility (169 )) {report_skip ("storage key removal facility is active" ); -else -test_migration (); + goto error; + } -migrate_once (); +parse_args (argc, argv); + +switch (arg_test_to_run) { +case TEST_SEQUENTIAL : +test_skey_migration_sequential (); +break ; +case TEST_PARALLEL : +test_skey_migration_parallel (); +break ; +default : +print_usage (); +break ; + } +error: +migrate_once ();report_prefix_pop ();return report_summary (); }
s390x/unittests.cfg +11 -4
@@ -185,10 +185,6 @@ smp = 2 file = migration-cmm.elf groups = migration -[migration-skey] -file = migration-skey.elf -groups = migration - [panic-loop-extint] file = panic-loop-extint.elf groups = panic
@@ -208,3 +204,14 @@ groups = migration [exittime] file = exittime.elf smp = 2 + +[migration-skey-sequential] +file = migration-skey.elf +groups = migration +extra_params = -append '--sequential' + +[migration-skey-parallel] +file = migration-skey.elf +smp = 2 +groups = migration +extra_params = -append '--parallel'
Signature · 1 line
2.36.1
Re: [kvm-unit-tests PATCH v3 1/1] s390x: add parallel skey migration test
Quoted 6 lines · On Wed, 14 Dec 2022 13:38:14 +0100 Nico Boehr <[email protected]> wrote:
Right now, we have a test which sets storage keys, then migrates the VM and - after migration finished - verifies the skeys are still there.
Add a new version of the test which changes storage keys while the migration is in progress. This is achieved by adding a command line argument to the existing migration-skey test.
just some small cosmetic issues
Quoted 12 lines
Signed-off-by Nico Boehr <[email protected]>--- s390x/migration-skey.c | 218 +++++++++++++++++++++++++++++++++++++---- s390x/unittests.cfg | 15 ++- 2 files changed, 210 insertions(+), 23 deletions(-)s390x/migration-skey.c +0
-0
s390x/migration-skey.c line 6
Quoted 5 lines
s390x/migration-skey.c +2
-0@@ -2,6 +2,12 @@/* * Storage Key migration tests * +* There are two variants of this test: +* - sequential: sets some storage keys on pages, migrates the VM and then
"set storage keys on some pages"?
or just "set some storage keys"
s390x/migration-skey.c line 7
Quoted 1 line
s390x/migration-skey.c +1
-0+* verifies the storage keys are still as we expect.
s/the/that the/
s390x/migration-skey.c line 8
Quoted 1 line
s390x/migration-skey.c +1
-0+* - parallel: start migration of a VM and set and check storage keys on some
s/of a VM/of VM/ or s/of a VM/of the VM/ or even just s/of the VM//
Quoted 14 lines
s390x/migration-skey.c +5
-1+* pages while migration is in process. +* * Copyright IBM Corp. 2022 * * Authors: @@ -13,16 +19,45 @@#include <asm/facility.h> #include <asm/page.h> #include <asm/mem.h> -#include <asm/interrupt.h> +#include <asm/barrier.h> #include <hardware.h> +#include <smp.h> +
[...]
s390x/migration-skey.c static void test_migration(void) · line 211
Quoted 4 lines
s390x/migration-skey.c +4
-0+ +static void print_usage (void ) +{ +report_info ("Usage: migration-skey [parallel|sequential]" );
--parallel and --sequential
Quoted 2 lines
s390x/migration-skey.c +2
-0+} +
[...]
Re: [kvm-unit-tests PATCH v3 1/1] s390x: add parallel skey migration test
In reply to Nico BoehrQuoted 8 lines · Nico Boehr
Right now, we have a test which sets storage keys, then migrates the VM and - after migration finished - verifies the skeys are still there.
Add a new version of the test which changes storage keys while the migration is in progress. This is achieved by adding a command line argument to the existing migration-skey test.
Signed-off-by Nico Boehr <[email protected]>
Indentation should be fixed IMO, feel free to ignore the rest.
Quoted 10 lines
--- s390x/migration-skey.c | 218 +++++++++++++++++++++++++++++++++++++---- s390x/unittests.cfg | 15 ++- 2 files changed, 210 insertions(+), 23 deletions(-)s390x/migration-skey.c +0
-0
[...]
Quoted 29 lines
s390x/migration-skey.c +21
-2+/* +* Verify storage keys on pagebuf. +* Storage keys must have been set by set_test_pattern on pagebuf before. +* set_test_pattern must have been called with the same seed value. +* +* If storage keys match the expected result, will return a verify_result +* with verify_failed false. All other fields are then invalid. +* If there is a mismatch, returned struct will have verify_failed true and will +* be filled with the details on the first mismatch encountered. +*/ +static struct verify_result verify_test_pattern (unsigned char seed) +{ +union skey expected_key, actual_key; +struct verify_result result = { + .verify_failed = true + }; +uint8_t *cur_page; +unsigned long i;for (i =0 ; i <NUM_PAGES ; i++) { - actual_key.val =get_storage_key (pagebuf[i]); - expected_key.val = i *2 ; + cur_page = pagebuf + i *PAGE_SIZE ; + actual_key.val =get_storage_key (cur_page); + expected_key.val = (i ^ seed) *2 ;/* * The PoP neither gives a guarantee that the reference bit is
s390x/migration-skey.c static void test_migration(void) · line 57
Quoted 7 lines
s390x/migration-skey.c +0
-3@@ -51,27 +105,153 @@ static void test_migration(void) actual_key.str.rf =0 ; expected_key.str.rf =0 ; -/* don't log anything when key matches to avoid spamming the log */ if (actual_key.val != expected_key.val) { - key_mismatches++; -report_fail ("page %d expected_key=0x%x actual_key=0x%x" , i, expected_key.val, actual_key.val);
I feel like setting verify_failed here also would be nicer. Could also do
return (struct verify_result) {
...
}Just a suggestion.
Quoted 4 lines
s390x/migration-skey.c +4
-0+ result.expected_key.val = expected_key.val; + result.actual_key.val = actual_key.val; + result.page_mismatch_idx = i; + result.page_mismatch_addr = (unsigned long )cur_page;
s390x/migration-skey.c static void test_migration(void) · line 121
Quoted 10 lines
s390x/migration-skey.c +6
-1+return result; } } -report (!key_mismatches,"skeys after migration match" ); + result.verify_failed = false; +return result; +} + +static void report_verify_result (struct verify_result *const result)
Why const? Why not also pointer to const?
s390x/migration-skey.c static void test_migration(void) · line 125
Quoted 4 lines
s390x/migration-skey.c +4
-0+{ +if (result->verify_failed) + report_fail("page skey mismatch: first page idx = %lu, addr = 0x%lx, " + "expected_key = 0x%x, actual_key = 0x%x",
Indent is off here.
I have a slight preference for %02x for the keys. Just a suggestion.
Quoted 7 lines
s390x/migration-skey.c +6
-0+ result->page_mismatch_idx, result->page_mismatch_addr, + result->expected_key.val, result->actual_key.val); + else + report_pass("skeys match"); +} +
[...]
Quoted 7 lines
s390x/migration-skey.c +2
-2-int main (void ) +int main (int argc,char **argv) {report_prefix_push ("migration-skey" ); -if (test_facility (169 )) +if (test_facility (169 )) {
s390x/migration-skey.c static void test_migration(void) · line 241
Quoted 10 lines
s390x/migration-skey.c +5
-3report_skip ("storage key removal facility is active" ); - else -test_migration (); + goto error; + } -migrate_once (); +parse_args (argc, argv); + +switch (arg_test_to_run) {
break statements should be indented.
Quoted 16 lines
s390x/migration-skey.c +12
-0+case TEST_SEQUENTIAL : +test_skey_migration_sequential (); +break ; +case TEST_PARALLEL : +test_skey_migration_parallel (); +break ; +default : +print_usage (); +break ; + } +error: +migrate_once ();report_prefix_pop ();return report_summary (); }
[...]
Re: [kvm-unit-tests PATCH v3 1/1] s390x: add parallel skey migration test
Quoting Nina Schoetterl-Glausch (2022-12-14 19:27:15)
Quoted 8 lines · Nico Boehr
Quoted 7 lines
Right now, we have a test which sets storage keys, then migrates the VM and - after migration finished - verifies the skeys are still there.
Add a new version of the test which changes storage keys while the migration is in progress. This is achieved by adding a command line argument to the existing migration-skey test.
Quoted 5 lines
Quoted 1 line
Signed-off-by Nico Boehr <[email protected]>Reviewed-by Nina Schoetterl-Glausch <[email protected]>Indentation should be fixed IMO, feel free to ignore the rest.
Thanks!
[...]
Quoted 5 lines
Quoted 5 lines
s390x/migration-skey.c +0
-0
[...]
Quoted 2 lines
Quoted 2 lines
s390x/migration-skey.c +2
-0+static struct verify_result verify_test_pattern (unsigned char seed) +{
[...]
Quoted 6 lines
s390x/migration-skey.c static void test_migration(void) · line 57
Quoted 4 lines
s390x/migration-skey.c +0
-3-/* don't log anything when key matches to avoid spamming the log */ if (actual_key.val != expected_key.val) { - key_mismatches++; -report_fail ("page %d expected_key=0x%x actual_key=0x%x" , i, expected_key.val, actual_key.val);I feel like setting
verify_failedhere also would be nicer.
I had this before, Claudio requested to remove it...
Quoted 5 lines
Could also do
return (struct verify_result) { ... }Just a suggestion.
No strong opinion. I'll do whatever you prefer.
[...]
Quoted 3 lines
Quoted 1 line
+static void
report_verify_result(struct verify_result * const result)Why const? Why not also pointer to const?
Yes right, I'll make this a const struct.
Quoted 6 lines
s390x/migration-skey.c static void test_migration(void) · line 125
Quoted 4 lines
s390x/migration-skey.c +4
-0+{ +if (result->verify_failed) + report_fail("page skey mismatch: first page idx = %lu, addr = 0x%lx, " + "expected_key = 0x%x, actual_key = 0x%x",Indent is off here.
Yes done.
Quoted 1 line
I have a slight preference for %02x for the keys. Just a suggestion.
Yes, changed.
Quoted 15 lines
[...]
Quoted 13 lines
s390x/migration-skey.c +4
-4-int main (void ) +int main (int argc,char **argv) {report_prefix_push ("migration-skey" ); -if (test_facility (169 )) +if (test_facility (169 )) {report_skip ("storage key removal facility is active" ); -else -test_migration (); + goto error; + }
Quoted 6 lines
s390x/migration-skey.c static void test_migration(void) · line 241
Quoted 4 lines
s390x/migration-skey.c +3
-1-migrate_once (); +parse_args (argc, argv); + +switch (arg_test_to_run) {break statements should be indented.
Fixed.