selftests/bpf: Improve the usability of test_progs
Subsystems (3)
v5 cover selftests/bpf: Improve the usability of test_progs
This short series adds two new "-a", "-d" switch to test_progs, supporting exact string match, as well as '*' wildcard. It also cleans up the output to make it possible to generate allowlist/denylist using common cli tools.
Yucong Sun (4):
selftests/bpf: skip loading bpf_testmod when using -l to list tests. selftests/bpf: correctly display subtest skip status selftests/bpf: also print test name in subtest status message selftests/bpf: Support glob matching for test selector.
tools/testing/selftests/bpf/test_progs.c | 101 ++++++++++++++++------- 1 file changed, 73 insertions(+), 28 deletions(-)
Signature · 1 line
2.30.2
PATCH v5 1/4 selftests/bpf: skip loading bpf_testmod when using -l to list tests.
When using "-l", test_progs often is executed as non-root user, load_bpf_testmod() will fail and output errors. This patch skip loading bpf testmod when "-l" is specified, making output cleaner.
--- tools/testing/selftests/bpf/test_progs.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
tools/testing/selftests/bpf/test_progs.c +2 -2
@@ -755,7 +755,7 @@ int main(int argc, char **argv)save_netns ();stdio_hijack (); env.has_testmod = true; -if (load_bpf_testmod ()) { +if (!env.list_test_names &&load_bpf_testmod ()) {fprintf (env.stderr,"WARNING! Selftests relying on bpf_testmod.ko will be skipped.\n" ); env.has_testmod = false; }
@@ -803,7 +803,7 @@ int main(int argc, char **argv)if (test->need_cgroup_cleanup)cleanup_cgroup_environment (); } -if (env.has_testmod) +if (!env.list_test_names && env.has_testmod)unload_bpf_testmod ();stdio_restore ();
Signature · 1 line
2.30.2
PATCH v5 2/4 selftests/bpf: correctly display subtest skip status
In reply to Yucong SunIn skip_account(), test->skip_cnt is set to 0 at the end, this makes next print statement never display SKIP status for the subtest. This patch moves the accounting logic after the print statement, fixing the issue.
This patch also added SKIP status display for normal tests.
--- tools/testing/selftests/bpf/test_progs.c | 25 ++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-)
tools/testing/selftests/bpf/test_progs.c +13 -12
@@ -148,18 +148,18 @@ void test__end_subtest()struct prog_test_def *test = env.test;int sub_error_cnt = test->error_cnt - test->old_error_cnt; -if (sub_error_cnt) - env.fail_cnt++; -else if (test->skip_cnt ==0 ) - env.sub_succ_cnt++; -skip_account (); -dump_test_log (test, sub_error_cnt);fprintf (env.stdout,"#%d/%d %s:%s\n" , test->test_num, test->subtest_num, test->subtest_name, sub_error_cnt ?"FAIL" : (test->skip_cnt ?"SKIP" :"OK" )); +if (sub_error_cnt) + env.fail_cnt++; +else if (test->skip_cnt ==0 ) + env.sub_succ_cnt++; +skip_account (); +free (test->subtest_name); test->subtest_name =NULL ; }
@@ -786,17 +786,18 @@ int main(int argc, char **argv)test__end_subtest (); test->tested = true; -if (test->error_cnt) - env.fail_cnt++; -else - env.succ_cnt++; -skip_account ();dump_test_log (test, test->error_cnt);fprintf (env.stdout,"#%d %s:%s\n" , test->test_num, test->test_name, - test->error_cnt ?"FAIL" :"OK" ); + test->error_cnt ?"FAIL" : (test->skip_cnt ?"SKIP" :"OK" )); + +if (test->error_cnt) + env.fail_cnt++; +else + env.succ_cnt++; +skip_account ();reset_affinity ();restore_netns ();
Signature · 1 line
2.30.2
PATCH v5 3/4 selftests/bpf: also print test name in subtest status message
In reply to Yucong SunThis patch add test name in subtest status message line, making it possible to grep ':OK' in the output to generate a list of passed test+subtest names, which can be processed to generate argument list to be used with "-a", "-d" exact string matching.
Example:
#1/1 align/mov:OK
..
#1/12 align/pointer variable subtraction:OK
#1 align:OK
--- tools/testing/selftests/bpf/test_progs.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
tools/testing/selftests/bpf/test_progs.c +2 -2
@@ -150,8 +150,8 @@ void test__end_subtest()dump_test_log (test, sub_error_cnt); -fprintf (env.stdout,"#%d/%d %s:%s\n" , - test->test_num, test->subtest_num, test->subtest_name, +fprintf (env.stdout,"#%d/%d %s/%s:%s\n" , + test->test_num, test->subtest_num, test->test_name, test->subtest_name, sub_error_cnt ?"FAIL" : (test->skip_cnt ?"SKIP" :"OK" ));if (sub_error_cnt)
Signature · 1 line
2.30.2
PATCH v5 4/4 selftests/bpf: Support glob matching for test selector.
In reply to Yucong SunThis patch adds '-a' and '-d' arguments, support exact string match, as well as using '*' wildcard in test/subtests selection. The old '-t' '-b' arguments still supports partial string match, but they can't be used together yet.
This patach also adds support for mulitple '-a' '-d' '-t' '-b' arguments.
Caveat: Same as the current substring matching mechanism, test and subtest selector applies independently, 'a*/b*' will execute all tests matching "a*", and with subtest name matching "b*", but tests matching "a*" that has no subtests will also be executed.
--- tools/testing/selftests/bpf/test_progs.c | 72 +++++++++++++++++++----- 1 file changed, 58 insertions(+), 14 deletions(-)
tools/testing/selftests/bpf/test_progs.c +58 -14
@@ -13,6 +13,28 @@#include <execinfo.h> /* backtrace */ #include <linux/membarrier.h> +/* Adapted from perf/util/string.c */ +static bool glob_match (const char *str,const char *pat) +{ +while (*str && *pat && *pat !='*' ) { +if (*str != *pat) +return false; + str++; + pat++; + } +/* Check wild card */ +if (*pat =='*' ) { +while (*pat =='*' ) + pat++; +if (!*pat)/* Tail wild card matches all */ +return true; +while (*str) +if (glob_match (str++, pat)) +return true; + } +return !*str && !*pat; +} +#define EXIT_NO_TEST 2#define EXIT_ERR_SETUP_INFRA 3
@@ -55,12 +77,12 @@ static bool should_run(struct test_selector *sel, int num, const char *name)int i;for (i =0 ; i < sel->blacklist.cnt; i++) { -if (strstr (name, sel->blacklist.strs[i])) +if (glob_match (name, sel->blacklist.strs[i]))return false; }for (i =0 ; i < sel->whitelist.cnt; i++) { -if (strstr (name, sel->whitelist.strs[i])) +if (glob_match (name, sel->whitelist.strs[i]))return true; }
@@ -450,6 +472,8 @@ enum ARG_KEYS {ARG_VERBOSE ='v' ,ARG_GET_TEST_CNT ='c' ,ARG_LIST_TEST_NAMES ='l' , +ARG_TEST_NAME_GLOB_ALLOWLIST ='a' , + ARG_TEST_NAME_GLOB_DENYLIST = 'd', }; static const struct argp_option opts[] = {
@@ -467,6 +491,10 @@ static const struct argp_option opts[] = { "Get number of selected top-level tests " }, {"list" ,ARG_LIST_TEST_NAMES ,NULL ,0 , "List test names that would run (without running them) " }, + { "allow", ARG_TEST_NAME_GLOB_ALLOWLIST, "NAMES", 0, + "Run tests with name matching the pattern (support *)." }, + { "deny", ARG_TEST_NAME_GLOB_DENYLIST, "NAMES", 0, + "Don't run tests with name matching the pattern (support *)." }, {}, };
@@ -491,7 +519,7 @@ static void free_str_set(const struct str_set *set)free (set->strs); } -static int parse_str_list (const char *s,struct str_set *set) +static int parse_str_list (const char *s,struct str_set *set,bool is_glob_pattern) {char *input, *state =NULL , *next, **tmp, **strs =NULL ;int cnt =0 ;
@@ -500,28 +528,38 @@ static int parse_str_list(const char *s, struct str_set *set)if (!input)return -ENOMEM ; - set->cnt =0 ; - set->strs =NULL ; -while ((next =strtok_r (state ?NULL : input,"," , &state))) { tmp =realloc (strs,sizeof (*strs) * (cnt +1 ));if (!tmp) goto err; strs = tmp; +if (is_glob_pattern) { + strs[cnt] =strdup (next); + }else { + strs[cnt] =malloc (strlen (next) +2 +1 ); +if (!strs[cnt]) + goto err; +sprintf (strs[cnt],"*%s*" , next); + } - strs[cnt] =strdup (next);if (!strs[cnt]) goto err; cnt++; } - set->cnt = cnt; - set->strs = (const char **)strs; + tmp =realloc (set->strs,sizeof (*strs) * (cnt + set->cnt)); +if (!tmp) + goto err; +memcpy (tmp + set->cnt, strs,sizeof (*strs) * (cnt)); + set->strs = (const char **)tmp; + set->cnt += cnt;free (input); +free (strs);return 0 ; err: -free (strs); +if (strs) +free (strs);free (input);return -ENOMEM ; }
@@ -553,29 +591,35 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) }break ; } +case ARG_TEST_NAME_GLOB_ALLOWLIST :case ARG_TEST_NAME : {char *subtest_str =strchr (arg,'/' );if (subtest_str) { *subtest_str ='\0' ;if (parse_str_list (subtest_str +1 , - &env->subtest_selector.whitelist)) + &env->subtest_selector.whitelist, + key ==ARG_TEST_NAME_GLOB_ALLOWLIST ))return -ENOMEM ; } -if (parse_str_list (arg, &env->test_selector.whitelist)) +if (parse_str_list (arg, &env->test_selector.whitelist, + key ==ARG_TEST_NAME_GLOB_ALLOWLIST ))return -ENOMEM ;break ; } +case ARG_TEST_NAME_GLOB_DENYLIST :case ARG_TEST_NAME_BLACKLIST : {char *subtest_str =strchr (arg,'/' );if (subtest_str) { *subtest_str ='\0' ;if (parse_str_list (subtest_str +1 , - &env->subtest_selector.blacklist)) + &env->subtest_selector.blacklist, + key ==ARG_TEST_NAME_GLOB_DENYLIST ))return -ENOMEM ; } -if (parse_str_list (arg, &env->test_selector.blacklist)) +if (parse_str_list (arg, &env->test_selector.blacklist, + key ==ARG_TEST_NAME_GLOB_DENYLIST ))return -ENOMEM ;break ; }
Signature · 1 line
2.30.2
Andrii Nakryiko
AppliedRe: [PATCH v5 bpf-next 4/4] selftests/bpf: Support glob matching for test selector.
Quoted 127 lines · Yucong Sun
This patch adds '-a' and '-d' arguments, support exact string match, as well as using '*' wildcard in test/subtests selection. The old '-t' '-b' arguments still supports partial string match, but they can't be used together yet.
This patach also adds support for mulitple '-a' '-d' '-t' '-b' arguments.
Caveat: Same as the current substring matching mechanism, test and subtest selector applies independently, 'a*/b*' will execute all tests matching "a*", and with subtest name matching "b*", but tests matching "a*" that has no subtests will also be executed.
Signed-off-by Yucong Sun <[email protected]>--- tools/testing/selftests/bpf/test_progs.c | 72 +++++++++++++++++++----- 1 file changed, 58 insertions(+), 14 deletions(-)tools/testing/selftests/bpf/test_progs.c +42
-9@@ -13,6 +13,28 @@#include <execinfo.h> /* backtrace */ #include <linux/membarrier.h> +/* Adapted from perf/util/string.c */ +static bool glob_match (const char *str,const char *pat) +{ +while (*str && *pat && *pat !='*' ) { +if (*str != *pat) +return false; + str++; + pat++; + } +/* Check wild card */ +if (*pat =='*' ) { +while (*pat =='*' ) + pat++; +if (!*pat)/* Tail wild card matches all */ +return true; +while (*str) +if (glob_match (str++, pat)) +return true; + } +return !*str && !*pat; +} +#define EXIT_NO_TEST 2#define EXIT_ERR_SETUP_INFRA 3@@ -55,12 +77,12 @@ static bool should_run(struct test_selector *sel, int num, const char *name)int i;for (i =0 ; i < sel->blacklist.cnt; i++) { -if (strstr (name, sel->blacklist.strs[i])) +if (glob_match (name, sel->blacklist.strs[i]))return false; }for (i =0 ; i < sel->whitelist.cnt; i++) { -if (strstr (name, sel->whitelist.strs[i])) +if (glob_match (name, sel->whitelist.strs[i]))return true; }@@ -450,6 +472,8 @@ enum ARG_KEYS {ARG_VERBOSE ='v' ,ARG_GET_TEST_CNT ='c' ,ARG_LIST_TEST_NAMES ='l' , +ARG_TEST_NAME_GLOB_ALLOWLIST ='a' , + ARG_TEST_NAME_GLOB_DENYLIST = 'd', }; static const struct argp_option opts[] = {@@ -467,6 +491,10 @@ static const struct argp_option opts[] = { "Get number of selected top-level tests " }, {"list" ,ARG_LIST_TEST_NAMES ,NULL ,0 , "List test names that would run (without running them) " }, + { "allow", ARG_TEST_NAME_GLOB_ALLOWLIST, "NAMES", 0, + "Run tests with name matching the pattern (support *)." }, + { "deny", ARG_TEST_NAME_GLOB_DENYLIST, "NAMES", 0, + "Don't run tests with name matching the pattern (support *)." }, {}, };@@ -491,7 +519,7 @@ static void free_str_set(const struct str_set *set)free (set->strs); } -static int parse_str_list (const char *s,struct str_set *set) +static int parse_str_list (const char *s,struct str_set *set,bool is_glob_pattern) {char *input, *state =NULL , *next, **tmp, **strs =NULL ;int cnt =0 ;@@ -500,28 +528,38 @@ static int parse_str_list(const char *s, struct str_set *set)if (!input)return -ENOMEM ; - set->cnt =0 ; - set->strs =NULL ; -while ((next =strtok_r (state ?NULL : input,"," , &state))) { tmp =realloc (strs,sizeof (*strs) * (cnt +1 ));if (!tmp) goto err; strs = tmp; +if (is_glob_pattern) { + strs[cnt] =strdup (next); + }else { + strs[cnt] =malloc (strlen (next) +2 +1 ); +if (!strs[cnt]) + goto err; +sprintf (strs[cnt],"*%s*" , next); + } - strs[cnt] =strdup (next);if (!strs[cnt]) goto err; cnt++; } - set->cnt = cnt; - set->strs = (const char **)strs; + tmp =realloc (set->strs,sizeof (*strs) * (cnt + set->cnt)); +if (!tmp) + goto err;
tools/testing/selftests/bpf/test_progs.c static int parse_str_list(const char *s, struct str_set *set) · line 562
Quoted 10 lines
tools/testing/selftests/bpf/test_progs.c +6
-1+memcpy (tmp + set->cnt, strs,sizeof (*strs) * (cnt)); + set->strs = (const char **)tmp; + set->cnt += cnt;free (input); +free (strs);return 0 ; err: -free (strs); +if (strs) +free (strs);
free(NULL) is noop, so no need to check if(strs)
You also missed the need to free all those strdup()'ed strings, so I added
for (i = 0; i < cnt; i++)
free(strs[i]);
We didn't need it originally, because eventually we call free_str_set(), but in this case those strings will be leaked, because we don't assign strs to set->strs.
BTW, a bit cleaner approach would be to just work on set->strs directly, and realloc()ing it as necessary, and only at the end updating set->cnt. You wouldn't need to memcpy and realloc one extra time at the end, and no need for that for + free loop. But it's fine the way it is as well.
Applied to bpf-next, thanks a lot for the improvements.
Quoted 46 lines
tools/testing/selftests/bpf/test_progs.c +10
-5free (input);return -ENOMEM ; }@@ -553,29 +591,35 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) }break ; } +case ARG_TEST_NAME_GLOB_ALLOWLIST :case ARG_TEST_NAME : {char *subtest_str =strchr (arg,'/' );if (subtest_str) { *subtest_str ='\0' ;if (parse_str_list (subtest_str +1 , - &env->subtest_selector.whitelist)) + &env->subtest_selector.whitelist, + key ==ARG_TEST_NAME_GLOB_ALLOWLIST ))return -ENOMEM ; } -if (parse_str_list (arg, &env->test_selector.whitelist)) +if (parse_str_list (arg, &env->test_selector.whitelist, + key ==ARG_TEST_NAME_GLOB_ALLOWLIST ))return -ENOMEM ;break ; } +case ARG_TEST_NAME_GLOB_DENYLIST :case ARG_TEST_NAME_BLACKLIST : {char *subtest_str =strchr (arg,'/' );if (subtest_str) { *subtest_str ='\0' ;if (parse_str_list (subtest_str +1 , - &env->subtest_selector.blacklist)) + &env->subtest_selector.blacklist, + key ==ARG_TEST_NAME_GLOB_DENYLIST ))return -ENOMEM ; } -if (parse_str_list (arg, &env->test_selector.blacklist)) +if (parse_str_list (arg, &env->test_selector.blacklist, + key ==ARG_TEST_NAME_GLOB_DENYLIST ))return -ENOMEM ;break ; } --2.30.2