Run tests in parallel
Ian Rogers
reviewerv4 cover Run tests in parallel
If command line option '-p' is provided, start/fork all tests in the main thread and then gather them in order at the end. On a laptop test time was reduced from 5 minutes 21 seconds to 1 minute 50 seconds. The option isn't default as the test shows up perf and test flakes.
The '-v' option is modified so that 'perf test -v' will give verbose output only from failing tests.
v4 - fix bug in width computation.
v3 - fix a crash if shell test directory isn't found, remove merged patch. v2 - fix parallel test output/verbose issue
v1 - initial PoC
Ian Rogers (8):
perf thread_map: Skip exited threads when scanning /proc perf list: Add scandirat compatibility function
perf tests: Avoid fork in perf_has_symbol test
tools subcmd: Add a no exec function call option
perf test: Rename builtin-test-list and add missed header guard perf tests: Use scandirat for shell script finding perf tests: Run time generate shell test suites
perf tests: Add option to run tests in parallel
tools/lib/subcmd/run-command.c | 2 + tools/lib/subcmd/run-command.h | 2 + tools/perf/tests/Build | 2 +- tools/perf/tests/builtin-test-list.c | 207 ---------- tools/perf/tests/builtin-test-list.h | 12 - tools/perf/tests/builtin-test.c | 378 ++++++++++-------- tools/perf/tests/shell/lib/perf_has_symbol.sh | 2 +- tools/perf/tests/tests-scripts.c | 257 ++++++++++++ tools/perf/tests/tests-scripts.h | 9 + tools/perf/util/print-events.c | 12 +- tools/perf/util/thread_map.c | 9 +- tools/perf/util/util.c | 19 + tools/perf/util/util.h | 8 + 13 files changed, 509 insertions(+), 410 deletions(-) delete mode 100644 tools/perf/tests/builtin-test-list.c delete mode 100644 tools/perf/tests/builtin-test-list.h create mode 100644 tools/perf/tests/tests-scripts.c create mode 100644 tools/perf/tests/tests-scripts.h
Signature · 1 line
2.43.0.687.g38aa6559b0-goog
Ian Rogers
reviewerPATCH v4 1/8 perf thread_map: Skip exited threads when scanning /proc
Scanning /proc is inherently racy. Scanning /proc/pid/task within that is also racy as the pid can terminate. Rather than failing in __thread_map__new_all_cpus, skip pids for such failures.
--- tools/perf/util/thread_map.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-)
tools/perf/util/thread_map.c +4 -5
@@ -109,9 +109,10 @@ static struct perf_thread_map *__thread_map__new_all_cpus(uid_t uid)snprintf (path,sizeof (path),"/proc/%d/task" , pid); items =scandir (path, &namelist, filter,NULL ); -if (items <=0 ) - goto out_free_closedir; - +if (items <=0 ) { +pr_debug ("scandir for %d returned empty, skipping\n" , pid); +continue ; + }while (threads->nr + items >= max_threads) { max_threads *=2 ; grow = true;
@@ -152,8 +153,6 @@ static struct perf_thread_map *__thread_map__new_all_cpus(uid_t uid)for (i =0 ; i < items; i++)zfree (&namelist[i]);free (namelist); - -out_free_closedir:zfree (&threads); goto out_closedir; }
Signature · 1 line
2.43.0.687.g38aa6559b0-goog
Ian Rogers
reviewerPATCH v4 2/8 perf list: Add scandirat compatibility function
In reply to Ian Rogersscandirat is used during the printing of tracepoint events but may be missing from certain libcs. Add a compatibility implementation that uses the symlink of an fd in /proc as a path for the reliably present scandir.
--- tools/perf/util/print-events.c | 12 +++--------- tools/perf/util/util.c | 19 +++++++++++++++++++ tools/perf/util/util.h | 8 ++++++++ 3 files changed, 30 insertions(+), 9 deletions(-)
tools/perf/util/print-events.c +3 -9
@@ -63,6 +63,8 @@ void print_tracepoint_events(const struct print_callbacks *print_cb __maybe_unus {char *events_path =get_tracing_file ("events" );int events_fd =open (events_path,O_PATH ); +struct dirent **sys_namelist =NULL ; +int sys_items;put_tracing_file (events_path);if (events_fd <0 ) {
@@ -70,10 +72,7 @@ void print_tracepoint_events(const struct print_callbacks *print_cb __maybe_unusreturn ; } -#ifdef HAVE_SCANDIRAT_SUPPORT -{ -struct dirent **sys_namelist =NULL ; -int sys_items =tracing_events__scandir_alphasort (&sys_namelist); + sys_items =tracing_events__scandir_alphasort (&sys_namelist);for (int i =0 ; i < sys_items; i++) {struct dirent *sys_dirent = sys_namelist[i];
@@ -130,11 +129,6 @@ void print_tracepoint_events(const struct print_callbacks *print_cb __maybe_unus }free (sys_namelist); -} -#else -printf ("\nWARNING: Your libc doesn't have the scandirat function, please ask its maintainers to implement it.\n" -" As a rough fallback, please do 'ls %s' to see the available tracepoint events.\n" , events_path); -#endif close (events_fd); }
tools/perf/util/util.c +19 -0
@@ -552,3 +552,22 @@ int sched_getcpu(void) return -1; }#endif + +#ifndef HAVE_SCANDIRAT_SUPPORT +int scandirat (int dirfd,const char *dirp, +struct dirent ***namelist, +int (*filter)(const struct dirent *), +int (*compar)(const struct dirent **,const struct dirent **)) +{ +char path[PATH_MAX ]; +int err, fd =openat (dirfd, dirp,O_PATH ); + +if (fd <0 ) +return fd; + +snprintf (path,sizeof (path),"/proc/%d/fd/%d" ,getpid (), fd); + err =scandir (path, namelist, filter, compar); +close (fd); +return err; +} +#endif
tools/perf/util/util.h +8 -0
@@ -6,6 +6,7 @@/* glibc 2.20 deprecates _BSD_SOURCE in favour of _DEFAULT_SOURCE */ #define _DEFAULT_SOURCE 1 +#include <dirent.h> #include <fcntl.h> #include <stdbool.h> #include <stddef.h>
@@ -56,6 +57,13 @@ int perf_tip(char **strp, const char *dirpath);int sched_getcpu (void );#endif +#ifndef HAVE_SCANDIRAT_SUPPORT +int scandirat (int dirfd,const char *dirp, +struct dirent ***namelist, +int (*filter)(const struct dirent *), +int (*compar)(const struct dirent **,const struct dirent **)); +#endif +extern bool perf_singlethreaded;void perf_set_singlethreaded (void );
Signature · 1 line
2.43.0.687.g38aa6559b0-goog
Ian Rogers
reviewerPATCH v4 3/8 perf tests: Avoid fork in perf_has_symbol test
In reply to Ian Rogersperf test -vv Symbols is used to indentify symbols within the perf binary. Add the -F flag so that the test command doesn't fork the test before running. This removes a little overhead.
--- tools/perf/tests/shell/lib/perf_has_symbol.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
tools/perf/tests/shell/lib/perf_has_symbol.sh +1 -1
@@ -3,7 +3,7 @@ perf_has_symbol() { - if perf test -vv "Symbols" 2>&1 | grep "[[:space:]]$1$"; then + if perf test -vv -F "Symbols" 2>&1 | grep "[[:space:]]$1$"; then echo "perf does have symbol '$1'" return 0 fi
Signature · 1 line
2.43.0.687.g38aa6559b0-goog
Ian Rogers
reviewerPATCH v4 4/8 tools subcmd: Add a no exec function call option
In reply to Ian RogersTools like perf fork tests in case they crash, but they don't want to exec a full binary. Add an option to call a function rather than do an exec. The child process exits with the result of the function call and is passed the struct of the run_command, things like container_of can then allow the child process function to determine additional arguments.
--- tools/lib/subcmd/run-command.c | 2 ++ tools/lib/subcmd/run-command.h | 2 ++ 2 files changed, 4 insertions(+)
tools/lib/subcmd/run-command.c +2 -0
@@ -122,6 +122,8 @@ int start_command(struct child_process *cmd) }if (cmd->preexec_cb) cmd->preexec_cb (); +if (cmd->no_exec_cmd) +exit (cmd->no_exec_cmd (cmd));if (cmd->exec_cmd) {execv_cmd (cmd->argv); }else {
tools/lib/subcmd/run-command.h +2 -0
@@ -47,6 +47,8 @@ struct child_process { unsigned exec_cmd:1; /* if this is to be external sub-command */ unsigned stdout_to_stderr:1;void (*preexec_cb)(void ); +/* If set, call function in child rather than doing an exec. */ +int (*no_exec_cmd)(struct child_process *process); };int start_command (struct child_process *);
Signature · 1 line
2.43.0.687.g38aa6559b0-goog
Ian Rogers
reviewerPATCH v4 5/8 perf test: Rename builtin-test-list and add missed header guard
In reply to Ian Rogersbuiltin-test-list is primarily concerned with shell script tests. Rename the file to better reflect this and add a missed header guard.
---
tools/perf/tests/Build | 2 +-
tools/perf/tests/builtin-test.c | 2 +-
tools/perf/tests/{builtin-test-list.c => tests-scripts.c} | 2 +-
tools/perf/tests/{builtin-test-list.h => tests-scripts.h} | 4 ++++
4 files changed, 7 insertions(+), 3 deletions(-)
rename tools/perf/tests/{builtin-test-list.c => tests-scripts.c} (99%)
rename tools/perf/tests/{builtin-test-list.h => tests-scripts.h} (79%)tools/perf/tests/Build +1 -1
@@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 perf-y += builtin-test.o -perf-y += builtin-test-list.o +perf-y += tests-scripts.o perf-y += parse-events.o perf-y += dso-data.o perf-y += attr.o
tools/perf/tests/builtin-test.c +1 -1
@@ -29,7 +29,7 @@#include <subcmd/exec-cmd.h> #include <linux/zalloc.h> -#include "builtin-test-list.h" +#include "tests-scripts.h" static bool dont_fork;const char *dso_to_test;
tools/perf/tests/tests-scripts.c renamed from tools/perf/tests/builtin-test-list.c +1 -1
@@ -15,7 +15,7 @@#include <sys/wait.h> #include <sys/stat.h> #include "builtin.h" -#include "builtin-test-list.h" +#include "tests-scripts.h" #include "color.h" #include "debug.h" #include "hist.h"
tools/perf/tests/tests-scripts.h renamed from tools/perf/tests/builtin-test-list.h +4 -0
@@ -1,4 +1,6 @@/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef TESTS_SCRIPTS_H +#define TESTS_SCRIPTS_H struct script_file { char *dir;
@@ -10,3 +12,5 @@ struct script_file {const struct script_file *list_script_files (void );/* Get maximum width of description string */ int list_script_max_width (void ); + +#endif /* TESTS_SCRIPTS_H */
Signature · 1 line
2.43.0.687.g38aa6559b0-goog
Ian Rogers
reviewerPATCH v4 6/8 perf tests: Use scandirat for shell script finding
In reply to Ian RogersAvoid filename appending buffers by using openat, faccessat and scandirat more widely. Turn the script's path back to a file name using readlink from /proc/<pid>/fd/<fd>.
Read the script's description using api/io.h to avoid fdopen conversions. Whilst reading perform additional sanity checks on the script's contents.
--- tools/perf/tests/builtin-test.c | 21 ++--- tools/perf/tests/tests-scripts.c | 144 ++++++++++++++++++------------- tools/perf/tests/tests-scripts.h | 1 - 3 files changed, 95 insertions(+), 71 deletions(-)
tools/perf/tests/builtin-test.c +9 -12
@@ -300,22 +300,20 @@ static int test_and_print(struct test_suite *t, int subtest) }struct shell_test { -const char *dir;const char *file; };static int shell_test__run (struct test_suite *test,int subdir__maybe_unused ) {int err; -char script[PATH_MAX ];struct shell_test *st = test->priv; +char *cmd; -path__join (script,sizeof (script) -3 , st->dir, st->file); - -if (verbose >0 ) -strncat (script," -v" ,sizeof (script) -strlen (script) -1 ); - - err =system (script); +asprintf (&cmd,"%s%s" , st->file, verbose ?" -v" :"" ); +if (!cmd) +return TEST_FAIL ; + err =system (cmd); +free (cmd);if (!err)return TEST_OK ;
@@ -331,7 +329,7 @@ static int run_shell_tests(int argc, const char *argv[], int i, int width, files =list_script_files ();if (!files)return 0 ; -for (file = files; file->dir; file++) { +for (file = files; file->file; file++) {int curr = i++;struct test_case test_cases[] = { {
@@ -345,13 +343,12 @@ static int run_shell_tests(int argc, const char *argv[], int i, int width, .test_cases = test_cases, .priv = &st, }; - st.dir = file->dir; + st.file = file->file;if (test_suite.desc ==NULL || !perf_test__matches (test_suite.desc, curr, argc, argv))continue ; - st.file = file->file;pr_info ("%3d: %-*s:" , i, width, test_suite.desc);if (intlist__find (skiplist, i)) {
@@ -455,7 +452,7 @@ static int perf_test__list_shell(int argc, const char **argv, int i) files =list_script_files ();if (!files)return 0 ; -for (file = files; file->dir; file++) { +for (file = files; file->file; file++) {int curr = i++;struct test_suite t = { .desc = file->desc
tools/perf/tests/tests-scripts.c +86 -58
235 lines not on this page · raw message
tools/perf/tests/tests-scripts.h +0 -1
@@ -3,7 +3,6 @@#define TESTS_SCRIPTS_H struct script_file { -char *dir;char *file;char *desc; };
Signature · 1 line
2.43.0.687.g38aa6559b0-goog
Namhyung Kim
maintainerRe: [PATCH v4 6/8] perf tests: Use scandirat for shell script finding
Quoted 34 lines · Ian Rogers
Avoid filename appending buffers by using openat, faccessat and scandirat more widely. Turn the script's path back to a file name using readlink from /proc/<pid>/fd/<fd>.
Read the script's description using
api/io.hto avoid fdopen conversions. Whilst reading perform additional sanity checks on the script's contents.Signed-off-by Ian Rogers <[email protected]>--- tools/perf/tests/builtin-test.c | 21 ++--- tools/perf/tests/tests-scripts.c | 144 ++++++++++++++++++------------- tools/perf/tests/tests-scripts.h | 1 - 3 files changed, 95 insertions(+), 71 deletions(-)tools/perf/tests/builtin-test.c +1
-2@@ -300,22 +300,20 @@ static int test_and_print(struct test_suite *t, int subtest) }struct shell_test { -const char *dir;const char *file; };static int shell_test__run (struct test_suite *test,int subdir__maybe_unused ) {int err; -char script[PATH_MAX ];struct shell_test *st = test->priv; +char *cmd;
tools/perf/tests/builtin-test.c static int test_and_print(struct test_suite *t, int subtest) · line 314
Quoted 10 lines
tools/perf/tests/builtin-test.c +3
-6-path__join (script,sizeof (script) -3 , st->dir, st->file); - -if (verbose >0 ) -strncat (script," -v" ,sizeof (script) -strlen (script) -1 ); - - err =system (script); +asprintf (&cmd,"%s%s" , st->file, verbose ?" -v" :"" ); +if (!cmd) +return TEST_FAIL ;
It fails to build.
tests/tests-scripts.c: In function ‘shell_test__run’:tests/tests-scripts.c:130:9: error: ignoring return value of ‘asprintf’ declared with attribute ‘warn_unused_result’ [-Werror=unused-result]
130 | asprintf(&cmd, "%s%s", file, verbose ? " -v" : "");
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~Thanks,
Namhyung
Quoted 5 lines
tools/perf/tests/builtin-test.c +2
-0+ err =system (cmd); +free (cmd);if (!err)return TEST_OK ;
Ian Rogers
reviewerRe: [PATCH v4 6/8] perf tests: Use scandirat for shell script finding
Quoted 49 lines · Namhyung Kim
Quoted 34 lines · Ian Rogers
Avoid filename appending buffers by using openat, faccessat and scandirat more widely. Turn the script's path back to a file name using readlink from /proc/<pid>/fd/<fd>.
Read the script's description using
api/io.hto avoid fdopen conversions. Whilst reading perform additional sanity checks on the script's contents.Signed-off-by Ian Rogers <[email protected]>--- tools/perf/tests/builtin-test.c | 21 ++--- tools/perf/tests/tests-scripts.c | 144 ++++++++++++++++++------------- tools/perf/tests/tests-scripts.h | 1 - 3 files changed, 95 insertions(+), 71 deletions(-)tools/perf/tests/builtin-test.c +1
-2@@ -300,22 +300,20 @@ static int test_and_print(struct test_suite *t, int subtest) }struct shell_test { -const char *dir;const char *file; };static int shell_test__run (struct test_suite *test,int subdir__maybe_unused ) {int err; -char script[PATH_MAX ];struct shell_test *st = test->priv; +char *cmd;tools/perf/tests/builtin-test.c static int test_and_print(struct test_suite *t, int subtest) · line 314
Quoted 10 lines
tools/perf/tests/builtin-test.c +3
-6-path__join (script,sizeof (script) -3 , st->dir, st->file); - -if (verbose >0 ) -strncat (script," -v" ,sizeof (script) -strlen (script) -1 ); - - err =system (script); +asprintf (&cmd,"%s%s" , st->file, verbose ?" -v" :"" ); +if (!cmd) +return TEST_FAIL ;It fails to build.
Quoted 6 lines
tests/tests-scripts.c: In function ‘shell_test__run’:tests/tests-scripts.c:130:9: error: ignoring return value of ‘asprintf’ declared with attribute ‘warn_unused_result’ [-Werror=unused-result]130 | asprintf(&cmd, "%s%s", file, verbose ? " -v" : ""); | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Thanks, 1 liner fix in v5 on its way. I'll add -Werror=unused-result to my EXTRA_CFLAGS.
Ian
Quoted 9 lines
Thanks,
NamhyungQuoted 5 lines
tools/perf/tests/builtin-test.c +2
-0+ err =system (cmd); +free (cmd);if (!err)return TEST_OK ;
Ian Rogers
reviewerPATCH v4 7/8 perf tests: Run time generate shell test suites
In reply to Ian RogersRather than special shell test logic, do a single pass to create an array of test suites. Hold the shell test file name in the test suite priv field. This makes the special shell test logic in builtin-test.c redundant so remove it.
--- tools/perf/tests/builtin-test.c | 91 +--------------------- tools/perf/tests/tests-scripts.c | 130 ++++++++++++++++++------------- tools/perf/tests/tests-scripts.h | 10 +-- 3 files changed, 81 insertions(+), 150 deletions(-)
tools/perf/tests/builtin-test.c +3 -88
@@ -130,6 +130,7 @@ static struct test_suite *generic_tests[] = {static struct test_suite **tests[] = { generic_tests, arch_tests, +NULL ,/* shell tests created at runtime. */ };static struct test_workload *workloads[] = {
@@ -299,74 +300,12 @@ static int test_and_print(struct test_suite *t, int subtest)return err; } -struct shell_test { -const char *file; -}; - -static int shell_test__run (struct test_suite *test,int subdir__maybe_unused ) -{ -int err; -struct shell_test *st = test->priv; -char *cmd; - -asprintf (&cmd,"%s%s" , st->file, verbose ?" -v" :"" ); -if (!cmd) -return TEST_FAIL ; - err =system (cmd); -free (cmd); -if (!err) -return TEST_OK ; - -return WEXITSTATUS (err) ==2 ?TEST_SKIP :TEST_FAIL ; -} - -static int run_shell_tests (int argc,const char *argv[],int i,int width, -struct intlist *skiplist) -{ -struct shell_test st; -const struct script_file *files, *file; - - files =list_script_files (); -if (!files) -return 0 ; -for (file = files; file->file; file++) { -int curr = i++; -struct test_case test_cases[] = { - { - .desc = file->desc, - .run_case = shell_test__run, - }, - { .name =NULL , } - }; -struct test_suite test_suite = { - .desc = test_cases[0 ].desc, - .test_cases = test_cases, - .priv = &st, - }; - st.file = file->file; - -if (test_suite.desc ==NULL || - !perf_test__matches (test_suite.desc, curr, argc, argv)) -continue ; - -pr_info ("%3d: %-*s:" , i, width, test_suite.desc); - -if (intlist__find (skiplist, i)) { -color_fprintf (stderr,PERF_COLOR_YELLOW ," Skip (user override)\n" ); -continue ; - } - -test_and_print (&test_suite,0 ); - } -return 0 ; -} -static int __cmd_test (int argc,const char *argv[],struct intlist *skiplist) {struct test_suite *t;unsigned int j, k;int i =0 ; -int width =list_script_max_width (); +int width =0 ;for_each_test (j, k, t) {int len =strlen (test_description (t,-1 ));
@@ -441,28 +380,6 @@ static int __cmd_test(int argc, const char *argv[], struct intlist *skiplist) } } } - -return run_shell_tests (argc, argv, i, width, skiplist); -} - -static int perf_test__list_shell (int argc,const char **argv,int i) -{ -const struct script_file *files, *file; - - files =list_script_files (); -if (!files) -return 0 ; -for (file = files; file->file; file++) { -int curr = i++; -struct test_suite t = { - .desc = file->desc - }; - -if (!perf_test__matches (t.desc, curr, argc, argv)) -continue ; - -pr_info ("%3d: %s\n" , i, t.desc); - }return 0 ; }
@@ -489,9 +406,6 @@ static int perf_test__list(int argc, const char **argv) test_description(t, subi)); } } - -perf_test__list_shell (argc, argv, i); -return 0 ; }
@@ -551,6 +465,7 @@ int cmd_test(int argc, const char **argv)/* Unbuffered output */ setvbuf (stdout,NULL , _IONBF,0 ); + tests[2 ] =create_script_test_suites (); argc =parse_options_subcommand (argc, argv, test_options, test_subcommands, test_usage,0 );if (argc >=1 && !strcmp (argv[0 ],"list" ))return perf_test__list (argc -1 , argv +1 );
tools/perf/tests/tests-scripts.c +76 -54
176 lines not on this page · raw message
tools/perf/tests/tests-scripts.h +2 -8
@@ -2,14 +2,8 @@#ifndef TESTS_SCRIPTS_H #define TESTS_SCRIPTS_H -struct script_file { -char *file; -char *desc; -}; +#include "tests.h" -/* List available script tests to run - singleton - never freed */ -const struct script_file *list_script_files (void ); -/* Get maximum width of description string */ -int list_script_max_width (void ); +struct test_suite **create_script_test_suites (void );#endif /* TESTS_SCRIPTS_H */
Signature · 1 line
2.43.0.687.g38aa6559b0-goog
Ian Rogers
reviewerPATCH v4 8/8 perf tests: Add option to run tests in parallel
In reply to Ian RogersBy default tests are forked, add an option (-p or --parallel) so that the forked tests are all started in parallel and then their output gathered serially. This is opt-in as running in parallel can cause test flakes.
Rather than fork within the code, the start_command/finish_command from libsubcmd are used. This changes how stderr and stdout are handled. The child stderr and stdout are always read to avoid the child blocking. If verbose is 1 (-v) then if the test fails the child stdout and stderr are displayed. If the verbose is >1 (e.g. -vv) then the stdout and stderr from the child are immediately displayed.
An unscientific test on my laptop shows the wall clock time for perf test without parallel being 5 minutes 21 seconds and with parallel (-p) being 1 minute 50 seconds.
---
v4 fix width computation with subtests bug caught by Namhyung. v1 of this code had a bug where stdout/stderr weren't read fully. This and additional issues/improvements are dealt with in v2.
--- tools/perf/tests/builtin-test.c | 314 ++++++++++++++++++++++---------- 1 file changed, 215 insertions(+), 99 deletions(-)
tools/perf/tests/builtin-test.c +215 -99
@@ -6,6 +6,7 @@*/ #include <fcntl.h> #include <errno.h> +#include <poll.h> #include <unistd.h> #include <string.h> #include <stdlib.h>
@@ -21,9 +22,11 @@#include "debug.h" #include "color.h" #include <subcmd/parse-options.h> +#include <subcmd/run-command.h> #include "string2.h" #include "symbol.h" #include "util/rlimit.h" +#include "util/strbuf.h" #include <linux/kernel.h> #include <linux/string.h> #include <subcmd/exec-cmd.h>
@@ -31,7 +34,13 @@#include "tests-scripts.h" +/* +* Command line option to not fork the test running in the same process and +* making them easier to debug. +*/ static bool dont_fork; +/* Fork the tests in parallel and then wait for their completion. */ +static bool parallel;const char *dso_to_test;const char *test_objdump_path ="objdump" ;
@@ -209,76 +218,36 @@ static bool perf_test__matches(const char *desc, int curr, int argc, const charreturn false; } -static int run_test (struct test_suite *test,int subtest) -{ -int status, err =-1 , child = dont_fork ?0 :fork (); -char sbuf[STRERR_BUFSIZE ]; - -if (child <0 ) { -pr_err ("failed to fork test: %s\n" , -str_error_r (errno, sbuf,sizeof (sbuf))); -return -1 ; - } - -if (!child) { -if (!dont_fork) { -pr_debug ("test child forked, pid %d\n" ,getpid ()); - -if (verbose <=0 ) { -int nullfd =open ("/dev/null" ,O_WRONLY ); - -if (nullfd >=0 ) { -close (STDERR_FILENO ); -close (STDOUT_FILENO ); - -dup2 (nullfd,STDOUT_FILENO ); -dup2 (STDOUT_FILENO ,STDERR_FILENO ); -close (nullfd); - } - }else { -signal (SIGSEGV , sighandler_dump_stack); -signal (SIGFPE , sighandler_dump_stack); - } - } - - err =test_function (test, subtest)(test, subtest); -if (!dont_fork) -exit (err); - } - -if (!dont_fork) { -wait (&status); +struct child_test { +struct child_process process; +struct test_suite *test; +int test_num; +int subtest; +}; -if (WIFEXITED (status)) { - err = (signed char )WEXITSTATUS (status); -pr_debug ("test child finished with %d\n" , err); - }else if (WIFSIGNALED (status)) { - err =-1 ; -pr_debug ("test child interrupted\n" ); - } - } +static int run_test_child (struct child_process *process) +{ + struct child_test *child = container_of(process, struct child_test, process); +int err; -return err; +pr_debug ("--- start ---\n" ); +pr_debug ("test child forked, pid %d\n" ,getpid ()); + err =test_function (child->test, child->subtest)(child->test, child->subtest); +pr_debug ("---- end(%d) ----\n" , err); +fflush (NULL ); +return -err; } -#define for_each_test (j, k, t) \ - for (j = 0, k = 0; j < ARRAY_SIZE(tests); j++, k = 0) \ - while ((t = tests[j][k++]) != NULL) - -static int test_and_print (struct test_suite *t,int subtest) +static int print_test_result (struct test_suite *t,int i,int subtest,int result,int width) { -int err; +if (has_subtests (t)) { +int subw = width >2 ? width -2 : width; -pr_debug ("\n--- start ---\n" ); - err =run_test (t, subtest); -pr_debug ("---- end ----\n" ); +pr_info ("%3d.%1d: %-*s:" , i +1 , subtest +1 , subw,test_description (t, subtest)); + }else +pr_info ("%3d: %-*s:" , i +1 , width,test_description (t, subtest)); -if (!has_subtests (t)) -pr_debug ("%s:" , t->desc); -else -pr_debug ("%s subtest %d:" , t->desc, subtest +1 ); - -switch (err) { +switch (result) {case TEST_OK :pr_info (" Ok\n" );break ;
@@ -297,34 +266,186 @@ static int test_and_print(struct test_suite *t, int subtest)break ; } -return err; +return 0 ; +} + +static int finish_test (struct child_test *child_test,int width) +{ +struct test_suite *t = child_test->test; +int i = child_test->test_num; +int subi = child_test->subtest; +int out = child_test->process.out; +int err = child_test->process.err; +bool out_done = out <=0 ; +bool err_done = err <=0 ; +struct strbuf out_output =STRBUF_INIT ; +struct strbuf err_output =STRBUF_INIT ; +int ret; + +/* +* For test suites with subtests, display the suite name ahead of the +* sub test names. +*/ +if (has_subtests (t) && subi ==0 ) +pr_info ("%3d: %-*s:\n" , i +1 , width,test_description (t,-1 )); + +/* +* Busy loop reading from the child's stdout and stderr that are set to +* be non-blocking until EOF. +*/ +if (!out_done) +fcntl (out,F_SETFL ,O_NONBLOCK ); +if (!err_done) +fcntl (err,F_SETFL ,O_NONBLOCK ); +if (verbose >1 ) { +if (has_subtests (t)) +pr_info ("%3d.%1d: %s:\n" , i +1 , subi +1 ,test_description (t, subi)); +else +pr_info ("%3d: %s:\n" , i +1 ,test_description (t,-1 )); + } +while (!out_done || !err_done) { +struct pollfd pfds[2 ] = { + { .fd = out, + .events =POLLIN |POLLERR |POLLHUP |POLLNVAL , + }, + { .fd = err, + .events =POLLIN |POLLERR |POLLHUP |POLLNVAL , + }, + }; +char buf[512 ]; +ssize_t len; + +/* Poll to avoid excessive spinning, timeout set for 1000ms. */ +poll (pfds,ARRAY_SIZE (pfds),/*timeout=*/ 1000 ); +if (!out_done && pfds[0 ].revents) { + errno =0 ; + len =read (out, buf,sizeof (buf) -1 ); + +if (len <=0 ) { + out_done = errno !=EAGAIN ; + }else { + buf[len] ='\0' ; +if (verbose >1 ) +fprintf (stdout,"%s" , buf); +else +strbuf_addstr (&out_output, buf); + } + } +if (!err_done && pfds[1 ].revents) { + errno =0 ; + len =read (err, buf,sizeof (buf) -1 ); + +if (len <=0 ) { + err_done = errno !=EAGAIN ; + }else { + buf[len] ='\0' ; +if (verbose >1 ) +fprintf (stdout,"%s" , buf); +else +strbuf_addstr (&err_output, buf); + } + } + } +/* Clean up child process. */ + ret =finish_command (&child_test->process); +if (verbose ==1 && ret ==TEST_FAIL ) { +/* Add header for test that was skipped above. */ +if (has_subtests (t)) +pr_info ("%3d.%1d: %s:\n" , i +1 , subi +1 ,test_description (t, subi)); +else +pr_info ("%3d: %s:\n" , i +1 ,test_description (t,-1 )); +fprintf (stdout,"%s" , out_output.buf); +fprintf (stderr,"%s" , err_output.buf); + } +strbuf_release (&out_output); +strbuf_release (&err_output); +print_test_result (t, i, subi, ret, width); +if (out >0 ) +close (out); +if (err >0 ) +close (err); +return 0 ; +} + +static int start_test (struct test_suite *test,int i,int subi,struct child_test **child, +int width) +{ +int err; + + *child =NULL ; +if (dont_fork) { +pr_debug ("--- start ---\n" ); + err =test_function (test, subi)(test, subi); +pr_debug ("---- end ----\n" ); +print_test_result (test, i, subi, err, width); +return 0 ; + } + + *child =zalloc (sizeof (**child)); +if (!*child) +return -ENOMEM ; + + (*child)->test = test; + (*child)->test_num = i; + (*child)->subtest = subi; + (*child)->process.pid =-1 ; + (*child)->process.no_stdin =1 ; +if (verbose <=0 ) { + (*child)->process.no_stdout =1 ; + (*child)->process.no_stderr =1 ; + }else { + (*child)->process.out =-1 ; + (*child)->process.err =-1 ; + } + (*child)->process.no_exec_cmd = run_test_child; + err =start_command (&(*child)->process); +if (err || parallel) +return err; +return finish_test (*child, width); } +#define for_each_test (j, k, t) \ + for (j = 0, k = 0; j < ARRAY_SIZE(tests); j++, k = 0) \ + while ((t = tests[j][k++]) != NULL) +static int __cmd_test (int argc,const char *argv[],struct intlist *skiplist) {struct test_suite *t;unsigned int j, k;int i =0 ;int width =0 ; +size_t num_tests =0 ; +struct child_test **child_tests; +int child_test_num =0 ;for_each_test (j, k, t) {int len =strlen (test_description (t,-1 ));if (width < len) width = len; + +if (has_subtests (t)) { +for (int subi =0 , subn =num_subtests (t); subi < subn; subi++) { + len =strlen (test_description (t, subi)); +if (width < len) + width = len; + num_tests++; + } + }else { + num_tests++; + } } + child_tests =calloc (num_tests,sizeof (*child_tests)); +if (!child_tests) +return -ENOMEM ;for_each_test (j, k, t) {int curr = i++; -int subi;if (!perf_test__matches (test_description (t,-1 ), curr, argc, argv)) {bool skip = true; -int subn; - - subn =num_subtests (t); -for (subi =0 ; subi < subn; subi++) { +for (int subi =0 , subn =num_subtests (t); subi < subn; subi++) {if (perf_test__matches (test_description (t, subi), curr, argc, argv)) skip = false;
@@ -334,52 +455,45 @@ static int __cmd_test(int argc, const char *argv[], struct intlist *skiplist)continue ; } -pr_info ("%3d: %-*s:" , i, width,test_description (t,-1 )); -if (intlist__find (skiplist, i)) { +pr_info ("%3d: %-*s:" , curr +1 , width,test_description (t,-1 ));color_fprintf (stderr,PERF_COLOR_YELLOW ," Skip (user override)\n" );continue ; }if (!has_subtests (t)) { -test_and_print (t,-1 ); - }else { -int subn =num_subtests (t); -/* -* minus 2 to align with normal testcases. -* For subtest we print additional '.x' in number. -* for example: -* -* 35: Test LLVM searching and compiling : -* 35.1: Basic BPF llvm compiling test : Ok -*/ -int subw = width >2 ? width -2 : width; - -if (subn <=0 ) { -color_fprintf (stderr,PERF_COLOR_YELLOW , -" Skip (not compiled in)\n" ); -continue ; - } -pr_info ("\n" ); +int err =start_test (t, curr,-1 , &child_tests[child_test_num++], width); -for (subi =0 ; subi < subn; subi++) { -int len =strlen (test_description (t, subi)); - -if (subw < len) - subw = len; +if (err) { +/* TODO: if parallel waitpid the already forked children. */ +free (child_tests); +return err; } + }else { +for (int subi =0 , subn =num_subtests (t); subi < subn; subi++) { +int err; -for (subi =0 ; subi < subn; subi++) {if (!perf_test__matches (test_description (t, subi), curr, argc, argv))continue ; -pr_info ("%3d.%1d: %-*s:" , i, subi +1 , subw, -test_description (t, subi)); -test_and_print (t, subi); + err =start_test (t, curr, subi, &child_tests[child_test_num++], + width); +if (err) +return err; } } } +for (i =0 ; i < child_test_num; i++) { +if (parallel) { +int ret =finish_test (child_tests[i], width); + +if (ret) +return ret; + } +free (child_tests[i]); + } +free (child_tests);return 0 ; }
@@ -447,6 +561,8 @@ int cmd_test(int argc, const char **argv) "be more verbose (show symbol address, etc)"),OPT_BOOLEAN ('F' ,"dont-fork" , &dont_fork,"Do not fork for testcase" ), +OPT_BOOLEAN ('p' ,"parallel" , ¶llel, +"Run the tests altogether in parallel" ),OPT_STRING ('w' ,"workload" , &workload,"work" ,"workload to run for testing" ),OPT_STRING (0 ,"dso" , &dso_to_test,"dso" ,"dso to test" ), OPT_STRING(0, "objdump", &test_objdump_path, "path",
Signature · 1 line
2.43.0.687.g38aa6559b0-goog
Hi Ian,
Quoted 5 lines · Ian Rogers
If command line option '-p' is provided, start/fork all tests in the main thread and then gather them in order at the end. On a laptop test time was reduced from 5 minutes 21 seconds to 1 minute 50 seconds. The option isn't default as the test shows up perf and test flakes.
It's a bit sad that tests flake when run in parallel but I think we can work on improving it later. So
Thanks,
Namhyung
Quoted 41 lines
The '-v' option is modified so that 'perf test -v' will give verbose output only from failing tests.
v4 - fix bug in width computation.
v3 - fix a crash if shell test directory isn't found, remove merged patch. v2 - fix parallel test output/verbose issue
v1 - initial PoCIan Rogers (8):
perfthread_map: Skip exited threads when scanning /proc perf list: Add scandirat compatibility function
perf tests: Avoid fork inperf_has_symboltest
tools subcmd: Add a no exec function call option
perf test: Rename builtin-test-list and add missed header guard perf tests: Use scandirat for shell script finding perf tests: Run time generate shell test suites
perf tests: Add option to run tests in paralleltools/lib/subcmd/run-command.c | 2 + tools/lib/subcmd/run-command.h | 2 + tools/perf/tests/Build | 2 +- tools/perf/tests/builtin-test-list.c | 207 ---------- tools/perf/tests/builtin-test-list.h | 12 - tools/perf/tests/builtin-test.c | 378 ++++++++++-------- tools/perf/tests/shell/lib/perf_has_symbol.sh | 2 +- tools/perf/tests/tests-scripts.c | 257 ++++++++++++ tools/perf/tests/tests-scripts.h | 9 + tools/perf/util/print-events.c | 12 +- tools/perf/util/thread_map.c | 9 +- tools/perf/util/util.c | 19 + tools/perf/util/util.h | 8 + 13 files changed, 509 insertions(+), 410 deletions(-) delete mode 100644 tools/perf/tests/builtin-test-list.c delete mode 100644 tools/perf/tests/builtin-test-list.h create mode 100644 tools/perf/tests/tests-scripts.c create mode 100644 tools/perf/tests/tests-scripts.h--
2.43.0.687.g38aa6559b0-goog