Stop running ptests in parallel
cover Stop running ptests in parallel
07d8a67 ("runner: Correctly handle running parallel tests") made an incorrect assumption that it was OK to run ptests in parallel and interleave the output. This is not correct since OE selftest expects all text between a BEGIN and END block to be for a single ptest. Revert the commit to run in parallel and add the correct fix for the bug with running multiple ptests in a single invocation, as well as a few other fixes
Joshua Watt (4):
Revert "runner: Correctly handle running parallel tests" Recreate pipe for each test
Report if child dies from a signal
Change test timeout to be total elapsed time
tests/data/hang/ptest/run-ptest | 1 + tests/data/signal/ptest/run-ptest | 10 ++ tests/utils.c | 4 +- utils.c | 290 +++++++++++++----------------- 4 files changed, 140 insertions(+), 165 deletions(-) create mode 100755 tests/data/signal/ptest/run-ptest
Signature · 1 line
2.33.0
PATCH 1/4 Revert "runner: Correctly handle running parallel tests"
This reverts commit 07d8a676aa962ecc5ec264ec33b0074adf2a8733.
This commit incorrectly assumed that ptest ran tests in parallel, which is not true. Doing so interleaves the test output in the log file, which the OE selftest parser cannot handle and thus breaks the test cases.
--- tests/utils.c | 39 +------- utils.c | 248 +++++++++++++++++++++----------------------------- 2 files changed, 105 insertions(+), 182 deletions(-)
tests/utils.c +3 -36
@@ -50,10 +50,9 @@ static char *ptests_found[] = {"glibc" ,"hang" ,"python" , -"signal" ,NULL }; -static int ptests_found_length =7 ; +static int ptests_found_length =6 ;static char *ptests_not_found[] = {"busybox" , "perl",
@@ -187,7 +186,6 @@ START_TEST(test_run_ptests) head =get_available_ptests (opts_directory);ptest_list_remove (head,"hang" ,1 );ptest_list_remove (head,"fail" ,1 ); -ptest_list_remove (head,"signal" ,1 ); rc =run_ptests (head, opts,"test_run_ptests" , fp_stdout, fp_stderr);ck_assert (rc ==0 );
@@ -217,8 +215,8 @@ search_for_timeout_and_duration(const int rp, FILE *fp_stdout) found_duration = found_duration ? found_duration :find_word (line, duration_str); } -ck_assert_msg (found_timeout == true,"TIMEOUT not found" ); -ck_assert_msg (found_duration == true,"DURATION not found" ); +ck_assert (found_timeout == true); +ck_assert (found_duration == true); }START_TEST (test_run_timeout_duration_ptest)
@@ -232,36 +230,6 @@ START_TEST(test_run_timeout_duration_ptest) }END_TEST -static void -search_for_signal_and_duration (const int rp,FILE *fp_stdout) -{ -char line_buf[PRINT_PTEST_BUF_SIZE ]; -bool found_error = false, found_duration = false; -char *line =NULL ; - -ck_assert (rp !=0 ); - -while ((line =fgets (line_buf,PRINT_PTEST_BUF_SIZE , fp_stdout)) !=NULL ) { -// once true, stay true - found_error = found_error ? found_error :find_word (line,"ERROR: Exited with signal" ); - found_duration = found_duration ? found_duration :find_word (line,"DURATION" ); - } - -ck_assert_msg (found_error == true,"ERROR not found" ); -ck_assert_msg (found_duration == true,"DURATION not found" ); -} - -START_TEST (test_run_signal_ptest) -{ -struct ptest_list *head =get_available_ptests (opts_directory); -unsigned int timeout =10 ; - -test_ptest_expected_failure (head, timeout,"signal" , search_for_signal_and_duration); - -ptest_list_free_all (head); -} -END_TEST - static voidsearch_for_fail (const int rp,FILE *fp_stdout) {
@@ -350,7 +318,6 @@ utils_suite(void)tcase_add_test (tc_core, test_filter_ptests);tcase_add_test (tc_core, test_run_ptests);tcase_add_test (tc_core, test_run_timeout_duration_ptest); -tcase_add_test (tc_core, test_run_signal_ptest);tcase_add_test (tc_core, test_run_fail_ptest);tcase_add_test (tc_core, test_xml_pass);tcase_add_test (tc_core, test_xml_fail);
utils.c +102 -146
313 lines not on this page · raw message
Signature · 1 line
2.33.0
PATCH 2/4 Recreate pipe for each test
In reply to Joshua WattThe write end of the pipe has to be closed by ptest-runner to make sure that it will get EOF when the child process is done with it. This means that a new pipe needs to be opened for each child so that the write ends can be passed to it.
Fixes the problem where tests would be reported with no output when ptest-runner was told to run multiple tests because they were passed invalid stdin/stderr pipes.
--- utils.c | 45 ++++++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 21 deletions(-)
utils.c +24 -21
@@ -61,7 +61,6 @@ static struct {unsigned int timeout;int timeouted; -pid_t pid;int padding1; } _child_reader;
@@ -414,8 +413,8 @@ run_ptests(struct ptest_list *head, const struct ptest_options opts,char stime[GET_STIME_BUF_SIZE ];pid_t child; -int pipefd_stdout[2 ]; -int pipefd_stderr[2 ]; +int pipefd_stdout[2 ] = {-1 ,-1 }; +int pipefd_stderr[2 ] = {-1 ,-1 };time_t sttime, entime;time_t duration;int slave;
@@ -429,28 +428,22 @@ run_ptests(struct ptest_list *head, const struct ptest_options opts, do { - if ((rc = pipe2(pipefd_stdout, 0)) == -1) - break; - - if ((rc = pipe2(pipefd_stderr, 0)) == -1) { - close(pipefd_stdout[0]); - close(pipefd_stdout[1]); - break; - } - if (isatty(0) && ioctl(0, TIOCNOTTY) == -1) { fprintf(fp, "ERROR: Unable to detach from controlling tty, %s\n", strerror(errno)); } - _child_reader.fds[0] = pipefd_stdout[0]; - _child_reader.fds[1] = pipefd_stderr[0]; - _child_reader.fps[0] = fp; - _child_reader.fps[1] = fp_stderr; - _child_reader.timeout = opts.timeout; - _child_reader.timeouted = 0; fprintf(fp, "START: %s\n", progname); PTEST_LIST_ITERATE_START(head, p) + if ((rc = pipe2(pipefd_stdout, 0)) == -1) + break; + + if ((rc = pipe2(pipefd_stderr, 0)) == -1) { + close(pipefd_stdout[0]); + close(pipefd_stdout[1]); + break; + } + char *ptest_dir = strdup(p->run_ptest); if (ptest_dir == NULL) { rc = -1;
@@ -497,7 +490,12 @@ run_ptests(struct ptest_list *head, const struct ptest_options opts,do_close (&pipefd_stdout[1 ]);do_close (&pipefd_stderr[1 ]); - _child_reader.pid = child; + _child_reader.fds[0 ] = pipefd_stdout[0 ]; + _child_reader.fds[1 ] = pipefd_stderr[0 ]; + _child_reader.fps[0 ] = fp; + _child_reader.fps[1 ] = fp_stderr; + _child_reader.timeout = opts.timeout; + _child_reader.timeouted =0 ;if (setpgid (child, pgid) ==-1 ) {fprintf (fp,"ERROR: setpgid() failed, %s\n" ,strerror (errno)); }
@@ -543,7 +541,7 @@ run_ptests(struct ptest_list *head, const struct ptest_options opts,* the pipes until EOF to make * sure we get all the output */ -kill (-_child_reader.pid,SIGKILL ); +kill (-child,SIGKILL ); _child_reader.timeouted =1 ; }
@@ -585,7 +583,7 @@ run_ptests(struct ptest_list *head, const struct ptest_options opts,* dead */ if (!_child_reader.timeouted) { -kill (-_child_reader.pid,SIGKILL ); +kill (-child,SIGKILL ); } status =wait_child (child);
@@ -607,6 +605,11 @@ run_ptests(struct ptest_list *head, const struct ptest_options opts,fprintf (fp,"%s\n" ,get_stime (stime,GET_STIME_BUF_SIZE , entime)); }free (ptest_dir); +do_close (&pipefd_stdout[0 ]); +do_close (&pipefd_stdout[1 ]); +do_close (&pipefd_stderr[0 ]); +do_close (&pipefd_stderr[1 ]); + PTEST_LIST_ITERATE_END fprintf(fp, "STOP: %s\n", progname);
Signature · 1 line
2.33.0
PATCH 3/4 Report if child dies from a signal
In reply to Joshua WattTests can exit due to a signal, which should also be reported in the test output.
--- tests/data/signal/ptest/run-ptest | 10 ++++++++ tests/utils.c | 39 +++++++++++++++++++++++++++--- utils.c | 40 +++++++++++++++++-------------- 3 files changed, 68 insertions(+), 21 deletions(-) create mode 100755 tests/data/signal/ptest/run-ptest
tests/data/signal/ptest/run-ptest new file +10 -0
@@ -0,0 +1,10 @@ +#!/bin/sh + +echo "signal" 1>&2 +echo "signal" + +kill -ABRT $$ + +while true; do + sleep 1 +done
tests/utils.c +36 -3
@@ -50,9 +50,10 @@ static char *ptests_found[] = {"glibc" ,"hang" ,"python" , +"signal" ,NULL }; -static int ptests_found_length =6 ; +static int ptests_found_length =7 ;static char *ptests_not_found[] = {"busybox" , "perl",
@@ -186,6 +187,7 @@ START_TEST(test_run_ptests) head =get_available_ptests (opts_directory);ptest_list_remove (head,"hang" ,1 );ptest_list_remove (head,"fail" ,1 ); +ptest_list_remove (head,"signal" ,1 ); rc =run_ptests (head, opts,"test_run_ptests" , fp_stdout, fp_stderr);ck_assert (rc ==0 );
@@ -215,8 +217,8 @@ search_for_timeout_and_duration(const int rp, FILE *fp_stdout) found_duration = found_duration ? found_duration :find_word (line, duration_str); } -ck_assert (found_timeout == true); -ck_assert (found_duration == true); +ck_assert_msg (found_timeout == true,"TIMEOUT not found" ); +ck_assert_msg (found_duration == true,"DURATION not found" ); }START_TEST (test_run_timeout_duration_ptest)
@@ -230,6 +232,36 @@ START_TEST(test_run_timeout_duration_ptest) }END_TEST +static void +search_for_signal_and_duration (const int rp,FILE *fp_stdout) +{ +char line_buf[PRINT_PTEST_BUF_SIZE ]; +bool found_error = false, found_duration = false; +char *line =NULL ; + +ck_assert (rp !=0 ); + +while ((line =fgets (line_buf,PRINT_PTEST_BUF_SIZE , fp_stdout)) !=NULL ) { +// once true, stay true + found_error = found_error ? found_error :find_word (line,"ERROR: Exited from signal" ); + found_duration = found_duration ? found_duration :find_word (line,"DURATION" ); + } + +ck_assert_msg (found_error == true,"ERROR not found" ); +ck_assert_msg (found_duration == true,"DURATION not found" ); +} + +START_TEST (test_run_signal_ptest) +{ +struct ptest_list *head =get_available_ptests (opts_directory); +unsigned int timeout =10 ; + +test_ptest_expected_failure (head, timeout,"signal" , search_for_signal_and_duration); + +ptest_list_free_all (head); +} +END_TEST + static voidsearch_for_fail (const int rp,FILE *fp_stdout) {
@@ -318,6 +350,7 @@ utils_suite(void)tcase_add_test (tc_core, test_filter_ptests);tcase_add_test (tc_core, test_run_ptests);tcase_add_test (tc_core, test_run_timeout_duration_ptest); +tcase_add_test (tc_core, test_run_signal_ptest);tcase_add_test (tc_core, test_run_fail_ptest);tcase_add_test (tc_core, test_xml_pass);tcase_add_test (tc_core, test_xml_fail);
utils.c +22 -18
@@ -343,18 +343,6 @@ run_child(char *run_ptest, int fd_stdout, int fd_stderr)/* exit(1); not needed? */ } -static inline int -wait_child (pid_t pid) -{ -int status =-1 ; - -waitpid (pid, &status,0 ); -if (WIFEXITED (status)) - status =WEXITSTATUS (status); - -return status; -} -/* Returns an integer file descriptor. * If it returns < 0, an error has occurred. * Otherwise, it has returned the slave pty file descriptor.
@@ -585,21 +573,37 @@ run_ptests(struct ptest_list *head, const struct ptest_options opts,if (!_child_reader.timeouted) {kill (-child,SIGKILL ); } - status =wait_child (child); +waitpid (child, &status,0 ); entime =time (NULL ); duration = entime - sttime; -if (status) { -fprintf (fp,"\nERROR: Exit status is %d\n" , status); - rc +=1 ; +int exit_code =0 ; + +if (!_child_reader.timeouted) { +if (WIFEXITED (status)) { + exit_code =WEXITSTATUS (status); +if (exit_code) { +fprintf (fp,"\nERROR: Exit status is %d\n" , exit_code); + rc +=1 ; + } + }else if (WIFSIGNALED (status)) { +int signal =WTERMSIG (status); +fprintf (fp,"\nERROR: Exited from signal %s (%d)\n" ,strsignal (signal), signal); + rc +=1 ; + }else { +fprintf (fp,"\nERROR: Exited for unknown reason (%d)\n" , status); + rc +=1 ; + } }fprintf (fp,"DURATION: %d\n" , (int ) duration); -if (_child_reader.timeouted) +if (_child_reader.timeouted) {fprintf (fp,"TIMEOUT: %s\n" , ptest_dir); + rc +=1 ; + }if (opts.xml_filename) -xml_add_case (xh, status, ptest_dir, _child_reader.timeouted, (int ) duration); +xml_add_case (xh, exit_code, ptest_dir, _child_reader.timeouted, (int ) duration);fprintf (fp,"END: %s\n" , ptest_dir);fprintf (fp,"%s\n" ,get_stime (stime,GET_STIME_BUF_SIZE , entime));
Signature · 1 line
2.33.0
PATCH 4/4 Change test timeout to be total elapsed time
In reply to Joshua WattChanges the way that tests time out to be the total elapsed time for the test, not just the time between receiving output from the test. This matches the implementation before 8259375 ("runner: Remove threads and mutexes").
Also update the timeout test case to test for this correctly.
--- tests/data/hang/ptest/run-ptest | 1 + tests/utils.c | 2 +- utils.c | 23 ++++++++++++----------- 3 files changed, 14 insertions(+), 12 deletions(-)
tests/data/hang/ptest/run-ptest +1 -0
@@ -3,5 +3,6 @@ echo "hang" 1>&2 echo "hang" while true; do + echo "hang" sleep 1 done
tests/utils.c +1 -1
@@ -224,7 +224,7 @@ search_for_timeout_and_duration(const int rp, FILE *fp_stdout)START_TEST (test_run_timeout_duration_ptest) {struct ptest_list *head =get_available_ptests (opts_directory); -unsigned int timeout =1 ; +unsigned int timeout =3 ;test_ptest_expected_failure (head, timeout,"hang" , search_for_timeout_and_duration);
utils.c +12 -11
@@ -403,7 +403,8 @@ run_ptests(struct ptest_list *head, const struct ptest_options opts,pid_t child;int pipefd_stdout[2 ] = {-1 ,-1 };int pipefd_stderr[2 ] = {-1 ,-1 }; -time_t sttime, entime; +time_t sttime, entime, now; +time_t timeout_deadline;time_t duration;int slave;int pgid =-1 ;
@@ -489,6 +490,7 @@ run_ptests(struct ptest_list *head, const struct ptest_options opts, } sttime =time (NULL ); + timeout_deadline = sttime + opts.timeout;fprintf (fp,"%s\n" ,get_stime (stime,GET_STIME_BUF_SIZE , sttime));fprintf (fp,"BEGIN: %s\n" , ptest_dir);
@@ -519,18 +521,17 @@ run_ptests(struct ptest_list *head, const struct ptest_options opts,if (done) {break ; } - -int ret =poll (pfds,2 , _child_reader.timeout*1000 ); - -if (ret ==0 && !_child_reader.timeouted) { -/* kill the child if we haven't -* already. Note that we -* continue to read data from -* the pipes until EOF to make -* sure we get all the output -*/ + now =time (NULL ); +if (now >= timeout_deadline) {kill (-child,SIGKILL ); _child_reader.timeouted =1 ; +break ; + } + +int ret =poll (pfds,2 , (timeout_deadline - now) *1000 ); + +if (ret ==0 ) { +continue ; }for (int i =0 ; i <2 ; i++) {
Signature · 1 line
2.33.0