Skip to contentSkip to conversation

ptest-runner: Add support to report duration of each ptest

PATCH In review Aníbal Limón 2 participants 4 messages Jan 7, 2019 – Jan 9, 2019

lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 ptest-runner: Add support to report duration of each ptest

In stdout reported as,

...
BEGIN: ptest-dir
...
DURATION: N
END: ptest-dir
...

In XML reported as,

...
<testcase classname='test1' name='run-ptest'>
<duration>N</duration>
</testcase>
...

---
 tests/data/reference.xml |  2 ++
 tests/utils.c            | 20 ++++++++++++--------
 utils.c                  | 19 ++++++++++++-------
 utils.h                  |  2 +-
 4 files changed, 27 insertions(+), 16 deletions(-)
tests/data/reference.xml +2 -0
index 17f91c2..f6ff389 100644
@@ -1,8 +1,10 @@
 <?xml version='1.0' encoding='UTF-8'?>
 <testsuite name='ptest' tests='2'>
 	<testcase classname='test1' name='run-ptest'>
+		<duration>5</duration>
 	</testcase>
 	<testcase classname='test2' name='run-ptest'>
+		<duration>10</duration>
 		<failure type='exit_code' message='run-ptest exited with code: 1'></failure>
 		<failure type='timeout'/>
 	</testcase>
tests/utils.c +12 -8
index 662abe8..2ccb1c0 100644
@@ -188,26 +188,30 @@ START_TEST(test_run_ptests)
 END_TEST
 
 static void
-search_for_timeout(const int rp, FILE *fp_stdout, FILE *fp_stderr)
+search_for_timeout_and_duration(const int rp, FILE *fp_stdout, FILE *fp_stderr)
 {
 	const char *timeout_str = "TIMEOUT";
+	const char *duration_str = "DURATION";
 	char line_buf[PRINT_PTEST_BUF_SIZE];
-	int found_timeout = 0;
+	int found_timeout = 0, found_duration = 0;
 	char *line = NULL;
 
 	ck_assert(rp != 0);
 
-	while ((line = fgets(line_buf, PRINT_PTEST_BUF_SIZE, fp_stdout)) != NULL)
+	while ((line = fgets(line_buf, PRINT_PTEST_BUF_SIZE, fp_stdout)) != NULL) {
 		find_word(&found_timeout, line, timeout_str);
+		find_word(&found_duration, line, duration_str);
+	}
 
 	ck_assert(found_timeout == 1);
+	ck_assert(found_duration == 1);
 }
 
-START_TEST(test_run_timeout_ptest)
+START_TEST(test_run_timeout_duration_ptest)
 	struct ptest_list *head = get_available_ptests(opts_directory);
 	int timeout = 1;
 
-	test_ptest_expected_failure(head, timeout, "hang", search_for_timeout);
+	test_ptest_expected_failure(head, timeout, "hang", search_for_timeout_and_duration);
 
 	ptest_list_free_all(head);
 END_TEST
@@ -257,8 +261,8 @@ START_TEST(test_xml_pass)
 	FILE *xp;
 	xp = xml_create(2, "./test.xml");
 	ck_assert(xp != NULL);
-	xml_add_case(xp, 0,"test1", 0);
-	xml_add_case(xp, 1,"test2", 1);
+	xml_add_case(xp, 0,"test1", 0, 5);
+	xml_add_case(xp, 1,"test2", 1, 10);
 	xml_finish(xp);
 
 	FILE *fp, *fr;
@@ -291,7 +295,7 @@ utils_suite()
 	tcase_add_test(tc_core, test_print_ptests);
 	tcase_add_test(tc_core, test_filter_ptests);
 	tcase_add_test(tc_core, test_run_ptests);
-	tcase_add_test(tc_core, test_run_timeout_ptest);
+	tcase_add_test(tc_core, test_run_timeout_duration_ptest);
 	tcase_add_test(tc_core, test_run_fail_ptest);
 	tcase_add_test(tc_core, test_xml_pass);
 	tcase_add_test(tc_core, test_xml_fail);
utils.c +12 -7
index 4a38ea1..01ee415 100644
@@ -45,12 +45,10 @@
 #define WAIT_CHILD_BUF_MAX_SIZE 1024
 
 static inline char *
-get_stime(char *stime, size_t size)
+get_stime(char *stime, size_t size, time_t t)
 {
-	time_t t;
 	struct tm *lt;
 
-	t = time(NULL);
 	lt = localtime(&t);
 	strftime(stime, size, "%Y-%m-%dT%H:%M", lt);
 
@@ -334,6 +332,8 @@ run_ptests(struct ptest_list *head, const struct ptest_options opts,
 	int pipefd_stdout[2];
 	int pipefd_stderr[2];
 	int timeouted;
+	time_t sttime, entime;
+	int duration;
 
 	if (opts.xml_filename) {
 		xh = xml_create(ptest_list_length(head), opts.xml_filename);
@@ -373,11 +373,15 @@ run_ptests(struct ptest_list *head, const struct ptest_options opts,
 				int fds[2]; fds[0] = pipefd_stdout[0]; fds[1] = pipefd_stderr[0];
 				FILE *fps[2]; fps[0] = fp; fps[1] = fp_stderr;
 
-				fprintf(fp, "%s\n", get_stime(stime, GET_STIME_BUF_SIZE));
+				sttime = time(NULL);
+				fprintf(fp, "%s\n", get_stime(stime, GET_STIME_BUF_SIZE, sttime));
 				fprintf(fp, "BEGIN: %s\n", ptest_dir);
 
 				status = wait_child(ptest_dir, p->run_ptest, child,
 						opts.timeout, fds, fps, &timeouted);
+				entime = time(NULL);
+				duration = entime - sttime;
+				fprintf(fps[0], "DURATION: %d\n", duration);
 
 				if (status) {
 					fprintf(fps[0], "\nERROR: Exit status is %d\n", status);
@@ -387,10 +391,10 @@ run_ptests(struct ptest_list *head, const struct ptest_options opts,
 					fprintf(fps[0], "TIMEOUT: %s\n", ptest_dir);
 
 				if (opts.xml_filename)
-					xml_add_case(xh, status, ptest_dir, timeouted);
+					xml_add_case(xh, status, ptest_dir, timeouted, duration);
 
 				fprintf(fp, "END: %s\n", ptest_dir);
-				fprintf(fp, "%s\n", get_stime(stime, GET_STIME_BUF_SIZE));
+				fprintf(fp, "%s\n", get_stime(stime, GET_STIME_BUF_SIZE, entime));
 			}
 		PTEST_LIST_ITERATE_END;
 		fprintf(fp, "STOP: %s\n", progname);
@@ -426,9 +430,10 @@ xml_create(int test_count, char *xml_filename)
 }
 
 void
-xml_add_case(FILE *xh, int status, const char *ptest_dir, int timeouted)
+xml_add_case(FILE *xh, int status, const char *ptest_dir, int timeouted, int duration)
 {
 	fprintf(xh, "\t<testcase classname='%s' name='run-ptest'>\n", ptest_dir);
+	fprintf(xh, "\t\t<duration>%d</duration>\n", duration);
 
 	if (status != 0) {
 		fprintf(xh, "\t\t<failure type='exit_code'");
utils.h +1 -1
index 880105f..5a3f44a 100644
@@ -48,7 +48,7 @@ extern int run_ptests(struct ptest_list *, const struct ptest_options,
 		const char *, FILE *, FILE *);
 
 extern FILE *xml_create(int, char *);
-extern void xml_add_case(FILE *, int, const char *, int);
+extern void xml_add_case(FILE *, int, const char *, int, int);
 extern void xml_finish(FILE *);
 
 #endif
Signature · 1 line
2.20.1
#1 lore raw
Quoted 17 lines · Aníbal Limón

In stdout reported as,

...
BEGIN: ptest-dir
...
DURATION: N
END: ptest-dir
...

In XML reported as,

...
<testcase classname='test1' name='run-ptest'>
<duration>N</duration>
</testcase>
...

Quoted 7 lines
---
 tests/data/reference.xml |  2 ++
 tests/utils.c            | 20 ++++++++++++--------
 utils.c                  | 19 ++++++++++++-------
 utils.h                  |  2 +-
 4 files changed, 27 insertions(+), 16 deletions(-)

I've not looked at the code detail but the above looks good to me and will significantly help ptest usage! :)

Once this is in are you able to help update the recipe and pass this data through to the the oe-selftest json report?

Cheers,

Richard

#2 lore raw
Quoted 26 lines · On Tue, 8 Jan 2019 at 14:00, <[email protected]> wrote:
Quoted 17 lines · Aníbal Limón

In stdout reported as,

...
BEGIN: ptest-dir
...
DURATION: N
END: ptest-dir
...

In XML reported as,

...
<testcase classname='test1' name='run-ptest'>
<duration>N</duration>
</testcase>
...

Quoted 7 lines
---
 tests/data/reference.xml |  2 ++
 tests/utils.c            | 20 ++++++++++++--------
 utils.c                  | 19 ++++++++++++-------
 utils.h                  |  2 +-
 4 files changed, 27 insertions(+), 16 deletions(-)
Quoted 6 lines

I've not looked at the code detail but the above looks good to me and will significantly help ptest usage! :)

Once this is in are you able to help update the recipe and pass this data through to the the oe-selftest json report?

I will send the recipe upgrade, What is the selftest json report?, I'm not aware.

Regards,
Anibal

Quoted 6 lines

Cheers,

Richard

#3 lore raw
Quoted 34 lines · Anibal Limon
Quoted 30 lines · On Tue, 8 Jan 2019 at 14:00, <[email protected]> wrote:
Quoted 17 lines · Aníbal Limón

In stdout reported as,

...
BEGIN: ptest-dir
...
DURATION: N
END: ptest-dir
...

In XML reported as,

...
<testcase classname='test1' name='run-ptest'>
<duration>N</duration>
</testcase>
...

Quoted 7 lines
---
 tests/data/reference.xml |  2 ++
 tests/utils.c            | 20 ++++++++++++--------
 utils.c                  | 19 ++++++++++++-------
 utils.h                  |  2 +-
 4 files changed, 27 insertions(+), 16 deletions(-)

I've not looked at the code detail but the above looks good to me and
will significantly help ptest usage! :)

Quoted 6 lines
Quoted 3 lines

Once this is in are you able to help update the recipe and pass this
data through to the the oe-selftest json report?

I will send the recipe upgrade, What is the selftest json report?, I'm not aware.

Sorry, its not from selftest, its from testimage running against an image where ptests are present. It generates something like:

https://autobuilder.yocto.io/pub/releases/yocto-2.7_M1.rc1/testresults/qemux86-64-ptest/testresults.json

Cheers,

Richard