selftests/bpf: add --json-summary option to test_progs
Subsystems (3)
PATCH v2 1/1 selftests/bpf: add --json-summary option to test_progs
Currently, test_progs outputs all stdout/stderr as it runs, and when it is done, prints a summary.
It is non-trivial for tooling to parse that output and extract meaningful information from it.
This change adds a new option, --json-summary/-J that let the caller specify a file where test_progs{,-no_alu32} can write a summary of the run in a json format that can later be parsed by tooling.
Currently, it creates a summary section with successes/skipped/failures followed by a list of failed tests and subtests.
A test contains the following fields:
- name: the name of the test
- number: the number of the test
- message: the log message that was printed by the test.
- failed: A boolean indicating whether the test failed or not. Currently we only output failed tests, but in the future, successful tests could be added.
- subtests: A list of subtests associated with this test.
A subtest contains the following fields:
- name: same as above
- number: sanme as above
- message: the log message that was printed by the subtest.
- failed: same as above but for the subtest
An example run and json content below:
```
$ sudo ./test_progs -a $(grep -v '^#' ./DENYLIST.aarch64 | awk '{print
$1","}' | tr -d '\n') -j -J /tmp/test_progs.json
$ jq < /tmp/test_progs.json | head -n 30
{
"success": 29,
"success_subtest": 23,
"skipped": 3,
"failed": 28,
"results": [
{
"name": "bpf_cookie",
"number": 10,
"message": "test_bpf_cookie:PASS:skel_open 0 nsec\n",
"failed": true,
"subtests": [
{
"name": "multi_kprobe_link_api",
"number": 2,
"message": "kprobe_multi_link_api_subtest:PASS:load_kallsyms 0
nsec\nlibbpf: extern 'bpf_testmod_fentry_test1' (strong): not
resolved\nlibbpf: failed to load object 'kprobe_multi'\nlibbpf: failed
to load BPF skeleton 'kprobe_multi':
-3\nkprobe_multi_link_api_subtest:FAIL:fentry_raw_skel_load unexpected
error: -3\n",
"failed": true
},
{
"name": "multi_kprobe_attach_api",
"number": 3,
"message": "libbpf: extern 'bpf_testmod_fentry_test1'
(strong): not resolved\nlibbpf: failed to load object
'kprobe_multi'\nlibbpf: failed to load BPF skeleton 'kprobe_multi':
-3\nkprobe_multi_attach_api_subtest:FAIL:fentry_raw_skel_load unexpected
error: -3\n",
"failed": true
},
{
"name": "lsm",
"number": 8,
"message": "lsm_subtest:PASS:lsm.link_create 0
nsec\nlsm_subtest:FAIL:stack_mprotect unexpected stack_mprotect: actual
0 != expected -1\n",
"failed": true
}
```The file can then be used to print a summary of the test run and list of failing tests/subtests:
```
$ jq -r < /tmp/test_progs.json '"Success:
\(.success)/\(.success_subtest), Skipped: \(.skipped), Failed:
\(.failed)"'
Success: 29/23, Skipped: 3, Failed: 28
$ jq -r < /tmp/test_progs.json '.results | map([
if .failed then "#\(.number) \(.name)" else empty end,
(
. as {name: $tname, number: $tnum} | .subtests | map(
if .failed then "#\($tnum)/\(.number) \($tname)/\(.name)"
else empty end
)
)
]) | flatten | .[]' | head -n 20
#10 bpf_cookie
#10/2 bpf_cookie/multi_kprobe_link_api
#10/3 bpf_cookie/multi_kprobe_attach_api
#10/8 bpf_cookie/lsm
#15 bpf_mod_race
#15/1 bpf_mod_race/ksym (used_btfs UAF)
#15/2 bpf_mod_race/kfunc (kfunc_btf_tab UAF)
#36 cgroup_hierarchical_stats
#61 deny_namespace
#61/1 deny_namespace/unpriv_userns_create_no_bpf
#73 fexit_stress
#83 get_func_ip_test
#99 kfunc_dynptr_param
#99/1 kfunc_dynptr_param/dynptr_data_null
#99/4 kfunc_dynptr_param/dynptr_data_null
#100 kprobe_multi_bench_attach
#100/1 kprobe_multi_bench_attach/kernel
#100/2 kprobe_multi_bench_attach/modules
#101 kprobe_multi_test
#101/1 kprobe_multi_test/skel_api
```---
v2:
* use test_failed, subtest_failed to populate "failed" field.
* Move to nested structure where subtests are added to the subtests array of a test.
* removed test/subtest prefixes now that an object identify either of them.
* addressed nits (comment, declaration at top of function)
* do not pretty output
--- tools/testing/selftests/bpf/Makefile | 4 +- tools/testing/selftests/bpf/json_writer.c | 1 + tools/testing/selftests/bpf/json_writer.h | 1 + tools/testing/selftests/bpf/test_progs.c | 86 +++++++++++++++++++++-- tools/testing/selftests/bpf/test_progs.h | 1 + 5 files changed, 87 insertions(+), 6 deletions(-) create mode 120000 tools/testing/selftests/bpf/json_writer.c create mode 120000 tools/testing/selftests/bpf/json_writer.h
tools/testing/selftests/bpf/Makefile +3 -1
@@ -234,6 +234,7 @@ $(TEST_GEN_PROGS) $(TEST_GEN_PROGS_EXTENDED): $(BPFOBJ) CGROUP_HELPERS := $(OUTPUT)/cgroup_helpers.o TESTING_HELPERS := $(OUTPUT)/testing_helpers.o TRACE_HELPERS := $(OUTPUT)/trace_helpers.o +JSON_WRITER := $(OUTPUT)/json_writer.o CAP_HELPERS := $(OUTPUT)/cap_helpers.o $(OUTPUT)/test_dev_cgroup: $(CGROUP_HELPERS) $(TESTING_HELPERS)
@@ -559,7 +560,8 @@ TRUNNER_BPF_PROGS_DIR := progs TRUNNER_EXTRA_SOURCES := test_progs.c cgroup_helpers.c trace_helpers.c \ network_helpers.c testing_helpers.c \ btf_helpers.c flow_dissector_load.h \ - cap_helpers.c test_loader.c xsk.c disasm.c + cap_helpers.c test_loader.c xsk.c disasm.c \ + json_writer.c TRUNNER_EXTRA_FILES := $(OUTPUT)/urandom_read $(OUTPUT)/bpf_testmod.ko \ $(OUTPUT)/liburandom_read.so \ $(OUTPUT)/xdp_synproxy \
tools/testing/selftests/bpf/json_writer.c new file +1 -0
@@ -0,0 +1 @@ +../../../bpf/bpftool/json_writer.c
tools/testing/selftests/bpf/json_writer.h new file +1 -0
@@ -0,0 +1 @@ +../../../bpf/bpftool/json_writer.h
tools/testing/selftests/bpf/test_progs.c +81 -5
174 lines not on this page · raw message
tools/testing/selftests/bpf/test_progs.h +1 -0
@@ -114,6 +114,7 @@ struct test_env {FILE *stdout;FILE *stderr;int nr_cpus; +FILE *json;int succ_cnt;/* successful tests */ int sub_succ_cnt;/* successful sub-tests */
Signature · 1 line
2.34.1
Andrii Nakryiko
AppliedQuoted 65 lines · Manu Bretelle
Currently,
test_progsoutputs all stdout/stderr as it runs, and when it is done, prints a summary.It is non-trivial for tooling to parse that output and extract meaningful information from it.
This change adds a new option,
--json-summary/-Jthat let the caller specify a file wheretest_progs{,-no_alu32}can write a summary of the run in a json format that can later be parsed by tooling.Currently, it creates a summary section with successes/skipped/failures followed by a list of failed tests and subtests.
A test contains the following fields:
- name: the name of the test
- number: the number of the test
- message: the log message that was printed by the test.
- failed: A boolean indicating whether the test failed or not. Currently we only output failed tests, but in the future, successful tests could be added.
- subtests: A list of subtests associated with this test.A subtest contains the following fields:
- name: same as above
- number: sanme as above
- message: the log message that was printed by the subtest.
- failed: same as above but for the subtestAn example run and json content below:
```$ sudo ./test_progs -a $(grep -v '^#' ./DENYLIST.aarch64 | awk '{print $1","}' | tr -d '\n') -j -J /tmp/test_progs.json $ jq < /tmp/test_progs.json | head -n 30 { "success": 29, "success_subtest": 23, "skipped": 3, "failed": 28, "results": [ { "name": "bpf_cookie", "number": 10, "message": "test_bpf_cookie:PASS:skel_open 0 nsec\n", "failed": true, "subtests": [ { "name": "multi_kprobe_link_api", "number": 2, "message": "kprobe_multi_link_api_subtest:PASS:load_kallsyms 0 nsec\nlibbpf: extern 'bpf_testmod_fentry_test1' (strong): not resolved\nlibbpf: failed to load object 'kprobe_multi'\nlibbpf: failed to load BPF skeleton 'kprobe_multi': -3\nkprobe_multi_link_api_subtest:FAIL:fentry_raw_skel_load unexpected error: -3\n", "failed": true }, { "name": "multi_kprobe_attach_api", "number": 3, "message": "libbpf: extern 'bpf_testmod_fentry_test1' (strong): not resolved\nlibbpf: failed to load object 'kprobe_multi'\nlibbpf: failed to load BPF skeleton 'kprobe_multi': -3\nkprobe_multi_attach_api_subtest:FAIL:fentry_raw_skel_load unexpected error: -3\n",
Quoted 8 lines
"failed": true
},
{
"name": "lsm",
"number": 8,
"message": "lsm_subtest:PASS:lsm.link_create 0
nsec\nlsm_subtest:FAIL:stack_mprotect unexpected stack_mprotect: actual
0 != expected -1\n",Undid this wrapping of message strings, and command line examples below. See also note about extra empty lines below.
Applied to bpf-next. So looking forward for this to be used in BPF CI!
Quoted 64 lines
"failed": true } ``` The file can then be used to print a summary of the test run and list of failing tests/subtests: ``` $ jq -r < /tmp/test_progs.json '"Success: \(.success)/\(.success_subtest), Skipped: \(.skipped), Failed: \(.failed)"'Success: 29/23, Skipped: 3, Failed: 28
$ jq -r < /tmp/test_progs.json '.results | map([ if .failed then "#\(.number) \(.name)" else empty end, ( . as {name: $tname, number: $tnum} | .subtests | map( if .failed then "#\($tnum)/\(.number) \($tname)/\(.name)" else empty end ) ) ]) | flatten | .[]' | head -n 20 #10 bpf_cookie #10/2 bpf_cookie/multi_kprobe_link_api #10/3 bpf_cookie/multi_kprobe_attach_api #10/8 bpf_cookie/lsm #15 bpf_mod_race #15/1 bpf_mod_race/ksym (used_btfs UAF) #15/2 bpf_mod_race/kfunc (kfunc_btf_tab UAF) #36 cgroup_hierarchical_stats #61 deny_namespace #61/1 deny_namespace/unpriv_userns_create_no_bpf #73 fexit_stress #83 get_func_ip_test #99 kfunc_dynptr_param #99/1 kfunc_dynptr_param/dynptr_data_null #99/4 kfunc_dynptr_param/dynptr_data_null #100 kprobe_multi_bench_attach #100/1 kprobe_multi_bench_attach/kernel #100/2 kprobe_multi_bench_attach/modules #101 kprobe_multi_test #101/1 kprobe_multi_test/skel_api ```Signed-off-by Manu Bretelle <[email protected]>---
v2:
* usetest_failed,subtest_failedto populate "failed" field.
* Move to nested structure where subtests are added to the subtests array of a test.
* removed test/subtest prefixes now that an object identify either of them.
* addressed nits (comment, declaration at top of function)
* do not pretty output--- tools/testing/selftests/bpf/Makefile | 4 +- tools/testing/selftests/bpf/json_writer.c | 1 + tools/testing/selftests/bpf/json_writer.h | 1 + tools/testing/selftests/bpf/test_progs.c | 86 +++++++++++++++++++++-- tools/testing/selftests/bpf/test_progs.h | 1 + 5 files changed, 87 insertions(+), 6 deletions(-) create mode 120000 tools/testing/selftests/bpf/json_writer.c create mode 120000 tools/testing/selftests/bpf/json_writer.h
[...]
Quoted 12 lines
tools/testing/selftests/bpf/test_progs.c +8
-0@@ -314,8 +338,24 @@ static void dump_test_log(const struct prog_test_def *test, test->test_name, subtest_state->name,test_result (subtest_state->error_cnt, subtest_state->skipped)); + +if (w && print_subtest) { +jsonw_start_object (w); +jsonw_string_field (w,"name" , subtest_state->name); +jsonw_uint_field (w,"number" , i+1 ); +jsonw_write_log_message (w, subtest_state->log_buf, subtest_state->log_cnt); +jsonw_bool_field (w,"failed" , subtest_failed); +jsonw_end_object (w);
tools/testing/selftests/bpf/test_progs.c static void dump_test_log(const struct prog_test_def *test, · line 358
Quoted 10 lines
tools/testing/selftests/bpf/test_progs.c +8
-0+ } + + } + +if (w && print_test) { +jsonw_end_array (w); +jsonw_end_object (w); } +
Undid this unnecessary line, same in few other places. Please don't add unnecessary whitespaces for future patches.
Quoted 3 lines
tools/testing/selftests/bpf/test_progs.c +0
-0print_test_result (test, test_state); }
[...]
This patch was applied to bpf/bpf-next.git (master) by Andrii Nakryiko <[email protected]>:
Hello:
This patch was applied to bpf/bpf-next.git (master) by Andrii Nakryiko <[email protected]>:
Quoted 11 lines · On Fri, 17 Mar 2023 09:32:56 -0700 you wrote:
Currently,
test_progsoutputs all stdout/stderr as it runs, and when it is done, prints a summary.It is non-trivial for tooling to parse that output and extract meaningful information from it.
This change adds a new option,
--json-summary/-Jthat let the caller specify a file wheretest_progs{,-no_alu32}can write a summary of the run in a json format that can later be parsed by tooling.[...]
Here is the summary with links:
- [bpf-next,v2] selftests/bpf: add --json-summary option to test_progs
https://git.kernel.org/bpf/bpf-next/c/2be7aa76cc69
You are awesome, thank you!
Signature · 2 lines
Deet-doot-dot, I am a bot. https://korg.docs.kernel.org/patchwork/pwbot.html