Add support for a test tags
cover Add support for a test tags
From: Cyril Hrubis <[email protected]>
This is a newest version of the test tags, first patch adds a support for test tags to a test library, the second one annotates tests with tags. Not everything is annotated at the moment but fair amount of tests is included.
These patches are also a stepping stone for the test metadata project that will allow us to do much more than this. You can have a look at:
https://github.com/metan-ucw/ltp/tree/master/docparse
Also if you run a make in the docparse directory it will generate test metadata along with a single html page with nice test metadata description.
While the whole project is work-in-progress, the test tags are finished and may add a value even without the rest of the changes, see the first commit for details.
Cyril Hrubis (2):
lib: Add support for test tags
testcases/kernel: Add tags
include/tst_test.h | 10 +++ lib/tst_test.c | 77 +++++++++++++++++-- testcases/cve/cve-2014-0196.c | 5 ++ testcases/cve/cve-2015-3290.c | 5 ++ testcases/cve/cve-2016-10044.c | 5 ++ testcases/cve/cve-2016-7042.c | 5 ++ testcases/cve/cve-2016-7117.c | 5 ++ testcases/cve/cve-2017-16939.c | 5 ++ testcases/cve/cve-2017-17052.c | 5 ++ testcases/cve/cve-2017-17053.c | 5 ++ testcases/cve/cve-2017-2618.c | 5 ++ testcases/cve/cve-2017-2671.c | 5 ++ testcases/cve/meltdown.c | 6 +- testcases/cve/stack_clash.c | 4 + testcases/kernel/crypto/af_alg01.c | 5 ++ testcases/kernel/crypto/af_alg02.c | 5 ++ testcases/kernel/crypto/af_alg03.c | 4 + testcases/kernel/crypto/af_alg04.c | 4 + testcases/kernel/crypto/af_alg05.c | 8 +- testcases/kernel/crypto/af_alg06.c | 4 + testcases/kernel/crypto/crypto_user01.c | 6 ++ testcases/kernel/crypto/crypto_user02.c | 4 + testcases/kernel/crypto/pcrypt_aead01.c | 5 ++ testcases/kernel/sound/snd_timer01.c | 6 ++ testcases/kernel/syscalls/accept/accept02.c | 5 ++ testcases/kernel/syscalls/acct/acct02.c | 4 + testcases/kernel/syscalls/add_key/add_key02.c | 5 ++ testcases/kernel/syscalls/bpf/bpf_prog02.c | 4 + .../syscalls/futex/futex_cmp_requeue02.c | 5 ++ testcases/kernel/syscalls/keyctl/keyctl02.c | 4 + testcases/kernel/syscalls/keyctl/keyctl04.c | 5 ++ testcases/kernel/syscalls/keyctl/keyctl05.c | 4 + .../kernel/syscalls/move_pages/move_pages12.c | 6 ++ .../kernel/syscalls/realpath/realpath01.c | 4 + .../syscalls/request_key/request_key05.c | 4 + testcases/kernel/syscalls/sbrk/sbrk03.c | 4 + testcases/kernel/syscalls/sendto/sendto02.c | 4 + .../kernel/syscalls/setrlimit/setrlimit06.c | 4 + 38 files changed, 251 insertions(+), 9 deletions(-)
Signature · 1 line
2.21.0
PATCH 1/2 lib: Add support for test tags
From: Cyril Hrubis <[email protected]>
The newly introduced test tags are generic name-value pairs that can hold test metadata, the intended use for now is to store kernel commit hashes for kernel reproducers as well as CVE ids. The mechanism is however choosen to be very generic so that it's easy to add basically any information later on.
As it is the main purpose is to print hints for a test failures. If a test that has been written as a kernel reproducer fails it prints nice URL pointing to a kernel commit that may be missing.
Example output:
--------------------------------------------------------------------------
tst_test.c:1145: INFO: Timeout per run is 0h 05m 00s add_key02.c:98: FAIL: unexpected error with key type 'asymmetric': EINVAL add_key02.c:98: FAIL: unexpected error with key type 'cifs.idmap': EINVAL add_key02.c:98: FAIL: unexpected error with key type 'cifs.spnego': EINVAL add_key02.c:98: FAIL: unexpected error with key type 'pkcs7_test': EINVAL add_key02.c:98: FAIL: unexpected error with key type 'rxrpc': EINVAL add_key02.c:98: FAIL: unexpected error with key type 'rxrpc_s': EINVAL add_key02.c:98: FAIL: unexpected error with key type 'user': EINVAL add_key02.c:98: FAIL: unexpected error with key type 'logon': EINVAL
HINT: This is a regression test for linux kernel, see commit:
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=5649645d725c
HINT: This test also tests for CVE-2017-15274, see:
https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-15274
Summary:
passed 0 failed 8 skipped 0
warnings 0
--------------------------------------------------------------------------
--- include/tst_test.h | 10 ++++++ lib/tst_test.c | 77 +++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 80 insertions(+), 7 deletions(-)
include/tst_test.h +10 -0
@@ -108,6 +108,11 @@ int tst_parse_int(const char *str, int *val, int min, int max);int tst_parse_long (const char *str,long *val,long min,long max);int tst_parse_float (const char *str,float *val,float min,float max); +struct tst_tag { +const char *name; +const char *value; +}; +extern unsigned int tst_variant;struct tst_test {
@@ -212,6 +217,11 @@ struct tst_test {* NULL-terminated array of capability settings */ struct tst_cap *caps; + +/* +* {NULL, NULL} terminated array of tags. +*/ +const struct tst_tag *tags; };/*
lib/tst_test.c +70 -7
120 lines not on this page · raw message
Signature · 1 line
2.21.0
PATCH 2/2 testcases/kernel: Add linux git and CVE tags
In reply to Cyril HrubisFrom: Cyril Hrubis <[email protected]>
Not all tests are annotated at the moment but the majority has been finished for now.
--- testcases/cve/cve-2014-0196.c | 5 +++++ testcases/cve/cve-2015-3290.c | 5 +++++ testcases/cve/cve-2016-10044.c | 5 +++++ testcases/cve/cve-2016-7042.c | 5 +++++ testcases/cve/cve-2016-7117.c | 5 +++++ testcases/cve/cve-2017-16939.c | 5 +++++ testcases/cve/cve-2017-17052.c | 5 +++++ testcases/cve/cve-2017-17053.c | 5 +++++ testcases/cve/cve-2017-2618.c | 5 +++++ testcases/cve/cve-2017-2671.c | 5 +++++ testcases/cve/meltdown.c | 6 +++++- testcases/cve/stack_clash.c | 4 ++++ testcases/kernel/crypto/af_alg01.c | 5 +++++ testcases/kernel/crypto/af_alg02.c | 5 +++++ testcases/kernel/crypto/af_alg03.c | 4 ++++ testcases/kernel/crypto/af_alg04.c | 4 ++++ testcases/kernel/crypto/af_alg05.c | 8 +++++++- testcases/kernel/crypto/af_alg06.c | 4 ++++ testcases/kernel/crypto/crypto_user01.c | 6 ++++++ testcases/kernel/crypto/crypto_user02.c | 4 ++++ testcases/kernel/crypto/pcrypt_aead01.c | 5 +++++ testcases/kernel/sound/snd_timer01.c | 6 ++++++ testcases/kernel/syscalls/accept/accept02.c | 5 +++++ testcases/kernel/syscalls/acct/acct02.c | 4 ++++ testcases/kernel/syscalls/add_key/add_key02.c | 5 +++++ testcases/kernel/syscalls/bpf/bpf_prog02.c | 4 ++++ testcases/kernel/syscalls/futex/futex_cmp_requeue02.c | 5 +++++ testcases/kernel/syscalls/keyctl/keyctl02.c | 4 ++++ testcases/kernel/syscalls/keyctl/keyctl04.c | 5 +++++ testcases/kernel/syscalls/keyctl/keyctl05.c | 4 ++++ testcases/kernel/syscalls/move_pages/move_pages12.c | 6 ++++++ testcases/kernel/syscalls/realpath/realpath01.c | 4 ++++ testcases/kernel/syscalls/request_key/request_key05.c | 4 ++++ testcases/kernel/syscalls/sbrk/sbrk03.c | 4 ++++ testcases/kernel/syscalls/sendto/sendto02.c | 4 ++++ testcases/kernel/syscalls/setrlimit/setrlimit06.c | 4 ++++ 36 files changed, 171 insertions(+), 2 deletions(-)
testcases/cve/cve-2014-0196.c +5 -0
@@ -141,4 +141,9 @@ static struct tst_test test = { .setup = setup, .cleanup = cleanup, .test_all = run, + .tags = (const struct tst_tag []) { + {"linux-git" ,"4291086b1f08" }, + {"CVE" ,"2014-0196" }, + {} + } };
testcases/cve/cve-2015-3290.c +5 -0
@@ -469,6 +469,11 @@ static struct tst_test test = { .setup = setup, .timeout = TIMEOUT, .test_all = run, + .tags = (const struct tst_tag []) { + {"linux-git" ,"9b6e6a8334d5" }, + {"CVE" ,"2015-3290" }, + {} + } };#else /* HAVE_PERF_EVENT_ATTR && (defined(__x86_64__) || defined(__i386__)) */
testcases/cve/cve-2016-10044.c +5 -0
@@ -62,4 +62,9 @@ static struct tst_test test = { .test_all = run, .cleanup = cleanup, .min_kver = "2.6.8", + .tags = (const struct tst_tag []) { + {"linux-git" ,"22f6b4d34fcf" }, + {"CVE" ,"2016-10044" }, + {} + } };
testcases/cve/cve-2016-7042.c +5 -0
@@ -71,4 +71,9 @@ static struct tst_test test = { .setup = setup, .cleanup = cleanup, .test_all = do_test, + .tags = (const struct tst_tag []) { + {"linux-git" ,"03dab869b7b2" }, + {"CVE" ,"2016-7042" }, + {} + } };
testcases/cve/cve-2016-7117.c +5 -0
@@ -150,4 +150,9 @@ static struct tst_test test = { .setup = setup, .cleanup = cleanup, .min_kver = "2.6.33", + .tags = (const struct tst_tag []) { + {"linux-git" ,"a2e2725541fa" }, + {"CVE" ,"2016-7117" }, + {} + } };
testcases/cve/cve-2017-16939.c +5 -0
@@ -77,4 +77,9 @@ static void run(void)static struct tst_test test = { .setup = setup, .test_all = run, + .tags = (const struct tst_tag []) { + {"linux-git" ,"1137b5e2529a" }, + {"CVE" ,"2017-16939" }, + {} + } };
testcases/cve/cve-2017-17052.c +5 -0
@@ -111,4 +111,9 @@ static struct tst_test test = { .cleanup = cleanup, .setup = setup, .test_all = run, + .tags = (const struct tst_tag []) { + {"linux-git" ,"2b7e8665b4ff" }, + {"CVE" ,"2017-17052" }, + {} + } };
testcases/cve/cve-2017-17053.c +5 -0
@@ -154,6 +154,11 @@ static struct tst_test test = { .setup = setup, .cleanup = cleanup, .test_all = run, + .tags = (const struct tst_tag []) { + {"linux-git" ,"ccd5b3235180" }, + {"CVE" ,"2017-17053" }, + {} + } };#else
testcases/cve/cve-2017-2618.c +5 -0
@@ -51,4 +51,9 @@ static struct tst_test test = { .forks_child = 1, .setup = setup, .test_all = do_test, + .tags = (const struct tst_tag []) { + {"linux-git" ,"0c461cb727d1" }, + {"CVE" ,"2017-2618" }, + {} + } };
testcases/cve/cve-2017-2671.c +5 -0
@@ -109,4 +109,9 @@ static struct tst_test test = { .test_all = run, .cleanup = cleanup, .needs_root = 1, + .tags = (const struct tst_tag []) { + {"linux-git" ,"43a6684519ab" }, + {"CVE" ,"2017-2671" }, + {} + } };
testcases/cve/meltdown.c +5 -1
@@ -377,7 +377,11 @@ static struct tst_test test = { .setup = setup, .test_all = run, .cleanup = cleanup, - .min_kver ="2.6.32" + .min_kver = "2.6.32", + .tags = (const struct tst_tag []) { + {"CVE" ,"2017-5754" }, + {} + } };#else /* #if defined(__x86_64__) || defined(__i386__) */
testcases/cve/stack_clash.c +4 -0
@@ -250,4 +250,8 @@ static struct tst_test test = { .needs_root = 1, .setup = setup, .test_all = stack_clash_test, + .tags = (const struct tst_tag []) { + {"CVE" ,"2017-1000364" }, + {} + } };
testcases/kernel/crypto/af_alg01.c +5 -0
@@ -76,4 +76,9 @@ static void do_test(unsigned int i)static struct tst_test test = { .test = do_test, .tcnt =ARRAY_SIZE (hash_algs), + .tags = (const struct tst_tag []) { + {"linux-git" ,"af3ff8045bbf" }, + {"CVE" ,"2017-17806" }, + {} + } };
testcases/kernel/crypto/af_alg02.c +5 -0
@@ -59,4 +59,9 @@ static struct tst_test test = { .test_all = run, .timeout = 20, .needs_checkpoints = 1, + .tags = (const struct tst_tag []) { + {"linux-git" ,"ecaaab564978" }, + {"CVE" ,"2017-17805" }, + {} + } };
testcases/kernel/crypto/af_alg03.c +4 -0
@@ -28,4 +28,8 @@ static void run(void)static struct tst_test test = { .test_all = run, + .tags = (const struct tst_tag []) { + {"linux-git" ,"e57121d08c38" }, + {} + } };
testcases/kernel/crypto/af_alg04.c +4 -0
@@ -55,4 +55,8 @@ static void run(void)static struct tst_test test = { .test_all = run, .forks_child =1 , + .tags = (const struct tst_tag []) { + {"linux-git" ,"bb2964810233" }, + {} + } };
testcases/kernel/crypto/af_alg05.c +7 -1
@@ -11,7 +11,7 @@* boundary. Based on the reproducer from the commit message. Note that this * issue only reproduces on certain architectures, such as arm and arm64. * -* On some older kernel without commit 160544075f2 ("crypto: scatterwalk - Hide +* On some older kernel without commit 160544075f2a ("crypto: scatterwalk - Hide * PageSlab call to optimise away flush_dcache_page") , it doesn't use * ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE macro. It can crash on all architectures. * Without skcipher walk interface, it is also a regresstion test for commit
@@ -46,4 +46,10 @@ static void run(void)static struct tst_test test = { .test_all = run, + .tags = (const struct tst_tag []) { + {"linux-git" ,"8088d3dd4d7c" }, + {"linux-git" ,"160544075f2a" }, + {"linux-git" ,"0868def3e410" }, + {} + } };
testcases/kernel/crypto/af_alg06.c +4 -0
@@ -48,4 +48,8 @@ static void run(void)static struct tst_test test = { .test_all = run, + .tags = (const struct tst_tag []) { + {"linux-git" ,"8f9c46934848" }, + {} + } };
testcases/kernel/crypto/crypto_user01.c +6 -0
@@ -192,4 +192,10 @@ static struct tst_test test = { .setup = setup, .test_all = run, .cleanup = cleanup, + .tags = (const struct tst_tag []) { + {"linux-git" ,"f43f39958beb" }, + {"CVE" ,"2013-2547" }, + {"CVE" ,"2018-19854" }, + {} + } };
testcases/kernel/crypto/crypto_user02.c +4 -0
@@ -105,4 +105,8 @@ static struct tst_test test = { .cleanup = cleanup, .needs_root = 1, .forks_child = 1, + .tags = (const struct tst_tag []) { + {"linux-git" ,"21d4120ec6f5" }, + {} + } };
testcases/kernel/crypto/pcrypt_aead01.c +5 -0
@@ -69,4 +69,9 @@ static struct tst_test test = { .test_all = run, .cleanup = cleanup, .needs_root = 1, + .tags = (const struct tst_tag []) { + {"linux-git" ,"d76c68109f37" }, + {"CVE" ,"2017-5754" }, + {} + } };
testcases/kernel/sound/snd_timer01.c +6 -0
@@ -140,4 +140,10 @@ static struct tst_test test = { .test_all = run, .setup = setup, .cleanup = cleanup, + .tags = (const struct tst_tag []) { + {"linux-git" ,"d11662f4f798" }, + {"linux-git" ,"ba3021b2c79b" }, + {"CVE" ,"2017-1000380" }, + {NULL ,NULL } + } };
testcases/kernel/syscalls/accept/accept02.c +5 -0
@@ -142,4 +142,9 @@ static struct tst_test test = { .setup = setup, .cleanup = cleanup, .needs_checkpoints = 1, + .tags = (const struct tst_tag []) { + {"CVE" ,"2017-8890" }, + {"linux-git" ,"657831ff" }, + {NULL ,NULL }, + } };
testcases/kernel/syscalls/acct/acct02.c +4 -0
@@ -263,4 +263,8 @@ static struct tst_test test = { .cleanup = cleanup, .needs_tmpdir = 1, .needs_root = 1, + .tags = (const struct tst_tag []) { + {"linux-git" ,"4d9570158b626" }, + {} + } };
testcases/kernel/syscalls/add_key/add_key02.c +5 -0
@@ -83,4 +83,9 @@ static void verify_add_key(unsigned int i)static struct tst_test test = { .tcnt =ARRAY_SIZE (tcases), .test = verify_add_key, + .tags = (const struct tst_tag []) { + {"linux-git" ,"5649645d725c" }, + {"CVE" ,"2017-15274" }, + {} + } };
testcases/kernel/syscalls/bpf/bpf_prog02.c +4 -0
@@ -190,5 +190,9 @@ static struct tst_test test = { {&attr, .size = sizeof(*attr)}, {&msg, .size = sizeof(MSG)}, {}, + }, + .tags = (const struct tst_tag []) { + {"linux-git" ,"3612af783cf5" }, + {} } };
testcases/kernel/syscalls/futex/futex_cmp_requeue02.c +5 -0
@@ -73,4 +73,9 @@ static struct tst_test test = { .cleanup = cleanup, .test = verify_futex_cmp_requeue, .tcnt = ARRAY_SIZE(tcases), + .tags = (const struct tst_tag []) { + {"CVE" ,"2018-6927" }, + {"linux-git" ,"fbe0e839d1e2" }, + {} + } };
testcases/kernel/syscalls/keyctl/keyctl02.c +4 -0
@@ -136,4 +136,8 @@ static struct tst_test test = { .setup = setup, .cleanup = cleanup, .test_all = do_test, + .tags = (const struct tst_tag []) { + {"linux-git" ,"b4a1b4f5047e" }, + {} + } };
testcases/kernel/syscalls/keyctl/keyctl04.c +5 -0
@@ -43,4 +43,9 @@ static void do_test(void)static struct tst_test test = { .test_all = do_test, + .tags = (const struct tst_tag []) { + {"CVE" ,"2017-7472" }, + {"linux-git" ,"c9f838d104fe" }, + {} + } };
testcases/kernel/syscalls/keyctl/keyctl05.c +4 -0
@@ -207,4 +207,8 @@ static struct tst_test test = { .tcnt = 3, .test = do_test, .forks_child = 1, + .tags = (const struct tst_tag []) { + {"linux-git" ,"63a0b0509e70" }, + {} + } };
testcases/kernel/syscalls/move_pages/move_pages12.c +6 -0
@@ -330,6 +330,12 @@ static struct tst_test test = { .cleanup = cleanup, .test = do_test, .tcnt = ARRAY_SIZE(tcases), + .tags = (const struct tst_tag []) { + {"linux-git" ,"e66f17ff7177" }, + {"linux-git" ,"c9d398fa2378" }, + {"linux-git" ,"4643d67e8cb0" }, + {} + } };#else
testcases/kernel/syscalls/realpath/realpath01.c +4 -0
@@ -41,4 +41,8 @@ static struct tst_test test = { .setup = setup, .needs_root = 1, .needs_tmpdir = 1, + .tags = (const struct tst_tag []) { + {"CVE" ,"2018-1000001" }, + {} + } };
testcases/kernel/syscalls/request_key/request_key05.c +4 -0
@@ -31,4 +31,8 @@ static void run(void)static struct tst_test test = { .test_all = run, + .tags = (const struct tst_tag []) { + {"CVE" ,"2017-6951" }, + {} + } };
testcases/kernel/syscalls/sbrk/sbrk03.c +4 -0
@@ -66,4 +66,8 @@ static void sbrk_test(void)static struct tst_test test = { .test_all = sbrk_test, + .tags = (const struct tst_tag []) { + {"linux-git" ,"473a06572fcd" }, + {} + } };
testcases/kernel/syscalls/sendto/sendto02.c +4 -0
@@ -77,4 +77,8 @@ static struct tst_test test = { .setup = setup, .cleanup = cleanup, .test_all = verify_sendto, + .tags = (const struct tst_tag []) { + {"linux-git" ,"6e51fe757259" }, + {} + } };
testcases/kernel/syscalls/setrlimit/setrlimit06.c +4 -0
@@ -114,4 +114,8 @@ static struct tst_test test = { .setup = setup, .cleanup = cleanup, .forks_child = 1, + .tags = (const struct tst_tag []) { + {"linux-git" ,"c3bca5d450b62" }, + {} + } };
Signature · 1 line
2.21.0
cover Add support for a test tags
In reply to Cyril HrubisHi!
Please ignore this patchset, I will resend it from my work email address.
Signature · 2 lines
Cyril Hrubis [email protected]