Support metadata groups
v3 cover Support metadata groups
---
Changes in v3:
- match @groups not groups
- drop const qualifier in eat_asterisk_space()
- WARN() when empty groups is provided
- error on strdup() failure
- we always add 1 group, so always add items to the hash
- Link to v2: https://lore.kernel.org/r/[email protected]
Changes in v2:
- switch from @group to @groups
- use Cyril snippet to folter out groups from description
- optimize code
- Link to v1: https://lore.kernel.org/r/[email protected]
---
Andrea Cervesato (2):
metadata: add tests grouping support
doc: conf.py: Show groups in test catalog
doc/conf.py | 10 +++ metadata/metaparse.c | 163 +++++++++++++++++++++++++++------- metadata/tests/array_size01.c.json | 2 + metadata/tests/array_size02.c.json | 2 + metadata/tests/array_size03.c.json | 2 + metadata/tests/array_size04.c.json | 2 + metadata/tests/empty_struct.c.json | 2 + metadata/tests/expand_flags.c.json | 2 + metadata/tests/groups.c | 11 +++ metadata/tests/groups.c.json | 12 +++ metadata/tests/include.c.json | 2 + metadata/tests/macro.c.json | 2 + metadata/tests/macro_str.c.json | 2 + metadata/tests/multiline_macro.c.json | 2 + metadata/tests/tags.c.json | 2 + 15 files changed, 187 insertions(+), 31 deletions(-)
---
base-commit: bc9cfca5c7b6599e501d580bb42b9538390fd9a6
change-id: 20260612-metadata_groups-dd2430e21901
Best regards,
Signature · 5 lines
Andrea Cervesato <[email protected]> -- Mailing list info: https://lists.linux.it/listinfo/ltp
PATCH v3 1/2 metadata: add tests grouping support
From: Andrea Cervesato <[email protected]>
Add groups field to metaparse JSON output, so we can filter out tests in kirk. Groups are derived from:
1. Source file path - the two nearest parent directories (immediate parent first), skipping 'kernel' as too generic. For example:
- testcases/kernel/syscalls/clone/clone01.c -> clone, syscalls
- testcases/kernel/kvm/kvm_pagefault01.c -> kvm
- testcases/cve/cve-2017-16939.c -> cve
2. @groups tags in the doc comment block, e.g.:
/*\
* Test description.
*
* @groups stress syscalls
* @groups memory
*/
Add test case for @groups tag parsing.
--- metadata/metaparse.c | 163 +++++++++++++++++++++++++++------- metadata/tests/array_size01.c.json | 2 + metadata/tests/array_size02.c.json | 2 + metadata/tests/array_size03.c.json | 2 + metadata/tests/array_size04.c.json | 2 + metadata/tests/empty_struct.c.json | 2 + metadata/tests/expand_flags.c.json | 2 + metadata/tests/groups.c | 11 +++ metadata/tests/groups.c.json | 12 +++ metadata/tests/include.c.json | 2 + metadata/tests/macro.c.json | 2 + metadata/tests/macro_str.c.json | 2 + metadata/tests/multiline_macro.c.json | 2 + metadata/tests/tags.c.json | 2 + 14 files changed, 177 insertions(+), 31 deletions(-)
metadata/metaparse.c +132 -31
@@ -17,6 +17,7 @@#include "data_storage.h" #define INCLUDE_PATH_MAX 5 +#define GROUPS_TAG "@groups"static int verbose;static char *cmdline_includepath[INCLUDE_PATH_MAX ];
@@ -34,7 +35,7 @@ static void remove_to_newline(FILE *f) }while (c !='\n' ); } -static const char *eat_asterisk_space (const char *c) +static char *eat_asterisk_space (char *c) {unsigned int i =0 ;
@@ -50,7 +51,51 @@ static const char *eat_asterisk_space(const char *c)return c; } -static void multiline_comment (FILE *f,struct data_node *doc) +/* +* Add a group to the groups array, skipping 'kernel' as it's too generic. +* Returns 0 if no group was added, 1 otherwise. +*/ +static int add_group (struct data_node *groups,const char *name) +{ +if (name &&strcmp (name,"kernel" )) { +data_node_array_add (groups,data_node_string (name)); +return 1 ; + } + +return 0 ; +} + +/* +* Parse a '@groups foo bar baz' doc comment line, adding each +* whitespace-separated name to the groups array. Returns 1 if the line +* was a @groups tag (and should be consumed), 0 otherwise. +*/ +static int parse_groups (struct data_node *groups,char *line) +{ +char *s; +char *name; + +if (!groups) +return 0 ; + + s =eat_asterisk_space (line); +if (strncmp (s,GROUPS_TAG ,sizeof (GROUPS_TAG ) -1 )) +return 0 ; + + s +=sizeof (GROUPS_TAG ) -1 ; +if (*s && *s !=' ' && *s !='\t' ) { +WARN ("Empty @group" ); +return 1 ; + } + +for (name =strtok (s," \t" ); name; name =strtok (NULL ," \t" )) +add_group (groups, name); + +return 1 ; +} + +static void multiline_comment (FILE *f,struct data_node *doc, +struct data_node *groups) {int c;int state =0 ;
@@ -62,12 +107,14 @@ static void multiline_comment(FILE *f, struct data_node *doc)if (doc) {if (c =='\n' ) { -struct data_node *line; +char *str; buf[bufp] =0 ; - line =data_node_string (eat_asterisk_space (buf)); -if (data_node_array_add (doc, line)) -WARN ("doc string comment truncated" ); + str =eat_asterisk_space (buf); bufp =0 ; +if (parse_groups (groups, str)) +continue ; +if (data_node_array_add (doc,data_node_string (str))) +WARN ("doc string comment truncated" );continue ; }
@@ -100,7 +147,8 @@ static void multiline_comment(FILE *f, struct data_node *doc)static const char doc_prefix[] ="\\\n" ; -static void maybe_doc_comment (FILE *f,struct data_node *doc) +static void maybe_doc_comment (FILE *f,struct data_node *doc, +struct data_node *groups) {int c, i;
@@ -113,14 +161,15 @@ static void maybe_doc_comment(FILE *f, struct data_node *doc)if (c =='*' )ungetc (c, f); -multiline_comment (f,NULL ); +multiline_comment (f,NULL ,NULL );return ; } -multiline_comment (f, doc); +multiline_comment (f, doc, groups); } -static void maybe_comment (FILE *f,struct data_node *doc) +static void maybe_comment (FILE *f,struct data_node *doc, +struct data_node *groups) {int c =getc (f);
@@ -129,7 +178,7 @@ static void maybe_comment(FILE *f, struct data_node *doc)remove_to_newline (f);break ;case '*' : -maybe_doc_comment (f, doc); +maybe_doc_comment (f, doc, groups);break ;default :ungetc (c, f);
@@ -137,7 +186,8 @@ static void maybe_comment(FILE *f, struct data_node *doc) } } -static char *next_token2 (FILE *f,char *buf,size_t buf_len,struct data_node *doc) +static char *next_token2 (FILE *f,char *buf,size_t buf_len, +struct data_node *doc,struct data_node *groups) {size_t i =0 ;int c;
@@ -194,7 +244,7 @@ static char *next_token2(FILE *f, char *buf, size_t buf_len, struct data_node *d buf[i++] = c;break ;case '/' : -maybe_comment (f, doc); +maybe_comment (f, doc, groups);break ;case '"' : in_str =1 ;
@@ -216,11 +266,11 @@ exit:return buf; } -static char *next_token (FILE *f,struct data_node *doc) +static char *next_token (FILE *f,struct data_node *doc,struct data_node *groups) {static char buf[4096 ]; -return next_token2 (f, buf,sizeof (buf), doc); +return next_token2 (f, buf,sizeof (buf), doc, groups); }static FILE *open_file (const char *dir,const char *fname)
@@ -383,7 +433,7 @@ static int array_is_hash(FILE *f)int in_id =1 ;char *token; -while ((token =next_token (f,NULL ))) { +while ((token =next_token (f,NULL ,NULL ))) {if (!strcmp (token,"}" )) {if (in_id && !comma_last)
@@ -402,7 +452,7 @@ static int array_is_hash(FILE *f)int level =1 ;for (;;) { - token =next_token (f,NULL ); + token =next_token (f,NULL ,NULL );if (!token) goto ret;
@@ -453,7 +503,7 @@ static int parse_array(FILE *f, const char *arr_id, struct data_node **ret) *ret =data_node_array ();for (;;) { -if (!(token =next_token (f,NULL ))) +if (!(token =next_token (f,NULL ,NULL )))return 1 ;if (!strcmp (token,"{" )) {
@@ -529,14 +579,14 @@ static int parse_get_array_len(FILE *f)const char *token;int cnt =0 , depth =0 , prev_comma =0 ; -if (!(token =next_token (f,NULL ))) +if (!(token =next_token (f,NULL ,NULL )))return 0 ;if (strcmp (token,"{" ))return 0 ;for (;;) { -if (!(token =next_token (f,NULL ))) +if (!(token =next_token (f,NULL ,NULL )))return 0 ;if (!strcmp (token,"{" ))
@@ -565,7 +615,7 @@ static void look_for_array_size(FILE *f, const char *arr_id, struct data_node **int prev_buf =1 ;for (;;) { -if (!(token =next_token2 (f, buf[cur_buf],sizeof (buf[cur_buf]),NULL ))) +if (!(token =next_token2 (f, buf[cur_buf],sizeof (buf[cur_buf]),NULL ,NULL )))break ;if (!strcmp (token,"=" ) && !strcmp (buf[prev_buf], arr_id)) {
@@ -595,13 +645,13 @@ static int parse_array_size(FILE *f, struct data_node **res) *res =NULL ; -if (!(token =next_token (f,NULL ))) +if (!(token =next_token (f,NULL ,NULL )))return 1 ;if (strcmp (token,"(" ))return 1 ; -if (!(token =next_token (f,NULL ))) +if (!(token =next_token (f,NULL ,NULL )))return 1 ; arr_id =strdup (token);
@@ -621,7 +671,7 @@ static int parse_array_size(FILE *f, struct data_node **res)rewind (f);for (;;) { -if (!(token =next_token (f,NULL ))) +if (!(token =next_token (f,NULL ,NULL )))break ;if (token[0 ] =='#' ) {
@@ -654,7 +704,8 @@ static int parse_array_size(FILE *f, struct data_node **res)return 0 ; } -static int parse_test_struct (FILE *f,struct data_node *doc,struct data_node *node) +static int parse_test_struct (FILE *f,struct data_node *doc, +struct data_node *groups,struct data_node *node) {char *token;char *id =NULL ;
@@ -662,7 +713,7 @@ static int parse_test_struct(FILE *f, struct data_node *doc, struct data_node *nstruct data_node *ret;for (;;) { -if (!(token =next_token (f, doc))) +if (!(token =next_token (f, doc, groups)))return 1 ;if (!strcmp (token,"}" ))
@@ -842,7 +893,7 @@ static void parse_include_macros(FILE *f, int level)if (!inc)return ; -while ((token =next_token (inc,NULL ))) { +while ((token =next_token (inc,NULL ,NULL ))) {if (token[0 ] =='#' ) { hash =1 ;continue ;
@@ -907,6 +958,49 @@ static void load_internal_macros(void)fprintf (stderr,"END PREDEFINED MACROS\n" ); } +/* +* Add groups derived from the source file path. +* +* Groups are the two nearest parent directories (immediate parent +* first), skipping 'kernel' as it's too generic: +* +* testcases/kernel/syscalls/clone/clone01.c -> clone, syscalls +* testcases/kernel/kvm/kvm_pagefault01.c -> kvm +* testcases/cve/cve-2017-16939.c -> cve +*/ +static void add_path_groups (struct data_node *groups,const char *fname) +{ +char *buf; +char *dirs[8 ]; +int ndirs =0 ; +char *p; + +if (strncmp (fname,"testcases/" ,10 )) +return ; + + buf =strdup (fname +10 ); +if (!buf) { +fprintf (stderr,"Allocation failed!\n" ); +exit (1 ); + } + + p =strtok (buf,"/" ); +while (p && ndirs <8 ) { + dirs[ndirs++] = p; + p =strtok (NULL ,"/" ); + } + +/* Last element is the filename, skip it */ + ndirs--; + +if (ndirs >=1 ) +add_group (groups, dirs[ndirs -1 ]); +if (ndirs >=2 ) +add_group (groups, dirs[ndirs -2 ]); + +free (buf); +} +static struct data_node *parse_file (const char *fname) {int state =0 , found =0 ;
@@ -923,15 +1017,18 @@ static struct data_node *parse_file(const char *fname)struct data_node *res =data_node_hash ();struct data_node *doc =data_node_array (); +struct data_node *groups =data_node_array (); + +add_path_groups (groups, fname);load_internal_macros (); -while ((token =next_token (f, doc))) { +while ((token =next_token (f, doc, groups))) {if (state <6 && !strcmp (tokens[state], token)) { state++; }else {if (token[0 ] =='#' ) { - token =next_token (f, doc); + token =next_token (f, doc, groups);if (token) {if (!strcmp (token,"define" ))parse_macro (f);
@@ -948,7 +1045,7 @@ static struct data_node *parse_file(const char *fname)continue ; found =1 ; -parse_test_struct (f, doc, res); +parse_test_struct (f, doc, groups, res); }if (data_node_array_len (doc)) {
@@ -958,6 +1055,9 @@ static struct data_node *parse_file(const char *fname)data_node_free (doc); } +/* inside the testcases/ folder we will always have at least one group */ +data_node_hash_add (res,"groups" , groups); +fclose (f);if (!found) {
@@ -985,7 +1085,7 @@ static void parse_must_files(void)if (!f)continue ; -while ((token =next_token (f,NULL ))) { +while ((token =next_token (f,NULL ,NULL ))) {if (!strcmp (token,"define" ))parse_macro (f); }
@@ -1238,6 +1338,7 @@ int main(int argc, char *argv[]) }data_node_hash_add (res,"fname" ,data_node_string (argv[optind])); +printf (" \"%s\": " ,strip_name (argv[optind]));data_to_json (res, stdout,2 );data_node_free (res);
metadata/tests/array_size01.c.json +2 -0
@@ -1,4 +1,6 @@ "array_size01": { "test_variants": 1, + "groups": [ + ], "fname": "array_size01.c" }
metadata/tests/array_size02.c.json +2 -0
@@ -1,4 +1,6 @@ "array_size02": { "test_variants": 3, + "groups": [ + ], "fname": "array_size02.c" }
metadata/tests/array_size03.c.json +2 -0
@@ -1,4 +1,6 @@ "array_size03": { "test_variants": 2, + "groups": [ + ], "fname": "array_size03.c" }
metadata/tests/array_size04.c.json +2 -0
@@ -1,4 +1,6 @@ "array_size04": { "test_variants": 3, + "groups": [ + ], "fname": "array_size04.c" }
metadata/tests/empty_struct.c.json +2 -0
@@ -1,3 +1,5 @@ "empty_struct": { + "groups": [ + ], "fname": "empty_struct.c" }
metadata/tests/expand_flags.c.json +2 -0
@@ -1,4 +1,6 @@ "expand_flags": { + "groups": [ + ], "all_filesystems": true, "needs_device": true, "needs_tmpdir": true,
metadata/tests/groups.c new file +11 -0
@@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +/*\ +* Test for @groups tag parsing. +* +* @groups stress regression +* @groups smoke +*/ + +static struct tst_test test = { +};
metadata/tests/groups.c.json new file +12 -0
@@ -0,0 +1,12 @@ + "groups": { + "doc": [ + "Test for @groups tag parsing.", + "" + ], + "groups": [ + "stress", + "regression", + "smoke" + ], + "fname": "groups.c" + }
metadata/tests/include.c.json +2 -0
@@ -1,4 +1,6 @@ "include": { + "groups": [ + ], "test_variants": 10, "fname": "include.c" }
metadata/tests/macro.c.json +2 -0
@@ -1,4 +1,6 @@ "macro": { + "groups": [ + ], "test_variants": 10, "fname": "macro.c" }
metadata/tests/macro_str.c.json +2 -0
@@ -1,4 +1,6 @@ "macro_str": { "syscall": "syscall(\"foo\")", + "groups": [ + ], "fname": "macro_str.c" }
metadata/tests/multiline_macro.c.json +2 -0
@@ -1,4 +1,6 @@ "multiline_macro": { + "groups": [ + ], "test_variants": 10, "fname": "multiline_macro.c" }
metadata/tests/tags.c.json +2 -0
@@ -9,5 +9,7 @@ "tag-value-2" ] ], + "groups": [ + ], "fname": "tags.c" }
Signature · 5 lines
2.51.0 -- Mailing list info: https://lists.linux.it/listinfo/ltp
Re: [LTP] metadata: add tests grouping support
Hi Andrea,
Quoted 1 line · Andrea Cervesato
metadata: add tests grouping support
--- [PATCH 1/2] ---
metadata/metaparse.c static const char *eat_asterisk_space(const char *c) · line 89
Quoted 7 lines
metadata/metaparse.c +7
-0+static int parse_groups (struct data_node *groups,char *line) +{ + s +=sizeof (GROUPS_TAG ) -1 ; +if (*s && *s !=' ' && *s !='\t' ) { +WARN ("Empty @group" ); +return 1 ; + }
Two things here:
The warning says "@group" (singular) but the tag is "@groups". More importantly, this condition fires when "@groups" is immediately followed by a non-whitespace character (e.g. "@groupsfoo"), which is not an empty tag but a word that happens to share a prefix with the tag. The message should probably say something like "invalid @groups suffix".
Separately, a truly empty "@groups" line (tag followed by nothing or only whitespace) falls through to strtok which produces zero tokens and returns 1 silently. Should that also warn?
metadata/metaparse.c static struct data_node *parse_file(const char *fname) · line 1059
Quoted 2 lines
metadata/metaparse.c +2
-0+/* inside the testcases/ folder we will always have at least one group */ +data_node_hash_add (res,"groups" , groups);
The comment is not accurate. A file like testcases/kernel/foo.c with no @groups tags produces zero groups since "kernel" is the only parent directory and is explicitly skipped.
Verdict - Needs revision
---
Note:
The agent can sometimes produce false positives although often its findings are genuine. If you find issues with the review, please comment this email or ignore the suggestions.
Regards,
LTP AI Reviewer
Signature · 1 line
Mailing list info: https://lists.linux.it/listinfo/ltp
Re: [LTP] [PATCH v3 1/2] metadata: add tests grouping support
In reply to Andrea CervesatoHi!
The code looks good to me now:
Please fix the minor issues pointed out by the ltp-agent before applying.
Signature · 5 lines
Cyril Hrubis [email protected] -- Mailing list info: https://lists.linux.it/listinfo/ltp
Re: [LTP] [PATCH v3 1/2] metadata: add tests grouping support
Merged with fixes on agent found issues, Thanks!
--
Andrea Cervesato
SUSE QE Automation Engineer Linux
[email protected]
Signature · 1 line
Mailing list info: https://lists.linux.it/listinfo/ltp
Re: [LTP] [PATCH v3 1/2] metadata: add tests grouping support
In reply to Cyril HrubisHi!
I'm looking at the metadata and we will need at least one follow up.
For tests that are under cve/ directory we add a 'cve' group now, but tests that have cve tag but are in a different place we do not do so.
For consistency we should do:
* Filter out cve as we do with kernel in the group by path
* Possibly add cve group based on cve tag (I would call this optional as it kind of duplicates the cve tag)
Signature · 5 lines
Cyril Hrubis [email protected] -- Mailing list info: https://lists.linux.it/listinfo/ltp
Re: [LTP] [PATCH v3 1/2] metadata: add tests grouping support
Hi Cyril,
Quoted 6 lines
Hi!
I'm looking at the metadata and we will need at least one follow up.For tests that are under cve/ directory we add a 'cve' group now, but tests that have cve tag but are in a different place we do not do so.
Quoted 6 lines
For consistency we should do:
* Filter out cve as we do with kernel in the group by path
* Possibly add cve group based on cve tag (I would call this optional as it kind of duplicates the cve tag)
Do you mean to take a look at the tst_test->tags and verify if we have a CVE tag? I think this is possible yes.
I will take a look at it.
--
Andrea Cervesato
SUSE QE Automation Engineer Linux
[email protected]
Signature · 1 line
Mailing list info: https://lists.linux.it/listinfo/ltp
Re: [LTP] [PATCH v3 1/2] metadata: add tests grouping support
Hi!
Quoted 7 lines
Quoted 7 lines
I'm looking at the metadata and we will need at least one follow up.
For tests that are under cve/ directory we add a 'cve' group now, but tests that have cve tag but are in a different place we do not do so.
For consistency we should do:
Quoted 7 lines
Quoted 4 lines
* Filter out cve as we do with kernel in the group by path
* Possibly add cve group based on cve tag (I would call this optional as it kind of duplicates the cve tag)
Do you mean to take a look at the
tst_test->tagsand verify if we have a CVE tag? I think this is possible yes.
Not really. We have quite a lot of cve reproducers that do not have a cve directory in their path. And then we have reproducers that do have it. The current metadata has cve group only for the reproducers in the cve directory which is inconsistent. So I proposed to filter out the "cve" directory the same way we filter out "kernel". And possibly add "cve" group based on tags, but that kind of duplicates the tag field, so it may not make sense to add it.
Signature · 5 lines
Cyril Hrubis [email protected] -- Mailing list info: https://lists.linux.it/listinfo/ltp
PATCH v3 2/2 doc: conf.py: Show groups in test catalog
In reply to Andrea CervesatoFrom: Andrea Cervesato <[email protected]>
Display the groups field from metadata in the generated test catalog pages. Groups appear after the source link and before the test description.
--- doc/conf.py | 10 ++++++++++ 1 file changed, 10 insertions(+)
doc/conf.py +10 -0
@@ -394,6 +394,7 @@ def _generate_setup_table(keys): 'needs_checkpoints', 'forks_child', 'tags', + 'groups', ] my_keys = {k: v for k, v in keys.items() if k not in exclude} if len(my_keys) == 0:
@@ -482,6 +483,15 @@ def generate_test_catalog(_): '' ]) + # groups information + groups = conf.get('groups', None) + if groups: + text.extend([ + '', + f"**Groups**: {', '.join(groups)}", + '' + ]) + # test description desc = conf.get('doc', None) if desc:
Signature · 5 lines
2.51.0 -- Mailing list info: https://lists.linux.it/listinfo/ltp