doc: support for links in syscalls
cover doc: support for links in syscalls
This is an optimization of the documentation, which adds a bit of refactoring and clickable syscalls names inside the statistics table. In this way, users can click on a syscall and being redirected to the source code which is testing it.
---
Andrea Cervesato (2):
doc: cleanup conf.py script
doc: support for clickable syscalls under stats
doc/conf.py | 158 ++++++++++++++++++++++++++++++++++-------------------------- 1 file changed, 91 insertions(+), 67 deletions(-)
---
base-commit: 8f2292af050fd64c447dd462880d0b0a19d79448
change-id: 20241204-doc_syscalls_link-a184545389ba
Best regards,
Signature · 5 lines
Andrea Cervesato <[email protected]> -- Mailing list info: https://lists.linux.it/listinfo/ltp
PATCH 1/2 doc: cleanup conf.py script
From: Andrea Cervesato <[email protected]>
Cleanup code in the documentation building script which is using bad patters and overcomplicating certain parts of the code.
--- doc/conf.py | 40 +++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 17 deletions(-)
doc/conf.py +23 -17
@@ -5,9 +5,9 @@ import os import re -import sphinx import socket import urllib.request +import sphinx # -- Project information ----------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
@@ -116,19 +116,19 @@ def generate_syscalls_stats(_): ] # fetch syscalls file + syscalls_url = "https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/plain/arch/mips/kernel/syscalls" error = False try: socket.setdefaulttimeout(3) urllib.request.urlretrieve( - "https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/plain/arch/mips/kernel/syscalls/syscall_n64.tbl", - "syscalls.tbl") - except Exception as err: + f"{syscalls_url}/syscall_n64.tbl", "syscalls.tbl") + except urllib.error.URLError as err: error = True logger = sphinx.util.logging.getLogger(__name__) - msg = "Can't download syscall_n64.tbl from kernel sources" + msg = f"Can't download syscall_n64.tbl from kernel sources ({err})" logger.warning(msg) - with open(output, 'w+') as stats: + with open(output, 'w+', encoding='utf-8') as stats: stats.write(f".. warning::\n\n {msg}") if error:
@@ -142,7 +142,7 @@ def generate_syscalls_stats(_): # collect all available kernel syscalls regexp = re.compile(r'\d+\s+n64\s+(?P<syscall>\w+)\s+\w+') ker_syscalls = [] - with open("syscalls.tbl", 'r') as data: + with open("syscalls.tbl", 'r', encoding='utf-8') as data: for line in data: match = regexp.search(line) if match:
@@ -150,7 +150,7 @@ def generate_syscalls_stats(_): # collect all LTP tested syscalls ltp_syscalls = [] - for root, _, files in os.walk('../testcases/kernel/syscalls'): + for _, _, files in os.walk('../testcases/kernel/syscalls'): for myfile in files: if myfile.endswith('.c'): ltp_syscalls.append(myfile)
@@ -196,41 +196,47 @@ def generate_syscalls_stats(_): ' :header-rows: 0\n\n', ] + max_columns = 3 + for sysname, tested in syscalls.items(): if tested: - if (index_tested % 3) == 0: + if (index_tested % max_columns) == 0: table_tested.append(f' * - {sysname}\n') else: table_tested.append(f' - {sysname}\n') index_tested += 1 else: - if (index_untest % 3) == 0: + if (index_untest % max_columns) == 0: table_untest.append(f' * - {sysname}\n') else: table_untest.append(f' - {sysname}\n') index_untest += 1 - left = index_tested % 3 + left = index_tested % max_columns if left > 0: - for index in range(0, 3 - left): - table_tested.append(f' -\n') + for _ in range(0, max_columns - left): + table_tested.append(' -\n') - left = index_untest % 3 + left = index_untest % max_columns if left > 0: - for index in range(0, 3 - left): - table_untest.append(f' -\n') + for _ in range(0, max_columns - left): + table_untest.append(' -\n') text.extend(table_tested) text.append('\n') text.extend(table_untest) # write the file - with open(output, 'w+') as stats: + with open(output, 'w+', encoding='utf-8') as stats: stats.writelines(text) def setup(app): + """ + Setup the current documentation, using self generated data and graphics + customizations. + """ app.add_css_file('custom.css') app.connect('builder-inited', generate_syscalls_stats)
Signature · 5 lines
2.43.0 -- Mailing list info: https://lists.linux.it/listinfo/ltp
Re: [LTP] [PATCH 1/2] doc: cleanup conf.py script
Hi Andrea,
Kind regards,
Petr
Signature · 1 line
Mailing list info: https://lists.linux.it/listinfo/ltp
PATCH 2/2 doc: support for clickable syscalls under stats
In reply to Andrea CervesatoFrom: Andrea Cervesato <[email protected]>
Under statistics tab, add possibility to click on a syscalls and being redirected to the source code which is testing them.
--- doc/conf.py | 120 ++++++++++++++++++++++++++++++++++-------------------------- 1 file changed, 69 insertions(+), 51 deletions(-)
doc/conf.py +69 -51
@@ -61,39 +61,38 @@ def generate_syscalls_stats(_): # because in some cases (i.e. io_ring) syscalls are tested, but they are # part of a more complex scenario. In the following list, we define syscalls # which we know they are 100% tested already. - white_list = [ - 'epoll_pwait2', - 'fadvise64', - 'fanotify_init', - 'fanotify_mark', - 'getdents64', - 'getmsg', - 'getpmsg', - 'inotify_add_watch', - 'inotify_rm_watch', - 'io_uring_enter', - 'io_uring_register', - 'io_uring_setup', - 'landlock_add_rule', - 'landlock_create_ruleset', - 'landlock_restrict_self', - 'lsetxattr', - 'newfstatat', - 'putmsg', - 'putpmsg', - 'pkey_alloc', - 'pkey_free', - 'pkey_mprotect', - 'prlimit64', - 'pread64', - 'pselect6', - 'pwrite64', - 'quotactl_fd', - 'rt_sigpending', - 'seccomp', - 'semtimedop', - 'sethostname', - ] + ltp_syscalls_path = "testcases/kernel/syscalls" + white_list = { + 'bpf': f'{ltp_syscalls_path}/bpf', + 'epoll_pwait2': f'{ltp_syscalls_path}/epoll_pwait', + 'fadvise64': f'{ltp_syscalls_path}/fadvise', + 'fanotify_init': f'{ltp_syscalls_path}/fanotify', + 'fanotify_mark': f'{ltp_syscalls_path}/fanotify', + 'futex': f'{ltp_syscalls_path}/futex', + 'getdents64': f'{ltp_syscalls_path}/gettdents', + 'inotify_add_watch': f'{ltp_syscalls_path}/inotify', + 'inotify_init': f'{ltp_syscalls_path}/inotify', + 'inotify_rm_watch': f'{ltp_syscalls_path}/inotify', + 'io_uring_enter': f'{ltp_syscalls_path}/io_uring', + 'io_uring_register': f'{ltp_syscalls_path}/io_uring', + 'io_uring_setup': f'{ltp_syscalls_path}/io_uring', + 'landlock_add_rule': f'{ltp_syscalls_path}/landlock', + 'landlock_create_ruleset': f'{ltp_syscalls_path}/landlock', + 'landlock_restrict_self': f'{ltp_syscalls_path}/landlock', + 'lsetxattr': f'{ltp_syscalls_path}/lgetxattr', + 'newfstatat': f'{ltp_syscalls_path}/fstatat', + 'pkey_alloc': f'{ltp_syscalls_path}/pkeys', + 'pkey_free': f'{ltp_syscalls_path}/pkeys', + 'pkey_mprotect': f'{ltp_syscalls_path}/pkeys', + 'prlimit64': f'{ltp_syscalls_path}/getrlimit', + 'pread64': f'{ltp_syscalls_path}/pread', + 'pselect6': f'{ltp_syscalls_path}/pselect', + 'pwrite64': f'{ltp_syscalls_path}/pwrite', + 'quotactl_fd': f'{ltp_syscalls_path}/quotactl', + 'rt_sigpending': f'{ltp_syscalls_path}/sigpending', + 'semtimedop': f'{ltp_syscalls_path}/ipc/semop', + 'sethostname': f'{ltp_syscalls_path}/sethostname' + } # populate with not implemented, reserved, unmaintained syscalls defined # inside the syscalls file
@@ -134,6 +133,7 @@ def generate_syscalls_stats(_): if error: return + syscalls_base_dir = "https://github.com/linux-test-project/ltp/tree/master" text = [ 'Syscalls\n', '--------\n\n',
@@ -145,15 +145,33 @@ def generate_syscalls_stats(_): with open("syscalls.tbl", 'r', encoding='utf-8') as data: for line in data: match = regexp.search(line) - if match: - ker_syscalls.append(match.group('syscall')) + if not match: + continue + + ker_syscalls.append(match.group('syscall')) # collect all LTP tested syscalls - ltp_syscalls = [] - for _, _, files in os.walk('../testcases/kernel/syscalls'): + name_patterns = [ + re.compile(r'(?P<name>[a-zA-Z_]+[^_])\d{2}\.c'), + re.compile(r'(?P<name>[a-zA-Z_]+[1-9])_\d{2}\.c'), + ] + ltp_syscalls = {} + for dirpath, _, files in os.walk(f'../{ltp_syscalls_path}'): for myfile in files: - if myfile.endswith('.c'): - ltp_syscalls.append(myfile) + match = None + for pattern in name_patterns: + match = pattern.search(myfile) + if match: + break + + if not match: + continue + + # we need to use relative path from the project root + path = dirpath.replace('../', '') + name = match.group('name') + + ltp_syscalls[name] = f'{syscalls_base_dir}/{path}' # compare kernel syscalls with LTP tested syscalls syscalls = {}
@@ -163,19 +181,19 @@ def generate_syscalls_stats(_): if kersc not in syscalls: if kersc in white_list: - syscalls[kersc] = True + syscalls[kersc] = f'{syscalls_base_dir}/{white_list[kersc]}' continue - syscalls[kersc] = False + syscalls[kersc] = None - for ltpsc in ltp_syscalls: - if ltpsc.startswith(kersc): - syscalls[kersc] = True + for ltpsc, ltpsp in ltp_syscalls.items(): + if ltpsc == kersc: + syscalls[kersc] = ltpsp # generate the statistics file - tested_syscalls = [key for key, val in syscalls.items() if val] - text.append( - 'syscalls which are tested under :master:`testcases/kernel/syscalls`:\n\n') + tested_syscalls = [key for key, val in syscalls.items() if val is not None] + text.append('syscalls which are tested under ' + ':master:`testcases/kernel/syscalls`:\n\n') text.append(f'* kernel syscalls: {len(ker_syscalls)}\n') text.append(f'* tested syscalls: {len(tested_syscalls)}\n\n')
@@ -198,12 +216,12 @@ def generate_syscalls_stats(_): max_columns = 3 - for sysname, tested in syscalls.items(): - if tested: + for sysname, path in syscalls.items(): + if path is not None: if (index_tested % max_columns) == 0: - table_tested.append(f' * - {sysname}\n') + table_tested.append(f' * - `{sysname} <{path}>`_\n') else: - table_tested.append(f' - {sysname}\n') + table_tested.append(f' - `{sysname} <{path}>`_\n') index_tested += 1 else:
Signature · 5 lines
2.43.0 -- Mailing list info: https://lists.linux.it/listinfo/ltp
Re: [LTP] [PATCH 2/2] doc: support for clickable syscalls under stats
Hi Andrea,
TL;DR
I'm glad that my original idea to have links did not get lost :).
Quoted 16 lines
From: Andrea Cervesato <[email protected]>
Under statistics tab, add possibility to click on a syscalls and being redirected to the source code which is testing them.
Signed-off-by Andrea Cervesato <[email protected]>--- doc/conf.py | 120 ++++++++++++++++++++++++++++++++++-------------------------- 1 file changed, 69 insertions(+), 51 deletions(-)doc/conf.py +0
-0@@ -61,39 +61,38 @@ def generate_syscalls_stats(_): # because in some cases (i.e. io_ring) syscalls are tested, but they are
doc/conf.py def generate_syscalls_stats(_): · line 71
Quoted 10 lines
doc/conf.py +0
-8# part of a more complex scenario. In the following list, we define syscalls # which we know they are 100% tested already. - white_list = [ - 'epoll_pwait2', - 'fadvise64', - 'fanotify_init', - 'fanotify_mark', - 'getdents64', - 'getmsg', - 'getpmsg',
You removed getmsg, getpmsg, putmsg and seccomp.
I originally put them in my patch to black_list (or I wanted)
https://patchwork.ozlabs.org/project/ltp/patch/[email protected]/
You haven't noticed you removed them (ok, not everything must be documented), but still it would be nice to remove them from "Untested syscalls" table, therefore I sent v2.
https://patchwork.ozlabs.org/project/ltp/patch/[email protected]/
Quoted 61 lines
doc/conf.py +32
-25- 'inotify_add_watch', - 'inotify_rm_watch', - 'io_uring_enter', - 'io_uring_register', - 'io_uring_setup', - 'landlock_add_rule', - 'landlock_create_ruleset', - 'landlock_restrict_self', - 'lsetxattr', - 'newfstatat', - 'putmsg', - 'putpmsg', - 'pkey_alloc', - 'pkey_free', - 'pkey_mprotect', - 'prlimit64', - 'pread64', - 'pselect6', - 'pwrite64', - 'quotactl_fd', - 'rt_sigpending', - 'seccomp', - 'semtimedop', - 'sethostname', - ] + ltp_syscalls_path = "testcases/kernel/syscalls" + white_list = { + 'bpf': f'{ltp_syscalls_path}/bpf', + 'epoll_pwait2': f'{ltp_syscalls_path}/epoll_pwait', + 'fadvise64': f'{ltp_syscalls_path}/fadvise', + 'fanotify_init': f'{ltp_syscalls_path}/fanotify', + 'fanotify_mark': f'{ltp_syscalls_path}/fanotify', + 'futex': f'{ltp_syscalls_path}/futex', + 'getdents64': f'{ltp_syscalls_path}/gettdents', + 'inotify_add_watch': f'{ltp_syscalls_path}/inotify', + 'inotify_init': f'{ltp_syscalls_path}/inotify', + 'inotify_rm_watch': f'{ltp_syscalls_path}/inotify', + 'io_uring_enter': f'{ltp_syscalls_path}/io_uring', + 'io_uring_register': f'{ltp_syscalls_path}/io_uring', + 'io_uring_setup': f'{ltp_syscalls_path}/io_uring', + 'landlock_add_rule': f'{ltp_syscalls_path}/landlock', + 'landlock_create_ruleset': f'{ltp_syscalls_path}/landlock', + 'landlock_restrict_self': f'{ltp_syscalls_path}/landlock', + 'lsetxattr': f'{ltp_syscalls_path}/lgetxattr', + 'newfstatat': f'{ltp_syscalls_path}/fstatat', + 'pkey_alloc': f'{ltp_syscalls_path}/pkeys', + 'pkey_free': f'{ltp_syscalls_path}/pkeys', + 'pkey_mprotect': f'{ltp_syscalls_path}/pkeys', + 'prlimit64': f'{ltp_syscalls_path}/getrlimit', + 'pread64': f'{ltp_syscalls_path}/pread', + 'pselect6': f'{ltp_syscalls_path}/pselect', + 'pwrite64': f'{ltp_syscalls_path}/pwrite', + 'quotactl_fd': f'{ltp_syscalls_path}/quotactl', + 'rt_sigpending': f'{ltp_syscalls_path}/sigpending', + 'semtimedop': f'{ltp_syscalls_path}/ipc/semop', + 'sethostname': f'{ltp_syscalls_path}/sethostname' + } # populate with not implemented, reserved, unmaintained syscalls defined # inside the syscalls file
doc/conf.py def generate_syscalls_stats(_): · line 136
Quoted 4 lines
doc/conf.py +1
-0@@ -134,6 +133,7 @@ def generate_syscalls_stats(_): if error: return + syscalls_base_dir = "https://github.com/linux-test-project/ltp/tree/master"
nit: we already have some variables for various https://github.com/linux-test-project/ltp
paths. I would personally define at the top variable holding "https://github.com/linux-test-project/ltp" and deriver others from it
(as a separate commit or a part of previous commit).
Quoted 10 lines
doc/conf.py +1
-2text = [ 'Syscalls\n', '--------\n\n',@@ -145,15 +145,33 @@ def generate_syscalls_stats(_): with open("syscalls.tbl", 'r', encoding='utf-8') as data: for line in data: match = regexp.search(line) - if match: - ker_syscalls.append(match.group('syscall')) + if not match:
doc/conf.py def generate_syscalls_stats(_): · line 156
Quoted 10 lines
doc/conf.py +6
-2+ continue + + ker_syscalls.append(match.group('syscall')) # collect all LTP tested syscalls - ltp_syscalls = [] - for _, _, files in os.walk('../testcases/kernel/syscalls'): + name_patterns = [ + re.compile(r'(?P<name>[a-zA-Z_]+[^_])\d{2}\.c'), + re.compile(r'(?P<name>[a-zA-Z_]+[1-9])_\d{2}\.c'),
Using regexp is probably better than my way to set paths (fewer things to maintain).
Thanks!
Kind regards,
Petr
Signature · 1 line
Mailing list info: https://lists.linux.it/listinfo/ltp
Re: [LTP] [PATCH 2/2] doc: support for clickable syscalls under stats
Hi Petr,
Quoted 33 lines · Petr Vorel
Hi Andrea,
TL;DR
Reviewed-by Petr Vorel <[email protected]>I'm glad that my original idea to have links did not get lost :).
Quoted 13 lines
From: Andrea Cervesato <[email protected]> Under statistics tab, add possibility to click on a syscalls and being redirected to the source code which is testing them.
Signed-off-by Andrea Cervesato <[email protected]>--- doc/conf.py | 120 ++++++++++++++++++++++++++++++++++-------------------------- 1 file changed, 69 insertions(+), 51 deletions(-)doc/conf.py +0
-0@@ -61,39 +61,38 @@ def generate_syscalls_stats(_): # because in some cases (i.e. io_ring) syscalls are tested, but they aredoc/conf.py def generate_syscalls_stats(_): · line 71
Quoted 10 lines
doc/conf.py +0
-8# part of a more complex scenario. In the following list, we define syscalls # which we know they are 100% tested already. - white_list = [ - 'epoll_pwait2', - 'fadvise64', - 'fanotify_init', - 'fanotify_mark', - 'getdents64', - 'getmsg', - 'getpmsg',You removed getmsg, getpmsg, putmsg and seccomp.
Quoted 7 lines
I originally put them in my patch to
black_list(or I wanted)
https://patchwork.ozlabs.org/project/ltp/patch/[email protected]/You haven't noticed you removed them (ok, not everything must be documented), but still it would be nice to remove them from "Untested syscalls" table, therefore I sent v2.
https://patchwork.ozlabs.org/project/ltp/patch/[email protected]/
Sure, I will send your follow up patch, thanks.
Quoted 59 lines
Quoted 59 lines
doc/conf.py +32
-25- 'inotify_add_watch', - 'inotify_rm_watch', - 'io_uring_enter', - 'io_uring_register', - 'io_uring_setup', - 'landlock_add_rule', - 'landlock_create_ruleset', - 'landlock_restrict_self', - 'lsetxattr', - 'newfstatat', - 'putmsg', - 'putpmsg', - 'pkey_alloc', - 'pkey_free', - 'pkey_mprotect', - 'prlimit64', - 'pread64', - 'pselect6', - 'pwrite64', - 'quotactl_fd', - 'rt_sigpending', - 'seccomp', - 'semtimedop', - 'sethostname', - ] + ltp_syscalls_path = "testcases/kernel/syscalls" + white_list = { + 'bpf': f'{ltp_syscalls_path}/bpf', + 'epoll_pwait2': f'{ltp_syscalls_path}/epoll_pwait', + 'fadvise64': f'{ltp_syscalls_path}/fadvise', + 'fanotify_init': f'{ltp_syscalls_path}/fanotify', + 'fanotify_mark': f'{ltp_syscalls_path}/fanotify', + 'futex': f'{ltp_syscalls_path}/futex', + 'getdents64': f'{ltp_syscalls_path}/gettdents', + 'inotify_add_watch': f'{ltp_syscalls_path}/inotify', + 'inotify_init': f'{ltp_syscalls_path}/inotify', + 'inotify_rm_watch': f'{ltp_syscalls_path}/inotify', + 'io_uring_enter': f'{ltp_syscalls_path}/io_uring', + 'io_uring_register': f'{ltp_syscalls_path}/io_uring', + 'io_uring_setup': f'{ltp_syscalls_path}/io_uring', + 'landlock_add_rule': f'{ltp_syscalls_path}/landlock', + 'landlock_create_ruleset': f'{ltp_syscalls_path}/landlock', + 'landlock_restrict_self': f'{ltp_syscalls_path}/landlock', + 'lsetxattr': f'{ltp_syscalls_path}/lgetxattr', + 'newfstatat': f'{ltp_syscalls_path}/fstatat', + 'pkey_alloc': f'{ltp_syscalls_path}/pkeys', + 'pkey_free': f'{ltp_syscalls_path}/pkeys', + 'pkey_mprotect': f'{ltp_syscalls_path}/pkeys', + 'prlimit64': f'{ltp_syscalls_path}/getrlimit', + 'pread64': f'{ltp_syscalls_path}/pread', + 'pselect6': f'{ltp_syscalls_path}/pselect', + 'pwrite64': f'{ltp_syscalls_path}/pwrite', + 'quotactl_fd': f'{ltp_syscalls_path}/quotactl', + 'rt_sigpending': f'{ltp_syscalls_path}/sigpending', + 'semtimedop': f'{ltp_syscalls_path}/ipc/semop', + 'sethostname': f'{ltp_syscalls_path}/sethostname' + } # populate with not implemented, reserved, unmaintained syscalls defined # inside the syscalls file
Quoted 8 lines
doc/conf.py def generate_syscalls_stats(_): · line 135
Quoted 4 lines
doc/conf.py +1
-0@@ -134,6 +133,7 @@ def generate_syscalls_stats(_): if error: return + syscalls_base_dir = "https://github.com/linux-test-project/ltp/tree/master"nit: we already have some variables for various https://github.com/linux-test-project/ltp
paths. I would personally define at the top variable holding "https://github.com/linux-test-project/ltp" and deriver others from it
(as a separate commit or a part of previous commit).
Good catch, I will include it in the last patch since it's required by stats generator specifically.
Quoted 23 lines
Quoted 10 lines
doc/conf.py +0
-2text = [ 'Syscalls\n', '--------\n\n',@@ -145,15 +145,33 @@ def generate_syscalls_stats(_): with open("syscalls.tbl", 'r', encoding='utf-8') as data: for line in data: match = regexp.search(line) - if match: - ker_syscalls.append(match.group('syscall'))doc/conf.py def generate_syscalls_stats(_): · line 155
Quoted 10 lines
doc/conf.py +7
-2+ if not match: + continue + + ker_syscalls.append(match.group('syscall')) # collect all LTP tested syscalls - ltp_syscalls = [] - for _, _, files in os.walk('../testcases/kernel/syscalls'): + name_patterns = [ + re.compile(r'(?P<name>[a-zA-Z_]+[^_])\d{2}\.c'), + re.compile(r'(?P<name>[a-zA-Z_]+[1-9])_\d{2}\.c'),Using regexp is probably better than my way to set paths (fewer things to maintain).
Quoted 4 lines
Thanks!
Kind regards,
Petr
I will push with the given suggestions and follow up patch, thanks.
Andrea
Signature · 1 line
Mailing list info: https://lists.linux.it/listinfo/ltp
Pushed with follow up patch for blacklist.
Thanks!
Andrea
Quoted 18 lines · Andrea Cervesato
This is an optimization of the documentation, which adds a bit of refactoring and clickable syscalls names inside the statistics table. In this way, users can click on a syscall and being redirected to the source code which is testing it.
Signed-off-by Andrea Cervesato <[email protected]>---
Andrea Cervesato (2):
doc: cleanupconf.pyscript
doc: support for clickable syscalls under statsdoc/conf.py | 158 ++++++++++++++++++++++++++++++++++-------------------------- 1 file changed, 91 insertions(+), 67 deletions(-)---
base-commit:8f2292af050fd64c447dd462880d0b0a19d79448
change-id: 20241204-doc_syscalls_link-a184545389baBest regards,
Signature · 1 line
Mailing list info: https://lists.linux.it/listinfo/ltp