tests/functional: add more CLI args
v2 cover tests/functional: add more CLI args
This series adds extra CLI args for functional tests, useful for developers that run test files directly.
It depends on a previous patch that adds a --debug CLI arg, and is encoded as a b4 change-id dependency, so it should be fetched and applied automatically when using b4.
---
Changes in v2:
- Fixed invalid -k value passing when -k was specified more than once
(thanks Paolo)
- Link to v1: https://lore.kernel.org/qemu-devel/[email protected]
---
Manos Pitsidianakis (3):
tests/functional: add --keep-scratch CLI arg
tests/functional: add --list-tests CLI arg
tests/functional: add -k TEST_NAME_PATTERN CLI argtests/functional/qemu_test/testcase.py | 51 +++++++++++++++++++++++++++++----- 1 file changed, 44 insertions(+), 7 deletions(-)
---
base-commit: 3656e761bcdd207b7759cdcd608212d2a6f9c12d
change-id: 20250718-functional_tests_args-12cdb5e56d84
prerequisite-change-id: 20250716-functional_tests_debug_arg-aa0a5f6b9375:v2
prerequisite-patch-id: 4ccc8f39ffb382d31c8e6450c43a5f8d177af044
--
γαῖα πυρί μιχθήτω
PATCH v2 1/3 tests/functional: add --keep-scratch CLI arg
Add CLI arg to keep scratch files after test execution, equivalent to setting QEMU_TEST_KEEP_SCRATCH env var.
--- tests/functional/qemu_test/testcase.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-)
tests/functional/qemu_test/testcase.py +13 -1
@@ -43,6 +43,13 @@ def parse_args(test_name: str) -> argparse.Namespace: help="Also print test and console logs on stdout. This will make the" " TAP output invalid and is meant for debugging only.", ) + parser.add_argument( + "--keep-scratch", + action="store_true", + help="Do not purge any scratch files created during the tests. " + "This is equivalent to setting QEMU_TEST_KEEP_SCRATCH=1 in the " + "environment.", + ) return parser.parse_args()
@@ -214,6 +221,9 @@ def setUp(self): path = os.path.basename(sys.argv[0])[:-3] args = parse_args(path) self.stdout_handler = None + self.keep_scratch = ( + "QEMU_TEST_KEEP_SCRATCH" in os.environ or args.keep_scratch + ) if args.debug: self.stdout_handler = logging.StreamHandler(sys.stdout) self.stdout_handler.setLevel(logging.DEBUG)
@@ -255,8 +265,10 @@ def setUp(self): self.skipTest('One or more assets is not available') def tearDown(self): - if "QEMU_TEST_KEEP_SCRATCH" not in os.environ: + if not self.keep_scratch: shutil.rmtree(self.workdir) + else: + self.log.info(f"Kept scratch files in {self.workdir}") if self.socketdir is not None: shutil.rmtree(self.socketdir.name) self.socketdir = None
Signature · 1 line
2.47.2
Re: [PATCH v2 1/3] tests/functional: add --keep-scratch CLI arg
Quoted 40 lines · Manos Pitsidianakis
Add CLI arg to keep scratch files after test execution, equivalent to setting
QEMU_TEST_KEEP_SCRATCHenv var.Suggested-by Alex Bennée <[email protected]>Signed-off-by Manos Pitsidianakis <[email protected]>--- tests/functional/qemu_test/testcase.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-)tests/functional/qemu_test/testcase.py +10
-0@@ -43,6 +43,13 @@ def parse_args(test_name: str) -> argparse.Namespace: help="Also print test and console logs on stdout. This will make the" " TAP output invalid and is meant for debugging only.", ) + parser.add_argument( + "--keep-scratch", + action="store_true", + help="Do not purge any scratch files created during the tests. " + "This is equivalent to setting QEMU_TEST_KEEP_SCRATCH=1 in the " + "environment.", + ) return parser.parse_args()@@ -214,6 +221,9 @@ def setUp(self): path = os.path.basename(sys.argv[0])[:-3] args = parse_args(path) self.stdout_handler = None + self.keep_scratch = ( + "QEMU_TEST_KEEP_SCRATCH" in os.environ or args.keep_scratch + ) if args.debug: self.stdout_handler = logging.StreamHandler(sys.stdout) self.stdout_handler.setLevel(logging.DEBUG)@@ -255,8 +265,10 @@ def setUp(self): self.skipTest('One or more assets is not available')
tests/functional/qemu_test/testcase.py def setUp(self): · line 274
Quoted 10 lines
tests/functional/qemu_test/testcase.py +3
-1def tearDown(self): - if "QEMU_TEST_KEEP_SCRATCH" not in os.environ: + if not self.keep_scratch: shutil.rmtree(self.workdir) + else: + self.log.info(f"Kept scratch files in {self.workdir}") if self.socketdir is not None: shutil.rmtree(self.socketdir.name) self.socketdir = None
PATCH v2 2/3 tests/functional: add --list-tests CLI arg
In reply to Manos PitsidianakisAdd CLI argument to list tests and exit.
Example output (current dir is build directory under root dir):
$ export PYTHONPATH=../python:../tests/functional $ export QEMU_TEST_QEMU_BINARY="$(pwd)/qemu-system-aarch64" $ ./pyvenv/bin/python3 ../tests/functional/test_aarch64_virt.py --list-tests test_aarch64_virt_gicv2 (test_aarch64_virt.Aarch64VirtMachine.test_aarch64_virt_gicv2) test_aarch64_virt_gicv3 (test_aarch64_virt.Aarch64VirtMachine.test_aarch64_virt_gicv3) test_alpine_virt_tcg_gic_max (test_aarch64_virt.Aarch64VirtMachine.test_alpine_virt_tcg_gic_max)
--- tests/functional/qemu_test/testcase.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-)
tests/functional/qemu_test/testcase.py +12 -4
@@ -50,6 +50,11 @@ def parse_args(test_name: str) -> argparse.Namespace: "This is equivalent to setting QEMU_TEST_KEEP_SCRATCH=1 in the " "environment.", ) + parser.add_argument( + "--list-tests", + action="store_true", + help="List all tests that would be executed and exit.", + ) return parser.parse_args()
@@ -280,10 +285,13 @@ def tearDown(self): def main(): path = os.path.basename(sys.argv[0])[:-3] - # If argparse receives --help or an unknown argument, it will raise a - # SystemExit which will get caught by the test runner. Parse the - # arguments here too to handle that case. - parse_args(path) + args = parse_args(path) + if args.list_tests: + loader = unittest.TestLoader() + for test_suite in loader.loadTestsFromName(path): + for test in test_suite: + print(test) + return cache = os.environ.get("QEMU_TEST_PRECACHE", None) if cache is not None:
Signature · 1 line
2.47.2
Re: [PATCH v2 2/3] tests/functional: add --list-tests CLI arg
Quoted 33 lines · Manos Pitsidianakis
Add CLI argument to list tests and exit.
Example output (current dir is build directory under root dir):
$ export PYTHONPATH=../python:../tests/functional $ export QEMU_TEST_QEMU_BINARY="$(pwd)/qemu-system-aarch64" $ ./pyvenv/bin/python3 ../tests/functional/test_aarch64_virt.py --list-tests test_aarch64_virt_gicv2 (test_aarch64_virt.Aarch64VirtMachine.test_aarch64_virt_gicv2) test_aarch64_virt_gicv3 (test_aarch64_virt.Aarch64VirtMachine.test_aarch64_virt_gicv3) test_alpine_virt_tcg_gic_max (test_aarch64_virt.Aarch64VirtMachine.test_alpine_virt_tcg_gic_max)Signed-off-by Manos Pitsidianakis <[email protected]>--- tests/functional/qemu_test/testcase.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-)tests/functional/qemu_test/testcase.py +5
-0@@ -50,6 +50,11 @@ def parse_args(test_name: str) -> argparse.Namespace: "This is equivalent to setting QEMU_TEST_KEEP_SCRATCH=1 in the " "environment.", ) + parser.add_argument( + "--list-tests", + action="store_true", + help="List all tests that would be executed and exit.", + ) return parser.parse_args()
tests/functional/qemu_test/testcase.py def tearDown(self): · line 288
Quoted 8 lines
tests/functional/qemu_test/testcase.py +1
-4@@ -280,10 +285,13 @@ def tearDown(self): def main(): path = os.path.basename(sys.argv[0])[:-3] - # If argparse receives --help or an unknown argument, it will raise a - # SystemExit which will get caught by the test runner. Parse the - # arguments here too to handle that case. - parse_args(path) + args = parse_args(path)
As mentioned for the initial --debug patch already: How's about changing that "args" into a class variable instead and get rid of the parse_args() in the setUp() function?
Thomas
Quoted 10 lines
tests/functional/qemu_test/testcase.py +6
-0+ if args.list_tests: + loader = unittest.TestLoader() + for test_suite in loader.loadTestsFromName(path): + for test in test_suite: + print(test) + return cache = os.environ.get("QEMU_TEST_PRECACHE", None) if cache is not None:
Re: [PATCH v2 2/3] tests/functional: add --list-tests CLI arg
Quoted 48 lines · Thomas Huth
Quoted 33 lines · Manos Pitsidianakis
Add CLI argument to list tests and exit.
Example output (current dir is build directory under root dir):
$ export PYTHONPATH=../python:../tests/functional $ export QEMU_TEST_QEMU_BINARY="$(pwd)/qemu-system-aarch64" $ ./pyvenv/bin/python3 ../tests/functional/test_aarch64_virt.py --list-tests test_aarch64_virt_gicv2 (test_aarch64_virt.Aarch64VirtMachine.test_aarch64_virt_gicv2) test_aarch64_virt_gicv3 (test_aarch64_virt.Aarch64VirtMachine.test_aarch64_virt_gicv3) test_alpine_virt_tcg_gic_max (test_aarch64_virt.Aarch64VirtMachine.test_alpine_virt_tcg_gic_max)Signed-off-by Manos Pitsidianakis <[email protected]>--- tests/functional/qemu_test/testcase.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-)tests/functional/qemu_test/testcase.py +5
-0@@ -50,6 +50,11 @@ def parse_args(test_name: str) -> argparse.Namespace: "This is equivalent to setting QEMU_TEST_KEEP_SCRATCH=1 in the " "environment.", ) + parser.add_argument( + "--list-tests", + action="store_true", + help="List all tests that would be executed and exit.", + ) return parser.parse_args()tests/functional/qemu_test/testcase.py def tearDown(self): · line 288
Quoted 8 lines
tests/functional/qemu_test/testcase.py +1
-4@@ -280,10 +285,13 @@ def tearDown(self): def main(): path = os.path.basename(sys.argv[0])[:-3] - # If argparse receives --help or an unknown argument, it will raise a - # SystemExit which will get caught by the test runner. Parse the - # arguments here too to handle that case. - parse_args(path) + args = parse_args(path)As mentioned for the initial --debug patch already: How's about changing that "args" into a class variable instead and get rid of the
parse_args()in thesetUp()function?
Quoted 1 line
Thomas
Yes that sounds good. Will merge the two patch series and resubmit, thanks Thomas!
Signature · 2 lines
Manos Pitsidianakis Emulation and Virtualization Engineer at Linaro Ltd
PATCH v2 3/3 tests/functional: add -k TEST_NAME_PATTERN CLI arg
In reply to Manos PitsidianakisAdd a CLI argument that takes fnmatch(3)-style patterns as value and can be specified many times. Only tests that match the pattern will be executed. This argument is passed to unittest.main which takes the same argument.
--- tests/functional/qemu_test/testcase.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-)
tests/functional/qemu_test/testcase.py +19 -2
@@ -16,6 +16,7 @@ import os from pathlib import Path import pycotap +import itertools import shutil from subprocess import run import sys
@@ -55,6 +56,14 @@ def parse_args(test_name: str) -> argparse.Namespace: action="store_true", help="List all tests that would be executed and exit.", ) + parser.add_argument( + "-k", + dest="test_name_patterns", + action="append", + type=str, + help="Only run tests which match the given substring. " + "This argument is passed to unittest.main verbatim.", + ) return parser.parse_args()
@@ -300,8 +309,16 @@ def main(): tr = pycotap.TAPTestRunner(message_log = pycotap.LogMode.LogToError, test_output_log = pycotap.LogMode.LogToError) - res = unittest.main(module = None, testRunner = tr, exit = False, - argv=["__dummy__", path]) + argv = ["__dummy__", path] + ( + list( + itertools.chain.from_iterable( + ["-k", x] for x in args.test_name_patterns + ) + ) + if args.test_name_patterns + else [] + ) + res = unittest.main(module=None, testRunner=tr, exit=False, argv=argv) for (test, message) in res.result.errors + res.result.failures: if hasattr(test, "log_filename"):
Signature · 1 line
2.47.2
Re: [PATCH v2 3/3] tests/functional: add -k TEST_NAME_PATTERN CLI arg
Quoted 47 lines · Manos Pitsidianakis
Add a CLI argument that takes fnmatch(3)-style patterns as value and can be specified many times. Only tests that match the pattern will be executed. This argument is passed to unittest.main which takes the same argument.
Signed-off-by Manos Pitsidianakis <[email protected]>--- tests/functional/qemu_test/testcase.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-)tests/functional/qemu_test/testcase.py +12
-2@@ -16,6 +16,7 @@ import os from pathlib import Path import pycotap +import itertools import shutil from subprocess import run import sys@@ -55,6 +56,14 @@ def parse_args(test_name: str) -> argparse.Namespace: action="store_true", help="List all tests that would be executed and exit.", ) + parser.add_argument( + "-k", + dest="test_name_patterns", + action="append", + type=str, + help="Only run tests which match the given substring. " + "This argument is passed to unittest.main verbatim.", + ) return parser.parse_args()@@ -300,8 +309,16 @@ def main(): tr = pycotap.TAPTestRunner(message_log = pycotap.LogMode.LogToError, test_output_log = pycotap.LogMode.LogToError) - res = unittest.main(module = None, testRunner = tr, exit = False, - argv=["__dummy__", path]) + argv = ["__dummy__", path] + ( + list( + itertools.chain.from_iterable(
tests/functional/qemu_test/testcase.py def main(): · line 324
Quoted 10 lines
tests/functional/qemu_test/testcase.py +7
-0+ ["-k", x] for x in args.test_name_patterns + ) + ) + if args.test_name_patterns + else [] + ) + res = unittest.main(module=None, testRunner=tr, exit=False, argv=argv) for (test, message) in res.result.errors + res.result.failures: if hasattr(test, "log_filename"):
Good idea!
Quoted 23 lines · Manos Pitsidianakis
This series adds extra CLI args for functional tests, useful for developers that run test files directly.
It depends on a previous patch that adds a --debug CLI arg, and is encoded as a b4 change-id dependency, so it should be fetched and applied automatically when using b4.
Signed-off-by Manos Pitsidianakis <[email protected]>---
Changes in v2:
- Fixed invalid -k value passing when -k was specified more than once
(thanks Paolo)
- Link to v1: https://lore.kernel.org/qemu-devel/[email protected]---
Manos Pitsidianakis (3):tests/functional: add --keep-scratch CLI arg tests/functional: add --list-tests CLI arg tests/functional: add -k TEST_NAME_PATTERN CLI argtests/functional/qemu_test/testcase.py | 51 +++++++++++++++++++++++++++++----- 1 file changed, 44 insertions(+), 7 deletions(-)
Quoted 9 lines
---
base-commit:3656e761bcdd207b7759cdcd608212d2a6f9c12d
change-id: 20250718-functional_tests_args-12cdb5e56d84
prerequisite-change-id: 20250716-functional_tests_debug_arg-aa0a5f6b9375:v2
prerequisite-patch-id:4ccc8f39ffb382d31c8e6450c43a5f8d177af044--
γαῖα πυρί μιχθήτω
Gentle ping