Convert the MIPS replay tests to the functional framework
cover Convert the MIPS replay tests to the functional framework
tests/avocado/replay_kernel.py is a rather big file with a lot of Avocado-based tests in it. But in the long run, we rather want to get away from Avocado, so we eventually have to convert these tests to the functional framework. For this, we have to separate the tests by target architecture, since the functional framework cannot detect this on the fly like Avocado (which e.g. looks at the tag annotations in the comments). Let's start the convertion by handling the MIPS-related tests first (the other replay_kernel tests will follow later).
Thomas Huth (5):
tests/functional: Add a decorator for skipping long running tests tests/functional: Add the ReplayKernelBase class tests/functional/test_mipsel_malta: Convert the mipsel replay tests tests/functional/test_mips64el_malta: Convert the mips64el replay
tests
tests/functional/test_mips_malta: Convert the mips big endian replay
tests
docs/devel/testing/functional.rst | 8 ++ tests/avocado/replay_kernel.py | 165 ----------------------- tests/functional/meson.build | 1 + tests/functional/qemu_test/__init__.py | 2 +- tests/functional/qemu_test/decorators.py | 14 ++ tests/functional/replay_kernel.py | 84 ++++++++++++ tests/functional/test_arm_quanta_gsj.py | 5 +- tests/functional/test_mips64el_malta.py | 26 +++- tests/functional/test_mips_malta.py | 29 +++- tests/functional/test_mipsel_malta.py | 30 ++++- 10 files changed, 189 insertions(+), 175 deletions(-) create mode 100644 tests/functional/replay_kernel.py
Signature · 1 line
2.48.1
PATCH 1/5 tests/functional: Add a decorator for skipping long running tests
Some tests have a very long runtime and might run into timeout issues e.g. when QEMU has been compiled with --enable-debug. Add a decorator for marking them more easily and document the corresponding environment variable that is used to enable the tests.
--- docs/devel/testing/functional.rst | 8 ++++++++ tests/functional/qemu_test/__init__.py | 2 +- tests/functional/qemu_test/decorators.py | 14 ++++++++++++++ tests/functional/test_arm_quanta_gsj.py | 5 +++-- 4 files changed, 26 insertions(+), 3 deletions(-)
docs/devel/testing/functional.rst +8 -0
@@ -351,5 +351,13 @@ the code snippet below: Tests should not live in this state forever and should either be fixed or eventually removed. +QEMU_TEST_TIMEOUT_EXPECTED +^^^^^^^^^^^^^^^^^^^^^^^^^^ +Tests that have a very long runtime and might run into timeout issues +e.g. if the QEMU binary has been compiled with debugging options enabled. +To avoid these timeout issues by default and to save some precious CPU +cycles during normal testing, such tests are disabled by default unless +the QEMU_TEST_TIMEOUT_EXPECTED environment variable has been set. + .. _unittest: https://docs.python.org/3/library/unittest.html
tests/functional/qemu_test/__init__.py +1 -1
@@ -14,7 +14,7 @@ from .testcase import QemuBaseTest, QemuUserTest, QemuSystemTest from .linuxkernel import LinuxKernelTest from .decorators import skipIfMissingCommands, skipIfNotMachine, \ - skipFlakyTest, skipUntrustedTest, skipBigDataTest, \ + skipFlakyTest, skipUntrustedTest, skipBigDataTest, skipLongRuntime, \ skipIfMissingImports from .archive import archive_extract from .uncompress import uncompress
tests/functional/qemu_test/decorators.py +14 -0
@@ -86,6 +86,20 @@ def skipBigDataTest(): return skipUnless(os.getenv('QEMU_TEST_ALLOW_LARGE_STORAGE'), 'Test requires large host storage space') +''' +Decorator to skip execution of tests which have a really long +runtime (and might e.g. time out if QEMU has been compiled with +debugging enabled) unless the $QEMU_TEST_TIMEOUT_EXPECTED +environment variable is set + +Example: + + @skipLongRuntime() +''' +def skipLongRuntime(): + return skipUnless(os.getenv('QEMU_TEST_TIMEOUT_EXPECTED'), + 'Test has a very long runtime and might time out') + ''' Decorator to skip execution of a test if the list of python imports is not available.
tests/functional/test_arm_quanta_gsj.py +3 -2
@@ -8,7 +8,8 @@ from qemu_test import LinuxKernelTest, Asset, exec_command_and_wait_for_pattern from qemu_test import interrupt_interactive_console_until_pattern -from unittest import skipUnless +from qemu_test import skipLongRuntime + class EmcraftSf2Machine(LinuxKernelTest):
@@ -32,7 +33,7 @@ class EmcraftSf2Machine(LinuxKernelTest): '20200711-gsj-qemu-0/nuvoton-npcm730-gsj.dtb'), '3249b2da787d4b9ad4e61f315b160abfceb87b5e1895a7ce898ce7f40c8d4045') - @skipUnless(os.getenv('QEMU_TEST_TIMEOUT_EXPECTED'), 'Test might timeout') + @skipLongRuntime() def test_arm_quanta_gsj(self): self.set_machine('quanta-gsj') image_path = self.uncompress(self.ASSET_IMAGE, format='gz')
Signature · 1 line
2.48.1
Re: [PATCH 1/5] tests/functional: Add a decorator for skipping long running tests
Quoted 6 lines · Thomas Huth
Some tests have a very long runtime and might run into timeout issues e.g. when QEMU has been compiled with --enable-debug. Add a decorator for marking them more easily and document the corresponding environment variable that is used to enable the tests.
Quoted 7 lines
Signed-off-by Thomas Huth <[email protected]>--- docs/devel/testing/functional.rst | 8 ++++++++ tests/functional/qemu_test/__init__.py | 2 +- tests/functional/qemu_test/decorators.py | 14 ++++++++++++++ tests/functional/test_arm_quanta_gsj.py | 5 +++-- 4 files changed, 26 insertions(+), 3 deletions(-)
Nice!
Re: [PATCH 1/5] tests/functional: Add a decorator for skipping long running tests
In reply to Thomas HuthQuoted 37 lines · Thomas Huth
Some tests have a very long runtime and might run into timeout issues e.g. when QEMU has been compiled with --enable-debug. Add a decorator for marking them more easily and document the corresponding environment variable that is used to enable the tests.
Signed-off-by Thomas Huth <[email protected]>--- docs/devel/testing/functional.rst | 8 ++++++++ tests/functional/qemu_test/__init__.py | 2 +- tests/functional/qemu_test/decorators.py | 14 ++++++++++++++ tests/functional/test_arm_quanta_gsj.py | 5 +++-- 4 files changed, 26 insertions(+), 3 deletions(-)docs/devel/testing/functional.rst +8
-0@@ -351,5 +351,13 @@ the code snippet below: Tests should not live in this state forever and should either be fixed or eventually removed. +QEMU_TEST_TIMEOUT_EXPECTED +^^^^^^^^^^^^^^^^^^^^^^^^^^ +Tests that have a very long runtime and might run into timeout issues +e.g. if the QEMU binary has been compiled with debugging options enabled. +To avoid these timeout issues by default and to save some precious CPU +cycles during normal testing, such tests are disabled by default unless +the QEMU_TEST_TIMEOUT_EXPECTED environment variable has been set. + .. _unittest: https://docs.python.org/3/library/unittest.htmltests/functional/qemu_test/__init__.py +0
-0
tests/functional/qemu_test/__init__.py line 17
Quoted 5 lines
tests/functional/qemu_test/__init__.py +1
-1@@ -14,7 +14,7 @@ from .testcase import QemuBaseTest, QemuUserTest, QemuSystemTest from .linuxkernel import LinuxKernelTest from .decorators import skipIfMissingCommands, skipIfNotMachine, \ - skipFlakyTest, skipUntrustedTest, skipBigDataTest, \ + skipFlakyTest, skipUntrustedTest, skipBigDataTest, skipLongRuntime, \
s/Runtime/RunningTime/, but actually in terms of naming convention, 'skipSlowTest' would fit better.
Quoted 12 lines
tests/functional/qemu_test/decorators.py +4
-0@@ -86,6 +86,20 @@ def skipBigDataTest(): return skipUnless(os.getenv('QEMU_TEST_ALLOW_LARGE_STORAGE'), 'Test requires large host storage space') +''' +Decorator to skip execution of tests which have a really long +runtime (and might e.g. time out if QEMU has been compiled with +debugging enabled) unless the $QEMU_TEST_TIMEOUT_EXPECTED
tests/functional/qemu_test/decorators.py def skipBigDataTest(): · line 102
Quoted 10 lines
tests/functional/qemu_test/decorators.py +10
-0+environment variable is set + +Example: + + @skipLongRuntime() +''' +def skipLongRuntime(): + return skipUnless(os.getenv('QEMU_TEST_TIMEOUT_EXPECTED'), + 'Test has a very long runtime and might time out') +
You're preserving the existnig env var which is good, but I have a little niggling desire to unify the naming conventions:
skipFlakyTest -> $QEMU_TEST_ALLOW_FLAKY
skipUntrustedTest -> $QEMU_TEST_ALLOW_UNTRUSTED
skipBigDataTest -> $QEMU_TEST_ALLOW_BIG_DATA
skipSlowTest -> $QEMU_TEST_ALLOW_SLOW
Could be a separate patch though if you like the idea.
Quoted 29 lines
tests/functional/qemu_test/decorators.py +0
-0''' Decorator to skip execution of a test if the list of python imports is not available.tests/functional/test_arm_quanta_gsj.py +3
-2@@ -8,7 +8,8 @@ from qemu_test import LinuxKernelTest, Asset, exec_command_and_wait_for_pattern from qemu_test import interrupt_interactive_console_until_pattern -from unittest import skipUnless +from qemu_test import skipLongRuntime + class EmcraftSf2Machine(LinuxKernelTest):@@ -32,7 +33,7 @@ class EmcraftSf2Machine(LinuxKernelTest): '20200711-gsj-qemu-0/nuvoton-npcm730-gsj.dtb'), '3249b2da787d4b9ad4e61f315b160abfceb87b5e1895a7ce898ce7f40c8d4045') - @skipUnless(os.getenv('QEMU_TEST_TIMEOUT_EXPECTED'), 'Test might timeout') + @skipLongRuntime() def test_arm_quanta_gsj(self): self.set_machine('quanta-gsj') image_path = self.uncompress(self.ASSET_IMAGE, format='gz')Signature · 1 line
2.48.1
With regards,
Daniel
Signature · 3 lines
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
Re: [PATCH 1/5] tests/functional: Add a decorator for skipping long running tests
Quoted 32 lines · Daniel P. Berrangé
Quoted 31 lines · Thomas Huth
Some tests have a very long runtime and might run into timeout issues e.g. when QEMU has been compiled with --enable-debug. Add a decorator for marking them more easily and document the corresponding environment variable that is used to enable the tests.
Signed-off-by Thomas Huth <[email protected]>--- docs/devel/testing/functional.rst | 8 ++++++++ tests/functional/qemu_test/__init__.py | 2 +- tests/functional/qemu_test/decorators.py | 14 ++++++++++++++ tests/functional/test_arm_quanta_gsj.py | 5 +++-- 4 files changed, 26 insertions(+), 3 deletions(-)docs/devel/testing/functional.rst +8
-0@@ -351,5 +351,13 @@ the code snippet below: Tests should not live in this state forever and should either be fixed or eventually removed. +QEMU_TEST_TIMEOUT_EXPECTED +^^^^^^^^^^^^^^^^^^^^^^^^^^ +Tests that have a very long runtime and might run into timeout issues +e.g. if the QEMU binary has been compiled with debugging options enabled. +To avoid these timeout issues by default and to save some precious CPU +cycles during normal testing, such tests are disabled by default unless +the QEMU_TEST_TIMEOUT_EXPECTED environment variable has been set. +
Quoted 14 lines
tests/functional/qemu_test/__init__.py line 17
Quoted 11 lines
docs/devel/testing/functional.rst +0
-0.. _unittest: https://docs.python.org/3/library/unittest.htmltests/functional/qemu_test/__init__.py +1
-1@@ -14,7 +14,7 @@ from .testcase import QemuBaseTest, QemuUserTest, QemuSystemTest from .linuxkernel import LinuxKernelTest from .decorators import skipIfMissingCommands, skipIfNotMachine, \ - skipFlakyTest, skipUntrustedTest, skipBigDataTest, \ + skipFlakyTest, skipUntrustedTest, skipBigDataTest, skipLongRuntime, \s/Runtime/RunningTime/, but actually in terms of naming convention, 'skipSlowTest' would fit better.
Ack, that sounds better, indeed.
Quoted 28 lines
Quoted 13 lines
tests/functional/qemu_test/decorators.py +4
-0@@ -86,6 +86,20 @@ def skipBigDataTest(): return skipUnless(os.getenv('QEMU_TEST_ALLOW_LARGE_STORAGE'), 'Test requires large host storage space') +''' +Decorator to skip execution of tests which have a really long +runtime (and might e.g. time out if QEMU has been compiled with +debugging enabled) unless the $QEMU_TEST_TIMEOUT_EXPECTEDtests/functional/qemu_test/decorators.py def skipBigDataTest(): · line 102
Quoted 10 lines
tests/functional/qemu_test/decorators.py +10
-0+environment variable is set + +Example: + + @skipLongRuntime() +''' +def skipLongRuntime(): + return skipUnless(os.getenv('QEMU_TEST_TIMEOUT_EXPECTED'), + 'Test has a very long runtime and might time out') +You're preserving the existnig env var which is good, but I have a little niggling desire to unify the naming conventions:
Quoted 6 lines
skipFlakyTest -> $QEMU_TEST_ALLOW_FLAKY skipUntrustedTest -> $QEMU_TEST_ALLOW_UNTRUSTED skipBigDataTest -> $QEMU_TEST_ALLOW_BIG_DATA skipSlowTest -> $QEMU_TEST_ALLOW_SLOWCould be a separate patch though if you like the idea.
I like the idea, some of the others are already starting with QEMU_TEST_ALLOW_, so the renaming makes sense to me, too. I'll change this patch in v2, but I'll leave the others for now - feel free to send a patch for them, if not I'll tackle them sometime later.
Thomas
PATCH 2/5 tests/functional: Add the ReplayKernelBase class
In reply to Thomas HuthCopy the ReplayKernelBase class from the avocado tests. We are going to need it to convert the related replay tests in the following patches.
--- tests/functional/replay_kernel.py | 84 +++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 tests/functional/replay_kernel.py
tests/functional/replay_kernel.py new file +84 -0
@@ -0,0 +1,84 @@ +# Record/replay test that boots a Linux kernel +# +# Copyright (c) 2020 ISP RAS +# +# Author: +# Pavel Dovgalyuk <[email protected]> +# +# This work is licensed under the terms of the GNU GPL, version 2 or +# later. See the COPYING file in the top-level directory. + +import os +import logging +import time +import subprocess + +from qemu_test.linuxkernel import LinuxKernelTest + +class ReplayKernelBase(LinuxKernelTest): + """ + Boots a Linux kernel in record mode and checks that the console + is operational and the kernel command line is properly passed + from QEMU to the kernel. + Then replays the same scenario and verifies, that QEMU correctly + terminates. + """ + + timeout = 180 + REPLAY_KERNEL_COMMAND_LINE = 'printk.time=1 panic=-1 ' + + def run_vm(self, kernel_path, kernel_command_line, console_pattern, + record, shift, args, replay_path): + # icount requires TCG to be available + self.require_accelerator('tcg') + + logger = logging.getLogger('replay') + start_time = time.time() + vm = self.get_vm() + vm.set_console() + if record: + logger.info('recording the execution...') + mode = 'record' + else: + logger.info('replaying the execution...') + mode = 'replay' + vm.add_args('-icount', 'shift=%s,rr=%s,rrfile=%s' % + (shift, mode, replay_path), + '-kernel', kernel_path, + '-append', kernel_command_line, + '-net', 'none', + '-no-reboot') + if args: + vm.add_args(*args) + vm.launch() + self.wait_for_console_pattern(console_pattern, vm) + if record: + vm.shutdown() + logger.info('finished the recording with log size %s bytes' + % os.path.getsize(replay_path)) + self.run_replay_dump(replay_path) + logger.info('successfully tested replay-dump.py') + else: + vm.wait() + logger.info('successfully finished the replay') + elapsed = time.time() - start_time + logger.info('elapsed time %.2f sec' % elapsed) + return elapsed + + def run_replay_dump(self, replay_path): + try: + subprocess.check_call(["./scripts/replay-dump.py", + "-f", replay_path], + stdout=subprocess.DEVNULL) + except subprocess.CalledProcessError: + self.fail('replay-dump.py failed') + + def run_rr(self, kernel_path, kernel_command_line, console_pattern, + shift=7, args=None): + replay_path = os.path.join(self.workdir, 'replay.bin') + t1 = self.run_vm(kernel_path, kernel_command_line, console_pattern, + True, shift, args, replay_path) + t2 = self.run_vm(kernel_path, kernel_command_line, console_pattern, + False, shift, args, replay_path) + logger = logging.getLogger('replay') + logger.info('replay overhead {:.2%}'.format(t2 / t1 - 1))
Signature · 1 line
2.48.1
Re: [PATCH 2/5] tests/functional: Add the ReplayKernelBase class
Quoted 8 lines · Thomas Huth
Copy the ReplayKernelBase class from the avocado tests. We are going to need it to convert the related replay tests in the following patches.
Signed-off-by Thomas Huth <[email protected]>--- tests/functional/replay_kernel.py | 84 +++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 tests/functional/replay_kernel.py
With regards,
Daniel
Signature · 3 lines
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
PATCH 3/5 tests/functional/test_mipsel_malta: Convert the mipsel replay tests
In reply to Thomas HuthMove the mipsel replay tests from tests/avocado/replay_kernel.py to the functional framework. Since the functional tests should be run per target, we cannot stick all replay tests in one file. Thus let's add these tests to the file where we already use the same asset already.
--- tests/avocado/replay_kernel.py | 54 --------------------------- tests/functional/meson.build | 1 + tests/functional/test_mipsel_malta.py | 30 +++++++++++++-- 3 files changed, 28 insertions(+), 57 deletions(-)
tests/avocado/replay_kernel.py +0 -54
@@ -521,57 +521,3 @@ def test_mips64el_malta_5KEc_cpio(self): console_pattern = 'Boot successful.' self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5, args=('-initrd', initrd_path)) - - def do_test_mips_malta32el_nanomips(self, kernel_path_xz): - kernel_path = self.workdir + "kernel" - with lzma.open(kernel_path_xz, 'rb') as f_in: - with open(kernel_path, 'wb') as f_out: - shutil.copyfileobj(f_in, f_out) - - kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE + - 'mem=256m@@0x0 ' - 'console=ttyS0') - console_pattern = 'Kernel command line: %s' % kernel_command_line - self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5) - - def test_mips_malta32el_nanomips_4k(self): - """ - :avocado: tags=arch:mipsel - :avocado: tags=machine:malta - :avocado: tags=endian:little - :avocado: tags=cpu:I7200 - """ - kernel_url = ('http://mipsdistros.mips.com/LinuxDistro/nanomips/' - 'kernels/v4.15.18-432-gb2eb9a8b07a1-20180627102142/' - 'generic_nano32r6el_page4k.xz') - kernel_hash = '477456aafd2a0f1ddc9482727f20fe9575565dd6' - kernel_path_xz = self.fetch_asset(kernel_url, asset_hash=kernel_hash) - self.do_test_mips_malta32el_nanomips(kernel_path_xz) - - def test_mips_malta32el_nanomips_16k_up(self): - """ - :avocado: tags=arch:mipsel - :avocado: tags=machine:malta - :avocado: tags=endian:little - :avocado: tags=cpu:I7200 - """ - kernel_url = ('http://mipsdistros.mips.com/LinuxDistro/nanomips/' - 'kernels/v4.15.18-432-gb2eb9a8b07a1-20180627102142/' - 'generic_nano32r6el_page16k_up.xz') - kernel_hash = 'e882868f944c71c816e832e2303b7874d044a7bc' - kernel_path_xz = self.fetch_asset(kernel_url, asset_hash=kernel_hash) - self.do_test_mips_malta32el_nanomips(kernel_path_xz) - - def test_mips_malta32el_nanomips_64k_dbg(self): - """ - :avocado: tags=arch:mipsel - :avocado: tags=machine:malta - :avocado: tags=endian:little - :avocado: tags=cpu:I7200 - """ - kernel_url = ('http://mipsdistros.mips.com/LinuxDistro/nanomips/' - 'kernels/v4.15.18-432-gb2eb9a8b07a1-20180627102142/' - 'generic_nano32r6el_page64k_dbg.xz') - kernel_hash = '18d1c68f2e23429e266ca39ba5349ccd0aeb7180' - kernel_path_xz = self.fetch_asset(kernel_url, asset_hash=kernel_hash) - self.do_test_mips_malta32el_nanomips(kernel_path_xz)
tests/functional/meson.build +1 -0
@@ -35,6 +35,7 @@ test_timeouts = { 'arm_sx1' : 360, 'intel_iommu': 300, 'mips_malta' : 120, + 'mipsel_malta' : 500, 'netdev_ethtool' : 180, 'ppc_40p' : 240, 'ppc64_hv' : 1000,
tests/functional/test_mipsel_malta.py +27 -3
@@ -9,12 +9,13 @@ # # SPDX-License-Identifier: GPL-2.0-or-later -from qemu_test import QemuSystemTest, LinuxKernelTest, Asset +from qemu_test import QemuSystemTest, Asset, wait_for_console_pattern from qemu_test import interrupt_interactive_console_until_pattern -from qemu_test import wait_for_console_pattern +from qemu_test import skipLongRuntime +from replay_kernel import ReplayKernelBase -class MaltaMachineConsole(LinuxKernelTest): +class MaltaMachineConsole(ReplayKernelBase): ASSET_KERNEL_4K = Asset( ('http://mipsdistros.mips.com/LinuxDistro/nanomips/'
@@ -57,6 +58,29 @@ def test_mips_malta32el_nanomips_16k_up(self): def test_mips_malta32el_nanomips_64k_dbg(self): self.do_test_mips_malta32el_nanomips(self.ASSET_KERNEL_64K) + def do_test_replay_mips_malta32el_nanomips(self, kernel_asset): + self.set_machine('malta') + self.cpu = 'I7200' + kernel_path = self.uncompress(kernel_asset) + + kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE + + 'mem=256m@@0x0 ' + 'console=ttyS0') + console_pattern = 'Kernel command line: %s' % kernel_command_line + self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5) + + @skipLongRuntime() + def test_replay_mips_malta32el_nanomips_4k(self): + self.do_test_replay_mips_malta32el_nanomips(self.ASSET_KERNEL_4K) + + @skipLongRuntime() + def test_replay_mips_malta32el_nanomips_16k_up(self): + self.do_test_replay_mips_malta32el_nanomips(self.ASSET_KERNEL_16K) + + @skipLongRuntime() + def test_replay_mips_malta32el_nanomips_64k_dbg(self): + self.do_test_replay_mips_malta32el_nanomips(self.ASSET_KERNEL_64K) + class MaltaMachineYAMON(QemuSystemTest):
Signature · 1 line
2.48.1
Re: [PATCH 3/5] tests/functional/test_mipsel_malta: Convert the mipsel replay tests
Quoted 4 lines · Thomas Huth
Move the mipsel replay tests from
tests/avocado/replay_kernel.pyto the functional framework. Since the functional tests should be run per target, we cannot stick all replay tests in one file. Thus let's add these tests to the file where we already use the same asset already.
Are the replay tests liable to impact running time much ?
The test timeouts are per-file, which could motivate a separate test_mipsel_malta_replay.py file ?
Quoted 17 lines
Signed-off-by Thomas Huth <[email protected]>--- tests/avocado/replay_kernel.py | 54 --------------------------- tests/functional/meson.build | 1 + tests/functional/test_mipsel_malta.py | 30 +++++++++++++-- 3 files changed, 28 insertions(+), 57 deletions(-)tests/functional/meson.build +1
-0@@ -35,6 +35,7 @@ test_timeouts = { 'arm_sx1' : 360, 'intel_iommu': 300, 'mips_malta' : 120, + 'mipsel_malta' : 500,
snip
tests/functional/meson.build
Quoted 12 lines
tests/functional/meson.build +12
-0+ + @skipLongRuntime() + def test_replay_mips_malta32el_nanomips_4k(self): + self.do_test_replay_mips_malta32el_nanomips(self.ASSET_KERNEL_4K) + + @skipLongRuntime() + def test_replay_mips_malta32el_nanomips_16k_up(self): + self.do_test_replay_mips_malta32el_nanomips(self.ASSET_KERNEL_16K) + + @skipLongRuntime() + def test_replay_mips_malta32el_nanomips_64k_dbg(self): + self.do_test_replay_mips_malta32el_nanomips(self.ASSET_KERNEL_64K)
Guess that answers my own question. I'd think a separate file for replay tests per target is nicer, so we leave the default executed malta tests with short timeout in meson.
With regards,
Daniel
Signature · 3 lines
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
Re: [PATCH 3/5] tests/functional/test_mipsel_malta: Convert the mipsel replay tests
Quoted 10 lines · Daniel P. Berrangé
Quoted 4 lines · Thomas Huth
Move the mipsel replay tests from
tests/avocado/replay_kernel.pyto the functional framework. Since the functional tests should be run per target, we cannot stick all replay tests in one file. Thus let's add these tests to the file where we already use the same asset already.Are the replay tests liable to impact running time much ?
The test timeouts are per-file, which could motivate a separate
test_mipsel_malta_replay.pyfile ?
The mipsel tests take almost 2 minutes on my fast laptop already, so yes, on CI and slow systems this will need a higher timeout setting.
Quoted 3 lines
Guess that answers my own question. I'd think a separate file for replay tests per target is nicer, so we leave the default executed malta tests with short timeout in meson.
Thinking about this twice, it's also nicer for the MAINTAINERS file if we have the replay related tests in separate files, so yes, let's go with separate files for the replay tests.
Thomas
PATCH 4/5 tests/functional/test_mips64el_malta: Convert the mips64el replay tests
In reply to Thomas HuthMove the mips64el replay tests from tests/avocado/replay_kernel.py to the functional framework. Since the functional tests should be run per target, we cannot stick all replay tests in one file. Thus let's add these tests to the file where we already use the same asset already.
--- tests/avocado/replay_kernel.py | 57 ------------------------- tests/functional/test_mips64el_malta.py | 26 ++++++++++- 2 files changed, 25 insertions(+), 58 deletions(-)
tests/avocado/replay_kernel.py +0 -57
@@ -162,33 +162,6 @@ def test_mips_malta(self): self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5) - def test_mips64el_malta(self): - """ - This test requires the ar tool to extract "data.tar.gz" from - the Debian package. - - The kernel can be rebuilt using this Debian kernel source [1] and - following the instructions on [2]. - - [1] http://snapshot.debian.org/package/linux-2.6/2.6.32-48/ - #linux-source-2.6.32_2.6.32-48 - [2] https://kernel-team.pages.debian.net/kernel-handbook/ - ch-common-tasks.html#s-common-official - - :avocado: tags=arch:mips64el - :avocado: tags=machine:malta - """ - deb_url = ('http://snapshot.debian.org/archive/debian/' - '20130217T032700Z/pool/main/l/linux-2.6/' - 'linux-image-2.6.32-5-5kc-malta_2.6.32-48_mipsel.deb') - deb_hash = '1aaec92083bf22fda31e0d27fa8d9a388e5fc3d5' - deb_path = self.fetch_asset(deb_url, asset_hash=deb_hash) - kernel_path = self.extract_from_deb(deb_path, - '/boot/vmlinux-2.6.32-5-5kc-malta') - kernel_command_line = self.KERNEL_COMMON_COMMAND_LINE + 'console=ttyS0' - console_pattern = 'Kernel command line: %s' % kernel_command_line - self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5) - def test_aarch64_virt(self): """ :avocado: tags=arch:aarch64
@@ -491,33 +464,3 @@ def test_mips_malta_cpio(self): console_pattern = 'Boot successful.' self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5, args=('-initrd', initrd_path)) - - @skipUnless(os.getenv('AVOCADO_ALLOW_UNTRUSTED_CODE'), 'untrusted code') - def test_mips64el_malta_5KEc_cpio(self): - """ - :avocado: tags=arch:mips64el - :avocado: tags=machine:malta - :avocado: tags=endian:little - :avocado: tags=slowness:high - :avocado: tags=cpu:5KEc - """ - kernel_url = ('https://github.com/philmd/qemu-testing-blob/' - 'raw/9ad2df38/mips/malta/mips64el/' - 'vmlinux-3.19.3.mtoman.20150408') - kernel_hash = '00d1d268fb9f7d8beda1de6bebcc46e884d71754' - kernel_path = self.fetch_asset(kernel_url, asset_hash=kernel_hash) - initrd_url = ('https://github.com/groeck/linux-build-test/' - 'raw/8584a59e/rootfs/' - 'mipsel64/rootfs.mipsel64r1.cpio.gz') - initrd_hash = '1dbb8a396e916847325284dbe2151167' - initrd_path_gz = self.fetch_asset(initrd_url, algorithm='md5', - asset_hash=initrd_hash) - initrd_path = self.workdir + "rootfs.cpio" - archive.gzip_uncompress(initrd_path_gz, initrd_path) - - kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE + - 'console=ttyS0 console=tty ' - 'rdinit=/sbin/init noreboot') - console_pattern = 'Boot successful.' - self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5, - args=('-initrd', initrd_path))
tests/functional/test_mips64el_malta.py +25 -1
@@ -15,9 +15,10 @@ from qemu_test import LinuxKernelTest, Asset from qemu_test import exec_command_and_wait_for_pattern from qemu_test import skipIfMissingImports, skipFlakyTest, skipUntrustedTest +from replay_kernel import ReplayKernelBase -class MaltaMachineConsole(LinuxKernelTest): +class MaltaMachineConsole(ReplayKernelBase): ASSET_KERNEL_2_63_2 = Asset( ('http://snapshot.debian.org/archive/debian/'
@@ -51,6 +52,15 @@ def test_mips64el_malta(self): console_pattern = 'Kernel command line: %s' % kernel_command_line self.wait_for_console_pattern(console_pattern) + def test_replay_mips64el_malta(self): + self.set_machine('malta') + kernel_path = self.archive_extract(self.ASSET_KERNEL_2_63_2, + member='boot/vmlinux-2.6.32-5-5kc-malta') + kernel_command_line = self.KERNEL_COMMON_COMMAND_LINE + 'console=ttyS0' + console_pattern = 'Kernel command line: %s' % kernel_command_line + self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5) + + ASSET_KERNEL_3_19_3 = Asset( ('https://github.com/philmd/qemu-testing-blob/' 'raw/9ad2df38/mips/malta/mips64el/'
@@ -90,6 +100,20 @@ def test_mips64el_malta_5KEc_cpio(self): # Wait for VM to shut down gracefully self.vm.wait() + @skipUntrustedTest() + def test_replay_mips64el_malta_5KEc_cpio(self): + self.set_machine('malta') + self.cpu = '5KEc' + kernel_path = self.ASSET_KERNEL_3_19_3.fetch() + initrd_path = self.uncompress(self.ASSET_CPIO_R1) + + kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE + + 'console=ttyS0 console=tty ' + 'rdinit=/sbin/init noreboot') + console_pattern = 'Boot successful.' + self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5, + args=('-initrd', initrd_path)) + @skipIfMissingImports('numpy', 'cv2') class MaltaMachineFramebuffer(LinuxKernelTest):
Signature · 1 line
2.48.1
PATCH 5/5 tests/functional/test_mips_malta: Convert the mips big endian replay tests
In reply to Thomas HuthMove the mips big endian replay tests from tests/avocado/replay_kernel.py to the functional framework. Since the functional tests should be run per target, we cannot stick all replay tests in one file. Thus let's add these tests to the file where we already use the same asset already.
--- tests/avocado/replay_kernel.py | 54 ----------------------------- tests/functional/test_mips_malta.py | 29 ++++++++++++++-- 2 files changed, 26 insertions(+), 57 deletions(-)
tests/avocado/replay_kernel.py +0 -54
@@ -144,24 +144,6 @@ def test_x86_64_q35(self): self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5) - def test_mips_malta(self): - """ - :avocado: tags=arch:mips - :avocado: tags=machine:malta - :avocado: tags=endian:big - """ - deb_url = ('http://snapshot.debian.org/archive/debian/' - '20130217T032700Z/pool/main/l/linux-2.6/' - 'linux-image-2.6.32-5-4kc-malta_2.6.32-48_mips.deb') - deb_hash = 'a8cfc28ad8f45f54811fc6cf74fc43ffcfe0ba04' - deb_path = self.fetch_asset(deb_url, asset_hash=deb_hash) - kernel_path = self.extract_from_deb(deb_path, - '/boot/vmlinux-2.6.32-5-4kc-malta') - kernel_command_line = self.KERNEL_COMMON_COMMAND_LINE + 'console=ttyS0' - console_pattern = 'Kernel command line: %s' % kernel_command_line - - self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5) - def test_aarch64_virt(self): """ :avocado: tags=arch:aarch64
@@ -428,39 +410,3 @@ def test_xtensa_lx60(self): '/qac-best-of-multiarch/download/day02.tar.xz') file_path = self.fetch_asset(tar_url, asset_hash=tar_hash) self.do_test_advcal_2018(file_path, 'santas-sleigh-ride.elf') - -@skipUnless(os.getenv('AVOCADO_TIMEOUT_EXPECTED'), 'Test might timeout') -class ReplayKernelSlow(ReplayKernelBase): - # Override the timeout, because this kernel includes an inner - # loop which is executed with TB recompilings during replay, - # making it very slow. - timeout = 180 - - def test_mips_malta_cpio(self): - """ - :avocado: tags=arch:mips - :avocado: tags=machine:malta - :avocado: tags=endian:big - :avocado: tags=slowness:high - """ - deb_url = ('http://snapshot.debian.org/archive/debian/' - '20160601T041800Z/pool/main/l/linux/' - 'linux-image-4.5.0-2-4kc-malta_4.5.5-1_mips.deb') - deb_hash = 'a3c84f3e88b54e06107d65a410d1d1e8e0f340f8' - deb_path = self.fetch_asset(deb_url, asset_hash=deb_hash) - kernel_path = self.extract_from_deb(deb_path, - '/boot/vmlinux-4.5.0-2-4kc-malta') - initrd_url = ('https://github.com/groeck/linux-build-test/raw/' - '8584a59ed9e5eb5ee7ca91f6d74bbb06619205b8/rootfs/' - 'mips/rootfs.cpio.gz') - initrd_hash = 'bf806e17009360a866bf537f6de66590de349a99' - initrd_path_gz = self.fetch_asset(initrd_url, asset_hash=initrd_hash) - initrd_path = self.workdir + "rootfs.cpio" - archive.gzip_uncompress(initrd_path_gz, initrd_path) - - kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE + - 'console=ttyS0 console=tty ' - 'rdinit=/sbin/init noreboot') - console_pattern = 'Boot successful.' - self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5, - args=('-initrd', initrd_path))
tests/functional/test_mips_malta.py +26 -3
@@ -6,11 +6,12 @@ # # SPDX-License-Identifier: GPL-2.0-or-later -from qemu_test import LinuxKernelTest, Asset +from qemu_test import Asset, skipLongRuntime from qemu_test import exec_command_and_wait_for_pattern +from replay_kernel import ReplayKernelBase -class MaltaMachineConsole(LinuxKernelTest): +class MaltaMachineConsole(ReplayKernelBase): ASSET_KERNEL_2_63_2 = Asset( ('http://snapshot.debian.org/archive/debian/'
@@ -32,6 +33,14 @@ def test_mips_malta(self): console_pattern = 'Kernel command line: %s' % kernel_command_line self.wait_for_console_pattern(console_pattern) + def test_replay_mips_malta(self): + self.set_machine('malta') + kernel_path = self.archive_extract(self.ASSET_KERNEL_2_63_2, + member='boot/vmlinux-2.6.32-5-4kc-malta') + kernel_command_line = self.KERNEL_COMMON_COMMAND_LINE + 'console=ttyS0' + console_pattern = 'Kernel command line: %s' % kernel_command_line + self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5) + ASSET_KERNEL_4_5_0 = Asset( ('http://snapshot.debian.org/archive/debian/' '20160601T041800Z/pool/main/l/linux/'
@@ -71,6 +80,20 @@ def test_mips_malta_cpio(self): # Wait for VM to shut down gracefully self.vm.wait() + @skipLongRuntime() + def test_replay_mips_malta_cpio(self): + self.set_machine('malta') + kernel_path = self.archive_extract(self.ASSET_KERNEL_4_5_0, + member='boot/vmlinux-4.5.0-2-4kc-malta') + initrd_path = self.uncompress(self.ASSET_INITRD) + + kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE + + 'console=ttyS0 console=tty ' + 'rdinit=/sbin/init noreboot') + console_pattern = 'Boot successful.' + self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5, + args=('-initrd', initrd_path)) + if __name__ == '__main__': - LinuxKernelTest.main() + ReplayKernelBase.main()
Signature · 1 line
2.48.1