Skip to contentSkip to conversation

Convert the MIPS replay tests to the functional framework

PATCH Superseded Thomas Huth 3 participants 12 messages Jan 24, 2025 – Jan 28, 2025 Arm/Amlogic Meson SoC support

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

cover Convert the MIPS replay tests to the functional framework

tests/avocado/replay_kernel.py is a rather big file with a lot of Avocado-based tests in it. But in the long run, we rather want to get away from Avocado, so we eventually have to convert these tests to the functional framework. For this, we have to separate the tests by target architecture, since the functional framework cannot detect this on the fly like Avocado (which e.g. looks at the tag annotations in the comments). Let's start the convertion by handling the MIPS-related tests first (the other replay_kernel tests will follow later).

Thomas Huth (5):
tests/functional: Add a decorator for skipping long running tests tests/functional: Add the ReplayKernelBase class tests/functional/test_mipsel_malta: Convert the mipsel replay tests tests/functional/test_mips64el_malta: Convert the mips64el replay
tests
tests/functional/test_mips_malta: Convert the mips big endian replay
tests

 docs/devel/testing/functional.rst        |   8 ++
 tests/avocado/replay_kernel.py           | 165 -----------------------
 tests/functional/meson.build             |   1 +
 tests/functional/qemu_test/__init__.py   |   2 +-
 tests/functional/qemu_test/decorators.py |  14 ++
 tests/functional/replay_kernel.py        |  84 ++++++++++++
 tests/functional/test_arm_quanta_gsj.py  |   5 +-
 tests/functional/test_mips64el_malta.py  |  26 +++-
 tests/functional/test_mips_malta.py      |  29 +++-
 tests/functional/test_mipsel_malta.py    |  30 ++++-
 10 files changed, 189 insertions(+), 175 deletions(-)
 create mode 100644 tests/functional/replay_kernel.py
Signature · 1 line
2.48.1
#1 lore raw

PATCH 1/5 tests/functional: Add a decorator for skipping long running tests

Some tests have a very long runtime and might run into timeout issues e.g. when QEMU has been compiled with --enable-debug. Add a decorator for marking them more easily and document the corresponding environment variable that is used to enable the tests.

---
 docs/devel/testing/functional.rst        |  8 ++++++++
 tests/functional/qemu_test/__init__.py   |  2 +-
 tests/functional/qemu_test/decorators.py | 14 ++++++++++++++
 tests/functional/test_arm_quanta_gsj.py  |  5 +++--
 4 files changed, 26 insertions(+), 3 deletions(-)
docs/devel/testing/functional.rst +8 -0
index ae238ed3fc..7d9396b696 100644
@@ -351,5 +351,13 @@ the code snippet below:
 Tests should not live in this state forever and should either be fixed
 or eventually removed.
 
+QEMU_TEST_TIMEOUT_EXPECTED
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+Tests that have a very long runtime and might run into timeout issues
+e.g. if the QEMU binary has been compiled with debugging options enabled.
+To avoid these timeout issues by default and to save some precious CPU
+cycles during normal testing, such tests are disabled by default unless
+the QEMU_TEST_TIMEOUT_EXPECTED environment variable has been set.
+
 
 .. _unittest: https://docs.python.org/3/library/unittest.html
tests/functional/qemu_test/__init__.py +1 -1
index da1830286d..b1a19d2a4b 100644
@@ -14,7 +14,7 @@
 from .testcase import QemuBaseTest, QemuUserTest, QemuSystemTest
 from .linuxkernel import LinuxKernelTest
 from .decorators import skipIfMissingCommands, skipIfNotMachine, \
-    skipFlakyTest, skipUntrustedTest, skipBigDataTest, \
+    skipFlakyTest, skipUntrustedTest, skipBigDataTest, skipLongRuntime, \
     skipIfMissingImports
 from .archive import archive_extract
 from .uncompress import uncompress
tests/functional/qemu_test/decorators.py +14 -0
index df088bc090..8f311e5309 100644
@@ -86,6 +86,20 @@ def skipBigDataTest():
     return skipUnless(os.getenv('QEMU_TEST_ALLOW_LARGE_STORAGE'),
                       'Test requires large host storage space')
 
+'''
+Decorator to skip execution of tests which have a really long
+runtime (and might e.g. time out if QEMU has been compiled with
+debugging enabled) unless the $QEMU_TEST_TIMEOUT_EXPECTED
+environment variable is set
+
+Example:
+
+  @skipLongRuntime()
+'''
+def skipLongRuntime():
+    return skipUnless(os.getenv('QEMU_TEST_TIMEOUT_EXPECTED'),
+                      'Test has a very long runtime and might time out')
+
 '''
 Decorator to skip execution of a test if the list
 of python imports is not available.
tests/functional/test_arm_quanta_gsj.py +3 -2
index 7b82e2185c..fe1d60d649 100755
@@ -8,7 +8,8 @@
 
 from qemu_test import LinuxKernelTest, Asset, exec_command_and_wait_for_pattern
 from qemu_test import interrupt_interactive_console_until_pattern
-from unittest import skipUnless
+from qemu_test import skipLongRuntime
+
 
 class EmcraftSf2Machine(LinuxKernelTest):
 
@@ -32,7 +33,7 @@ class EmcraftSf2Machine(LinuxKernelTest):
          '20200711-gsj-qemu-0/nuvoton-npcm730-gsj.dtb'),
         '3249b2da787d4b9ad4e61f315b160abfceb87b5e1895a7ce898ce7f40c8d4045')
 
-    @skipUnless(os.getenv('QEMU_TEST_TIMEOUT_EXPECTED'), 'Test might timeout')
+    @skipLongRuntime()
     def test_arm_quanta_gsj(self):
         self.set_machine('quanta-gsj')
         image_path = self.uncompress(self.ASSET_IMAGE, format='gz')
Signature · 1 line
2.48.1
Reviewed-by on v1 1/5
#2 lore raw

Re: [PATCH 1/5] tests/functional: Add a decorator for skipping long running tests

Quoted 6 lines · Thomas Huth

Some tests have a very long runtime and might run into timeout issues e.g. when QEMU has been compiled with --enable-debug. Add a decorator for marking them more easily and document the corresponding environment variable that is used to enable the tests.

Quoted 7 lines
---
  docs/devel/testing/functional.rst        |  8 ++++++++
  tests/functional/qemu_test/__init__.py   |  2 +-
  tests/functional/qemu_test/decorators.py | 14 ++++++++++++++
  tests/functional/test_arm_quanta_gsj.py  |  5 +++--
  4 files changed, 26 insertions(+), 3 deletions(-)

Nice!

#3 lore raw

Re: [PATCH 1/5] tests/functional: Add a decorator for skipping long running tests

In reply to Thomas Huth
Quoted 37 lines · Thomas Huth

Some tests have a very long runtime and might run into timeout issues e.g. when QEMU has been compiled with --enable-debug. Add a decorator for marking them more easily and document the corresponding environment variable that is used to enable the tests.

---
 docs/devel/testing/functional.rst        |  8 ++++++++
 tests/functional/qemu_test/__init__.py   |  2 +-
 tests/functional/qemu_test/decorators.py | 14 ++++++++++++++
 tests/functional/test_arm_quanta_gsj.py  |  5 +++--
 4 files changed, 26 insertions(+), 3 deletions(-)
docs/devel/testing/functional.rst +8 -0
index ae238ed3fc..7d9396b696 100644
@@ -351,5 +351,13 @@ the code snippet below:
 Tests should not live in this state forever and should either be fixed
 or eventually removed.
 
+QEMU_TEST_TIMEOUT_EXPECTED
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+Tests that have a very long runtime and might run into timeout issues
+e.g. if the QEMU binary has been compiled with debugging options enabled.
+To avoid these timeout issues by default and to save some precious CPU
+cycles during normal testing, such tests are disabled by default unless
+the QEMU_TEST_TIMEOUT_EXPECTED environment variable has been set.
+
 
 .. _unittest: https://docs.python.org/3/library/unittest.html
tests/functional/qemu_test/__init__.py +0 -0
index da1830286d..b1a19d2a4b 100644

tests/functional/qemu_test/__init__.py line 17

Quoted 5 lines
tests/functional/qemu_test/__init__.py +1 -1
@@ -14,7 +14,7 @@
 from .testcase import QemuBaseTest, QemuUserTest, QemuSystemTest
 from .linuxkernel import LinuxKernelTest
 from .decorators import skipIfMissingCommands, skipIfNotMachine, \
-    skipFlakyTest, skipUntrustedTest, skipBigDataTest, \
+    skipFlakyTest, skipUntrustedTest, skipBigDataTest, skipLongRuntime, \

s/Runtime/RunningTime/, but actually in terms of naming convention, 'skipSlowTest' would fit better.

Quoted 12 lines
tests/functional/qemu_test/decorators.py +4 -0
index df088bc090..8f311e5309 100644
@@ -86,6 +86,20 @@ def skipBigDataTest():
     return skipUnless(os.getenv('QEMU_TEST_ALLOW_LARGE_STORAGE'),
                       'Test requires large host storage space')
 
+'''
+Decorator to skip execution of tests which have a really long
+runtime (and might e.g. time out if QEMU has been compiled with
+debugging enabled) unless the $QEMU_TEST_TIMEOUT_EXPECTED

tests/functional/qemu_test/decorators.py def skipBigDataTest(): · line 102

Quoted 10 lines
tests/functional/qemu_test/decorators.py +10 -0
+environment variable is set
+
+Example:
+
+  @skipLongRuntime()
+'''
+def skipLongRuntime():
+    return skipUnless(os.getenv('QEMU_TEST_TIMEOUT_EXPECTED'),
+                      'Test has a very long runtime and might time out')
+

You're preserving the existnig env var which is good, but I have a little niggling desire to unify the naming conventions:

skipFlakyTest -> $QEMU_TEST_ALLOW_FLAKY
skipUntrustedTest -> $QEMU_TEST_ALLOW_UNTRUSTED
skipBigDataTest -> $QEMU_TEST_ALLOW_BIG_DATA
skipSlowTest -> $QEMU_TEST_ALLOW_SLOW

Could be a separate patch though if you like the idea.

Quoted 29 lines
tests/functional/qemu_test/decorators.py +0 -0
 '''
 Decorator to skip execution of a test if the list
 of python imports is not available.
tests/functional/test_arm_quanta_gsj.py +3 -2
index 7b82e2185c..fe1d60d649 100755
@@ -8,7 +8,8 @@
 
 from qemu_test import LinuxKernelTest, Asset, exec_command_and_wait_for_pattern
 from qemu_test import interrupt_interactive_console_until_pattern
-from unittest import skipUnless
+from qemu_test import skipLongRuntime
+
 
 class EmcraftSf2Machine(LinuxKernelTest):
 
@@ -32,7 +33,7 @@ class EmcraftSf2Machine(LinuxKernelTest):
          '20200711-gsj-qemu-0/nuvoton-npcm730-gsj.dtb'),
         '3249b2da787d4b9ad4e61f315b160abfceb87b5e1895a7ce898ce7f40c8d4045')
 
-    @skipUnless(os.getenv('QEMU_TEST_TIMEOUT_EXPECTED'), 'Test might timeout')
+    @skipLongRuntime()
     def test_arm_quanta_gsj(self):
         self.set_machine('quanta-gsj')
         image_path = self.uncompress(self.ASSET_IMAGE, format='gz')
Signature · 1 line
2.48.1

With regards,
Daniel

Signature · 3 lines
|: https://berrange.com      -o-    https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org         -o-            https://fstop138.berrange.com :|
|: https://entangle-photo.org    -o-    https://www.instagram.com/dberrange :|
#4 lore raw

Re: [PATCH 1/5] tests/functional: Add a decorator for skipping long running tests

Quoted 32 lines · Daniel P. Berrangé
Quoted 31 lines · Thomas Huth

Some tests have a very long runtime and might run into timeout issues e.g. when QEMU has been compiled with --enable-debug. Add a decorator for marking them more easily and document the corresponding environment variable that is used to enable the tests.

---
  docs/devel/testing/functional.rst        |  8 ++++++++
  tests/functional/qemu_test/__init__.py   |  2 +-
  tests/functional/qemu_test/decorators.py | 14 ++++++++++++++
  tests/functional/test_arm_quanta_gsj.py  |  5 +++--
  4 files changed, 26 insertions(+), 3 deletions(-)
docs/devel/testing/functional.rst +8 -0
index ae238ed3fc..7d9396b696 100644
@@ -351,5 +351,13 @@ the code snippet below:
  Tests should not live in this state forever and should either be fixed
  or eventually removed.
 
+QEMU_TEST_TIMEOUT_EXPECTED
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+Tests that have a very long runtime and might run into timeout issues
+e.g. if the QEMU binary has been compiled with debugging options enabled.
+To avoid these timeout issues by default and to save some precious CPU
+cycles during normal testing, such tests are disabled by default unless
+the QEMU_TEST_TIMEOUT_EXPECTED environment variable has been set.
+
 
Quoted 14 lines

tests/functional/qemu_test/__init__.py line 17

Quoted 11 lines
docs/devel/testing/functional.rst +0 -0
  .. _unittest: https://docs.python.org/3/library/unittest.html
tests/functional/qemu_test/__init__.py +1 -1
index da1830286d..b1a19d2a4b 100644
@@ -14,7 +14,7 @@
  from .testcase import QemuBaseTest, QemuUserTest, QemuSystemTest
  from .linuxkernel import LinuxKernelTest
  from .decorators import skipIfMissingCommands, skipIfNotMachine, \
-    skipFlakyTest, skipUntrustedTest, skipBigDataTest, \
+    skipFlakyTest, skipUntrustedTest, skipBigDataTest, skipLongRuntime, \

s/Runtime/RunningTime/, but actually in terms of naming convention, 'skipSlowTest' would fit better.

Ack, that sounds better, indeed.

Quoted 28 lines
Quoted 13 lines
tests/functional/qemu_test/decorators.py +4 -0
index df088bc090..8f311e5309 100644
@@ -86,6 +86,20 @@ def skipBigDataTest():
      return skipUnless(os.getenv('QEMU_TEST_ALLOW_LARGE_STORAGE'),
                        'Test requires large host storage space')
 
+'''
+Decorator to skip execution of tests which have a really long
+runtime (and might e.g. time out if QEMU has been compiled with
+debugging enabled) unless the $QEMU_TEST_TIMEOUT_EXPECTED

tests/functional/qemu_test/decorators.py def skipBigDataTest(): · line 102

Quoted 10 lines
tests/functional/qemu_test/decorators.py +10 -0
+environment variable is set
+
+Example:
+
+  @skipLongRuntime()
+'''
+def skipLongRuntime():
+    return skipUnless(os.getenv('QEMU_TEST_TIMEOUT_EXPECTED'),
+                      'Test has a very long runtime and might time out')
+

You're preserving the existnig env var which is good, but I have a little niggling desire to unify the naming conventions:

Quoted 6 lines
   skipFlakyTest     -> $QEMU_TEST_ALLOW_FLAKY
   skipUntrustedTest -> $QEMU_TEST_ALLOW_UNTRUSTED
   skipBigDataTest   -> $QEMU_TEST_ALLOW_BIG_DATA
   skipSlowTest      -> $QEMU_TEST_ALLOW_SLOW

Could be a separate patch though if you like the idea.

I like the idea, some of the others are already starting with QEMU_TEST_ALLOW_, so the renaming makes sense to me, too. I'll change this patch in v2, but I'll leave the others for now - feel free to send a patch for them, if not I'll tackle them sometime later.

Thomas

#5 lore raw

PATCH 2/5 tests/functional: Add the ReplayKernelBase class

In reply to Thomas Huth

Copy the ReplayKernelBase class from the avocado tests. We are going to need it to convert the related replay tests in the following patches.

---
 tests/functional/replay_kernel.py | 84 +++++++++++++++++++++++++++++++
 1 file changed, 84 insertions(+)
 create mode 100644 tests/functional/replay_kernel.py
tests/functional/replay_kernel.py new file +84 -0
new file mode 100644
index 0000000000..8e8ac7d052
@@ -0,0 +1,84 @@
+# Record/replay test that boots a Linux kernel
+#
+# Copyright (c) 2020 ISP RAS
+#
+# Author:
+#  Pavel Dovgalyuk <[email protected]>
+#
+# This work is licensed under the terms of the GNU GPL, version 2 or
+# later.  See the COPYING file in the top-level directory.
+
+import os
+import logging
+import time
+import subprocess
+
+from qemu_test.linuxkernel import LinuxKernelTest
+
+class ReplayKernelBase(LinuxKernelTest):
+    """
+    Boots a Linux kernel in record mode and checks that the console
+    is operational and the kernel command line is properly passed
+    from QEMU to the kernel.
+    Then replays the same scenario and verifies, that QEMU correctly
+    terminates.
+    """
+
+    timeout = 180
+    REPLAY_KERNEL_COMMAND_LINE = 'printk.time=1 panic=-1 '
+
+    def run_vm(self, kernel_path, kernel_command_line, console_pattern,
+               record, shift, args, replay_path):
+        # icount requires TCG to be available
+        self.require_accelerator('tcg')
+
+        logger = logging.getLogger('replay')
+        start_time = time.time()
+        vm = self.get_vm()
+        vm.set_console()
+        if record:
+            logger.info('recording the execution...')
+            mode = 'record'
+        else:
+            logger.info('replaying the execution...')
+            mode = 'replay'
+        vm.add_args('-icount', 'shift=%s,rr=%s,rrfile=%s' %
+                    (shift, mode, replay_path),
+                    '-kernel', kernel_path,
+                    '-append', kernel_command_line,
+                    '-net', 'none',
+                    '-no-reboot')
+        if args:
+            vm.add_args(*args)
+        vm.launch()
+        self.wait_for_console_pattern(console_pattern, vm)
+        if record:
+            vm.shutdown()
+            logger.info('finished the recording with log size %s bytes'
+                        % os.path.getsize(replay_path))
+            self.run_replay_dump(replay_path)
+            logger.info('successfully tested replay-dump.py')
+        else:
+            vm.wait()
+            logger.info('successfully finished the replay')
+        elapsed = time.time() - start_time
+        logger.info('elapsed time %.2f sec' % elapsed)
+        return elapsed
+
+    def run_replay_dump(self, replay_path):
+        try:
+            subprocess.check_call(["./scripts/replay-dump.py",
+                                   "-f", replay_path],
+                                  stdout=subprocess.DEVNULL)
+        except subprocess.CalledProcessError:
+            self.fail('replay-dump.py failed')
+
+    def run_rr(self, kernel_path, kernel_command_line, console_pattern,
+               shift=7, args=None):
+        replay_path = os.path.join(self.workdir, 'replay.bin')
+        t1 = self.run_vm(kernel_path, kernel_command_line, console_pattern,
+                         True, shift, args, replay_path)
+        t2 = self.run_vm(kernel_path, kernel_command_line, console_pattern,
+                         False, shift, args, replay_path)
+        logger = logging.getLogger('replay')
+        logger.info('replay overhead {:.2%}'.format(t2 / t1 - 1))
Signature · 1 line
2.48.1
Reviewed-by on v1 2/5
#6 lore raw

Re: [PATCH 2/5] tests/functional: Add the ReplayKernelBase class

Quoted 8 lines · Thomas Huth

Copy the ReplayKernelBase class from the avocado tests. We are going to need it to convert the related replay tests in the following patches.

---
 tests/functional/replay_kernel.py | 84 +++++++++++++++++++++++++++++++
 1 file changed, 84 insertions(+)
 create mode 100644 tests/functional/replay_kernel.py

With regards,
Daniel

Signature · 3 lines
|: https://berrange.com      -o-    https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org         -o-            https://fstop138.berrange.com :|
|: https://entangle-photo.org    -o-    https://www.instagram.com/dberrange :|
#7 lore raw

PATCH 3/5 tests/functional/test_mipsel_malta: Convert the mipsel replay tests

In reply to Thomas Huth

Move the mipsel replay tests from tests/avocado/replay_kernel.py to the functional framework. Since the functional tests should be run per target, we cannot stick all replay tests in one file. Thus let's add these tests to the file where we already use the same asset already.

---
 tests/avocado/replay_kernel.py        | 54 ---------------------------
 tests/functional/meson.build          |  1 +
 tests/functional/test_mipsel_malta.py | 30 +++++++++++++--
 3 files changed, 28 insertions(+), 57 deletions(-)
tests/avocado/replay_kernel.py +0 -54
index e22c200a36..4f50f48163 100644
@@ -521,57 +521,3 @@ def test_mips64el_malta_5KEc_cpio(self):
         console_pattern = 'Boot successful.'
         self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5,
                     args=('-initrd', initrd_path))
-
-    def do_test_mips_malta32el_nanomips(self, kernel_path_xz):
-        kernel_path = self.workdir + "kernel"
-        with lzma.open(kernel_path_xz, 'rb') as f_in:
-            with open(kernel_path, 'wb') as f_out:
-                shutil.copyfileobj(f_in, f_out)
-
-        kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE +
-                               'mem=256m@@0x0 '
-                               'console=ttyS0')
-        console_pattern = 'Kernel command line: %s' % kernel_command_line
-        self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5)
-
-    def test_mips_malta32el_nanomips_4k(self):
-        """
-        :avocado: tags=arch:mipsel
-        :avocado: tags=machine:malta
-        :avocado: tags=endian:little
-        :avocado: tags=cpu:I7200
-        """
-        kernel_url = ('http://mipsdistros.mips.com/LinuxDistro/nanomips/'
-                      'kernels/v4.15.18-432-gb2eb9a8b07a1-20180627102142/'
-                      'generic_nano32r6el_page4k.xz')
-        kernel_hash = '477456aafd2a0f1ddc9482727f20fe9575565dd6'
-        kernel_path_xz = self.fetch_asset(kernel_url, asset_hash=kernel_hash)
-        self.do_test_mips_malta32el_nanomips(kernel_path_xz)
-
-    def test_mips_malta32el_nanomips_16k_up(self):
-        """
-        :avocado: tags=arch:mipsel
-        :avocado: tags=machine:malta
-        :avocado: tags=endian:little
-        :avocado: tags=cpu:I7200
-        """
-        kernel_url = ('http://mipsdistros.mips.com/LinuxDistro/nanomips/'
-                      'kernels/v4.15.18-432-gb2eb9a8b07a1-20180627102142/'
-                      'generic_nano32r6el_page16k_up.xz')
-        kernel_hash = 'e882868f944c71c816e832e2303b7874d044a7bc'
-        kernel_path_xz = self.fetch_asset(kernel_url, asset_hash=kernel_hash)
-        self.do_test_mips_malta32el_nanomips(kernel_path_xz)
-
-    def test_mips_malta32el_nanomips_64k_dbg(self):
-        """
-        :avocado: tags=arch:mipsel
-        :avocado: tags=machine:malta
-        :avocado: tags=endian:little
-        :avocado: tags=cpu:I7200
-        """
-        kernel_url = ('http://mipsdistros.mips.com/LinuxDistro/nanomips/'
-                      'kernels/v4.15.18-432-gb2eb9a8b07a1-20180627102142/'
-                      'generic_nano32r6el_page64k_dbg.xz')
-        kernel_hash = '18d1c68f2e23429e266ca39ba5349ccd0aeb7180'
-        kernel_path_xz = self.fetch_asset(kernel_url, asset_hash=kernel_hash)
-        self.do_test_mips_malta32el_nanomips(kernel_path_xz)
tests/functional/meson.build +1 -0
index b7719ab85f..7d233213c1 100644
@@ -35,6 +35,7 @@ test_timeouts = {
   'arm_sx1' : 360,
   'intel_iommu': 300,
   'mips_malta' : 120,
+  'mipsel_malta' : 500,
   'netdev_ethtool' : 180,
   'ppc_40p' : 240,
   'ppc64_hv' : 1000,
tests/functional/test_mipsel_malta.py +27 -3
index fe9c3a172e..fff42c7f4f 100755
@@ -9,12 +9,13 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-from qemu_test import QemuSystemTest, LinuxKernelTest, Asset
+from qemu_test import QemuSystemTest, Asset, wait_for_console_pattern
 from qemu_test import interrupt_interactive_console_until_pattern
-from qemu_test import wait_for_console_pattern
+from qemu_test import skipLongRuntime
+from replay_kernel import ReplayKernelBase
 
 
-class MaltaMachineConsole(LinuxKernelTest):
+class MaltaMachineConsole(ReplayKernelBase):
 
     ASSET_KERNEL_4K = Asset(
         ('http://mipsdistros.mips.com/LinuxDistro/nanomips/'
@@ -57,6 +58,29 @@ def test_mips_malta32el_nanomips_16k_up(self):
     def test_mips_malta32el_nanomips_64k_dbg(self):
         self.do_test_mips_malta32el_nanomips(self.ASSET_KERNEL_64K)
 
+    def do_test_replay_mips_malta32el_nanomips(self, kernel_asset):
+        self.set_machine('malta')
+        self.cpu = 'I7200'
+        kernel_path = self.uncompress(kernel_asset)
+
+        kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE +
+                               'mem=256m@@0x0 '
+                               'console=ttyS0')
+        console_pattern = 'Kernel command line: %s' % kernel_command_line
+        self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5)
+
+    @skipLongRuntime()
+    def test_replay_mips_malta32el_nanomips_4k(self):
+        self.do_test_replay_mips_malta32el_nanomips(self.ASSET_KERNEL_4K)
+
+    @skipLongRuntime()
+    def test_replay_mips_malta32el_nanomips_16k_up(self):
+        self.do_test_replay_mips_malta32el_nanomips(self.ASSET_KERNEL_16K)
+
+    @skipLongRuntime()
+    def test_replay_mips_malta32el_nanomips_64k_dbg(self):
+        self.do_test_replay_mips_malta32el_nanomips(self.ASSET_KERNEL_64K)
+
 
 class MaltaMachineYAMON(QemuSystemTest):
 
Signature · 1 line
2.48.1
#8 lore raw

Re: [PATCH 3/5] tests/functional/test_mipsel_malta: Convert the mipsel replay tests

Quoted 4 lines · Thomas Huth

Move the mipsel replay tests from tests/avocado/replay_kernel.py to the functional framework. Since the functional tests should be run per target, we cannot stick all replay tests in one file. Thus let's add these tests to the file where we already use the same asset already.

Are the replay tests liable to impact running time much ?

The test timeouts are per-file, which could motivate a separate test_mipsel_malta_replay.py file ?

Quoted 17 lines
---
 tests/avocado/replay_kernel.py        | 54 ---------------------------
 tests/functional/meson.build          |  1 +
 tests/functional/test_mipsel_malta.py | 30 +++++++++++++--
 3 files changed, 28 insertions(+), 57 deletions(-)
tests/functional/meson.build +1 -0
index b7719ab85f..7d233213c1 100644
@@ -35,6 +35,7 @@ test_timeouts = {
   'arm_sx1' : 360,
   'intel_iommu': 300,
   'mips_malta' : 120,
+  'mipsel_malta' : 500,

snip

tests/functional/meson.build

Quoted 12 lines
tests/functional/meson.build +12 -0
+
+    @skipLongRuntime()
+    def test_replay_mips_malta32el_nanomips_4k(self):
+        self.do_test_replay_mips_malta32el_nanomips(self.ASSET_KERNEL_4K)
+
+    @skipLongRuntime()
+    def test_replay_mips_malta32el_nanomips_16k_up(self):
+        self.do_test_replay_mips_malta32el_nanomips(self.ASSET_KERNEL_16K)
+
+    @skipLongRuntime()
+    def test_replay_mips_malta32el_nanomips_64k_dbg(self):
+        self.do_test_replay_mips_malta32el_nanomips(self.ASSET_KERNEL_64K)

Guess that answers my own question. I'd think a separate file for replay tests per target is nicer, so we leave the default executed malta tests with short timeout in meson.

With regards,
Daniel

Signature · 3 lines
|: https://berrange.com      -o-    https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org         -o-            https://fstop138.berrange.com :|
|: https://entangle-photo.org    -o-    https://www.instagram.com/dberrange :|
#9 lore raw

Re: [PATCH 3/5] tests/functional/test_mipsel_malta: Convert the mipsel replay tests

Quoted 10 lines · Daniel P. Berrangé
Quoted 4 lines · Thomas Huth

Move the mipsel replay tests from tests/avocado/replay_kernel.py to the functional framework. Since the functional tests should be run per target, we cannot stick all replay tests in one file. Thus let's add these tests to the file where we already use the same asset already.

Are the replay tests liable to impact running time much ?

The test timeouts are per-file, which could motivate a separate test_mipsel_malta_replay.py file ?

The mipsel tests take almost 2 minutes on my fast laptop already, so yes, on CI and slow systems this will need a higher timeout setting.

Quoted 3 lines

Guess that answers my own question. I'd think a separate file for replay tests per target is nicer, so we leave the default executed malta tests with short timeout in meson.

Thinking about this twice, it's also nicer for the MAINTAINERS file if we have the replay related tests in separate files, so yes, let's go with separate files for the replay tests.

Thomas

#10 lore raw

PATCH 4/5 tests/functional/test_mips64el_malta: Convert the mips64el replay tests

In reply to Thomas Huth

Move the mips64el replay tests from tests/avocado/replay_kernel.py to the functional framework. Since the functional tests should be run per target, we cannot stick all replay tests in one file. Thus let's add these tests to the file where we already use the same asset already.

---
 tests/avocado/replay_kernel.py          | 57 -------------------------
 tests/functional/test_mips64el_malta.py | 26 ++++++++++-
 2 files changed, 25 insertions(+), 58 deletions(-)
tests/avocado/replay_kernel.py +0 -57
index 4f50f48163..a45881b9a6 100644
@@ -162,33 +162,6 @@ def test_mips_malta(self):
 
         self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5)
 
-    def test_mips64el_malta(self):
-        """
-        This test requires the ar tool to extract "data.tar.gz" from
-        the Debian package.
-
-        The kernel can be rebuilt using this Debian kernel source [1] and
-        following the instructions on [2].
-
-        [1] http://snapshot.debian.org/package/linux-2.6/2.6.32-48/
-            #linux-source-2.6.32_2.6.32-48
-        [2] https://kernel-team.pages.debian.net/kernel-handbook/
-            ch-common-tasks.html#s-common-official
-
-        :avocado: tags=arch:mips64el
-        :avocado: tags=machine:malta
-        """
-        deb_url = ('http://snapshot.debian.org/archive/debian/'
-                   '20130217T032700Z/pool/main/l/linux-2.6/'
-                   'linux-image-2.6.32-5-5kc-malta_2.6.32-48_mipsel.deb')
-        deb_hash = '1aaec92083bf22fda31e0d27fa8d9a388e5fc3d5'
-        deb_path = self.fetch_asset(deb_url, asset_hash=deb_hash)
-        kernel_path = self.extract_from_deb(deb_path,
-                                            '/boot/vmlinux-2.6.32-5-5kc-malta')
-        kernel_command_line = self.KERNEL_COMMON_COMMAND_LINE + 'console=ttyS0'
-        console_pattern = 'Kernel command line: %s' % kernel_command_line
-        self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5)
-
     def test_aarch64_virt(self):
         """
         :avocado: tags=arch:aarch64
@@ -491,33 +464,3 @@ def test_mips_malta_cpio(self):
         console_pattern = 'Boot successful.'
         self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5,
                     args=('-initrd', initrd_path))
-
-    @skipUnless(os.getenv('AVOCADO_ALLOW_UNTRUSTED_CODE'), 'untrusted code')
-    def test_mips64el_malta_5KEc_cpio(self):
-        """
-        :avocado: tags=arch:mips64el
-        :avocado: tags=machine:malta
-        :avocado: tags=endian:little
-        :avocado: tags=slowness:high
-        :avocado: tags=cpu:5KEc
-        """
-        kernel_url = ('https://github.com/philmd/qemu-testing-blob/'
-                      'raw/9ad2df38/mips/malta/mips64el/'
-                      'vmlinux-3.19.3.mtoman.20150408')
-        kernel_hash = '00d1d268fb9f7d8beda1de6bebcc46e884d71754'
-        kernel_path = self.fetch_asset(kernel_url, asset_hash=kernel_hash)
-        initrd_url = ('https://github.com/groeck/linux-build-test/'
-                      'raw/8584a59e/rootfs/'
-                      'mipsel64/rootfs.mipsel64r1.cpio.gz')
-        initrd_hash = '1dbb8a396e916847325284dbe2151167'
-        initrd_path_gz = self.fetch_asset(initrd_url, algorithm='md5',
-                                          asset_hash=initrd_hash)
-        initrd_path = self.workdir + "rootfs.cpio"
-        archive.gzip_uncompress(initrd_path_gz, initrd_path)
-
-        kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE +
-                               'console=ttyS0 console=tty '
-                               'rdinit=/sbin/init noreboot')
-        console_pattern = 'Boot successful.'
-        self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5,
-                    args=('-initrd', initrd_path))
tests/functional/test_mips64el_malta.py +25 -1
index a8da15a26b..1eac62b452 100755
@@ -15,9 +15,10 @@
 from qemu_test import LinuxKernelTest, Asset
 from qemu_test import exec_command_and_wait_for_pattern
 from qemu_test import skipIfMissingImports, skipFlakyTest, skipUntrustedTest
+from replay_kernel import ReplayKernelBase
 
 
-class MaltaMachineConsole(LinuxKernelTest):
+class MaltaMachineConsole(ReplayKernelBase):
 
     ASSET_KERNEL_2_63_2 = Asset(
         ('http://snapshot.debian.org/archive/debian/'
@@ -51,6 +52,15 @@ def test_mips64el_malta(self):
         console_pattern = 'Kernel command line: %s' % kernel_command_line
         self.wait_for_console_pattern(console_pattern)
 
+    def test_replay_mips64el_malta(self):
+        self.set_machine('malta')
+        kernel_path = self.archive_extract(self.ASSET_KERNEL_2_63_2,
+                                    member='boot/vmlinux-2.6.32-5-5kc-malta')
+        kernel_command_line = self.KERNEL_COMMON_COMMAND_LINE + 'console=ttyS0'
+        console_pattern = 'Kernel command line: %s' % kernel_command_line
+        self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5)
+
+
     ASSET_KERNEL_3_19_3 = Asset(
         ('https://github.com/philmd/qemu-testing-blob/'
          'raw/9ad2df38/mips/malta/mips64el/'
@@ -90,6 +100,20 @@ def test_mips64el_malta_5KEc_cpio(self):
         # Wait for VM to shut down gracefully
         self.vm.wait()
 
+    @skipUntrustedTest()
+    def test_replay_mips64el_malta_5KEc_cpio(self):
+        self.set_machine('malta')
+        self.cpu = '5KEc'
+        kernel_path = self.ASSET_KERNEL_3_19_3.fetch()
+        initrd_path = self.uncompress(self.ASSET_CPIO_R1)
+
+        kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE +
+                               'console=ttyS0 console=tty '
+                               'rdinit=/sbin/init noreboot')
+        console_pattern = 'Boot successful.'
+        self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5,
+                    args=('-initrd', initrd_path))
+
 
 @skipIfMissingImports('numpy', 'cv2')
 class MaltaMachineFramebuffer(LinuxKernelTest):
Signature · 1 line
2.48.1
#11 lore raw

PATCH 5/5 tests/functional/test_mips_malta: Convert the mips big endian replay tests

In reply to Thomas Huth

Move the mips big endian replay tests from tests/avocado/replay_kernel.py to the functional framework. Since the functional tests should be run per target, we cannot stick all replay tests in one file. Thus let's add these tests to the file where we already use the same asset already.

---
 tests/avocado/replay_kernel.py      | 54 -----------------------------
 tests/functional/test_mips_malta.py | 29 ++++++++++++++--
 2 files changed, 26 insertions(+), 57 deletions(-)
tests/avocado/replay_kernel.py +0 -54
index a45881b9a6..b9b54a8793 100644
@@ -144,24 +144,6 @@ def test_x86_64_q35(self):
 
         self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5)
 
-    def test_mips_malta(self):
-        """
-        :avocado: tags=arch:mips
-        :avocado: tags=machine:malta
-        :avocado: tags=endian:big
-        """
-        deb_url = ('http://snapshot.debian.org/archive/debian/'
-                   '20130217T032700Z/pool/main/l/linux-2.6/'
-                   'linux-image-2.6.32-5-4kc-malta_2.6.32-48_mips.deb')
-        deb_hash = 'a8cfc28ad8f45f54811fc6cf74fc43ffcfe0ba04'
-        deb_path = self.fetch_asset(deb_url, asset_hash=deb_hash)
-        kernel_path = self.extract_from_deb(deb_path,
-                                            '/boot/vmlinux-2.6.32-5-4kc-malta')
-        kernel_command_line = self.KERNEL_COMMON_COMMAND_LINE + 'console=ttyS0'
-        console_pattern = 'Kernel command line: %s' % kernel_command_line
-
-        self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5)
-
     def test_aarch64_virt(self):
         """
         :avocado: tags=arch:aarch64
@@ -428,39 +410,3 @@ def test_xtensa_lx60(self):
                    '/qac-best-of-multiarch/download/day02.tar.xz')
         file_path = self.fetch_asset(tar_url, asset_hash=tar_hash)
         self.do_test_advcal_2018(file_path, 'santas-sleigh-ride.elf')
-
-@skipUnless(os.getenv('AVOCADO_TIMEOUT_EXPECTED'), 'Test might timeout')
-class ReplayKernelSlow(ReplayKernelBase):
-    # Override the timeout, because this kernel includes an inner
-    # loop which is executed with TB recompilings during replay,
-    # making it very slow.
-    timeout = 180
-
-    def test_mips_malta_cpio(self):
-        """
-        :avocado: tags=arch:mips
-        :avocado: tags=machine:malta
-        :avocado: tags=endian:big
-        :avocado: tags=slowness:high
-        """
-        deb_url = ('http://snapshot.debian.org/archive/debian/'
-                   '20160601T041800Z/pool/main/l/linux/'
-                   'linux-image-4.5.0-2-4kc-malta_4.5.5-1_mips.deb')
-        deb_hash = 'a3c84f3e88b54e06107d65a410d1d1e8e0f340f8'
-        deb_path = self.fetch_asset(deb_url, asset_hash=deb_hash)
-        kernel_path = self.extract_from_deb(deb_path,
-                                            '/boot/vmlinux-4.5.0-2-4kc-malta')
-        initrd_url = ('https://github.com/groeck/linux-build-test/raw/'
-                      '8584a59ed9e5eb5ee7ca91f6d74bbb06619205b8/rootfs/'
-                      'mips/rootfs.cpio.gz')
-        initrd_hash = 'bf806e17009360a866bf537f6de66590de349a99'
-        initrd_path_gz = self.fetch_asset(initrd_url, asset_hash=initrd_hash)
-        initrd_path = self.workdir + "rootfs.cpio"
-        archive.gzip_uncompress(initrd_path_gz, initrd_path)
-
-        kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE +
-                               'console=ttyS0 console=tty '
-                               'rdinit=/sbin/init noreboot')
-        console_pattern = 'Boot successful.'
-        self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5,
-                    args=('-initrd', initrd_path))
tests/functional/test_mips_malta.py +26 -3
index 3b15038d89..a745f21dd2 100755
@@ -6,11 +6,12 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-from qemu_test import LinuxKernelTest, Asset
+from qemu_test import Asset, skipLongRuntime
 from qemu_test import exec_command_and_wait_for_pattern
+from replay_kernel import ReplayKernelBase
 
 
-class MaltaMachineConsole(LinuxKernelTest):
+class MaltaMachineConsole(ReplayKernelBase):
 
     ASSET_KERNEL_2_63_2 = Asset(
         ('http://snapshot.debian.org/archive/debian/'
@@ -32,6 +33,14 @@ def test_mips_malta(self):
         console_pattern = 'Kernel command line: %s' % kernel_command_line
         self.wait_for_console_pattern(console_pattern)
 
+    def test_replay_mips_malta(self):
+        self.set_machine('malta')
+        kernel_path = self.archive_extract(self.ASSET_KERNEL_2_63_2,
+                                     member='boot/vmlinux-2.6.32-5-4kc-malta')
+        kernel_command_line = self.KERNEL_COMMON_COMMAND_LINE + 'console=ttyS0'
+        console_pattern = 'Kernel command line: %s' % kernel_command_line
+        self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5)
+
     ASSET_KERNEL_4_5_0 = Asset(
         ('http://snapshot.debian.org/archive/debian/'
          '20160601T041800Z/pool/main/l/linux/'
@@ -71,6 +80,20 @@ def test_mips_malta_cpio(self):
         # Wait for VM to shut down gracefully
         self.vm.wait()
 
+    @skipLongRuntime()
+    def test_replay_mips_malta_cpio(self):
+        self.set_machine('malta')
+        kernel_path = self.archive_extract(self.ASSET_KERNEL_4_5_0,
+                                      member='boot/vmlinux-4.5.0-2-4kc-malta')
+        initrd_path = self.uncompress(self.ASSET_INITRD)
+
+        kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE +
+                               'console=ttyS0 console=tty '
+                               'rdinit=/sbin/init noreboot')
+        console_pattern = 'Boot successful.'
+        self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5,
+                    args=('-initrd', initrd_path))
+
 
 if __name__ == '__main__':
-    LinuxKernelTest.main()
+    ReplayKernelBase.main()
Signature · 1 line
2.48.1