tests/acceptance: Only run tests tagged 'gating-ci' on GitLab CI
cover tests/acceptance: Only run tests tagged 'gating-ci' on GitLab CI
This series try an idea from the CI BoF last week:
- allow adding all test to mainstream repository
- promote test to "gating" once it is proven stable
- only run 'gating-ci' on GitLab CI
Philippe Mathieu-Daudé (5):
tests/acceptance: Restrict virtio_check_params tests to X86 target tests/acceptance: Restore MIPS Malta multicore tests tests/acceptance: Skip incomplete virtio_version tests using '@skip' RFC tests/acceptance: Only run tests tagged 'gating-ci' on GitLab CI tests/acceptance: Let stable tests use the 'gating-ci' tag
tests/Makefile.include | 2 +- tests/acceptance/boot_linux.py | 10 ++++-- tests/acceptance/boot_linux_console.py | 38 ++++++++++++++++++++ tests/acceptance/cpu_queries.py | 1 + tests/acceptance/empty_cpu_model.py | 3 ++ tests/acceptance/linux_initrd.py | 4 +-- tests/acceptance/machine_arm_canona1100.py | 1 + tests/acceptance/machine_arm_integratorcp.py | 2 ++ tests/acceptance/machine_arm_n8x0.py | 2 ++ tests/acceptance/machine_avr6.py | 1 + tests/acceptance/machine_m68k_nextcube.py | 9 +++++ tests/acceptance/machine_mips_malta.py | 6 ++-- tests/acceptance/machine_rx_gdbsim.py | 3 -- tests/acceptance/machine_sparc64_sun4u.py | 1 + tests/acceptance/machine_sparc_leon3.py | 1 + tests/acceptance/migration.py | 12 ++++++- tests/acceptance/pc_cpu_hotplug_props.py | 3 ++ tests/acceptance/ppc_prep_40p.py | 3 +- tests/acceptance/replay_kernel.py | 17 +++++++-- tests/acceptance/version.py | 1 + tests/acceptance/virtio_check_params.py | 7 +++- tests/acceptance/virtio_version.py | 17 ++++++--- tests/acceptance/vnc.py | 9 +++++ tests/acceptance/x86_cpu_model_versions.py | 1 + 24 files changed, 133 insertions(+), 21 deletions(-)
Signature · 1 line
2.26.2
PATCH 1/5 tests/acceptance: Restrict virtio_check_params tests to X86 target
We disabled the virtio_check_params tests in commit 2d6a6e238a2 ("tests/acceptance/virtio_check_params: Disable the test"), because these tests were making multiarch CI fail (virtio is multiarch).
We took the big hammer, as we can restrict the tests to the X86 arch. Restore them on this single arch.
--- tests/acceptance/virtio_check_params.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-)
tests/acceptance/virtio_check_params.py +6 -1
@@ -42,6 +42,12 @@ class VirtioMaxSegSettingsCheck(Test): + """ + This test should be multi-arch, however only X86 is implemented. + + :avocado: tags=arch:x86_64 + """ + @staticmethod def make_pattern(props): pattern_items = ['{0} = \w+'.format(prop) for prop in props]
@@ -117,7 +123,6 @@ def seg_max_adjust_enabled(mt): return True return False - @skip("break multi-arch CI") def test_machine_types(self): # collect all machine types except 'none', 'isapc', 'microvm' with QEMUMachine(self.qemu_bin) as vm:
Signature · 1 line
2.26.2
Re: [PATCH-for-5.2? 1/5] tests/acceptance: Restrict virtio_check_params tests to X86 target
ping?
Quoted 39 lines · Philippe Mathieu-Daudé
We disabled the
virtio_check_paramstests in commit2d6a6e238a2("tests/acceptance/virtio_check_params: Disable the test"), because these tests were making multiarch CI fail (virtio is multiarch).We took the big hammer, as we can restrict the tests to the X86 arch. Restore them on this single arch.
Signed-off-by Philippe Mathieu-Daudé <[email protected]>--- tests/acceptance/virtio_check_params.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-)tests/acceptance/virtio_check_params.py +6
-1@@ -42,6 +42,12 @@ class VirtioMaxSegSettingsCheck(Test): + """ + This test should be multi-arch, however only X86 is implemented. + + :avocado: tags=arch:x86_64 + """ + @staticmethod def make_pattern(props): pattern_items = ['{0} = \w+'.format(prop) for prop in props]@@ -117,7 +123,6 @@ def seg_max_adjust_enabled(mt): return True return False - @skip("break multi-arch CI") def test_machine_types(self): # collect all machine types except 'none', 'isapc', 'microvm' with QEMUMachine(self.qemu_bin) as vm:
Re: [PATCH-for-5.2? 1/5] tests/acceptance: Restrict virtio_check_params tests to X86 target
In reply to Philippe Mathieu-DaudéQuoted 40 lines · Philippe Mathieu-Daudé
We disabled the
virtio_check_paramstests in commit2d6a6e238a2("tests/acceptance/virtio_check_params: Disable the test"), because these tests were making multiarch CI fail (virtio is multiarch).We took the big hammer, as we can restrict the tests to the X86 arch. Restore them on this single arch.
Signed-off-by Philippe Mathieu-Daudé <[email protected]>--- tests/acceptance/virtio_check_params.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-)tests/acceptance/virtio_check_params.py +6
-2@@ -42,6 +42,12 @@ class VirtioMaxSegSettingsCheck(Test): + """ + This test should be multi-arch, however only X86 is implemented. + + :avocado: tags=arch:x86_64 + """ + @staticmethod def make_pattern(props): pattern_items = ['{0} = \w+'.format(prop) for prop in props]@@ -117,7 +123,6 @@ def seg_max_adjust_enabled(mt): return True return False - @skip("break multi-arch CI") def test_machine_types(self): # collect all machine types except 'none', 'isapc', 'microvm' with QEMUMachine(self.qemu_bin) as vm: --
Quoted 3 lines
2.26.2
PATCH 2/5 tests/acceptance: Restore MIPS Malta multicore tests
In reply to Philippe Mathieu-DaudéSince 42a052333a6 ("hw/misc/mips_cpc: Start vCPU when powered on") the multicore support of the MIPS Malta board has been fixed.
This reverts commit 61bbce96fe4c8e3a2b7df5a67ba7dc6ba418e54b.
--- tests/acceptance/machine_mips_malta.py | 2 -- 1 file changed, 2 deletions(-)
tests/acceptance/machine_mips_malta.py +0 -2
@@ -100,7 +100,6 @@ def test_mips_malta_i6400_framebuffer_logo_1core(self): """ self.do_test_i6400_framebuffer_logo(1) - @skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab') def test_mips_malta_i6400_framebuffer_logo_7cores(self): """ :avocado: tags=arch:mips64el
@@ -110,7 +109,6 @@ def test_mips_malta_i6400_framebuffer_logo_7cores(self): """ self.do_test_i6400_framebuffer_logo(7) - @skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab') def test_mips_malta_i6400_framebuffer_logo_8cores(self): """ :avocado: tags=arch:mips64el
Signature · 1 line
2.26.2
Re: [PATCH-for-5.2? 2/5] tests/acceptance: Restore MIPS Malta multicore tests
Quoted 23 lines · Philippe Mathieu-Daudé
Since
42a052333a6("hw/misc/mips_cpc: Start vCPU when powered on") the multicore support of the MIPS Malta board has been fixed.This reverts commit
61bbce96fe4c8e3a2b7df5a67ba7dc6ba418e54b.Signed-off-by Philippe Mathieu-Daudé <[email protected]>--- tests/acceptance/machine_mips_malta.py | 2 -- 1 file changed, 2 deletions(-)tests/acceptance/machine_mips_malta.py +0
-1@@ -100,7 +100,6 @@ def test_mips_malta_i6400_framebuffer_logo_1core(self): """ self.do_test_i6400_framebuffer_logo(1) - @skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab') def test_mips_malta_i6400_framebuffer_logo_7cores(self): """ :avocado: tags=arch:mips64el
tests/acceptance/machine_mips_malta.py def test_mips_malta_i6400_framebuffer_logo_7cores(self): · line 112
Quoted 5 lines
tests/acceptance/machine_mips_malta.py +0
-1@@ -110,7 +109,6 @@ def test_mips_malta_i6400_framebuffer_logo_7cores(self): """ self.do_test_i6400_framebuffer_logo(7) - @skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab') def test_mips_malta_i6400_framebuffer_logo_8cores(self):
And this still doesn't work... :(
https://gitlab.com/philmd/qemu/-/jobs/826406235#L218
PATCH 3/5 tests/acceptance: Skip incomplete virtio_version tests using '@skip'
In reply to Philippe Mathieu-DaudéPrefer skipping incomplete tests with the "@skip" keyword, rather than commenting the code.
--- tests/acceptance/virtio_version.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-)
tests/acceptance/virtio_version.py +7 -4
@@ -140,17 +140,20 @@ def check_all_variants(self, qemu_devtype, virtio_devid): self.assertIn('conventional-pci-device', trans_ifaces) self.assertNotIn('pci-express-device', trans_ifaces) + @skip("virtio-blk requires 'driver' parameter") + def test_conventional_devs_driver(self): + self.check_all_variants('virtio-blk-pci', VIRTIO_BLOCK) + + @skip("virtio-9p requires 'fsdev' parameter") + def test_conventional_devs_fsdev(self): + self.check_all_variants('virtio-9p-pci', VIRTIO_9P) def test_conventional_devs(self): self.check_all_variants('virtio-net-pci', VIRTIO_NET) - # virtio-blk requires 'driver' parameter - #self.check_all_variants('virtio-blk-pci', VIRTIO_BLOCK) self.check_all_variants('virtio-serial-pci', VIRTIO_CONSOLE) self.check_all_variants('virtio-rng-pci', VIRTIO_RNG) self.check_all_variants('virtio-balloon-pci', VIRTIO_BALLOON) self.check_all_variants('virtio-scsi-pci', VIRTIO_SCSI) - # virtio-9p requires 'fsdev' parameter - #self.check_all_variants('virtio-9p-pci', VIRTIO_9P) def check_modern_only(self, qemu_devtype, virtio_devid): """Check if a modern-only virtio device type behaves as expected"""
Signature · 1 line
2.26.2
Re: [PATCH-for-5.2? 3/5] tests/acceptance: Skip incomplete virtio_version tests using '@skip'
tests/acceptance/virtio_version.py
Quoted 12 lines · Philippe Mathieu-Daudé
Prefer skipping incomplete tests with the "@skip" keyword, rather than commenting the code.
Signed-off-by Philippe Mathieu-Daudé <[email protected]>--- tests/acceptance/virtio_version.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-)tests/acceptance/virtio_version.py +0
-0
And I forgot:
-- >8 --
@@ -14,6 +14,7 @@
sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..',
'python'))
from qemu.machine import QEMUMachine
from avocado_qemu import Test
+from avocado import skip
# Virtio Device IDs:
VIRTIO_NET = 1
---
Quoted 26 lines
tests/acceptance/virtio_version.py +7
-4@@ -140,17 +140,20 @@ def check_all_variants(self, qemu_devtype, virtio_devid): self.assertIn('conventional-pci-device', trans_ifaces) self.assertNotIn('pci-express-device', trans_ifaces) + @skip("virtio-blk requires 'driver' parameter") + def test_conventional_devs_driver(self): + self.check_all_variants('virtio-blk-pci', VIRTIO_BLOCK) + + @skip("virtio-9p requires 'fsdev' parameter") + def test_conventional_devs_fsdev(self): + self.check_all_variants('virtio-9p-pci', VIRTIO_9P) def test_conventional_devs(self): self.check_all_variants('virtio-net-pci', VIRTIO_NET) - # virtio-blk requires 'driver' parameter - #self.check_all_variants('virtio-blk-pci', VIRTIO_BLOCK) self.check_all_variants('virtio-serial-pci', VIRTIO_CONSOLE) self.check_all_variants('virtio-rng-pci', VIRTIO_RNG) self.check_all_variants('virtio-balloon-pci', VIRTIO_BALLOON) self.check_all_variants('virtio-scsi-pci', VIRTIO_SCSI) - # virtio-9p requires 'fsdev' parameter - #self.check_all_variants('virtio-9p-pci', VIRTIO_9P) def check_modern_only(self, qemu_devtype, virtio_devid): """Check if a modern-only virtio device type behaves as expected"""
Re: [PATCH-for-5.2? 3/5] tests/acceptance: Skip incomplete virtio_version tests using '@skip'
In reply to Philippe Mathieu-DaudéQuoted 13 lines · Philippe Mathieu-Daudé
Prefer skipping incomplete tests with the "@skip" keyword, rather than commenting the code.
Signed-off-by Philippe Mathieu-Daudé <[email protected]>--- tests/acceptance/virtio_version.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-)tests/acceptance/virtio_version.py +0
-0
tests/acceptance/virtio_version.py def check_all_variants(self, qemu_devtype, virtio_devid): · line 143
Quoted 4 lines
tests/acceptance/virtio_version.py +1
-0@@ -140,17 +140,20 @@ def check_all_variants(self, qemu_devtype, virtio_devid): self.assertIn('conventional-pci-device', trans_ifaces) self.assertNotIn('pci-express-device', trans_ifaces) + @skip("virtio-blk requires 'driver' parameter")
Shouldn't that be 'drive' instead of 'driver' ?
Quoted 7 lines
tests/acceptance/virtio_version.py +6
-0+ def test_conventional_devs_driver(self): + self.check_all_variants('virtio-blk-pci', VIRTIO_BLOCK) + + @skip("virtio-9p requires 'fsdev' parameter") + def test_conventional_devs_fsdev(self): + self.check_all_variants('virtio-9p-pci', VIRTIO_9P)
tests/acceptance/virtio_version.py def check_all_variants(self, qemu_devtype, virtio_devid): · line 153
Quoted 10 lines
tests/acceptance/virtio_version.py +0
-4def test_conventional_devs(self): self.check_all_variants('virtio-net-pci', VIRTIO_NET) - # virtio-blk requires 'driver' parameter - #self.check_all_variants('virtio-blk-pci', VIRTIO_BLOCK) self.check_all_variants('virtio-serial-pci', VIRTIO_CONSOLE) self.check_all_variants('virtio-rng-pci', VIRTIO_RNG) self.check_all_variants('virtio-balloon-pci', VIRTIO_BALLOON) self.check_all_variants('virtio-scsi-pci', VIRTIO_SCSI) - # virtio-9p requires 'fsdev' parameter - #self.check_all_variants('virtio-9p-pci', VIRTIO_9P)
I think I'd prefer to keep the stuff commented ... otherwise it will show up in the logs, giving the impression that you could run the tests somehow if you just provided the right environment, which is just not possible right now.
Thomas
Re: [PATCH-for-5.2? 3/5] tests/acceptance: Skip incomplete virtio_version tests using '@skip'
Quoted 17 lines · Thomas Huth
Quoted 16 lines · Philippe Mathieu-Daudé
Prefer skipping incomplete tests with the "@skip" keyword, rather than commenting the code.
Signed-off-by Philippe Mathieu-Daudé <[email protected]>--- tests/acceptance/virtio_version.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-)tests/acceptance/virtio_version.py +0
-0@@ -140,17 +140,20 @@ def check_all_variants(self, qemu_devtype, virtio_devid): self.assertIn('conventional-pci-device', trans_ifaces) self.assertNotIn('pci-express-device', trans_ifaces)
Quoted 3 lines
tests/acceptance/virtio_version.py def check_all_variants(self, qemu_devtype, virtio_devid): · line 143
Quoted 1 line
tests/acceptance/virtio_version.py +1
-0+ @skip("virtio-blk requires 'driver' parameter")Shouldn't that be 'drive' instead of 'driver' ?
No clue, this is the previous commented line.
Quoted 19 lines
Quoted 8 lines
tests/acceptance/virtio_version.py +6
-0+ def test_conventional_devs_driver(self): + self.check_all_variants('virtio-blk-pci', VIRTIO_BLOCK) + + @skip("virtio-9p requires 'fsdev' parameter") + def test_conventional_devs_fsdev(self): + self.check_all_variants('virtio-9p-pci', VIRTIO_9P)tests/acceptance/virtio_version.py def check_all_variants(self, qemu_devtype, virtio_devid): · line 153
Quoted 10 lines
tests/acceptance/virtio_version.py +0
-4def test_conventional_devs(self): self.check_all_variants('virtio-net-pci', VIRTIO_NET) - # virtio-blk requires 'driver' parameter - #self.check_all_variants('virtio-blk-pci', VIRTIO_BLOCK) self.check_all_variants('virtio-serial-pci', VIRTIO_CONSOLE) self.check_all_variants('virtio-rng-pci', VIRTIO_RNG) self.check_all_variants('virtio-balloon-pci', VIRTIO_BALLOON) self.check_all_variants('virtio-scsi-pci', VIRTIO_SCSI) - # virtio-9p requires 'fsdev' parameter - #self.check_all_variants('virtio-9p-pci', VIRTIO_9P)
Quoted 3 lines
I think I'd prefer to keep the stuff commented ... otherwise it will show up in the logs, giving the impression that you could run the tests somehow if you just provided the right environment, which is just not possible right now.
Well, we usually don't commit commented code like that. If it is committed, it is important, then it has to show up in the log. If you don't want it logged, why not remove it then?
Re: [PATCH-for-5.2? 3/5] tests/acceptance: Skip incomplete virtio_version tests using '@skip'
Quoted 43 lines · Philippe Mathieu-Daudé
Quoted 17 lines · Thomas Huth
Quoted 16 lines · Philippe Mathieu-Daudé
Prefer skipping incomplete tests with the "@skip" keyword, rather than commenting the code.
Signed-off-by Philippe Mathieu-Daudé <[email protected]>--- tests/acceptance/virtio_version.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-)tests/acceptance/virtio_version.py +0
-0@@ -140,17 +140,20 @@ def check_all_variants(self, qemu_devtype, virtio_devid): self.assertIn('conventional-pci-device', trans_ifaces) self.assertNotIn('pci-express-device', trans_ifaces)Quoted 3 lines
tests/acceptance/virtio_version.py def check_all_variants(self, qemu_devtype, virtio_devid): · line 143
Quoted 1 line
tests/acceptance/virtio_version.py +1
-0+ @skip("virtio-blk requires 'driver' parameter")Shouldn't that be 'drive' instead of 'driver' ?
No clue, this is the previous commented line.
Quoted 19 lines
Quoted 18 lines
tests/acceptance/virtio_version.py +6
-4+ def test_conventional_devs_driver(self): + self.check_all_variants('virtio-blk-pci', VIRTIO_BLOCK) + + @skip("virtio-9p requires 'fsdev' parameter") + def test_conventional_devs_fsdev(self): + self.check_all_variants('virtio-9p-pci', VIRTIO_9P) def test_conventional_devs(self): self.check_all_variants('virtio-net-pci', VIRTIO_NET) - # virtio-blk requires 'driver' parameter - #self.check_all_variants('virtio-blk-pci', VIRTIO_BLOCK) self.check_all_variants('virtio-serial-pci', VIRTIO_CONSOLE) self.check_all_variants('virtio-rng-pci', VIRTIO_RNG) self.check_all_variants('virtio-balloon-pci', VIRTIO_BALLOON) self.check_all_variants('virtio-scsi-pci', VIRTIO_SCSI) - # virtio-9p requires 'fsdev' parameter - #self.check_all_variants('virtio-9p-pci', VIRTIO_9P)
Quoted 7 lines
Quoted 3 lines
I think I'd prefer to keep the stuff commented ... otherwise it will show up in the logs, giving the impression that you could run the tests somehow if you just provided the right environment, which is just not possible right now.
Well, we usually don't commit commented code like that. If it is committed, it is important, then it has to show up in the log. If you don't want it logged, why not remove it then?
Then I'd rather remove it. Or leave a comment saying "we cannot exercise virtio-9p-pci and virtio-blk-pci here (yet) since they need additional parameters to be set".
Thomas
RFC 4/5 tests/acceptance: Only run tests tagged 'gating-ci' on GitLab CI
In reply to Philippe Mathieu-DaudéTo avoid breaking our CI each time a test is added, switch from the "run all but disable some" to "only run the tagged tests on CI". This way we can add a test to the repository, and promote it to 'gating-ci' once it is proven stable enough.
---
TODO: where to add documentation?
- docs/devel/testing.rst (too big, split?)
- tests/acceptance/README.rst
--- tests/Makefile.include | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
tests/Makefile.include +1 -1
@@ -126,7 +126,7 @@ check-acceptance: check-venv $(TESTS_RESULTS_DIR) get-vm-images $(TESTS_VENV_DIR)/bin/python -m avocado \ --show=$(AVOCADO_SHOW) run --job-results-dir=$(TESTS_RESULTS_DIR) \ --filter-by-tags-include-empty --filter-by-tags-include-empty-key \ - $(AVOCADO_TAGS) \ + $(AVOCADO_TAGS) -t gating-ci \ $(if $(GITLAB_CI),,--failfast) tests/acceptance, \ "AVOCADO", "tests/acceptance")
Signature · 1 line
2.26.2
Quoted 22 lines · Philippe Mathieu-Daudé
To avoid breaking our CI each time a test is added, switch from the "run all but disable some" to "only run the tagged tests on CI". This way we can add a test to the repository, and promote it to 'gating-ci' once it is proven stable enough.
Signed-off-by Philippe Mathieu-Daudé <[email protected]>---
TODO: where to add documentation?
-
docs/devel/testing.rst(too big, split?)
-tests/acceptance/README.rst--- tests/Makefile.include | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)tests/Makefile.include +0
-0
tests/Makefile.include check-acceptance: check-venv $(TESTS_RESULTS_DIR) get-vm-images · line 129
Quoted 5 lines
tests/Makefile.include +1
-1@@ -126,7 +126,7 @@ check-acceptance: check-venv $(TESTS_RESULTS_DIR) get-vm-images $(TESTS_VENV_DIR)/bin/python -m avocado \ --show=$(AVOCADO_SHOW) run --job-results-dir=$(TESTS_RESULTS_DIR) \ --filter-by-tags-include-empty --filter-by-tags-include-empty-key \ - $(AVOCADO_TAGS) \ + $(AVOCADO_TAGS) -t gating-ci \
This doesn't work as expected, since we have:
AVOCADO_TAGS=$(patsubst %-softmmu,-t arch:%, $(filter %-softmmu,$(TARGET_DIRS)))
And avocado '-t' works as logical OR, not logical AND.
OTOH it seems this variable predate the auto-skip feature (when a binary is not present).
So I'll test this instead, which is simpler:
-- >8 --
@@ -90,7 +90,7 @@ TESTS_RESULTS_DIR=$(BUILD_DIR)/tests/results
# Any number of command separated loggers are accepted. For more
# information please refer to "avocado --help". AVOCADO_SHOW=app -AVOCADO_TAGS=$(patsubst %-softmmu,-t arch:%, $(filter %-softmmu,$(TARGET_DIRS))) +AVOCADO_TAGS=-t gating-ci
$(TESTS_VENV_DIR): $(TESTS_VENV_REQ)
$(call quiet-command, \
---
Quoted 43 lines · Philippe Mathieu-Daudé
Quoted 22 lines · Philippe Mathieu-Daudé
To avoid breaking our CI each time a test is added, switch from the "run all but disable some" to "only run the tagged tests on CI". This way we can add a test to the repository, and promote it to 'gating-ci' once it is proven stable enough.
Signed-off-by Philippe Mathieu-Daudé <[email protected]>---
TODO: where to add documentation?
-
docs/devel/testing.rst(too big, split?)
-tests/acceptance/README.rst--- tests/Makefile.include | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)tests/Makefile.include +0
-0tests/Makefile.include check-acceptance: check-venv $(TESTS_RESULTS_DIR) get-vm-images · line 129
Quoted 5 lines
tests/Makefile.include +1
-1@@ -126,7 +126,7 @@ check-acceptance: check-venv $(TESTS_RESULTS_DIR) get-vm-images $(TESTS_VENV_DIR)/bin/python -m avocado \ --show=$(AVOCADO_SHOW) run --job-results-dir=$(TESTS_RESULTS_DIR) \ --filter-by-tags-include-empty --filter-by-tags-include-empty-key \ - $(AVOCADO_TAGS) \ + $(AVOCADO_TAGS) -t gating-ci \This doesn't work as expected, since we have:
AVOCADO_TAGS=$(patsubst %-softmmu,-t arch:%, $(filter %-softmmu,$(TARGET_DIRS)))
And avocado '-t' works as logical OR, not logical AND.
OTOH it seems this variable predate the auto-skip feature (when a binary is not present).
So I'll test this instead, which is simpler:
-- >8 --
Review TESTS_RESULTS_DIR=$(BUILD_DIR)/tests/results · line 95
Quoted 7 lines
(quoted diff) +1
-1@@ -90,7 +90,7 @@ TESTS_RESULTS_DIR=$(BUILD_DIR)/tests/results # Any number of command separated loggers are accepted. For more # information please refer to "avocado --help". AVOCADO_SHOW=app -AVOCADO_TAGS=$(patsubst %-softmmu,-t arch:%, $(filter %-softmmu,$(TARGET_DIRS))) +AVOCADO_TAGS=-t gating-ci
But you can *add* to the resulting list with the arches, and then you get a logical AND. So it'd end up as:
-t arch:x86_64,gating-ci -t arch:aarch64,gating-ci [...]
To avoid having only "gating-ci" tests running on other environments that call "make check-acceptance", we can look at the environment variable that GitLab CI sets and optionally include the "gating-ci" tag.
Regards,
- Cleber.
Quoted 22 lines · Philippe Mathieu-Daudé
To avoid breaking our CI each time a test is added, switch from the "run all but disable some" to "only run the tagged tests on CI". This way we can add a test to the repository, and promote it to 'gating-ci' once it is proven stable enough.
Signed-off-by Philippe Mathieu-Daudé <[email protected]>---
TODO: where to add documentation?
-
docs/devel/testing.rst(too big, split?)
-tests/acceptance/README.rst--- tests/Makefile.include | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)tests/Makefile.include +0
-0
tests/Makefile.include check-acceptance: check-venv $(TESTS_RESULTS_DIR) get-vm-images · line 131
Quoted 7 lines
tests/Makefile.include +1
-1@@ -126,7 +126,7 @@ check-acceptance: check-venv $(TESTS_RESULTS_DIR) get-vm-images $(TESTS_VENV_DIR)/bin/python -m avocado \ --show=$(AVOCADO_SHOW) run --job-results-dir=$(TESTS_RESULTS_DIR) \ --filter-by-tags-include-empty --filter-by-tags-include-empty-key \ - $(AVOCADO_TAGS) \ + $(AVOCADO_TAGS) -t gating-ci \ $(if $(GITLAB_CI),,--failfast) tests/acceptance, \ "AVOCADO", "tests/acceptance")
self-NAck, this has to be added in acceptance_test_job_template in .gitlab-ci.d/buildtest-template.yml, not in tests/Makefile.include.
PATCH 5/5 tests/acceptance: Let stable tests use the 'gating-ci' tag
In reply to Philippe Mathieu-DaudéSwitch from "run all tests but disable some" to "only run the tests tagged as stable on CI".
Declare our current set of tests as stable with the 'gating-ci' tag.
--- tests/acceptance/boot_linux.py | 10 ++++-- tests/acceptance/boot_linux_console.py | 38 ++++++++++++++++++++ tests/acceptance/cpu_queries.py | 1 + tests/acceptance/empty_cpu_model.py | 3 ++ tests/acceptance/linux_initrd.py | 4 +-- tests/acceptance/machine_arm_canona1100.py | 1 + tests/acceptance/machine_arm_integratorcp.py | 2 ++ tests/acceptance/machine_arm_n8x0.py | 2 ++ tests/acceptance/machine_avr6.py | 1 + tests/acceptance/machine_m68k_nextcube.py | 9 +++++ tests/acceptance/machine_mips_malta.py | 4 ++- tests/acceptance/machine_rx_gdbsim.py | 3 -- tests/acceptance/machine_sparc64_sun4u.py | 1 + tests/acceptance/machine_sparc_leon3.py | 1 + tests/acceptance/migration.py | 12 ++++++- tests/acceptance/pc_cpu_hotplug_props.py | 3 ++ tests/acceptance/ppc_prep_40p.py | 3 +- tests/acceptance/replay_kernel.py | 17 +++++++-- tests/acceptance/version.py | 1 + tests/acceptance/virtio_version.py | 6 ++++ tests/acceptance/vnc.py | 9 +++++ tests/acceptance/x86_cpu_model_versions.py | 1 + 22 files changed, 119 insertions(+), 13 deletions(-)
tests/acceptance/boot_linux.py +8 -2
@@ -20,7 +20,6 @@ from avocado.utils import vmimage from avocado.utils import datadrainer from avocado.utils.path import find_command -from avocado import skipIf ACCEL_NOT_AVAILABLE_FMT = "%s accelerator does not seem to be available" KVM_NOT_AVAILABLE = ACCEL_NOT_AVAILABLE_FMT % "KVM"
@@ -114,6 +113,7 @@ class BootLinuxX8664(BootLinux): def test_pc_i440fx_tcg(self): """ + :avocado: tags=gating-ci :avocado: tags=machine:pc :avocado: tags=accel:tcg """
@@ -124,6 +124,7 @@ def test_pc_i440fx_tcg(self): def test_pc_i440fx_kvm(self): """ + :avocado: tags=gating-ci :avocado: tags=machine:pc :avocado: tags=accel:kvm """
@@ -134,6 +135,7 @@ def test_pc_i440fx_kvm(self): def test_pc_q35_tcg(self): """ + :avocado: tags=gating-ci :avocado: tags=machine:q35 :avocado: tags=accel:tcg """
@@ -144,6 +146,7 @@ def test_pc_q35_tcg(self): def test_pc_q35_kvm(self): """ + :avocado: tags=gating-ci :avocado: tags=machine:q35 :avocado: tags=accel:kvm """
@@ -171,6 +174,7 @@ def add_common_args(self): def test_virt_tcg(self): """ + :avocado: tags=gating-ci :avocado: tags=accel:tcg :avocado: tags=cpu:max """
@@ -184,6 +188,7 @@ def test_virt_tcg(self): def test_virt_kvm_gicv2(self): """ + :avocado: tags=gating-ci :avocado: tags=accel:kvm :avocado: tags=cpu:host :avocado: tags=device:gicv2
@@ -198,6 +203,7 @@ def test_virt_kvm_gicv2(self): def test_virt_kvm_gicv3(self): """ + :avocado: tags=gating-ci :avocado: tags=accel:kvm :avocado: tags=cpu:host :avocado: tags=device:gicv3
@@ -220,6 +226,7 @@ class BootLinuxPPC64(BootLinux): def test_pseries_tcg(self): """ + :avocado: tags=gating-ci :avocado: tags=machine:pseries :avocado: tags=accel:tcg """
@@ -236,7 +243,6 @@ class BootLinuxS390X(BootLinux): chksum = '4caaab5a434fd4d1079149a072fdc7891e354f834d355069ca982fdcaf5a122d' - @skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab') def test_s390_ccw_virtio_tcg(self): """ :avocado: tags=machine:s390-ccw-virtio
tests/acceptance/boot_linux_console.py +38 -0
@@ -98,6 +98,7 @@ class BootLinuxConsole(LinuxKernelTest): def test_x86_64_pc(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:x86_64 :avocado: tags=machine:pc """
@@ -117,6 +118,7 @@ def test_x86_64_pc(self): def test_mips_malta(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:mips :avocado: tags=machine:malta :avocado: tags=endian:big
@@ -150,6 +152,7 @@ def test_mips64el_malta(self): [2] https://kernel-team.pages.debian.net/kernel-handbook/ ch-common-tasks.html#s-common-official + :avocado: tags=gating-ci :avocado: tags=arch:mips64el :avocado: tags=machine:malta """
@@ -171,6 +174,7 @@ def test_mips64el_malta(self): def test_mips_malta_cpio(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:mips :avocado: tags=machine:malta :avocado: tags=endian:big
@@ -213,6 +217,7 @@ def test_mips_malta_cpio(self): @skipUnless(os.getenv('AVOCADO_ALLOW_UNTRUSTED_CODE'), 'untrusted code') def test_mips64el_malta_5KEc_cpio(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:mips64el :avocado: tags=machine:malta :avocado: tags=endian:little
@@ -273,6 +278,7 @@ def do_test_mips_malta32el_nanomips(self, kernel_url, kernel_hash): def test_mips_malta32el_nanomips_4k(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:mipsel :avocado: tags=machine:malta :avocado: tags=endian:little
@@ -285,6 +291,7 @@ def test_mips_malta32el_nanomips_4k(self): def test_mips_malta32el_nanomips_16k_up(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:mipsel :avocado: tags=machine:malta :avocado: tags=endian:little
@@ -297,6 +304,7 @@ def test_mips_malta32el_nanomips_16k_up(self): def test_mips_malta32el_nanomips_64k_dbg(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:mipsel :avocado: tags=machine:malta :avocado: tags=endian:little
@@ -309,6 +317,7 @@ def test_mips_malta32el_nanomips_64k_dbg(self): def test_aarch64_virt(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:aarch64 :avocado: tags=machine:virt """
@@ -330,6 +339,7 @@ def test_aarch64_virt(self): def test_aarch64_xlnx_versal_virt(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:aarch64 :avocado: tags=machine:xlnx-versal-virt :avocado: tags=device:pl011
@@ -355,6 +365,7 @@ def test_aarch64_xlnx_versal_virt(self): def test_arm_virt(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:arm :avocado: tags=machine:virt """
@@ -375,6 +386,7 @@ def test_arm_virt(self): def test_arm_emcraft_sf2(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:arm :avocado: tags=machine:emcraft-sf2 :avocado: tags=endian:little
@@ -439,6 +451,7 @@ def do_test_arm_raspi2(self, uart_id): def test_arm_raspi2_uart0(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:arm :avocado: tags=machine:raspi2 :avocado: tags=device:pl011
@@ -447,6 +460,7 @@ def test_arm_raspi2_uart0(self): def test_arm_exynos4210_initrd(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:arm :avocado: tags=machine:smdkc210 """
@@ -489,6 +503,7 @@ def test_arm_exynos4210_initrd(self): 'Test artifacts fetched from unreliable apt.armbian.com') def test_arm_cubieboard_initrd(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:arm :avocado: tags=machine:cubieboard """
@@ -531,6 +546,7 @@ def test_arm_cubieboard_initrd(self): 'Test artifacts fetched from unreliable apt.armbian.com') def test_arm_cubieboard_sata(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:arm :avocado: tags=machine:cubieboard """
@@ -622,6 +638,7 @@ def test_arm_quanta_gsj(self): def test_arm_quanta_gsj_initrd(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:arm :avocado: tags=machine:quanta-gsj """
@@ -660,6 +677,7 @@ def test_arm_quanta_gsj_initrd(self): 'Test artifacts fetched from unreliable apt.armbian.com') def test_arm_orangepi(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:arm :avocado: tags=machine:orangepi-pc """
@@ -687,6 +705,7 @@ def test_arm_orangepi(self): 'Test artifacts fetched from unreliable apt.armbian.com') def test_arm_orangepi_initrd(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:arm :avocado: tags=machine:orangepi-pc """
@@ -731,6 +750,7 @@ def test_arm_orangepi_initrd(self): 'Test artifacts fetched from unreliable apt.armbian.com') def test_arm_orangepi_sd(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:arm :avocado: tags=machine:orangepi-pc :avocado: tags=device:sd
@@ -786,6 +806,7 @@ def test_arm_orangepi_sd(self): @skipUnless(P7ZIP_AVAILABLE, '7z not installed') def test_arm_orangepi_bionic(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:arm :avocado: tags=machine:orangepi-pc :avocado: tags=device:sd
@@ -829,6 +850,7 @@ def test_arm_orangepi_bionic(self): @skipUnless(os.getenv('AVOCADO_ALLOW_LARGE_STORAGE'), 'storage limited') def test_arm_orangepi_uboot_netbsd9(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:arm :avocado: tags=machine:orangepi-pc :avocado: tags=device:sd
@@ -893,6 +915,7 @@ def test_arm_orangepi_uboot_netbsd9(self): def test_aarch64_raspi3_atf(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:aarch64 :avocado: tags=machine:raspi3 :avocado: tags=cpu:cortex-a53
@@ -915,6 +938,7 @@ def test_aarch64_raspi3_atf(self): def test_s390x_s390_ccw_virtio(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:s390x :avocado: tags=machine:s390-ccw-virtio """
@@ -935,6 +959,7 @@ def test_s390x_s390_ccw_virtio(self): def test_alpha_clipper(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:alpha :avocado: tags=machine:clipper """
@@ -956,6 +981,7 @@ def test_alpha_clipper(self): def test_ppc64_pseries(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:ppc64 :avocado: tags=machine:pseries """
@@ -975,6 +1001,7 @@ def test_ppc64_pseries(self): def test_m68k_q800(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:m68k :avocado: tags=machine:q800 """
@@ -1010,6 +1037,7 @@ def do_test_advcal_2018(self, day, tar_hash, kernel_name, console=0): def test_arm_vexpressa9(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:arm :avocado: tags=machine:vexpress-a9 """
@@ -1019,6 +1047,7 @@ def test_arm_vexpressa9(self): def test_m68k_mcf5208evb(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:m68k :avocado: tags=machine:mcf5208evb """
@@ -1027,6 +1056,7 @@ def test_m68k_mcf5208evb(self): def test_microblaze_s3adsp1800(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:microblaze :avocado: tags=machine:petalogix-s3adsp1800 """
@@ -1035,6 +1065,7 @@ def test_microblaze_s3adsp1800(self): def test_or1k_sim(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:or1k :avocado: tags=machine:or1k-sim """
@@ -1043,6 +1074,7 @@ def test_or1k_sim(self): def test_nios2_10m50(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:nios2 :avocado: tags=machine:10m50-ghrd """
@@ -1051,6 +1083,7 @@ def test_nios2_10m50(self): def test_ppc64_e500(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:ppc64 :avocado: tags=machine:ppce500 """
@@ -1060,6 +1093,7 @@ def test_ppc64_e500(self): def test_ppc_g3beige(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:ppc :avocado: tags=machine:g3beige """
@@ -1069,6 +1103,7 @@ def test_ppc_g3beige(self): def test_ppc_mac99(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:ppc :avocado: tags=machine:mac99 """
@@ -1078,6 +1113,7 @@ def test_ppc_mac99(self): def test_sh4_r2d(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:sh4 :avocado: tags=machine:r2d """
@@ -1087,6 +1123,7 @@ def test_sh4_r2d(self): def test_sparc_ss20(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:sparc :avocado: tags=machine:SS-20 """
@@ -1095,6 +1132,7 @@ def test_sparc_ss20(self): def test_xtensa_lx60(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:xtensa :avocado: tags=machine:lx60 """
tests/acceptance/cpu_queries.py +1 -0
@@ -19,6 +19,7 @@ class QueryCPUModelExpansion(Test): def test(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:x86_64 :avocado: tags=machine:none """
tests/acceptance/empty_cpu_model.py +3 -0
@@ -11,6 +11,9 @@ class EmptyCPUModel(Test): def test(self): + """ + :avocado: tags=gating-ci + """ self.vm.add_args('-S', '-display', 'none', '-machine', 'none', '-cpu', '') self.vm.set_qmp_monitor(enabled=False) self.vm.launch()
tests/acceptance/linux_initrd.py +2 -2
@@ -13,7 +13,6 @@ import tempfile from avocado_qemu import Test -from avocado import skipIf class LinuxInitrd(Test):
@@ -32,6 +31,8 @@ def test_with_2gib_file_should_exit_error_msg_with_linux_v3_6(self): and expect it exits with error message. Fedora-18 shipped with linux-3.6 which have not supported xloadflags cannot support more than 2GiB initrd. + + :avocado: tags=gating-ci """ kernel_url = ('https://archives.fedoraproject.org/pub/archive/fedora/li' 'nux/releases/18/Fedora/x86_64/os/images/pxeboot/vmlinuz')
@@ -53,7 +54,6 @@ def test_with_2gib_file_should_exit_error_msg_with_linux_v3_6(self): max_size + 1) self.assertRegex(self.vm.get_log(), expected_msg) - @skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab') def test_with_2gib_file_should_work_with_linux_v4_16(self): """ QEMU has supported up to 4 GiB initrd for recent kernel
tests/acceptance/machine_arm_canona1100.py +1 -0
@@ -19,6 +19,7 @@ class CanonA1100Machine(Test): def test_arm_canona1100(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:arm :avocado: tags=machine:canon-a1100 :avocado: tags=device:pflash_cfi02
tests/acceptance/machine_arm_integratorcp.py +2 -0
@@ -54,6 +54,7 @@ def boot_integratorcp(self): def test_integratorcp_console(self): """ Boots the Linux kernel and checks that the console is operational + :avocado: tags=gating-ci :avocado: tags=arch:arm :avocado: tags=machine:integratorcp :avocado: tags=device:pl011
@@ -67,6 +68,7 @@ def test_integratorcp_console(self): def test_framebuffer_tux_logo(self): """ Boot Linux and verify the Tux logo is displayed on the framebuffer. + :avocado: tags=gating-ci :avocado: tags=arch:arm :avocado: tags=machine:integratorcp :avocado: tags=device:pl110
tests/acceptance/machine_arm_n8x0.py +2 -0
@@ -35,6 +35,7 @@ def __do_test_n8x0(self): @skipUnless(os.getenv('AVOCADO_ALLOW_UNTRUSTED_CODE'), 'untrusted code') def test_n800(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:arm :avocado: tags=machine:n800 """
@@ -43,6 +44,7 @@ def test_n800(self): @skipUnless(os.getenv('AVOCADO_ALLOW_UNTRUSTED_CODE'), 'untrusted code') def test_n810(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:arm :avocado: tags=machine:n810 """
tests/acceptance/machine_avr6.py +1 -0
@@ -26,6 +26,7 @@ class AVR6Machine(Test): def test_freertos(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:avr :avocado: tags=machine:arduino-mega-2560-v3 """
tests/acceptance/machine_m68k_nextcube.py +9 -0
@@ -69,6 +69,9 @@ def check_bootrom_framebuffer(self, screenshot_path): @skipUnless(PIL_AVAILABLE, 'Python PIL not installed') def test_bootrom_framebuffer_size(self): + """ + :avocado: tags=gating-ci + """ screenshot_path = os.path.join(self.workdir, "dump.ppm") self.check_bootrom_framebuffer(screenshot_path)
@@ -78,6 +81,9 @@ def test_bootrom_framebuffer_size(self): @skipUnless(tesseract_available(3), 'tesseract v3 OCR tool not available') def test_bootrom_framebuffer_ocr_with_tesseract_v3(self): + """ + :avocado: tags=gating-ci + """ screenshot_path = os.path.join(self.workdir, "dump.ppm") self.check_bootrom_framebuffer(screenshot_path)
@@ -94,6 +100,9 @@ def test_bootrom_framebuffer_ocr_with_tesseract_v3(self): # that it is still alpha-level software. @skipUnless(tesseract_available(4), 'tesseract v4 OCR tool not available') def test_bootrom_framebuffer_ocr_with_tesseract_v4(self): + """ + :avocado: tags=gating-ci + """ screenshot_path = os.path.join(self.workdir, "dump.ppm") self.check_bootrom_framebuffer(screenshot_path)
tests/acceptance/machine_mips_malta.py +3 -1
@@ -15,7 +15,6 @@ from avocado_qemu import Test from avocado_qemu import wait_for_console_pattern from avocado.utils import archive -from avocado import skipIf NUMPY_AVAILABLE = True
@@ -94,6 +93,7 @@ def do_test_i6400_framebuffer_logo(self, cpu_cores_count): def test_mips_malta_i6400_framebuffer_logo_1core(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:mips64el :avocado: tags=machine:malta :avocado: tags=cpu:i6400
@@ -102,6 +102,7 @@ def test_mips_malta_i6400_framebuffer_logo_1core(self): def test_mips_malta_i6400_framebuffer_logo_7cores(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:mips64el :avocado: tags=machine:malta :avocado: tags=cpu:i6400
@@ -111,6 +112,7 @@ def test_mips_malta_i6400_framebuffer_logo_7cores(self): def test_mips_malta_i6400_framebuffer_logo_8cores(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:mips64el :avocado: tags=machine:malta :avocado: tags=cpu:i6400
tests/acceptance/machine_rx_gdbsim.py +0 -3
@@ -10,7 +10,6 @@ import os -from avocado import skipIf from avocado_qemu import Test from avocado_qemu import exec_command_and_wait_for_pattern from avocado_qemu import wait_for_console_pattern
@@ -22,7 +21,6 @@ class RxGdbSimMachine(Test): timeout = 30 KERNEL_COMMON_COMMAND_LINE = 'printk.time=0 ' - @skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab') def test_uboot(self): """ U-Boot and checks that the console is operational.
@@ -46,7 +44,6 @@ def test_uboot(self): # FIXME limit baudrate on chardev, else we type too fast #exec_command_and_wait_for_pattern(self, 'version', gcc_version) - @skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab') def test_linux_sash(self): """ Boots a Linux kernel and checks that the console is operational.
tests/acceptance/machine_sparc64_sun4u.py +1 -0
@@ -21,6 +21,7 @@ class Sun4uMachine(LinuxKernelTest): def test_sparc64_sun4u(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:sparc64 :avocado: tags=machine:sun4u """
tests/acceptance/machine_sparc_leon3.py +1 -0
@@ -19,6 +19,7 @@ class Leon3Machine(Test): # and QEMU exit calling cpu_abort(), which makes this test to fail. def test_leon3_helenos_uimage(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:sparc :avocado: tags=machine:leon3_generic :avocado: tags=binfmt:uimage
tests/acceptance/migration.py +11 -1
@@ -64,17 +64,27 @@ def _get_free_port(self): def test_migration_with_tcp_localhost(self): + """ + :avocado: tags=gating-ci + """ dest_uri = 'tcp:localhost:%u' % self._get_free_port() self.do_migrate(dest_uri) def test_migration_with_unix(self): + """ + :avocado: tags=gating-ci + """ with tempfile.TemporaryDirectory(prefix='socket_') as socket_path: dest_uri = 'unix:%s/qemu-test.sock' % socket_path self.do_migrate(dest_uri) @skipUnless(find_command('nc', default=False), "'nc' command not found") def test_migration_with_exec(self): - """The test works for both netcat-traditional and netcat-openbsd packages.""" + """ + The test works for both netcat-traditional and netcat-openbsd packages. + + :avocado: tags=gating-ci + """ free_port = self._get_free_port() dest_uri = 'exec:nc -l localhost %u' % free_port src_uri = 'exec:nc localhost %u' % free_port
tests/acceptance/pc_cpu_hotplug_props.py +3 -0
@@ -27,6 +27,9 @@ class OmittedCPUProps(Test): :avocado: tags=arch:x86_64 """ def test_no_die_id(self): + """ + :avocado: tags=gating-ci + """ self.vm.add_args('-nodefaults', '-S') self.vm.add_args('-smp', '1,sockets=2,cores=2,threads=2,maxcpus=8') self.vm.add_args('-cpu', 'qemu64')
tests/acceptance/ppc_prep_40p.py +2 -1
@@ -7,7 +7,6 @@ import os -from avocado import skipIf from avocado import skipUnless from avocado_qemu import Test from avocado_qemu import wait_for_console_pattern
@@ -49,6 +48,7 @@ def test_factory_firmware_and_netbsd(self): def test_openbios_192m(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:ppc :avocado: tags=machine:40p """
@@ -62,6 +62,7 @@ def test_openbios_192m(self): def test_openbios_and_netbsd(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:ppc :avocado: tags=machine:40p """
tests/acceptance/replay_kernel.py +14 -3
@@ -14,7 +14,6 @@ import logging import time -from avocado import skipIf from avocado import skipUnless from avocado_qemu import wait_for_console_pattern from avocado.utils import archive
@@ -77,7 +76,6 @@ def run_rr(self, kernel_path, kernel_command_line, console_pattern, logger.info('replay overhead {:.2%}'.format(t2 / t1 - 1)) class ReplayKernelNormal(ReplayKernelBase): - @skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab') def test_x86_64_pc(self): """ :avocado: tags=arch:x86_64
@@ -96,6 +94,7 @@ def test_x86_64_pc(self): def test_mips_malta(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:mips :avocado: tags=machine:malta :avocado: tags=endian:big
@@ -125,6 +124,7 @@ def test_mips64el_malta(self): [2] https://kernel-team.pages.debian.net/kernel-handbook/ ch-common-tasks.html#s-common-official + :avocado: tags=gating-ci :avocado: tags=arch:mips64el :avocado: tags=machine:malta """
@@ -141,6 +141,7 @@ def test_mips64el_malta(self): def test_aarch64_virt(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:aarch64 :avocado: tags=machine:virt :avocado: tags=cpu:cortex-a53
@@ -160,6 +161,7 @@ def test_aarch64_virt(self): def test_arm_virt(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:arm :avocado: tags=machine:virt """
@@ -175,7 +177,6 @@ def test_arm_virt(self): self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=1) - @skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab') @skipUnless(os.getenv('ARMBIAN_ARTIFACTS_CACHED'), 'Test artifacts fetched from unreliable apt.armbian.com') def test_arm_cubieboard_initrd(self):
@@ -211,6 +212,7 @@ def test_arm_cubieboard_initrd(self): def test_ppc64_pseries(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:ppc64 :avocado: tags=machine:pseries """
@@ -227,6 +229,7 @@ def test_ppc64_pseries(self): def test_m68k_q800(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:m68k :avocado: tags=machine:q800 """
@@ -258,6 +261,7 @@ def do_test_advcal_2018(self, file_path, kernel_name, args=None): def test_arm_vexpressa9(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:arm :avocado: tags=machine:vexpress-a9 """
@@ -271,6 +275,7 @@ def test_arm_vexpressa9(self): def test_m68k_mcf5208evb(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:m68k :avocado: tags=machine:mcf5208evb """
@@ -282,6 +287,7 @@ def test_m68k_mcf5208evb(self): def test_microblaze_s3adsp1800(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:microblaze :avocado: tags=machine:petalogix-s3adsp1800 """
@@ -293,6 +299,7 @@ def test_microblaze_s3adsp1800(self): def test_ppc64_e500(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:ppc64 :avocado: tags=machine:ppce500 :avocado: tags=cpu:e5500
@@ -305,6 +312,7 @@ def test_ppc64_e500(self): def test_ppc_g3beige(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:ppc :avocado: tags=machine:g3beige """
@@ -317,6 +325,7 @@ def test_ppc_g3beige(self): def test_ppc_mac99(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:ppc :avocado: tags=machine:mac99 """
@@ -329,6 +338,7 @@ def test_ppc_mac99(self): def test_sparc_ss20(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:sparc :avocado: tags=machine:SS-20 """
@@ -340,6 +350,7 @@ def test_sparc_ss20(self): def test_xtensa_lx60(self): """ + :avocado: tags=gating-ci :avocado: tags=arch:xtensa :avocado: tags=machine:lx60 :avocado: tags=cpu:dc233c
tests/acceptance/version.py +1 -0
@@ -14,6 +14,7 @@ class Version(Test): """ + :avocado: tags=gating-ci :avocado: tags=quick """ def test_qmp_human_info_version(self):
tests/acceptance/virtio_version.py +6 -0
@@ -149,6 +149,9 @@ def test_conventional_devs_fsdev(self): self.check_all_variants('virtio-9p-pci', VIRTIO_9P) def test_conventional_devs(self): + """ + :avocado: tags=gating-ci + """ self.check_all_variants('virtio-net-pci', VIRTIO_NET) self.check_all_variants('virtio-serial-pci', VIRTIO_CONSOLE) self.check_all_variants('virtio-rng-pci', VIRTIO_RNG)
@@ -171,6 +174,9 @@ def check_modern_only(self, qemu_devtype, virtio_devid): self.assertIn('pci-express-device', ifaces) def test_modern_only_devs(self): + """ + :avocado: tags=gating-ci + """ self.check_modern_only('virtio-vga', VIRTIO_GPU) self.check_modern_only('virtio-gpu-pci', VIRTIO_GPU) self.check_modern_only('virtio-mouse-pci', VIRTIO_INPUT)
tests/acceptance/vnc.py +9 -0
@@ -21,6 +21,9 @@ def test_no_vnc(self): self.assertFalse(self.vm.qmp('query-vnc')['return']['enabled']) def test_no_vnc_change_password(self): + """ + :avocado: tags=gating-ci + """ self.vm.add_args('-nodefaults', '-S') self.vm.launch() self.assertFalse(self.vm.qmp('query-vnc')['return']['enabled'])
@@ -35,6 +38,9 @@ def test_no_vnc_change_password(self): 'Could not set password') def test_change_password_requires_a_password(self): + """ + :avocado: tags=gating-ci + """ self.vm.add_args('-nodefaults', '-S', '-vnc', ':0') self.vm.launch() self.assertTrue(self.vm.qmp('query-vnc')['return']['enabled'])
@@ -49,6 +55,9 @@ def test_change_password_requires_a_password(self): 'Could not set password') def test_change_password(self): + """ + :avocado: tags=gating-ci + """ self.vm.add_args('-nodefaults', '-S', '-vnc', ':0,password') self.vm.launch() self.assertTrue(self.vm.qmp('query-vnc')['return']['enabled'])
tests/acceptance/x86_cpu_model_versions.py +1 -0
@@ -28,6 +28,7 @@ class X86CPUModelAliases(avocado_qemu.Test): """ Validation of PC CPU model versions and CPU model aliases + :avocado: tags=gating-ci :avocado: tags=arch:x86_64 """ def validate_aliases(self, cpus):
Signature · 1 line
2.26.2