Manual test cleanup
cover Manual test cleanup
This moves some maunal tests to runtime. Delete others being run in runtime.
Armin Kuster (7):
manual tests migrate to selftest
manual qa/oe-core: remove packageconfig flag tests manual qa/bsp-qemu: remove rpm tests already done in runtime manual qa/bsp-qemu: remove KVM enabled which is already done in
selftest runqemu
manual/bsp-qemu: drop xserver test
manual/bsp-qemu: remove connmand test done in runtime test manual sdk: remove this manual sdk test
meta/lib/oeqa/manual/bsp-qemu.json | 142 ------------------------------ meta/lib/oeqa/manual/oe-core.json | 36 +------- meta/lib/oeqa/manual/sdk.json | 32 ------- meta/lib/oeqa/selftest/cases/oescripts.py | 89 +++++++++++++++++++ 4 files changed, 90 insertions(+), 209 deletions(-) delete mode 100644 meta/lib/oeqa/manual/sdk.json
Signature · 1 line
2.7.4
PATCH 1/7 manual tests migrate to selftest
--- meta/lib/oeqa/selftest/cases/oescripts.py | 89 +++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+)
meta/lib/oeqa/selftest/cases/oescripts.py +89 -0
@@ -1,8 +1,11 @@ +import os, sys from oeqa.selftest.case import OESelftestTestCase +from oeqa.core.decorator.depends import OETestDepends from oeqa.selftest.cases.buildhistory import BuildhistoryBase from oeqa.utils.commands import Command, runCmd, bitbake, get_bb_var, get_test_layer from oeqa.core.decorator.oeid import OETestID + class BuildhistoryDiffTests(BuildhistoryBase): @OETestID(295)
@@ -26,3 +29,89 @@ class BuildhistoryDiffTests(BuildhistoryBase): self.fail('Unexpected line:\n%s\nExpected line endings:\n %s' % (line, '\n '.join(expected_endlines))) if expected_endlines: self.fail('Missing expected line endings:\n %s' % '\n '.join(expected_endlines)) + + +class OEScriptTests(OESelftestTestCase): + def check_endlines(self, results, expected_endlines): + for line in results.output.splitlines(): + for el in expected_endlines: + if line == el: + expected_endlines.remove(el) + break + + if expected_endlines: + self.fail('Missing expected line endings:\n %s' % '\n '.join(expected_endlines)) + + +class OEPybootchartguyTests(OEScriptTests): + def test_pybootchartguy_help(self): + runCmd('../scripts/pybootchartgui/pybootchartgui.py --help') + + def test_pybootchartguy_to_generate_build_png_output(self): + tmpdir = get_bb_var('TMPDIR') + runCmd('../scripts/pybootchartgui/pybootchartgui.py %s/buildstats -o %s/charts -f png' % (tmpdir, tmpdir)) + + def test_pybootchartguy_to_generate_build_svg_output(self): + tmpdir = get_bb_var('TMPDIR') + runCmd('../scripts/pybootchartgui/pybootchartgui.py %s/buildstats -o %s/charts -f svg' % (tmpdir, tmpdir)) + + def test_pybootchartguy_to_generate_build_pdf_output(self): + tmpdir = get_bb_var('TMPDIR') + runCmd('../scripts/pybootchartgui/pybootchartgui.py %s/buildstats -o %s/charts -f pdf' % (tmpdir, tmpdir)) + + +class OEListPackageconfigTests(OEScriptTests): + #oe-core.scripts.List_all_the_PACKAGECONFIG's_flags + def test_packageconfig_flags_help(self): + runCmd('../scripts/contrib/list-packageconfig-flags.py -h') + + def test_packageconfig_flags_default(self): + results = runCmd('../scripts/contrib/list-packageconfig-flags.py') + expected_endlines = [] + expected_endlines.append("RECIPE NAME PACKAGECONFIG FLAGS") + expected_endlines.append("xserver-xorg-1.20.3 dri dri2 dri3 gcrypt glamor glx nettle openssl systemd systemd-logind udev unwind xinerama xmlto xshmfence xwayland") + expected_endlines.append("znc-1.7.1 ipv6") + + self.check_endlines(results, expected_endlines) + + + def test_packageconfig_flags_option_flags(self): + results = runCmd('../scripts/contrib/list-packageconfig-flags.py -f') + expected_endlines = [] + expected_endlines.append("PACKAGECONFIG FLAG RECIPE NAMES") + expected_endlines.append("xshmfence xserver-xorg-1.20.3") + + self.check_endlines(results, expected_endlines) + + def test_packageconfig_flags_option_all(self): + results = runCmd('../scripts/contrib/list-packageconfig-flags.py -a') + expected_endlines = [] + expected_endlines.append("xserver-xorg-1.20.3") + expected_endlines.append("PACKAGECONFIG dri2 udev openssl") + expected_endlines.append("PACKAGECONFIG[udev] --enable-config-udev,--disable-config-udev,udev") + expected_endlines.append("PACKAGECONFIG[dri] --enable-dri,--disable-dri,xorgproto virtual/mesa") + expected_endlines.append("PACKAGECONFIG[dri2] --enable-dri2,--disable-dri2,xorgproto") + expected_endlines.append("PACKAGECONFIG[dri3] --enable-dri3,--disable-dri3,xorgproto") + expected_endlines.append("PACKAGECONFIG[glx] --enable-glx,--disable-glx,xorgproto virtual/libgl virtual/libx11") + expected_endlines.append("PACKAGECONFIG[glamor] --enable-glamor,--disable-glamor,libepoxy virtual/libgbm,libegl") + expected_endlines.append("PACKAGECONFIG[unwind] --enable-libunwind,--disable-libunwind,libunwind") + expected_endlines.append("PACKAGECONFIG[xshmfence] --enable-xshmfence,--disable-xshmfence,libxshmfence") + expected_endlines.append("PACKAGECONFIG[xmlto] --with-xmlto, --without-xmlto, xmlto-native docbook-xml-dtd4-native docbook-xsl-stylesheets-native") + expected_endlines.append("PACKAGECONFIG[systemd-logind] --enable-systemd-logind=yes,--enable-systemd-logind=no,dbus,") + expected_endlines.append("PACKAGECONFIG[systemd] --with-systemd-daemon,--without-systemd-daemon,systemd") + expected_endlines.append("PACKAGECONFIG[xinerama] --enable-xinerama,--disable-xinerama,xorgproto") + expected_endlines.append("PACKAGECONFIG[xwayland] --enable-xwayland,--disable-xwayland,wayland wayland-native wayland-protocols libepoxy") + expected_endlines.append("PACKAGECONFIG[openssl] --with-sha1=libcrypto,,openssl") + expected_endlines.append("PACKAGECONFIG[nettle] --with-sha1=libnettle,,nettle") + expected_endlines.append("PACKAGECONFIG[gcrypt] --with-sha1=libgcrypt,,libgcrypt") + + self.check_endlines(results, expected_endlines) + + def test_packageconfig_flags_optiins_preferred_only(self): + results = runCmd('../scripts/contrib/list-packageconfig-flags.py -p') + expected_endlines = [] + expected_endlines.append("RECIPE NAME PACKAGECONFIG FLAGS") + expected_endlines.append("xserver-xorg-1.20.3 dri dri2 dri3 gcrypt glamor glx nettle openssl systemd systemd-logind udev unwind xinerama xmlto xshmfence xwayland") + + self.check_endlines(results, expected_endlines) +
Signature · 1 line
2.7.4
Re: [QA][PATCH 1/7] manual tests migrate to selftest
Quoted 5 lines · Armin Kuster
Signed-off-by Armin Kuster <[email protected]>--- meta/lib/oeqa/selftest/cases/oescripts.py | 89 +++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+)
This threw a few errors on the autobuilder:
https://autobuilder.yoctoproject.org/typhoon/#/builders/56/builds/425
One issue is missing pycairo but there seem to be others.
Cheers,
Richard
PATCH 2/7 manual qa/oe-core: remove packageconfig flag tests
In reply to Armin Kuster--- meta/lib/oeqa/manual/oe-core.json | 36 +----------------------------------- 1 file changed, 1 insertion(+), 35 deletions(-)
meta/lib/oeqa/manual/oe-core.json +1 -35
@@ -69,40 +69,6 @@ }, { "test": { - "@alias": "oe-core.scripts.List_all_the_PACKAGECONFIG's_flags", - "author": [ - { - "email": "[email protected]", - "name": "[email protected]" - } - ], - "execution": { - "1": { - "action": " Download the poky source and setup the environment. ", - "expected_results": "" - }, - "2": { - "action": "Run \"../scripts/contrib/list-packageconfig-flags.py\" ", - "expected_results": "In step 2, will list available pkgs which have PACKAGECONFIG flags: \nPACKAGE NAME (or RECIPE NAME) PACKAGECONFIG FLAGS \n============================================================== \nalsa-tools-1.0.26.1 defaultval gtk+ \navahi-ui-0.6.31 defaultval python \nbluez4-4.101 alsa defaultval pie \n" - }, - "3": { - "action": "Run \"../scripts/contrib/list-packageconfig-flags.py -f\" ", - "expected_results": "In step 3, will list available PACKAGECONFIG flags and all affected pkgs \nPACKAGECONFIG FLAG PACKAGE NAMES (or RECIPE NAMES) \n==================================== \n3g connman-1.16 \n \navahi cups-1.6.3 pulseaudio-4.0 \nbeecrypt rpm-5.4.9 rpm-native-5.4.9 \n" - }, - "4": { - "action": "Run \"../scripts/contrib/list-packageconfig-flags.py -a\" ", - "expected_results": "In step 4, will list all pkgs and PACKAGECONFIG information: \n================================================== \ngtk+-2.24.18 \n/home/jiahongxu/yocto/poky/meta/recipes-gnome/gtk+/gtk+_2.24.18.bb \nPACKAGECONFIG x11 \nPACKAGECONFIG[x11] --with-x=yes --with-gdktarget=x11,--with-x=no,${X11DEPENDS} \nxf86-video-intel-2.21.9 \n/home/jiahongxu/yocto/poky/meta/recipes-graphics/xorg-driver/xf86-video-intel_2.21.9.bb \nPACKAGECONFIG None \nPACKAGECONFIG[xvmc] --enable-xvmc,--disable-xvmc,libxvmc \nPACKAGECONFIG[sna] --enable-sna,--disable-sna \n" - }, - "5": { - "action": "Run \"../scripts/contrib/list-packageconfig-flags.py -p\" ", - "expected_results": "In step 5, will list pkgs with preferred version: \nPACKAGE NAME (or RECIPE NAME) PACKAGECONFIG FLAGS \n=================================================== \nalsa-tools-1.0.26.1 defaultval gtk+ \navahi-ui-0.6.31 defaultval python \nbluez4-4.101 alsa defaultval pie \nbluez5-5.7 alsa defaultval obex-profiles \n\n\n\n " - } - }, - "summary": "List_all_the_PACKAGECONFIG's_flags" - } - }, - { - "test": { "@alias": "oe-core.bitbake.Test_bitbake_menuconfig", "author": [ {
@@ -257,4 +223,4 @@ "summary": "test_bitbake_sane_error_for_invalid_layer" } } -] +]
Signature · 1 line
2.7.4
PATCH 3/7 manual qa/bsp-qemu: remove rpm tests already done in runtime
In reply to Armin Kuster--- meta/lib/oeqa/manual/bsp-qemu.json | 60 -------------------------------------- 1 file changed, 60 deletions(-)
meta/lib/oeqa/manual/bsp-qemu.json +0 -60
@@ -91,66 +91,6 @@ }, { "test": { - "@alias": "bsps-qemu.bsps-tools.rpm_-__install_dependency_package", - "author": [ - { - "email": "[email protected]", - "name": "[email protected]" - } - ], - "execution": { - "1": { - "action": "Get a not previously installed RPM package or build one on local machine, which should have run-time dependency.For example, \"mc\" (Midnight Commander, which is a visual file manager) should depend on \"ncurses-terminfo\". \n\n$ bitbake mc \n\n\n", - "expected_results": "" - }, - "2": { - "action": "Copy the package into a system folder (for example /home/root/rpm_packages). \n\n\n", - "expected_results": "" - }, - "3": { - "action": "Run \"rpm -ivh package_name\" and check the output, for example \"rpm -ivh mc.rpm*\" should report the dependency on \"ncurses-terminfo\".\n\n\n\n", - "expected_results": "3 . rpm command should report message when some RPM installation depends on other packages." - } - }, - "summary": "rpm_-__install_dependency_package" - } - }, - { - "test": { - "@alias": "bsps-qemu.bsps-tools.Check_rpm_install/removal_log_file_size(auto)", - "author": [ - { - "email": "[email protected]", - "name": "[email protected]" - } - ], - "execution": { - "1": { - "action": "Get some rpm or other kind of installation packages. \n\n", - "expected_results": "Steps 1- 4 (more than 2.3) \nEach file will occupy around 10MB, and there should be some method to keep rpm log in a small size. (the size of the db of RPMs must not be taking so much space) \nStep 5 (less than or equal to 2.3)\nThe size on /var/lib/rpm/ must keep around 30MB" - }, - "2": { - "action": "After system is up, check the size of log file named as \"log.xxxxxx\" on /var/lib/rpm/log \n\n", - "expected_results": "" - }, - "3": { - "action": "After several install/removal of packages, with either of the install/removal commands (rpm/smart/zypper/dnf install/removal), check again the size of log file. \n\n", - "expected_results": "" - }, - "4": { - "action": "For packages installation, there will be some database files under /var/lib/rpm/, named as \"__db.xxx\" and there will be some log files \nunder /var/lib/rpm/log, named as \"\"log.xxxxxx\"\". \n\nNote: You will only see the log.xxxx on /var/lib/rpm/log mentioned above if the poky version is minor than 2.3.For poky 2.3 or major versions this has been modified and the package RPM4 does not show the logs.xxxx. if major, follow the next step. \n\n", - "expected_results": "" - }, - "5": { - "action": "Repeat steps (1 and 3) and check the size of /var/lib/rpm/ \n\nMore info: https://bugzilla.yoctoproject.org/show_bug.cgi?id=9259", - "expected_results": "" - } - }, - "summary": "Check_rpm_install/removal_log_file_size" - } - }, - { - "test": { "@alias": "bsps-qemu.bsps-runtime.only_one_connmand_in_background(auto)", "author": [ {
Signature · 1 line
2.7.4
PATCH 4/7 manual qa/bsp-qemu: remove KVM enabled which is already done in selftest runqemu
In reply to Armin Kuster--- meta/lib/oeqa/manual/bsp-qemu.json | 30 ------------------------------ 1 file changed, 30 deletions(-)
meta/lib/oeqa/manual/bsp-qemu.json +0 -30
@@ -1,36 +1,6 @@ [ { "test": { - "@alias": "bsps-qemu.bsps-tools.qemu_can_be_started_with_KVM_enabled", - "author": [ - { - "email": "[email protected]", - "name": "[email protected]" - } - ], - "execution": { - "1": { - "action": "Build a kernel with KVM enabled \n\nIn Local.conf add \n\nQEMU_USE_KVM = \"${@ '1' if os.access('/dev/kvm', os.R_OK|os.W_OK) else '0' }\" \n\n ", - "expected_results": "" - }, - "2": { - "action": "Start qemu with option \"kvm\" with runqemu \n a. If you start qemu with kvm failed, maybe it is because host not install kvm and vhost_net module. Follow below link to install them. \n b. vhost_test refer: https://wiki.yoctoproject.org/wiki/Running_an_x86_Yocto_Linux_image_under_QEMU_KVM \n c. kvm refer: https://wiki.yoctoproject.org/wiki/How_to_enable_KVM_for_Poky_qemu", - "expected_results": "" - }, - "3": { - "action": "Check if qemu starts up and if kvm_intel module is used", - "expected_results": "" - }, - "4": { - "action": "If kvm_intel module is not used when starting qemu, it will show 0 in \"Used by\" column when you run \"lsmod | grep kvm_intel\" ", - "expected_results": "KVM enabled with qemu \nExecute \"lsmod | grep kvm_intel\" from your host twice, before and after you \nstart the qemu with kvm option. Before start, the number should be 0, \nafter start, the number should bigger than 0." - } - }, - "summary": "qemu_can_be_started_with_KVM_enabled" - } - }, - { - "test": { "@alias": "bsps-qemu.bsps-tools.Post-installation_logging", "author": [ {
Signature · 1 line
2.7.4
PATCH 5/7 manual/bsp-qemu: drop xserver test
In reply to Armin Kusterxserver testing is done at runtime and selftest via sato image
--- meta/lib/oeqa/manual/bsp-qemu.json | 22 ---------------------- 1 file changed, 22 deletions(-)
meta/lib/oeqa/manual/bsp-qemu.json +0 -22
@@ -91,28 +91,6 @@ }, { "test": { - "@alias": "bsps-qemu.bsps-runtime.X_server_can_start_up_with_runlevel_5_boot", - "author": [ - { - "email": "[email protected]", - "name": "[email protected]" - } - ], - "execution": { - "1": { - "action": "boot up system with default runlevel \n\n", - "expected_results": "X server can start up well and desktop display has no problem . \n\n" - }, - "2": { - "action": "type runlevel at command prompt", - "expected_results": "Output:N 5" - } - }, - "summary": "X_server_can_start_up_with_runlevel_5_boot" - } - }, - { - "test": { "@alias": "bsps-qemu.bsps-runtime.check_bash_in_image", "author": [ {
Signature · 1 line
2.7.4
PATCH 6/7 manual/bsp-qemu: remove connmand test done in runtime test
In reply to Armin Kuster--- meta/lib/oeqa/manual/bsp-qemu.json | 30 ------------------------------ 1 file changed, 30 deletions(-)
meta/lib/oeqa/manual/bsp-qemu.json +0 -30
@@ -61,36 +61,6 @@ }, { "test": { - "@alias": "bsps-qemu.bsps-runtime.only_one_connmand_in_background(auto)", - "author": [ - { - "email": "[email protected]", - "name": "[email protected]" - } - ], - "execution": { - "1": { - "action": "Boot system", - "expected_results": "" - }, - "2": { - "action": "Run \"ps aux |grep connmand\" or \"ps -ef | grep connmand\" or \"ps | grep connmand\"", - "expected_results": "Connmand (connection manager, used to manage internet connections) should be shown as an active process \n\n" - }, - "3": { - "action": "Run command \"connmand\" to try to launch to a second connmand process", - "expected_results": "" - }, - "4": { - "action": "Check, with \"ps\" connmand if a second connmand can be generated ", - "expected_results": "There should be only one connmand process instance in background ." - } - }, - "summary": "only_one_connmand_in_background" - } - }, - { - "test": { "@alias": "bsps-qemu.bsps-runtime.check_bash_in_image", "author": [ {
Signature · 1 line
2.7.4
PATCH 7/7 manual sdk: remove this manual sdk test
In reply to Armin KusterThis is done during nearly all qemu testimage
This seems to test "release" artifacts on the web
--- meta/lib/oeqa/manual/sdk.json | 32 -------------------------------- 1 file changed, 32 deletions(-) delete mode 100644 meta/lib/oeqa/manual/sdk.json
meta/lib/oeqa/manual/sdk.json deleted +0 -32
@@ -1,32 +0,0 @@ -[ - { - "test": { - "@alias": "sdk.sdk_runqemu.test_install_cross_toolchain_can_run_multiple_qemu_for_x86", - "author": [ - { - "email": "[email protected]", - "name": "[email protected]" - } - ], - "execution": { - "1": { - "action": "Prepare kernel, rootfs tar.bz2 image, and qemu configuration \n \ta. Download kernel, rootfs tar.bz2 image and qemu configuration from public autobuilder webpage \n \tb. Goto https://autobuilder.yocto.io/pub/releases/<target_release>/machines/qemu/qemux86/ \n \tc. Download \n \t \ti. rootfs tar.bz2: core-image-sato-sdk-qemux86.tar.bz2 \n \t\tii. kernel: bzImage-qemux86.bin \n \t\tiii. qemu configuration: core-image-sato-sdk-qemux86.qemuboot.conf ", - "expected_results": "Download completes successfully." - }, - "2": { - "action": "Download & install toolchain tarball matching your host from public autobuilder \n \ta. Goto https://autobuilder.yocto.io/pub/releases/<target_release>/toolchain/x86_64/ \n \tb. Download poky-glibc-x86_64-core-image-sato-<type-arch>-toolchain-<release-version>.sh \n \tc. Run command: poky-glibc-x86_64-core-image-sato-<type-arch>-toolchain-<release-version>.sh \n \td. After installation toolchain Run source command : source /toolchain-installed-path/environment-setup-<architecture name>-poky-linux", - "expected_results": "Toolchain gets installed successfully." - }, - "3": { - "action": "Extract rootfs twice into two images \n \ta. Run 2 commands below: \n runqemu-extract-sdk core-image-sato-sdk-qemux86.tar.bz2 qemux86_rootfs_image1 \n runqemu-extract-sdk core-image-sato-sdk-qemux86.tar.bz2 qemux86_rootfs_image2", - "expected_results": "Both images build successfully." - }, - "4": { - "action": " From the 2 terminals, start qemu to boot up both two images \n \ta. Run 2 commands below: \n runqemu <kernel-name> core-image-sato-sdk-qemux86.qemuboot.conf qemux86_rootfs_image1 \n runqemu <kernel-name> core-image-sato-sdk-qemux86.qemuboot.conf qemux86_rootfs_image2 ", - "expected_results": "Expect both qemu to boot up successfully." - } - }, - "summary": "test_install_cross_toolchain_can_run_multiple_qemu_for_x86" - } - } -]
Signature · 1 line
2.7.4
Re: [QA][PATCH 7/7] manual sdk: remove this manual sdk test
Quoted 2 lines · Armin Kuster
This is done during nearly all qemu testimage
This seems to test "release" artifacts on the web
This does two things, it tests release artefacts and it also tests we can run two qemus at once.
Whether those are useful things to test I'm not sure about but we should make a concious statement in the commit message if we are deciding not to test that.
Cheers,
Richard
Re: [QA][PATCH 7/7] manual sdk: remove this manual sdk test
Quoted 4 lines · Richard Purdie
Quoted 2 lines · Armin Kuster
This is done during nearly all qemu testimage
This seems to test "release" artifacts on the webThis does two things, it tests release artefacts
Correct. I think this is more of a Release process step and I think it can be automated.
Quoted 2 lines
and it also tests we
can run two qemus at once.
I would think this could be automated as well.
Quoted 4 lines
Whether those are useful things to test I'm not sure about but we should make a concious statement in the commit message if we are deciding not to test that.
sometimes patches are way to get a conversion started and here we are : )
- armin
Quoted 7 lines
Cheers,
Richard