Skip to contentSkip to conversation

Run tests on a remote machine

PATCH New Stefan Stanacar 1 participant 3 messages Nov 26, 2013

lore
Share & tools
lore
#0 lore raw

v2 cover Run tests on a remote machine

Hello,

These patches add some abstraction on how we control a target, so that we can also run the tests on real hardware.
The "backend" is controlled by setting TEST_TARGET to "qemu" or "simpleremote". (a very simple remote implementation which does not deploy the image but runs the tests on a remote machine via ssh to a specified IP address).

Cheers,
Stefan

Changed in v2:
- yay, typos: FunFailed -> FuncFailed (there's a gremlin in my computer :))
- small changes in commit messages
- one try/except block in targetcontrol.py was too big, scale it down.

The following changes since commit 381cd842caa3db5eabf8973305b6979014d9d0a8:

linux-firmware: add missing linux-firmware-iwlwifi-7260-7 package (2013-11-25 16:14:48 +0000)

are available in the git repository at:

git://git.yoctoproject.org/poky-contrib stefans/hwtest3
http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=stefans/hwtest3

Stefan Stanacar (2):
lib/oeqa: targetcontrol.py: add abstraction for running tests on
different targets
testimage: use the new targetcontrol.py module for running tests

 meta/classes/testimage-auto.bbclass |   3 +-
 meta/classes/testimage.bbclass      | 104 ++++++++++-----------------
 meta/lib/oeqa/oetest.py             |  20 +++---
 meta/lib/oeqa/runtime/ping.py       |   2 +-
 meta/lib/oeqa/runtime/smart.py      |   4 +-
 meta/lib/oeqa/targetcontrol.py      | 137 ++++++++++++++++++++++++++++++++++++
 meta/lib/oeqa/utils/qemurunner.py   |  50 +++++++------
 meta/lib/oeqa/utils/sshcontrol.py   |  20 ++----
 8 files changed, 224 insertions(+), 116 deletions(-)
 create mode 100644 meta/lib/oeqa/targetcontrol.py
Signature · 1 line
1.8.3.1
#1 lore raw

PATCH v2 1/2 lib/oeqa: targetcontrol.py: add abstraction for running tests on different targets

Add a new module which abstracts the target object used by testimage.bbclass

The purpose of this module is to move the deployment of a target from testimage.bbclass, basically abstracting different implementations of how we setup a target and how it runs commands. It allows to select one implementation or another by setting TEST_TARGET (currently to: "qemu" and "simpleremote").

QemuTarget is used to start a qemu instance (as it's currently done in testimage.bbclass) SimpleRemoteTarget is meant for a remote machine (by setting TEST_TARGET_IP) that's already up and running with network and ssh.
Simply put, it opens the door for running the tests on different types of targets by adding new classes (maybe qemu-nfsroot or remote-special etc.). One could also override BaseTarget which currently uses the existing SSHControl module and add a serial implementation.

[ YOCTO #5554 ]

---
 meta/lib/oeqa/targetcontrol.py | 137 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 137 insertions(+)
 create mode 100644 meta/lib/oeqa/targetcontrol.py
meta/lib/oeqa/targetcontrol.py new file +137 -0
new file mode 100644
index 0000000..dee38ec
@@ -0,0 +1,137 @@
+# Copyright (C) 2013 Intel Corporation
+#
+# Released under the MIT license (see COPYING.MIT)
+
+# This module is used by testimage.bbclass for setting up and controlling a target machine.
+
+import os
+import shutil
+import subprocess
+import bb
+
+from oeqa.utils.sshcontrol import SSHControl
+from oeqa.utils.qemurunner import QemuRunner
+
+
+def get_target_controller(d):
+    if d.getVar("TEST_TARGET", True) == "qemu":
+        return QemuTarget(d)
+    elif d.getVar("TEST_TARGET", True) == "simpleremote":
+        return SimpleRemoteTarget(d)
+    else:
+        bb.fatal("Please set a valid TEST_TARGET")
+
+
+class BaseTarget(object):
+
+    def __init__(self, d):
+        self.connection = None
+        self.ip = None
+        self.server_ip = None
+        self.datetime = d.getVar('DATETIME', True)
+        self.testdir = d.getVar("TEST_LOG_DIR", True)
+        self.pn = d.getVar("PN", True)
+
+    def deploy(self):
+
+        self.sshlog = os.path.join(self.testdir, "ssh_target_log.%s" % self.datetime)
+        sshloglink = os.path.join(self.testdir, "ssh_target_log")
+        if os.path.islink(sshloglink):
+            os.unlink(sshloglink)
+        os.symlink(self.sshlog, sshloglink)
+        bb.note("SSH log file: %s" %  self.sshlog)
+
+    def run(self, cmd, timeout=None):
+        return self.connection.run(cmd, timeout)
+
+    def copy_to(self, localpath, remotepath):
+        return self.connection.copy_to(localpath, remotepath)
+
+    def copy_from(self, remotepath, localpath):
+        return self.connection.copy_from(remotepath, localpath)
+
+
+
+class QemuTarget(BaseTarget):
+
+    def __init__(self, d):
+
+        super(QemuTarget, self).__init__(d)
+
+        self.qemulog = os.path.join(self.testdir, "qemu_boot_log.%s" % self.datetime)
+        self.origrootfs = os.path.join(d.getVar("DEPLOY_DIR_IMAGE", True),  d.getVar("IMAGE_LINK_NAME", True) + '.ext3')
+        self.rootfs = os.path.join(self.testdir, d.getVar("IMAGE_LINK_NAME", True) + '-testimage.ext3')
+
+        self.runner = QemuRunner(machine=d.getVar("MACHINE", True),
+                        rootfs=self.rootfs,
+                        tmpdir = d.getVar("TMPDIR", True),
+                        deploy_dir_image = d.getVar("DEPLOY_DIR_IMAGE", True),
+                        display = d.getVar("BB_ORIGENV", False).getVar("DISPLAY", True),
+                        logfile = self.qemulog,
+                        boottime = int(d.getVar("TEST_QEMUBOOT_TIMEOUT", True)))
+
+    def deploy(self):
+        try:
+            shutil.copyfile(self.origrootfs, self.rootfs)
+        except Exception as e:
+            bb.fatal("Error copying rootfs: %s" % e)
+
+        qemuloglink = os.path.join(self.testdir, "qemu_boot_log")
+        if os.path.islink(qemuloglink):
+            os.unlink(qemuloglink)
+        os.symlink(self.qemulog, qemuloglink)
+
+        bb.note("rootfs file: %s" %  self.rootfs)
+        bb.note("Qemu log file: %s" % self.qemulog)
+        super(QemuTarget, self).deploy()
+
+    def start(self, params=None):
+        if self.runner.start(params):
+            self.ip = self.runner.ip
+            self.server_ip = self.runner.server_ip
+            self.connection = SSHControl(ip=self.ip, logfile=self.sshlog)
+        else:
+            raise bb.build.FuncFailed("%s - FAILED to start qemu - check the task log and the boot log" % self.pn)
+
+    def stop(self):
+        self.runner.stop()
+        self.connection = None
+        self.ip = None
+        self.server_ip = None
+
+    def restart(self, params=None):
+        if self.runner.restart(params):
+            self.ip = self.runner.ip
+            self.server_ip = self.runner.server_ip
+            self.connection = SSHControl(ip=self.ip, logfile=self.sshlog)
+        else:
+            raise bb.build.FuncFailed("%s - FAILED to re-start qemu - check the task log and the boot log" % self.pn)
+
+
+class SimpleRemoteTarget(BaseTarget):
+
+    def __init__(self, d):
+        super(SimpleRemoteTarget, self).__init__(d)
+        self.ip = d.getVar("TEST_TARGET_IP", True) or bb.fatal('Please set TEST_TARGET_IP with the IP address of the machine you want to run the tests on.')
+        bb.note("Target IP: %s" % self.ip)
+        self.server_ip = d.getVar("TEST_SERVER_IP", True)
+        if not self.server_ip:
+            try:
+                self.server_ip = subprocess.check_output(['ip', 'route', 'get', self.ip ]).split()[6]
+            except Exception as e:
+                bb.fatal("Failed to determine the host IP address (alternatively you can set TEST_SERVER_IP with the IP address of this machine): %s" % e)
+        bb.note("Server IP: %s" % self.server_ip)
+
+    def deploy(self):
+        super(SimpleRemoteTarget, self).deploy()
+
+    def start(self, params=None):
+        self.connection = SSHControl(self.ip, logfile=self.sshlog)
+
+    def stop(self):
+        self.connection = None
+        self.ip = None
+        self.server_ip = None
+
+    def restart(self, params=None):
+        pass
Signature · 1 line
1.8.3.1
#2 lore raw

PATCH v2 2/2 testimage: use the new targetcontrol.py module for running tests

In reply to Stefan Stanacar

This patch makes the necessary changes for using the targetcontrol.py module so that one can run the same tests on a qemu instance or a remote machine based on the value of TEST_TARGET variable: "qemu" or "simpleremote". The default value is "qemu" which starts a qemu instance and it's the with what we currently have.

With "simpleremote", the remote machine must be up with network and ssh and you need to set TEST_TARGET_IP with the IP address of the remote machine (it can still be a qemu instance that was manually started).

Basically testimage.bbclass now does something along the lines of:
- load tests -> deploy (prepare) / start target -> run tests.
There were a couple of changes necessary for tests and
also some cleanups/renames that were needed to adjust this change. (use
ip everywhere when refering to target and server_ip when refering to host/build machine) Also two unnecessary and unsed methods were dropped from sshcontrol.

[ YOCTO #5554 ]

---
 meta/classes/testimage-auto.bbclass |   3 +-
 meta/classes/testimage.bbclass      | 104 +++++++++++++-----------------------
 meta/lib/oeqa/oetest.py             |  20 +++----
 meta/lib/oeqa/runtime/ping.py       |   2 +-
 meta/lib/oeqa/runtime/smart.py      |   4 +-
 meta/lib/oeqa/utils/qemurunner.py   |  50 ++++++++++-------
 meta/lib/oeqa/utils/sshcontrol.py   |  20 +++----
 7 files changed, 87 insertions(+), 116 deletions(-)
meta/classes/testimage-auto.bbclass +1 -2
index 3d0e289..a5b8f7f 100644
@@ -19,5 +19,4 @@ python do_testimage_auto() {
     testimage_main(d)
 }
 addtask testimage_auto before do_build after do_rootfs
-do_testimage_auto[depends] += "qemu-native:do_populate_sysroot"
-do_testimage_auto[depends] += "qemu-helper-native:do_populate_sysroot"
+do_testimage_auto[depends] += "${TESTIMAGEDEPENDS}"
meta/classes/testimage.bbclass +36 -68

145 lines not on this page · raw message

meta/lib/oeqa/oetest.py +10 -10
index 3bb3589..38f9aef 100644
@@ -14,7 +14,7 @@ import bb
 from oeqa.utils.sshcontrol import SSHControl
 
 
-def runTests(tc):
+def loadTests(tc):
 
     # set the context object passed from the test class
     setattr(oeTest, "tc", tc)
@@ -24,12 +24,16 @@ def runTests(tc):
     suite = unittest.TestSuite()
     testloader = unittest.TestLoader()
     testloader.sortTestMethodsUsing = None
-    runner = unittest.TextTestRunner(verbosity=2)
+    suite = testloader.loadTestsFromNames(tc.testslist)
+
+    return suite
+
+def runTests(tc):
 
+    suite = loadTests(tc)
     bb.note("Test modules  %s" % tc.testslist)
-    suite = testloader.loadTestsFromNames(tc.testslist)
     bb.note("Found %s tests" % suite.countTestCases())
-
+    runner = unittest.TextTestRunner(verbosity=2)
     result = runner.run(suite)
 
     return result
@@ -87,11 +91,7 @@ class oeRuntimeTest(oeTest):
 
     @classmethod
     def restartTarget(self,params=None):
-
-        if oeRuntimeTest.tc.qemu.restart(params):
-            oeRuntimeTest.tc.target.host = oeRuntimeTest.tc.qemu.ip
-        else:
-            raise Exception("Restarting target failed")
+        oeRuntimeTest.tc.target.restart(params)
 
 
 def getmodule(pos=2):
@@ -109,7 +109,7 @@ def skipModule(reason, pos=2):
     else:
         raise Exception("\nTest %s wants to be skipped.\nReason is: %s" \
                 "\nTest was required in TEST_SUITES, so either the condition for skipping is wrong" \
-                "\nor the image really doesn't have the requred feature/package when it should." % (modname, reason))
+                "\nor the image really doesn't have the required feature/package when it should." % (modname, reason))
 
 def skipModuleIf(cond, reason):
 
meta/lib/oeqa/runtime/ping.py +1 -1
index 0d028f9..a73c724 100644
@@ -11,7 +11,7 @@ class PingTest(oeRuntimeTest):
         count = 0
         endtime = time.time() + 60
         while count < 5 and time.time() < endtime:
-            proc = subprocess.Popen("ping -c 1 %s" % oeRuntimeTest.tc.qemu.ip, shell=True, stdout=subprocess.PIPE)
+            proc = subprocess.Popen("ping -c 1 %s" % self.target.ip, shell=True, stdout=subprocess.PIPE)
             output += proc.communicate()[0]
             if proc.poll() == 0:
                 count += 1
meta/lib/oeqa/runtime/smart.py +2 -2
index c3fdf7d..7ef4b0e 100644
@@ -46,7 +46,7 @@ class SmartRepoTest(SmartTest):
 
     @classmethod
     def setUpClass(self):
-        self.repo_server = HTTPService(oeRuntimeTest.tc.d.getVar('DEPLOY_DIR', True), oeRuntimeTest.tc.qemu.host_ip)
+        self.repo_server = HTTPService(oeRuntimeTest.tc.d.getVar('DEPLOY_DIR', True), oeRuntimeTest.tc.target.server_ip)
         self.repo_server.start()
 
     @classmethod
@@ -58,7 +58,7 @@ class SmartRepoTest(SmartTest):
 
     def test_smart_channel_add(self):
         image_pkgtype = self.tc.d.getVar('IMAGE_PKGTYPE', True)
-        deploy_url = 'http://%s:%s/%s' %(self.tc.qemu.host_ip, self.repo_server.port, image_pkgtype)
+        deploy_url = 'http://%s:%s/%s' %(self.target.server_ip, self.repo_server.port, image_pkgtype)
         pkgarchs = self.tc.d.getVar('PACKAGE_ARCHS', True)
         for arch in os.listdir('%s/%s' % (self.repo_server.root_dir, image_pkgtype)):
             if arch in pkgarchs:
meta/lib/oeqa/utils/qemurunner.py +30 -20

103 lines not on this page · raw message

meta/lib/oeqa/utils/sshcontrol.py +7 -13
index 07257b8..a0dcf02 100644
@@ -13,8 +13,8 @@ import select
 
 class SSHControl(object):
 
-    def __init__(self, host=None, timeout=300, logfile=None):
-        self.host = host
+    def __init__(self, ip=None, timeout=300, logfile=None):
+        self.ip = ip
         self.timeout = timeout
         self._starttime = None
         self._out = ''
@@ -35,7 +35,7 @@ class SSHControl(object):
     def _internal_run(self, cmd):
         # We need this for a proper PATH
         cmd = ". /etc/profile; " + cmd
-        command = self.ssh + [self.host, cmd]
+        command = self.ssh + [self.ip, cmd]
         self.log("[Running]$ %s" % " ".join(command))
         self._starttime = time.time()
         # ssh hangs without os.setsid
@@ -48,10 +48,10 @@ class SSHControl(object):
         if time is 0 will let cmd run until it finishes.
         Time can be passed to here or can be set per class instance."""
 
-        if self.host:
+        if self.ip:
             sshconn = self._internal_run(cmd)
         else:
-            raise Exception("Remote IP/host hasn't been set, I can't run ssh without one.")
+            raise Exception("Remote IP hasn't been set, I can't run ssh without one.")
 
         # run the command forever
         if timeout == 0:
@@ -108,15 +108,9 @@ class SSHControl(object):
         return (ret, out)
 
     def copy_to(self, localpath, remotepath):
-        actualcmd = [localpath, 'root@%s:%s' % (self.host, remotepath)]
+        actualcmd = [localpath, 'root@%s:%s' % (self.ip, remotepath)]
         return self._internal_scp(actualcmd)
 
     def copy_from(self, remotepath, localpath):
-        actualcmd = ['root@%s:%s' % (self.host, remotepath), localpath]
+        actualcmd = ['root@%s:%s' % (self.ip, remotepath), localpath]
         return self._internal_scp(actualcmd)
-
-    def get_status(self):
-        return self._ret
-
-    def get_output(self):
-        return self._out
Signature · 1 line
1.8.3.1