Add dumps when qemurunner fails
cover Add dumps when qemurunner fails
From: Mariano Lopez <[email protected]>
These patches add dumps when qemurunner ends unexpectedly and make runexported test compatible with HostDumper.
The previous version of this patch was more intrusive with qemurunner, now the instance of HostDumper is created inside qemurunner with parameters sent to the constructor.
Also, now the Dumper constructors doesn't require the datastore, It is needed to send the commands and the dump dir as strings.
This also makes compatible the HostDumper with runexported test.
The following changes since commit 22afc047dd1b8f83e56c4a9862710776c509e3c5:
oe-selftest: test devtool build-image plugin (2015-08-31 11:37:14 +0100)
are available in the git repository at:
git://git.yoctoproject.org/poky-contrib mariano/bug8118v8
http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=mariano/bug8118v8
Mariano Lopez (2):
qemurunner: Added host dumps when there are errors
runexported: Make it compatible with host dump
meta/classes/testimage.bbclass | 6 +++++- meta/lib/oeqa/oetest.py | 13 ++++++------- meta/lib/oeqa/runexported.py | 8 +++++++- meta/lib/oeqa/targetcontrol.py | 9 +++++++-- meta/lib/oeqa/utils/dump.py | 28 +++++++++------------------- meta/lib/oeqa/utils/qemurunner.py | 15 ++++++++++++++- 6 files changed, 48 insertions(+), 31 deletions(-)
Signature · 1 line
1.8.4.5
PATCH v2 1/2 qemurunner: Added host dumps when there are errors
From: Mariano Lopez <[email protected]>
This adds an instance of HostDumper to qemurunner, with this instance now is possible to get dumps from the host when there is an error.
This adds dump points in the next cases:
- runqemu exits before seeing qemu pid
- Fail to get qemu process arguments
- Not reach login banner before timeout
- qemu pid never appears
This also modifies the constructors of BaseDumper, HostDumper and TargetDumper, they don't require the datastore anymore, but the feature to replace datastore variables has been lost (never used)
[YOCTO #8118]
--- meta/lib/oeqa/oetest.py | 10 +++++----- meta/lib/oeqa/targetcontrol.py | 9 +++++++-- meta/lib/oeqa/utils/dump.py | 28 +++++++++------------------- meta/lib/oeqa/utils/qemurunner.py | 15 ++++++++++++++- 4 files changed, 35 insertions(+), 27 deletions(-)
meta/lib/oeqa/oetest.py +5 -5
@@ -123,14 +123,14 @@ class oeRuntimeTest(oeTest): # If a test fails or there is an exception if not exc_info() == (None, None, None): exc_clear() - self.tc.host_dumper.create_dir(self._testMethodName) - self.tc.host_dumper.dump_host() - #Only QemuTarget has a serial console + #Only dump for QemuTarget if (isinstance(self.target, QemuTarget)): + self.tc.host_dumper.create_dir(self._testMethodName) + self.tc.host_dumper.dump_host() self.target.target_dumper.dump_target( self.tc.host_dumper.dump_dir) - print ("%s dump data stored in %s" % (self._testMethodName, - self.tc.host_dumper.dump_dir)) + print ("%s dump data stored in %s" % (self._testMethodName, + self.tc.host_dumper.dump_dir)) #TODO: use package_manager.py to install packages on any type of image def install_packages(self, packagelist):
meta/lib/oeqa/targetcontrol.py +7 -2
@@ -124,6 +124,9 @@ class QemuTarget(BaseTarget): self.origrootfs = os.path.join(d.getVar("DEPLOY_DIR_IMAGE", True), d.getVar("IMAGE_LINK_NAME", True) + '.' + self.image_fstype) self.rootfs = os.path.join(self.testdir, d.getVar("IMAGE_LINK_NAME", True) + '-testimage.' + self.image_fstype) self.kernel = os.path.join(d.getVar("DEPLOY_DIR_IMAGE", True), d.getVar("KERNEL_IMAGETYPE", False) + '-' + d.getVar('MACHINE', False) + '.bin') + dump_target_cmds = d.getVar("testimage_dump_target", True) + dump_host_cmds = d.getVar("testimage_dump_host", True) + dump_dir = d.getVar("TESTIMAGE_DUMP_DIR", True) # Log QemuRunner log output to a file import oe.path
@@ -151,9 +154,11 @@ class QemuTarget(BaseTarget): deploy_dir_image = d.getVar("DEPLOY_DIR_IMAGE", True), display = d.getVar("BB_ORIGENV", False).getVar("DISPLAY", True), logfile = self.qemulog, - boottime = int(d.getVar("TEST_QEMUBOOT_TIMEOUT", True))) + boottime = int(d.getVar("TEST_QEMUBOOT_TIMEOUT", True)), + dump_dir = dump_dir, + dump_host_cmds = d.getVar("testimage_dump_host", True)) - self.target_dumper = TargetDumper(d, self.runner) + self.target_dumper = TargetDumper(dump_target_cmds, dump_dir, self.runner) def deploy(self): try:
meta/lib/oeqa/utils/dump.py +9 -19
@@ -6,30 +6,22 @@ import itertools from commands import runCmd def get_host_dumper(d): - return HostDumper(d) + cmds = d.getVar("testimage_dump_host", True) + parent_dir = d.getVar("TESTIMAGE_DUMP_DIR", True) + return HostDumper(cmds, parent_dir) class BaseDumper(object): - def __init__(self, d, cmds): + def __init__(self, cmds, parent_dir): self.cmds = [] - self.parent_dir = d.getVar("TESTIMAGE_DUMP_DIR", True) + self.parent_dir = parent_dir if not cmds: return for cmd in cmds.split('\n'): cmd = cmd.lstrip() if not cmd or cmd[0] == '#': continue - # Replae variables from the datastore - while True: - index_start = cmd.find("${") - if index_start == -1: - break - index_start += 2 - index_end = cmd.find("}", index_start) - var = cmd[index_start:index_end] - value = d.getVar(var, True) - cmd = cmd.replace("${%s}" % var, value) self.cmds.append(cmd) def create_dir(self, dir_suffix):
@@ -62,9 +54,8 @@ class BaseDumper(object): class HostDumper(BaseDumper): - def __init__(self, d): - host_cmds = d.getVar("testimage_dump_host", True) - super(HostDumper, self).__init__(d, host_cmds) + def __init__(self, cmds, parent_dir): + super(HostDumper, self).__init__(cmds, parent_dir) def dump_host(self, dump_dir=""): if dump_dir:
@@ -76,9 +67,8 @@ class HostDumper(BaseDumper): class TargetDumper(BaseDumper): - def __init__(self, d, qemurunner): - target_cmds = d.getVar("testimage_dump_target", True) - super(TargetDumper, self).__init__(d, target_cmds) + def __init__(self, cmds, parent_dir, qemurunner): + super(TargetDumper, self).__init__(cmds, parent_dir) self.runner = qemurunner def dump_target(self, dump_dir=""):
meta/lib/oeqa/utils/qemurunner.py +14 -1
@@ -14,13 +14,14 @@ import socket import select import errno import threading +from oeqa.utils.dump import HostDumper import logging logger = logging.getLogger("BitBake.QemuRunner") class QemuRunner: - def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, boottime): + def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, boottime, dump_dir, dump_host_cmds): # Popen object for runqemu self.runqemu = None
@@ -42,6 +43,7 @@ class QemuRunner: self.thread = None self.runqemutime = 60 + self.host_dumper = HostDumper(dump_host_cmds, dump_dir) def create_socket(self): try:
@@ -118,6 +120,7 @@ class QemuRunner: if self.runqemu.returncode: # No point waiting any longer logger.info('runqemu exited with code %d' % self.runqemu.returncode) + self._dump_host() self.stop() logger.info("Output from runqemu:\n%s" % getOutput(output)) return False
@@ -137,6 +140,7 @@ class QemuRunner: self.server_ip = ips[1] except IndexError, ValueError: logger.info("Couldn't get ip from qemu process arguments! Here is the qemu command line used:\n%s\nand output from runqemu:\n%s" % (cmdline, getOutput(output))) + self._dump_host() self.stop() return False logger.info("qemu cmdline used:\n{}".format(cmdline))
@@ -189,6 +193,7 @@ class QemuRunner: lines = "\n".join(bootlog.splitlines()[-25:]) logger.info("Last 25 lines of text:\n%s" % lines) logger.info("Check full boot log: %s" % self.logfile) + self._dump_host() self.stop() return False
@@ -202,6 +207,7 @@ class QemuRunner: else: logger.info("Qemu pid didn't appeared in %s seconds" % self.runqemutime) + self._dump_host() self.stop() logger.info("Output from runqemu:\n%s" % getOutput(output)) return False
@@ -334,6 +340,13 @@ class QemuRunner: status = 1 return (status, str(data)) + + def _dump_host(self): + self.host_dumper.create_dir("qemu") + logger.error("Qemu ended unexpectedly, dump data from host" + " is in %s" % self.host_dumper.dump_dir) + self.host_dumper.dump_host() + # This class is for reading data from a socket and passing it to logfunc # to be processed. It's completely event driven and has a straightforward # event loop. The mechanism for stopping the thread is a simple pipe which
Signature · 1 line
1.8.4.5
PATCH v2 2/2 runexported: Make it compatible with host dump
In reply to [email protected]From: Mariano Lopez <[email protected]>
Currently it is not possible to run a exported test, but this patch will allow to use the HosDumper class when running a exported test, otherwise the HostDumper class will break runexpored test.
[YOCTO #8118]
--- meta/classes/testimage.bbclass | 6 +++++- meta/lib/oeqa/oetest.py | 3 +-- meta/lib/oeqa/runexported.py | 8 +++++++- 3 files changed, 13 insertions(+), 4 deletions(-)
meta/classes/testimage.bbclass +5 -1
@@ -173,9 +173,10 @@ def exportTests(d,tc): savedata = {} savedata["d"] = {} savedata["target"] = {} + savedata["host_dumper"] = {} for key in tc.__dict__: # special cases - if key != "d" and key != "target": + if key != "d" and key != "target" and key != "host_dumper": savedata[key] = getattr(tc, key) savedata["target"]["ip"] = tc.target.ip or d.getVar("TEST_TARGET_IP", True) savedata["target"]["server_ip"] = tc.target.server_ip or d.getVar("TEST_SERVER_IP", True)
@@ -189,6 +190,9 @@ def exportTests(d,tc): # we don't care about those anyway pass + savedata["host_dumper"]["parent_dir"] = tc.host_dumper.parent_dir + savedata["host_dumper"]["cmds"] = tc.host_dumper.cmds + with open(os.path.join(exportpath, "testdata.json"), "w") as f: json.dump(savedata, f, skipkeys=True, indent=4, sort_keys=True)
meta/lib/oeqa/oetest.py +1 -2
@@ -13,7 +13,6 @@ import inspect import subprocess import bb from oeqa.utils.decorators import LogResults -from oeqa.targetcontrol import QemuTarget from sys import exc_info, exc_clear def loadTests(tc, type="runtime"):
@@ -124,7 +123,7 @@ class oeRuntimeTest(oeTest): if not exc_info() == (None, None, None): exc_clear() #Only dump for QemuTarget - if (isinstance(self.target, QemuTarget)): + if (type(self.target).__name__ == "QemuTarget"): self.tc.host_dumper.create_dir(self._testMethodName) self.tc.host_dumper.dump_host() self.target.target_dumper.dump_target(
meta/lib/oeqa/runexported.py +7 -1
@@ -32,6 +32,7 @@ sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "oeqa")) from oeqa.oetest import runTests from oeqa.utils.sshcontrol import SSHControl +from oeqa.utils.dump import get_host_dumper # this isn't pretty but we need a fake target object # for running the tests externally as we don't care
@@ -118,11 +119,16 @@ def main(): for key in loaded["target"].keys(): setattr(target, key, loaded["target"][key]) + host_dumper = get_host_dumper(d) + host_dumper.parent_dir = loaded["host_dumper"]["parent_dir"] + host_dumper.cmds = loaded["host_dumper"]["cmds"] + tc = TestContext() setattr(tc, "d", d) setattr(tc, "target", target) + setattr(tc, "host_dumper", host_dumper) for key in loaded.keys(): - if key != "d" and key != "target": + if key != "d" and key != "target" and key != "host_dumper": setattr(tc, key, loaded[key]) target.exportStart()
Signature · 1 line
1.8.4.5