Enable QMP Dumping for testimage
cover Enable QMP Dumping for testimage
This is the second pass at enabling getting debug information from QEMU via the Qemu Machine Protocol interface. The Qemu source provides a qmp.py module which I current install in tmp/hosttools as a known location. I am sure there are various possible locations.
The initial commands that I issue to qmp with a failure is detected
is: query-status and query-block. The output goes into formated json files as follows:
tmp/log/runtime-hostdump/202101061054_qmp
├── qmp_00_query-block ├── qmp_00_query-status ├── qmp_01_query-block └── qmp_01_query-status
Comments welcome, I tested this by calling the monitor_dump code directly in the target/ssh.py code instead of waiting for a given failure.
Sau!
Saul Wold (2):
qemu-system-native: install qmp python module
qemurunner: Add support for qmp commands
meta/classes/testimage.bbclass | 6 ++++ meta/lib/oeqa/core/target/qemu.py | 6 ++++ meta/lib/oeqa/core/target/ssh.py | 17 +++++++++- meta/lib/oeqa/targetcontrol.py | 3 ++ meta/lib/oeqa/utils/dump.py | 31 +++++++++++++++++-- meta/lib/oeqa/utils/qemurunner.py | 29 ++++++++++++++++- .../qemu/qemu-system-native_5.1.0.bb | 4 +++ 7 files changed, 91 insertions(+), 5 deletions(-)
Signature · 1 line
2.25.1
RFC PATCH 1/2 qemu-system-native: install qmp python module
The qmp python module supports the Qemu Machine Protocol [0]. This module needs to be installed in a known location so the qemurunner python script can find the qmp module.
hosttools seemed like a logical static location, for reference the usage in qemurunner is:
sys.path.append('./tmp/hosttools')
import qmp
[0] https://github.com/qemu/qemu/blob/master/docs/interop/qmp-spec.txt
--- meta/recipes-devtools/qemu/qemu-system-native_5.1.0.bb | 4 ++++ 1 file changed, 4 insertions(+)
meta/recipes-devtools/qemu/qemu-system-native_5.1.0.bb +4 -0
@@ -2,6 +2,7 @@ BPN = "qemu" require qemu-native.inc +inherit python3-dir # As some of the files installed by qemu-native and qemu-system-native # are the same, we depend on qemu-native to get the full installation set # and avoid file clashes
@@ -23,4 +24,7 @@ do_install_append() { rm -f ${D}${datadir}/qemu/trace-events-all rm -rf ${D}${datadir}/qemu/keymaps rm -rf ${D}${datadir}/icons/ + + # Install qmp.py to be used with testimage + install -D ${S}/python/qemu/qmp.py ${HOSTTOOLS_DIR}/qmp.py }
Signature · 1 line
2.25.1
RFC PATCH 2/2 qemurunner: Add support for qmp commands
In reply to Saul WoldThis adds support for the Qemu Machine Protocol [0] extending the current dump process for Host and Target. The commands are added in the testimage.bbclass.
Currently, we setup qemu to stall until qmp gets connected and sends the initialization and continue commands, this works correctly.
With this version, the monitor_dumper is created in OEQemuTarget but then set in OESSHTarget as that's where we get the SSH failure happens. Python's @property is used to create a setter/getter type of setup in OESSHTarget to get overridden by OEQemuTarget.
By default the data is currently dumped to files for each command in TMPDIR/log/runtime-hostdump/<date>_qmp/unknown_<seq>_qemu_monitor as this is the naming convenstion in the dump.py code.
We use the qmp.py from qemu, which needs to get installed in a known static location.
[0] https://github.com/qemu/qemu/blob/master/docs/interop/qmp-spec.txt
--- meta/classes/testimage.bbclass | 6 ++++++ meta/lib/oeqa/core/target/qemu.py | 6 ++++++ meta/lib/oeqa/core/target/ssh.py | 17 ++++++++++++++++- meta/lib/oeqa/targetcontrol.py | 3 +++ meta/lib/oeqa/utils/dump.py | 31 ++++++++++++++++++++++++++++--- meta/lib/oeqa/utils/qemurunner.py | 29 ++++++++++++++++++++++++++++- 6 files changed, 87 insertions(+), 5 deletions(-)
meta/classes/testimage.bbclass +6 -0
@@ -127,6 +127,11 @@ testimage_dump_host () { netstat -an } +testimage_dump_monitor () { + query-status + query-block +} + python do_testimage() { testimage_main(d) }
@@ -319,6 +324,7 @@ def testimage_main(d): target_kwargs['powercontrol_extra_args'] = d.getVar("TEST_POWERCONTROL_EXTRA_ARGS") or "" target_kwargs['serialcontrol_cmd'] = d.getVar("TEST_SERIALCONTROL_CMD") or None target_kwargs['serialcontrol_extra_args'] = d.getVar("TEST_SERIALCONTROL_EXTRA_ARGS") or "" + target_kwargs['testimage_dump_monitor'] = d.getVar("testimage_dump_monitor") or "" target_kwargs['testimage_dump_target'] = d.getVar("testimage_dump_target") or "" def export_ssh_agent(d):
meta/lib/oeqa/core/target/qemu.py +6 -0
@@ -12,6 +12,7 @@ from collections import defaultdict from .ssh import OESSHTarget from oeqa.utils.qemurunner import QemuRunner +from oeqa.utils.dump import MonitorDumper from oeqa.utils.dump import TargetDumper supported_fstypes = ['ext3', 'ext4', 'cpio.gz', 'wic']
@@ -43,6 +44,11 @@ class OEQemuTarget(OESSHTarget): dump_host_cmds=dump_host_cmds, logger=logger, serial_ports=serial_ports, boot_patterns = boot_patterns, use_ovmf=ovmf) + dump_monitor_cmds = kwargs.get("testimage_dump_monitor") + self.monitor_dumper = MonitorDumper(dump_monitor_cmds, dump_dir, self.runner) + if self.monitor_dumper: + self.monitor_dumper.create_dir("qmp") + dump_target_cmds = kwargs.get("testimage_dump_target") self.target_dumper = TargetDumper(dump_target_cmds, dump_dir, self.runner) self.target_dumper.create_dir("qemu")
meta/lib/oeqa/core/target/ssh.py +16 -1
@@ -43,6 +43,7 @@ class OESSHTarget(OETarget): if port: self.ssh = self.ssh + [ '-p', port ] self.scp = self.scp + [ '-P', port ] + self._monitor_dumper = None def start(self, **kwargs): pass
@@ -50,6 +51,15 @@ class OESSHTarget(OETarget): def stop(self, **kwargs): pass + @property + def monitor_dumper(self): + return self._monitor_dumper + + @monitor_dumper.setter + def monitor_dumper(self, dumper): + self._monitor_dumper = dumper + self.monitor_dumper.dump_monitor() + def _run(self, command, timeout=None, ignore_status=True): """ Runs command in target using SSHProcess.
@@ -87,9 +97,14 @@ class OESSHTarget(OETarget): processTimeout = self.timeout status, output = self._run(sshCmd, processTimeout, True) - self.logger.debug('Command: %s\nOutput: %s\n' % (command, output)) + self.logger.debug('Command: %s\nStatus: %d Output: %s\n' % (command, status, output)) if (status == 255) and (('No route to host') in output): + if self.monitor_dumper: + self.monitor_dumper.dump_monitor() + if status == 255: self.target_dumper.dump_target() + if self.monitor_dumper: + self.monitor_dumper.dump_monitor() return (status, output) def copyTo(self, localSrc, remoteDst):
meta/lib/oeqa/targetcontrol.py +3 -0
@@ -17,6 +17,7 @@ from oeqa.utils.sshcontrol import SSHControl from oeqa.utils.qemurunner import QemuRunner from oeqa.utils.qemutinyrunner import QemuTinyRunner from oeqa.utils.dump import TargetDumper +from oeqa.utils.dump import MonitorDumper from oeqa.controllers.testtargetloader import TestTargetLoader from abc import ABCMeta, abstractmethod
@@ -108,6 +109,7 @@ class QemuTarget(BaseTarget): self.qemulog = os.path.join(self.testdir, "qemu_boot_log.%s" % self.datetime) dump_target_cmds = d.getVar("testimage_dump_target") dump_host_cmds = d.getVar("testimage_dump_host") + dump_monitor_cmds = d.getVar("testimage_dump_monitor") dump_dir = d.getVar("TESTIMAGE_DUMP_DIR") if not dump_dir: dump_dir = os.path.join(d.getVar('LOG_DIR'), 'runtime-hostdump')
@@ -147,6 +149,7 @@ class QemuTarget(BaseTarget): serial_ports = len(d.getVar("SERIAL_CONSOLES").split())) self.target_dumper = TargetDumper(dump_target_cmds, dump_dir, self.runner) + self.monitor_dumper = MonitorDumper(dump_monitor_cmds, dump_dir, self.runner) def deploy(self): bb.utils.mkdirhier(self.testdir)
meta/lib/oeqa/utils/dump.py +28 -3
@@ -4,6 +4,7 @@ import os import sys +import json import errno import datetime import itertools
@@ -51,6 +52,8 @@ class BaseDumper(object): prefix = "host" elif isinstance(self, TargetDumper): prefix = "target" + elif isinstance(self, MonitorDumper): + prefix = "qmp" else: prefix = "unknown" for i in itertools.count():
@@ -58,9 +61,12 @@ class BaseDumper(object): fullname = os.path.join(self.dump_dir, filename) if not os.path.exists(fullname): break - with open(fullname, 'w') as dump_file: - dump_file.write(output) - + if isinstance(self, MonitorDumper): + with open(fullname, 'w') as json_file: + json.dump(output, json_file, indent=4) + else: + with open(fullname, 'w') as dump_file: + dump_file.write(output) class HostDumper(BaseDumper): """ Class to get dumps from the host running the tests """
@@ -96,3 +102,22 @@ class TargetDumper(BaseDumper): except: print("Tried to dump info from target but " "serial console failed") + print("Failed CMD: %s" % (cmd)) + +class MonitorDumper(BaseDumper): + """ Class to get dumps via the Qemu Monitor, it only works with QemuRunner """ + + def __init__(self, cmds, parent_dir, runner): + super(MonitorDumper, self).__init__(cmds, parent_dir) + self.runner = runner + + def dump_monitor(self, dump_dir=""): + if dump_dir: + self.dump_dir = dump_dir + for cmd in self.cmds: + try: + output = self.runner.run_monitor(cmd) + self._write_dump(cmd, output) + except: + print("Failed to dump montor data") + print("Failed CMD: %s" % (cmd))
meta/lib/oeqa/utils/qemurunner.py +28 -1
@@ -20,9 +20,13 @@ import string import threading import codecs import logging +import tempfile from oeqa.utils.dump import HostDumper from collections import defaultdict +sys.path.append('./tmp/hosttools') +import qmp + # Get Unicode non printable control chars control_range = list(range(0,32))+list(range(127,160)) control_chars = [chr(x) for x in control_range
@@ -168,6 +172,8 @@ class QemuRunner: return self.launch(launch_cmd, qemuparams=qemuparams, get_ip=get_ip, extra_bootparams=extra_bootparams, env=env) def launch(self, launch_cmd, get_ip = True, qemuparams = None, extra_bootparams = None, env = None): + qmp_port = self.tmpdir + "/." + next(tempfile._get_candidate_names()) + try: if self.serial_ports >= 2: self.threadsock, threadport = self.create_socket()
@@ -185,6 +191,8 @@ class QemuRunner: if os.path.exists(self.qemu_pidfile): os.remove(self.qemu_pidfile) self.qemuparams = 'bootparams="{0}" qemuparams="-pidfile {1}"'.format(bootparams, self.qemu_pidfile) + qemuparams += ' -S -qmp unix:%s,server,wait' % (qmp_port) + qemuparams += ' -monitor tcp:localhost:4444,server,nowait' if qemuparams: self.qemuparams = self.qemuparams[:-1] + " " + qemuparams + " " + '\"'
@@ -251,6 +259,19 @@ class QemuRunner: if self.runqemu_exited: return False + # Create the client socket for the QEMU Monitor Control Socket + # This will allow us to read status from Qemu if the the process + # is still alive + try: + self.qmp = qmp.QEMUMonitorProtocol(qmp_port) + self.qmp.connect() + except socket.error as msg: + self.logger.error("Failed to connect qemu monitor socket: %s" % msg[1]) + return False + + # Release the qemu porcess to continue running + self.run_monitor('cont') + if not self.is_alive(): self.logger.error("Qemu pid didn't appear in %s seconds (%s)" % (self.runqemutime, time.strftime("%D %H:%M:%S")))
@@ -376,7 +397,6 @@ class QemuRunner: sock.close() stopread = True - if not reachedlogin: if time.time() >= endtime: self.logger.warning("Target didn't reach login banner in %d seconds (%s)" %
@@ -391,6 +411,7 @@ class QemuRunner: self.stop() return False + self._dump_host() # If we are not able to login the tests can continue try: (status, output) = self.run_serial(self.boot_patterns['send_login_user'], raw=True, timeout=120)
@@ -437,6 +458,9 @@ class QemuRunner: self.runqemu.stdout.close() self.runqemu_exited = True + if hasattr(self, 'qmp') and self.qmp: + self.qmp.close() + self.qmp = None if hasattr(self, 'server_socket') and self.server_socket: self.server_socket.close() self.server_socket = None
@@ -495,6 +519,9 @@ class QemuRunner: return True return False + def run_monitor(self, command, timeout=60): + return self.qmp.cmd(command) + def run_serial(self, command, raw=False, timeout=60): # We assume target system have echo to get command status if not raw:
Signature · 1 line
2.25.1
PATCH 1/1 qemurunner: Add support for qmp commands
In reply to Saul WoldThis adds support for the Qemu Machine Protocol [0] extending the current dump process for Host and Target. The commands are added in the testimage.bbclass.
Currently, we setup qemu to stall until qmp gets connected and sends the initialization and continue commands, this works correctly.
With this version, the monitor_dumper is created in OEQemuTarget but then set in OESSHTarget as that's where we get the SSH failure happens. Python's @property is used to create a setter/getter type of setup in OESSHTarget to get overridden by OEQemuTarget.
By default the data is currently dumped to files for each command in TMPDIR/log/runtime-hostdump/<date>_qmp/unknown_<seq>_qemu_monitor as this is the naming convenstion in the dump.py code.
We use the qmp.py from qemu, which needs to get installed in a known static location.
[0] https://github.com/qemu/qemu/blob/master/docs/interop/qmp-spec.txt
--- meta/classes/testimage.bbclass | 6 ++++++ meta/lib/oeqa/core/target/qemu.py | 6 ++++++ meta/lib/oeqa/core/target/ssh.py | 17 ++++++++++++++++- meta/lib/oeqa/targetcontrol.py | 3 +++ meta/lib/oeqa/utils/dump.py | 31 ++++++++++++++++++++++++++++--- meta/lib/oeqa/utils/qemurunner.py | 29 ++++++++++++++++++++++++++++- 6 files changed, 87 insertions(+), 5 deletions(-)
meta/classes/testimage.bbclass +6 -0
@@ -127,6 +127,11 @@ testimage_dump_host () { netstat -an } +testimage_dump_monitor () { + query-status + query-block +} + python do_testimage() { testimage_main(d) }
@@ -319,6 +324,7 @@ def testimage_main(d): target_kwargs['powercontrol_extra_args'] = d.getVar("TEST_POWERCONTROL_EXTRA_ARGS") or "" target_kwargs['serialcontrol_cmd'] = d.getVar("TEST_SERIALCONTROL_CMD") or None target_kwargs['serialcontrol_extra_args'] = d.getVar("TEST_SERIALCONTROL_EXTRA_ARGS") or "" + target_kwargs['testimage_dump_monitor'] = d.getVar("testimage_dump_monitor") or "" target_kwargs['testimage_dump_target'] = d.getVar("testimage_dump_target") or "" def export_ssh_agent(d):
meta/lib/oeqa/core/target/qemu.py +6 -0
@@ -12,6 +12,7 @@ from collections import defaultdict from .ssh import OESSHTarget from oeqa.utils.qemurunner import QemuRunner +from oeqa.utils.dump import MonitorDumper from oeqa.utils.dump import TargetDumper supported_fstypes = ['ext3', 'ext4', 'cpio.gz', 'wic']
@@ -43,6 +44,11 @@ class OEQemuTarget(OESSHTarget): dump_host_cmds=dump_host_cmds, logger=logger, serial_ports=serial_ports, boot_patterns = boot_patterns, use_ovmf=ovmf) + dump_monitor_cmds = kwargs.get("testimage_dump_monitor") + self.monitor_dumper = MonitorDumper(dump_monitor_cmds, dump_dir, self.runner) + if self.monitor_dumper: + self.monitor_dumper.create_dir("qmp") + dump_target_cmds = kwargs.get("testimage_dump_target") self.target_dumper = TargetDumper(dump_target_cmds, dump_dir, self.runner) self.target_dumper.create_dir("qemu")
meta/lib/oeqa/core/target/ssh.py +16 -1
@@ -43,6 +43,7 @@ class OESSHTarget(OETarget): if port: self.ssh = self.ssh + [ '-p', port ] self.scp = self.scp + [ '-P', port ] + self._monitor_dumper = None def start(self, **kwargs): pass
@@ -50,6 +51,15 @@ class OESSHTarget(OETarget): def stop(self, **kwargs): pass + @property + def monitor_dumper(self): + return self._monitor_dumper + + @monitor_dumper.setter + def monitor_dumper(self, dumper): + self._monitor_dumper = dumper + self.monitor_dumper.dump_monitor() + def _run(self, command, timeout=None, ignore_status=True): """ Runs command in target using SSHProcess.
@@ -87,9 +97,14 @@ class OESSHTarget(OETarget): processTimeout = self.timeout status, output = self._run(sshCmd, processTimeout, True) - self.logger.debug('Command: %s\nOutput: %s\n' % (command, output)) + self.logger.debug('Command: %s\nStatus: %d Output: %s\n' % (command, status, output)) if (status == 255) and (('No route to host') in output): + if self.monitor_dumper: + self.monitor_dumper.dump_monitor() + if status == 255: self.target_dumper.dump_target() + if self.monitor_dumper: + self.monitor_dumper.dump_monitor() return (status, output) def copyTo(self, localSrc, remoteDst):
meta/lib/oeqa/targetcontrol.py +3 -0
@@ -17,6 +17,7 @@ from oeqa.utils.sshcontrol import SSHControl from oeqa.utils.qemurunner import QemuRunner from oeqa.utils.qemutinyrunner import QemuTinyRunner from oeqa.utils.dump import TargetDumper +from oeqa.utils.dump import MonitorDumper from oeqa.controllers.testtargetloader import TestTargetLoader from abc import ABCMeta, abstractmethod
@@ -108,6 +109,7 @@ class QemuTarget(BaseTarget): self.qemulog = os.path.join(self.testdir, "qemu_boot_log.%s" % self.datetime) dump_target_cmds = d.getVar("testimage_dump_target") dump_host_cmds = d.getVar("testimage_dump_host") + dump_monitor_cmds = d.getVar("testimage_dump_monitor") dump_dir = d.getVar("TESTIMAGE_DUMP_DIR") if not dump_dir: dump_dir = os.path.join(d.getVar('LOG_DIR'), 'runtime-hostdump')
@@ -147,6 +149,7 @@ class QemuTarget(BaseTarget): serial_ports = len(d.getVar("SERIAL_CONSOLES").split())) self.target_dumper = TargetDumper(dump_target_cmds, dump_dir, self.runner) + self.monitor_dumper = MonitorDumper(dump_monitor_cmds, dump_dir, self.runner) def deploy(self): bb.utils.mkdirhier(self.testdir)
meta/lib/oeqa/utils/dump.py +28 -3
@@ -4,6 +4,7 @@ import os import sys +import json import errno import datetime import itertools
@@ -51,6 +52,8 @@ class BaseDumper(object): prefix = "host" elif isinstance(self, TargetDumper): prefix = "target" + elif isinstance(self, MonitorDumper): + prefix = "qmp" else: prefix = "unknown" for i in itertools.count():
@@ -58,9 +61,12 @@ class BaseDumper(object): fullname = os.path.join(self.dump_dir, filename) if not os.path.exists(fullname): break - with open(fullname, 'w') as dump_file: - dump_file.write(output) - + if isinstance(self, MonitorDumper): + with open(fullname, 'w') as json_file: + json.dump(output, json_file, indent=4) + else: + with open(fullname, 'w') as dump_file: + dump_file.write(output) class HostDumper(BaseDumper): """ Class to get dumps from the host running the tests """
@@ -96,3 +102,22 @@ class TargetDumper(BaseDumper): except: print("Tried to dump info from target but " "serial console failed") + print("Failed CMD: %s" % (cmd)) + +class MonitorDumper(BaseDumper): + """ Class to get dumps via the Qemu Monitor, it only works with QemuRunner """ + + def __init__(self, cmds, parent_dir, runner): + super(MonitorDumper, self).__init__(cmds, parent_dir) + self.runner = runner + + def dump_monitor(self, dump_dir=""): + if dump_dir: + self.dump_dir = dump_dir + for cmd in self.cmds: + try: + output = self.runner.run_monitor(cmd) + self._write_dump(cmd, output) + except: + print("Failed to dump montor data") + print("Failed CMD: %s" % (cmd))
meta/lib/oeqa/utils/qemurunner.py +28 -1
@@ -20,9 +20,13 @@ import string import threading import codecs import logging +import tempfile from oeqa.utils.dump import HostDumper from collections import defaultdict +sys.path.append('./tmp/hosttools') +import qmp + # Get Unicode non printable control chars control_range = list(range(0,32))+list(range(127,160)) control_chars = [chr(x) for x in control_range
@@ -168,6 +172,8 @@ class QemuRunner: return self.launch(launch_cmd, qemuparams=qemuparams, get_ip=get_ip, extra_bootparams=extra_bootparams, env=env) def launch(self, launch_cmd, get_ip = True, qemuparams = None, extra_bootparams = None, env = None): + qmp_port = self.tmpdir + "/." + next(tempfile._get_candidate_names()) + try: if self.serial_ports >= 2: self.threadsock, threadport = self.create_socket()
@@ -185,6 +191,8 @@ class QemuRunner: if os.path.exists(self.qemu_pidfile): os.remove(self.qemu_pidfile) self.qemuparams = 'bootparams="{0}" qemuparams="-pidfile {1}"'.format(bootparams, self.qemu_pidfile) + qemuparams += ' -S -qmp unix:%s,server,wait' % (qmp_port) + qemuparams += ' -monitor tcp:localhost:4444,server,nowait' if qemuparams: self.qemuparams = self.qemuparams[:-1] + " " + qemuparams + " " + '\"'
@@ -251,6 +259,19 @@ class QemuRunner: if self.runqemu_exited: return False + # Create the client socket for the QEMU Monitor Control Socket + # This will allow us to read status from Qemu if the the process + # is still alive + try: + self.qmp = qmp.QEMUMonitorProtocol(qmp_port) + self.qmp.connect() + except socket.error as msg: + self.logger.error("Failed to connect qemu monitor socket: %s" % msg[1]) + return False + + # Release the qemu porcess to continue running + self.run_monitor('cont') + if not self.is_alive(): self.logger.error("Qemu pid didn't appear in %s seconds (%s)" % (self.runqemutime, time.strftime("%D %H:%M:%S")))
@@ -376,7 +397,6 @@ class QemuRunner: sock.close() stopread = True - if not reachedlogin: if time.time() >= endtime: self.logger.warning("Target didn't reach login banner in %d seconds (%s)" %
@@ -391,6 +411,7 @@ class QemuRunner: self.stop() return False + self._dump_host() # If we are not able to login the tests can continue try: (status, output) = self.run_serial(self.boot_patterns['send_login_user'], raw=True, timeout=120)
@@ -437,6 +458,9 @@ class QemuRunner: self.runqemu.stdout.close() self.runqemu_exited = True + if hasattr(self, 'qmp') and self.qmp: + self.qmp.close() + self.qmp = None if hasattr(self, 'server_socket') and self.server_socket: self.server_socket.close() self.server_socket = None
@@ -495,6 +519,9 @@ class QemuRunner: return True return False + def run_monitor(self, command, timeout=60): + return self.qmp.cmd(command) + def run_serial(self, command, raw=False, timeout=60): # We assume target system have echo to get command status if not raw:
Signature · 1 line
2.25.1