Allow to dump commands from serial console when a test fails
cover Allow to dump commands from serial console when a test fails
From: Mariano Lopez <[email protected]>
These changes allows to run commands on the qemu serial console when a test fails. Right now the tests are hardcoded but the idea is to be able to customize them.
qemurunner.py: Added and tunned run_serial method to allow to run serial commands on the target
oetest.py: Added tearDown method to oeRuntimeTest, this allows to catch when a test fails and run the diagnostic commands on the target
The following changes since commit 0f7df92e3d2dbbb4c94299171d5d0287887e0d28:
clutter-gst: update to 3.0.8 (2015-08-11 09:28:52 -0700)
are available in the git repository at:
git://git.yoctoproject.org/poky-contrib mariano/bug8118v2
http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=mariano/bug8118v2
Mariano Lopez (6):
qemurunner.py: Add method run_serial qemurunner.py: Added login to start method qemurunner.py: Added raw mode in run_serial qemurunner.py: Performance improvements in run_serial oetest.py: Added method tearDown for oeRuntimeTest oetest.py: Don't wait to write dump files
meta/lib/oeqa/oetest.py | 35 +++++++++++++++++++++++++++ meta/lib/oeqa/utils/qemurunner.py | 50 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+)
Signature · 1 line
1.8.4.5
PATCH 1/6 qemurunner.py: Add method run_serial
From: Mariano Lopez <[email protected]>
The only need for the console before this patch was to check if the target has booted. This allows to send commands to the terminal.
This new method is based on the method with the same name of the QemuTinyRunner class. The difference here is it will remove the command and the prompt. The other diference is it will send an echo $? to check if the last command was successful.
[YOCTO #8118]
--- meta/lib/oeqa/utils/qemurunner.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+)
meta/lib/oeqa/utils/qemurunner.py +30 -0
@@ -262,3 +262,33 @@ class QemuRunner: basecmd = os.path.basename(basecmd) if "qemu-system" in basecmd and "-serial tcp" in commands[p]: return [int(p),commands[p]] + + def run_serial(self, command): + # We assume target system have echo to get command status + self.server_socket.sendall("%s; echo $?\n" % command) + data = '' + status = 0 + stopread = False + endtime = time.time()+5 + while time.time()<endtime and not stopread: + sread, _, _ = select.select([self.server_socket],[],[],5) + for sock in sread: + answer = sock.recv(1024) + if answer: + data += answer + else: + sock.close() + stopread = True + if data: + # Remove first line (command line) and last line (prompt) + data = data[data.find('$?\r\n')+4:data.rfind('\r\n')] + index = data.rfind('\r\n') + if index == -1: + status_cmd = data + data = "" + else: + status_cmd = data[index+2:] + data = data[:index] + if (status_cmd == "0"): + status = 1 + return (status, str(data))
Signature · 1 line
1.8.4.5
PATCH 2/6 qemurunner.py: Added login to start method
In reply to [email protected]From: Mariano Lopez <[email protected]>
This adds the automatic login after the target finished booting. If the automatic login fails it won't stop the target or any test, it would only send a log to the file.
[YOCTO #8118]
--- meta/lib/oeqa/utils/qemurunner.py | 11 +++++++++++ 1 file changed, 11 insertions(+)
meta/lib/oeqa/utils/qemurunner.py +11 -0
@@ -37,6 +37,7 @@ class QemuRunner: self.deploy_dir_image = deploy_dir_image self.logfile = logfile self.boottime = boottime + self.logged = False self.runqemutime = 60
@@ -159,6 +160,7 @@ class QemuRunner: self.log(data) self.bootlog += data if re.search(".* login:", self.bootlog): + self.server_socket = self.qemusock stopread = True reachedlogin = True logger.info("Reached login banner")
@@ -174,6 +176,15 @@ class QemuRunner: logger.info("Check full boot log: %s" % self.logfile) self.stop() return False + + (status, output) = self.run_serial("root\n") + if re.search("root@[a-zA-Z0-9\-]+:~#", output): + self.logged = True + logger.info("Logged as root in serial console") + else: + logger.info("Couldn't login into serial console" + " as root using blank password") + else: logger.info("Qemu pid didn't appeared in %s seconds" % self.runqemutime) self.stop()
Signature · 1 line
1.8.4.5
PATCH 3/6 qemurunner.py: Added raw mode in run_serial
In reply to [email protected]From: Mariano Lopez <[email protected]>
Raw mode allows to send the command without sending 'echo $?' for validation; Also this doesn't remove the command or the prompt from the output returned. In raw mode validation is done if there is output.
This raw mode would be useful for validate the prompt when a user logs in.
[YOCTO #8118]
--- meta/lib/oeqa/utils/qemurunner.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-)
meta/lib/oeqa/utils/qemurunner.py +18 -13
@@ -177,7 +177,7 @@ class QemuRunner: self.stop() return False - (status, output) = self.run_serial("root\n") + (status, output) = self.run_serial("root\n", raw=True) if re.search("root@[a-zA-Z0-9\-]+:~#", output): self.logged = True logger.info("Logged as root in serial console")
@@ -274,9 +274,11 @@ class QemuRunner: if "qemu-system" in basecmd and "-serial tcp" in commands[p]: return [int(p),commands[p]] - def run_serial(self, command): + def run_serial(self, command, raw=False): # We assume target system have echo to get command status - self.server_socket.sendall("%s; echo $?\n" % command) + if not raw: + command = "%s; echo $?\n" % command + self.server_socket.sendall(command) data = '' status = 0 stopread = False
@@ -291,15 +293,18 @@ class QemuRunner: sock.close() stopread = True if data: - # Remove first line (command line) and last line (prompt) - data = data[data.find('$?\r\n')+4:data.rfind('\r\n')] - index = data.rfind('\r\n') - if index == -1: - status_cmd = data - data = "" - else: - status_cmd = data[index+2:] - data = data[:index] - if (status_cmd == "0"): + if raw: status = 1 + else: + # Remove first line (command line) and last line (prompt) + data = data[data.find('$?\r\n')+4:data.rfind('\r\n')] + index = data.rfind('\r\n') + if index == -1: + status_cmd = data + data = "" + else: + status_cmd = data[index+2:] + data = data[:index] + if (status_cmd == "0"): + status = 1 return (status, str(data))
Signature · 1 line
1.8.4.5
PATCH 4/6 qemurunner.py: Performance improvements in run_serial
In reply to [email protected]From: Mariano Lopez <[email protected]>
This allow to search for the prompt after a command is run so it can avoid waiting for the timeout.
Also corrected identation issues
[YOCTO #8118]
--- meta/lib/oeqa/utils/qemurunner.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-)
meta/lib/oeqa/utils/qemurunner.py +12 -8
@@ -284,14 +284,18 @@ class QemuRunner: stopread = False endtime = time.time()+5 while time.time()<endtime and not stopread: - sread, _, _ = select.select([self.server_socket],[],[],5) - for sock in sread: - answer = sock.recv(1024) - if answer: - data += answer - else: - sock.close() - stopread = True + sread, _, _ = select.select([self.server_socket],[],[],5) + for sock in sread: + answer = sock.recv(1024) + if answer: + data += answer + # Search the prompt to stop + if re.search("[a-zA-Z0-9]+@[a-zA-Z0-9\-]+:~#", data): + stopread = True + break + else: + sock.close() + stopread = True if data: if raw: status = 1
Signature · 1 line
1.8.4.5
PATCH 5/6 oetest.py: Added method tearDown for oeRuntimeTest
In reply to [email protected]From: Mariano Lopez <[email protected]>
The tearDown method is triggered when a tests ends it doesn't matter if fails or succeeds. Inside this method added an evalution to check if fails and then run some commands in the target to get the data for later debugging.
[YOCTO #8118]
--- meta/lib/oeqa/oetest.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+)
meta/lib/oeqa/oetest.py +37 -0
@@ -11,6 +11,7 @@ import os, re, mmap import unittest import inspect import subprocess +import datetime import bb from oeqa.utils.decorators import LogResults
@@ -117,6 +118,42 @@ class oeRuntimeTest(oeTest): self.target = oeRuntimeTest.tc.target super(oeRuntimeTest, self).__init__(methodName) + def tearDown(self): + # If a test fails or there is an exception + if (self._resultForDoCleanups.failures or + self._resultForDoCleanups.errors): + commands = ["top -bn1", "ps", "free", "df", "_ping", "dmesg", "netstat -a", "ifconfig -a", "_logs"] + dump_dir = "/tmp/oe-saved-tests" + dump_dir = os.path.join(dump_dir, + datetime.datetime.now().strftime('%Y%m%d%H%M')) + os.makedirs(dump_dir) + bb.warn("Test failed, getting data from target " + "and saving it in %s" % dump_dir) + output = self.run_bulk_commands(commands) + for key,msg in output.iteritems(): + filename = key.split()[0] + with open(os.path.join(dump_dir, filename), 'w') as f: + f.write(msg) + + def run_bulk_commands(self, commands): + all_output = {} + for command in commands: + # This will ping the host from target + if command == "_ping": + comm = "ping -c3 %s" % self.target.server_ip + # This will get all the logs from /var/log/ + elif command == "_logs": + comm = 'find /var/log/ -type f 2>/dev/null ' + comm = '%s-exec echo "%s" \\; ' % (comm, '='*20) + comm = '%s-exec echo {} \\; ' % comm + comm = '%s-exec echo "%s" \\; ' % (comm, '='*20) + comm = '%s-exec cat {} \\; -exec echo "" \\;' % comm + else: + comm = command + (status, output) = self.target.run_serial(comm) + all_output[command] = output + return all_output + #TODO: use package_manager.py to install packages on any type of image def install_packages(self, packagelist): for package in packagelist:
Signature · 1 line
1.8.4.5
PATCH 6/6 oetest.py: Don't wait to write dump files
In reply to [email protected]From: Mariano Lopez <[email protected]>
This allows to write the dump files immediately after get the data from the target. Before this, it would run all the commands and write the files. The old behavior could cause no log written at all if the serial console gets stuck.
--- meta/lib/oeqa/oetest.py | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-)
meta/lib/oeqa/oetest.py +15 -17
@@ -122,21 +122,18 @@ class oeRuntimeTest(oeTest): # If a test fails or there is an exception if (self._resultForDoCleanups.failures or self._resultForDoCleanups.errors): - commands = ["top -bn1", "ps", "free", "df", "_ping", "dmesg", "netstat -a", "ifconfig -a", "_logs"] - dump_dir = "/tmp/oe-saved-tests" - dump_dir = os.path.join(dump_dir, - datetime.datetime.now().strftime('%Y%m%d%H%M')) - os.makedirs(dump_dir) - bb.warn("Test failed, getting data from target " - "and saving it in %s" % dump_dir) - output = self.run_bulk_commands(commands) - for key,msg in output.iteritems(): - filename = key.split()[0] - with open(os.path.join(dump_dir, filename), 'w') as f: - f.write(msg) - - def run_bulk_commands(self, commands): - all_output = {} + self.dump_target_logs() + + def dump_target_logs(self): + commands = ["top -bn1", "ps", "free", "df", "_ping", "dmesg", "netstat -a", "ifconfig -a", "_logs"] + dump_dir = "/tmp/oe-saved-tests" + dump_sub_dir = ("%s_%s" % ( + datetime.datetime.now().strftime('%Y%m%d%H%M'), + self._testMethodName)) + dump_dir = os.path.join(dump_dir, dump_sub_dir) + os.makedirs(dump_dir) + bb.warn("%s failed: getting data from target and " + "saving into %s" % (self._testMethodName, dump_dir)) for command in commands: # This will ping the host from target if command == "_ping":
@@ -151,8 +148,9 @@ class oeRuntimeTest(oeTest): else: comm = command (status, output) = self.target.run_serial(comm) - all_output[command] = output - return all_output + filename = command.split()[0] + with open(os.path.join(dump_dir, filename), 'w') as f: + f.write(output) #TODO: use package_manager.py to install packages on any type of image def install_packages(self, packagelist):
Signature · 1 line
1.8.4.5