oeqa/runtime/parselogs improvements
v2 cover oeqa/runtime/parselogs improvements
From: Ross Burton <[email protected]>
Mostly the same as before, but this time with a missing casefold() added.
Ross
Ross Burton (5):
oeqa/runtime/parselogs: improve find call oeqa/runtime/parselogs: don't pass around members oeqa/runtime/parselogs: move some variables out of global scope oeqa/runtime/parselogs: select the correct machine-specific ignores
early
oeqa/runtime/parselogs: parse the logs with Python, not grep
meta/lib/oeqa/runtime/cases/parselogs.py | 141 +++++++++++------------ 1 file changed, 67 insertions(+), 74 deletions(-)
Signature · 1 line
2.34.1
PATCH v2 1/5 oeqa/runtime/parselogs: improve find call
From: Ross Burton <[email protected]>
getLogList() uses remote find invocations to find the logs. Instead of relying on shell expansion of wildcards and redundant use of -maxdepth (pointless as the shell expansion means the find is passed the files to return), invoke find idiomatically by telling it what directory to search for and escape the glob so find processes it.
Also remove many pointless str() calls.
--- meta/lib/oeqa/runtime/cases/parselogs.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-)
meta/lib/oeqa/runtime/cases/parselogs.py +5 -5
@@ -229,18 +229,18 @@ class ParseLogsTest(OERuntimeTestCase): def getLogList(self, log_locations): logs = [] for location in log_locations: - status, _ = self.target.run('test -f ' + str(location)) + status, _ = self.target.run('test -f %s' % location) if status == 0: - logs.append(str(location)) + logs.append(location) else: - status, _ = self.target.run('test -d ' + str(location)) + status, _ = self.target.run('test -d %s' % location) if status == 0: - cmd = 'find ' + str(location) + '/*.log -maxdepth 1 -type f' + cmd = 'find %s -name \\*.log -maxdepth 1 -type f' % location status, output = self.target.run(cmd) if status == 0: output = output.splitlines() for logfile in output: - logs.append(os.path.join(location, str(logfile))) + logs.append(os.path.join(location, logfile)) return logs # Copy the log files to be parsed locally
Signature · 1 line
2.34.1
PATCH v2 2/5 oeqa/runtime/parselogs: don't pass around members
In reply to [email protected]From: Ross Burton <[email protected]>
There's no point in passing around member fields, just access them directly.
--- meta/lib/oeqa/runtime/cases/parselogs.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-)
meta/lib/oeqa/runtime/cases/parselogs.py +14 -10
@@ -263,19 +263,19 @@ class ParseLogsTest(OERuntimeTestCase): return logs # Build the grep command to be used with filters and exclusions - def build_grepcmd(self, errors, ignore_errors, log): + def build_grepcmd(self, log): grepcmd = 'grep ' grepcmd += '-Ei "' - for error in errors: + for error in self.errors: grepcmd += r'\<' + error + r'\>' + '|' grepcmd = grepcmd[:-1] grepcmd += '" ' + str(log) + " | grep -Eiv \'" try: - errorlist = ignore_errors[self.td.get('MACHINE')] + errorlist = self.ignore_errors[self.td.get('MACHINE')] except KeyError: self.msg += 'No ignore list found for this machine, using default\n' - errorlist = ignore_errors['default'] + errorlist = self.ignore_errors['default'] for ignore_error in errorlist: ignore_error = ignore_error.replace('(', r'\(')
@@ -292,17 +292,21 @@ class ParseLogsTest(OERuntimeTestCase): return grepcmd - # Grep only the errors so that their context could be collected. - # Default context is 10 lines before and after the error itself - def parse_logs(self, errors, ignore_errors, logs, - lines_before = 10, lines_after = 10): + def parse_logs(self, logs, lines_before=10, lines_after=10): + """ + Search the log files @logs looking for error lines (marked by + @self.errors), ignoring anything listed in @self.ignore_errors. + + Returns a dictionary of log filenames to a dictionary of error lines to + the error context (controlled by @lines_before and @lines_after). + """ results = {} rez = [] grep_output = '' for log in logs: result = None - thegrep = self.build_grepcmd(errors, ignore_errors, log) + thegrep = self.build_grepcmd(log) try: result = check_output(thegrep, shell=True).decode('utf-8')
@@ -333,7 +337,7 @@ class ParseLogsTest(OERuntimeTestCase): def test_parselogs(self): self.write_dmesg() log_list = self.get_local_log_list(self.log_locations) - result = self.parse_logs(self.errors, self.ignore_errors, log_list) + result = self.parse_logs(log_list) errcount = 0 for log in result: self.msg += 'Log: ' + log + '\n'
Signature · 1 line
2.34.1
PATCH v2 3/5 oeqa/runtime/parselogs: move some variables out of global scope
In reply to [email protected]From: Ross Burton <[email protected]>
errors and log_locations can be trivially set in the class directly, instead of being defined in the module and then copied into the class.
--- meta/lib/oeqa/runtime/cases/parselogs.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-)
meta/lib/oeqa/runtime/cases/parselogs.py +9 -10
@@ -11,9 +11,6 @@ from shutil import rmtree from oeqa.runtime.case import OERuntimeTestCase from oeqa.core.decorator.depends import OETestDepends -#in the future these lists could be moved outside of module -errors = ["error", "cannot", "can\'t", "failed"] - common_errors = [ "(WW) warning, (EE) error, (NI) not implemented, (??) unknown.", "dma timeout",
@@ -201,17 +198,19 @@ ignore_errors = { ] + common_errors, } -log_locations = ["/var/log/","/var/log/dmesg", "/tmp/dmesg_output.log"] - class ParseLogsTest(OERuntimeTestCase): + # Which log files should be collected + log_locations = ["/var/log/", "/var/log/dmesg", "/tmp/dmesg_output.log"] + + # The keywords that identify error messages in the log files + errors = ["error", "cannot", "can't", "failed"] + @classmethod def setUpClass(cls): - cls.errors = errors - # When systemd is enabled we need to notice errors on # circular dependencies in units. - if 'systemd' in cls.td.get('DISTRO_FEATURES', ''): + if 'systemd' in cls.td.get('DISTRO_FEATURES'): cls.errors.extend([ 'Found ordering cycle on', 'Breaking ordering cycle by deleting job',
@@ -220,8 +219,6 @@ class ParseLogsTest(OERuntimeTestCase): ]) cls.ignore_errors = ignore_errors - cls.log_locations = log_locations - cls.msg = '' # Go through the log locations provided and if it's a folder # create a list with all the .log files in it, if it's a file
@@ -338,7 +335,9 @@ class ParseLogsTest(OERuntimeTestCase): self.write_dmesg() log_list = self.get_local_log_list(self.log_locations) result = self.parse_logs(log_list) + errcount = 0 + self.msg = "" for log in result: self.msg += 'Log: ' + log + '\n' self.msg += '-----------------------\n'
Signature · 1 line
2.34.1
PATCH v2 4/5 oeqa/runtime/parselogs: select the correct machine-specific ignores early
In reply to [email protected]From: Ross Burton <[email protected]>
This has no impact to the execution, but makes the following changes neater.
--- meta/lib/oeqa/runtime/cases/parselogs.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-)
meta/lib/oeqa/runtime/cases/parselogs.py +6 -7
@@ -218,7 +218,11 @@ class ParseLogsTest(OERuntimeTestCase): 'Ordering cycle found, skipping', ]) - cls.ignore_errors = ignore_errors + try: + cls.ignore_errors = ignore_errors[cls.td.get('MACHINE')] + except KeyError: + cls.logger.info('No ignore list found for this machine, using default') + cls.ignore_errors = ignore_errors['default'] # Go through the log locations provided and if it's a folder # create a list with all the .log files in it, if it's a file
@@ -268,13 +272,8 @@ class ParseLogsTest(OERuntimeTestCase): grepcmd = grepcmd[:-1] grepcmd += '" ' + str(log) + " | grep -Eiv \'" - try: - errorlist = self.ignore_errors[self.td.get('MACHINE')] - except KeyError: - self.msg += 'No ignore list found for this machine, using default\n' - errorlist = self.ignore_errors['default'] - for ignore_error in errorlist: + for ignore_error in self.ignore_errors: ignore_error = ignore_error.replace('(', r'\(') ignore_error = ignore_error.replace(')', r'\)') ignore_error = ignore_error.replace("'", '.')
Signature · 1 line
2.34.1
PATCH v2 5/5 oeqa/runtime/parselogs: parse the logs with Python, not grep
In reply to [email protected]From: Ross Burton <[email protected]>
Instead of constructing huge grep statements, we can simply open the logs in Python and do the relevant string operations directly.
The trick is to remember to casefold() all of the strings, so that the "in" operator can be used.
Just one of the ignores needs to be adjusted because it uses a regular expression and the new logic doesn't support that. This is handled by simply reducing the size of the ignore match.
--- meta/lib/oeqa/runtime/cases/parselogs.py | 91 +++++++++++------------- 1 file changed, 41 insertions(+), 50 deletions(-)
meta/lib/oeqa/runtime/cases/parselogs.py +41 -50
@@ -4,9 +4,9 @@ # SPDX-License-Identifier: MIT # +import collections import os -from subprocess import check_output from shutil import rmtree from oeqa.runtime.case import OERuntimeTestCase from oeqa.core.decorator.depends import OETestDepends
@@ -64,6 +64,7 @@ common_errors = [ "xf86OpenConsole: Switching VT failed", "Failed to read LoaderConfigTimeoutOneShot variable, ignoring: Operation not supported", "Failed to read LoaderEntryOneShot variable, ignoring: Operation not supported", + "invalid BAR (can't size)", ] x86_common = [
@@ -97,16 +98,13 @@ ignore_errors = { 'default' : common_errors, 'qemux86' : [ 'Failed to access perfctr msr (MSR', - 'pci 0000:00:00.0: [Firmware Bug]: reg 0x..: invalid BAR (can\'t size)', ] + qemux86_common, 'qemux86-64' : qemux86_common, 'qemumips' : [ 'Failed to load module "glx"', - 'pci 0000:00:00.0: [Firmware Bug]: reg 0x..: invalid BAR (can\'t size)', 'cacheinfo: Failed to find cpu0 device node', ] + common_errors, 'qemumips64' : [ - 'pci 0000:00:00.0: [Firmware Bug]: reg 0x..: invalid BAR (can\'t size)', 'cacheinfo: Failed to find cpu0 device node', ] + common_errors, 'qemuppc' : [
@@ -218,11 +216,13 @@ class ParseLogsTest(OERuntimeTestCase): 'Ordering cycle found, skipping', ]) + cls.errors = [s.casefold() for s in cls.errors] + try: - cls.ignore_errors = ignore_errors[cls.td.get('MACHINE')] + cls.ignore_errors = [s.casefold() for s in ignore_errors[cls.td.get('MACHINE')]] except KeyError: cls.logger.info('No ignore list found for this machine, using default') - cls.ignore_errors = ignore_errors['default'] + cls.ignore_errors = [s.casefold() for s in ignore_errors['default']] # Go through the log locations provided and if it's a folder # create a list with all the .log files in it, if it's a file
@@ -263,30 +263,33 @@ class ParseLogsTest(OERuntimeTestCase): logs = [f for f in dir_files if os.path.isfile(f)] return logs - # Build the grep command to be used with filters and exclusions - def build_grepcmd(self, log): - grepcmd = 'grep ' - grepcmd += '-Ei "' - for error in self.errors: - grepcmd += r'\<' + error + r'\>' + '|' - grepcmd = grepcmd[:-1] - grepcmd += '" ' + str(log) + " | grep -Eiv \'" + def get_context(self, lines, index, before=6, after=3): + """ + Given a set of lines and the index of the line that is important, return + a number of lines surrounding that line. + """ + last = len(lines) + start = index - before + end = index + after + 1 - for ignore_error in self.ignore_errors: - ignore_error = ignore_error.replace('(', r'\(') - ignore_error = ignore_error.replace(')', r'\)') - ignore_error = ignore_error.replace("'", '.') - ignore_error = ignore_error.replace('?', r'\?') - ignore_error = ignore_error.replace('[', r'\[') - ignore_error = ignore_error.replace(']', r'\]') - ignore_error = ignore_error.replace('*', r'\*') - ignore_error = ignore_error.replace('0-9', '[0-9]') - grepcmd += ignore_error + '|' - grepcmd = grepcmd[:-1] - grepcmd += "\'" + if start < 0: + end -= start + start = 0 + if end > last: + start -= end - last + end = last - return grepcmd + return lines[start:end] + + def test_get_context(self): + """ + A test case for the test case. + """ + lines = list(range(0,10)) + self.assertEqual(self.get_context(lines, 0, 2, 1), [0, 1, 2, 3]) + self.assertEqual(self.get_context(lines, 5, 2, 1), [3, 4, 5, 6]) + self.assertEqual(self.get_context(lines, 9, 2, 1), [6, 7, 8, 9]) def parse_logs(self, logs, lines_before=10, lines_after=10): """
@@ -296,31 +299,19 @@ class ParseLogsTest(OERuntimeTestCase): Returns a dictionary of log filenames to a dictionary of error lines to the error context (controlled by @lines_before and @lines_after). """ - results = {} - rez = [] - grep_output = '' + results = collections.defaultdict(dict) for log in logs: - result = None - thegrep = self.build_grepcmd(log) - - try: - result = check_output(thegrep, shell=True).decode('utf-8') - except: - pass + with open(log) as f: + lines = f.readlines() - if result is not None: - results[log] = {} - rez = result.splitlines() + for i, line in enumerate(lines): + line = line.strip() + line_lower = line.casefold() - for xrez in rez: - try: - cmd = ['grep', '-F', xrez, '-B', str(lines_before)] - cmd += ['-A', str(lines_after), log] - grep_output = check_output(cmd).decode('utf-8') - except: - pass - results[log][xrez]=grep_output + if any(keyword in line_lower for keyword in self.errors): + if not any(ignore in line_lower for ignore in self.ignore_errors): + results[log][line] = "".join(self.get_context(lines, i, lines_before, lines_after)) return results
@@ -342,9 +333,9 @@ class ParseLogsTest(OERuntimeTestCase): self.msg += '-----------------------\n' for error in result[log]: errcount += 1 - self.msg += 'Central error: ' + str(error) + '\n' + self.msg += 'Central error: ' + error + '\n' self.msg += '***********************\n' - self.msg += result[str(log)][str(error)] + '\n' + self.msg += result[log][error] + '\n' self.msg += '***********************\n' self.msg += '%s errors found in logs.' % errcount self.assertEqual(errcount, 0, msg=self.msg)
Signature · 1 line
2.34.1