Skip to contentSkip to conversation

oeqa/runtime/parselogs improvements

PATCH New [email protected] 1 participant 6 messages Sep 23, 2023

lore
Share & tools
lore
#0 lore raw

v2 cover oeqa/runtime/parselogs improvements

From: Ross Burton <[email protected]>

Mostly the same as before, but this time with a missing casefold() added.

Ross

Ross Burton (5):
oeqa/runtime/parselogs: improve find call oeqa/runtime/parselogs: don't pass around members oeqa/runtime/parselogs: move some variables out of global scope oeqa/runtime/parselogs: select the correct machine-specific ignores
early
oeqa/runtime/parselogs: parse the logs with Python, not grep

 meta/lib/oeqa/runtime/cases/parselogs.py | 141 +++++++++++------------
 1 file changed, 67 insertions(+), 74 deletions(-)
Signature · 1 line
2.34.1
#1 lore raw

PATCH v2 1/5 oeqa/runtime/parselogs: improve find call

From: Ross Burton <[email protected]>

getLogList() uses remote find invocations to find the logs. Instead of relying on shell expansion of wildcards and redundant use of -maxdepth (pointless as the shell expansion means the find is passed the files to return), invoke find idiomatically by telling it what directory to search for and escape the glob so find processes it.

Also remove many pointless str() calls.

---
 meta/lib/oeqa/runtime/cases/parselogs.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)
meta/lib/oeqa/runtime/cases/parselogs.py +5 -5
index 6e5dc753060..93782b844b9 100644
@@ -229,18 +229,18 @@ class ParseLogsTest(OERuntimeTestCase):
     def getLogList(self, log_locations):
         logs = []
         for location in log_locations:
-            status, _ = self.target.run('test -f ' + str(location))
+            status, _ = self.target.run('test -f %s' % location)
             if status == 0:
-                logs.append(str(location))
+                logs.append(location)
             else:
-                status, _ = self.target.run('test -d ' + str(location))
+                status, _ = self.target.run('test -d %s' % location)
                 if status == 0:
-                    cmd = 'find ' + str(location) + '/*.log -maxdepth 1 -type f'
+                    cmd = 'find %s -name \\*.log -maxdepth 1 -type f' % location
                     status, output = self.target.run(cmd)
                     if status == 0:
                         output = output.splitlines()
                         for logfile in output:
-                            logs.append(os.path.join(location, str(logfile)))
+                            logs.append(os.path.join(location, logfile))
         return logs
 
     # Copy the log files to be parsed locally
Signature · 1 line
2.34.1
#2 lore raw

PATCH v2 2/5 oeqa/runtime/parselogs: don't pass around members

In reply to [email protected]

From: Ross Burton <[email protected]>

There's no point in passing around member fields, just access them directly.

---
 meta/lib/oeqa/runtime/cases/parselogs.py | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)
meta/lib/oeqa/runtime/cases/parselogs.py +14 -10
index 93782b844b9..0262f574d1a 100644
@@ -263,19 +263,19 @@ class ParseLogsTest(OERuntimeTestCase):
         return logs
 
     # Build the grep command to be used with filters and exclusions
-    def build_grepcmd(self, errors, ignore_errors, log):
+    def build_grepcmd(self, log):
         grepcmd = 'grep '
         grepcmd += '-Ei "'
-        for error in errors:
+        for error in self.errors:
             grepcmd += r'\<' + error + r'\>' + '|'
         grepcmd = grepcmd[:-1]
         grepcmd += '" ' + str(log) + " | grep -Eiv \'"
 
         try:
-            errorlist = ignore_errors[self.td.get('MACHINE')]
+            errorlist = self.ignore_errors[self.td.get('MACHINE')]
         except KeyError:
             self.msg += 'No ignore list found for this machine, using default\n'
-            errorlist = ignore_errors['default']
+            errorlist = self.ignore_errors['default']
 
         for ignore_error in errorlist:
             ignore_error = ignore_error.replace('(', r'\(')
@@ -292,17 +292,21 @@ class ParseLogsTest(OERuntimeTestCase):
 
         return grepcmd
 
-    # Grep only the errors so that their context could be collected.
-    # Default context is 10 lines before and after the error itself
-    def parse_logs(self, errors, ignore_errors, logs,
-                   lines_before = 10, lines_after = 10):
+    def parse_logs(self, logs, lines_before=10, lines_after=10):
+        """
+        Search the log files @logs looking for error lines (marked by
+        @self.errors), ignoring anything listed in @self.ignore_errors.
+
+        Returns a dictionary of log filenames to a dictionary of error lines to
+        the error context (controlled by @lines_before and @lines_after).
+        """
         results = {}
         rez = []
         grep_output = ''
 
         for log in logs:
             result = None
-            thegrep = self.build_grepcmd(errors, ignore_errors, log)
+            thegrep = self.build_grepcmd(log)
 
             try:
                 result = check_output(thegrep, shell=True).decode('utf-8')
@@ -333,7 +337,7 @@ class ParseLogsTest(OERuntimeTestCase):
     def test_parselogs(self):
         self.write_dmesg()
         log_list = self.get_local_log_list(self.log_locations)
-        result = self.parse_logs(self.errors, self.ignore_errors, log_list)
+        result = self.parse_logs(log_list)
         errcount = 0
         for log in result:
             self.msg += 'Log: ' + log + '\n'
Signature · 1 line
2.34.1
#3 lore raw

PATCH v2 3/5 oeqa/runtime/parselogs: move some variables out of global scope

In reply to [email protected]

From: Ross Burton <[email protected]>

errors and log_locations can be trivially set in the class directly, instead of being defined in the module and then copied into the class.

---
 meta/lib/oeqa/runtime/cases/parselogs.py | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)
meta/lib/oeqa/runtime/cases/parselogs.py +9 -10
index 0262f574d1a..3f205661ea9 100644
@@ -11,9 +11,6 @@ from shutil import rmtree
 from oeqa.runtime.case import OERuntimeTestCase
 from oeqa.core.decorator.depends import OETestDepends
 
-#in the future these lists could be moved outside of module
-errors = ["error", "cannot", "can\'t", "failed"]
-
 common_errors = [
     "(WW) warning, (EE) error, (NI) not implemented, (??) unknown.",
     "dma timeout",
@@ -201,17 +198,19 @@ ignore_errors = {
         ] + common_errors,
 }
 
-log_locations = ["/var/log/","/var/log/dmesg", "/tmp/dmesg_output.log"]
-
 class ParseLogsTest(OERuntimeTestCase):
 
+    # Which log files should be collected
+    log_locations = ["/var/log/", "/var/log/dmesg", "/tmp/dmesg_output.log"]
+
+    # The keywords that identify error messages in the log files
+    errors = ["error", "cannot", "can't", "failed"]
+
     @classmethod
     def setUpClass(cls):
-        cls.errors = errors
-
         # When systemd is enabled we need to notice errors on
         # circular dependencies in units.
-        if 'systemd' in cls.td.get('DISTRO_FEATURES', ''):
+        if 'systemd' in cls.td.get('DISTRO_FEATURES'):
             cls.errors.extend([
                 'Found ordering cycle on',
                 'Breaking ordering cycle by deleting job',
@@ -220,8 +219,6 @@ class ParseLogsTest(OERuntimeTestCase):
                 ])
 
         cls.ignore_errors = ignore_errors
-        cls.log_locations = log_locations
-        cls.msg = ''
 
     # Go through the log locations provided and if it's a folder
     # create a list with all the .log files in it, if it's a file
@@ -338,7 +335,9 @@ class ParseLogsTest(OERuntimeTestCase):
         self.write_dmesg()
         log_list = self.get_local_log_list(self.log_locations)
         result = self.parse_logs(log_list)
+
         errcount = 0
+        self.msg = ""
         for log in result:
             self.msg += 'Log: ' + log + '\n'
             self.msg += '-----------------------\n'
Signature · 1 line
2.34.1
#4 lore raw

PATCH v2 4/5 oeqa/runtime/parselogs: select the correct machine-specific ignores early

In reply to [email protected]

From: Ross Burton <[email protected]>

This has no impact to the execution, but makes the following changes neater.

---
 meta/lib/oeqa/runtime/cases/parselogs.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)
meta/lib/oeqa/runtime/cases/parselogs.py +6 -7
index 3f205661ea9..3980582dda2 100644
@@ -218,7 +218,11 @@ class ParseLogsTest(OERuntimeTestCase):
                 'Ordering cycle found, skipping',
                 ])
 
-        cls.ignore_errors = ignore_errors
+        try:
+            cls.ignore_errors = ignore_errors[cls.td.get('MACHINE')]
+        except KeyError:
+            cls.logger.info('No ignore list found for this machine, using default')
+            cls.ignore_errors = ignore_errors['default']
 
     # Go through the log locations provided and if it's a folder
     # create a list with all the .log files in it, if it's a file
@@ -268,13 +272,8 @@ class ParseLogsTest(OERuntimeTestCase):
         grepcmd = grepcmd[:-1]
         grepcmd += '" ' + str(log) + " | grep -Eiv \'"
 
-        try:
-            errorlist = self.ignore_errors[self.td.get('MACHINE')]
-        except KeyError:
-            self.msg += 'No ignore list found for this machine, using default\n'
-            errorlist = self.ignore_errors['default']
 
-        for ignore_error in errorlist:
+        for ignore_error in self.ignore_errors:
             ignore_error = ignore_error.replace('(', r'\(')
             ignore_error = ignore_error.replace(')', r'\)')
             ignore_error = ignore_error.replace("'", '.')
Signature · 1 line
2.34.1
#5 lore raw

PATCH v2 5/5 oeqa/runtime/parselogs: parse the logs with Python, not grep

In reply to [email protected]

From: Ross Burton <[email protected]>

Instead of constructing huge grep statements, we can simply open the logs in Python and do the relevant string operations directly.

The trick is to remember to casefold() all of the strings, so that the "in" operator can be used.

Just one of the ignores needs to be adjusted because it uses a regular expression and the new logic doesn't support that. This is handled by simply reducing the size of the ignore match.

---
 meta/lib/oeqa/runtime/cases/parselogs.py | 91 +++++++++++-------------
 1 file changed, 41 insertions(+), 50 deletions(-)
meta/lib/oeqa/runtime/cases/parselogs.py +41 -50
index 3980582dda2..a805edd79dd 100644
@@ -4,9 +4,9 @@
 # SPDX-License-Identifier: MIT
 #
 
+import collections
 import os
 
-from subprocess import check_output
 from shutil import rmtree
 from oeqa.runtime.case import OERuntimeTestCase
 from oeqa.core.decorator.depends import OETestDepends
@@ -64,6 +64,7 @@ common_errors = [
     "xf86OpenConsole: Switching VT failed",
     "Failed to read LoaderConfigTimeoutOneShot variable, ignoring: Operation not supported",
     "Failed to read LoaderEntryOneShot variable, ignoring: Operation not supported",
+    "invalid BAR (can't size)",
     ]
 
 x86_common = [
@@ -97,16 +98,13 @@ ignore_errors = {
     'default' : common_errors,
     'qemux86' : [
         'Failed to access perfctr msr (MSR',
-        'pci 0000:00:00.0: [Firmware Bug]: reg 0x..: invalid BAR (can\'t size)',
         ] + qemux86_common,
     'qemux86-64' : qemux86_common,
     'qemumips' : [
         'Failed to load module "glx"',
-        'pci 0000:00:00.0: [Firmware Bug]: reg 0x..: invalid BAR (can\'t size)',
         'cacheinfo: Failed to find cpu0 device node',
         ] + common_errors,
     'qemumips64' : [
-        'pci 0000:00:00.0: [Firmware Bug]: reg 0x..: invalid BAR (can\'t size)',
         'cacheinfo: Failed to find cpu0 device node',
          ] + common_errors,
     'qemuppc' : [
@@ -218,11 +216,13 @@ class ParseLogsTest(OERuntimeTestCase):
                 'Ordering cycle found, skipping',
                 ])
 
+        cls.errors = [s.casefold() for s in cls.errors]
+
         try:
-            cls.ignore_errors = ignore_errors[cls.td.get('MACHINE')]
+            cls.ignore_errors = [s.casefold() for s in ignore_errors[cls.td.get('MACHINE')]]
         except KeyError:
             cls.logger.info('No ignore list found for this machine, using default')
-            cls.ignore_errors = ignore_errors['default']
+            cls.ignore_errors = [s.casefold() for s in ignore_errors['default']]
 
     # Go through the log locations provided and if it's a folder
     # create a list with all the .log files in it, if it's a file
@@ -263,30 +263,33 @@ class ParseLogsTest(OERuntimeTestCase):
         logs = [f for f in dir_files if os.path.isfile(f)]
         return logs
 
-    # Build the grep command to be used with filters and exclusions
-    def build_grepcmd(self, log):
-        grepcmd = 'grep '
-        grepcmd += '-Ei "'
-        for error in self.errors:
-            grepcmd += r'\<' + error + r'\>' + '|'
-        grepcmd = grepcmd[:-1]
-        grepcmd += '" ' + str(log) + " | grep -Eiv \'"
+    def get_context(self, lines, index, before=6, after=3):
+        """
+        Given a set of lines and the index of the line that is important, return
+        a number of lines surrounding that line.
+        """
+        last = len(lines)
 
+        start = index - before
+        end = index + after + 1
 
-        for ignore_error in self.ignore_errors:
-            ignore_error = ignore_error.replace('(', r'\(')
-            ignore_error = ignore_error.replace(')', r'\)')
-            ignore_error = ignore_error.replace("'", '.')
-            ignore_error = ignore_error.replace('?', r'\?')
-            ignore_error = ignore_error.replace('[', r'\[')
-            ignore_error = ignore_error.replace(']', r'\]')
-            ignore_error = ignore_error.replace('*', r'\*')
-            ignore_error = ignore_error.replace('0-9', '[0-9]')
-            grepcmd += ignore_error + '|'
-        grepcmd = grepcmd[:-1]
-        grepcmd += "\'"
+        if start < 0:
+            end -= start
+            start = 0
+        if end > last:
+            start -= end - last
+            end = last
 
-        return grepcmd
+        return lines[start:end]
+
+    def test_get_context(self):
+        """
+        A test case for the test case.
+        """
+        lines = list(range(0,10))
+        self.assertEqual(self.get_context(lines, 0, 2, 1), [0, 1, 2, 3])
+        self.assertEqual(self.get_context(lines, 5, 2, 1), [3, 4, 5, 6])
+        self.assertEqual(self.get_context(lines, 9, 2, 1), [6, 7, 8, 9])
 
     def parse_logs(self, logs, lines_before=10, lines_after=10):
         """
@@ -296,31 +299,19 @@ class ParseLogsTest(OERuntimeTestCase):
         Returns a dictionary of log filenames to a dictionary of error lines to
         the error context (controlled by @lines_before and @lines_after).
         """
-        results = {}
-        rez = []
-        grep_output = ''
+        results = collections.defaultdict(dict)
 
         for log in logs:
-            result = None
-            thegrep = self.build_grepcmd(log)
-
-            try:
-                result = check_output(thegrep, shell=True).decode('utf-8')
-            except:
-                pass
+            with open(log) as f:
+                lines = f.readlines()
 
-            if result is not None:
-                results[log] = {}
-                rez = result.splitlines()
+            for i, line in enumerate(lines):
+                line = line.strip()
+                line_lower = line.casefold()
 
-                for xrez in rez:
-                    try:
-                        cmd = ['grep', '-F', xrez, '-B', str(lines_before)]
-                        cmd += ['-A', str(lines_after), log]
-                        grep_output = check_output(cmd).decode('utf-8')
-                    except:
-                        pass
-                    results[log][xrez]=grep_output
+                if any(keyword in line_lower for keyword in self.errors):
+                    if not any(ignore in line_lower for ignore in self.ignore_errors):
+                        results[log][line] = "".join(self.get_context(lines, i, lines_before, lines_after))
 
         return results
 
@@ -342,9 +333,9 @@ class ParseLogsTest(OERuntimeTestCase):
             self.msg += '-----------------------\n'
             for error in result[log]:
                 errcount += 1
-                self.msg += 'Central error: ' + str(error) + '\n'
+                self.msg += 'Central error: ' + error + '\n'
                 self.msg +=  '***********************\n'
-                self.msg +=  result[str(log)][str(error)] + '\n'
+                self.msg +=  result[log][error] + '\n'
                 self.msg +=  '***********************\n'
         self.msg += '%s errors found in logs.' % errcount
         self.assertEqual(errcount, 0, msg=self.msg)
Signature · 1 line
2.34.1