Skip to contentSkip to conversation

resulttool: Handle multiple series containing ptestresults

PATCH New Nathan Rossi 1 participant 4 messages Sep 11, 2019

lore
Share & tools
lore
#0 lore raw

PATCH v2 1/4 resulttool: Handle multiple series containing ptestresults

Handle multiple results series having ptestresults content. The contents are merged on a per-result basis where duplicates are ignored (with a warning message printed). The 'ptestresults.sections' collection is also merged on a per-suite basis.

---
 scripts/lib/resulttool/report.py | 44 +++++++++++++++++++++++++++++-----------
 1 file changed, 32 insertions(+), 12 deletions(-)
scripts/lib/resulttool/report.py +32 -12
index 8b03717d29..883b52517b 100644
@@ -32,16 +32,22 @@ class ResultsTextReport(object):
             # Ensure tests without any test results still show up on the report
             for suite in result['ptestresult.sections']:
                 if suite not in self.ptests[machine]:
-                    self.ptests[machine][suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []}
+                    self.ptests[machine][suite] = {
+                            'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-',
+                            'failed_testcases': [], "testcases": set(),
+                            }
                 if 'duration' in result['ptestresult.sections'][suite]:
                     self.ptests[machine][suite]['duration'] = result['ptestresult.sections'][suite]['duration']
                 if 'timeout' in result['ptestresult.sections'][suite]:
                     self.ptests[machine][suite]['duration'] += " T"
-            return
+            return True
+
+        # process test result
         try:
             _, suite, test = k.split(".", 2)
         except ValueError:
-            return
+            return True
+
         # Handle 'glib-2.0'
         if 'ptestresult.sections' in result and suite not in result['ptestresult.sections']:
             try:
@@ -50,11 +56,23 @@ class ResultsTextReport(object):
                     suite = suite + "." + suite1
             except ValueError:
                 pass
+
         if suite not in self.ptests[machine]:
-            self.ptests[machine][suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []}
+            self.ptests[machine][suite] = {
+                    'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-',
+                    'failed_testcases': [], "testcases": set(),
+                    }
+
+        # do not process duplicate results
+        if test in self.ptests[machine][suite]["testcases"]:
+            print("Warning duplicate ptest result '{}.{}' for {}".format(suite, test, machine))
+            return False
+
         for tk in self.result_types:
             if status in self.result_types[tk]:
                 self.ptests[machine][suite][tk] += 1
+        self.ptests[machine][suite]["testcases"].add(test)
+        return True
 
     def handle_ltptest_result(self, k, status, result, machine):
         if machine not in self.ltptests:
@@ -124,17 +142,20 @@ class ResultsTextReport(object):
         result = testresult.get('result', [])
         for k in result:
             test_status = result[k].get('status', [])
+            if k.startswith("ptestresult."):
+                if not self.handle_ptest_result(k, test_status, result, machine):
+                    continue
+            elif k.startswith("ltpresult."):
+                self.handle_ltptest_result(k, test_status, result, machine)
+            elif k.startswith("ltpposixresult."):
+                self.handle_ltpposixtest_result(k, test_status, result, machine)
+
+            # process result if it was not skipped by a handler
             for tk in self.result_types:
                 if test_status in self.result_types[tk]:
                     test_count_report[tk] += 1
             if test_status in self.result_types['failed']:
                 test_count_report['failed_testcases'].append(k)
-            if k.startswith("ptestresult."):
-                self.handle_ptest_result(k, test_status, result, machine)
-            if k.startswith("ltpresult."):
-                self.handle_ltptest_result(k, test_status, result, machine)
-            if k.startswith("ltpposixresult."):
-                self.handle_ltpposixtest_result(k, test_status, result, machine)
         return test_count_report
 
     def print_test_report(self, template_file_name, test_count_reports):
@@ -210,8 +231,7 @@ class ResultsTextReport(object):
                 # Check to see if there is already results for these kinds of tests for the machine
                 for key in result['result'].keys():
                     testtype = str(key).split('.')[0]
-                    if ((machine in self.ptests and testtype == "ptestresult" and self.ptests[machine]) or
-                        (machine in self.ltptests and testtype == "ltpiresult" and self.ltptests[machine]) or
+                    if ((machine in self.ltptests and testtype == "ltpiresult" and self.ltptests[machine]) or
                         (machine in self.ltpposixtests and testtype == "ltpposixresult" and self.ltpposixtests[machine])):
                         print("Already have test results for %s on %s, skipping %s" %(str(key).split('.')[0], machine, resultid))
                         skip = True

---
2.23.0.rc1

#1 lore raw

PATCH v2 2/4 gcc-cross.inc: Process binaries in build dir to be relocatable

Process binaries within the build directory before stashing to be relocatable with ORIGIN relative rpaths. This corrects issues with rpaths being invalid when trying to use the binaries from an unstashed build directory (e.g. gcc-runtime).

---
Changes in v2:
- Nest the stash directory within 'stashed-builddir' to match the nesting of the copied build directory, this ensures that the relative rpaths are correct when extracting to a similar build directory.

---
 meta/recipes-devtools/gcc/gcc-cross.inc | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)
meta/recipes-devtools/gcc/gcc-cross.inc +9 -1
index 95af6d89a9..8855bb1f34 100644
@@ -206,9 +206,17 @@ do_package_write_ipk[noexec] = "1"
 do_package_write_rpm[noexec] = "1"
 do_package_write_deb[noexec] = "1"
 
-BUILDDIRSTASH = "${WORKDIR}/stashed-builddir"
+inherit chrpath
+
+python gcc_stash_builddir_fixrpaths() {
+    # rewrite rpaths, breaking hardlinks as required
+    process_dir("/", d.getVar("BUILDDIRSTASH"), d, break_hardlinks = True)
+}
+
+BUILDDIRSTASH = "${WORKDIR}/stashed-builddir/build"
 do_gcc_stash_builddir[dirs] = "${B}"
 do_gcc_stash_builddir[cleandirs] = "${BUILDDIRSTASH}"
+do_gcc_stash_builddir[postfuncs] += "gcc_stash_builddir_fixrpaths"
 do_gcc_stash_builddir () {
 	dest=${BUILDDIRSTASH}
 	hardlinkdir . $dest

---
2.23.0.rc1

#2 lore raw

PATCH v2 3/4 oeqa/core/case.py: Add OEPTestResultTestCase for ptestresult helpers

In reply to Nathan Rossi

Add the OEPTestResultTestCase class as a mix-in class to provide helper functions for interacting with ptestresults within the extraresults object generated by the test case.

This class also provides default compression of log text and log files.

Also add support to resulttool for decoding/decompressing log files embedded in the test results.

---
Changes in v2:
- Fix up resultutils so that it can decode compressed logs
- Move common log decoding from log to resultutils module, and simplify uses of log access

---
 meta/lib/oeqa/core/case.py            | 49 +++++++++++++++++++++++++++++++++++
 scripts/lib/resulttool/log.py         | 43 +++++++++++++++---------------
 scripts/lib/resulttool/resultutils.py | 45 +++++++++++++++++++++++++++-----
 3 files changed, 109 insertions(+), 28 deletions(-)
meta/lib/oeqa/core/case.py +49 -0
index aca144e9dc..180635ac6c 100644
@@ -4,6 +4,8 @@
 # SPDX-License-Identifier: MIT
 #
 
+import base64
+import zlib
 import unittest
 
 from oeqa.core.exception import OEQAMissingVariable
@@ -49,3 +51,50 @@ class OETestCase(unittest.TestCase):
         for d in self.decorators:
             d.tearDownDecorator()
         self.tearDownMethod()
+
+class OEPTestResultTestCase:
+    """
+    Mix-in class to provide functions to make interacting with extraresults for
+    the purposes of storing ptestresult data.
+    """
+    @staticmethod
+    def _compress_log(log):
+        logdata = log.encode("utf-8")
+        logdata = zlib.compress(logdata)
+        logdata = base64.b64encode(logdata).decode("utf-8")
+        return {"compressed" : logdata}
+
+    def ptest_rawlog(self, log):
+        if not hasattr(self, "extraresults"):
+            self.extraresults = {"ptestresult.sections" : {}}
+        self.extraresults["ptestresult.rawlogs"] = {"log" : self._compress_log(log)}
+
+    def ptest_section(self, section, duration = None, log = None, logfile = None, exitcode = None):
+        if not hasattr(self, "extraresults"):
+            self.extraresults = {"ptestresult.sections" : {}}
+
+        sections = self.extraresults.get("ptestresult.sections")
+        if section not in sections:
+            sections[section] = {}
+
+        if log is not None:
+            sections[section]["log"] = self._compress_log(log)
+        elif logfile is not None:
+            with open(logfile, "r") as f:
+                sections[section]["log"] = self._compress_log(f.read())
+
+        if duration is not None:
+            sections[section]["duration"] = duration
+        if exitcode is not None:
+            sections[section]["exitcode"] = exitcode
+
+    def ptest_result(self, section, test, result):
+        if not hasattr(self, "extraresults"):
+            self.extraresults = {"ptestresult.sections" : {}}
+
+        sections = self.extraresults.get("ptestresult.sections")
+        if section not in sections:
+            sections[section] = {}
+        resultname = "ptestresult.{}.{}".format(section, test)
+        self.extraresults[resultname] = {"status" : result}
+
scripts/lib/resulttool/log.py +21 -22

64 lines not on this page · raw message

scripts/lib/resulttool/resultutils.py +39 -6

65 lines not on this page · raw message

---
2.23.0.rc1

#3 lore raw

PATCH v2 4/4 oeqa/selftest: Rework toolchain tests to use OEPTestResultTestCase

In reply to Nathan Rossi

Use OEPTestResultTestCase to collect results and add logfile collection and compression.

---
Changes in v2:
- Fixed up missing logpath setting for gcc.py.

---
 meta/lib/oeqa/selftest/cases/binutils.py | 11 ++++++-----
 meta/lib/oeqa/selftest/cases/gcc.py      |  9 +++++----
 meta/lib/oeqa/selftest/cases/glibc.py    |  7 ++++---
 3 files changed, 15 insertions(+), 12 deletions(-)
meta/lib/oeqa/selftest/cases/binutils.py +6 -5
index 9bc752040f..821f52f5a8 100644
@@ -4,6 +4,7 @@ import sys
 import re
 import logging
 from oeqa.core.decorator import OETestTag
+from oeqa.core.case import OEPTestResultTestCase
 from oeqa.selftest.case import OESelftestTestCase
 from oeqa.utils.commands import bitbake, get_bb_var, get_bb_vars
 
@@ -15,7 +16,7 @@ def parse_values(content):
                 break
 
 @OETestTag("toolchain-user", "toolchain-system")
-class BinutilsCrossSelfTest(OESelftestTestCase):
+class BinutilsCrossSelfTest(OESelftestTestCase, OEPTestResultTestCase):
     def test_binutils(self):
         self.run_binutils("binutils")
 
@@ -36,14 +37,14 @@ class BinutilsCrossSelfTest(OESelftestTestCase):
 
         bitbake("{0} -c check".format(recipe))
 
-        ptestsuite = "binutils-{}".format(suite) if suite != "binutils" else suite
-        self.extraresults = {"ptestresult.sections" : {ptestsuite : {}}}
-
         sumspath = os.path.join(builddir, suite, "{0}.sum".format(suite))
         if not os.path.exists(sumspath):
             sumspath = os.path.join(builddir, suite, "testsuite", "{0}.sum".format(suite))
+        logpath = os.path.splitext(sumspath)[0] + ".log"
 
+        ptestsuite = "binutils-{}".format(suite) if suite != "binutils" else suite
+        self.ptest_section(ptestsuite, logfile = logpath)
         with open(sumspath, "r") as f:
             for test, result in parse_values(f):
-                self.extraresults["ptestresult.{}.{}".format(ptestsuite, test)] = {"status" : result}
+                self.ptest_result(ptestsuite, test, result)
 
meta/lib/oeqa/selftest/cases/gcc.py +5 -4
index 2c25b5904c..5a917b9c44 100644
@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: MIT
 import os
 from oeqa.core.decorator import OETestTag
+from oeqa.core.case import OEPTestResultTestCase
 from oeqa.selftest.case import OESelftestTestCase
 from oeqa.utils.commands import bitbake, get_bb_var, get_bb_vars, runqemu, Command
 
@@ -11,7 +12,7 @@ def parse_values(content):
                 yield i[len(v) + 2:].strip(), v
                 break
 
-class GccSelfTestBase(OESelftestTestCase):
+class GccSelfTestBase(OESelftestTestCase, OEPTestResultTestCase):
     def check_skip(self, suite):
         targets = get_bb_var("RUNTIMETARGET", "gcc-runtime").split()
         if suite not in targets:
@@ -41,20 +42,20 @@ class GccSelfTestBase(OESelftestTestCase):
         bb_vars = get_bb_vars(["B", "TARGET_SYS"], recipe)
         builddir, target_sys = bb_vars["B"], bb_vars["TARGET_SYS"]
 
-        self.extraresults = {"ptestresult.sections" : {}}
         for suite in suites:
             sumspath = os.path.join(builddir, "gcc", "testsuite", suite, "{0}.sum".format(suite))
             if not os.path.exists(sumspath): # check in target dirs
                 sumspath = os.path.join(builddir, target_sys, suite, "testsuite", "{0}.sum".format(suite))
             if not os.path.exists(sumspath): # handle libstdc++-v3 -> libstdc++
                 sumspath = os.path.join(builddir, target_sys, suite, "testsuite", "{0}.sum".format(suite.split("-")[0]))
+            logpath = os.path.splitext(sumspath)[0] + ".log"
 
             ptestsuite = "gcc-{}".format(suite) if suite != "gcc" else suite
             ptestsuite = ptestsuite + "-user" if ssh is None else ptestsuite
-            self.extraresults["ptestresult.sections"][ptestsuite] = {}
+            self.ptest_section(ptestsuite, logfile = logpath)
             with open(sumspath, "r") as f:
                 for test, result in parse_values(f):
-                    self.extraresults["ptestresult.{}.{}".format(ptestsuite, test)] = {"status" : result}
+                    self.ptest_result(ptestsuite, test, result)
 
     def run_check_emulated(self, *args, **kwargs):
         # build core-image-minimal with required packages
meta/lib/oeqa/selftest/cases/glibc.py +4 -3
index 2e42485dd6..c687f6ef93 100644
@@ -2,6 +2,7 @@
 import os
 import contextlib
 from oeqa.core.decorator import OETestTag
+from oeqa.core.case import OEPTestResultTestCase
 from oeqa.selftest.case import OESelftestTestCase
 from oeqa.utils.commands import bitbake, get_bb_var, get_bb_vars, runqemu, Command
 from oeqa.utils.nfs import unfs_server
@@ -13,7 +14,7 @@ def parse_values(content):
                 yield i[len(v) + 2:].strip(), v
                 break
 
-class GlibcSelfTestBase(OESelftestTestCase):
+class GlibcSelfTestBase(OESelftestTestCase, OEPTestResultTestCase):
     def run_check(self, ssh = None):
         # configure ssh target
         features = []
@@ -31,10 +32,10 @@ class GlibcSelfTestBase(OESelftestTestCase):
         builddir = get_bb_var("B", "glibc-testsuite")
 
         ptestsuite = "glibc-user" if ssh is None else "glibc"
-        self.extraresults = {"ptestresult.sections" : {ptestsuite : {}}}
+        self.ptest_section(ptestsuite)
         with open(os.path.join(builddir, "tests.sum"), "r") as f:
             for test, result in parse_values(f):
-                self.extraresults["ptestresult.{}.{}".format(ptestsuite, test)] = {"status" : result}
+                self.ptest_result(ptestsuite, test, result)
 
     def run_check_emulated(self):
         with contextlib.ExitStack() as s:

---
2.23.0.rc1