Skip to contentSkip to conversation

resulttool: Handle multiple series containing ptestresults

PATCH Superseded Nathan Rossi 1 participant 6 messages Sep 10, 2019

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

PATCH 1/6 resulttool: Handle multiple series containing ptestresults

Handle multiple results series having ptestresults content. The contents are merged on a per-result basis where duplicates are ignored (with a warning message printed). The 'ptestresults.sections' collection is also merged on a per-suite basis.

---
 scripts/lib/resulttool/report.py | 44 +++++++++++++++++++++++++++++-----------
 1 file changed, 32 insertions(+), 12 deletions(-)
scripts/lib/resulttool/report.py +32 -12
index 8b03717d29..883b52517b 100644
@@ -32,16 +32,22 @@ class ResultsTextReport(object):
             # Ensure tests without any test results still show up on the report
             for suite in result['ptestresult.sections']:
                 if suite not in self.ptests[machine]:
-                    self.ptests[machine][suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []}
+                    self.ptests[machine][suite] = {
+                            'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-',
+                            'failed_testcases': [], "testcases": set(),
+                            }
                 if 'duration' in result['ptestresult.sections'][suite]:
                     self.ptests[machine][suite]['duration'] = result['ptestresult.sections'][suite]['duration']
                 if 'timeout' in result['ptestresult.sections'][suite]:
                     self.ptests[machine][suite]['duration'] += " T"
-            return
+            return True
+
+        # process test result
         try:
             _, suite, test = k.split(".", 2)
         except ValueError:
-            return
+            return True
+
         # Handle 'glib-2.0'
         if 'ptestresult.sections' in result and suite not in result['ptestresult.sections']:
             try:
@@ -50,11 +56,23 @@ class ResultsTextReport(object):
                     suite = suite + "." + suite1
             except ValueError:
                 pass
+
         if suite not in self.ptests[machine]:
-            self.ptests[machine][suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []}
+            self.ptests[machine][suite] = {
+                    'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-',
+                    'failed_testcases': [], "testcases": set(),
+                    }
+
+        # do not process duplicate results
+        if test in self.ptests[machine][suite]["testcases"]:
+            print("Warning duplicate ptest result '{}.{}' for {}".format(suite, test, machine))
+            return False
+
         for tk in self.result_types:
             if status in self.result_types[tk]:
                 self.ptests[machine][suite][tk] += 1
+        self.ptests[machine][suite]["testcases"].add(test)
+        return True
 
     def handle_ltptest_result(self, k, status, result, machine):
         if machine not in self.ltptests:
@@ -124,17 +142,20 @@ class ResultsTextReport(object):
         result = testresult.get('result', [])
         for k in result:
             test_status = result[k].get('status', [])
+            if k.startswith("ptestresult."):
+                if not self.handle_ptest_result(k, test_status, result, machine):
+                    continue
+            elif k.startswith("ltpresult."):
+                self.handle_ltptest_result(k, test_status, result, machine)
+            elif k.startswith("ltpposixresult."):
+                self.handle_ltpposixtest_result(k, test_status, result, machine)
+
+            # process result if it was not skipped by a handler
             for tk in self.result_types:
                 if test_status in self.result_types[tk]:
                     test_count_report[tk] += 1
             if test_status in self.result_types['failed']:
                 test_count_report['failed_testcases'].append(k)
-            if k.startswith("ptestresult."):
-                self.handle_ptest_result(k, test_status, result, machine)
-            if k.startswith("ltpresult."):
-                self.handle_ltptest_result(k, test_status, result, machine)
-            if k.startswith("ltpposixresult."):
-                self.handle_ltpposixtest_result(k, test_status, result, machine)
         return test_count_report
 
     def print_test_report(self, template_file_name, test_count_reports):
@@ -210,8 +231,7 @@ class ResultsTextReport(object):
                 # Check to see if there is already results for these kinds of tests for the machine
                 for key in result['result'].keys():
                     testtype = str(key).split('.')[0]
-                    if ((machine in self.ptests and testtype == "ptestresult" and self.ptests[machine]) or
-                        (machine in self.ltptests and testtype == "ltpiresult" and self.ltptests[machine]) or
+                    if ((machine in self.ltptests and testtype == "ltpiresult" and self.ltptests[machine]) or
                         (machine in self.ltpposixtests and testtype == "ltpposixresult" and self.ltpposixtests[machine])):
                         print("Already have test results for %s on %s, skipping %s" %(str(key).split('.')[0], machine, resultid))
                         skip = True

---
2.23.0.rc1

#1 lore raw

PATCH 2/6 oeqa/selftest/context.py: For -t/-T use append argparse action

Use the 'append' action of argparse instead of nargs. This changes the behaviour of the option from "-t foo bar -r" to "-t foo -t bar -r".

Additionally rename the long form options to be consistent with behaviour, such that they specifying a single tag at a time.

---
 meta/lib/oeqa/selftest/context.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)
meta/lib/oeqa/selftest/context.py +6 -6
index 3126ada716..c4eb5d614e 100644
@@ -78,12 +78,12 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
         parser.add_argument('--machine', required=False, choices=['random', 'all'],
                             help='Run tests on different machines (random/all).')
 
-        parser.add_argument('-t', '--select-tags', dest="select_tags",
-                nargs='*', default=None,
-                help='Filter all (unhidden) tests to any that match any of the specified tags.')
-        parser.add_argument('-T', '--exclude-tags', dest="exclude_tags",
-                nargs='*', default=None,
-                help='Exclude all (unhidden) tests that match any of the specified tags. (exclude applies before select)')
+        parser.add_argument('-t', '--select-tag', dest="select_tags",
+                action='append', default=None,
+                help='Filter all (unhidden) tests to any that match any of the specified tag(s).')
+        parser.add_argument('-T', '--exclude-tag', dest="exclude_tags",
+                action='append', default=None,
+                help='Exclude all (unhidden) tests that match any of the specified tag(s). (exclude applies before select)')
 
         parser.set_defaults(func=self.run)
 

---
2.23.0.rc1

#2 lore raw

PATCH 3/6 chrpath.bbclass: Add break_hardlinks kwarg to allow breaking hardlinks

In reply to Nathan Rossi

Add the break_hardlinks kwarg to break hardlinks when modifying files. This uses the bb.utils.break_hardlinks function to break hardlinks. The default is to maintain existing behaviour and leave hardlinks in place.

---
 meta/classes/chrpath.bbclass | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)
meta/classes/chrpath.bbclass +13 -6
index ad3c3975a5..2870c10d51 100644
@@ -1,7 +1,7 @@
 CHRPATH_BIN ?= "chrpath"
 PREPROCESS_RELOCATE_DIRS ?= ""
 
-def process_file_linux(cmd, fpath, rootdir, baseprefix, tmpdir, d):
+def process_file_linux(cmd, fpath, rootdir, baseprefix, tmpdir, d, break_hardlinks = False):
     import subprocess as sub
 
     p = sub.Popen([cmd, '-l', fpath],stdout=sub.PIPE,stderr=sub.PIPE)
@@ -39,6 +39,9 @@ def process_file_linux(cmd, fpath, rootdir, baseprefix, tmpdir, d):
 
     # if we have modified some rpaths call chrpath to update the binary
     if modified:
+        if break_hardlinks:
+            bb.utils.break_hardlinks(fpath)
+
         args = ":".join(new_rpaths)
         #bb.note("Setting rpath for %s to %s" %(fpath, args))
         p = sub.Popen([cmd, '-r', args, fpath],stdout=sub.PIPE,stderr=sub.PIPE)
@@ -46,7 +49,7 @@ def process_file_linux(cmd, fpath, rootdir, baseprefix, tmpdir, d):
         if p.returncode != 0:
             bb.fatal("%s: chrpath command failed with exit code %d:\n%s%s" % (d.getVar('PN'), p.returncode, out, err))
 
-def process_file_darwin(cmd, fpath, rootdir, baseprefix, tmpdir, d):
+def process_file_darwin(cmd, fpath, rootdir, baseprefix, tmpdir, d, break_hardlinks = False):
     import subprocess as sub
 
     p = sub.Popen([d.expand("${HOST_PREFIX}otool"), '-L', fpath],stdout=sub.PIPE,stderr=sub.PIPE)
@@ -61,11 +64,14 @@ def process_file_darwin(cmd, fpath, rootdir, baseprefix, tmpdir, d):
         if baseprefix not in rpath:
             continue
 
+        if break_hardlinks:
+            bb.utils.break_hardlinks(fpath)
+
         newpath = "@loader_path/" + os.path.relpath(rpath, os.path.dirname(fpath.replace(rootdir, "/")))
         p = sub.Popen([d.expand("${HOST_PREFIX}install_name_tool"), '-change', rpath, newpath, fpath],stdout=sub.PIPE,stderr=sub.PIPE)
         out, err = p.communicate()
 
-def process_dir (rootdir, directory, d):
+def process_dir(rootdir, directory, d, break_hardlinks = False):
     import stat
 
     rootdir = os.path.normpath(rootdir)
@@ -95,7 +101,7 @@ def process_dir (rootdir, directory, d):
             continue
 
         if os.path.isdir(fpath):
-            process_dir(rootdir, fpath, d)
+            process_dir(rootdir, fpath, d, break_hardlinks = break_hardlinks)
         else:
             #bb.note("Testing %s for relocatability" % fpath)
 
@@ -108,8 +114,9 @@ def process_dir (rootdir, directory, d):
             else:
                 # Temporarily make the file writeable so we can chrpath it
                 os.chmod(fpath, perms|stat.S_IRWXU)
-            process_file(cmd, fpath, rootdir, baseprefix, tmpdir, d)
-                
+
+            process_file(cmd, fpath, rootdir, baseprefix, tmpdir, d, break_hardlinks = break_hardlinks)
+
             if perms:
                 os.chmod(fpath, perms)
 

---
2.23.0.rc1

#3 lore raw

PATCH 4/6 gcc-cross.inc: Process binaries in build dir to be relocatable

In reply to Nathan Rossi

Process binaries within the build directory before stashing to be relocatable with ORIGIN relative rpaths. This corrects issues with rpaths being invalid when trying to use the binaries from an unstashed build directory (e.g. gcc-runtime).

---
 meta/recipes-devtools/gcc/gcc-cross.inc | 8 ++++++++
 1 file changed, 8 insertions(+)
meta/recipes-devtools/gcc/gcc-cross.inc +8 -0
index 95af6d89a9..e20f6cb6c1 100644
@@ -206,9 +206,17 @@ do_package_write_ipk[noexec] = "1"
 do_package_write_rpm[noexec] = "1"
 do_package_write_deb[noexec] = "1"
 
+inherit chrpath
+
+python gcc_stash_builddir_fixrpaths() {
+    # rewrite rpaths, breaking hardlinks as required
+    process_dir("/", d.getVar("BUILDDIRSTASH"), d, break_hardlinks = True)
+}
+
 BUILDDIRSTASH = "${WORKDIR}/stashed-builddir"
 do_gcc_stash_builddir[dirs] = "${B}"
 do_gcc_stash_builddir[cleandirs] = "${BUILDDIRSTASH}"
+do_gcc_stash_builddir[postfuncs] += "gcc_stash_builddir_fixrpaths"
 do_gcc_stash_builddir () {
 	dest=${BUILDDIRSTASH}
 	hardlinkdir . $dest

---
2.23.0.rc1

#4 lore raw

PATCH 5/6 oeqa/core/case.py: Add OEPTestResultTestCase for ptestresult helpers

In reply to Nathan Rossi

Add the OEPTestResultTestCase class as a mix-in class to provide helper functions for interacting with ptestresults within the extraresults object generated by the test case.

This class also provides default compression of log text and log files.

Also add support to resulttool for decoding/decompressing log files embedded in the test results.

---
 meta/lib/oeqa/core/case.py    | 49 +++++++++++++++++++++++++++++++++++++++++++
 scripts/lib/resulttool/log.py | 12 ++++++++++-
 2 files changed, 60 insertions(+), 1 deletion(-)
meta/lib/oeqa/core/case.py +49 -0
index aca144e9dc..180635ac6c 100644
@@ -4,6 +4,8 @@
 # SPDX-License-Identifier: MIT
 #
 
+import base64
+import zlib
 import unittest
 
 from oeqa.core.exception import OEQAMissingVariable
@@ -49,3 +51,50 @@ class OETestCase(unittest.TestCase):
         for d in self.decorators:
             d.tearDownDecorator()
         self.tearDownMethod()
+
+class OEPTestResultTestCase:
+    """
+    Mix-in class to provide functions to make interacting with extraresults for
+    the purposes of storing ptestresult data.
+    """
+    @staticmethod
+    def _compress_log(log):
+        logdata = log.encode("utf-8")
+        logdata = zlib.compress(logdata)
+        logdata = base64.b64encode(logdata).decode("utf-8")
+        return {"compressed" : logdata}
+
+    def ptest_rawlog(self, log):
+        if not hasattr(self, "extraresults"):
+            self.extraresults = {"ptestresult.sections" : {}}
+        self.extraresults["ptestresult.rawlogs"] = {"log" : self._compress_log(log)}
+
+    def ptest_section(self, section, duration = None, log = None, logfile = None, exitcode = None):
+        if not hasattr(self, "extraresults"):
+            self.extraresults = {"ptestresult.sections" : {}}
+
+        sections = self.extraresults.get("ptestresult.sections")
+        if section not in sections:
+            sections[section] = {}
+
+        if log is not None:
+            sections[section]["log"] = self._compress_log(log)
+        elif logfile is not None:
+            with open(logfile, "r") as f:
+                sections[section]["log"] = self._compress_log(f.read())
+
+        if duration is not None:
+            sections[section]["duration"] = duration
+        if exitcode is not None:
+            sections[section]["exitcode"] = exitcode
+
+    def ptest_result(self, section, test, result):
+        if not hasattr(self, "extraresults"):
+            self.extraresults = {"ptestresult.sections" : {}}
+
+        sections = self.extraresults.get("ptestresult.sections")
+        if section not in sections:
+            sections[section] = {}
+        resultname = "ptestresult.{}.{}".format(section, test)
+        self.extraresults[resultname] = {"status" : result}
+
scripts/lib/resulttool/log.py +11 -1
index 2352c767d9..99663772a3 100644
@@ -5,6 +5,8 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 import os
+import base64
+import zlib
 import resulttool.resultutils as resultutils
 
 def show_ptest(result, ptest, logger):
@@ -25,6 +27,9 @@ def show_reproducible(result, reproducible, logger):
         print("reproducible '%s' not found" % reproducible)
         return 1
 
+def decode_compressed_log(logdata):
+    data = base64.b64decode(logdata.encode("utf-8"))
+    return zlib.decompress(data).decode("utf-8")
 
 def log(args, logger):
     results = resultutils.load_resultsdata(args.source)
@@ -39,6 +44,7 @@ def log(args, logger):
             if 'ptestresult.sections' in r:
                 for name, ptest in r['ptestresult.sections'].items():
                     if 'log' in ptest:
+                        logdata = ptest['log']
                         dest_dir = args.dump_ptest
                         if args.prepend_run:
                             dest_dir = os.path.join(dest_dir, run_name)
@@ -48,7 +54,11 @@ def log(args, logger):
                         dest = os.path.join(dest_dir, '%s.log' % name)
                         print(dest)
                         with open(dest, 'w') as f:
-                            f.write(ptest['log'])
+                            if isinstance(logdata, str):
+                                f.write(logdata)
+                            elif isinstance(logdata, dict):
+                                if "compressed" in logdata:
+                                    f.write(decode_compressed_log(logdata.get("compressed")))
 
         if args.raw_ptest:
             if 'ptestresult.rawlogs' in r:

---
2.23.0.rc1

#5 lore raw

PATCH 6/6 oeqa/selftest: Rework toolchain tests to use OEPTestResultTestCase

In reply to Nathan Rossi

Use OEPTestResultTestCase to collect results and add logfile collection and compression.

---
 meta/lib/oeqa/selftest/cases/binutils.py | 11 ++++++-----
 meta/lib/oeqa/selftest/cases/gcc.py      |  8 ++++----
 meta/lib/oeqa/selftest/cases/glibc.py    |  7 ++++---
 3 files changed, 14 insertions(+), 12 deletions(-)
meta/lib/oeqa/selftest/cases/binutils.py +6 -5
index 9bc752040f..821f52f5a8 100644
@@ -4,6 +4,7 @@ import sys
 import re
 import logging
 from oeqa.core.decorator import OETestTag
+from oeqa.core.case import OEPTestResultTestCase
 from oeqa.selftest.case import OESelftestTestCase
 from oeqa.utils.commands import bitbake, get_bb_var, get_bb_vars
 
@@ -15,7 +16,7 @@ def parse_values(content):
                 break
 
 @OETestTag("toolchain-user", "toolchain-system")
-class BinutilsCrossSelfTest(OESelftestTestCase):
+class BinutilsCrossSelfTest(OESelftestTestCase, OEPTestResultTestCase):
     def test_binutils(self):
         self.run_binutils("binutils")
 
@@ -36,14 +37,14 @@ class BinutilsCrossSelfTest(OESelftestTestCase):
 
         bitbake("{0} -c check".format(recipe))
 
-        ptestsuite = "binutils-{}".format(suite) if suite != "binutils" else suite
-        self.extraresults = {"ptestresult.sections" : {ptestsuite : {}}}
-
         sumspath = os.path.join(builddir, suite, "{0}.sum".format(suite))
         if not os.path.exists(sumspath):
             sumspath = os.path.join(builddir, suite, "testsuite", "{0}.sum".format(suite))
+        logpath = os.path.splitext(sumspath)[0] + ".log"
 
+        ptestsuite = "binutils-{}".format(suite) if suite != "binutils" else suite
+        self.ptest_section(ptestsuite, logfile = logpath)
         with open(sumspath, "r") as f:
             for test, result in parse_values(f):
-                self.extraresults["ptestresult.{}.{}".format(ptestsuite, test)] = {"status" : result}
+                self.ptest_result(ptestsuite, test, result)
 
meta/lib/oeqa/selftest/cases/gcc.py +4 -4
index 2c25b5904c..6daa8d9e15 100644
@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: MIT
 import os
 from oeqa.core.decorator import OETestTag
+from oeqa.core.case import OEPTestResultTestCase
 from oeqa.selftest.case import OESelftestTestCase
 from oeqa.utils.commands import bitbake, get_bb_var, get_bb_vars, runqemu, Command
 
@@ -11,7 +12,7 @@ def parse_values(content):
                 yield i[len(v) + 2:].strip(), v
                 break
 
-class GccSelfTestBase(OESelftestTestCase):
+class GccSelfTestBase(OESelftestTestCase, OEPTestResultTestCase):
     def check_skip(self, suite):
         targets = get_bb_var("RUNTIMETARGET", "gcc-runtime").split()
         if suite not in targets:
@@ -41,7 +42,6 @@ class GccSelfTestBase(OESelftestTestCase):
         bb_vars = get_bb_vars(["B", "TARGET_SYS"], recipe)
         builddir, target_sys = bb_vars["B"], bb_vars["TARGET_SYS"]
 
-        self.extraresults = {"ptestresult.sections" : {}}
         for suite in suites:
             sumspath = os.path.join(builddir, "gcc", "testsuite", suite, "{0}.sum".format(suite))
             if not os.path.exists(sumspath): # check in target dirs
@@ -51,10 +51,10 @@ class GccSelfTestBase(OESelftestTestCase):
 
             ptestsuite = "gcc-{}".format(suite) if suite != "gcc" else suite
             ptestsuite = ptestsuite + "-user" if ssh is None else ptestsuite
-            self.extraresults["ptestresult.sections"][ptestsuite] = {}
+            self.ptest_section(ptestsuite, logfile = logpath)
             with open(sumspath, "r") as f:
                 for test, result in parse_values(f):
-                    self.extraresults["ptestresult.{}.{}".format(ptestsuite, test)] = {"status" : result}
+                    self.ptest_result(ptestsuite, test, result)
 
     def run_check_emulated(self, *args, **kwargs):
         # build core-image-minimal with required packages
meta/lib/oeqa/selftest/cases/glibc.py +4 -3
index 2e42485dd6..c687f6ef93 100644
@@ -2,6 +2,7 @@
 import os
 import contextlib
 from oeqa.core.decorator import OETestTag
+from oeqa.core.case import OEPTestResultTestCase
 from oeqa.selftest.case import OESelftestTestCase
 from oeqa.utils.commands import bitbake, get_bb_var, get_bb_vars, runqemu, Command
 from oeqa.utils.nfs import unfs_server
@@ -13,7 +14,7 @@ def parse_values(content):
                 yield i[len(v) + 2:].strip(), v
                 break
 
-class GlibcSelfTestBase(OESelftestTestCase):
+class GlibcSelfTestBase(OESelftestTestCase, OEPTestResultTestCase):
     def run_check(self, ssh = None):
         # configure ssh target
         features = []
@@ -31,10 +32,10 @@ class GlibcSelfTestBase(OESelftestTestCase):
         builddir = get_bb_var("B", "glibc-testsuite")
 
         ptestsuite = "glibc-user" if ssh is None else "glibc"
-        self.extraresults = {"ptestresult.sections" : {ptestsuite : {}}}
+        self.ptest_section(ptestsuite)
         with open(os.path.join(builddir, "tests.sum"), "r") as f:
             for test, result in parse_values(f):
-                self.extraresults["ptestresult.{}.{}".format(ptestsuite, test)] = {"status" : result}
+                self.ptest_result(ptestsuite, test, result)
 
     def run_check_emulated(self):
         with contextlib.ExitStack() as s:

---
2.23.0.rc1