resulttool: Handle multiple series containing ptestresults
PATCH 1/6 resulttool: Handle multiple series containing ptestresults
Handle multiple results series having ptestresults content. The contents are merged on a per-result basis where duplicates are ignored (with a warning message printed). The 'ptestresults.sections' collection is also merged on a per-suite basis.
--- scripts/lib/resulttool/report.py | 44 +++++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 12 deletions(-)
scripts/lib/resulttool/report.py +32 -12
@@ -32,16 +32,22 @@ class ResultsTextReport(object): # Ensure tests without any test results still show up on the report for suite in result['ptestresult.sections']: if suite not in self.ptests[machine]: - self.ptests[machine][suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []} + self.ptests[machine][suite] = { + 'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', + 'failed_testcases': [], "testcases": set(), + } if 'duration' in result['ptestresult.sections'][suite]: self.ptests[machine][suite]['duration'] = result['ptestresult.sections'][suite]['duration'] if 'timeout' in result['ptestresult.sections'][suite]: self.ptests[machine][suite]['duration'] += " T" - return + return True + + # process test result try: _, suite, test = k.split(".", 2) except ValueError: - return + return True + # Handle 'glib-2.0' if 'ptestresult.sections' in result and suite not in result['ptestresult.sections']: try:
@@ -50,11 +56,23 @@ class ResultsTextReport(object): suite = suite + "." + suite1 except ValueError: pass + if suite not in self.ptests[machine]: - self.ptests[machine][suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []} + self.ptests[machine][suite] = { + 'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', + 'failed_testcases': [], "testcases": set(), + } + + # do not process duplicate results + if test in self.ptests[machine][suite]["testcases"]: + print("Warning duplicate ptest result '{}.{}' for {}".format(suite, test, machine)) + return False + for tk in self.result_types: if status in self.result_types[tk]: self.ptests[machine][suite][tk] += 1 + self.ptests[machine][suite]["testcases"].add(test) + return True def handle_ltptest_result(self, k, status, result, machine): if machine not in self.ltptests:
@@ -124,17 +142,20 @@ class ResultsTextReport(object): result = testresult.get('result', []) for k in result: test_status = result[k].get('status', []) + if k.startswith("ptestresult."): + if not self.handle_ptest_result(k, test_status, result, machine): + continue + elif k.startswith("ltpresult."): + self.handle_ltptest_result(k, test_status, result, machine) + elif k.startswith("ltpposixresult."): + self.handle_ltpposixtest_result(k, test_status, result, machine) + + # process result if it was not skipped by a handler for tk in self.result_types: if test_status in self.result_types[tk]: test_count_report[tk] += 1 if test_status in self.result_types['failed']: test_count_report['failed_testcases'].append(k) - if k.startswith("ptestresult."): - self.handle_ptest_result(k, test_status, result, machine) - if k.startswith("ltpresult."): - self.handle_ltptest_result(k, test_status, result, machine) - if k.startswith("ltpposixresult."): - self.handle_ltpposixtest_result(k, test_status, result, machine) return test_count_report def print_test_report(self, template_file_name, test_count_reports):
@@ -210,8 +231,7 @@ class ResultsTextReport(object): # Check to see if there is already results for these kinds of tests for the machine for key in result['result'].keys(): testtype = str(key).split('.')[0] - if ((machine in self.ptests and testtype == "ptestresult" and self.ptests[machine]) or - (machine in self.ltptests and testtype == "ltpiresult" and self.ltptests[machine]) or + if ((machine in self.ltptests and testtype == "ltpiresult" and self.ltptests[machine]) or (machine in self.ltpposixtests and testtype == "ltpposixresult" and self.ltpposixtests[machine])): print("Already have test results for %s on %s, skipping %s" %(str(key).split('.')[0], machine, resultid)) skip = True
---
2.23.0.rc1
PATCH 2/6 oeqa/selftest/context.py: For -t/-T use append argparse action
Use the 'append' action of argparse instead of nargs. This changes the behaviour of the option from "-t foo bar -r" to "-t foo -t bar -r".
Additionally rename the long form options to be consistent with behaviour, such that they specifying a single tag at a time.
--- meta/lib/oeqa/selftest/context.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-)
meta/lib/oeqa/selftest/context.py +6 -6
@@ -78,12 +78,12 @@ class OESelftestTestContextExecutor(OETestContextExecutor): parser.add_argument('--machine', required=False, choices=['random', 'all'], help='Run tests on different machines (random/all).') - parser.add_argument('-t', '--select-tags', dest="select_tags", - nargs='*', default=None, - help='Filter all (unhidden) tests to any that match any of the specified tags.') - parser.add_argument('-T', '--exclude-tags', dest="exclude_tags", - nargs='*', default=None, - help='Exclude all (unhidden) tests that match any of the specified tags. (exclude applies before select)') + parser.add_argument('-t', '--select-tag', dest="select_tags", + action='append', default=None, + help='Filter all (unhidden) tests to any that match any of the specified tag(s).') + parser.add_argument('-T', '--exclude-tag', dest="exclude_tags", + action='append', default=None, + help='Exclude all (unhidden) tests that match any of the specified tag(s). (exclude applies before select)') parser.set_defaults(func=self.run)
---
2.23.0.rc1
PATCH 3/6 chrpath.bbclass: Add break_hardlinks kwarg to allow breaking hardlinks
In reply to Nathan RossiAdd the break_hardlinks kwarg to break hardlinks when modifying files. This uses the bb.utils.break_hardlinks function to break hardlinks. The default is to maintain existing behaviour and leave hardlinks in place.
--- meta/classes/chrpath.bbclass | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-)
meta/classes/chrpath.bbclass +13 -6
@@ -1,7 +1,7 @@ CHRPATH_BIN ?= "chrpath" PREPROCESS_RELOCATE_DIRS ?= "" -def process_file_linux(cmd, fpath, rootdir, baseprefix, tmpdir, d): +def process_file_linux(cmd, fpath, rootdir, baseprefix, tmpdir, d, break_hardlinks = False): import subprocess as sub p = sub.Popen([cmd, '-l', fpath],stdout=sub.PIPE,stderr=sub.PIPE)
@@ -39,6 +39,9 @@ def process_file_linux(cmd, fpath, rootdir, baseprefix, tmpdir, d): # if we have modified some rpaths call chrpath to update the binary if modified: + if break_hardlinks: + bb.utils.break_hardlinks(fpath) + args = ":".join(new_rpaths) #bb.note("Setting rpath for %s to %s" %(fpath, args)) p = sub.Popen([cmd, '-r', args, fpath],stdout=sub.PIPE,stderr=sub.PIPE)
@@ -46,7 +49,7 @@ def process_file_linux(cmd, fpath, rootdir, baseprefix, tmpdir, d): if p.returncode != 0: bb.fatal("%s: chrpath command failed with exit code %d:\n%s%s" % (d.getVar('PN'), p.returncode, out, err)) -def process_file_darwin(cmd, fpath, rootdir, baseprefix, tmpdir, d): +def process_file_darwin(cmd, fpath, rootdir, baseprefix, tmpdir, d, break_hardlinks = False): import subprocess as sub p = sub.Popen([d.expand("${HOST_PREFIX}otool"), '-L', fpath],stdout=sub.PIPE,stderr=sub.PIPE)
@@ -61,11 +64,14 @@ def process_file_darwin(cmd, fpath, rootdir, baseprefix, tmpdir, d): if baseprefix not in rpath: continue + if break_hardlinks: + bb.utils.break_hardlinks(fpath) + newpath = "@loader_path/" + os.path.relpath(rpath, os.path.dirname(fpath.replace(rootdir, "/"))) p = sub.Popen([d.expand("${HOST_PREFIX}install_name_tool"), '-change', rpath, newpath, fpath],stdout=sub.PIPE,stderr=sub.PIPE) out, err = p.communicate() -def process_dir (rootdir, directory, d): +def process_dir(rootdir, directory, d, break_hardlinks = False): import stat rootdir = os.path.normpath(rootdir)
@@ -95,7 +101,7 @@ def process_dir (rootdir, directory, d): continue if os.path.isdir(fpath): - process_dir(rootdir, fpath, d) + process_dir(rootdir, fpath, d, break_hardlinks = break_hardlinks) else: #bb.note("Testing %s for relocatability" % fpath)
@@ -108,8 +114,9 @@ def process_dir (rootdir, directory, d): else: # Temporarily make the file writeable so we can chrpath it os.chmod(fpath, perms|stat.S_IRWXU) - process_file(cmd, fpath, rootdir, baseprefix, tmpdir, d) - + + process_file(cmd, fpath, rootdir, baseprefix, tmpdir, d, break_hardlinks = break_hardlinks) + if perms: os.chmod(fpath, perms)
---
2.23.0.rc1
PATCH 4/6 gcc-cross.inc: Process binaries in build dir to be relocatable
In reply to Nathan RossiProcess binaries within the build directory before stashing to be relocatable with ORIGIN relative rpaths. This corrects issues with rpaths being invalid when trying to use the binaries from an unstashed build directory (e.g. gcc-runtime).
--- meta/recipes-devtools/gcc/gcc-cross.inc | 8 ++++++++ 1 file changed, 8 insertions(+)
meta/recipes-devtools/gcc/gcc-cross.inc +8 -0
@@ -206,9 +206,17 @@ do_package_write_ipk[noexec] = "1" do_package_write_rpm[noexec] = "1" do_package_write_deb[noexec] = "1" +inherit chrpath + +python gcc_stash_builddir_fixrpaths() { + # rewrite rpaths, breaking hardlinks as required + process_dir("/", d.getVar("BUILDDIRSTASH"), d, break_hardlinks = True) +} + BUILDDIRSTASH = "${WORKDIR}/stashed-builddir" do_gcc_stash_builddir[dirs] = "${B}" do_gcc_stash_builddir[cleandirs] = "${BUILDDIRSTASH}" +do_gcc_stash_builddir[postfuncs] += "gcc_stash_builddir_fixrpaths" do_gcc_stash_builddir () { dest=${BUILDDIRSTASH} hardlinkdir . $dest
---
2.23.0.rc1
PATCH 5/6 oeqa/core/case.py: Add OEPTestResultTestCase for ptestresult helpers
In reply to Nathan RossiAdd the OEPTestResultTestCase class as a mix-in class to provide helper functions for interacting with ptestresults within the extraresults object generated by the test case.
This class also provides default compression of log text and log files.
Also add support to resulttool for decoding/decompressing log files embedded in the test results.
--- meta/lib/oeqa/core/case.py | 49 +++++++++++++++++++++++++++++++++++++++++++ scripts/lib/resulttool/log.py | 12 ++++++++++- 2 files changed, 60 insertions(+), 1 deletion(-)
meta/lib/oeqa/core/case.py +49 -0
@@ -4,6 +4,8 @@ # SPDX-License-Identifier: MIT # +import base64 +import zlib import unittest from oeqa.core.exception import OEQAMissingVariable
@@ -49,3 +51,50 @@ class OETestCase(unittest.TestCase): for d in self.decorators: d.tearDownDecorator() self.tearDownMethod() + +class OEPTestResultTestCase: + """ + Mix-in class to provide functions to make interacting with extraresults for + the purposes of storing ptestresult data. + """ + @staticmethod + def _compress_log(log): + logdata = log.encode("utf-8") + logdata = zlib.compress(logdata) + logdata = base64.b64encode(logdata).decode("utf-8") + return {"compressed" : logdata} + + def ptest_rawlog(self, log): + if not hasattr(self, "extraresults"): + self.extraresults = {"ptestresult.sections" : {}} + self.extraresults["ptestresult.rawlogs"] = {"log" : self._compress_log(log)} + + def ptest_section(self, section, duration = None, log = None, logfile = None, exitcode = None): + if not hasattr(self, "extraresults"): + self.extraresults = {"ptestresult.sections" : {}} + + sections = self.extraresults.get("ptestresult.sections") + if section not in sections: + sections[section] = {} + + if log is not None: + sections[section]["log"] = self._compress_log(log) + elif logfile is not None: + with open(logfile, "r") as f: + sections[section]["log"] = self._compress_log(f.read()) + + if duration is not None: + sections[section]["duration"] = duration + if exitcode is not None: + sections[section]["exitcode"] = exitcode + + def ptest_result(self, section, test, result): + if not hasattr(self, "extraresults"): + self.extraresults = {"ptestresult.sections" : {}} + + sections = self.extraresults.get("ptestresult.sections") + if section not in sections: + sections[section] = {} + resultname = "ptestresult.{}.{}".format(section, test) + self.extraresults[resultname] = {"status" : result} +
scripts/lib/resulttool/log.py +11 -1
@@ -5,6 +5,8 @@ # SPDX-License-Identifier: GPL-2.0-only # import os +import base64 +import zlib import resulttool.resultutils as resultutils def show_ptest(result, ptest, logger):
@@ -25,6 +27,9 @@ def show_reproducible(result, reproducible, logger): print("reproducible '%s' not found" % reproducible) return 1 +def decode_compressed_log(logdata): + data = base64.b64decode(logdata.encode("utf-8")) + return zlib.decompress(data).decode("utf-8") def log(args, logger): results = resultutils.load_resultsdata(args.source)
@@ -39,6 +44,7 @@ def log(args, logger): if 'ptestresult.sections' in r: for name, ptest in r['ptestresult.sections'].items(): if 'log' in ptest: + logdata = ptest['log'] dest_dir = args.dump_ptest if args.prepend_run: dest_dir = os.path.join(dest_dir, run_name)
@@ -48,7 +54,11 @@ def log(args, logger): dest = os.path.join(dest_dir, '%s.log' % name) print(dest) with open(dest, 'w') as f: - f.write(ptest['log']) + if isinstance(logdata, str): + f.write(logdata) + elif isinstance(logdata, dict): + if "compressed" in logdata: + f.write(decode_compressed_log(logdata.get("compressed"))) if args.raw_ptest: if 'ptestresult.rawlogs' in r:
---
2.23.0.rc1
PATCH 6/6 oeqa/selftest: Rework toolchain tests to use OEPTestResultTestCase
In reply to Nathan RossiUse OEPTestResultTestCase to collect results and add logfile collection and compression.
--- meta/lib/oeqa/selftest/cases/binutils.py | 11 ++++++----- meta/lib/oeqa/selftest/cases/gcc.py | 8 ++++---- meta/lib/oeqa/selftest/cases/glibc.py | 7 ++++--- 3 files changed, 14 insertions(+), 12 deletions(-)
meta/lib/oeqa/selftest/cases/binutils.py +6 -5
@@ -4,6 +4,7 @@ import sys import re import logging from oeqa.core.decorator import OETestTag +from oeqa.core.case import OEPTestResultTestCase from oeqa.selftest.case import OESelftestTestCase from oeqa.utils.commands import bitbake, get_bb_var, get_bb_vars
@@ -15,7 +16,7 @@ def parse_values(content): break @OETestTag("toolchain-user", "toolchain-system") -class BinutilsCrossSelfTest(OESelftestTestCase): +class BinutilsCrossSelfTest(OESelftestTestCase, OEPTestResultTestCase): def test_binutils(self): self.run_binutils("binutils")
@@ -36,14 +37,14 @@ class BinutilsCrossSelfTest(OESelftestTestCase): bitbake("{0} -c check".format(recipe)) - ptestsuite = "binutils-{}".format(suite) if suite != "binutils" else suite - self.extraresults = {"ptestresult.sections" : {ptestsuite : {}}} - sumspath = os.path.join(builddir, suite, "{0}.sum".format(suite)) if not os.path.exists(sumspath): sumspath = os.path.join(builddir, suite, "testsuite", "{0}.sum".format(suite)) + logpath = os.path.splitext(sumspath)[0] + ".log" + ptestsuite = "binutils-{}".format(suite) if suite != "binutils" else suite + self.ptest_section(ptestsuite, logfile = logpath) with open(sumspath, "r") as f: for test, result in parse_values(f): - self.extraresults["ptestresult.{}.{}".format(ptestsuite, test)] = {"status" : result} + self.ptest_result(ptestsuite, test, result)
meta/lib/oeqa/selftest/cases/gcc.py +4 -4
@@ -1,6 +1,7 @@ # SPDX-License-Identifier: MIT import os from oeqa.core.decorator import OETestTag +from oeqa.core.case import OEPTestResultTestCase from oeqa.selftest.case import OESelftestTestCase from oeqa.utils.commands import bitbake, get_bb_var, get_bb_vars, runqemu, Command
@@ -11,7 +12,7 @@ def parse_values(content): yield i[len(v) + 2:].strip(), v break -class GccSelfTestBase(OESelftestTestCase): +class GccSelfTestBase(OESelftestTestCase, OEPTestResultTestCase): def check_skip(self, suite): targets = get_bb_var("RUNTIMETARGET", "gcc-runtime").split() if suite not in targets:
@@ -41,7 +42,6 @@ class GccSelfTestBase(OESelftestTestCase): bb_vars = get_bb_vars(["B", "TARGET_SYS"], recipe) builddir, target_sys = bb_vars["B"], bb_vars["TARGET_SYS"] - self.extraresults = {"ptestresult.sections" : {}} for suite in suites: sumspath = os.path.join(builddir, "gcc", "testsuite", suite, "{0}.sum".format(suite)) if not os.path.exists(sumspath): # check in target dirs
@@ -51,10 +51,10 @@ class GccSelfTestBase(OESelftestTestCase): ptestsuite = "gcc-{}".format(suite) if suite != "gcc" else suite ptestsuite = ptestsuite + "-user" if ssh is None else ptestsuite - self.extraresults["ptestresult.sections"][ptestsuite] = {} + self.ptest_section(ptestsuite, logfile = logpath) with open(sumspath, "r") as f: for test, result in parse_values(f): - self.extraresults["ptestresult.{}.{}".format(ptestsuite, test)] = {"status" : result} + self.ptest_result(ptestsuite, test, result) def run_check_emulated(self, *args, **kwargs): # build core-image-minimal with required packages
meta/lib/oeqa/selftest/cases/glibc.py +4 -3
@@ -2,6 +2,7 @@ import os import contextlib from oeqa.core.decorator import OETestTag +from oeqa.core.case import OEPTestResultTestCase from oeqa.selftest.case import OESelftestTestCase from oeqa.utils.commands import bitbake, get_bb_var, get_bb_vars, runqemu, Command from oeqa.utils.nfs import unfs_server
@@ -13,7 +14,7 @@ def parse_values(content): yield i[len(v) + 2:].strip(), v break -class GlibcSelfTestBase(OESelftestTestCase): +class GlibcSelfTestBase(OESelftestTestCase, OEPTestResultTestCase): def run_check(self, ssh = None): # configure ssh target features = []
@@ -31,10 +32,10 @@ class GlibcSelfTestBase(OESelftestTestCase): builddir = get_bb_var("B", "glibc-testsuite") ptestsuite = "glibc-user" if ssh is None else "glibc" - self.extraresults = {"ptestresult.sections" : {ptestsuite : {}}} + self.ptest_section(ptestsuite) with open(os.path.join(builddir, "tests.sum"), "r") as f: for test, result in parse_values(f): - self.extraresults["ptestresult.{}.{}".format(ptestsuite, test)] = {"status" : result} + self.ptest_result(ptestsuite, test, result) def run_check_emulated(self): with contextlib.ExitStack() as s:
---
2.23.0.rc1