oeqa/core/runner.py: Fix OETestTag listing
PATCH 1/7 oeqa/core/runner.py: Fix OETestTag listing
Use the __oeqa_testtags attribute added by OETestTag and display no, one or more tags separated by a comma. Also change the formatting of the output so that the list of tests is formatted as "<test> (<tags>)" and remove the table header for "id" (it is no longer available).
--- meta/lib/oeqa/core/runner.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-)
meta/lib/oeqa/core/runner.py +8 -11
@@ -205,23 +205,20 @@ class OETestRunner(_TestRunner): self._walked_cases = self._walked_cases + 1 def _list_tests_name(self, suite): - from oeqa.core.decorator.oetag import OETestTag - self._walked_cases = 0 def _list_cases(logger, case): - oetag = None - - if hasattr(case, 'decorators'): - for d in case.decorators: - if isinstance(d, OETestTag): - oetag = d.oetag - - logger.info("%s\t\t%s" % (oetag, case.id())) + oetags = [] + if hasattr(case, '__oeqa_testtags'): + oetags = getattr(case, '__oeqa_testtags') + if oetags: + logger.info("%s (%s)" % (case.id(), ",".join(oetags))) + else: + logger.info("%s" % (case.id())) self.tc.logger.info("Listing all available tests:") self._walked_cases = 0 - self.tc.logger.info("id\ttag\t\ttest") + self.tc.logger.info("test (tags)") self.tc.logger.info("-" * 80) self._walk_suite(suite, _list_cases) self.tc.logger.info("-" * 80)
---
2.23.0.rc1
PATCH 2/7 oeqa/core: Implement proper extra result collection and serialization
Implement handling of extra result (e.g. ptestresult) collection with the addition of a "extraresults" extraction function in OETestResult. In order to be able to serialize and deserialize the extraresults data, allow OETestResult add* calls to take a details kwarg. The subunit module can handle cross-process transfer of binary data for the details kwarg. With a TestResult proxy class to sit inbetween to encode and decode to and from json.
--- meta/lib/oeqa/core/runner.py | 41 +++++++++++++++++-- meta/lib/oeqa/core/utils/concurrencytest.py | 61 ++++++++++++++++++++++++++++- 2 files changed, 96 insertions(+), 6 deletions(-)
meta/lib/oeqa/core/runner.py +37 -4
@@ -43,6 +43,7 @@ class OETestResult(_TestResult): self.starttime = {} self.endtime = {} self.progressinfo = {} + self.extraresults = {} # Inject into tc so that TestDepends decorator can see results tc.results = self
@@ -129,19 +130,51 @@ class OETestResult(_TestResult): return 'UNKNOWN', None - def addSuccess(self, test): + def extractExtraResults(self, test, details = None): + extraresults = None + if details is not None and "extraresults" in details: + extraresults = details.get("extraresults", {}) + elif hasattr(test, "extraresults"): + extraresults = test.extraresults + + if extraresults is not None: + for k, v in extraresults.items(): + # handle updating already existing entries (e.g. ptestresults.sections) + if k in self.extraresults: + self.extraresults[k].update(v) + else: + self.extraresults[k] = v + + def addError(self, test, *args, details = None): + self.extractExtraResults(test, details = details) + return super(OETestResult, self).addError(test, *args) + + def addFailure(self, test, *args, details = None): + self.extractExtraResults(test, details = details) + return super(OETestResult, self).addFailure(test, *args) + + def addSuccess(self, test, details = None): #Added so we can keep track of successes too self.successes.append((test, None)) - super(OETestResult, self).addSuccess(test) + self.extractExtraResults(test, details = details) + return super(OETestResult, self).addSuccess(test) + + def addExpectedFailure(self, test, *args, details = None): + self.extractExtraResults(test, details = details) + return super(OETestResult, self).addExpectedFailure(test, *args) + + def addUnexpectedSuccess(self, test, details = None): + self.extractExtraResults(test, details = details) + return super(OETestResult, self).addUnexpectedSuccess(test) def logDetails(self, json_file_dir=None, configuration=None, result_id=None, dump_streams=False): self.tc.logger.info("RESULTS:") - result = {} + result = self.extraresults logs = {} if hasattr(self.tc, "extraresults"): - result = self.tc.extraresults + result.update(self.tc.extraresults) for case_name in self.tc._registry['cases']: case = self.tc._registry['cases'][case_name]
meta/lib/oeqa/core/utils/concurrencytest.py +59 -2
91 lines not on this page · raw message
---
2.23.0.rc1
PATCH 3/7 oeqa/selftest: Use extraresults on self instead of self.tc
In reply to Nathan RossiIn order to take advantage of multiprocess execution of tests the extraresults must be passed through the TestResult. With changes to how oeqa/core handles test cases the extraresults attribute of the testcase is passed to the TestResult, with passing across process boundaries handled automatically.
--- meta/lib/oeqa/selftest/cases/binutils.py | 13 ++----------- meta/lib/oeqa/selftest/cases/gcc.py | 14 +++----------- meta/lib/oeqa/selftest/cases/glibc.py | 13 ++----------- meta/lib/oeqa/selftest/cases/reproducible.py | 13 +++++-------- 4 files changed, 12 insertions(+), 41 deletions(-)
meta/lib/oeqa/selftest/cases/binutils.py +2 -11
@@ -16,15 +16,6 @@ def parse_values(content): @OETestTag("machine") class BinutilsCrossSelfTest(OESelftestTestCase): - @classmethod - def setUpClass(cls): - super().setUpClass() - if not hasattr(cls.tc, "extraresults"): - cls.tc.extraresults = {} - - if "ptestresult.sections" not in cls.tc.extraresults: - cls.tc.extraresults["ptestresult.sections"] = {} - def test_binutils(self): self.run_binutils("binutils")
@@ -46,7 +37,7 @@ class BinutilsCrossSelfTest(OESelftestTestCase): bitbake("{0} -c check".format(recipe)) ptestsuite = "binutils-{}".format(suite) if suite != "binutils" else suite - self.tc.extraresults["ptestresult.sections"][ptestsuite] = {} + self.extraresults = {"ptestresult.sections" : {ptestsuite : {}}} sumspath = os.path.join(builddir, suite, "{0}.sum".format(suite)) if not os.path.exists(sumspath):
@@ -54,5 +45,5 @@ class BinutilsCrossSelfTest(OESelftestTestCase): with open(sumspath, "r") as f: for test, result in parse_values(f): - self.tc.extraresults["ptestresult.{}.{}".format(ptestsuite, test)] = {"status" : result} + self.extraresults["ptestresult.{}.{}".format(ptestsuite, test)] = {"status" : result}
meta/lib/oeqa/selftest/cases/gcc.py +3 -11
@@ -13,15 +13,6 @@ def parse_values(content): @OETestTag("machine") class GccSelfTest(OESelftestTestCase): - @classmethod - def setUpClass(cls): - super().setUpClass() - if not hasattr(cls.tc, "extraresults"): - cls.tc.extraresults = {} - - if "ptestresult.sections" not in cls.tc.extraresults: - cls.tc.extraresults["ptestresult.sections"] = {} - def gcc_runtime_check_skip(self, suite): targets = get_bb_var("RUNTIMETARGET", "gcc-runtime").split() if suite not in targets:
@@ -71,6 +62,7 @@ class GccSelfTest(OESelftestTestCase): bb_vars = get_bb_vars(["B", "TARGET_SYS"], recipe) builddir, target_sys = bb_vars["B"], bb_vars["TARGET_SYS"] + self.extraresults = {"ptestresult.sections" : {}} for suite in suites: sumspath = os.path.join(builddir, "gcc", "testsuite", suite, "{0}.sum".format(suite)) if not os.path.exists(sumspath): # check in target dirs
@@ -80,10 +72,10 @@ class GccSelfTest(OESelftestTestCase): ptestsuite = "gcc-{}".format(suite) if suite != "gcc" else suite ptestsuite = ptestsuite + "-user" if ssh is None else ptestsuite - self.tc.extraresults["ptestresult.sections"][ptestsuite] = {} + self.extraresults["ptestresult.sections"][ptestsuite] = {} with open(sumspath, "r") as f: for test, result in parse_values(f): - self.tc.extraresults["ptestresult.{}.{}".format(ptestsuite, test)] = {"status" : result} + self.extraresults["ptestresult.{}.{}".format(ptestsuite, test)] = {"status" : result} class GccSelfTestSystemEmulated(GccSelfTest): default_installed_packages = ["libgcc", "libstdc++", "libatomic", "libgomp"]
meta/lib/oeqa/selftest/cases/glibc.py +2 -11
@@ -15,15 +15,6 @@ def parse_values(content): @OETestTag("machine") class GlibcSelfTest(OESelftestTestCase): - @classmethod - def setUpClass(cls): - super().setUpClass() - if not hasattr(cls.tc, "extraresults"): - cls.tc.extraresults = {} - - if "ptestresult.sections" not in cls.tc.extraresults: - cls.tc.extraresults["ptestresult.sections"] = {} - def test_glibc(self): self.glibc_run_check()
@@ -44,10 +35,10 @@ class GlibcSelfTest(OESelftestTestCase): builddir = get_bb_var("B", "glibc-testsuite") ptestsuite = "glibc-user" if ssh is None else "glibc" - self.tc.extraresults["ptestresult.sections"][ptestsuite] = {} + self.extraresults = {"ptestresult.sections" : {ptestsuite : {}}} with open(os.path.join(builddir, "tests.sum"), "r") as f: for test, result in parse_values(f): - self.tc.extraresults["ptestresult.{}.{}".format(ptestsuite, test)] = {"status" : result} + self.extraresults["ptestresult.{}.{}".format(ptestsuite, test)] = {"status" : result} class GlibcSelfTestSystemEmulated(GlibcSelfTest): default_installed_packages = [
meta/lib/oeqa/selftest/cases/reproducible.py +5 -8
@@ -82,15 +82,12 @@ class ReproducibleTests(OESelftestTestCase): for v in needed_vars: setattr(self, v.lower(), bb_vars[v]) - if not hasattr(self.tc, "extraresults"): - self.tc.extraresults = {} - self.extras = self.tc.extraresults - - self.extras.setdefault('reproducible.rawlogs', {})['log'] = '' - self.extras.setdefault('reproducible', {}).setdefault('files', {}) + self.extrasresults = {} + self.extrasresults.setdefault('reproducible.rawlogs', {})['log'] = '' + self.extrasresults.setdefault('reproducible', {}).setdefault('files', {}) def append_to_log(self, msg): - self.extras['reproducible.rawlogs']['log'] += msg + self.extrasresults['reproducible.rawlogs']['log'] += msg def compare_packages(self, reference_dir, test_dir, diffutils_sysroot): result = PackageCompareResults()
@@ -117,7 +114,7 @@ class ReproducibleTests(OESelftestTestCase): return result def write_package_list(self, package_class, name, packages): - self.extras['reproducible']['files'].setdefault(package_class, {})[name] = [ + self.extrasresults['reproducible']['files'].setdefault(package_class, {})[name] = [ {'reference': p.reference, 'test': p.test} for p in packages] def test_reproducible_builds(self):
---
2.23.0.rc1
PATCH 4/7 oeqa/selftest/context.py: Change -t/-T args to be optional
In reply to Nathan RossiChange the -t/-T args to be optional so that they can be used together with the existing -r/-a/... args to run a more flexible filtering of test tags.
--- meta/lib/oeqa/selftest/context.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-)
meta/lib/oeqa/selftest/context.py +19 -11
@@ -61,12 +61,6 @@ class OESelftestTestContextExecutor(OETestContextExecutor): group.add_argument('-r', '--run-tests', required=False, action='store', nargs='+', dest="run_tests", default=None, help='Select what tests to run (modules, classes or test methods). Format should be: <module>.<class>.<test_method>') - group.add_argument('-t', '--run-only-tags', action='store', - nargs='+', dest="run_only_tags", default=None, - help='Run all (unhidden) tests which match any of the specified tags.') - group.add_argument('-T', '--run-exclude-tags', action='store', - nargs='+', dest="run_exclude_tags", default=None, - help='Run all (unhidden) tests excluding any that match any of the specified tags.') group.add_argument('-m', '--list-modules', required=False, action="store_true", default=False,
@@ -83,7 +77,14 @@ class OESelftestTestContextExecutor(OETestContextExecutor): parser.add_argument('--machine', required=False, choices=['random', 'all'], help='Run tests on different machines (random/all).') - + + parser.add_argument('-t', '--select-tags', dest="select_tags", + nargs='*', default=None, + help='Filter all (unhidden) tests to any that match any of the specified tags.') + parser.add_argument('-T', '--exclude-tags', dest="exclude_tags", + nargs='*', default=None, + help='Exclude all (unhidden) tests that match any of the specified tags. (exclude applies before select)') + parser.set_defaults(func=self.run) def _get_available_machines(self):
@@ -155,10 +156,17 @@ class OESelftestTestContextExecutor(OETestContextExecutor): copyfile(self.tc_kwargs['init']['config_paths']['bblayers'], self.tc_kwargs['init']['config_paths']['bblayers_backup']) - if args.run_only_tags: - self.tc_kwargs['load']['tags_filter'] = lambda tags: not tags or not any(tag in args.run_only_tags for tag in tags) - if args.run_exclude_tags: - self.tc_kwargs['load']['tags_filter'] = lambda tags: any(tag in args.run_exclude_tags for tag in tags) + def tag_filter(tags): + if args.exclude_tags: + if any(tag in args.exclude_tags for tag in tags): + return True + if args.select_tags: + if not tags or not any(tag in args.select_tags for tag in tags): + return True + return False + + if args.select_tags or args.exclude_tags: + self.tc_kwargs['load']['tags_filter'] = tag_filter self.tc_kwargs['run']['skips'] = args.skips self.tc_kwargs['run']['processes'] = args.processes
---
2.23.0.rc1
PATCH 5/7 oeqa/core/decorator: Fix super class modifying subclass tags
In reply to Nathan Rossi--- meta/lib/oeqa/core/decorator/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
meta/lib/oeqa/core/decorator/__init__.py +2 -1
@@ -70,7 +70,8 @@ def OETestTag(*tags): expandedtags += strToList(tag) def decorator(item): if hasattr(item, "__oeqa_testtags"): - item.__oeqa_testtags += expandedtags + # do not append, create a new list (to handle classes with inheritance) + item.__oeqa_testtags = list(item.__oeqa_testtags) + expandedtags else: item.__oeqa_testtags = expandedtags return item
---
2.23.0.rc1
PATCH 6/7 oeqa/selftest/cases/gcc.py: Split into classes for parallelism
In reply to Nathan RossiSplit the gcc selftest cases into multiple classes one for each test. This is done in order to make it easy to execute multiple gcc tests in parallel when using oe-selftest with the '-j' arg.
Additionally tag the user tests with "toolchain-user" and the system emulation (qemu system) tests with "toolchain-system".
--- meta/lib/oeqa/selftest/cases/gcc.py | 99 ++++++++++++++++++++++++++----------- 1 file changed, 70 insertions(+), 29 deletions(-)
meta/lib/oeqa/selftest/cases/gcc.py +70 -29
@@ -11,34 +11,13 @@ def parse_values(content): yield i[len(v) + 2:].strip(), v break -@OETestTag("machine") -class GccSelfTest(OESelftestTestCase): - def gcc_runtime_check_skip(self, suite): +class GccSelfTestBase(OESelftestTestCase): + def check_skip(self, suite): targets = get_bb_var("RUNTIMETARGET", "gcc-runtime").split() if suite not in targets: self.skipTest("Target does not use {0}".format(suite)) - def test_cross_gcc(self): - self.gcc_run_check("gcc", "g++") - - def test_libatomic(self): - self.gcc_run_check("libatomic") - - def test_libgomp(self): - self.gcc_run_check("libgomp") - - def test_libstdcxx(self): - self.gcc_run_check("libstdc++-v3") - - def test_libssp(self): - self.gcc_runtime_check_skip("libssp") - self.gcc_run_check("libssp") - - def test_libitm(self): - self.gcc_runtime_check_skip("libitm") - self.gcc_run_check("libitm") - - def gcc_run_check(self, *suites, ssh = None): + def run_check(self, *suites, ssh = None): targets = set() for s in suites: if s in ["gcc", "g++"]:
@@ -77,11 +56,9 @@ class GccSelfTest(OESelftestTestCase): for test, result in parse_values(f): self.extraresults["ptestresult.{}.{}".format(ptestsuite, test)] = {"status" : result} -class GccSelfTestSystemEmulated(GccSelfTest): - default_installed_packages = ["libgcc", "libstdc++", "libatomic", "libgomp"] - - def gcc_run_check(self, *args, **kwargs): + def run_check_emulated(self, *args, **kwargs): # build core-image-minimal with required packages + default_installed_packages = ["libgcc", "libstdc++", "libatomic", "libgomp"] features = [] features.append('IMAGE_FEATURES += "ssh-server-openssh"') features.append('CORE_IMAGE_EXTRA_INSTALL += "{0}"'.format(" ".join(self.default_installed_packages)))
@@ -94,5 +71,69 @@ class GccSelfTestSystemEmulated(GccSelfTest): status, _ = qemu.run("uname") self.assertEqual(status, 0) - return super().gcc_run_check(*args, **kwargs, ssh = qemu.ip) + return self.run_check(*args, **kwargs, ssh = qemu.ip) + +@OETestTag("machine", "toolchain-user") +class GccCrossSelfTest(GccSelfTestBase): + def test_cross_gcc(self): + self.run_check("gcc", "g++") + +@OETestTag("machine", "toolchain-user") +class GccLibAtomicSelfTest(GccSelfTestBase): + def test_libatomic(self): + self.run_check("libatomic") + +@OETestTag("machine", "toolchain-user") +class GccLibGompSelfTest(GccSelfTestBase): + def test_libgomp(self): + self.run_check("libgomp") + +@OETestTag("machine", "toolchain-user") +class GccLibStdCxxSelfTest(GccSelfTestBase): + def test_libstdcxx(self): + self.run_check("libstdc++-v3") + +@OETestTag("machine", "toolchain-user") +class GccLibSspSelfTest(GccSelfTestBase): + def test_libssp(self): + self.check_skip("libssp") + self.run_check("libssp") + +@OETestTag("machine", "toolchain-user") +class GccLibItmSelfTest(GccSelfTestBase): + def test_libitm(self): + self.check_skip("libitm") + self.run_check("libitm") + +@OETestTag("machine", "toolchain-system") +class GccCrossSelfTestSystemEmulated(GccSelfTestBase): + def test_cross_gcc(self): + self.run_check_emulated("gcc", "g++") + +@OETestTag("machine", "toolchain-system") +class GccLibAtomicSelfTestSystemEmulated(GccSelfTestBase): + def test_libatomic(self): + self.run_check_emulated("libatomic") + +@OETestTag("machine", "toolchain-system") +class GccLibGompSelfTestSystemEmulated(GccSelfTestBase): + def test_libgomp(self): + self.run_check_emulated("libgomp") + +@OETestTag("machine", "toolchain-system") +class GccLibStdCxxSelfTestSystemEmulated(GccSelfTestBase): + def test_libstdcxx(self): + self.run_check_emulated("libstdc++-v3") + +@OETestTag("machine", "toolchain-system") +class GccLibSspSelfTestSystemEmulated(GccSelfTestBase): + def test_libssp(self): + self.check_skip("libssp") + self.run_check_emulated("libssp") + +@OETestTag("machine", "toolchain-system") +class GccLibItmSelfTestSystemEmulated(GccSelfTestBase): + def test_libitm(self): + self.check_skip("libitm") + self.run_check_emulated("libitm")
---
2.23.0.rc1
PATCH 7/7 oeqa/selftest/cases/glibc.py: Rework and tag with toolchain-user/system
In reply to Nathan RossiRework the glibc execution into a common base class. Additionally tag the tests with "toolchain-user" and "toolchain-system".
--- meta/lib/oeqa/selftest/cases/glibc.py | 44 +++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 20 deletions(-)
meta/lib/oeqa/selftest/cases/glibc.py +24 -20
@@ -13,12 +13,8 @@ def parse_values(content): yield i[len(v) + 2:].strip(), v break -@OETestTag("machine") -class GlibcSelfTest(OESelftestTestCase): - def test_glibc(self): - self.glibc_run_check() - - def glibc_run_check(self, ssh = None): +class GlibcSelfTestBase(OESelftestTestCase): + def run_check(self, ssh = None): # configure ssh target features = [] if ssh is not None:
@@ -40,25 +36,23 @@ class GlibcSelfTest(OESelftestTestCase): for test, result in parse_values(f): self.extraresults["ptestresult.{}.{}".format(ptestsuite, test)] = {"status" : result} -class GlibcSelfTestSystemEmulated(GlibcSelfTest): - default_installed_packages = [ - "glibc-charmaps", - "libgcc", - "libstdc++", - "libatomic", - "libgomp", - # "python3", - # "python3-pexpect", - "nfs-utils", - ] - - def glibc_run_check(self): + def run_check_emulated(self): with contextlib.ExitStack() as s: # use the base work dir, as the nfs mount, since the recipe directory may not exist tmpdir = get_bb_var("BASE_WORKDIR") nfsport, mountport = s.enter_context(unfs_server(tmpdir)) # build core-image-minimal with required packages + default_installed_packages = [ + "glibc-charmaps", + "libgcc", + "libstdc++", + "libatomic", + "libgomp", + # "python3", + # "python3-pexpect", + "nfs-utils", + ] features = [] features.append('IMAGE_FEATURES += "ssh-server-openssh"') features.append('CORE_IMAGE_EXTRA_INSTALL += "{0}"'.format(" ".join(self.default_installed_packages)))
@@ -80,5 +74,15 @@ class GlibcSelfTestSystemEmulated(GlibcSelfTest): if status != 0: raise Exception("Failed to setup NFS mount on target ({})".format(repr(output))) - super().glibc_run_check(ssh = qemu.ip) + self.run_check(ssh = qemu.ip) + +@OETestTag("machine", "toolchain-user") +class GlibcSelfTest(GlibcSelfTestBase): + def test_glibc(self): + self.run_check() + +@OETestTag("machine", "toolchain-system") +class GlibcSelfTestSystemEmulated(GlibcSelfTestBase): + def test_glibc(self): + self.run_check_emulated()
---
2.23.0.rc1
✗ patchtest: failure for "oeqa/core/runner.py: Fix OETes..." and 6 more
In reply to Nathan Rossi== Series Details ==
Series: "oeqa/core/runner.py: Fix OETes..." and 6 more
Revision: 1
URL : https://patchwork.openembedded.org/series/19766/
State : failure
== Summary ==
Thank you for submitting this patch series to OpenEmbedded Core. This is an automated response. Several tests have been executed on the proposed series by patchtest resulting in the following failures:
* Issue Series does not apply on top of target branch [test_series_merge_on_head]
Suggested fix Rebase your series on top of targeted branch
Targeted branch master (currently at 868e84c57e)
If you believe any of these test results are incorrect, please reply to the mailing list ([email protected]) raising your concerns. Otherwise we would appreciate you correcting the issues and submitting a new version of the patchset if applicable. Please ensure you add/increment the version number when sending the new version (i.e. [PATCH] -> [PATCH v2] -> [PATCH v3] -> ...).
---
Guidelines: https://www.openembedded.org/wiki/Commit_Patch_Message_Guidelines Test framework: http://git.yoctoproject.org/cgit/cgit.cgi/patchtest Test suite: http://git.yoctoproject.org/cgit/cgit.cgi/patchtest-oe