oeqa: Fix subTest result reporting
PATCH 1/1 oeqa: Fix subTest result reporting
Fixes the way that subTest results are reported so that each subTest is reported independently, with it's own status. The base test case is also reported, but only if it actually had a status.
--- meta/lib/oeqa/core/runner.py | 44 +++++++++++++++++---- meta/lib/oeqa/core/utils/concurrencytest.py | 3 ++ 2 files changed, 39 insertions(+), 8 deletions(-)
meta/lib/oeqa/core/runner.py +36 -8
@@ -40,6 +40,7 @@ class OETestResult(_TestResult): super(OETestResult, self).__init__(*args, **kwargs) self.successes = [] + self.subtests = {} self.starttime = {} self.endtime = {} self.progressinfo = {}
@@ -145,6 +146,13 @@ class OETestResult(_TestResult): else: self.extraresults[k] = v + def addSubTest(self, test, subtest, err, details=None): + self.extractExtraResults(test, details=details) + self.subtests.setdefault(test, list()).append(subtest) + if err is None: + self.successes.append((subtest, None)) + return super(OETestResult, self).addSubTest(test, subtest, err) + def addError(self, test, *args, details = None): self.extractExtraResults(test, details = details) return super(OETestResult, self).addError(test, *args)
@@ -169,18 +177,18 @@ class OETestResult(_TestResult): def logDetails(self, json_file_dir=None, configuration=None, result_id=None, dump_streams=False): - self.tc.logger.info("RESULTS:") - - result = self.extraresults - logs = {} - if hasattr(self.tc, "extraresults"): - result.update(self.tc.extraresults) - for case_name in self.tc._registry['cases']: - case = self.tc._registry['cases'][case_name] + def logCaseDetails(case, required): + nonlocal self + nonlocal result + nonlocal dump_streams + nonlocal logs (status, log) = self._getTestResultDetails(case) + if status == "UNKNOWN" and not required: + return + t = "" if case.id() in self.starttime and case.id() in self.endtime: t = " (" + "{0:.2f}".format(self.endtime[case.id()] - self.starttime[case.id()]) + "s)"
@@ -197,6 +205,26 @@ class OETestResult(_TestResult): report['stderr'] = stderr result[case.id()] = report + self.tc.logger.info("RESULTS:") + + result = self.extraresults + logs = {} + if hasattr(self.tc, "extraresults"): + result.update(self.tc.extraresults) + + for case_name in self.tc._registry['cases']: + case = self.tc._registry['cases'][case_name] + + if case in self.subtests: + for subtest in self.subtests[case]: + logCaseDetails(subtest, True) + + # The base case may not have any reported details. If it comes + # up as unknown, don't report anything. + logCaseDetails(case, False) + else: + logCaseDetails(case, True) + for i in ['PASSED', 'SKIPPED', 'EXPECTEDFAIL', 'ERROR', 'FAILED', 'UNKNOWN']: if i not in logs: continue
meta/lib/oeqa/core/utils/concurrencytest.py +3 -0
@@ -81,6 +81,9 @@ class ProxyTestResult: def _addResult(self, method, test, *args, exception = False, **kwargs): return method(test, *args, **kwargs) + def addSubTest(self, test, subtest, outcome, **kwargs): + self._addResult(self.result.addSubTest, test, subtest, outcome, exception = (outcome is not None), **kwargs) + def addError(self, test, err = None, **kwargs): self._addResult(self.result.addError, test, err, exception = True, **kwargs)
Signature · 1 line
2.23.0
Hi Joshua
Quoted 6 lines · Joshua Watt
Fixes the way that subTest results are reported so that each subTest is
reported independently, with it's own status. The base test case is also
reported, but only if it actually had a status.
Quoted 7 lines
Signed-off-by Joshua Watt <[email protected]>--- meta/lib/oeqa/core/runner.py | 44 +++++++++++++++++ ---- meta/lib/oeqa/core/utils/concurrencytest.py | 3 ++ 2 files changed, 39 insertions(+), 8 deletions(-)
This is causing errors:
https://autobuilder.yoctoproject.org/typhoon/#/builders/79/builds/476
https://errors.yoctoproject.org/Errors/Details/275668/
Traceback (most recent call last):
File "/home/pokybuild/yocto-worker/oe-selftest-centos/build/meta/lib/oeqa/core/utils/concurrencytest.py", line 299, in fork_for_tests
process_suite.run(ExtraResultsEncoderTestResult(subunit_result))
File "/usr/lib64/python3.6/unittest/suite.py", line 122, in run
test(result)
File "/usr/lib64/python3.6/unittest/case.py", line 670, in __call__
return self.run(*args, **kwds)
File "/usr/lib64/python3.6/unittest/case.py", line 630, in run
self._feedErrorsToResult(result, outcome.errors)
File "/usr/lib64/python3.6/unittest/case.py", line 555, in_feedErrorsToResult
result.addSubTest(test.test_case, test, exc_info)
File "/home/pokybuild/yocto-worker/oe-selftest-centos/build/meta/lib/oeqa/core/utils/concurrencytest.py", line 85, in addSubTest
self._addResult(self.result.addSubTest, test, subtest, outcome,
exception = (outcome is not None), **kwargs)
AttributeError: 'AutoTimingTestResultDecorator' object has no attribute 'addSubTest'
2019-11-11 19:14:43,928 - oe-selftest - INFO -
runtime_test.Postinst.test_failing_postinst (subunit.RemotedTestCas
Thanks,
Anuj