Skip to contentSkip to conversation

oeqa: Fix subTest result reporting

PATCH In review Joshua Watt 2 participants 2 messages Nov 8, 2019 – Nov 12, 2019

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 oeqa: Fix subTest result reporting

Fixes the way that subTest results are reported so that each subTest is reported independently, with it's own status. The base test case is also reported, but only if it actually had a status.

---
 meta/lib/oeqa/core/runner.py                | 44 +++++++++++++++++----
 meta/lib/oeqa/core/utils/concurrencytest.py |  3 ++
 2 files changed, 39 insertions(+), 8 deletions(-)
meta/lib/oeqa/core/runner.py +36 -8
index f656e1a9c5f..ee9db6f7951 100644
@@ -40,6 +40,7 @@ class OETestResult(_TestResult):
         super(OETestResult, self).__init__(*args, **kwargs)
 
         self.successes = []
+        self.subtests = {}
         self.starttime = {}
         self.endtime = {}
         self.progressinfo = {}
@@ -145,6 +146,13 @@ class OETestResult(_TestResult):
                 else:
                     self.extraresults[k] = v
 
+    def addSubTest(self, test, subtest, err, details=None):
+        self.extractExtraResults(test, details=details)
+        self.subtests.setdefault(test, list()).append(subtest)
+        if err is None:
+            self.successes.append((subtest, None))
+        return super(OETestResult, self).addSubTest(test, subtest, err)
+
     def addError(self, test, *args, details = None):
         self.extractExtraResults(test, details = details)
         return super(OETestResult, self).addError(test, *args)
@@ -169,18 +177,18 @@ class OETestResult(_TestResult):
 
     def logDetails(self, json_file_dir=None, configuration=None, result_id=None,
             dump_streams=False):
-        self.tc.logger.info("RESULTS:")
-
-        result = self.extraresults
-        logs = {}
-        if hasattr(self.tc, "extraresults"):
-            result.update(self.tc.extraresults)
 
-        for case_name in self.tc._registry['cases']:
-            case = self.tc._registry['cases'][case_name]
+        def logCaseDetails(case, required):
+            nonlocal self
+            nonlocal result
+            nonlocal dump_streams
+            nonlocal logs
 
             (status, log) = self._getTestResultDetails(case)
 
+            if status == "UNKNOWN" and not required:
+                return
+
             t = ""
             if case.id() in self.starttime and case.id() in self.endtime:
                 t = " (" + "{0:.2f}".format(self.endtime[case.id()] - self.starttime[case.id()]) + "s)"
@@ -197,6 +205,26 @@ class OETestResult(_TestResult):
                 report['stderr'] = stderr
             result[case.id()] = report
 
+        self.tc.logger.info("RESULTS:")
+
+        result = self.extraresults
+        logs = {}
+        if hasattr(self.tc, "extraresults"):
+            result.update(self.tc.extraresults)
+
+        for case_name in self.tc._registry['cases']:
+            case = self.tc._registry['cases'][case_name]
+
+            if case in self.subtests:
+                for subtest in self.subtests[case]:
+                    logCaseDetails(subtest, True)
+
+                # The base case may not have any reported details. If it comes
+                # up as unknown, don't report anything.
+                logCaseDetails(case, False)
+            else:
+                logCaseDetails(case, True)
+
         for i in ['PASSED', 'SKIPPED', 'EXPECTEDFAIL', 'ERROR', 'FAILED', 'UNKNOWN']:
             if i not in logs:
                 continue
meta/lib/oeqa/core/utils/concurrencytest.py +3 -0
index 0f7b3dcc113..ae68fd50902 100644
@@ -81,6 +81,9 @@ class ProxyTestResult:
     def _addResult(self, method, test, *args, exception = False, **kwargs):
         return method(test, *args, **kwargs)
 
+    def addSubTest(self, test, subtest, outcome, **kwargs):
+        self._addResult(self.result.addSubTest, test, subtest, outcome, exception = (outcome is not None), **kwargs)
+
     def addError(self, test, err = None, **kwargs):
         self._addResult(self.result.addError, test, err, exception = True, **kwargs)
 
Signature · 1 line
2.23.0
#1 lore raw

Hi Joshua

Quoted 6 lines · Joshua Watt

Fixes the way that subTest results are reported so that each subTest is
reported independently, with it's own status. The base test case is also
reported, but only if it actually had a status.

Quoted 7 lines
---
 meta/lib/oeqa/core/runner.py                | 44 +++++++++++++++++
----
 meta/lib/oeqa/core/utils/concurrencytest.py |  3 ++
 2 files changed, 39 insertions(+), 8 deletions(-)

This is causing errors:

https://autobuilder.yoctoproject.org/typhoon/#/builders/79/builds/476
https://errors.yoctoproject.org/Errors/Details/275668/

Traceback (most recent call last):
File "/home/pokybuild/yocto-worker/oe-selftest-
centos/build/meta/lib/oeqa/core/utils/concurrencytest.py", line 299, in fork_for_tests
process_suite.run(ExtraResultsEncoderTestResult(subunit_result))
File "/usr/lib64/python3.6/unittest/suite.py", line 122, in run
test(result)
File "/usr/lib64/python3.6/unittest/case.py", line 670, in __call__
return self.run(*args, **kwds)
File "/usr/lib64/python3.6/unittest/case.py", line 630, in run
self._feedErrorsToResult(result, outcome.errors)
File "/usr/lib64/python3.6/unittest/case.py", line 555, in
_feedErrorsToResult
result.addSubTest(test.test_case, test, exc_info)
File "/home/pokybuild/yocto-worker/oe-selftest-
centos/build/meta/lib/oeqa/core/utils/concurrencytest.py", line 85, in addSubTest
self._addResult(self.result.addSubTest, test, subtest, outcome,
exception = (outcome is not None), **kwargs)
AttributeError: 'AutoTimingTestResultDecorator' object has no attribute 'addSubTest'

2019-11-11 19:14:43,928 - oe-selftest - INFO -

runtime_test.Postinst.test_failing_postinst (subunit.RemotedTestCas

Thanks,

Anuj