test-runner: fix test skipping
PATCH 1/2 test-runner: fix test skipping
Tests that called skipTest would result in an exception which would hault execution as it was uncaught. In addition this wouldn't result in an skipped test.
Now the actual test run is surrounded in a try/except block, skipped exceptions are handled specifically, and a stack trace is printed if some other exception occurs.
--- tools/test-runner | 48 +++++++++++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 18 deletions(-)
tools/test-runner +30 -18
@@ -13,9 +13,11 @@ import time import unittest import importlib import signal +from unittest.result import TestResult import pyroute2 import multiprocessing import re +import traceback from configparser import ConfigParser from prettytable import PrettyTable
@@ -1156,34 +1158,44 @@ def start_test(ctx, subtests, rqueue): # file = file.strip('()').split('.')[0] + '.py' - # Set up class only on first test - if index == 0: - dbg(file) - t.setUpClass() + # Create an empty result here in case the test fails + result = TestResult() - dbg("\t%s RUNNING" % str(func), end='') - sys.__stdout__.flush() - - # Run test (setUp/tearDown run automatically) - result = t() - - # Tear down class only on last test - if index == len(tlist) - 1: - t.tearDownClass() + try: + # Set up class only on first test + if index == 0: + dbg("%s\n\t%s RUNNING" % (file, str(func)), end='') + t.setUpClass() + else: + dbg("\t%s RUNNING" % str(func), end='') + + sys.__stdout__.flush() + + # Run test (setUp/tearDown run automatically) + result = t() + + # Tear down class only on last test + if index == len(tlist) - 1: + t.tearDownClass() + except unittest.SkipTest as e: + result.skipped.append(t) + except Exception as e: + dbg('\n%s threw an uncaught exception:' % func) + traceback.print_exc(file=sys.__stdout__) run += result.testsRun errors += len(result.errors) failures += len(result.failures) skipped += len(result.skipped) - if len(result.errors) > 0 or len(result.failures) > 0: + if len(result.skipped) > 0: + dbg(colored(" SKIPPED", "cyan")) + elif run == 0 or len(result.errors) > 0 or len(result.failures) > 0: dbg(colored(" FAILED", "red")) for e in result.errors: dbg(e[1]) for f in result.failures: dbg(f[1]) - elif len(result.skipped) > 0: - dbg(colored(" SKIPPED", "cyan")) else: dbg(colored(" PASSED", "green"))
@@ -1396,8 +1408,8 @@ def run_auto_tests(ctx, args): ctx.results[os.path.basename(test)] = rqueue.get() except Exception as ex: - dbg(ex) - dbg("Uncaught exception thrown for %s" % test) + dbg("%s threw an uncaught exception" % test) + traceback.print_exc(file=sys.__stdout__) ctx.results[os.path.basename(test)] = SimpleResult(run=0, failures=0, errors=0, skipped=0, time=0) finally:
Signature · 1 line
2.31.1
Hi James,
Quoted 11 lines · James Prestwood
Tests that called skipTest would result in an exception which would hault execution as it was uncaught. In addition this wouldn't result in an skipped test.
Now the actual test run is surrounded in a try/except block, skipped exceptions are handled specifically, and a stack trace is printed if some other exception occurs.
--- tools/test-runner | 48 +++++++++++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 18 deletions(-)
All applied, thanks
Regards,
-Denis
PATCH 2/2 auto-t: skip testEAP-PEAP-SIM if ofonod is not running
In reply to James Prestwood--- autotests/testEAP-PEAP-SIM/connection_test.py | 4 ++++ 1 file changed, 4 insertions(+)
autotests/testEAP-PEAP-SIM/connection_test.py +4 -0
@@ -10,6 +10,7 @@ from iwd import IWD from iwd import NetworkType from hlrauc import AuthCenter from ofono import Ofono +from config import ctx class Test(unittest.TestCase): def validate_connection(self, wd):
@@ -46,6 +47,9 @@ class Test(unittest.TestCase): @classmethod def setUpClass(cls): + if not ctx.is_process_running('ofonod'): + cls.skipTest(cls, "ofono not running") + cls.auth = AuthCenter('/tmp/hlrauc.sock', '/tmp/sim.db') IWD.copy_to_storage('ssidEAP-PEAP-SIM.8021x')
Signature · 1 line
2.31.1