Skip to contentSkip to conversation

test-runner: fix test skipping

PATCH In review James Prestwood 2 participants 3 messages Aug 13, 2021

lore
Share & tools
lore
#0 lore raw

PATCH 1/2 test-runner: fix test skipping

Tests that called skipTest would result in an exception which would hault execution as it was uncaught. In addition this wouldn't result in an skipped test.

Now the actual test run is surrounded in a try/except block, skipped exceptions are handled specifically, and a stack trace is printed if some other exception occurs.

---
 tools/test-runner | 48 +++++++++++++++++++++++++++++------------------
 1 file changed, 30 insertions(+), 18 deletions(-)
tools/test-runner +30 -18
index 0ffaec77..3e898dd4 100755
@@ -13,9 +13,11 @@ import time
 import unittest
 import importlib
 import signal
+from unittest.result import TestResult
 import pyroute2
 import multiprocessing
 import re
+import traceback
 
 from configparser import ConfigParser
 from prettytable import PrettyTable
@@ -1156,34 +1158,44 @@ def start_test(ctx, subtests, rqueue):
 				#
 				file = file.strip('()').split('.')[0] + '.py'
 
-				# Set up class only on first test
-				if index == 0:
-					dbg(file)
-					t.setUpClass()
+				# Create an empty result here in case the test fails
+				result = TestResult()
 
-				dbg("\t%s RUNNING" % str(func), end='')
-				sys.__stdout__.flush()
-
-				# Run test (setUp/tearDown run automatically)
-				result = t()
-
-				# Tear down class only on last test
-				if index == len(tlist) - 1:
-					t.tearDownClass()
+				try:
+					# Set up class only on first test
+					if index == 0:
+						dbg("%s\n\t%s RUNNING" % (file, str(func)), end='')
+						t.setUpClass()
+					else:
+						dbg("\t%s RUNNING" % str(func), end='')
+
+					sys.__stdout__.flush()
+
+					# Run test (setUp/tearDown run automatically)
+					result = t()
+
+					# Tear down class only on last test
+					if index == len(tlist) - 1:
+						t.tearDownClass()
+				except unittest.SkipTest as e:
+					result.skipped.append(t)
+				except Exception as e:
+					dbg('\n%s threw an uncaught exception:' % func)
+					traceback.print_exc(file=sys.__stdout__)
 
 				run += result.testsRun
 				errors += len(result.errors)
 				failures += len(result.failures)
 				skipped += len(result.skipped)
 
-				if len(result.errors) > 0 or len(result.failures) > 0:
+				if len(result.skipped) > 0:
+					dbg(colored(" SKIPPED", "cyan"))
+				elif run == 0 or len(result.errors) > 0 or len(result.failures) > 0:
 					dbg(colored(" FAILED", "red"))
 					for e in result.errors:
 						dbg(e[1])
 					for f in result.failures:
 						dbg(f[1])
-				elif len(result.skipped) > 0:
-					dbg(colored(" SKIPPED", "cyan"))
 				else:
 					dbg(colored(" PASSED", "green"))
 
@@ -1396,8 +1408,8 @@ def run_auto_tests(ctx, args):
 				ctx.results[os.path.basename(test)] = rqueue.get()
 
 		except Exception as ex:
-			dbg(ex)
-			dbg("Uncaught exception thrown for %s" % test)
+			dbg("%s threw an uncaught exception" % test)
+			traceback.print_exc(file=sys.__stdout__)
 			ctx.results[os.path.basename(test)] = SimpleResult(run=0, failures=0,
 								errors=0, skipped=0, time=0)
 		finally:
Signature · 1 line
2.31.1
#1 lore raw

Hi James,

Quoted 11 lines · James Prestwood

Tests that called skipTest would result in an exception which would hault execution as it was uncaught. In addition this wouldn't result in an skipped test.

Now the actual test run is surrounded in a try/except block, skipped exceptions are handled specifically, and a stack trace is printed if some other exception occurs.

---
  tools/test-runner | 48 +++++++++++++++++++++++++++++------------------
  1 file changed, 30 insertions(+), 18 deletions(-)

All applied, thanks

Regards,
-Denis

#2 lore raw

PATCH 2/2 auto-t: skip testEAP-PEAP-SIM if ofonod is not running

In reply to James Prestwood
---
 autotests/testEAP-PEAP-SIM/connection_test.py | 4 ++++
 1 file changed, 4 insertions(+)
autotests/testEAP-PEAP-SIM/connection_test.py +4 -0
index ebf22a46..84b95c7a 100644
@@ -10,6 +10,7 @@ from iwd import IWD
 from iwd import NetworkType
 from hlrauc import AuthCenter
 from ofono import Ofono
+from config import ctx
 
 class Test(unittest.TestCase):
     def validate_connection(self, wd):
@@ -46,6 +47,9 @@ class Test(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
+        if not ctx.is_process_running('ofonod'):
+            cls.skipTest(cls, "ofono not running")
+
         cls.auth = AuthCenter('/tmp/hlrauc.sock', '/tmp/sim.db')
 
         IWD.copy_to_storage('ssidEAP-PEAP-SIM.8021x')
Signature · 1 line
2.31.1