Skip to contentSkip to conversation

test-runner: add option to specify subtests to run

PATCH In review James Prestwood 2 participants 9 messages Nov 16, 2020

lore
Share & tools
lore
#0 lore raw

PATCH 1/8 test-runner: add option to specify subtests to run

You can now specify a limited list of subtests to run out of a full auto-test using --sub-tests,-S. This option is limited in that it is only meant to be used with a single autotest (since it doesn't make much sense otherwise).

The subtest can be specified both with or without the file extension.

Example usage:

./test-runner -A testAP -S failure_test,dhcp_test.py

This will only run the two subtests and exclude any other *.py tests present in the test directory.

---
 tools/test-runner | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)
tools/test-runner +39 -0
index b4840fa4..97d0e32c 100755
@@ -892,6 +892,18 @@ def pre_test(ctx, test):
 		else:
 			shutil.copy(f, '/tmp')
 
+	# Prune down any subtests if needed
+	if ctx.args.sub_tests:
+		ctx.args.sub_tests = ctx.args.sub_tests.split(',')
+		pruned = []
+
+		for s in subtests:
+			# Allow test name both with and without the extension
+			if s in ctx.args.sub_tests or os.path.splitext(s)[0] in ctx.args.sub_tests:
+				pruned.append(s)
+
+		subtests = pruned
+
 	ctx.start_dbus_monitor()
 	ctx.start_radios()
 	ctx.start_hostapd()
@@ -996,6 +1008,10 @@ def run_auto_tests(ctx, args):
 		try:
 			copied, subtests = pre_test(ctx, test)
 
+			if len(subtests) < 1:
+				dbg("No tests to run")
+				exit()
+
 			rqueue = multiprocessing.Queue()
 			p = multiprocessing.Process(target=start_test, args=(ctx, subtests, rqueue))
 			p.start()
@@ -1060,6 +1076,7 @@ def run_tests():
 	parser.add_argument('--log-uid')
 	parser.add_argument('--hw')
 	parser.add_argument('--monitor')
+	parser.add_argument('--sub_tests')
 
 	args = parser.parse_args(options)
 
@@ -1128,6 +1145,9 @@ class Main:
 				help='Use physical adapters for tests (passthrough)')
 		self.parser.add_argument('--monitor', '-m', type=str,
 				help='Enables iwmon output to file')
+		self.parser.add_argument('--sub-tests', '-S', metavar='<subtests>',
+				type=str, nargs=1, help='List of subtests to run',
+				default=None, dest='sub_tests')
 
 		# Prevent --autotest/--unittest from being used together
 		auto_unit_group = self.parser.add_mutually_exclusive_group()
@@ -1153,10 +1173,25 @@ class Main:
 
 		self.args = self.parser.parse_args()
 
+		if self.args.auto_tests:
+			self.args.auto_tests = self.args.auto_tests[0].split(',')
+
+		if self.args.sub_tests:
+			self.args.sub_tests = self.args.sub_tests[0].split(',')
+
 		if self.args.log and self.args.unit_tests:
 			dbg("Cannot use --log with --unit-tests")
 			quit()
 
+		if self.args.sub_tests:
+			if not self.args.auto_tests:
+				dbg("--sub-tests must be used with --auto-tests")
+				quit()
+
+			if len(self.args.auto_tests) > 1:
+				dbg("--sub-tests must be used with a single auto test")
+				quit()
+
 	def start(self):
 		usb_adapters = None
 
@@ -1220,6 +1255,9 @@ class Main:
 		if self.args.auto_tests:
 			options += ' --auto_tests %s' % ','.join(self.args.auto_tests)
 
+		if self.args.sub_tests:
+			options += ' --sub_tests %s' % ','.join(self.args.sub_tests)
+
 		if self.args.log:
 			if os.environ.get('SUDO_GID', None) is None:
 				print("--log can only be used as root user")
@@ -1242,6 +1280,7 @@ class Main:
 
 		denylist = [
 			'auto_tests',
+			'sub_tests',
 			'qemu',
 			'kernel'
 		]
Signature · 1 line
2.26.2
#1 lore raw

PATCH 2/8 auto-t: refactor get_instance from iwd.py

When network namespaces are introduced there may be multiple IWD class instances. This makes IWD.get_instance ambiguous when namespaces are involved. iwd.py has been refactored to not use IWD.get_instance, but testutil still needs it since its purely based off interface names. Rather than remove it and modify every test to pass the IWD object we can just maintain the existing behavior for only the root namespace.

---
 autotests/util/iwd.py | 31 ++++++++++++++++++-------------
 1 file changed, 18 insertions(+), 13 deletions(-)
autotests/util/iwd.py +18 -13
index d7235f2d..af3ecd05 100755
@@ -399,14 +399,12 @@ class Device(IWDDBusAbstract):
         elif not scan_if_needed:
             return None
 
-        iwd = IWD.get_instance()
-
         self.scan()
 
         condition = 'obj.scanning'
-        iwd.wait_for_object_condition(self, condition)
+        IWD._wait_for_object_condition(self, condition)
         condition = 'not obj.scanning'
-        iwd.wait_for_object_condition(self, condition)
+        IWD._wait_for_object_condition(self, condition)
 
         for bus_obj in self._station.GetOrderedNetworks():
             ordered_network = OrderedNetwork(bus_obj)
@@ -940,7 +938,7 @@ class IWD(AsyncOpAbstract):
     _agent_manager_if = None
     _iwd_proc = None
     _devices = None
-    _instance = None
+    _default_instance = None
     psk_agent = None
 
     def __init__(self, start_iwd_daemon = False, iwd_config_dir = '/tmp'):
@@ -964,8 +962,11 @@ class IWD(AsyncOpAbstract):
         self._devices = DeviceList(self)
 
         # Weak to make sure the test's reference to @self is the only counted
-        # reference so that __del__ gets called when it's released
-        IWD._instance = weakref.ref(self)
+        # reference so that __del__ gets called when it's released. This is only
+        # done for the root namespace in order to allow testutil to function
+        # correctly in non-namespace tests.
+        if self.namespace.name == "root":
+            IWD._default_instance = weakref.ref(self)
 
     def __del__(self):
         if self.psk_agent:
@@ -998,10 +999,11 @@ class IWD(AsyncOpAbstract):
                                IWD_AGENT_MANAGER_INTERFACE)
         return self._agent_manager_if
 
-    def wait_for_object_condition(self, obj, condition_str, max_wait = 50):
-        self._wait_timed_out = False
+    @staticmethod
+    def _wait_for_object_condition(obj, condition_str, max_wait = 50):
+        _wait_timed_out = False
         def wait_timeout_cb():
-            self._wait_timed_out = True
+            _wait_timed_out = True
             return False
 
         try:
@@ -1009,14 +1011,17 @@ class IWD(AsyncOpAbstract):
             context = ctx.mainloop.get_context()
             while not eval(condition_str):
                 context.iteration(may_block=True)
-                if self._wait_timed_out and ctx.args.gdb == None:
+                if _wait_timed_out and ctx.args.gdb == None:
                     raise TimeoutError('[' + condition_str + ']'\
                                        ' condition was not met in '\
                                        + str(max_wait) + ' sec')
         finally:
-            if not self._wait_timed_out:
+            if not _wait_timed_out:
                 GLib.source_remove(timeout)
 
+    def wait_for_object_condition(self, *args, **kwargs):
+        self._wait_for_object_condition(*args, **kwargs)
+
     def wait(self, time):
         self._wait_timed_out = False
         def wait_timeout_cb():
@@ -1129,4 +1134,4 @@ class IWD(AsyncOpAbstract):
 
     @staticmethod
     def get_instance():
-        return IWD._instance()
+        return IWD._default_instance()
Signature · 1 line
2.26.2
#2 lore raw

PATCH 3/8 test-runner: store process stdout

In reply to James Prestwood

If a blocking process is started store the output in case the caller needs it. Output will be stored in Process.out.

---
 tools/test-runner | 28 +++++++++++++++++++---------
 1 file changed, 19 insertions(+), 9 deletions(-)
tools/test-runner +19 -9
index 97d0e32c..77b2c9fc 100755
@@ -209,17 +209,27 @@ class Process:
 					self.stdout = sys.__stdout__
 					self.stderr = sys.__stderr__
 
-		if not wait and not check:
-			self.pid = subprocess.Popen(self.args, stdout=self.stdout, \
+
+		self.pid = subprocess.Popen(self.args, stdout=self.stdout, \
 							stderr=self.stderr, env=env, \
 							cwd=os.getcwd())
-			print("Starting process {}".format(self.pid.args))
-		else:
-			self.ret = subprocess.call(self.args, stdout=self.stdout, \
-							stderr=self.stderr)
-			print("%s returned %d" % (args[0], self.ret))
-			if check and self.ret != 0:
-				raise subprocess.CalledProcessError(returncode=self.ret, cmd=self.args)
+
+		print("Starting process {}".format(self.pid.args))
+
+		if not wait and not check:
+			return
+
+		self.pid.wait()
+
+		self.out, _ = self.pid.communicate()
+		self.ret = self.pid.returncode
+
+		if self.out:
+			self.out = self.out.decode('utf-8')
+
+		print("%s returned %d" % (args[0], self.ret))
+		if check and self.ret != 0:
+			raise subprocess.CalledProcessError(returncode=self.ret, cmd=self.args)
 
 	def __del__(self):
 		print("Del process %s" % self.args)
Signature · 1 line
2.26.2
#3 lore raw

PATCH 4/8 test-runner: delay starting the shell until after pre_test

In reply to James Prestwood

This makes --shell somewhat more useful by copying all the test files and starting test processes before dumping the user into a shell.

---
 tools/test-runner | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)
tools/test-runner +9 -9
index 77b2c9fc..610f1702 100755
@@ -1005,19 +1005,19 @@ def run_auto_tests(ctx, args):
 	shutil.copytree(args.testhome + '/autotests/misc/secrets', '/tmp/secrets')
 	shutil.copy(args.testhome + '/autotests/misc/phonesim/phonesim.conf', '/tmp')
 
-	if args.shell:
-		#
-		# Shell really isn't meant to be used with multiple tests. If
-		# a set of tests was passed in just start out in the first.
-		#
-		os.chdir(tests[0])
-		os.system('/bin/bash')
-		exit()
-
 	for test in tests:
 		try:
 			copied, subtests = pre_test(ctx, test)
 
+			if args.shell:
+				#
+				# Shell really isn't meant to be used with multiple tests. If
+				# a set of tests was passed in just start out in the first.
+				#
+				os.chdir(tests[0])
+				os.system('/bin/bash')
+				exit()
+
 			if len(subtests) < 1:
 				dbg("No tests to run")
 				exit()
Signature · 1 line
2.26.2
#4 lore raw

PATCH 5/8 auto-t: update hostapd.list_sta to use Process.out

In reply to James Prestwood

Instead of list_sta() waiting for the PID itself it can just use wait=True and grab the output out of the Process class itself.

---
 autotests/util/hostapd.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)
autotests/util/hostapd.py +5 -4
index cf82c010..3f48bd57 100644
@@ -152,11 +152,12 @@ class HostapdCLI:
         ctx.start_process(self.cmdline + ['enable'], wait=True)
 
     def list_sta(self):
-        proc = ctx.start_process(self.cmdline + ['list_sta'])
-        proc.pid.wait()
-        lines = proc.pid.stdout.read().decode('utf-8')
+        proc = ctx.start_process(self.cmdline + ['list_sta'], wait=True)
 
-        return [line for line in lines.split('\n') if line]
+        if not proc.out:
+            return []
+
+        return [line for line in proc.out.split('\n') if line]
 
     def set_neighbor(self, addr, ssid, nr):
         cmd = self.cmdline + ['set_neighbor', addr, 'ssid="%s"' % ssid, 'nr=%s' % nr]
Signature · 1 line
2.26.2
#5 lore raw

PATCH 6/8 auto-t: catch exceptions in post_test

In reply to James Prestwood
---
 tools/test-runner | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)
tools/test-runner +11 -7
index 610f1702..61555b77 100755
@@ -945,15 +945,19 @@ def post_test(ctx, to_copy):
 	'''
 		Remove copied files, and stop test processes.
 	'''
-	for f in to_copy:
-		if os.path.isdir('/tmp/' + f):
-			shutil.rmtree('/tmp/' + f)
-		else:
-			os.remove('/tmp/' + f)
+	try:
+		for f in to_copy:
+			if os.path.isdir('/tmp/' + f):
+				shutil.rmtree('/tmp/' + f)
+			else:
+				os.remove('/tmp/' + f)
 
-	Process(['ifconfig', 'lo', 'down'], wait=True)
+		Process(['ifconfig', 'lo', 'down'], wait=True)
+	except Exception as e:
+		print("Exception thrown in post_test")
+	finally:
+		ctx.stop_test_processes()
 
-	ctx.stop_test_processes()
 	if ctx.args.valgrind:
 		with open('/tmp/valgrind.log', 'r') as f:
 				dbg(f.read())
Signature · 1 line
2.26.2
#6 lore raw

PATCH 7/8 auto-t: do away with -1 test results

In reply to James Prestwood

This can result in strange test results since there was no less than zero checks before subtracting the total tests from failed tests. In case of an internal exception we can just set all values to zero. This will be handled specially as we do for timeout errors.

---
 tools/test-runner | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)
tools/test-runner +10 -6
index 61555b77..69e7fe7e 100755
@@ -973,16 +973,20 @@ def print_results(results):
 	total_time = 0
 
 	for test, result in results.items():
-		if result.time != TEST_MAX_TIMEOUT:
+
+		if result.time == TEST_MAX_TIMEOUT:
+			failed = "Timed out"
+			passed = "Timed out"
+		elif result.time == 0:
+			failed = "Exception"
+			passed = "Exception"
+		else:
 			failed = result.failures + result.errors
 			passed = result.run - failed
 
 			total_pass += passed
 			total_fail += failed
 			total_skip += result.skipped
-		else:
-			failed = "Timed out"
-			passed = "Timed out"
 
 		total_time += result.time
 
@@ -1046,8 +1050,8 @@ def run_auto_tests(ctx, args):
 		except Exception as ex:
 			print(ex)
 			print("Uncaught exception thrown for %s" % test)
-			ctx.results[os.path.basename(test)] = SimpleResult(run=-1, failures=-1,
-								errors=-1, skipped=-1, time=-1)
+			ctx.results[os.path.basename(test)] = SimpleResult(run=0, failures=0,
+								errors=0, skipped=0, time=0)
 		finally:
 			post_test(ctx, copied)
 
Signature · 1 line
2.26.2
#7 lore raw

PATCH 8/8 auto-t: introduce new autotest syntax

In reply to James Prestwood

Sometimes improperly written tests can end up causing future tests to fail. For faster debugging you can now add a '+' after a given autotest which will start that test and run all tests which come alphabetically after it (as if you are running a full autotest suite).

Example:

./test-runner -A testWPA+

This will run testWPA, testWPA2, testWPA2-no-CCMP, testWPA2-SHA256, and testWPA2withMFP.

---
 tools/test-runner | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)
tools/test-runner +9 -4
index 69e7fe7e..803d4da5 100755
@@ -802,6 +802,7 @@ def build_test_list(args):
 	'''
 	tests = []
 	test_root = args.testhome + '/autotests'
+	full_list = sorted(os.listdir(test_root))
 
 	# Run all tests
 	if not args.auto_tests:
@@ -809,15 +810,19 @@ def build_test_list(args):
 		if args.shell:
 			return [test_root + '/shell']
 
-		tests = os.listdir(test_root)
 		# Pair down any non-tests and append full path
-		tests = [test_root + '/' + t for t in tests if t.startswith('test')]
+		tests = [test_root + '/' + t for t in full_list if t.startswith('test')]
 	else:
 		print("Generating partial test list")
 		for t in args.auto_tests.split(','):
 			path = '%s/%s' % (test_root, t)
-			# Full test path specified
-			if os.path.exists(t):
+			if t.endswith('+'):
+				t = t.split('+')[0]
+				i = full_list.index(t)
+
+				tests = [test_root + '/' + x for x in full_list[i:] \
+							if x.startswith('test')]
+			elif os.path.exists(t):
 				tests.append(t)
 			elif os.path.exists(path):
 				tests.append(path)
Signature · 1 line
2.26.2
#8 lore raw
In reply to James Prestwood

Hi James,

Quoted 10 lines · James Prestwood

You can now specify a limited list of subtests to run out of a full auto-test using --sub-tests,-S. This option is limited in that it is only meant to be used with a single autotest (since it doesn't make much sense otherwise).

The subtest can be specified both with or without the file extension.

Example usage:

Quoted 8 lines

./test-runner -A testAP -S failure_test,dhcp_test.py

This will only run the two subtests and exclude any other *.py tests present in the test directory.

---
  tools/test-runner | 39 +++++++++++++++++++++++++++++++++++++++
  1 file changed, 39 insertions(+)

All applied, thanks.

Regards,
-Denis