Skip to contentSkip to conversation

test-runner: Move environment setup into own module

PATCH In review James Prestwood 2 participants 15 messages Mar 31, 2022

lore
Share & tools
lore

James Prestwood

author
#0 lore raw

PATCH v2 1/14 test-runner: Move environment setup into own module

This (as well as subsequent commits) will separate test-runner into two parts:

1. Environment setup
2. Running tests

Spurred by interest in adding UML/host support, test-runner was in need of a refactor to separate out the environment setup and actually running the tests.

The environment (currently only Qemu) requires quite a bit of special handling (ctypes mounting/reboot, 9p mounts, tons of kernel options etc) which nobody writing tests should need to see or care about. This has all been moved into 'runner.py'.

Running the tests (inside test-runner) won't change much.

The new 'runner.py' module adds an abstraction class which allows different Runner's to be implemented, and setup their own environment as they see fit. This is in preparation for UML and Host runners.

---
 tools/runner.py | 485 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 485 insertions(+)
 create mode 100644 tools/runner.py
tools/runner.py new file +485 -0
new file mode 100644
index 00000000..743d74b1
@@ -0,0 +1,485 @@
+#!/usr/bin/python3
+
+from argparse import ArgumentParser
+from argparse import Namespace
+from configparser import ConfigParser
+from collections import namedtuple
+from shutil import copy, copytree, which, rmtree
+from glob import glob
+
+import os
+import ctypes
+import fcntl
+import sys
+
+libc = ctypes.cdll['libc.so.6']
+libc.mount.argtypes = (ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p, \
+			ctypes.c_ulong, ctypes.c_char_p)
+
+# Using ctypes to load the libc library is somewhat low level. Because of this
+# we need to define our own flags/options for use with mounting.
+MS_NOSUID = 2
+MS_NODEV = 4
+MS_NOEXEC = 8
+MS_STRICTATIME = 1 << 24
+STDIN_FILENO = 0
+TIOCSTTY = 0x540E
+
+MountInfo = namedtuple('MountInfo', 'fstype target options flags')
+DevInfo = namedtuple('DevInfo', 'target linkpath')
+
+mounts_common = [
+	MountInfo('sysfs', '/sys', '', MS_NOSUID|MS_NOEXEC|MS_NODEV),
+	MountInfo('proc', '/proc', '', MS_NOSUID|MS_NOEXEC|MS_NODEV),
+	MountInfo('devpts', '/dev/pts', 'mode=0620', MS_NOSUID|MS_NOEXEC),
+	MountInfo('tmpfs', '/dev/shm', 'mode=1777',
+					MS_NOSUID|MS_NODEV|MS_STRICTATIME),
+	MountInfo('tmpfs', '/run', 'mode=0755',
+					MS_NOSUID|MS_NODEV|MS_STRICTATIME),
+	MountInfo('tmpfs', '/tmp', '', 0),
+	MountInfo('tmpfs', '/usr/share/dbus-1', 'mode=0755',
+					MS_NOSUID|MS_NOEXEC|MS_NODEV|MS_STRICTATIME),
+]
+
+dev_table = [
+	DevInfo('/proc/self/fd', '/dev/fd'),
+	DevInfo('/proc/self/fd/0', '/dev/stdin'),
+	DevInfo('/proc/self/fd/1', '/dev/stdout'),
+	DevInfo('/proc/self/fd/2', '/dev/stderr')
+]
+
+def mount(source, target, fs, flags, options=''):
+	'''
+		Python wrapper for libc mount()
+	'''
+	ret = libc.mount(source.encode(), target.encode(), fs.encode(), flags,
+				options.encode())
+	if ret < 0:
+		errno = ctypes.get_errno()
+		raise Exception("Could not mount %s (%d)" % (target, errno))
+
+#
+# Custom argparse.Namespace class to stringify arguments in a way that can be
+# directly passed to the test environment as kernel arguments. This also removes
+# any None, False, or [] arguments.
+#
+class RunnerNamespace(Namespace):
+	def __init__(self, *args, **kwargs):
+		super().__init__(*args, **kwargs)
+
+	def to_cmd(self):
+		ret = ''
+		for k, v in self.__dict__.items():
+			if v in [None, False, [], '']:
+				continue
+
+			ret += '%s=%s ' % (k, str(v))
+
+		return ret.strip()
+
+#
+# The core arguments needed both inside and outside the test environment
+#
+class RunnerCoreArgParse(ArgumentParser):
+	def __init__(self, *args, **kwargs):
+		super().__init__(self, *args, **kwargs)
+
+		self.add_argument('--start', '-s',
+				help='Custom init process in virtual environment',
+				dest='start',
+				default=None)
+		self.add_argument('--verbose', '-v', metavar='<list>',
+				type=str,
+				help='Comma separated list of applications',
+				dest='verbose',
+				default=[])
+		self.add_argument('--debug', '--dbg', '-d',
+				action='store_true',
+				help='Enable test-runner debugging',
+				dest='dbg')
+		self.add_argument('--log', '-l',
+				type=str,
+				help='Directory for log files')
+		self.add_argument('--monitor', '-m',
+				type=str,
+				help='Enables iwmon output to file')
+		self.add_argument('--sub-tests', '-S',
+				metavar='<subtests>',
+				type=str, nargs=1, help='List of subtests to run',
+				default=None, dest='sub_tests')
+		self.add_argument('--result', '-e', type=str,
+				help='Writes PASS/FAIL to results file')
+		self.add_argument('--hw', '-w',
+				type=str,
+				help='Use physical adapters for tests (passthrough)')
+		self.add_argument('--testhome')
+
+		# Prevent --autotest/--unittest from being used together
+		auto_unit_group = self.add_mutually_exclusive_group()
+		auto_unit_group.add_argument('--autotests', '-A',
+				metavar='<tests>',
+				type=str,
+				help='List of tests to run',
+				default=None,
+				dest='autotests')
+		auto_unit_group.add_argument('--unit-tests', '-U',
+				metavar='<tests>',
+				type=str,
+				nargs='?',
+				const='*',
+				help='List of unit tests to run',
+				dest='unit_tests')
+
+		# Prevent --valgrind/--gdb from being used together
+		valgrind_gdb_group = self.add_mutually_exclusive_group()
+		valgrind_gdb_group.add_argument('--gdb', '-g',
+				metavar='<exec>',
+				type=str,
+				nargs=1,
+				help='Run gdb on specified executable',
+				dest='gdb')
+		valgrind_gdb_group.add_argument('--valgrind', '-V',
+				action='store_true',
+				help='Run valgrind on IWD',
+				dest='valgrind')
+
+	# Overwrite to use a custom namespace class and parse from env
+	def parse_args(self, *args, namespace=RunnerNamespace()):
+		if len(sys.argv) > 1:
+			return super().parse_args(*args, namespace=namespace)
+
+		options = []
+		for k, v in os.environ.items():
+			options.append('--' + k)
+			options.append(v)
+
+		return self.parse_known_args(args=options, namespace=namespace)[0]
+
+#
+# Arguments only needed outside the test environment
+#
+class RunnerArgParse(RunnerCoreArgParse):
+	def __init__(self, *args, **kwargs):
+		super().__init__(*args, **kwargs)
+
+		self.add_argument('--runner', '-r',
+				metavar='<runner type>',
+				type=str,
+				help='Type of runner to use (qemu, uml, host)',
+				dest='runner',
+				default=None)
+		self.add_argument('--kernel', '-k',
+				metavar='<kernel>',
+				type=str,
+				help='Path to kernel/uml image',
+				dest='kernel',
+				default=None)
+
+#
+# Class to sort out what type of runner this is, returns the RunnerAbstract
+# implementation.
+#
+class Runner:
+	def __new__(self):
+		parser = RunnerArgParse(description='IWD Test Runner')
+
+		args = parser.parse_args()
+
+		# Common options
+		args.PATH = os.environ['PATH']
+
+		if 'testhome' not in args.to_cmd():
+			if os.getcwd().endswith('tools'):
+				args.testhome = '%s/../' % os.getcwd()
+			else:
+				args.testhome = os.getcwd()
+
+		if args.start is None:
+			if os.path.exists('run-tests'):
+				args.start = os.path.abspath('run-tests')
+			elif os.path.exists('tools/run-tests'):
+				args.start = os.path.abspath('tools/run-tests')
+			else:
+				raise Exception("Cannot locate run-tests binary")
+
+		# If no runner is specified but we have a kernel image assume
+		# if the kernel is executable its UML, otherwise qemu
+		if not args.runner:
+			if not args.kernel:
+				raise Exception("Please specify --runner/--kernel")
+
+			if os.access(args.kernel, os.X_OK):
+				args.runner = 'uml'
+			else:
+				args.runner = 'qemu'
+
+		if args.runner == 'qemu':
+			return QemuRunner(args)
+		else:
+			raise Exception("Unknown runner %s" % args.runner)
+
+class RunnerAbstract:
+	cmdline = []
+	env = None
+	name = "unnamed"
+
+	def __init__(self, args):
+		self.args = args
+
+		if self.args.log:
+			self.args.log = os.path.abspath(self.args.log)
+
+	def start(self):
+		print("Starting %s" % self.name)
+		os.execlpe(self.cmdline[0], *self.cmdline, self.env)
+
+	def prepare_environment(self):
+		path = os.environ['PATH']
+		os.environ['PATH'] = '%s/src' % self.args.testhome
+		os.environ['PATH'] += ':%s/tools' % self.args.testhome
+		os.environ['PATH'] += ':%s/client' % self.args.testhome
+		os.environ['PATH'] += ':%s/monitor' % self.args.testhome
+		os.environ['PATH'] += ':%s/wired' % self.args.testhome
+		os.environ['PATH'] += ':' + path
+
+		sys.path.append(self.args.testhome + '/autotests/util')
+
+		if not os.path.exists('/tmp/iwd'):
+			os.mkdir('/tmp/iwd')
+
+		#
+		# This prevents any print() calls in this script from printing unless
+		# --debug is passed. For an 'always print' option use dbg()
+		#
+		if not self.args.dbg:
+			sys.stdout = open(os.devnull, 'w')
+
+		# Copy autotests/misc/{certs,secrets,phonesim} so any test can refer to them
+		if os.path.exists('/tmp/certs'):
+			rmtree('/tmp/certs')
+
+		if os.path.exists('/tmp/secrets'):
+			rmtree('/tmp/secrets')
+
+		copytree(self.args.testhome + '/autotests/misc/certs', '/tmp/certs')
+		copytree(self.args.testhome + '/autotests/misc/secrets', '/tmp/secrets')
+		copy(self.args.testhome + '/autotests/misc/phonesim/phonesim.conf', '/tmp')
+
+	def cleanup_environment(self):
+		rmtree('/tmp/iwd')
+		rmtree('/tmp/certs')
+		rmtree('/tmp/secrets')
+		os.remove('/tmp/phonesim.conf')
+
+		os.sync()
+
+	# For QEMU/UML runners
+	def _prepare_mounts(self):
+		for entry in mounts_common:
+			try:
+				os.lstat(entry.target)
+			except:
+				os.mkdir(entry.target, 755)
+
+			mount(entry.fstype, entry.target, entry.fstype, entry.flags,
+				entry.options)
+
+		for entry in dev_table:
+			os.symlink(entry.target, entry.linkpath)
+
+		os.setsid()
+
+	def stop(self):
+		exit()
+
+class QemuRunner(RunnerAbstract):
+	name = "Qemu Runner"
+
+	def __init__(self, args):
+		def mount_options(id):
+			return 'mount_tag=%s,security_model=passthrough,id=%s' % (id, id)
+
+		usb_adapters = None
+		pci_adapters = None
+		gid = None
+		append_gid_uid = False
+
+		super().__init__(args)
+
+		if len(sys.argv) <= 1:
+			return
+
+		if not which('qemu-system-x86_64'):
+			raise Exception('Cannot locate qemu binary')
+
+		if not args.kernel or not os.path.exists(args.kernel):
+			raise Exception('Cannot locate kernel image %s' % args.kernel)
+
+		if args.hw:
+			hw_conf = ConfigParser()
+			hw_conf.read(args.hw)
+
+			if hw_conf.has_section('USBAdapters'):
+				# The actual key name of the adapter
+				# doesn't matter since all we need is the
+				# bus/address. This gets named by the kernel
+				# anyways once in the VM.
+				usb_adapters = [v for v in hw_conf['USBAdapters'].values()]
+
+			if hw_conf.has_section('PCIAdapters'):
+				pci_adapters = [v for v in hw_conf['PCIAdapters'].values()]
+
+		if os.environ.get('SUDO_GID', None):
+			uid = int(os.environ['SUDO_UID'])
+			gid = int(os.environ['SUDO_GID'])
+
+		if args.log:
+			if os.getuid() != 0:
+				print("--log can only be used as root user")
+				quit()
+
+			append_gid_uid = True
+
+			args.log = os.path.abspath(args.log)
+			if not os.path.exists(self.args.log):
+				os.mkdir(self.args.log)
+
+				if gid:
+					os.chown(self.args.log, uid, gid)
+
+		if args.monitor:
+			if os.getuid() != 0:
+				print("--monitor can only be used as root user")
+				quit()
+
+			append_gid_uid = True
+
+			args.monitor = os.path.abspath(args.monitor)
+			monitor_parent_dir = os.path.abspath(os.path.join(self.args.monitor,
+										os.pardir))
+
+		if args.result:
+			if os.getuid() != 0:
+				print("--result can only be used as root user")
+				quit()
+
+			append_gid_uid = True
+
+			args.result = os.path.abspath(args.result)
+			result_parent_dir = os.path.abspath(os.path.join(self.args.result,
+										os.pardir))
+
+		if append_gid_uid:
+			args.SUDO_UID = uid
+			args.SUDO_GID = gid
+
+		kern_log = "ignore_loglevel" if "kernel" in args.verbose else "quiet"
+
+		qemu_cmdline = [
+			'qemu-system-x86_64',
+			'-machine', 'type=q35,accel=kvm:tcg',
+			'-nodefaults', '-no-user-config', '-monitor', 'none',
+			'-display', 'none', '-m', '256M', '-nographic', '-vga',
+			'none', '-no-acpi', '-no-hpet',
+			'-no-reboot', '-fsdev',
+			'local,id=fsdev-root,path=/,readonly=on,security_model=none,multidevs=remap',
+			'-device',
+			'virtio-9p-pci,fsdev=fsdev-root,mount_tag=/dev/root',
+			'-chardev', 'stdio,id=chardev-serial0,signal=off',
+			'-device', 'pci-serial,chardev=chardev-serial0',
+			'-device', 'virtio-rng-pci',
+			'-kernel', args.kernel,
+			'-smp', '2',
+			'-append',
+			'console=ttyS0,115200n8 earlyprintk=serial \
+				rootfstype=9p root=/dev/root \
+				rootflags=trans=virtio \
+				acpi=off pci=noacpi %s ro \
+				mac80211_hwsim.radios=0 init=%s %s' %
+						(kern_log, args.start, args.to_cmd()),
+		]
+
+		# Add two ethernet devices for testing EAD
+		qemu_cmdline.extend([
+				'-net', 'nic,model=virtio',
+				'-net', 'nic,model=virtio',
+				'-net', 'user'
+		])
+
+		if usb_adapters:
+			for bus, addr in [s.split(',') for s in usb_adapters]:
+				qemu_cmdline.extend(['-usb',
+							'-device',
+							'usb-host,hostbus=%s,hostaddr=%s' % \
+							(bus, addr)])
+		if pci_adapters:
+			qemu_cmdline.extend(['-enable-kvm'])
+			for addr in pci_adapters:
+				qemu_cmdline.extend(['-device', 'vfio-pci,host=%s' % addr])
+
+		if args.log:
+			#
+			# Creates a virtfs device that can be mounted. This mount
+			# will point back to the provided log directory and is
+			# writable unlike the rest of the mounted file system.
+			#
+			qemu_cmdline.extend([
+				'-virtfs',
+				'local,path=%s,%s' % (args.log, mount_options('logdir'))
+			])
+
+		if args.monitor:
+			qemu_cmdline.extend([
+				'-virtfs',
+				'local,path=%s,%s' % (monitor_parent_dir, mount_options('mondir'))
+			])
+
+		if args.result:
+			qemu_cmdline.extend([
+				'-virtfs',
+				'local,path=%s,%s' % (result_parent_dir, mount_options('resultdir'))
+			])
+
+		self.cmdline = qemu_cmdline
+
+	def prepare_environment(self):
+		mounts_common.extend([
+			MountInfo('debugfs', '/sys/kernel/debug', '', 0)
+		])
+
+		self._prepare_mounts()
+
+		super().prepare_environment()
+
+		fcntl.ioctl(STDIN_FILENO, TIOCSTTY, 1)
+
+		if self.args.log:
+			mount('logdir', self.args.log, '9p', 0,
+					'trans=virtio,version=9p2000.L,msize=10240')
+			# Clear out any log files from other test runs
+			for f in glob('%s/*' % self.args.log):
+				print("removing %s" % f)
+
+				if os.path.isdir(f):
+					rmtree(f)
+				else:
+					os.remove(f)
+		elif self.args.monitor:
+			parent = os.path.abspath(os.path.join(self.args.monitor, os.pardir))
+			mount('mondir', parent, '9p', 0,
+					'trans=virtio,version=9p2000.L,msize=10240')
+
+		if self.args.result:
+			parent = os.path.abspath(os.path.join(self.args.result, os.pardir))
+			mount('resultdir', parent, '9p', 0,
+					'trans=virtio,version=9p2000.L,msize=10240')
+
+	def stop(self):
+		RB_AUTOBOOT = 0x01234567
+		#
+		# Calling 'reboot' or 'shutdown' from a shell (e.g. os.system('reboot'))
+		# is not the same the POSIX reboot() and will cause a kernel panic since
+		# we are the init process. The libc.reboot() allows the VM to exit
+		# gracefully.
+		#
+		libc.reboot(RB_AUTOBOOT)
Signature · 1 line
2.34.1
#1 lore raw

Hi James,

Quoted 17 lines · James Prestwood

This (as well as subsequent commits) will separate test-runner into two parts:

1. Environment setup
2. Running tests

Spurred by interest in adding UML/host support, test-runner was in need of a refactor to separate out the environment setup and actually running the tests.

The environment (currently only Qemu) requires quite a bit of special handling (ctypes mounting/reboot, 9p mounts, tons of kernel options etc) which nobody writing tests should need to see or care about. This has all been moved into 'runner.py'.

Running the tests (inside test-runner) won't change much.

Quoted 8 lines

The new 'runner.py' module adds an abstraction class which allows different Runner's to be implemented, and setup their own environment as they see fit. This is in preparation for UML and Host runners.

---
  tools/runner.py | 485 ++++++++++++++++++++++++++++++++++++++++++++++++
  1 file changed, 485 insertions(+)

create mode 100644 tools/runner.py

All applied, thanks.

Regards,
-Denis

author
#2 lore raw

PATCH v2 2/14 test-runner: remove environment specific code

In reply to James Prestwood

This removes all the Qemu/environment related code as this has been moved into runner.py.

---
 tools/test-runner | 506 ++--------------------------------------------
 1 file changed, 17 insertions(+), 489 deletions(-)
tools/test-runner +17 -489
index 81624953..c6f7f550 100755
@@ -1,24 +1,21 @@
 #!/usr/bin/python3
 
-import argparse
 import os
 import shutil
-import ctypes
 import fcntl
-import shlex
 import sys
 import subprocess
 import atexit
 import time
 import unittest
 import importlib
-import signal
 from unittest.result import TestResult
-import pyroute2
 import multiprocessing
 import re
 import traceback
 
+from runner import Runner
+
 from configparser import ConfigParser
 from prettytable import PrettyTable
 from termcolor import colored
@@ -29,19 +26,6 @@ import dbus.mainloop.glib
 from gi.repository import GLib
 from weakref import WeakValueDictionary
 
-libc = ctypes.cdll['libc.so.6']
-libc.mount.argtypes = (ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p, \
-			ctypes.c_ulong, ctypes.c_char_p)
-
-# Using ctypes to load the libc library is somewhat low level. Because of this
-# we need to define our own flags/options for use with mounting.
-MS_NOSUID = 2
-MS_NODEV = 4
-MS_NOEXEC = 8
-MS_STRICTATIME = 1 << 24
-STDIN_FILENO = 0
-TIOCSTTY = 0x540E
-
 config = None
 intf_id = 0
 
@@ -73,14 +57,7 @@ def exit_vm():
 
 	os.sync()
 
-	RB_AUTOBOOT = 0x01234567
-	#
-	# Calling 'reboot' or 'shutdown' from a shell (e.g. os.system('reboot'))
-	# is not the same the POSIX reboot() and will cause a kernel panic since
-	# we are the init process. The libc.reboot() allows the VM to exit
-	# gracefully.
-	#
-	libc.reboot(RB_AUTOBOOT)
+	runner.stop()
 
 def path_exists(path):
 	'''
@@ -104,38 +81,6 @@ def find_binary(list):
 			return path
 	return None
 
-def mount(source, target, fs, flags, options=''):
-	'''
-		Python wrapper for libc mount()
-	'''
-	ret = libc.mount(source.encode(), target.encode(), fs.encode(), flags,
-				options.encode())
-	if ret < 0:
-		errno = ctypes.get_errno()
-		raise Exception("Could not mount %s (%d)" % (target, errno))
-
-MountInfo = namedtuple('MountInfo', 'fstype target options flags')
-
-mount_table = [
-	MountInfo('sysfs', '/sys', '', MS_NOSUID|MS_NOEXEC|MS_NODEV),
-	MountInfo('proc', '/proc', '', MS_NOSUID|MS_NOEXEC|MS_NODEV),
-	MountInfo('devpts', '/dev/pts', 'mode=0620', MS_NOSUID|MS_NOEXEC),
-	MountInfo('tmpfs', '/dev/shm', 'mode=1777', MS_NOSUID|MS_NODEV|MS_STRICTATIME),
-	MountInfo('tmpfs', '/run', 'mode=0755', MS_NOSUID|MS_NODEV|MS_STRICTATIME),
-	MountInfo('tmpfs', '/tmp', '', 0),
-	MountInfo('tmpfs', '/usr/share/dbus-1', 'mode=0755', MS_NOSUID|MS_NOEXEC|MS_NODEV|MS_STRICTATIME),
-	MountInfo('debugfs', '/sys/kernel/debug', '', 0)
-]
-
-DevInfo = namedtuple('DevInfo', 'target linkpath')
-
-dev_table = [
-	DevInfo('/proc/self/fd', '/dev/fd'),
-	DevInfo('/proc/self/fd/0', '/dev/stdin'),
-	DevInfo('/proc/self/fd/1', '/dev/stdout'),
-	DevInfo('/proc/self/fd/2', '/dev/stderr')
-]
-
 # Partial DBus config. The remainder (<listen>) will be filled in for each
 # namespace that is created so each individual dbus-daemon has its own socket
 # and address.
@@ -910,6 +855,8 @@ class TestContext(Namespace):
 			self.cur_radio_id += 1
 
 	def discover_radios(self):
+		import pyroute2
+
 		phys = []
 
 		try:
@@ -1129,27 +1076,6 @@ class TestContext(Namespace):
 
 		return ret
 
-def prepare_sandbox():
-	print('Preparing sandbox')
-
-	for entry in mount_table:
-		try:
-			os.lstat(entry.target)
-		except:
-			os.mkdir(entry.target, 755)
-
-		mount(entry.fstype, entry.target, entry.fstype, entry.flags,
-			entry.options)
-
-	for entry in dev_table:
-		os.symlink(entry.target, entry.linkpath)
-
-	os.mkdir('/tmp/iwd')
-
-	os.setsid()
-
-	fcntl.ioctl(STDIN_FILENO, TIOCSTTY, 1)
-
 def build_unit_list(args):
 	'''
 		Build list of unit tests based on passed arguments. This first
@@ -1187,11 +1113,7 @@ def build_test_list(args):
 	test_root = args.testhome + '/autotests'
 
 	# Run all tests
-	if not args.auto_tests:
-		# --shell with no tests implies 'shell' test
-		if args.shell:
-			return [test_root + '/shell']
-
+	if not args.autotests:
 		# Get list of all autotests (committed in git)
 		tests = os.popen('git -C %s ls-files autotests/ | cut -f2 -d"/" \
 					| grep "test*" | uniq' % args.testhome).read() \
@@ -1202,7 +1124,7 @@ def build_test_list(args):
 
 		full_list = sorted(os.listdir(test_root))
 
-		for t in args.auto_tests.split(','):
+		for t in args.autotests.split(','):
 			path = '%s/%s' % (test_root, t)
 			if t.endswith('+'):
 				t = t.split('+')[0]
@@ -1527,26 +1449,11 @@ def print_results(results):
 def run_auto_tests(ctx, args):
 	tests = build_test_list(args)
 
-	# Copy autotests/misc/{certs,secrets,phonesim} so any test can refer to them
-	shutil.copytree(args.testhome + '/autotests/misc/certs', '/tmp/certs')
-	shutil.copytree(args.testhome + '/autotests/misc/secrets', '/tmp/secrets')
-	shutil.copy(args.testhome + '/autotests/misc/phonesim/phonesim.conf', '/tmp')
-
 	for test in tests:
 		copied = []
 		try:
 			subtests = pre_test(ctx, test, copied)
 
-			if args.shell:
-				#
-				# Shell really isn't meant to be used with multiple tests. If
-				# a set of tests was passed in just start out in the first.
-				#
-				os.chdir(tests[0])
-				os.environ['DBUS_SYSTEM_BUS_ADDRESS'] = ctx.dbus_address
-				os.system('/bin/bash')
-				sys.exit()
-
 			if len(subtests) < 1:
 				dbg("No tests to run")
 				sys.exit()
@@ -1576,11 +1483,6 @@ def run_auto_tests(ctx, args):
 		finally:
 			post_test(ctx, copied)
 
-	shutil.rmtree('/tmp/iwd')
-	shutil.rmtree('/tmp/certs')
-	shutil.rmtree('/tmp/secrets')
-	os.remove('/tmp/phonesim.conf')
-
 def run_unit_tests(ctx, args):
 	os.chdir(args.testhome + '/unit')
 	units = build_unit_list(args)
@@ -1592,54 +1494,10 @@ def run_unit_tests(ctx, args):
 		else:
 			dbg("Unit test %s passed" % os.path.basename(u))
 
-def run_tests():
+def run_tests(args):
 	global config
 
-	with open('/proc/cmdline', 'r') as f:
-		cmdline = f.read()
-
-	start = cmdline.find('--testhome')
-
-	options = shlex.split(cmdline[start:])
-
-	parser = argparse.ArgumentParser()
-	parser.add_argument('--testhome')
-	parser.add_argument('--auto_tests')
-	parser.add_argument('--unit_tests')
-	parser.add_argument('--verbose', default=[])
-	parser.add_argument('--debug')
-	parser.add_argument('--path')
-	parser.add_argument('--valgrind')
-	parser.add_argument('--gdb')
-	parser.add_argument('--shell')
-	parser.add_argument('--log')
-	parser.add_argument('--log-gid')
-	parser.add_argument('--log-uid')
-	parser.add_argument('--hw')
-	parser.add_argument('--monitor')
-	parser.add_argument('--sub_tests')
-	parser.add_argument('--result')
-
-	args = parser.parse_args(options)
-
-	#
-	# This prevents any print() calls in this script from printing unless
-	# --debug is passed. For an 'always print' option use dbg()
-	#
-	if not args.debug:
-		sys.stdout = open(os.devnull, 'w')
-
-	if args.verbose != []:
-		args.verbose = args.verbose.split(',')
-
-	os.environ['PATH'] = '%s/src' % args.testhome
-	os.environ['PATH'] += ':%s/tools' % args.testhome
-	os.environ['PATH'] += ':%s/client' % args.testhome
-	os.environ['PATH'] += ':%s/monitor' % args.testhome
-	os.environ['PATH'] += ':%s/wired' % args.testhome
-	os.environ['PATH'] += ':' + args.path
-
-	sys.path.append(args.testhome + '/autotests/util')
+	os.chdir(args.testhome)
 
 	#
 	# This allows all autotest utils (iwd/hostapd/etc) to access the
@@ -1654,347 +1512,17 @@ def run_tests():
 	config.hwsim = importlib.import_module('hwsim')
 	config.hostapd = importlib.import_module('hostapd')
 
-	if args.log:
-		mount('logdir', args.log, '9p', 0, 'trans=virtio,version=9p2000.L,msize=10240')
-		# Clear out any log files from other test runs
-		for f in glob('%s/*' % args.log):
-			print("removing %s" % f)
-
-			if os.path.isdir(f):
-				shutil.rmtree(f)
-			else:
-				os.remove(f)
-
-		# Start writing out kernel log
-		config.ctx.start_process(["dmesg", '--follow'])
-	elif args.monitor:
-		parent = os.path.abspath(os.path.join(args.monitor, os.pardir))
-		mount('mondir', parent, '9p', 0, 'trans=virtio,version=9p2000.L,msize=10240')
+	# Start writing out kernel log
+	config.ctx.start_process(["dmesg", '--follow'])
 
-	if args.result:
-		parent = os.path.abspath(os.path.join(args.result, os.pardir))
-		mount('resultdir', parent, '9p', 0, 'trans=virtio,version=9p2000.L,msize=10240')
-
-	if config.ctx.args.unit_tests is None:
+	if args.unit_tests is None:
 		run_auto_tests(config.ctx, args)
 	else:
 		run_unit_tests(config.ctx, args)
 
-class Main:
-	def __init__(self):
-		self.parser = argparse.ArgumentParser(
-				description='IWD Test Runner')
-
-		self.parser.add_argument('--qemu', '-q',
-				metavar='<QEMU binary>', type=str,
-				help='QEMU binary to use',
-				dest='qemu',
-				default=None)
-		self.parser.add_argument('--kernel', '-k', metavar='<kernel>',
-				type=str,
-				help='Path to kernel image',
-				dest='kernel',
-				default=None)
-		self.parser.add_argument('--verbose', '-v', metavar='<list>',
-				type=str,
-				help='Comma separated list of applications',
-				dest='verbose',
-				default=[])
-		self.parser.add_argument('--debug', '-d',
-				action='store_true',
-				help='Enable test-runner debugging',
-				dest='debug')
-		self.parser.add_argument('--shell', '-s', action='store_true',
-				help='Boot into shell', dest='shell')
-		self.parser.add_argument('--log', '-l', type=str,
-				help='Directory for log files')
-		self.parser.add_argument('--hw', '-w', type=str, nargs=1,
-				help='Use physical adapters for tests (passthrough)')
-		self.parser.add_argument('--monitor', '-m', type=str,
-				help='Enables iwmon output to file')
-		self.parser.add_argument('--sub-tests', '-S', metavar='<subtests>',
-				type=str, nargs=1, help='List of subtests to run',
-				default=None, dest='sub_tests')
-		self.parser.add_argument('--result', '-r', type=str,
-				help='Writes PASS/FAIL to results file')
-
-		# Prevent --autotest/--unittest from being used together
-		auto_unit_group = self.parser.add_mutually_exclusive_group()
-		auto_unit_group.add_argument('--auto-tests', '-A',
-				metavar='<tests>', type=str, nargs=1,
-				help='List of tests to run',
-				default=None,
-				dest='auto_tests')
-		auto_unit_group.add_argument('--unit-tests', '-U',
-				metavar='<tests>', type=str, nargs='?',
-				const='*',
-				help='List of unit tests to run',
-				dest='unit_tests')
-
-		# Prevent --valgrind/--gdb from being used together
-		valgrind_gdb_group = self.parser.add_mutually_exclusive_group()
-		valgrind_gdb_group.add_argument('--gdb', '-g', metavar='<exec>',
-				type=str, nargs=1,
-				help='Run gdb on specified executable',
-				dest='gdb')
-		valgrind_gdb_group.add_argument('--valgrind', '-V', action='store_true',
-				help='Run valgrind on IWD', dest='valgrind')
-
-		self.args = self.parser.parse_args()
-
-		if self.args.auto_tests:
-			self.args.auto_tests = self.args.auto_tests[0].split(',')
-
-		if self.args.sub_tests:
-			self.args.sub_tests = self.args.sub_tests[0].split(',')
-
-		if self.args.log and self.args.unit_tests:
-			dbg("Cannot use --log with --unit-tests")
-			quit()
-
-		if self.args.sub_tests:
-			if not self.args.auto_tests:
-				dbg("--sub-tests must be used with --auto-tests")
-				quit()
-
-			if len(self.args.auto_tests) > 1:
-				dbg("--sub-tests must be used with a single auto test")
-				quit()
-
-	def start(self):
-		usb_adapters = None
-		pci_adapters = None
-
-		qemu_table = [
-			'qemu-system-x86_64',
-			'/usr/bin/qemu-system-x86_64'
-		]
-
-		kernel_table = [
-			'bzImage',
-			'arch/x86/boot/bzImage',
-			'vmlinux',
-			'arch/x86/boot/vmlinux'
-		]
-
-		if self.args.qemu is None:
-			qemu_binary = find_binary(qemu_table)
-			if not qemu_binary:
-				print("Could not find qemu binary")
-				quit()
-		else:
-			if path_exists(self.args.qemu):
-				qemu_binary = self.args.qemu
-			else:
-				print("QEMU binary %s does not exist" % \
-						self.args.qemu)
-				quit()
-
-		if self.args.kernel is None:
-			kernel_binary = find_binary(kernel_table)
-			if not kernel_binary:
-				print("Could not find kernel image")
-				quit()
-		else:
-			if path_exists(self.args.kernel):
-				kernel_binary = self.args.kernel
-			else:
-				print("Kernel image %s does not exist" % \
-						self.args.kernel)
-				quit()
-
-		if self.args.hw:
-			hw_conf = ConfigParser()
-			hw_conf.read(self.args.hw)
-
-			if hw_conf.has_section('USBAdapters'):
-				# The actual key name of the adapter
-				# doesn't matter since all we need is the
-				# bus/address. This gets named by the kernel
-				# anyways once in the VM.
-				usb_adapters = [v for v in hw_conf['USBAdapters'].values()]
-
-			if hw_conf.has_section('PCIAdapters'):
-				pci_adapters = [v for v in hw_conf['PCIAdapters'].values()]
-
-		#
-		# Additional arguments not provided to test-runner which are
-		# needed once booted into the kernel.
-		#
-		options = 'init=%s' % os.path.realpath(sys.argv[0])
-
-		# Support running from top level as well as tools
-		if os.getcwd().endswith('tools'):
-			options += ' --testhome %s/../' % os.getcwd()
-		else:
-			options += ' --testhome %s' % os.getcwd()
-
-		options += ' --path "%s"' % os.environ['PATH']
-
-		if self.args.auto_tests:
-			options += ' --auto_tests %s' % ','.join(self.args.auto_tests)
-
-		if self.args.sub_tests:
-			options += ' --sub_tests %s' % ','.join(self.args.sub_tests)
-
-		gid = None
-		if os.environ.get('SUDO_GID', None):
-			uid = int(os.environ['SUDO_UID'])
-			gid = int(os.environ['SUDO_GID'])
-
-		append_gid_uid = False
-
-		if self.args.log:
-			if os.getuid() != 0:
-				print("--log can only be used as root user")
-				quit()
-
-			self.args.log = os.path.abspath(self.args.log)
-			append_gid_uid = True
-
-			if not path_exists(self.args.log):
-				os.mkdir(self.args.log)
-				if gid:
-					os.chown(self.args.log, uid, gid)
-
-		if self.args.monitor:
-			if os.getuid() != 0:
-				print("--monitor can only be used as root user")
-				quit()
+runner = Runner(from_env=True)
 
-			self.args.monitor = os.path.abspath(self.args.monitor)
-			mon_parent_dir = os.path.abspath(os.path.join(self.args.monitor, os.pardir))
-			append_gid_uid = True
-
-		if self.args.result:
-			if os.getuid() != 0:
-				print("--result can only be used as root user")
-				quit()
-
-			self.args.result = os.path.abspath(self.args.result)
-			result_parent_dir = os.path.abspath(os.path.join(self.args.result, os.pardir))
-			append_gid_uid = True
-
-		if append_gid_uid and gid:
-			options += ' --log-gid %u' % (gid,)
-			options += ' --log-uid %u' % (uid,)
-
-		denylist = [
-			'auto_tests',
-			'sub_tests',
-			'qemu',
-			'kernel'
-		]
-
-		nproc = multiprocessing.cpu_count()
-
-		#
-		# Specially handle CPU systems with minimal cores, otherwise
-		# use half the host cores.
-		#
-		if nproc < 2:
-			smp = 1
-		else:
-			smp = int(nproc / 2)
-
-		ram = 256
-
-		print("Using %d cores, %d RAM for VM" % (smp, ram))
-
-		#
-		# This passes through most of the command line options to
-		# the kernel command line. Some are not relevant (e.g. qemu)
-		# so similar options are added in the denylist above. This excludes
-		# any unset options which are assumed to be None or False. This
-		# is done so default arguments can be filled once in the VM. If
-		# we pass and basic types (None, False etc.) they are turned into
-		# a string representation ('None', 'False', etc.) which is not
-		# desirable.
-		#
-		for arg in vars(self.args):
-			if arg in denylist or getattr(self.args, arg) in [None, False, []]:
-				continue
-			options += ' --%s %s' % (arg, str(getattr(self.args, arg)))
-
-		kern_log = "ignore_loglevel" if "kernel" in self.args.verbose else "quiet"
-
-		qemu_cmdline = [
-			qemu_binary,
-			'-machine', 'type=q35,accel=kvm:tcg',
-			'-nodefaults', '-no-user-config', '-monitor', 'none',
-			'-display', 'none', '-m', '%dM' % ram, '-nographic', '-vga',
-			'none', '-no-acpi', '-no-hpet',
-			'-no-reboot', '-fsdev',
-			'local,id=fsdev-root,path=/,readonly=on,security_model=none,multidevs=remap',
-			'-device',
-			'virtio-9p-pci,fsdev=fsdev-root,mount_tag=/dev/root',
-			'-chardev', 'stdio,id=chardev-serial0,signal=off',
-			'-device', 'pci-serial,chardev=chardev-serial0',
-			'-device', 'virtio-rng-pci',
-			'-kernel',
-			kernel_binary,
-			'-append',
-			'console=ttyS0,115200n8 earlyprintk=serial \
-				rootfstype=9p root=/dev/root \
-				rootflags=trans=virtio \
-				acpi=off pci=noacpi %s ro \
-				mac80211_hwsim.radios=0 %s' % (kern_log, options),
-			'-smp', str(smp)
-		]
-
-		# Add two ethernet devices for testing EAD
-		qemu_cmdline.extend([
-				'-net', 'nic,model=virtio',
-				'-net', 'nic,model=virtio',
-				'-net', 'user'
-		])
-
-		if usb_adapters:
-			for bus, addr in [s.split(',') for s in usb_adapters]:
-				qemu_cmdline.extend(['-usb',
-							'-device',
-							'usb-host,hostbus=%s,hostaddr=%s' % \
-							(bus, addr)])
-		if pci_adapters:
-			qemu_cmdline.extend(['-enable-kvm'])
-			for addr in pci_adapters:
-				qemu_cmdline.extend(['-device', 'vfio-pci,host=%s' % addr])
-
-		if self.args.log:
-			#
-			# Creates a virtfs device that can be mounted. This mount
-			# will point back to the provided log directory and is
-			# writable unlike the rest of the mounted file system.
-			#
-			qemu_cmdline.extend([
-				'-virtfs',
-				'local,path=%s,mount_tag=logdir,security_model=passthrough,id=logdir' \
-						% self.args.log
-			])
-
-		if self.args.monitor:
-			qemu_cmdline.extend([
-				'-virtfs',
-				'local,path=%s,mount_tag=mondir,security_model=passthrough,id=mondir' \
-						% mon_parent_dir
-			])
-
-		if self.args.result:
-			qemu_cmdline.extend([
-				'-virtfs',
-				'local,path=%s,mount_tag=resultdir,security_model=passthrough,id=resultdir' \
-						% result_parent_dir
-			])
-
-		os.execlp(qemu_cmdline[0], *qemu_cmdline)
-
-if __name__ == '__main__':
-	if os.getpid() == 1 and os.getppid() == 0:
-		atexit.register(exit_vm)
-		prepare_sandbox()
-		run_tests()
-
-		sys.exit()
-
-	main = Main()
-	main.start()
+atexit.register(exit_vm)
+runner.prepare_environment()
+run_tests(runner.args)
+runner.cleanup_environment()
Signature · 1 line
2.34.1
author
#3 lore raw

PATCH v2 3/14 test-runner: rename test-runner, add run-tests

In reply to James Prestwood

In order to keep all test-runner dev scripts working and to work with the new runner.py system some file renaming was required.

test-runner was renamed to run-tests
A new test-runner was added which only creates the Runner() class.

---
 tools/run-tests   | 1526 +++++++++++++++++++++++++++++++++++++++++++++
 tools/test-runner | 1525 +-------------------------------------------
 2 files changed, 1527 insertions(+), 1524 deletions(-)
 create mode 100755 tools/run-tests
tools/run-tests new file +1526 -0
new file mode 100755
index 00000000..716126d0
@@ -0,0 +1,1526 @@
+#!/usr/bin/python3
+
+import os
+import shutil
+import fcntl
+import sys
+import subprocess
+import atexit
+import time
+import unittest
+import importlib
+from unittest.result import TestResult
+import multiprocessing
+import re
+import traceback
+
+from runner import Runner
+
+from configparser import ConfigParser
+from prettytable import PrettyTable
+from termcolor import colored
+from glob import glob
+from collections import namedtuple
+from time import sleep
+import dbus.mainloop.glib
+from gi.repository import GLib
+from weakref import WeakValueDictionary
+
+config = None
+intf_id = 0
+
+TEST_MAX_TIMEOUT = 240
+
+dbus.mainloop.glib.DBusGMainLoop(set_as_default=True)
+
+def dbg(*s, **kwargs):
+	'''
+		Allows prints if stdout has been re-directed
+	'''
+	print(*s, **kwargs, file=sys.__stdout__)
+
+def exit_vm():
+	if config:
+		for p in Process.get_all():
+			print("Process %s still running!" % p.args[0])
+			p.kill()
+
+		if config.ctx and config.ctx.results:
+			success = print_results(config.ctx.results)
+		else:
+			success = False
+
+		if config.ctx.args.result:
+			result = 'PASS' if success else 'FAIL'
+			with open(config.ctx.args.result, 'w') as f:
+				f.write(result)
+
+	os.sync()
+
+	runner.stop()
+
+def path_exists(path):
+	'''
+		Searches PATH as well as absolute paths.
+	'''
+	if shutil.which(path):
+		return True
+	try:
+		os.stat(path)
+	except:
+		return False
+	return True
+
+def find_binary(list):
+	'''
+		Returns a binary from 'list' if its found in PATH or on a
+		valid absolute path.
+	'''
+	for path in list:
+		if path_exists(path):
+			return path
+	return None
+
+# Partial DBus config. The remainder (<listen>) will be filled in for each
+# namespace that is created so each individual dbus-daemon has its own socket
+# and address.
+dbus_config = '''
+<!DOCTYPE busconfig PUBLIC \
+"-//freedesktop//DTD D-Bus Bus Configuration 1.0//EN" \
+"http://www.freedesktop.org/standards/dbus/1.0/\
+busconfig.dtd\">
+<busconfig>
+<type>system</type>
+<limit name=\"reply_timeout\">2147483647</limit>
+<auth>ANONYMOUS</auth>
+<allow_anonymous/>
+<policy context=\"default\">
+<allow user=\"*\"/>
+<allow own=\"*\"/>
+<allow send_type=\"method_call\"/>
+<allow send_type=\"signal\"/>
+<allow send_type=\"method_return\"/>
+<allow send_type=\"error\"/>
+<allow receive_type=\"method_call\"/>
+<allow receive_type=\"signal\"/>
+<allow receive_type=\"method_return\"/>
+<allow receive_type=\"error\"/>
+<allow send_destination=\"*\" eavesdrop=\"true\"/>
+<allow eavesdrop=\"true\"/>
+</policy>
+'''
+
+class Process(subprocess.Popen):
+	processes = WeakValueDictionary()
+	ctx = None
+
+	def __new__(cls, *args, **kwargs):
+		obj = super().__new__(cls)
+		cls.processes[id(obj)] = obj
+		return obj
+
+	def __init__(self, args, namespace=None, outfile=None, env=None, check=False, cleanup=None):
+		self.write_fds = []
+		self.io_watch = None
+		self.cleanup = cleanup
+		self.verbose = False
+		self.out = ''
+		self.hup = False
+		self.killed = False
+		self.namespace = namespace
+
+		if not self.ctx:
+			global config
+			self.ctx = config.ctx
+
+		if self.ctx.is_verbose(args[0], log=False):
+			self.verbose = True
+
+		if namespace:
+			args = ['ip', 'netns', 'exec', namespace] + args
+
+		if outfile:
+			# outfile is only used by iwmon, in which case we don't want
+			# to append to an existing file.
+			self._append_outfile(outfile, append=False)
+
+		if self.ctx.args.log:
+			logfile = '%s/%s/%s' % (self.ctx.args.log,
+						os.path.basename(os.getcwd()),
+						args[0])
+			self._append_outfile(logfile)
+
+		super().__init__(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+					env=env, cwd=os.getcwd())
+
+		# Set as non-blocking so read() in the IO callback doesn't block forever
+		fl = fcntl.fcntl(self.stdout, fcntl.F_GETFL)
+		fcntl.fcntl(self.stdout, fcntl.F_SETFL, fl | os.O_NONBLOCK)
+
+		self.io_watch = GLib.io_add_watch(self.stdout, GLib.IO_IN |
+						GLib.IO_HUP | GLib.IO_ERR, self.process_io)
+
+		print("Starting process {}".format(self.args))
+
+		if check:
+			self.wait(10)
+			self.killed = True
+			if self.returncode != 0:
+				raise subprocess.CalledProcessError(returncode=self.returncode,
+									cmd=args)
+
+	@classmethod
+	def get_all(cls):
+		return cls.processes.values()
+
+	@classmethod
+	def kill_all(cls):
+		for p in cls.processes.values():
+			p.kill()
+
+	@staticmethod
+	def _write_io(instance, data, stdout=True):
+		for f in instance.write_fds:
+			f.write(data)
+
+			# Write out a separator so multiple process calls per
+			# test are easer to read.
+			if instance.hup:
+				f.write("Terminated: {}\n\n".format(instance.args))
+
+			f.flush()
+
+		if instance.verbose and stdout:
+			sys.__stdout__.write(data)
+			sys.__stdout__.flush()
+
+	@classmethod
+	def write_separators(cls, sep):
+		for proc in cls.processes.values():
+			if proc.killed:
+				continue
+
+			cls._write_io(proc, sep, stdout=False)
+
+	def process_io(self, source, condition):
+		if condition & GLib.IO_HUP:
+			self.hup = True
+
+		data = source.read()
+
+		if not data:
+			return True
+
+		data = data.decode('utf-8')
+
+		# Save data away in case the caller needs it (e.g. list_sta)
+		self.out += data
+
+		self._write_io(self, data)
+
+		return True
+
+	def _append_outfile(self, file, append=True):
+		gid = int(os.environ.get('SUDO_GID', os.getgid()))
+		uid = int(os.environ.get('SUDO_UID', os.getuid()))
+		dir = os.path.dirname(file)
+
+		if not path_exists(dir):
+			os.mkdir(dir)
+			os.chown(dir, uid, gid)
+
+		file = os.path.join(dir,file)
+
+		# If the out file exists, append. Useful for processes like
+		# hostapd_cli where it is called multiple times independently.
+		if os.path.isfile(file) and append:
+			mode = 'a'
+		else:
+			mode = 'w'
+
+		try:
+			f = open(file, mode)
+		except Exception as e:
+			traceback.print_exc()
+			sys.exit(0)
+
+		os.fchown(f.fileno(), uid, gid)
+
+		self.write_fds.append(f)
+
+	def wait_for_socket(self, socket, wait):
+		Namespace.non_block_wait(os.path.exists, wait, socket)
+
+	# Wait for both process termination and HUP signal
+	def __wait(self, timeout):
+		try:
+			super().wait(timeout)
+			if not self.hup:
+				return False
+
+			return True
+		except:
+			return False
+
+	# Override wait() so it can do so non-blocking
+	def wait(self, timeout=10):
+		Namespace.non_block_wait(self.__wait, timeout, 1)
+		self._cleanup()
+
+	def _cleanup(self):
+		if self.cleanup:
+			self.cleanup()
+
+		self.write_fds = []
+
+		if self.io_watch:
+			GLib.source_remove(self.io_watch)
+			self.io_watch = None
+
+		self.cleanup = None
+		self.killed = True
+
+	# Override kill()
+	def kill(self, force=False):
+		if self.killed:
+			return
+
+		print("Killing process {}".format(self.args))
+
+		if force:
+			super().kill()
+		else:
+			self.terminate()
+
+		try:
+			self.wait(timeout=15)
+		except:
+			dbg("Process %s did not complete in 15 seconds!" % self.name)
+			super().kill()
+
+		self._cleanup()
+
+	def __str__(self):
+		return str(self.args) + '\n'
+
+class Interface:
+	def __init__(self, name, config):
+		self.name = name
+		self.ctrl_interface = '/var/run/hostapd/' + name
+		self.config = config
+
+	def __del__(self):
+		Process(['iw', 'dev', self.name, 'del']).wait()
+
+	def set_interface_state(self, state):
+		Process(['ip', 'link', 'set', self.name, state]).wait()
+
+class Radio:
+	def __init__(self, name):
+		self.name = name
+		# hostapd will reset this if this radio is used by it
+		self.use = 'iwd'
+		self.interface = None
+
+	def __del__(self):
+		print("Removing radio %s" % self.name)
+		self.interface = None
+
+	def create_interface(self, config, use):
+		global intf_id
+
+		ifname = 'wln%s' % intf_id
+
+		intf_id += 1
+
+		self.interface = Interface(ifname, config)
+		self.use = use
+
+		Process(['iw', 'phy', self.name, 'interface', 'add', ifname,
+				'type', 'managed']).wait()
+
+		return self.interface
+
+	def __str__(self):
+		ret = self.name + ':\n'
+		ret += '\tUsed By: %s ' % self.use
+		if self.interface:
+			ret += '(%s)' % self.interface.name
+
+		ret += '\n'
+
+		return ret
+
+class VirtualRadio(Radio):
+	'''
+		A subclass of 'Radio' specific to mac80211_hwsim radios.
+
+		TODO: Using D-Bus to create and destroy radios is more desireable
+		than the command line.
+	'''
+
+	def __init__(self, name, cfg=None):
+		global config
+
+		self.disable_cipher = None
+		self.disable_iftype = None
+
+		self.hwsim = config.hwsim.Hwsim()
+
+		if cfg:
+			self.disable_iftype = cfg.get('iftype_disable', None)
+			self.disable_cipher = cfg.get('cipher_disable', None)
+
+		self._radio = self.hwsim.radios.create(name, p2p_device=True,
+					iftype_disable=self.disable_iftype,
+					cipher_disable=self.disable_cipher)
+
+		super().__init__(self._radio.name)
+
+	def __del__(self):
+		super().__del__()
+
+		# If the radio was moved into a namespace this will fail
+		try:
+			self._radio.remove()
+		except:
+			pass
+
+		self._radio = None
+
+	def __str__(self):
+		ret = super().__str__()
+
+		if self.disable_iftype:
+			ret += '\tDisabled interface types: %s\n' % self.disable_iftype
+
+		if self.disable_cipher:
+			ret += '\tDisabled ciphers: %s\n' % self.disable_cipher
+
+		ret += '\tPath: %s' % self._radio.path
+
+		ret += '\n'
+
+		return ret
+
+class HostapdInstance:
+	'''
+		A single instance of hostapd. In reality all hostapd instances
+		are started as a single process. This class just makes things
+		convenient for communicating with one of the hostapd APs.
+	'''
+	def __init__(self, config, radio):
+		self.radio = radio
+		self.config = config
+		self.cli = None
+
+		self.intf = radio.create_interface(self.config, 'hostapd')
+		self.intf.set_interface_state('up')
+
+	def __del__(self):
+		print("Removing HostapdInstance %s" % self.config)
+		self.intf.set_interface_state('down')
+		self.radio = None
+		self.intf = None
+
+	def __str__(self):
+		ret = 'Hostapd (%s)\n' % self.intf.name
+		ret += '\tConfig: %s\n' % self.config
+
+		return ret
+
+class Hostapd:
+	'''
+		A set of running hostapd instances. This is really just a single
+		process since hostapd can be started with multiple config files.
+	'''
+	def __init__(self, ctx, radios, configs, radius):
+		self.ctx = ctx
+
+		if len(configs) != len(radios):
+			raise Exception("Config (%d) and radio (%d) list length not equal" % \
+						(len(configs), len(radios)))
+
+		print("Initializing hostapd instances")
+
+		Process(['ip', 'link', 'set', 'eth0', 'up']).wait()
+		Process(['ip', 'link', 'set', 'eth1', 'up']).wait()
+
+		self.global_ctrl_iface = '/var/run/hostapd/ctrl'
+
+		self.instances = [HostapdInstance(c, r) for c, r in zip(configs, radios)]
+
+		ifaces = [rad.interface.name for rad in radios]
+		ifaces = ','.join(ifaces)
+
+		args = ['hostapd', '-g', self.global_ctrl_iface]
+
+		if ifaces:
+			args.extend(['-i', ifaces])
+
+		#
+		# Config files should already be present in /tmp. This appends
+		# ctrl_interface and does any variable replacement. Currently
+		# this is just any $ifaceN occurrences.
+		#
+		for c in configs:
+			full_path = '/tmp/%s' % c
+			args.append(full_path)
+
+			self._rewrite_config(full_path)
+
+		if radius:
+			args.append(radius)
+
+		if ctx.is_verbose('hostapd'):
+			args.append('-d')
+
+		self.process = Process(args)
+
+		self.process.wait_for_socket(self.global_ctrl_iface, 30)
+
+		for hapd in self.instances:
+			self.process.wait_for_socket(hapd.intf.ctrl_interface, 30)
+
+	def attach_cli(self):
+		global config
+
+		for hapd in self.instances:
+			hapd.cli = config.hostapd.HostapdCLI(config=hapd.config)
+
+	def _rewrite_config(self, config):
+		'''
+			Replaces any $ifaceN values with the correct interface
+			names as well as appends the ctrl_interface path to
+			the config file.
+		'''
+		with open(config, 'r+') as f:
+			data = f.read()
+			to_replace = []
+			for match in re.finditer(r'\$iface[0-9]+', data):
+				tag = data[match.start():match.end()]
+				idx = tag.split('iface')[1]
+
+				to_replace.append((tag, self.instances[int(idx)].intf.name))
+
+			for r in to_replace:
+				data = data.replace(r[0], r[1], 1)
+
+			data += '\nctrl_interface=/var/run/hostapd\n'
+
+			f.write(data)
+
+	def __getitem__(self, config):
+		if not config:
+			return self.instances[0]
+
+		for hapd in self.instances:
+			if hapd.config == config:
+				return hapd
+
+		return None
+
+	def __del__(self):
+		print("Removing Hostapd")
+		try:
+			os.remove(self.global_ctrl_iface)
+		except:
+			print("Failed to remove %s" % self.global_ctrl_iface)
+
+		self.instances = None
+
+		# Hostapd may have already been stopped
+		if self.process:
+			self.ctx.stop_process(self.process)
+
+		self.ctx = None
+
+		# Hostapd creates simdb sockets for EAP-SIM/AKA tests but does not
+		# clean them up.
+		for f in glob("/tmp/eap_sim_db*"):
+			os.remove(f)
+
+dbus_count = 0
+
+class Namespace:
+	def __init__(self, args, name, radios):
+		self.dbus_address = None
+		self.name = name
+		self.radios = radios
+		self.args = args
+
+		Process(['ip', 'netns', 'add', name]).wait()
+		for r in radios:
+			Process(['iw', 'phy', r.name, 'set', 'netns', 'name', name]).wait()
+
+		self.start_dbus()
+
+	def reset(self):
+		self._bus = None
+
+		for r in self.radios:
+			r._radio = None
+
+		self.radios = []
+
+		Process.kill_all()
+
+	def __del__(self):
+		if self.name:
+			print("Removing namespace %s" % self.name)
+
+			Process(['ip', 'netns', 'del', self.name]).wait()
+
+	def get_bus(self):
+		return self._bus
+
+	def start_process(self, args, env=None, **kwargs):
+		if not env:
+			env = os.environ.copy()
+
+		if hasattr(self, "dbus_address"):
+			# In case this process needs DBus...
+			env['DBUS_SYSTEM_BUS_ADDRESS'] = self.dbus_address
+
+		return Process(args, namespace=self.name, env=env, **kwargs)
+
+	def stop_process(self, p, force=False):
+		p.kill(force)
+
+	def is_process_running(self, process):
+		for p in Process.get_all():
+			if p.namespace == self.name and p.args[0] == process:
+				return True
+		return False
+
+	def _cleanup_dbus(self):
+		try:
+			os.remove(self.dbus_address.split('=')[1])
+		except:
+			pass
+
+		os.remove(self.dbus_cfg)
+
+	def start_dbus(self):
+		global dbus_count
+
+		self.dbus_address = 'unix:path=/tmp/dbus%d' % dbus_count
+		self.dbus_cfg = '/tmp/dbus%d.conf' % dbus_count
+		dbus_count += 1
+
+		with open(self.dbus_cfg, 'w+') as f:
+			f.write(dbus_config)
+			f.write('<listen>%s</listen>\n' % self.dbus_address)
+			f.write('</busconfig>\n')
+
+		p = self.start_process(['dbus-daemon', '--config-file=%s' % self.dbus_cfg],
+					cleanup=self._cleanup_dbus)
+
+		p.wait_for_socket(self.dbus_address.split('=')[1], 5)
+
+		self._bus = dbus.bus.BusConnection(address_or_type=self.dbus_address)
+
+	def start_iwd(self, config_dir = '/tmp', storage_dir = '/tmp/iwd'):
+		args = []
+		iwd_radios = ','.join([r.name for r in self.radios if r.use == 'iwd'])
+
+		if self.args.valgrind:
+			args.extend(['valgrind', '--leak-check=full', '--track-origins=yes',
+					'--show-leak-kinds=all',
+					'--log-file=/tmp/valgrind.log.%p'])
+
+		args.extend(['iwd', '-E'])
+
+		if iwd_radios != '':
+			args.extend(['-p', iwd_radios])
+
+		if self.is_verbose(args[0]):
+			args.append('-d')
+
+		env = os.environ.copy()
+
+		env['CONFIGURATION_DIRECTORY'] = config_dir
+		env['STATE_DIRECTORY'] = storage_dir
+
+		if self.is_verbose('iwd-dhcp'):
+			env['IWD_DHCP_DEBUG'] = '1'
+
+		if self.is_verbose('iwd-tls'):
+			env['IWD_TLS_DEBUG'] = '1'
+
+		if self.is_verbose('iwd-acd'):
+			env['IWD_ACD_DEBUG'] = '1'
+
+		return self.start_process(args, env=env)
+
+	def is_verbose(self, process, log=True):
+		process = os.path.basename(process)
+
+		if self.args is None:
+			return False
+
+		# every process is verbose when logging is enabled
+		if log and self.args.log:
+			return True
+
+		if process in self.args.verbose:
+			return True
+
+		# Special case here to enable verbose output with valgrind running
+		if process == 'valgrind' and 'iwd' in self.args.verbose:
+			return True
+
+		# Handle any glob matches
+		for item in self.args.verbose:
+			if process in glob(item):
+				return True
+
+		return False
+
+	@staticmethod
+	def non_block_wait(func, timeout, *args, exception=True):
+		'''
+			Convenience function for waiting in a non blocking
+			manor using GLibs context iteration i.e. does not block
+			the main loop while waiting.
+
+			'func' will be called at least once and repeatedly until
+			either it returns success, throws an exception, or the
+			'timeout' expires.
+
+			'timeout' is the ultimate timeout in seconds
+
+			'*args' will be passed to 'func'
+
+			If 'exception' is an Exception type it will be raised.
+			If 'exception' is True a generic TimeoutError will be raised.
+			Any other value will not result in an exception.
+		'''
+		# Simple class for signaling the wait timeout
+		class Bool:
+			def __init__(self, value):
+				self.value = value
+
+		def wait_timeout_cb(done):
+			done.value = True
+			return False
+
+		mainloop = GLib.MainLoop()
+		done = Bool(False)
+
+		timeout = GLib.timeout_add_seconds(timeout, wait_timeout_cb, done)
+		context = mainloop.get_context()
+
+		while True:
+			context.iteration(may_block=False)
+
+			try:
+				ret = func(*args)
+				if ret:
+					if not done.value:
+						GLib.source_remove(timeout)
+					return ret
+			except Exception as e:
+				if not done.value:
+					GLib.source_remove(timeout)
+				raise e
+
+			sleep(0.1)
+
+			if done.value == True:
+				if isinstance(exception, Exception):
+					raise exception
+				elif type(exception) == bool and exception:
+					raise TimeoutError("Timeout on non_block_wait")
+				else:
+					return
+
+	def __str__(self):
+		ret = 'Namespace: %s\n' % self.name
+		ret += 'Processes:\n'
+		for p in Process.get_all():
+			ret += '\t%s' % str(p)
+
+		ret += 'Radios:\n'
+		if len(self.radios) > 0:
+			for r in self.radios:
+				ret += '\t%s\n' % str(r)
+		else:
+			ret += '\tNo Radios\n'
+
+		ret += 'DBus Address: %s\n' % self.dbus_address
+		ret += '===================================================\n\n'
+
+		return ret
+
+class BarChart():
+	def __init__(self, height=10, max_width=80):
+		self._height = height
+		self._max_width = max_width
+		self._values = []
+		self._max_value = 0
+		self._min_value = 0
+
+	def add_value(self, value):
+		if len(self._values) == 0:
+			self._max_value = int(1.01 * value)
+			self._min_value = int(0.99 * value)
+		elif value > self._max_value:
+			self._max_value = int(1.01 * value)
+		elif value < self._min_value:
+			self._min_value = int(0.99 * value)
+
+		self._values.append(value)
+
+	def _value_to_stars(self, value):
+		# Need to scale value (range of min_value -> max_value) to
+		# a range of 0 -> height
+		#
+		# Scaled = ((value - min_value) / ( max_value - min_value)) * (Height - 0) + 0
+
+		return int(((value - self._min_value) /
+			(self._max_value - self._min_value)) * self._height)
+
+	def __str__(self):
+		# Need to map value from range 0 - self._height
+		ret = ''
+
+		for i, value in enumerate(self._values):
+			stars = self._value_to_stars(value)
+			ret += '[%3u] ' % i + '%-10s' % ('*' * stars) + '\t\t\t%d\n' % value
+
+		ret += '\n'
+
+		return ret
+
+
+class TestContext(Namespace):
+	'''
+		Contains all information for a given set of tests being run
+		such as processes, radios, interfaces and test results.
+	'''
+	def __init__(self, args):
+		self.name = None
+		self.args = args
+		self.hw_config = None
+		self.hostapd = None
+		self.wpas_interfaces = None
+		self.cur_radio_id = 0
+		self.cur_iface_id = 0
+		self.radios = []
+		self.loopback_started = False
+		self.results = {}
+		self.mainloop = GLib.MainLoop()
+		self.namespaces = []
+		self._last_mem_available = 0
+		self._mem_chart = BarChart()
+
+	def start_dbus_monitor(self):
+		if not self.is_verbose('dbus-monitor'):
+			return
+
+		self.start_process(['dbus-monitor', '--address', self.dbus_address])
+
+	def start_haveged(self):
+		self.start_process(['haveged', '-F'])
+
+	def create_radios(self):
+		setup = self.hw_config['SETUP']
+		nradios = int(setup['num_radios'])
+		args = ['hwsim']
+
+		if self.hw_config['SETUP'].get('hwsim_medium', 'no') in ['no', '0', 'false']:
+			# register hwsim as medium
+			args.extend(['--no-register'])
+
+		self.start_process(args)
+		self.non_block_wait(self._bus.name_has_owner, 20, 'net.connman.hwsim',
+					exception=TimeoutError('net.connman.hwsim did not appear'))
+
+		for i in range(nradios):
+			name = 'rad%u' % i
+
+			# Get any [radX] sections. These are for configuring
+			# any special radios. This no longer requires a
+			# radio_conf list, we just assume radios start rad0
+			# and increment.
+			rad_config = None
+			if self.hw_config.has_section(name):
+				rad_config = self.hw_config[name]
+
+			self.radios.append(VirtualRadio(name, rad_config))
+			self.cur_radio_id += 1
+
+	def discover_radios(self):
+		import pyroute2
+
+		phys = []
+
+		try:
+			iw = pyroute2.iwutil.IW()
+		except:
+			iw = pyroute2.IW()
+
+		attrs = [phy['attrs'] for phy in iw.list_wiphy()]
+
+		for attr in attrs:
+			for key, value in attr:
+				if key == 'NL80211_ATTR_WIPHY_NAME':
+					if value not in phys:
+						phys.append(value)
+					break
+
+		print('Discovered radios: %s' % str(phys))
+		self.radios = [Radio(name) for name in phys]
+
+	def start_radios(self):
+		reg_domain = self.hw_config['SETUP'].get('reg_domain', None)
+		if reg_domain:
+			Process(['iw', 'reg', 'set', reg_domain]).wait()
+
+		if self.args.hw:
+			self.discover_radios()
+		else:
+			self.create_radios()
+
+	def start_hostapd(self):
+		if not 'HOSTAPD' in self.hw_config:
+			return
+
+		settings = self.hw_config['HOSTAPD']
+
+		if self.args.hw:
+			# Just grab the first N radios. It gets rather
+			# complicated trying to map radX radios specified in
+			# hw.conf so any passed through physical adapters are
+			# just given to hostapd/IWD as they appear during
+			# discovery.
+			#
+			# TODO: It may be desireable to map PCI/USB adapters to
+			#       specific radX radios specified in the config but
+			#       there are really 2 separate use cases here.
+			#       1. You want to test a *specific* radio with IWD
+			#          or hostapd. For this you would want radX
+			#          to map to a specific radio
+			#       2. You have many adapters in use to run multiple
+			#          tests. In this case you would not care what
+			#          was using each radio, just that there was
+			#          enough to run all tests.
+			nradios = 0
+			for k, _ in settings.items():
+				if k == 'radius_server':
+					continue
+				nradios += 1
+
+			hapd_radios = self.radios[:nradios]
+
+		else:
+			hapd_radios = [rad for rad in self.radios if rad.name in settings]
+
+		hapd_configs = [conf for rad, conf in settings.items() if rad != 'radius_server']
+
+		radius_config = settings.get('radius_server', None)
+
+		self.hostapd = Hostapd(self, hapd_radios, hapd_configs, radius_config)
+		self.hostapd.attach_cli()
+
+	def get_frequencies(self):
+		frequencies = []
+
+		for hapd in self.hostapd.instances:
+			frequencies.append(hapd.cli.frequency)
+
+		return frequencies
+
+	def start_wpas_interfaces(self):
+
+		if 'WPA_SUPPLICANT' not in self.hw_config:
+			return
+
+		settings = self.hw_config['WPA_SUPPLICANT']
+
+		if self.args.hw:
+			nradios = len(settings.items())
+
+			wpas_radios = self.radios[:nradios]
+			self.wpas_interfaces = []
+
+			#
+			# Physical radios most likely will use a different name
+			# than 'rad#' but the config file is referenced by these
+			# 'rad#' names. Iterate through both the settings and
+			# physical radios to create interfaces associated with
+			# each config file.
+			#
+			for vrad, hwrad in zip(settings.items(), wpas_radios):
+				self.wpas_interfaces.append(hwrad.create_interface(vrad[1], 'wpas'))
+
+		else:
+			wpas_radios = [rad for rad in self.radios if rad.name in settings]
+			self.wpas_interfaces = [rad.create_interface(settings[rad.name], 'wpas') \
+						for rad in wpas_radios]
+
+	def start_ofono(self):
+		sim_keys = self.hw_config['SETUP'].get('sim_keys', None)
+		if not sim_keys:
+			print("Ofono not requred")
+			return
+		elif sim_keys != 'ofono':
+			os.environ['IWD_SIM_KEYS'] = sim_keys
+			return
+
+		if not find_binary(['ofonod']) or not find_binary(['phonesim']):
+			print("Ofono or Phonesim not found, skipping test")
+			return
+
+		Process(['ip', 'link', 'set', 'lo', 'up']).wait()
+
+		os.environ['OFONO_PHONESIM_CONFIG'] = '/tmp/phonesim.conf'
+
+		phonesim_args = ['phonesim', '-p', '12345', '/usr/share/phonesim/default.xml']
+
+		self.start_process(phonesim_args)
+
+		#
+		# TODO:
+		# Is there something to wait for? Without this phonesim rejects
+		# connections on all but the fist test.
+		#
+		time.sleep(3)
+
+		ofono_args = ['ofonod', '-n', '--plugin=atmodem,phonesim']
+		if self.is_verbose('ofonod'):
+			ofono_args.append('-d')
+
+		self.start_process(ofono_args)
+
+		print("Ofono started")
+
+	def create_namespaces(self):
+		if not self.hw_config.has_section('NameSpaces'):
+			return
+
+		for key, value in self.hw_config.items('NameSpaces'):
+			radio_names = value.split(',')
+			# Gather up radio objects for this namespace
+			radios = [rad for rad in self.radios if rad.name in radio_names]
+
+			# Remove radios from 'root' namespace
+			self.radios = list(set(self.radios) - set(radios))
+
+			self.namespaces.append(Namespace(self.args, key, radios))
+
+	def get_namespace(self, ns):
+		for n in self.namespaces:
+			if n.name == ns:
+				return n
+
+		return None
+
+	def stop_test_processes(self):
+		for n in self.namespaces:
+			n.reset()
+
+		self.namespaces = []
+		self.hostapd = None
+		self.wpas_interfaces = None
+
+		self.reset()
+
+	def meminfo_to_dict(self):
+		def removesuffix(string, suffix):
+			if string.endswith(suffix):
+				return string[:-len(suffix)]
+			return string
+
+		ret = {}
+
+		with open('/proc/meminfo', 'r') as f:
+			data = f.read().strip().split('\n')
+
+		for l in data:
+			entry = l.split(':')
+			ret[entry[0]] = int(removesuffix(entry[1], 'kB'))
+
+		return ret
+
+	def __str__(self):
+		ret = 'Arguments:\n'
+		for arg in vars(self.args):
+			ret += '\t --%s %s\n' % (arg, str(getattr(self.args, arg)))
+
+		ret += 'Hostapd:\n'
+		if self.hostapd:
+			for h in self.hostapd.instances:
+				ret += '\t%s\n' % str(h)
+		else:
+			ret += '\tNo Hostapd instances\n'
+
+		info = self.meminfo_to_dict()
+		self._mem_chart.add_value(info['MemAvailable'])
+
+		ret += 'Available Memory: %u kB\n' % info['MemAvailable']
+		ret += 'Last Test Delta: %+d kB\n' % (info['MemAvailable'] - self._last_mem_available)
+		ret += 'Per-test Usage:\n'
+		ret += str(self._mem_chart)
+
+		self._last_mem_available = info['MemAvailable']
+
+		ret += super().__str__()
+
+		for n in self.namespaces:
+			ret += n.__str__()
+
+		return ret
+
+def build_unit_list(args):
+	'''
+		Build list of unit tests based on passed arguments. This first
+		checks for literal names provided in the arguments, then if
+		no matches were found, checks for a glob match.
+	'''
+	tests = []
+	test_root = args.testhome + '/unit'
+
+	for unit in args.unit_tests.split(','):
+		path = '%s/%s' % (test_root, unit)
+		if os.access(unit, os.X_OK):
+			tests.append(unit)
+		elif os.access(path, os.X_OK):
+			tests.append(path)
+		else:
+			# Full list or glob, first build up valid list of tests
+			matches = glob(path)
+			if matches == []:
+				raise Exception("Could not find test %s" % unit)
+
+			matches = [exe for exe in matches if os.access(exe, os.X_OK)]
+
+			tests.extend(matches)
+
+	return sorted(tests)
+
+def build_test_list(args):
+	'''
+		Build list of auto test directories based on passed arguments.
+		First check for absolute paths, then look in <iwd>/autotests,
+		then glob match.
+	'''
+	tests = []
+	test_root = args.testhome + '/autotests'
+
+	# Run all tests
+	if not args.autotests:
+		# Get list of all autotests (committed in git)
+		tests = os.popen('git -C %s ls-files autotests/ | cut -f2 -d"/" \
+					| grep "test*" | uniq' % args.testhome).read() \
+					.strip().split('\n')
+		tests = [test_root + '/' + t for t in tests]
+	else:
+		print("Generating partial test list")
+
+		full_list = sorted(os.listdir(test_root))
+
+		for t in args.autotests.split(','):
+			path = '%s/%s' % (test_root, t)
+			if t.endswith('+'):
+				t = t.split('+')[0]
+				i = full_list.index(t)
+
+				tests = [test_root + '/' + x for x in full_list[i:] \
+							if x.startswith('test')]
+			elif os.path.exists(t):
+				if t not in tests:
+					tests.append(t)
+			elif os.path.exists(path):
+				if path not in tests:
+					tests.append(path)
+			else:
+				matches = glob(path)
+				if matches == []:
+					raise Exception("Could not find test %s" % t)
+
+				tests.extend(list(set(matches) - set(tests)))
+
+	return sorted(tests)
+
+SimpleResult = namedtuple('SimpleResult', 'run failures errors skipped time')
+
+def start_test(ctx, subtests, rqueue):
+	'''
+		Run an individual test. 'subtests' are parsed prior to calling
+		but these effectively make up a single test. 'rqueue' is the
+		results queue which is required since this is using
+		multiprocessing.
+	'''
+	run = 0
+	errors = 0
+	failures = 0
+	skipped = 0
+
+	start = time.time()
+	#
+	# Iterate through each individual python test.
+	#
+	for s in subtests:
+		loader = unittest.TestLoader()
+		try:
+			module = importlib.import_module(os.path.splitext(s)[0])
+		except OSError as e:
+			dbg(subprocess.check_output("cat /proc/buddyinfo", shell=True).decode('utf-8'))
+			dbg(subprocess.check_output("dmesg | tail -80", shell=True).decode('utf-8'))
+			print(ctx)
+			raise e
+
+		subtest = loader.loadTestsFromModule(module)
+
+		# The test suite is being (ab)used to get a bit more granularity
+		# with individual tests. The 'normal' way to use unittest is to
+		# just create a test suite and run them. The problem here is that
+		# test results are queued and printed at the very end so its
+		# difficult to know *where* a test failed (python gives a stack
+		# trace but printing the exception/failure immediately shows
+		# where in the debug logs something failed). Moreso if there are
+		# several test functions inside a single python file they run
+		# as a single test and it is difficult (again) to know where
+		# something failed.
+
+		# Iterating through each python test file
+		for test in subtest:
+			limit_funcs = []
+
+			if ctx.args.sub_tests:
+				for i in ctx.args.sub_tests:
+					if len(i.split('.')) == 2:
+						limit_funcs.append(i.split('.')[1])
+
+			# Iterating through individual test functions inside a
+			# Test() class. Due to the nature of unittest we have
+			# to jump through some hoops to set up the test class
+			# only once by turning the enumeration into a list, then
+			# enumerating (again) to keep track of the index (just
+			# enumerating the test class doesn't allow len() because
+			# it is not a list).
+			tlist = list(enumerate(test))
+			for index, t in enumerate(tlist):
+				# enumerate is returning a tuple, index 1 is our
+				# actual object.
+				t = t[1]
+
+				func, file = str(t).split(' ')
+				#
+				# TODO: There may be a better way of doing this
+				# but strigifying the test class gives us a string:
+				# <function> (<file>.<class>)
+				#
+				file = file.strip('()').split('.')[0] + '.py'
+
+				# Create an empty result here in case the test fails
+				result = TestResult()
+
+				try:
+					skip = len(limit_funcs) > 0 and func not in limit_funcs
+
+					# Set up class only on first test
+					if index == 0:
+						if not skip:
+							dbg("%s\n\t%s RUNNING" % (file, str(func)), end='')
+						t.setUpClass()
+					else:
+						if not skip:
+							dbg("\t%s RUNNING" % str(func), end='')
+
+					sys.__stdout__.flush()
+
+					Process.write_separators("\n====== %s:%s ======\n\n" % (file, func))
+
+					if not skip:
+						# Run test (setUp/tearDown run automatically)
+						result = t()
+
+					# Tear down class only on last test
+					if index == len(tlist) - 1:
+						t.tearDownClass()
+
+					if skip:
+						continue
+				except unittest.SkipTest as e:
+					result.skipped.append(t)
+				except Exception as e:
+					dbg('\n%s threw an uncaught exception:' % func)
+					traceback.print_exc(file=sys.__stdout__)
+
+				run += result.testsRun
+				errors += len(result.errors)
+				failures += len(result.failures)
+				skipped += len(result.skipped)
+
+				if len(result.skipped) > 0:
+					dbg(colored(" SKIPPED", "cyan"))
+				elif run == 0 or len(result.errors) > 0 or len(result.failures) > 0:
+					dbg(colored(" FAILED", "red"))
+					for e in result.errors:
+						dbg(e[1])
+					for f in result.failures:
+						dbg(f[1])
+				else:
+					dbg(colored(" PASSED", "green"))
+
+		# Prevents future test modules with the same name (e.g.
+		# connection_test.py) from being loaded from the cache
+		sys.modules.pop(module.__name__)
+
+	#
+	# The multiprocessing queue is picky with what objects it will serialize
+	# and send between processes. Because of this we put the important bits
+	# of the result into our own 'SimpleResult' tuple.
+	#
+	sresult = SimpleResult(run=run, failures=failures, errors=errors,
+				skipped=skipped, time=time.time() - start)
+	rqueue.put(sresult)
+
+	# This may not be required since we are manually popping sys.modules
+	importlib.invalidate_caches()
+
+def pre_test(ctx, test, copied):
+	'''
+		Copy test files, start processes, and any other pre test work.
+	'''
+	os.chdir(test)
+
+	dbg("\nStarting %s" % colored(os.path.basename(test), "white", attrs=['bold']))
+	if not os.path.exists(test + '/hw.conf'):
+		raise Exception("No hw.conf found for %s" % test)
+
+	ctx.hw_config = ConfigParser()
+	ctx.hw_config.read(test + '/hw.conf')
+	#
+	# We have two types of test files: tests and everything else. Rather
+	# than require each test to specify the files needing to be copied to
+	# /tmp (previously 'tmpfs_extra_stuff'), we just copy everything which
+	# isn't a test. There is really no reason not to do this as any file
+	# present in a test directory should be needed by the test.
+	#
+	# All files
+	files = os.listdir(test)
+	# Tests (starts or ends with 'test')
+	subtests = [f for f in files if f.startswith('test') or \
+			os.path.splitext(f)[0].endswith('test')]
+	# Everything else (except .py files)
+	to_copy = [f for f in list(set(files) - set(subtests)) if not f.endswith('.py') \
+								and f != '__pycache__']
+	for f in to_copy:
+		if os.path.isdir(f):
+			shutil.copytree(f, '/tmp/' + f)
+		else:
+			shutil.copy(f, '/tmp')
+		copied.append(f)
+
+	# Prune down any subtests if needed
+	if ctx.args.sub_tests:
+		ctx.args.sub_tests = ctx.args.sub_tests.split(',')
+
+		to_run = [x.split('.')[0] for x in ctx.args.sub_tests]
+		pruned = []
+
+		for s in subtests:
+			no_ext = s
+			# Handle <file>.<test function> format
+			if '.' in s:
+				no_ext = s.split('.')[0]
+
+			if no_ext in to_run:
+				pruned.append(no_ext + '.py')
+
+		subtests = pruned
+
+	if ctx.args.log:
+		ctx.start_process(['iwmon', '--nowiphy'])
+	elif ctx.args.monitor:
+		ctx.start_process(['iwmon'], outfile=ctx.args.monitor)
+
+	ctx.start_dbus()
+	ctx.start_haveged()
+	ctx.start_dbus_monitor()
+	ctx.start_radios()
+	ctx.create_namespaces()
+	ctx.start_hostapd()
+	ctx.start_wpas_interfaces()
+	ctx.start_ofono()
+
+	if ctx.hw_config.has_option('SETUP', 'start_iwd'):
+		start = ctx.hw_config.getboolean('SETUP', 'start_iwd')
+	else:
+		start = True
+
+	if start:
+		ctx.start_iwd()
+	else:
+		print("Not starting IWD from test-runner")
+
+	print(ctx)
+
+	sys.path.insert(1, test)
+
+	return sorted(subtests)
+
+def post_test(ctx, to_copy):
+	'''
+		Remove copied files, and stop test processes.
+	'''
+	try:
+		for f in to_copy:
+			if os.path.isdir('/tmp/' + f):
+				shutil.rmtree('/tmp/' + f)
+			else:
+				os.remove('/tmp/' + f)
+
+		Process(['ip', 'link', 'set', 'lo', 'down']).wait()
+	except Exception as e:
+		print("Exception thrown in post_test")
+	finally:
+		ctx.stop_test_processes()
+
+	if ctx.args.valgrind:
+		for f in os.listdir('/tmp'):
+			if f.startswith("valgrind.log."):
+				dbg(f)
+				with open('/tmp/' + f, 'r') as v:
+					dbg(v.read())
+				dbg("\n")
+				os.remove('/tmp/' + f)
+
+	# Special case for when logging is enabled
+	if os.path.isfile('/tmp/iwd-tls-debug-server-cert.pem'):
+		os.remove('/tmp/iwd-tls-debug-server-cert.pem')
+
+	allowed = ['phonesim.conf', 'certs', 'secrets', 'iwd']
+	for f in [f for f in os.listdir('/tmp') if f not in allowed]:
+		dbg("File %s was not cleaned up!" % f)
+		try:
+			os.remove('/tmp/' + f)
+		except:
+			pass
+
+def print_results(results):
+	table = PrettyTable(['Test', colored('Passed', 'green'), colored('Failed', 'red'), \
+				colored('Skipped', 'cyan'), colored('Time', 'yellow')])
+
+	total_pass = 0
+	total_fail = 0
+	total_skip = 0
+	total_time = 0
+
+	for test, result in results.items():
+
+		if result.time == TEST_MAX_TIMEOUT:
+			failed = "Timed out"
+			passed = "Timed out"
+		elif result.time == 0:
+			failed = "Exception"
+			passed = "Exception"
+		else:
+			failed = result.failures + result.errors
+			passed = result.run - failed
+
+			total_pass += passed
+			total_fail += failed
+			total_skip += result.skipped
+
+		total_time += result.time
+
+		time = '%.2f' % result.time
+
+		table.add_row([test, colored(passed, 'green'), colored(failed, 'red'), \
+				colored(result.skipped, 'cyan'), colored(time, 'yellow')])
+
+	total_time = '%.2f' % total_time
+
+	table.add_row(['Total', colored(total_pass, 'green'), colored(total_fail, 'red'), \
+			colored(total_skip, 'cyan'), colored(total_time, 'yellow')])
+
+	dbg(table)
+
+	return total_fail == 0
+
+def run_auto_tests(ctx, args):
+	tests = build_test_list(args)
+
+	for test in tests:
+		copied = []
+		try:
+			subtests = pre_test(ctx, test, copied)
+
+			if len(subtests) < 1:
+				dbg("No tests to run")
+				sys.exit()
+
+			rqueue = multiprocessing.Queue()
+			p = multiprocessing.Process(target=start_test, args=(ctx, subtests, rqueue))
+			p.start()
+			# Rather than time each subtest we just time the total but
+			# mutiply the default time by the number of tests being run.
+			p.join(TEST_MAX_TIMEOUT * len(subtests))
+
+			if p.is_alive():
+				# Timeout
+				p.terminate()
+
+				ctx.results[os.path.basename(test)] = SimpleResult(run=0,
+								failures=0, errors=0,
+								skipped=0, time=TEST_MAX_TIMEOUT)
+			else:
+				ctx.results[os.path.basename(test)] = rqueue.get()
+
+		except Exception as ex:
+			dbg("%s threw an uncaught exception" % test)
+			traceback.print_exc(file=sys.__stdout__)
+			ctx.results[os.path.basename(test)] = SimpleResult(run=0, failures=0,
+								errors=0, skipped=0, time=0)
+		finally:
+			post_test(ctx, copied)
+
+def run_unit_tests(ctx, args):
+	os.chdir(args.testhome + '/unit')
+	units = build_unit_list(args)
+
+	for u in units:
+		p = ctx.start_process([u]).wait()
+		if p.returncode != 0:
+			dbg("Unit test %s failed" % os.path.basename(u))
+		else:
+			dbg("Unit test %s passed" % os.path.basename(u))
+
+def run_tests(args):
+	global config
+
+	os.chdir(args.testhome)
+
+	#
+	# This allows all autotest utils (iwd/hostapd/etc) to access the
+	# TestContext. Any other module or script (in the same interpreter) can
+	# simply import config.ctx and access all live test information,
+	# start/stop processes, see active radios etc.
+	#
+	config = importlib.import_module('config')
+	config.ctx = TestContext(args)
+
+	# Must import these after config so ctx gets set
+	config.hwsim = importlib.import_module('hwsim')
+	config.hostapd = importlib.import_module('hostapd')
+
+	# Start writing out kernel log
+	config.ctx.start_process(["dmesg", '--follow'])
+
+	if args.unit_tests is None:
+		run_auto_tests(config.ctx, args)
+	else:
+		run_unit_tests(config.ctx, args)
+
+runner = Runner()
+
+atexit.register(exit_vm)
+runner.prepare_environment()
+run_tests(runner.args)
+runner.cleanup_environment()
+
tools/test-runner +1 -1524

1,474 lines not on this page · raw message

The rest of this diff is not shown. View the raw message.

Signature · 1 line
2.34.1

Part of the diff is not shown here. View the raw message.

author
#4 lore raw

PATCH v2 4/14 test-runner: add initial UmlRunner implementation

In reply to James Prestwood

This allows test-runner to run inside a UML binary which has some advantages, specifically time-travel/infinite CPU speed. This should fix any scheduler related failures we have on slower systems.

Currently this runner does not suppor the same features as the Qemu runner, specifically:

- No hardware passthrough
- No logging/monitor (UML -> host mounting isn't implemented yet)

---
 tools/runner.py | 30 +++++++++++++++++++++++++++++-
 1 file changed, 29 insertions(+), 1 deletion(-)
tools/runner.py +29 -1
index 743d74b1..33c10917 100644
@@ -213,7 +213,9 @@ class Runner:
 			else:
 				args.runner = 'qemu'
 
-		if args.runner == 'qemu':
+		if args.runner == 'uml':
+			return UmlRunner(args)
+		elif args.runner == 'qemu':
 			return QemuRunner(args)
 		else:
 			raise Exception("Unknown runner %s" % args.runner)
@@ -483,3 +485,29 @@ class QemuRunner(RunnerAbstract):
 		# gracefully.
 		#
 		libc.reboot(RB_AUTOBOOT)
+
+class UmlRunner(RunnerAbstract):
+	name = "UML Runner"
+
+	def __init__(self, args):
+		super().__init__(args)
+
+		if len(sys.argv) <= 1:
+			return
+
+		if not which(args.kernel):
+			raise Exception('Cannot locate UML binary %s' % args.kernel)
+
+		kern_log = "ignore_loglevel" if "kernel" in args.verbose else "quiet"
+
+		cmd = [args.kernel, 'rootfstype=hostfs', 'ro', 'mem=256M', 'mac80211_hwsim.radios=0',
+				'time-travel=inf-cpu', 'eth0=mcast', 'eth1=mcast',
+				'%s '% kern_log, 'init=%s' % args.start]
+		cmd.extend(args.to_cmd().split(' '))
+
+		self.cmdline = cmd
+
+	def prepare_environment(self):
+		self._prepare_mounts()
+
+		super().prepare_environment()
Signature · 1 line
2.34.1
author
#5 lore raw

PATCH v2 5/14 test-runner: fix logging for namespaces and pre-test processes

In reply to James Prestwood

Namespace process logs were appearing under 'ip' (and also overwriting actual 'ip' logs) since they were executed with 'ip netns exec <namespace>'. Instead special case this and append '-<namespace>' to the log file name.

In addition processes executed prior to any tests were being put under a folder (name of testhome directory). Now this case is detected and these logs are put at the top level log directory.

---
 tools/run-tests | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)
tools/run-tests +13 -3
index 716126d0..94b71ea4 100755
@@ -129,6 +129,8 @@ class Process(subprocess.Popen):
 		self.killed = False
 		self.namespace = namespace
 
+		logfile = args[0]
+
 		if not self.ctx:
 			global config
 			self.ctx = config.ctx
@@ -138,6 +140,7 @@ class Process(subprocess.Popen):
 
 		if namespace:
 			args = ['ip', 'netns', 'exec', namespace] + args
+			logfile += '-%s' % namespace
 
 		if outfile:
 			# outfile is only used by iwmon, in which case we don't want
@@ -145,9 +148,16 @@ class Process(subprocess.Popen):
 			self._append_outfile(outfile, append=False)
 
 		if self.ctx.args.log:
-			logfile = '%s/%s/%s' % (self.ctx.args.log,
-						os.path.basename(os.getcwd()),
-						args[0])
+			testdir = os.getcwd()
+
+			# Special case any processes started prior to a test
+			# (i.e. from testhome). Put these in the root log directory
+			if testdir == self.ctx.args.testhome:
+				testdir = '.'
+			else:
+				testdir = os.path.basename(testdir)
+
+			logfile = '%s/%s/%s' % (self.ctx.args.log, testdir, logfile)
 			self._append_outfile(logfile)
 
 		super().__init__(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
Signature · 1 line
2.34.1
author
#6 lore raw

PATCH v2 6/14 test-runner: use may_block=True for context iteration (and move location)

In reply to James Prestwood

This allows the callers condition to be checked immediately without the mainloop running. In addition may_block=True allows the mainloop to poll/sleep rather than immediately return back to the caller. This handles async IO much better than may_block=False, at least for our use-case.

---
 tools/run-tests | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)
tools/run-tests +2 -4
index 94b71ea4..e2993b1b 100755
@@ -722,8 +722,6 @@ class Namespace:
 		context = mainloop.get_context()
 
 		while True:
-			context.iteration(may_block=False)
-
 			try:
 				ret = func(*args)
 				if ret:
@@ -735,8 +733,6 @@ class Namespace:
 					GLib.source_remove(timeout)
 				raise e
 
-			sleep(0.1)
-
 			if done.value == True:
 				if isinstance(exception, Exception):
 					raise exception
@@ -745,6 +741,8 @@ class Namespace:
 				else:
 					return
 
+			context.iteration(may_block=True)
+
 	def __str__(self):
 		ret = 'Namespace: %s\n' % self.name
 		ret += 'Processes:\n'
Signature · 1 line
2.34.1
author
#7 lore raw

PATCH v2 7/14 test-runner: don't kill dmesg after individual tests

In reply to James Prestwood

This prevents any kernel logging from being available after the first test is finished.

---
 tools/run-tests | 3 +++
 1 file changed, 3 insertions(+)
tools/run-tests +3 -0
index e2993b1b..c4d133a6 100755
@@ -186,6 +186,9 @@ class Process(subprocess.Popen):
 	@classmethod
 	def kill_all(cls):
 		for p in cls.processes.values():
+			if p.args[0] == 'dmesg':
+				continue
+
 			p.kill()
 
 	@staticmethod
Signature · 1 line
2.34.1
author
#8 lore raw

PATCH v2 8/14 test-runner: add time-travel to kernel config

In reply to James Prestwood

This lets UML work with time-travel[=inf-cpu] options.

---
 tools/test_runner_kernel_config | 1 +
 1 file changed, 1 insertion(+)
tools/test_runner_kernel_config +1 -0
index f29c1781..7806fdb2 100755
@@ -50,3 +50,4 @@ scripts/config --enable CONFIG_SECURITYFS
 # Options needed for UML
 scripts/config --enable CONFIG_BINFMT_ELF
 scripts/config --enable CONFIG_HOSTFS
+scripts/config --enable CONFIG_UML_TIME_TRAVEL_SUPPORT
Signature · 1 line
2.34.1
author
#9 lore raw

PATCH v2 9/14 auto-t: fix testBSSBlacklist typo

In reply to James Prestwood

The property is 'enabled' not 'enable'

---
 autotests/testBSSBlacklist/connection_test.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
autotests/testBSSBlacklist/connection_test.py +3 -3
index 8e239f44..9631a322 100644
@@ -110,11 +110,11 @@ class Test(unittest.TestCase):
 
         # Have both APs drop all packets, both should get blacklisted
         rule0.drop = True
-        rule0.enable = True
+        rule0.enabled = True
         rule1.drop = True
-        rule1.enable = True
+        rule1.enabled = True
         rule2.drop = True
-        rule2.enable = True
+        rule2.enabled = True
 
         with self.assertRaises(iwd.FailedEx):
             ordered_network.network_object.connect()
Signature · 1 line
2.34.1
author
#10 lore raw

PATCH v2 10/14 auto-t: wait for roam events in proper order

In reply to James Prestwood

These were out of order and would sometimes fail the check.

---
 autotests/testPSK-roam/roam_ap_disconnect_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
autotests/testPSK-roam/roam_ap_disconnect_test.py +1 -1
index 9cef3090..760c25fa 100644
@@ -32,8 +32,8 @@ class Test(unittest.TestCase):
 
         # Since both BSS's have low signal, the roam should fail and trigger
         # another roam scan.
-        device.wait_for_event('no-roam-candidates', timeout=30)
         device.wait_for_event('roam-scan-triggered', timeout=30)
+        device.wait_for_event('no-roam-candidates', timeout=30)
 
         # Hostapd sends disconnect
         self.bss_hostapd[0].disable()
Signature · 1 line
2.34.1
author
#11 lore raw

PATCH v2 11/14 auto-t: cleanup testRRM

In reply to James Prestwood

This removes prints which were never supposed to make it upstream as well as changes sleep() to wd.wait() as well as increase the wait period to fix issues with how fast UML runs the tests.

---
 autotests/testRRM/connection_test.py | 32 ++++++++++------------------
 1 file changed, 11 insertions(+), 21 deletions(-)
autotests/testRRM/connection_test.py +11 -21
index 278c69ce..4d894958 100644
@@ -11,8 +11,6 @@ from iwd import NetworkType
 from hostapd import HostapdCLI
 import testutil
 
-from time import sleep
-
 # Table beacon with wildcard BSSID
 basic_beacon = '51000000000002ffffffffffff020100'
 # Table beacon with wildcard BSSID and SSID filter
@@ -53,51 +51,43 @@ class Test(unittest.TestCase):
 
         hapd.wait_for_event('AP-STA-CONNECTED')
 
+        wd.wait(1)
+
         # This should return both APs
         hapd.req_beacon(device.address, basic_beacon)
 
         for e in ['BEACON-RESP-RX', 'BEACON-RESP-RX']:
-            event = hapd.wait_for_event(e)
-            if event:
-                print(event)
+            hapd.wait_for_event(e)
 
-        sleep(0.5)
+        wd.wait(1)
 
         # This should return just ssidRRM
         hapd.req_beacon(device.address, beacon_with_ssid)
-        event = hapd.wait_for_event('BEACON-RESP-RX')
-        if event:
-            print(event)
+        hapd.wait_for_event('BEACON-RESP-RX')
 
-        sleep(0.5)
+        wd.wait(1)
 
         # This should passive scan on channel 11, returning otherSSID
         hapd.req_beacon(device.address, beacon_passive)
         # TODO: See if we are scanning here (scan not initiated from station)
 
-        event = hapd.wait_for_event('BEACON-RESP-RX')
-        if event:
-            print(event)
+        hapd.wait_for_event('BEACON-RESP-RX')
 
-        sleep(0.5)
+        wd.wait(1)
 
         # This should active scan on channel 11, returning otherSSID
         hapd.req_beacon(device.address, beacon_active)
         # TODO: See if we are scanning here (scan not initiated from station)
 
-        event = hapd.wait_for_event('BEACON-RESP-RX')
-        if event:
-            print(event)
+        hapd.wait_for_event('BEACON-RESP-RX')
 
-        sleep(0.5)
+        wd.wait(1)
 
          # This should passive scan on channel 11, returning otherSSID
         hapd.req_beacon(device.address, beacon_passive_duration)
         # TODO: See if we are scanning here (scan not initiated from station)
 
-        event = hapd.wait_for_event('BEACON-RESP-RX')
-        if event:
-            print(event)
+        hapd.wait_for_event('BEACON-RESP-RX')
 
         device.disconnect()
 
Signature · 1 line
2.34.1
author
#12 lore raw

PATCH v2 12/14 auto-t: hostapd.py: use IO watch for hostapd events

In reply to James Prestwood

With how fast UML is hostapd events were being sent out prior to ever calling wait_for_event. Instead set an IO watch on the control socket and cache all events as they come. Then, when wait_for_event is called, it can reference this list. If the event is found any older events are purged from the list.

The AP-ENABLED event needed a special case because hostapd gets started before the IO watch can be registered. To fix this an enabled property was added which queries the state directly. This is checked first, and if not enabled wait_for_event continues normally.

---
 autotests/util/hostapd.py | 68 ++++++++++++++++++++++++++-------------
 tools/run-tests           |  3 +-
 2 files changed, 48 insertions(+), 23 deletions(-)
autotests/util/hostapd.py +46 -22
index 4ae7c42d..6a9f37c7 100644
@@ -81,27 +81,48 @@ class HostapdCLI(object):
 
         self.ctrl_sock.connect(self.socket_path + '/' + self.ifname)
 
+        self.events = []
+        self.io_watch = GLib.io_add_watch(self.ctrl_sock, GLib.IO_IN, self._handle_data_in)
+
         if 'OK' not in self._ctrl_request('ATTACH'):
             raise Exception('ATTACH failed')
 
         ctrl_count = ctrl_count + 1
 
-    def _poll_event(self, event):
-        if not self._data_available(0.25):
-            return False
+    def _handle_data_in(self, sock, *args):
+        newdata = sock.recv(4096)
+
+        decoded = newdata.decode('utf-8')
+        if len(decoded) >= 3 and decoded[0] == '<' and decoded[2] == '>':
+            decoded = decoded[3:]
+        while len(decoded) and decoded[-1] == '\n':
+            decoded = decoded[:-1]
+
+        self.events.insert(0, decoded)
+
+        return True
 
-        data = self.ctrl_sock.recv(4096).decode('utf-8')
-        if event in data:
-            return data
+    def _poll_event(self, event):
+        # Look through the list (most recent is first) until the even is found.
+        # Once found consume this event and any older ones as to not
+        # accidentally trigger a false positive later on.
+        for idx, e in enumerate(self.events):
+            if event in e:
+                self.events = self.events[:idx]
+                return e
 
         return False
 
     def wait_for_event(self, event, timeout=10):
+        if event == 'AP-ENABLED':
+            if self.enabled:
+                return 'AP-ENABLED'
+
         return ctx.non_block_wait(self._poll_event, timeout, event,
                                     exception=TimeoutError("waiting for event"))
 
-    def _data_available(self, timeout=2):
-        [r, w, e] = select.select([self.ctrl_sock], [], [], timeout)
+    def _data_available(self):
+        [r, w, e] = select.select([self.ctrl_sock], [], [])
         if r:
             return True
         return False
@@ -228,25 +249,28 @@ class HostapdCLI(object):
         ctx.start_process(cmd).wait()
         self.wait_for_event('AP-CSA-FINISHED')
 
-    @property
-    def bssid(self):
+    def _get_status(self):
+        ret = {}
+
         cmd = self.cmdline + ['status']
         proc = ctx.start_process(cmd)
         proc.wait()
-        status = proc.out.split('\n')
+        status = proc.out.strip().split('\n')
 
-        bssid = [x for x in status if x.startswith('bssid')]
-        bssid = bssid[0].split('=')
-        return bssid[1]
+        for kv in status:
+            k, v = kv.split('=')
+            ret[k] = v
+
+        return ret
 
     @property
-    def frequency(self):
-        cmd = self.cmdline + ['status']
-        proc = ctx.start_process(cmd)
-        proc.wait()
-        status = proc.out.split('\n')
+    def bssid(self):
+        return self._get_status()['bssid[0]']
 
-        frequency = [x for x in status if x.startswith('freq')][0]
-        frequency = frequency.split('=')[1]
+    @property
+    def frequency(self):
+        return int(self._get_status()['freq'])
 
-        return int(frequency)
+    @property
+    def enabled(self):
+        return self._get_status()['state'] == 'ENABLED'
tools/run-tests +2 -1
index c4d133a6..2b9290ed 100755
@@ -540,7 +540,8 @@ class Hostapd:
 		except:
 			print("Failed to remove %s" % self.global_ctrl_iface)
 
-		self.instances = None
+		for hapd in self.instances:
+			GLib.source_remove(hapd.cli.io_watch)
 
 		# Hostapd may have already been stopped
 		if self.process:
Signature · 1 line
2.34.1
author
#13 lore raw

PATCH v2 13/14 auto-t: iwd.py: use IO watch for station debug events

In reply to James Prestwood

Similarly to hostapd.wait_for_event, IWD's variant needed to act on an IO watch because events were being received prior to even calling wait_for_event.

---
 autotests/util/iwd.py | 27 ++++++++++++---------------
 1 file changed, 12 insertions(+), 15 deletions(-)
autotests/util/iwd.py +12 -15
index 253fe25e..a2062258 100755
@@ -232,16 +232,14 @@ class StationDebug(IWDDBusAbstract):
     _iface_name = IWD_STATION_DEBUG_INTERFACE
 
     def __init__(self, *args, **kwargs):
-        self._last_event = None
-        self._last_event_data = []
+        self._events = []
 
         IWDDBusAbstract.__init__(self, *args, **kwargs)
 
         self._iface.connect_to_signal("Event", self._event_handler)
 
     def _event_handler(self, event, data):
-        self._last_event = event
-        self._last_event_data = data
+        self._events.insert(0, (event, data))
 
     @property
     def autoconnect(self):
@@ -257,19 +255,18 @@ class StationDebug(IWDDBusAbstract):
         frequencies = dbus.Array([dbus.UInt16(f) for f in frequencies])
         self._iface.Scan(frequencies)
 
-    def wait_for_event(self, event, timeout=10):
-        try:
-            if self._last_event is not None:
-                return self._last_event_data
+    def _poll_event(self, event):
+        for idx, e in enumerate(self._events):
+            if event == e[0]:
+                # Consume any older events
+                self._events = self._events[:idx]
+                return True
 
-            ctx.non_block_wait(lambda s, e : s._last_event == e, timeout, self, event,
-                                exception=TimeoutError("waiting for StationDebug.Event"))
+        return False
 
-            return self._last_event_data
-        finally:
-            # Consume the event
-            self._last_event_data = None
-            self._last_event = None
+    def wait_for_event(self, event, timeout=10):
+        return ctx.non_block_wait(self._poll_event, timeout, event,
+                                    exception=TimeoutError("waiting for event"))
 
 class DeviceProvisioning(IWDDBusAbstract):
     '''
Signature · 1 line
2.34.1
author
#14 lore raw

PATCH v2 14/14 rrm: use ULL for static values

In reply to James Prestwood
---
 src/rrm.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
src/rrm.c +2 -2
index d1c7029d..d32ffbff 100644
@@ -46,9 +46,9 @@
 #include "linux/nl80211.h"
 
 /* Limit requests per second */
-#define MAX_REQUESTS_PER_SEC		2
+#define MAX_REQUESTS_PER_SEC		2ULL
 /* Microseconds between requests */
-#define MIN_MICROS_BETWEEN_REQUESTS	(1000000 / MAX_REQUESTS_PER_SEC)
+#define MIN_MICROS_BETWEEN_REQUESTS	(1000000ULL / MAX_REQUESTS_PER_SEC)
 
 /* 802.11-2016 Table 9-90 */
 #define REPORT_DETAIL_NO_FIELDS_OR_ELEMS		0
Signature · 1 line
2.34.1