auto-t: make test timeout configurable
PATCH 1/3 auto-t: make test timeout configurable
With the addition of DPP PKEX autotests some of the timeouts are quite long and hit test-runners maximum timeouts. For UML we should allow this since time-travel lets us skip idle waits. Move the test timeout out of a global define and into the argument list so QEMU and UML can define it differently.
--- tools/run-tests | 12 +++++------- tools/runner.py | 7 +++++++ 2 files changed, 12 insertions(+), 7 deletions(-)
tools/run-tests +5 -7
@@ -27,8 +27,6 @@ from utils import Process, Namespace, BarChart config = None intf_id = 0 -TEST_MAX_TIMEOUT = 240 - dbus.mainloop.glib.DBusGMainLoop(set_as_default=True) def dbg(*s, **kwargs):
@@ -52,7 +50,7 @@ def exit_vm(): p.kill() if config.ctx and config.ctx.results: - success = print_results(config.ctx.results) + success = print_results(config.ctx.results, int(config.ctx.args.timeout)) if config.ctx.args.result: write_results(config.ctx.args.result, config.ctx.results)
@@ -917,7 +915,7 @@ def post_test(ctx, to_copy): except: pass -def print_results(results): +def print_results(results, max_timeout): table = PrettyTable(['Test', colored('Passed', 'green'), colored('Failed', 'red'), \ colored('Skipped', 'cyan'), colored('Time', 'yellow')])
@@ -928,7 +926,7 @@ def print_results(results): for test, result in results.items(): - if result.time == TEST_MAX_TIMEOUT: + if result.time == max_timeout: failed = "Timed out" passed = "Timed out" elif result.time == 0:
@@ -975,7 +973,7 @@ def run_auto_tests(ctx, args): p.start() # Rather than time each subtest we just time the total but # mutiply the default time by the number of tests being run. - p.join(TEST_MAX_TIMEOUT * len(subtests)) + p.join(int(args.timeout) * len(subtests)) if p.is_alive(): # Timeout
@@ -983,7 +981,7 @@ def run_auto_tests(ctx, args): ctx.results[os.path.basename(test)] = SimpleResult(run=0, failures=0, errors=0, - skipped=0, time=TEST_MAX_TIMEOUT) + skipped=0, time=int(args.timeout)) else: ctx.results[os.path.basename(test)] = rqueue.get()
tools/runner.py +7 -0
@@ -120,9 +120,12 @@ class RunnerCoreArgParse(ArgumentParser): const=True, action='store', help='Use physical adapters for tests (passthrough)') + + # Hidden options only meant to be passed to the kernel self.add_argument('--testhome', help=SUPPRESS) self.add_argument('--monitor-parent', help=SUPPRESS) self.add_argument('--result-parent', help=SUPPRESS) + self.add_argument('--timeout', help=SUPPRESS) # Prevent --autotest/--unittest from being used together auto_unit_group = self.add_mutually_exclusive_group()
@@ -385,6 +388,8 @@ class QemuRunner(RunnerAbstract): self._prepare_outfiles() + self.args.timeout = 240 + if args.hw: if os.path.isfile(args.hw): hw_conf = ConfigParser()
@@ -543,6 +548,8 @@ class UmlRunner(RunnerAbstract): self._prepare_outfiles() + self.args.timeout = 1000 + kern_log = "ignore_loglevel" if "kernel" in args.verbose else "quiet" cmd = [args.kernel, 'rootfstype=hostfs', 'ro', 'mem=256M', 'mac80211_hwsim.radios=0',
Signature · 1 line
2.25.1
PATCH 2/3 auto-t: add stop APIs and fix some issues wpas.py
- wait_for_event was returning a list in certain cases, not the event itself
- The configurator ID was not being printed (',' instead of '%')
- The DPP ID was not being properly waited for with PKEX
--- autotests/util/wpas.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-)
autotests/util/wpas.py +12 -2
@@ -76,7 +76,7 @@ class Wpas: for e in self._rx_data: if event in e: - return self._rx_data + return e return False
@@ -281,7 +281,7 @@ class Wpas: self._dpp_conf_id = self.wait_for_result() if not uri: - print("DPP Configurator ID: %s", self._dpp_conf_id) + print("DPP Configurator ID: %s" % self._dpp_conf_id) return self._rx_data = []
@@ -315,6 +315,8 @@ class Wpas: self._rx_data = [] self._ctrl_request(cmd) self._dpp_id = self.wait_for_result() + while not self._dpp_id.isnumeric(): + self._dpp_id = self.wait_for_result() def dpp_pkex_add(self, code, identifier=None, version=None, initiator=False, role=None): cmd = f'DPP_PKEX_ADD own={self._dpp_id}'
@@ -336,10 +338,18 @@ class Wpas: self._rx_data = [] self._ctrl_request(cmd) + def dpp_pkex_remove(self): + self._rx_data = [] + self._ctrl_request("DPP_PKEX_REMOVE *") + def dpp_listen(self, freq): self._rx_data = [] self._ctrl_request(f'DPP_LISTEN {freq}') + def dpp_stop_listen(self): + self._rx_data = [] + self._ctrl_request("DPP_STOP_LISTEN") + def dpp_configurator_remove(self): self._ctrl_request('DPP_CONFIGURATOR_REMOVE *') self.wait_for_result()
Signature · 1 line
2.25.1
PATCH 3/3 auto-t: get DPP PKEX test running reliably
In reply to James PrestwoodSeveral tests do not pass due to some additional changes that have not been merged. Remove these cases and add some hardening after discovering some unfortunate wpa_supplicant behavior.
- Disable p2p in wpa_supplicant. With p2p enabled an extra device is created which starts receiving DPP frames and printing confusing messages.
- Remove extra asserts which don't make sense currently. These will be added back later as future additions to PKEX are upstreamed.
- Work around wpa_supplicant retransmit limitation. This is described in detail in the comment in pkex_test.py
--- autotests/testDPP/pkex_test.py | 56 ++++++++++++++++++++++++++++------ autotests/testDPP/wpas.conf | 1 + 2 files changed, 48 insertions(+), 9 deletions(-)
autotests/testDPP/pkex_test.py +47 -9
@@ -21,6 +21,11 @@ class Test(unittest.TestCase): self.wpas.dpp_configurator_create() self.wpas.dpp_listen(2437) + def stop_wpas_pkex(self): + self.wpas.dpp_pkex_remove() + self.wpas.dpp_stop_listen() + self.wpas.dpp_configurator_remove() + def start_iwd_pkex_configurator(self, device, agent=False): self.hapd.reload() self.hapd.wait_for_event('AP-ENABLED')
@@ -38,11 +43,51 @@ class Test(unittest.TestCase): else: device.dpp_pkex_configure_enrollee('secret123', identifier="test") + # + # WPA Supplicant has awful handling of retransmissions and no-ACK + # conditions. It only handles retransmitting the exchange request when + # there is no-ACK, which makes zero sense since its a broadcast... + # + # So, really, testing against wpa_supplicant is fragile and dependent on + # how the scheduling works out. If IWD doesn't ACK due to being on the + # next frequency or in between offchannel requests wpa_supplicant gets + # into a state where it thinks a PKEX session has been started (having + # received the exchange request) but will only accept commit-reveal + # frames. IWD is unaware because it never got a response so it keeps + # sending exchange requests which are ignored. + # + # Nevertheless we should still test against wpa_supplicant for + # compatibility so attempt to detect this case and restart the + # wpa_supplicant configurator. + # + def restart_wpas_if_needed(self): + i = 0 + + while i < 10: + data = self.wpas.wait_for_event("DPP-RX") + self.assertIn("type=7", data) + + data = self.wpas.wait_for_event("DPP-TX") + self.assertIn("type=8", data) + + data = self.wpas.wait_for_event("DPP-TX-STATUS") + if "result=no-ACK" in data: + self.stop_wpas_pkex() + self.start_wpas_pkex('secret123', identifier="test") + else: + return + + i += 1 + + raise Exception("wpa_supplicant could not complete PKEX after 10 retries") + def test_pkex_iwd_as_enrollee(self): self.start_wpas_pkex('secret123', identifier="test") self.device[0].dpp_pkex_enroll('secret123', identifier="test") + self.restart_wpas_if_needed() + self.wpas.wait_for_event("DPP-AUTH-SUCCESS") def test_pkex_iwd_as_enrollee_retransmit(self):
@@ -52,6 +97,8 @@ class Test(unittest.TestCase): self.device[0].dpp_pkex_enroll('secret123', identifier="test") + self.restart_wpas_if_needed() + self.wpas.wait_for_event("DPP-AUTH-SUCCESS") def test_pkex_unsupported_version(self):
@@ -121,15 +168,6 @@ class Test(unittest.TestCase): condition = 'obj.state == DeviceState.connected' self.wd.wait_for_object_condition(self.device[1], condition) - self.assertTrue(os.path.exists('/tmp/ns0/ssidCCMP.psk')) - - with open('/tmp/ns0/ssidCCMP.psk') as f: - data = f.read() - - self.assertIn("SendHostname", data) - self.assertIn("SharedCode=secret123", data) - self.assertIn("ExactConfig=true", data) - def test_pkex_configurator_with_agent(self): self.start_iwd_pkex_configurator(self.device[0], agent=True)
autotests/testDPP/wpas.conf +1 -0
@@ -3,3 +3,4 @@ ctrl_interface=/tmp/rad1-p2p-wpas update_config=0 pmf=2 dpp_config_processing=2 +p2p_disabled=1
Signature · 1 line
2.25.1
Hi James,
Quoted 10 lines · James Prestwood
With the addition of DPP PKEX autotests some of the timeouts are quite long and hit test-runners maximum timeouts. For UML we should allow this since time-travel lets us skip idle waits. Move the test timeout out of a global define and into the argument list so QEMU and UML can define it differently.
--- tools/run-tests | 12 +++++------- tools/runner.py | 7 +++++++ 2 files changed, 12 insertions(+), 7 deletions(-)
All applied, thanks.
Regards,
-Denis