OEQA framework add support of threaded mode and enable in SDK
cover OEQA framework add support of threaded mode and enable in SDK
This series enables support of OEQA framework threaded runs, it implements a new module oeqa.core.threaded with OETest{Context, Loader, Runner, StreamLogger, Result} versions supporting parallelized runs.
The {e,}SDK test component was enable to use threaded mode by default.
The following changes since commit 381897c64069ea43d595380a3ae913bcc79cf7e1:
build-appliance-image: Update to master head revision (2017-05-01 08:56:47 +0100)
are available in the git repository at:
git://git.yoctoproject.org/poky-contrib alimon/oeqa_threaded
http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=alimon/oeqa_threaded
Aníbal Limón (17):
testsdk.bbclass: Remove unused import of OEStreamLogger oeqa/sdk/context.py: Import argparse_oe at OESDKTestContext.run method oeqa/sdk/context.py: Add return to OESDKTestContext.run() method oeqa/core: Don't expose OEStreamLogger in OETestContext oeqa/core/runner: OETestResult remove unneeded override of startTest oeqa/core: Move OETestContext.log{Summary, Details} into OETestResult oeqa/core/threaded: Add new module with OETestLoaderThreaded oeqa/core/threaded: Add OEStreamLoggerThreaded class oeqa/core/runner: OETestResult add internal _tc_map_results oeqa/core/threaded: Add OETestResultThreaded{,Internal} classes oeqa/core/threaded: Add support of OETestRunnerThreaded oeqa/core/threaded: Add OETestContextThreaded class oeqa/core/decorator/depends: Add support for threading mode oeqa/core/decorator/oetimeout: Add support for OEQA threaded mode oeqa/core/tests: Add tests of OEQA Threaded mode oeqa/sdkext/cases: Move sdk_update test into devtool module oeqa/sdk: Enable usage of OEQA thread mode
meta/classes/testimage.bbclass | 4 +- meta/classes/testsdk.bbclass | 13 +- meta/lib/oeqa/core/context.py | 97 +------- meta/lib/oeqa/core/decorator/depends.py | 8 +- meta/lib/oeqa/core/decorator/oetimeout.py | 40 ++- meta/lib/oeqa/core/runner.py | 94 ++++++- .../core/tests/cases/loader/threaded/threaded.py | 12 + .../tests/cases/loader/threaded/threaded_alone.py | 8 + .../cases/loader/threaded/threaded_depends.py | 10 + .../tests/cases/loader/threaded/threaded_module.py | 12 + meta/lib/oeqa/core/tests/common.py | 10 + meta/lib/oeqa/core/tests/test_decorators.py | 12 + meta/lib/oeqa/core/tests/test_loader.py | 30 ++- meta/lib/oeqa/core/threaded.py | 274 +++++++++++++++++++++ meta/lib/oeqa/sdk/context.py | 11 +- meta/lib/oeqa/sdkext/cases/devtool.py | 32 +++ meta/lib/oeqa/sdkext/cases/sdk_update.py | 39 --- 17 files changed, 548 insertions(+), 158 deletions(-) create mode 100644 meta/lib/oeqa/core/tests/cases/loader/threaded/threaded.py create mode 100644 meta/lib/oeqa/core/tests/cases/loader/threaded/threaded_alone.py create mode 100644 meta/lib/oeqa/core/tests/cases/loader/threaded/threaded_depends.py create mode 100644 meta/lib/oeqa/core/tests/cases/loader/threaded/threaded_module.py create mode 100644 meta/lib/oeqa/core/threaded.py delete mode 100644 meta/lib/oeqa/sdkext/cases/sdk_update.py
Signature · 1 line
2.1.4
PATCH 1/17 testsdk.bbclass: Remove unused import of OEStreamLogger
--- meta/classes/testsdk.bbclass | 1 - 1 file changed, 1 deletion(-)
meta/classes/testsdk.bbclass +0 -1
@@ -21,7 +21,6 @@ def testsdk_main(d): import logging from bb.utils import export_proxies - from oeqa.core.runner import OEStreamLogger from oeqa.sdk.context import OESDKTestContext, OESDKTestContextExecutor from oeqa.utils import make_logger_bitbake_compatible
Signature · 1 line
2.1.4
PATCH 2/17 oeqa/sdk/context.py: Import argparse_oe at OESDKTestContext.run method
In reply to Aníbal LimónThis import was at level of OESDKTestContext.register_commands but OESDKTestContext.run method need it to raise exceptions.
--- meta/lib/oeqa/sdk/context.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
meta/lib/oeqa/sdk/context.py +2 -2
@@ -44,8 +44,6 @@ class OESDKTestContextExecutor(OETestContextExecutor): default_test_data = None def register_commands(self, logger, subparsers): - import argparse_oe - super(OESDKTestContextExecutor, self).register_commands(logger, subparsers) sdk_group = self.parser.add_argument_group('sdk options')
@@ -109,6 +107,8 @@ class OESDKTestContextExecutor(OETestContextExecutor): log(env) def run(self, logger, args): + import argparse_oe + if not args.sdk_dir: raise argparse_oe.ArgumentUsageError("No SDK directory "\ "specified please do, --sdk-dir SDK_DIR", self.name)
Signature · 1 line
2.1.4
PATCH 3/17 oeqa/sdk/context.py: Add return to OESDKTestContext.run() method
In reply to Aníbal LimónThe run() methods of a OETestContext's are expected to return the results.
--- meta/lib/oeqa/sdk/context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
meta/lib/oeqa/sdk/context.py +1 -1
@@ -128,6 +128,6 @@ class OESDKTestContextExecutor(OETestContextExecutor): "environment (%s) specified" % args.sdk_env, self.name) self.sdk_env = sdk_envs[args.sdk_env] - super(OESDKTestContextExecutor, self).run(logger, args) + return super(OESDKTestContextExecutor, self).run(logger, args) _executor_class = OESDKTestContextExecutor
Signature · 1 line
2.1.4
PATCH 4/17 oeqa/core: Don't expose OEStreamLogger in OETestContext
In reply to Aníbal LimónThe OEStreamLogger class is used for redirect PyUnit output to a certain logger so there is not need to expose at level of OETestContext because only OETestRunner needs to know.
[YOCTO #11450]
--- meta/lib/oeqa/core/context.py | 6 ++---- meta/lib/oeqa/core/runner.py | 3 +++ 2 files changed, 5 insertions(+), 4 deletions(-)
meta/lib/oeqa/core/context.py +2 -4
@@ -10,12 +10,11 @@ import collections import re from oeqa.core.loader import OETestLoader -from oeqa.core.runner import OETestRunner, OEStreamLogger, xmlEnabled +from oeqa.core.runner import OETestRunner, xmlEnabled class OETestContext(object): loaderClass = OETestLoader runnerClass = OETestRunner - streamLoggerClass = OEStreamLogger files_dir = os.path.abspath(os.path.join(os.path.dirname( os.path.abspath(__file__)), "../files"))
@@ -52,8 +51,7 @@ class OETestContext(object): self.suites = self.loader.discover() def runTests(self): - streamLogger = self.streamLoggerClass(self.logger) - self.runner = self.runnerClass(self, stream=streamLogger, verbosity=2) + self.runner = self.runnerClass(self, verbosity=2) self._run_start_time = time.time() result = self.runner.run(self.suites)
meta/lib/oeqa/core/runner.py +3 -0
@@ -48,12 +48,15 @@ class OETestResult(_TestResult): super(OETestResult, self).startTest(test) class OETestRunner(_TestRunner): + streamLoggerClass = OEStreamLogger + def __init__(self, tc, *args, **kwargs): if xmlEnabled: if not kwargs.get('output'): kwargs['output'] = os.path.join(os.getcwd(), 'TestResults_%s_%s' % (time.strftime("%Y%m%d%H%M%S"), os.getpid())) + kwargs['stream'] = self.streamLoggerClass(tc.logger) super(OETestRunner, self).__init__(*args, **kwargs) self.tc = tc self.resultclass = OETestResult
Signature · 1 line
2.1.4
PATCH 5/17 oeqa/core/runner: OETestResult remove unneeded override of startTest
In reply to Aníbal LimónI override this method before for keep track of results and forget to remove it, now isn't need.
--- meta/lib/oeqa/core/runner.py | 3 --- 1 file changed, 3 deletions(-)
meta/lib/oeqa/core/runner.py +0 -3
@@ -44,9 +44,6 @@ class OETestResult(_TestResult): self.tc._results['skipped'] = self.skipped self.tc._results['expectedFailures'] = self.expectedFailures - def startTest(self, test): - super(OETestResult, self).startTest(test) - class OETestRunner(_TestRunner): streamLoggerClass = OEStreamLogger
Signature · 1 line
2.1.4
PATCH 6/17 oeqa/core: Move OETestContext.log{Summary, Details} into OETestResult
In reply to Aníbal LimónThose methods are used to write in the log the results so it makes sense to have defined at OETestResult because is a format of the result itself.
[YOCTO #11450]
--- meta/classes/testimage.bbclass | 4 +- meta/classes/testsdk.bbclass | 8 ++-- meta/lib/oeqa/core/context.py | 93 ++---------------------------------------- meta/lib/oeqa/core/runner.py | 88 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 97 insertions(+), 96 deletions(-)
meta/classes/testimage.bbclass +2 -2
@@ -292,8 +292,8 @@ def testimage_main(d): # Show results (if we have them) if not results: bb.fatal('%s - FAILED - tests were interrupted during execution' % pn) - tc.logSummary(results, pn) - tc.logDetails() + results.logSummary(pn) + results.logDetails() if not results.wasSuccessful(): bb.fatal('%s - FAILED - check the task log and the ssh log' % pn)
meta/classes/testsdk.bbclass +4 -4
@@ -70,8 +70,8 @@ def testsdk_main(d): component = "%s %s" % (pn, OESDKTestContextExecutor.name) context_msg = "%s:%s" % (os.path.basename(tcname), os.path.basename(sdk_env)) - tc.logSummary(result, component, context_msg) - tc.logDetails() + result.logSummary(component, context_msg) + result.logDetails() if not result.wasSuccessful(): fail = True
@@ -172,8 +172,8 @@ def testsdkext_main(d): component = "%s %s" % (pn, OESDKExtTestContextExecutor.name) context_msg = "%s:%s" % (os.path.basename(tcname), os.path.basename(sdk_env)) - tc.logSummary(result, component, context_msg) - tc.logDetails() + result.logSummary(component, context_msg) + result.logDetails() if not result.wasSuccessful(): fail = True
meta/lib/oeqa/core/context.py +3 -90
113 lines not on this page · raw message
meta/lib/oeqa/core/runner.py +88 -0
100 lines not on this page · raw message
Signature · 1 line
2.1.4
PATCH 7/17 oeqa/core/threaded: Add new module with OETestLoaderThreaded
In reply to Aníbal LimónThe threded module will implement Thread support into the OEQA framework.
The OETestLoaderThreaded overrides discover to return a list of suites every suite will be run by a separate Thread.
Some design considerations are:
- All the tests of a certain module needs to be run at one thread because unittest framework needs to keep track of setUp{Module,
Class,} and tearDown{Module, Class,}.
- Tests that depends on other needs to be run at the same thread because OEQA framework look at the status of dependant test to decide if skip or not, this constraint can be change in the future but a sync mechanishm is needed to implement between the Threads.
[YOCTO #11450]
--- meta/lib/oeqa/core/threaded.py | 91 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 meta/lib/oeqa/core/threaded.py
meta/lib/oeqa/core/threaded.py new file +91 -0
@@ -0,0 +1,91 @@ +# Copyright (C) 2017 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import multiprocessing + +from unittest.suite import TestSuite +from oeqa.core.loader import OETestLoader + +class OETestLoaderThreaded(OETestLoader): + def __init__(self, tc, module_paths, modules, tests, modules_required, + filters, process_num=0, *args, **kwargs): + super(OETestLoaderThreaded, self).__init__(tc, module_paths, modules, + tests, modules_required, filters, *args, **kwargs) + + self.process_num = process_num + + def discover(self): + suite = super(OETestLoaderThreaded, self).discover() + + if self.process_num <= 0: + self.process_num = min(multiprocessing.cpu_count(), + len(suite._tests)) + + suites = [] + for _ in range(self.process_num): + suites.append(self.suiteClass()) + + def _search_for_module_idx(suites, case): + """ + Cases in the same module needs to be run + in the same thread because PyUnit keeps track + of setUp{Module, Class,} and tearDown{Module, Class,}. + """ + + for idx in range(self.process_num): + suite = suites[idx] + for c in suite._tests: + if case.__module__ == c.__module__: + return idx + + return -1 + + def _search_for_depend_idx(suites, depends): + """ + Dependency cases needs to be run in the same + thread, because OEQA framework look at the state + of dependant test to figure out if skip or not. + """ + + for idx in range(self.process_num): + suite = suites[idx] + + for case in suite._tests: + if case.id() in depends: + return idx + return -1 + + def _get_best_idx(suites): + sizes = [len(suite._tests) for suite in suites] + return sizes.index(min(sizes)) + + def _fill_suites(suite): + idx = -1 + for case in suite: + if isinstance(case, TestSuite): + _fill_suites(case) + else: + idx = _search_for_module_idx(suites, case) + + depends = {} + if 'depends' in self.tc._registry: + depends = self.tc._registry['depends'] + + if idx == -1 and case.id() in depends: + case_depends = depends[case.id()] + idx = _search_for_depend_idx(suites, case_depends) + + if idx == -1: + idx = _get_best_idx(suites) + + suites[idx].addTest(case) + _fill_suites(suite) + + suites_tmp = suites + suites = [] + for suite in suites_tmp: + if len(suite._tests) > 0: + suites.append(suite) + + return suites +
Signature · 1 line
2.1.4
PATCH 8/17 oeqa/core/threaded: Add OEStreamLoggerThreaded class
In reply to Aníbal LimónThe OEStreamLoggerThreaded overrides OEStreamLogger to redirect the PyUnit output to a logger.
Instead of log every line when comes the OEStreamLoggerThreaded will buffer the PyUnit output and write everything at end of every suite execution to don't have mixed suite outputs.
[YOCTO #11450]
--- meta/lib/oeqa/core/threaded.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+)
meta/lib/oeqa/core/threaded.py +25 -0
@@ -1,10 +1,13 @@ # Copyright (C) 2017 Intel Corporation # Released under the MIT license (see COPYING.MIT) +import threading import multiprocessing from unittest.suite import TestSuite + from oeqa.core.loader import OETestLoader +from oeqa.core.runner import OEStreamLogger class OETestLoaderThreaded(OETestLoader): def __init__(self, tc, module_paths, modules, tests, modules_required,
@@ -89,3 +92,25 @@ class OETestLoaderThreaded(OETestLoader): return suites +class OEStreamLoggerThreaded(OEStreamLogger): + _lock = threading.Lock() + buffers = {} + + def write(self, msg): + tid = threading.get_ident() + + if not tid in self.buffers: + self.buffers[tid] = "" + + if msg: + self.buffers[tid] += msg + + def finish(self): + tid = threading.get_ident() + + self._lock.acquire() + self.logger.info('THREAD: %d' % tid) + self.logger.info('-' * 70) + for line in self.buffers[tid].split('\n'): + self.logger.info(line) + self._lock.release()
Signature · 1 line
2.1.4
PATCH 9/17 oeqa/core/runner: OETestResult add internal _tc_map_results
In reply to Aníbal LimónThis method is to assign results into the TestContext, create an internal one to support change implementation in Thread version.
[YOCTO #11450]
--- meta/lib/oeqa/core/runner.py | 2 ++ 1 file changed, 2 insertions(+)
meta/lib/oeqa/core/runner.py +2 -0
@@ -39,7 +39,9 @@ class OETestResult(_TestResult): super(OETestResult, self).__init__(*args, **kwargs) self.tc = tc + self._tc_map_results() + def _tc_map_results(self): self.tc._results['failures'] = self.failures self.tc._results['errors'] = self.errors self.tc._results['skipped'] = self.skipped
Signature · 1 line
2.1.4
PATCH 10/17 oeqa/core/threaded: Add OETestResultThreaded{, Internal} classes
In reply to Aníbal LimónThe OETestResultThreadedInternal extends OETestResult to stores results by Thread.
The OETestResultThreaded is a simple class that provides the implementation of interfaces needed by outside like wasSuccesful, stop, logSummary, logDetails.
[YOCTO #11450]
--- meta/lib/oeqa/core/threaded.py | 73 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-)
meta/lib/oeqa/core/threaded.py +72 -1
@@ -7,7 +7,7 @@ import multiprocessing from unittest.suite import TestSuite from oeqa.core.loader import OETestLoader -from oeqa.core.runner import OEStreamLogger +from oeqa.core.runner import OEStreamLogger, OETestResult class OETestLoaderThreaded(OETestLoader): def __init__(self, tc, module_paths, modules, tests, modules_required,
@@ -114,3 +114,74 @@ class OEStreamLoggerThreaded(OEStreamLogger): for line in self.buffers[tid].split('\n'): self.logger.info(line) self._lock.release() + +class OETestResultThreadedInternal(OETestResult): + def _tc_map_results(self): + tid = threading.get_ident() + + # PyUnit generates a result for every test module run, test + # if the thread already has an entry to avoid lose the previous + # test module results. + if not tid in self.tc._results: + self.tc._results[tid] = {} + self.tc._results[tid]['failures'] = self.failures + self.tc._results[tid]['errors'] = self.errors + self.tc._results[tid]['skipped'] = self.skipped + self.tc._results[tid]['expectedFailures'] = self.expectedFailures + +class OETestResultThreaded(object): + _results = {} + _lock = threading.Lock() + + def __init__(self, tc): + self.tc = tc + + def _fill_tc_results(self): + tids = list(self.tc._results.keys()) + fields = ['failures', 'errors', 'skipped', 'expectedFailures'] + + for tid in tids: + result = self.tc._results[tid] + for field in fields: + if not field in self.tc._results: + self.tc._results[field] = [] + self.tc._results[field].extend(result[field]) + + def addResult(self, result, run_start_time, run_end_time): + tid = threading.get_ident() + + self._lock.acquire() + self._results[tid] = {} + self._results[tid]['result'] = result + self._results[tid]['run_start_time'] = run_start_time + self._results[tid]['run_end_time'] = run_end_time + self._results[tid]['result'] = result + self._lock.release() + + def wasSuccessful(self): + wasSuccessful = True + for tid in self._results.keys(): + wasSuccessful = wasSuccessful and \ + self._results[tid]['result'].wasSuccessful() + return wasSuccessful + + def stop(self): + for tid in self._results.keys(): + self._results[tid]['result'].stop() + + def logSummary(self, component, context_msg=''): + elapsed_time = (self.tc._run_end_time - self.tc._run_start_time) + + self.tc.logger.info("SUMMARY:") + self.tc.logger.info("%s (%s) - Ran %d tests in %.3fs" % (component, + context_msg, len(self.tc._registry['cases']), elapsed_time)) + if self.wasSuccessful(): + msg = "%s - OK - All required tests passed" % component + else: + msg = "%s - FAIL - Required tests failed" % component + self.tc.logger.info(msg) + + def logDetails(self): + tid = list(self._results)[0] + result = self._results[tid]['result'] + result.logDetails()
Signature · 1 line
2.1.4
PATCH 11/17 oeqa/core/threaded: Add support of OETestRunnerThreaded
In reply to Aníbal LimónThe OETestRunnerThreaded overrides the run method of OETestRunner it recieves a list of suites to be executed by a ThreadPool.
The new run method handles the ThreadPool creation and the OETestResultThreaded fill.
[YOCTO #11450]
--- meta/lib/oeqa/core/threaded.py | 75 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-)
meta/lib/oeqa/core/threaded.py +74 -1
@@ -3,11 +3,13 @@ import threading import multiprocessing +import queue +import time from unittest.suite import TestSuite from oeqa.core.loader import OETestLoader -from oeqa.core.runner import OEStreamLogger, OETestResult +from oeqa.core.runner import OEStreamLogger, OETestResult, OETestRunner class OETestLoaderThreaded(OETestLoader): def __init__(self, tc, module_paths, modules, tests, modules_required,
@@ -185,3 +187,74 @@ class OETestResultThreaded(object): tid = list(self._results)[0] result = self._results[tid]['result'] result.logDetails() + +class _Worker(threading.Thread): + """Thread executing tasks from a given tasks queue""" + def __init__(self, tasks, result, stream): + threading.Thread.__init__(self) + self.tasks = tasks + + self.result = result + self.stream = stream + + def run(self): + while True: + try: + func, args, kargs = self.tasks.get(block=False) + except queue.Empty: + break + + try: + run_start_time = time.time() + rc = func(*args, **kargs) + run_end_time = time.time() + self.result.addResult(rc, run_start_time, run_end_time) + self.stream.finish() + except Exception as e: + print(e) + finally: + self.tasks.task_done() + +class _ThreadedPool: + """Pool of threads consuming tasks from a queue""" + def __init__(self, num_workers, num_tasks, stream=None, result=None): + self.tasks = queue.Queue(num_tasks) + self.workers = [] + + for _ in range(num_workers): + worker = _Worker(self.tasks, result, stream) + self.workers.append(worker) + + def start(self): + for worker in self.workers: + worker.start() + + def add_task(self, func, *args, **kargs): + """Add a task to the queue""" + self.tasks.put((func, args, kargs)) + + def wait_completion(self): + """Wait for completion of all the tasks in the queue""" + self.tasks.join() + for worker in self.workers: + worker.join() + +class OETestRunnerThreaded(OETestRunner): + streamLoggerClass = OEStreamLoggerThreaded + + def __init__(self, tc, *args, **kwargs): + super(OETestRunnerThreaded, self).__init__(tc, *args, **kwargs) + self.resultclass = OETestResultThreadedInternal # XXX: XML reporting overrides at __init__ + + def run(self, suites): + result = OETestResultThreaded(self.tc) + + pool = _ThreadedPool(len(suites), len(suites), stream=self.stream, + result=result) + for s in suites: + pool.add_task(super(OETestRunnerThreaded, self).run, s) + pool.start() + pool.wait_completion() + result._fill_tc_results() + + return result
Signature · 1 line
2.1.4
PATCH 12/17 oeqa/core/threaded: Add OETestContextThreaded class
In reply to Aníbal LimónThis class sets the {loader,runner}Class to the threaded versions and overrides loadTests method to be able specify the process_num.
[YOCTO #11450]
--- meta/lib/oeqa/core/threaded.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+)
meta/lib/oeqa/core/threaded.py +14 -0
@@ -10,6 +10,7 @@ from unittest.suite import TestSuite from oeqa.core.loader import OETestLoader from oeqa.core.runner import OEStreamLogger, OETestResult, OETestRunner +from oeqa.core.context import OETestContext class OETestLoaderThreaded(OETestLoader): def __init__(self, tc, module_paths, modules, tests, modules_required,
@@ -258,3 +259,16 @@ class OETestRunnerThreaded(OETestRunner): result._fill_tc_results() return result + +class OETestContextThreaded(OETestContext): + loaderClass = OETestLoaderThreaded + runnerClass = OETestRunnerThreaded + + def loadTests(self, module_paths, modules=[], tests=[], + modules_manifest="", modules_required=[], filters={}, process_num=0): + if modules_manifest: + modules = self._read_modules_from_manifest(modules_manifest) + + self.loader = self.loaderClass(self, module_paths, modules, tests, + modules_required, filters, process_num) + self.suites = self.loader.discover()
Signature · 1 line
2.1.4
PATCH 13/17 oeqa/core/decorator/depends: Add support for threading mode
In reply to Aníbal LimónThe _skipTestDependency needs to know if the thread mode is enabled because the _results are by thread.
[YOCTO #11450]
--- meta/lib/oeqa/core/decorator/depends.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-)
meta/lib/oeqa/core/decorator/depends.py +7 -1
@@ -3,6 +3,7 @@ from unittest import SkipTest +from oeqa.core.threaded import OETestRunnerThreaded from oeqa.core.exception import OEQADependency from . import OETestDiscover, registerDecorator
@@ -63,7 +64,12 @@ def _order_test_case_by_depends(cases, depends): return [cases[case_id] for case_id in cases_ordered] def _skipTestDependency(case, depends): - results = case.tc._results + if isinstance(case.tc.runner, OETestRunnerThreaded): + import threading + results = case.tc._results[threading.get_ident()] + else: + results = case.tc._results + skipReasons = ['errors', 'failures', 'skipped'] for reason in skipReasons:
Signature · 1 line
2.1.4
PATCH 14/17 oeqa/core/decorator/oetimeout: Add support for OEQA threaded mode
In reply to Aníbal LimónIn python signals are only allowed to delivery into the main thread, to support the threading mode test if the runner is threaded and use threading.Timer instead.
There are some considerations like SIGALRM interrupts the execution after N seconds but the Timer only starts a Thread to notice the timeout and the exception will be raised when the test run ends.
[YOCTO #11450]
--- meta/lib/oeqa/core/decorator/oetimeout.py | 40 +++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 10 deletions(-)
meta/lib/oeqa/core/decorator/oetimeout.py +30 -10
@@ -1,8 +1,12 @@ # Copyright (C) 2016 Intel Corporation # Released under the MIT license (see COPYING.MIT) -import signal from . import OETestDecorator, registerDecorator + +import signal +from threading import Timer + +from oeqa.core.threaded import OETestRunnerThreaded from oeqa.core.exception import OEQATimeoutError @registerDecorator
@@ -10,16 +14,32 @@ class OETimeout(OETestDecorator): attrs = ('oetimeout',) def setUpDecorator(self): - timeout = self.oetimeout - def _timeoutHandler(signum, frame): - raise OEQATimeoutError("Timed out after %s " + self.logger.debug("Setting up a %d second(s) timeout" % self.oetimeout) + + if isinstance(self.case.tc.runner, OETestRunnerThreaded): + self.timeouted = False + def _timeoutHandler(): + self.timeouted = True + + self.timer = Timer(self.oetimeout, _timeoutHandler) + self.timer.start() + else: + timeout = self.oetimeout + def _timeoutHandler(signum, frame): + raise OEQATimeoutError("Timed out after %s " "seconds of execution" % timeout) - self.logger.debug("Setting up a %d second(s) timeout" % self.oetimeout) - self.alarmSignal = signal.signal(signal.SIGALRM, _timeoutHandler) - signal.alarm(self.oetimeout) + self.alarmSignal = signal.signal(signal.SIGALRM, _timeoutHandler) + signal.alarm(self.oetimeout) def tearDownDecorator(self): - signal.alarm(0) - signal.signal(signal.SIGALRM, self.alarmSignal) - self.logger.debug("Removed SIGALRM handler") + if isinstance(self.case.tc.runner, OETestRunnerThreaded): + self.timer.cancel() + self.logger.debug("Removed Timer handler") + if self.timeouted: + raise OEQATimeoutError("Timed out after %s " + "seconds of execution" % self.oetimeout) + else: + signal.alarm(0) + signal.signal(signal.SIGALRM, self.alarmSignal) + self.logger.debug("Removed SIGALRM handler")
Signature · 1 line
2.1.4
PATCH 15/17 oeqa/core/tests: Add tests of OEQA Threaded mode
In reply to Aníbal LimónAdd needed tests to validate the OEQA Threaded mode, the remaining parts are tested by the OEQA without Threaded mode.
- test_loader.py: Add a test to validate rules when creating the list of test suites.
- test_decorators.py: Add oetimeout test because the threaded mode uses Timer instead of signal.
[YOCTO #11450]
--- .../core/tests/cases/loader/threaded/threaded.py | 12 +++++++++ .../tests/cases/loader/threaded/threaded_alone.py | 8 ++++++ .../cases/loader/threaded/threaded_depends.py | 10 ++++++++ .../tests/cases/loader/threaded/threaded_module.py | 12 +++++++++ meta/lib/oeqa/core/tests/common.py | 10 ++++++++ meta/lib/oeqa/core/tests/test_decorators.py | 12 +++++++++ meta/lib/oeqa/core/tests/test_loader.py | 30 +++++++++++++++++++++- 7 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 meta/lib/oeqa/core/tests/cases/loader/threaded/threaded.py create mode 100644 meta/lib/oeqa/core/tests/cases/loader/threaded/threaded_alone.py create mode 100644 meta/lib/oeqa/core/tests/cases/loader/threaded/threaded_depends.py create mode 100644 meta/lib/oeqa/core/tests/cases/loader/threaded/threaded_module.py
meta/lib/oeqa/core/tests/cases/loader/threaded/threaded.py new file +12 -0
@@ -0,0 +1,12 @@ +# Copyright (C) 2017 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +from oeqa.core.case import OETestCase + +class ThreadedTest(OETestCase): + def test_threaded_no_depends(self): + self.assertTrue(True, msg='How is this possible?') + +class ThreadedTest2(OETestCase): + def test_threaded_same_module(self): + self.assertTrue(True, msg='How is this possible?')
meta/lib/oeqa/core/tests/cases/loader/threaded/threaded_alone.py new file +8 -0
@@ -0,0 +1,8 @@ +# Copyright (C) 2017 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +from oeqa.core.case import OETestCase + +class ThreadedTestAlone(OETestCase): + def test_threaded_alone(self): + self.assertTrue(True, msg='How is this possible?')
meta/lib/oeqa/core/tests/cases/loader/threaded/threaded_depends.py new file +10 -0
@@ -0,0 +1,10 @@ +# Copyright (C) 2017 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +from oeqa.core.case import OETestCase +from oeqa.core.decorator.depends import OETestDepends + +class ThreadedTest3(OETestCase): + @OETestDepends(['threaded.ThreadedTest.test_threaded_no_depends']) + def test_threaded_depends(self): + self.assertTrue(True, msg='How is this possible?')
meta/lib/oeqa/core/tests/cases/loader/threaded/threaded_module.py new file +12 -0
@@ -0,0 +1,12 @@ +# Copyright (C) 2017 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +from oeqa.core.case import OETestCase + +class ThreadedTestModule(OETestCase): + def test_threaded_module(self): + self.assertTrue(True, msg='How is this possible?') + +class ThreadedTestModule2(OETestCase): + def test_threaded_module2(self): + self.assertTrue(True, msg='How is this possible?')
meta/lib/oeqa/core/tests/common.py +10 -0
@@ -33,3 +33,13 @@ class TestBase(unittest.TestCase): tc.loadTests(self.cases_path, modules=modules, tests=tests, filters=filters) return tc + + def _testLoaderThreaded(self, d={}, modules=[], + tests=[], filters={}): + from oeqa.core.threaded import OETestContextThreaded + + tc = OETestContextThreaded(d, self.logger) + tc.loadTests(self.cases_path, modules=modules, tests=tests, + filters=filters) + + return tc
meta/lib/oeqa/core/tests/test_decorators.py +12 -0
@@ -131,5 +131,17 @@ class TestTimeoutDecorator(TestBase): msg = "OETestTimeout didn't restore SIGALRM" self.assertIs(alarm_signal, signal.getsignal(signal.SIGALRM), msg=msg) + def test_timeout_thread(self): + tests = ['timeout.TimeoutTest.testTimeoutPass'] + msg = 'Failed to run test using OETestTimeout' + tc = self._testLoaderThreaded(modules=self.modules, tests=tests) + self.assertTrue(tc.runTests().wasSuccessful(), msg=msg) + + def test_timeout_threaded_fail(self): + tests = ['timeout.TimeoutTest.testTimeoutFail'] + msg = "OETestTimeout test didn't timeout as expected" + tc = self._testLoaderThreaded(modules=self.modules, tests=tests) + self.assertFalse(tc.runTests().wasSuccessful(), msg=msg) + if __name__ == '__main__': unittest.main()
meta/lib/oeqa/core/tests/test_loader.py +29 -1
@@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright (C) 2016 Intel Corporation +# Copyright (C) 2016-2017 Intel Corporation # Released under the MIT license (see COPYING.MIT) import os
@@ -82,5 +82,33 @@ class TestLoader(TestBase): msg = 'Expected modules from two different paths' self.assertEqual(modules, expected_modules, msg=msg) + def test_loader_threaded(self): + cases_path = self.cases_path + + self.cases_path = [os.path.join(self.cases_path, 'loader', 'threaded')] + + tc = self._testLoaderThreaded() + self.assertEqual(len(tc.suites), 3, "Expected to be 3 suites") + + case_ids = ['threaded.ThreadedTest.test_threaded_no_depends', + 'threaded.ThreadedTest2.test_threaded_same_module', + 'threaded_depends.ThreadedTest3.test_threaded_depends'] + for case in tc.suites[0]._tests: + self.assertEqual(case.id(), + case_ids[tc.suites[0]._tests.index(case)]) + + case_ids = ['threaded_alone.ThreadedTestAlone.test_threaded_alone'] + for case in tc.suites[1]._tests: + self.assertEqual(case.id(), + case_ids[tc.suites[1]._tests.index(case)]) + + case_ids = ['threaded_module.ThreadedTestModule.test_threaded_module', + 'threaded_module.ThreadedTestModule2.test_threaded_module2'] + for case in tc.suites[2]._tests: + self.assertEqual(case.id(), + case_ids[tc.suites[2]._tests.index(case)]) + + self.cases_path = cases_path + if __name__ == '__main__': unittest.main()
Signature · 1 line
2.1.4
PATCH 16/17 oeqa/sdkext/cases: Move sdk_update test into devtool module
In reply to Aníbal LimónWith the new OEQA thread support there are problems running devtool twice at the same time because only one instance of bitbake/devtool is allowed.
[YOCTO #11450]
--- meta/lib/oeqa/sdkext/cases/devtool.py | 32 ++++++++++++++++++++++++++ meta/lib/oeqa/sdkext/cases/sdk_update.py | 39 -------------------------------- 2 files changed, 32 insertions(+), 39 deletions(-) delete mode 100644 meta/lib/oeqa/sdkext/cases/sdk_update.py
meta/lib/oeqa/sdkext/cases/devtool.py +32 -0
@@ -1,12 +1,14 @@ # Copyright (C) 2016 Intel Corporation # Released under the MIT license (see COPYING.MIT) +import os import shutil import subprocess from oeqa.sdkext.case import OESDKExtTestCase from oeqa.core.decorator.depends import OETestDepends from oeqa.core.decorator.oeid import OETestID +from oeqa.utils.httpserver import HTTPService class DevtoolTest(OESDKExtTestCase): @classmethod
@@ -95,3 +97,33 @@ class DevtoolTest(OESDKExtTestCase): self._run('devtool build %s ' % package_nodejs) finally: self._run('devtool reset %s '% package_nodejs) + +class SdkUpdateTest(OESDKExtTestCase): + @classmethod + def setUpClass(self): + self.publish_dir = os.path.join(self.tc.sdk_dir, 'esdk_publish') + if os.path.exists(self.publish_dir): + shutil.rmtree(self.publish_dir) + os.mkdir(self.publish_dir) + + base_tcname = "%s/%s" % (self.td.get("SDK_DEPLOY", ''), + self.td.get("TOOLCHAINEXT_OUTPUTNAME", '')) + tcname_new = "%s-new.sh" % base_tcname + if not os.path.exists(tcname_new): + tcname_new = "%s.sh" % base_tcname + + cmd = 'oe-publish-sdk %s %s' % (tcname_new, self.publish_dir) + subprocess.check_output(cmd, shell=True) + + self.http_service = HTTPService(self.publish_dir) + self.http_service.start() + + self.http_url = "http://127.0.0.1:%d" % self.http_service.port + + def test_sdk_update_http(self): + output = self._run("devtool sdk-update \"%s\"" % self.http_url) + + @classmethod + def tearDownClass(self): + self.http_service.stop() + shutil.rmtree(self.publish_dir)
meta/lib/oeqa/sdkext/cases/sdk_update.py deleted +0 -39
@@ -1,39 +0,0 @@ -# Copyright (C) 2016 Intel Corporation -# Released under the MIT license (see COPYING.MIT) - -import os -import shutil -import subprocess - -from oeqa.sdkext.case import OESDKExtTestCase -from oeqa.utils.httpserver import HTTPService - -class SdkUpdateTest(OESDKExtTestCase): - @classmethod - def setUpClass(self): - self.publish_dir = os.path.join(self.tc.sdk_dir, 'esdk_publish') - if os.path.exists(self.publish_dir): - shutil.rmtree(self.publish_dir) - os.mkdir(self.publish_dir) - - base_tcname = "%s/%s" % (self.td.get("SDK_DEPLOY", ''), - self.td.get("TOOLCHAINEXT_OUTPUTNAME", '')) - tcname_new = "%s-new.sh" % base_tcname - if not os.path.exists(tcname_new): - tcname_new = "%s.sh" % base_tcname - - cmd = 'oe-publish-sdk %s %s' % (tcname_new, self.publish_dir) - subprocess.check_output(cmd, shell=True) - - self.http_service = HTTPService(self.publish_dir) - self.http_service.start() - - self.http_url = "http://127.0.0.1:%d" % self.http_service.port - - def test_sdk_update_http(self): - output = self._run("devtool sdk-update \"%s\"" % self.http_url) - - @classmethod - def tearDownClass(self): - self.http_service.stop() - shutil.rmtree(self.publish_dir)
Signature · 1 line
2.1.4
PATCH 17/17 oeqa/sdk: Enable usage of OEQA thread mode
In reply to Aníbal Limónoeqa/sdk/context.py - Use OETestContextThreaded. classes/testsdk.bbclass - Enable bb event thread mode to avoid corrupt the PIPE when multiple threads writes.
[YOCTO #11450]
--- meta/classes/testsdk.bbclass | 4 ++++ meta/lib/oeqa/sdk/context.py | 5 +++-- 2 files changed, 7 insertions(+), 2 deletions(-)
meta/classes/testsdk.bbclass +4 -0
@@ -24,6 +24,8 @@ def testsdk_main(d): from oeqa.sdk.context import OESDKTestContext, OESDKTestContextExecutor from oeqa.utils import make_logger_bitbake_compatible + bb.event.enable_threadlock() + pn = d.getVar("PN") logger = make_logger_bitbake_compatible(logging.getLogger("BitBake"))
@@ -97,6 +99,8 @@ def testsdkext_main(d): from oeqa.utils import avoid_paths_in_environ, make_logger_bitbake_compatible, subprocesstweak from oeqa.sdkext.context import OESDKExtTestContext, OESDKExtTestContextExecutor + bb.event.enable_threadlock() + pn = d.getVar("PN") logger = make_logger_bitbake_compatible(logging.getLogger("BitBake"))
meta/lib/oeqa/sdk/context.py +3 -2
@@ -6,9 +6,10 @@ import sys import glob import re -from oeqa.core.context import OETestContext, OETestContextExecutor +from oeqa.core.context import OETestContextExecutor +from oeqa.core.threaded import OETestContextThreaded -class OESDKTestContext(OETestContext): +class OESDKTestContext(OETestContextThreaded): sdk_files_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "files") def __init__(self, td=None, logger=None, sdk_dir=None, sdk_env=None,
Signature · 1 line
2.1.4