OEQA Framework Refactor & Improvements

new file mode 100644
index 0000000..e69de29

46 lines not on this page · raw message

new file mode 100644
index 0000000..2dfd840
@@ -0,0 +1,14 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+class OEQAException(Exception):
+    pass
+
+class OEQATimeoutError(OEQAException):
+    pass
+
+class OEQAMissingVariable(OEQAException):
+    pass
+
+class OEQADependency(OEQAException):
+    pass

76 lines not on this page · raw message

new file mode 100644
index 0000000..e69de29

new file mode 100644
index 0000000..6ae58ad
@@ -0,0 +1,37 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+def toList(obj, obj_type, obj_name="Object"):
+    if isinstance(obj, obj_type):
+        return [obj]
+    elif isinstance(obj, list):
+        return obj
+    else:
+        raise TypeError("%s must be %s or list" % (obj_name, obj_type))
+
+def toSet(obj, obj_type, obj_name="Object"):
+    if isinstance(obj, obj_type):
+        return {obj}
+    elif isinstance(obj, list):
+        return set(obj)
+    elif isinstance(obj, set):
+        return obj
+    else:
+        raise TypeError("%s must be %s or set" % (obj_name, obj_type))
+
+def strToList(obj, obj_name="Object"):
+    return toList(obj, str, obj_name)
+
+def strToSet(obj, obj_name="Object"):
+    return toSet(obj, str, obj_name)
+
+def intToList(obj, obj_name="Object"):
+    return toList(obj, int, obj_name)
+
+def dataStoteToDict(d, variables):
+    data = {}
+
+    for v in variables:
+        data[v] = d.getVar(v, True)
+
+    return data

new file mode 100644
index 0000000..cb06523
@@ -0,0 +1,14 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+import os
+import sys
+
+def findFile(file_name, directory):
+    """
+        Search for a file in directory and returns its complete path.
+    """
+    for r, d, f in os.walk(directory):
+        if file_name in f:
+            return os.path.join(r, file_name)
+    return None

71 lines not on this page · raw message

new file mode 100644
index 0000000..d5caf53
@@ -0,0 +1,225 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+import os
+import sys
+import json
+import time
+import logging
+import collections
+import re
+
+from oeqa.core.loader import OETestLoader
+from oeqa.core.runner import OETestRunner, OEStreamLogger, xmlEnabled
+
+class OETestContext(object):
+    loaderClass = OETestLoader
+    runnerClass = OETestRunner
+    streamLoggerClass = OEStreamLogger
+
+    files_dir = os.path.abspath(os.path.join(os.path.dirname(
+        os.path.abspath(__file__)), "../files"))
+
+    def __init__(self, d=None, logger=None):
+        if not type(d) is dict:
+            raise TypeError("d isn't dictionary type")
+
+        self.d = d
+        self.logger = logger
+        self._registry = {}
+        self._registry['cases'] = collections.OrderedDict()
+        self._results = {}
+
+    def _read_modules_from_manifest(self, manifest):
+        if not os.path.exists(manifest):
+            raise
+
+        modules = []
+        for line in open(manifest).readlines():
+            line = line.strip()
+            if line and not line.startswith("#"):
+                modules.append(line)
+
+        return modules
+
+    def loadTests(self, module_paths, modules=[], tests=[],
+            modules_manifest="", modules_required=[], filters={}):
+        if modules_manifest:
+            modules = self._read_modules_from_manifest(modules_manifest)
+
+        self.loader = self.loaderClass(self, module_paths, modules, tests,
+                modules_required, filters)
+        self.suites = self.loader.discover()
+
+    def runTests(self):
+        streamLogger = self.streamLoggerClass(self.logger)
+        self.runner = self.runnerClass(self, stream=streamLogger, verbosity=2)
+
+        self._run_start_time = time.time()
+        result = self.runner.run(self.suites)
+        self._run_end_time = time.time()
+
+        return result
+
+    def logSummary(self, result, component, context_msg=''):
+        self.logger.info("SUMMARY:")
+        self.logger.info("%s (%s) - Ran %d test%s in %.3fs" % (component,
+            context_msg, result.testsRun, result.testsRun != 1 and "s" or "",
+            (self._run_end_time - self._run_start_time)))
+
+        if result.wasSuccessful():
+            msg = "%s - OK - All required tests passed" % component
+        else:
+            msg = "%s - FAIL - Required tests failed" % component
+        skipped = len(self._results['skipped'])
+        if skipped: 
+            msg += " (skipped=%d)" % skipped
+        self.logger.info(msg)
+
+    def _logDetailsNotPassed(self, case, type, desc):
+        found = False
+
+        for (scase, msg) in self._results[type]:
+            # XXX: When XML reporting is enabled scase is
+            # xmlrunner.result._TestInfo instance instead of
+            # string.
+            if xmlEnabled:
+                if case.id() == scase.test_id:
+                    found = True
+                    break
+                scase_str = scase.test_id
+            else:
+                if case == scase:
+                    found = True
+                    break
+                scase_str = str(scase)
+
+            # When fails at module or class level the class name is passed as string
+            # so figure out to see if match
+            m = re.search("^setUpModule \((?P<module_name>.*)\)$", scase_str)
+            if m:
+                if case.__class__.__module__ == m.group('module_name'):
+                    found = True
+                    break
+
+            m = re.search("^setUpClass \((?P<class_name>.*)\)$", scase_str)
+            if m:
+                class_name = "%s.%s" % (case.__class__.__module__,
+                        case.__class__.__name__)
+
+                if class_name == m.group('class_name'):
+                    found = True
+                    break
+
+        if found:
+            self.logger.info("RESULTS - %s - Testcase %s: %s" % (case.id(),
+                case.oe_id if hasattr(case, 'oeid') else '-1', desc))
+            if msg:
+                self.logger.info(msg)
+
+        return found
+
+    def logDetails(self):
+        self.logger.info("RESULTS:")
+        for case_name in self._registry['cases']:
+            case = self._registry['cases'][case_name]
+
+            result_types = ['failures', 'errors', 'skipped', 'expectedFailures']
+            result_desc = ['FAILED', 'ERROR', 'SKIPPED', 'EXPECTEDFAIL']
+
+            match = False
+            for idx, name in enumerate(result_types):
+                match = self._logDetailsNotPassed(case, result_types[idx],
+                        result_desc[idx])
+                if match:
+                    break
+
+            if not match:
+                self.logger.info("RESULTS - %s - Testcase %s: %s" % (case.id(),
+                    case.oe_id if hasattr(case, 'oeid') else '-1',
+                    'PASSED'))
+
+class OETestContextExecutor(object):
+    _context_class = OETestContext
+
+    name = 'core'
+    help = 'core test component example'
+    description = 'executes core test suite example'
+
+    default_cases = [os.path.join(os.path.abspath(os.path.dirname(__file__)),
+            'cases/example')]
+    default_test_data = os.path.join(default_cases[0], 'data.json')
+
+    def register_commands(self, logger, subparsers):
+        self.parser = subparsers.add_parser(self.name, help=self.help,
+                description=self.description, group='components')
+
+        self.default_output_log = '%s-results-%s.log' % (self.name,
+                time.strftime("%Y%m%d%H%M%S"))
+        self.parser.add_argument('--output-log', action='store',
+                default=self.default_output_log,
+                help="results output log, default: %s" % self.default_output_log)
+
+        if self.default_test_data:
+            self.parser.add_argument('--test-data-file', action='store',
+                    default=self.default_test_data,
+                    help="data file to load, default: %s" % self.default_test_data)
+        else:
+            self.parser.add_argument('--test-data-file', action='store',
+                    help="data file to load")
+
+        if self.default_cases:
+            self.parser.add_argument('CASES_PATHS', action='store',
+                    default=self.default_cases, nargs='*',
+                    help="paths to directories with test cases, default: %s"\
+                            % self.default_cases)
+        else:
+            self.parser.add_argument('CASES_PATHS', action='store',
+                    nargs='+', help="paths to directories with test cases")
+
+        self.parser.set_defaults(func=self.run)
+
+    def _setup_logger(self, logger, args):
+        formatter = logging.Formatter('%(asctime)s - ' + self.name + \
+                ' - %(levelname)s - %(message)s')
+        sh = logger.handlers[0]
+        sh.setFormatter(formatter)
+        fh = logging.FileHandler(args.output_log)
+        fh.setFormatter(formatter)
+        logger.addHandler(fh)
+
+        return logger
+
+    def _process_args(self, logger, args):
+        self.tc_kwargs = {}
+        self.tc_kwargs['init'] = {}
+        self.tc_kwargs['load'] = {}
+        self.tc_kwargs['run'] = {}
+
+        self.tc_kwargs['init']['logger'] = self._setup_logger(logger, args)
+        if args.test_data_file:
+            self.tc_kwargs['init']['td'] = json.load(
+                    open(args.test_data_file, "r"))
+        else:
+            self.tc_kwargs['init']['td'] = {}
+
+        self.module_paths = args.CASES_PATHS
+
+    def run(self, logger, args):
+        self._process_args(logger, args)
+
+        self.tc = self._context_class(**self.tc_kwargs['init'])
+        self.tc.loadTests(self.module_paths, **self.tc_kwargs['load'])
+        rc = self.tc.runTests(**self.tc_kwargs['run'])
+        self.tc.logSummary(rc, self.name)
+        self.tc.logDetails()
+
+        output_link = os.path.join(os.path.dirname(args.output_log),
+                "%s-results.log" % self.name)
+        if os.path.exists(output_link):
+            os.remove(output_link)
+        os.symlink(args.output_log, output_link)
+
+        return rc
+
+_executor_class = OETestContextExecutor

71 lines not on this page · raw message

235 lines not on this page · raw message

new file mode 100644
index 0000000..195711c
@@ -0,0 +1,94 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+from unittest import SkipTest
+
+from oeqa.core.exception import OEQADependency
+
+from . import OETestDiscover, registerDecorator
+
+def _add_depends(registry, case, depends):
+    module_name = case.__module__
+    class_name = case.__class__.__name__
+
+    case_id = case.id()
+
+    for depend in depends:
+        dparts = depend.split('.')
+
+        if len(dparts) == 1:
+            depend_id = ".".join((module_name, class_name, dparts[0]))
+        elif len(dparts) == 2:
+            depend_id = ".".join((module_name, dparts[0], dparts[1]))
+        else:
+            depend_id = depend
+
+        if not case_id in registry:
+            registry[case_id] = []
+        if not depend_id in registry[case_id]:
+            registry[case_id].append(depend_id)
+
+def _validate_test_case_depends(cases, depends):
+    for case in depends:
+        if not case in cases:
+            continue
+        for dep in depends[case]:
+            if not dep in cases:
+                raise OEQADependency("TestCase %s depends on %s and isn't available"\
+                       ", cases available %s." % (case, dep, str(cases.keys())))
+
+def _order_test_case_by_depends(cases, depends):
+    def _dep_resolve(graph, node, resolved, seen):
+        seen.append(node)
+        for edge in graph[node]:
+            if edge not in resolved:
+                if edge in seen:
+                    raise OEQADependency("Test cases %s and %s have a circular" \
+                                       " dependency." % (node, edge))
+                _dep_resolve(graph, edge, resolved, seen)
+        resolved.append(node)
+
+    dep_graph = {}
+    dep_graph['__root__'] = cases.keys()
+    for case in cases:
+        if case in depends:
+            dep_graph[case] = depends[case]
+        else:
+            dep_graph[case] = []
+
+    cases_ordered = []
+    _dep_resolve(dep_graph, '__root__', cases_ordered, [])
+    cases_ordered.remove('__root__')
+
+    return [cases[case_id] for case_id in cases_ordered]
+
+def _skipTestDependency(case, depends):
+    results = case.tc._results
+    skipReasons = ['errors', 'failures', 'skipped']
+
+    for reason in skipReasons:
+        for test, _ in results[reason]:
+            if test.id() in depends:
+                raise SkipTest("Test case %s depends on %s and was in %s." \
+                        % (case.id(), test.id(), reason))
+
+@registerDecorator
+class OETestDepends(OETestDiscover):
+    attrs = ('depends',)
+
+    def bind(self, registry, case):
+        super(OETestDepends, self).bind(registry, case)
+        if not registry.get('depends'):
+            registry['depends'] = {}
+        _add_depends(registry['depends'], case, self.depends)
+
+    @staticmethod
+    def discover(registry):
+        if registry.get('depends'):
+            _validate_test_case_depends(registry['cases'], registry['depends'])
+            return _order_test_case_by_depends(registry['cases'], registry['depends'])
+        else:
+            return [registry['cases'][case_id] for case_id in registry['cases']]
+
+    def setUpDecorator(self):
+        _skipTestDependency(self.case, self.depends)

new file mode 100644
index 0000000..ea8017a
@@ -0,0 +1,23 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+from . import OETestFilter, registerDecorator
+from oeqa.core.utils.misc import intToList
+
+def _idFilter(oeid, filters):
+     return False if oeid in filters else True
+
+@registerDecorator
+class OETestID(OETestFilter):
+    attrs = ('oeid',)
+
+    def bind(self, registry, case):
+        super(OETestID, self).bind(registry, case)
+
+    def filtrate(self, filters):
+        if filters.get('oeid'):
+            filterx = intToList(filters['oeid'], 'oeid')
+            del filters['oeid']
+            if _idFilter(self.oeid, filterx):
+                return True
+        return False

new file mode 100644
index 0000000..ad38ab7
@@ -0,0 +1,24 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+from . import OETestFilter, registerDecorator
+from oeqa.core.utils.misc import strToList
+
+def _tagFilter(tags, filters):
+    return False if set(tags) & set(filters) else True
+
+@registerDecorator
+class OETestTag(OETestFilter):
+    attrs = ('oetag',)
+
+    def bind(self, registry, case):
+        super(OETestTag, self).bind(registry, case)
+        self.oetag = strToList(self.oetag, 'oetag')
+
+    def filtrate(self, filters):
+        if filters.get('oetag'):
+            filterx = strToList(filters['oetag'], 'oetag')
+            del filters['oetag']
+            if _tagFilter(self.oetag, filterx):
+                return True
+        return False

new file mode 100644
index 0000000..a247583
@@ -0,0 +1,25 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+import signal
+from . import OETestDecorator, registerDecorator
+from oeqa.core.exception import OEQATimeoutError
+
+@registerDecorator
+class OETimeout(OETestDecorator):
+    attrs = ('oetimeout',)
+
+    def setUpDecorator(self):
+        timeout = self.oetimeout
+        def _timeoutHandler(signum, frame):
+            raise OEQATimeoutError("Timed out after %s "
+                    "seconds of execution" % timeout)
+
+        self.logger.debug("Setting up a %d second(s) timeout" % self.oetimeout)
+        self.alarmSignal = signal.signal(signal.SIGALRM, _timeoutHandler)
+        signal.alarm(self.oetimeout)
+
+    def tearDownDecorator(self):
+        signal.alarm(0)
+        signal.signal(signal.SIGALRM, self.alarmSignal)
+        self.logger.debug("Removed SIGALRM handler")

new file mode 100644
index 0000000..51ef6fe
@@ -0,0 +1,36 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+from oeqa.core.exception import OEQAMissingVariable
+
+from . import OETestDecorator, registerDecorator
+
+@registerDecorator
+class skipIfDataVar(OETestDecorator):
+    """
+        Skip test based on value of a data store's variable.
+
+        It will get the info of var from the data store and will
+        check it against value; if are equal it will skip the test
+        with msg as the reason.
+    """
+
+    attrs = ('var', 'value', 'msg')
+
+    def setUpDecorator(self):
+        msg = 'Checking if %r value is %r to skip test' % (self.var, self.value)
+        self.logger.debug(msg)
+        if self.case.tc.d.get(self.var) == self.value:
+            self.case.skipTest(self.msg)
+
+@registerDecorator
+class OETestDataDepends(OETestDecorator):
+    attrs = ('data_depends',)
+
+    def setUpDecorator(self):
+        for v in self.data_depends:
+            try:
+                value = self.case.d[v]
+            except KeyError:
+                raise OEQAMissingVariable("Test case need %s variable but"\
+                        " isn't into d" % v)

new file mode 100644
index 0000000..e69de29

new file mode 100644
index 0000000..4d8fad0
@@ -0,0 +1,20 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+from oeqa.core.case import OETestCase
+from oeqa.core.decorator.oetag import OETestTag
+from oeqa.core.decorator.data import OETestDataDepends
+
+class DataTest(OETestCase):
+    data_vars = ['IMAGE', 'ARCH']
+
+    @OETestDataDepends(['MACHINE',])
+    @OETestTag('dataTestOk')
+    def testDataOk(self):
+        self.assertEqual(self.d.get('IMAGE'), 'core-image-minimal')
+        self.assertEqual(self.d.get('ARCH'), 'x86')
+        self.assertEqual(self.d.get('MACHINE'), 'qemuarm')
+
+    @OETestTag('dataTestFail')
+    def testDataFail(self):
+        pass

new file mode 100644
index 0000000..17cdd90
@@ -0,0 +1,38 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+from oeqa.core.case import OETestCase
+from oeqa.core.decorator.depends import OETestDepends
+
+class DependsTest(OETestCase):
+
+    def testDependsFirst(self):
+        self.assertTrue(True, msg='How is this possible?')
+
+    @OETestDepends(['testDependsFirst'])
+    def testDependsSecond(self):
+        self.assertTrue(True, msg='How is this possible?')
+
+    @OETestDepends(['testDependsSecond'])
+    def testDependsThird(self):
+        self.assertTrue(True, msg='How is this possible?')
+
+    @OETestDepends(['testDependsSecond'])
+    def testDependsFourth(self):
+        self.assertTrue(True, msg='How is this possible?')
+
+    @OETestDepends(['testDependsThird', 'testDependsFourth'])
+    def testDependsFifth(self):
+        self.assertTrue(True, msg='How is this possible?')
+
+    @OETestDepends(['testDependsCircular3'])
+    def testDependsCircular1(self):
+        self.assertTrue(True, msg='How is this possible?')
+
+    @OETestDepends(['testDependsCircular1'])
+    def testDependsCircular2(self):
+        self.assertTrue(True, msg='How is this possible?')
+
+    @OETestDepends(['testDependsCircular2'])
+    def testDependsCircular3(self):
+        self.assertTrue(True, msg='How is this possible?')

new file mode 100644
index 0000000..038d445
@@ -0,0 +1,15 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+from oeqa.core.case import OETestCase
+
+class AnotherIDTest(OETestCase):
+
+    def testAnotherIdGood(self):
+        self.assertTrue(True, msg='How is this possible?')
+
+    def testAnotherIdOther(self):
+        self.assertTrue(True, msg='How is this possible?')
+
+    def testAnotherIdNone(self):
+        self.assertTrue(True, msg='How is this possible?')

new file mode 100644
index 0000000..c9ffd17
@@ -0,0 +1,9 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+from oeqa.core.case import OETestCase
+
+class AnotherTest(OETestCase):
+
+    def testAnother(self):
+        self.assertTrue(True, msg='How is this possible?')

new file mode 100644
index 0000000..c2d3d32
@@ -0,0 +1,18 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+from oeqa.core.case import OETestCase
+from oeqa.core.decorator.oeid import OETestID
+
+class IDTest(OETestCase):
+
+    @OETestID(101)
+    def testIdGood(self):
+        self.assertTrue(True, msg='How is this possible?')
+
+    @OETestID(102)
+    def testIdOther(self):
+        self.assertTrue(True, msg='How is this possible?')
+
+    def testIdNone(self):
+        self.assertTrue(True, msg='How is this possible?')

new file mode 100644
index 0000000..0cae02e
@@ -0,0 +1,18 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+from oeqa.core.case import OETestCase
+from oeqa.core.decorator.oetag import OETestTag
+
+class TagTest(OETestCase):
+
+    @OETestTag('goodTag')
+    def testTagGood(self):
+        self.assertTrue(True, msg='How is this possible?')
+
+    @OETestTag('otherTag')
+    def testTagOther(self):
+        self.assertTrue(True, msg='How is this possible?')
+
+    def testTagNone(self):
+        self.assertTrue(True, msg='How is this possible?')

new file mode 100644
index 0000000..870c3157
@@ -0,0 +1,18 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+from time import sleep
+
+from oeqa.core.case import OETestCase
+from oeqa.core.decorator.oetimeout import OETimeout
+
+class TimeoutTest(OETestCase):
+
+    @OETimeout(1)
+    def testTimeoutPass(self):
+        self.assertTrue(True, msg='How is this possible?')
+
+    @OETimeout(1)
+    def testTimeoutFail(self):
+        sleep(2)
+        self.assertTrue(True, msg='How is this possible?')

new file mode 100644
index 0000000..52b18a1
@@ -0,0 +1,35 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+import sys
+import os
+
+import unittest
+import logging
+import os
+
+logger = logging.getLogger("oeqa")
+logger.setLevel(logging.INFO)
+consoleHandler = logging.StreamHandler()
+formatter = logging.Formatter('OEQATest: %(message)s')
+consoleHandler.setFormatter(formatter)
+logger.addHandler(consoleHandler)
+
+def setup_sys_path():
+    directory = os.path.dirname(os.path.abspath(__file__))
+    oeqa_lib = os.path.realpath(os.path.join(directory, '../../../'))
+    if not oeqa_lib in sys.path:
+        sys.path.insert(0, oeqa_lib)
+
+class TestBase(unittest.TestCase):
+    def setUp(self):
+        self.logger = logger
+        directory = os.path.dirname(os.path.abspath(__file__))
+        self.cases_path = os.path.join(directory, 'cases')
+
+    def _testLoader(self, d={}, modules=[], tests=[], filters={}):
+        from oeqa.core.context import OETestContext
+        tc = OETestContext(d, self.logger)
+        tc.loadTests(self.cases_path, modules=modules, tests=tests,
+                     filters=filters)
+        return tc

51 lines not on this page · raw message

135 lines not on this page · raw message

86 lines not on this page · raw message

new file mode 100755
index 0000000..a3f3861
@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+import unittest
+import logging
+import tempfile
+
+from common import setup_sys_path, TestBase
+setup_sys_path()
+
+from oeqa.core.runner import OEStreamLogger
+
+class TestRunner(TestBase):
+    def test_stream_logger(self):
+        fp = tempfile.TemporaryFile(mode='w+')
+
+        logging.basicConfig(format='%(message)s', stream=fp)
+        logger = logging.getLogger()
+        logger.setLevel(logging.INFO)
+
+        oeSL = OEStreamLogger(logger)
+
+        lines = ['init', 'bigline_' * 65535, 'morebigline_' * 65535 * 4, 'end']
+        for line in lines:
+            oeSL.write(line)
+
+        fp.seek(0)
+        fp_lines = fp.readlines()
+        for i, fp_line in enumerate(fp_lines):
+            fp_line = fp_line.strip()
+            self.assertEqual(lines[i], fp_line)
+
+        fp.close()
+
+if __name__ == '__main__':
+    unittest.main()

new file mode 100755
index 0000000..9222dea
@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+
+# OpenEmbedded test tool
+#
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+import os
+import sys
+import argparse
+import importlib
+import logging
+
+scripts_path = os.path.dirname(os.path.realpath(__file__))
+lib_path = scripts_path + '/lib'
+sys.path = sys.path + [lib_path]
+import argparse_oe
+import scriptutils
+import scriptpath
+scriptpath.add_oe_lib_path()
+
+from oeqa.core.context import OETestContextExecutor
+
+logger = scriptutils.logger_create('oe-test')
+
+def _load_test_components(logger):
+    components = {}
+
+    for path in sys.path:
+        base_dir = os.path.join(path, 'oeqa')
+        if os.path.exists(base_dir) and os.path.isdir(base_dir):
+            for file in os.listdir(base_dir):
+                comp_name = file
+                comp_context = os.path.join(base_dir, file, 'context.py')
+                if os.path.exists(comp_context):
+                    comp_plugin = importlib.import_module('oeqa.%s.%s' % \
+                            (comp_name, 'context'))
+                    try:
+                        if not issubclass(comp_plugin._executor_class,
+                                OETestContextExecutor):
+                            raise TypeError("Component %s in %s, _executor_class "\
+                                "isn't derived from OETestContextExecutor."\
+                                % (comp_name, comp_context))
+
+                        components[comp_name] = comp_plugin._executor_class()
+                    except AttributeError:
+                        raise AttributeError("Component %s in %s don't have "\
+                                "_executor_class defined." % (comp_name, comp_context))
+
+    return components
+
+def main():
+    parser = argparse_oe.ArgumentParser(description="OpenEmbedded test tool",
+                                        add_help=False,
+                                        epilog="Use %(prog)s <subcommand> --help to get help on a specific command")
+    parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true')
+    parser.add_argument('-q', '--quiet', help='Print only errors', action='store_true')
+    global_args, unparsed_args = parser.parse_known_args()
+
+    # Help is added here rather than via add_help=True, as we don't want it to
+    # be handled by parse_known_args()
+    parser.add_argument('-h', '--help', action='help', default=argparse.SUPPRESS,
+                        help='show this help message and exit')
+
+    if global_args.debug:
+        logger.setLevel(logging.DEBUG)
+    elif global_args.quiet:
+        logger.setLevel(logging.ERROR)
+
+    components = _load_test_components(logger)
+
+    subparsers = parser.add_subparsers(dest="subparser_name", title='subcommands', metavar='<subcommand>')
+    subparsers.add_subparser_group('components', 'Test components')
+    subparsers.required = True
+    for comp_name in sorted(components.keys()):
+        comp = components[comp_name]
+        comp.register_commands(logger, subparsers)
+
+    try:
+        args = parser.parse_args(unparsed_args, namespace=global_args)
+        ret = args.func(logger, args)
+    except SystemExit as err:
+        if err.code != 0:
+            raise err
+        ret = err.code
+    except argparse_oe.ArgumentUsageError as ae:
+        parser.error_subcommand(ae.message, ae.subcommand)
+
+    return ret
+
+if __name__ == '__main__':
+    try:
+        ret = main()
+    except Exception:
+        ret = 1
+        import traceback
+        traceback.print_exc()
+    sys.exit(ret)

index d5caf53..c7d6db3 100644
@@ -148,7 +148,7 @@ class OETestContextExecutor(object):
 
     default_cases = [os.path.join(os.path.abspath(os.path.dirname(__file__)),
             'cases/example')]
-    default_test_data = os.path.join(default_cases[0], 'data.json')
+    default_data = os.path.join(default_cases[0], 'data.json')
 
     def register_commands(self, logger, subparsers):
         self.parser = subparsers.add_parser(self.name, help=self.help,

@@ -160,12 +160,12 @@ class OETestContextExecutor(object):
                 default=self.default_output_log,
                 help="results output log, default: %s" % self.default_output_log)
 
-        if self.default_test_data:
-            self.parser.add_argument('--test-data-file', action='store',
-                    default=self.default_test_data,
-                    help="data file to load, default: %s" % self.default_test_data)
+        if self.default_data:
+            self.parser.add_argument('--data-file', action='store',
+                    default=self.default_data,
+                    help="data file to load, default: %s" % self.default_data)
         else:
-            self.parser.add_argument('--test-data-file', action='store',
+            self.parser.add_argument('--data-file', action='store',
                     help="data file to load")
 
         if self.default_cases:

@@ -197,11 +197,11 @@ class OETestContextExecutor(object):
         self.tc_kwargs['run'] = {}
 
         self.tc_kwargs['init']['logger'] = self._setup_logger(logger, args)
-        if args.test_data_file:
-            self.tc_kwargs['init']['td'] = json.load(
-                    open(args.test_data_file, "r"))
+        if args.data_file:
+            self.tc_kwargs['init']['d'] = json.load(
+                    open(args.data_file, "r"))
         else:
-            self.tc_kwargs['init']['td'] = {}
+            self.tc_kwargs['init']['d'] = {}
 
         self.module_paths = args.CASES_PATHS
 

new file mode 100644
index 0000000..e69de29

new file mode 100644
index 0000000..21d6b16
@@ -0,0 +1 @@
+{"ARCH": "x86", "IMAGE": "core-image-minimal"}
\ No newline at end of file

new file mode 100644
index 0000000..8b404fe
@@ -0,0 +1,20 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+from oeqa.core.case import OETestCase
+from oeqa.core.decorator.depends import OETestDepends
+
+class OETestExample(OETestCase):
+    def test_example(self):
+        self.logger.info('IMAGE: %s' % self.d.get('IMAGE'))
+        self.assertEqual('core-image-minimal', self.d.get('IMAGE'))
+        self.logger.info('ARCH: %s' % self.d.get('ARCH'))
+        self.assertEqual('x86', self.d.get('ARCH'))
+
+class OETestExampleDepend(OETestCase):
+    @OETestDepends(['OETestExample.test_example'])
+    def test_example_depends(self):
+        pass
+
+    def test_example_no_depends(self):
+        pass

new file mode 100644
index 0000000..0c859fd
@@ -0,0 +1,38 @@
+= OEQA Framework =
+
+== Introduction ==
+
+This is the new OEQA framework the base clases of the framework
+are in this module oeqa/core the subsequent components needs to
+extend this classes.
+
+A new/unique runner was created called oe-test and is under scripts/
+oe-test, this new runner scans over oeqa module searching for test
+components that supports OETestContextExecutor implemented in context
+module (i.e. oeqa/core/context.py).
+
+For execute an example:
+
+$ source oe-init-build-env
+$ oe-test core
+
+For list supported components:
+
+$ oe-test -h
+
+== Create new Test component ==
+
+Usally for add a new Test component the developer needs to extend
+OETestContext/OETestContextExecutor in context.py and OETestCase in
+case.py.
+
+== How to run the testing of the OEQA framework ==
+
+Run all tests:
+
+$ PATH=$PATH:../../ python3 -m unittest discover -s tests
+
+Run some test:
+
+$ cd tests/
+$ ./test_data.py

index ee48950..db3bd6d 100644
@@ -1,3 +1,4 @@
+import json
 import oe.maketype
 
 def typed_value(key, d):

@@ -15,3 +16,30 @@ def typed_value(key, d):
         return oe.maketype.create(d.getVar(key, True) or '', var_type, **flags)
     except (TypeError, ValueError) as exc:
         bb.msg.fatal("Data", "%s: %s" % (key, str(exc)))
+
+def export2json(d, json_file, expand=True):
+    data2export = {}
+    keys2export = []
+
+    for key in d.keys():
+        if key.startswith("_"):
+            continue
+        elif key.startswith("BB"):
+            continue
+        elif key.startswith("B_pn"):
+            continue
+        elif key.startswith("do_"):
+            continue
+        elif d.getVarFlag(key, "func", True):
+            continue
+
+        keys2export.append(key)
+
+    for key in keys2export:
+        try:
+            data2export[key] = d.getVar(key, expand)
+        except bb.data_smart.ExpansionError:
+            data2export[key] = ''
+
+    with open(json_file, "w") as f:
+        json.dump(data2export, f, skipkeys=True, indent=4, sort_keys=True)

index 0c7ceea..5cfd2fe 100644
@@ -14,6 +14,9 @@ ROOTFS_POSTPROCESS_COMMAND += "rootfs_update_timestamp ; "
 # Tweak the mount options for rootfs in /etc/fstab if read-only-rootfs is enabled
 ROOTFS_POSTPROCESS_COMMAND += '${@bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs", "read_only_rootfs_hook; ", "",d)}'
 
+# Generates test data file with data store variables expanded in json format
+ROOTFS_POSTPROCESS_COMMAND += "write_image_test_data ; "
+
 # Write manifest
 IMAGE_MANIFEST = "${IMGDEPLOYDIR}/${IMAGE_NAME}.rootfs.manifest"
 ROOTFS_POSTUNINSTALL_COMMAND =+ "write_image_manifest ; "

@@ -278,3 +281,18 @@ rootfs_check_host_user_contaminated () {
 rootfs_sysroot_relativelinks () {
 	sysroot-relativelinks.py ${SDK_OUTPUT}/${SDKTARGETSYSROOT}
 }
+
+# Generated test data json file
+python write_image_test_data() {
+    from oe.data import export2json
+
+    testdata = "%s/%s.testdata.json" % (d.getVar('DEPLOY_DIR_IMAGE', True), d.getVar('IMAGE_NAME', True))
+    testdata_link = "%s/%s.testdata.json" % (d.getVar('DEPLOY_DIR_IMAGE', True), d.getVar('IMAGE_LINK_NAME', True))
+
+    bb.utils.mkdirhier(os.path.dirname(testdata))
+    export2json(d, testdata)
+
+    if os.path.lexists(testdata_link):
+       os.remove(testdata_link)
+    os.symlink(os.path.basename(testdata), testdata_link)
+}

index 220cde6..f0fc089 100644
@@ -73,6 +73,13 @@ python write_target_sdk_manifest () {
         output.write(format_pkg_list(pkgs, 'ver'))
 }
 
+python write_sdk_test_data() {
+    from oe.data import export2json
+    testdata = "%s/%s.testdata.json" % (d.getVar('SDKDEPLOYDIR', True), d.getVar('TOOLCHAIN_OUTPUTNAME', True))
+    bb.utils.mkdirhier(os.path.dirname(testdata))
+    export2json(d, testdata)
+}
+
 python write_host_sdk_manifest () {
     from oe.sdk import sdk_list_installed_packages
     from oe.utils import format_pkg_list

@@ -84,7 +91,7 @@ python write_host_sdk_manifest () {
         output.write(format_pkg_list(pkgs, 'ver'))
 }
 
-POPULATE_SDK_POST_TARGET_COMMAND_append = " write_target_sdk_manifest ; "
+POPULATE_SDK_POST_TARGET_COMMAND_append = " write_target_sdk_manifest ; write_sdk_test_data ; "
 POPULATE_SDK_POST_HOST_COMMAND_append = " write_host_sdk_manifest; "
 SDK_PACKAGING_COMMAND = "${@'${SDK_PACKAGING_FUNC};' if '${SDK_PACKAGING_FUNC}' else ''}"
 SDK_POSTPROCESS_COMMAND = " create_sdk_files; check_sdk_sysroots; tar_sdk; ${SDK_PACKAGING_COMMAND} "

index a47cb19..d2dbf20 100644
@@ -5,30 +5,30 @@ import unittest
 
 from oeqa.core.exception import OEQAMissingVariable
 
-def _validate_data_vars(d, data_vars, type_msg):
-    if data_vars:
-        for v in data_vars:
-            if not v in d:
+def _validate_td_vars(td, td_vars, type_msg):
+    if td_vars:
+        for v in td_vars:
+            if not v in td:
                 raise OEQAMissingVariable("Test %s need %s variable but"\
-                        " isn't into d" % (type_msg, v))
+                        " isn't into td" % (type_msg, v))
 
 class OETestCase(unittest.TestCase):
     # TestContext and Logger instance set by OETestLoader.
     tc = None
     logger = None
 
-    # d has all the variables needed by the test cases
+    # td has all the variables needed by the test cases
     # is the same across all the test cases.
-    d = None
+    td = None
 
-    # data_vars has the variables needed by a test class
-    # or test case instance, if some var isn't into d a
+    # td_vars has the variables needed by a test class
+    # or test case instance, if some var isn't into td a
     # OEMissingVariable exception is raised
-    data_vars = None
+    td_vars = None
 
     @classmethod
     def _oeSetUpClass(clss):
-        _validate_data_vars(clss.d, clss.data_vars, "class")
+        _validate_td_vars(clss.td, clss.td_vars, "class")
         clss.setUpClassMethod()
 
     @classmethod

index 8b404fe..11cf380 100644
@@ -6,10 +6,10 @@ from oeqa.core.decorator.depends import OETestDepends
 
 class OETestExample(OETestCase):
     def test_example(self):
-        self.logger.info('IMAGE: %s' % self.d.get('IMAGE'))
-        self.assertEqual('core-image-minimal', self.d.get('IMAGE'))
-        self.logger.info('ARCH: %s' % self.d.get('ARCH'))
-        self.assertEqual('x86', self.d.get('ARCH'))
+        self.logger.info('IMAGE: %s' % self.td.get('IMAGE'))
+        self.assertEqual('core-image-minimal', self.td.get('IMAGE'))
+        self.logger.info('ARCH: %s' % self.td.get('ARCH'))
+        self.assertEqual('x86', self.td.get('ARCH'))
 
 class OETestExampleDepend(OETestCase):
     @OETestDepends(['OETestExample.test_example'])

index c7d6db3..316f90f 100644
@@ -20,11 +20,11 @@ class OETestContext(object):
     files_dir = os.path.abspath(os.path.join(os.path.dirname(
         os.path.abspath(__file__)), "../files"))
 
-    def __init__(self, d=None, logger=None):
-        if not type(d) is dict:
-            raise TypeError("d isn't dictionary type")
+    def __init__(self, td=None, logger=None):
+        if not type(td) is dict:
+            raise TypeError("td isn't dictionary type")
 
-        self.d = d
+        self.td = td
         self.logger = logger
         self._registry = {}
         self._registry['cases'] = collections.OrderedDict()

@@ -148,7 +148,7 @@ class OETestContextExecutor(object):
 
     default_cases = [os.path.join(os.path.abspath(os.path.dirname(__file__)),
             'cases/example')]
-    default_data = os.path.join(default_cases[0], 'data.json')
+    default_test_data = os.path.join(default_cases[0], 'data.json')
 
     def register_commands(self, logger, subparsers):
         self.parser = subparsers.add_parser(self.name, help=self.help,

@@ -160,12 +160,12 @@ class OETestContextExecutor(object):
                 default=self.default_output_log,
                 help="results output log, default: %s" % self.default_output_log)
 
-        if self.default_data:
-            self.parser.add_argument('--data-file', action='store',
-                    default=self.default_data,
-                    help="data file to load, default: %s" % self.default_data)
+        if self.default_test_data:
+            self.parser.add_argument('--test-data-file', action='store',
+                    default=self.default_test_data,
+                    help="data file to load, default: %s" % self.default_test_data)
         else:
-            self.parser.add_argument('--data-file', action='store',
+            self.parser.add_argument('--test-data-file', action='store',
                     help="data file to load")
 
         if self.default_cases:

@@ -197,11 +197,11 @@ class OETestContextExecutor(object):
         self.tc_kwargs['run'] = {}
 
         self.tc_kwargs['init']['logger'] = self._setup_logger(logger, args)
-        if args.data_file:
-            self.tc_kwargs['init']['d'] = json.load(
-                    open(args.data_file, "r"))
+        if args.test_data_file:
+            self.tc_kwargs['init']['td'] = json.load(
+                    open(args.test_data_file, "r"))
         else:
-            self.tc_kwargs['init']['d'] = {}
+            self.tc_kwargs['init']['td'] = {}
 
         self.module_paths = args.CASES_PATHS
 

index 51ef6fe..73cca88 100644
@@ -20,17 +20,17 @@ class skipIfDataVar(OETestDecorator):
     def setUpDecorator(self):
         msg = 'Checking if %r value is %r to skip test' % (self.var, self.value)
         self.logger.debug(msg)
-        if self.case.tc.d.get(self.var) == self.value:
+        if self.case.td.get(self.var) == self.value:
             self.case.skipTest(self.msg)
 
 @registerDecorator
 class OETestDataDepends(OETestDecorator):
-    attrs = ('data_depends',)
+    attrs = ('td_depends',)
 
     def setUpDecorator(self):
-        for v in self.data_depends:
+        for v in self.td_depends:
             try:
-                value = self.case.d[v]
+                value = self.case.td[v]
             except KeyError:
                 raise OEQAMissingVariable("Test case need %s variable but"\
-                        " isn't into d" % v)
+                        " isn't into td" % v)

index 94f71ba..c73ef9a 100644
@@ -66,7 +66,7 @@ class OETestLoader(unittest.TestLoader):
     def _patchCaseClass(self, testCaseClass):
         # Adds custom attributes to the OETestCase class
         setattr(testCaseClass, 'tc', self.tc)
-        setattr(testCaseClass, 'd', self.tc.d)
+        setattr(testCaseClass, 'td', self.tc.td)
         setattr(testCaseClass, 'logger', self.tc.logger)
 
     def _validateFilters(self, filters, decorator_filters):

index 4d8fad0..88003a6 100644
@@ -11,9 +11,9 @@ class DataTest(OETestCase):
     @OETestDataDepends(['MACHINE',])
     @OETestTag('dataTestOk')
     def testDataOk(self):
-        self.assertEqual(self.d.get('IMAGE'), 'core-image-minimal')
-        self.assertEqual(self.d.get('ARCH'), 'x86')
-        self.assertEqual(self.d.get('MACHINE'), 'qemuarm')
+        self.assertEqual(self.td.get('IMAGE'), 'core-image-minimal')
+        self.assertEqual(self.td.get('ARCH'), 'x86')
+        self.assertEqual(self.td.get('MACHINE'), 'qemuarm')
 
     @OETestTag('dataTestFail')
     def testDataFail(self):

index cb06523..a21caad 100644
@@ -12,3 +12,8 @@ def findFile(file_name, directory):
         if file_name in f:
             return os.path.join(r, file_name)
     return None
+
+def remove_safe(path):
+    if os.path.exists(path):
+        os.remove(path)
+

similarity index 100%
rename from meta/lib/oeqa/runtime/files/test.cpp
rename to meta/lib/oeqa/files/test.cpp

similarity index 100%
rename from meta/lib/oeqa/runtime/files/test.pl
rename to meta/lib/oeqa/files/test.pl

similarity index 100%
rename from meta/lib/oeqa/runtime/files/test.py
rename to meta/lib/oeqa/files/test.py

similarity index 100%
rename from meta/lib/oeqa/sdk/__init__.py
rename to meta/lib/oeqa/sdk/cases/__init__.py

similarity index 100%
rename from meta/lib/oeqa/sdk/buildcvs.py
rename to meta/lib/oeqa/sdk/cases/buildcvs.py

similarity index 100%
rename from meta/lib/oeqa/sdk/buildgalculator.py
rename to meta/lib/oeqa/sdk/cases/buildgalculator.py

similarity index 100%
rename from meta/lib/oeqa/sdk/buildiptables.py
rename to meta/lib/oeqa/sdk/cases/buildiptables.py

similarity index 100%
rename from meta/lib/oeqa/sdk/gcc.py
rename to meta/lib/oeqa/sdk/cases/gcc.py

similarity index 100%
rename from meta/lib/oeqa/sdk/perl.py
rename to meta/lib/oeqa/sdk/cases/perl.py

similarity index 100%
rename from meta/lib/oeqa/sdk/python.py
rename to meta/lib/oeqa/sdk/cases/python.py

similarity index 100%
rename from meta/lib/oeqa/runtime/files/testsdkmakefile
rename to meta/lib/oeqa/sdk/files/testsdkmakefile

index 95d3bf7..4a98b0f1 100644
@@ -145,20 +145,6 @@ class OETestCalledProcessError(subprocess.CalledProcessError):
 
 subprocess.CalledProcessError = OETestCalledProcessError
 
-class oeSDKTest(oeTest):
-    def __init__(self, methodName='runTest'):
-        self.sdktestdir = oeSDKTest.tc.sdktestdir
-        super(oeSDKTest, self).__init__(methodName)
-
-    @classmethod
-    def hasHostPackage(self, pkg):
-        if re.search(pkg, oeTest.tc.hostpkgmanifest):
-            return True
-        return False
-
-    def _run(self, cmd):
-        return subprocess.check_output(". %s > /dev/null; %s;" % (self.tc.sdkenv, cmd), shell=True, stderr=subprocess.STDOUT).decode("utf-8")
-
 class oeSDKExtTest(oeSDKTest):
     def _run(self, cmd):
         # extensible sdk shows a warning if found bitbake in the path

@@ -657,43 +643,6 @@ class ExportTestContext(RuntimeTestContext):
         pkg_dir = os.path.join(export_dir, extracted_dir)
         super(ExportTestContext, self).install_uninstall_packages(test_id, pkg_dir, install)
 
-class SDKTestContext(TestContext):
-    def __init__(self, d, sdktestdir, sdkenv, tcname, *args):
-        super(SDKTestContext, self).__init__(d)
-
-        self.sdktestdir = sdktestdir
-        self.sdkenv = sdkenv
-        self.tcname = tcname
-
-        if not hasattr(self, 'target_manifest'):
-            self.target_manifest = d.getVar("SDK_TARGET_MANIFEST", True)
-        try:
-            self.pkgmanifest = {}
-            with open(self.target_manifest) as f:
-                for line in f:
-                    (pkg, arch, version) = line.strip().split()
-                    self.pkgmanifest[pkg] = (version, arch)
-        except IOError as e:
-            bb.fatal("No package manifest file found. Did you build the sdk image?\n%s" % e)
-
-        if not hasattr(self, 'host_manifest'):
-            self.host_manifest = d.getVar("SDK_HOST_MANIFEST", True)
-        try:
-            with open(self.host_manifest) as f:
-                self.hostpkgmanifest = f.read()
-        except IOError as e:
-            bb.fatal("No host package manifest file found. Did you build the sdk image?\n%s" % e)
-
-    def _get_test_namespace(self):
-        return "sdk"
-
-    def _get_test_suites(self):
-        return (self.d.getVar("TEST_SUITES_SDK", True) or "auto").split()
-
-    def _get_test_suites_required(self):
-        return [t for t in (self.d.getVar("TEST_SUITES_SDK", True) or \
-                "auto").split() if t != "auto"]
-
 class SDKExtTestContext(SDKTestContext):
     def __init__(self, d, sdktestdir, sdkenv, tcname, *args):
         self.target_manifest = d.getVar("SDK_EXT_TARGET_MANIFEST", True)

new file mode 100644
index 0000000..e69de29

new file mode 100644
index 0000000..782db8b
@@ -0,0 +1,12 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+import subprocess
+
+from oeqa.core.case import OETestCase
+
+class OESDKTestCase(OETestCase):
+    def _run(self, cmd):
+        return subprocess.check_output(". %s > /dev/null; %s;" % \
+                (self.tc.sdk_env, cmd), shell=True,
+                stderr=subprocess.STDOUT).decode("utf-8")

133 lines not on this page · raw message

index 06b4c50..959fb38 100644
@@ -50,30 +50,74 @@ def run_test_context(CTestContext, d, testdir, tcname, pn, *args):
 
 def testsdk_main(d):
     import os
-    import oeqa.sdk
     import subprocess
-    from oeqa.oetest import SDKTestContext
+    import json
+    import logging
+
+    from bb.utils import export_proxies
+    from oeqa.core.runner import OEStreamLogger
+    from oeqa.sdk.context import OESDKTestContext, OESDKTestContextExecutor
 
     pn = d.getVar("PN", True)
-    bb.utils.mkdirhier(d.getVar("TEST_LOG_DIR", True))
+    logger = logging.getLogger("BitBake")
+
+    # sdk use network for download projects for build
+    export_proxies(d)
+
+    test_log_dir = d.getVar("TEST_LOG_DIR", True)
+
+    bb.utils.mkdirhier(test_log_dir)
 
     tcname = d.expand("${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.sh")
     if not os.path.exists(tcname):
         bb.fatal("The toolchain %s is not built. Build it before running the tests: 'bitbake <image> -c populate_sdk' ." % tcname)
 
-    sdktestdir = d.expand("${WORKDIR}/testimage-sdk/")
-    bb.utils.remove(sdktestdir, True)
-    bb.utils.mkdirhier(sdktestdir)
+    tdname = d.expand("${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.testdata.json")
+    test_data = json.load(open(tdname, "r"))
+    test_data['TEST_LOG_DIR'] = test_log_dir
+
+    target_pkg_manifest = OESDKTestContextExecutor._load_manifest(
+        d.expand("${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.target.manifest"))
+    host_pkg_manifest = OESDKTestContextExecutor._load_manifest(
+        d.expand("${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.host.manifest"))
+
+    sdk_dir = d.expand("${WORKDIR}/testimage-sdk/")
+    bb.utils.remove(sdk_dir, True)
+    bb.utils.mkdirhier(sdk_dir)
     try:
-        subprocess.check_output("cd %s; %s <<EOF\n./tc\nY\nEOF" % (sdktestdir, tcname), shell=True)
+        subprocess.check_output("cd %s; %s <<EOF\n./\nY\nEOF" % (sdk_dir, tcname), shell=True)
     except subprocess.CalledProcessError as e:
         bb.fatal("Couldn't install the SDK:\n%s" % e.output.decode("utf-8"))
 
-    try:
-        run_test_context(SDKTestContext, d, sdktestdir, tcname, pn)
-    finally:
-        bb.utils.remove(sdktestdir, True)
+    fail = False
+    sdk_envs = OESDKTestContextExecutor._get_sdk_environs(sdk_dir)
+    for s in sdk_envs:
+        sdk_env = sdk_envs[s]
+        bb.plain("SDK testing environment: %s" % s)
+        tc = OESDKTestContext(td=test_data, logger=logger, sdk_dir=sdk_dir,
+            sdk_env=sdk_env, target_pkg_manifest=target_pkg_manifest,
+            host_pkg_manifest=host_pkg_manifest)
 
+        try:
+            tc.loadTests(OESDKTestContextExecutor.default_cases)
+        except Exception as e:
+            import traceback
+            bb.fatal("Loading tests failed:\n%s" % traceback.format_exc())
+
+        result = tc.runTests()
+
+        component = "%s %s" % (pn, OESDKTestContextExecutor.name)
+        context_msg = "%s:%s" % (os.path.basename(tcname), os.path.basename(sdk_env))
+
+        tc.logSummary(result, component, context_msg)
+        tc.logDetails()
+
+        if not result.wasSuccessful():
+            fail = True
+
+    if fail:
+        bb.fatal("%s - FAILED - check the task log and the commands log" % pn)
+  
 testsdk_main[vardepsexclude] =+ "BB_ORIGENV"
 
 python do_testsdk() {

@@ -94,7 +138,6 @@ def testsdkext_main(d):
     from oeqa.oetest import SDKTestContext, SDKExtTestContext
     from oeqa.utils import avoid_paths_in_environ
 
-
     # extensible sdk use network
     export_proxies(d)
 

new file mode 100644
index 0000000..e69de29

new file mode 100644
index 0000000..138b5ef
@@ -0,0 +1,33 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+from oeqa.utils.buildproject import BuildProject
+
+class TargetBuildProject(BuildProject):
+
+    def __init__(self, target, d, uri, foldername=None):
+        self.target = target
+        self.targetdir = "~/"
+        BuildProject.__init__(self, d, uri, foldername, tmpdir="/tmp")
+
+    def download_archive(self):
+
+        self._download_archive()
+
+        (status, output) = self.target.copy_to(self.localarchive, self.targetdir)
+        if status != 0:
+            raise Exception("Failed to copy archive to target, output: %s" % output)
+
+        (status, output) = self.target.run('tar xf %s%s -C %s' % (self.targetdir, self.archive, self.targetdir))
+        if status != 0:
+            raise Exception("Failed to extract archive, output: %s" % output)
+
+        #Change targetdir to project folder
+        self.targetdir = self.targetdir + self.fname
+
+    # The timeout parameter of target.run is set to 0 to make the ssh command
+    # run with no timeout.
+    def _run(self, cmd):
+        return self.target.run(cmd, 0)[0]
+
+

new file mode 100644
index 0000000..e69de29

45 lines not on this page · raw message

74 lines not on this page · raw message

index 138b5ef..006d4d4 100644
@@ -5,13 +5,13 @@ from oeqa.utils.buildproject import BuildProject
 
 class TargetBuildProject(BuildProject):
 
-    def __init__(self, target, d, uri, foldername=None):
+    def __init__(self, target, uri, foldername=None, dl_dir=None):
         self.target = target
         self.targetdir = "~/"
-        BuildProject.__init__(self, d, uri, foldername, tmpdir="/tmp")
+        BuildProject.__init__(self, uri, foldername, tmpdir="/tmp",
+                dl_dir=dl_dir)
 
     def download_archive(self):
-
         self._download_archive()
 
         (status, output) = self.target.copy_to(self.localarchive, self.targetdir)

index 1aa8a69..cc34e0c 100644
@@ -7,17 +7,17 @@ import subprocess
 from oeqa.utils.buildproject import BuildProject
 
 class SDKBuildProject(BuildProject):
-
-    def __init__(self, testpath, sdkenv, d, uri, foldername=None):
+    def __init__(self, testpath, sdkenv, uri, testlogdir, builddatetime,
+            foldername=None, dl_dir=None):
         self.sdkenv = sdkenv
         self.testdir = testpath
         self.targetdir = testpath
-        bb.utils.mkdirhier(testpath)
-        self.datetime = d.getVar('DATETIME', True)
-        self.testlogdir = d.getVar("TEST_LOG_DIR", True)
-        bb.utils.mkdirhier(self.testlogdir)
+        os.makedirs(testpath, exist_ok=True)
+        self.datetime = builddatetime
+        self.testlogdir = testlogdir
+        os.makedirs(self.testlogdir, exist_ok=True)
         self.logfile = os.path.join(self.testlogdir, "sdk_target_log.%s" % self.datetime)
-        BuildProject.__init__(self, d, uri, foldername, tmpdir=testpath)
+        BuildProject.__init__(self, uri, foldername, tmpdir=testpath, dl_dir=dl_dir)
 
     def download_archive(self):
 

index 0e1ed8a..386a927 100644
@@ -6,17 +6,17 @@
 
 import os
 import re
-import bb.utils
 import subprocess
+import shutil
+
 from abc import ABCMeta, abstractmethod
 
 class BuildProject(metaclass=ABCMeta):
-
-    def __init__(self, d, uri, foldername=None, tmpdir="/tmp/"):
-        self.d = d
+    def __init__(self, uri, foldername=None, tmpdir="/tmp/", dl_dir=None):
         self.uri = uri
         self.archive = os.path.basename(uri)
         self.localarchive = os.path.join(tmpdir,self.archive)
+        self.dl_dir = dl_dir
         if foldername:
             self.fname = foldername
         else:

@@ -24,27 +24,11 @@ class BuildProject(metaclass=ABCMeta):
 
     # Download self.archive to self.localarchive
     def _download_archive(self):
-
-        dl_dir = self.d.getVar("DL_DIR", True)
-        if dl_dir and os.path.exists(os.path.join(dl_dir, self.archive)):
-            bb.utils.copyfile(os.path.join(dl_dir, self.archive), self.localarchive)
+        if self.dl_dir and os.path.exists(os.path.join(self.dl_dir, self.archive)):
+            shutil.copyfile(os.path.join(self.dl_dir, self.archive), self.localarchive)
             return
 
-        exportvars = ['HTTP_PROXY', 'http_proxy',
-                      'HTTPS_PROXY', 'https_proxy',
-                      'FTP_PROXY', 'ftp_proxy',
-                      'FTPS_PROXY', 'ftps_proxy',
-                      'NO_PROXY', 'no_proxy',
-                      'ALL_PROXY', 'all_proxy',
-                      'SOCKS5_USER', 'SOCKS5_PASSWD']
-
-        cmd = ''
-        for var in exportvars:
-            val = self.d.getVar(var, True)
-            if val:
-                cmd = 'export ' + var + '=\"%s\"; %s' % (val, cmd)
-
-        cmd = cmd + "wget -O %s %s" % (self.localarchive, self.uri)
+        cmd = "wget -O %s %s" % (self.localarchive, self.uri)
         subprocess.check_call(cmd, shell=True)
 
     # This method should provide a way to run a command in the desired environment.

@@ -66,4 +50,3 @@ class BuildProject(metaclass=ABCMeta):
     def clean(self):
         self._run('rm -rf %s' % self.targetdir)
         subprocess.call('rm -f %s' % self.localarchive, shell=True)
-        pass

deleted file mode 100644
index 4cf3fa7..0000000
@@ -1,3 +0,0 @@
-# Enable other layers to have tests in the same named directory
-from pkgutil import extend_path
-__path__ = extend_path(__path__, __name__)

index c7146fa..ee7fb73 100644
@@ -1,13 +1,16 @@
-from oeqa.oetest import oeSDKTest, skipModule
-from oeqa.utils.decorators import *
-from oeqa.utils.targetbuild import SDKBuildProject
+from oeqa.sdk.case import OESDKTestCase
+from oeqa.sdk.utils.sdkbuildproject import SDKBuildProject
 
-class BuildCvsTest(oeSDKTest):
+class BuildCvsTest(OESDKTestCase):
+    td_vars = ['TEST_LOG_DIR', 'DATETIME']
 
     @classmethod
     def setUpClass(self):
-        self.project = SDKBuildProject(oeSDKTest.tc.sdktestdir + "/cvs/", oeSDKTest.tc.sdkenv, oeSDKTest.tc.d,
-                        "http://ftp.gnu.org/non-gnu/cvs/source/feature/1.12.13/cvs-1.12.13.tar.bz2")
+        dl_dir = self.td.get('DL_DIR', None)
+
+        self.project = SDKBuildProject(self.tc.sdk_dir + "/cvs/", self.tc.sdk_env,
+                        "http://ftp.gnu.org/non-gnu/cvs/source/feature/1.12.13/cvs-1.12.13.tar.bz2",
+                        self.td['TEST_LOG_DIR'], self.td['DATETIME'], dl_dir=dl_dir)
         self.project.download_archive()
 
     def test_cvs(self):

index dc2fa9c..d2c1189 100644
@@ -1,17 +1,25 @@
-from oeqa.oetest import oeSDKTest, skipModule
-from oeqa.utils.decorators import *
-from oeqa.utils.targetbuild import SDKBuildProject
+import unittest
 
-def setUpModule():
-    if not (oeSDKTest.hasPackage("gtk+3") or oeSDKTest.hasPackage("libgtk-3.0")):
-        skipModule("Image doesn't have gtk+3 in manifest")
+from oeqa.sdk.case import OESDKTestCase
+from oeqa.sdk.utils.sdkbuildproject import SDKBuildProject
+
+class GalculatorTest(OESDKTestCase):
+    td_vars = ['TEST_LOG_DIR', 'DATETIME']
+
+    @classmethod
+    def setUpClass(self):
+        if not (self.tc.hasTargetPackage("gtk+3") or\
+                self.tc.hasTargetPackage("libgtk-3.0")):
+            raise unittest.SkipTest("%s class: SDK don't support gtk+3" % self.__name__)
 
-class GalculatorTest(oeSDKTest):
     def test_galculator(self):
+        dl_dir = self.td.get('DL_DIR', None)
+        project = None
         try:
-            project = SDKBuildProject(oeSDKTest.tc.sdktestdir + "/galculator/",
-                                      oeSDKTest.tc.sdkenv, oeSDKTest.tc.d,
-                                      "http://galculator.mnim.org/downloads/galculator-2.1.4.tar.bz2")
+            project = SDKBuildProject(self.tc.sdk_dir + "/galculator/",
+                                      self.tc.sdk_env,
+                                      "http://galculator.mnim.org/downloads/galculator-2.1.4.tar.bz2",
+                                      self.td['TEST_LOG_DIR'], self.td['DATETIME'], dl_dir=dl_dir)
 
             project.download_archive()
 

index f0cb8a4..a50fb5d 100644
@@ -1,14 +1,16 @@
-from oeqa.oetest import oeSDKTest
-from oeqa.utils.decorators import *
-from oeqa.utils.targetbuild import SDKBuildProject
+from oeqa.sdk.case import OESDKTestCase
+from oeqa.sdk.utils.sdkbuildproject import SDKBuildProject
 
-
-class BuildIptablesTest(oeSDKTest):
+class BuildIptablesTest(OESDKTestCase):
+    td_vars = ['TEST_LOG_DIR', 'DATETIME']
 
     @classmethod
     def setUpClass(self):
-        self.project = SDKBuildProject(oeSDKTest.tc.sdktestdir + "/iptables/", oeSDKTest.tc.sdkenv, oeSDKTest.tc.d,
-                        "http://downloads.yoctoproject.org/mirror/sources/iptables-1.4.13.tar.bz2")
+        dl_dir = self.td.get('DL_DIR', None)
+
+        self.project = SDKBuildProject(self.tc.sdk_dir + "/iptables/", self.tc.sdk_env, 
+                        "http://downloads.yoctoproject.org/mirror/sources/iptables-1.4.13.tar.bz2",
+                        self.td['TEST_LOG_DIR'], self.td['DATETIME'], dl_dir=dl_dir)
         self.project.download_archive()
 
     def test_iptables(self):

60 lines not on this page · raw message

index 45f422e..e1bded2 100644
@@ -1,28 +1,27 @@
-import unittest
 import os
 import shutil
-from oeqa.oetest import oeSDKTest, skipModule
-from oeqa.utils.decorators import *
-
-def setUpModule():
-    if not oeSDKTest.hasHostPackage("nativesdk-perl"):
-        skipModule("No perl package in the SDK")
-
+import unittest
 
-class PerlTest(oeSDKTest):
+from oeqa.core.utils.path import remove_safe
+from oeqa.sdk.case import OESDKTestCase
 
+class PerlTest(OESDKTestCase):
     @classmethod
     def setUpClass(self):
+        if not self.tc.hasHostPackage("nativesdk-perl"):
+            raise unittest.SkipTest("No perl package in the SDK")
+
         for f in ['test.pl']:
-            shutil.copyfile(os.path.join(self.tc.filesdir, f), self.tc.sdktestdir + f)
-        self.testfile = self.tc.sdktestdir + "test.pl"
+            shutil.copyfile(os.path.join(self.tc.files_dir, f),
+                    os.path.join(self.tc.sdk_dir, f))
+        self.testfile = os.path.join(self.tc.sdk_dir, "test.pl")
 
     def test_perl_exists(self):
         self._run('which perl')
 
     def test_perl_works(self):
-        self._run('perl %s/test.pl' % self.tc.sdktestdir)
+        self._run('perl %s' % self.testfile)
 
     @classmethod
     def tearDownClass(self):
-        bb.utils.remove("%s/test.pl" % self.tc.sdktestdir)
+        remove_safe(self.testfile)

index 896fab4..94a296f 100644
@@ -1,26 +1,25 @@
-import unittest
 import os
 import shutil
-from oeqa.oetest import oeSDKTest, skipModule
-from oeqa.utils.decorators import *
-
-def setUpModule():
-    if not oeSDKTest.hasHostPackage("nativesdk-python"):
-        skipModule("No python package in the SDK")
-
+import unittest
 
-class PythonTest(oeSDKTest):
+from oeqa.core.utils.path import remove_safe
+from oeqa.sdk.case import OESDKTestCase
 
+class PythonTest(OESDKTestCase):
     @classmethod
     def setUpClass(self):
+        if not self.tc.hasHostPackage("nativesdk-python"):
+            raise unittest.SkipTest("No python package in the SDK")
+
         for f in ['test.py']:
-            shutil.copyfile(os.path.join(self.tc.filesdir, f), self.tc.sdktestdir + f)
+            shutil.copyfile(os.path.join(self.tc.files_dir, f),
+                   os.path.join(self.tc.sdk_dir, f))
 
     def test_python_exists(self):
         self._run('which python')
 
     def test_python_stdout(self):
-        output = self._run('python %s/test.py' % self.tc.sdktestdir)
+        output = self._run('python %s/test.py' % self.tc.sdk_dir)
         self.assertEqual(output.strip(), "the value of a is 0.01", msg="Incorrect output: %s" % output)
 
     def test_python_testfile(self):

@@ -28,5 +27,5 @@ class PythonTest(oeSDKTest):
 
     @classmethod
     def tearDownClass(self):
-        bb.utils.remove("%s/test.py" % self.tc.sdktestdir)
-        bb.utils.remove("/tmp/testfile.python")
+        remove_safe("%s/test.py" % self.tc.sdk_dir)
+        remove_safe("/tmp/testfile.python")

index 959fb38..4c4df10 100644
@@ -14,7 +14,6 @@
 #
 # where "<image-name>" is an image like core-image-sato.
 
-TEST_LOG_DIR ?= "${WORKDIR}/testimage"
 TESTSDKLOCK = "${TMPDIR}/testsdk.lock"
 
 def run_test_context(CTestContext, d, testdir, tcname, pn, *args):

@@ -64,17 +63,12 @@ def testsdk_main(d):
     # sdk use network for download projects for build
     export_proxies(d)
 
-    test_log_dir = d.getVar("TEST_LOG_DIR", True)
-
-    bb.utils.mkdirhier(test_log_dir)
-
     tcname = d.expand("${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.sh")
     if not os.path.exists(tcname):
         bb.fatal("The toolchain %s is not built. Build it before running the tests: 'bitbake <image> -c populate_sdk' ." % tcname)
 
     tdname = d.expand("${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.testdata.json")
     test_data = json.load(open(tdname, "r"))
-    test_data['TEST_LOG_DIR'] = test_log_dir
 
     target_pkg_manifest = OESDKTestContextExecutor._load_manifest(
         d.expand("${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.target.manifest"))

index ee7fb73..6222a8e 100644
@@ -2,7 +2,7 @@ from oeqa.sdk.case import OESDKTestCase
 from oeqa.sdk.utils.sdkbuildproject import SDKBuildProject
 
 class BuildCvsTest(OESDKTestCase):
-    td_vars = ['TEST_LOG_DIR', 'DATETIME']
+    td_vars = ['DATETIME']
 
     @classmethod
     def setUpClass(self):

@@ -10,7 +10,7 @@ class BuildCvsTest(OESDKTestCase):
 
         self.project = SDKBuildProject(self.tc.sdk_dir + "/cvs/", self.tc.sdk_env,
                         "http://ftp.gnu.org/non-gnu/cvs/source/feature/1.12.13/cvs-1.12.13.tar.bz2",
-                        self.td['TEST_LOG_DIR'], self.td['DATETIME'], dl_dir=dl_dir)
+                        self.tc.sdk_dir, self.td['DATETIME'], dl_dir=dl_dir)
         self.project.download_archive()
 
     def test_cvs(self):

index d2c1189..bdc8b6a 100644
@@ -4,7 +4,7 @@ from oeqa.sdk.case import OESDKTestCase
 from oeqa.sdk.utils.sdkbuildproject import SDKBuildProject
 
 class GalculatorTest(OESDKTestCase):
-    td_vars = ['TEST_LOG_DIR', 'DATETIME']
+    td_vars = ['DATETIME']
 
     @classmethod
     def setUpClass(self):

@@ -19,7 +19,7 @@ class GalculatorTest(OESDKTestCase):
             project = SDKBuildProject(self.tc.sdk_dir + "/galculator/",
                                       self.tc.sdk_env,
                                       "http://galculator.mnim.org/downloads/galculator-2.1.4.tar.bz2",
-                                      self.td['TEST_LOG_DIR'], self.td['DATETIME'], dl_dir=dl_dir)
+                                      self.tc.sdk_dir, self.td['DATETIME'], dl_dir=dl_dir)
 
             project.download_archive()
 

index a50fb5d..532b5de 100644
@@ -2,7 +2,7 @@ from oeqa.sdk.case import OESDKTestCase
 from oeqa.sdk.utils.sdkbuildproject import SDKBuildProject
 
 class BuildIptablesTest(OESDKTestCase):
-    td_vars = ['TEST_LOG_DIR', 'DATETIME']
+    td_vars = ['DATETIME']
 
     @classmethod
     def setUpClass(self):

@@ -10,7 +10,7 @@ class BuildIptablesTest(OESDKTestCase):
 
         self.project = SDKBuildProject(self.tc.sdk_dir + "/iptables/", self.tc.sdk_env, 
                         "http://downloads.yoctoproject.org/mirror/sources/iptables-1.4.13.tar.bz2",
-                        self.td['TEST_LOG_DIR'], self.td['DATETIME'], dl_dir=dl_dir)
+                        self.tc.sdk_dir, self.td['DATETIME'], dl_dir=dl_dir)
         self.project.download_archive()
 
     def test_iptables(self):

index 4cf3fa7..e69de29 100644
@@ -1,3 +0,0 @@
-# Enable other layers to have tests in the same named directory
-from pkgutil import extend_path
-__path__ = extend_path(__path__, __name__)

similarity index 100%
rename from meta/lib/oeqa/sdkext/devtool.py
rename to meta/lib/oeqa/sdkext/cases/devtool.py

similarity index 100%
rename from meta/lib/oeqa/sdkext/sdk_update.py
rename to meta/lib/oeqa/sdkext/cases/sdk_update.py

index 4c4df10..24529ca 100644
@@ -16,37 +16,6 @@
 
 TESTSDKLOCK = "${TMPDIR}/testsdk.lock"
 
-def run_test_context(CTestContext, d, testdir, tcname, pn, *args):
-    import glob
-    import time
-
-    targets = glob.glob(d.expand(testdir + "/tc/environment-setup-*"))
-    for sdkenv in targets:
-        bb.plain("Testing %s" % sdkenv)
-        tc = CTestContext(d, testdir, sdkenv, tcname, args)
-
-        # this is a dummy load of tests
-        # we are doing that to find compile errors in the tests themselves
-        # before booting the image
-        try:
-            tc.loadTests()
-        except Exception as e:
-            import traceback
-            bb.fatal("Loading tests failed:\n%s" % traceback.format_exc())
-
-        starttime = time.time()
-        result = tc.runTests()
-        stoptime = time.time()
-        if result.wasSuccessful():
-            bb.plain("%s SDK(%s):%s - Ran %d test%s in %.3fs" % (pn, os.path.basename(tcname), os.path.basename(sdkenv),result.testsRun, result.testsRun != 1 and "s" or "", stoptime - starttime))
-            msg = "%s - OK - All required tests passed" % pn
-            skipped = len(result.skipped)
-            if skipped:
-                msg += " (skipped=%d)" % skipped
-            bb.plain(msg)
-        else:
-            bb.fatal("%s - FAILED - check the task log and the commands log" % pn)
-
 def testsdk_main(d):
     import os
     import subprocess

@@ -121,16 +90,20 @@ addtask testsdk
 do_testsdk[nostamp] = "1"
 do_testsdk[lockfiles] += "${TESTSDKLOCK}"
 
-TEST_LOG_SDKEXT_DIR ?= "${WORKDIR}/testsdkext"
 TESTSDKEXTLOCK = "${TMPDIR}/testsdkext.lock"
 
 def testsdkext_main(d):
     import os
-    import oeqa.sdkext
+    import json
     import subprocess
+    import logging
+
     from bb.utils import export_proxies
-    from oeqa.oetest import SDKTestContext, SDKExtTestContext
     from oeqa.utils import avoid_paths_in_environ
+    from oeqa.sdk.context import OESDKExtTestContext, OESDKExtTestContextExecutor
+
+    pn = d.getVar("PN", True)
+    logger = logging.getLogger("BitBake")
 
     # extensible sdk use network
     export_proxies(d)

@@ -141,20 +114,24 @@ def testsdkext_main(d):
                       d.getVar('BASE_WORKDIR', True)]
     os.environ['PATH'] = avoid_paths_in_environ(paths_to_avoid)
 
-    pn = d.getVar("PN", True)
-    bb.utils.mkdirhier(d.getVar("TEST_LOG_SDKEXT_DIR", True))
-
     tcname = d.expand("${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.sh")
     if not os.path.exists(tcname):
         bb.fatal("The toolchain ext %s is not built. Build it before running the" \
                  " tests: 'bitbake <image> -c populate_sdk_ext' ." % tcname)
 
-    testdir = d.expand("${WORKDIR}/testsdkext/")
-    bb.utils.remove(testdir, True)
-    bb.utils.mkdirhier(testdir)
-    sdkdir = os.path.join(testdir, 'tc')
+    tdname = d.expand("${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.testdata.json")
+    test_data = json.load(open(tdname, "r"))
+
+    target_pkg_manifest = OESDKExtTestContextExecutor._load_manifest(
+        d.expand("${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.target.manifest"))
+    host_pkg_manifest = OESDKExtTestContextExecutor._load_manifest(
+        d.expand("${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.host.manifest"))
+
+    sdk_dir = d.expand("${WORKDIR}/testsdkext/")
+    bb.utils.remove(sdk_dir, True)
+    bb.utils.mkdirhier(sdk_dir)
     try:
-        subprocess.check_output("%s -y -d %s" % (tcname, sdkdir), shell=True)
+        subprocess.check_output("%s -y -d %s" % (tcname, sdk_dir), shell=True)
     except subprocess.CalledProcessError as e:
         msg = "Couldn't install the extensible SDK:\n%s" % e.output.decode("utf-8")
         logfn = os.path.join(sdkdir, 'preparing_build_system.log')

@@ -165,19 +142,35 @@ def testsdkext_main(d):
                     msg += line
         bb.fatal(msg)
 
-    try:
-        bb.plain("Running SDK Compatibility tests ...")
-        run_test_context(SDKExtTestContext, d, testdir, tcname, pn, True)
-    finally:
-        pass
+    fail = False
+    sdk_envs = OESDKExtTestContextExecutor._get_sdk_environs(sdk_dir)
+    for s in sdk_envs:
+        bb.plain("Extensible SDK testing environment: %s" % s)
 
-    try:
-        bb.plain("Running Extensible SDK tests ...")
-        run_test_context(SDKExtTestContext, d, testdir, tcname, pn)
-    finally:
-        pass
+        sdk_env = sdk_envs[s]
+        tc = OESDKExtTestContext(td=test_data, logger=logger, sdk_dir=sdk_dir,
+            sdk_env=sdk_env, target_pkg_manifest=target_pkg_manifest,
+            host_pkg_manifest=host_pkg_manifest)
 
-    bb.utils.remove(testdir, True)
+        try:
+            tc.loadTests(OESDKExtTestContextExecutor.default_cases)
+        except Exception as e:
+            import traceback
+            bb.fatal("Loading tests failed:\n%s" % traceback.format_exc())
+
+        result = tc.runTests()
+
+        component = "%s %s" % (pn, OESDKExtTestContextExecutor.name)
+        context_msg = "%s:%s" % (os.path.basename(tcname), os.path.basename(sdk_env))
+
+        tc.logSummary(result, component, context_msg)
+        tc.logDetails()
+
+        if not result.wasSuccessful():
+            fail = True
+
+    if fail:
+        bb.fatal("%s - FAILED - check the task log and the commands log" % pn)
 
 testsdkext_main[vardepsexclude] =+ "BB_ORIGENV"
 

index 4a98b0f1..f9e9025 100644
@@ -27,7 +27,6 @@ try:
 except ImportError:
     pass
 from oeqa.utils.decorators import LogResults, gettag, getResults
-from oeqa.utils import avoid_paths_in_environ
 
 logger = logging.getLogger("BitBake")
 

@@ -145,18 +144,6 @@ class OETestCalledProcessError(subprocess.CalledProcessError):
 
 subprocess.CalledProcessError = OETestCalledProcessError
 
-class oeSDKExtTest(oeSDKTest):
-    def _run(self, cmd):
-        # extensible sdk shows a warning if found bitbake in the path
-        # because can cause contamination, i.e. use devtool from
-        # poky/scripts instead of eSDK one.
-        env = os.environ.copy()
-        paths_to_avoid = ['bitbake/bin', 'poky/scripts']
-        env['PATH'] = avoid_paths_in_environ(paths_to_avoid)
-
-        return subprocess.check_output(". %s > /dev/null;"\
-            " %s;" % (self.tc.sdkenv, cmd), stderr=subprocess.STDOUT, shell=True, env=env).decode("utf-8")
-
 def getmodule(pos=2):
     # stack returns a list of tuples containg frame information
     # First element of the list the is current frame, caller is 1

@@ -642,30 +629,3 @@ class ExportTestContext(RuntimeTestContext):
         extracted_dir = self.d.getVar("TEST_EXPORT_EXTRACTED_DIR", True)
         pkg_dir = os.path.join(export_dir, extracted_dir)
         super(ExportTestContext, self).install_uninstall_packages(test_id, pkg_dir, install)
-
-class SDKExtTestContext(SDKTestContext):
-    def __init__(self, d, sdktestdir, sdkenv, tcname, *args):
-        self.target_manifest = d.getVar("SDK_EXT_TARGET_MANIFEST", True)
-        self.host_manifest = d.getVar("SDK_EXT_HOST_MANIFEST", True)
-        if args:
-            self.cm = args[0] # Compatibility mode for run SDK tests
-        else:
-            self.cm = False
-
-        super(SDKExtTestContext, self).__init__(d, sdktestdir, sdkenv, tcname)
-
-        self.sdkextfilesdir = os.path.join(os.path.dirname(os.path.abspath(
-            oeqa.sdkext.__file__)), "files")
-
-    def _get_test_namespace(self):
-        if self.cm:
-            return "sdk"
-        else:
-            return "sdkext"
-
-    def _get_test_suites(self):
-        return (self.d.getVar("TEST_SUITES_SDK_EXT", True) or "auto").split()
-
-    def _get_test_suites_required(self):
-        return [t for t in (self.d.getVar("TEST_SUITES_SDK_EXT", True) or \
-                "auto").split() if t != "auto"]

new file mode 100644
index 0000000..6f708aa
@@ -0,0 +1,21 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+import os
+import subprocess
+
+from oeqa.utils import avoid_paths_in_environ
+from oeqa.sdk.case import OESDKTestCase
+
+class OESDKExtTestCase(OESDKTestCase):
+    def _run(self, cmd):
+        # extensible sdk shows a warning if found bitbake in the path
+        # because can cause contamination, i.e. use devtool from
+        # poky/scripts instead of eSDK one.
+        env = os.environ.copy()
+        paths_to_avoid = ['bitbake/bin', 'poky/scripts']
+        env['PATH'] = avoid_paths_in_environ(paths_to_avoid)
+
+        return subprocess.check_output(". %s > /dev/null;"\
+            " %s;" % (self.tc.sdk_env, cmd), stderr=subprocess.STDOUT,
+            shell=True, env=env).decode("utf-8")

new file mode 100644
index 0000000..8dbcd80
@@ -0,0 +1,21 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+import os
+from oeqa.sdk.context import OESDKTestContext, OESDKTestContextExecutor
+
+class OESDKExtTestContext(OESDKTestContext):
+    esdk_files_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "files")
+
+class OESDKExtTestContextExecutor(OESDKTestContextExecutor):
+    _context_class = OESDKExtTestContext
+
+    name = 'esdk'
+    help = 'esdk test component'
+    description = 'executes esdk tests'
+
+    default_cases = [OESDKTestContextExecutor.default_cases[0],
+            os.path.join(os.path.abspath(os.path.dirname(__file__)), 'cases')]
+    default_test_data = None
+
+_executor_class = OESDKExtTestContextExecutor

index 24529ca..1d6547a 100644
@@ -100,7 +100,7 @@ def testsdkext_main(d):
 
     from bb.utils import export_proxies
     from oeqa.utils import avoid_paths_in_environ
-    from oeqa.sdk.context import OESDKExtTestContext, OESDKExtTestContextExecutor
+    from oeqa.sdkext.context import OESDKExtTestContext, OESDKExtTestContextExecutor
 
     pn = d.getVar("PN", True)
     logger = logging.getLogger("BitBake")

@@ -119,13 +119,13 @@ def testsdkext_main(d):
         bb.fatal("The toolchain ext %s is not built. Build it before running the" \
                  " tests: 'bitbake <image> -c populate_sdk_ext' ." % tcname)
 
-    tdname = d.expand("${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.testdata.json")
+    tdname = d.expand("${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.testdata.json")
     test_data = json.load(open(tdname, "r"))
 
     target_pkg_manifest = OESDKExtTestContextExecutor._load_manifest(
-        d.expand("${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.target.manifest"))
+        d.expand("${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.target.manifest"))
     host_pkg_manifest = OESDKExtTestContextExecutor._load_manifest(
-        d.expand("${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.host.manifest"))
+        d.expand("${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.host.manifest"))
 
     sdk_dir = d.expand("${WORKDIR}/testsdkext/")
     bb.utils.remove(sdk_dir, True)

@@ -134,7 +134,7 @@ def testsdkext_main(d):
         subprocess.check_output("%s -y -d %s" % (tcname, sdk_dir), shell=True)
     except subprocess.CalledProcessError as e:
         msg = "Couldn't install the extensible SDK:\n%s" % e.output.decode("utf-8")
-        logfn = os.path.join(sdkdir, 'preparing_build_system.log')
+        logfn = os.path.join(sdk_dir, 'preparing_build_system.log')
         if os.path.exists(logfn):
             msg += '\n\nContents of preparing_build_system.log:\n'
             with open(logfn, 'r') as f:

index bdc8b6a..b9b24ee 100644
@@ -10,7 +10,7 @@ class GalculatorTest(OESDKTestCase):
     def setUpClass(self):
         if not (self.tc.hasTargetPackage("gtk+3") or\
                 self.tc.hasTargetPackage("libgtk-3.0")):
-            raise unittest.SkipTest("%s class: SDK don't support gtk+3" % self.__name__)
+            raise unittest.SkipTest("%s class: SDK don't support gtk+3" % self.__class__.__name__)
 
     def test_galculator(self):
         dl_dir = self.td.get('DL_DIR', None)

index e06af4c..36725e3 100644
@@ -20,7 +20,7 @@ class GccCompileTest(OESDKTestCase):
         machine = self.td.get("MACHINE")
         if not self.tc.hasHostPackage("packagegroup-cross-canadian-%s" % machine):
             raise unittest.SkipTest("%s class: SDK doesn't contain a cross-canadian toolchain",
-                    self.__name__)
+                    self.__class__.__name__)
 
     def test_gcc_compile(self):
         self._run('$CC %s/test.c -o %s/test -lm' % (self.tc.sdk_dir, self.tc.sdk_dir))

98 lines not on this page · raw message

index 2ade839..4e70c8f 100644
@@ -1,21 +1,23 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
 import os
 import shutil
 import subprocess
 
-from oeqa.oetest import oeSDKExtTest
+from oeqa.sdkext.case import OESDKExtTestCase
 from oeqa.utils.httpserver import HTTPService
 
-class SdkUpdateTest(oeSDKExtTest):
-
+class SdkUpdateTest(OESDKExtTestCase):
     @classmethod
     def setUpClass(self):
-        self.publish_dir = os.path.join(self.tc.sdktestdir, 'esdk_publish')
+        self.publish_dir = os.path.join(self.tc.sdk_dir, 'esdk_publish')
         if os.path.exists(self.publish_dir):
             shutil.rmtree(self.publish_dir)
         os.mkdir(self.publish_dir)
 
-        tcname_new = self.tc.d.expand(
-            "${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}-new.sh")
+        tcname_new = "%s/%s-new.sh" % (self.td.get("SDK_DEPLOY", ''),
+            self.td.get("TOOLCHAINEXT_OUTPUTNAME", ''))
         if not os.path.exists(tcname_new):
             tcname_new = self.tc.tcname
 

index 6b6781d..89ed003 100644
@@ -129,11 +129,15 @@ def testimage_main(d):
     from oeqa.oetest import ImageTestContext
     from oeqa.targetcontrol import get_target_controller
     from oeqa.utils.dump import get_host_dumper
+    from bb.utils import export_proxies
 
     pn = d.getVar("PN", True)
     bb.utils.mkdirhier(d.getVar("TEST_LOG_DIR", True))
     test_create_extract_dirs(d)
 
+    # runtime use network for download projects for build
+    export_proxies(d)
+
     # we need the host dumper in test context
     host_dumper = get_host_dumper(d)
 

index fe6cbfb..a5ca3a5 100644
@@ -1,6 +1,6 @@
 from oeqa.oetest import oeRuntimeTest, skipModule
 from oeqa.utils.decorators import *
-from oeqa.utils.targetbuild import TargetBuildProject
+from oeqa.runtime.utils.targetbuildproject import TargetBuildProject
 
 def setUpModule():
     if not oeRuntimeTest.hasFeature("tools-sdk"):

@@ -10,9 +10,10 @@ class BuildCvsTest(oeRuntimeTest):
 
     @classmethod
     def setUpClass(self):
-        self.project = TargetBuildProject(oeRuntimeTest.tc.target, oeRuntimeTest.tc.d,
-                        "http://ftp.gnu.org/non-gnu/cvs/source/feature/1.12.13/cvs-1.12.13.tar.bz2")
-        self.project.download_archive()
+        dl_dir = oeRuntimeTest.tc.d.getVar('DL_DIR', True)
+        self.project = TargetBuildProject(oeRuntimeTest.tc.target,
+                        "http://ftp.gnu.org/non-gnu/cvs/source/feature/1.12.13/cvs-1.12.13.tar.bz2",
+                        dl_dir=dl_dir)
 
     @testcase(205)
     @skipUnlessPassed("test_ssh")

index 220101d..20f0a79 100644
@@ -1,6 +1,6 @@
 from oeqa.oetest import oeRuntimeTest, skipModule
 from oeqa.utils.decorators import *
-from oeqa.utils.targetbuild import TargetBuildProject
+from oeqa.runtime.utils.targetbuildproject import TargetBuildProject
 
 def setUpModule():
     if not oeRuntimeTest.hasFeature("tools-sdk"):

@@ -10,9 +10,11 @@ class GalculatorTest(oeRuntimeTest):
     @testcase(1526)
     @skipUnlessPassed("test_ssh")
     def test_galculator(self):
+        dl_dir = oeRuntimeTest.tc.d.getVar('DL_DIR', True)
         try:
-            project = TargetBuildProject(oeRuntimeTest.tc.target, oeRuntimeTest.tc.d,
-                                      "http://galculator.mnim.org/downloads/galculator-2.1.4.tar.bz2")
+            project = TargetBuildProject(oeRuntimeTest.tc.target,
+                                      "http://galculator.mnim.org/downloads/galculator-2.1.4.tar.bz2",
+                                      dl_dir=dl_dir)
             project.download_archive()
 
             self.assertEqual(project.run_configure(), 0,

index bc75d0a..a0e82f0 100644
@@ -1,6 +1,6 @@
 from oeqa.oetest import oeRuntimeTest, skipModule
 from oeqa.utils.decorators import *
-from oeqa.utils.targetbuild import TargetBuildProject
+from oeqa.runtime.utils.targetbuildproject import TargetBuildProject
 
 def setUpModule():
     if not oeRuntimeTest.hasFeature("tools-sdk"):

@@ -10,8 +10,10 @@ class BuildIptablesTest(oeRuntimeTest):
 
     @classmethod
     def setUpClass(self):
-        self.project = TargetBuildProject(oeRuntimeTest.tc.target, oeRuntimeTest.tc.d,
-                        "http://downloads.yoctoproject.org/mirror/sources/iptables-1.4.13.tar.bz2")
+        dl_dir = oeRuntimeTest.tc.d.getVar('DL_DIR', True)
+        self.project = TargetBuildProject(oeRuntimeTest.tc.target,
+                        "http://downloads.yoctoproject.org/mirror/sources/iptables-1.4.13.tar.bz2",
+                        dl_dir=dl_dir)
         self.project.download_archive()
 
     @testcase(206)

index 5147020..a3ccd63 100644
@@ -87,6 +87,7 @@ def exportTests(d,tc):
     #   - the contents of oeqa/utils and oeqa/runtime/files
     #   - oeqa/oetest.py and oeqa/runexport.py (this will get copied to exportpath not exportpath/oeqa)
     #   - __init__.py files
+    bb.utils.mkdirhier(os.path.join(exportpath, "oeqa/files"))
     bb.utils.mkdirhier(os.path.join(exportpath, "oeqa/runtime/files"))
     bb.utils.mkdirhier(os.path.join(exportpath, "oeqa/utils"))
     # copy test modules, this should cover tests in other layers too

@@ -124,6 +125,10 @@ def exportTests(d,tc):
         for f in files:
             if f.endswith(".py"):
                 shutil.copy2(os.path.join(root, f), os.path.join(exportpath, "oeqa/utils"))
+    # copy oeqa/files/*
+    for root, dirs, files in os.walk(os.path.join(oeqadir, "files")):
+        for f in files:
+            shutil.copy2(os.path.join(root, f), os.path.join(exportpath, "oeqa/files"))
     # copy oeqa/runtime/files/*
     for root, dirs, files in os.walk(os.path.join(oeqadir, "runtime/files")):
         for f in files:

index f9e9025..48bcafc 100644
@@ -201,6 +201,7 @@ class TestContext(object):
         self.testsrequired = self._get_test_suites_required()
 
         self.filesdir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "runtime/files")
+        self.corefilesdir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "files")
         self.imagefeatures = d.getVar("IMAGE_FEATURES", True).split()
         self.distrofeatures = d.getVar("DISTRO_FEATURES", True).split()
 

index d90cd17..6edb89f 100644
@@ -12,9 +12,9 @@ class GccCompileTest(oeRuntimeTest):
 
     @classmethod
     def setUpClass(self):
-        oeRuntimeTest.tc.target.copy_to(os.path.join(oeRuntimeTest.tc.filesdir, "test.c"), "/tmp/test.c")
+        oeRuntimeTest.tc.target.copy_to(os.path.join(oeRuntimeTest.tc.corefilesdir, "test.c"), "/tmp/test.c")
         oeRuntimeTest.tc.target.copy_to(os.path.join(oeRuntimeTest.tc.filesdir, "testmakefile"), "/tmp/testmakefile")
-        oeRuntimeTest.tc.target.copy_to(os.path.join(oeRuntimeTest.tc.filesdir, "test.cpp"), "/tmp/test.cpp")
+        oeRuntimeTest.tc.target.copy_to(os.path.join(oeRuntimeTest.tc.corefilesdir, "test.cpp"), "/tmp/test.cpp")
 
     @testcase(203)
     def test_gcc_compile(self):

index e044d0a..6bf98f1 100644
@@ -12,7 +12,7 @@ class PerlTest(oeRuntimeTest):
 
     @classmethod
     def setUpClass(self):
-        oeRuntimeTest.tc.target.copy_to(os.path.join(oeRuntimeTest.tc.filesdir, "test.pl"), "/tmp/test.pl")
+        oeRuntimeTest.tc.target.copy_to(os.path.join(oeRuntimeTest.tc.corefilesdir, "test.pl"), "/tmp/test.pl")
 
     @testcase(1141)
     def test_perl_exists(self):

index 29a231c..93e822c 100644
@@ -12,7 +12,7 @@ class PythonTest(oeRuntimeTest):
 
     @classmethod
     def setUpClass(self):
-        oeRuntimeTest.tc.target.copy_to(os.path.join(oeRuntimeTest.tc.filesdir, "test.py"), "/tmp/test.py")
+        oeRuntimeTest.tc.target.copy_to(os.path.join(oeRuntimeTest.tc.corefilesdir, "test.py"), "/tmp/test.py")
 
     @testcase(1145)
     def test_python_exists(self):