Allow to install packages in the DUT during runtime testing

index 0aac030..e06c668 100644
@@ -43,10 +43,6 @@ def exportTests(d,tc):
     savedata = {}
     savedata["d"] = {}
     savedata["target"] = {}
-    for key in tc.__dict__:
-        # special cases
-        if key not in ['d', 'target', 'suite']:
-            savedata[key] = getattr(tc, key)
     savedata["target"]["ip"] = tc.target.ip or d.getVar("TEST_TARGET_IP", True)
     savedata["target"]["server_ip"] = tc.target.server_ip or d.getVar("TEST_SERVER_IP", True)
 

index 70db119..8ad3715 100644
@@ -264,6 +264,22 @@ class TestContext(object):
 
         return testslist
 
+    def getTests(self, test):
+        '''Return all individual tests executed when running the suite.'''
+        # Unfortunately unittest does not have an API for this, so we have
+        # to rely on implementation details. This only needs to work
+        # for TestSuite containing TestCase.
+        method = getattr(test, '_testMethodName', None)
+        if method:
+            # leaf case: a TestCase
+            yield test
+        else:
+            # Look into TestSuite.
+            tests = getattr(test, '_tests', [])
+            for t1 in tests:
+                for t2 in self.getTests(t1):
+                    yield t2
+
     def loadTests(self):
         setattr(oeTest, "tc", self)
 

@@ -272,36 +288,20 @@ class TestContext(object):
         suites = [testloader.loadTestsFromName(name) for name in self.testslist]
         suites = filterByTagExp(suites, getattr(self, "tagexp", None))
 
-        def getTests(test):
-            '''Return all individual tests executed when running the suite.'''
-            # Unfortunately unittest does not have an API for this, so we have
-            # to rely on implementation details. This only needs to work
-            # for TestSuite containing TestCase.
-            method = getattr(test, '_testMethodName', None)
-            if method:
-                # leaf case: a TestCase
-                yield test
-            else:
-                # Look into TestSuite.
-                tests = getattr(test, '_tests', [])
-                for t1 in tests:
-                    for t2 in getTests(t1):
-                        yield t2
-
         # Determine dependencies between suites by looking for @skipUnlessPassed
         # method annotations. Suite A depends on suite B if any method in A
         # depends on a method on B.
         for suite in suites:
             suite.dependencies = []
             suite.depth = 0
-            for test in getTests(suite):
+            for test in self.getTests(suite):
                 methodname = getattr(test, '_testMethodName', None)
                 if methodname:
                     method = getattr(test, methodname)
                     depends_on = getattr(method, '_depends_on', None)
                     if depends_on:
                         for dep_suite in suites:
-                            if depends_on in [getattr(t, '_testMethodName', None) for t in getTests(dep_suite)]:
+                            if depends_on in [getattr(t, '_testMethodName', None) for t in self.getTests(dep_suite)]:
                                 if dep_suite not in suite.dependencies and \
                                    dep_suite is not suite:
                                     suite.dependencies.append(dep_suite)

new file mode 100644
index 0000000..099ecc9
@@ -0,0 +1,29 @@
+def get_package_manager(d, root_path):
+    """
+    Returns an OE package manager that can install packages in root_path.
+    """
+    from oe.package_manager import RpmPM, OpkgPM, DpkgPM
+
+    pkg_class = d.getVar("IMAGE_PKGTYPE", True)
+    if pkg_class == "rpm":
+        pm = RpmPM(d,
+                   root_path,
+                   d.getVar('TARGET_VENDOR', True))
+        pm.create_configs()
+
+    elif pkg_class == "ipk":
+        pm = OpkgPM(d,
+                    root_path,
+                    d.getVar("IPKGCONF_TARGET", True),
+                    d.getVar("ALL_MULTILIB_PACKAGE_ARCHS", True))
+
+    elif pkg_class == "deb":
+        pm = DpkgPM(d,
+                    root_path,
+                    d.getVar('PACKAGE_ARCHS', True),
+                    d.getVar('DPKG_ARCH', True))
+
+    pm.write_index()
+    pm.update()
+
+    return pm

index 8ad3715..7abd850 100644
@@ -385,6 +385,43 @@ class RuntimeTestContext(TestContext):
         if oeTest.hasPackage("procps"):
             oeRuntimeTest.pscmd = "ps -ef"
 
+    def _getJsonFile(self, module):
+        """
+        Returns the path of the JSON file for a module, empty if doesn't exitst.
+        """
+
+        module_file = module.filename
+        json_file = "%s.json" % module_file.rsplit(".", 1)[0]
+        if os.path.isfile(module_file) and os.path.isfile(json_file):
+            return json_file
+        else:
+            return ""
+
+    def _getNeededPackages(self, json_file, test=None):
+        """
+        Returns a dict with needed packages based on a JSON file.
+
+
+        If a test is specified it will return the dict just for that test.
+        """
+
+        import json
+
+        needed_packages = {}
+
+        with open(json_file) as f:
+            test_packages = json.load(f)
+        for key,value in test_packages.items():
+            needed_packages[key] = value
+
+        if test:
+            if test in needed_packages:
+                needed_packages = needed_packages[test]
+            else:
+                needed_packages = {}
+
+        return needed_packages
+
 class ImageTestContext(RuntimeTestContext):
     def __init__(self, d, target, host_dumper):
         super(ImageTestContext, self).__init__(d, target)

index 7abd850..8dd494a 100644
@@ -12,6 +12,7 @@ import unittest
 import inspect
 import subprocess
 import signal
+import shutil
 try:
     import bb
 except ImportError:

@@ -264,6 +265,22 @@ class TestContext(object):
 
         return testslist
 
+    def getTestModules(self):
+        """
+        Returns all the test modules in the testlist.
+        """
+
+        import pkgutil
+
+        modules = []
+        for test in self.testslist:
+            if re.search("\w+\.\w+\.test_\S+", test):
+                test = '.'.join(t.split('.')[:3])
+            module = pkgutil.get_loader(test)
+            modules.append(module)
+
+        return modules
+
     def getTests(self, test):
         '''Return all individual tests executed when running the suite.'''
         # Unfortunately unittest does not have an API for this, so we have

@@ -385,6 +402,44 @@ class RuntimeTestContext(TestContext):
         if oeTest.hasPackage("procps"):
             oeRuntimeTest.pscmd = "ps -ef"
 
+    def extract_packages(self):
+        """
+        Find and extract packages that will be needed during runtime.
+        """
+
+        needed_packages = {}
+        extracted_path = self.d.getVar("TEST_EXTRACTED_DIR", True)
+        modules = self.getTestModules()
+        bbpaths = self.d.getVar("BBPATH", True).split(":")
+
+        for module in modules:
+            json_file = self._getJsonFile(module)
+            if json_file:
+                needed_packages = self._getNeededPackages(json_file)
+
+        for key,value in needed_packages.items():
+            packages = ()
+            if isinstance(value, dict):
+                packages = (value, )
+            elif isinstance(value, list):
+                packages = value
+            else:
+                bb.fatal("Failed to process needed packages for %s; "
+                         "Value must be a dict or list" % key)
+
+            for package in packages:
+                pkg = package["pkg"]
+                rm = package.get("rm", False)
+                extract = package.get("extract", True)
+                if extract:
+                    dst_dir = os.path.join(extracted_path, pkg)
+
+                # Extract package and copy it to TEST_EXTRACTED_DIR
+                if extract and not os.path.exists(dst_dir):
+                    pkg_dir = self._extract_in_tmpdir(pkg)
+                    shutil.copytree(pkg_dir, dst_dir)
+                    shutil.rmtree(pkg_dir)
+
     def _getJsonFile(self, module):
         """
         Returns the path of the JSON file for a module, empty if doesn't exitst.

@@ -422,6 +477,21 @@ class RuntimeTestContext(TestContext):
 
         return needed_packages
 
+    def _extract_in_tmpdir(self, pkg):
+        """"
+        Returns path to a temp directory where the package was
+        extracted without dependencies.
+        """
+
+        from oeqa.utils.package_manager import get_package_manager
+
+        pkg_path = os.path.join(self.d.getVar("TEST_INSTALL_TMP_DIR", True), pkg)
+        pm = get_package_manager(self.d, pkg_path)
+        extract_dir = pm.extract(pkg)
+        shutil.rmtree(pkg_path)
+
+        return extract_dir
+
 class ImageTestContext(RuntimeTestContext):
     def __init__(self, d, target, host_dumper):
         super(ImageTestContext, self).__init__(d, target)

index 8dd494a..a50f9d8 100644
@@ -409,6 +409,7 @@ class RuntimeTestContext(TestContext):
 
         needed_packages = {}
         extracted_path = self.d.getVar("TEST_EXTRACTED_DIR", True)
+        packaged_path = self.d.getVar("TEST_PACKAGED_DIR", True)
         modules = self.getTestModules()
         bbpaths = self.d.getVar("BBPATH", True).split(":")
 

@@ -433,6 +434,8 @@ class RuntimeTestContext(TestContext):
                 extract = package.get("extract", True)
                 if extract:
                     dst_dir = os.path.join(extracted_path, pkg)
+                else:
+                    dst_dir = os.path.join(packaged_path)
 
                 # Extract package and copy it to TEST_EXTRACTED_DIR
                 if extract and not os.path.exists(dst_dir):

@@ -440,6 +443,10 @@ class RuntimeTestContext(TestContext):
                     shutil.copytree(pkg_dir, dst_dir)
                     shutil.rmtree(pkg_dir)
 
+                # Copy package to TEST_PACKAGED_DIR
+                elif not extract:
+                    self._copy_package(pkg)
+
     def _getJsonFile(self, module):
         """
         Returns the path of the JSON file for a module, empty if doesn't exitst.

@@ -492,6 +499,21 @@ class RuntimeTestContext(TestContext):
 
         return extract_dir
 
+    def _copy_package(self, pkg):
+        """
+        Copy the RPM, DEB or IPK package to dst_dir
+        """
+
+        from oeqa.utils.package_manager import get_package_manager
+
+        pkg_path = os.path.join(self.d.getVar("TEST_INSTALL_TMP_DIR", True), pkg)
+        dst_dir = self.d.getVar("TEST_PACKAGED_DIR", True)
+        pm = get_package_manager(self.d, pkg_path)
+        pkg_info = pm.package_info(pkg)
+        file_path = pkg_info[pkg]["filepath"]
+        shutil.copy2(file_path, dst_dir)
+        shutil.rmtree(pkg_path)
+
 class ImageTestContext(RuntimeTestContext):
     def __init__(self, d, target, host_dumper):
         super(ImageTestContext, self).__init__(d, target)

index 2d7da96..79b3206 100644
@@ -1444,8 +1444,10 @@ class RpmPM(PackageManager):
                 break
 
         # To have the same data type than other package_info methods
+        filepath = os.path.join(self.deploy_dir, arch, filename)
         pkg_dict = {}
-        pkg_dict[pkg] = {"arch":arch, "ver":ver, "filename":filename}
+        pkg_dict[pkg] = {"arch":arch, "ver":ver, "filename":filename,
+                         "filepath": filepath}
 
         return pkg_dict
 

@@ -1461,9 +1463,7 @@ class RpmPM(PackageManager):
             bb.fatal("Unable to get information for package '%s' while "
                      "trying to extract the package."  % pkg)
 
-        pkg_arch = pkg_info[pkg]["arch"]
-        pkg_filename = pkg_info[pkg]["filename"]
-        pkg_path = os.path.join(self.deploy_dir, pkg_arch, pkg_filename)
+        pkg_path = pkg_info[pkg]["filepath"]
 
         cpio_cmd = bb.utils.which(os.getenv("PATH"), "cpio")
         rpm2cpio_cmd = bb.utils.which(os.getenv("PATH"), "rpm2cpio")

@@ -1522,10 +1522,11 @@ class OpkgDpkgPM(PackageManager):
 
     This method extracts the common parts for Opkg and Dpkg
     """
-    def extract(self, pkg, pkg_path):
+    def extract(self, pkg, pkg_info):
 
         ar_cmd = bb.utils.which(os.getenv("PATH"), "ar")
         tar_cmd = bb.utils.which(os.getenv("PATH"), "tar")
+        pkg_path = pkg_info[pkg]["filepath"]
 
         if not os.path.isfile(pkg_path):
             bb.fatal("Unable to extract package for '%s'."

@@ -1897,7 +1898,14 @@ class OpkgPM(OpkgDpkgPM):
     """
     def package_info(self, pkg):
         cmd = "%s %s info %s" % (self.opkg_cmd, self.opkg_args, pkg)
-        return super(OpkgPM, self).package_info(pkg, cmd)
+        pkg_info = super(OpkgPM, self).package_info(pkg, cmd)
+
+        pkg_arch = pkg_info[pkg]["arch"]
+        pkg_filename = pkg_info[pkg]["filename"]
+        pkg_info[pkg]["filepath"] = \
+                os.path.join(self.deploy_dir, pkg_arch, pkg_filename)
+
+        return pkg_info
 
     """
     Returns the path to a tmpdir where resides the contents of a package.

@@ -1910,11 +1918,7 @@ class OpkgPM(OpkgDpkgPM):
             bb.fatal("Unable to get information for package '%s' while "
                      "trying to extract the package."  % pkg)
 
-        pkg_arch = pkg_info[pkg]["arch"]
-        pkg_filename = pkg_info[pkg]["filename"]
-        pkg_path = os.path.join(self.deploy_dir, pkg_arch, pkg_filename)
-
-        tmp_dir = super(OpkgPM, self).extract(pkg, pkg_path)
+        tmp_dir = super(OpkgPM, self).extract(pkg, pkg_info)
         bb.utils.remove(os.path.join(tmp_dir, "data.tar.gz"))
 
         return tmp_dir

@@ -2219,7 +2223,14 @@ class DpkgPM(OpkgDpkgPM):
     """
     def package_info(self, pkg):
         cmd = "%s show %s" % (self.apt_cache_cmd, pkg)
-        return super(DpkgPM, self).package_info(pkg, cmd)
+        pkg_info = super(DpkgPM, self).package_info(pkg, cmd)
+
+        pkg_arch = pkg_info[pkg]["pkgarch"]
+        pkg_filename = pkg_info[pkg]["filename"]
+        pkg_info[pkg]["filepath"] = \
+                os.path.join(self.deploy_dir, pkg_arch, pkg_filename)
+
+        return pkg_info
 
     """
     Returns the path to a tmpdir where resides the contents of a package.

@@ -2232,11 +2243,7 @@ class DpkgPM(OpkgDpkgPM):
             bb.fatal("Unable to get information for package '%s' while "
                      "trying to extract the package."  % pkg)
 
-        pkg_arch = pkg_info[pkg]["pkgarch"]
-        pkg_filename = pkg_info[pkg]["filename"]
-        pkg_path = os.path.join(self.deploy_dir, pkg_arch, pkg_filename)
-
-        tmp_dir = super(DpkgPM, self).extract(pkg, pkg_path)
+        tmp_dir = super(DpkgPM, self).extract(pkg, pkg_info)
         bb.utils.remove(os.path.join(tmp_dir, "data.tar.xz"))
 
         return tmp_dir

index e06c668..21a2bf7 100644
@@ -16,6 +16,9 @@
 
 TEST_LOG_DIR ?= "${WORKDIR}/testexport"
 TEST_EXPORT_DIR ?= "${TMPDIR}/testexport/${PN}"
+TEST_EXPORT_PACKAGED_DIR ?= "packages/packaged"
+TEST_EXPORT_EXTRACTED_DIR ?= "packages/extracted"
+
 TEST_TARGET ?= "simpleremote"
 TEST_TARGET_IP ?= ""
 TEST_SERVER_IP ?= ""

@@ -27,9 +30,9 @@ python do_testexport() {
     testexport_main(d)
 }
 
-addtask testexport
+addtask testexport after do_test_extract_packages
 do_testimage[nostamp] = "1"
-do_testimage[depends] += "${TEST_EXPORT_DEPENDS}"
+do_testimage[depends] += "${TEST_EXPORT_DEPENDS} ${TESTIMAGEDEPENDS}"
 do_testimage[lockfiles] += "${TEST_EXPORT_LOCK}"
 
 def exportTests(d,tc):

@@ -96,6 +99,10 @@ def exportTests(d,tc):
                         shutil.copytree(foldername, target_folder)
         if not isfolder:
             shutil.copy2(mod.filename, os.path.join(exportpath, "oeqa/runtime"))
+            json_file = "%s.json" % mod.filename.rsplit(".", 1)[0]
+            if os.path.isfile(json_file):
+                shutil.copy2(json_file, os.path.join(exportpath, "oeqa/runtime"))
+
     # copy __init__.py files
     oeqadir = pkgutil.get_loader("oeqa").filename
     shutil.copy2(os.path.join(oeqadir, "__init__.py"), os.path.join(exportpath, "oeqa"))

@@ -113,6 +120,19 @@ def exportTests(d,tc):
         for f in files:
             shutil.copy2(os.path.join(root, f), os.path.join(exportpath, "oeqa/runtime/files"))
 
+    # Copy packages needed for runtime testing
+    export_pkg_dir = os.path.join(d.getVar("TEST_EXPORT_DIR", True), "packages")
+    test_pkg_dir = d.getVar("TEST_NEEDED_PACKAGES_DIR", True)
+    for root, subdirs, files in os.walk(test_pkg_dir):
+        for subdir in subdirs:
+            tmp_dir = os.path.join(root.replace(test_pkg_dir, "").lstrip("/"), subdir)
+            new_dir = os.path.join(export_pkg_dir, tmp_dir)
+            bb.utils.mkdirhier(new_dir)
+
+        for f in files:
+            src_f = os.path.join(root, f)
+            dst_f = os.path.join(export_pkg_dir, root.replace(test_pkg_dir, "").lstrip("/"), f)
+            shutil.copy2(src_f, dst_f)
     bb.plain("Exported tests to: %s" % exportpath)
 
 def testexport_main(d):

@@ -120,9 +140,10 @@ def testexport_main(d):
     from oeqa.targetcontrol import get_target_controller
     from oeqa.utils.dump import get_host_dumper
 
+    export_dir = d.getVar("TEST_EXPORT_DIR", True)
     bb.utils.mkdirhier(d.getVar("TEST_LOG_DIR", True))
-    bb.utils.remove(d.getVar("TEST_EXPORT_DIR", True), recurse=True)
-    bb.utils.mkdirhier(d.getVar("TEST_EXPORT_DIR", True))
+    bb.utils.remove(export_dir, recurse=True)
+    bb.utils.mkdirhier(export_dir)
 
     # the robot dance
     target = get_target_controller(d)

63 lines not on this page · raw message

109 lines not on this page · raw message

index 1bdc617..31adb71 100644
@@ -39,3 +39,5 @@ ALTERNATIVE_LINK_NAME[cpio] = "${base_bindir}/cpio"
 
 ALTERNATIVE_PRIORITY[rmt] = "50"
 ALTERNATIVE_LINK_NAME[rmt] = "${base_sbindir}/rmt"
+
+BBCLASSEXTEND = "native"

index 436dec4..4f2abb8 100644
@@ -68,6 +68,8 @@ TEST_TARGET ?= "qemu"
 
 TESTIMAGEDEPENDS = ""
 TESTIMAGEDEPENDS_qemuall = "qemu-native:do_populate_sysroot qemu-helper-native:do_populate_sysroot"
+TESTIMAGEDEPENDS += "${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'cpio-native:do_populate_sysroot', '', d)}"
+TESTIMAGEDEPENDS_qemuall += "${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'cpio-native:do_populate_sysroot', '', d)}"
 
 TESTIMAGELOCK = "${TMPDIR}/testimage.lock"
 TESTIMAGELOCK_qemuall = ""