Enable installation of packages in DUT

index 3b1d24f..4aaff8c 100644
@@ -1411,7 +1411,7 @@ class RpmPM(PackageManager):
     def package_info(self, pkg):
         cmd = "%s %s info --urls %s" % (self.smart_cmd, self.smart_opt, pkg)
         try:
-            output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
+            output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8")
         except subprocess.CalledProcessError as e:
             bb.fatal("Unable to list available packages. Command '%s' "
                      "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8")))

@@ -1506,7 +1506,7 @@ class OpkgDpkgPM(PackageManager):
     def package_info(self, pkg, cmd):
 
         try:
-            output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
+            output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8")
         except subprocess.CalledProcessError as e:
             bb.fatal("Unable to list available packages. Command '%s' "
                      "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8")))

index 51b7d93..c86eaac 100644
@@ -96,11 +96,13 @@ def exportTests(d,tc):
                         shutil.copytree(foldername, target_folder)
         if not isfolder:
             shutil.copy2(mod.path, os.path.join(exportpath, "oeqa/runtime"))
-    # copy __init__.py files
-    oeqadir = os.path.dirname(pkgutil.get_loader("oeqa").path)
-    shutil.copy2(os.path.join(oeqadir, "__init__.py"), os.path.join(exportpath, "oeqa"))
-    shutil.copy2(os.path.join(oeqadir, "runtime/__init__.py"), os.path.join(exportpath, "oeqa/runtime"))
+    # Get meta layer
+    for layer in d.getVar("BBLAYERS", True).split():
+        if os.path.basename(layer) == "meta":
+            meta_layer = layer
+            break
     # copy oeqa/oetest.py and oeqa/runexported.py
+    oeqadir = os.path.join(meta_layer, "lib/oeqa")
     shutil.copy2(os.path.join(oeqadir, "oetest.py"), os.path.join(exportpath, "oeqa"))
     shutil.copy2(os.path.join(oeqadir, "runexported.py"), exportpath)
     # copy oeqa/utils/*.py

deleted file mode 100644
index e69de29..0000000

index b4cf34b..819f959 100644
@@ -203,8 +203,7 @@ class TestContext(object):
         self.testslist = self._get_tests_list(path, extrapath)
         self.testsrequired = self._get_test_suites_required()
 
-        self.filesdir = os.path.join(os.path.dirname(os.path.abspath(
-            oeqa.runtime.__file__)), "files")
+        self.filesdir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "runtime/files")
         self.imagefeatures = d.getVar("IMAGE_FEATURES", True).split()
         self.distrofeatures = d.getVar("DISTRO_FEATURES", True).split()
 

@@ -460,7 +459,7 @@ class RuntimeTestContext(TestContext):
         Returns the path of the JSON file for a module, empty if doesn't exitst.
         """
 
-        module_file = module.filename
+        module_file = module.path
         json_file = "%s.json" % module_file.rsplit(".", 1)[0]
         if os.path.isfile(module_file) and os.path.isfile(json_file):
             return json_file

deleted file mode 100644
index 4cf3fa7..0000000
@@ -1,3 +0,0 @@
-# Enable other layers to have tests in the same named directory
-from pkgutil import extend_path
-__path__ = extend_path(__path__, __name__)

index c86eaac..8046e29 100644
@@ -16,6 +16,9 @@
 
 TEST_LOG_DIR ?= "${WORKDIR}/testexport"
 TEST_EXPORT_DIR ?= "${TMPDIR}/testexport/${PN}"
+TEST_EXPORT_PACKAGED_DIR ?= "packages/packaged"
+TEST_EXPORT_EXTRACTED_DIR ?= "packages/extracted"
+
 TEST_TARGET ?= "simpleremote"
 TEST_TARGET_IP ?= ""
 TEST_SERVER_IP ?= ""

@@ -28,9 +31,9 @@ python do_testexport() {
 }
 
 addtask testexport
-do_testimage[nostamp] = "1"
-do_testimage[depends] += "${TEST_EXPORT_DEPENDS}"
-do_testimage[lockfiles] += "${TEST_EXPORT_LOCK}"
+do_testexport[nostamp] = "1"
+do_testexport[depends] += "${TEST_EXPORT_DEPENDS} ${TESTIMAGEDEPENDS}"
+do_testexport[lockfiles] += "${TEST_EXPORT_LOCK}"
 
 def exportTests(d,tc):
     import json

@@ -96,6 +99,9 @@ def exportTests(d,tc):
                         shutil.copytree(foldername, target_folder)
         if not isfolder:
             shutil.copy2(mod.path, os.path.join(exportpath, "oeqa/runtime"))
+            json_file = "%s.json" % mod.path.rsplit(".", 1)[0]
+            if os.path.isfile(json_file):
+                shutil.copy2(json_file, os.path.join(exportpath, "oeqa/runtime"))
     # Get meta layer
     for layer in d.getVar("BBLAYERS", True).split():
         if os.path.basename(layer) == "meta":

@@ -115,6 +121,20 @@ def exportTests(d,tc):
         for f in files:
             shutil.copy2(os.path.join(root, f), os.path.join(exportpath, "oeqa/runtime/files"))
 
+    # Copy packages needed for runtime testing
+    export_pkg_dir = os.path.join(d.getVar("TEST_EXPORT_DIR", True), "packages")
+    test_pkg_dir = d.getVar("TEST_NEEDED_PACKAGES_DIR", True)
+    for root, subdirs, files in os.walk(test_pkg_dir):
+        for subdir in subdirs:
+            tmp_dir = os.path.join(root.replace(test_pkg_dir, "").lstrip("/"), subdir)
+            new_dir = os.path.join(export_pkg_dir, tmp_dir)
+            bb.utils.mkdirhier(new_dir)
+
+        for f in files:
+            src_f = os.path.join(root, f)
+            dst_f = os.path.join(export_pkg_dir, root.replace(test_pkg_dir, "").lstrip("/"), f)
+            shutil.copy2(src_f, dst_f)
+
     bb.plain("Exported tests to: %s" % exportpath)
 
 def testexport_main(d):

@@ -122,9 +142,11 @@ def testexport_main(d):
     from oeqa.targetcontrol import get_target_controller
     from oeqa.utils.dump import get_host_dumper
 
+    test_create_extract_dirs(d)
+    export_dir = d.getVar("TEST_EXPORT_DIR", True)
     bb.utils.mkdirhier(d.getVar("TEST_LOG_DIR", True))
-    bb.utils.remove(d.getVar("TEST_EXPORT_DIR", True), recurse=True)
-    bb.utils.mkdirhier(d.getVar("TEST_EXPORT_DIR", True))
+    bb.utils.remove(export_dir, recurse=True)
+    bb.utils.mkdirhier(export_dir)
 
     # the robot dance
     target = get_target_controller(d)

@@ -141,6 +163,7 @@ def testexport_main(d):
         import traceback
         bb.fatal("Loading tests failed:\n%s" % traceback.format_exc())
 
+    tc.extract_packages()
     exportTests(d,tc)
 
 testexport_main[vardepsexclude] =+ "BB_ORIGENV"

index a2e13df..a70d3a8 100644
@@ -30,6 +30,10 @@
 TEST_LOG_DIR ?= "${WORKDIR}/testimage"
 
 TEST_EXPORT_DIR ?= "${TMPDIR}/testimage/${PN}"
+TEST_INSTALL_TMP_DIR ?= "${WORKDIR}/testimage/install_tmp"
+TEST_NEEDED_PACKAGES_DIR ?= "${WORKDIR}/testimage/packages"
+TEST_EXTRACTED_DIR ?= "${TEST_NEEDED_PACKAGES_DIR}/extracted"
+TEST_PACKAGED_DIR ?= "${TEST_NEEDED_PACKAGES_DIR}/packaged"
 
 RPMTESTSUITE = "${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'smart rpm', '', d)}"
 MINTESTSUITE = "ping"

@@ -100,6 +104,7 @@ testimage_dump_host () {
 python do_testimage() {
     testimage_main(d)
 }
+
 addtask testimage
 do_testimage[nostamp] = "1"
 do_testimage[depends] += "${TESTIMAGEDEPENDS}"

@@ -117,6 +122,7 @@ def testimage_main(d):
 
     pn = d.getVar("PN", True)
     bb.utils.mkdirhier(d.getVar("TEST_LOG_DIR", True))
+    test_create_extract_dirs(d)
 
     # we need the host dumper in test context
     host_dumper = get_host_dumper(d)

@@ -136,6 +142,7 @@ def testimage_main(d):
         import traceback
         bb.fatal("Loading tests failed:\n%s" % traceback.format_exc())
 
+    tc.extract_packages()
     target.deploy()
     try:
         target.start()

@@ -155,6 +162,17 @@ def testimage_main(d):
         signal.signal(signal.SIGTERM, tc.origsigtermhandler)
         target.stop()
 
+def test_create_extract_dirs(d):
+    install_path = d.getVar("TEST_INSTALL_TMP_DIR", True)
+    package_path = d.getVar("TEST_PACKAGED_DIR", True)
+    extracted_path = d.getVar("TEST_EXTRACTED_DIR", True)
+    bb.utils.mkdirhier(d.getVar("TEST_LOG_DIR", True))
+    bb.utils.remove(package_path, recurse=True)
+    bb.utils.mkdirhier(install_path)
+    bb.utils.mkdirhier(package_path)
+    bb.utils.mkdirhier(extracted_path)
+
+
 testimage_main[vardepsexclude] =+ "BB_ORIGENV"
 
 inherit testsdk

109 lines not on this page · raw message

index 8046e29..a320880 100644
@@ -24,6 +24,7 @@ TEST_TARGET_IP ?= ""
 TEST_SERVER_IP ?= ""
 
 TEST_EXPORT_DEPENDS = ""
+TEST_EXPORT_DEPENDS += "${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'cpio-native:do_populate_sysroot', '', d)}"
 TEST_EXPORT_LOCK = "${TMPDIR}/testimage.lock"
 
 python do_testexport() {

index a70d3a8..7fd0f62 100644
@@ -68,6 +68,8 @@ TEST_TARGET ?= "qemu"
 
 TESTIMAGEDEPENDS = ""
 TESTIMAGEDEPENDS_qemuall = "qemu-native:do_populate_sysroot qemu-helper-native:do_populate_sysroot"
+TESTIMAGEDEPENDS += "${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'cpio-native:do_populate_sysroot', '', d)}"
+TESTIMAGEDEPENDS_qemuall += "${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'cpio-native:do_populate_sysroot', '', d)}"
 
 TESTIMAGELOCK = "${TMPDIR}/testimage.lock"
 TESTIMAGELOCK_qemuall = ""

new file mode 100644
index 0000000..951d3be
@@ -0,0 +1,56 @@
+DESCRIPTION = "SDK type target for standalone tarball containing packages defined by TEST_EXPORT_TOOLS. The \
+               tarball can be used to run missing programs on testing systems which don't have such tools.\
+               This recipe is almost the same as buildtools-tarball"
+SUMMARY = "Standalone tarball for test systems with missing software"
+LICENSE = "MIT"
+LIC_FILES_CHKSUM = "file://${COREBASE}/LICENSE;md5=4d92cd373abda3937c2bc47fbc49d690 \
+                    file://${COREBASE}/meta/COPYING.MIT;md5=3da9cfbcb788c80a0384361b4de20420"
+
+TOOLCHAIN_TARGET_TASK ?= ""
+
+TOOLCHAIN_HOST_TASK ?= "${TEST_EXPORT_SDK_PACKAGES}"
+
+SDK_PACKAGE_ARCHS += "tesexport-tools-${SDKPKGSUFFIX}"
+
+TOOLCHAIN_OUTPUTNAME ?= "${TEST_EXPORT_SDK_NAME}"
+
+SDK_TITLE = "Testexport tools"
+
+RDEPENDS = "${TOOLCHAIN_HOST_TASK}"
+
+EXCLUDE_FROM_WORLD = "1"
+
+inherit meta
+inherit populate_sdk
+inherit toolchain-scripts
+
+create_sdk_files_append () {
+	rm -f ${SDK_OUTPUT}/${SDKPATH}/site-config-*
+	rm -f ${SDK_OUTPUT}/${SDKPATH}/environment-setup-*
+	rm -f ${SDK_OUTPUT}/${SDKPATH}/version-*
+
+	# Generate new (mini) sdk-environment-setup file
+	script=${1:-${SDK_OUTPUT}/${SDKPATH}/environment-setup-${SDK_SYS}}
+	touch $script
+	echo 'export PATH=${SDKPATHNATIVE}${bindir_nativesdk}:$PATH' >> $script
+	# In order for the self-extraction script to correctly extract and set up things,
+	# we need a 'OECORE_NATIVE_SYSROOT=xxx' line in environment setup script.
+	# However, testexport-tarball is inherently a tool set instead of a fully functional SDK,
+	# so instead of exporting the variable, we use a comment here.
+	echo '#OECORE_NATIVE_SYSROOT="${SDKPATHNATIVE}"' >> $script
+	toolchain_create_sdk_version ${SDK_OUTPUT}/${SDKPATH}/version-${SDK_SYS}
+
+	echo 'export GIT_SSL_CAINFO="${SDKPATHNATIVE}${sysconfdir}/ssl/certs/ca-certificates.crt"' >>$script
+
+	if [ "${SDKMACHINE}" = "i686" ]; then
+		echo 'export NO32LIBS="0"' >>$script
+		echo 'echo "$BB_ENV_EXTRAWHITE" | grep -q "NO32LIBS"' >>$script
+		echo '[ $? != 0 ] && export BB_ENV_EXTRAWHITE="NO32LIBS $BB_ENV_EXTRAWHITE"' >>$script
+	fi
+}
+
+# testexport-tarball doesn't need config site
+TOOLCHAIN_NEED_CONFIGSITE_CACHE = ""
+
+# The recipe doesn't need any default deps
+INHIBIT_DEFAULT_DEPS = "1"

index a320880..6009349 100644
@@ -23,8 +23,14 @@ TEST_TARGET ?= "simpleremote"
 TEST_TARGET_IP ?= ""
 TEST_SERVER_IP ?= ""
 
+TEST_EXPORT_SDK_PACKAGES ?= ""
+TEST_EXPORT_SDK_ENABLED ?= "0"
+TEST_EXPORT_SDK_NAME ?= "testexport-tools-nativesdk"
+TEST_EXPORT_SDK_DIR ?= "sdk"
+
 TEST_EXPORT_DEPENDS = ""
 TEST_EXPORT_DEPENDS += "${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'cpio-native:do_populate_sysroot', '', d)}"
+TEST_EXPORT_DEPENDS += "${@bb.utils.contains('TEST_EXPORT_SDK_ENABLED', '1', 'testexport-tarball:do_populate_sdk', '', d)}"
 TEST_EXPORT_LOCK = "${TMPDIR}/testimage.lock"
 
 python do_testexport() {

@@ -136,6 +142,16 @@ def exportTests(d,tc):
             dst_f = os.path.join(export_pkg_dir, root.replace(test_pkg_dir, "").lstrip("/"), f)
             shutil.copy2(src_f, dst_f)
 
+    # Copy SDK
+    if d.getVar("TEST_EXPORT_SDK_ENABLED", True) == "1":
+        sdk_deploy = d.getVar("SDK_DEPLOY", True)
+        tarball_name = "%s.sh" % d.getVar("TEST_EXPORT_SDK_NAME", True)
+        tarball_path = os.path.join(sdk_deploy, tarball_name)
+        export_sdk_dir = os.path.join(d.getVar("TEST_EXPORT_DIR", True),
+                                      d.getVar("TEST_EXPORT_SDK_DIR", True))
+        bb.utils.mkdirhier(export_sdk_dir)
+        shutil.copy2(tarball_path, export_sdk_dir)
+
     bb.plain("Exported tests to: %s" % exportpath)
 
 def testexport_main(d):

index f1366a6..5886739 100755
@@ -31,8 +31,8 @@ except ImportError:
 sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "oeqa")))
 
 from oeqa.oetest import ExportTestContext
+from oeqa.utils.commands import runCmd
 from oeqa.utils.sshcontrol import SSHControl
-from oeqa.utils.dump import get_host_dumper
 
 # this isn't pretty but we need a fake target object
 # for running the tests externally as we don't care

@@ -107,6 +107,8 @@ def main():
         if not os.path.isdir(d["DEPLOY_DIR"]):
             print("WARNING: The path to DEPLOY_DIR does not exist: %s" % d["DEPLOY_DIR"])
 
+    extract_sdk(d)
+
     target = FakeTarget(d)
     for key in loaded["target"].keys():
         setattr(target, key, loaded["target"][key])

@@ -118,6 +120,37 @@ def main():
 
     return 0
 
+def extract_sdk(d):
+    """
+    Extract SDK if needed
+    """
+
+    export_dir = os.path.dirname(os.path.realpath(__file__))
+    tools_dir = d.getVar("TEST_EXPORT_SDK_DIR", True)
+    tarball_name = "%s.sh" % d.getVar("TEST_EXPORT_SDK_NAME", True)
+    tarball_path = os.path.join(export_dir, tools_dir, tarball_name)
+    extract_path = os.path.join(export_dir, "sysroot")
+    if os.path.isfile(tarball_path):
+        print ("Found SDK tarball %s. Extracting..." % tarball_path)
+        result = runCmd("%s -y -d %s" % (tarball_path, extract_path))
+        for f in os.listdir(extract_path):
+            if f.startswith("environment-setup"):
+                print("Setting up SDK environment...")
+                env_file = os.path.join(extract_path, f)
+                update_env(env_file)
+
+def update_env(env_file):
+    """
+    Source a file and update environment
+    """
+
+    cmd = ". %s; env -0" % env_file
+    result = runCmd(cmd)
+
+    for line in result.output.split("\0"):
+        (key, _, value) = line.partition("=")
+        os.environ[key] = value
+
 if __name__ == "__main__":
     try:
         ret = main()

index 6009349..99602a3 100644
@@ -128,6 +128,9 @@ def exportTests(d,tc):
         for f in files:
             shutil.copy2(os.path.join(root, f), os.path.join(exportpath, "oeqa/runtime/files"))
 
+    # Create tar file for common parts of testexport
+    create_tarball(d, "testexport.tar.gz", d.getVar("TEST_EXPORT_DIR", True))
+
     # Copy packages needed for runtime testing
     export_pkg_dir = os.path.join(d.getVar("TEST_EXPORT_DIR", True), "packages")
     test_pkg_dir = d.getVar("TEST_NEEDED_PACKAGES_DIR", True)

@@ -142,6 +145,9 @@ def exportTests(d,tc):
             dst_f = os.path.join(export_pkg_dir, root.replace(test_pkg_dir, "").lstrip("/"), f)
             shutil.copy2(src_f, dst_f)
 
+    # Create tar file for packages needed by the DUT
+    create_tarball(d, "testexport_packages_%s.tar.gz" % d.getVar("MACHINE", True), export_pkg_dir)
+
     # Copy SDK
     if d.getVar("TEST_EXPORT_SDK_ENABLED", True) == "1":
         sdk_deploy = d.getVar("SDK_DEPLOY", True)

@@ -152,6 +158,9 @@ def exportTests(d,tc):
         bb.utils.mkdirhier(export_sdk_dir)
         shutil.copy2(tarball_path, export_sdk_dir)
 
+    # Create tar file for the sdk
+    create_tarball(d, "testexport_sdk_%s.tar.gz" % d.getVar("SDK_ARCH", True), export_sdk_dir)
+
     bb.plain("Exported tests to: %s" % exportpath)
 
 def testexport_main(d):

@@ -183,6 +192,23 @@ def testexport_main(d):
     tc.extract_packages()
     exportTests(d,tc)
 
+def create_tarball(d, tar_name, src_dir):
+
+    import tarfile
+
+    tar_path = os.path.join(d.getVar("TEST_EXPORT_DIR", True), tar_name)
+    current_dir = os.getcwd()
+    src_dir = src_dir.rstrip('/')
+    dir_name = os.path.dirname(src_dir)
+    base_name = os.path.basename(src_dir)
+
+    os.chdir(dir_name)
+    tar = tarfile.open(tar_path, "w:gz")
+    tar.add(base_name)
+    tar.close()
+    os.chdir(current_dir)
+
+
 testexport_main[vardepsexclude] =+ "BB_ORIGENV"
 
 inherit testimage

index 6009349..15fa470 100644
@@ -128,9 +128,13 @@ def exportTests(d,tc):
         for f in files:
             shutil.copy2(os.path.join(root, f), os.path.join(exportpath, "oeqa/runtime/files"))
 
+    # Create tar file for common parts of testexport
+    create_tarball(d, "testexport.tar.gz", d.getVar("TEST_EXPORT_DIR", True))
+
     # Copy packages needed for runtime testing
     export_pkg_dir = os.path.join(d.getVar("TEST_EXPORT_DIR", True), "packages")
     test_pkg_dir = d.getVar("TEST_NEEDED_PACKAGES_DIR", True)
+    need_pkg_dir = False
     for root, subdirs, files in os.walk(test_pkg_dir):
         for subdir in subdirs:
             tmp_dir = os.path.join(root.replace(test_pkg_dir, "").lstrip("/"), subdir)

@@ -138,10 +142,18 @@ def exportTests(d,tc):
             bb.utils.mkdirhier(new_dir)
 
         for f in files:
+            need_pkg_dir = True
             src_f = os.path.join(root, f)
             dst_f = os.path.join(export_pkg_dir, root.replace(test_pkg_dir, "").lstrip("/"), f)
             shutil.copy2(src_f, dst_f)
 
+    if need_pkg_dir:
+        # Create tar file for packages needed by the DUT
+        create_tarball(d, "testexport_packages_%s.tar.gz" % d.getVar("MACHINE", True), export_pkg_dir)
+    else:
+        # Remov packages dir from exported test
+        bb.utils.remove(export_pkg_dir, True)
+
     # Copy SDK
     if d.getVar("TEST_EXPORT_SDK_ENABLED", True) == "1":
         sdk_deploy = d.getVar("SDK_DEPLOY", True)

@@ -152,6 +164,9 @@ def exportTests(d,tc):
         bb.utils.mkdirhier(export_sdk_dir)
         shutil.copy2(tarball_path, export_sdk_dir)
 
+        # Create tar file for the sdk
+        create_tarball(d, "testexport_sdk_%s.tar.gz" % d.getVar("SDK_ARCH", True), export_sdk_dir)
+
     bb.plain("Exported tests to: %s" % exportpath)
 
 def testexport_main(d):

@@ -183,6 +198,23 @@ def testexport_main(d):
     tc.extract_packages()
     exportTests(d,tc)
 
+def create_tarball(d, tar_name, src_dir):
+
+    import tarfile
+
+    tar_path = os.path.join(d.getVar("TEST_EXPORT_DIR", True), tar_name)
+    current_dir = os.getcwd()
+    src_dir = src_dir.rstrip('/')
+    dir_name = os.path.dirname(src_dir)
+    base_name = os.path.basename(src_dir)
+
+    os.chdir(dir_name)
+    tar = tarfile.open(tar_path, "w:gz")
+    tar.add(base_name)
+    tar.close()
+    os.chdir(current_dir)
+
+
 testexport_main[vardepsexclude] =+ "BB_ORIGENV"
 
 inherit testimage