Skip to contentSkip to conversation

Allows to copy symlinks to DUTs with testimage

PATCH In review [email protected] 2 participants 4 messages Jul 26, 2016

lore
Share & tools
lore

[email protected]

author
#0 lore raw

v2 cover Allows to copy symlinks to DUTs with testimage

From: Mariano Lopez <[email protected]>

Currently when using the install functionality in the DUT without package manager, if you copy a symlink the function will fail.

These series will change the behaviour and will allow to recreate the symlinks in the DUT.

Changes in v2:

- Use oe.path.copytree instead of shutil.copytree
- Don't create function oeqa_copy, not needed anymore

[YOCTO #9932]

The following changes since commit 039f47ad197a9a53109c9f3deadd9c35e62c056d:

uclibc: remove meta-yocto-bsp append (2016-07-26 08:56:32 +0100)

are available in the git repository at:

git://git.yoctoproject.org/poky-contrib mariano/bug9932v2
http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=mariano/bug9932v2

Mariano Lopez (2):
oeqa/oetest.py: Allow to export packages using symlinks oeqa/utils/sshcontrol.py: Allows to copy symlinks to target

 meta/classes/testexport.bbclass   | 24 +++++-------------------
 meta/lib/oeqa/oetest.py           | 17 +++++++++++++----
 meta/lib/oeqa/utils/sshcontrol.py |  9 +++++++--
 3 files changed, 25 insertions(+), 25 deletions(-)
Signature · 1 line
2.6.6
author
#1 lore raw

PATCH v2 1/2 oeqa/oetest.py: Allow to export packages using symlinks

From: Mariano Lopez <[email protected]>

Currently packages that contains symlinks can't be extracted and exported. This allows to export extracted such packages.

A nice side effect is improved readability.

[YOCTO #9932]

Signed-off-by Mariano Lopez <[email protected]>
---
 meta/classes/testexport.bbclass | 24 +++++-------------------
 meta/lib/oeqa/oetest.py         | 17 +++++++++++++----
 2 files changed, 18 insertions(+), 23 deletions(-)
meta/classes/testexport.bbclass +5 -19
index 15fa470..5147020 100644
@@ -47,6 +47,7 @@ def exportTests(d,tc):
     import shutil
     import pkgutil
     import re
+    import oe.path
 
     exportpath = d.getVar("TEST_EXPORT_DIR", True)
 

@@ -103,7 +104,7 @@ def exportTests(d,tc):
                     isfolder = True
                     target_folder = os.path.join(exportpath, "oeqa", "runtime", os.path.basename(foldername))
                     if not os.path.exists(target_folder):
-                        shutil.copytree(foldername, target_folder)
+                        oe.path.copytree(foldername, target_folder)
         if not isfolder:
             shutil.copy2(mod.path, os.path.join(exportpath, "oeqa/runtime"))
             json_file = "%s.json" % mod.path.rsplit(".", 1)[0]

@@ -132,27 +133,12 @@ def exportTests(d,tc):
     create_tarball(d, "testexport.tar.gz", d.getVar("TEST_EXPORT_DIR", True))
 
     # Copy packages needed for runtime testing
-    export_pkg_dir = os.path.join(d.getVar("TEST_EXPORT_DIR", True), "packages")
     test_pkg_dir = d.getVar("TEST_NEEDED_PACKAGES_DIR", True)
-    need_pkg_dir = False
-    for root, subdirs, files in os.walk(test_pkg_dir):
-        for subdir in subdirs:
-            tmp_dir = os.path.join(root.replace(test_pkg_dir, "").lstrip("/"), subdir)
-            new_dir = os.path.join(export_pkg_dir, tmp_dir)
-            bb.utils.mkdirhier(new_dir)
-
-        for f in files:
-            need_pkg_dir = True
-            src_f = os.path.join(root, f)
-            dst_f = os.path.join(export_pkg_dir, root.replace(test_pkg_dir, "").lstrip("/"), f)
-            shutil.copy2(src_f, dst_f)
-
-    if need_pkg_dir:
+    if os.listdir(test_pkg_dir):
+        export_pkg_dir = os.path.join(d.getVar("TEST_EXPORT_DIR", True), "packages")
+        oe.path.copytree(test_pkg_dir, export_pkg_dir)
         # Create tar file for packages needed by the DUT
         create_tarball(d, "testexport_packages_%s.tar.gz" % d.getVar("MACHINE", True), export_pkg_dir)
-    else:
-        # Remov packages dir from exported test
-        bb.utils.remove(export_pkg_dir, True)
 
     # Copy SDK
     if d.getVar("TEST_EXPORT_SDK_ENABLED", True) == "1":
meta/lib/oeqa/oetest.py +13 -4
index e63ca56..3bf3643 100644
@@ -447,6 +447,8 @@ class RuntimeTestContext(TestContext):
         modules = self.getTestModules()
         bbpaths = self.d.getVar("BBPATH", True).split(":")
 
+        shutil.rmtree(self.d.getVar("TEST_EXTRACTED_DIR", True))
+        shutil.rmtree(self.d.getVar("TEST_PACKAGED_DIR", True))
         for module in modules:
             json_file = self._getJsonFile(module)
             if json_file:

@@ -458,6 +460,8 @@ class RuntimeTestContext(TestContext):
         Extract packages that will be needed during runtime.
         """
 
+        import oe.path
+
         extracted_path = self.d.getVar("TEST_EXTRACTED_DIR", True)
         packaged_path = self.d.getVar("TEST_PACKAGED_DIR", True)
 

@@ -481,13 +485,18 @@ class RuntimeTestContext(TestContext):
                     dst_dir = os.path.join(packaged_path)
 
                 # Extract package and copy it to TEST_EXTRACTED_DIR
-                if extract and not os.path.exists(dst_dir):
-                    pkg_dir = self._extract_in_tmpdir(pkg)
-                    shutil.copytree(pkg_dir, dst_dir)
+                pkg_dir = self._extract_in_tmpdir(pkg)
+                if extract:
+
+                    # Same package used for more than one test,
+                    # don't need to extract again.
+                    if os.path.exists(dst_dir):
+                        continue
+                    oe.path.copytree(pkg_dir, dst_dir)
                     shutil.rmtree(pkg_dir)
 
                 # Copy package to TEST_PACKAGED_DIR
-                elif not extract:
+                else:
                     self._copy_package(pkg)
 
     def _getJsonFile(self, module):
Signature · 1 line
2.6.6
#2 lore raw

Re: [PATCHv2 1/2] oeqa/oetest.py: Allow to export packages using symlinks

Quoted 83 lines · On Tue, 2016-07-26 at 09:38 +0000, [email protected] wrote:

From: Mariano Lopez <[email protected]>

Currently packages that contains symlinks can't be extracted and exported. This allows to export extracted such packages.

A nice side effect is improved readability.

[YOCTO #9932]

Signed-off-by Mariano Lopez <[email protected]>

---
 meta/classes/testexport.bbclass | 24 +++++------------------- meta/lib/oeqa/oetest.py         | 17 +++++++++++++----

 2 files changed, 18 insertions(+), 23 deletions(-)
meta/classes/testexport.bbclass +0 -0
index 15fa470..5147020 100644
@@ -47,6 +47,7 @@ def exportTests(d,tc):

     import shutil
     import pkgutil
     import re
+    import oe.path

     exportpath = d.getVar("TEST_EXPORT_DIR", True)

meta/classes/testexport.bbclass +0 -0
@@ -103,7 +104,7 @@ def exportTests(d,tc):

                     isfolder = True
                     target_folder = os.path.join(exportpath, "oeqa", "runtime", os.path.basename(foldername))
                     if not os.path.exists(target_folder):
-                        shutil.copytree(foldername, target_folder) +                        oe.path.copytree(foldername, target_folder) if not isfolder:
             shutil.copy2(mod.path, os.path.join(exportpath, "oeqa/runtime"))
             json_file = "%s.json" % mod.path.rsplit(".", 1)[0]

meta/classes/testexport.bbclass +0 -0
@@ -132,27 +133,12 @@ def exportTests(d,tc):
      create_tarball(d, "testexport.tar.gz",
 d.getVar("TEST_EXPORT_DIR", True))


     # Copy packages needed for runtime testing
-    export_pkg_dir = os.path.join(d.getVar("TEST_EXPORT_DIR", True), "packages")
     test_pkg_dir = d.getVar("TEST_NEEDED_PACKAGES_DIR", True) -    need_pkg_dir = False
-    for root, subdirs, files in os.walk(test_pkg_dir):
-        for subdir in subdirs:
-            tmp_dir = os.path.join(root.replace(test_pkg_dir, "").lstrip("/"), subdir)
-            new_dir = os.path.join(export_pkg_dir, tmp_dir) -            bb.utils.mkdirhier(new_dir)
-
-        for f in files:
-            need_pkg_dir = True
-            src_f = os.path.join(root, f)
-            dst_f = os.path.join(export_pkg_dir, root.replace(test_pkg_dir, "").lstrip("/"), f)
-            shutil.copy2(src_f, dst_f)
-
-    if need_pkg_dir:
+    if os.listdir(test_pkg_dir):
+        export_pkg_dir = os.path.join(d.getVar("TEST_EXPORT_DIR", True), "packages")
+        oe.path.copytree(test_pkg_dir, export_pkg_dir)
         # Create tar file for packages needed by the DUT create_tarball(d, "testexport_packages_%s.tar.gz" % d.getVar("MACHINE", True), export_pkg_dir)
-    else:
-        # Remov packages dir from exported test
-        bb.utils.remove(export_pkg_dir, True)

     # Copy SDK
     if d.getVar("TEST_EXPORT_SDK_ENABLED", True) == "1":

meta/lib/oeqa/oetest.py +0 -0
index e63ca56..3bf3643 100644
@@ -447,6 +447,8 @@ class RuntimeTestContext(TestContext):
          modules = self.getTestModules()
          bbpaths = self.d.getVar("BBPATH", True).split(":")
Quoted 3 lines


+        shutil.rmtree(self.d.getVar("TEST_EXTRACTED_DIR", True)) +        shutil.rmtree(self.d.getVar("TEST_PACKAGED_DIR", True))

Do we use the shutil module anywhere else in oetest.py? Could we drop it completely if we use oe.path.remove() rather than shutil.rmtree()? Perhaps you could send a follow-on patch?

Quoted 38 lines

         for module in modules:
             json_file = self._getJsonFile(module) if json_file:

meta/lib/oeqa/oetest.py +0 -0
@@ -458,6 +460,8 @@ class RuntimeTestContext(TestContext):

         Extract packages that will be needed during runtime. """

+        import oe.path
+
         extracted_path = self.d.getVar("TEST_EXTRACTED_DIR", True) packaged_path = self.d.getVar("TEST_PACKAGED_DIR", True)

meta/lib/oeqa/oetest.py +0 -0
@@ -481,13 +485,18 @@ class RuntimeTestContext(TestContext):
                      dst_dir = os.path.join(packaged_path)


                 # Extract package and copy it to TEST_EXTRACTED_DIR -                if extract and not os.path.exists(dst_dir):
-                    pkg_dir = self._extract_in_tmpdir(pkg) -                    shutil.copytree(pkg_dir, dst_dir)
+                pkg_dir = self._extract_in_tmpdir(pkg)
+                if extract:
+
+                    # Same package used for more than one test, +                    # don't need to extract again.
+                    if os.path.exists(dst_dir):
+                        continue
+                    oe.path.copytree(pkg_dir, dst_dir) shutil.rmtree(pkg_dir)

                 # Copy package to TEST_PACKAGED_DIR
-                elif not extract:
+                else:
                     self._copy_package(pkg)

     def _getJsonFile(self, module):
--
2.6.6

author
#3 lore raw

PATCH v2 2/2 oeqa/utils/sshcontrol.py: Allows to copy symlinks to target

In reply to [email protected]

From: Mariano Lopez <[email protected]>

Currently when copying a symlink to the target it will fail throwing an exception. This will recreate symlinks from the system performing the tests to the device under tests.

[YOCTO #9932]

Signed-off-by Mariano Lopez <[email protected]>
---
 meta/lib/oeqa/utils/sshcontrol.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)
meta/lib/oeqa/utils/sshcontrol.py +7 -2
index f5d46e0..da485ee 100644
@@ -147,8 +147,13 @@ class SSHControl(object):
         return self._internal_run(command, timeout, self.ignore_status)
 
     def copy_to(self, localpath, remotepath):
-        command = self.scp + [localpath, '%s@%s:%s' % (self.user, self.ip, remotepath)]
-        return self._internal_run(command, ignore_status=False)
+        if os.path.islink(localpath):
+            link = os.readlink(localpath)
+            dst_dir, dst_base = os.path.split(remotepath)
+            return self.run("cd %s; ln -s %s %s" % (dst_dir, link, dst_base))
+        else:
+            command = self.scp + [localpath, '%s@%s:%s' % (self.user, self.ip, remotepath)]
+            return self._internal_run(command, ignore_status=False)
 
     def copy_from(self, remotepath, localpath):
         command = self.scp + ['%s@%s:%s' % (self.user, self.ip, remotepath), localpath]
Signature · 1 line
2.6.6