Add extract() method to package manager

From: Mariano Lopez <[email protected]>

When using the opkg and apt-get package managers the function opkg_query() can be useful when query for package information.

This change moves the function outside the Indexer class so the Indexer, OpkgPM, DpkgPM can benefit from it.

[YOCTO #9569]

Signed-off-by Mariano Lopez <[email protected]>

---
 meta/lib/oe/package_manager.py | 104 ++++++++++++++++++++-----------
----------

 1 file changed, 51 insertions(+), 53 deletions(-)

meta/lib/oe/package_manager.py +0 -0

index b4b359a..427518d 100644
@@ -27,6 +27,55 @@ def create_index(arg):

     return None

+"""

+    for line in cmd_output.splitlines():
+        line = line.rstrip()
+        if ':' in line:
+            if line.startswith("Package: "):
+                pkg = line.split(": ")[1]
+            elif line.startswith("Architecture: "):
+                arch = line.split(": ")[1]
+            elif line.startswith("Version: "):
+                ver = line.split(": ")[1]
+            elif line.startswith("File: "):
+                filename = line.split(": ")[1]
+            elif line.startswith("Depends: "):
+                depends = verregex.sub('', line.split(": ")[1]) +                for depend in depends.split(", "):
+                    dep.append(depend)
+            elif line.startswith("Recommends: "):
+                recommends = verregex.sub('', line.split(": ")[1]) +                for recommend in recommends.split(", "):
+                    dep.append("%s [REC]" % recommend) +        else:
+            # IPK doesn't include the filename
+            if not filename:
+                filename = "%s_%s_%s.ipk" % (pkg, ver, arch) +            if pkg:
+                output[pkg] = {"arch":arch, "ver":ver, +                        "filename":filename, "deps": dep } +            pkg = ""
+            filename = ""
+            dep = []

meta/lib/oe/package_manager.py +9 -0

+
+    if pkg:
+        if not filename:
+            filename = "%s_%s_%s.ipk" % (pkg, ver, arch)
+        output[pkg] = {"arch":arch, "ver":ver,
+                "filename":filename, "deps": dep }
+
+    return output
+

 class Indexer(object):
     __metaclass__ = ABCMeta

meta/lib/oe/package_manager.py +0 -0

@@ -293,57 +342,6 @@ class PkgsList(object):

         pass


-    """
-    This method parse the output from the package manager -    and return a dictionary with the information of the -    installed packages. This is used whne the packages are -    in deb or ipk format
-    """
-    def opkg_query(self, cmd_output):
-        verregex = re.compile(' \([=<>]* [^ )]*\)')
-        output = dict()
-        filename = ""
-        dep = []
-        pkg = ""
-        for line in cmd_output.splitlines():
-            line = line.rstrip()
-            if ':' in line:
-                if line.startswith("Package: "):
-                    pkg = line.split(": ")[1]
-                elif line.startswith("Architecture: "):
-                    arch = line.split(": ")[1]
-                elif line.startswith("Version: "):
-                    ver = line.split(": ")[1]
-                elif line.startswith("File: "):
-                    filename = line.split(": ")[1]
-                elif line.startswith("Depends: "):
-                    depends = verregex.sub('', line.split(": ")[1]) -                    for depend in depends.split(", "):
-                        dep.append(depend)
-                elif line.startswith("Recommends: "):
-                    recommends = verregex.sub('', line.split(":
")[1])
-                    for recommend in recommends.split(", "):
-                        dep.append("%s [REC]" % recommend) -            else:
-                # IPK doesn't include the filename
-                if not filename:
-                    filename = "%s_%s_%s.ipk" % (pkg, ver, arch) -                if pkg:
-                    output[pkg] = {"arch":arch, "ver":ver, -                            "filename":filename, "deps": dep } -                pkg = ""
-                filename = ""
-                dep = []

meta/lib/oe/package_manager.py +0 -10

-
-        if pkg:
-            if not filename:
-                filename = "%s_%s_%s.ipk" % (pkg, ver, arch)
-            output[pkg] = {"arch":arch, "ver":ver,
-                    "filename":filename, "deps": dep }
-
-        return output
-
-

 class RpmPkgsList(PkgsList):
     def __init__(self, d, rootfs_dir, arch_var=None, os_var=None):
         super(RpmPkgsList, self).__init__(d, rootfs_dir)

meta/lib/oe/package_manager.py +0 -0

@@ -479,7 +477,7 @@ class OpkgPkgsList(PkgsList):

             bb.fatal("Cannot get the installed packages list. Command '%s' "
                      "returned %d and stderr:\n%s" % (cmd, p.returncode, cmd_stderr))

-        return self.opkg_query(cmd_output)
+        return opkg_query(cmd_output)


 class DpkgPkgsList(PkgsList):

meta/lib/oe/package_manager.py +0 -0

@@ -497,7 +495,7 @@ class DpkgPkgsList(PkgsList):

             bb.fatal("Cannot get the installed packages list. Command '%s' "
                      "returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output))

-        return self.opkg_query(cmd_output)
+        return opkg_query(cmd_output)


 class PackageManager(object):
--
2.6.6

From: Mariano Lopez <[email protected]>

Sometimes it is needed to have the content of a package outside the recipe context.  This new method extract the content of an IPK/DEB file to a tmpdir, without actually installing the package.

A new OpkgDpkgPM class was added to share the code for opkg and dpkg.

There were need some changes to opkg_query() in order to use it with apt-cache output.

[YOCTO #9569]

Signed-off-by Mariano Lopez <[email protected]>

---
 meta/lib/oe/package_manager.py | 134
+++++++++++++++++++++++++++++++++++++++--

 1 file changed, 129 insertions(+), 5 deletions(-)

meta/lib/oe/package_manager.py +0 -0

index 427518d..0830da9 100644
@@ -38,6 +38,7 @@ def opkg_query(cmd_output):

     filename = ""
     dep = []
     pkg = ""
+    pkgarch = ""

     for line in cmd_output.splitlines():
         line = line.rstrip()
         if ':' in line:

meta/lib/oe/package_manager.py +0 -0

@@ -47,8 +48,10 @@ def opkg_query(cmd_output):

                 arch = line.split(": ")[1]
             elif line.startswith("Version: "):
                 ver = line.split(": ")[1]
-            elif line.startswith("File: "):
+            elif line.startswith("File: ") or line.startswith("Filename:"):
                 filename = line.split(": ")[1]
+                if "/" in filename:
+                    filename = os.path.basename(filename) elif line.startswith("Depends: "):
                 depends = verregex.sub('', line.split(": ")[1]) for depend in depends.split(", "):

meta/lib/oe/package_manager.py +0 -0

@@ -57,15 +60,20 @@ def opkg_query(cmd_output):
                  recommends = verregex.sub('', line.split(": ")[1])

                 for recommend in recommends.split(", "):
                     dep.append("%s [REC]" % recommend) -        else:
+            elif line.startswith("PackageArch: "):
+                pkgarch = line.split(": ")[1]
+
+        # When there is a blank line save the package information +        elif not line:
             # IPK doesn't include the filename
             if not filename:
                 filename = "%s_%s_%s.ipk" % (pkg, ver, arch) if pkg:
                 output[pkg] = {"arch":arch, "ver":ver, -                        "filename":filename, "deps": dep } +                        "filename":filename, "deps": dep, "pkgarch":pkgarch }
             pkg = ""
             filename = ""
+            pkgarch = ""
             dep = []

     if pkg:

meta/lib/oe/package_manager.py +0 -0

@@ -1397,7 +1405,70 @@ class RpmPM(PackageManager):
              bb.utils.remove(f, True)

meta/lib/oe/package_manager.py +16 -1

-class OpkgPM(PackageManager):
+class OpkgDpkgPM(PackageManager):
+    """
+    This is an abstract class. Do not instantiate this directly.
+    """
+    def __init__(self, d):
+        super(OpkgDpkgPM, self).__init__(d)
+
+    """
+    Returns a dictionary with the package info.
+
+    This method extracts the common parts for Opkg and Dpkg
+    """
+    def package_info(self, pkg, cmd):
+
+        try:
+            output = subprocess.check_output(cmd,

stderr=subprocess.STDOUT, shell=True)
+        except subprocess.CalledProcessError as e:
+            bb.fatal("Unable to list available packages. Command '%s' "
+                     "returned %d:\n%s" % (cmd, e.returncode, e.output))
+        return opkg_query(output)
+
+    """
+    Returns the path to a tmpdir where resides the contents of a package.

meta/lib/oe/package_manager.py +21 -0

+
+    Deleting the tmpdir is responsability of the caller.
+
+    This method extracts the common parts for Opkg and Dpkg
+    """
+    def extract(self, pkg, pkg_path):
+
+        ar_cmd = bb.utils.which(os.getenv("PATH"), "ar")
+        tar_cmd = bb.utils.which(os.getenv("PATH"), "tar")
+
+        if not os.path.isfile(pkg_path):
+            bb.fatal("Unable to extract package for '%s'."
+                     "File %s doesn't exists" % (pkg, pkg_path))
+
+        tmp_dir = tempfile.mkdtemp()
+        current_dir = os.getcwd()
+        os.chdir(tmp_dir)
+
+        try:
+            cmd = "%s x %s" % (ar_cmd, pkg_path)
+            output = subprocess.check_output(cmd,

stderr=subprocess.STDOUT, shell=True)
+            cmd = "%s xf data.tar.*" % tar_cmd
+            output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
+        except subprocess.CalledProcessError as e:
+            bb.utils.remove(tmp_dir, recurse=True)
+            bb.fatal("Unable to extract %s package. Command '%s' " +                     "returned %d:\n%s" % (pkg_path, cmd, e.returncode, e.output))
+        except OSError as e:
+            bb.utils.remove(tmp_dir, recurse=True)
+            bb.fatal("Unable to extract %s package. Command '%s' " +                     "returned %d:\n%s at %s" % (pkg_path, cmd, e.errno, e.strerror, e.filename))

meta/lib/oe/package_manager.py +10 -0

+
+        bb.note("Extracted %s to %s" % (pkg_path, tmp_dir))
+        bb.utils.remove(os.path.join(tmp_dir, "debian-binary"))
+        bb.utils.remove(os.path.join(tmp_dir, "control.tar.gz"))
+        os.chdir(current_dir)
+
+        return tmp_dir
+
+
+class OpkgPM(OpkgDpkgPM):

     def __init__(self, d, target_rootfs, config_file, archs, task_name='target'):
         super(OpkgPM, self).__init__(d)

meta/lib/oe/package_manager.py +0 -0

@@ -1732,8 +1803,34 @@ class OpkgPM(PackageManager):
                              self.opkg_dir,
                              symlinks=True)

+    """
+    Returns a dictionary with the package info.
+    """
+    def package_info(self, pkg):
+        cmd = "%s %s info %s" % (self.opkg_cmd, self.opkg_args, pkg) +        return super(OpkgPM, self).package_info(pkg, cmd) +
+    """
+    Returns the path to a tmpdir where resides the contents of a package.
+
+    Deleting the tmpdir is responsability of the caller. +    """
+    def extract(self, pkg):
+        pkg_info = self.package_info(pkg)
+        if not pkg_info:
+            bb.fatal("Unable to get information for package '%s' while "
+                     "trying to extract the package."  % pkg)
-class DpkgPM(PackageManager):
+        pkg_arch = pkg_info[pkg]["arch"]
+        pkg_filename = pkg_info[pkg]["filename"]
+        pkg_path = os.path.join(self.deploy_dir, pkg_arch, pkg_filename)

meta/lib/oe/package_manager.py +7 -0

+
+        tmp_dir = super(OpkgPM, self).extract(pkg, pkg_path)
+        bb.utils.remove(os.path.join(tmp_dir, "data.tar.gz"))
+
+        return tmp_dir
+
+class DpkgPM(OpkgDpkgPM):

     def __init__(self, d, target_rootfs, archs, base_archs, apt_conf_dir=None):
         super(DpkgPM, self).__init__(d) self.target_rootfs = target_rootfs

meta/lib/oe/package_manager.py +0 -0

@@ -1744,6 +1841,7 @@ class DpkgPM(PackageManager):

             self.apt_conf_dir = apt_conf_dir self.apt_conf_file = os.path.join(self.apt_conf_dir, "apt.conf")
         self.apt_get_cmd = bb.utils.which(os.getenv('PATH'), "apt- get")
+        self.apt_cache_cmd = bb.utils.which(os.getenv('PATH'), "apt- cache")

         self.apt_args = d.getVar("APT_ARGS", True)

meta/lib/oe/package_manager.py +0 -0

@@ -2027,6 +2125,32 @@ class DpkgPM(PackageManager):

     def list_installed(self):
         return DpkgPkgsList(self.d, self.target_rootfs).list_pkgs()
+    """
+    Returns a dictionary with the package info.
+    """
+    def package_info(self, pkg):
+        cmd = "%s show %s" % (self.apt_cache_cmd, pkg)
+        return super(DpkgPM, self).package_info(pkg, cmd) +
+    """
+    Returns the path to a tmpdir where resides the contents of a package.
+
+    Deleting the tmpdir is responsability of the caller. +    """
+    def extract(self, pkg):
+        pkg_info = self.package_info(pkg)
+        if not pkg_info:
+            bb.fatal("Unable to get information for package '%s' while "
+                     "trying to extract the package."  % pkg) +
+        pkg_arch = pkg_info[pkg]["pkgarch"]
+        pkg_filename = pkg_info[pkg]["filename"]
+        pkg_path = os.path.join(self.deploy_dir, pkg_arch, pkg_filename)

meta/lib/oe/package_manager.py +5 -0

+
+        tmp_dir = super(DpkgPM, self).extract(pkg, pkg_path)
+        bb.utils.remove(os.path.join(tmp_dir, "data.tar.xz"))
+
+        return tmp_dir

 def generate_index_files(d):
     classes = d.getVar('PACKAGE_CLASSES', True).replace("package_", "").split()
--
2.6.6

From: Mariano Lopez <[email protected]>

This new method extract the content of RPM file to a tmpdir, without actually installing the package.

[YOCTO #9569]

Signed-off-by Mariano Lopez <[email protected]>

---
 meta/lib/oe/package_manager.py | 80
++++++++++++++++++++++++++++++++++++++++++

 1 file changed, 80 insertions(+)

meta/lib/oe/package_manager.py +0 -0

index 0830da9..d95564a 100644
@@ -1404,6 +1404,86 @@ class RpmPM(PackageManager):

         for f in rpm_db_locks:
             bb.utils.remove(f, True)

+    """
+    Returns a dictionary with the package info.
+    """
+    def package_info(self, pkg):
+        cmd = "%s %s info --urls %s" % (self.smart_cmd, self.smart_opt, pkg)
+        try:
+            output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
+        except subprocess.CalledProcessError as e:
+            bb.fatal("Unable to list available packages. Command '%s' "
+                     "returned %d:\n%s" % (cmd, e.returncode, e.output))
+
+        #Parse output

+        for line in output.splitlines():
+            line = line.rstrip()
+            if line.startswith("Name:"):
+                pkg = line.split(": ")[1]
+            elif line.startswith("Version:"):
+                tmp_str = line.split(": ")[1]
+                ver, arch = tmp_str.split("@")
+                break

meta/lib/oe/package_manager.py +13 -0

+
+        # Get filename
+        index = re.search("^URLs", output, re.MULTILINE)
+        tmp_str = output[index.end():]
+        for line in tmp_str.splitlines():
+            if "/" in line:
+                line = line.lstrip()
+                filename = line.split(" ")[0]
+                break
+
+        # To have the same data type than other package_info methods
+        pkg_dict = {}
+        pkg_dict[pkg] = {"arch":arch, "ver":ver,

"filename":filename}

meta/lib/oe/package_manager.py +5 -0

+
+        return pkg_dict
+
+    """
+    Returns the path to a tmpdir where resides the contents of a

package.

meta/lib/oe/package_manager.py +8 -0

+
+    Deleting the tmpdir is responsability of the caller.
+
+    """
+    def extract(self, pkg):
+        pkg_info = self.package_info(pkg)
+        if not pkg_info:
+            bb.fatal("Unable to get information for package '%s'

while "
+                     "trying to extract the package."  % pkg) +
+        pkg_arch = pkg_info[pkg]["arch"]
+        pkg_filename = pkg_info[pkg]["filename"]
+        pkg_path = os.path.join(self.deploy_dir, pkg_arch, pkg_filename)

meta/lib/oe/package_manager.py +14 -0

+
+        cpio_cmd = bb.utils.which(os.getenv("PATH"), "cpio")
+        rpm2cpio_cmd = bb.utils.which(os.getenv("PATH"), "rpm2cpio")
+
+        if not os.path.isfile(pkg_path):
+            bb.fatal("Unable to extract package for '%s'."
+                     "File %s doesn't exists" % (pkg, pkg_path))
+
+        tmp_dir = tempfile.mkdtemp()
+        current_dir = os.getcwd()
+        os.chdir(tmp_dir)
+
+        try:
+            cmd = "%s %s | %s -idmv" % (rpm2cpio_cmd, pkg_path,

cpio_cmd)
+            output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
+        except subprocess.CalledProcessError as e:
+            bb.utils.remove(tmp_dir, recurse=True)
+            bb.fatal("Unable to extract %s package. Command '%s' " +                     "returned %d:\n%s" % (pkg_path, cmd, e.returncode, e.output))
+        except OSError as e:
+            bb.utils.remove(tmp_dir, recurse=True)
+            bb.fatal("Unable to extract %s package. Command '%s' " +                     "returned %d:\n%s at %s" % (pkg_path, cmd, e.errno, e.strerror, e.filename))

meta/lib/oe/package_manager.py +6 -0

+
+        bb.note("Extracted %s to %s" % (pkg_path, tmp_dir))
+        os.chdir(current_dir)
+
+        return tmp_dir
+

 class OpkgDpkgPM(PackageManager):
     """
--
2.6.6

meta/lib/oe/package_manager.py class RpmPM(PackageManager): · line 1426

Quoted 11 lines · On Wed, 2016-05-11 at 12:31 +0000, [email protected] wrote:

From: Mariano Lopez <[email protected]>

meta/lib/oe/package_manager.py +9 -0

+        #Parse output
+        for line in output.splitlines():
+            line = line.rstrip()
+            if line.startswith("Name:"):
+                pkg = line.split(": ")[1]
+            elif line.startswith("Version:"):
+                tmp_str = line.split(": ")[1]
+                ver, arch = tmp_str.split("@")
+                break

Could pkg, ver and arch be potentially undefined here?