Add extract() method to package manager
v2 cover Add extract() method to package manager
From: Mariano Lopez <[email protected]>
Sometimes it is needed to have the content of a package outside of an image, these series add this capability using a temp directory.
Changes in v2:
- Add default values when parsing package information. This way it won't throw an exception when trying to save an undefined variable
The following changes since commit 28433319ad8299aa23b1fcfdddbe100b29e86517:
bitbake: toaster: tests browser Add test for creating a project (2016-05-11 11:32:58 +0100)
are available in the git repository at:
git://git.yoctoproject.org/poky-contrib mariano/bug9569v2
http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=mariano/bug9569v2
Mariano Lopez (3):
package_manager.py: Move opkg_query() outside of Indexer class package_manager.py: Add extract() method for opkg and dpkg package_manager.py: Add extract() method for RPM package manager
meta/lib/oe/package_manager.py | 321 ++++++++++++++++++++++++++++++++++------- 1 file changed, 266 insertions(+), 55 deletions(-)
Signature · 1 line
2.6.6
PATCH v2 1/3 package_manager.py: Move opkg_query() outside of Indexer class
From: Mariano Lopez <[email protected]>
When using the opkg and apt-get package managers the function opkg_query() can be useful when query for package information.
This change moves the function outside the Indexer class so the Indexer, OpkgPM, DpkgPM can benefit from it.
[YOCTO #9569]
--- meta/lib/oe/package_manager.py | 104 ++++++++++++++++++++--------------------- 1 file changed, 51 insertions(+), 53 deletions(-)
meta/lib/oe/package_manager.py +51 -53
@@ -27,6 +27,55 @@ def create_index(arg): return None +""" +This method parse the output from the package managerand return +a dictionary with the information of the packages. This is used +when the packages are in deb or ipk format. +""" +def opkg_query(cmd_output): + verregex = re.compile(' \([=<>]* [^ )]*\)') + output = dict() + filename = "" + dep = [] + pkg = "" + for line in cmd_output.splitlines(): + line = line.rstrip() + if ':' in line: + if line.startswith("Package: "): + pkg = line.split(": ")[1] + elif line.startswith("Architecture: "): + arch = line.split(": ")[1] + elif line.startswith("Version: "): + ver = line.split(": ")[1] + elif line.startswith("File: "): + filename = line.split(": ")[1] + elif line.startswith("Depends: "): + depends = verregex.sub('', line.split(": ")[1]) + for depend in depends.split(", "): + dep.append(depend) + elif line.startswith("Recommends: "): + recommends = verregex.sub('', line.split(": ")[1]) + for recommend in recommends.split(", "): + dep.append("%s [REC]" % recommend) + else: + # IPK doesn't include the filename + if not filename: + filename = "%s_%s_%s.ipk" % (pkg, ver, arch) + if pkg: + output[pkg] = {"arch":arch, "ver":ver, + "filename":filename, "deps": dep } + pkg = "" + filename = "" + dep = [] + + if pkg: + if not filename: + filename = "%s_%s_%s.ipk" % (pkg, ver, arch) + output[pkg] = {"arch":arch, "ver":ver, + "filename":filename, "deps": dep } + + return output + class Indexer(object): __metaclass__ = ABCMeta
@@ -293,57 +342,6 @@ class PkgsList(object): pass - """ - This method parse the output from the package manager - and return a dictionary with the information of the - installed packages. This is used whne the packages are - in deb or ipk format - """ - def opkg_query(self, cmd_output): - verregex = re.compile(' \([=<>]* [^ )]*\)') - output = dict() - filename = "" - dep = [] - pkg = "" - for line in cmd_output.splitlines(): - line = line.rstrip() - if ':' in line: - if line.startswith("Package: "): - pkg = line.split(": ")[1] - elif line.startswith("Architecture: "): - arch = line.split(": ")[1] - elif line.startswith("Version: "): - ver = line.split(": ")[1] - elif line.startswith("File: "): - filename = line.split(": ")[1] - elif line.startswith("Depends: "): - depends = verregex.sub('', line.split(": ")[1]) - for depend in depends.split(", "): - dep.append(depend) - elif line.startswith("Recommends: "): - recommends = verregex.sub('', line.split(": ")[1]) - for recommend in recommends.split(", "): - dep.append("%s [REC]" % recommend) - else: - # IPK doesn't include the filename - if not filename: - filename = "%s_%s_%s.ipk" % (pkg, ver, arch) - if pkg: - output[pkg] = {"arch":arch, "ver":ver, - "filename":filename, "deps": dep } - pkg = "" - filename = "" - dep = [] - - if pkg: - if not filename: - filename = "%s_%s_%s.ipk" % (pkg, ver, arch) - output[pkg] = {"arch":arch, "ver":ver, - "filename":filename, "deps": dep } - - return output - - class RpmPkgsList(PkgsList): def __init__(self, d, rootfs_dir, arch_var=None, os_var=None): super(RpmPkgsList, self).__init__(d, rootfs_dir)
@@ -479,7 +477,7 @@ class OpkgPkgsList(PkgsList): bb.fatal("Cannot get the installed packages list. Command '%s' " "returned %d and stderr:\n%s" % (cmd, p.returncode, cmd_stderr)) - return self.opkg_query(cmd_output) + return opkg_query(cmd_output) class DpkgPkgsList(PkgsList):
@@ -497,7 +495,7 @@ class DpkgPkgsList(PkgsList): bb.fatal("Cannot get the installed packages list. Command '%s' " "returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output)) - return self.opkg_query(cmd_output) + return opkg_query(cmd_output) class PackageManager(object):
Signature · 1 line
2.6.6
PATCH v2 2/3 package_manager.py: Add extract() method for opkg and dpkg
In reply to [email protected]From: Mariano Lopez <[email protected]>
Sometimes it is needed to have the content of a package outside the recipe context. This new method extract the content of an IPK/DEB file to a tmpdir, without actually installing the package.
A new OpkgDpkgPM class was added to share the code for opkg and dpkg.
There were need some changes to opkg_query() in order to use it with apt-cache output. Also set default values to avoid UnboundLocalError
[YOCTO #9569]
--- meta/lib/oe/package_manager.py | 140 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 134 insertions(+), 6 deletions(-)
meta/lib/oe/package_manager.py +134 -6
@@ -35,9 +35,12 @@ when the packages are in deb or ipk format. def opkg_query(cmd_output): verregex = re.compile(' \([=<>]* [^ )]*\)') output = dict() + pkg = "" + arch = "" + ver = "" filename = "" dep = [] - pkg = "" + pkgarch = "" for line in cmd_output.splitlines(): line = line.rstrip() if ':' in line:
@@ -47,8 +50,10 @@ def opkg_query(cmd_output): arch = line.split(": ")[1] elif line.startswith("Version: "): ver = line.split(": ")[1] - elif line.startswith("File: "): + elif line.startswith("File: ") or line.startswith("Filename:"): filename = line.split(": ")[1] + if "/" in filename: + filename = os.path.basename(filename) elif line.startswith("Depends: "): depends = verregex.sub('', line.split(": ")[1]) for depend in depends.split(", "):
@@ -57,16 +62,23 @@ def opkg_query(cmd_output): recommends = verregex.sub('', line.split(": ")[1]) for recommend in recommends.split(", "): dep.append("%s [REC]" % recommend) - else: + elif line.startswith("PackageArch: "): + pkgarch = line.split(": ")[1] + + # When there is a blank line save the package information + elif not line: # IPK doesn't include the filename if not filename: filename = "%s_%s_%s.ipk" % (pkg, ver, arch) if pkg: output[pkg] = {"arch":arch, "ver":ver, - "filename":filename, "deps": dep } + "filename":filename, "deps": dep, "pkgarch":pkgarch } pkg = "" + arch = "" + ver = "" filename = "" dep = [] + pkgarch = "" if pkg: if not filename:
@@ -1397,7 +1409,70 @@ class RpmPM(PackageManager): bb.utils.remove(f, True) -class OpkgPM(PackageManager): +class OpkgDpkgPM(PackageManager): + """ + This is an abstract class. Do not instantiate this directly. + """ + def __init__(self, d): + super(OpkgDpkgPM, self).__init__(d) + + """ + Returns a dictionary with the package info. + + This method extracts the common parts for Opkg and Dpkg + """ + def package_info(self, pkg, cmd): + + try: + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) + except subprocess.CalledProcessError as e: + bb.fatal("Unable to list available packages. Command '%s' " + "returned %d:\n%s" % (cmd, e.returncode, e.output)) + return opkg_query(output) + + """ + Returns the path to a tmpdir where resides the contents of a package. + + Deleting the tmpdir is responsability of the caller. + + This method extracts the common parts for Opkg and Dpkg + """ + def extract(self, pkg, pkg_path): + + ar_cmd = bb.utils.which(os.getenv("PATH"), "ar") + tar_cmd = bb.utils.which(os.getenv("PATH"), "tar") + + if not os.path.isfile(pkg_path): + bb.fatal("Unable to extract package for '%s'." + "File %s doesn't exists" % (pkg, pkg_path)) + + tmp_dir = tempfile.mkdtemp() + current_dir = os.getcwd() + os.chdir(tmp_dir) + + try: + cmd = "%s x %s" % (ar_cmd, pkg_path) + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) + cmd = "%s xf data.tar.*" % tar_cmd + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) + except subprocess.CalledProcessError as e: + bb.utils.remove(tmp_dir, recurse=True) + bb.fatal("Unable to extract %s package. Command '%s' " + "returned %d:\n%s" % (pkg_path, cmd, e.returncode, e.output)) + except OSError as e: + bb.utils.remove(tmp_dir, recurse=True) + bb.fatal("Unable to extract %s package. Command '%s' " + "returned %d:\n%s at %s" % (pkg_path, cmd, e.errno, e.strerror, e.filename)) + + bb.note("Extracted %s to %s" % (pkg_path, tmp_dir)) + bb.utils.remove(os.path.join(tmp_dir, "debian-binary")) + bb.utils.remove(os.path.join(tmp_dir, "control.tar.gz")) + os.chdir(current_dir) + + return tmp_dir + + +class OpkgPM(OpkgDpkgPM): def __init__(self, d, target_rootfs, config_file, archs, task_name='target'): super(OpkgPM, self).__init__(d)
@@ -1732,8 +1807,34 @@ class OpkgPM(PackageManager): self.opkg_dir, symlinks=True) + """ + Returns a dictionary with the package info. + """ + def package_info(self, pkg): + cmd = "%s %s info %s" % (self.opkg_cmd, self.opkg_args, pkg) + return super(OpkgPM, self).package_info(pkg, cmd) + + """ + Returns the path to a tmpdir where resides the contents of a package. + + Deleting the tmpdir is responsability of the caller. + """ + def extract(self, pkg): + pkg_info = self.package_info(pkg) + if not pkg_info: + bb.fatal("Unable to get information for package '%s' while " + "trying to extract the package." % pkg) + + pkg_arch = pkg_info[pkg]["arch"] + pkg_filename = pkg_info[pkg]["filename"] + pkg_path = os.path.join(self.deploy_dir, pkg_arch, pkg_filename) + + tmp_dir = super(OpkgPM, self).extract(pkg, pkg_path) + bb.utils.remove(os.path.join(tmp_dir, "data.tar.gz")) + + return tmp_dir -class DpkgPM(PackageManager): +class DpkgPM(OpkgDpkgPM): def __init__(self, d, target_rootfs, archs, base_archs, apt_conf_dir=None): super(DpkgPM, self).__init__(d) self.target_rootfs = target_rootfs
@@ -1744,6 +1845,7 @@ class DpkgPM(PackageManager): self.apt_conf_dir = apt_conf_dir self.apt_conf_file = os.path.join(self.apt_conf_dir, "apt.conf") self.apt_get_cmd = bb.utils.which(os.getenv('PATH'), "apt-get") + self.apt_cache_cmd = bb.utils.which(os.getenv('PATH'), "apt-cache") self.apt_args = d.getVar("APT_ARGS", True)
@@ -2027,6 +2129,32 @@ class DpkgPM(PackageManager): def list_installed(self): return DpkgPkgsList(self.d, self.target_rootfs).list_pkgs() + """ + Returns a dictionary with the package info. + """ + def package_info(self, pkg): + cmd = "%s show %s" % (self.apt_cache_cmd, pkg) + return super(DpkgPM, self).package_info(pkg, cmd) + + """ + Returns the path to a tmpdir where resides the contents of a package. + + Deleting the tmpdir is responsability of the caller. + """ + def extract(self, pkg): + pkg_info = self.package_info(pkg) + if not pkg_info: + bb.fatal("Unable to get information for package '%s' while " + "trying to extract the package." % pkg) + + pkg_arch = pkg_info[pkg]["pkgarch"] + pkg_filename = pkg_info[pkg]["filename"] + pkg_path = os.path.join(self.deploy_dir, pkg_arch, pkg_filename) + + tmp_dir = super(DpkgPM, self).extract(pkg, pkg_path) + bb.utils.remove(os.path.join(tmp_dir, "data.tar.xz")) + + return tmp_dir def generate_index_files(d): classes = d.getVar('PACKAGE_CLASSES', True).replace("package_", "").split()
Signature · 1 line
2.6.6
PATCH v2 3/3 package_manager.py: Add extract() method for RPM package manager
In reply to [email protected]From: Mariano Lopez <[email protected]>
This new method extract the content of RPM file to a tmpdir, without actually installing the package.
[YOCTO #9569]
--- meta/lib/oe/package_manager.py | 85 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+)
meta/lib/oe/package_manager.py +85 -0
@@ -1408,6 +1408,91 @@ class RpmPM(PackageManager): for f in rpm_db_locks: bb.utils.remove(f, True) + """ + Returns a dictionary with the package info. + """ + def package_info(self, pkg): + cmd = "%s %s info --urls %s" % (self.smart_cmd, self.smart_opt, pkg) + try: + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) + except subprocess.CalledProcessError as e: + bb.fatal("Unable to list available packages. Command '%s' " + "returned %d:\n%s" % (cmd, e.returncode, e.output)) + + # Set default values to avoid UnboundLocalError + arch = "" + ver = "" + filename = "" + + #Parse output + for line in output.splitlines(): + line = line.rstrip() + if line.startswith("Name:"): + pkg = line.split(": ")[1] + elif line.startswith("Version:"): + tmp_str = line.split(": ")[1] + ver, arch = tmp_str.split("@") + break + + # Get filename + index = re.search("^URLs", output, re.MULTILINE) + tmp_str = output[index.end():] + for line in tmp_str.splitlines(): + if "/" in line: + line = line.lstrip() + filename = line.split(" ")[0] + break + + # To have the same data type than other package_info methods + pkg_dict = {} + pkg_dict[pkg] = {"arch":arch, "ver":ver, "filename":filename} + + return pkg_dict + + """ + Returns the path to a tmpdir where resides the contents of a package. + + Deleting the tmpdir is responsability of the caller. + + """ + def extract(self, pkg): + pkg_info = self.package_info(pkg) + if not pkg_info: + bb.fatal("Unable to get information for package '%s' while " + "trying to extract the package." % pkg) + + pkg_arch = pkg_info[pkg]["arch"] + pkg_filename = pkg_info[pkg]["filename"] + pkg_path = os.path.join(self.deploy_dir, pkg_arch, pkg_filename) + + cpio_cmd = bb.utils.which(os.getenv("PATH"), "cpio") + rpm2cpio_cmd = bb.utils.which(os.getenv("PATH"), "rpm2cpio") + + if not os.path.isfile(pkg_path): + bb.fatal("Unable to extract package for '%s'." + "File %s doesn't exists" % (pkg, pkg_path)) + + tmp_dir = tempfile.mkdtemp() + current_dir = os.getcwd() + os.chdir(tmp_dir) + + try: + cmd = "%s %s | %s -idmv" % (rpm2cpio_cmd, pkg_path, cpio_cmd) + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) + except subprocess.CalledProcessError as e: + bb.utils.remove(tmp_dir, recurse=True) + bb.fatal("Unable to extract %s package. Command '%s' " + "returned %d:\n%s" % (pkg_path, cmd, e.returncode, e.output)) + except OSError as e: + bb.utils.remove(tmp_dir, recurse=True) + bb.fatal("Unable to extract %s package. Command '%s' " + "returned %d:\n%s at %s" % (pkg_path, cmd, e.errno, e.strerror, e.filename)) + + bb.note("Extracted %s to %s" % (pkg_path, tmp_dir)) + os.chdir(current_dir) + + return tmp_dir + class OpkgDpkgPM(PackageManager): """
Signature · 1 line
2.6.6