fix package-index build issue
cover fix package-index build issue
After rootfs code refactoring, 'bitbake package-index' fails because it's calling the old bash indexing routines. This patchset should fix that.
laurentiu
The following changes since commit b60ed2d0fd86a3ae5ce7facbb044677aa7ec2889:
bitbake: runqueue: Ensure we do run 'masked' setscene tasks if specified as targets (2014-02-13 17:57:20 +0000)
are available in the git repository at:
git://mirror.rb.intel.com/git.yoctoproject.org/poky-contrib lpalcu/package_index_recipe_fix
for you to fetch changes up to db1e6488ef483ab76f7eca961b214bd8f8758b2e:
package_*.bbclass: remove references to the old bash indexing routines (2014-02-14 12:47:53 +0200)
----------------------------------------------------------------
Laurentiu Palcu (3):
package_manager.py, rootfs.py, sdk.py: add Indexer class package-index.bb: use the new python indexing routines package_*.bbclass: remove references to the old bash indexing routines
meta/classes/package_deb.bbclass | 1 - meta/classes/package_ipk.bbclass | 1 - meta/classes/package_rpm.bbclass | 1 - meta/lib/oe/package_manager.py | 339 ++++++++++++++++++++----------- meta/lib/oe/rootfs.py | 58 ------ meta/lib/oe/sdk.py | 48 +---- meta/recipes-core/meta/package-index.bb | 7 +- 7 files changed, 224 insertions(+), 231 deletions(-)
Laurentiu Palcu (3):
package_manager.py, rootfs.py, sdk.py: add Indexer class package-index.bb: use the new python indexing routines package_*.bbclass: remove references to the old bash indexing
routines
meta/classes/package_deb.bbclass | 1 - meta/classes/package_ipk.bbclass | 1 - meta/classes/package_rpm.bbclass | 1 - meta/lib/oe/package_manager.py | 339 ++++++++++++++++++++----------- meta/lib/oe/rootfs.py | 58 ------ meta/lib/oe/sdk.py | 48 +---- meta/recipes-core/meta/package-index.bb | 7 +- 7 files changed, 224 insertions(+), 231 deletions(-)
Signature · 1 line
1.7.9.5
PATCH 1/3 package_manager.py, rootfs.py, sdk.py: add Indexer class
Because the package-index.bb needs to create package indexes outside do_rootfs environment, move the indexing capability out of PackageManager class to a smaller Indexer class.
This commit:
* simply moves the indexing functions for ipk/deb with no changes;
* rewrites the RPM indexing function so that it can be easily moved out of the PackageManager class;
* removes some RPM duplicate code, moves it into a method inside RpmPM class and changes the RpmPM constructor so that the new method is effective;
--- meta/lib/oe/package_manager.py | 339 +++++++++++++++++++++++++--------------- meta/lib/oe/rootfs.py | 58 ------- meta/lib/oe/sdk.py | 48 +----- 3 files changed, 221 insertions(+), 224 deletions(-)
meta/lib/oe/package_manager.py +216 -123
@@ -22,6 +22,144 @@ def create_index(arg): return None +class Indexer(object): + __metaclass__ = ABCMeta + + def __init__(self, d, deploy_dir): + self.d = d + self.deploy_dir = deploy_dir + + @abstractmethod + def write_index(self): + pass + + +class RpmIndexer(Indexer): + def write_index(self): + sdk_pkg_archs = (self.d.getVar('SDK_PACKAGE_ARCHS', True) or "").replace('-', '_').split() + mlb_prefix_list = (self.d.getVar('MULTILIB_PREFIX_LIST', True) or "").replace('-', '_').split() + all_mlb_pkg_archs = (self.d.getVar('ALL_MULTILIB_PACKAGE_ARCHS', True) or "").replace('-', '_').split() + + archs = set() + for item in mlb_prefix_list: + archs = archs.union(set(item.split(':')[1:])) + + if len(archs) == 0: + archs = archs.union(set(all_mlb_pkg_archs)) + + archs = archs.union(set(sdk_pkg_archs)) + + rpm_createrepo = bb.utils.which(os.getenv('PATH'), "createrepo") + index_cmds = [] + rpm_dirs_found = False + for arch in archs: + arch_dir = os.path.join(self.deploy_dir, arch) + if not os.path.isdir(arch_dir): + continue + + index_cmds.append("%s --update -q %s" % (rpm_createrepo, arch_dir)) + + rpm_dirs_found = True + + if not rpm_dirs_found: + return("There are no packages in %s" % self.deploy_dir) + + nproc = multiprocessing.cpu_count() + pool = bb.utils.multiprocessingpool(nproc) + results = list(pool.imap(create_index, index_cmds)) + pool.close() + pool.join() + + for result in results: + if result is not None: + return(result) + + +class OpkgIndexer(Indexer): + def write_index(self): + arch_vars = ["ALL_MULTILIB_PACKAGE_ARCHS", + "SDK_PACKAGE_ARCHS", + "MULTILIB_ARCHS"] + + opkg_index_cmd = bb.utils.which(os.getenv('PATH'), "opkg-make-index") + + if not os.path.exists(os.path.join(self.deploy_dir, "Packages")): + open(os.path.join(self.deploy_dir, "Packages"), "w").close() + + index_cmds = [] + for arch_var in arch_vars: + archs = self.d.getVar(arch_var, True) + if archs is None: + continue + + for arch in archs.split(): + pkgs_dir = os.path.join(self.deploy_dir, arch) + pkgs_file = os.path.join(pkgs_dir, "Packages") + + if not os.path.isdir(pkgs_dir): + continue + + if not os.path.exists(pkgs_file): + open(pkgs_file, "w").close() + + index_cmds.append('%s -r %s -p %s -m %s' % + (opkg_index_cmd, pkgs_file, pkgs_file, pkgs_dir)) + + if len(index_cmds) == 0: + return("There are no packages in %s!" % self.deploy_dir) + + nproc = multiprocessing.cpu_count() + pool = bb.utils.multiprocessingpool(nproc) + results = list(pool.imap(create_index, index_cmds)) + pool.close() + pool.join() + + for result in results: + if result is not None: + return(result) + + +class DpkgIndexer(Indexer): + def write_index(self): + pkg_archs = self.d.getVar('PACKAGE_ARCHS', True) + if pkg_archs is not None: + arch_list = pkg_archs.split() + sdk_pkg_archs = self.d.getVar('SDK_PACKAGE_ARCHS', True) + if sdk_pkg_archs is not None: + arch_list += sdk_pkg_archs.split() + + dpkg_scanpackages = bb.utils.which(os.getenv('PATH'), "dpkg-scanpackages") + gzip = bb.utils.which(os.getenv('PATH'), "gzip") + + index_cmds = [] + deb_dirs_found = False + for arch in arch_list: + arch_dir = os.path.join(self.deploy_dir, arch) + if not os.path.isdir(arch_dir): + continue + + with open(os.path.join(arch_dir, "Release"), "w+") as release: + release.write("Label: %s" % arch) + + index_cmds.append("cd %s; %s . | %s > Packages.gz" % + (arch_dir, dpkg_scanpackages, gzip)) + + deb_dirs_found = True + + if not deb_dirs_found: + return("There are no packages in %s" % self.deploy_dir) + + nproc = multiprocessing.cpu_count() + pool = bb.utils.multiprocessingpool(nproc) + results = list(pool.imap(create_index, index_cmds)) + pool.close() + pool.join() + + for result in results: + if result is not None: + return(result) + + class PackageManager(object): """ This is an abstract class. Do not instantiate this directly.
@@ -136,14 +274,13 @@ class RpmPM(PackageManager): def __init__(self, d, target_rootfs, - package_archs, - target_os, target_vendor, task_name='target', - providename=None): + providename=None, + arch_var=None, + os_var=None): super(RpmPM, self).__init__(d) self.target_rootfs = target_rootfs - self.ml_os_list = target_os self.target_vendor = target_vendor self.task_name = task_name self.providename = providename
@@ -164,19 +301,58 @@ class RpmPM(PackageManager): if not os.path.exists(self.d.expand('${T}/saved')): bb.utils.mkdirhier(self.d.expand('${T}/saved')) - # arch order is reversed. This ensures the -best- match is - # listed first! - self.ml_prefix_list = dict() + self.indexer = RpmIndexer(self.d, self.deploy_dir) + + self.ml_prefix_list, self.ml_os_list = self._get_prefix_and_os_list(arch_var, os_var) + + def _get_prefix_and_os_list(self, arch_var, os_var): + package_archs = { + 'default': [], + } + + target_os = { + 'default': "", + } + + if arch_var is not None and os_var is not None: + package_archs['default'] = self.d.getVar(arch_var, True).split() + package_archs['default'].reverse() + target_os['default'] = self.d.getVar(os_var, True).strip() + else: + package_archs['default'] = self.d.getVar("PACKAGE_ARCHS", True).split() + # arch order is reversed. This ensures the -best- match is + # listed first! + package_archs['default'].reverse() + target_os['default'] = self.d.getVar("TARGET_OS", True).strip() + multilibs = self.d.getVar('MULTILIBS', True) or "" + for ext in multilibs.split(): + eext = ext.split(':') + if len(eext) > 1 and eext[0] == 'multilib': + localdata = bb.data.createCopy(self.d) + default_tune_key = "DEFAULTTUNE_virtclass-multilib-" + eext[1] + default_tune = localdata.getVar(default_tune_key, False) + if default_tune: + localdata.setVar("DEFAULTTUNE", default_tune) + bb.data.update_data(localdata) + package_archs[eext[1]] = localdata.getVar('PACKAGE_ARCHS', + True).split() + package_archs[eext[1]].reverse() + target_os[eext[1]] = localdata.getVar("TARGET_OS", + True).strip() + + ml_prefix_list = dict() for mlib in package_archs: if mlib == 'default': - self.ml_prefix_list[mlib] = package_archs[mlib] + ml_prefix_list[mlib] = package_archs[mlib] else: - self.ml_prefix_list[mlib] = list() + ml_prefix_list[mlib] = list() for arch in package_archs[mlib]: if arch in ['all', 'noarch', 'any']: - self.ml_prefix_list[mlib].append(arch) + ml_prefix_list[mlib].append(arch) else: - self.ml_prefix_list[mlib].append(mlib + "_" + arch) + ml_prefix_list[mlib].append(mlib + "_" + arch) + + return (ml_prefix_list, target_os) ''' Create configs for rpm and smart, and multilib is supported
@@ -552,39 +728,10 @@ class RpmPM(PackageManager): self._invoke_smart('upgrade') def write_index(self): - arch_list = set() - for mlib in self.ml_prefix_list: - for arch in self.ml_prefix_list[mlib]: - if arch not in arch_list: - arch_list.add(arch.replace('-', '_')) + result = self.indexer.write_index() - sdk_pkg_archs = (self.d.getVar('SDK_PACKAGE_ARCHS', True) or "").replace('-', '_') - arch_list = arch_list.union(set(sdk_pkg_archs.split())) - - rpm_createrepo = bb.utils.which(os.getenv('PATH'), "createrepo") - index_cmds = [] - rpm_dirs_found = False - for arch in arch_list: - arch_dir = os.path.join(self.deploy_dir, arch) - if not os.path.isdir(arch_dir): - continue - - index_cmds.append("%s --update -q %s" % (rpm_createrepo, arch_dir)) - - rpm_dirs_found = True - - if not rpm_dirs_found: - bb.fatal("There are no packages in %s" % self.deploy_dir) - - nproc = multiprocessing.cpu_count() - pool = bb.utils.multiprocessingpool(nproc) - results = list(pool.imap(create_index, index_cmds)) - pool.close() - pool.join() - - for result in results: - if result is not None: - bb.fatal(result) + if result is not None: + bb.fatal(result) def remove_packaging_data(self): bb.utils.remove(self.image_rpmlib, True)
@@ -810,6 +957,8 @@ class OpkgPM(PackageManager): else: self._create_custom_config() + self.indexer = OpkgIndexer(self.d, self.deploy_dir) + """ This function will change a package's status in /var/lib/opkg/status file. If 'packages' is None then the new_status will be applied to all
@@ -943,53 +1092,14 @@ class OpkgPM(PackageManager): "returned %d:\n%s" % (e.cmd, e.returncode, e.output)) def write_index(self): - arch_vars = ["ALL_MULTILIB_PACKAGE_ARCHS", - "SDK_PACKAGE_ARCHS", - "MULTILIB_ARCHS"] - - tmpdir = self.d.getVar('TMPDIR', True) - self.deploy_dir_lock() - opkg_index_cmd = bb.utils.which(os.getenv('PATH'), "opkg-make-index") - - if not os.path.exists(os.path.join(self.deploy_dir, "Packages")): - open(os.path.join(self.deploy_dir, "Packages"), "w").close() - - index_cmds = [] - for arch_var in arch_vars: - archs = self.d.getVar(arch_var, True) - if archs is None: - continue - - for arch in archs.split(): - pkgs_dir = os.path.join(self.deploy_dir, arch) - pkgs_file = os.path.join(pkgs_dir, "Packages") - - if not os.path.isdir(pkgs_dir): - continue - - if not os.path.exists(pkgs_file): - open(pkgs_file, "w").close() - - index_cmds.append('%s -r %s -p %s -m %s' % - (opkg_index_cmd, pkgs_file, pkgs_file, pkgs_dir)) - - if len(index_cmds) == 0: - self.deploy_dir_unlock() - bb.fatal("There are no packages in %s!" % self.deploy_dir) - - nproc = multiprocessing.cpu_count() - pool = bb.utils.multiprocessingpool(nproc) - results = list(pool.imap(create_index, index_cmds)) - pool.close() - pool.join() + result = self.indexer.write_index() self.deploy_dir_unlock() - for result in results: - if result is not None: - bb.fatal(result) + if result is not None: + bb.fatal(result) def remove_packaging_data(self): bb.utils.remove(self.opkg_dir, True)
@@ -1078,6 +1188,8 @@ class DpkgPM(PackageManager): self._create_configs(archs, base_archs) + self.indexer = DpkgIndexer(self.d, self.deploy_dir) + """ This function will change a package's status in /var/lib/dpkg/status file. If 'packages' is None then the new_status will be applied to all
@@ -1215,49 +1327,14 @@ class DpkgPM(PackageManager): "returned %d:\n%s" % (e.cmd, e.returncode, e.output)) def write_index(self): - tmpdir = self.d.getVar('TMPDIR', True) - - pkg_archs = self.d.getVar('PACKAGE_ARCHS', True) - if pkg_archs is not None: - arch_list = pkg_archs.split() - sdk_pkg_archs = self.d.getVar('SDK_PACKAGE_ARCHS', True) - if sdk_pkg_archs is not None: - arch_list += sdk_pkg_archs.split() - - dpkg_scanpackages = bb.utils.which(os.getenv('PATH'), "dpkg-scanpackages") - gzip = bb.utils.which(os.getenv('PATH'), "gzip") - self.deploy_dir_lock() - index_cmds = [] - deb_dirs_found = False - for arch in arch_list: - arch_dir = os.path.join(self.deploy_dir, arch) - if not os.path.isdir(arch_dir): - continue - - with open(os.path.join(arch_dir, "Release"), "w+") as release: - release.write("Label: %s" % arch) - - index_cmds.append("cd %s; %s . | %s > Packages.gz" % - (arch_dir, dpkg_scanpackages, gzip)) - - deb_dirs_found = True - - if not deb_dirs_found: - bb.fatal("There are no packages in %s" % self.deploy_dir) - - nproc = multiprocessing.cpu_count() - pool = bb.utils.multiprocessingpool(nproc) - results = list(pool.imap(create_index, index_cmds)) - pool.close() - pool.join() + result = self.indexer.write_index() self.deploy_dir_unlock() - for result in results: - if result is not None: - bb.fatal(result) + if result is not None: + bb.fatal(result) def _create_configs(self, archs, base_archs): base_archs = re.sub("_", "-", base_archs)
@@ -1365,6 +1442,22 @@ class DpkgPM(PackageManager): return output + +def generate_index_files(d): + img_type = d.getVar('IMAGE_PKGTYPE', True) + + result = None + + if img_type == "rpm": + result = RpmIndexer(d, d.getVar('DEPLOY_DIR_RPM', True)).write_index() + elif img_type == "ipk": + result = OpkgIndexer(d, d.getVar('DEPLOY_DIR_IPK', True)).write_index() + elif img_type == "deb": + result = DpkgIndexer(d, d.getVar('DEPLOY_DIR_DEB', True)).write_index() + + if result is not None: + bb.fatal(result) + if __name__ == "__main__": """ We should be able to run this as a standalone script, from outside bitbake
meta/lib/oe/rootfs.py +0 -58
74 lines not on this page · raw message
meta/lib/oe/sdk.py +5 -43
77 lines not on this page · raw message
Signature · 1 line
1.7.9.5
PATCH 2/3 package-index.bb: use the new python indexing routines
In reply to Laurentiu Palcu--- meta/recipes-core/meta/package-index.bb | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-)
meta/recipes-core/meta/package-index.bb +3 -4
@@ -21,10 +21,9 @@ do_populate_sysroot[noexec] = "1" do_package_index[nostamp] = "1" do_package_index[depends] += "${PACKAGEINDEXDEPS}" -do_package_index() { - set -ex - ${PACKAGEINDEXES} - set +ex +python do_package_index() { + from oe.rootfs import generate_index_files + generate_index_files(d) } addtask do_package_index before do_build EXCLUDE_FROM_WORLD = "1"
Signature · 1 line
1.7.9.5
PATCH 3/3 package_*.bbclass: remove references to the old bash indexing routines
In reply to Laurentiu PalcuPackage indexing is done in python and package-index.bb uses the new routines.
--- meta/classes/package_deb.bbclass | 1 - meta/classes/package_ipk.bbclass | 1 - meta/classes/package_rpm.bbclass | 1 - 3 files changed, 3 deletions(-)
meta/classes/package_deb.bbclass +0 -1
@@ -309,6 +309,5 @@ do_package_write_deb[umask] = "022" addtask package_write_deb before do_package_write after do_packagedata do_package -PACKAGEINDEXES += "[ ! -e ${DEPLOY_DIR_DEB} ] || package_update_index_deb;" PACKAGEINDEXDEPS += "dpkg-native:do_populate_sysroot" PACKAGEINDEXDEPS += "apt-native:do_populate_sysroot"
meta/classes/package_ipk.bbclass +0 -1
@@ -255,6 +255,5 @@ do_package_write_ipk[cleandirs] = "${PKGWRITEDIRIPK}" do_package_write_ipk[umask] = "022" addtask package_write_ipk before do_package_write after do_packagedata do_package -PACKAGEINDEXES += "[ ! -e ${DEPLOY_DIR_IPK} ] || package_update_index_ipk;" PACKAGEINDEXDEPS += "opkg-utils-native:do_populate_sysroot" PACKAGEINDEXDEPS += "opkg-native:do_populate_sysroot"
meta/classes/package_rpm.bbclass +0 -1
@@ -734,6 +734,5 @@ do_package_write_rpm[cleandirs] = "${PKGWRITEDIRRPM}" do_package_write_rpm[umask] = "022" addtask package_write_rpm before do_package_write after do_packagedata do_package -PACKAGEINDEXES += "[ ! -e ${DEPLOY_DIR_RPM} ] || package_update_index_rpm;" PACKAGEINDEXDEPS += "rpm-native:do_populate_sysroot" PACKAGEINDEXDEPS += "createrepo-native:do_populate_sysroot"
Signature · 1 line
1.7.9.5