do_rootfs: Added PACKAGE_FEED_URIS functionality
PATCH 1/2 do_rootfs: Added PACKAGE_FEED_URIS functionality
Adding a common interface to add predefined package manager channels to prebuilt rootfs:es.
Adding PACKAGE_FEED_URIS = "http://myre.po/repo/, will assume repo directories named (rpm,ipk,deb) as subdirectories and statically add them to the rootfs, using the same PKG_ARCHs as the build which produced the images.
Tested with RPM, IPK and DEB.
deb feed functionality seem broken, is anyone using this ?
--- meta/lib/oe/package_manager.py | 89 +++++++++++++++++++++++++++++++++++++++++- meta/lib/oe/rootfs.py | 16 ++------ 2 files changed, 92 insertions(+), 13 deletions(-)
meta/lib/oe/package_manager.py +88 -1
@@ -223,6 +223,7 @@ class PackageManager(object): self.d = d self.deploy_dir = None self.deploy_lock = None + self.feed_uris = self.d.getVar('PACKAGE_FEED_URIS', True) or "" """ Update the package manager package database.
@@ -262,6 +263,10 @@ class PackageManager(object): def list_installed(self, format=None): pass + @abstractmethod + def insert_feeds_uris(self): + pass + """ Install complementary packages based upon the list of currently installed packages e.g. locales, *-dev, *-dbg, etc. This will only attempt to install
@@ -358,6 +363,46 @@ class RpmPM(PackageManager): self.ml_prefix_list, self.ml_os_list = self.indexer.get_ml_prefix_and_os_list(arch_var, os_var) + + def insert_feeds_uris(self): + if self.feed_uris == "": + return + + # List must be prefered to least preferred order + default_platform_extra = set() + platform_extra = set() + bbextendvariant = self.d.getVar('BBEXTENDVARIANT', True) or "" + for mlib in self.ml_os_list: + for arch in self.ml_prefix_list[mlib]: + plt = arch.replace('-', '_') + '-.*-' + self.ml_os_list[mlib] + if mlib == bbextendvariant: + default_platform_extra.add(plt) + else: + platform_extra.add(plt) + + platform_extra = platform_extra.union(default_platform_extra) + + arch_list = [] + for canonical_arch in platform_extra: + arch = canonical_arch.split('-')[0] + if not os.path.exists(os.path.join(self.deploy_dir, arch)): + continue + arch_list.append(arch) + + uri_iterator = 0 + channel_priority = 10 + 5 * len(self.feed_uris.split()) * len(arch_list) + + for uri in self.feed_uris.split(): + for arch in arch_list: + bb.note('Note: adding Smart channel url%d%s (%s)' % + (uri_iterator, arch, channel_priority)) + self._invoke_smart('channel --add url%d-%s type=rpm-md baseurl=%s/rpm/%s -y' + % (uri_iterator, arch, uri, arch)) + self._invoke_smart('channel --set url%d-%s priority=%d' % + (uri_iterator, arch, channel_priority)) + channel_priority -= 5 + uri_iterator += 1 + ''' Create configs for rpm and smart, and multilib is supported '''
@@ -944,7 +989,6 @@ class OpkgPM(PackageManager): self.deploy_dir = self.d.getVar("DEPLOY_DIR_IPK", True) self.deploy_lock_file = os.path.join(self.deploy_dir, "deploy.lock") - self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg-cl") self.opkg_args = "-f %s -o %s " % (self.config_file, target_rootfs) self.opkg_args += self.d.getVar("OPKG_ARGS", True)
@@ -1050,6 +1094,29 @@ class OpkgPM(PackageManager): config_file.write("src oe-%s file:%s\n" % (arch, pkgs_dir)) + def insert_feeds_uris(self): + if self.feed_uris == "": + return + + rootfs_config = os.path.join('%s/etc/opkg/base-feeds.conf' + % self.target_rootfs) + + with open(rootfs_config, "w+") as config_file: + uri_iterator = 0 + for uri in self.feed_uris.split(): + config_file.write("src/gz url-%d %s/ipk\n" % + (uri_iterator, uri)) + + for arch in self.pkg_archs.split(): + if not os.path.exists(os.path.join(self.deploy_dir, arch)): + continue + bb.note('Note: adding opkg channel url-%s-%d (%s)' % + (arch, uri_iterator, uri)) + + config_file.write("src/gz uri-%s-%d %s/ipk/%s\n" % + (arch, uri_iterator, uri, arch)) + uri_iterator += 1 + def update(self): self.deploy_dir_lock()
@@ -1410,6 +1477,26 @@ class DpkgPM(PackageManager): if result is not None: bb.fatal(result) + def insert_feeds_uris(self): + if self.feed_uris == "": + return + + sources_conf = os.path.join("%s/etc/apt/sources.list" + % self.target_rootfs) + arch_list = [] + archs = self.d.getVar('PACKAGE_ARCHS', True) + for arch in archs.split(): + if not os.path.exists(os.path.join(self.deploy_dir, arch)): + continue + arch_list.append(arch) + + with open(sources_conf, "w+") as sources_file: + for uri in self.feed_uris.split(): + for arch in arch_list: + bb.note('Note: adding dpkg channel at (%s)' % uri) + sources_file.write("deb %s/deb/%s ./\n" % + (uri, arch)) + def _create_configs(self, archs, base_archs): base_archs = re.sub("_", "-", base_archs)
meta/lib/oe/rootfs.py +4 -12
@@ -41,9 +41,10 @@ class Rootfs(object): def _log_check(self): pass - @abstractmethod def _insert_feed_uris(self): - pass + if base_contains("IMAGE_FEATURES", "package-management", + True, False, self.d): + self.pm.insert_feeds_uris() @abstractmethod def _handle_intercept_failure(self, failed_script):
@@ -349,9 +350,6 @@ class RpmRootfs(Rootfs): if found_error == 6: bb.fatal(message) - def _insert_feed_uris(self): - pass - def _handle_intercept_failure(self, registered_pkgs): rpm_postinsts_dir = self.image_rootfs + self.d.expand('${sysconfdir}/rpm-postinsts/') bb.utils.mkdirhier(rpm_postinsts_dir)
@@ -372,6 +370,7 @@ class DpkgRootfs(Rootfs): d.getVar('PACKAGE_ARCHS', True), d.getVar('DPKG_ARCH', True)) + def _create(self): pkgs_to_install = self.manifest.parse_initial_manifest()
@@ -432,9 +431,6 @@ class DpkgRootfs(Rootfs): def _log_check(self): pass - def _insert_feed_uris(self): - pass - class OpkgRootfs(Rootfs): def __init__(self, d, manifest_dir):
@@ -698,10 +694,6 @@ class OpkgRootfs(Rootfs): def _log_check(self): pass - def _insert_feed_uris(self): - pass - - def create_rootfs(d, manifest_dir=None): env_bkp = os.environ.copy()
Signature · 1 line
1.8.3.2
PATCH 2/2 do_rootfs: Add PACKAGE_FEED_URIS as a vardep
A bit uncertain on where to put this, suggestions welcome. Needed to automatically rebuild do_rootfs when PACKAGE_FEED_URIS change.
--- meta/classes/rootfs_deb.bbclass | 1 + meta/classes/rootfs_ipk.bbclass | 1 + meta/classes/rootfs_rpm.bbclass | 1 + 3 files changed, 3 insertions(+)
meta/classes/rootfs_deb.bbclass +1 -0
@@ -8,6 +8,7 @@ ROOTFS_PKGMANAGE_BOOTSTRAP = "run-postinsts" do_rootfs[depends] += "dpkg-native:do_populate_sysroot apt-native:do_populate_sysroot" do_rootfs[recrdeptask] += "do_package_write_deb" rootfs_deb_do_rootfs[vardepsexclude] += "BUILDNAME" +do_rootfs[vardeps] += "PACKAGE_FEED_URIS" do_rootfs[lockfiles] += "${DEPLOY_DIR_DEB}/deb.lock"
meta/classes/rootfs_ipk.bbclass +1 -0
@@ -11,6 +11,7 @@ ROOTFS_PKGMANAGE_BOOTSTRAP = "run-postinsts" do_rootfs[depends] += "opkg-native:do_populate_sysroot opkg-utils-native:do_populate_sysroot" do_rootfs[recrdeptask] += "do_package_write_ipk" +do_rootfs[vardeps] += "PACKAGE_FEED_URIS" rootfs_ipk_do_rootfs[vardepsexclude] += "BUILDNAME" do_rootfs[lockfiles] += "${WORKDIR}/ipk.lock"
meta/classes/rootfs_rpm.bbclass +1 -0
@@ -23,6 +23,7 @@ do_rootfs[depends] += "createrepo-native:do_populate_sysroot" do_rootfs[recrdeptask] += "do_package_write_rpm" rootfs_rpm_do_rootfs[vardepsexclude] += "BUILDNAME" +do_rootfs[vardeps] += "PACKAGE_FEED_URIS" # RPM doesn't work with multiple rootfs generation at once due to collisions in the use of files # in ${DEPLOY_DIR_RPM}. This can be removed if package_update_index_rpm can be called concurrently
Signature · 1 line
1.8.3.2
Quoted 9 lines · On Thursday 27 February 2014 21:20:37 David Nyström wrote:
Adding a common interface to add predefined package manager channels to prebuilt rootfs:es.
Adding
PACKAGE_FEED_URIS= "http://myre.po/repo/, will assume repo directories named (rpm,ipk,deb) as subdirectories and statically add them to the rootfs, using the samePKG_ARCHsas the build which produced the images.Tested with RPM, IPK and DEB.
Looks good. The only thing that looks like it might be missing here is there appears to be no way to name the package repo entries individually, though I'm not sure what syntax we would use to allow that.
Thoughts?
Cheers,
Paul
Signature · 2 lines
Paul Eggleton Intel Open Source Technology Centre
Quoted 9 lines · Paul Eggleton
Quoted 8 lines · On Thursday 27 February 2014 21:20:37 David Nyström wrote:
Adding a common interface to add predefined package manager channels to prebuilt rootfs:es.
Adding
PACKAGE_FEED_URIS= "http://myre.po/repo/, will assume repo directories named (rpm,ipk,deb) as subdirectories and statically add them to the rootfs, using the samePKG_ARCHsas the build which produced the images.
Quoted 7 lines
Quoted 1 line
Tested with RPM, IPK and DEB.
Looks good. The only thing that looks like it might be missing here is there appears to be no way to name the package repo entries individually, though I'm not sure what syntax we would use to allow that.
Thoughts?
Well, there is the old moblin syntax for IPK_FEED_URIS:
For qemux86-64
--IPK_FEED_URIS = " \
all##http://downloads.yoctoproject.org/releases/yocto/yocto-1.5/ipk/all \
x86_64##http://downloads.yoctoproject.org/releases/yocto/yocto-1.5/ipk/x86_64 \
core2-64##http://downloads.yoctoproject.org/releases/yocto/yocto-1.5/ipk/core2-64 \
qemux86_64##http://downloads.yoctoproject.org/releases/yocto/yocto-1.5/ipk/qemux86_64 \
--
I don't particularly agree with above format.
When using the deploy directory as input it makes the feed_syntax MACHINE dependent, and difficult to keep in sync for new machines and upstream changes.
IMHO, bitbake should support the same input format as it generates as output format for the repos. If someone want to share code for the post-processing of outputted bitbake repos(deb,ipk,rpm), I'll happily reconsider.
Having multiple feed formats is confusing though, we should fix this so we have the same format everywhere.
What could be done is to add names to the top level repos directory, i.e. PACKAGE_FEED_URIS ="stable##http://myre.op/deploy/ \
"experimental#http://not_trusted.to/ext_deploy/"
But I'm not sure what this will accomplish. The real channel names would become experimental-all, stable-all instead of current hardcoded url0-all, url1-all et.c. Is this what you were suggesting ?
With smart and apt, you can prioritize channels, which the implementation will do, in descending order, keeping the pkg_arch inter-priorities intact. However, for opkg, you prioritize archs rather than channels afaik.
Quoted 3 lines
Cheers,
Paul
Does this resolve issue 5407?
https://bugzilla.yoctoproject.org/show_bug.cgi?id=5407
Quoted 2 lines · Trevor Woerner
Does this resolve issue 5407?
https://bugzilla.yoctoproject.org/show_bug.cgi?id=5407
Seems so :D
Signature · 3 lines
Otavio Salvador O.S. Systems http://www.ossystems.com.br http://code.ossystems.com.br Mobile: +55 (53) 9981-7854 Mobile: +1 (347) 903-9750
Quoted 3 lines · Trevor Woerner
Does this resolve issue 5407?
https://bugzilla.yoctoproject.org/show_bug.cgi?id=5407
This patch satisfies a small part of 5407, when this patch has been reworked into a recipe and merged, there is still the issue to synchonize the formatting for repositories in f.ex. IPK_FEED_URIS, and a general cleanup of code made obsolete(poky-feed-config-ipk, distro-feeds et.c.)
Br,
David
Quoted 6 lines · On Tuesday 04 March 2014 09:45:09 David Nyström wrote:
Quoted 2 lines · Trevor Woerner
Does this resolve issue 5407?
https://bugzilla.yoctoproject.org/show_bug.cgi?id=5407This patch satisfies a small part of 5407, when this patch has been reworked into a recipe and merged
I'm still confused as to why reworking this into a recipe is beneficial... ?
Cheers,
Paul
Signature · 2 lines
Paul Eggleton Intel Open Source Technology Centre
Quoted 9 lines · Paul Eggleton
Quoted 6 lines · On Tuesday 04 March 2014 09:45:09 David Nyström wrote:
Quoted 2 lines · Trevor Woerner
Does this resolve issue 5407?
https://bugzilla.yoctoproject.org/show_bug.cgi?id=5407This patch satisfies a small part of 5407, when this patch has been reworked into a recipe and merged
I'm still confused as to why reworking this into a recipe is beneficial... ?
Because if the feed info is stored in a package, it can be updated without reflashing.
We used this once to stop updates for selected machines from a shared feed (due to memory constraints), but allowed other machines to continue to receive updates by pointing them to a new feed location.
Regards,
Andreas
Quoted 12 lines · On Tuesday 04 March 2014 12:02:41 Andreas Oberritter wrote:
Quoted 10 lines · Paul Eggleton
Quoted 6 lines · On Tuesday 04 March 2014 09:45:09 David Nyström wrote:
Quoted 2 lines · Trevor Woerner
Does this resolve issue 5407?
https://bugzilla.yoctoproject.org/show_bug.cgi?id=5407This patch satisfies a small part of 5407, when this patch has been reworked into a recipe and merged
I'm still confused as to why reworking this into a recipe is beneficial... ?
Quoted 6 lines
Because if the feed info is stored in a package, it can be updated without reflashing.
We used this once to stop updates for selected machines from a shared feed (due to memory constraints), but allowed other machines to continue to receive updates by pointing them to a new feed location.
I see - makes sense.
Thanks,
Paul
Signature · 2 lines
Paul Eggleton Intel Open Source Technology Centre
Quoted 217 lines · David Nyström
Adding a common interface to add predefined package manager channels to prebuilt rootfs:es.
Adding
PACKAGE_FEED_URIS= "http://myre.po/repo/, will assume repo directories named (rpm,ipk,deb) as subdirectories and statically add them to the rootfs, using the samePKG_ARCHsas the build which produced the images.Tested with RPM, IPK and DEB.
deb feed functionality seem broken, is anyone using this ?
Signed-off-by David Nyström <[email protected]>Signed-off-by David Nyström <[email protected]>--- meta/lib/oe/package_manager.py | 89 +++++++++++++++++++++++++++++++++++++++++- meta/lib/oe/rootfs.py | 16 ++------ 2 files changed, 92 insertions(+), 13 deletions(-)meta/lib/oe/package_manager.py +88
-1@@ -223,6 +223,7 @@ class PackageManager(object): self.d = d self.deploy_dir = None self.deploy_lock = None + self.feed_uris = self.d.getVar('PACKAGE_FEED_URIS', True) or "" """ Update the package manager package database.@@ -262,6 +263,10 @@ class PackageManager(object): def list_installed(self, format=None): pass + @abstractmethod + def insert_feeds_uris(self): + pass + """ Install complementary packages based upon the list of currently installed packages e.g. locales, *-dev, *-dbg, etc. This will only attempt to install@@ -358,6 +363,46 @@ class RpmPM(PackageManager): self.ml_prefix_list, self.ml_os_list = self.indexer.get_ml_prefix_and_os_list(arch_var, os_var) + + def insert_feeds_uris(self): + if self.feed_uris == "": + return + + # List must be prefered to least preferred order + default_platform_extra = set() + platform_extra = set() + bbextendvariant = self.d.getVar('BBEXTENDVARIANT', True) or "" + for mlib in self.ml_os_list: + for arch in self.ml_prefix_list[mlib]: + plt = arch.replace('-', '_') + '-.*-' + self.ml_os_list[mlib] + if mlib == bbextendvariant: + default_platform_extra.add(plt) + else: + platform_extra.add(plt) + + platform_extra = platform_extra.union(default_platform_extra) + + arch_list = [] + for canonical_arch in platform_extra: + arch = canonical_arch.split('-')[0] + if not os.path.exists(os.path.join(self.deploy_dir, arch)): + continue + arch_list.append(arch) + + uri_iterator = 0 + channel_priority = 10 + 5 * len(self.feed_uris.split()) * len(arch_list) + + for uri in self.feed_uris.split(): + for arch in arch_list: + bb.note('Note: adding Smart channel url%d%s (%s)' % + (uri_iterator, arch, channel_priority)) + self._invoke_smart('channel --add url%d-%s type=rpm-md baseurl=%s/rpm/%s -y' + % (uri_iterator, arch, uri, arch)) + self._invoke_smart('channel --set url%d-%s priority=%d' % + (uri_iterator, arch, channel_priority)) + channel_priority -= 5 + uri_iterator += 1 + ''' Create configs for rpm and smart, and multilib is supported '''@@ -944,7 +989,6 @@ class OpkgPM(PackageManager): self.deploy_dir = self.d.getVar("DEPLOY_DIR_IPK", True) self.deploy_lock_file = os.path.join(self.deploy_dir, "deploy.lock") - self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg-cl") self.opkg_args = "-f %s -o %s " % (self.config_file, target_rootfs) self.opkg_args += self.d.getVar("OPKG_ARGS", True)@@ -1050,6 +1094,29 @@ class OpkgPM(PackageManager): config_file.write("src oe-%s file:%s\n" % (arch, pkgs_dir)) + def insert_feeds_uris(self): + if self.feed_uris == "": + return + + rootfs_config = os.path.join('%s/etc/opkg/base-feeds.conf' + % self.target_rootfs) + + with open(rootfs_config, "w+") as config_file: + uri_iterator = 0 + for uri in self.feed_uris.split(): + config_file.write("src/gz url-%d %s/ipk\n" % + (uri_iterator, uri)) + + for arch in self.pkg_archs.split(): + if not os.path.exists(os.path.join(self.deploy_dir, arch)): + continue + bb.note('Note: adding opkg channel url-%s-%d (%s)' % + (arch, uri_iterator, uri)) + + config_file.write("src/gz uri-%s-%d %s/ipk/%s\n" % + (arch, uri_iterator, uri, arch)) + uri_iterator += 1 + def update(self): self.deploy_dir_lock()@@ -1410,6 +1477,26 @@ class DpkgPM(PackageManager): if result is not None: bb.fatal(result) + def insert_feeds_uris(self): + if self.feed_uris == "": + return + + sources_conf = os.path.join("%s/etc/apt/sources.list" + % self.target_rootfs) + arch_list = [] + archs = self.d.getVar('PACKAGE_ARCHS', True) + for arch in archs.split(): + if not os.path.exists(os.path.join(self.deploy_dir, arch)): + continue + arch_list.append(arch) + + with open(sources_conf, "w+") as sources_file: + for uri in self.feed_uris.split(): + for arch in arch_list: + bb.note('Note: adding dpkg channel at (%s)' % uri) + sources_file.write("deb %s/deb/%s ./\n" % + (uri, arch)) + def _create_configs(self, archs, base_archs): base_archs = re.sub("_", "-", base_archs)meta/lib/oe/rootfs.py +4
-12@@ -41,9 +41,10 @@ class Rootfs(object): def _log_check(self): pass - @abstractmethod def _insert_feed_uris(self): - pass + if base_contains("IMAGE_FEATURES", "package-management", + True, False, self.d): + self.pm.insert_feeds_uris() @abstractmethod def _handle_intercept_failure(self, failed_script):@@ -349,9 +350,6 @@ class RpmRootfs(Rootfs): if found_error == 6: bb.fatal(message) - def _insert_feed_uris(self): - pass - def _handle_intercept_failure(self, registered_pkgs): rpm_postinsts_dir = self.image_rootfs + self.d.expand('${sysconfdir}/rpm-postinsts/') bb.utils.mkdirhier(rpm_postinsts_dir)@@ -372,6 +370,7 @@ class DpkgRootfs(Rootfs): d.getVar('PACKAGE_ARCHS', True), d.getVar('DPKG_ARCH', True)) + def _create(self): pkgs_to_install = self.manifest.parse_initial_manifest()@@ -432,9 +431,6 @@ class DpkgRootfs(Rootfs): def _log_check(self): pass - def _insert_feed_uris(self): - pass - class OpkgRootfs(Rootfs): def __init__(self, d, manifest_dir):@@ -698,10 +694,6 @@ class OpkgRootfs(Rootfs): def _log_check(self): pass - def _insert_feed_uris(self): - pass - - def create_rootfs(d, manifest_dir=None): env_bkp = os.environ.copy()Signature · 6 lines
1.8.3.2 _______________________________________________ Openembedded-core mailing list [email protected] http://lists.openembedded.org/mailman/listinfo/openembedded-core