Skip to contentSkip to conversation

do_rootfs: Added PACKAGE_FEED_URIS functionality

PATCH In review David Nyström 7 participants 11 messages Feb 27, 2014 – Mar 4, 2014

lore
Share & tools
lore
#0 lore raw

PATCH 1/2 do_rootfs: Added PACKAGE_FEED_URIS functionality

Adding a common interface to add predefined package manager channels to prebuilt rootfs:es.

Adding PACKAGE_FEED_URIS = "http://myre.po/repo/, will assume repo directories named (rpm,ipk,deb) as subdirectories and statically add them to the rootfs, using the same PKG_ARCHs as the build which produced the images.

Tested with RPM, IPK and DEB.

deb feed functionality seem broken, is anyone using this ?

---
 meta/lib/oe/package_manager.py | 89 +++++++++++++++++++++++++++++++++++++++++-
 meta/lib/oe/rootfs.py          | 16 ++------
 2 files changed, 92 insertions(+), 13 deletions(-)
meta/lib/oe/package_manager.py +88 -1
index ff4f1de..c930572 100644
@@ -223,6 +223,7 @@ class PackageManager(object):
         self.d = d
         self.deploy_dir = None
         self.deploy_lock = None
+        self.feed_uris = self.d.getVar('PACKAGE_FEED_URIS', True) or ""
 
     """
     Update the package manager package database.
@@ -262,6 +263,10 @@ class PackageManager(object):
     def list_installed(self, format=None):
         pass
 
+    @abstractmethod
+    def insert_feeds_uris(self):
+        pass
+
     """
     Install complementary packages based upon the list of currently installed
     packages e.g. locales, *-dev, *-dbg, etc. This will only attempt to install
@@ -358,6 +363,46 @@ class RpmPM(PackageManager):
 
         self.ml_prefix_list, self.ml_os_list = self.indexer.get_ml_prefix_and_os_list(arch_var, os_var)
 
+
+    def insert_feeds_uris(self):
+        if self.feed_uris == "":
+            return
+
+        # List must be prefered to least preferred order
+        default_platform_extra = set()
+        platform_extra = set()
+        bbextendvariant = self.d.getVar('BBEXTENDVARIANT', True) or ""
+        for mlib in self.ml_os_list:
+            for arch in self.ml_prefix_list[mlib]:
+                plt = arch.replace('-', '_') + '-.*-' + self.ml_os_list[mlib]
+                if mlib == bbextendvariant:
+                        default_platform_extra.add(plt)
+                else:
+                        platform_extra.add(plt)
+
+        platform_extra = platform_extra.union(default_platform_extra)
+
+        arch_list = []
+        for canonical_arch in platform_extra:
+            arch = canonical_arch.split('-')[0]
+            if not os.path.exists(os.path.join(self.deploy_dir, arch)):
+                continue
+            arch_list.append(arch)
+
+        uri_iterator = 0
+        channel_priority = 10 + 5 * len(self.feed_uris.split()) * len(arch_list)
+
+        for uri in self.feed_uris.split():
+            for arch in arch_list:
+                bb.note('Note: adding Smart channel url%d%s (%s)' %
+                        (uri_iterator, arch, channel_priority))
+                self._invoke_smart('channel --add url%d-%s type=rpm-md baseurl=%s/rpm/%s -y'
+                                   % (uri_iterator, arch, uri, arch))
+                self._invoke_smart('channel --set url%d-%s priority=%d' %
+                                   (uri_iterator, arch, channel_priority))
+                channel_priority -= 5
+            uri_iterator += 1
+
     '''
     Create configs for rpm and smart, and multilib is supported
     '''
@@ -944,7 +989,6 @@ class OpkgPM(PackageManager):
 
         self.deploy_dir = self.d.getVar("DEPLOY_DIR_IPK", True)
         self.deploy_lock_file = os.path.join(self.deploy_dir, "deploy.lock")
-
         self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg-cl")
         self.opkg_args = "-f %s -o %s " % (self.config_file, target_rootfs)
         self.opkg_args += self.d.getVar("OPKG_ARGS", True)
@@ -1050,6 +1094,29 @@ class OpkgPM(PackageManager):
                     config_file.write("src oe-%s file:%s\n" %
                                       (arch, pkgs_dir))
 
+    def insert_feeds_uris(self):
+        if self.feed_uris == "":
+            return
+
+        rootfs_config = os.path.join('%s/etc/opkg/base-feeds.conf'
+                                  % self.target_rootfs)
+
+        with open(rootfs_config, "w+") as config_file:
+            uri_iterator = 0
+            for uri in self.feed_uris.split():
+                config_file.write("src/gz url-%d %s/ipk\n" %
+                                  (uri_iterator, uri))
+
+                for arch in self.pkg_archs.split():
+                    if not os.path.exists(os.path.join(self.deploy_dir, arch)):
+                        continue
+                    bb.note('Note: adding opkg channel url-%s-%d (%s)' %
+                        (arch, uri_iterator, uri))
+
+                    config_file.write("src/gz uri-%s-%d %s/ipk/%s\n" %
+                                      (arch, uri_iterator, uri, arch))
+                uri_iterator += 1
+
     def update(self):
         self.deploy_dir_lock()
 
@@ -1410,6 +1477,26 @@ class DpkgPM(PackageManager):
         if result is not None:
             bb.fatal(result)
 
+    def insert_feeds_uris(self):
+        if self.feed_uris == "":
+            return
+
+        sources_conf = os.path.join("%s/etc/apt/sources.list"
+                                    % self.target_rootfs)
+        arch_list = []
+        archs = self.d.getVar('PACKAGE_ARCHS', True)
+        for arch in archs.split():
+            if not os.path.exists(os.path.join(self.deploy_dir, arch)):
+                continue
+            arch_list.append(arch)
+
+        with open(sources_conf, "w+") as sources_file:
+            for uri in self.feed_uris.split():
+                for arch in arch_list:
+                    bb.note('Note: adding dpkg channel at (%s)' % uri)
+                    sources_file.write("deb %s/deb/%s ./\n" %
+                                       (uri, arch))
+
     def _create_configs(self, archs, base_archs):
         base_archs = re.sub("_", "-", base_archs)
 
meta/lib/oe/rootfs.py +4 -12
index be0afa6..cb0a739 100644
@@ -41,9 +41,10 @@ class Rootfs(object):
     def _log_check(self):
         pass
 
-    @abstractmethod
     def _insert_feed_uris(self):
-        pass
+        if base_contains("IMAGE_FEATURES", "package-management",
+                         True, False, self.d):
+            self.pm.insert_feeds_uris()
 
     @abstractmethod
     def _handle_intercept_failure(self, failed_script):
@@ -349,9 +350,6 @@ class RpmRootfs(Rootfs):
                 if found_error == 6:
                     bb.fatal(message)
 
-    def _insert_feed_uris(self):
-        pass
-
     def _handle_intercept_failure(self, registered_pkgs):
         rpm_postinsts_dir = self.image_rootfs + self.d.expand('${sysconfdir}/rpm-postinsts/')
         bb.utils.mkdirhier(rpm_postinsts_dir)
@@ -372,6 +370,7 @@ class DpkgRootfs(Rootfs):
                          d.getVar('PACKAGE_ARCHS', True),
                          d.getVar('DPKG_ARCH', True))
 
+
     def _create(self):
         pkgs_to_install = self.manifest.parse_initial_manifest()
 
@@ -432,9 +431,6 @@ class DpkgRootfs(Rootfs):
     def _log_check(self):
         pass
 
-    def _insert_feed_uris(self):
-        pass
-
 
 class OpkgRootfs(Rootfs):
     def __init__(self, d, manifest_dir):
@@ -698,10 +694,6 @@ class OpkgRootfs(Rootfs):
     def _log_check(self):
         pass
 
-    def _insert_feed_uris(self):
-        pass
-
-
 def create_rootfs(d, manifest_dir=None):
     env_bkp = os.environ.copy()
 
Signature · 1 line
1.8.3.2
#1 lore raw

PATCH 2/2 do_rootfs: Add PACKAGE_FEED_URIS as a vardep

A bit uncertain on where to put this, suggestions welcome. Needed to automatically rebuild do_rootfs when PACKAGE_FEED_URIS change.

---
 meta/classes/rootfs_deb.bbclass | 1 +
 meta/classes/rootfs_ipk.bbclass | 1 +
 meta/classes/rootfs_rpm.bbclass | 1 +
 3 files changed, 3 insertions(+)
meta/classes/rootfs_deb.bbclass +1 -0
index 728ea0e..a42a472 100644
@@ -8,6 +8,7 @@ ROOTFS_PKGMANAGE_BOOTSTRAP  = "run-postinsts"
 do_rootfs[depends] += "dpkg-native:do_populate_sysroot apt-native:do_populate_sysroot"
 do_rootfs[recrdeptask] += "do_package_write_deb"
 rootfs_deb_do_rootfs[vardepsexclude] += "BUILDNAME"
+do_rootfs[vardeps] += "PACKAGE_FEED_URIS"
 
 do_rootfs[lockfiles] += "${DEPLOY_DIR_DEB}/deb.lock"
 
meta/classes/rootfs_ipk.bbclass +1 -0
index 1887dd3..f5fef00 100644
@@ -11,6 +11,7 @@ ROOTFS_PKGMANAGE_BOOTSTRAP  = "run-postinsts"
 
 do_rootfs[depends] += "opkg-native:do_populate_sysroot opkg-utils-native:do_populate_sysroot"
 do_rootfs[recrdeptask] += "do_package_write_ipk"
+do_rootfs[vardeps] += "PACKAGE_FEED_URIS"
 rootfs_ipk_do_rootfs[vardepsexclude] += "BUILDNAME"
 
 do_rootfs[lockfiles] += "${WORKDIR}/ipk.lock"
meta/classes/rootfs_rpm.bbclass +1 -0
index 032dabc..4b02247 100644
@@ -23,6 +23,7 @@ do_rootfs[depends] += "createrepo-native:do_populate_sysroot"
 
 do_rootfs[recrdeptask] += "do_package_write_rpm"
 rootfs_rpm_do_rootfs[vardepsexclude] += "BUILDNAME"
+do_rootfs[vardeps] += "PACKAGE_FEED_URIS"
 
 # RPM doesn't work with multiple rootfs generation at once due to collisions in the use of files 
 # in ${DEPLOY_DIR_RPM}. This can be removed if package_update_index_rpm can be called concurrently
Signature · 1 line
1.8.3.2
#2 lore raw
In reply to David Nyström
Quoted 9 lines · On Thursday 27 February 2014 21:20:37 David Nyström wrote:

Adding a common interface to add predefined package manager channels to prebuilt rootfs:es.

Adding PACKAGE_FEED_URIS = "http://myre.po/repo/, will assume repo directories named (rpm,ipk,deb) as subdirectories and statically add them to the rootfs, using the same PKG_ARCHs as the build which produced the images.

Tested with RPM, IPK and DEB.

Looks good. The only thing that looks like it might be missing here is there appears to be no way to name the package repo entries individually, though I'm not sure what syntax we would use to allow that.

Thoughts?

Cheers,
Paul

Signature · 2 lines
Paul Eggleton
Intel Open Source Technology Centre
#3 lore raw
Quoted 9 lines · Paul Eggleton
Quoted 8 lines · On Thursday 27 February 2014 21:20:37 David Nyström wrote:

Adding a common interface to add predefined package manager channels to prebuilt rootfs:es.

Adding PACKAGE_FEED_URIS = "http://myre.po/repo/, will assume repo directories named (rpm,ipk,deb) as subdirectories and statically add them to the rootfs, using the same PKG_ARCHs as the build which produced the images.

Quoted 7 lines
Quoted 1 line

Tested with RPM, IPK and DEB.

Looks good. The only thing that looks like it might be missing here is there appears to be no way to name the package repo entries individually, though I'm not sure what syntax we would use to allow that.

Thoughts?

Well, there is the old moblin syntax for IPK_FEED_URIS:

For qemux86-64
--
IPK_FEED_URIS = " \

all##http://downloads.yoctoproject.org/releases/yocto/yocto-1.5/ipk/all \

x86_64##http://downloads.yoctoproject.org/releases/yocto/yocto-1.5/ipk/x86_64 \

core2-64##http://downloads.yoctoproject.org/releases/yocto/yocto-1.5/ipk/core2-64 \

qemux86_64##http://downloads.yoctoproject.org/releases/yocto/yocto-1.5/ipk/qemux86_64 \
--
I don't particularly agree with above format.
When using the deploy directory as input it makes the feed_syntax MACHINE dependent, and difficult to keep in sync for new machines and upstream changes.

IMHO, bitbake should support the same input format as it generates as output format for the repos. If someone want to share code for the post-processing of outputted bitbake repos(deb,ipk,rpm), I'll happily reconsider.

Having multiple feed formats is confusing though, we should fix this so we have the same format everywhere.

What could be done is to add names to the top level repos directory, i.e. PACKAGE_FEED_URIS ="stable##http://myre.op/deploy/ \
"experimental#http://not_trusted.to/ext_deploy/"

But I'm not sure what this will accomplish. The real channel names would become experimental-all, stable-all instead of current hardcoded url0-all, url1-all et.c. Is this what you were suggesting ?

With smart and apt, you can prioritize channels, which the implementation will do, in descending order, keeping the pkg_arch inter-priorities intact. However, for opkg, you prioritize archs rather than channels afaik.

Quoted 3 lines

Cheers,
Paul

#7 lore raw
Quoted 6 lines · On Tuesday 04 March 2014 09:45:09 David Nyström wrote:
Quoted 2 lines · Trevor Woerner

Does this resolve issue 5407?
https://bugzilla.yoctoproject.org/show_bug.cgi?id=5407

This patch satisfies a small part of 5407, when this patch has been reworked into a recipe and merged

I'm still confused as to why reworking this into a recipe is beneficial... ?

Cheers,
Paul

Signature · 2 lines
Paul Eggleton
Intel Open Source Technology Centre
#8 lore raw
Quoted 9 lines · Paul Eggleton
Quoted 6 lines · On Tuesday 04 March 2014 09:45:09 David Nyström wrote:
Quoted 2 lines · Trevor Woerner

Does this resolve issue 5407?
https://bugzilla.yoctoproject.org/show_bug.cgi?id=5407

This patch satisfies a small part of 5407, when this patch has been reworked into a recipe and merged

I'm still confused as to why reworking this into a recipe is beneficial... ?

Because if the feed info is stored in a package, it can be updated without reflashing.

We used this once to stop updates for selected machines from a shared feed (due to memory constraints), but allowed other machines to continue to receive updates by pointing them to a new feed location.

Regards,
Andreas

#9 lore raw
Quoted 12 lines · On Tuesday 04 March 2014 12:02:41 Andreas Oberritter wrote:
Quoted 10 lines · Paul Eggleton
Quoted 6 lines · On Tuesday 04 March 2014 09:45:09 David Nyström wrote:
Quoted 2 lines · Trevor Woerner

Does this resolve issue 5407?
https://bugzilla.yoctoproject.org/show_bug.cgi?id=5407

This patch satisfies a small part of 5407, when this patch has been reworked into a recipe and merged

I'm still confused as to why reworking this into a recipe is beneficial... ?

Quoted 6 lines

Because if the feed info is stored in a package, it can be updated without reflashing.

We used this once to stop updates for selected machines from a shared feed (due to memory constraints), but allowed other machines to continue to receive updates by pointing them to a new feed location.

I see - makes sense.

Thanks,
Paul

Signature · 2 lines
Paul Eggleton
Intel Open Source Technology Centre
Reviewed-by on v1 1/2
#10 lore raw
In reply to David Nyström
Quoted 217 lines · David Nyström

Adding a common interface to add predefined package manager channels to prebuilt rootfs:es.

Adding PACKAGE_FEED_URIS = "http://myre.po/repo/, will assume repo directories named (rpm,ipk,deb) as subdirectories and statically add them to the rootfs, using the same PKG_ARCHs as the build which produced the images.

Tested with RPM, IPK and DEB.

deb feed functionality seem broken, is anyone using this ?

---
 meta/lib/oe/package_manager.py | 89 +++++++++++++++++++++++++++++++++++++++++-
 meta/lib/oe/rootfs.py          | 16 ++------
 2 files changed, 92 insertions(+), 13 deletions(-)
meta/lib/oe/package_manager.py +88 -1
index ff4f1de..c930572 100644
@@ -223,6 +223,7 @@ class PackageManager(object):
         self.d = d
         self.deploy_dir = None
         self.deploy_lock = None
+        self.feed_uris = self.d.getVar('PACKAGE_FEED_URIS', True) or ""
 
     """
     Update the package manager package database.
@@ -262,6 +263,10 @@ class PackageManager(object):
     def list_installed(self, format=None):
         pass
 
+    @abstractmethod
+    def insert_feeds_uris(self):
+        pass
+
     """
     Install complementary packages based upon the list of currently installed
     packages e.g. locales, *-dev, *-dbg, etc. This will only attempt to install
@@ -358,6 +363,46 @@ class RpmPM(PackageManager):
 
         self.ml_prefix_list, self.ml_os_list = self.indexer.get_ml_prefix_and_os_list(arch_var, os_var)
 
+
+    def insert_feeds_uris(self):
+        if self.feed_uris == "":
+            return
+
+        # List must be prefered to least preferred order
+        default_platform_extra = set()
+        platform_extra = set()
+        bbextendvariant = self.d.getVar('BBEXTENDVARIANT', True) or ""
+        for mlib in self.ml_os_list:
+            for arch in self.ml_prefix_list[mlib]:
+                plt = arch.replace('-', '_') + '-.*-' + self.ml_os_list[mlib]
+                if mlib == bbextendvariant:
+                        default_platform_extra.add(plt)
+                else:
+                        platform_extra.add(plt)
+
+        platform_extra = platform_extra.union(default_platform_extra)
+
+        arch_list = []
+        for canonical_arch in platform_extra:
+            arch = canonical_arch.split('-')[0]
+            if not os.path.exists(os.path.join(self.deploy_dir, arch)):
+                continue
+            arch_list.append(arch)
+
+        uri_iterator = 0
+        channel_priority = 10 + 5 * len(self.feed_uris.split()) * len(arch_list)
+
+        for uri in self.feed_uris.split():
+            for arch in arch_list:
+                bb.note('Note: adding Smart channel url%d%s (%s)' %
+                        (uri_iterator, arch, channel_priority))
+                self._invoke_smart('channel --add url%d-%s type=rpm-md baseurl=%s/rpm/%s -y'
+                                   % (uri_iterator, arch, uri, arch))
+                self._invoke_smart('channel --set url%d-%s priority=%d' %
+                                   (uri_iterator, arch, channel_priority))
+                channel_priority -= 5
+            uri_iterator += 1
+
     '''
     Create configs for rpm and smart, and multilib is supported
     '''
@@ -944,7 +989,6 @@ class OpkgPM(PackageManager):
 
         self.deploy_dir = self.d.getVar("DEPLOY_DIR_IPK", True)
         self.deploy_lock_file = os.path.join(self.deploy_dir, "deploy.lock")
-
         self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg-cl")
         self.opkg_args = "-f %s -o %s " % (self.config_file, target_rootfs)
         self.opkg_args += self.d.getVar("OPKG_ARGS", True)
@@ -1050,6 +1094,29 @@ class OpkgPM(PackageManager):
                     config_file.write("src oe-%s file:%s\n" %
                                       (arch, pkgs_dir))
 
+    def insert_feeds_uris(self):
+        if self.feed_uris == "":
+            return
+
+        rootfs_config = os.path.join('%s/etc/opkg/base-feeds.conf'
+                                  % self.target_rootfs)
+
+        with open(rootfs_config, "w+") as config_file:
+            uri_iterator = 0
+            for uri in self.feed_uris.split():
+                config_file.write("src/gz url-%d %s/ipk\n" %
+                                  (uri_iterator, uri))
+
+                for arch in self.pkg_archs.split():
+                    if not os.path.exists(os.path.join(self.deploy_dir, arch)):
+                        continue
+                    bb.note('Note: adding opkg channel url-%s-%d (%s)' %
+                        (arch, uri_iterator, uri))
+
+                    config_file.write("src/gz uri-%s-%d %s/ipk/%s\n" %
+                                      (arch, uri_iterator, uri, arch))
+                uri_iterator += 1
+
     def update(self):
         self.deploy_dir_lock()
 
@@ -1410,6 +1477,26 @@ class DpkgPM(PackageManager):
         if result is not None:
             bb.fatal(result)
 
+    def insert_feeds_uris(self):
+        if self.feed_uris == "":
+            return
+
+        sources_conf = os.path.join("%s/etc/apt/sources.list"
+                                    % self.target_rootfs)
+        arch_list = []
+        archs = self.d.getVar('PACKAGE_ARCHS', True)
+        for arch in archs.split():
+            if not os.path.exists(os.path.join(self.deploy_dir, arch)):
+                continue
+            arch_list.append(arch)
+
+        with open(sources_conf, "w+") as sources_file:
+            for uri in self.feed_uris.split():
+                for arch in arch_list:
+                    bb.note('Note: adding dpkg channel at (%s)' % uri)
+                    sources_file.write("deb %s/deb/%s ./\n" %
+                                       (uri, arch))
+
     def _create_configs(self, archs, base_archs):
         base_archs = re.sub("_", "-", base_archs)
 
meta/lib/oe/rootfs.py +4 -12
index be0afa6..cb0a739 100644
@@ -41,9 +41,10 @@ class Rootfs(object):
     def _log_check(self):
         pass
 
-    @abstractmethod
     def _insert_feed_uris(self):
-        pass
+        if base_contains("IMAGE_FEATURES", "package-management",
+                         True, False, self.d):
+            self.pm.insert_feeds_uris()
 
     @abstractmethod
     def _handle_intercept_failure(self, failed_script):
@@ -349,9 +350,6 @@ class RpmRootfs(Rootfs):
                 if found_error == 6:
                     bb.fatal(message)
 
-    def _insert_feed_uris(self):
-        pass
-
     def _handle_intercept_failure(self, registered_pkgs):
         rpm_postinsts_dir = self.image_rootfs + self.d.expand('${sysconfdir}/rpm-postinsts/')
         bb.utils.mkdirhier(rpm_postinsts_dir)
@@ -372,6 +370,7 @@ class DpkgRootfs(Rootfs):
                          d.getVar('PACKAGE_ARCHS', True),
                          d.getVar('DPKG_ARCH', True))
 
+
     def _create(self):
         pkgs_to_install = self.manifest.parse_initial_manifest()
 
@@ -432,9 +431,6 @@ class DpkgRootfs(Rootfs):
     def _log_check(self):
         pass
 
-    def _insert_feed_uris(self):
-        pass
-
 
 class OpkgRootfs(Rootfs):
     def __init__(self, d, manifest_dir):
@@ -698,10 +694,6 @@ class OpkgRootfs(Rootfs):
     def _log_check(self):
         pass
 
-    def _insert_feed_uris(self):
-        pass
-
-
 def create_rootfs(d, manifest_dir=None):
     env_bkp = os.environ.copy()
 
Signature · 6 lines
1.8.3.2

_______________________________________________
Openembedded-core mailing list
[email protected]
http://lists.openembedded.org/mailman/listinfo/openembedded-core