Skip to contentSkip to conversation

package_manager/ipk: skip checksums for unsigned local feeds

PATCH Superseded [email protected] 2 participants 4 messages Jul 29 – Jul 30

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

PATCH 1/2 package_manager/ipk: skip checksums for unsigned local feeds

From: Etienne Cordonnier <[email protected]>

Computing checksums in the opkg Packages index requires reading every .ipk file in full. For a large image with 7000+ packages (including multi-gigabyte debug packages), this adds 150-300s to every do_rootfs run.

Checksums in the Packages index are only meaningful for signed feeds (PACKAGE_FEED_SIGN=1): the GPG signature covers the Packages index which contains the SHA256Sum of each .ipk, forming a chain of trust that prevents tampered packages being swapped on a remote feed.

For unsigned local file:// feeds the packages are installed directly from the build host filesystem where there is no tampering risk. Skip all checksum generation in that case by passing no --checksum flags to opkg-make-index (the tool's default behaviour when given no flags). Pass --force-checksum to opkg so it does not error on the absent checksum fields.

On a test image with 7000+ packages (including a 2.3 GB debug package): write_index time reduced from ~180s to ~22s (8x speedup) when opkg-make-index is configured to produce no checksums by default.

See https://git.openembedded.org/openembedded-core/commit/?id=e462f47489f35902b6972f9837d9adfa542fc796
("Enable sha256 checksums in opkg indexer", 2019) for the original rationale.

AI-Generated: Claude Sonnet 4.6

---
 meta/lib/oe/package_manager/ipk/__init__.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)
meta/lib/oe/package_manager/ipk/__init__.py +9 -5
index 344e0852177..6c0b896a086 100644
@@ -12,7 +12,7 @@ from oe.package_manager import *
 from oe.package_manager.common_deb_ipk import OpkgDpkgPM
 
 class OpkgIndexer(Indexer):
-    def write_index(self):
+    def write_index(self, skip_checksums=False):
         arch_vars = ["ALL_MULTILIB_PACKAGE_ARCHS",
                      "SDK_PACKAGE_ARCHS",
                      ]
@@ -44,8 +44,9 @@ class OpkgIndexer(Indexer):
                 if not os.path.exists(pkgs_file):
                     open(pkgs_file, "w").close()
 
-                index_cmds.add('%s --checksum md5 --checksum sha256 -r %s -p %s -m %s %s' %
-                                  (opkg_index_cmd, pkgs_file, pkgs_file, pkgs_dir, opkg_index_cmd_extra_params))
+                checksum_args = '' if skip_checksums else '--checksum md5 --checksum sha256 '
+                index_cmds.add('%s %s-r %s -p %s -m %s %s' %
+                                  (opkg_index_cmd, checksum_args, pkgs_file, pkgs_file, pkgs_dir, opkg_index_cmd_extra_params))
 
                 index_sign_files.add(pkgs_file)
 
@@ -103,8 +104,12 @@ class OpkgPM(OpkgDpkgPM):
         self.deploy_dir = oe.path.join(self.d.getVar('WORKDIR'), ipk_repo_workdir)
         self.deploy_lock_file = os.path.join(self.deploy_dir, "deploy.lock")
         self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg")
+        self.from_feeds = (self.d.getVar('BUILD_IMAGES_FROM_FEEDS') or "") == "1"
+        self._skip_checksums = self.d.getVar('PACKAGE_FEED_SIGN') != '1' and not self.from_feeds
         self.opkg_args = ['--volatile-cache', '-f', config_file, '-t', self.d.expand('${T}/ipktemp/'), '-o', target_rootfs]
         self.opkg_args.extend(shlex.split(self.d.getVar("OPKG_ARGS")))
+        if self._skip_checksums:
+            self.opkg_args.append('--force-checksum')
 
         if prepare_index:
             create_packages_dir(self.d, self.deploy_dir, d.getVar("DEPLOY_DIR_IPK"), "package_write_ipk", filterbydependencies)
@@ -116,7 +121,6 @@ class OpkgPM(OpkgDpkgPM):
         if not os.path.exists(self.d.expand('${T}/saved')):
             bb.utils.mkdirhier(self.d.expand('${T}/saved'))
 
-        self.from_feeds = (self.d.getVar('BUILD_IMAGES_FROM_FEEDS') or "") == "1"
         if self.from_feeds:
             self._create_custom_config()
         else:
@@ -346,7 +350,7 @@ class OpkgPM(OpkgDpkgPM):
     def write_index(self):
         self.deploy_dir_lock()
 
-        result = self.indexer.write_index()
+        result = self.indexer.write_index(skip_checksums=self._skip_checksums)
 
         self.deploy_dir_unlock()
 
Signature · 1 line
2.43.0
#1 lore raw

PATCH 2/2 opkg: enable sha256 checksum verification by default

From: Etienne Cordonnier <[email protected]>

WITH_SHA256=ON adds no external dependency: sha256.c is a self-contained 528-line pure C implementation already present in the source tree.

oe-core already generates SHA256Sum fields in the Packages index (via --checksum sha256 in OpkgIndexer.write_index), but opkg silently ignores them unless compiled with WITH_SHA256=ON. Enabling sha256 by default makes those checksums actually verified at install time, improving package integrity checking at zero dependency cost.

Once SHA256 is universally available, --checksum md5 can also be
dropped from OpkgIndexer.write_index() -- MD5 is collision-prone and
should not be relied upon for integrity checking -- completing the
deprecation noted in https://git.openembedded.org/openembedded-core/commit/?id=e462f47489f35902b6972f9837d9adfa542fc796
(2019).

AI-Generated: Claude Sonnet 4.6

---
 meta/recipes-devtools/opkg/opkg_0.10.0.bb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
meta/recipes-devtools/opkg/opkg_0.10.0.bb +1 -1
index debcd94a300..e3c2dd790dd 100644
@@ -30,7 +30,7 @@ inherit cmake pkgconfig ptest
 target_localstatedir := "${localstatedir}"
 OPKGLIBDIR ??= "${target_localstatedir}/lib"
 
-PACKAGECONFIG ??= "libsolv ${@bb.utils.filter('DISTRO_FEATURES', 'acl xattr', d)}"
+PACKAGECONFIG ??= "libsolv sha256 ${@bb.utils.filter('DISTRO_FEATURES', 'acl xattr', d)}"
 
 PACKAGECONFIG[gpg] = "-DWITH_GPGME=ON,-DWITH_GPGME=OFF,\
     gnupg gpgme libgpg-error,\
Signature · 1 line
2.43.0
#2 lore raw
In reply to [email protected]

Etienne Cordonnier via lists.openembedded.org schrieb am Mi 29. Jul, 17:29 (+0200):

Quoted 36 lines

From: Etienne Cordonnier <[email protected]>

Computing checksums in the opkg Packages index requires reading every .ipk file in full. For a large image with 7000+ packages (including multi-gigabyte debug packages), this adds 150-300s to every do_rootfs run.

Checksums in the Packages index are only meaningful for signed feeds (PACKAGE_FEED_SIGN=1): the GPG signature covers the Packages index which contains the SHA256Sum of each .ipk, forming a chain of trust that prevents tampered packages being swapped on a remote feed.

For unsigned local file:// feeds the packages are installed directly from the build host filesystem where there is no tampering risk. Skip all checksum generation in that case by passing no --checksum flags to opkg-make-index (the tool's default behaviour when given no flags). Pass --force-checksum to opkg so it does not error on the absent checksum fields.

On a test image with 7000+ packages (including a 2.3 GB debug package): write_index time reduced from ~180s to ~22s (8x speedup) when opkg-make-index is configured to produce no checksums by default.

See https://git.openembedded.org/openembedded-core/commit/?id=e462f47489f35902b6972f9837d9adfa542fc796
("Enable sha256 checksums in opkg indexer", 2019) for the original rationale.

AI-Generated: Claude Sonnet 4.6

---
 meta/lib/oe/package_manager/ipk/__init__.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)
meta/lib/oe/package_manager/ipk/__init__.py +0 -0
index 344e0852177..6c0b896a086 100644

meta/lib/oe/package_manager/ipk/__init__.py from oe.package_manager import * · line 15

Quoted 5 lines
meta/lib/oe/package_manager/ipk/__init__.py +1 -1
@@ -12,7 +12,7 @@ from oe.package_manager import *
 from oe.package_manager.common_deb_ipk import OpkgDpkgPM
 
 class OpkgIndexer(Indexer):
-    def write_index(self):
+    def write_index(self, skip_checksums=False):

Suggestion: I would use the positive case (like with_checksums or use_checksums or build_checksums), because reasoning about/reading code with the inverse is more difficult.

Quoted 16 lines
meta/lib/oe/package_manager/ipk/__init__.py +3 -2
         arch_vars = ["ALL_MULTILIB_PACKAGE_ARCHS",
                      "SDK_PACKAGE_ARCHS",
                      ]
@@ -44,8 +44,9 @@ class OpkgIndexer(Indexer):
                 if not os.path.exists(pkgs_file):
                     open(pkgs_file, "w").close()
 
-                index_cmds.add('%s --checksum md5 --checksum sha256 -r %s -p %s -m %s %s' %
-                                  (opkg_index_cmd, pkgs_file, pkgs_file, pkgs_dir, opkg_index_cmd_extra_params))
+                checksum_args = '' if skip_checksums else '--checksum md5 --checksum sha256 '
+                index_cmds.add('%s %s-r %s -p %s -m %s %s' %
+                                  (opkg_index_cmd, checksum_args, pkgs_file, pkgs_file, pkgs_dir, opkg_index_cmd_extra_params))
 
                 index_sign_files.add(pkgs_file)
 

meta/lib/oe/package_manager/ipk/__init__.py class OpkgPM(OpkgDpkgPM): · line 107

Quoted 4 lines
meta/lib/oe/package_manager/ipk/__init__.py +1 -0
@@ -103,8 +104,12 @@ class OpkgPM(OpkgDpkgPM):
         self.deploy_dir = oe.path.join(self.d.getVar('WORKDIR'), ipk_repo_workdir)
         self.deploy_lock_file = os.path.join(self.deploy_dir, "deploy.lock")
         self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg")
+        self.from_feeds = (self.d.getVar('BUILD_IMAGES_FROM_FEEDS') or "") == "1"

Or: bb.utils.to_boolean(...)

meta/lib/oe/package_manager/ipk/__init__.py class OpkgPM(OpkgDpkgPM): · line 108

Quoted 1 line
meta/lib/oe/package_manager/ipk/__init__.py +1 -0
+        self._skip_checksums = self.d.getVar('PACKAGE_FEED_SIGN') != '1' and not self.from_feeds

I would swap them because evaluation of the second one is cheaper.

Quoted 16 lines
meta/lib/oe/package_manager/ipk/__init__.py +2 -1
         self.opkg_args = ['--volatile-cache', '-f', config_file, '-t', self.d.expand('${T}/ipktemp/'), '-o', target_rootfs]
         self.opkg_args.extend(shlex.split(self.d.getVar("OPKG_ARGS")))
+        if self._skip_checksums:
+            self.opkg_args.append('--force-checksum')
 
         if prepare_index:
             create_packages_dir(self.d, self.deploy_dir, d.getVar("DEPLOY_DIR_IPK"), "package_write_ipk", filterbydependencies)
@@ -116,7 +121,6 @@ class OpkgPM(OpkgDpkgPM):
         if not os.path.exists(self.d.expand('${T}/saved')):
             bb.utils.mkdirhier(self.d.expand('${T}/saved'))
 
-        self.from_feeds = (self.d.getVar('BUILD_IMAGES_FROM_FEEDS') or "") == "1"
         if self.from_feeds:
             self._create_custom_config()
         else:

meta/lib/oe/package_manager/ipk/__init__.py class OpkgPM(OpkgDpkgPM): · line 355

Quoted 7 lines
meta/lib/oe/package_manager/ipk/__init__.py +1 -1
@@ -346,7 +350,7 @@ class OpkgPM(OpkgDpkgPM):
     def write_index(self):
         self.deploy_dir_lock()
 
-        result = self.indexer.write_index()
+        result = self.indexer.write_index(skip_checksums=self._skip_checksums)
 
         self.deploy_dir_unlock()

Jörg

Signature · 6 lines
Navimatix GmbH           T: 03641 - 327 99 0
Tatzendpromenade 2       F: 03641 - 526 306
07745 Jena               www.navimatix.de

Geschäftsführer: Steffen Späthe, Jan Rommeley
Registergericht: Amtsgericht Jena, HRB 501480
#3 lore raw

Hi Jörg,
- I've changed to positive logic
- I would rather not change to to_boolean. I'm just moving a line, and introducing to_boolean would make it inconsistent with all the other lines where this variable is used. Also BUILD_IMAGES_FROM_FEEDS is undocumented (it is barely used upstream as far as I understand from
https://lists.yoctoproject.org/g/yocto/topic/install_externally_built_ipk/112649954
), so acceptable values are not 100% clear to me (other lines using this variable treat "False" as a positive value since they call "if d.getVar('BUILD_IMAGES_FROM_FEEDS'):", while to_boolean would do the opposite).
- I've swapped the evaluation order

Quoted 127 lines · Jörg Sommer

Etienne Cordonnier via lists.openembedded.org schrieb am Mi 29. Jul, 17:29 (+0200):

Quoted 5 lines

From: Etienne Cordonnier <[email protected]>

Computing checksums in the opkg Packages index requires reading every .ipk file in full. For a large image with 7000+ packages (including multi-gigabyte debug packages), this adds 150-300s to every do_rootfs

run.

Quoted 13 lines

Checksums in the Packages index are only meaningful for signed feeds (PACKAGE_FEED_SIGN=1): the GPG signature covers the Packages index which contains the SHA256Sum of each .ipk, forming a chain of trust that prevents tampered packages being swapped on a remote feed.

For unsigned local file:// feeds the packages are installed directly from the build host filesystem where there is no tampering risk. Skip all checksum generation in that case by passing no --checksum flags to opkg-make-index (the tool's default behaviour when given no flags). Pass --force-checksum to opkg so it does not error on the absent checksum fields.

Quoted 5 lines

On a test image with 7000+ packages (including a 2.3 GB debug package): write_index time reduced from ~180s to ~22s (8x speedup) when opkg-make-index is configured to produce no checksums by default.

See

https://git.openembedded.org/openembedded-core/commit/?id=e462f47489f35902b6972f9837d9adfa542fc796

Quoted 1 line

("Enable sha256 checksums in opkg indexer", 2019) for the original

rationale.

Review

Quoted 8 lines

AI-Generated: Claude Sonnet 4.6

---
 meta/lib/oe/package_manager/ipk/__init__.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)
(quoted diff) +0 -0

b/meta/lib/oe/package_manager/ipk/__init__.py

meta/lib/oe/package_manager/ipk/__init__.py from oe.package_manager import * · line 15

Quoted 9 lines

index 344e0852177..6c0b896a086 100644

meta/lib/oe/package_manager/ipk/__init__.py +1 -1
@@ -12,7 +12,7 @@ from oe.package_manager import *
 from oe.package_manager.common_deb_ipk import OpkgDpkgPM
 
 class OpkgIndexer(Indexer):
-    def write_index(self):
+    def write_index(self, skip_checksums=False):

Suggestion: I would use the positive case (like with_checksums or use_checksums or build_checksums), because reasoning about/reading code with
the inverse is more difficult.

meta/lib/oe/package_manager/ipk/__init__.py class OpkgIndexer(Indexer): · line 47

Quoted 8 lines
meta/lib/oe/package_manager/ipk/__init__.py +0 -1
         arch_vars = ["ALL_MULTILIB_PACKAGE_ARCHS",
                      "SDK_PACKAGE_ARCHS",
                      ]
@@ -44,8 +44,9 @@ class OpkgIndexer(Indexer):
                 if not os.path.exists(pkgs_file):
                     open(pkgs_file, "w").close()
 
-                index_cmds.add('%s --checksum md5 --checksum sha256 -r

%s -p %s -m %s %s' %

meta/lib/oe/package_manager/ipk/__init__.py class OpkgIndexer(Indexer): · line 48

Quoted 1 line
meta/lib/oe/package_manager/ipk/__init__.py +0 -1
-                                  (opkg_index_cmd, pkgs_file,

pkgs_file, pkgs_dir, opkg_index_cmd_extra_params))

meta/lib/oe/package_manager/ipk/__init__.py class OpkgIndexer(Indexer): · line 47

Quoted 1 line
meta/lib/oe/package_manager/ipk/__init__.py +1 -0
+                checksum_args = '' if skip_checksums else '--checksum

md5 --checksum sha256 '

meta/lib/oe/package_manager/ipk/__init__.py class OpkgIndexer(Indexer): · line 49

Quoted 2 lines
meta/lib/oe/package_manager/ipk/__init__.py +2 -0
+                index_cmds.add('%s %s-r %s -p %s -m %s %s' %
+                                  (opkg_index_cmd, checksum_args,

pkgs_file, pkgs_file, pkgs_dir, opkg_index_cmd_extra_params))

meta/lib/oe/package_manager/ipk/__init__.py class OpkgPM(OpkgDpkgPM): · line 104

Quoted 5 lines
meta/lib/oe/package_manager/ipk/__init__.py +0 -0
                 index_sign_files.add(pkgs_file)
 
@@ -103,8 +104,12 @@ class OpkgPM(OpkgDpkgPM):
         self.deploy_dir = oe.path.join(self.d.getVar('WORKDIR'),

ipk_repo_workdir)

meta/lib/oe/package_manager/ipk/__init__.py class OpkgPM(OpkgDpkgPM): · line 105

Quoted 1 line
meta/lib/oe/package_manager/ipk/__init__.py +0 -0
         self.deploy_lock_file = os.path.join(self.deploy_dir,

"deploy.lock")

meta/lib/oe/package_manager/ipk/__init__.py class OpkgPM(OpkgDpkgPM): · line 107

Quoted 2 lines
meta/lib/oe/package_manager/ipk/__init__.py +1 -0
         self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg")
+        self.from_feeds = (self.d.getVar('BUILD_IMAGES_FROM_FEEDS') or

"") == "1"

Or: bb.utils.to_boolean(...)

meta/lib/oe/package_manager/ipk/__init__.py class OpkgPM(OpkgDpkgPM): · line 108

Quoted 1 line
meta/lib/oe/package_manager/ipk/__init__.py +1 -0
+        self._skip_checksums = self.d.getVar('PACKAGE_FEED_SIGN') !=

'1' and not self.from_feeds

I would swap them because evaluation of the second one is cheaper.

meta/lib/oe/package_manager/ipk/__init__.py class OpkgPM(OpkgDpkgPM): · line 109

Quoted 1 line
meta/lib/oe/package_manager/ipk/__init__.py +0 -0
         self.opkg_args = ['--volatile-cache', '-f', config_file, '-t',

self.d.expand('${T}/ipktemp/'), '-o', target_rootfs]

meta/lib/oe/package_manager/ipk/__init__.py class OpkgPM(OpkgDpkgPM):

Quoted 6 lines
meta/lib/oe/package_manager/ipk/__init__.py +2 -0
         self.opkg_args.extend(shlex.split(self.d.getVar("OPKG_ARGS")))
+        if self._skip_checksums:
+            self.opkg_args.append('--force-checksum')
 
         if prepare_index:
             create_packages_dir(self.d, self.deploy_dir,

d.getVar("DEPLOY_DIR_IPK"), "package_write_ipk", filterbydependencies)

meta/lib/oe/package_manager/ipk/__init__.py class OpkgPM(OpkgDpkgPM): · line 119

Quoted 5 lines
meta/lib/oe/package_manager/ipk/__init__.py +0 -1
@@ -116,7 +121,6 @@ class OpkgPM(OpkgDpkgPM):
         if not os.path.exists(self.d.expand('${T}/saved')):
             bb.utils.mkdirhier(self.d.expand('${T}/saved'))
 
-        self.from_feeds = (self.d.getVar('BUILD_IMAGES_FROM_FEEDS') or

"") == "1"

meta/lib/oe/package_manager/ipk/__init__.py class OpkgPM(OpkgDpkgPM): · line 353

Quoted 9 lines
meta/lib/oe/package_manager/ipk/__init__.py +1 -1
         if self.from_feeds:
             self._create_custom_config()
         else:
@@ -346,7 +350,7 @@ class OpkgPM(OpkgDpkgPM):
     def write_index(self):
         self.deploy_dir_lock()
 
-        result = self.indexer.write_index()
+        result =

self.indexer.write_index(skip_checksums=self._skip_checksums)

meta/lib/oe/package_manager/ipk/__init__.py class OpkgPM(OpkgDpkgPM): · line 355

Quoted 2 lines
meta/lib/oe/package_manager/ipk/__init__.py +0 -0
         self.deploy_dir_unlock()

Jörg

--
Navimatix GmbH T: 03641 - 327 99 0
Tatzendpromenade 2 F: 03641 - 526 306
07745 Jena www.navimatix.de

Geschäftsführer: Steffen Späthe, Jan Rommeley
Registergericht: Amtsgericht Jena, HRB 501480