package_manager/ipk: skip checksums for unsigned local feeds
PATCH 1/2 package_manager/ipk: skip checksums for unsigned local feeds
From: Etienne Cordonnier <[email protected]>
Computing checksums in the opkg Packages index requires reading every .ipk file in full. For a large image with 7000+ packages (including multi-gigabyte debug packages), this adds 150-300s to every do_rootfs run.
Checksums in the Packages index are only meaningful for signed feeds (PACKAGE_FEED_SIGN=1): the GPG signature covers the Packages index which contains the SHA256Sum of each .ipk, forming a chain of trust that prevents tampered packages being swapped on a remote feed.
For unsigned local file:// feeds the packages are installed directly from the build host filesystem where there is no tampering risk. Skip all checksum generation in that case by passing no --checksum flags to opkg-make-index (the tool's default behaviour when given no flags). Pass --force-checksum to opkg so it does not error on the absent checksum fields.
On a test image with 7000+ packages (including a 2.3 GB debug package): write_index time reduced from ~180s to ~22s (8x speedup) when opkg-make-index is configured to produce no checksums by default.
See https://git.openembedded.org/openembedded-core/commit/?id=e462f47489f35902b6972f9837d9adfa542fc796
("Enable sha256 checksums in opkg indexer", 2019) for the original rationale.
AI-Generated: Claude Sonnet 4.6
--- meta/lib/oe/package_manager/ipk/__init__.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-)
meta/lib/oe/package_manager/ipk/__init__.py +9 -5
@@ -12,7 +12,7 @@ from oe.package_manager import * from oe.package_manager.common_deb_ipk import OpkgDpkgPM class OpkgIndexer(Indexer): - def write_index(self): + def write_index(self, skip_checksums=False): arch_vars = ["ALL_MULTILIB_PACKAGE_ARCHS", "SDK_PACKAGE_ARCHS", ]
@@ -44,8 +44,9 @@ class OpkgIndexer(Indexer): if not os.path.exists(pkgs_file): open(pkgs_file, "w").close() - index_cmds.add('%s --checksum md5 --checksum sha256 -r %s -p %s -m %s %s' % - (opkg_index_cmd, pkgs_file, pkgs_file, pkgs_dir, opkg_index_cmd_extra_params)) + checksum_args = '' if skip_checksums else '--checksum md5 --checksum sha256 ' + index_cmds.add('%s %s-r %s -p %s -m %s %s' % + (opkg_index_cmd, checksum_args, pkgs_file, pkgs_file, pkgs_dir, opkg_index_cmd_extra_params)) index_sign_files.add(pkgs_file)
@@ -103,8 +104,12 @@ class OpkgPM(OpkgDpkgPM): self.deploy_dir = oe.path.join(self.d.getVar('WORKDIR'), ipk_repo_workdir) self.deploy_lock_file = os.path.join(self.deploy_dir, "deploy.lock") self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg") + self.from_feeds = (self.d.getVar('BUILD_IMAGES_FROM_FEEDS') or "") == "1" + self._skip_checksums = self.d.getVar('PACKAGE_FEED_SIGN') != '1' and not self.from_feeds self.opkg_args = ['--volatile-cache', '-f', config_file, '-t', self.d.expand('${T}/ipktemp/'), '-o', target_rootfs] self.opkg_args.extend(shlex.split(self.d.getVar("OPKG_ARGS"))) + if self._skip_checksums: + self.opkg_args.append('--force-checksum') if prepare_index: create_packages_dir(self.d, self.deploy_dir, d.getVar("DEPLOY_DIR_IPK"), "package_write_ipk", filterbydependencies)
@@ -116,7 +121,6 @@ class OpkgPM(OpkgDpkgPM): if not os.path.exists(self.d.expand('${T}/saved')): bb.utils.mkdirhier(self.d.expand('${T}/saved')) - self.from_feeds = (self.d.getVar('BUILD_IMAGES_FROM_FEEDS') or "") == "1" if self.from_feeds: self._create_custom_config() else:
@@ -346,7 +350,7 @@ class OpkgPM(OpkgDpkgPM): def write_index(self): self.deploy_dir_lock() - result = self.indexer.write_index() + result = self.indexer.write_index(skip_checksums=self._skip_checksums) self.deploy_dir_unlock()
Signature · 1 line
2.43.0
PATCH 2/2 opkg: enable sha256 checksum verification by default
From: Etienne Cordonnier <[email protected]>
WITH_SHA256=ON adds no external dependency: sha256.c is a self-contained 528-line pure C implementation already present in the source tree.
oe-core already generates SHA256Sum fields in the Packages index (via --checksum sha256 in OpkgIndexer.write_index), but opkg silently ignores them unless compiled with WITH_SHA256=ON. Enabling sha256 by default makes those checksums actually verified at install time, improving package integrity checking at zero dependency cost.
Once SHA256 is universally available, --checksum md5 can also be
dropped from OpkgIndexer.write_index() -- MD5 is collision-prone and
should not be relied upon for integrity checking -- completing the
deprecation noted in https://git.openembedded.org/openembedded-core/commit/?id=e462f47489f35902b6972f9837d9adfa542fc796
(2019).
AI-Generated: Claude Sonnet 4.6
--- meta/recipes-devtools/opkg/opkg_0.10.0.bb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
meta/recipes-devtools/opkg/opkg_0.10.0.bb +1 -1
@@ -30,7 +30,7 @@ inherit cmake pkgconfig ptest target_localstatedir := "${localstatedir}" OPKGLIBDIR ??= "${target_localstatedir}/lib" -PACKAGECONFIG ??= "libsolv ${@bb.utils.filter('DISTRO_FEATURES', 'acl xattr', d)}" +PACKAGECONFIG ??= "libsolv sha256 ${@bb.utils.filter('DISTRO_FEATURES', 'acl xattr', d)}" PACKAGECONFIG[gpg] = "-DWITH_GPGME=ON,-DWITH_GPGME=OFF,\ gnupg gpgme libgpg-error,\
Signature · 1 line
2.43.0
Etienne Cordonnier via lists.openembedded.org schrieb am Mi 29. Jul, 17:29 (+0200):
Quoted 36 lines
From: Etienne Cordonnier <[email protected]>
Computing checksums in the opkg Packages index requires reading every .ipk file in full. For a large image with 7000+ packages (including multi-gigabyte debug packages), this adds 150-300s to every
do_rootfsrun.Checksums in the Packages index are only meaningful for signed feeds (PACKAGE_FEED_SIGN=1): the GPG signature covers the Packages index which contains the SHA256Sum of each .ipk, forming a chain of trust that prevents tampered packages being swapped on a remote feed.
For unsigned local file:// feeds the packages are installed directly from the build host filesystem where there is no tampering risk. Skip all checksum generation in that case by passing no --checksum flags to opkg-make-index (the tool's default behaviour when given no flags). Pass --force-checksum to opkg so it does not error on the absent checksum fields.
On a test image with 7000+ packages (including a 2.3 GB debug package):
write_indextime reduced from ~180s to ~22s (8x speedup) when opkg-make-index is configured to produce no checksums by default.See https://git.openembedded.org/openembedded-core/commit/?id=e462f47489f35902b6972f9837d9adfa542fc796
("Enable sha256 checksums in opkg indexer", 2019) for the original rationale.AI-Generated: Claude Sonnet 4.6
Signed-off-by Etienne Cordonnier <[email protected]>--- meta/lib/oe/package_manager/ipk/__init__.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-)meta/lib/oe/package_manager/ipk/__init__.py +0
-0
meta/lib/oe/package_manager/ipk/__init__.py from oe.package_manager import * · line 15
Quoted 5 lines
meta/lib/oe/package_manager/ipk/__init__.py +1
-1@@ -12,7 +12,7 @@ from oe.package_manager import * from oe.package_manager.common_deb_ipk import OpkgDpkgPM class OpkgIndexer(Indexer): - def write_index(self): + def write_index(self, skip_checksums=False):
Suggestion: I would use the positive case (like with_checksums or use_checksums or build_checksums), because reasoning about/reading code with the inverse is more difficult.
Quoted 16 lines
meta/lib/oe/package_manager/ipk/__init__.py +3
-2arch_vars = ["ALL_MULTILIB_PACKAGE_ARCHS", "SDK_PACKAGE_ARCHS", ]@@ -44,8 +44,9 @@ class OpkgIndexer(Indexer): if not os.path.exists(pkgs_file): open(pkgs_file, "w").close() - index_cmds.add('%s --checksum md5 --checksum sha256 -r %s -p %s -m %s %s' % - (opkg_index_cmd, pkgs_file, pkgs_file, pkgs_dir, opkg_index_cmd_extra_params)) + checksum_args = '' if skip_checksums else '--checksum md5 --checksum sha256 ' + index_cmds.add('%s %s-r %s -p %s -m %s %s' % + (opkg_index_cmd, checksum_args, pkgs_file, pkgs_file, pkgs_dir, opkg_index_cmd_extra_params)) index_sign_files.add(pkgs_file)
meta/lib/oe/package_manager/ipk/__init__.py class OpkgPM(OpkgDpkgPM): · line 107
Quoted 4 lines
meta/lib/oe/package_manager/ipk/__init__.py +1
-0@@ -103,8 +104,12 @@ class OpkgPM(OpkgDpkgPM): self.deploy_dir = oe.path.join(self.d.getVar('WORKDIR'), ipk_repo_workdir) self.deploy_lock_file = os.path.join(self.deploy_dir, "deploy.lock") self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg") + self.from_feeds = (self.d.getVar('BUILD_IMAGES_FROM_FEEDS') or "") == "1"
Or: bb.utils.to_boolean(...)
meta/lib/oe/package_manager/ipk/__init__.py class OpkgPM(OpkgDpkgPM): · line 108
Quoted 1 line
meta/lib/oe/package_manager/ipk/__init__.py +1
-0+ self._skip_checksums = self.d.getVar('PACKAGE_FEED_SIGN') != '1' and not self.from_feeds
I would swap them because evaluation of the second one is cheaper.
Quoted 16 lines
meta/lib/oe/package_manager/ipk/__init__.py +2
-1self.opkg_args = ['--volatile-cache', '-f', config_file, '-t', self.d.expand('${T}/ipktemp/'), '-o', target_rootfs] self.opkg_args.extend(shlex.split(self.d.getVar("OPKG_ARGS"))) + if self._skip_checksums: + self.opkg_args.append('--force-checksum') if prepare_index: create_packages_dir(self.d, self.deploy_dir, d.getVar("DEPLOY_DIR_IPK"), "package_write_ipk", filterbydependencies)@@ -116,7 +121,6 @@ class OpkgPM(OpkgDpkgPM): if not os.path.exists(self.d.expand('${T}/saved')): bb.utils.mkdirhier(self.d.expand('${T}/saved')) - self.from_feeds = (self.d.getVar('BUILD_IMAGES_FROM_FEEDS') or "") == "1" if self.from_feeds: self._create_custom_config() else:
meta/lib/oe/package_manager/ipk/__init__.py class OpkgPM(OpkgDpkgPM): · line 355
Quoted 7 lines
meta/lib/oe/package_manager/ipk/__init__.py +1
-1@@ -346,7 +350,7 @@ class OpkgPM(OpkgDpkgPM): def write_index(self): self.deploy_dir_lock() - result = self.indexer.write_index() + result = self.indexer.write_index(skip_checksums=self._skip_checksums) self.deploy_dir_unlock()
Jörg
Signature · 6 lines
Navimatix GmbH T: 03641 - 327 99 0 Tatzendpromenade 2 F: 03641 - 526 306 07745 Jena www.navimatix.de Geschäftsführer: Steffen Späthe, Jan Rommeley Registergericht: Amtsgericht Jena, HRB 501480
Hi Jörg,
- I've changed to positive logic
- I would rather not change to to_boolean. I'm just moving a line, and introducing to_boolean would make it inconsistent with all the other lines where this variable is used. Also BUILD_IMAGES_FROM_FEEDS is undocumented (it is barely used upstream as far as I understand from
https://lists.yoctoproject.org/g/yocto/topic/install_externally_built_ipk/112649954
), so acceptable values are not 100% clear to me (other lines using this variable treat "False" as a positive value since they call "if d.getVar('BUILD_IMAGES_FROM_FEEDS'):", while to_boolean would do the opposite).
- I've swapped the evaluation order
Quoted 127 lines · Jörg Sommer
Etienne Cordonnier via lists.openembedded.org schrieb am Mi 29. Jul, 17:29 (+0200):
Quoted 5 lines
From: Etienne Cordonnier <[email protected]>
Computing checksums in the opkg Packages index requires reading every .ipk file in full. For a large image with 7000+ packages (including multi-gigabyte debug packages), this adds 150-300s to every
do_rootfsrun.
Quoted 13 lines
Checksums in the Packages index are only meaningful for signed feeds (PACKAGE_FEED_SIGN=1): the GPG signature covers the Packages index which contains the SHA256Sum of each .ipk, forming a chain of trust that prevents tampered packages being swapped on a remote feed.
For unsigned local file:// feeds the packages are installed directly from the build host filesystem where there is no tampering risk. Skip all checksum generation in that case by passing no --checksum flags to opkg-make-index (the tool's default behaviour when given no flags). Pass --force-checksum to opkg so it does not error on the absent checksum fields.
Quoted 5 lines
On a test image with 7000+ packages (including a 2.3 GB debug package):
write_indextime reduced from ~180s to ~22s (8x speedup) when opkg-make-index is configured to produce no checksums by default.See
https://git.openembedded.org/openembedded-core/commit/?id=e462f47489f35902b6972f9837d9adfa542fc796
Quoted 1 line
("Enable sha256 checksums in opkg indexer", 2019) for the original
rationale.
Review
Quoted 8 lines
AI-Generated: Claude Sonnet 4.6
Signed-off-by Etienne Cordonnier <[email protected]>--- meta/lib/oe/package_manager/ipk/__init__.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-)(quoted diff) +0
-0
b/meta/lib/oe/package_manager/ipk/__init__.pymeta/lib/oe/package_manager/ipk/__init__.py from oe.package_manager import * · line 15
Quoted 9 lines
index
344e0852177..6c0b896a086100644meta/lib/oe/package_manager/ipk/__init__.py +1
-1@@ -12,7 +12,7 @@ from oe.package_manager import * from oe.package_manager.common_deb_ipk import OpkgDpkgPM class OpkgIndexer(Indexer): - def write_index(self): + def write_index(self, skip_checksums=False):Suggestion: I would use the positive case (like
with_checksumsoruse_checksumsorbuild_checksums), because reasoning about/reading code with
the inverse is more difficult.meta/lib/oe/package_manager/ipk/__init__.py class OpkgIndexer(Indexer): · line 47
Quoted 8 lines
meta/lib/oe/package_manager/ipk/__init__.py +0
-1arch_vars = ["ALL_MULTILIB_PACKAGE_ARCHS", "SDK_PACKAGE_ARCHS", ]@@ -44,8 +44,9 @@ class OpkgIndexer(Indexer): if not os.path.exists(pkgs_file): open(pkgs_file, "w").close() - index_cmds.add('%s --checksum md5 --checksum sha256 -r%s -p %s -m %s %s' %
meta/lib/oe/package_manager/ipk/__init__.py class OpkgIndexer(Indexer): · line 48
Quoted 1 line
meta/lib/oe/package_manager/ipk/__init__.py +0
-1- (opkg_index_cmd, pkgs_file,
pkgs_file,pkgs_dir,opkg_index_cmd_extra_params))meta/lib/oe/package_manager/ipk/__init__.py class OpkgIndexer(Indexer): · line 47
Quoted 1 line
meta/lib/oe/package_manager/ipk/__init__.py +1
-0+ checksum_args = '' if skip_checksums else '--checksummd5 --checksum sha256 '
meta/lib/oe/package_manager/ipk/__init__.py class OpkgIndexer(Indexer): · line 49
Quoted 2 lines
meta/lib/oe/package_manager/ipk/__init__.py +2
-0+ index_cmds.add('%s %s-r %s -p %s -m %s %s' % + (opkg_index_cmd, checksum_args,
pkgs_file,pkgs_file,pkgs_dir,opkg_index_cmd_extra_params))meta/lib/oe/package_manager/ipk/__init__.py class OpkgPM(OpkgDpkgPM): · line 104
Quoted 5 lines
meta/lib/oe/package_manager/ipk/__init__.py +0
-0index_sign_files.add(pkgs_file)@@ -103,8 +104,12 @@ class OpkgPM(OpkgDpkgPM): self.deploy_dir = oe.path.join(self.d.getVar('WORKDIR'),
ipk_repo_workdir)meta/lib/oe/package_manager/ipk/__init__.py class OpkgPM(OpkgDpkgPM): · line 105
Quoted 1 line
meta/lib/oe/package_manager/ipk/__init__.py +0
-0self.deploy_lock_file = os.path.join(self.deploy_dir,"deploy.lock")
meta/lib/oe/package_manager/ipk/__init__.py class OpkgPM(OpkgDpkgPM): · line 107
Quoted 2 lines
meta/lib/oe/package_manager/ipk/__init__.py +1
-0self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg") + self.from_feeds = (self.d.getVar('BUILD_IMAGES_FROM_FEEDS') or"") == "1"
Or:
bb.utils.to_boolean(...)meta/lib/oe/package_manager/ipk/__init__.py class OpkgPM(OpkgDpkgPM): · line 108
Quoted 1 line
meta/lib/oe/package_manager/ipk/__init__.py +1
-0+ self._skip_checksums = self.d.getVar('PACKAGE_FEED_SIGN') !='1' and not self.from_feeds
I would swap them because evaluation of the second one is cheaper.
meta/lib/oe/package_manager/ipk/__init__.py class OpkgPM(OpkgDpkgPM): · line 109
Quoted 1 line
meta/lib/oe/package_manager/ipk/__init__.py +0
-0self.opkg_args = ['--volatile-cache', '-f', config_file, '-t',
self.d.expand('${T}/ipktemp/'), '-o',target_rootfs]meta/lib/oe/package_manager/ipk/__init__.py class OpkgPM(OpkgDpkgPM):
Quoted 6 lines
meta/lib/oe/package_manager/ipk/__init__.py +2
-0self.opkg_args.extend(shlex.split(self.d.getVar("OPKG_ARGS"))) + if self._skip_checksums: + self.opkg_args.append('--force-checksum') if prepare_index: create_packages_dir(self.d, self.deploy_dir,
d.getVar("DEPLOY_DIR_IPK"), "package_write_ipk", filterbydependencies)meta/lib/oe/package_manager/ipk/__init__.py class OpkgPM(OpkgDpkgPM): · line 119
Quoted 5 lines
meta/lib/oe/package_manager/ipk/__init__.py +0
-1@@ -116,7 +121,6 @@ class OpkgPM(OpkgDpkgPM): if not os.path.exists(self.d.expand('${T}/saved')): bb.utils.mkdirhier(self.d.expand('${T}/saved')) - self.from_feeds = (self.d.getVar('BUILD_IMAGES_FROM_FEEDS') or"") == "1"
meta/lib/oe/package_manager/ipk/__init__.py class OpkgPM(OpkgDpkgPM): · line 353
Quoted 9 lines
meta/lib/oe/package_manager/ipk/__init__.py +1
-1if self.from_feeds: self._create_custom_config() else:@@ -346,7 +350,7 @@ class OpkgPM(OpkgDpkgPM): def write_index(self): self.deploy_dir_lock() - result = self.indexer.write_index() + result =
self.indexer.write_index(skip_checksums=self._skip_checksums)meta/lib/oe/package_manager/ipk/__init__.py class OpkgPM(OpkgDpkgPM): · line 355
Quoted 2 lines
meta/lib/oe/package_manager/ipk/__init__.py +0
-0self.deploy_dir_unlock()Jörg
--
Navimatix GmbH T: 03641 - 327 99 0
Tatzendpromenade 2 F: 03641 - 526 306
07745 Jena www.navimatix.deGeschäftsführer: Steffen Späthe, Jan Rommeley
Registergericht: Amtsgericht Jena, HRB 501480