ipk: Switch to using zstd compression
PATCH 1/1 ipk: Switch to using zstd compression
Converts IPK package generation to use zstd instead of xz. zstd has a much larger compression/speed tradeoff range allowing users to choose what suits them best. In addition, it support parallel compression for additional compression speed up.
A new variable called ZSTD_DEFAULTS is provided to set the defaults for places that want to use zstd for compression; the zst image conversion command is also modified to use this.
Finally, in order for this to function properly, opkg must include zstd support, so it is enabled all the time with no PACKAGECONFIG to turn it off.
--- meta/classes-global/package_ipk.bbclass | 2 +- meta/classes-recipe/image_types.bbclass | 4 +--- meta/conf/bitbake.conf | 4 ++++ meta/lib/oe/package_manager/ipk/__init__.py | 4 ++-- meta/recipes-devtools/opkg/opkg_0.6.2.bb | 6 +++--- 5 files changed, 11 insertions(+), 9 deletions(-)
meta/classes-global/package_ipk.bbclass +1 -1
@@ -15,7 +15,7 @@ IPKGCONF_SDK_TARGET = "${WORKDIR}/opkg-sdk-target.conf" PKGWRITEDIRIPK = "${WORKDIR}/deploy-ipks" # Program to be used to build opkg packages -OPKGBUILDCMD ??= 'opkg-build -Z xz -a "${XZ_DEFAULTS}"' +OPKGBUILDCMD ??= 'opkg-build -Z zstd -a "${ZSTD_DEFAULTS}"' OPKG_ARGS += "--force_postinstall --prefer-arch-to-version" OPKG_ARGS += "${@['', '--no-install-recommends'][d.getVar("NO_RECOMMENDATIONS") == "1"]}"
meta/classes-recipe/image_types.bbclass +1 -3
@@ -63,8 +63,6 @@ ZIP_COMPRESSION_LEVEL ?= "-9" 7ZIP_COMPRESSION_METHOD ?= "BZip2" 7ZIP_EXTENSION ?= "7z" -ZSTD_COMPRESSION_LEVEL ?= "-3" - JFFS2_SUM_EXTRA_ARGS ?= "" IMAGE_CMD:jffs2 = "mkfs.jffs2 --root=${IMAGE_ROOTFS} --faketime --output=${IMGDEPLOYDIR}/${IMAGE_NAME}.jffs2 ${EXTRA_IMAGECMD}"
@@ -337,7 +335,7 @@ CONVERSION_CMD:lz4 = "lz4 -9 -z -l ${IMAGE_NAME}.${type} ${IMAGE_NAME}.${type}.l CONVERSION_CMD:lzo = "lzop -9 ${IMAGE_NAME}.${type}" CONVERSION_CMD:zip = "zip ${ZIP_COMPRESSION_LEVEL} ${IMAGE_NAME}.${type}.zip ${IMAGE_NAME}.${type}" CONVERSION_CMD:7zip = "7za a -mx=${7ZIP_COMPRESSION_LEVEL} -mm=${7ZIP_COMPRESSION_METHOD} ${IMAGE_NAME}.${type}.${7ZIP_EXTENSION} ${IMAGE_NAME}.${type}" -CONVERSION_CMD:zst = "zstd -f -k -T0 -c ${ZSTD_COMPRESSION_LEVEL} ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.zst" +CONVERSION_CMD:zst = "zstd -f -k -c ${ZSTD_DEFAULTS} ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.zst" CONVERSION_CMD:sum = "sumtool -i ${IMAGE_NAME}.${type} -o ${IMAGE_NAME}.${type}.sum ${JFFS2_SUM_EXTRA_ARGS}" CONVERSION_CMD:md5sum = "md5sum ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.md5sum" CONVERSION_CMD:sha1sum = "sha1sum ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.sha1sum"
meta/conf/bitbake.conf +4 -0
@@ -878,6 +878,10 @@ XZ_DEFAULTS[vardepsexclude] += "XZ_MEMLIMIT XZ_THREADS" ZSTD_THREADS ?= "${@oe.utils.cpu_count(at_least=2)}" ZSTD_THREADS[vardepvalue] = "1" +ZSTD_COMPRESSION_LEVEL ?= "-3" +ZSTD_DEFAULTS ?= "--threads=${ZSTD_THREADS} ${ZSTD_COMPRESSION_LEVEL}" +ZSTD_DEFAULTS[vardepsexclude] = "ZSTD_COMPRESSION_LEVEL ZSTD_THREADS" + # Limit the number of threads that OpenMP libraries will use. Otherwise they # may fallback to using all CPUs export OMP_NUM_THREADS = "${BB_NUMBER_THREADS}"
meta/lib/oe/package_manager/ipk/__init__.py +2 -2
@@ -133,7 +133,7 @@ class OpkgDpkgPM(PackageManager): tmp_dir = tempfile.mkdtemp() current_dir = os.getcwd() os.chdir(tmp_dir) - data_tar = 'data.tar.xz' + data_tar = 'data.tar.zst' try: cmd = [ar_cmd, 'x', pkg_path]
@@ -505,6 +505,6 @@ class OpkgPM(OpkgDpkgPM): "trying to extract the package." % pkg) tmp_dir = super(OpkgPM, self).extract(pkg, pkg_info) - bb.utils.remove(os.path.join(tmp_dir, "data.tar.xz")) + bb.utils.remove(os.path.join(tmp_dir, "data.tar.zst")) return tmp_dir
meta/recipes-devtools/opkg/opkg_0.6.2.bb +3 -3
@@ -8,7 +8,7 @@ LICENSE = "GPL-2.0-or-later" LIC_FILES_CHKSUM = "file://COPYING;md5=94d55d512a9ba36caa9b7df079bae19f \ file://src/opkg.c;beginline=4;endline=18;md5=d6200b0f2b41dee278aa5fad333eecae" -DEPENDS = "libarchive" +DEPENDS = "libarchive zstd" PE = "1"
@@ -40,10 +40,10 @@ PACKAGECONFIG[gpg] = "--enable-gpg,--disable-gpg,\ PACKAGECONFIG[curl] = "--enable-curl,--disable-curl,curl" PACKAGECONFIG[ssl-curl] = "--enable-ssl-curl,--disable-ssl-curl,curl openssl" PACKAGECONFIG[sha256] = "--enable-sha256,--disable-sha256" -PACKAGECONFIG[zstd] = "--enable-zstd,--disable-zstd,zstd" PACKAGECONFIG[libsolv] = "--with-libsolv,--without-libsolv,libsolv" -EXTRA_OECONF:class-native = "--localstatedir=/${@os.path.relpath('${localstatedir}', '${STAGING_DIR_NATIVE}')} --sysconfdir=/${@os.path.relpath('${sysconfdir}', '${STAGING_DIR_NATIVE}')}" +EXTRA_OECONF = "--enable-zstd" +EXTRA_OECONF:append:class-native = " --localstatedir=/${@os.path.relpath('${localstatedir}', '${STAGING_DIR_NATIVE}')} --sysconfdir=/${@os.path.relpath('${sysconfdir}', '${STAGING_DIR_NATIVE}')}" do_install:append () { install -d ${D}${sysconfdir}/opkg
Signature · 1 line
2.34.1
Quoted 15 lines · Joshua Watt
meta/classes-recipe/image_types.bbclass +1
-1@@ -337,7 +335,7 @@ CONVERSION_CMD:lz4 = "lz4 -9 -z -l ${IMAGE_NAME}.${type} ${IMAGE_NAME}.${type}.l CONVERSION_CMD:lzo = "lzop -9 ${IMAGE_NAME}.${type}" CONVERSION_CMD:zip = "zip ${ZIP_COMPRESSION_LEVEL} ${IMAGE_NAME}.${type}.zip ${IMAGE_NAME}.${type}" CONVERSION_CMD:7zip = "7za a -mx=${7ZIP_COMPRESSION_LEVEL} -mm=${7ZIP_COMPRESSION_METHOD} ${IMAGE_NAME}.${type}.${7ZIP_EXTENSION} ${IMAGE_NAME}.${type}" -CONVERSION_CMD:zst = "zstd -f -k -T0 -c ${ZSTD_COMPRESSION_LEVEL} ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.zst" +CONVERSION_CMD:zst = "zstd -f -k -c ${ZSTD_DEFAULTS} ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.zst" CONVERSION_CMD:sum = "sumtool -i ${IMAGE_NAME}.${type} -o ${IMAGE_NAME}.${type}.sum ${JFFS2_SUM_EXTRA_ARGS}" CONVERSION_CMD:md5sum = "md5sum ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.md5sum" CONVERSION_CMD:sha1sum = "sha1sum ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.sha1sum"meta/conf/bitbake.conf +0
-0
meta/conf/bitbake.conf XZ_DEFAULTS[vardepsexclude] += "XZ_MEMLIMIT XZ_THREADS" · line 884
Quoted 7 lines
meta/conf/bitbake.conf +4
-0@@ -878,6 +878,10 @@ XZ_DEFAULTS[vardepsexclude] += "XZ_MEMLIMIT XZ_THREADS" ZSTD_THREADS ?= "${@oe.utils.cpu_count(at_least=2)}" ZSTD_THREADS[vardepvalue] = "1" +ZSTD_COMPRESSION_LEVEL ?= "-3" +ZSTD_DEFAULTS ?= "--threads=${ZSTD_THREADS} ${ZSTD_COMPRESSION_LEVEL}" +ZSTD_DEFAULTS[vardepsexclude] = "ZSTD_COMPRESSION_LEVEL ZSTD_THREADS" +
Threads, sure but the compression level does change the output...
Cheers,
Richard
Quoted 19 lines · Richard Purdie
Quoted 18 lines · Joshua Watt
meta/classes-recipe/image_types.bbclass +1
-1@@ -337,7 +335,7 @@ CONVERSION_CMD:lz4 = "lz4 -9 -z -l ${IMAGE_NAME}.${type} ${IMAGE_NAME}.${type}.l CONVERSION_CMD:lzo = "lzop -9 ${IMAGE_NAME}.${type}" CONVERSION_CMD:zip = "zip ${ZIP_COMPRESSION_LEVEL} ${IMAGE_NAME}.${type}.zip ${IMAGE_NAME}.${type}" CONVERSION_CMD:7zip = "7za a -mx=${7ZIP_COMPRESSION_LEVEL} -mm=${7ZIP_COMPRESSION_METHOD} ${IMAGE_NAME}.${type}.${7ZIP_EXTENSION} ${IMAGE_NAME}.${type}" -CONVERSION_CMD:zst = "zstd -f -k -T0 -c ${ZSTD_COMPRESSION_LEVEL} ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.zst" +CONVERSION_CMD:zst = "zstd -f -k -c ${ZSTD_DEFAULTS} ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.zst" CONVERSION_CMD:sum = "sumtool -i ${IMAGE_NAME}.${type} -o ${IMAGE_NAME}.${type}.sum ${JFFS2_SUM_EXTRA_ARGS}" CONVERSION_CMD:md5sum = "md5sum ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.md5sum" CONVERSION_CMD:sha1sum = "sha1sum ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.sha1sum"meta/conf/bitbake.conf +0
-0@@ -878,6 +878,10 @@ XZ_DEFAULTS[vardepsexclude] += "XZ_MEMLIMIT XZ_THREADS" ZSTD_THREADS ?= "${@oe.utils.cpu_count(at_least=2)}" ZSTD_THREADS[vardepvalue] = "1"
Quoted 6 lines
meta/conf/bitbake.conf XZ_DEFAULTS[vardepsexclude] += "XZ_MEMLIMIT XZ_THREADS" · line 884
Quoted 4 lines
meta/conf/bitbake.conf +4
-0+ZSTD_COMPRESSION_LEVEL ?= "-3" +ZSTD_DEFAULTS ?= "--threads=${ZSTD_THREADS} ${ZSTD_COMPRESSION_LEVEL}" +ZSTD_DEFAULTS[vardepsexclude] = "ZSTD_COMPRESSION_LEVEL ZSTD_THREADS" +Threads, sure but the compression level does change the output...
Indeed: https://autobuilder.yoctoproject.org/typhoon/#/builders/117/builds/4098/steps/12/logs/stdio
Quoted 14 lines
Cheers,
Richard
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#192550): https://lists.openembedded.org/g/openembedded-core/message/192550
Mute This Topic: https://lists.openembedded.org/mt/103198693/3617179
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [[email protected]]
-=-=-=-=-=-=-=-=-=-=-=-
Signature · 3 lines
Alexandre Belloni, co-owner and COO, Bootlin Embedded Linux and Kernel engineering https://bootlin.com
"In addition, it support parallel compression for additional compression speed up."
It is not really clear what you mean with this sentence: xz also supports parallel compression (in poky via XZ_THREADS), so zstd support for parallel compression is not a reason to switch. zstd is rather known for fast decompression, and in my own testing was much faster than xz for compressing very big files (see this thread where I had done some benchmarking of xz vs zstd for poky:
https://lists.openembedded.org/g/openembedded-core/topic/93654146#170610 )
Etienne
Quoted 69 lines · Alexandre Belloni via lists.openembedded.org
Quoted 3 lines · Richard Purdie
Review CONVERSION_CMD:lz4 = "lz4 -9 -z -l
Quoted 2 lines · Joshua Watt
(quoted diff) +0
-0@@ -337,7 +335,7 @@ CONVERSION_CMD:lz4 = "lz4 -9 -z -l${IMAGE_NAME}.${type} ${IMAGE_NAME}.${type}.l
Quoted 2 lines
Review CONVERSION_CMD:lz4 = "lz4 -9 -z -l · line 336
Quoted 2 lines
(quoted diff) +0
-0CONVERSION_CMD:lzo = "lzop -9 ${IMAGE_NAME}.${type}" CONVERSION_CMD:zip = "zip ${ZIP_COMPRESSION_LEVEL}${IMAGE_NAME}.${type}.zip ${IMAGE_NAME}.${type}"
Quoted 1 line
Review CONVERSION_CMD:lz4 = "lz4 -9 -z -l · line 337
Quoted 1 line
(quoted diff) +0
-0CONVERSION_CMD:7zip = "7za a -mx=${7ZIP_COMPRESSION_LEVEL}-mm=${7ZIP_COMPRESSION_METHOD} ${IMAGE_NAME}.${type}.${7ZIP_EXTENSION} ${IMAGE_NAME}.${type}"
Quoted 1 line
Review CONVERSION_CMD:lz4 = "lz4 -9 -z -l · line 340
Quoted 1 line
(quoted diff) +0
-1-CONVERSION_CMD:zst = "zstd -f -k -T0 -c ${ZSTD_COMPRESSION_LEVEL}${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.zst"
Quoted 1 line
Review CONVERSION_CMD:lz4 = "lz4 -9 -z -l · line 338
Quoted 1 line
(quoted diff) +1
-0+CONVERSION_CMD:zst = "zstd -f -k -c ${ZSTD_DEFAULTS}${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.zst"
Quoted 1 line
Review CONVERSION_CMD:lz4 = "lz4 -9 -z -l · line 339
Quoted 1 line
(quoted diff) +0
-0CONVERSION_CMD:sum = "sumtool -i ${IMAGE_NAME}.${type} -o${IMAGE_NAME}.${type}.sum ${JFFS2_SUM_EXTRA_ARGS}"
Quoted 1 line
Review CONVERSION_CMD:lz4 = "lz4 -9 -z -l · line 340
Quoted 1 line
(quoted diff) +0
-0CONVERSION_CMD:md5sum = "md5sum ${IMAGE_NAME}.${type} >${IMAGE_NAME}.${type}.md5sum"
Quoted 1 line
Review CONVERSION_CMD:lz4 = "lz4 -9 -z -l · line 341
Quoted 1 line
(quoted diff) +0
-0CONVERSION_CMD:sha1sum = "sha1sum ${IMAGE_NAME}.${type} >${IMAGE_NAME}.${type}.sha1sum"
Quoted 5 lines
meta/conf/bitbake.conf XZ_DEFAULTS[vardepsexclude] += "XZ_MEMLIMIT
Quoted 5 lines
meta/conf/bitbake.conf +0
-0@@ -878,6 +878,10 @@ XZ_DEFAULTS[vardepsexclude] += "XZ_MEMLIMIT
XZ_THREADS"Quoted 9 lines
meta/conf/bitbake.conf XZ_DEFAULTS[vardepsexclude] += "XZ_MEMLIMIT XZ_THREADS" · line 884
Quoted 7 lines
meta/conf/bitbake.conf +4
-0ZSTD_THREADS ?= "${@oe.utils.cpu_count(at_least=2)}" ZSTD_THREADS[vardepvalue] = "1" +ZSTD_COMPRESSION_LEVEL ?= "-3" +ZSTD_DEFAULTS ?= "--threads=${ZSTD_THREADS} ${ZSTD_COMPRESSION_LEVEL}" +ZSTD_DEFAULTS[vardepsexclude] = "ZSTD_COMPRESSION_LEVEL ZSTD_THREADS" +Threads, sure but the compression level does change the output...
Quoted 8 lines
Cheers,
Richard
--
Alexandre Belloni, co-owner and COO, Bootlin
Embedded Linux and Kernel engineering-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group. View/Reply Online (#192630):
https://urldefense.proofpoint.com/v2/url?u=https-3A__lists.openembedded.org_g_openembedded-2Dcore_message_192630&d=DwIFaQ&c=ncDTmphkJTvjIDPh0hpF_4vCHvabgGkICC2epckfdiw&r=AhkbNonVuMIGRfPx_Qj9TsRih1DULJTKUkSGa66m67E&m=VXbIqF3VsSJLhfuxHVnkEJmbESqEa7d1xzRDMF_I8T6QV7FiZqqrai93ni13zQ_Q&s=k685ISY4i-G6Hp4zqVgYfnk-xew6vKgC-JT9kW2WLKw&e=
Mute This Topic:
https://urldefense.proofpoint.com/v2/url?u=https-3A__lists.openembedded.org_mt_103198693_7048771&d=DwIFaQ&c=ncDTmphkJTvjIDPh0hpF_4vCHvabgGkICC2epckfdiw&r=AhkbNonVuMIGRfPx_Qj9TsRih1DULJTKUkSGa66m67E&m=VXbIqF3VsSJLhfuxHVnkEJmbESqEa7d1xzRDMF_I8T6QV7FiZqqrai93ni13zQ_Q&s=70x-KZ_YKkAOtaXmyL5phwvgr8YbbWVpLg4RZFtitBA&e=
Group Owner: [email protected] Unsubscribe:
https://urldefense.proofpoint.com/v2/url?u=https-3A__lists.openembedded.org_g_openembedded-2Dcore_unsub&d=DwIFaQ&c=ncDTmphkJTvjIDPh0hpF_4vCHvabgGkICC2epckfdiw&r=AhkbNonVuMIGRfPx_Qj9TsRih1DULJTKUkSGa66m67E&m=VXbIqF3VsSJLhfuxHVnkEJmbESqEa7d1xzRDMF_I8T6QV7FiZqqrai93ni13zQ_Q&s=Ij8_R9oYgqphELn5UASAx5RKXsEUDZmuGqppiqL63cw&e=
[[email protected]]
-=-=-=-=-=-=-=-=-=-=-=-
PATCH v2 1/1 ipk: Switch to using zstd compression
In reply to Joshua WattConverts IPK package generation to use zstd instead of xz. zstd has a much larger compression/speed tradeoff range allowing users to choose what suits them best, and fast decompression speeds. It also continues to support parallel compression as xz did.
A new variable called ZSTD_DEFAULTS is provided to set the defaults for places that want to use zstd for compression; the zst image conversion command is also modified to use this.
Finally, in order for this to function properly, opkg must include zstd support, so it is enabled all the time with no PACKAGECONFIG to turn it off.
--- meta/classes-global/package_ipk.bbclass | 2 +- meta/classes-recipe/image_types.bbclass | 4 +--- meta/conf/bitbake.conf | 4 ++++ meta/lib/oe/package_manager/ipk/__init__.py | 4 ++-- meta/recipes-devtools/opkg/opkg_0.6.2.bb | 6 +++--- 5 files changed, 11 insertions(+), 9 deletions(-)
meta/classes-global/package_ipk.bbclass +1 -1
@@ -15,7 +15,7 @@ IPKGCONF_SDK_TARGET = "${WORKDIR}/opkg-sdk-target.conf" PKGWRITEDIRIPK = "${WORKDIR}/deploy-ipks" # Program to be used to build opkg packages -OPKGBUILDCMD ??= 'opkg-build -Z xz -a "${XZ_DEFAULTS}"' +OPKGBUILDCMD ??= 'opkg-build -Z zstd -a "${ZSTD_DEFAULTS}"' OPKG_ARGS += "--force_postinstall --prefer-arch-to-version" OPKG_ARGS += "${@['', '--no-install-recommends'][d.getVar("NO_RECOMMENDATIONS") == "1"]}"
meta/classes-recipe/image_types.bbclass +1 -3
@@ -63,8 +63,6 @@ ZIP_COMPRESSION_LEVEL ?= "-9" 7ZIP_COMPRESSION_METHOD ?= "BZip2" 7ZIP_EXTENSION ?= "7z" -ZSTD_COMPRESSION_LEVEL ?= "-3" - JFFS2_SUM_EXTRA_ARGS ?= "" IMAGE_CMD:jffs2 = "mkfs.jffs2 --root=${IMAGE_ROOTFS} --faketime --output=${IMGDEPLOYDIR}/${IMAGE_NAME}.jffs2 ${EXTRA_IMAGECMD}"
@@ -337,7 +335,7 @@ CONVERSION_CMD:lz4 = "lz4 -9 -z -l ${IMAGE_NAME}.${type} ${IMAGE_NAME}.${type}.l CONVERSION_CMD:lzo = "lzop -9 ${IMAGE_NAME}.${type}" CONVERSION_CMD:zip = "zip ${ZIP_COMPRESSION_LEVEL} ${IMAGE_NAME}.${type}.zip ${IMAGE_NAME}.${type}" CONVERSION_CMD:7zip = "7za a -mx=${7ZIP_COMPRESSION_LEVEL} -mm=${7ZIP_COMPRESSION_METHOD} ${IMAGE_NAME}.${type}.${7ZIP_EXTENSION} ${IMAGE_NAME}.${type}" -CONVERSION_CMD:zst = "zstd -f -k -T0 -c ${ZSTD_COMPRESSION_LEVEL} ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.zst" +CONVERSION_CMD:zst = "zstd -f -k -c ${ZSTD_DEFAULTS} ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.zst" CONVERSION_CMD:sum = "sumtool -i ${IMAGE_NAME}.${type} -o ${IMAGE_NAME}.${type}.sum ${JFFS2_SUM_EXTRA_ARGS}" CONVERSION_CMD:md5sum = "md5sum ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.md5sum" CONVERSION_CMD:sha1sum = "sha1sum ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.sha1sum"
meta/conf/bitbake.conf +4 -0
@@ -878,6 +878,10 @@ XZ_DEFAULTS[vardepsexclude] += "XZ_MEMLIMIT XZ_THREADS" ZSTD_THREADS ?= "${@oe.utils.cpu_count(at_least=2)}" ZSTD_THREADS[vardepvalue] = "1" +ZSTD_COMPRESSION_LEVEL ?= "-3" +ZSTD_DEFAULTS ?= "--threads=${ZSTD_THREADS} ${ZSTD_COMPRESSION_LEVEL}" +ZSTD_DEFAULTS[vardepsexclude] = "ZSTD_THREADS" + # Limit the number of threads that OpenMP libraries will use. Otherwise they # may fallback to using all CPUs export OMP_NUM_THREADS = "${BB_NUMBER_THREADS}"
meta/lib/oe/package_manager/ipk/__init__.py +2 -2
@@ -133,7 +133,7 @@ class OpkgDpkgPM(PackageManager): tmp_dir = tempfile.mkdtemp() current_dir = os.getcwd() os.chdir(tmp_dir) - data_tar = 'data.tar.xz' + data_tar = 'data.tar.zst' try: cmd = [ar_cmd, 'x', pkg_path]
@@ -505,6 +505,6 @@ class OpkgPM(OpkgDpkgPM): "trying to extract the package." % pkg) tmp_dir = super(OpkgPM, self).extract(pkg, pkg_info) - bb.utils.remove(os.path.join(tmp_dir, "data.tar.xz")) + bb.utils.remove(os.path.join(tmp_dir, "data.tar.zst")) return tmp_dir
meta/recipes-devtools/opkg/opkg_0.6.2.bb +3 -3
@@ -8,7 +8,7 @@ LICENSE = "GPL-2.0-or-later" LIC_FILES_CHKSUM = "file://COPYING;md5=94d55d512a9ba36caa9b7df079bae19f \ file://src/opkg.c;beginline=4;endline=18;md5=d6200b0f2b41dee278aa5fad333eecae" -DEPENDS = "libarchive" +DEPENDS = "libarchive zstd" PE = "1"
@@ -40,10 +40,10 @@ PACKAGECONFIG[gpg] = "--enable-gpg,--disable-gpg,\ PACKAGECONFIG[curl] = "--enable-curl,--disable-curl,curl" PACKAGECONFIG[ssl-curl] = "--enable-ssl-curl,--disable-ssl-curl,curl openssl" PACKAGECONFIG[sha256] = "--enable-sha256,--disable-sha256" -PACKAGECONFIG[zstd] = "--enable-zstd,--disable-zstd,zstd" PACKAGECONFIG[libsolv] = "--with-libsolv,--without-libsolv,libsolv" -EXTRA_OECONF:class-native = "--localstatedir=/${@os.path.relpath('${localstatedir}', '${STAGING_DIR_NATIVE}')} --sysconfdir=/${@os.path.relpath('${sysconfdir}', '${STAGING_DIR_NATIVE}')}" +EXTRA_OECONF = "--enable-zstd" +EXTRA_OECONF:append:class-native = " --localstatedir=/${@os.path.relpath('${localstatedir}', '${STAGING_DIR_NATIVE}')} --sysconfdir=/${@os.path.relpath('${sysconfdir}', '${STAGING_DIR_NATIVE}')}" do_install:append () { install -d ${D}${sysconfdir}/opkg
Signature · 1 line
2.34.1
Hi Josh,
This still causes reproducibility issues:
https://autobuilder.yoctoproject.org/typhoon/#/builders/117/builds/4113/steps/13/logs/stdio
https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20231219-mrw5vs1g/packages/diff-html/
Quoted 132 lines · Joshua Watt
Converts IPK package generation to use zstd instead of xz. zstd has a much larger compression/speed tradeoff range allowing users to choose what suits them best, and fast decompression speeds. It also continues to support parallel compression as xz did.
A new variable called
ZSTD_DEFAULTSis provided to set the defaults for places that want to use zstd for compression; the zst image conversion command is also modified to use this.Finally, in order for this to function properly, opkg must include zstd support, so it is enabled all the time with no PACKAGECONFIG to turn it off.
Signed-off-by Joshua Watt <[email protected]>--- meta/classes-global/package_ipk.bbclass | 2 +- meta/classes-recipe/image_types.bbclass | 4 +--- meta/conf/bitbake.conf | 4 ++++ meta/lib/oe/package_manager/ipk/__init__.py | 4 ++-- meta/recipes-devtools/opkg/opkg_0.6.2.bb | 6 +++--- 5 files changed, 11 insertions(+), 9 deletions(-)meta/classes-global/package_ipk.bbclass +1
-1@@ -15,7 +15,7 @@ IPKGCONF_SDK_TARGET = "${WORKDIR}/opkg-sdk-target.conf" PKGWRITEDIRIPK = "${WORKDIR}/deploy-ipks" # Program to be used to build opkg packages -OPKGBUILDCMD ??= 'opkg-build -Z xz -a "${XZ_DEFAULTS}"' +OPKGBUILDCMD ??= 'opkg-build -Z zstd -a "${ZSTD_DEFAULTS}"' OPKG_ARGS += "--force_postinstall --prefer-arch-to-version" OPKG_ARGS += "${@['', '--no-install-recommends'][d.getVar("NO_RECOMMENDATIONS") == "1"]}"meta/classes-recipe/image_types.bbclass +1
-3@@ -63,8 +63,6 @@ ZIP_COMPRESSION_LEVEL ?= "-9" 7ZIP_COMPRESSION_METHOD ?= "BZip2" 7ZIP_EXTENSION ?= "7z" -ZSTD_COMPRESSION_LEVEL ?= "-3" - JFFS2_SUM_EXTRA_ARGS ?= "" IMAGE_CMD:jffs2 = "mkfs.jffs2 --root=${IMAGE_ROOTFS} --faketime --output=${IMGDEPLOYDIR}/${IMAGE_NAME}.jffs2 ${EXTRA_IMAGECMD}"@@ -337,7 +335,7 @@ CONVERSION_CMD:lz4 = "lz4 -9 -z -l ${IMAGE_NAME}.${type} ${IMAGE_NAME}.${type}.l CONVERSION_CMD:lzo = "lzop -9 ${IMAGE_NAME}.${type}" CONVERSION_CMD:zip = "zip ${ZIP_COMPRESSION_LEVEL} ${IMAGE_NAME}.${type}.zip ${IMAGE_NAME}.${type}" CONVERSION_CMD:7zip = "7za a -mx=${7ZIP_COMPRESSION_LEVEL} -mm=${7ZIP_COMPRESSION_METHOD} ${IMAGE_NAME}.${type}.${7ZIP_EXTENSION} ${IMAGE_NAME}.${type}" -CONVERSION_CMD:zst = "zstd -f -k -T0 -c ${ZSTD_COMPRESSION_LEVEL} ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.zst" +CONVERSION_CMD:zst = "zstd -f -k -c ${ZSTD_DEFAULTS} ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.zst" CONVERSION_CMD:sum = "sumtool -i ${IMAGE_NAME}.${type} -o ${IMAGE_NAME}.${type}.sum ${JFFS2_SUM_EXTRA_ARGS}" CONVERSION_CMD:md5sum = "md5sum ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.md5sum" CONVERSION_CMD:sha1sum = "sha1sum ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.sha1sum"meta/conf/bitbake.conf +4
-0@@ -878,6 +878,10 @@ XZ_DEFAULTS[vardepsexclude] += "XZ_MEMLIMIT XZ_THREADS" ZSTD_THREADS ?= "${@oe.utils.cpu_count(at_least=2)}" ZSTD_THREADS[vardepvalue] = "1" +ZSTD_COMPRESSION_LEVEL ?= "-3" +ZSTD_DEFAULTS ?= "--threads=${ZSTD_THREADS} ${ZSTD_COMPRESSION_LEVEL}" +ZSTD_DEFAULTS[vardepsexclude] = "ZSTD_THREADS" + # Limit the number of threads that OpenMP libraries will use. Otherwise they # may fallback to using all CPUs export OMP_NUM_THREADS = "${BB_NUMBER_THREADS}"meta/lib/oe/package_manager/ipk/__init__.py +2
-2@@ -133,7 +133,7 @@ class OpkgDpkgPM(PackageManager): tmp_dir = tempfile.mkdtemp() current_dir = os.getcwd() os.chdir(tmp_dir) - data_tar = 'data.tar.xz' + data_tar = 'data.tar.zst' try: cmd = [ar_cmd, 'x', pkg_path]@@ -505,6 +505,6 @@ class OpkgPM(OpkgDpkgPM): "trying to extract the package." % pkg) tmp_dir = super(OpkgPM, self).extract(pkg, pkg_info) - bb.utils.remove(os.path.join(tmp_dir, "data.tar.xz")) + bb.utils.remove(os.path.join(tmp_dir, "data.tar.zst")) return tmp_dirmeta/recipes-devtools/opkg/opkg_0.6.2.bb +3
-3@@ -8,7 +8,7 @@ LICENSE = "GPL-2.0-or-later" LIC_FILES_CHKSUM = "file://COPYING;md5=94d55d512a9ba36caa9b7df079bae19f \ file://src/opkg.c;beginline=4;endline=18;md5=d6200b0f2b41dee278aa5fad333eecae" -DEPENDS = "libarchive" +DEPENDS = "libarchive zstd" PE = "1"@@ -40,10 +40,10 @@ PACKAGECONFIG[gpg] = "--enable-gpg,--disable-gpg,\ PACKAGECONFIG[curl] = "--enable-curl,--disable-curl,curl" PACKAGECONFIG[ssl-curl] = "--enable-ssl-curl,--disable-ssl-curl,curl openssl" PACKAGECONFIG[sha256] = "--enable-sha256,--disable-sha256" -PACKAGECONFIG[zstd] = "--enable-zstd,--disable-zstd,zstd" PACKAGECONFIG[libsolv] = "--with-libsolv,--without-libsolv,libsolv" -EXTRA_OECONF:class-native = "--localstatedir=/${@os.path.relpath('${localstatedir}', '${STAGING_DIR_NATIVE}')} --sysconfdir=/${@os.path.relpath('${sysconfdir}', '${STAGING_DIR_NATIVE}')}" +EXTRA_OECONF = "--enable-zstd" +EXTRA_OECONF:append:class-native = " --localstatedir=/${@os.path.relpath('${localstatedir}', '${STAGING_DIR_NATIVE}')} --sysconfdir=/${@os.path.relpath('${sysconfdir}', '${STAGING_DIR_NATIVE}')}" do_install:append () { install -d ${D}${sysconfdir}/opkgSignature · 11 lines
2.34.1 -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#192657): https://lists.openembedded.org/g/openembedded-core/message/192657 Mute This Topic: https://lists.openembedded.org/mt/103244564/3617179 Group Owner: [email protected] Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [[email protected]] -=-=-=-=-=-=-=-=-=-=-=-
Signature · 3 lines
Alexandre Belloni, co-owner and COO, Bootlin Embedded Linux and Kernel engineering https://bootlin.com
PATCH v3 1/1 ipk: Switch to using zstd compression
In reply to Joshua WattConverts IPK package generation to use zstd instead of xz. zstd has a much larger compression/speed tradeoff range allowing users to choose what suits them best, and fast decompression speeds. It also continues to support parallel compression as xz did.
A new variable called ZSTD_DEFAULTS is provided to set the defaults for places that want to use zstd for compression; the zst image conversion command is also modified to use this.
Finally, in order for this to function properly, opkg must include zstd support, so it is enabled all the time with no PACKAGECONFIG to turn it off.
--- meta/classes-global/package_ipk.bbclass | 4 ++-- meta/classes-recipe/image_types.bbclass | 4 +--- meta/conf/bitbake.conf | 4 ++++ meta/lib/oe/package_manager/ipk/__init__.py | 4 ++-- meta/recipes-devtools/opkg/opkg_0.6.2.bb | 6 +++--- 5 files changed, 12 insertions(+), 10 deletions(-)
meta/classes-global/package_ipk.bbclass +2 -2
@@ -15,7 +15,7 @@ IPKGCONF_SDK_TARGET = "${WORKDIR}/opkg-sdk-target.conf" PKGWRITEDIRIPK = "${WORKDIR}/deploy-ipks" # Program to be used to build opkg packages -OPKGBUILDCMD ??= 'opkg-build -Z xz -a "${XZ_DEFAULTS}"' +OPKGBUILDCMD ??= 'opkg-build -Z zstd -a "${ZSTD_DEFAULTS}"' OPKG_ARGS += "--force_postinstall --prefer-arch-to-version" OPKG_ARGS += "${@['', '--no-install-recommends'][d.getVar("NO_RECOMMENDATIONS") == "1"]}"
@@ -277,7 +277,7 @@ addtask do_package_write_ipk_setscene python () { if d.getVar('PACKAGES') != '': - deps = ' opkg-utils-native:do_populate_sysroot virtual/fakeroot-native:do_populate_sysroot xz-native:do_populate_sysroot' + deps = ' opkg-utils-native:do_populate_sysroot virtual/fakeroot-native:do_populate_sysroot zstd-native:do_populate_sysroot' d.appendVarFlag('do_package_write_ipk', 'depends', deps) d.setVarFlag('do_package_write_ipk', 'fakeroot', "1")
meta/classes-recipe/image_types.bbclass +1 -3
@@ -63,8 +63,6 @@ ZIP_COMPRESSION_LEVEL ?= "-9" 7ZIP_COMPRESSION_METHOD ?= "BZip2" 7ZIP_EXTENSION ?= "7z" -ZSTD_COMPRESSION_LEVEL ?= "-3" - JFFS2_SUM_EXTRA_ARGS ?= "" IMAGE_CMD:jffs2 = "mkfs.jffs2 --root=${IMAGE_ROOTFS} --faketime --output=${IMGDEPLOYDIR}/${IMAGE_NAME}.jffs2 ${EXTRA_IMAGECMD}"
@@ -337,7 +335,7 @@ CONVERSION_CMD:lz4 = "lz4 -9 -z -l ${IMAGE_NAME}.${type} ${IMAGE_NAME}.${type}.l CONVERSION_CMD:lzo = "lzop -9 ${IMAGE_NAME}.${type}" CONVERSION_CMD:zip = "zip ${ZIP_COMPRESSION_LEVEL} ${IMAGE_NAME}.${type}.zip ${IMAGE_NAME}.${type}" CONVERSION_CMD:7zip = "7za a -mx=${7ZIP_COMPRESSION_LEVEL} -mm=${7ZIP_COMPRESSION_METHOD} ${IMAGE_NAME}.${type}.${7ZIP_EXTENSION} ${IMAGE_NAME}.${type}" -CONVERSION_CMD:zst = "zstd -f -k -T0 -c ${ZSTD_COMPRESSION_LEVEL} ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.zst" +CONVERSION_CMD:zst = "zstd -f -k -c ${ZSTD_DEFAULTS} ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.zst" CONVERSION_CMD:sum = "sumtool -i ${IMAGE_NAME}.${type} -o ${IMAGE_NAME}.${type}.sum ${JFFS2_SUM_EXTRA_ARGS}" CONVERSION_CMD:md5sum = "md5sum ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.md5sum" CONVERSION_CMD:sha1sum = "sha1sum ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.sha1sum"
meta/conf/bitbake.conf +4 -0
@@ -878,6 +878,10 @@ XZ_DEFAULTS[vardepsexclude] += "XZ_MEMLIMIT XZ_THREADS" ZSTD_THREADS ?= "${@oe.utils.cpu_count(at_least=2)}" ZSTD_THREADS[vardepvalue] = "1" +ZSTD_COMPRESSION_LEVEL ?= "-3" +ZSTD_DEFAULTS ?= "--threads=${ZSTD_THREADS} ${ZSTD_COMPRESSION_LEVEL}" +ZSTD_DEFAULTS[vardepsexclude] = "ZSTD_THREADS" + # Limit the number of threads that OpenMP libraries will use. Otherwise they # may fallback to using all CPUs export OMP_NUM_THREADS = "${BB_NUMBER_THREADS}"
meta/lib/oe/package_manager/ipk/__init__.py +2 -2
@@ -133,7 +133,7 @@ class OpkgDpkgPM(PackageManager): tmp_dir = tempfile.mkdtemp() current_dir = os.getcwd() os.chdir(tmp_dir) - data_tar = 'data.tar.xz' + data_tar = 'data.tar.zst' try: cmd = [ar_cmd, 'x', pkg_path]
@@ -505,6 +505,6 @@ class OpkgPM(OpkgDpkgPM): "trying to extract the package." % pkg) tmp_dir = super(OpkgPM, self).extract(pkg, pkg_info) - bb.utils.remove(os.path.join(tmp_dir, "data.tar.xz")) + bb.utils.remove(os.path.join(tmp_dir, "data.tar.zst")) return tmp_dir
meta/recipes-devtools/opkg/opkg_0.6.2.bb +3 -3
@@ -8,7 +8,7 @@ LICENSE = "GPL-2.0-or-later" LIC_FILES_CHKSUM = "file://COPYING;md5=94d55d512a9ba36caa9b7df079bae19f \ file://src/opkg.c;beginline=4;endline=18;md5=d6200b0f2b41dee278aa5fad333eecae" -DEPENDS = "libarchive" +DEPENDS = "libarchive zstd" PE = "1"
@@ -40,10 +40,10 @@ PACKAGECONFIG[gpg] = "--enable-gpg,--disable-gpg,\ PACKAGECONFIG[curl] = "--enable-curl,--disable-curl,curl" PACKAGECONFIG[ssl-curl] = "--enable-ssl-curl,--disable-ssl-curl,curl openssl" PACKAGECONFIG[sha256] = "--enable-sha256,--disable-sha256" -PACKAGECONFIG[zstd] = "--enable-zstd,--disable-zstd,zstd" PACKAGECONFIG[libsolv] = "--with-libsolv,--without-libsolv,libsolv" -EXTRA_OECONF:class-native = "--localstatedir=/${@os.path.relpath('${localstatedir}', '${STAGING_DIR_NATIVE}')} --sysconfdir=/${@os.path.relpath('${sysconfdir}', '${STAGING_DIR_NATIVE}')}" +EXTRA_OECONF = "--enable-zstd" +EXTRA_OECONF:append:class-native = " --localstatedir=/${@os.path.relpath('${localstatedir}', '${STAGING_DIR_NATIVE}')} --sysconfdir=/${@os.path.relpath('${sysconfdir}', '${STAGING_DIR_NATIVE}')}" do_install:append () { install -d ${D}${sysconfdir}/opkg
Signature · 1 line
2.34.1