Skip to contentSkip to conversation

ipk: Switch to using zstd compression

PATCH Superseded Joshua Watt 4 participants 7 messages Dec 15, 2023 – Dec 20, 2023

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 ipk: Switch to using zstd compression

Converts IPK package generation to use zstd instead of xz. zstd has a much larger compression/speed tradeoff range allowing users to choose what suits them best. In addition, it support parallel compression for additional compression speed up.

A new variable called ZSTD_DEFAULTS is provided to set the defaults for places that want to use zstd for compression; the zst image conversion command is also modified to use this.

Finally, in order for this to function properly, opkg must include zstd support, so it is enabled all the time with no PACKAGECONFIG to turn it off.

---
 meta/classes-global/package_ipk.bbclass     | 2 +-
 meta/classes-recipe/image_types.bbclass     | 4 +---
 meta/conf/bitbake.conf                      | 4 ++++
 meta/lib/oe/package_manager/ipk/__init__.py | 4 ++--
 meta/recipes-devtools/opkg/opkg_0.6.2.bb    | 6 +++---
 5 files changed, 11 insertions(+), 9 deletions(-)
meta/classes-global/package_ipk.bbclass +1 -1
index 71ffdd522ac..025dba49313 100644
@@ -15,7 +15,7 @@ IPKGCONF_SDK_TARGET = "${WORKDIR}/opkg-sdk-target.conf"
 PKGWRITEDIRIPK = "${WORKDIR}/deploy-ipks"
 
 # Program to be used to build opkg packages
-OPKGBUILDCMD ??= 'opkg-build -Z xz -a "${XZ_DEFAULTS}"'
+OPKGBUILDCMD ??= 'opkg-build -Z zstd -a "${ZSTD_DEFAULTS}"'
 
 OPKG_ARGS += "--force_postinstall --prefer-arch-to-version"
 OPKG_ARGS += "${@['', '--no-install-recommends'][d.getVar("NO_RECOMMENDATIONS") == "1"]}"
meta/classes-recipe/image_types.bbclass +1 -3
index d615b41ed1e..3733bdfc20c 100644
@@ -63,8 +63,6 @@ ZIP_COMPRESSION_LEVEL ?= "-9"
 7ZIP_COMPRESSION_METHOD ?= "BZip2"
 7ZIP_EXTENSION ?= "7z"
 
-ZSTD_COMPRESSION_LEVEL ?= "-3"
-
 JFFS2_SUM_EXTRA_ARGS ?= ""
 IMAGE_CMD:jffs2 = "mkfs.jffs2 --root=${IMAGE_ROOTFS} --faketime --output=${IMGDEPLOYDIR}/${IMAGE_NAME}.jffs2 ${EXTRA_IMAGECMD}"
 
@@ -337,7 +335,7 @@ CONVERSION_CMD:lz4 = "lz4 -9 -z -l ${IMAGE_NAME}.${type} ${IMAGE_NAME}.${type}.l
 CONVERSION_CMD:lzo = "lzop -9 ${IMAGE_NAME}.${type}"
 CONVERSION_CMD:zip = "zip ${ZIP_COMPRESSION_LEVEL} ${IMAGE_NAME}.${type}.zip ${IMAGE_NAME}.${type}"
 CONVERSION_CMD:7zip = "7za a -mx=${7ZIP_COMPRESSION_LEVEL} -mm=${7ZIP_COMPRESSION_METHOD} ${IMAGE_NAME}.${type}.${7ZIP_EXTENSION} ${IMAGE_NAME}.${type}"
-CONVERSION_CMD:zst = "zstd -f -k -T0 -c ${ZSTD_COMPRESSION_LEVEL} ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.zst"
+CONVERSION_CMD:zst = "zstd -f -k -c ${ZSTD_DEFAULTS} ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.zst"
 CONVERSION_CMD:sum = "sumtool -i ${IMAGE_NAME}.${type} -o ${IMAGE_NAME}.${type}.sum ${JFFS2_SUM_EXTRA_ARGS}"
 CONVERSION_CMD:md5sum = "md5sum ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.md5sum"
 CONVERSION_CMD:sha1sum = "sha1sum ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.sha1sum"
meta/conf/bitbake.conf +4 -0
index 55166d01784..4b1fe28c85a 100644
@@ -878,6 +878,10 @@ XZ_DEFAULTS[vardepsexclude] += "XZ_MEMLIMIT XZ_THREADS"
 ZSTD_THREADS ?= "${@oe.utils.cpu_count(at_least=2)}"
 ZSTD_THREADS[vardepvalue] = "1"
 
+ZSTD_COMPRESSION_LEVEL ?= "-3"
+ZSTD_DEFAULTS ?= "--threads=${ZSTD_THREADS} ${ZSTD_COMPRESSION_LEVEL}"
+ZSTD_DEFAULTS[vardepsexclude] = "ZSTD_COMPRESSION_LEVEL ZSTD_THREADS"
+
 # Limit the number of threads that OpenMP libraries will use. Otherwise they
 # may fallback to using all CPUs
 export OMP_NUM_THREADS = "${BB_NUMBER_THREADS}"
meta/lib/oe/package_manager/ipk/__init__.py +2 -2
index e6f9c08e2b6..8fcbad56aab 100644
@@ -133,7 +133,7 @@ class OpkgDpkgPM(PackageManager):
         tmp_dir = tempfile.mkdtemp()
         current_dir = os.getcwd()
         os.chdir(tmp_dir)
-        data_tar = 'data.tar.xz'
+        data_tar = 'data.tar.zst'
 
         try:
             cmd = [ar_cmd, 'x', pkg_path]
@@ -505,6 +505,6 @@ class OpkgPM(OpkgDpkgPM):
                      "trying to extract the package."  % pkg)
 
         tmp_dir = super(OpkgPM, self).extract(pkg, pkg_info)
-        bb.utils.remove(os.path.join(tmp_dir, "data.tar.xz"))
+        bb.utils.remove(os.path.join(tmp_dir, "data.tar.zst"))
 
         return tmp_dir
meta/recipes-devtools/opkg/opkg_0.6.2.bb +3 -3
index b2e675ae48b..71c0b44aed0 100644
@@ -8,7 +8,7 @@ LICENSE = "GPL-2.0-or-later"
 LIC_FILES_CHKSUM = "file://COPYING;md5=94d55d512a9ba36caa9b7df079bae19f \
                     file://src/opkg.c;beginline=4;endline=18;md5=d6200b0f2b41dee278aa5fad333eecae"
 
-DEPENDS = "libarchive"
+DEPENDS = "libarchive zstd"
 
 PE = "1"
 
@@ -40,10 +40,10 @@ PACKAGECONFIG[gpg] = "--enable-gpg,--disable-gpg,\
 PACKAGECONFIG[curl] = "--enable-curl,--disable-curl,curl"
 PACKAGECONFIG[ssl-curl] = "--enable-ssl-curl,--disable-ssl-curl,curl openssl"
 PACKAGECONFIG[sha256] = "--enable-sha256,--disable-sha256"
-PACKAGECONFIG[zstd] = "--enable-zstd,--disable-zstd,zstd"
 PACKAGECONFIG[libsolv] = "--with-libsolv,--without-libsolv,libsolv"
 
-EXTRA_OECONF:class-native = "--localstatedir=/${@os.path.relpath('${localstatedir}', '${STAGING_DIR_NATIVE}')} --sysconfdir=/${@os.path.relpath('${sysconfdir}', '${STAGING_DIR_NATIVE}')}"
+EXTRA_OECONF = "--enable-zstd"
+EXTRA_OECONF:append:class-native = " --localstatedir=/${@os.path.relpath('${localstatedir}', '${STAGING_DIR_NATIVE}')} --sysconfdir=/${@os.path.relpath('${sysconfdir}', '${STAGING_DIR_NATIVE}')}"
 
 do_install:append () {
 	install -d ${D}${sysconfdir}/opkg
Signature · 1 line
2.34.1
#1 lore raw
Quoted 15 lines · Joshua Watt
meta/classes-recipe/image_types.bbclass +1 -1
@@ -337,7 +335,7 @@ CONVERSION_CMD:lz4 = "lz4 -9 -z -l ${IMAGE_NAME}.${type} ${IMAGE_NAME}.${type}.l
 CONVERSION_CMD:lzo = "lzop -9 ${IMAGE_NAME}.${type}"
 CONVERSION_CMD:zip = "zip ${ZIP_COMPRESSION_LEVEL} ${IMAGE_NAME}.${type}.zip ${IMAGE_NAME}.${type}"
 CONVERSION_CMD:7zip = "7za a -mx=${7ZIP_COMPRESSION_LEVEL} -mm=${7ZIP_COMPRESSION_METHOD} ${IMAGE_NAME}.${type}.${7ZIP_EXTENSION} ${IMAGE_NAME}.${type}"
-CONVERSION_CMD:zst = "zstd -f -k -T0 -c ${ZSTD_COMPRESSION_LEVEL} ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.zst"
+CONVERSION_CMD:zst = "zstd -f -k -c ${ZSTD_DEFAULTS} ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.zst"
 CONVERSION_CMD:sum = "sumtool -i ${IMAGE_NAME}.${type} -o ${IMAGE_NAME}.${type}.sum ${JFFS2_SUM_EXTRA_ARGS}"
 CONVERSION_CMD:md5sum = "md5sum ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.md5sum"
 CONVERSION_CMD:sha1sum = "sha1sum ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.sha1sum"
meta/conf/bitbake.conf +0 -0
index 55166d01784..4b1fe28c85a 100644

meta/conf/bitbake.conf XZ_DEFAULTS[vardepsexclude] += "XZ_MEMLIMIT XZ_THREADS" · line 884

Quoted 7 lines
meta/conf/bitbake.conf +4 -0
@@ -878,6 +878,10 @@ XZ_DEFAULTS[vardepsexclude] += "XZ_MEMLIMIT XZ_THREADS"
 ZSTD_THREADS ?= "${@oe.utils.cpu_count(at_least=2)}"
 ZSTD_THREADS[vardepvalue] = "1"
 
+ZSTD_COMPRESSION_LEVEL ?= "-3"
+ZSTD_DEFAULTS ?= "--threads=${ZSTD_THREADS} ${ZSTD_COMPRESSION_LEVEL}"
+ZSTD_DEFAULTS[vardepsexclude] = "ZSTD_COMPRESSION_LEVEL ZSTD_THREADS"
+

Threads, sure but the compression level does change the output...

Cheers,

Richard

#2 lore raw
Quoted 19 lines · Richard Purdie
Quoted 18 lines · Joshua Watt
meta/classes-recipe/image_types.bbclass +1 -1
@@ -337,7 +335,7 @@ CONVERSION_CMD:lz4 = "lz4 -9 -z -l ${IMAGE_NAME}.${type} ${IMAGE_NAME}.${type}.l
 CONVERSION_CMD:lzo = "lzop -9 ${IMAGE_NAME}.${type}"
 CONVERSION_CMD:zip = "zip ${ZIP_COMPRESSION_LEVEL} ${IMAGE_NAME}.${type}.zip ${IMAGE_NAME}.${type}"
 CONVERSION_CMD:7zip = "7za a -mx=${7ZIP_COMPRESSION_LEVEL} -mm=${7ZIP_COMPRESSION_METHOD} ${IMAGE_NAME}.${type}.${7ZIP_EXTENSION} ${IMAGE_NAME}.${type}"
-CONVERSION_CMD:zst = "zstd -f -k -T0 -c ${ZSTD_COMPRESSION_LEVEL} ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.zst"
+CONVERSION_CMD:zst = "zstd -f -k -c ${ZSTD_DEFAULTS} ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.zst"
 CONVERSION_CMD:sum = "sumtool -i ${IMAGE_NAME}.${type} -o ${IMAGE_NAME}.${type}.sum ${JFFS2_SUM_EXTRA_ARGS}"
 CONVERSION_CMD:md5sum = "md5sum ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.md5sum"
 CONVERSION_CMD:sha1sum = "sha1sum ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.sha1sum"
meta/conf/bitbake.conf +0 -0
index 55166d01784..4b1fe28c85a 100644
@@ -878,6 +878,10 @@ XZ_DEFAULTS[vardepsexclude] += "XZ_MEMLIMIT XZ_THREADS"
 ZSTD_THREADS ?= "${@oe.utils.cpu_count(at_least=2)}"
 ZSTD_THREADS[vardepvalue] = "1"
 
Quoted 6 lines

meta/conf/bitbake.conf XZ_DEFAULTS[vardepsexclude] += "XZ_MEMLIMIT XZ_THREADS" · line 884

Quoted 4 lines
meta/conf/bitbake.conf +4 -0
+ZSTD_COMPRESSION_LEVEL ?= "-3"
+ZSTD_DEFAULTS ?= "--threads=${ZSTD_THREADS} ${ZSTD_COMPRESSION_LEVEL}"
+ZSTD_DEFAULTS[vardepsexclude] = "ZSTD_COMPRESSION_LEVEL ZSTD_THREADS"
+

Threads, sure but the compression level does change the output...

Indeed: https://autobuilder.yoctoproject.org/typhoon/#/builders/117/builds/4098/steps/12/logs/stdio

Quoted 14 lines

Cheers,

Richard

-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#192550): https://lists.openembedded.org/g/openembedded-core/message/192550
Mute This Topic: https://lists.openembedded.org/mt/103198693/3617179
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [[email protected]]
-=-=-=-=-=-=-=-=-=-=-=-

Signature · 3 lines
Alexandre Belloni, co-owner and COO, Bootlin
Embedded Linux and Kernel engineering
https://bootlin.com
#3 lore raw

"In addition, it support parallel compression for additional compression speed up."

It is not really clear what you mean with this sentence: xz also supports parallel compression (in poky via XZ_THREADS), so zstd support for parallel compression is not a reason to switch. zstd is rather known for fast decompression, and in my own testing was much faster than xz for compressing very big files (see this thread where I had done some benchmarking of xz vs zstd for poky:
https://lists.openembedded.org/g/openembedded-core/topic/93654146#170610 )

Etienne

Quoted 69 lines · Alexandre Belloni via lists.openembedded.org
Quoted 3 lines · Richard Purdie

Review CONVERSION_CMD:lz4 = "lz4 -9 -z -l

Quoted 2 lines · Joshua Watt
(quoted diff) +0 -0
@@ -337,7 +335,7 @@ CONVERSION_CMD:lz4 = "lz4 -9 -z -l

${IMAGE_NAME}.${type} ${IMAGE_NAME}.${type}.l

Quoted 2 lines

Review CONVERSION_CMD:lz4 = "lz4 -9 -z -l · line 336

Quoted 2 lines
(quoted diff) +0 -0
 CONVERSION_CMD:lzo = "lzop -9 ${IMAGE_NAME}.${type}"
 CONVERSION_CMD:zip = "zip ${ZIP_COMPRESSION_LEVEL}

${IMAGE_NAME}.${type}.zip ${IMAGE_NAME}.${type}"

Quoted 1 line

Review CONVERSION_CMD:lz4 = "lz4 -9 -z -l · line 337

Quoted 1 line
(quoted diff) +0 -0
 CONVERSION_CMD:7zip = "7za a -mx=${7ZIP_COMPRESSION_LEVEL}

-mm=${7ZIP_COMPRESSION_METHOD} ${IMAGE_NAME}.${type}.${7ZIP_EXTENSION} ${IMAGE_NAME}.${type}"

Quoted 1 line

Review CONVERSION_CMD:lz4 = "lz4 -9 -z -l · line 340

Quoted 1 line
(quoted diff) +0 -1
-CONVERSION_CMD:zst = "zstd -f -k -T0 -c ${ZSTD_COMPRESSION_LEVEL}

${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.zst"

Quoted 1 line

Review CONVERSION_CMD:lz4 = "lz4 -9 -z -l · line 338

Quoted 1 line
(quoted diff) +1 -0
+CONVERSION_CMD:zst = "zstd -f -k -c ${ZSTD_DEFAULTS}

${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.zst"

Quoted 1 line

Review CONVERSION_CMD:lz4 = "lz4 -9 -z -l · line 339

Quoted 1 line
(quoted diff) +0 -0
 CONVERSION_CMD:sum = "sumtool -i ${IMAGE_NAME}.${type} -o

${IMAGE_NAME}.${type}.sum ${JFFS2_SUM_EXTRA_ARGS}"

Quoted 1 line

Review CONVERSION_CMD:lz4 = "lz4 -9 -z -l · line 340

Quoted 1 line
(quoted diff) +0 -0
 CONVERSION_CMD:md5sum = "md5sum ${IMAGE_NAME}.${type} >

${IMAGE_NAME}.${type}.md5sum"

Quoted 1 line

Review CONVERSION_CMD:lz4 = "lz4 -9 -z -l · line 341

Quoted 1 line
(quoted diff) +0 -0
 CONVERSION_CMD:sha1sum = "sha1sum ${IMAGE_NAME}.${type} >

${IMAGE_NAME}.${type}.sha1sum"

Quoted 5 lines

meta/conf/bitbake.conf XZ_DEFAULTS[vardepsexclude] += "XZ_MEMLIMIT

Quoted 5 lines
meta/conf/bitbake.conf +0 -0
index 55166d01784..4b1fe28c85a 100644
@@ -878,6 +878,10 @@ XZ_DEFAULTS[vardepsexclude] += "XZ_MEMLIMIT

XZ_THREADS"

Quoted 9 lines

meta/conf/bitbake.conf XZ_DEFAULTS[vardepsexclude] += "XZ_MEMLIMIT XZ_THREADS" · line 884

Quoted 7 lines
meta/conf/bitbake.conf +4 -0
 ZSTD_THREADS ?= "${@oe.utils.cpu_count(at_least=2)}"
 ZSTD_THREADS[vardepvalue] = "1"
 
+ZSTD_COMPRESSION_LEVEL ?= "-3"
+ZSTD_DEFAULTS ?= "--threads=${ZSTD_THREADS} ${ZSTD_COMPRESSION_LEVEL}"
+ZSTD_DEFAULTS[vardepsexclude] = "ZSTD_COMPRESSION_LEVEL ZSTD_THREADS"
+

Threads, sure but the compression level does change the output...

Indeed:
https://urldefense.proofpoint.com/v2/url?u=https-3A__autobuilder.yoctoproject.org_typhoon_-23_builders_117_builds_4098_steps_12_logs_stdio&d=DwIBAg&c=ncDTmphkJTvjIDPh0hpF_4vCHvabgGkICC2epckfdiw&r=AhkbNonVuMIGRfPx_Qj9TsRih1DULJTKUkSGa66m67E&m=VXbIqF3VsSJLhfuxHVnkEJmbESqEa7d1xzRDMF_I8T6QV7FiZqqrai93ni13zQ_Q&s=goqwfFCv1hUsFcfGiURrGxqTB99GLv91ufggcLAL_ps&e=

Quoted 8 lines

Cheers,

Richard

--
Alexandre Belloni, co-owner and COO, Bootlin
Embedded Linux and Kernel engineering

https://urldefense.proofpoint.com/v2/url?u=https-3A__bootlin.com&d=DwIBAg&c=ncDTmphkJTvjIDPh0hpF_4vCHvabgGkICC2epckfdiw&r=AhkbNonVuMIGRfPx_Qj9TsRih1DULJTKUkSGa66m67E&m=VXbIqF3VsSJLhfuxHVnkEJmbESqEa7d1xzRDMF_I8T6QV7FiZqqrai93ni13zQ_Q&s=T9Ac5z4SgDitQWd1B_cBzo1oOydXnG0V9MEr95tDUmQ&e=

-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group. View/Reply Online (#192630):
https://urldefense.proofpoint.com/v2/url?u=https-3A__lists.openembedded.org_g_openembedded-2Dcore_message_192630&d=DwIFaQ&c=ncDTmphkJTvjIDPh0hpF_4vCHvabgGkICC2epckfdiw&r=AhkbNonVuMIGRfPx_Qj9TsRih1DULJTKUkSGa66m67E&m=VXbIqF3VsSJLhfuxHVnkEJmbESqEa7d1xzRDMF_I8T6QV7FiZqqrai93ni13zQ_Q&s=k685ISY4i-G6Hp4zqVgYfnk-xew6vKgC-JT9kW2WLKw&e=
Mute This Topic:
https://urldefense.proofpoint.com/v2/url?u=https-3A__lists.openembedded.org_mt_103198693_7048771&d=DwIFaQ&c=ncDTmphkJTvjIDPh0hpF_4vCHvabgGkICC2epckfdiw&r=AhkbNonVuMIGRfPx_Qj9TsRih1DULJTKUkSGa66m67E&m=VXbIqF3VsSJLhfuxHVnkEJmbESqEa7d1xzRDMF_I8T6QV7FiZqqrai93ni13zQ_Q&s=70x-KZ_YKkAOtaXmyL5phwvgr8YbbWVpLg4RZFtitBA&e=
Group Owner: [email protected] Unsubscribe:
https://urldefense.proofpoint.com/v2/url?u=https-3A__lists.openembedded.org_g_openembedded-2Dcore_unsub&d=DwIFaQ&c=ncDTmphkJTvjIDPh0hpF_4vCHvabgGkICC2epckfdiw&r=AhkbNonVuMIGRfPx_Qj9TsRih1DULJTKUkSGa66m67E&m=VXbIqF3VsSJLhfuxHVnkEJmbESqEa7d1xzRDMF_I8T6QV7FiZqqrai93ni13zQ_Q&s=Ij8_R9oYgqphELn5UASAx5RKXsEUDZmuGqppiqL63cw&e=
[[email protected]]
-=-=-=-=-=-=-=-=-=-=-=-

#4 lore raw

PATCH v2 1/1 ipk: Switch to using zstd compression

In reply to Joshua Watt

Converts IPK package generation to use zstd instead of xz. zstd has a much larger compression/speed tradeoff range allowing users to choose what suits them best, and fast decompression speeds. It also continues to support parallel compression as xz did.

A new variable called ZSTD_DEFAULTS is provided to set the defaults for places that want to use zstd for compression; the zst image conversion command is also modified to use this.

Finally, in order for this to function properly, opkg must include zstd support, so it is enabled all the time with no PACKAGECONFIG to turn it off.

---
 meta/classes-global/package_ipk.bbclass     | 2 +-
 meta/classes-recipe/image_types.bbclass     | 4 +---
 meta/conf/bitbake.conf                      | 4 ++++
 meta/lib/oe/package_manager/ipk/__init__.py | 4 ++--
 meta/recipes-devtools/opkg/opkg_0.6.2.bb    | 6 +++---
 5 files changed, 11 insertions(+), 9 deletions(-)
meta/classes-global/package_ipk.bbclass +1 -1
index 71ffdd522ac..025dba49313 100644
@@ -15,7 +15,7 @@ IPKGCONF_SDK_TARGET = "${WORKDIR}/opkg-sdk-target.conf"
 PKGWRITEDIRIPK = "${WORKDIR}/deploy-ipks"
 
 # Program to be used to build opkg packages
-OPKGBUILDCMD ??= 'opkg-build -Z xz -a "${XZ_DEFAULTS}"'
+OPKGBUILDCMD ??= 'opkg-build -Z zstd -a "${ZSTD_DEFAULTS}"'
 
 OPKG_ARGS += "--force_postinstall --prefer-arch-to-version"
 OPKG_ARGS += "${@['', '--no-install-recommends'][d.getVar("NO_RECOMMENDATIONS") == "1"]}"
meta/classes-recipe/image_types.bbclass +1 -3
index d615b41ed1e..3733bdfc20c 100644
@@ -63,8 +63,6 @@ ZIP_COMPRESSION_LEVEL ?= "-9"
 7ZIP_COMPRESSION_METHOD ?= "BZip2"
 7ZIP_EXTENSION ?= "7z"
 
-ZSTD_COMPRESSION_LEVEL ?= "-3"
-
 JFFS2_SUM_EXTRA_ARGS ?= ""
 IMAGE_CMD:jffs2 = "mkfs.jffs2 --root=${IMAGE_ROOTFS} --faketime --output=${IMGDEPLOYDIR}/${IMAGE_NAME}.jffs2 ${EXTRA_IMAGECMD}"
 
@@ -337,7 +335,7 @@ CONVERSION_CMD:lz4 = "lz4 -9 -z -l ${IMAGE_NAME}.${type} ${IMAGE_NAME}.${type}.l
 CONVERSION_CMD:lzo = "lzop -9 ${IMAGE_NAME}.${type}"
 CONVERSION_CMD:zip = "zip ${ZIP_COMPRESSION_LEVEL} ${IMAGE_NAME}.${type}.zip ${IMAGE_NAME}.${type}"
 CONVERSION_CMD:7zip = "7za a -mx=${7ZIP_COMPRESSION_LEVEL} -mm=${7ZIP_COMPRESSION_METHOD} ${IMAGE_NAME}.${type}.${7ZIP_EXTENSION} ${IMAGE_NAME}.${type}"
-CONVERSION_CMD:zst = "zstd -f -k -T0 -c ${ZSTD_COMPRESSION_LEVEL} ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.zst"
+CONVERSION_CMD:zst = "zstd -f -k -c ${ZSTD_DEFAULTS} ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.zst"
 CONVERSION_CMD:sum = "sumtool -i ${IMAGE_NAME}.${type} -o ${IMAGE_NAME}.${type}.sum ${JFFS2_SUM_EXTRA_ARGS}"
 CONVERSION_CMD:md5sum = "md5sum ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.md5sum"
 CONVERSION_CMD:sha1sum = "sha1sum ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.sha1sum"
meta/conf/bitbake.conf +4 -0
index 83b12cbc158..b3f1f18a0d7 100644
@@ -878,6 +878,10 @@ XZ_DEFAULTS[vardepsexclude] += "XZ_MEMLIMIT XZ_THREADS"
 ZSTD_THREADS ?= "${@oe.utils.cpu_count(at_least=2)}"
 ZSTD_THREADS[vardepvalue] = "1"
 
+ZSTD_COMPRESSION_LEVEL ?= "-3"
+ZSTD_DEFAULTS ?= "--threads=${ZSTD_THREADS} ${ZSTD_COMPRESSION_LEVEL}"
+ZSTD_DEFAULTS[vardepsexclude] = "ZSTD_THREADS"
+
 # Limit the number of threads that OpenMP libraries will use. Otherwise they
 # may fallback to using all CPUs
 export OMP_NUM_THREADS = "${BB_NUMBER_THREADS}"
meta/lib/oe/package_manager/ipk/__init__.py +2 -2
index e6f9c08e2b6..8fcbad56aab 100644
@@ -133,7 +133,7 @@ class OpkgDpkgPM(PackageManager):
         tmp_dir = tempfile.mkdtemp()
         current_dir = os.getcwd()
         os.chdir(tmp_dir)
-        data_tar = 'data.tar.xz'
+        data_tar = 'data.tar.zst'
 
         try:
             cmd = [ar_cmd, 'x', pkg_path]
@@ -505,6 +505,6 @@ class OpkgPM(OpkgDpkgPM):
                      "trying to extract the package."  % pkg)
 
         tmp_dir = super(OpkgPM, self).extract(pkg, pkg_info)
-        bb.utils.remove(os.path.join(tmp_dir, "data.tar.xz"))
+        bb.utils.remove(os.path.join(tmp_dir, "data.tar.zst"))
 
         return tmp_dir
meta/recipes-devtools/opkg/opkg_0.6.2.bb +3 -3
index b2e675ae48b..71c0b44aed0 100644
@@ -8,7 +8,7 @@ LICENSE = "GPL-2.0-or-later"
 LIC_FILES_CHKSUM = "file://COPYING;md5=94d55d512a9ba36caa9b7df079bae19f \
                     file://src/opkg.c;beginline=4;endline=18;md5=d6200b0f2b41dee278aa5fad333eecae"
 
-DEPENDS = "libarchive"
+DEPENDS = "libarchive zstd"
 
 PE = "1"
 
@@ -40,10 +40,10 @@ PACKAGECONFIG[gpg] = "--enable-gpg,--disable-gpg,\
 PACKAGECONFIG[curl] = "--enable-curl,--disable-curl,curl"
 PACKAGECONFIG[ssl-curl] = "--enable-ssl-curl,--disable-ssl-curl,curl openssl"
 PACKAGECONFIG[sha256] = "--enable-sha256,--disable-sha256"
-PACKAGECONFIG[zstd] = "--enable-zstd,--disable-zstd,zstd"
 PACKAGECONFIG[libsolv] = "--with-libsolv,--without-libsolv,libsolv"
 
-EXTRA_OECONF:class-native = "--localstatedir=/${@os.path.relpath('${localstatedir}', '${STAGING_DIR_NATIVE}')} --sysconfdir=/${@os.path.relpath('${sysconfdir}', '${STAGING_DIR_NATIVE}')}"
+EXTRA_OECONF = "--enable-zstd"
+EXTRA_OECONF:append:class-native = " --localstatedir=/${@os.path.relpath('${localstatedir}', '${STAGING_DIR_NATIVE}')} --sysconfdir=/${@os.path.relpath('${sysconfdir}', '${STAGING_DIR_NATIVE}')}"
 
 do_install:append () {
 	install -d ${D}${sysconfdir}/opkg
Signature · 1 line
2.34.1
#5 lore raw

Hi Josh,

This still causes reproducibility issues:

https://autobuilder.yoctoproject.org/typhoon/#/builders/117/builds/4113/steps/13/logs/stdio
https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20231219-mrw5vs1g/packages/diff-html/

Quoted 132 lines · Joshua Watt

Converts IPK package generation to use zstd instead of xz. zstd has a much larger compression/speed tradeoff range allowing users to choose what suits them best, and fast decompression speeds. It also continues to support parallel compression as xz did.

A new variable called ZSTD_DEFAULTS is provided to set the defaults for places that want to use zstd for compression; the zst image conversion command is also modified to use this.

Finally, in order for this to function properly, opkg must include zstd support, so it is enabled all the time with no PACKAGECONFIG to turn it off.

---
 meta/classes-global/package_ipk.bbclass     | 2 +-
 meta/classes-recipe/image_types.bbclass     | 4 +---
 meta/conf/bitbake.conf                      | 4 ++++
 meta/lib/oe/package_manager/ipk/__init__.py | 4 ++--
 meta/recipes-devtools/opkg/opkg_0.6.2.bb    | 6 +++---
 5 files changed, 11 insertions(+), 9 deletions(-)
meta/classes-global/package_ipk.bbclass +1 -1
index 71ffdd522ac..025dba49313 100644
@@ -15,7 +15,7 @@ IPKGCONF_SDK_TARGET = "${WORKDIR}/opkg-sdk-target.conf"
 PKGWRITEDIRIPK = "${WORKDIR}/deploy-ipks"
 
 # Program to be used to build opkg packages
-OPKGBUILDCMD ??= 'opkg-build -Z xz -a "${XZ_DEFAULTS}"'
+OPKGBUILDCMD ??= 'opkg-build -Z zstd -a "${ZSTD_DEFAULTS}"'
 
 OPKG_ARGS += "--force_postinstall --prefer-arch-to-version"
 OPKG_ARGS += "${@['', '--no-install-recommends'][d.getVar("NO_RECOMMENDATIONS") == "1"]}"
meta/classes-recipe/image_types.bbclass +1 -3
index d615b41ed1e..3733bdfc20c 100644
@@ -63,8 +63,6 @@ ZIP_COMPRESSION_LEVEL ?= "-9"
 7ZIP_COMPRESSION_METHOD ?= "BZip2"
 7ZIP_EXTENSION ?= "7z"
 
-ZSTD_COMPRESSION_LEVEL ?= "-3"
-
 JFFS2_SUM_EXTRA_ARGS ?= ""
 IMAGE_CMD:jffs2 = "mkfs.jffs2 --root=${IMAGE_ROOTFS} --faketime --output=${IMGDEPLOYDIR}/${IMAGE_NAME}.jffs2 ${EXTRA_IMAGECMD}"
 
@@ -337,7 +335,7 @@ CONVERSION_CMD:lz4 = "lz4 -9 -z -l ${IMAGE_NAME}.${type} ${IMAGE_NAME}.${type}.l
 CONVERSION_CMD:lzo = "lzop -9 ${IMAGE_NAME}.${type}"
 CONVERSION_CMD:zip = "zip ${ZIP_COMPRESSION_LEVEL} ${IMAGE_NAME}.${type}.zip ${IMAGE_NAME}.${type}"
 CONVERSION_CMD:7zip = "7za a -mx=${7ZIP_COMPRESSION_LEVEL} -mm=${7ZIP_COMPRESSION_METHOD} ${IMAGE_NAME}.${type}.${7ZIP_EXTENSION} ${IMAGE_NAME}.${type}"
-CONVERSION_CMD:zst = "zstd -f -k -T0 -c ${ZSTD_COMPRESSION_LEVEL} ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.zst"
+CONVERSION_CMD:zst = "zstd -f -k -c ${ZSTD_DEFAULTS} ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.zst"
 CONVERSION_CMD:sum = "sumtool -i ${IMAGE_NAME}.${type} -o ${IMAGE_NAME}.${type}.sum ${JFFS2_SUM_EXTRA_ARGS}"
 CONVERSION_CMD:md5sum = "md5sum ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.md5sum"
 CONVERSION_CMD:sha1sum = "sha1sum ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.sha1sum"
meta/conf/bitbake.conf +4 -0
index 83b12cbc158..b3f1f18a0d7 100644
@@ -878,6 +878,10 @@ XZ_DEFAULTS[vardepsexclude] += "XZ_MEMLIMIT XZ_THREADS"
 ZSTD_THREADS ?= "${@oe.utils.cpu_count(at_least=2)}"
 ZSTD_THREADS[vardepvalue] = "1"
 
+ZSTD_COMPRESSION_LEVEL ?= "-3"
+ZSTD_DEFAULTS ?= "--threads=${ZSTD_THREADS} ${ZSTD_COMPRESSION_LEVEL}"
+ZSTD_DEFAULTS[vardepsexclude] = "ZSTD_THREADS"
+
 # Limit the number of threads that OpenMP libraries will use. Otherwise they
 # may fallback to using all CPUs
 export OMP_NUM_THREADS = "${BB_NUMBER_THREADS}"
meta/lib/oe/package_manager/ipk/__init__.py +2 -2
index e6f9c08e2b6..8fcbad56aab 100644
@@ -133,7 +133,7 @@ class OpkgDpkgPM(PackageManager):
         tmp_dir = tempfile.mkdtemp()
         current_dir = os.getcwd()
         os.chdir(tmp_dir)
-        data_tar = 'data.tar.xz'
+        data_tar = 'data.tar.zst'
 
         try:
             cmd = [ar_cmd, 'x', pkg_path]
@@ -505,6 +505,6 @@ class OpkgPM(OpkgDpkgPM):
                      "trying to extract the package."  % pkg)
 
         tmp_dir = super(OpkgPM, self).extract(pkg, pkg_info)
-        bb.utils.remove(os.path.join(tmp_dir, "data.tar.xz"))
+        bb.utils.remove(os.path.join(tmp_dir, "data.tar.zst"))
 
         return tmp_dir
meta/recipes-devtools/opkg/opkg_0.6.2.bb +3 -3
index b2e675ae48b..71c0b44aed0 100644
@@ -8,7 +8,7 @@ LICENSE = "GPL-2.0-or-later"
 LIC_FILES_CHKSUM = "file://COPYING;md5=94d55d512a9ba36caa9b7df079bae19f \
                     file://src/opkg.c;beginline=4;endline=18;md5=d6200b0f2b41dee278aa5fad333eecae"
 
-DEPENDS = "libarchive"
+DEPENDS = "libarchive zstd"
 
 PE = "1"
 
@@ -40,10 +40,10 @@ PACKAGECONFIG[gpg] = "--enable-gpg,--disable-gpg,\
 PACKAGECONFIG[curl] = "--enable-curl,--disable-curl,curl"
 PACKAGECONFIG[ssl-curl] = "--enable-ssl-curl,--disable-ssl-curl,curl openssl"
 PACKAGECONFIG[sha256] = "--enable-sha256,--disable-sha256"
-PACKAGECONFIG[zstd] = "--enable-zstd,--disable-zstd,zstd"
 PACKAGECONFIG[libsolv] = "--with-libsolv,--without-libsolv,libsolv"
 
-EXTRA_OECONF:class-native = "--localstatedir=/${@os.path.relpath('${localstatedir}', '${STAGING_DIR_NATIVE}')} --sysconfdir=/${@os.path.relpath('${sysconfdir}', '${STAGING_DIR_NATIVE}')}"
+EXTRA_OECONF = "--enable-zstd"
+EXTRA_OECONF:append:class-native = " --localstatedir=/${@os.path.relpath('${localstatedir}', '${STAGING_DIR_NATIVE}')} --sysconfdir=/${@os.path.relpath('${sysconfdir}', '${STAGING_DIR_NATIVE}')}"
 
 do_install:append () {
 	install -d ${D}${sysconfdir}/opkg
Signature · 11 lines
2.34.1



-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#192657): https://lists.openembedded.org/g/openembedded-core/message/192657
Mute This Topic: https://lists.openembedded.org/mt/103244564/3617179
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [[email protected]]
-=-=-=-=-=-=-=-=-=-=-=-
Signature · 3 lines
Alexandre Belloni, co-owner and COO, Bootlin
Embedded Linux and Kernel engineering
https://bootlin.com
#6 lore raw

PATCH v3 1/1 ipk: Switch to using zstd compression

In reply to Joshua Watt

Converts IPK package generation to use zstd instead of xz. zstd has a much larger compression/speed tradeoff range allowing users to choose what suits them best, and fast decompression speeds. It also continues to support parallel compression as xz did.

A new variable called ZSTD_DEFAULTS is provided to set the defaults for places that want to use zstd for compression; the zst image conversion command is also modified to use this.

Finally, in order for this to function properly, opkg must include zstd support, so it is enabled all the time with no PACKAGECONFIG to turn it off.

---
 meta/classes-global/package_ipk.bbclass     | 4 ++--
 meta/classes-recipe/image_types.bbclass     | 4 +---
 meta/conf/bitbake.conf                      | 4 ++++
 meta/lib/oe/package_manager/ipk/__init__.py | 4 ++--
 meta/recipes-devtools/opkg/opkg_0.6.2.bb    | 6 +++---
 5 files changed, 12 insertions(+), 10 deletions(-)
meta/classes-global/package_ipk.bbclass +2 -2
index 71ffdd522ac..209568ae5f1 100644
@@ -15,7 +15,7 @@ IPKGCONF_SDK_TARGET = "${WORKDIR}/opkg-sdk-target.conf"
 PKGWRITEDIRIPK = "${WORKDIR}/deploy-ipks"
 
 # Program to be used to build opkg packages
-OPKGBUILDCMD ??= 'opkg-build -Z xz -a "${XZ_DEFAULTS}"'
+OPKGBUILDCMD ??= 'opkg-build -Z zstd -a "${ZSTD_DEFAULTS}"'
 
 OPKG_ARGS += "--force_postinstall --prefer-arch-to-version"
 OPKG_ARGS += "${@['', '--no-install-recommends'][d.getVar("NO_RECOMMENDATIONS") == "1"]}"
@@ -277,7 +277,7 @@ addtask do_package_write_ipk_setscene
 
 python () {
     if d.getVar('PACKAGES') != '':
-        deps = ' opkg-utils-native:do_populate_sysroot virtual/fakeroot-native:do_populate_sysroot xz-native:do_populate_sysroot'
+        deps = ' opkg-utils-native:do_populate_sysroot virtual/fakeroot-native:do_populate_sysroot zstd-native:do_populate_sysroot'
         d.appendVarFlag('do_package_write_ipk', 'depends', deps)
         d.setVarFlag('do_package_write_ipk', 'fakeroot', "1")
 
meta/classes-recipe/image_types.bbclass +1 -3
index d615b41ed1e..3733bdfc20c 100644
@@ -63,8 +63,6 @@ ZIP_COMPRESSION_LEVEL ?= "-9"
 7ZIP_COMPRESSION_METHOD ?= "BZip2"
 7ZIP_EXTENSION ?= "7z"
 
-ZSTD_COMPRESSION_LEVEL ?= "-3"
-
 JFFS2_SUM_EXTRA_ARGS ?= ""
 IMAGE_CMD:jffs2 = "mkfs.jffs2 --root=${IMAGE_ROOTFS} --faketime --output=${IMGDEPLOYDIR}/${IMAGE_NAME}.jffs2 ${EXTRA_IMAGECMD}"
 
@@ -337,7 +335,7 @@ CONVERSION_CMD:lz4 = "lz4 -9 -z -l ${IMAGE_NAME}.${type} ${IMAGE_NAME}.${type}.l
 CONVERSION_CMD:lzo = "lzop -9 ${IMAGE_NAME}.${type}"
 CONVERSION_CMD:zip = "zip ${ZIP_COMPRESSION_LEVEL} ${IMAGE_NAME}.${type}.zip ${IMAGE_NAME}.${type}"
 CONVERSION_CMD:7zip = "7za a -mx=${7ZIP_COMPRESSION_LEVEL} -mm=${7ZIP_COMPRESSION_METHOD} ${IMAGE_NAME}.${type}.${7ZIP_EXTENSION} ${IMAGE_NAME}.${type}"
-CONVERSION_CMD:zst = "zstd -f -k -T0 -c ${ZSTD_COMPRESSION_LEVEL} ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.zst"
+CONVERSION_CMD:zst = "zstd -f -k -c ${ZSTD_DEFAULTS} ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.zst"
 CONVERSION_CMD:sum = "sumtool -i ${IMAGE_NAME}.${type} -o ${IMAGE_NAME}.${type}.sum ${JFFS2_SUM_EXTRA_ARGS}"
 CONVERSION_CMD:md5sum = "md5sum ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.md5sum"
 CONVERSION_CMD:sha1sum = "sha1sum ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.sha1sum"
meta/conf/bitbake.conf +4 -0
index 83b12cbc158..b3f1f18a0d7 100644
@@ -878,6 +878,10 @@ XZ_DEFAULTS[vardepsexclude] += "XZ_MEMLIMIT XZ_THREADS"
 ZSTD_THREADS ?= "${@oe.utils.cpu_count(at_least=2)}"
 ZSTD_THREADS[vardepvalue] = "1"
 
+ZSTD_COMPRESSION_LEVEL ?= "-3"
+ZSTD_DEFAULTS ?= "--threads=${ZSTD_THREADS} ${ZSTD_COMPRESSION_LEVEL}"
+ZSTD_DEFAULTS[vardepsexclude] = "ZSTD_THREADS"
+
 # Limit the number of threads that OpenMP libraries will use. Otherwise they
 # may fallback to using all CPUs
 export OMP_NUM_THREADS = "${BB_NUMBER_THREADS}"
meta/lib/oe/package_manager/ipk/__init__.py +2 -2
index e6f9c08e2b6..8fcbad56aab 100644
@@ -133,7 +133,7 @@ class OpkgDpkgPM(PackageManager):
         tmp_dir = tempfile.mkdtemp()
         current_dir = os.getcwd()
         os.chdir(tmp_dir)
-        data_tar = 'data.tar.xz'
+        data_tar = 'data.tar.zst'
 
         try:
             cmd = [ar_cmd, 'x', pkg_path]
@@ -505,6 +505,6 @@ class OpkgPM(OpkgDpkgPM):
                      "trying to extract the package."  % pkg)
 
         tmp_dir = super(OpkgPM, self).extract(pkg, pkg_info)
-        bb.utils.remove(os.path.join(tmp_dir, "data.tar.xz"))
+        bb.utils.remove(os.path.join(tmp_dir, "data.tar.zst"))
 
         return tmp_dir
meta/recipes-devtools/opkg/opkg_0.6.2.bb +3 -3
index b2e675ae48b..71c0b44aed0 100644
@@ -8,7 +8,7 @@ LICENSE = "GPL-2.0-or-later"
 LIC_FILES_CHKSUM = "file://COPYING;md5=94d55d512a9ba36caa9b7df079bae19f \
                     file://src/opkg.c;beginline=4;endline=18;md5=d6200b0f2b41dee278aa5fad333eecae"
 
-DEPENDS = "libarchive"
+DEPENDS = "libarchive zstd"
 
 PE = "1"
 
@@ -40,10 +40,10 @@ PACKAGECONFIG[gpg] = "--enable-gpg,--disable-gpg,\
 PACKAGECONFIG[curl] = "--enable-curl,--disable-curl,curl"
 PACKAGECONFIG[ssl-curl] = "--enable-ssl-curl,--disable-ssl-curl,curl openssl"
 PACKAGECONFIG[sha256] = "--enable-sha256,--disable-sha256"
-PACKAGECONFIG[zstd] = "--enable-zstd,--disable-zstd,zstd"
 PACKAGECONFIG[libsolv] = "--with-libsolv,--without-libsolv,libsolv"
 
-EXTRA_OECONF:class-native = "--localstatedir=/${@os.path.relpath('${localstatedir}', '${STAGING_DIR_NATIVE}')} --sysconfdir=/${@os.path.relpath('${sysconfdir}', '${STAGING_DIR_NATIVE}')}"
+EXTRA_OECONF = "--enable-zstd"
+EXTRA_OECONF:append:class-native = " --localstatedir=/${@os.path.relpath('${localstatedir}', '${STAGING_DIR_NATIVE}')} --sysconfdir=/${@os.path.relpath('${sysconfdir}', '${STAGING_DIR_NATIVE}')}"
 
 do_install:append () {
 	install -d ${D}${sysconfdir}/opkg
Signature · 1 line
2.34.1