zstd: move recipe to oe-core from meta-oe
PATCH v4 1/2 zstd: move recipe to oe-core from meta-oe
From: Stefan Agner <[email protected]>
Zstd is a very fast compression algorithm and has gained quite some popularity recently. An upcoming patch allows to compress rootfs using Zstd. Moving Zstd to oe-core allows this new conversion formats to be automatically tested by the oe-selftest imagefeatures.ImageFeatures.test_image_fstypes test.
Furthermore there are three packages in oe-core which allow to enable Zstd support through packageconfig currently.
--- meta/conf/distro/include/maintainers.inc | 1 + meta/recipes-extended/zstd/zstd_1.4.4.bb | 35 ++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 meta/recipes-extended/zstd/zstd_1.4.4.bb
meta/conf/distro/include/maintainers.inc +1 -0
@@ -774,3 +774,4 @@ RECIPE_MAINTAINER_pn-xwininfo = "Armin Kuster <[email protected]>" RECIPE_MAINTAINER_pn-xz = "Denys Dmytriyenko <[email protected]>" RECIPE_MAINTAINER_pn-zip = "Denys Dmytriyenko <[email protected]>" RECIPE_MAINTAINER_pn-zlib = "Denys Dmytriyenko <[email protected]>" +RECIPE_MAINTAINER_pn-zstd = "Alex Kiernan <[email protected]>"
meta/recipes-extended/zstd/zstd_1.4.4.bb new file +35 -0
@@ -0,0 +1,35 @@ +SUMMARY = "Zstandard - Fast real-time compression algorithm" +DESCRIPTION = "Zstandard is a fast lossless compression algorithm, targeting \ +real-time compression scenarios at zlib-level and better compression ratios. \ +It's backed by a very fast entropy stage, provided by Huff0 and FSE library." +HOMEPAGE = "http://www.zstd.net/" +SECTION = "console/utils" + +LICENSE = "BSD-3-Clause & GPLv2" +LIC_FILES_CHKSUM = "file://LICENSE;md5=c7f0b161edbe52f5f345a3d1311d0b32 \ + file://COPYING;md5=39bba7d2cf0ba1036f2a6e2be52fe3f0" + +SRC_URI = "git://github.com/facebook/zstd.git;nobranch=1" + +SRCREV = "10f0e6993f9d2f682da6d04aa2385b7d53cbb4ee" +UPSTREAM_CHECK_GITTAGREGEX = "v(?P<pver>\d+(\.\d+)+)" + +S = "${WORKDIR}/git" + +PACKAGECONFIG ??= "" +PACKAGECONFIG[lz4] = "HAVE_LZ4=1,HAVE_LZ4=0,lz4" +PACKAGECONFIG[lzma] = "HAVE_LZMA=1,HAVE_LZMA=0,xz" +PACKAGECONFIG[zlib] = "HAVE_ZLIB=1,HAVE_ZLIB=0,zlib" + +# See programs/README.md for how to use this +ZSTD_LEGACY_SUPPORT ??= "4" + +do_compile () { + oe_runmake ${PACKAGECONFIG_CONFARGS} ZSTD_LEGACY_SUPPORT=${ZSTD_LEGACY_SUPPORT} +} + +do_install () { + oe_runmake install 'DESTDIR=${D}' +} + +BBCLASSEXTEND = "native nativesdk"
Signature · 1 line
2.17.1
PATCH v4 2/2 image_types: add Zstandard conversion support
From: Stefan Agner <[email protected]>
Add Zstandard (or just Zstd) compression support. This allows to create Zstd compressed tarballs by using tar.zst as IMAGE_FSTYPES.
--- meta/classes/image_types.bbclass | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-)
meta/classes/image_types.bbclass +6 -2
@@ -59,6 +59,8 @@ XZ_INTEGRITY_CHECK ?= "crc32" ZIP_COMPRESSION_LEVEL ?= "-9" +ZSTD_COMPRESSION_LEVEL ?= "-3" + JFFS2_SUM_EXTRA_ARGS ?= "" IMAGE_CMD_jffs2 = "mkfs.jffs2 --root=${IMAGE_ROOTFS} --faketime --output=${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.jffs2 ${EXTRA_IMAGECMD}"
@@ -269,7 +271,7 @@ IMAGE_TYPES = " \ hddimg \ squashfs squashfs-xz squashfs-lzo squashfs-lz4 \ ubi ubifs multiubi \ - tar tar.gz tar.bz2 tar.xz tar.lz4 \ + tar tar.gz tar.bz2 tar.xz tar.lz4 tar.zst \ cpio cpio.gz cpio.xz cpio.lzma cpio.lz4 \ wic wic.gz wic.bz2 wic.lzma \ container \
@@ -282,7 +284,7 @@ IMAGE_TYPES = " \ # CONVERSION_CMD/DEPENDS. COMPRESSIONTYPES ?= "" -CONVERSIONTYPES = "gz bz2 lzma xz lz4 lzo zip sum md5sum sha1sum sha224sum sha256sum sha384sum sha512sum bmap u-boot vmdk vdi qcow2 base64 ${COMPRESSIONTYPES}" +CONVERSIONTYPES = "gz bz2 lzma xz lz4 lzo zip zst sum md5sum sha1sum sha224sum sha256sum sha384sum sha512sum bmap u-boot vmdk vdi qcow2 base64 ${COMPRESSIONTYPES}" CONVERSION_CMD_lzma = "lzma -k -f -7 ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}" CONVERSION_CMD_gz = "gzip -f -9 -n -c --rsyncable ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.gz" CONVERSION_CMD_bz2 = "pbzip2 -f -k ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}"
@@ -290,6 +292,7 @@ CONVERSION_CMD_xz = "xz -f -k -c ${XZ_COMPRESSION_LEVEL} ${XZ_DEFAULTS} --check= CONVERSION_CMD_lz4 = "lz4 -9 -z -l ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.lz4" CONVERSION_CMD_lzo = "lzop -9 ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}" CONVERSION_CMD_zip = "zip ${ZIP_COMPRESSION_LEVEL} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.zip ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}" +CONVERSION_CMD_zst = "zstd -f -k -c ${ZSTD_COMPRESSION_LEVEL} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.zst" CONVERSION_CMD_sum = "sumtool -i ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} -o ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.sum ${JFFS2_SUM_EXTRA_ARGS}" CONVERSION_CMD_md5sum = "md5sum ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.md5sum" CONVERSION_CMD_sha1sum = "sha1sum ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.sha1sum"
@@ -310,6 +313,7 @@ CONVERSION_DEPENDS_xz = "xz-native" CONVERSION_DEPENDS_lz4 = "lz4-native" CONVERSION_DEPENDS_lzo = "lzop-native" CONVERSION_DEPENDS_zip = "zip-native" +CONVERSION_DEPENDS_zst = "zstd-native" CONVERSION_DEPENDS_sum = "mtd-utils-native" CONVERSION_DEPENDS_bmap = "bmap-tools-native" CONVERSION_DEPENDS_u-boot = "u-boot-tools-native"
Signature · 1 line
2.17.1
Re: [PATCH v4 2/2] image_types: add Zstandard conversion support
meta/classes/image_types.bbclass XZ_INTEGRITY_CHECK ?= "crc32" · line 63
Quoted 10 lines · Stefan Agner
...
meta/classes/image_types.bbclass +2
-0@@ -59,6 +59,8 @@ XZ_INTEGRITY_CHECK ?= "crc32" ZIP_COMPRESSION_LEVEL ?= "-9" +ZSTD_COMPRESSION_LEVEL ?= "-3" +
Why -3 and not -19 ?
Compression speed is not a real issue, and AFAIR decompression tends to be slightly faster with smaller compressed size. Plus the benefit that smaller images are faster to load from network/storage.
cu
Adrian
Re: [PATCH v4 2/2] image_types: add Zstandard conversion support
Quoted 15 lines · Adrian Bunk
meta/classes/image_types.bbclass XZ_INTEGRITY_CHECK ?= "crc32" · line 63
Quoted 10 lines · Stefan Agner
...
meta/classes/image_types.bbclass +2
-0@@ -59,6 +59,8 @@ XZ_INTEGRITY_CHECK ?= "crc32" ZIP_COMPRESSION_LEVEL ?= "-9" +ZSTD_COMPRESSION_LEVEL ?= "-3" +Why -3 and not -19 ?
Quoted 5 lines
Compression speed is not a real issue, and AFAIR decompression tends to be slightly faster with smaller compressed size. Plus the benefit that smaller images are faster to load from network/storage.
I've been carrying similar functionality locally and at some point pushed to --ultra -22 and with a large image (~2GiB) the compression stage then took literally hours. Not sure I tested with -19 though
Signature · 1 line
Alex Kiernan
Re: [PATCH v4 2/2] image_types: add Zstandard conversion support
In reply to Stefan AgnerQuoted 43 lines · Stefan Agner
From: Stefan Agner <[email protected]>
Add Zstandard (or just Zstd) compression support. This allows to create Zstd compressed tarballs by using tar.zst as
IMAGE_FSTYPES.Signed-off-by Stefan Agner <[email protected]>--- meta/classes/image_types.bbclass | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-)meta/classes/image_types.bbclass +4
-2@@ -59,6 +59,8 @@ XZ_INTEGRITY_CHECK ?= "crc32" ZIP_COMPRESSION_LEVEL ?= "-9" +ZSTD_COMPRESSION_LEVEL ?= "-3" + JFFS2_SUM_EXTRA_ARGS ?= "" IMAGE_CMD_jffs2 = "mkfs.jffs2 --root=${IMAGE_ROOTFS} --faketime --output=${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.jffs2 ${EXTRA_IMAGECMD}"@@ -269,7 +271,7 @@ IMAGE_TYPES = " \ hddimg \ squashfs squashfs-xz squashfs-lzo squashfs-lz4 \ ubi ubifs multiubi \ - tar tar.gz tar.bz2 tar.xz tar.lz4 \ + tar tar.gz tar.bz2 tar.xz tar.lz4 tar.zst \ cpio cpio.gz cpio.xz cpio.lzma cpio.lz4 \ wic wic.gz wic.bz2 wic.lzma \ container \@@ -282,7 +284,7 @@ IMAGE_TYPES = " \ # CONVERSION_CMD/DEPENDS. COMPRESSIONTYPES ?= "" -CONVERSIONTYPES = "gz bz2 lzma xz lz4 lzo zip sum md5sum sha1sum sha224sum sha256sum sha384sum sha512sum bmap u-boot vmdk vdi qcow2 base64 ${COMPRESSIONTYPES}" +CONVERSIONTYPES = "gz bz2 lzma xz lz4 lzo zip zst sum md5sum sha1sum sha224sum sha256sum sha384sum sha512sum bmap u-boot vmdk vdi qcow2 base64 ${COMPRESSIONTYPES}" CONVERSION_CMD_lzma = "lzma -k -f -7 ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}" CONVERSION_CMD_gz = "gzip -f -9 -n -c --rsyncable ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.gz" CONVERSION_CMD_bz2 = "pbzip2 -f -k ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}"
meta/classes/image_types.bbclass CONVERSION_CMD_xz = "xz -f -k -c ${XZ_COMPRESSION_LEVEL} ${XZ_DEFAULTS} --check= · line 295
Quoted 4 lines
meta/classes/image_types.bbclass +1
-0@@ -290,6 +292,7 @@ CONVERSION_CMD_xz = "xz -f -k -c ${XZ_COMPRESSION_LEVEL} ${XZ_DEFAULTS} --check= CONVERSION_CMD_lz4 = "lz4 -9 -z -l ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.lz4" CONVERSION_CMD_lzo = "lzop -9 ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}" CONVERSION_CMD_zip = "zip ${ZIP_COMPRESSION_LEVEL} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.zip ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}" +CONVERSION_CMD_zst = "zstd -f -k -c ${ZSTD_COMPRESSION_LEVEL} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.zst"
Add -T0 so we default to parallel?
Signature · 1 line
Alex Kiernan
Re: [PATCH v4 2/2] image_types: add Zstandard conversion support
Quoted 38 lines · Alex Kiernan
Quoted 37 lines · Stefan Agner
From: Stefan Agner <[email protected]>
Add Zstandard (or just Zstd) compression support. This allows to create Zstd compressed tarballs by using tar.zst as
IMAGE_FSTYPES.Signed-off-by Stefan Agner <[email protected]>--- meta/classes/image_types.bbclass | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-)meta/classes/image_types.bbclass +3
-1@@ -59,6 +59,8 @@ XZ_INTEGRITY_CHECK ?= "crc32" ZIP_COMPRESSION_LEVEL ?= "-9" +ZSTD_COMPRESSION_LEVEL ?= "-3" + JFFS2_SUM_EXTRA_ARGS ?= "" IMAGE_CMD_jffs2 = "mkfs.jffs2 --root=${IMAGE_ROOTFS} --faketime --output=${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.jffs2 ${EXTRA_IMAGECMD}"@@ -269,7 +271,7 @@ IMAGE_TYPES = " \ hddimg \ squashfs squashfs-xz squashfs-lzo squashfs-lz4 \ ubi ubifs multiubi \ - tar tar.gz tar.bz2 tar.xz tar.lz4 \ + tar tar.gz tar.bz2 tar.xz tar.lz4 tar.zst \ cpio cpio.gz cpio.xz cpio.lzma cpio.lz4 \ wic wic.gz wic.bz2 wic.lzma \ container \@@ -282,7 +284,7 @@ IMAGE_TYPES = " \ # CONVERSION_CMD/DEPENDS. COMPRESSIONTYPES ?= ""
Quoted 12 lines
meta/classes/image_types.bbclass CONVERSION_CMD_xz = "xz -f -k -c ${XZ_COMPRESSION_LEVEL} ${XZ_DEFAULTS} --check= · line 295
Quoted 10 lines
meta/classes/image_types.bbclass +2
-1-CONVERSIONTYPES = "gz bz2 lzma xz lz4 lzo zip sum md5sum sha1sum sha224sum sha256sum sha384sum sha512sum bmap u-boot vmdk vdi qcow2 base64 ${COMPRESSIONTYPES}" +CONVERSIONTYPES = "gz bz2 lzma xz lz4 lzo zip zst sum md5sum sha1sum sha224sum sha256sum sha384sum sha512sum bmap u-boot vmdk vdi qcow2 base64 ${COMPRESSIONTYPES}" CONVERSION_CMD_lzma = "lzma -k -f -7 ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}" CONVERSION_CMD_gz = "gzip -f -9 -n -c --rsyncable ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.gz" CONVERSION_CMD_bz2 = "pbzip2 -f -k ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}"@@ -290,6 +292,7 @@ CONVERSION_CMD_xz = "xz -f -k -c ${XZ_COMPRESSION_LEVEL} ${XZ_DEFAULTS} --check= CONVERSION_CMD_lz4 = "lz4 -9 -z -l ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.lz4" CONVERSION_CMD_lzo = "lzop -9 ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}" CONVERSION_CMD_zip = "zip ${ZIP_COMPRESSION_LEVEL} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.zip ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}" +CONVERSION_CMD_zst = "zstd -f -k -c ${ZSTD_COMPRESSION_LEVEL} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.zst"Add -T0 so we default to parallel?
Sensible, will send a v5.
--
Stefan