erofs-utils: add XZ algorithm support
RFC PATCH 1/1 erofs-utils: add XZ algorithm support
It's silly but roughtly usable *_destsize implementation for XZ by using liblzma stream API and fitblk-like approach, whose generated data can be directly decompressed by kernel.
I can investigate it deeply by hacking lzma2/XZ internals if I have more extra time to do, that is another story for now.
Currently, MT compression support would be a higher priority stuff and even for fixed-sized output compression it can be multi-threaded by fixed-sized input segments:
_________________________________________________________
| thread 0 | thread 1 | thread 0 | thread 1 | ... |____x MB____|____x MB_____|____x MB____|____x MB____|____
MT compression can also be implemented on per-file basis of course.
---
A brief fitblk-like XZ approach compression ratio comparsion:
459382784 enwik9_4k.xz.squashfs.img
401649664 enwik9_8k.xz.squashfs.img
378892288 enwik9_4k.xz.erofs.img (bisect is forcely true) 360972288 enwik9_16k.xz.squashfs.img
configure.ac | 12 ++ include/erofs_fs.h | 1 + lib/Makefile.am | 4 + lib/compress.c | 32 +++-- lib/compressor.c | 3 + lib/compressor.h | 2 + lib/compressor_xz.c | 277 ++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 320 insertions(+), 11 deletions(-) create mode 100644 lib/compressor_xz.c
configure.ac +12 -0
@@ -154,9 +154,17 @@ if test "x$enable_lz4" = "xyes"; then CPPFLAGS=${saved_CPPFLAGS} fi +AC_CHECK_HEADERS(lzma.h, + AC_CHECK_LIB(lzma, lzma_code,[ + have_liblzma="yes" + LZMA_LIBS="-llzma" + AC_DEFINE([XZ_ENABLED], [1], [Define to 1 if xz is enabled.]) + ], [have_liblzma="no"])) + # Set up needed symbols, conditionals and compiler/linker flags AM_CONDITIONAL([ENABLE_LZ4], [test "x${have_lz4}" = "xyes"]) AM_CONDITIONAL([ENABLE_LZ4HC], [test "x${have_lz4hc}" = "xyes"]) +AM_CONDITIONAL([ENABLE_XZ], [test "x${have_liblzma}" = "xyes"]) if test "x${have_lz4}" = "xyes"; then AC_DEFINE([LZ4_ENABLED], [1], [Define to 1 if lz4 is enabled.])
@@ -173,6 +181,10 @@ if test "x${have_lz4}" = "xyes"; then LIBS="$LZ4_LIBS $LIBS" fi +if test "x${have_liblzma}" = "xyes"; then +LIBS="$LZMA_LIBS $LIBS" +fi + AC_CONFIG_FILES([Makefile man/Makefile lib/Makefile
include/erofs_fs.h +1 -0
@@ -194,6 +194,7 @@ static inline unsigned int erofs_xattr_entry_size(struct erofs_xattr_entry *e)/* available compression algorithm types (for h_algorithmtype) */ enum {Z_EROFS_COMPRESSION_LZ4 =0 , +Z_EROFS_COMPRESSION_XZ =1 ,Z_EROFS_COMPRESSION_MAX };
lib/Makefile.am +4 -0
@@ -12,4 +12,8 @@ if ENABLE_LZ4HC liberofs_la_SOURCES += compressor_lz4hc.c endif endif +if ENABLE_XZ +liberofs_la_CFLAGS += ${LZMA_CFLAGS} +liberofs_la_SOURCES += compressor_xz.c +endif
lib/compress.c +21 -11
@@ -23,7 +23,7 @@ static int compressionlevel;static struct z_erofs_map_header mapheader;struct z_erofs_vle_compress_ctx { -u8 *metacur; +u8 *metacur, *dstbuf;u8 queue[EROFS_CONFIG_COMPR_MAX_SZ *2 ];unsigned int head, tail;
@@ -114,8 +114,7 @@ static void vle_write_indexes(struct z_erofs_vle_compress_ctx *ctx, }static int write_uncompressed_block (struct z_erofs_vle_compress_ctx *ctx, -unsigned int *len, -char *dst) +unsigned int *len,u8 *dst) {int ret;unsigned int count;
@@ -148,11 +147,10 @@ static int vle_compress_one(struct erofs_inode *inode, bool final) {struct erofs_compress *const h = &compresshandle; +u8 *const dst = ctx->dstbuf;unsigned int len = ctx->tail - ctx->head;unsigned int count;int ret; -static char dstbuf[EROFS_BLKSIZ *2 ]; -char *const dst = dstbuf +EROFS_BLKSIZ ;while (len) {bool raw;
@@ -389,21 +387,28 @@ int z_erofs_convert_to_compacted_format(struct erofs_inode *inode,int erofs_write_compressed_file (struct erofs_inode *inode) { -struct erofs_buffer_head *bh;struct z_erofs_vle_compress_ctx ctx; +u8 *compressmeta, *dstbuf; +int ret, fd; +struct erofs_buffer_head *bh;erofs_off_t remaining;erofs_blk_t blkaddr, compressed_blocks;unsigned int legacymetasize; -int ret, fd; -u8 *compressmeta =malloc (vle_compressmeta_capacity (inode->i_size)); -if (!compressmeta) + dstbuf =malloc ((EROFS_BLKSIZ + compresshandle.paddingdstsz) *2 ); +if (!dstbuf)return -ENOMEM ; + compressmeta =malloc (vle_compressmeta_capacity (inode->i_size)); +if (!compressmeta) { + ret = -ENOMEM ; + goto err_freedstbuf; + } + fd =open (inode->i_srcpath,O_RDONLY |O_BINARY );if (fd <0 ) { ret = -errno; - goto err_free; + goto err_freemeta; }/* allocate main data buffer */
@@ -418,6 +423,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode) blkaddr =erofs_mapbh (bh->block, true);/* start_blkaddr */ ctx.blkaddr = blkaddr; ctx.metacur = compressmeta +Z_EROFS_LEGACY_MAP_HEADER_SIZE ; + ctx.dstbuf = dstbuf +EROFS_BLKSIZ + compresshandle.paddingdstsz; ctx.head = ctx.tail =0 ; ctx.clusterofs =0 ; remaining = inode->i_size;
@@ -486,8 +492,10 @@ err_bdrop:erofs_bdrop (bh, true);/* revoke buffer */ err_close:close (fd); -err_free: +err_freemeta:free (compressmeta); +err_freedstbuf: +free (dstbuf);return ret; }
@@ -495,6 +503,8 @@ static int erofs_get_compress_algorithm_id(const char *name) {if (!strcmp (name,"lz4" ) || !strcmp (name,"lz4hc" ))return Z_EROFS_COMPRESSION_LZ4 ; +if (!strcmp (name,"xz" )) +return Z_EROFS_COMPRESSION_XZ ;return -ENOTSUP ; }
lib/compressor.c +3 -0
@@ -19,6 +19,9 @@ static struct erofs_compressor *compressors[] = {#endif &erofs_compressor_lz4,#endif +#if XZ_ENABLED + &erofs_compressor_xz, +#endif }; int erofs_compress_destsize(struct erofs_compress *c,
lib/compressor.h +2 -0
@@ -38,12 +38,14 @@ struct erofs_compress {unsigned int destsize_redzone_begin;unsigned int destsize_redzone_end; +unsigned int paddingdstsz;void *private_data; };/* list of compression algorithms */ extern struct erofs_compressor erofs_compressor_lz4;extern struct erofs_compressor erofs_compressor_lz4hc; +extern struct erofs_compressor erofs_compressor_xz; int erofs_compress_destsize(struct erofs_compress *c, int compression_level, void *src, unsigned int *srcsize,
lib/compressor_xz.c new file +277 -0
277 lines not on this page · raw message
Signature · 1 line
2.17.1