Skip to contentSkip to conversation

erofs-utils: add XZ algorithm support

RFC PATCH New Gao Xiang 1 participant 1 message Nov 14, 2019 Library code

lore
Share & tools
lore

Gao Xiang

author
#0 lore raw

RFC PATCH 1/1 erofs-utils: add XZ algorithm support

It's silly but roughtly usable *_destsize implementation for XZ by using liblzma stream API and fitblk-like approach, whose generated data can be directly decompressed by kernel.

I can investigate it deeply by hacking lzma2/XZ internals if I have more extra time to do, that is another story for now.

Currently, MT compression support would be a higher priority stuff and even for fixed-sized output compression it can be multi-threaded by fixed-sized input segments:
_________________________________________________________

|  thread 0  |  thread 1   |  thread 0  |  thread 1  | ...
|____x MB____|____x MB_____|____x MB____|____x MB____|____

MT compression can also be implemented on per-file basis of course.

---

A brief fitblk-like XZ approach compression ratio comparsion:
459382784 enwik9_4k.xz.squashfs.img
401649664 enwik9_8k.xz.squashfs.img
378892288 enwik9_4k.xz.erofs.img (bisect is forcely true) 360972288 enwik9_16k.xz.squashfs.img

 configure.ac        |  12 ++
 include/erofs_fs.h  |   1 +
 lib/Makefile.am     |   4 +
 lib/compress.c      |  32 +++--
 lib/compressor.c    |   3 +
 lib/compressor.h    |   2 +
 lib/compressor_xz.c | 277 ++++++++++++++++++++++++++++++++++++++++++++
 7 files changed, 320 insertions(+), 11 deletions(-)
 create mode 100644 lib/compressor_xz.c
configure.ac +12 -0
index a5adf172cc43..f8cc8efebf2f 100644
@@ -154,9 +154,17 @@ if test "x$enable_lz4" = "xyes"; then
   CPPFLAGS=${saved_CPPFLAGS}
 fi
 
+AC_CHECK_HEADERS(lzma.h,
+  AC_CHECK_LIB(lzma, lzma_code,[
+      have_liblzma="yes"
+      LZMA_LIBS="-llzma"
+      AC_DEFINE([XZ_ENABLED], [1], [Define to 1 if xz is enabled.])
+    ], [have_liblzma="no"]))
+
 # Set up needed symbols, conditionals and compiler/linker flags
 AM_CONDITIONAL([ENABLE_LZ4], [test "x${have_lz4}" = "xyes"])
 AM_CONDITIONAL([ENABLE_LZ4HC], [test "x${have_lz4hc}" = "xyes"])
+AM_CONDITIONAL([ENABLE_XZ], [test "x${have_liblzma}" = "xyes"])
 
 if test "x${have_lz4}" = "xyes"; then
   AC_DEFINE([LZ4_ENABLED], [1], [Define to 1 if lz4 is enabled.])
@@ -173,6 +181,10 @@ if test "x${have_lz4}" = "xyes"; then
   LIBS="$LZ4_LIBS $LIBS"
 fi
 
+if test "x${have_liblzma}" = "xyes"; then
+LIBS="$LZMA_LIBS $LIBS"
+fi
+
 AC_CONFIG_FILES([Makefile
 		 man/Makefile
 		 lib/Makefile
include/erofs_fs.h +1 -0
index bcc4f0c630ad..0310f1d07f1c 100644
@@ -194,6 +194,7 @@ static inline unsigned int erofs_xattr_entry_size(struct erofs_xattr_entry *e)
 /* available compression algorithm types (for h_algorithmtype) */
 enum {
 	Z_EROFS_COMPRESSION_LZ4	= 0,
+	Z_EROFS_COMPRESSION_XZ = 1,
 	Z_EROFS_COMPRESSION_MAX
 };
 
lib/Makefile.am +4 -0
index 1ff81f9eebfe..b978b361fb72 100644
@@ -12,4 +12,8 @@ if ENABLE_LZ4HC
 liberofs_la_SOURCES += compressor_lz4hc.c
 endif
 endif
+if ENABLE_XZ
+liberofs_la_CFLAGS += ${LZMA_CFLAGS}
+liberofs_la_SOURCES += compressor_xz.c
+endif
 
lib/compress.c +21 -11
index 99fd527ead50..b7106c081712 100644
@@ -23,7 +23,7 @@ static int compressionlevel;
 static struct z_erofs_map_header mapheader;
 
 struct z_erofs_vle_compress_ctx {
-	u8 *metacur;
+	u8 *metacur, *dstbuf;
 
 	u8 queue[EROFS_CONFIG_COMPR_MAX_SZ * 2];
 	unsigned int head, tail;
@@ -114,8 +114,7 @@ static void vle_write_indexes(struct z_erofs_vle_compress_ctx *ctx,
 }
 
 static int write_uncompressed_block(struct z_erofs_vle_compress_ctx *ctx,
-				    unsigned int *len,
-				    char *dst)
+				    unsigned int *len, u8 *dst)
 {
 	int ret;
 	unsigned int count;
@@ -148,11 +147,10 @@ static int vle_compress_one(struct erofs_inode *inode,
 			    bool final)
 {
 	struct erofs_compress *const h = &compresshandle;
+	u8 *const dst = ctx->dstbuf;
 	unsigned int len = ctx->tail - ctx->head;
 	unsigned int count;
 	int ret;
-	static char dstbuf[EROFS_BLKSIZ * 2];
-	char *const dst = dstbuf + EROFS_BLKSIZ;
 
 	while (len) {
 		bool raw;
@@ -389,21 +387,28 @@ int z_erofs_convert_to_compacted_format(struct erofs_inode *inode,
 
 int erofs_write_compressed_file(struct erofs_inode *inode)
 {
-	struct erofs_buffer_head *bh;
 	struct z_erofs_vle_compress_ctx ctx;
+	u8 *compressmeta, *dstbuf;
+	int ret, fd;
+	struct erofs_buffer_head *bh;
 	erofs_off_t remaining;
 	erofs_blk_t blkaddr, compressed_blocks;
 	unsigned int legacymetasize;
-	int ret, fd;
 
-	u8 *compressmeta = malloc(vle_compressmeta_capacity(inode->i_size));
-	if (!compressmeta)
+	dstbuf = malloc((EROFS_BLKSIZ + compresshandle.paddingdstsz) * 2);
+	if (!dstbuf)
 		return -ENOMEM;
 
+	compressmeta = malloc(vle_compressmeta_capacity(inode->i_size));
+	if (!compressmeta) {
+		ret = -ENOMEM;
+		goto err_freedstbuf;
+	}
+
 	fd = open(inode->i_srcpath, O_RDONLY | O_BINARY);
 	if (fd < 0) {
 		ret = -errno;
-		goto err_free;
+		goto err_freemeta;
 	}
 
 	/* allocate main data buffer */
@@ -418,6 +423,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
 	blkaddr = erofs_mapbh(bh->block, true);	/* start_blkaddr */
 	ctx.blkaddr = blkaddr;
 	ctx.metacur = compressmeta + Z_EROFS_LEGACY_MAP_HEADER_SIZE;
+	ctx.dstbuf = dstbuf + EROFS_BLKSIZ + compresshandle.paddingdstsz;
 	ctx.head = ctx.tail = 0;
 	ctx.clusterofs = 0;
 	remaining = inode->i_size;
@@ -486,8 +492,10 @@ err_bdrop:
 	erofs_bdrop(bh, true);	/* revoke buffer */
 err_close:
 	close(fd);
-err_free:
+err_freemeta:
 	free(compressmeta);
+err_freedstbuf:
+	free(dstbuf);
 	return ret;
 }
 
@@ -495,6 +503,8 @@ static int erofs_get_compress_algorithm_id(const char *name)
 {
 	if (!strcmp(name, "lz4") || !strcmp(name, "lz4hc"))
 		return Z_EROFS_COMPRESSION_LZ4;
+	if (!strcmp(name, "xz"))
+		return Z_EROFS_COMPRESSION_XZ;
 	return -ENOTSUP;
 }
 
lib/compressor.c +3 -0
index b2434e0e5418..daf413bf9a6c 100644
@@ -19,6 +19,9 @@ static struct erofs_compressor *compressors[] = {
 #endif
 		&erofs_compressor_lz4,
 #endif
+#if XZ_ENABLED
+		&erofs_compressor_xz,
+#endif
 };
 
 int erofs_compress_destsize(struct erofs_compress *c,
lib/compressor.h +2 -0
index b2471c41ca4e..6be4210caf8a 100644
@@ -38,12 +38,14 @@ struct erofs_compress {
 	unsigned int destsize_redzone_begin;
 	unsigned int destsize_redzone_end;
 
+	unsigned int paddingdstsz;
 	void *private_data;
 };
 
 /* list of compression algorithms */
 extern struct erofs_compressor erofs_compressor_lz4;
 extern struct erofs_compressor erofs_compressor_lz4hc;
+extern struct erofs_compressor erofs_compressor_xz;
 
 int erofs_compress_destsize(struct erofs_compress *c, int compression_level,
 			    void *src, unsigned int *srcsize,
lib/compressor_xz.c new file +277 -0

277 lines not on this page · raw message

Signature · 1 line
2.17.1