Skip to contentSkip to conversation

erofs-utils: support tail-packing inline compressed data

PATCH New Gao Xiang 1 participant 3 messages Jan 14, 2022 Library code

lore
Share & tools
lore

Gao Xiang

author
#0 lore raw

v9 cover erofs-utils: support tail-packing inline compressed data

v8: https://lore.kernel.org/r/[email protected]

This is a follow-up of previous v6 Yue Hu's patchset. It implements tail-packing inline for compressed files. In particular, called tail pcluster inline.

Here is some evaluation of Linux 5.10.87 source code (75368 inodes):
linux-5.10.87 (erofs, uncompressed) 972570624

linux-5.10.87 (erofs, lz4hc,9 4k tailpacking) 391696384
linux-5.10.87 (erofs, lz4hc,9 8k tailpacking) 368807936
linux-5.10.87 (erofs, lz4hc,9 16k tailpacking) 345649152

linux-5.10.87 (erofs, lz4hc,9 4k vanilla) 416079872
linux-5.10.87 (erofs, lz4hc,9 8k vanilla) 395493376
linux-5.10.87 (erofs, lz4hc,9 16k vanilla) 383213568

Usage:
mkfs.erofs -zlz4hc -Eztailpacking foo.erofs.img foo/

Thanks,
Gao Xiang

changes since v8:
- [1/2] sync up with the latest in-kernel version;
- [2/2] fix an assertion failure in debugging mode.

Yue Hu (2):
erofs-utils: fuse: support tail-packing inline compressed data
erofs-utils: mkfs: support tail-packing inline compressed data

 include/erofs/config.h   |   1 +
 include/erofs/internal.h |   6 ++
 include/erofs_fs.h       |  10 +++-
 lib/compress.c           | 122 ++++++++++++++++++++++++++++++---------
 lib/compressor.c         |  16 +++--
 lib/compressor.h         |   2 +-
 lib/decompress.c         |   5 +-
 lib/inode.c              |  57 +++++++++++++-----
 lib/namei.c              |   2 +-
 lib/zmap.c               |  99 +++++++++++++++++++++++--------
 mkfs/main.c              |   8 +++
 11 files changed, 251 insertions(+), 77 deletions(-)
Signature · 1 line
2.24.4
author
#1 lore raw

PATCH v9 1/2 erofs-utils: fuse: support tail-packing inline compressed data

From: Yue Hu <[email protected]>

Add tail-packing inline compressed data support for erofsfuse.

---
 include/erofs/internal.h |  5 ++
 include/erofs_fs.h       | 10 +++-
 lib/decompress.c         |  5 +-
 lib/namei.c              |  2 +-
 lib/zmap.c               | 99 ++++++++++++++++++++++++++++++----------
 5 files changed, 92 insertions(+), 29 deletions(-)
include/erofs/internal.h +5 -0
index 2c7b611..68cab50 100644
@@ -131,6 +131,7 @@ EROFS_FEATURE_FUNCS(compr_cfgs, incompat, INCOMPAT_COMPR_CFGS)
 EROFS_FEATURE_FUNCS(big_pcluster, incompat, INCOMPAT_BIG_PCLUSTER)
 EROFS_FEATURE_FUNCS(chunked_file, incompat, INCOMPAT_CHUNKED_FILE)
 EROFS_FEATURE_FUNCS(device_table, incompat, INCOMPAT_DEVICE_TABLE)
+EROFS_FEATURE_FUNCS(ztailpacking, incompat, INCOMPAT_ZTAILPACKING)
 EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM)
 
 #define EROFS_I_EA_INITED	(1 << 0)
@@ -192,6 +193,9 @@ struct erofs_inode {
 			uint8_t  z_algorithmtype[2];
 			uint8_t  z_logical_clusterbits;
 			uint8_t  z_physical_clusterblks;
+			uint64_t z_tailextent_headlcn;
+			unsigned int    z_idataoff;
+#define z_idata_size	idata_size
 		};
 	};
 #ifdef WITH_ANDROID
@@ -295,6 +299,7 @@ struct erofs_map_blocks {
  * approach instead if possible since it's more metadata lightweight.)
  */
 #define EROFS_GET_BLOCKS_FIEMAP	0x0002
+#define EROFS_GET_BLOCKS_FINDTAIL	0x0008
 
 enum {
 	Z_EROFS_COMPRESSION_SHIFTED = Z_EROFS_COMPRESSION_MAX,
include/erofs_fs.h +8 -2
index 9a91877..59d9bbb 100644
@@ -23,12 +23,14 @@
 #define EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER	0x00000002
 #define EROFS_FEATURE_INCOMPAT_CHUNKED_FILE	0x00000004
 #define EROFS_FEATURE_INCOMPAT_DEVICE_TABLE	0x00000008
+#define EROFS_FEATURE_INCOMPAT_ZTAILPACKING	0x00000010
 #define EROFS_ALL_FEATURE_INCOMPAT		\
 	(EROFS_FEATURE_INCOMPAT_LZ4_0PADDING | \
 	 EROFS_FEATURE_INCOMPAT_COMPR_CFGS | \
 	 EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER | \
 	 EROFS_FEATURE_INCOMPAT_CHUNKED_FILE | \
-	 EROFS_FEATURE_INCOMPAT_DEVICE_TABLE)
+	 EROFS_FEATURE_INCOMPAT_DEVICE_TABLE | \
+	 EROFS_FEATURE_INCOMPAT_ZTAILPACKING)
 
 #define EROFS_SB_EXTSLOT_SIZE	16
 
@@ -290,13 +292,17 @@ struct z_erofs_lzma_cfgs {
  *                                  (4B) + 2B + (4B) if compacted 2B is on.
  * bit 1 : HEAD1 big pcluster (0 - off; 1 - on)
  * bit 2 : HEAD2 big pcluster (0 - off; 1 - on)
+ * bit 3 : tailpacking inline pcluster (0 - off; 1 - on)
  */
 #define Z_EROFS_ADVISE_COMPACTED_2B		0x0001
 #define Z_EROFS_ADVISE_BIG_PCLUSTER_1		0x0002
 #define Z_EROFS_ADVISE_BIG_PCLUSTER_2		0x0004
+#define Z_EROFS_ADVISE_INLINE_PCLUSTER		0x0008
 
 struct z_erofs_map_header {
-	__le32	h_reserved1;
+	__le16	h_reserved1;
+	/* record the size of tailpacking data */
+	__le16  h_idata_size;
 	__le16	h_advise;
 	/*
 	 * bit 0-3 : algorithm type of head 1 (logical cluster type 01);
lib/decompress.c +4 -1
index 359dae7..1661f91 100644
@@ -110,6 +110,9 @@ static int z_erofs_decompress_lz4(struct z_erofs_decompress_req *rq)
 					  rq->decodedlength);
 
 	if (ret != (int)rq->decodedlength) {
+		erofs_err("failed to %s decompress %d in[%u, %u] out[%u]",
+			  rq->partial_decoding ? "partial" : "full",
+			  ret, rq->inputsize, inputmargin, rq->decodedlength);
 		ret = -EIO;
 		goto out;
 	}
@@ -129,7 +132,7 @@ out:
 int z_erofs_decompress(struct z_erofs_decompress_req *rq)
 {
 	if (rq->alg == Z_EROFS_COMPRESSION_SHIFTED) {
-		if (rq->inputsize != EROFS_BLKSIZ)
+		if (rq->inputsize > EROFS_BLKSIZ)
 			return -EFSCORRUPTED;
 
 		DBG_BUGON(rq->decodedlength > EROFS_BLKSIZ);
lib/namei.c +1 -1
index 4124170..97f0f80 100644
@@ -137,7 +137,7 @@ int erofs_read_inode_from_disk(struct erofs_inode *vi)
 		vi->u.chunkbits = LOG_BLOCK_SIZE +
 			(vi->u.chunkformat & EROFS_CHUNK_FORMAT_BLKBITS_MASK);
 	} else if (erofs_inode_is_data_compressed(vi->datalayout))
-		z_erofs_fill_inode(vi);
+		return z_erofs_fill_inode(vi);
 	return 0;
 bogusimode:
 	erofs_err("bogus i_mode (%o) @ nid %llu", vi->i_mode, vi->nid | 0ULL);
lib/zmap.c +74 -25

195 lines not on this page · raw message

Signature · 1 line
2.24.4
author
#2 lore raw

PATCH v9 2/2 erofs-utils: mkfs: support tail-packing inline compressed data

In reply to Gao Xiang

From: Yue Hu <[email protected]>

Currently, we only support tail-packing inline data for uncompressed files, let's support it as well for compressed files.

---
 include/erofs/config.h   |   1 +
 include/erofs/internal.h |   1 +
 lib/compress.c           | 122 ++++++++++++++++++++++++++++++---------
 lib/compressor.c         |  16 +++--
 lib/compressor.h         |   2 +-
 lib/inode.c              |  57 +++++++++++++-----
 mkfs/main.c              |   8 +++
 7 files changed, 159 insertions(+), 48 deletions(-)
include/erofs/config.h +1 -0
index e41c079..a3f9bac 100644
@@ -43,6 +43,7 @@ struct erofs_configure {
 	char c_timeinherit;
 	char c_chunkbits;
 	bool c_noinline_data;
+	bool c_ztailpacking;
 
 #ifdef HAVE_LIBSELINUX
 	struct selabel_handle *sehnd;
include/erofs/internal.h +1 -0
index 68cab50..3615297 100644
@@ -175,6 +175,7 @@ struct erofs_inode {
 	unsigned char inode_isize;
 	/* inline tail-end packing size */
 	unsigned short idata_size;
+	bool compressed_idata;
 
 	unsigned int xattr_isize;
 	unsigned int extent_isize;
lib/compress.c +96 -26

212 lines not on this page · raw message

lib/compressor.c +11 -5
index 3e04a3a..3666496 100644
@@ -24,23 +24,29 @@ static const struct erofs_compressor *compressors[] = {
 
 int erofs_compress_destsize(const struct erofs_compress *c,
 			    const void *src, unsigned int *srcsize,
-			    void *dst, unsigned int dstsize)
+			    void *dst, unsigned int dstsize, bool inblocks)
 {
-	unsigned int uncompressed_size;
+	unsigned int uncompressed_capacity, compressed_size;
 	int ret;
 
 	DBG_BUGON(!c->alg);
 	if (!c->alg->compress_destsize)
 		return -ENOTSUP;
 
+	uncompressed_capacity = *srcsize;
 	ret = c->alg->compress_destsize(c, src, srcsize, dst, dstsize);
 	if (ret < 0)
 		return ret;
 
+	/* XXX: ret >= EROFS_BLKSIZ is a temporary hack for ztailpacking */
+	if (inblocks || ret >= EROFS_BLKSIZ ||
+	    uncompressed_capacity != *srcsize)
+		compressed_size = roundup(ret, EROFS_BLKSIZ);
+	else
+		compressed_size = ret;
+	DBG_BUGON(c->compress_threshold < 100);
 	/* check if there is enough gains to compress */
-	uncompressed_size = *srcsize;
-	if (roundup(ret, EROFS_BLKSIZ) >= uncompressed_size *
-	    c->compress_threshold / 100)
+	if (*srcsize <= compressed_size * c->compress_threshold / 100)
 		return -EAGAIN;
 	return ret;
 }
lib/compressor.h +1 -1
index 728dd0b..cf063f1 100644
@@ -47,7 +47,7 @@ extern const struct erofs_compressor erofs_compressor_lzma;
 
 int erofs_compress_destsize(const struct erofs_compress *c,
 			    const void *src, unsigned int *srcsize,
-			    void *dst, unsigned int dstsize);
+			    void *dst, unsigned int dstsize, bool inblocks);
 
 int erofs_compressor_setlevel(struct erofs_compress *c, int compression_level);
 int erofs_compressor_init(struct erofs_compress *c, char *alg_name);
lib/inode.c +41 -16

91 lines not on this page · raw message

mkfs/main.c +8 -0
index 90cedde..9eb696a 100644
@@ -187,6 +187,12 @@ static int parse_extended_opts(const char *opts)
 				return -EINVAL;
 			cfg.c_force_chunkformat = FORCE_INODE_CHUNK_INDEXES;
 		}
+
+		if (MATCH_EXTENTED_OPT("ztailpacking", token, keylen)) {
+			if (vallen)
+				return -EINVAL;
+			cfg.c_ztailpacking = true;
+		}
 	}
 	return 0;
 }
@@ -625,6 +631,8 @@ int main(int argc, char **argv)
 	erofs_show_config();
 	if (erofs_sb_has_chunked_file())
 		erofs_warn("EXPERIMENTAL chunked file feature in use. Use at your own risk!");
+	if (cfg.c_ztailpacking)
+		erofs_warn("EXPERIMENTAL compressed inline data feature in use. Use at your own risk!");
 	erofs_set_fs_root(cfg.c_src_path);
 #ifndef NDEBUG
 	if (cfg.c_random_pclusterblks)
Signature · 1 line
2.24.4