erofs-utils: support tail-packing inline compressed data
v9 cover erofs-utils: support tail-packing inline compressed data
v8: https://lore.kernel.org/r/[email protected]
This is a follow-up of previous v6 Yue Hu's patchset. It implements tail-packing inline for compressed files. In particular, called tail pcluster inline.
Here is some evaluation of Linux 5.10.87 source code (75368 inodes):
linux-5.10.87 (erofs, uncompressed) 972570624
linux-5.10.87 (erofs, lz4hc,9 4k tailpacking) 391696384
linux-5.10.87 (erofs, lz4hc,9 8k tailpacking) 368807936
linux-5.10.87 (erofs, lz4hc,9 16k tailpacking) 345649152
linux-5.10.87 (erofs, lz4hc,9 4k vanilla) 416079872
linux-5.10.87 (erofs, lz4hc,9 8k vanilla) 395493376
linux-5.10.87 (erofs, lz4hc,9 16k vanilla) 383213568
Usage:
mkfs.erofs -zlz4hc -Eztailpacking foo.erofs.img foo/
Thanks,
Gao Xiang
changes since v8:
- [1/2] sync up with the latest in-kernel version;
- [2/2] fix an assertion failure in debugging mode.
Yue Hu (2):
erofs-utils: fuse: support tail-packing inline compressed data
erofs-utils: mkfs: support tail-packing inline compressed data
include/erofs/config.h | 1 + include/erofs/internal.h | 6 ++ include/erofs_fs.h | 10 +++- lib/compress.c | 122 ++++++++++++++++++++++++++++++--------- lib/compressor.c | 16 +++-- lib/compressor.h | 2 +- lib/decompress.c | 5 +- lib/inode.c | 57 +++++++++++++----- lib/namei.c | 2 +- lib/zmap.c | 99 +++++++++++++++++++++++-------- mkfs/main.c | 8 +++ 11 files changed, 251 insertions(+), 77 deletions(-)
Signature · 1 line
2.24.4
PATCH v9 1/2 erofs-utils: fuse: support tail-packing inline compressed data
From: Yue Hu <[email protected]>
Add tail-packing inline compressed data support for erofsfuse.
--- include/erofs/internal.h | 5 ++ include/erofs_fs.h | 10 +++- lib/decompress.c | 5 +- lib/namei.c | 2 +- lib/zmap.c | 99 ++++++++++++++++++++++++++++++---------- 5 files changed, 92 insertions(+), 29 deletions(-)
include/erofs/internal.h +5 -0
@@ -131,6 +131,7 @@ EROFS_FEATURE_FUNCS(compr_cfgs, incompat, INCOMPAT_COMPR_CFGS) EROFS_FEATURE_FUNCS(big_pcluster, incompat, INCOMPAT_BIG_PCLUSTER) EROFS_FEATURE_FUNCS(chunked_file, incompat, INCOMPAT_CHUNKED_FILE) EROFS_FEATURE_FUNCS(device_table, incompat, INCOMPAT_DEVICE_TABLE) +EROFS_FEATURE_FUNCS(ztailpacking, incompat, INCOMPAT_ZTAILPACKING) EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM)#define EROFS_I_EA_INITED (1 << 0)
@@ -192,6 +193,9 @@ struct erofs_inode {uint8_t z_algorithmtype[2 ];uint8_t z_logical_clusterbits;uint8_t z_physical_clusterblks; +uint64_t z_tailextent_headlcn; +unsigned int z_idataoff; +#define z_idata_size idata_size }; };#ifdef WITH_ANDROID
@@ -295,6 +299,7 @@ struct erofs_map_blocks {* approach instead if possible since it's more metadata lightweight.) */ #define EROFS_GET_BLOCKS_FIEMAP 0x0002 +#define EROFS_GET_BLOCKS_FINDTAIL 0x0008enum {Z_EROFS_COMPRESSION_SHIFTED =Z_EROFS_COMPRESSION_MAX ,
include/erofs_fs.h +8 -2
@@ -23,12 +23,14 @@#define EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER 0x00000002#define EROFS_FEATURE_INCOMPAT_CHUNKED_FILE 0x00000004#define EROFS_FEATURE_INCOMPAT_DEVICE_TABLE 0x00000008 +#define EROFS_FEATURE_INCOMPAT_ZTAILPACKING 0x00000010#define EROFS_ALL_FEATURE_INCOMPAT \ (EROFS_FEATURE_INCOMPAT_LZ4_0PADDING | \ EROFS_FEATURE_INCOMPAT_COMPR_CFGS | \ EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER | \ EROFS_FEATURE_INCOMPAT_CHUNKED_FILE | \ - EROFS_FEATURE_INCOMPAT_DEVICE_TABLE) + EROFS_FEATURE_INCOMPAT_DEVICE_TABLE | \ + EROFS_FEATURE_INCOMPAT_ZTAILPACKING)#define EROFS_SB_EXTSLOT_SIZE 16
@@ -290,13 +292,17 @@ struct z_erofs_lzma_cfgs {* (4B) + 2B + (4B) if compacted 2B is on. * bit 1 : HEAD1 big pcluster (0 - off; 1 - on) * bit 2 : HEAD2 big pcluster (0 - off; 1 - on) +* bit 3 : tailpacking inline pcluster (0 - off; 1 - on) */ #define Z_EROFS_ADVISE_COMPACTED_2B 0x0001#define Z_EROFS_ADVISE_BIG_PCLUSTER_1 0x0002#define Z_EROFS_ADVISE_BIG_PCLUSTER_2 0x0004 +#define Z_EROFS_ADVISE_INLINE_PCLUSTER 0x0008struct z_erofs_map_header { -__le32 h_reserved1; +__le16 h_reserved1; +/* record the size of tailpacking data */ +__le16 h_idata_size;__le16 h_advise;/* * bit 0-3 : algorithm type of head 1 (logical cluster type 01);
lib/decompress.c +4 -1
@@ -110,6 +110,9 @@ static int z_erofs_decompress_lz4(struct z_erofs_decompress_req *rq) rq->decodedlength);if (ret != (int )rq->decodedlength) { +erofs_err ("failed to %s decompress %d in[%u, %u] out[%u]" , + rq->partial_decoding ?"partial" :"full" , + ret, rq->inputsize, inputmargin, rq->decodedlength); ret = -EIO ; goto out; }
@@ -129,7 +132,7 @@ out:int z_erofs_decompress (struct z_erofs_decompress_req *rq) {if (rq->alg ==Z_EROFS_COMPRESSION_SHIFTED ) { -if (rq->inputsize !=EROFS_BLKSIZ ) +if (rq->inputsize >EROFS_BLKSIZ )return -EFSCORRUPTED ;DBG_BUGON (rq->decodedlength >EROFS_BLKSIZ );
lib/namei.c +1 -1
@@ -137,7 +137,7 @@ int erofs_read_inode_from_disk(struct erofs_inode *vi) vi->u.chunkbits =LOG_BLOCK_SIZE + (vi->u.chunkformat &EROFS_CHUNK_FORMAT_BLKBITS_MASK ); }else if (erofs_inode_is_data_compressed (vi->datalayout)) -z_erofs_fill_inode (vi); +return z_erofs_fill_inode (vi);return 0 ; bogusimode:erofs_err ("bogus i_mode (%o) @ nid %llu" , vi->i_mode, vi->nid |0ULL );
lib/zmap.c +74 -25
195 lines not on this page · raw message
Signature · 1 line
2.24.4
PATCH v9 2/2 erofs-utils: mkfs: support tail-packing inline compressed data
In reply to Gao XiangFrom: Yue Hu <[email protected]>
Currently, we only support tail-packing inline data for uncompressed files, let's support it as well for compressed files.
--- include/erofs/config.h | 1 + include/erofs/internal.h | 1 + lib/compress.c | 122 ++++++++++++++++++++++++++++++--------- lib/compressor.c | 16 +++-- lib/compressor.h | 2 +- lib/inode.c | 57 +++++++++++++----- mkfs/main.c | 8 +++ 7 files changed, 159 insertions(+), 48 deletions(-)
include/erofs/config.h +1 -0
@@ -43,6 +43,7 @@ struct erofs_configure {char c_timeinherit;char c_chunkbits;bool c_noinline_data; +bool c_ztailpacking;#ifdef HAVE_LIBSELINUX struct selabel_handle *sehnd;
include/erofs/internal.h +1 -0
@@ -175,6 +175,7 @@ struct erofs_inode {unsigned char inode_isize;/* inline tail-end packing size */ unsigned short idata_size; +bool compressed_idata;unsigned int xattr_isize;unsigned int extent_isize;
lib/compress.c +96 -26
212 lines not on this page · raw message
lib/compressor.c +11 -5
@@ -24,23 +24,29 @@ static const struct erofs_compressor *compressors[] = {int erofs_compress_destsize (const struct erofs_compress *c,const void *src,unsigned int *srcsize, -void *dst,unsigned int dstsize) +void *dst,unsigned int dstsize,bool inblocks) { -unsigned int uncompressed_size; +unsigned int uncompressed_capacity, compressed_size;int ret;DBG_BUGON (!c->alg);if (!c->alg->compress_destsize)return -ENOTSUP ; + uncompressed_capacity = *srcsize; ret = c->alg->compress_destsize (c, src, srcsize, dst, dstsize);if (ret <0 )return ret; +/* XXX: ret >= EROFS_BLKSIZ is a temporary hack for ztailpacking */ +if (inblocks || ret >=EROFS_BLKSIZ || + uncompressed_capacity != *srcsize) + compressed_size =roundup (ret,EROFS_BLKSIZ ); +else + compressed_size = ret; +DBG_BUGON (c->compress_threshold <100 );/* check if there is enough gains to compress */ - uncompressed_size = *srcsize; -if (roundup (ret,EROFS_BLKSIZ ) >= uncompressed_size * - c->compress_threshold /100 ) +if (*srcsize <= compressed_size * c->compress_threshold /100 )return -EAGAIN ;return ret; }
lib/compressor.h +1 -1
@@ -47,7 +47,7 @@ extern const struct erofs_compressor erofs_compressor_lzma;int erofs_compress_destsize (const struct erofs_compress *c,const void *src,unsigned int *srcsize, -void *dst,unsigned int dstsize); +void *dst,unsigned int dstsize,bool inblocks);int erofs_compressor_setlevel (struct erofs_compress *c,int compression_level);int erofs_compressor_init (struct erofs_compress *c,char *alg_name);
lib/inode.c +41 -16
91 lines not on this page · raw message
mkfs/main.c +8 -0
@@ -187,6 +187,12 @@ static int parse_extended_opts(const char *opts)return -EINVAL ; cfg.c_force_chunkformat =FORCE_INODE_CHUNK_INDEXES ; } + +if (MATCH_EXTENTED_OPT ("ztailpacking" , token, keylen)) { +if (vallen) +return -EINVAL ; + cfg.c_ztailpacking = true; + } }return 0 ; }
@@ -625,6 +631,8 @@ int main(int argc, char **argv)erofs_show_config ();if (erofs_sb_has_chunked_file ())erofs_warn ("EXPERIMENTAL chunked file feature in use. Use at your own risk!" ); +if (cfg.c_ztailpacking) +erofs_warn ("EXPERIMENTAL compressed inline data feature in use. Use at your own risk!" );erofs_set_fs_root (cfg.c_src_path);#ifndef NDEBUG if (cfg.c_random_pclusterblks)
Signature · 1 line
2.24.4