erofs-utils: compressed fragments feature
v4 cover erofs-utils: compressed fragments feature
From: Yue Hu <[email protected]>
In order to achieve greater compression ratio, let's introduce compressed fragments feature which can merge tail of per-file or the whole files into one special inode to reach the target.
And we can also set pcluster size to fragments inode for different compression requirments.
In this patchset, we also improve the uncompressed data layout of compressed files. Just write it from 'clusterofs' instead of 0 since it can benefit from in-place I/O. For now, it only goes with fragments.
The main idea above is from Xiang.
Here is some test data of Linux 5.10.87 source code under Ubuntu 18.04:
linux-5.10.87 (erofs, uncompressed) 1.1G
linux-5.10.87 (erofs, lz4hc,12 4k fragments,4k) 301M linux-5.10.87 (erofs, lz4hc,12 8k fragments,8k) 268M linux-5.10.87 (erofs, lz4hc,12 16k fragments,16k) 242M linux-5.10.87 (erofs, lz4hc,12 32k fragments,32k) 225M linux-5.10.87 (erofs, lz4hc,12 64k fragments,64k) 217M
linux-5.10.87 (erofs, lz4hc,12 4k vanilla) 396M linux-5.10.87 (erofs, lz4hc,12 8k vanilla) 376M linux-5.10.87 (erofs, lz4hc,12 16k vanilla) 364M linux-5.10.87 (erofs, lz4hc,12 32k vanilla) 359M linux-5.10.87 (erofs, lz4hc,12 64k vanilla) 358M
Usage:
mkfs.erofs -zlz4hc,12 -C65536 -Efragments,65536 foo.erofs.img foo/
Changes since v3:
- fuse minor change and modify fragments naming related suggested by Xiang;
- refator fragments build code;
- rebase to latest commit 547bea3cb71a.
Changes since v2:
- mainly reimplment the decompression logic for fragment inode due to kernel side;
- fix compatibility issue to old image with ztailpacking feature;
- move code of super.c in patch 3/3 to patch 1/3;
- minor naming change.
Changes since v1:
- mainly optimize index space for fragment inode;
- add merging tail with len <= pclustersize into fragments directly;
- use a inode instead of nid to avoid multiple load fragments;
- fix memory leak of building fragments;
- minor change to diff special fragments with normal inode.
- rebase to commit cb058526 with patch [1];
- code cleanup.
Note that inode will be extended version (64 bytes) due to mtime, may use 'force-inode-compact' option to reduce the size if mtime careless.
[1] https://lore.kernel.org/linux-erofs/[email protected]/
Yue Hu (3):
erofs-utils: lib: add support for fragments data decompression
erofs-utils: lib: support on-disk offset for shifted decompression
erofs-utils: introduce compressed fragments support
include/erofs/compress.h | 3 +- include/erofs/config.h | 3 +- include/erofs/decompress.h | 3 ++ include/erofs/fragments.h | 25 +++++++++ include/erofs/inode.h | 1 + include/erofs/internal.h | 7 +++ include/erofs_fs.h | 27 +++++++--- lib/Makefile.am | 4 +- lib/compress.c | 108 +++++++++++++++++++++++++++---------- lib/data.c | 28 +++++++++- lib/decompress.c | 10 +++- lib/fragments.c | 58 ++++++++++++++++++++ lib/inode.c | 59 +++++++++++++++----- lib/super.c | 24 ++++++++- lib/zmap.c | 26 +++++++++ mkfs/main.c | 64 +++++++++++++++++++--- 16 files changed, 390 insertions(+), 60 deletions(-) create mode 100644 include/erofs/fragments.h create mode 100644 lib/fragments.c
Signature · 1 line
2.17.1
RFC PATCH v4 1/3 erofs-utils: lib: add support for fragments data decompression
From: Yue Hu <[email protected]>
Add compressed fragments support for erofsfuse.
--- include/erofs/internal.h | 6 ++++++ include/erofs_fs.h | 26 ++++++++++++++++++++------ lib/data.c | 20 ++++++++++++++++++++ lib/super.c | 24 +++++++++++++++++++++++- lib/zmap.c | 26 ++++++++++++++++++++++++++ 5 files changed, 95 insertions(+), 7 deletions(-)
include/erofs/internal.h +6 -0
@@ -102,6 +102,7 @@ struct erofs_sb_info {u16 devt_slotoff;/* used for mkfs */ u16 device_id_mask;/* used for others */ }; +struct erofs_inode *packed_inode; };/* global sbi */
@@ -132,6 +133,7 @@ EROFS_FEATURE_FUNCS(big_pcluster, incompat, INCOMPAT_BIG_PCLUSTER) EROFS_FEATURE_FUNCS(chunked_file, incompat, INCOMPAT_CHUNKED_FILE) EROFS_FEATURE_FUNCS(device_table, incompat, INCOMPAT_DEVICE_TABLE) EROFS_FEATURE_FUNCS(ztailpacking, incompat, INCOMPAT_ZTAILPACKING) +EROFS_FEATURE_FUNCS(fragments, incompat, INCOMPAT_FRAGMENTS) EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM)#define EROFS_I_EA_INITED (1 << 0)
@@ -209,6 +211,7 @@ struct erofs_inode {#ifdef WITH_ANDROID uint64_t capabilities;#endif +erofs_off_t fragmentoff; };static inline bool is_inode_layout_compression (struct erofs_inode *inode)
@@ -279,6 +282,7 @@ enum { BH_Mapped, BH_Encoded, BH_FullMapped, + BH_Fragment, };/* Has a disk mapping */
@@ -289,6 +293,8 @@ enum {#define EROFS_MAP_ENCODED (1 << BH_Encoded)/* The length of extent is full */ #define EROFS_MAP_FULL_MAPPED (1 << BH_FullMapped) +/* Located in the special packed inode */ +#define EROFS_MAP_FRAGMENT (1 << BH_Fragment)struct erofs_map_blocks {char mpage[EROFS_BLKSIZ ];
include/erofs_fs.h +20 -6
@@ -25,13 +25,15 @@#define EROFS_FEATURE_INCOMPAT_CHUNKED_FILE 0x00000004#define EROFS_FEATURE_INCOMPAT_DEVICE_TABLE 0x00000008#define EROFS_FEATURE_INCOMPAT_ZTAILPACKING 0x00000010 +#define EROFS_FEATURE_INCOMPAT_FRAGMENTS 0x00000020#define EROFS_ALL_FEATURE_INCOMPAT \ (EROFS_FEATURE_INCOMPAT_LZ4_0PADDING | \ EROFS_FEATURE_INCOMPAT_COMPR_CFGS | \ EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER | \ EROFS_FEATURE_INCOMPAT_CHUNKED_FILE | \ EROFS_FEATURE_INCOMPAT_DEVICE_TABLE | \ - EROFS_FEATURE_INCOMPAT_ZTAILPACKING) + EROFS_FEATURE_INCOMPAT_ZTAILPACKING | \ + EROFS_FEATURE_INCOMPAT_FRAGMENTS)#define EROFS_SB_EXTSLOT_SIZE 16
@@ -73,7 +75,9 @@ struct erofs_super_block { }__packed u1;__le16 extra_devices;/* # of devices besides the primary device */ __le16 devt_slotoff;/* startoff = devt_slotoff * devt_slotsize */ -__u8 reserved2[38 ]; +__u8 reserved[6 ]; +__le64 packed_nid;/* nid of the special packed inode */ +__u8 reserved2[24 ]; };/*
@@ -294,16 +298,25 @@ struct z_erofs_lzma_cfgs {* bit 1 : HEAD1 big pcluster (0 - off; 1 - on) * bit 2 : HEAD2 big pcluster (0 - off; 1 - on) * bit 3 : tailpacking inline pcluster (0 - off; 1 - on) +* bit 4 : fragment pcluster (0 - off; 1 - on) */ #define Z_EROFS_ADVISE_COMPACTED_2B 0x0001#define Z_EROFS_ADVISE_BIG_PCLUSTER_1 0x0002#define Z_EROFS_ADVISE_BIG_PCLUSTER_2 0x0004#define Z_EROFS_ADVISE_INLINE_PCLUSTER 0x0008 +#define Z_EROFS_ADVISE_FRAGMENT_PCLUSTER 0x0010 +#define Z_EROFS_FRAGMENT_INODE_BIT 7struct z_erofs_map_header { -__le16 h_reserved1; -/* record the size of tailpacking data */ -__le16 h_idata_size; +union { +/* direct addressing for fragment offset */ +__le32 h_fragmentoff; +struct { +__le16 h_reserved1; +/* record the size of tailpacking data */ +__le16 h_idata_size; + }; + };__le16 h_advise;/* * bit 0-3 : algorithm type of head 1 (logical cluster type 01);
@@ -312,7 +325,8 @@ struct z_erofs_map_header {__u8 h_algorithmtype;/* * bit 0-2 : logical cluster bits - 12, e.g. 0 for 4096; -* bit 3-7 : reserved. +* bit 3-6 : reserved; +* bit 7 : move the whole file into packed inode or not. */ __u8 h_clusterbits; };
lib/data.c +20 -0
@@ -275,6 +275,26 @@ static int z_erofs_read_data(struct erofs_inode *inode, char *buffer,continue ; } +if (map.m_flags &EROFS_MAP_FRAGMENT ) { +char *out; + + out =malloc (length - skip); +if (!out) { + ret = -ENOMEM ; +break ; + } + ret =z_erofs_read_data (sbi.packed_inode, out, + length - skip, + inode->fragmentoff + skip); +if (ret <0 ) { +free (out); +break ; + } +memcpy (buffer + end - offset, out, length - skip); +free (out); +continue ; + } +if (map.m_plen > bufsize) { bufsize = map.m_plen; raw =realloc (raw, bufsize);
lib/super.c +23 -1
@@ -104,6 +104,21 @@ int erofs_read_superblock(void) sbi.xattr_blkaddr =le32_to_cpu (dsb->xattr_blkaddr); sbi.islotbits =EROFS_ISLOTBITS ; sbi.root_nid =le16_to_cpu (dsb->root_nid); + sbi.packed_inode =NULL ; +if (erofs_sb_has_fragments ()) { +struct erofs_inode *inode; + + inode =calloc (1 ,sizeof (struct erofs_inode )); +if (!inode) +return -ENOMEM ; + inode->nid =le64_to_cpu (dsb->packed_nid); + ret =erofs_read_inode_from_disk (inode); +if (ret) { +free (inode); +return ret; + } + sbi.packed_inode = inode; + } sbi.inos =le64_to_cpu (dsb->inos); sbi.checksum =le32_to_cpu (dsb->checksum);
@@ -111,11 +126,18 @@ int erofs_read_superblock(void) sbi.build_time_nsec =le32_to_cpu (dsb->build_time_nsec);memcpy (&sbi.uuid, dsb->uuid,sizeof (dsb->uuid)); -return erofs_init_devices (&sbi, dsb); + + ret =erofs_init_devices (&sbi, dsb); +if (ret && sbi.packed_inode) +free (sbi.packed_inode); +return ret; }void erofs_put_super (void ) {if (sbi.devs)free (sbi.devs); + +if (sbi.packed_inode) +free (sbi.packed_inode); }
lib/zmap.c +26 -0
@@ -83,6 +83,20 @@ static int z_erofs_fill_inode_lazy(struct erofs_inode *vi)if (ret <0 )return ret; } +if (vi->z_advise &Z_EROFS_ADVISE_FRAGMENT_PCLUSTER ) { + vi->fragmentoff =le32_to_cpu (h->h_fragmentoff); + +if (h->h_clusterbits >>Z_EROFS_FRAGMENT_INODE_BIT ) { + vi->z_tailextent_headlcn =0 ; + }else { +struct erofs_map_blocks map = { .index =UINT_MAX }; + + ret =z_erofs_do_map_blocks (vi, &map, +EROFS_GET_BLOCKS_FINDTAIL ); +if (ret <0 ) +return ret; + } + } vi->flags |=EROFS_I_Z_INITED ;return 0 ; }
@@ -546,6 +560,7 @@ static int z_erofs_do_map_blocks(struct erofs_inode *vi, int flags) {bool ztailpacking = vi->z_advise &Z_EROFS_ADVISE_INLINE_PCLUSTER ; +bool inpacked = vi->z_advise &Z_EROFS_ADVISE_FRAGMENT_PCLUSTER ;struct z_erofs_maprecorder m = { .inode = vi, .map = map,
@@ -609,6 +624,9 @@ static int z_erofs_do_map_blocks(struct erofs_inode *vi, map->m_flags |=EROFS_MAP_META ; map->m_pa = vi->z_idataoff; map->m_plen = vi->z_idata_size; + }else if (inpacked && m.lcn == vi->z_tailextent_headlcn) { + map->m_flags |=EROFS_MAP_FRAGMENT ; +DBG_BUGON (!map->m_la); }else { map->m_pa =blknr_to_addr (m.pblk); err =z_erofs_get_extent_compressedlen (&m, initial_lcn);
@@ -652,6 +670,14 @@ int z_erofs_map_blocks_iter(struct erofs_inode *vi,if (err) goto out; +if ((vi->z_advise &Z_EROFS_ADVISE_FRAGMENT_PCLUSTER ) && + !vi->z_tailextent_headlcn) { + map->m_llen = map->m_la +1 ; + map->m_la =0 ; + map->m_flags =EROFS_MAP_MAPPED |EROFS_MAP_FRAGMENT ; + goto out; + } + err =z_erofs_do_map_blocks (vi, map, flags); out:DBG_BUGON (err <0 && err != -ENOMEM );
Signature · 1 line
2.17.1
RFC PATCH v4 2/3 erofs-utils: lib: support on-disk offset for shifted decompression
In reply to [email protected]From: Yue Hu <[email protected]>
Add support to uncompressed data layout with on-disk offset for compressed files.
--- include/erofs/decompress.h | 3 +++ lib/data.c | 8 +++++++- lib/decompress.c | 10 ++++++++-- 3 files changed, 18 insertions(+), 3 deletions(-)
include/erofs/decompress.h +3 -0
@@ -23,6 +23,9 @@ struct z_erofs_decompress_req {unsigned int decodedskip;unsigned int inputsize, decodedlength; +/* head offset of uncompressed data */ +unsigned int shiftedhead; +/* indicate the algorithm will be used for decompression */ unsigned int alg;bool partial_decoding;
lib/data.c +7 -1
@@ -226,7 +226,7 @@ static int z_erofs_read_data(struct erofs_inode *inode, char *buffer, };struct erofs_map_dev mdev;bool partial; -unsigned int bufsize =0 ; +unsigned int bufsize =0 , head;char *raw =NULL ;int ret =0 ;
@@ -307,10 +307,16 @@ static int z_erofs_read_data(struct erofs_inode *inode, char *buffer,if (ret <0 )break ; + head =0 ; +if (erofs_sb_has_fragments () && + map.m_algorithmformat ==Z_EROFS_COMPRESSION_SHIFTED ) + head =erofs_blkoff (end); + ret = z_erofs_decompress(&(struct z_erofs_decompress_req) { .in = raw, .out = buffer + end - offset, .decodedskip = skip, + .shiftedhead = head, .inputsize = map.m_plen, .decodedlength = length, .alg = map.m_algorithmformat,
lib/decompress.c +8 -2
@@ -132,14 +132,20 @@ out:int z_erofs_decompress (struct z_erofs_decompress_req *rq) {if (rq->alg ==Z_EROFS_COMPRESSION_SHIFTED ) { +unsigned int count, rightpart; +if (rq->inputsize >EROFS_BLKSIZ )return -EFSCORRUPTED ;DBG_BUGON (rq->decodedlength >EROFS_BLKSIZ );DBG_BUGON (rq->decodedlength < rq->decodedskip); -memcpy (rq->out, rq->in + rq->decodedskip, - rq->decodedlength - rq->decodedskip); + count = rq->decodedlength - rq->decodedskip; + rightpart =min (EROFS_BLKSIZ - rq->shiftedhead, count); + +memcpy (rq->out, rq->in + (erofs_sb_has_fragments () ? + rq->shiftedhead : rq->decodedskip), rightpart); +memcpy (rq->out + rightpart, rq->in, count - rightpart);return 0 ; }
Signature · 1 line
2.17.1
Re: [RFC PATCH v4 2/3] erofs-utils: lib: support on-disk offset for shifted decompression
Hi Yue,
Quoted 4 lines · On Sun, Aug 21, 2022 at 09:57:24PM +0800, [email protected] wrote:
From: Yue Hu <[email protected]>
Add support to uncompressed data layout with on-disk offset for compressed files.
Sorry for some delay.
Add interlaced uncompressed data layout for compressed files.
Quoted 13 lines
Signed-off-by Yue Hu <[email protected]>--- include/erofs/decompress.h | 3 +++ lib/data.c | 8 +++++++- lib/decompress.c | 10 ++++++++-- 3 files changed, 18 insertions(+), 3 deletions(-)include/erofs/decompress.h +0
-0
include/erofs/decompress.h struct z_erofs_decompress_req { · line 27
Quoted 5 lines
include/erofs/decompress.h +2
-0@@ -23,6 +23,9 @@ struct z_erofs_decompress_req {unsigned int decodedskip;unsigned int inputsize, decodedlength; +/* head offset of uncompressed data */ +unsigned int shiftedhead;
/* cut point of interlaced uncompressed data */ unsigned int interlaced_offset;
Quoted 18 lines
include/erofs/decompress.h +1
-0+/* indicate the algorithm will be used for decompression */ unsigned int alg;bool partial_decoding;lib/data.c +1
-1@@ -226,7 +226,7 @@ static int z_erofs_read_data(struct erofs_inode *inode, char *buffer, };struct erofs_map_dev mdev;bool partial; -unsigned int bufsize =0 ; +unsigned int bufsize =0 , head;char *raw =NULL ;int ret =0 ;
lib/data.c static int z_erofs_read_data(struct erofs_inode *inode, char *buffer, · line 311
Quoted 5 lines
lib/data.c +2
-0@@ -307,10 +307,16 @@ static int z_erofs_read_data(struct erofs_inode *inode, char *buffer,if (ret <0 )break ; + head =0 ; + if (erofs_sb_has_fragments() &&
Can we add another Z_EROFS_ADVISE_ for this, such as Z_EROFS_ADVISE_INTERLACED_UNCOMPRESSED_DATA ?
Quoted 23 lines
lib/data.c +4
-0+ map.m_algorithmformat == Z_EROFS_COMPRESSION_SHIFTED) + head =erofs_blkoff (end); + ret = z_erofs_decompress(&(struct z_erofs_decompress_req) { .in = raw, .out = buffer + end - offset, .decodedskip = skip, + .shiftedhead = head, .inputsize = map.m_plen, .decodedlength = length, .alg = map.m_algorithmformat,lib/decompress.c +2
-0@@ -132,14 +132,20 @@ out:int z_erofs_decompress (struct z_erofs_decompress_req *rq) {if (rq->alg ==Z_EROFS_COMPRESSION_SHIFTED ) { +unsigned int count, rightpart; +if (rq->inputsize >EROFS_BLKSIZ )return -EFSCORRUPTED ;
lib/decompress.c out: · line 146
Quoted 10 lines
lib/decompress.c +4
-2DBG_BUGON (rq->decodedlength >EROFS_BLKSIZ );DBG_BUGON (rq->decodedlength < rq->decodedskip); -memcpy (rq->out, rq->in + rq->decodedskip, - rq->decodedlength - rq->decodedskip); + count = rq->decodedlength - rq->decodedskip; + rightpart =min (EROFS_BLKSIZ - rq->shiftedhead, count); + + memcpy(rq->out, rq->in + (erofs_sb_has_fragments() ?
same here.
Thanks,
Gao Xiang
Quoted 7 lines
lib/decompress.c +2
-0+ rq->shiftedhead : rq->decodedskip), rightpart); +memcpy (rq->out + rightpart, rq->in, count - rightpart);return 0 ; }Signature · 1 line
2.17.1
Re: [RFC PATCH v4 2/3] erofs-utils: lib: support on-disk offset for shifted decompression
Quoted 55 lines · On Wed, 24 Aug 2022 16:45:59 +0800 Gao Xiang <[email protected]>…
Hi Yue,
Quoted 4 lines · On Sun, Aug 21, 2022 at 09:57:24PM +0800, [email protected] wrote:
From: Yue Hu <[email protected]>
Add support to uncompressed data layout with on-disk offset for compressed files.
Sorry for some delay.
Add interlaced uncompressed data layout for compressed files.
Quoted 13 lines
Signed-off-by Yue Hu <[email protected]>--- include/erofs/decompress.h | 3 +++ lib/data.c | 8 +++++++- lib/decompress.c | 10 ++++++++-- 3 files changed, 18 insertions(+), 3 deletions(-)include/erofs/decompress.h +0
-0include/erofs/decompress.h struct z_erofs_decompress_req { · line 27
Quoted 5 lines
include/erofs/decompress.h +2
-0@@ -23,6 +23,9 @@ struct z_erofs_decompress_req {unsigned int decodedskip;unsigned int inputsize, decodedlength; +/* head offset of uncompressed data */ +unsigned int shiftedhead;/* cut point of interlaced uncompressed data */ unsigned int interlaced_offset;Quoted 21 lines
include/erofs/decompress.h +1
-0+/* indicate the algorithm will be used for decompression */ unsigned int alg;bool partial_decoding;lib/data.c +1
-1@@ -226,7 +226,7 @@ static int z_erofs_read_data(struct erofs_inode *inode, char *buffer, };struct erofs_map_dev mdev;bool partial; -unsigned int bufsize =0 ; +unsigned int bufsize =0 , head;char *raw =NULL ;int ret =0 ;@@ -307,10 +307,16 @@ static int z_erofs_read_data(struct erofs_inode *inode, char *buffer,if (ret <0 )break ;
Quoted 5 lines
lib/data.c static int z_erofs_read_data(struct erofs_inode *inode, char *buffer, · line 311
Quoted 2 lines
lib/data.c +2
-0+ head =0 ; + if (erofs_sb_has_fragments() &&Can we add another
Z_EROFS_ADVISE_for this, such asZ_EROFS_ADVISE_INTERLACED_UNCOMPRESSED_DATA?
Sounds good, let's do this.
Thanks.
Quoted 47 lines
Quoted 24 lines
lib/data.c +4
-0+ map.m_algorithmformat == Z_EROFS_COMPRESSION_SHIFTED) + head =erofs_blkoff (end); + ret = z_erofs_decompress(&(struct z_erofs_decompress_req) { .in = raw, .out = buffer + end - offset, .decodedskip = skip, + .shiftedhead = head, .inputsize = map.m_plen, .decodedlength = length, .alg = map.m_algorithmformat,lib/decompress.c +2
-0@@ -132,14 +132,20 @@ out:int z_erofs_decompress (struct z_erofs_decompress_req *rq) {if (rq->alg ==Z_EROFS_COMPRESSION_SHIFTED ) { +unsigned int count, rightpart; +if (rq->inputsize >EROFS_BLKSIZ )return -EFSCORRUPTED ;lib/decompress.c out: · line 146
Quoted 10 lines
lib/decompress.c +4
-2DBG_BUGON (rq->decodedlength >EROFS_BLKSIZ );DBG_BUGON (rq->decodedlength < rq->decodedskip); -memcpy (rq->out, rq->in + rq->decodedskip, - rq->decodedlength - rq->decodedskip); + count = rq->decodedlength - rq->decodedskip; + rightpart =min (EROFS_BLKSIZ - rq->shiftedhead, count); + + memcpy(rq->out, rq->in + (erofs_sb_has_fragments() ?same here.
Thanks,
Gao XiangQuoted 7 lines
lib/decompress.c +2
-0+ rq->shiftedhead : rq->decodedskip), rightpart); +memcpy (rq->out + rightpart, rq->in, count - rightpart);return 0 ; }Signature · 1 line
2.17.1
RFC PATCH v4 3/3 erofs-utils: introduce compressed fragments support
In reply to [email protected]From: Yue Hu <[email protected]>
This approach can merge tail pclusters or the whole files into a special inode in order to achieve greater compression ratio. And an option of pcluster size is provided for different compression requirments.
Meanwhile, we change to write the uncompressed data from 'clusterofs' when compressing files since it can benefit from in-place I/O. For now, this change goes with the fragments.
--- include/erofs/compress.h | 3 +- include/erofs/config.h | 3 +- include/erofs/fragments.h | 25 +++++++++ include/erofs/inode.h | 1 + include/erofs/internal.h | 1 + include/erofs_fs.h | 1 + lib/Makefile.am | 4 +- lib/compress.c | 108 ++++++++++++++++++++++++++++---------- lib/fragments.c | 58 ++++++++++++++++++++ lib/inode.c | 59 ++++++++++++++++----- mkfs/main.c | 64 +++++++++++++++++++--- 11 files changed, 277 insertions(+), 50 deletions(-) create mode 100644 include/erofs/fragments.h create mode 100644 lib/fragments.c
include/erofs/compress.h +2 -1
@@ -18,7 +18,8 @@ extern "C"#define EROFS_CONFIG_COMPR_MIN_SZ (32 * 1024)void z_erofs_drop_inline_pcluster (struct erofs_inode *inode); -int erofs_write_compressed_file (struct erofs_inode *inode); +int erofs_write_compressed_file_from_fd (struct erofs_inode *inode,int fd, +bool is_src);int z_erofs_compress_init (struct erofs_buffer_head *bh);int z_erofs_compress_exit (void );
include/erofs/config.h +2 -1
@@ -44,6 +44,7 @@ struct erofs_configure {char c_chunkbits;bool c_noinline_data;bool c_ztailpacking; +bool c_fragments;bool c_ignore_mtime;bool c_showprogress;
@@ -62,7 +63,7 @@ struct erofs_configure {/* < 0, xattr disabled and INT_MAX, always use inline xattrs */ int c_inline_xattr_tolerance; -u32 c_pclusterblks_max, c_pclusterblks_def; +u32 c_pclusterblks_max, c_pclusterblks_def, c_pclusterblks_packed;u32 c_max_decompressed_extent_bytes;u32 c_dict_size;u64 c_unix_timestamp;
include/erofs/fragments.h new file +25 -0
@@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ +/* +* Copyright (C), 2022, Coolpad Group Limited. +*/ +#ifndef __EROFS_FRAGMENTS_H +#define __EROFS_FRAGMENTS_H + +#ifdef __cplusplus +extern "C" +{ +#endif + +#include "erofs/internal.h" + +int z_erofs_pack_fragments (struct erofs_inode *inode,void *data, +unsigned int len); +struct erofs_inode *erofs_mkfs_build_fragments (void ); +int erofs_fragments_init (void ); +void erofs_fragments_exit (void ); + +#ifdef __cplusplus +} +#endif + +#endif
include/erofs/inode.h +1 -0
@@ -22,6 +22,7 @@ unsigned int erofs_iput(struct erofs_inode *inode);erofs_nid_t erofs_lookupnid (struct erofs_inode *inode);struct erofs_inode *erofs_mkfs_build_tree_from_path (struct erofs_inode *parent,const char *path); +struct erofs_inode *erofs_mkfs_build_special_from_fd (int fd,const char *name); #ifdef __cplusplus }
include/erofs/internal.h +1 -0
@@ -212,6 +212,7 @@ struct erofs_inode {uint64_t capabilities;#endif erofs_off_t fragmentoff; +unsigned int fragment_size; };static inline bool is_inode_layout_compression (struct erofs_inode *inode)
include/erofs_fs.h +1 -0
@@ -267,6 +267,7 @@ struct erofs_inode_chunk_index {/* maximum supported size of a physical compression cluster */ #define Z_EROFS_PCLUSTER_MAX_SIZE (1024 * 1024) +#define Z_EROFS_PCLUSTER_MAX_BLKS (Z_EROFS_PCLUSTER_MAX_SIZE / EROFS_BLKSIZ)/* available compression algorithm types (for h_algorithmtype) */ enum {
lib/Makefile.am +3 -1
@@ -22,12 +22,14 @@ noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \ $(top_srcdir)/include/erofs/trace.h \ $(top_srcdir)/include/erofs/xattr.h \ $(top_srcdir)/include/erofs/compress_hints.h \ + $(top_srcdir)/include/erofs/fragments.h \ $(top_srcdir)/lib/liberofs_private.h noinst_HEADERS += compressor.h liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \ namei.c data.c compress.c compressor.c zmap.c decompress.c \ - compress_hints.c hashmap.c sha256.c blobchunk.c dir.c + compress_hints.c hashmap.c sha256.c blobchunk.c dir.c \ + fragments.c liberofs_la_CFLAGS = -Wall -I$(top_srcdir)/include if ENABLE_LZ4 liberofs_la_CFLAGS += ${LZ4_CFLAGS}
lib/compress.c +79 -29
251 lines not on this page · raw message
lib/fragments.c new file +58 -0
58 lines not on this page · raw message
lib/inode.c +47 -12
80 lines not on this page · raw message
mkfs/main.c +58 -6
120 lines not on this page · raw message
Signature · 1 line
2.17.1
Re: [RFC PATCH v4 3/3] erofs-utils: introduce compressed fragments support
Quoted 162 lines · On Sun, Aug 21, 2022 at 09:57:25PM +0800, [email protected] wrote:
From: Yue Hu <[email protected]>
This approach can merge tail pclusters or the whole files into a special inode in order to achieve greater compression ratio. And an option of pcluster size is provided for different compression requirments.
Meanwhile, we change to write the uncompressed data from 'clusterofs' when compressing files since it can benefit from in-place I/O. For now, this change goes with the fragments.
Signed-off-by Yue Hu <[email protected]>--- include/erofs/compress.h | 3 +- include/erofs/config.h | 3 +- include/erofs/fragments.h | 25 +++++++++ include/erofs/inode.h | 1 + include/erofs/internal.h | 1 + include/erofs_fs.h | 1 + lib/Makefile.am | 4 +- lib/compress.c | 108 ++++++++++++++++++++++++++++---------- lib/fragments.c | 58 ++++++++++++++++++++ lib/inode.c | 59 ++++++++++++++++----- mkfs/main.c | 64 +++++++++++++++++++--- 11 files changed, 277 insertions(+), 50 deletions(-) create mode 100644 include/erofs/fragments.h create mode 100644 lib/fragments.cinclude/erofs/compress.h +2
-1@@ -18,7 +18,8 @@ extern "C"#define EROFS_CONFIG_COMPR_MIN_SZ (32 * 1024)void z_erofs_drop_inline_pcluster (struct erofs_inode *inode); -int erofs_write_compressed_file (struct erofs_inode *inode); +int erofs_write_compressed_file_from_fd (struct erofs_inode *inode,int fd, +bool is_src);int z_erofs_compress_init (struct erofs_buffer_head *bh);int z_erofs_compress_exit (void );include/erofs/config.h +2
-1@@ -44,6 +44,7 @@ struct erofs_configure {char c_chunkbits;bool c_noinline_data;bool c_ztailpacking; +bool c_fragments;bool c_ignore_mtime;bool c_showprogress;@@ -62,7 +63,7 @@ struct erofs_configure {/* < 0, xattr disabled and INT_MAX, always use inline xattrs */ int c_inline_xattr_tolerance; -u32 c_pclusterblks_max, c_pclusterblks_def; +u32 c_pclusterblks_max, c_pclusterblks_def, c_pclusterblks_packed;u32 c_max_decompressed_extent_bytes;u32 c_dict_size;u64 c_unix_timestamp;include/erofs/fragments.h new file +25
-0@@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ +/* +* Copyright (C), 2022, Coolpad Group Limited. +*/ +#ifndef __EROFS_FRAGMENTS_H +#define __EROFS_FRAGMENTS_H + +#ifdef __cplusplus +extern "C" +{ +#endif + +#include "erofs/internal.h" + +int z_erofs_pack_fragments (struct erofs_inode *inode,void *data, +unsigned int len); +struct erofs_inode *erofs_mkfs_build_fragments (void ); +int erofs_fragments_init (void ); +void erofs_fragments_exit (void ); + +#ifdef __cplusplus +} +#endif + +#endif include/erofs/inode.h +1
-0@@ -22,6 +22,7 @@ unsigned int erofs_iput(struct erofs_inode *inode);erofs_nid_t erofs_lookupnid (struct erofs_inode *inode);struct erofs_inode *erofs_mkfs_build_tree_from_path (struct erofs_inode *parent,const char *path); +struct erofs_inode *erofs_mkfs_build_special_from_fd (int fd,const char *name); #ifdef __cplusplus }include/erofs/internal.h +1
-0@@ -212,6 +212,7 @@ struct erofs_inode {uint64_t capabilities;#endif erofs_off_t fragmentoff; +unsigned int fragment_size; };static inline bool is_inode_layout_compression (struct erofs_inode *inode)include/erofs_fs.h +1
-0@@ -267,6 +267,7 @@ struct erofs_inode_chunk_index {/* maximum supported size of a physical compression cluster */ #define Z_EROFS_PCLUSTER_MAX_SIZE (1024 * 1024) +#define Z_EROFS_PCLUSTER_MAX_BLKS (Z_EROFS_PCLUSTER_MAX_SIZE / EROFS_BLKSIZ)/* available compression algorithm types (for h_algorithmtype) */ enum {lib/Makefile.am +3
-1@@ -22,12 +22,14 @@ noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \ $(top_srcdir)/include/erofs/trace.h \ $(top_srcdir)/include/erofs/xattr.h \ $(top_srcdir)/include/erofs/compress_hints.h \ + $(top_srcdir)/include/erofs/fragments.h \ $(top_srcdir)/lib/liberofs_private.h noinst_HEADERS += compressor.h liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \ namei.c data.c compress.c compressor.c zmap.c decompress.c \ - compress_hints.c hashmap.c sha256.c blobchunk.c dir.c + compress_hints.c hashmap.c sha256.c blobchunk.c dir.c \ + fragments.c liberofs_la_CFLAGS = -Wall -I$(top_srcdir)/include if ENABLE_LZ4 liberofs_la_CFLAGS += ${LZ4_CFLAGS}lib/compress.c +1
-0@@ -18,6 +18,7 @@#include "compressor.h" #include "erofs/block_list.h" #include "erofs/compress_hints.h" +#include "erofs/fragments.h" static struct erofs_compress compresshandle;static unsigned int algorithmtype[2 ];
lib/compress.c static void vle_write_indexes(struct z_erofs_vle_compress_ctx *ctx, · line 80
Quoted 8 lines
lib/compress.c +2
-2@@ -74,9 +75,9 @@ static void vle_write_indexes(struct z_erofs_vle_compress_ctx *ctx,if (!d1) {/* * A lcluster cannot have three parts with the middle one which -* is well-compressed for !ztailpacking cases. +* is well-compressed for !ztailpacking and !fragments cases. */ -DBG_BUGON (!raw && !cfg.c_ztailpacking); +DBG_BUGON (!raw && !cfg.c_ztailpacking && !cfg.c_fragments);
This should be per-inode?
Quoted 16 lines
lib/compress.c +1
-1type = raw ?Z_EROFS_VLE_CLUSTER_TYPE_PLAIN :Z_EROFS_VLE_CLUSTER_TYPE_HEAD ; advise =cpu_to_le16 (type <<Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT );@@ -143,7 +144,7 @@ static int write_uncompressed_extent(struct z_erofs_vle_compress_ctx *ctx, unsigned int *len, char *dst) { int ret; -unsigned int count; +unsigned int count, offset, rcopied, rzeroed;/* reset clusterofs to 0 if permitted */ if (!erofs_sb_has_lz4_0padding() && ctx->clusterofs &&@@ -153,11 +154,21 @@ static int write_uncompressed_extent(struct z_erofs_vle_compress_ctx *ctx, ctx->clusterofs =0 ; }
lib/compress.c static int write_uncompressed_extent(struct z_erofs_vle_compress_ctx *ctx, · line 163
Quoted 10 lines
lib/compress.c +5
-3-/* write uncompressed data */ +/* +* write uncompressed data from clusterofs which can benefit from +* in-place I/O, loop shift right when to exceed EROFS_BLKSIZ. +*/ count =min (EROFS_BLKSIZ , *len); -memcpy (dst, ctx->queue + ctx->head, count); -memset (dst + count,0 ,EROFS_BLKSIZ - count); + offset = cfg.c_fragments ? ctx->clusterofs :0 ;
interlaced_offset;
Quoted 12 lines
lib/compress.c +8
-0+ rcopied =min (EROFS_BLKSIZ - offset, count); + rzeroed =EROFS_BLKSIZ - offset - rcopied; + +memcpy (dst + offset, ctx->queue + ctx->head, rcopied); +memcpy (dst, ctx->queue + ctx->head + rcopied, count - rcopied); + +memset (dst + offset + rcopied,0 , rzeroed); +memset (dst + count - rcopied,0 ,EROFS_BLKSIZ - count - rzeroed);erofs_dbg ("Writing %u uncompressed data to block %u" , count, ctx->blkaddr);
lib/compress.c static int write_uncompressed_extent(struct z_erofs_vle_compress_ctx *ctx, · line 182
Quoted 6 lines
lib/compress.c +2
-1@@ -167,8 +178,11 @@ static int write_uncompressed_extent(struct z_erofs_vle_compress_ctx *ctx,return count; } -static unsigned int z_erofs_get_max_pclusterblks (struct erofs_inode *inode) +static unsigned int z_erofs_get_max_pclusterblks (struct erofs_inode *inode, +bool is_src)
is_src should be in erofs_inode as, for example, bool is_packed_inode, or other likewise rather than passing another argument here.
Quoted 80 lines
lib/compress.c +31
-4{ +if (cfg.c_fragments && !is_src) +return cfg.c_pclusterblks_packed;#ifndef NDEBUG if (cfg.c_random_pclusterblks)return 1 +rand () % cfg.c_pclusterblks_max;@@ -224,7 +238,7 @@ static void tryrecompress_trailing(void *in, unsigned int *insize,static int vle_compress_one (struct erofs_inode *inode,struct z_erofs_vle_compress_ctx *ctx, -bool final) +bool final,bool is_src) {struct erofs_compress *const h = &compresshandle;unsigned int len = ctx->tail - ctx->head;@@ -234,14 +248,19 @@ static int vle_compress_one(struct erofs_inode *inode,char *const dst = dstbuf +EROFS_BLKSIZ ;while (len) { -unsigned int pclustersize = -z_erofs_get_max_pclusterblks (inode) *EROFS_BLKSIZ ; +unsigned int pclustersize =EROFS_BLKSIZ * +z_erofs_get_max_pclusterblks (inode, is_src);bool may_inline = (cfg.c_ztailpacking && final); +bool may_packing = (cfg.c_fragments && final && is_src);bool raw;if (len <= pclustersize) {if (!final)break ; +if (may_packing) { + count = len; + goto frag_packing; + }if (!may_inline && len <=EROFS_BLKSIZ ) goto nocompression; }@@ -294,6 +313,19 @@ nocompression:return ret; ctx->compressedblks =1 ; raw = false; + }else if (may_packing && len == count && ret < pclustersize) { +frag_packing: + ret =z_erofs_pack_fragments (inode, + ctx->queue + ctx->head, + len); +if (ret <0 ) +return ret; +if (inode->i_size == inode->fragment_size) { + ctx->head += len; +return 0 ; + } + ctx->compressedblks =0 ; + raw = false; }else {unsigned int tailused, padding;@@ -546,13 +578,20 @@ static void z_erofs_write_mapheader(struct erofs_inode *inode, {struct z_erofs_map_header h = { .h_advise =cpu_to_le16 (inode->z_advise), - .h_idata_size =cpu_to_le16 (inode->idata_size), .h_algorithmtype = inode->z_algorithmtype[1 ] <<4 | inode->z_algorithmtype[0 ],/* lclustersize */ .h_clusterbits = inode->z_logical_clusterbits -12 , }; +if (cfg.c_fragments) + h.h_fragmentoff =cpu_to_le32 (inode->fragmentoff); +else + h.h_idata_size =cpu_to_le16 (inode->idata_size); + +if (inode->fragment_size && inode->i_size == inode->fragment_size) + h.h_clusterbits |=1 <<Z_EROFS_FRAGMENT_INODE_BIT ; +memset (compressmeta,0 ,Z_EROFS_LEGACY_MAP_HEADER_SIZE );/* write out map header */ memcpy (compressmeta, &h,sizeof (struct z_erofs_map_header ));
lib/compress.c void z_erofs_drop_inline_pcluster(struct erofs_inode *inode) · line 648
Quoted 6 lines
lib/compress.c +2
-1@@ -605,30 +644,25 @@ void z_erofs_drop_inline_pcluster(struct erofs_inode *inode) inode->eof_tailraw =NULL ; } -int erofs_write_compressed_file (struct erofs_inode *inode) +int erofs_write_compressed_file_from_fd (struct erofs_inode *inode,int fd, +bool is_src)
same here.
Quoted 92 lines
lib/compress.c +12
-15{struct erofs_buffer_head *bh;static struct z_erofs_vle_compress_ctx ctx;erofs_off_t remaining;erofs_blk_t blkaddr, compressed_blocks;unsigned int legacymetasize; -int ret, fd; +int ret;u8 *compressmeta =malloc (vle_compressmeta_capacity (inode->i_size));if (!compressmeta)return -ENOMEM ; - fd =open (inode->i_srcpath,O_RDONLY |O_BINARY ); -if (fd <0 ) { - ret = -errno; - goto err_free_meta; - } -/* allocate main data buffer */ bh =erofs_balloc (DATA ,0 ,0 ,0 );if (IS_ERR (bh)) { ret =PTR_ERR (bh); - goto err_close; + goto err_free_meta; }/* initialize per-file compression setting */ @@ -649,6 +683,9 @@ int erofs_write_compressed_file(struct erofs_inode *inode) inode->z_algorithmtype[1 ] = algorithmtype[1 ]; inode->z_logical_clusterbits =LOG_BLOCK_SIZE ; + inode->idata_size =0 ; + inode->fragment_size =0 ; + blkaddr =erofs_mapbh (bh->block);/* start_blkaddr */ ctx.blkaddr = blkaddr; ctx.metacur = compressmeta +Z_EROFS_LEGACY_MAP_HEADER_SIZE ;@@ -668,7 +705,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode) remaining -= readcount; ctx.tail += readcount; - ret =vle_compress_one (inode, &ctx, !remaining); + ret =vle_compress_one (inode, &ctx, !remaining, is_src);if (ret) goto err_free_idata; }@@ -682,19 +719,20 @@ int erofs_write_compressed_file(struct erofs_inode *inode)vle_write_indexes_final (&ctx); legacymetasize = ctx.metacur - compressmeta;/* estimate if data compression saves space or not */ -if (compressed_blocks *EROFS_BLKSIZ + inode->idata_size + +if (!inode->fragment_size && + compressed_blocks *EROFS_BLKSIZ + inode->idata_size + legacymetasize >= inode->i_size) { ret = -ENOSPC ; goto err_free_idata; }z_erofs_write_mapheader (inode, compressmeta); -close (fd);if (compressed_blocks) { ret =erofs_bh_balloon (bh,blknr_to_addr (compressed_blocks));DBG_BUGON (ret !=EROFS_BLKSIZ ); }else { -DBG_BUGON (!inode->idata_size); +if (!cfg.c_fragments) +DBG_BUGON (!inode->idata_size); } erofs_info("compressed %s (%llu bytes) into %u blocks",@@ -717,7 +755,8 @@ int erofs_write_compressed_file(struct erofs_inode *inode)DBG_BUGON (ret); } inode->compressmeta = compressmeta; -erofs_droid_blocklist_write (inode, blkaddr, compressed_blocks); +if (is_src) +erofs_droid_blocklist_write (inode, blkaddr, compressed_blocks);return 0 ; err_free_idata:@@ -727,8 +766,6 @@ err_free_idata: } err_bdrop:erofs_bdrop (bh, true);/* revoke buffer */ -err_close: -close (fd); err_free_meta:free (compressmeta);return ret;@@ -834,14 +871,27 @@ int z_erofs_compress_init(struct erofs_buffer_head *sb_bh)* to be loaded in order to get those compressed block counts.
lib/compress.c int z_erofs_compress_init(struct erofs_buffer_head *sb_bh) · line 879
Quoted 10 lines
lib/compress.c +2
-2*/ if (cfg.c_pclusterblks_max >1 ) { -if (cfg.c_pclusterblks_max > -Z_EROFS_PCLUSTER_MAX_SIZE /EROFS_BLKSIZ ) { +if (cfg.c_pclusterblks_max >Z_EROFS_PCLUSTER_MAX_BLKS ) {erofs_err ("unsupported clusterblks %u (too large)" , cfg.c_pclusterblks_max);return -EINVAL ; } +if (cfg.c_pclusterblks_packed >Z_EROFS_PCLUSTER_MAX_BLKS ) {
c_pclusterblks_packed should be smaller than c_pclusterblks_max
lib/compress.c int z_erofs_compress_init(struct erofs_buffer_head *sb_bh) · line 887
Quoted 8 lines
lib/compress.c +8
-0+erofs_err ("unsupported clusterblks %u (too large for fragments)" , + cfg.c_pclusterblks_packed); +return -EINVAL ; + } +if (cfg.c_pclusterblks_packed ==1 ) { +erofs_err ("physical cluster size of fragments should > 4096 bytes" ); +return -EINVAL ; + }
How can this happen? why judging this here?
lib/compress.c int z_erofs_compress_init(struct erofs_buffer_head *sb_bh) · line 894
Quoted 7 lines
lib/compress.c +5
-0erofs_sb_set_big_pcluster (); } +if (!erofs_sb_has_big_pcluster () && cfg.c_pclusterblks_packed >1 ) { +erofs_err ("invalid clusterblks %u (for fragments)" , + cfg.c_pclusterblks_packed); +return -EINVAL ; + }
The only condition I think would be
if (c_pclusterblks_packed > cfg.c_pclusterblks_max) {
erofs_err("invalid physical cluster size for the packed file");
}Quoted 85 lines
lib/compress.c +0
-0if (ret !=Z_EROFS_COMPRESSION_LZ4 )erofs_sb_set_compr_cfgs ();lib/fragments.c new file +58
-0@@ -0,0 +1,58 @@ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 +/* +* Copyright (C), 2022, Coolpad Group Limited. +* Created by Yue Hu <[email protected]> +*/ +#define _GNU_SOURCE +#include <stdlib.h> +#include <unistd.h> +#include "erofs/err.h" +#include "erofs/inode.h" +#include "erofs/compress.h" +#include "erofs/print.h" +#include "erofs/fragments.h" + +static FILE *packedfile; + +int z_erofs_pack_fragments (struct erofs_inode *inode,void *data, +unsigned int len) +{ + inode->z_advise |=Z_EROFS_ADVISE_FRAGMENT_PCLUSTER ; + inode->fragmentoff =ftell (packedfile); + inode->fragment_size = len; + +if (write (fileno (packedfile), data, len) <0 ) +return -EIO ; + +erofs_sb_set_fragments (); + +erofs_dbg ("Recording %u fragment data at %lu" , inode->fragment_size, + inode->fragmentoff); +return len; +} + +struct erofs_inode *erofs_mkfs_build_fragments (void ) +{ +fseek (packedfile,0 ,SEEK_SET ); + +return erofs_mkfs_build_special_from_fd (fileno (packedfile), +"packed_file" ); +} + +void erofs_fragments_exit (void ) +{ +if (packedfile) +fclose (packedfile); +} + +int erofs_fragments_init (void ) +{ +#ifdef HAVE_TMPFILE64 + packedfile =tmpfile64 (); +#else + packedfile =tmpfile (); +#endif +if (!packedfile) +return -ENOMEM ; +return 0 ; +}lib/inode.c +5
-1@@ -424,7 +424,11 @@ int erofs_write_file(struct erofs_inode *inode) }if (cfg.c_compr_alg_master &&erofs_file_is_compressible (inode)) { - ret =erofs_write_compressed_file (inode); + fd =open (inode->i_srcpath,O_RDONLY |O_BINARY ); +if (fd <0 ) +return -errno; + ret =erofs_write_compressed_file_from_fd (inode, fd, true); +close (fd);if (!ret || ret != -ENOSPC )return ret;
lib/inode.c static struct erofs_inode *erofs_new_inode(void) · line 943
Quoted 5 lines
lib/inode.c +2
-0@@ -935,6 +939,24 @@ static struct erofs_inode *erofs_new_inode(void)return inode; } +static struct erofs_inode *erofs_generate_inode (struct stat64 *st, +const char *path)
let's avoid such helper, since it doesn't simplify a lot.
Thanks,
Gao Xiang
Quoted 200 lines
lib/inode.c +40
-11+{ +struct erofs_inode *inode; +int ret; + + inode =erofs_new_inode (); +if (IS_ERR (inode)) +return inode; + + ret =erofs_fill_inode (inode, st, path); +if (ret) { +free (inode); +return ERR_PTR (ret); + } +return inode; +} +/* get the inode from the (source) path */ static struct erofs_inode *erofs_iget_from_path (const char *path,bool is_src) {@@ -962,17 +984,7 @@ static struct erofs_inode *erofs_iget_from_path(const char *path, bool is_src) }/* cannot find in the inode cache */ - inode =erofs_new_inode (); -if (IS_ERR (inode)) -return inode; - - ret =erofs_fill_inode (inode, &st, path); -if (ret) { -free (inode); -return ERR_PTR (ret); - } - -return inode; +return erofs_generate_inode (&st, path); }static void erofs_fixup_meta_blkaddr (struct erofs_inode *rootdir)@@ -1180,3 +1192,26 @@ struct erofs_inode *erofs_mkfs_build_tree_from_path(struct erofs_inode *parent,return erofs_mkfs_build_tree (inode); } + +struct erofs_inode *erofs_mkfs_build_special_from_fd (int fd,const char *name) +{ +struct stat64 st; +struct erofs_inode *inode; +int ret; + + ret =fstat64 (fd, &st); +if (ret) +return ERR_PTR (-errno); + + inode =erofs_generate_inode (&st, name); +if (IS_ERR (inode)) +return inode; + +/* only for compressed file now */ + ret =erofs_write_compressed_file_from_fd (inode, fd, false); +if (ret) +return ERR_PTR (ret); + +erofs_prepare_inode_buffer (inode); +return inode; +}mkfs/main.c +58
-6@@ -23,6 +23,7 @@#include "erofs/block_list.h" #include "erofs/compress_hints.h" #include "erofs/blobchunk.h" +#include "erofs/fragments.h" #include "../lib/liberofs_private.h" #ifdef HAVE_LIBUUID @@ -133,9 +134,9 @@ static int parse_extended_opts(const char *opts)const char *p =strchr (token,',' ); next =NULL ; -if (p) +if (p) { next = p +1 ; -else { + }else { p = token +strlen (token); next = p; }@@ -202,7 +203,34 @@ static int parse_extended_opts(const char *opts)return -EINVAL ; cfg.c_ztailpacking = true; } + +if (MATCH_EXTENTED_OPT ("fragments" , token, keylen)) { +char *endptr; +u64 i; + +if (vallen || cfg.c_ztailpacking) +return -EINVAL ; + cfg.c_fragments = true; + + i =strtoull (next, &endptr,0 ); +if (i ==0 || (*endptr !=',' && *endptr !='\0' )) { + cfg.c_pclusterblks_packed =1 ; +continue ; + } +if (i %EROFS_BLKSIZ ) { +erofs_err ("invalid physical clustersize %llu" , + i); +return -EINVAL ; + } + cfg.c_pclusterblks_packed = i /EROFS_BLKSIZ ; + +if (*endptr ==',' ) + next =strchr (next,',' ) +1 ; +else + goto out; + } } +out:return 0 ; }@@ -458,7 +486,8 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])int erofs_mkfs_update_super_block (struct erofs_buffer_head *bh,erofs_nid_t root_nid, -erofs_blk_t *blocks) +erofs_blk_t *blocks, +erofs_nid_t packed_nid) {struct erofs_super_block sb = { .magic =cpu_to_le32 (EROFS_SUPER_MAGIC_V1 ),@@ -482,6 +511,7 @@ int erofs_mkfs_update_super_block(struct erofs_buffer_head *bh, *blocks =erofs_mapbh (NULL ); sb.blocks =cpu_to_le32 (*blocks); sb.root_nid =cpu_to_le16 (root_nid); + sb.packed_nid =cpu_to_le64 (packed_nid);memcpy (sb.uuid, sbi.uuid,sizeof (sb.uuid));if (erofs_sb_has_compr_cfgs ())@@ -599,8 +629,8 @@ int main(int argc, char **argv) {int err =0 ;struct erofs_buffer_head *sb_bh; -struct erofs_inode *root_inode; -erofs_nid_t root_nid; +struct erofs_inode *root_inode, *packed_inode; +erofs_nid_t root_nid, packed_nid;struct stat64 st;erofs_blk_t nblocks;struct timeval t;@@ -670,6 +700,14 @@ int main(int argc, char **argv)erofs_warn ("EXPERIMENTAL chunked file feature in use. Use at your own risk!" );if (cfg.c_ztailpacking)erofs_warn ("EXPERIMENTAL compressed inline data feature in use. Use at your own risk!" ); +if (cfg.c_fragments) { + err =erofs_fragments_init (); +if (err) { +erofs_err ("failed to initialize fragments" ); +return 1 ; + } +erofs_warn ("EXPERIMENTAL compressed fragments feature in use. Use at your own risk!" ); + }erofs_set_fs_root (cfg.c_src_path);#ifndef NDEBUG if (cfg.c_random_pclusterblks)@@ -739,7 +777,19 @@ int main(int argc, char **argv) goto exit; } - err =erofs_mkfs_update_super_block (sb_bh, root_nid, &nblocks); + packed_nid =0 ; +if (cfg.c_fragments) { + packed_inode =erofs_mkfs_build_fragments (); +if (IS_ERR (packed_inode)) { + err =PTR_ERR (packed_inode); + goto exit; + } + packed_nid =erofs_lookupnid (packed_inode); +erofs_iput (packed_inode); + } + + err =erofs_mkfs_update_super_block (sb_bh, root_nid, &nblocks, + packed_nid);if (err) goto exit;@@ -761,6 +811,8 @@ exit:erofs_cleanup_exclude_rules ();if (cfg.c_chunkbits)erofs_blob_exit (); +if (cfg.c_fragments) +erofs_fragments_exit ();erofs_exit_configure ();if (err) {Signature · 1 line
2.17.1
Re: [RFC PATCH v4 3/3] erofs-utils: introduce compressed fragments support
Hi Xiang,
Quoted 160 lines · On Wed, 24 Aug 2022 17:00:07 +0800 Gao Xiang <[email protected]>…
Quoted 159 lines · On Sun, Aug 21, 2022 at 09:57:25PM +0800, [email protected] wrote:
From: Yue Hu <[email protected]>
This approach can merge tail pclusters or the whole files into a special inode in order to achieve greater compression ratio. And an option of pcluster size is provided for different compression requirments.
Meanwhile, we change to write the uncompressed data from 'clusterofs' when compressing files since it can benefit from in-place I/O. For now, this change goes with the fragments.
Signed-off-by Yue Hu <[email protected]>--- include/erofs/compress.h | 3 +- include/erofs/config.h | 3 +- include/erofs/fragments.h | 25 +++++++++ include/erofs/inode.h | 1 + include/erofs/internal.h | 1 + include/erofs_fs.h | 1 + lib/Makefile.am | 4 +- lib/compress.c | 108 ++++++++++++++++++++++++++++---------- lib/fragments.c | 58 ++++++++++++++++++++ lib/inode.c | 59 ++++++++++++++++----- mkfs/main.c | 64 +++++++++++++++++++--- 11 files changed, 277 insertions(+), 50 deletions(-) create mode 100644 include/erofs/fragments.h create mode 100644 lib/fragments.cinclude/erofs/compress.h +2
-1@@ -18,7 +18,8 @@ extern "C"#define EROFS_CONFIG_COMPR_MIN_SZ (32 * 1024)void z_erofs_drop_inline_pcluster (struct erofs_inode *inode); -int erofs_write_compressed_file (struct erofs_inode *inode); +int erofs_write_compressed_file_from_fd (struct erofs_inode *inode,int fd, +bool is_src);int z_erofs_compress_init (struct erofs_buffer_head *bh);int z_erofs_compress_exit (void );include/erofs/config.h +2
-1@@ -44,6 +44,7 @@ struct erofs_configure {char c_chunkbits;bool c_noinline_data;bool c_ztailpacking; +bool c_fragments;bool c_ignore_mtime;bool c_showprogress;@@ -62,7 +63,7 @@ struct erofs_configure {/* < 0, xattr disabled and INT_MAX, always use inline xattrs */ int c_inline_xattr_tolerance; -u32 c_pclusterblks_max, c_pclusterblks_def; +u32 c_pclusterblks_max, c_pclusterblks_def, c_pclusterblks_packed;u32 c_max_decompressed_extent_bytes;u32 c_dict_size;u64 c_unix_timestamp;include/erofs/fragments.h new file +25
-0@@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ +/* +* Copyright (C), 2022, Coolpad Group Limited. +*/ +#ifndef __EROFS_FRAGMENTS_H +#define __EROFS_FRAGMENTS_H + +#ifdef __cplusplus +extern "C" +{ +#endif + +#include "erofs/internal.h" + +int z_erofs_pack_fragments (struct erofs_inode *inode,void *data, +unsigned int len); +struct erofs_inode *erofs_mkfs_build_fragments (void ); +int erofs_fragments_init (void ); +void erofs_fragments_exit (void ); + +#ifdef __cplusplus +} +#endif + +#endif include/erofs/inode.h +1
-0@@ -22,6 +22,7 @@ unsigned int erofs_iput(struct erofs_inode *inode);erofs_nid_t erofs_lookupnid (struct erofs_inode *inode);struct erofs_inode *erofs_mkfs_build_tree_from_path (struct erofs_inode *parent,const char *path); +struct erofs_inode *erofs_mkfs_build_special_from_fd (int fd,const char *name); #ifdef __cplusplus }include/erofs/internal.h +1
-0@@ -212,6 +212,7 @@ struct erofs_inode {uint64_t capabilities;#endif erofs_off_t fragmentoff; +unsigned int fragment_size; };static inline bool is_inode_layout_compression (struct erofs_inode *inode)include/erofs_fs.h +1
-0@@ -267,6 +267,7 @@ struct erofs_inode_chunk_index {/* maximum supported size of a physical compression cluster */ #define Z_EROFS_PCLUSTER_MAX_SIZE (1024 * 1024) +#define Z_EROFS_PCLUSTER_MAX_BLKS (Z_EROFS_PCLUSTER_MAX_SIZE / EROFS_BLKSIZ)/* available compression algorithm types (for h_algorithmtype) */ enum {lib/Makefile.am +3
-1@@ -22,12 +22,14 @@ noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \ $(top_srcdir)/include/erofs/trace.h \ $(top_srcdir)/include/erofs/xattr.h \ $(top_srcdir)/include/erofs/compress_hints.h \ + $(top_srcdir)/include/erofs/fragments.h \ $(top_srcdir)/lib/liberofs_private.h noinst_HEADERS += compressor.h liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \ namei.c data.c compress.c compressor.c zmap.c decompress.c \ - compress_hints.c hashmap.c sha256.c blobchunk.c dir.c + compress_hints.c hashmap.c sha256.c blobchunk.c dir.c \ + fragments.c liberofs_la_CFLAGS = -Wall -I$(top_srcdir)/include if ENABLE_LZ4 liberofs_la_CFLAGS += ${LZ4_CFLAGS}lib/compress.c +1
-0@@ -18,6 +18,7 @@#include "compressor.h" #include "erofs/block_list.h" #include "erofs/compress_hints.h" +#include "erofs/fragments.h"
Quoted 13 lines
lib/compress.c static void vle_write_indexes(struct z_erofs_vle_compress_ctx *ctx, · line 80
Quoted 11 lines
lib/compress.c +2
-2static struct erofs_compress compresshandle;static unsigned int algorithmtype[2 ];@@ -74,9 +75,9 @@ static void vle_write_indexes(struct z_erofs_vle_compress_ctx *ctx,if (!d1) {/* * A lcluster cannot have three parts with the middle one which -* is well-compressed for !ztailpacking cases. +* is well-compressed for !ztailpacking and !fragments cases. */ -DBG_BUGON (!raw && !cfg.c_ztailpacking); +DBG_BUGON (!raw && !cfg.c_ztailpacking && !cfg.c_fragments);This should be per-inode?
Thanks for reviewing the patch, let me check this and below.
Thanks.
Quoted 569 lines
Quoted 17 lines
lib/compress.c +1
-1type = raw ?Z_EROFS_VLE_CLUSTER_TYPE_PLAIN :Z_EROFS_VLE_CLUSTER_TYPE_HEAD ; advise =cpu_to_le16 (type <<Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT );@@ -143,7 +144,7 @@ static int write_uncompressed_extent(struct z_erofs_vle_compress_ctx *ctx, unsigned int *len, char *dst) { int ret; -unsigned int count; +unsigned int count, offset, rcopied, rzeroed;/* reset clusterofs to 0 if permitted */ if (!erofs_sb_has_lz4_0padding() && ctx->clusterofs &&@@ -153,11 +154,21 @@ static int write_uncompressed_extent(struct z_erofs_vle_compress_ctx *ctx, ctx->clusterofs =0 ; }lib/compress.c static int write_uncompressed_extent(struct z_erofs_vle_compress_ctx *ctx, · line 163
Quoted 10 lines
lib/compress.c +5
-3-/* write uncompressed data */ +/* +* write uncompressed data from clusterofs which can benefit from +* in-place I/O, loop shift right when to exceed EROFS_BLKSIZ. +*/ count =min (EROFS_BLKSIZ , *len); -memcpy (dst, ctx->queue + ctx->head, count); -memset (dst + count,0 ,EROFS_BLKSIZ - count); + offset = cfg.c_fragments ? ctx->clusterofs :0 ;
interlaced_offset;Quoted 12 lines
lib/compress.c +8
-0+ rcopied =min (EROFS_BLKSIZ - offset, count); + rzeroed =EROFS_BLKSIZ - offset - rcopied; + +memcpy (dst + offset, ctx->queue + ctx->head, rcopied); +memcpy (dst, ctx->queue + ctx->head + rcopied, count - rcopied); + +memset (dst + offset + rcopied,0 , rzeroed); +memset (dst + count - rcopied,0 ,EROFS_BLKSIZ - count - rzeroed);erofs_dbg ("Writing %u uncompressed data to block %u" , count, ctx->blkaddr);lib/compress.c static int write_uncompressed_extent(struct z_erofs_vle_compress_ctx *ctx, · line 182
Quoted 6 lines
lib/compress.c +2
-1@@ -167,8 +178,11 @@ static int write_uncompressed_extent(struct z_erofs_vle_compress_ctx *ctx,return count; } -static unsigned int z_erofs_get_max_pclusterblks (struct erofs_inode *inode) +static unsigned int z_erofs_get_max_pclusterblks (struct erofs_inode *inode, +bool is_src)
is_srcshould be inerofs_inodeas, for example, boolis_packed_inode, or other likewise rather than passing another argument here.Quoted 80 lines
lib/compress.c +31
-4{ +if (cfg.c_fragments && !is_src) +return cfg.c_pclusterblks_packed;#ifndef NDEBUG if (cfg.c_random_pclusterblks)return 1 +rand () % cfg.c_pclusterblks_max;@@ -224,7 +238,7 @@ static void tryrecompress_trailing(void *in, unsigned int *insize,static int vle_compress_one (struct erofs_inode *inode,struct z_erofs_vle_compress_ctx *ctx, -bool final) +bool final,bool is_src) {struct erofs_compress *const h = &compresshandle;unsigned int len = ctx->tail - ctx->head;@@ -234,14 +248,19 @@ static int vle_compress_one(struct erofs_inode *inode,char *const dst = dstbuf +EROFS_BLKSIZ ;while (len) { -unsigned int pclustersize = -z_erofs_get_max_pclusterblks (inode) *EROFS_BLKSIZ ; +unsigned int pclustersize =EROFS_BLKSIZ * +z_erofs_get_max_pclusterblks (inode, is_src);bool may_inline = (cfg.c_ztailpacking && final); +bool may_packing = (cfg.c_fragments && final && is_src);bool raw;if (len <= pclustersize) {if (!final)break ; +if (may_packing) { + count = len; + goto frag_packing; + }if (!may_inline && len <=EROFS_BLKSIZ ) goto nocompression; }@@ -294,6 +313,19 @@ nocompression:return ret; ctx->compressedblks =1 ; raw = false; + }else if (may_packing && len == count && ret < pclustersize) { +frag_packing: + ret =z_erofs_pack_fragments (inode, + ctx->queue + ctx->head, + len); +if (ret <0 ) +return ret; +if (inode->i_size == inode->fragment_size) { + ctx->head += len; +return 0 ; + } + ctx->compressedblks =0 ; + raw = false; }else {unsigned int tailused, padding;@@ -546,13 +578,20 @@ static void z_erofs_write_mapheader(struct erofs_inode *inode, {struct z_erofs_map_header h = { .h_advise =cpu_to_le16 (inode->z_advise), - .h_idata_size =cpu_to_le16 (inode->idata_size), .h_algorithmtype = inode->z_algorithmtype[1 ] <<4 | inode->z_algorithmtype[0 ],/* lclustersize */ .h_clusterbits = inode->z_logical_clusterbits -12 , }; +if (cfg.c_fragments) + h.h_fragmentoff =cpu_to_le32 (inode->fragmentoff); +else + h.h_idata_size =cpu_to_le16 (inode->idata_size); + +if (inode->fragment_size && inode->i_size == inode->fragment_size) + h.h_clusterbits |=1 <<Z_EROFS_FRAGMENT_INODE_BIT ; +memset (compressmeta,0 ,Z_EROFS_LEGACY_MAP_HEADER_SIZE );/* write out map header */ memcpy (compressmeta, &h,sizeof (struct z_erofs_map_header ));lib/compress.c void z_erofs_drop_inline_pcluster(struct erofs_inode *inode) · line 648
Quoted 6 lines
lib/compress.c +2
-1@@ -605,30 +644,25 @@ void z_erofs_drop_inline_pcluster(struct erofs_inode *inode) inode->eof_tailraw =NULL ; } -int erofs_write_compressed_file (struct erofs_inode *inode) +int erofs_write_compressed_file_from_fd (struct erofs_inode *inode,int fd, +bool is_src)same here.
Quoted 92 lines
lib/compress.c +12
-15{struct erofs_buffer_head *bh;static struct z_erofs_vle_compress_ctx ctx;erofs_off_t remaining;erofs_blk_t blkaddr, compressed_blocks;unsigned int legacymetasize; -int ret, fd; +int ret;u8 *compressmeta =malloc (vle_compressmeta_capacity (inode->i_size));if (!compressmeta)return -ENOMEM ; - fd =open (inode->i_srcpath,O_RDONLY |O_BINARY ); -if (fd <0 ) { - ret = -errno; - goto err_free_meta; - } -/* allocate main data buffer */ bh =erofs_balloc (DATA ,0 ,0 ,0 );if (IS_ERR (bh)) { ret =PTR_ERR (bh); - goto err_close; + goto err_free_meta; }/* initialize per-file compression setting */ @@ -649,6 +683,9 @@ int erofs_write_compressed_file(struct erofs_inode *inode) inode->z_algorithmtype[1 ] = algorithmtype[1 ]; inode->z_logical_clusterbits =LOG_BLOCK_SIZE ; + inode->idata_size =0 ; + inode->fragment_size =0 ; + blkaddr =erofs_mapbh (bh->block);/* start_blkaddr */ ctx.blkaddr = blkaddr; ctx.metacur = compressmeta +Z_EROFS_LEGACY_MAP_HEADER_SIZE ;@@ -668,7 +705,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode) remaining -= readcount; ctx.tail += readcount; - ret =vle_compress_one (inode, &ctx, !remaining); + ret =vle_compress_one (inode, &ctx, !remaining, is_src);if (ret) goto err_free_idata; }@@ -682,19 +719,20 @@ int erofs_write_compressed_file(struct erofs_inode *inode)vle_write_indexes_final (&ctx); legacymetasize = ctx.metacur - compressmeta;/* estimate if data compression saves space or not */ -if (compressed_blocks *EROFS_BLKSIZ + inode->idata_size + +if (!inode->fragment_size && + compressed_blocks *EROFS_BLKSIZ + inode->idata_size + legacymetasize >= inode->i_size) { ret = -ENOSPC ; goto err_free_idata; }z_erofs_write_mapheader (inode, compressmeta); -close (fd);if (compressed_blocks) { ret =erofs_bh_balloon (bh,blknr_to_addr (compressed_blocks));DBG_BUGON (ret !=EROFS_BLKSIZ ); }else { -DBG_BUGON (!inode->idata_size); +if (!cfg.c_fragments) +DBG_BUGON (!inode->idata_size); } erofs_info("compressed %s (%llu bytes) into %u blocks",@@ -717,7 +755,8 @@ int erofs_write_compressed_file(struct erofs_inode *inode)DBG_BUGON (ret); } inode->compressmeta = compressmeta; -erofs_droid_blocklist_write (inode, blkaddr, compressed_blocks); +if (is_src) +erofs_droid_blocklist_write (inode, blkaddr, compressed_blocks);return 0 ; err_free_idata:@@ -727,8 +766,6 @@ err_free_idata: } err_bdrop:erofs_bdrop (bh, true);/* revoke buffer */ -err_close: -close (fd); err_free_meta:free (compressmeta);return ret;@@ -834,14 +871,27 @@ int z_erofs_compress_init(struct erofs_buffer_head *sb_bh)* to be loaded in order to get those compressed block counts. lib/compress.c int z_erofs_compress_init(struct erofs_buffer_head *sb_bh) · line 879
Quoted 10 lines
lib/compress.c +2
-2*/ if (cfg.c_pclusterblks_max >1 ) { -if (cfg.c_pclusterblks_max > -Z_EROFS_PCLUSTER_MAX_SIZE /EROFS_BLKSIZ ) { +if (cfg.c_pclusterblks_max >Z_EROFS_PCLUSTER_MAX_BLKS ) {erofs_err ("unsupported clusterblks %u (too large)" , cfg.c_pclusterblks_max);return -EINVAL ; } +if (cfg.c_pclusterblks_packed >Z_EROFS_PCLUSTER_MAX_BLKS ) {
c_pclusterblks_packedshould be smaller thanc_pclusterblks_maxlib/compress.c int z_erofs_compress_init(struct erofs_buffer_head *sb_bh) · line 887
Quoted 8 lines
lib/compress.c +8
-0+erofs_err ("unsupported clusterblks %u (too large for fragments)" , + cfg.c_pclusterblks_packed); +return -EINVAL ; + } +if (cfg.c_pclusterblks_packed ==1 ) { +erofs_err ("physical cluster size of fragments should > 4096 bytes" ); +return -EINVAL ; + }How can this happen? why judging this here?
lib/compress.c int z_erofs_compress_init(struct erofs_buffer_head *sb_bh) · line 894
Quoted 7 lines
lib/compress.c +5
-0erofs_sb_set_big_pcluster (); } +if (!erofs_sb_has_big_pcluster () && cfg.c_pclusterblks_packed >1 ) { +erofs_err ("invalid clusterblks %u (for fragments)" , + cfg.c_pclusterblks_packed); +return -EINVAL ; + }The only condition I think would be
if (c_pclusterblks_packed > cfg.c_pclusterblks_max) { erofs_err("invalid physical cluster size for the packed file"); }Quoted 85 lines
lib/compress.c +0
-0if (ret !=Z_EROFS_COMPRESSION_LZ4 )erofs_sb_set_compr_cfgs ();lib/fragments.c new file +58
-0@@ -0,0 +1,58 @@ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 +/* +* Copyright (C), 2022, Coolpad Group Limited. +* Created by Yue Hu <[email protected]> +*/ +#define _GNU_SOURCE +#include <stdlib.h> +#include <unistd.h> +#include "erofs/err.h" +#include "erofs/inode.h" +#include "erofs/compress.h" +#include "erofs/print.h" +#include "erofs/fragments.h" + +static FILE *packedfile; + +int z_erofs_pack_fragments (struct erofs_inode *inode,void *data, +unsigned int len) +{ + inode->z_advise |=Z_EROFS_ADVISE_FRAGMENT_PCLUSTER ; + inode->fragmentoff =ftell (packedfile); + inode->fragment_size = len; + +if (write (fileno (packedfile), data, len) <0 ) +return -EIO ; + +erofs_sb_set_fragments (); + +erofs_dbg ("Recording %u fragment data at %lu" , inode->fragment_size, + inode->fragmentoff); +return len; +} + +struct erofs_inode *erofs_mkfs_build_fragments (void ) +{ +fseek (packedfile,0 ,SEEK_SET ); + +return erofs_mkfs_build_special_from_fd (fileno (packedfile), +"packed_file" ); +} + +void erofs_fragments_exit (void ) +{ +if (packedfile) +fclose (packedfile); +} + +int erofs_fragments_init (void ) +{ +#ifdef HAVE_TMPFILE64 + packedfile =tmpfile64 (); +#else + packedfile =tmpfile (); +#endif +if (!packedfile) +return -ENOMEM ; +return 0 ; +}lib/inode.c +5
-1@@ -424,7 +424,11 @@ int erofs_write_file(struct erofs_inode *inode) }if (cfg.c_compr_alg_master &&erofs_file_is_compressible (inode)) { - ret =erofs_write_compressed_file (inode); + fd =open (inode->i_srcpath,O_RDONLY |O_BINARY ); +if (fd <0 ) +return -errno; + ret =erofs_write_compressed_file_from_fd (inode, fd, true); +close (fd);if (!ret || ret != -ENOSPC )return ret;lib/inode.c static struct erofs_inode *erofs_new_inode(void) · line 943
Quoted 5 lines
lib/inode.c +2
-0@@ -935,6 +939,24 @@ static struct erofs_inode *erofs_new_inode(void)return inode; } +static struct erofs_inode *erofs_generate_inode (struct stat64 *st, +const char *path)let's avoid such helper, since it doesn't simplify a lot.
Thanks,
Gao XiangQuoted 200 lines
lib/inode.c +40
-11+{ +struct erofs_inode *inode; +int ret; + + inode =erofs_new_inode (); +if (IS_ERR (inode)) +return inode; + + ret =erofs_fill_inode (inode, st, path); +if (ret) { +free (inode); +return ERR_PTR (ret); + } +return inode; +} +/* get the inode from the (source) path */ static struct erofs_inode *erofs_iget_from_path (const char *path,bool is_src) {@@ -962,17 +984,7 @@ static struct erofs_inode *erofs_iget_from_path(const char *path, bool is_src) }/* cannot find in the inode cache */ - inode =erofs_new_inode (); -if (IS_ERR (inode)) -return inode; - - ret =erofs_fill_inode (inode, &st, path); -if (ret) { -free (inode); -return ERR_PTR (ret); - } - -return inode; +return erofs_generate_inode (&st, path); }static void erofs_fixup_meta_blkaddr (struct erofs_inode *rootdir)@@ -1180,3 +1192,26 @@ struct erofs_inode *erofs_mkfs_build_tree_from_path(struct erofs_inode *parent,return erofs_mkfs_build_tree (inode); } + +struct erofs_inode *erofs_mkfs_build_special_from_fd (int fd,const char *name) +{ +struct stat64 st; +struct erofs_inode *inode; +int ret; + + ret =fstat64 (fd, &st); +if (ret) +return ERR_PTR (-errno); + + inode =erofs_generate_inode (&st, name); +if (IS_ERR (inode)) +return inode; + +/* only for compressed file now */ + ret =erofs_write_compressed_file_from_fd (inode, fd, false); +if (ret) +return ERR_PTR (ret); + +erofs_prepare_inode_buffer (inode); +return inode; +}mkfs/main.c +58
-6@@ -23,6 +23,7 @@#include "erofs/block_list.h" #include "erofs/compress_hints.h" #include "erofs/blobchunk.h" +#include "erofs/fragments.h" #include "../lib/liberofs_private.h" #ifdef HAVE_LIBUUID @@ -133,9 +134,9 @@ static int parse_extended_opts(const char *opts)const char *p =strchr (token,',' ); next =NULL ; -if (p) +if (p) { next = p +1 ; -else { + }else { p = token +strlen (token); next = p; }@@ -202,7 +203,34 @@ static int parse_extended_opts(const char *opts)return -EINVAL ; cfg.c_ztailpacking = true; } + +if (MATCH_EXTENTED_OPT ("fragments" , token, keylen)) { +char *endptr; +u64 i; + +if (vallen || cfg.c_ztailpacking) +return -EINVAL ; + cfg.c_fragments = true; + + i =strtoull (next, &endptr,0 ); +if (i ==0 || (*endptr !=',' && *endptr !='\0' )) { + cfg.c_pclusterblks_packed =1 ; +continue ; + } +if (i %EROFS_BLKSIZ ) { +erofs_err ("invalid physical clustersize %llu" , + i); +return -EINVAL ; + } + cfg.c_pclusterblks_packed = i /EROFS_BLKSIZ ; + +if (*endptr ==',' ) + next =strchr (next,',' ) +1 ; +else + goto out; + } } +out:return 0 ; }@@ -458,7 +486,8 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])int erofs_mkfs_update_super_block (struct erofs_buffer_head *bh,erofs_nid_t root_nid, -erofs_blk_t *blocks) +erofs_blk_t *blocks, +erofs_nid_t packed_nid) {struct erofs_super_block sb = { .magic =cpu_to_le32 (EROFS_SUPER_MAGIC_V1 ),@@ -482,6 +511,7 @@ int erofs_mkfs_update_super_block(struct erofs_buffer_head *bh, *blocks =erofs_mapbh (NULL ); sb.blocks =cpu_to_le32 (*blocks); sb.root_nid =cpu_to_le16 (root_nid); + sb.packed_nid =cpu_to_le64 (packed_nid);memcpy (sb.uuid, sbi.uuid,sizeof (sb.uuid));if (erofs_sb_has_compr_cfgs ())@@ -599,8 +629,8 @@ int main(int argc, char **argv) {int err =0 ;struct erofs_buffer_head *sb_bh; -struct erofs_inode *root_inode; -erofs_nid_t root_nid; +struct erofs_inode *root_inode, *packed_inode; +erofs_nid_t root_nid, packed_nid;struct stat64 st;erofs_blk_t nblocks;struct timeval t;@@ -670,6 +700,14 @@ int main(int argc, char **argv)erofs_warn ("EXPERIMENTAL chunked file feature in use. Use at your own risk!" );if (cfg.c_ztailpacking)erofs_warn ("EXPERIMENTAL compressed inline data feature in use. Use at your own risk!" ); +if (cfg.c_fragments) { + err =erofs_fragments_init (); +if (err) { +erofs_err ("failed to initialize fragments" ); +return 1 ; + } +erofs_warn ("EXPERIMENTAL compressed fragments feature in use. Use at your own risk!" ); + }erofs_set_fs_root (cfg.c_src_path);#ifndef NDEBUG if (cfg.c_random_pclusterblks)@@ -739,7 +777,19 @@ int main(int argc, char **argv) goto exit; } - err =erofs_mkfs_update_super_block (sb_bh, root_nid, &nblocks); + packed_nid =0 ; +if (cfg.c_fragments) { + packed_inode =erofs_mkfs_build_fragments (); +if (IS_ERR (packed_inode)) { + err =PTR_ERR (packed_inode); + goto exit; + } + packed_nid =erofs_lookupnid (packed_inode); +erofs_iput (packed_inode); + } + + err =erofs_mkfs_update_super_block (sb_bh, root_nid, &nblocks, + packed_nid);if (err) goto exit;@@ -761,6 +811,8 @@ exit:erofs_cleanup_exclude_rules ();if (cfg.c_chunkbits)erofs_blob_exit (); +if (cfg.c_fragments) +erofs_fragments_exit ();erofs_exit_configure ();if (err) {Signature · 1 line
2.17.1