erofs-utils: introduce segment compression
PATCH v4 1/1 erofs-utils: introduce segment compression
Support segment compression which seperates files in several logic units (segments) and each segment is compressed independently.
Advantages:
- more friendly for data differencing;
- it can also be used for parallel compression in the same file later.
---
Changes since v3 suggest by Gao Xiang<[email protected]>:
- add 'S#' parameter to custome compression segment size
- move limit logic to size decrease
include/erofs/config.h | 1 + lib/compress.c | 8 ++++++-- lib/config.c | 1 + mkfs/main.c | 16 +++++++++++++++- 4 files changed, 23 insertions(+), 3 deletions(-)
include/erofs/config.h +1 -0
@@ -36,6 +36,7 @@ struct erofs_configure {char *c_src_path;char *c_compr_alg_master;int c_compr_level_master; +unsigned int c_compr_seg_size;/* max segment compress size */ int c_force_inodeversion;/* < 0, xattr disabled and INT_MAX, always use inline xattrs */ int c_inline_xattr_tolerance;
lib/compress.c +6 -2
@@ -32,6 +32,7 @@ struct z_erofs_vle_compress_ctx {erofs_blk_t blkaddr;/* pointing to the next blkaddr */ u16 clusterofs; +unsigned int comprlimits; };#define Z_EROFS_LEGACY_MAP_HEADER_SIZE \
@@ -163,8 +164,7 @@ static int vle_compress_one(struct erofs_inode *inode, goto nocompression;break ; } - - count = len; + count =min (len, ctx->comprlimits); ret =erofs_compress_destsize (h, compressionlevel, ctx->queue + ctx->head, &count, dst,EROFS_BLKSIZ );
@@ -202,6 +202,9 @@ nocompression: ++ctx->blkaddr; len -= count; + ctx->comprlimits -= count; +if (!ctx->comprlimits) + ctx->comprlimits = cfg.c_compr_seg_size;if (!final && ctx->head >=EROFS_CONFIG_COMPR_MAX_SZ ) { const unsigned int qh_aligned =
@@ -422,6 +425,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode) ctx.head = ctx.tail =0 ; ctx.clusterofs =0 ; remaining = inode->i_size; + ctx.comprlimits = cfg.c_compr_seg_size;while (remaining) { const u64 readcount = min_t(u64, remaining,
lib/config.c +1 -0
@@ -23,6 +23,7 @@ void erofs_init_configure(void) cfg.c_force_inodeversion =0 ; cfg.c_inline_xattr_tolerance =2 ; cfg.c_unix_timestamp =-1 ; + cfg.c_compr_seg_size =1024U *EROFS_BLKSIZ ; }void erofs_show_config (void )
mkfs/main.c +15 -1
@@ -61,6 +61,7 @@ static void usage(void)" -x# set xattr tolerance to # (< 0, disable xattrs; default 2)\n" " -EX[,...] X=extended options\n" " -T# set a fixed UNIX timestamp # to all files\n" +" -S# set the max input stream size # to one compress\n" " --exclude-path=X avoid including file X (X = exact literal path)\n" " --exclude-regex=X avoid including files that match X (X = regular expression)\n" #ifdef HAVE_LIBSELINUX
@@ -138,7 +139,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])char *endptr;int opt, i; -while ((opt =getopt_long (argc, argv,"d:x:z:E:T:" , +while ((opt =getopt_long (argc, argv,"d:x:z:E:T:S:" , long_options,NULL )) !=-1 ) {switch (opt) {case 'z' :
@@ -188,6 +189,19 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])return -EINVAL ; }break ; +case 'S' : + cfg.c_compr_seg_size =strtol (optarg, &endptr,0 ); +if (*endptr !='\0' ) { +erofs_err ("invalid compress segment size %s" , + optarg); +return -EINVAL ; + } +if (cfg.c_compr_seg_size %EROFS_BLKSIZ !=0 ) { +erofs_err ("segment size:%u should be align to %u" , + cfg.c_compr_seg_size,EROFS_BLKSIZ ); +return -EINVAL ; + } +break ;case 2 : opt =erofs_parse_exclude_path (optarg, false);if (opt) {
Signature · 1 line
2.17.1
Hi Guifu,
Quoted 13 lines · Li Guifu via Linux-erofs
Support segment compression which seperates files in several logic units (segments) and each segment is compressed independently.
Advantages:
- more friendly for data differencing;
- it can also be used for parallel compression in the same file later.Signed-off-by Li Guifu <[email protected]>---
Changes since v3 suggest by Gao Xiang<[email protected]>:
- add 'S#' parameter to custome compression segment size
- move limit logic to size decrease
Quoted 4 lines
include/erofs/config.h | 1 + lib/compress.c | 8 ++++++-- lib/config.c | 1 + mkfs/main.c | 16 +++++++++++++++-
Just do a quick response for this, and will test it later.
First, You might need to update the manpage as well.
include/erofs/config.h struct erofs_configure { · line 38
Quoted 10 lines
4 files changed, 23 insertions(+), 3 deletions(-)include/erofs/config.h +0
-0@@ -36,6 +36,7 @@ struct erofs_configure {char *c_src_path;char *c_compr_alg_master;int c_compr_level_master;
u64 c_compr_segsize;
lib/compress.c struct z_erofs_vle_compress_ctx { · line 35
Quoted 13 lines
include/erofs/config.h +1
-0+unsigned int c_compr_seg_size;/* max segment compress size */ int c_force_inodeversion;/* < 0, xattr disabled and INT_MAX, always use inline xattrs */ int c_inline_xattr_tolerance;lib/compress.c +1
-0@@ -32,6 +32,7 @@ struct z_erofs_vle_compress_ctx {erofs_blk_t blkaddr;/* pointing to the next blkaddr */ u16 clusterofs; +unsigned int comprlimits;
How about the name "segavail"; ?
u64 segavail;
lib/compress.c static int vle_compress_one(struct erofs_inode *inode, · line 166
Quoted 7 lines
lib/compress.c +0
-0};#define Z_EROFS_LEGACY_MAP_HEADER_SIZE \@@ -163,8 +164,7 @@ static int vle_compress_one(struct erofs_inode *inode, goto nocompression;break ; }
I think we might add "if (segavail < EROFS_BLKSIZE) goto nocompression;" since it seems better.
Quoted 29 lines
lib/compress.c +5
-2- - count = len; + count =min (len, ctx->comprlimits); ret =erofs_compress_destsize (h, compressionlevel, ctx->queue + ctx->head, &count, dst,EROFS_BLKSIZ );@@ -202,6 +202,9 @@ nocompression: ++ctx->blkaddr; len -= count; + ctx->comprlimits -= count; +if (!ctx->comprlimits) + ctx->comprlimits = cfg.c_compr_seg_size;if (!final && ctx->head >=EROFS_CONFIG_COMPR_MAX_SZ ) { const unsigned int qh_aligned =@@ -422,6 +425,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode) ctx.head = ctx.tail =0 ; ctx.clusterofs =0 ; remaining = inode->i_size; + ctx.comprlimits = cfg.c_compr_seg_size;while (remaining) { const u64 readcount = min_t(u64, remaining,lib/config.c +0
-0
lib/config.c void erofs_init_configure(void) · line 26
Quoted 4 lines
lib/config.c +1
-0@@ -23,6 +23,7 @@ void erofs_init_configure(void) cfg.c_force_inodeversion =0 ; cfg.c_inline_xattr_tolerance =2 ; cfg.c_unix_timestamp =-1 ; + cfg.c_compr_seg_size =1024U *EROFS_BLKSIZ ;
We don't need that limit by default, so
cfg.c_compr_segsize = -1ULL;
Quoted 30 lines
lib/config.c +0
-0}void erofs_show_config (void )mkfs/main.c +4
-1@@ -61,6 +61,7 @@ static void usage(void)" -x# set xattr tolerance to # (< 0, disable xattrs; default 2)\n" " -EX[,...] X=extended options\n" " -T# set a fixed UNIX timestamp # to all files\n" +" -S# set the max input stream size # to one compress\n" " --exclude-path=X avoid including file X (X = exact literal path)\n" " --exclude-regex=X avoid including files that match X (X = regular expression)\n" #ifdef HAVE_LIBSELINUX@@ -138,7 +139,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])char *endptr;int opt, i; -while ((opt =getopt_long (argc, argv,"d:x:z:E:T:" , +while ((opt =getopt_long (argc, argv,"d:x:z:E:T:S:" , long_options,NULL )) !=-1 ) {switch (opt) {case 'z' :@@ -188,6 +189,19 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])return -EINVAL ; }break ; +case 'S' : + cfg.c_compr_seg_size =strtol (optarg, &endptr,0 );
mkfs/main.c static int mkfs_parse_options_cfg(int argc, char *argv[]) · line 203
Quoted 10 lines
mkfs/main.c +10
-0+if (*endptr !='\0' ) { +erofs_err ("invalid compress segment size %s" , + optarg); +return -EINVAL ; + } +if (cfg.c_compr_seg_size %EROFS_BLKSIZ !=0 ) { +erofs_err ("segment size:%u should be align to %u" , + cfg.c_compr_seg_size,EROFS_BLKSIZ ); +return -EINVAL ; + }
if (!cfg.c_compr_segsize)
cfg.c_compr_segsize = -1ULL;
else if (cfg.c_compr_segsize % EROFS_BLKSIZ) {
erofs_err("segmentsize %u should be aligned with blocksize %u",
cfg.c_compr_seg_size, EROFS_BLKSIZ);
return -EINVAL;
}Thanks,
Gao Xiang
PATCH v5 1/1 erofs-utils: introduce segment compression
In reply to Li Guifu via Linux-erofsSupport segment compression which seperates files in several logic units (segments) and each segment is compressed independently.
Advantages:
- more friendly for data differencing;
- it can also be used for parallel compression in the same file later.
--- include/erofs/config.h | 1 + lib/compress.c | 16 ++++++++++++++-- lib/config.c | 1 + man/mkfs.erofs.1 | 4 ++++ mkfs/main.c | 16 +++++++++++++++- 5 files changed, 35 insertions(+), 3 deletions(-)
include/erofs/config.h +1 -0
@@ -36,6 +36,7 @@ struct erofs_configure {char *c_src_path;char *c_compr_alg_master;int c_compr_level_master; +u64 c_compr_seg_size;int c_force_inodeversion;/* < 0, xattr disabled and INT_MAX, always use inline xattrs */ int c_inline_xattr_tolerance;
lib/compress.c +14 -2
@@ -32,6 +32,7 @@ struct z_erofs_vle_compress_ctx {erofs_blk_t blkaddr;/* pointing to the next blkaddr */ u16 clusterofs; +u64 segavail; };#define Z_EROFS_LEGACY_MAP_HEADER_SIZE \
@@ -158,13 +159,19 @@ static int vle_compress_one(struct erofs_inode *inode,while (len) {bool raw; + count =min_t (u64, len, ctx->segavail); +if (ctx->segavail <=EROFS_BLKSIZ ) { +if (len < ctx->segavail && !final) +break ; + goto nocompression; + } +if (len <=EROFS_BLKSIZ ) {if (final) goto nocompression;break ; } - count = len; ret =erofs_compress_destsize (h, compressionlevel, ctx->queue + ctx->head, &count, dst,EROFS_BLKSIZ );
@@ -174,8 +181,9 @@ static int vle_compress_one(struct erofs_inode *inode, inode->i_srcpath, erofs_strerror(ret)); } + count = len; nocompression: - ret =write_uncompressed_block (ctx, &len, dst); + ret =write_uncompressed_block (ctx, &count, dst);if (ret <0 )return ret; count = ret;
@@ -202,6 +210,9 @@ nocompression: ++ctx->blkaddr; len -= count; + ctx->segavail -= count; +if (!ctx->segavail) + ctx->segavail = cfg.c_compr_seg_size;if (!final && ctx->head >=EROFS_CONFIG_COMPR_MAX_SZ ) { const unsigned int qh_aligned =
@@ -422,6 +433,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode) ctx.head = ctx.tail =0 ; ctx.clusterofs =0 ; remaining = inode->i_size; + ctx.segavail = cfg.c_compr_seg_size;while (remaining) { const u64 readcount = min_t(u64, remaining,
lib/config.c +1 -0
@@ -23,6 +23,7 @@ void erofs_init_configure(void) cfg.c_force_inodeversion =0 ; cfg.c_inline_xattr_tolerance =2 ; cfg.c_unix_timestamp =-1 ; + cfg.c_compr_seg_size =UINT64_MAX ; }void erofs_show_config (void )
man/mkfs.erofs.1 +4 -0
@@ -52,6 +52,10 @@ Forcely generate extended inodes (64-byte inodes) to output. Set all files to the given UNIX timestamp. Reproducible builds requires setting all to a specific one. .TP +.BI "\-S " # +Set the max input stream size at one compression. The default is unsigned 64bit MAX. +It must be algin to EROFS block size(4096). +.TP .BI "\-\-exclude-path=" path Ignore file that matches the exact literal path. You may give multiple `--exclude-path' options.
mkfs/main.c +15 -1
@@ -61,6 +61,7 @@ static void usage(void)" -x# set xattr tolerance to # (< 0, disable xattrs; default 2)\n" " -EX[,...] X=extended options\n" " -T# set a fixed UNIX timestamp # to all files\n" +" -S# set the max input stream size # at one compression\n" " --exclude-path=X avoid including file X (X = exact literal path)\n" " --exclude-regex=X avoid including files that match X (X = regular expression)\n" #ifdef HAVE_LIBSELINUX
@@ -138,7 +139,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])char *endptr;int opt, i; -while ((opt =getopt_long (argc, argv,"d:x:z:E:T:" , +while ((opt =getopt_long (argc, argv,"d:x:z:E:T:S:" , long_options,NULL )) !=-1 ) {switch (opt) {case 'z' :
@@ -188,6 +189,19 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])return -EINVAL ; }break ; +case 'S' : + cfg.c_compr_seg_size =strtoll (optarg, &endptr,0 ); +if (*endptr !='\0' || !cfg.c_compr_seg_size) { +erofs_err ("invalid compress segment size %s" , + optarg); +return -EINVAL ; + } +if (cfg.c_compr_seg_size %EROFS_BLKSIZ ) { +erofs_err ("segment size:%" PRIu64" should be align to %u" , + cfg.c_compr_seg_size,EROFS_BLKSIZ ); +return -EINVAL ; + } +break ;case 2 : opt =erofs_parse_exclude_path (optarg, false);if (opt) {
Signature · 1 line
2.17.1
Quoted 21 lines · Li Guifu via Linux-erofs
Support segment compression which seperates files in several logic units (segments) and each segment is compressed independently.
Advantages:
- more friendly for data differencing;
- it can also be used for parallel compression in the same file later.Signed-off-by Li Guifu <[email protected]>--- include/erofs/config.h | 1 + lib/compress.c | 16 ++++++++++++++-- lib/config.c | 1 + man/mkfs.erofs.1 | 4 ++++ mkfs/main.c | 16 +++++++++++++++- 5 files changed, 35 insertions(+), 3 deletions(-)include/erofs/config.h +0
-0
include/erofs/config.h struct erofs_configure { · line 39
Quoted 4 lines
include/erofs/config.h +1
-0@@ -36,6 +36,7 @@ struct erofs_configure {char *c_src_path;char *c_compr_alg_master;int c_compr_level_master; +u64 c_compr_seg_size;
Could you please move this variable up a bit? Thanks.
char *c_compr_alg_master;
u64 c_compr_seg_size;
int c_compr_level_master;
Quoted 16 lines
include/erofs/config.h +0
-0int c_force_inodeversion;/* < 0, xattr disabled and INT_MAX, always use inline xattrs */ int c_inline_xattr_tolerance;lib/compress.c +1
-0@@ -32,6 +32,7 @@ struct z_erofs_vle_compress_ctx {erofs_blk_t blkaddr;/* pointing to the next blkaddr */ u16 clusterofs; +u64 segavail; };#define Z_EROFS_LEGACY_MAP_HEADER_SIZE \
lib/compress.c static int vle_compress_one(struct erofs_inode *inode, · line 160
Quoted 2 lines
lib/compress.c +0
-0@@ -158,13 +159,19 @@ static int vle_compress_one(struct erofs_inode *inode,while (len) {bool raw;
unsigned int limit;
lib/compress.c static int vle_compress_one(struct erofs_inode *inode, · line 162
Quoted 2 lines
lib/compress.c +1
-0+ count =min_t (u64, len, ctx->segavail);
kill this line.
lib/compress.c static int vle_compress_one(struct erofs_inode *inode, · line 165
Quoted 3 lines
lib/compress.c +3
-0+if (ctx->segavail <=EROFS_BLKSIZ ) { +if (len < ctx->segavail && !final) +break ;
limit = ctx->segavail;
lib/compress.c static int vle_compress_one(struct erofs_inode *inode, · line 167
Quoted 10 lines
lib/compress.c +3
-1+ goto nocompression; + } +if (len <=EROFS_BLKSIZ ) {if (final) goto nocompression;break ; } - count = len;
count = min_t(u64, len, ctx->segavail);
lib/compress.c static int vle_compress_one(struct erofs_inode *inode, · line 184
Quoted 8 lines
lib/compress.c +1
-0ret =erofs_compress_destsize (h, compressionlevel, ctx->queue + ctx->head, &count, dst,EROFS_BLKSIZ );@@ -174,8 +181,9 @@ static int vle_compress_one(struct erofs_inode *inode, inode->i_srcpath, erofs_strerror(ret)); } + count = len;
kill this line and add limit = EROFS_BLKSIZ;
lib/compress.c static int vle_compress_one(struct erofs_inode *inode, · line 186
Quoted 3 lines
lib/compress.c +1
-1nocompression: - ret =write_uncompressed_block (ctx, &len, dst); + ret =write_uncompressed_block (ctx, &count, dst);
ret = write_uncompressed_block(ctx, &count, limit, dst);
and update write_uncompressed_block as well.
Quoted 26 lines
lib/compress.c +4
-0if (ret <0 )return ret; count = ret;@@ -202,6 +210,9 @@ nocompression: ++ctx->blkaddr; len -= count; + ctx->segavail -= count; +if (!ctx->segavail) + ctx->segavail = cfg.c_compr_seg_size;if (!final && ctx->head >=EROFS_CONFIG_COMPR_MAX_SZ ) { const unsigned int qh_aligned =@@ -422,6 +433,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode) ctx.head = ctx.tail =0 ; ctx.clusterofs =0 ; remaining = inode->i_size; + ctx.segavail = cfg.c_compr_seg_size;while (remaining) { const u64 readcount = min_t(u64, remaining,lib/config.c +0
-0
lib/config.c void erofs_init_configure(void) · line 26
Quoted 4 lines
lib/config.c +1
-0@@ -23,6 +23,7 @@ void erofs_init_configure(void) cfg.c_force_inodeversion =0 ; cfg.c_inline_xattr_tolerance =2 ; cfg.c_unix_timestamp =-1 ; + cfg.c_compr_seg_size =UINT64_MAX ;
cfg.c_compr_seg_size = -1;
since it is a very simple way to assign UINT_MAX by implicit sign extension without taking care for the specific data type.
Quoted 8 lines
lib/config.c +0
-0}void erofs_show_config (void )man/mkfs.erofs.1 +0
-0
man/mkfs.erofs.1 Forcely generate extended inodes (64-byte inodes) to output. · line 57
Quoted 6 lines
man/mkfs.erofs.1 +3
-0@@ -52,6 +52,10 @@ Forcely generate extended inodes (64-byte inodes) to output. Set all files to the given UNIX timestamp. Reproducible builds requires setting all to a specific one. .TP +.BI "\-S " # +Set the max input stream size at one compression. The default is unsigned 64bit MAX. +It must be algin to EROFS block size(4096).
it's hard for end users to type "max 64-bit unsigned value"...
I'd suggest "Set max input stream size for each individual segment (disabled if 0). The default value is 0. It should be aligned with blocksize."
mkfs/main.c static void usage(void) · line 64
Quoted 13 lines
man/mkfs.erofs.1 +1
-0+.TP .BI "\-\-exclude-path=" path Ignore file that matches the exact literal path. You may give multiple `--exclude-path' options.mkfs/main.c +1
-0@@ -61,6 +61,7 @@ static void usage(void)" -x# set xattr tolerance to # (< 0, disable xattrs; default 2)\n" " -EX[,...] X=extended options\n" " -T# set a fixed UNIX timestamp # to all files\n" +" -S# set the max input stream size # at one compression\n"
-S# Set max input stream size # for each individual segment\n
Quoted 13 lines
mkfs/main.c +1
-1" --exclude-path=X avoid including file X (X = exact literal path)\n" " --exclude-regex=X avoid including files that match X (X = regular expression)\n" #ifdef HAVE_LIBSELINUX@@ -138,7 +139,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])char *endptr;int opt, i; -while ((opt =getopt_long (argc, argv,"d:x:z:E:T:" , +while ((opt =getopt_long (argc, argv,"d:x:z:E:T:S:" , long_options,NULL )) !=-1 ) {switch (opt) {case 'z' :
mkfs/main.c static int mkfs_parse_options_cfg(int argc, char *argv[]) · line 194
Quoted 6 lines
mkfs/main.c +3
-0@@ -188,6 +189,19 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])return -EINVAL ; }break ; +case 'S' : + cfg.c_compr_seg_size =strtoll (optarg, &endptr,0 ); +if (*endptr !='\0' || !cfg.c_compr_seg_size) {
Disable this if cfg.c_compr_seg_size == 0
mkfs/main.c static int mkfs_parse_options_cfg(int argc, char *argv[]) · line 200
Quoted 6 lines
mkfs/main.c +6
-0+erofs_err ("invalid compress segment size %s" , + optarg); +return -EINVAL ; + } +if (cfg.c_compr_seg_size %EROFS_BLKSIZ ) { + erofs_err("segment size:%"PRIu64" should be align to %u",
Could you follow my advice in the previous reply?
Although I'm not good at English, but I don't think the above message is _reasonable_.
Thanks,
Gao Xiang
Quoted 10 lines
mkfs/main.c +4
-1+ cfg.c_compr_seg_size, EROFS_BLKSIZ); +return -EINVAL ; + } +break ;case 2 : opt =erofs_parse_exclude_path (optarg, false);if (opt) { --2.17.1
PATCH v6 1/1 erofs-utils: introduce segment limits compression
In reply to Li Guifu via Linux-erofsSupport segment compression which seperates files in several logic units (segments) and each segment is compressed independently.
Advantages:
- more friendly for data differencing;
- it can also be used for parallel compression in the same file later.
---
Changes since v3 suggest by Gao Xiang<[email protected]>:
- add a limits varialbe to give the limits size in the write_uncompress_block
- Set comments more readable
include/erofs/config.h | 1 + lib/compress.c | 29 +++++++++++++++++++++-------- lib/config.c | 1 + man/mkfs.erofs.1 | 4 ++++ mkfs/main.c | 18 +++++++++++++++++- 5 files changed, 44 insertions(+), 9 deletions(-)
include/erofs/config.h +1 -0
@@ -35,6 +35,7 @@ struct erofs_configure {char *c_img_path;char *c_src_path;char *c_compr_alg_master; +u64 c_compr_seg_size;int c_compr_level_master;int c_force_inodeversion;/* < 0, xattr disabled and INT_MAX, always use inline xattrs */
lib/compress.c +21 -8
@@ -32,6 +32,7 @@ struct z_erofs_vle_compress_ctx {erofs_blk_t blkaddr;/* pointing to the next blkaddr */ u16 clusterofs; +u64 segavail; };#define Z_EROFS_LEGACY_MAP_HEADER_SIZE \
@@ -116,23 +117,21 @@ static void vle_write_indexes(struct z_erofs_vle_compress_ctx *ctx, }static int write_uncompressed_block (struct z_erofs_vle_compress_ctx *ctx, -unsigned int *len, +unsigned int *len,unsigned int limits,char *dst) {int ret; -unsigned int count; +unsigned int count =min (limits, *len);/* write uncompressed data */ /* reset clusterofs to 0 if permitted */ if (!erofs_sb_has_lz4_0padding () && ctx->head >= ctx->clusterofs) { ctx->head -= ctx->clusterofs; *len += ctx->clusterofs; + count += ctx->clusterofs; ctx->clusterofs =0 ; } -/* write uncompressed data */ - count =min (EROFS_BLKSIZ , *len); -memcpy (dst, ctx->queue + ctx->head, count);memset (dst + count,0 ,EROFS_BLKSIZ - count);
@@ -157,14 +156,22 @@ static int vle_compress_one(struct erofs_inode *inode,while (len) {bool raw; +unsigned int limits =EROFS_BLKSIZ ; + +if (ctx->segavail <=EROFS_BLKSIZ ) { +if (len < ctx->segavail && !final) +break ; + + limits = ctx->segavail; + goto nocompression; + }if (len <=EROFS_BLKSIZ ) {if (final) goto nocompression;break ; } - - count = len; + count =min_t (u64, len, ctx->segavail); ret =erofs_compress_destsize (h, compressionlevel, ctx->queue + ctx->head, &count, dst,EROFS_BLKSIZ );
@@ -175,7 +182,7 @@ static int vle_compress_one(struct erofs_inode *inode, erofs_strerror(ret)); } nocompression: - ret =write_uncompressed_block (ctx, &len, dst); + ret =write_uncompressed_block (ctx, &len, limits, dst);if (ret <0 )return ret; count = ret;
@@ -203,6 +210,11 @@ nocompression: ++ctx->blkaddr; len -= count; +if (count >= ctx->segavail) + ctx->segavail = cfg.c_compr_seg_size; +else + ctx->segavail -= count; +if (!final && ctx->head >=EROFS_CONFIG_COMPR_MAX_SZ ) {const unsigned int qh_aligned =round_down (ctx->head,EROFS_BLKSIZ );
@@ -422,6 +434,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode) ctx.head = ctx.tail =0 ; ctx.clusterofs =0 ; remaining = inode->i_size; + ctx.segavail = cfg.c_compr_seg_size;while (remaining) { const u64 readcount = min_t(u64, remaining,
lib/config.c +1 -0
@@ -23,6 +23,7 @@ void erofs_init_configure(void) cfg.c_force_inodeversion =0 ; cfg.c_inline_xattr_tolerance =2 ; cfg.c_unix_timestamp =-1 ; + cfg.c_compr_seg_size =-1UL ; }void erofs_show_config (void )
man/mkfs.erofs.1 +4 -0
@@ -52,6 +52,10 @@ Forcely generate extended inodes (64-byte inodes) to output. Set all files to the given UNIX timestamp. Reproducible builds requires setting all to a specific one. .TP +.BI "\-S " # +Set max input stream size for each individual segment (disabled if 0). +The default value is 0. It should be aligned with blocksize. +.TP .BI "\-\-exclude-path=" path Ignore file that matches the exact literal path. You may give multiple `--exclude-path' options.
mkfs/main.c +17 -1
@@ -61,6 +61,7 @@ static void usage(void)" -x# set xattr tolerance to # (< 0, disable xattrs; default 2)\n" " -EX[,...] X=extended options\n" " -T# set a fixed UNIX timestamp # to all files\n" +" -S# Set max input stream size # for each individual segment\n" " --exclude-path=X avoid including file X (X = exact literal path)\n" " --exclude-regex=X avoid including files that match X (X = regular expression)\n" #ifdef HAVE_LIBSELINUX
@@ -138,7 +139,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])char *endptr;int opt, i; -while ((opt =getopt_long (argc, argv,"d:x:z:E:T:" , +while ((opt =getopt_long (argc, argv,"d:x:z:E:T:S:" , long_options,NULL )) !=-1 ) {switch (opt) {case 'z' :
@@ -188,6 +189,21 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])return -EINVAL ; }break ; +case 'S' : + cfg.c_compr_seg_size =strtoll (optarg, &endptr,0 ); +if (*endptr !='\0' ) { +erofs_err ("invalid compress segment size %s" , + optarg); +return -EINVAL ; + } +if (!cfg.c_compr_seg_size) { + cfg.c_compr_seg_size =-1UL ; + }else if (cfg.c_compr_seg_size %EROFS_BLKSIZ ) { +erofs_err ("segment size:%" PRIu64" should be align to %u" , + cfg.c_compr_seg_size,EROFS_BLKSIZ ); +return -EINVAL ; + } +break ;case 2 : opt =erofs_parse_exclude_path (optarg, false);if (opt) {
Signature · 1 line
2.17.1
PATCH v8 1/1 erofs-utils: introduce segment compression
Support segment compression which seperates files in several logic units (segments) and each segment is compressed independently.
Advantages:
- more friendly for data differencing;
- it can also be used for parallel compression in the same file later.
--- include/erofs/config.h | 1 + lib/compress.c | 47 ++++++++++++++++++++++++++---------------- lib/config.c | 1 + man/mkfs.erofs.1 | 4 ++++ mkfs/main.c | 18 +++++++++++++++- 5 files changed, 52 insertions(+), 19 deletions(-)
include/erofs/config.h +1 -0
@@ -35,6 +35,7 @@ struct erofs_configure {char *c_img_path;char *c_src_path;char *c_compr_alg_master; +u64 c_compr_seg_size;int c_compr_level_master;int c_force_inodeversion;/* < 0, xattr disabled and INT_MAX, always use inline xattrs */
lib/compress.c +29 -18
94 lines not on this page · raw message
lib/config.c +1 -0
@@ -23,6 +23,7 @@ void erofs_init_configure(void) cfg.c_force_inodeversion =0 ; cfg.c_inline_xattr_tolerance =2 ; cfg.c_unix_timestamp =-1 ; + cfg.c_compr_seg_size =-1 ; }void erofs_show_config (void )
man/mkfs.erofs.1 +4 -0
@@ -52,6 +52,10 @@ Forcely generate extended inodes (64-byte inodes) to output. Set all files to the given UNIX timestamp. Reproducible builds requires setting all to a specific one. .TP +.BI "\-S " # +Set max input stream size for each individual segment (disabled if 0). +The default value is 0. It should be aligned with blocksize. +.TP .BI "\-\-exclude-path=" path Ignore file that matches the exact literal path. You may give multiple `--exclude-path' options.
mkfs/main.c +17 -1
@@ -61,6 +61,7 @@ static void usage(void)" -x# set xattr tolerance to # (< 0, disable xattrs; default 2)\n" " -EX[,...] X=extended options\n" " -T# set a fixed UNIX timestamp # to all files\n" +" -S# Set max input stream size # for each individual segment\n" " --exclude-path=X avoid including file X (X = exact literal path)\n" " --exclude-regex=X avoid including files that match X (X = regular expression)\n" #ifdef HAVE_LIBSELINUX
@@ -138,7 +139,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])char *endptr;int opt, i; -while ((opt =getopt_long (argc, argv,"d:x:z:E:T:" , +while ((opt =getopt_long (argc, argv,"d:x:z:E:T:S:" , long_options,NULL )) !=-1 ) {switch (opt) {case 'z' :
@@ -188,6 +189,21 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])return -EINVAL ; }break ; +case 'S' : + cfg.c_compr_seg_size =strtoll (optarg, &endptr,0 ); +if (*endptr !='\0' ) { +erofs_err ("invalid compress segment size %s" , + optarg); +return -EINVAL ; + } +if (!cfg.c_compr_seg_size) { + cfg.c_compr_seg_size =-1 ; + }else if (cfg.c_compr_seg_size %EROFS_BLKSIZ ) { +erofs_err ("segment size:%" PRIu64" should be align with %u" , + cfg.c_compr_seg_size,EROFS_BLKSIZ ); +return -EINVAL ; + } +break ;case 2 : opt =erofs_parse_exclude_path (optarg, false);if (opt) {
Signature · 1 line
2.17.1
On Sun, Jun 21, 2020 at 08:27:45PM +0800, Li Guifu via Linux-erofs wrote:
...
lib/compress.c static int vle_compress_one(struct erofs_inode *inode, · line 181
Quoted 9 lines
nocompression:
lib/compress.c +7
-1- ret =write_uncompressed_block (ctx, &len, dst); +/* reset clusterofs to 0 if permitted */ +if (!erofs_sb_has_lz4_0padding () && + ctx->head >= ctx->clusterofs) { + ctx->head -= ctx->clusterofs; + len += ctx->clusterofs; + limit += ctx->clusterofs; + ctx->clusterofs =0 ;
TL;DR: it seems still buggy here.
Pls rethink carefully and send a usable patch then...
Thanks,
Gao Xiang
PATCH v10 1/1 erofs-utils: introduce segment compression
Support segment compression which seperates files in several logic units (segments) and each segment is compressed independently.
Advantages:
- more friendly for data differencing;
- it can also be used for parallel compression in the same file later.
--- include/erofs/config.h | 1 + lib/compress.c | 29 +++++++++++++++++++++++------ lib/config.c | 1 + man/mkfs.erofs.1 | 4 ++++ mkfs/main.c | 16 +++++++++++++++- 5 files changed, 44 insertions(+), 7 deletions(-)
include/erofs/config.h +1 -0
@@ -35,6 +35,7 @@ struct erofs_configure {char *c_img_path;char *c_src_path;char *c_compr_alg_master; +u64 c_compr_seg_size;int c_compr_level_master;int c_force_inodeversion;/* < 0, xattr disabled and INT_MAX, always use inline xattrs */
lib/compress.c +23 -6
@@ -32,6 +32,7 @@ struct z_erofs_vle_compress_ctx {erofs_blk_t blkaddr;/* pointing to the next blkaddr */ u16 clusterofs; +u64 segavail; };#define Z_EROFS_LEGACY_MAP_HEADER_SIZE \
@@ -116,7 +117,7 @@ static void vle_write_indexes(struct z_erofs_vle_compress_ctx *ctx, }static int write_uncompressed_block (struct z_erofs_vle_compress_ctx *ctx, -unsigned int *len, +unsigned int *len,unsigned int *ucomproft,char *dst) {int ret;
@@ -125,14 +126,19 @@ static int write_uncompressed_block(struct z_erofs_vle_compress_ctx *ctx,/* reset clusterofs to 0 if permitted */ if (!erofs_sb_has_lz4_0padding () && ctx->head >= ctx->clusterofs) { + *ucomproft = ctx->clusterofs; ctx->head -= ctx->clusterofs; *len += ctx->clusterofs; ctx->clusterofs =0 ; + count =min (EROFS_BLKSIZ , *len); + }else { + *ucomproft =0 ; + count =min_t (u64, ctx->segavail, *len); +if (count >EROFS_BLKSIZ ) + count =EROFS_BLKSIZ ; }/* write uncompressed data */ - count =min (EROFS_BLKSIZ , *len); -memcpy (dst, ctx->queue + ctx->head, count);memset (dst + count,0 ,EROFS_BLKSIZ - count);
@@ -157,14 +163,21 @@ static int vle_compress_one(struct erofs_inode *inode,while (len) {bool raw; +unsigned int ucomproft =0 ; + +if (ctx->segavail <=EROFS_BLKSIZ ) { +if (len < ctx->segavail && !final) +break ; + + goto nocompression; + }if (len <=EROFS_BLKSIZ ) {if (final) goto nocompression;break ; } - - count = len; + count =min_t (u64, len, ctx->segavail); ret =erofs_compress_destsize (h, compressionlevel, ctx->queue + ctx->head, &count, dst,EROFS_BLKSIZ );
@@ -175,7 +188,7 @@ static int vle_compress_one(struct erofs_inode *inode, erofs_strerror(ret)); } nocompression: - ret =write_uncompressed_block (ctx, &len, dst); + ret =write_uncompressed_block (ctx, &len, &ucomproft, dst);if (ret <0 )return ret; count = ret;
@@ -202,6 +215,9 @@ nocompression: ++ctx->blkaddr; len -= count; + ctx->segavail -= count - ucomproft; +if (!ctx->segavail) + ctx->segavail = cfg.c_compr_seg_size;if (!final && ctx->head >=EROFS_CONFIG_COMPR_MAX_SZ ) { const unsigned int qh_aligned =
@@ -422,6 +438,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode) ctx.head = ctx.tail =0 ; ctx.clusterofs =0 ; remaining = inode->i_size; + ctx.segavail = cfg.c_compr_seg_size;while (remaining) { const u64 readcount = min_t(u64, remaining,
lib/config.c +1 -0
@@ -23,6 +23,7 @@ void erofs_init_configure(void) cfg.c_force_inodeversion =0 ; cfg.c_inline_xattr_tolerance =2 ; cfg.c_unix_timestamp =-1 ; + cfg.c_compr_seg_size =-1 ; }void erofs_show_config (void )
man/mkfs.erofs.1 +4 -0
@@ -52,6 +52,10 @@ Forcely generate extended inodes (64-byte inodes) to output. Set all files to the given UNIX timestamp. Reproducible builds requires setting all to a specific one. .TP +.BI "\-S " # +Set max input stream size for each individual segment (disabled if 0). +The default value is 0. It has be aligned with blocksize. +.TP .BI "\-\-exclude-path=" path Ignore file that matches the exact literal path. You may give multiple `--exclude-path' options.
mkfs/main.c +15 -1
@@ -61,6 +61,7 @@ static void usage(void)" -x# set xattr tolerance to # (< 0, disable xattrs; default 2)\n" " -EX[,...] X=extended options\n" " -T# set a fixed UNIX timestamp # to all files\n" +" -S# Set max input stream size # for each individual segment\n" " --exclude-path=X avoid including file X (X = exact literal path)\n" " --exclude-regex=X avoid including files that match X (X = regular expression)\n" #ifdef HAVE_LIBSELINUX
@@ -138,7 +139,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])char *endptr;int opt, i; -while ((opt =getopt_long (argc, argv,"d:x:z:E:T:" , +while ((opt =getopt_long (argc, argv,"d:x:z:E:T:S:" , long_options,NULL )) !=-1 ) {switch (opt) {case 'z' :
@@ -188,6 +189,19 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])return -EINVAL ; }break ; +case 'S' : + i =strtoll (optarg, &endptr,0 ); +if (*endptr !='\0' ) { +erofs_err ("invalid compress segment size %s" , + optarg); +return -EINVAL ; + } +if (!i) { + cfg.c_compr_seg_size =-1 ; + }else { + cfg.c_compr_seg_size = i *EROFS_BLKSIZ ; + } +break ;case 2 : opt =erofs_parse_exclude_path (optarg, false);if (opt) {
Signature · 1 line
2.17.1
PATCH v11 1/1 erofs-utils: introduce segment compression
Support segment compression which seperates files in several logic units (segments) and each segment is compressed independently.
Advantages:
- more friendly for data differencing;
- it can also be used for parallel compression in the same file later.
---
Changes from v10
- chang variable uncomprofs to clusterofs which only used when write uncompress block
include/erofs/config.h | 1 + lib/compress.c | 30 ++++++++++++++++++++++++------ lib/config.c | 1 + man/mkfs.erofs.1 | 4 ++++ mkfs/main.c | 16 +++++++++++++++- 5 files changed, 45 insertions(+), 7 deletions(-)
include/erofs/config.h +1 -0
@@ -35,6 +35,7 @@ struct erofs_configure {char *c_img_path;char *c_src_path;char *c_compr_alg_master; +u64 c_compr_seg_size;int c_compr_level_master;int c_force_inodeversion;/* < 0, xattr disabled and INT_MAX, always use inline xattrs */
lib/compress.c +24 -6
@@ -32,6 +32,7 @@ struct z_erofs_vle_compress_ctx {erofs_blk_t blkaddr;/* pointing to the next blkaddr */ u16 clusterofs; +u64 segavail; };#define Z_EROFS_LEGACY_MAP_HEADER_SIZE \
@@ -116,7 +117,7 @@ static void vle_write_indexes(struct z_erofs_vle_compress_ctx *ctx, }static int write_uncompressed_block (struct z_erofs_vle_compress_ctx *ctx, -unsigned int *len, +unsigned int *len,unsigned int *clusterofs,char *dst) {int ret;
@@ -125,14 +126,19 @@ static int write_uncompressed_block(struct z_erofs_vle_compress_ctx *ctx,/* reset clusterofs to 0 if permitted */ if (!erofs_sb_has_lz4_0padding () && ctx->head >= ctx->clusterofs) { + *clusterofs = ctx->clusterofs; ctx->head -= ctx->clusterofs; *len += ctx->clusterofs; ctx->clusterofs =0 ; + count =min (EROFS_BLKSIZ , *len); + }else { + *clusterofs =0 ; + count =min_t (u64, ctx->segavail, *len); +if (count >EROFS_BLKSIZ ) + count =EROFS_BLKSIZ ; }/* write uncompressed data */ - count =min (EROFS_BLKSIZ , *len); -memcpy (dst, ctx->queue + ctx->head, count);memset (dst + count,0 ,EROFS_BLKSIZ - count);
@@ -157,14 +163,21 @@ static int vle_compress_one(struct erofs_inode *inode,while (len) {bool raw; +unsigned int clusterofs; + +if (ctx->segavail <=EROFS_BLKSIZ ) { +if (len < ctx->segavail && !final) +break ; + + goto nocompression; + }if (len <=EROFS_BLKSIZ ) {if (final) goto nocompression;break ; } - - count = len; + count =min_t (u64, len, ctx->segavail); ret =erofs_compress_destsize (h, compressionlevel, ctx->queue + ctx->head, &count, dst,EROFS_BLKSIZ );
@@ -175,11 +188,12 @@ static int vle_compress_one(struct erofs_inode *inode, erofs_strerror(ret)); } nocompression: - ret =write_uncompressed_block (ctx, &len, dst); + ret =write_uncompressed_block (ctx, &len, &clusterofs, dst);if (ret <0 )return ret; count = ret; raw = true; + ctx->segavail -= count - clusterofs; }else {/* write compressed data */ erofs_dbg("Writing %u compressed data to block %u",
@@ -194,6 +208,7 @@ nocompression:if (ret)return ret; raw = false; + ctx->segavail -= count; } ctx->head += count;
@@ -202,6 +217,8 @@ nocompression: ++ctx->blkaddr; len -= count; +if (!ctx->segavail) + ctx->segavail = cfg.c_compr_seg_size;if (!final && ctx->head >=EROFS_CONFIG_COMPR_MAX_SZ ) { const unsigned int qh_aligned =
@@ -422,6 +439,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode) ctx.head = ctx.tail =0 ; ctx.clusterofs =0 ; remaining = inode->i_size; + ctx.segavail = cfg.c_compr_seg_size;while (remaining) { const u64 readcount = min_t(u64, remaining,
lib/config.c +1 -0
@@ -23,6 +23,7 @@ void erofs_init_configure(void) cfg.c_force_inodeversion =0 ; cfg.c_inline_xattr_tolerance =2 ; cfg.c_unix_timestamp =-1 ; + cfg.c_compr_seg_size =-1 ; }void erofs_show_config (void )
man/mkfs.erofs.1 +4 -0
@@ -52,6 +52,10 @@ Forcely generate extended inodes (64-byte inodes) to output. Set all files to the given UNIX timestamp. Reproducible builds requires setting all to a specific one. .TP +.BI "\-S " # +Set max input stream size for each individual segment (disabled if 0). +The default value is 0. It has be aligned with blocksize. +.TP .BI "\-\-exclude-path=" path Ignore file that matches the exact literal path. You may give multiple `--exclude-path' options.
mkfs/main.c +15 -1
@@ -61,6 +61,7 @@ static void usage(void)" -x# set xattr tolerance to # (< 0, disable xattrs; default 2)\n" " -EX[,...] X=extended options\n" " -T# set a fixed UNIX timestamp # to all files\n" +" -S# Set max input stream size # for each individual segment\n" " --exclude-path=X avoid including file X (X = exact literal path)\n" " --exclude-regex=X avoid including files that match X (X = regular expression)\n" #ifdef HAVE_LIBSELINUX
@@ -138,7 +139,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])char *endptr;int opt, i; -while ((opt =getopt_long (argc, argv,"d:x:z:E:T:" , +while ((opt =getopt_long (argc, argv,"d:x:z:E:T:S:" , long_options,NULL )) !=-1 ) {switch (opt) {case 'z' :
@@ -188,6 +189,19 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])return -EINVAL ; }break ; +case 'S' : + i =strtoll (optarg, &endptr,0 ); +if (*endptr !='\0' ) { +erofs_err ("invalid compress segment size %s" , + optarg); +return -EINVAL ; + } +if (!i) { + cfg.c_compr_seg_size =-1 ; + }else { + cfg.c_compr_seg_size = i *EROFS_BLKSIZ ; + } +break ;case 2 : opt =erofs_parse_exclude_path (optarg, false);if (opt) {
Signature · 1 line
2.17.1
Hi Guifu,
Quoted 7 lines · Li Guifu via Linux-erofs
Support segment compression which seperates files in several logic units (segments) and each segment is compressed independently.
Advantages:
- more friendly for data differencing;
- it can also be used for parallel compression in the same file later.
Quoted 5 lines
Signed-off-by Li Guifu <[email protected]>---
Changes from v10
- chang variable uncomprofs to clusterofs which only used when write uncompress block
Could you please test the following patch if you're available? Does it work?
From 0436ed04717853351e13d68db6f170f60e25fc12 Mon Sep 17 00:00:00 2001
From: Li Guifu <[email protected]>
Date: Sun, 5 Jul 2020 16:32:30 +0800
Subject: [PATCH v12] erofs-utils: introduce segment compression
Support segment compression which seperates files in several logic units (segments) and each segment is compressed independently.
Advantages:
- more friendly for data differencing;
- it can also be used for parallel compression in the same file later.
--- include/erofs/config.h | 2 ++ lib/compress.c | 38 ++++++++++++++++++++++++++++++-------- lib/config.c | 1 + man/mkfs.erofs.1 | 4 ++++ mkfs/main.c | 12 +++++++++++- 5 files changed, 48 insertions(+), 9 deletions(-)
include/erofs/config.h +2 -0
@@ -35,6 +35,8 @@ struct erofs_configure {char *c_img_path;char *c_src_path;char *c_compr_alg_master; +u64 c_compr_segsize; +int c_compr_level_master;int c_force_inodeversion;/* < 0, xattr disabled and INT_MAX, always use inline xattrs */
lib/compress.c +30 -8
@@ -32,6 +32,7 @@ struct z_erofs_vle_compress_ctx {erofs_blk_t blkaddr;/* pointing to the next blkaddr */ u16 clusterofs; +u64 segavail; };#define Z_EROFS_LEGACY_MAP_HEADER_SIZE \
@@ -124,24 +125,33 @@ static int write_uncompressed_block(struct z_erofs_vle_compress_ctx *ctx,/* reset clusterofs to 0 if permitted */ if (!erofs_sb_has_lz4_0padding () && - ctx->head >= ctx->clusterofs) { + ctx->clusterofs && ctx->head >= ctx->clusterofs) { ctx->head -= ctx->clusterofs; *len += ctx->clusterofs; + + ctx->segavail += ctx->clusterofs; +DBG_BUGON (ctx->segavail > cfg.c_compr_segsize); + +DBG_BUGON (ctx->segavail <EROFS_BLKSIZ ); +/* so only *len will be the candidate instead of segavail */ + count = *len; + ctx->clusterofs =0 ; + }else { + count =min_t (u64, ctx->segavail, *len); } -/* write uncompressed data */ - count =min (EROFS_BLKSIZ , *len); +if (count >EROFS_BLKSIZ ) + count =EROFS_BLKSIZ ; +/* fill zero if the uncompressed block isn't full */ memcpy (dst, ctx->queue + ctx->head, count);memset (dst + count,0 ,EROFS_BLKSIZ - count);erofs_dbg ("Writing %u uncompressed data to block %u" , count, ctx->blkaddr); ret =blk_write (dst, ctx->blkaddr,1 ); -if (ret) -return ret; -return count; +return ret ? ret : count; } static int vle_compress_one(struct erofs_inode *inode,
@@ -158,13 +168,20 @@ static int vle_compress_one(struct erofs_inode *inode,while (len) {bool raw; +if (ctx->segavail <=EROFS_BLKSIZ ) { +if (len < ctx->segavail && !final) +break ; + + goto nocompression; + } +if (len <=EROFS_BLKSIZ ) {if (final) goto nocompression;break ; } - count = len; + count =min_t (u64, len, ctx->segavail); ret =erofs_compress_destsize (h, compressionlevel, ctx->queue + ctx->head, &count, dst,EROFS_BLKSIZ );
@@ -195,8 +212,12 @@ nocompression:return ret; raw = false; } - ctx->head += count; +DBG_BUGON (ctx->segavail < count); + ctx->segavail -= count; +if (!ctx->segavail) + ctx->segavail = cfg.c_compr_segsize; +/* write compression indexes for this blkaddr */ vle_write_indexes (ctx, count, raw);
@@ -421,6 +442,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode) ctx.metacur = compressmeta +Z_EROFS_LEGACY_MAP_HEADER_SIZE ; ctx.head = ctx.tail =0 ; ctx.clusterofs =0 ; + ctx.segavail = cfg.c_compr_segsize; remaining = inode->i_size;while (remaining) {
lib/config.c +1 -0
@@ -23,6 +23,7 @@ void erofs_init_configure(void) cfg.c_force_inodeversion =0 ; cfg.c_inline_xattr_tolerance =2 ; cfg.c_unix_timestamp =-1 ; + cfg.c_compr_segsize =-1 ; }void erofs_show_config (void )
man/mkfs.erofs.1 +4 -0
@@ -48,6 +48,10 @@ Forcely generate compact inodes (32-byte inodes) to output. Forcely generate extended inodes (64-byte inodes) to output. .RE .TP +.BI "\-S " # +Set maximum blocks for each individual compress segment. +The default is 0 (disabled). +.TP .BI "\-T " # Set all files to the given UNIX timestamp. Reproducible builds requires setting all to a specific one.
mkfs/main.c +11 -1
@@ -61,6 +61,7 @@ static void usage(void)" -x# set xattr tolerance to # (< 0, disable xattrs; default 2)\n" " -EX[,...] X=extended options\n" " -T# set a fixed UNIX timestamp # to all files\n" +" -S# Set maximum blocks for each individual compress segment\n" " --exclude-path=X avoid including file X (X = exact literal path)\n" " --exclude-regex=X avoid including files that match X (X = regular expression)\n" #ifdef HAVE_LIBSELINUX
@@ -138,7 +139,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])char *endptr;int opt, i; -while ((opt =getopt_long (argc, argv,"d:x:z:E:T:" , +while ((opt =getopt_long (argc, argv,"d:x:z:E:T:S:" , long_options,NULL )) !=-1 ) {switch (opt) {case 'z' :
@@ -188,6 +189,15 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])return -EINVAL ; }break ; +case 'S' : + i =strtoll (optarg, &endptr,0 ); +if (*endptr !='\0' ) { +erofs_err ("invalid blocks per compress segment %s" , + optarg); +return -EINVAL ; + } + cfg.c_compr_segsize = i ?blknr_to_addr (i) :-1 ; +break ;case 2 : opt =erofs_parse_exclude_path (optarg, false);if (opt) {
Signature · 1 line
2.24.0
Guifu, some progress on this as well?
Quoted 219 lines · Gao Xiang via Linux-erofs
Hi Guifu,
Quoted 7 lines · Li Guifu via Linux-erofs
Support segment compression which seperates files in several logic units (segments) and each segment is compressed independently.
Advantages:
- more friendly for data differencing;
- it can also be used for parallel compression in the same file later.Quoted 5 lines
Signed-off-by Li Guifu <[email protected]>---
Changes from v10
- chang variable uncomprofs to clusterofs which only used when write uncompress blockCould you please test the following patch if you're available? Does it work?
From
0436ed04717853351e13d68db6f170f60e25fc12Mon Sep 17 00:00:00 2001
From: Li Guifu <[email protected]>
Date: Sun, 5 Jul 2020 16:32:30 +0800
Subject: [PATCH v12] erofs-utils: introduce segment compressionSupport segment compression which seperates files in several logic units (segments) and each segment is compressed independently.
Advantages:
- more friendly for data differencing;
- it can also be used for parallel compression in the same file later.Signed-off-by Li Guifu <[email protected]>Signed-off-by Gao Xiang <[email protected]>--- include/erofs/config.h | 2 ++ lib/compress.c | 38 ++++++++++++++++++++++++++++++-------- lib/config.c | 1 + man/mkfs.erofs.1 | 4 ++++ mkfs/main.c | 12 +++++++++++- 5 files changed, 48 insertions(+), 9 deletions(-)include/erofs/config.h +2
-0@@ -35,6 +35,8 @@ struct erofs_configure {char *c_img_path;char *c_src_path;char *c_compr_alg_master; +u64 c_compr_segsize; +int c_compr_level_master;int c_force_inodeversion;/* < 0, xattr disabled and INT_MAX, always use inline xattrs */ lib/compress.c +30
-8@@ -32,6 +32,7 @@ struct z_erofs_vle_compress_ctx {erofs_blk_t blkaddr;/* pointing to the next blkaddr */ u16 clusterofs; +u64 segavail; };#define Z_EROFS_LEGACY_MAP_HEADER_SIZE \@@ -124,24 +125,33 @@ static int write_uncompressed_block(struct z_erofs_vle_compress_ctx *ctx,/* reset clusterofs to 0 if permitted */ if (!erofs_sb_has_lz4_0padding () && - ctx->head >= ctx->clusterofs) { + ctx->clusterofs && ctx->head >= ctx->clusterofs) { ctx->head -= ctx->clusterofs; *len += ctx->clusterofs; + + ctx->segavail += ctx->clusterofs; +DBG_BUGON (ctx->segavail > cfg.c_compr_segsize); + +DBG_BUGON (ctx->segavail <EROFS_BLKSIZ ); +/* so only *len will be the candidate instead of segavail */ + count = *len; + ctx->clusterofs =0 ; + }else { + count =min_t (u64, ctx->segavail, *len); } -/* write uncompressed data */ - count =min (EROFS_BLKSIZ , *len); +if (count >EROFS_BLKSIZ ) + count =EROFS_BLKSIZ ; +/* fill zero if the uncompressed block isn't full */ memcpy (dst, ctx->queue + ctx->head, count);memset (dst + count,0 ,EROFS_BLKSIZ - count);erofs_dbg ("Writing %u uncompressed data to block %u" , count, ctx->blkaddr); ret =blk_write (dst, ctx->blkaddr,1 ); -if (ret) -return ret; -return count; +return ret ? ret : count; } static int vle_compress_one(struct erofs_inode *inode,@@ -158,13 +168,20 @@ static int vle_compress_one(struct erofs_inode *inode,while (len) {bool raw; +if (ctx->segavail <=EROFS_BLKSIZ ) { +if (len < ctx->segavail && !final) +break ; + + goto nocompression; + } +if (len <=EROFS_BLKSIZ ) {if (final) goto nocompression;break ; } - count = len; + count =min_t (u64, len, ctx->segavail); ret =erofs_compress_destsize (h, compressionlevel, ctx->queue + ctx->head, &count, dst,EROFS_BLKSIZ );@@ -195,8 +212,12 @@ nocompression:return ret; raw = false; } - ctx->head += count; +DBG_BUGON (ctx->segavail < count); + ctx->segavail -= count; +if (!ctx->segavail) + ctx->segavail = cfg.c_compr_segsize; +/* write compression indexes for this blkaddr */ vle_write_indexes (ctx, count, raw);@@ -421,6 +442,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode) ctx.metacur = compressmeta +Z_EROFS_LEGACY_MAP_HEADER_SIZE ; ctx.head = ctx.tail =0 ; ctx.clusterofs =0 ; + ctx.segavail = cfg.c_compr_segsize; remaining = inode->i_size;while (remaining) {lib/config.c +1
-0@@ -23,6 +23,7 @@ void erofs_init_configure(void) cfg.c_force_inodeversion =0 ; cfg.c_inline_xattr_tolerance =2 ; cfg.c_unix_timestamp =-1 ; + cfg.c_compr_segsize =-1 ; }void erofs_show_config (void )man/mkfs.erofs.1 +4
-0@@ -48,6 +48,10 @@ Forcely generate compact inodes (32-byte inodes) to output. Forcely generate extended inodes (64-byte inodes) to output. .RE .TP +.BI "\-S " # +Set maximum blocks for each individual compress segment. +The default is 0 (disabled). +.TP .BI "\-T " # Set all files to the given UNIX timestamp. Reproducible builds requires setting all to a specific one.mkfs/main.c +11
-1@@ -61,6 +61,7 @@ static void usage(void)" -x# set xattr tolerance to # (< 0, disable xattrs; default 2)\n" " -EX[,...] X=extended options\n" " -T# set a fixed UNIX timestamp # to all files\n" +" -S# Set maximum blocks for each individual compress segment\n" " --exclude-path=X avoid including file X (X = exact literal path)\n" " --exclude-regex=X avoid including files that match X (X = regular expression)\n" #ifdef HAVE_LIBSELINUX@@ -138,7 +139,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])char *endptr;int opt, i; -while ((opt =getopt_long (argc, argv,"d:x:z:E:T:" , +while ((opt =getopt_long (argc, argv,"d:x:z:E:T:S:" , long_options,NULL )) !=-1 ) {switch (opt) {case 'z' :@@ -188,6 +189,15 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])return -EINVAL ; }break ; +case 'S' : + i =strtoll (optarg, &endptr,0 ); +if (*endptr !='\0' ) { +erofs_err ("invalid blocks per compress segment %s" , + optarg); +return -EINVAL ; + } + cfg.c_compr_segsize = i ?blknr_to_addr (i) :-1 ; +break ;case 2 : opt =erofs_parse_exclude_path (optarg, false);if (opt) {Signature · 1 line
2.24.0