btrfs/zstd: enable negative compression levels mount option
Subsystems (2)
PATCH v3 1/1 btrfs/zstd: enable negative compression levels mount option
This patch allows using the fast modes (negative compression levels) of zstd as a mount option.
As per the results, the compression ratio is lower:
for level in {-10..-1} 1 2 3; \
do printf "level %3d\n" $level; \
mount -o compress=zstd:$level /dev/sdb /mnt/test/; \
grep sdb /proc/mounts; \
cp -r /usr/bin /mnt/test/; sync; compsize /mnt/test/bin; \
cp -r /usr/share/doc /mnt/test/; sync; compsize /mnt/test/doc; \
cp enwik9 /mnt/test/; sync; compsize /mnt/test/enwik9; \
cp linux-6.13.tar /mnt/test/; sync; compsize /mnt/test/linux-6.13.tar; \
rm -r /mnt/test/{bin,doc,enwik9,linux-6.13.tar}; \
umount /mnt/test/; \
done |& tee results | \
awk '/^level/{print}/^TOTAL/{print$3"\t"$2" |"}' | paste - - - - -
266M bin | 45M doc | 953M wiki | 1.4G source =============================+===============+===============+===============+ level -10 171M 64% | 28M 62% | 631M 66% | 512M 34% | level -9 165M 62% | 27M 61% | 615M 64% | 493M 33% | level -8 161M 60% | 27M 59% | 598M 62% | 475M 32% | level -7 155M 58% | 26M 58% | 582M 61% | 457M 30% | level -6 151M 56% | 25M 56% | 565M 59% | 437M 29% | level -5 145M 54% | 24M 55% | 545M 57% | 417M 28% | level -4 139M 52% | 23M 52% | 520M 54% | 391M 26% | level -3 135M 50% | 22M 50% | 495M 51% | 369M 24% | level -2 127M 47% | 22M 48% | 470M 49% | 349M 23% | level -1 120M 45% | 21M 47% | 452M 47% | 332M 22% | level 1 110M 41% | 17M 39% | 362M 38% | 290M 19% | level 2 106M 40% | 17M 38% | 349M 36% | 288M 19% | level 3 104M 39% | 16M 37% | 340M 35% | 276M 18% |
---
Changes in v3:
* Correct the accidentally replaced string in clip_level() function.
Changes in v2:
* Set the minimal level to -10 and add a clip_level() helper as suggested by Dave.
* Add more filetypes to comparison in commit message.
The enwik9 test corpus is a dump of wikipedia site as used by various benchmarks.
--- fs/btrfs/compression.c | 18 +++++++--------- fs/btrfs/compression.h | 25 +++++++--------------- fs/btrfs/fs.h | 2 +- fs/btrfs/inode.c | 2 +- fs/btrfs/super.c | 2 +- fs/btrfs/zlib.c | 1 + fs/btrfs/zstd.c | 48 +++++++++++++++++++++++++----------------- 7 files changed, 49 insertions(+), 49 deletions(-)
fs/btrfs/compression.c +8 -10
@@ -740,7 +740,7 @@ static const struct btrfs_compress_op * const btrfs_compress_op[] = { &btrfs_zstd_compress, }; -static struct list_head *alloc_workspace (int type,unsigned int level) +static struct list_head *alloc_workspace (int type,int level) {switch (type) {case BTRFS_COMPRESS_NONE :return alloc_heuristic_ws ();
@@ -818,7 +818,7 @@ static void btrfs_cleanup_workspace_manager(int type)* Preallocation makes a forward progress guarantees and we do not return * errors. */ -struct list_head *btrfs_get_workspace (int type,unsigned int level) +struct list_head *btrfs_get_workspace (int type,int level) {struct workspace_manager *wsm;struct list_head *workspace;
@@ -968,14 +968,14 @@ static void put_workspace(int type, struct list_head *ws)* Adjust @level according to the limits of the compression algorithm or * fallback to default */ -static unsigned int btrfs_compress_set_level (int type,unsigned level) +static int btrfs_compress_set_level (unsigned int type,int level) {const struct btrfs_compress_op *ops = btrfs_compress_op[type];if (level ==0 ) level = ops->default_level;else - level =min (level, ops->max_level); + level =min (max (level, ops->min_level), ops->max_level);return level; }
@@ -1023,12 +1023,10 @@ int btrfs_compress_filemap_get_folio(struct address_space *mapping, u64 start,* @total_out is an in/out parameter, must be set to the input length and will * be also used to return the total number of compressed bytes */ -int btrfs_compress_folios (unsigned int type_level,struct address_space *mapping, +int btrfs_compress_folios (unsigned int type,int level,struct address_space *mapping,u64 start,struct folio **folios,unsigned long *out_folios,unsigned long *total_in,unsigned long *total_out) { -int type =btrfs_compress_type (type_level); -int level =btrfs_compress_level (type_level);const unsigned long orig_len = *total_out;struct list_head *workspace;int ret;
@@ -1592,16 +1590,16 @@ int btrfs_compress_heuristic(struct btrfs_inode *inode, u64 start, u64 end)* Convert the compression suffix (eg. after "zlib" starting with ":") to * level, unrecognized string will set the default level */ -unsigned int btrfs_compress_str2level (unsigned int type,const char *str) +int btrfs_compress_str2level (unsigned int type,const char *str) { -unsigned int level =0 ; +int level =0 ;int ret;if (!type)return 0 ;if (str[0 ] ==':' ) { - ret =kstrtouint (str +1 ,10 , &level); + ret =kstrtoint (str +1 ,10 , &level);if (ret) level =0 ; }
fs/btrfs/compression.h +8 -17
@@ -72,16 +72,6 @@ struct compressed_bio {struct btrfs_bio bbio; }; -static inline unsigned int btrfs_compress_type (unsigned int type_level) -{ -return (type_level &0xF ); -} - -static inline unsigned int btrfs_compress_level (unsigned int type_level) -{ -return ((type_level &0xF0 ) >>4 ); -} -/* @range_end must be exclusive. */ static inline u32 btrfs_calc_input_length (u64 range_end,u64 cur) {
@@ -93,7 +83,7 @@ static inline u32 btrfs_calc_input_length(u64 range_end, u64 cur)int __init btrfs_init_compress (void );void __cold btrfs_exit_compress (void ); -int btrfs_compress_folios (unsigned int type_level,struct address_space *mapping, +int btrfs_compress_folios (unsigned int type,int level,struct address_space *mapping,u64 start,struct folio **folios,unsigned long *out_folios,unsigned long *total_in,unsigned long *total_out); int btrfs_decompress(int type, const u8 *data_in, struct folio *dest_folio,
@@ -107,7 +97,7 @@ void btrfs_submit_compressed_write(struct btrfs_ordered_extent *ordered, bool writeback);void btrfs_submit_compressed_read (struct btrfs_bio *bbio); -unsigned int btrfs_compress_str2level (unsigned int type,const char *str); +int btrfs_compress_str2level (unsigned int type,const char *str);struct folio *btrfs_alloc_compr_folio (void );void btrfs_free_compr_folio (struct folio *folio);
@@ -131,14 +121,15 @@ struct workspace_manager {wait_queue_head_t ws_wait; }; -struct list_head *btrfs_get_workspace (int type,unsigned int level); +struct list_head *btrfs_get_workspace (int type,int level);void btrfs_put_workspace (int type,struct list_head *ws);struct btrfs_compress_op {struct workspace_manager *workspace_manager;/* Maximum level supported by the compression algorithm */ -unsigned int max_level; -unsigned int default_level; +int min_level; +int max_level; +int default_level; };/* The heuristic workspaces are managed via the 0th workspace manager */
@@ -187,9 +178,9 @@ int zstd_decompress(struct list_head *ws, const u8 *data_in, size_t destlen);void zstd_init_workspace_manager (void );void zstd_cleanup_workspace_manager (void ); -struct list_head *zstd_alloc_workspace (unsigned int level); +struct list_head *zstd_alloc_workspace (int level);void zstd_free_workspace (struct list_head *ws); -struct list_head *zstd_get_workspace (unsigned int level); +struct list_head *zstd_get_workspace (int level);void zstd_put_workspace (struct list_head *ws);#endif
fs/btrfs/fs.h +1 -1
@@ -486,7 +486,7 @@ struct btrfs_fs_info {unsigned long long mount_opt; unsigned long compress_type:4; -unsigned int compress_level; +int compress_level;u32 commit_interval;/* * It is a suggestive number, the read side is safe even it gets a
fs/btrfs/inode.c +1 -1
@@ -1013,7 +1013,7 @@ static void compress_file_range(struct btrfs_work *work) compress_type = inode->prop_compress;/* Compression level is applied here. */ - ret =btrfs_compress_folios (compress_type | (fs_info->compress_level <<4 ), + ret =btrfs_compress_folios (compress_type, fs_info->compress_level, mapping, start, folios, &nr_folios, &total_in, &total_compressed);if (ret)
fs/btrfs/super.c +1 -1
@@ -84,7 +84,7 @@ struct btrfs_fs_context {u32 thread_pool_size;unsigned long long mount_opt; unsigned long compress_type:4; -unsigned int compress_level; +int compress_level;refcount_t refs; };
fs/btrfs/zlib.c +1 -0
@@ -463,6 +463,7 @@ int zlib_decompress(struct list_head *ws, const u8 *data_in,const struct btrfs_compress_op btrfs_zlib_compress = { .workspace_manager = &wsm, + .min_level =1 , .max_level =9 , .default_level =BTRFS_ZLIB_DEFAULT_LEVEL , };
fs/btrfs/zstd.c +29 -19
144 lines not on this page · raw message
Signature · 1 line
2.45.2
In next patch iteration if there's one please fix the subject so there's "btrfs: zstd: ...".
Quoted 18 lines · Daniel Vacek
This patch allows using the fast modes (negative compression levels) of zstd as a mount option.
As per the results, the compression ratio is lower:
for level in {-10..-1} 1 2 3; \
do printf "level %3d\n" $level; \
mount -o compress=zstd:$level/dev/sdb/mnt/test/; \
grep sdb/proc/mounts; \
cp -r/usr/bin/mnt/test/; sync; compsize/mnt/test/bin; \
cp -r/usr/share/doc/mnt/test/; sync; compsize/mnt/test/doc; \
cp enwik9/mnt/test/; sync; compsize/mnt/test/enwik9; \
cp linux-6.13.tar/mnt/test/; sync; compsize/mnt/test/linux-6.13.tar; \
rm -r/mnt/test/{bin,doc,enwik9,linux-6.13.tar}; \
umount/mnt/test/; \
done |& tee results | \
awk '/^level/{print}/^TOTAL/{print$3"\t"$2" |"}' | paste - - - - -
Quoted 15 lines
266M bin | 45M doc | 953M wiki | 1.4G source =============================+===============+===============+===============+ level -10 171M 64% | 28M 62% | 631M 66% | 512M 34% | level -9 165M 62% | 27M 61% | 615M 64% | 493M 33% | level -8 161M 60% | 27M 59% | 598M 62% | 475M 32% | level -7 155M 58% | 26M 58% | 582M 61% | 457M 30% | level -6 151M 56% | 25M 56% | 565M 59% | 437M 29% | level -5 145M 54% | 24M 55% | 545M 57% | 417M 28% | level -4 139M 52% | 23M 52% | 520M 54% | 391M 26% | level -3 135M 50% | 22M 50% | 495M 51% | 369M 24% | level -2 127M 47% | 22M 48% | 470M 49% | 349M 23% | level -1 120M 45% | 21M 47% | 452M 47% | 332M 22% | level 1 110M 41% | 17M 39% | 362M 38% | 290M 19% | level 2 106M 40% | 17M 38% | 349M 36% | 288M 19% | level 3 104M 39% | 16M 37% | 340M 35% | 276M 18% |
Thanks, this is a good overview. The ratio comparing the now default level 3 against the fastest level -10 is between 1.6-1.9x, with roughly linear progression with each level.
We still need the compression times. This may be harder to get from kernel, so one possibility is to add tracepoints or gather the statistics inside kernel before/after compression calls and then print to the log.
Or maybe the 'zstd' utility can be configured to run its benchmarks close to what kernel does, ie. the block size, memory limit and maybe more. I've skimmed the manual page and there's enough to choose from.
The last thing is to implement a userspace microbenchmark. Recently mkfs.btrfs learned compression support so you could reuse this code for that.
Quoted 5 lines
Signed-off-by Daniel Vacek <[email protected]>---
Changes in v3:
* Correct the accidentally replaced string inclip_level()function.
The code looks good now. Thanks.