f2fs-tools: sload compression support
v3 cover f2fs-tools: sload compression support
From: Robin Hsu <[email protected]>
* 3 patch set:
#1: added some #ifdef for easier support
#2: main code change
#3: automake changes
v2 fix (from v1): fixed a bug and a more elegant error handling flow. v3 fix (from v2): ./configure (automake) automatically determine to compile in lzo and/or lz4 compression support depending on the presence of liblzo2-dev and/or liblz4-dev
Robin Hsu (3):
f2fs-tools: Added #ifdef WITH_func
f2fs-tools:sload.f2fs compression support
f2fs-tools:sload.f2fs compress: Fixed automake
configure.ac | 12 +++ fsck/Makefile.am | 9 +- fsck/compress_wrapper.c | 102 ++++++++++++++++++++ fsck/compress_wrapper.h | 22 +++++ fsck/fsck.h | 15 +++ fsck/main.c | 157 ++++++++++++++++++++++++++++++- fsck/segment.c | 202 +++++++++++++++++++++++++++++++++++++--- fsck/sload.c | 67 +++++++++++++ include/f2fs_fs.h | 76 ++++++++++++++- lib/libf2fs_io.c | 33 +++++++ 10 files changed, 678 insertions(+), 17 deletions(-) create mode 100644 fsck/compress_wrapper.c create mode 100644 fsck/compress_wrapper.h
Signature · 8 lines
2.29.2.576.ga3fc446d84-goog _______________________________________________ Linux-f2fs-devel mailing list [email protected] https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
PATCH v3 1/3 f2fs-tools: Added #ifdef WITH_func
From: Robin Hsu <[email protected]>
Add proprocessor defines (options) 'WITH_func',
where func = DUMP, DEFRAG, RESIZE, or SLOAD
--- fsck/main.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+)
fsck/main.c +16 -0
@@ -345,6 +345,7 @@ void f2fs_parse_options(int argc, char *argv[])break ; } }else if (!strcmp ("dump.f2fs" , prog)) { +#ifdef WITH_DUMP const char *option_string ="d:i:n:s:Sa:b:V" ;static struct dump_option dump_opt = { .nid = 0, /* default root ino */
@@ -426,7 +427,9 @@ void f2fs_parse_options(int argc, char *argv[]) } c.private = &dump_opt; +#endif }else if (!strcmp ("defrag.f2fs" , prog)) { +#ifdef WITH_DEFRAG const char *option_string ="d:s:Sl:t:iV" ; c.func =DEFRAG ;
@@ -484,7 +487,9 @@ void f2fs_parse_options(int argc, char *argv[])if (err !=NOERROR ) break; } +#endif }else if (!strcmp ("resize.f2fs" , prog)) { +#ifdef WITH_RESIZE const char *option_string ="d:st:iV" ; c.func =RESIZE ;
@@ -526,7 +531,9 @@ void f2fs_parse_options(int argc, char *argv[])if (err !=NOERROR )break ; } +#endif }else if (!strcmp ("sload.f2fs" , prog)) { +#ifdef WITH_SLOAD const char *option_string ="C:d:f:p:s:St:T:V" ;#ifdef HAVE_LIBSELINUX int max_nr_opt = (int)sizeof(c.seopt_file) /
@@ -595,6 +602,7 @@ void f2fs_parse_options(int argc, char *argv[])if (err !=NOERROR ) break; } +#endif /* WITH_SLOAD */ }if (err ==NOERROR ) {
@@ -707,6 +715,7 @@ static int do_fsck(struct f2fs_sb_info *sbi)return FSCK_ERRORS_LEFT_UNCORRECTED ; } +#ifdef WITH_DUMP static void do_dump (struct f2fs_sb_info *sbi) {struct dump_option *opt = (struct dump_option *)c.private;
@@ -733,7 +742,9 @@ static void do_dump(struct f2fs_sb_info *sbi)print_cp_state (flag); } +#endif +#ifdef WITH_DEFRAG static int do_defrag (struct f2fs_sb_info *sbi) {struct f2fs_super_block *sb =F2FS_RAW_SUPER (sbi);
@@ -782,7 +793,9 @@ out_range: c.defrag_target);return -1 ; } +#endif +#ifdef WITH_RESIZE static int do_resize (struct f2fs_sb_info *sbi) {if (!c.target_sectors)
@@ -796,7 +809,9 @@ static int do_resize(struct f2fs_sb_info *sbi)return f2fs_resize (sbi); } +#endif +#ifdef WITH_SLOAD static int do_sload (struct f2fs_sb_info *sbi) {if (!c.from_dir) {
@@ -808,6 +823,7 @@ static int do_sload(struct f2fs_sb_info *sbi) return f2fs_sload(sbi); } +#endif #if defined(__APPLE__)static u64 get_boottime_ns ()
Signature · 8 lines
2.29.2.576.ga3fc446d84-goog _______________________________________________ Linux-f2fs-devel mailing list [email protected] https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
Re: [f2fs-dev] [PATCH v3 1/3] f2fs-tools: Added #ifdef WITH_func
Quoted 6 lines · Robin Hsu
From: Robin Hsu <[email protected]>
Add proprocessor defines (options) '
WITH_func',
where func = DUMP, DEFRAG, RESIZE, or SLOADSigned-off-by Robin Hsu <[email protected]>
Thanks,
_______________________________________________
Linux-f2fs-devel mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
PATCH v3 2/3 f2fs-tools:sload.f2fs compression support
In reply to Robin HsuFrom: Robin Hsu <[email protected]>
Add F2FS compression support for sload
* Support file extension filter, either default-accept or default-deny policy
* Support choice of compression algorithm, LZO (version 2) or LZ4 (default)
* Support custom log of cluster size
* Support minimum number of compressed blocks per cluster (default 1). A cluster will not be compressed if the number can not be met.
* suuport -r (read-only) option
--- fsck/compress_wrapper.c | 102 ++++++++++++++++++++ fsck/compress_wrapper.h | 22 +++++ fsck/fsck.h | 15 +++ fsck/main.c | 141 +++++++++++++++++++++++++++- fsck/segment.c | 202 +++++++++++++++++++++++++++++++++++++--- fsck/sload.c | 67 +++++++++++++ include/f2fs_fs.h | 76 ++++++++++++++- lib/libf2fs_io.c | 33 +++++++ 8 files changed, 644 insertions(+), 14 deletions(-) create mode 100644 fsck/compress_wrapper.c create mode 100644 fsck/compress_wrapper.h
fsck/compress_wrapper.c new file +102 -0
@@ -0,0 +1,102 @@ +/** +* compress_wrapper.c +* +* Copyright (c) 2020 Google Inc. +* Robin Hsu <[email protected]> +* : initial created, for sload compression support +* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License version 2 as +* published by the Free Software Foundation. +*/ +#include "f2fs.h" /* for config.h for general environment (non-Android) */ + +#include "compress_wrapper.h" +#ifdef HAVE_LIBLZO2 +#include <lzo/lzo1x.h> /* for lzo1x_1_15_compress() */ +#endif +#ifdef HAVE_LIBLZ4 +#include <lz4.h> /* for LZ4_compress_fast_extState() */ +#endif + +/* +* macro/constants borrowed from kernel header (GPL-2.0): +* include/linux/lzo.h, and include/linux/lz4.h +*/ +#ifdef HAVE_LIBLZO2 +#define lzo1x_worst_compress (x) ((x) + (x) / 16 + 64 + 3 + 2) +#define LZO_WORK_SIZE ALIGN_UP(LZO1X_1_15_MEM_COMPRESS, 8) +#endif +#ifdef HAVE_LIBLZ4 +#define LZ4_MEMORY_USAGE 14 +#define LZ4_MAX_INPUT_SIZE 0x7E000000/* 2 113 929 216 bytes */ +#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(long long)) +#define LZ4_MEM_COMPRESS LZ4_STREAMSIZE +#define LZ4_ACCELERATION_DEFAULT 1 +#define LZ4_WORK_SIZE ALIGN_UP(LZ4_MEM_COMPRESS, 8) +#endif + +#ifdef HAVE_LIBLZO2 +static void lzo_compress_init (struct compress_ctx *cc) +{ +size_t size = cc->cluster_size *F2FS_BLKSIZE ; +size_t alloc = size +lzo1x_worst_compress (size) + +COMPRESS_HEADER_SIZE +LZO_WORK_SIZE ; +ASSERT ((cc->private =qbuf_alloc (alloc)) !=NULL ); + cc->rbuf = (char *) cc->private +LZO_WORK_SIZE ; + cc->cbuf = (struct compress_data *)((char *) cc->rbuf + size); +} + +static int lzo_compress (struct compress_ctx *cc) +{ +int ret =lzo1x_1_15_compress (cc->rbuf, cc->rlen, cc->cbuf->cdata, + (lzo_uintp)(&cc->clen), cc->private); + cc->cbuf->clen =cpu_to_le32 (cc->clen); +return ret; +} +#endif + +#ifdef HAVE_LIBLZ4 +static void lz4_compress_init (struct compress_ctx *cc) +{ +size_t size = cc->cluster_size *F2FS_BLKSIZE ; +size_t alloc = size +LZ4_COMPRESSBOUND (size) + +COMPRESS_HEADER_SIZE +LZ4_WORK_SIZE ; +ASSERT ((cc->private =qbuf_alloc (alloc)) !=NULL ); + cc->rbuf = (char *) cc->private +LZ4_WORK_SIZE ; + cc->cbuf = (struct compress_data *)((char *) cc->rbuf + size); +} + +static int lz4_compress (struct compress_ctx *cc) +{ + cc->clen =LZ4_compress_fast_extState (cc->private, cc->rbuf, + (char *)cc->cbuf->cdata, cc->rlen, + cc->rlen -F2FS_BLKSIZE * c.sldc_min_cbpc, +LZ4_ACCELERATION_DEFAULT ); + +if (!cc->clen) +return 1 ; + + cc->cbuf->clen =cpu_to_le32 (cc->clen); +return 0 ; +} +#endif + +const char *ca_names[] = { +"LZO" , +"LZ4" , +"" ,/* end of the name list */ +}; + +compress_ops compr_ops[] = { +#ifdef HAVE_LIBLZO2 + {lzo_compress_init, lzo_compress}, +#else + {NULL, NULL}, +#endif +#ifdef HAVE_LIBLZ4 + {lz4_compress_init, lz4_compress}, +#else + {NULL, NULL}, +#endif +};
fsck/compress_wrapper.h new file +22 -0
@@ -0,0 +1,22 @@ +/** +* compress_wrapper.h +* +* Copyright (c) 2020 Google Inc. +* Robin Hsu <[email protected]> +* : initial created, for sload compression support +* Copyright (c) 2020 Google Inc. +* Robin Hsu <[email protected]> +* : add sload compression support +* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License version 2 as +* published by the Free Software Foundation. +*/ + +#ifndef COMPRESS_WRAPPER_H +#define COMPRESS_WRAPPER_H + +#include "f2fs_fs.h" +extern compress_ops compr_ops[];/* [0]: LZO, [1]: LZ4, */ + +#endif /* COMPRESS_WRAPPER_H */
fsck/fsck.h +15 -0
@@ -3,6 +3,9 @@* * Copyright (c) 2013 Samsung Electronics Co., Ltd. * http://www.samsung.com/ +* Copyright (c) 2020 Google Inc. +* Robin Hsu <[email protected]> +* : add sload compression support * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as
@@ -266,6 +269,9 @@ int f2fs_resize(struct f2fs_sb_info *);/* sload.c */ int f2fs_sload (struct f2fs_sb_info *); +void sldc_erase_bufs (struct compress_ctx *cc); +void sload_countblk (void ); +extern struct ext_tbl_op ext_filter;/* segment.c */ int reserve_new_block(struct f2fs_sb_info *, block_t *,
@@ -282,7 +288,16 @@ block_t new_node_block(struct f2fs_sb_info *,struct quota_file ;u64 f2fs_quota_size (struct quota_file *);u64 f2fs_read (struct f2fs_sb_info *,nid_t ,u8 *,u64 ,pgoff_t ); +enum wr_addr_type { +WR_NORMAL =1 , +WR_COMPRESS_DATA =2 , +WR_NULL_ADDR =NULL_ADDR ,/* 0 */ +WR_NEW_ADDR =NEW_ADDR ,/* -1U */ +WR_COMPRESS_ADDR =COMPRESS_ADDR ,/* -2U */ +};u64 f2fs_write (struct f2fs_sb_info *,nid_t ,u8 *,u64 ,pgoff_t ); +u64 f2fs_write_compress_data (struct f2fs_sb_info *,nid_t ,u8 *,u64 ,pgoff_t ); +u64 f2fs_write_addrtag (struct f2fs_sb_info *,nid_t ,pgoff_t ,unsigned int );void f2fs_filesize_update (struct f2fs_sb_info *,nid_t ,u64 ); int get_dnode_of_data(struct f2fs_sb_info *, struct dnode_of_data *,
fsck/main.c +140 -1
197 lines not on this page · raw message
fsck/segment.c +190 -12
284 lines not on this page · raw message
fsck/sload.c +67 -0
79 lines not on this page · raw message
include/f2fs_fs.h +75 -1
118 lines not on this page · raw message
lib/libf2fs_io.c +33 -0
@@ -5,6 +5,9 @@* http://www.samsung.com/ * Copyright (c) 2019 Google Inc. * http://www.google.com/ +* Copyright (c) 2020 Google Inc. +* Robin Hsu <[email protected]> +* : add quick-buffer for sload compression support * * Dual licensed under the GPL or LGPL version 2 licenses. */
@@ -106,6 +109,36 @@ static long dcache_relocate_offset0[] = { };static int dcache_relocate_offset[16 ]; +/* quick (shared) buffer */ +static bool qbuf_initialized; +static void *qbuf; +static size_t qbufsize; +void qbuf_free (void ) +{ +ASSERT (qbuf_initialized); +if (qbuf !=NULL ) { +free (qbuf); + qbuf =NULL ; + qbufsize =0 ; + } +} +void *qbuf_alloc (size_t size) +{ +ASSERT (qbuf_initialized); +if (size > qbufsize) { +qbuf_free (); + qbuf =malloc (size); + } +return qbuf; +} +void qbuf_init (void ) +{ +if (qbuf_initialized) +return ; +atexit (qbuf_free); + qbuf_initialized = true; +} +static void dcache_print_statistics (void ) {long i;
Signature · 8 lines
2.29.2.576.ga3fc446d84-goog _______________________________________________ Linux-f2fs-devel mailing list [email protected] https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
Re: [PATCH v3 2/3] f2fs-tools:sload.f2fs compression support
Hi Robin,
I found some bugs and want to suggest a fix.
I'll apply this on your patch and let me know, if you want to modify more on top of this.
fsck/segment.c +7 -3
@@ -115,6 +115,8 @@ int new_data_block(struct f2fs_sb_info *sbi, void *block,get_node_info (sbi, dn->nid, &ni);set_summary (&sum, dn->nid, dn->ofs_in_node, ni.version); + + dn->data_blkaddr = blkaddr; ret =reserve_new_block (sbi, &dn->data_blkaddr, &sum, type,0 );if (ret) { c.alloc_failed =1 ;
@@ -517,8 +519,8 @@ int f2fs_build_file(struct f2fs_sb_info *sbi, struct dentry *de) COMPRESS_HEADER_SIZE, BLOCK_SZ);unsigned int cur_cblk; -if (ret || n < (int )(csize +BLOCK_SZ * - c.sldc_min_cbpc)) { +if (ret || n < (1 << c.sldc_cc.log_cluster_size) *BLOCK_SZ || + n < (int )(csize +BLOCK_SZ * c.sldc_min_cbpc)) { wlen =f2fs_write (sbi, de->ino, rbuf, n, off);ASSERT ((int )wlen == n); }else {
@@ -549,8 +551,10 @@ int f2fs_build_file(struct f2fs_sb_info *sbi, struct dentry *de)ASSERT (dev_read_block (node_blk, ni.blk_addr) >=0 );/* update inode meta */ node_blk->i.i_size =cpu_to_le64 (off); -if (!c.sldc_immutable) +if (!c.sldc_immutable) { node_blk->i.i_compr_blocks =cpu_to_le64 (cblocks); + node_blk->i.i_blocks +=cpu_to_le64 (cblocks); + }ASSERT (write_inode (node_blk, ni.blk_addr) >=0 );free (node_blk);
Quoted 979 lines · Robin Hsu
From: Robin Hsu <[email protected]>
Add F2FS compression support for sload
* Support file extension filter, either default-accept or default-deny policy
* Support choice of compression algorithm, LZO (version 2) or LZ4 (default)
* Support custom log of cluster size
* Support minimum number of compressed blocks per cluster (default 1). A cluster will not be compressed if the number can not be met.
* suuport -r (read-only) optionSigned-off-by Robin Hsu <[email protected]>--- fsck/compress_wrapper.c | 102 ++++++++++++++++++++ fsck/compress_wrapper.h | 22 +++++ fsck/fsck.h | 15 +++ fsck/main.c | 141 +++++++++++++++++++++++++++- fsck/segment.c | 202 +++++++++++++++++++++++++++++++++++++--- fsck/sload.c | 67 +++++++++++++ include/f2fs_fs.h | 76 ++++++++++++++- lib/libf2fs_io.c | 33 +++++++ 8 files changed, 644 insertions(+), 14 deletions(-) create mode 100644 fsck/compress_wrapper.c create mode 100644 fsck/compress_wrapper.hfsck/compress_wrapper.c new file +102
-0@@ -0,0 +1,102 @@ +/** +* compress_wrapper.c +* +* Copyright (c) 2020 Google Inc. +* Robin Hsu <[email protected]> +* : initial created, for sload compression support +* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License version 2 as +* published by the Free Software Foundation. +*/ +#include "f2fs.h" /* for config.h for general environment (non-Android) */ + +#include "compress_wrapper.h" +#ifdef HAVE_LIBLZO2 +#include <lzo/lzo1x.h> /* for lzo1x_1_15_compress() */ +#endif +#ifdef HAVE_LIBLZ4 +#include <lz4.h> /* for LZ4_compress_fast_extState() */ +#endif + +/* +* macro/constants borrowed from kernel header (GPL-2.0): +* include/linux/lzo.h, and include/linux/lz4.h +*/ +#ifdef HAVE_LIBLZO2 +#define lzo1x_worst_compress (x) ((x) + (x) / 16 + 64 + 3 + 2) +#define LZO_WORK_SIZE ALIGN_UP(LZO1X_1_15_MEM_COMPRESS, 8) +#endif +#ifdef HAVE_LIBLZ4 +#define LZ4_MEMORY_USAGE 14 +#define LZ4_MAX_INPUT_SIZE 0x7E000000/* 2 113 929 216 bytes */ +#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(long long)) +#define LZ4_MEM_COMPRESS LZ4_STREAMSIZE +#define LZ4_ACCELERATION_DEFAULT 1 +#define LZ4_WORK_SIZE ALIGN_UP(LZ4_MEM_COMPRESS, 8) +#endif + +#ifdef HAVE_LIBLZO2 +static void lzo_compress_init (struct compress_ctx *cc) +{ +size_t size = cc->cluster_size *F2FS_BLKSIZE ; +size_t alloc = size +lzo1x_worst_compress (size) + +COMPRESS_HEADER_SIZE +LZO_WORK_SIZE ; +ASSERT ((cc->private =qbuf_alloc (alloc)) !=NULL ); + cc->rbuf = (char *) cc->private +LZO_WORK_SIZE ; + cc->cbuf = (struct compress_data *)((char *) cc->rbuf + size); +} + +static int lzo_compress (struct compress_ctx *cc) +{ +int ret =lzo1x_1_15_compress (cc->rbuf, cc->rlen, cc->cbuf->cdata, + (lzo_uintp)(&cc->clen), cc->private); + cc->cbuf->clen =cpu_to_le32 (cc->clen); +return ret; +} +#endif + +#ifdef HAVE_LIBLZ4 +static void lz4_compress_init (struct compress_ctx *cc) +{ +size_t size = cc->cluster_size *F2FS_BLKSIZE ; +size_t alloc = size +LZ4_COMPRESSBOUND (size) + +COMPRESS_HEADER_SIZE +LZ4_WORK_SIZE ; +ASSERT ((cc->private =qbuf_alloc (alloc)) !=NULL ); + cc->rbuf = (char *) cc->private +LZ4_WORK_SIZE ; + cc->cbuf = (struct compress_data *)((char *) cc->rbuf + size); +} + +static int lz4_compress (struct compress_ctx *cc) +{ + cc->clen =LZ4_compress_fast_extState (cc->private, cc->rbuf, + (char *)cc->cbuf->cdata, cc->rlen, + cc->rlen -F2FS_BLKSIZE * c.sldc_min_cbpc, +LZ4_ACCELERATION_DEFAULT ); + +if (!cc->clen) +return 1 ; + + cc->cbuf->clen =cpu_to_le32 (cc->clen); +return 0 ; +} +#endif + +const char *ca_names[] = { +"LZO" , +"LZ4" , +"" ,/* end of the name list */ +}; + +compress_ops compr_ops[] = { +#ifdef HAVE_LIBLZO2 + {lzo_compress_init, lzo_compress}, +#else + {NULL, NULL}, +#endif +#ifdef HAVE_LIBLZ4 + {lz4_compress_init, lz4_compress}, +#else + {NULL, NULL}, +#endif +};fsck/compress_wrapper.h new file +22
-0@@ -0,0 +1,22 @@ +/** +* compress_wrapper.h +* +* Copyright (c) 2020 Google Inc. +* Robin Hsu <[email protected]> +* : initial created, for sload compression support +* Copyright (c) 2020 Google Inc. +* Robin Hsu <[email protected]> +* : add sload compression support +* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License version 2 as +* published by the Free Software Foundation. +*/ + +#ifndef COMPRESS_WRAPPER_H +#define COMPRESS_WRAPPER_H + +#include "f2fs_fs.h" +extern compress_ops compr_ops[];/* [0]: LZO, [1]: LZ4, */ + +#endif /* COMPRESS_WRAPPER_H */ fsck/fsck.h +15
-0@@ -3,6 +3,9 @@* * Copyright (c) 2013 Samsung Electronics Co., Ltd. * http://www.samsung.com/ +* Copyright (c) 2020 Google Inc. +* Robin Hsu <[email protected]> +* : add sload compression support * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -266,6 +269,9 @@ int f2fs_resize(struct f2fs_sb_info *);/* sload.c */ int f2fs_sload (struct f2fs_sb_info *); +void sldc_erase_bufs (struct compress_ctx *cc); +void sload_countblk (void ); +extern struct ext_tbl_op ext_filter;/* segment.c */ int reserve_new_block(struct f2fs_sb_info *, block_t *,@@ -282,7 +288,16 @@ block_t new_node_block(struct f2fs_sb_info *,struct quota_file ;u64 f2fs_quota_size (struct quota_file *);u64 f2fs_read (struct f2fs_sb_info *,nid_t ,u8 *,u64 ,pgoff_t ); +enum wr_addr_type { +WR_NORMAL =1 , +WR_COMPRESS_DATA =2 , +WR_NULL_ADDR =NULL_ADDR ,/* 0 */ +WR_NEW_ADDR =NEW_ADDR ,/* -1U */ +WR_COMPRESS_ADDR =COMPRESS_ADDR ,/* -2U */ +};u64 f2fs_write (struct f2fs_sb_info *,nid_t ,u8 *,u64 ,pgoff_t ); +u64 f2fs_write_compress_data (struct f2fs_sb_info *,nid_t ,u8 *,u64 ,pgoff_t ); +u64 f2fs_write_addrtag (struct f2fs_sb_info *,nid_t ,pgoff_t ,unsigned int );void f2fs_filesize_update (struct f2fs_sb_info *,nid_t ,u64 ); int get_dnode_of_data(struct f2fs_sb_info *, struct dnode_of_data *,fsck/main.c +140
-1@@ -13,6 +13,9 @@* Copyright (c) 2019 Google Inc. * Robin Hsu <[email protected]> * : add cache layer +* Copyright (c) 2020 Google Inc. +* Robin Hsu <[email protected]> +* : add sload compression support * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -25,6 +28,7 @@#include <getopt.h> #include <stdbool.h> #include "quotaio.h" +#include "compress_wrapper.h" struct f2fs_fsck gfsck;@@ -134,6 +138,17 @@ void sload_usage()MSG (0 ," -S sparse_mode\n" );MSG (0 ," -t mount point [prefix of target fs path, default:/]\n" );MSG (0 ," -T timestamp\n" ); +MSG (0 ," -c enable compression (default allow policy)\n" ); +MSG (0 ," ------------ Compression sub-options -----------------\n" ); +MSG (0 ," -L <log-of-blocks-per-cluster>, default 2\n" ); +MSG (0 ," -a <algorithm> compression algorithm, default LZ4\n" ); +MSG (0 ," -x <ext> compress files except for these extensions.\n" ); +MSG (0 ," -i <ext> compress files with these extensions only.\n" ); +MSG (0 ," * -i or -x: use it many times for multiple extensions.\n" ); +MSG (0 ," * -i and -x cannot be used together..\n" ); +MSG (0 ," -m <num> min compressed blocks per cluster\n" ); +MSG (0 ," -r readonly (IMMUTABLE) for compressed files\n" ); +MSG (0 ," ------------------------------------------------------\n" );MSG (0 ," -d debug level [default:0]\n" );MSG (0 ," -V print the version number and exit\n" );exit (1 );@@ -534,7 +549,7 @@ void f2fs_parse_options(int argc, char *argv[])#endif }else if (!strcmp ("sload.f2fs" , prog)) {#ifdef WITH_SLOAD -const char *option_string ="C:d:f:p:s:St:T:V" ; +const char *option_string ="cL:a:i:x:m:rC:d:f:p:s:St:T:V" ;#ifdef HAVE_LIBSELINUX int max_nr_opt = (int )sizeof (c.seopt_file) /sizeof (c.seopt_file[0 ]);@@ -543,8 +558,82 @@ void f2fs_parse_options(int argc, char *argv[])char *p; c.func =SLOAD ; + c.sldc_cc.log_cluster_size =2 ; + c.sldc_ca =CA_LZ4 ; + c.sldc_min_cbpc =1 ; + c.sldc_ef = &ext_filter;while ((option =getopt (argc, argv, option_string)) !=EOF ) { +unsigned int i; +int val; +switch (option) { +case 'c' :/* compression support */ + c.sldc_en = true; +break ; +case 'L' :/* compression: log of blocks-per-cluster */ + c.sldc_got_opt = true; + val =atoi (optarg); +if (val <MIN_COMPRESS_LOG_SIZE || + val >MAX_COMPRESS_LOG_SIZE ) { +MSG (0 ,"\tError: log of blocks per" +" cluster must be in the range" +" of %d .. %d.\n" , +MIN_COMPRESS_LOG_SIZE , +MAX_COMPRESS_LOG_SIZE ); +error_out (prog); + } + c.sldc_cc.log_cluster_size = val; +break ; +case 'a' :/* compression: choose algorithm */ + c.sldc_got_opt = true; + c.sldc_ca = (u8)-1 ; +for (i =0 ; ca_names[i][0 ] !=0 ; i++) { +if (!strcmp (ca_names[i], optarg)) { + c.sldc_ca = i; +break ; + } + } +if (c.sldc_ca == (u8)-1 ) { +MSG (0 ,"\tError: Unknown compression" +" algorithm %s\n" , optarg); +error_out (prog); + } +break ; +case 'i' :/* compress only these extensions */ + c.sldc_got_opt = true; +if (c.sldc_policy ==FP_ALLOW ) { +MSG (0 ,"\tError: could not mix option" +" -i and -x\n" ); +error_out (prog); + } + c.sldc_policy =FP_DENY ; + c.sldc_ef->add (optarg); +break ; +case 'x' :/* compress except for these extensions */ + c.sldc_got_opt = true; +if (c.sldc_policy ==FP_DENY ) { +MSG (0 ,"\tError: could not mix option" +" -i and -x\n" ); +error_out (prog); + } + c.sldc_policy =FP_ALLOW ; + c.sldc_ef->add (optarg); +break ; +case 'm' :/* minimum compressed blocks per cluster */ + c.sldc_got_opt = true; + val =atoi (optarg); +if (val <=0 ) { +MSG (0 ,"\tError: minimum compressed" +" blocks per cluster must be" +" positive.\n" ); +error_out (prog); + } + c.sldc_min_cbpc = val; +break ; +case 'r' :/* compress file to set IMMUTABLE */ + c.sldc_got_opt = true; + c.sldc_immutable = true; +break ;case 'C' : c.fs_config_file =absolute_path (optarg);break ;@@ -602,6 +691,27 @@ void f2fs_parse_options(int argc, char *argv[])if (err !=NOERROR ) break; } +if (c.sldc_got_opt && !c.sldc_en) { +MSG (0 ,"\tError: compression sub-options are used" +" without the compression enable (-c) option\n" + ); +error_out (prog); + } +if (err ==NOERROR && c.sldc_en) { + c.sldc_cc.cluster_size =1 + << c.sldc_cc.log_cluster_size; +if (c.sldc_policy ==FP_UNASSIGNED ) + c.sldc_policy =FP_ALLOW ; +if (c.sldc_min_cbpc >= c.sldc_cc.cluster_size) { +MSG (0 ,"\tError: minimum reduced blocks by" +" compression per cluster must be at" +" most one less than blocks per" +" cluster, i.e. %d\n" , + c.sldc_cc.cluster_size -1 ); +error_out (prog); + } +qbuf_init (); + }#endif /* WITH_SLOAD */ }@@ -812,6 +922,27 @@ static int do_resize(struct f2fs_sb_info *sbi)#endif #ifdef WITH_SLOAD +int init_compr (struct f2fs_sb_info *sbi) +{ +if (!(sbi->raw_super->feature + &cpu_to_le32 (F2FS_FEATURE_COMPRESSION ))) { +MSG (0 ,"Error: Compression (-c) was requested " +"but the file system is not created " +"with such feature.\n" ); +return -1 ; + } +if (compr_ops[c.sldc_ca].init ==NULL ) { +MSG (0 ,"Error: The selected compression algorithm is not" +" supported\n" ); +return -1 ; + } + c.sldc_compr = compr_ops + c.sldc_ca; + c.sldc_compr->init (&c.sldc_cc); +sldc_erase_bufs (&c.sldc_cc); + c.sldc_cc.rlen = c.sldc_cc.cluster_size *F2FS_BLKSIZE ; +return 0 ; +} +static int do_sload (struct f2fs_sb_info *sbi) {if (!c.from_dir) {@@ -821,6 +952,11 @@ static int do_sload(struct f2fs_sb_info *sbi)if (!c.mount_point) c.mount_point ="/" ; +if (c.sldc_en) { +if (init_compr (sbi)) +return -1 ; + } + return f2fs_sload(sbi); }#endif @@ -971,6 +1107,9 @@ retry:return ret2; } +if (c.func ==SLOAD ) + c.sldc_ef->destroy (); +printf ("\nDone: %lf secs\n" , (get_boottime_ns () - start) /1000000000.0 );return ret;fsck/segment.c +190
-12@@ -8,6 +8,9 @@* Hou Pengyang <[email protected]> * Liu Shuoran <[email protected]> * Jaegeuk Kim <[email protected]> +* Copyright (c) 2020 Google Inc. +* Robin Hsu <[email protected]> +* : add sload compression support * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -16,6 +19,7 @@#include "fsck.h" #include "node.h" #include "quotaio.h" +#include "compress_wrapper.h" int reserve_new_block (struct f2fs_sb_info *sbi,block_t *to,struct f2fs_summary *sum,int type,bool is_inode)@@ -228,8 +232,14 @@ u64 f2fs_read(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,return read_count; } -u64 f2fs_write (struct f2fs_sb_info *sbi,nid_t ino,u8 *buffer, -u64 count,pgoff_t offset) +/* +* Do not call this function directly. Instead, call one of the following: +* u64 f2fs_write(); +* u64 f2fs_write_compress_data(); +* u64 f2fs_write_addrtag(); +*/ +static u64 f2fs_write_ex (struct f2fs_sb_info *sbi,nid_t ino,u8 *buffer, +u64 count,pgoff_t offset,enum wr_addr_type addr_type) {struct dnode_of_data dn;struct node_info ni;@@ -243,6 +253,19 @@ u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,void * index_node =NULL ;int idirty =0 ;int err; +bool has_data = (addr_type ==WR_NORMAL + || addr_type ==WR_COMPRESS_DATA ); + +if (count ==0 ) +return 0 ; + +/* +* Enforce calling from f2fs_write(), f2fs_write_compress_data(), +* and f2fs_write_addrtag(). Beside, check if is properly called. +*/ +ASSERT ((!has_data && buffer ==NULL ) || (has_data && buffer !=NULL )); +if (addr_type !=WR_NORMAL ) +ASSERT (offset %F2FS_BLKSIZE ==0 );/* block boundary only */ /* Memory allocation for block buffer and inode. */ blk_buffer =calloc (BLOCK_SZ ,2 );@@ -265,15 +288,26 @@ u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,if (err)break ; idirty |= dn.idirty; -if (index_node) -free (index_node); +free (index_node); index_node = (dn.node_blk == dn.inode_blk) ? -NULL : dn.node_blk; +NULL : dn.node_blk; remained_blkentries =ADDRS_PER_PAGE (sbi, - dn.node_blk, dn.inode_blk); + dn.node_blk, dn.inode_blk) - + dn.ofs_in_node; }ASSERT (remained_blkentries >0 ); +if (!has_data) { + dn.data_blkaddr = addr_type; +set_data_blkaddr (&dn); + idirty |= dn.idirty; +if (dn.ndirty) +ASSERT (dev_write_block (dn.node_blk, + dn.node_blkaddr) >=0 ); + written_count =0 ; +break ; + } + blkaddr =datablock_addr (dn.node_blk, dn.ofs_in_node);if (blkaddr ==NULL_ADDR || blkaddr ==NEW_ADDR ) { err = new_data_block(sbi, blk_buffer,@@ -281,6 +315,7 @@ u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,if (err)break ; blkaddr = dn.data_blkaddr; + idirty |= dn.idirty; } off_in_blk = offset %BLOCK_SZ ;@@ -305,9 +340,10 @@ u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer, dn.ofs_in_node++;if ((--remained_blkentries ==0 || count ==0 ) && (dn.ndirty)) -ASSERT (dev_write_block (dn.node_blk, dn.node_blkaddr) >=0 ); +ASSERT (dev_write_block (dn.node_blk, dn.node_blkaddr) + >=0 ); } -if (offset >le64_to_cpu (inode->i.i_size)) { +if (addr_type ==WR_NORMAL && offset >le64_to_cpu (inode->i.i_size)) { inode->i.i_size =cpu_to_le64 (offset); idirty =1 ; }@@ -315,13 +351,33 @@ u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,ASSERT (inode == dn.inode_blk);ASSERT (write_inode (inode, ni.blk_addr) >=0 ); } -if (index_node) -free (index_node); + +free (index_node);free (blk_buffer);return written_count; } +u64 f2fs_write (struct f2fs_sb_info *sbi,nid_t ino,u8 *buffer, +u64 count,pgoff_t offset) +{ +return f2fs_write_ex (sbi, ino, buffer, count, offset,WR_NORMAL ); +} + +u64 f2fs_write_compress_data (struct f2fs_sb_info *sbi,nid_t ino,u8 *buffer, +u64 count,pgoff_t offset) +{ +return f2fs_write_ex (sbi, ino, buffer, count, offset,WR_COMPRESS_DATA ); +} + +u64 f2fs_write_addrtag (struct f2fs_sb_info *sbi,nid_t ino,pgoff_t offset, +unsigned int addrtag) +{ +ASSERT (addrtag ==COMPRESS_ADDR || addrtag ==NEW_ADDR + || addrtag ==NULL_ADDR ); +return f2fs_write_ex (sbi, ino,NULL ,F2FS_BLKSIZE , offset, addrtag); +} +/* This function updates only inode->i.i_size */ void f2fs_filesize_update (struct f2fs_sb_info *sbi,nid_t ino,u64 filesize) {@@ -342,11 +398,59 @@ void f2fs_filesize_update(struct f2fs_sb_info *sbi, nid_t ino, u64 filesize)free (inode); } +#define MAX_BULKR_RETRY 5 +int bulkread (int fd,void *rbuf,size_t rsize,bool *eof) +{ +int n =0 ; +int retry =MAX_BULKR_RETRY ; +int cur; + +if (!rsize) +return 0 ; + +if (eof !=NULL ) + *eof = false; +while (rsize && (cur =read (fd, rbuf, rsize)) !=0 ) { +if (cur ==-1 ) { +if (errno ==EINTR && retry--) +continue ; +return -1 ; + } + retry =MAX_BULKR_RETRY ; + + rsize -= cur; + n += cur; + } +if (eof !=NULL ) + *eof = (cur ==0 ); +return n; +} + +u64 f2fs_fix_mutable (struct f2fs_sb_info *sbi,nid_t ino,pgoff_t offset, +unsigned int compressed) +{ +unsigned int i; +u64 wlen; + +if (c.sldc_immutable) +return 0 ; + +for (i =0 ; i < compressed -1 ; i++) { + wlen =f2fs_write_addrtag (sbi, ino, + offset + (i <<F2FS_BLKSIZE_BITS ),NEW_ADDR ); +if (wlen) +return wlen; + } +return 0 ; +} +int f2fs_build_file (struct f2fs_sb_info *sbi,struct dentry *de) {int fd, n;pgoff_t off =0 ;u8 buffer[BLOCK_SZ ]; +struct node_info ni; +struct f2fs_node *node_blk;if (de->ino ==0 )return -1 ;@@ -359,8 +463,6 @@ int f2fs_build_file(struct f2fs_sb_info *sbi, struct dentry *de)/* inline_data support */ if (de->size <=DEF_MAX_INLINE_DATA ) { -struct node_info ni; -struct f2fs_node *node_blk;int ret;get_node_info (sbi, de->ino, &ni);@@ -385,6 +487,82 @@ int f2fs_build_file(struct f2fs_sb_info *sbi, struct dentry *de) node_blk->i.i_size =cpu_to_le64 (de->size);ASSERT (write_inode (node_blk, ni.blk_addr) >=0 );free (node_blk); +#ifdef WITH_SLOAD + }else if (c.func ==SLOAD && c.sldc_en && + c.sldc_ef->filter (de->full_path)) { +bool eof = false; +u8 *rbuf = c.sldc_cc.rbuf; +unsigned int cblocks =0 ; + + node_blk =calloc (BLOCK_SZ ,1 ); +ASSERT (node_blk); + +/* read inode */ +get_node_info (sbi, de->ino, &ni); +ASSERT (dev_read_block (node_blk, ni.blk_addr) >=0 ); +/* update inode meta */ + node_blk->i.i_compress_algrithm = c.sldc_ca; + node_blk->i.i_log_cluster_size = + c.sldc_cc.log_cluster_size; + node_blk->i.i_flags =cpu_to_le32 ( +F2FS_COMPR_FL | + (c.sldc_immutable ?FS_IMMUTABLE_FL :0 )); +ASSERT (write_inode (node_blk, ni.blk_addr) >=0 ); + +while (!eof && (n =bulkread (fd, rbuf, c.sldc_cc.rlen, + &eof)) >0 ) { +int ret = c.sldc_compr->compress (&c.sldc_cc); +u64 wlen; +u32 csize =ALIGN_UP (c.sldc_cc.clen + +COMPRESS_HEADER_SIZE ,BLOCK_SZ ); +unsigned int cur_cblk; + +if (ret || n < (int )(csize +BLOCK_SZ * + c.sldc_min_cbpc)) { + wlen =f2fs_write (sbi, de->ino, rbuf, n, off); +ASSERT ((int )wlen == n); + }else { + wlen =f2fs_write_addrtag (sbi, de->ino, off, +WR_COMPRESS_ADDR ); +ASSERT (!wlen); + wlen =f2fs_write_compress_data (sbi, de->ino, + (u8 *)c.sldc_cc.cbuf, + csize, off +BLOCK_SZ ); +ASSERT (wlen == csize); +sldc_erase_bufs (&c.sldc_cc); + cur_cblk = (c.sldc_cc.rlen - csize) /BLOCK_SZ ; + cblocks += cur_cblk; + wlen =f2fs_fix_mutable (sbi, de->ino, + off +BLOCK_SZ + csize, + cur_cblk); +ASSERT (!wlen); + } + off += n; + } +if (n ==-1 ) { +fprintf (stderr,"Load file '%s' failed: " , + de->full_path); +perror (NULL ); + } +/* read inode */ +get_node_info (sbi, de->ino, &ni); +ASSERT (dev_read_block (node_blk, ni.blk_addr) >=0 ); +/* update inode meta */ + node_blk->i.i_size =cpu_to_le64 (off); +if (!c.sldc_immutable) + node_blk->i.i_compr_blocks =cpu_to_le64 (cblocks); +ASSERT (write_inode (node_blk, ni.blk_addr) >=0 ); +free (node_blk); + +if (!c.sldc_immutable) { + sbi->total_valid_block_count += cblocks; +if (sbi->total_valid_block_count >= + sbi->user_block_count) { +ERR_MSG ("Not enough space\n" ); +ASSERT (0 ); + } + } +#endif }else {while ((n =read (fd, buffer,BLOCK_SZ )) >0 ) {f2fs_write (sbi, de->ino, buffer, n, off);fsck/sload.c +67
-0@@ -6,6 +6,9 @@* Hou Pengyang <[email protected]> * Liu Shuoran <[email protected]> * Jaegeuk Kim <[email protected]> +* Copyright (c) 2020 Google Inc. +* Robin Hsu <[email protected]> +* : add sload compression support * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -317,6 +320,70 @@ skip:return 0 ; } +typedef struct _ext_tbl { +const char *ext; +struct _ext_tbl *next;/* linked list */ +}ext_tbl_t ; +static ext_tbl_t *ext_tbl; + +static bool ext_found (const char *ext) +{ +ext_tbl_t *p = ext_tbl; + +while (p !=NULL &&strcmp (ext, p->ext)) + p = p->next; +return (p !=NULL ); +} + +static const char *get_ext (const char *path) +{ +char *p =strrchr (path,'.' ); +return p ==NULL ? path +strlen (path) : p +1 ; +} + +static bool ext_do_filter (const char *path) +{ +return (ext_found (get_ext (path)) == true) ^ (c.sldc_policy ==FP_ALLOW ); +} + +static void ext_filter_add (const char *ext) +{ +ext_tbl_t *node; + +ASSERT (ext !=NULL ); +if (ext_found (ext)) +return ;/* ext was already registered */ + node =malloc (sizeof (ext_tbl_t)); +ASSERT (node !=NULL ); + node->ext = ext; + node->next = ext_tbl; + ext_tbl = node; +} + +static void ext_filter_destroy (void ) +{ +ext_tbl_t *p; + +while (ext_tbl !=NULL ) { + p = ext_tbl; + ext_tbl = p->next; +free (p); + } +} + +struct ext_tbl_op ext_filter = { + .add = ext_filter_add, + .destroy = ext_filter_destroy, + .filter = ext_do_filter, +}; + +void sldc_erase_bufs (struct compress_ctx *cc) +{ +memset (cc->rbuf,0 , cc->cluster_size *F2FS_BLKSIZE ); +memset (cc->cbuf->cdata,0 , cc->cluster_size *F2FS_BLKSIZE + -F2FS_BLKSIZE ); +} +int f2fs_sload (struct f2fs_sb_info *sbi) {int ret =0 ;include/f2fs_fs.h +75
-1@@ -5,6 +5,9 @@* http://www.samsung.com/ * Copyright (c) 2019 Google Inc. * http://www.google.com/ +* Copyright (c) 2020 Google Inc. +* Robin Hsu <[email protected]> +* : add sload compression support * * Dual licensed under the GPL or LGPL version 2 licenses. * @@ -68,6 +71,10 @@ typedef uint16_t u_int16_t;typedef uint8_t u_int8_t ;#endif +/* codes from kernel's f2fs.h, GPL-v2.0 */ +#define MIN_COMPRESS_LOG_SIZE 2 +#define MAX_COMPRESS_LOG_SIZE 8 +typedef u_int64_t u64 ;typedef u_int32_t u32 ;typedef u_int16_t u16 ;@@ -93,6 +100,31 @@ typedef u32 __be32;typedef u64 __be64 ;#endif +/* +* code borrowed from kernel f2fs dirver: f2fs.h, GPL-2.0 +* : definitions of COMPRESS_DATA_RESERVED_SIZE, +* struct compress_data, COMPRESS_HEADER_SIZE, +* and struct compress_ctx +*/ +#define COMPRESS_DATA_RESERVED_SIZE 4 +struct compress_data { +__le32 clen;/* compressed data size */ +__le32 chksum;/* checksum of compressed data */ +__le32 reserved[COMPRESS_DATA_RESERVED_SIZE ];/* reserved */ +u8 cdata[];/* compressed data */ +}; +#define COMPRESS_HEADER_SIZE (sizeof(struct compress_data)) +/* compress context */ +struct compress_ctx { +unsigned int cluster_size;/* page count in cluster */ +unsigned int log_cluster_size;/* log of cluster size */ +void *rbuf;/* compression input buffer */ +struct compress_data *cbuf;/* comprsssion output header + data */ +size_t rlen;/* valid data length in rbuf */ +size_t clen;/* valid data length in cbuf */ +void *private;/* work buf for compress algorithm */ +}; +#if HAVE_BYTESWAP_H #include <byteswap.h> #else @@ -345,6 +377,25 @@ typedef struct {bool dbg_en; } dev_cache_config_t; +/* f2fs_configration: sldc_ca, the sload compress algorithm */ +enum {CA_LZO ,CA_LZ4 }; +extern const char *ca_names[]; + +typedef struct { +void (*init)(struct compress_ctx *cc); +int (*compress)(struct compress_ctx *cc); +}compress_ops ; + +#define ALIGN_UP (value, size) ((value) + ((value) % (size) > 0 ? \ + (size) - (value) % (size) : 0)) + +enum filter_policy {FP_UNASSIGNED =0 ,FP_ALLOW ,FP_DENY }; +struct ext_tbl_op { +void (*add)(const char *); +void (*destroy)(void ); +bool (*filter)(const char *); +}; +struct f2fs_configuration {u_int32_t reserved_segments;u_int32_t new_reserved_segments;@@ -441,6 +492,24 @@ struct f2fs_configuration {/* cache parameters */ dev_cache_config_t cache_config; + +/* quick dynamic buffer */ +bool qbuf_initialized; +size_t qbufsize; +void *qbuf; + +/* sldc: sload compression support */ +bool sldc_en; +bool sldc_use_allow_list;/* default false to use the deny list */ +struct compress_ctx sldc_cc; +u8 sldc_ca;/* compress algorithm: 0 = LZO, 1 = LZ4 */ +compress_ops *sldc_compr; +enum filter_policy sldc_policy; +/* max_cppc can used to specify minimum compression rate */ +unsigned int sldc_min_cbpc;/* min compressed pages per cluster */ +bool sldc_got_opt; +bool sldc_immutable; +struct ext_tbl_op *sldc_ef;/* extension filter */ };#ifdef CONFIG_64BIT @@ -1226,6 +1295,11 @@ extern void f2fs_release_sparse_resource(void);extern int f2fs_finalize_device (void );extern int f2fs_fsync_device (void ); +/* quick (shared) buffer */ +extern void qbuf_free (void ); +extern void *qbuf_alloc (size_t size); +extern void qbuf_init (void ); +extern void dcache_init (void );extern void dcache_release (void );@@ -1377,7 +1451,7 @@ int f2fs_reset_zone(int, void *);extern int f2fs_reset_zones (int );extern uint32_t f2fs_get_usable_segments (struct f2fs_super_block *sb); -#define SIZE_ALIGN (val, size) ((val) + (size) - 1) / (size) +#define SIZE_ALIGN (val, size) (((val) + (size) - 1) / (size))#define SEG_ALIGN (blks) SIZE_ALIGN(blks, c.blks_per_seg)#define ZONE_ALIGN (blks) SIZE_ALIGN(blks, c.blks_per_seg * \ c.segs_per_zone)lib/libf2fs_io.c +33
-0@@ -5,6 +5,9 @@* http://www.samsung.com/ * Copyright (c) 2019 Google Inc. * http://www.google.com/ +* Copyright (c) 2020 Google Inc. +* Robin Hsu <[email protected]> +* : add quick-buffer for sload compression support * * Dual licensed under the GPL or LGPL version 2 licenses. */ @@ -106,6 +109,36 @@ static long dcache_relocate_offset0[] = { };static int dcache_relocate_offset[16 ]; +/* quick (shared) buffer */ +static bool qbuf_initialized; +static void *qbuf; +static size_t qbufsize; +void qbuf_free (void ) +{ +ASSERT (qbuf_initialized); +if (qbuf !=NULL ) { +free (qbuf); + qbuf =NULL ; + qbufsize =0 ; + } +} +void *qbuf_alloc (size_t size) +{ +ASSERT (qbuf_initialized); +if (size > qbufsize) { +qbuf_free (); + qbuf =malloc (size); + } +return qbuf; +} +void qbuf_init (void ) +{ +if (qbuf_initialized) +return ; +atexit (qbuf_free); + qbuf_initialized = true; +} +static void dcache_print_statistics (void ) {long i;Signature · 1 line
2.29.2.576.ga3fc446d84-goog
_______________________________________________
Linux-f2fs-devel mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
Re: [f2fs-dev] [PATCH v3 2/3] f2fs-tools:sload.f2fs compression support
In reply to Robin HsuQuoted 11 lines · Robin Hsu
From: Robin Hsu <[email protected]>
Add F2FS compression support for sload
* Support file extension filter, either default-accept or default-deny policy
* Support choice of compression algorithm, LZO (version 2) or LZ4 (default)
* Support custom log of cluster size
* Support minimum number of compressed blocks per cluster (default 1). A cluster will not be compressed if the number can not be met.
* suuport -r (read-only) option
Could you please update manual as well?
include/f2fs_fs.h struct f2fs_configuration { · line 501
Quoted 2 lines
include/f2fs_fs.h +2
-0+ +/* sldc: sload compression support */
Personally, I don't like the naming method of adding "sldc_" prefix... :(
include/f2fs_fs.h struct f2fs_configuration { · line 512
Quoted 11 lines
include/f2fs_fs.h +11
-0+bool sldc_en; +bool sldc_use_allow_list;/* default false to use the deny list */ +struct compress_ctx sldc_cc; +u8 sldc_ca;/* compress algorithm: 0 = LZO, 1 = LZ4 */ +compress_ops *sldc_compr; +enum filter_policy sldc_policy; +/* max_cppc can used to specify minimum compression rate */ +unsigned int sldc_min_cbpc;/* min compressed pages per cluster */ +bool sldc_got_opt; +bool sldc_immutable; +struct ext_tbl_op *sldc_ef;/* extension filter */
The variables name like sldc_en, sldc_ca, min_cbpc, sldc_ef makes developers hard to understand w/o comments, and also there is no comments for several variable like sldc_en, sldc_cc...
Could you please improve the naming like f2fs-tools style?
Thanks,
_______________________________________________
Linux-f2fs-devel mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
A message that is not in the archive (sent off-list or not yet received): CAKnFrsLmEROi+ZwVCmoC=W7u+rVoZfWGC9Lr9_y=oLMUZMw63Q@mail.gmail.com
Re: [f2fs-dev] [PATCH v3 2/3] f2fs-tools:sload.f2fs compression support
Quoted 10 lines · 徐瑞斌
Hi, Jaegeuk,
I comment here the patch your provided (3 parts, since the patch contains 3 fixes):
1. + dn->data_blkaddr = blkaddr; ret = reserve_new_block(sbi, &dn->data_blkaddr, &sum, type, 0);We cannot assign
dn->data_blkaddrhere. The old one is to be used inreserve_new_block()function. Also,reserve_new_block()function actually will setdn->data_blkaddrto blkaddr in the end.
This tries to avoid deleting the block address used in the previous offset. Otherwise, we'll see wrong i_blocks.
Quoted 8 lines
2. Added condition "n < (1 << c.sldc_cc.log_cluster_size) *
BLOCK_SZ"The semantic meaning of the whole if statement is to say:
When the compression fail (ret != 0) or the original read size is
smaller than the compressed size plus (the minimum block saved (specified by the user) x block size), we will not do compression but just write the data as is.
This is missing the last block having < 4Kb.
Quoted 8 lines
The right hand side (RHS) of your added condition is exactly the read size, i.e. the cluster size. That means the condition is always false except the read of the last part of the file, when the file size is not exactly the multiple of the cluster size. That means we will never try to compress the last part of the file (when the last part is not a multiple of the cluster size)
Quoted 8 lines
IMHO, the original implementation should be correct.
3.
node_blk->i.i_blocks+=cpu_to_le64(cblocks);I am not quite sure of the
i_blockscount. Did you mean that when the file
is mutable, meaning that the file reserves some blocks for future write,
we will add count toi_blocksto mark the block as a used block by the file, right? I thought we only need to increment the allocated count...
Should add it.
Quoted 48 lines
Regards,
Robin Hsu 徐瑞斌Quoted 41 lines · Chao Yu
Quoted 11 lines · Robin Hsu
From: Robin Hsu <[email protected]>
Add F2FS compression support for sload
* Support file extension filter, either default-accept or default-deny policy
* Support choice of compression algorithm, LZO (version 2) or LZ4 (default)
* Support custom log of cluster size
* Support minimum number of compressed blocks per cluster (default 1). A cluster will not be compressed if the number can not be met.
* suuport -r (read-only) optionCould you please update manual as well?
include/f2fs_fs.h struct f2fs_configuration { · line 501
Quoted 2 lines
include/f2fs_fs.h +2
-0+ +/* sldc: sload compression support */ Personally, I don't like the naming method of adding "
sldc_" prefix... :(include/f2fs_fs.h struct f2fs_configuration { · line 512
Quoted 11 lines
include/f2fs_fs.h +11
-0+bool sldc_en; +bool sldc_use_allow_list;/* default false to use the deny list */ +struct compress_ctx sldc_cc; +u8 sldc_ca;/* compress algorithm: 0 = LZO, 1 = LZ4 */ +compress_ops *sldc_compr; +enum filter_policy sldc_policy; +/* max_cppc can used to specify minimum compression rate */ +unsigned int sldc_min_cbpc;/* min compressed pages per cluster */ +bool sldc_got_opt; +bool sldc_immutable; +struct ext_tbl_op *sldc_ef;/* extension filter */ The variables name like
sldc_en,sldc_ca,min_cbpc,sldc_efmakes developers
hard to understand w/o comments, and also there is no comments for several variable likesldc_en,sldc_cc...Could you please improve the naming like f2fs-tools style?
Thanks,
_______________________________________________
Linux-f2fs-devel mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
Re: [f2fs-dev] [PATCH v3 2/3] f2fs-tools:sload.f2fs compression support
Chao and Robin,
I refactored the patch to modify the names and structures. Please take a look at this.
https://git.kernel.org/pub/scm/linux/kernel/git/jaegeuk/f2fs-tools.git/commit/?h=dev&id=5cd8e5fcc79611c0692f18c7e7e13d6b4742d3c0
Quoted 99 lines · Jaegeuk Kim
Quoted 10 lines · 徐瑞斌
Hi, Jaegeuk,
I comment here the patch your provided (3 parts, since the patch contains 3 fixes):
1. + dn->data_blkaddr = blkaddr; ret = reserve_new_block(sbi, &dn->data_blkaddr, &sum, type, 0);We cannot assign
dn->data_blkaddrhere. The old one is to be used inreserve_new_block()function. Also,reserve_new_block()function actually will setdn->data_blkaddrto blkaddr in the end.This tries to avoid deleting the block address used in the previous offset. Otherwise, we'll see wrong
i_blocks.Quoted 8 lines
2. Added condition "n < (1 << c.sldc_cc.log_cluster_size) *
BLOCK_SZ"The semantic meaning of the whole if statement is to say:
When the compression fail (ret != 0) or the original read size is
smaller than the compressed size plus (the minimum block saved (specified by the user) x block size), we will not do compression but just write the data as is.This is missing the last block having < 4Kb.
Quoted 8 lines
The right hand side (RHS) of your added condition is exactly the read size, i.e. the cluster size. That means the condition is always false except the read of the last part of the file, when the file size is not exactly the multiple of the cluster size. That means we will never try to compress the last part of the file (when the last part is not a multiple of the cluster size)
Quoted 8 lines
IMHO, the original implementation should be correct.
3.
node_blk->i.i_blocks+=cpu_to_le64(cblocks);I am not quite sure of the
i_blockscount. Did you mean that when the file
is mutable, meaning that the file reserves some blocks for future write,
we will add count toi_blocksto mark the block as a used block by the file, right? I thought we only need to increment the allocated count...Should add it.
Quoted 48 lines
Regards,
Robin Hsu 徐瑞斌Quoted 41 lines · Chao Yu
Quoted 11 lines · Robin Hsu
From: Robin Hsu <[email protected]>
Add F2FS compression support for sload
* Support file extension filter, either default-accept or default-deny policy
* Support choice of compression algorithm, LZO (version 2) or LZ4 (default)
* Support custom log of cluster size
* Support minimum number of compressed blocks per cluster (default 1). A cluster will not be compressed if the number can not be met.
* suuport -r (read-only) optionCould you please update manual as well?
include/f2fs_fs.h struct f2fs_configuration { · line 501
Quoted 2 lines
include/f2fs_fs.h +2
-0+ +/* sldc: sload compression support */ Personally, I don't like the naming method of adding "
sldc_" prefix... :(include/f2fs_fs.h struct f2fs_configuration { · line 512
Quoted 11 lines
include/f2fs_fs.h +11
-0+bool sldc_en; +bool sldc_use_allow_list;/* default false to use the deny list */ +struct compress_ctx sldc_cc; +u8 sldc_ca;/* compress algorithm: 0 = LZO, 1 = LZ4 */ +compress_ops *sldc_compr; +enum filter_policy sldc_policy; +/* max_cppc can used to specify minimum compression rate */ +unsigned int sldc_min_cbpc;/* min compressed pages per cluster */ +bool sldc_got_opt; +bool sldc_immutable; +struct ext_tbl_op *sldc_ef;/* extension filter */ The variables name like
sldc_en,sldc_ca,min_cbpc,sldc_efmakes developers
hard to understand w/o comments, and also there is no comments for several variable likesldc_en,sldc_cc...Could you please improve the naming like f2fs-tools style?
Thanks,
_______________________________________________
Linux-f2fs-devel mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
Re: [f2fs-dev] [PATCH v3 2/3] f2fs-tools:sload.f2fs compression support
In reply to Robin HsuQuoted 881 lines · Robin Hsu
From: Robin Hsu <[email protected]>
Add F2FS compression support for sload
* Support file extension filter, either default-accept or default-deny policy
* Support choice of compression algorithm, LZO (version 2) or LZ4 (default)
* Support custom log of cluster size
* Support minimum number of compressed blocks per cluster (default 1). A cluster will not be compressed if the number can not be met.
* suuport -r (read-only) optionSigned-off-by Robin Hsu <[email protected]>--- fsck/compress_wrapper.c | 102 ++++++++++++++++++++ fsck/compress_wrapper.h | 22 +++++ fsck/fsck.h | 15 +++ fsck/main.c | 141 +++++++++++++++++++++++++++- fsck/segment.c | 202 +++++++++++++++++++++++++++++++++++++--- fsck/sload.c | 67 +++++++++++++ include/f2fs_fs.h | 76 ++++++++++++++- lib/libf2fs_io.c | 33 +++++++ 8 files changed, 644 insertions(+), 14 deletions(-) create mode 100644 fsck/compress_wrapper.c create mode 100644 fsck/compress_wrapper.hfsck/compress_wrapper.c new file +102
-0@@ -0,0 +1,102 @@ +/** +* compress_wrapper.c +* +* Copyright (c) 2020 Google Inc. +* Robin Hsu <[email protected]> +* : initial created, for sload compression support +* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License version 2 as +* published by the Free Software Foundation. +*/ +#include "f2fs.h" /* for config.h for general environment (non-Android) */ + +#include "compress_wrapper.h" +#ifdef HAVE_LIBLZO2 +#include <lzo/lzo1x.h> /* for lzo1x_1_15_compress() */ +#endif +#ifdef HAVE_LIBLZ4 +#include <lz4.h> /* for LZ4_compress_fast_extState() */ +#endif + +/* +* macro/constants borrowed from kernel header (GPL-2.0): +* include/linux/lzo.h, and include/linux/lz4.h +*/ +#ifdef HAVE_LIBLZO2 +#define lzo1x_worst_compress (x) ((x) + (x) / 16 + 64 + 3 + 2) +#define LZO_WORK_SIZE ALIGN_UP(LZO1X_1_15_MEM_COMPRESS, 8) +#endif +#ifdef HAVE_LIBLZ4 +#define LZ4_MEMORY_USAGE 14 +#define LZ4_MAX_INPUT_SIZE 0x7E000000/* 2 113 929 216 bytes */ +#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(long long)) +#define LZ4_MEM_COMPRESS LZ4_STREAMSIZE +#define LZ4_ACCELERATION_DEFAULT 1 +#define LZ4_WORK_SIZE ALIGN_UP(LZ4_MEM_COMPRESS, 8) +#endif + +#ifdef HAVE_LIBLZO2 +static void lzo_compress_init (struct compress_ctx *cc) +{ +size_t size = cc->cluster_size *F2FS_BLKSIZE ; +size_t alloc = size +lzo1x_worst_compress (size) + +COMPRESS_HEADER_SIZE +LZO_WORK_SIZE ; +ASSERT ((cc->private =qbuf_alloc (alloc)) !=NULL ); + cc->rbuf = (char *) cc->private +LZO_WORK_SIZE ; + cc->cbuf = (struct compress_data *)((char *) cc->rbuf + size); +} + +static int lzo_compress (struct compress_ctx *cc) +{ +int ret =lzo1x_1_15_compress (cc->rbuf, cc->rlen, cc->cbuf->cdata, + (lzo_uintp)(&cc->clen), cc->private); + cc->cbuf->clen =cpu_to_le32 (cc->clen); +return ret; +} +#endif + +#ifdef HAVE_LIBLZ4 +static void lz4_compress_init (struct compress_ctx *cc) +{ +size_t size = cc->cluster_size *F2FS_BLKSIZE ; +size_t alloc = size +LZ4_COMPRESSBOUND (size) + +COMPRESS_HEADER_SIZE +LZ4_WORK_SIZE ; +ASSERT ((cc->private =qbuf_alloc (alloc)) !=NULL ); + cc->rbuf = (char *) cc->private +LZ4_WORK_SIZE ; + cc->cbuf = (struct compress_data *)((char *) cc->rbuf + size); +} + +static int lz4_compress (struct compress_ctx *cc) +{ + cc->clen =LZ4_compress_fast_extState (cc->private, cc->rbuf, + (char *)cc->cbuf->cdata, cc->rlen, + cc->rlen -F2FS_BLKSIZE * c.sldc_min_cbpc, +LZ4_ACCELERATION_DEFAULT ); + +if (!cc->clen) +return 1 ; + + cc->cbuf->clen =cpu_to_le32 (cc->clen); +return 0 ; +} +#endif + +const char *ca_names[] = { +"LZO" , +"LZ4" , +"" ,/* end of the name list */ +}; + +compress_ops compr_ops[] = { +#ifdef HAVE_LIBLZO2 + {lzo_compress_init, lzo_compress}, +#else + {NULL, NULL}, +#endif +#ifdef HAVE_LIBLZ4 + {lz4_compress_init, lz4_compress}, +#else + {NULL, NULL}, +#endif +};fsck/compress_wrapper.h new file +22
-0@@ -0,0 +1,22 @@ +/** +* compress_wrapper.h +* +* Copyright (c) 2020 Google Inc. +* Robin Hsu <[email protected]> +* : initial created, for sload compression support +* Copyright (c) 2020 Google Inc. +* Robin Hsu <[email protected]> +* : add sload compression support +* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License version 2 as +* published by the Free Software Foundation. +*/ + +#ifndef COMPRESS_WRAPPER_H +#define COMPRESS_WRAPPER_H + +#include "f2fs_fs.h" +extern compress_ops compr_ops[];/* [0]: LZO, [1]: LZ4, */ + +#endif /* COMPRESS_WRAPPER_H */ fsck/fsck.h +15
-0@@ -3,6 +3,9 @@* * Copyright (c) 2013 Samsung Electronics Co., Ltd. * http://www.samsung.com/ +* Copyright (c) 2020 Google Inc. +* Robin Hsu <[email protected]> +* : add sload compression support * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -266,6 +269,9 @@ int f2fs_resize(struct f2fs_sb_info *);/* sload.c */ int f2fs_sload (struct f2fs_sb_info *); +void sldc_erase_bufs (struct compress_ctx *cc); +void sload_countblk (void ); +extern struct ext_tbl_op ext_filter;/* segment.c */ int reserve_new_block(struct f2fs_sb_info *, block_t *,@@ -282,7 +288,16 @@ block_t new_node_block(struct f2fs_sb_info *,struct quota_file ;u64 f2fs_quota_size (struct quota_file *);u64 f2fs_read (struct f2fs_sb_info *,nid_t ,u8 *,u64 ,pgoff_t ); +enum wr_addr_type { +WR_NORMAL =1 , +WR_COMPRESS_DATA =2 , +WR_NULL_ADDR =NULL_ADDR ,/* 0 */ +WR_NEW_ADDR =NEW_ADDR ,/* -1U */ +WR_COMPRESS_ADDR =COMPRESS_ADDR ,/* -2U */ +};u64 f2fs_write (struct f2fs_sb_info *,nid_t ,u8 *,u64 ,pgoff_t ); +u64 f2fs_write_compress_data (struct f2fs_sb_info *,nid_t ,u8 *,u64 ,pgoff_t ); +u64 f2fs_write_addrtag (struct f2fs_sb_info *,nid_t ,pgoff_t ,unsigned int );void f2fs_filesize_update (struct f2fs_sb_info *,nid_t ,u64 ); int get_dnode_of_data(struct f2fs_sb_info *, struct dnode_of_data *,fsck/main.c +140
-1@@ -13,6 +13,9 @@* Copyright (c) 2019 Google Inc. * Robin Hsu <[email protected]> * : add cache layer +* Copyright (c) 2020 Google Inc. +* Robin Hsu <[email protected]> +* : add sload compression support * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -25,6 +28,7 @@#include <getopt.h> #include <stdbool.h> #include "quotaio.h" +#include "compress_wrapper.h" struct f2fs_fsck gfsck;@@ -134,6 +138,17 @@ void sload_usage()MSG (0 ," -S sparse_mode\n" );MSG (0 ," -t mount point [prefix of target fs path, default:/]\n" );MSG (0 ," -T timestamp\n" ); +MSG (0 ," -c enable compression (default allow policy)\n" ); +MSG (0 ," ------------ Compression sub-options -----------------\n" ); +MSG (0 ," -L <log-of-blocks-per-cluster>, default 2\n" ); +MSG (0 ," -a <algorithm> compression algorithm, default LZ4\n" ); +MSG (0 ," -x <ext> compress files except for these extensions.\n" ); +MSG (0 ," -i <ext> compress files with these extensions only.\n" ); +MSG (0 ," * -i or -x: use it many times for multiple extensions.\n" ); +MSG (0 ," * -i and -x cannot be used together..\n" ); +MSG (0 ," -m <num> min compressed blocks per cluster\n" ); +MSG (0 ," -r readonly (IMMUTABLE) for compressed files\n" ); +MSG (0 ," ------------------------------------------------------\n" );MSG (0 ," -d debug level [default:0]\n" );MSG (0 ," -V print the version number and exit\n" );exit (1 );@@ -534,7 +549,7 @@ void f2fs_parse_options(int argc, char *argv[])#endif }else if (!strcmp ("sload.f2fs" , prog)) {#ifdef WITH_SLOAD -const char *option_string ="C:d:f:p:s:St:T:V" ; +const char *option_string ="cL:a:i:x:m:rC:d:f:p:s:St:T:V" ;#ifdef HAVE_LIBSELINUX int max_nr_opt = (int )sizeof (c.seopt_file) /sizeof (c.seopt_file[0 ]);@@ -543,8 +558,82 @@ void f2fs_parse_options(int argc, char *argv[])char *p; c.func =SLOAD ; + c.sldc_cc.log_cluster_size =2 ; + c.sldc_ca =CA_LZ4 ; + c.sldc_min_cbpc =1 ; + c.sldc_ef = &ext_filter;while ((option =getopt (argc, argv, option_string)) !=EOF ) { +unsigned int i; +int val; +switch (option) { +case 'c' :/* compression support */ + c.sldc_en = true; +break ; +case 'L' :/* compression: log of blocks-per-cluster */ + c.sldc_got_opt = true; + val =atoi (optarg); +if (val <MIN_COMPRESS_LOG_SIZE || + val >MAX_COMPRESS_LOG_SIZE ) { +MSG (0 ,"\tError: log of blocks per" +" cluster must be in the range" +" of %d .. %d.\n" , +MIN_COMPRESS_LOG_SIZE , +MAX_COMPRESS_LOG_SIZE ); +error_out (prog); + } + c.sldc_cc.log_cluster_size = val; +break ; +case 'a' :/* compression: choose algorithm */ + c.sldc_got_opt = true; + c.sldc_ca = (u8)-1 ; +for (i =0 ; ca_names[i][0 ] !=0 ; i++) { +if (!strcmp (ca_names[i], optarg)) { + c.sldc_ca = i; +break ; + } + } +if (c.sldc_ca == (u8)-1 ) { +MSG (0 ,"\tError: Unknown compression" +" algorithm %s\n" , optarg); +error_out (prog); + } +break ; +case 'i' :/* compress only these extensions */ + c.sldc_got_opt = true; +if (c.sldc_policy ==FP_ALLOW ) { +MSG (0 ,"\tError: could not mix option" +" -i and -x\n" ); +error_out (prog); + } + c.sldc_policy =FP_DENY ; + c.sldc_ef->add (optarg); +break ; +case 'x' :/* compress except for these extensions */ + c.sldc_got_opt = true; +if (c.sldc_policy ==FP_DENY ) { +MSG (0 ,"\tError: could not mix option" +" -i and -x\n" ); +error_out (prog); + } + c.sldc_policy =FP_ALLOW ; + c.sldc_ef->add (optarg); +break ; +case 'm' :/* minimum compressed blocks per cluster */ + c.sldc_got_opt = true; + val =atoi (optarg); +if (val <=0 ) { +MSG (0 ,"\tError: minimum compressed" +" blocks per cluster must be" +" positive.\n" ); +error_out (prog); + } + c.sldc_min_cbpc = val; +break ; +case 'r' :/* compress file to set IMMUTABLE */ + c.sldc_got_opt = true; + c.sldc_immutable = true; +break ;case 'C' : c.fs_config_file =absolute_path (optarg);break ;@@ -602,6 +691,27 @@ void f2fs_parse_options(int argc, char *argv[])if (err !=NOERROR ) break; } +if (c.sldc_got_opt && !c.sldc_en) { +MSG (0 ,"\tError: compression sub-options are used" +" without the compression enable (-c) option\n" + ); +error_out (prog); + } +if (err ==NOERROR && c.sldc_en) { + c.sldc_cc.cluster_size =1 + << c.sldc_cc.log_cluster_size; +if (c.sldc_policy ==FP_UNASSIGNED ) + c.sldc_policy =FP_ALLOW ; +if (c.sldc_min_cbpc >= c.sldc_cc.cluster_size) { +MSG (0 ,"\tError: minimum reduced blocks by" +" compression per cluster must be at" +" most one less than blocks per" +" cluster, i.e. %d\n" , + c.sldc_cc.cluster_size -1 ); +error_out (prog); + } +qbuf_init (); + }#endif /* WITH_SLOAD */ }@@ -812,6 +922,27 @@ static int do_resize(struct f2fs_sb_info *sbi)#endif #ifdef WITH_SLOAD +int init_compr (struct f2fs_sb_info *sbi) +{ +if (!(sbi->raw_super->feature + &cpu_to_le32 (F2FS_FEATURE_COMPRESSION ))) { +MSG (0 ,"Error: Compression (-c) was requested " +"but the file system is not created " +"with such feature.\n" ); +return -1 ; + } +if (compr_ops[c.sldc_ca].init ==NULL ) { +MSG (0 ,"Error: The selected compression algorithm is not" +" supported\n" ); +return -1 ; + } + c.sldc_compr = compr_ops + c.sldc_ca; + c.sldc_compr->init (&c.sldc_cc); +sldc_erase_bufs (&c.sldc_cc); + c.sldc_cc.rlen = c.sldc_cc.cluster_size *F2FS_BLKSIZE ; +return 0 ; +} +static int do_sload (struct f2fs_sb_info *sbi) {if (!c.from_dir) {@@ -821,6 +952,11 @@ static int do_sload(struct f2fs_sb_info *sbi)if (!c.mount_point) c.mount_point ="/" ; +if (c.sldc_en) { +if (init_compr (sbi)) +return -1 ; + } + return f2fs_sload(sbi); }#endif @@ -971,6 +1107,9 @@ retry:return ret2; } +if (c.func ==SLOAD ) + c.sldc_ef->destroy (); +printf ("\nDone: %lf secs\n" , (get_boottime_ns () - start) /1000000000.0 );return ret;fsck/segment.c +190
-12@@ -8,6 +8,9 @@* Hou Pengyang <[email protected]> * Liu Shuoran <[email protected]> * Jaegeuk Kim <[email protected]> +* Copyright (c) 2020 Google Inc. +* Robin Hsu <[email protected]> +* : add sload compression support * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -16,6 +19,7 @@#include "fsck.h" #include "node.h" #include "quotaio.h" +#include "compress_wrapper.h" int reserve_new_block (struct f2fs_sb_info *sbi,block_t *to,struct f2fs_summary *sum,int type,bool is_inode)@@ -228,8 +232,14 @@ u64 f2fs_read(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,return read_count; } -u64 f2fs_write (struct f2fs_sb_info *sbi,nid_t ino,u8 *buffer, -u64 count,pgoff_t offset) +/* +* Do not call this function directly. Instead, call one of the following: +* u64 f2fs_write(); +* u64 f2fs_write_compress_data(); +* u64 f2fs_write_addrtag(); +*/ +static u64 f2fs_write_ex (struct f2fs_sb_info *sbi,nid_t ino,u8 *buffer, +u64 count,pgoff_t offset,enum wr_addr_type addr_type) {struct dnode_of_data dn;struct node_info ni;@@ -243,6 +253,19 @@ u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,void * index_node =NULL ;int idirty =0 ;int err; +bool has_data = (addr_type ==WR_NORMAL + || addr_type ==WR_COMPRESS_DATA ); + +if (count ==0 ) +return 0 ; + +/* +* Enforce calling from f2fs_write(), f2fs_write_compress_data(), +* and f2fs_write_addrtag(). Beside, check if is properly called. +*/ +ASSERT ((!has_data && buffer ==NULL ) || (has_data && buffer !=NULL )); +if (addr_type !=WR_NORMAL ) +ASSERT (offset %F2FS_BLKSIZE ==0 );/* block boundary only */ /* Memory allocation for block buffer and inode. */ blk_buffer =calloc (BLOCK_SZ ,2 );@@ -265,15 +288,26 @@ u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,if (err)break ; idirty |= dn.idirty; -if (index_node) -free (index_node); +free (index_node); index_node = (dn.node_blk == dn.inode_blk) ? -NULL : dn.node_blk; +NULL : dn.node_blk; remained_blkentries =ADDRS_PER_PAGE (sbi, - dn.node_blk, dn.inode_blk); + dn.node_blk, dn.inode_blk) - + dn.ofs_in_node; }ASSERT (remained_blkentries >0 ); +if (!has_data) { + dn.data_blkaddr = addr_type; +set_data_blkaddr (&dn); + idirty |= dn.idirty; +if (dn.ndirty) +ASSERT (dev_write_block (dn.node_blk, + dn.node_blkaddr) >=0 ); + written_count =0 ; +break ; + } + blkaddr =datablock_addr (dn.node_blk, dn.ofs_in_node);if (blkaddr ==NULL_ADDR || blkaddr ==NEW_ADDR ) { err = new_data_block(sbi, blk_buffer,@@ -281,6 +315,7 @@ u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,if (err)break ; blkaddr = dn.data_blkaddr; + idirty |= dn.idirty; } off_in_blk = offset %BLOCK_SZ ;@@ -305,9 +340,10 @@ u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer, dn.ofs_in_node++;if ((--remained_blkentries ==0 || count ==0 ) && (dn.ndirty)) -ASSERT (dev_write_block (dn.node_blk, dn.node_blkaddr) >=0 ); +ASSERT (dev_write_block (dn.node_blk, dn.node_blkaddr) + >=0 ); } -if (offset >le64_to_cpu (inode->i.i_size)) { +if (addr_type ==WR_NORMAL && offset >le64_to_cpu (inode->i.i_size)) { inode->i.i_size =cpu_to_le64 (offset); idirty =1 ; }@@ -315,13 +351,33 @@ u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,ASSERT (inode == dn.inode_blk);ASSERT (write_inode (inode, ni.blk_addr) >=0 ); } -if (index_node) -free (index_node); + +free (index_node);free (blk_buffer);return written_count; } +u64 f2fs_write (struct f2fs_sb_info *sbi,nid_t ino,u8 *buffer, +u64 count,pgoff_t offset) +{ +return f2fs_write_ex (sbi, ino, buffer, count, offset,WR_NORMAL ); +} + +u64 f2fs_write_compress_data (struct f2fs_sb_info *sbi,nid_t ino,u8 *buffer, +u64 count,pgoff_t offset) +{ +return f2fs_write_ex (sbi, ino, buffer, count, offset,WR_COMPRESS_DATA ); +} + +u64 f2fs_write_addrtag (struct f2fs_sb_info *sbi,nid_t ino,pgoff_t offset, +unsigned int addrtag) +{ +ASSERT (addrtag ==COMPRESS_ADDR || addrtag ==NEW_ADDR + || addrtag ==NULL_ADDR ); +return f2fs_write_ex (sbi, ino,NULL ,F2FS_BLKSIZE , offset, addrtag); +} +/* This function updates only inode->i.i_size */ void f2fs_filesize_update (struct f2fs_sb_info *sbi,nid_t ino,u64 filesize) {@@ -342,11 +398,59 @@ void f2fs_filesize_update(struct f2fs_sb_info *sbi, nid_t ino, u64 filesize)free (inode); } +#define MAX_BULKR_RETRY 5 +int bulkread (int fd,void *rbuf,size_t rsize,bool *eof) +{ +int n =0 ; +int retry =MAX_BULKR_RETRY ; +int cur; + +if (!rsize) +return 0 ; + +if (eof !=NULL ) + *eof = false; +while (rsize && (cur =read (fd, rbuf, rsize)) !=0 ) { +if (cur ==-1 ) { +if (errno ==EINTR && retry--) +continue ; +return -1 ; + } + retry =MAX_BULKR_RETRY ; + + rsize -= cur; + n += cur; + } +if (eof !=NULL ) + *eof = (cur ==0 ); +return n; +} + +u64 f2fs_fix_mutable (struct f2fs_sb_info *sbi,nid_t ino,pgoff_t offset, +unsigned int compressed) +{ +unsigned int i; +u64 wlen; + +if (c.sldc_immutable) +return 0 ; + +for (i =0 ; i < compressed -1 ; i++) { + wlen =f2fs_write_addrtag (sbi, ino, + offset + (i <<F2FS_BLKSIZE_BITS ),NEW_ADDR ); +if (wlen) +return wlen; + } +return 0 ; +} +int f2fs_build_file (struct f2fs_sb_info *sbi,struct dentry *de) {int fd, n;pgoff_t off =0 ;u8 buffer[BLOCK_SZ ]; +struct node_info ni; +struct f2fs_node *node_blk;if (de->ino ==0 )return -1 ;@@ -359,8 +463,6 @@ int f2fs_build_file(struct f2fs_sb_info *sbi, struct dentry *de)/* inline_data support */ if (de->size <=DEF_MAX_INLINE_DATA ) { -struct node_info ni; -struct f2fs_node *node_blk;int ret;get_node_info (sbi, de->ino, &ni);@@ -385,6 +487,82 @@ int f2fs_build_file(struct f2fs_sb_info *sbi, struct dentry *de) node_blk->i.i_size =cpu_to_le64 (de->size);ASSERT (write_inode (node_blk, ni.blk_addr) >=0 );free (node_blk); +#ifdef WITH_SLOAD + }else if (c.func ==SLOAD && c.sldc_en && + c.sldc_ef->filter (de->full_path)) { +bool eof = false; +u8 *rbuf = c.sldc_cc.rbuf; +unsigned int cblocks =0 ; + + node_blk =calloc (BLOCK_SZ ,1 ); +ASSERT (node_blk); + +/* read inode */ +get_node_info (sbi, de->ino, &ni); +ASSERT (dev_read_block (node_blk, ni.blk_addr) >=0 ); +/* update inode meta */ + node_blk->i.i_compress_algrithm = c.sldc_ca; + node_blk->i.i_log_cluster_size = + c.sldc_cc.log_cluster_size; + node_blk->i.i_flags =cpu_to_le32 ( +F2FS_COMPR_FL | + (c.sldc_immutable ?FS_IMMUTABLE_FL :0 )); +ASSERT (write_inode (node_blk, ni.blk_addr) >=0 ); + +while (!eof && (n =bulkread (fd, rbuf, c.sldc_cc.rlen, + &eof)) >0 ) { +int ret = c.sldc_compr->compress (&c.sldc_cc); +u64 wlen; +u32 csize =ALIGN_UP (c.sldc_cc.clen + +COMPRESS_HEADER_SIZE ,BLOCK_SZ ); +unsigned int cur_cblk; + +if (ret || n < (int )(csize +BLOCK_SZ * + c.sldc_min_cbpc)) { + wlen =f2fs_write (sbi, de->ino, rbuf, n, off); +ASSERT ((int )wlen == n); + }else { + wlen =f2fs_write_addrtag (sbi, de->ino, off, +WR_COMPRESS_ADDR ); +ASSERT (!wlen); + wlen =f2fs_write_compress_data (sbi, de->ino, + (u8 *)c.sldc_cc.cbuf, + csize, off +BLOCK_SZ ); +ASSERT (wlen == csize); +sldc_erase_bufs (&c.sldc_cc); + cur_cblk = (c.sldc_cc.rlen - csize) /BLOCK_SZ ; + cblocks += cur_cblk; + wlen =f2fs_fix_mutable (sbi, de->ino, + off +BLOCK_SZ + csize, + cur_cblk); +ASSERT (!wlen); + } + off += n; + } +if (n ==-1 ) { +fprintf (stderr,"Load file '%s' failed: " , + de->full_path); +perror (NULL ); + } +/* read inode */ +get_node_info (sbi, de->ino, &ni); +ASSERT (dev_read_block (node_blk, ni.blk_addr) >=0 ); +/* update inode meta */ + node_blk->i.i_size =cpu_to_le64 (off); +if (!c.sldc_immutable) + node_blk->i.i_compr_blocks =cpu_to_le64 (cblocks); +ASSERT (write_inode (node_blk, ni.blk_addr) >=0 ); +free (node_blk); + +if (!c.sldc_immutable) { + sbi->total_valid_block_count += cblocks; +if (sbi->total_valid_block_count >= + sbi->user_block_count) { +ERR_MSG ("Not enough space\n" ); +ASSERT (0 ); + } + } +#endif }else {while ((n =read (fd, buffer,BLOCK_SZ )) >0 ) {f2fs_write (sbi, de->ino, buffer, n, off);fsck/sload.c +67
-0@@ -6,6 +6,9 @@* Hou Pengyang <[email protected]> * Liu Shuoran <[email protected]> * Jaegeuk Kim <[email protected]> +* Copyright (c) 2020 Google Inc. +* Robin Hsu <[email protected]> +* : add sload compression support * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -317,6 +320,70 @@ skip:return 0 ; } +typedef struct _ext_tbl { +const char *ext; +struct _ext_tbl *next;/* linked list */ +}ext_tbl_t ; +static ext_tbl_t *ext_tbl; + +static bool ext_found (const char *ext) +{ +ext_tbl_t *p = ext_tbl; + +while (p !=NULL &&strcmp (ext, p->ext)) + p = p->next; +return (p !=NULL ); +} + +static const char *get_ext (const char *path) +{ +char *p =strrchr (path,'.' ); +return p ==NULL ? path +strlen (path) : p +1 ; +} + +static bool ext_do_filter (const char *path) +{ +return (ext_found (get_ext (path)) == true) ^ (c.sldc_policy ==FP_ALLOW ); +} + +static void ext_filter_add (const char *ext) +{ +ext_tbl_t *node; + +ASSERT (ext !=NULL ); +if (ext_found (ext)) +return ;/* ext was already registered */ + node =malloc (sizeof (ext_tbl_t)); +ASSERT (node !=NULL ); + node->ext = ext; + node->next = ext_tbl; + ext_tbl = node; +} + +static void ext_filter_destroy (void ) +{ +ext_tbl_t *p; + +while (ext_tbl !=NULL ) { + p = ext_tbl; + ext_tbl = p->next; +free (p); + } +} + +struct ext_tbl_op ext_filter = { + .add = ext_filter_add, + .destroy = ext_filter_destroy, + .filter = ext_do_filter, +}; + +void sldc_erase_bufs (struct compress_ctx *cc) +{ +memset (cc->rbuf,0 , cc->cluster_size *F2FS_BLKSIZE ); +memset (cc->cbuf->cdata,0 , cc->cluster_size *F2FS_BLKSIZE + -F2FS_BLKSIZE ); +} +int f2fs_sload (struct f2fs_sb_info *sbi) {int ret =0 ;include/f2fs_fs.h +51
-0@@ -5,6 +5,9 @@* http://www.samsung.com/ * Copyright (c) 2019 Google Inc. * http://www.google.com/ +* Copyright (c) 2020 Google Inc. +* Robin Hsu <[email protected]> +* : add sload compression support * * Dual licensed under the GPL or LGPL version 2 licenses. * @@ -68,6 +71,10 @@ typedef uint16_t u_int16_t;typedef uint8_t u_int8_t ;#endif +/* codes from kernel's f2fs.h, GPL-v2.0 */ +#define MIN_COMPRESS_LOG_SIZE 2 +#define MAX_COMPRESS_LOG_SIZE 8 +typedef u_int64_t u64 ;typedef u_int32_t u32 ;typedef u_int16_t u16 ;@@ -93,6 +100,31 @@ typedef u32 __be32;typedef u64 __be64 ;#endif +/* +* code borrowed from kernel f2fs dirver: f2fs.h, GPL-2.0 +* : definitions of COMPRESS_DATA_RESERVED_SIZE, +* struct compress_data, COMPRESS_HEADER_SIZE, +* and struct compress_ctx +*/ +#define COMPRESS_DATA_RESERVED_SIZE 4 +struct compress_data { +__le32 clen;/* compressed data size */ +__le32 chksum;/* checksum of compressed data */ +__le32 reserved[COMPRESS_DATA_RESERVED_SIZE ];/* reserved */ +u8 cdata[];/* compressed data */ +}; +#define COMPRESS_HEADER_SIZE (sizeof(struct compress_data)) +/* compress context */ +struct compress_ctx { +unsigned int cluster_size;/* page count in cluster */ +unsigned int log_cluster_size;/* log of cluster size */ +void *rbuf;/* compression input buffer */ +struct compress_data *cbuf;/* comprsssion output header + data */ +size_t rlen;/* valid data length in rbuf */ +size_t clen;/* valid data length in cbuf */ +void *private;/* work buf for compress algorithm */ +}; +#if HAVE_BYTESWAP_H #include <byteswap.h> #else @@ -345,6 +377,25 @@ typedef struct {bool dbg_en; } dev_cache_config_t; +/* f2fs_configration: sldc_ca, the sload compress algorithm */ +enum {CA_LZO ,CA_LZ4 }; +extern const char *ca_names[]; + +typedef struct { +void (*init)(struct compress_ctx *cc); +int (*compress)(struct compress_ctx *cc); +}compress_ops ; + +#define ALIGN_UP (value, size) ((value) + ((value) % (size) > 0 ? \ + (size) - (value) % (size) : 0)) + +enum filter_policy {FP_UNASSIGNED =0 ,FP_ALLOW ,FP_DENY }; +struct ext_tbl_op { +void (*add)(const char *); +void (*destroy)(void ); +bool (*filter)(const char *); +}; +struct f2fs_configuration {u_int32_t reserved_segments;u_int32_t new_reserved_segments;
include/f2fs_fs.h struct f2fs_configuration { · line 499
Quoted 8 lines
include/f2fs_fs.h +5
-0@@ -441,6 +492,24 @@ struct f2fs_configuration {/* cache parameters */ dev_cache_config_t cache_config; + +/* quick dynamic buffer */ +bool qbuf_initialized; +size_t qbufsize; +void *qbuf;
I don't think these qbuf* variables inside the f2fs_configuration are used anywhere - shouldn't they be removed?
Quoted 64 lines
include/f2fs_fs.h +19
-1+ +/* sldc: sload compression support */ +bool sldc_en; +bool sldc_use_allow_list;/* default false to use the deny list */ +struct compress_ctx sldc_cc; +u8 sldc_ca;/* compress algorithm: 0 = LZO, 1 = LZ4 */ +compress_ops *sldc_compr; +enum filter_policy sldc_policy; +/* max_cppc can used to specify minimum compression rate */ +unsigned int sldc_min_cbpc;/* min compressed pages per cluster */ +bool sldc_got_opt; +bool sldc_immutable; +struct ext_tbl_op *sldc_ef;/* extension filter */ };#ifdef CONFIG_64BIT @@ -1226,6 +1295,11 @@ extern void f2fs_release_sparse_resource(void);extern int f2fs_finalize_device (void );extern int f2fs_fsync_device (void ); +/* quick (shared) buffer */ +extern void qbuf_free (void ); +extern void *qbuf_alloc (size_t size); +extern void qbuf_init (void ); +extern void dcache_init (void );extern void dcache_release (void );@@ -1377,7 +1451,7 @@ int f2fs_reset_zone(int, void *);extern int f2fs_reset_zones (int );extern uint32_t f2fs_get_usable_segments (struct f2fs_super_block *sb); -#define SIZE_ALIGN (val, size) ((val) + (size) - 1) / (size) +#define SIZE_ALIGN (val, size) (((val) + (size) - 1) / (size))#define SEG_ALIGN (blks) SIZE_ALIGN(blks, c.blks_per_seg)#define ZONE_ALIGN (blks) SIZE_ALIGN(blks, c.blks_per_seg * \ c.segs_per_zone)lib/libf2fs_io.c +12
-0@@ -5,6 +5,9 @@* http://www.samsung.com/ * Copyright (c) 2019 Google Inc. * http://www.google.com/ +* Copyright (c) 2020 Google Inc. +* Robin Hsu <[email protected]> +* : add quick-buffer for sload compression support * * Dual licensed under the GPL or LGPL version 2 licenses. */ @@ -106,6 +109,36 @@ static long dcache_relocate_offset0[] = { };static int dcache_relocate_offset[16 ]; +/* quick (shared) buffer */ +static bool qbuf_initialized; +static void *qbuf; +static size_t qbufsize; +void qbuf_free (void ) +{ +ASSERT (qbuf_initialized); +if (qbuf !=NULL ) { +free (qbuf);
lib/libf2fs_io.c static long dcache_relocate_offset0[] = { · line 130
Quoted 10 lines
lib/libf2fs_io.c +10
-0+ qbuf =NULL ; + qbufsize =0 ; + } +} +void *qbuf_alloc (size_t size) +{ +ASSERT (qbuf_initialized); +if (size > qbufsize) { +qbuf_free (); + qbuf =malloc (size);
qbufsize needs to be updated to size here.
lib/libf2fs_io.c static long dcache_relocate_offset0[] = { · line 140
Quoted 10 lines
lib/libf2fs_io.c +10
-0+ } +return qbuf; +} +void qbuf_init (void ) +{ +if (qbuf_initialized) +return ; +atexit (qbuf_free); + qbuf_initialized = true; +}
Is there really a need for all this qbuf code? As far as I can tell, qbuf_alloc() is only ever called once during any invocation of sload, and it'd be better/simpler to replace that with a malloc()...
Quoted 13 lines
lib/libf2fs_io.c +1
-0+static void dcache_print_statistics (void ) {long i;Signature · 8 lines
2.29.2.576.ga3fc446d84-goog _______________________________________________ Linux-f2fs-devel mailing list [email protected] https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
_______________________________________________
Linux-f2fs-devel mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
Re: [f2fs-dev] [PATCH v3 2/3] f2fs-tools:sload.f2fs compression support
Quoted 967 lines · Satya Tangirala
Quoted 881 lines · Robin Hsu
From: Robin Hsu <[email protected]>
Add F2FS compression support for sload
* Support file extension filter, either default-accept or default-deny policy
* Support choice of compression algorithm, LZO (version 2) or LZ4 (default)
* Support custom log of cluster size
* Support minimum number of compressed blocks per cluster (default 1). A cluster will not be compressed if the number can not be met.
* suuport -r (read-only) optionSigned-off-by Robin Hsu <[email protected]>--- fsck/compress_wrapper.c | 102 ++++++++++++++++++++ fsck/compress_wrapper.h | 22 +++++ fsck/fsck.h | 15 +++ fsck/main.c | 141 +++++++++++++++++++++++++++- fsck/segment.c | 202 +++++++++++++++++++++++++++++++++++++--- fsck/sload.c | 67 +++++++++++++ include/f2fs_fs.h | 76 ++++++++++++++- lib/libf2fs_io.c | 33 +++++++ 8 files changed, 644 insertions(+), 14 deletions(-) create mode 100644 fsck/compress_wrapper.c create mode 100644 fsck/compress_wrapper.hfsck/compress_wrapper.c new file +102
-0@@ -0,0 +1,102 @@ +/** +* compress_wrapper.c +* +* Copyright (c) 2020 Google Inc. +* Robin Hsu <[email protected]> +* : initial created, for sload compression support +* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License version 2 as +* published by the Free Software Foundation. +*/ +#include "f2fs.h" /* for config.h for general environment (non-Android) */ + +#include "compress_wrapper.h" +#ifdef HAVE_LIBLZO2 +#include <lzo/lzo1x.h> /* for lzo1x_1_15_compress() */ +#endif +#ifdef HAVE_LIBLZ4 +#include <lz4.h> /* for LZ4_compress_fast_extState() */ +#endif + +/* +* macro/constants borrowed from kernel header (GPL-2.0): +* include/linux/lzo.h, and include/linux/lz4.h +*/ +#ifdef HAVE_LIBLZO2 +#define lzo1x_worst_compress (x) ((x) + (x) / 16 + 64 + 3 + 2) +#define LZO_WORK_SIZE ALIGN_UP(LZO1X_1_15_MEM_COMPRESS, 8) +#endif +#ifdef HAVE_LIBLZ4 +#define LZ4_MEMORY_USAGE 14 +#define LZ4_MAX_INPUT_SIZE 0x7E000000/* 2 113 929 216 bytes */ +#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(long long)) +#define LZ4_MEM_COMPRESS LZ4_STREAMSIZE +#define LZ4_ACCELERATION_DEFAULT 1 +#define LZ4_WORK_SIZE ALIGN_UP(LZ4_MEM_COMPRESS, 8) +#endif + +#ifdef HAVE_LIBLZO2 +static void lzo_compress_init (struct compress_ctx *cc) +{ +size_t size = cc->cluster_size *F2FS_BLKSIZE ; +size_t alloc = size +lzo1x_worst_compress (size) + +COMPRESS_HEADER_SIZE +LZO_WORK_SIZE ; +ASSERT ((cc->private =qbuf_alloc (alloc)) !=NULL ); + cc->rbuf = (char *) cc->private +LZO_WORK_SIZE ; + cc->cbuf = (struct compress_data *)((char *) cc->rbuf + size); +} + +static int lzo_compress (struct compress_ctx *cc) +{ +int ret =lzo1x_1_15_compress (cc->rbuf, cc->rlen, cc->cbuf->cdata, + (lzo_uintp)(&cc->clen), cc->private); + cc->cbuf->clen =cpu_to_le32 (cc->clen); +return ret; +} +#endif + +#ifdef HAVE_LIBLZ4 +static void lz4_compress_init (struct compress_ctx *cc) +{ +size_t size = cc->cluster_size *F2FS_BLKSIZE ; +size_t alloc = size +LZ4_COMPRESSBOUND (size) + +COMPRESS_HEADER_SIZE +LZ4_WORK_SIZE ; +ASSERT ((cc->private =qbuf_alloc (alloc)) !=NULL ); + cc->rbuf = (char *) cc->private +LZ4_WORK_SIZE ; + cc->cbuf = (struct compress_data *)((char *) cc->rbuf + size); +} + +static int lz4_compress (struct compress_ctx *cc) +{ + cc->clen =LZ4_compress_fast_extState (cc->private, cc->rbuf, + (char *)cc->cbuf->cdata, cc->rlen, + cc->rlen -F2FS_BLKSIZE * c.sldc_min_cbpc, +LZ4_ACCELERATION_DEFAULT ); + +if (!cc->clen) +return 1 ; + + cc->cbuf->clen =cpu_to_le32 (cc->clen); +return 0 ; +} +#endif + +const char *ca_names[] = { +"LZO" , +"LZ4" , +"" ,/* end of the name list */ +}; + +compress_ops compr_ops[] = { +#ifdef HAVE_LIBLZO2 + {lzo_compress_init, lzo_compress}, +#else + {NULL, NULL}, +#endif +#ifdef HAVE_LIBLZ4 + {lz4_compress_init, lz4_compress}, +#else + {NULL, NULL}, +#endif +};fsck/compress_wrapper.h new file +22
-0@@ -0,0 +1,22 @@ +/** +* compress_wrapper.h +* +* Copyright (c) 2020 Google Inc. +* Robin Hsu <[email protected]> +* : initial created, for sload compression support +* Copyright (c) 2020 Google Inc. +* Robin Hsu <[email protected]> +* : add sload compression support +* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License version 2 as +* published by the Free Software Foundation. +*/ + +#ifndef COMPRESS_WRAPPER_H +#define COMPRESS_WRAPPER_H + +#include "f2fs_fs.h" +extern compress_ops compr_ops[];/* [0]: LZO, [1]: LZ4, */ + +#endif /* COMPRESS_WRAPPER_H */ fsck/fsck.h +15
-0@@ -3,6 +3,9 @@* * Copyright (c) 2013 Samsung Electronics Co., Ltd. * http://www.samsung.com/ +* Copyright (c) 2020 Google Inc. +* Robin Hsu <[email protected]> +* : add sload compression support * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -266,6 +269,9 @@ int f2fs_resize(struct f2fs_sb_info *);/* sload.c */ int f2fs_sload (struct f2fs_sb_info *); +void sldc_erase_bufs (struct compress_ctx *cc); +void sload_countblk (void ); +extern struct ext_tbl_op ext_filter;/* segment.c */ int reserve_new_block(struct f2fs_sb_info *, block_t *,@@ -282,7 +288,16 @@ block_t new_node_block(struct f2fs_sb_info *,struct quota_file ;u64 f2fs_quota_size (struct quota_file *);u64 f2fs_read (struct f2fs_sb_info *,nid_t ,u8 *,u64 ,pgoff_t ); +enum wr_addr_type { +WR_NORMAL =1 , +WR_COMPRESS_DATA =2 , +WR_NULL_ADDR =NULL_ADDR ,/* 0 */ +WR_NEW_ADDR =NEW_ADDR ,/* -1U */ +WR_COMPRESS_ADDR =COMPRESS_ADDR ,/* -2U */ +};u64 f2fs_write (struct f2fs_sb_info *,nid_t ,u8 *,u64 ,pgoff_t ); +u64 f2fs_write_compress_data (struct f2fs_sb_info *,nid_t ,u8 *,u64 ,pgoff_t ); +u64 f2fs_write_addrtag (struct f2fs_sb_info *,nid_t ,pgoff_t ,unsigned int );void f2fs_filesize_update (struct f2fs_sb_info *,nid_t ,u64 ); int get_dnode_of_data(struct f2fs_sb_info *, struct dnode_of_data *,fsck/main.c +140
-1@@ -13,6 +13,9 @@* Copyright (c) 2019 Google Inc. * Robin Hsu <[email protected]> * : add cache layer +* Copyright (c) 2020 Google Inc. +* Robin Hsu <[email protected]> +* : add sload compression support * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -25,6 +28,7 @@#include <getopt.h> #include <stdbool.h> #include "quotaio.h" +#include "compress_wrapper.h" struct f2fs_fsck gfsck;@@ -134,6 +138,17 @@ void sload_usage()MSG (0 ," -S sparse_mode\n" );MSG (0 ," -t mount point [prefix of target fs path, default:/]\n" );MSG (0 ," -T timestamp\n" ); +MSG (0 ," -c enable compression (default allow policy)\n" ); +MSG (0 ," ------------ Compression sub-options -----------------\n" ); +MSG (0 ," -L <log-of-blocks-per-cluster>, default 2\n" ); +MSG (0 ," -a <algorithm> compression algorithm, default LZ4\n" ); +MSG (0 ," -x <ext> compress files except for these extensions.\n" ); +MSG (0 ," -i <ext> compress files with these extensions only.\n" ); +MSG (0 ," * -i or -x: use it many times for multiple extensions.\n" ); +MSG (0 ," * -i and -x cannot be used together..\n" ); +MSG (0 ," -m <num> min compressed blocks per cluster\n" ); +MSG (0 ," -r readonly (IMMUTABLE) for compressed files\n" ); +MSG (0 ," ------------------------------------------------------\n" );MSG (0 ," -d debug level [default:0]\n" );MSG (0 ," -V print the version number and exit\n" );exit (1 );@@ -534,7 +549,7 @@ void f2fs_parse_options(int argc, char *argv[])#endif }else if (!strcmp ("sload.f2fs" , prog)) {#ifdef WITH_SLOAD -const char *option_string ="C:d:f:p:s:St:T:V" ; +const char *option_string ="cL:a:i:x:m:rC:d:f:p:s:St:T:V" ;#ifdef HAVE_LIBSELINUX int max_nr_opt = (int )sizeof (c.seopt_file) /sizeof (c.seopt_file[0 ]);@@ -543,8 +558,82 @@ void f2fs_parse_options(int argc, char *argv[])char *p; c.func =SLOAD ; + c.sldc_cc.log_cluster_size =2 ; + c.sldc_ca =CA_LZ4 ; + c.sldc_min_cbpc =1 ; + c.sldc_ef = &ext_filter;while ((option =getopt (argc, argv, option_string)) !=EOF ) { +unsigned int i; +int val; +switch (option) { +case 'c' :/* compression support */ + c.sldc_en = true; +break ; +case 'L' :/* compression: log of blocks-per-cluster */ + c.sldc_got_opt = true; + val =atoi (optarg); +if (val <MIN_COMPRESS_LOG_SIZE || + val >MAX_COMPRESS_LOG_SIZE ) { +MSG (0 ,"\tError: log of blocks per" +" cluster must be in the range" +" of %d .. %d.\n" , +MIN_COMPRESS_LOG_SIZE , +MAX_COMPRESS_LOG_SIZE ); +error_out (prog); + } + c.sldc_cc.log_cluster_size = val; +break ; +case 'a' :/* compression: choose algorithm */ + c.sldc_got_opt = true; + c.sldc_ca = (u8)-1 ; +for (i =0 ; ca_names[i][0 ] !=0 ; i++) { +if (!strcmp (ca_names[i], optarg)) { + c.sldc_ca = i; +break ; + } + } +if (c.sldc_ca == (u8)-1 ) { +MSG (0 ,"\tError: Unknown compression" +" algorithm %s\n" , optarg); +error_out (prog); + } +break ; +case 'i' :/* compress only these extensions */ + c.sldc_got_opt = true; +if (c.sldc_policy ==FP_ALLOW ) { +MSG (0 ,"\tError: could not mix option" +" -i and -x\n" ); +error_out (prog); + } + c.sldc_policy =FP_DENY ; + c.sldc_ef->add (optarg); +break ; +case 'x' :/* compress except for these extensions */ + c.sldc_got_opt = true; +if (c.sldc_policy ==FP_DENY ) { +MSG (0 ,"\tError: could not mix option" +" -i and -x\n" ); +error_out (prog); + } + c.sldc_policy =FP_ALLOW ; + c.sldc_ef->add (optarg); +break ; +case 'm' :/* minimum compressed blocks per cluster */ + c.sldc_got_opt = true; + val =atoi (optarg); +if (val <=0 ) { +MSG (0 ,"\tError: minimum compressed" +" blocks per cluster must be" +" positive.\n" ); +error_out (prog); + } + c.sldc_min_cbpc = val; +break ; +case 'r' :/* compress file to set IMMUTABLE */ + c.sldc_got_opt = true; + c.sldc_immutable = true; +break ;case 'C' : c.fs_config_file =absolute_path (optarg);break ;@@ -602,6 +691,27 @@ void f2fs_parse_options(int argc, char *argv[])if (err !=NOERROR ) break; } +if (c.sldc_got_opt && !c.sldc_en) { +MSG (0 ,"\tError: compression sub-options are used" +" without the compression enable (-c) option\n" + ); +error_out (prog); + } +if (err ==NOERROR && c.sldc_en) { + c.sldc_cc.cluster_size =1 + << c.sldc_cc.log_cluster_size; +if (c.sldc_policy ==FP_UNASSIGNED ) + c.sldc_policy =FP_ALLOW ; +if (c.sldc_min_cbpc >= c.sldc_cc.cluster_size) { +MSG (0 ,"\tError: minimum reduced blocks by" +" compression per cluster must be at" +" most one less than blocks per" +" cluster, i.e. %d\n" , + c.sldc_cc.cluster_size -1 ); +error_out (prog); + } +qbuf_init (); + }#endif /* WITH_SLOAD */ }@@ -812,6 +922,27 @@ static int do_resize(struct f2fs_sb_info *sbi)#endif #ifdef WITH_SLOAD +int init_compr (struct f2fs_sb_info *sbi) +{ +if (!(sbi->raw_super->feature + &cpu_to_le32 (F2FS_FEATURE_COMPRESSION ))) { +MSG (0 ,"Error: Compression (-c) was requested " +"but the file system is not created " +"with such feature.\n" ); +return -1 ; + } +if (compr_ops[c.sldc_ca].init ==NULL ) { +MSG (0 ,"Error: The selected compression algorithm is not" +" supported\n" ); +return -1 ; + } + c.sldc_compr = compr_ops + c.sldc_ca; + c.sldc_compr->init (&c.sldc_cc); +sldc_erase_bufs (&c.sldc_cc); + c.sldc_cc.rlen = c.sldc_cc.cluster_size *F2FS_BLKSIZE ; +return 0 ; +} +static int do_sload (struct f2fs_sb_info *sbi) {if (!c.from_dir) {@@ -821,6 +952,11 @@ static int do_sload(struct f2fs_sb_info *sbi)if (!c.mount_point) c.mount_point ="/" ; +if (c.sldc_en) { +if (init_compr (sbi)) +return -1 ; + } + return f2fs_sload(sbi); }#endif @@ -971,6 +1107,9 @@ retry:return ret2; } +if (c.func ==SLOAD ) + c.sldc_ef->destroy (); +printf ("\nDone: %lf secs\n" , (get_boottime_ns () - start) /1000000000.0 );return ret;fsck/segment.c +190
-12@@ -8,6 +8,9 @@* Hou Pengyang <[email protected]> * Liu Shuoran <[email protected]> * Jaegeuk Kim <[email protected]> +* Copyright (c) 2020 Google Inc. +* Robin Hsu <[email protected]> +* : add sload compression support * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -16,6 +19,7 @@#include "fsck.h" #include "node.h" #include "quotaio.h" +#include "compress_wrapper.h" int reserve_new_block (struct f2fs_sb_info *sbi,block_t *to,struct f2fs_summary *sum,int type,bool is_inode)@@ -228,8 +232,14 @@ u64 f2fs_read(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,return read_count; } -u64 f2fs_write (struct f2fs_sb_info *sbi,nid_t ino,u8 *buffer, -u64 count,pgoff_t offset) +/* +* Do not call this function directly. Instead, call one of the following: +* u64 f2fs_write(); +* u64 f2fs_write_compress_data(); +* u64 f2fs_write_addrtag(); +*/ +static u64 f2fs_write_ex (struct f2fs_sb_info *sbi,nid_t ino,u8 *buffer, +u64 count,pgoff_t offset,enum wr_addr_type addr_type) {struct dnode_of_data dn;struct node_info ni;@@ -243,6 +253,19 @@ u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,void * index_node =NULL ;int idirty =0 ;int err; +bool has_data = (addr_type ==WR_NORMAL + || addr_type ==WR_COMPRESS_DATA ); + +if (count ==0 ) +return 0 ; + +/* +* Enforce calling from f2fs_write(), f2fs_write_compress_data(), +* and f2fs_write_addrtag(). Beside, check if is properly called. +*/ +ASSERT ((!has_data && buffer ==NULL ) || (has_data && buffer !=NULL )); +if (addr_type !=WR_NORMAL ) +ASSERT (offset %F2FS_BLKSIZE ==0 );/* block boundary only */ /* Memory allocation for block buffer and inode. */ blk_buffer =calloc (BLOCK_SZ ,2 );@@ -265,15 +288,26 @@ u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,if (err)break ; idirty |= dn.idirty; -if (index_node) -free (index_node); +free (index_node); index_node = (dn.node_blk == dn.inode_blk) ? -NULL : dn.node_blk; +NULL : dn.node_blk; remained_blkentries =ADDRS_PER_PAGE (sbi, - dn.node_blk, dn.inode_blk); + dn.node_blk, dn.inode_blk) - + dn.ofs_in_node; }ASSERT (remained_blkentries >0 ); +if (!has_data) { + dn.data_blkaddr = addr_type; +set_data_blkaddr (&dn); + idirty |= dn.idirty; +if (dn.ndirty) +ASSERT (dev_write_block (dn.node_blk, + dn.node_blkaddr) >=0 ); + written_count =0 ; +break ; + } + blkaddr =datablock_addr (dn.node_blk, dn.ofs_in_node);if (blkaddr ==NULL_ADDR || blkaddr ==NEW_ADDR ) { err = new_data_block(sbi, blk_buffer,@@ -281,6 +315,7 @@ u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,if (err)break ; blkaddr = dn.data_blkaddr; + idirty |= dn.idirty; } off_in_blk = offset %BLOCK_SZ ;@@ -305,9 +340,10 @@ u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer, dn.ofs_in_node++;if ((--remained_blkentries ==0 || count ==0 ) && (dn.ndirty)) -ASSERT (dev_write_block (dn.node_blk, dn.node_blkaddr) >=0 ); +ASSERT (dev_write_block (dn.node_blk, dn.node_blkaddr) + >=0 ); } -if (offset >le64_to_cpu (inode->i.i_size)) { +if (addr_type ==WR_NORMAL && offset >le64_to_cpu (inode->i.i_size)) { inode->i.i_size =cpu_to_le64 (offset); idirty =1 ; }@@ -315,13 +351,33 @@ u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,ASSERT (inode == dn.inode_blk);ASSERT (write_inode (inode, ni.blk_addr) >=0 ); } -if (index_node) -free (index_node); + +free (index_node);free (blk_buffer);return written_count; } +u64 f2fs_write (struct f2fs_sb_info *sbi,nid_t ino,u8 *buffer, +u64 count,pgoff_t offset) +{ +return f2fs_write_ex (sbi, ino, buffer, count, offset,WR_NORMAL ); +} + +u64 f2fs_write_compress_data (struct f2fs_sb_info *sbi,nid_t ino,u8 *buffer, +u64 count,pgoff_t offset) +{ +return f2fs_write_ex (sbi, ino, buffer, count, offset,WR_COMPRESS_DATA ); +} + +u64 f2fs_write_addrtag (struct f2fs_sb_info *sbi,nid_t ino,pgoff_t offset, +unsigned int addrtag) +{ +ASSERT (addrtag ==COMPRESS_ADDR || addrtag ==NEW_ADDR + || addrtag ==NULL_ADDR ); +return f2fs_write_ex (sbi, ino,NULL ,F2FS_BLKSIZE , offset, addrtag); +} +/* This function updates only inode->i.i_size */ void f2fs_filesize_update (struct f2fs_sb_info *sbi,nid_t ino,u64 filesize) {@@ -342,11 +398,59 @@ void f2fs_filesize_update(struct f2fs_sb_info *sbi, nid_t ino, u64 filesize)free (inode); } +#define MAX_BULKR_RETRY 5 +int bulkread (int fd,void *rbuf,size_t rsize,bool *eof) +{ +int n =0 ; +int retry =MAX_BULKR_RETRY ; +int cur; + +if (!rsize) +return 0 ; + +if (eof !=NULL ) + *eof = false; +while (rsize && (cur =read (fd, rbuf, rsize)) !=0 ) { +if (cur ==-1 ) { +if (errno ==EINTR && retry--) +continue ; +return -1 ; + } + retry =MAX_BULKR_RETRY ; + + rsize -= cur; + n += cur; + } +if (eof !=NULL ) + *eof = (cur ==0 ); +return n; +} + +u64 f2fs_fix_mutable (struct f2fs_sb_info *sbi,nid_t ino,pgoff_t offset, +unsigned int compressed) +{ +unsigned int i; +u64 wlen; + +if (c.sldc_immutable) +return 0 ; + +for (i =0 ; i < compressed -1 ; i++) { + wlen =f2fs_write_addrtag (sbi, ino, + offset + (i <<F2FS_BLKSIZE_BITS ),NEW_ADDR ); +if (wlen) +return wlen; + } +return 0 ; +} +int f2fs_build_file (struct f2fs_sb_info *sbi,struct dentry *de) {int fd, n;pgoff_t off =0 ;u8 buffer[BLOCK_SZ ]; +struct node_info ni; +struct f2fs_node *node_blk;if (de->ino ==0 )return -1 ;@@ -359,8 +463,6 @@ int f2fs_build_file(struct f2fs_sb_info *sbi, struct dentry *de)/* inline_data support */ if (de->size <=DEF_MAX_INLINE_DATA ) { -struct node_info ni; -struct f2fs_node *node_blk;int ret;get_node_info (sbi, de->ino, &ni);@@ -385,6 +487,82 @@ int f2fs_build_file(struct f2fs_sb_info *sbi, struct dentry *de) node_blk->i.i_size =cpu_to_le64 (de->size);ASSERT (write_inode (node_blk, ni.blk_addr) >=0 );free (node_blk); +#ifdef WITH_SLOAD + }else if (c.func ==SLOAD && c.sldc_en && + c.sldc_ef->filter (de->full_path)) { +bool eof = false; +u8 *rbuf = c.sldc_cc.rbuf; +unsigned int cblocks =0 ; + + node_blk =calloc (BLOCK_SZ ,1 ); +ASSERT (node_blk); + +/* read inode */ +get_node_info (sbi, de->ino, &ni); +ASSERT (dev_read_block (node_blk, ni.blk_addr) >=0 ); +/* update inode meta */ + node_blk->i.i_compress_algrithm = c.sldc_ca; + node_blk->i.i_log_cluster_size = + c.sldc_cc.log_cluster_size; + node_blk->i.i_flags =cpu_to_le32 ( +F2FS_COMPR_FL | + (c.sldc_immutable ?FS_IMMUTABLE_FL :0 )); +ASSERT (write_inode (node_blk, ni.blk_addr) >=0 ); + +while (!eof && (n =bulkread (fd, rbuf, c.sldc_cc.rlen, + &eof)) >0 ) { +int ret = c.sldc_compr->compress (&c.sldc_cc); +u64 wlen; +u32 csize =ALIGN_UP (c.sldc_cc.clen + +COMPRESS_HEADER_SIZE ,BLOCK_SZ ); +unsigned int cur_cblk; + +if (ret || n < (int )(csize +BLOCK_SZ * + c.sldc_min_cbpc)) { + wlen =f2fs_write (sbi, de->ino, rbuf, n, off); +ASSERT ((int )wlen == n); + }else { + wlen =f2fs_write_addrtag (sbi, de->ino, off, +WR_COMPRESS_ADDR ); +ASSERT (!wlen); + wlen =f2fs_write_compress_data (sbi, de->ino, + (u8 *)c.sldc_cc.cbuf, + csize, off +BLOCK_SZ ); +ASSERT (wlen == csize); +sldc_erase_bufs (&c.sldc_cc); + cur_cblk = (c.sldc_cc.rlen - csize) /BLOCK_SZ ; + cblocks += cur_cblk; + wlen =f2fs_fix_mutable (sbi, de->ino, + off +BLOCK_SZ + csize, + cur_cblk); +ASSERT (!wlen); + } + off += n; + } +if (n ==-1 ) { +fprintf (stderr,"Load file '%s' failed: " , + de->full_path); +perror (NULL ); + } +/* read inode */ +get_node_info (sbi, de->ino, &ni); +ASSERT (dev_read_block (node_blk, ni.blk_addr) >=0 ); +/* update inode meta */ + node_blk->i.i_size =cpu_to_le64 (off); +if (!c.sldc_immutable) + node_blk->i.i_compr_blocks =cpu_to_le64 (cblocks); +ASSERT (write_inode (node_blk, ni.blk_addr) >=0 ); +free (node_blk); + +if (!c.sldc_immutable) { + sbi->total_valid_block_count += cblocks; +if (sbi->total_valid_block_count >= + sbi->user_block_count) { +ERR_MSG ("Not enough space\n" ); +ASSERT (0 ); + } + } +#endif }else {while ((n =read (fd, buffer,BLOCK_SZ )) >0 ) {f2fs_write (sbi, de->ino, buffer, n, off);fsck/sload.c +67
-0@@ -6,6 +6,9 @@* Hou Pengyang <[email protected]> * Liu Shuoran <[email protected]> * Jaegeuk Kim <[email protected]> +* Copyright (c) 2020 Google Inc. +* Robin Hsu <[email protected]> +* : add sload compression support * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -317,6 +320,70 @@ skip:return 0 ; } +typedef struct _ext_tbl { +const char *ext; +struct _ext_tbl *next;/* linked list */ +}ext_tbl_t ; +static ext_tbl_t *ext_tbl; + +static bool ext_found (const char *ext) +{ +ext_tbl_t *p = ext_tbl; + +while (p !=NULL &&strcmp (ext, p->ext)) + p = p->next; +return (p !=NULL ); +} + +static const char *get_ext (const char *path) +{ +char *p =strrchr (path,'.' ); +return p ==NULL ? path +strlen (path) : p +1 ; +} + +static bool ext_do_filter (const char *path) +{ +return (ext_found (get_ext (path)) == true) ^ (c.sldc_policy ==FP_ALLOW ); +} + +static void ext_filter_add (const char *ext) +{ +ext_tbl_t *node; + +ASSERT (ext !=NULL ); +if (ext_found (ext)) +return ;/* ext was already registered */ + node =malloc (sizeof (ext_tbl_t)); +ASSERT (node !=NULL ); + node->ext = ext; + node->next = ext_tbl; + ext_tbl = node; +} + +static void ext_filter_destroy (void ) +{ +ext_tbl_t *p; + +while (ext_tbl !=NULL ) { + p = ext_tbl; + ext_tbl = p->next; +free (p); + } +} + +struct ext_tbl_op ext_filter = { + .add = ext_filter_add, + .destroy = ext_filter_destroy, + .filter = ext_do_filter, +}; + +void sldc_erase_bufs (struct compress_ctx *cc) +{ +memset (cc->rbuf,0 , cc->cluster_size *F2FS_BLKSIZE ); +memset (cc->cbuf->cdata,0 , cc->cluster_size *F2FS_BLKSIZE + -F2FS_BLKSIZE ); +} +int f2fs_sload (struct f2fs_sb_info *sbi) {int ret =0 ;include/f2fs_fs.h +51
-0@@ -5,6 +5,9 @@* http://www.samsung.com/ * Copyright (c) 2019 Google Inc. * http://www.google.com/ +* Copyright (c) 2020 Google Inc. +* Robin Hsu <[email protected]> +* : add sload compression support * * Dual licensed under the GPL or LGPL version 2 licenses. * @@ -68,6 +71,10 @@ typedef uint16_t u_int16_t;typedef uint8_t u_int8_t ;#endif +/* codes from kernel's f2fs.h, GPL-v2.0 */ +#define MIN_COMPRESS_LOG_SIZE 2 +#define MAX_COMPRESS_LOG_SIZE 8 +typedef u_int64_t u64 ;typedef u_int32_t u32 ;typedef u_int16_t u16 ;@@ -93,6 +100,31 @@ typedef u32 __be32;typedef u64 __be64 ;#endif +/* +* code borrowed from kernel f2fs dirver: f2fs.h, GPL-2.0 +* : definitions of COMPRESS_DATA_RESERVED_SIZE, +* struct compress_data, COMPRESS_HEADER_SIZE, +* and struct compress_ctx +*/ +#define COMPRESS_DATA_RESERVED_SIZE 4 +struct compress_data { +__le32 clen;/* compressed data size */ +__le32 chksum;/* checksum of compressed data */ +__le32 reserved[COMPRESS_DATA_RESERVED_SIZE ];/* reserved */ +u8 cdata[];/* compressed data */ +}; +#define COMPRESS_HEADER_SIZE (sizeof(struct compress_data)) +/* compress context */ +struct compress_ctx { +unsigned int cluster_size;/* page count in cluster */ +unsigned int log_cluster_size;/* log of cluster size */ +void *rbuf;/* compression input buffer */ +struct compress_data *cbuf;/* comprsssion output header + data */ +size_t rlen;/* valid data length in rbuf */ +size_t clen;/* valid data length in cbuf */ +void *private;/* work buf for compress algorithm */ +}; +#if HAVE_BYTESWAP_H #include <byteswap.h> #else @@ -345,6 +377,25 @@ typedef struct {bool dbg_en; } dev_cache_config_t; +/* f2fs_configration: sldc_ca, the sload compress algorithm */ +enum {CA_LZO ,CA_LZ4 }; +extern const char *ca_names[]; + +typedef struct { +void (*init)(struct compress_ctx *cc); +int (*compress)(struct compress_ctx *cc); +}compress_ops ; + +#define ALIGN_UP (value, size) ((value) + ((value) % (size) > 0 ? \ + (size) - (value) % (size) : 0)) + +enum filter_policy {FP_UNASSIGNED =0 ,FP_ALLOW ,FP_DENY }; +struct ext_tbl_op { +void (*add)(const char *); +void (*destroy)(void ); +bool (*filter)(const char *); +}; +struct f2fs_configuration {u_int32_t reserved_segments;u_int32_t new_reserved_segments;include/f2fs_fs.h struct f2fs_configuration { · line 499
Quoted 8 lines
include/f2fs_fs.h +5
-0@@ -441,6 +492,24 @@ struct f2fs_configuration {/* cache parameters */ dev_cache_config_t cache_config; + +/* quick dynamic buffer */ +bool qbuf_initialized; +size_t qbufsize; +void *qbuf;I don't think these qbuf* variables inside the
f2fs_configurationare used anywhere - shouldn't they be removed?Quoted 64 lines
include/f2fs_fs.h +19
-1+ +/* sldc: sload compression support */ +bool sldc_en; +bool sldc_use_allow_list;/* default false to use the deny list */ +struct compress_ctx sldc_cc; +u8 sldc_ca;/* compress algorithm: 0 = LZO, 1 = LZ4 */ +compress_ops *sldc_compr; +enum filter_policy sldc_policy; +/* max_cppc can used to specify minimum compression rate */ +unsigned int sldc_min_cbpc;/* min compressed pages per cluster */ +bool sldc_got_opt; +bool sldc_immutable; +struct ext_tbl_op *sldc_ef;/* extension filter */ };#ifdef CONFIG_64BIT @@ -1226,6 +1295,11 @@ extern void f2fs_release_sparse_resource(void);extern int f2fs_finalize_device (void );extern int f2fs_fsync_device (void ); +/* quick (shared) buffer */ +extern void qbuf_free (void ); +extern void *qbuf_alloc (size_t size); +extern void qbuf_init (void ); +extern void dcache_init (void );extern void dcache_release (void );@@ -1377,7 +1451,7 @@ int f2fs_reset_zone(int, void *);extern int f2fs_reset_zones (int );extern uint32_t f2fs_get_usable_segments (struct f2fs_super_block *sb); -#define SIZE_ALIGN (val, size) ((val) + (size) - 1) / (size) +#define SIZE_ALIGN (val, size) (((val) + (size) - 1) / (size))#define SEG_ALIGN (blks) SIZE_ALIGN(blks, c.blks_per_seg)#define ZONE_ALIGN (blks) SIZE_ALIGN(blks, c.blks_per_seg * \ c.segs_per_zone)lib/libf2fs_io.c +12
-0@@ -5,6 +5,9 @@* http://www.samsung.com/ * Copyright (c) 2019 Google Inc. * http://www.google.com/ +* Copyright (c) 2020 Google Inc. +* Robin Hsu <[email protected]> +* : add quick-buffer for sload compression support * * Dual licensed under the GPL or LGPL version 2 licenses. */ @@ -106,6 +109,36 @@ static long dcache_relocate_offset0[] = { };static int dcache_relocate_offset[16 ]; +/* quick (shared) buffer */ +static bool qbuf_initialized; +static void *qbuf; +static size_t qbufsize; +void qbuf_free (void ) +{ +ASSERT (qbuf_initialized); +if (qbuf !=NULL ) { +free (qbuf);lib/libf2fs_io.c static long dcache_relocate_offset0[] = { · line 130
Quoted 10 lines
lib/libf2fs_io.c +10
-0+ qbuf =NULL ; + qbufsize =0 ; + } +} +void *qbuf_alloc (size_t size) +{ +ASSERT (qbuf_initialized); +if (size > qbufsize) { +qbuf_free (); + qbuf =malloc (size);qbufsize needs to be updated to size here.
Quoted 13 lines
lib/libf2fs_io.c static long dcache_relocate_offset0[] = { · line 140
Quoted 10 lines
lib/libf2fs_io.c +10
-0+ } +return qbuf; +} +void qbuf_init (void ) +{ +if (qbuf_initialized) +return ; +atexit (qbuf_free); + qbuf_initialized = true; +}Is there really a need for all this qbuf code? As far as I can tell,
qbuf_alloc()is only ever called once during any invocation of sload, and it'd be better/simpler to replace that with amalloc()...
Yeah, it seems we don't need qbuf. Let me apply this change. Please check -dev branch.
Thanks,
--- fsck/compress.c | 6 ++++-- fsck/main.c | 1 - include/f2fs_fs.h | 10 ---------- lib/libf2fs_io.c | 30 ------------------------------ 4 files changed, 4 insertions(+), 43 deletions(-)
fsck/compress.c +4 -2
@@ -55,7 +55,8 @@ static void lzo_compress_init(struct compress_ctx *cc)size_t size = cc->cluster_size *F2FS_BLKSIZE ;size_t alloc = size +lzo1x_worst_compress (size) +COMPRESS_HEADER_SIZE +LZO_WORK_SIZE ; -ASSERT ((cc->private =qbuf_alloc (alloc)) !=NULL ); + cc->private =malloc (alloc); +ASSERT (cc->private); cc->rbuf = (char *) cc->private +LZO_WORK_SIZE ; cc->cbuf = (struct compress_data *)((char *) cc->rbuf + size); }
@@ -75,7 +76,8 @@ static void lz4_compress_init(struct compress_ctx *cc)size_t size = cc->cluster_size *F2FS_BLKSIZE ;size_t alloc = size +LZ4_COMPRESSBOUND (size) +COMPRESS_HEADER_SIZE +LZ4_WORK_SIZE ; -ASSERT ((cc->private =qbuf_alloc (alloc)) !=NULL ); + cc->private =malloc (alloc); +ASSERT (cc->private); cc->rbuf = (char *) cc->private +LZ4_WORK_SIZE ; cc->cbuf = (struct compress_data *)((char *) cc->rbuf + size); }
fsck/main.c +0 -1
@@ -712,7 +712,6 @@ void f2fs_parse_options(int argc, char *argv[]) c.compress.cc.cluster_size - 1); error_out(prog); } -qbuf_init (); }#endif /* WITH_SLOAD */ }
include/f2fs_fs.h +0 -10
@@ -515,11 +515,6 @@ struct f2fs_configuration {/* cache parameters */ dev_cache_config_t cache_config; -/* quick dynamic buffer */ -bool qbuf_initialized; -size_t qbufsize; -void *qbuf; -/* compression support for sload.f2fs */ compress_config_t compress; };
@@ -1307,11 +1302,6 @@ extern void f2fs_release_sparse_resource(void);extern int f2fs_finalize_device (void );extern int f2fs_fsync_device (void ); -/* quick (shared) buffer */ -extern void qbuf_free (void ); -extern void *qbuf_alloc (size_t size); -extern void qbuf_init (void ); -extern void dcache_init (void );extern void dcache_release (void );
lib/libf2fs_io.c +0 -30
@@ -109,36 +109,6 @@ static long dcache_relocate_offset0[] = { };static int dcache_relocate_offset[16 ]; -/* quick (shared) buffer */ -static bool qbuf_initialized; -static void *qbuf; -static size_t qbufsize; -void qbuf_free (void ) -{ -ASSERT (qbuf_initialized); -if (qbuf !=NULL ) { -free (qbuf); - qbuf =NULL ; - qbufsize =0 ; - } -} -void *qbuf_alloc (size_t size) -{ -ASSERT (qbuf_initialized); -if (size > qbufsize) { -qbuf_free (); - qbuf =malloc (size); - } -return qbuf; -} -void qbuf_init (void ) -{ -if (qbuf_initialized) -return ; -atexit (qbuf_free); - qbuf_initialized = true; -} -static void dcache_print_statistics (void ) {long i;
Signature · 8 lines
2.29.2.729.g45daf8777d-goog _______________________________________________ Linux-f2fs-devel mailing list [email protected] https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
PATCH v3 3/3 f2fs-tools:sload.f2fs compress: Fixed automake
In reply to Robin HsuFrom: Robin Hsu <[email protected]>
Fixed automake for sload.f2fs compression support
./configure automatcally compile in liblzo2 (for sload to support -a LZO) and liblz4 (for sload to support -a LZ4), whhen the libraries present.
--- configure.ac | 12 ++++++++++++ fsck/Makefile.am | 9 ++++++--- 2 files changed, 18 insertions(+), 3 deletions(-)
configure.ac +12 -0
@@ -52,6 +52,18 @@ AC_PATH_PROG([LDCONFIG], [ldconfig], [$PATH:/sbin]) # Checks for libraries. +AC_CHECK_LIB([lzo2], [main], + [AC_SUBST([liblzo2_LIBS], ["-llzo2"]) + AC_DEFINE([HAVE_LIBLZO2], [1], + [Define if you have liblzo2]) + ], [], []) + +AC_CHECK_LIB([lz4], [main], + [AC_SUBST([liblz4_LIBS], ["-llz4"]) + AC_DEFINE([HAVE_LIBLZ4], [1], + [Define if you have liblz4]) + ], [], []) + PKG_CHECK_MODULES([libuuid], [uuid]) AS_IF([test "x$with_selinux" != "xno"],
fsck/Makefile.am +6 -3
@@ -3,12 +3,15 @@ AM_CPPFLAGS = ${libuuid_CFLAGS} -I$(top_srcdir)/include AM_CFLAGS = -Wall sbin_PROGRAMS = fsck.f2fs -noinst_HEADERS = common.h dict.h dqblk_v2.h f2fs.h fsck.h node.h quotaio.h quotaio_tree.h quotaio_v2.h xattr.h +noinst_HEADERS = common.h dict.h dqblk_v2.h f2fs.h fsck.h node.h quotaio.h \ + quotaio_tree.h quotaio_v2.h xattr.h compress_wrapper.h include_HEADERS = $(top_srcdir)/include/quota.h fsck_f2fs_SOURCES = main.c fsck.c dump.c mount.c defrag.c resize.c \ - node.c segment.c dir.c sload.c xattr.c \ + node.c segment.c dir.c sload.c xattr.c compress_wrapper.c \ dict.c mkquota.c quotaio.c quotaio_tree.c quotaio_v2.c -fsck_f2fs_LDADD = ${libselinux_LIBS} ${libuuid_LIBS} $(top_builddir)/lib/libf2fs.la +fsck_f2fs_LDADD = ${libselinux_LIBS} ${libuuid_LIBS} \ + ${liblzo2_LIBS} ${liblz4_LIBS} \ + $(top_builddir)/lib/libf2fs.la install-data-hook: ln -sf fsck.f2fs $(DESTDIR)/$(sbindir)/dump.f2fs
Signature · 8 lines
2.29.2.576.ga3fc446d84-goog _______________________________________________ Linux-f2fs-devel mailing list [email protected] https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel