f2fs-tools: sload compression support
v3 cover f2fs-tools: sload compression support
From: Robin Hsu <[email protected]>
* 3 patch set:
#1: added some #ifdef for easier support
#2: main code change
#3: automake changes
v2 fix (from v1): fixed a bug and a more elegant error handling flow. v3 fix (from v2): ./configure (automake) automatically determine to compile in lzo and/or lz4 compression support depending on the presence of liblzo2-dev and/or liblz4-dev
Robin Hsu (3):
f2fs-tools: Added #ifdef WITH_func
f2fs-tools:sload.f2fs compression support
f2fs-tools:sload.f2fs compress: Fixed automake
configure.ac | 12 +++ fsck/Makefile.am | 9 +- fsck/compress_wrapper.c | 102 ++++++++++++++++++++ fsck/compress_wrapper.h | 22 +++++ fsck/fsck.h | 15 +++ fsck/main.c | 157 ++++++++++++++++++++++++++++++- fsck/segment.c | 202 +++++++++++++++++++++++++++++++++++++--- fsck/sload.c | 67 +++++++++++++ include/f2fs_fs.h | 76 ++++++++++++++- lib/libf2fs_io.c | 33 +++++++ 10 files changed, 678 insertions(+), 17 deletions(-) create mode 100644 fsck/compress_wrapper.c create mode 100644 fsck/compress_wrapper.h
Signature · 8 lines
2.29.2.576.ga3fc446d84-goog _______________________________________________ Linux-f2fs-devel mailing list [email protected] https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
PATCH v3 1/3 f2fs-tools: Added #ifdef WITH_func
From: Robin Hsu <[email protected]>
Add proprocessor defines (options) 'WITH_func',
where func = DUMP, DEFRAG, RESIZE, or SLOAD
--- fsck/main.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+)
fsck/main.c +16 -0
@@ -345,6 +345,7 @@ void f2fs_parse_options(int argc, char *argv[])break ; } }else if (!strcmp ("dump.f2fs" , prog)) { +#ifdef WITH_DUMP const char *option_string ="d:i:n:s:Sa:b:V" ;static struct dump_option dump_opt = { .nid = 0, /* default root ino */
@@ -426,7 +427,9 @@ void f2fs_parse_options(int argc, char *argv[]) } c.private = &dump_opt; +#endif }else if (!strcmp ("defrag.f2fs" , prog)) { +#ifdef WITH_DEFRAG const char *option_string ="d:s:Sl:t:iV" ; c.func =DEFRAG ;
@@ -484,7 +487,9 @@ void f2fs_parse_options(int argc, char *argv[])if (err !=NOERROR ) break; } +#endif }else if (!strcmp ("resize.f2fs" , prog)) { +#ifdef WITH_RESIZE const char *option_string ="d:st:iV" ; c.func =RESIZE ;
@@ -526,7 +531,9 @@ void f2fs_parse_options(int argc, char *argv[])if (err !=NOERROR )break ; } +#endif }else if (!strcmp ("sload.f2fs" , prog)) { +#ifdef WITH_SLOAD const char *option_string ="C:d:f:p:s:St:T:V" ;#ifdef HAVE_LIBSELINUX int max_nr_opt = (int)sizeof(c.seopt_file) /
@@ -595,6 +602,7 @@ void f2fs_parse_options(int argc, char *argv[])if (err !=NOERROR ) break; } +#endif /* WITH_SLOAD */ }if (err ==NOERROR ) {
@@ -707,6 +715,7 @@ static int do_fsck(struct f2fs_sb_info *sbi)return FSCK_ERRORS_LEFT_UNCORRECTED ; } +#ifdef WITH_DUMP static void do_dump (struct f2fs_sb_info *sbi) {struct dump_option *opt = (struct dump_option *)c.private;
@@ -733,7 +742,9 @@ static void do_dump(struct f2fs_sb_info *sbi)print_cp_state (flag); } +#endif +#ifdef WITH_DEFRAG static int do_defrag (struct f2fs_sb_info *sbi) {struct f2fs_super_block *sb =F2FS_RAW_SUPER (sbi);
@@ -782,7 +793,9 @@ out_range: c.defrag_target);return -1 ; } +#endif +#ifdef WITH_RESIZE static int do_resize (struct f2fs_sb_info *sbi) {if (!c.target_sectors)
@@ -796,7 +809,9 @@ static int do_resize(struct f2fs_sb_info *sbi)return f2fs_resize (sbi); } +#endif +#ifdef WITH_SLOAD static int do_sload (struct f2fs_sb_info *sbi) {if (!c.from_dir) {
@@ -808,6 +823,7 @@ static int do_sload(struct f2fs_sb_info *sbi) return f2fs_sload(sbi); } +#endif #if defined(__APPLE__)static u64 get_boottime_ns ()
Signature · 8 lines
2.29.2.576.ga3fc446d84-goog _______________________________________________ Linux-f2fs-devel mailing list [email protected] https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
PATCH v3 2/3 f2fs-tools:sload.f2fs compression support
In reply to Robin HsuFrom: Robin Hsu <[email protected]>
Add F2FS compression support for sload
* Support file extension filter, either default-accept or default-deny policy
* Support choice of compression algorithm, LZO (version 2) or LZ4 (default)
* Support custom log of cluster size
* Support minimum number of compressed blocks per cluster (default 1). A cluster will not be compressed if the number can not be met.
* suuport -r (read-only) option
--- fsck/compress_wrapper.c | 102 ++++++++++++++++++++ fsck/compress_wrapper.h | 22 +++++ fsck/fsck.h | 15 +++ fsck/main.c | 141 +++++++++++++++++++++++++++- fsck/segment.c | 202 +++++++++++++++++++++++++++++++++++++--- fsck/sload.c | 67 +++++++++++++ include/f2fs_fs.h | 76 ++++++++++++++- lib/libf2fs_io.c | 33 +++++++ 8 files changed, 644 insertions(+), 14 deletions(-) create mode 100644 fsck/compress_wrapper.c create mode 100644 fsck/compress_wrapper.h
fsck/compress_wrapper.c new file +102 -0
@@ -0,0 +1,102 @@ +/** +* compress_wrapper.c +* +* Copyright (c) 2020 Google Inc. +* Robin Hsu <[email protected]> +* : initial created, for sload compression support +* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License version 2 as +* published by the Free Software Foundation. +*/ +#include "f2fs.h" /* for config.h for general environment (non-Android) */ + +#include "compress_wrapper.h" +#ifdef HAVE_LIBLZO2 +#include <lzo/lzo1x.h> /* for lzo1x_1_15_compress() */ +#endif +#ifdef HAVE_LIBLZ4 +#include <lz4.h> /* for LZ4_compress_fast_extState() */ +#endif + +/* +* macro/constants borrowed from kernel header (GPL-2.0): +* include/linux/lzo.h, and include/linux/lz4.h +*/ +#ifdef HAVE_LIBLZO2 +#define lzo1x_worst_compress (x) ((x) + (x) / 16 + 64 + 3 + 2) +#define LZO_WORK_SIZE ALIGN_UP(LZO1X_1_15_MEM_COMPRESS, 8) +#endif +#ifdef HAVE_LIBLZ4 +#define LZ4_MEMORY_USAGE 14 +#define LZ4_MAX_INPUT_SIZE 0x7E000000/* 2 113 929 216 bytes */ +#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(long long)) +#define LZ4_MEM_COMPRESS LZ4_STREAMSIZE +#define LZ4_ACCELERATION_DEFAULT 1 +#define LZ4_WORK_SIZE ALIGN_UP(LZ4_MEM_COMPRESS, 8) +#endif + +#ifdef HAVE_LIBLZO2 +static void lzo_compress_init (struct compress_ctx *cc) +{ +size_t size = cc->cluster_size *F2FS_BLKSIZE ; +size_t alloc = size +lzo1x_worst_compress (size) + +COMPRESS_HEADER_SIZE +LZO_WORK_SIZE ; +ASSERT ((cc->private =qbuf_alloc (alloc)) !=NULL ); + cc->rbuf = (char *) cc->private +LZO_WORK_SIZE ; + cc->cbuf = (struct compress_data *)((char *) cc->rbuf + size); +} + +static int lzo_compress (struct compress_ctx *cc) +{ +int ret =lzo1x_1_15_compress (cc->rbuf, cc->rlen, cc->cbuf->cdata, + (lzo_uintp)(&cc->clen), cc->private); + cc->cbuf->clen =cpu_to_le32 (cc->clen); +return ret; +} +#endif + +#ifdef HAVE_LIBLZ4 +static void lz4_compress_init (struct compress_ctx *cc) +{ +size_t size = cc->cluster_size *F2FS_BLKSIZE ; +size_t alloc = size +LZ4_COMPRESSBOUND (size) + +COMPRESS_HEADER_SIZE +LZ4_WORK_SIZE ; +ASSERT ((cc->private =qbuf_alloc (alloc)) !=NULL ); + cc->rbuf = (char *) cc->private +LZ4_WORK_SIZE ; + cc->cbuf = (struct compress_data *)((char *) cc->rbuf + size); +} + +static int lz4_compress (struct compress_ctx *cc) +{ + cc->clen =LZ4_compress_fast_extState (cc->private, cc->rbuf, + (char *)cc->cbuf->cdata, cc->rlen, + cc->rlen -F2FS_BLKSIZE * c.sldc_min_cbpc, +LZ4_ACCELERATION_DEFAULT ); + +if (!cc->clen) +return 1 ; + + cc->cbuf->clen =cpu_to_le32 (cc->clen); +return 0 ; +} +#endif + +const char *ca_names[] = { +"LZO" , +"LZ4" , +"" ,/* end of the name list */ +}; + +compress_ops compr_ops[] = { +#ifdef HAVE_LIBLZO2 + {lzo_compress_init, lzo_compress}, +#else + {NULL, NULL}, +#endif +#ifdef HAVE_LIBLZ4 + {lz4_compress_init, lz4_compress}, +#else + {NULL, NULL}, +#endif +};
fsck/compress_wrapper.h new file +22 -0
@@ -0,0 +1,22 @@ +/** +* compress_wrapper.h +* +* Copyright (c) 2020 Google Inc. +* Robin Hsu <[email protected]> +* : initial created, for sload compression support +* Copyright (c) 2020 Google Inc. +* Robin Hsu <[email protected]> +* : add sload compression support +* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License version 2 as +* published by the Free Software Foundation. +*/ + +#ifndef COMPRESS_WRAPPER_H +#define COMPRESS_WRAPPER_H + +#include "f2fs_fs.h" +extern compress_ops compr_ops[];/* [0]: LZO, [1]: LZ4, */ + +#endif /* COMPRESS_WRAPPER_H */
fsck/fsck.h +15 -0
@@ -3,6 +3,9 @@* * Copyright (c) 2013 Samsung Electronics Co., Ltd. * http://www.samsung.com/ +* Copyright (c) 2020 Google Inc. +* Robin Hsu <[email protected]> +* : add sload compression support * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as
@@ -266,6 +269,9 @@ int f2fs_resize(struct f2fs_sb_info *);/* sload.c */ int f2fs_sload (struct f2fs_sb_info *); +void sldc_erase_bufs (struct compress_ctx *cc); +void sload_countblk (void ); +extern struct ext_tbl_op ext_filter;/* segment.c */ int reserve_new_block(struct f2fs_sb_info *, block_t *,
@@ -282,7 +288,16 @@ block_t new_node_block(struct f2fs_sb_info *,struct quota_file ;u64 f2fs_quota_size (struct quota_file *);u64 f2fs_read (struct f2fs_sb_info *,nid_t ,u8 *,u64 ,pgoff_t ); +enum wr_addr_type { +WR_NORMAL =1 , +WR_COMPRESS_DATA =2 , +WR_NULL_ADDR =NULL_ADDR ,/* 0 */ +WR_NEW_ADDR =NEW_ADDR ,/* -1U */ +WR_COMPRESS_ADDR =COMPRESS_ADDR ,/* -2U */ +};u64 f2fs_write (struct f2fs_sb_info *,nid_t ,u8 *,u64 ,pgoff_t ); +u64 f2fs_write_compress_data (struct f2fs_sb_info *,nid_t ,u8 *,u64 ,pgoff_t ); +u64 f2fs_write_addrtag (struct f2fs_sb_info *,nid_t ,pgoff_t ,unsigned int );void f2fs_filesize_update (struct f2fs_sb_info *,nid_t ,u64 ); int get_dnode_of_data(struct f2fs_sb_info *, struct dnode_of_data *,
fsck/main.c +140 -1
197 lines not on this page · raw message
fsck/segment.c +190 -12
284 lines not on this page · raw message
fsck/sload.c +67 -0
79 lines not on this page · raw message
include/f2fs_fs.h +75 -1
118 lines not on this page · raw message
lib/libf2fs_io.c +33 -0
@@ -5,6 +5,9 @@* http://www.samsung.com/ * Copyright (c) 2019 Google Inc. * http://www.google.com/ +* Copyright (c) 2020 Google Inc. +* Robin Hsu <[email protected]> +* : add quick-buffer for sload compression support * * Dual licensed under the GPL or LGPL version 2 licenses. */
@@ -106,6 +109,36 @@ static long dcache_relocate_offset0[] = { };static int dcache_relocate_offset[16 ]; +/* quick (shared) buffer */ +static bool qbuf_initialized; +static void *qbuf; +static size_t qbufsize; +void qbuf_free (void ) +{ +ASSERT (qbuf_initialized); +if (qbuf !=NULL ) { +free (qbuf); + qbuf =NULL ; + qbufsize =0 ; + } +} +void *qbuf_alloc (size_t size) +{ +ASSERT (qbuf_initialized); +if (size > qbufsize) { +qbuf_free (); + qbuf =malloc (size); + } +return qbuf; +} +void qbuf_init (void ) +{ +if (qbuf_initialized) +return ; +atexit (qbuf_free); + qbuf_initialized = true; +} +static void dcache_print_statistics (void ) {long i;
Signature · 8 lines
2.29.2.576.ga3fc446d84-goog _______________________________________________ Linux-f2fs-devel mailing list [email protected] https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
PATCH v3 3/3 f2fs-tools:sload.f2fs compress: Fixed automake
In reply to Robin HsuFrom: Robin Hsu <[email protected]>
Fixed automake for sload.f2fs compression support
./configure automatcally compile in liblzo2 (for sload to support -a LZO) and liblz4 (for sload to support -a LZ4), whhen the libraries present.
--- configure.ac | 12 ++++++++++++ fsck/Makefile.am | 9 ++++++--- 2 files changed, 18 insertions(+), 3 deletions(-)
configure.ac +12 -0
@@ -52,6 +52,18 @@ AC_PATH_PROG([LDCONFIG], [ldconfig], [$PATH:/sbin]) # Checks for libraries. +AC_CHECK_LIB([lzo2], [main], + [AC_SUBST([liblzo2_LIBS], ["-llzo2"]) + AC_DEFINE([HAVE_LIBLZO2], [1], + [Define if you have liblzo2]) + ], [], []) + +AC_CHECK_LIB([lz4], [main], + [AC_SUBST([liblz4_LIBS], ["-llz4"]) + AC_DEFINE([HAVE_LIBLZ4], [1], + [Define if you have liblz4]) + ], [], []) + PKG_CHECK_MODULES([libuuid], [uuid]) AS_IF([test "x$with_selinux" != "xno"],
fsck/Makefile.am +6 -3
@@ -3,12 +3,15 @@ AM_CPPFLAGS = ${libuuid_CFLAGS} -I$(top_srcdir)/include AM_CFLAGS = -Wall sbin_PROGRAMS = fsck.f2fs -noinst_HEADERS = common.h dict.h dqblk_v2.h f2fs.h fsck.h node.h quotaio.h quotaio_tree.h quotaio_v2.h xattr.h +noinst_HEADERS = common.h dict.h dqblk_v2.h f2fs.h fsck.h node.h quotaio.h \ + quotaio_tree.h quotaio_v2.h xattr.h compress_wrapper.h include_HEADERS = $(top_srcdir)/include/quota.h fsck_f2fs_SOURCES = main.c fsck.c dump.c mount.c defrag.c resize.c \ - node.c segment.c dir.c sload.c xattr.c \ + node.c segment.c dir.c sload.c xattr.c compress_wrapper.c \ dict.c mkquota.c quotaio.c quotaio_tree.c quotaio_v2.c -fsck_f2fs_LDADD = ${libselinux_LIBS} ${libuuid_LIBS} $(top_builddir)/lib/libf2fs.la +fsck_f2fs_LDADD = ${libselinux_LIBS} ${libuuid_LIBS} \ + ${liblzo2_LIBS} ${liblz4_LIBS} \ + $(top_builddir)/lib/libf2fs.la install-data-hook: ln -sf fsck.f2fs $(DESTDIR)/$(sbindir)/dump.f2fs
Signature · 8 lines
2.29.2.576.ga3fc446d84-goog _______________________________________________ Linux-f2fs-devel mailing list [email protected] https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel