Skip to contentSkip to conversation

f2fs-tools: sload.f2fs to support compression

PATCH Superseded Robin Hsu 1 participant 4 messages Dec 2, 2020 Library code

v1 of 3 Superseded Latest is v3 →

lore
Share & tools
lore

Robin Hsu

author
#0 lore raw

cover f2fs-tools: sload.f2fs to support compression

From: Robin Hsu <[email protected]>

* 3 patch set:
#1: added some #ifdef for easier support
#2: main code change
#3: automake changes

Robin Hsu (3):
f2fs-tools: Added #ifdef WITH_func
f2fs-tools:sload.f2fs compression support
f2fs-tools:sload.f2fs compress: Fixed automake

 configure.ac            |  32 +++++++
 fsck/Makefile.am        |   9 +-
 fsck/compress_wrapper.c | 102 ++++++++++++++++++++
 fsck/compress_wrapper.h |  22 +++++
 fsck/fsck.h             |  15 +++
 fsck/main.c             | 154 +++++++++++++++++++++++++++++-
 fsck/segment.c          | 202 +++++++++++++++++++++++++++++++++++++---
 fsck/sload.c            |  67 +++++++++++++
 include/f2fs_fs.h       |  76 ++++++++++++++-
 lib/libf2fs_io.c        |  33 +++++++
 10 files changed, 695 insertions(+), 17 deletions(-)
 create mode 100644 fsck/compress_wrapper.c
 create mode 100644 fsck/compress_wrapper.h
Signature · 8 lines
2.29.2.454.gaff20da3a2-goog



_______________________________________________
Linux-f2fs-devel mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
author
#1 lore raw

PATCH 1/3 f2fs-tools: Added #ifdef WITH_func

From: Robin Hsu <[email protected]>

Add proprocessor defines (options) 'WITH_func',
where func = DUMP, DEFRAG, RESIZE, or SLOAD

---
 fsck/main.c | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
fsck/main.c +16 -0
index e70048b..b20498f 100644
@@ -345,6 +345,7 @@ void f2fs_parse_options(int argc, char *argv[])
 				break;
 		}
 	} else if (!strcmp("dump.f2fs", prog)) {
+#ifdef WITH_DUMP
 		const char *option_string = "d:i:n:s:Sa:b:V";
 		static struct dump_option dump_opt = {
 			.nid = 0,	/* default root ino */
@@ -426,7 +427,9 @@ void f2fs_parse_options(int argc, char *argv[])
 		}
 
 		c.private = &dump_opt;
+#endif
 	} else if (!strcmp("defrag.f2fs", prog)) {
+#ifdef WITH_DEFRAG
 		const char *option_string = "d:s:Sl:t:iV";
 
 		c.func = DEFRAG;
@@ -484,7 +487,9 @@ void f2fs_parse_options(int argc, char *argv[])
 			if (err != NOERROR)
 				break;
 		}
+#endif
 	} else if (!strcmp("resize.f2fs", prog)) {
+#ifdef WITH_RESIZE
 		const char *option_string = "d:st:iV";
 
 		c.func = RESIZE;
@@ -526,7 +531,9 @@ void f2fs_parse_options(int argc, char *argv[])
 			if (err != NOERROR)
 				break;
 		}
+#endif
 	} else if (!strcmp("sload.f2fs", prog)) {
+#ifdef WITH_SLOAD
 		const char *option_string = "C:d:f:p:s:St:T:V";
 #ifdef HAVE_LIBSELINUX
 		int max_nr_opt = (int)sizeof(c.seopt_file) /
@@ -595,6 +602,7 @@ void f2fs_parse_options(int argc, char *argv[])
 			if (err != NOERROR)
 				break;
 		}
+#endif /* WITH_SLOAD */
 	}
 
 	if (err == NOERROR) {
@@ -707,6 +715,7 @@ static int do_fsck(struct f2fs_sb_info *sbi)
 	return FSCK_ERRORS_LEFT_UNCORRECTED;
 }
 
+#ifdef WITH_DUMP
 static void do_dump(struct f2fs_sb_info *sbi)
 {
 	struct dump_option *opt = (struct dump_option *)c.private;
@@ -733,7 +742,9 @@ static void do_dump(struct f2fs_sb_info *sbi)
 	print_cp_state(flag);
 
 }
+#endif
 
+#ifdef WITH_DEFRAG
 static int do_defrag(struct f2fs_sb_info *sbi)
 {
 	struct f2fs_super_block *sb = F2FS_RAW_SUPER(sbi);
@@ -782,7 +793,9 @@ out_range:
 				c.defrag_target);
 	return -1;
 }
+#endif
 
+#ifdef WITH_RESIZE
 static int do_resize(struct f2fs_sb_info *sbi)
 {
 	if (!c.target_sectors)
@@ -796,7 +809,9 @@ static int do_resize(struct f2fs_sb_info *sbi)
 
 	return f2fs_resize(sbi);
 }
+#endif
 
+#ifdef WITH_SLOAD
 static int do_sload(struct f2fs_sb_info *sbi)
 {
 	if (!c.from_dir) {
@@ -808,6 +823,7 @@ static int do_sload(struct f2fs_sb_info *sbi)
 
 	return f2fs_sload(sbi);
 }
+#endif
 
 #if defined(__APPLE__)
 static u64 get_boottime_ns()
Signature · 8 lines
2.29.2.454.gaff20da3a2-goog



_______________________________________________
Linux-f2fs-devel mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
author
#2 lore raw

PATCH 2/3 f2fs-tools:sload.f2fs compression support

In reply to Robin Hsu

From: Robin Hsu <[email protected]>

Add F2FS compression support for sload
* Support file extension filter, either default-accept or default-deny policy
* Support choice of compression algorithm, LZO (version 2) or LZ4 (default)
* Support custom log of cluster size
* Support minimum number of compressed blocks per cluster (default 1). A cluster will not be compressed if the number can not be met.
* suuport -r (read-only) option

---
 fsck/compress_wrapper.c | 102 ++++++++++++++++++++
 fsck/compress_wrapper.h |  22 +++++
 fsck/fsck.h             |  15 +++
 fsck/main.c             | 138 ++++++++++++++++++++++++++-
 fsck/segment.c          | 202 +++++++++++++++++++++++++++++++++++++---
 fsck/sload.c            |  67 +++++++++++++
 include/f2fs_fs.h       |  76 ++++++++++++++-
 lib/libf2fs_io.c        |  33 +++++++
 8 files changed, 641 insertions(+), 14 deletions(-)
 create mode 100644 fsck/compress_wrapper.c
 create mode 100644 fsck/compress_wrapper.h
fsck/compress_wrapper.c new file +102 -0
new file mode 100644
index 0000000..2cdc4fd
@@ -0,0 +1,102 @@
+/**
+ * compress_wrapper.c
+ *
+ * Copyright (c) 2020 Google Inc.
+ *   Robin Hsu <[email protected]>
+ *   : initial created, for sload compression support
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#include "f2fs.h" /* for config.h for general environment (non-Android) */
+
+#include "compress_wrapper.h"
+#ifdef HAVE_LIBLZO2
+#include <lzo/lzo1x.h> /* for lzo1x_1_15_compress() */
+#endif
+#ifdef HAVE_LIBLZ4
+#include <lz4.h> /* for LZ4_compress_fast_extState() */
+#endif
+
+/*
+ * macro/constants borrowed from kernel header (GPL-2.0):
+ * include/linux/lzo.h, and include/linux/lz4.h
+ */
+#ifdef HAVE_LIBLZO2
+#define lzo1x_worst_compress(x)		((x) + (x) / 16 + 64 + 3 + 2)
+#define LZO_WORK_SIZE			ALIGN_UP(LZO1X_1_15_MEM_COMPRESS, 8)
+#endif
+#ifdef HAVE_LIBLZ4
+#define LZ4_MEMORY_USAGE		14
+#define LZ4_MAX_INPUT_SIZE		0x7E000000 /* 2 113 929 216 bytes */
+#define LZ4_STREAMSIZE			(LZ4_STREAMSIZE_U64 * sizeof(long long))
+#define LZ4_MEM_COMPRESS		LZ4_STREAMSIZE
+#define LZ4_ACCELERATION_DEFAULT	1
+#define LZ4_WORK_SIZE			ALIGN_UP(LZ4_MEM_COMPRESS, 8)
+#endif
+
+#ifdef HAVE_LIBLZO2
+static void lzo_compress_init(struct compress_ctx *cc)
+{
+	size_t size = cc->cluster_size * F2FS_BLKSIZE;
+	size_t alloc = size + lzo1x_worst_compress(size)
+			+ COMPRESS_HEADER_SIZE + LZO_WORK_SIZE;
+	ASSERT((cc->private = qbuf_alloc(alloc)) != NULL);
+	cc->rbuf = (char *) cc->private + LZO_WORK_SIZE;
+	cc->cbuf = (struct compress_data *)((char *) cc->rbuf + size);
+}
+
+static int lzo_compress(struct compress_ctx *cc)
+{
+	int ret = lzo1x_1_15_compress(cc->rbuf, cc->rlen, cc->cbuf->cdata,
+			(lzo_uintp)(&cc->clen), cc->private);
+	cc->cbuf->clen = cpu_to_le32(cc->clen);
+	return ret;
+}
+#endif
+
+#ifdef HAVE_LIBLZ4
+static void lz4_compress_init(struct compress_ctx *cc)
+{
+	size_t size = cc->cluster_size * F2FS_BLKSIZE;
+	size_t alloc = size + LZ4_COMPRESSBOUND(size)
+			+ COMPRESS_HEADER_SIZE + LZ4_WORK_SIZE;
+	ASSERT((cc->private = qbuf_alloc(alloc)) != NULL);
+	cc->rbuf = (char *) cc->private + LZ4_WORK_SIZE;
+	cc->cbuf = (struct compress_data *)((char *) cc->rbuf + size);
+}
+
+static int lz4_compress(struct compress_ctx *cc)
+{
+	cc->clen = LZ4_compress_fast_extState(cc->private, cc->rbuf,
+			(char *)cc->cbuf->cdata, cc->rlen,
+			cc->rlen - F2FS_BLKSIZE * c.sldc_min_cbpc,
+			LZ4_ACCELERATION_DEFAULT);
+
+	if (!cc->clen)
+		return 1;
+
+	cc->cbuf->clen = cpu_to_le32(cc->clen);
+	return 0;
+}
+#endif
+
+const char *ca_names[] = {
+	"LZO",
+	"LZ4",
+	"", /* end of the name list */
+};
+
+compress_ops compr_ops[] = {
+#ifdef HAVE_LIBLZO2
+	{lzo_compress_init, lzo_compress},
+#else
+	{NULL, NULL},
+#endif
+#ifdef HAVE_LIBLZ4
+	{lz4_compress_init, lz4_compress},
+#else
+	{NULL, NULL},
+#endif
+};
fsck/compress_wrapper.h new file +22 -0
new file mode 100644
index 0000000..ec33d43
@@ -0,0 +1,22 @@
+/**
+ * compress_wrapper.h
+ *
+ * Copyright (c) 2020 Google Inc.
+ *   Robin Hsu <[email protected]>
+ *   : initial created, for sload compression support
+ * Copyright (c) 2020 Google Inc.
+ *   Robin Hsu <[email protected]>
+ *  : add sload compression support
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef COMPRESS_WRAPPER_H
+#define COMPRESS_WRAPPER_H
+
+#include "f2fs_fs.h"
+extern compress_ops compr_ops[]; /* [0]: LZO, [1]: LZ4, */
+
+#endif /* COMPRESS_WRAPPER_H */
fsck/fsck.h +15 -0
index c5e85fe..4e866ec 100644
@@ -3,6 +3,9 @@
  *
  * Copyright (c) 2013 Samsung Electronics Co., Ltd.
  *             http://www.samsung.com/
+ * Copyright (c) 2020 Google Inc.
+ *   Robin Hsu <[email protected]>
+ *  : add sload compression support
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
@@ -266,6 +269,9 @@ int f2fs_resize(struct f2fs_sb_info *);
 
 /* sload.c */
 int f2fs_sload(struct f2fs_sb_info *);
+void sldc_erase_bufs(struct compress_ctx *cc);
+void sload_countblk(void);
+extern struct ext_tbl_op ext_filter;
 
 /* segment.c */
 int reserve_new_block(struct f2fs_sb_info *, block_t *,
@@ -282,7 +288,16 @@ block_t new_node_block(struct f2fs_sb_info *,
 struct quota_file;
 u64 f2fs_quota_size(struct quota_file *);
 u64 f2fs_read(struct f2fs_sb_info *, nid_t, u8 *, u64, pgoff_t);
+enum wr_addr_type {
+	WR_NORMAL = 1,
+	WR_COMPRESS_DATA = 2,
+	WR_NULL_ADDR = NULL_ADDR,  /* 0 */
+	WR_NEW_ADDR = NEW_ADDR, /* -1U */
+	WR_COMPRESS_ADDR = COMPRESS_ADDR, /* -2U */
+};
 u64 f2fs_write(struct f2fs_sb_info *, nid_t, u8 *, u64, pgoff_t);
+u64 f2fs_write_compress_data(struct f2fs_sb_info *, nid_t, u8 *, u64, pgoff_t);
+u64 f2fs_write_addrtag(struct f2fs_sb_info *, nid_t, pgoff_t, unsigned int);
 void f2fs_filesize_update(struct f2fs_sb_info *, nid_t, u64);
 
 int get_dnode_of_data(struct f2fs_sb_info *, struct dnode_of_data *,
fsck/main.c +137 -1

194 lines not on this page · raw message

fsck/segment.c +190 -12

284 lines not on this page · raw message

fsck/sload.c +67 -0

79 lines not on this page · raw message

include/f2fs_fs.h +75 -1

118 lines not on this page · raw message

lib/libf2fs_io.c +33 -0
index 138285d..0280896 100644
@@ -5,6 +5,9 @@
  *             http://www.samsung.com/
  * Copyright (c) 2019 Google Inc.
  *             http://www.google.com/
+ * Copyright (c) 2020 Google Inc.
+ *   Robin Hsu <[email protected]>
+ *  : add quick-buffer for sload compression support
  *
  * Dual licensed under the GPL or LGPL version 2 licenses.
  */
@@ -106,6 +109,36 @@ static long dcache_relocate_offset0[] = {
 };
 static int dcache_relocate_offset[16];
 
+/* quick (shared) buffer */
+static bool qbuf_initialized;
+static void *qbuf;
+static size_t qbufsize;
+void qbuf_free(void)
+{
+	ASSERT(qbuf_initialized);
+	if (qbuf != NULL) {
+		free(qbuf);
+		qbuf = NULL;
+		qbufsize = 0;
+	}
+}
+void *qbuf_alloc(size_t size)
+{
+	ASSERT(qbuf_initialized);
+	if (size > qbufsize) {
+		qbuf_free();
+		qbuf = malloc(size);
+	}
+	return qbuf;
+}
+void qbuf_init(void)
+{
+	if (qbuf_initialized)
+		return;
+	atexit(qbuf_free);
+	qbuf_initialized = true;
+}
+
 static void dcache_print_statistics(void)
 {
 	long i;
Signature · 8 lines
2.29.2.454.gaff20da3a2-goog



_______________________________________________
Linux-f2fs-devel mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
author
#3 lore raw

PATCH 3/3 f2fs-tools:sload.f2fs compress: Fixed automake

In reply to Robin Hsu

From: Robin Hsu <[email protected]>

Fixed automake for sload.f2fs compression support

./configure now will by default depends on liblzo2 and liblz4.

To compile without liblzo2 (and thus not support liblzo2 compression), run ./configure with '--without-lzo2' option.

To compile without liblz4 (and thus not support liblz4 compression), run ./configure with '--without-lz4' option.

---
 configure.ac     | 32 ++++++++++++++++++++++++++++++++
 fsck/Makefile.am |  9 ++++++---
 2 files changed, 38 insertions(+), 3 deletions(-)
configure.ac +32 -0
index 1e5619d..3a357b2 100644
@@ -44,6 +44,14 @@ AC_ARG_WITH([blkid],
 	AS_HELP_STRING([--without-blkid],
 	  [Ignore presence of libblkid and disable blkid support]))
 
+AC_ARG_WITH([lzo2],
+	AS_HELP_STRING([--without-lzo2],
+	  [Ignore liblzo ver.2 for sload (not supporting lzo compression)]))
+
+AC_ARG_WITH([lz4],
+	AS_HELP_STRING([--without-lz4],
+	  [Ignore liblz4 for sload (not supporting lz4 compression)]))
+
 # Checks for programs.
 AC_PROG_CC
 AC_PROG_LIBTOOL
@@ -52,6 +60,30 @@ AC_PATH_PROG([LDCONFIG], [ldconfig],
        [$PATH:/sbin])
 
 # Checks for libraries.
+AS_IF([test "x$with_lzo2" != xno],
+            [AC_CHECK_LIB([lzo2], [main],
+              [AC_SUBST([liblzo2_LIBS], ["-llzo2"])
+               AC_DEFINE([HAVE_LIBLZO2], [1],
+                         [Define if you have liblzo2])
+              ],
+              [if test "x$with_lzo2" != xcheck; then
+                 AC_MSG_FAILURE(
+                   [--with-lzo2 was given (or by default), but test for liblzo2 failed])
+               fi
+              ], [])])
+
+AS_IF([test "x$with_lz4" != xno],
+            [AC_CHECK_LIB([lz4], [main],
+              [AC_SUBST([liblz4_LIBS], ["-llz4"])
+               AC_DEFINE([HAVE_LIBLZ4], [1],
+                         [Define if you have liblz4])
+              ],
+              [if test "x$with_lz4" != xcheck; then
+                 AC_MSG_FAILURE(
+                   [--with-lz4 was given (or by default), but test for liblz4 failed])
+               fi
+              ], [])])
+
 PKG_CHECK_MODULES([libuuid], [uuid])
 
 AS_IF([test "x$with_selinux" != "xno"],
fsck/Makefile.am +6 -3
index 1fc7310..74bc4b8 100644
@@ -3,12 +3,15 @@
 AM_CPPFLAGS = ${libuuid_CFLAGS} -I$(top_srcdir)/include
 AM_CFLAGS = -Wall
 sbin_PROGRAMS = fsck.f2fs
-noinst_HEADERS = common.h dict.h dqblk_v2.h f2fs.h fsck.h node.h quotaio.h quotaio_tree.h quotaio_v2.h xattr.h
+noinst_HEADERS = common.h dict.h dqblk_v2.h f2fs.h fsck.h node.h quotaio.h \
+		quotaio_tree.h quotaio_v2.h xattr.h compress_wrapper.h
 include_HEADERS = $(top_srcdir)/include/quota.h
 fsck_f2fs_SOURCES = main.c fsck.c dump.c mount.c defrag.c resize.c \
-		node.c segment.c dir.c sload.c xattr.c \
+		node.c segment.c dir.c sload.c xattr.c compress_wrapper.c \
 		dict.c mkquota.c quotaio.c quotaio_tree.c quotaio_v2.c
-fsck_f2fs_LDADD = ${libselinux_LIBS} ${libuuid_LIBS} $(top_builddir)/lib/libf2fs.la
+fsck_f2fs_LDADD = ${libselinux_LIBS} ${libuuid_LIBS} \
+	${liblzo2_LIBS} ${liblz4_LIBS} \
+	$(top_builddir)/lib/libf2fs.la
 
 install-data-hook:
 	ln -sf fsck.f2fs $(DESTDIR)/$(sbindir)/dump.f2fs
Signature · 8 lines
2.29.2.454.gaff20da3a2-goog



_______________________________________________
Linux-f2fs-devel mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel