Skip to contentSkip to conversation

erofs_utils: new mkfs framework

PATCH New Gao Xiang 1 participant 14 messages May 31, 2019

Subsystems (2)
lore
Share & tools
lore

Gao Xiang

author
#0 lore raw

cover erofs_utils: new mkfs framework

Hi all,
I spent nearly a month to make big changes to erofs-mkfs, which mainly introduces a new mkfs framework.

I personally think it is much cleaner and more scalable than the old one, which is hack and not clean enough.

In-place compression is still work-in-progress, thus it isn't included in this series.

Comments are welcome!

Thanks,
Gao Xiang

Gao Xiang (8):
erofs-utils: introduce buffer cache
erofs-utils: introduce inode operations
erofs-utils: introduce generic compression framework
erofs-utils: introduce lz4/lz4hc compression algorithm
erofs-utils: introduce compression for regular files
erofs-utils: propagate compressed size to its callers
erofs-utils: add a README
erofs-utils: fix potential erofs_bh_balloon failure

Li Guifu (5):
erofs-utils: add erofs on-disk layout
erofs-utils: introduce erofs-utils basic headers
erofs-utils: introduce miscellaneous files
erofs-utils: add input/output functions
erofs-utils: introduce mkfs support

Signature · 1 line
2.17.1
author
#1 lore raw

PATCH 1/13 erofs-utils: add erofs on-disk layout

From: Li Guifu <[email protected]>

This patch adds a header file describing the erofs on-disk layout, which should be kept in line with the kernel implementation all the time.

Signed-off-by Li Guifu <bluce.liguifu at huawei.com>
Signed-off-by Miao Xie <miaoxie at huawei.com>
Signed-off-by Fang Wei <fangwei1 at huawei.com>
Signed-off-by Gao Xiang <gaoxiang25 at huawei.com>
---
 include/erofs_fs.h | 275 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 275 insertions(+)
 create mode 100644 include/erofs_fs.h
include/erofs_fs.h new file +275 -0
new file mode 100644
index 0000000..4b6c1d1
@@ -0,0 +1,275 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Apache-2.0 */
+/*
+ * erofs_utils/include/erofs_fs.h
+ *
+ * Copyright (C) 2017-2018 HUAWEI, Inc.
+ *             http://www.huawei.com/
+ * Created by Gao Xiang <gaoxiang25 at huawei.com>
+ *
+ * This file is dual-licensed; you may select either the GNU General Public
+ * License version 2 or Apache License, Version 2.0. See the file COPYING
+ * in the main directory of the Linux distribution for more details.
+ */
+#ifndef __EROFS_FS_H
+#define __EROFS_FS_H
+
+/* Enhanced(Extended) ROM File System */
+#define EROFS_SUPER_MAGIC_V1    0xE0F5E1E2
+#define EROFS_SUPER_OFFSET      1024
+
+struct erofs_super_block {
+/*  0 */__le32 magic;           /* in the little endian */
+/*  4 */__le32 checksum;        /* crc32c(super_block) */
+/*  8 */__le32 features;
+/* 12 */__u8 blkszbits;         /* support block_size == PAGE_SIZE only */
+/* 13 */__u8 reserved;
+
+/* 14 */__le16 root_nid;
+/* 16 */__le64 inos;            /* total valid ino # (== f_files - f_favail) */
+
+/* 24 */__le64 build_time;      /* inode v1 time derivation */
+/* 32 */__le32 build_time_nsec;
+/* 36 */__le32 blocks;          /* used for statfs */
+/* 40 */__le32 meta_blkaddr;
+/* 44 */__le32 xattr_blkaddr;
+/* 48 */__u8 uuid[16];          /* 128-bit uuid for volume */
+/* 64 */__u8 volume_name[16];   /* volume name */
+
+/* 80 */__u8 reserved2[48];     /* 128 bytes */
+} __packed;
+
+/*
+ * erofs inode data mapping:
+ * 0 - inode plain without inline data A:
+ * inode, [xattrs], ... | ... | no-holed data
+ * 1 - inode VLE compression B:
+ * inode, [xattrs], extents ... | ...
+ * 2 - inode plain with inline data C:
+ * inode, [xattrs], last_inline_data, ... | ... | no-holed data
+ * 3~7 - reserved
+ */
+enum {
+	EROFS_INODE_LAYOUT_PLAIN,
+	EROFS_INODE_LAYOUT_COMPRESSION,
+	EROFS_INODE_LAYOUT_INLINE,
+	EROFS_INODE_LAYOUT_MAX
+};
+
+/* bit definitions of inode i_advise */
+#define EROFS_I_VERSION_BITS            1
+#define EROFS_I_DATA_MAPPING_BITS       3
+
+#define EROFS_I_VERSION_BIT             0
+#define EROFS_I_DATA_MAPPING_BIT        1
+
+struct erofs_inode_v1 {
+/*  0 */__le16 i_advise;
+
+/* 1 header + n-1 * 4 bytes inline xattr to keep continuity */
+/*  2 */__le16 i_xattr_icount;
+/*  4 */__le16 i_mode;
+/*  6 */__le16 i_nlink;
+/*  8 */__le32 i_size;
+/* 12 */__le32 i_reserved;
+/* 16 */union {
+		/* file total compressed blocks for data mapping 1 */
+		__le32 compressed_blocks;
+		__le32 raw_blkaddr;
+
+		/* for device files, used to indicate old/new device # */
+		__le32 rdev;
+	} i_u __packed;
+/* 20 */__le32 i_ino;           /* only used for 32-bit stat compatibility */
+/* 24 */__le16 i_uid;
+/* 26 */__le16 i_gid;
+/* 28 */__le32 i_checksum;
+} __packed;
+
+/* 32 bytes on-disk inode */
+#define EROFS_INODE_LAYOUT_V1   0
+/* 64 bytes on-disk inode */
+#define EROFS_INODE_LAYOUT_V2   1
+
+struct erofs_inode_v2 {
+	__le16 i_advise;
+
+	/* 1 header + n-1 * 4 bytes inline xattr to keep continuity */
+	__le16 i_xattr_icount;
+	__le16 i_mode;
+	__le16 i_reserved;      /* 8 bytes */
+	__le64 i_size;          /* 16 bytes */
+	union {
+		/* file total compressed blocks for data mapping 1 */
+		__le32 compressed_blocks;
+		__le32 raw_blkaddr;
+
+		/* for device files, used to indicate old/new device # */
+		__le32 rdev;
+	} i_u __packed;
+
+	/* only used for 32-bit stat compatibility */
+	__le32 i_ino;           /* 24 bytes */
+
+	__le32 i_uid;
+	__le32 i_gid;
+	__le64 i_ctime;         /* 32 bytes */
+	__le32 i_ctime_nsec;
+	__le32 i_nlink;
+	__u8   i_reserved2[12];
+	__le32 i_checksum;      /* 64 bytes */
+} __packed;
+
+#define EROFS_MAX_SHARED_XATTRS         (128)
+/* h_shared_count between 129 ... 255 are special # */
+#define EROFS_SHARED_XATTR_EXTENT       (255)
+
+/*
+ * inline xattrs (n == i_xattr_icount):
+ * erofs_xattr_ibody_header(1) + (n - 1) * 4 bytes
+ *          12 bytes           /                   \
+ *                            /                     \
+ *                           /-----------------------\
+ *                           |  erofs_xattr_entries+ |
+ *                           +-----------------------+
+ * inline xattrs must starts in erofs_xattr_ibody_header,
+ * for read-only fs, no need to introduce h_refcount
+ */
+struct erofs_xattr_ibody_header {
+	__le32 h_checksum;
+	__u8   h_shared_count;
+	__u8   h_reserved[7];
+	__le32 h_shared_xattrs[0];      /* shared xattr id array */
+} __packed;
+
+/* Name indexes */
+#define EROFS_XATTR_INDEX_USER              1
+#define EROFS_XATTR_INDEX_POSIX_ACL_ACCESS  2
+#define EROFS_XATTR_INDEX_POSIX_ACL_DEFAULT 3
+#define EROFS_XATTR_INDEX_TRUSTED           4
+#define EROFS_XATTR_INDEX_LUSTRE            5
+#define EROFS_XATTR_INDEX_SECURITY          6
+
+/* xattr entry (for both inline & shared xattrs) */
+struct erofs_xattr_entry {
+	__u8   e_name_len;      /* length of name */
+	__u8   e_name_index;    /* attribute name index */
+	__le16 e_value_size;    /* size of attribute value */
+	/* followed by e_name and e_value */
+	char   e_name[0];       /* attribute name */
+} __packed;
+
+#define ondisk_xattr_ibody_size(count)	({\
+	u32 __count = le16_to_cpu(count); \
+	((__count) == 0) ? 0 : \
+	sizeof(struct erofs_xattr_ibody_header) + \
+		sizeof(__u32) * ((__count) - 1); })
+
+#define EROFS_XATTR_ALIGN(size) round_up(size, sizeof(struct erofs_xattr_entry))
+#define EROFS_XATTR_ENTRY_SIZE(entry) EROFS_XATTR_ALIGN( \
+	sizeof(struct erofs_xattr_entry) + \
+	(entry)->e_name_len + le16_to_cpu((entry)->e_value_size))
+
+/* have to be aligned with 8 bytes on disk */
+struct erofs_extent_header {
+	__le32 eh_checksum;
+	__le32 eh_reserved[3];
+} __packed;
+
+/*
+ * Z_EROFS Variable-sized Logical Extent cluster type:
+ *    0 - literal (uncompressed) cluster
+ *    1 - compressed cluster (for the head logical cluster)
+ *    2 - compressed cluster (for the other logical clusters)
+ *
+ * In detail,
+ *    0 - literal (uncompressed) cluster,
+ *        di_advise = 0
+ *        di_clusterofs = the literal data offset of the cluster
+ *        di_blkaddr = the blkaddr of the literal cluster
+ *
+ *    1 - compressed cluster (for the head logical cluster)
+ *        di_advise = 1
+ *        di_clusterofs = the decompressed data offset of the cluster
+ *        di_blkaddr = the blkaddr of the compressed cluster
+ *
+ *    2 - compressed cluster (for the other logical clusters)
+ *        di_advise = 2
+ *        di_clusterofs =
+ *           the decompressed data offset in its own head cluster
+ *        di_u.delta[0] = distance to its corresponding head cluster
+ *        di_u.delta[1] = distance to its corresponding tail cluster
+ *                (di_advise could be 0, 1 or 2)
+ */
+enum {
+	Z_EROFS_VLE_CLUSTER_TYPE_PLAIN,
+	Z_EROFS_VLE_CLUSTER_TYPE_HEAD,
+	Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD,
+	Z_EROFS_VLE_CLUSTER_TYPE_RESERVED,
+	Z_EROFS_VLE_CLUSTER_TYPE_MAX
+};
+
+#define Z_EROFS_VLE_DI_CLUSTER_TYPE_BITS        2
+#define Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT         0
+
+struct z_erofs_vle_decompressed_index {
+	__le16 di_advise;
+	/* where to decompress in the head cluster */
+	__le16 di_clusterofs;
+
+	union {
+		/* for the head cluster */
+		__le32 blkaddr;
+		/*
+		 * for the rest clusters
+		 * eg. for 4k page-sized cluster, maximum 4K*64k = 256M)
+		 * [0] - pointing to the head cluster
+		 * [1] - pointing to the tail cluster
+		 */
+		__le16 delta[2];
+	} di_u __packed;		/* 8 bytes */
+} __packed;
+
+#define Z_EROFS_VLE_EXTENT_ALIGN(size) round_up(size, \
+	sizeof(struct z_erofs_vle_decompressed_index))
+
+/* dirent sorts in alphabet order, thus we can do binary search */
+struct erofs_dirent {
+	__le64 nid;     /*  0, node number */
+	__le16 nameoff; /*  8, start offset of file name */
+	__u8 file_type; /* 10, file type */
+	__u8 reserved;  /* 11, reserved */
+} __packed;
+
+/* file types used in inode_info->flags */
+enum {
+	EROFS_FT_UNKNOWN,
+	EROFS_FT_REG_FILE,
+	EROFS_FT_DIR,
+	EROFS_FT_CHRDEV,
+	EROFS_FT_BLKDEV,
+	EROFS_FT_FIFO,
+	EROFS_FT_SOCK,
+	EROFS_FT_SYMLINK,
+	EROFS_FT_MAX
+};
+
+#define EROFS_NAME_LEN      255
+
+/* check the EROFS on-disk layout strictly at compile time */
+static inline void erofs_check_ondisk_layout_definitions(void)
+{
+	BUILD_BUG_ON(sizeof(struct erofs_super_block) != 128);
+	BUILD_BUG_ON(sizeof(struct erofs_inode_v1) != 32);
+	BUILD_BUG_ON(sizeof(struct erofs_inode_v2) != 64);
+	BUILD_BUG_ON(sizeof(struct erofs_xattr_ibody_header) != 12);
+	BUILD_BUG_ON(sizeof(struct erofs_xattr_entry) != 4);
+	BUILD_BUG_ON(sizeof(struct erofs_extent_header) != 16);
+	BUILD_BUG_ON(sizeof(struct z_erofs_vle_decompressed_index) != 8);
+	BUILD_BUG_ON(sizeof(struct erofs_dirent) != 12);
+
+	BUILD_BUG_ON(BIT(Z_EROFS_VLE_DI_CLUSTER_TYPE_BITS) <
+		     Z_EROFS_VLE_CLUSTER_TYPE_MAX - 1);
+}
+
+#endif
+
Signature · 1 line
2.17.1
author
#2 lore raw

PATCH 2/13 erofs-utils: introduce erofs-utils basic headers

In reply to Gao Xiang

From: Li Guifu <[email protected]>

This patch adds basic definitions, and a simple
kernel-like list implementaion.

Signed-off-by Li Guifu <bluce.liguifu at huawei.com>
Signed-off-by Miao Xie <miaoxie at huawei.com>
Signed-off-by Fang Wei <fangwei1 at huawei.com>
Signed-off-by Gao Xiang <gaoxiang25 at huawei.com>
---
 include/erofs/defs.h     | 159 +++++++++++++++++++++++++++++++++++++++
 include/erofs/err.h      |  34 +++++++++
 include/erofs/internal.h | 126 +++++++++++++++++++++++++++++++
 include/erofs/list.h     | 110 +++++++++++++++++++++++++++
 4 files changed, 429 insertions(+)
 create mode 100644 include/erofs/defs.h
 create mode 100644 include/erofs/err.h
 create mode 100644 include/erofs/internal.h
 create mode 100644 include/erofs/list.h
include/erofs/defs.h new file +159 -0
new file mode 100644
index 0000000..111b703
@@ -0,0 +1,159 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * erofs_utils/include/erofs/defs.h
+ *
+ * Copyright (C) 2018 HUAWEI, Inc.
+ *             http://www.huawei.com/
+ * Created by Li Guifu <bluce.liguifu at huawei.com>
+ * Modified by Gao Xiang <gaoxiang25 at huawei.com>
+ */
+#ifndef __EROFS_DEFS_H
+#define __EROFS_DEFS_H
+
+#include <stddef.h>
+#include <stdint.h>
+#include <assert.h>
+#include <inttypes.h>
+
+#include <stdbool.h>
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#ifdef HAVE_LINUX_TYPES_H
+#include <linux/types.h>
+#endif
+
+/*
+ * container_of - cast a member of a structure out to the containing structure
+ * @ptr:	the pointer to the member.
+ * @type:	the type of the container struct this is embedded in.
+ * @member:	the name of the member within the struct.
+ */
+#define container_of(ptr, type, member) ({			\
+	const typeof(((type *)0)->member) *__mptr = (ptr);	\
+	(type *)((char *)__mptr - offsetof(type, member)); })
+
+typedef uint8_t         u8;
+typedef uint16_t        u16;
+typedef uint32_t        u32;
+typedef uint64_t        u64;
+
+#ifndef HAVE_LINUX_TYPES_H
+typedef u8	__u8;
+typedef u16	__u16;
+typedef u32	__u32;
+typedef u64	__u64;
+typedef u16	__le16;
+typedef u32	__le32;
+typedef u64	__le64;
+typedef u16	__be16;
+typedef u32	__be32;
+typedef u64	__be64;
+#endif
+
+typedef int8_t          s8;
+typedef int16_t         s16;
+typedef int32_t         s32;
+typedef int64_t         s64;
+
+
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+/*
+ * The host byte order is the same as network byte order,
+ * so these functions are all just identity.
+ */
+#define cpu_to_le16(x) ((__u16)(x))
+#define cpu_to_le32(x) ((__u32)(x))
+#define cpu_to_le64(x) ((__u64)(x))
+#define le16_to_cpu(x) ((__u16)(x))
+#define le32_to_cpu(x) ((__u32)(x))
+#define le64_to_cpu(x) ((__u64)(x))
+
+#else
+#if __BYTE_ORDER == __BIG_ENDIAN
+#define cpu_to_le16(x) (__builtin_bswap16(x))
+#define cpu_to_le32(x) (__builtin_bswap32(x))
+#define cpu_to_le64(x) (__builtin_bswap64(x))
+#define le16_to_cpu(x) (__builtin_bswap16(x))
+#define le32_to_cpu(x) (__builtin_bswap32(x))
+#define le64_to_cpu(x) (__builtin_bswap64(x))
+#else
+#pragma error
+#endif
+#endif
+
+#ifndef __OPTIMIZE__
+#define BUILD_BUG_ON(condition) ((void)sizeof(char[1 - 2 * !!(condition)]))
+#else
+#define BUILD_BUG_ON(condition) assert(condition)
+#endif
+
+#define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
+
+#define __round_mask(x, y)      ((__typeof__(x))((y)-1))
+#define round_up(x, y)          ((((x)-1) | __round_mask(x, y))+1)
+#define round_down(x, y)        ((x) & ~__round_mask(x, y))
+
+/* The `const' in roundup() prevents gcc-3.3 from calling __divdi3 */
+#define roundup(x, y) (					\
+{							\
+	const typeof(y) __y = y;			\
+	(((x) + (__y - 1)) / __y) * __y;		\
+}							\
+)
+#define rounddown(x, y) (				\
+{							\
+	typeof(x) __x = (x);				\
+	__x - (__x % (y));				\
+}							\
+)
+
+#define min(x, y) ({				\
+	typeof(x) _min1 = (x);			\
+	typeof(y) _min2 = (y);			\
+	(void) (&_min1 == &_min2);		\
+	_min1 < _min2 ? _min1 : _min2; })
+
+#define max(x, y) ({				\
+	typeof(x) _max1 = (x);			\
+	typeof(y) _max2 = (y);			\
+	(void) (&_max1 == &_max2);		\
+	_max1 > _max2 ? _max1 : _max2; })
+
+/*
+ * ..and if you can't take the strict types, you can specify one yourself.
+ * Or don't use min/max at all, of course.
+ */
+#define min_t(type, x, y) ({			\
+	type __min1 = (x);			\
+	type __min2 = (y);			\
+	__min1 < __min2 ? __min1: __min2; })
+
+#define max_t(type, x, y) ({			\
+	type __max1 = (x);			\
+	type __max2 = (y);			\
+	__max1 > __max2 ? __max1: __max2; })
+
+#define ARRAY_SIZE(arr)	(sizeof(arr) / sizeof((arr)[0]))
+
+#define BIT(nr)             (1UL << (nr))
+#define BIT_ULL(nr)         (1ULL << (nr))
+#define BIT_MASK(nr)        (1UL << ((nr) % BITS_PER_LONG))
+#define BIT_WORD(nr)        ((nr) / BITS_PER_LONG)
+#define BIT_ULL_MASK(nr)    (1ULL << ((nr) % BITS_PER_LONG_LONG))
+#define BIT_ULL_WORD(nr)    ((nr) / BITS_PER_LONG_LONG)
+#define BITS_PER_BYTE       8
+#define BITS_TO_LONGS(nr)   DIV_ROUND_UP(nr, BITS_PER_BYTE * sizeof(long))
+
+#define BUG_ON(cond)        assert(!(cond))
+
+#ifdef NDEBUG
+#define DBG_BUGON(condition)	((void)(condition))
+#else
+#define DBG_BUGON(condition)	BUG_ON(condition)
+#endif
+
+#endif
+
include/erofs/err.h new file +34 -0
new file mode 100644
index 0000000..fd4c873
@@ -0,0 +1,34 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * erofs_utils/include/erofs/err.h
+ *
+ * Copyright (C) 2018 HUAWEI, Inc.
+ *             http://www.huawei.com/
+ * Created by Li Guifu <bluce.liguifu at huawei.com>
+ */
+#ifndef __EROFS_ERR_H
+#define __EROFS_ERR_H
+
+#include <errno.h>
+
+#define MAX_ERRNO (4095)
+#define IS_ERR_VALUE(x)                                                        \
+	((unsigned long)(void *)(x) >= (unsigned long)-MAX_ERRNO)
+
+static inline void *ERR_PTR(long error)
+{
+	return (void *)error;
+}
+
+static inline int IS_ERR(const void *ptr)
+{
+	return IS_ERR_VALUE((unsigned long)ptr);
+}
+
+static inline long PTR_ERR(const void *ptr)
+{
+	return (long) ptr;
+}
+
+#endif
+
include/erofs/internal.h new file +126 -0

126 lines not on this page · raw message

include/erofs/list.h new file +110 -0

110 lines not on this page · raw message

Signature · 1 line
2.17.1
author
#3 lore raw

PATCH 3/13 erofs-utils: introduce miscellaneous files

In reply to Gao Xiang

From: Li Guifu <[email protected]>

This patch introduces a global configuration scheme, some functions which can print messages in different level, build scripts and .gitignore file.

Signed-off-by Li Guifu <bluce.liguifu at huawei.com>
Signed-off-by Miao Xie <miaoxie at huawei.com>
Signed-off-by Fang Wei <fangwei1 at huawei.com>
Signed-off-by Gao Xiang <gaoxiang25 at huawei.com>
---
 .gitignore             |  25 ++++++++++
 Makefile.am            |   6 +++
 VERSION                |   2 +
 autogen.sh             |   9 ++++
 configure.ac           | 107 +++++++++++++++++++++++++++++++++++++++++
 include/erofs/config.h |  31 ++++++++++++
 include/erofs/print.h  |  67 ++++++++++++++++++++++++++
 lib/Makefile.am        |   7 +++
 lib/config.c           |  36 ++++++++++++++
 9 files changed, 290 insertions(+)
 create mode 100644 .gitignore
 create mode 100644 Makefile.am
 create mode 100644 VERSION
 create mode 100755 autogen.sh
 create mode 100644 configure.ac
 create mode 100644 include/erofs/config.h
 create mode 100644 include/erofs/print.h
 create mode 100644 lib/Makefile.am
 create mode 100644 lib/config.c
.gitignore new file +25 -0
new file mode 100644
index 0000000..3a39a1e
@@ -0,0 +1,25 @@
+.*
+*~
+*.diff
+*.o
+*.la
+*.a
+*.patch
+*.rej
+
+#
+# Generated files
+#
+aclocal.m4
+autom4te.cache
+config.*
+Makefile
+Makefile.in
+config/
+m4/
+configure
+configure.scan
+libtool
+stamp-h
+stamp-h1
+
Makefile.am new file +6 -0
new file mode 100644
index 0000000..ee5fd92
@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: GPL-2.0+
+# Makefile.am
+
+ACLOCAL_AMFLAGS = -I m4
+
+SUBDIRS=lib
VERSION new file +2 -0
new file mode 100644
index 0000000..a27f297
@@ -0,0 +1,2 @@
+0.1
+2019-04-20
autogen.sh new file +9 -0
new file mode 100755
index 0000000..fdda7e1
@@ -0,0 +1,9 @@
+#!/bin/sh
+# SPDX-License-Identifier: GPL-2.0+
+
+aclocal && \
+autoheader && \
+autoconf && \
+libtoolize && \
+automake -a -c
+
configure.ac new file +107 -0

107 lines not on this page · raw message

include/erofs/config.h new file +31 -0
new file mode 100644
index 0000000..e31732b
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * erofs_utils/include/erofs/config.h
+ *
+ * Copyright (C) 2018-2019 HUAWEI, Inc.
+ *             http://www.huawei.com/
+ * Created by Li Guifu <bluce.liguifu at huawei.com>
+ */
+#ifndef __EROFS_CONFIG_H
+#define __EROFS_CONFIG_H
+
+#include "defs.h"
+
+struct erofs_configure {
+	const char *c_version;
+	int c_dbg_lvl;
+	bool c_dry_run;
+
+	/* related arguments for mkfs.erofs */
+	char *c_img_path;
+	char *c_src_path;
+};
+
+extern struct erofs_configure cfg;
+
+void erofs_init_configure(void);
+void erofs_show_config(void);
+void erofs_exit_configure(void);
+
+#endif
+
include/erofs/print.h new file +67 -0

67 lines not on this page · raw message

lib/Makefile.am new file +7 -0
new file mode 100644
index 0000000..6f1da26
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: GPL-2.0+
+# Makefile.am
+
+noinst_LTLIBRARIES = liberofs.la
+liberofs_la_SOURCES = config.c
+liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
+
lib/config.c new file +36 -0
new file mode 100644
index 0000000..6ff8e4d
@@ -0,0 +1,36 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs_utils/lib/config.c
+ *
+ * Copyright (C) 2018-2019 HUAWEI, Inc.
+ *             http://www.huawei.com/
+ * Created by Li Guifu <bluce.liguifu at huawei.com>
+ */
+#include <string.h>
+#include "erofs/print.h"
+
+struct erofs_configure cfg;
+
+void erofs_init_configure(void)
+{
+	memset(&cfg, 0, sizeof(cfg));
+
+	cfg.c_dbg_lvl  = 0;
+	cfg.c_version  = PACKAGE_VERSION;
+	cfg.c_dry_run  = false;
+}
+
+void erofs_show_config(void)
+{
+	const struct erofs_configure *c = &cfg;
+
+	erofs_dump("\tc_version:           [%8s]\n", c->c_version);
+	erofs_dump("\tc_dbg_lvl:           [%8d]\n", c->c_dbg_lvl);
+	erofs_dump("\tc_dry_run:           [%8d]\n", c->c_dry_run);
+}
+
+void erofs_exit_configure(void)
+{
+
+}
+
Signature · 1 line
2.17.1
author
#4 lore raw

PATCH 4/13 erofs-utils: add input/output functions

In reply to Gao Xiang

From: Li Guifu <[email protected]>

This patch adds definitions and functions which are mainly used for reading and writing target image files.

Signed-off-by Li Guifu <bluce.liguifu at huawei.com>
Signed-off-by Miao Xie <miaoxie at huawei.com>
Signed-off-by Fang Wei <fangwei1 at huawei.com>
Signed-off-by Gao Xiang <gaoxiang25 at huawei.com>
---
 include/erofs/io.h |  33 ++++++++++++
 lib/Makefile.am    |   2 +-
 lib/io.c           | 123 +++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 157 insertions(+), 1 deletion(-)
 create mode 100644 include/erofs/io.h
 create mode 100644 lib/io.c
include/erofs/io.h new file +33 -0
new file mode 100644
index 0000000..9471388
@@ -0,0 +1,33 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * erofs_utils/include/erofs/io.h
+ *
+ * Copyright (C) 2018-2019 HUAWEI, Inc.
+ *             http://www.huawei.com/
+ * Created by Li Guifu <bluce.liguifu at huawei.com>
+ */
+#ifndef __EROFS_IO_H
+#define __EROFS_IO_H
+
+#include <unistd.h>
+#include "internal.h"
+
+#ifndef O_BINARY
+#define O_BINARY	0
+#endif
+
+int dev_open(const char *devname);
+void dev_close(void);
+int dev_write(const void *buf, u64 offset, size_t len);
+int dev_fsync(void);
+u64 dev_length(void);
+
+static inline int blk_write(const void *buf, erofs_blk_t blkaddr,
+			    u32 nblocks)
+{
+	return dev_write(buf, blknr_to_addr(blkaddr),
+			 blknr_to_addr(nblocks));
+}
+
+#endif
+
lib/Makefile.am +1 -1
index 6f1da26..a2c1b24 100644
@@ -2,6 +2,6 @@
 # Makefile.am
 
 noinst_LTLIBRARIES = liberofs.la
-liberofs_la_SOURCES = config.c
+liberofs_la_SOURCES = config.c io.c
 liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
 
lib/io.c new file +123 -0

123 lines not on this page · raw message

Signature · 1 line
2.17.1
author
#5 lore raw

PATCH 5/13 erofs-utils: introduce buffer cache

In reply to Gao Xiang

From: Gao Xiang <[email protected]>

This patch adds a buffer cache mainly to manage incomplete metadata blocks. mkfs logic is also simplified with the help of buffer cache.

Signed-off-by Miao Xie <miaoxie at huawei.com>
[ Gao Xiang: with heavy changes. ]
Signed-off-by Gao Xiang <gaoxiang25 at huawei.com>
---
 include/erofs/cache.h | 104 +++++++++++++
 lib/Makefile.am       |   2 +-
 lib/cache.c           | 343 ++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 448 insertions(+), 1 deletion(-)
 create mode 100644 include/erofs/cache.h
 create mode 100644 lib/cache.c
include/erofs/cache.h new file +104 -0
new file mode 100644
index 0000000..108757a
@@ -0,0 +1,104 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * erofs_utils/include/erofs/cache.h
+ *
+ * Copyright (C) 2018 HUAWEI, Inc.
+ *             http://www.huawei.com/
+ * Created by Miao Xie <miaoxie at huawei.com>
+ * with heavy changes by Gao Xiang <gaoxiang25 at huawei.com>
+ */
+#ifndef __EROFS_CACHE_H
+#define __EROFS_CACHE_H
+
+#include "internal.h"
+
+struct erofs_buffer_head;
+struct erofs_buffer_block;
+
+#define DATA		0
+#define META		1
+/* including inline xattrs, extent */
+#define INODE		2
+/* shared xattrs */
+#define XATTR		3
+
+struct erofs_bhops {
+	bool (*preflush)(struct erofs_buffer_head *bh);
+	bool (*flush)(struct erofs_buffer_head *bh);
+};
+
+struct erofs_buffer_head {
+	struct list_head list;
+	struct erofs_buffer_block *block;
+
+	unsigned int off;
+	struct erofs_bhops *op;
+
+	void *fsprivate;
+};
+
+struct erofs_buffer_block {
+	struct list_head list;
+
+	erofs_blk_t blkaddr;
+	int type;
+
+	struct erofs_buffer_head buffers;
+};
+
+static inline const int get_alignsize(int type, int *type_ret)
+{
+	if (type == DATA)
+		return EROFS_BLKSIZ;
+
+	if (type == INODE) {
+		*type_ret = META;
+		return sizeof(struct erofs_inode_v1);
+	} else if (type == XATTR) {
+		*type_ret = META;
+		return sizeof(struct erofs_xattr_entry);
+	}
+
+	if (type == META)
+		return 1;
+	return -EINVAL;
+}
+
+extern struct erofs_bhops erofs_drop_directly_bhops;
+extern struct erofs_bhops erofs_skip_write_bhops;
+extern struct erofs_bhops erofs_buf_write_bhops;
+
+static inline erofs_off_t erofs_btell(struct erofs_buffer_head *bh, bool end)
+{
+	const struct erofs_buffer_block *bb = bh->block;
+
+	if (bb->blkaddr == NULL_ADDR)
+		return NULL_ADDR_UL;
+
+	return blknr_to_addr(bb->blkaddr) +
+		(end ? list_next_entry(bh, list)->off : bh->off);
+}
+
+static inline bool erofs_bh_flush_generic_end(struct erofs_buffer_head *bh)
+{
+	list_del(&bh->list);
+	free(bh);
+	return true;
+}
+
+struct erofs_buffer_head *erofs_buffer_init(void);
+int erofs_bh_balloon(struct erofs_buffer_head *bh, unsigned int incr);
+
+struct erofs_buffer_head *erofs_balloc(int type, unsigned int size,
+				       unsigned int required_ext,
+				       unsigned int inline_ext);
+struct erofs_buffer_head *erofs_battach(struct erofs_buffer_head *bh,
+					int type, int size);
+
+erofs_blk_t erofs_mapbh(struct erofs_buffer_block *bb, bool end);
+bool erofs_bflush(struct erofs_buffer_block *bb);
+
+void erofs_bdrop(struct erofs_buffer_head *bh, bool tryrevoke);
+
+#endif
+
lib/Makefile.am +1 -1
index a2c1b24..8508660 100644
@@ -2,6 +2,6 @@
 # Makefile.am
 
 noinst_LTLIBRARIES = liberofs.la
-liberofs_la_SOURCES = config.c io.c
+liberofs_la_SOURCES = config.c io.c cache.c
 liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
 
lib/cache.c new file +343 -0

343 lines not on this page · raw message

Signature · 1 line
2.17.1
author
#6 lore raw

PATCH 6/13 erofs-utils: introduce inode operations

In reply to Gao Xiang

From: Gao Xiang <[email protected]>

This patch adds core inode and dentry operations to build the target image.

Signed-off-by Li Guifu <bluce.liguifu at huawei.com>
[ Gao Xiang: with heavy changes. ]
Signed-off-by Gao Xiang <gaoxiang25 at huawei.com>
---
 include/erofs/inode.h    |  21 ++
 include/erofs/internal.h |  10 +-
 lib/Makefile.am          |   2 +-
 lib/inode.c              | 752 +++++++++++++++++++++++++++++++++++++++
 4 files changed, 782 insertions(+), 3 deletions(-)
 create mode 100644 include/erofs/inode.h
 create mode 100644 lib/inode.c
include/erofs/inode.h new file +21 -0
new file mode 100644
index 0000000..43aee93
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * erofs_utils/include/erofs/inode.h
+ *
+ * Copyright (C) 2018-2019 HUAWEI, Inc.
+ *             http://www.huawei.com/
+ * Created by Li Guifu <bluce.liguifu at huawei.com>
+ * with heavy changes by Gao Xiang <gaoxiang25 at huawei.com>
+ */
+#ifndef __EROFS_INODE_H
+#define __EROFS_INODE_H
+
+#include "erofs/internal.h"
+
+void erofs_inode_manager_init(void);
+unsigned int erofs_iput(struct erofs_inode *inode);
+erofs_nid_t erofs_lookupnid(struct erofs_inode *inode);
+struct erofs_inode *erofs_mkfs_build_tree_from_path(struct erofs_inode *parent,
+						    const char *path);
+
+#endif
include/erofs/internal.h +8 -2
index 238a076..dfa6173 100644
@@ -82,8 +82,6 @@ struct erofs_inode {
 	char i_srcpath[PATH_MAX + 1];
 
 	unsigned char data_mapping_mode;
-	bool compression_disabled;
-
 	unsigned char inode_isize;
 	/* inline tail-end packing size */
 	unsigned short idata_size;
@@ -111,6 +109,14 @@ struct erofs_dentry {
 	};
 };
 
+static inline bool is_dot_dotdot(const char *name)
+{
+	if (name[0] != '.')
+		return false;
+
+	return name[1] == '\0' || (name[1] == '.' && name[2] == '\0');
+}
+
 #include <stdio.h>
 #include <string.h>
 
lib/Makefile.am +1 -1
index 8508660..5257d71 100644
@@ -2,6 +2,6 @@
 # Makefile.am
 
 noinst_LTLIBRARIES = liberofs.la
-liberofs_la_SOURCES = config.c io.c cache.c
+liberofs_la_SOURCES = config.c io.c cache.c inode.c
 liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
 
lib/inode.c new file +752 -0

752 lines not on this page · raw message

Signature · 1 line
2.17.1
author
#7 lore raw

PATCH 7/13 erofs-utils: introduce mkfs support

In reply to Gao Xiang

From: Li Guifu <[email protected]>

This patch adds mkfs support to erofs-utils, and it's able to build uncompressed images at the moment.

Signed-off-by Li Guifu <bluce.liguifu at huawei.com>
Signed-off-by Gao Xiang <gaoxiang25 at huawei.com>
---
 Makefile.am      |   2 +-
 configure.ac     |   3 +-
 mkfs/Makefile.am |   9 +++
 mkfs/main.c      | 179 +++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 191 insertions(+), 2 deletions(-)
 create mode 100644 mkfs/Makefile.am
 create mode 100644 mkfs/main.c
Makefile.am +1 -1
index ee5fd92..d94ab73 100644
@@ -3,4 +3,4 @@
 
 ACLOCAL_AMFLAGS = -I m4
 
-SUBDIRS=lib
+SUBDIRS=lib mkfs
configure.ac +2 -1
index 9c6d8bb..49f1a7d 100644
@@ -103,5 +103,6 @@ AC_CHECK_DECL(lseek64,[AC_DEFINE(HAVE_LSEEK64_PROTOTYPE, 1,
 AC_CHECK_FUNCS([gettimeofday memset realpath strdup strerror strrchr strtoull])
 
 AC_CONFIG_FILES([Makefile
-		 lib/Makefile])
+		 lib/Makefile
+		 mkfs/Makefile])
 AC_OUTPUT
mkfs/Makefile.am new file +9 -0
new file mode 100644
index 0000000..257f864
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0+
+# Makefile.am
+
+AUTOMAKE_OPTIONS = foreign
+bin_PROGRAMS     = mkfs.erofs
+mkfs_erofs_SOURCES = main.c
+mkfs_erofs_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
+mkfs_erofs_LDADD = $(top_builddir)/lib/liberofs.la
+
mkfs/main.c new file +179 -0

179 lines not on this page · raw message

Signature · 1 line
2.17.1
author
#8 lore raw

PATCH 8/13 erofs-utils: introduce generic compression framework

In reply to Gao Xiang

From: Gao Xiang <[email protected]>

This patch adds a new flexable compression framework to user-space utilities, which is designed in order to integrate more compression algorithms easily.

Signed-off-by Gao Xiang <gaoxiang25 at huawei.com>
---
 lib/Makefile.am  |  2 +-
 lib/compressor.c | 74 ++++++++++++++++++++++++++++++++++++++++++++++++
 lib/compressor.h | 48 +++++++++++++++++++++++++++++++
 3 files changed, 123 insertions(+), 1 deletion(-)
 create mode 100644 lib/compressor.c
 create mode 100644 lib/compressor.h
lib/Makefile.am +1 -1
index 5257d71..64da708 100644
@@ -2,6 +2,6 @@
 # Makefile.am
 
 noinst_LTLIBRARIES = liberofs.la
-liberofs_la_SOURCES = config.c io.c cache.c inode.c
+liberofs_la_SOURCES = config.c io.c cache.c inode.c compressor.c
 liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
 
lib/compressor.c new file +74 -0

74 lines not on this page · raw message

lib/compressor.h new file +48 -0

48 lines not on this page · raw message

Signature · 1 line
2.17.1
author
#9 lore raw

PATCH 9/13 erofs-utils: introduce lz4/lz4hc compression algorithm

In reply to Gao Xiang

From: Gao Xiang <[email protected]>

This patch adds lz4/lz4hc compression algorithms to erofs-utils compression framework in order to enable the fixed-output size compression.

Signed-off-by Gao Xiang <gaoxiang25 at huawei.com>
---
 configure.ac           | 65 ++++++++++++++++++++++++++++++++++++++++++
 lib/Makefile.am        |  7 +++++
 lib/compressor.c       |  6 ++++
 lib/compressor.h       |  6 ++++
 lib/compressor_lz4.c   | 48 +++++++++++++++++++++++++++++++
 lib/compressor_lz4hc.c | 61 +++++++++++++++++++++++++++++++++++++++
 6 files changed, 193 insertions(+)
 create mode 100644 lib/compressor_lz4.c
 create mode 100644 lib/compressor_lz4hc.c
configure.ac +65 -0
index 49f1a7d..6cc0c60 100644
@@ -50,7 +50,22 @@ AC_DEFUN([EROFS_UTILS_PARSE_DIRECTORY],
  fi
 ])
 
+AC_ARG_ENABLE(lz4,
+   [AS_HELP_STRING([--disable-lz4], [disable LZ4 compression support @<:@default=enabled@:>@])],
+   [enable_lz4="$enableval"], [enable_lz4="yes"])
+
 # Checks for libraries.
+# Use customized LZ4 library path when specified.
+AC_ARG_WITH(lz4-incdir,
+   [AS_HELP_STRING([--with-lz4-incdir=DIR], [LZ4 include directory])], [
+   EROFS_UTILS_PARSE_DIRECTORY(["$withval"],[withval])])
+
+AC_ARG_WITH(lz4-libdir,
+   [AS_HELP_STRING([--with-lz4-libdir=DIR], [LZ4 lib directory])], [
+   EROFS_UTILS_PARSE_DIRECTORY(["$withval"],[withval])])
+
+AC_ARG_VAR([LZ4_CFLAGS], [C compiler flags for lz4])
+AC_ARG_VAR([LZ4_LIBS], [linker flags for lz4])
 
 # Checks for header files.
 AC_CHECK_HEADERS(m4_flatten([
@@ -102,7 +117,57 @@ AC_CHECK_DECL(lseek64,[AC_DEFINE(HAVE_LSEEK64_PROTOTYPE, 1,
 # Checks for library functions.
 AC_CHECK_FUNCS([gettimeofday memset realpath strdup strerror strrchr strtoull])
 
+# Configure lz4
+test -z $LZ4_LIBS && LZ4_LIBS='-llz4'
+
+if test "x$enable_lz4" = "xyes"; then
+  test -z "${with_lz4_incdir}" || LZ4_CFLAGS="-I$with_lz4_incdir $LZ4_CFLAGS"
+  test -z "${with_lz4_libdir}" || LZ4_LIBS="-L$with_lz4_libdir $LZ4_LIBS"
+
+  saved_CPPFLAGS=${CPPFLAGS}
+  CPPFLAGS="${LZ4_CFLAGS} ${CFLAGS}"
+
+  AC_CHECK_HEADERS([lz4.h],[have_lz4h="yes"], [])
+
+  if test "x${have_lz4h}" = "xyes" ; then
+    saved_LDFLAGS=${LDFLAGS}
+    LDFLAGS="-L$with_lz4_libdir ${LDFLAGS}"
+    AC_CHECK_LIB(lz4, LZ4_compress_destSize, [
+      have_lz4="yes"
+      have_lz4hc="yes"
+      AC_CHECK_LIB(lz4, LZ4_compress_HC_destSize, [], [
+        AC_CHECK_DECL(LZ4_compress_HC_destSize, [lz4_force_static="yes"],
+          [have_lz4hc="no"], [[
+#define LZ4_HC_STATIC_LINKING_ONLY (1)
+#include <lz4hc.h>
+        ]])
+      ])
+    ], [AC_MSG_ERROR([Cannot find proper lz4 version (>= 1.8.0)])])
+    LDFLAGS=${saved_LDFLAGS}
+
+    if test "x${have_lz4}" = "xyes"; then
+      AC_DEFINE([LZ4_ENABLED], [1], [Define to 1 if lz4 is enabled.])
+
+      if test "x${have_lz4hc}" = "xyes"; then
+        AC_DEFINE([LZ4HC_ENABLED], [1], [Define to 1 if lz4hc is enabled.])
+      fi
+
+      if test "x${lz4_force_static}" = "xyes"; then
+        LDFLAGS="-all-static ${LDFLAGS}"
+      else
+	test -z "${with_lz4_libdir}" || LZ4_LIBS="-R ${with_lz4_libdir} $LZ4_LIBS"
+      fi
+      LIBS="$LZ4_LIBS $LIBS"
+    fi
+  fi
+  CFLAGS=${saved_CPPFLAGS}
+fi
+
+AM_CONDITIONAL([ENABLE_LZ4], [test "x${have_lz4}" = "xyes"])
+AM_CONDITIONAL([ENABLE_LZ4HC], [test "x${have_lz4hc}" = "xyes"])
+
 AC_CONFIG_FILES([Makefile
 		 lib/Makefile
 		 mkfs/Makefile])
 AC_OUTPUT
+
lib/Makefile.am +7 -0
index 64da708..17dc9e1 100644
@@ -4,4 +4,11 @@
 noinst_LTLIBRARIES = liberofs.la
 liberofs_la_SOURCES = config.c io.c cache.c inode.c compressor.c
 liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
+if ENABLE_LZ4
+liberofs_la_CFLAGS += ${LZ4_CFLAGS}
+liberofs_la_SOURCES += compressor_lz4.c
+if ENABLE_LZ4HC
+liberofs_la_SOURCES += compressor_lz4hc.c
+endif
+endif
 
lib/compressor.c +6 -0
index cc97cfb..a6138c6 100644
@@ -39,6 +39,12 @@ int erofs_compressor_init(struct erofs_compress *c,
 			  char *alg_name)
 {
 	static struct erofs_compressor *compressors[] = {
+#if LZ4_ENABLED
+#if LZ4HC_ENABLED
+		&erofs_compressor_lz4hc,
+#endif
+		&erofs_compressor_lz4,
+#endif
 	};
 
 	int ret, i;
lib/compressor.h +6 -0
index 8ad9d11..814ef2b 100644
@@ -35,8 +35,14 @@ struct erofs_compress {
 	unsigned int destsize_alignsize;
 	unsigned int destsize_redzone_begin;
 	unsigned int destsize_redzone_end;
+
+	void *private;
 };
 
+/* list of compression algorithms */
+extern struct erofs_compressor erofs_compressor_lz4;
+extern struct erofs_compressor erofs_compressor_lz4hc;
+
 int erofs_compress_destsize(struct erofs_compress *c, int compression_level,
 			    void *src, unsigned int *srcsize,
 			    void *dst, unsigned int dstsize);
lib/compressor_lz4.c new file +48 -0

48 lines not on this page · raw message

lib/compressor_lz4hc.c new file +61 -0

61 lines not on this page · raw message

Signature · 1 line
2.17.1
author
#10 lore raw

PATCH 10/13 erofs-utils: introduce compression for regular files

In reply to Gao Xiang

From: Gao Xiang <[email protected]>

Yay! Let's try to transparent file compression
for all regular files if necessary.

Signed-off-by Gao Xiang <gaoxiang25 at huawei.com>
---
 include/erofs/compress.h |  25 ++++
 include/erofs/config.h   |   2 +
 include/erofs/internal.h |   1 +
 lib/Makefile.am          |   2 +-
 lib/compress.c           | 303 +++++++++++++++++++++++++++++++++++++++
 lib/config.c             |   1 +
 lib/inode.c              |  18 ++-
 mkfs/main.c              |  21 +++
 8 files changed, 369 insertions(+), 4 deletions(-)
 create mode 100644 include/erofs/compress.h
 create mode 100644 lib/compress.c
include/erofs/compress.h new file +25 -0
new file mode 100644
index 0000000..e0abb8f
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * erofs_utils/include/erofs/compress.h
+ *
+ * Copyright (C) 2019 HUAWEI, Inc.
+ *             http://www.huawei.com/
+ * Created by Gao Xiang <gaoxiang25 at huawei.com>
+ */
+#ifndef __EROFS_COMPRESS_H
+#define __EROFS_COMPRESS_H
+
+#include "internal.h"
+
+/* workaround for an upstream lz4 compression issue, which can crash us */
+/* #define EROFS_CONFIG_COMPR_MAX_SZ        (1024 * 1024) */
+#define EROFS_CONFIG_COMPR_MAX_SZ           (900  * 1024)
+#define EROFS_CONFIG_COMPR_MIN_SZ           (32   * 1024)
+
+int erofs_write_compressed_file(struct erofs_inode *inode);
+
+int z_erofs_compress_init(void);
+int z_erofs_compress_exit(void);
+
+#endif
+
include/erofs/config.h +2 -0
index e31732b..177017b 100644
@@ -19,6 +19,8 @@ struct erofs_configure {
 	/* related arguments for mkfs.erofs */
 	char *c_img_path;
 	char *c_src_path;
+	char *c_compr_alg_master;
+	int c_compr_level_master;
 };
 
 extern struct erofs_configure cfg;
include/erofs/internal.h +1 -0
index dfa6173..78da589 100644
@@ -94,6 +94,7 @@ struct erofs_inode {
 	struct erofs_buffer_head *bh_inline, *bh_data;
 
 	void *idata;
+	void *compressmeta;
 };
 
 #define IS_ROOT(x)	((x) == (x)->i_parent)
lib/Makefile.am +1 -1
index 17dc9e1..dea82f7 100644
@@ -2,7 +2,7 @@
 # Makefile.am
 
 noinst_LTLIBRARIES = liberofs.la
-liberofs_la_SOURCES = config.c io.c cache.c inode.c compressor.c
+liberofs_la_SOURCES = config.c io.c cache.c inode.c compress.c compressor.c
 liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
 if ENABLE_LZ4
 liberofs_la_CFLAGS += ${LZ4_CFLAGS}
lib/compress.c new file +303 -0

303 lines not on this page · raw message

lib/config.c +1 -0
index 6ff8e4d..3312c9b 100644
@@ -18,6 +18,7 @@ void erofs_init_configure(void)
 	cfg.c_dbg_lvl  = 0;
 	cfg.c_version  = PACKAGE_VERSION;
 	cfg.c_dry_run  = false;
+	cfg.c_compr_level_master = -1;
 }
 
 void erofs_show_config(void)
lib/inode.c +15 -3
index d3e5ed8..b7fcf0e 100644
@@ -17,6 +17,7 @@
 #include "erofs/inode.h"
 #include "erofs/cache.h"
 #include "erofs/io.h"
+#include "erofs/compress.h"
 
 struct erofs_sb_info sbi;
 
@@ -286,8 +287,11 @@ int erofs_write_file(struct erofs_inode *inode)
 	unsigned int nblocks, i;
 	int ret, fd;
 
-	if (erofs_file_is_compressible(inode)) {
-		/* to be implemented */
+	if (cfg.c_compr_alg_master && erofs_file_is_compressible(inode)) {
+		ret = erofs_write_compressed_file(inode);
+
+		if (!ret || ret != -ENOSPC)
+			return ret;
 	}
 
 	/* fallback to all data uncompressed */
@@ -340,7 +344,7 @@ fail:
 static bool erofs_bh_flush_write_inode(struct erofs_buffer_head *bh)
 {
 	struct erofs_inode *const inode = bh->fsprivate;
-	const erofs_off_t off = erofs_btell(bh, false);
+	erofs_off_t off = erofs_btell(bh, false);
 
 	/* let's support v1 currently */
 	struct erofs_inode_v1 v1 = {0};
@@ -379,6 +383,14 @@ static bool erofs_bh_flush_write_inode(struct erofs_buffer_head *bh)
 	if (ret)
 		return false;
 
+	if (inode->extent_isize) {
+		/* write compression metadata */
+		off = Z_EROFS_VLE_EXTENT_ALIGN(off + inode->inode_isize);
+		ret = dev_write(inode->compressmeta, off, inode->extent_isize);
+		if (ret)
+			return false;
+	}
+
 	erofs_iput(inode);
 	return erofs_bh_flush_generic_end(bh);
 }
mkfs/main.c +21 -0
index 1ed15d2..ce64ece 100644
@@ -17,6 +17,7 @@
 #include "erofs/cache.h"
 #include "erofs/inode.h"
 #include "erofs/io.h"
+#include "erofs/compress.h"
 
 #define EROFS_SUPER_END (EROFS_SUPER_OFFSET + sizeof(struct erofs_super_block))
 
@@ -25,6 +26,7 @@ static void usage(char *execpath)
 	fprintf(stderr, "%s %s\n", basename(execpath), cfg.c_version);
 	fprintf(stderr, "usage: [options] FILE DIRECTORY\n\n");
 	fprintf(stderr, "Generate erofs image from DIRECTORY to FILE, and [options] are:\n");
+	fprintf(stderr, " -zX[,Y]  X=compressor (Y=compression level, optional)\n");
 	fprintf(stderr, " -d#      set output message level to # (maximum 9)\n");
 }
 
@@ -44,6 +46,23 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
 
 	while ((opt = getopt(argc, argv, "d:z:")) != -1) {
 		switch (opt) {
+		case 'z':
+			if (!optarg) {
+				cfg.c_compr_alg_master = "(default)";
+				break;
+			}
+			/* get specified compression level */
+			for (i = 0; optarg[i] != '\0'; ++i) {
+				if (optarg[i] == ',') {
+					cfg.c_compr_level_master =
+						atoi(optarg + i + 1);
+					optarg[i] = '\0';
+					break;
+				}
+			}
+			cfg.c_compr_alg_master = strndup(optarg, i);
+			break;
+
 		case 'd':
 			cfg.c_dbg_lvl = parse_num_from_str(optarg);
 			break;
@@ -148,6 +167,7 @@ int main(int argc, char **argv)
 		goto exit;
 	}
 
+	z_erofs_compress_init();
 	erofs_inode_manager_init();
 
 	root_inode = erofs_mkfs_build_tree_from_path(NULL, cfg.c_src_path);
@@ -167,6 +187,7 @@ int main(int argc, char **argv)
 	if (!erofs_bflush(NULL))
 		err = -EIO;
 exit:
+	z_erofs_compress_exit();
 	dev_close();
 	erofs_exit_configure();
 
Signature · 1 line
2.17.1
author
#11 lore raw

PATCH 11/13 erofs-utils: propagate compressed size to its callers

In reply to Gao Xiang

From: Gao Xiang <[email protected]>

It will be later used for marking whether it can
decompress in-place.

Signed-off-by Gao Xiang <gaoxiang25 at huawei.com>
---
 lib/compress.c         | 2 +-
 lib/compressor.c       | 4 ++--
 lib/compressor_lz4.c   | 2 +-
 lib/compressor_lz4hc.c | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)
lib/compress.c +1 -1
index 205a4f2..7bfb8ce 100644
@@ -131,7 +131,7 @@ static int vle_compress_one(struct erofs_inode *inode,
 		ret = erofs_compress_destsize(h, compressionlevel,
 					      ctx->queue + ctx->head,
 					      &count, dst, EROFS_BLKSIZ);
-		if (ret) {
+		if (ret <= 0) {
 			if (ret != -EAGAIN) {
 				erofs_err("failed to compress %s: %s",
 					  inode->i_srcpath,
lib/compressor.c +2 -2
index a6138c6..570db14 100644
@@ -26,13 +26,13 @@ int erofs_compress_destsize(struct erofs_compress *c,
 
 	ret = c->alg->compress_destsize(c, compression_level,
 					src, srcsize, dst, dstsize);
-	if (ret)
+	if (ret < 0)
 		return ret;
 
 	/* check if there is enough gains to compress */
 	if (*srcsize <= dstsize * c->compress_threshold / 100)
 		return -EAGAIN;
-	return 0;
+	return ret;
 }
 
 int erofs_compressor_init(struct erofs_compress *c,
lib/compressor_lz4.c +1 -1
index eacd21c..0d33223 100644
@@ -21,7 +21,7 @@ static int lz4_compress_destsize(struct erofs_compress *c,
 	if (!rc)
 		return -EFAULT;
 	*srcsize = srcSize;
-	return 0;
+	return rc;
 }
 
 static int compressor_lz4_exit(struct erofs_compress *c)
lib/compressor_lz4hc.c +1 -1
index 3bbb754..d9413e2 100644
@@ -25,7 +25,7 @@ static int lz4hc_compress_destsize(struct erofs_compress *c,
 	if (!rc)
 		return -EFAULT;
 	*srcsize = srcSize;
-	return 0;
+	return rc;
 }
 
 static int compressor_lz4hc_exit(struct erofs_compress *c)
Signature · 1 line
2.17.1
author
#12 lore raw

PATCH 12/13 erofs-utils: add a README

In reply to Gao Xiang

From: Gao Xiang <[email protected]>

This patch adds a README and some warnings here.

Signed-off-by Gao Xiang <gaoxiang25 at huawei.com>
---
 README | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 README
README new file +11 -0
new file mode 100644
index 0000000..4d509be
@@ -0,0 +1,11 @@
+
+The new erofs-utils introduces a completely new framework,
+which is more cleaner but still under development for now.
+
+Use the old the mkfs-dev branch of erofs-utils temporarily
+to build erofs image instead:
+git clone git://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs-utils.git -b mkfs-dev
+
+Thanks,
+Gao Xiang
+
Signature · 1 line
2.17.1
author
#13 lore raw

PATCH 13/13 erofs-utils: fix potential erofs_bh_balloon failure

In reply to Gao Xiang

From: Gao Xiang <[email protected]>

erofs_write_tail_end is called at the last end of erofs_mkfs_build_tree, which is of course after traversing all sub-tree inodes.

Therefore, erofs_bh_balloon could fail if corresponding block is allocated, fix it.

Signed-off-by Gao Xiang <gaoxiang25 at huawei.com>

---
Will be wrapped in the original patch.

 lib/inode.c | 61 +++++++++++++++++++++++++++++++++++------------------
 1 file changed, 40 insertions(+), 21 deletions(-)
lib/inode.c +40 -21
index b7fcf0e..9f7a4e4 100644
@@ -399,6 +399,35 @@ static struct erofs_bhops erofs_write_inode_bhops = {
 	.flush = erofs_bh_flush_write_inode,
 };
 
+int erofs_prepare_tail_block(struct erofs_inode *inode)
+{
+	struct erofs_buffer_head *bh;
+	int ret;
+
+	if (!inode->idata_size)
+		return 0;
+
+	bh = inode->bh_data;
+	if (!bh) {
+		bh = erofs_balloc(DATA, EROFS_BLKSIZ, 0, 0);
+		if (IS_ERR(bh))
+			return PTR_ERR(bh);
+		bh->op = &erofs_skip_write_bhops;
+
+		/* get blkaddr of bh */
+		ret = erofs_mapbh(bh->block, true);
+		DBG_BUGON(ret < 0);
+		inode->u.i_blkaddr = bh->block->blkaddr;
+
+		inode->bh_data = bh;
+		return 0;
+	}
+	/* expend a block as the tail block (should be successful) */
+	ret = erofs_bh_balloon(bh, EROFS_BLKSIZ);
+	DBG_BUGON(ret);
+	return 0;
+}
+
 int erofs_prepare_inode_buffer(struct erofs_inode *inode)
 {
 	unsigned int inodesize;
@@ -421,11 +450,18 @@ int erofs_prepare_inode_buffer(struct erofs_inode *inode)
 
 	bh = erofs_balloc(INODE, inodesize, 0, inode->idata_size);
 	if (bh == ERR_PTR(-ENOSPC)) {
+		int ret;
+
 		inode->data_mapping_mode = EROFS_INODE_LAYOUT_PLAIN;
 noinline:
+		/* expend an extra block for tail-end data */
+		ret = erofs_prepare_tail_block(inode);
+		if (ret)
+			return ret;
 		bh = erofs_balloc(INODE, inodesize, 0, 0);
 		if (IS_ERR(bh))
 			return PTR_ERR(bh);
+		DBG_BUGON(inode->bh_inline);
 	} else if (IS_ERR(bh)) {
 		return PTR_ERR(bh);
 	} else if (inode->idata_size) {
@@ -485,28 +521,11 @@ int erofs_write_tail_end(struct erofs_inode *inode)
 		ibh->op = &erofs_write_inline_bhops;
 	} else {
 		int ret;
-		erofs_off_t off;
-
-		if (bh) {
-			/* expend a block (should be successful) */
-			ret = erofs_bh_balloon(bh, EROFS_BLKSIZ);
-			DBG_BUGON(ret);
-
-			erofs_mapbh(bh->block, true);
-			off = erofs_btell(bh, true) - EROFS_BLKSIZ;
-		} else {
-			bh = erofs_balloc(DATA, EROFS_BLKSIZ, 0, 0);
-			if (IS_ERR(bh))
-				return PTR_ERR(bh);
-
-			bh->op = &erofs_skip_write_bhops;
-			erofs_mapbh(bh->block, true);
-			off = erofs_btell(bh, false);
-			inode->u.i_blkaddr = erofs_blknr(off);
-			inode->bh_data = bh;
-		}
 
-		ret = dev_write(inode->idata, off, inode->idata_size);
+		erofs_mapbh(bh->block, true);
+		ret = dev_write(inode->idata,
+				erofs_btell(bh, true) - EROFS_BLKSIZ,
+				inode->idata_size);
 		if (ret)
 			return ret;
 
Signature · 1 line
2.17.1