Skip to contentSkip to conversation

erofs-utils: mkfs: introduce rebuild mode

PATCH Superseded Jingbo Xu 2 participants 14 messages Aug 23, 2023 Library code

v4 of 8 Superseded Latest is v8 →

lore
Share & tools
lore

Jingbo Xu

author
#0 lore raw

v4 cover erofs-utils: mkfs: introduce rebuild mode

v4:
- patch 2: the original two v3 patches[1][2] are merged as patch 2; also update commit message
- patch 10: newly added; introduce "--keep-whiteout=[0|1]" option for tarfs and rebuild mode controlling whether whiteout files are kept in the generated image.
- patch 9: add documentation for tarfs and rebuild mode in the usage message of mkfs.erofs

[1] https://lore.kernel.org/all/[email protected]/
[2] https://lore.kernel.org/all/[email protected]/

v1: https://lore.kernel.org/all/[email protected]/
v2: https://lore.kernel.org/all/[email protected]/
v3: https://lore.kernel.org/all/[email protected]/

-------------------------

Introduce a new rebuild mode merging multiple erofs images generated from either tarerfs index mode (--tar=i):

	mkfs.erofs --tar=i --aufs layer0.erofs layer0.tar
	...
	mkfs.erofs --tar=i --aufs layerN.erofs layerN.tar

	mkfs.erofs merge.erofs layerN.erofs ... layer0.erofs

or tarerofs non-index mode (--tar=f):

	mkfs.erofs --tar=f -Enoinline_data --aufs layer0.erofs layer0.tar
	...
	mkfs.erofs --tar=f -Enoinline_data --aufs layerN.erofs layerN.tar

	mkfs.erofs merge.erofs layerN.erofs ... layer0.erofs

Jingbo Xu (10):
erofs-utils: lib: fix dirent type of whiteout in tarerofs
erofs-utils: lib: keep erofs_init_devices in sync with kernel
erofs-utils: lib: add list_splice_tail() helper
erofs-utils: lib: make erofs_get_unhashed_chunk() global
erofs-utils: lib: add erofs_read_xattrs_from_disk() helper
erofs-utils: lib: add erofs_insert_ihash() helper
erofs-utils: lib: add erofs_rebuild_get_dentry() helper
erofs-utils: lib: add erofs_rebuild_load_tree() helper
erofs-utils: mkfs: introduce rebuild mode
erofs-utils: mkfs: add --keep-whiteout option for tarfs and rebuild
mode

 include/erofs/blobchunk.h |   2 +
 include/erofs/config.h    |   1 +
 include/erofs/inode.h     |   5 +-
 include/erofs/internal.h  |   6 +
 include/erofs/list.h      |  20 ++
 include/erofs/rebuild.h   |  21 +++
 include/erofs/xattr.h     |  33 +---
 lib/Makefile.am           |   3 +-
 lib/blobchunk.c           |   2 +-
 lib/config.c              |   1 +
 lib/inode.c               |  31 ++-
 lib/rebuild.c             | 388 ++++++++++++++++++++++++++++++++++++++
 lib/super.c               |   9 +-
 lib/tar.c                 | 119 +-----------
 lib/xattr.c               | 112 +++++++++++
 mkfs/main.c               | 221 +++++++++++++++++-----
 16 files changed, 772 insertions(+), 202 deletions(-)
 create mode 100644 include/erofs/rebuild.h
 create mode 100644 lib/rebuild.c
Signature · 1 line
2.19.1.6.gb485710b
author
#1 lore raw

PATCH v4 1/10 erofs-utils: lib: fix dirent type of whiteout in tarerofs

Set the correct dirent type for whiteout.

Fixes 95d315fd7958 ("erofs-utils: introduce tarerofs")
---
 lib/tar.c | 1 +
 1 file changed, 1 insertion(+)
lib/tar.c +1 -0
index 42590d2..328ab98 100644
@@ -694,6 +694,7 @@ new_inode:
 	if (whout) {
 		inode->i_mode = (inode->i_mode & ~S_IFMT) | S_IFCHR;
 		inode->u.i_rdev = EROFS_WHITEOUT_DEV;
+		d->type = EROFS_FT_CHRDEV;
 	} else {
 		inode->i_mode = st.st_mode;
 		if (S_ISBLK(st.st_mode) || S_ISCHR(st.st_mode))
Signature · 1 line
2.19.1.6.gb485710b
author
#3 lore raw

PATCH v4 2/10 erofs-utils: lib: keep erofs_init_devices in sync with kernel

In reply to Jingbo Xu

Keep erofs_init_devices() in sync with kernel erofs_scan_devices()[1], which scans the devtable (if any) automatically if sbi->extra_devices is not explicitly specified.

Also fix the missing le32_to_cpu() when parsing the device slot. Read and cache the number of blocks of each device for later use.

Fixes 0ce853a01123 ("erofs-utils: fuse: add multiple device support")

[1] https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=ba73eadd23d1c

---
 lib/super.c | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)
lib/super.c +6 -3
index 21dc51f..fc709fc 100644
@@ -37,7 +37,8 @@ static int erofs_init_devices(struct erofs_sb_info *sbi,
 	else
 		ondisk_extradevs = le16_to_cpu(dsb->extra_devices);
 
-	if (ondisk_extradevs != sbi->extra_devices) {
+	if (sbi->extra_devices &&
+	    ondisk_extradevs != sbi->extra_devices) {
 		erofs_err("extra devices don't match (ondisk %u, given %u)",
 			  ondisk_extradevs, sbi->extra_devices);
 		return -EINVAL;
@@ -45,6 +46,7 @@ static int erofs_init_devices(struct erofs_sb_info *sbi,
 	if (!ondisk_extradevs)
 		return 0;
 
+	sbi->extra_devices = ondisk_extradevs;
 	sbi->device_id_mask = roundup_pow_of_two(ondisk_extradevs + 1) - 1;
 	sbi->devs = calloc(ondisk_extradevs, sizeof(*sbi->devs));
 	if (!sbi->devs)
@@ -60,8 +62,9 @@ static int erofs_init_devices(struct erofs_sb_info *sbi,
 			return ret;
 		}
 
-		sbi->devs[i].mapped_blkaddr = dis.mapped_blkaddr;
-		sbi->total_blocks += dis.blocks;
+		sbi->devs[i].mapped_blkaddr = le32_to_cpu(dis.mapped_blkaddr);
+		sbi->devs[i].blocks = le32_to_cpu(dis.blocks);
+		sbi->total_blocks += sbi->devs[i].blocks;
 		pos += EROFS_DEVT_SLOT_SIZE;
 	}
 	return 0;
Signature · 1 line
2.19.1.6.gb485710b
Reviewed-by on v4 2/10
#4 lore raw

Re: [PATCH v4 02/10] erofs-utils: lib: keep erofs_init_devices in sync with kernel

Quoted 10 lines · Jingbo Xu

Keep erofs_init_devices() in sync with kernel erofs_scan_devices()[1], which scans the devtable (if any) automatically if sbi->extra_devices is not explicitly specified.

Also fix the missing le32_to_cpu() when parsing the device slot. Read and cache the number of blocks of each device for later use.

Fixes 0ce853a01123 ("erofs-utils: fuse: add multiple device support")

[1] https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=ba73eadd23d1c

Thanks,
Gao Xiang

author
#5 lore raw

PATCH v4 3/10 erofs-utils: lib: add list_splice_tail() helper

In reply to Jingbo Xu

Add list_splice_tail() helper.

---
 include/erofs/list.h | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)
include/erofs/list.h +20 -0
index 3f5da1a..d7a9fee 100644
@@ -70,6 +70,26 @@ static inline int list_empty(struct list_head *head)
 	return head->next == head;
 }
 
+static inline void __list_splice(struct list_head *list,
+		struct list_head *prev, struct list_head *next)
+{
+	struct list_head *first = list->next;
+	struct list_head *last = list->prev;
+
+	first->prev = prev;
+	prev->next = first;
+
+	last->next = next;
+	next->prev = last;
+}
+
+static inline void list_splice_tail(struct list_head *list,
+				    struct list_head *head)
+{
+	if (!list_empty(list))
+		__list_splice(list, head->prev, head);
+}
+
 #define list_entry(ptr, type, member) container_of(ptr, type, member)
 
 #define list_first_entry(ptr, type, member)                                    \
Signature · 1 line
2.19.1.6.gb485710b
author
#6 lore raw

PATCH v4 4/10 erofs-utils: lib: make erofs_get_unhashed_chunk() global

In reply to Jingbo Xu

... so that it could be called from outside blobchunk.c later.

---
 include/erofs/blobchunk.h | 2 ++
 lib/blobchunk.c           | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)
include/erofs/blobchunk.h +2 -0
index 010aee1..fb85d8e 100644
@@ -14,6 +14,8 @@ extern "C"
 
 #include "erofs/internal.h"
 
+struct erofs_blobchunk *erofs_get_unhashed_chunk(unsigned int device_id,
+		erofs_blk_t blkaddr, erofs_off_t sourceoffset);
 int erofs_blob_write_chunk_indexes(struct erofs_inode *inode, erofs_off_t off);
 int erofs_blob_write_chunked_file(struct erofs_inode *inode, int fd);
 int tarerofs_write_chunkes(struct erofs_inode *inode, erofs_off_t data_offset);
lib/blobchunk.c +1 -1
index cada5bb..07f18bd 100644
@@ -38,7 +38,7 @@ struct erofs_blobchunk erofs_holechunk = {
 };
 static LIST_HEAD(unhashed_blobchunks);
 
-static struct erofs_blobchunk *erofs_get_unhashed_chunk(unsigned int device_id,
+struct erofs_blobchunk *erofs_get_unhashed_chunk(unsigned int device_id,
 		erofs_blk_t blkaddr, erofs_off_t sourceoffset)
 {
 	struct erofs_blobchunk *chunk;
Signature · 1 line
2.19.1.6.gb485710b
author
#7 lore raw

PATCH v4 5/10 erofs-utils: lib: add erofs_read_xattrs_from_disk() helper

In reply to Jingbo Xu

Add erofs_read_xattrs_from_disk() helper reading extended attributes from disk, add checking if it's an opaque directory.

Move all xattr name related macros to xattr.c and introduce erofs_set_opaque_xattr() helper to hide all these details.

---
 include/erofs/internal.h |   1 +
 include/erofs/xattr.h    |  33 +-----------
 lib/tar.c                |   7 +--
 lib/xattr.c              | 112 +++++++++++++++++++++++++++++++++++++++
 4 files changed, 116 insertions(+), 37 deletions(-)
include/erofs/internal.h +1 -0
index 3e73eef..455a73a 100644
@@ -190,6 +190,7 @@ struct erofs_inode {
 	bool compressed_idata;
 	bool lazy_tailblock;
 	bool with_tmpfile;
+	bool opaque;
 
 	unsigned int xattr_isize;
 	unsigned int extent_isize;
include/erofs/xattr.h +2 -31
index 748442a..2c8f8dd 100644
@@ -43,37 +43,6 @@ static inline unsigned int xattrblock_offset(struct erofs_inode *vi,
 	(_size - sizeof(struct erofs_xattr_ibody_header)) / \
 	sizeof(struct erofs_xattr_entry) + 1; })
 
-#ifndef XATTR_SYSTEM_PREFIX
-#define XATTR_SYSTEM_PREFIX	"system."
-#endif
-#ifndef XATTR_SYSTEM_PREFIX_LEN
-#define XATTR_SYSTEM_PREFIX_LEN (sizeof(XATTR_SYSTEM_PREFIX) - 1)
-#endif
-#ifndef XATTR_USER_PREFIX
-#define XATTR_USER_PREFIX	"user."
-#endif
-#ifndef XATTR_USER_PREFIX_LEN
-#define XATTR_USER_PREFIX_LEN (sizeof(XATTR_USER_PREFIX) - 1)
-#endif
-#ifndef XATTR_SECURITY_PREFIX
-#define XATTR_SECURITY_PREFIX	"security."
-#endif
-#ifndef XATTR_SECURITY_PREFIX_LEN
-#define XATTR_SECURITY_PREFIX_LEN (sizeof(XATTR_SECURITY_PREFIX) - 1)
-#endif
-#ifndef XATTR_TRUSTED_PREFIX
-#define XATTR_TRUSTED_PREFIX	"trusted."
-#endif
-#ifndef XATTR_TRUSTED_PREFIX_LEN
-#define XATTR_TRUSTED_PREFIX_LEN (sizeof(XATTR_TRUSTED_PREFIX) - 1)
-#endif
-#ifndef XATTR_NAME_POSIX_ACL_ACCESS
-#define XATTR_NAME_POSIX_ACL_ACCESS "system.posix_acl_access"
-#endif
-#ifndef XATTR_NAME_POSIX_ACL_DEFAULT
-#define XATTR_NAME_POSIX_ACL_DEFAULT "system.posix_acl_default"
-#endif
-
 int erofs_scan_file_xattrs(struct erofs_inode *inode);
 int erofs_prepare_xattr_ibody(struct erofs_inode *inode);
 char *erofs_export_xattr_ibody(struct list_head *ixattrs, unsigned int size);
@@ -87,6 +56,8 @@ int erofs_xattr_prefixes_init(struct erofs_sb_info *sbi);
 
 int erofs_setxattr(struct erofs_inode *inode, char *key,
 		   const void *value, size_t size);
+int erofs_set_opaque_xattr(struct erofs_inode *inode);
+int erofs_read_xattrs_from_disk(struct erofs_inode *inode);
 
 #ifdef __cplusplus
 }
lib/tar.c +1 -6
index 328ab98..ce54d17 100644
@@ -19,11 +19,6 @@
 #include "erofs/xattr.h"
 #include "erofs/blobchunk.h"
 
-#define OVL_XATTR_NAMESPACE "overlay."
-#define OVL_XATTR_TRUSTED_PREFIX XATTR_TRUSTED_PREFIX OVL_XATTR_NAMESPACE
-#define OVL_XATTR_OPAQUE_POSTFIX "opaque"
-#define OVL_XATTR_OPAQUE OVL_XATTR_TRUSTED_PREFIX OVL_XATTR_OPAQUE_POSTFIX
-
 #define EROFS_WHITEOUT_DEV	0
 
 static char erofs_libbuf[16384];
@@ -633,7 +628,7 @@ restart:
 	} else if (opq) {
 		DBG_BUGON(d->type == EROFS_FT_UNKNOWN);
 		DBG_BUGON(!d->inode);
-		ret = erofs_setxattr(d->inode, OVL_XATTR_OPAQUE, "y", 1);
+		ret = erofs_set_opaque_xattr(d->inode);
 		goto out;
 	} else if (th.typeflag == '1') {	/* hard link cases */
 		struct erofs_dentry *d2;
lib/xattr.c +112 -0

130 lines not on this page · raw message

Signature · 1 line
2.19.1.6.gb485710b
author
#8 lore raw

PATCH v4 6/10 erofs-utils: lib: add erofs_insert_ihash() helper

In reply to Jingbo Xu

Add erofs_insert_ihash() helper inserting inode into inode hash table.

Also add prototypes of erofs_iget() and erofs_iget_by_nid() in the header file.

---
 include/erofs/inode.h |  3 +++
 lib/inode.c           | 10 +++++++---
 2 files changed, 10 insertions(+), 3 deletions(-)
include/erofs/inode.h +3 -0
index e8a5670..1c602a8 100644
@@ -25,6 +25,9 @@ u32 erofs_new_encode_dev(dev_t dev);
 unsigned char erofs_mode_to_ftype(umode_t mode);
 unsigned char erofs_ftype_to_dtype(unsigned int filetype);
 void erofs_inode_manager_init(void);
+void erofs_insert_ihash(struct erofs_inode *inode, dev_t dev, ino_t ino);
+struct erofs_inode *erofs_iget(dev_t dev, ino_t ino);
+struct erofs_inode *erofs_iget_by_nid(erofs_nid_t nid);
 unsigned int erofs_iput(struct erofs_inode *inode);
 erofs_nid_t erofs_lookupnid(struct erofs_inode *inode);
 struct erofs_dentry *erofs_d_alloc(struct erofs_inode *parent,
lib/inode.c +7 -3
index d54f84f..b967aab 100644
@@ -75,6 +75,12 @@ void erofs_inode_manager_init(void)
 		init_list_head(&inode_hashtable[i]);
 }
 
+void erofs_insert_ihash(struct erofs_inode *inode, dev_t dev, ino_t ino)
+{
+	list_add(&inode->i_hash,
+		 &inode_hashtable[(ino ^ dev) % NR_INODE_HASHTABLE]);
+}
+
 /* get the inode from the (source) inode # */
 struct erofs_inode *erofs_iget(dev_t dev, ino_t ino)
 {
@@ -976,9 +982,7 @@ static int erofs_fill_inode(struct erofs_inode *inode, struct stat *st,
 		inode->inode_isize = sizeof(struct erofs_inode_compact);
 	}
 
-	list_add(&inode->i_hash,
-		 &inode_hashtable[(st->st_ino ^ st->st_dev) %
-				  NR_INODE_HASHTABLE]);
+	erofs_insert_ihash(inode, st->st_dev, st->st_ino);
 	return 0;
 }
 
Signature · 1 line
2.19.1.6.gb485710b
author
#9 lore raw

PATCH v4 7/10 erofs-utils: lib: add erofs_rebuild_get_dentry() helper

In reply to Jingbo Xu

Rename tarerofs_get_dentry() to erofs_rebuild_get_dentry().

Also make whout and 'opq' parameter optional when aufs is false.

---
 include/erofs/rebuild.h |  19 +++++++
 lib/Makefile.am         |   3 +-
 lib/rebuild.c           | 117 ++++++++++++++++++++++++++++++++++++++++
 lib/tar.c               | 109 ++-----------------------------------
 4 files changed, 141 insertions(+), 107 deletions(-)
 create mode 100644 include/erofs/rebuild.h
 create mode 100644 lib/rebuild.c
include/erofs/rebuild.h new file +19 -0
new file mode 100644
index 0000000..92873c9
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */
+#ifndef __EROFS_REBUILD_H
+#define __EROFS_REBUILD_H
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+#include "internal.h"
+
+struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd,
+		char *path, bool aufs, bool *whout, bool *opq);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
lib/Makefile.am +2 -1
index 7a5dc03..4bcc09b 100644
@@ -24,6 +24,7 @@ noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \
       $(top_srcdir)/include/erofs/xattr.h \
       $(top_srcdir)/include/erofs/compress_hints.h \
       $(top_srcdir)/include/erofs/fragments.h \
+      $(top_srcdir)/include/erofs/rebuild.h \
       $(top_srcdir)/lib/liberofs_private.h
 
 noinst_HEADERS += compressor.h
@@ -31,7 +32,7 @@ liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \
 		      namei.c data.c compress.c compressor.c zmap.c decompress.c \
 		      compress_hints.c hashmap.c sha256.c blobchunk.c dir.c \
 		      fragments.c rb_tree.c dedupe.c uuid_unparse.c uuid.c tar.c \
-		      block_list.c
+		      block_list.c rebuild.c
 
 liberofs_la_CFLAGS = -Wall ${libuuid_CFLAGS} -I$(top_srcdir)/include
 if ENABLE_LZ4
lib/rebuild.c new file +117 -0

117 lines not on this page · raw message

lib/tar.c +3 -106

139 lines not on this page · raw message

Signature · 1 line
2.19.1.6.gb485710b
author
#10 lore raw

PATCH v4 8/10 erofs-utils: lib: add erofs_rebuild_load_tree() helper

In reply to Jingbo Xu

Add erofs_rebuild_load_tree() helper loading inode tree from given erofs image, and making it merged into a given inode tree in an overlayfs like model.

Since we need to read the content of the symlink file from disk when loading tree, add dependency on zlib_LIBS for mkfs.erofs.

Also rename tarerofs_dump_tree() to erofs_rebuild_dump_tree(), so that it could also be called from rebuild mode.

---
 include/erofs/inode.h    |   2 +-
 include/erofs/internal.h |   1 +
 include/erofs/rebuild.h  |   2 +
 lib/inode.c              |   4 +-
 lib/rebuild.c            | 271 +++++++++++++++++++++++++++++++++++++++
 mkfs/main.c              |   2 +-
 6 files changed, 278 insertions(+), 4 deletions(-)
include/erofs/inode.h +1 -1
index 1c602a8..fe9dda2 100644
@@ -32,7 +32,7 @@ unsigned int erofs_iput(struct erofs_inode *inode);
 erofs_nid_t erofs_lookupnid(struct erofs_inode *inode);
 struct erofs_dentry *erofs_d_alloc(struct erofs_inode *parent,
 				   const char *name);
-int tarerofs_dump_tree(struct erofs_inode *dir);
+int erofs_rebuild_dump_tree(struct erofs_inode *dir);
 int erofs_init_empty_dir(struct erofs_inode *dir);
 struct erofs_inode *erofs_new_inode(void);
 struct erofs_inode *erofs_mkfs_build_tree_from_path(const char *path);
include/erofs/internal.h +1 -0
index 455a73a..457c3dd 100644
@@ -108,6 +108,7 @@ struct erofs_sb_info {
 
 	int devfd;
 	u64 devsz;
+	dev_t dev;
 	unsigned int nblobs;
 	unsigned int blobfd[256];
 };
include/erofs/rebuild.h +2 -0
index 92873c9..3ac074c 100644
@@ -12,6 +12,8 @@ extern "C"
 struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd,
 		char *path, bool aufs, bool *whout, bool *opq);
 
+int erofs_rebuild_load_tree(struct erofs_inode *root, struct erofs_sb_info *sbi);
+
 #ifdef __cplusplus
 }
 #endif
lib/inode.c +2 -2
index b967aab..1cd69ec 100644
@@ -1323,7 +1323,7 @@ struct erofs_inode *erofs_mkfs_build_special_from_fd(int fd, const char *name)
 	return inode;
 }
 
-int tarerofs_dump_tree(struct erofs_inode *dir)
+int erofs_rebuild_dump_tree(struct erofs_inode *dir)
 {
 	struct erofs_dentry *d;
 	unsigned int nr_subdirs;
@@ -1391,7 +1391,7 @@ int tarerofs_dump_tree(struct erofs_inode *dir)
 			continue;
 
 		inode = erofs_igrab(d->inode);
-		ret = tarerofs_dump_tree(inode);
+		ret = erofs_rebuild_dump_tree(inode);
 		dir->i_nlink += (erofs_mode_to_ftype(inode->i_mode) == EROFS_FT_DIR);
 		erofs_iput(inode);
 		if (ret)
lib/rebuild.c +271 -0

283 lines not on this page · raw message

mkfs/main.c +1 -1
index c03a7a8..628af59 100644
@@ -946,7 +946,7 @@ int main(int argc, char **argv)
 		if (err < 0)
 			goto exit;
 
-		err = tarerofs_dump_tree(root_inode);
+		err = erofs_rebuild_dump_tree(root_inode);
 		if (err < 0)
 			goto exit;
 	}
Signature · 1 line
2.19.1.6.gb485710b
#11 lore raw

Re: [PATCH v4 08/10] erofs-utils: lib: add erofs_rebuild_load_tree() helper

Quoted 104 lines · Jingbo Xu

Add erofs_rebuild_load_tree() helper loading inode tree from given erofs image, and making it merged into a given inode tree in an overlayfs like model.

Since we need to read the content of the symlink file from disk when loading tree, add dependency on zlib_LIBS for mkfs.erofs.

Also rename tarerofs_dump_tree() to erofs_rebuild_dump_tree(), so that it could also be called from rebuild mode.

---
  include/erofs/inode.h    |   2 +-
  include/erofs/internal.h |   1 +
  include/erofs/rebuild.h  |   2 +
  lib/inode.c              |   4 +-
  lib/rebuild.c            | 271 +++++++++++++++++++++++++++++++++++++++
  mkfs/main.c              |   2 +-
  6 files changed, 278 insertions(+), 4 deletions(-)
include/erofs/inode.h +1 -1
index 1c602a8..fe9dda2 100644
@@ -32,7 +32,7 @@ unsigned int erofs_iput(struct erofs_inode *inode);
  erofs_nid_t erofs_lookupnid(struct erofs_inode *inode);
  struct erofs_dentry *erofs_d_alloc(struct erofs_inode *parent,
  				   const char *name);
-int tarerofs_dump_tree(struct erofs_inode *dir);
+int erofs_rebuild_dump_tree(struct erofs_inode *dir);
  int erofs_init_empty_dir(struct erofs_inode *dir);
  struct erofs_inode *erofs_new_inode(void);
  struct erofs_inode *erofs_mkfs_build_tree_from_path(const char *path);
include/erofs/internal.h +1 -0
index 455a73a..457c3dd 100644
@@ -108,6 +108,7 @@ struct erofs_sb_info {
 
  	int devfd;
  	u64 devsz;
+	dev_t dev;
  	unsigned int nblobs;
  	unsigned int blobfd[256];
  };
include/erofs/rebuild.h +2 -0
index 92873c9..3ac074c 100644
@@ -12,6 +12,8 @@ extern "C"
  struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd,
  		char *path, bool aufs, bool *whout, bool *opq);
 
+int erofs_rebuild_load_tree(struct erofs_inode *root, struct erofs_sb_info *sbi);
+
  #ifdef __cplusplus
  }
  #endif
lib/inode.c +2 -2
index b967aab..1cd69ec 100644
@@ -1323,7 +1323,7 @@ struct erofs_inode *erofs_mkfs_build_special_from_fd(int fd, const char *name)
  	return inode;
  }
 
-int tarerofs_dump_tree(struct erofs_inode *dir)
+int erofs_rebuild_dump_tree(struct erofs_inode *dir)
  {
  	struct erofs_dentry *d;
  	unsigned int nr_subdirs;
@@ -1391,7 +1391,7 @@ int tarerofs_dump_tree(struct erofs_inode *dir)
  			continue;
 
  		inode = erofs_igrab(d->inode);
-		ret = tarerofs_dump_tree(inode);
+		ret = erofs_rebuild_dump_tree(inode);
  		dir->i_nlink += (erofs_mode_to_ftype(inode->i_mode) == EROFS_FT_DIR);
  		erofs_iput(inode);
  		if (ret)
lib/rebuild.c +9 -0
index 7aaa071..e2525fe 100644
@@ -3,9 +3,18 @@
  #include <unistd.h>
  #include <stdlib.h>
  #include <string.h>
+#include <sys/stat.h>
+#include <config.h>
+#if defined(HAVE_SYS_SYSMACROS_H)
+#include <sys/sysmacros.h>
+#endif
  #include "erofs/print.h"
  #include "erofs/inode.h"
  #include "erofs/rebuild.h"
+#include "erofs/io.h"
+#include "erofs/dir.h"
+#include "erofs/xattr.h"
+#include "erofs/blobchunk.h"
  #include "erofs/internal.h"
 
  #ifdef HAVE_LINUX_AUFS_TYPE_H

lib/rebuild.c struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd, · line 129

Quoted 6 lines
lib/rebuild.c +3 -0
@@ -115,3 +124,265 @@ struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd,
  	}
  	return d;
  }
+
+static int erofs_rebuild_fill_inode_map(struct erofs_inode *inode,
+					struct erofs_inode *vi)

erofs_rebuilt_fixup_inode_indexes? Also could you use (dst, src) instead of (inode, vi)?

Quoted 60 lines
lib/rebuild.c +60 -0
+{
+	int ret;
+	struct erofs_sb_info *sbi = vi->sbi;
+	unsigned int count, unit, chunkbits, i;
+	unsigned int deviceid = inode->dev;
+	erofs_off_t chunksize;
+	erofs_blk_t blkaddr;
+	struct erofs_inode_chunk_index *idx;
+
+	/* TODO: fill data map in other layouts */
+	if (vi->datalayout != EROFS_INODE_CHUNK_BASED &&
+	    vi->datalayout != EROFS_INODE_FLAT_PLAIN) {
+		erofs_err("unsupported datalayout %d", vi->datalayout);
+		return -EOPNOTSUPP;
+	}
+
+	if (sbi->extra_devices) {
+		chunkbits = vi->u.chunkbits;
+		inode->u.chunkformat = vi->u.chunkformat;
+	} else {
+		chunkbits = ilog2(inode->i_size - 1) + 1;
+		if (chunkbits < sbi->blkszbits)
+			chunkbits = sbi->blkszbits;
+		if (chunkbits - sbi->blkszbits > EROFS_CHUNK_FORMAT_BLKBITS_MASK)
+			chunkbits = EROFS_CHUNK_FORMAT_BLKBITS_MASK + sbi->blkszbits;
+		inode->u.chunkformat = EROFS_CHUNK_FORMAT_INDEXES;
+		inode->u.chunkformat |= chunkbits - sbi->blkszbits;
+	}
+	chunksize = 1ULL << chunkbits;
+	count = DIV_ROUND_UP(inode->i_size, chunksize);
+
+	unit = sizeof(struct erofs_inode_chunk_index);
+	inode->extent_isize = count * unit;
+	idx = malloc(max(sizeof(*idx), sizeof(void *)));
+	if (!idx)
+		return -ENOMEM;
+	inode->chunkindexes = idx;
+
+	for (i = 0; i < count; i++) {
+		struct erofs_blobchunk *chunk;
+		struct erofs_map_blocks map = {
+			.index = UINT_MAX,
+		};
+
+		map.m_la = i << chunkbits;
+		ret = erofs_map_blocks(vi, &map, 0);
+		if (ret)
+			goto err;
+
+		blkaddr = erofs_blknr(sbi, map.m_pa);
+		chunk = erofs_get_unhashed_chunk(deviceid, blkaddr, 0);
+		if (IS_ERR(chunk)) {
+			ret = PTR_ERR(chunk);
+			goto err;
+		}
+		*(void **)idx++ = chunk;
+		erofs_dbg("\t%s: chunk %d (deviceid %u, blkaddr %u)",
+			inode->i_srcpath, i, deviceid, blkaddr);
+
+	}

lib/rebuild.c struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd, · line 199

Quoted 10 lines
lib/rebuild.c +10 -0
+	inode->datalayout = EROFS_INODE_CHUNK_BASED;
+	return 0;
+err:
+	free(idx);
+	inode->chunkindexes = NULL;
+	return ret;
+}
+
+static int erofs_rebuild_fill_inode(struct erofs_inode *inode,
+				    struct erofs_inode *vi)

same here.

Quoted 52 lines
lib/rebuild.c +52 -0
+{
+	int ret = 0;
+
+	inode->i_srcpath = strdup(vi->i_srcpath);
+	inode->i_mode = vi->i_mode;
+	inode->i_uid = vi->i_uid;
+	inode->i_gid = vi->i_gid;
+	inode->i_mtime = vi->i_mtime;
+	inode->i_ino[1] = vi->nid;
+	inode->i_nlink = 1;
+	inode->opaque = vi->opaque;
+	list_splice_tail(&vi->i_xattrs, &inode->i_xattrs);
+
+	switch (inode->i_mode & S_IFMT) {
+	case S_IFCHR:
+	case S_IFBLK:
+	case S_IFIFO:
+	case S_IFSOCK:
+		inode->u.i_rdev = erofs_new_encode_dev(vi->u.i_rdev);
+		inode->i_size = 0;
+		erofs_dbg("\tdev: %d %d", major(vi->u.i_rdev),
+			  minor(vi->u.i_rdev));
+		break;
+	case S_IFDIR:
+		inode->i_size = 0;
+		ret = erofs_init_empty_dir(inode);
+		break;
+	case S_IFLNK:
+		inode->i_size = vi->i_size;
+		inode->i_link = malloc(inode->i_size + 1);
+		ret = erofs_pread(vi, inode->i_link, inode->i_size, 0);
+		erofs_dbg("\tsymlink: %s -> %s", inode->i_srcpath, inode->i_link);
+		break;
+	case S_IFREG:
+		inode->i_size = vi->i_size;
+		if (inode->i_size)
+			ret = erofs_rebuild_fill_inode_map(inode, vi);
+		else
+			inode->u.i_blkaddr = NULL_ADDR;
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+	return ret;
+}
+
+struct erofs_rebuild_dir_context {
+	struct erofs_dir_context ctx;
+	struct erofs_inode *root;
+	dev_t dev;
+};

lib/rebuild.c struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd, · line 261

Quoted 10 lines
lib/rebuild.c +10 -0
+
+static int erofs_rebuild_parse_inode(struct erofs_rebuild_dir_context *rctx,
+				     struct erofs_inode *vi)
+{
+	struct erofs_inode *inode;
+	struct erofs_dentry *d;
+	int ret;
+
+	d = erofs_rebuild_get_dentry(rctx->root, vi->i_srcpath,
+				     false, NULL, NULL);

I'm not sure if it's effective, I think we should just record parent nid and lookup subdir instead.

Quoted 10 lines
lib/rebuild.c +10 -0
+	if (IS_ERR(d))
+		return PTR_ERR(d);
+
+	if (d->type != EROFS_FT_UNKNOWN) {
+		inode = d->inode;
+		DBG_BUGON((inode->i_mode & S_IFMT) != (vi->i_mode & S_IFMT));
+		if (!S_ISDIR(inode->i_mode) || inode->opaque) {
+			erofs_dbg("file %s: %s (%d) exists",
+				  vi->i_srcpath, inode->i_srcpath,
+				  erofs_mode_to_ftype(inode->i_mode));

lib/rebuild.c struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd, · line 281

Quoted 10 lines
lib/rebuild.c +10 -0
+			return 0;
+		}
+		erofs_dbg("dir %s: %s merging", vi->i_srcpath, inode->i_srcpath);
+	} else {
+		erofs_dbg("loading file: %s (%d) (nid %llu)", vi->i_srcpath,
+			  erofs_mode_to_ftype(vi->i_mode), vi->nid);
+		if (S_ISREG(vi->i_mode) && vi->i_nlink > 1 &&
+		    (inode = erofs_iget(rctx->dev, vi->nid))) {
+			/* hardlink file */
+			if (S_ISDIR(inode->i_mode)) {

why this can happen?

Quoted 37 lines
lib/rebuild.c +37 -0
+				erofs_dbg("hardlink directory not supported");
+				ret = -EISDIR;
+				goto put_inode;
+			}
+			inode->i_nlink++;
+			erofs_dbg("\thardlink: %s -> %s",
+				  vi->i_srcpath, inode->i_srcpath);
+		} else {
+			inode = erofs_new_inode();
+			if (IS_ERR(inode))
+				return PTR_ERR(inode);
+
+			ret = erofs_read_xattrs_from_disk(vi);
+			if (ret)
+				goto put_inode;
+
+			inode->i_parent = d->inode;
+			inode->dev = rctx->dev;
+			ret = erofs_rebuild_fill_inode(inode, vi);
+			if (ret)
+				goto put_inode;
+
+			erofs_insert_ihash(inode, rctx->dev, vi->nid);
+		}
+
+		d->inode = inode;
+		d->type = erofs_mode_to_ftype(inode->i_mode);
+	}
+
+	ret = 0;
+	if (S_ISDIR(vi->i_mode)) {
+		struct erofs_rebuild_dir_context nctx = *rctx;
+		nctx.ctx.dir = vi;
+		ret = erofs_iterate_dir(&nctx.ctx, false);
+	}
+	return ret;
+

lib/rebuild.c struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd, · line 328

Quoted 10 lines
lib/rebuild.c +10 -0
+put_inode:
+	erofs_iput(inode);
+	return ret;
+}
+
+static int erofs_rebuild_dirent_iter(struct erofs_dir_context *ctx)
+{
+	struct erofs_rebuild_dir_context *rctx = (void *)ctx;
+	struct erofs_inode *dir = ctx->dir;
+	struct erofs_inode vi = {};

why we cannot reuse the new allocated inode -- erofs_new_inode()?

Quoted 22 lines
lib/rebuild.c +22 -0
+	char *path;
+	int ret;
+
+	if (ctx->dot_dotdot)
+		return 0;
+
+	erofs_dbg("file %s/%.*s", dir->i_srcpath, ctx->de_namelen, ctx->dname);
+	vi.nid = ctx->de_nid;
+	vi.sbi = dir->sbi;
+	ret = erofs_read_inode_from_disk(&vi);
+	if (ret)
+		return ret;
+
+	ret = asprintf(&path, "%s/%.*s", dir->i_srcpath, ctx->de_namelen,
+		       ctx->dname);
+	if (ret < 0)
+		return ret;
+	vi.i_srcpath = path;
+
+	ret = erofs_rebuild_parse_inode(rctx, &vi);
+	free(path);
+	return ret;

lib/rebuild.c struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd, · line 360

Quoted 10 lines
lib/rebuild.c +10 -0
+}
+
+int erofs_rebuild_load_tree(struct erofs_inode *root, struct erofs_sb_info *sbi)
+{
+	struct erofs_inode inode = {};
+	struct erofs_rebuild_dir_context ctx;
+	int ret;
+
+	if (!sbi->devname) {
+		erofs_err("please open the device first");
		erofs_err("failed to find a device for rebuilding");

lib/rebuild.c struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd, · line 366

Quoted 6 lines
lib/rebuild.c +6 -0
+		return -EINVAL;
+	}
+
+	ret = erofs_read_superblock(sbi);
+	if (ret) {
+		erofs_err("failed to read superblock of img %s", sbi->devname);

let's avoid using abbreviation "img"

"failed to read superblock of %s"

lib/rebuild.c struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd, · line 374

Quoted 8 lines
lib/rebuild.c +8 -0
+		return ret;
+	}
+
+	inode.nid = sbi->root_nid;
+	inode.sbi = sbi;
+	ret = erofs_read_inode_from_disk(&inode);
+	if (ret) {
+		erofs_err("failed to read root inode of img %s", sbi->devname);

"failed to read root inode of %s"

Thanks,
Gao Xiang

author
#12 lore raw

PATCH v4 9/10 erofs-utils: mkfs: introduce rebuild mode

In reply to Jingbo Xu

Introduce a new rebuild mode merging multiple erofs images generated from either tarerfs index mode (--tar=i):

	mkfs.erofs --tar=i --aufs layer0.erofs layer0.tar
	...
	mkfs.erofs --tar=i --aufs layerN.erofs layerN.tar

	mkfs.erofs merge.erofs layerN.erofs ... layer0.erofs

or tarerofs non-index mode (--tar=f):

	mkfs.erofs --tar=f -Enoinline_data --aufs layer0.erofs layer0.tar
	...
	mkfs.erofs --tar=f -Enoinline_data --aufs layerN.erofs layerN.tar

	mkfs.erofs merge.erofs layerN.erofs ... layer0.erofs

When calling rebuild mode, the top most layer must be specified firstly in the cmdline, while the bottom most layer i.e. layer0 the lastly.

The rebuild mode doesn't support flat inline datalayout yet.

---
 include/erofs/internal.h |   2 +
 mkfs/main.c              | 207 +++++++++++++++++++++++++++++++--------
 2 files changed, 166 insertions(+), 43 deletions(-)
include/erofs/internal.h +2 -0
index 457c3dd..3596689 100644
@@ -111,6 +111,8 @@ struct erofs_sb_info {
 	dev_t dev;
 	unsigned int nblobs;
 	unsigned int blobfd[256];
+
+	struct list_head list;
 };
 
 /* make sure that any user of the erofs headers has atleast 64bit off_t type */
mkfs/main.c +164 -43

292 lines not on this page · raw message

Signature · 1 line
2.19.1.6.gb485710b
author
#13 lore raw

PATCH v4 10/10 erofs-utils: mkfs: add --keep-whiteout option for tarfs and rebuild mode

In reply to Jingbo Xu

Add "--keep-whiteout=[0|1]" option for tarfs and rebuild mode, controlling whether whiteout files are kept in the generated image.

This option is enabled by default for both tarfs and rebuild mode, i.e. as if "--keep-whiteout=1" is specified by default.

To hide whiteout files, specify "--keep-whiteout=0" explicitly.

---
 include/erofs/config.h   |  1 +
 include/erofs/internal.h |  2 ++
 lib/config.c             |  1 +
 lib/inode.c              | 17 +++++++++++++++--
 lib/tar.c                |  2 --
 mkfs/main.c              | 12 ++++++++++++
 6 files changed, 31 insertions(+), 4 deletions(-)
include/erofs/config.h +1 -0
index 8f52d2c..91c9d7d 100644
@@ -53,6 +53,7 @@ struct erofs_configure {
 	bool c_ignore_mtime;
 	bool c_showprogress;
 	bool c_extra_ea_name_prefixes;
+	bool c_keep_whiteout;
 
 #ifdef HAVE_LIBSELINUX
 	struct selabel_handle *sehnd;
include/erofs/internal.h +2 -0
index 3596689..7cc3864 100644
@@ -426,6 +426,8 @@ static inline u32 erofs_crc32c(u32 crc, const u8 *in, size_t len)
 	return crc;
 }
 
+#define EROFS_WHITEOUT_DEV	0
+
 #ifdef __cplusplus
 }
 #endif
lib/config.c +1 -0
index a3235c8..ae24a17 100644
@@ -33,6 +33,7 @@ void erofs_init_configure(void)
 	cfg.c_pclusterblks_max = 1;
 	cfg.c_pclusterblks_def = 1;
 	cfg.c_max_decompressed_extent_bytes = -1;
+	cfg.c_keep_whiteout = true;
 }
 
 void erofs_show_config(void)
lib/inode.c +15 -2
index 1cd69ec..c14c30d 100644
@@ -1323,9 +1323,14 @@ struct erofs_inode *erofs_mkfs_build_special_from_fd(int fd, const char *name)
 	return inode;
 }
 
+static bool erofs_inode_is_whiteout(struct erofs_inode *inode)
+{
+	return S_ISCHR(inode->i_mode) && inode->u.i_rdev == EROFS_WHITEOUT_DEV;
+}
+
 int erofs_rebuild_dump_tree(struct erofs_inode *dir)
 {
-	struct erofs_dentry *d;
+	struct erofs_dentry *d, *n;
 	unsigned int nr_subdirs;
 	int ret;
 
@@ -1369,8 +1374,16 @@ int erofs_rebuild_dump_tree(struct erofs_inode *dir)
 	}
 
 	nr_subdirs = 0;
-	list_for_each_entry(d, &dir->i_subdirs, d_child)
+	list_for_each_entry_safe(d, n, &dir->i_subdirs, d_child) {
+		if (!cfg.c_keep_whiteout && erofs_inode_is_whiteout(d->inode)) {
+			erofs_dbg("skip whiteout %s", d->inode->i_srcpath);
+			list_del(&d->d_child);
+			erofs_d_invalidate(d);
+			free(d);
+			continue;
+		}
 		++nr_subdirs;
+	}
 
 	ret = erofs_prepare_dir_layout(dir, nr_subdirs);
 	if (ret)
lib/tar.c +0 -2
index cf3497f..26830f4 100644
@@ -13,8 +13,6 @@
 #include "erofs/blobchunk.h"
 #include "erofs/rebuild.h"
 
-#define EROFS_WHITEOUT_DEV	0
-
 static char erofs_libbuf[16384];
 
 struct tar_header {
mkfs/main.c +12 -0
index 961dfba..e5f3485 100644
@@ -57,6 +57,7 @@ static struct option long_options[] = {
 	{"gid-offset", required_argument, NULL, 17},
 	{"tar", optional_argument, NULL, 20},
 	{"aufs", no_argument, NULL, 21},
+	{"keep-whiteout", required_argument, NULL, 22},
 	{"mount-point", required_argument, NULL, 512},
 	{"xattr-prefix", required_argument, NULL, 19},
 #ifdef WITH_ANDROID
@@ -121,6 +122,7 @@ static void usage(void)
 	      " --preserve-mtime      keep per-file modification time strictly\n"
 	      " --aufs                replace aufs special files with overlayfs metadata\n"
 	      " --tar=[fi]            generate an image from tarball(s)\n"
+	      " --keep-whiteout=[01]  keep whiteout in tarfs or rebuild mode\n"
 	      " --quiet               quiet execution (do not write anything to standard output.)\n"
 #ifndef NDEBUG
 	      " --random-pclusterblks randomize pclusterblks for big pcluster (debugging only)\n"
@@ -521,6 +523,16 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
 		case 21:
 			erofstar.aufs = true;
 			break;
+		case 22:
+			if (!strcmp(optarg, "0"))
+				cfg.c_keep_whiteout = false;
+			else if (!strcmp(optarg, "1"))
+				cfg.c_keep_whiteout = true;
+			else {
+				erofs_err("invalid keep-whiteout %s", optarg);
+				return -EINVAL;
+			}
+			break;
 		case 1:
 			usage();
 			exit(0);
Signature · 1 line
2.19.1.6.gb485710b