Skip to contentSkip to conversation

erofs-utils: mkfs: introduce rebuild mode

PATCH Superseded Jingbo Xu 2 participants 15 messages Sep 5, 2023 – Sep 7, 2023 Library code

v6 of 8 Superseded Latest is v8 →

lore
Share & tools
lore

Jingbo Xu

author
#0 lore raw

v6 cover erofs-utils: mkfs: introduce rebuild mode

v6:
- newly add cleanup patches (patch 1 and 9)
- patch 8: improve the code of parsing multiple sources with "do {} while" loop
- patch 10: newly added to work around the potential issue of exposing whiteouts in the overlayfs mount by setting origin xattr on parent directory of whiteouts
- patch 11: the former "--keep-whiteout" option is renamed to "--overlay-strip" option; apart from whiteouts, origin xattrs will also be stripped when the option is enabled

[1] https://lore.kernel.org/all/[email protected]/
[2] https://lore.kernel.org/all/[email protected]/

v1: https://lore.kernel.org/all/[email protected]/
v2: https://lore.kernel.org/all/[email protected]/
v3: https://lore.kernel.org/all/[email protected]/
v4: https://lore.kernel.org/all/[email protected]/
v5: https://lore.kernel.org/all/[email protected]/

-------------------------

Introduce a new rebuild mode merging multiple erofs images generated from either tarerfs index mode (--tar=i):

	mkfs.erofs --tar=i --aufs layer0.erofs layer0.tar
	...
	mkfs.erofs --tar=i --aufs layerN.erofs layerN.tar

	mkfs.erofs merge.erofs layerN.erofs ... layer0.erofs

or tarerofs non-index mode (--tar=f):

	mkfs.erofs --tar=f -Enoinline_data --aufs layer0.erofs layer0.tar
	...
	mkfs.erofs --tar=f -Enoinline_data --aufs layerN.erofs layerN.tar

	mkfs.erofs merge.erofs layerN.erofs ... layer0.erofs

Jingbo Xu (11):
erofs-utils: lib: remove unneeded NULL checking for returned item
erofs-utils: lib: add list_splice_tail() helper
erofs-utils: lib: make erofs_get_unhashed_chunk() global
erofs-utils: lib: add erofs_read_xattrs_from_disk() helper
erofs-utils: lib: add erofs_insert_ihash() helper
erofs-utils: lib: add erofs_rebuild_get_dentry() helper
erofs-utils: lib: add erofs_rebuild_load_tree() helper
erofs-utils: mkfs: introduce rebuild mode
erofs-utils: lib: add erofs_inode_is_whiteout() helper
erofs-utils: lib: set origin xattr on parent directory of whiteout
erofs-utils: mkfs: add --overlay-strip option for tarfs and rebuild
mode

 include/erofs/blobchunk.h |   2 +
 include/erofs/config.h    |   1 +
 include/erofs/inode.h     |   5 +-
 include/erofs/internal.h  |  11 +
 include/erofs/list.h      |  20 ++
 include/erofs/rebuild.h   |  21 ++
 include/erofs/xattr.h     |  36 +---
 lib/Makefile.am           |   3 +-
 lib/blobchunk.c           |   2 +-
 lib/inode.c               |  26 ++-
 lib/rebuild.c             | 409 ++++++++++++++++++++++++++++++++++++++
 lib/tar.c                 | 129 ++----------
 lib/xattr.c               | 144 +++++++++++++-
 mkfs/main.c               | 228 ++++++++++++++++-----
 14 files changed, 827 insertions(+), 210 deletions(-)
 create mode 100644 include/erofs/rebuild.h
 create mode 100644 lib/rebuild.c
Signature · 1 line
2.19.1.6.gb485710b
author
#1 lore raw

PATCH v6 1/11 erofs-utils: lib: remove unneeded NULL checking for returned item

get_xattritem() will in no way return NULL.

---
 lib/xattr.c | 4 ----
 1 file changed, 4 deletions(-)
lib/xattr.c +0 -4
index 0cab29f..5acc1ae 100644
@@ -437,8 +437,6 @@ int erofs_setxattr(struct erofs_inode *inode, char *key,
 	item = get_xattritem(prefix, kvbuf, len);
 	if (IS_ERR(item))
 		return PTR_ERR(item);
-	if (!item)
-		return 0;
 
 	return erofs_xattr_add(&inode->i_xattrs, item);
 }
@@ -473,8 +471,6 @@ static int erofs_droid_xattr_set_caps(struct erofs_inode *inode)
 	item = get_xattritem(EROFS_XATTR_INDEX_SECURITY, kvbuf, len);
 	if (IS_ERR(item))
 		return PTR_ERR(item);
-	if (!item)
-		return 0;
 
 	return erofs_xattr_add(&inode->i_xattrs, item);
 }
Signature · 1 line
2.19.1.6.gb485710b
author
#2 lore raw

PATCH v6 2/11 erofs-utils: lib: add list_splice_tail() helper

In reply to Jingbo Xu

Add list_splice_tail() helper.

---
 include/erofs/list.h | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)
include/erofs/list.h +20 -0
index 3f5da1a..d7a9fee 100644
@@ -70,6 +70,26 @@ static inline int list_empty(struct list_head *head)
 	return head->next == head;
 }
 
+static inline void __list_splice(struct list_head *list,
+		struct list_head *prev, struct list_head *next)
+{
+	struct list_head *first = list->next;
+	struct list_head *last = list->prev;
+
+	first->prev = prev;
+	prev->next = first;
+
+	last->next = next;
+	next->prev = last;
+}
+
+static inline void list_splice_tail(struct list_head *list,
+				    struct list_head *head)
+{
+	if (!list_empty(list))
+		__list_splice(list, head->prev, head);
+}
+
 #define list_entry(ptr, type, member) container_of(ptr, type, member)
 
 #define list_first_entry(ptr, type, member)                                    \
Signature · 1 line
2.19.1.6.gb485710b
author
#3 lore raw

PATCH v6 3/11 erofs-utils: lib: make erofs_get_unhashed_chunk() global

In reply to Jingbo Xu

... so that it could be called from outside blobchunk.c later.

---
 include/erofs/blobchunk.h | 2 ++
 lib/blobchunk.c           | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)
include/erofs/blobchunk.h +2 -0
index 010aee1..fb85d8e 100644
@@ -14,6 +14,8 @@ extern "C"
 
 #include "erofs/internal.h"
 
+struct erofs_blobchunk *erofs_get_unhashed_chunk(unsigned int device_id,
+		erofs_blk_t blkaddr, erofs_off_t sourceoffset);
 int erofs_blob_write_chunk_indexes(struct erofs_inode *inode, erofs_off_t off);
 int erofs_blob_write_chunked_file(struct erofs_inode *inode, int fd);
 int tarerofs_write_chunkes(struct erofs_inode *inode, erofs_off_t data_offset);
lib/blobchunk.c +1 -1
index cada5bb..07f18bd 100644
@@ -38,7 +38,7 @@ struct erofs_blobchunk erofs_holechunk = {
 };
 static LIST_HEAD(unhashed_blobchunks);
 
-static struct erofs_blobchunk *erofs_get_unhashed_chunk(unsigned int device_id,
+struct erofs_blobchunk *erofs_get_unhashed_chunk(unsigned int device_id,
 		erofs_blk_t blkaddr, erofs_off_t sourceoffset)
 {
 	struct erofs_blobchunk *chunk;
Signature · 1 line
2.19.1.6.gb485710b
author
#4 lore raw

PATCH v6 4/11 erofs-utils: lib: add erofs_read_xattrs_from_disk() helper

In reply to Jingbo Xu

Add erofs_read_xattrs_from_disk() helper reading extended attributes from disk, add checking if it's an opaque directory.

Move all xattr name related macros to xattr.c and introduce erofs_set_opaque_xattr() helper to hide all these details.

---
 include/erofs/internal.h |   1 +
 include/erofs/xattr.h    |  33 +-----------
 lib/tar.c                |   7 +--
 lib/xattr.c              | 112 +++++++++++++++++++++++++++++++++++++++
 4 files changed, 116 insertions(+), 37 deletions(-)
include/erofs/internal.h +1 -0
index 382024a..6020553 100644
@@ -191,6 +191,7 @@ struct erofs_inode {
 	bool compressed_idata;
 	bool lazy_tailblock;
 	bool with_tmpfile;
+	bool opaque;
 
 	unsigned int xattr_isize;
 	unsigned int extent_isize;
include/erofs/xattr.h +2 -31
index cf02257..9fa1ad1 100644
@@ -43,37 +43,6 @@ static inline unsigned int xattrblock_offset(struct erofs_inode *vi,
 	(_size - sizeof(struct erofs_xattr_ibody_header)) / \
 	sizeof(struct erofs_xattr_entry) + 1; })
 
-#ifndef XATTR_SYSTEM_PREFIX
-#define XATTR_SYSTEM_PREFIX	"system."
-#endif
-#ifndef XATTR_SYSTEM_PREFIX_LEN
-#define XATTR_SYSTEM_PREFIX_LEN (sizeof(XATTR_SYSTEM_PREFIX) - 1)
-#endif
-#ifndef XATTR_USER_PREFIX
-#define XATTR_USER_PREFIX	"user."
-#endif
-#ifndef XATTR_USER_PREFIX_LEN
-#define XATTR_USER_PREFIX_LEN (sizeof(XATTR_USER_PREFIX) - 1)
-#endif
-#ifndef XATTR_SECURITY_PREFIX
-#define XATTR_SECURITY_PREFIX	"security."
-#endif
-#ifndef XATTR_SECURITY_PREFIX_LEN
-#define XATTR_SECURITY_PREFIX_LEN (sizeof(XATTR_SECURITY_PREFIX) - 1)
-#endif
-#ifndef XATTR_TRUSTED_PREFIX
-#define XATTR_TRUSTED_PREFIX	"trusted."
-#endif
-#ifndef XATTR_TRUSTED_PREFIX_LEN
-#define XATTR_TRUSTED_PREFIX_LEN (sizeof(XATTR_TRUSTED_PREFIX) - 1)
-#endif
-#ifndef XATTR_NAME_POSIX_ACL_ACCESS
-#define XATTR_NAME_POSIX_ACL_ACCESS "system.posix_acl_access"
-#endif
-#ifndef XATTR_NAME_POSIX_ACL_DEFAULT
-#define XATTR_NAME_POSIX_ACL_DEFAULT "system.posix_acl_default"
-#endif
-
 int erofs_scan_file_xattrs(struct erofs_inode *inode);
 int erofs_prepare_xattr_ibody(struct erofs_inode *inode);
 char *erofs_export_xattr_ibody(struct erofs_inode *inode);
@@ -87,6 +56,8 @@ int erofs_xattr_prefixes_init(struct erofs_sb_info *sbi);
 
 int erofs_setxattr(struct erofs_inode *inode, char *key,
 		   const void *value, size_t size);
+int erofs_set_opaque_xattr(struct erofs_inode *inode);
+int erofs_read_xattrs_from_disk(struct erofs_inode *inode);
 
 #ifdef __cplusplus
 }
lib/tar.c +1 -6
index 53196c1..5b63cf5 100644
@@ -19,11 +19,6 @@
 #include "erofs/xattr.h"
 #include "erofs/blobchunk.h"
 
-#define OVL_XATTR_NAMESPACE "overlay."
-#define OVL_XATTR_TRUSTED_PREFIX XATTR_TRUSTED_PREFIX OVL_XATTR_NAMESPACE
-#define OVL_XATTR_OPAQUE_POSTFIX "opaque"
-#define OVL_XATTR_OPAQUE OVL_XATTR_TRUSTED_PREFIX OVL_XATTR_OPAQUE_POSTFIX
-
 #define EROFS_WHITEOUT_DEV	0
 
 static char erofs_libbuf[16384];
@@ -788,7 +783,7 @@ restart:
 	} else if (opq) {
 		DBG_BUGON(d->type == EROFS_FT_UNKNOWN);
 		DBG_BUGON(!d->inode);
-		ret = erofs_setxattr(d->inode, OVL_XATTR_OPAQUE, "y", 1);
+		ret = erofs_set_opaque_xattr(d->inode);
 		goto out;
 	} else if (th.typeflag == '1') {	/* hard link cases */
 		struct erofs_dentry *d2;
lib/xattr.c +112 -0

130 lines not on this page · raw message

Signature · 1 line
2.19.1.6.gb485710b
Reviewed-by on v6 4/11
#5 lore raw

Re: [PATCH v6 04/11] erofs-utils: lib: add erofs_read_xattrs_from_disk() helper

Quoted 7 lines · Jingbo Xu

Add erofs_read_xattrs_from_disk() helper reading extended attributes from disk, add checking if it's an opaque directory.

Move all xattr name related macros to xattr.c and introduce erofs_set_opaque_xattr() helper to hide all these details.

Add erofs_read_xattrs_from_disk() helper to read extended attributes from disk as well as a helper to check if it's an opaque directory.

Move all xattr name related macros to xattr.c and introduce erofs_set_opaque_xattr() helper to hide all these details.

I will update this manually, otherwise it looks good to me,

Thanks,
Gao Xiang

author
#6 lore raw

PATCH v6 5/11 erofs-utils: lib: add erofs_insert_ihash() helper

In reply to Jingbo Xu

Add erofs_insert_ihash() helper inserting inode into inode hash table.

Also add prototypes of erofs_iget() and erofs_iget_by_nid() in the header file.

---
 include/erofs/inode.h |  3 +++
 lib/inode.c           | 10 +++++++---
 2 files changed, 10 insertions(+), 3 deletions(-)
include/erofs/inode.h +3 -0
index e8a5670..1c602a8 100644
@@ -25,6 +25,9 @@ u32 erofs_new_encode_dev(dev_t dev);
 unsigned char erofs_mode_to_ftype(umode_t mode);
 unsigned char erofs_ftype_to_dtype(unsigned int filetype);
 void erofs_inode_manager_init(void);
+void erofs_insert_ihash(struct erofs_inode *inode, dev_t dev, ino_t ino);
+struct erofs_inode *erofs_iget(dev_t dev, ino_t ino);
+struct erofs_inode *erofs_iget_by_nid(erofs_nid_t nid);
 unsigned int erofs_iput(struct erofs_inode *inode);
 erofs_nid_t erofs_lookupnid(struct erofs_inode *inode);
 struct erofs_dentry *erofs_d_alloc(struct erofs_inode *parent,
lib/inode.c +7 -3
index a3a643f..8fab964 100644
@@ -75,6 +75,12 @@ void erofs_inode_manager_init(void)
 		init_list_head(&inode_hashtable[i]);
 }
 
+void erofs_insert_ihash(struct erofs_inode *inode, dev_t dev, ino_t ino)
+{
+	list_add(&inode->i_hash,
+		 &inode_hashtable[(ino ^ dev) % NR_INODE_HASHTABLE]);
+}
+
 /* get the inode from the (source) inode # */
 struct erofs_inode *erofs_iget(dev_t dev, ino_t ino)
 {
@@ -976,9 +982,7 @@ static int erofs_fill_inode(struct erofs_inode *inode, struct stat *st,
 		inode->inode_isize = sizeof(struct erofs_inode_compact);
 	}
 
-	list_add(&inode->i_hash,
-		 &inode_hashtable[(st->st_ino ^ st->st_dev) %
-				  NR_INODE_HASHTABLE]);
+	erofs_insert_ihash(inode, st->st_dev, st->st_ino);
 	return 0;
 }
 
Signature · 1 line
2.19.1.6.gb485710b
Reviewed-by on v6 5/11
#7 lore raw

Re: [PATCH v6 05/11] erofs-utils: lib: add erofs_insert_ihash() helper

Quoted 6 lines · Jingbo Xu

Add erofs_insert_ihash() helper inserting inode into inode hash table.

Also add prototypes of erofs_iget() and erofs_iget_by_nid() in the header file.

Thanks,
Gao Xiang

author
#8 lore raw

PATCH v6 6/11 erofs-utils: lib: add erofs_rebuild_get_dentry() helper

In reply to Jingbo Xu

Rename tarerofs_get_dentry() to erofs_rebuild_get_dentry().

Also make whout and 'opq' parameter optional when aufs is false.

---
 include/erofs/rebuild.h |  19 +++++++
 lib/Makefile.am         |   3 +-
 lib/rebuild.c           | 117 ++++++++++++++++++++++++++++++++++++++++
 lib/tar.c               | 109 ++-----------------------------------
 4 files changed, 141 insertions(+), 107 deletions(-)
 create mode 100644 include/erofs/rebuild.h
 create mode 100644 lib/rebuild.c
include/erofs/rebuild.h new file +19 -0
new file mode 100644
index 0000000..92873c9
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */
+#ifndef __EROFS_REBUILD_H
+#define __EROFS_REBUILD_H
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+#include "internal.h"
+
+struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd,
+		char *path, bool aufs, bool *whout, bool *opq);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
lib/Makefile.am +2 -1
index 3e09516..8a45bd6 100644
@@ -25,6 +25,7 @@ noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \
       $(top_srcdir)/include/erofs/compress_hints.h \
       $(top_srcdir)/include/erofs/fragments.h \
       $(top_srcdir)/include/erofs/xxhash.h \
+      $(top_srcdir)/include/erofs/rebuild.h \
       $(top_srcdir)/lib/liberofs_private.h
 
 noinst_HEADERS += compressor.h
@@ -32,7 +33,7 @@ liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \
 		      namei.c data.c compress.c compressor.c zmap.c decompress.c \
 		      compress_hints.c hashmap.c sha256.c blobchunk.c dir.c \
 		      fragments.c rb_tree.c dedupe.c uuid_unparse.c uuid.c tar.c \
-		      block_list.c xxhash.c
+		      block_list.c xxhash.c rebuild.c
 
 liberofs_la_CFLAGS = -Wall ${libuuid_CFLAGS} -I$(top_srcdir)/include
 if ENABLE_LZ4
lib/rebuild.c new file +117 -0

117 lines not on this page · raw message

lib/tar.c +3 -106

139 lines not on this page · raw message

Signature · 1 line
2.19.1.6.gb485710b
#9 lore raw

Re: [PATCH v6 06/11] erofs-utils: lib: add erofs_rebuild_get_dentry() helper

Quoted 111 lines · Jingbo Xu

Rename tarerofs_get_dentry() to erofs_rebuild_get_dentry().

Also make whout and 'opq' parameter optional when aufs is false.

---
  include/erofs/rebuild.h |  19 +++++++
  lib/Makefile.am         |   3 +-
  lib/rebuild.c           | 117 ++++++++++++++++++++++++++++++++++++++++
  lib/tar.c               | 109 ++-----------------------------------
  4 files changed, 141 insertions(+), 107 deletions(-)

create mode 100644 include/erofs/rebuild.h
create mode 100644 lib/rebuild.c

include/erofs/rebuild.h new file +19 -0
new file mode 100644
index 0000000..92873c9
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */
+#ifndef __EROFS_REBUILD_H
+#define __EROFS_REBUILD_H
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+#include "internal.h"
+
+struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd,
+		char *path, bool aufs, bool *whout, bool *opq);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
lib/Makefile.am +2 -1
index 3e09516..8a45bd6 100644
@@ -25,6 +25,7 @@ noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \
        $(top_srcdir)/include/erofs/compress_hints.h \
        $(top_srcdir)/include/erofs/fragments.h \
        $(top_srcdir)/include/erofs/xxhash.h \
+      $(top_srcdir)/include/erofs/rebuild.h \
        $(top_srcdir)/lib/liberofs_private.h
 
  noinst_HEADERS += compressor.h
@@ -32,7 +33,7 @@ liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \
  		      namei.c data.c compress.c compressor.c zmap.c decompress.c \
  		      compress_hints.c hashmap.c sha256.c blobchunk.c dir.c \
  		      fragments.c rb_tree.c dedupe.c uuid_unparse.c uuid.c tar.c \
-		      block_list.c xxhash.c
+		      block_list.c xxhash.c rebuild.c
 
  liberofs_la_CFLAGS = -Wall ${libuuid_CFLAGS} -I$(top_srcdir)/include
  if ENABLE_LZ4
lib/rebuild.c new file +45 -0
new file mode 100644
index 0000000..7aaa071
@@ -0,0 +1,117 @@
+// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0
+#define _GNU_SOURCE
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+#include "erofs/print.h"
+#include "erofs/inode.h"
+#include "erofs/rebuild.h"
+#include "erofs/internal.h"
+
+#ifdef HAVE_LINUX_AUFS_TYPE_H
+#include <linux/aufs_type.h>
+#else
+#define AUFS_WH_PFX		".wh."
+#define AUFS_DIROPQ_NAME	AUFS_WH_PFX ".opq"
+#define AUFS_WH_DIROPQ		AUFS_WH_PFX AUFS_DIROPQ_NAME
+#endif
+
+static struct erofs_dentry *erofs_rebuild_mkdir(struct erofs_inode *dir,
+						const char *s)
+{
+	struct erofs_inode *inode;
+	struct erofs_dentry *d;
+
+	inode = erofs_new_inode();
+	if (IS_ERR(inode))
+		return ERR_CAST(inode);
+
+	inode->i_mode = S_IFDIR | 0755;
+	inode->i_parent = dir;
+	inode->i_uid = getuid();
+	inode->i_gid = getgid();
+	inode->i_mtime = inode->sbi->build_time;
+	inode->i_mtime_nsec = inode->sbi->build_time_nsec;
+	erofs_init_empty_dir(inode);
+
+	d = erofs_d_alloc(dir, s);
+	if (!IS_ERR(d)) {
+		d->type = EROFS_FT_DIR;
+		d->inode = inode;
+	}
+	return d;
+}
+
+struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd,

lib/rebuild.c line 55

Quoted 10 lines
lib/rebuild.c new file +10 -0
+		char *path, bool aufs, bool *whout, bool *opq)
+{
+	struct erofs_dentry *d = NULL;
+	unsigned int len = strlen(path);
+	char *s = path;
+
+	if (aufs) {
+		*whout = false;
+		*opq = false;
+	}

I'd suggest adding a boolean dumb instead here, otherwise it looks good to me.

Thanks,
Gao Xiang

author
#10 lore raw

PATCH v6 7/11 erofs-utils: lib: add erofs_rebuild_load_tree() helper

In reply to Jingbo Xu

Add erofs_rebuild_load_tree() helper loading inode tree from given erofs image, and making it merged into a given inode tree in an overlayfs like model.

Since we need to read the content of the symlink file from disk when loading tree, add dependency on zlib_LIBS for mkfs.erofs.

Also rename tarerofs_dump_tree() to erofs_rebuild_dump_tree(), so that it could also be called from rebuild mode.

---
 include/erofs/inode.h    |   2 +-
 include/erofs/internal.h |   1 +
 include/erofs/rebuild.h  |   2 +
 lib/inode.c              |   4 +-
 lib/rebuild.c            | 284 +++++++++++++++++++++++++++++++++++++++
 mkfs/main.c              |   2 +-
 6 files changed, 291 insertions(+), 4 deletions(-)
include/erofs/inode.h +1 -1
index 1c602a8..fe9dda2 100644
@@ -32,7 +32,7 @@ unsigned int erofs_iput(struct erofs_inode *inode);
 erofs_nid_t erofs_lookupnid(struct erofs_inode *inode);
 struct erofs_dentry *erofs_d_alloc(struct erofs_inode *parent,
 				   const char *name);
-int tarerofs_dump_tree(struct erofs_inode *dir);
+int erofs_rebuild_dump_tree(struct erofs_inode *dir);
 int erofs_init_empty_dir(struct erofs_inode *dir);
 struct erofs_inode *erofs_new_inode(void);
 struct erofs_inode *erofs_mkfs_build_tree_from_path(const char *path);
include/erofs/internal.h +1 -0
index 6020553..3d5c337 100644
@@ -108,6 +108,7 @@ struct erofs_sb_info {
 
 	int devfd;
 	u64 devsz;
+	dev_t dev;
 	unsigned int nblobs;
 	unsigned int blobfd[256];
 };
include/erofs/rebuild.h +2 -0
index 92873c9..3ac074c 100644
@@ -12,6 +12,8 @@ extern "C"
 struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd,
 		char *path, bool aufs, bool *whout, bool *opq);
 
+int erofs_rebuild_load_tree(struct erofs_inode *root, struct erofs_sb_info *sbi);
+
 #ifdef __cplusplus
 }
 #endif
lib/inode.c +2 -2
index 8fab964..7e5d464 100644
@@ -1324,7 +1324,7 @@ struct erofs_inode *erofs_mkfs_build_special_from_fd(int fd, const char *name)
 	return inode;
 }
 
-int tarerofs_dump_tree(struct erofs_inode *dir)
+int erofs_rebuild_dump_tree(struct erofs_inode *dir)
 {
 	struct erofs_dentry *d;
 	unsigned int nr_subdirs;
@@ -1392,7 +1392,7 @@ int tarerofs_dump_tree(struct erofs_inode *dir)
 			continue;
 
 		inode = erofs_igrab(d->inode);
-		ret = tarerofs_dump_tree(inode);
+		ret = erofs_rebuild_dump_tree(inode);
 		dir->i_nlink += (erofs_mode_to_ftype(inode->i_mode) == EROFS_FT_DIR);
 		erofs_iput(inode);
 		if (ret)
lib/rebuild.c +284 -0

296 lines not on this page · raw message

mkfs/main.c +1 -1
index 607c883..94f6407 100644
@@ -962,7 +962,7 @@ int main(int argc, char **argv)
 		if (err < 0)
 			goto exit;
 
-		err = tarerofs_dump_tree(root_inode);
+		err = erofs_rebuild_dump_tree(root_inode);
 		if (err < 0)
 			goto exit;
 	}
Signature · 1 line
2.19.1.6.gb485710b
author
#11 lore raw

PATCH v6 8/11 erofs-utils: mkfs: introduce rebuild mode

In reply to Jingbo Xu

Introduce a new rebuild mode merging multiple erofs images generated from either tarerfs index mode (--tar=i):

	mkfs.erofs --tar=i --aufs layer0.erofs layer0.tar
	...
	mkfs.erofs --tar=i --aufs layerN.erofs layerN.tar

	mkfs.erofs merge.erofs layerN.erofs ... layer0.erofs

or tarerofs non-index mode (--tar=f):

	mkfs.erofs --tar=f -Enoinline_data --aufs layer0.erofs layer0.tar
	...
	mkfs.erofs --tar=f -Enoinline_data --aufs layerN.erofs layerN.tar

	mkfs.erofs merge.erofs layerN.erofs ... layer0.erofs

When calling rebuild mode, the top most layer must be specified firstly in the cmdline, while the bottom most layer i.e. layer0 the lastly.

The rebuild mode doesn't support flat inline datalayout yet.

---
 include/erofs/internal.h |   2 +
 mkfs/main.c              | 216 ++++++++++++++++++++++++++++++---------
 2 files changed, 170 insertions(+), 48 deletions(-)
include/erofs/internal.h +2 -0
index 3d5c337..fcc0710 100644
@@ -111,6 +111,8 @@ struct erofs_sb_info {
 	dev_t dev;
 	unsigned int nblobs;
 	unsigned int blobfd[256];
+
+	struct list_head list;
 };
 
 /* make sure that any user of the erofs headers has atleast 64bit off_t type */
mkfs/main.c +168 -48

297 lines not on this page · raw message

Signature · 1 line
2.19.1.6.gb485710b
author
#12 lore raw

PATCH v6 9/11 erofs-utils: lib: add erofs_inode_is_whiteout() helper

In reply to Jingbo Xu

Add erofs_inode_is_whiteout() helper checking if given inode is a whiteout.

---
 include/erofs/internal.h | 7 +++++++
 lib/tar.c                | 2 --
 2 files changed, 7 insertions(+), 2 deletions(-)
include/erofs/internal.h +7 -0
index fcc0710..5df689c 100644
@@ -20,6 +20,7 @@ typedef unsigned short umode_t;
 #include "erofs_fs.h"
 #include <fcntl.h>
 #include <sys/types.h> /* for off_t definition */
+#include <sys/stat.h> /* for S_ISCHR definition */
 #include <stdio.h>
 
 #ifndef PATH_MAX
@@ -427,6 +428,12 @@ static inline u32 erofs_crc32c(u32 crc, const u8 *in, size_t len)
 	return crc;
 }
 
+#define EROFS_WHITEOUT_DEV	0
+static inline bool erofs_inode_is_whiteout(struct erofs_inode *inode)
+{
+	return S_ISCHR(inode->i_mode) && inode->u.i_rdev == EROFS_WHITEOUT_DEV;
+}
+
 #ifdef __cplusplus
 }
 #endif
lib/tar.c +0 -2
index e0fe2a2..8cb392c 100644
@@ -13,8 +13,6 @@
 #include "erofs/blobchunk.h"
 #include "erofs/rebuild.h"
 
-#define EROFS_WHITEOUT_DEV	0
-
 static char erofs_libbuf[16384];
 
 struct tar_header {
Signature · 1 line
2.19.1.6.gb485710b
author
#13 lore raw

PATCH v6 10/11 erofs-utils: lib: set origin xattr on parent directory of whiteout

In reply to Jingbo Xu

Whiteouts will be exposed in the overlayfs mount if the whiteouts are from non-merged directory without origin xattr set on the directory.

To fix this, set origin xattr (with empty value) on parent directory of whiteouts, marking the parent directory as copied-up to avoid exposing whiteouts in the overlayfs mount.

To avoid setting multiple origin xattrs on the same directory when there are multiple whiteouts under the directory, enhance erofs_setxattr() so that it could skip duplicate xattrs.

---
 include/erofs/xattr.h |  3 ++-
 lib/rebuild.c         |  6 ++++++
 lib/tar.c             |  9 ++++++++-
 lib/xattr.c           | 27 ++++++++++++++++++++++++---
 4 files changed, 40 insertions(+), 5 deletions(-)
include/erofs/xattr.h +2 -1
index 9fa1ad1..62d0b8b 100644
@@ -55,8 +55,9 @@ void erofs_xattr_prefixes_cleanup(struct erofs_sb_info *sbi);
 int erofs_xattr_prefixes_init(struct erofs_sb_info *sbi);
 
 int erofs_setxattr(struct erofs_inode *inode, char *key,
-		   const void *value, size_t size);
+		   const void *value, size_t size, bool nodup);
 int erofs_set_opaque_xattr(struct erofs_inode *inode);
+int erofs_set_origin_xattr(struct erofs_inode *inode);
 int erofs_read_xattrs_from_disk(struct erofs_inode *inode);
 
 #ifdef __cplusplus
lib/rebuild.c +6 -0
index 477d68d..b254dea 100644
@@ -200,6 +200,12 @@ static int erofs_rebuild_fill_inode(struct erofs_inode *inode)
 
 	switch (inode->i_mode & S_IFMT) {
 	case S_IFCHR:
+		if (erofs_inode_is_whiteout(inode)) {
+			ret = erofs_set_origin_xattr(inode->i_parent);
+			if (ret)
+				break;
+		}
+		/* fallthrough */
 	case S_IFBLK:
 	case S_IFIFO:
 	case S_IFSOCK:
lib/tar.c +8 -1
index 8cb392c..b75b723 100644
@@ -206,7 +206,7 @@ int tarerofs_apply_xattrs(struct erofs_inode *inode, struct list_head *xattrs)
 		item->kv[item->namelen] = '\0';
 		erofs_dbg("Recording xattr(%s)=\"%s\" (of %u bytes) to file %s",
 			  item->kv, v, vsz, inode->i_srcpath);
-		ret = erofs_setxattr(inode, item->kv, v, vsz);
+		ret = erofs_setxattr(inode, item->kv, v, vsz, false);
 		if (ret == -ENODATA)
 			erofs_err("Failed to set xattr(%s)=%s to file %s",
 				  item->kv, v, inode->i_srcpath);
@@ -740,6 +740,13 @@ new_inode:
 		inode->i_mode = (inode->i_mode & ~S_IFMT) | S_IFCHR;
 		inode->u.i_rdev = EROFS_WHITEOUT_DEV;
 		d->type = EROFS_FT_CHRDEV;
+		/*
+		 * Mark parent directory as copied-up to avoid exposing
+		 * whiteouts in the overlayfs mount.
+		 */
+		ret = erofs_set_origin_xattr(inode->i_parent);
+		if (ret)
+			goto out;
 	} else {
 		inode->i_mode = st.st_mode;
 		if (S_ISBLK(st.st_mode) || S_ISCHR(st.st_mode))
lib/xattr.c +24 -3
index cc6a393..feafe96 100644
@@ -57,12 +57,18 @@
 #ifndef OVL_XATTR_OPAQUE_POSTFIX
 #define OVL_XATTR_OPAQUE_POSTFIX "opaque"
 #endif
+#ifndef OVL_XATTR_ORIGIN_POSTFIX
+#define OVL_XATTR_ORIGIN_POSTFIX "origin"
+#endif
 #ifndef OVL_XATTR_TRUSTED_PREFIX
 #define OVL_XATTR_TRUSTED_PREFIX XATTR_TRUSTED_PREFIX OVL_XATTR_NAMESPACE
 #endif
 #ifndef OVL_XATTR_OPAQUE
 #define OVL_XATTR_OPAQUE OVL_XATTR_TRUSTED_PREFIX OVL_XATTR_OPAQUE_POSTFIX
 #endif
+#ifndef OVL_XATTR_ORIGIN
+#define OVL_XATTR_ORIGIN OVL_XATTR_TRUSTED_PREFIX OVL_XATTR_ORIGIN_POSTFIX
+#endif
 
 #define EA_HASHTABLE_BITS 16
 
@@ -455,11 +461,12 @@ err:
 }
 
 int erofs_setxattr(struct erofs_inode *inode, char *key,
-		   const void *value, size_t size)
+		   const void *value, size_t size, bool nodup)
 {
 	char *kvbuf;
 	unsigned int len[2];
 	struct xattr_item *item;
+	struct inode_xattr_node *node;
 	u8 prefix;
 	u16 prefixlen;
 
@@ -481,12 +488,26 @@ int erofs_setxattr(struct erofs_inode *inode, char *key,
 	if (IS_ERR(item))
 		return PTR_ERR(item);
 
+	if (nodup) {
+		list_for_each_entry(node, &inode->i_xattrs, list) {
+			if (node->item == item) {
+				put_xattritem(item);
+				return 0;
+			}
+		}
+	}
+
 	return erofs_xattr_add(&inode->i_xattrs, item);
 }
 
 int erofs_set_opaque_xattr(struct erofs_inode *inode)
 {
-	return erofs_setxattr(inode, OVL_XATTR_OPAQUE, "y", 1);
+	return erofs_setxattr(inode, OVL_XATTR_OPAQUE, "y", 1, false);
+}
+
+int erofs_set_origin_xattr(struct erofs_inode *inode)
+{
+	return erofs_setxattr(inode, OVL_XATTR_ORIGIN, NULL, 0, true);
 }
 
 #ifdef WITH_ANDROID
@@ -599,7 +620,7 @@ int erofs_read_xattrs_from_disk(struct erofs_inode *inode)
 			DBG_BUGON(ret != size);
 		}
 
-		ret = erofs_setxattr(inode, key, value, size);
+		ret = erofs_setxattr(inode, key, value, size, false);
 		free(value);
 		if (ret)
 			break;
Signature · 1 line
2.19.1.6.gb485710b
author
#14 lore raw

PATCH v6 11/11 erofs-utils: mkfs: add --overlay-strip option for tarfs and rebuild mode

In reply to Jingbo Xu

Add "--overlay-strip=[0|1]" option for tarfs and rebuild mode, controlling whether overlayfs related stuff, e.g. whiteout files and origin xattrs, are stripped from the generated image.

This option is disabled by default for both tarfs and rebuild mode, that is, whiteout files and origin xattrs are kept in the image by default. Specify "--overlay-strip=1" explicitly to strip these stuff.

---
 include/erofs/config.h |  1 +
 lib/inode.c            | 12 ++++++++++--
 lib/rebuild.c          | 10 ++++++----
 lib/tar.c              | 16 +++++++++-------
 lib/xattr.c            |  5 +++++
 mkfs/main.c            | 12 ++++++++++++
 6 files changed, 43 insertions(+), 13 deletions(-)
include/erofs/config.h +1 -0
index c51f0cd..f244607 100644
@@ -54,6 +54,7 @@ struct erofs_configure {
 	bool c_showprogress;
 	bool c_extra_ea_name_prefixes;
 	bool c_xattr_name_filter;
+	bool c_overlay_strip;
 
 #ifdef HAVE_LIBSELINUX
 	struct selabel_handle *sehnd;
lib/inode.c +10 -2
index 7e5d464..42a4c2f 100644
@@ -1326,7 +1326,7 @@ struct erofs_inode *erofs_mkfs_build_special_from_fd(int fd, const char *name)
 
 int erofs_rebuild_dump_tree(struct erofs_inode *dir)
 {
-	struct erofs_dentry *d;
+	struct erofs_dentry *d, *n;
 	unsigned int nr_subdirs;
 	int ret;
 
@@ -1370,8 +1370,16 @@ int erofs_rebuild_dump_tree(struct erofs_inode *dir)
 	}
 
 	nr_subdirs = 0;
-	list_for_each_entry(d, &dir->i_subdirs, d_child)
+	list_for_each_entry_safe(d, n, &dir->i_subdirs, d_child) {
+		if (cfg.c_overlay_strip && erofs_inode_is_whiteout(d->inode)) {
+			erofs_dbg("strip whiteout %s", d->inode->i_srcpath);
+			list_del(&d->d_child);
+			erofs_d_invalidate(d);
+			free(d);
+			continue;
+		}
 		++nr_subdirs;
+	}
 
 	ret = erofs_prepare_dir_layout(dir, nr_subdirs);
 	if (ret)
lib/rebuild.c +6 -4
index b254dea..1d8c7bc 100644
@@ -200,10 +200,12 @@ static int erofs_rebuild_fill_inode(struct erofs_inode *inode)
 
 	switch (inode->i_mode & S_IFMT) {
 	case S_IFCHR:
-		if (erofs_inode_is_whiteout(inode)) {
-			ret = erofs_set_origin_xattr(inode->i_parent);
-			if (ret)
-				break;
+		if (!cfg.c_overlay_strip) {
+			if (erofs_inode_is_whiteout(inode)) {
+				ret = erofs_set_origin_xattr(inode->i_parent);
+				if (ret)
+					break;
+			}
 		}
 		/* fallthrough */
 	case S_IFBLK:
lib/tar.c +9 -7
index b75b723..3389bf6 100644
@@ -740,13 +740,15 @@ new_inode:
 		inode->i_mode = (inode->i_mode & ~S_IFMT) | S_IFCHR;
 		inode->u.i_rdev = EROFS_WHITEOUT_DEV;
 		d->type = EROFS_FT_CHRDEV;
-		/*
-		 * Mark parent directory as copied-up to avoid exposing
-		 * whiteouts in the overlayfs mount.
-		 */
-		ret = erofs_set_origin_xattr(inode->i_parent);
-		if (ret)
-			goto out;
+		if (!cfg.c_overlay_strip) {
+			/*
+			 * Mark parent directory as copied-up to avoid exposing
+			 * whiteouts in the overlayfs mount.
+			 */
+			ret = erofs_set_origin_xattr(inode->i_parent);
+			if (ret)
+				goto out;
+		}
 	} else {
 		inode->i_mode = st.st_mode;
 		if (S_ISBLK(st.st_mode) || S_ISCHR(st.st_mode))
lib/xattr.c +5 -0
index feafe96..d7f4e3a 100644
@@ -601,6 +601,11 @@ int erofs_read_xattrs_from_disk(struct erofs_inode *inode)
 			inode->opaque = true;
 		}
 
+		if (cfg.c_overlay_strip && !strcmp(key, OVL_XATTR_ORIGIN)) {
+			ret = 0;
+			continue;
+		}
+
 		ret = erofs_getxattr(inode, key, NULL, 0);
 		if (ret < 0)
 			goto out;
mkfs/main.c +12 -0
index b51ef37..4af07f2 100644
@@ -57,6 +57,7 @@ static struct option long_options[] = {
 	{"gid-offset", required_argument, NULL, 17},
 	{"tar", optional_argument, NULL, 20},
 	{"aufs", no_argument, NULL, 21},
+	{"overlay-strip", required_argument, NULL, 22},
 	{"mount-point", required_argument, NULL, 512},
 	{"xattr-prefix", required_argument, NULL, 19},
 #ifdef WITH_ANDROID
@@ -121,6 +122,7 @@ static void usage(void)
 	      " --preserve-mtime      keep per-file modification time strictly\n"
 	      " --aufs                replace aufs special files with overlayfs metadata\n"
 	      " --tar=[fi]            generate an image from tarball(s)\n"
+	      " --overlay-strip=[01]  strip overlayfs stuff e.g. whiteout in tarfs or rebuild mode\n"
 	      " --quiet               quiet execution (do not write anything to standard output.)\n"
 #ifndef NDEBUG
 	      " --random-pclusterblks randomize pclusterblks for big pcluster (debugging only)\n"
@@ -538,6 +540,16 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
 		case 21:
 			erofstar.aufs = true;
 			break;
+		case 22:
+			if (!strcmp(optarg, "0"))
+				cfg.c_overlay_strip = false;
+			else if (!strcmp(optarg, "1"))
+				cfg.c_overlay_strip = true;
+			else {
+				erofs_err("invalid overlay-strip %s", optarg);
+				return -EINVAL;
+			}
+			break;
 		case 1:
 			usage();
 			exit(0);
Signature · 1 line
2.19.1.6.gb485710b