erofs-utils: mkfs: introduce rebuild mode
v6 cover erofs-utils: mkfs: introduce rebuild mode
v6:
- newly add cleanup patches (patch 1 and 9)
- patch 8: improve the code of parsing multiple sources with "do {} while" loop
- patch 10: newly added to work around the potential issue of exposing whiteouts in the overlayfs mount by setting origin xattr on parent directory of whiteouts
- patch 11: the former "--keep-whiteout" option is renamed to "--overlay-strip" option; apart from whiteouts, origin xattrs will also be stripped when the option is enabled
[1] https://lore.kernel.org/all/[email protected]/
[2] https://lore.kernel.org/all/[email protected]/
v1: https://lore.kernel.org/all/[email protected]/
v2: https://lore.kernel.org/all/[email protected]/
v3: https://lore.kernel.org/all/[email protected]/
v4: https://lore.kernel.org/all/[email protected]/
v5: https://lore.kernel.org/all/[email protected]/
-------------------------
Introduce a new rebuild mode merging multiple erofs images generated from either tarerfs index mode (--tar=i):
mkfs.erofs --tar=i --aufs layer0.erofs layer0.tar ... mkfs.erofs --tar=i --aufs layerN.erofs layerN.tar mkfs.erofs merge.erofs layerN.erofs ... layer0.erofs
or tarerofs non-index mode (--tar=f):
mkfs.erofs --tar=f -Enoinline_data --aufs layer0.erofs layer0.tar ... mkfs.erofs --tar=f -Enoinline_data --aufs layerN.erofs layerN.tar mkfs.erofs merge.erofs layerN.erofs ... layer0.erofs
Jingbo Xu (11):
erofs-utils: lib: remove unneeded NULL checking for returned item
erofs-utils: lib: add list_splice_tail() helper
erofs-utils: lib: make erofs_get_unhashed_chunk() global
erofs-utils: lib: add erofs_read_xattrs_from_disk() helper
erofs-utils: lib: add erofs_insert_ihash() helper
erofs-utils: lib: add erofs_rebuild_get_dentry() helper
erofs-utils: lib: add erofs_rebuild_load_tree() helper
erofs-utils: mkfs: introduce rebuild mode
erofs-utils: lib: add erofs_inode_is_whiteout() helper
erofs-utils: lib: set origin xattr on parent directory of whiteout
erofs-utils: mkfs: add --overlay-strip option for tarfs and rebuild
mode
include/erofs/blobchunk.h | 2 + include/erofs/config.h | 1 + include/erofs/inode.h | 5 +- include/erofs/internal.h | 11 + include/erofs/list.h | 20 ++ include/erofs/rebuild.h | 21 ++ include/erofs/xattr.h | 36 +--- lib/Makefile.am | 3 +- lib/blobchunk.c | 2 +- lib/inode.c | 26 ++- lib/rebuild.c | 409 ++++++++++++++++++++++++++++++++++++++ lib/tar.c | 129 ++---------- lib/xattr.c | 144 +++++++++++++- mkfs/main.c | 228 ++++++++++++++++----- 14 files changed, 827 insertions(+), 210 deletions(-) create mode 100644 include/erofs/rebuild.h create mode 100644 lib/rebuild.c
Signature · 1 line
2.19.1.6.gb485710b
PATCH v6 1/11 erofs-utils: lib: remove unneeded NULL checking for returned item
get_xattritem() will in no way return NULL.
--- lib/xattr.c | 4 ---- 1 file changed, 4 deletions(-)
lib/xattr.c +0 -4
@@ -437,8 +437,6 @@ int erofs_setxattr(struct erofs_inode *inode, char *key, item =get_xattritem (prefix, kvbuf, len);if (IS_ERR (item))return PTR_ERR (item); -if (!item) -return 0 ;return erofs_xattr_add (&inode->i_xattrs, item); }
@@ -473,8 +471,6 @@ static int erofs_droid_xattr_set_caps(struct erofs_inode *inode) item =get_xattritem (EROFS_XATTR_INDEX_SECURITY , kvbuf, len);if (IS_ERR (item))return PTR_ERR (item); -if (!item) -return 0 ;return erofs_xattr_add (&inode->i_xattrs, item); }
Signature · 1 line
2.19.1.6.gb485710b
PATCH v6 2/11 erofs-utils: lib: add list_splice_tail() helper
In reply to Jingbo XuAdd list_splice_tail() helper.
--- include/erofs/list.h | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+)
include/erofs/list.h +20 -0
@@ -70,6 +70,26 @@ static inline int list_empty(struct list_head *head)return head->next == head; } +static inline void __list_splice (struct list_head *list, +struct list_head *prev,struct list_head *next) +{ +struct list_head *first = list->next; +struct list_head *last = list->prev; + + first->prev = prev; + prev->next = first; + + last->next = next; + next->prev = last; +} + +static inline void list_splice_tail (struct list_head *list, +struct list_head *head) +{ +if (!list_empty (list)) +__list_splice (list, head->prev, head); +} +#define list_entry (ptr, type, member) container_of(ptr, type, member)#define list_first_entry (ptr, type, member) \
Signature · 1 line
2.19.1.6.gb485710b
PATCH v6 3/11 erofs-utils: lib: make erofs_get_unhashed_chunk() global
In reply to Jingbo Xu... so that it could be called from outside blobchunk.c later.
--- include/erofs/blobchunk.h | 2 ++ lib/blobchunk.c | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-)
include/erofs/blobchunk.h +2 -0
@@ -14,6 +14,8 @@ extern "C"#include "erofs/internal.h" +struct erofs_blobchunk *erofs_get_unhashed_chunk (unsigned int device_id, +erofs_blk_t blkaddr,erofs_off_t sourceoffset);int erofs_blob_write_chunk_indexes (struct erofs_inode *inode,erofs_off_t off);int erofs_blob_write_chunked_file (struct erofs_inode *inode,int fd);int tarerofs_write_chunkes (struct erofs_inode *inode,erofs_off_t data_offset);
lib/blobchunk.c +1 -1
@@ -38,7 +38,7 @@ struct erofs_blobchunk erofs_holechunk = { };static LIST_HEAD (unhashed_blobchunks); -static struct erofs_blobchunk *erofs_get_unhashed_chunk (unsigned int device_id, +struct erofs_blobchunk *erofs_get_unhashed_chunk (unsigned int device_id,erofs_blk_t blkaddr,erofs_off_t sourceoffset) {struct erofs_blobchunk *chunk;
Signature · 1 line
2.19.1.6.gb485710b
PATCH v6 4/11 erofs-utils: lib: add erofs_read_xattrs_from_disk() helper
In reply to Jingbo XuAdd erofs_read_xattrs_from_disk() helper reading extended attributes from disk, add checking if it's an opaque directory.
Move all xattr name related macros to xattr.c and introduce erofs_set_opaque_xattr() helper to hide all these details.
--- include/erofs/internal.h | 1 + include/erofs/xattr.h | 33 +----------- lib/tar.c | 7 +-- lib/xattr.c | 112 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 116 insertions(+), 37 deletions(-)
include/erofs/internal.h +1 -0
@@ -191,6 +191,7 @@ struct erofs_inode {bool compressed_idata;bool lazy_tailblock;bool with_tmpfile; +bool opaque;unsigned int xattr_isize;unsigned int extent_isize;
include/erofs/xattr.h +2 -31
@@ -43,37 +43,6 @@ static inline unsigned int xattrblock_offset(struct erofs_inode *vi, (_size -sizeof (struct erofs_xattr_ibody_header )) / \ sizeof(struct erofs_xattr_entry) + 1; }) -#ifndef XATTR_SYSTEM_PREFIX -#define XATTR_SYSTEM_PREFIX "system." -#endif -#ifndef XATTR_SYSTEM_PREFIX_LEN -#define XATTR_SYSTEM_PREFIX_LEN (sizeof(XATTR_SYSTEM_PREFIX) - 1) -#endif -#ifndef XATTR_USER_PREFIX -#define XATTR_USER_PREFIX "user." -#endif -#ifndef XATTR_USER_PREFIX_LEN -#define XATTR_USER_PREFIX_LEN (sizeof(XATTR_USER_PREFIX) - 1) -#endif -#ifndef XATTR_SECURITY_PREFIX -#define XATTR_SECURITY_PREFIX "security." -#endif -#ifndef XATTR_SECURITY_PREFIX_LEN -#define XATTR_SECURITY_PREFIX_LEN (sizeof(XATTR_SECURITY_PREFIX) - 1) -#endif -#ifndef XATTR_TRUSTED_PREFIX -#define XATTR_TRUSTED_PREFIX "trusted." -#endif -#ifndef XATTR_TRUSTED_PREFIX_LEN -#define XATTR_TRUSTED_PREFIX_LEN (sizeof(XATTR_TRUSTED_PREFIX) - 1) -#endif -#ifndef XATTR_NAME_POSIX_ACL_ACCESS -#define XATTR_NAME_POSIX_ACL_ACCESS "system.posix_acl_access" -#endif -#ifndef XATTR_NAME_POSIX_ACL_DEFAULT -#define XATTR_NAME_POSIX_ACL_DEFAULT "system.posix_acl_default" -#endif -int erofs_scan_file_xattrs (struct erofs_inode *inode);int erofs_prepare_xattr_ibody (struct erofs_inode *inode);char *erofs_export_xattr_ibody (struct erofs_inode *inode);
@@ -87,6 +56,8 @@ int erofs_xattr_prefixes_init(struct erofs_sb_info *sbi);int erofs_setxattr (struct erofs_inode *inode,char *key,const void *value,size_t size); +int erofs_set_opaque_xattr (struct erofs_inode *inode); +int erofs_read_xattrs_from_disk (struct erofs_inode *inode); #ifdef __cplusplus }
lib/tar.c +1 -6
@@ -19,11 +19,6 @@#include "erofs/xattr.h" #include "erofs/blobchunk.h" -#define OVL_XATTR_NAMESPACE "overlay." -#define OVL_XATTR_TRUSTED_PREFIX XATTR_TRUSTED_PREFIX OVL_XATTR_NAMESPACE -#define OVL_XATTR_OPAQUE_POSTFIX "opaque" -#define OVL_XATTR_OPAQUE OVL_XATTR_TRUSTED_PREFIX OVL_XATTR_OPAQUE_POSTFIX -#define EROFS_WHITEOUT_DEV 0static char erofs_libbuf[16384 ];
@@ -788,7 +783,7 @@ restart: }else if (opq) {DBG_BUGON (d->type ==EROFS_FT_UNKNOWN );DBG_BUGON (!d->inode); - ret =erofs_setxattr (d->inode,OVL_XATTR_OPAQUE ,"y" ,1 ); + ret =erofs_set_opaque_xattr (d->inode); goto out; }else if (th.typeflag =='1' ) {/* hard link cases */ struct erofs_dentry *d2;
lib/xattr.c +112 -0
130 lines not on this page · raw message
Signature · 1 line
2.19.1.6.gb485710b
Re: [PATCH v6 04/11] erofs-utils: lib: add erofs_read_xattrs_from_disk() helper
Quoted 7 lines · Jingbo Xu
Add
erofs_read_xattrs_from_disk()helper reading extended attributes from disk, add checking if it's an opaque directory.Move all xattr name related macros to
xattr.cand introduceerofs_set_opaque_xattr()helper to hide all these details.Signed-off-by Jingbo Xu <[email protected]>
Add erofs_read_xattrs_from_disk() helper to read extended attributes from disk as well as a helper to check if it's an opaque directory.
Move all xattr name related macros to xattr.c and introduce erofs_set_opaque_xattr() helper to hide all these details.
I will update this manually, otherwise it looks good to me,
Thanks,
Gao Xiang
PATCH v6 5/11 erofs-utils: lib: add erofs_insert_ihash() helper
In reply to Jingbo XuAdd erofs_insert_ihash() helper inserting inode into inode hash table.
Also add prototypes of erofs_iget() and erofs_iget_by_nid() in the header file.
--- include/erofs/inode.h | 3 +++ lib/inode.c | 10 +++++++--- 2 files changed, 10 insertions(+), 3 deletions(-)
include/erofs/inode.h +3 -0
@@ -25,6 +25,9 @@ u32 erofs_new_encode_dev(dev_t dev);unsigned char erofs_mode_to_ftype (umode_t mode);unsigned char erofs_ftype_to_dtype (unsigned int filetype);void erofs_inode_manager_init (void ); +void erofs_insert_ihash (struct erofs_inode *inode,dev_t dev,ino_t ino); +struct erofs_inode *erofs_iget (dev_t dev,ino_t ino); +struct erofs_inode *erofs_iget_by_nid (erofs_nid_t nid);unsigned int erofs_iput (struct erofs_inode *inode);erofs_nid_t erofs_lookupnid (struct erofs_inode *inode); struct erofs_dentry *erofs_d_alloc(struct erofs_inode *parent,
lib/inode.c +7 -3
@@ -75,6 +75,12 @@ void erofs_inode_manager_init(void)init_list_head (&inode_hashtable[i]); } +void erofs_insert_ihash (struct erofs_inode *inode,dev_t dev,ino_t ino) +{ +list_add (&inode->i_hash, + &inode_hashtable[(ino ^ dev) %NR_INODE_HASHTABLE ]); +} +/* get the inode from the (source) inode # */ struct erofs_inode *erofs_iget (dev_t dev,ino_t ino) {
@@ -976,9 +982,7 @@ static int erofs_fill_inode(struct erofs_inode *inode, struct stat *st, inode->inode_isize =sizeof (struct erofs_inode_compact ); } -list_add (&inode->i_hash, - &inode_hashtable[(st->st_ino ^ st->st_dev) % -NR_INODE_HASHTABLE ]); +erofs_insert_ihash (inode, st->st_dev, st->st_ino);return 0 ; }
Signature · 1 line
2.19.1.6.gb485710b
Re: [PATCH v6 05/11] erofs-utils: lib: add erofs_insert_ihash() helper
Quoted 6 lines · Jingbo Xu
Add
erofs_insert_ihash()helper inserting inode into inode hash table.Also add prototypes of
erofs_iget()anderofs_iget_by_nid()in the header file.Signed-off-by Jingbo Xu <[email protected]>
Thanks,
Gao Xiang
PATCH v6 6/11 erofs-utils: lib: add erofs_rebuild_get_dentry() helper
In reply to Jingbo XuRename tarerofs_get_dentry() to erofs_rebuild_get_dentry().
Also make whout and 'opq' parameter optional when aufs is false.
--- include/erofs/rebuild.h | 19 +++++++ lib/Makefile.am | 3 +- lib/rebuild.c | 117 ++++++++++++++++++++++++++++++++++++++++ lib/tar.c | 109 ++----------------------------------- 4 files changed, 141 insertions(+), 107 deletions(-) create mode 100644 include/erofs/rebuild.h create mode 100644 lib/rebuild.c
include/erofs/rebuild.h new file +19 -0
@@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ +#ifndef __EROFS_REBUILD_H +#define __EROFS_REBUILD_H + +#ifdef __cplusplus +extern "C" +{ +#endif + +#include "internal.h" + +struct erofs_dentry *erofs_rebuild_get_dentry (struct erofs_inode *pwd, +char *path,bool aufs,bool *whout,bool *opq); + +#ifdef __cplusplus +} +#endif + +#endif
lib/Makefile.am +2 -1
@@ -25,6 +25,7 @@ noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \ $(top_srcdir)/include/erofs/compress_hints.h \ $(top_srcdir)/include/erofs/fragments.h \ $(top_srcdir)/include/erofs/xxhash.h \ + $(top_srcdir)/include/erofs/rebuild.h \ $(top_srcdir)/lib/liberofs_private.h noinst_HEADERS += compressor.h
@@ -32,7 +33,7 @@ liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \ namei.c data.c compress.c compressor.c zmap.c decompress.c \ compress_hints.c hashmap.c sha256.c blobchunk.c dir.c \ fragments.c rb_tree.c dedupe.c uuid_unparse.c uuid.c tar.c \ - block_list.c xxhash.c + block_list.c xxhash.c rebuild.c liberofs_la_CFLAGS = -Wall ${libuuid_CFLAGS} -I$(top_srcdir)/include if ENABLE_LZ4
lib/rebuild.c new file +117 -0
117 lines not on this page · raw message
lib/tar.c +3 -106
139 lines not on this page · raw message
Signature · 1 line
2.19.1.6.gb485710b
Re: [PATCH v6 06/11] erofs-utils: lib: add erofs_rebuild_get_dentry() helper
Quoted 111 lines · Jingbo Xu
Rename
tarerofs_get_dentry()toerofs_rebuild_get_dentry().Also make
whoutand 'opq' parameter optional whenaufsis false.Signed-off-by Jingbo Xu <[email protected]>--- include/erofs/rebuild.h | 19 +++++++ lib/Makefile.am | 3 +- lib/rebuild.c | 117 ++++++++++++++++++++++++++++++++++++++++ lib/tar.c | 109 ++----------------------------------- 4 files changed, 141 insertions(+), 107 deletions(-)create mode 100644
include/erofs/rebuild.h
create mode 100644lib/rebuild.cinclude/erofs/rebuild.h new file +19
-0@@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ +#ifndef __EROFS_REBUILD_H +#define __EROFS_REBUILD_H + +#ifdef __cplusplus +extern "C" +{ +#endif + +#include "internal.h" + +struct erofs_dentry *erofs_rebuild_get_dentry (struct erofs_inode *pwd, +char *path,bool aufs,bool *whout,bool *opq); + +#ifdef __cplusplus +} +#endif + +#endif lib/Makefile.am +2
-1@@ -25,6 +25,7 @@ noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \ $(top_srcdir)/include/erofs/compress_hints.h \ $(top_srcdir)/include/erofs/fragments.h \ $(top_srcdir)/include/erofs/xxhash.h \ + $(top_srcdir)/include/erofs/rebuild.h \ $(top_srcdir)/lib/liberofs_private.h noinst_HEADERS += compressor.h@@ -32,7 +33,7 @@ liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \ namei.c data.c compress.c compressor.c zmap.c decompress.c \ compress_hints.c hashmap.c sha256.c blobchunk.c dir.c \ fragments.c rb_tree.c dedupe.c uuid_unparse.c uuid.c tar.c \ - block_list.c xxhash.c + block_list.c xxhash.c rebuild.c liberofs_la_CFLAGS = -Wall ${libuuid_CFLAGS} -I$(top_srcdir)/include if ENABLE_LZ4lib/rebuild.c new file +45
-0@@ -0,0 +1,117 @@ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 +#define _GNU_SOURCE +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include "erofs/print.h" +#include "erofs/inode.h" +#include "erofs/rebuild.h" +#include "erofs/internal.h" + +#ifdef HAVE_LINUX_AUFS_TYPE_H +#include <linux/aufs_type.h> +#else +#define AUFS_WH_PFX ".wh." +#define AUFS_DIROPQ_NAME AUFS_WH_PFX ".opq" +#define AUFS_WH_DIROPQ AUFS_WH_PFX AUFS_DIROPQ_NAME +#endif + +static struct erofs_dentry *erofs_rebuild_mkdir (struct erofs_inode *dir, +const char *s) +{ +struct erofs_inode *inode; +struct erofs_dentry *d; + + inode =erofs_new_inode (); +if (IS_ERR (inode)) +return ERR_CAST (inode); + + inode->i_mode =S_IFDIR |0755 ; + inode->i_parent = dir; + inode->i_uid =getuid (); + inode->i_gid =getgid (); + inode->i_mtime = inode->sbi->build_time; + inode->i_mtime_nsec = inode->sbi->build_time_nsec; +erofs_init_empty_dir (inode); + + d =erofs_d_alloc (dir, s); +if (!IS_ERR (d)) { + d->type =EROFS_FT_DIR ; + d->inode = inode; + } +return d; +} + +struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd,
lib/rebuild.c line 55
Quoted 10 lines
lib/rebuild.c new file +10
-0+ char *path, bool aufs, bool *whout, bool *opq) +{ + struct erofs_dentry *d = NULL; +unsigned int len =strlen (path); +char *s = path; + +if (aufs) { + *whout = false; + *opq = false; + }
I'd suggest adding a boolean dumb instead here, otherwise it looks good to me.
Thanks,
Gao Xiang
PATCH v6 7/11 erofs-utils: lib: add erofs_rebuild_load_tree() helper
In reply to Jingbo XuAdd erofs_rebuild_load_tree() helper loading inode tree from given erofs image, and making it merged into a given inode tree in an overlayfs like model.
Since we need to read the content of the symlink file from disk when loading tree, add dependency on zlib_LIBS for mkfs.erofs.
Also rename tarerofs_dump_tree() to erofs_rebuild_dump_tree(), so that it could also be called from rebuild mode.
--- include/erofs/inode.h | 2 +- include/erofs/internal.h | 1 + include/erofs/rebuild.h | 2 + lib/inode.c | 4 +- lib/rebuild.c | 284 +++++++++++++++++++++++++++++++++++++++ mkfs/main.c | 2 +- 6 files changed, 291 insertions(+), 4 deletions(-)
include/erofs/inode.h +1 -1
@@ -32,7 +32,7 @@ unsigned int erofs_iput(struct erofs_inode *inode);erofs_nid_t erofs_lookupnid (struct erofs_inode *inode);struct erofs_dentry *erofs_d_alloc (struct erofs_inode *parent,const char *name); -int tarerofs_dump_tree (struct erofs_inode *dir); +int erofs_rebuild_dump_tree (struct erofs_inode *dir);int erofs_init_empty_dir (struct erofs_inode *dir);struct erofs_inode *erofs_new_inode (void );struct erofs_inode *erofs_mkfs_build_tree_from_path (const char *path);
include/erofs/internal.h +1 -0
@@ -108,6 +108,7 @@ struct erofs_sb_info {int devfd;u64 devsz; +dev_t dev;unsigned int nblobs;unsigned int blobfd[256 ]; };
include/erofs/rebuild.h +2 -0
@@ -12,6 +12,8 @@ extern "C"struct erofs_dentry *erofs_rebuild_get_dentry (struct erofs_inode *pwd,char *path,bool aufs,bool *whout,bool *opq); +int erofs_rebuild_load_tree (struct erofs_inode *root,struct erofs_sb_info *sbi); + #ifdef __cplusplus }#endif
lib/inode.c +2 -2
@@ -1324,7 +1324,7 @@ struct erofs_inode *erofs_mkfs_build_special_from_fd(int fd, const char *name)return inode; } -int tarerofs_dump_tree (struct erofs_inode *dir) +int erofs_rebuild_dump_tree (struct erofs_inode *dir) {struct erofs_dentry *d;unsigned int nr_subdirs;
@@ -1392,7 +1392,7 @@ int tarerofs_dump_tree(struct erofs_inode *dir)continue ; inode =erofs_igrab (d->inode); - ret =tarerofs_dump_tree (inode); + ret =erofs_rebuild_dump_tree (inode); dir->i_nlink += (erofs_mode_to_ftype (inode->i_mode) ==EROFS_FT_DIR );erofs_iput (inode);if (ret)
lib/rebuild.c +284 -0
296 lines not on this page · raw message
mkfs/main.c +1 -1
@@ -962,7 +962,7 @@ int main(int argc, char **argv)if (err <0 ) goto exit; - err =tarerofs_dump_tree (root_inode); + err =erofs_rebuild_dump_tree (root_inode);if (err <0 ) goto exit; }
Signature · 1 line
2.19.1.6.gb485710b
PATCH v6 8/11 erofs-utils: mkfs: introduce rebuild mode
In reply to Jingbo XuIntroduce a new rebuild mode merging multiple erofs images generated from either tarerfs index mode (--tar=i):
mkfs.erofs --tar=i --aufs layer0.erofs layer0.tar ... mkfs.erofs --tar=i --aufs layerN.erofs layerN.tar mkfs.erofs merge.erofs layerN.erofs ... layer0.erofs
or tarerofs non-index mode (--tar=f):
mkfs.erofs --tar=f -Enoinline_data --aufs layer0.erofs layer0.tar ... mkfs.erofs --tar=f -Enoinline_data --aufs layerN.erofs layerN.tar mkfs.erofs merge.erofs layerN.erofs ... layer0.erofs
When calling rebuild mode, the top most layer must be specified firstly in the cmdline, while the bottom most layer i.e. layer0 the lastly.
The rebuild mode doesn't support flat inline datalayout yet.
--- include/erofs/internal.h | 2 + mkfs/main.c | 216 ++++++++++++++++++++++++++++++--------- 2 files changed, 170 insertions(+), 48 deletions(-)
include/erofs/internal.h +2 -0
@@ -111,6 +111,8 @@ struct erofs_sb_info {dev_t dev;unsigned int nblobs;unsigned int blobfd[256 ]; + +struct list_head list; };/* make sure that any user of the erofs headers has atleast 64bit off_t type */
mkfs/main.c +168 -48
297 lines not on this page · raw message
Signature · 1 line
2.19.1.6.gb485710b
PATCH v6 9/11 erofs-utils: lib: add erofs_inode_is_whiteout() helper
In reply to Jingbo XuAdd erofs_inode_is_whiteout() helper checking if given inode is a whiteout.
--- include/erofs/internal.h | 7 +++++++ lib/tar.c | 2 -- 2 files changed, 7 insertions(+), 2 deletions(-)
include/erofs/internal.h +7 -0
@@ -20,6 +20,7 @@ typedef unsigned short umode_t;#include "erofs_fs.h" #include <fcntl.h> #include <sys/types.h> /* for off_t definition */ +#include <sys/stat.h> /* for S_ISCHR definition */ #include <stdio.h> #ifndef PATH_MAX
@@ -427,6 +428,12 @@ static inline u32 erofs_crc32c(u32 crc, const u8 *in, size_t len)return crc; } +#define EROFS_WHITEOUT_DEV 0 +static inline bool erofs_inode_is_whiteout (struct erofs_inode *inode) +{ +return S_ISCHR (inode->i_mode) && inode->u.i_rdev ==EROFS_WHITEOUT_DEV ; +} + #ifdef __cplusplus }#endif
lib/tar.c +0 -2
@@ -13,8 +13,6 @@#include "erofs/blobchunk.h" #include "erofs/rebuild.h" -#define EROFS_WHITEOUT_DEV 0 -static char erofs_libbuf[16384 ];struct tar_header {
Signature · 1 line
2.19.1.6.gb485710b
PATCH v6 10/11 erofs-utils: lib: set origin xattr on parent directory of whiteout
In reply to Jingbo XuWhiteouts will be exposed in the overlayfs mount if the whiteouts are from non-merged directory without origin xattr set on the directory.
To fix this, set origin xattr (with empty value) on parent directory of whiteouts, marking the parent directory as copied-up to avoid exposing whiteouts in the overlayfs mount.
To avoid setting multiple origin xattrs on the same directory when there are multiple whiteouts under the directory, enhance erofs_setxattr() so that it could skip duplicate xattrs.
--- include/erofs/xattr.h | 3 ++- lib/rebuild.c | 6 ++++++ lib/tar.c | 9 ++++++++- lib/xattr.c | 27 ++++++++++++++++++++++++--- 4 files changed, 40 insertions(+), 5 deletions(-)
include/erofs/xattr.h +2 -1
@@ -55,8 +55,9 @@ void erofs_xattr_prefixes_cleanup(struct erofs_sb_info *sbi);int erofs_xattr_prefixes_init (struct erofs_sb_info *sbi);int erofs_setxattr (struct erofs_inode *inode,char *key, -const void *value,size_t size); +const void *value,size_t size,bool nodup);int erofs_set_opaque_xattr (struct erofs_inode *inode); +int erofs_set_origin_xattr (struct erofs_inode *inode);int erofs_read_xattrs_from_disk (struct erofs_inode *inode);#ifdef __cplusplus
lib/rebuild.c +6 -0
@@ -200,6 +200,12 @@ static int erofs_rebuild_fill_inode(struct erofs_inode *inode)switch (inode->i_mode &S_IFMT ) {case S_IFCHR : +if (erofs_inode_is_whiteout (inode)) { + ret =erofs_set_origin_xattr (inode->i_parent); +if (ret) +break ; + } +/* fallthrough */ case S_IFBLK :case S_IFIFO :case S_IFSOCK :
lib/tar.c +8 -1
@@ -206,7 +206,7 @@ int tarerofs_apply_xattrs(struct erofs_inode *inode, struct list_head *xattrs) item->kv[item->namelen] ='\0' ;erofs_dbg ("Recording xattr(%s)=\"%s\" (of %u bytes) to file %s" , item->kv, v, vsz, inode->i_srcpath); - ret =erofs_setxattr (inode, item->kv, v, vsz); + ret =erofs_setxattr (inode, item->kv, v, vsz, false);if (ret == -ENODATA )erofs_err ("Failed to set xattr(%s)=%s to file %s" , item->kv, v, inode->i_srcpath);
@@ -740,6 +740,13 @@ new_inode: inode->i_mode = (inode->i_mode & ~S_IFMT ) |S_IFCHR ; inode->u.i_rdev =EROFS_WHITEOUT_DEV ; d->type =EROFS_FT_CHRDEV ; +/* +* Mark parent directory as copied-up to avoid exposing +* whiteouts in the overlayfs mount. +*/ + ret =erofs_set_origin_xattr (inode->i_parent); +if (ret) + goto out; }else { inode->i_mode = st.st_mode;if (S_ISBLK (st.st_mode) ||S_ISCHR (st.st_mode))
lib/xattr.c +24 -3
@@ -57,12 +57,18 @@#ifndef OVL_XATTR_OPAQUE_POSTFIX #define OVL_XATTR_OPAQUE_POSTFIX "opaque"#endif +#ifndef OVL_XATTR_ORIGIN_POSTFIX +#define OVL_XATTR_ORIGIN_POSTFIX "origin" +#endif #ifndef OVL_XATTR_TRUSTED_PREFIX #define OVL_XATTR_TRUSTED_PREFIX XATTR_TRUSTED_PREFIX OVL_XATTR_NAMESPACE#endif #ifndef OVL_XATTR_OPAQUE #define OVL_XATTR_OPAQUE OVL_XATTR_TRUSTED_PREFIX OVL_XATTR_OPAQUE_POSTFIX#endif +#ifndef OVL_XATTR_ORIGIN +#define OVL_XATTR_ORIGIN OVL_XATTR_TRUSTED_PREFIX OVL_XATTR_ORIGIN_POSTFIX +#endif #define EA_HASHTABLE_BITS 16
@@ -455,11 +461,12 @@ err: }int erofs_setxattr (struct erofs_inode *inode,char *key, -const void *value,size_t size) +const void *value,size_t size,bool nodup) {char *kvbuf;unsigned int len[2 ];struct xattr_item *item; +struct inode_xattr_node *node;u8 prefix;u16 prefixlen;
@@ -481,12 +488,26 @@ int erofs_setxattr(struct erofs_inode *inode, char *key,if (IS_ERR (item))return PTR_ERR (item); +if (nodup) { +list_for_each_entry (node, &inode->i_xattrs, list) { +if (node->item == item) { +put_xattritem (item); +return 0 ; + } + } + } +return erofs_xattr_add (&inode->i_xattrs, item); }int erofs_set_opaque_xattr (struct erofs_inode *inode) { -return erofs_setxattr (inode,OVL_XATTR_OPAQUE ,"y" ,1 ); +return erofs_setxattr (inode,OVL_XATTR_OPAQUE ,"y" ,1 , false); +} + +int erofs_set_origin_xattr (struct erofs_inode *inode) +{ +return erofs_setxattr (inode,OVL_XATTR_ORIGIN ,NULL ,0 , true); }#ifdef WITH_ANDROID
@@ -599,7 +620,7 @@ int erofs_read_xattrs_from_disk(struct erofs_inode *inode)DBG_BUGON (ret != size); } - ret =erofs_setxattr (inode, key, value, size); + ret =erofs_setxattr (inode, key, value, size, false);free (value);if (ret)break ;
Signature · 1 line
2.19.1.6.gb485710b
PATCH v6 11/11 erofs-utils: mkfs: add --overlay-strip option for tarfs and rebuild mode
In reply to Jingbo XuAdd "--overlay-strip=[0|1]" option for tarfs and rebuild mode, controlling whether overlayfs related stuff, e.g. whiteout files and origin xattrs, are stripped from the generated image.
This option is disabled by default for both tarfs and rebuild mode, that is, whiteout files and origin xattrs are kept in the image by default. Specify "--overlay-strip=1" explicitly to strip these stuff.
--- include/erofs/config.h | 1 + lib/inode.c | 12 ++++++++++-- lib/rebuild.c | 10 ++++++---- lib/tar.c | 16 +++++++++------- lib/xattr.c | 5 +++++ mkfs/main.c | 12 ++++++++++++ 6 files changed, 43 insertions(+), 13 deletions(-)
include/erofs/config.h +1 -0
@@ -54,6 +54,7 @@ struct erofs_configure {bool c_showprogress;bool c_extra_ea_name_prefixes;bool c_xattr_name_filter; +bool c_overlay_strip;#ifdef HAVE_LIBSELINUX struct selabel_handle *sehnd;
lib/inode.c +10 -2
@@ -1326,7 +1326,7 @@ struct erofs_inode *erofs_mkfs_build_special_from_fd(int fd, const char *name)int erofs_rebuild_dump_tree (struct erofs_inode *dir) { -struct erofs_dentry *d; +struct erofs_dentry *d, *n;unsigned int nr_subdirs;int ret;
@@ -1370,8 +1370,16 @@ int erofs_rebuild_dump_tree(struct erofs_inode *dir) } nr_subdirs =0 ; -list_for_each_entry (d, &dir->i_subdirs, d_child) +list_for_each_entry_safe (d, n, &dir->i_subdirs, d_child) { +if (cfg.c_overlay_strip &&erofs_inode_is_whiteout (d->inode)) { +erofs_dbg ("strip whiteout %s" , d->inode->i_srcpath); +list_del (&d->d_child); +erofs_d_invalidate (d); +free (d); +continue ; + } ++nr_subdirs; + } ret =erofs_prepare_dir_layout (dir, nr_subdirs);if (ret)
lib/rebuild.c +6 -4
@@ -200,10 +200,12 @@ static int erofs_rebuild_fill_inode(struct erofs_inode *inode)switch (inode->i_mode &S_IFMT ) {case S_IFCHR : -if (erofs_inode_is_whiteout (inode)) { - ret =erofs_set_origin_xattr (inode->i_parent); -if (ret) -break ; +if (!cfg.c_overlay_strip) { +if (erofs_inode_is_whiteout (inode)) { + ret =erofs_set_origin_xattr (inode->i_parent); +if (ret) +break ; + } }/* fallthrough */ case S_IFBLK :
lib/tar.c +9 -7
@@ -740,13 +740,15 @@ new_inode: inode->i_mode = (inode->i_mode & ~S_IFMT ) |S_IFCHR ; inode->u.i_rdev =EROFS_WHITEOUT_DEV ; d->type =EROFS_FT_CHRDEV ; -/* -* Mark parent directory as copied-up to avoid exposing -* whiteouts in the overlayfs mount. -*/ - ret =erofs_set_origin_xattr (inode->i_parent); -if (ret) - goto out; +if (!cfg.c_overlay_strip) { +/* +* Mark parent directory as copied-up to avoid exposing +* whiteouts in the overlayfs mount. +*/ + ret =erofs_set_origin_xattr (inode->i_parent); +if (ret) + goto out; + } }else { inode->i_mode = st.st_mode;if (S_ISBLK (st.st_mode) ||S_ISCHR (st.st_mode))
lib/xattr.c +5 -0
@@ -601,6 +601,11 @@ int erofs_read_xattrs_from_disk(struct erofs_inode *inode) inode->opaque = true; } +if (cfg.c_overlay_strip && !strcmp (key,OVL_XATTR_ORIGIN )) { + ret =0 ; +continue ; + } + ret =erofs_getxattr (inode, key,NULL ,0 );if (ret <0 ) goto out;
mkfs/main.c +12 -0
@@ -57,6 +57,7 @@ static struct option long_options[] = { {"gid-offset", required_argument, NULL, 17}, {"tar", optional_argument, NULL, 20}, {"aufs", no_argument, NULL, 21}, + {"overlay-strip", required_argument, NULL, 22}, {"mount-point", required_argument, NULL, 512}, {"xattr-prefix", required_argument, NULL, 19}, #ifdef WITH_ANDROID
@@ -121,6 +122,7 @@ static void usage(void)" --preserve-mtime keep per-file modification time strictly\n" " --aufs replace aufs special files with overlayfs metadata\n" " --tar=[fi] generate an image from tarball(s)\n" +" --overlay-strip=[01] strip overlayfs stuff e.g. whiteout in tarfs or rebuild mode\n" " --quiet quiet execution (do not write anything to standard output.)\n" #ifndef NDEBUG" --random-pclusterblks randomize pclusterblks for big pcluster (debugging only)\n"
@@ -538,6 +540,16 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])case 21 : erofstar.aufs = true;break ; +case 22 : +if (!strcmp (optarg,"0" )) + cfg.c_overlay_strip = false; +else if (!strcmp (optarg,"1" )) + cfg.c_overlay_strip = true; +else { +erofs_err ("invalid overlay-strip %s" , optarg); +return -EINVAL ; + } +break ;case 1 :usage ();exit (0 );
Signature · 1 line
2.19.1.6.gb485710b