erofs-utils: mkfs: introduce rebuild mode
v4 cover erofs-utils: mkfs: introduce rebuild mode
v4:
- patch 2: the original two v3 patches[1][2] are merged as patch 2; also update commit message
- patch 10: newly added; introduce "--keep-whiteout=[0|1]" option for tarfs and rebuild mode controlling whether whiteout files are kept in the generated image.
- patch 9: add documentation for tarfs and rebuild mode in the usage message of mkfs.erofs
[1] https://lore.kernel.org/all/[email protected]/
[2] https://lore.kernel.org/all/[email protected]/
v1: https://lore.kernel.org/all/[email protected]/
v2: https://lore.kernel.org/all/[email protected]/
v3: https://lore.kernel.org/all/[email protected]/
-------------------------
Introduce a new rebuild mode merging multiple erofs images generated from either tarerfs index mode (--tar=i):
mkfs.erofs --tar=i --aufs layer0.erofs layer0.tar ... mkfs.erofs --tar=i --aufs layerN.erofs layerN.tar mkfs.erofs merge.erofs layerN.erofs ... layer0.erofs
or tarerofs non-index mode (--tar=f):
mkfs.erofs --tar=f -Enoinline_data --aufs layer0.erofs layer0.tar ... mkfs.erofs --tar=f -Enoinline_data --aufs layerN.erofs layerN.tar mkfs.erofs merge.erofs layerN.erofs ... layer0.erofs
Jingbo Xu (10):
erofs-utils: lib: fix dirent type of whiteout in tarerofs
erofs-utils: lib: keep erofs_init_devices in sync with kernel
erofs-utils: lib: add list_splice_tail() helper
erofs-utils: lib: make erofs_get_unhashed_chunk() global
erofs-utils: lib: add erofs_read_xattrs_from_disk() helper
erofs-utils: lib: add erofs_insert_ihash() helper
erofs-utils: lib: add erofs_rebuild_get_dentry() helper
erofs-utils: lib: add erofs_rebuild_load_tree() helper
erofs-utils: mkfs: introduce rebuild mode
erofs-utils: mkfs: add --keep-whiteout option for tarfs and rebuild
mode
include/erofs/blobchunk.h | 2 + include/erofs/config.h | 1 + include/erofs/inode.h | 5 +- include/erofs/internal.h | 6 + include/erofs/list.h | 20 ++ include/erofs/rebuild.h | 21 +++ include/erofs/xattr.h | 33 +--- lib/Makefile.am | 3 +- lib/blobchunk.c | 2 +- lib/config.c | 1 + lib/inode.c | 31 ++- lib/rebuild.c | 388 ++++++++++++++++++++++++++++++++++++++ lib/super.c | 9 +- lib/tar.c | 119 +----------- lib/xattr.c | 112 +++++++++++ mkfs/main.c | 221 +++++++++++++++++----- 16 files changed, 772 insertions(+), 202 deletions(-) create mode 100644 include/erofs/rebuild.h create mode 100644 lib/rebuild.c
Signature · 1 line
2.19.1.6.gb485710b
PATCH v4 1/10 erofs-utils: lib: fix dirent type of whiteout in tarerofs
Set the correct dirent type for whiteout.
--- lib/tar.c | 1 + 1 file changed, 1 insertion(+)
lib/tar.c +1 -0
@@ -694,6 +694,7 @@ new_inode:if (whout) { inode->i_mode = (inode->i_mode & ~S_IFMT ) |S_IFCHR ; inode->u.i_rdev =EROFS_WHITEOUT_DEV ; + d->type =EROFS_FT_CHRDEV ; }else { inode->i_mode = st.st_mode;if (S_ISBLK (st.st_mode) ||S_ISCHR (st.st_mode))
Signature · 1 line
2.19.1.6.gb485710b
Re: [PATCH v4 01/10] erofs-utils: lib: fix dirent type of whiteout in tarerofs
Quoted 4 lines · Jingbo Xu
Set the correct dirent type for whiteout.
Fixes 95d315fd7958 ("erofs-utils: introduce tarerofs")Signed-off-by Jingbo Xu <[email protected]>
Thanks,
Gao Xiang
PATCH v4 2/10 erofs-utils: lib: keep erofs_init_devices in sync with kernel
In reply to Jingbo XuKeep erofs_init_devices() in sync with kernel erofs_scan_devices()[1], which scans the devtable (if any) automatically if sbi->extra_devices is not explicitly specified.
Also fix the missing le32_to_cpu() when parsing the device slot. Read and cache the number of blocks of each device for later use.
[1] https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=ba73eadd23d1c
--- lib/super.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-)
lib/super.c +6 -3
@@ -37,7 +37,8 @@ static int erofs_init_devices(struct erofs_sb_info *sbi,else ondisk_extradevs =le16_to_cpu (dsb->extra_devices); -if (ondisk_extradevs != sbi->extra_devices) { +if (sbi->extra_devices && + ondisk_extradevs != sbi->extra_devices) {erofs_err ("extra devices don't match (ondisk %u, given %u)" , ondisk_extradevs, sbi->extra_devices);return -EINVAL ;
@@ -45,6 +46,7 @@ static int erofs_init_devices(struct erofs_sb_info *sbi,if (!ondisk_extradevs)return 0 ; + sbi->extra_devices = ondisk_extradevs; sbi->device_id_mask =roundup_pow_of_two (ondisk_extradevs +1 ) -1 ; sbi->devs =calloc (ondisk_extradevs,sizeof (*sbi->devs));if (!sbi->devs)
@@ -60,8 +62,9 @@ static int erofs_init_devices(struct erofs_sb_info *sbi,return ret; } - sbi->devs[i].mapped_blkaddr = dis.mapped_blkaddr; - sbi->total_blocks += dis.blocks; + sbi->devs[i].mapped_blkaddr =le32_to_cpu (dis.mapped_blkaddr); + sbi->devs[i].blocks =le32_to_cpu (dis.blocks); + sbi->total_blocks += sbi->devs[i].blocks; pos +=EROFS_DEVT_SLOT_SIZE ; }return 0 ;
Signature · 1 line
2.19.1.6.gb485710b
Re: [PATCH v4 02/10] erofs-utils: lib: keep erofs_init_devices in sync with kernel
Quoted 10 lines · Jingbo Xu
Keep
erofs_init_devices()in sync with kernel erofs_scan_devices()[1], which scans the devtable (if any) automatically ifsbi->extra_devicesis not explicitly specified.Also fix the missing
le32_to_cpu()when parsing the device slot. Read and cache the number of blocks of each device for later use.Fixes 0ce853a01123 ("erofs-utils: fuse: add multiple device support")[1] https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=ba73eadd23d1c
Signed-off-by Jingbo Xu <[email protected]>
Thanks,
Gao Xiang
PATCH v4 3/10 erofs-utils: lib: add list_splice_tail() helper
In reply to Jingbo XuAdd list_splice_tail() helper.
--- include/erofs/list.h | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+)
include/erofs/list.h +20 -0
@@ -70,6 +70,26 @@ static inline int list_empty(struct list_head *head)return head->next == head; } +static inline void __list_splice (struct list_head *list, +struct list_head *prev,struct list_head *next) +{ +struct list_head *first = list->next; +struct list_head *last = list->prev; + + first->prev = prev; + prev->next = first; + + last->next = next; + next->prev = last; +} + +static inline void list_splice_tail (struct list_head *list, +struct list_head *head) +{ +if (!list_empty (list)) +__list_splice (list, head->prev, head); +} +#define list_entry (ptr, type, member) container_of(ptr, type, member)#define list_first_entry (ptr, type, member) \
Signature · 1 line
2.19.1.6.gb485710b
PATCH v4 4/10 erofs-utils: lib: make erofs_get_unhashed_chunk() global
In reply to Jingbo Xu... so that it could be called from outside blobchunk.c later.
--- include/erofs/blobchunk.h | 2 ++ lib/blobchunk.c | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-)
include/erofs/blobchunk.h +2 -0
@@ -14,6 +14,8 @@ extern "C"#include "erofs/internal.h" +struct erofs_blobchunk *erofs_get_unhashed_chunk (unsigned int device_id, +erofs_blk_t blkaddr,erofs_off_t sourceoffset);int erofs_blob_write_chunk_indexes (struct erofs_inode *inode,erofs_off_t off);int erofs_blob_write_chunked_file (struct erofs_inode *inode,int fd);int tarerofs_write_chunkes (struct erofs_inode *inode,erofs_off_t data_offset);
lib/blobchunk.c +1 -1
@@ -38,7 +38,7 @@ struct erofs_blobchunk erofs_holechunk = { };static LIST_HEAD (unhashed_blobchunks); -static struct erofs_blobchunk *erofs_get_unhashed_chunk (unsigned int device_id, +struct erofs_blobchunk *erofs_get_unhashed_chunk (unsigned int device_id,erofs_blk_t blkaddr,erofs_off_t sourceoffset) {struct erofs_blobchunk *chunk;
Signature · 1 line
2.19.1.6.gb485710b
PATCH v4 5/10 erofs-utils: lib: add erofs_read_xattrs_from_disk() helper
In reply to Jingbo XuAdd erofs_read_xattrs_from_disk() helper reading extended attributes from disk, add checking if it's an opaque directory.
Move all xattr name related macros to xattr.c and introduce erofs_set_opaque_xattr() helper to hide all these details.
--- include/erofs/internal.h | 1 + include/erofs/xattr.h | 33 +----------- lib/tar.c | 7 +-- lib/xattr.c | 112 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 116 insertions(+), 37 deletions(-)
include/erofs/internal.h +1 -0
@@ -190,6 +190,7 @@ struct erofs_inode {bool compressed_idata;bool lazy_tailblock;bool with_tmpfile; +bool opaque;unsigned int xattr_isize;unsigned int extent_isize;
include/erofs/xattr.h +2 -31
@@ -43,37 +43,6 @@ static inline unsigned int xattrblock_offset(struct erofs_inode *vi, (_size -sizeof (struct erofs_xattr_ibody_header )) / \ sizeof(struct erofs_xattr_entry) + 1; }) -#ifndef XATTR_SYSTEM_PREFIX -#define XATTR_SYSTEM_PREFIX "system." -#endif -#ifndef XATTR_SYSTEM_PREFIX_LEN -#define XATTR_SYSTEM_PREFIX_LEN (sizeof(XATTR_SYSTEM_PREFIX) - 1) -#endif -#ifndef XATTR_USER_PREFIX -#define XATTR_USER_PREFIX "user." -#endif -#ifndef XATTR_USER_PREFIX_LEN -#define XATTR_USER_PREFIX_LEN (sizeof(XATTR_USER_PREFIX) - 1) -#endif -#ifndef XATTR_SECURITY_PREFIX -#define XATTR_SECURITY_PREFIX "security." -#endif -#ifndef XATTR_SECURITY_PREFIX_LEN -#define XATTR_SECURITY_PREFIX_LEN (sizeof(XATTR_SECURITY_PREFIX) - 1) -#endif -#ifndef XATTR_TRUSTED_PREFIX -#define XATTR_TRUSTED_PREFIX "trusted." -#endif -#ifndef XATTR_TRUSTED_PREFIX_LEN -#define XATTR_TRUSTED_PREFIX_LEN (sizeof(XATTR_TRUSTED_PREFIX) - 1) -#endif -#ifndef XATTR_NAME_POSIX_ACL_ACCESS -#define XATTR_NAME_POSIX_ACL_ACCESS "system.posix_acl_access" -#endif -#ifndef XATTR_NAME_POSIX_ACL_DEFAULT -#define XATTR_NAME_POSIX_ACL_DEFAULT "system.posix_acl_default" -#endif -int erofs_scan_file_xattrs (struct erofs_inode *inode);int erofs_prepare_xattr_ibody (struct erofs_inode *inode);char *erofs_export_xattr_ibody (struct list_head *ixattrs,unsigned int size);
@@ -87,6 +56,8 @@ int erofs_xattr_prefixes_init(struct erofs_sb_info *sbi);int erofs_setxattr (struct erofs_inode *inode,char *key,const void *value,size_t size); +int erofs_set_opaque_xattr (struct erofs_inode *inode); +int erofs_read_xattrs_from_disk (struct erofs_inode *inode); #ifdef __cplusplus }
lib/tar.c +1 -6
@@ -19,11 +19,6 @@#include "erofs/xattr.h" #include "erofs/blobchunk.h" -#define OVL_XATTR_NAMESPACE "overlay." -#define OVL_XATTR_TRUSTED_PREFIX XATTR_TRUSTED_PREFIX OVL_XATTR_NAMESPACE -#define OVL_XATTR_OPAQUE_POSTFIX "opaque" -#define OVL_XATTR_OPAQUE OVL_XATTR_TRUSTED_PREFIX OVL_XATTR_OPAQUE_POSTFIX -#define EROFS_WHITEOUT_DEV 0static char erofs_libbuf[16384 ];
@@ -633,7 +628,7 @@ restart: }else if (opq) {DBG_BUGON (d->type ==EROFS_FT_UNKNOWN );DBG_BUGON (!d->inode); - ret =erofs_setxattr (d->inode,OVL_XATTR_OPAQUE ,"y" ,1 ); + ret =erofs_set_opaque_xattr (d->inode); goto out; }else if (th.typeflag =='1' ) {/* hard link cases */ struct erofs_dentry *d2;
lib/xattr.c +112 -0
130 lines not on this page · raw message
Signature · 1 line
2.19.1.6.gb485710b
PATCH v4 6/10 erofs-utils: lib: add erofs_insert_ihash() helper
In reply to Jingbo XuAdd erofs_insert_ihash() helper inserting inode into inode hash table.
Also add prototypes of erofs_iget() and erofs_iget_by_nid() in the header file.
--- include/erofs/inode.h | 3 +++ lib/inode.c | 10 +++++++--- 2 files changed, 10 insertions(+), 3 deletions(-)
include/erofs/inode.h +3 -0
@@ -25,6 +25,9 @@ u32 erofs_new_encode_dev(dev_t dev);unsigned char erofs_mode_to_ftype (umode_t mode);unsigned char erofs_ftype_to_dtype (unsigned int filetype);void erofs_inode_manager_init (void ); +void erofs_insert_ihash (struct erofs_inode *inode,dev_t dev,ino_t ino); +struct erofs_inode *erofs_iget (dev_t dev,ino_t ino); +struct erofs_inode *erofs_iget_by_nid (erofs_nid_t nid);unsigned int erofs_iput (struct erofs_inode *inode);erofs_nid_t erofs_lookupnid (struct erofs_inode *inode); struct erofs_dentry *erofs_d_alloc(struct erofs_inode *parent,
lib/inode.c +7 -3
@@ -75,6 +75,12 @@ void erofs_inode_manager_init(void)init_list_head (&inode_hashtable[i]); } +void erofs_insert_ihash (struct erofs_inode *inode,dev_t dev,ino_t ino) +{ +list_add (&inode->i_hash, + &inode_hashtable[(ino ^ dev) %NR_INODE_HASHTABLE ]); +} +/* get the inode from the (source) inode # */ struct erofs_inode *erofs_iget (dev_t dev,ino_t ino) {
@@ -976,9 +982,7 @@ static int erofs_fill_inode(struct erofs_inode *inode, struct stat *st, inode->inode_isize =sizeof (struct erofs_inode_compact ); } -list_add (&inode->i_hash, - &inode_hashtable[(st->st_ino ^ st->st_dev) % -NR_INODE_HASHTABLE ]); +erofs_insert_ihash (inode, st->st_dev, st->st_ino);return 0 ; }
Signature · 1 line
2.19.1.6.gb485710b
PATCH v4 7/10 erofs-utils: lib: add erofs_rebuild_get_dentry() helper
In reply to Jingbo XuRename tarerofs_get_dentry() to erofs_rebuild_get_dentry().
Also make whout and 'opq' parameter optional when aufs is false.
--- include/erofs/rebuild.h | 19 +++++++ lib/Makefile.am | 3 +- lib/rebuild.c | 117 ++++++++++++++++++++++++++++++++++++++++ lib/tar.c | 109 ++----------------------------------- 4 files changed, 141 insertions(+), 107 deletions(-) create mode 100644 include/erofs/rebuild.h create mode 100644 lib/rebuild.c
include/erofs/rebuild.h new file +19 -0
@@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ +#ifndef __EROFS_REBUILD_H +#define __EROFS_REBUILD_H + +#ifdef __cplusplus +extern "C" +{ +#endif + +#include "internal.h" + +struct erofs_dentry *erofs_rebuild_get_dentry (struct erofs_inode *pwd, +char *path,bool aufs,bool *whout,bool *opq); + +#ifdef __cplusplus +} +#endif + +#endif
lib/Makefile.am +2 -1
@@ -24,6 +24,7 @@ noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \ $(top_srcdir)/include/erofs/xattr.h \ $(top_srcdir)/include/erofs/compress_hints.h \ $(top_srcdir)/include/erofs/fragments.h \ + $(top_srcdir)/include/erofs/rebuild.h \ $(top_srcdir)/lib/liberofs_private.h noinst_HEADERS += compressor.h
@@ -31,7 +32,7 @@ liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \ namei.c data.c compress.c compressor.c zmap.c decompress.c \ compress_hints.c hashmap.c sha256.c blobchunk.c dir.c \ fragments.c rb_tree.c dedupe.c uuid_unparse.c uuid.c tar.c \ - block_list.c + block_list.c rebuild.c liberofs_la_CFLAGS = -Wall ${libuuid_CFLAGS} -I$(top_srcdir)/include if ENABLE_LZ4
lib/rebuild.c new file +117 -0
117 lines not on this page · raw message
lib/tar.c +3 -106
139 lines not on this page · raw message
Signature · 1 line
2.19.1.6.gb485710b
PATCH v4 8/10 erofs-utils: lib: add erofs_rebuild_load_tree() helper
In reply to Jingbo XuAdd erofs_rebuild_load_tree() helper loading inode tree from given erofs image, and making it merged into a given inode tree in an overlayfs like model.
Since we need to read the content of the symlink file from disk when loading tree, add dependency on zlib_LIBS for mkfs.erofs.
Also rename tarerofs_dump_tree() to erofs_rebuild_dump_tree(), so that it could also be called from rebuild mode.
--- include/erofs/inode.h | 2 +- include/erofs/internal.h | 1 + include/erofs/rebuild.h | 2 + lib/inode.c | 4 +- lib/rebuild.c | 271 +++++++++++++++++++++++++++++++++++++++ mkfs/main.c | 2 +- 6 files changed, 278 insertions(+), 4 deletions(-)
include/erofs/inode.h +1 -1
@@ -32,7 +32,7 @@ unsigned int erofs_iput(struct erofs_inode *inode);erofs_nid_t erofs_lookupnid (struct erofs_inode *inode);struct erofs_dentry *erofs_d_alloc (struct erofs_inode *parent,const char *name); -int tarerofs_dump_tree (struct erofs_inode *dir); +int erofs_rebuild_dump_tree (struct erofs_inode *dir);int erofs_init_empty_dir (struct erofs_inode *dir);struct erofs_inode *erofs_new_inode (void );struct erofs_inode *erofs_mkfs_build_tree_from_path (const char *path);
include/erofs/internal.h +1 -0
@@ -108,6 +108,7 @@ struct erofs_sb_info {int devfd;u64 devsz; +dev_t dev;unsigned int nblobs;unsigned int blobfd[256 ]; };
include/erofs/rebuild.h +2 -0
@@ -12,6 +12,8 @@ extern "C"struct erofs_dentry *erofs_rebuild_get_dentry (struct erofs_inode *pwd,char *path,bool aufs,bool *whout,bool *opq); +int erofs_rebuild_load_tree (struct erofs_inode *root,struct erofs_sb_info *sbi); + #ifdef __cplusplus }#endif
lib/inode.c +2 -2
@@ -1323,7 +1323,7 @@ struct erofs_inode *erofs_mkfs_build_special_from_fd(int fd, const char *name)return inode; } -int tarerofs_dump_tree (struct erofs_inode *dir) +int erofs_rebuild_dump_tree (struct erofs_inode *dir) {struct erofs_dentry *d;unsigned int nr_subdirs;
@@ -1391,7 +1391,7 @@ int tarerofs_dump_tree(struct erofs_inode *dir)continue ; inode =erofs_igrab (d->inode); - ret =tarerofs_dump_tree (inode); + ret =erofs_rebuild_dump_tree (inode); dir->i_nlink += (erofs_mode_to_ftype (inode->i_mode) ==EROFS_FT_DIR );erofs_iput (inode);if (ret)
lib/rebuild.c +271 -0
283 lines not on this page · raw message
mkfs/main.c +1 -1
@@ -946,7 +946,7 @@ int main(int argc, char **argv)if (err <0 ) goto exit; - err =tarerofs_dump_tree (root_inode); + err =erofs_rebuild_dump_tree (root_inode);if (err <0 ) goto exit; }
Signature · 1 line
2.19.1.6.gb485710b
Re: [PATCH v4 08/10] erofs-utils: lib: add erofs_rebuild_load_tree() helper
Quoted 104 lines · Jingbo Xu
Add
erofs_rebuild_load_tree()helper loading inode tree from given erofs image, and making it merged into a given inode tree in an overlayfs like model.Since we need to read the content of the symlink file from disk when loading tree, add dependency on
zlib_LIBSfor mkfs.erofs.Also rename
tarerofs_dump_tree()toerofs_rebuild_dump_tree(), so that it could also be called from rebuild mode.Signed-off-by Jingbo Xu <[email protected]>--- include/erofs/inode.h | 2 +- include/erofs/internal.h | 1 + include/erofs/rebuild.h | 2 + lib/inode.c | 4 +- lib/rebuild.c | 271 +++++++++++++++++++++++++++++++++++++++ mkfs/main.c | 2 +- 6 files changed, 278 insertions(+), 4 deletions(-)include/erofs/inode.h +1
-1@@ -32,7 +32,7 @@ unsigned int erofs_iput(struct erofs_inode *inode);erofs_nid_t erofs_lookupnid (struct erofs_inode *inode);struct erofs_dentry *erofs_d_alloc (struct erofs_inode *parent,const char *name); -int tarerofs_dump_tree (struct erofs_inode *dir); +int erofs_rebuild_dump_tree (struct erofs_inode *dir);int erofs_init_empty_dir (struct erofs_inode *dir);struct erofs_inode *erofs_new_inode (void );struct erofs_inode *erofs_mkfs_build_tree_from_path (const char *path);include/erofs/internal.h +1
-0@@ -108,6 +108,7 @@ struct erofs_sb_info {int devfd;u64 devsz; +dev_t dev;unsigned int nblobs;unsigned int blobfd[256 ]; };include/erofs/rebuild.h +2
-0@@ -12,6 +12,8 @@ extern "C"struct erofs_dentry *erofs_rebuild_get_dentry (struct erofs_inode *pwd,char *path,bool aufs,bool *whout,bool *opq); +int erofs_rebuild_load_tree (struct erofs_inode *root,struct erofs_sb_info *sbi); + #ifdef __cplusplus }#endif lib/inode.c +2
-2@@ -1323,7 +1323,7 @@ struct erofs_inode *erofs_mkfs_build_special_from_fd(int fd, const char *name)return inode; } -int tarerofs_dump_tree (struct erofs_inode *dir) +int erofs_rebuild_dump_tree (struct erofs_inode *dir) {struct erofs_dentry *d;unsigned int nr_subdirs;@@ -1391,7 +1391,7 @@ int tarerofs_dump_tree(struct erofs_inode *dir)continue ; inode =erofs_igrab (d->inode); - ret =tarerofs_dump_tree (inode); + ret =erofs_rebuild_dump_tree (inode); dir->i_nlink += (erofs_mode_to_ftype (inode->i_mode) ==EROFS_FT_DIR );erofs_iput (inode);if (ret)lib/rebuild.c +9
-0@@ -3,9 +3,18 @@#include <unistd.h> #include <stdlib.h> #include <string.h> +#include <sys/stat.h> +#include <config.h> +#if defined(HAVE_SYS_SYSMACROS_H ) +#include <sys/sysmacros.h> +#endif #include "erofs/print.h" #include "erofs/inode.h" #include "erofs/rebuild.h" +#include "erofs/io.h" +#include "erofs/dir.h" +#include "erofs/xattr.h" +#include "erofs/blobchunk.h" #include "erofs/internal.h" #ifdef HAVE_LINUX_AUFS_TYPE_H
lib/rebuild.c struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd, · line 129
Quoted 6 lines
lib/rebuild.c +3
-0@@ -115,3 +124,265 @@ struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd, }return d; } + +static int erofs_rebuild_fill_inode_map (struct erofs_inode *inode, +struct erofs_inode *vi)
erofs_rebuilt_fixup_inode_indexes? Also could you use (dst, src) instead of (inode, vi)?
Quoted 60 lines
lib/rebuild.c +60
-0+{ +int ret; +struct erofs_sb_info *sbi = vi->sbi; +unsigned int count, unit, chunkbits, i; +unsigned int deviceid = inode->dev; +erofs_off_t chunksize; +erofs_blk_t blkaddr; +struct erofs_inode_chunk_index *idx; + +/* TODO: fill data map in other layouts */ +if (vi->datalayout !=EROFS_INODE_CHUNK_BASED && + vi->datalayout !=EROFS_INODE_FLAT_PLAIN ) { +erofs_err ("unsupported datalayout %d" , vi->datalayout); +return -EOPNOTSUPP ; + } + +if (sbi->extra_devices) { + chunkbits = vi->u.chunkbits; + inode->u.chunkformat = vi->u.chunkformat; + }else { + chunkbits =ilog2 (inode->i_size -1 ) +1 ; +if (chunkbits < sbi->blkszbits) + chunkbits = sbi->blkszbits; +if (chunkbits - sbi->blkszbits >EROFS_CHUNK_FORMAT_BLKBITS_MASK ) + chunkbits =EROFS_CHUNK_FORMAT_BLKBITS_MASK + sbi->blkszbits; + inode->u.chunkformat =EROFS_CHUNK_FORMAT_INDEXES ; + inode->u.chunkformat |= chunkbits - sbi->blkszbits; + } + chunksize =1ULL << chunkbits; + count =DIV_ROUND_UP (inode->i_size, chunksize); + + unit =sizeof (struct erofs_inode_chunk_index ); + inode->extent_isize = count * unit; + idx =malloc (max (sizeof (*idx),sizeof (void *))); +if (!idx) +return -ENOMEM ; + inode->chunkindexes = idx; + +for (i =0 ; i < count; i++) { +struct erofs_blobchunk *chunk; +struct erofs_map_blocks map = { + .index =UINT_MAX , + }; + + map.m_la = i << chunkbits; + ret =erofs_map_blocks (vi, &map,0 ); +if (ret) + goto err; + + blkaddr =erofs_blknr (sbi, map.m_pa); + chunk =erofs_get_unhashed_chunk (deviceid, blkaddr,0 ); +if (IS_ERR (chunk)) { + ret =PTR_ERR (chunk); + goto err; + } + *(void **)idx++ = chunk; +erofs_dbg ("\t%s: chunk %d (deviceid %u, blkaddr %u)" , + inode->i_srcpath, i, deviceid, blkaddr); + + }
lib/rebuild.c struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd, · line 199
Quoted 10 lines
lib/rebuild.c +10
-0+ inode->datalayout =EROFS_INODE_CHUNK_BASED ; +return 0 ; +err: +free (idx); + inode->chunkindexes =NULL ; +return ret; +} + +static int erofs_rebuild_fill_inode (struct erofs_inode *inode, +struct erofs_inode *vi)
same here.
Quoted 52 lines
lib/rebuild.c +52
-0+{ +int ret =0 ; + + inode->i_srcpath =strdup (vi->i_srcpath); + inode->i_mode = vi->i_mode; + inode->i_uid = vi->i_uid; + inode->i_gid = vi->i_gid; + inode->i_mtime = vi->i_mtime; + inode->i_ino[1 ] = vi->nid; + inode->i_nlink =1 ; + inode->opaque = vi->opaque; +list_splice_tail (&vi->i_xattrs, &inode->i_xattrs); + +switch (inode->i_mode &S_IFMT ) { +case S_IFCHR : +case S_IFBLK : +case S_IFIFO : +case S_IFSOCK : + inode->u.i_rdev =erofs_new_encode_dev (vi->u.i_rdev); + inode->i_size =0 ; +erofs_dbg ("\tdev: %d %d" ,major (vi->u.i_rdev), +minor (vi->u.i_rdev)); +break ; +case S_IFDIR : + inode->i_size =0 ; + ret =erofs_init_empty_dir (inode); +break ; +case S_IFLNK : + inode->i_size = vi->i_size; + inode->i_link =malloc (inode->i_size +1 ); + ret =erofs_pread (vi, inode->i_link, inode->i_size,0 ); +erofs_dbg ("\tsymlink: %s -> %s" , inode->i_srcpath, inode->i_link); +break ; +case S_IFREG : + inode->i_size = vi->i_size; +if (inode->i_size) + ret =erofs_rebuild_fill_inode_map (inode, vi); +else + inode->u.i_blkaddr =NULL_ADDR ; +break ; +default : + ret = -EINVAL ; +break ; + } +return ret; +} + +struct erofs_rebuild_dir_context { +struct erofs_dir_context ctx; +struct erofs_inode *root; +dev_t dev; +};
lib/rebuild.c struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd, · line 261
Quoted 10 lines
lib/rebuild.c +10
-0+ +static int erofs_rebuild_parse_inode (struct erofs_rebuild_dir_context *rctx, +struct erofs_inode *vi) +{ +struct erofs_inode *inode; +struct erofs_dentry *d; +int ret; + + d =erofs_rebuild_get_dentry (rctx->root, vi->i_srcpath, + false,NULL ,NULL );
I'm not sure if it's effective, I think we should just record parent nid and lookup subdir instead.
Quoted 10 lines
lib/rebuild.c +10
-0+if (IS_ERR (d)) +return PTR_ERR (d); + +if (d->type !=EROFS_FT_UNKNOWN ) { + inode = d->inode; +DBG_BUGON ((inode->i_mode &S_IFMT ) != (vi->i_mode &S_IFMT )); +if (!S_ISDIR (inode->i_mode) || inode->opaque) { +erofs_dbg ("file %s: %s (%d) exists" , + vi->i_srcpath, inode->i_srcpath, +erofs_mode_to_ftype (inode->i_mode));
lib/rebuild.c struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd, · line 281
Quoted 10 lines
lib/rebuild.c +10
-0+return 0 ; + } +erofs_dbg ("dir %s: %s merging" , vi->i_srcpath, inode->i_srcpath); + }else { +erofs_dbg ("loading file: %s (%d) (nid %llu)" , vi->i_srcpath, +erofs_mode_to_ftype (vi->i_mode), vi->nid); +if (S_ISREG (vi->i_mode) && vi->i_nlink >1 && + (inode =erofs_iget (rctx->dev, vi->nid))) { +/* hardlink file */ +if (S_ISDIR (inode->i_mode)) {
why this can happen?
Quoted 37 lines
lib/rebuild.c +37
-0+erofs_dbg ("hardlink directory not supported" ); + ret = -EISDIR ; + goto put_inode; + } + inode->i_nlink++; +erofs_dbg ("\thardlink: %s -> %s" , + vi->i_srcpath, inode->i_srcpath); + }else { + inode =erofs_new_inode (); +if (IS_ERR (inode)) +return PTR_ERR (inode); + + ret =erofs_read_xattrs_from_disk (vi); +if (ret) + goto put_inode; + + inode->i_parent = d->inode; + inode->dev = rctx->dev; + ret =erofs_rebuild_fill_inode (inode, vi); +if (ret) + goto put_inode; + +erofs_insert_ihash (inode, rctx->dev, vi->nid); + } + + d->inode = inode; + d->type =erofs_mode_to_ftype (inode->i_mode); + } + + ret =0 ; +if (S_ISDIR (vi->i_mode)) { +struct erofs_rebuild_dir_context nctx = *rctx; + nctx.ctx.dir = vi; + ret =erofs_iterate_dir (&nctx.ctx, false); + } +return ret; +
lib/rebuild.c struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd, · line 328
Quoted 10 lines
lib/rebuild.c +10
-0+put_inode: +erofs_iput (inode); +return ret; +} + +static int erofs_rebuild_dirent_iter (struct erofs_dir_context *ctx) +{ +struct erofs_rebuild_dir_context *rctx = (void *)ctx; +struct erofs_inode *dir = ctx->dir; +struct erofs_inode vi = {};
why we cannot reuse the new allocated inode -- erofs_new_inode()?
Quoted 22 lines
lib/rebuild.c +22
-0+char *path; +int ret; + +if (ctx->dot_dotdot) +return 0 ; + +erofs_dbg ("file %s/%.*s" , dir->i_srcpath, ctx->de_namelen, ctx->dname); + vi.nid = ctx->de_nid; + vi.sbi = dir->sbi; + ret =erofs_read_inode_from_disk (&vi); +if (ret) +return ret; + + ret =asprintf (&path,"%s/%.*s" , dir->i_srcpath, ctx->de_namelen, + ctx->dname); +if (ret <0 ) +return ret; + vi.i_srcpath = path; + + ret =erofs_rebuild_parse_inode (rctx, &vi); +free (path); +return ret;
lib/rebuild.c struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd, · line 360
Quoted 10 lines
lib/rebuild.c +10
-0+} + +int erofs_rebuild_load_tree (struct erofs_inode *root,struct erofs_sb_info *sbi) +{ +struct erofs_inode inode = {}; +struct erofs_rebuild_dir_context ctx; +int ret; + +if (!sbi->devname) { +erofs_err ("please open the device first" );
erofs_err("failed to find a device for rebuilding");lib/rebuild.c struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd, · line 366
Quoted 6 lines
lib/rebuild.c +6
-0+return -EINVAL ; + } + + ret =erofs_read_superblock (sbi); +if (ret) { +erofs_err ("failed to read superblock of img %s" , sbi->devname);
let's avoid using abbreviation "img"
"failed to read superblock of %s"
lib/rebuild.c struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd, · line 374
Quoted 8 lines
lib/rebuild.c +8
-0+return ret; + } + + inode.nid = sbi->root_nid; + inode.sbi = sbi; + ret =erofs_read_inode_from_disk (&inode); +if (ret) { +erofs_err ("failed to read root inode of img %s" , sbi->devname);
"failed to read root inode of %s"
Thanks,
Gao Xiang
PATCH v4 9/10 erofs-utils: mkfs: introduce rebuild mode
In reply to Jingbo XuIntroduce a new rebuild mode merging multiple erofs images generated from either tarerfs index mode (--tar=i):
mkfs.erofs --tar=i --aufs layer0.erofs layer0.tar ... mkfs.erofs --tar=i --aufs layerN.erofs layerN.tar mkfs.erofs merge.erofs layerN.erofs ... layer0.erofs
or tarerofs non-index mode (--tar=f):
mkfs.erofs --tar=f -Enoinline_data --aufs layer0.erofs layer0.tar ... mkfs.erofs --tar=f -Enoinline_data --aufs layerN.erofs layerN.tar mkfs.erofs merge.erofs layerN.erofs ... layer0.erofs
When calling rebuild mode, the top most layer must be specified firstly in the cmdline, while the bottom most layer i.e. layer0 the lastly.
The rebuild mode doesn't support flat inline datalayout yet.
--- include/erofs/internal.h | 2 + mkfs/main.c | 207 +++++++++++++++++++++++++++++++-------- 2 files changed, 166 insertions(+), 43 deletions(-)
include/erofs/internal.h +2 -0
@@ -111,6 +111,8 @@ struct erofs_sb_info {dev_t dev;unsigned int nblobs;unsigned int blobfd[256 ]; + +struct list_head list; };/* make sure that any user of the erofs headers has atleast 64bit off_t type */
mkfs/main.c +164 -43
292 lines not on this page · raw message
Signature · 1 line
2.19.1.6.gb485710b
PATCH v4 10/10 erofs-utils: mkfs: add --keep-whiteout option for tarfs and rebuild mode
In reply to Jingbo XuAdd "--keep-whiteout=[0|1]" option for tarfs and rebuild mode, controlling whether whiteout files are kept in the generated image.
This option is enabled by default for both tarfs and rebuild mode, i.e. as if "--keep-whiteout=1" is specified by default.
To hide whiteout files, specify "--keep-whiteout=0" explicitly.
--- include/erofs/config.h | 1 + include/erofs/internal.h | 2 ++ lib/config.c | 1 + lib/inode.c | 17 +++++++++++++++-- lib/tar.c | 2 -- mkfs/main.c | 12 ++++++++++++ 6 files changed, 31 insertions(+), 4 deletions(-)
include/erofs/config.h +1 -0
@@ -53,6 +53,7 @@ struct erofs_configure {bool c_ignore_mtime;bool c_showprogress;bool c_extra_ea_name_prefixes; +bool c_keep_whiteout;#ifdef HAVE_LIBSELINUX struct selabel_handle *sehnd;
include/erofs/internal.h +2 -0
@@ -426,6 +426,8 @@ static inline u32 erofs_crc32c(u32 crc, const u8 *in, size_t len)return crc; } +#define EROFS_WHITEOUT_DEV 0 + #ifdef __cplusplus }#endif
lib/config.c +1 -0
@@ -33,6 +33,7 @@ void erofs_init_configure(void) cfg.c_pclusterblks_max =1 ; cfg.c_pclusterblks_def =1 ; cfg.c_max_decompressed_extent_bytes =-1 ; + cfg.c_keep_whiteout = true; }void erofs_show_config (void )
lib/inode.c +15 -2
@@ -1323,9 +1323,14 @@ struct erofs_inode *erofs_mkfs_build_special_from_fd(int fd, const char *name)return inode; } +static bool erofs_inode_is_whiteout (struct erofs_inode *inode) +{ +return S_ISCHR (inode->i_mode) && inode->u.i_rdev ==EROFS_WHITEOUT_DEV ; +} +int erofs_rebuild_dump_tree (struct erofs_inode *dir) { -struct erofs_dentry *d; +struct erofs_dentry *d, *n;unsigned int nr_subdirs;int ret;
@@ -1369,8 +1374,16 @@ int erofs_rebuild_dump_tree(struct erofs_inode *dir) } nr_subdirs =0 ; -list_for_each_entry (d, &dir->i_subdirs, d_child) +list_for_each_entry_safe (d, n, &dir->i_subdirs, d_child) { +if (!cfg.c_keep_whiteout &&erofs_inode_is_whiteout (d->inode)) { +erofs_dbg ("skip whiteout %s" , d->inode->i_srcpath); +list_del (&d->d_child); +erofs_d_invalidate (d); +free (d); +continue ; + } ++nr_subdirs; + } ret =erofs_prepare_dir_layout (dir, nr_subdirs);if (ret)
lib/tar.c +0 -2
@@ -13,8 +13,6 @@#include "erofs/blobchunk.h" #include "erofs/rebuild.h" -#define EROFS_WHITEOUT_DEV 0 -static char erofs_libbuf[16384 ];struct tar_header {
mkfs/main.c +12 -0
@@ -57,6 +57,7 @@ static struct option long_options[] = { {"gid-offset", required_argument, NULL, 17}, {"tar", optional_argument, NULL, 20}, {"aufs", no_argument, NULL, 21}, + {"keep-whiteout", required_argument, NULL, 22}, {"mount-point", required_argument, NULL, 512}, {"xattr-prefix", required_argument, NULL, 19}, #ifdef WITH_ANDROID
@@ -121,6 +122,7 @@ static void usage(void)" --preserve-mtime keep per-file modification time strictly\n" " --aufs replace aufs special files with overlayfs metadata\n" " --tar=[fi] generate an image from tarball(s)\n" +" --keep-whiteout=[01] keep whiteout in tarfs or rebuild mode\n" " --quiet quiet execution (do not write anything to standard output.)\n" #ifndef NDEBUG" --random-pclusterblks randomize pclusterblks for big pcluster (debugging only)\n"
@@ -521,6 +523,16 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])case 21 : erofstar.aufs = true;break ; +case 22 : +if (!strcmp (optarg,"0" )) + cfg.c_keep_whiteout = false; +else if (!strcmp (optarg,"1" )) + cfg.c_keep_whiteout = true; +else { +erofs_err ("invalid keep-whiteout %s" , optarg); +return -EINVAL ; + } +break ;case 1 :usage ();exit (0 );
Signature · 1 line
2.19.1.6.gb485710b