erofs-utils: mkfs: introduce rebuild mode
v8 cover erofs-utils: mkfs: introduce rebuild mode
v8:
- patch 7: fix the missing call for erofs_mkfs_dump_blobs() in rebuild mode by also calling erofs_mkfs_dump_blobs() when sbi->extra_devices is not zero; otherwise the content of the on-disk device slot is all 0
- patch 8: strip overlay.opaque xattr when --ovlfs-strip=1; update the commit message accordingly
v1: https://lore.kernel.org/all/[email protected]/
v2: https://lore.kernel.org/all/[email protected]/
v3: https://lore.kernel.org/all/[email protected]/
v4: https://lore.kernel.org/all/[email protected]/
v5: https://lore.kernel.org/all/[email protected]/
v6: https://lore.kernel.org/all/[email protected]/
v7: https://lore.kernel.org/all/[email protected]/
-------------------------
Introduce a new rebuild mode merging multiple erofs images generated from either tarerfs index mode (--tar=i):
mkfs.erofs --tar=i --aufs layer0.erofs layer0.tar ... mkfs.erofs --tar=i --aufs layerN.erofs layerN.tar mkfs.erofs merge.erofs layer0.erofs ... layerN.erofs
or tarerofs non-index mode (--tar=f):
mkfs.erofs --tar=f -Enoinline_data --aufs layer0.erofs layer0.tar ... mkfs.erofs --tar=f -Enoinline_data --aufs layerN.erofs layerN.tar mkfs.erofs merge.erofs layer0.erofs ... layerN.erofs
Jingbo Xu (8):
erofs-utils: lib: add list_splice_tail() helper
erofs-utils: lib: make erofs_get_unhashed_chunk() global
erofs-utils: lib: add erofs_read_xattrs_from_disk() helper
erofs-utils: lib: add erofs_insert_ihash() helper
erofs-utils: lib: add erofs_rebuild_get_dentry() helper
erofs-utils: lib: add erofs_rebuild_load_tree() helper
erofs-utils: mkfs: introduce rebuild mode
erofs-utils: mkfs: add --ovlfs-strip option
include/erofs/blobchunk.h | 2 + include/erofs/config.h | 1 + include/erofs/inode.h | 5 +- include/erofs/internal.h | 7 +- include/erofs/list.h | 20 ++ include/erofs/rebuild.h | 21 ++ include/erofs/xattr.h | 2 + lib/Makefile.am | 3 +- lib/blobchunk.c | 2 +- lib/inode.c | 31 ++- lib/rebuild.c | 404 ++++++++++++++++++++++++++++++++++++++ lib/tar.c | 109 +--------- lib/xattr.c | 87 ++++++++ mkfs/main.c | 227 ++++++++++++++++----- 14 files changed, 751 insertions(+), 170 deletions(-) create mode 100644 include/erofs/rebuild.h create mode 100644 lib/rebuild.c
Signature · 1 line
2.19.1.6.gb485710b
PATCH v8 1/8 erofs-utils: lib: add list_splice_tail() helper
Add list_splice_tail() helper.
--- include/erofs/list.h | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+)
include/erofs/list.h +20 -0
@@ -70,6 +70,26 @@ static inline int list_empty(struct list_head *head)return head->next == head; } +static inline void __list_splice (struct list_head *list, +struct list_head *prev,struct list_head *next) +{ +struct list_head *first = list->next; +struct list_head *last = list->prev; + + first->prev = prev; + prev->next = first; + + last->next = next; + next->prev = last; +} + +static inline void list_splice_tail (struct list_head *list, +struct list_head *head) +{ +if (!list_empty (list)) +__list_splice (list, head->prev, head); +} +#define list_entry (ptr, type, member) container_of(ptr, type, member)#define list_first_entry (ptr, type, member) \
Signature · 1 line
2.19.1.6.gb485710b
PATCH v8 2/8 erofs-utils: lib: make erofs_get_unhashed_chunk() global
In reply to Jingbo Xu... so that it could be called from outside blobchunk.c later.
--- include/erofs/blobchunk.h | 2 ++ lib/blobchunk.c | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-)
include/erofs/blobchunk.h +2 -0
@@ -14,6 +14,8 @@ extern "C"#include "erofs/internal.h" +struct erofs_blobchunk *erofs_get_unhashed_chunk (unsigned int device_id, +erofs_blk_t blkaddr,erofs_off_t sourceoffset);int erofs_blob_write_chunk_indexes (struct erofs_inode *inode,erofs_off_t off);int erofs_blob_write_chunked_file (struct erofs_inode *inode,int fd);int tarerofs_write_chunkes (struct erofs_inode *inode,erofs_off_t data_offset);
lib/blobchunk.c +1 -1
@@ -38,7 +38,7 @@ struct erofs_blobchunk erofs_holechunk = { };static LIST_HEAD (unhashed_blobchunks); -static struct erofs_blobchunk *erofs_get_unhashed_chunk (unsigned int device_id, +struct erofs_blobchunk *erofs_get_unhashed_chunk (unsigned int device_id,erofs_blk_t blkaddr,erofs_off_t sourceoffset) {struct erofs_blobchunk *chunk;
Signature · 1 line
2.19.1.6.gb485710b
PATCH v8 3/8 erofs-utils: lib: add erofs_read_xattrs_from_disk() helper
In reply to Jingbo XuAdd erofs_read_xattrs_from_disk() helper to reading extended attributes from disk.
--- include/erofs/internal.h | 1 + include/erofs/xattr.h | 1 + lib/xattr.c | 69 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+)
include/erofs/internal.h +1 -0
@@ -196,6 +196,7 @@ struct erofs_inode {bool compressed_idata;bool lazy_tailblock;bool with_tmpfile; +bool opaque;/* OVL: non-merge dir that may contain whiteout entries */ bool whiteouts;
include/erofs/xattr.h +1 -0
@@ -58,6 +58,7 @@ int erofs_setxattr(struct erofs_inode *inode, char *key, const void *value, size_t size);int erofs_set_opaque_xattr (struct erofs_inode *inode);int erofs_set_origin_xattr (struct erofs_inode *inode); +int erofs_read_xattrs_from_disk (struct erofs_inode *inode); #ifdef __cplusplus }
lib/xattr.c +69 -0
75 lines not on this page · raw message
Signature · 1 line
2.19.1.6.gb485710b
PATCH v8 4/8 erofs-utils: lib: add erofs_insert_ihash() helper
In reply to Jingbo XuAdd erofs_insert_ihash() helper inserting inode into inode hash table.
Also add prototypes of erofs_iget() and erofs_iget_by_nid() in the header file.
--- include/erofs/inode.h | 3 +++ lib/inode.c | 10 +++++++--- 2 files changed, 10 insertions(+), 3 deletions(-)
include/erofs/inode.h +3 -0
@@ -25,6 +25,9 @@ u32 erofs_new_encode_dev(dev_t dev);unsigned char erofs_mode_to_ftype (umode_t mode);unsigned char erofs_ftype_to_dtype (unsigned int filetype);void erofs_inode_manager_init (void ); +void erofs_insert_ihash (struct erofs_inode *inode,dev_t dev,ino_t ino); +struct erofs_inode *erofs_iget (dev_t dev,ino_t ino); +struct erofs_inode *erofs_iget_by_nid (erofs_nid_t nid);unsigned int erofs_iput (struct erofs_inode *inode);erofs_nid_t erofs_lookupnid (struct erofs_inode *inode); struct erofs_dentry *erofs_d_alloc(struct erofs_inode *parent,
lib/inode.c +7 -3
@@ -75,6 +75,12 @@ void erofs_inode_manager_init(void)init_list_head (&inode_hashtable[i]); } +void erofs_insert_ihash (struct erofs_inode *inode,dev_t dev,ino_t ino) +{ +list_add (&inode->i_hash, + &inode_hashtable[(ino ^ dev) %NR_INODE_HASHTABLE ]); +} +/* get the inode from the (source) inode # */ struct erofs_inode *erofs_iget (dev_t dev,ino_t ino) {
@@ -976,9 +982,7 @@ static int erofs_fill_inode(struct erofs_inode *inode, struct stat *st, inode->inode_isize =sizeof (struct erofs_inode_compact ); } -list_add (&inode->i_hash, - &inode_hashtable[(st->st_ino ^ st->st_dev) % -NR_INODE_HASHTABLE ]); +erofs_insert_ihash (inode, st->st_dev, st->st_ino);return 0 ; }
Signature · 1 line
2.19.1.6.gb485710b
PATCH v8 5/8 erofs-utils: lib: add erofs_rebuild_get_dentry() helper
In reply to Jingbo XuRename tarerofs_get_dentry() to erofs_rebuild_get_dentry() and move it into lib/rebuild.c.
No logic changes.
--- include/erofs/rebuild.h | 19 +++++++ lib/Makefile.am | 3 +- lib/rebuild.c | 116 ++++++++++++++++++++++++++++++++++++++++ lib/tar.c | 109 ++----------------------------------- 4 files changed, 140 insertions(+), 107 deletions(-) create mode 100644 include/erofs/rebuild.h create mode 100644 lib/rebuild.c
include/erofs/rebuild.h new file +19 -0
@@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ +#ifndef __EROFS_REBUILD_H +#define __EROFS_REBUILD_H + +#ifdef __cplusplus +extern "C" +{ +#endif + +#include "internal.h" + +struct erofs_dentry *erofs_rebuild_get_dentry (struct erofs_inode *pwd, +char *path,bool aufs,bool *whout,bool *opq); + +#ifdef __cplusplus +} +#endif + +#endif
lib/Makefile.am +2 -1
@@ -25,6 +25,7 @@ noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \ $(top_srcdir)/include/erofs/compress_hints.h \ $(top_srcdir)/include/erofs/fragments.h \ $(top_srcdir)/include/erofs/xxhash.h \ + $(top_srcdir)/include/erofs/rebuild.h \ $(top_srcdir)/lib/liberofs_private.h noinst_HEADERS += compressor.h
@@ -32,7 +33,7 @@ liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \ namei.c data.c compress.c compressor.c zmap.c decompress.c \ compress_hints.c hashmap.c sha256.c blobchunk.c dir.c \ fragments.c rb_tree.c dedupe.c uuid_unparse.c uuid.c tar.c \ - block_list.c xxhash.c + block_list.c xxhash.c rebuild.c liberofs_la_CFLAGS = -Wall ${libuuid_CFLAGS} -I$(top_srcdir)/include if ENABLE_LZ4
lib/rebuild.c new file +116 -0
116 lines not on this page · raw message
lib/tar.c +3 -106
139 lines not on this page · raw message
Signature · 1 line
2.19.1.6.gb485710b
PATCH v8 6/8 erofs-utils: lib: add erofs_rebuild_load_tree() helper
In reply to Jingbo XuAdd erofs_rebuild_load_tree() helper to load the inode tree from a given erofs image, and make it merged acting as an overlayfs-like model.
Since the content of the symlink file needs to be read when loading tree, let's add a dependency on zlib_LIBS for mkfs.erofs.
Also rename tarerofs_dump_tree() to erofs_rebuild_dump_tree() since it is also called in the rebuild mode.
--- include/erofs/inode.h | 2 +- include/erofs/internal.h | 6 +- include/erofs/rebuild.h | 2 + lib/inode.c | 4 +- lib/rebuild.c | 288 +++++++++++++++++++++++++++++++++++++++ mkfs/main.c | 2 +- 6 files changed, 298 insertions(+), 6 deletions(-)
include/erofs/inode.h +1 -1
@@ -32,7 +32,7 @@ unsigned int erofs_iput(struct erofs_inode *inode);erofs_nid_t erofs_lookupnid (struct erofs_inode *inode);struct erofs_dentry *erofs_d_alloc (struct erofs_inode *parent,const char *name); -int tarerofs_dump_tree (struct erofs_inode *dir); +int erofs_rebuild_dump_tree (struct erofs_inode *dir);int erofs_init_empty_dir (struct erofs_inode *dir);struct erofs_inode *erofs_new_inode (void );struct erofs_inode *erofs_mkfs_build_tree_from_path (const char *path);
include/erofs/internal.h +4 -2
@@ -109,6 +109,7 @@ struct erofs_sb_info {int devfd;u64 devsz; +dev_t dev;unsigned int nblobs;unsigned int blobfd[256 ];
@@ -155,8 +156,6 @@ struct erofs_inode {union {/* (erofsfuse) runtime flags */ unsigned int flags; -/* (mkfs.erofs) device ID containing source file */ -u32 dev;/* (mkfs.erofs) queued sub-directories blocking dump */ u32 subdirs_queued; };
@@ -164,6 +163,9 @@ struct erofs_inode {struct erofs_sb_info *sbi;struct erofs_inode *i_parent; +/* (mkfs.erofs) device ID containing source file */ +u32 dev; +umode_t i_mode;erofs_off_t i_size;
include/erofs/rebuild.h +2 -0
@@ -12,6 +12,8 @@ extern "C"struct erofs_dentry *erofs_rebuild_get_dentry (struct erofs_inode *pwd,char *path,bool aufs,bool *whout,bool *opq); +int erofs_rebuild_load_tree (struct erofs_inode *root,struct erofs_sb_info *sbi); + #ifdef __cplusplus }#endif
lib/inode.c +2 -2
@@ -1324,7 +1324,7 @@ struct erofs_inode *erofs_mkfs_build_special_from_fd(int fd, const char *name)return inode; } -int tarerofs_dump_tree (struct erofs_inode *dir) +int erofs_rebuild_dump_tree (struct erofs_inode *dir) {struct erofs_dentry *d;unsigned int nr_subdirs;
@@ -1395,7 +1395,7 @@ int tarerofs_dump_tree(struct erofs_inode *dir)continue ; inode =erofs_igrab (d->inode); - ret =tarerofs_dump_tree (inode); + ret =erofs_rebuild_dump_tree (inode); dir->i_nlink += (erofs_mode_to_ftype (inode->i_mode) ==EROFS_FT_DIR );erofs_iput (inode);if (ret)
lib/rebuild.c +288 -0
300 lines not on this page · raw message
mkfs/main.c +1 -1
@@ -965,7 +965,7 @@ int main(int argc, char **argv)if (err <0 ) goto exit; - err =tarerofs_dump_tree (root_inode); + err =erofs_rebuild_dump_tree (root_inode);if (err <0 ) goto exit; }
Signature · 1 line
2.19.1.6.gb485710b
PATCH v8 7/8 erofs-utils: mkfs: introduce rebuild mode
In reply to Jingbo XuIntroduce a new EXPERIMENTAL rebuild mode, which can be used to generate a meta-only multidev manifest image with an overlayfs-like merged tree from multiple specific EROFS images either of
tarerofs index mode (--tar=i):
mkfs.erofs --tar=i --aufs layer0.erofs layer0.tar ... mkfs.erofs --tar=i --aufs layerN.erofs layerN-1.tar
or mkfs.erofs uncompressed mode without inline data:
mkfs.erofs --tar=f -Enoinline_data --aufs layer0.erofs layer0.tar ... mkfs.erofs --tar=f -Enoinline_data --aufs layerN-1.erofs layerN-1.tar
To merge these layers, just type:
mkfs.erofs merged.erofs layer0.erofs ... layerN-1.erofs
It doesn't support compression and/or flat inline datalayout yet.
--- mkfs/main.c | 219 ++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 168 insertions(+), 51 deletions(-)
mkfs/main.c +168 -51
@@ -26,6 +26,7 @@#include "erofs/compress_hints.h" #include "erofs/blobchunk.h" #include "erofs/fragments.h" +#include "erofs/rebuild.h" #include "../lib/liberofs_private.h" #include "../lib/liberofs_uuid.h"
@@ -83,8 +84,8 @@ static void print_available_compressors(FILE *f, const char *delim)static void usage (void ) { -fputs ("usage: [options] FILE DIRECTORY\n\n" -"Generate erofs image from DIRECTORY to FILE, and [options] are:\n" +fputs ("usage: [options] FILE SOURCE(s)\n" +"Generate EROFS image (FILE) from DIRECTORY, TARBALL and/or EROFS images. And [options] are:\n" " -b# set block size to # (# = page size by default)\n" " -d# set output message level to # (maximum 9)\n" " -x# set xattr tolerance to # (< 0, disable xattrs; default 2)\n"
@@ -135,7 +136,10 @@ static unsigned int pclustersize_packed, pclustersize_max;static struct erofs_tarfile erofstar = { .global.xattrs =LIST_HEAD_INIT (erofstar.global.xattrs) }; -static bool tar_mode; +static bool tar_mode, rebuild_mode; + +static unsigned int rebuild_src_count; +static LIST_HEAD (rebuild_src_list);static int parse_extended_opts (const char *opts) {
@@ -277,10 +281,23 @@ static int mkfs_parse_compress_algs(char *algs)return 0 ; } +static void erofs_rebuild_cleanup (void ) +{ +struct erofs_sb_info *src, *n; + +list_for_each_entry_safe (src, n, &rebuild_src_list, list) { +list_del (&src->list); +erofs_put_super (src); +dev_close (src); +free (src); + } + rebuild_src_count =0 ; +} + static int mkfs_parse_options_cfg(int argc, char *argv[]) { char *endptr; - int opt, i; + int opt, i, err; bool quiet = false; while ((opt = getopt_long(argc, argv, "C:E:L:T:U:b:d:x:z:",
@@ -531,12 +548,14 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])if (optind >= argc) {if (!tar_mode) { -erofs_err ("missing argument: DIRECTORY" ); +erofs_err ("missing argument: SOURCE(s)" );return -EINVAL ; }else { erofstar.fd =STDIN_FILENO ; } - }else { + }else { +struct stat st; + cfg.c_src_path =realpath (argv[optind++],NULL );if (!cfg.c_src_path) { erofs_err("failed to parse source directory: %s",
@@ -544,7 +563,47 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])return -ENOENT ; } -if (optind < argc) { +if (tar_mode) { + erofstar.fd =open (cfg.c_src_path,O_RDONLY ); +if (erofstar.fd <0 ) { +erofs_err ("failed to open file: %s" , cfg.c_src_path); +usage (); +return -errno; + } + }else { + err =lstat (cfg.c_src_path, &st); +if (err) +return -errno; +if (S_ISDIR (st.st_mode)) +erofs_set_fs_root (cfg.c_src_path); +else + rebuild_mode = true; + } + +if (rebuild_mode) { +char *srcpath = cfg.c_src_path; + +do { +struct erofs_sb_info *src; + + src =calloc (1 ,sizeof (struct erofs_sb_info )); +if (!src) { +erofs_rebuild_cleanup (); +return -ENOMEM ; + } + + err =dev_open_ro (src, srcpath); +if (err) { +free (src); +erofs_rebuild_cleanup (); +return err; + } + +/* extra device index starts from 1 */ + src->dev = ++rebuild_src_count; +list_add (&src->list, &rebuild_src_list); + }while (optind < argc && (srcpath = argv[optind++])); + }else if (optind < argc) {erofs_err ("unexpected argument: %s\n" , argv[optind]);return -EINVAL ; }
@@ -735,6 +794,61 @@ void erofs_show_progs(int argc, char *argv[])if (cfg.c_dbg_lvl >=EROFS_WARN )printf ("%s %s\n" ,basename (argv[0 ]), cfg.c_version); } +static struct erofs_inode *erofs_alloc_root_inode (void ) +{ +struct erofs_inode *root; + + root =erofs_new_inode (); +if (IS_ERR (root)) +return root; + root->i_srcpath =strdup ("/" ); + root->i_mode =S_IFDIR |0777 ; + root->i_parent = root; + root->i_mtime = root->sbi->build_time; + root->i_mtime_nsec = root->sbi->build_time_nsec; +erofs_init_empty_dir (root); +return root; +} + +static int erofs_rebuild_load_trees (struct erofs_inode *root) +{ +struct erofs_sb_info *src; +unsigned int extra_devices =0 ; +erofs_blk_t nblocks; +int ret, i =0 ; + +list_for_each_entry (src, &rebuild_src_list, list) { + ret =erofs_rebuild_load_tree (root, src); +if (ret) { +erofs_err ("failed to load %s" , src->devname); +return ret; + } +if (src->extra_devices >1 ) { +erofs_err ("%s: unsupported number of extra devices" , + src->devname, src->extra_devices); +return -EOPNOTSUPP ; + } + extra_devices += src->extra_devices; + } + +if (extra_devices && extra_devices != rebuild_src_count) { +erofs_err ("unsupported mix of source images" ); +return -EOPNOTSUPP ; + } + + ret =erofs_mkfs_init_devices (&sbi, rebuild_src_count); +if (ret) +return ret; + +list_for_each_entry (src, &rebuild_src_list, list) { +if (extra_devices) + nblocks = src->devs[0 ].blocks; +else + nblocks = src->primarydevice_blocks; + sbi.devs[i++].blocks = nblocks; + } +return 0 ; +}static void erofs_mkfs_showsummaries (erofs_blk_t nblocks) {
@@ -761,7 +875,6 @@ int main(int argc, char **argv)struct erofs_buffer_head *sb_bh;struct erofs_inode *root_inode, *packed_inode;erofs_nid_t root_nid, packed_nid; -struct stat st;erofs_blk_t nblocks;struct timeval t;FILE *packedfile =NULL ;
@@ -783,26 +896,6 @@ int main(int argc, char **argv)return 1 ; } -if (!tar_mode) { - err =lstat (cfg.c_src_path, &st); -if (err) -return 1 ; -if (!S_ISDIR (st.st_mode)) { -erofs_err ("root of the filesystem is not a directory - %s" , - cfg.c_src_path); -usage (); -return 1 ; - } -erofs_set_fs_root (cfg.c_src_path); - }else if (cfg.c_src_path) { - erofstar.fd =open (cfg.c_src_path,O_RDONLY ); -if (erofstar.fd <0 ) { -erofs_err ("failed to open file: %s" , cfg.c_src_path); -usage (); -return 1 ; - } - } -if (cfg.c_unix_timestamp !=-1 ) { sbi.build_time = cfg.c_unix_timestamp; sbi.build_time_nsec =0 ;
@@ -866,6 +959,22 @@ int main(int argc, char **argv) } } +if (rebuild_mode) { +struct erofs_sb_info *src; + +erofs_warn ("EXPERIMENTAL rebuild mode in use. Use at your own risk!" ); + + src = list_first_entry(&rebuild_src_list, struct erofs_sb_info, list); +if (!src) + goto exit; + err =erofs_read_superblock (src); +if (err) { +erofs_err ("failed to read superblock of %s" , src->devname); + goto exit; + } + sbi.blkszbits = src->blkszbits; + } + sb_bh =erofs_buffer_init ();if (IS_ERR (sb_bh)) { err =PTR_ERR (sb_bh);
@@ -931,7 +1040,35 @@ int main(int argc, char **argv)erofs_inode_manager_init (); -if (!tar_mode) { +if (tar_mode) { + root_inode =erofs_alloc_root_inode (); +if (IS_ERR (root_inode)) { + err =PTR_ERR (root_inode); + goto exit; + } + +while (!(err =tarerofs_parse_tar (root_inode, &erofstar))); + +if (err <0 ) + goto exit; + + err =erofs_rebuild_dump_tree (root_inode); +if (err <0 ) + goto exit; + }else if (rebuild_mode) { + root_inode =erofs_alloc_root_inode (); +if (IS_ERR (root_inode)) { + err =PTR_ERR (root_inode); + goto exit; + } + + err =erofs_rebuild_load_trees (root_inode); +if (err) + goto exit; + err =erofs_rebuild_dump_tree (root_inode); +if (err) + goto exit; + }else { err =erofs_build_shared_xattrs_from_path (&sbi, cfg.c_src_path);if (err) { erofs_err("failed to build shared xattrs: %s",
@@ -947,32 +1084,11 @@ int main(int argc, char **argv) err =PTR_ERR (root_inode); goto exit; } - }else { - root_inode =erofs_new_inode (); -if (IS_ERR (root_inode)) { - err =PTR_ERR (root_inode); - goto exit; - } - root_inode->i_srcpath =strdup ("/" ); - root_inode->i_mode =S_IFDIR |0777 ; - root_inode->i_parent = root_inode; - root_inode->i_mtime = sbi.build_time; - root_inode->i_mtime_nsec = sbi.build_time_nsec; -erofs_init_empty_dir (root_inode); - -while (!(err =tarerofs_parse_tar (root_inode, &erofstar))); - -if (err <0 ) - goto exit; - - err =erofs_rebuild_dump_tree (root_inode); -if (err <0 ) - goto exit; } root_nid =erofs_lookupnid (root_inode);erofs_iput (root_inode); -if (erofstar.index_mode || cfg.c_chunkbits) { +if (erofstar.index_mode || cfg.c_chunkbits || sbi.extra_devices) {if (erofstar.index_mode && !erofstar.mapfile) sbi.devs[0 ].blocks =BLK_ROUND_UP (&sbi, erofstar.offset);
@@ -1026,6 +1142,7 @@ exit:z_erofs_fragments_exit ();erofs_packedfile_exit ();erofs_xattr_cleanup_name_prefixes (); +erofs_rebuild_cleanup ();erofs_exit_configure ();if (err) {
Signature · 1 line
2.19.1.6.gb485710b
Quoted 16 lines · Jingbo Xu
Introduce a new EXPERIMENTAL rebuild mode, which can be used to generate a meta-only multidev manifest image with an overlayfs-like merged tree from multiple specific EROFS images either of
tarerofs index mode (--tar=i):
mkfs.erofs --tar=i --aufs layer0.erofs layer0.tar ... mkfs.erofs --tar=i --aufs layerN.erofs layerN-1.taror mkfs.erofs uncompressed mode without inline data:
mkfs.erofs --tar=f -Enoinline_data --aufs layer0.erofs layer0.tar ... mkfs.erofs --tar=f -Enoinline_data --aufs layerN-1.erofs layerN-1.tar
Quoted 7 lines
To merge these layers, just type:
mkfs.erofs merged.erofs layer0.erofs ... layerN-1.erofsIt doesn't support compression and/or flat inline datalayout yet.
Signed-off-by Jingbo Xu <[email protected]>Signed-off-by Gao Xiang <[email protected]>
As Jingbo reported, applying the following diff to fix the device table:
mkfs/main.c +3 -4
@@ -590,10 +590,9 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])if (rebuild_mode) {char *srcpath = cfg.c_src_path; +struct erofs_sb_info *src; do { - struct erofs_sb_info *src; - src = calloc(1, sizeof(struct erofs_sb_info)); if (!src) { erofs_rebuild_cleanup();
@@ -823,7 +822,7 @@ static int erofs_rebuild_load_trees(struct erofs_inode *root)struct erofs_sb_info *src;unsigned int extra_devices =0 ;erofs_blk_t nblocks; -int ret, i =0 ; +int ret;list_for_each_entry (src, &rebuild_src_list, list) { ret =erofs_rebuild_load_tree (root, src);
@@ -853,7 +852,7 @@ static int erofs_rebuild_load_trees(struct erofs_inode *root) nblocks = src->devs[0 ].blocks; else nblocks = src->primarydevice_blocks; - sbi.devs[i++].blocks = nblocks; + sbi.devs[src->dev -1 ].blocks = nblocks; }return 0 ; }
Signature · 1 line
2.19.1.6.gb485710b
PATCH v8 8/8 erofs-utils: mkfs: add `--ovlfs-strip` option
In reply to Jingbo XuAdd --ovlfs-strip=[0|1] option for tarfs and rebuild mode for now in order to control whether some overlayfs related stuffs (e.g. whiteout files, OVL_XATTR_OPAQUE and OVL_XATTR_ORIGIN xattrs) are finally stripped.
This option is disabled by default for mkfs, that is, the overlayfs related stuffs described above are kept in the image by default.
Specify --ovlfs-strip=1 explicitly to strip these stuffs.
--- include/erofs/config.h | 1 + include/erofs/xattr.h | 1 + lib/inode.c | 17 ++++++++++++++--- lib/xattr.c | 18 ++++++++++++++++++ mkfs/main.c | 8 ++++++++ 5 files changed, 42 insertions(+), 3 deletions(-)
include/erofs/config.h +1 -0
@@ -54,6 +54,7 @@ struct erofs_configure {bool c_showprogress;bool c_extra_ea_name_prefixes;bool c_xattr_name_filter; +bool c_ovlfs_strip;#ifdef HAVE_LIBSELINUX struct selabel_handle *sehnd;
include/erofs/xattr.h +1 -0
@@ -57,6 +57,7 @@ int erofs_xattr_prefixes_init(struct erofs_sb_info *sbi);int erofs_setxattr (struct erofs_inode *inode,char *key,const void *value,size_t size);int erofs_set_opaque_xattr (struct erofs_inode *inode); +void erofs_clear_opaque_xattr (struct erofs_inode *inode);int erofs_set_origin_xattr (struct erofs_inode *inode);int erofs_read_xattrs_from_disk (struct erofs_inode *inode);
lib/inode.c +14 -3
@@ -1326,7 +1326,7 @@ struct erofs_inode *erofs_mkfs_build_special_from_fd(int fd, const char *name)int erofs_rebuild_dump_tree (struct erofs_inode *dir) { -struct erofs_dentry *d; +struct erofs_dentry *d, *n;unsigned int nr_subdirs;int ret;
@@ -1341,7 +1341,10 @@ int erofs_rebuild_dump_tree(struct erofs_inode *dir) dir->inode_isize =sizeof (struct erofs_inode_compact ); } -if (dir->whiteouts) +/* strip all unnecessary overlayfs xattrs when ovlfs_strip is enabled */ +if (cfg.c_ovlfs_strip) +erofs_clear_opaque_xattr (dir); +else if (dir->whiteouts)erofs_set_origin_xattr (dir); ret =erofs_prepare_xattr_ibody (dir);
@@ -1373,8 +1376,16 @@ int erofs_rebuild_dump_tree(struct erofs_inode *dir) } nr_subdirs =0 ; -list_for_each_entry (d, &dir->i_subdirs, d_child) +list_for_each_entry_safe (d, n, &dir->i_subdirs, d_child) { +if (cfg.c_ovlfs_strip &&erofs_inode_is_whiteout (d->inode)) { +erofs_dbg ("remove whiteout %s" , d->inode->i_srcpath); +list_del (&d->d_child); +erofs_d_invalidate (d); +free (d); +continue ; + } ++nr_subdirs; + } ret =erofs_prepare_dir_layout (dir, nr_subdirs);if (ret)
lib/xattr.c +18 -0
@@ -499,11 +499,29 @@ int erofs_setxattr(struct erofs_inode *inode, char *key,return erofs_xattr_add (&inode->i_xattrs, item); } +static void erofs_clearxattr (struct erofs_inode *inode,const char *key) +{ +struct inode_xattr_node *node, *n; + +list_for_each_entry_safe (node, n, &inode->i_xattrs, list) { +if (!strcmp (node->item->kvbuf, key)) { +list_del (&node->list); +put_xattritem (node->item); +free (node); + } + } +} +int erofs_set_opaque_xattr (struct erofs_inode *inode) {return erofs_setxattr (inode,OVL_XATTR_OPAQUE ,"y" ,1 ); } +void erofs_clear_opaque_xattr (struct erofs_inode *inode) +{ +erofs_clearxattr (inode,OVL_XATTR_OPAQUE ); +} +int erofs_set_origin_xattr (struct erofs_inode *inode) {return erofs_setxattr (inode,OVL_XATTR_ORIGIN ,NULL ,0 );
mkfs/main.c +8 -0
@@ -64,6 +64,7 @@ static struct option long_options[] = { {"fs-config-file", required_argument, NULL, 514}, {"block-list-file", required_argument, NULL, 515}, #endif + {"ovlfs-strip", optional_argument, NULL, 516}, {0, 0, 0, 0}, };
@@ -115,6 +116,7 @@ static void usage(void)" --preserve-mtime keep per-file modification time strictly\n" " --aufs replace aufs special files with overlayfs metadata\n" " --tar=[fi] generate an image from tarball(s)\n" +" --ovlfs-strip=[01] strip overlayfs metadata in the target image (e.g. whiteouts)\n" " --quiet quiet execution (do not write anything to standard output.)\n" #ifndef NDEBUG" --random-pclusterblks randomize pclusterblks for big pcluster (debugging only)\n"
@@ -516,6 +518,12 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])case 21 : erofstar.aufs = true;break ; +case 516 : +if (!optarg || !strcmp (optarg,"1" )) + cfg.c_ovlfs_strip = true; +else + cfg.c_ovlfs_strip = false; +break ;case 1 :usage ();exit (0 );
Signature · 1 line
2.19.1.6.gb485710b
Re: [PATCH v8 8/8] erofs-utils: mkfs: add `--ovlfs-strip` option
Quoted 13 lines · Jingbo Xu
Add
--ovlfs-strip=[0|1]option for tarfs and rebuild mode for now in order to control whether some overlayfs related stuffs (e.g. whiteout files,OVL_XATTR_OPAQUEandOVL_XATTR_ORIGINxattrs) are finally stripped.This option is disabled by default for mkfs, that is, the overlayfs related stuffs described above are kept in the image by default.
Specify
--ovlfs-strip=1explicitly to strip these stuffs.Signed-off-by Jingbo Xu <[email protected]>Signed-off-by Gao Xiang <[email protected]>---
...
lib/xattr.c int erofs_setxattr(struct erofs_inode *inode, char *key, · line 502
Quoted 9 lines
lib/xattr.c +1
-0@@ -499,11 +499,29 @@ int erofs_setxattr(struct erofs_inode *inode, char *key,return erofs_xattr_add (&inode->i_xattrs, item); } +static void erofs_clearxattr (struct erofs_inode *inode,const char *key)
I rename it as erofs_removexattr().
Thanks,
Gao Xiang
PATCH v3 1/1 erofs-utils: lib: refactor extended attribute name prefixes
In reply to Jingbo XuPreviously, the extended attribute name in xattr_item was actually the part with the matched prefix stripped, which makes it clumsy to strip or delete a specific attribute from one file.
To fix this, make the complete attribute name stored in xattr_item. One thing worth noting is that the attribute name in xattr_item has a trailing '\0' for ease of name comparison, while the trailing '\0' will get stripped when writing to on-disk erofs_xattr_entry.
---
v3:
- introduce EROFS_XATTR_KSIZE() and EROFS_XATTR_KVSIZE() macro
- make the full long name prefix (instead of the infix in v2) stored in ea_type_node; and thus compare with the full long name prefix in get_xattritem() directly
- convert the data type of in-memory prefix index and prefix_len from u8/u16 to unsigned int
--- lib/xattr.c | 379 ++++++++++++++++++++++------------------------------ 1 file changed, 161 insertions(+), 218 deletions(-)
lib/xattr.c +161 -218
@@ -51,6 +51,12 @@#ifndef XATTR_NAME_POSIX_ACL_DEFAULT #define XATTR_NAME_POSIX_ACL_DEFAULT "system.posix_acl_default"#endif +#ifndef XATTR_NAME_SECURITY_SELINUX +#define XATTR_NAME_SECURITY_SELINUX "security.selinux" +#endif +#ifndef XATTR_NAME_SECURITY_CAPABILITY +#define XATTR_NAME_SECURITY_CAPABILITY "security.capability" +#endif #ifndef OVL_XATTR_NAMESPACE #define OVL_XATTR_NAMESPACE "overlay."#endif
@@ -72,12 +78,23 @@#define EA_HASHTABLE_BITS 16 +/* extra one byte for the trailing `\0` of attribute name */ +#define EROFS_XATTR_KSIZE (kvlen) (kvlen[0] + 1) +#define EROFS_XATTR_KVSIZE (kvlen) (EROFS_XATTR_KSIZE(kvlen) + kvlen[1]) + +/* +* @base_index: the index of matched predefined short prefix +* @prefix: the index of matched long prefix if any; +* same as base_index otherwise +* @prefix_len: the length of matched long prefix if any; +* the length of matched predefined short prefix otherwise +*/ struct xattr_item {struct xattr_item *next_shared_xattr;const char *kvbuf;unsigned int hash[2 ], len[2 ], count;int shared_xattr_id; -u8 prefix; +unsigned int prefix, base_index, prefix_len;struct hlist_node node; };
@@ -93,7 +110,7 @@ static unsigned int shared_xattrs_count;static struct xattr_prefix {const char *prefix; -u8 prefix_len; +unsigned int prefix_len; } xattr_types[] = { [EROFS_XATTR_INDEX_USER ] = {XATTR_USER_PREFIX ,
@@ -116,11 +133,27 @@ static struct xattr_prefix {struct ea_type_node {struct list_head list;struct xattr_prefix type; -u8 index; +unsigned int index, base_index, base_len; }; +static LIST_HEAD (ea_name_prefixes);static unsigned int ea_prefix_count; +static bool match_prefix (const char *key,unsigned int *index, +unsigned int *len) +{ +struct xattr_prefix *p; + +for (p = xattr_types; p < xattr_types +ARRAY_SIZE (xattr_types); ++p) { +if (p->prefix && !strncmp (p->prefix, key, p->prefix_len)) { + *len = p->prefix_len; + *index = p - xattr_types; +return true; + } + } +return false; +} +static unsigned int BKDRHash (char *str,unsigned int len) {const unsigned int seed =131313 ;
@@ -133,13 +166,12 @@ static unsigned int BKDRHash(char *str, unsigned int len)return hash; } -static unsigned int xattr_item_hash (u8 prefix,char *buf, -unsigned int len[2 ],unsigned int hash[2 ]) +static unsigned int xattr_item_hash (char *buf,unsigned int len[2 ], +unsigned int hash[2 ]) { hash[0 ] =BKDRHash (buf, len[0 ]);/* key */ hash[1 ] =BKDRHash (buf + len[0 ], len[1 ]);/* value */ - -return prefix ^ hash[0 ] ^ hash[1 ]; +return hash[0 ] ^ hash[1 ]; }static unsigned int put_xattritem (struct xattr_item *item)
@@ -150,17 +182,14 @@ static unsigned int put_xattritem(struct xattr_item *item)return 0 ; } -static struct xattr_item *get_xattritem (u8 prefix,char *kvbuf, -unsigned int len[2 ]) +static struct xattr_item *get_xattritem (char *kvbuf,unsigned int len[2 ]) {struct xattr_item *item;unsigned int hash[2 ], hkey; - hkey =xattr_item_hash (prefix, kvbuf, len, hash); - + hkey =xattr_item_hash (kvbuf, len, hash);hash_for_each_possible (ea_hashtable, item, node, hkey) { -if (prefix == item->prefix && - item->len[0 ] == len[0 ] && item->len[1 ] == len[1 ] && +if (item->len[0 ] == len[0 ] && item->len[1 ] == len[1 ] && item->hash[0 ] == hash[0 ] && item->hash[1 ] == hash[1 ] && !memcmp (kvbuf, item->kvbuf, len[0 ] + len[1 ])) {free (kvbuf);
@@ -174,6 +203,14 @@ static struct xattr_item *get_xattritem(u8 prefix, char *kvbuf,free (kvbuf);return ERR_PTR (-ENOMEM ); } + +if (!match_prefix (kvbuf, &item->base_index, &item->prefix_len)) { +free (item); +free (kvbuf); +return ERR_PTR (-ENODATA ); + } +DBG_BUGON (len[0 ] < item->prefix_len); +INIT_HLIST_NODE (&item->node); item->count =1 ; item->kvbuf = kvbuf;
@@ -182,56 +219,37 @@ static struct xattr_item *get_xattritem(u8 prefix, char *kvbuf, item->hash[0 ] = hash[0 ]; item->hash[1 ] = hash[1 ]; item->shared_xattr_id =-1 ; - item->prefix = prefix; -hash_add (ea_hashtable, &item->node, hkey); -return item; -} + item->prefix = item->base_index; -static bool match_base_prefix (const char *key,u8 *index,u16 *len) -{ -struct xattr_prefix *p; +if (cfg.c_extra_ea_name_prefixes) { +struct ea_type_node *tnode; -for (p = xattr_types; p < xattr_types +ARRAY_SIZE (xattr_types); ++p) { -if (p->prefix && !strncmp (p->prefix, key, p->prefix_len)) { - *len = p->prefix_len; - *index = p - xattr_types; -return true; +list_for_each_entry (tnode, &ea_name_prefixes, list) { +if (item->base_index == tnode->base_index && + !strncmp (tnode->type.prefix, kvbuf, + tnode->type.prefix_len)) { + item->prefix = tnode->index; + item->prefix_len = tnode->type.prefix_len; +break ; + } } } -return false; -} -static bool match_prefix (const char *key,u8 *index,u16 *len) -{ -struct xattr_prefix *p; -struct ea_type_node *tnode; - -list_for_each_entry (tnode, &ea_name_prefixes, list) { - p = &tnode->type; -if (p->prefix && !strncmp (p->prefix, key, p->prefix_len)) { - *len = p->prefix_len; - *index = tnode->index; -return true; - } - } -return match_base_prefix (key, index, len); +hash_add (ea_hashtable, &item->node, hkey); +return item; }static struct xattr_item *parse_one_xattr (const char *path,const char *key,unsigned int keylen) {ssize_t ret; -u8 prefix; -u16 prefixlen;unsigned int len[2 ];char *kvbuf;erofs_dbg ("parse xattr [%s] of %s" , path, key); -if (!match_prefix (key, &prefix, &prefixlen)) -return ERR_PTR (-ENODATA ); - -DBG_BUGON (keylen < prefixlen); +/* length of the key */ + len[0 ] = keylen;/* determine length of the value */ #ifdef HAVE_LGETXATTR
@@ -246,19 +264,18 @@ static struct xattr_item *parse_one_xattr(const char *path, const char *key, len[1 ] = ret;/* allocate key-value buffer */ - len[0 ] = keylen - prefixlen; - - kvbuf =malloc (len[0 ] + len[1 ]); + kvbuf =malloc (EROFS_XATTR_KVSIZE (len));if (!kvbuf)return ERR_PTR (-ENOMEM ); -memcpy (kvbuf, key + prefixlen, len[0 ]); +sprintf (kvbuf,"%s" , key);if (len[1 ]) {/* copy value to buffer */ #ifdef HAVE_LGETXATTR - ret =lgetxattr (path, key, kvbuf + len[0 ], len[1 ]); + ret =lgetxattr (path, key, kvbuf +EROFS_XATTR_KSIZE (len), + len[1 ]);#elif defined(__APPLE__) - ret =getxattr (path, key, kvbuf + len[0 ], len[1 ],0 , -XATTR_NOFOLLOW ); + ret =getxattr (path, key, kvbuf +EROFS_XATTR_KSIZE (len), + len[1 ],0 ,XATTR_NOFOLLOW );#else free (kvbuf);return ERR_PTR (-EOPNOTSUPP );
@@ -273,7 +290,7 @@ static struct xattr_item *parse_one_xattr(const char *path, const char *key, len[1 ] = ret; } } -return get_xattritem (prefix, kvbuf, len); +return get_xattritem (kvbuf, len); } static struct xattr_item *erofs_get_selabel_xattr(const char *srcpath,
@@ -308,16 +325,17 @@ static struct xattr_item *erofs_get_selabel_xattr(const char *srcpath, return NULL; } - len[0 ] =sizeof ("selinux" ) -1 ; + len[0 ] =sizeof (XATTR_NAME_SECURITY_SELINUX ) -1 ; len[1 ] =strlen (secontext); - kvbuf =malloc (len[0 ] + len[1 ] +1 ); + kvbuf =malloc (EROFS_XATTR_KVSIZE (len));if (!kvbuf) {freecon (secontext);return ERR_PTR (-ENOMEM ); } -sprintf (kvbuf,"selinux%s" , secontext); +sprintf (kvbuf,"%s" ,XATTR_NAME_SECURITY_SELINUX ); +memcpy (kvbuf +EROFS_XATTR_KSIZE (len), secontext, len[1 ]);freecon (secontext); -return get_xattritem (EROFS_XATTR_INDEX_SECURITY , kvbuf, len); +return get_xattritem (kvbuf, len); }#endif return NULL ;
@@ -466,24 +484,18 @@ int erofs_setxattr(struct erofs_inode *inode, char *key,char *kvbuf;unsigned int len[2 ];struct xattr_item *item; -u8 prefix; -u16 prefixlen; - -if (!match_prefix (key, &prefix, &prefixlen)) -return -ENODATA ; + len[0 ] =strlen (key); len[1 ] = size; -/* allocate key-value buffer */ - len[0 ] =strlen (key) - prefixlen; - kvbuf =malloc (len[0 ] + len[1 ]); + kvbuf =malloc (EROFS_XATTR_KVSIZE (len));if (!kvbuf)return -ENOMEM ; -memcpy (kvbuf, key + prefixlen, len[0 ]); -memcpy (kvbuf + len[0 ], value, size); +sprintf (kvbuf,"%s" , key); +memcpy (kvbuf +EROFS_XATTR_KSIZE (len), value, size); - item =get_xattritem (prefix, kvbuf, len); + item =get_xattritem (kvbuf, len);if (IS_ERR (item))return PTR_ERR (item);DBG_BUGON (!item);
@@ -513,22 +525,22 @@ static int erofs_droid_xattr_set_caps(struct erofs_inode *inode)if (!capabilities)return 0 ; - len[0 ] =sizeof ("capability" ) -1 ; + len[0 ] =sizeof (XATTR_NAME_SECURITY_CAPABILITY ) -1 ; len[1 ] =sizeof (caps); - kvbuf =malloc (len[0 ] + len[1 ]); + kvbuf =malloc (EROFS_XATTR_KVSIZE (len));if (!kvbuf)return -ENOMEM ; -memcpy (kvbuf,"capability" , len[0 ]); +sprintf (kvbuf,"%s" ,XATTR_NAME_SECURITY_CAPABILITY ); caps.magic_etc =VFS_CAP_REVISION_2 |VFS_CAP_FLAGS_EFFECTIVE ; caps.data[0 ].permitted = (u32 ) capabilities; caps.data[0 ].inheritable =0 ; caps.data[1 ].permitted = (u32) (capabilities >>32 ); caps.data[1 ].inheritable =0 ; -memcpy (kvbuf + len[0 ], &caps, len[1 ]); +memcpy (kvbuf +EROFS_XATTR_KSIZE (len), &caps, len[1 ]); - item =get_xattritem (EROFS_XATTR_INDEX_SECURITY , kvbuf, len); + item =get_xattritem (kvbuf, len);if (IS_ERR (item))return PTR_ERR (item);DBG_BUGON (!item);
@@ -558,6 +570,13 @@ int erofs_scan_file_xattrs(struct erofs_inode *inode)return erofs_droid_xattr_set_caps (inode); } +static inline unsigned int erofs_next_xattr_align (unsigned int pos, +struct xattr_item *item) +{ +return EROFS_XATTR_ALIGN (pos +sizeof (struct erofs_xattr_entry ) + + item->len[0 ] + item->len[1 ] - item->prefix_len); +} +int erofs_prepare_xattr_ibody (struct erofs_inode *inode) {int ret;
@@ -572,14 +591,13 @@ int erofs_prepare_xattr_ibody(struct erofs_inode *inode)/* get xattr ibody size */ ret =sizeof (struct erofs_xattr_ibody_header );list_for_each_entry (node, ixattrs, list) { -const struct xattr_item *item = node->item; +struct xattr_item *item = node->item;if (item->shared_xattr_id >=0 ) { ret +=sizeof (__le32);continue ; } - ret +=sizeof (struct erofs_xattr_entry ); - ret =EROFS_XATTR_ALIGN (ret + item->len[0 ] + item->len[1 ]); + ret =erofs_next_xattr_align (ret, item); } inode->xattr_isize = ret;return ret;
@@ -688,25 +706,9 @@ static int comp_shared_xattr_item(const void *a, const void *b)return la > lb; } -static inline int erofs_xattr_index_by_prefix (const char *prefix,int *len) -{ -if (!strncmp (prefix,XATTR_USER_PREFIX ,XATTR_USER_PREFIX_LEN )){ - *len =XATTR_USER_PREFIX_LEN ; -return EROFS_XATTR_INDEX_USER ; - }else if (!strncmp (prefix,XATTR_TRUSTED_PREFIX ,XATTR_TRUSTED_PREFIX_LEN )) { - *len =XATTR_TRUSTED_PREFIX_LEN ; -return EROFS_XATTR_INDEX_TRUSTED ; - }else if (!strncmp (prefix,XATTR_SECURITY_PREFIX ,XATTR_SECURITY_PREFIX_LEN )) { - *len =XATTR_SECURITY_PREFIX_LEN ; -return EROFS_XATTR_INDEX_SECURITY ; - } -return -ENODATA ; -} -int erofs_xattr_write_name_prefixes (struct erofs_sb_info *sbi,FILE *f) {struct ea_type_node *tnode; -struct xattr_prefix *p;off_t offset;if (!ea_prefix_count)
@@ -732,16 +734,13 @@ int erofs_xattr_write_name_prefixes(struct erofs_sb_info *sbi, FILE *f)u8 data[EROFS_NAME_LEN +2 +sizeof (struct erofs_xattr_long_prefix )]; } u; -int ret, len; +int len, infix_len; - p = &tnode->type; - ret =erofs_xattr_index_by_prefix (p->prefix, &len); -if (ret <0 ) -return ret; - u.s.prefix.base_index = ret; -memcpy (u.s.prefix.infix, p->prefix + len, p->prefix_len - len); - len =sizeof (struct erofs_xattr_long_prefix ) + - p->prefix_len - len; + u.s.prefix.base_index = tnode->base_index; + infix_len = tnode->type.prefix_len - tnode->base_len; +memcpy (u.s.prefix.infix, tnode->type.prefix + tnode->base_len, + infix_len); + len =sizeof (struct erofs_xattr_long_prefix ) + infix_len; u.s.size =cpu_to_le16 (len);if (fwrite (&u.s,sizeof (__le16) + len,1 , f) !=1 )return -EIO ;
@@ -754,11 +753,30 @@ int erofs_xattr_write_name_prefixes(struct erofs_sb_info *sbi, FILE *f)return 0 ; } +static void erofs_write_xattr_entry (char *buf,struct xattr_item *item) +{ +struct erofs_xattr_entry entry = { + .e_name_index = item->prefix, + .e_name_len = item->len[0 ] - item->prefix_len, + .e_value_size =cpu_to_le16 (item->len[1 ]), + }; + +memcpy (buf, &entry,sizeof (entry)); + buf +=sizeof (struct erofs_xattr_entry ); +memcpy (buf, item->kvbuf + item->prefix_len, + item->len[0 ] - item->prefix_len); + buf += item->len[0 ] - item->prefix_len; +memcpy (buf, item->kvbuf + item->len[0 ] +1 , item->len[1 ]); + +erofs_dbg ("writing xattr %d %s (%d %s)" , item->base_index, item->kvbuf, + item->prefix, item->kvbuf + item->prefix_len); +} +int erofs_build_shared_xattrs_from_path (struct erofs_sb_info *sbi,const char *path) {int ret;struct erofs_buffer_head *bh; -struct xattr_item *n, **sorted_n; +struct xattr_item *item, *n, **sorted_n;char *buf;unsigned int p, i;erofs_off_t off;
@@ -787,13 +805,11 @@ int erofs_build_shared_xattrs_from_path(struct erofs_sb_info *sbi, const char *p i =0 ;while (shared_xattrs_list) { -struct xattr_item *item = shared_xattrs_list; - + item = shared_xattrs_list; sorted_n[i++] = item; shared_xattrs_list = item->next_shared_xattr; - shared_xattrs_size +=sizeof (struct erofs_xattr_entry ); - shared_xattrs_size =EROFS_XATTR_ALIGN (shared_xattrs_size + - item->len[0 ] + item->len[1 ]); + shared_xattrs_size =erofs_next_xattr_align (shared_xattrs_size, + item); }DBG_BUGON (i != shared_xattrs_count); sorted_n[i] =NULL ;
@@ -820,20 +836,11 @@ int erofs_build_shared_xattrs_from_path(struct erofs_sb_info *sbi, const char *p off %=erofs_blksiz (sbi); p =0 ;for (i =0 ; i < shared_xattrs_count; i++) { -struct xattr_item *item = sorted_n[i]; -const struct erofs_xattr_entry entry = { - .e_name_index = item->prefix, - .e_name_len = item->len[0 ], - .e_value_size =cpu_to_le16 (item->len[1 ]) - }; - + item = sorted_n[i]; +erofs_write_xattr_entry (buf + p, item); item->next_shared_xattr = sorted_n[i +1 ]; item->shared_xattr_id = (off + p) /sizeof (__le32); - -memcpy (buf + p, &entry,sizeof (entry)); - p +=sizeof (struct erofs_xattr_entry ); -memcpy (buf + p, item->kvbuf, item->len[0 ] + item->len[1 ]); - p =EROFS_XATTR_ALIGN (p + item->len[0 ] + item->len[1 ]); + p =erofs_next_xattr_align (p, item); } shared_xattrs_list = sorted_n[0 ];free (sorted_n);
@@ -846,70 +853,12 @@ out:return ret; } -static int erofs_xattr_filter_hashbit (struct xattr_item *item) -{ -u8 prefix = item->prefix; -const char *key = item->kvbuf; -unsigned int len = item->len[0 ]; -char *name =NULL ; -uint32_t hashbit; - -if (prefix &EROFS_XATTR_LONG_PREFIX ) { -struct ea_type_node *tnode; -u16 prefix_len; -int ret; - -list_for_each_entry (tnode, &ea_name_prefixes, list) { -if (tnode->index == item->prefix) { - ret =asprintf (&name,"%s%.*s" , - tnode->type.prefix, len, key); -if (ret <0 ) -return -ENOMEM ; -break ; - } - } -if (!name) -return -ENOENT ; - -if (!match_base_prefix (name, &prefix, &prefix_len)) { -free (name); -return -ENOENT ; - } - key = name + prefix_len; - len =strlen (key); - } - - hashbit =xxh32 (key, len,EROFS_XATTR_FILTER_SEED + prefix) & - (EROFS_XATTR_FILTER_BITS -1 ); -if (name) -free (name); -return hashbit; -} - -static u32 erofs_xattr_filter_map (struct list_head *ixattrs) -{ -struct inode_xattr_node *node, *n; -u32 name_filter; -int hashbit; - - name_filter =0 ; -list_for_each_entry_safe (node, n, ixattrs, list) { - hashbit =erofs_xattr_filter_hashbit (node->item); -if (hashbit <0 ) { -erofs_warn ("failed to generate xattr name filter: %s" , -strerror (-hashbit)); -return 0 ; - } - name_filter |= (1UL << hashbit); - } -return EROFS_XATTR_FILTER_DEFAULT & ~name_filter; -} -char *erofs_export_xattr_ibody (struct erofs_inode *inode) {struct list_head *ixattrs = &inode->i_xattrs;unsigned int size = inode->xattr_isize;struct inode_xattr_node *node, *n; +struct xattr_item *item;struct erofs_xattr_ibody_header *header;LIST_HEAD (ilst);unsigned int p;
@@ -922,16 +871,29 @@ char *erofs_export_xattr_ibody(struct erofs_inode *inode) header->h_shared_count =0 ;if (cfg.c_xattr_name_filter) { - header->h_name_filter = -cpu_to_le32 (erofs_xattr_filter_map (ixattrs)); +u32 name_filter =0 ; +int hashbit; +unsigned int base_len; + +list_for_each_entry (node, ixattrs, list) { + item = node->item; + base_len = xattr_types[item->base_index].prefix_len; + hashbit =xxh32 (item->kvbuf + base_len, + item->len[0 ] - base_len, +EROFS_XATTR_FILTER_SEED + item->base_index) & + (EROFS_XATTR_FILTER_BITS -1 ); + name_filter |= (1UL << hashbit); + } + name_filter =EROFS_XATTR_FILTER_DEFAULT & ~name_filter; + + header->h_name_filter =cpu_to_le32 (name_filter);if (header->h_name_filter)erofs_sb_set_xattr_filter (inode->sbi); } p =sizeof (struct erofs_xattr_ibody_header );list_for_each_entry_safe (node, n, ixattrs, list) { -struct xattr_item *const item = node->item; - + item = node->item;list_del (&node->list);/* move inline xattrs to the onstack list */
@@ -948,18 +910,9 @@ char *erofs_export_xattr_ibody(struct erofs_inode *inode) }list_for_each_entry_safe (node, n, &ilst, list) { -struct xattr_item *const item = node->item; -const struct erofs_xattr_entry entry = { - .e_name_index = item->prefix, - .e_name_len = item->len[0 ], - .e_value_size =cpu_to_le16 (item->len[1 ]) - }; - -memcpy (buf + p, &entry,sizeof (entry)); - p +=sizeof (struct erofs_xattr_entry ); -memcpy (buf + p, item->kvbuf, item->len[0 ] + item->len[1 ]); - p =EROFS_XATTR_ALIGN (p + item->len[0 ] + item->len[1 ]); - + item = node->item; +erofs_write_xattr_entry (buf + p, item); + p =erofs_next_xattr_align (p, item);list_del (&node->list);free (node);put_xattritem (item);
@@ -1354,8 +1307,7 @@ int erofs_getxattr(struct erofs_inode *vi, const char *name, char *buffer, size_t buffer_size) {int ret; -u8 prefix; -u16 prefixlen; +unsigned int prefix, prefixlen;struct getxattr_iter it;if (!name)
@@ -1365,7 +1317,7 @@ int erofs_getxattr(struct erofs_inode *vi, const char *name, char *buffer,if (ret)return ret; -if (!match_base_prefix (name, &prefix, &prefixlen)) +if (!match_prefix (name, &prefix, &prefixlen))return -ENODATA ; it.it.sbi = vi->sbi;
@@ -1532,34 +1484,25 @@ int erofs_listxattr(struct erofs_inode *vi, char *buffer, size_t buffer_size)int erofs_xattr_insert_name_prefix (const char *prefix) {struct ea_type_node *tnode; -struct xattr_prefix *p; -bool matched = false; -char *s;if (ea_prefix_count >=0x80 ||strlen (prefix) >UINT8_MAX )return -EOVERFLOW ; -for (p = xattr_types; p < xattr_types +ARRAY_SIZE (xattr_types); ++p) { -if (!strncmp (p->prefix, prefix, p->prefix_len)) { - matched = true; -break ; - } - } -if (!matched) -return -ENODATA ; - - s =strdup (prefix); -if (!s) + tnode =calloc (1 ,sizeof (*tnode)); +if (!tnode)return -ENOMEM ; - tnode =malloc (sizeof (*tnode)); -if (!tnode) { -free (s); -return -ENOMEM ; +if (!match_prefix (prefix, &tnode->base_index, &tnode->base_len)) { +free (tnode); +return -ENODATA ; } - tnode->type.prefix = s; tnode->type.prefix_len =strlen (prefix); + tnode->type.prefix =strdup (prefix); +if (!tnode->type.prefix) { +free (tnode); +return -ENOMEM ; + } tnode->index =EROFS_XATTR_LONG_PREFIX | ea_prefix_count; ea_prefix_count++;
Signature · 1 line
2.19.1.6.gb485710b
Re: [PATCH v3] erofs-utils: lib: refactor extended attribute name prefixes
Sorry I send this by mistake. It has no difference with the previously sent v3 here[*]
Sorry for the noise.
[*]
https://lore.kernel.org/all/[email protected]/
Signature · 2 lines
Thanks, Jingbo