erofs-utils: mkfs: introduce rebuild mode
PATCH 2/13 erofs-utils: lib: add list_splice_tail() helper
In reply to Jingbo XuAdd list_splice_tail() helper.
--- include/erofs/list.h | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+)
include/erofs/list.h +20 -0
@@ -70,6 +70,26 @@ static inline int list_empty(struct list_head *head)return head->next == head; } +static inline void __list_splice (struct list_head *list, +struct list_head *prev,struct list_head *next) +{ +struct list_head *first = list->next; +struct list_head *last = list->prev; + + first->prev = prev; + prev->next = first; + + last->next = next; + next->prev = last; +} + +static inline void list_splice_tail (struct list_head *list, +struct list_head *head) +{ +if (!list_empty (list)) +__list_splice (list, head->prev, head); +} +#define list_entry (ptr, type, member) container_of(ptr, type, member)#define list_first_entry (ptr, type, member) \
Signature · 1 line
2.19.1.6.gb485710b
PATCH 1/13 erofs-utils: fix overriding of i_rdev for special device
In reply to Jingbo XuIn index mode for tarerofs, tarerofs_write_chunk_indexes() is still called even for files with 0 i_size, and inode->u.chunkformat is initialized accordingly. This will make the previously set inode->u.i_rdev be overridden as u.chunkformat and u.i_rdev are reused in one union.
To tidy up the code of writing file, extract two helpers writing file in index and non-index mode. Also fix the missing erofs_iput() when tarerofs_write_chunkes() fails.
--- lib/tar.c | 77 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 45 insertions(+), 32 deletions(-)
lib/tar.c +45 -32
@@ -359,6 +359,44 @@ void tarerofs_remove_inode(struct erofs_inode *inode) --inode->i_parent->i_nlink; } +static int tarerofs_write_file_data (struct erofs_inode *inode, +struct erofs_tarfile *tar) +{ +unsigned int j, rem; +char buf[65536 ]; + +if (!inode->i_tmpfile) { + inode->i_tmpfile =tmpfile (); +if (!inode->i_tmpfile) +return -ENOSPC ; + } + +for (j = inode->i_size; j; ) { + rem = min_t(unsigned int, sizeof(buf), j); + +if (erofs_read_from_fd (tar->fd, buf, rem) != rem || +fwrite (buf, rem,1 , inode->i_tmpfile) !=1 ) +return -EIO ; + j -= rem; + } +fseek (inode->i_tmpfile,0 ,SEEK_SET ); + inode->with_tmpfile = true; +return 0 ; +} + +static int tarerofs_write_file_index (struct erofs_inode *inode, +struct erofs_tarfile *tar,erofs_off_t data_offset) +{ +int ret; + + ret =tarerofs_write_chunkes (inode, data_offset); +if (ret) +return ret; +if (erofs_lskip (tar->fd, inode->i_size)) +return -EIO ; +return 0 ; +} +int tarerofs_parse_tar (struct erofs_inode *root,struct erofs_tarfile *tar) {char path[PATH_MAX ];
@@ -672,41 +710,16 @@ new_inode: inode->i_size =strlen (eh.link); inode->i_link =malloc (inode->i_size +1 );memcpy (inode->i_link, eh.link, inode->i_size +1 ); - }else if (tar->index_mode) { - ret =tarerofs_write_chunkes (inode, data_offset); -if (ret) - goto out; -if (erofs_lskip (tar->fd, inode->i_size)) { + }else if (inode->i_size) { +if (tar->index_mode) + ret =tarerofs_write_file_index (inode, tar, + data_offset); +else + ret =tarerofs_write_file_data (inode, tar); +if (ret) {erofs_iput (inode); - ret = -EIO ; goto out; } - }else { -char buf[65536 ]; - -if (!inode->i_tmpfile) { - inode->i_tmpfile =tmpfile (); - -if (!inode->i_tmpfile) { -erofs_iput (inode); - ret = -ENOSPC ; - goto out; - } - } - -for (j = inode->i_size; j; ) { - rem =min_t (int,sizeof (buf), j); - -if (erofs_read_from_fd (tar->fd, buf, rem) != rem || -fwrite (buf, rem,1 , inode->i_tmpfile) !=1 ) { -erofs_iput (inode); - ret = -EIO ; - goto out; - } - j -= rem; - } -fseek (inode->i_tmpfile,0 ,SEEK_SET ); - inode->with_tmpfile = true; } inode->i_nlink++; ret =0 ;
Signature · 1 line
2.19.1.6.gb485710b
cover erofs-utils: mkfs: introduce rebuild mode
Introduce a new rebuild mode merging multiple erofs images generated from either tarerfs index mode (--tar=i):
mkfs.erofs --tar=i --aufs layer0.erofs layer0.tar ... mkfs.erofs --tar=i --aufs layerN.erofs layerN.tar mkfs.erofs merge.erofs layerN.erofs ... layer0.erofs
or tarerofs non-index mode (--tar=f):
mkfs.erofs --tar=f -Enoinline_data --aufs layer0.erofs layer0.tar ... mkfs.erofs --tar=f -Enoinline_data --aufs layerN.erofs layerN.tar mkfs.erofs merge.erofs layerN.erofs ... layer0.erofs
The rebuild mode doesn't support flat inline datalayout yet.
Jingbo Xu (13):
erofs-utils: fix overriding of i_rdev for special device
erofs-utils: lib: add list_splice_tail() helper
erofs-utils: lib: read i_ino in erofs_read_inode_from_disk()
erofs-utils: lib: fix erofs_init_devices() in multidev mode
erofs-utils: lib: keep self maintained devname
erofs-utils: lib: make erofs_get_unhashed_chunk() global
erofs-utils: lib: add erofs_read_xattrs_from_disk() helper
erofs-utils: lib: add erofs_inode_tag_opaque() helper
erofs-utils: lib: add erofs_insert_ihash() helper
erofs-utils: lib: add erofs_rebuild_dump_tree() helper
erofs-utils: lib: add erofs_rebuild_get_dentry() helper
erofs-utils: lib: add erofs_rebuild_load_tree() helper
erofs-utils: mkfs: introduce rebuild mode
include/erofs/blobchunk.h | 2 + include/erofs/inode.h | 5 +- include/erofs/internal.h | 9 +- include/erofs/list.h | 20 ++ include/erofs/rebuild.h | 21 +++ include/erofs/xattr.h | 23 +++ lib/Makefile.am | 3 +- lib/blobchunk.c | 2 +- lib/inode.c | 33 +++- lib/io.c | 14 +- lib/namei.c | 2 + lib/rebuild.c | 387 ++++++++++++++++++++++++++++++++++++++ lib/super.c | 11 +- lib/tar.c | 193 +++++-------------- lib/xattr.c | 120 ++++++++++++ mkfs/Makefile.am | 3 +- mkfs/main.c | 198 ++++++++++++++----- 17 files changed, 839 insertions(+), 207 deletions(-) create mode 100644 include/erofs/rebuild.h create mode 100644 lib/rebuild.c
Signature · 1 line
2.19.1.6.gb485710b
PATCH 3/13 erofs-utils: lib: read i_ino in erofs_read_inode_from_disk()
i_ino[0] is a unique inode serial number in the erofs filesystem where the inode resides, and also serves as the on-disk inode number, while i_ino[1] is a unique number identifying the source inode in the source directory, which is usually derived from st->st_ino.
Read on-disk ino and store it in i_ino[0] given the above background.
--- lib/namei.c | 2 ++ 1 file changed, 2 insertions(+)
lib/namei.c +2 -0
@@ -60,6 +60,7 @@ int erofs_read_inode_from_disk(struct erofs_inode *vi) die = (struct erofs_inode_extended *)buf; vi->xattr_isize =erofs_xattr_ibody_size (die->i_xattr_icount); vi->i_mode =le16_to_cpu (die->i_mode); + vi->i_ino[0 ] =le32_to_cpu (die->i_ino);switch (vi->i_mode &S_IFMT ) {case S_IFREG :
@@ -95,6 +96,7 @@ int erofs_read_inode_from_disk(struct erofs_inode *vi) vi->inode_isize =sizeof (struct erofs_inode_compact ); vi->xattr_isize =erofs_xattr_ibody_size (dic->i_xattr_icount); vi->i_mode =le16_to_cpu (dic->i_mode); + vi->i_ino[0 ] =le32_to_cpu (dic->i_ino);switch (vi->i_mode &S_IFMT ) {case S_IFREG :
Signature · 1 line
2.19.1.6.gb485710b
PATCH 4/13 erofs-utils: lib: fix erofs_init_devices() in multidev mode
In reply to Jingbo XuFix the missing le32_to_cpu(). Read and cache blocks of each device for later use.
Initialize sbi->extra_devices from on-disk extra_devices.
--- lib/super.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-)
lib/super.c +4 -7
@@ -36,11 +36,7 @@ static int erofs_init_devices(struct erofs_sb_info *sbi,else ondisk_extradevs =le16_to_cpu (dsb->extra_devices); -if (ondisk_extradevs != sbi->extra_devices) { -erofs_err ("extra devices don't match (ondisk %u, given %u)" , - ondisk_extradevs, sbi->extra_devices); -return -EINVAL ; - } + sbi->extra_devices = ondisk_extradevs;if (!ondisk_extradevs)return 0 ;
@@ -59,8 +55,9 @@ static int erofs_init_devices(struct erofs_sb_info *sbi,return ret; } - sbi->devs[i].mapped_blkaddr = dis.mapped_blkaddr; - sbi->total_blocks += dis.blocks; + sbi->devs[i].mapped_blkaddr =le32_to_cpu (dis.mapped_blkaddr); + sbi->devs[i].blocks =le32_to_cpu (dis.blocks); + sbi->total_blocks += sbi->devs[i].blocks; pos +=EROFS_DEVT_SLOT_SIZE ; }return 0 ;
Signature · 1 line
2.19.1.6.gb485710b
PATCH 5/13 erofs-utils: lib: keep self maintained devname
In reply to Jingbo XuKeep self allocated and maintained devname in erofs_sb_info.
--- include/erofs/internal.h | 2 +- lib/io.c | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-)
include/erofs/internal.h +1 -1
@@ -61,7 +61,7 @@ struct erofs_device_info {struct erofs_sb_info {struct erofs_device_info *devs; -const char *devname; +char *devname;u64 total_blocks;u64 primarydevice_blocks;
lib/io.c +12 -2
@@ -10,6 +10,7 @@#ifndef _GNU_SOURCE#define _GNU_SOURCE#endif +#include <stdlib.h> #include <sys/stat.h> #include <sys/ioctl.h> #include "erofs/io.h"
@@ -46,6 +47,7 @@ static int dev_get_blkdev_size(int fd, u64 *bytes)void dev_close (struct erofs_sb_info *sbi) {close (sbi->devfd); +free (sbi->devname); sbi->devname =NULL ; sbi->devfd =-1 ; sbi->devsz =0 ;
@@ -95,7 +97,11 @@ int dev_open(struct erofs_sb_info *sbi, const char *dev)return -EINVAL ; } - sbi->devname = dev; + sbi->devname =strdup (dev); +if (!sbi->devname) { +close (fd); +return -ENOMEM ; + } sbi->devfd = fd;erofs_info ("successfully to open %s" , dev);
@@ -136,8 +142,12 @@ int dev_open_ro(struct erofs_sb_info *sbi, const char *dev)return -errno; } + sbi->devname =strdup (dev); +if (!sbi->devname) { +close (fd); +return -ENOMEM ; + } sbi->devfd = fd; - sbi->devname = dev; sbi->devsz =INT64_MAX ;return 0 ; }
Signature · 1 line
2.19.1.6.gb485710b
PATCH 6/13 erofs-utils: lib: make erofs_get_unhashed_chunk() global
In reply to Jingbo Xu... so that it could be called from outside blobchunk.c later.
--- include/erofs/blobchunk.h | 2 ++ lib/blobchunk.c | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-)
include/erofs/blobchunk.h +2 -0
@@ -14,6 +14,8 @@ extern "C"#include "erofs/internal.h" +struct erofs_blobchunk *erofs_get_unhashed_chunk (unsigned int device_id, +erofs_blk_t blkaddr,erofs_off_t sourceoffset);int erofs_blob_write_chunk_indexes (struct erofs_inode *inode,erofs_off_t off);int erofs_blob_write_chunked_file (struct erofs_inode *inode,int fd);int tarerofs_write_chunkes (struct erofs_inode *inode,erofs_off_t data_offset);
lib/blobchunk.c +1 -1
@@ -38,7 +38,7 @@ struct erofs_blobchunk erofs_holechunk = { };static LIST_HEAD (unhashed_blobchunks); -static struct erofs_blobchunk *erofs_get_unhashed_chunk (unsigned int device_id, +struct erofs_blobchunk *erofs_get_unhashed_chunk (unsigned int device_id,erofs_blk_t blkaddr,erofs_off_t sourceoffset) {struct erofs_blobchunk *chunk;
Signature · 1 line
2.19.1.6.gb485710b
PATCH 7/13 erofs-utils: lib: add erofs_read_xattrs_from_disk() helper
In reply to Jingbo XuAdd erofs_read_xattrs_from_disk() helper to read extended attributes from disk.
--- include/erofs/xattr.h | 1 + lib/xattr.c | 76 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+)
include/erofs/xattr.h +1 -0
@@ -85,6 +85,7 @@ int erofs_xattr_write_name_prefixes(struct erofs_sb_info *sbi, FILE *f);int erofs_setxattr (struct erofs_inode *inode,char *key,const void *value,size_t size); +int erofs_read_xattrs_from_disk (struct erofs_inode *inode); #ifdef __cplusplus }
lib/xattr.c +76 -0
82 lines not on this page · raw message
Signature · 1 line
2.19.1.6.gb485710b
PATCH 8/13 erofs-utils: lib: add erofs_inode_tag_opaque() helper
In reply to Jingbo XuAdd erofs_inode_tag_opaque() helper checking if it's an opaque directory.
--- include/erofs/internal.h | 2 ++ include/erofs/xattr.h | 22 ++++++++++++++++++++ lib/tar.c | 5 ----- lib/xattr.c | 44 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 5 deletions(-)
include/erofs/internal.h +2 -0
@@ -219,6 +219,8 @@ struct erofs_inode {#endif erofs_off_t fragmentoff;unsigned int fragment_size; + +bool opaque; };static inline erofs_off_t erofs_iloc (struct erofs_inode *inode)
include/erofs/xattr.h +22 -0
@@ -73,6 +73,27 @@ static inline unsigned int xattrblock_offset(struct erofs_inode *vi,#ifndef XATTR_NAME_POSIX_ACL_DEFAULT #define XATTR_NAME_POSIX_ACL_DEFAULT "system.posix_acl_default"#endif +#ifndef OVL_XATTR_NAMESPACE +#define OVL_XATTR_NAMESPACE "overlay." +#endif +#ifndef OVL_XATTR_OPAQUE_POSTFIX +#define OVL_XATTR_OPAQUE_POSTFIX "opaque" +#endif +#ifndef OVL_XATTR_TRUSTED_PREFIX +#define OVL_XATTR_TRUSTED_PREFIX XATTR_TRUSTED_PREFIX OVL_XATTR_NAMESPACE +#endif +#ifndef OVL_XATTR_OPAQUE_SUFFIX +#define OVL_XATTR_OPAQUE_SUFFIX OVL_XATTR_NAMESPACE OVL_XATTR_OPAQUE_POSTFIX +#endif +#ifndef OVL_XATTR_OPAQUE_SUFFIX_LEN +#define OVL_XATTR_OPAQUE_SUFFIX_LEN (sizeof(OVL_XATTR_OPAQUE_SUFFIX) - 1) +#endif +#ifndef OVL_XATTR_OPAQUE +#define OVL_XATTR_OPAQUE OVL_XATTR_TRUSTED_PREFIX OVL_XATTR_OPAQUE_POSTFIX +#endif +#ifndef OVL_XATTR_OPAQUE_LEN +#define OVL_XATTR_OPAQUE_LEN (sizeof(OVL_XATTR_OPAQUE) - 1) +#endif int erofs_scan_file_xattrs (struct erofs_inode *inode);int erofs_prepare_xattr_ibody (struct erofs_inode *inode);
@@ -86,6 +107,7 @@ int erofs_xattr_write_name_prefixes(struct erofs_sb_info *sbi, FILE *f);int erofs_setxattr (struct erofs_inode *inode,char *key,const void *value,size_t size);int erofs_read_xattrs_from_disk (struct erofs_inode *inode); +void erofs_inode_tag_opaque (struct erofs_inode *inode); #ifdef __cplusplus }
lib/tar.c +0 -5
@@ -19,11 +19,6 @@#include "erofs/xattr.h" #include "erofs/blobchunk.h" -#define OVL_XATTR_NAMESPACE "overlay." -#define OVL_XATTR_TRUSTED_PREFIX XATTR_TRUSTED_PREFIX OVL_XATTR_NAMESPACE -#define OVL_XATTR_OPAQUE_POSTFIX "opaque" -#define OVL_XATTR_OPAQUE OVL_XATTR_TRUSTED_PREFIX OVL_XATTR_OPAQUE_POSTFIX -#define EROFS_WHITEOUT_DEV 0static char erofs_libbuf[16384 ];
lib/xattr.c +44 -0
50 lines not on this page · raw message
Signature · 1 line
2.19.1.6.gb485710b
PATCH 9/13 erofs-utils: lib: add erofs_insert_ihash() helper
In reply to Jingbo XuAdd erofs_insert_ihash() helper inserting inode into inode hash table.
Also add prototypes of erofs_iget() and erofs_iget_by_nid() in the header file.
--- include/erofs/inode.h | 3 +++ lib/inode.c | 10 +++++++--- 2 files changed, 10 insertions(+), 3 deletions(-)
include/erofs/inode.h +3 -0
@@ -25,6 +25,9 @@ u32 erofs_new_encode_dev(dev_t dev);unsigned char erofs_mode_to_ftype (umode_t mode);unsigned char erofs_ftype_to_dtype (unsigned int filetype);void erofs_inode_manager_init (void ); +void erofs_insert_ihash (struct erofs_inode *inode,dev_t dev,ino_t ino); +struct erofs_inode *erofs_iget (dev_t dev,ino_t ino); +struct erofs_inode *erofs_iget_by_nid (erofs_nid_t nid);unsigned int erofs_iput (struct erofs_inode *inode);erofs_nid_t erofs_lookupnid (struct erofs_inode *inode); struct erofs_dentry *erofs_d_alloc(struct erofs_inode *parent,
lib/inode.c +7 -3
@@ -75,6 +75,12 @@ void erofs_inode_manager_init(void)init_list_head (&inode_hashtable[i]); } +void erofs_insert_ihash (struct erofs_inode *inode,dev_t dev,ino_t ino) +{ +list_add (&inode->i_hash, + &inode_hashtable[(ino ^ dev) %NR_INODE_HASHTABLE ]); +} +/* get the inode from the (source) inode # */ struct erofs_inode *erofs_iget (dev_t dev,ino_t ino) {
@@ -976,9 +982,7 @@ static int erofs_fill_inode(struct erofs_inode *inode, struct stat *st, inode->inode_isize =sizeof (struct erofs_inode_compact ); } -list_add (&inode->i_hash, - &inode_hashtable[(st->st_ino ^ st->st_dev) % -NR_INODE_HASHTABLE ]); +erofs_insert_ihash (inode, st->st_dev, st->st_ino);return 0 ; }
Signature · 1 line
2.19.1.6.gb485710b
PATCH 10/13 erofs-utils: lib: add erofs_rebuild_dump_tree() helper
In reply to Jingbo XuEnhance tarerofs_dump_tree() so that it could optionally skip whiteout files when it's supposed to output image in merged view. Rename it to erofs_rebuild_dump_tree().
--- include/erofs/inode.h | 2 +- include/erofs/internal.h | 2 ++ lib/inode.c | 23 ++++++++++++++++++++--- lib/tar.c | 2 -- mkfs/main.c | 2 +- 5 files changed, 24 insertions(+), 7 deletions(-)
include/erofs/inode.h +1 -1
@@ -32,7 +32,7 @@ unsigned int erofs_iput(struct erofs_inode *inode);erofs_nid_t erofs_lookupnid (struct erofs_inode *inode);struct erofs_dentry *erofs_d_alloc (struct erofs_inode *parent,const char *name); -int tarerofs_dump_tree (struct erofs_inode *dir); +int erofs_rebuild_dump_tree (struct erofs_inode *dir,bool merged_view);int erofs_init_empty_dir (struct erofs_inode *dir);struct erofs_inode *erofs_new_inode (void );struct erofs_inode *erofs_mkfs_build_tree_from_path (const char *path);
include/erofs/internal.h +2 -0
@@ -416,6 +416,8 @@ static inline u32 erofs_crc32c(u32 crc, const u8 *in, size_t len)return crc; } +#define EROFS_WHITEOUT_DEV 0 + #ifdef __cplusplus }#endif
lib/inode.c +20 -3
@@ -1323,9 +1323,14 @@ struct erofs_inode *erofs_mkfs_build_special_from_fd(int fd, const char *name)return inode; } -int tarerofs_dump_tree (struct erofs_inode *dir) +static bool erofs_inode_is_whiteout (struct erofs_inode *inode) { -struct erofs_dentry *d; +return S_ISCHR (inode->i_mode) && inode->u.i_rdev ==EROFS_WHITEOUT_DEV ; +} + +int erofs_rebuild_dump_tree (struct erofs_inode *dir,bool merged_view) +{ +struct erofs_dentry *d, *n;unsigned int nr_subdirs;int ret;
@@ -1368,6 +1373,18 @@ int tarerofs_dump_tree(struct erofs_inode *dir)return 0 ; } +if (merged_view) { +list_for_each_entry_safe (d, n, &dir->i_subdirs, d_child) { +if (erofs_inode_is_whiteout (d->inode)) { +erofs_dbg ("skip whiteout %s" , d->inode->i_srcpath); +erofs_d_invalidate (d); +list_del (&d->d_child); +free (d); +continue ; + } + } + } + nr_subdirs =0 ;list_for_each_entry (d, &dir->i_subdirs, d_child) ++nr_subdirs;
@@ -1391,7 +1408,7 @@ int tarerofs_dump_tree(struct erofs_inode *dir)continue ; inode =erofs_igrab (d->inode); - ret =tarerofs_dump_tree (inode); + ret =erofs_rebuild_dump_tree (inode, merged_view); dir->i_nlink += (erofs_mode_to_ftype (inode->i_mode) ==EROFS_FT_DIR );erofs_iput (inode);if (ret)
lib/tar.c +0 -2
@@ -19,8 +19,6 @@#include "erofs/xattr.h" #include "erofs/blobchunk.h" -#define EROFS_WHITEOUT_DEV 0 -static char erofs_libbuf[16384 ];struct tar_header {
mkfs/main.c +1 -1
@@ -946,7 +946,7 @@ int main(int argc, char **argv)if (err <0 ) goto exit; - err =tarerofs_dump_tree (root_inode); + err =erofs_rebuild_dump_tree (root_inode, false);if (err <0 ) goto exit; }
Signature · 1 line
2.19.1.6.gb485710b
PATCH 11/13 erofs-utils: lib: add erofs_rebuild_get_dentry() helper
In reply to Jingbo XuRename tarerofs_get_dentry() to erofs_rebuild_get_dentry().
Also make whout and 'opq' parameter optional when aufs is false.
--- include/erofs/rebuild.h | 19 +++++++ lib/Makefile.am | 3 +- lib/rebuild.c | 117 ++++++++++++++++++++++++++++++++++++++++ lib/tar.c | 109 ++----------------------------------- 4 files changed, 141 insertions(+), 107 deletions(-) create mode 100644 include/erofs/rebuild.h create mode 100644 lib/rebuild.c
include/erofs/rebuild.h new file +19 -0
@@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ +#ifndef __EROFS_REBUILD_H +#define __EROFS_REBUILD_H + +#ifdef __cplusplus +extern "C" +{ +#endif + +#include "internal.h" + +struct erofs_dentry *erofs_rebuild_get_dentry (struct erofs_inode *pwd, +char *path,bool aufs,bool *whout,bool *opq); + +#ifdef __cplusplus +} +#endif + +#endif
lib/Makefile.am +2 -1
@@ -24,6 +24,7 @@ noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \ $(top_srcdir)/include/erofs/xattr.h \ $(top_srcdir)/include/erofs/compress_hints.h \ $(top_srcdir)/include/erofs/fragments.h \ + $(top_srcdir)/include/erofs/rebuild.h \ $(top_srcdir)/lib/liberofs_private.h noinst_HEADERS += compressor.h
@@ -31,7 +32,7 @@ liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \ namei.c data.c compress.c compressor.c zmap.c decompress.c \ compress_hints.c hashmap.c sha256.c blobchunk.c dir.c \ fragments.c rb_tree.c dedupe.c uuid_unparse.c uuid.c tar.c \ - block_list.c + block_list.c rebuild.c liberofs_la_CFLAGS = -Wall ${libuuid_CFLAGS} -I$(top_srcdir)/include if ENABLE_LZ4
lib/rebuild.c new file +117 -0
117 lines not on this page · raw message
lib/tar.c +3 -106
139 lines not on this page · raw message
Signature · 1 line
2.19.1.6.gb485710b
PATCH 12/13 erofs-utils: lib: add erofs_rebuild_load_tree() helper
In reply to Jingbo XuAdd erofs_rebuild_load_tree() helper loading inode tree from given erofs image, and making it merged into a given inode tree in an overlayfs like model.
Since we need to read the content of the symlink file from disk when loading tree, add dependency on zlib_LIBS for mkfs.erofs.
--- include/erofs/internal.h | 1 + include/erofs/rebuild.h | 2 + lib/rebuild.c | 270 +++++++++++++++++++++++++++++++++++++++ mkfs/Makefile.am | 3 +- 4 files changed, 275 insertions(+), 1 deletion(-)
include/erofs/internal.h +1 -0
@@ -102,6 +102,7 @@ struct erofs_sb_info {int devfd;u64 devsz; +dev_t dev;unsigned int nblobs;unsigned int blobfd[256 ]; };
include/erofs/rebuild.h +2 -0
@@ -12,6 +12,8 @@ extern "C"struct erofs_dentry *erofs_rebuild_get_dentry (struct erofs_inode *pwd,char *path,bool aufs,bool *whout,bool *opq); +int erofs_rebuild_load_tree (struct erofs_inode *root,struct erofs_sb_info *sbi); + #ifdef __cplusplus }#endif
lib/rebuild.c +270 -0
282 lines not on this page · raw message
mkfs/Makefile.am +2 -1
@@ -6,4 +6,5 @@ AM_CPPFLAGS = ${libselinux_CFLAGS} mkfs_erofs_SOURCES = main.c mkfs_erofs_CFLAGS = -Wall -I$(top_srcdir)/include mkfs_erofs_LDADD = $(top_builddir)/lib/liberofs.la ${libselinux_LIBS} \ - ${libuuid_LIBS} ${liblz4_LIBS} ${liblzma_LIBS} ${libdeflate_LIBS} + ${libuuid_LIBS} ${liblz4_LIBS} ${liblzma_LIBS} ${zlib_LIBS} \ + ${libdeflate_LIBS}
Signature · 1 line
2.19.1.6.gb485710b
PATCH 13/13 erofs-utils: mkfs: introduce rebuild mode
In reply to Jingbo XuIntroduce a new rebuild mode merging multiple erofs images generated from either tarerfs index mode (--tar=i):
mkfs.erofs --tar=i --aufs layer0.erofs layer0.tar ... mkfs.erofs --tar=i --aufs layerN.erofs layerN.tar mkfs.erofs merge.erofs layerN.erofs ... layer0.erofs
or tarerofs non-index mode:
mkfs.erofs --tar=f -Enoinline_data --aufs layer0.erofs layer0.tar ... mkfs.erofs --tar=f -Enoinline_data --aufs layerN.erofs layerN.tar mkfs.erofs merge.erofs layerN.erofs ... layer0.erofs
When calling rebuild mode, the top most layer must be specified firstly in the cmdline, while the bottom most layer i.e. layer0 the lastly.
The rebuild mode doesn't support flat inline datalayout yet.
--- include/erofs/internal.h | 2 + mkfs/main.c | 196 +++++++++++++++++++++++++++++++-------- 2 files changed, 157 insertions(+), 41 deletions(-)
include/erofs/internal.h +2 -0
@@ -105,6 +105,8 @@ struct erofs_sb_info {dev_t dev;unsigned int nblobs;unsigned int blobfd[256 ]; + +struct list_head list; };/* make sure that any user of the erofs headers has atleast 64bit off_t type */
mkfs/main.c +155 -41
275 lines not on this page · raw message
Signature · 1 line
2.19.1.6.gb485710b
Re: [PATCH 01/13] erofs-utils: fix overriding of i_rdev for special device
In reply to Jingbo XuQuoted 11 lines · Jingbo Xu
In index mode for tarerofs,
tarerofs_write_chunk_indexes()is still called even for files with 0i_size, andinode->u.chunkformatis initialized accordingly. This will make the previously setinode->u.i_rdevbe overridden as u.chunkformat and u.i_rdev are reused in one union.To tidy up the code of writing file, extract two helpers writing file in index and non-index mode. Also fix the missing
erofs_iput()whentarerofs_write_chunkes()fails.Fixes 95d315fd7958 ("erofs-utils: introduce tarerofs")Signed-off-by Jingbo Xu <[email protected]>
Thanks,
Gao Xiang
Re: [PATCH 02/13] erofs-utils: lib: add list_splice_tail() helper
In reply to Jingbo XuQuoted 3 lines · Jingbo Xu
Add
list_splice_tail()helper.Signed-off-by Jingbo Xu <[email protected]>
Thanks,
Gao Xiang
Re: [PATCH 03/13] erofs-utils: lib: read i_ino in erofs_read_inode_from_disk()
In reply to Jingbo XuQuoted 8 lines · Jingbo Xu
i_ino[0] is a unique inode serial number in the erofs filesystem where the inode resides, and also serves as the on-disk inode number, while i_ino[1] is a unique number identifying the source inode in the source directory, which is usually derived from
st->st_ino.Read on-disk ino and store it in i_ino[0] given the above background.
Signed-off-by Jingbo Xu <[email protected]>
Thanks,
Gao Xiang
Re: [PATCH 04/13] erofs-utils: lib: fix erofs_init_devices() in multidev mode
In reply to Jingbo XuQuoted 6 lines · Jingbo Xu
Fix the missing
le32_to_cpu(). Read and cache blocks of each device for later use.Initialize
sbi->extra_devicesfrom on-diskextra_devices.Signed-off-by Jingbo Xu <[email protected]>
Thanks,
Gao Xiang
Re: [PATCH 05/13] erofs-utils: lib: keep self maintained devname
In reply to Jingbo XuQuoted 42 lines · Jingbo Xu
Keep self allocated and maintained devname in
erofs_sb_info.Signed-off-by Jingbo Xu <[email protected]>--- include/erofs/internal.h | 2 +- lib/io.c | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-)include/erofs/internal.h +1
-1@@ -61,7 +61,7 @@ struct erofs_device_info {struct erofs_sb_info {struct erofs_device_info *devs; -const char *devname; +char *devname;u64 total_blocks;u64 primarydevice_blocks;lib/io.c +2
-0@@ -10,6 +10,7 @@#ifndef _GNU_SOURCE#define _GNU_SOURCE#endif +#include <stdlib.h> #include <sys/stat.h> #include <sys/ioctl.h> #include "erofs/io.h" @@ -46,6 +47,7 @@ static int dev_get_blkdev_size(int fd, u64 *bytes)void dev_close (struct erofs_sb_info *sbi) {close (sbi->devfd); +free (sbi->devname); sbi->devname =NULL ; sbi->devfd =-1 ; sbi->devsz =0 ;
lib/io.c int dev_open(struct erofs_sb_info *sbi, const char *dev) · line 100
Quoted 5 lines
lib/io.c +1
-1@@ -95,7 +97,11 @@ int dev_open(struct erofs_sb_info *sbi, const char *dev)return -EINVAL ; } - sbi->devname = dev; + sbi->devname =strdup (dev);
Could we move sbi->devname assignment to the beginning of the function? e.g.
..
sbi->devname = strdup(dev);
if (!sbi->devname)
return -ENOMEM;
fd = open(dev, O_RDWR | O_CREAT | O_BINARY, 0644);
...Quoted 8 lines
lib/io.c +4
-0+if (!sbi->devname) { +close (fd); +return -ENOMEM ; + } sbi->devfd = fd;erofs_info ("successfully to open %s" , dev);
lib/io.c int dev_open_ro(struct erofs_sb_info *sbi, const char *dev) · line 149
Quoted 8 lines
lib/io.c +5
-0@@ -136,8 +142,12 @@ int dev_open_ro(struct erofs_sb_info *sbi, const char *dev)return -errno; } + sbi->devname =strdup (dev); +if (!sbi->devname) { +close (fd); +return -ENOMEM ; + }
Same here.
Thanks,
Gao Xiang
Re: [PATCH 07/13] erofs-utils: lib: add erofs_read_xattrs_from_disk() helper
In reply to Jingbo XuQuoted 53 lines · Jingbo Xu
Add
erofs_read_xattrs_from_disk()helper to read extended attributes from disk.Signed-off-by Jingbo Xu <[email protected]>--- include/erofs/xattr.h | 1 + lib/xattr.c | 76 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+)include/erofs/xattr.h +1
-0@@ -85,6 +85,7 @@ int erofs_xattr_write_name_prefixes(struct erofs_sb_info *sbi, FILE *f);int erofs_setxattr (struct erofs_inode *inode,char *key,const void *value,size_t size); +int erofs_read_xattrs_from_disk (struct erofs_inode *inode); #ifdef __cplusplus }lib/xattr.c +24
-0@@ -493,6 +493,82 @@ int erofs_scan_file_xattrs(struct erofs_inode *inode)return erofs_droid_xattr_set_caps (inode); } +static struct xattr_item *erofs_read_xattr_from_disk (struct erofs_inode *inode, +char *key) +{ +ssize_t ret; +u8 prefix; +u16 prefixlen; +unsigned int len[2 ]; +char *kvbuf; + +if (!match_prefix (key, &prefix, &prefixlen)) +return ERR_PTR (-ENODATA ); + + ret =erofs_getxattr (inode, key,NULL ,0 ); +if (ret <0 ) +return ERR_PTR (-errno); + +/* allocate key-value buffer */ + len[0 ] =strlen (key) - prefixlen; + len[1 ] = ret; + kvbuf =malloc (len[0 ] + len[1 ]); +if (!kvbuf) +return ERR_PTR (-ENOMEM ); +memcpy (kvbuf, key + prefixlen, len[0 ]); +if (len[1 ]) {
lib/xattr.c int erofs_scan_file_xattrs(struct erofs_inode *inode) · line 529
Quoted 10 lines
lib/xattr.c +10
-0+ ret =erofs_getxattr (inode, key, kvbuf + len[0 ], len[1 ]); +if (ret <0 ) { +free (kvbuf); +return ERR_PTR (-errno); + } +if (ret != len[1 ]) { +erofs_err ("size of xattr value got changed just now (%u-> %ld)" , + len[1 ], (long )ret); + len[1 ] = ret; + }
Can this actually happen? Maybe just:
DBG_BUGON(ret != len[1]);
Thanks,
Gao Xiang
Gao Xiang
Reviewed-byRe: [PATCH 05/13] erofs-utils: lib: keep self maintained devname
In reply to Gao XiangQuoted 7 lines · Gao Xiang
Quoted 4 lines · Jingbo Xu
Keep self allocated and maintained devname in
erofs_sb_info.Signed-off-by Jingbo Xu <[email protected]>---
...
Quoted 9 lines
lib/io.c int dev_open(struct erofs_sb_info *sbi, const char *dev) · line 99
Quoted 5 lines
lib/io.c +1
-1@@ -95,7 +97,11 @@ int dev_open(struct erofs_sb_info *sbi, const char *dev) return -EINVAL; } - sbi->devname = dev; + sbi->devname = strdup(dev);Could we move
sbi->devnameassignment to the beginning of the function? e.g.
Quoted 13 lines
..
sbi->devname= strdup(dev);
if (!sbi->devname)
return-ENOMEM;fd =
open(dev, O_RDWR | O_CREAT | O_BINARY, 0644);
...Quoted 4 lines
lib/io.c +4
-0+ if (!sbi->devname) { + close(fd); + return -ENOMEM; + }
After a second thought, since there are already several close(fd); in the error paths and they are unavoidable. So it looks good to me now:
Thanks,
Gao Xiang
Re: [PATCH 07/13] erofs-utils: lib: add erofs_read_xattrs_from_disk() helper
In reply to Gao XiangQuoted 62 lines · Gao Xiang
Quoted 59 lines · Jingbo Xu
Add
erofs_read_xattrs_from_disk()helper to read extended attributes from disk.Signed-off-by Jingbo Xu <[email protected]>---
include/erofs/xattr.h| 1 +
lib/xattr.c | 76 +++++++++++++++++++++++++++++++++++++++++++2 files changed, 77 insertions(+)include/erofs/xattr.h +1
-0@@ -85,6 +85,7 @@ int erofs_xattr_write_name_prefixes(struct erofs_sb_info *sbi, FILE *f); int erofs_setxattr(struct erofs_inode *inode, char *key, const void *value, size_t size); +int erofs_read_xattrs_from_disk (struct erofs_inode *inode); #ifdef __cplusplus }lib/xattr.c +0
-0@@ -493,6 +493,82 @@ int erofs_scan_file_xattrs(struct erofs_inode *inode) return erofs_droid_xattr_set_caps(inode); }+static struct
xattr_item*erofs_read_xattr_from_disk(structerofs_inode*inode,
+ char *key)lib/xattr.c +28
-0+{ + ssize_t ret; + u8 prefix; + u16 prefixlen; + unsigned int len[2]; + char *kvbuf; + + if (!match_prefix(key, &prefix, &prefixlen)) + return ERR_PTR(-ENODATA); + + ret = erofs_getxattr(inode, key, NULL, 0); + if (ret < 0) + return ERR_PTR(-errno); + + /* allocate key-value buffer */ + len[0] = strlen(key) - prefixlen; + len[1] = ret; + kvbuf = malloc(len[0] + len[1]); + if (!kvbuf) + return ERR_PTR(-ENOMEM); + memcpy(kvbuf, key + prefixlen, len[0]); + if (len[1]) { + ret = erofs_getxattr(inode, key, kvbuf + len[0], len[1]); + if (ret < 0) { + free(kvbuf); + return ERR_PTR(-errno); + } + if (ret != len[1]) {
Quoted 8 lines
Quoted 5 lines
+ erofs_err("size of xattr value got changed just now (%u-> %ld)",
+ len[1], (long)ret);
+ len[1] = ret;
+ }Can this actually happen? Maybe just:
DBG_BUGON(ret != len[1]);
Make sense.
Signature · 2 lines
Thanks, Jingbo
Re: [PATCH 08/13] erofs-utils: lib: add erofs_inode_tag_opaque() helper
In reply to Jingbo XuQuoted 15 lines · Jingbo Xu
Add
erofs_inode_tag_opaque()helper checking if it's an opaque directory.Signed-off-by Jingbo Xu <[email protected]>--- include/erofs/internal.h | 2 ++ include/erofs/xattr.h | 22 ++++++++++++++++++++ lib/tar.c | 5 ----- lib/xattr.c | 44 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 5 deletions(-)include/erofs/internal.h +0
-0
include/erofs/internal.h struct erofs_inode { · line 223
Quoted 5 lines
include/erofs/internal.h +2
-0@@ -219,6 +219,8 @@ struct erofs_inode {#endif erofs_off_t fragmentoff;unsigned int fragment_size; + +bool opaque;
could we move it upwards? for example, just under
bool with_tmpfile;
Quoted 22 lines
include/erofs/internal.h +0
-0};static inline erofs_off_t erofs_iloc (struct erofs_inode *inode)include/erofs/xattr.h +11
-0@@ -73,6 +73,27 @@ static inline unsigned int xattrblock_offset(struct erofs_inode *vi,#ifndef XATTR_NAME_POSIX_ACL_DEFAULT #define XATTR_NAME_POSIX_ACL_DEFAULT "system.posix_acl_default"#endif +#ifndef OVL_XATTR_NAMESPACE +#define OVL_XATTR_NAMESPACE "overlay." +#endif +#ifndef OVL_XATTR_OPAQUE_POSTFIX +#define OVL_XATTR_OPAQUE_POSTFIX "opaque" +#endif +#ifndef OVL_XATTR_TRUSTED_PREFIX +#define OVL_XATTR_TRUSTED_PREFIX XATTR_TRUSTED_PREFIX OVL_XATTR_NAMESPACE +#endif +#ifndef OVL_XATTR_OPAQUE_SUFFIX +#define OVL_XATTR_OPAQUE_SUFFIX OVL_XATTR_NAMESPACE OVL_XATTR_OPAQUE_POSTFIX
include/erofs/xattr.h static inline unsigned int xattrblock_offset(struct erofs_inode *vi, · line 96
Quoted 10 lines
include/erofs/xattr.h +10
-0+#endif +#ifndef OVL_XATTR_OPAQUE_SUFFIX_LEN +#define OVL_XATTR_OPAQUE_SUFFIX_LEN (sizeof(OVL_XATTR_OPAQUE_SUFFIX) - 1) +#endif +#ifndef OVL_XATTR_OPAQUE +#define OVL_XATTR_OPAQUE OVL_XATTR_TRUSTED_PREFIX OVL_XATTR_OPAQUE_POSTFIX +#endif +#ifndef OVL_XATTR_OPAQUE_LEN +#define OVL_XATTR_OPAQUE_LEN (sizeof(OVL_XATTR_OPAQUE) - 1) +#endif
I'd like to avoid export these symbols in the headers, how about moving all related logic into lib/xattr.c?
Quoted 46 lines
include/erofs/xattr.h +1
-0int erofs_scan_file_xattrs (struct erofs_inode *inode);int erofs_prepare_xattr_ibody (struct erofs_inode *inode);@@ -86,6 +107,7 @@ int erofs_xattr_write_name_prefixes(struct erofs_sb_info *sbi, FILE *f);int erofs_setxattr (struct erofs_inode *inode,char *key,const void *value,size_t size);int erofs_read_xattrs_from_disk (struct erofs_inode *inode); +void erofs_inode_tag_opaque (struct erofs_inode *inode); #ifdef __cplusplus }lib/tar.c +0
-5@@ -19,11 +19,6 @@#include "erofs/xattr.h" #include "erofs/blobchunk.h" -#define OVL_XATTR_NAMESPACE "overlay." -#define OVL_XATTR_TRUSTED_PREFIX XATTR_TRUSTED_PREFIX OVL_XATTR_NAMESPACE -#define OVL_XATTR_OPAQUE_POSTFIX "opaque" -#define OVL_XATTR_OPAQUE OVL_XATTR_TRUSTED_PREFIX OVL_XATTR_OPAQUE_POSTFIX -#define EROFS_WHITEOUT_DEV 0static char erofs_libbuf[16384 ];lib/xattr.c +11
-0@@ -1421,6 +1421,50 @@ int erofs_listxattr(struct erofs_inode *vi, char *buffer, size_t buffer_size)return shared_listxattr (vi, &it); } +static bool erofs_xattr_is_opaque (struct xattr_item *item) +{ +struct ea_type_node *tnode; +unsigned int plen; +const char *prefix; + +if (item->prefix ==EROFS_XATTR_INDEX_TRUSTED && + !strncmp (item->kvbuf,OVL_XATTR_OPAQUE_SUFFIX , +OVL_XATTR_OPAQUE_SUFFIX_LEN )) +return true; +
lib/xattr.c int erofs_listxattr(struct erofs_inode *vi, char *buffer, size_t buffer_size) · line 1444
Quoted 10 lines
lib/xattr.c +10
-0+if (item->prefix &EROFS_XATTR_LONG_PREFIX ) { +list_for_each_entry (tnode, &ea_name_prefixes, list) { + prefix = tnode->type.prefix; + plen = tnode->type.prefix_len; +if (tnode->index == item->prefix && + plen + item->len[0 ] ==OVL_XATTR_OPAQUE_LEN && + !strncmp (prefix,OVL_XATTR_OPAQUE , plen) && + !strncmp (item->kvbuf,OVL_XATTR_OPAQUE + plen, + item->len[0 ])) +return true;
These looks too complicated to me, how about just connect prefix and suffix strings and do a string comparsion together?
Thanks,
Gao Xiang