erofs-utils: introduce tarerofs
PATCH v4 1/1 erofs-utils: introduce tarerofs
Let's try to add a new mode "tarerofs" for mkfs.erofs.
It mainly aims at two use cases:
- Convert a tarball (or later tarballs with a merged view) into a full EROFS image [--tar=f];
- Generate an EROFS manifest image to reuse tar data [--tar=i], which also enables EROFS 512-byte blocks.
The second use case is mainly prepared for OCI direct mount without OCI blob unpacking.
[ Note that --tar=f generates lots of temporary files for now which
can impact performance since the original tar stream(s) may be non-seekable. ]
---
changes since v3:
- fix NULL dereference due to '.' tar header in some archives.
configure.ac | 1 + include/erofs/blobchunk.h | 2 +- include/erofs/inode.h | 11 + include/erofs/internal.h | 7 +- include/erofs/tar.h | 29 ++ lib/Makefile.am | 2 +- lib/blobchunk.c | 35 +- lib/inode.c | 176 ++++++--- lib/tar.c | 734 ++++++++++++++++++++++++++++++++++++++ mkfs/main.c | 122 +++++-- 10 files changed, 1024 insertions(+), 95 deletions(-) create mode 100644 include/erofs/tar.h create mode 100644 lib/tar.c
configure.ac +1 -0
@@ -124,6 +124,7 @@ AC_CHECK_HEADERS(m4_flatten([ fcntl.h getopt.h inttypes.h + linux/aufs_type.h linux/falloc.h linux/fs.h linux/types.h
include/erofs/blobchunk.h +1 -1
@@ -15,7 +15,7 @@ extern "C"#include "erofs/internal.h" int erofs_blob_write_chunk_indexes (struct erofs_inode *inode,erofs_off_t off); -int erofs_blob_write_chunked_file (struct erofs_inode *inode); +int erofs_blob_write_chunked_file (struct erofs_inode *inode,int fd);int erofs_blob_remap (void );void erofs_blob_exit (void );int erofs_blob_init (const char *blobfile_path);
include/erofs/inode.h +11 -0
@@ -15,11 +15,22 @@ extern "C"#include "erofs/internal.h" +static inline struct erofs_inode *erofs_igrab (struct erofs_inode *inode) +{ + ++inode->i_count; +return inode; +} +unsigned char erofs_mode_to_ftype (umode_t mode);unsigned char erofs_ftype_to_dtype (unsigned int filetype);void erofs_inode_manager_init (void );unsigned int erofs_iput (struct erofs_inode *inode);erofs_nid_t erofs_lookupnid (struct erofs_inode *inode); +struct erofs_dentry *erofs_d_alloc (struct erofs_inode *parent, +const char *name); +int tarerofs_dump_tree (struct erofs_inode *dir); +int erofs_init_empty_dir (struct erofs_inode *dir); +struct erofs_inode *erofs_new_inode (void );struct erofs_inode *erofs_mkfs_build_tree_from_path (const char *path);struct erofs_inode *erofs_mkfs_build_special_from_fd (int fd,const char *name);
include/erofs/internal.h +6 -1
@@ -22,6 +22,7 @@ typedef unsigned short umode_t;#include "erofs_fs.h" #include <fcntl.h> #include <sys/types.h> /* for off_t definition */ +#include <stdio.h> #ifndef PATH_MAX #define PATH_MAX 4096/* # chars in a path name including nul */
@@ -174,13 +175,17 @@ struct erofs_inode { } u;char *i_srcpath; - +union { +char *i_link; +FILE *i_tmpfile; + };unsigned char datalayout;unsigned char inode_isize;/* inline tail-end packing size */ unsigned short idata_size;bool compressed_idata;bool lazy_tailblock; +bool with_tmpfile;unsigned int xattr_isize;unsigned int extent_isize;
include/erofs/tar.h new file +29 -0
@@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ +#ifndef __EROFS_TAR_H +#define __EROFS_TAR_H + +#include <sys/stat.h> + +struct erofs_pax_header { +struct stat st; +bool use_mtime; +bool use_size; +bool use_uid; +bool use_gid; +char *path, *link; +}; + +struct erofs_tarfile { +struct erofs_pax_header global; + +int fd; +u64 offset; +bool index_mode, aufs; +}; + +int tarerofs_init_empty_dir (struct erofs_inode *inode); +int tarerofs_parse_tar (struct erofs_inode *root,struct erofs_tarfile *tar); +int tarerofs_reverse_devtable (unsigned int devices); +int tarerofs_write_devtable (struct erofs_tarfile *tar); + +#endif
lib/Makefile.am +1 -1
@@ -29,7 +29,7 @@ noinst_HEADERS += compressor.h liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \ namei.c data.c compress.c compressor.c zmap.c decompress.c \ compress_hints.c hashmap.c sha256.c blobchunk.c dir.c \ - fragments.c rb_tree.c dedupe.c + fragments.c rb_tree.c dedupe.c tar.c liberofs_la_CFLAGS = -Wall -I$(top_srcdir)/include if ENABLE_LZ4
lib/blobchunk.c +20 -15
@@ -128,12 +128,20 @@ int erofs_blob_write_chunk_indexes(struct erofs_inode *inode, src += sizeof(void *), dst += unit) {struct erofs_blobchunk *chunk; - chunk = *(void **)(inode->chunkindexes + src); +if (blobfile) { + chunk = *(void **)(inode->chunkindexes + src); -if (chunk->blkaddr !=EROFS_NULL_ADDR ) - idx.blkaddr = base_blkaddr + chunk->blkaddr; -else - idx.blkaddr =EROFS_NULL_ADDR ; +if (chunk->blkaddr !=EROFS_NULL_ADDR ) + idx.blkaddr = base_blkaddr + chunk->blkaddr; +else + idx.blkaddr =EROFS_NULL_ADDR ; + }else if (unit ==EROFS_BLOCK_MAP_ENTRY_SIZE ) { + idx.blkaddr =le32_to_cpu (*(__le32 *)(inode->chunkindexes + dst)); + }else { +struct erofs_inode_chunk_index *i2 = inode->chunkindexes + dst; + + idx.blkaddr =le32_to_cpu (i2->blkaddr); + }if (extent_start !=EROFS_NULL_ADDR && idx.blkaddr == extent_end +1 ) {
@@ -149,6 +157,10 @@ int erofs_blob_write_chunk_indexes(struct erofs_inode *inode, extent_start = idx.blkaddr; extent_end = idx.blkaddr; } + +if (!blobfile) +continue ; +if (unit ==EROFS_BLOCK_MAP_ENTRY_SIZE )memcpy (inode->chunkindexes + dst, &idx.blkaddr, unit);else
@@ -166,17 +178,14 @@ int erofs_blob_write_chunk_indexes(struct erofs_inode *inode,return dev_write (inode->chunkindexes, off, inode->extent_isize); } -int erofs_blob_write_chunked_file (struct erofs_inode *inode) +int erofs_blob_write_chunked_file (struct erofs_inode *inode,int fd) {unsigned int chunkbits = cfg.c_chunkbits;unsigned int count, unit;struct erofs_inode_chunk_index *idx;erofs_off_t pos, len, chunksize; -int fd, ret; +int ret; - fd =open (inode->i_srcpath,O_RDONLY |O_BINARY ); -if (fd <0 ) -return -errno;#ifdef SEEK_DATA /* if the file is fully sparsed, use one big chunk instead */ if (lseek (fd,0 ,SEEK_DATA ) <0 && errno ==ENXIO ) {
@@ -200,10 +209,8 @@ int erofs_blob_write_chunked_file(struct erofs_inode *inode) inode->extent_isize = count * unit; idx =malloc (count *max (sizeof (*idx),sizeof (void *))); -if (!idx) { -close (fd); +if (!idx)return -ENOMEM ; - } inode->chunkindexes = idx;for (pos =0 ; pos < inode->i_size; pos += len) {
@@ -240,10 +247,8 @@ int erofs_blob_write_chunked_file(struct erofs_inode *inode) *(void **)idx++ = chunk; } inode->datalayout =EROFS_INODE_CHUNK_BASED ; -close (fd);return 0 ; err: -close (fd);free (inode->chunkindexes); inode->chunkindexes =NULL ;return ret;
lib/inode.c +133 -43
258 lines not on this page · raw message
lib/tar.c new file +734 -0
734 lines not on this page · raw message
mkfs/main.c +88 -34
192 lines not on this page · raw message
Signature · 1 line
2.24.4