Skip to contentSkip to conversation

erofs-utils: introduce tarerofs

PATCH Superseded Gao Xiang 1 participant 1 message Mar 28, 2023 Library code

v2 of 6 Superseded Latest is v6 →

lore
Share & tools
lore

Gao Xiang

author
#0 lore raw

PATCH v2 1/1 erofs-utils: introduce tarerofs

Let's try to add a new mode "tarerofs" for mkfs.erofs.

It mainly aims at two use cases:
- Convert a tarball (or later tarballs with a merged view) into a full EROFS image [--tar=full];

- Generate an EROFS manifest image to reuse tar data [--tar=index], which also enables EROFS 512-byte blocks.

The second use case is mainly prepared for OCI direct mount without OCI blob unpacking.

[ Note that --tar=full generates lots of temporary files for now
which can impact performance since the original tar stream(s) may be non-seekable. ]

---
RFC: https://lore.kernel.org/r/[email protected]/

changes since RFC:
- fix hardlinks, symlinks;
- more pax support;
- various bugfixes.

 include/erofs/blobchunk.h |   2 +-
 include/erofs/inode.h     |  11 +
 include/erofs/internal.h  |   7 +-
 include/erofs/tar.h       |  29 ++
 lib/Makefile.am           |   2 +-
 lib/blobchunk.c           |  35 +-
 lib/inode.c               | 175 +++++++---
 lib/tar.c                 | 692 ++++++++++++++++++++++++++++++++++++++
 mkfs/main.c               | 122 +++++--
 9 files changed, 980 insertions(+), 95 deletions(-)
 create mode 100644 include/erofs/tar.h
 create mode 100644 lib/tar.c
include/erofs/blobchunk.h +1 -1
index 49cb7bf..d7deb33 100644
@@ -15,7 +15,7 @@ extern "C"
 #include "erofs/internal.h"
 
 int erofs_blob_write_chunk_indexes(struct erofs_inode *inode, erofs_off_t off);
-int erofs_blob_write_chunked_file(struct erofs_inode *inode);
+int erofs_blob_write_chunked_file(struct erofs_inode *inode, int fd);
 int erofs_blob_remap(void);
 void erofs_blob_exit(void);
 int erofs_blob_init(const char *blobfile_path);
include/erofs/inode.h +11 -0
index 058a235..1c3e3cb 100644
@@ -15,11 +15,22 @@ extern "C"
 
 #include "erofs/internal.h"
 
+static inline struct erofs_inode *erofs_igrab(struct erofs_inode *inode)
+{
+	++inode->i_count;
+	return inode;
+}
+
 unsigned char erofs_mode_to_ftype(umode_t mode);
 unsigned char erofs_ftype_to_dtype(unsigned int filetype);
 void erofs_inode_manager_init(void);
 unsigned int erofs_iput(struct erofs_inode *inode);
 erofs_nid_t erofs_lookupnid(struct erofs_inode *inode);
+struct erofs_dentry *erofs_d_alloc(struct erofs_inode *parent,
+				   const char *name);
+int tarerofs_dump_tree(struct erofs_inode *dir);
+int erofs_init_empty_dir(struct erofs_inode *dir);
+struct erofs_inode *erofs_new_inode(void);
 struct erofs_inode *erofs_mkfs_build_tree_from_path(const char *path);
 struct erofs_inode *erofs_mkfs_build_special_from_fd(int fd, const char *name);
 
include/erofs/internal.h +6 -1
index 641a795..e54922d 100644
@@ -22,6 +22,7 @@ typedef unsigned short umode_t;
 #include "erofs_fs.h"
 #include <fcntl.h>
 #include <sys/types.h> /* for off_t definition */
+#include <stdio.h>
 
 #ifndef PATH_MAX
 #define PATH_MAX        4096    /* # chars in a path name including nul */
@@ -174,13 +175,17 @@ struct erofs_inode {
 	} u;
 
 	char *i_srcpath;
-
+	union {
+		char *i_link;
+		FILE *i_tmpfile;
+	};
 	unsigned char datalayout;
 	unsigned char inode_isize;
 	/* inline tail-end packing size */
 	unsigned short idata_size;
 	bool compressed_idata;
 	bool lazy_tailblock;
+	bool with_tmpfile;
 
 	unsigned int xattr_isize;
 	unsigned int extent_isize;
include/erofs/tar.h new file +29 -0
new file mode 100644
index 0000000..c837801
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */
+#ifndef __EROFS_TAR_H
+#define __EROFS_TAR_H
+
+#include <sys/stat.h>
+
+struct erofs_pax_header {
+	struct stat st;
+	bool use_mtime;
+	bool use_size;
+	bool use_uid;
+	bool use_gid;
+	char *path, *link;
+};
+
+struct erofs_tarfile {
+	struct erofs_pax_header global;
+
+	int fd;
+	u64 offset;
+	bool index_mode, aufs;
+};
+
+int tarerofs_init_empty_dir(struct erofs_inode *inode);
+int tarerofs_parse_tar(struct erofs_inode *root, struct erofs_tarfile *tar);
+int tarerofs_reverse_devtable(unsigned int devices);
+int tarerofs_write_devtable(struct erofs_tarfile *tar);
+
+#endif
lib/Makefile.am +1 -1
index faa7311..a377370 100644
@@ -29,7 +29,7 @@ noinst_HEADERS += compressor.h
 liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \
 		      namei.c data.c compress.c compressor.c zmap.c decompress.c \
 		      compress_hints.c hashmap.c sha256.c blobchunk.c dir.c \
-		      fragments.c rb_tree.c dedupe.c
+		      fragments.c rb_tree.c dedupe.c tar.c
 
 liberofs_la_CFLAGS = -Wall -I$(top_srcdir)/include
 if ENABLE_LZ4
lib/blobchunk.c +20 -15
index 8142cc3..9665d63 100644
@@ -128,12 +128,20 @@ int erofs_blob_write_chunk_indexes(struct erofs_inode *inode,
 	     src += sizeof(void *), dst += unit) {
 		struct erofs_blobchunk *chunk;
 
-		chunk = *(void **)(inode->chunkindexes + src);
+		if (blobfile) {
+			chunk = *(void **)(inode->chunkindexes + src);
 
-		if (chunk->blkaddr != EROFS_NULL_ADDR)
-			idx.blkaddr = base_blkaddr + chunk->blkaddr;
-		else
-			idx.blkaddr = EROFS_NULL_ADDR;
+			if (chunk->blkaddr != EROFS_NULL_ADDR)
+				idx.blkaddr = base_blkaddr + chunk->blkaddr;
+			else
+				idx.blkaddr = EROFS_NULL_ADDR;
+		} else if (unit == EROFS_BLOCK_MAP_ENTRY_SIZE) {
+			idx.blkaddr = le32_to_cpu(*(__le32 *)(inode->chunkindexes + dst));
+		} else {
+			struct erofs_inode_chunk_index *i2 = inode->chunkindexes + dst;
+
+			idx.blkaddr = le32_to_cpu(i2->blkaddr);
+		}
 
 		if (extent_start != EROFS_NULL_ADDR &&
 		    idx.blkaddr == extent_end + 1) {
@@ -149,6 +157,10 @@ int erofs_blob_write_chunk_indexes(struct erofs_inode *inode,
 			extent_start = idx.blkaddr;
 			extent_end = idx.blkaddr;
 		}
+
+		if (!blobfile)
+			continue;
+
 		if (unit == EROFS_BLOCK_MAP_ENTRY_SIZE)
 			memcpy(inode->chunkindexes + dst, &idx.blkaddr, unit);
 		else
@@ -166,17 +178,14 @@ int erofs_blob_write_chunk_indexes(struct erofs_inode *inode,
 	return dev_write(inode->chunkindexes, off, inode->extent_isize);
 }
 
-int erofs_blob_write_chunked_file(struct erofs_inode *inode)
+int erofs_blob_write_chunked_file(struct erofs_inode *inode, int fd)
 {
 	unsigned int chunkbits = cfg.c_chunkbits;
 	unsigned int count, unit;
 	struct erofs_inode_chunk_index *idx;
 	erofs_off_t pos, len, chunksize;
-	int fd, ret;
+	int ret;
 
-	fd = open(inode->i_srcpath, O_RDONLY | O_BINARY);
-	if (fd < 0)
-		return -errno;
 #ifdef SEEK_DATA
 	/* if the file is fully sparsed, use one big chunk instead */
 	if (lseek(fd, 0, SEEK_DATA) < 0 && errno == ENXIO) {
@@ -200,10 +209,8 @@ int erofs_blob_write_chunked_file(struct erofs_inode *inode)
 
 	inode->extent_isize = count * unit;
 	idx = malloc(count * max(sizeof(*idx), sizeof(void *)));
-	if (!idx) {
-		close(fd);
+	if (!idx)
 		return -ENOMEM;
-	}
 	inode->chunkindexes = idx;
 
 	for (pos = 0; pos < inode->i_size; pos += len) {
@@ -240,10 +247,8 @@ int erofs_blob_write_chunked_file(struct erofs_inode *inode)
 		*(void **)idx++ = chunk;
 	}
 	inode->datalayout = EROFS_INODE_CHUNK_BASED;
-	close(fd);
 	return 0;
 err:
-	close(fd);
 	free(inode->chunkindexes);
 	inode->chunkindexes = NULL;
 	return ret;
lib/inode.c +132 -43

257 lines not on this page · raw message

lib/tar.c new file +692 -0

692 lines not on this page · raw message

mkfs/main.c +88 -34

192 lines not on this page · raw message

Signature · 1 line
2.24.4