Skip to contentSkip to conversation

erofs-utils: introduce tarerofs

RFC PATCH Superseded Gao Xiang 1 participant 1 message Mar 24, 2023 Library code

v1 of 6 Superseded Latest is v6 →

lore
Share & tools
lore

Gao Xiang

author
#0 lore raw

RFC PATCH 1/1 erofs-utils: introduce tarerofs

Let's add a new mode "tarerofs" for mkfs.erofs.

It mainly has two use cases:
- Convert a tarball (or later tarballs with a merged view) into a full erofs image;

- Generate an EROFS image as a manifest to refer to exist tarballs.

The second use case is mainly prepared for OCI direct mount without OCI blob unpacking.

---

 include/erofs/blobchunk.h |   2 +-
 include/erofs/inode.h     |  10 +
 include/erofs/internal.h  |   7 +-
 lib/Makefile.am           |   2 +-
 lib/blobchunk.c           |  35 ++-
 lib/inode.c               | 147 ++++++---
 lib/tar.c                 | 616 ++++++++++++++++++++++++++++++++++++++
 mkfs/main.c               | 121 +++++---
 8 files changed, 847 insertions(+), 93 deletions(-)
 create mode 100644 lib/tar.c
include/erofs/blobchunk.h +1 -1
index 49cb7bf..d7deb33 100644
@@ -15,7 +15,7 @@ extern "C"
 #include "erofs/internal.h"
 
 int erofs_blob_write_chunk_indexes(struct erofs_inode *inode, erofs_off_t off);
-int erofs_blob_write_chunked_file(struct erofs_inode *inode);
+int erofs_blob_write_chunked_file(struct erofs_inode *inode, int fd);
 int erofs_blob_remap(void);
 void erofs_blob_exit(void);
 int erofs_blob_init(const char *blobfile_path);
include/erofs/inode.h +10 -0
index 058a235..b9b1f9c 100644
@@ -15,11 +15,21 @@ extern "C"
 
 #include "erofs/internal.h"
 
+static inline struct erofs_inode *erofs_igrab(struct erofs_inode *inode)
+{
+	++inode->i_count;
+	return inode;
+}
+
 unsigned char erofs_mode_to_ftype(umode_t mode);
 unsigned char erofs_ftype_to_dtype(unsigned int filetype);
 void erofs_inode_manager_init(void);
 unsigned int erofs_iput(struct erofs_inode *inode);
 erofs_nid_t erofs_lookupnid(struct erofs_inode *inode);
+struct erofs_dentry *erofs_d_alloc(struct erofs_inode *parent,
+				   const char *name);
+int tarerofs_dump_tree(struct erofs_inode *dir);
+struct erofs_inode *erofs_new_inode(void);
 struct erofs_inode *erofs_mkfs_build_tree_from_path(const char *path);
 struct erofs_inode *erofs_mkfs_build_special_from_fd(int fd, const char *name);
 
include/erofs/internal.h +6 -1
index 1f1e730..c184bc4 100644
@@ -22,6 +22,7 @@ typedef unsigned short umode_t;
 #include "erofs_fs.h"
 #include <fcntl.h>
 #include <sys/types.h> /* for off_t definition */
+#include <stdio.h>
 
 #ifndef PATH_MAX
 #define PATH_MAX        4096    /* # chars in a path name including nul */
@@ -174,12 +175,16 @@ struct erofs_inode {
 	} u;
 
 	char *i_srcpath;
-
+	union {
+		char *i_link;
+		FILE *i_tmpfile;
+	};
 	unsigned char datalayout;
 	unsigned char inode_isize;
 	/* inline tail-end packing size */
 	unsigned short idata_size;
 	bool compressed_idata;
+	bool with_tmpfile;
 
 	unsigned int xattr_isize;
 	unsigned int extent_isize;
lib/Makefile.am +1 -1
index faa7311..a377370 100644
@@ -29,7 +29,7 @@ noinst_HEADERS += compressor.h
 liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \
 		      namei.c data.c compress.c compressor.c zmap.c decompress.c \
 		      compress_hints.c hashmap.c sha256.c blobchunk.c dir.c \
-		      fragments.c rb_tree.c dedupe.c
+		      fragments.c rb_tree.c dedupe.c tar.c
 
 liberofs_la_CFLAGS = -Wall -I$(top_srcdir)/include
 if ENABLE_LZ4
lib/blobchunk.c +20 -15
index 8142cc3..9665d63 100644
@@ -128,12 +128,20 @@ int erofs_blob_write_chunk_indexes(struct erofs_inode *inode,
 	     src += sizeof(void *), dst += unit) {
 		struct erofs_blobchunk *chunk;
 
-		chunk = *(void **)(inode->chunkindexes + src);
+		if (blobfile) {
+			chunk = *(void **)(inode->chunkindexes + src);
 
-		if (chunk->blkaddr != EROFS_NULL_ADDR)
-			idx.blkaddr = base_blkaddr + chunk->blkaddr;
-		else
-			idx.blkaddr = EROFS_NULL_ADDR;
+			if (chunk->blkaddr != EROFS_NULL_ADDR)
+				idx.blkaddr = base_blkaddr + chunk->blkaddr;
+			else
+				idx.blkaddr = EROFS_NULL_ADDR;
+		} else if (unit == EROFS_BLOCK_MAP_ENTRY_SIZE) {
+			idx.blkaddr = le32_to_cpu(*(__le32 *)(inode->chunkindexes + dst));
+		} else {
+			struct erofs_inode_chunk_index *i2 = inode->chunkindexes + dst;
+
+			idx.blkaddr = le32_to_cpu(i2->blkaddr);
+		}
 
 		if (extent_start != EROFS_NULL_ADDR &&
 		    idx.blkaddr == extent_end + 1) {
@@ -149,6 +157,10 @@ int erofs_blob_write_chunk_indexes(struct erofs_inode *inode,
 			extent_start = idx.blkaddr;
 			extent_end = idx.blkaddr;
 		}
+
+		if (!blobfile)
+			continue;
+
 		if (unit == EROFS_BLOCK_MAP_ENTRY_SIZE)
 			memcpy(inode->chunkindexes + dst, &idx.blkaddr, unit);
 		else
@@ -166,17 +178,14 @@ int erofs_blob_write_chunk_indexes(struct erofs_inode *inode,
 	return dev_write(inode->chunkindexes, off, inode->extent_isize);
 }
 
-int erofs_blob_write_chunked_file(struct erofs_inode *inode)
+int erofs_blob_write_chunked_file(struct erofs_inode *inode, int fd)
 {
 	unsigned int chunkbits = cfg.c_chunkbits;
 	unsigned int count, unit;
 	struct erofs_inode_chunk_index *idx;
 	erofs_off_t pos, len, chunksize;
-	int fd, ret;
+	int ret;
 
-	fd = open(inode->i_srcpath, O_RDONLY | O_BINARY);
-	if (fd < 0)
-		return -errno;
 #ifdef SEEK_DATA
 	/* if the file is fully sparsed, use one big chunk instead */
 	if (lseek(fd, 0, SEEK_DATA) < 0 && errno == ENXIO) {
@@ -200,10 +209,8 @@ int erofs_blob_write_chunked_file(struct erofs_inode *inode)
 
 	inode->extent_isize = count * unit;
 	idx = malloc(count * max(sizeof(*idx), sizeof(void *)));
-	if (!idx) {
-		close(fd);
+	if (!idx)
 		return -ENOMEM;
-	}
 	inode->chunkindexes = idx;
 
 	for (pos = 0; pos < inode->i_size; pos += len) {
@@ -240,10 +247,8 @@ int erofs_blob_write_chunked_file(struct erofs_inode *inode)
 		*(void **)idx++ = chunk;
 	}
 	inode->datalayout = EROFS_INODE_CHUNK_BASED;
-	close(fd);
 	return 0;
 err:
-	close(fd);
 	free(inode->chunkindexes);
 	inode->chunkindexes = NULL;
 	return ret;
lib/inode.c +106 -41

217 lines not on this page · raw message

lib/tar.c new file +616 -0

616 lines not on this page · raw message

mkfs/main.c +87 -34

185 lines not on this page · raw message

Signature · 1 line
2.24.4