Skip to contentSkip to conversation

erofs-utils: generate preallocated extents for tarerofs

PATCH Superseded Gao Xiang 2 participants 6 messages Jul 29, 2023 – Aug 1, 2023 Library code

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore

Gao Xiang

author
#0 lore raw

PATCH 1/1 erofs-utils: generate preallocated extents for tarerofs

It is also useful to fill data blocks from tarballs at runtime in order to implement image lazy pulling.

Let's generate an additional mapfile for such use cases:
mkfs.erofs --tar=0,MAPFILE IMAGE TARBALL

---
 include/erofs/blobchunk.h  |   7 +-
 include/erofs/block_list.h |   7 +-
 include/erofs/tar.h        |   1 +
 lib/Makefile.am            |   3 +-
 lib/blobchunk.c            | 159 +++++++++++++++++++++++++++++--------
 lib/block_list.c           |  23 ++++--
 lib/tar.c                  |  88 +-------------------
 mkfs/main.c                |  41 ++++++----
 8 files changed, 182 insertions(+), 147 deletions(-)
include/erofs/blobchunk.h +3 -4
index 7c5645e..b15599b 100644
@@ -14,14 +14,13 @@ extern "C"
 
 #include "erofs/internal.h"
 
-struct erofs_blobchunk *erofs_get_unhashed_chunk(erofs_off_t chunksize,
-			unsigned int device_id, erofs_blk_t blkaddr);
 int erofs_blob_write_chunk_indexes(struct erofs_inode *inode, erofs_off_t off);
 int erofs_blob_write_chunked_file(struct erofs_inode *inode, int fd);
-int erofs_blob_remap(struct erofs_sb_info *sbi);
+int tarerofs_write_chunk_data(struct erofs_inode *inode, erofs_off_t data_offset);
+int erofs_mkfs_dump_blobs(struct erofs_sb_info *sbi);
 void erofs_blob_exit(void);
 int erofs_blob_init(const char *blobfile_path);
-int erofs_generate_devtable(struct erofs_sb_info *sbi);
+int erofs_mkfs_init_devices(struct erofs_sb_info *sbi, unsigned int devices);
 
 #ifdef __cplusplus
 }
include/erofs/block_list.h +5 -2
index 78fab44..9f9975e 100644
@@ -13,9 +13,12 @@ extern "C"
 
 #include "internal.h"
 
+int erofs_blocklist_open(char *filename, bool srcmap);
+void erofs_blocklist_close(void);
+
+void tarerofs_blocklist_write(erofs_blk_t blkaddr, erofs_blk_t nblocks,
+			      erofs_off_t srcoff);
 #ifdef WITH_ANDROID
-int erofs_droid_blocklist_fopen(void);
-void erofs_droid_blocklist_fclose(void);
 void erofs_droid_blocklist_write(struct erofs_inode *inode,
 				 erofs_blk_t blk_start, erofs_blk_t nblocks);
 void erofs_droid_blocklist_write_tail_end(struct erofs_inode *inode,
include/erofs/tar.h +1 -0
index a14f8ac..8d3f8de 100644
@@ -15,6 +15,7 @@ struct erofs_pax_header {
 
 struct erofs_tarfile {
 	struct erofs_pax_header global;
+	char *mapfile;
 
 	int fd;
 	u64 offset;
lib/Makefile.am +2 -1
index ebe466b..7a5dc03 100644
@@ -30,7 +30,8 @@ noinst_HEADERS += compressor.h
 liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \
 		      namei.c data.c compress.c compressor.c zmap.c decompress.c \
 		      compress_hints.c hashmap.c sha256.c blobchunk.c dir.c \
-		      fragments.c rb_tree.c dedupe.c uuid_unparse.c uuid.c tar.c
+		      fragments.c rb_tree.c dedupe.c uuid_unparse.c uuid.c tar.c \
+		      block_list.c
 
 liberofs_la_CFLAGS = -Wall ${libuuid_CFLAGS} -I$(top_srcdir)/include
 if ENABLE_LZ4
lib/blobchunk.c +124 -35

260 lines not on this page · raw message

lib/block_list.c +18 -5
index 896fb01..b45b553 100644
@@ -3,7 +3,6 @@
  * Copyright (C), 2021, Coolpad Group Limited.
  * Created by Yue Hu <[email protected]>
  */
-#ifdef WITH_ANDROID
 #include <stdio.h>
 #include <sys/stat.h>
 #include "erofs/block_list.h"
@@ -12,17 +11,19 @@
 #include "erofs/print.h"
 
 static FILE *block_list_fp;
+bool srcmap_enabled;
 
-int erofs_droid_blocklist_fopen(void)
+int erofs_blocklist_open(char *filename, bool srcmap)
 {
-	block_list_fp = fopen(cfg.block_list_file, "w");
+	block_list_fp = fopen(filename, "w");
 
 	if (!block_list_fp)
-		return -1;
+		return -errno;
+	srcmap_enabled = srcmap;
 	return 0;
 }
 
-void erofs_droid_blocklist_fclose(void)
+void erofs_blocklist_close(void)
 {
 	if (!block_list_fp)
 		return;
@@ -31,6 +32,18 @@ void erofs_droid_blocklist_fclose(void)
 	block_list_fp = NULL;
 }
 
+/* XXX: really need to be cleaned up */
+void tarerofs_blocklist_write(erofs_blk_t blkaddr, erofs_blk_t nblocks,
+			      erofs_off_t srcoff)
+{
+	if (!block_list_fp || !nblocks || !srcmap_enabled)
+		return;
+
+	fprintf(block_list_fp, "%08x %8x %08" PRIx64 "\n",
+		blkaddr, nblocks, srcoff);
+}
+
+#ifdef WITH_ANDROID
 static void blocklist_write(const char *path, erofs_blk_t blk_start,
 			    erofs_blk_t nblocks, bool first_extent,
 			    bool last_extent)
lib/tar.c +3 -85

109 lines not on this page · raw message

mkfs/main.c +26 -15

79 lines not on this page · raw message

Signature · 1 line
2.24.4
#1 lore raw

mkfs/main.c int main(int argc, char **argv) · line 802

Quoted 7 lines · Gao Xiang
mkfs/main.c +2 -1
@@ -795,7 +798,8 @@ int main(int argc, char **argv)
 		return 1;
 	}
 
-	if (cfg.block_list_file && erofs_droid_blocklist_fopen() < 0) {
+	if (cfg.block_list_file &&
+	    erofs_blocklist_open(block_list_file, false) < 0) {

erofs_blocklist_open(cfg.block_list_file, false) ?

Signature · 2 lines
Thanks,
Jingbo
author
#2 lore raw
Quoted 7 lines · Jingbo Xu
Quoted 4 lines · Gao Xiang
mkfs/main.c +0 -0
@@ -795,7 +798,8 @@ int main(int argc, char **argv)
  		return 1;
  	}
 
Quoted 5 lines

mkfs/main.c int main(int argc, char **argv) · line 802

Quoted 3 lines
mkfs/main.c +2 -1
-	if (cfg.block_list_file && erofs_droid_blocklist_fopen() < 0) {
+	if (cfg.block_list_file &&
+	    erofs_blocklist_open(block_list_file, false) < 0) {

erofs_blocklist_open(cfg.block_list_file, false) ?

Nice catch, will fix it.

Thanks,
Gao Xiang

#3 lore raw
In reply to Gao Xiang

lib/blobchunk.c err: · line 338

Quoted 6 lines · Gao Xiang
lib/blobchunk.c +4 -1
-int erofs_blob_remap(struct erofs_sb_info *sbi)
+int tarerofs_write_chunk_data(struct erofs_inode *inode, erofs_off_t data_offset)
+{
+	struct erofs_sb_info *sbi = inode->sbi;
+	unsigned int chunkbits = ilog2(inode->i_size - 1) + 1;

What if inode->i_size is 0 ?

Signature · 2 lines
Thanks,
Jingbo
author
#4 lore raw
Quoted 11 lines · Jingbo Xu

lib/blobchunk.c err: · line 338

Quoted 6 lines · Gao Xiang
lib/blobchunk.c +4 -1
-int erofs_blob_remap(struct erofs_sb_info *sbi)
+int tarerofs_write_chunk_data(struct erofs_inode *inode, erofs_off_t data_offset)
+{
+	struct erofs_sb_info *sbi = inode->sbi;
+	unsigned int chunkbits = ilog2(inode->i_size - 1) + 1;

What if inode->i_size is 0 ?

if (chunkbits - sbi->blkszbits > EROFS_CHUNK_FORMAT_BLKBITS_MASK)

	chunkbits = EROFS_CHUNK_FORMAT_BLKBITS_MASK + sbi->blkszbits;

will just generate a large chunk for this.

Actually I think we shouldn't use chunk-based format if inode->i_size is 0, it needs to be fixed in a seperate commit.

Thanks,
Gao Xiang

#5 lore raw
In reply to Gao Xiang
Quoted 37 lines · Gao Xiang
lib/blobchunk.c +36 -1
-int erofs_blob_remap(struct erofs_sb_info *sbi)
+int tarerofs_write_chunk_data(struct erofs_inode *inode, erofs_off_t data_offset)
+{
+	struct erofs_sb_info *sbi = inode->sbi;
+	unsigned int chunkbits = ilog2(inode->i_size - 1) + 1;
+	unsigned int count, unit, device_id;
+	erofs_off_t chunksize, len, pos;
+	erofs_blk_t blkaddr;
+	struct erofs_inode_chunk_index *idx;
+
+	if (chunkbits < sbi->blkszbits)
+		chunkbits = sbi->blkszbits;
+	if (chunkbits - sbi->blkszbits > EROFS_CHUNK_FORMAT_BLKBITS_MASK)
+		chunkbits = EROFS_CHUNK_FORMAT_BLKBITS_MASK + sbi->blkszbits;
+
+	inode->u.chunkformat |= chunkbits - sbi->blkszbits;
+	if (sbi->extra_devices) {
+		device_id = 1;
+		inode->u.chunkformat |= EROFS_CHUNK_FORMAT_INDEXES;
+		unit = sizeof(struct erofs_inode_chunk_index);
+		DBG_BUGON(erofs_blkoff(sbi, data_offset));
+		blkaddr = erofs_blknr(sbi, data_offset);
+	} else {
+		device_id = 0;
+		unit = EROFS_BLOCK_MAP_ENTRY_SIZE;
+		DBG_BUGON(erofs_blkoff(sbi, datablob_size));
+		blkaddr = erofs_blknr(sbi, datablob_size);
+		datablob_size += round_up(inode->i_size, erofs_blksiz(sbi));
+	}
+	chunksize = 1ULL << chunkbits;
+	count = DIV_ROUND_UP(inode->i_size, chunksize);
+
+	inode->extent_isize = count * unit;
+	idx = calloc(count, max(sizeof(*idx), sizeof(void *)));
+	if (!idx)
+		return -ENOMEM;
+	inode->chunkindexes = idx;

lib/blobchunk.c err: · line 380

Quoted 10 lines
lib/blobchunk.c +10 -0
+
+	for (pos = 0; pos < inode->i_size; pos += len) {
+		struct erofs_blobchunk *chunk;
+
+		len = min_t(erofs_off_t, inode->i_size - pos, chunksize);
+
+		chunk = erofs_get_unhashed_chunk(device_id, blkaddr,
+						 data_offset);
+		if (IS_ERR(chunk))
+			return PTR_ERR(chunk);

inode->chunkindexes needs to be freed on error path.

Quoted 9 lines
lib/blobchunk.c +9 -0
+
+		*(void **)idx++ = chunk;
+		blkaddr += erofs_blknr(sbi, len);
+		data_offset += len;
+	}
+	inode->datalayout = EROFS_INODE_CHUNK_BASED;
+	return 0;
+}
+
Signature · 2 lines
Thanks,
Jingbo