erofs-utils: generate preallocated extents for tarerofs
PATCH 1/1 erofs-utils: generate preallocated extents for tarerofs
It is also useful to fill data blocks from tarballs at runtime in order to implement image lazy pulling.
Let's generate an additional mapfile for such use cases:
mkfs.erofs --tar=0,MAPFILE IMAGE TARBALL
--- include/erofs/blobchunk.h | 7 +- include/erofs/block_list.h | 7 +- include/erofs/tar.h | 1 + lib/Makefile.am | 3 +- lib/blobchunk.c | 159 +++++++++++++++++++++++++++++-------- lib/block_list.c | 23 ++++-- lib/tar.c | 88 +------------------- mkfs/main.c | 41 ++++++---- 8 files changed, 182 insertions(+), 147 deletions(-)
include/erofs/blobchunk.h +3 -4
@@ -14,14 +14,13 @@ extern "C"#include "erofs/internal.h" -struct erofs_blobchunk *erofs_get_unhashed_chunk (erofs_off_t chunksize, -unsigned int device_id,erofs_blk_t blkaddr);int erofs_blob_write_chunk_indexes (struct erofs_inode *inode,erofs_off_t off);int erofs_blob_write_chunked_file (struct erofs_inode *inode,int fd); -int erofs_blob_remap (struct erofs_sb_info *sbi); +int tarerofs_write_chunk_data (struct erofs_inode *inode,erofs_off_t data_offset); +int erofs_mkfs_dump_blobs (struct erofs_sb_info *sbi);void erofs_blob_exit (void );int erofs_blob_init (const char *blobfile_path); -int erofs_generate_devtable (struct erofs_sb_info *sbi); +int erofs_mkfs_init_devices (struct erofs_sb_info *sbi,unsigned int devices); #ifdef __cplusplus }
include/erofs/block_list.h +5 -2
@@ -13,9 +13,12 @@ extern "C"#include "internal.h" +int erofs_blocklist_open (char *filename,bool srcmap); +void erofs_blocklist_close (void ); + +void tarerofs_blocklist_write (erofs_blk_t blkaddr,erofs_blk_t nblocks, +erofs_off_t srcoff);#ifdef WITH_ANDROID -int erofs_droid_blocklist_fopen (void ); -void erofs_droid_blocklist_fclose (void );void erofs_droid_blocklist_write (struct erofs_inode *inode,erofs_blk_t blk_start,erofs_blk_t nblocks); void erofs_droid_blocklist_write_tail_end(struct erofs_inode *inode,
include/erofs/tar.h +1 -0
@@ -15,6 +15,7 @@ struct erofs_pax_header {struct erofs_tarfile {struct erofs_pax_header global; +char *mapfile;int fd;u64 offset;
lib/Makefile.am +2 -1
@@ -30,7 +30,8 @@ noinst_HEADERS += compressor.h liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \ namei.c data.c compress.c compressor.c zmap.c decompress.c \ compress_hints.c hashmap.c sha256.c blobchunk.c dir.c \ - fragments.c rb_tree.c dedupe.c uuid_unparse.c uuid.c tar.c + fragments.c rb_tree.c dedupe.c uuid_unparse.c uuid.c tar.c \ + block_list.c liberofs_la_CFLAGS = -Wall ${libuuid_CFLAGS} -I$(top_srcdir)/include if ENABLE_LZ4
lib/blobchunk.c +124 -35
260 lines not on this page · raw message
lib/block_list.c +18 -5
@@ -3,7 +3,6 @@* Copyright (C), 2021, Coolpad Group Limited. * Created by Yue Hu <[email protected]> */ -#ifdef WITH_ANDROID #include <stdio.h> #include <sys/stat.h> #include "erofs/block_list.h"
@@ -12,17 +11,19 @@#include "erofs/print.h" static FILE *block_list_fp; +bool srcmap_enabled; -int erofs_droid_blocklist_fopen (void ) +int erofs_blocklist_open (char *filename,bool srcmap) { - block_list_fp =fopen (cfg.block_list_file,"w" ); + block_list_fp =fopen (filename,"w" );if (!block_list_fp) -return -1 ; +return -errno; + srcmap_enabled = srcmap;return 0 ; } -void erofs_droid_blocklist_fclose (void ) +void erofs_blocklist_close (void ) {if (!block_list_fp)return ;
@@ -31,6 +32,18 @@ void erofs_droid_blocklist_fclose(void) block_list_fp =NULL ; } +/* XXX: really need to be cleaned up */ +void tarerofs_blocklist_write (erofs_blk_t blkaddr,erofs_blk_t nblocks, +erofs_off_t srcoff) +{ +if (!block_list_fp || !nblocks || !srcmap_enabled) +return ; + +fprintf (block_list_fp,"%08x %8x %08" PRIx64"\n" , + blkaddr, nblocks, srcoff); +} + +#ifdef WITH_ANDROID static void blocklist_write (const char *path,erofs_blk_t blk_start,erofs_blk_t nblocks,bool first_extent,bool last_extent)
lib/tar.c +3 -85
109 lines not on this page · raw message
mkfs/main.c +26 -15
79 lines not on this page · raw message
Signature · 1 line
2.24.4
mkfs/main.c int main(int argc, char **argv) · line 802
Quoted 7 lines · Gao Xiang
mkfs/main.c +2
-1@@ -795,7 +798,8 @@ int main(int argc, char **argv)return 1 ; } -if (cfg.block_list_file &&erofs_droid_blocklist_fopen () <0 ) { +if (cfg.block_list_file && +erofs_blocklist_open (block_list_file, false) <0 ) {
erofs_blocklist_open(cfg.block_list_file, false) ?
Signature · 2 lines
Thanks, Jingbo
Quoted 7 lines · Jingbo Xu
Quoted 4 lines · Gao Xiang
mkfs/main.c +0
-0@@ -795,7 +798,8 @@ int main(int argc, char **argv)return 1 ; }
Quoted 5 lines
mkfs/main.c int main(int argc, char **argv) · line 802
Quoted 3 lines
mkfs/main.c +2
-1-if (cfg.block_list_file &&erofs_droid_blocklist_fopen () <0 ) { +if (cfg.block_list_file && +erofs_blocklist_open (block_list_file, false) <0 ) {
erofs_blocklist_open(cfg.block_list_file, false)?
Nice catch, will fix it.
Thanks,
Gao Xiang
lib/blobchunk.c err: · line 338
Quoted 6 lines · Gao Xiang
lib/blobchunk.c +4
-1-int erofs_blob_remap (struct erofs_sb_info *sbi) +int tarerofs_write_chunk_data (struct erofs_inode *inode,erofs_off_t data_offset) +{ +struct erofs_sb_info *sbi = inode->sbi; +unsigned int chunkbits =ilog2 (inode->i_size -1 ) +1 ;
What if inode->i_size is 0 ?
Signature · 2 lines
Thanks, Jingbo
Quoted 11 lines · Jingbo Xu
lib/blobchunk.c err: · line 338
Quoted 6 lines · Gao Xiang
lib/blobchunk.c +4
-1-int erofs_blob_remap (struct erofs_sb_info *sbi) +int tarerofs_write_chunk_data (struct erofs_inode *inode,erofs_off_t data_offset) +{ +struct erofs_sb_info *sbi = inode->sbi; +unsigned int chunkbits =ilog2 (inode->i_size -1 ) +1 ;What if
inode->i_sizeis 0 ?
if (chunkbits - sbi->blkszbits > EROFS_CHUNK_FORMAT_BLKBITS_MASK)
chunkbits = EROFS_CHUNK_FORMAT_BLKBITS_MASK + sbi->blkszbits;
will just generate a large chunk for this.
Actually I think we shouldn't use chunk-based format if inode->i_size is 0, it needs to be fixed in a seperate commit.
Thanks,
Gao Xiang
Quoted 37 lines · Gao Xiang
lib/blobchunk.c +36
-1-int erofs_blob_remap (struct erofs_sb_info *sbi) +int tarerofs_write_chunk_data (struct erofs_inode *inode,erofs_off_t data_offset) +{ +struct erofs_sb_info *sbi = inode->sbi; +unsigned int chunkbits =ilog2 (inode->i_size -1 ) +1 ; +unsigned int count, unit, device_id; +erofs_off_t chunksize, len, pos; +erofs_blk_t blkaddr; +struct erofs_inode_chunk_index *idx; + +if (chunkbits < sbi->blkszbits) + chunkbits = sbi->blkszbits; +if (chunkbits - sbi->blkszbits >EROFS_CHUNK_FORMAT_BLKBITS_MASK ) + chunkbits =EROFS_CHUNK_FORMAT_BLKBITS_MASK + sbi->blkszbits; + + inode->u.chunkformat |= chunkbits - sbi->blkszbits; +if (sbi->extra_devices) { + device_id =1 ; + inode->u.chunkformat |=EROFS_CHUNK_FORMAT_INDEXES ; + unit =sizeof (struct erofs_inode_chunk_index ); +DBG_BUGON (erofs_blkoff (sbi, data_offset)); + blkaddr =erofs_blknr (sbi, data_offset); + }else { + device_id =0 ; + unit =EROFS_BLOCK_MAP_ENTRY_SIZE ; +DBG_BUGON (erofs_blkoff (sbi, datablob_size)); + blkaddr =erofs_blknr (sbi, datablob_size); + datablob_size +=round_up (inode->i_size,erofs_blksiz (sbi)); + } + chunksize =1ULL << chunkbits; + count =DIV_ROUND_UP (inode->i_size, chunksize); + + inode->extent_isize = count * unit; + idx =calloc (count,max (sizeof (*idx),sizeof (void *))); +if (!idx) +return -ENOMEM ; + inode->chunkindexes = idx;
lib/blobchunk.c err: · line 380
Quoted 10 lines
lib/blobchunk.c +10
-0+ +for (pos =0 ; pos < inode->i_size; pos += len) { +struct erofs_blobchunk *chunk; + + len =min_t (erofs_off_t, inode->i_size - pos, chunksize); + + chunk =erofs_get_unhashed_chunk (device_id, blkaddr, + data_offset); +if (IS_ERR (chunk)) +return PTR_ERR (chunk);
inode->chunkindexes needs to be freed on error path.
Quoted 9 lines
lib/blobchunk.c +9
-0+ + *(void **)idx++ = chunk; + blkaddr +=erofs_blknr (sbi, len); + data_offset += len; + } + inode->datalayout =EROFS_INODE_CHUNK_BASED ; +return 0 ; +} +
Signature · 2 lines
Thanks, Jingbo