AOSP: erofs-utils: add block list support
PATCH 1/1 AOSP: erofs-utils: add block list support
From: Yue Hu <[email protected]>
Android update engine will treat EROFS filesystem image as one single file. Let's add block list support to optimize OTA size.
--- include/erofs/block_list.h | 19 ++++++++++ include/erofs/config.h | 1 + lib/block_list.c | 86 ++++++++++++++++++++++++++++++++++++++++++++++ lib/compress.c | 8 +++++ lib/inode.c | 21 ++++++++++- mkfs/main.c | 17 +++++++++ 6 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 include/erofs/block_list.h create mode 100644 lib/block_list.c
include/erofs/block_list.h new file +19 -0
@@ -0,0 +1,19 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* +* erofs-utils/include/erofs/block_list.h +* +* Copyright (C), 2021, Coolpad Group Limited. +* Created by Yue Hu <[email protected]> +*/ +#ifndef __EROFS_BLOCK_LIST_H +#define __EROFS_BLOCK_LIST_H + +#include "internal.h" + +int block_list_fopen (void ); +void block_list_fclose (void ); +void write_block_list (const char *path,erofs_blk_t blk_start, +erofs_blk_t nblocks,bool has_tail); +void write_block_list_tail_end (const char *path,erofs_blk_t nblocks, +bool inline_data,erofs_blk_t blkaddr); +#endif
include/erofs/config.h +1 -0
@@ -65,6 +65,7 @@ struct erofs_configure {char *mount_point;char *target_out_path;char *fs_config_file; +char *block_list_file;#endif };
lib/block_list.c new file +86 -0
86 lines not on this page · raw message
lib/compress.c +8 -0
@@ -19,6 +19,10 @@#include "erofs/compress.h" #include "compressor.h" +#ifdef WITH_ANDROID +#include "erofs/block_list.h" +#endif +static struct erofs_compress compresshandle;static int compressionlevel;
@@ -553,6 +557,10 @@ int erofs_write_compressed_file(struct erofs_inode *inode) inode->i_srcpath, (unsigned long long )inode->i_size, compressed_blocks); +#ifdef WITH_ANDROID +write_block_list (inode->i_srcpath, blkaddr, compressed_blocks, false); +#endif +/* * TODO: need to move erofs_bdrop to erofs_write_tail_end * when both mkfs & kernel support compression inline.
lib/inode.c +20 -1
@@ -22,6 +22,10 @@#include "erofs/xattr.h" #include "erofs/exclude.h" +#ifdef WITH_ANDROID +#include "erofs/block_list.h" +#endif +#define S_SHIFT 12static unsigned char erofs_ftype_by_mode[S_IFMT >>S_SHIFT ] = { [S_IFREG >>S_SHIFT ] =EROFS_FT_REG_FILE ,
@@ -369,6 +373,12 @@ static int write_uncompressed_file_from_fd(struct erofs_inode *inode, int fd)return -EIO ; } } + +#ifdef WITH_ANDROID +if (nblocks) +write_block_list (inode->i_srcpath, inode->u.i_blkaddr, + nblocks, inode->idata_size ? true : false); +#endif return 0 ; }
@@ -626,6 +636,7 @@ static struct erofs_bhops erofs_write_inline_bhops = {int erofs_write_tail_end (struct erofs_inode *inode) {struct erofs_buffer_head *bh, *ibh; +erofs_off_t pos; bh = inode->bh_data;
@@ -640,7 +651,6 @@ int erofs_write_tail_end(struct erofs_inode *inode) ibh->op = &erofs_write_inline_bhops; }else {int ret; -erofs_off_t pos;erofs_mapbh (bh->block); pos =erofs_btell (bh, true) -EROFS_BLKSIZ ;
@@ -658,6 +668,15 @@ int erofs_write_tail_end(struct erofs_inode *inode)free (inode->idata); inode->idata =NULL ; } + +#ifdef WITH_ANDROID +if (!S_ISDIR (inode->i_mode) && !S_ISLNK (inode->i_mode)) +write_block_list_tail_end (inode->i_srcpath, + inode->i_size /EROFS_BLKSIZ , + inode->bh_inline ? true: false, +erofs_blknr (pos)); +#endif + out:/* now bh_data can drop directly */ if (bh) {
mkfs/main.c +17 -0
@@ -27,6 +27,10 @@#include <uuid.h> #endif +#ifdef WITH_ANDROID +#include "erofs/block_list.h" +#endif +#define EROFS_SUPER_END (EROFS_SUPER_OFFSET + sizeof(struct erofs_super_block))static struct option long_options[] = {
@@ -47,6 +51,7 @@ static struct option long_options[] = { {"mount-point", required_argument, NULL, 10}, {"product-out", required_argument, NULL, 11}, {"fs-config-file", required_argument, NULL, 12}, + {"block-list-file", required_argument, NULL, 13}, #endif {0, 0, 0, 0}, };
@@ -95,6 +100,7 @@ static void usage(void)" --mount-point=X X=prefix of target fs path (default: /)\n" " --product-out=X X=product_out directory\n" " --fs-config-file=X X=fs_config file\n" + " --block-list-file=X X=block_list file\n"#endif "\nAvailable compressors are: ", stderr);print_available_compressors (stderr,", " );
@@ -293,6 +299,9 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])case 12 : cfg.fs_config_file = optarg;break ; +case 13 : + cfg.block_list_file = optarg; +break ;#endif case 'C' : i =strtoull (optarg, &endptr,0 );
@@ -541,6 +550,11 @@ int main(int argc, char **argv)erofs_err ("failed to load fs config %s" , cfg.fs_config_file); return 1; } + +if (cfg.block_list_file &&block_list_fopen () <0 ) { +erofs_err ("failed to open %s" , cfg.block_list_file); +return 1 ; + }#endif erofs_show_config ();
@@ -607,6 +621,9 @@ int main(int argc, char **argv) err =erofs_mkfs_superblock_csum_set (); exit:z_erofs_compress_exit (); +#ifdef WITH_ANDROID +block_list_fclose (); +#endif dev_close ();erofs_cleanup_exclude_rules ();erofs_exit_configure ();
Signature · 1 line
1.9.1
Hi Yue,
Thanks for your contribution!
Quoted 6 lines · Yue Hu
From: Yue Hu <[email protected]>
Android update engine will treat EROFS filesystem image as one single file. Let's add block list support to optimize OTA size.
Change-Id: I21d6177dff0ee65d3c57023b102e991d40873f0d
Change-Id is not strictly necessary here.
I'm not quite familiar with Android block list format (Guifu knows better than me), may be Jianan and Guifu could review it first. I'll seek time on this off work.
Hi Jianan,
I heard that you guys have some internal implementation as well, could you also help review this patch?
Hi Guifu,
Could you also take a look at this against the previous huawei in-house Android block list implementation and review this as well?
Thanks,
Gao Xiang
Quoted 292 lines
Signed-off-by Yue Hu <[email protected]>--- include/erofs/block_list.h | 19 ++++++++++ include/erofs/config.h | 1 + lib/block_list.c | 86 ++++++++++++++++++++++++++++++++++++++++++++++ lib/compress.c | 8 +++++ lib/inode.c | 21 ++++++++++- mkfs/main.c | 17 +++++++++ 6 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 include/erofs/block_list.h create mode 100644 lib/block_list.cinclude/erofs/block_list.h new file +19
-0@@ -0,0 +1,19 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* +* erofs-utils/include/erofs/block_list.h +* +* Copyright (C), 2021, Coolpad Group Limited. +* Created by Yue Hu <[email protected]> +*/ +#ifndef __EROFS_BLOCK_LIST_H +#define __EROFS_BLOCK_LIST_H + +#include "internal.h" + +int block_list_fopen (void ); +void block_list_fclose (void ); +void write_block_list (const char *path,erofs_blk_t blk_start, +erofs_blk_t nblocks,bool has_tail); +void write_block_list_tail_end (const char *path,erofs_blk_t nblocks, +bool inline_data,erofs_blk_t blkaddr); +#endif include/erofs/config.h +1
-0@@ -65,6 +65,7 @@ struct erofs_configure {char *mount_point;char *target_out_path;char *fs_config_file; +char *block_list_file;#endif };lib/block_list.c new file +86
-0@@ -0,0 +1,86 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* +* erofs-utils/lib/block_list.c +* +* Copyright (C), 2021, Coolpad Group Limited. +* Created by Yue Hu <[email protected]> +*/ +#ifdef WITH_ANDROID +#include <stdio.h> + +#include "erofs/block_list.h" + +#define pr_fmt (fmt) "EROFS block_list: " FUNC_LINE_FMT fmt "\n" +#include "erofs/print.h" + +static FILE *block_list_fp =NULL ; + +int block_list_fopen (void ) +{ +if (block_list_fp) +return 0 ; + + block_list_fp =fopen (cfg.block_list_file,"w" ); + +if (block_list_fp ==NULL ) +return -1 ; + +return 0 ; +} + +void block_list_fclose (void ) +{ +if (block_list_fp) { +fclose (block_list_fp); + block_list_fp =NULL ; + } +} + +void write_block_list (const char *path,erofs_blk_t blk_start, +erofs_blk_t nblocks,bool has_tail) +{ +const char *fspath =erofs_fspath (path); + +if (!block_list_fp || !cfg.mount_point) +return ; + +/* only tail-end data */ +if (!nblocks) +return ; + +fprintf (block_list_fp,"/%s" , cfg.mount_point); + +if (fspath[0 ] !='/' ) +fprintf (block_list_fp,"/" ); + +if (nblocks ==1 ) { +fprintf (block_list_fp,"%s %u" , fspath, blk_start); + }else { +fprintf (block_list_fp,"%s %u-%u" , fspath, blk_start, + blk_start + nblocks -1 ); + } + +if (!has_tail) +fprintf (block_list_fp,"\n" ); +} + +void write_block_list_tail_end (const char *path,erofs_blk_t nblocks, +bool inline_data,erofs_blk_t blkaddr) +{ +if (!block_list_fp || !cfg.mount_point) +return ; + +if (!nblocks && !inline_data) { +erofs_dbg ("%s : only tail-end non-inline data" , path); +write_block_list (path, blkaddr,1 , false); +return ; + } + +if (nblocks) { +if (!inline_data) +fprintf (block_list_fp," %u" , blkaddr); + +fprintf (block_list_fp,"\n" ); + } +} +#endif lib/compress.c +8
-0@@ -19,6 +19,10 @@#include "erofs/compress.h" #include "compressor.h" +#ifdef WITH_ANDROID +#include "erofs/block_list.h" +#endif +static struct erofs_compress compresshandle;static int compressionlevel;@@ -553,6 +557,10 @@ int erofs_write_compressed_file(struct erofs_inode *inode) inode->i_srcpath, (unsigned long long )inode->i_size, compressed_blocks); +#ifdef WITH_ANDROID +write_block_list (inode->i_srcpath, blkaddr, compressed_blocks, false); +#endif +/* * TODO: need to move erofs_bdrop to erofs_write_tail_end * when both mkfs & kernel support compression inline. lib/inode.c +20
-1@@ -22,6 +22,10 @@#include "erofs/xattr.h" #include "erofs/exclude.h" +#ifdef WITH_ANDROID +#include "erofs/block_list.h" +#endif +#define S_SHIFT 12static unsigned char erofs_ftype_by_mode[S_IFMT >>S_SHIFT ] = { [S_IFREG >>S_SHIFT ] =EROFS_FT_REG_FILE ,@@ -369,6 +373,12 @@ static int write_uncompressed_file_from_fd(struct erofs_inode *inode, int fd)return -EIO ; } } + +#ifdef WITH_ANDROID +if (nblocks) +write_block_list (inode->i_srcpath, inode->u.i_blkaddr, + nblocks, inode->idata_size ? true : false); +#endif return 0 ; }@@ -626,6 +636,7 @@ static struct erofs_bhops erofs_write_inline_bhops = {int erofs_write_tail_end (struct erofs_inode *inode) {struct erofs_buffer_head *bh, *ibh; +erofs_off_t pos; bh = inode->bh_data;@@ -640,7 +651,6 @@ int erofs_write_tail_end(struct erofs_inode *inode) ibh->op = &erofs_write_inline_bhops; }else {int ret; -erofs_off_t pos;erofs_mapbh (bh->block); pos =erofs_btell (bh, true) -EROFS_BLKSIZ ;@@ -658,6 +668,15 @@ int erofs_write_tail_end(struct erofs_inode *inode)free (inode->idata); inode->idata =NULL ; } + +#ifdef WITH_ANDROID +if (!S_ISDIR (inode->i_mode) && !S_ISLNK (inode->i_mode)) +write_block_list_tail_end (inode->i_srcpath, + inode->i_size /EROFS_BLKSIZ , + inode->bh_inline ? true: false, +erofs_blknr (pos)); +#endif + out:/* now bh_data can drop directly */ if (bh) {mkfs/main.c +17
-0@@ -27,6 +27,10 @@#include <uuid.h> #endif +#ifdef WITH_ANDROID +#include "erofs/block_list.h" +#endif +#define EROFS_SUPER_END (EROFS_SUPER_OFFSET + sizeof(struct erofs_super_block))static struct option long_options[] = {@@ -47,6 +51,7 @@ static struct option long_options[] = { {"mount-point", required_argument, NULL, 10}, {"product-out", required_argument, NULL, 11}, {"fs-config-file", required_argument, NULL, 12}, + {"block-list-file", required_argument, NULL, 13}, #endif {0, 0, 0, 0}, };@@ -95,6 +100,7 @@ static void usage(void)" --mount-point=X X=prefix of target fs path (default: /)\n" " --product-out=X X=product_out directory\n" " --fs-config-file=X X=fs_config file\n" + " --block-list-file=X X=block_list file\n"#endif "\nAvailable compressors are: ", stderr);print_available_compressors (stderr,", " );@@ -293,6 +299,9 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])case 12 : cfg.fs_config_file = optarg;break ; +case 13 : + cfg.block_list_file = optarg; +break ;#endif case 'C' : i =strtoull (optarg, &endptr,0 );@@ -541,6 +550,11 @@ int main(int argc, char **argv)erofs_err ("failed to load fs config %s" , cfg.fs_config_file); return 1; } + +if (cfg.block_list_file &&block_list_fopen () <0 ) { +erofs_err ("failed to open %s" , cfg.block_list_file); +return 1 ; + }#endif erofs_show_config ();@@ -607,6 +621,9 @@ int main(int argc, char **argv) err =erofs_mkfs_superblock_csum_set (); exit:z_erofs_compress_exit (); +#ifdef WITH_ANDROID +block_list_fclose (); +#endif dev_close ();erofs_cleanup_exclude_rules ();erofs_exit_configure ();Signature · 1 line
1.9.1
Hi all,
This patch works well for us.
Thanks,
Jianan
Quoted 296 lines · Yue Hu
From: Yue Hu <[email protected]>
Android update engine will treat EROFS filesystem image as one single file. Let's add block list support to optimize OTA size.
Change-Id I21d6177dff0ee65d3c57023b102e991d40873f0dSigned-off-by Yue Hu <[email protected]>--- include/erofs/block_list.h | 19 ++++++++++ include/erofs/config.h | 1 + lib/block_list.c | 86 ++++++++++++++++++++++++++++++++++++++++++++++ lib/compress.c | 8 +++++ lib/inode.c | 21 ++++++++++- mkfs/main.c | 17 +++++++++ 6 files changed, 151 insertions(+), 1 deletion(-)create mode 100644
include/erofs/block_list.h
create mode 100644lib/block_list.cinclude/erofs/block_list.h new file +19
-0@@ -0,0 +1,19 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* +* erofs-utils/include/erofs/block_list.h +* +* Copyright (C), 2021, Coolpad Group Limited. +* Created by Yue Hu <[email protected]> +*/ +#ifndef __EROFS_BLOCK_LIST_H +#define __EROFS_BLOCK_LIST_H + +#include "internal.h" + +int block_list_fopen (void ); +void block_list_fclose (void ); +void write_block_list (const char *path,erofs_blk_t blk_start, +erofs_blk_t nblocks,bool has_tail); +void write_block_list_tail_end (const char *path,erofs_blk_t nblocks, +bool inline_data,erofs_blk_t blkaddr); +#endif include/erofs/config.h +1
-0@@ -65,6 +65,7 @@ struct erofs_configure {char *mount_point;char *target_out_path;char *fs_config_file; +char *block_list_file;#endif };lib/block_list.c new file +86
-0@@ -0,0 +1,86 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* +* erofs-utils/lib/block_list.c +* +* Copyright (C), 2021, Coolpad Group Limited. +* Created by Yue Hu <[email protected]> +*/ +#ifdef WITH_ANDROID +#include <stdio.h> + +#include "erofs/block_list.h" + +#define pr_fmt (fmt) "EROFS block_list: " FUNC_LINE_FMT fmt "\n" +#include "erofs/print.h" + +static FILE *block_list_fp =NULL ; + +int block_list_fopen (void ) +{ +if (block_list_fp) +return 0 ; + + block_list_fp =fopen (cfg.block_list_file,"w" ); + +if (block_list_fp ==NULL ) +return -1 ; + +return 0 ; +} + +void block_list_fclose (void ) +{ +if (block_list_fp) { +fclose (block_list_fp); + block_list_fp =NULL ; + } +} + +void write_block_list (const char *path,erofs_blk_t blk_start, +erofs_blk_t nblocks,bool has_tail) +{ +const char *fspath =erofs_fspath (path); + +if (!block_list_fp || !cfg.mount_point) +return ; + +/* only tail-end data */ +if (!nblocks) +return ; + +fprintf (block_list_fp,"/%s" , cfg.mount_point); + +if (fspath[0 ] !='/' ) +fprintf (block_list_fp,"/" ); + +if (nblocks ==1 ) { +fprintf (block_list_fp,"%s %u" , fspath, blk_start); + }else { +fprintf (block_list_fp,"%s %u-%u" , fspath, blk_start, + blk_start + nblocks -1 ); + } + +if (!has_tail) +fprintf (block_list_fp,"\n" ); +} + +void write_block_list_tail_end (const char *path,erofs_blk_t nblocks, +bool inline_data,erofs_blk_t blkaddr) +{ +if (!block_list_fp || !cfg.mount_point) +return ; + +if (!nblocks && !inline_data) { +erofs_dbg ("%s : only tail-end non-inline data" , path); +write_block_list (path, blkaddr,1 , false); +return ; + } + +if (nblocks) { +if (!inline_data) +fprintf (block_list_fp," %u" , blkaddr); + +fprintf (block_list_fp,"\n" ); + } +} +#endif lib/compress.c +8
-0@@ -19,6 +19,10 @@#include "erofs/compress.h" #include "compressor.h" +#ifdef WITH_ANDROID +#include "erofs/block_list.h" +#endif +static struct erofs_compress compresshandle;static int compressionlevel;@@ -553,6 +557,10 @@ int erofs_write_compressed_file(struct erofs_inode *inode) inode->i_srcpath, (unsigned long long )inode->i_size, compressed_blocks); +#ifdef WITH_ANDROID +write_block_list (inode->i_srcpath, blkaddr, compressed_blocks, false); +#endif +/* * TODO: need to move erofs_bdrop to erofs_write_tail_end * when both mkfs & kernel support compression inline. lib/inode.c +20
-1@@ -22,6 +22,10 @@#include "erofs/xattr.h" #include "erofs/exclude.h" +#ifdef WITH_ANDROID +#include "erofs/block_list.h" +#endif +#define S_SHIFT 12static unsigned char erofs_ftype_by_mode[S_IFMT >>S_SHIFT ] = { [S_IFREG >>S_SHIFT ] =EROFS_FT_REG_FILE ,@@ -369,6 +373,12 @@ static int write_uncompressed_file_from_fd(struct erofs_inode *inode, int fd)return -EIO ; } } + +#ifdef WITH_ANDROID +if (nblocks) +write_block_list (inode->i_srcpath, inode->u.i_blkaddr, + nblocks, inode->idata_size ? true : false); +#endif return 0 ; }@@ -626,6 +636,7 @@ static struct erofs_bhops erofs_write_inline_bhops = {int erofs_write_tail_end (struct erofs_inode *inode) {struct erofs_buffer_head *bh, *ibh; +erofs_off_t pos; bh = inode->bh_data;@@ -640,7 +651,6 @@ int erofs_write_tail_end(struct erofs_inode *inode) ibh->op = &erofs_write_inline_bhops; }else {int ret; -erofs_off_t pos;erofs_mapbh (bh->block); pos =erofs_btell (bh, true) -EROFS_BLKSIZ ;@@ -658,6 +668,15 @@ int erofs_write_tail_end(struct erofs_inode *inode)free (inode->idata); inode->idata =NULL ; } + +#ifdef WITH_ANDROID +if (!S_ISDIR (inode->i_mode) && !S_ISLNK (inode->i_mode)) +write_block_list_tail_end (inode->i_srcpath, + inode->i_size /EROFS_BLKSIZ , + inode->bh_inline ? true: false, +erofs_blknr (pos)); +#endif + out:/* now bh_data can drop directly */ if (bh) {mkfs/main.c +17
-0@@ -27,6 +27,10 @@#include <uuid.h> #endif +#ifdef WITH_ANDROID +#include "erofs/block_list.h" +#endif +#define EROFS_SUPER_END (EROFS_SUPER_OFFSET + sizeof(struct erofs_super_block))static struct option long_options[] = {@@ -47,6 +51,7 @@ static struct option long_options[] = { {"mount-point", required_argument, NULL, 10}, {"product-out", required_argument, NULL, 11}, {"fs-config-file", required_argument, NULL, 12}, + {"block-list-file", required_argument, NULL, 13}, #endif {0, 0, 0, 0}, };@@ -95,6 +100,7 @@ static void usage(void)" --mount-point=X X=prefix of target fs path (default: /)\n" " --product-out=X X=product_out directory\n" " --fs-config-file=X X=fs_config file\n" + " --block-list-file=X X=block_list file\n"#endif "\nAvailable compressors are: ", stderr);print_available_compressors (stderr,", " );@@ -293,6 +299,9 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])case 12 : cfg.fs_config_file = optarg;break ; +case 13 : + cfg.block_list_file = optarg; +break ;#endif case 'C' : i =strtoull (optarg, &endptr,0 );@@ -541,6 +550,11 @@ int main(int argc, char **argv)erofs_err ("failed to load fs config %s" , cfg.fs_config_file); return 1; } + +if (cfg.block_list_file &&block_list_fopen () <0 ) { +erofs_err ("failed to open %s" , cfg.block_list_file); +return 1 ; + }#endif erofs_show_config ();@@ -607,6 +621,9 @@ int main(int argc, char **argv) err =erofs_mkfs_superblock_csum_set (); exit:z_erofs_compress_exit (); +#ifdef WITH_ANDROID +block_list_fclose (); +#endif dev_close ();erofs_cleanup_exclude_rules ();erofs_exit_configure ();
Hi Jianan,
Quoted 5 lines · Huang Jianan
Hi all,
This patch works well for us.
Tested-by Huang Jianan <[email protected]>
Thanks for confirmation.
Quoted 5 lines
Thanks,
Jianan
Hi Yue,
Quoted 35 lines · Yue Hu
Quoted 34 lines
From: Yue Hu <[email protected]>
Android update engine will treat EROFS filesystem image as one single file. Let's add block list support to optimize OTA size.
Change-Id I21d6177dff0ee65d3c57023b102e991d40873f0dSigned-off-by Yue Hu <[email protected]>--- include/erofs/block_list.h | 19 ++++++++++ include/erofs/config.h | 1 + lib/block_list.c | 86 ++++++++++++++++++++++++++++++++++++++++++++++ lib/compress.c | 8 +++++ lib/inode.c | 21 ++++++++++- mkfs/main.c | 17 +++++++++ 6 files changed, 151 insertions(+), 1 deletion(-)create mode 100644
include/erofs/block_list.h
create mode 100644lib/block_list.cinclude/erofs/block_list.h new file +10
-0@@ -0,0 +1,19 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* +* erofs-utils/include/erofs/block_list.h +* +* Copyright (C), 2021, Coolpad Group Limited. +* Created by Yue Hu <[email protected]> +*/ +#ifndef __EROFS_BLOCK_LIST_H +#define __EROFS_BLOCK_LIST_H +
Quoted 8 lines
include/erofs/block_list.h line 18
Quoted 8 lines
include/erofs/block_list.h new file +8
-0+#include "internal.h" + +int block_list_fopen (void ); +void block_list_fclose (void ); +void write_block_list (const char *path,erofs_blk_t blk_start, +erofs_blk_t nblocks,bool has_tail); +void write_block_list_tail_end (const char *path,erofs_blk_t nblocks, +bool inline_data,erofs_blk_t blkaddr);
some nitpick...
How about use "erofs_droid_blocklist_" prefix here?
even though it may not be quite useful, but I'd like to avoid potential namespace polluation...
Quoted 78 lines
Quoted 78 lines
include/erofs/block_list.h new file +1
-0+#endif include/erofs/config.h +1
-0@@ -65,6 +65,7 @@ struct erofs_configure {char *mount_point;char *target_out_path;char *fs_config_file; +char *block_list_file;#endif };lib/block_list.c new file +60
-0@@ -0,0 +1,86 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* +* erofs-utils/lib/block_list.c +* +* Copyright (C), 2021, Coolpad Group Limited. +* Created by Yue Hu <[email protected]> +*/ +#ifdef WITH_ANDROID +#include <stdio.h> + +#include "erofs/block_list.h" + +#define pr_fmt (fmt) "EROFS block_list: " FUNC_LINE_FMT fmt "\n" +#include "erofs/print.h" + +static FILE *block_list_fp =NULL ; + +int block_list_fopen (void ) +{ +if (block_list_fp) +return 0 ; + + block_list_fp =fopen (cfg.block_list_file,"w" ); + +if (block_list_fp ==NULL ) +return -1 ; + +return 0 ; +} + +void block_list_fclose (void ) +{ +if (block_list_fp) { +fclose (block_list_fp); + block_list_fp =NULL ; + } +} + +void write_block_list (const char *path,erofs_blk_t blk_start, +erofs_blk_t nblocks,bool has_tail) +{ +const char *fspath =erofs_fspath (path); + +if (!block_list_fp || !cfg.mount_point) +return ; + +/* only tail-end data */ +if (!nblocks) +return ; + +fprintf (block_list_fp,"/%s" , cfg.mount_point); + +if (fspath[0 ] !='/' ) +fprintf (block_list_fp,"/" ); + +if (nblocks ==1 ) { +fprintf (block_list_fp,"%s %u" , fspath, blk_start); + }else { +fprintf (block_list_fp,"%s %u-%u" , fspath, blk_start, + blk_start + nblocks -1 );
Quoted 8 lines
lib/block_list.c line 68
Quoted 8 lines
lib/block_list.c new file +8
-0+ } + +if (!has_tail) +fprintf (block_list_fp,"\n" ); +} + +void write_block_list_tail_end (const char *path,erofs_blk_t nblocks, +bool inline_data,erofs_blk_t blkaddr)
This makes me wondering...
1) how about passing in struct erofs_inode *inode directly instead?
2) how about getting rid of inline_data boolean and using blkaddr != NULL_ADDR to replace such case, e.g.
Quoted 5 lines
lib/block_list.c line 73
Quoted 5 lines
lib/block_list.c new file +5
-0+{ +if (!block_list_fp || !cfg.mount_point) +return ; + +if (!nblocks && !inline_data) {
if (!nblocks && blkaddr != NULL_ADDR) {Quoted 74 lines
Quoted 74 lines
lib/block_list.c new file +13
-0+erofs_dbg ("%s : only tail-end non-inline data" , path); +write_block_list (path, blkaddr,1 , false); + return; + } + +if (nblocks) { +if (!inline_data) +fprintf (block_list_fp," %u" , blkaddr); + +fprintf (block_list_fp,"\n" ); + } +} +#endif lib/compress.c +8
-0@@ -19,6 +19,10 @@#include "erofs/compress.h" #include "compressor.h" +#ifdef WITH_ANDROID +#include "erofs/block_list.h" +#endif +static struct erofs_compress compresshandle;static int compressionlevel;@@ -553,6 +557,10 @@ int erofs_write_compressed_file(struct erofs_inode *inode) inode->i_srcpath, (unsigned long long )inode->i_size, compressed_blocks); +#ifdef WITH_ANDROID +write_block_list (inode->i_srcpath, blkaddr, compressed_blocks, false); +#endif +/* * TODO: need to move erofs_bdrop to erofs_write_tail_end * when both mkfs & kernel support compression inline. lib/inode.c +11
-1@@ -22,6 +22,10 @@#include "erofs/xattr.h" #include "erofs/exclude.h" +#ifdef WITH_ANDROID +#include "erofs/block_list.h" +#endif +#define S_SHIFT 12static unsigned char erofs_ftype_by_mode[S_IFMT >>S_SHIFT ] = { [S_IFREG >>S_SHIFT ] =EROFS_FT_REG_FILE ,@@ -369,6 +373,12 @@ static int write_uncompressed_file_from_fd(struct erofs_inode *inode, int fd)return -EIO ; } } + +#ifdef WITH_ANDROID +if (nblocks) +write_block_list (inode->i_srcpath, inode->u.i_blkaddr, + nblocks, inode->idata_size ? true : false); +#endif return 0 ; }@@ -626,6 +636,7 @@ static struct erofs_bhops erofs_write_inline_bhops = {int erofs_write_tail_end (struct erofs_inode *inode) {struct erofs_buffer_head *bh, *ibh; +erofs_off_t pos; bh = inode->bh_data;@@ -640,7 +651,6 @@ int erofs_write_tail_end(struct erofs_inode *inode) ibh->op = &erofs_write_inline_bhops; }else {int ret; -erofs_off_t pos;erofs_mapbh (bh->block);
Quoted 8 lines
lib/inode.c int erofs_write_tail_end(struct erofs_inode *inode) · line 673
Quoted 8 lines
lib/inode.c +3
-0pos =erofs_btell (bh, true) -EROFS_BLKSIZ ;@@ -658,6 +668,15 @@ int erofs_write_tail_end(struct erofs_inode *inode)free (inode->idata); inode->idata =NULL ; } + +#ifdef WITH_ANDROID +if (!S_ISDIR (inode->i_mode) && !S_ISLNK (inode->i_mode))
hmmm... that seems some hacky, but I can leave it as-is for now. but how about moving it into erofs_droid_blocklist_write_tail_end()?
Quoted 4 lines
lib/inode.c int erofs_write_tail_end(struct erofs_inode *inode) · line 677
Quoted 4 lines
lib/inode.c +4
-0+write_block_list_tail_end (inode->i_srcpath, + inode->i_size /EROFS_BLKSIZ , + inode->bh_inline ? true: false, +erofs_blknr (pos));
erofs_droid_blocklist_write_tail_end(inode, inode->bh_inline ? NULL_ADDR : erofs_blknr(pos));
I will send a rough delta patch later, please check it out...
Thanks,
Gao Xiang
Quoted 65 lines
Quoted 65 lines
lib/inode.c +2
-0+#endif + out:/* now bh_data can drop directly */ if (bh) {mkfs/main.c +17
-0@@ -27,6 +27,10 @@#include <uuid.h> #endif +#ifdef WITH_ANDROID +#include "erofs/block_list.h" +#endif +#define EROFS_SUPER_END (EROFS_SUPER_OFFSET + sizeof(struct erofs_super_block))static struct option long_options[] = {@@ -47,6 +51,7 @@ static struct option long_options[] = { {"mount-point", required_argument, NULL, 10}, {"product-out", required_argument, NULL, 11}, {"fs-config-file", required_argument, NULL, 12}, + {"block-list-file", required_argument, NULL, 13}, #endif {0, 0, 0, 0}, };@@ -95,6 +100,7 @@ static void usage(void)" --mount-point=X X=prefix of target fs path (default: /)\n" " --product-out=X X=product_out directory\n" " --fs-config-file=X X=fs_config file\n" + " --block-list-file=X X=block_list file\n"#endif "\nAvailable compressors are: ", stderr);print_available_compressors (stderr,", " );@@ -293,6 +299,9 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])case 12 : cfg.fs_config_file = optarg;break ; +case 13 : + cfg.block_list_file = optarg; +break ;#endif case 'C' : i =strtoull (optarg, &endptr,0 );@@ -541,6 +550,11 @@ int main(int argc, char **argv)erofs_err ("failed to load fs config %s" , cfg.fs_config_file); return 1; } + +if (cfg.block_list_file &&block_list_fopen () <0 ) { +erofs_err ("failed to open %s" , cfg.block_list_file); +return 1 ; + }#endif erofs_show_config ();@@ -607,6 +621,9 @@ int main(int argc, char **argv) err =erofs_mkfs_superblock_csum_set (); exit:z_erofs_compress_exit (); +#ifdef WITH_ANDROID +block_list_fclose (); +#endif dev_close ();erofs_cleanup_exclude_rules ();erofs_exit_configure ();
PATCH v2 1/1 AOSP: erofs-utils: add block list support
In reply to Yue HuFrom: Yue Hu <[email protected]>
Android update engine will treat EROFS filesystem image as one single file. Let's add block list support to optimize OTA size.
---
some cleanups.
Hi all,
please kindly check it again if it works since I dont have such environment. If it doesn't work, please help fix it..
Thanks,
Gao Xiang
include/erofs/block_list.h | 27 +++++++++++++++ include/erofs/config.h | 1 + lib/block_list.c | 86 ++++++++++++++++++++++++++++++++++++++++++++++ lib/compress.c | 2 ++ lib/inode.c | 6 ++++ mkfs/main.c | 14 ++++++++ 6 files changed, 136 insertions(+) create mode 100644 include/erofs/block_list.h create mode 100644 lib/block_list.c
include/erofs/block_list.h new file +27 -0
@@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* +* erofs-utils/include/erofs/block_list.h +* +* Copyright (C), 2021, Coolpad Group Limited. +* Created by Yue Hu <[email protected]> +*/ +#ifndef __EROFS_BLOCK_LIST_H +#define __EROFS_BLOCK_LIST_H + +#include "internal.h" + +#ifdef WITH_ANDROID +int erofs_droid_blocklist_fopen (void ); +void erofs_droid_blocklist_fclose (void ); +void erofs_droid_blocklist_write (struct erofs_inode *inode, +erofs_blk_t blk_start,erofs_blk_t nblocks); +void erofs_droid_blocklist_write_tail_end (struct erofs_inode *inode, +erofs_blk_t blkaddr); +#else +static inline void erofs_droid_blocklist_write (struct erofs_inode *inode, +erofs_blk_t blk_start,erofs_blk_t nblocks) {} +static inline +void erofs_droid_blocklist_write_tail_end (struct erofs_inode *inode, +erofs_blk_t blkaddr) {} +#endif +#endif
include/erofs/config.h +1 -0
@@ -65,6 +65,7 @@ struct erofs_configure {char *mount_point;char *target_out_path;char *fs_config_file; +char *block_list_file;#endif };
lib/block_list.c new file +86 -0
86 lines not on this page · raw message
lib/compress.c +2 -0
@@ -18,6 +18,7 @@#include "erofs/cache.h" #include "erofs/compress.h" #include "compressor.h" +#include "erofs/block_list.h" static struct erofs_compress compresshandle;static int compressionlevel;
@@ -571,6 +572,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode)DBG_BUGON (ret); } inode->compressmeta = compressmeta; +erofs_droid_blocklist_write (inode, blkaddr, compressed_blocks);return 0 ; err_bdrop:
lib/inode.c +6 -0
@@ -21,6 +21,7 @@#include "erofs/compress.h" #include "erofs/xattr.h" #include "erofs/exclude.h" +#include "erofs/block_list.h" #define S_SHIFT 12static unsigned char erofs_ftype_by_mode[S_IFMT >>S_SHIFT ] = {
@@ -369,6 +370,7 @@ static int write_uncompressed_file_from_fd(struct erofs_inode *inode, int fd)return -EIO ; } } +erofs_droid_blocklist_write (inode, inode->u.i_blkaddr, nblocks);return 0 ; }
@@ -638,6 +640,8 @@ int erofs_write_tail_end(struct erofs_inode *inode) ibh->fsprivate =erofs_igrab (inode); ibh->op = &erofs_write_inline_bhops; + +erofs_droid_blocklist_write_tail_end (inode,NULL_ADDR ); }else {int ret;erofs_off_t pos;
@@ -657,6 +661,8 @@ int erofs_write_tail_end(struct erofs_inode *inode) inode->idata_size =0 ;free (inode->idata); inode->idata =NULL ; + +erofs_droid_blocklist_write_tail_end (inode,erofs_blknr (pos)); } out:/* now bh_data can drop directly */
mkfs/main.c +14 -0
@@ -22,6 +22,7 @@#include "erofs/compress.h" #include "erofs/xattr.h" #include "erofs/exclude.h" +#include "erofs/block_list.h" #ifdef HAVE_LIBUUID #include <uuid.h>
@@ -47,6 +48,7 @@ static struct option long_options[] = { {"mount-point", required_argument, NULL, 10}, {"product-out", required_argument, NULL, 11}, {"fs-config-file", required_argument, NULL, 12}, + {"block-list-file", required_argument, NULL, 13}, #endif {0, 0, 0, 0}, };
@@ -95,6 +97,7 @@ static void usage(void)" --mount-point=X X=prefix of target fs path (default: /)\n" " --product-out=X X=product_out directory\n" " --fs-config-file=X X=fs_config file\n" + " --block-list-file=X X=block_list file\n"#endif "\nAvailable compressors are: ", stderr);print_available_compressors (stderr,", " );
@@ -293,6 +296,9 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])case 12 : cfg.fs_config_file = optarg;break ; +case 13 : + cfg.block_list_file = optarg; +break ;#endif case 'C' : i =strtoull (optarg, &endptr,0 );
@@ -541,6 +547,11 @@ int main(int argc, char **argv)erofs_err ("failed to load fs config %s" , cfg.fs_config_file); return 1; } + +if (cfg.block_list_file &&erofs_droid_blocklist_fopen () <0 ) { +erofs_err ("failed to open %s" , cfg.block_list_file); +return 1 ; + }#endif erofs_show_config ();
@@ -607,6 +618,9 @@ int main(int argc, char **argv) err =erofs_mkfs_superblock_csum_set (); exit:z_erofs_compress_exit (); +#ifdef WITH_ANDROID +erofs_droid_blocklist_fclose (); +#endif dev_close ();erofs_cleanup_exclude_rules ();erofs_exit_configure ();
Signature · 1 line
1.8.3.1
Quoted 123 lines · On Thu, 24 Jun 2021 01:31:29 +0800 Gao Xiang <[email protected]>…
From: Yue Hu <[email protected]>
Android update engine will treat EROFS filesystem image as one single file. Let's add block list support to optimize OTA size.
Signed-off-by Yue Hu <[email protected]>Signed-off-by Gao Xiang <[email protected]>---
some cleanups.Hi all,
please kindly check it again if it works since I dont have such environment. If it doesn't work, please help fix it..Thanks,
Gao Xianginclude/erofs/block_list.h | 27 +++++++++++++++ include/erofs/config.h | 1 + lib/block_list.c | 86 ++++++++++++++++++++++++++++++++++++++++++++++ lib/compress.c | 2 ++ lib/inode.c | 6 ++++ mkfs/main.c | 14 ++++++++ 6 files changed, 136 insertions(+) create mode 100644 include/erofs/block_list.h create mode 100644 lib/block_list.cinclude/erofs/block_list.h new file +27
-0@@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* +* erofs-utils/include/erofs/block_list.h +* +* Copyright (C), 2021, Coolpad Group Limited. +* Created by Yue Hu <[email protected]> +*/ +#ifndef __EROFS_BLOCK_LIST_H +#define __EROFS_BLOCK_LIST_H + +#include "internal.h" + +#ifdef WITH_ANDROID +int erofs_droid_blocklist_fopen (void ); +void erofs_droid_blocklist_fclose (void ); +void erofs_droid_blocklist_write (struct erofs_inode *inode, +erofs_blk_t blk_start,erofs_blk_t nblocks); +void erofs_droid_blocklist_write_tail_end (struct erofs_inode *inode, +erofs_blk_t blkaddr); +#else +static inline void erofs_droid_blocklist_write (struct erofs_inode *inode, +erofs_blk_t blk_start,erofs_blk_t nblocks) {} +static inline +void erofs_droid_blocklist_write_tail_end (struct erofs_inode *inode, +erofs_blk_t blkaddr) {} +#endif +#endif include/erofs/config.h +1
-0@@ -65,6 +65,7 @@ struct erofs_configure {char *mount_point;char *target_out_path;char *fs_config_file; +char *block_list_file;#endif };lib/block_list.c new file +45
-0@@ -0,0 +1,86 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* +* erofs-utils/lib/block_list.c +* +* Copyright (C), 2021, Coolpad Group Limited. +* Created by Yue Hu <[email protected]> +*/ +#ifdef WITH_ANDROID +#include <stdio.h> +#include <sys/stat.h> +#include "erofs/block_list.h" + +#define pr_fmt (fmt) "EROFS block_list: " FUNC_LINE_FMT fmt "\n" +#include "erofs/print.h" + +static FILE *block_list_fp =NULL ; + +int erofs_droid_blocklist_fopen (void ) +{ +if (block_list_fp) +return 0 ; + + block_list_fp =fopen (cfg.block_list_file,"w" ); + +if (!block_list_fp) +return -1 ; +return 0 ; +} + +void erofs_droid_blocklist_fclose (void ) +{ +if (!block_list_fp) +return ; + +fclose (block_list_fp); + block_list_fp =NULL ; +} + +static void blocklist_write (const char *path,erofs_blk_t blk_start, +erofs_blk_t nblocks,bool has_tail) +{ +const char *fspath =erofs_fspath (path); + +fprintf (block_list_fp,"/%s" , cfg.mount_point); +
lib/block_list.c line 55
Quoted 10 lines
lib/block_list.c new file +10
-0+if (fspath[0 ] !='/' ) +fprintf (block_list_fp,"/" ); + +if (nblocks ==1 ) +fprintf (block_list_fp,"%s %u" , fspath, blk_start); +else if (nblocks >1 ) +fprintf (block_list_fp,"%s %u-%u" , fspath, blk_start, + blk_start + nblocks -1 ); +else +fprintf (block_list_fp,"%s" , fspath);
Seems the last else branch is not correct, it will only record the fspath without block map due to only inline tail-end data. It's not intention in my v1 patch.
Quoted 149 lines
lib/block_list.c new file +31
-0+ +if (!has_tail) + fprintf(block_list_fp, "\n"); +} + +void erofs_droid_blocklist_write (struct erofs_inode *inode, +erofs_blk_t blk_start,erofs_blk_t nblocks) +{ +if (!block_list_fp || !cfg.mount_point) +return ; + +blocklist_write (inode->i_srcpath, blk_start, nblocks, + !!inode->idata_size); +} + +void erofs_droid_blocklist_write_tail_end (struct erofs_inode *inode, +erofs_blk_t blkaddr) +{ +if (!block_list_fp || !cfg.mount_point) +return ; + +/* XXX: a bit hacky.. may need a better approach */ +if (S_ISDIR (inode->i_mode) ||S_ISLNK (inode->i_mode)) +return ; + +if (blkaddr !=NULL_ADDR ) +fprintf (block_list_fp," %u\n" , blkaddr); +else +fprintf (block_list_fp,"\n" ); +} +#endif lib/compress.c +2
-0@@ -18,6 +18,7 @@#include "erofs/cache.h" #include "erofs/compress.h" #include "compressor.h" +#include "erofs/block_list.h" static struct erofs_compress compresshandle;static int compressionlevel;@@ -571,6 +572,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode)DBG_BUGON (ret); } inode->compressmeta = compressmeta; +erofs_droid_blocklist_write (inode, blkaddr, compressed_blocks);return 0 ; err_bdrop:lib/inode.c +6
-0@@ -21,6 +21,7 @@#include "erofs/compress.h" #include "erofs/xattr.h" #include "erofs/exclude.h" +#include "erofs/block_list.h" #define S_SHIFT 12static unsigned char erofs_ftype_by_mode[S_IFMT >>S_SHIFT ] = {@@ -369,6 +370,7 @@ static int write_uncompressed_file_from_fd(struct erofs_inode *inode, int fd)return -EIO ; } } +erofs_droid_blocklist_write (inode, inode->u.i_blkaddr, nblocks);return 0 ; }@@ -638,6 +640,8 @@ int erofs_write_tail_end(struct erofs_inode *inode) ibh->fsprivate =erofs_igrab (inode); ibh->op = &erofs_write_inline_bhops; + +erofs_droid_blocklist_write_tail_end (inode,NULL_ADDR ); }else {int ret;erofs_off_t pos;@@ -657,6 +661,8 @@ int erofs_write_tail_end(struct erofs_inode *inode) inode->idata_size =0 ;free (inode->idata); inode->idata =NULL ; + +erofs_droid_blocklist_write_tail_end (inode,erofs_blknr (pos)); } out:/* now bh_data can drop directly */ mkfs/main.c +14
-0@@ -22,6 +22,7 @@#include "erofs/compress.h" #include "erofs/xattr.h" #include "erofs/exclude.h" +#include "erofs/block_list.h" #ifdef HAVE_LIBUUID #include <uuid.h> @@ -47,6 +48,7 @@ static struct option long_options[] = { {"mount-point", required_argument, NULL, 10}, {"product-out", required_argument, NULL, 11}, {"fs-config-file", required_argument, NULL, 12}, + {"block-list-file", required_argument, NULL, 13}, #endif {0, 0, 0, 0}, };@@ -95,6 +97,7 @@ static void usage(void)" --mount-point=X X=prefix of target fs path (default: /)\n" " --product-out=X X=product_out directory\n" " --fs-config-file=X X=fs_config file\n" + " --block-list-file=X X=block_list file\n"#endif "\nAvailable compressors are: ", stderr);print_available_compressors (stderr,", " );@@ -293,6 +296,9 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])case 12 : cfg.fs_config_file = optarg;break ; +case 13 : + cfg.block_list_file = optarg; +break ;#endif case 'C' : i =strtoull (optarg, &endptr,0 );@@ -541,6 +547,11 @@ int main(int argc, char **argv)erofs_err ("failed to load fs config %s" , cfg.fs_config_file); return 1; } + +if (cfg.block_list_file &&erofs_droid_blocklist_fopen () <0 ) { +erofs_err ("failed to open %s" , cfg.block_list_file); +return 1 ; + }#endif erofs_show_config ();@@ -607,6 +618,9 @@ int main(int argc, char **argv) err =erofs_mkfs_superblock_csum_set (); exit:z_erofs_compress_exit (); +#ifdef WITH_ANDROID +erofs_droid_blocklist_fclose (); +#endif dev_close ();erofs_cleanup_exclude_rules ();erofs_exit_configure ();
Hi Yue,
Quoted 131 lines · Yue Hu
Quoted 128 lines · On Thu, 24 Jun 2021 01:31:29 +0800 Gao Xiang <[email protected]>…
From: Yue Hu <[email protected]>
Android update engine will treat EROFS filesystem image as one single file. Let's add block list support to optimize OTA size.
Signed-off-by Yue Hu <[email protected]>Signed-off-by Gao Xiang <[email protected]>---
some cleanups.Hi all,
please kindly check it again if it works since I dont have such environment. If it doesn't work, please help fix it..Thanks,
Gao Xianginclude/erofs/block_list.h | 27 +++++++++++++++ include/erofs/config.h | 1 + lib/block_list.c | 86 ++++++++++++++++++++++++++++++++++++++++++++++ lib/compress.c | 2 ++ lib/inode.c | 6 ++++ mkfs/main.c | 14 ++++++++ 6 files changed, 136 insertions(+) create mode 100644 include/erofs/block_list.h create mode 100644 lib/block_list.cinclude/erofs/block_list.h new file +27
-0@@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* +* erofs-utils/include/erofs/block_list.h +* +* Copyright (C), 2021, Coolpad Group Limited. +* Created by Yue Hu <[email protected]> +*/ +#ifndef __EROFS_BLOCK_LIST_H +#define __EROFS_BLOCK_LIST_H + +#include "internal.h" + +#ifdef WITH_ANDROID +int erofs_droid_blocklist_fopen (void ); +void erofs_droid_blocklist_fclose (void ); +void erofs_droid_blocklist_write (struct erofs_inode *inode, +erofs_blk_t blk_start,erofs_blk_t nblocks); +void erofs_droid_blocklist_write_tail_end (struct erofs_inode *inode, +erofs_blk_t blkaddr); +#else +static inline void erofs_droid_blocklist_write (struct erofs_inode *inode, +erofs_blk_t blk_start,erofs_blk_t nblocks) {} +static inline +void erofs_droid_blocklist_write_tail_end (struct erofs_inode *inode, +erofs_blk_t blkaddr) {} +#endif +#endif include/erofs/config.h +1
-0@@ -65,6 +65,7 @@ struct erofs_configure {char *mount_point;char *target_out_path;char *fs_config_file; +char *block_list_file;#endif };lib/block_list.c new file +50
-0@@ -0,0 +1,86 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* +* erofs-utils/lib/block_list.c +* +* Copyright (C), 2021, Coolpad Group Limited. +* Created by Yue Hu <[email protected]> +*/ +#ifdef WITH_ANDROID +#include <stdio.h> +#include <sys/stat.h> +#include "erofs/block_list.h" + +#define pr_fmt (fmt) "EROFS block_list: " FUNC_LINE_FMT fmt "\n" +#include "erofs/print.h" + +static FILE *block_list_fp =NULL ; + +int erofs_droid_blocklist_fopen (void ) +{ +if (block_list_fp) +return 0 ; + + block_list_fp =fopen (cfg.block_list_file,"w" ); + +if (!block_list_fp) +return -1 ; +return 0 ; +} + +void erofs_droid_blocklist_fclose (void ) +{ +if (!block_list_fp) +return ; + +fclose (block_list_fp); + block_list_fp =NULL ; +} + +static void blocklist_write (const char *path,erofs_blk_t blk_start, +erofs_blk_t nblocks,bool has_tail) +{ +const char *fspath =erofs_fspath (path); + +fprintf (block_list_fp,"/%s" , cfg.mount_point); + +if (fspath[0 ] !='/' ) +fprintf (block_list_fp,"/" ); + +if (nblocks ==1 ) +fprintf (block_list_fp,"%s %u" , fspath, blk_start);
Quoted 8 lines
lib/block_list.c line 55
Quoted 5 lines
lib/block_list.c new file +5
-0+else if (nblocks >1 ) +fprintf (block_list_fp,"%s %u-%u" , fspath, blk_start, + blk_start + nblocks -1 ); +else +fprintf (block_list_fp,"%s" , fspath);Seems the last else branch is not correct, it will only record the fspath without block map due to only inline tail-end data. It's not intention in my v1 patch.
Thanks for pointing out... yeah, that is a bit awkward, my intention was to simplify it as much as possible, some better idea for this?
Thanks,
Gao Xiang
Quoted 150 lines
Quoted 150 lines
lib/block_list.c new file +31
-0+ +if (!has_tail) + fprintf(block_list_fp, "\n"); +} + +void erofs_droid_blocklist_write (struct erofs_inode *inode, +erofs_blk_t blk_start,erofs_blk_t nblocks) +{ +if (!block_list_fp || !cfg.mount_point) +return ; + +blocklist_write (inode->i_srcpath, blk_start, nblocks, + !!inode->idata_size); +} + +void erofs_droid_blocklist_write_tail_end (struct erofs_inode *inode, +erofs_blk_t blkaddr) +{ +if (!block_list_fp || !cfg.mount_point) +return ; + +/* XXX: a bit hacky.. may need a better approach */ +if (S_ISDIR (inode->i_mode) ||S_ISLNK (inode->i_mode)) +return ; + +if (blkaddr !=NULL_ADDR ) +fprintf (block_list_fp," %u\n" , blkaddr); +else +fprintf (block_list_fp,"\n" ); +} +#endif lib/compress.c +2
-0@@ -18,6 +18,7 @@#include "erofs/cache.h" #include "erofs/compress.h" #include "compressor.h" +#include "erofs/block_list.h" static struct erofs_compress compresshandle;static int compressionlevel;@@ -571,6 +572,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode)DBG_BUGON (ret); } inode->compressmeta = compressmeta; +erofs_droid_blocklist_write (inode, blkaddr, compressed_blocks);return 0 ; err_bdrop:lib/inode.c +6
-0@@ -21,6 +21,7 @@#include "erofs/compress.h" #include "erofs/xattr.h" #include "erofs/exclude.h" +#include "erofs/block_list.h" #define S_SHIFT 12static unsigned char erofs_ftype_by_mode[S_IFMT >>S_SHIFT ] = {@@ -369,6 +370,7 @@ static int write_uncompressed_file_from_fd(struct erofs_inode *inode, int fd)return -EIO ; } } +erofs_droid_blocklist_write (inode, inode->u.i_blkaddr, nblocks);return 0 ; }@@ -638,6 +640,8 @@ int erofs_write_tail_end(struct erofs_inode *inode) ibh->fsprivate =erofs_igrab (inode); ibh->op = &erofs_write_inline_bhops; + +erofs_droid_blocklist_write_tail_end (inode,NULL_ADDR ); }else {int ret;erofs_off_t pos;@@ -657,6 +661,8 @@ int erofs_write_tail_end(struct erofs_inode *inode) inode->idata_size =0 ;free (inode->idata); inode->idata =NULL ; + +erofs_droid_blocklist_write_tail_end (inode,erofs_blknr (pos)); } out:/* now bh_data can drop directly */ mkfs/main.c +14
-0@@ -22,6 +22,7 @@#include "erofs/compress.h" #include "erofs/xattr.h" #include "erofs/exclude.h" +#include "erofs/block_list.h" #ifdef HAVE_LIBUUID #include <uuid.h> @@ -47,6 +48,7 @@ static struct option long_options[] = { {"mount-point", required_argument, NULL, 10}, {"product-out", required_argument, NULL, 11}, {"fs-config-file", required_argument, NULL, 12}, + {"block-list-file", required_argument, NULL, 13}, #endif {0, 0, 0, 0}, };@@ -95,6 +97,7 @@ static void usage(void)" --mount-point=X X=prefix of target fs path (default: /)\n" " --product-out=X X=product_out directory\n" " --fs-config-file=X X=fs_config file\n" + " --block-list-file=X X=block_list file\n"#endif "\nAvailable compressors are: ", stderr);print_available_compressors (stderr,", " );@@ -293,6 +296,9 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])case 12 : cfg.fs_config_file = optarg;break ; +case 13 : + cfg.block_list_file = optarg; +break ;#endif case 'C' : i =strtoull (optarg, &endptr,0 );@@ -541,6 +547,11 @@ int main(int argc, char **argv)erofs_err ("failed to load fs config %s" , cfg.fs_config_file); return 1; } + +if (cfg.block_list_file &&erofs_droid_blocklist_fopen () <0 ) { +erofs_err ("failed to open %s" , cfg.block_list_file); +return 1 ; + }#endif erofs_show_config ();@@ -607,6 +618,9 @@ int main(int argc, char **argv) err =erofs_mkfs_superblock_csum_set (); exit:z_erofs_compress_exit (); +#ifdef WITH_ANDROID +erofs_droid_blocklist_fclose (); +#endif dev_close ();erofs_cleanup_exclude_rules ();erofs_exit_configure ();
Gao Xiang
There are three condition need to be fixed.
The whole inline file should not be printed anything even *\n* The last inline block should add *\n* to trigger a new line The last block and not inline should be print independently
Quoted 153 lines · Gao Xiang
From: Yue Hu <[email protected]>
Android update engine will treat EROFS filesystem image as one single file. Let's add block list support to optimize OTA size.
Signed-off-by Yue Hu <[email protected]>Signed-off-by Gao Xiang <[email protected]>---
some cleanups.Hi all,
please kindly check it again if it works since I dont have such environment. If it doesn't work, please help fix it..Thanks,
Gao Xianginclude/erofs/block_list.h | 27 +++++++++++++++ include/erofs/config.h | 1 + lib/block_list.c | 86 ++++++++++++++++++++++++++++++++++++++++++++++ lib/compress.c | 2 ++ lib/inode.c | 6 ++++ mkfs/main.c | 14 ++++++++ 6 files changed, 136 insertions(+)create mode 100644
include/erofs/block_list.h
create mode 100644lib/block_list.cinclude/erofs/block_list.h new file +27
-0@@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* +* erofs-utils/include/erofs/block_list.h +* +* Copyright (C), 2021, Coolpad Group Limited. +* Created by Yue Hu <[email protected]> +*/ +#ifndef __EROFS_BLOCK_LIST_H +#define __EROFS_BLOCK_LIST_H + +#include "internal.h" + +#ifdef WITH_ANDROID +int erofs_droid_blocklist_fopen (void ); +void erofs_droid_blocklist_fclose (void ); +void erofs_droid_blocklist_write (struct erofs_inode *inode, +erofs_blk_t blk_start,erofs_blk_t nblocks); +void erofs_droid_blocklist_write_tail_end (struct erofs_inode *inode, +erofs_blk_t blkaddr); +#else +static inline void erofs_droid_blocklist_write (struct erofs_inode *inode, +erofs_blk_t blk_start,erofs_blk_t nblocks) {} +static inline +void erofs_droid_blocklist_write_tail_end (struct erofs_inode *inode, +erofs_blk_t blkaddr) {} +#endif +#endif include/erofs/config.h +1
-0@@ -65,6 +65,7 @@ struct erofs_configure {char *mount_point;char *target_out_path;char *fs_config_file; +char *block_list_file;#endif };lib/block_list.c new file +75
-0@@ -0,0 +1,86 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* +* erofs-utils/lib/block_list.c +* +* Copyright (C), 2021, Coolpad Group Limited. +* Created by Yue Hu <[email protected]> +*/ +#ifdef WITH_ANDROID +#include <stdio.h> +#include <sys/stat.h> +#include "erofs/block_list.h" + +#define pr_fmt (fmt) "EROFS block_list: " FUNC_LINE_FMT fmt "\n" +#include "erofs/print.h" + +static FILE *block_list_fp =NULL ; + +int erofs_droid_blocklist_fopen (void ) +{ +if (block_list_fp) +return 0 ; + + block_list_fp =fopen (cfg.block_list_file,"w" ); + +if (!block_list_fp) +return -1 ; +return 0 ; +} + +void erofs_droid_blocklist_fclose (void ) +{ +if (!block_list_fp) +return ; + +fclose (block_list_fp); + block_list_fp =NULL ; +} + +static void blocklist_write (const char *path,erofs_blk_t blk_start, +erofs_blk_t nblocks,bool has_tail) +{ +const char *fspath =erofs_fspath (path); + +fprintf (block_list_fp,"/%s" , cfg.mount_point); + +if (fspath[0 ] !='/' ) +fprintf (block_list_fp,"/" ); + +if (nblocks ==1 ) +fprintf (block_list_fp,"%s %u" , fspath, blk_start); +else if (nblocks >1 ) +fprintf (block_list_fp,"%s %u-%u" , fspath, blk_start, + blk_start + nblocks -1 ); +else +fprintf (block_list_fp,"%s" , fspath); + +if (!has_tail) +fprintf (block_list_fp,"\n" ); +} + +void erofs_droid_blocklist_write (struct erofs_inode *inode, +erofs_blk_t blk_start,erofs_blk_t nblocks) +{ +if (!block_list_fp || !cfg.mount_point) +return ; + +blocklist_write (inode->i_srcpath, blk_start, nblocks, + !!inode->idata_size); +} + +void erofs_droid_blocklist_write_tail_end (struct erofs_inode *inode, +erofs_blk_t blkaddr) +{ +if (!block_list_fp || !cfg.mount_point) +return ;
lib/block_list.c line 85
Quoted 10 lines
lib/block_list.c new file +10
-0+ +/* XXX: a bit hacky.. may need a better approach */ +if (S_ISDIR (inode->i_mode) ||S_ISLNK (inode->i_mode)) +return ; + +if (blkaddr !=NULL_ADDR ) +fprintf (block_list_fp," %u\n" , blkaddr); +else +fprintf (block_list_fp,"\n" ); +}
void erofs_droid_blocklist_write_tail_end(struct erofs_inode *inode,
erofs_blk_t blkaddr)
{
if (!block_list_fp || !cfg.mount_point)
return;
/* XXX: a bit hacky.. may need a better approach */
if (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode))
return;
if (erofs_blknr(inode->i_size)) { // its block list has been output before
if (blkaddr == NULL_ADDR)
fprintf(block_list_fp, "\n");
else
fprintf(block_list_fp, " %u\n", blkaddr);
return;
}
if (blkaddr != NULL_ADDR)
blocklist_write(inode->i_srcpath, blkaddr, 1, false);
}Quoted 120 lines
lib/block_list.c new file +1
-0+#endif lib/compress.c +2
-0@@ -18,6 +18,7 @@#include "erofs/cache.h" #include "erofs/compress.h" #include "compressor.h" +#include "erofs/block_list.h" static struct erofs_compress compresshandle;static int compressionlevel;@@ -571,6 +572,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode)DBG_BUGON (ret); } inode->compressmeta = compressmeta; +erofs_droid_blocklist_write (inode, blkaddr, compressed_blocks);return 0 ; err_bdrop:lib/inode.c +6
-0@@ -21,6 +21,7 @@#include "erofs/compress.h" #include "erofs/xattr.h" #include "erofs/exclude.h" +#include "erofs/block_list.h" #define S_SHIFT 12static unsigned char erofs_ftype_by_mode[S_IFMT >>S_SHIFT ] = {@@ -369,6 +370,7 @@ static int write_uncompressed_file_from_fd(struct erofs_inode *inode, int fd)return -EIO ; } } +erofs_droid_blocklist_write (inode, inode->u.i_blkaddr, nblocks);return 0 ; }@@ -638,6 +640,8 @@ int erofs_write_tail_end(struct erofs_inode *inode) ibh->fsprivate =erofs_igrab (inode); ibh->op = &erofs_write_inline_bhops; + +erofs_droid_blocklist_write_tail_end (inode,NULL_ADDR ); }else {int ret;erofs_off_t pos;@@ -657,6 +661,8 @@ int erofs_write_tail_end(struct erofs_inode *inode) inode->idata_size =0 ;free (inode->idata); inode->idata =NULL ; + +erofs_droid_blocklist_write_tail_end (inode,erofs_blknr (pos)); } out:/* now bh_data can drop directly */ mkfs/main.c +14
-0@@ -22,6 +22,7 @@#include "erofs/compress.h" #include "erofs/xattr.h" #include "erofs/exclude.h" +#include "erofs/block_list.h" #ifdef HAVE_LIBUUID #include <uuid.h> @@ -47,6 +48,7 @@ static struct option long_options[] = { {"mount-point", required_argument, NULL, 10}, {"product-out", required_argument, NULL, 11}, {"fs-config-file", required_argument, NULL, 12}, + {"block-list-file", required_argument, NULL, 13}, #endif {0, 0, 0, 0}, };@@ -95,6 +97,7 @@ static void usage(void)" --mount-point=X X=prefix of target fs path (default: /)\n" " --product-out=X X=product_out directory\n" " --fs-config-file=X X=fs_config file\n" + " --block-list-file=X X=block_list file\n"#endif "\nAvailable compressors are: ", stderr);print_available_compressors (stderr,", " );@@ -293,6 +296,9 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])case 12 : cfg.fs_config_file = optarg;break ; +case 13 : + cfg.block_list_file = optarg; +break ;#endif case 'C' : i =strtoull (optarg, &endptr,0 );@@ -541,6 +547,11 @@ int main(int argc, char **argv)erofs_err ("failed to load fs config %s" , cfg.fs_config_file); return 1; } + +if (cfg.block_list_file &&erofs_droid_blocklist_fopen () <0 ) { +erofs_err ("failed to open %s" , cfg.block_list_file); +return 1 ; + }#endif erofs_show_config ();@@ -607,6 +618,9 @@ int main(int argc, char **argv) err =erofs_mkfs_superblock_csum_set (); exit:z_erofs_compress_exit (); +#ifdef WITH_ANDROID +erofs_droid_blocklist_fclose (); +#endif dev_close ();erofs_cleanup_exclude_rules ();erofs_exit_configure ();
Quoted 3 lines · Li Guifu
Gao Xiang
There are three condition need to be fixed.
The whole inline file should not be printed anything even *\n*
Yes.
Quoted 1 line
The last inline block should add *\n* to trigger a new line
doesn't it?
Quoted 1 line
The last block and not inline should be print independently
Could you give some output example of this?
btw, do you also have some idea or help fix/improve this patch? I'd like to merge it asap if it gets into shape but I don't have aosp environment at hand.
Quoted 176 lines · Gao Xiang
Quoted 152 lines
From: Yue Hu <[email protected]>
Android update engine will treat EROFS filesystem image as one single file. Let's add block list support to optimize OTA size.
Signed-off-by Yue Hu <[email protected]>Signed-off-by Gao Xiang <[email protected]>---
some cleanups.Hi all,
please kindly check it again if it works since I dont have such environment. If it doesn't work, please help fix it..Thanks,
Gao Xianginclude/erofs/block_list.h | 27 +++++++++++++++ include/erofs/config.h | 1 + lib/block_list.c | 86 ++++++++++++++++++++++++++++++++++++++++++++++ lib/compress.c | 2 ++ lib/inode.c | 6 ++++ mkfs/main.c | 14 ++++++++ 6 files changed, 136 insertions(+)create mode 100644
include/erofs/block_list.h
create mode 100644lib/block_list.cinclude/erofs/block_list.h new file +27
-0@@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* +* erofs-utils/include/erofs/block_list.h +* +* Copyright (C), 2021, Coolpad Group Limited. +* Created by Yue Hu <[email protected]> +*/ +#ifndef __EROFS_BLOCK_LIST_H +#define __EROFS_BLOCK_LIST_H + +#include "internal.h" + +#ifdef WITH_ANDROID +int erofs_droid_blocklist_fopen (void ); +void erofs_droid_blocklist_fclose (void ); +void erofs_droid_blocklist_write (struct erofs_inode *inode, +erofs_blk_t blk_start,erofs_blk_t nblocks); +void erofs_droid_blocklist_write_tail_end (struct erofs_inode *inode, +erofs_blk_t blkaddr); +#else +static inline void erofs_droid_blocklist_write (struct erofs_inode *inode, +erofs_blk_t blk_start,erofs_blk_t nblocks) {} +static inline +void erofs_droid_blocklist_write_tail_end (struct erofs_inode *inode, +erofs_blk_t blkaddr) {} +#endif +#endif include/erofs/config.h +1
-0@@ -65,6 +65,7 @@ struct erofs_configure {char *mount_point;char *target_out_path;char *fs_config_file; +char *block_list_file;#endif };lib/block_list.c new file +75
-0@@ -0,0 +1,86 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* +* erofs-utils/lib/block_list.c +* +* Copyright (C), 2021, Coolpad Group Limited. +* Created by Yue Hu <[email protected]> +*/ +#ifdef WITH_ANDROID +#include <stdio.h> +#include <sys/stat.h> +#include "erofs/block_list.h" + +#define pr_fmt (fmt) "EROFS block_list: " FUNC_LINE_FMT fmt "\n" +#include "erofs/print.h" + +static FILE *block_list_fp =NULL ; + +int erofs_droid_blocklist_fopen (void ) +{ +if (block_list_fp) +return 0 ; + + block_list_fp =fopen (cfg.block_list_file,"w" ); + +if (!block_list_fp) +return -1 ; +return 0 ; +} + +void erofs_droid_blocklist_fclose (void ) +{ +if (!block_list_fp) +return ; + +fclose (block_list_fp); + block_list_fp =NULL ; +} + +static void blocklist_write (const char *path,erofs_blk_t blk_start, +erofs_blk_t nblocks,bool has_tail) +{ +const char *fspath =erofs_fspath (path); + +fprintf (block_list_fp,"/%s" , cfg.mount_point); + +if (fspath[0 ] !='/' ) +fprintf (block_list_fp,"/" ); + +if (nblocks ==1 ) +fprintf (block_list_fp,"%s %u" , fspath, blk_start); +else if (nblocks >1 ) +fprintf (block_list_fp,"%s %u-%u" , fspath, blk_start, + blk_start + nblocks -1 ); +else +fprintf (block_list_fp,"%s" , fspath); + +if (!has_tail) +fprintf (block_list_fp,"\n" ); +} + +void erofs_droid_blocklist_write (struct erofs_inode *inode, +erofs_blk_t blk_start,erofs_blk_t nblocks) +{ +if (!block_list_fp || !cfg.mount_point) +return ; + +blocklist_write (inode->i_srcpath, blk_start, nblocks, + !!inode->idata_size); +} + +void erofs_droid_blocklist_write_tail_end (struct erofs_inode *inode, +erofs_blk_t blkaddr) +{ +if (!block_list_fp || !cfg.mount_point) +return ;lib/block_list.c line 85
Quoted 10 lines
lib/block_list.c new file +10
-0+ +/* XXX: a bit hacky.. may need a better approach */ +if (S_ISDIR (inode->i_mode) ||S_ISLNK (inode->i_mode)) +return ; + +if (blkaddr !=NULL_ADDR ) +fprintf (block_list_fp," %u\n" , blkaddr); +else +fprintf (block_list_fp,"\n" ); +}void erofs_droid_blocklist_write_tail_end(struct
erofs_inode*inode,
erofs_blk_tblkaddr){ if (!block_list_fp || !cfg.mount_point) return; /* XXX: a bit hacky.. may need a better approach */ if (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode)) return; if (erofs_blknr(inode->i_size)) { // its block list has been output before
Quoted 8 lines
if (blkaddr == NULL_ADDR) fprintf(block_list_fp, "\n"); else fprintf(block_list_fp, " %u\n", blkaddr); return; } if (blkaddr != NULL_ADDR) blocklist_write(inode->i_srcpath, blkaddr, 1, false);
That still sounds a bit hacky. But if no better idea, we could leave it as-is... honestly, I don't quite like this.
Thanks,
Gao Xiang
Quoted 116 lines
}
Quoted 113 lines
lib/block_list.c new file +1
-0+#endif lib/compress.c +2
-0@@ -18,6 +18,7 @@#include "erofs/cache.h" #include "erofs/compress.h" #include "compressor.h" +#include "erofs/block_list.h" static struct erofs_compress compresshandle;static int compressionlevel;@@ -571,6 +572,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode)DBG_BUGON (ret); } inode->compressmeta = compressmeta; +erofs_droid_blocklist_write (inode, blkaddr, compressed_blocks);return 0 ; err_bdrop:lib/inode.c +6
-0@@ -21,6 +21,7 @@#include "erofs/compress.h" #include "erofs/xattr.h" #include "erofs/exclude.h" +#include "erofs/block_list.h" #define S_SHIFT 12static unsigned char erofs_ftype_by_mode[S_IFMT >>S_SHIFT ] = {@@ -369,6 +370,7 @@ static int write_uncompressed_file_from_fd(struct erofs_inode *inode, int fd)return -EIO ; } } +erofs_droid_blocklist_write (inode, inode->u.i_blkaddr, nblocks);return 0 ; }@@ -638,6 +640,8 @@ int erofs_write_tail_end(struct erofs_inode *inode) ibh->fsprivate =erofs_igrab (inode); ibh->op = &erofs_write_inline_bhops; + +erofs_droid_blocklist_write_tail_end (inode,NULL_ADDR ); }else {int ret;erofs_off_t pos;@@ -657,6 +661,8 @@ int erofs_write_tail_end(struct erofs_inode *inode) inode->idata_size =0 ;free (inode->idata); inode->idata =NULL ; + +erofs_droid_blocklist_write_tail_end (inode,erofs_blknr (pos)); } out:/* now bh_data can drop directly */ mkfs/main.c +14
-0@@ -22,6 +22,7 @@#include "erofs/compress.h" #include "erofs/xattr.h" #include "erofs/exclude.h" +#include "erofs/block_list.h" #ifdef HAVE_LIBUUID #include <uuid.h> @@ -47,6 +48,7 @@ static struct option long_options[] = { {"mount-point", required_argument, NULL, 10}, {"product-out", required_argument, NULL, 11}, {"fs-config-file", required_argument, NULL, 12}, + {"block-list-file", required_argument, NULL, 13}, #endif {0, 0, 0, 0}, };@@ -95,6 +97,7 @@ static void usage(void)" --mount-point=X X=prefix of target fs path (default: /)\n" " --product-out=X X=product_out directory\n" " --fs-config-file=X X=fs_config file\n" + " --block-list-file=X X=block_list file\n"#endif "\nAvailable compressors are: ", stderr);print_available_compressors (stderr,", " );@@ -293,6 +296,9 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])case 12 : cfg.fs_config_file = optarg;break ; +case 13 : + cfg.block_list_file = optarg; +break ;#endif case 'C' : i =strtoull (optarg, &endptr,0 );@@ -541,6 +547,11 @@ int main(int argc, char **argv)erofs_err ("failed to load fs config %s" , cfg.fs_config_file); return 1; } + +if (cfg.block_list_file &&erofs_droid_blocklist_fopen () <0 ) { +erofs_err ("failed to open %s" , cfg.block_list_file); +return 1 ; + }#endif erofs_show_config ();@@ -607,6 +618,9 @@ int main(int argc, char **argv) err =erofs_mkfs_superblock_csum_set (); exit:z_erofs_compress_exit (); +#ifdef WITH_ANDROID +erofs_droid_blocklist_fclose (); +#endif dev_close ();erofs_cleanup_exclude_rules ();erofs_exit_configure ();
Quoted 9 lines · On Thu, 24 Jun 2021 13:37:53 +0800 Gao Xiang <[email protected]>…
Quoted 3 lines · Li Guifu
Gao Xiang
There are three condition need to be fixed.
The whole inline file should not be printed anything even *\n*Yes.
Quoted 1 line
The last inline block should add *\n* to trigger a new line
Quoted 7 lines
doesn't it?
Quoted 1 line
The last block and not inline should be print independently
Could you give some output example of this?
btw, do you also have some idea or help fix/improve this patch?
Gao Xiang,
I'm improving the patch based on v2 and some suggestions from Guifu. Of course, i will test it firstly then send it out.
Thanks.
Quoted 310 lines
I'd like to merge it asap if it gets into shape but I don't have aosp environment at hand.
Quoted 176 lines · Gao Xiang
Quoted 152 lines
From: Yue Hu <[email protected]>
Android update engine will treat EROFS filesystem image as one single file. Let's add block list support to optimize OTA size.
Signed-off-by Yue Hu <[email protected]>Signed-off-by Gao Xiang <[email protected]>---
some cleanups.Hi all,
please kindly check it again if it works since I dont have such environment. If it doesn't work, please help fix it..Thanks,
Gao Xianginclude/erofs/block_list.h | 27 +++++++++++++++ include/erofs/config.h | 1 + lib/block_list.c | 86 ++++++++++++++++++++++++++++++++++++++++++++++ lib/compress.c | 2 ++ lib/inode.c | 6 ++++ mkfs/main.c | 14 ++++++++ 6 files changed, 136 insertions(+)create mode 100644
include/erofs/block_list.h
create mode 100644lib/block_list.cinclude/erofs/block_list.h new file +27
-0@@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* +* erofs-utils/include/erofs/block_list.h +* +* Copyright (C), 2021, Coolpad Group Limited. +* Created by Yue Hu <[email protected]> +*/ +#ifndef __EROFS_BLOCK_LIST_H +#define __EROFS_BLOCK_LIST_H + +#include "internal.h" + +#ifdef WITH_ANDROID +int erofs_droid_blocklist_fopen (void ); +void erofs_droid_blocklist_fclose (void ); +void erofs_droid_blocklist_write (struct erofs_inode *inode, +erofs_blk_t blk_start,erofs_blk_t nblocks); +void erofs_droid_blocklist_write_tail_end (struct erofs_inode *inode, +erofs_blk_t blkaddr); +#else +static inline void erofs_droid_blocklist_write (struct erofs_inode *inode, +erofs_blk_t blk_start,erofs_blk_t nblocks) {} +static inline +void erofs_droid_blocklist_write_tail_end (struct erofs_inode *inode, +erofs_blk_t blkaddr) {} +#endif +#endif include/erofs/config.h +1
-0@@ -65,6 +65,7 @@ struct erofs_configure {char *mount_point;char *target_out_path;char *fs_config_file; +char *block_list_file;#endif };lib/block_list.c new file +75
-0@@ -0,0 +1,86 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* +* erofs-utils/lib/block_list.c +* +* Copyright (C), 2021, Coolpad Group Limited. +* Created by Yue Hu <[email protected]> +*/ +#ifdef WITH_ANDROID +#include <stdio.h> +#include <sys/stat.h> +#include "erofs/block_list.h" + +#define pr_fmt (fmt) "EROFS block_list: " FUNC_LINE_FMT fmt "\n" +#include "erofs/print.h" + +static FILE *block_list_fp =NULL ; + +int erofs_droid_blocklist_fopen (void ) +{ +if (block_list_fp) +return 0 ; + + block_list_fp =fopen (cfg.block_list_file,"w" ); + +if (!block_list_fp) +return -1 ; +return 0 ; +} + +void erofs_droid_blocklist_fclose (void ) +{ +if (!block_list_fp) +return ; + +fclose (block_list_fp); + block_list_fp =NULL ; +} + +static void blocklist_write (const char *path,erofs_blk_t blk_start, +erofs_blk_t nblocks,bool has_tail) +{ +const char *fspath =erofs_fspath (path); + +fprintf (block_list_fp,"/%s" , cfg.mount_point); + +if (fspath[0 ] !='/' ) +fprintf (block_list_fp,"/" ); + +if (nblocks ==1 ) +fprintf (block_list_fp,"%s %u" , fspath, blk_start); +else if (nblocks >1 ) +fprintf (block_list_fp,"%s %u-%u" , fspath, blk_start, + blk_start + nblocks -1 ); +else +fprintf (block_list_fp,"%s" , fspath); + +if (!has_tail) +fprintf (block_list_fp,"\n" ); +} + +void erofs_droid_blocklist_write (struct erofs_inode *inode, +erofs_blk_t blk_start,erofs_blk_t nblocks) +{ +if (!block_list_fp || !cfg.mount_point) +return ; + +blocklist_write (inode->i_srcpath, blk_start, nblocks, + !!inode->idata_size); +} + +void erofs_droid_blocklist_write_tail_end (struct erofs_inode *inode, +erofs_blk_t blkaddr) +{ +if (!block_list_fp || !cfg.mount_point) +return ;lib/block_list.c line 85
Quoted 10 lines
lib/block_list.c new file +10
-0+ +/* XXX: a bit hacky.. may need a better approach */ +if (S_ISDIR (inode->i_mode) ||S_ISLNK (inode->i_mode)) +return ; + +if (blkaddr !=NULL_ADDR ) +fprintf (block_list_fp," %u\n" , blkaddr); +else +fprintf (block_list_fp,"\n" ); +}void erofs_droid_blocklist_write_tail_end(struct
erofs_inode*inode,
erofs_blk_tblkaddr){ if (!block_list_fp || !cfg.mount_point) return; /* XXX: a bit hacky.. may need a better approach */ if (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode)) return; if (erofs_blknr(inode->i_size)) { // its block list has been output beforeQuoted 8 lines
if (blkaddr == NULL_ADDR) fprintf(block_list_fp, "\n"); else fprintf(block_list_fp, " %u\n", blkaddr); return; } if (blkaddr != NULL_ADDR) blocklist_write(inode->i_srcpath, blkaddr, 1, false);That still sounds a bit hacky. But if no better idea, we could leave it as-is... honestly, I don't quite like this.
Thanks,
Gao XiangQuoted 116 lines
}
Quoted 113 lines
lib/block_list.c new file +1
-0+#endif lib/compress.c +2
-0@@ -18,6 +18,7 @@#include "erofs/cache.h" #include "erofs/compress.h" #include "compressor.h" +#include "erofs/block_list.h" static struct erofs_compress compresshandle;static int compressionlevel;@@ -571,6 +572,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode)DBG_BUGON (ret); } inode->compressmeta = compressmeta; +erofs_droid_blocklist_write (inode, blkaddr, compressed_blocks);return 0 ; err_bdrop:lib/inode.c +6
-0@@ -21,6 +21,7 @@#include "erofs/compress.h" #include "erofs/xattr.h" #include "erofs/exclude.h" +#include "erofs/block_list.h" #define S_SHIFT 12static unsigned char erofs_ftype_by_mode[S_IFMT >>S_SHIFT ] = {@@ -369,6 +370,7 @@ static int write_uncompressed_file_from_fd(struct erofs_inode *inode, int fd)return -EIO ; } } +erofs_droid_blocklist_write (inode, inode->u.i_blkaddr, nblocks);return 0 ; }@@ -638,6 +640,8 @@ int erofs_write_tail_end(struct erofs_inode *inode) ibh->fsprivate =erofs_igrab (inode); ibh->op = &erofs_write_inline_bhops; + +erofs_droid_blocklist_write_tail_end (inode,NULL_ADDR ); }else {int ret;erofs_off_t pos;@@ -657,6 +661,8 @@ int erofs_write_tail_end(struct erofs_inode *inode) inode->idata_size =0 ;free (inode->idata); inode->idata =NULL ; + +erofs_droid_blocklist_write_tail_end (inode,erofs_blknr (pos)); } out:/* now bh_data can drop directly */ mkfs/main.c +14
-0@@ -22,6 +22,7 @@#include "erofs/compress.h" #include "erofs/xattr.h" #include "erofs/exclude.h" +#include "erofs/block_list.h" #ifdef HAVE_LIBUUID #include <uuid.h> @@ -47,6 +48,7 @@ static struct option long_options[] = { {"mount-point", required_argument, NULL, 10}, {"product-out", required_argument, NULL, 11}, {"fs-config-file", required_argument, NULL, 12}, + {"block-list-file", required_argument, NULL, 13}, #endif {0, 0, 0, 0}, };@@ -95,6 +97,7 @@ static void usage(void)" --mount-point=X X=prefix of target fs path (default: /)\n" " --product-out=X X=product_out directory\n" " --fs-config-file=X X=fs_config file\n" + " --block-list-file=X X=block_list file\n"#endif "\nAvailable compressors are: ", stderr);print_available_compressors (stderr,", " );@@ -293,6 +296,9 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])case 12 : cfg.fs_config_file = optarg;break ; +case 13 : + cfg.block_list_file = optarg; +break ;#endif case 'C' : i =strtoull (optarg, &endptr,0 );@@ -541,6 +547,11 @@ int main(int argc, char **argv)erofs_err ("failed to load fs config %s" , cfg.fs_config_file); return 1; } + +if (cfg.block_list_file &&erofs_droid_blocklist_fopen () <0 ) { +erofs_err ("failed to open %s" , cfg.block_list_file); +return 1 ; + }#endif erofs_show_config ();@@ -607,6 +618,9 @@ int main(int argc, char **argv) err =erofs_mkfs_superblock_csum_set (); exit:z_erofs_compress_exit (); +#ifdef WITH_ANDROID +erofs_droid_blocklist_fclose (); +#endif dev_close ();erofs_cleanup_exclude_rules ();erofs_exit_configure ();
PATCH v3 1/1 AOSP: erofs-utils: add block list support
In reply to Gao XiangFrom: Yue Hu <[email protected]>
Android update engine will treat EROFS filesystem image as one single file. Let's add block list support to optimize OTA size.
--- include/erofs/block_list.h | 27 +++++++++++ include/erofs/config.h | 1 + lib/block_list.c | 91 ++++++++++++++++++++++++++++++++++++++ lib/compress.c | 2 + lib/inode.c | 6 +++ mkfs/main.c | 14 ++++++ 6 files changed, 141 insertions(+) create mode 100644 include/erofs/block_list.h create mode 100644 lib/block_list.c
include/erofs/block_list.h new file +27 -0
@@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* +* erofs-utils/include/erofs/block_list.h +* +* Copyright (C), 2021, Coolpad Group Limited. +* Created by Yue Hu <[email protected]> +*/ +#ifndef __EROFS_BLOCK_LIST_H +#define __EROFS_BLOCK_LIST_H + +#include "internal.h" + +#ifdef WITH_ANDROID +int erofs_droid_blocklist_fopen (void ); +void erofs_droid_blocklist_fclose (void ); +void erofs_droid_blocklist_write (struct erofs_inode *inode, +erofs_blk_t blk_start,erofs_blk_t nblocks); +void erofs_droid_blocklist_write_tail_end (struct erofs_inode *inode, +erofs_blk_t blkaddr); +#else +static inline void erofs_droid_blocklist_write (struct erofs_inode *inode, +erofs_blk_t blk_start,erofs_blk_t nblocks) {} +static inline +void erofs_droid_blocklist_write_tail_end (struct erofs_inode *inode, +erofs_blk_t blkaddr) {} +#endif +#endif
include/erofs/config.h +1 -0
@@ -65,6 +65,7 @@ struct erofs_configure {char *mount_point;char *target_out_path;char *fs_config_file; +char *block_list_file;#endif };
lib/block_list.c new file +91 -0
91 lines not on this page · raw message
lib/compress.c +2 -0
@@ -18,6 +18,7 @@#include "erofs/cache.h" #include "erofs/compress.h" #include "compressor.h" +#include "erofs/block_list.h" static struct erofs_compress compresshandle;static int compressionlevel;
@@ -571,6 +572,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode)DBG_BUGON (ret); } inode->compressmeta = compressmeta; +erofs_droid_blocklist_write (inode, blkaddr, compressed_blocks);return 0 ; err_bdrop:
lib/inode.c +6 -0
@@ -21,6 +21,7 @@#include "erofs/compress.h" #include "erofs/xattr.h" #include "erofs/exclude.h" +#include "erofs/block_list.h" #define S_SHIFT 12static unsigned char erofs_ftype_by_mode[S_IFMT >>S_SHIFT ] = {
@@ -369,6 +370,7 @@ static int write_uncompressed_file_from_fd(struct erofs_inode *inode, int fd)return -EIO ; } } +erofs_droid_blocklist_write (inode, inode->u.i_blkaddr, nblocks);return 0 ; }
@@ -638,6 +640,8 @@ static int erofs_write_tail_end(struct erofs_inode *inode) ibh->fsprivate =erofs_igrab (inode); ibh->op = &erofs_write_inline_bhops; + +erofs_droid_blocklist_write_tail_end (inode,NULL_ADDR ); }else {int ret;erofs_off_t pos;
@@ -657,6 +661,8 @@ static int erofs_write_tail_end(struct erofs_inode *inode) inode->idata_size =0 ;free (inode->idata); inode->idata =NULL ; + +erofs_droid_blocklist_write_tail_end (inode,erofs_blknr (pos)); } out:/* now bh_data can drop directly */
mkfs/main.c +14 -0
@@ -22,6 +22,7 @@#include "erofs/compress.h" #include "erofs/xattr.h" #include "erofs/exclude.h" +#include "erofs/block_list.h" #ifdef HAVE_LIBUUID #include <uuid.h>
@@ -47,6 +48,7 @@ static struct option long_options[] = { {"mount-point", required_argument, NULL, 10}, {"product-out", required_argument, NULL, 11}, {"fs-config-file", required_argument, NULL, 12}, + {"block-list-file", required_argument, NULL, 13}, #endif {0, 0, 0, 0}, };
@@ -95,6 +97,7 @@ static void usage(void)" --mount-point=X X=prefix of target fs path (default: /)\n" " --product-out=X X=product_out directory\n" " --fs-config-file=X X=fs_config file\n" + " --block-list-file=X X=block_list file\n"#endif "\nAvailable compressors are: ", stderr);print_available_compressors (stderr,", " );
@@ -293,6 +296,9 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])case 12 : cfg.fs_config_file = optarg;break ; +case 13 : + cfg.block_list_file = optarg; +break ;#endif case 'C' : i =strtoull (optarg, &endptr,0 );
@@ -541,6 +547,11 @@ int main(int argc, char **argv)erofs_err ("failed to load fs config %s" , cfg.fs_config_file); return 1; } + +if (cfg.block_list_file &&erofs_droid_blocklist_fopen () <0 ) { +erofs_err ("failed to open %s" , cfg.block_list_file); +return 1 ; + }#endif erofs_show_config ();
@@ -607,6 +618,9 @@ int main(int argc, char **argv) err =erofs_mkfs_superblock_csum_set (); exit:z_erofs_compress_exit (); +#ifdef WITH_ANDROID +erofs_droid_blocklist_fclose (); +#endif dev_close ();erofs_cleanup_exclude_rules ();erofs_exit_configure ();
Signature · 1 line
2.25.1
Quoted 115 lines · On Thu, 24 Jun 2021 02:03:05 -0400 Li GuiFu <[email protected]> wrote:
From: Yue Hu <[email protected]>
Android update engine will treat EROFS filesystem image as one single file. Let's add block list support to optimize OTA size.
Signed-off-by Yue Hu <[email protected]>Signed-off-by Gao Xiang <[email protected]>Signed-off-by Li Guifu <[email protected]>--- include/erofs/block_list.h | 27 +++++++++++ include/erofs/config.h | 1 + lib/block_list.c | 91 ++++++++++++++++++++++++++++++++++++++ lib/compress.c | 2 + lib/inode.c | 6 +++ mkfs/main.c | 14 ++++++ 6 files changed, 141 insertions(+) create mode 100644 include/erofs/block_list.h create mode 100644 lib/block_list.cinclude/erofs/block_list.h new file +27
-0@@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* +* erofs-utils/include/erofs/block_list.h +* +* Copyright (C), 2021, Coolpad Group Limited. +* Created by Yue Hu <[email protected]> +*/ +#ifndef __EROFS_BLOCK_LIST_H +#define __EROFS_BLOCK_LIST_H + +#include "internal.h" + +#ifdef WITH_ANDROID +int erofs_droid_blocklist_fopen (void ); +void erofs_droid_blocklist_fclose (void ); +void erofs_droid_blocklist_write (struct erofs_inode *inode, +erofs_blk_t blk_start,erofs_blk_t nblocks); +void erofs_droid_blocklist_write_tail_end (struct erofs_inode *inode, +erofs_blk_t blkaddr); +#else +static inline void erofs_droid_blocklist_write (struct erofs_inode *inode, +erofs_blk_t blk_start,erofs_blk_t nblocks) {} +static inline +void erofs_droid_blocklist_write_tail_end (struct erofs_inode *inode, +erofs_blk_t blkaddr) {} +#endif +#endif include/erofs/config.h +1
-0@@ -65,6 +65,7 @@ struct erofs_configure {char *mount_point;char *target_out_path;char *fs_config_file; +char *block_list_file;#endif };lib/block_list.c new file +45
-0@@ -0,0 +1,91 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* +* erofs-utils/lib/block_list.c +* +* Copyright (C), 2021, Coolpad Group Limited. +* Created by Yue Hu <[email protected]> +*/ +#ifdef WITH_ANDROID +#include <stdio.h> +#include <sys/stat.h> +#include "erofs/block_list.h" + +#define pr_fmt (fmt) "EROFS block_list: " FUNC_LINE_FMT fmt "\n" +#include "erofs/print.h" + +static FILE *block_list_fp =NULL ; + +int erofs_droid_blocklist_fopen (void ) +{ +if (block_list_fp) +return 0 ; + + block_list_fp =fopen (cfg.block_list_file,"w" ); + +if (!block_list_fp) +return -1 ; +return 0 ; +} + +void erofs_droid_blocklist_fclose (void ) +{ +if (!block_list_fp) +return ; + +fclose (block_list_fp); + block_list_fp =NULL ; +} + +static void blocklist_write (const char *path,erofs_blk_t blk_start, +erofs_blk_t nblocks,bool has_tail) +{ +const char *fspath =erofs_fspath (path); + +fprintf (block_list_fp,"/%s" , cfg.mount_point); +
lib/block_list.c line 55
Quoted 10 lines
lib/block_list.c new file +10
-0+if (fspath[0 ] !='/' ) +fprintf (block_list_fp,"/" ); + +if (nblocks ==1 ) +fprintf (block_list_fp,"%s %u" , fspath, blk_start); +else if (nblocks >1 ) +fprintf (block_list_fp,"%s %u-%u" , fspath, blk_start, + blk_start + nblocks -1 ); +else +fprintf (block_list_fp,"%s" , fspath);
The last else is not needed since nblocks = 0 is not possible.
Quoted 154 lines
lib/block_list.c new file +36
-0+ +if (!has_tail) +fprintf (block_list_fp,"\n" ); +} + +void erofs_droid_blocklist_write (struct erofs_inode *inode, +erofs_blk_t blk_start,erofs_blk_t nblocks) +{ +if (!block_list_fp || !cfg.mount_point || !nblocks) +return ; + +blocklist_write (inode->i_srcpath, blk_start, nblocks, + !!inode->idata_size); +} + +void erofs_droid_blocklist_write_tail_end (struct erofs_inode *inode, +erofs_blk_t blkaddr) +{ +if (!block_list_fp || !cfg.mount_point) +return ; + +/* XXX: a bit hacky.. may need a better approach */ +if (S_ISDIR (inode->i_mode) ||S_ISLNK (inode->i_mode)) +return ; + +if (erofs_blknr (inode->i_size)) {// its block list has been output before +if (blkaddr ==NULL_ADDR ) +fprintf (block_list_fp,"\n" ); +else +fprintf (block_list_fp," %u\n" , blkaddr); +return ; + } +if (blkaddr !=NULL_ADDR ) +blocklist_write (inode->i_srcpath, blkaddr,1 , false); +} +#endif lib/compress.c +2
-0@@ -18,6 +18,7 @@#include "erofs/cache.h" #include "erofs/compress.h" #include "compressor.h" +#include "erofs/block_list.h" static struct erofs_compress compresshandle;static int compressionlevel;@@ -571,6 +572,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode)DBG_BUGON (ret); } inode->compressmeta = compressmeta; +erofs_droid_blocklist_write (inode, blkaddr, compressed_blocks);return 0 ; err_bdrop:lib/inode.c +6
-0@@ -21,6 +21,7 @@#include "erofs/compress.h" #include "erofs/xattr.h" #include "erofs/exclude.h" +#include "erofs/block_list.h" #define S_SHIFT 12static unsigned char erofs_ftype_by_mode[S_IFMT >>S_SHIFT ] = {@@ -369,6 +370,7 @@ static int write_uncompressed_file_from_fd(struct erofs_inode *inode, int fd)return -EIO ; } } +erofs_droid_blocklist_write (inode, inode->u.i_blkaddr, nblocks);return 0 ; }@@ -638,6 +640,8 @@ static int erofs_write_tail_end(struct erofs_inode *inode) ibh->fsprivate =erofs_igrab (inode); ibh->op = &erofs_write_inline_bhops; + +erofs_droid_blocklist_write_tail_end (inode,NULL_ADDR ); }else {int ret;erofs_off_t pos;@@ -657,6 +661,8 @@ static int erofs_write_tail_end(struct erofs_inode *inode) inode->idata_size =0 ;free (inode->idata); inode->idata =NULL ; + +erofs_droid_blocklist_write_tail_end (inode,erofs_blknr (pos)); } out:/* now bh_data can drop directly */ mkfs/main.c +14
-0@@ -22,6 +22,7 @@#include "erofs/compress.h" #include "erofs/xattr.h" #include "erofs/exclude.h" +#include "erofs/block_list.h" #ifdef HAVE_LIBUUID #include <uuid.h> @@ -47,6 +48,7 @@ static struct option long_options[] = { {"mount-point", required_argument, NULL, 10}, {"product-out", required_argument, NULL, 11}, {"fs-config-file", required_argument, NULL, 12}, + {"block-list-file", required_argument, NULL, 13}, #endif {0, 0, 0, 0}, };@@ -95,6 +97,7 @@ static void usage(void)" --mount-point=X X=prefix of target fs path (default: /)\n" " --product-out=X X=product_out directory\n" " --fs-config-file=X X=fs_config file\n" + " --block-list-file=X X=block_list file\n"#endif "\nAvailable compressors are: ", stderr);print_available_compressors (stderr,", " );@@ -293,6 +296,9 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])case 12 : cfg.fs_config_file = optarg;break ; +case 13 : + cfg.block_list_file = optarg; +break ;#endif case 'C' : i =strtoull (optarg, &endptr,0 );@@ -541,6 +547,11 @@ int main(int argc, char **argv)erofs_err ("failed to load fs config %s" , cfg.fs_config_file); return 1; } + +if (cfg.block_list_file &&erofs_droid_blocklist_fopen () <0 ) { +erofs_err ("failed to open %s" , cfg.block_list_file); +return 1 ; + }#endif erofs_show_config ();@@ -607,6 +618,9 @@ int main(int argc, char **argv) err =erofs_mkfs_superblock_csum_set (); exit:z_erofs_compress_exit (); +#ifdef WITH_ANDROID +erofs_droid_blocklist_fclose (); +#endif dev_close ();erofs_cleanup_exclude_rules ();erofs_exit_configure ();
Hu Yue
Thanks to your contribution.
Base test shows it could record block map list correctly. Some codes need be refactored for further ahead.
Quoted 15 lines · Yue Hu
From: Yue Hu <[email protected]>
Android update engine will treat EROFS filesystem image as one single file. Let's add block list support to optimize OTA size.
Change-Id I21d6177dff0ee65d3c57023b102e991d40873f0dSigned-off-by Yue Hu <[email protected]>--- include/erofs/block_list.h | 19 ++++++++++ include/erofs/config.h | 1 + lib/block_list.c | 86 ++++++++++++++++++++++++++++++++++++++++++++++ lib/compress.c | 8 +++++ lib/inode.c | 21 ++++++++++- mkfs/main.c | 17 +++++++++ 6 files changed, 151 insertions(+), 1 deletion(-)
Quoted 2 lines
create mode 100644
include/erofs/block_list.h
create mode 100644lib/block_list.c
step1:
block_list.c/h are new file, please add them to lib/Makefile.am or
it will not be built.
You would also add a Android.mk/bp for android target build
Quoted 60 lines
include/erofs/block_list.h new file +19
-0@@ -0,0 +1,19 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* +* erofs-utils/include/erofs/block_list.h +* +* Copyright (C), 2021, Coolpad Group Limited. +* Created by Yue Hu <[email protected]> +*/ +#ifndef __EROFS_BLOCK_LIST_H +#define __EROFS_BLOCK_LIST_H + +#include "internal.h" + +int block_list_fopen (void ); +void block_list_fclose (void ); +void write_block_list (const char *path,erofs_blk_t blk_start, +erofs_blk_t nblocks,bool has_tail); +void write_block_list_tail_end (const char *path,erofs_blk_t nblocks, +bool inline_data,erofs_blk_t blkaddr); +#endif include/erofs/config.h +1
-0@@ -65,6 +65,7 @@ struct erofs_configure {char *mount_point;char *target_out_path;char *fs_config_file; +char *block_list_file;#endif };lib/block_list.c new file +16
-0@@ -0,0 +1,86 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* +* erofs-utils/lib/block_list.c +* +* Copyright (C), 2021, Coolpad Group Limited. +* Created by Yue Hu <[email protected]> +*/ +#ifdef WITH_ANDROID +#include <stdio.h> + +#include "erofs/block_list.h" + +#define pr_fmt (fmt) "EROFS block_list: " FUNC_LINE_FMT fmt "\n" +#include "erofs/print.h" + +static FILE *block_list_fp =NULL ;
lib/block_list.c line 26
Quoted 10 lines
lib/block_list.c new file +10
-0+ +int block_list_fopen (void ) +{ +if (block_list_fp) +return 0 ; + + block_list_fp =fopen (cfg.block_list_file,"w" ); + +if (block_list_fp ==NULL ) +return -1 ;
step2:
The return code would be replace with errno please. So further error message could be showed up.
Quoted 35 lines
lib/block_list.c new file +35
-0+ +return 0 ; +} + +void block_list_fclose (void ) +{ +if (block_list_fp) { +fclose (block_list_fp); + block_list_fp =NULL ; + } +} + +void write_block_list (const char *path,erofs_blk_t blk_start, +erofs_blk_t nblocks,bool has_tail) +{ +const char *fspath =erofs_fspath (path); + +if (!block_list_fp || !cfg.mount_point) +return ; + +/* only tail-end data */ +if (!nblocks) +return ; + +fprintf (block_list_fp,"/%s" , cfg.mount_point); + +if (fspath[0 ] !='/' ) +fprintf (block_list_fp,"/" ); + +if (nblocks ==1 ) { +fprintf (block_list_fp,"%s %u" , fspath, blk_start); + }else { +fprintf (block_list_fp,"%s %u-%u" , fspath, blk_start, + blk_start + nblocks -1 ); + }
lib/block_list.c line 71
Quoted 10 lines
lib/block_list.c new file +10
-0+ +if (!has_tail) +fprintf (block_list_fp,"\n" ); +} + +void write_block_list_tail_end (const char *path,erofs_blk_t nblocks, +bool inline_data,erofs_blk_t blkaddr) +{ +if (!block_list_fp || !cfg.mount_point) +return ;
step3:
if (!block_list_fp || !cfg.mount_point)
These codes are double checked at write_block_list() function. Could you do a further optimize ?
Quoted 4 lines
lib/block_list.c new file +4
-0+ +if (!nblocks && !inline_data) { +erofs_dbg ("%s : only tail-end non-inline data" , path); +write_block_list (path, blkaddr,1 , false);
lib/block_list.c line 85
Quoted 10 lines
lib/block_list.c new file +10
-0+return ; + } + +if (nblocks) { +if (!inline_data) +fprintf (block_list_fp," %u" , blkaddr); + +fprintf (block_list_fp,"\n" ); + } +}
step4:
write_block_list_tail_end() has much if-condition nesting, It could return early at the begin of it.
Quoted 72 lines
lib/block_list.c new file +1
-0+#endif lib/compress.c +8
-0@@ -19,6 +19,10 @@#include "erofs/compress.h" #include "compressor.h" +#ifdef WITH_ANDROID +#include "erofs/block_list.h" +#endif +static struct erofs_compress compresshandle;static int compressionlevel;@@ -553,6 +557,10 @@ int erofs_write_compressed_file(struct erofs_inode *inode) inode->i_srcpath, (unsigned long long )inode->i_size, compressed_blocks); +#ifdef WITH_ANDROID +write_block_list (inode->i_srcpath, blkaddr, compressed_blocks, false); +#endif +/* * TODO: need to move erofs_bdrop to erofs_write_tail_end * when both mkfs & kernel support compression inline. lib/inode.c +11
-1@@ -22,6 +22,10 @@#include "erofs/xattr.h" #include "erofs/exclude.h" +#ifdef WITH_ANDROID +#include "erofs/block_list.h" +#endif +#define S_SHIFT 12static unsigned char erofs_ftype_by_mode[S_IFMT >>S_SHIFT ] = { [S_IFREG >>S_SHIFT ] =EROFS_FT_REG_FILE ,@@ -369,6 +373,12 @@ static int write_uncompressed_file_from_fd(struct erofs_inode *inode, int fd)return -EIO ; } } + +#ifdef WITH_ANDROID +if (nblocks) +write_block_list (inode->i_srcpath, inode->u.i_blkaddr, + nblocks, inode->idata_size ? true : false); +#endif return 0 ; }@@ -626,6 +636,7 @@ static struct erofs_bhops erofs_write_inline_bhops = {int erofs_write_tail_end (struct erofs_inode *inode) {struct erofs_buffer_head *bh, *ibh; +erofs_off_t pos; bh = inode->bh_data;@@ -640,7 +651,6 @@ int erofs_write_tail_end(struct erofs_inode *inode) ibh->op = &erofs_write_inline_bhops; }else {int ret; -erofs_off_t pos;erofs_mapbh (bh->block); pos =erofs_btell (bh, true) -EROFS_BLKSIZ ;
lib/inode.c int erofs_write_tail_end(struct erofs_inode *inode) · line 677
Quoted 10 lines
lib/inode.c +7
-0@@ -658,6 +668,15 @@ int erofs_write_tail_end(struct erofs_inode *inode)free (inode->idata); inode->idata =NULL ; } + +#ifdef WITH_ANDROID +if (!S_ISDIR (inode->i_mode) && !S_ISLNK (inode->i_mode)) +write_block_list_tail_end (inode->i_srcpath, + inode->i_size /EROFS_BLKSIZ , + inode->bh_inline ? true: false, +erofs_blknr (pos));
step5:
inode->i_size / EROFS_BLKSIZ,
It could be replace with erofs_blknr() just like pos. The *pos* here would cause build errorly with message, "./include/erofs/internal.h:55:41: error: ‘pos’ may be used
uninitialized in this function [-Werror=maybe-uninitialized]"
Quoted 69 lines
lib/inode.c +2
-0+#endif + out:/* now bh_data can drop directly */ if (bh) {mkfs/main.c +17
-0@@ -27,6 +27,10 @@#include <uuid.h> #endif +#ifdef WITH_ANDROID +#include "erofs/block_list.h" +#endif +#define EROFS_SUPER_END (EROFS_SUPER_OFFSET + sizeof(struct erofs_super_block))static struct option long_options[] = {@@ -47,6 +51,7 @@ static struct option long_options[] = { {"mount-point", required_argument, NULL, 10}, {"product-out", required_argument, NULL, 11}, {"fs-config-file", required_argument, NULL, 12}, + {"block-list-file", required_argument, NULL, 13}, #endif {0, 0, 0, 0}, };@@ -95,6 +100,7 @@ static void usage(void)" --mount-point=X X=prefix of target fs path (default: /)\n" " --product-out=X X=product_out directory\n" " --fs-config-file=X X=fs_config file\n" + " --block-list-file=X X=block_list file\n"#endif "\nAvailable compressors are: ", stderr);print_available_compressors (stderr,", " );@@ -293,6 +299,9 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])case 12 : cfg.fs_config_file = optarg;break ; +case 13 : + cfg.block_list_file = optarg; +break ;#endif case 'C' : i =strtoull (optarg, &endptr,0 );@@ -541,6 +550,11 @@ int main(int argc, char **argv)erofs_err ("failed to load fs config %s" , cfg.fs_config_file); return 1; } + +if (cfg.block_list_file &&block_list_fopen () <0 ) { +erofs_err ("failed to open %s" , cfg.block_list_file); +return 1 ; + }#endif erofs_show_config ();@@ -607,6 +621,9 @@ int main(int argc, char **argv) err =erofs_mkfs_superblock_csum_set (); exit:z_erofs_compress_exit (); +#ifdef WITH_ANDROID +block_list_fclose (); +#endif dev_close ();erofs_cleanup_exclude_rules ();erofs_exit_configure ();
Quoted 5 lines · Li Guifu
Hu Yue
Thanks to your contribution.
Base test shows it could record block map list correctly. Some codes need be refactored for further ahead.
Guifu,
could you check my v2 as well?
https://lore.kernel.org/linux-erofs/[email protected]
does it look good to you?
Thanks,
Gao Xiang