erofs-utils: introduce hashmap from git source
PATCH 1/4 erofs-utils: introduce hashmap from git source
Copied from git source (it's already workable).
--- include/erofs/flex-array.h | 147 +++++++++++++++++++ include/erofs/hashmap.h | 103 ++++++++++++++ lib/Makefile.am | 3 +- lib/hashmap.c | 284 +++++++++++++++++++++++++++++++++++++ 4 files changed, 536 insertions(+), 1 deletion(-) create mode 100644 include/erofs/flex-array.h create mode 100644 include/erofs/hashmap.h create mode 100644 lib/hashmap.c
include/erofs/flex-array.h new file +147 -0
@@ -0,0 +1,147 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __EROFS_FLEX_ARRAY_H +#define __EROFS_FLEX_ARRAY_H + +#include <stdio.h> +#include <stdlib.h> +#include <limits.h> +#include <stdint.h> + +#include "defs.h" +#include "print.h" + +/* +* flex-array.h +* +* Some notes to make sense of the code. +* +* Flex-arrays: +* - Flex-arrays became standard in C99 and are defined by "array[]" (at the +* end of a struct) +* - Pre-C99 flex-arrays can be accomplished by "array[1]" +* - There is a GNU extension where they are defined using "array[0]" +* Allegedly there is/was a bug in gcc whereby foo[1] generated incorrect +* code, so it's safest to use [0] (https://lkml.org/lkml/2015/2/18/407). +* +* For C89 and C90, __STDC__ is 1 +* For later standards, __STDC_VERSION__ is defined according to the standard. +* For example: 199901L or 201112L +* +* Whilst we're on the subject, in version 5 of gcc, the default std was +* changed from gnu89 to gnu11. In jgmenu, CFLAGS therefore contains -std=gnu89 +* You can check your default gcc std by doing: +* gcc -dM -E - </dev/null | grep '__STDC_VERSION__\|__STDC__' +* +* The code below is copied from git's git-compat-util.h in support of +* hashmap.c +*/ + +#ifndef FLEX_ARRAY +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >=199901L ) && \ + (!defined(__SUNPRO_C) || (__SUNPRO_C >0x580 )) +# define FLEX_ARRAY /* empty */ +#elif defined(__GNUC__) +# if (__GNUC__ >=3 ) +# define FLEX_ARRAY /* empty */ +# else +# define FLEX_ARRAY 0/* older GNU extension */ +# endif +#endif + +/* Otherwise, default to safer but a bit wasteful traditional style */ +#ifndef FLEX_ARRAY +# define FLEX_ARRAY 1 +#endif +#endif + +#define bitsizeof (x) (CHAR_BIT * sizeof(x)) + +#define maximum_signed_value_of_type (a) \ + (INTMAX_MAX >> (bitsizeof(intmax_t) - bitsizeof(a))) + +#define maximum_unsigned_value_of_type (a) \ + (UINTMAX_MAX >> (bitsizeof(uintmax_t) - bitsizeof(a))) + +/* +* Signed integer overflow is undefined in C, so here's a helper macro +* to detect if the sum of two integers will overflow. +* Requires: a >= 0, typeof(a) equals typeof(b) +*/ +#define signed_add_overflows (a, b) \ + ((b) > maximum_signed_value_of_type(a) - (a)) + +#define unsigned_add_overflows (a, b) \ + ((b) > maximum_unsigned_value_of_type(a) - (a)) + +static inline size_t st_add (size_t a,size_t b) +{ +if (unsigned_add_overflows (a, b)) { +erofs_err ("size_t overflow: %llu + %llu" , a |0ULL , b |0ULL ); +BUG_ON (1 ); +return -1 ; + } +return a + b; +} + +#define st_add3 (a, b, c) st_add(st_add((a), (b)), (c)) +#define st_add4 (a, b, c, d) st_add(st_add3((a), (b), (c)), (d)) + +/* +* These functions help you allocate structs with flex arrays, and copy +* the data directly into the array. For example, if you had: +* +* struct foo { +* int bar; +* char name[FLEX_ARRAY]; +* }; +* +* you can do: +* +* struct foo *f; +* FLEX_ALLOC_MEM(f, name, src, len); +* +* to allocate a "foo" with the contents of "src" in the "name" field. +* The resulting struct is automatically zero'd, and the flex-array field +* is NUL-terminated (whether the incoming src buffer was or not). +* +* The FLEXPTR_* variants operate on structs that don't use flex-arrays, +* but do want to store a pointer to some extra data in the same allocated +* block. For example, if you have: +* +* struct foo { +* char *name; +* int bar; +* }; +* +* you can do: +* +* struct foo *f; +* FLEXPTR_ALLOC_STR(f, name, src); +* +* and "name" will point to a block of memory after the struct, which will be +* freed along with the struct (but the pointer can be repointed anywhere). +* +* The *_STR variants accept a string parameter rather than a ptr/len +* combination. +* +* Note that these macros will evaluate the first parameter multiple +* times, and it must be assignable as an lvalue. +*/ +#define FLEX_ALLOC_MEM (x, flexname, buf, len) do { \ + size_t flex_array_len_ = (len); \ + (x) = calloc(1, st_add3(sizeof(*(x)), flex_array_len_, 1)); \ + BUG_ON(!(x)); \ + memcpy((void *)(x)->flexname, (buf), flex_array_len_); \ +} while (0) +#define FLEXPTR_ALLOC_MEM (x, ptrname, buf, len) do { \ + size_t flex_array_len_ = (len); \ + (x) = xcalloc(1, st_add3(sizeof(*(x)), flex_array_len_, 1)); \ + memcpy((x) + 1, (buf), flex_array_len_); \ + (x)->ptrname = (void *)((x) + 1); \ +} while (0) +#define FLEX_ALLOC_STR (x, flexname, str) \ + FLEX_ALLOC_MEM((x), flexname, (str), strlen(str)) +#define FLEXPTR_ALLOC_STR (x, ptrname, str) \ + FLEXPTR_ALLOC_MEM((x), ptrname, (str), strlen(str)) + +#endif
include/erofs/hashmap.h new file +103 -0
103 lines not on this page · raw message
lib/Makefile.am +2 -1
@@ -21,7 +21,8 @@ noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \ noinst_HEADERS += compressor.h liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \ - namei.c data.c compress.c compressor.c zmap.c decompress.c + namei.c data.c compress.c compressor.c zmap.c decompress.c \ + hashmap.c liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include if ENABLE_LZ4 liberofs_la_CFLAGS += ${LZ4_CFLAGS}
lib/hashmap.c new file +284 -0
284 lines not on this page · raw message
Signature · 1 line
2.24.4
PATCH 2/4 erofs-utils: introduce sha256
A simple sha256 approach copied from e2fsprogs.
--- lib/Makefile.am | 2 +- lib/sha256.c | 248 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 249 insertions(+), 1 deletion(-) create mode 100644 lib/sha256.c
lib/Makefile.am +1 -1
@@ -22,7 +22,7 @@ noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \ noinst_HEADERS += compressor.h liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \ namei.c data.c compress.c compressor.c zmap.c decompress.c \ - hashmap.c + hashmap.c sha256.c liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include if ENABLE_LZ4 liberofs_la_CFLAGS += ${LZ4_CFLAGS}
lib/sha256.c new file +248 -0
248 lines not on this page · raw message
Signature · 1 line
2.24.4
PATCH 3/4 erofs-utils: introduce copy_file_range
In reply to Gao XiangAdd copy_file_range support. Emulate it instead if libc doesn't support it or have no emulation.
--- configure.ac | 1 + include/erofs/io.h | 5 +++ lib/io.c | 95 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+)
configure.ac +1 -0
@@ -158,6 +158,7 @@ AC_CHECK_DECL(lseek64,[AC_DEFINE(HAVE_LSEEK64_PROTOTYPE, 1, # Checks for library functions. AC_CHECK_FUNCS(m4_flatten([ backtrace + copy_file_range fallocate gettimeofday lgetxattr
include/erofs/io.h +5 -0
@@ -9,6 +9,7 @@#ifndef __EROFS_IO_H#define __EROFS_IO_H +#define _GNU_SOURCE#include <unistd.h> #include "internal.h"
@@ -26,6 +27,10 @@ int dev_fsync(void);int dev_resize (erofs_blk_t nblocks);u64 dev_length (void ); +int erofs_copy_file_range (int fd_in,erofs_off_t *off_in, +int fd_out,erofs_off_t *off_out, +size_t length); +static inline int blk_write (const void *buf,erofs_blk_t blkaddr,u32 nblocks) {
lib/io.c +95 -0
98 lines not on this page · raw message
Signature · 1 line
2.24.4
PATCH 4/4 erofs-utils: support chunk-based uncompressed files
In reply to Gao XiangAdd mkfs support to new chunk-based uncompressed files.
--- include/erofs/blobchunk.h | 18 ++++ include/erofs/config.h | 2 +- include/erofs/internal.h | 1 + include/erofs/io.h | 2 + include/erofs_fs.h | 39 +++++++- lib/Makefile.am | 2 +- lib/blobchunk.c | 200 ++++++++++++++++++++++++++++++++++++++ lib/inode.c | 35 +++++-- lib/io.c | 2 +- mkfs/main.c | 41 +++++++- 10 files changed, 328 insertions(+), 14 deletions(-) create mode 100644 include/erofs/blobchunk.h create mode 100644 lib/blobchunk.c
include/erofs/blobchunk.h new file +18 -0
@@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* +* erofs-utils/lib/blobchunk.h +* +* Copyright (C) 2021, Alibaba Cloud +*/ +#ifndef __EROFS_BLOBCHUNK_H +#define __EROFS_BLOBCHUNK_H + +#include "erofs/internal.h" + +int erofs_blob_write_chunk_indexes (struct erofs_inode *inode,erofs_off_t off); +int erofs_blob_write_chunked_file (struct erofs_inode *inode); +int erofs_blob_remap (void ); +void erofs_blob_exit (void ); +int erofs_blob_init (void ); + +#endif
include/erofs/config.h +1 -1
@@ -44,6 +44,7 @@ struct erofs_configure {bool c_random_pclusterblks;#endif char c_timeinherit; +char c_chunkbits;bool c_noinline_data;#ifdef HAVE_LIBSELINUX
@@ -57,7 +58,6 @@ struct erofs_configure {int c_force_inodeversion;/* < 0, xattr disabled and INT_MAX, always use inline xattrs */ int c_inline_xattr_tolerance; -u32 c_physical_clusterblks;u32 c_max_decompressed_extent_bytes;u64 c_unix_timestamp;
include/erofs/internal.h +1 -0
@@ -160,6 +160,7 @@ struct erofs_inode {union {void *compressmeta; +void *chunkindexes;struct {uint16_t z_advise;uint8_t z_algorithmtype[2 ];
include/erofs/io.h +2 -0
@@ -27,6 +27,8 @@ int dev_fsync(void);int dev_resize (erofs_blk_t nblocks);u64 dev_length (void ); +extern int erofs_devfd; +int erofs_copy_file_range (int fd_in,erofs_off_t *off_in,int fd_out,erofs_off_t *off_out,size_t length);
include/erofs_fs.h +35 -4
@@ -5,7 +5,7 @@* * Copyright (C) 2017-2018 HUAWEI, Inc. * http://www.huawei.com/ -* Created by Gao Xiang <[email protected]> +* Copyright (C) 2021, Alibaba Cloud */ #ifndef __EROFS_FS_H#define __EROFS_FS_H
@@ -22,10 +22,12 @@#define EROFS_FEATURE_INCOMPAT_LZ4_0PADDING 0x00000001#define EROFS_FEATURE_INCOMPAT_COMPR_CFGS 0x00000002#define EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER 0x00000002 +#define EROFS_FEATURE_INCOMPAT_CHUNKED_FILE 0x00000004#define EROFS_ALL_FEATURE_INCOMPAT \ (EROFS_FEATURE_INCOMPAT_LZ4_0PADDING | \ EROFS_FEATURE_INCOMPAT_COMPR_CFGS | \ - EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER) + EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER | \ + EROFS_FEATURE_INCOMPAT_CHUNKED_FILE)#define EROFS_SB_EXTSLOT_SIZE 16
@@ -67,13 +69,16 @@ struct erofs_super_block {* inode, [xattrs], last_inline_data, ... | ... | no-holed data * 3 - inode compression D: * inode, [xattrs], map_header, extents ... | ... -* 4~7 - reserved +* 4 - inode chunk-based E: +* inode, [xattrs], chunk indexes ... | ... +* 5~7 - reserved */ enum {EROFS_INODE_FLAT_PLAIN =0 ,EROFS_INODE_FLAT_COMPRESSION_LEGACY =1 ,EROFS_INODE_FLAT_INLINE =2 ,EROFS_INODE_FLAT_COMPRESSION =3 , +EROFS_INODE_CHUNK_BASED =4 ,EROFS_INODE_DATALAYOUT_MAX };
@@ -93,6 +98,16 @@ static inline bool erofs_inode_is_data_compressed(unsigned int datamode)#define EROFS_I_ALL \ ((1 << (EROFS_I_DATALAYOUT_BIT + EROFS_I_DATALAYOUT_BITS)) - 1) +/* indicate chunk blkbits, thus `chunksize = blocksize << chunk blkbits' */ +#define EROFS_CHUNK_FORMAT_BLKBITS 0x001F +/* with chunk indexes or just a 4-byte blkaddr array */ +#define EROFS_CHUNK_FORMAT_INDEXES 0x0020 + +struct erofs_inode_chunk_info { +__le16 format;/* chunk blkbits */ +__le16 reserved; +}; +/* 32-byte reduced form of an ondisk inode */ struct erofs_inode_compact {__le16 i_format;/* inode format hints */
@@ -110,6 +125,9 @@ struct erofs_inode_compact {/* for device files, used to indicate old/new device # */ __le32 rdev; + +/* for chunk-based files, it contains the summary info */ +struct erofs_inode_chunk_info c; } i_u;__le32 i_ino;/* only used for 32-bit stat compatibility */ __le16 i_uid;
@@ -138,6 +156,9 @@ struct erofs_inode_extended {/* for device files, used to indicate old/new device # */ __le32 rdev; + +/* for chunk-based files, it contains the summary info */ +struct erofs_inode_chunk_info c; } i_u;/* only used for 32-bit stat compatibility */
@@ -207,6 +228,15 @@ static inline unsigned int erofs_xattr_entry_size(struct erofs_xattr_entry *e) e->e_name_len + le16_to_cpu(e->e_value_size)); } +/* represent a zeroed chunk (hole) */ +#define EROFS_NULL_ADDR -1 + +struct erofs_inode_chunk_index { +__le32 blkaddr; +__le16 device_id;/* back-end storage id, always 0 for now */ +__le16 reserved;/* reserved, don't care */ +}; +/* maximum supported size of a physical compression cluster */ #define Z_EROFS_PCLUSTER_MAX_SIZE (1024 * 1024)
@@ -351,6 +381,8 @@ static inline void erofs_check_ondisk_layout_definitions(void)BUILD_BUG_ON (sizeof (struct erofs_inode_extended ) !=64 );BUILD_BUG_ON (sizeof (struct erofs_xattr_ibody_header ) !=12 );BUILD_BUG_ON (sizeof (struct erofs_xattr_entry ) !=4 ); +BUILD_BUG_ON (sizeof (struct erofs_inode_chunk_info ) !=4 ); +BUILD_BUG_ON (sizeof (struct erofs_inode_chunk_index ) !=8 );BUILD_BUG_ON (sizeof (struct z_erofs_map_header ) !=8 );BUILD_BUG_ON (sizeof (struct z_erofs_vle_decompressed_index ) !=8 );BUILD_BUG_ON (sizeof (struct erofs_dirent ) !=12 );
@@ -360,4 +392,3 @@ static inline void erofs_check_ondisk_layout_definitions(void) }#endif -
lib/Makefile.am +1 -1
@@ -22,7 +22,7 @@ noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \ noinst_HEADERS += compressor.h liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \ namei.c data.c compress.c compressor.c zmap.c decompress.c \ - hashmap.c sha256.c + hashmap.c sha256.c blobchunk.c liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include if ENABLE_LZ4 liberofs_la_CFLAGS += ${LZ4_CFLAGS}
lib/blobchunk.c new file +200 -0
200 lines not on this page · raw message
lib/inode.c +29 -6
@@ -25,6 +25,7 @@#include "erofs/xattr.h" #include "erofs/exclude.h" #include "erofs/block_list.h" +#include "erofs/blobchunk.h" #define S_SHIFT 12static unsigned char erofs_ftype_by_mode[S_IFMT >>S_SHIFT ] = {
@@ -386,6 +387,9 @@ int erofs_write_file(struct erofs_inode *inode)return 0 ; } +if (cfg.c_chunkbits) +return erofs_blob_write_chunked_file (inode); +if (cfg.c_compr_alg_master &&erofs_file_is_compressible (inode)) { ret =erofs_write_compressed_file (inode);
@@ -439,6 +443,11 @@ static bool erofs_bh_flush_write_inode(struct erofs_buffer_head *bh)if (is_inode_layout_compression (inode)) u.dic.i_u.compressed_blocks =cpu_to_le32 (inode->u.i_blocks); +else if (inode->datalayout == +EROFS_INODE_CHUNK_BASED ) + u.dic.i_u.c.format = (cfg.c_chunkbits - +LOG_BLOCK_SIZE ) | +EROFS_CHUNK_FORMAT_INDEXES ;else u.dic.i_u.raw_blkaddr =cpu_to_le32 (inode->u.i_blkaddr);
@@ -472,6 +481,11 @@ static bool erofs_bh_flush_write_inode(struct erofs_buffer_head *bh)if (is_inode_layout_compression (inode)) u.die.i_u.compressed_blocks =cpu_to_le32 (inode->u.i_blocks); +else if (inode->datalayout == +EROFS_INODE_CHUNK_BASED ) + u.die.i_u.c.format = (cfg.c_chunkbits - +LOG_BLOCK_SIZE ) | +EROFS_CHUNK_FORMAT_INDEXES ;else u.die.i_u.raw_blkaddr =cpu_to_le32 (inode->u.i_blkaddr);
@@ -504,12 +518,19 @@ static bool erofs_bh_flush_write_inode(struct erofs_buffer_head *bh) }if (inode->extent_isize) { -/* write compression metadata */ - off =Z_EROFS_VLE_EXTENT_ALIGN (off); - ret =dev_write (inode->compressmeta, off, inode->extent_isize); -if (ret) -return false; -free (inode->compressmeta); +if (inode->datalayout ==EROFS_INODE_CHUNK_BASED ) { + ret =erofs_blob_write_chunk_indexes (inode, off); +if (ret) +return false; + }else { +/* write compression metadata */ + off =Z_EROFS_VLE_EXTENT_ALIGN (off); + ret =dev_write (inode->compressmeta, off, + inode->extent_isize); +if (ret) +return false; +free (inode->compressmeta); + } } inode->bh =NULL ;
@@ -564,6 +585,8 @@ static int erofs_prepare_inode_buffer(struct erofs_inode *inode)if (is_inode_layout_compression (inode)) goto noinline; +if (inode->datalayout ==EROFS_INODE_CHUNK_BASED ) + goto noinline;if (cfg.c_noinline_data &&S_ISREG (inode->i_mode)) { inode->datalayout =EROFS_INODE_FLAT_PLAIN ;
lib/io.c +1 -1
@@ -26,7 +26,7 @@#include "erofs/print.h" static const char *erofs_devname; -static int erofs_devfd =-1 ; +int erofs_devfd =-1 ;static u64 erofs_devsz;int dev_get_blkdev_size (int fd,u64 *bytes)
mkfs/main.c +40 -1
89 lines not on this page · raw message
Signature · 1 line
2.24.4
PATCH v2 4/4 erofs-utils: support chunk-based uncompressed files
Add mkfs support to new chunk-based uncompressed files.
---
changes since v1:
- update the member order of struct erofs_inode_chunk_index in sync with the kernel side.
include/erofs/blobchunk.h | 18 ++++ include/erofs/config.h | 2 +- include/erofs/internal.h | 1 + include/erofs/io.h | 2 + include/erofs_fs.h | 42 +++++++- lib/Makefile.am | 2 +- lib/blobchunk.c | 200 ++++++++++++++++++++++++++++++++++++++ lib/inode.c | 35 +++++-- lib/io.c | 2 +- mkfs/main.c | 39 ++++++++ 10 files changed, 330 insertions(+), 13 deletions(-) create mode 100644 include/erofs/blobchunk.h create mode 100644 lib/blobchunk.c
include/erofs/blobchunk.h new file +18 -0
@@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* +* erofs-utils/lib/blobchunk.h +* +* Copyright (C) 2021, Alibaba Cloud +*/ +#ifndef __EROFS_BLOBCHUNK_H +#define __EROFS_BLOBCHUNK_H + +#include "erofs/internal.h" + +int erofs_blob_write_chunk_indexes (struct erofs_inode *inode,erofs_off_t off); +int erofs_blob_write_chunked_file (struct erofs_inode *inode); +int erofs_blob_remap (void ); +void erofs_blob_exit (void ); +int erofs_blob_init (void ); + +#endif
include/erofs/config.h +1 -1
@@ -44,6 +44,7 @@ struct erofs_configure {bool c_random_pclusterblks;#endif char c_timeinherit; +char c_chunkbits;bool c_noinline_data;#ifdef HAVE_LIBSELINUX
@@ -57,7 +58,6 @@ struct erofs_configure {int c_force_inodeversion;/* < 0, xattr disabled and INT_MAX, always use inline xattrs */ int c_inline_xattr_tolerance; -u32 c_physical_clusterblks;u32 c_max_decompressed_extent_bytes;u64 c_unix_timestamp;
include/erofs/internal.h +1 -0
@@ -160,6 +160,7 @@ struct erofs_inode {union {void *compressmeta; +void *chunkindexes;struct {uint16_t z_advise;uint8_t z_algorithmtype[2 ];
include/erofs/io.h +2 -0
@@ -27,6 +27,8 @@ int dev_fsync(void);int dev_resize (erofs_blk_t nblocks);u64 dev_length (void ); +extern int erofs_devfd; +int erofs_copy_file_range (int fd_in,erofs_off_t *off_in,int fd_out,erofs_off_t *off_out,size_t length);
include/erofs_fs.h +38 -4
105 lines not on this page · raw message
lib/Makefile.am +1 -1
@@ -22,7 +22,7 @@ noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \ noinst_HEADERS += compressor.h liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \ namei.c data.c compress.c compressor.c zmap.c decompress.c \ - hashmap.c sha256.c + hashmap.c sha256.c blobchunk.c liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include if ENABLE_LZ4 liberofs_la_CFLAGS += ${LZ4_CFLAGS}
lib/blobchunk.c new file +200 -0
200 lines not on this page · raw message
lib/inode.c +29 -6
@@ -25,6 +25,7 @@#include "erofs/xattr.h" #include "erofs/exclude.h" #include "erofs/block_list.h" +#include "erofs/blobchunk.h" #define S_SHIFT 12static unsigned char erofs_ftype_by_mode[S_IFMT >>S_SHIFT ] = {
@@ -386,6 +387,9 @@ int erofs_write_file(struct erofs_inode *inode)return 0 ; } +if (cfg.c_chunkbits) +return erofs_blob_write_chunked_file (inode); +if (cfg.c_compr_alg_master &&erofs_file_is_compressible (inode)) { ret =erofs_write_compressed_file (inode);
@@ -439,6 +443,11 @@ static bool erofs_bh_flush_write_inode(struct erofs_buffer_head *bh)if (is_inode_layout_compression (inode)) u.dic.i_u.compressed_blocks =cpu_to_le32 (inode->u.i_blocks); +else if (inode->datalayout == +EROFS_INODE_CHUNK_BASED ) + u.dic.i_u.c.format = (cfg.c_chunkbits - +LOG_BLOCK_SIZE ) | +EROFS_CHUNK_FORMAT_INDEXES ;else u.dic.i_u.raw_blkaddr =cpu_to_le32 (inode->u.i_blkaddr);
@@ -472,6 +481,11 @@ static bool erofs_bh_flush_write_inode(struct erofs_buffer_head *bh)if (is_inode_layout_compression (inode)) u.die.i_u.compressed_blocks =cpu_to_le32 (inode->u.i_blocks); +else if (inode->datalayout == +EROFS_INODE_CHUNK_BASED ) + u.die.i_u.c.format = (cfg.c_chunkbits - +LOG_BLOCK_SIZE ) | +EROFS_CHUNK_FORMAT_INDEXES ;else u.die.i_u.raw_blkaddr =cpu_to_le32 (inode->u.i_blkaddr);
@@ -504,12 +518,19 @@ static bool erofs_bh_flush_write_inode(struct erofs_buffer_head *bh) }if (inode->extent_isize) { -/* write compression metadata */ - off =Z_EROFS_VLE_EXTENT_ALIGN (off); - ret =dev_write (inode->compressmeta, off, inode->extent_isize); -if (ret) -return false; -free (inode->compressmeta); +if (inode->datalayout ==EROFS_INODE_CHUNK_BASED ) { + ret =erofs_blob_write_chunk_indexes (inode, off); +if (ret) +return false; + }else { +/* write compression metadata */ + off =Z_EROFS_VLE_EXTENT_ALIGN (off); + ret =dev_write (inode->compressmeta, off, + inode->extent_isize); +if (ret) +return false; +free (inode->compressmeta); + } } inode->bh =NULL ;
@@ -564,6 +585,8 @@ static int erofs_prepare_inode_buffer(struct erofs_inode *inode)if (is_inode_layout_compression (inode)) goto noinline; +if (inode->datalayout ==EROFS_INODE_CHUNK_BASED ) + goto noinline;if (cfg.c_noinline_data &&S_ISREG (inode->i_mode)) { inode->datalayout =EROFS_INODE_FLAT_PLAIN ;
lib/io.c +1 -1
@@ -26,7 +26,7 @@#include "erofs/print.h" static const char *erofs_devname; -static int erofs_devfd =-1 ; +int erofs_devfd =-1 ;static u64 erofs_devsz;int dev_get_blkdev_size (int fd,u64 *bytes)
mkfs/main.c +39 -0
@@ -23,6 +23,8 @@#include "erofs/xattr.h" #include "erofs/exclude.h" #include "erofs/block_list.h" +#include "erofs/hashtable.h" +#include "erofs/blobchunk.h" #ifdef HAVE_LIBUUID #include <uuid.h>
@@ -44,6 +46,7 @@ static struct option long_options[] = { {"random-pclusterblks", no_argument, NULL, 8}, #endif {"max-extent-bytes", required_argument, NULL, 9}, + {"chunksize", required_argument, NULL, 256}, #ifdef WITH_ANDROID {"mount-point", required_argument, NULL, 10}, {"product-out", required_argument, NULL, 11},
@@ -79,6 +82,7 @@ static void usage(void)#ifdef HAVE_LIBUUID " -UX use a given filesystem UUID\n" #endif +" --chunksize=X set chunk size to X bytes and use chunk-based files instead\n" " --exclude-path=X avoid including file X (X = exact literal path)\n" " --exclude-regex=X avoid including files that match X (X = regular expression)\n" #ifdef HAVE_LIBSELINUX
@@ -316,6 +320,24 @@ static int mkfs_parse_options_cfg(int argc, char *argv[]) } cfg.c_physical_clusterblks = i /EROFS_BLKSIZ ;break ; +case 256 : + i =strtol (optarg, &endptr,0 ); +if (*endptr !='\0' ) { +erofs_err ("invalid chunksize %s" , optarg); +return -EINVAL ; + } + cfg.c_chunkbits =ilog2 (i); +if ((1 << cfg.c_chunkbits) != i) { +erofs_err ("chunksize %s must be a power of two" , + optarg); +return -EINVAL ; + } +if (i <EROFS_BLKSIZ ) { +erofs_err ("chunksize %s must be larger than block size" , + optarg); +return -EINVAL ; + } +break ;case 1 :usage ();
@@ -523,6 +545,12 @@ int main(int argc, char **argv)return 1 ; } +if (cfg.c_chunkbits) { + err =erofs_blob_init (); +if (err) +return 1 ; + } + err =lstat64 (cfg.c_src_path, &st);if (err)return 1 ;
@@ -610,6 +638,13 @@ int main(int argc, char **argv) root_nid =erofs_lookupnid (root_inode);erofs_iput (root_inode); +if (cfg.c_chunkbits) { +erofs_info ("total metadata: %u blocks" ,erofs_mapbh (NULL )); + err =erofs_blob_remap (); +if (err) + goto exit; + } + err =erofs_mkfs_update_super_block (sb_bh, root_nid, &nblocks);if (err) goto exit;
@@ -629,6 +664,10 @@ exit:#endif dev_close ();erofs_cleanup_exclude_rules (); + +if (cfg.c_chunkbits) +erofs_blob_exit (); +erofs_exit_configure ();if (err) {
Signature · 1 line
2.24.4