Skip to contentSkip to conversation

erofs-utils: lib: fix wrong forward declaration of `erofs_metamgr`

PATCH In review Gao Xiang 2 participants 6 messages Aug 26 – Aug 27 Library code

lore
Share & tools
lore

Gao Xiang

author
#0 lore raw

PATCH 1/4 erofs-utils: lib: fix wrong forward declaration of `erofs_metamgr`

it is forward-declared as "struct erofs_metaboxmgr", but the actual struct was already named as "erofs_metamgr" in lib/metabox.c.

This could confuse the compiler; update it to match the real name.

---
 include/erofs/internal.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
include/erofs/internal.h +1 -1
index c9c7873..514cfbd 100644
@@ -89,7 +89,7 @@ struct erofs_mkfs_dfops;
 struct erofs_packed_inode;
 struct erofs_xattrmgr;
 struct z_erofs_mgr;
-struct erofs_metaboxmgr;
+struct erofs_metamgr;
 
 struct erofs_sb_info {
 	struct erofs_sb_lz4_info lz4;
Signature · 1 line
2.47.3
author
#1 lore raw

PATCH 2/4 erofs-utils: lib: use `struct erofs_device_info` for the primary device

Source kernel commit: 7b00af2c5414dc01e0718deef7ead81102867636

---
 include/erofs/internal.h |  4 +---
 lib/importer.c           |  3 +--
 lib/io.c                 |  8 ++++----
 lib/rebuild.c            |  6 +++---
 lib/super.c              | 18 +++++++++---------
 lib/vmdk.c               |  4 ++--
 mkfs/main.c              | 12 ++++++------
 7 files changed, 26 insertions(+), 29 deletions(-)
include/erofs/internal.h +1 -3
index 514cfbd..30dae41 100644
@@ -93,11 +93,9 @@ struct erofs_metamgr;
 
 struct erofs_sb_info {
 	struct erofs_sb_lz4_info lz4;
-	struct erofs_device_info *devs;
-	char *devname;
+	struct erofs_device_info *devs, dif0;
 
 	u64 total_blocks;
-	u64 primarydevice_blocks;
 
 	s32 meta_blkaddr;
 	u32 xattr_blkaddr;
lib/importer.c +1 -2
index c404b0f..2a635d2 100644
@@ -126,8 +126,7 @@ int erofs_importer_flush_all(struct erofs_importer *im)
 
 	fsalignblks = im->params->fsalignblks ?
 		roundup_pow_of_two(im->params->fsalignblks) : 1;
-	sbi->primarydevice_blocks = roundup(erofs_mapbh(sbi->bmgr, NULL),
-					    fsalignblks);
+	sbi->dif0.blocks = roundup(erofs_mapbh(sbi->bmgr, NULL), fsalignblks);
 	err = erofs_write_device_table(sbi);
 	if (err)
 		return err;
lib/io.c +4 -4
index 3ba45cc..5c5449f 100644
@@ -368,8 +368,8 @@ repeat:
 	}
 
 out:
-	sbi->devname = strdup(dev);
-	if (!sbi->devname) {
+	sbi->dif0.src_path = strdup(dev);
+	if (!sbi->dif0.src_path) {
 		close(fd);
 		return -ENOMEM;
 	}
@@ -381,8 +381,8 @@ out:
 void erofs_dev_close(struct erofs_sb_info *sbi)
 {
 	erofs_io_close(&sbi->bdev);
-	free(sbi->devname);
-	sbi->devname = NULL;
+	free(sbi->dif0.src_path);
+	sbi->dif0.src_path = NULL;
 	sbi->bdev.fd = -1;
 }
 
lib/rebuild.c +3 -3
index a5308dc..6b03a52 100644
@@ -233,7 +233,7 @@ static int erofs_rebuild_write_full_data(struct erofs_inode *inode)
 				return -EFSCORRUPTED;
 			return 0;
 		}
-		inode->rebuild_blobpath = strdup(src_sbi->devname);
+		inode->rebuild_blobpath = strdup(src_sbi->dif0.src_path);
 		if (!inode->rebuild_blobpath)
 			return -ENOMEM;
 		inode->rebuild_src_dataoff =
@@ -244,7 +244,7 @@ static int erofs_rebuild_write_full_data(struct erofs_inode *inode)
 		unsigned int inline_size = inode->i_size % erofs_blksiz(src_sbi);
 
 		if (nblocks > 0 && inode->u.i_blkaddr != EROFS_NULL_ADDR) {
-			inode->rebuild_blobpath = strdup(src_sbi->devname);
+			inode->rebuild_blobpath = strdup(src_sbi->dif0.src_path);
 			if (!inode->rebuild_blobpath)
 				return -ENOMEM;
 			inode->rebuild_src_dataoff =
@@ -500,7 +500,7 @@ int erofs_rebuild_load_tree(struct erofs_inode *root, struct erofs_sb_info *sbi,
 	struct erofs_inode inode = {};
 	struct erofs_rebuild_dir_context ctx;
 	char uuid_str[37];
-	char *fsid = sbi->devname;
+	char *fsid = sbi->dif0.src_path;
 	int ret;
 
 	if (!fsid) {
lib/super.c +9 -9
index d138f84..ead4170 100644
@@ -33,7 +33,7 @@ static int erofs_init_devices(struct erofs_sb_info *sbi,
 	erofs_off_t pos;
 	bool _48bit = erofs_sb_has_48bit(sbi);
 
-	sbi->total_blocks = sbi->primarydevice_blocks;
+	sbi->total_blocks = sbi->dif0.blocks;
 
 	if (!erofs_sb_has_device_table(sbi))
 		ondisk_extradevs = 0;
@@ -118,14 +118,14 @@ int erofs_read_superblock(struct erofs_sb_info *sbi)
 		erofs_err("invalid sb_extslots %u", dsb->sb_extslots);
 		return -EINVAL;
 	}
-	sbi->primarydevice_blocks = le32_to_cpu(dsb->blocks_lo);
+	sbi->dif0.blocks = le32_to_cpu(dsb->blocks_lo);
 	sbi->meta_blkaddr = le32_to_cpu(dsb->meta_blkaddr);
 	sbi->xattr_blkaddr = le32_to_cpu(dsb->xattr_blkaddr);
 	sbi->xattr_prefix_start = le32_to_cpu(dsb->xattr_prefix_start);
 	sbi->xattr_prefix_count = dsb->xattr_prefix_count;
 	if (erofs_sb_has_48bit(sbi) && dsb->rootnid_8b) {
 		sbi->root_nid = le64_to_cpu(dsb->rootnid_8b);
-		sbi->primarydevice_blocks = sbi->primarydevice_blocks |
+		sbi->dif0.blocks = sbi->dif0.blocks |
 				((u64)le16_to_cpu(dsb->rb.blocks_hi) << 32);
 	} else {
 		sbi->root_nid = le16_to_cpu(dsb->rb.rootnid_2b);
@@ -230,10 +230,10 @@ int erofs_writesb(struct erofs_sb_info *sbi)
 	char *buf;
 	int ret;
 
-	sb.blocks_lo	= cpu_to_le32(sbi->primarydevice_blocks);
-	if (sbi->primarydevice_blocks > UINT32_MAX ||
+	sb.blocks_lo	= cpu_to_le32(sbi->dif0.blocks);
+	if (sbi->dif0.blocks > UINT32_MAX ||
 	    sbi->root_nid > UINT16_MAX) {
-		sb.rb.blocks_hi = cpu_to_le16(sbi->primarydevice_blocks >> 32);
+		sb.rb.blocks_hi = cpu_to_le16(sbi->dif0.blocks >> 32);
 		sb.rootnid_8b = cpu_to_le64(sbi->root_nid);
 	}
 	memcpy(sb.uuid, sbi->uuid, sizeof(sb.uuid));
@@ -404,7 +404,7 @@ int erofs_mkfs_init_devices(struct erofs_sb_info *sbi, unsigned int devices)
 
 int erofs_write_device_table(struct erofs_sb_info *sbi)
 {
-	erofs_blk_t nblocks = sbi->primarydevice_blocks;
+	erofs_blk_t nblocks = sbi->dif0.blocks;
 	struct erofs_buffer_head *bh = sbi->bh_devt;
 	erofs_off_t pos;
 	unsigned int i, ret;
@@ -484,7 +484,7 @@ int erofs_mkfs_load_fs(struct erofs_sb_info *sbi, unsigned int dsunit)
 	erofs_warn("EXPERIMENTAL incremental build in use. Use at your own risk!");
 	err = erofs_read_superblock(sbi);
 	if (err) {
-		erofs_err("failed to read superblock of %s: %s", sbi->devname,
+		erofs_err("failed to read superblock of %s: %s", sbi->dif0.src_path,
 			  erofs_strerror(err));
 		return err;
 	}
@@ -493,7 +493,7 @@ int erofs_mkfs_load_fs(struct erofs_sb_info *sbi, unsigned int dsunit)
 	if (!err && S_ISREG(u.st.st_mode))
 		u.startblk = DIV_ROUND_UP(u.st.st_size, erofs_blksiz(sbi));
 	else
-		u.startblk = sbi->primarydevice_blocks;
+		u.startblk = sbi->dif0.blocks;
 
 	bmgr = erofs_buffer_init(sbi, u.startblk, NULL);
 	if (!bmgr)
lib/vmdk.c +2 -2
index 316a8ff..28381c8 100644
@@ -51,8 +51,8 @@ int erofs_dump_vmdk_desc(FILE *f, struct erofs_sb_info *sbi)
 	int ret, i;
 
 	fprintf(f, desc_template_1, cid, parent_cid, subformat);
-	sectors = sbi->primarydevice_blocks << (sbi->blkszbits - 9);
-	ret = erofs_vmdk_desc_add_extent(f, sectors, (char *)sbi->devname, 0);
+	sectors = sbi->dif0.blocks << (sbi->blkszbits - 9);
+	ret = erofs_vmdk_desc_add_extent(f, sectors, (char *)sbi->dif0.blocks, 0);
 	if (ret)
 		return ret;
 	total_sectors = sectors;
mkfs/main.c +6 -6
index 929ce7c..5bf7b8f 100644
@@ -1685,12 +1685,12 @@ static int erofs_mkfs_rebuild_load_trees(struct erofs_inode *root)
 		ret = erofs_rebuild_load_tree(root, src, datamode);
 		src->xamgr = NULL;
 		if (ret) {
-			erofs_err("failed to load %s", src->devname);
+			erofs_err("failed to load %s", src->dif0.src_path);
 			return ret;
 		}
 		if (src->extra_devices > 1) {
 			erofs_err("%s: unsupported number %u of extra devices",
-				  src->devname, src->extra_devices);
+				  src->dif0.src_path, src->extra_devices);
 			return -EOPNOTSUPP;
 		}
 		extra_devices += src->extra_devices;
@@ -1723,8 +1723,8 @@ static int erofs_mkfs_rebuild_load_trees(struct erofs_inode *root)
 			nblocks = src->devs[0].blocks;
 			tag = src->devs[0].tag;
 		} else {
-			nblocks = src->primarydevice_blocks;
-			devs[idx].src_path = strdup(src->devname);
+			nblocks = src->dif0.blocks;
+			devs[idx].src_path = strdup(src->dif0.src_path);
 		}
 		devs[idx].blocks = nblocks;
 		if (tag && *tag)
@@ -1849,7 +1849,7 @@ int main(int argc, char **argv)
 			goto exit;
 		err = erofs_read_superblock(src);
 		if (err) {
-			erofs_err("failed to read superblock of %s", src->devname);
+			erofs_err("failed to read superblock of %s", src->dif0.src_path);
 			goto exit;
 		}
 		mkfs_blkszbits = src->blkszbits;
@@ -2051,7 +2051,7 @@ int main(int argc, char **argv)
 	if (err)
 		goto exit;
 
-	err = erofs_dev_resize(&g_sbi, g_sbi.primarydevice_blocks);
+	err = erofs_dev_resize(&g_sbi, g_sbi.dif0.blocks);
 
 	if (!err && erofs_sb_has_sb_chksum(&g_sbi)) {
 		err = erofs_enable_sb_chksum(&g_sbi, &crc);
Signature · 1 line
2.47.3
author
#5 lore raw

PATCH 4/4 erofs-utils: get rid of hashmap

In reply to Gao Xiang

This is the last piece of GPL-only 3rd-party code in liberofs.

We may introduce a new dynamic hashmap implementation without GPL-only restriction later.

---
 include/erofs/flex-array.h | 156 ---------------------
 include/erofs/hashmap.h    | 115 ---------------
 lib/Makefile.am            |   4 +-
 lib/hashmap.c              | 280 -------------------------------------
 4 files changed, 1 insertion(+), 554 deletions(-)
 delete mode 100644 include/erofs/flex-array.h
 delete mode 100644 include/erofs/hashmap.h
 delete mode 100644 lib/hashmap.c
include/erofs/flex-array.h deleted +0 -156
deleted file mode 100644
index 9b1642f..0000000
@@ -1,156 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef __EROFS_FLEX_ARRAY_H
-#define __EROFS_FLEX_ARRAY_H
-
-#ifdef __cplusplus
-extern "C"
-{
-#endif
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <limits.h>
-#include <stdint.h>
-
-#include "defs.h"
-#include "print.h"
-
-/*
- * flex-array.h
- *
- * Some notes to make sense of the code.
- *
- * Flex-arrays:
- *   - Flex-arrays became standard in C99 and are defined by "array[]" (at the
- *     end of a struct)
- *   - Pre-C99 flex-arrays can be accomplished by "array[1]"
- *   - There is a GNU extension where they are defined using "array[0]"
- *     Allegedly there is/was a bug in gcc whereby foo[1] generated incorrect
- *     code, so it's safest to use [0] (https://lkml.org/lkml/2015/2/18/407).
- *
- * For C89 and C90, __STDC__ is 1
- * For later standards, __STDC_VERSION__ is defined according to the standard.
- * For example: 199901L or 201112L
- *
- * Whilst we're on the subject, in version 5 of gcc, the default std was
- * changed from gnu89 to gnu11. In jgmenu, CFLAGS therefore contains -std=gnu89
- * You can check your default gcc std by doing:
- * gcc -dM -E - </dev/null | grep '__STDC_VERSION__\|__STDC__'
- *
- * The code below is copied from git's git-compat-util.h in support of
- * hashmap.c
- */
-
-#ifndef FLEX_ARRAY
-#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && \
-	(!defined(__SUNPRO_C) || (__SUNPRO_C > 0x580))
-# define FLEX_ARRAY /* empty */
-#elif defined(__GNUC__)
-# if (__GNUC__ >= 3)
-#  define FLEX_ARRAY /* empty */
-# else
-#  define FLEX_ARRAY 0 /* older GNU extension */
-# endif
-#endif
-
-/* Otherwise, default to safer but a bit wasteful traditional style */
-#ifndef FLEX_ARRAY
-# define FLEX_ARRAY 1
-#endif
-#endif
-
-#define bitsizeof(x) (CHAR_BIT * sizeof(x))
-
-#define maximum_signed_value_of_type(a) \
-	(INTMAX_MAX >> (bitsizeof(intmax_t) - bitsizeof(a)))
-
-#define maximum_unsigned_value_of_type(a) \
-	(UINTMAX_MAX >> (bitsizeof(uintmax_t) - bitsizeof(a)))
-
-/*
- * Signed integer overflow is undefined in C, so here's a helper macro
- * to detect if the sum of two integers will overflow.
- * Requires: a >= 0, typeof(a) equals typeof(b)
- */
-#define signed_add_overflows(a, b) \
-	((b) > maximum_signed_value_of_type(a) - (a))
-
-#define unsigned_add_overflows(a, b) \
-	((b) > maximum_unsigned_value_of_type(a) - (a))
-
-static inline size_t st_add(size_t a, size_t b)
-{
-	if (unsigned_add_overflows(a, b)) {
-		erofs_err("size_t overflow: %llu + %llu", a | 0ULL, b | 0ULL);
-		BUG_ON(1);
-		return -1;
-	}
-	return a + b;
-}
-
-#define st_add3(a, b, c) st_add(st_add((a), (b)), (c))
-#define st_add4(a, b, c, d) st_add(st_add3((a), (b), (c)), (d))
-
-/*
- * These functions help you allocate structs with flex arrays, and copy
- * the data directly into the array. For example, if you had:
- *
- *   struct foo {
- *     int bar;
- *     char name[FLEX_ARRAY];
- *   };
- *
- * you can do:
- *
- *   struct foo *f;
- *   FLEX_ALLOC_MEM(f, name, src, len);
- *
- * to allocate a "foo" with the contents of "src" in the "name" field.
- * The resulting struct is automatically zero'd, and the flex-array field
- * is NUL-terminated (whether the incoming src buffer was or not).
- *
- * The FLEXPTR_* variants operate on structs that don't use flex-arrays,
- * but do want to store a pointer to some extra data in the same allocated
- * block. For example, if you have:
- *
- *   struct foo {
- *     char *name;
- *     int bar;
- *   };
- *
- * you can do:
- *
- *   struct foo *f;
- *   FLEXPTR_ALLOC_STR(f, name, src);
- *
- * and "name" will point to a block of memory after the struct, which will be
- * freed along with the struct (but the pointer can be repointed anywhere).
- *
- * The *_STR variants accept a string parameter rather than a ptr/len
- * combination.
- *
- * Note that these macros will evaluate the first parameter multiple
- * times, and it must be assignable as an lvalue.
- */
-#define FLEX_ALLOC_MEM(x, flexname, buf, len) do { \
-	size_t flex_array_len_ = (len); \
-	(x) = calloc(1, st_add3(sizeof(*(x)), flex_array_len_, 1)); \
-	BUG_ON(!(x)); \
-	memcpy((void *)(x)->flexname, (buf), flex_array_len_); \
-} while (0)
-#define FLEXPTR_ALLOC_MEM(x, ptrname, buf, len) do { \
-	size_t flex_array_len_ = (len); \
-	(x) = xcalloc(1, st_add3(sizeof(*(x)), flex_array_len_, 1)); \
-	memcpy((x) + 1, (buf), flex_array_len_); \
-	(x)->ptrname = (void *)((x) + 1); \
-} while (0)
-#define FLEX_ALLOC_STR(x, flexname, str) \
-	FLEX_ALLOC_MEM((x), flexname, (str), strlen(str))
-#define FLEXPTR_ALLOC_STR(x, ptrname, str) \
-	FLEXPTR_ALLOC_MEM((x), ptrname, (str), strlen(str))
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
include/erofs/hashmap.h deleted +0 -115

115 lines not on this page · raw message

lib/Makefile.am +1 -3
index 5ef2122..25d08f2 100644
@@ -10,8 +10,6 @@ noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \
       $(top_srcdir)/include/erofs/diskbuf.h \
       $(top_srcdir)/include/erofs/err.h \
       $(top_srcdir)/include/erofs/exclude.h \
-      $(top_srcdir)/include/erofs/flex-array.h \
-      $(top_srcdir)/include/erofs/hashmap.h \
       $(top_srcdir)/include/erofs/inode.h \
       $(top_srcdir)/include/erofs/internal.h \
       $(top_srcdir)/include/erofs/io.h \
@@ -43,7 +41,7 @@ endif
 
 liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \
 		      namei.c data.c compress.c compressor.c zmap.c decompress.c \
-		      compress_hints.c hashmap.c sha256.c blobchunk.c dir.c \
+		      compress_hints.c sha256.c blobchunk.c dir.c \
 		      fragments.c dedupe.c uuid_unparse.c uuid.c tar.c \
 		      block_list.c rebuild.c diskbuf.c bitops.c dedupe_ext.c \
 		      vmdk.c metabox.c global.c importer.c base64.c
lib/hashmap.c deleted +0 -280

280 lines not on this page · raw message

Signature · 1 line
2.47.3
author
#4 lore raw

PATCH 3/4 erofs-utils: lib: refactor unencoded chunk handling

In reply to Gao Xiang

Switch global variables to per-sb and/or per-device structures, replace the hashmap with a bucket-list hash table, and replace open-coded blobfile raw I/O with the buffer allocator and vfile I/O so that blob chunks won't rely on temporary files on /tmp [1].

Future follow-ups: Optimize to the PLAIN/INLINE inode layout if no valid deduplication chunk is found.

[ Gao Xiang: need to check if --dsunit works as expected in this new
implementation. ]

---
 include/erofs/blobchunk.h |  11 +-
 include/erofs/internal.h  |   5 +-
 lib/blobchunk.c           | 422 ++++++++++++++++++++------------------
 lib/rebuild.c             |   4 +-
 lib/super.c               |  12 +-
 mkfs/main.c               |  27 ++-
 6 files changed, 262 insertions(+), 219 deletions(-)
include/erofs/blobchunk.h +7 -4
index 1761fdd..1c8d77d 100644
@@ -14,8 +14,9 @@ extern "C"
 
 #include "erofs/internal.h"
 
-struct erofs_blobchunk *erofs_get_unhashed_chunk(unsigned int device_id,
-		erofs_blk_t blkaddr, erofs_off_t sourceoffset);
+struct erofs_chunkitem *erofs_get_unhashed_chunk(struct erofs_sb_info *sbi,
+		unsigned int device_id, erofs_blk_t blkaddr,
+		erofs_off_t sourceoffset);
 void erofs_inode_fixup_chunkformat(struct erofs_inode *inode);
 int erofs_write_chunk_indexes(struct erofs_inode *inode, struct erofs_vfile *vf,
 			      erofs_off_t off);
@@ -24,8 +25,10 @@ int erofs_blob_write_chunked_file(struct erofs_inode *inode, int fd,
 int erofs_write_zero_inode(struct erofs_inode *inode);
 int tarerofs_write_chunkes(struct erofs_inode *inode, erofs_off_t data_offset);
 int erofs_mkfs_dump_blobs(struct erofs_sb_info *sbi);
-void erofs_blob_exit(void);
-int erofs_blob_init(const char *blobfile_path, erofs_off_t chunksize);
+int erofs_blob_exit(struct erofs_sb_info *sbi);
+int erofs_blob_init(struct erofs_sb_info *sbi, int blobdev_id,
+		    unsigned int chunkbits_def);
+int erofs_blob_init_device(struct erofs_sb_info *sbi, int device_id);
 
 #ifdef __cplusplus
 }
include/erofs/internal.h +4 -1
index 30dae41..95f5627 100644
@@ -67,8 +67,9 @@ struct erofs_buffer_head;
 struct erofs_bufmgr;
 
 struct erofs_device_info {
-	char *src_path;
 	u8 tag[64];
+	char *src_path;
+	struct erofs_bufmgr *bmgr;
 	erofs_blk_t blocks;
 	erofs_blk_t uniaddr;
 };
@@ -90,6 +91,7 @@ struct erofs_packed_inode;
 struct erofs_xattrmgr;
 struct z_erofs_mgr;
 struct erofs_metamgr;
+struct erofs_chunkmgr;
 
 struct erofs_sb_info {
 	struct erofs_sb_lz4_info lz4;
@@ -151,6 +153,7 @@ struct erofs_sb_info {
 	struct erofs_bufmgr *bmgr;
 	struct erofs_xattrmgr *xamgr;
 	struct z_erofs_mgr *zmgr;
+	struct erofs_chunkmgr *chunkmgr;
 	struct erofs_metamgr *m2gr, *mxgr;
 	struct erofs_packed_inode *packedinode;
 	struct erofs_buffer_head *bh_sb;
lib/blobchunk.c +219 -203

599 lines not on this page · raw message

lib/rebuild.c +2 -2
index 6b03a52..55681f9 100644
@@ -193,7 +193,7 @@ static int erofs_rebuild_write_blob_index(struct erofs_sb_info *dst_sb,
 	inode->chunkindexes = idx;
 
 	for (i = 0; i < count; i++) {
-		struct erofs_blobchunk *chunk;
+		struct erofs_chunkitem *chunk;
 		struct erofs_map_blocks map = {
 			.buf = __EROFS_BUF_INITIALIZER,
 		};
@@ -204,7 +204,7 @@ static int erofs_rebuild_write_blob_index(struct erofs_sb_info *dst_sb,
 			goto err;
 
 		blkaddr = erofs_blknr(dst_sb, map.m_pa);
-		chunk = erofs_get_unhashed_chunk(inode->dev, blkaddr, 0);
+		chunk = erofs_get_unhashed_chunk(dst_sb, inode->dev, blkaddr, 0);
 		if (IS_ERR(chunk)) {
 			ret = PTR_ERR(chunk);
 			goto err;
lib/super.c +11 -1
index ead4170..25116fe 100644
@@ -189,8 +189,18 @@ void erofs_put_super(struct erofs_sb_info *sbi)
 		int i;
 
 		DBG_BUGON(!sbi->extra_devices);
-		for (i = 0; i < sbi->extra_devices; ++i)
+		for (i = 0; i < sbi->extra_devices; ++i) {
+			struct erofs_bufmgr *bmgr = sbi->devs[i].bmgr;
+			struct erofs_vfile *vf;
+
+			if (bmgr) {
+				vf = bmgr->vf;
+				erofs_buffer_exit(bmgr);
+				close(vf->fd);
+				free(vf);
+			}
 			free(sbi->devs[i].src_path);
+		}
 		free(sbi->devs);
 		sbi->devs = NULL;
 	}
mkfs/main.c +19 -8
index 5bf7b8f..b6d215b 100644
@@ -1915,12 +1915,6 @@ int main(int argc, char **argv)
 	}
 
 	cfg.c_dedupe = importer_params.dedupe;
-	if (cfg.c_chunkbits) {
-		err = erofs_blob_init(cfg.c_blobdev_path, 1 << cfg.c_chunkbits);
-		if (err)
-			goto exit;
-	}
-
 	if (tar_index_512b || cfg.c_blobdev_path) {
 		err = erofs_mkfs_init_devices(&g_sbi, 1);
 		if (err) {
@@ -1930,6 +1924,24 @@ int main(int argc, char **argv)
 		}
 	}
 
+	if (tar_index_512b || cfg.c_chunkbits) {
+		if (g_sbi.extra_devices && cfg.c_blobdev_path) {
+			g_sbi.devs[0].src_path = strdup(cfg.c_blobdev_path);
+			if (!g_sbi.devs[0].src_path) {
+				err = -ENOMEM;
+				goto exit;
+			}
+
+			err = erofs_blob_init_device(&g_sbi, 1);
+			if (err)
+				goto exit;
+
+		}
+		err = erofs_blob_init(&g_sbi, cfg.c_blobdev_path ? 1 : 0, cfg.c_chunkbits);
+		if (err)
+			goto exit;
+	}
+
 	if (source_mode == EROFS_MKFS_SOURCE_LOCALDIR) {
 		err = erofs_load_shared_xattrs_from_path(&g_sbi, cfg.c_src_path,
 						mkfscfg.inlinexattr_tolerance);
@@ -2072,8 +2084,7 @@ exit:
 		fclose(blklst);
 	erofs_cleanup_compress_hints();
 	erofs_cleanup_exclude_rules();
-	if (cfg.c_chunkbits || source_mode == EROFS_MKFS_SOURCE_REBUILD)
-		erofs_blob_exit();
+	erofs_blob_exit(&g_sbi);
 	erofs_xattr_cleanup_name_prefixes();
 	erofs_rebuild_cleanup();
 	erofs_diskbuf_exit();
Signature · 1 line
2.47.3
#2 lore raw

Re: [PATCH 2/4] erofs-utils: lib: use `struct erofs_device_info` for the primary device

In reply to Gao Xiang
Quoted 177 lines · Gao Xiang

Source kernel commit: 7b00af2c5414dc01e0718deef7ead81102867636

---
  include/erofs/internal.h |  4 +---
  lib/importer.c           |  3 +--
  lib/io.c                 |  8 ++++----
  lib/rebuild.c            |  6 +++---
  lib/super.c              | 18 +++++++++---------
  lib/vmdk.c               |  4 ++--
  mkfs/main.c              | 12 ++++++------
  7 files changed, 26 insertions(+), 29 deletions(-)
include/erofs/internal.h +1 -3
index 514cfbd..30dae41 100644
@@ -93,11 +93,9 @@ struct erofs_metamgr;
 
  struct erofs_sb_info {
  	struct erofs_sb_lz4_info lz4;
-	struct erofs_device_info *devs;
-	char *devname;
+	struct erofs_device_info *devs, dif0;
 
  	u64 total_blocks;
-	u64 primarydevice_blocks;
 
  	s32 meta_blkaddr;
  	u32 xattr_blkaddr;
lib/importer.c +1 -2
index c404b0f..2a635d2 100644
@@ -126,8 +126,7 @@ int erofs_importer_flush_all(struct erofs_importer *im)
 
  	fsalignblks = im->params->fsalignblks ?
  		roundup_pow_of_two(im->params->fsalignblks) : 1;
-	sbi->primarydevice_blocks = roundup(erofs_mapbh(sbi->bmgr, NULL),
-					    fsalignblks);
+	sbi->dif0.blocks = roundup(erofs_mapbh(sbi->bmgr, NULL), fsalignblks);
  	err = erofs_write_device_table(sbi);
  	if (err)
  		return err;
lib/io.c +4 -4
index 3ba45cc..5c5449f 100644
@@ -368,8 +368,8 @@ repeat:
  	}
 
  out:
-	sbi->devname = strdup(dev);
-	if (!sbi->devname) {
+	sbi->dif0.src_path = strdup(dev);
+	if (!sbi->dif0.src_path) {
  		close(fd);
  		return -ENOMEM;
  	}
@@ -381,8 +381,8 @@ out:
  void erofs_dev_close(struct erofs_sb_info *sbi)
  {
  	erofs_io_close(&sbi->bdev);
-	free(sbi->devname);
-	sbi->devname = NULL;
+	free(sbi->dif0.src_path);
+	sbi->dif0.src_path = NULL;
  	sbi->bdev.fd = -1;
  }
 
lib/rebuild.c +3 -3
index a5308dc..6b03a52 100644
@@ -233,7 +233,7 @@ static int erofs_rebuild_write_full_data(struct erofs_inode *inode)
  				return -EFSCORRUPTED;
  			return 0;
  		}
-		inode->rebuild_blobpath = strdup(src_sbi->devname);
+		inode->rebuild_blobpath = strdup(src_sbi->dif0.src_path);
  		if (!inode->rebuild_blobpath)
  			return -ENOMEM;
  		inode->rebuild_src_dataoff =
@@ -244,7 +244,7 @@ static int erofs_rebuild_write_full_data(struct erofs_inode *inode)
  		unsigned int inline_size = inode->i_size % erofs_blksiz(src_sbi);
 
  		if (nblocks > 0 && inode->u.i_blkaddr != EROFS_NULL_ADDR) {
-			inode->rebuild_blobpath = strdup(src_sbi->devname);
+			inode->rebuild_blobpath = strdup(src_sbi->dif0.src_path);
  			if (!inode->rebuild_blobpath)
  				return -ENOMEM;
  			inode->rebuild_src_dataoff =
@@ -500,7 +500,7 @@ int erofs_rebuild_load_tree(struct erofs_inode *root, struct erofs_sb_info *sbi,
  	struct erofs_inode inode = {};
  	struct erofs_rebuild_dir_context ctx;
  	char uuid_str[37];
-	char *fsid = sbi->devname;
+	char *fsid = sbi->dif0.src_path;
  	int ret;
 
  	if (!fsid) {
lib/super.c +9 -9
index d138f84..ead4170 100644
@@ -33,7 +33,7 @@ static int erofs_init_devices(struct erofs_sb_info *sbi,
  	erofs_off_t pos;
  	bool _48bit = erofs_sb_has_48bit(sbi);
 
-	sbi->total_blocks = sbi->primarydevice_blocks;
+	sbi->total_blocks = sbi->dif0.blocks;
 
  	if (!erofs_sb_has_device_table(sbi))
  		ondisk_extradevs = 0;
@@ -118,14 +118,14 @@ int erofs_read_superblock(struct erofs_sb_info *sbi)
  		erofs_err("invalid sb_extslots %u", dsb->sb_extslots);
  		return -EINVAL;
  	}
-	sbi->primarydevice_blocks = le32_to_cpu(dsb->blocks_lo);
+	sbi->dif0.blocks = le32_to_cpu(dsb->blocks_lo);
  	sbi->meta_blkaddr = le32_to_cpu(dsb->meta_blkaddr);
  	sbi->xattr_blkaddr = le32_to_cpu(dsb->xattr_blkaddr);
  	sbi->xattr_prefix_start = le32_to_cpu(dsb->xattr_prefix_start);
  	sbi->xattr_prefix_count = dsb->xattr_prefix_count;
  	if (erofs_sb_has_48bit(sbi) && dsb->rootnid_8b) {
  		sbi->root_nid = le64_to_cpu(dsb->rootnid_8b);
-		sbi->primarydevice_blocks = sbi->primarydevice_blocks |
+		sbi->dif0.blocks = sbi->dif0.blocks |
  				((u64)le16_to_cpu(dsb->rb.blocks_hi) << 32);
  	} else {
  		sbi->root_nid = le16_to_cpu(dsb->rb.rootnid_2b);
@@ -230,10 +230,10 @@ int erofs_writesb(struct erofs_sb_info *sbi)
  	char *buf;
  	int ret;
 
-	sb.blocks_lo	= cpu_to_le32(sbi->primarydevice_blocks);
-	if (sbi->primarydevice_blocks > UINT32_MAX ||
+	sb.blocks_lo	= cpu_to_le32(sbi->dif0.blocks);
+	if (sbi->dif0.blocks > UINT32_MAX ||
  	    sbi->root_nid > UINT16_MAX) {
-		sb.rb.blocks_hi = cpu_to_le16(sbi->primarydevice_blocks >> 32);
+		sb.rb.blocks_hi = cpu_to_le16(sbi->dif0.blocks >> 32);
  		sb.rootnid_8b = cpu_to_le64(sbi->root_nid);
  	}
  	memcpy(sb.uuid, sbi->uuid, sizeof(sb.uuid));
@@ -404,7 +404,7 @@ int erofs_mkfs_init_devices(struct erofs_sb_info *sbi, unsigned int devices)
 
  int erofs_write_device_table(struct erofs_sb_info *sbi)
  {
-	erofs_blk_t nblocks = sbi->primarydevice_blocks;
+	erofs_blk_t nblocks = sbi->dif0.blocks;
  	struct erofs_buffer_head *bh = sbi->bh_devt;
  	erofs_off_t pos;
  	unsigned int i, ret;
@@ -484,7 +484,7 @@ int erofs_mkfs_load_fs(struct erofs_sb_info *sbi, unsigned int dsunit)
  	erofs_warn("EXPERIMENTAL incremental build in use. Use at your own risk!");
  	err = erofs_read_superblock(sbi);
  	if (err) {
-		erofs_err("failed to read superblock of %s: %s", sbi->devname,
+		erofs_err("failed to read superblock of %s: %s", sbi->dif0.src_path,
  			  erofs_strerror(err));
  		return err;
  	}
@@ -493,7 +493,7 @@ int erofs_mkfs_load_fs(struct erofs_sb_info *sbi, unsigned int dsunit)
  	if (!err && S_ISREG(u.st.st_mode))
  		u.startblk = DIV_ROUND_UP(u.st.st_size, erofs_blksiz(sbi));
  	else
-		u.startblk = sbi->primarydevice_blocks;
+		u.startblk = sbi->dif0.blocks;
 
  	bmgr = erofs_buffer_init(sbi, u.startblk, NULL);
  	if (!bmgr)
lib/vmdk.c +0 -0
index 316a8ff..28381c8 100644

lib/vmdk.c int erofs_dump_vmdk_desc(FILE *f, struct erofs_sb_info *sbi) · line 55

Quoted 7 lines
lib/vmdk.c +2 -2
@@ -51,8 +51,8 @@ int erofs_dump_vmdk_desc(FILE *f, struct erofs_sb_info *sbi)
  	int ret, i;
 
  	fprintf(f, desc_template_1, cid, parent_cid, subformat);
-	sectors = sbi->primarydevice_blocks << (sbi->blkszbits - 9);
-	ret = erofs_vmdk_desc_add_extent(f, sectors, (char *)sbi->devname, 0);
+	sectors = sbi->dif0.blocks << (sbi->blkszbits - 9);
+	ret = erofs_vmdk_desc_add_extent(f, sectors, (char *)sbi->dif0.blocks, 0);

mistake? should use src_path instead of blocks

Thanks,

Yifan Zhao

Quoted 51 lines
lib/vmdk.c +0 -0
  	if (ret)
  		return ret;
  	total_sectors = sectors;
mkfs/main.c +6 -6
index 929ce7c..5bf7b8f 100644
@@ -1685,12 +1685,12 @@ static int erofs_mkfs_rebuild_load_trees(struct erofs_inode *root)
  		ret = erofs_rebuild_load_tree(root, src, datamode);
  		src->xamgr = NULL;
  		if (ret) {
-			erofs_err("failed to load %s", src->devname);
+			erofs_err("failed to load %s", src->dif0.src_path);
  			return ret;
  		}
  		if (src->extra_devices > 1) {
  			erofs_err("%s: unsupported number %u of extra devices",
-				  src->devname, src->extra_devices);
+				  src->dif0.src_path, src->extra_devices);
  			return -EOPNOTSUPP;
  		}
  		extra_devices += src->extra_devices;
@@ -1723,8 +1723,8 @@ static int erofs_mkfs_rebuild_load_trees(struct erofs_inode *root)
  			nblocks = src->devs[0].blocks;
  			tag = src->devs[0].tag;
  		} else {
-			nblocks = src->primarydevice_blocks;
-			devs[idx].src_path = strdup(src->devname);
+			nblocks = src->dif0.blocks;
+			devs[idx].src_path = strdup(src->dif0.src_path);
  		}
  		devs[idx].blocks = nblocks;
  		if (tag && *tag)
@@ -1849,7 +1849,7 @@ int main(int argc, char **argv)
  			goto exit;
  		err = erofs_read_superblock(src);
  		if (err) {
-			erofs_err("failed to read superblock of %s", src->devname);
+			erofs_err("failed to read superblock of %s", src->dif0.src_path);
  			goto exit;
  		}
  		mkfs_blkszbits = src->blkszbits;
@@ -2051,7 +2051,7 @@ int main(int argc, char **argv)
  	if (err)
  		goto exit;
 
-	err = erofs_dev_resize(&g_sbi, g_sbi.primarydevice_blocks);
+	err = erofs_dev_resize(&g_sbi, g_sbi.dif0.blocks);
 
  	if (!err && erofs_sb_has_sb_chksum(&g_sbi)) {
  		err = erofs_enable_sb_chksum(&g_sbi, &crc);
author
#3 lore raw

Re: [PATCH 2/4] erofs-utils: lib: use `struct erofs_device_info` for the primary device

Hi Yifan,

Quoted 6 lines · Yifan Zhao
Quoted 4 lines · Gao Xiang

Source kernel commit: 7b00af2c5414dc01e0718deef7ead81102867636

---

...

Quoted 5 lines
Quoted 5 lines
lib/vmdk.c +0 -0
index 316a8ff..28381c8 100644
@@ -51,8 +51,8 @@ int erofs_dump_vmdk_desc(FILE *f, struct erofs_sb_info *sbi)
Quoted 8 lines

lib/vmdk.c int erofs_dump_vmdk_desc(FILE *f, struct erofs_sb_info *sbi) · line 55

Quoted 6 lines
lib/vmdk.c +2 -2
  	int ret, i;
  	fprintf(f, desc_template_1, cid, parent_cid, subformat);
-	sectors = sbi->primarydevice_blocks << (sbi->blkszbits - 9);
-	ret = erofs_vmdk_desc_add_extent(f, sectors, (char *)sbi->devname, 0);
+	sectors = sbi->dif0.blocks << (sbi->blkszbits - 9);
+	ret = erofs_vmdk_desc_add_extent(f, sectors, (char *)sbi->dif0.blocks, 0);

mistake? should use src_path instead of blocks

It's already got fixed.

Thanks,
Gao Xiang

Quoted 5 lines

Thanks,

Yifan Zhao