Skip to contentSkip to conversation

erofs-utils: support 64-bit internal buffer cache

PATCH In review Gao Xiang via Linux-erofs 2 participants 4 messages Oct 4, 2019 – Oct 5, 2019 Library code

lore
Share & tools
lore
#0 lore raw

PATCH 1/2 erofs-utils: support 64-bit internal buffer cache

From: Gao Xiang <[email protected]>

Previously, the type of off in struct erofs_buffer_head is unsigned int, it's not enough for large files.

Fix to a 64-bit field in order to support large files.

---
Hi Guifu,
Please take some time to review these patches since I'd like to release erofs-utils 1.0 recently.

Thanks,
Gao Xiang

 include/erofs/cache.h |  8 ++++----
 include/erofs/defs.h  |  4 ++++
 lib/cache.c           | 15 ++++++++-------
 3 files changed, 16 insertions(+), 11 deletions(-)
include/erofs/cache.h +4 -4
index 71df811..10a6aac 100644
@@ -31,7 +31,7 @@ struct erofs_buffer_head {
 	struct list_head list;
 	struct erofs_buffer_block *block;
 
-	unsigned int off;
+	erofs_off_t off;
 	struct erofs_bhops *op;
 
 	void *fsprivate;
@@ -87,13 +87,13 @@ static inline bool erofs_bh_flush_generic_end(struct erofs_buffer_head *bh)
 }
 
 struct erofs_buffer_head *erofs_buffer_init(void);
-int erofs_bh_balloon(struct erofs_buffer_head *bh, unsigned int incr);
+int erofs_bh_balloon(struct erofs_buffer_head *bh, erofs_off_t incr);
 
-struct erofs_buffer_head *erofs_balloc(int type, unsigned int size,
+struct erofs_buffer_head *erofs_balloc(int type, erofs_off_t size,
 				       unsigned int required_ext,
 				       unsigned int inline_ext);
 struct erofs_buffer_head *erofs_battach(struct erofs_buffer_head *bh,
-					int type, int size);
+					int type, unsigned int size);
 
 erofs_blk_t erofs_mapbh(struct erofs_buffer_block *bb, bool end);
 bool erofs_bflush(struct erofs_buffer_block *bb);
include/erofs/defs.h +4 -0
index 0d9910c..15db4e3 100644
@@ -136,6 +136,10 @@ typedef int64_t         s64;
 	type __max2 = (y);			\
 	__max1 > __max2 ? __max1: __max2; })
 
+#define sgn(x) ({		\
+	typeof(x) _x = (x);	\
+(_x > 0) - (_x < 0); })
+
 #define ARRAY_SIZE(arr)	(sizeof(arr) / sizeof((arr)[0]))
 
 #define BIT(nr)             (1UL << (nr))
lib/cache.c +8 -7
index 4f03cb9..41d2d5d 100644
@@ -74,14 +74,15 @@ struct erofs_buffer_head *erofs_buffer_init(void)
 /* return occupied bytes in specific buffer block if succeed */
 static int __erofs_battach(struct erofs_buffer_block *bb,
 			   struct erofs_buffer_head *bh,
-			   unsigned int incr,
+			   erofs_off_t incr,
 			   unsigned int alignsize,
 			   unsigned int extrasize,
 			   bool dryrun)
 {
-	const unsigned int alignedoffset = roundup(bb->buffers.off, alignsize);
-	const int oob = alignedoffset + incr + extrasize -
-			roundup(bb->buffers.off + 1, EROFS_BLKSIZ);
+	const erofs_off_t alignedoffset = roundup(bb->buffers.off, alignsize);
+	const int oob = sgn(roundup(bb->buffers.off % EROFS_BLKSIZ,
+				    alignsize) + incr + extrasize -
+			    EROFS_BLKSIZ);
 	bool tailupdate = false;
 	erofs_blk_t blkaddr;
 
@@ -113,7 +114,7 @@ static int __erofs_battach(struct erofs_buffer_block *bb,
 	return (alignedoffset + incr) % EROFS_BLKSIZ;
 }
 
-int erofs_bh_balloon(struct erofs_buffer_head *bh, unsigned int incr)
+int erofs_bh_balloon(struct erofs_buffer_head *bh, erofs_off_t incr)
 {
 	struct erofs_buffer_block *const bb = bh->block;
 
@@ -124,7 +125,7 @@ int erofs_bh_balloon(struct erofs_buffer_head *bh, unsigned int incr)
 	return __erofs_battach(bb, NULL, incr, 1, 0, false);
 }
 
-struct erofs_buffer_head *erofs_balloc(int type, unsigned int size,
+struct erofs_buffer_head *erofs_balloc(int type, erofs_off_t size,
 				       unsigned int required_ext,
 				       unsigned int inline_ext)
 {
@@ -214,7 +215,7 @@ found:
 }
 
 struct erofs_buffer_head *erofs_battach(struct erofs_buffer_head *bh,
-					int type, int size)
+					int type, unsigned int size)
 {
 	struct erofs_buffer_block *const bb = bh->block;
 	struct erofs_buffer_head *nbh;
Signature · 1 line
2.17.1
#1 lore raw

PATCH 2/2 erofs-utils: complete extended inode support

From: Gao Xiang <[email protected]>

There is no requirement to use extended inode (v2) for Android, but it's obviously useful for wider use cases.

extended inode was partially supported by obsoleted_mkfs, complete for new erofs-utils as well.

---
 include/erofs/config.h   |   6 ++
 include/erofs/internal.h |   2 +
 lib/compress.c           |   4 +-
 lib/config.c             |   1 +
 lib/inode.c              | 119 +++++++++++++++++++++++++++++++--------
 mkfs/main.c              |  26 +++++++--
 6 files changed, 125 insertions(+), 33 deletions(-)
include/erofs/config.h +6 -0
index 05fe6b2..8b09430 100644
@@ -11,6 +11,11 @@
 
 #include "defs.h"
 
+enum {
+	FORCE_INODE_COMPACT = 1,
+	FORCE_INODE_EXTENDED,
+};
+
 struct erofs_configure {
 	const char *c_version;
 	int c_dbg_lvl;
@@ -22,6 +27,7 @@ struct erofs_configure {
 	char *c_src_path;
 	char *c_compr_alg_master;
 	int c_compr_level_master;
+	int c_force_inodeversion;
 };
 
 extern struct erofs_configure cfg;
include/erofs/internal.h +2 -0
index 07a32d2..5384946 100644
@@ -53,6 +53,8 @@ struct erofs_sb_info {
 	erofs_blk_t xattr_blkaddr;
 
 	u32 feature_incompat;
+	u64 build_time;
+	u32 build_time_nsec;
 };
 
 /* global sbi */
lib/compress.c +2 -2
index 7935fce..7f65e5e 100644
@@ -423,8 +423,8 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
 	remaining = inode->i_size;
 
 	while (remaining) {
-		const uint readcount = min_t(uint, remaining,
-					     sizeof(ctx.queue) - ctx.tail);
+		const u64 readcount = min_t(u64, remaining,
+					    sizeof(ctx.queue) - ctx.tail);
 
 		ret = read(fd, ctx.queue + ctx.tail, readcount);
 		if (ret != readcount) {
lib/config.c +1 -0
index 9c78142..110c8b6 100644
@@ -22,6 +22,7 @@ void erofs_init_configure(void)
 	cfg.c_legacy_compress = false;
 	cfg.c_compr_level_master = -1;
 	sbi.feature_incompat = EROFS_FEATURE_INCOMPAT_LZ4_0PADDING;
+	cfg.c_force_inodeversion = 0;
 }
 
 void erofs_show_config(void)
lib/inode.c +94 -25

159 lines not on this page · raw message

mkfs/main.c +20 -6
index effc26b..4b279c0 100644
@@ -72,6 +72,18 @@ static int parse_extended_opts(const char *opts)
 			sbi.feature_incompat &=
 				~EROFS_FEATURE_INCOMPAT_LZ4_0PADDING;
 		}
+
+		if (MATCH_EXTENTED_OPT("force-inode-compact", token, keylen)) {
+			if (vallen)
+				return -EINVAL;
+			cfg.c_force_inodeversion = FORCE_INODE_COMPACT;
+		}
+
+		if (MATCH_EXTENTED_OPT("force-inode-extended", token, keylen)) {
+			if (vallen)
+				return -EINVAL;
+			cfg.c_force_inodeversion = FORCE_INODE_EXTENDED;
+		}
 	}
 	return 0;
 }
@@ -153,6 +165,8 @@ int erofs_mkfs_update_super_block(struct erofs_buffer_head *bh,
 		.magic     = cpu_to_le32(EROFS_SUPER_MAGIC_V1),
 		.blkszbits = LOG_BLOCK_SIZE,
 		.inos   = 0,
+		.build_time = cpu_to_le64(sbi.build_time),
+		.build_time_nsec = cpu_to_le32(sbi.build_time_nsec),
 		.blocks = 0,
 		.meta_blkaddr  = sbi.meta_blkaddr,
 		.xattr_blkaddr = 0,
@@ -161,12 +175,6 @@ int erofs_mkfs_update_super_block(struct erofs_buffer_head *bh,
 	const unsigned int sb_blksize =
 		round_up(EROFS_SUPER_END, EROFS_BLKSIZ);
 	char *buf;
-	struct timeval t;
-
-	if (!gettimeofday(&t, NULL)) {
-		sb.build_time      = cpu_to_le64(t.tv_sec);
-		sb.build_time_nsec = cpu_to_le32(t.tv_usec);
-	}
 
 	*blocks         = erofs_mapbh(NULL, true);
 	sb.blocks       = cpu_to_le32(*blocks);
@@ -193,6 +201,7 @@ int main(int argc, char **argv)
 	erofs_nid_t root_nid;
 	struct stat64 st;
 	erofs_blk_t nblocks;
+	struct timeval t;
 
 	erofs_init_configure();
 	fprintf(stderr, "%s %s\n", basename(argv[0]), cfg.c_version);
@@ -214,6 +223,11 @@ int main(int argc, char **argv)
 		return 1;
 	}
 
+	if (!gettimeofday(&t, NULL)) {
+		sbi.build_time      = t.tv_sec;
+		sbi.build_time_nsec = t.tv_usec;
+	}
+
 	err = dev_open(cfg.c_img_path);
 	if (err) {
 		usage();
Signature · 1 line
2.17.1
Reviewed-by Tested-by on v1 2/2
#2 lore raw

Re: [PATCH 2/2] erofs-utils: complete extended inode support

Quoted 10 lines

From: Gao Xiang <[email protected]>

There is no requirement to use extended inode (v2) for Android, but it's obviously useful for wider use cases.

extended inode was partially supported by obsoleted_mkfs, complete for new erofs-utils as well.

It looks good

Thanks,