Skip to contentSkip to conversation

erofs-utils: lib: fix improper alignment for chunked sparse files

PATCH New Gao Xiang 1 participant 3 messages Jul 24, 2023 Library code

lore
Share & tools
lore

Gao Xiang

author
#0 lore raw

PATCH 1/3 erofs-utils: lib: fix improper alignment for chunked sparse files

File position should be fixed to aligned with the chunk boundary. Otherwise, incorrect data could be read.

Fixes 7c49e8b195ad ("erofs-utils: support chunk-based sparse files")
---
 lib/blobchunk.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)
lib/blobchunk.c +5 -1
index 44541d5..c0df2f7 100644
@@ -228,8 +228,12 @@ int erofs_blob_write_chunked_file(struct erofs_inode *inode, int fd)
 				offset = pos;
 			else
 				offset = ((pos >> chunkbits) + 1) << chunkbits;
-		} else {
+		} else if (offset != (offset & ~(chunksize - 1))) {
 			offset &= ~(chunksize - 1);
+			if (lseek(fd, offset, SEEK_SET) != offset) {
+				ret = -EIO;
+				goto err;
+			}
 		}
 
 		if (offset > pos) {
Signature · 1 line
2.24.4
author
#1 lore raw

PATCH 2/3 erofs-utils: lib: tidy up erofs_blob_getchunk()

Mainly get rid of memory allocation on each chunk.

---
 lib/blobchunk.c | 92 +++++++++++++++++++++++++++----------------------
 1 file changed, 51 insertions(+), 41 deletions(-)
lib/blobchunk.c +51 -41
index c0df2f7..4619057 100644
@@ -51,60 +51,57 @@ struct erofs_blobchunk *erofs_get_unhashed_chunk(erofs_off_t chunksize,
 }
 
 static struct erofs_blobchunk *erofs_blob_getchunk(struct erofs_sb_info *sbi,
-		int fd, erofs_off_t chunksize)
+						u8 *buf, erofs_off_t chunksize)
 {
 	static u8 zeroed[EROFS_MAX_BLOCK_SIZE];
-	u8 *chunkdata, sha256[32];
-	int ret;
-	unsigned int hash;
-	erofs_off_t blkpos;
 	struct erofs_blobchunk *chunk;
+	unsigned int hash, padding;
+	u8 sha256[32];
+	erofs_off_t blkpos;
+	int ret;
 
-	chunkdata = malloc(chunksize);
-	if (!chunkdata)
-		return ERR_PTR(-ENOMEM);
-
-	ret = read(fd, chunkdata, chunksize);
-	if (ret < chunksize) {
-		chunk = ERR_PTR(-EIO);
-		goto out;
-	}
-	erofs_sha256(chunkdata, chunksize, sha256);
+	erofs_sha256(buf, chunksize, sha256);
 	hash = memhash(sha256, sizeof(sha256));
 	chunk = hashmap_get_from_hash(&blob_hashmap, hash, sha256);
 	if (chunk) {
 		DBG_BUGON(chunksize != chunk->chunksize);
-		goto out;
+		erofs_dbg("Found duplicated chunk at %u", chunk->blkaddr);
+		return chunk;
 	}
+
 	chunk = malloc(sizeof(struct erofs_blobchunk));
-	if (!chunk) {
-		chunk = ERR_PTR(-ENOMEM);
-		goto out;
-	}
+	if (!chunk)
+		return ERR_PTR(-ENOMEM);
 
 	chunk->chunksize = chunksize;
+	memcpy(chunk->sha256, sha256, sizeof(sha256));
 	blkpos = ftell(blobfile);
 	DBG_BUGON(erofs_blkoff(sbi, blkpos));
-	chunk->device_id = 0;
+
+	if (multidev)
+		chunk->device_id = 1;
+	else
+		chunk->device_id = 0;
 	chunk->blkaddr = erofs_blknr(sbi, blkpos);
-	memcpy(chunk->sha256, sha256, sizeof(sha256));
-	hashmap_entry_init(&chunk->ent, hash);
-	hashmap_add(&blob_hashmap, chunk);
 
 	erofs_dbg("Writing chunk (%u bytes) to %u", chunksize, chunk->blkaddr);
-	ret = fwrite(chunkdata, chunksize, 1, blobfile);
-	if (ret == 1 && erofs_blkoff(sbi, chunksize))
-		ret = fwrite(zeroed,
-			     erofs_blksiz(sbi) - erofs_blkoff(sbi, chunksize),
-			     1, blobfile);
+	ret = fwrite(buf, chunksize, 1, blobfile);
+	padding = erofs_blkoff(sbi, chunksize);
+	if (ret == 1) {
+		padding = erofs_blkoff(sbi, chunksize);
+		if (padding) {
+			padding = erofs_blksiz(sbi) - padding;
+			ret = fwrite(zeroed, padding, 1, blobfile);
+		}
+	}
+
 	if (ret < 1) {
-		hashmap_remove(&blob_hashmap, &chunk->ent);
 		free(chunk);
-		chunk = ERR_PTR(-ENOSPC);
-		goto out;
+		return ERR_PTR(-ENOSPC);
 	}
-out:
-	free(chunkdata);
+
+	hashmap_entry_init(&chunk->ent, hash);
+	hashmap_add(&blob_hashmap, chunk);
 	return chunk;
 }
 
@@ -189,6 +186,7 @@ int erofs_blob_write_chunked_file(struct erofs_inode *inode, int fd)
 	unsigned int count, unit;
 	struct erofs_inode_chunk_index *idx;
 	erofs_off_t pos, len, chunksize;
+	u8 *chunkdata;
 	int ret;
 
 #ifdef SEEK_DATA
@@ -212,11 +210,17 @@ int erofs_blob_write_chunked_file(struct erofs_inode *inode, int fd)
 	else
 		unit = EROFS_BLOCK_MAP_ENTRY_SIZE;
 
-	inode->extent_isize = count * unit;
-	idx = malloc(count * max(sizeof(*idx), sizeof(void *)));
-	if (!idx)
+	chunkdata = malloc(chunksize);
+	if (!chunkdata)
 		return -ENOMEM;
-	inode->chunkindexes = idx;
+
+	inode->extent_isize = count * unit;
+	inode->chunkindexes = malloc(count * max(sizeof(*idx), sizeof(void *)));
+	if (!inode->chunkindexes) {
+		ret = -ENOMEM;
+		goto err;
+	}
+	idx = inode->chunkindexes;
 
 	for (pos = 0; pos < inode->i_size; pos += len) {
 		struct erofs_blobchunk *chunk;
@@ -248,20 +252,26 @@ int erofs_blob_write_chunked_file(struct erofs_inode *inode, int fd)
 #endif
 
 		len = min_t(u64, inode->i_size - pos, chunksize);
-		chunk = erofs_blob_getchunk(sbi, fd, len);
+		ret = read(fd, chunkdata, len);
+		if (ret < len) {
+			ret = -EIO;
+			goto err;
+		}
+
+		chunk = erofs_blob_getchunk(sbi, chunkdata, len);
 		if (IS_ERR(chunk)) {
 			ret = PTR_ERR(chunk);
 			goto err;
 		}
-		if (multidev)
-			chunk->device_id = 1;
 		*(void **)idx++ = chunk;
 	}
 	inode->datalayout = EROFS_INODE_CHUNK_BASED;
+	free(chunkdata);
 	return 0;
 err:
 	free(inode->chunkindexes);
 	inode->chunkindexes = NULL;
+	free(chunkdata);
 	return ret;
 }
 
Signature · 1 line
2.24.4
author
#2 lore raw

PATCH 3/3 erofs-utils: merge consecutive chunks if possible

In reply to Gao Xiang

Since EROFS chunk size can be configured on a per-file basis, let's generate indexes with the best chunk size.

---
 include/erofs/defs.h |  5 +++++
 lib/blobchunk.c      | 53 ++++++++++++++++++++++++++++++++++++++++----
 2 files changed, 54 insertions(+), 4 deletions(-)
include/erofs/defs.h +5 -0
index 44af557..20f9741 100644
@@ -286,6 +286,11 @@ static inline unsigned int fls_long(unsigned long x)
 	return x ? sizeof(x) * 8 - __builtin_clz(x) : 0;
 }
 
+static inline unsigned long lowbit(unsigned long n)
+{
+	return n & -n;
+}
+
 /**
  * __roundup_pow_of_two() - round up to nearest power of two
  * @n: value to round up
lib/blobchunk.c +49 -4

97 lines not on this page · raw message

Signature · 1 line
2.24.4