Skip to contentSkip to conversation

erofs-utils: fix handling of sparse files

PATCH Superseded Sandeep Dhavale via Linux-erofs 2 participants 4 messages Apr 3, 2024 Library code

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

cover erofs-utils: fix handling of sparse files

Hi,
While working on a enhancement where blocks filled with zeros can be treated as a hole (sparse file), I stumbled upon a bug. I found during mkfs.erofs, the utility handles the sparse files by looking at the holes and marking them as a erofs_holechunk. However the calculation of minextblks need to consider the contiguous valid data blocks only else we end up wrongly merging the blobchunks and creating inconsistent erofs image.

This can be easily reproduced with below script which creates a file and punches few holes.

$ cat repro.sh
#!/bin/bash
dd if=/dev/urandom of=sample_sparse_file bs=4096 count=49
fallocate --punch-hole --offset 36864 --length 28672 sample_sparse_file
fallocate --punch-hole --offset 143360 --length 53248 sample_sparse_file
filefrag -v sample_sparse_file
mkdir erofs_image_data
cp --sparse=always sample_sparse_file erofs_image_data/
mkfs.erofs --chunksize=4096 problem_erofs.img erofs_image_data/

You can see that if you mount such image, you will have IO errors.

$ md5sum mountpt/*
md5sum: mountpt/sample_sparse_file: Input/output error

The patch addresses this by tracking the start of contiguous data blocks and calculating minextblks correctly. So merging of chunks still happens if possible.

Sandeep Dhavale (1):
erofs-utils: lib: Fix calculation of minextblks when working with
sparse files

 lib/blobchunk.c | 28 ++++++++++++++++++++++------
 1 file changed, 22 insertions(+), 6 deletions(-)
Signature · 1 line
2.44.0.478.gd926399ef9-goog
#1 lore raw

PATCH 1/1 erofs-utils: lib: Fix calculation of minextblks when working with sparse files

When we work with sparse files (files with holes), we need to consider when the contiguous data block starts after each hole to correctly calculate minextblks so we can merge consecutive chunks later.
Now that we need to recalculate minextblks multiple places, put the logic in helper function for avoiding repetition and easier reading.

Fixes 7b46f7a0160a (erofs-utils: lib: merge consecutive chunks if possible)
---
 lib/blobchunk.c | 28 ++++++++++++++++++++++------
 1 file changed, 22 insertions(+), 6 deletions(-)
lib/blobchunk.c +22 -6
index ee12194..41bee19 100644
@@ -223,6 +223,16 @@ out:
 	return 0;
 }
 
+static erofs_blk_t erofs_minblks_from_interval(struct erofs_sb_info *sbi,
+			erofs_off_t start, erofs_off_t end, erofs_blk_t cur_min)
+{
+	erofs_blk_t lb;
+	lb = lowbit((end - start) >> sbi->blkszbits);
+	if (lb && lb < cur_min)
+		return lb;
+	return cur_min;
+}
+
 int erofs_blob_write_chunked_file(struct erofs_inode *inode, int fd,
 				  erofs_off_t startoff)
 {
@@ -231,8 +241,8 @@ int erofs_blob_write_chunked_file(struct erofs_inode *inode, int fd,
 	unsigned int count, unit;
 	struct erofs_blobchunk *chunk, *lastch;
 	struct erofs_inode_chunk_index *idx;
-	erofs_off_t pos, len, chunksize;
-	erofs_blk_t lb, minextblks;
+	erofs_off_t pos, len, chunksize, interval_start;
+	erofs_blk_t minextblks;
 	u8 *chunkdata;
 	int ret;
 
@@ -267,9 +277,10 @@ int erofs_blob_write_chunked_file(struct erofs_inode *inode, int fd,
 		goto err;
 	}
 	idx = inode->chunkindexes;
-
 	lastch = NULL;
 	minextblks = BLK_ROUND_UP(sbi, inode->i_size);
+	interval_start = 0;
+
 	for (pos = 0; pos < inode->i_size; pos += len) {
 #ifdef SEEK_DATA
 		off_t offset = lseek(fd, pos + startoff, SEEK_DATA);
@@ -294,12 +305,15 @@ int erofs_blob_write_chunked_file(struct erofs_inode *inode, int fd,
 
 		if (offset > pos) {
 			len = 0;
+			minextblks = erofs_minblks_from_interval(sbi,
+					   interval_start, pos, minextblks);
 			do {
 				*(void **)idx++ = &erofs_holechunk;
 				pos += chunksize;
 			} while (pos < offset);
 			DBG_BUGON(pos != offset);
 			lastch = NULL;
+			interval_start = pos;
 			continue;
 		}
 #endif
@@ -320,13 +334,15 @@ int erofs_blob_write_chunked_file(struct erofs_inode *inode, int fd,
 		if (lastch && (lastch->device_id != chunk->device_id ||
 		    erofs_pos(sbi, lastch->blkaddr) + lastch->chunksize !=
 		    erofs_pos(sbi, chunk->blkaddr))) {
-			lb = lowbit(pos >> sbi->blkszbits);
-			if (lb && lb < minextblks)
-				minextblks = lb;
+			minextblks = erofs_minblks_from_interval(sbi,
+					    interval_start, pos, minextblks);
+			interval_start = pos;
 		}
 		*(void **)idx++ = chunk;
 		lastch = chunk;
 	}
+	minextblks = erofs_minblks_from_interval(sbi, interval_start, pos,
+					minextblks);
 	inode->datalayout = EROFS_INODE_CHUNK_BASED;
 	free(chunkdata);
 	return erofs_blob_mergechunks(inode, chunkbits,
Signature · 1 line
2.44.0.478.gd926399ef9-goog
#2 lore raw

Re: [PATCH v1 1/1] erofs-utils: lib: Fix calculation of minextblks when working with sparse files

Hi Sandeep,

Quoted 8 lines · Sandeep Dhavale

When we work with sparse files (files with holes), we need to consider when the contiguous data block starts after each hole to correctly calculate minextblks so we can merge consecutive chunks later.
Now that we need to recalculate minextblks multiple places, put the logic in helper function for avoiding repetition and easier reading.

Fixes 7b46f7a0160a (erofs-utils: lib: merge consecutive chunks if possible)

Thanks for catching this!

Quoted 11 lines
---
  lib/blobchunk.c | 28 ++++++++++++++++++++++------
  1 file changed, 22 insertions(+), 6 deletions(-)
lib/blobchunk.c +0 -0
index ee12194..41bee19 100644
@@ -223,6 +223,16 @@ out:
  	return 0;
  }

lib/blobchunk.c out: · line 234

Quoted 10 lines
lib/blobchunk.c +9 -0
 
+static erofs_blk_t erofs_minblks_from_interval(struct erofs_sb_info *sbi,
+			erofs_off_t start, erofs_off_t end, erofs_blk_t cur_min)
+{
+	erofs_blk_t lb;
+	lb = lowbit((end - start) >> sbi->blkszbits);
+	if (lb && lb < cur_min)
+		return lb;
+	return cur_min;
+}

Just a minor thought, maybe

static erofs_blk_t erofs_update_minextblks(struct erofs_sb_info *sbi,
erofs_off_t start, erofs_off_t end, erofs_blk_t *minextblks)

{
	erofs_blk_t lb;

	lb = lowbit((end - start) >> sbi->blkszbits);
	if (lb && lb < *minextblks)
		*minextblks = lb;
}

could be better (or just my own perference) since no input + output on a same variable, otherwise it looks good to me.

If you agree on this minor nit, I could revise it directly or you could send another version. Both are fine with me :)

Thanks,
Gao Xiang

Quoted 62 lines
lib/blobchunk.c +13 -6
+
  int erofs_blob_write_chunked_file(struct erofs_inode *inode, int fd,
  				  erofs_off_t startoff)
  {
@@ -231,8 +241,8 @@ int erofs_blob_write_chunked_file(struct erofs_inode *inode, int fd,
  	unsigned int count, unit;
  	struct erofs_blobchunk *chunk, *lastch;
  	struct erofs_inode_chunk_index *idx;
-	erofs_off_t pos, len, chunksize;
-	erofs_blk_t lb, minextblks;
+	erofs_off_t pos, len, chunksize, interval_start;
+	erofs_blk_t minextblks;
  	u8 *chunkdata;
  	int ret;
 
@@ -267,9 +277,10 @@ int erofs_blob_write_chunked_file(struct erofs_inode *inode, int fd,
  		goto err;
  	}
  	idx = inode->chunkindexes;
-
  	lastch = NULL;
  	minextblks = BLK_ROUND_UP(sbi, inode->i_size);
+	interval_start = 0;
+
  	for (pos = 0; pos < inode->i_size; pos += len) {
  #ifdef SEEK_DATA
  		off_t offset = lseek(fd, pos + startoff, SEEK_DATA);
@@ -294,12 +305,15 @@ int erofs_blob_write_chunked_file(struct erofs_inode *inode, int fd,
 
  		if (offset > pos) {
  			len = 0;
+			minextblks = erofs_minblks_from_interval(sbi,
+					   interval_start, pos, minextblks);
  			do {
  				*(void **)idx++ = &erofs_holechunk;
  				pos += chunksize;
  			} while (pos < offset);
  			DBG_BUGON(pos != offset);
  			lastch = NULL;
+			interval_start = pos;
  			continue;
  		}
  #endif
@@ -320,13 +334,15 @@ int erofs_blob_write_chunked_file(struct erofs_inode *inode, int fd,
  		if (lastch && (lastch->device_id != chunk->device_id ||
  		    erofs_pos(sbi, lastch->blkaddr) + lastch->chunksize !=
  		    erofs_pos(sbi, chunk->blkaddr))) {
-			lb = lowbit(pos >> sbi->blkszbits);
-			if (lb && lb < minextblks)
-				minextblks = lb;
+			minextblks = erofs_minblks_from_interval(sbi,
+					    interval_start, pos, minextblks);
+			interval_start = pos;
  		}
  		*(void **)idx++ = chunk;
  		lastch = chunk;
  	}
+	minextblks = erofs_minblks_from_interval(sbi, interval_start, pos,
+					minextblks);
  	inode->datalayout = EROFS_INODE_CHUNK_BASED;
  	free(chunkdata);
  	return erofs_blob_mergechunks(inode, chunkbits,
#3 lore raw

Re: [PATCH v1 1/1] erofs-utils: lib: Fix calculation of minextblks when working with sparse files

Hi Gao,
Thank you for the review. I will update and send the v2.

Thanks,
Sandeep.