Skip to contentSkip to conversation

erofs-utils: fsck: support fragments

RFC PATCH Superseded Yue Hu 2 participants 7 messages Dec 24, 2022 – Jan 4, 2023 Library code

v1 of 3 Superseded Latest is v3 →

lore
Share & tools
lore

Yue Hu

author
#0 lore raw

RFC PATCH 1/1 erofs-utils: fsck: support fragments

From: Yue Hu <[email protected]>

Add compressed fragments support for fsck.erofs.

---
 fsck/main.c | 41 +++++++++++++++++++++++++++++++++++++++--
 lib/zmap.c  |  1 +
 2 files changed, 40 insertions(+), 2 deletions(-)
fsck/main.c +39 -2
index 2a9c501..9babc61 100644
@@ -421,6 +421,31 @@ static int erofs_verify_inode_data(struct erofs_inode *inode, int outfd)
 		if (!(map.m_flags & EROFS_MAP_MAPPED) || !fsckcfg.check_decomp)
 			continue;
 
+		if (map.m_flags & EROFS_MAP_FRAGMENT) {
+			struct erofs_inode packed_inode = {
+				.nid = sbi.packed_nid,
+			};
+
+			ret = erofs_read_inode_from_disk(&packed_inode);
+			if (ret) {
+				erofs_err("failed to read packed inode from disk");
+				goto out;
+			}
+
+			if (!buffer || map.m_llen > buffer_size) {
+				buffer_size = map.m_llen;
+				buffer = realloc(buffer, map.m_llen);
+				BUG_ON(!buffer);
+			}
+			ret = erofs_pread(&packed_inode, buffer, map.m_llen,
+					  inode->fragmentoff);
+			if (ret)
+				goto out;
+
+			compressed = true;	/* force using buffer */
+			goto write_outfd;
+		}
+
 		if (map.m_plen > raw_size) {
 			raw_size = map.m_plen;
 			raw = realloc(raw, raw_size);
@@ -476,6 +501,7 @@ static int erofs_verify_inode_data(struct erofs_inode *inode, int outfd)
 			}
 		}
 
+write_outfd:
 		if (outfd >= 0 && write(outfd, compressed ? buffer : raw,
 					map.m_llen) < 0) {
 			erofs_err("I/O error occurred when verifying data chunk @ nid %llu",
@@ -486,8 +512,9 @@ static int erofs_verify_inode_data(struct erofs_inode *inode, int outfd)
 	}
 
 	if (fsckcfg.print_comp_ratio) {
-		fsckcfg.logical_blocks += BLK_ROUND_UP(inode->i_size);
 		fsckcfg.physical_blocks += BLK_ROUND_UP(pchunk_len);
+		if (!erofs_is_packed_inode(inode))
+			fsckcfg.logical_blocks += BLK_ROUND_UP(inode->i_size);
 	}
 out:
 	if (raw)
@@ -732,6 +759,8 @@ static int erofsfsck_check_inode(erofs_nid_t pnid, erofs_nid_t nid)
 			ret = erofs_extract_dir(&inode);
 			break;
 		case S_IFREG:
+			if (erofs_is_packed_inode(&inode))
+				goto verify;
 			ret = erofs_extract_file(&inode);
 			break;
 		case S_IFLNK:
@@ -767,7 +796,7 @@ verify:
 		ret = erofs_iterate_dir(&ctx, true);
 	}
 
-	if (!ret)
+	if (!ret && !erofs_is_packed_inode(&inode))
 		erofsfsck_set_attributes(&inode, fsckcfg.extract_path);
 
 	if (ret == -ECANCELED)
@@ -822,6 +851,14 @@ int main(int argc, char **argv)
 		goto exit_put_super;
 	}
 
+	if (erofs_sb_has_fragments()) {
+		err = erofsfsck_check_inode(sbi.packed_nid, sbi.packed_nid);
+		if (err) {
+			erofs_err("failed to verify packed file");
+			goto exit_put_super;
+		}
+	}
+
 	err = erofsfsck_check_inode(sbi.root_nid, sbi.root_nid);
 	if (fsckcfg.corrupted) {
 		if (!fsckcfg.extract_path)
lib/zmap.c +1 -0
index 41e0713..ca65038 100644
@@ -639,6 +639,7 @@ static int z_erofs_do_map_blocks(struct erofs_inode *vi,
 		map->m_plen = vi->z_idata_size;
 	} else if (fragment && m.lcn == vi->z_tailextent_headlcn) {
 		map->m_flags |= EROFS_MAP_FRAGMENT;
+		map->m_plen = 0;
 	} else {
 		map->m_pa = blknr_to_addr(m.pblk);
 		err = z_erofs_get_extent_compressedlen(&m, initial_lcn);
Signature · 1 line
2.17.1
#1 lore raw

Hi Yue,

Quoted 15 lines · Yue Hu

From: Yue Hu <[email protected]>

Add compressed fragments support for fsck.erofs.

---
  fsck/main.c | 41 +++++++++++++++++++++++++++++++++++++++--
  lib/zmap.c  |  1 +
  2 files changed, 40 insertions(+), 2 deletions(-)
fsck/main.c +0 -0
index 2a9c501..9babc61 100644

fsck/main.c static int erofs_verify_inode_data(struct erofs_inode *inode, int outfd) · line 427

Quoted 7 lines
fsck/main.c +4 -0
@@ -421,6 +421,31 @@ static int erofs_verify_inode_data(struct erofs_inode *inode, int outfd)
  		if (!(map.m_flags & EROFS_MAP_MAPPED) || !fsckcfg.check_decomp)
  			continue;
 
+		if (map.m_flags & EROFS_MAP_FRAGMENT) {
+			struct erofs_inode packed_inode = {
+				.nid = sbi.packed_nid,
+			};

Sorry for late response.

a question... why don't we just rely on z_erofs_read_data()?

Thanks,
Gao Xiang

Quoted 88 lines
fsck/main.c +35 -2
+
+			ret = erofs_read_inode_from_disk(&packed_inode);
+			if (ret) {
+				erofs_err("failed to read packed inode from disk");
+				goto out;
+			}
+
+			if (!buffer || map.m_llen > buffer_size) {
+				buffer_size = map.m_llen;
+				buffer = realloc(buffer, map.m_llen);
+				BUG_ON(!buffer);
+			}
+			ret = erofs_pread(&packed_inode, buffer, map.m_llen,
+					  inode->fragmentoff);
+			if (ret)
+				goto out;
+
+			compressed = true;	/* force using buffer */
+			goto write_outfd;
+		}
+
  		if (map.m_plen > raw_size) {
  			raw_size = map.m_plen;
  			raw = realloc(raw, raw_size);
@@ -476,6 +501,7 @@ static int erofs_verify_inode_data(struct erofs_inode *inode, int outfd)
  			}
  		}
 
+write_outfd:
  		if (outfd >= 0 && write(outfd, compressed ? buffer : raw,
  					map.m_llen) < 0) {
  			erofs_err("I/O error occurred when verifying data chunk @ nid %llu",
@@ -486,8 +512,9 @@ static int erofs_verify_inode_data(struct erofs_inode *inode, int outfd)
  	}
 
  	if (fsckcfg.print_comp_ratio) {
-		fsckcfg.logical_blocks += BLK_ROUND_UP(inode->i_size);
  		fsckcfg.physical_blocks += BLK_ROUND_UP(pchunk_len);
+		if (!erofs_is_packed_inode(inode))
+			fsckcfg.logical_blocks += BLK_ROUND_UP(inode->i_size);
  	}
  out:
  	if (raw)
@@ -732,6 +759,8 @@ static int erofsfsck_check_inode(erofs_nid_t pnid, erofs_nid_t nid)
  			ret = erofs_extract_dir(&inode);
  			break;
  		case S_IFREG:
+			if (erofs_is_packed_inode(&inode))
+				goto verify;
  			ret = erofs_extract_file(&inode);
  			break;
  		case S_IFLNK:
@@ -767,7 +796,7 @@ verify:
  		ret = erofs_iterate_dir(&ctx, true);
  	}
 
-	if (!ret)
+	if (!ret && !erofs_is_packed_inode(&inode))
  		erofsfsck_set_attributes(&inode, fsckcfg.extract_path);
 
  	if (ret == -ECANCELED)
@@ -822,6 +851,14 @@ int main(int argc, char **argv)
  		goto exit_put_super;
  	}
 
+	if (erofs_sb_has_fragments()) {
+		err = erofsfsck_check_inode(sbi.packed_nid, sbi.packed_nid);
+		if (err) {
+			erofs_err("failed to verify packed file");
+			goto exit_put_super;
+		}
+	}
+
  	err = erofsfsck_check_inode(sbi.root_nid, sbi.root_nid);
  	if (fsckcfg.corrupted) {
  		if (!fsckcfg.extract_path)
lib/zmap.c +1 -0
index 41e0713..ca65038 100644
@@ -639,6 +639,7 @@ static int z_erofs_do_map_blocks(struct erofs_inode *vi,
  		map->m_plen = vi->z_idata_size;
  	} else if (fragment && m.lcn == vi->z_tailextent_headlcn) {
  		map->m_flags |= EROFS_MAP_FRAGMENT;
+		map->m_plen = 0;
  	} else {
  		map->m_pa = blknr_to_addr(m.pblk);
  		err = z_erofs_get_extent_compressedlen(&m, initial_lcn);
author
#2 lore raw

Hi Xiang,

Quoted 21 lines · On Wed, 4 Jan 2023 10:48:40 +0800 Xiang Gao <[email protected]> wrote:

Hi Yue,

Quoted 18 lines · Yue Hu

From: Yue Hu <[email protected]>

Add compressed fragments support for fsck.erofs.

---
  fsck/main.c | 41 +++++++++++++++++++++++++++++++++++++++--
  lib/zmap.c  |  1 +
  2 files changed, 40 insertions(+), 2 deletions(-)
fsck/main.c +0 -0
index 2a9c501..9babc61 100644
@@ -421,6 +421,31 @@ static int erofs_verify_inode_data(struct erofs_inode *inode, int outfd)
  		if (!(map.m_flags & EROFS_MAP_MAPPED) || !fsckcfg.check_decomp)
  			continue;
 
Quoted 8 lines

fsck/main.c static int erofs_verify_inode_data(struct erofs_inode *inode, int outfd) · line 427

Quoted 4 lines
fsck/main.c +4 -0
+		if (map.m_flags & EROFS_MAP_FRAGMENT) {
+			struct erofs_inode packed_inode = {
+				.nid = sbi.packed_nid,
+			};  

Sorry for late response.

a question... why don't we just rely on z_erofs_read_data()?

We may fall back to uncompressed mode for packed inode due to 'ENOSPC'.

Quoted 93 lines

Thanks,
Gao Xiang

Quoted 88 lines
fsck/main.c +35 -2
+
+			ret = erofs_read_inode_from_disk(&packed_inode);
+			if (ret) {
+				erofs_err("failed to read packed inode from disk");
+				goto out;
+			}
+
+			if (!buffer || map.m_llen > buffer_size) {
+				buffer_size = map.m_llen;
+				buffer = realloc(buffer, map.m_llen);
+				BUG_ON(!buffer);
+			}
+			ret = erofs_pread(&packed_inode, buffer, map.m_llen,
+					  inode->fragmentoff);
+			if (ret)
+				goto out;
+
+			compressed = true;	/* force using buffer */
+			goto write_outfd;
+		}
+
  		if (map.m_plen > raw_size) {
  			raw_size = map.m_plen;
  			raw = realloc(raw, raw_size);
@@ -476,6 +501,7 @@ static int erofs_verify_inode_data(struct erofs_inode *inode, int outfd)
  			}
  		}
 
+write_outfd:
  		if (outfd >= 0 && write(outfd, compressed ? buffer : raw,
  					map.m_llen) < 0) {
  			erofs_err("I/O error occurred when verifying data chunk @ nid %llu",
@@ -486,8 +512,9 @@ static int erofs_verify_inode_data(struct erofs_inode *inode, int outfd)
  	}
 
  	if (fsckcfg.print_comp_ratio) {
-		fsckcfg.logical_blocks += BLK_ROUND_UP(inode->i_size);
  		fsckcfg.physical_blocks += BLK_ROUND_UP(pchunk_len);
+		if (!erofs_is_packed_inode(inode))
+			fsckcfg.logical_blocks += BLK_ROUND_UP(inode->i_size);
  	}
  out:
  	if (raw)
@@ -732,6 +759,8 @@ static int erofsfsck_check_inode(erofs_nid_t pnid, erofs_nid_t nid)
  			ret = erofs_extract_dir(&inode);
  			break;
  		case S_IFREG:
+			if (erofs_is_packed_inode(&inode))
+				goto verify;
  			ret = erofs_extract_file(&inode);
  			break;
  		case S_IFLNK:
@@ -767,7 +796,7 @@ verify:
  		ret = erofs_iterate_dir(&ctx, true);
  	}
 
-	if (!ret)
+	if (!ret && !erofs_is_packed_inode(&inode))
  		erofsfsck_set_attributes(&inode, fsckcfg.extract_path);
 
  	if (ret == -ECANCELED)
@@ -822,6 +851,14 @@ int main(int argc, char **argv)
  		goto exit_put_super;
  	}
 
+	if (erofs_sb_has_fragments()) {
+		err = erofsfsck_check_inode(sbi.packed_nid, sbi.packed_nid);
+		if (err) {
+			erofs_err("failed to verify packed file");
+			goto exit_put_super;
+		}
+	}
+
  	err = erofsfsck_check_inode(sbi.root_nid, sbi.root_nid);
  	if (fsckcfg.corrupted) {
  		if (!fsckcfg.extract_path)
lib/zmap.c +1 -0
index 41e0713..ca65038 100644
@@ -639,6 +639,7 @@ static int z_erofs_do_map_blocks(struct erofs_inode *vi,
  		map->m_plen = vi->z_idata_size;
  	} else if (fragment && m.lcn == vi->z_tailextent_headlcn) {
  		map->m_flags |= EROFS_MAP_FRAGMENT;
+		map->m_plen = 0;
  	} else {
  		map->m_pa = blknr_to_addr(m.pblk);
  		err = z_erofs_get_extent_compressedlen(&m, initial_lcn);  
#3 lore raw
Quoted 33 lines · Yue Hu

Hi Xiang,

Quoted 28 lines · On Wed, 4 Jan 2023 10:48:40 +0800 Xiang Gao <[email protected]> wrote:

Hi Yue,

Quoted 15 lines · Yue Hu

From: Yue Hu <[email protected]>

Add compressed fragments support for fsck.erofs.

---
   fsck/main.c | 41 +++++++++++++++++++++++++++++++++++++++--
   lib/zmap.c  |  1 +
   2 files changed, 40 insertions(+), 2 deletions(-)
fsck/main.c +0 -0
index 2a9c501..9babc61 100644

fsck/main.c static int erofs_verify_inode_data(struct erofs_inode *inode, int outfd) · line 427

Quoted 7 lines
fsck/main.c +4 -0
@@ -421,6 +421,31 @@ static int erofs_verify_inode_data(struct erofs_inode *inode, int outfd)
   		if (!(map.m_flags & EROFS_MAP_MAPPED) || !fsckcfg.check_decomp)
   			continue;
 
+		if (map.m_flags & EROFS_MAP_FRAGMENT) {
+			struct erofs_inode packed_inode = {
+				.nid = sbi.packed_nid,
+			};

Sorry for late response.

Quoted 3 lines
Quoted 1 line

a question... why don't we just rely on z_erofs_read_data()?

We may fall back to uncompressed mode for packed inode due to 'ENOSPC'.

I think we could just export parts of erofs_pread() to clean up the whole erofs_verify_inode_data()...

Thanks,
Gao Xiang

author
#4 lore raw
Quoted 34 lines · On Wed, 4 Jan 2023 11:32:36 +0800 Xiang Gao <[email protected]> wrote:
Quoted 33 lines · Yue Hu

Hi Xiang,

Quoted 28 lines · On Wed, 4 Jan 2023 10:48:40 +0800 Xiang Gao <[email protected]> wrote:

Hi Yue,

Quoted 15 lines · Yue Hu

From: Yue Hu <[email protected]>

Add compressed fragments support for fsck.erofs.

---
   fsck/main.c | 41 +++++++++++++++++++++++++++++++++++++++--
   lib/zmap.c  |  1 +
   2 files changed, 40 insertions(+), 2 deletions(-)
fsck/main.c +0 -0
index 2a9c501..9babc61 100644

fsck/main.c static int erofs_verify_inode_data(struct erofs_inode *inode, int outfd) · line 427

Quoted 7 lines
fsck/main.c +4 -0
@@ -421,6 +421,31 @@ static int erofs_verify_inode_data(struct erofs_inode *inode, int outfd)
   		if (!(map.m_flags & EROFS_MAP_MAPPED) || !fsckcfg.check_decomp)
   			continue;
 
+		if (map.m_flags & EROFS_MAP_FRAGMENT) {
+			struct erofs_inode packed_inode = {
+				.nid = sbi.packed_nid,
+			};  

Sorry for late response.

Quoted 6 lines
Quoted 3 lines
Quoted 1 line

a question... why don't we just rely on z_erofs_read_data()?

We may fall back to uncompressed mode for packed inode due to 'ENOSPC'.

I think we could just export parts of erofs_pread() to clean up the whole erofs_verify_inode_data()...

What's the clean up referring to?

However, i think erofs_verify_inode_data() looks a little duplicate compared to erofs_read_raw_data/z_erofs_read_data().

Quoted 5 lines

Thanks,
Gao Xiang

#5 lore raw

On 2023/1/4 15:26, Yue Hu wrote:

...

Quoted 7 lines
Quoted 3 lines

I think we could just export parts of erofs_pread() to clean up the whole erofs_verify_inode_data()...

What's the clean up referring to?

However, i think erofs_verify_inode_data() looks a little duplicate compared to erofs_read_raw_data/z_erofs_read_data().

We should reuse the main part (for example, by introducing an interface to accept mapping) of erofs_read_raw_data() and z_erofs_read_data() to avoid duplicated code in erofs_verify_inode_data().

Thanks,
Gao Xiang

Quoted 6 lines
Quoted 6 lines

Thanks,
Gao Xiang

author
#6 lore raw
Quoted 8 lines · On Wed, 4 Jan 2023 15:28:42 +0800 Xiang Gao <[email protected]> wrote:

On 2023/1/4 15:26, Yue Hu wrote:

...

Quoted 4 lines
Quoted 3 lines

I think we could just export parts of erofs_pread() to clean up the whole erofs_verify_inode_data()...

Quoted 8 lines
Quoted 3 lines

What's the clean up referring to?

However, i think erofs_verify_inode_data() looks a little duplicate compared to erofs_read_raw_data/z_erofs_read_data().

We should reuse the main part (for example, by introducing an interface to accept mapping) of
erofs_read_raw_data() and z_erofs_read_data() to avoid duplicated code in erofs_verify_inode_data().

Yes, it's. Let's do it.

Quoted 10 lines

Thanks,
Gao Xiang

Quoted 6 lines
Quoted 6 lines

Thanks,
Gao Xiang