btrfs: Allow read-only mount with corrupted extent tree
RFC 1/1 btrfs: Allow read-only mount with corrupted extent tree
Currently if there's any corruption at all in extent tree (eg. even single bit) then mounting will fail with:
"failed to read block groups: -5" (-EIO)
It happens because we immediately abort on first error when searching in extent tree for block groups.
Now with this patch if ignorebadroots option is specified then we handle such case and continue by removing already created block groups and creating dummy block groups.
--- fs/btrfs/block-group.c | 14 ++++++++++++++ fs/btrfs/disk-io.c | 4 ++-- fs/btrfs/disk-io.h | 2 ++ 3 files changed, 18 insertions(+), 2 deletions(-)
fs/btrfs/block-group.c +14 -0
@@ -2048,6 +2048,20 @@ int btrfs_read_block_groups(struct btrfs_fs_info *info) ret =check_chunk_block_group_mappings (info); error:btrfs_free_path (path); + +if (ret == -EIO &&btrfs_test_opt (info,IGNOREBADROOTS )) { +btrfs_put_block_group_cache (info); +btrfs_stop_all_workers (info); +btrfs_free_block_groups (info); + ret =btrfs_init_workqueues (info,NULL ); +if (ret) +return ret; + ret =btrfs_init_space_info (info); +if (ret) +return ret; +return fill_dummy_bgs (info); + } +return ret; }
fs/btrfs/disk-io.c +2 -2
@@ -1955,7 +1955,7 @@ static int read_backup_root(struct btrfs_fs_info *fs_info, u8 priority) }/* helper to cleanup workers */ -static void btrfs_stop_all_workers (struct btrfs_fs_info *fs_info) +void btrfs_stop_all_workers (struct btrfs_fs_info *fs_info) {btrfs_destroy_workqueue (fs_info->fixup_workers);btrfs_destroy_workqueue (fs_info->delalloc_workers);
@@ -2122,7 +2122,7 @@ static void btrfs_init_qgroup(struct btrfs_fs_info *fs_info)mutex_init (&fs_info->qgroup_rescan_lock); } -static int btrfs_init_workqueues (struct btrfs_fs_info *fs_info, +int btrfs_init_workqueues (struct btrfs_fs_info *fs_info,struct btrfs_fs_devices *fs_devices) {u32 max_active = fs_info->thread_pool_size;
fs/btrfs/disk-io.h +2 -0
@@ -137,6 +137,8 @@ int btrfs_find_free_objectid(struct btrfs_root *root, u64 *objectid);int btrfs_find_highest_objectid (struct btrfs_root *root,u64 *objectid);int __init btrfs_end_io_wq_init (void );void __cold btrfs_end_io_wq_exit (void ); +void btrfs_stop_all_workers (struct btrfs_fs_info *fs_info); +int btrfs_init_workqueues (struct btrfs_fs_info *fs_info,struct btrfs_fs_devices *fs_devices);#ifdef CONFIG_DEBUG_LOCK_ALLOC void btrfs_set_buffer_lockdep_class(u64 objectid,
Signature · 1 line
2.30.2
trešd., 2021. g. 17. marts, plkst. 03:18 — lietotājs Dāvis Mosāns (<[email protected]>) rakstīja:
Quoted 28 lines
Currently if there's any corruption at all in extent tree (eg. even single bit) then mounting will fail with:
"failed to read block groups: -5" (-EIO)
It happens because we immediately abort on first error when searching in extent tree for block groups.Now with this patch if
ignorebadrootsoption is specified then we handle such case and continue by removing already created block groups and creating dummy block groups.Signed-off-by Dāvis Mosāns <[email protected]>--- fs/btrfs/block-group.c | 14 ++++++++++++++ fs/btrfs/disk-io.c | 4 ++-- fs/btrfs/disk-io.h | 2 ++ 3 files changed, 18 insertions(+), 2 deletions(-)fs/btrfs/block-group.c +2
-0@@ -2048,6 +2048,20 @@ int btrfs_read_block_groups(struct btrfs_fs_info *info) ret =check_chunk_block_group_mappings (info); error:btrfs_free_path (path); + +if (ret == -EIO &&btrfs_test_opt (info,IGNOREBADROOTS )) {
fs/btrfs/block-group.c int btrfs_read_block_groups(struct btrfs_fs_info *info) · line 2062
Quoted 10 lines
fs/btrfs/block-group.c +10
-0+btrfs_put_block_group_cache (info); +btrfs_stop_all_workers (info); +btrfs_free_block_groups (info); + ret =btrfs_init_workqueues (info,NULL ); +if (ret) +return ret; + ret =btrfs_init_space_info (info); +if (ret) +return ret; +return fill_dummy_bgs (info);
This isn't that nice, but I don't really know how to properly clean up everything related to already created block groups so this was easiest way. It seems to work fine.
But looks like need to do something about replay log aswell because if it's not disabled then it fails with:
[ 1397.246869] BTRFS info (device sde): start tree-log replay [ 1398.218685] BTRFS warning (device sde): sde checksum verify failed on 21057127661568 wanted 0xd1506ed9 found 0x22ab750a level 0 [ 1398.218803] BTRFS warning (device sde): sde checksum verify failed on 21057127661568 wanted 0xd1506ed9 found 0x7dd54bb9 level 0 [ 1398.218813] BTRFS: error (device sde) in __btrfs_free_extent:3054: errno=-5 IO failure [ 1398.218828] BTRFS: error (device sde) in btrfs_run_delayed_refs:2124: errno=-5 IO failure [ 1398.219002] BTRFS: error (device sde) in btrfs_replay_log:2254: errno=-5 IO failure (Failed to recover log tree) [ 1398.229048] BTRFS error (device sde): open_ctree failed
Ideally it should replay everything except extent refs.
I also noticed that after unmount there is:
[11000.562504] BTRFS warning (device sde): page private not zero on page 21057098481664 [11000.562510] BTRFS warning (device sde): page private not zero on
page 21057098485760
not sure what it means.
Best regards,
Dāvis
Quoted 32 lines · Dāvis Mosāns
trešd., 2021. g. 17. marts, plkst. 03:18 — lietotājs Dāvis Mosāns (<[email protected]>) rakstīja:
Quoted 30 lines
Currently if there's any corruption at all in extent tree (eg. even single bit) then mounting will fail with:
"failed to read block groups: -5" (-EIO)
It happens because we immediately abort on first error when searching in extent tree for block groups.Now with this patch if
ignorebadrootsoption is specified then we handle such case and continue by removing already created block groups and creating dummy block groups.Signed-off-by Dāvis Mosāns <[email protected]>--- fs/btrfs/block-group.c | 14 ++++++++++++++ fs/btrfs/disk-io.c | 4 ++-- fs/btrfs/disk-io.h | 2 ++ 3 files changed, 18 insertions(+), 2 deletions(-)fs/btrfs/block-group.c +4
-0@@ -2048,6 +2048,20 @@ int btrfs_read_block_groups(struct btrfs_fs_info *info) ret =check_chunk_block_group_mappings (info); error:btrfs_free_path (path); + +if (ret == -EIO &&btrfs_test_opt (info,IGNOREBADROOTS )) { +btrfs_put_block_group_cache (info); +btrfs_stop_all_workers (info);
Quoted 8 lines
fs/btrfs/block-group.c int btrfs_read_block_groups(struct btrfs_fs_info *info) · line 2062
Quoted 8 lines
fs/btrfs/block-group.c +8
-0+btrfs_free_block_groups (info); + ret =btrfs_init_workqueues (info,NULL ); +if (ret) +return ret; + ret =btrfs_init_space_info (info); +if (ret) +return ret; +return fill_dummy_bgs (info);
When we hit bad things in extent tree, we should ensure we're mounting the fs RO, or we can't continue.
And we should also refuse to mount back to RW if we hit such case, so that we don't need anything complex, just ignore the whole extent tree and create the dummy block groups.
Quoted 7 lines
This isn't that nice, but I don't really know how to properly clean up everything related to already created block groups so this was easiest way. It seems to work fine.
But looks like need to do something about replay log aswell because if it's not disabled then it fails with:
Quoted 12 lines
[ 1397.246869] BTRFS info (device sde): start tree-log replay [ 1398.218685] BTRFS warning (device sde): sde checksum verify failed on 21057127661568 wanted 0xd1506ed9 found 0x22ab750a level 0 [ 1398.218803] BTRFS warning (device sde): sde checksum verify failed on 21057127661568 wanted 0xd1506ed9 found 0x7dd54bb9 level 0 [ 1398.218813] BTRFS: error (device sde) in __btrfs_free_extent:3054: errno=-5 IO failure [ 1398.218828] BTRFS: error (device sde) in btrfs_run_delayed_refs:2124: errno=-5 IO failure [ 1398.219002] BTRFS: error (device sde) in btrfs_replay_log:2254: errno=-5 IO failure (Failed to recover log tree) [ 1398.229048] BTRFS error (device sde): open_ctree failed
This is because we shouldn't allow to do anything write to the fs if we have anything wrong in extent tree.
Thanks,
Qu
Quoted 16 lines
Ideally it should replay everything except extent refs. >
I also noticed that after unmount there is:
[11000.562504] BTRFS warning (device sde): page private not zero on page 21057098481664 [11000.562510] BTRFS warning (device sde): page private not zero onpage 21057098485760
not sure what it means.
Best regards,
Dāvis
trešd., 2021. g. 17. marts, plkst. 12:28 — lietotājs Qu Wenruo (<[email protected]>) rakstīja:
Quoted 48 lines · Dāvis Mosāns
Quoted 32 lines
trešd., 2021. g. 17. marts, plkst. 03:18 — lietotājs Dāvis Mosāns (<[email protected]>) rakstīja:
Quoted 30 lines
Currently if there's any corruption at all in extent tree (eg. even single bit) then mounting will fail with:
"failed to read block groups: -5" (-EIO)
It happens because we immediately abort on first error when searching in extent tree for block groups.Now with this patch if
ignorebadrootsoption is specified then we handle such case and continue by removing already created block groups and creating dummy block groups.Signed-off-by Dāvis Mosāns <[email protected]>--- fs/btrfs/block-group.c | 14 ++++++++++++++ fs/btrfs/disk-io.c | 4 ++-- fs/btrfs/disk-io.h | 2 ++ 3 files changed, 18 insertions(+), 2 deletions(-)fs/btrfs/block-group.c +4
-0@@ -2048,6 +2048,20 @@ int btrfs_read_block_groups(struct btrfs_fs_info *info) ret =check_chunk_block_group_mappings (info); error:btrfs_free_path (path); + +if (ret == -EIO &&btrfs_test_opt (info,IGNOREBADROOTS )) { +btrfs_put_block_group_cache (info); +btrfs_stop_all_workers (info);Quoted 8 lines
fs/btrfs/block-group.c int btrfs_read_block_groups(struct btrfs_fs_info *info) · line 2062
Quoted 8 lines
fs/btrfs/block-group.c +8
-0+btrfs_free_block_groups (info); + ret =btrfs_init_workqueues (info,NULL ); +if (ret) +return ret; + ret =btrfs_init_space_info (info); +if (ret) +return ret; +return fill_dummy_bgs (info);When we hit bad things in extent tree, we should ensure we're mounting the fs RO, or we can't continue.
Quoted 4 lines
And we should also refuse to mount back to RW if we hit such case, so that we don't need anything complex, just ignore the whole extent tree and create the dummy block groups.
That's what we're doing here, ignorebadroots implies RO mount and without specifying it doesn't mount at all.
Quoted 20 lines
Quoted 7 lines
This isn't that nice, but I don't really know how to properly clean up everything related to already created block groups so this was easiest way. It seems to work fine.
But looks like need to do something about replay log aswell because if it's not disabled then it fails with:Quoted 12 lines
[ 1397.246869] BTRFS info (device sde): start tree-log replay [ 1398.218685] BTRFS warning (device sde): sde checksum verify failed on 21057127661568 wanted 0xd1506ed9 found 0x22ab750a level 0 [ 1398.218803] BTRFS warning (device sde): sde checksum verify failed on 21057127661568 wanted 0xd1506ed9 found 0x7dd54bb9 level 0 [ 1398.218813] BTRFS: error (device sde) in __btrfs_free_extent:3054: errno=-5 IO failure [ 1398.218828] BTRFS: error (device sde) in btrfs_run_delayed_refs:2124: errno=-5 IO failure [ 1398.219002] BTRFS: error (device sde) in btrfs_replay_log:2254: errno=-5 IO failure (Failed to recover log tree) [ 1398.229048] BTRFS error (device sde): open_ctree failed
Quoted 3 lines
This is because we shouldn't allow to do anything write to the fs if we have anything wrong in extent tree.
This is happening when mounting read-only. My assumption is that it only tries to replay in memory without writing anything to disk.
Quoted 78 lines · Dāvis Mosāns
trešd., 2021. g. 17. marts, plkst. 12:28 — lietotājs Qu Wenruo (<[email protected]>) rakstīja:
Quoted 48 lines · Dāvis Mosāns
Quoted 32 lines
trešd., 2021. g. 17. marts, plkst. 03:18 — lietotājs Dāvis Mosāns (<[email protected]>) rakstīja:
Quoted 30 lines
Currently if there's any corruption at all in extent tree (eg. even single bit) then mounting will fail with:
"failed to read block groups: -5" (-EIO)
It happens because we immediately abort on first error when searching in extent tree for block groups.Now with this patch if
ignorebadrootsoption is specified then we handle such case and continue by removing already created block groups and creating dummy block groups.Signed-off-by Dāvis Mosāns <[email protected]>--- fs/btrfs/block-group.c | 14 ++++++++++++++ fs/btrfs/disk-io.c | 4 ++-- fs/btrfs/disk-io.h | 2 ++ 3 files changed, 18 insertions(+), 2 deletions(-)fs/btrfs/block-group.c +4
-0@@ -2048,6 +2048,20 @@ int btrfs_read_block_groups(struct btrfs_fs_info *info) ret =check_chunk_block_group_mappings (info); error:btrfs_free_path (path); + +if (ret == -EIO &&btrfs_test_opt (info,IGNOREBADROOTS )) { +btrfs_put_block_group_cache (info); +btrfs_stop_all_workers (info);Quoted 8 lines
fs/btrfs/block-group.c int btrfs_read_block_groups(struct btrfs_fs_info *info) · line 2062
Quoted 8 lines
fs/btrfs/block-group.c +8
-0+btrfs_free_block_groups (info); + ret =btrfs_init_workqueues (info,NULL ); +if (ret) +return ret; + ret =btrfs_init_space_info (info); +if (ret) +return ret; +return fill_dummy_bgs (info);When we hit bad things in extent tree, we should ensure we're mounting the fs RO, or we can't continue.
Quoted 4 lines
And we should also refuse to mount back to RW if we hit such case, so that we don't need anything complex, just ignore the whole extent tree and create the dummy block groups.
That's what we're doing here,
ignorebadrootsimplies RO mount and without specifying it doesn't mount at all.Quoted 20 lines
Quoted 19 lines
This isn't that nice, but I don't really know how to properly clean up everything related to already created block groups so this was easiest way. It seems to work fine.
But looks like need to do something about replay log aswell because if it's not disabled then it fails with:[ 1397.246869] BTRFS info (device sde): start tree-log replay [ 1398.218685] BTRFS warning (device sde): sde checksum verify failed on 21057127661568 wanted 0xd1506ed9 found 0x22ab750a level 0 [ 1398.218803] BTRFS warning (device sde): sde checksum verify failed on 21057127661568 wanted 0xd1506ed9 found 0x7dd54bb9 level 0 [ 1398.218813] BTRFS: error (device sde) in __btrfs_free_extent:3054: errno=-5 IO failure [ 1398.218828] BTRFS: error (device sde) in btrfs_run_delayed_refs:2124: errno=-5 IO failure [ 1398.219002] BTRFS: error (device sde) in btrfs_replay_log:2254: errno=-5 IO failure (Failed to recover log tree) [ 1398.229048] BTRFS error (device sde): open_ctree failed
Quoted 7 lines
Quoted 3 lines
This is because we shouldn't allow to do anything write to the fs if we have anything wrong in extent tree.
This is happening when mounting read-only. My assumption is that it only tries to replay in memory without writing anything to disk.
We lacks the check on log tree.
Normally for such forced RO mount, log replay is not allowed.
We should output a warning to prompt user to use nologreplay, and reject the mount.
Thanks,
Qu
ceturtd., 2021. g. 18. marts, plkst. 01:49 — lietotājs Qu Wenruo (<[email protected]>) rakstīja:
Quoted 90 lines · Dāvis Mosāns
Quoted 78 lines
trešd., 2021. g. 17. marts, plkst. 12:28 — lietotājs Qu Wenruo (<[email protected]>) rakstīja:
Quoted 48 lines · Dāvis Mosāns
Quoted 32 lines
trešd., 2021. g. 17. marts, plkst. 03:18 — lietotājs Dāvis Mosāns (<[email protected]>) rakstīja:
Quoted 30 lines
Currently if there's any corruption at all in extent tree (eg. even single bit) then mounting will fail with:
"failed to read block groups: -5" (-EIO)
It happens because we immediately abort on first error when searching in extent tree for block groups.Now with this patch if
ignorebadrootsoption is specified then we handle such case and continue by removing already created block groups and creating dummy block groups.Signed-off-by Dāvis Mosāns <[email protected]>--- fs/btrfs/block-group.c | 14 ++++++++++++++ fs/btrfs/disk-io.c | 4 ++-- fs/btrfs/disk-io.h | 2 ++ 3 files changed, 18 insertions(+), 2 deletions(-)fs/btrfs/block-group.c +4
-0@@ -2048,6 +2048,20 @@ int btrfs_read_block_groups(struct btrfs_fs_info *info) ret =check_chunk_block_group_mappings (info); error:btrfs_free_path (path); + +if (ret == -EIO &&btrfs_test_opt (info,IGNOREBADROOTS )) { +btrfs_put_block_group_cache (info); +btrfs_stop_all_workers (info);Quoted 8 lines
fs/btrfs/block-group.c int btrfs_read_block_groups(struct btrfs_fs_info *info) · line 2062
Quoted 8 lines
fs/btrfs/block-group.c +8
-0+btrfs_free_block_groups (info); + ret =btrfs_init_workqueues (info,NULL ); +if (ret) +return ret; + ret =btrfs_init_space_info (info); +if (ret) +return ret; +return fill_dummy_bgs (info);When we hit bad things in extent tree, we should ensure we're mounting the fs RO, or we can't continue.
Quoted 4 lines
And we should also refuse to mount back to RW if we hit such case, so that we don't need anything complex, just ignore the whole extent tree and create the dummy block groups.
That's what we're doing here,
ignorebadrootsimplies RO mount and without specifying it doesn't mount at all.Quoted 20 lines
Quoted 19 lines
This isn't that nice, but I don't really know how to properly clean up everything related to already created block groups so this was easiest way. It seems to work fine.
But looks like need to do something about replay log aswell because if it's not disabled then it fails with:[ 1397.246869] BTRFS info (device sde): start tree-log replay [ 1398.218685] BTRFS warning (device sde): sde checksum verify failed on 21057127661568 wanted 0xd1506ed9 found 0x22ab750a level 0 [ 1398.218803] BTRFS warning (device sde): sde checksum verify failed on 21057127661568 wanted 0xd1506ed9 found 0x7dd54bb9 level 0 [ 1398.218813] BTRFS: error (device sde) in __btrfs_free_extent:3054: errno=-5 IO failure [ 1398.218828] BTRFS: error (device sde) in btrfs_run_delayed_refs:2124: errno=-5 IO failure [ 1398.219002] BTRFS: error (device sde) in btrfs_replay_log:2254: errno=-5 IO failure (Failed to recover log tree) [ 1398.229048] BTRFS error (device sde): open_ctree failedQuoted 7 lines
Quoted 3 lines
This is because we shouldn't allow to do anything write to the fs if we have anything wrong in extent tree.
This is happening when mounting read-only. My assumption is that it only tries to replay in memory without writing anything to disk.
Quoted 7 lines
We lacks the check on log tree.
Normally for such forced RO mount, log replay is not allowed.
We should output a warning to prompt user to use nologreplay, and reject the mount.
I'm not familiar with log replay but couldn't there be something useful (ignoring ref counts) that would still be worth replaying in memory?
Quoted 96 lines · Dāvis Mosāns
ceturtd., 2021. g. 18. marts, plkst. 01:49 — lietotājs Qu Wenruo (<[email protected]>) rakstīja:
Quoted 94 lines · Dāvis Mosāns
Quoted 78 lines
trešd., 2021. g. 17. marts, plkst. 12:28 — lietotājs Qu Wenruo (<[email protected]>) rakstīja:
Quoted 48 lines · Dāvis Mosāns
Quoted 32 lines
trešd., 2021. g. 17. marts, plkst. 03:18 — lietotājs Dāvis Mosāns (<[email protected]>) rakstīja:
Quoted 30 lines
Currently if there's any corruption at all in extent tree (eg. even single bit) then mounting will fail with:
"failed to read block groups: -5" (-EIO)
It happens because we immediately abort on first error when searching in extent tree for block groups.Now with this patch if
ignorebadrootsoption is specified then we handle such case and continue by removing already created block groups and creating dummy block groups.Signed-off-by Dāvis Mosāns <[email protected]>--- fs/btrfs/block-group.c | 14 ++++++++++++++ fs/btrfs/disk-io.c | 4 ++-- fs/btrfs/disk-io.h | 2 ++ 3 files changed, 18 insertions(+), 2 deletions(-)fs/btrfs/block-group.c +4
-0@@ -2048,6 +2048,20 @@ int btrfs_read_block_groups(struct btrfs_fs_info *info) ret =check_chunk_block_group_mappings (info); error:btrfs_free_path (path); + +if (ret == -EIO &&btrfs_test_opt (info,IGNOREBADROOTS )) { +btrfs_put_block_group_cache (info); +btrfs_stop_all_workers (info);Quoted 8 lines
fs/btrfs/block-group.c int btrfs_read_block_groups(struct btrfs_fs_info *info) · line 2062
Quoted 8 lines
fs/btrfs/block-group.c +8
-0+btrfs_free_block_groups (info); + ret =btrfs_init_workqueues (info,NULL ); +if (ret) +return ret; + ret =btrfs_init_space_info (info); +if (ret) +return ret; +return fill_dummy_bgs (info);When we hit bad things in extent tree, we should ensure we're mounting the fs RO, or we can't continue.
Quoted 4 lines
And we should also refuse to mount back to RW if we hit such case, so that we don't need anything complex, just ignore the whole extent tree and create the dummy block groups.
That's what we're doing here,
ignorebadrootsimplies RO mount and without specifying it doesn't mount at all.Quoted 20 lines
Quoted 19 lines
This isn't that nice, but I don't really know how to properly clean up everything related to already created block groups so this was easiest way. It seems to work fine.
But looks like need to do something about replay log aswell because if it's not disabled then it fails with:[ 1397.246869] BTRFS info (device sde): start tree-log replay [ 1398.218685] BTRFS warning (device sde): sde checksum verify failed on 21057127661568 wanted 0xd1506ed9 found 0x22ab750a level 0 [ 1398.218803] BTRFS warning (device sde): sde checksum verify failed on 21057127661568 wanted 0xd1506ed9 found 0x7dd54bb9 level 0 [ 1398.218813] BTRFS: error (device sde) in __btrfs_free_extent:3054: errno=-5 IO failure [ 1398.218828] BTRFS: error (device sde) in btrfs_run_delayed_refs:2124: errno=-5 IO failure [ 1398.219002] BTRFS: error (device sde) in btrfs_replay_log:2254: errno=-5 IO failure (Failed to recover log tree) [ 1398.229048] BTRFS error (device sde): open_ctree failedQuoted 7 lines
Quoted 3 lines
This is because we shouldn't allow to do anything write to the fs if we have anything wrong in extent tree.
This is happening when mounting read-only. My assumption is that it only tries to replay in memory without writing anything to disk.
We lacks the check on log tree.
Normally for such forced RO mount, log replay is not allowed.
Quoted 8 lines
Quoted 3 lines
We should output a warning to prompt user to use nologreplay, and reject the mount.
I'm not familiar with log replay but couldn't there be something useful (ignoring ref counts) that would still be worth replaying in memory?
Log replay means metadata write.
Any write needs a valid extent tree to find out free space for new metadata/data.
So no, we can't do anything but completely ignoring the log.
Thanks,
Qu
sestd., 2021. g. 20. marts, plkst. 02:34 — lietotājs Qu Wenruo (<[email protected]>) rakstīja:
Quoted 108 lines · Dāvis Mosāns
Quoted 96 lines
ceturtd., 2021. g. 18. marts, plkst. 01:49 — lietotājs Qu Wenruo (<[email protected]>) rakstīja:
Quoted 94 lines · Dāvis Mosāns
Quoted 78 lines
trešd., 2021. g. 17. marts, plkst. 12:28 — lietotājs Qu Wenruo (<[email protected]>) rakstīja:
Quoted 48 lines · Dāvis Mosāns
Quoted 32 lines
trešd., 2021. g. 17. marts, plkst. 03:18 — lietotājs Dāvis Mosāns (<[email protected]>) rakstīja:
Quoted 30 lines
Currently if there's any corruption at all in extent tree (eg. even single bit) then mounting will fail with:
"failed to read block groups: -5" (-EIO)
It happens because we immediately abort on first error when searching in extent tree for block groups.Now with this patch if
ignorebadrootsoption is specified then we handle such case and continue by removing already created block groups and creating dummy block groups.Signed-off-by Dāvis Mosāns <[email protected]>--- fs/btrfs/block-group.c | 14 ++++++++++++++ fs/btrfs/disk-io.c | 4 ++-- fs/btrfs/disk-io.h | 2 ++ 3 files changed, 18 insertions(+), 2 deletions(-)fs/btrfs/block-group.c +4
-0@@ -2048,6 +2048,20 @@ int btrfs_read_block_groups(struct btrfs_fs_info *info) ret =check_chunk_block_group_mappings (info); error:btrfs_free_path (path); + +if (ret == -EIO &&btrfs_test_opt (info,IGNOREBADROOTS )) { +btrfs_put_block_group_cache (info); +btrfs_stop_all_workers (info);Quoted 8 lines
fs/btrfs/block-group.c int btrfs_read_block_groups(struct btrfs_fs_info *info) · line 2062
Quoted 8 lines
fs/btrfs/block-group.c +8
-0+btrfs_free_block_groups (info); + ret =btrfs_init_workqueues (info,NULL ); +if (ret) +return ret; + ret =btrfs_init_space_info (info); +if (ret) +return ret; +return fill_dummy_bgs (info);When we hit bad things in extent tree, we should ensure we're mounting the fs RO, or we can't continue.
Quoted 4 lines
And we should also refuse to mount back to RW if we hit such case, so that we don't need anything complex, just ignore the whole extent tree and create the dummy block groups.
That's what we're doing here,
ignorebadrootsimplies RO mount and without specifying it doesn't mount at all.Quoted 20 lines
Quoted 19 lines
This isn't that nice, but I don't really know how to properly clean up everything related to already created block groups so this was easiest way. It seems to work fine.
But looks like need to do something about replay log aswell because if it's not disabled then it fails with:[ 1397.246869] BTRFS info (device sde): start tree-log replay [ 1398.218685] BTRFS warning (device sde): sde checksum verify failed on 21057127661568 wanted 0xd1506ed9 found 0x22ab750a level 0 [ 1398.218803] BTRFS warning (device sde): sde checksum verify failed on 21057127661568 wanted 0xd1506ed9 found 0x7dd54bb9 level 0 [ 1398.218813] BTRFS: error (device sde) in __btrfs_free_extent:3054: errno=-5 IO failure [ 1398.218828] BTRFS: error (device sde) in btrfs_run_delayed_refs:2124: errno=-5 IO failure [ 1398.219002] BTRFS: error (device sde) in btrfs_replay_log:2254: errno=-5 IO failure (Failed to recover log tree) [ 1398.229048] BTRFS error (device sde): open_ctree failedQuoted 7 lines
Quoted 3 lines
This is because we shouldn't allow to do anything write to the fs if we have anything wrong in extent tree.
This is happening when mounting read-only. My assumption is that it only tries to replay in memory without writing anything to disk.
We lacks the check on log tree.
Normally for such forced RO mount, log replay is not allowed.
Quoted 8 lines
Quoted 3 lines
We should output a warning to prompt user to use nologreplay, and reject the mount.
I'm not familiar with log replay but couldn't there be something useful (ignoring ref counts) that would still be worth replaying in memory?
Quoted 7 lines
Log replay means metadata write.
Any write needs a valid extent tree to find out free space for new metadata/data.
So no, we can't do anything but completely ignoring the log.
I see, updated patch. But even then it seems it could be possible to add new ramdisk and make allocations there (eg. create new extent tree there) thus allowing replay. I guess that's way too much work. Anyway thanks for feedback!
Best regards,
Dāvis
Quoted 115 lines · Dāvis Mosāns
sestd., 2021. g. 20. marts, plkst. 02:34 — lietotājs Qu Wenruo (<[email protected]>) rakstīja:
Quoted 113 lines · Dāvis Mosāns
Quoted 96 lines
ceturtd., 2021. g. 18. marts, plkst. 01:49 — lietotājs Qu Wenruo (<[email protected]>) rakstīja:
Quoted 94 lines · Dāvis Mosāns
Quoted 78 lines
trešd., 2021. g. 17. marts, plkst. 12:28 — lietotājs Qu Wenruo (<[email protected]>) rakstīja:
Quoted 48 lines · Dāvis Mosāns
Quoted 32 lines
trešd., 2021. g. 17. marts, plkst. 03:18 — lietotājs Dāvis Mosāns (<[email protected]>) rakstīja:
Quoted 30 lines
Currently if there's any corruption at all in extent tree (eg. even single bit) then mounting will fail with:
"failed to read block groups: -5" (-EIO)
It happens because we immediately abort on first error when searching in extent tree for block groups.Now with this patch if
ignorebadrootsoption is specified then we handle such case and continue by removing already created block groups and creating dummy block groups.Signed-off-by Dāvis Mosāns <[email protected]>--- fs/btrfs/block-group.c | 14 ++++++++++++++ fs/btrfs/disk-io.c | 4 ++-- fs/btrfs/disk-io.h | 2 ++ 3 files changed, 18 insertions(+), 2 deletions(-)fs/btrfs/block-group.c +4
-0@@ -2048,6 +2048,20 @@ int btrfs_read_block_groups(struct btrfs_fs_info *info) ret =check_chunk_block_group_mappings (info); error:btrfs_free_path (path); + +if (ret == -EIO &&btrfs_test_opt (info,IGNOREBADROOTS )) { +btrfs_put_block_group_cache (info); +btrfs_stop_all_workers (info);Quoted 8 lines
fs/btrfs/block-group.c int btrfs_read_block_groups(struct btrfs_fs_info *info) · line 2062
Quoted 8 lines
fs/btrfs/block-group.c +8
-0+btrfs_free_block_groups (info); + ret =btrfs_init_workqueues (info,NULL ); +if (ret) +return ret; + ret =btrfs_init_space_info (info); +if (ret) +return ret; +return fill_dummy_bgs (info);When we hit bad things in extent tree, we should ensure we're mounting the fs RO, or we can't continue.
Quoted 4 lines
And we should also refuse to mount back to RW if we hit such case, so that we don't need anything complex, just ignore the whole extent tree and create the dummy block groups.
That's what we're doing here,
ignorebadrootsimplies RO mount and without specifying it doesn't mount at all.Quoted 20 lines
Quoted 19 lines
This isn't that nice, but I don't really know how to properly clean up everything related to already created block groups so this was easiest way. It seems to work fine.
But looks like need to do something about replay log aswell because if it's not disabled then it fails with:[ 1397.246869] BTRFS info (device sde): start tree-log replay [ 1398.218685] BTRFS warning (device sde): sde checksum verify failed on 21057127661568 wanted 0xd1506ed9 found 0x22ab750a level 0 [ 1398.218803] BTRFS warning (device sde): sde checksum verify failed on 21057127661568 wanted 0xd1506ed9 found 0x7dd54bb9 level 0 [ 1398.218813] BTRFS: error (device sde) in __btrfs_free_extent:3054: errno=-5 IO failure [ 1398.218828] BTRFS: error (device sde) in btrfs_run_delayed_refs:2124: errno=-5 IO failure [ 1398.219002] BTRFS: error (device sde) in btrfs_replay_log:2254: errno=-5 IO failure (Failed to recover log tree) [ 1398.229048] BTRFS error (device sde): open_ctree failedQuoted 7 lines
Quoted 3 lines
This is because we shouldn't allow to do anything write to the fs if we have anything wrong in extent tree.
This is happening when mounting read-only. My assumption is that it only tries to replay in memory without writing anything to disk.
We lacks the check on log tree.
Normally for such forced RO mount, log replay is not allowed.
Quoted 8 lines
Quoted 3 lines
We should output a warning to prompt user to use nologreplay, and reject the mount.
I'm not familiar with log replay but couldn't there be something useful (ignoring ref counts) that would still be worth replaying in memory?
Log replay means metadata write.
Any write needs a valid extent tree to find out free space for new metadata/data.
Quoted 6 lines
Quoted 2 lines
So no, we can't do anything but completely ignoring the log.
I see, updated patch. But even then it seems it could be possible to add new ramdisk and make allocations there (eg. create new extent tree there) thus allowing replay.
The problem here is, since the extent tree is corrupted, we won't know which range has metadata already.
While metadata CoW, just like its name, needs to CoW, which means it can't writeback (even just in memory) to anywhere we have metadata.
The worst case is, we choose a bytenr for the new metadata to be (in memory), but it turns out later read needs to read metadata from the exactly same location.
Quoted 1 line
I guess that's way too much work.
And we gain nothing but tons of potential bugs.
BTW, I'm curious what's your test cases? As it seems you're using log-replay but if we hit anything wrong for the replayed data, it means btrfs kernel module has something wrong.
Did you add extra corruption for the replayed data, or it's some bug unexposed?
Thanks,
Qu
Quoted 5 lines
Anyway thanks for feedback!
Best regards,
Dāvis
pirmd., 2021. g. 22. marts, plkst. 02:25 — lietotājs Qu Wenruo (<[email protected]>) rakstīja:
Quoted 126 lines · Dāvis Mosāns
Quoted 115 lines
sestd., 2021. g. 20. marts, plkst. 02:34 — lietotājs Qu Wenruo (<[email protected]>) rakstīja:
Quoted 113 lines · Dāvis Mosāns
Quoted 96 lines
ceturtd., 2021. g. 18. marts, plkst. 01:49 — lietotājs Qu Wenruo (<[email protected]>) rakstīja:
Quoted 94 lines · Dāvis Mosāns
Quoted 78 lines
trešd., 2021. g. 17. marts, plkst. 12:28 — lietotājs Qu Wenruo (<[email protected]>) rakstīja:
Quoted 48 lines · Dāvis Mosāns
Quoted 32 lines
trešd., 2021. g. 17. marts, plkst. 03:18 — lietotājs Dāvis Mosāns (<[email protected]>) rakstīja:
Quoted 30 lines
Currently if there's any corruption at all in extent tree (eg. even single bit) then mounting will fail with:
"failed to read block groups: -5" (-EIO)
It happens because we immediately abort on first error when searching in extent tree for block groups.Now with this patch if
ignorebadrootsoption is specified then we handle such case and continue by removing already created block groups and creating dummy block groups.Signed-off-by Dāvis Mosāns <[email protected]>--- fs/btrfs/block-group.c | 14 ++++++++++++++ fs/btrfs/disk-io.c | 4 ++-- fs/btrfs/disk-io.h | 2 ++ 3 files changed, 18 insertions(+), 2 deletions(-)fs/btrfs/block-group.c +4
-0@@ -2048,6 +2048,20 @@ int btrfs_read_block_groups(struct btrfs_fs_info *info) ret =check_chunk_block_group_mappings (info); error:btrfs_free_path (path); + +if (ret == -EIO &&btrfs_test_opt (info,IGNOREBADROOTS )) { +btrfs_put_block_group_cache (info); +btrfs_stop_all_workers (info);Quoted 8 lines
fs/btrfs/block-group.c int btrfs_read_block_groups(struct btrfs_fs_info *info) · line 2062
Quoted 8 lines
fs/btrfs/block-group.c +8
-0+btrfs_free_block_groups (info); + ret =btrfs_init_workqueues (info,NULL ); +if (ret) +return ret; + ret =btrfs_init_space_info (info); +if (ret) +return ret; +return fill_dummy_bgs (info);When we hit bad things in extent tree, we should ensure we're mounting the fs RO, or we can't continue.
Quoted 4 lines
And we should also refuse to mount back to RW if we hit such case, so that we don't need anything complex, just ignore the whole extent tree and create the dummy block groups.
That's what we're doing here,
ignorebadrootsimplies RO mount and without specifying it doesn't mount at all.Quoted 20 lines
Quoted 19 lines
This isn't that nice, but I don't really know how to properly clean up everything related to already created block groups so this was easiest way. It seems to work fine.
But looks like need to do something about replay log aswell because if it's not disabled then it fails with:[ 1397.246869] BTRFS info (device sde): start tree-log replay [ 1398.218685] BTRFS warning (device sde): sde checksum verify failed on 21057127661568 wanted 0xd1506ed9 found 0x22ab750a level 0 [ 1398.218803] BTRFS warning (device sde): sde checksum verify failed on 21057127661568 wanted 0xd1506ed9 found 0x7dd54bb9 level 0 [ 1398.218813] BTRFS: error (device sde) in __btrfs_free_extent:3054: errno=-5 IO failure [ 1398.218828] BTRFS: error (device sde) in btrfs_run_delayed_refs:2124: errno=-5 IO failure [ 1398.219002] BTRFS: error (device sde) in btrfs_replay_log:2254: errno=-5 IO failure (Failed to recover log tree) [ 1398.229048] BTRFS error (device sde): open_ctree failedQuoted 7 lines
Quoted 3 lines
This is because we shouldn't allow to do anything write to the fs if we have anything wrong in extent tree.
This is happening when mounting read-only. My assumption is that it only tries to replay in memory without writing anything to disk.
We lacks the check on log tree.
Normally for such forced RO mount, log replay is not allowed.
Quoted 8 lines
Quoted 3 lines
We should output a warning to prompt user to use nologreplay, and reject the mount.
I'm not familiar with log replay but couldn't there be something useful (ignoring ref counts) that would still be worth replaying in memory?
Log replay means metadata write.
Any write needs a valid extent tree to find out free space for new metadata/data.
Quoted 6 lines
Quoted 2 lines
So no, we can't do anything but completely ignoring the log.
I see, updated patch. But even then it seems it could be possible to add new ramdisk and make allocations there (eg. create new extent tree there) thus allowing replay.
Quoted 9 lines
The problem here is, since the extent tree is corrupted, we won't know which range has metadata already.
While metadata CoW, just like its name, needs to CoW, which means it can't writeback (even just in memory) to anywhere we have metadata.The worst case is, we choose a bytenr for the new metadata to be (in memory), but it turns out later read needs to read metadata from the exactly same location.
The idea is if we add new disk then we would put it after last bytenr (which isn't mapped to any existing disks) thus there wouldn't be any overlap.
Quoted 6 lines
BTW, I'm curious what's your test cases? As it seems you're using log-replay but if we hit anything wrong for the replayed data, it means btrfs kernel module has something wrong.
Did you add extra corruption for the replayed data, or it's some bug unexposed?
Basically I've a corrupted btrfs due to HBA card fault and before I nuke it I want to copy as much usable data as possible. So I was thinking if whatever is in replay log could be restored. The replay tree log itself is perfectly fine with valid checksum and there isn't any issues regarding that. I looked at it with btrfs inspect dump-tree and saw that there isn't anything important so it's fine ignoring it.
Quoted 133 lines · Dāvis Mosāns
pirmd., 2021. g. 22. marts, plkst. 02:25 — lietotājs Qu Wenruo (<[email protected]>) rakstīja:
Quoted 131 lines · Dāvis Mosāns
Quoted 115 lines
sestd., 2021. g. 20. marts, plkst. 02:34 — lietotājs Qu Wenruo (<[email protected]>) rakstīja:
Quoted 113 lines · Dāvis Mosāns
Quoted 96 lines
ceturtd., 2021. g. 18. marts, plkst. 01:49 — lietotājs Qu Wenruo (<[email protected]>) rakstīja:
Quoted 94 lines · Dāvis Mosāns
Quoted 78 lines
trešd., 2021. g. 17. marts, plkst. 12:28 — lietotājs Qu Wenruo (<[email protected]>) rakstīja:
Quoted 48 lines · Dāvis Mosāns
Quoted 32 lines
trešd., 2021. g. 17. marts, plkst. 03:18 — lietotājs Dāvis Mosāns (<[email protected]>) rakstīja:
Quoted 30 lines
Currently if there's any corruption at all in extent tree (eg. even single bit) then mounting will fail with:
"failed to read block groups: -5" (-EIO)
It happens because we immediately abort on first error when searching in extent tree for block groups.Now with this patch if
ignorebadrootsoption is specified then we handle such case and continue by removing already created block groups and creating dummy block groups.Signed-off-by Dāvis Mosāns <[email protected]>--- fs/btrfs/block-group.c | 14 ++++++++++++++ fs/btrfs/disk-io.c | 4 ++-- fs/btrfs/disk-io.h | 2 ++ 3 files changed, 18 insertions(+), 2 deletions(-)fs/btrfs/block-group.c +4
-0@@ -2048,6 +2048,20 @@ int btrfs_read_block_groups(struct btrfs_fs_info *info) ret =check_chunk_block_group_mappings (info); error:btrfs_free_path (path); + +if (ret == -EIO &&btrfs_test_opt (info,IGNOREBADROOTS )) { +btrfs_put_block_group_cache (info); +btrfs_stop_all_workers (info);Quoted 8 lines
fs/btrfs/block-group.c int btrfs_read_block_groups(struct btrfs_fs_info *info) · line 2062
Quoted 8 lines
fs/btrfs/block-group.c +8
-0+btrfs_free_block_groups (info); + ret =btrfs_init_workqueues (info,NULL ); +if (ret) +return ret; + ret =btrfs_init_space_info (info); +if (ret) +return ret; +return fill_dummy_bgs (info);When we hit bad things in extent tree, we should ensure we're mounting the fs RO, or we can't continue.
Quoted 4 lines
And we should also refuse to mount back to RW if we hit such case, so that we don't need anything complex, just ignore the whole extent tree and create the dummy block groups.
That's what we're doing here,
ignorebadrootsimplies RO mount and without specifying it doesn't mount at all.Quoted 20 lines
Quoted 19 lines
This isn't that nice, but I don't really know how to properly clean up everything related to already created block groups so this was easiest way. It seems to work fine.
But looks like need to do something about replay log aswell because if it's not disabled then it fails with:[ 1397.246869] BTRFS info (device sde): start tree-log replay [ 1398.218685] BTRFS warning (device sde): sde checksum verify failed on 21057127661568 wanted 0xd1506ed9 found 0x22ab750a level 0 [ 1398.218803] BTRFS warning (device sde): sde checksum verify failed on 21057127661568 wanted 0xd1506ed9 found 0x7dd54bb9 level 0 [ 1398.218813] BTRFS: error (device sde) in __btrfs_free_extent:3054: errno=-5 IO failure [ 1398.218828] BTRFS: error (device sde) in btrfs_run_delayed_refs:2124: errno=-5 IO failure [ 1398.219002] BTRFS: error (device sde) in btrfs_replay_log:2254: errno=-5 IO failure (Failed to recover log tree) [ 1398.229048] BTRFS error (device sde): open_ctree failedQuoted 7 lines
Quoted 3 lines
This is because we shouldn't allow to do anything write to the fs if we have anything wrong in extent tree.
This is happening when mounting read-only. My assumption is that it only tries to replay in memory without writing anything to disk.
We lacks the check on log tree.
Normally for such forced RO mount, log replay is not allowed.
Quoted 8 lines
Quoted 3 lines
We should output a warning to prompt user to use nologreplay, and reject the mount.
I'm not familiar with log replay but couldn't there be something useful (ignoring ref counts) that would still be worth replaying in memory?
Log replay means metadata write.
Any write needs a valid extent tree to find out free space for new metadata/data.
Quoted 6 lines
Quoted 2 lines
So no, we can't do anything but completely ignoring the log.
I see, updated patch. But even then it seems it could be possible to add new ramdisk and make allocations there (eg. create new extent tree there) thus allowing replay.
The problem here is, since the extent tree is corrupted, we won't know which range has metadata already.
While metadata CoW, just like its name, needs to CoW, which means it can't writeback (even just in memory) to anywhere we have metadata.
Quoted 8 lines
Quoted 4 lines
The worst case is, we choose a bytenr for the new metadata to be (in memory), but it turns out later read needs to read metadata from the exactly same location.
The idea is if we add new disk then we would put it after last bytenr (which isn't mapped to any existing disks) thus there wouldn't be any overlap.
I wonder if that idea can be turned into an online recovery tool. Rebuild the metadata by more or less reflinking the old filesystem's data into a new filesystem created in the unallocated spaces between the old filesystem's block groups (building new subvol trees in the process). It would need the device and chunk trees to be intact, but you have to have those for rescue= to work at all, and it's relatively rarer for those to get damaged. There's a complicated dance that has to be done to flip a block group from the old filesystem to the new one, but maybe that can be done by just making the entire old filesystem into a giant file image, then making ordinary reflinks to it, then finish by deleting the old image and running defrag to discard the unreferenced blocks that remain. Think "btrfs-convert" but using a busted btrfs as source instead of ext4.
OK, way out of scope for this thread.
Quoted 6 lines
Quoted 5 lines
BTW, I'm curious what's your test cases? As it seems you're using log-replay but if we hit anything wrong for the replayed data, it means btrfs kernel module has something wrong.
Did you add extra corruption for the replayed data, or it's some bug unexposed?
Quoted 7 lines
Basically I've a corrupted btrfs due to HBA card fault and before I nuke it I want to copy as much usable data as possible. So I was thinking if whatever is in replay log could be restored. The replay tree log itself is perfectly fine with valid checksum and there isn't any issues regarding that. I looked at it with
btrfs inspect dump-treeand saw that there isn't anything important so it's fine ignoring it.
The log tree will only contain things that were fsync()ed after the last completed transaction commit. Unless you're hitting a delayed refs latency issue or have non-default mount options, that's data from only the last 30 seconds before the filesystem failed.
It might be desirable to replay the log tree if you had very high-value data there, but it's the last 0.001% of the filesystem that requires the last 99% of the development effort to recover.
pirmd., 2021. g. 22. marts, plkst. 06:48 — lietotājs Zygo Blaxell (<[email protected]>) rakstīja:
Quoted 144 lines · Dāvis Mosāns
Quoted 133 lines
pirmd., 2021. g. 22. marts, plkst. 02:25 — lietotājs Qu Wenruo (<[email protected]>) rakstīja:
Quoted 131 lines · Dāvis Mosāns
Quoted 115 lines
sestd., 2021. g. 20. marts, plkst. 02:34 — lietotājs Qu Wenruo (<[email protected]>) rakstīja:
Quoted 113 lines · Dāvis Mosāns
Quoted 96 lines
ceturtd., 2021. g. 18. marts, plkst. 01:49 — lietotājs Qu Wenruo (<[email protected]>) rakstīja:
Quoted 94 lines · Dāvis Mosāns
Quoted 78 lines
trešd., 2021. g. 17. marts, plkst. 12:28 — lietotājs Qu Wenruo (<[email protected]>) rakstīja:
Quoted 48 lines · Dāvis Mosāns
Quoted 32 lines
trešd., 2021. g. 17. marts, plkst. 03:18 — lietotājs Dāvis Mosāns (<[email protected]>) rakstīja:
Quoted 30 lines
Currently if there's any corruption at all in extent tree (eg. even single bit) then mounting will fail with:
"failed to read block groups: -5" (-EIO)
It happens because we immediately abort on first error when searching in extent tree for block groups.Now with this patch if
ignorebadrootsoption is specified then we handle such case and continue by removing already created block groups and creating dummy block groups.Signed-off-by Dāvis Mosāns <[email protected]>--- fs/btrfs/block-group.c | 14 ++++++++++++++ fs/btrfs/disk-io.c | 4 ++-- fs/btrfs/disk-io.h | 2 ++ 3 files changed, 18 insertions(+), 2 deletions(-)fs/btrfs/block-group.c +4
-0@@ -2048,6 +2048,20 @@ int btrfs_read_block_groups(struct btrfs_fs_info *info) ret =check_chunk_block_group_mappings (info); error:btrfs_free_path (path); + +if (ret == -EIO &&btrfs_test_opt (info,IGNOREBADROOTS )) { +btrfs_put_block_group_cache (info); +btrfs_stop_all_workers (info);Quoted 8 lines
fs/btrfs/block-group.c int btrfs_read_block_groups(struct btrfs_fs_info *info) · line 2062
Quoted 8 lines
fs/btrfs/block-group.c +8
-0+btrfs_free_block_groups (info); + ret =btrfs_init_workqueues (info,NULL ); +if (ret) +return ret; + ret =btrfs_init_space_info (info); +if (ret) +return ret; +return fill_dummy_bgs (info);When we hit bad things in extent tree, we should ensure we're mounting the fs RO, or we can't continue.
Quoted 4 lines
And we should also refuse to mount back to RW if we hit such case, so that we don't need anything complex, just ignore the whole extent tree and create the dummy block groups.
That's what we're doing here,
ignorebadrootsimplies RO mount and without specifying it doesn't mount at all.Quoted 20 lines
Quoted 19 lines
This isn't that nice, but I don't really know how to properly clean up everything related to already created block groups so this was easiest way. It seems to work fine.
But looks like need to do something about replay log aswell because if it's not disabled then it fails with:[ 1397.246869] BTRFS info (device sde): start tree-log replay [ 1398.218685] BTRFS warning (device sde): sde checksum verify failed on 21057127661568 wanted 0xd1506ed9 found 0x22ab750a level 0 [ 1398.218803] BTRFS warning (device sde): sde checksum verify failed on 21057127661568 wanted 0xd1506ed9 found 0x7dd54bb9 level 0 [ 1398.218813] BTRFS: error (device sde) in __btrfs_free_extent:3054: errno=-5 IO failure [ 1398.218828] BTRFS: error (device sde) in btrfs_run_delayed_refs:2124: errno=-5 IO failure [ 1398.219002] BTRFS: error (device sde) in btrfs_replay_log:2254: errno=-5 IO failure (Failed to recover log tree) [ 1398.229048] BTRFS error (device sde): open_ctree failedQuoted 7 lines
Quoted 3 lines
This is because we shouldn't allow to do anything write to the fs if we have anything wrong in extent tree.
This is happening when mounting read-only. My assumption is that it only tries to replay in memory without writing anything to disk.
We lacks the check on log tree.
Normally for such forced RO mount, log replay is not allowed.
Quoted 8 lines
Quoted 3 lines
We should output a warning to prompt user to use nologreplay, and reject the mount.
I'm not familiar with log replay but couldn't there be something useful (ignoring ref counts) that would still be worth replaying in memory?
Log replay means metadata write.
Any write needs a valid extent tree to find out free space for new metadata/data.
Quoted 6 lines
Quoted 2 lines
So no, we can't do anything but completely ignoring the log.
I see, updated patch. But even then it seems it could be possible to add new ramdisk and make allocations there (eg. create new extent tree there) thus allowing replay.
The problem here is, since the extent tree is corrupted, we won't know which range has metadata already.
While metadata CoW, just like its name, needs to CoW, which means it can't writeback (even just in memory) to anywhere we have metadata.Quoted 8 lines
Quoted 4 lines
The worst case is, we choose a bytenr for the new metadata to be (in memory), but it turns out later read needs to read metadata from the exactly same location.
The idea is if we add new disk then we would put it after last bytenr (which isn't mapped to any existing disks) thus there wouldn't be any overlap.
Quoted 13 lines
I wonder if that idea can be turned into an online recovery tool. Rebuild the metadata by more or less reflinking the old filesystem's data into a new filesystem created in the unallocated spaces between the old filesystem's block groups (building new subvol trees in the process). It would need the device and chunk trees to be intact, but you have to have those for rescue= to work at all, and it's relatively rarer for those to get damaged. There's a complicated dance that has to be done to flip a block group from the old filesystem to the new one, but maybe that can be done by just making the entire old filesystem into a giant file image, then making ordinary reflinks to it, then finish by deleting the old image and running defrag to discard the unreferenced blocks that remain. Think "btrfs-convert" but using a busted btrfs as source instead of ext4.
That does sound pretty cool and could be doable, but it's way outside of my knowledge about this all to even attempt anything like this.
Quoted 9 lines
The log tree will only contain things that were fsync()ed after the last completed transaction commit. Unless you're hitting a delayed refs latency issue or have non-default mount options, that's data from only the last 30 seconds before the filesystem failed.
It might be desirable to replay the log tree if you had very high-value data there, but it's the last 0.001% of the filesystem that requires the last 99% of the development effort to recover.
Yea for me there was only some XATTR_ITEMs and single INODE_ITEM so nothing that useful. Also I think easier way would be to parse it offline and then apply those changes on data that's copied to new filesystem. That doesn't seem that complicated if someone really needs it.
PATCH 1/1 btrfs: Allow read-only mount with corrupted extent tree
In reply to Dāvis MosānsCurrently if there's any corruption at all in extent tree (eg. even single bit) then mounting will fail with:
"failed to read block groups: -5" (-EIO)
It happens because we immediately abort on first error when searching in extent tree for block groups.
Now with this patch if ignorebadroots option is specified then we handle such case and continue by removing already created block groups and creating dummy block groups.
--- fs/btrfs/block-group.c | 20 ++++++++++++++++++++ fs/btrfs/disk-io.c | 4 ++-- fs/btrfs/disk-io.h | 2 ++ 3 files changed, 24 insertions(+), 2 deletions(-)
fs/btrfs/block-group.c +20 -0
@@ -2048,6 +2048,26 @@ int btrfs_read_block_groups(struct btrfs_fs_info *info) ret =check_chunk_block_group_mappings (info); error:btrfs_free_path (path); + +if (ret == -EIO &&btrfs_test_opt (info,IGNOREBADROOTS )) { + +if (btrfs_super_log_root (info->super_copy) !=0 ) { +btrfs_warn (info,"Ignoring tree-log replay due to extent tree corruption!" ); +btrfs_set_super_log_root (info->super_copy,0 ); + } + +btrfs_put_block_group_cache (info); +btrfs_stop_all_workers (info); +btrfs_free_block_groups (info); + ret =btrfs_init_workqueues (info,NULL ); +if (ret) +return ret; + ret =btrfs_init_space_info (info); +if (ret) +return ret; +return fill_dummy_bgs (info); + } +return ret; }
fs/btrfs/disk-io.c +2 -2
@@ -1955,7 +1955,7 @@ static int read_backup_root(struct btrfs_fs_info *fs_info, u8 priority) }/* helper to cleanup workers */ -static void btrfs_stop_all_workers (struct btrfs_fs_info *fs_info) +void btrfs_stop_all_workers (struct btrfs_fs_info *fs_info) {btrfs_destroy_workqueue (fs_info->fixup_workers);btrfs_destroy_workqueue (fs_info->delalloc_workers);
@@ -2122,7 +2122,7 @@ static void btrfs_init_qgroup(struct btrfs_fs_info *fs_info)mutex_init (&fs_info->qgroup_rescan_lock); } -static int btrfs_init_workqueues (struct btrfs_fs_info *fs_info, +int btrfs_init_workqueues (struct btrfs_fs_info *fs_info,struct btrfs_fs_devices *fs_devices) {u32 max_active = fs_info->thread_pool_size;
fs/btrfs/disk-io.h +2 -0
@@ -137,6 +137,8 @@ int btrfs_find_free_objectid(struct btrfs_root *root, u64 *objectid);int btrfs_find_highest_objectid (struct btrfs_root *root,u64 *objectid);int __init btrfs_end_io_wq_init (void );void __cold btrfs_end_io_wq_exit (void ); +void btrfs_stop_all_workers (struct btrfs_fs_info *fs_info); +int btrfs_init_workqueues (struct btrfs_fs_info *fs_info,struct btrfs_fs_devices *fs_devices);#ifdef CONFIG_DEBUG_LOCK_ALLOC void btrfs_set_buffer_lockdep_class(u64 objectid,
Signature · 1 line
2.30.2
svētd., 2021. g. 21. marts, plkst. 23:46 — lietotājs Dāvis Mosāns (<[email protected]>) rakstīja:
Quoted 85 lines
Currently if there's any corruption at all in extent tree (eg. even single bit) then mounting will fail with:
"failed to read block groups: -5" (-EIO)
It happens because we immediately abort on first error when searching in extent tree for block groups.Now with this patch if
ignorebadrootsoption is specified then we handle such case and continue by removing already created block groups and creating dummy block groups.Signed-off-by Dāvis Mosāns <[email protected]>--- fs/btrfs/block-group.c | 20 ++++++++++++++++++++ fs/btrfs/disk-io.c | 4 ++-- fs/btrfs/disk-io.h | 2 ++ 3 files changed, 24 insertions(+), 2 deletions(-)fs/btrfs/block-group.c +20
-0@@ -2048,6 +2048,26 @@ int btrfs_read_block_groups(struct btrfs_fs_info *info) ret =check_chunk_block_group_mappings (info); error:btrfs_free_path (path); + +if (ret == -EIO &&btrfs_test_opt (info,IGNOREBADROOTS )) { + +if (btrfs_super_log_root (info->super_copy) !=0 ) { +btrfs_warn (info,"Ignoring tree-log replay due to extent tree corruption!" ); +btrfs_set_super_log_root (info->super_copy,0 ); + } + +btrfs_put_block_group_cache (info); +btrfs_stop_all_workers (info); +btrfs_free_block_groups (info); + ret =btrfs_init_workqueues (info,NULL ); +if (ret) +return ret; + ret =btrfs_init_space_info (info); +if (ret) +return ret; +return fill_dummy_bgs (info); + } +return ret; }fs/btrfs/disk-io.c +2
-2@@ -1955,7 +1955,7 @@ static int read_backup_root(struct btrfs_fs_info *fs_info, u8 priority) }/* helper to cleanup workers */ -static void btrfs_stop_all_workers (struct btrfs_fs_info *fs_info) +void btrfs_stop_all_workers (struct btrfs_fs_info *fs_info) {btrfs_destroy_workqueue (fs_info->fixup_workers);btrfs_destroy_workqueue (fs_info->delalloc_workers);@@ -2122,7 +2122,7 @@ static void btrfs_init_qgroup(struct btrfs_fs_info *fs_info)mutex_init (&fs_info->qgroup_rescan_lock); } -static int btrfs_init_workqueues (struct btrfs_fs_info *fs_info, +int btrfs_init_workqueues (struct btrfs_fs_info *fs_info,struct btrfs_fs_devices *fs_devices) {u32 max_active = fs_info->thread_pool_size;fs/btrfs/disk-io.h +2
-1@@ -137,6 +137,8 @@ int btrfs_find_free_objectid(struct btrfs_root *root, u64 *objectid);int btrfs_find_highest_objectid (struct btrfs_root *root,u64 *objectid);int __init btrfs_end_io_wq_init (void );void __cold btrfs_end_io_wq_exit (void ); +void btrfs_stop_all_workers (struct btrfs_fs_info *fs_info); +int btrfs_init_workqueues (struct btrfs_fs_info *fs_info,struct btrfs_fs_devices *fs_devices);#ifdef CONFIG_DEBUG_LOCK_ALLOC void btrfs_set_buffer_lockdep_class(u64 objectid, --
Quoted 2 lines
2.30.2
Ping? Could anyone take a look?