btrfs-progs: Fix the nobarrier behavior of write
cover btrfs-progs: Fix the nobarrier behavior of write
This urgent patchset can be fetched from github:
https://github.com/adam900710/btrfs-progs/tree/flush_super
Which is based on v4.20.2.
Before this patch, btrfs-progs writes to the fs has no barrier at all. All metadata and superblock are just buffered write, no barrier between super blocks and metadata writes at all.
No wonder why even clear space cache can cause serious transid corruption to the originally good fs.
Please merge this fix as soon as possible as I really don't want to see btrfs-progs corrupting any fs any more.
Qu Wenruo (2):
btrfs-progs: disk-io: Make super block write error more easy to read
btrfs-progs: disk-io: Flush to ensure super block write is FUA
disk-io.c | 45 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 39 insertions(+), 6 deletions(-)
Signature · 1 line
2.21.0
PATCH 1/2 btrfs-progs: disk-io: Make super block write error more easy to read
When we failed to write super blocks, we just output something like:
WARNING: failed to write sb: I/O error
Or
WARNING: failed to write all sb data
There is no info about which device failed and there are two different error message for the same write error.
This patch will change it to something more detailed:
ERROR: failed to write super block: I/O error
ERROR: failed to write super block for devid 1
This provides the basis for later super block flush error handling.
--- disk-io.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-)
disk-io.c +11 -6
@@ -1599,8 +1599,12 @@ static int write_dev_supers(struct btrfs_fs_info *fs_info, ret =pwrite64 (device->fd, fs_info->super_copy,BTRFS_SUPER_INFO_SIZE , fs_info->super_bytenr); -if (ret !=BTRFS_SUPER_INFO_SIZE ) +if (ret !=BTRFS_SUPER_INFO_SIZE ) { + errno =EIO ; + ret = -errno; +error ("failed to write super block: %m" ); goto write_err; + }return 0 ; }
@@ -1622,17 +1626,18 @@ static int write_dev_supers(struct btrfs_fs_info *fs_info,*/ ret =pwrite64 (device->fd, fs_info->super_copy,BTRFS_SUPER_INFO_SIZE , bytenr); -if (ret !=BTRFS_SUPER_INFO_SIZE ) +if (ret !=BTRFS_SUPER_INFO_SIZE ) { + errno =EIO ; + ret = -errno; +error ("failed to write super block %i: %m" , i); goto write_err; + } }return 0 ; write_err: -if (ret >0 ) -fprintf (stderr,"WARNING: failed to write all sb data\n" ); -else -fprintf (stderr,"WARNING: failed to write sb: %m\n" ); +error ("failed to write super block for devid %llu" , device->devid);return ret; }
Signature · 1 line
2.21.0
Re: [PATCH URGENT 1/2] btrfs-progs: disk-io: Make super block write error more easy to read
Quoted 11 lines · Qu Wenruo
When we failed to write super blocks, we just output something like:
WARNING: failed to write sb: I/O error
Or
WARNING: failed to write all sb dataThere is no info about which device failed and there are two different error message for the same write error.
This patch will change it to something more detailed:
ERROR: failed to write super block: I/O error
ERROR: failed to write super block for devid 1
But now we print 2 messages for 1 error. I'd rather have the information conveyed on a single line. I.e call fprintf directly from both callsites that do goto write_err with respective format and rename write_err label to out.
Quoted 51 lines
This provides the basis for later super block flush error handling.
Signed-off-by Qu Wenruo <[email protected]>--- disk-io.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-)disk-io.c +11
-6@@ -1599,8 +1599,12 @@ static int write_dev_supers(struct btrfs_fs_info *fs_info, ret =pwrite64 (device->fd, fs_info->super_copy,BTRFS_SUPER_INFO_SIZE , fs_info->super_bytenr); -if (ret !=BTRFS_SUPER_INFO_SIZE ) +if (ret !=BTRFS_SUPER_INFO_SIZE ) { + errno =EIO ; + ret = -errno; +error ("failed to write super block: %m" ); goto write_err; + }return 0 ; }@@ -1622,17 +1626,18 @@ static int write_dev_supers(struct btrfs_fs_info *fs_info,*/ ret =pwrite64 (device->fd, fs_info->super_copy,BTRFS_SUPER_INFO_SIZE , bytenr); -if (ret !=BTRFS_SUPER_INFO_SIZE ) +if (ret !=BTRFS_SUPER_INFO_SIZE ) { + errno =EIO ; + ret = -errno; +error ("failed to write super block %i: %m" , i); goto write_err; + } }return 0 ; write_err: -if (ret >0 ) -fprintf (stderr,"WARNING: failed to write all sb data\n" ); -else -fprintf (stderr,"WARNING: failed to write sb: %m\n" ); +error ("failed to write super block for devid %llu" , device->devid);return ret; }
Re: [PATCH URGENT 1/2] btrfs-progs: disk-io: Make super block write error more easy to read
Quoted 11 lines · Nikolay Borisov
Quoted 8 lines · Qu Wenruo
When we failed to write super blocks, we just output something like:
WARNING: failed to write sb: I/O error
Or
WARNING: failed to write all sb dataThere is no info about which device failed and there are two different error message for the same write error.
Quoted 8 lines
Quoted 3 lines
This patch will change it to something more detailed:
ERROR: failed to write super block: I/O error
ERROR: failed to write super block for devid 1But now we print 2 messages for 1 error. I'd rather have the information conveyed on a single line. I.e call fprintf directly from both callsites that do goto
write_errwith respective format and renamewrite_errlabel to out.
Makes sense.
I'll go that way.
Thanks,
Qu
Quoted 52 lines
Quoted 52 lines
This provides the basis for later super block flush error handling.
Signed-off-by Qu Wenruo <[email protected]>--- disk-io.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-)disk-io.c +11
-6@@ -1599,8 +1599,12 @@ static int write_dev_supers(struct btrfs_fs_info *fs_info, ret =pwrite64 (device->fd, fs_info->super_copy,BTRFS_SUPER_INFO_SIZE , fs_info->super_bytenr); -if (ret !=BTRFS_SUPER_INFO_SIZE ) +if (ret !=BTRFS_SUPER_INFO_SIZE ) { + errno =EIO ; + ret = -errno; +error ("failed to write super block: %m" ); goto write_err; + }return 0 ; }@@ -1622,17 +1626,18 @@ static int write_dev_supers(struct btrfs_fs_info *fs_info,*/ ret =pwrite64 (device->fd, fs_info->super_copy,BTRFS_SUPER_INFO_SIZE , bytenr); -if (ret !=BTRFS_SUPER_INFO_SIZE ) +if (ret !=BTRFS_SUPER_INFO_SIZE ) { + errno =EIO ; + ret = -errno; +error ("failed to write super block %i: %m" , i); goto write_err; + } }return 0 ; write_err: -if (ret >0 ) -fprintf (stderr,"WARNING: failed to write all sb data\n" ); -else -fprintf (stderr,"WARNING: failed to write sb: %m\n" ); +error ("failed to write super block for devid %llu" , device->devid);return ret; }
PATCH 2/2 btrfs-progs: disk-io: Flush to ensure super block write is FUA
In reply to Qu Wenruo[BUG]
There are tons of reports of btrfs-progs screwing up the fs, the most recent one is "btrfs check --clear-space-cache v1" triggered BUG_ON() and then leaving the fs with transid mismatch problem.
[CAUSE]
In kernel, we have block layer handing the flush work, even on devices without FUA support (like most SATA device using default libata settings), kernel handles FUA write by flushing the device, then normal write, and finish it with another flush.
The pre-flush, write, post-flush works pretty well to implement FUA write.
However in btrfs-progs we just use pwrite(), there is nothing keeping the write order.
So even for basic v1 free space cache clearing, we have different vision on the write sequence from kernel bio layer (by dm-log-writes) and user space pwrite() calls.
In btrfs-progs, with extra debug output in write_tree_block() and write_dev_supers(), we can see btrfs-progs follows the right write sequence:
Opening filesystem to check...
Checking filesystem on /dev/mapper/log
UUID: 3feb3c8b-4eb3-42f3-8e9c-0af22dd58ecf
write tree block start=1708130304 gen=39
write tree block start=1708146688 gen=39
write tree block start=1708163072 gen=39
write super devid=1 gen=39
write tree block start=1708179456 gen=40
write tree block start=1708195840 gen=40
write super devid=1 gen=40
write tree block start=1708130304 gen=41
write tree block start=1708146688 gen=41
write tree block start=1708228608 gen=41
write super devid=1 gen=41
write tree block start=1708163072 gen=42
write tree block start=1708179456 gen=42
write super devid=1 gen=42
write tree block start=1708130304 gen=43
write tree block start=1708146688 gen=43
write super devid=1 gen=43
Free space cache cleared
But from dm-log-writes, the bio sequence is a different story:
replaying 1742: sector 131072, size 4096, flags 0(NONE) replaying 1743: sector 128, size 4096, flags 0(NONE) <<< Only one sb write replaying 1744: sector 2828480, size 4096, flags 0(NONE) replaying 1745: sector 2828488, size 4096, flags 0(NONE) replaying 1746: sector 2828496, size 4096, flags 0(NONE) replaying 1787: sector 2304120, size 4096, flags 0(NONE)
......
replaying 1790: sector 2304144, size 4096, flags 0(NONE) replaying 1791: sector 2304152, size 4096, flags 0(NONE) replaying 1792: sector 0, size 0, flags 8(MARK)
During the free space cache clearing, we committed 3 transaction but dm-log-write only caught one super block write.
This means all the 3 writes were merged into the last super block write. And the super block write was the 2nd write, before all tree block writes, completely screwing up the metadata CoW protection.
No wonder crashed btrfs-progs can make things worse.
[FIX]
Fix this super serious problem by implementing pre and post flush for the primary super block in btrfs-progs.
--- disk-io.c | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+)
disk-io.c +28 -0
@@ -1585,6 +1585,16 @@ static int write_dev_supers(struct btrfs_fs_info *fs_info,u32 crc;int i, ret; +/* +* We need to write super block after all metadata written. +* This is the equivalent of kernel pre-flush for FUA. +*/ + ret =fsync (device->fd); +if (ret <0 ) { +error ("failed to flush: %m" ); + ret = -errno; + goto write_err; + }if (fs_info->super_bytenr !=BTRFS_SUPER_INFO_OFFSET ) {btrfs_set_super_bytenr (sb, fs_info->super_bytenr); crc = ~(u32 )0 ;
@@ -1605,6 +1615,12 @@ static int write_dev_supers(struct btrfs_fs_info *fs_info,error ("failed to write super block: %m" ); goto write_err; } + ret =fsync (device->fd); +if (ret <0 ) { +error ("failed to flush: %m" ); + ret = -errno; + goto write_err; + }return 0 ; }
@@ -1632,6 +1648,18 @@ static int write_dev_supers(struct btrfs_fs_info *fs_info,error ("failed to write super block %i: %m" , i); goto write_err; } +/* +* Flush after the primary sb write, this is the equivalent of +* kernel post-flush for FUA write. +*/ +if (i ==0 ) { + ret =fsync (device->fd); +if (ret <0 ) { +error ("failed to flush: %m" ); + ret = -errno; + goto write_err; + } + } }return 0 ;
Signature · 1 line
2.21.0
Re: [PATCH URGENT 2/2] btrfs-progs: disk-io: Flush to ensure super block write is FUA
Quoted 11 lines · Qu Wenruo
[BUG]
There are tons of reports of btrfs-progs screwing up the fs, the most recent one is "btrfs check --clear-space-cache v1" triggeredBUG_ON()and then leaving the fs with transid mismatch problem.[CAUSE]
In kernel, we have block layer handing the flush work, even on devices without FUA support (like most SATA device using default libata settings), kernel handles FUA write by flushing the device, then normal write, and finish it with another flush.
Quoted 5 lines
The pre-flush, write, post-flush works pretty well to implement FUA write.
However in btrfs-progs we just use
pwrite(), there is nothing keeping the write order.
This could be expanded to include the following information:
1. The device fs is opened with only O_RD or O_RDWR i.e we don't pass any special arguments when opening the fd.
2. We do not use fsync anywhere
Quoted 51 lines
So even for basic v1 free space cache clearing, we have different vision on the write sequence from kernel bio layer (by dm-log-writes) and user space
pwrite()calls.In btrfs-progs, with extra debug output in
write_tree_block()andwrite_dev_supers(), we can see btrfs-progs follows the right write sequence:Opening filesystem to check...
Checking filesystem on/dev/mapper/log
UUID: 3feb3c8b-4eb3-42f3-8e9c-0af22dd58ecf
write tree block start=1708130304 gen=39
write tree block start=1708146688 gen=39
write tree block start=1708163072 gen=39
write super devid=1 gen=39
write tree block start=1708179456 gen=40
write tree block start=1708195840 gen=40
write super devid=1 gen=40
write tree block start=1708130304 gen=41
write tree block start=1708146688 gen=41
write tree block start=1708228608 gen=41
write super devid=1 gen=41
write tree block start=1708163072 gen=42
write tree block start=1708179456 gen=42
write super devid=1 gen=42
write tree block start=1708130304 gen=43
write tree block start=1708146688 gen=43
write super devid=1 gen=43
Free space cache clearedBut from dm-log-writes, the bio sequence is a different story:
replaying 1742: sector 131072, size 4096, flags 0(NONE) replaying 1743: sector 128, size 4096, flags 0(NONE) <<< Only one sb write replaying 1744: sector 2828480, size 4096, flags 0(NONE) replaying 1745: sector 2828488, size 4096, flags 0(NONE) replaying 1746: sector 2828496, size 4096, flags 0(NONE) replaying 1787: sector 2304120, size 4096, flags 0(NONE)
......
replaying 1790: sector 2304144, size 4096, flags 0(NONE) replaying 1791: sector 2304152, size 4096, flags 0(NONE) replaying 1792: sector 0, size 0, flags 8(MARK)During the free space cache clearing, we committed 3 transaction but dm-log-write only caught one super block write.
This means all the 3 writes were merged into the last super block write. And the super block write was the 2nd write, before all tree block writes, completely screwing up the metadata CoW protection.
Quoted 7 lines
No wonder crashed btrfs-progs can make things worse.
[FIX]
Fix this super serious problem by implementing pre and post flush for the primary super block in btrfs-progs.Signed-off-by Qu Wenruo <[email protected]>
Quoted 58 lines
--- disk-io.c | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+)disk-io.c +28
-0@@ -1585,6 +1585,16 @@ static int write_dev_supers(struct btrfs_fs_info *fs_info,u32 crc;int i, ret; +/* +* We need to write super block after all metadata written. +* This is the equivalent of kernel pre-flush for FUA. +*/ + ret =fsync (device->fd); +if (ret <0 ) { +error ("failed to flush: %m" ); + ret = -errno; + goto write_err; + }if (fs_info->super_bytenr !=BTRFS_SUPER_INFO_OFFSET ) {btrfs_set_super_bytenr (sb, fs_info->super_bytenr); crc = ~(u32 )0 ;@@ -1605,6 +1615,12 @@ static int write_dev_supers(struct btrfs_fs_info *fs_info,error ("failed to write super block: %m" ); goto write_err; } + ret =fsync (device->fd); +if (ret <0 ) { +error ("failed to flush: %m" ); + ret = -errno; + goto write_err; + }return 0 ; }@@ -1632,6 +1648,18 @@ static int write_dev_supers(struct btrfs_fs_info *fs_info,error ("failed to write super block %i: %m" , i); goto write_err; } +/* +* Flush after the primary sb write, this is the equivalent of +* kernel post-flush for FUA write. +*/ +if (i ==0 ) { + ret =fsync (device->fd); +if (ret <0 ) { +error ("failed to flush: %m" ); + ret = -errno; + goto write_err; + } + } }return 0 ;
Re: [PATCH URGENT 2/2] btrfs-progs: disk-io: Flush to ensure super block write is FUA
Quoted 17 lines · Nikolay Borisov
Quoted 14 lines · Qu Wenruo
[BUG]
There are tons of reports of btrfs-progs screwing up the fs, the most recent one is "btrfs check --clear-space-cache v1" triggeredBUG_ON()and then leaving the fs with transid mismatch problem.[CAUSE]
In kernel, we have block layer handing the flush work, even on devices without FUA support (like most SATA device using default libata settings), kernel handles FUA write by flushing the device, then normal write, and finish it with another flush.The pre-flush, write, post-flush works pretty well to implement FUA write.
Quoted 7 lines
Quoted 2 lines
However in btrfs-progs we just use
pwrite(), there is nothing keeping the write order.This could be expanded to include the following information:
1. The device fs is opened with only
O_RDorO_RDWRi.e we don't pass any special arguments when opening the fd.
In fact you can pass O_SYNC to solve the problem, but with possible performance penalty.
In fact, the fsync() call here is completely compatible with any open flag except O_RDONLY.
So I'm not sure why extra comment on this is needed.
Quoted 2 lines
2. We do not use fsync anywhere
Even we use, it won't cause any problem.
Here we only need to ensure one barrier for super block, if another part of btrfs-progs wants an extra barrier, it's completely fine to do so.
So I don't see much point to explain both points.
Quoted 120 lines
Quoted 52 lines
So even for basic v1 free space cache clearing, we have different vision on the write sequence from kernel bio layer (by dm-log-writes) and user space
pwrite()calls.In btrfs-progs, with extra debug output in
write_tree_block()andwrite_dev_supers(), we can see btrfs-progs follows the right write sequence:Opening filesystem to check...
Checking filesystem on/dev/mapper/log
UUID: 3feb3c8b-4eb3-42f3-8e9c-0af22dd58ecf
write tree block start=1708130304 gen=39
write tree block start=1708146688 gen=39
write tree block start=1708163072 gen=39
write super devid=1 gen=39
write tree block start=1708179456 gen=40
write tree block start=1708195840 gen=40
write super devid=1 gen=40
write tree block start=1708130304 gen=41
write tree block start=1708146688 gen=41
write tree block start=1708228608 gen=41
write super devid=1 gen=41
write tree block start=1708163072 gen=42
write tree block start=1708179456 gen=42
write super devid=1 gen=42
write tree block start=1708130304 gen=43
write tree block start=1708146688 gen=43
write super devid=1 gen=43
Free space cache clearedBut from dm-log-writes, the bio sequence is a different story:
replaying 1742: sector 131072, size 4096, flags 0(NONE) replaying 1743: sector 128, size 4096, flags 0(NONE) <<< Only one sb write replaying 1744: sector 2828480, size 4096, flags 0(NONE) replaying 1745: sector 2828488, size 4096, flags 0(NONE) replaying 1746: sector 2828496, size 4096, flags 0(NONE) replaying 1787: sector 2304120, size 4096, flags 0(NONE)
......
replaying 1790: sector 2304144, size 4096, flags 0(NONE) replaying 1791: sector 2304152, size 4096, flags 0(NONE) replaying 1792: sector 0, size 0, flags 8(MARK)During the free space cache clearing, we committed 3 transaction but dm-log-write only caught one super block write.
This means all the 3 writes were merged into the last super block write. And the super block write was the 2nd write, before all tree block writes, completely screwing up the metadata CoW protection.
Quoted 7 lines
No wonder crashed btrfs-progs can make things worse.
[FIX]
Fix this super serious problem by implementing pre and post flush for the primary super block in btrfs-progs.Signed-off-by Qu Wenruo <[email protected]>Reviewed-by Nikolay Borisov <[email protected]>Quoted 58 lines
--- disk-io.c | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+)disk-io.c +28
-0@@ -1585,6 +1585,16 @@ static int write_dev_supers(struct btrfs_fs_info *fs_info,u32 crc;int i, ret; +/* +* We need to write super block after all metadata written. +* This is the equivalent of kernel pre-flush for FUA. +*/ + ret =fsync (device->fd); +if (ret <0 ) { +error ("failed to flush: %m" ); + ret = -errno; + goto write_err; + }if (fs_info->super_bytenr !=BTRFS_SUPER_INFO_OFFSET ) {btrfs_set_super_bytenr (sb, fs_info->super_bytenr); crc = ~(u32 )0 ;@@ -1605,6 +1615,12 @@ static int write_dev_supers(struct btrfs_fs_info *fs_info,error ("failed to write super block: %m" ); goto write_err; } + ret =fsync (device->fd); +if (ret <0 ) { +error ("failed to flush: %m" ); + ret = -errno; + goto write_err; + }return 0 ; }@@ -1632,6 +1648,18 @@ static int write_dev_supers(struct btrfs_fs_info *fs_info,error ("failed to write super block %i: %m" , i); goto write_err; } +/* +* Flush after the primary sb write, this is the equivalent of +* kernel post-flush for FUA write. +*/ +if (i ==0 ) { + ret =fsync (device->fd); +if (ret <0 ) { +error ("failed to flush: %m" ); + ret = -errno; + goto write_err; + } + } }return 0 ;