fstests: prevent dm-log-writes out of order replay issues
cover fstests: prevent dm-log-writes out of order replay issues
Here's a stab at addressing the out-of-order write issues reproduced by generic/482 on XFS v5 filesystems. Patch 1 refactors the code to allow for use of an arbitrary device and patch 2 fixes up generic/482 to use a thinly provisioned replay device.
This could be factored in different ways and/or be implemented across the other tests, but the focus here is on generic/482.
Brian
Brian Foster (2):
common/dmlogwrites: genericize log writes target device generic/482: use thin volume as data device
common/dmlogwrites | 24 ++++++++++++++++++------ tests/generic/455 | 8 ++++---- tests/generic/457 | 8 ++++---- tests/generic/470 | 4 ++-- tests/generic/482 | 20 ++++++++++++++------ 5 files changed, 42 insertions(+), 22 deletions(-)
Signature · 1 line
2.17.2
PATCH 1/2 common/dmlogwrites: genericize log writes target device
The dm-log-writes infrastructure is currently implemented to use SCRATCH_DEV as a hardcoded data device. In preparation to allow use of specialized devices in certain circumstances, genericize the code to allow an arbitrary data device. This requires passing the target device as a parameter to several helper functions from various tests. No functional changes.
--- common/dmlogwrites | 24 ++++++++++++++++++------ tests/generic/455 | 8 ++++---- tests/generic/457 | 8 ++++---- tests/generic/470 | 4 ++-- tests/generic/482 | 4 ++-- 5 files changed, 30 insertions(+), 18 deletions(-)
common/dmlogwrites +18 -6
@@ -32,7 +32,7 @@ _require_log_writes_dax() _require_test_program "log-writes/replay-log" local ret=0 - _log_writes_init + _log_writes_init $SCRATCH_DEV _log_writes_mkfs > /dev/null 2>&1 _log_writes_mount -o dax > /dev/null 2>&1 # Check options to be sure. XFS ignores dax option
@@ -47,10 +47,15 @@ _require_log_writes_dax() _log_writes_init() { - local BLK_DEV_SIZE=`blockdev --getsz $SCRATCH_DEV` + blkdev=$1 + + [ -z "$blkdev" ] && _fail \ + "block dev must be specified for _log_writes_init" + + local BLK_DEV_SIZE=`blockdev --getsz $blkdev` LOGWRITES_NAME=logwrites-test LOGWRITES_DMDEV=/dev/mapper/$LOGWRITES_NAME - LOGWRITES_TABLE="0 $BLK_DEV_SIZE log-writes $SCRATCH_DEV $LOGWRITES_DEV" + LOGWRITES_TABLE="0 $BLK_DEV_SIZE log-writes $blkdev $LOGWRITES_DEV" _dmsetup_create $LOGWRITES_NAME --table "$LOGWRITES_TABLE" || \ _fail "failed to create log-writes device" }
@@ -82,17 +87,21 @@ _log_writes_unmount() # _log_writes_replay_log <mark> # -# This replays the log contained on $LOGWRITES_DEV onto $SCRATCH_DEV upto the +# This replays the log contained on $LOGWRITES_DEV onto blkdev upto the # mark passed in. _log_writes_replay_log() { _mark=$1 + _blkdev=$2 + + [ -z "$_blkdev" ] && _fail \ + "block dev must be specified for _log_writes_replay_log" $here/src/log-writes/replay-log --log $LOGWRITES_DEV --find \ --end-mark $_mark >> $seqres.full 2>&1 [ $? -ne 0 ] && _fail "mark '$_mark' does not exist" - $here/src/log-writes/replay-log --log $LOGWRITES_DEV --replay $SCRATCH_DEV \ + $here/src/log-writes/replay-log --log $LOGWRITES_DEV --replay $_blkdev \ --end-mark $_mark >> $seqres.full 2>&1 [ $? -ne 0 ] && _fail "replay failed" }
@@ -150,16 +159,19 @@ _log_writes_find_next_fua() _log_writes_replay_log_range() { local end=$1 + local blkdev=$2 [ -z "$end" ] && _fail \ "end entry must be specified for _log_writes_replay_log_range" + [ -z "$blkdev" ] && _fail \ + "block dev must be specified for _log_writes_replay_log_range" # To ensure we replay the last entry, # we need to manually increase the end entry number to ensure # it's played echo "=== replay to $end ===" >> $seqres.full $here/src/log-writes/replay-log --log $LOGWRITES_DEV \ - --replay $SCRATCH_DEV --limit $(($end + 1)) \ + --replay $blkdev --limit $(($end + 1)) \ >> $seqres.full 2>&1 [ $? -ne 0 ] && _fail "replay failed" }
tests/generic/455 +4 -4
@@ -42,7 +42,7 @@ check_files() local filename=$(basename $i) local mark="${filename##*.}" echo "checking $filename" >> $seqres.full - _log_writes_replay_log $filename + _log_writes_replay_log $filename $SCRATCH_DEV _scratch_mount local expected_md5=$(_md5_checksum $i) local md5=$(_md5_checksum $SCRATCH_MNT/$name)
@@ -57,7 +57,7 @@ rm -rf $SANITY_DIR mkdir $SANITY_DIR # Create the log -_log_writes_init +_log_writes_init $SCRATCH_DEV _log_writes_mkfs >> $seqres.full 2>&1
@@ -92,7 +92,7 @@ _check_scratch_fs # check pre umount echo "checking pre umount" >> $seqres.full -_log_writes_replay_log last +_log_writes_replay_log last $SCRATCH_DEV _scratch_mount _scratch_unmount _check_scratch_fs
@@ -103,7 +103,7 @@ done # Check the end echo "checking post umount" >> $seqres.full -_log_writes_replay_log end +_log_writes_replay_log end $SCRATCH_DEV _scratch_mount for j in `seq 0 $((NUM_FILES-1))`; do md5=$(_md5_checksum $SCRATCH_MNT/testfile$j)
tests/generic/457 +4 -4
@@ -44,7 +44,7 @@ check_files() local filename=$(basename $i) local mark="${filename##*.}" echo "checking $filename" >> $seqres.full - _log_writes_replay_log $filename + _log_writes_replay_log $filename $SCRATCH_DEV _scratch_mount local expected_md5=$(_md5_checksum $i) local md5=$(_md5_checksum $SCRATCH_MNT/$name)
@@ -59,7 +59,7 @@ rm -rf $SANITY_DIR mkdir $SANITY_DIR # Create the log -_log_writes_init +_log_writes_init $SCRATCH_DEV _log_writes_mkfs >> $seqres.full 2>&1
@@ -96,7 +96,7 @@ _check_scratch_fs # check pre umount echo "checking pre umount" >> $seqres.full -_log_writes_replay_log last +_log_writes_replay_log last $SCRATCH_DEV _scratch_mount _scratch_unmount _check_scratch_fs
@@ -107,7 +107,7 @@ done # Check the end echo "checking post umount" >> $seqres.full -_log_writes_replay_log end +_log_writes_replay_log end $SCRATCH_DEV _scratch_mount for j in `seq 0 $((NUM_FILES-1))`; do md5=$(_md5_checksum $SCRATCH_MNT/testfile$j)
tests/generic/470 +2 -2
@@ -39,7 +39,7 @@ _require_log_writes_dax _require_xfs_io_command "mmap" "-S" _require_xfs_io_command "log_writes" -_log_writes_init +_log_writes_init $SCRATCH_DEV _log_writes_mkfs >> $seqres.full 2>&1 _log_writes_mount -o dax
@@ -58,7 +58,7 @@ _check_scratch_fs _scratch_mkfs >> $seqres.full 2>&1 # check pre-unmap state -_log_writes_replay_log preunmap +_log_writes_replay_log preunmap $SCRATCH_DEV _scratch_mount # We should see $SCRATCH_MNT/test as having 1 MiB in block allocations
tests/generic/482 +2 -2
@@ -55,7 +55,7 @@ fsstress_args=$(_scale_fsstress_args -w -d $SCRATCH_MNT -n 512 -p $nr_cpus \ $FSSTRESS_AVOID) _test_unmount -_log_writes_init +_log_writes_init $SCRATCH_DEV _log_writes_mkfs >> $seqres.full 2>&1 _log_writes_mark mkfs
@@ -70,7 +70,7 @@ cur=$(_log_writes_find_next_fua $prev) [ -z "$cur" ] && _fail "failed to locate next FUA write" while [ ! -z "$cur" ]; do - _log_writes_replay_log_range $cur >> $seqres.full + _log_writes_replay_log_range $cur $SCRATCH_DEV >> $seqres.full # Here we need extra mount to replay the log, mainly for journal based # fs, as their fsck will report dirty log as error.
Signature · 1 line
2.17.2
PATCH 2/2 generic/482: use thin volume as data device
In reply to Brian FosterThe dm-log-writes replay mechanism issues discards to provide zeroing functionality to prevent out-of-order replay issues. These discards don't always result in zeroing bevavior, however, depending on the underlying physical device. In turn, this causes test failures on XFS v5 filesystems that enforce metadata log recovery ordering if the filesystem ends up with stale data from the future with respect to the active log at a particular recovery point.
To ensure reliable discard zeroing behavior, use a thinly provisioned volume as the data device instead of using the scratch device directly. This slows the test down slightly, but provides reliable functional behavior at a reduced cost from active snapshot management or forced zeroing.
--- tests/generic/482 | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-)
tests/generic/482 +14 -6
@@ -22,12 +22,14 @@ _cleanup() cd / $KILLALL_PROG -KILL -q $FSSTRESS_PROG &> /dev/null _log_writes_cleanup &> /dev/null + _dmthin_cleanup rm -f $tmp.* } # get standard environment, filters and checks . ./common/rc . ./common/filter +. ./common/dmthin . ./common/dmlogwrites # remove previous $seqres.full before test
@@ -44,6 +46,7 @@ _require_command "$KILLALL_PROG" killall _require_scratch # and we need extra device as log device _require_log_writes +_require_dm_target thin-pool nr_cpus=$("$here/src/feature" -o)
@@ -53,9 +56,15 @@ if [ $nr_cpus -gt 8 ]; then fi fsstress_args=$(_scale_fsstress_args -w -d $SCRATCH_MNT -n 512 -p $nr_cpus \ $FSSTRESS_AVOID) +devsize=$((1024*1024*200 / 512)) # 200m phys/virt size +csize=$((1024*64 / 512)) # 64k cluster size +lowspace=$((1024*1024 / 512)) # 1m low space threshold +# Use a thin device to provide deterministic discard behavior. Discards are used +# by the log replay tool for fast zeroing to prevent out-of-order replay issues. _test_unmount -_log_writes_init $SCRATCH_DEV +_dmthin_init $devsize $devsize $csize $lowspace +_log_writes_init $DMTHIN_VOL_DEV _log_writes_mkfs >> $seqres.full 2>&1 _log_writes_mark mkfs
@@ -70,16 +79,15 @@ cur=$(_log_writes_find_next_fua $prev) [ -z "$cur" ] && _fail "failed to locate next FUA write" while [ ! -z "$cur" ]; do - _log_writes_replay_log_range $cur $SCRATCH_DEV >> $seqres.full + _log_writes_replay_log_range $cur $DMTHIN_VOL_DEV >> $seqres.full # Here we need extra mount to replay the log, mainly for journal based # fs, as their fsck will report dirty log as error. - # We don't care to preserve any data on $SCRATCH_DEV, as we can replay + # We don't care to preserve any data on the replay dev, as we can replay # back to the point we need, and in fact sometimes creating/deleting # snapshots repeatedly can be slower than replaying the log. - _scratch_mount - _scratch_unmount - _check_scratch_fs + _dmthin_mount + _dmthin_check_fs prev=$cur cur=$(_log_writes_find_next_fua $(($cur + 1)))
Signature · 1 line
2.17.2
Re: [PATCH 1/2] common/dmlogwrites: genericize log writes target device
In reply to Brian FosterQuoted 9 lines · Brian Foster
The dm-log-writes infrastructure is currently implemented to use
SCRATCH_DEVas a hardcoded data device. In preparation to allow use of specialized devices in certain circumstances, genericize the code to allow an arbitrary data device. This requires passing the target device as a parameter to several helper functions from various tests. No functional changes.Signed-off-by Brian Foster <[email protected]>
Looks ok.
Quoted 211 lines
--- common/dmlogwrites | 24 ++++++++++++++++++------ tests/generic/455 | 8 ++++---- tests/generic/457 | 8 ++++---- tests/generic/470 | 4 ++-- tests/generic/482 | 4 ++-- 5 files changed, 30 insertions(+), 18 deletions(-)common/dmlogwrites +18
-6@@ -32,7 +32,7 @@ _require_log_writes_dax() _require_test_program "log-writes/replay-log" local ret=0 - _log_writes_init + _log_writes_init $SCRATCH_DEV _log_writes_mkfs > /dev/null 2>&1 _log_writes_mount -o dax > /dev/null 2>&1 # Check options to be sure. XFS ignores dax option@@ -47,10 +47,15 @@ _require_log_writes_dax() _log_writes_init() { - local BLK_DEV_SIZE=`blockdev --getsz $SCRATCH_DEV` + blkdev=$1 + + [ -z "$blkdev" ] && _fail \ + "block dev must be specified for _log_writes_init" + + local BLK_DEV_SIZE=`blockdev --getsz $blkdev` LOGWRITES_NAME=logwrites-test LOGWRITES_DMDEV=/dev/mapper/$LOGWRITES_NAME - LOGWRITES_TABLE="0 $BLK_DEV_SIZE log-writes $SCRATCH_DEV $LOGWRITES_DEV" + LOGWRITES_TABLE="0 $BLK_DEV_SIZE log-writes $blkdev $LOGWRITES_DEV" _dmsetup_create $LOGWRITES_NAME --table "$LOGWRITES_TABLE" || \ _fail "failed to create log-writes device" }@@ -82,17 +87,21 @@ _log_writes_unmount() # _log_writes_replay_log <mark> # -# This replays the log contained on $LOGWRITES_DEV onto $SCRATCH_DEV upto the +# This replays the log contained on $LOGWRITES_DEV onto blkdev upto the # mark passed in. _log_writes_replay_log() { _mark=$1 + _blkdev=$2 + + [ -z "$_blkdev" ] && _fail \ + "block dev must be specified for _log_writes_replay_log" $here/src/log-writes/replay-log --log $LOGWRITES_DEV --find \ --end-mark $_mark >> $seqres.full 2>&1 [ $? -ne 0 ] && _fail "mark '$_mark' does not exist" - $here/src/log-writes/replay-log --log $LOGWRITES_DEV --replay $SCRATCH_DEV \ + $here/src/log-writes/replay-log --log $LOGWRITES_DEV --replay $_blkdev \ --end-mark $_mark >> $seqres.full 2>&1 [ $? -ne 0 ] && _fail "replay failed" }@@ -150,16 +159,19 @@ _log_writes_find_next_fua() _log_writes_replay_log_range() { local end=$1 + local blkdev=$2 [ -z "$end" ] && _fail \ "end entry must be specified for _log_writes_replay_log_range" + [ -z "$blkdev" ] && _fail \ + "block dev must be specified for _log_writes_replay_log_range" # To ensure we replay the last entry, # we need to manually increase the end entry number to ensure # it's played echo "=== replay to $end ===" >> $seqres.full $here/src/log-writes/replay-log --log $LOGWRITES_DEV \ - --replay $SCRATCH_DEV --limit $(($end + 1)) \ + --replay $blkdev --limit $(($end + 1)) \ >> $seqres.full 2>&1 [ $? -ne 0 ] && _fail "replay failed" }tests/generic/455 +4
-4@@ -42,7 +42,7 @@ check_files() local filename=$(basename $i) local mark="${filename##*.}" echo "checking $filename" >> $seqres.full - _log_writes_replay_log $filename + _log_writes_replay_log $filename $SCRATCH_DEV _scratch_mount local expected_md5=$(_md5_checksum $i) local md5=$(_md5_checksum $SCRATCH_MNT/$name)@@ -57,7 +57,7 @@ rm -rf $SANITY_DIR mkdir $SANITY_DIR # Create the log -_log_writes_init +_log_writes_init $SCRATCH_DEV _log_writes_mkfs >> $seqres.full 2>&1@@ -92,7 +92,7 @@ _check_scratch_fs # check pre umount echo "checking pre umount" >> $seqres.full -_log_writes_replay_log last +_log_writes_replay_log last $SCRATCH_DEV _scratch_mount _scratch_unmount _check_scratch_fs@@ -103,7 +103,7 @@ done # Check the end echo "checking post umount" >> $seqres.full -_log_writes_replay_log end +_log_writes_replay_log end $SCRATCH_DEV _scratch_mount for j in `seq 0 $((NUM_FILES-1))`; do md5=$(_md5_checksum $SCRATCH_MNT/testfile$j)tests/generic/457 +4
-4@@ -44,7 +44,7 @@ check_files() local filename=$(basename $i) local mark="${filename##*.}" echo "checking $filename" >> $seqres.full - _log_writes_replay_log $filename + _log_writes_replay_log $filename $SCRATCH_DEV _scratch_mount local expected_md5=$(_md5_checksum $i) local md5=$(_md5_checksum $SCRATCH_MNT/$name)@@ -59,7 +59,7 @@ rm -rf $SANITY_DIR mkdir $SANITY_DIR # Create the log -_log_writes_init +_log_writes_init $SCRATCH_DEV _log_writes_mkfs >> $seqres.full 2>&1@@ -96,7 +96,7 @@ _check_scratch_fs # check pre umount echo "checking pre umount" >> $seqres.full -_log_writes_replay_log last +_log_writes_replay_log last $SCRATCH_DEV _scratch_mount _scratch_unmount _check_scratch_fs@@ -107,7 +107,7 @@ done # Check the end echo "checking post umount" >> $seqres.full -_log_writes_replay_log end +_log_writes_replay_log end $SCRATCH_DEV _scratch_mount for j in `seq 0 $((NUM_FILES-1))`; do md5=$(_md5_checksum $SCRATCH_MNT/testfile$j)tests/generic/470 +2
-2@@ -39,7 +39,7 @@ _require_log_writes_dax _require_xfs_io_command "mmap" "-S" _require_xfs_io_command "log_writes" -_log_writes_init +_log_writes_init $SCRATCH_DEV _log_writes_mkfs >> $seqres.full 2>&1 _log_writes_mount -o dax@@ -58,7 +58,7 @@ _check_scratch_fs _scratch_mkfs >> $seqres.full 2>&1 # check pre-unmap state -_log_writes_replay_log preunmap +_log_writes_replay_log preunmap $SCRATCH_DEV _scratch_mount # We should see $SCRATCH_MNT/test as having 1 MiB in block allocationstests/generic/482 +2
-3@@ -55,7 +55,7 @@ fsstress_args=$(_scale_fsstress_args -w -d $SCRATCH_MNT -n 512 -p $nr_cpus \ $FSSTRESS_AVOID) _test_unmount -_log_writes_init +_log_writes_init $SCRATCH_DEV _log_writes_mkfs >> $seqres.full 2>&1 _log_writes_mark mkfs@@ -70,7 +70,7 @@ cur=$(_log_writes_find_next_fua $prev) [ -z "$cur" ] && _fail "failed to locate next FUA write" while [ ! -z "$cur" ]; do - _log_writes_replay_log_range $cur >> $seqres.full + _log_writes_replay_log_range $cur $SCRATCH_DEV >> $seqres.full # Here we need extra mount to replay the log, mainly for journal based # fs, as their fsck will report dirty log as error. --2.17.2
Re: [PATCH 2/2] generic/482: use thin volume as data device
In reply to Brian FosterQuoted 9 lines · Brian Foster
The dm-log-writes replay mechanism issues discards to provide zeroing functionality to prevent out-of-order replay issues. These discards don't always result in zeroing bevavior, however, depending on the underlying physical device. In turn, this causes test failures on XFS v5 filesystems that enforce metadata log recovery ordering if the filesystem ends up with stale data from the future with respect to the active log at a particular recovery point.
Quoted 7 lines
To ensure reliable discard zeroing behavior, use a thinly provisioned volume as the data device instead of using the scratch device directly. This slows the test down slightly, but provides reliable functional behavior at a reduced cost from active snapshot management or forced zeroing.
Signed-off-by Brian Foster <[email protected]>
Looks ok. Apart from minor nit below
Quoted 24 lines
--- tests/generic/482 | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-)tests/generic/482 +2
-0@@ -22,12 +22,14 @@ _cleanup() cd / $KILLALL_PROG -KILL -q $FSSTRESS_PROG &> /dev/null _log_writes_cleanup &> /dev/null + _dmthin_cleanup rm -f $tmp.* } # get standard environment, filters and checks . ./common/rc . ./common/filter +. ./common/dmthin . ./common/dmlogwrites # remove previous $seqres.full before test
tests/generic/482 _require_command "$KILLALL_PROG" killall · line 46
Quoted 1 line
tests/generic/482 +0
-0@@ -44,6 +46,7 @@ _require_command "$KILLALL_PROG" killall _require_scratch
_require_scratch_nocheck
Quoted 47 lines
tests/generic/482 +12
-7# and we need extra device as log device _require_log_writes +_require_dm_target thin-pool nr_cpus=$("$here/src/feature" -o)@@ -53,9 +56,15 @@ if [ $nr_cpus -gt 8 ]; then fi fsstress_args=$(_scale_fsstress_args -w -d $SCRATCH_MNT -n 512 -p $nr_cpus \ $FSSTRESS_AVOID) +devsize=$((1024*1024*200 / 512)) # 200m phys/virt size +csize=$((1024*64 / 512)) # 64k cluster size +lowspace=$((1024*1024 / 512)) # 1m low space threshold +# Use a thin device to provide deterministic discard behavior. Discards are used +# by the log replay tool for fast zeroing to prevent out-of-order replay issues. _test_unmount -_log_writes_init $SCRATCH_DEV +_dmthin_init $devsize $devsize $csize $lowspace +_log_writes_init $DMTHIN_VOL_DEV _log_writes_mkfs >> $seqres.full 2>&1 _log_writes_mark mkfs@@ -70,16 +79,15 @@ cur=$(_log_writes_find_next_fua $prev) [ -z "$cur" ] && _fail "failed to locate next FUA write" while [ ! -z "$cur" ]; do - _log_writes_replay_log_range $cur $SCRATCH_DEV >> $seqres.full + _log_writes_replay_log_range $cur $DMTHIN_VOL_DEV >> $seqres.full # Here we need extra mount to replay the log, mainly for journal based # fs, as their fsck will report dirty log as error. - # We don't care to preserve any data on $SCRATCH_DEV, as we can replay + # We don't care to preserve any data on the replay dev, as we can replay # back to the point we need, and in fact sometimes creating/deleting # snapshots repeatedly can be slower than replaying the log. - _scratch_mount - _scratch_unmount - _check_scratch_fs + _dmthin_mount + _dmthin_check_fs prev=$cur cur=$(_log_writes_find_next_fua $(($cur + 1))) --2.17.2