Skip to contentSkip to conversation

fix up generic dmlogwrites tests to work with XFS

PATCH In review Brian Foster 2 participants 7 messages Sep 1, 2020

lore
Share & tools
lore
#0 lore raw

v3 cover fix up generic dmlogwrites tests to work with XFS

Hi all,

This version removes the discard zeroing check as Christoph points out that discard behavior is generally not predictable. The remaining generic dm-logwrites tests are still updated to use dm-thinp such that discards from the recovery tool clear block ranges reasonably reliably. This allows the tests to function correctly on filesystems like XFS while a more generic solution is investigated.

Brian

v3:
- Re-add dm-thinp changes.
- Drop discard checks (from v1) and XFS disablement (from v2).
- Use _require_scratch_nocheck in generic/470 to avoid spurious repair noise now that the test no longer runs mkfs on the scratch dev.
v2: https://lore.kernel.org/fstests/[email protected]/
- Drop all dmthinp changes. Unconditionally disable tests on XFS.
v1: https://lore.kernel.org/fstests/[email protected]/

Brian Foster (3):
generic/455: use thin volume for dmlogwrites target device generic/457: use thin volume for dmlogwrites target device generic/470: use thin volume for dmlogwrites target device

 tests/generic/455 | 36 ++++++++++++++++++++++--------------
 tests/generic/457 | 33 +++++++++++++++++++++------------
 tests/generic/470 | 23 +++++++++++++++++------
 3 files changed, 60 insertions(+), 32 deletions(-)
Signature · 1 line
2.25.4
#1 lore raw

PATCH v3 1/3 generic/455: use thin volume for dmlogwrites target device

dmlogwrites support for XFS depends on discard zeroing support of the intended target device. Update the test to use a thin volume and allow it to run consistently and reliably on XFS.

---
 tests/generic/455 | 36 ++++++++++++++++++++++--------------
 1 file changed, 22 insertions(+), 14 deletions(-)
tests/generic/455 +22 -14
index 05621220..72a44fda 100755
@@ -16,12 +16,14 @@ status=1	# failure is the default!
 _cleanup()
 {
 	_log_writes_cleanup
+	_dmthin_cleanup
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 
 # get standard environment, filters and checks
 . ./common/rc
 . ./common/filter
+. ./common/dmthin
 . ./common/dmlogwrites
 
 # real QA test starts here
@@ -30,6 +32,7 @@ _supported_os Linux
 _require_test
 _require_scratch_nocheck
 _require_log_writes
+_require_dm_target thin-pool
 
 rm -f $seqres.full
 
@@ -42,13 +45,12 @@ check_files()
 		local filename=$(basename $i)
 		local mark="${filename##*.}"
 		echo "checking $filename" >> $seqres.full
-		_log_writes_replay_log $filename $SCRATCH_DEV
-		_scratch_mount
+		_log_writes_replay_log $filename $DMTHIN_VOL_DEV
+		_dmthin_mount
 		local expected_md5=$(_md5_checksum $i)
 		local md5=$(_md5_checksum $SCRATCH_MNT/$name)
 		[ "${md5}" != "${expected_md5}" ] && _fail "$filename md5sum mismatched"
-		_scratch_unmount
-		_check_scratch_fs
+		_dmthin_check_fs
 	done
 }
 
@@ -56,8 +58,16 @@ SANITY_DIR=$TEST_DIR/fsxtests
 rm -rf $SANITY_DIR
 mkdir $SANITY_DIR
 
+devsize=$((1024*1024*200 / 512))        # 200m phys/virt size
+csize=$((1024*64 / 512))                # 64k cluster size
+lowspace=$((1024*1024 / 512))           # 1m low space threshold
+
+# Use a thin device to provide deterministic discard behavior. Discards are used
+# by the log replay tool for fast zeroing to prevent out-of-order replay issues.
+_dmthin_init $devsize $devsize $csize $lowspace
+
 # Create the log
-_log_writes_init $SCRATCH_DEV
+_log_writes_init $DMTHIN_VOL_DEV
 
 _log_writes_mkfs >> $seqres.full 2>&1
 
@@ -88,14 +98,13 @@ _log_writes_mark last
 _log_writes_unmount
 _log_writes_mark end
 _log_writes_remove
-_check_scratch_fs
+_dmthin_check_fs
 
 # check pre umount
 echo "checking pre umount" >> $seqres.full
-_log_writes_replay_log last $SCRATCH_DEV
-_scratch_mount
-_scratch_unmount
-_check_scratch_fs
+_log_writes_replay_log last $DMTHIN_VOL_DEV
+_dmthin_mount
+_dmthin_check_fs
 
 for j in `seq 0 $((NUM_FILES-1))`; do
 	check_files testfile$j
@@ -103,14 +112,13 @@ done
 
 # Check the end
 echo "checking post umount" >> $seqres.full
-_log_writes_replay_log end $SCRATCH_DEV
-_scratch_mount
+_log_writes_replay_log end $DMTHIN_VOL_DEV
+_dmthin_mount
 for j in `seq 0 $((NUM_FILES-1))`; do
 	md5=$(_md5_checksum $SCRATCH_MNT/testfile$j)
 	[ "${md5}" != "${test_md5[$j]}" ] && _fail "testfile$j end md5sum mismatched"
 done
-_scratch_unmount
-_check_scratch_fs
+_dmthin_check_fs
 
 echo "Silence is golden"
 status=0
Signature · 1 line
2.25.4
Reviewed-by on v3 1/3
#2 lore raw

Re: [PATCH v3 1/3] generic/455: use thin volume for dmlogwrites target device

Quoted 7 lines · Brian Foster

dmlogwrites support for XFS depends on discard zeroing support of the intended target device. Update the test to use a thin volume and allow it to run consistently and reliably on XFS.

---

Quoted 105 lines
 tests/generic/455 | 36 ++++++++++++++++++++++--------------
 1 file changed, 22 insertions(+), 14 deletions(-)
tests/generic/455 +22 -15
index 05621220..72a44fda 100755
@@ -16,12 +16,14 @@ status=1    # failure is the default!
 _cleanup()
 {
        _log_writes_cleanup
+       _dmthin_cleanup
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 
 # get standard environment, filters and checks
 . ./common/rc
 . ./common/filter
+. ./common/dmthin
 . ./common/dmlogwrites
 
 # real QA test starts here
@@ -30,6 +32,7 @@ _supported_os Linux
 _require_test
 _require_scratch_nocheck
 _require_log_writes
+_require_dm_target thin-pool
 
 rm -f $seqres.full
 
@@ -42,13 +45,12 @@ check_files()
                local filename=$(basename $i)
                local mark="${filename##*.}"
                echo "checking $filename" >> $seqres.full
-               _log_writes_replay_log $filename $SCRATCH_DEV
-               _scratch_mount
+               _log_writes_replay_log $filename $DMTHIN_VOL_DEV
+               _dmthin_mount
                local expected_md5=$(_md5_checksum $i)
                local md5=$(_md5_checksum $SCRATCH_MNT/$name)
                [ "${md5}" != "${expected_md5}" ] && _fail "$filename md5sum mismatched"
-               _scratch_unmount
-               _check_scratch_fs
+               _dmthin_check_fs
        done
 }
 
@@ -56,8 +58,16 @@ SANITY_DIR=$TEST_DIR/fsxtests
 rm -rf $SANITY_DIR
 mkdir $SANITY_DIR
 
+devsize=$((1024*1024*200 / 512))        # 200m phys/virt size
+csize=$((1024*64 / 512))                # 64k cluster size
+lowspace=$((1024*1024 / 512))           # 1m low space threshold
+
+# Use a thin device to provide deterministic discard behavior. Discards are used
+# by the log replay tool for fast zeroing to prevent out-of-order replay issues.
+_dmthin_init $devsize $devsize $csize $lowspace
+
 # Create the log
-_log_writes_init $SCRATCH_DEV
+_log_writes_init $DMTHIN_VOL_DEV
 
 _log_writes_mkfs >> $seqres.full 2>&1
 
@@ -88,14 +98,13 @@ _log_writes_mark last
 _log_writes_unmount
 _log_writes_mark end
 _log_writes_remove
-_check_scratch_fs
+_dmthin_check_fs
 
 # check pre umount
 echo "checking pre umount" >> $seqres.full
-_log_writes_replay_log last $SCRATCH_DEV
-_scratch_mount
-_scratch_unmount
-_check_scratch_fs
+_log_writes_replay_log last $DMTHIN_VOL_DEV
+_dmthin_mount
+_dmthin_check_fs
 
 for j in `seq 0 $((NUM_FILES-1))`; do
        check_files testfile$j
@@ -103,14 +112,13 @@ done
 
 # Check the end
 echo "checking post umount" >> $seqres.full
-_log_writes_replay_log end $SCRATCH_DEV
-_scratch_mount
+_log_writes_replay_log end $DMTHIN_VOL_DEV
+_dmthin_mount
 for j in `seq 0 $((NUM_FILES-1))`; do
        md5=$(_md5_checksum $SCRATCH_MNT/testfile$j)
        [ "${md5}" != "${test_md5[$j]}" ] && _fail "testfile$j end md5sum mismatched"
 done
-_scratch_unmount
-_check_scratch_fs
+_dmthin_check_fs
 
 echo "Silence is golden"
 status=0
--

2.25.4

#3 lore raw

PATCH v3 2/3 generic/457: use thin volume for dmlogwrites target device

In reply to Brian Foster

dmlogwrites support for XFS depends on discard zeroing support of the intended target device. Update the test to use a thin volume and allow it to run consistently and reliably on XFS.

---
 tests/generic/457 | 33 +++++++++++++++++++++------------
 1 file changed, 21 insertions(+), 12 deletions(-)
tests/generic/457 +21 -12
index 82367304..42a064d8 100755
@@ -16,6 +16,7 @@ status=1	# failure is the default!
 _cleanup()
 {
 	_log_writes_cleanup
+	_dmthin_cleanup
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 
@@ -23,6 +24,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 . ./common/rc
 . ./common/filter
 . ./common/reflink
+. ./common/dmthin
 . ./common/dmlogwrites
 
 # real QA test starts here
@@ -32,6 +34,7 @@ _require_test
 _require_scratch_reflink
 _require_cp_reflink
 _require_log_writes
+_require_dm_target thin-pool
 
 rm -f $seqres.full
 
@@ -44,13 +47,12 @@ check_files()
 		local filename=$(basename $i)
 		local mark="${filename##*.}"
 		echo "checking $filename" >> $seqres.full
-		_log_writes_replay_log $filename $SCRATCH_DEV
-		_scratch_mount
+		_log_writes_replay_log $filename $DMTHIN_VOL_DEV
+		_dmthin_mount
 		local expected_md5=$(_md5_checksum $i)
 		local md5=$(_md5_checksum $SCRATCH_MNT/$name)
 		[ "${md5}" != "${expected_md5}" ] && _fail "$filename md5sum mismatched"
-		_scratch_unmount
-		_check_scratch_fs
+		_dmthin_check_fs
 	done
 }
 
@@ -58,8 +60,16 @@ SANITY_DIR=$TEST_DIR/fsxtests
 rm -rf $SANITY_DIR
 mkdir $SANITY_DIR
 
+devsize=$((1024*1024*200 / 512))        # 200m phys/virt size
+csize=$((1024*64 / 512))                # 64k cluster size
+lowspace=$((1024*1024 / 512))           # 1m low space threshold
+
+# Use a thin device to provide deterministic discard behavior. Discards are used
+# by the log replay tool for fast zeroing to prevent out-of-order replay issues.
+_dmthin_init $devsize $devsize $csize $lowspace
+
 # Create the log
-_log_writes_init $SCRATCH_DEV
+_log_writes_init $DMTHIN_VOL_DEV
 
 _log_writes_mkfs >> $seqres.full 2>&1
 
@@ -92,14 +102,13 @@ _log_writes_mark last
 _log_writes_unmount
 _log_writes_mark end
 _log_writes_remove
-_check_scratch_fs
+_dmthin_check_fs
 
 # check pre umount
 echo "checking pre umount" >> $seqres.full
-_log_writes_replay_log last $SCRATCH_DEV
-_scratch_mount
-_scratch_unmount
-_check_scratch_fs
+_log_writes_replay_log last $DMTHIN_VOL_DEV
+_dmthin_mount
+_dmthin_check_fs
 
 for j in `seq 0 $((NUM_FILES-1))`; do
 	check_files testfile$j
@@ -107,8 +116,8 @@ done
 
 # Check the end
 echo "checking post umount" >> $seqres.full
-_log_writes_replay_log end $SCRATCH_DEV
-_scratch_mount
+_log_writes_replay_log end $DMTHIN_VOL_DEV
+_dmthin_mount
 for j in `seq 0 $((NUM_FILES-1))`; do
 	md5=$(_md5_checksum $SCRATCH_MNT/testfile$j)
 	[ "${md5}" != "${test_md5[$j]}" ] && _fail "testfile$j end md5sum mismatched"
Signature · 1 line
2.25.4
Reviewed-by on v3 2/3
#4 lore raw

Re: [PATCH v3 2/3] generic/457: use thin volume for dmlogwrites target device

Quoted 7 lines · Brian Foster

dmlogwrites support for XFS depends on discard zeroing support of the intended target device. Update the test to use a thin volume and allow it to run consistently and reliably on XFS.

---

Quoted 99 lines
 tests/generic/457 | 33 +++++++++++++++++++++------------
 1 file changed, 21 insertions(+), 12 deletions(-)
tests/generic/457 +21 -13
index 82367304..42a064d8 100755
@@ -16,6 +16,7 @@ status=1      # failure is the default!
 _cleanup()
 {
        _log_writes_cleanup
+       _dmthin_cleanup
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 
@@ -23,6 +24,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 . ./common/rc
 . ./common/filter
 . ./common/reflink
+. ./common/dmthin
 . ./common/dmlogwrites
 
 # real QA test starts here
@@ -32,6 +34,7 @@ _require_test
 _require_scratch_reflink
 _require_cp_reflink
 _require_log_writes
+_require_dm_target thin-pool
 
 rm -f $seqres.full
 
@@ -44,13 +47,12 @@ check_files()
                local filename=$(basename $i)
                local mark="${filename##*.}"
                echo "checking $filename" >> $seqres.full
-               _log_writes_replay_log $filename $SCRATCH_DEV
-               _scratch_mount
+               _log_writes_replay_log $filename $DMTHIN_VOL_DEV
+               _dmthin_mount
                local expected_md5=$(_md5_checksum $i)
                local md5=$(_md5_checksum $SCRATCH_MNT/$name)
                [ "${md5}" != "${expected_md5}" ] && _fail "$filename md5sum mismatched"
-               _scratch_unmount
-               _check_scratch_fs
+               _dmthin_check_fs
        done
 }
 
@@ -58,8 +60,16 @@ SANITY_DIR=$TEST_DIR/fsxtests
 rm -rf $SANITY_DIR
 mkdir $SANITY_DIR
 
+devsize=$((1024*1024*200 / 512))        # 200m phys/virt size
+csize=$((1024*64 / 512))                # 64k cluster size
+lowspace=$((1024*1024 / 512))           # 1m low space threshold
+
+# Use a thin device to provide deterministic discard behavior. Discards are used
+# by the log replay tool for fast zeroing to prevent out-of-order replay issues.
+_dmthin_init $devsize $devsize $csize $lowspace
+
 # Create the log
-_log_writes_init $SCRATCH_DEV
+_log_writes_init $DMTHIN_VOL_DEV
 
 _log_writes_mkfs >> $seqres.full 2>&1
 
@@ -92,14 +102,13 @@ _log_writes_mark last
 _log_writes_unmount
 _log_writes_mark end
 _log_writes_remove
-_check_scratch_fs
+_dmthin_check_fs
 
 # check pre umount
 echo "checking pre umount" >> $seqres.full
-_log_writes_replay_log last $SCRATCH_DEV
-_scratch_mount
-_scratch_unmount
-_check_scratch_fs
+_log_writes_replay_log last $DMTHIN_VOL_DEV
+_dmthin_mount
+_dmthin_check_fs
 
 for j in `seq 0 $((NUM_FILES-1))`; do
        check_files testfile$j
@@ -107,8 +116,8 @@ done
 
 # Check the end
 echo "checking post umount" >> $seqres.full
-_log_writes_replay_log end $SCRATCH_DEV
-_scratch_mount
+_log_writes_replay_log end $DMTHIN_VOL_DEV
+_dmthin_mount
 for j in `seq 0 $((NUM_FILES-1))`; do
        md5=$(_md5_checksum $SCRATCH_MNT/testfile$j)
        [ "${md5}" != "${test_md5[$j]}" ] && _fail "testfile$j end md5sum mismatched"
--

2.25.4

#5 lore raw

PATCH v3 3/3 generic/470: use thin volume for dmlogwrites target device

In reply to Brian Foster

dmlogwrites support for XFS depends on discard zeroing support of the intended target device. Update the test to use a thin volume and allow it to run consistently and reliably on XFS.

---
 tests/generic/470 | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)
tests/generic/470 +17 -6
index fd6da563..c77499a2 100755
@@ -20,12 +20,14 @@ _cleanup()
 {
 	cd /
 	_log_writes_cleanup
+	_dmthin_cleanup
 	rm -f $tmp.*
 }
 
 # get standard environment, filters and checks
 . ./common/rc
 . ./common/filter
+. ./common/dmthin
 . ./common/dmlogwrites
 
 # remove previous $seqres.full before test
@@ -34,12 +36,21 @@ rm -f $seqres.full
 # real QA test starts here
 _supported_fs generic
 _supported_os Linux
-_require_scratch
+_require_scratch_nocheck
 _require_log_writes_dax_mountopt "dax"
+_require_dm_target thin-pool
 _require_xfs_io_command "mmap" "-S"
 _require_xfs_io_command "log_writes"
 
-_log_writes_init $SCRATCH_DEV
+devsize=$((1024*1024*200 / 512))        # 200m phys/virt size
+csize=$((1024*64 / 512))                # 64k cluster size
+lowspace=$((1024*1024 / 512))           # 1m low space threshold
+
+# Use a thin device to provide deterministic discard behavior. Discards are used
+# by the log replay tool for fast zeroing to prevent out-of-order replay issues.
+_dmthin_init $devsize $devsize $csize $lowspace
+
+_log_writes_init $DMTHIN_VOL_DEV
 _log_writes_mkfs >> $seqres.full 2>&1
 _log_writes_mount -o dax
 
@@ -52,14 +63,14 @@ $XFS_IO_PROG -t -c "truncate $LEN" -c "mmap -S 0 $LEN" -c "mwrite 0 $LEN" \
 # Unmount the scratch dir and tear down the log writes target
 _log_writes_unmount
 _log_writes_remove
-_check_scratch_fs
+_dmthin_check_fs
 
 # destroy previous filesystem so we can be sure our rebuild works
-_scratch_mkfs >> $seqres.full 2>&1
+_mkfs_dev $DMTHIN_VOL_DEV >> $seqres.full 2>&1
 
 # check pre-unmap state
-_log_writes_replay_log preunmap $SCRATCH_DEV
-_scratch_mount
+_log_writes_replay_log preunmap $DMTHIN_VOL_DEV
+_dmthin_mount
 
 # We should see $SCRATCH_MNT/test as having 1 MiB in block allocations
 du -sh $SCRATCH_MNT/test | _filter_scratch | _filter_spaces
Signature · 1 line
2.25.4
Reviewed-by on v3 3/3
#6 lore raw

Re: [PATCH v3 3/3] generic/470: use thin volume for dmlogwrites target device

Quoted 7 lines · Brian Foster

dmlogwrites support for XFS depends on discard zeroing support of the intended target device. Update the test to use a thin volume and allow it to run consistently and reliably on XFS.

---

Quoted 68 lines
 tests/generic/470 | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)
tests/generic/470 +17 -7
index fd6da563..c77499a2 100755
@@ -20,12 +20,14 @@ _cleanup()
 {
        cd /
        _log_writes_cleanup
+       _dmthin_cleanup
        rm -f $tmp.*
 }
 
 # get standard environment, filters and checks
 . ./common/rc
 . ./common/filter
+. ./common/dmthin
 . ./common/dmlogwrites
 
 # remove previous $seqres.full before test
@@ -34,12 +36,21 @@ rm -f $seqres.full
 # real QA test starts here
 _supported_fs generic
 _supported_os Linux
-_require_scratch
+_require_scratch_nocheck
 _require_log_writes_dax_mountopt "dax"
+_require_dm_target thin-pool
 _require_xfs_io_command "mmap" "-S"
 _require_xfs_io_command "log_writes"
 
-_log_writes_init $SCRATCH_DEV
+devsize=$((1024*1024*200 / 512))        # 200m phys/virt size
+csize=$((1024*64 / 512))                # 64k cluster size
+lowspace=$((1024*1024 / 512))           # 1m low space threshold
+
+# Use a thin device to provide deterministic discard behavior. Discards are used
+# by the log replay tool for fast zeroing to prevent out-of-order replay issues.
+_dmthin_init $devsize $devsize $csize $lowspace
+
+_log_writes_init $DMTHIN_VOL_DEV
 _log_writes_mkfs >> $seqres.full 2>&1
 _log_writes_mount -o dax
 
@@ -52,14 +63,14 @@ $XFS_IO_PROG -t -c "truncate $LEN" -c "mmap -S 0 $LEN" -c "mwrite 0 $LEN" \
 # Unmount the scratch dir and tear down the log writes target
 _log_writes_unmount
 _log_writes_remove
-_check_scratch_fs
+_dmthin_check_fs
 
 # destroy previous filesystem so we can be sure our rebuild works
-_scratch_mkfs >> $seqres.full 2>&1
+_mkfs_dev $DMTHIN_VOL_DEV >> $seqres.full 2>&1
 
 # check pre-unmap state
-_log_writes_replay_log preunmap $SCRATCH_DEV
-_scratch_mount
+_log_writes_replay_log preunmap $DMTHIN_VOL_DEV
+_dmthin_mount
 
 # We should see $SCRATCH_MNT/test as having 1 MiB in block allocations
 du -sh $SCRATCH_MNT/test | _filter_scratch | _filter_spaces
--

2.25.4