Add more tests for multi fs block atomic writes

The main motivation of adding this function on top of _require_fio is that there has been a case in fio where atomic= option was added but later it was changed to noop since kernel didn't yet have support for atomic writes. It was then again utilized to do atomic writes in a later version, once kernel got the support. Due to this there is a point in fio where _require_fio w/ atomic=1 will succeed even though it would not be doing atomic writes.

Hence, add an internal helper __require_fio_version to require specific versions of fio to work past such issues. Further, add the high level _require_fio_atomic_writes helper which tests can use to ensure fio has the right version for atomic writes.

Reviewed-by Zorro Lang <[email protected]>

Reviewed-by John Garry <[email protected]>

Signed-off-by Ojaswin Mujoo <[email protected]>

---
 common/rc | 43 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)

common/rc +43 -0

index 28fbbcbb..8a023b9d 100644
@@ -6000,6 +6000,49 @@ _max() {
 	echo $ret
 }
 
+# Due to reasons explained in fio commit 40f1fc11d, fio version between
+# v3.33 and v3.38 have atomic= feature but it is a no-op and doesn't do
+# RWF_ATOMIC write. Hence, use this helper to ensure fio has the
+# required support. Currently, the simplest way we have is to ensure
+# the version.
+_require_fio_atomic_writes() {
+	__require_fio_version "3.38+"
+}
+
+# Check the required fio version. Examples:
+#   __require_fio_version 3.38 (matches 3.38 only)
+#   __require_fio_version 3.38+ (matches 3.38 and above)
+#   __require_fio_version 3.38- (matches 3.38 and below)
+#
+# Internal helper, avoid using directly in tests.
+__require_fio_version() {
+	local req_ver="$1"
+	local fio_ver
+
+	_require_fio
+	_require_math
+
+	fio_ver=$(fio -v | cut -d"-" -f2)
+
+	case "$req_ver" in
+	*+)
+		req_ver=${req_ver%+}
+		test $(_math "$fio_ver >= $req_ver") -eq 1 || \
+			_notrun "need fio >= $req_ver (found $fio_ver)"
+		;;
+	*-)
+		req_ver=${req_ver%-}
+		test $(_math "$fio_ver <= $req_ver") -eq 1 || \
+			_notrun "need fio <= $req_ver (found $fio_ver)"
+		;;
+	*)
+		req_ver=${req_ver%-}
+		test $(_math "$fio_ver == $req_ver") -eq 1 || \
+			_notrun "need fio = $req_ver (found $fio_ver)"
+		;;
+	esac
+}
+
 ################################################################################
 # make sure this script returns success
 /bin/true

Signature · 1 line

2.49.0

Implement atomic write support to help fuzz atomic writes with fsx.

Suggested-by Ritesh Harjani (IBM) <[email protected]>

Reviewed-by Darrick J. Wong <[email protected]>

Reviewed-by John Garry <[email protected]>

Signed-off-by Ojaswin Mujoo <[email protected]>

---
 ltp/fsx.c | 115 +++++++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 110 insertions(+), 5 deletions(-)

ltp/fsx.c +60 -3

index 163b9453..bdb87ca9 100644
@@ -40,6 +40,7 @@
 #include <liburing.h>
 #endif
 #include <sys/syscall.h>
+#include "statx.h"
 
 #ifndef MAP_FILE
 # define MAP_FILE 0

@@ -49,6 +50,10 @@
 #define RWF_DONTCACHE	0x80
 #endif
 
+#ifndef RWF_ATOMIC
+#define RWF_ATOMIC	0x40
+#endif
+
 #define NUMPRINTCOLUMNS 32	/* # columns of data to print on each line */
 
 /* Operation flags (bitmask) */

@@ -110,6 +115,7 @@ enum {
 	OP_READ_DONTCACHE,
 	OP_WRITE,
 	OP_WRITE_DONTCACHE,
+	OP_WRITE_ATOMIC,
 	OP_MAPREAD,
 	OP_MAPWRITE,
 	OP_MAX_LITE,

@@ -200,6 +206,11 @@ int	uring = 0;
 int	mark_nr = 0;
 int	dontcache_io = 1;
 int	hugepages = 0;                  /* -h flag */
+int	do_atomic_writes = 1;		/* -a flag disables */
+
+/* User for atomic writes */
+int awu_min = 0;
+int awu_max = 0;
 
 /* Stores info needed to periodically collapse hugepages */
 struct hugepages_collapse_info {

@@ -288,6 +299,7 @@ static const char *op_names[] = {
 	[OP_READ_DONTCACHE] = "read_dontcache",
 	[OP_WRITE] = "write",
 	[OP_WRITE_DONTCACHE] = "write_dontcache",
+	[OP_WRITE_ATOMIC] = "write_atomic",
 	[OP_MAPREAD] = "mapread",
 	[OP_MAPWRITE] = "mapwrite",
 	[OP_TRUNCATE] = "truncate",

@@ -422,6 +434,7 @@ logdump(void)
 				prt("\t***RRRR***");
 			break;
 		case OP_WRITE_DONTCACHE:
+		case OP_WRITE_ATOMIC:
 		case OP_WRITE:
 			prt("WRITE    0x%x thru 0x%x\t(0x%x bytes)",
 			    lp->args[0], lp->args[0] + lp->args[1] - 1,

@@ -1073,6 +1086,25 @@ update_file_size(unsigned offset, unsigned size)
 	file_size = offset + size;
 }
 
+static int is_power_of_2(unsigned n) {
+	return ((n & (n - 1)) == 0);
+}
+
+/*
+ * Round down n to nearest power of 2.
+ * If n is already a power of 2, return n;
+ */
+static int rounddown_pow_of_2(int n) {
+	int i = 0;
+
+	if (is_power_of_2(n))
+		return n;
+
+	for (; (1 << i) < n; i++);
+
+	return 1 << (i - 1);
+}
+
 void
 dowrite(unsigned offset, unsigned size, int flags)
 {

@@ -1081,6 +1113,27 @@ dowrite(unsigned offset, unsigned size, int flags)
 	offset -= offset % writebdy;
 	if (o_direct)
 		size -= size % writebdy;
+	if (flags & RWF_ATOMIC) {
+		/* atomic write len must be between awu_min and awu_max */
+		if (size < awu_min)
+			size = awu_min;
+		if (size > awu_max)
+			size = awu_max;
+
+		/* atomic writes need power-of-2 sizes */
+		size = rounddown_pow_of_2(size);
+
+		/* atomic writes need naturally aligned offsets */
+		offset -= offset % size;
+
+		/* Skip the write if we are crossing max filesize */
+		if ((offset + size) > maxfilelen) {
+			if (!quiet && testcalls > simulatedopcount)
+				prt("skipping atomic write past maxfilelen\n");
+			log4(OP_WRITE_ATOMIC, offset, size, FL_SKIPPED);
+			return;
+		}
+	}
 	if (size == 0) {
 		if (!quiet && testcalls > simulatedopcount && !o_direct)
 			prt("skipping zero size write\n");

@@ -1088,7 +1141,10 @@ dowrite(unsigned offset, unsigned size, int flags)
 		return;
 	}
 
-	log4(OP_WRITE, offset, size, FL_NONE);
+	if (flags & RWF_ATOMIC)
+		log4(OP_WRITE_ATOMIC, offset, size, FL_NONE);
+	else
+		log4(OP_WRITE, offset, size, FL_NONE);
 
 	gendata(original_buf, good_buf, offset, size);
 	if (offset + size > file_size) {

@@ -1108,8 +1164,9 @@ dowrite(unsigned offset, unsigned size, int flags)
 		       (monitorstart == -1 ||
 			(offset + size > monitorstart &&
 			(monitorend == -1 || offset <= monitorend))))))
-		prt("%lld write\t0x%x thru\t0x%x\t(0x%x bytes)\tdontcache=%d\n", testcalls,
-		    offset, offset + size - 1, size, (flags & RWF_DONTCACHE) != 0);
+		prt("%lld write\t0x%x thru\t0x%x\t(0x%x bytes)\tdontcache=%d atomic_wr=%d\n", testcalls,
+		    offset, offset + size - 1, size, (flags & RWF_DONTCACHE) != 0,
+		    (flags & RWF_ATOMIC) != 0);
 	iret = fsxwrite(fd, good_buf + offset, size, offset, flags);
 	if (iret != size) {
 		if (iret == -1)

ltp/fsx.c +10 -0

+		return 0;
+	}
+
+	ret = xfstests_statx(AT_FDCWD, fname, 0, STATX_WRITE_ATOMIC, &stx);
+	if (ret < 0) {
+		fprintf(stderr, "main: Statx failed with %d."
+			" Failed to determine atomic write limits, "
+			" disabling!\n", ret);
+		return 0;
+	}

ltp/fsx.c +23 -2

+	return 0;
+}
+
 #ifdef HAVE_COPY_FILE_RANGE
 int
 test_copy_range(void)

@@ -2356,6 +2443,12 @@ have_op:
 			goto out;
 		}
 		break;
+	case OP_WRITE_ATOMIC:
+		if (!do_atomic_writes) {
+			log4(OP_WRITE_ATOMIC, offset, size, FL_SKIPPED);
+			goto out;
+		}
+		break;
 	}
 
 	switch (op) {

@@ -2385,6 +2478,11 @@ have_op:
 			dowrite(offset, size, 0);
 		break;
 
+	case OP_WRITE_ATOMIC:
+		TRIM_OFF_LEN(offset, size, maxfilelen);
+		dowrite(offset, size, RWF_ATOMIC);
+		break;
+
 	case OP_MAPREAD:
 		TRIM_OFF_LEN(offset, size, file_size);
 		domapread(offset, size);

@@ -2511,13 +2609,14 @@ void
 usage(void)
 {
 	fprintf(stdout, "usage: %s",
-		"fsx [-dfhknqxyzBEFHIJKLORWXZ0]\n\
+		"fsx [-adfhknqxyzBEFHIJKLORWXZ0]\n\
 	   [-b opnum] [-c Prob] [-g filldata] [-i logdev] [-j logid]\n\
 	   [-l flen] [-m start:end] [-o oplen] [-p progressinterval]\n\
 	   [-r readbdy] [-s style] [-t truncbdy] [-w writebdy]\n\
 	   [-A|-U] [-D startingop] [-N numops] [-P dirpath] [-S seed]\n\
 	   [--replay-ops=opsfile] [--record-ops[=opsfile]] [--duration=seconds]\n\
 	   ... fname\n\
+	-a: disable atomic writes\n\
 	-b opnum: beginning operation number (default 1)\n\
 	-c P: 1 in P chance of file close+open at each op (default infinity)\n\
 	-d: debug output for all operations\n\

@@ -3059,9 +3158,13 @@ main(int argc, char **argv)
 	setvbuf(stdout, (char *)0, _IOLBF, 0); /* line buffered stdout */
 
 	while ((ch = getopt_long(argc, argv,
-				 "0b:c:de:fg:hi:j:kl:m:no:p:qr:s:t:uw:xyABD:EFJKHzCILN:OP:RS:UWXZ",
+				 "0ab:c:de:fg:hi:j:kl:m:no:p:qr:s:t:uw:xyABD:EFJKHzCILN:OP:RS:UWXZ",
 				 longopts, NULL)) != EOF)
 		switch (ch) {
+		case 'a':
+			prt("main(): Atomic writes disabled\n");
+			do_atomic_writes = 0;
+			break;
 		case 'b':
 			simulatedopcount = getnum(optarg, &endp);
 			if (!quiet)

@@ -3475,6 +3578,8 @@ main(int argc, char **argv)
 		exchange_range_calls = test_exchange_range();
 	if (dontcache_io)
 		dontcache_io = test_dontcache_io();
+	if (do_atomic_writes)
+		do_atomic_writes = test_atomic_writes();
 
 	while (keep_running())
 		if (!test())

Signature · 1 line

2.49.0

Implement atomic write support to help fuzz atomic writes with fsx.

Suggested-by Ritesh Harjani (IBM) <[email protected]>

Reviewed-by Darrick J. Wong <[email protected]>

Reviewed-by John Garry <[email protected]>

Signed-off-by Ojaswin Mujoo <[email protected]>

---

 ltp/fsx.c | 115 +++++++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 110 insertions(+), 5 deletions(-)

ltp/fsx.c +110 -5

index 163b9453..bdb87ca9 100644
@@ -40,6 +40,7 @@
 #include <liburing.h>
 #endif
 #include <sys/syscall.h>
+#include "statx.h"
 
 #ifndef MAP_FILE
 # define MAP_FILE 0

@@ -49,6 +50,10 @@
 #define RWF_DONTCACHE	0x80
 #endif
 
+#ifndef RWF_ATOMIC
+#define RWF_ATOMIC	0x40
+#endif
+
 #define NUMPRINTCOLUMNS 32	/* # columns of data to print on each line */
 
 /* Operation flags (bitmask) */

@@ -110,6 +115,7 @@ enum {
 	OP_READ_DONTCACHE,
 	OP_WRITE,
 	OP_WRITE_DONTCACHE,
+	OP_WRITE_ATOMIC,
 	OP_MAPREAD,
 	OP_MAPWRITE,
 	OP_MAX_LITE,

@@ -200,6 +206,11 @@ int	uring = 0;
 int	mark_nr = 0;
 int	dontcache_io = 1;
 int	hugepages = 0;                  /* -h flag */
+int	do_atomic_writes = 1;		/* -a flag disables */
+
+/* User for atomic writes */
+int awu_min = 0;
+int awu_max = 0;
 
 /* Stores info needed to periodically collapse hugepages */
 struct hugepages_collapse_info {

@@ -288,6 +299,7 @@ static const char *op_names[] = {
 	[OP_READ_DONTCACHE] = "read_dontcache",
 	[OP_WRITE] = "write",
 	[OP_WRITE_DONTCACHE] = "write_dontcache",
+	[OP_WRITE_ATOMIC] = "write_atomic",
 	[OP_MAPREAD] = "mapread",
 	[OP_MAPWRITE] = "mapwrite",
 	[OP_TRUNCATE] = "truncate",

@@ -422,6 +434,7 @@ logdump(void)
 				prt("\t***RRRR***");
 			break;
 		case OP_WRITE_DONTCACHE:
+		case OP_WRITE_ATOMIC:
 		case OP_WRITE:
 			prt("WRITE    0x%x thru 0x%x\t(0x%x bytes)",
 			    lp->args[0], lp->args[0] + lp->args[1] - 1,

@@ -1073,6 +1086,25 @@ update_file_size(unsigned offset, unsigned size)
 	file_size = offset + size;
 }
 
+static int is_power_of_2(unsigned n) {
+	return ((n & (n - 1)) == 0);
+}
+
+/*
+ * Round down n to nearest power of 2.
+ * If n is already a power of 2, return n;
+ */
+static int rounddown_pow_of_2(int n) {
+	int i = 0;
+
+	if (is_power_of_2(n))
+		return n;
+
+	for (; (1 << i) < n; i++);
+
+	return 1 << (i - 1);
+}
+
 void
 dowrite(unsigned offset, unsigned size, int flags)
 {

@@ -1081,6 +1113,27 @@ dowrite(unsigned offset, unsigned size, int flags)
 	offset -= offset % writebdy;
 	if (o_direct)
 		size -= size % writebdy;
+	if (flags & RWF_ATOMIC) {
+		/* atomic write len must be between awu_min and awu_max */
+		if (size < awu_min)
+			size = awu_min;
+		if (size > awu_max)
+			size = awu_max;
+
+		/* atomic writes need power-of-2 sizes */
+		size = rounddown_pow_of_2(size);
+
+		/* atomic writes need naturally aligned offsets */
+		offset -= offset % size;
+
+		/* Skip the write if we are crossing max filesize */
+		if ((offset + size) > maxfilelen) {
+			if (!quiet && testcalls > simulatedopcount)
+				prt("skipping atomic write past maxfilelen\n");
+			log4(OP_WRITE_ATOMIC, offset, size, FL_SKIPPED);
+			return;
+		}
+	}
 	if (size == 0) {
 		if (!quiet && testcalls > simulatedopcount && !o_direct)
 			prt("skipping zero size write\n");

@@ -1088,7 +1141,10 @@ dowrite(unsigned offset, unsigned size, int flags)
 		return;
 	}
 
-	log4(OP_WRITE, offset, size, FL_NONE);
+	if (flags & RWF_ATOMIC)
+		log4(OP_WRITE_ATOMIC, offset, size, FL_NONE);
+	else
+		log4(OP_WRITE, offset, size, FL_NONE);
 
 	gendata(original_buf, good_buf, offset, size);
 	if (offset + size > file_size) {

@@ -1108,8 +1164,9 @@ dowrite(unsigned offset, unsigned size, int flags)
 		       (monitorstart == -1 ||
 			(offset + size > monitorstart &&
 			(monitorend == -1 || offset <= monitorend))))))
-		prt("%lld write\t0x%x thru\t0x%x\t(0x%x bytes)\tdontcache=%d\n", testcalls,
-		    offset, offset + size - 1, size, (flags & RWF_DONTCACHE) != 0);
+		prt("%lld write\t0x%x thru\t0x%x\t(0x%x bytes)\tdontcache=%d atomic_wr=%d\n", testcalls,
+		    offset, offset + size - 1, size, (flags & RWF_DONTCACHE) != 0,
+		    (flags & RWF_ATOMIC) != 0);
 	iret = fsxwrite(fd, good_buf + offset, size, offset, flags);
 	if (iret != size) {
 		if (iret == -1)

@@ -1785,6 +1842,36 @@ do_dedupe_range(unsigned offset, unsigned length, unsigned dest)
 }
 #endif
 
+int test_atomic_writes(void) {
+	int ret;
+	struct statx stx;
+
+	if (o_direct != O_DIRECT) {
+		fprintf(stderr, "main: atomic writes need O_DIRECT (-Z), "
+				"disabling!\n");
+		return 0;
+	}
+
+	ret = xfstests_statx(AT_FDCWD, fname, 0, STATX_WRITE_ATOMIC, &stx);
+	if (ret < 0) {
+		fprintf(stderr, "main: Statx failed with %d."
+			" Failed to determine atomic write limits, "
+			" disabling!\n", ret);
+		return 0;
+	}
+
+	if (stx.stx_attributes & STATX_ATTR_WRITE_ATOMIC &&
+	    stx.stx_atomic_write_unit_min > 0) {
+		awu_min = stx.stx_atomic_write_unit_min;
+		awu_max = stx.stx_atomic_write_unit_max;
+		return 1;
+	}
+
+	fprintf(stderr, "main: IO Stack does not support "
+			"atomic writes, disabling!\n");
+	return 0;
+}
+
 #ifdef HAVE_COPY_FILE_RANGE
 int
 test_copy_range(void)

@@ -2356,6 +2443,12 @@ have_op:
 			goto out;
 		}
 		break;
+	case OP_WRITE_ATOMIC:
+		if (!do_atomic_writes) {
+			log4(OP_WRITE_ATOMIC, offset, size, FL_SKIPPED);
+			goto out;
+		}
+		break;
 	}
 
 	switch (op) {

@@ -2385,6 +2478,11 @@ have_op:
 			dowrite(offset, size, 0);
 		break;
 
+	case OP_WRITE_ATOMIC:
+		TRIM_OFF_LEN(offset, size, maxfilelen);
+		dowrite(offset, size, RWF_ATOMIC);
+		break;
+
 	case OP_MAPREAD:
 		TRIM_OFF_LEN(offset, size, file_size);
 		domapread(offset, size);

@@ -2511,13 +2609,14 @@ void
 usage(void)
 {
 	fprintf(stdout, "usage: %s",
-		"fsx [-dfhknqxyzBEFHIJKLORWXZ0]\n\
+		"fsx [-adfhknqxyzBEFHIJKLORWXZ0]\n\
 	   [-b opnum] [-c Prob] [-g filldata] [-i logdev] [-j logid]\n\
 	   [-l flen] [-m start:end] [-o oplen] [-p progressinterval]\n\
 	   [-r readbdy] [-s style] [-t truncbdy] [-w writebdy]\n\
 	   [-A|-U] [-D startingop] [-N numops] [-P dirpath] [-S seed]\n\
 	   [--replay-ops=opsfile] [--record-ops[=opsfile]] [--duration=seconds]\n\
 	   ... fname\n\
+	-a: disable atomic writes\n\
 	-b opnum: beginning operation number (default 1)\n\
 	-c P: 1 in P chance of file close+open at each op (default infinity)\n\
 	-d: debug output for all operations\n\

@@ -3059,9 +3158,13 @@ main(int argc, char **argv)
 	setvbuf(stdout, (char *)0, _IOLBF, 0); /* line buffered stdout */
 
 	while ((ch = getopt_long(argc, argv,
-				 "0b:c:de:fg:hi:j:kl:m:no:p:qr:s:t:uw:xyABD:EFJKHzCILN:OP:RS:UWXZ",
+				 "0ab:c:de:fg:hi:j:kl:m:no:p:qr:s:t:uw:xyABD:EFJKHzCILN:OP:RS:UWXZ",
 				 longopts, NULL)) != EOF)
 		switch (ch) {
+		case 'a':
+			prt("main(): Atomic writes disabled\n");
+			do_atomic_writes = 0;
+			break;
 		case 'b':
 			simulatedopcount = getnum(optarg, &endp);
 			if (!quiet)

@@ -3475,6 +3578,8 @@ main(int argc, char **argv)
 		exchange_range_calls = test_exchange_range();
 	if (dontcache_io)
 		dontcache_io = test_dontcache_io();
+	if (do_atomic_writes)
+		do_atomic_writes = test_atomic_writes();
 
 	while (keep_running())
 		if (!test())

Signature · 1 line

2.49.0

Quoted 8 lines · Ojaswin Mujoo

Implement atomic write support to help fuzz atomic writes with fsx.

Suggested-by Ritesh Harjani (IBM) <[email protected]>

Reviewed-by Darrick J. Wong <[email protected]>

Reviewed-by John Garry <[email protected]>

Signed-off-by Ojaswin Mujoo <[email protected]>

---

Hmm... this patch causes more regular fsx test cases fail on old kernel, (e.g. g/760, g/617, g/263 ...) except set "FSX_AVOID=-a". Is there a way to disable "atomic write" automatically if it's not supported by current system?

Thanks,
Zorro

Quoted 245 lines

 ltp/fsx.c | 115 +++++++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 110 insertions(+), 5 deletions(-)

ltp/fsx.c +110 -5

index 163b9453..bdb87ca9 100644
@@ -40,6 +40,7 @@
 #include <liburing.h>
 #endif
 #include <sys/syscall.h>
+#include "statx.h"
 
 #ifndef MAP_FILE
 # define MAP_FILE 0

@@ -49,6 +50,10 @@
 #define RWF_DONTCACHE	0x80
 #endif
 
+#ifndef RWF_ATOMIC
+#define RWF_ATOMIC	0x40
+#endif
+
 #define NUMPRINTCOLUMNS 32	/* # columns of data to print on each line */
 
 /* Operation flags (bitmask) */

@@ -110,6 +115,7 @@ enum {
 	OP_READ_DONTCACHE,
 	OP_WRITE,
 	OP_WRITE_DONTCACHE,
+	OP_WRITE_ATOMIC,
 	OP_MAPREAD,
 	OP_MAPWRITE,
 	OP_MAX_LITE,

@@ -200,6 +206,11 @@ int	uring = 0;
 int	mark_nr = 0;
 int	dontcache_io = 1;
 int	hugepages = 0;                  /* -h flag */
+int	do_atomic_writes = 1;		/* -a flag disables */
+
+/* User for atomic writes */
+int awu_min = 0;
+int awu_max = 0;
 
 /* Stores info needed to periodically collapse hugepages */
 struct hugepages_collapse_info {

@@ -288,6 +299,7 @@ static const char *op_names[] = {
 	[OP_READ_DONTCACHE] = "read_dontcache",
 	[OP_WRITE] = "write",
 	[OP_WRITE_DONTCACHE] = "write_dontcache",
+	[OP_WRITE_ATOMIC] = "write_atomic",
 	[OP_MAPREAD] = "mapread",
 	[OP_MAPWRITE] = "mapwrite",
 	[OP_TRUNCATE] = "truncate",

@@ -422,6 +434,7 @@ logdump(void)
 				prt("\t***RRRR***");
 			break;
 		case OP_WRITE_DONTCACHE:
+		case OP_WRITE_ATOMIC:
 		case OP_WRITE:
 			prt("WRITE    0x%x thru 0x%x\t(0x%x bytes)",
 			    lp->args[0], lp->args[0] + lp->args[1] - 1,

@@ -1073,6 +1086,25 @@ update_file_size(unsigned offset, unsigned size)
 	file_size = offset + size;
 }
 
+static int is_power_of_2(unsigned n) {
+	return ((n & (n - 1)) == 0);
+}
+
+/*
+ * Round down n to nearest power of 2.
+ * If n is already a power of 2, return n;
+ */
+static int rounddown_pow_of_2(int n) {
+	int i = 0;
+
+	if (is_power_of_2(n))
+		return n;
+
+	for (; (1 << i) < n; i++);
+
+	return 1 << (i - 1);
+}
+
 void
 dowrite(unsigned offset, unsigned size, int flags)
 {

@@ -1081,6 +1113,27 @@ dowrite(unsigned offset, unsigned size, int flags)
 	offset -= offset % writebdy;
 	if (o_direct)
 		size -= size % writebdy;
+	if (flags & RWF_ATOMIC) {
+		/* atomic write len must be between awu_min and awu_max */
+		if (size < awu_min)
+			size = awu_min;
+		if (size > awu_max)
+			size = awu_max;
+
+		/* atomic writes need power-of-2 sizes */
+		size = rounddown_pow_of_2(size);
+
+		/* atomic writes need naturally aligned offsets */
+		offset -= offset % size;
+
+		/* Skip the write if we are crossing max filesize */
+		if ((offset + size) > maxfilelen) {
+			if (!quiet && testcalls > simulatedopcount)
+				prt("skipping atomic write past maxfilelen\n");
+			log4(OP_WRITE_ATOMIC, offset, size, FL_SKIPPED);
+			return;
+		}
+	}
 	if (size == 0) {
 		if (!quiet && testcalls > simulatedopcount && !o_direct)
 			prt("skipping zero size write\n");

@@ -1088,7 +1141,10 @@ dowrite(unsigned offset, unsigned size, int flags)
 		return;
 	}
 
-	log4(OP_WRITE, offset, size, FL_NONE);
+	if (flags & RWF_ATOMIC)
+		log4(OP_WRITE_ATOMIC, offset, size, FL_NONE);
+	else
+		log4(OP_WRITE, offset, size, FL_NONE);
 
 	gendata(original_buf, good_buf, offset, size);
 	if (offset + size > file_size) {

@@ -1108,8 +1164,9 @@ dowrite(unsigned offset, unsigned size, int flags)
 		       (monitorstart == -1 ||
 			(offset + size > monitorstart &&
 			(monitorend == -1 || offset <= monitorend))))))
-		prt("%lld write\t0x%x thru\t0x%x\t(0x%x bytes)\tdontcache=%d\n", testcalls,
-		    offset, offset + size - 1, size, (flags & RWF_DONTCACHE) != 0);
+		prt("%lld write\t0x%x thru\t0x%x\t(0x%x bytes)\tdontcache=%d atomic_wr=%d\n", testcalls,
+		    offset, offset + size - 1, size, (flags & RWF_DONTCACHE) != 0,
+		    (flags & RWF_ATOMIC) != 0);
 	iret = fsxwrite(fd, good_buf + offset, size, offset, flags);
 	if (iret != size) {
 		if (iret == -1)

@@ -1785,6 +1842,36 @@ do_dedupe_range(unsigned offset, unsigned length, unsigned dest)
 }
 #endif
 
+int test_atomic_writes(void) {
+	int ret;
+	struct statx stx;
+
+	if (o_direct != O_DIRECT) {
+		fprintf(stderr, "main: atomic writes need O_DIRECT (-Z), "
+				"disabling!\n");
+		return 0;
+	}
+
+	ret = xfstests_statx(AT_FDCWD, fname, 0, STATX_WRITE_ATOMIC, &stx);
+	if (ret < 0) {
+		fprintf(stderr, "main: Statx failed with %d."
+			" Failed to determine atomic write limits, "
+			" disabling!\n", ret);
+		return 0;
+	}
+
+	if (stx.stx_attributes & STATX_ATTR_WRITE_ATOMIC &&
+	    stx.stx_atomic_write_unit_min > 0) {
+		awu_min = stx.stx_atomic_write_unit_min;
+		awu_max = stx.stx_atomic_write_unit_max;
+		return 1;
+	}
+
+	fprintf(stderr, "main: IO Stack does not support "
+			"atomic writes, disabling!\n");
+	return 0;
+}
+
 #ifdef HAVE_COPY_FILE_RANGE
 int
 test_copy_range(void)

@@ -2356,6 +2443,12 @@ have_op:
 			goto out;
 		}
 		break;
+	case OP_WRITE_ATOMIC:
+		if (!do_atomic_writes) {
+			log4(OP_WRITE_ATOMIC, offset, size, FL_SKIPPED);
+			goto out;
+		}
+		break;
 	}
 
 	switch (op) {

@@ -2385,6 +2478,11 @@ have_op:
 			dowrite(offset, size, 0);
 		break;
 
+	case OP_WRITE_ATOMIC:
+		TRIM_OFF_LEN(offset, size, maxfilelen);
+		dowrite(offset, size, RWF_ATOMIC);
+		break;
+
 	case OP_MAPREAD:
 		TRIM_OFF_LEN(offset, size, file_size);
 		domapread(offset, size);

@@ -2511,13 +2609,14 @@ void
 usage(void)
 {
 	fprintf(stdout, "usage: %s",
-		"fsx [-dfhknqxyzBEFHIJKLORWXZ0]\n\
+		"fsx [-adfhknqxyzBEFHIJKLORWXZ0]\n\
 	   [-b opnum] [-c Prob] [-g filldata] [-i logdev] [-j logid]\n\
 	   [-l flen] [-m start:end] [-o oplen] [-p progressinterval]\n\
 	   [-r readbdy] [-s style] [-t truncbdy] [-w writebdy]\n\
 	   [-A|-U] [-D startingop] [-N numops] [-P dirpath] [-S seed]\n\
 	   [--replay-ops=opsfile] [--record-ops[=opsfile]] [--duration=seconds]\n\
 	   ... fname\n\
+	-a: disable atomic writes\n\
 	-b opnum: beginning operation number (default 1)\n\
 	-c P: 1 in P chance of file close+open at each op (default infinity)\n\
 	-d: debug output for all operations\n\

@@ -3059,9 +3158,13 @@ main(int argc, char **argv)
 	setvbuf(stdout, (char *)0, _IOLBF, 0); /* line buffered stdout */
 
 	while ((ch = getopt_long(argc, argv,
-				 "0b:c:de:fg:hi:j:kl:m:no:p:qr:s:t:uw:xyABD:EFJKHzCILN:OP:RS:UWXZ",
+				 "0ab:c:de:fg:hi:j:kl:m:no:p:qr:s:t:uw:xyABD:EFJKHzCILN:OP:RS:UWXZ",
 				 longopts, NULL)) != EOF)
 		switch (ch) {
+		case 'a':
+			prt("main(): Atomic writes disabled\n");
+			do_atomic_writes = 0;
+			break;
 		case 'b':
 			simulatedopcount = getnum(optarg, &endp);
 			if (!quiet)

@@ -3475,6 +3578,8 @@ main(int argc, char **argv)
 		exchange_range_calls = test_exchange_range();
 	if (dontcache_io)
 		dontcache_io = test_dontcache_io();
+	if (do_atomic_writes)
+		do_atomic_writes = test_atomic_writes();
 
 	while (keep_running())
 		if (!test())

Signature · 1 line

2.49.0

Quoted 10 lines · Zorro Lang

Quoted 8 lines · Ojaswin Mujoo

Implement atomic write support to help fuzz atomic writes with fsx.

Suggested-by Ritesh Harjani (IBM) <[email protected]>

Reviewed-by Darrick J. Wong <[email protected]>

Reviewed-by John Garry <[email protected]>

Signed-off-by Ojaswin Mujoo <[email protected]>

---

Quoted 4 lines

Hmm... this patch causes more regular fsx test cases fail on old kernel, (e.g. g/760, g/617, g/263 ...) except set "FSX_AVOID=-a". Is there a way to disable "atomic write" automatically if it's not supported by current system?

Hi Zorro,
Sorry for being late, I've been on vacation this week.

Yes so by design we should be automatically disabling atomic writes when they are not supported by the stack but seems like the issue is that when we do disable it we print some extra messages to stdout/err which show up in the xfstests output causing failure.

I can think of 2 ways around this:

1. Don't print anything and just silently drop atomic writes if stack doesn't support them.

2. Make atomic writes as a default off instead of default on feature but his loses a bit of coverage as existing tests wont get atomic write testing free of cost any more.

Regards,
ojaswin

Quoted 249 lines

Thanks,
Zorro

Quoted 245 lines

 ltp/fsx.c | 115 +++++++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 110 insertions(+), 5 deletions(-)

ltp/fsx.c +110 -5

index 163b9453..bdb87ca9 100644
@@ -40,6 +40,7 @@
 #include <liburing.h>
 #endif
 #include <sys/syscall.h>
+#include "statx.h"
 
 #ifndef MAP_FILE
 # define MAP_FILE 0

@@ -49,6 +50,10 @@
 #define RWF_DONTCACHE	0x80
 #endif
 
+#ifndef RWF_ATOMIC
+#define RWF_ATOMIC	0x40
+#endif
+
 #define NUMPRINTCOLUMNS 32	/* # columns of data to print on each line */
 
 /* Operation flags (bitmask) */

@@ -110,6 +115,7 @@ enum {
 	OP_READ_DONTCACHE,
 	OP_WRITE,
 	OP_WRITE_DONTCACHE,
+	OP_WRITE_ATOMIC,
 	OP_MAPREAD,
 	OP_MAPWRITE,
 	OP_MAX_LITE,

@@ -200,6 +206,11 @@ int	uring = 0;
 int	mark_nr = 0;
 int	dontcache_io = 1;
 int	hugepages = 0;                  /* -h flag */
+int	do_atomic_writes = 1;		/* -a flag disables */
+
+/* User for atomic writes */
+int awu_min = 0;
+int awu_max = 0;
 
 /* Stores info needed to periodically collapse hugepages */
 struct hugepages_collapse_info {

@@ -288,6 +299,7 @@ static const char *op_names[] = {
 	[OP_READ_DONTCACHE] = "read_dontcache",
 	[OP_WRITE] = "write",
 	[OP_WRITE_DONTCACHE] = "write_dontcache",
+	[OP_WRITE_ATOMIC] = "write_atomic",
 	[OP_MAPREAD] = "mapread",
 	[OP_MAPWRITE] = "mapwrite",
 	[OP_TRUNCATE] = "truncate",

@@ -422,6 +434,7 @@ logdump(void)
 				prt("\t***RRRR***");
 			break;
 		case OP_WRITE_DONTCACHE:
+		case OP_WRITE_ATOMIC:
 		case OP_WRITE:
 			prt("WRITE    0x%x thru 0x%x\t(0x%x bytes)",
 			    lp->args[0], lp->args[0] + lp->args[1] - 1,

@@ -1073,6 +1086,25 @@ update_file_size(unsigned offset, unsigned size)
 	file_size = offset + size;
 }
 
+static int is_power_of_2(unsigned n) {
+	return ((n & (n - 1)) == 0);
+}
+
+/*
+ * Round down n to nearest power of 2.
+ * If n is already a power of 2, return n;
+ */
+static int rounddown_pow_of_2(int n) {
+	int i = 0;
+
+	if (is_power_of_2(n))
+		return n;
+
+	for (; (1 << i) < n; i++);
+
+	return 1 << (i - 1);
+}
+
 void
 dowrite(unsigned offset, unsigned size, int flags)
 {

@@ -1081,6 +1113,27 @@ dowrite(unsigned offset, unsigned size, int flags)
 	offset -= offset % writebdy;
 	if (o_direct)
 		size -= size % writebdy;
+	if (flags & RWF_ATOMIC) {
+		/* atomic write len must be between awu_min and awu_max */
+		if (size < awu_min)
+			size = awu_min;
+		if (size > awu_max)
+			size = awu_max;
+
+		/* atomic writes need power-of-2 sizes */
+		size = rounddown_pow_of_2(size);
+
+		/* atomic writes need naturally aligned offsets */
+		offset -= offset % size;
+
+		/* Skip the write if we are crossing max filesize */
+		if ((offset + size) > maxfilelen) {
+			if (!quiet && testcalls > simulatedopcount)
+				prt("skipping atomic write past maxfilelen\n");
+			log4(OP_WRITE_ATOMIC, offset, size, FL_SKIPPED);
+			return;
+		}
+	}
 	if (size == 0) {
 		if (!quiet && testcalls > simulatedopcount && !o_direct)
 			prt("skipping zero size write\n");

@@ -1088,7 +1141,10 @@ dowrite(unsigned offset, unsigned size, int flags)
 		return;
 	}
 
-	log4(OP_WRITE, offset, size, FL_NONE);
+	if (flags & RWF_ATOMIC)
+		log4(OP_WRITE_ATOMIC, offset, size, FL_NONE);
+	else
+		log4(OP_WRITE, offset, size, FL_NONE);
 
 	gendata(original_buf, good_buf, offset, size);
 	if (offset + size > file_size) {

@@ -1108,8 +1164,9 @@ dowrite(unsigned offset, unsigned size, int flags)
 		       (monitorstart == -1 ||
 			(offset + size > monitorstart &&
 			(monitorend == -1 || offset <= monitorend))))))
-		prt("%lld write\t0x%x thru\t0x%x\t(0x%x bytes)\tdontcache=%d\n", testcalls,
-		    offset, offset + size - 1, size, (flags & RWF_DONTCACHE) != 0);
+		prt("%lld write\t0x%x thru\t0x%x\t(0x%x bytes)\tdontcache=%d atomic_wr=%d\n", testcalls,
+		    offset, offset + size - 1, size, (flags & RWF_DONTCACHE) != 0,
+		    (flags & RWF_ATOMIC) != 0);
 	iret = fsxwrite(fd, good_buf + offset, size, offset, flags);
 	if (iret != size) {
 		if (iret == -1)

@@ -1785,6 +1842,36 @@ do_dedupe_range(unsigned offset, unsigned length, unsigned dest)
 }
 #endif
 
+int test_atomic_writes(void) {
+	int ret;
+	struct statx stx;
+
+	if (o_direct != O_DIRECT) {
+		fprintf(stderr, "main: atomic writes need O_DIRECT (-Z), "
+				"disabling!\n");
+		return 0;
+	}
+
+	ret = xfstests_statx(AT_FDCWD, fname, 0, STATX_WRITE_ATOMIC, &stx);
+	if (ret < 0) {
+		fprintf(stderr, "main: Statx failed with %d."
+			" Failed to determine atomic write limits, "
+			" disabling!\n", ret);
+		return 0;
+	}
+
+	if (stx.stx_attributes & STATX_ATTR_WRITE_ATOMIC &&
+	    stx.stx_atomic_write_unit_min > 0) {
+		awu_min = stx.stx_atomic_write_unit_min;
+		awu_max = stx.stx_atomic_write_unit_max;
+		return 1;
+	}
+
+	fprintf(stderr, "main: IO Stack does not support "
+			"atomic writes, disabling!\n");
+	return 0;
+}
+
 #ifdef HAVE_COPY_FILE_RANGE
 int
 test_copy_range(void)

@@ -2356,6 +2443,12 @@ have_op:
 			goto out;
 		}
 		break;
+	case OP_WRITE_ATOMIC:
+		if (!do_atomic_writes) {
+			log4(OP_WRITE_ATOMIC, offset, size, FL_SKIPPED);
+			goto out;
+		}
+		break;
 	}
 
 	switch (op) {

@@ -2385,6 +2478,11 @@ have_op:
 			dowrite(offset, size, 0);
 		break;
 
+	case OP_WRITE_ATOMIC:
+		TRIM_OFF_LEN(offset, size, maxfilelen);
+		dowrite(offset, size, RWF_ATOMIC);
+		break;
+
 	case OP_MAPREAD:
 		TRIM_OFF_LEN(offset, size, file_size);
 		domapread(offset, size);

@@ -2511,13 +2609,14 @@ void
 usage(void)
 {
 	fprintf(stdout, "usage: %s",
-		"fsx [-dfhknqxyzBEFHIJKLORWXZ0]\n\
+		"fsx [-adfhknqxyzBEFHIJKLORWXZ0]\n\
 	   [-b opnum] [-c Prob] [-g filldata] [-i logdev] [-j logid]\n\
 	   [-l flen] [-m start:end] [-o oplen] [-p progressinterval]\n\
 	   [-r readbdy] [-s style] [-t truncbdy] [-w writebdy]\n\
 	   [-A|-U] [-D startingop] [-N numops] [-P dirpath] [-S seed]\n\
 	   [--replay-ops=opsfile] [--record-ops[=opsfile]] [--duration=seconds]\n\
 	   ... fname\n\
+	-a: disable atomic writes\n\
 	-b opnum: beginning operation number (default 1)\n\
 	-c P: 1 in P chance of file close+open at each op (default infinity)\n\
 	-d: debug output for all operations\n\

@@ -3059,9 +3158,13 @@ main(int argc, char **argv)
 	setvbuf(stdout, (char *)0, _IOLBF, 0); /* line buffered stdout */
 
 	while ((ch = getopt_long(argc, argv,
-				 "0b:c:de:fg:hi:j:kl:m:no:p:qr:s:t:uw:xyABD:EFJKHzCILN:OP:RS:UWXZ",
+				 "0ab:c:de:fg:hi:j:kl:m:no:p:qr:s:t:uw:xyABD:EFJKHzCILN:OP:RS:UWXZ",
 				 longopts, NULL)) != EOF)
 		switch (ch) {
+		case 'a':
+			prt("main(): Atomic writes disabled\n");
+			do_atomic_writes = 0;
+			break;
 		case 'b':
 			simulatedopcount = getnum(optarg, &endp);
 			if (!quiet)

@@ -3475,6 +3578,8 @@ main(int argc, char **argv)
 		exchange_range_calls = test_exchange_range();
 	if (dontcache_io)
 		dontcache_io = test_dontcache_io();
+	if (do_atomic_writes)
+		do_atomic_writes = test_atomic_writes();
 
 	while (keep_running())
 		if (!test())

Signature · 1 line

2.49.0

Quoted 24 lines · Ojaswin Mujoo

Quoted 10 lines · Zorro Lang

Quoted 8 lines · Ojaswin Mujoo

Implement atomic write support to help fuzz atomic writes with fsx.

Suggested-by Ritesh Harjani (IBM) <[email protected]>

Reviewed-by Darrick J. Wong <[email protected]>

Reviewed-by John Garry <[email protected]>

Signed-off-by Ojaswin Mujoo <[email protected]>

---

Quoted 4 lines

Hmm... this patch causes more regular fsx test cases fail on old kernel, (e.g. g/760, g/617, g/263 ...) except set "FSX_AVOID=-a". Is there a way to disable "atomic write" automatically if it's not supported by current system?

Hi Zorro,
Sorry for being late, I've been on vacation this week.

Yes so by design we should be automatically disabling atomic writes when they are not supported by the stack but seems like the issue is that when we do disable it we print some extra messages to stdout/err which show up in the xfstests output causing failure.

Quoted 8 lines

I can think of 2 ways around this:

1. Don't print anything and just silently drop atomic writes if stack doesn't support them.

2. Make atomic writes as a default off instead of default on feature but his loses a bit of coverage as existing tests wont get atomic write testing free of cost any more.

Hi Ojaswin,

Please have a nice vacation :)

It's not the "extra messages" cause failure, those "quiet" failures can be fixed by:

ltp/fsx.c +6 -4

index bdb87ca90..0a035b37b 100644
@@ -1847,8 +1847,9 @@ int test_atomic_writes(void) {
        struct statx stx;
 
        if (o_direct != O_DIRECT) {
-               fprintf(stderr, "main: atomic writes need O_DIRECT (-Z), "
-                               "disabling!\n");
+               if (!quiet)
+                       fprintf(stderr, "main: atomic writes need O_DIRECT (-Z), "
+                                       "disabling!\n");
                return 0;
        }
 

@@ -1867,8 +1868,9 @@ int test_atomic_writes(void) {
                return 1;
        }
 
-       fprintf(stderr, "main: IO Stack does not support "
-                       "atomic writes, disabling!\n");
+       if (!quiet)
+               fprintf(stderr, "main: IO Stack does not support "
+                               "atomic writes, disabling!\n");
        return 0;
 }
 

But I hit more read or write failures e.g. [1], this failure can't be reproduced with FSX_AVOID=-a. Is it a atomic write bug or an unexpected test failure?

Thanks,
Zorro

+244(244 mod 256): SKIPPED (no operation)
+245(245 mod 256): FALLOC 0x695c5 thru 0x6a2e6 (0xd21 bytes) INTERIOR
+246(246 mod 256): MAPWRITE 0x5ac00 thru 0x5b185 (0x586 bytes)
+247(247 mod 256): WRITE 0x31200 thru 0x313ff (0x200 bytes)
+248(248 mod 256): SKIPPED (no operation)
+249(249 mod 256): TRUNCATE DOWN from 0x78242 to 0xf200 ******WWWW
+250(250 mod 256): FALLOC 0x65000 thru 0x66f26 (0x1f26 bytes) PAST_EOF
+251(251 mod 256): WRITE 0x45400 thru 0x467ff (0x1400 bytes) HOLE ***WWWW
+252(252 mod 256): SKIPPED (no operation)
+253(253 mod 256): SKIPPED (no operation)
+254(254 mod 256): MAPWRITE 0x4be00 thru 0x4daee (0x1cef bytes)
+255(255 mod 256): MAPREAD 0xc000 thru 0xcae9 (0xaea bytes)

+256(  0 mod 256): READ     0x3e000 thru 0x3efff	(0x1000 bytes)
+257(  1 mod 256): SKIPPED (no operation)
+258(  2 mod 256): INSERT 0x45000 thru 0x45fff	(0x1000 bytes)
+259(  3 mod 256): ZERO     0x1d7d5 thru 0x1f399	(0x1bc5 bytes)	******ZZZZ
+260(  4 mod 256): TRUNCATE DOWN	from 0x4eaef to 0x11200	******WWWW
+261(  5 mod 256): WRITE    0x43000 thru 0x43fff	(0x1000 bytes) HOLE	***WWWW
+262(  6 mod 256): WRITE    0x2200 thru 0x31ff	(0x1000 bytes)
+263(  7 mod 256): WRITE    0x15000 thru 0x15fff	(0x1000 bytes)
+264(  8 mod 256): WRITE    0x2e400 thru 0x2e7ff	(0x400 bytes)
+265(  9 mod 256): COPY 0xd000 thru 0xdfff	(0x1000 bytes) to 0x1d800 thru 0x1e7ff	******EEEE

+266( 10 mod 256): CLONE 0x2a000 thru 0x2afff (0x1000 bytes) to 0x21000 thru 0x21fff
+267( 11 mod 256): MAPREAD 0x31000 thru 0x31d0a (0xd0b bytes)
+268( 12 mod 256): SKIPPED (no operation)
+269( 13 mod 256): WRITE 0x25000 thru 0x25fff (0x1000 bytes)
+270( 14 mod 256): SKIPPED (no operation)

+271( 15 mod 256): MAPREAD  0x30000 thru 0x30577	(0x578 bytes)
+272( 16 mod 256): PUNCH    0x1a267 thru 0x1c093	(0x1e2d bytes)
+273( 17 mod 256): MAPREAD  0x1f000 thru 0x1f9c9	(0x9ca bytes)
+274( 18 mod 256): WRITE    0x40800 thru 0x40dff	(0x600 bytes)

+275( 19 mod 256): SKIPPED (no operation)
+276( 20 mod 256): MAPWRITE 0x20600 thru 0x22115 (0x1b16 bytes)
+277( 21 mod 256): MAPWRITE 0x3d000 thru 0x3ee5a (0x1e5b bytes)
+278( 22 mod 256): WRITE 0x2ee00 thru 0x2efff (0x200 bytes)
+279( 23 mod 256): WRITE 0x76200 thru 0x769ff (0x800 bytes) HOLE
+280( 24 mod 256): SKIPPED (no operation)
+281( 25 mod 256): SKIPPED (no operation)
+282( 26 mod 256): MAPREAD 0xa000 thru 0xa5e7 (0x5e8 bytes)
+283( 27 mod 256): SKIPPED (no operation)
+284( 28 mod 256): SKIPPED (no operation)
+285( 29 mod 256): SKIPPED (no operation)
+286( 30 mod 256): SKIPPED (no operation)
+287( 31 mod 256): COLLAPSE 0x11000 thru 0x11fff (0x1000 bytes)
+288( 32 mod 256): COPY 0x5d000 thru 0x5dfff (0x1000 bytes) to 0x4ca00 thru 0x4d9ff
+289( 33 mod 256): TRUNCATE DOWN from 0x75a00 to 0x1e400
+290( 34 mod 256): MAPREAD 0x1c000 thru 0x1d802 (0x1803 bytes) ***RRRR***

ltp/fsx.c +3 -0

+Log of operations saved to "/mnt/xfstests/test/junk.fsxops"; replay with --replay-ops
+Correct content saved for comparison
+(maybe hexdump "/mnt/xfstests/test/junk" vs "/mnt/xfstests/test/junk.fsxgood")

Thanks,
Zorro

Quoted 34 lines · Zorro Lang

Quoted 24 lines · Ojaswin Mujoo

Quoted 10 lines · Zorro Lang

Quoted 8 lines · Ojaswin Mujoo

Implement atomic write support to help fuzz atomic writes with fsx.

Suggested-by Ritesh Harjani (IBM) <[email protected]>

Reviewed-by Darrick J. Wong <[email protected]>

Reviewed-by John Garry <[email protected]>

Signed-off-by Ojaswin Mujoo <[email protected]>

---

Quoted 4 lines

Hmm... this patch causes more regular fsx test cases fail on old kernel, (e.g. g/760, g/617, g/263 ...) except set "FSX_AVOID=-a". Is there a way to disable "atomic write" automatically if it's not supported by current system?

Hi Zorro,
Sorry for being late, I've been on vacation this week.

Yes so by design we should be automatically disabling atomic writes when they are not supported by the stack but seems like the issue is that when we do disable it we print some extra messages to stdout/err which show up in the xfstests output causing failure.

Quoted 8 lines

I can think of 2 ways around this:

1. Don't print anything and just silently drop atomic writes if stack doesn't support them.

2. Make atomic writes as a default off instead of default on feature but his loses a bit of coverage as existing tests wont get atomic write testing free of cost any more.

Quoted 6 lines

Hi Ojaswin,

Please have a nice vacation :)

It's not the "extra messages" cause failure, those "quiet" failures can be fixed by:

Oh okay got it.

Quoted 37 lines

ltp/fsx.c +6 -4

index bdb87ca90..0a035b37b 100644
@@ -1847,8 +1847,9 @@ int test_atomic_writes(void) {
        struct statx stx;
 
        if (o_direct != O_DIRECT) {
-               fprintf(stderr, "main: atomic writes need O_DIRECT (-Z), "
-                               "disabling!\n");
+               if (!quiet)
+                       fprintf(stderr, "main: atomic writes need O_DIRECT (-Z), "
+                                       "disabling!\n");
                return 0;
        }
 

@@ -1867,8 +1868,9 @@ int test_atomic_writes(void) {
                return 1;
        }
 
-       fprintf(stderr, "main: IO Stack does not support "
-                       "atomic writes, disabling!\n");
+       if (!quiet)
+               fprintf(stderr, "main: IO Stack does not support "
+                               "atomic writes, disabling!\n");
        return 0;
 }
 

But I hit more read or write failures e.g. [1], this failure can't be reproduced with FSX_AVOID=-a. Is it a atomic write bug or an unexpected test failure?

Thanks,
Zorro

<...>

Quoted 51 lines

+244(244 mod 256): SKIPPED (no operation)
+245(245 mod 256): FALLOC 0x695c5 thru 0x6a2e6 (0xd21 bytes) INTERIOR
+246(246 mod 256): MAPWRITE 0x5ac00 thru 0x5b185 (0x586 bytes)
+247(247 mod 256): WRITE 0x31200 thru 0x313ff (0x200 bytes)
+248(248 mod 256): SKIPPED (no operation)
+249(249 mod 256): TRUNCATE DOWN from 0x78242 to 0xf200 ******WWWW
+250(250 mod 256): FALLOC 0x65000 thru 0x66f26 (0x1f26 bytes) PAST_EOF
+251(251 mod 256): WRITE 0x45400 thru 0x467ff (0x1400 bytes) HOLE ***WWWW
+252(252 mod 256): SKIPPED (no operation)
+253(253 mod 256): SKIPPED (no operation)
+254(254 mod 256): MAPWRITE 0x4be00 thru 0x4daee (0x1cef bytes)
+255(255 mod 256): MAPREAD 0xc000 thru 0xcae9 (0xaea bytes)

+256(  0 mod 256): READ     0x3e000 thru 0x3efff	(0x1000 bytes)
+257(  1 mod 256): SKIPPED (no operation)
+258(  2 mod 256): INSERT 0x45000 thru 0x45fff	(0x1000 bytes)
+259(  3 mod 256): ZERO     0x1d7d5 thru 0x1f399	(0x1bc5 bytes)	******ZZZZ
+260(  4 mod 256): TRUNCATE DOWN	from 0x4eaef to 0x11200	******WWWW
+261(  5 mod 256): WRITE    0x43000 thru 0x43fff	(0x1000 bytes) HOLE	***WWWW
+262(  6 mod 256): WRITE    0x2200 thru 0x31ff	(0x1000 bytes)
+263(  7 mod 256): WRITE    0x15000 thru 0x15fff	(0x1000 bytes)
+264(  8 mod 256): WRITE    0x2e400 thru 0x2e7ff	(0x400 bytes)
+265(  9 mod 256): COPY 0xd000 thru 0xdfff	(0x1000 bytes) to 0x1d800 thru 0x1e7ff	******EEEE

+266( 10 mod 256): CLONE 0x2a000 thru 0x2afff (0x1000 bytes) to 0x21000 thru 0x21fff
+267( 11 mod 256): MAPREAD 0x31000 thru 0x31d0a (0xd0b bytes)
+268( 12 mod 256): SKIPPED (no operation)
+269( 13 mod 256): WRITE 0x25000 thru 0x25fff (0x1000 bytes)
+270( 14 mod 256): SKIPPED (no operation)

+271( 15 mod 256): MAPREAD  0x30000 thru 0x30577	(0x578 bytes)
+272( 16 mod 256): PUNCH    0x1a267 thru 0x1c093	(0x1e2d bytes)
+273( 17 mod 256): MAPREAD  0x1f000 thru 0x1f9c9	(0x9ca bytes)
+274( 18 mod 256): WRITE    0x40800 thru 0x40dff	(0x600 bytes)

+275( 19 mod 256): SKIPPED (no operation)
+276( 20 mod 256): MAPWRITE 0x20600 thru 0x22115 (0x1b16 bytes)
+277( 21 mod 256): MAPWRITE 0x3d000 thru 0x3ee5a (0x1e5b bytes)
+278( 22 mod 256): WRITE 0x2ee00 thru 0x2efff (0x200 bytes)
+279( 23 mod 256): WRITE 0x76200 thru 0x769ff (0x800 bytes) HOLE
+280( 24 mod 256): SKIPPED (no operation)
+281( 25 mod 256): SKIPPED (no operation)
+282( 26 mod 256): MAPREAD 0xa000 thru 0xa5e7 (0x5e8 bytes)
+283( 27 mod 256): SKIPPED (no operation)
+284( 28 mod 256): SKIPPED (no operation)
+285( 29 mod 256): SKIPPED (no operation)
+286( 30 mod 256): SKIPPED (no operation)
+287( 31 mod 256): COLLAPSE 0x11000 thru 0x11fff (0x1000 bytes)
+288( 32 mod 256): COPY 0x5d000 thru 0x5dfff (0x1000 bytes) to 0x4ca00 thru 0x4d9ff
+289( 33 mod 256): TRUNCATE DOWN from 0x75a00 to 0x1e400
+290( 34 mod 256): MAPREAD 0x1c000 thru 0x1d802 (0x1803 bytes) ***RRRR***

ltp/fsx.c +3 -0

+Log of operations saved to "/mnt/xfstests/test/junk.fsxops"; replay with --replay-ops
+Correct content saved for comparison
+(maybe hexdump "/mnt/xfstests/test/junk" vs "/mnt/xfstests/test/junk.fsxgood")

Quoted 2 lines

Thanks,
Zorro

Hi Zorro, just to confirm is this on an older kernel that doesnt support RWF_ATOMIC or on a kernle that does support it.

Regards,
ojaswin

Quoted 135 lines · Ojaswin Mujoo

Quoted 34 lines · Zorro Lang

Quoted 24 lines · Ojaswin Mujoo

Quoted 10 lines · Zorro Lang

Quoted 8 lines · Ojaswin Mujoo

Implement atomic write support to help fuzz atomic writes with fsx.

Suggested-by Ritesh Harjani (IBM) <[email protected]>

Reviewed-by Darrick J. Wong <[email protected]>

Reviewed-by John Garry <[email protected]>

Signed-off-by Ojaswin Mujoo <[email protected]>

---

Quoted 4 lines

Hmm... this patch causes more regular fsx test cases fail on old kernel, (e.g. g/760, g/617, g/263 ...) except set "FSX_AVOID=-a". Is there a way to disable "atomic write" automatically if it's not supported by current system?

Hi Zorro,
Sorry for being late, I've been on vacation this week.

Yes so by design we should be automatically disabling atomic writes when they are not supported by the stack but seems like the issue is that when we do disable it we print some extra messages to stdout/err which show up in the xfstests output causing failure.

Quoted 8 lines

I can think of 2 ways around this:

1. Don't print anything and just silently drop atomic writes if stack doesn't support them.

2. Make atomic writes as a default off instead of default on feature but his loses a bit of coverage as existing tests wont get atomic write testing free of cost any more.

Quoted 6 lines

Hi Ojaswin,

Please have a nice vacation :)

It's not the "extra messages" cause failure, those "quiet" failures can be fixed by:

Oh okay got it.

Quoted 37 lines

ltp/fsx.c +6 -4

index bdb87ca90..0a035b37b 100644
@@ -1847,8 +1847,9 @@ int test_atomic_writes(void) {
        struct statx stx;
 
        if (o_direct != O_DIRECT) {
-               fprintf(stderr, "main: atomic writes need O_DIRECT (-Z), "
-                               "disabling!\n");
+               if (!quiet)
+                       fprintf(stderr, "main: atomic writes need O_DIRECT (-Z), "
+                                       "disabling!\n");
                return 0;
        }
 

@@ -1867,8 +1868,9 @@ int test_atomic_writes(void) {
                return 1;
        }
 
-       fprintf(stderr, "main: IO Stack does not support "
-                       "atomic writes, disabling!\n");
+       if (!quiet)
+               fprintf(stderr, "main: IO Stack does not support "
+                               "atomic writes, disabling!\n");
        return 0;
 }
 

But I hit more read or write failures e.g. [1], this failure can't be reproduced with FSX_AVOID=-a. Is it a atomic write bug or an unexpected test failure?

Thanks,
Zorro

<...>

Quoted 51 lines

+244(244 mod 256): SKIPPED (no operation)
+245(245 mod 256): FALLOC 0x695c5 thru 0x6a2e6 (0xd21 bytes) INTERIOR
+246(246 mod 256): MAPWRITE 0x5ac00 thru 0x5b185 (0x586 bytes)
+247(247 mod 256): WRITE 0x31200 thru 0x313ff (0x200 bytes)
+248(248 mod 256): SKIPPED (no operation)
+249(249 mod 256): TRUNCATE DOWN from 0x78242 to 0xf200 ******WWWW
+250(250 mod 256): FALLOC 0x65000 thru 0x66f26 (0x1f26 bytes) PAST_EOF
+251(251 mod 256): WRITE 0x45400 thru 0x467ff (0x1400 bytes) HOLE ***WWWW
+252(252 mod 256): SKIPPED (no operation)
+253(253 mod 256): SKIPPED (no operation)
+254(254 mod 256): MAPWRITE 0x4be00 thru 0x4daee (0x1cef bytes)
+255(255 mod 256): MAPREAD 0xc000 thru 0xcae9 (0xaea bytes)

+256(  0 mod 256): READ     0x3e000 thru 0x3efff	(0x1000 bytes)
+257(  1 mod 256): SKIPPED (no operation)
+258(  2 mod 256): INSERT 0x45000 thru 0x45fff	(0x1000 bytes)
+259(  3 mod 256): ZERO     0x1d7d5 thru 0x1f399	(0x1bc5 bytes)	******ZZZZ
+260(  4 mod 256): TRUNCATE DOWN	from 0x4eaef to 0x11200	******WWWW
+261(  5 mod 256): WRITE    0x43000 thru 0x43fff	(0x1000 bytes) HOLE	***WWWW
+262(  6 mod 256): WRITE    0x2200 thru 0x31ff	(0x1000 bytes)
+263(  7 mod 256): WRITE    0x15000 thru 0x15fff	(0x1000 bytes)
+264(  8 mod 256): WRITE    0x2e400 thru 0x2e7ff	(0x400 bytes)
+265(  9 mod 256): COPY 0xd000 thru 0xdfff	(0x1000 bytes) to 0x1d800 thru 0x1e7ff	******EEEE

+266( 10 mod 256): CLONE 0x2a000 thru 0x2afff (0x1000 bytes) to 0x21000 thru 0x21fff
+267( 11 mod 256): MAPREAD 0x31000 thru 0x31d0a (0xd0b bytes)
+268( 12 mod 256): SKIPPED (no operation)
+269( 13 mod 256): WRITE 0x25000 thru 0x25fff (0x1000 bytes)
+270( 14 mod 256): SKIPPED (no operation)

+271( 15 mod 256): MAPREAD  0x30000 thru 0x30577	(0x578 bytes)
+272( 16 mod 256): PUNCH    0x1a267 thru 0x1c093	(0x1e2d bytes)
+273( 17 mod 256): MAPREAD  0x1f000 thru 0x1f9c9	(0x9ca bytes)
+274( 18 mod 256): WRITE    0x40800 thru 0x40dff	(0x600 bytes)

+275( 19 mod 256): SKIPPED (no operation)
+276( 20 mod 256): MAPWRITE 0x20600 thru 0x22115 (0x1b16 bytes)
+277( 21 mod 256): MAPWRITE 0x3d000 thru 0x3ee5a (0x1e5b bytes)
+278( 22 mod 256): WRITE 0x2ee00 thru 0x2efff (0x200 bytes)
+279( 23 mod 256): WRITE 0x76200 thru 0x769ff (0x800 bytes) HOLE
+280( 24 mod 256): SKIPPED (no operation)
+281( 25 mod 256): SKIPPED (no operation)
+282( 26 mod 256): MAPREAD 0xa000 thru 0xa5e7 (0x5e8 bytes)
+283( 27 mod 256): SKIPPED (no operation)
+284( 28 mod 256): SKIPPED (no operation)
+285( 29 mod 256): SKIPPED (no operation)
+286( 30 mod 256): SKIPPED (no operation)
+287( 31 mod 256): COLLAPSE 0x11000 thru 0x11fff (0x1000 bytes)
+288( 32 mod 256): COPY 0x5d000 thru 0x5dfff (0x1000 bytes) to 0x4ca00 thru 0x4d9ff
+289( 33 mod 256): TRUNCATE DOWN from 0x75a00 to 0x1e400
+290( 34 mod 256): MAPREAD 0x1c000 thru 0x1d802 (0x1803 bytes) ***RRRR***

ltp/fsx.c +3 -0

+Log of operations saved to "/mnt/xfstests/test/junk.fsxops"; replay with --replay-ops
+Correct content saved for comparison
+(maybe hexdump "/mnt/xfstests/test/junk" vs "/mnt/xfstests/test/junk.fsxgood")

Quoted 5 lines

Quoted 2 lines

Thanks,
Zorro

Hi Zorro, just to confirm is this on an older kernel that doesnt support RWF_ATOMIC or on a kernle that does support it.

I tested on linux 6.16 and current latest linux v6.17+ (will be 6.18-rc1 later). About the RWF_ATOMIC flag in my system:

# grep -rsn RWF_ATOMIC /usr/include/
/usr/include/bits/uio-ext.h:51:#define RWF_ATOMIC       0x00000040 /* Write is to be issued with torn-write
/usr/include/linux/fs.h:424:#define RWF_ATOMIC  ((__kernel_rwf_t)0x00000040)
/usr/include/linux/fs.h:431:                     RWF_APPEND | RWF_NOAPPEND | RWF_ATOMIC |\
/usr/include/xfs/linux.h:236:#ifndef RWF_ATOMIC
/usr/include/xfs/linux.h:237:#define RWF_ATOMIC ((__kernel_rwf_t)0x00000040)

Thanks,
Zorro

Quoted 4 lines

Regards,
ojaswin

Quoted 145 lines · Zorro Lang

Quoted 135 lines · Ojaswin Mujoo

Quoted 34 lines · Zorro Lang

Quoted 24 lines · Ojaswin Mujoo

Quoted 10 lines · Zorro Lang

Quoted 8 lines · Ojaswin Mujoo

Implement atomic write support to help fuzz atomic writes with fsx.

Suggested-by Ritesh Harjani (IBM) <[email protected]>

Reviewed-by Darrick J. Wong <[email protected]>

Reviewed-by John Garry <[email protected]>

Signed-off-by Ojaswin Mujoo <[email protected]>

---

Quoted 4 lines

Hmm... this patch causes more regular fsx test cases fail on old kernel, (e.g. g/760, g/617, g/263 ...) except set "FSX_AVOID=-a". Is there a way to disable "atomic write" automatically if it's not supported by current system?

Hi Zorro,
Sorry for being late, I've been on vacation this week.

Yes so by design we should be automatically disabling atomic writes when they are not supported by the stack but seems like the issue is that when we do disable it we print some extra messages to stdout/err which show up in the xfstests output causing failure.

Quoted 8 lines

I can think of 2 ways around this:

1. Don't print anything and just silently drop atomic writes if stack doesn't support them.

2. Make atomic writes as a default off instead of default on feature but his loses a bit of coverage as existing tests wont get atomic write testing free of cost any more.

Quoted 6 lines

Hi Ojaswin,

Please have a nice vacation :)

It's not the "extra messages" cause failure, those "quiet" failures can be fixed by:

Oh okay got it.

Quoted 37 lines

ltp/fsx.c +6 -4

index bdb87ca90..0a035b37b 100644
@@ -1847,8 +1847,9 @@ int test_atomic_writes(void) {
        struct statx stx;
 
        if (o_direct != O_DIRECT) {
-               fprintf(stderr, "main: atomic writes need O_DIRECT (-Z), "
-                               "disabling!\n");
+               if (!quiet)
+                       fprintf(stderr, "main: atomic writes need O_DIRECT (-Z), "
+                                       "disabling!\n");
                return 0;
        }
 

@@ -1867,8 +1868,9 @@ int test_atomic_writes(void) {
                return 1;
        }
 
-       fprintf(stderr, "main: IO Stack does not support "
-                       "atomic writes, disabling!\n");
+       if (!quiet)
+               fprintf(stderr, "main: IO Stack does not support "
+                               "atomic writes, disabling!\n");
        return 0;
 }
 

But I hit more read or write failures e.g. [1], this failure can't be reproduced with FSX_AVOID=-a. Is it a atomic write bug or an unexpected test failure?

Thanks,
Zorro

<...>

Quoted 51 lines

+244(244 mod 256): SKIPPED (no operation)
+245(245 mod 256): FALLOC 0x695c5 thru 0x6a2e6 (0xd21 bytes) INTERIOR
+246(246 mod 256): MAPWRITE 0x5ac00 thru 0x5b185 (0x586 bytes)
+247(247 mod 256): WRITE 0x31200 thru 0x313ff (0x200 bytes)
+248(248 mod 256): SKIPPED (no operation)
+249(249 mod 256): TRUNCATE DOWN from 0x78242 to 0xf200 ******WWWW
+250(250 mod 256): FALLOC 0x65000 thru 0x66f26 (0x1f26 bytes) PAST_EOF
+251(251 mod 256): WRITE 0x45400 thru 0x467ff (0x1400 bytes) HOLE ***WWWW
+252(252 mod 256): SKIPPED (no operation)
+253(253 mod 256): SKIPPED (no operation)
+254(254 mod 256): MAPWRITE 0x4be00 thru 0x4daee (0x1cef bytes)
+255(255 mod 256): MAPREAD 0xc000 thru 0xcae9 (0xaea bytes)

+256(  0 mod 256): READ     0x3e000 thru 0x3efff	(0x1000 bytes)
+257(  1 mod 256): SKIPPED (no operation)
+258(  2 mod 256): INSERT 0x45000 thru 0x45fff	(0x1000 bytes)
+259(  3 mod 256): ZERO     0x1d7d5 thru 0x1f399	(0x1bc5 bytes)	******ZZZZ
+260(  4 mod 256): TRUNCATE DOWN	from 0x4eaef to 0x11200	******WWWW
+261(  5 mod 256): WRITE    0x43000 thru 0x43fff	(0x1000 bytes) HOLE	***WWWW
+262(  6 mod 256): WRITE    0x2200 thru 0x31ff	(0x1000 bytes)
+263(  7 mod 256): WRITE    0x15000 thru 0x15fff	(0x1000 bytes)
+264(  8 mod 256): WRITE    0x2e400 thru 0x2e7ff	(0x400 bytes)
+265(  9 mod 256): COPY 0xd000 thru 0xdfff	(0x1000 bytes) to 0x1d800 thru 0x1e7ff	******EEEE

+266( 10 mod 256): CLONE 0x2a000 thru 0x2afff (0x1000 bytes) to 0x21000 thru 0x21fff
+267( 11 mod 256): MAPREAD 0x31000 thru 0x31d0a (0xd0b bytes)
+268( 12 mod 256): SKIPPED (no operation)
+269( 13 mod 256): WRITE 0x25000 thru 0x25fff (0x1000 bytes)
+270( 14 mod 256): SKIPPED (no operation)

+271( 15 mod 256): MAPREAD  0x30000 thru 0x30577	(0x578 bytes)
+272( 16 mod 256): PUNCH    0x1a267 thru 0x1c093	(0x1e2d bytes)
+273( 17 mod 256): MAPREAD  0x1f000 thru 0x1f9c9	(0x9ca bytes)
+274( 18 mod 256): WRITE    0x40800 thru 0x40dff	(0x600 bytes)

+275( 19 mod 256): SKIPPED (no operation)
+276( 20 mod 256): MAPWRITE 0x20600 thru 0x22115 (0x1b16 bytes)
+277( 21 mod 256): MAPWRITE 0x3d000 thru 0x3ee5a (0x1e5b bytes)
+278( 22 mod 256): WRITE 0x2ee00 thru 0x2efff (0x200 bytes)
+279( 23 mod 256): WRITE 0x76200 thru 0x769ff (0x800 bytes) HOLE
+280( 24 mod 256): SKIPPED (no operation)
+281( 25 mod 256): SKIPPED (no operation)
+282( 26 mod 256): MAPREAD 0xa000 thru 0xa5e7 (0x5e8 bytes)
+283( 27 mod 256): SKIPPED (no operation)
+284( 28 mod 256): SKIPPED (no operation)
+285( 29 mod 256): SKIPPED (no operation)
+286( 30 mod 256): SKIPPED (no operation)
+287( 31 mod 256): COLLAPSE 0x11000 thru 0x11fff (0x1000 bytes)
+288( 32 mod 256): COPY 0x5d000 thru 0x5dfff (0x1000 bytes) to 0x4ca00 thru 0x4d9ff
+289( 33 mod 256): TRUNCATE DOWN from 0x75a00 to 0x1e400
+290( 34 mod 256): MAPREAD 0x1c000 thru 0x1d802 (0x1803 bytes) ***RRRR***

ltp/fsx.c +3 -0

+Log of operations saved to "/mnt/xfstests/test/junk.fsxops"; replay with --replay-ops
+Correct content saved for comparison
+(maybe hexdump "/mnt/xfstests/test/junk" vs "/mnt/xfstests/test/junk.fsxgood")

Quoted 5 lines

Quoted 2 lines

Thanks,
Zorro

Hi Zorro, just to confirm is this on an older kernel that doesnt support RWF_ATOMIC or on a kernle that does support it.

I tested on linux 6.16 and current latest linux v6.17+ (will be 6.18-rc1 later). About the RWF_ATOMIC flag in my system:

Quoted 6 lines

# grep -rsn RWF_ATOMIC /usr/include/
/usr/include/bits/uio-ext.h:51:#define RWF_ATOMIC       0x00000040 /* Write is to be issued with torn-write
/usr/include/linux/fs.h:424:#define RWF_ATOMIC  ((__kernel_rwf_t)0x00000040)
/usr/include/linux/fs.h:431:                     RWF_APPEND | RWF_NOAPPEND | RWF_ATOMIC |\
/usr/include/xfs/linux.h:236:#ifndef RWF_ATOMIC
/usr/include/xfs/linux.h:237:#define RWF_ATOMIC ((__kernel_rwf_t)0x00000040)

Hi Zorro, thanks for checking this. So correct me if im wrong but I understand that you have run this test on an atomic writes enabled kernel where the stack also supports atomic writes.

Looking at the bad data log:

	+READ BAD DATA: offset = 0x1c000, size = 0x1803, fname = /mnt/xfstests/test/junk
	+OFFSET      GOOD    BAD     RANGE
	+0x1c000     0x0000  0xcdcd  0x0
	+operation# (mod 256) for the bad data may be 205

We see that 0x0000 was expected but we got 0xcdcd. Now the operation that caused this is indicated to be 205, but looking at that operation:

+205(205 mod 256): ZERO 0x6dbe6 thru 0x6e6aa (0xac5 bytes)

This doesn't even overlap the range that is bad. (0x1c000 to 0x1c00f). Infact, it does seem like an unlikely coincidence that the actual data in the bad range is 0xcdcd which is something xfs_io -c "pwrite" writes to default (fsx writes random data in even offsets and operation num in odd).

I am able to replicate this but only on XFS but not on ext4 (atleast not in 20 runs). I'm trying to better understand if this is a test issue or not. Will keep you update.

I'm not sure how this will affect the upcoming release, if you want shall I send a small patch to make the atomic writes feature default off instead of default on till we root cause this?

Regards,
Ojaswin

Quoted 10 lines

Thanks,
Zorro

Quoted 4 lines

Regards,
ojaswin

Quoted 179 lines · Ojaswin Mujoo

Quoted 145 lines · Zorro Lang

Quoted 135 lines · Ojaswin Mujoo

Quoted 34 lines · Zorro Lang

Quoted 24 lines · Ojaswin Mujoo

Quoted 10 lines · Zorro Lang

Quoted 8 lines · Ojaswin Mujoo

Implement atomic write support to help fuzz atomic writes with fsx.

Suggested-by Ritesh Harjani (IBM) <[email protected]>

Reviewed-by Darrick J. Wong <[email protected]>

Reviewed-by John Garry <[email protected]>

Signed-off-by Ojaswin Mujoo <[email protected]>

---

Quoted 4 lines

Hmm... this patch causes more regular fsx test cases fail on old kernel, (e.g. g/760, g/617, g/263 ...) except set "FSX_AVOID=-a". Is there a way to disable "atomic write" automatically if it's not supported by current system?

Hi Zorro,
Sorry for being late, I've been on vacation this week.

Yes so by design we should be automatically disabling atomic writes when they are not supported by the stack but seems like the issue is that when we do disable it we print some extra messages to stdout/err which show up in the xfstests output causing failure.

Quoted 8 lines

I can think of 2 ways around this:

1. Don't print anything and just silently drop atomic writes if stack doesn't support them.

2. Make atomic writes as a default off instead of default on feature but his loses a bit of coverage as existing tests wont get atomic write testing free of cost any more.

Quoted 6 lines

Hi Ojaswin,

Please have a nice vacation :)

It's not the "extra messages" cause failure, those "quiet" failures can be fixed by:

Oh okay got it.

Quoted 37 lines

ltp/fsx.c +6 -4

index bdb87ca90..0a035b37b 100644
@@ -1847,8 +1847,9 @@ int test_atomic_writes(void) {
        struct statx stx;
 
        if (o_direct != O_DIRECT) {
-               fprintf(stderr, "main: atomic writes need O_DIRECT (-Z), "
-                               "disabling!\n");
+               if (!quiet)
+                       fprintf(stderr, "main: atomic writes need O_DIRECT (-Z), "
+                                       "disabling!\n");
                return 0;
        }
 

@@ -1867,8 +1868,9 @@ int test_atomic_writes(void) {
                return 1;
        }
 
-       fprintf(stderr, "main: IO Stack does not support "
-                       "atomic writes, disabling!\n");
+       if (!quiet)
+               fprintf(stderr, "main: IO Stack does not support "
+                               "atomic writes, disabling!\n");
        return 0;
 }
 

But I hit more read or write failures e.g. [1], this failure can't be reproduced with FSX_AVOID=-a. Is it a atomic write bug or an unexpected test failure?

Thanks,
Zorro

<...>

Quoted 51 lines

+244(244 mod 256): SKIPPED (no operation)
+245(245 mod 256): FALLOC 0x695c5 thru 0x6a2e6 (0xd21 bytes) INTERIOR
+246(246 mod 256): MAPWRITE 0x5ac00 thru 0x5b185 (0x586 bytes)
+247(247 mod 256): WRITE 0x31200 thru 0x313ff (0x200 bytes)
+248(248 mod 256): SKIPPED (no operation)
+249(249 mod 256): TRUNCATE DOWN from 0x78242 to 0xf200 ******WWWW
+250(250 mod 256): FALLOC 0x65000 thru 0x66f26 (0x1f26 bytes) PAST_EOF
+251(251 mod 256): WRITE 0x45400 thru 0x467ff (0x1400 bytes) HOLE ***WWWW
+252(252 mod 256): SKIPPED (no operation)
+253(253 mod 256): SKIPPED (no operation)
+254(254 mod 256): MAPWRITE 0x4be00 thru 0x4daee (0x1cef bytes)
+255(255 mod 256): MAPREAD 0xc000 thru 0xcae9 (0xaea bytes)

+256(  0 mod 256): READ     0x3e000 thru 0x3efff	(0x1000 bytes)
+257(  1 mod 256): SKIPPED (no operation)
+258(  2 mod 256): INSERT 0x45000 thru 0x45fff	(0x1000 bytes)
+259(  3 mod 256): ZERO     0x1d7d5 thru 0x1f399	(0x1bc5 bytes)	******ZZZZ
+260(  4 mod 256): TRUNCATE DOWN	from 0x4eaef to 0x11200	******WWWW
+261(  5 mod 256): WRITE    0x43000 thru 0x43fff	(0x1000 bytes) HOLE	***WWWW
+262(  6 mod 256): WRITE    0x2200 thru 0x31ff	(0x1000 bytes)
+263(  7 mod 256): WRITE    0x15000 thru 0x15fff	(0x1000 bytes)
+264(  8 mod 256): WRITE    0x2e400 thru 0x2e7ff	(0x400 bytes)
+265(  9 mod 256): COPY 0xd000 thru 0xdfff	(0x1000 bytes) to 0x1d800 thru 0x1e7ff	******EEEE

+266( 10 mod 256): CLONE 0x2a000 thru 0x2afff (0x1000 bytes) to 0x21000 thru 0x21fff
+267( 11 mod 256): MAPREAD 0x31000 thru 0x31d0a (0xd0b bytes)
+268( 12 mod 256): SKIPPED (no operation)
+269( 13 mod 256): WRITE 0x25000 thru 0x25fff (0x1000 bytes)
+270( 14 mod 256): SKIPPED (no operation)

+271( 15 mod 256): MAPREAD  0x30000 thru 0x30577	(0x578 bytes)
+272( 16 mod 256): PUNCH    0x1a267 thru 0x1c093	(0x1e2d bytes)
+273( 17 mod 256): MAPREAD  0x1f000 thru 0x1f9c9	(0x9ca bytes)
+274( 18 mod 256): WRITE    0x40800 thru 0x40dff	(0x600 bytes)

+275( 19 mod 256): SKIPPED (no operation)
+276( 20 mod 256): MAPWRITE 0x20600 thru 0x22115 (0x1b16 bytes)
+277( 21 mod 256): MAPWRITE 0x3d000 thru 0x3ee5a (0x1e5b bytes)
+278( 22 mod 256): WRITE 0x2ee00 thru 0x2efff (0x200 bytes)
+279( 23 mod 256): WRITE 0x76200 thru 0x769ff (0x800 bytes) HOLE
+280( 24 mod 256): SKIPPED (no operation)
+281( 25 mod 256): SKIPPED (no operation)
+282( 26 mod 256): MAPREAD 0xa000 thru 0xa5e7 (0x5e8 bytes)
+283( 27 mod 256): SKIPPED (no operation)
+284( 28 mod 256): SKIPPED (no operation)
+285( 29 mod 256): SKIPPED (no operation)
+286( 30 mod 256): SKIPPED (no operation)
+287( 31 mod 256): COLLAPSE 0x11000 thru 0x11fff (0x1000 bytes)
+288( 32 mod 256): COPY 0x5d000 thru 0x5dfff (0x1000 bytes) to 0x4ca00 thru 0x4d9ff
+289( 33 mod 256): TRUNCATE DOWN from 0x75a00 to 0x1e400
+290( 34 mod 256): MAPREAD 0x1c000 thru 0x1d802 (0x1803 bytes) ***RRRR***

ltp/fsx.c +3 -0

+Log of operations saved to "/mnt/xfstests/test/junk.fsxops"; replay with --replay-ops
+Correct content saved for comparison
+(maybe hexdump "/mnt/xfstests/test/junk" vs "/mnt/xfstests/test/junk.fsxgood")

Quoted 5 lines

Quoted 2 lines

Thanks,
Zorro

Hi Zorro, just to confirm is this on an older kernel that doesnt support RWF_ATOMIC or on a kernle that does support it.

I tested on linux 6.16 and current latest linux v6.17+ (will be 6.18-rc1 later). About the RWF_ATOMIC flag in my system:

Quoted 6 lines

# grep -rsn RWF_ATOMIC /usr/include/
/usr/include/bits/uio-ext.h:51:#define RWF_ATOMIC       0x00000040 /* Write is to be issued with torn-write
/usr/include/linux/fs.h:424:#define RWF_ATOMIC  ((__kernel_rwf_t)0x00000040)
/usr/include/linux/fs.h:431:                     RWF_APPEND | RWF_NOAPPEND | RWF_ATOMIC |\
/usr/include/xfs/linux.h:236:#ifndef RWF_ATOMIC
/usr/include/xfs/linux.h:237:#define RWF_ATOMIC ((__kernel_rwf_t)0x00000040)

Hi Zorro, thanks for checking this. So correct me if im wrong but I understand that you have run this test on an atomic writes enabled kernel where the stack also supports atomic writes.

Looking at the bad data log:

	+READ BAD DATA: offset = 0x1c000, size = 0x1803, fname = /mnt/xfstests/test/junk
	+OFFSET      GOOD    BAD     RANGE
	+0x1c000     0x0000  0xcdcd  0x0
	+operation# (mod 256) for the bad data may be 205

We see that 0x0000 was expected but we got 0xcdcd. Now the operation that caused this is indicated to be 205, but looking at that operation:

+205(205 mod 256): ZERO 0x6dbe6 thru 0x6e6aa (0xac5 bytes)

This doesn't even overlap the range that is bad. (0x1c000 to 0x1c00f). Infact, it does seem like an unlikely coincidence that the actual data in the bad range is 0xcdcd which is something xfs_io -c "pwrite" writes to default (fsx writes random data in even offsets and operation num in odd).

I am able to replicate this but only on XFS but not on ext4 (atleast not in 20 runs). I'm trying to better understand if this is a test issue or not. Will keep you update.

Quoted 6 lines

I'm not sure how this will affect the upcoming release, if you want shall I send a small patch to make the atomic writes feature default off instead of default on till we root cause this?

Regards,
Ojaswin

Hi Zorro,

So I'm able to narrow down the opoerations and replicate it via the following replay file:

# -----

# replay.fsxops
# -----
write_atomic 0x57000 0x1000 0x69690
write_atomic 0x66000 0x1000 0x4de00
write_atomic 0x18000 0x1000 0x2c800
copy_range 0x20000 0x1000 0xe00 0x70e00
write_atomic 0x18000 0x1000 0x70e00
copy_range 0x21000 0x1000 0x23000 0x74218
truncate 0x0 0x11200 0x4daef *
write_atomic 0x43000 0x1000 0x11200 *
write_atomic 0x15000 0x1000 0x44000
copy_range 0xd000 0x1000 0x1d800 0x44000
mapread 0x1c000 0x1803 0x1e400 *

Command: ./ltp/fsx -N 10000 -o 8192 -l 500000 -r 4096 -t 512 -w 512 -Z -FKuHzI --replay-ops replay.fsxops $MNT/junk

$MNT/junk is always opened O_TRUNC and is an on an XFS FS where the disk is non-atomic so all RWF_ATOMIC writes are software emulated.

Here are the logs generated for this run:

Seed set to 1
main: filesystem does not support exchange range, disabling!

READ BAD DATA: offset = 0x1c000, size = 0x1803, fname = /mnt/test/junk

OFFSET      GOOD    BAD     RANGE
0x1d000     0x0000  0xf322  0x0

operation# (mod 256) for the bad data may be 243
0x1d001 0x0000 0x22f3 0x1
operation# (mod 256) for the bad data may be 243
0x1d002 0x0000 0xf391 0x2
operation# (mod 256) for the bad data may be 243
0x1d003 0x0000 0x91f3 0x3
<... a few more such lines ..>

LOG DUMP (11 total operations):
openat(AT_FDCWD, "/mnt/test/junk.fsxops", O_WRONLY|O_CREAT|O_TRUNC, 0666) = 7

1(  1 mod 256): WRITE    0x57000 thru 0x57fff   (0x1000 bytes) HOLE     ***WWWW ATOMIC
2(  2 mod 256): WRITE    0x66000 thru 0x66fff   (0x1000 bytes) HOLE ATOMIC
3(  3 mod 256): WRITE    0x18000 thru 0x18fff   (0x1000 bytes) ATOMIC
4(  4 mod 256): COPY 0x20000 thru 0x20fff       (0x1000 bytes) to 0xe00 thru 0x1dff
5(  5 mod 256): WRITE    0x18000 thru 0x18fff   (0x1000 bytes) ATOMIC
6(  6 mod 256): COPY 0x21000 thru 0x21fff       (0x1000 bytes) to 0x23000 thru 0x23fff
7(  7 mod 256): TRUNCATE DOWN   from 0x67000 to 0x11200 ******WWWW
8(  8 mod 256): WRITE    0x43000 thru 0x43fff   (0x1000 bytes) HOLE     ***WWWW ATOMIC
9(  9 mod 256): WRITE    0x15000 thru 0x15fff   (0x1000 bytes) ATOMIC

10( 10 mod 256): COPY 0xd000 thru 0xdfff (0x1000 bytes) to 0x1d800 thru 0x1e7ff
11( 11 mod 256): MAPREAD 0x1c000 thru 0x1d802 (0x1803 bytes) ***RRRR*** Log of operations saved to "/mnt/test/junk.fsxops"; replay with --replay-ops Correct content saved for comparison
(maybe hexdump "/mnt/test/junk" vs "/mnt/test/junk.fsxgood")
+++ exited with 110 +++

We can see that the bad data is detected in the final MAPREAD operation and and bad offset is at 0x1d000. If we look at the operations dump above its clear that none of the operations should be modifying the 0x1d000 so we should have been reading 0s but yet we see some junk data there in the file:

$ hexdump /mnt/test/junk -s 0x1c000 -n0x1020
001c000 0000 0000 0000 0000 0000 0000 0000 0000
*
001d000 22f3 91f3 7ff3 3af3 39f3 23f3 6df3 c2f3
001d010 c5f3 f6f3 a6f3 1ef3 58f3 40f3 32f3 5ff3
001d020

Another thing to not is that I can't reproduce the above on scsi-debug device. @Darrick, @John, could this be an issue in kernel?

Regards,
ojaswin

Quoted 10 lines

Quoted 10 lines

Thanks,
Zorro

Quoted 4 lines

Regards,
ojaswin

Quoted 260 lines · Ojaswin Mujoo

Quoted 179 lines · Ojaswin Mujoo

Quoted 145 lines · Zorro Lang

Quoted 135 lines · Ojaswin Mujoo

Quoted 34 lines · Zorro Lang

Quoted 24 lines · Ojaswin Mujoo

Quoted 10 lines · Zorro Lang

Quoted 8 lines · Ojaswin Mujoo

Implement atomic write support to help fuzz atomic writes with fsx.

Suggested-by Ritesh Harjani (IBM) <[email protected]>

Reviewed-by Darrick J. Wong <[email protected]>

Reviewed-by John Garry <[email protected]>

Signed-off-by Ojaswin Mujoo <[email protected]>

---

Quoted 4 lines

Hmm... this patch causes more regular fsx test cases fail on old kernel, (e.g. g/760, g/617, g/263 ...) except set "FSX_AVOID=-a". Is there a way to disable "atomic write" automatically if it's not supported by current system?

Hi Zorro,
Sorry for being late, I've been on vacation this week.

Yes so by design we should be automatically disabling atomic writes when they are not supported by the stack but seems like the issue is that when we do disable it we print some extra messages to stdout/err which show up in the xfstests output causing failure.

Quoted 8 lines

I can think of 2 ways around this:

1. Don't print anything and just silently drop atomic writes if stack doesn't support them.

2. Make atomic writes as a default off instead of default on feature but his loses a bit of coverage as existing tests wont get atomic write testing free of cost any more.

Quoted 6 lines

Hi Ojaswin,

Please have a nice vacation :)

It's not the "extra messages" cause failure, those "quiet" failures can be fixed by:

Oh okay got it.

Quoted 37 lines

ltp/fsx.c +6 -4

index bdb87ca90..0a035b37b 100644
@@ -1847,8 +1847,9 @@ int test_atomic_writes(void) {
        struct statx stx;
 
        if (o_direct != O_DIRECT) {
-               fprintf(stderr, "main: atomic writes need O_DIRECT (-Z), "
-                               "disabling!\n");
+               if (!quiet)
+                       fprintf(stderr, "main: atomic writes need O_DIRECT (-Z), "
+                                       "disabling!\n");
                return 0;
        }
 

@@ -1867,8 +1868,9 @@ int test_atomic_writes(void) {
                return 1;
        }
 
-       fprintf(stderr, "main: IO Stack does not support "
-                       "atomic writes, disabling!\n");
+       if (!quiet)
+               fprintf(stderr, "main: IO Stack does not support "
+                               "atomic writes, disabling!\n");
        return 0;
 }
 

But I hit more read or write failures e.g. [1], this failure can't be reproduced with FSX_AVOID=-a. Is it a atomic write bug or an unexpected test failure?

Thanks,
Zorro

<...>

Quoted 51 lines

+244(244 mod 256): SKIPPED (no operation)
+245(245 mod 256): FALLOC 0x695c5 thru 0x6a2e6 (0xd21 bytes) INTERIOR
+246(246 mod 256): MAPWRITE 0x5ac00 thru 0x5b185 (0x586 bytes)
+247(247 mod 256): WRITE 0x31200 thru 0x313ff (0x200 bytes)
+248(248 mod 256): SKIPPED (no operation)
+249(249 mod 256): TRUNCATE DOWN from 0x78242 to 0xf200 ******WWWW
+250(250 mod 256): FALLOC 0x65000 thru 0x66f26 (0x1f26 bytes) PAST_EOF
+251(251 mod 256): WRITE 0x45400 thru 0x467ff (0x1400 bytes) HOLE ***WWWW
+252(252 mod 256): SKIPPED (no operation)
+253(253 mod 256): SKIPPED (no operation)
+254(254 mod 256): MAPWRITE 0x4be00 thru 0x4daee (0x1cef bytes)
+255(255 mod 256): MAPREAD 0xc000 thru 0xcae9 (0xaea bytes)

+256(  0 mod 256): READ     0x3e000 thru 0x3efff	(0x1000 bytes)
+257(  1 mod 256): SKIPPED (no operation)
+258(  2 mod 256): INSERT 0x45000 thru 0x45fff	(0x1000 bytes)
+259(  3 mod 256): ZERO     0x1d7d5 thru 0x1f399	(0x1bc5 bytes)	******ZZZZ
+260(  4 mod 256): TRUNCATE DOWN	from 0x4eaef to 0x11200	******WWWW
+261(  5 mod 256): WRITE    0x43000 thru 0x43fff	(0x1000 bytes) HOLE	***WWWW
+262(  6 mod 256): WRITE    0x2200 thru 0x31ff	(0x1000 bytes)
+263(  7 mod 256): WRITE    0x15000 thru 0x15fff	(0x1000 bytes)
+264(  8 mod 256): WRITE    0x2e400 thru 0x2e7ff	(0x400 bytes)
+265(  9 mod 256): COPY 0xd000 thru 0xdfff	(0x1000 bytes) to 0x1d800 thru 0x1e7ff	******EEEE

+266( 10 mod 256): CLONE 0x2a000 thru 0x2afff (0x1000 bytes) to 0x21000 thru 0x21fff
+267( 11 mod 256): MAPREAD 0x31000 thru 0x31d0a (0xd0b bytes)
+268( 12 mod 256): SKIPPED (no operation)
+269( 13 mod 256): WRITE 0x25000 thru 0x25fff (0x1000 bytes)
+270( 14 mod 256): SKIPPED (no operation)

+271( 15 mod 256): MAPREAD  0x30000 thru 0x30577	(0x578 bytes)
+272( 16 mod 256): PUNCH    0x1a267 thru 0x1c093	(0x1e2d bytes)
+273( 17 mod 256): MAPREAD  0x1f000 thru 0x1f9c9	(0x9ca bytes)
+274( 18 mod 256): WRITE    0x40800 thru 0x40dff	(0x600 bytes)

+275( 19 mod 256): SKIPPED (no operation)
+276( 20 mod 256): MAPWRITE 0x20600 thru 0x22115 (0x1b16 bytes)
+277( 21 mod 256): MAPWRITE 0x3d000 thru 0x3ee5a (0x1e5b bytes)
+278( 22 mod 256): WRITE 0x2ee00 thru 0x2efff (0x200 bytes)
+279( 23 mod 256): WRITE 0x76200 thru 0x769ff (0x800 bytes) HOLE
+280( 24 mod 256): SKIPPED (no operation)
+281( 25 mod 256): SKIPPED (no operation)
+282( 26 mod 256): MAPREAD 0xa000 thru 0xa5e7 (0x5e8 bytes)
+283( 27 mod 256): SKIPPED (no operation)
+284( 28 mod 256): SKIPPED (no operation)
+285( 29 mod 256): SKIPPED (no operation)
+286( 30 mod 256): SKIPPED (no operation)
+287( 31 mod 256): COLLAPSE 0x11000 thru 0x11fff (0x1000 bytes)
+288( 32 mod 256): COPY 0x5d000 thru 0x5dfff (0x1000 bytes) to 0x4ca00 thru 0x4d9ff
+289( 33 mod 256): TRUNCATE DOWN from 0x75a00 to 0x1e400
+290( 34 mod 256): MAPREAD 0x1c000 thru 0x1d802 (0x1803 bytes) ***RRRR***

ltp/fsx.c +3 -0

+Log of operations saved to "/mnt/xfstests/test/junk.fsxops"; replay with --replay-ops
+Correct content saved for comparison
+(maybe hexdump "/mnt/xfstests/test/junk" vs "/mnt/xfstests/test/junk.fsxgood")

Quoted 5 lines

Quoted 2 lines

Thanks,
Zorro

Hi Zorro, just to confirm is this on an older kernel that doesnt support RWF_ATOMIC or on a kernle that does support it.

I tested on linux 6.16 and current latest linux v6.17+ (will be 6.18-rc1 later). About the RWF_ATOMIC flag in my system:

Quoted 6 lines

# grep -rsn RWF_ATOMIC /usr/include/
/usr/include/bits/uio-ext.h:51:#define RWF_ATOMIC       0x00000040 /* Write is to be issued with torn-write
/usr/include/linux/fs.h:424:#define RWF_ATOMIC  ((__kernel_rwf_t)0x00000040)
/usr/include/linux/fs.h:431:                     RWF_APPEND | RWF_NOAPPEND | RWF_ATOMIC |\
/usr/include/xfs/linux.h:236:#ifndef RWF_ATOMIC
/usr/include/xfs/linux.h:237:#define RWF_ATOMIC ((__kernel_rwf_t)0x00000040)

Hi Zorro, thanks for checking this. So correct me if im wrong but I understand that you have run this test on an atomic writes enabled kernel where the stack also supports atomic writes.

Looking at the bad data log:

	+READ BAD DATA: offset = 0x1c000, size = 0x1803, fname = /mnt/xfstests/test/junk
	+OFFSET      GOOD    BAD     RANGE
	+0x1c000     0x0000  0xcdcd  0x0
	+operation# (mod 256) for the bad data may be 205

We see that 0x0000 was expected but we got 0xcdcd. Now the operation that caused this is indicated to be 205, but looking at that operation:

+205(205 mod 256): ZERO 0x6dbe6 thru 0x6e6aa (0xac5 bytes)

This doesn't even overlap the range that is bad. (0x1c000 to 0x1c00f). Infact, it does seem like an unlikely coincidence that the actual data in the bad range is 0xcdcd which is something xfs_io -c "pwrite" writes to default (fsx writes random data in even offsets and operation num in odd).

I am able to replicate this but only on XFS but not on ext4 (atleast not in 20 runs). I'm trying to better understand if this is a test issue or not. Will keep you update.

Quoted 6 lines

I'm not sure how this will affect the upcoming release, if you want shall I send a small patch to make the atomic writes feature default off instead of default on till we root cause this?

Regards,
Ojaswin

Hi Zorro,

So I'm able to narrow down the opoerations and replicate it via the following replay file:

# -----

# replay.fsxops
# -----
write_atomic 0x57000 0x1000 0x69690
write_atomic 0x66000 0x1000 0x4de00
write_atomic 0x18000 0x1000 0x2c800
copy_range 0x20000 0x1000 0xe00 0x70e00
write_atomic 0x18000 0x1000 0x70e00
copy_range 0x21000 0x1000 0x23000 0x74218
truncate 0x0 0x11200 0x4daef *
write_atomic 0x43000 0x1000 0x11200 *
write_atomic 0x15000 0x1000 0x44000
copy_range 0xd000 0x1000 0x1d800 0x44000
mapread 0x1c000 0x1803 0x1e400 *

Command: ./ltp/fsx -N 10000 -o 8192 -l 500000 -r 4096 -t 512 -w 512 -Z -FKuHzI --replay-ops replay.fsxops $MNT/junk

$MNT/junk is always opened O_TRUNC and is an on an XFS FS where the disk is non-atomic so all RWF_ATOMIC writes are software emulated.

Here are the logs generated for this run:

Seed set to 1
main: filesystem does not support exchange range, disabling!

READ BAD DATA: offset = 0x1c000, size = 0x1803, fname = /mnt/test/junk

OFFSET      GOOD    BAD     RANGE
0x1d000     0x0000  0xf322  0x0

operation# (mod 256) for the bad data may be 243
0x1d001 0x0000 0x22f3 0x1
operation# (mod 256) for the bad data may be 243
0x1d002 0x0000 0xf391 0x2
operation# (mod 256) for the bad data may be 243
0x1d003 0x0000 0x91f3 0x3
<... a few more such lines ..>

LOG DUMP (11 total operations):
openat(AT_FDCWD, "/mnt/test/junk.fsxops", O_WRONLY|O_CREAT|O_TRUNC, 0666) = 7

1(  1 mod 256): WRITE    0x57000 thru 0x57fff   (0x1000 bytes) HOLE     ***WWWW ATOMIC
2(  2 mod 256): WRITE    0x66000 thru 0x66fff   (0x1000 bytes) HOLE ATOMIC
3(  3 mod 256): WRITE    0x18000 thru 0x18fff   (0x1000 bytes) ATOMIC
4(  4 mod 256): COPY 0x20000 thru 0x20fff       (0x1000 bytes) to 0xe00 thru 0x1dff
5(  5 mod 256): WRITE    0x18000 thru 0x18fff   (0x1000 bytes) ATOMIC
6(  6 mod 256): COPY 0x21000 thru 0x21fff       (0x1000 bytes) to 0x23000 thru 0x23fff
7(  7 mod 256): TRUNCATE DOWN   from 0x67000 to 0x11200 ******WWWW
8(  8 mod 256): WRITE    0x43000 thru 0x43fff   (0x1000 bytes) HOLE     ***WWWW ATOMIC
9(  9 mod 256): WRITE    0x15000 thru 0x15fff   (0x1000 bytes) ATOMIC

10( 10 mod 256): COPY 0xd000 thru 0xdfff (0x1000 bytes) to 0x1d800 thru 0x1e7ff
11( 11 mod 256): MAPREAD 0x1c000 thru 0x1d802 (0x1803 bytes) ***RRRR*** Log of operations saved to "/mnt/test/junk.fsxops"; replay with --replay-ops Correct content saved for comparison
(maybe hexdump "/mnt/test/junk" vs "/mnt/test/junk.fsxgood")
+++ exited with 110 +++

We can see that the bad data is detected in the final MAPREAD operation and and bad offset is at 0x1d000. If we look at the operations dump above its clear that none of the operations should be modifying the 0x1d000 so we should have been reading 0s but yet we see some junk data there in the file:

$ hexdump /mnt/test/junk -s 0x1c000 -n0x1020
001c000 0000 0000 0000 0000 0000 0000 0000 0000
*
001d000 22f3 91f3 7ff3 3af3 39f3 23f3 6df3 c2f3
001d010 c5f3 f6f3 a6f3 1ef3 58f3 40f3 32f3 5ff3
001d020

Quoted 2 lines

Another thing to not is that I can't reproduce the above on scsi-debug device. @Darrick, @John, could this be an issue in kernel?

Hi Ojaswin,

If we can be sure this's a kernel bug, rather than a fstests (patch) issue, I think we can merge this patchset to expose this bug. Does this make sense to you and others?

Thanks,
Zorro

Quoted 14 lines

Regards,
ojaswin

Quoted 10 lines

Quoted 10 lines

Thanks,
Zorro

Quoted 4 lines

Regards,
ojaswin

Quoted 266 lines · Zorro Lang

Quoted 260 lines · Ojaswin Mujoo

Quoted 179 lines · Ojaswin Mujoo

Quoted 145 lines · Zorro Lang

Quoted 135 lines · Ojaswin Mujoo

Quoted 34 lines · Zorro Lang

Quoted 24 lines · Ojaswin Mujoo

Quoted 10 lines · Zorro Lang

Quoted 8 lines · Ojaswin Mujoo

Implement atomic write support to help fuzz atomic writes with fsx.

Suggested-by Ritesh Harjani (IBM) <[email protected]>

Reviewed-by Darrick J. Wong <[email protected]>

Reviewed-by John Garry <[email protected]>

Signed-off-by Ojaswin Mujoo <[email protected]>

---

Quoted 4 lines

Hmm... this patch causes more regular fsx test cases fail on old kernel, (e.g. g/760, g/617, g/263 ...) except set "FSX_AVOID=-a". Is there a way to disable "atomic write" automatically if it's not supported by current system?

Hi Zorro,
Sorry for being late, I've been on vacation this week.

Yes so by design we should be automatically disabling atomic writes when they are not supported by the stack but seems like the issue is that when we do disable it we print some extra messages to stdout/err which show up in the xfstests output causing failure.

Quoted 8 lines

I can think of 2 ways around this:

1. Don't print anything and just silently drop atomic writes if stack doesn't support them.

2. Make atomic writes as a default off instead of default on feature but his loses a bit of coverage as existing tests wont get atomic write testing free of cost any more.

Quoted 6 lines

Hi Ojaswin,

Please have a nice vacation :)

It's not the "extra messages" cause failure, those "quiet" failures can be fixed by:

Oh okay got it.

Quoted 37 lines

ltp/fsx.c +6 -4

index bdb87ca90..0a035b37b 100644
@@ -1847,8 +1847,9 @@ int test_atomic_writes(void) {
        struct statx stx;
 
        if (o_direct != O_DIRECT) {
-               fprintf(stderr, "main: atomic writes need O_DIRECT (-Z), "
-                               "disabling!\n");
+               if (!quiet)
+                       fprintf(stderr, "main: atomic writes need O_DIRECT (-Z), "
+                                       "disabling!\n");
                return 0;
        }
 

@@ -1867,8 +1868,9 @@ int test_atomic_writes(void) {
                return 1;
        }
 
-       fprintf(stderr, "main: IO Stack does not support "
-                       "atomic writes, disabling!\n");
+       if (!quiet)
+               fprintf(stderr, "main: IO Stack does not support "
+                               "atomic writes, disabling!\n");
        return 0;
 }
 

But I hit more read or write failures e.g. [1], this failure can't be reproduced with FSX_AVOID=-a. Is it a atomic write bug or an unexpected test failure?

Thanks,
Zorro

<...>

Quoted 51 lines

+244(244 mod 256): SKIPPED (no operation)
+245(245 mod 256): FALLOC 0x695c5 thru 0x6a2e6 (0xd21 bytes) INTERIOR
+246(246 mod 256): MAPWRITE 0x5ac00 thru 0x5b185 (0x586 bytes)
+247(247 mod 256): WRITE 0x31200 thru 0x313ff (0x200 bytes)
+248(248 mod 256): SKIPPED (no operation)
+249(249 mod 256): TRUNCATE DOWN from 0x78242 to 0xf200 ******WWWW
+250(250 mod 256): FALLOC 0x65000 thru 0x66f26 (0x1f26 bytes) PAST_EOF
+251(251 mod 256): WRITE 0x45400 thru 0x467ff (0x1400 bytes) HOLE ***WWWW
+252(252 mod 256): SKIPPED (no operation)
+253(253 mod 256): SKIPPED (no operation)
+254(254 mod 256): MAPWRITE 0x4be00 thru 0x4daee (0x1cef bytes)
+255(255 mod 256): MAPREAD 0xc000 thru 0xcae9 (0xaea bytes)

+256(  0 mod 256): READ     0x3e000 thru 0x3efff	(0x1000 bytes)
+257(  1 mod 256): SKIPPED (no operation)
+258(  2 mod 256): INSERT 0x45000 thru 0x45fff	(0x1000 bytes)
+259(  3 mod 256): ZERO     0x1d7d5 thru 0x1f399	(0x1bc5 bytes)	******ZZZZ
+260(  4 mod 256): TRUNCATE DOWN	from 0x4eaef to 0x11200	******WWWW
+261(  5 mod 256): WRITE    0x43000 thru 0x43fff	(0x1000 bytes) HOLE	***WWWW
+262(  6 mod 256): WRITE    0x2200 thru 0x31ff	(0x1000 bytes)
+263(  7 mod 256): WRITE    0x15000 thru 0x15fff	(0x1000 bytes)
+264(  8 mod 256): WRITE    0x2e400 thru 0x2e7ff	(0x400 bytes)
+265(  9 mod 256): COPY 0xd000 thru 0xdfff	(0x1000 bytes) to 0x1d800 thru 0x1e7ff	******EEEE

+266( 10 mod 256): CLONE 0x2a000 thru 0x2afff (0x1000 bytes) to 0x21000 thru 0x21fff
+267( 11 mod 256): MAPREAD 0x31000 thru 0x31d0a (0xd0b bytes)
+268( 12 mod 256): SKIPPED (no operation)
+269( 13 mod 256): WRITE 0x25000 thru 0x25fff (0x1000 bytes)
+270( 14 mod 256): SKIPPED (no operation)

+271( 15 mod 256): MAPREAD  0x30000 thru 0x30577	(0x578 bytes)
+272( 16 mod 256): PUNCH    0x1a267 thru 0x1c093	(0x1e2d bytes)
+273( 17 mod 256): MAPREAD  0x1f000 thru 0x1f9c9	(0x9ca bytes)
+274( 18 mod 256): WRITE    0x40800 thru 0x40dff	(0x600 bytes)

+275( 19 mod 256): SKIPPED (no operation)
+276( 20 mod 256): MAPWRITE 0x20600 thru 0x22115 (0x1b16 bytes)
+277( 21 mod 256): MAPWRITE 0x3d000 thru 0x3ee5a (0x1e5b bytes)
+278( 22 mod 256): WRITE 0x2ee00 thru 0x2efff (0x200 bytes)
+279( 23 mod 256): WRITE 0x76200 thru 0x769ff (0x800 bytes) HOLE
+280( 24 mod 256): SKIPPED (no operation)
+281( 25 mod 256): SKIPPED (no operation)
+282( 26 mod 256): MAPREAD 0xa000 thru 0xa5e7 (0x5e8 bytes)
+283( 27 mod 256): SKIPPED (no operation)
+284( 28 mod 256): SKIPPED (no operation)
+285( 29 mod 256): SKIPPED (no operation)
+286( 30 mod 256): SKIPPED (no operation)
+287( 31 mod 256): COLLAPSE 0x11000 thru 0x11fff (0x1000 bytes)
+288( 32 mod 256): COPY 0x5d000 thru 0x5dfff (0x1000 bytes) to 0x4ca00 thru 0x4d9ff
+289( 33 mod 256): TRUNCATE DOWN from 0x75a00 to 0x1e400
+290( 34 mod 256): MAPREAD 0x1c000 thru 0x1d802 (0x1803 bytes) ***RRRR***

ltp/fsx.c +3 -0

+Log of operations saved to "/mnt/xfstests/test/junk.fsxops"; replay with --replay-ops
+Correct content saved for comparison
+(maybe hexdump "/mnt/xfstests/test/junk" vs "/mnt/xfstests/test/junk.fsxgood")

Quoted 5 lines

Quoted 2 lines

Thanks,
Zorro

Hi Zorro, just to confirm is this on an older kernel that doesnt support RWF_ATOMIC or on a kernle that does support it.

I tested on linux 6.16 and current latest linux v6.17+ (will be 6.18-rc1 later). About the RWF_ATOMIC flag in my system:

Quoted 6 lines

# grep -rsn RWF_ATOMIC /usr/include/
/usr/include/bits/uio-ext.h:51:#define RWF_ATOMIC       0x00000040 /* Write is to be issued with torn-write
/usr/include/linux/fs.h:424:#define RWF_ATOMIC  ((__kernel_rwf_t)0x00000040)
/usr/include/linux/fs.h:431:                     RWF_APPEND | RWF_NOAPPEND | RWF_ATOMIC |\
/usr/include/xfs/linux.h:236:#ifndef RWF_ATOMIC
/usr/include/xfs/linux.h:237:#define RWF_ATOMIC ((__kernel_rwf_t)0x00000040)

Hi Zorro, thanks for checking this. So correct me if im wrong but I understand that you have run this test on an atomic writes enabled kernel where the stack also supports atomic writes.

Looking at the bad data log:

	+READ BAD DATA: offset = 0x1c000, size = 0x1803, fname = /mnt/xfstests/test/junk
	+OFFSET      GOOD    BAD     RANGE
	+0x1c000     0x0000  0xcdcd  0x0
	+operation# (mod 256) for the bad data may be 205

We see that 0x0000 was expected but we got 0xcdcd. Now the operation that caused this is indicated to be 205, but looking at that operation:

+205(205 mod 256): ZERO 0x6dbe6 thru 0x6e6aa (0xac5 bytes)

This doesn't even overlap the range that is bad. (0x1c000 to 0x1c00f). Infact, it does seem like an unlikely coincidence that the actual data in the bad range is 0xcdcd which is something xfs_io -c "pwrite" writes to default (fsx writes random data in even offsets and operation num in odd).

I am able to replicate this but only on XFS but not on ext4 (atleast not in 20 runs). I'm trying to better understand if this is a test issue or not. Will keep you update.

Quoted 6 lines

I'm not sure how this will affect the upcoming release, if you want shall I send a small patch to make the atomic writes feature default off instead of default on till we root cause this?

Regards,
Ojaswin

Hi Zorro,

So I'm able to narrow down the opoerations and replicate it via the following replay file:

# -----

# replay.fsxops
# -----
write_atomic 0x57000 0x1000 0x69690
write_atomic 0x66000 0x1000 0x4de00
write_atomic 0x18000 0x1000 0x2c800
copy_range 0x20000 0x1000 0xe00 0x70e00
write_atomic 0x18000 0x1000 0x70e00
copy_range 0x21000 0x1000 0x23000 0x74218
truncate 0x0 0x11200 0x4daef *
write_atomic 0x43000 0x1000 0x11200 *
write_atomic 0x15000 0x1000 0x44000
copy_range 0xd000 0x1000 0x1d800 0x44000
mapread 0x1c000 0x1803 0x1e400 *

Command: ./ltp/fsx -N 10000 -o 8192 -l 500000 -r 4096 -t 512 -w 512 -Z -FKuHzI --replay-ops replay.fsxops $MNT/junk

$MNT/junk is always opened O_TRUNC and is an on an XFS FS where the disk is non-atomic so all RWF_ATOMIC writes are software emulated.

Here are the logs generated for this run:

Seed set to 1
main: filesystem does not support exchange range, disabling!

READ BAD DATA: offset = 0x1c000, size = 0x1803, fname = /mnt/test/junk

OFFSET      GOOD    BAD     RANGE
0x1d000     0x0000  0xf322  0x0

operation# (mod 256) for the bad data may be 243
0x1d001 0x0000 0x22f3 0x1
operation# (mod 256) for the bad data may be 243
0x1d002 0x0000 0xf391 0x2
operation# (mod 256) for the bad data may be 243
0x1d003 0x0000 0x91f3 0x3
<... a few more such lines ..>

LOG DUMP (11 total operations):
openat(AT_FDCWD, "/mnt/test/junk.fsxops", O_WRONLY|O_CREAT|O_TRUNC, 0666) = 7

1(  1 mod 256): WRITE    0x57000 thru 0x57fff   (0x1000 bytes) HOLE     ***WWWW ATOMIC
2(  2 mod 256): WRITE    0x66000 thru 0x66fff   (0x1000 bytes) HOLE ATOMIC
3(  3 mod 256): WRITE    0x18000 thru 0x18fff   (0x1000 bytes) ATOMIC
4(  4 mod 256): COPY 0x20000 thru 0x20fff       (0x1000 bytes) to 0xe00 thru 0x1dff
5(  5 mod 256): WRITE    0x18000 thru 0x18fff   (0x1000 bytes) ATOMIC
6(  6 mod 256): COPY 0x21000 thru 0x21fff       (0x1000 bytes) to 0x23000 thru 0x23fff
7(  7 mod 256): TRUNCATE DOWN   from 0x67000 to 0x11200 ******WWWW
8(  8 mod 256): WRITE    0x43000 thru 0x43fff   (0x1000 bytes) HOLE     ***WWWW ATOMIC
9(  9 mod 256): WRITE    0x15000 thru 0x15fff   (0x1000 bytes) ATOMIC

10( 10 mod 256): COPY 0xd000 thru 0xdfff (0x1000 bytes) to 0x1d800 thru 0x1e7ff
11( 11 mod 256): MAPREAD 0x1c000 thru 0x1d802 (0x1803 bytes) ***RRRR*** Log of operations saved to "/mnt/test/junk.fsxops"; replay with --replay-ops Correct content saved for comparison
(maybe hexdump "/mnt/test/junk" vs "/mnt/test/junk.fsxgood")
+++ exited with 110 +++

We can see that the bad data is detected in the final MAPREAD operation and and bad offset is at 0x1d000. If we look at the operations dump above its clear that none of the operations should be modifying the 0x1d000 so we should have been reading 0s but yet we see some junk data there in the file:

$ hexdump /mnt/test/junk -s 0x1c000 -n0x1020
001c000 0000 0000 0000 0000 0000 0000 0000 0000
*
001d000 22f3 91f3 7ff3 3af3 39f3 23f3 6df3 c2f3
001d010 c5f3 f6f3 a6f3 1ef3 58f3 40f3 32f3 5ff3
001d020

Quoted 2 lines

Another thing to not is that I can't reproduce the above on scsi-debug device. @Darrick, @John, could this be an issue in kernel?

Hi Ojaswin,

Quoted 2 lines

If we can be sure this's a kernel bug, rather than a fstests (patch) issue, I think we can merge this patchset to expose this bug. Does this make sense to you and others?

Looks like a kernel bug to me...

--D

Quoted 19 lines

Thanks,
Zorro

Quoted 14 lines

Regards,
ojaswin

Quoted 10 lines

Quoted 10 lines

Thanks,
Zorro

Quoted 4 lines

Regards,
ojaswin

Quoted 270 lines · Darrick J. Wong

Quoted 268 lines · Zorro Lang

Quoted 260 lines · Ojaswin Mujoo

Quoted 179 lines · Ojaswin Mujoo

Quoted 145 lines · Zorro Lang

Quoted 135 lines · Ojaswin Mujoo

Quoted 34 lines · Zorro Lang

Quoted 24 lines · Ojaswin Mujoo

Quoted 10 lines · Zorro Lang

Quoted 8 lines · Ojaswin Mujoo

Implement atomic write support to help fuzz atomic writes with fsx.

Suggested-by Ritesh Harjani (IBM) <[email protected]>

Reviewed-by Darrick J. Wong <[email protected]>

Reviewed-by John Garry <[email protected]>

Signed-off-by Ojaswin Mujoo <[email protected]>

---

Quoted 4 lines

Hmm... this patch causes more regular fsx test cases fail on old kernel, (e.g. g/760, g/617, g/263 ...) except set "FSX_AVOID=-a". Is there a way to disable "atomic write" automatically if it's not supported by current system?

Hi Zorro,
Sorry for being late, I've been on vacation this week.

Yes so by design we should be automatically disabling atomic writes when they are not supported by the stack but seems like the issue is that when we do disable it we print some extra messages to stdout/err which show up in the xfstests output causing failure.

Quoted 8 lines

I can think of 2 ways around this:

1. Don't print anything and just silently drop atomic writes if stack doesn't support them.

2. Make atomic writes as a default off instead of default on feature but his loses a bit of coverage as existing tests wont get atomic write testing free of cost any more.

Quoted 6 lines

Hi Ojaswin,

Please have a nice vacation :)

It's not the "extra messages" cause failure, those "quiet" failures can be fixed by:

Oh okay got it.

Quoted 37 lines

ltp/fsx.c +6 -4

index bdb87ca90..0a035b37b 100644
@@ -1847,8 +1847,9 @@ int test_atomic_writes(void) {
        struct statx stx;
 
        if (o_direct != O_DIRECT) {
-               fprintf(stderr, "main: atomic writes need O_DIRECT (-Z), "
-                               "disabling!\n");
+               if (!quiet)
+                       fprintf(stderr, "main: atomic writes need O_DIRECT (-Z), "
+                                       "disabling!\n");
                return 0;
        }
 

@@ -1867,8 +1868,9 @@ int test_atomic_writes(void) {
                return 1;
        }
 
-       fprintf(stderr, "main: IO Stack does not support "
-                       "atomic writes, disabling!\n");
+       if (!quiet)
+               fprintf(stderr, "main: IO Stack does not support "
+                               "atomic writes, disabling!\n");
        return 0;
 }
 

But I hit more read or write failures e.g. [1], this failure can't be reproduced with FSX_AVOID=-a. Is it a atomic write bug or an unexpected test failure?

Thanks,
Zorro

<...>

Quoted 51 lines

+244(244 mod 256): SKIPPED (no operation)
+245(245 mod 256): FALLOC 0x695c5 thru 0x6a2e6 (0xd21 bytes) INTERIOR
+246(246 mod 256): MAPWRITE 0x5ac00 thru 0x5b185 (0x586 bytes)
+247(247 mod 256): WRITE 0x31200 thru 0x313ff (0x200 bytes)
+248(248 mod 256): SKIPPED (no operation)
+249(249 mod 256): TRUNCATE DOWN from 0x78242 to 0xf200 ******WWWW
+250(250 mod 256): FALLOC 0x65000 thru 0x66f26 (0x1f26 bytes) PAST_EOF
+251(251 mod 256): WRITE 0x45400 thru 0x467ff (0x1400 bytes) HOLE ***WWWW
+252(252 mod 256): SKIPPED (no operation)
+253(253 mod 256): SKIPPED (no operation)
+254(254 mod 256): MAPWRITE 0x4be00 thru 0x4daee (0x1cef bytes)
+255(255 mod 256): MAPREAD 0xc000 thru 0xcae9 (0xaea bytes)

+256(  0 mod 256): READ     0x3e000 thru 0x3efff	(0x1000 bytes)
+257(  1 mod 256): SKIPPED (no operation)
+258(  2 mod 256): INSERT 0x45000 thru 0x45fff	(0x1000 bytes)
+259(  3 mod 256): ZERO     0x1d7d5 thru 0x1f399	(0x1bc5 bytes)	******ZZZZ
+260(  4 mod 256): TRUNCATE DOWN	from 0x4eaef to 0x11200	******WWWW
+261(  5 mod 256): WRITE    0x43000 thru 0x43fff	(0x1000 bytes) HOLE	***WWWW
+262(  6 mod 256): WRITE    0x2200 thru 0x31ff	(0x1000 bytes)
+263(  7 mod 256): WRITE    0x15000 thru 0x15fff	(0x1000 bytes)
+264(  8 mod 256): WRITE    0x2e400 thru 0x2e7ff	(0x400 bytes)
+265(  9 mod 256): COPY 0xd000 thru 0xdfff	(0x1000 bytes) to 0x1d800 thru 0x1e7ff	******EEEE

+266( 10 mod 256): CLONE 0x2a000 thru 0x2afff (0x1000 bytes) to 0x21000 thru 0x21fff
+267( 11 mod 256): MAPREAD 0x31000 thru 0x31d0a (0xd0b bytes)
+268( 12 mod 256): SKIPPED (no operation)
+269( 13 mod 256): WRITE 0x25000 thru 0x25fff (0x1000 bytes)
+270( 14 mod 256): SKIPPED (no operation)

+271( 15 mod 256): MAPREAD  0x30000 thru 0x30577	(0x578 bytes)
+272( 16 mod 256): PUNCH    0x1a267 thru 0x1c093	(0x1e2d bytes)
+273( 17 mod 256): MAPREAD  0x1f000 thru 0x1f9c9	(0x9ca bytes)
+274( 18 mod 256): WRITE    0x40800 thru 0x40dff	(0x600 bytes)

+275( 19 mod 256): SKIPPED (no operation)
+276( 20 mod 256): MAPWRITE 0x20600 thru 0x22115 (0x1b16 bytes)
+277( 21 mod 256): MAPWRITE 0x3d000 thru 0x3ee5a (0x1e5b bytes)
+278( 22 mod 256): WRITE 0x2ee00 thru 0x2efff (0x200 bytes)
+279( 23 mod 256): WRITE 0x76200 thru 0x769ff (0x800 bytes) HOLE
+280( 24 mod 256): SKIPPED (no operation)
+281( 25 mod 256): SKIPPED (no operation)
+282( 26 mod 256): MAPREAD 0xa000 thru 0xa5e7 (0x5e8 bytes)
+283( 27 mod 256): SKIPPED (no operation)
+284( 28 mod 256): SKIPPED (no operation)
+285( 29 mod 256): SKIPPED (no operation)
+286( 30 mod 256): SKIPPED (no operation)
+287( 31 mod 256): COLLAPSE 0x11000 thru 0x11fff (0x1000 bytes)
+288( 32 mod 256): COPY 0x5d000 thru 0x5dfff (0x1000 bytes) to 0x4ca00 thru 0x4d9ff
+289( 33 mod 256): TRUNCATE DOWN from 0x75a00 to 0x1e400
+290( 34 mod 256): MAPREAD 0x1c000 thru 0x1d802 (0x1803 bytes) ***RRRR***

ltp/fsx.c +3 -0

+Log of operations saved to "/mnt/xfstests/test/junk.fsxops"; replay with --replay-ops
+Correct content saved for comparison
+(maybe hexdump "/mnt/xfstests/test/junk" vs "/mnt/xfstests/test/junk.fsxgood")

Quoted 5 lines

Quoted 2 lines

Thanks,
Zorro

Hi Zorro, just to confirm is this on an older kernel that doesnt support RWF_ATOMIC or on a kernle that does support it.

I tested on linux 6.16 and current latest linux v6.17+ (will be 6.18-rc1 later). About the RWF_ATOMIC flag in my system:

Quoted 6 lines

# grep -rsn RWF_ATOMIC /usr/include/
/usr/include/bits/uio-ext.h:51:#define RWF_ATOMIC       0x00000040 /* Write is to be issued with torn-write
/usr/include/linux/fs.h:424:#define RWF_ATOMIC  ((__kernel_rwf_t)0x00000040)
/usr/include/linux/fs.h:431:                     RWF_APPEND | RWF_NOAPPEND | RWF_ATOMIC |\
/usr/include/xfs/linux.h:236:#ifndef RWF_ATOMIC
/usr/include/xfs/linux.h:237:#define RWF_ATOMIC ((__kernel_rwf_t)0x00000040)

Hi Zorro, thanks for checking this. So correct me if im wrong but I understand that you have run this test on an atomic writes enabled kernel where the stack also supports atomic writes.

Looking at the bad data log:

	+READ BAD DATA: offset = 0x1c000, size = 0x1803, fname = /mnt/xfstests/test/junk
	+OFFSET      GOOD    BAD     RANGE
	+0x1c000     0x0000  0xcdcd  0x0
	+operation# (mod 256) for the bad data may be 205

We see that 0x0000 was expected but we got 0xcdcd. Now the operation that caused this is indicated to be 205, but looking at that operation:

+205(205 mod 256): ZERO 0x6dbe6 thru 0x6e6aa (0xac5 bytes)

This doesn't even overlap the range that is bad. (0x1c000 to 0x1c00f). Infact, it does seem like an unlikely coincidence that the actual data in the bad range is 0xcdcd which is something xfs_io -c "pwrite" writes to default (fsx writes random data in even offsets and operation num in odd).

I am able to replicate this but only on XFS but not on ext4 (atleast not in 20 runs). I'm trying to better understand if this is a test issue or not. Will keep you update.

Quoted 6 lines

I'm not sure how this will affect the upcoming release, if you want shall I send a small patch to make the atomic writes feature default off instead of default on till we root cause this?

Regards,
Ojaswin

Hi Zorro,

So I'm able to narrow down the opoerations and replicate it via the following replay file:

# -----

# replay.fsxops
# -----
write_atomic 0x57000 0x1000 0x69690
write_atomic 0x66000 0x1000 0x4de00
write_atomic 0x18000 0x1000 0x2c800
copy_range 0x20000 0x1000 0xe00 0x70e00
write_atomic 0x18000 0x1000 0x70e00
copy_range 0x21000 0x1000 0x23000 0x74218
truncate 0x0 0x11200 0x4daef *
write_atomic 0x43000 0x1000 0x11200 *
write_atomic 0x15000 0x1000 0x44000
copy_range 0xd000 0x1000 0x1d800 0x44000
mapread 0x1c000 0x1803 0x1e400 *

Command: ./ltp/fsx -N 10000 -o 8192 -l 500000 -r 4096 -t 512 -w 512 -Z -FKuHzI --replay-ops replay.fsxops $MNT/junk

$MNT/junk is always opened O_TRUNC and is an on an XFS FS where the disk is non-atomic so all RWF_ATOMIC writes are software emulated.

Here are the logs generated for this run:

Seed set to 1
main: filesystem does not support exchange range, disabling!

READ BAD DATA: offset = 0x1c000, size = 0x1803, fname = /mnt/test/junk

OFFSET      GOOD    BAD     RANGE
0x1d000     0x0000  0xf322  0x0

operation# (mod 256) for the bad data may be 243
0x1d001 0x0000 0x22f3 0x1
operation# (mod 256) for the bad data may be 243
0x1d002 0x0000 0xf391 0x2
operation# (mod 256) for the bad data may be 243
0x1d003 0x0000 0x91f3 0x3
<... a few more such lines ..>

LOG DUMP (11 total operations):
openat(AT_FDCWD, "/mnt/test/junk.fsxops", O_WRONLY|O_CREAT|O_TRUNC, 0666) = 7

1(  1 mod 256): WRITE    0x57000 thru 0x57fff   (0x1000 bytes) HOLE     ***WWWW ATOMIC
2(  2 mod 256): WRITE    0x66000 thru 0x66fff   (0x1000 bytes) HOLE ATOMIC
3(  3 mod 256): WRITE    0x18000 thru 0x18fff   (0x1000 bytes) ATOMIC
4(  4 mod 256): COPY 0x20000 thru 0x20fff       (0x1000 bytes) to 0xe00 thru 0x1dff
5(  5 mod 256): WRITE    0x18000 thru 0x18fff   (0x1000 bytes) ATOMIC
6(  6 mod 256): COPY 0x21000 thru 0x21fff       (0x1000 bytes) to 0x23000 thru 0x23fff
7(  7 mod 256): TRUNCATE DOWN   from 0x67000 to 0x11200 ******WWWW
8(  8 mod 256): WRITE    0x43000 thru 0x43fff   (0x1000 bytes) HOLE     ***WWWW ATOMIC
9(  9 mod 256): WRITE    0x15000 thru 0x15fff   (0x1000 bytes) ATOMIC

10( 10 mod 256): COPY 0xd000 thru 0xdfff (0x1000 bytes) to 0x1d800 thru 0x1e7ff
11( 11 mod 256): MAPREAD 0x1c000 thru 0x1d802 (0x1803 bytes) ***RRRR*** Log of operations saved to "/mnt/test/junk.fsxops"; replay with --replay-ops Correct content saved for comparison
(maybe hexdump "/mnt/test/junk" vs "/mnt/test/junk.fsxgood")
+++ exited with 110 +++

We can see that the bad data is detected in the final MAPREAD operation and and bad offset is at 0x1d000. If we look at the operations dump above its clear that none of the operations should be modifying the 0x1d000 so we should have been reading 0s but yet we see some junk data there in the file:

$ hexdump /mnt/test/junk -s 0x1c000 -n0x1020
001c000 0000 0000 0000 0000 0000 0000 0000 0000
*
001d000 22f3 91f3 7ff3 3af3 39f3 23f3 6df3 c2f3
001d010 c5f3 f6f3 a6f3 1ef3 58f3 40f3 32f3 5ff3
001d020

Quoted 2 lines

Another thing to not is that I can't reproduce the above on scsi-debug device. @Darrick, @John, could this be an issue in kernel?

Hi Ojaswin,

If we can be sure this's a kernel bug, rather than a fstests (patch) issue, I think we can merge this patchset to expose this bug. Does this make sense to you and others?