Skip to contentSkip to conversation

xfstests: Introduce a new SEEK_DATA/SEEK_HOLE tester

PATCH In review Jeff Liu 2 participants 3 messages Dec 29, 2011 – Feb 2, 2012

lore
Share & tools
lore

Jeff Liu

author
#0 lore raw

PATCH 1/1 xfstests: Introduce a new SEEK_DATA/SEEK_HOLE tester

Hello,

This is another SEEK_DATA/SEEK_HOLE tester which is intended to cover multiple extents checking.
I have ran it against btrfs to ensure the tester works, and ran it against XFS to ensure the SEEK_DATA/SEEK_HOLE patch works too.

There have four test cases in xfstest-277.
1. test file created in preallocation mode with dirty pages.
2. test file created in preallocation mode with writeback pages.
3. test file created as normal, but make holes with pwrite(2), full with zeros(data extent) at the end of file.
4. test file created as normal, same as 3, but make holes at the end of file.

Any feekback are appreciated!

---
 277                    |  165 ++++++++++++
 group                  |    1 +
 src/Makefile           |    3 +-
 src/seek_copy_tester.c |  674 ++++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 842 insertions(+), 1 deletions(-)
 create mode 100755 277
 create mode 100755 src/seek_copy_tester.c
277 new file +165 -0
new file mode 100755
index 0000000..04eb32d
@@ -0,0 +1,165 @@
+#! /bin/bash
+# FS QA Test No. 276
+#
+# SEEK_DATA/SEEK_HOLE tester.
+# Create a sparse file with a couple of holes and data extents.
+# Simulates the copy process to verify SEEK_DATA/SEEK_HOLE stuff works
+# as expected.
+#
+#-----------------------------------------------------------------------
+# Copyright (c) 2011 Oracle Inc.  All Rights Reserved.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it would be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write the Free Software Foundation,
+# Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#
+#-----------------------------------------------------------------------
+#
+# creator
[email protected]
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+fail=0
+status=0	# success is the default!
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+# real QA test starts here
+_supported_fs generic
+_supported_os Linux Solaris
+
+src=$TEST_DIR/seek_copy_testfile
+dest=$TEST_DIR/seek_copy_testfile.dest
+
+[ -x $here/src/seek_copy_tester ] || _notrun "seek_copy_tester not built"
+
+_cleanup()
+{
+	rm -f $src $dest
+}
+
+# seek_copy_test_01()
+# create a 100Mytes file in preallocation mode.
+# fallocate offset start from 0.
+# the first data extent offset start from 80991, write 4Kbytes,
+# and then skip 195001 bytes for next write.
+# this is intended to test data buffer lookup for DIRTY pages.
+# verify results:
+# 1. file size is identical.
+# 2. perform cmp(1) to compare SRC and DEST file byte by byte.
+test01()
+{
+	rm -f $src $dest
+
+	$here/src/seek_copy_tester -P -O 0 -L 100m -s 80991 -k 195001 -l 4k $src $dest
+
+	test $(stat --printf "%s" $src) = $(stat --printf "%s" $dest) ||
+		echo "FAIL_TEST01: file size no equal" >> $seq.out    &&
+		fail=1
+	cmp $src $dest | tee -a $seq.out
+	cmp $src $dest						      ||
+		echo "FAIL_TEST01: file compare failed" >> $seq.out   &&
+		fail=1
+	test $fail -eq 1 && status=1
+}
+
+# seek_copy_test_02()
+# create a 100Mytes file in preallocation mode.
+# fallocate offset start from 0.
+# the first data extent offset start from 0, write 16Kbytes,
+# and then skip 8Mbytes for next write.
+# Try flushing DIRTY pages to WRITEBACK mode, this is intended to
+# test data buffer lookup in WRITEBACK pages.
+# verify results:
+# 1. file size is identical.
+# 2. perform cmp(1) to compare SRC and DEST file byte by byte.
+test02()
+{
+	rm -rf $src $dest
+	fail=0
+
+	$here/src/seek_copy_tester -S -P -O 0 -L 100m -s 0 -k 8m -l 16k $src $dest
+
+	test $(stat --printf "%s" $src) = $(stat --printf "%s" $dest) ||
+		echo "FAIL_TEST02: file size no equal" >> $seq.out    &&
+		fail=1
+	cmp $src $dest | tee -a $seq.out
+	cmp $src $dest						      ||
+		echo "FAIL_TEST02: file compare failed" >>$seq.out    &&
+		fail=1
+	test $fail -eq 1 && status=1
+}
+
+# seek_copy_test_03
+# create a 100Mytes sparse file.
+# the first data extent offset start from 512, write 4Kbytes,
+# and then skip 1Mbytes for next write.
+# don't make holes at the end of file.
+# verify results:
+# 1. file size is identical.
+# 2. perform cmp(1) to compare SRC and DEST file byte by byte.
+test03()
+{
+	rm -f $src $dest
+	fail=0
+
+	$here/src/seek_copy_tester -M 100m -s 512 -k 1m -l 4k $src $dest
+
+	test $(stat --printf "%s" $src) = $(stat --printf "%s" $dest) ||
+		echo "FAIL_TEST03: file size no equal" >> $seq.out    &&
+		fail=1
+	cmp $src $dest | tee -a $seq.out
+	cmp $src $dest						      ||
+		echo "FAIL_TEST03: file compare failed" >>$seq.out    &&
+		fail=1
+	test $fail -eq 1 && status=1
+}
+
+# seek_copy_test_04
+# create a 1Gbytes sparse file.
+# the first data extent offset start from 512, write 4Kbytes,
+# and then skip 1Mbytes for next write.
+# make holes at the end of file.
+# verify results:
+# 1. file size is identical.
+# 2. perform cmp(1) to compare SRC and DEST file byte by byte.
+test04()
+{
+	rm -f $src $dest
+	fail=0
+
+	$here/src/seek_copy_tester -M 1g -s 512 -k 1m -l 4k -E $src $dest
+
+	test $(stat --printf "%s" $src) = $(stat --printf "%s" $dest) ||
+		echo "FAIL_TEST04: file size no equal" >> $seq.out    &&
+		fail=1
+	cmp $src $dest | tee -a $seq.out
+	cmp $src $dest						      ||
+		echo "FAIL_TEST04: file compare failed" >>$seq.out    &&
+		fail=1
+	test $fail -eq 1 && status=1
+}
+echo "QA output created by $seq" >> $seq.out
+
+test01
+test02
+test03
+test04
+
+status=0
+exit
group +1 -0
index be4bd9b..5228abe 100644
@@ -390,3 +390,4 @@ deprecated
 274 auto rw
 275 auto rw
 276 auto quick
+277 auto rw
src/Makefile +2 -1
index 6797064..4a24d3c 100644
@@ -17,7 +17,8 @@ LINUX_TARGETS = xfsctl bstat t_mtab getdevicesize preallo_rw_pattern_reader \
 	preallo_rw_pattern_writer ftrunc trunc fs_perms testx looptest \
 	locktest unwritten_mmap bulkstat_unlink_test t_stripealign \
 	bulkstat_unlink_test_modified t_dir_offset t_futimens t_immutable \
-	stale_handle pwrite_mmap_blocked fstrim t_dir_offset2 seek_sanity_tester
+	stale_handle pwrite_mmap_blocked fstrim t_dir_offset2 seek_sanity_tester \
+	seek_copy_tester
 
 SUBDIRS =
 
src/seek_copy_tester.c new file +674 -0

674 lines not on this page · raw message

Signature · 6 lines
1.7.4.1

_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs
#1 lore raw

src/seek_copy_tester.c

Quoted 12 lines · Jeff Liu

Hello,

This is another SEEK_DATA/SEEK_HOLE tester which is intended to cover multiple extents checking.
I have ran it against btrfs to ensure the tester works, and ran it against XFS to ensure the SEEK_DATA/SEEK_HOLE patch works too.

src/seek_copy_tester.c new file +0 -0
new file mode 100755
index 0000000..4971f34
@@ -0,0 +1,674 @@

Do you want to add Author/Copyright and description?

Quoted 4 lines
src/seek_copy_tester.c new file +4 -0
+#include<stdio.h>
+#include<stdlib.h>
+#include<stdint.h>
+#include<stdarg.h>

...

Quoted 8 lines
src/seek_copy_tester.c new file +8 -0
+int
+full_write(int fd, const void *buf, size_t count)
+{
+	int ret = 0;
+	const char *ptr = (const char *) buf;
+
+	while (count>  0) {
+		ssize_t n = write(fd, ptr, count);

src/seek_copy_tester.c line 101

Quoted 10 lines
src/seek_copy_tester.c new file +10 -0
+		if (n<  0) {
+			if (errno == EINTR)
+				continue;
+			error("full_write failed as %s", strerror(errno));
+			ret = -1;
+			break;
+		}
+
+		if (n == 0)
+			break;

Callers of this routine expect the count number of bytes to be written. Write a message if leaving this routine early? An error?

Quoted 7 lines
src/seek_copy_tester.c new file +7 -0
+
+		ptr += n;
+		count -= n;
+	}
+
+	return ret;
+}

...

Quoted 6 lines
src/seek_copy_tester.c new file +6 -0
+int
+create_data_and_holes(int fd, size_t nr_total_bytes, off_t start_offset,
+		      uint64_t nr_skip_bytes, uint64_t nr_data_bytes,
+		      int wrote_hole_at_eof)
+{
+	int ret = 0;

src/seek_copy_tester.c line 215

Quoted 10 lines
src/seek_copy_tester.c new file +10 -0
+	off_t total = nr_total_bytes;
+	off_t data_len = nr_data_bytes;
+	off_t off = start_offset;
+	char buf[4096];
+
+	memset(buf, 'A', sizeof(buf));
+
+	total -= start_offset;
+	while (total>  0) {
+		do {

You can actually write more than total byte on the last data write. If writing exact total is important, then give do_pwrite() the count:
cnt = MIN(total, sizeof(buf))

src/seek_copy_tester.c line 224

Quoted 9 lines
src/seek_copy_tester.c new file +9 -0
+			ssize_t nr_write = do_pwrite(fd, buf, sizeof(buf), off);
+			if (nr_write<  0) {
+				error("do_pwrite() failed as %s", strerror(errno));
+				ret = -1;
+				goto out;
+			}
+			if (nr_write == 0)
+				break;
+

do_pwrite will return 0 if not an error.

src/seek_copy_tester.c line 226

Quoted 2 lines
src/seek_copy_tester.c new file +2 -0
+			off += nr_write;
+			data_len -= nr_write;

These are probably sizeof(buf0 or my cnt not nr_write

Quoted 4 lines
src/seek_copy_tester.c new file +4 -0
+		} while (data_len>  0);
+
+		off += (nr_skip_bytes + nr_data_bytes);
+		total -= off;

...

Quoted 34 lines
src/seek_copy_tester.c new file +34 -0
+
+/*
+ * Copy a data extent from source file to dest file.
+ * @data_off: data offset
+ * @hole_off: hole offset
+ * The length of this extent is (hole_off - data_off).
+ */
+int
+do_extent_copy(int src_fd, int dest_fd, off_t data_off, off_t hole_off)
+{
+	uint64_t len = (uint64_t)(hole_off - data_off);
+	char buf[BUF_SIZE];
+	int ret;
+
+	/* Seek to data_off for data reading */
+	ret = lseek(src_fd, data_off, SEEK_SET);
+	if (ret<  0) {
+		error("seek source file to %llu failed as %s",
+		       (uint64_t)data_off, strerror(errno));
+		return ret;
+	}
+
+	/* Seek to data_off for data writing, make holes as well */
+	ret = lseek(dest_fd, data_off, SEEK_SET);
+	if (ret<  0) {
+		error("seek dest file to %llu failed as %s",
+		       (uint64_t)data_off, strerror(errno));
+		return ret;
+	}
+
+	while (len>  0) {
+		memset(buf, 0, sizeof(buf));
+		ssize_t n_read = read(src_fd, buf, BUF_SIZE);
+		if (n_read<  0) {

src/seek_copy_tester.c line 293

Quoted 10 lines
src/seek_copy_tester.c new file +10 -0
+			if (errno == EINTR)
+				continue;
+
+			error("read source file extent failed as %s",
+			      strerror(errno));	
+			return n_read;
+		}
+
+		if (n_read == 0)
+			break;

Message? Error?

--Mark Tinguely

_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs

author
#2 lore raw

Hi Mark,

Thanks for your review! My response is inline below.

Quoted 9 lines · Mark Tinguely
Quoted 7 lines · Jeff Liu

Hello,

This is another SEEK_DATA/SEEK_HOLE tester which is intended to cover multiple extents checking.
I have ran it against btrfs to ensure the tester works, and ran it against XFS to ensure the SEEK_DATA/SEEK_HOLE patch works too.

Quoted 8 lines

src/seek_copy_tester.c

Quoted 6 lines
src/seek_copy_tester.c new file +0 -0
new file mode 100755
index 0000000..4971f34
@@ -0,0 +1,674 @@

Do you want to add Author/Copyright and description?

Sure. :)

Quoted 21 lines
Quoted 5 lines
src/seek_copy_tester.c new file +4 -0
+#include<stdio.h>
+#include<stdlib.h>
+#include<stdint.h>
+#include<stdarg.h>

...

Quoted 13 lines
src/seek_copy_tester.c new file +13 -0
+int
+full_write(int fd, const void *buf, size_t count)
+{
+    int ret = 0;
+    const char *ptr = (const char *) buf;
+
+    while (count>  0) {
+        ssize_t n = write(fd, ptr, count);
+        if (n<  0) {
+            if (errno == EINTR)
+                continue;
+            error("full_write failed as %s", strerror(errno));
+            ret = -1;
Quoted 8 lines

src/seek_copy_tester.c line 101

Quoted 5 lines
src/seek_copy_tester.c new file +5 -0
+            break;
+        }
+
+        if (n == 0)
+            break;

Callers of this routine expect the count number of bytes to be written. Write a message if leaving this routine early? An error?

It's prone to be an error if nothing was written, an it's better to print out an error message here.

Also, I am prefer to revise full_write() to return the number of bytes actually wrote done. i.e, size_t full_write().
The caller can simply comparing the return value for each write by:

if (full_write(fd, buf, count) != count) {
	error();
	...
}
Quoted 42 lines
Quoted 8 lines
src/seek_copy_tester.c new file +7 -0
+
+        ptr += n;
+        count -= n;
+    }
+
+    return ret;
+}

...

Quoted 6 lines
src/seek_copy_tester.c new file +6 -0
+int
+create_data_and_holes(int fd, size_t nr_total_bytes, off_t start_offset,
+              uint64_t nr_skip_bytes, uint64_t nr_data_bytes,
+              int wrote_hole_at_eof)
+{
+    int ret = 0;

src/seek_copy_tester.c line 215

Quoted 10 lines
src/seek_copy_tester.c new file +10 -0
+    off_t total = nr_total_bytes;
+    off_t data_len = nr_data_bytes;
+    off_t off = start_offset;
+    char buf[4096];
+
+    memset(buf, 'A', sizeof(buf));
+
+    total -= start_offset;
+    while (total>  0) {
+        do {

You can actually write more than total byte on the last data write. If writing exact total is important, then give do_pwrite() the count:
cnt = MIN(total, sizeof(buf))

src/seek_copy_tester.c line 224

Quoted 9 lines
src/seek_copy_tester.c new file +9 -0
+            ssize_t nr_write = do_pwrite(fd, buf, sizeof(buf), off);
+            if (nr_write<  0) {
+                error("do_pwrite() failed as %s", strerror(errno));
+                ret = -1;
+                goto out;
+            }
+            if (nr_write == 0)
+                break;
+
Quoted 1 line

do_pwrite will return 0 if not an error.

To simplify error checking, I'd like to change "ssize_t do_pwrite()" to "size_t full_pwrite()"; let it return the number of wrote bytes same as full_write(), and print out an error message if the return value is not equal to the desired(sizeof(buf)).

Quoted 3 lines
Quoted 2 lines
+            off += nr_write;
            data_len -= nr_write;

These are probably sizeof(buf0 or my cnt not nr_write

With above modification, nr_write can be replaced by sizeof(buf). Ah, I just realized that I should use BUF_SIZE macro here.

Quoted 45 lines
Quoted 4 lines
src/seek_copy_tester.c new file +4 -0
+        } while (data_len>  0);
+
+        off += (nr_skip_bytes + nr_data_bytes);
+        total -= off;

...

Quoted 38 lines
src/seek_copy_tester.c new file +38 -0
+
+/*
+ * Copy a data extent from source file to dest file.
+ * @data_off: data offset
+ * @hole_off: hole offset
+ * The length of this extent is (hole_off - data_off).
+ */
+int
+do_extent_copy(int src_fd, int dest_fd, off_t data_off, off_t hole_off)
+{
+    uint64_t len = (uint64_t)(hole_off - data_off);
+    char buf[BUF_SIZE];
+    int ret;
+
+    /* Seek to data_off for data reading */
+    ret = lseek(src_fd, data_off, SEEK_SET);
+    if (ret<  0) {
+        error("seek source file to %llu failed as %s",
+               (uint64_t)data_off, strerror(errno));
+        return ret;
+    }
+
+    /* Seek to data_off for data writing, make holes as well */
+    ret = lseek(dest_fd, data_off, SEEK_SET);
+    if (ret<  0) {
+        error("seek dest file to %llu failed as %s",
+               (uint64_t)data_off, strerror(errno));
+        return ret;
+    }
+
+    while (len>  0) {
+        memset(buf, 0, sizeof(buf));
+        ssize_t n_read = read(src_fd, buf, BUF_SIZE);
+        if (n_read<  0) {
+            if (errno == EINTR)
+                continue;
+
+            error("read source file extent failed as %s",
Quoted 8 lines

src/seek_copy_tester.c line 293

Quoted 6 lines
src/seek_copy_tester.c new file +6 -0
+                  strerror(errno));   
+            return n_read;
+        }
+
+        if (n_read == 0)
+            break;

Message? Error?

Hmm, not an error. maybe drop a message when read hit EOF is useful for debugging purpose.

Thanks,
-Jeff

Quoted 7 lines

--Mark Tinguely

_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs

_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs