Skip to contentSkip to conversation

writes not always synchronous on JFS with O_SYNC?

CHECKER Ben Pfaff 2 participants 4 messages Mar 21, 2005 – Mar 22, 2005

lore
Share & tools
lore

Ben Pfaff

author
#0 lore raw

Hi. We've been running some tests on JFS and other file systems and believe we've found an issue whereby O_SYNC does not always cause data to be committed synchronously. On Linux 2.6.11, we found that the program appended below causes /mnt/sbd0/0006/0010/0029/0033 to contain all zeros, despite the use of O_SYNC on the write calls and the fsyncs on the directories. It seems to be pretty sensitive to the existence of the rmdir calls: if I omit one of them, the data does get written.

Note that /dev/sbd0 is essentially a ramdisk that we've developed for testing this kind of thing: it allows a snapshot of a disk's momentary contents to be copied out, so that we don't have to do a reboot.

Can you confirm this?

Thanks,

Ben.

----------------------------------------------------------------------

#define _GNU_SOURCE
#include <sys/types.h>
#include <sys/ioctl.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <stdarg.h>
#include <assert.h>
#include <unistd.h>

#define CHECK(ret) if(ret < 0) {perror(0); assert(0);} #define MAX_SIG_WRITE (20)
typedef unsigned int signature_t;

typedef struct write_s {
	off_t off;
	size_t len;
	char buf[MAX_SIG_WRITE * sizeof(signature_t)];
	off_t size; // file size;
	signature_t sig; // file sig;
} write_t;
write_t writes[] = {
	{0, 4,{-23, -69, 101, -119, }, 4, 228307655},
	{1023, 8,{63, -3, -104, -45, 4, -110, -59, 88, }, 1031, 4168021214},
	{4093, 12,{98, 11, -106, 98, 79, -107, -88, 102, 1, 100, -6, 85, }, 4105, 2386671457},
	{32755, 20,{-83, 49, -128, -96, 90, -7, 13, 69, 62, 127, -99, 125, 7, 53, -70, 62, -113, 63, 72, -104, }, 32775, 2170096179},
	{1073741793, 40,{-69, -57, -42, 80, -70, 54, 127, 17, -113, 34, 72, 31, 27, 5, -28, -18, -73, -79, -1, -52, -106, 8, 86, 100, 72, 28, 98, -27, -52, -104, -65, 3, 56, 51, 68, -66, 65, -87, 62, 51, }, 1073741833, 255668987},
};

int systemf(const char *fmt, ...)

{
	static char cmd[1024];

	va_list ap;
	va_start(ap, fmt);
	vsprintf(cmd, fmt, ap);
	va_end(ap);
	
	fprintf(stderr, "running cmd \"%s\"\n", cmd);
	return system(cmd);
}

int test_write(const char *pathname, unsigned i)

{
	int fd = open(pathname, O_WRONLY | O_SYNC);
	CHECK(fd);
	int ret = pwrite(fd, writes[i].buf, 
			 writes[i].len, writes[i].off);
	CHECK(ret);
	ret = close(fd);
	CHECK(ret);
	return ret;
}

void test_fsync(const char* pathname)

{
	int fd, ret;
	fd = open(pathname, 0);
	CHECK(fd);
	ret = fsync(fd);
	CHECK(ret);
	ret = close(fd);
	CHECK(ret);
}

void test_creat(const char* pathname, mode_t m)

{
	int fd, ret;
	fd = creat(pathname, m);
	CHECK(fd);
	ret = close(fd);
	CHECK(ret);
}

int main(int argc, char *argv[])

{
	int ret, fd;
	systemf("umount /dev/sbd0");
	systemf("sbin/mkfs.jfs -q /dev/sbd0 4096");
	systemf("sbin/fsck.jfs -a -p /dev/sbd0");
	systemf("mount -t jfs /dev/sbd0 /mnt/sbd0 ");
	systemf("umount /dev/sbd0");
	systemf("mount -t jfs /dev/sbd0 /mnt/sbd0 ");
	
	ret = mkdir("/mnt/sbd0/0006", 511);
	CHECK(ret);
	ret = mkdir("/mnt/sbd0/0006/0010", 511);
	CHECK(ret);
	ret = mkdir("/mnt/sbd0/0014", 511);
	CHECK(ret);
	ret = mkdir("/mnt/sbd0/0015", 511);
	CHECK(ret);
	ret = mkdir("/mnt/sbd0/0015/0016", 511);
	CHECK(ret);
	ret = mkdir("/mnt/sbd0/0006/0010/0017", 511);
	CHECK(ret);
	ret = rmdir("/mnt/sbd0/0015/0016");
	CHECK(ret);
	ret = mkdir("/mnt/sbd0/0006/0010/0017/0021", 511);
	CHECK(ret);
	ret = rmdir("/mnt/sbd0/0014");
	CHECK(ret);
	ret = rmdir("/mnt/sbd0/0015");
	CHECK(ret);
	ret = test_creat("/mnt/sbd0/0006/0028", 511);
	CHECK(ret);
	ret = test_write("/mnt/sbd0/0006/0028", 0);
	CHECK(ret);
	ret = mkdir("/mnt/sbd0/0006/0010/0029", 511);
	CHECK(ret);
	ret = test_creat("/mnt/sbd0/0006/0010/0029/0033", 511);
	CHECK(ret);
	ret = test_write("/mnt/sbd0/0006/0010/0029/0033", 0);
	CHECK(ret);
	ret = test_fsync("/mnt/sbd0/0006/0010/0029");
	CHECK(ret);
	ret = test_fsync("/mnt/sbd0/0006/0010");
	CHECK(ret);
	ret = test_fsync("/mnt/sbd0/0006");
	CHECK(ret);
	ret = test_fsync("/mnt/sbd0");
	CHECK(ret);
#if 0
	{
		#include "../sbd/sbd.h"
		int sbd_fd = open("/dev/sbd0", O_RDONLY);
		ret = ioctl(sbd_fd, SBD_COPY_DISK, 1);
		CHECK(ret);
		close(sbd_fd);
	}
#else
	systemf("reboot -f -n");
#endif
	return 0;
}
Signature · 3 lines
Ben Pfaff 
email: [email protected]
web: http://benpfaff.org
#1 lore raw
Quoted 9 lines · Ben Pfaff

Hi. We've been running some tests on JFS and other file systems and believe we've found an issue whereby O_SYNC does not always cause data to be committed synchronously. On Linux 2.6.11, we found that the program appended below causes /mnt/sbd0/0006/0010/0029/0033 to contain all zeros, despite the use of O_SYNC on the write calls and the fsyncs on the directories. It seems to be pretty sensitive to the existence of the rmdir calls: if I omit one of them, the data does get written.

I can reproduce this behavior, but what I'm seeing is a little more alarming than the data being written asynchronously. After fsck replays the journal, the xtree for the file is corrupt, which is why it appears to contain zeros. The syslog is showing that a corrupted xtree has been found. I am still investigating.

Quoted 4 lines

Note that /dev/sbd0 is essentially a ramdisk that we've developed for testing this kind of thing: it allows a snapshot of a disk's momentary contents to be copied out, so that we don't have to do a reboot.

I'm getting similar results writing to a regular disk partition and rebooting.

Quoted 1 line

Can you confirm this?

Confirmed.

Quoted 1 line
	ret = test_creat("/mnt/sbd0/0006/0028", 511);

test_creat is type void.

Quoted 10 lines
	CHECK(ret);
	ret = test_write("/mnt/sbd0/0006/0028", 0);
	CHECK(ret);
	ret = mkdir("/mnt/sbd0/0006/0010/0029", 511);
	CHECK(ret);
	ret = test_creat("/mnt/sbd0/0006/0010/0029/0033", 511);
	CHECK(ret);
	ret = test_write("/mnt/sbd0/0006/0010/0029/0033", 0);
	CHECK(ret);
	ret = test_fsync("/mnt/sbd0/0006/0010/0029");

so is test_fsync

Quoted 1 line
	CHECK(ret);
Signature · 2 lines
David Kleikamp
IBM Linux Technology Center
#2 lore raw
Quoted 10 lines · Dave Kleikamp
Quoted 9 lines · Ben Pfaff

Hi. We've been running some tests on JFS and other file systems and believe we've found an issue whereby O_SYNC does not always cause data to be committed synchronously. On Linux 2.6.11, we found that the program appended below causes /mnt/sbd0/0006/0010/0029/0033 to contain all zeros, despite the use of O_SYNC on the write calls and the fsyncs on the directories. It seems to be pretty sensitive to the existence of the rmdir calls: if I omit one of them, the data does get written.

The problem is in the journal-replay code and it is triggered by the reuse of a directory inode by a regular file inode, so reproducing the bug requires the journal containing both changes to the inode as a directory and then as a file.

This patch to jfs_fsck fixes the problem. It wasn't really an issue with O_SYNC at all, although I believe there are O_SYNC issues that need to be resolved. The data itself should be sync'd correctly, as generic_file_write checks for the O_SYNC flag. The missing piece in jfs is that metadata changes to the inode may not always be making it to the disk when they should.

Signature · 2 lines
David Kleikamp
IBM Linux Technology Center

Index: jfsutils/libfs/log_work.c
===================================================================
RCS file: /cvsroot/jfs/jfsutils/libfs/log_work.c,v retrieving revision 1.21

libfs/log_work.c +11 -1
diff -u -p -r1.21 log_work.c
@@ -1,5 +1,5 @@
 /*
- *   Copyright (c) International Business Machines Corp., 2000-2002
+ *   Copyright (C) International Business Machines Corp., 2000-2005
  *
  *   This program is free software;  you can redistribute it and/or modify
  *   it under the terms of the GNU General Public License as published by
@@ -2758,6 +2758,11 @@ int updatePage(struct lrd *ld, int32_t l
 						off += 1;
 						data += linesize;
 						seglen -= linesize;
+						/*
+						 * i_data overlaps btroot.
+						 * Strip of 32 bytes
+						 */
+						seglen -= 32;
 					} else if (db->db_idata & mask_8) {
 						/* Only update ibase */
 						db->db_ibase |= mask_8;
@@ -2766,6 +2771,11 @@ int updatePage(struct lrd *ld, int32_t l
 						/* update both */
 						db->db_ibase |= mask_8;
 						db->db_idata |= mask_8;
+						/*
+						 * i_data overlaps btroot.
+						 * Strip of 32 bytes
+						 */
+						seglen -= 32;
 					}
 				}
 			} else if (ino_rem == 1) {	/* inline data */
author
#3 lore raw
Quoted 6 lines · Dave Kleikamp

This patch to jfs_fsck fixes the problem. It wasn't really an issue with O_SYNC at all, although I believe there are O_SYNC issues that need to be resolved. The data itself should be sync'd correctly, as generic_file_write checks for the O_SYNC flag. The missing piece in jfs is that metadata changes to the inode may not always be making it to the disk when they should.

Wow, that's an amazingly fast fix. Thanks! If I have time today, I'll re-run the checker after applying your patch.

Signature · 3 lines
Ben Pfaff 
email: [email protected]
web: http://benpfaff.org