: filelock1: Test restore of adivsory locks during restart

This test currently fails during restart on ckpt-v19-rc2.

On Serge's cr-next it fails cleanly during checkpoint due to:

	commit 5d1f1227384876dd13a66cad1f286d98f9b1891b
	Author: Dave Hansen <[email protected]>
	Date:   Thu Dec 17 09:35:13 2009 -0800

	    ckpt-files: error out on file locks and leases

---
From: Sukadev Bhattiprolu <[email protected]>
Date: Fri, 15 Jan 2010 15:33:55 -0800
Subject: [PATCH] filelock1: Test restore of adivsory locks during restart

Test that any byte-range locks held by a process at the time of checkpoint are restored correctly after restart.

Signed-off-by Sukadev Bhattiprolu <[email protected]>

---
 fileio/Makefile         |    9 +-
 fileio/filelock1.c      |  383 +++++++++++++++++++++++++++++++++++++++++++++++
 fileio/run-filelock1.sh |  218 +++++++++++++++++++++++++++
 3 files changed, 608 insertions(+), 2 deletions(-)
 create mode 100644 fileio/filelock1.c
 create mode 100755 fileio/run-filelock1.sh

fileio/Makefile +7 -2

index 071a9eb..40d19da 100644
@@ -1,6 +1,11 @@
-targets = fileio1
+targets = fileio1 filelock1
 
-all: $(targets)
+INCLUDE   = ../libcrtest
+LIBCRTEST = ../libcrtest/common.o
+CFLAGS    = -I $(INCLUDE)
+LDFLAGS   = $(LIBCRTEST)
+
+all: $(LIBCRTEST) $(targets)
 
 clean:
 	rm -f $(targets)

fileio/filelock1.c new file +383 -0

new file mode 100644
index 0000000..305cbeb
@@ -0,0 +1,383 @@
+#include <stdio.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <string.h>
+#include <signal.h>
+#include <errno.h>
+#include "libcrtest.h"
+
+#define TEST_FILE	"data.d/data.filelock1"
+#define LOG_FILE	"logs.d/log.filelock1"
+
+typedef unsigned long long u64;
+
+extern FILE *logfp;
+int test_fd;
+int event_fd1;
+int event_fd2;
+
+/*
+ * Description:
+ * 	Ensure that F_RDLCK and F_WRLCK byte-range locks held by a process at
+ * 	the time of checkpoint are properly restored when the process is
+ * 	restarted from the checkpoint.
+ *
+ * Implementation:
+ * 	Two processes, P0 and P1 acquire the set of locks described by
+ * 	locks_list[] below. Then, they notify the parent that they are ready for
+ * 	checkpoint and wait for checkpoint to be done.  When they are restarted
+ * 	(i.e when test_done() is TRUE), each process verifies that it has the
+ * 	locks it had at the time of checkpoint and that it cannot grab a lock
+ * 	held by the other process.
+ */
+
+setup_notification()
+{
+	int efd;
+
+	efd = eventfd(0, 0);
+	if (efd < 0) {
+		fprintf(logfp, "ERROR: eventfd(): %s\n", strerror(errno));
+		do_exit(1);
+	}
+	return efd;
+}
+
+wait_for_events(int efd, u64 total)
+{
+	int n;
+	u64 events;
+	u64 count = (u64)0;
+
+	do {
+		fprintf(logfp, "%d: wait_for_events: fd %d, reading for %llu\n",
+				getpid(), efd, total);
+		fflush(logfp);
+
+		n = read(efd, &events, sizeof(events));
+		if (n != sizeof(events)) {
+			fprintf(logfp, "ERROR: read(event_fd) %s\n",
+						strerror(errno));
+			do_exit(1);
+		}
+		fprintf(logfp, "%d: wait_for_events: fd %d read %llu\n",
+				getpid(), efd, events);
+
+		count += events;
+	} while (count < total);
+}
+
+notify_one_event(int efd)
+{
+	int n;
+	u64 event = (u64)1;
+
+	fprintf(logfp, "%d: Notifying one event on fd %d\n", getpid(), efd);
+	fflush(logfp);
+
+	n = write(efd, &event, sizeof(event));
+	if (n != sizeof(event)) {
+		fprintf(logfp, "ERROR: write(event_fd) %s\n", strerror(errno));
+		do_exit(1);
+	}
+}
+
+struct test_arg {
+	int child_idx;
+	int type;
+	int start;
+	int len;
+};
+
+struct test_arg locks_list[] = {
+	{ 0, F_WRLCK, 0, 17 },
+	{ 1, F_WRLCK, 18, 16 },
+	{ 0, F_WRLCK, 35, 27 },
+	{ 1, F_WRLCK, 63, 17 },
+	{ 0, F_RDLCK, 81, 25 },
+	{ 1, F_RDLCK, 81, 25 },
+};
+
+void set_lock(int fd, struct test_arg *tlock)
+{
+	int rc;
+	struct flock lock;
+
+	lock.l_type = tlock->type;
+	lock.l_whence = SEEK_SET;
+	lock.l_start = (off_t)tlock->start;
+	lock.l_len = (off_t)tlock->len;
+
+	rc = fcntl(fd, F_SETLK, &lock);
+	if (rc < 0 && errno != EAGAIN) {
+		fprintf(logfp, "%d: set_lock(): ERROR [%d, %llu, %llu]: %s\n",
+				getpid(), tlock->type, (u64)tlock->start,
+				(u64)tlock->len, strerror(errno));
+		fflush(logfp);
+		kill(getppid(), SIGUSR1);
+		do_exit(1);
+	}
+
+	fprintf(logfp, "%d: set_lock(): [%d, %llu, %llu] %s\n", getpid(),
+			tlock->type, (u64)tlock->start, (u64)tlock->len,
+			rc < 0 ? strerror(errno) : "done");
+}
+/*
+ * If @set is TRUE, ensure that the given lock is set.
+ * If @set is FALSE, ensure that the given lock is NOT set.
+ */
+void test_lock(int fd, int locked_by_me, struct test_arg *tlock)
+{
+	int rc;
+	int conflict;
+	struct flock lock;
+	char lock_info[512];
+
+	lock.l_type = tlock->type;
+	lock.l_whence = SEEK_SET;
+	lock.l_start = (off_t)tlock->start;
+	lock.l_len = (off_t)tlock->len;
+	lock.l_pid = 0;
+
+	sprintf(lock_info, "lock [%d, %llu, %llu] ", tlock->type,
+			(u64)tlock->start, (u64)tlock->len);
+
+	conflict = 0;
+	rc = fcntl(fd, F_SETLK, &lock);
+	if (rc < 0 && (errno == EAGAIN || errno == EACCES)) {
+		rc = fcntl(fd, F_GETLK, &lock);
+		if (rc < 0) {
+			fprintf(logfp, "ERROR: fcntl(F_GETLK): %s, error %s\n",
+					lock_info, strerror(errno));
+			goto error;
+		}
+
+		if (lock.l_type == F_UNLCK || lock.l_pid == 0) {
+			fprintf(logfp, "%d: ERROR: %s F_SETLK / F_GETLK "
+					"mismatch !!!\n", getpid(), lock_info);
+			goto error;
+		}
+		conflict = 1;
+	} else if (rc < 0) {
+		fprintf(logfp, "ERROR: fcntl(F_SETLK): %s, error %s\n",
+				lock_info, strerror(errno));
+		goto error;
+	}
+
+	fprintf(logfp, "%d: %s, locked_by_me: %d, conflict %d\n", getpid(),
+			lock_info, locked_by_me, conflict);
+
+	if (locked_by_me && conflict) {
+		fprintf(logfp, "%d: FAIL: %s is NOT set by me !!!\n", getpid(),
+				lock_info);
+		goto error;
+	} else if (!locked_by_me && !conflict) {
+		fprintf(logfp, "%d: FAIL: %s is NOT set by peer !!!\n",
+				getpid(), lock_info);
+		goto error;
+	} else {
+		fprintf(logfp, "%d: PASS: %s is %sset by me\n",
+				getpid(), lock_info, conflict ? "not " : "");
+		return;
+	}
+
+error:
+	fflush(logfp);
+	kill(getppid(), SIGUSR1);
+	do_exit(1);
+}
+
+void handler(int sig)
+{
+	/*
+	 * We completed the test and siblings have completed their test.
+	 * So, safe to drop our locks and exit.
+	 */
+	fprintf(logfp, "%d: Ok to exit...\n", getpid());
+	fflush(logfp);
+	do_exit(0);
+}
+
+int do_child1(int idx)
+{
+	int rc;
+	int locked_by_me;
+	int i;
+	int num_locks;
+	int failed;
+	
+	signal(SIGINT, handler);
+
+	num_locks = sizeof(locks_list) / sizeof(struct test_arg);
+
+	for (i = 0; i < num_locks; i++) {
+		if (idx != locks_list[i].child_idx)
+			continue;
+
+		set_lock(test_fd, &locks_list[i]);
+	}
+
+	/*
+	 * Tell parent we are ready for checkpoint...
+	 */
+	notify_one_event(event_fd1);
+
+	/*
+	 * Wait for checkpoint/restart
+	 */
+	fprintf(logfp, "%d: waiting for test-done\n", idx);
+	fflush(logfp);
+	while(!test_done()) {
+		sleep(1);
+	}
+	fprintf(logfp, "%d: Found test-done\n", idx);
+	fflush(logfp);
+
+	for (i = 0; i < num_locks; i++) {
+		/*
+		 * If we had (not) set the lock earlier, ensure we still have
+		 * it (not) set.
+		 */
+		locked_by_me = 0;
+		if (idx == locks_list[i].child_idx ||
+					locks_list[i].type == F_RDLCK)
+			locked_by_me = 1;
+
+		test_lock(test_fd, locked_by_me, &locks_list[i]);
+	}
+
+	/*
+	 * Notify parent that we are done testing the locks.
+	 */
+	notify_one_event(event_fd2);
+
+	/*
+	 * Hold onto our locks and wait for siblings to complete their
+	 * test on our locks. Parent will SIGINT us when it is safe to
+	 * exit.
+	 */
+	pause();
+
+	do_exit(0);
+}
+
+/*
+ * Populate the test file so the children can lock some portions of
+ * the file
+ */
+void setup_test_file()
+{
+	char buf[256];
+
+	test_fd = open(TEST_FILE, O_RDWR|O_CREAT|O_TRUNC, 0666);
+	if (test_fd < 0) {
+		fprintf(logfp, "ERROR: open(%s): %s\n", TEST_FILE,
+				strerror(errno));
+		do_exit(1);
+	}
+
+	memset(buf, 0, sizeof(buf));
+	write(test_fd, buf, sizeof(buf));
+}
+
+int pid1, pid2;
+void child_handler(int sig)
+{
+	/*
+	 * Wait for the child that exited prematurely
+	 */
+	fprintf(logfp, "%d: Got signal %d\n", getpid(), sig);
+	fflush(logfp);
+
+	if (sig == SIGCHLD)
+		do_wait(1);
+	fprintf(logfp, "%d: Test case FAILED\n", getpid());
+	fflush(logfp);
+	/*
+	 * Kill (remaining) children and exit.
+	 */
+	kill(pid1, SIGKILL);
+	kill(pid2, SIGKILL);
+
+	do_exit(-1);
+}
+
+main(int argc, char *argv[])
+{
+	int i, status, rc;
+
+	if (test_done()) {
+		printf("Remove %s before running test\n", TEST_DONE);
+		do_exit(1);
+	}
+
+	logfp = fopen(LOG_FILE, "w");
+	if (!logfp) {
+		perror("open() logfile");
+		do_exit(1);
+	}
+
+	printf("%s: Closing stdio fds and writing messages to %s\n",
+			argv[0], LOG_FILE);
+
+	for (i=0; i<100; i++)  {
+		if (fileno(logfp) != i)
+			close(i);
+	}
+
+	setup_test_file();
+	event_fd1 = setup_notification();
+	event_fd2 = setup_notification();
+
+	/*
+	 * Before waiting for events below, ensure we will be notified
+	 * if a child encounters an error and/or exits prematurely.
+	 */
+	signal(SIGUSR1, child_handler);
+	signal(SIGCHLD, child_handler);
+
+	/*
+	 * Create the first child and wait for it take its record locks
+	 */
+	pid1 = fork();
+	if (pid1 == 0)
+		do_child1(0);
+	wait_for_events(event_fd1, 1);
+
+	/*
+	 * Create the second child and wait for it take its locks.
+	 */
+	pid2 = fork();
+	if (pid2 == 0)
+		do_child1(1);
+	wait_for_events(event_fd1, 1);
+
+	/*
+	 * Now that the test processes are ready, tell any wrapper scripts,
+	 * we are ready for checkpoint
+	 */
+	set_checkpoint_ready();
+
+	fprintf(logfp, "***** %d: Ready for checkpoint\n", getpid());
+	fflush(logfp);
+
+	/*
+	 * Wait for all children to test the locks. Since a processes locks
+	 * are dropped on exit, if process P1 exits before process P2 has
+	 * completed testing a conflicting lock, P2 may acquire the lock
+	 * supposed to be held by P1 and wrongly assume that test failed.
+	 */
+	wait_for_events(event_fd2, 2);
+
+	signal(SIGCHLD, SIG_IGN);
+
+	/*
+	 * Tell children it is safe to exit
+	 */
+	kill(pid1, SIGINT);
+	kill(pid2, SIGINT);
+
+	do_wait(2);
+
+	do_exit(0);
+}

fileio/run-filelock1.sh new file +218 -0

new file mode 100755
index 0000000..0ba2d18
@@ -0,0 +1,218 @@
+#!/bin/bash
+
+source ../common.sh
+
+dir=`mktemp -p . -d -t cr_filelock1_XXXXXXX` || (echo "mktemp failed"; exit 1)
+
+# NOTE: As of ckpt-v15-dev, the --container option to 'ckpt' causes this
+#	test to fail with "container not isolated" message due to the
+#	log-file being shared between the application threads.
+#
+CHECKPOINT="`which checkpoint` --container"
+RESTART=`which restart`
+ECHO="/bin/echo -e"
+
+TEST_CMD="../filelock1"
+TEST_ARGS=""
+TEST_LOG="logs.d/log.filelock1"
+SCRIPT_LOG="logs.d/log.run-filelock1"
+TEST_PID_FILE="pid.filelock1";
+
+SNAPSHOT_DIR="snap1.d"
+
+TEST_DONE="test-done"
+CHECKPOINT_FILE="checkpoint-filelock1";
+CHECKPOINT_READY="checkpoint-ready"
+CHECKPOINT_DONE="checkpoint-done"
+
+LOGS_DIR="logs.d"
+DATA_DIR="data.d"
+
+NS_EXEC="../../ns_exec"
+NS_EXEC_ARGS="-cgpuimP $TEST_PID_FILE"
+
+checkpoint()
+{
+	local pid=$1
+
+	$ECHO "\t- Checkpoint: $CHECKPOINT $pid \> $CHECKPOINT_FILE"
+	$CHECKPOINT $pid > $CHECKPOINT_FILE
+	ret=$?
+	if [ $ret -ne 0 ]; then
+		$ECHO "***** FAIL: Checkpoint of $pid failed"
+		ps -efL |grep $TEST_CMD >> $SCRIPT_LOG
+		exit 1;
+	fi
+}
+
+function wait_for_checkpoint_ready()
+{
+	# Wait for test to finish setup
+	while [ ! -f $CHECKPOINT_READY ]; do
+		$ECHO "\t- Waiting for $CHECKPOINT_READY"
+		sleep 1;
+	done;
+}
+
+function create_container()
+{
+	local pid;
+
+	cmdline="$NS_EXEC $NS_EXEC_ARGS -- $TEST_CMD $TEST_ARGS"
+
+	$ECHO "\t- Creating container:"
+	$ECHO "\t- $cmdline"
+
+	$cmdline &
+
+	wait_for_checkpoint_ready;
+
+	# Find global pid of container-init
+	pid=`cat $TEST_PID_FILE`;
+	if [  "x$pid" == "x" ]; then
+		$ECHO "***** FAIL: Invalid container-init pid $pid"
+		ps -efL |grep $TEST_CMD >> $SCRIPT_LOG
+		exit 1
+	fi
+	$ECHO "Created container with pid $pid" >> $SCRIPT_LOG
+}
+
+function restart_container
+{
+	local ret;
+
+	cmdline="$RESTART --pids --pidns --wait"
+	$ECHO "\t- $cmdline"
+
+	sleep 1
+
+	$cmdline < $CHECKPOINT_FILE >> $SCRIPT_LOG 2>&1 &
+	ret=$?
+
+	if [ $ret -ne 0 ]; then
+		$ECHO "***** FAIL: Restart of $pid failed"
+		ps -efL |grep $TEST_CMD >> $SCRIPT_LOG
+		exit 1;
+	fi
+
+}
+
+function create_fs_snapshot()
+{
+	# Prepare for snapshot
+	if [ -d $SNAPSHOT_DIR ]; then
+		rm -rf ${SNAPSHOT_DIR}.prev
+		mv $SNAPSHOT_DIR ${SNAPSHOT_DIR}.prev
+		mkdir $SNAPSHOT_DIR
+	fi
+
+	# Snapshot the log files
+	cp ${LOGS_DIR}/* $SNAPSHOT_DIR
+}
+
+function restore_fs_snapshot()
+{
+	# Restore the snapshot after the main process has been killed
+	/bin/cp ${SNAPSHOT_DIR}/* $LOGS_DIR
+}
+
+cd $dir
+echo "Current directory: `pwd`"
+
+if [ ! -d $LOGS_DIR ]; then
+	mkdir $LOGS_DIR
+fi
+
+if [ ! -d $DATA_DIR ]; then
+	mkdir $DATA_DIR
+fi
+
+if [ ! -d $SNAPSHOT_DIR ]; then
+	mkdir $SNAPSHOT_DIR
+fi
+
+if [ ! -f $INPUT_DATA ]; then
+	$FILEIO -C $INPUT_DATA
+fi
+
+# Make sure no stray filelock1 process from another run is still going
+killall $TEST_CMD > $SCRIPT_LOG 2>&1
+
+> $SCRIPT_LOG;
+cnt=1
+while [ $cnt -lt 20 ]; do
+	$ECHO "===== Iteration $cnt"
+
+	# Remove any 'state' files, start the app and let it tell us
+	# when it is ready
+	rm -f $CHECKPOINT_READY $TEST_DONE $TEST_PID_FILE
+
+	create_container
+	wait_for_checkpoint_ready
+
+	pid=`cat $TEST_PID_FILE`
+
+	$ECHO "\t- Done creating container, cinit-pid $pid"
+
+	ps -efL |grep $TEST_CMD >> $SCRIPT_LOG
+
+	# override default freezerdir
+	if [ -d $freezerdir ]; then
+		rmdir $freezerdir
+	fi
+	freezerdir=$freezermountpoint/$pid
+	freeze_pid $pid
+
+	num_pids1=`ps -efL |grep $TEST_CMD | wc -l`
+
+	create_fs_snapshot
+
+	checkpoint $pid
+
+	touch $CHECKPOINT_DONE
+
+	killall -9 `basename $TEST_CMD`
+
+	thaw
+
+	sleep 3
+
+	restore_fs_snapshot
+
+	restart_container
+
+	sleep 3;
+
+	num_pids2=`ps -efL |grep $TEST_CMD | wc -l`
+	ps -efL |grep $TEST_CMD >> $SCRIPT_LOG
+	$ECHO "\t- num_pids1 $num_pids1, num_pids2 $num_pids2";
+
+	# ns_exec pid is parent-pid of restarted-container-init
+	nspid=`pidof restart`
+
+	if [ "x$nspid" == "x" ]; then
+		$ECHO "***** FAIL: Can't find pid of $RESTART"
+		exit 1;
+	fi
+
+	# End test gracefully
+	touch $TEST_DONE
+
+	$ECHO "\t- Waiting for restarted container to exit (gloabl-pid $nspid)"
+	wait $nspid;
+	ret=$?
+
+	grep --binary-files=text FAIL $PWD/$TEST_LOG > /dev/null 2>&1
+	if [ $? -eq 0 ]; then
+		$ECHO "\t***** Application FAILED after restart" >> $SCRIPT_LOG
+		$ECHO "\t***** See $TEST_LOG for details" >> $SCRIPT_LOG
+
+		$ECHO "\t***** Application FAILED after restart"
+		$ECHO "\tSee $PWD/$TEST_LOG for details"
+		exit 1;
+	fi
+
+	$ECHO "\t- Container exited, status $ret"
+
+	cnt=$((cnt+1))
+done

Signature · 1 line

1.6.0.4