sleepers don't wake up on restart

Tried this with v14-rc2. This is probably not implemented yet...

I created a simple 1-level process tree (parent with 10 children). The parent just waits for the children to exit. The children do (error checks removed).

do_child():

        sprintf(cfile, "child-log-%d", cnum);

        i = 0;
        while (!test_done()) {
                cfp = fopen(cfile, "a");
                fprintf(cfp, "i = %d\n", i++);
                fflush(cfp);
                fclose(cfp);
                sleep(1);
        }

test_done():

	rc = access("test-done", F_OK)
	return rc == 0;

After freezing and restarting (using mktree), only one off 10 children makes progress. Others are stuck in:

	ptree1        S f63ef170     0  8272      1
	 f5f17780 00000082 c2b1d5c8 f63ef170 f63ef2dc c2b20260 f5e0ff44 1c18de95
	 00000282 c0233594 c2b1d5c8 00000000 00000282 0000c350 00000001 0000c350
	 00000001 f5e0ff44 0000c350 c051b229 00000000 00000001 0000c350 bff32f44
	Call Trace:
	 [<c0233594>] hrtimer_start_range_ns+0x105/0x111
	 [<c051b229>] do_nanosleep+0x54/0x8c
	 [<c023362f>] hrtimer_nanosleep+0x8f/0xee
	 [<c0233210>] hrtimer_wakeup+0x0/0x18
	 [<c051b20f>] do_nanosleep+0x3a/0x8c
	 [<c02336cf>] sys_nanosleep+0x41/0x51
	 [<c0202ce5>] sysenter_do_call+0x12/0x25

One of them, (the 8th child) makes progress and continues to write to its file

	ptree1        D f6bb7170     0  8280      1
	 f5de8b00 00000086 00000000 f6bb7170 f6bb72dc c2b17260 c0288ead 000165a5
	 c05322de c02dde8a ffffffff f6b34800 00000000 00000000 00000000 f6255d94
	 f5dd0700 c2b02314 f552d208 c02de045 00000000 f6e21478 00000000 f6b14600
	Call Trace:
	 [<c0288ead>] __find_get_block+0x121/0x12b
	 [<c02dde8a>] journal_stop+0x231/0x23d
	 [<c02de045>] do_get_write_access+0x1af/0x33b
	 [<c02310a9>] wake_bit_function+0x0/0x3c
	 [<c02de1e9>] journal_get_write_access+0x18/0x26
	 [<c02d3653>] __ext3_journal_get_write_access+0x13/0x32
	 [<c02c9285>] ext3_reserve_inode_write+0x2d/0x5d
	 [<c02c92cf>] ext3_mark_inode_dirty+0x1a/0x30
	 [<c02cb891>] ext3_dirty_inode+0x50/0x63
	 [<c0285d85>] __mark_inode_dirty+0x21/0x129
	 [<c027f7fa>] file_update_time+0x7e/0xa7
	 [<c0251c45>] __generic_file_aio_write_nolock+0x32a/0x4a1
	 [<c0281589>] mntput_no_expire+0x13/0xd7
	 [<c0251ffd>] generic_file_aio_write+0x4f/0xa6
	 [<c02c7bf9>] ext3_file_write+0x19/0x83
	 [<c0270a4d>] do_sync_write+0xbf/0x100
	 [<c023107c>] autoremove_wake_function+0x0/0x2d
	 [<c026672f>] page_add_new_anon_rmap+0x20/0x3b
	 [<c0260df9>] handle_mm_fault+0x1df/0x50c
	 [<c031d42d>] security_file_permission+0xc/0xd
	 [<c027098e>] do_sync_write+0x0/0x100
	 [<c02710dd>] vfs_write+0x83/0xf6
	 [<c02715a2>] sys_write+0x3c/0x63

When I create the 'test-done' file, this (8th child) exits but others are still sleeping (confirms they never woke up from the sleep(1))

Sukadev

Suka,

can you please post the entire test program so I can try to reproduce it here ?

thanks,

Oren.

Oren Laadan [[email protected]] wrote:

| 
| Suka,
| 
| can you please post the entire test program so I can try to reproduce
| it here ?

Sure. Attached.

Suka

#include <stdio.h>
#include <unistd.h>
#include <wait.h>
#include <errno.h>

#define NUM_PROCS 10
#define CKPT_READY "ckpt-ready"
#define CKPT_DONE "ckpt-done"
#define TEST_DONE "test-done"
#define LOG_FILE "log.ptree1"

FILE *logfp;

void do_exit(int status)

{
	if (logfp) {
		fflush(logfp);
		fclose(logfp);
	}
	_Exit(status);
}

int test_done()

{
	int rc;

	rc = access(TEST_DONE, F_OK);
	if (rc == 0)
		return 1;
	else if (errno == ENOENT)
		return 0;

	fprintf(logfp, "access(%s) failed, %s\n", TEST_DONE, strerror(errno));
	do_exit(1);
}

int checkpoint_done()

{
	int rc;

	rc = access(CKPT_DONE, F_OK);
	if (rc == 0)
		return 1;
	else if (errno == ENOENT)
		return 0;

	fprintf(logfp, "access(%s) failed, %s\n", CKPT_DONE, strerror(errno));
	do_exit(1);
}

void checkpoint_ready()

{
	int fd;

	fd = creat(CKPT_READY, 0666, 0);
	if (fd < 0) {
		fprintf(logfp, "creat(%s) failed, %s\n", CKPT_READY,
				strerror(errno));
		do_exit(1);
	}
	close(fd);
}

do_child(int cnum)

{
	int i;
	FILE *cfp;
	char cfile[256];

	sprintf(cfile, "child-log-%d", cnum);

	i = 0;
	while (!test_done()) {
		cfp = fopen(cfile, "a");
		if (!cfp) {
			fprintf(logfp, "fopen(%s) failed, error %s\n", cfile,
					strerror(errno));
			do_exit(1);
		}
		fprintf(cfp, "i = %d\n", i++);
		fflush(cfp);
		fclose(cfp);
		sleep(1);
	}

	do_exit(0);
}

print_exit_status(int pid, int status)

{
	fprintf(logfp, "Pid %d unexpected exit - ", pid);
	if (WIFEXITED(status)) {
		fprintf(logfp, "exit status %d\n", WEXITSTATUS(status));
	} else if (WIFSIGNALED(status)) {
		fprintf(logfp, "got signal %d\n", WTERMSIG(status));
	} else {
		fprintf(logfp, "stopped/continued ?\n");
	}
}

main(int argc, char *argv[])

{
	int i;
	int status;
	int pid_list[NUM_PROCS];

	if (test_done()) {
		printf("Remove %s before running test\n", TEST_DONE);
		do_exit(1);
	}

	logfp = fopen(LOG_FILE, "w");
	if (!logfp) {
		fprintf(stderr, "fopen(%s) failed, %s\n", LOG_FILE,
					strerror(errno));
		fflush(stderr);
		do_exit(1);
	}
	close(0);close(1);close(2);

	for (i = 0; i < NUM_PROCS; i++) {
		if ((pid_list[i] = fork()) == 0)
			do_child(i);
	}

	/*
 	 * Now that we closed the special files and created process tree
	 * tell any wrapper scripts, we are ready for checkpoint
	 */
	checkpoint_ready();

	while(!checkpoint_done())
		sleep(1);

	for (i = 0; i < NUM_PROCS; i++) {
		if (waitpid(pid_list[i], &status, 0) < 0) {
			fprintf(logfp, "waitpid(%d) failed, error %s\n",
					pid_list[i], strerror(errno));
		}

		if (!WIFEXITED(status) || WEXITSTATUS(status) != 0)
			print_exit_status(pid_list[i], status);	
	}
}

_______________________________________________
Containers mailing list Containers-cunTk1MwBs9QetFLy7KEm3xJsTq8ys+cHZ5vskTnxNA@public.gmane.org
https://lists.linux-foundation.org/mailman/listinfo/containers

I just posted v14-rc3 which includes the c/r of restart-blocks. That should improve the situation.

However, depending on which syscalls one uses, process may still seem "stuck" after restart because the current code still does not save signals nor task timers; If a signal was pending (SIGALRM for example) after freezing but before checkpoint, it will be lost. If a timer was set at checkpoint, it will not be restored.

So depending on your program, you may still experience issues until I add patches to handle that.

Oren.

Oren Laadan [[email protected]] wrote:

| 
| I just posted v14-rc3 which includes the c/r of restart-blocks.
| That should improve the situation.
| 
| However, depending on which syscalls one uses, process may still
| seem "stuck" after restart because the current code still does
| not save signals nor task timers; If a signal was pending (SIGALRM
| for example) after freezing but before checkpoint, it will be lost.
| If a timer was set at checkpoint, it will not be restored.
| 
| So depending on your program, you may still experience issues
| until I add patches to handle that.

Ok, Just an fyi, the original program seemed to work fine, but when I try to restart a small process tree, I get stuck on restart again.

I am running on v14-rc3 branch. Has this got anything to do with pending SIGCHLD ? Seems to be easier to repro with larger process trees (2 children per process, 4 or more levels deep).

Test programs (attached) (they need some cleanup though)

	ptree2.c
	p2.loop

--------- Processes after restart:

$ ps -ef|grep ptree

root     10461 10459  0 22:07 pts/0    00:00:00 ./ptree2 -n 1 -d 2
root     10465 10461  0 22:07 pts/0    00:00:00 ./ptree2 -n 1 -d 2
root     10466 10465  0 22:07 pts/0    00:00:00 [ptree2] <defunct>
root     10479  8220  0 22:09 pts/1    00:00:00 grep ptree

---------- Process stacks

tree2 S f6270a90 0 10461 10459
f5e59380 00000082 08048a86 f6270a90 f6270bfc c2b32260 00000000 0000d9d3 f5f423b0 00000000 ffffffff 00000000 00000000 00000001 00000000 f6270a88 00000000 f6270a90 00000000 c02243aa 00000004 00000003 0000000c 00000006

Call Trace:
 [<c02243aa>] do_wait+0x1dd/0x2f6
 [<c021cd14>] default_wake_function+0x0/0x8
 [<c0224542>] sys_wait4+0x7f/0x92
 [<c0224568>] sys_waitpid+0x13/0x17
 [<c0202ce5>] sysenter_do_call+0x12/0x25
 [<c0510000>] rtl8139_init_one+0x5ae/0x887

ptree2 S f5f423b0 0 10465 10461
f6002180 00000082 c2b265c8 f5f423b0 f5f4251c c2b29260 f67b1f44 e06d0177 00000282 c023363c c2b265c8 00000000 00000282 0000c350 00000001 0000c350 00000001 f67b1f44 0000c350 c051be99 00000000 00000001 0000c350 bf9d0e04

Call Trace:
 [<c023363c>] hrtimer_start_range_ns+0x105/0x111
 [<c051be99>] do_nanosleep+0x54/0x8c
 [<c02336d7>] hrtimer_nanosleep+0x8f/0xee
 [<c02332b8>] hrtimer_wakeup+0x0/0x18
 [<c051be7f>] do_nanosleep+0x3a/0x8c
 [<c0233777>] sys_nanosleep+0x41/0x51
 [<c0202ce5>] sysenter_do_call+0x12/0x25

ptree2 ? f6bee040 0 10466 10465
f638cb80 00000046 00200200 f6bee040 f6bee1ac c2b17260 f6bee038 0000dd77 00000000 c022f576 ffffffff 00000303 00000000 00000001 00000000 00000012 f5a61e84 f6bee040 f6bee038 c0224c29 f6270a90 00000001 f6bee038 f5a61f88

Call Trace:
 [<c022f576>] wakeme_after_rcu+0x0/0x8
 [<c0224c29>] do_exit+0x638/0x63c
 [<c0224c87>] do_group_exit+0x5a/0x83
 [<c0224cbd>] sys_exit_group+0xd/0x10
 [<c0202ce5>] sysenter_do_call+0x12/0x25

#include <stdio.h>
#include <unistd.h>
#include <wait.h>
#include <errno.h>
#include <string.h>

int max_depth = 3;
int num_children = 3;

#define CKPT_READY "checkpoint-ready"
#define CKPT_DONE "checkpoint-done"
#define TEST_DONE "test-done"
#define LOG_FILE "log-ptree2"

#undef SYS_GETGPID

#ifdef SYS_GETGPID
static inline int sys_getgpid()
{

#define	__NR_getgpid	335
        return syscall(__NR_getgpid);
}

#else
#define sys_getgpid getpid
#endif

FILE *logfp;

void do_exit(int status)

{
	if (logfp) {
		fflush(logfp);
		fclose(logfp);
	}
	_Exit(status);
}

int test_done()

{
	int rc;

	rc = access(TEST_DONE, F_OK);
	if (rc == 0)
		return 1;
	else if (errno == ENOENT)
		return 0;

	fprintf(logfp, "access(%s) failed, %s\n", TEST_DONE, strerror(errno));
	do_exit(1);
}

int checkpoint_done()

{
	int rc;

	rc = access(CKPT_DONE, F_OK);
	if (rc == 0)
		return 1;
	else if (errno == ENOENT)
		return 0;

	fprintf(logfp, "access(%s) failed, %s\n", CKPT_DONE, strerror(errno));
	do_exit(1);
}

void checkpoint_ready()

{
	int fd;

	fd = creat(CKPT_READY, 0666, 0);
	if (fd < 0) {
		fprintf(logfp, "creat(%s) failed, %s\n", CKPT_READY,
				strerror(errno));
		do_exit(1);
	}
	close(fd);
}

print_exit_status(int pid, int status)

{
	fprintf(logfp, "Pid %d unexpected exit - ", pid);
	if (WIFEXITED(status)) {
		fprintf(logfp, "exit status %d\n", WEXITSTATUS(status));
	} else if (WIFSIGNALED(status)) {
		fprintf(logfp, "got signal %d\n", WTERMSIG(status));
	} else {
		fprintf(logfp, "stopped/continued ?\n");
	}
}

void do_wait()

{
	int rc;
	int n;
	int status;

	n = 0;
	while(1) {
		rc = waitpid(-1, &status, 0);
		if (rc < 0)
			break;

		n++;
		if (!WIFEXITED(status) || WEXITSTATUS(status) != 0)
			print_exit_status(rc, status);	
	}

	if (errno != ECHILD) {
		fprintf(logfp, "waitpid(%d) failed, error %s\n",
					rc, strerror(errno));
		do_exit(1);
	}

	if (getpid() == 1 && n != num_children * max_depth) {
		fprintf(logfp, "Only %d of %d children exited ?\n",
			num_children, num_children * max_depth);
		do_exit(1);
	}

	do_exit(0);
}

static do_child(int depth, char *suffix);

create_children(int depth, char *parent_suffix)

{
	int i;
	int child_pid;
	char suffix[1024];

	for (i = 0; i < num_children; i++) {
		sprintf(suffix, "%s-%d", parent_suffix, i);

		child_pid = fork();
		if (child_pid == 0)
			do_child(depth, suffix);
		else if (child_pid < 0) {
			fprintf(logfp, "fork() failed, depth %d, "
				"child %d, error %s\n", depth, i,
				strerror(errno));
			do_exit(1);
		}
	}
}

do_child(int depth, char *suffix)

{
	int i;
	FILE *cfp;
	char cfile[256];
	char *mode = "w";

	/*
	 * Recursively calls do_child() and both parent and child
	 * execute the code below
	 */
	if (depth < max_depth)
		create_children(depth+1, suffix);

	sprintf(cfile, "%s%s", LOG_FILE, suffix);

	i = 0;
	while (!test_done()) {
		/* truncate the first time, append after that */
		cfp = fopen(cfile, mode);
		mode = "a";
		if (!cfp) {
			fprintf(logfp, "fopen(%s) failed, error %s\n", cfile,
					strerror(errno));
			do_exit(1);
		}
		fprintf(cfp, "gpid %d, pid %d: i %d\n", sys_getgpid(),
				getpid(), i++);
		fflush(cfp);
		sleep(1);
		fprintf(cfp, "gpid %d: woke up from sleep(1)\n", sys_getgpid());
		fflush(cfp);
		fclose(cfp);
	}

	/* Wait for any children that pre-deceased us */
	do_wait();

	do_exit(0);
}

static void usage(char *argv[])

{
	printf("%s [h] [-d max-depth] [-n max-children]\n", argv[0]);
	printf("\t <max-depth> max depth of process tree, default 3\n");
	printf("\t <num-children> # of children per process, default 3\n");
	do_exit(1);
}

main(int argc, char *argv[])

{
	int c;
	int i;
	int status;

	if (test_done()) {
		printf("Remove %s before running test\n", TEST_DONE);
		do_exit(1);
	}

	while ((c = getopt(argc, argv, "hd:n:")) != EOF) {
		switch (c) {
		case 'd': max_depth = atoi(optarg); break;
		case 'n': num_children = atoi(optarg); break;
		case 'h':
		default:
			usage(argv);
		}
	};

	logfp = fopen(LOG_FILE, "w");
	if (!logfp) {
		fprintf(stderr, "fopen(%s) failed, %s\n", LOG_FILE,
					strerror(errno));
		fflush(stderr);
		do_exit(1);
	}
	close(0);close(1);close(2);

	create_children(1, "");

	/*
 	 * Now that we closed the special files and created process tree
	 * tell any wrapper scripts, we are ready for checkpoint
	 */
	checkpoint_ready();

#if 0
	while(!checkpoint_done())
		sleep(1);

#endif

	do_wait();
}

#!/bin/bash

freezermountpoint=/cgroups
CHECKPOINT=".."
NS_EXEC="$CHECKPOINT/bin/ns_exec"
CR="$CHECKPOINT/bin/cr"
RSTR="$CHECKPOINT/bin/rstr"
MKTREE="$CHECKPOINT/bin/mktree"
ECHO="/bin/echo -e"

TEST_CMD="./ptree2"
TEST_ARGS="-n 1 -d 2" # -n: children per process, -d: depth of process tree
SCRIPT_LOG="log-p2-loop"
TEST_PID_FILE="pid.ptree2";

LOG_FILE="loop-ptree2.log"
SNAPSHOT_DIR="snap1"

TEST_DONE="test-done"
CHECKPOINT_FILE="checkpoint-ptree2";
CHECKPOINT_READY="checkpoint-ready"
CHECKPOINT_DONE="checkpoint-done"
TEST_LOG_PREFIX="log-ptree2"
TEST_LOG_SNAP="${TEST_LOG_PREFIX}.snap"

freeze()

{
	$ECHO "\t - Freezing $1"
	$ECHO FROZEN > ${freezermountpoint}/$1/freezer.state
	ret=$?
	if [ $ret -ne 0 ]; then
		$ECHO "***** FAIL: \'echo FROZEN \> $state\' returned $ret"
	fi
}

unfreeze()

{
	$ECHO "\t - Unfreezing $1"
	$ECHO THAWED > ${freezermountpoint}/$1/freezer.state
	ret=$?
	if [ $ret -ne 0 ]; then
		$ECHO "***** FAIL: \'echo THAWED \> $state\' returned $ret"
	fi
}

cleancgroup()

{
	$ECHO "\t - Clean cgroup of $1"
	rmdir ${freezermountpoint}/$1
	if [ -d ${freezermountpoint}/$1 ]; then
		$ECHO ***** WARNING ${freezermountpoint}/$1 remains
	fi
}

checkpoint()
{
local pid=$1

$ECHO "Checkpoint: $CR $pid $CHECKPOINT_FILE" $CR $pid $CHECKPOINT_FILE
ret=$?
if [ $ret -ne 0 ]; then
$ECHO "***** FAIL: Checkpoint of $pid failed" ps aux |grep $TEST_CMD >> $SCRIPT_LOG exit 1;
fi
}

function create_container
{
local pid;

$ECHO "\t - $NS_EXEC -cpmP $TEST_PID_FILE -- $TEST_CMD $TEST_ARGS" $NS_EXEC -cpmP $TEST_PID_FILE -- $TEST_CMD $TEST_ARGS &

# Wait for test to finish setup
while [ ! -f $CHECKPOINT_READY ]; do
/bin/$ECHO -e "\t - Waiting for $CHECKPOINT_READY" sleep 1;
done;

# Find global pid of container-init
pid=cat $TEST_PID_FILE;
if [ "x$pid" == "x" ]; then
$ECHO "***** FAIL: Invalid container-init pid $pid" ps -ef |grep $TEST_CMD >> $SCRIPT_LOG exit 1
fi
$ECHO "Created container with pid $pid" >> $SCRIPT_LOG echo $pid
}

function restart_container

{
	local ret;

	$ECHO "\t - Exec $NS_EXEC -cpuim -- $MKTREE --no-pids < $CHECKPOINT_FILE"

	sleep 1

	$NS_EXEC -cpuim -- $MKTREE --no-pids < $CHECKPOINT_FILE >> $SCRIPT_LOG 2>&1 &
	ret=$?

	if [ $ret -ne 0 ]; then
		$ECHO "***** FAIL: Restart of $pid failed"
		ps aux |grep $TEST_CMD >> $SCRIPT_LOG
		exit 1;
	fi
}

# Check freezer mount point
line=grep freezer /proc/mounts
$ECHO $line | grep "\<ns\>"
if [ $? -ne 0 ]; then
$ECHO "please mount freezer and ns cgroups" $ECHO " mkdir /cgroups"
$ECHO " mount -t cgroup -o freezer,ns cgroup /cgroups" exit 1
fi
#freezermountpoint=$ECHO $line | awk '{ print $2 '}

# Make sure no stray e2 from another run is still going killall $TEST_CMD > $SCRIPT_LOG 2>&1

cnt=1
while [ 1 ]; do
> $SCRIPT_LOG;
dmesg -c > /dev/null

$ECHO "===== Iteration $cnt"

# Remove any 'state' files, start the app and let it tell us # when it is ready
rm -f $CHECKPOINT_READY $TEST_DONE $TEST_PID_FILE

$NS_EXEC -cpumP $TEST_PID_FILE -- $TEST_CMD $TEST_ARGS& $ECHO "\t - $NS_EXEC -cpumP $TEST_PID_FILE -- $TEST_CMD $TEST_ARGS"

# Wait for test to finish setup
while [ ! -f $CHECKPOINT_READY ]; do
$ECHO "\t - Waiting for $CHECKPOINT_READY" sleep 1;
done;

ps -ef |grep ptree2 >> $SCRIPT_LOG

# Find global pid of container-init
pid=cat $TEST_PID_FILE;
if [ "x$pid" == "x" ]; then
$ECHO "***** FAIL: Invalid container-init pid $pid" ps -ef |grep $TEST_CMD
exit 1
fi
$ECHO $pid
#pid=create_container
$ECHO "\t - Done creating container"

# Prepare for snapshot
if [ -d $SNAPSHOT_DIR ]; then
rm -rf ${SNAPSHOT_DIR}.prev mv $SNAPSHOT_DIR ${SNAPSHOT_DIR}.prev mkdir $SNAPSHOT_DIR
fi

freeze $pid

num_pids1=ps -ef |grep $TEST_CMD | wc -l

checkpoint $pid

#$ECHO t > /proc/sysrq-trigger
#dmesg > dmesg-1.out

# Snapshot the log files
cp ${TEST_LOG_PREFIX}* $SNAPSHOT_DIR
touch $CHECKPOINT_DONE

killall -9 basename $TEST_CMD

unfreeze $pid

sleep 3

cleancgroup $pid

# Restore the snapshot after the main process has been killed /bin/cp ${SNAPSHOT_DIR}/* .

# Restart.
restart_container

sleep 3;
num_pids2=ps -ef |grep $TEST_CMD | wc -l

ps -ef |grep ptree2 >> $SCRIPT_LOG

$ECHO "\t - num_pids1 $num_pids1, num_pids2 $num_pids2";

# Find global-pid of container-init nspid=pidof $NS_EXEC
if [ "x$nspid" == "x" ]; then
$ECHO "***** FAIL: Can't find pid of $NS_EXEC" exit 1;
fi

# End test gracefully
touch $TEST_DONE

$ECHO "\t - Restart: Waiting for container-init (gloabl-pid $nspid) to exit" wait $nspid;
ret=$?

$ECHO "Container-init (global-pid $nspid) exited, status $ret"

if [ -d /cgroups/$pid ]; then
cleancgroup $pid
fi

cnt=$((cnt+1))
done

_______________________________________________
Containers mailing list Containers-cunTk1MwBs9QetFLy7KEm3xJsTq8ys+cHZ5vskTnxNA@public.gmane.org
https://lists.linux-foundation.org/mailman/listinfo/containers

Hi,

Could be. You can verify by adding a couple of lines of code to the checkpoint to complain if there are signals pending on a task that is being checkpointed.

BTW, current code disregards Zombie processes.

Support for both (signals and zombies) is in the queue.

Oren.