Link LXC with USERCR

cover

	$ cd /root
	$ git-clone git://git.ncl.cs.columbia.edu/pub/git/user-cr.git user-cr
	$ cd user-cr
	$ git-checkout ckpt-v20-dev
	  	Tested with commit e275f77e4a82d228c1df14dbeb691342e32cdac2
		as HEAD.
	
	# Apply following two patches:

	https://lists.linux-foundation.org/pipermail/containers/2010-March/024037.html
	https://lists.linux-foundation.org/pipermail/containers/2010-March/024038.html

	$ cd /root/user-cr

	$ KERNELSRC=/root/linux-2.6/ make 

		Build USERCR by pointing to corresponding kernel-source.
		This should create restart.o and checkpoint.o needed by LXC.
	$ cd /root/lxc.git
	$ lxc-execute --name foo --rcfile lxc-no-netns.conf -- /bin/sleep 1000
	$ lxc-checkpoint --name foo --image /root/lxc-foo.ckpt
	$ lxc-stop --name foo
	$ lxc-restart --name foo --image /root/lxc-foo.ckpt
	$ ./autogen.sh
	$ ./configure --help |grep libcr
	--with-libcr=dir     use the Checkpoint/Restart implementation in 'dir'
	$ ls /home/guest/user-cr/
	app-checkpoint.h    checkpoint.o    restart.o
	$ ./configure --with-libcr=/home/guest/user-cr
---
 configure.ac        |   19 +++++++++++++++++++
 src/lxc/Makefile.am |   10 +++++++++-
 2 files changed, 28 insertions(+), 1 deletions(-)
index f82e7df..fe6584c 100644
@@ -12,6 +12,25 @@ AM_PROG_CC_C_O
 AC_GNU_SOURCE
 AC_CHECK_PROG(SETCAP, setcap, yes, no, $PATH$PATH_SEPARATOR/sbin)
 
+AC_ARG_WITH(libcr, [AS_HELP_STRING([--with-libcr=dir], \
+           [use the Checkpoint/Restart implementation in 'dir'])], [], \
+	   [with_libcr=no])
+
+if test "x$with_libcr" != "xno"; then
+       AS_AC_EXPAND(LIBCR_OBJS, "${with_libcr}/checkpoint.o ${with_libcr}/restart.o ${with_libcr}/libeclone.a")
+       AS_AC_EXPAND(LIBCR_CFLAGS, "-DLIBCR -I${with_libcr} -I$(with_libcr}/include")
+
+       AC_CHECK_FILE([$with_libcr/app-checkpoint.h], [], \
+               AC_MSG_ERROR([--with-libcr specified directory $with_libcr but $with_libcr/app-checkpoint.h was not found]))
+
+       AC_CHECK_FILE([${with_libcr}/checkpoint.o], [], \
+               AC_MSG_ERROR([--with-libcr specified directory $with_libcr but ${with_libcr}/checkpoint.o was not found]))
+
+       AC_CHECK_FILE([${with_libcr}/restart.o], [], \
+               AC_MSG_ERROR([--with-libcr specified directory $with_libcr but ${with_libcr}/restart.o was not found]))
+fi
+
+
 AC_ARG_ENABLE([doc],
 	[AC_HELP_STRING([--enable-doc], [make mans (require docbook2man installed) [default=auto]])],
 	[], [enable_doc=auto])

index 890f706..699c355 100644
@@ -46,12 +46,20 @@ liblxc_so_SOURCES = \
 	mainloop.c mainloop.h \
 	af_unix.c af_unix.h
 
-AM_CFLAGS=-I$(top_srcdir)/src
+# We only need $(LIBCR_CFLAGS) for lxc_checkpoint and lxc_restart files
+# but for now, just set it for all.
+AM_CFLAGS=-I$(top_srcdir)/src $(LIBCR_CFLAGS)
 
 liblxc_so_CFLAGS = -fPIC -DPIC $(AM_CFLAGS)
 
+# TODO: Adding $(LIBCR_OBJS) here ensures we don't have undefined references
+# 	when building liblxc.so, but this has the side-effect of putting the
+# 	app_checkpoint/restart functions in liblxc.so. Or alternatively,
+# 	we could remove src/lxc/{checkpoint.o,restart.o} from liblxc.so
+# 	and link lxc-checkpoint/lxc-restart with them directly.
 liblxc_so_LDFLAGS = \
 	-shared \
+	$(LIBCR_OBJS) \
 	-Wl,-soname,liblxc.so.$(firstword $(subst ., ,$(VERSION)))
 
 liblxc_so_LDADD = -lutil

1.6.6.1
	$ ./autogen.sh
	$ ./configure --help |grep libcr
	--with-libcr=dir     use the Checkpoint/Restart implementation in 'dir'
	$ ls /home/guest/user-cr/
	app-checkpoint.h    checkpoint.o    restart.o
	$ ./configure --with-libcr=/home/guest/user-cr
---
 configure.ac        |   19 +++++++++++++++++++
 src/lxc/Makefile.am |   10 +++++++++-
 2 files changed, 28 insertions(+), 1 deletions(-)
index f82e7df..fe6584c 100644

@@ -12,6 +12,25 @@ AM_PROG_CC_C_O
 AC_GNU_SOURCE
 AC_CHECK_PROG(SETCAP, setcap, yes, no, $PATH$PATH_SEPARATOR/sbin)
 
+AC_ARG_WITH(libcr, [AS_HELP_STRING([--with-libcr=dir], \
+           [use the Checkpoint/Restart implementation in 'dir'])], [], \
+	   [with_libcr=no])
+
+if test "x$with_libcr" != "xno"; then
+       AS_AC_EXPAND(LIBCR_OBJS, "${with_libcr}/checkpoint.o ${with_libcr}/restart.o ${with_libcr}/libeclone.a")
+       AS_AC_EXPAND(LIBCR_CFLAGS, "-DLIBCR -I${with_libcr} -I$(with_libcr}/include")

+
+       AC_CHECK_FILE([$with_libcr/app-checkpoint.h], [], \
+               AC_MSG_ERROR([--with-libcr specified directory $with_libcr but $with_libcr/app-checkpoint.h was not found]))
+
+       AC_CHECK_FILE([${with_libcr}/checkpoint.o], [], \
+               AC_MSG_ERROR([--with-libcr specified directory $with_libcr but ${with_libcr}/checkpoint.o was not found]))
+
+       AC_CHECK_FILE([${with_libcr}/restart.o], [], \
+               AC_MSG_ERROR([--with-libcr specified directory $with_libcr but ${with_libcr}/restart.o was not found]))
+fi
+
+
 AC_ARG_ENABLE([doc],
 	[AC_HELP_STRING([--enable-doc], [make mans (require docbook2man installed) [default=auto]])],
 	[], [enable_doc=auto])

index 890f706..699c355 100644
@@ -46,12 +46,20 @@ liblxc_so_SOURCES = \
 	mainloop.c mainloop.h \
 	af_unix.c af_unix.h
 
-AM_CFLAGS=-I$(top_srcdir)/src
+# We only need $(LIBCR_CFLAGS) for lxc_checkpoint and lxc_restart files
+# but for now, just set it for all.
+AM_CFLAGS=-I$(top_srcdir)/src $(LIBCR_CFLAGS)
 
 liblxc_so_CFLAGS = -fPIC -DPIC $(AM_CFLAGS)
 
+# TODO: Adding $(LIBCR_OBJS) here ensures we don't have undefined references

+# 	when building liblxc.so, but this has the side-effect of putting the
+# 	app_checkpoint/restart functions in liblxc.so. Or alternatively,
+# 	we could remove src/lxc/{checkpoint.o,restart.o} from liblxc.so
+# 	and link lxc-checkpoint/lxc-restart with them directly.
 liblxc_so_LDFLAGS = \
 	-shared \
+	$(LIBCR_OBJS) \
 	-Wl,-soname,liblxc.so.$(firstword $(subst ., ,$(VERSION)))
 
 liblxc_so_LDADD = -lutil

---
 src/lxc/lxc_restart.c |   34 +++++++++++++++++++++++++---------
 1 files changed, 25 insertions(+), 9 deletions(-)
index 7db1d85..de4b421 100644
@@ -38,13 +38,21 @@
 lxc_log_define(lxc_restart_ui, lxc_restart);
 
 static struct lxc_list defines;
+static char *statedir;
 
 static int my_checker(const struct lxc_arguments* args)
 {
-	if (!args->statefile) {
-		lxc_error(args, "no statefile specified");
-		return -1;
-	}
+	int d, f;
+	
+	/* make them boolean */
+	d = !!(statedir);
+	f = !!(args->statefile);
+
+        if (!(d ^ f)) {
+                lxc_error(args, "Must specify exactly one of --directory "
+                                "and --statefile options");
+                return -1;
+        }
 
 	return 0;
 }

@@ -52,8 +60,9 @@ static int my_checker(const struct lxc_arguments* args)
 static int my_parser(struct lxc_arguments* args, int c, char* arg)
 {
 	switch (c) {
-	case 'd': args->statefile = arg; break;
+	case 'd': statedir = arg; break;
 	case 'f': args->rcfile = arg; break;
+	case 'S': args->statefile = arg; break;
 	case 'p': args->flags = LXC_FLAG_PAUSE; break;
 	case 's': return lxc_config_define_add(&defines, arg);
 	}

@@ -66,21 +75,24 @@ static const struct option my_longopts[] = {
 	{"rcfile", required_argument, 0, 'f'},
 	{"pause", no_argument, 0, 'p'},
 	{"define", required_argument, 0, 's'},
+	{"statefile", required_argument, 0, 'S'},
 	LXC_COMMON_OPTIONS
 };
 
 static struct lxc_arguments my_args = {
 	.progname = "lxc-restart",
 	.help     = "\
---name=NAME --directory STATEFILE\n\
+--name=NAME --directory STATEFILE (deprecated)\n\
+\tlxc_restart --name=NAME --statefile=STATEFILE\n\
 \n\
 lxc-restart restarts from STATEFILE the NAME container\n\
 \n\
 Options :\n\
   -n, --name=NAME      NAME for name of the container\n\
   -p, --pause          do not release the container after the restart\n\
-  -d, --directory=STATEFILE for name of statefile\n\
+  -d, --directory=STATEFILE for name of statefile (legacy mode, deprecated)\n\
   -f, --rcfile=FILE Load configuration file FILE\n\
+  -i, --statefile=STATEFILE Load the application state from STATEFILE (libcr mode)\n\
   -s, --define KEY=VAL Assign VAL to configuration variable KEY\n",
 	.options  = my_longopts,
 	.parser   = my_parser,

@@ -90,6 +102,7 @@ Options :\n\
 int main(int argc, char *argv[])
 {
 	char *rcfile = NULL;
+	const char *statefile;
 	struct lxc_conf *conf;
 
 	lxc_list_init(&defines);

@@ -131,6 +144,9 @@ int main(int argc, char *argv[])
 	if (lxc_config_define_load(&defines, conf))
 		return -1;
 
-	return lxc_restart(my_args.name, my_args.statefile, conf,
-			   my_args.flags);
+	statefile = my_args.statefile;
+	if (statedir)
+		statefile = statedir;
+
+	return lxc_restart(my_args.name, statefile, conf, my_args.flags);
 }

1.6.6.1
---
 src/lxc/lxc_restart.c |   34 +++++++++++++++++++++++++---------
 1 files changed, 25 insertions(+), 9 deletions(-)
index 7db1d85..de4b421 100644
@@ -38,13 +38,21 @@
 lxc_log_define(lxc_restart_ui, lxc_restart);
 
 static struct lxc_list defines;
+static char *statedir;
 
 static int my_checker(const struct lxc_arguments* args)
 {
-	if (!args->statefile) {
-		lxc_error(args, "no statefile specified");
-		return -1;
-	}
+	int d, f;
+	
+	/* make them boolean */
+	d = !!(statedir);
+	f = !!(args->statefile);
+
+        if (!(d ^ f)) {
+                lxc_error(args, "Must specify exactly one of --directory "
+                                "and --statefile options");
+                return -1;
+        }
 
 	return 0;
 }

@@ -52,8 +60,9 @@ static int my_checker(const struct lxc_arguments* args)
 static int my_parser(struct lxc_arguments* args, int c, char* arg)
 {
 	switch (c) {
-	case 'd': args->statefile = arg; break;
+	case 'd': statedir = arg; break;
 	case 'f': args->rcfile = arg; break;
+	case 'S': args->statefile = arg; break;
 	case 'p': args->flags = LXC_FLAG_PAUSE; break;
 	case 's': return lxc_config_define_add(&defines, arg);
 	}

@@ -66,21 +75,24 @@ static const struct option my_longopts[] = {
 	{"rcfile", required_argument, 0, 'f'},
 	{"pause", no_argument, 0, 'p'},
 	{"define", required_argument, 0, 's'},
+	{"statefile", required_argument, 0, 'S'},
 	LXC_COMMON_OPTIONS
 };
 
 static struct lxc_arguments my_args = {
 	.progname = "lxc-restart",
 	.help     = "\
---name=NAME --directory STATEFILE\n\
+--name=NAME --directory STATEFILE (deprecated)\n\
+\tlxc_restart --name=NAME --statefile=STATEFILE\n\
 \n\
 lxc-restart restarts from STATEFILE the NAME container\n\
 \n\
 Options :\n\
   -n, --name=NAME      NAME for name of the container\n\
   -p, --pause          do not release the container after the restart\n\
-  -d, --directory=STATEFILE for name of statefile\n\
+  -d, --directory=STATEFILE for name of statefile (legacy mode, deprecated)\n\
   -f, --rcfile=FILE Load configuration file FILE\n\
+  -i, --statefile=STATEFILE Load the application state from STATEFILE (libcr mode)\n\
   -s, --define KEY=VAL Assign VAL to configuration variable KEY\n",
 	.options  = my_longopts,
 	.parser   = my_parser,

@@ -90,6 +102,7 @@ Options :\n\
 int main(int argc, char *argv[])
 {
 	char *rcfile = NULL;
+	const char *statefile;
 	struct lxc_conf *conf;
 
 	lxc_list_init(&defines);

@@ -131,6 +144,9 @@ int main(int argc, char *argv[])
 	if (lxc_config_define_load(&defines, conf))

 		return -1;
 
-	return lxc_restart(my_args.name, my_args.statefile, conf,
-			   my_args.flags);
+	statefile = my_args.statefile;
+	if (statedir)
+		statefile = statedir;
+
+	return lxc_restart(my_args.name, statefile, conf, my_args.flags);
 }

---
 src/lxc/lxc_checkpoint.c |   40 +++++++++++++++++++++++++++++-----------
 1 files changed, 29 insertions(+), 11 deletions(-)
index a8c74a9..5e8506f 100644
@@ -37,12 +37,21 @@
 
 lxc_log_define(lxc_checkpoint_ui, lxc_checkpoint);
 
+static char *statedir;
+
 static int my_checker(const struct lxc_arguments* args)
 {
-	if (!args->statefile) {
-		lxc_error(args, "no statefile specified");
-		return -1;
-	}
+	int d, f;
+	
+	/* make them boolean */
+	d = !!(statedir);
+	f = !!(args->statefile);
+
+        if (!(d ^ f)) {
+                lxc_error(args, "Must specify exactly one of --directory "
+                                "and --statefile options");
+                return -1;
+        }
 
 	return 0;
 }

@@ -52,7 +61,8 @@ static int my_parser(struct lxc_arguments* args, int c, char* arg)
 	switch (c) {
 	case 'k': args->flags = LXC_FLAG_HALT; break;
 	case 'p': args->flags = LXC_FLAG_PAUSE; break;
-	case 'd': args->statefile = arg; break;
+	case 'd': statedir = arg; break;
+	case 'S': args->statefile = arg; break;
 	}
 	return 0;
 }

@@ -61,13 +71,15 @@ static const struct option my_longopts[] = {
 	{"kill", no_argument, 0, 'k'},
 	{"pause", no_argument, 0, 'p'},
 	{"directory", required_argument, 0, 'd'},
+	{"statefile", required_argument, 0, 'S'},
 	LXC_COMMON_OPTIONS
 };
 
 static struct lxc_arguments my_args = {
 	.progname = "lxc-checkpoint",
 	.help     = "\
---name=NAME --directory STATEFILE\n\
+--name=NAME --directory STATEFILE (deprecated)\n\
+\tlxc_checkpoint --name=NAME --statefile=STATEFILE\n\
 \n\
 lxc-checkpoint checkpoints in STATEFILE the NAME container\n\
 \n\

@@ -75,7 +87,8 @@ Options :\n\
   -n, --name=NAME      NAME for name of the container\n\
   -k, --kill           stop the container after checkpoint\n\
   -p, --pause          don't unfreeze the container after the checkpoint\n\
-  -d, --directory=STATEFILE where to store the statefile\n",
+  -d, --directory=STATEFILE where to store the statefile (deprecated)\n\
+  -i, --statefile=STATEFILE where to store the checkpoint-image (LIBCR mode)\n",
 
 	.options  = my_longopts,
 	.parser   = my_parser,

@@ -97,6 +110,7 @@ static int create_statefile(const char *dir)
 int main(int argc, char *argv[])
 {
 	int ret;
+	const char *statefile;
 
 	ret = lxc_arguments_parse(&my_args, argc, argv);
 	if (ret)

@@ -107,11 +121,15 @@ int main(int argc, char *argv[])
 	if (ret)
 		return ret;
 
-	ret = create_statefile(my_args.statefile);
-	if (ret)
-		return ret;
+	statefile = my_args.statefile;
+	if (statedir) {
+		statefile = statedir;
+		ret = create_statefile(statefile);
+		if (ret)
+			return ret;
+	}
 
-	ret = lxc_checkpoint(my_args.name, my_args.statefile, my_args.flags);
+	ret = lxc_checkpoint(my_args.name, statefile, my_args.flags);
 	if (ret)
 		ERROR("failed to checkpoint '%s'", my_args.name);
 	else

1.6.6.1
---
 src/lxc/checkpoint.c |   29 +++++++++++++++++++++++++++++
 src/lxc/lxc.h        |    6 ++++++
 src/lxc/lxc_attach.c |   29 +----------------------------
 3 files changed, 36 insertions(+), 28 deletions(-)
index 7e8a93e..4e75cb6 100644
@@ -22,9 +22,38 @@
  */
 #include <lxc/lxc.h>
 #include <lxc/log.h>
+#include <lxc/commands.h>
 
 lxc_log_define(lxc_checkpoint, lxc);
 
+pid_t get_init_pid(const char *name)
+{
+	struct lxc_command command = {
+		.request = { .type = LXC_COMMAND_PID },
+	};
+
+	int ret, stopped = 0;
+
+	ret = lxc_command(name, &command, &stopped);
+	if (ret < 0 && stopped) {
+		INFO("'%s' is already stopped", name);
+		return 0;
+	}
+
+	if (ret < 0) {
+		ERROR("failed to send command");
+		return -1;
+	}
+
+	if (command.answer.ret) {
+		ERROR("failed to retrieve the init pid: %s",
+		      strerror(-command.answer.ret));
+		return -1;
+	}
+
+	return command.answer.pid;
+}
+
 int lxc_checkpoint(const char *name, const char *statefile, int flags)
 {
 	return 0;

index b0b9f4e..bd87bdb 100644
@@ -27,6 +27,7 @@
 extern "C" {
 #endif
 
+#include <unistd.h>
 #include <stddef.h>
 #include <lxc/state.h>
 

@@ -56,6 +57,11 @@ extern int lxc_start(const char *name, char *const argv[], struct lxc_conf *);
 extern int lxc_stop(const char *name);
 
 /*
+ * Get the pid of the root application process tree in parent-pid namespace
+ */
+extern pid_t get_init_pid(const char *name);
+
+/*
  * Open the monitoring mechanism for a specific container
  * The function will return an fd corresponding to the events
  * Returns a file descriptor on success, < 0 otherwise

index a012c2c..3d2cdd5 100644
@@ -28,6 +28,7 @@
 #include "commands.h"
 #include "arguments.h"
 #include "namespace.h"
+#include "lxc.h"
 #include "log.h"
 
 lxc_log_define(lxc_attach_ui, lxc);

@@ -50,34 +51,6 @@ Options :\n\
 	.checker  = NULL,
 };
 
-pid_t get_init_pid(const char *name)
-{
-	struct lxc_command command = {
-		.request = { .type = LXC_COMMAND_PID },
-	};
-
-	int ret, stopped = 0;
-
-	ret = lxc_command(name, &command, &stopped);
-	if (ret < 0 && stopped) {
-		INFO("'%s' is already stopped", name);
-		return 0;
-	}
-
-	if (ret < 0) {
-		ERROR("failed to send command");
-		return -1;
-	}
-
-	if (command.answer.ret) {
-		ERROR("failed to retrieve the init pid: %s",
-		      strerror(-command.answer.ret));
-		return -1;
-	}
-
-	return command.answer.pid;
-}
-
 int main(int argc, char *argv[], char *envp[])
 {
 	int ret;

1.6.6.1
---
 src/lxc/restart.c |  129 ++++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 files changed, 128 insertions(+), 1 deletions(-)
index 467489e..220acbf 100644
@@ -22,11 +22,138 @@
  */
 #include <lxc/lxc.h>
 #include <lxc/log.h>
+#include <lxc/start.h>
+#include <lxc/namespace.h>
+#include <lxc/cgroup.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <signal.h>
+#include <sys/prctl.h>
+#include <sys/wait.h>
+#include "error.h"
+#ifdef LIBCR
+#include <app-checkpoint.h>
+#endif
 
 lxc_log_define(lxc_restart, lxc);
 
+#ifdef LIBCR
+
+struct lxc_restart_arg {
+	const char *name;
+	const char *statefile;
+	char *const argv;
+	struct lxc_handler *handler;
+	int lxc_flags;
+};
+
+static int do_restart(struct lxc_restart_arg *lxcarg)
+{
+	int pid;
+	int lxc_flags = lxcarg->lxc_flags;
+	struct lxc_handler *handler = lxcarg->handler;
+	const char *statefile = lxcarg->statefile;
+	struct app_restart_args restart_args;
+
+        if (sigprocmask(SIG_SETMASK, &handler->oldmask, NULL)) {
+                SYSERROR("failed to set sigprocmask");
+                return -1;
+        }
+
+        if (prctl(PR_SET_PDEATHSIG, SIGKILL, 0, 0, 0)) {
+                SYSERROR("failed to set pdeath signal");
+                return -1;
+        }
+
+	memset(&restart_args, 0, sizeof(restart_args));
+
+	restart_args.infd = open(statefile, O_RDONLY, 0);
+	if (restart_args.infd < 0) {
+		SYSERROR("Failed to open statefile %s\n", statefile);
+		return -1;
+	}
+
+	restart_args.pids = 1;
+	restart_args.pidns = 1;
+	restart_args.mnt_pty = 1;
+	restart_args.mntns = 1;
+	restart_args.klogfd = -1;
+	restart_args.ulogfd = lxc_log_fd;
+	restart_args.uerrfd = fileno(stderr);
+	restart_args.debug = 1;
+	restart_args.wait = 0;
+
+	if (lxc_flags & LXC_FLAG_PAUSE)
+		restart_args.keep_frozen = 1;
+
+        pid = app_restart(&restart_args);
+
+        return pid;
+}
+
 int lxc_restart(const char *name, const char *statefile, struct lxc_conf *conf,
-		int flags)
+		int lxc_flags)
 {
+	int err;
+	int status;
+	struct lxc_handler *handler;
+	struct lxc_restart_arg lxcarg = {
+		.name = name,
+		.statefile = statefile,
+		.lxc_flags = lxc_flags,
+		.handler = NULL,
+	};
+
+	if (lxc_check_inherited())
+		return -1;
+
+	handler = lxc_init(name, conf);
+	if (!handler) {
+		ERROR("failed to initialize the container");
+		return -1;
+	}
+
+	lxcarg.handler = handler;
+	handler->pid = do_restart(&lxcarg);
+
+	INFO("do_restart(): returns pid %d\n", handler->pid);
+	lxc_rename_nsgroup(name, handler);
+
+	err = lxc_poll(name, handler);
+	if (err) {
+		ERROR("mainloop exited with an error");
+		goto out_abort;
+	}
+
+	while (waitpid(handler->pid, &status, 0) < 0 && errno == EINTR)
+		continue;
+
+	if (!(lxc_flags & LXC_FLAG_PAUSE)) {
+		err = lxc_unfreeze(name);
+		if (err) {
+			ERROR("lxc_restart(): Unable to unfreeze\n");
+			goto out_fini;
+		}
+	}
+
+	err =  lxc_error_set_and_log(handler->pid, status);
+
+out_fini:
+	lxc_fini(name, handler);
+	return err;
+
+out_abort:
+	lxc_abort(name, handler);
+	goto out_fini;
+
 	return 0;
 }
+#else
+int lxc_restart(const char *name, const char *statefile, struct lxc_conf *conf,
+		int lxc_flags)
+{
+	ERROR("'restart' function not configured");
+	ERROR("Try --with-libcr option to 'configure' script");
+	return -1;
+}
+#endif

1.6.6.1
---
 src/lxc/checkpoint.c |   88 ++++++++++++++++++++++++++++++++++++++++++++++++-
 src/lxc/state.c      |    1 -
 2 files changed, 86 insertions(+), 3 deletions(-)
index 4e75cb6..6e905ff 100644
@@ -22,7 +22,17 @@
  */
 #include <lxc/lxc.h>
 #include <lxc/log.h>
-#include <lxc/commands.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <linux/checkpoint.h>
+
+#include "commands.h"
+#include "arguments.h"
+#include "namespace.h"
+#ifdef LIBCR
+#include "app-checkpoint.h"
+#endif
 
 lxc_log_define(lxc_checkpoint, lxc);
 

@@ -54,7 +64,81 @@ pid_t get_init_pid(const char *name)
 	return command.answer.pid;
 }
 
-int lxc_checkpoint(const char *name, const char *statefile, int flags)
+#ifdef LIBCR
+int lxc_checkpoint(const char *name, const char *statefile, int lxc_flags)
 {
+	int ret;
+	int pid;
+	int flags;
+	struct app_checkpoint_args crargs;
+
+	pid = get_init_pid(name);
+	if (pid < 0) {
+		ERROR("Unable to find cinit pid");
+		return -1;
+	}
+
+	ret = lxc_freeze(name);
+	if (ret < 0)
+		return ret;
+
+	memset(&crargs, 0, sizeof(crargs));
+
+	ret = open(statefile, O_CREAT|O_RDWR|O_EXCL, 0600);
+	if (ret < 0) {
+		ERROR("open(%s) failed, %s\n", statefile, strerror(errno));
+		return -1;
+	}
+
+	crargs.outfd = ret;
+	crargs.logfd = lxc_log_fd;
+	crargs.uerrfd = lxc_log_fd;
+	/*
+	 * TODO: Set this to 0 for now - otherwise we get an objhash leak
+	 * 	 due to mismatched references to current PTY which needs to
+	 * 	 be investigated.
+	 *
+	 * TODO: Map @lxc_flags to user-cr flags ?
+	 *
+	 * TODO: We can probably drop the ->container field since @flags
+	 * 	 can provide the same selection. 
+	 *
+	 * TODO: Do we may need a --container option to lxc_checkpoint or
+	 * 	 assume that we always work with full containers ?
+	 */
+	crargs.container = 0;
+
+	/*
+	 * TODO: Set the CHECKPOINT_NONETNS unconditionally for now. Otherwise
+	 * 	 it would require running the application in a private netns.
+	 * 	 Implement a command-line option to allow user selection.
+	 */
+	flags = CHECKPOINT_SUBTREE|CHECKPOINT_NONETNS;
+
+	ret = app_checkpoint(pid, flags, &crargs);
+	if (ret < 0) {
+		ERROR("checkpoint of %s (pid %d) failed\n", name, pid);
+		return -1;
+	}
+
+	if (lxc_flags & LXC_FLAG_HALT) {
+		ret = lxc_stop(name);
+		if (ret < 0)
+			return ret;
+
+		return lxc_unfreeze(name);
+	}
+
+	if (!(lxc_flags & LXC_FLAG_PAUSE))
+		return lxc_unfreeze(name);
+
 	return 0;
 }
+#else
+int lxc_checkpoint(const char *name, const char *statefile, int lxc_flags)
+{
+	ERROR("'checkpoint' not configured");
+	ERROR("Try --with-libcr option to 'configure' script");
+	return -1;
+}
+#endif

index b29ae09..1e4c7e1 100644
@@ -167,4 +167,3 @@ extern int lxc_state_callback(int fd, struct lxc_request *request,
 out:
 	return ret;
 }
-

1.6.6.1
---
 src/lxc/checkpoint.c |   88 ++++++++++++++++++++++++++++++++++++++++++++++++-
 src/lxc/state.c      |    1 -
 2 files changed, 86 insertions(+), 3 deletions(-)
index 4e75cb6..6e905ff 100644
@@ -22,7 +22,17 @@
  */
 #include <lxc/lxc.h>
 #include <lxc/log.h>
-#include <lxc/commands.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <linux/checkpoint.h>
+
+#include "commands.h"
+#include "arguments.h"
+#include "namespace.h"
+#ifdef LIBCR
+#include "app-checkpoint.h"
+#endif
 
 lxc_log_define(lxc_checkpoint, lxc);
 

@@ -54,7 +64,81 @@ pid_t get_init_pid(const char *name)
 	return command.answer.pid;
 }
 
-int lxc_checkpoint(const char *name, const char *statefile, int flags)
+#ifdef LIBCR
+int lxc_checkpoint(const char *name, const char *statefile, int lxc_flags)
 {
+	int ret;
+	int pid;
+	int flags;
+	struct app_checkpoint_args crargs;
+
+	pid = get_init_pid(name);
+	if (pid < 0) {
+		ERROR("Unable to find cinit pid");
+		return -1;
+	}
+
+	ret = lxc_freeze(name);
+	if (ret < 0)
+		return ret;
+
+	memset(&crargs, 0, sizeof(crargs));
+
+	ret = open(statefile, O_CREAT|O_RDWR|O_EXCL, 0600);
+	if (ret < 0) {
+		ERROR("open(%s) failed, %s\n", statefile, strerror(errno));
+		return -1;
+	}
+
+	crargs.outfd = ret;
+	crargs.logfd = lxc_log_fd;
+	crargs.uerrfd = lxc_log_fd;
+	/*
+	 * TODO: Set this to 0 for now - otherwise we get an objhash leak
+	 * 	 due to mismatched references to current PTY which needs to
+	 * 	 be investigated.
+	 *
+	 * TODO: Map @lxc_flags to user-cr flags ?
+	 *
+	 * TODO: We can probably drop the ->container field since @flags
+	 * 	 can provide the same selection. 
+	 *
+	 * TODO: Do we may need a --container option to lxc_checkpoint or
+	 * 	 assume that we always work with full containers ?
+	 */
+	crargs.container = 0;
+
+	/*
+	 * TODO: Set the CHECKPOINT_NONETNS unconditionally for now. Otherwise
+	 * 	 it would require running the application in a private netns.
+	 * 	 Implement a command-line option to allow user selection.
+	 */
+	flags = CHECKPOINT_SUBTREE|CHECKPOINT_NONETNS;
+
+	ret = app_checkpoint(pid, flags, &crargs);
+	if (ret < 0) {
+		ERROR("checkpoint of %s (pid %d) failed\n", name, pid);
+		return -1;
+	}
+
+	if (lxc_flags & LXC_FLAG_HALT) {
+		ret = lxc_stop(name);
+		if (ret < 0)
+			return ret;
+
+		return lxc_unfreeze(name);
+	}
+
+	if (!(lxc_flags & LXC_FLAG_PAUSE))
+		return lxc_unfreeze(name);
+
 	return 0;
 }
+#else
+int lxc_checkpoint(const char *name, const char *statefile, int lxc_flags)
+{
+	ERROR("'checkpoint' not configured");
+	ERROR("Try --with-libcr option to 'configure' script");
+	return -1;
+}
+#endif

index b29ae09..1e4c7e1 100644

@@ -167,4 +167,3 @@ extern int lxc_state_callback(int fd, struct lxc_request *request,
 out:
 	return ret;
 }
-

index b29ae09..1e4c7e1 100644
@@ -167,4 +167,3 @@ extern int lxc_state_callback(int fd, struct lxc_request *request,
  out:
  	return ret;
  }
-

	$ cd /root
	$ git-clone git://git.ncl.cs.columbia.edu/pub/git/user-cr.git user-cr
	$ cd user-cr
	$ git-checkout ckpt-v20-dev
	  	Tested with commit e275f77e4a82d228c1df14dbeb691342e32cdac2
		as HEAD.
	
	# Apply following two patches:

	https://lists.linux-foundation.org/pipermail/containers/2010-March/024037.html
	https://lists.linux-foundation.org/pipermail/containers/2010-March/024038.html

	$ cd /root/user-cr

	$ KERNELSRC=/root/linux-2.6/ make 

		Build USERCR by pointing to corresponding kernel-source.
		This should create restart.o and checkpoint.o needed by LXC.
	$ cd /root/lxc.git
	$ lxc-execute --name foo --rcfile lxc-no-netns.conf -- /bin/sleep 1000
	$ lxc-checkpoint --name foo --image /root/lxc-foo.ckpt
	$ lxc-stop --name foo
	$ lxc-restart --name foo --image /root/lxc-foo.ckpt
	$ lxc-execute --name foo --rcfile lxc-no-netns.conf -- /bin/sleep 1000
	$ lxc-checkpoint --name foo --image /root/lxc-foo.ckpt
	$ lxc-stop --name foo
	$ lxc-restart --name foo --image /root/lxc-foo.ckpt
	# pi1.sh
	#
	# IBM Confidential
	#
	# OCO Source Materials
	#
	# P91223
	#
	# (C) Copyright IBM Corp. 2003, 2008
	#
	# The source code for this program is not published or otherwise
	# divested of its trade secrets, irrespective of what has been
	# deposited with the U.S. Copyright Office.
	#!/bin/sh

	me=`basename "$0" 2>/dev/null`
	ndigit=${1:-1000}

	echo $me - $ndigit digits
	echo "scale=$ndigit;a(1)*4" | bc -l
	$ lxc-execute -n pi1 -- ./pi1.sh 3000 &
	$ lxc-checkpoint -n pi1 -d ./ckpt -k
	$ lxc-restart -n pi1 -d ./ckpt
	# pi1.sh
	#
	# IBM Confidential
	#
	# OCO Source Materials
	#
	# P91223
	#
	# (C) Copyright IBM Corp. 2003, 2008
	#
	# The source code for this program is not published or otherwise
	# divested of its trade secrets, irrespective of what has been
	# deposited with the U.S. Copyright Office.
	#!/bin/sh

	me=`basename "$0" 2>/dev/null`
	ndigit=${1:-1000}

	echo $me - $ndigit digits
	echo "scale=$ndigit;a(1)*4" | bc -l
	$ lxc-execute -n pi1 -- ./pi1.sh 3000&
	$ lxc-checkpoint -n pi1 -d ./ckpt -k
	$ lxc-restart -n pi1 -d ./ckpt
	#!/bin/sh

	me=`basename "$0" 2>/dev/null`
	ndigit=${1:-1000}

	echo $me - $ndigit digits
	echo "scale=$ndigit;a(1)*4" | bc -l
	$ lxc-execute -n pi1 -- ./pi1.sh 3000&
	$ lxc-checkpoint -n pi1 -d ./ckpt -k
	$ lxc-restart -n pi1 -d ./ckpt
	$ cd /root
	$ git-clone git://git.ncl.cs.columbia.edu/pub/git/user-cr.git user-cr
	$ cd user-cr
	$ git-checkout ckpt-v20-dev
	  	Tested with commit e275f77e4a82d228c1df14dbeb691342e32cdac2
		as HEAD.
	
	# Apply following two patches:

	https://lists.linux-foundation.org/pipermail/containers/2010-March/024037.html
	https://lists.linux-foundation.org/pipermail/containers/2010-March/024038.html

	$ cd /root/user-cr

	$ KERNELSRC=/root/linux-2.6/ make 
	$ cd /root
	$ git-clone git://git.ncl.cs.columbia.edu/pub/git/user-cr.git user-cr
	$ cd user-cr
	$ git-checkout ckpt-v20-dev
	  	Tested with commit e275f77e4a82d228c1df14dbeb691342e32cdac2
		as HEAD.
	
	# Apply following two patches:

	https://lists.linux-foundation.org/pipermail/containers/2010-March/024037.html
	https://lists.linux-foundation.org/pipermail/containers/2010-March/024038.html

	$ cd /root/user-cr
	$ KERNELSRC=/root/linux-2.6/ make 
   http://www.linux-cr.org/git/?p=linux-cr.git;a=summary
	$ cd /root
	$ git-clone git://git.ncl.cs.columbia.edu/pub/git/user-cr.git user-cr
	$ cd user-cr
	$ git-checkout ckpt-v20-dev
	  	Tested with commit e275f77e4a82d228c1df14dbeb691342e32cdac2
		as HEAD.
	
	# Apply following two patches:

	https://lists.linux-foundation.org/pipermail/containers/2010-March/024037.html
	https://lists.linux-foundation.org/pipermail/containers/2010-March/024038.html

	$ cd /root/user-cr

	$ KERNELSRC=/root/linux-2.6/ make 

		Build USERCR by pointing to corresponding kernel-source.
		This should create restart.o and checkpoint.o needed by LXC.
	$ cd /root/lxc.git
	Apply attached patches to LXC (I tested with these patches applied
	to commit 9ea8066aa67b808f71f46e346bd7a215e2a355f3)
	$ lxc-execute --name foo --rcfile lxc-no-netns.conf -- /bin/sleep 1000
	$ lxc-checkpoint --name foo --image /root/lxc-foo.ckpt
	$ lxc-execute --name foo --rcfile lxc-no-netns.conf -- /bin/sleep 1000
	$ lxc-checkpoint --name foo --image /root/lxc-foo.ckpt
	$ lxc-stop --name foo
	$ lxc-restart --name foo --image /root/lxc-foo.ckpt
	$ lxc-execute --name foo --rcfile lxc-no-netns.conf -- /bin/sleep 1000
	$ lxc-checkpoint --name foo --image /root/lxc-foo.ckpt
	$ lxc-stop --name foo
	$ lxc-restart --name foo --image /root/lxc-foo.ckpt
	$ lxc-execute --name foo --rcfile lxc-no-netns.conf -- /bin/sleep 1000
	$ lxc-checkpoint --name foo --image /root/lxc-foo.ckpt
	$ lxc-stop --name foo
	$ lxc-restart --name foo --image /root/lxc-foo.ckpt
	$ lxc-execute --name foo --rcfile lxc-no-netns.conf -- /bin/sleep 1000
	$ lxc-checkpoint --name foo --image /root/lxc-foo.ckpt
	$ lxc-stop --name foo
	$ lxc-restart --name foo --image /root/lxc-foo.ckpt
{
        int i, n, fd;
        char buf[256];

        fd = open("/tmp/foo", O_CREAT|O_RDWR|O_TRUNC, 0666);
        if (fd < 0) {
                perror("open()");
                exit(1);
        }

        for (i = 0; i < 1000; i++) {
                sprintf(buf, "i %d\n", i);
                n = write(fd, buf, strlen(buf));
                if (n != strlen(buf))
                        perror("write()");
                sleep(1);
        }
}
	$ lxc-execute --name foo --rcfile lxc-no-netns.conf -- /bin/sleep 1000
	$ lxc-checkpoint --name foo --image /root/lxc-foo.ckpt
	$ lxc-stop --name foo
	$ lxc-restart --name foo --image /root/lxc-foo.ckpt

Link LXC with USERCR

Sukadev Bhattiprolu

Sukadev Bhattiprolu

Michel Normand

Sukadev Bhattiprolu

Sukadev Bhattiprolu

Michel Normand

Sukadev Bhattiprolu

Sukadev Bhattiprolu

Cedric Le Goater

Sukadev Bhattiprolu

Sukadev Bhattiprolu

Michel Normand

Cedric Le Goater

Michel Normand

Cedric Le Goater

Cedric Le Goater

Sukadev Bhattiprolu

Daniel Lezcano

Oren Laadan

Daniel Lezcano

Daniel Lezcano

Serge E. Hallyn

Daniel Lezcano

Daniel Lezcano

Sukadev Bhattiprolu

Daniel Lezcano

Sukadev Bhattiprolu

Oren Laadan