broken out c/r patches

RFC PATCH New Dave Hansen 1 participant 4 messages Aug 4, 2008

Subsystems (2)

x86 Kbuild

lore

Share & tools

lore

Dave Hansen

author

2008-08-04 20:31 UTC #0 lore raw

cover broken out c/r patches

I've done a bit of refactoring to Oren's patches.

I wonder if they're in a state that people think we can share on LKML like Ted suggested. Thoughts?

At the containers mini-conference before OLS, the consensus among all the stakeholders was that doing checkpoint/restart in the kernel as much as possible was the best approach. With this approach, the kernel will export a relatively opaque 'blob' of data to userspace which can then be handed to the new kernel at restore time.

This is different that what had been proposed before, which was that a userspace application would be responsible for collecting all of this data. We were also planning on adding lots of new, little kernel interfaces for all of the things that needed checkpointing. This unites those into a single, grand interface.

The 'blob' will contain copies of select portions of kernel structures such as vmas and mm_structs. It will also contain copies of the actual memory that the process uses. Any changes in this blob's format between kernel revisions can be handled by an in-userspace conversion program.

This is a similar approach to virtually all of the commercial checkpoint/restart products out there, as well as the research project Zap.

These patches basically serialize internel kernel state and write it out to a file descriptor. The checkpoint and restore are done with two new system calls: sys_checkpoint and sys_restart.

In this incarnation, they can only work checkpoint and restore a single task. The task's address space may consist of only private, simple vma's - anonymous or file-mapped.

Dave Hansen

author

2008-08-04 20:31 UTC #1 lore raw

RFC PATCH 1/3 kernel-based checkpoint-restart: general infrastructure

From: Oren Laadan <[email protected]>

This patch adds those interfaces, as well as all of the helpers needed to easily manage the file format.

The code is roughly broken out as follows:

ckpt/sys.c - user/kernel data transfer, as well as setting up of the
checkpoint/restart context (a per-checkpoint data structure for housekeeping)
ckpt/checkpoint.c - output wrappers and basic checkpoint handling ckpt/restart.c - input wrappers and basic restart handling

Patches to add the per-architecture support as well as the actual work to do the memory checkpoint follow in subsequent patches.

Signed-off-by Oren Laadan <[email protected]>

---

 linux-2.6.git-dave/Makefile          |    2 
 linux-2.6.git-dave/ckpt/Makefile     |    1 
 linux-2.6.git-dave/ckpt/checkpoint.c |  207 +++++++++++++++++++++++++++++++
 linux-2.6.git-dave/ckpt/ckpt.h       |   82 ++++++++++++
 linux-2.6.git-dave/ckpt/ckpt_hdr.h   |   69 ++++++++++
 linux-2.6.git-dave/ckpt/restart.c    |  189 ++++++++++++++++++++++++++++
 linux-2.6.git-dave/ckpt/sys.c        |  233 +++++++++++++++++++++++++++++++++++
 7 files changed, 782 insertions(+), 1 deletion(-)

ckpt/checkpoint.c new file +207 -0

diff -puN /dev/null ckpt/checkpoint.c
@@ -0,0 +1,207 @@
+/*
+ *  Checkpoint logic and helpers
+ *
+ *  Copyright (C) 2008 Oren Laadan
+ *
+ *  This file is subject to the terms and conditions of the GNU General Public
+ *  License.  See the file COPYING in the main directory of the Linux
+ *  distribution for more details.
+ */
+
+#include <linux/version.h>
+#include <linux/sched.h>
+#include <linux/time.h>
+#include <linux/fs.h>
+#include <linux/file.h>
+#include <linux/dcache.h>
+#include <linux/mount.h>
+#include <asm/ptrace.h>
+
+#include "ckpt.h"
+#include "ckpt_hdr.h"
+
+/**
+ * cr_get_fname - return pathname of a given file
+ * @file: file pointer
+ * @buf: buffer for pathname
+ * @n: buffer length (in) and pathname length (out)
+ *
+ * if the buffer provivded by the caller is too small, allocate a new
+ * buffer; caller should call cr_put_pathname() for cleanup
+ */
+char *cr_get_fname(struct path *path, struct path *root, char *buf, int *n)
+{
+	char *fname;
+
+	fname = __d_path(path, root, buf, *n);
+
+	if (IS_ERR(fname) && PTR_ERR(fname) == -ENAMETOOLONG) {
+		 if (!(buf = (char *) __get_free_pages(GFP_KERNEL, 0)))
+			 return ERR_PTR(-ENOMEM);
+		fname = __d_path(path, root, buf, PAGE_SIZE);
+		if (IS_ERR(fname))
+			free_pages((unsigned long) buf, 0);
+	}
+	if (!IS_ERR(fname))
+		*n = (buf + *n - fname);
+
+	return fname;
+}
+
+/**
+ * cr_put_fname - (possibly) cleanup pathname buffer
+ * @buf: original buffer that was given to cr_get_pathname()
+ * @fname: resulting pathname from cr_get_pathname()
+ * @n: length of original buffer
+ */
+void cr_put_fname(char *buf, char *fname, int n)
+{
+	if (fname && (fname < buf || fname >= buf + n))
+		free_pages((unsigned long) buf, 0);
+}
+
+/**
+ * cr_write_obj - write a record described by a cr_hdr
+ * @ctx: checkpoint context
+ * @h: record descriptor
+ * @buf: record buffer
+ */
+int cr_write_obj(struct cr_ctx *ctx, struct cr_hdr *h, void *buf)
+{
+	int ret;
+
+	if ((ret = cr_kwrite(ctx, h, sizeof(*h))) < 0)
+		return ret;
+	return cr_kwrite(ctx, buf, h->len);
+}
+
+/**
+ * cr_write_str - write a string record
+ * @ctx: checkpoint context
+ * @str: string buffer
+ * @n: string length
+ */
+int cr_write_str(struct cr_ctx *ctx, char *str, int n)
+{
+	struct cr_hdr h;
+
+	h.type = CR_HDR_STR;
+	h.len = n;
+	h.id = 0;
+
+	return cr_write_obj(ctx, &h, str);
+}
+
+/* write the checkpoint header */
+static int cr_write_hdr(struct cr_ctx *ctx)
+{
+	struct cr_hdr h;
+	struct cr_hdr_head *hh = ctx->tbuf;
+	struct timeval ktv;
+
+	h.type = CR_HDR_HEAD;
+	h.len = sizeof(*hh);
+	h.id = 0;
+
+	do_gettimeofday(&ktv);
+
+	hh->magic = 0x00a2d200;
+	hh->major = (LINUX_VERSION_CODE >> 16) & 0xff;
+	hh->minor = (LINUX_VERSION_CODE >> 8) & 0xff;
+	hh->patch = (LINUX_VERSION_CODE) & 0xff;
+
+	hh->version = 1;
+
+	hh->flags = ctx->flags;
+	hh->time = ktv.tv_sec;
+
+	return cr_write_obj(ctx, &h, hh);
+}
+
+/* write the checkpoint trailer */
+static int cr_write_tail(struct cr_ctx *ctx)
+{
+	struct cr_hdr h;
+	struct cr_hdr_tail *hh = ctx->tbuf;
+
+	h.type = CR_HDR_TAIL;
+	h.len = sizeof(*hh);
+	h.id = 0;
+
+	hh->magic = 0x002d2a00;
+	hh->cksum[0] = hh->cksum[1] = 1;	/* TBD ... */
+
+	return cr_write_obj(ctx, &h, hh);
+}
+
+/* dump the task_struct of a given task */
+static int cr_write_task_struct(struct cr_ctx *ctx, struct task_struct *t)
+{
+	struct cr_hdr h;
+	struct cr_hdr_task *hh = ctx->tbuf;
+
+	h.type = CR_HDR_TASK;
+	h.len = sizeof(*hh);
+	h.id = ctx->pid;
+
+	hh->state = t->state;
+	hh->exit_state = t->exit_state;
+	hh->exit_code = t->exit_code;
+	hh->exit_signal = t->exit_signal;
+
+	hh->pid = t->pid;
+	hh->tgid = t->tgid;
+
+	hh->utime = t->utime;
+	hh->stime = t->stime;
+	hh->utimescaled = t->utimescaled;
+	hh->stimescaled = t->stimescaled;
+	hh->gtime = t->gtime;
+	hh->prev_utime = t->prev_utime;
+	hh->prev_stime = t->prev_stime;
+	hh->nvcsw = t->nvcsw;
+	hh->nivcsw = t->nivcsw;
+	hh->start_time_sec = t->start_time.tv_sec;
+	hh->start_time_nsec = t->start_time.tv_nsec;
+	hh->real_start_time_sec = t->real_start_time.tv_sec;
+	hh->real_start_time_nsec = t->real_start_time.tv_nsec;
+	hh->min_flt = t->min_flt;
+	hh->maj_flt = t->maj_flt;
+
+	hh->task_comm_len = TASK_COMM_LEN;
+	memcpy(hh->comm, t->comm, TASK_COMM_LEN);
+
+	return cr_write_obj(ctx, &h, hh);
+}
+
+/* dump the entire state of a given task */
+static int cr_write_task(struct cr_ctx *ctx, struct task_struct *t)
+{
+	int ret ;
+
+	BUG_ON(t->state == TASK_DEAD);
+
+	ret = cr_write_task_struct(ctx, t);
+	CR_PRINTK("ret (task_struct) %d\n", ret);
+
+	return ret;
+}
+
+int do_checkpoint(struct cr_ctx *ctx)
+{
+	int ret;
+
+	/* FIX: need to test whether container is checkpointable */
+
+	ret = cr_write_hdr(ctx);
+	if (!ret)
+		ret = cr_write_task(ctx, current);
+	if (!ret)
+		ret = cr_write_tail(ctx);
+
+	/* on success, return (unique) checkpoint identifier */
+	if (!ret)
+		ret = ctx->crid;
+
+	return ret;
+}

ckpt/ckpt.h new file +82 -0

82 lines not on this page · raw message

ckpt/ckpt_hdr.h new file +69 -0

69 lines not on this page · raw message

ckpt/Makefile new file +1 -0

diff -puN /dev/null ckpt/Makefile
@@ -0,0 +1 @@
+obj-y += sys.o checkpoint.o restart.o

ckpt/restart.c new file +189 -0

189 lines not on this page · raw message

ckpt/sys.c new file +233 -0

233 lines not on this page · raw message

Makefile renamed from Makefile~handle_a_single_task_with_private_memory_maps +1 -1

diff -puN Makefile~handle_a_single_task_with_private_memory_maps Makefile
@@ -611,7 +611,7 @@ export mod_strip_cmd
 
 
 ifeq ($(KBUILD_EXTMOD),)
-core-y		+= kernel/ mm/ fs/ ipc/ security/ crypto/ block/
+core-y		+= kernel/ mm/ fs/ ipc/ security/ crypto/ block/ ckpt/
 
 vmlinux-dirs	:= $(patsubst %/,%,$(filter %/, $(init-y) $(init-m) \
 		     $(core-y) $(core-m) $(drivers-y) $(drivers-m) \

Dave Hansen

author

2008-08-04 20:31 UTC #2 lore raw

RFC PATCH 2/3 checkpoint/restart: x86 support

In reply to Dave Hansen

The original version of Oren's patch contained a good hunk of #ifdefs. I've extracted all of those and created a bit of an API for new architectures to follow.

Leaving Oren's sign-off because this is all still his code, even though he hasn't seen it mangled like this before.

Signed-off-by Oren Laadan <[email protected]>

---

 linux-2.6.git-dave/ckpt/Makefile          |    1 
 linux-2.6.git-dave/ckpt/checkpoint.c      |    7 
 linux-2.6.git-dave/ckpt/ckpt_arch.h       |    6 
 linux-2.6.git-dave/ckpt/restart.c         |    7 
 linux-2.6.git-dave/ckpt/x86.c             |  269 ++++++++++++++++++++++++++++++
 linux-2.6.git-dave/include/asm-x86/ckpt.h |   46 +++++
 6 files changed, 336 insertions(+)

ckpt/checkpoint.c renamed from ckpt/checkpoint.c~x86_part +7 -0

diff -puN ckpt/checkpoint.c~x86_part ckpt/checkpoint.c
@@ -19,6 +19,7 @@
 
 #include "ckpt.h"
 #include "ckpt_hdr.h"
+#include "ckpt_arch.h"
 
 /**
  * cr_get_fname - return pathname of a given file

@@ -183,6 +184,12 @@ static int cr_write_task(struct cr_ctx *
 
 	ret = cr_write_task_struct(ctx, t);
 	CR_PRINTK("ret (task_struct) %d\n", ret);
+	if (!ret)
+		ret = cr_write_thread(ctx, t);
+	CR_PRINTK("ret (thread) %d\n", ret);
+	if (!ret)
+		ret = cr_write_cpu(ctx, t);
+	CR_PRINTK("ret (cpu) %d\n", ret);
 
 	return ret;
 }

ckpt/ckpt_arch.h new file +6 -0

diff -puN /dev/null ckpt/ckpt_arch.h
@@ -0,0 +1,6 @@
+#include "ckpt.h"
+
+int cr_write_thread(struct cr_ctx *ctx, struct task_struct *t);
+int cr_write_cpu(struct cr_ctx *ctx, struct task_struct *t);
+int cr_read_thread(struct cr_ctx *ctx);
+int cr_read_cpu(struct cr_ctx *ctx);

ckpt/Makefile renamed from ckpt/Makefile~x86_part +1 -0

diff -puN ckpt/Makefile~x86_part ckpt/Makefile
@@ -1 +1,2 @@
 obj-y += sys.o checkpoint.o restart.o
+obj-$(CONFIG_X86) += x86.o

ckpt/restart.c renamed from ckpt/restart.c~x86_part +7 -0

diff -puN ckpt/restart.c~x86_part ckpt/restart.c
@@ -21,6 +21,7 @@
 
 #include "ckpt.h"
 #include "ckpt_hdr.h"
+#include "ckpt_arch.h"
 
 /**
  * cr_hbuf_get - reserve space on the hbuf

@@ -171,6 +172,12 @@ static int cr_read_task(struct cr_ctx *c
 
 	ret = cr_read_task_struct(ctx);
 	CR_PRINTK("ret (task_struct) %d\n", ret);
+	if (!ret)
+		ret = cr_read_thread(ctx);
+	CR_PRINTK("ret (thread) %d\n", ret);
+	if (!ret)
+		ret = cr_read_cpu(ctx);
+	CR_PRINTK("ret (cpu) %d\n", ret);
 
 	return ret;
 }

ckpt/x86.c new file +269 -0

269 lines not on this page · raw message

include/asm-x86/ckpt.h new file +46 -0

46 lines not on this page · raw message

Dave Hansen

author

2008-08-04 20:31 UTC #3 lore raw

RFC PATCH 3/3 checkpoint/restart: memory management

In reply to Dave Hansen

For each vma, there is a 'struct cr_vma'; if the vma is file-mapped, it will be followed by the file name. The cr_vma->npages will tell how many pages were dumped for this vma. Then it will be followed by the actual data: first a dump of the addresses of all dumped pages (npages entries) followed by a dump of the contents of all dumped pages (npages pages). Then will come the next vma and so on.

I guess I could also separate out the x86-specific bits here, but they're pretty small, comparatively.

Signed-off-by Oren Laadan <[email protected]>

---

 linux-2.6.git-dave/arch/x86/kernel/ldt.c  |    2 
 linux-2.6.git-dave/ckpt/Makefile          |    2 
 linux-2.6.git-dave/ckpt/ckpt_arch.h       |    2 
 linux-2.6.git-dave/ckpt/ckpt_hdr.h        |   21 +
 linux-2.6.git-dave/ckpt/ckpt_mem.c        |  388 ++++++++++++++++++++++++++++++
 linux-2.6.git-dave/ckpt/ckpt_mem.h        |   32 ++
 linux-2.6.git-dave/ckpt/rstr_mem.c        |  354 +++++++++++++++++++++++++++
 linux-2.6.git-dave/ckpt/sys.c             |    3 
 linux-2.6.git-dave/ckpt/x86.c             |   83 ++++++
 linux-2.6.git-dave/include/asm-x86/ckpt.h |    5 
 10 files changed, 890 insertions(+), 2 deletions(-)

arch/x86/kernel/ldt.c renamed from arch/x86/kernel/ldt.c~memory_part +1 -1

diff -puN arch/x86/kernel/ldt.c~memory_part arch/x86/kernel/ldt.c
@@ -183,7 +183,7 @@ static int read_default_ldt(void __user 
 	return bytecount;
 }
 
-static int write_ldt(void __user *ptr, unsigned long bytecount, int oldmode)
+int write_ldt(void __user *ptr, unsigned long bytecount, int oldmode)
 {
 	struct mm_struct *mm = current->mm;
 	struct desc_struct ldt;

ckpt/ckpt_arch.h renamed from ckpt/ckpt_arch.h~memory_part +2 -0

diff -puN ckpt/ckpt_arch.h~memory_part ckpt/ckpt_arch.h
@@ -4,3 +4,5 @@ int cr_write_thread(struct cr_ctx *ctx, 
 int cr_write_cpu(struct cr_ctx *ctx, struct task_struct *t);
 int cr_read_thread(struct cr_ctx *ctx);
 int cr_read_cpu(struct cr_ctx *ctx);
+int cr_write_mm_context(struct cr_ctx *ctx, struct mm_struct *mm);
+int cr_read_mm_context(struct cr_ctx *ctx, struct mm_struct *mm);

ckpt/ckpt_hdr.h renamed from ckpt/ckpt_hdr.h~memory_part +21 -0

diff -puN ckpt/ckpt_hdr.h~memory_part ckpt/ckpt_hdr.h
@@ -67,3 +67,24 @@ struct cr_hdr_task {
 };
 
 
+
+struct cr_hdr_mm {
+	__u32 tag;	/* sharing identifier */
+	__u64 start_code, end_code, start_data, end_data;
+	__u64 start_brk, brk, start_stack;
+	__u64 arg_start, arg_end, env_start, env_end;
+	__s16 map_count;
+};
+
+struct cr_hdr_vma {
+	__u32 how;
+
+	__u64 vm_start;
+	__u64 vm_end;
+	__u64 vm_page_prot;
+	__u64 vm_flags;
+	__u64 vm_pgoff;
+
+	__s16 npages;
+	__s16 namelen;
+};

ckpt/ckpt_mem.c new file +388 -0

388 lines not on this page · raw message

ckpt/ckpt_mem.h new file +32 -0

diff -puN /dev/null ckpt/ckpt_mem.h
@@ -0,0 +1,32 @@
+/*
+ *  Generic container checkpoint-restart
+ *
+ *  Copyright (C) 2008 Oren Laadan
+ *
+ *  This file is subject to the terms and conditions of the GNU General Public
+ *  License.  See the file COPYING in the main directory of the Linux
+ *  distribution for more details.
+ */
+
+#include <linux/mm_types.h>
+
+/* page-array chains: each pgarr hols a list of <addr,page> tuples */
+struct cr_pgarr {
+	unsigned long *addrs;
+	struct page **pages;
+	struct cr_pgarr *next;
+	unsigned short nleft;
+	unsigned short nused;
+};
+
+/* vma subtypes */
+enum {
+	CR_VMA_ANON = 1,
+	CR_VMA_FILE
+};
+
+extern void _cr_pgarr_release(struct cr_ctx *ctx, struct cr_pgarr *pgarr);
+extern void cr_pgarr_release(struct cr_ctx *ctx);
+extern void cr_pgarr_free(struct cr_ctx *ctx);
+extern struct cr_pgarr *cr_pgarr_alloc(struct cr_ctx *ctx, struct cr_pgarr **pgnew);
+extern struct cr_pgarr *cr_pgarr_prep(struct cr_ctx *ctx);

ckpt/Makefile renamed from ckpt/Makefile~memory_part +1 -1

diff -puN ckpt/Makefile~memory_part ckpt/Makefile
@@ -1,2 +1,2 @@
-obj-y += sys.o checkpoint.o restart.o
+obj-y += sys.o checkpoint.o restart.o ckpt_mem.o rstr_mem.o
 obj-$(CONFIG_X86) += x86.o

ckpt/rstr_mem.c new file +354 -0

354 lines not on this page · raw message

ckpt/sys.c renamed from ckpt/sys.c~memory_part +3 -0

diff -puN ckpt/sys.c~memory_part ckpt/sys.c
@@ -15,6 +15,7 @@
 #include <linux/capability.h>
 
 #include "ckpt.h"
+#include "ckpt_mem.h"
 
 /*
  * helpers to write/read to/from the image file descriptor

@@ -118,6 +119,8 @@ void cr_ctx_free(struct cr_ctx *ctx)
 	if (ctx->vfsroot)
 		path_put(ctx->vfsroot);
 
+	cr_pgarr_free(ctx);
+
 	free_pages((unsigned long) ctx->tbuf, CR_ORDER_TBUF);
 	free_pages((unsigned long) ctx->hbuf, CR_ORDER_HBUF);

ckpt/x86.c renamed from ckpt/x86.c~memory_part +83 -0

91 lines not on this page · raw message

include/asm-x86/ckpt.h renamed from include/asm-x86/ckpt.h~memory_part +5 -0

diff -puN include/asm-x86/ckpt.h~memory_part include/asm-x86/ckpt.h
@@ -43,4 +43,9 @@ struct cr_hdr_cpu {
 	union thread_xstate xstate;	/* i387 */
 };
 
+struct cr_hdr_mm_context {
+	__s16 ldt_entry_size;
+	__s16 nldt;
+};
+
 #endif /* __ASM_X86_CKPT_H */