Skip to contentSkip to conversation

user-cr: Routing information in userspace

RFC In review Dan Smith 3 participants 7 messages May 26, 2010 – Jun 15, 2010 3 messages not in the archive Kbuild

lore
Share & tools
lore

Dan Smith

author
#0 lore raw

I've been working on doing the c/r of network routes in userspace and this set is what I've got so far. It depends on a patch to iproute2 that the maintainer seemed happy with (quoted in the last patch). It also depends on Eric's setns() patches.

It adds the beginnings of a hook system to checkpoint and restart and then implements a network hook to do the actual work.

If this approach is palatable, I'll finish up the bits to make it conditional at the command-line level, etc and re-submit.

A message that is not in the archive (sent off-list or not yet received): 1274884956-21005-1-git-send-email-danms-r/[email protected]

author
#2 lore raw

RFC PATCH 1/3 user-cr: Add a basic hooks system

In order to allow users to perform parts of the checkpoint/restore themselves, we need a system of "hooks" for the user-cr applications to call at various times. This patch adds a little bit of infrastructure for that, and defines two hooks: "checkpoint-pre" and "restart-post". The checkpoint-pre hook gets run before we call sys_checkpoint() and the restart-post hook gets run after we've restored the task(s).

The infrastructure calls the hooks by name by exec'ing a shell. The environment is set up with information about the task at hand, with variables prefixed by "CR_". For now, only two are defined. CR_ROOT_PID is the pid of the root of the checkpoint or the pid of the root of the restored task tree. CR_BASE_FILE is the name of the file we're checkpointing to or restoring from (or empty if using stdio).

I think it's to be expected that any checkpointing or restarting done by the hooks will be stored separately from the main checkpoint file, so I expect that at least initially we'll use a permutation of the base file's name for those.

Included are two shell scripts in the hooks/ directory which can be modified to add the actual work to be done (and potentially some logic to decide when and how to run some of them).

Thoughts?

---
 Makefile             |    5 +-
 app-checkpoint.h     |    7 +++
 checkpoint-main.c    |    1 +
 checkpoint.c         |    7 +++
 hooks.c              |  118 ++++++++++++++++++++++++++++++++++++++++++++++++++
 hooks.h              |   21 +++++++++
 hooks/checkpoint-pre |    5 ++
 hooks/restart-post   |    5 ++
 restart-main.c       |    1 +
 restart.c            |   11 ++++-
 10 files changed, 178 insertions(+), 3 deletions(-)
 create mode 100644 hooks.c
 create mode 100644 hooks.h
 create mode 100644 hooks/checkpoint-pre
 create mode 100644 hooks/restart-post
Makefile +3 -2
index 6c9ff93..ea86406 100644
@@ -45,6 +45,7 @@ LDLIBS = -lm
 
 all: $(PROGS)
 	@$(MAKE) -C test
+	@$(MAKE) -C hooks
 
 $(LIB_ECLONE):
 	$(AR) ruv $(LIB_ECLONE) $^
@@ -54,9 +55,9 @@ restart: CFLAGS += -D__REENTRANT -pthread
 
 $(CR_OBJS): common.h app-checkpoint.h
 
-restart: restart.o restart-main.o
+restart: restart.o restart-main.o hooks.o
 
-checkpoint: checkpoint.o checkpoint-main.o
+checkpoint: checkpoint.o checkpoint-main.o hooks.o
 
 # eclone() is architecture specific
 ifneq ($(SUBARCH),)
app-checkpoint.h +7 -0
index 8c8ebbb..fe08560 100644
@@ -1,8 +1,12 @@
+#ifndef __APP_CHECKPOINT_H
+#define __APP_CHECKPOINT_H
+
 #include <linux/checkpoint.h>
 #include <linux/checkpoint_hdr.h>
 
 struct app_checkpoint_args {
 	int outfd;
+	char *output;
 	int logfd;
 	int uerrfd;
 	int container;
@@ -24,6 +28,7 @@ struct app_restart_args {
 	char *freezer;
 	int keep_frozen;
 	int infd;
+	char *input;
 	int klogfd;
 	int ulogfd;
 	int uerrfd;
@@ -44,3 +49,5 @@ extern int app_checkpoint(int pid, unsigned long flags,
 				struct app_checkpoint_args *args);
 
 extern int app_restart(struct app_restart_args *args);
+
+#endif
checkpoint-main.c +1 -0
index 24d89eb..1329f84 100644
@@ -131,6 +131,7 @@ static void parse_args(struct app_checkpoint_args *args, int argc, char *argv[])
 			ckpt_perror("open output file");
 			exit(1);
 		}
+		args->output = output;
 	}
 
 	if (logfile && args->logfd >= 0) {
checkpoint.c +7 -0
index e0290c9..2a36df0 100644
@@ -23,6 +23,7 @@
 
 #include "app-checkpoint.h"
 #include "common.h"
+#include "hooks.h"
 
 static int global_uerrfd = -1;
 
@@ -36,6 +37,12 @@ int app_checkpoint(int pid, unsigned long flags,
 {
 	int ret;
 
+	ret = call_checkpoint_hook("pre", pid, args);
+	if (ret) {
+		ckpt_err("Hook checkpoint-pre failed: %i\n", ret);
+		return ret;
+	}
+
 	global_uerrfd = args->uerrfd;
 
 	/* output file descriptor (default: stdout) */
hooks.c new file +118 -0

118 lines not on this page · raw message

hooks.h new file +21 -0
new file mode 100644
index 0000000..132d60e
@@ -0,0 +1,21 @@
+/*
+ *  hooks.h: interfaces for calling hooks
+ *
+ *  Copyright 2010 IBM Corporation
+ *
+ *  This file is subject to the terms and conditions of the GNU General Public
+ *  License.  See the file COPYING in the main directory of the Linux
+ *  distribution for more details.
+ */
+
+#ifndef __HOOKS_H
+#define __HOOKS_H
+
+#include "app-checkpoint.h"
+
+int call_restart_hook(const char *hook, int root_pid,
+		      struct app_restart_args *args);
+int call_checkpoint_hook(const char *hook, int root_pid,
+			 struct app_checkpoint_args *args);
+
+#endif
hooks/checkpoint-pre new file +5 -0
new file mode 100644
index 0000000..c586275
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+echo "========= CHECKPOINT PRE HOOK ============"
+
+echo "========= CHECKPOINT PRE HOOK ============"
hooks/restart-post new file +5 -0
new file mode 100644
index 0000000..dbce024
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+echo "========= RESTART POST HOOK ============"
+
+echo "========= RESTART POST HOOK ============"
restart-main.c +1 -0
index 656d49f..a019bc3 100644
@@ -279,6 +279,7 @@ static void parse_args(struct app_restart_args *args, int argc, char *argv[])
 			ckpt_perror("open input file");
 			exit(1);
 		}
+		args->input = strdup(input);
 	}
 
 	if (klogfile && args->klogfd >= 0) {
restart.c +10 -1
index c5c65a2..c29206a 100644
@@ -42,6 +42,7 @@
 #include "compat.h"
 #include "app-checkpoint.h"
 #include "common.h"
+#include "hooks.h"
 
 /*
  * By default, 'restart' creates a new pid namespace in which the
@@ -877,8 +878,10 @@ static int ckpt_coordinator_pidns(struct ckpt_ctx *ctx)
 	ctx->args->copy_status = copy;
 
 	ret = ckpt_coordinator_status(ctx);
+	if (ret != 0)
+		return ret;
 
-	if (ret == 0 && ctx->args->wait)
+	if (ctx->args->wait)
 		ret = ckpt_collect_child(ctx);
 
 	return ret;
@@ -928,6 +931,12 @@ static int ckpt_coordinator(struct ckpt_ctx *ctx)
 		exit(1);
 	}
 
+	ret = call_restart_hook("post", root_pid, ctx->args);
+	if (ret < 0) {
+		printf("Hook restart-post failed: %i\n", ret);
+		return ret;
+	}
+
 	ckpt_verbose("Success\n");
 	ckpt_dbg("restart succeeded\n");
 
Signature · 1 line
1.7.0.4
author
#3 lore raw

PATCH 2/3 Make extract_headers.sh bring out the __NR_setns symbol

In reply to a message not in the archive
---
 scripts/extract-headers.sh |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)
scripts/extract-headers.sh +1 -1
index 8c8ae69..0deed98 100755
@@ -144,7 +144,7 @@ echo '#endif /* _CHECKPOINT_CKPT_HDR_H_ */' >> "${OUTPUT_INCLUDES}/linux/checkpo
 # We use ARCH_COND to break up architecture-specific sections of the header.
 #
 ARCH_COND='#if'
-REGEX='[[:space:]]*#[[:space:]]*define[[:space:]]*__NR_(checkpoint|restart|clone_with_pids)[[:space:]]+[0-9]+'
+REGEX='[[:space:]]*#[[:space:]]*define[[:space:]]*__NR_(checkpoint|restart|clone_with_pids|setns)[[:space:]]+[0-9]+'
 
 cat - <<-EOFOE
 /*
Signature · 1 line
1.7.0.4

A message that is not in the archive (sent off-list or not yet received): 1274884956-21005-3-git-send-email-danms-r/[email protected]

#5 lore raw

Re: [PATCH 2/3] Make extract_headers.sh bring out the __NR_setns symbol

Quoted 10 lines · Dan Smith
---
 scripts/extract-headers.sh |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)
scripts/extract-headers.sh +0 -0
index 8c8ae69..0deed98 100755

scripts/extract-headers.sh echo '#endif /* _CHECKPOINT_CKPT_HDR_H_ */' >> "${OUTPUT_INCLUDES}/linux/checkpo · line 147

Quoted 5 lines
scripts/extract-headers.sh +1 -1
@@ -144,7 +144,7 @@ echo '#endif /* _CHECKPOINT_CKPT_HDR_H_ */' >> "${OUTPUT_INCLUDES}/linux/checkpo
 # We use ARCH_COND to break up architecture-specific sections of the header.
 #
 ARCH_COND='#if'
-REGEX='[[:space:]]*#[[:space:]]*define[[:space:]]*__NR_(checkpoint|restart|clone_with_pids)[[:space:]]+[0-9]+'
+REGEX='[[:space:]]*#[[:space:]]*define[[:space:]]*__NR_(checkpoint|restart|clone_with_pids|setns)[[:space:]]+[0-9]+'

Which reminds me that "clone_with_pids" is now "eclone" ...

Oren.

Quoted 3 lines
scripts/extract-headers.sh +0 -0
 cat - <<-EOFOE
 /*
author
#6 lore raw

RFC PATCH 3/3 user-cr: Hook implementation for userspace network bits

In reply to a message not in the archive

This patch adds a "net" hook application that (for now) handles the c/r of network routes in a container. At the moment, it just jumps into the netns of the root task and saves or restores the routing information. I think that's a reasonable first-step.

Note that it requires my patch to iproute2 to implement "save" and "restore":

http://marc.info/?l=linux-netdev&m=127446032232507&w=3

On checkpoint, it does a setns() into the network namespace of the root task, and then opens a file and saves the current mapping of device names to indexes, followed by the output of "ip route save".

On restart, it again does a setns() into the network namespace of the root task, and opens the file. This time, it maps the current device names and indexes, but also reads the stored version, and then updates the saved routing information (correcting indexes where necessary) as it feeds the stream to "ip route restore".

There needs to be some more work done, at least the following:
1. Install the binaries into a libexec directory of some sort
2. Allow conditional execution of the net hook
3. Coordinate with the unfreezing of the tasks so that we can restore the routing information first
4. Disable the hook if the checkpoint or restart is working against stdio, or come up with a directory-based approach to storing the checkpoint stream and associated metadata

With this patch, I'm able to checkpoint/restart a container and have any routes saved and restored along with it.

---
 hooks/Makefile       |   17 +++
 hooks/checkpoint-pre |    2 +
 hooks/map_ifs.c      |  268 +++++++++++++++++++++++++++++++++++++
 hooks/map_ifs.h      |   41 ++++++
 hooks/net.c          |  362 ++++++++++++++++++++++++++++++++++++++++++++++++++
 hooks/restart-post   |    2 +
 6 files changed, 692 insertions(+), 0 deletions(-)
 create mode 100644 hooks/Makefile
 create mode 100644 hooks/map_ifs.c
 create mode 100644 hooks/map_ifs.h
 create mode 100644 hooks/net.c
hooks/Makefile new file +17 -0
new file mode 100644
index 0000000..2766388
@@ -0,0 +1,17 @@
+CFLAGS += -I../include
+
+all: net
+
+net.o: map_ifs.c map_ifs.h
+map_ifs.o: map_ifs.c map_ifs.h
+
+net: LDFLAGS += -lnl
+net: map_ifs.o net.o
+
+clean:
+	rm -f net map_ifs *.o
+
+# Test program for map_ifs
+map_ifs: CFLAGS += -D__TEST -lnl
+map_ifs: map_ifs.c map_ifs.h
+	$(CC) -o $@ $^ $(CFLAGS)
hooks/checkpoint-pre +2 -0
index c586275..a4251ca 100644
@@ -2,4 +2,6 @@
 
 echo "========= CHECKPOINT PRE HOOK ============"
 
+net -c -f ${CR_BASE_FILE}.netns ${CR_ROOT_PID}
+
 echo "========= CHECKPOINT PRE HOOK ============"
hooks/map_ifs.c new file +268 -0

268 lines not on this page · raw message

hooks/map_ifs.h new file +41 -0

41 lines not on this page · raw message

hooks/net.c new file +362 -0

362 lines not on this page · raw message

hooks/restart-post +2 -0
index dbce024..a2fa0cb 100644
@@ -2,4 +2,6 @@
 
 echo "========= RESTART POST HOOK ============"
 
+net -r -f ${CR_BASE_FILE}.netns ${CR_ROOT_PID}
+
 echo "========= RESTART POST HOOK ============"
Signature · 1 line
1.7.0.4
#7 lore raw
In reply to a message not in the archive

* Dan Smith <danms-r/[email protected]> wrote:

Hi,

Quoted 4 lines

I've been working on doing the c/r of network routes in userspace and this set is what I've got so far. It depends on a patch to iproute2 that the maintainer seemed happy with (quoted in the last patch). It also depends on Eric's setns() patches.

I'd prefer the routing tables being available via sysfs (eg. /sys/net/ipv4/routes). Containers could live under their own subdirectory (eg. /sys/containers/<id>/).

cu

Signature · 8 lines
---------------------------------------------------------------------
 Enrico Weigelt    ==   metux IT service - http://www.metux.de/
---------------------------------------------------------------------
 Please visit the OpenSource QM Taskforce:
 	http://wiki.metux.de/public/OpenSource_QM_Taskforce
 Patches / Fixes for a lot dozens of packages in dozens of versions:
	http://patches.metux.de/
---------------------------------------------------------------------

A message that is not in the archive (sent off-list or not yet received): [email protected]

author
#9 lore raw

EW> I'd prefer the routing tables being available via sysfs EW> (eg. /sys/net/ipv4/routes). Containers could live under their own EW> subdirectory (eg. /sys/containers/<id>/).

Well, that's a bit of extra work. IIRC, the multiple routing table implementations converge at about the RTNL layer, making it rather difficult to expose them via another interface.

Regardless, it wouldn't change this patch set but by a couple of lines.

Signature · 3 lines
Dan Smith
IBM Linux Technology Center
email: danms-r/[email protected]