Skip to contentSkip to conversation

Sockets as proper objhash objects

PATCH In review Dan Smith 2 participants 7 messages Sep 14, 2009 3 messages not in the archive

Subsystems (3)
lore
Share & tools
lore

Dan Smith

author
#0 lore raw

This version of the set contains a small fix to the first patch, a change suggested by Serge to the second, and a new third to avoid leaking the struct socket object on DEAD sockets. As previously mentioned, this still doesn't implement the collect bits for socket objects, which will follow later. Note that it depends on the yet-to-be-pushed post-files deferqueue patch that Oren has accepted.

A message that is not in the archive (sent off-list or not yet received): 1252946924-4401-1-git-send-email-danms-r/[email protected]

author
#2 lore raw

PATCH 1/3 Make sockets proper objhash objects and use checkpoint_obj() on them (v3)

This changes the checkpoint/restart procedure for sockets a bit. The socket file header is now checkpointed separately from the socket itself, which allows us to checkpoint a socket without arriving at it from a file descriptor. Thus, most sockets will be checkpointed as a result of processing the file table, calling sock_file_checkpoint(fd), which in turn calls checkpoint_obj(socket).

However, we may arrive at some sockets while checkpointing other objects, such as the other end of an AF_UNIX socket with buffers in flight. This patch just opens that door, which is utilized by the next patch.

Changes in v3:
- Fix failure to actually set the SOCK_DEAD flag on restore

Changes in v2:
- If we attempt to checkpoint an orphan socket, create a struct socket to adopt it for the purposes of the checkpoint

---
 checkpoint/objhash.c           |    2 +
 include/linux/checkpoint_hdr.h |    6 +-
 include/linux/net.h            |    4 +-
 include/net/af_unix.h          |    4 +-
 include/net/sock.h             |    2 +
 net/checkpoint.c               |  153 +++++++++++++++++++++++++++++++---------
 net/unix/checkpoint.c          |   11 ++--
 7 files changed, 136 insertions(+), 46 deletions(-)
checkpoint/objhash.c +2 -0
index a9a10d1..a410346 100644
@@ -381,6 +381,8 @@ static struct ckpt_obj_ops ckpt_obj_ops[] = {
 		.obj_type = CKPT_OBJ_SOCK,
 		.ref_drop = obj_sock_drop,
 		.ref_grab = obj_sock_grab,
+		.checkpoint = checkpoint_sock,
+		.restore = restore_sock,
 	},
 };
 
include/linux/checkpoint_hdr.h +4 -2
index 06bc6e2..b75562c 100644
@@ -70,6 +70,7 @@ enum {
 	CKPT_HDR_USER,
 	CKPT_HDR_GROUPINFO,
 	CKPT_HDR_TASK_CREDS,
+	CKPT_HDR_SOCKET,
 
 	/* 201-299: reserved for arch-dependent */
 
@@ -368,7 +369,8 @@ struct ckpt_hdr_file_pipe {
 } __attribute__((aligned(8)));
 
 /* socket */
-struct ckpt_socket {
+struct ckpt_hdr_socket {
+	struct ckpt_hdr h;
 	struct { /* struct socket */
 		__u64 flags;
 		__u8 state;
@@ -428,7 +430,7 @@ struct ckpt_hdr_socket_unix {
 
 struct ckpt_hdr_file_socket {
 	struct ckpt_hdr_file common;
-	struct ckpt_socket socket;
+	__s32 sock_objref;
 } __attribute__((aligned(8)));
 
 struct ckpt_hdr_utsns {
include/linux/net.h +2 -2
index 27187a4..96c7e22 100644
@@ -148,7 +148,7 @@ struct msghdr;
 struct module;
 
 struct ckpt_ctx;
-struct ckpt_socket;
+struct ckpt_hdr_socket;
 
 struct proto_ops {
 	int		family;
@@ -197,7 +197,7 @@ struct proto_ops {
 	int		(*checkpoint)(struct ckpt_ctx *ctx,
 				      struct socket *sock);
 	int		(*restore)(struct ckpt_ctx *ctx, struct socket *sock,
-				   struct ckpt_socket *h);
+				   struct ckpt_hdr_socket *h);
 };
 
 struct net_proto_family {
include/net/af_unix.h +2 -2
index 1a1fd20..61f666b 100644
@@ -71,10 +71,10 @@ static inline void unix_sysctl_unregister(struct net *net) {}
 
 #ifdef CONFIG_CHECKPOINT
 struct ckpt_ctx;
-struct ckpt_socket;
+struct ckpt_hdr_socket;
 extern int unix_checkpoint(struct ckpt_ctx *ctx, struct socket *sock);
 extern int unix_restore(struct ckpt_ctx *ctx, struct socket *sock,
-			struct ckpt_socket *h);
+			struct ckpt_hdr_socket *h);
 #else
 #define unix_checkpoint NULL
 #define unix_restore NULL
include/net/sock.h +2 -0
index 8e3b050..0db1ca3 100644
@@ -1644,6 +1644,8 @@ extern __u32 sysctl_rmem_default;
 /* Checkpoint/Restart Functions */
 struct ckpt_ctx;
 struct ckpt_hdr_file;
+extern int checkpoint_sock(struct ckpt_ctx *ctx, void *ptr);
+extern void *restore_sock(struct ckpt_ctx *ctx);
 extern int sock_file_checkpoint(struct ckpt_ctx *ctx, struct file *file);
 extern struct file *sock_file_restore(struct ckpt_ctx *ctx,
 				      struct ckpt_hdr_file *h);
net/checkpoint.c +119 -34

244 lines not on this page · raw message

net/unix/checkpoint.c +5 -6
index 08e664b..395f6fd 100644
@@ -57,7 +57,6 @@ static int unix_write_cwd(struct ckpt_ctx *ctx,
 int unix_checkpoint(struct ckpt_ctx *ctx, struct socket *sock)
 {
 	struct unix_sock *sk = unix_sk(sock->sk);
-	struct unix_sock *pr = unix_sk(sk->peer);
 	struct ckpt_hdr_socket_unix *un;
 	int new;
 	int ret = -ENOMEM;
@@ -86,7 +85,7 @@ int unix_checkpoint(struct ckpt_ctx *ctx, struct socket *sock)
 		goto out;
 
 	if (sk->peer)
-		un->peer = ckpt_obj_lookup_add(ctx, pr, CKPT_OBJ_SOCK, &new);
+		un->peer = checkpoint_obj(ctx, sk->peer, CKPT_OBJ_SOCK);
 	else
 		un->peer = 0;
 
@@ -237,7 +236,7 @@ static int unix_join(struct ckpt_ctx *ctx,
 }
 
 static int unix_restore_connected(struct ckpt_ctx *ctx,
-				  struct ckpt_socket *h,
+				  struct ckpt_hdr_socket *h,
 				  struct ckpt_hdr_socket_unix *un,
 				  struct socket *sock)
 {
@@ -423,7 +422,7 @@ static int unix_fakebind(struct socket *sock,
 	return 0;
 }
 
-static int unix_restore_bind(struct ckpt_socket *h,
+static int unix_restore_bind(struct ckpt_hdr_socket *h,
 			     struct ckpt_hdr_socket_unix *un,
 			     struct socket *sock,
 			     const char *path)
@@ -440,7 +439,7 @@ static int unix_restore_bind(struct ckpt_socket *h,
 }
 
 /* Some easy pre-flight checks before we get underway */
-static int unix_precheck(struct socket *sock, struct ckpt_socket *h)
+static int unix_precheck(struct socket *sock, struct ckpt_hdr_socket *h)
 {
 	struct net *net = sock_net(sock->sk);
 
@@ -471,7 +470,7 @@ static int unix_precheck(struct socket *sock, struct ckpt_socket *h)
 }
 
 int unix_restore(struct ckpt_ctx *ctx, struct socket *sock,
-		      struct ckpt_socket *h)
+		      struct ckpt_hdr_socket *h)
 
 {
 	struct ckpt_hdr_socket_unix *un;
Signature · 1 line
1.6.2.5
author
#3 lore raw

PATCH 2/3 Track socket buffer owners (v3)

In reply to a message not in the archive

This patch is a superset of the previous attempt to store socket buffers with their owners. It defers the writing and reading of the socket buffers until after the end of the file phase to avoid a recursive nose-dive of checkpointing socket owners.

This also moves the join logic to the deferqueue as well, since that too can lead us down a deep hole. The buffer restore logic may perform a join if it decides that the join is inevitable (but not yet performed) and necessary.

Changes in v3:
- Reverse order of length and capable(CAP_NET_ADMIN) checks to avoid setting PF_SUPERPRIV too early, per Serge's recommendation

Changes in v2:
- Add comment about using deferqueue_add() in a function that could have been called during deferqueue_run()
- Change 'ref' to 'objref' in several variable names
- Move SOCK_DEAD flag detection to initial patch
- Catch errors from unix_read_buffers(), even on the should-be-missing send buffers

---
 include/linux/checkpoint.h     |    3 -
 include/linux/checkpoint_hdr.h |    6 +
 net/checkpoint.c               |  120 +++++++++------
 net/unix/checkpoint.c          |  336 ++++++++++++++++++++++++++--------------
 4 files changed, 303 insertions(+), 162 deletions(-)
include/linux/checkpoint.h +0 -3
index 761cad5..88861a1 100644
@@ -84,9 +84,6 @@ extern int ckpt_sock_getnames(struct ckpt_ctx *ctx,
 			      struct socket *socket,
 			      struct sockaddr *loc, unsigned *loc_len,
 			      struct sockaddr *rem, unsigned *rem_len);
-extern struct ckpt_hdr_socket_queue *
-ckpt_sock_read_buffer_hdr(struct ckpt_ctx *ctx,
-			  uint32_t *bufsize);
 
 /* ckpt kflags */
 #define ckpt_set_ctx_kflag(__ctx, __kflag)  \
include/linux/checkpoint_hdr.h +6 -0
index b75562c..f70a9d0 100644
@@ -414,6 +414,12 @@ struct ckpt_hdr_socket_queue {
 	__u32 total_bytes;
 } __attribute__ ((aligned(8)));
 
+struct ckpt_hdr_socket_buffer {
+	struct ckpt_hdr h;
+	__s32 sk_objref;
+	__s32 pr_objref;
+};
+
 #define CKPT_UNIX_LINKED 1
 struct ckpt_hdr_socket_unix {
 	struct ckpt_hdr h;
net/checkpoint.c +76 -44

181 lines not on this page · raw message

net/unix/checkpoint.c +221 -115

458 lines not on this page · raw message

Signature · 1 line
1.6.2.5
author
#4 lore raw

PATCH 3/3 Make sure we free the struct socket of SOCK_DEAD sockets

In reply to a message not in the archive

Avoid the leak reported by Serge for sockets that don't get attached to a file. Do this by orphaning the socket on restore if it is marked as SOCK_DEAD. If it is needed later for sending a buffer, temporarily adopt it for that process.

---
 net/checkpoint.c      |   13 ++++++++++++-
 net/unix/checkpoint.c |   16 ++++++++++++++++
 2 files changed, 28 insertions(+), 1 deletions(-)
net/checkpoint.c +12 -1
index d22341a..0bfca77 100644
@@ -627,6 +627,7 @@ struct sock *do_sock_restore(struct ckpt_ctx *ctx)
 {
 	struct ckpt_hdr_socket *h;
 	struct socket *sock;
+	struct sock *sk;
 	int ret;
 
 	h = ckpt_read_obj_type(ctx, sizeof(*h), CKPT_HDR_SOCKET);
@@ -661,7 +662,14 @@ struct sock *do_sock_restore(struct ckpt_ctx *ctx)
 
 	ckpt_hdr_put(ctx, h);
 
-	return sock->sk;
+	sk = sock->sk;
+	if (sock_flag(sk, SOCK_DEAD)) {
+		sock_orphan(sk);
+		sock->sk = NULL;
+		sock_release(sock);
+	}
+
+	return sk;
  err:
 	ckpt_hdr_put(ctx, h);
 	sock_release(sock);
@@ -688,6 +696,9 @@ struct file *sock_file_restore(struct ckpt_ctx *ctx, struct ckpt_hdr_file *ptr)
 	if (IS_ERR(sk))
 		return ERR_PTR(PTR_ERR(sk));
 
+	/* We should have a struct socket if we were attached to a file */
+	BUG_ON(!sk->sk_socket);
+
 	file = sock_alloc_attach_fd(sk->sk_socket);
 	if (IS_ERR(file))
 		return file;
net/unix/checkpoint.c +16 -0
index 60f6a0c..da2b408 100644
@@ -180,6 +180,7 @@ static int sock_read_buffer_sendmsg(struct ckpt_ctx *ctx,
 	struct kvec kvec;
 	struct ckpt_hdr_socket_buffer *h;
 	struct sock *sk;
+	struct socket *sock = NULL;
 	uint8_t sock_shutdown;
 	uint8_t peer_shutdown = 0;
 	void *buf;
@@ -229,6 +230,15 @@ static int sock_read_buffer_sendmsg(struct ckpt_ctx *ctx,
 			ckpt_debug("Failed to join: %i\n", ret);
 			goto out;
 		}
+	} else
+
+	if (!sk->sk_socket) {
+		ret = sock_create(sk->sk_family, sk->sk_type, 0, &sock);
+		if (ret < 0)
+			goto out;
+		if (unix_sk(sk)->peer)
+			sock->state = SS_CONNECTED;
+		sock_graft(sk, sock);
 	}
 
 	kvec.iov_len = len;
@@ -275,6 +285,12 @@ static int sock_read_buffer_sendmsg(struct ckpt_ctx *ctx,
 	ckpt_hdr_put(ctx, h);
 	kfree(buf);
 
+	if (sock) {
+		sock_orphan(sk);
+		sock->sk = NULL;
+		sock_release(sock);
+	}
+
 	return ret;
 }
 
Signature · 1 line
1.6.2.5

A message that is not in the archive (sent off-list or not yet received): 1252946924-4401-4-git-send-email-danms-r/[email protected]

#6 lore raw

Re: [PATCH 3/3] Make sure we free the struct socket of SOCK_DEAD sockets

Quoted 6 lines · Dan Smith

Avoid the leak reported by Serge for sockets that don't get attached to a file. Do this by orphaning the socket on restore if it is marked as SOCK_DEAD. If it is needed later for sending a buffer, temporarily adopt it for that process.

This works well for the case of successful restart, but I suspect it doesn't cover two other cases:

1) Malicious user removes the SOCK_DEAD marking off a socket.
2) Restart fails after a socket is restored but before it is attached to a process.

In both cases, the sk->sk_socket of a non-dead socket isn't freed.

Note that we could detect the inconsistency of case 1 at the end of restart, if there are non-attached sockets that aren't dead. But I doubt if it makes any difference, because it is harmless (assuming that the cleanup is indeed fixed).

Oren.

Quoted 84 lines
---
 net/checkpoint.c      |   13 ++++++++++++-
 net/unix/checkpoint.c |   16 ++++++++++++++++
 2 files changed, 28 insertions(+), 1 deletions(-)
net/checkpoint.c +12 -1
index d22341a..0bfca77 100644
@@ -627,6 +627,7 @@ struct sock *do_sock_restore(struct ckpt_ctx *ctx)
 {
 	struct ckpt_hdr_socket *h;
 	struct socket *sock;
+	struct sock *sk;
 	int ret;
 
 	h = ckpt_read_obj_type(ctx, sizeof(*h), CKPT_HDR_SOCKET);
@@ -661,7 +662,14 @@ struct sock *do_sock_restore(struct ckpt_ctx *ctx)
 
 	ckpt_hdr_put(ctx, h);
 
-	return sock->sk;
+	sk = sock->sk;
+	if (sock_flag(sk, SOCK_DEAD)) {
+		sock_orphan(sk);
+		sock->sk = NULL;
+		sock_release(sock);
+	}
+
+	return sk;
  err:
 	ckpt_hdr_put(ctx, h);
 	sock_release(sock);
@@ -688,6 +696,9 @@ struct file *sock_file_restore(struct ckpt_ctx *ctx, struct ckpt_hdr_file *ptr)
 	if (IS_ERR(sk))
 		return ERR_PTR(PTR_ERR(sk));
 
+	/* We should have a struct socket if we were attached to a file */
+	BUG_ON(!sk->sk_socket);
+
 	file = sock_alloc_attach_fd(sk->sk_socket);
 	if (IS_ERR(file))
 		return file;
net/unix/checkpoint.c +16 -0
index 60f6a0c..da2b408 100644
@@ -180,6 +180,7 @@ static int sock_read_buffer_sendmsg(struct ckpt_ctx *ctx,
 	struct kvec kvec;
 	struct ckpt_hdr_socket_buffer *h;
 	struct sock *sk;
+	struct socket *sock = NULL;
 	uint8_t sock_shutdown;
 	uint8_t peer_shutdown = 0;
 	void *buf;
@@ -229,6 +230,15 @@ static int sock_read_buffer_sendmsg(struct ckpt_ctx *ctx,
 			ckpt_debug("Failed to join: %i\n", ret);
 			goto out;
 		}
+	} else
+
+	if (!sk->sk_socket) {
+		ret = sock_create(sk->sk_family, sk->sk_type, 0, &sock);
+		if (ret < 0)
+			goto out;
+		if (unix_sk(sk)->peer)
+			sock->state = SS_CONNECTED;
+		sock_graft(sk, sock);
 	}
 
 	kvec.iov_len = len;
@@ -275,6 +285,12 @@ static int sock_read_buffer_sendmsg(struct ckpt_ctx *ctx,
 	ckpt_hdr_put(ctx, h);
 	kfree(buf);
 
+	if (sock) {
+		sock_orphan(sk);
+		sock->sk = NULL;
+		sock_release(sock);
+	}
+
 	return ret;
 }
 

A message that is not in the archive (sent off-list or not yet received): [email protected]

author
#8 lore raw

Re: [PATCH 3/3] Make sure we free the struct socket of SOCK_DEAD sockets

OL> This works well for the case of successful restart, but I suspect it OL> doesn't cover two other cases:

OL> 1) Malicious user removes the SOCK_DEAD marking off a socket. OL> 2) Restart fails after a socket is restored but before it is OL> attached to a process.

Aww, come on... when are users ever malicious? :)

Okay, I think I can handle both of these with a carefully-crafted obj->ops->drop() replacement. I don't want to get back to the orphan-all-by-default situation because it becomes hard to ensure that the socket object we later allocate receives proper treatment.

Thanks.

Signature · 3 lines
Dan Smith
IBM Linux Technology Center
email: danms-r/[email protected]
#9 lore raw
In reply to a message not in the archive

Ok, first two look fine (or I've been watching them for too long), so I queued them for ckpt-v18.

Oren.

Quoted 11 lines · Dan Smith

This version of the set contains a small fix to the first patch, a change suggested by Serge to the second, and a new third to avoid leaking the struct socket object on DEAD sockets. As previously mentioned, this still doesn't implement the collect bits for socket objects, which will follow later. Note that it depends on the yet-to-be-pushed post-files deferqueue patch that Oren has accepted.

_______________________________________________
Containers mailing list Containers-cunTk1MwBs9QetFLy7KEm3xJsTq8ys+cHZ5vskTnxNA@public.gmane.org
https://lists.linux-foundation.org/mailman/listinfo/containers