Skip to contentSkip to conversation

Track socket buffer owners

PATCH In review Dan Smith 3 participants 13 messages Sep 9, 2009 – Sep 14, 2009 9 messages not in the archive

Subsystems (3)
lore
Share & tools
lore

Dan Smith

author
#0 lore raw

This is the updated patch set that turns sockets into proper objhash objects and tracks the source and destination of socket buffers so that we can restore them from their proper owner to maintain sender information.

This is a roll-up of the comments from the last round.

A message that is not in the archive (sent off-list or not yet received): 1252508756-4278-1-git-send-email-danms-r/[email protected]

author
#2 lore raw

PATCH 1/3 Make sockets proper objhash objects and use checkpoint_obj() on them (v2)

This changes the checkpoint/restart procedure for sockets a bit. The socket file header is now checkpointed separately from the socket itself, which allows us to checkpoint a socket without arriving at it from a file descriptor. Thus, most sockets will be checkpointed as a result of processing the file table, calling sock_file_checkpoint(fd), which in turn calls checkpoint_obj(socket).

However, we may arrive at some sockets while checkpointing other objects, such as the other end of an AF_UNIX socket with buffers in flight. This patch just opens that door, which is utilized by the next patch.

Changes in v2:
- If we attempt to checkpoint an orphan socket, create a struct socket to adopt it for the purposes of the checkpoint

---
 checkpoint/objhash.c           |    2 +
 include/linux/checkpoint_hdr.h |    6 +-
 include/linux/net.h            |    4 +-
 include/net/af_unix.h          |    4 +-
 include/net/sock.h             |    2 +
 net/checkpoint.c               |  153 +++++++++++++++++++++++++++++++---------
 net/unix/checkpoint.c          |   11 ++--
 7 files changed, 136 insertions(+), 46 deletions(-)
checkpoint/objhash.c +2 -0
index a9a10d1..a410346 100644
@@ -381,6 +381,8 @@ static struct ckpt_obj_ops ckpt_obj_ops[] = {
 		.obj_type = CKPT_OBJ_SOCK,
 		.ref_drop = obj_sock_drop,
 		.ref_grab = obj_sock_grab,
+		.checkpoint = checkpoint_sock,
+		.restore = restore_sock,
 	},
 };
 
include/linux/checkpoint_hdr.h +4 -2
index 06bc6e2..b75562c 100644
@@ -70,6 +70,7 @@ enum {
 	CKPT_HDR_USER,
 	CKPT_HDR_GROUPINFO,
 	CKPT_HDR_TASK_CREDS,
+	CKPT_HDR_SOCKET,
 
 	/* 201-299: reserved for arch-dependent */
 
@@ -368,7 +369,8 @@ struct ckpt_hdr_file_pipe {
 } __attribute__((aligned(8)));
 
 /* socket */
-struct ckpt_socket {
+struct ckpt_hdr_socket {
+	struct ckpt_hdr h;
 	struct { /* struct socket */
 		__u64 flags;
 		__u8 state;
@@ -428,7 +430,7 @@ struct ckpt_hdr_socket_unix {
 
 struct ckpt_hdr_file_socket {
 	struct ckpt_hdr_file common;
-	struct ckpt_socket socket;
+	__s32 sock_objref;
 } __attribute__((aligned(8)));
 
 struct ckpt_hdr_utsns {
include/linux/net.h +2 -2
index 27187a4..96c7e22 100644
@@ -148,7 +148,7 @@ struct msghdr;
 struct module;
 
 struct ckpt_ctx;
-struct ckpt_socket;
+struct ckpt_hdr_socket;
 
 struct proto_ops {
 	int		family;
@@ -197,7 +197,7 @@ struct proto_ops {
 	int		(*checkpoint)(struct ckpt_ctx *ctx,
 				      struct socket *sock);
 	int		(*restore)(struct ckpt_ctx *ctx, struct socket *sock,
-				   struct ckpt_socket *h);
+				   struct ckpt_hdr_socket *h);
 };
 
 struct net_proto_family {
include/net/af_unix.h +2 -2
index 1a1fd20..61f666b 100644
@@ -71,10 +71,10 @@ static inline void unix_sysctl_unregister(struct net *net) {}
 
 #ifdef CONFIG_CHECKPOINT
 struct ckpt_ctx;
-struct ckpt_socket;
+struct ckpt_hdr_socket;
 extern int unix_checkpoint(struct ckpt_ctx *ctx, struct socket *sock);
 extern int unix_restore(struct ckpt_ctx *ctx, struct socket *sock,
-			struct ckpt_socket *h);
+			struct ckpt_hdr_socket *h);
 #else
 #define unix_checkpoint NULL
 #define unix_restore NULL
include/net/sock.h +2 -0
index 8e3b050..0db1ca3 100644
@@ -1644,6 +1644,8 @@ extern __u32 sysctl_rmem_default;
 /* Checkpoint/Restart Functions */
 struct ckpt_ctx;
 struct ckpt_hdr_file;
+extern int checkpoint_sock(struct ckpt_ctx *ctx, void *ptr);
+extern void *restore_sock(struct ckpt_ctx *ctx);
 extern int sock_file_checkpoint(struct ckpt_ctx *ctx, struct file *file);
 extern struct file *sock_file_restore(struct ckpt_ctx *ctx,
 				      struct ckpt_hdr_file *h);
net/checkpoint.c +119 -34

244 lines not on this page · raw message

net/unix/checkpoint.c +5 -6
index 08e664b..395f6fd 100644
@@ -57,7 +57,6 @@ static int unix_write_cwd(struct ckpt_ctx *ctx,
 int unix_checkpoint(struct ckpt_ctx *ctx, struct socket *sock)
 {
 	struct unix_sock *sk = unix_sk(sock->sk);
-	struct unix_sock *pr = unix_sk(sk->peer);
 	struct ckpt_hdr_socket_unix *un;
 	int new;
 	int ret = -ENOMEM;
@@ -86,7 +85,7 @@ int unix_checkpoint(struct ckpt_ctx *ctx, struct socket *sock)
 		goto out;
 
 	if (sk->peer)
-		un->peer = ckpt_obj_lookup_add(ctx, pr, CKPT_OBJ_SOCK, &new);
+		un->peer = checkpoint_obj(ctx, sk->peer, CKPT_OBJ_SOCK);
 	else
 		un->peer = 0;
 
@@ -237,7 +236,7 @@ static int unix_join(struct ckpt_ctx *ctx,
 }
 
 static int unix_restore_connected(struct ckpt_ctx *ctx,
-				  struct ckpt_socket *h,
+				  struct ckpt_hdr_socket *h,
 				  struct ckpt_hdr_socket_unix *un,
 				  struct socket *sock)
 {
@@ -423,7 +422,7 @@ static int unix_fakebind(struct socket *sock,
 	return 0;
 }
 
-static int unix_restore_bind(struct ckpt_socket *h,
+static int unix_restore_bind(struct ckpt_hdr_socket *h,
 			     struct ckpt_hdr_socket_unix *un,
 			     struct socket *sock,
 			     const char *path)
@@ -440,7 +439,7 @@ static int unix_restore_bind(struct ckpt_socket *h,
 }
 
 /* Some easy pre-flight checks before we get underway */
-static int unix_precheck(struct socket *sock, struct ckpt_socket *h)
+static int unix_precheck(struct socket *sock, struct ckpt_hdr_socket *h)
 {
 	struct net *net = sock_net(sock->sk);
 
@@ -471,7 +470,7 @@ static int unix_precheck(struct socket *sock, struct ckpt_socket *h)
 }
 
 int unix_restore(struct ckpt_ctx *ctx, struct socket *sock,
-		      struct ckpt_socket *h)
+		      struct ckpt_hdr_socket *h)
 
 {
 	struct ckpt_hdr_socket_unix *un;
Signature · 1 line
1.6.2.5

A message that is not in the archive (sent off-list or not yet received): 1252508756-4278-2-git-send-email-danms-r/[email protected]

#4 lore raw

Re: [PATCH 1/3] Make sockets proper objhash objects and use checkpoint_obj() on them (v2)

Quoted 11 lines · Dan Smith

This changes the checkpoint/restart procedure for sockets a bit. The socket file header is now checkpointed separately from the socket itself, which allows us to checkpoint a socket without arriving at it from a file descriptor. Thus, most sockets will be checkpointed as a result of processing the file table, calling sock_file_checkpoint(fd), which in turn calls checkpoint_obj(socket).

However, we may arrive at some sockets while checkpointing other objects, such as the other end of an AF_UNIX socket with buffers in flight. This patch just opens that door, which is utilized by the next patch.

Quoted 5 lines

Changes in v2:
- If we attempt to checkpoint an orphan socket, create a struct socket to adopt it for the purposes of the checkpoint

Looks good !

Did you also address this ?
https://lists.linux-foundation.org/pipermail/containers/2009-September/020385.html

-----
SH> Yes. On the struct sock. But what will drop the ref on the SH> struct socket? Or has one of your later patches, not yet in SH> ckpt-v17-dev, added that? Or, am I just missing a place where SH> sock_put() will actually sock_release(sk->sk_socket)?

Hmm, I see what you mean. I can't find any path where sock_put() will release the struct socket. What's weird is that there is a WARN_ON() in af_unix.c:354 that should get tripped if we call sk_free() when we still have a socket. I don't see that, but now I'm not sure why.

Perhaps what we should do is orphan the struct sock before we add it to the hash and then graft it onto a new struct socket before attaching it to a struct file?
-----

[The reason sk_free() expects no sk_socket is because the socket at this point must have been released already via proto_ops->release(). The callback is assumed to orphan the socket].

Oren.

A message that is not in the archive (sent off-list or not yet received): [email protected]

author
#6 lore raw

Re: [PATCH 1/3] Make sockets proper objhash objects and use checkpoint_obj() on them (v2)

OL> Did you also address this ?

Sorry, I meant to put something in the intro mail about this.

OL> [The reason sk_free() expects no sk_socket is because the socket OL> at this point must have been released already via
OL> proto_ops->release(). The callback is assumed to orphan the OL> socket].

Right, which is why we're not seeing the warning. I've got a follow-on patch to the current set started but wanted to avoid packing more onto these two patches. If you'd like me to resubmit the set and include that patch (when finished) I'll do that.

Signature · 3 lines
Dan Smith
IBM Linux Technology Center
email: danms-r/[email protected]

A message that is not in the archive (sent off-list or not yet received): 87d45xlfhz.fsf-FLMGYpZoEPULwtHQx/[email protected]

#8 lore raw

Re: [PATCH 1/3] Make sockets proper objhash objects and use checkpoint_obj() on them (v2)

Quoted 9 lines · Dan Smith

OL> Did you also address this ?

Sorry, I meant to put something in the intro mail about this.

OL> [The reason sk_free() expects no sk_socket is because the socket OL> at this point must have been released already via
OL> proto_ops->release(). The callback is assumed to orphan the OL> socket].

Quoted 5 lines

Right, which is why we're not seeing the warning. I've got a follow-on patch to the current set started but wanted to avoid packing more onto these two patches. If you'd like me to resubmit the set and include that patch (when finished) I'll do that.

I prefer not to pull in code that we know has a leak, but rather wait for the fixed version. I'll take the 2nd patch (post-file deferqueue) now.

Oren.

author
#9 lore raw

PATCH 2/3 Add post-file deferqueue (v2)

In reply to a message not in the archive

This is the deferq bit of Matt's patch, which is needed by the subsequent socket patch.

Changes in v2:
- Allocate and destroy the deferqueue at context alloc/free time

Cc Matt Helsley <matthltc-r/[email protected]>
---
 checkpoint/files.c               |   17 +++++++++++++++++
 checkpoint/sys.c                 |    7 +++++++
 include/linux/checkpoint_types.h |    1 +
 3 files changed, 25 insertions(+), 0 deletions(-)
checkpoint/files.c +17 -0
index 204055b..3d99823 100644
@@ -21,6 +21,7 @@
 #include <linux/syscalls.h>
 #include <linux/checkpoint.h>
 #include <linux/checkpoint_hdr.h>
+#include <linux/deferqueue.h>
 #include <net/sock.h>
 
 
@@ -294,6 +295,14 @@ static int do_checkpoint_file_table(struct ckpt_ctx *ctx,
 		if (ret < 0)
 			break;
 	}
+	if (!ret) {
+		ret = deferqueue_run(ctx->files_deferq);
+		if (ret > 0) {
+			pr_warning("c/r: files deferqueue had %d entries\n",
+				   ret);
+			ret = 0;
+		}
+	}
  out:
 	kfree(fdtable);
 	return ret;
@@ -697,6 +706,14 @@ static struct files_struct *do_restore_file_table(struct ckpt_ctx *ctx)
 		if (ret < 0)
 			break;
 	}
+	if (!ret) {
+		ret = deferqueue_run(ctx->files_deferq);
+		if (ret > 0) {
+			pr_warning("c/r: files deferqueue had %d entries\n",
+				   ret);
+			ret = 0;
+		}
+	}
  out:
 	ckpt_hdr_put(ctx, h);
 	if (!ret) {
checkpoint/sys.c +7 -0
index 525182a..31a188d 100644
@@ -201,6 +201,9 @@ static void ckpt_ctx_free(struct ckpt_ctx *ctx)
 		deferqueue_destroy(ctx->deferqueue);
 	}
 
+	if (ctx->files_deferq)
+		deferqueue_destroy(ctx->files_deferq);
+
 	if (ctx->file)
 		fput(ctx->file);
 
@@ -254,6 +257,10 @@ static struct ckpt_ctx *ckpt_ctx_alloc(int fd, unsigned long uflags,
 	if (!ctx->deferqueue)
 		goto err;
 
+	ctx->files_deferq = deferqueue_create();
+	if (!ctx->files_deferq)
+		goto err;
+
 	atomic_inc(&ctx->refcount);
 	return ctx;
  err:
include/linux/checkpoint_types.h +1 -0
index a18846f..b6f130c 100644
@@ -48,6 +48,7 @@ struct ckpt_ctx {
 
 	struct ckpt_obj_hash *obj_hash;	/* repository for shared objects */
 	struct deferqueue_head *deferqueue;	/* queue of deferred work */
+	struct deferqueue_head *files_deferq;
 
 	struct path fs_mnt;     /* container root (FIXME) */
 
Signature · 1 line
1.6.2.5

A message that is not in the archive (sent off-list or not yet received): 1252508756-4278-3-git-send-email-danms-r/[email protected]

Reviewed-by
#11 lore raw

Re: [PATCH 2/3] Add post-file deferqueue (v2)

Quoted 8 lines · Dan Smith

This is the deferq bit of Matt's patch, which is needed by the subsequent socket patch.

Changes in v2:
- Allocate and destroy the deferqueue at context alloc/free time

Cc Matt Helsley <matthltc-r/[email protected]>

(but see comment below).

Quoted 19 lines
---
 checkpoint/files.c               |   17 +++++++++++++++++
 checkpoint/sys.c                 |    7 +++++++
 include/linux/checkpoint_types.h |    1 +
 3 files changed, 25 insertions(+), 0 deletions(-)
checkpoint/files.c +1 -0
index 204055b..3d99823 100644
@@ -21,6 +21,7 @@
 #include <linux/syscalls.h>
 #include <linux/checkpoint.h>
 #include <linux/checkpoint_hdr.h>
+#include <linux/deferqueue.h>
 #include <net/sock.h>
 
 

checkpoint/files.c static int do_checkpoint_file_table(struct ckpt_ctx *ctx, · line 303

Quoted 9 lines
checkpoint/files.c +6 -0
@@ -294,6 +295,14 @@ static int do_checkpoint_file_table(struct ckpt_ctx *ctx,
 		if (ret < 0)
 			break;
 	}
+	if (!ret) {
+		ret = deferqueue_run(ctx->files_deferq);
+		if (ret > 0) {
+			pr_warning("c/r: files deferqueue had %d entries\n",
+				   ret);
+			ret = 0;

I'm unsure why the warning; perhaps you mean "ckpt_debug()" ?

Quoted 6 lines
checkpoint/files.c +2 -0
+		}
+	}
  out:
 	kfree(fdtable);
 	return ret;

checkpoint/files.c static struct files_struct *do_restore_file_table(struct ckpt_ctx *ctx) · line 714

Quoted 9 lines
checkpoint/files.c +6 -0
@@ -697,6 +706,14 @@ static struct files_struct *do_restore_file_table(struct ckpt_ctx *ctx)
 		if (ret < 0)
 			break;
 	}
+	if (!ret) {
+		ret = deferqueue_run(ctx->files_deferq);
+		if (ret > 0) {
+			pr_warning("c/r: files deferqueue had %d entries\n",
+				   ret);
+			ret = 0;

Ditto.

Oren.

A message that is not in the archive (sent off-list or not yet received): [email protected]

author
#13 lore raw

Re: [PATCH 2/3] Add post-file deferqueue (v2)

OL> I'm unsure why the warning; perhaps you mean "ckpt_debug()" ?

This is just copied from Matt's patch. Feel free to change it when you put it in or I can send another one.

Signature · 3 lines
Dan Smith
IBM Linux Technology Center
email: danms-r/[email protected]

A message that is not in the archive (sent off-list or not yet received): 87hbv9lfup.fsf-FLMGYpZoEPULwtHQx/[email protected]

#15 lore raw

Re: [PATCH 2/3] Add post-file deferqueue (v2)

Quoted 5 lines · Dan Smith

OL> I'm unsure why the warning; perhaps you mean "ckpt_debug()" ?

This is just copied from Matt's patch. Feel free to change it when you put it in or I can send another one.

Nah ... no need to resend.

author
#17 lore raw

PATCH 3/3 Track socket buffer owners (v2)

In reply to a message not in the archive

This patch is a superset of the previous attempt to store socket buffers with their owners. It defers the writing and reading of the socket buffers until after the end of the file phase to avoid a recursive nose-dive of checkpointing socket owners.

This also moves the join logic to the deferqueue as well, since that too can lead us down a deep hole. The buffer restore logic may perform a join if it decides that the join is inevitable (but not yet performed) and necessary.

Changes in v2:
- Add comment about using deferqueue_add() in a function that could have been called during deferqueue_run()
- Change 'ref' to 'objref' in several variable names
- Move SOCK_DEAD flag detection to initial patch
- Catch errors from unix_read_buffers(), even on the should-be-missing send buffers

---
 include/linux/checkpoint.h     |    3 -
 include/linux/checkpoint_hdr.h |    6 +
 net/checkpoint.c               |  120 +++++++++------
 net/unix/checkpoint.c          |  336 ++++++++++++++++++++++++++--------------
 4 files changed, 303 insertions(+), 162 deletions(-)
include/linux/checkpoint.h +0 -3
index 761cad5..88861a1 100644
@@ -84,9 +84,6 @@ extern int ckpt_sock_getnames(struct ckpt_ctx *ctx,
 			      struct socket *socket,
 			      struct sockaddr *loc, unsigned *loc_len,
 			      struct sockaddr *rem, unsigned *rem_len);
-extern struct ckpt_hdr_socket_queue *
-ckpt_sock_read_buffer_hdr(struct ckpt_ctx *ctx,
-			  uint32_t *bufsize);
 
 /* ckpt kflags */
 #define ckpt_set_ctx_kflag(__ctx, __kflag)  \
include/linux/checkpoint_hdr.h +6 -0
index b75562c..f70a9d0 100644
@@ -414,6 +414,12 @@ struct ckpt_hdr_socket_queue {
 	__u32 total_bytes;
 } __attribute__ ((aligned(8)));
 
+struct ckpt_hdr_socket_buffer {
+	struct ckpt_hdr h;
+	__s32 sk_objref;
+	__s32 pr_objref;
+};
+
 #define CKPT_UNIX_LINKED 1
 struct ckpt_hdr_socket_unix {
 	struct ckpt_hdr h;
net/checkpoint.c +76 -44

181 lines not on this page · raw message

net/unix/checkpoint.c +221 -115

458 lines not on this page · raw message

Signature · 1 line
1.6.2.5

A message that is not in the archive (sent off-list or not yet received): 1252508756-4278-4-git-send-email-danms-r/[email protected]

#19 lore raw

Re: [PATCH 3/3] Track socket buffer owners (v2)

Quoted 18 lines · Dan Smith

This patch is a superset of the previous attempt to store socket buffers with their owners. It defers the writing and reading of the socket buffers until after the end of the file phase to avoid a recursive nose-dive of checkpointing socket owners.

This also moves the join logic to the deferqueue as well, since that too can lead us down a deep hole. The buffer restore logic may perform a join if it decides that the join is inevitable (but not yet performed) and necessary.

Changes in v2:
- Add comment about using deferqueue_add() in a function that could have been called during deferqueue_run()
- Change 'ref' to 'objref' in several variable names
- Move SOCK_DEAD flag detection to initial patch
- Catch errors from unix_read_buffers(), even on the should-be-missing send buffers

Quoted 1 line

Looks fine to my unqualified eyes, except for:

Quoted 11 lines
net/unix/checkpoint.c +7 -0
@@ -139,54 +241,125 @@ static int sock_read_buffer_sendmsg(struct ckpt_ctx *ctx, struct sock *sk)
 	if (ret < 0)
 		goto out;
 
+	msg.msg_name = addr;
+	msg.msg_namelen = addrlen;
+
+	/* If peer is shutdown, unshutdown it for this process */
+	sock_shutdown = sk->sk_shutdown;
+	sk->sk_shutdown &= ~SHUTDOWN_MASK;
+

net/unix/checkpoint.c static int sock_read_buffer_sendmsg(struct ckpt_ctx *ctx, struct sock *sk) · line 260

Quoted 10 lines
net/unix/checkpoint.c +10 -0
+	/* Unshutdown peer too, if necessary */
+	if (unix_sk(sk)->peer) {
+		peer_shutdown = unix_sk(sk)->peer->sk_shutdown;
+		unix_sk(sk)->peer->sk_shutdown &= ~SHUTDOWN_MASK;
+	}
+
+	/* Make sure there's room in the send buffer */
+	sndbuf = sk->sk_sndbuf;
+	if (capable(CAP_NET_ADMIN) &&
+	    ((sk->sk_sndbuf - atomic_read(&sk->sk_wmem_alloc)) < len))

'capable' actually has an adverse effect of setting the PF_SUPERPRIV flag on current. So if I don't misread this, you'll want to do the length check first, then the capable check, in order to make sure that PF_SUPERPRIV doesn't get set unless the privilege was actually needed.

net/unix/checkpoint.c static int sock_read_buffer_sendmsg(struct ckpt_ctx *ctx, struct sock *sk) · line 263

Quoted 3 lines
net/unix/checkpoint.c +3 -0
+		sk->sk_sndbuf += len;
+	else
+		sk->sk_sndbuf = sysctl_wmem_max;

-serge

A message that is not in the archive (sent off-list or not yet received): 20090911020206.GA15845-r/[email protected]

author
#21 lore raw

Re: [PATCH 3/3] Track socket buffer owners (v2)

SH> 'capable' actually has an adverse effect of setting the SH> PF_SUPERPRIV flag on current. So if I don't misread this, you'll SH> want to do the length check first, then the capable check, in SH> order to make sure that PF_SUPERPRIV doesn't get set unless the SH> privilege was actually needed.

Ah, okay, thanks, I'll switch those.

Signature · 3 lines
Dan Smith
IBM Linux Technology Center
email: danms-r/[email protected]