Sockets as proper objhash objects
Subsystems (3)
This version of the set contains a small fix to the first patch, a change suggested by Serge to the second, and a new third to avoid leaking the struct socket object on DEAD sockets. As previously mentioned, this still doesn't implement the collect bits for socket objects, which will follow later. Note that it depends on the yet-to-be-pushed post-files deferqueue patch that Oren has accepted.
A message that is not in the archive (sent off-list or not yet received): 1252946924-4401-1-git-send-email-danms-r/[email protected]
PATCH 1/3 Make sockets proper objhash objects and use checkpoint_obj() on them (v3)
This changes the checkpoint/restart procedure for sockets a bit. The socket file header is now checkpointed separately from the socket itself, which allows us to checkpoint a socket without arriving at it from a file descriptor. Thus, most sockets will be checkpointed as a result of processing the file table, calling sock_file_checkpoint(fd), which in turn calls checkpoint_obj(socket).
However, we may arrive at some sockets while checkpointing other objects, such as the other end of an AF_UNIX socket with buffers in flight. This patch just opens that door, which is utilized by the next patch.
Changes in v3:
- Fix failure to actually set the SOCK_DEAD flag on restore
Changes in v2:
- If we attempt to checkpoint an orphan socket, create a struct socket to adopt it for the purposes of the checkpoint
--- checkpoint/objhash.c | 2 + include/linux/checkpoint_hdr.h | 6 +- include/linux/net.h | 4 +- include/net/af_unix.h | 4 +- include/net/sock.h | 2 + net/checkpoint.c | 153 +++++++++++++++++++++++++++++++--------- net/unix/checkpoint.c | 11 ++-- 7 files changed, 136 insertions(+), 46 deletions(-)
checkpoint/objhash.c +2 -0
@@ -381,6 +381,8 @@ static struct ckpt_obj_ops ckpt_obj_ops[] = { .obj_type = CKPT_OBJ_SOCK, .ref_drop = obj_sock_drop, .ref_grab = obj_sock_grab, + .checkpoint = checkpoint_sock, + .restore = restore_sock, }, };
include/linux/checkpoint_hdr.h +4 -2
@@ -70,6 +70,7 @@ enum {CKPT_HDR_USER ,CKPT_HDR_GROUPINFO ,CKPT_HDR_TASK_CREDS , + CKPT_HDR_SOCKET,/* 201-299: reserved for arch-dependent */
@@ -368,7 +369,8 @@ struct ckpt_hdr_file_pipe { } __attribute__((aligned(8)));/* socket */ -struct ckpt_socket { +struct ckpt_hdr_socket { +struct ckpt_hdr h;struct {/* struct socket */ __u64 flags;__u8 state;
@@ -428,7 +430,7 @@ struct ckpt_hdr_socket_unix {struct ckpt_hdr_file_socket {struct ckpt_hdr_file common; -struct ckpt_socket socket; +__s32 sock_objref; } __attribute__((aligned (8 )));struct ckpt_hdr_utsns {
include/linux/net.h +2 -2
@@ -148,7 +148,7 @@ struct msghdr;struct module ;struct ckpt_ctx ; -struct ckpt_socket ; +struct ckpt_hdr_socket ;struct proto_ops {int family;
@@ -197,7 +197,7 @@ struct proto_ops {int (*checkpoint)(struct ckpt_ctx *ctx,struct socket *sock);int (*restore)(struct ckpt_ctx *ctx,struct socket *sock, -struct ckpt_socket *h); +struct ckpt_hdr_socket *h); };struct net_proto_family {
include/net/af_unix.h +2 -2
@@ -71,10 +71,10 @@ static inline void unix_sysctl_unregister(struct net *net) {}#ifdef CONFIG_CHECKPOINT struct ckpt_ctx ; -struct ckpt_socket ; +struct ckpt_hdr_socket ;extern int unix_checkpoint (struct ckpt_ctx *ctx,struct socket *sock);extern int unix_restore (struct ckpt_ctx *ctx,struct socket *sock, -struct ckpt_socket *h); +struct ckpt_hdr_socket *h);#else #define unix_checkpoint NULL#define unix_restore NULL
include/net/sock.h +2 -0
@@ -1644,6 +1644,8 @@ extern __u32 sysctl_rmem_default;/* Checkpoint/Restart Functions */ struct ckpt_ctx ;struct ckpt_hdr_file ; +extern int checkpoint_sock (struct ckpt_ctx *ctx,void *ptr); +extern void *restore_sock (struct ckpt_ctx *ctx);extern int sock_file_checkpoint (struct ckpt_ctx *ctx,struct file *file);extern struct file *sock_file_restore (struct ckpt_ctx *ctx,struct ckpt_hdr_file *h);
net/checkpoint.c +119 -34
244 lines not on this page · raw message
net/unix/checkpoint.c +5 -6
@@ -57,7 +57,6 @@ static int unix_write_cwd(struct ckpt_ctx *ctx,int unix_checkpoint (struct ckpt_ctx *ctx,struct socket *sock) {struct unix_sock *sk =unix_sk (sock->sk); -struct unix_sock *pr =unix_sk (sk->peer);struct ckpt_hdr_socket_unix *un;int new;int ret = -ENOMEM ;
@@ -86,7 +85,7 @@ int unix_checkpoint(struct ckpt_ctx *ctx, struct socket *sock) goto out;if (sk->peer) - un->peer =ckpt_obj_lookup_add (ctx, pr,CKPT_OBJ_SOCK , &new); + un->peer =checkpoint_obj (ctx, sk->peer,CKPT_OBJ_SOCK );else un->peer =0 ;
@@ -237,7 +236,7 @@ static int unix_join(struct ckpt_ctx *ctx, }static int unix_restore_connected (struct ckpt_ctx *ctx, -struct ckpt_socket *h, +struct ckpt_hdr_socket *h,struct ckpt_hdr_socket_unix *un,struct socket *sock) {
@@ -423,7 +422,7 @@ static int unix_fakebind(struct socket *sock,return 0 ; } -static int unix_restore_bind (struct ckpt_socket *h, +static int unix_restore_bind (struct ckpt_hdr_socket *h,struct ckpt_hdr_socket_unix *un,struct socket *sock,const char *path)
@@ -440,7 +439,7 @@ static int unix_restore_bind(struct ckpt_socket *h, }/* Some easy pre-flight checks before we get underway */ -static int unix_precheck (struct socket *sock,struct ckpt_socket *h) +static int unix_precheck (struct socket *sock,struct ckpt_hdr_socket *h) {struct net *net =sock_net (sock->sk);
@@ -471,7 +470,7 @@ static int unix_precheck(struct socket *sock, struct ckpt_socket *h) }int unix_restore (struct ckpt_ctx *ctx,struct socket *sock, -struct ckpt_socket *h) +struct ckpt_hdr_socket *h) {struct ckpt_hdr_socket_unix *un;
Signature · 1 line
1.6.2.5
PATCH 2/3 Track socket buffer owners (v3)
In reply to a message not in the archiveThis patch is a superset of the previous attempt to store socket buffers with their owners. It defers the writing and reading of the socket buffers until after the end of the file phase to avoid a recursive nose-dive of checkpointing socket owners.
This also moves the join logic to the deferqueue as well, since that too can lead us down a deep hole. The buffer restore logic may perform a join if it decides that the join is inevitable (but not yet performed) and necessary.
Changes in v3:
- Reverse order of length and capable(CAP_NET_ADMIN) checks to avoid setting PF_SUPERPRIV too early, per Serge's recommendation
Changes in v2:
- Add comment about using deferqueue_add() in a function that could have been called during deferqueue_run()
- Change 'ref' to 'objref' in several variable names
- Move SOCK_DEAD flag detection to initial patch
- Catch errors from unix_read_buffers(), even on the should-be-missing send buffers
--- include/linux/checkpoint.h | 3 - include/linux/checkpoint_hdr.h | 6 + net/checkpoint.c | 120 +++++++++------ net/unix/checkpoint.c | 336 ++++++++++++++++++++++++++-------------- 4 files changed, 303 insertions(+), 162 deletions(-)
include/linux/checkpoint.h +0 -3
@@ -84,9 +84,6 @@ extern int ckpt_sock_getnames(struct ckpt_ctx *ctx,struct socket *socket, struct sockaddr *loc, unsigned *loc_len, struct sockaddr *rem, unsigned *rem_len); -extern struct ckpt_hdr_socket_queue * -ckpt_sock_read_buffer_hdr (struct ckpt_ctx *ctx, -uint32_t *bufsize);/* ckpt kflags */ #define ckpt_set_ctx_kflag (__ctx, __kflag) \
include/linux/checkpoint_hdr.h +6 -0
@@ -414,6 +414,12 @@ struct ckpt_hdr_socket_queue {__u32 total_bytes; } __attribute__ ((aligned(8))); +struct ckpt_hdr_socket_buffer { +struct ckpt_hdr h; +__s32 sk_objref; +__s32 pr_objref; +}; +#define CKPT_UNIX_LINKED 1struct ckpt_hdr_socket_unix {struct ckpt_hdr h;
net/checkpoint.c +76 -44
181 lines not on this page · raw message
net/unix/checkpoint.c +221 -115
458 lines not on this page · raw message
Signature · 1 line
1.6.2.5
PATCH 3/3 Make sure we free the struct socket of SOCK_DEAD sockets
In reply to a message not in the archiveAvoid the leak reported by Serge for sockets that don't get attached to a file. Do this by orphaning the socket on restore if it is marked as SOCK_DEAD. If it is needed later for sending a buffer, temporarily adopt it for that process.
--- net/checkpoint.c | 13 ++++++++++++- net/unix/checkpoint.c | 16 ++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletions(-)
net/checkpoint.c +12 -1
@@ -627,6 +627,7 @@ struct sock *do_sock_restore(struct ckpt_ctx *ctx) {struct ckpt_hdr_socket *h;struct socket *sock; +struct sock *sk;int ret; h =ckpt_read_obj_type (ctx,sizeof (*h),CKPT_HDR_SOCKET );
@@ -661,7 +662,14 @@ struct sock *do_sock_restore(struct ckpt_ctx *ctx)ckpt_hdr_put (ctx, h); -return sock->sk; + sk = sock->sk; +if (sock_flag (sk,SOCK_DEAD )) { +sock_orphan (sk); + sock->sk =NULL ; +sock_release (sock); + } + +return sk; err:ckpt_hdr_put (ctx, h);sock_release (sock);
@@ -688,6 +696,9 @@ struct file *sock_file_restore(struct ckpt_ctx *ctx, struct ckpt_hdr_file *ptr)if (IS_ERR (sk))return ERR_PTR (PTR_ERR (sk)); +/* We should have a struct socket if we were attached to a file */ +BUG_ON (!sk->sk_socket); + file =sock_alloc_attach_fd (sk->sk_socket);if (IS_ERR (file))return file;
net/unix/checkpoint.c +16 -0
@@ -180,6 +180,7 @@ static int sock_read_buffer_sendmsg(struct ckpt_ctx *ctx,struct kvec kvec;struct ckpt_hdr_socket_buffer *h;struct sock *sk; +struct socket *sock =NULL ;uint8_t sock_shutdown;uint8_t peer_shutdown =0 ;void *buf;
@@ -229,6 +230,15 @@ static int sock_read_buffer_sendmsg(struct ckpt_ctx *ctx,ckpt_debug ("Failed to join: %i\n" , ret); goto out; } + }else + +if (!sk->sk_socket) { + ret =sock_create (sk->sk_family, sk->sk_type,0 , &sock); +if (ret <0 ) + goto out; +if (unix_sk (sk)->peer) + sock->state =SS_CONNECTED ; +sock_graft (sk, sock); } kvec.iov_len = len;
@@ -275,6 +285,12 @@ static int sock_read_buffer_sendmsg(struct ckpt_ctx *ctx,ckpt_hdr_put (ctx, h);kfree (buf); +if (sock) { +sock_orphan (sk); + sock->sk =NULL ; +sock_release (sock); + } +return ret; }
Signature · 1 line
1.6.2.5
A message that is not in the archive (sent off-list or not yet received): 1252946924-4401-4-git-send-email-danms-r/[email protected]
Re: [PATCH 3/3] Make sure we free the struct socket of SOCK_DEAD sockets
Quoted 6 lines · Dan Smith
Avoid the leak reported by Serge for sockets that don't get attached to a file. Do this by orphaning the socket on restore if it is marked as
SOCK_DEAD. If it is needed later for sending a buffer, temporarily adopt it for that process.Signed-off-by Dan Smith <danms-r/[email protected]>
This works well for the case of successful restart, but I suspect it doesn't cover two other cases:
1) Malicious user removes the SOCK_DEAD marking off a socket.
2) Restart fails after a socket is restored but before it is attached to a process.
In both cases, the sk->sk_socket of a non-dead socket isn't freed.
Note that we could detect the inconsistency of case 1 at the end of restart, if there are non-attached sockets that aren't dead. But I doubt if it makes any difference, because it is harmless (assuming that the cleanup is indeed fixed).
Oren.
Quoted 84 lines
--- net/checkpoint.c | 13 ++++++++++++- net/unix/checkpoint.c | 16 ++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletions(-)net/checkpoint.c +12
-1@@ -627,6 +627,7 @@ struct sock *do_sock_restore(struct ckpt_ctx *ctx) {struct ckpt_hdr_socket *h;struct socket *sock; +struct sock *sk;int ret; h =ckpt_read_obj_type (ctx,sizeof (*h),CKPT_HDR_SOCKET );@@ -661,7 +662,14 @@ struct sock *do_sock_restore(struct ckpt_ctx *ctx)ckpt_hdr_put (ctx, h); -return sock->sk; + sk = sock->sk; +if (sock_flag (sk,SOCK_DEAD )) { +sock_orphan (sk); + sock->sk =NULL ; +sock_release (sock); + } + +return sk; err:ckpt_hdr_put (ctx, h);sock_release (sock);@@ -688,6 +696,9 @@ struct file *sock_file_restore(struct ckpt_ctx *ctx, struct ckpt_hdr_file *ptr)if (IS_ERR (sk))return ERR_PTR (PTR_ERR (sk)); +/* We should have a struct socket if we were attached to a file */ +BUG_ON (!sk->sk_socket); + file =sock_alloc_attach_fd (sk->sk_socket);if (IS_ERR (file))return file;net/unix/checkpoint.c +16
-0@@ -180,6 +180,7 @@ static int sock_read_buffer_sendmsg(struct ckpt_ctx *ctx,struct kvec kvec;struct ckpt_hdr_socket_buffer *h;struct sock *sk; +struct socket *sock =NULL ;uint8_t sock_shutdown;uint8_t peer_shutdown =0 ;void *buf;@@ -229,6 +230,15 @@ static int sock_read_buffer_sendmsg(struct ckpt_ctx *ctx,ckpt_debug ("Failed to join: %i\n" , ret); goto out; } + }else + +if (!sk->sk_socket) { + ret =sock_create (sk->sk_family, sk->sk_type,0 , &sock); +if (ret <0 ) + goto out; +if (unix_sk (sk)->peer) + sock->state =SS_CONNECTED ; +sock_graft (sk, sock); } kvec.iov_len = len;@@ -275,6 +285,12 @@ static int sock_read_buffer_sendmsg(struct ckpt_ctx *ctx,ckpt_hdr_put (ctx, h);kfree (buf); +if (sock) { +sock_orphan (sk); + sock->sk =NULL ; +sock_release (sock); + } +return ret; }
A message that is not in the archive (sent off-list or not yet received): [email protected]
Re: [PATCH 3/3] Make sure we free the struct socket of SOCK_DEAD sockets
OL> This works well for the case of successful restart, but I suspect it OL> doesn't cover two other cases:
OL> 1) Malicious user removes the SOCK_DEAD marking off a socket. OL> 2) Restart fails after a socket is restored but before it is OL> attached to a process.
Aww, come on... when are users ever malicious? :)
Okay, I think I can handle both of these with a carefully-crafted obj->ops->drop() replacement. I don't want to get back to the orphan-all-by-default situation because it becomes hard to ensure that the socket object we later allocate receives proper treatment.
Thanks.
Signature · 3 lines
Dan Smith IBM Linux Technology Center email: danms-r/[email protected]
Ok, first two look fine (or I've been watching them for too long), so I queued them for ckpt-v18.
Oren.
Quoted 11 lines · Dan Smith
This version of the set contains a small fix to the first patch, a change suggested by Serge to the second, and a new third to avoid leaking the struct socket object on DEAD sockets. As previously mentioned, this still doesn't implement the collect bits for socket objects, which will follow later. Note that it depends on the yet-to-be-pushed post-files deferqueue patch that Oren has accepted.
_______________________________________________
Containers mailing list Containers-cunTk1MwBs9QetFLy7KEm3xJsTq8ys+cHZ5vskTnxNA@public.gmane.org
https://lists.linux-foundation.org/mailman/listinfo/containers