mptcp: plug mptcp_accept memory leaks
Subsystems (2)
Florian Westphal
cover mptcp: plug mptcp_accept memory leaks
Hi,
This series resolves the memory leaks that occur when running mptcp_connect.sh.
I don't mind if they get squashed.
I've decided to split this into 3 distinct changes to make it a bit clearer what is going on.
Second patch is not a bug fix, i can resend and drop it if the consensus is to continue using the sk->sk_socket back reference.
Third patch is a bit large due to the change in how the new mptcp socket is created.
See the commit messages for more details.
After these changes kmemleak doesn't report any leaks when the test script has completed.
Cheers,
Florian
Florian Westphal
PATCH 1/3 mptcp: release mptcp socket in mptcp_close
kmemleak reports several stray objects.
This fixes following splats:
unreferenced object 0xffff88810e27f240 (size 2016):
comm "mptcp_connect", pid 2187, jiffies 4295218907 (age 1371.080s)
hex dump (first 32 bytes):
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
02 00 01 41 00 00 00 00 00 00 00 00 00 00 00 00 ...A............
backtrace:
[<00000000298ae054>] sk_alloc+0x4c/0x600
[<000000001cd7d411>] inet_create+0x1f4/0x6a0
[<00000000ac4d1289>] __sock_create+0x1b4/0x3c0
[<00000000cf2663ed>] __sys_socket+0xb6/0x170
[<00000000b8efb3a9>] __x64_sys_socket+0x3d/0x50
[<000000006e83b432>] do_syscall_64+0x72/0x270
[<0000000007655965>] entry_SYSCALL_64_after_hwframe+0x49/0xbe
[<0000000073c2278e>] 0xffffffffffffffffThis is because mptcp_close forgot to put the last reference.
We also need to set the state to CLOSED and orphan the socket, else the put will trigger warning about 'alive' inet socket being torn down.
--- net/mptcp/protocol.c | 5 +++++ 1 file changed, 5 insertions(+)
net/mptcp/protocol.c +5 -0
@@ -578,6 +578,8 @@ static void mptcp_close(struct sock *sk, long timeout)struct socket *ssk =NULL ;struct list_head list; +inet_sk_state_store (sk,TCP_CLOSE ); +spin_lock_bh (&msk->conn_list_lock);if (msk->subflow) { ssk = msk->subflow;
@@ -601,6 +603,9 @@ static void mptcp_close(struct sock *sk, long timeout)pr_debug ("conn_list->subflow=%p" , subflow);sock_release (sock_sk (subflow)->sk_socket); }; + +sock_orphan (sk); +sock_put (sk); } static struct sock *mptcp_accept(struct sock *sk, int flags, int *err,
Signature · 1 line
2.21.0
Florian Westphal
PATCH 2/3 mptcp: store tcp socket, not tcp sock in subflow ctx
In reply to Florian WestphalInstead of using sk->sk_socket to get back the socket reference, store the 'socket' pointer that is coming from kernel_accept().
This doesn't fix a bug, but it appears cleaner to me.
--- net/mptcp/protocol.c | 19 ++++++++++--------- net/mptcp/protocol.h | 6 +++--- net/mptcp/subflow.c | 10 +++++++--- 3 files changed, 20 insertions(+), 15 deletions(-)
net/mptcp/protocol.c +10 -9
@@ -27,7 +27,7 @@ static inline bool before64(__u64 seq1, __u64 seq2)static bool mptcp_subflow_hold (struct subflow_context *subflow) { -struct sock *sk =sock_sk (subflow); +struct sock *sk =mptcp_subflow_socket (subflow)->sk;return refcount_inc_not_zero (&sk->sk_refcnt); }
@@ -40,7 +40,7 @@ static struct sock *mptcp_subflow_get_ref(const struct mptcp_sock *msk)mptcp_for_each_subflow (msk, subflow) {if (mptcp_subflow_hold (subflow)) {rcu_read_unlock (); -return sock_sk (subflow); +return mptcp_subflow_socket (subflow)->sk; } }
@@ -232,7 +232,7 @@ static u64 expand_seq(u64 old_seq, u16 old_data_len, u64 seq)static u64 get_mapped_dsn (struct subflow_context *subflow) { -u32 map_offset = (tcp_sk (sock_sk (subflow))->copied_seq - +u32 map_offset = (tcp_sk (mptcp_subflow_socket (subflow)->sk)->copied_seq - subflow->ssn_offset - subflow->map_subflow_seq);
@@ -601,8 +601,8 @@ static void mptcp_close(struct sock *sk, long timeout)list_for_each_entry_safe (subflow, tmp, &list, node) {pr_debug ("conn_list->subflow=%p" , subflow); -sock_release (sock_sk (subflow)->sk_socket); - }; +sock_release (mptcp_subflow_socket (subflow)); + }sock_orphan (sk);sock_put (sk);
@@ -656,6 +656,7 @@ static struct sock *mptcp_accept(struct sock *sk, int flags, int *err, subflow->map_subflow_seq =1 ; subflow->rel_write_seq =1 ; subflow->conn = new_mptcp_sock->sk; + subflow->tcp_sock = new_sock; }else { msk->subflow = new_sock; }
@@ -934,8 +935,8 @@ static __poll_t mptcp_poll(struct file *file, struct socket *sock,if (!tmp)break ; - ret |=tcp_poll (file,sock_sk (tmp)->sk_socket, wait); -sock_put (sock_sk (tmp)); + ret |=tcp_poll (file,mptcp_subflow_socket (tmp), wait); +sock_put (mptcp_subflow_socket (tmp)->sk); }return ret;
@@ -964,9 +965,9 @@ static int mptcp_shutdown(struct socket *sock, int how)lock_sock (sock->sk);rcu_read_lock ();list_for_each_entry_rcu (subflow, &msk->conn_list, node) { -rcu_read_unlock ();pr_debug ("conn_list->subflow=%p" , subflow); - ret =kernel_sock_shutdown (sock_sk (subflow)->sk_socket, how); +rcu_read_unlock (); + ret =kernel_sock_shutdown (mptcp_subflow_socket (subflow), how);rcu_read_lock (); }rcu_read_unlock ();
net/mptcp/protocol.h +3 -3
@@ -90,7 +90,7 @@ struct subflow_context { map_valid :1 , use_checksum : 1; -struct sock *sk;/* underlying tcp_sock */ +struct socket *tcp_sock;/* underlying tcp_sock */ struct sock *conn;/* parent mptcp_sock */ void (*tcp_sk_data_ready)(struct sock *sk); };
@@ -101,9 +101,9 @@ static inline struct subflow_context *subflow_ctx(const struct sock *sk)return (struct subflow_context *)icsk->icsk_ulp_data; } -static inline struct sock *sock_sk (const struct subflow_context *subflow) +static inline struct socket *mptcp_subflow_socket (const struct subflow_context *subflow) { -return subflow->sk; +return subflow->tcp_sock; }int subflow_init (void );
net/mptcp/subflow.c +7 -3
@@ -174,6 +174,7 @@ static void subflow_data_ready(struct sock *sk) }static struct subflow_context *subflow_create_ctx (struct sock *sk, +struct socket *sock,gfp_t priority) {struct inet_connection_sock *icsk =inet_csk (sk);
@@ -187,7 +188,8 @@ static struct subflow_context *subflow_create_ctx(struct sock *sk,pr_debug ("subflow=%p" , ctx); - ctx->sk = sk; +/* might be NULL */ + ctx->tcp_sock = sock;return ctx; }
@@ -199,7 +201,7 @@ static int subflow_ulp_init(struct sock *sk)struct subflow_context *ctx;int err =0 ; - ctx =subflow_create_ctx (sk,GFP_KERNEL ); + ctx =subflow_create_ctx (sk, sk->sk_socket,GFP_KERNEL );if (!ctx) { err = -ENOMEM ; goto out;
@@ -234,7 +236,9 @@ static void subflow_ulp_clone(const struct request_sock *req,struct subflow_context *new_ctx; old_ctx =inet_csk (newsk)->icsk_ulp_data; - new_ctx =subflow_create_ctx (newsk, priority); + +/* newsk->sk_socket is NULL at this point */ + new_ctx =subflow_create_ctx (newsk,NULL , priority);if (!new_ctx)return ;
Signature · 1 line
2.21.0
Krystad
Re: [MPTCP] [PATCH mptcp 2/3] mptcp: store tcp socket, not tcp sock in subflow ctx
Quoted 99 lines · Florian Westphal
Instead of using
sk->sk_socketto get back the socket reference, store the 'socket' pointer that is coming fromkernel_accept().This doesn't fix a bug, but it appears cleaner to me.
Signed-off-by Florian Westphal <fw(a)strlen.de>--- net/mptcp/protocol.c | 19 ++++++++++--------- net/mptcp/protocol.h | 6 +++--- net/mptcp/subflow.c | 10 +++++++--- 3 files changed, 20 insertions(+), 15 deletions(-)net/mptcp/protocol.c +10
-9@@ -27,7 +27,7 @@ static inline bool before64(__u64 seq1, __u64 seq2)static bool mptcp_subflow_hold (struct subflow_context *subflow) { -struct sock *sk =sock_sk (subflow); +struct sock *sk =mptcp_subflow_socket (subflow)->sk;return refcount_inc_not_zero (&sk->sk_refcnt); }@@ -40,7 +40,7 @@ static struct sock *mptcp_subflow_get_ref(const struct mptcp_sock *msk)mptcp_for_each_subflow (msk, subflow) {if (mptcp_subflow_hold (subflow)) {rcu_read_unlock (); -return sock_sk (subflow); +return mptcp_subflow_socket (subflow)->sk; } }@@ -232,7 +232,7 @@ static u64 expand_seq(u64 old_seq, u16 old_data_len, u64 seq)static u64 get_mapped_dsn (struct subflow_context *subflow) { -u32 map_offset = (tcp_sk (sock_sk (subflow))->copied_seq - +u32 map_offset = (tcp_sk (mptcp_subflow_socket (subflow)->sk)->copied_seq - subflow->ssn_offset - subflow->map_subflow_seq);@@ -601,8 +601,8 @@ static void mptcp_close(struct sock *sk, long timeout)list_for_each_entry_safe (subflow, tmp, &list, node) {pr_debug ("conn_list->subflow=%p" , subflow); -sock_release (sock_sk (subflow)->sk_socket); - }; +sock_release (mptcp_subflow_socket (subflow)); + }sock_orphan (sk);sock_put (sk);@@ -656,6 +656,7 @@ static struct sock *mptcp_accept(struct sock *sk, int flags, int *err, subflow->map_subflow_seq =1 ; subflow->rel_write_seq =1 ; subflow->conn = new_mptcp_sock->sk; + subflow->tcp_sock = new_sock; }else { msk->subflow = new_sock; }@@ -934,8 +935,8 @@ static __poll_t mptcp_poll(struct file *file, struct socket *sock,if (!tmp)break ; - ret |=tcp_poll (file,sock_sk (tmp)->sk_socket, wait); -sock_put (sock_sk (tmp)); + ret |=tcp_poll (file,mptcp_subflow_socket (tmp), wait); +sock_put (mptcp_subflow_socket (tmp)->sk); }return ret;@@ -964,9 +965,9 @@ static int mptcp_shutdown(struct socket *sock, int how)lock_sock (sock->sk);rcu_read_lock ();list_for_each_entry_rcu (subflow, &msk->conn_list, node) { -rcu_read_unlock ();pr_debug ("conn_list->subflow=%p" , subflow); - ret =kernel_sock_shutdown (sock_sk (subflow)->sk_socket, how); +rcu_read_unlock (); + ret =kernel_sock_shutdown (mptcp_subflow_socket (subflow), how);rcu_read_lock (); }rcu_read_unlock ();net/mptcp/protocol.h +1
-1@@ -90,7 +90,7 @@ struct subflow_context { map_valid :1 , use_checksum : 1; -struct sock *sk;/* underlying tcp_sock */ +struct socket *tcp_sock;/* underlying tcp_sock */ struct sock *conn;/* parent mptcp_sock */ void (*tcp_sk_data_ready)(struct sock *sk); };
net/mptcp/protocol.h static inline struct subflow_context *subflow_ctx(const struct sock *sk) · line 105
Quoted 6 lines
net/mptcp/protocol.h +1
-1@@ -101,9 +101,9 @@ static inline struct subflow_context *subflow_ctx(const struct sock *sk)return (struct subflow_context *)icsk->icsk_ulp_data; } -static inline struct sock *sock_sk (const struct subflow_context *subflow) +static inline struct socket *mptcp_subflow_socket (const struct subflow_context *subflow) {
This name is a unclear, the intent is to return the "underlying tcp socket", so how about subflow_tcp_socket? Then this one and the previous are clearly the inverse of each other.
Peter.
Quoted 47 lines
net/mptcp/protocol.h +1
-1-return subflow->sk; +return subflow->tcp_sock; }int subflow_init (void );net/mptcp/subflow.c +7
-3@@ -174,6 +174,7 @@ static void subflow_data_ready(struct sock *sk) }static struct subflow_context *subflow_create_ctx (struct sock *sk, +struct socket *sock,gfp_t priority) {struct inet_connection_sock *icsk =inet_csk (sk);@@ -187,7 +188,8 @@ static struct subflow_context *subflow_create_ctx(struct sock *sk,pr_debug ("subflow=%p" , ctx); - ctx->sk = sk; +/* might be NULL */ + ctx->tcp_sock = sock;return ctx; }@@ -199,7 +201,7 @@ static int subflow_ulp_init(struct sock *sk)struct subflow_context *ctx;int err =0 ; - ctx =subflow_create_ctx (sk,GFP_KERNEL ); + ctx =subflow_create_ctx (sk, sk->sk_socket,GFP_KERNEL );if (!ctx) { err = -ENOMEM ; goto out;@@ -234,7 +236,9 @@ static void subflow_ulp_clone(const struct request_sock *req,struct subflow_context *new_ctx; old_ctx =inet_csk (newsk)->icsk_ulp_data; - new_ctx =subflow_create_ctx (newsk, priority); + +/* newsk->sk_socket is NULL at this point */ + new_ctx =subflow_create_ctx (newsk,NULL , priority);if (!new_ctx)return ;
Florian Westphal
PATCH 3/3 mptcp: accept: don't leak mptcp socket structure
In reply to Florian Westphalaccept() is supposed to prepare and return a 'struct sock'.
The caller holds a new inode/socket, and will associate the returned sock with it.
mptcp_accept however will allocate it via 'struct socket', then returns socket->sk.
This then leaks the outer socket struct inode returned by sock_create():
unreferenced object 0xffff88810512e8c0 (size 936):
comm "mptcp_connect", [..]
backtrace:
[<00000000872561ba>] alloc_inode+0x35/0xe0
[<00000000646e04ed>] new_inode_pseudo+0x12/0x80
[<00000000e2e77036>] sock_alloc+0x26/0x100
[<00000000870a8688>] __sock_create+0x8f/0x3c0
[<000000000558b3fa>] mptcp_accept+0x140/0x530
[<0000000044718d60>] inet_accept+0xac/0x470
[<00000000da8f3979>] mptcp_stream_accept+0x62/0xa0
[<00000000c9010499>] __sys_accept4+0x228/0x3c0To fix this, make several (unfortunately, intrusive) changes:
1. Instead of allocating a new mptcp socket, clone the mptcp listen socket socket->sk.
This gives us a mptcp sock without the inode container. We return this sock struct to the caller, the caller will then complete creation of the mptcp socket.
2. For the 'compat' (old tcp, not mp capable) case, return the tcp socket directly and release the socket coming from kernel_accept().
We can use mptcp_getname() to override the socket->ops to tcp. This will make mptcp_accept work like tcp accept, and mptcp won't be involved anymore for the particular socket.
--- net/mptcp/protocol.c | 58 ++++++++++++++++++++++++++++++-------------- 1 file changed, 40 insertions(+), 18 deletions(-)
net/mptcp/protocol.c +40 -18
@@ -613,9 +613,9 @@ static struct sock *mptcp_accept(struct sock *sk, int flags, int *err, {struct mptcp_sock *msk =mptcp_sk (sk);struct socket *listener = msk->subflow; -struct socket *new_sock; -struct socket *new_mptcp_sock;struct subflow_context *subflow; +struct socket *new_sock; +struct sock *newsk;pr_debug ("msk=%p, listener=%p" , msk,subflow_ctx (listener->sk)); *err =kernel_accept (listener, &new_sock, flags);
@@ -625,28 +625,32 @@ static struct sock *mptcp_accept(struct sock *sk, int flags, int *err, subflow =subflow_ctx (new_sock->sk);pr_debug ("new subflow=%p" , subflow); - *err =sock_create (PF_INET ,SOCK_STREAM ,IPPROTO_MPTCP , &new_mptcp_sock); -if (*err <0 ) { -kernel_sock_shutdown (new_sock,SHUT_RDWR ); -sock_release (new_sock); -return NULL ; - } - - msk =mptcp_sk (new_mptcp_sock->sk);pr_debug ("new msk=%p" , msk);if (subflow->mp_capable) { +struct sock *new_mptcp_sock;u64 ack_seq; +local_bh_disable (); + new_mptcp_sock =sk_clone_lock (sk,GFP_ATOMIC ); +if (!new_sock) { + *err = -ENOBUFS ; +local_bh_enable (); +kernel_sock_shutdown (new_sock,SHUT_RDWR ); +sock_release (new_sock); +return NULL ; + } + + msk =mptcp_sk (new_mptcp_sock); msk->remote_key = subflow->remote_key; msk->local_key = subflow->local_key; msk->token = subflow->token;pr_debug ("token=%u" , msk->token); -token_update_accept (new_sock->sk, new_mptcp_sock->sk); -spin_lock_bh (&msk->conn_list_lock); +token_update_accept (new_sock->sk, new_mptcp_sock); +spin_lock (&msk->conn_list_lock);list_add_rcu (&subflow->node, &msk->conn_list); msk->subflow =NULL ; -spin_unlock_bh (&msk->conn_list_lock); +spin_unlock (&msk->conn_list_lock);crypto_key_sha1 (msk->remote_key,NULL , &ack_seq); msk->write_seq = subflow->idsn +1 ;
@@ -655,14 +659,20 @@ static struct sock *mptcp_accept(struct sock *sk, int flags, int *err, subflow->map_seq = ack_seq; subflow->map_subflow_seq =1 ; subflow->rel_write_seq =1 ; - subflow->conn = new_mptcp_sock->sk; subflow->tcp_sock = new_sock; + newsk = new_mptcp_sock; + subflow->conn = new_mptcp_sock; +bh_unlock_sock (new_mptcp_sock); +local_bh_enable (); +inet_sk_state_store (newsk,TCP_ESTABLISHED ); }else { - msk->subflow = new_sock; + newsk = new_sock->sk; +tcp_sk (newsk)->is_mptcp =0 ; + new_sock->sk =NULL ; +sock_release (new_sock); } -inet_sk_state_store (new_mptcp_sock->sk,TCP_ESTABLISHED ); -return new_mptcp_sock->sk; +return newsk; }static void mptcp_destroy (struct sock *sk)
@@ -852,6 +862,18 @@ static int mptcp_getname(struct socket *sock, struct sockaddr *uaddr,pr_debug ("msk=%p" , msk); +if (sock->sk->sk_prot == &tcp_prot) { +/* we are being invoked from __sys_accept4, after +* mptcp_accept() has just accepted a non-mp-capable +* flow: sk is a tcp_sk, not an mptcp one. +* +* Hand the socket over to tcp so all further socket ops +* bypass mptcp. +*/ + sock->ops = &inet_stream_ops; +return inet_getname (sock, uaddr, peer); + } +if (msk->subflow) {pr_debug ("subflow=%p" , msk->subflow->sk);return inet_getname (msk->subflow, uaddr, peer);
@@ -965,8 +987,8 @@ static int mptcp_shutdown(struct socket *sock, int how)lock_sock (sock->sk);rcu_read_lock ();list_for_each_entry_rcu (subflow, &msk->conn_list, node) { -pr_debug ("conn_list->subflow=%p" , subflow);rcu_read_unlock (); +pr_debug ("conn_list->subflow=%p" , subflow); ret =kernel_sock_shutdown (mptcp_subflow_socket (subflow), how);rcu_read_lock (); }
Signature · 1 line
2.21.0
Krystad
Hi Florian -
Thanks for this, it looks good to me with just one small comment on #2.
Peter.
Quoted 27 lines · Florian Westphal
Hi,
This series resolves the memory leaks that occur when running
mptcp_connect.sh.I don't mind if they get squashed.
I've decided to split this into 3 distinct changes to make it a bit clearer what is going on.Second patch is not a bug fix, i can resend and drop it if the consensus is to continue using the
sk->sk_socketback reference.Third patch is a bit large due to the change in how the new mptcp socket is created.
See the commit messages for more details.
After these changes kmemleak doesn't report any leaks when the test script has completed.Cheers,
Florian_______________________________________________
mptcp mailing list
mptcp(a)lists.01.org
https://lists.01.org/mailman/listinfo/mptcp
Matthieu Baerts
Hi Peter,
Thank you for the review! And thank you Florian for the patch-set!
Quoted 4 lines · Peter
Hi Florian -
Thanks for this, it looks good to me with just one small comment on #2.
Is the v2 OK for you?
Thanks,
Matt
Signature · 5 lines
Matthieu Baerts | R&D Engineer matthieu.baerts(a)tessares.net Tessares SA | Hybrid Access Solutions www.tessares.net 1 Avenue Jean Monnet, 1348 Louvain-la-Neuve, Belgium