mptcp: token container refactor
Subsystems (2)
Paolo Abeni
cover mptcp: token container refactor
This series supersede the RTC patch for token refactor. The relevant patch is unchanged, but this additional bring in some related minor cleanup (patch 1/4) and KUNIT (!!!) self-tests (4/4). To make things cleaner it also moves the existing in-kernel tests (crypto) to KUNIT.
Paolo Abeni (4):
mptcp: add __init annotation on setup functions
mptcp: refactor token container.
mptcp: move crypto test to KUNIT
mptcp: introduce token KUNIT self-tests
net/mptcp/Kconfig | 19 ++- net/mptcp/Makefile | 4 + net/mptcp/crypto.c | 65 +--------- net/mptcp/crypto_test.c | 71 +++++++++++ net/mptcp/protocol.c | 21 +--- net/mptcp/protocol.h | 14 ++- net/mptcp/subflow.c | 12 +- net/mptcp/token.c | 255 ++++++++++++++++++++++++++++------------ net/mptcp/token_test.c | 138 ++++++++++++++++++++++ 9 files changed, 431 insertions(+), 168 deletions(-) create mode 100644 net/mptcp/crypto_test.c create mode 100644 net/mptcp/token_test.c
Signature · 1 line
2.21.3
Paolo Abeni
PATCH 1/4 mptcp: add __init annotation on setup functions
Add the missing annotation in some setup-only
functions.
--- net/mptcp/protocol.c | 5 +++-- net/mptcp/protocol.h | 9 +++++---- net/mptcp/subflow.c | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-)
net/mptcp/protocol.c +3 -2
@@ -2028,7 +2028,7 @@ static struct inet_protosw mptcp_protosw = { .flags = INET_PROTOSW_ICSK, }; -void mptcp_proto_init (void ) +void __init mptcp_proto_init (void ) { mptcp_prot.h.hashinfo = tcp_prot.h.hashinfo;
@@ -2037,6 +2037,7 @@ void mptcp_proto_init(void)mptcp_subflow_init ();mptcp_pm_init (); +mptcp_token_init ();if (proto_register (&mptcp_prot,1 ) !=0 )panic ("Failed to register MPTCP proto.\n" );
@@ -2089,7 +2090,7 @@ static struct inet_protosw mptcp_v6_protosw = { .flags = INET_PROTOSW_ICSK, }; -int mptcp_proto_v6_init (void ) +int __init mptcp_proto_v6_init (void ) {int err;
net/mptcp/protocol.h +5 -4
@@ -342,7 +342,7 @@ mptcp_subflow_get_mapped_dsn(const struct mptcp_subflow_context *subflow)int mptcp_is_enabled (struct net *net);bool mptcp_subflow_data_available (struct sock *sk); -void mptcp_subflow_init (void ); +void __init mptcp_subflow_init (void );/* called with sk socket lock held */ int __mptcp_subflow_connect(struct sock *sk, int ifindex,
@@ -365,9 +365,9 @@ extern const struct inet_connection_sock_af_ops ipv4_specific;extern const struct inet_connection_sock_af_ops ipv6_specific;#endif -void mptcp_proto_init (void ); +void __init mptcp_proto_init (void );#if IS_ENABLED (CONFIG_MPTCP_IPV6 ) -int mptcp_proto_v6_init (void ); +int __init mptcp_proto_v6_init (void );#endif struct sock *mptcp_sk_clone(const struct sock *sk,
@@ -382,6 +382,7 @@ bool mptcp_finish_join(struct sock *sk);void mptcp_data_acked (struct sock *sk);void mptcp_subflow_eof (struct sock *sk); +void __init mptcp_token_init (void );int mptcp_token_new_request (struct request_sock *req);void mptcp_token_destroy_request (u32 token);int mptcp_token_new_connect (struct sock *sk);
@@ -404,7 +405,7 @@ static inline void mptcp_crypto_key_gen_sha(u64 *key, u32 *token, u64 *idsn)void mptcp_crypto_hmac_sha (u64 key1,u64 key2,u8 *msg,int len,void *hmac); -void mptcp_pm_init (void ); +void __init mptcp_pm_init (void );void mptcp_pm_data_init (struct mptcp_sock *msk);void mptcp_pm_close (struct mptcp_sock *msk);void mptcp_pm_new_connection (struct mptcp_sock *msk,int server_side);
net/mptcp/subflow.c +1 -1
@@ -1239,7 +1239,7 @@ static int subflow_ops_init(struct request_sock_ops *subflow_ops)return 0 ; } -void mptcp_subflow_init (void ) +void __init mptcp_subflow_init (void ) { subflow_request_sock_ops = tcp_request_sock_ops;if (subflow_ops_init (&subflow_request_sock_ops) !=0 )
Signature · 1 line
2.21.3
Christoph Paasch
[MPTCP] Re: [PATCH 1/4] mptcp: add __init annotation on setup functions
Hello Paolo,
Quoted 66 lines · Paolo Abeni
Add the missing annotation in some setup-only
functions.Signed-off-by Paolo Abeni <pabeni(a)redhat.com>--- net/mptcp/protocol.c | 5 +++-- net/mptcp/protocol.h | 9 +++++---- net/mptcp/subflow.c | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-)net/mptcp/protocol.c +3
-2@@ -2028,7 +2028,7 @@ static struct inet_protosw mptcp_protosw = { .flags = INET_PROTOSW_ICSK, }; -void mptcp_proto_init (void ) +void __init mptcp_proto_init (void ) { mptcp_prot.h.hashinfo = tcp_prot.h.hashinfo;@@ -2037,6 +2037,7 @@ void mptcp_proto_init(void)mptcp_subflow_init ();mptcp_pm_init (); +mptcp_token_init ();if (proto_register (&mptcp_prot,1 ) !=0 )panic ("Failed to register MPTCP proto.\n" );@@ -2089,7 +2090,7 @@ static struct inet_protosw mptcp_v6_protosw = { .flags = INET_PROTOSW_ICSK, }; -int mptcp_proto_v6_init (void ) +int __init mptcp_proto_v6_init (void ) {int err;net/mptcp/protocol.h +3
-3@@ -342,7 +342,7 @@ mptcp_subflow_get_mapped_dsn(const struct mptcp_subflow_context *subflow)int mptcp_is_enabled (struct net *net);bool mptcp_subflow_data_available (struct sock *sk); -void mptcp_subflow_init (void ); +void __init mptcp_subflow_init (void );/* called with sk socket lock held */ int __mptcp_subflow_connect(struct sock *sk, int ifindex,@@ -365,9 +365,9 @@ extern const struct inet_connection_sock_af_ops ipv4_specific;extern const struct inet_connection_sock_af_ops ipv6_specific;#endif -void mptcp_proto_init (void ); +void __init mptcp_proto_init (void );#if IS_ENABLED (CONFIG_MPTCP_IPV6 ) -int mptcp_proto_v6_init (void ); +int __init mptcp_proto_v6_init (void );#endif struct sock *mptcp_sk_clone(const struct sock *sk,
net/mptcp/protocol.h bool mptcp_finish_join(struct sock *sk); · line 385
Quoted 4 lines
net/mptcp/protocol.h +1
-0@@ -382,6 +382,7 @@ bool mptcp_finish_join(struct sock *sk);void mptcp_data_acked (struct sock *sk);void mptcp_subflow_eof (struct sock *sk); +void __init mptcp_token_init (void );
I think you need to define the empty mptcp_token_init() as otherwise this patch won't compile.
Christoph
Paolo Abeni
PATCH 2/4 mptcp: refactor token container.
In reply to Paolo AbeniReplace the radix tree with an hash table allocated at boot time. The radix tree has some short coming:
a single lock is contented by all the mptcp operation, the lookup currently use such lock, and traversing all the items would require lock, too.
With hash table instead we trade a little memory to address all the above - a per bucket lock is used.
Additionally refactor the token creation to code to:
- limit the number of consecutive attempts to a fixed maximum. Hitting an hash bucket with long chain is considered a failed attempt
- accept() no longer can fail to to token management.
- if token creation fails at connect() time, we do fallback to TCP (before the connection was closed)
--- net/mptcp/protocol.c | 16 +-- net/mptcp/protocol.h | 5 +- net/mptcp/subflow.c | 10 +- net/mptcp/token.c | 246 ++++++++++++++++++++++++++++++------------- 4 files changed, 184 insertions(+), 93 deletions(-)
net/mptcp/protocol.c +2 -14
@@ -1424,19 +1424,7 @@ struct sock *mptcp_sk_clone(const struct sock *sk, msk->token = subflow_req->token; msk->subflow =NULL ; -if (unlikely (mptcp_token_new_accept (subflow_req->token, nsk))) { - nsk->sk_state =TCP_CLOSE ; -bh_unlock_sock (nsk); - -/* we can't call into mptcp_close() here - possible BH context -* free the sock directly. -* sk_clone_lock() sets nsk refcnt to two, hence call sk_free() -* too. -*/ -sk_common_release (nsk); -sk_free (nsk); -return NULL ; - } +mptcp_token_accept (subflow_req, msk); msk->write_seq = subflow_req->idsn +1 ;atomic64_set (&msk->snd_una, msk->write_seq);
@@ -1654,7 +1642,6 @@ void mptcp_finish_connect(struct sock *ssk)*/ WRITE_ONCE (msk->remote_key, subflow->remote_key);WRITE_ONCE (msk->local_key, subflow->local_key); -WRITE_ONCE (msk->token, subflow->token);WRITE_ONCE (msk->write_seq, subflow->idsn +1 );WRITE_ONCE (msk->ack_seq, ack_seq);WRITE_ONCE (msk->can_ack,1 );
@@ -1738,6 +1725,7 @@ static struct proto mptcp_prot = { .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem), .sysctl_mem = sysctl_tcp_mem, .obj_size = sizeof(struct mptcp_sock), + .slab_flags = SLAB_TYPESAFE_BY_RCU, .no_autobind = true, };
net/mptcp/protocol.h +4 -1
@@ -211,6 +211,7 @@ struct mptcp_sock {struct socket *subflow;/* outgoing connect/listener/!mp_capable */ struct sock *first;struct mptcp_pm_data pm; +struct hlist_nulls_node token_node; };#define mptcp_for_each_subflow (__msk, __subflow) \
@@ -255,6 +256,7 @@ struct mptcp_subflow_request_sock {u64 thmac;u32 local_nonce;u32 remote_nonce; +struct hlist_nulls_node token_node; }; static inline struct mptcp_subflow_request_sock *
@@ -386,7 +388,8 @@ void __init mptcp_token_init(void);int mptcp_token_new_request (struct request_sock *req);void mptcp_token_destroy_request (u32 token);int mptcp_token_new_connect (struct sock *sk); -int mptcp_token_new_accept (u32 token,struct sock *conn); +void mptcp_token_accept (struct mptcp_subflow_request_sock *r, +struct mptcp_sock *msk);struct mptcp_sock *mptcp_token_get_sock (u32 token);void mptcp_token_destroy (u32 token);
net/mptcp/subflow.c +5 -5
@@ -31,11 +31,14 @@ static void SUBFLOW_REQ_INC_STATS(struct request_sock *req,static int subflow_rebuild_header (struct sock *sk) {struct mptcp_subflow_context *subflow =mptcp_subflow_ctx (sk); -int local_id, err =0 ; +int local_id;if (subflow->request_mptcp && !subflow->token) {pr_debug ("subflow=%p" , sk); - err =mptcp_token_new_connect (sk); +if (mptcp_token_new_connect (sk)) { + subflow->mp_capable =0 ; + goto out; + } }else if (subflow->request_join && !subflow->local_nonce) {struct mptcp_sock *msk = (struct mptcp_sock *)subflow->conn;
@@ -56,9 +59,6 @@ static int subflow_rebuild_header(struct sock *sk) } out: -if (err) -return err; -return subflow->icsk_af_ops->rebuild_header (sk); }
net/mptcp/token.c +173 -73
304 lines not on this page · raw message
Signature · 1 line
2.21.3
Christoph Paasch
[MPTCP] Re: [PATCH 2/4] mptcp: refactor token container.
Hello,
Quoted 62 lines · Paolo Abeni
Replace the radix tree with an hash table allocated at boot time. The radix tree has some short coming:
a single lock is contented by all the mptcp operation, the lookup currently use such lock, and traversing all the items would require lock, too.With hash table instead we trade a little memory to address all the above - a per bucket lock is used.
Additionally refactor the token creation to code to:
- limit the number of consecutive attempts to a fixed maximum. Hitting an hash bucket with long chain is considered a failed attempt
-
accept()no longer can fail to to token management.- if token creation fails at
connect()time, we do fallback to TCP (before the connection was closed)Signed-off-by Paolo Abeni <pabeni(a)redhat.com>--- net/mptcp/protocol.c | 16 +-- net/mptcp/protocol.h | 5 +- net/mptcp/subflow.c | 10 +- net/mptcp/token.c | 246 ++++++++++++++++++++++++++++++------------- 4 files changed, 184 insertions(+), 93 deletions(-)net/mptcp/protocol.c +1
-14@@ -1424,19 +1424,7 @@ struct sock *mptcp_sk_clone(const struct sock *sk, msk->token = subflow_req->token; msk->subflow =NULL ; -if (unlikely (mptcp_token_new_accept (subflow_req->token, nsk))) { - nsk->sk_state =TCP_CLOSE ; -bh_unlock_sock (nsk); - -/* we can't call into mptcp_close() here - possible BH context -* free the sock directly. -* sk_clone_lock() sets nsk refcnt to two, hence call sk_free() -* too. -*/ -sk_common_release (nsk); -sk_free (nsk); -return NULL ; - } +mptcp_token_accept (subflow_req, msk); msk->write_seq = subflow_req->idsn +1 ;atomic64_set (&msk->snd_una, msk->write_seq);@@ -1654,7 +1642,6 @@ void mptcp_finish_connect(struct sock *ssk)*/ WRITE_ONCE (msk->remote_key, subflow->remote_key);WRITE_ONCE (msk->local_key, subflow->local_key); -WRITE_ONCE (msk->token, subflow->token);WRITE_ONCE (msk->write_seq, subflow->idsn +1 );WRITE_ONCE (msk->ack_seq, ack_seq);WRITE_ONCE (msk->can_ack,1 );
net/mptcp/protocol.c static struct proto mptcp_prot = { · line 1728
Quoted 4 lines
net/mptcp/protocol.c +1
-0@@ -1738,6 +1725,7 @@ static struct proto mptcp_prot = { .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem), .sysctl_mem = sysctl_tcp_mem, .obj_size = sizeof(struct mptcp_sock), + .slab_flags = SLAB_TYPESAFE_BY_RCU,
I wonder if you now need to be careful when allocating and zero'ing the socket, same way it is happening in sock_copy ?
In out-of-tree I had to take care of that by bringing back the clear_sk callback in struct proto which will then be called in sk_prot_alloc:
Quoted 293 lines
net/mptcp/protocol.c +0
-0.no_autobind = true, };net/mptcp/protocol.h +4
-1@@ -211,6 +211,7 @@ struct mptcp_sock {struct socket *subflow;/* outgoing connect/listener/!mp_capable */ struct sock *first;struct mptcp_pm_data pm; +struct hlist_nulls_node token_node; };#define mptcp_for_each_subflow (__msk, __subflow) \@@ -255,6 +256,7 @@ struct mptcp_subflow_request_sock {u64 thmac;u32 local_nonce;u32 remote_nonce; +struct hlist_nulls_node token_node; }; static inline struct mptcp_subflow_request_sock *@@ -386,7 +388,8 @@ void __init mptcp_token_init(void);int mptcp_token_new_request (struct request_sock *req);void mptcp_token_destroy_request (u32 token);int mptcp_token_new_connect (struct sock *sk); -int mptcp_token_new_accept (u32 token,struct sock *conn); +void mptcp_token_accept (struct mptcp_subflow_request_sock *r, +struct mptcp_sock *msk);struct mptcp_sock *mptcp_token_get_sock (u32 token);void mptcp_token_destroy (u32 token);net/mptcp/subflow.c +5
-5@@ -31,11 +31,14 @@ static void SUBFLOW_REQ_INC_STATS(struct request_sock *req,static int subflow_rebuild_header (struct sock *sk) {struct mptcp_subflow_context *subflow =mptcp_subflow_ctx (sk); -int local_id, err =0 ; +int local_id;if (subflow->request_mptcp && !subflow->token) {pr_debug ("subflow=%p" , sk); - err =mptcp_token_new_connect (sk); +if (mptcp_token_new_connect (sk)) { + subflow->mp_capable =0 ; + goto out; + } }else if (subflow->request_join && !subflow->local_nonce) {struct mptcp_sock *msk = (struct mptcp_sock *)subflow->conn;@@ -56,9 +59,6 @@ static int subflow_rebuild_header(struct sock *sk) } out: -if (err) -return err; -return subflow->icsk_af_ops->rebuild_header (sk); }net/mptcp/token.c +112
-65@@ -24,7 +24,7 @@#include <linux/kernel.h> #include <linux/module.h> -#include <linux/radix-tree.h> +#include <linux/memblock.h> #include <linux/ip.h> #include <linux/tcp.h> #include <net/sock.h> @@ -33,10 +33,53 @@#include <net/mptcp.h> #include "protocol.h" -static RADIX_TREE(token_tree, GFP_ATOMIC); -static RADIX_TREE(token_req_tree, GFP_ATOMIC); -static DEFINE_SPINLOCK (token_tree_lock); -static int token_used__read_mostly ; +#define TOKEN_MAX_RETRIES 4 +#define TOKEN_MAX_CHAIN_LEN 4 + +struct token_bucket { +spinlock_t lock; +int chain_len; +struct hlist_nulls_head req_chain; +struct hlist_nulls_head msk_chain; +}; + +static struct token_bucket *token_hash__read_mostly ; +static unsigned int token_mask__read_mostly ; + +static struct token_bucket *token_bucket (u32 token) +{ +return &token_hash[token & token_mask]; +} + +static struct mptcp_subflow_request_sock * +__token_lookup_req (struct token_bucket *t,u32 token) +{ +struct mptcp_subflow_request_sock *req; +struct hlist_nulls_node *pos; + +hlist_nulls_for_each_entry_rcu (req, pos, &t->req_chain, token_node) +if (req->token == token) +return req; +return NULL ; +} + +static struct mptcp_sock * +__token_lookup_msk (struct token_bucket *t,u32 token) +{ +struct hlist_nulls_node *pos; +struct mptcp_sock *msk; + +hlist_nulls_for_each_entry_rcu (msk, pos, &t->msk_chain, token_node) +if (msk->token == token) +return msk; +return NULL ; +} + +bool __token_bucket_busy (struct token_bucket *t,u32 token) +{ +return !token || t->chain_len >=TOKEN_MAX_CHAIN_LEN || +__token_lookup_req (t, token) ||__token_lookup_msk (t, token); +}/** * mptcp_token_new_request - create new key/idsn/token for subflow_request @@ -52,30 +95,32 @@ static int token_used __read_mostly;int mptcp_token_new_request (struct request_sock *req) {struct mptcp_subflow_request_sock *subflow_req =mptcp_subflow_rsk (req); -int err; - -while (1 ) { -u32 token; - -mptcp_crypto_key_gen_sha (&subflow_req->local_key, - &subflow_req->token, - &subflow_req->idsn); -pr_debug ("req=%p local_key=%llu, token=%u, idsn=%llu\n" , - req, subflow_req->local_key, subflow_req->token, - subflow_req->idsn); - - token = subflow_req->token; -spin_lock_bh (&token_tree_lock); -if (!radix_tree_lookup (&token_req_tree, token) && - !radix_tree_lookup (&token_tree, token)) -break ; -spin_unlock_bh (&token_tree_lock); +int retries =TOKEN_MAX_RETRIES ; +struct token_bucket *bucket; +u32 token; + +again: +mptcp_crypto_key_gen_sha (&subflow_req->local_key, + &subflow_req->token, + &subflow_req->idsn); +pr_debug ("req=%p local_key=%llu, token=%u, idsn=%llu\n" , + req, subflow_req->local_key, subflow_req->token, + subflow_req->idsn); + + token = subflow_req->token; + bucket =token_bucket (token); +spin_lock_bh (&bucket->lock); +if (__token_bucket_busy (bucket, token)) { +spin_unlock_bh (&bucket->lock); +if (!--retries) +return -EBUSY ; + goto again; } - err =radix_tree_insert (&token_req_tree, - subflow_req->token, &token_used); -spin_unlock_bh (&token_tree_lock); -return err; +hlist_nulls_add_tail_rcu (&subflow_req->token_node, &bucket->req_chain); + bucket->chain_len++; +spin_unlock_bh (&bucket->lock); +return 0 ; }/** @@ -97,48 +142,54 @@ int mptcp_token_new_request(struct request_sock *req)int mptcp_token_new_connect (struct sock *sk) {struct mptcp_subflow_context *subflow =mptcp_subflow_ctx (sk); -struct sock *mptcp_sock = subflow->conn; -int err; +struct mptcp_sock *msk =mptcp_sk (subflow->conn); +int retries =TOKEN_MAX_RETRIES ; +struct token_bucket *bucket; -while (1 ) { -u32 token; +pr_debug ("ssk=%p, local_key=%llu, token=%u, idsn=%llu\n" , + sk, subflow->local_key, subflow->token, subflow->idsn); -mptcp_crypto_key_gen_sha (&subflow->local_key, &subflow->token, - &subflow->idsn); +again: +mptcp_crypto_key_gen_sha (&subflow->local_key, &subflow->token, + &subflow->idsn); -pr_debug ("ssk=%p, local_key=%llu, token=%u, idsn=%llu\n" , - sk, subflow->local_key, subflow->token, subflow->idsn); - - token = subflow->token; -spin_lock_bh (&token_tree_lock); -if (!radix_tree_lookup (&token_req_tree, token) && - !radix_tree_lookup (&token_tree, token)) -break ; -spin_unlock_bh (&token_tree_lock); + bucket =token_bucket (subflow->token); +spin_lock_bh (&bucket->lock); +if (__token_bucket_busy (bucket, subflow->token)) { +spin_unlock_bh (&bucket->lock); +if (!--retries) +return -EBUSY ; + goto again; } - err =radix_tree_insert (&token_tree, subflow->token, mptcp_sock); -spin_unlock_bh (&token_tree_lock); -return err; +WRITE_ONCE (msk->token, subflow->token); +hlist_nulls_add_tail_rcu (&msk->token_node, &bucket->msk_chain); + bucket->chain_len++; +spin_unlock_bh (&bucket->lock); +return 0 ; }/** -* mptcp_token_new_accept - insert token for later processing -* @token: the token to insert to the tree -* @conn: the just cloned socket linked to the new connection +* mptcp_token_accept - replace a req sk with full sock in token hash +* @req: the request socket to be removed +* @msk: the just cloned socket linked to the new connection * * Called when a SYN packet creates a new logical connection, i.e. * is not a join request. */ -int mptcp_token_new_accept (u32 token,struct sock *conn) +void mptcp_token_accept (struct mptcp_subflow_request_sock *req, +struct mptcp_sock *msk) { -int err; - -spin_lock_bh (&token_tree_lock); - err =radix_tree_insert (&token_tree, token, conn); -spin_unlock_bh (&token_tree_lock); +struct mptcp_subflow_request_sock *pos; +struct token_bucket *bucket; -return err; + bucket =token_bucket (req->token); +spin_lock_bh (&bucket->lock); + pos =__token_lookup_req (bucket, req->token); +if (!WARN_ON_ONCE (pos != req)) +hlist_nulls_del_rcu (&req->token_node); +hlist_nulls_add_tail_rcu (&msk->token_node, &bucket->msk_chain); +spin_unlock_bh (&bucket->lock); }/** @@ -152,20 +203,33 @@ int mptcp_token_new_accept(u32 token, struct sock *conn)*/ struct mptcp_sock *mptcp_token_get_sock (u32 token) { -struct sock *conn; - -spin_lock_bh (&token_tree_lock); - conn =radix_tree_lookup (&token_tree, token); -if (conn) { -/* token still reserved? */ -if (conn == (struct sock *)&token_used) - conn =NULL ; -else -sock_hold (conn); +struct hlist_nulls_node *pos; +struct token_bucket *bucket; +struct mptcp_sock *msk; + +rcu_read_lock (); + bucket =token_bucket (token);
net/mptcp/token.c int mptcp_token_new_accept(u32 token, struct sock *conn) · line 221
Quoted 10 lines
net/mptcp/token.c +10
-0+ +again: +hlist_nulls_for_each_entry_rcu (msk, pos, &bucket->msk_chain, + token_node) { +if (READ_ONCE (msk->token) != token) +continue ; +if (!refcount_inc_not_zero (&((struct sock *)msk)->sk_refcnt)) + goto not_found; +if (READ_ONCE (msk->token) != token) + goto again;
I think you need to drop the refcount when going to "again" again
Christoph
Paolo Abeni
[MPTCP] Re: [PATCH 2/4] mptcp: refactor token container.
Quoted 73 lines · Christoph Paasch
Hello,
Quoted 62 lines · Paolo Abeni
Replace the radix tree with an hash table allocated at boot time. The radix tree has some short coming:
a single lock is contented by all the mptcp operation, the lookup currently use such lock, and traversing all the items would require lock, too.With hash table instead we trade a little memory to address all the above - a per bucket lock is used.
Additionally refactor the token creation to code to:
- limit the number of consecutive attempts to a fixed maximum. Hitting an hash bucket with long chain is considered a failed attempt
-
accept()no longer can fail to to token management.- if token creation fails at
connect()time, we do fallback to TCP (before the connection was closed)Signed-off-by Paolo Abeni <pabeni(a)redhat.com>--- net/mptcp/protocol.c | 16 +-- net/mptcp/protocol.h | 5 +- net/mptcp/subflow.c | 10 +- net/mptcp/token.c | 246 ++++++++++++++++++++++++++++++------------- 4 files changed, 184 insertions(+), 93 deletions(-)net/mptcp/protocol.c +1
-14@@ -1424,19 +1424,7 @@ struct sock *mptcp_sk_clone(const struct sock *sk, msk->token = subflow_req->token; msk->subflow =NULL ; -if (unlikely (mptcp_token_new_accept (subflow_req->token, nsk))) { - nsk->sk_state =TCP_CLOSE ; -bh_unlock_sock (nsk); - -/* we can't call into mptcp_close() here - possible BH context -* free the sock directly. -* sk_clone_lock() sets nsk refcnt to two, hence call sk_free() -* too. -*/ -sk_common_release (nsk); -sk_free (nsk); -return NULL ; - } +mptcp_token_accept (subflow_req, msk); msk->write_seq = subflow_req->idsn +1 ;atomic64_set (&msk->snd_una, msk->write_seq);@@ -1654,7 +1642,6 @@ void mptcp_finish_connect(struct sock *ssk)*/ WRITE_ONCE (msk->remote_key, subflow->remote_key);WRITE_ONCE (msk->local_key, subflow->local_key); -WRITE_ONCE (msk->token, subflow->token);WRITE_ONCE (msk->write_seq, subflow->idsn +1 );WRITE_ONCE (msk->ack_seq, ack_seq);WRITE_ONCE (msk->can_ack,1 );net/mptcp/protocol.c static struct proto mptcp_prot = { · line 1728
Quoted 4 lines
net/mptcp/protocol.c +1
-0@@ -1738,6 +1725,7 @@ static struct proto mptcp_prot = { .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem), .sysctl_mem = sysctl_tcp_mem, .obj_size = sizeof(struct mptcp_sock), + .slab_flags = SLAB_TYPESAFE_BY_RCU,I wonder if you now need to be careful when allocating and zero'ing the socket, same way it is happening in
sock_copy?
Quoted 6 lines
In out-of-tree I had to take care of that by bringing back the
clear_skcallback in struct proto which will then be called insk_prot_alloc:
Indeed! I felt like I was missing some relevant point!
re-adding the clear_sk callback for mptcp's sake does not look 110% nice. There are a few alternatives, which sounds equally suboptimal too me:
1) use plain RCU (with kfree_rcu() and all the relevant memory overhead) for the whole msk sockets (~1600 bytes for IPv4). We already to that for the subflow contexts (~200 bytes). This is simple, but the memory overhead could be relevant ?!?
1.a) additionally srink the mptcp_sock structure, e.g. I'm wild
guessing we can use 'struct sock' as the base type and add mptcp custom fields there ?!?
2) use ULP for msk, too. Move the token there (and possibly all mptcp- specific data), and use plain RCU to handle the context. As a downside we will need 2 allocations per accept() (the msk socket and the msk ulp context)
Any other better option?!?
Thanks!
Paolo
Christoph Paasch
[MPTCP] Re: [PATCH 2/4] mptcp: refactor token container.
Quoted 96 lines · Paolo Abeni
Quoted 73 lines · Christoph Paasch
Hello,
Quoted 62 lines · Paolo Abeni
Replace the radix tree with an hash table allocated at boot time. The radix tree has some short coming:
a single lock is contented by all the mptcp operation, the lookup currently use such lock, and traversing all the items would require lock, too.With hash table instead we trade a little memory to address all the above - a per bucket lock is used.
Additionally refactor the token creation to code to:
- limit the number of consecutive attempts to a fixed maximum. Hitting an hash bucket with long chain is considered a failed attempt
-
accept()no longer can fail to to token management.- if token creation fails at
connect()time, we do fallback to TCP (before the connection was closed)Signed-off-by Paolo Abeni <pabeni(a)redhat.com>--- net/mptcp/protocol.c | 16 +-- net/mptcp/protocol.h | 5 +- net/mptcp/subflow.c | 10 +- net/mptcp/token.c | 246 ++++++++++++++++++++++++++++++------------- 4 files changed, 184 insertions(+), 93 deletions(-)net/mptcp/protocol.c +1
-14@@ -1424,19 +1424,7 @@ struct sock *mptcp_sk_clone(const struct sock *sk, msk->token = subflow_req->token; msk->subflow =NULL ; -if (unlikely (mptcp_token_new_accept (subflow_req->token, nsk))) { - nsk->sk_state =TCP_CLOSE ; -bh_unlock_sock (nsk); - -/* we can't call into mptcp_close() here - possible BH context -* free the sock directly. -* sk_clone_lock() sets nsk refcnt to two, hence call sk_free() -* too. -*/ -sk_common_release (nsk); -sk_free (nsk); -return NULL ; - } +mptcp_token_accept (subflow_req, msk); msk->write_seq = subflow_req->idsn +1 ;atomic64_set (&msk->snd_una, msk->write_seq);@@ -1654,7 +1642,6 @@ void mptcp_finish_connect(struct sock *ssk)*/ WRITE_ONCE (msk->remote_key, subflow->remote_key);WRITE_ONCE (msk->local_key, subflow->local_key); -WRITE_ONCE (msk->token, subflow->token);WRITE_ONCE (msk->write_seq, subflow->idsn +1 );WRITE_ONCE (msk->ack_seq, ack_seq);WRITE_ONCE (msk->can_ack,1 );net/mptcp/protocol.c static struct proto mptcp_prot = { · line 1728
Quoted 4 lines
net/mptcp/protocol.c +1
-0@@ -1738,6 +1725,7 @@ static struct proto mptcp_prot = { .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem), .sysctl_mem = sysctl_tcp_mem, .obj_size = sizeof(struct mptcp_sock), + .slab_flags = SLAB_TYPESAFE_BY_RCU,I wonder if you now need to be careful when allocating and zero'ing the socket, same way it is happening in
sock_copy?Quoted 6 lines
In out-of-tree I had to take care of that by bringing back the
clear_skcallback in struct proto which will then be called insk_prot_alloc:Indeed! I felt like I was missing some relevant point!
re-adding the
clear_skcallback for mptcp's sake does not look 110% nice. There are a few alternatives, which sounds equally suboptimal too me:1) use plain RCU (with
kfree_rcu()and all the relevant memory overhead) for the whole msk sockets (~1600 bytes for IPv4). We already to that for the subflow contexts (~200 bytes). This is simple, but the memory overhead could be relevant ?!?1.a) additionally srink the
mptcp_sockstructure, e.g. I'm wild
guessing we can use 'struct sock' as the base type and add mptcp custom fields there ?!?
Quoted 6 lines
2) use ULP for msk, too. Move the token there (and possibly all mptcp- specific data), and use plain RCU to handle the context. As a downside we will need 2 allocations per
accept()(the msk socket and the msk ulp context)Any other better option?!?
If we exclude clear_sk, I can't see another option that the ones you described. Both have the problem that on a busy node we have the memory-overheads.
Christoph
Mat Martineau
[MPTCP] Re: [PATCH 2/4] mptcp: refactor token container.
In reply to Paolo AbeniQuoted 96 lines · Paolo Abeni
Quoted 73 lines · Christoph Paasch
Hello,
Quoted 62 lines · Paolo Abeni
Replace the radix tree with an hash table allocated at boot time. The radix tree has some short coming:
a single lock is contented by all the mptcp operation, the lookup currently use such lock, and traversing all the items would require lock, too.With hash table instead we trade a little memory to address all the above - a per bucket lock is used.
Additionally refactor the token creation to code to:
- limit the number of consecutive attempts to a fixed maximum. Hitting an hash bucket with long chain is considered a failed attempt
-
accept()no longer can fail to to token management.- if token creation fails at
connect()time, we do fallback to TCP (before the connection was closed)Signed-off-by Paolo Abeni <pabeni(a)redhat.com>--- net/mptcp/protocol.c | 16 +-- net/mptcp/protocol.h | 5 +- net/mptcp/subflow.c | 10 +- net/mptcp/token.c | 246 ++++++++++++++++++++++++++++++------------- 4 files changed, 184 insertions(+), 93 deletions(-)net/mptcp/protocol.c +1
-14@@ -1424,19 +1424,7 @@ struct sock *mptcp_sk_clone(const struct sock *sk, msk->token = subflow_req->token; msk->subflow =NULL ; -if (unlikely (mptcp_token_new_accept (subflow_req->token, nsk))) { - nsk->sk_state =TCP_CLOSE ; -bh_unlock_sock (nsk); - -/* we can't call into mptcp_close() here - possible BH context -* free the sock directly. -* sk_clone_lock() sets nsk refcnt to two, hence call sk_free() -* too. -*/ -sk_common_release (nsk); -sk_free (nsk); -return NULL ; - } +mptcp_token_accept (subflow_req, msk); msk->write_seq = subflow_req->idsn +1 ;atomic64_set (&msk->snd_una, msk->write_seq);@@ -1654,7 +1642,6 @@ void mptcp_finish_connect(struct sock *ssk)*/ WRITE_ONCE (msk->remote_key, subflow->remote_key);WRITE_ONCE (msk->local_key, subflow->local_key); -WRITE_ONCE (msk->token, subflow->token);WRITE_ONCE (msk->write_seq, subflow->idsn +1 );WRITE_ONCE (msk->ack_seq, ack_seq);WRITE_ONCE (msk->can_ack,1 );net/mptcp/protocol.c static struct proto mptcp_prot = { · line 1728
Quoted 4 lines
net/mptcp/protocol.c +1
-0@@ -1738,6 +1725,7 @@ static struct proto mptcp_prot = { .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem), .sysctl_mem = sysctl_tcp_mem, .obj_size = sizeof(struct mptcp_sock), + .slab_flags = SLAB_TYPESAFE_BY_RCU,I wonder if you now need to be careful when allocating and zero'ing the socket, same way it is happening in
sock_copy?Quoted 6 lines
In out-of-tree I had to take care of that by bringing back the
clear_skcallback in struct proto which will then be called insk_prot_alloc:Indeed! I felt like I was missing some relevant point!
re-adding the
clear_skcallback for mptcp's sake does not look 110% nice. There are a few alternatives, which sounds equally suboptimal too me:1) use plain RCU (with
kfree_rcu()and all the relevant memory overhead) for the whole msk sockets (~1600 bytes for IPv4). We already to that for the subflow contexts (~200 bytes). This is simple, but the memory overhead could be relevant ?!?1.a) additionally srink the
mptcp_sockstructure, e.g. I'm wild
guessing we can use 'struct sock' as the base type and add mptcp custom fields there ?!?
Quoted 7 lines
2) use ULP for msk, too. Move the token there (and possibly all mptcp- specific data), and use plain RCU to handle the context. As a downside we will need 2 allocations per
accept()(the msk socket and the msk ulp context)Any other better option?!?
I lean toward option 1 so we don't have to change around ULP to not expose MPTCP stuff (we rely on a kernel sock check to keep the subflow ULP from being exposed to userspace). I'd also like to keep ULP available for possible TLS support someday.
--
Mat Martineau
Intel
Paolo Abeni
[MPTCP] Re: [PATCH 2/4] mptcp: refactor token container.
Quoted 103 lines · Mat Martineau
Quoted 101 lines · Paolo Abeni
Quoted 73 lines · Christoph Paasch
Hello,
Quoted 62 lines · Paolo Abeni
Replace the radix tree with an hash table allocated at boot time. The radix tree has some short coming:
a single lock is contented by all the mptcp operation, the lookup currently use such lock, and traversing all the items would require lock, too.With hash table instead we trade a little memory to address all the above - a per bucket lock is used.
Additionally refactor the token creation to code to:
- limit the number of consecutive attempts to a fixed maximum. Hitting an hash bucket with long chain is considered a failed attempt
-
accept()no longer can fail to to token management.- if token creation fails at
connect()time, we do fallback to TCP (before the connection was closed)Signed-off-by Paolo Abeni <pabeni(a)redhat.com>--- net/mptcp/protocol.c | 16 +-- net/mptcp/protocol.h | 5 +- net/mptcp/subflow.c | 10 +- net/mptcp/token.c | 246 ++++++++++++++++++++++++++++++------------- 4 files changed, 184 insertions(+), 93 deletions(-)net/mptcp/protocol.c +1
-14@@ -1424,19 +1424,7 @@ struct sock *mptcp_sk_clone(const struct sock *sk, msk->token = subflow_req->token; msk->subflow =NULL ; -if (unlikely (mptcp_token_new_accept (subflow_req->token, nsk))) { - nsk->sk_state =TCP_CLOSE ; -bh_unlock_sock (nsk); - -/* we can't call into mptcp_close() here - possible BH context -* free the sock directly. -* sk_clone_lock() sets nsk refcnt to two, hence call sk_free() -* too. -*/ -sk_common_release (nsk); -sk_free (nsk); -return NULL ; - } +mptcp_token_accept (subflow_req, msk); msk->write_seq = subflow_req->idsn +1 ;atomic64_set (&msk->snd_una, msk->write_seq);@@ -1654,7 +1642,6 @@ void mptcp_finish_connect(struct sock *ssk)*/ WRITE_ONCE (msk->remote_key, subflow->remote_key);WRITE_ONCE (msk->local_key, subflow->local_key); -WRITE_ONCE (msk->token, subflow->token);WRITE_ONCE (msk->write_seq, subflow->idsn +1 );WRITE_ONCE (msk->ack_seq, ack_seq);WRITE_ONCE (msk->can_ack,1 );net/mptcp/protocol.c static struct proto mptcp_prot = { · line 1728
Quoted 4 lines
net/mptcp/protocol.c +1
-0@@ -1738,6 +1725,7 @@ static struct proto mptcp_prot = { .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem), .sysctl_mem = sysctl_tcp_mem, .obj_size = sizeof(struct mptcp_sock), + .slab_flags = SLAB_TYPESAFE_BY_RCU,I wonder if you now need to be careful when allocating and zero'ing the socket, same way it is happening in
sock_copy?Quoted 6 lines
In out-of-tree I had to take care of that by bringing back the
clear_skcallback in struct proto which will then be called insk_prot_alloc:Indeed! I felt like I was missing some relevant point!
re-adding the
clear_skcallback for mptcp's sake does not look 110% nice. There are a few alternatives, which sounds equally suboptimal too me:1) use plain RCU (with
kfree_rcu()and all the relevant memory overhead) for the whole msk sockets (~1600 bytes for IPv4). We already to that for the subflow contexts (~200 bytes). This is simple, but the memory overhead could be relevant ?!?1.a) additionally srink the
mptcp_sockstructure, e.g. I'm wild
guessing we can use 'struct sock' as the base type and add mptcp custom fields there ?!?2) use ULP for msk, too. Move the token there (and possibly all mptcp- specific data), and use plain RCU to handle the context. As a downside we will need 2 allocations per
accept()(the msk socket and the msk ulp context)
Quoted 7 lines
Quoted 2 lines
Any other better option?!?
I lean toward option 1 so we don't have to change around ULP to not expose MPTCP stuff (we rely on a kernel sock check to keep the subflow ULP from being exposed to userspace). I'd also like to keep ULP available for possible TLS support someday.
I just occurred to me a likely crazy 3rd alternative. What if we define a new list_nulls variant with 'nulls' values using the least significative bit zeroed? (the current implementation requires the opposite).
Something alike:
struct list_0_node *list_0_next(struct list_0_node *node)
{
return (struct list_0_node *)(list->next & ~1);
}bool list_0_is_null(struct list_0_node *node)
{
return !((unsigned long __force)node & 1);
}[...]
This way, memsetting a struct to 0 will preserve the NULL value and we should not need any additional care. Do I miss something relevant ?!?
Thanks,
Paolo
Christoph Paasch
[MPTCP] Re: [PATCH 2/4] mptcp: refactor token container.
Quoted 125 lines · Paolo Abeni
Quoted 103 lines · Mat Martineau
Quoted 101 lines · Paolo Abeni
Quoted 73 lines · Christoph Paasch
Hello,
Quoted 62 lines · Paolo Abeni
Replace the radix tree with an hash table allocated at boot time. The radix tree has some short coming:
a single lock is contented by all the mptcp operation, the lookup currently use such lock, and traversing all the items would require lock, too.With hash table instead we trade a little memory to address all the above - a per bucket lock is used.
Additionally refactor the token creation to code to:
- limit the number of consecutive attempts to a fixed maximum. Hitting an hash bucket with long chain is considered a failed attempt
-
accept()no longer can fail to to token management.- if token creation fails at
connect()time, we do fallback to TCP (before the connection was closed)Signed-off-by Paolo Abeni <pabeni(a)redhat.com>--- net/mptcp/protocol.c | 16 +-- net/mptcp/protocol.h | 5 +- net/mptcp/subflow.c | 10 +- net/mptcp/token.c | 246 ++++++++++++++++++++++++++++++------------- 4 files changed, 184 insertions(+), 93 deletions(-)net/mptcp/protocol.c +1
-14@@ -1424,19 +1424,7 @@ struct sock *mptcp_sk_clone(const struct sock *sk, msk->token = subflow_req->token; msk->subflow =NULL ; -if (unlikely (mptcp_token_new_accept (subflow_req->token, nsk))) { - nsk->sk_state =TCP_CLOSE ; -bh_unlock_sock (nsk); - -/* we can't call into mptcp_close() here - possible BH context -* free the sock directly. -* sk_clone_lock() sets nsk refcnt to two, hence call sk_free() -* too. -*/ -sk_common_release (nsk); -sk_free (nsk); -return NULL ; - } +mptcp_token_accept (subflow_req, msk); msk->write_seq = subflow_req->idsn +1 ;atomic64_set (&msk->snd_una, msk->write_seq);@@ -1654,7 +1642,6 @@ void mptcp_finish_connect(struct sock *ssk)*/ WRITE_ONCE (msk->remote_key, subflow->remote_key);WRITE_ONCE (msk->local_key, subflow->local_key); -WRITE_ONCE (msk->token, subflow->token);WRITE_ONCE (msk->write_seq, subflow->idsn +1 );WRITE_ONCE (msk->ack_seq, ack_seq);WRITE_ONCE (msk->can_ack,1 );net/mptcp/protocol.c static struct proto mptcp_prot = { · line 1728
Quoted 4 lines
net/mptcp/protocol.c +1
-0@@ -1738,6 +1725,7 @@ static struct proto mptcp_prot = { .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem), .sysctl_mem = sysctl_tcp_mem, .obj_size = sizeof(struct mptcp_sock), + .slab_flags = SLAB_TYPESAFE_BY_RCU,I wonder if you now need to be careful when allocating and zero'ing the socket, same way it is happening in
sock_copy?Quoted 6 lines
In out-of-tree I had to take care of that by bringing back the
clear_skcallback in struct proto which will then be called insk_prot_alloc:Indeed! I felt like I was missing some relevant point!
re-adding the
clear_skcallback for mptcp's sake does not look 110% nice. There are a few alternatives, which sounds equally suboptimal too me:1) use plain RCU (with
kfree_rcu()and all the relevant memory overhead) for the whole msk sockets (~1600 bytes for IPv4). We already to that for the subflow contexts (~200 bytes). This is simple, but the memory overhead could be relevant ?!?1.a) additionally srink the
mptcp_sockstructure, e.g. I'm wild
guessing we can use 'struct sock' as the base type and add mptcp custom fields there ?!?2) use ULP for msk, too. Move the token there (and possibly all mptcp- specific data), and use plain RCU to handle the context. As a downside we will need 2 allocations per
accept()(the msk socket and the msk ulp context)Quoted 7 lines
Quoted 2 lines
Any other better option?!?
I lean toward option 1 so we don't have to change around ULP to not expose MPTCP stuff (we rely on a kernel sock check to keep the subflow ULP from being exposed to userspace). I'd also like to keep ULP available for possible TLS support someday.
I just occurred to me a likely crazy 3rd alternative. What if we define a new
list_nullsvariant with 'nulls' values using the least significative bit zeroed? (the current implementation requires the opposite).Something alike:
struct
list_0_node*list_0_next(structlist_0_node*node){ return (struct list_0_node *)(list->next & ~1); }bool
list_0_is_null(struct list_0_node *node)
Quoted 8 lines
{ return !((unsigned long __force)node & 1); }[...]
This way, memsetting a struct to 0 will preserve the NULL value and we should not need any additional care. Do I miss something relevant ?!?
Meaning, when bytes get zero'd the nulls value becomes 0 and we always "goto again"?
That's a neat idea IMO. I think it could work.
But it means that a 0 nulls-value is invalid, right? (which is easy to take care of)
Christoph
Paolo Abeni
[MPTCP] Re: [PATCH 2/4] mptcp: refactor token container.
Quoted 130 lines · Christoph Paasch
Quoted 129 lines · Paolo Abeni
Quoted 103 lines · Mat Martineau
Quoted 101 lines · Paolo Abeni
Quoted 73 lines · Christoph Paasch
Hello,
Quoted 62 lines · Paolo Abeni
Replace the radix tree with an hash table allocated at boot time. The radix tree has some short coming:
a single lock is contented by all the mptcp operation, the lookup currently use such lock, and traversing all the items would require lock, too.With hash table instead we trade a little memory to address all the above - a per bucket lock is used.
Additionally refactor the token creation to code to:
- limit the number of consecutive attempts to a fixed maximum. Hitting an hash bucket with long chain is considered a failed attempt
-
accept()no longer can fail to to token management.- if token creation fails at
connect()time, we do fallback to TCP (before the connection was closed)Signed-off-by Paolo Abeni <pabeni(a)redhat.com>--- net/mptcp/protocol.c | 16 +-- net/mptcp/protocol.h | 5 +- net/mptcp/subflow.c | 10 +- net/mptcp/token.c | 246 ++++++++++++++++++++++++++++++------------- 4 files changed, 184 insertions(+), 93 deletions(-)net/mptcp/protocol.c +1
-14@@ -1424,19 +1424,7 @@ struct sock *mptcp_sk_clone(const struct sock *sk, msk->token = subflow_req->token; msk->subflow =NULL ; -if (unlikely (mptcp_token_new_accept (subflow_req->token, nsk))) { - nsk->sk_state =TCP_CLOSE ; -bh_unlock_sock (nsk); - -/* we can't call into mptcp_close() here - possible BH context -* free the sock directly. -* sk_clone_lock() sets nsk refcnt to two, hence call sk_free() -* too. -*/ -sk_common_release (nsk); -sk_free (nsk); -return NULL ; - } +mptcp_token_accept (subflow_req, msk); msk->write_seq = subflow_req->idsn +1 ;atomic64_set (&msk->snd_una, msk->write_seq);@@ -1654,7 +1642,6 @@ void mptcp_finish_connect(struct sock *ssk)*/ WRITE_ONCE (msk->remote_key, subflow->remote_key);WRITE_ONCE (msk->local_key, subflow->local_key); -WRITE_ONCE (msk->token, subflow->token);WRITE_ONCE (msk->write_seq, subflow->idsn +1 );WRITE_ONCE (msk->ack_seq, ack_seq);WRITE_ONCE (msk->can_ack,1 );net/mptcp/protocol.c static struct proto mptcp_prot = { · line 1728
Quoted 4 lines
net/mptcp/protocol.c +1
-0@@ -1738,6 +1725,7 @@ static struct proto mptcp_prot = { .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem), .sysctl_mem = sysctl_tcp_mem, .obj_size = sizeof(struct mptcp_sock), + .slab_flags = SLAB_TYPESAFE_BY_RCU,I wonder if you now need to be careful when allocating and zero'ing the socket, same way it is happening in
sock_copy?Quoted 6 lines
In out-of-tree I had to take care of that by bringing back the
clear_skcallback in struct proto which will then be called insk_prot_alloc:Indeed! I felt like I was missing some relevant point!
re-adding the
clear_skcallback for mptcp's sake does not look 110% nice. There are a few alternatives, which sounds equally suboptimal too me:1) use plain RCU (with
kfree_rcu()and all the relevant memory overhead) for the whole msk sockets (~1600 bytes for IPv4). We already to that for the subflow contexts (~200 bytes). This is simple, but the memory overhead could be relevant ?!?1.a) additionally srink the
mptcp_sockstructure, e.g. I'm wild
guessing we can use 'struct sock' as the base type and add mptcp custom fields there ?!?2) use ULP for msk, too. Move the token there (and possibly all mptcp- specific data), and use plain RCU to handle the context. As a downside we will need 2 allocations per
accept()(the msk socket and the msk ulp context)Quoted 7 lines
Quoted 2 lines
Any other better option?!?
I lean toward option 1 so we don't have to change around ULP to not expose MPTCP stuff (we rely on a kernel sock check to keep the subflow ULP from being exposed to userspace). I'd also like to keep ULP available for possible TLS support someday.
I just occurred to me a likely crazy 3rd alternative. What if we define a new
list_nullsvariant with 'nulls' values using the least significative bit zeroed? (the current implementation requires the opposite).Something alike:
struct
list_0_node*list_0_next(structlist_0_node*node){ return (struct list_0_node *)(list->next & ~1); }bool
list_0_is_null(struct list_0_node *node){ return !((unsigned long __force)node & 1); }
Quoted 8 lines
Quoted 4 lines
[...]
This way, memsetting a struct to 0 will preserve the NULL value and we should not need any additional care. Do I miss something relevant ?!?
Meaning, when bytes get zero'd the nulls value becomes 0 and we always "goto again"?
That's a neat idea IMO. I think it could work.
That was the idea ;)
Quoted 2 lines
But it means that a 0 nulls-value is invalid, right? (which is easy to take care of)
Why? (I like the idea that NULL is snull ;) I don't see why it can't !??)
Anyhow I opted for a different, hopefully less invasive solution - using the currently unused msk->sk_node to insert the msk into the token hash - please see v2 :)
Thanks,
Paolo
Christoph Paasch
[MPTCP] Re: [PATCH 2/4] mptcp: refactor token container.
Quoted 140 lines · Paolo Abeni
Quoted 130 lines · Christoph Paasch
Quoted 129 lines · Paolo Abeni
Quoted 103 lines · Mat Martineau
Quoted 101 lines · Paolo Abeni
Quoted 73 lines · Christoph Paasch
Hello,
Quoted 62 lines · Paolo Abeni
Replace the radix tree with an hash table allocated at boot time. The radix tree has some short coming:
a single lock is contented by all the mptcp operation, the lookup currently use such lock, and traversing all the items would require lock, too.With hash table instead we trade a little memory to address all the above - a per bucket lock is used.
Additionally refactor the token creation to code to:
- limit the number of consecutive attempts to a fixed maximum. Hitting an hash bucket with long chain is considered a failed attempt
-
accept()no longer can fail to to token management.- if token creation fails at
connect()time, we do fallback to TCP (before the connection was closed)Signed-off-by Paolo Abeni <pabeni(a)redhat.com>--- net/mptcp/protocol.c | 16 +-- net/mptcp/protocol.h | 5 +- net/mptcp/subflow.c | 10 +- net/mptcp/token.c | 246 ++++++++++++++++++++++++++++++------------- 4 files changed, 184 insertions(+), 93 deletions(-)net/mptcp/protocol.c +1
-14@@ -1424,19 +1424,7 @@ struct sock *mptcp_sk_clone(const struct sock *sk, msk->token = subflow_req->token; msk->subflow =NULL ; -if (unlikely (mptcp_token_new_accept (subflow_req->token, nsk))) { - nsk->sk_state =TCP_CLOSE ; -bh_unlock_sock (nsk); - -/* we can't call into mptcp_close() here - possible BH context -* free the sock directly. -* sk_clone_lock() sets nsk refcnt to two, hence call sk_free() -* too. -*/ -sk_common_release (nsk); -sk_free (nsk); -return NULL ; - } +mptcp_token_accept (subflow_req, msk); msk->write_seq = subflow_req->idsn +1 ;atomic64_set (&msk->snd_una, msk->write_seq);@@ -1654,7 +1642,6 @@ void mptcp_finish_connect(struct sock *ssk)*/ WRITE_ONCE (msk->remote_key, subflow->remote_key);WRITE_ONCE (msk->local_key, subflow->local_key); -WRITE_ONCE (msk->token, subflow->token);WRITE_ONCE (msk->write_seq, subflow->idsn +1 );WRITE_ONCE (msk->ack_seq, ack_seq);WRITE_ONCE (msk->can_ack,1 );net/mptcp/protocol.c static struct proto mptcp_prot = { · line 1728
Quoted 4 lines
net/mptcp/protocol.c +1
-0@@ -1738,6 +1725,7 @@ static struct proto mptcp_prot = { .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem), .sysctl_mem = sysctl_tcp_mem, .obj_size = sizeof(struct mptcp_sock), + .slab_flags = SLAB_TYPESAFE_BY_RCU,I wonder if you now need to be careful when allocating and zero'ing the socket, same way it is happening in
sock_copy?Quoted 6 lines
In out-of-tree I had to take care of that by bringing back the
clear_skcallback in struct proto which will then be called insk_prot_alloc:Indeed! I felt like I was missing some relevant point!
re-adding the
clear_skcallback for mptcp's sake does not look 110% nice. There are a few alternatives, which sounds equally suboptimal too me:1) use plain RCU (with
kfree_rcu()and all the relevant memory overhead) for the whole msk sockets (~1600 bytes for IPv4). We already to that for the subflow contexts (~200 bytes). This is simple, but the memory overhead could be relevant ?!?1.a) additionally srink the
mptcp_sockstructure, e.g. I'm wild
guessing we can use 'struct sock' as the base type and add mptcp custom fields there ?!?2) use ULP for msk, too. Move the token there (and possibly all mptcp- specific data), and use plain RCU to handle the context. As a downside we will need 2 allocations per
accept()(the msk socket and the msk ulp context)Quoted 7 lines
Quoted 2 lines
Any other better option?!?
I lean toward option 1 so we don't have to change around ULP to not expose MPTCP stuff (we rely on a kernel sock check to keep the subflow ULP from being exposed to userspace). I'd also like to keep ULP available for possible TLS support someday.
I just occurred to me a likely crazy 3rd alternative. What if we define a new
list_nullsvariant with 'nulls' values using the least significative bit zeroed? (the current implementation requires the opposite).Something alike:
struct
list_0_node*list_0_next(structlist_0_node*node){ return (struct list_0_node *)(list->next & ~1); }bool
list_0_is_null(struct list_0_node *node){ return !((unsigned long __force)node & 1); }Quoted 8 lines
Quoted 4 lines
[...]
This way, memsetting a struct to 0 will preserve the NULL value and we should not need any additional care. Do I miss something relevant ?!?
Meaning, when bytes get zero'd the nulls value becomes 0 and we always "goto again"?
That's a neat idea IMO. I think it could work.
Quoted 7 lines
That was the idea ;)
Quoted 2 lines
But it means that a 0 nulls-value is invalid, right? (which is easy to take care of)
Why? (I like the idea that NULL is snull ;) I don't see why it can't !??)
I was wondering how in the lookup you handle the end-case when you hit a NULL. Would you be able to know whether either the token is not present or the socket has been zero'ed?
Quoted 3 lines
Anyhow I opted for a different, hopefully less invasive solution - using the currently unused
msk->sk_nodeto insert the msk into the token hash - please see v2 :)
Nice, that solution is even cleaner!
Christoph
Mat Martineau
[MPTCP] Re: [PATCH 2/4] mptcp: refactor token container.
In reply to Paolo AbeniQuoted 150 lines · Paolo Abeni
Replace the radix tree with an hash table allocated at boot time. The radix tree has some short coming:
a single lock is contented by all the mptcp operation, the lookup currently use such lock, and traversing all the items would require lock, too.With hash table instead we trade a little memory to address all the above - a per bucket lock is used.
Additionally refactor the token creation to code to:
- limit the number of consecutive attempts to a fixed maximum. Hitting an hash bucket with long chain is considered a failed attempt
-
accept()no longer can fail to to token management.- if token creation fails at
connect()time, we do fallback to TCP (before the connection was closed)Signed-off-by Paolo Abeni <pabeni(a)redhat.com>---
net/mptcp/protocol.c | 16 +-- net/mptcp/protocol.h | 5 +- net/mptcp/subflow.c | 10 +- net/mptcp/token.c | 246 ++++++++++++++++++++++++++++++------------- 4 files changed, 184 insertions(+), 93 deletions(-)net/mptcp/protocol.c +2
-14@@ -1424,19 +1424,7 @@ struct sock *mptcp_sk_clone(const struct sock *sk, msk->token = subflow_req->token; msk->subflow =NULL ; -if (unlikely (mptcp_token_new_accept (subflow_req->token, nsk))) { - nsk->sk_state =TCP_CLOSE ; -bh_unlock_sock (nsk); - -/* we can't call into mptcp_close() here - possible BH context -* free the sock directly. -* sk_clone_lock() sets nsk refcnt to two, hence call sk_free() -* too. -*/ -sk_common_release (nsk); -sk_free (nsk); -return NULL ; - } +mptcp_token_accept (subflow_req, msk); msk->write_seq = subflow_req->idsn +1 ;atomic64_set (&msk->snd_una, msk->write_seq);@@ -1654,7 +1642,6 @@ void mptcp_finish_connect(struct sock *ssk)*/ WRITE_ONCE (msk->remote_key, subflow->remote_key);WRITE_ONCE (msk->local_key, subflow->local_key); -WRITE_ONCE (msk->token, subflow->token);WRITE_ONCE (msk->write_seq, subflow->idsn +1 );WRITE_ONCE (msk->ack_seq, ack_seq);WRITE_ONCE (msk->can_ack,1 );@@ -1738,6 +1725,7 @@ static struct proto mptcp_prot = { .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem), .sysctl_mem = sysctl_tcp_mem, .obj_size = sizeof(struct mptcp_sock), + .slab_flags = SLAB_TYPESAFE_BY_RCU, .no_autobind = true, };net/mptcp/protocol.h +2
-0@@ -211,6 +211,7 @@ struct mptcp_sock {struct socket *subflow;/* outgoing connect/listener/!mp_capable */ struct sock *first;struct mptcp_pm_data pm; +struct hlist_nulls_node token_node; };#define mptcp_for_each_subflow (__msk, __subflow) \@@ -255,6 +256,7 @@ struct mptcp_subflow_request_sock {u64 thmac;u32 local_nonce;u32 remote_nonce; +struct hlist_nulls_node token_node; };static inline struct
mptcp_subflow_request_sock*net/mptcp/protocol.h +2
-1@@ -386,7 +388,8 @@ void __init mptcp_token_init(void);int mptcp_token_new_request (struct request_sock *req);void mptcp_token_destroy_request (u32 token);int mptcp_token_new_connect (struct sock *sk); -int mptcp_token_new_accept (u32 token,struct sock *conn); +void mptcp_token_accept (struct mptcp_subflow_request_sock *r, +struct mptcp_sock *msk);struct mptcp_sock *mptcp_token_get_sock (u32 token);void mptcp_token_destroy (u32 token);net/mptcp/subflow.c +5
-2@@ -31,11 +31,14 @@ static void SUBFLOW_REQ_INC_STATS(struct request_sock *req,static int subflow_rebuild_header (struct sock *sk) {struct mptcp_subflow_context *subflow =mptcp_subflow_ctx (sk); -int local_id, err =0 ; +int local_id;if (subflow->request_mptcp && !subflow->token) {pr_debug ("subflow=%p" , sk); - err =mptcp_token_new_connect (sk); +if (mptcp_token_new_connect (sk)) { + subflow->mp_capable =0 ; + goto out; + } }else if (subflow->request_join && !subflow->local_nonce) {struct mptcp_sock *msk = (struct mptcp_sock *)subflow->conn;@@ -56,9 +59,6 @@ static int subflow_rebuild_header(struct sock *sk) }out:
net/mptcp/subflow.c +0
-3-if (err) -return err; -return subflow->icsk_af_ops->rebuild_header(sk); }net/mptcp/token.c +1
-4@@ -24,7 +24,7 @@#include <linux/kernel.h> #include <linux/module.h> -#include <linux/radix-tree.h> +#include <linux/memblock.h> #include <linux/ip.h> #include <linux/tcp.h> #include <net/sock.h> @@ -33,10 +33,53 @@#include <net/mptcp.h> #include "protocol.h" -static RADIX_TREE(token_tree, GFP_ATOMIC); -static RADIX_TREE(token_req_tree, GFP_ATOMIC); -static DEFINE_SPINLOCK (token_tree_lock);
net/mptcp/token.c line 44
Quoted 10 lines
net/mptcp/token.c +9
-1-static int token_used__read_mostly ; +#define TOKEN_MAX_RETRIES 4 +#define TOKEN_MAX_CHAIN_LEN 4 + +struct token_bucket { +spinlock_t lock; +int chain_len; +struct hlist_nulls_head req_chain; +struct hlist_nulls_head msk_chain; +};
I remember a discussion a while ago about how we didn't really need two radix trees for req & msk, since there's only one "token space". Similarly, seems like we only need one linked list per bucket if we can define a common structure to use in both sock types, like:
struct mptcp_token_node {
struct hlist_nulls_node node;
u32 value;
bool is_req;
};
It would use a little more space per mptcp_sock/req_sock and would require extra code in the lookup to return the appropriate container pointer (req or mptcp_sock), but would shrink the hash table or give us more buckets in the same memory, which seems like a win.
Quoted 24 lines
net/mptcp/token.c +24
-0+ +static struct token_bucket *token_hash__read_mostly ; +static unsigned int token_mask__read_mostly ; + +static struct token_bucket *token_bucket (u32 token) +{ +return &token_hash[token & token_mask]; +} + +static struct mptcp_subflow_request_sock * +__token_lookup_req (struct token_bucket *t,u32 token) +{ +struct mptcp_subflow_request_sock *req; +struct hlist_nulls_node *pos; + +hlist_nulls_for_each_entry_rcu (req, pos, &t->req_chain, token_node) +if (req->token == token) +return req; +return NULL ; +} + +static struct mptcp_sock * +__token_lookup_msk (struct token_bucket *t,u32 token) +{
net/mptcp/token.c line 78
Quoted 10 lines
net/mptcp/token.c +10
-0+struct hlist_nulls_node *pos; +struct mptcp_sock *msk; + +hlist_nulls_for_each_entry_rcu (msk, pos, &t->msk_chain, token_node) +if (msk->token == token) +return msk; +return NULL ; +} + +bool __token_bucket_busy (struct token_bucket *t,u32 token)
static?
Quoted 165 lines
net/mptcp/token.c +68
-60+{ +return !token || t->chain_len >=TOKEN_MAX_CHAIN_LEN || +__token_lookup_req (t, token) ||__token_lookup_msk (t, token); +}/** * mptcp_token_new_request - create new key/idsn/token for subflow_request @@ -52,30 +95,32 @@ static int token_used __read_mostly;int mptcp_token_new_request (struct request_sock *req) {struct mptcp_subflow_request_sock *subflow_req =mptcp_subflow_rsk (req); -int err; - -while (1 ) { -u32 token; - -mptcp_crypto_key_gen_sha (&subflow_req->local_key, - &subflow_req->token, - &subflow_req->idsn); -pr_debug ("req=%p local_key=%llu, token=%u, idsn=%llu\n" , - req, subflow_req->local_key, subflow_req->token, - subflow_req->idsn); - - token = subflow_req->token; -spin_lock_bh (&token_tree_lock); -if (!radix_tree_lookup (&token_req_tree, token) && - !radix_tree_lookup (&token_tree, token)) -break ; -spin_unlock_bh (&token_tree_lock); +int retries =TOKEN_MAX_RETRIES ; +struct token_bucket *bucket; +u32 token; + +again: +mptcp_crypto_key_gen_sha (&subflow_req->local_key, + &subflow_req->token, + &subflow_req->idsn); +pr_debug ("req=%p local_key=%llu, token=%u, idsn=%llu\n" , + req, subflow_req->local_key, subflow_req->token, + subflow_req->idsn); + + token = subflow_req->token; + bucket =token_bucket (token); +spin_lock_bh (&bucket->lock); +if (__token_bucket_busy (bucket, token)) { +spin_unlock_bh (&bucket->lock); +if (!--retries) +return -EBUSY ; + goto again; } - err =radix_tree_insert (&token_req_tree, - subflow_req->token, &token_used); -spin_unlock_bh (&token_tree_lock); -return err; +hlist_nulls_add_tail_rcu (&subflow_req->token_node, &bucket->req_chain); + bucket->chain_len++; +spin_unlock_bh (&bucket->lock); +return 0 ; }/** @@ -97,48 +142,54 @@ int mptcp_token_new_request(struct request_sock *req)int mptcp_token_new_connect (struct sock *sk) {struct mptcp_subflow_context *subflow =mptcp_subflow_ctx (sk); -struct sock *mptcp_sock = subflow->conn; -int err; +struct mptcp_sock *msk =mptcp_sk (subflow->conn); +int retries =TOKEN_MAX_RETRIES ; +struct token_bucket *bucket; -while (1 ) { -u32 token; +pr_debug ("ssk=%p, local_key=%llu, token=%u, idsn=%llu\n" , + sk, subflow->local_key, subflow->token, subflow->idsn); -mptcp_crypto_key_gen_sha (&subflow->local_key, &subflow->token, - &subflow->idsn); +again: +mptcp_crypto_key_gen_sha (&subflow->local_key, &subflow->token, + &subflow->idsn); -pr_debug ("ssk=%p, local_key=%llu, token=%u, idsn=%llu\n" , - sk, subflow->local_key, subflow->token, subflow->idsn); - - token = subflow->token; -spin_lock_bh (&token_tree_lock); -if (!radix_tree_lookup (&token_req_tree, token) && - !radix_tree_lookup (&token_tree, token)) -break ; -spin_unlock_bh (&token_tree_lock); + bucket =token_bucket (subflow->token); +spin_lock_bh (&bucket->lock); +if (__token_bucket_busy (bucket, subflow->token)) { +spin_unlock_bh (&bucket->lock); +if (!--retries) +return -EBUSY ; + goto again; } - err =radix_tree_insert (&token_tree, subflow->token, mptcp_sock); -spin_unlock_bh (&token_tree_lock); -return err; +WRITE_ONCE (msk->token, subflow->token); +hlist_nulls_add_tail_rcu (&msk->token_node, &bucket->msk_chain); + bucket->chain_len++; +spin_unlock_bh (&bucket->lock); +return 0 ; }/** -* mptcp_token_new_accept - insert token for later processing -* @token: the token to insert to the tree -* @conn: the just cloned socket linked to the new connection +* mptcp_token_accept - replace a req sk with full sock in token hash +* @req: the request socket to be removed +* @msk: the just cloned socket linked to the new connection * * Called when a SYN packet creates a new logical connection, i.e. * is not a join request. */ -int mptcp_token_new_accept (u32 token,struct sock *conn) +void mptcp_token_accept (struct mptcp_subflow_request_sock *req, +struct mptcp_sock *msk) { -int err; - -spin_lock_bh (&token_tree_lock); - err =radix_tree_insert (&token_tree, token, conn); -spin_unlock_bh (&token_tree_lock); +struct mptcp_subflow_request_sock *pos; +struct token_bucket *bucket; -return err; + bucket =token_bucket (req->token); +spin_lock_bh (&bucket->lock); + pos =__token_lookup_req (bucket, req->token); +if (!WARN_ON_ONCE (pos != req)) +hlist_nulls_del_rcu (&req->token_node); +hlist_nulls_add_tail_rcu (&msk->token_node, &bucket->msk_chain); +spin_unlock_bh (&bucket->lock); }/** @@ -152,20 +203,33 @@ int mptcp_token_new_accept(u32 token, struct sock *conn)*/ struct mptcp_sock *mptcp_token_get_sock (u32 token) { -struct sock *conn; - -spin_lock_bh (&token_tree_lock); - conn =radix_tree_lookup (&token_tree, token); -if (conn) { -/* token still reserved? */ -if (conn == (struct sock *)&token_used) - conn =NULL ; -else -sock_hold (conn); +struct hlist_nulls_node *pos; +struct token_bucket *bucket; +struct mptcp_sock *msk; + +rcu_read_lock (); + bucket =token_bucket (token);
net/mptcp/token.c int mptcp_token_new_accept(u32 token, struct sock *conn) · line 221
Quoted 10 lines
net/mptcp/token.c +10
-0+ +again: +hlist_nulls_for_each_entry_rcu (msk, pos, &bucket->msk_chain, + token_node) { +if (READ_ONCE (msk->token) != token) +continue ; +if (!refcount_inc_not_zero (&((struct sock *)msk)->sk_refcnt)) + goto not_found; +if (READ_ONCE (msk->token) != token) + goto again;
Checking msk->token twice here, but it doesn't look like it ever gets changed after it's set?
Thanks,
Mat
Quoted 78 lines
net/mptcp/token.c +51
-8+ goto found; } -spin_unlock_bh (&token_tree_lock); +if (get_nulls_value (pos) != (token & token_mask)) + goto again; -return mptcp_sk (conn); +not_found: + msk =NULL ; + +found: +rcu_read_unlock (); +return msk; }/** @@ -177,9 +241,17 @@ struct mptcp_sock *mptcp_token_get_sock(u32 token)*/ void mptcp_token_destroy_request (u32 token) { -spin_lock_bh (&token_tree_lock); -radix_tree_delete (&token_req_tree, token); -spin_unlock_bh (&token_tree_lock); +struct mptcp_subflow_request_sock *pos; +struct token_bucket *bucket; + + bucket =token_bucket (token); +spin_lock_bh (&bucket->lock); + pos =__token_lookup_req (bucket, token); +if (pos) { +hlist_nulls_del_rcu (&pos->token_node); + bucket->chain_len--; + } +spin_unlock_bh (&bucket->lock); }/** @@ -190,7 +262,35 @@ void mptcp_token_destroy_request(u32 token)*/ void mptcp_token_destroy (u32 token) { -spin_lock_bh (&token_tree_lock); -radix_tree_delete (&token_tree, token); -spin_unlock_bh (&token_tree_lock); +struct token_bucket *bucket; +struct mptcp_sock *pos; + + bucket =token_bucket (token); +spin_lock_bh (&bucket->lock); + pos =__token_lookup_msk (bucket, token); +if (pos) { +hlist_nulls_del_rcu (&pos->token_node); + bucket->chain_len--; + } +spin_unlock_bh (&bucket->lock); +} + +void __init mptcp_token_init (void ) +{ +int i; + + token_hash =alloc_large_system_hash ("MPTCP token" , +sizeof (struct token_bucket ), +0 , +20 ,/* one slot per 1MB of memory */ +0 , +NULL , + &token_mask, +0 , +64 *1024 ); +for (i =0 ; i < token_mask +1 ; ++i) { +INIT_HLIST_NULLS_HEAD (&token_hash[i].req_chain, i); +INIT_HLIST_NULLS_HEAD (&token_hash[i].msk_chain, i); +spin_lock_init (&token_hash[i].lock); + } }Signature · 1 line
2.21.3
--
Mat Martineau
Intel
Paolo Abeni
[MPTCP] Re: [PATCH 2/4] mptcp: refactor token container.
Hi,
Quoted 18 lines · Mat Martineau
net/mptcp/token.c line 44
Quoted 6 lines
net/mptcp/token.c +6
-0+struct token_bucket { +spinlock_t lock; +int chain_len; +struct hlist_nulls_head req_chain; +struct hlist_nulls_head msk_chain; +};I remember a discussion a while ago about how we didn't really need two radix trees for req & msk, since there's only one "token space". Similarly, seems like we only need one linked list per bucket if we can define a common structure to use in both sock types, like:
struct
mptcp_token_node{(quoted diff) +0
-0struct hlist_nulls_node node; u32 value; bool is_req;};
Quoted 4 lines
It would use a little more space per mptcp_sock/req_sock and would require extra code in the lookup to return the appropriate container pointer (req or
mptcp_sock), but would shrink the hash table or give us more buckets in the same memory, which seems like a win.
LGTM, I'll try that.
[...]
Quoted 23 lines
Quoted 23 lines
struct
mptcp_sock*mptcp_token_get_sock(u32 token) {net/mptcp/token.c +11
-10-struct sock *conn; - -spin_lock_bh (&token_tree_lock); - conn =radix_tree_lookup (&token_tree, token); -if (conn) { -/* token still reserved? */ -if (conn == (struct sock *)&token_used) - conn =NULL ; -else -sock_hold (conn); +struct hlist_nulls_node *pos; +struct token_bucket *bucket; +struct mptcp_sock *msk; + +rcu_read_lock (); + bucket =token_bucket (token); + +again: +hlist_nulls_for_each_entry_rcu (msk, pos, &bucket->msk_chain, + token_node) { +if (READ_ONCE (msk->token) != token)
Quoted 8 lines
net/mptcp/token.c int mptcp_token_new_accept(u32 token, struct sock *conn) · line 221
Quoted 5 lines
net/mptcp/token.c +5
-0+continue ; +if (!refcount_inc_not_zero (&((struct sock *)msk)->sk_refcnt)) + goto not_found; +if (READ_ONCE (msk->token) != token) + goto again;Checking
msk->tokentwice here, but it doesn't look like it ever gets changed after it's set?
AFAICT, this is the common list_nulls usage pattern. The problem is that a msk can be freed and re-allocated concurrently by another CPU. Since this lookup is lockless, the token value can possibly change 'under-the-wood' till we get a reference to the token - and forbit msk de-allocation.
Cheers,
Paolo
Paolo Abeni
[MPTCP] Re: [PATCH 2/4] mptcp: refactor token container.
In reply to Mat MartineauQuoted 26 lines · Mat Martineau
net/mptcp/token.c line 44
Quoted 13 lines · Paolo Abeni
net/mptcp/token.c +9
-4-static RADIX_TREE(token_tree, GFP_ATOMIC); -static RADIX_TREE(token_req_tree, GFP_ATOMIC); -static DEFINE_SPINLOCK (token_tree_lock); -static int token_used__read_mostly ; +#define TOKEN_MAX_RETRIES 4 +#define TOKEN_MAX_CHAIN_LEN 4 + +struct token_bucket { +spinlock_t lock; +int chain_len; +struct hlist_nulls_head req_chain; +struct hlist_nulls_head msk_chain; +};I remember a discussion a while ago about how we didn't really need two radix trees for req & msk, since there's only one "token space". Similarly, seems like we only need one linked list per bucket if we can define a common structure to use in both sock types, like:
struct
mptcp_token_node{(quoted diff) +0
-0struct hlist_nulls_node node; u32 value; bool is_req;};
Quoted 4 lines
It would use a little more space per mptcp_sock/req_sock and would require extra code in the lookup to return the appropriate container pointer (req or
mptcp_sock), but would shrink the hash table or give us more buckets in the same memory, which seems like a win.
In v2 I had to drop the above idea because it would not fit with the current solution for the list corruption issue.
Anyhow I have an addional doubt, how can the above fit the list traversal helper ? Don't we need to allocate 'mptcp_token_node' separatelly and keep a back ptr to the main socket/req???
Otherwise 'mptcp_token_node' should have the same offset inside 'struct mptcp_sock' and inside 'struct mptcp_subflow_request_sock' to fit hlist_nulls_entry(), right?
Thanks,
Paolo
Mat Martineau
[MPTCP] Re: [PATCH 2/4] mptcp: refactor token container.
Quoted 27 lines · Paolo Abeni
Quoted 26 lines · Mat Martineau
net/mptcp/token.c line 44
Quoted 13 lines · Paolo Abeni
net/mptcp/token.c +9
-4-static RADIX_TREE(token_tree, GFP_ATOMIC); -static RADIX_TREE(token_req_tree, GFP_ATOMIC); -static DEFINE_SPINLOCK (token_tree_lock); -static int token_used__read_mostly ; +#define TOKEN_MAX_RETRIES 4 +#define TOKEN_MAX_CHAIN_LEN 4 + +struct token_bucket { +spinlock_t lock; +int chain_len; +struct hlist_nulls_head req_chain; +struct hlist_nulls_head msk_chain; +};I remember a discussion a while ago about how we didn't really need two radix trees for req & msk, since there's only one "token space". Similarly, seems like we only need one linked list per bucket if we can define a common structure to use in both sock types, like:
struct
mptcp_token_node{(quoted diff) +0
-0struct hlist_nulls_node node; u32 value; bool is_req;};
Quoted 8 lines
Quoted 4 lines
It would use a little more space per mptcp_sock/req_sock and would require extra code in the lookup to return the appropriate container pointer (req or
mptcp_sock), but would shrink the hash table or give us more buckets in the same memory, which seems like a win.In v2 I had to drop the above idea because it would not fit with the current solution for the list corruption issue.
Yes, the list corruption issue is more important than the above optimization!
Quoted 8 lines
Anyhow I have an addional doubt, how can the above fit the list traversal helper ? Don't we need to allocate '
mptcp_token_node' separatelly and keep a back ptr to the main socket/req???Otherwise '
mptcp_token_node' should have the same offset inside 'structmptcp_sock' and inside 'structmptcp_subflow_request_sock' to fithlist_nulls_entry(), right?
The idea was to wrap up the information the list traversal helper needed in mptcp_token_node. Then use is_req to determine which structure type (mptcp_sock or mptcp_subflow_request_sock) to use with container_of().
--
Mat Martineau
Intel
Paolo Abeni
PATCH 3/4 mptcp: move crypto test to KUNIT
In reply to Paolo Abenicurrently MPTCP uses a custom hook to executed unit test at boot time. Let's use the KUNIT framework instead. Additionally move the relevant code to a separate file and export the function needed by the test when self-tests are build as a module.
--- net/mptcp/Kconfig | 19 +++++++---- net/mptcp/Makefile | 3 ++ net/mptcp/crypto.c | 65 ++----------------------------------- net/mptcp/crypto_test.c | 71 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 90 insertions(+), 68 deletions(-) create mode 100644 net/mptcp/crypto_test.c
net/mptcp/Kconfig +13 -6
@@ -19,12 +19,19 @@ config MPTCP_IPV6 select IPV6 default y -config MPTCP_HMAC_TEST - bool "Tests for MPTCP HMAC implementation" +endif + +config MPTCP_KUNIT_TESTS + tristate "This builds the MPTCP KUnit tests" + select MPTCP + depends on KUNIT help - This option enable boot time self-test for the HMAC implementation - used by the MPTCP code + Currently covers the MPTCP crypto helpers. + Only useful for kernel devs running KUnit test harness and are not + for inclusion into a production build. - Say N if you are unsure. + For more information on KUnit and unit tests in general please refer + to the KUnit documentation in Documentation/dev-tools/kunit/. + + If unsure, say N. -endif
net/mptcp/Makefile +3 -0
@@ -3,3 +3,6 @@ obj-$(CONFIG_MPTCP) += mptcp.o mptcp-y := protocol.o subflow.o options.o token.o crypto.o ctrl.o pm.o diag.o \ mib.o pm_netlink.o + +mptcp_crypto_test-objs := crypto_test.o +obj-$(CONFIG_MPTCP_KUNIT_TESTS) += mptcp_crypto_test.o
net/mptcp/crypto.c +3 -62
69 lines not on this page · raw message
net/mptcp/crypto_test.c new file +71 -0
71 lines not on this page · raw message
Signature · 1 line
2.21.3
Davide Caratti
[MPTCP] Re: [PATCH 3/4] mptcp: move crypto test to KUNIT
Quoted 15 lines · Paolo Abeni
currently MPTCP uses a custom hook to executed unit test at boot time. Let's use the KUNIT framework instead. Additionally move the relevant code to a separate file and export the function needed by the test when self-tests are build as a module.
Signed-off-by Paolo Abeni <pabeni(a)redhat.com>--- net/mptcp/Kconfig | 19 +++++++---- net/mptcp/Makefile | 3 ++ net/mptcp/crypto.c | 65 ++----------------------------------- net/mptcp/crypto_test.c | 71 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 90 insertions(+), 68 deletions(-) create mode 100644 net/mptcp/crypto_test.c
[...]
net/mptcp/crypto_test.c line 1
Quoted 7 lines
net/mptcp/crypto_test.c new file +1
-0@@ -0,0 +1,71 @@ +#include <kunit/test.h>
Nit: I think we need a SPDX line here (or non-conventional users of checkpatch might see a warning)
Quoted 70 lines
net/mptcp/crypto_test.c new file +70
-0+ +#include "protocol.h" + +struct test_case { +char *key; +char *msg; +char *result; +}; + +/* we can't reuse RFC 4231 test vectors, as we have constraint on the +* input and key size, and we truncate the output. +*/ +static struct test_case tests[] = { + { + .key ="0b0b0b0b0b0b0b0b" , + .msg ="48692054" , + .result ="8385e24fb4235ac37556b6b886db106284a1da67" , + }, + { + .key ="aaaaaaaaaaaaaaaa" , + .msg ="dddddddd" , + .result ="2c5e219164ff1dca1c4a92318d847bb6b9d44492" , + }, + { + .key ="0102030405060708" , + .msg ="cdcdcdcd" , + .result ="e73b9ba9969969cefb04aa0d6df18ec2fcc075b6" , + }, +}; + +static void mptcp_crypto_test_basic (struct kunit *test) +{ +char hmac[20 ], hmac_hex[41 ]; +u32 nonce1, nonce2; +u64 key1, key2; +u8 msg[8 ]; +int i, j; + +for (i =0 ; i <ARRAY_SIZE (tests); ++i) { +/* mptcp hmap will convert to be before computing the hmac */ + key1 =be64_to_cpu (*((__be64 *)&tests[i].key[0 ])); + key2 =be64_to_cpu (*((__be64 *)&tests[i].key[8 ])); + nonce1 =be32_to_cpu (*((__be32 *)&tests[i].msg[0 ])); + nonce2 =be32_to_cpu (*((__be32 *)&tests[i].msg[4 ])); + +put_unaligned_be32 (nonce1, &msg[0 ]); +put_unaligned_be32 (nonce2, &msg[4 ]); + +mptcp_crypto_hmac_sha (key1, key2, msg,8 , hmac); +for (j =0 ; j <20 ; ++j) +sprintf (&hmac_hex[j <<1 ],"%02x" , hmac[j] &0xff ); + hmac_hex[40 ] =0 ; + +KUNIT_EXPECT_STREQ (test, &hmac_hex[0 ], tests[i].result); + } +} + +static struct kunit_case mptcp_crypto_test_cases[] = { +KUNIT_CASE (mptcp_crypto_test_basic), + {} +}; + +static struct kunit_suite mptcp_crypto_suite = { + .name ="mptcp-crypto" , + .test_cases = mptcp_crypto_test_cases, +}; + +kunit_test_suite (mptcp_crypto_suite); + +MODULE_LICENSE ("GPL" );
Paolo Abeni
[MPTCP] Re: [PATCH 3/4] mptcp: move crypto test to KUNIT
Quoted 19 lines · Davide Caratti
Quoted 15 lines · Paolo Abeni
currently MPTCP uses a custom hook to executed unit test at boot time. Let's use the KUNIT framework instead. Additionally move the relevant code to a separate file and export the function needed by the test when self-tests are build as a module.
Signed-off-by Paolo Abeni <pabeni(a)redhat.com>--- net/mptcp/Kconfig | 19 +++++++---- net/mptcp/Makefile | 3 ++ net/mptcp/crypto.c | 65 ++----------------------------------- net/mptcp/crypto_test.c | 71 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 90 insertions(+), 68 deletions(-) create mode 100644 net/mptcp/crypto_test.c[...]
Quoted 10 lines
net/mptcp/crypto_test.c line 1
Quoted 7 lines
net/mptcp/crypto_test.c new file +1
-0@@ -0,0 +1,71 @@ +#include <kunit/test.h> Nit: I think we need a SPDX line here (or non-conventional users of checkpatch might see a warning)
Right you are! I'll fix that in the next iteration!
Thanks,
Paolo
Paolo Abeni
PATCH 4/4 mptcp: introduce token KUNIT self-tests
In reply to Paolo AbeniUnit tests for the internal MPTCP token APIs, using KUNIT
--- net/mptcp/Makefile | 3 +- net/mptcp/token.c | 9 +++ net/mptcp/token_test.c | 138 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 net/mptcp/token_test.c
net/mptcp/Makefile +2 -1
@@ -5,4 +5,5 @@ mptcp-y := protocol.o subflow.o options.o token.o crypto.o ctrl.o pm.o diag.o \ mib.o pm_netlink.o mptcp_crypto_test-objs := crypto_test.o -obj-$(CONFIG_MPTCP_KUNIT_TESTS) += mptcp_crypto_test.o +mptcp_token_test-objs := token_test.o +obj-$(CONFIG_MPTCP_KUNIT_TESTS) += mptcp_crypto_test.o mptcp_token_test.o
net/mptcp/token.c +9 -0
@@ -294,3 +294,12 @@ void __init mptcp_token_init(void)spin_lock_init (&token_hash[i].lock); } } + +#if IS_MODULE (CONFIG_MPTCP_KUNIT_TESTS ) +EXPORT_SYMBOL_GPL (mptcp_token_new_request); +EXPORT_SYMBOL_GPL (mptcp_token_new_connect); +EXPORT_SYMBOL_GPL (mptcp_token_accept); +EXPORT_SYMBOL_GPL (mptcp_token_get_sock); +EXPORT_SYMBOL_GPL (mptcp_token_destroy_request); +EXPORT_SYMBOL_GPL (mptcp_token_destroy); +#endif
net/mptcp/token_test.c new file +138 -0
139 lines not on this page · raw message
Signature · 1 line
2.21.3
Davide Caratti
[MPTCP] Re: [PATCH 4/4] mptcp: introduce token KUNIT self-tests
Quoted 9 lines · Paolo Abeni
Unit tests for the internal MPTCP token APIs, using KUNIT
Signed-off-by Paolo Abeni <pabeni(a)redhat.com>--- net/mptcp/Makefile | 3 +- net/mptcp/token.c | 9 +++ net/mptcp/token_test.c | 138 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 net/mptcp/token_test.c
[...]
net/mptcp/token_test.c line 1
Quoted 7 lines
net/mptcp/token_test.c new file +1
-0@@ -0,0 +1,138 @@ +#include <kunit/test.h>
same as patch 3/4: I think we need a SPDX line here (or non-conventional users of checkpatch might see a warning)
Quoted 138 lines
net/mptcp/token_test.c new file +137
-0+ +#include "protocol.h" + +static struct mptcp_subflow_request_sock *build_req_sock (struct kunit *test) +{ +struct mptcp_subflow_request_sock *req; + + req =kunit_kzalloc (test,sizeof (struct mptcp_subflow_request_sock ), +GFP_USER ); +KUNIT_EXPECT_NOT_ERR_OR_NULL (test, req); +return req; +} + +static void mptcp_token_test_req_basic (struct kunit *test) +{ +struct mptcp_subflow_request_sock *req =build_req_sock (test); +struct mptcp_sock *null_msk =NULL ; + +KUNIT_ASSERT_EQ (test,0 , +mptcp_token_new_request ((struct request_sock *)req)); +KUNIT_EXPECT_NE (test,0 , (int )req->token); +KUNIT_EXPECT_PTR_EQ (test, null_msk,mptcp_token_get_sock (req->token)); + +/* cleanup */ +mptcp_token_destroy_request (req->token); +} + +static struct inet_connection_sock *build_icsk (struct kunit *test) +{ +struct inet_connection_sock *icsk; + + icsk =kunit_kzalloc (test,sizeof (struct inet_connection_sock ), +GFP_USER ); +KUNIT_EXPECT_NOT_ERR_OR_NULL (test, icsk); +return icsk; +} + +static struct mptcp_subflow_context *build_ctx (struct kunit *test) +{ +struct mptcp_subflow_context *ctx; + + ctx =kunit_kzalloc (test,sizeof (struct mptcp_subflow_context ), +GFP_USER ); +KUNIT_EXPECT_NOT_ERR_OR_NULL (test, ctx); +return ctx; +} + +static struct mptcp_sock *build_msk (struct kunit *test) +{ +struct mptcp_sock *msk; + + msk =kunit_kzalloc (test,sizeof (struct mptcp_sock ),GFP_USER ); +KUNIT_EXPECT_NOT_ERR_OR_NULL (test, msk); +refcount_set (&((struct sock *)msk)->sk_refcnt,1 ); +return msk; +} + +static void mptcp_token_test_msk_basic (struct kunit *test) +{ +struct inet_connection_sock *icsk =build_icsk (test); +struct mptcp_subflow_context *ctx =build_ctx (test); +struct mptcp_sock *msk =build_msk (test); +struct mptcp_sock *null_msk =NULL ; +struct sock *sk; + + icsk->icsk_ulp_data = ctx; + ctx->conn = (struct sock *)msk; + sk = (struct sock *)msk; + +KUNIT_ASSERT_EQ (test,0 , +mptcp_token_new_connect ((struct sock *)icsk)); +KUNIT_EXPECT_NE (test,0 , (int )ctx->token); +KUNIT_EXPECT_EQ (test, ctx->token, msk->token); +KUNIT_EXPECT_PTR_EQ (test, msk,mptcp_token_get_sock (ctx->token)); +KUNIT_EXPECT_EQ (test,2 , (int )refcount_read (&sk->sk_refcnt)); + +mptcp_token_destroy (ctx->token); +KUNIT_EXPECT_PTR_EQ (test, null_msk,mptcp_token_get_sock (ctx->token)); +} + +static void mptcp_token_test_accept (struct kunit *test) +{ +struct mptcp_subflow_request_sock *req =build_req_sock (test); +struct mptcp_sock *msk =build_msk (test); + +KUNIT_ASSERT_EQ (test,0 , +mptcp_token_new_request ((struct request_sock *)req)); + msk->token = req->token; +mptcp_token_accept (req, msk); +KUNIT_EXPECT_PTR_EQ (test, msk,mptcp_token_get_sock (msk->token)); + +/* this is now a no-op */ +mptcp_token_destroy_request (req->token); +KUNIT_EXPECT_PTR_EQ (test, msk,mptcp_token_get_sock (msk->token)); + +/* cleanup */ +mptcp_token_destroy (msk->token); +} + +static void mptcp_token_test_destroyed (struct kunit *test) +{ +struct mptcp_subflow_request_sock *req =build_req_sock (test); +struct mptcp_sock *msk =build_msk (test); +struct mptcp_sock *null_msk =NULL ; +struct sock *sk; + + sk = (struct sock *)msk; + +KUNIT_ASSERT_EQ (test,0 , +mptcp_token_new_request ((struct request_sock *)req)); + msk->token = req->token; +mptcp_token_accept (req, msk); + +/* simulate race on removal */ +refcount_set (&sk->sk_refcnt,0 ); +KUNIT_EXPECT_PTR_EQ (test, null_msk,mptcp_token_get_sock (msk->token)); + +/* cleanup */ +mptcp_token_destroy (msk->token); +} + +static struct kunit_case mptcp_token_test_cases[] = { +KUNIT_CASE (mptcp_token_test_req_basic), +KUNIT_CASE (mptcp_token_test_msk_basic), +KUNIT_CASE (mptcp_token_test_accept), +KUNIT_CASE (mptcp_token_test_destroyed), + {} +}; + +static struct kunit_suite mptcp_token_suite = { + .name ="mptcp-token" , + .test_cases = mptcp_token_test_cases, +}; + +kunit_test_suite (mptcp_token_suite); + +MODULE_LICENSE ("GPL" );