Add a bhash2 table hashed by port + address
Subsystems (5)
v3 cover Add a bhash2 table hashed by port + address
This patchset proposes adding a bhash2 table that hashes by port and address. The motivation behind bhash2 is to expedite bind requests in situations where the port has many sockets in its bhash table entry, which makes checking bind conflicts costly especially given that we acquire the table entry spinlock while doing so, which can cause softirq cpu lockups and can prevent new tcp connections.
We ran into this problem at Meta where the traffic team binds a large number of IPs to port 443 and the bind() call took a significant amount of time which led to cpu softirq lockups, which caused packet drops and other failures on the machine
The patches are as follows:
1/2 - Adds a second bhash table (bhash2) hashed by port and address 2/2 - Adds a test for timing how long an additional bind request takes when the bhash entry is populated
When experimentally testing this on a local server for ~24k sockets bound to the port, the results seen were:
ipv4:
before - 0.002317 seconds
with bhash2 - 0.000018 seconds
ipv6:
before - 0.002431 seconds
with bhash2 - 0.000021 seconds
v2 -> v3:
v2:
https://lore.kernel.org/netdev/[email protected]/
* Fix bhash2 allocation error handling for dccp
* Rebase onto net-next/master
v1 -> v2:
v1:
https://lore.kernel.org/netdev/[email protected]/
* Attached test for timing bind request
Joanne Koong (2):
net: Add a second bind table hashed by port and address
selftests: Add test for timing a bind request to a port with a
populated bhash entry
include/net/inet_connection_sock.h | 3 + include/net/inet_hashtables.h | 56 ++++- include/net/sock.h | 14 ++ net/dccp/proto.c | 34 ++- net/ipv4/inet_connection_sock.c | 227 +++++++++++++----- net/ipv4/inet_hashtables.c | 188 ++++++++++++++- net/ipv4/tcp.c | 14 +- tools/testing/selftests/net/.gitignore | 1 + tools/testing/selftests/net/Makefile | 2 + tools/testing/selftests/net/bind_bhash_test.c | 119 +++++++++ 10 files changed, 576 insertions(+), 82 deletions(-) create mode 100644 tools/testing/selftests/net/bind_bhash_test.c
Signature · 1 line
2.30.2
PATCH v3 1/2 net: Add a second bind table hashed by port and address
We currently have one tcp bind table (bhash) which hashes by port number only. In the socket bind path, we check for bind conflicts by traversing the specified port's inet_bind2_bucket while holding the bucket's spinlock (see inet_csk_get_port() and inet_csk_bind_conflict()).
In instances where there are tons of sockets hashed to the same port at different addresses, checking for a bind conflict is time-intensive and can cause softirq cpu lockups, as well as stops new tcp connections since __inet_inherit_port() also contests for the spinlock.
This patch proposes adding a second bind table, bhash2, that hashes by port and ip address. Searching the bhash2 table leads to significantly faster conflict resolution and less time holding the spinlock.
--- include/net/inet_connection_sock.h | 3 + include/net/inet_hashtables.h | 56 ++++++- include/net/sock.h | 14 ++ net/dccp/proto.c | 34 ++++- net/ipv4/inet_connection_sock.c | 227 +++++++++++++++++++++-------- net/ipv4/inet_hashtables.c | 188 ++++++++++++++++++++++-- net/ipv4/tcp.c | 14 +- 7 files changed, 454 insertions(+), 82 deletions(-)
include/net/inet_connection_sock.h +3 -0
@@ -25,6 +25,7 @@#undef INET_CSK_CLEAR_TIMERSstruct inet_bind_bucket ; +struct inet_bind2_bucket ;struct tcp_congestion_ops ;/*
@@ -57,6 +58,7 @@ struct inet_connection_sock_af_ops {* * @icsk_accept_queue: FIFO of established children * @icsk_bind_hash: Bind node +* @icsk_bind2_hash: Bind node in the bhash2 table * @icsk_timeout: Timeout * @icsk_retransmit_timer: Resend (no ack) * @icsk_rto: Retransmit timeout
@@ -84,6 +86,7 @@ struct inet_connection_sock {struct inet_sock icsk_inet;struct request_sock_queue icsk_accept_queue;struct inet_bind_bucket *icsk_bind_hash; +struct inet_bind2_bucket *icsk_bind2_hash;unsigned long icsk_timeout;struct timer_list icsk_retransmit_timer;struct timer_list icsk_delack_timer;
include/net/inet_hashtables.h +55 -1
91 lines not on this page · raw message
include/net/sock.h +14 -0
@@ -351,6 +351,7 @@ struct sk_filter;* @sk_txtime_report_errors: set report errors mode for SO_TXTIME * @sk_txtime_unused: unused txtime flags * @ns_tracker: tracker for netns reference +* @sk_bind2_node: bind node in the bhash2 table */ struct sock {/*
@@ -540,6 +541,7 @@ struct sock {#endif struct rcu_head sk_rcu;netns_tracker ns_tracker; +struct hlist_node sk_bind2_node; };enum sk_pacing {
@@ -820,6 +822,16 @@ static inline void sk_add_bind_node(struct sock *sk,hlist_add_head (&sk->sk_bind_node, list); } +static inline void __sk_del_bind2_node (struct sock *sk) +{ +__hlist_del (&sk->sk_bind2_node); +} + +static inline void sk_add_bind2_node (struct sock *sk,struct hlist_head *list) +{ +hlist_add_head (&sk->sk_bind2_node, list); +} +#define sk_for_each (__sk, list) \ hlist_for_each_entry(__sk, list, sk_node)#define sk_for_each_rcu (__sk, list) \
@@ -837,6 +849,8 @@ static inline void sk_add_bind_node(struct sock *sk, hlist_for_each_entry_safe(__sk, tmp, list, sk_node)#define sk_for_each_bound (__sk, list) \ hlist_for_each_entry(__sk, list, sk_bind_node) +#define sk_for_each_bound_bhash2 (__sk, list) \ + hlist_for_each_entry(__sk, list, sk_bind2_node)/** * sk_for_each_entry_offset_rcu - iterate over a list at a given struct offset
net/dccp/proto.c +29 -5
@@ -1121,6 +1121,12 @@ static int __init dccp_init(void) SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT, NULL);if (!dccp_hashinfo.bind_bucket_cachep) goto out_free_hashinfo2; + dccp_hashinfo.bind2_bucket_cachep = +kmem_cache_create ("dccp_bind2_bucket" , +sizeof (struct inet_bind2_bucket ),0 , +SLAB_HWCACHE_ALIGN |SLAB_ACCOUNT ,NULL ); +if (!dccp_hashinfo.bind2_bucket_cachep) + goto out_free_bind_bucket_cachep;/* * Size and allocate the main established and bind bucket
@@ -1151,7 +1157,7 @@ static int __init dccp_init(void)if (!dccp_hashinfo.ehash) {DCCP_CRIT ("Failed to allocate DCCP established hash table" ); - goto out_free_bind_bucket_cachep; + goto out_free_bind2_bucket_cachep; }for (i =0 ; i <= dccp_hashinfo.ehash_mask; i++)
@@ -1170,21 +1176,31 @@ static int __init dccp_init(void)continue ; dccp_hashinfo.bhash = (struct inet_bind_hashbucket *)__get_free_pages (GFP_ATOMIC |__GFP_NOWARN, bhash_order); + dccp_hashinfo.bhash2 = (struct inet_bind2_hashbucket *) +__get_free_pages (GFP_ATOMIC | __GFP_NOWARN, bhash_order); }while (!dccp_hashinfo.bhash && --bhash_order >=0 );if (!dccp_hashinfo.bhash) {DCCP_CRIT ("Failed to allocate DCCP bind hash table" ); +if (dccp_hashinfo.bhash2) +free_pages ((unsigned long )dccp_hashinfo.bhash2, bhash_order); goto out_free_dccp_locks; } +if (!dccp_hashinfo.bhash2) { +DCCP_CRIT ("Failed to allocate DCCP bind2 hash table" ); + goto out_free_dccp_bhash; + } +for (i =0 ; i < dccp_hashinfo.bhash_size; i++) {spin_lock_init (&dccp_hashinfo.bhash[i].lock);INIT_HLIST_HEAD (&dccp_hashinfo.bhash[i].chain); +INIT_HLIST_HEAD (&dccp_hashinfo.bhash2[i].chain); } rc =dccp_mib_init ();if (rc) - goto out_free_dccp_bhash; + goto out_free_dccp_bhash2; rc =dccp_ackvec_init ();if (rc)
@@ -1208,30 +1224,38 @@ static int __init dccp_init(void)dccp_ackvec_exit (); out_free_dccp_mib:dccp_mib_exit (); +out_free_dccp_bhash2: +free_pages ((unsigned long )dccp_hashinfo.bhash2, bhash_order); out_free_dccp_bhash:free_pages ((unsigned long )dccp_hashinfo.bhash, bhash_order); out_free_dccp_locks:inet_ehash_locks_free (&dccp_hashinfo); out_free_dccp_ehash:free_pages ((unsigned long )dccp_hashinfo.ehash, ehash_order); +out_free_bind2_bucket_cachep: +kmem_cache_destroy (dccp_hashinfo.bind2_bucket_cachep); out_free_bind_bucket_cachep:kmem_cache_destroy (dccp_hashinfo.bind_bucket_cachep); out_free_hashinfo2:inet_hashinfo2_free_mod (&dccp_hashinfo); out_fail: dccp_hashinfo.bhash =NULL ; + dccp_hashinfo.bhash2 =NULL ; dccp_hashinfo.ehash =NULL ; dccp_hashinfo.bind_bucket_cachep =NULL ; + dccp_hashinfo.bind2_bucket_cachep =NULL ;return rc; } static void __exit dccp_fini(void) { +int bhash_order =get_order (dccp_hashinfo.bhash_size * +sizeof (struct inet_bind_hashbucket )); +ccid_cleanup_builtins ();dccp_mib_exit (); -free_pages ((unsigned long )dccp_hashinfo.bhash, -get_order (dccp_hashinfo.bhash_size * -sizeof (struct inet_bind_hashbucket ))); +free_pages ((unsigned long )dccp_hashinfo.bhash, bhash_order); +free_pages ((unsigned long )dccp_hashinfo.bhash2, bhash_order);free_pages ((unsigned long )dccp_hashinfo.ehash,get_order ((dccp_hashinfo.ehash_mask +1 ) *sizeof (struct inet_ehash_bucket )));
net/ipv4/inet_connection_sock.c +164 -63
314 lines not on this page · raw message
net/ipv4/inet_hashtables.c +177 -11
295 lines not on this page · raw message
net/ipv4/tcp.c +12 -2
@@ -4605,6 +4605,12 @@ void __init tcp_init(void)SLAB_HWCACHE_ALIGN |SLAB_PANIC |SLAB_ACCOUNT , NULL); + tcp_hashinfo.bind2_bucket_cachep = +kmem_cache_create ("tcp_bind2_bucket" , +sizeof (struct inet_bind2_bucket ),0 , +SLAB_HWCACHE_ALIGN |SLAB_PANIC | +SLAB_ACCOUNT , +NULL );/* Size and allocate the main established and bind bucket * hash tables.
@@ -4627,8 +4633,9 @@ void __init tcp_init(void)if (inet_ehash_locks_alloc (&tcp_hashinfo))panic ("TCP: failed to alloc ehash_locks" ); tcp_hashinfo.bhash = - alloc_large_system_hash("TCP bind", - sizeof(struct inet_bind_hashbucket), + alloc_large_system_hash("TCP bind bhash tables", + sizeof(struct inet_bind_hashbucket) + + sizeof(struct inet_bind2_hashbucket), tcp_hashinfo.ehash_mask + 1, 17, /* one slot per 128 KB of memory */ 0,
@@ -4637,9 +4644,12 @@ void __init tcp_init(void)0 , 64 * 1024); tcp_hashinfo.bhash_size =1U << tcp_hashinfo.bhash_size; + tcp_hashinfo.bhash2 = + (struct inet_bind2_hashbucket *)(tcp_hashinfo.bhash + tcp_hashinfo.bhash_size);for (i =0 ; i < tcp_hashinfo.bhash_size; i++) {spin_lock_init (&tcp_hashinfo.bhash[i].lock);INIT_HLIST_HEAD (&tcp_hashinfo.bhash[i].chain); +INIT_HLIST_HEAD (&tcp_hashinfo.bhash2[i].chain); }
Signature · 1 line
2.30.2
Re: [PATCH net-next v3 1/2] net: Add a second bind table hashed by port and address
Quoted 227 lines · Joanne Koong
We currently have one tcp bind table (bhash) which hashes by port number only. In the socket bind path, we check for bind conflicts by traversing the specified port's
inet_bind2_bucketwhile holding the bucket's spinlock (seeinet_csk_get_port()andinet_csk_bind_conflict()).In instances where there are tons of sockets hashed to the same port at different addresses, checking for a bind conflict is time-intensive and can cause softirq cpu lockups, as well as stops new tcp connections since
__inet_inherit_port()also contests for the spinlock.This patch proposes adding a second bind table, bhash2, that hashes by port and ip address. Searching the bhash2 table leads to significantly faster conflict resolution and less time holding the spinlock.
Signed-off-by Joanne Koong <[email protected]>--- include/net/inet_connection_sock.h | 3 + include/net/inet_hashtables.h | 56 ++++++- include/net/sock.h | 14 ++ net/dccp/proto.c | 34 ++++- net/ipv4/inet_connection_sock.c | 227 +++++++++++++++++++++-------- net/ipv4/inet_hashtables.c | 188 ++++++++++++++++++++++-- net/ipv4/tcp.c | 14 +- 7 files changed, 454 insertions(+), 82 deletions(-)include/net/inet_connection_sock.h +3
-0@@ -25,6 +25,7 @@#undef INET_CSK_CLEAR_TIMERSstruct inet_bind_bucket ; +struct inet_bind2_bucket ;struct tcp_congestion_ops ;/* @@ -57,6 +58,7 @@ struct inet_connection_sock_af_ops {* * @icsk_accept_queue: FIFO of established children * @icsk_bind_hash: Bind node +* @icsk_bind2_hash: Bind node in the bhash2 table * @icsk_timeout: Timeout * @icsk_retransmit_timer: Resend (no ack) * @icsk_rto: Retransmit timeout @@ -84,6 +86,7 @@ struct inet_connection_sock {struct inet_sock icsk_inet;struct request_sock_queue icsk_accept_queue;struct inet_bind_bucket *icsk_bind_hash; +struct inet_bind2_bucket *icsk_bind2_hash;unsigned long icsk_timeout;struct timer_list icsk_retransmit_timer;struct timer_list icsk_delack_timer;include/net/inet_hashtables.h +55
-1@@ -90,11 +90,30 @@ struct inet_bind_bucket {struct hlist_head owners; }; +struct inet_bind2_bucket { +possible_net_t ib_net; +int l3mdev; +unsigned short port; +union { +#if IS_ENABLED (CONFIG_IPV6 ) +struct in6_addr v6_rcv_saddr; +#endif +__be32 rcv_saddr; + }; +struct hlist_node node;/* Node in the inet2_bind_hashbucket chain */ +struct hlist_head owners;/* List of sockets hashed to this bucket */ +}; +static inline struct net *ib_net (struct inet_bind_bucket *ib) {return read_pnet (&ib->ib_net); } +static inline struct net *ib2_net (struct inet_bind2_bucket *ib) +{ +return read_pnet (&ib->ib_net); +} +#define inet_bind_bucket_for_each (tb, head) \ hlist_for_each_entry(tb, head, node)@@ -103,6 +122,15 @@ struct inet_bind_hashbucket {struct hlist_head chain; }; +/* This is synchronized using the inet_bind_hashbucket's spinlock. +* Instead of having separate spinlocks, the inet_bind2_hashbucket can share +* the inet_bind_hashbucket's given that in every case where the bhash2 table +* is useful, a lookup in the bhash table also occurs. +*/ +struct inet_bind2_hashbucket { +struct hlist_head chain; +}; +/* Sockets can be hashed in established or listening table. * We must use different 'nulls' end-of-chain value for all hash buckets : * A socket might transition from ESTABLISH to LISTEN state without @@ -138,6 +166,11 @@ struct inet_hashinfo {*/ struct kmem_cache *bind_bucket_cachep;struct inet_bind_hashbucket *bhash; +/* The 2nd binding table hashed by port and address. +* This is used primarily for expediting the resolution of bind conflicts. +*/ +struct kmem_cache *bind2_bucket_cachep; +struct inet_bind2_hashbucket *bhash2;unsigned int bhash_size;/* The 2nd listener table hashed by local port and address */ @@ -221,6 +254,27 @@ inet_bind_bucket_create(struct kmem_cache *cachep, struct net *net,void inet_bind_bucket_destroy (struct kmem_cache *cachep,struct inet_bind_bucket *tb); +static inline bool check_bind_bucket_match (struct inet_bind_bucket *tb,struct net *net, +const unsigned short port,int l3mdev) +{ +return net_eq (ib_net (tb), net) && tb->port == port && tb->l3mdev == l3mdev; +} + +struct inet_bind2_bucket * +inet_bind2_bucket_create (struct kmem_cache *cachep,struct net *net, +struct inet_bind2_hashbucket *head,const unsigned short port, +int l3mdev,const struct sock *sk); + +void inet_bind2_bucket_destroy (struct kmem_cache *cachep,struct inet_bind2_bucket *tb); + +struct inet_bind2_bucket * +inet_bind2_bucket_find (struct inet_hashinfo *hinfo,struct net *net,const unsigned short port, +int l3mdev,struct sock *sk,struct inet_bind2_hashbucket **head); + +bool check_bind2_bucket_match_nulladdr (struct inet_bind2_bucket *tb,struct net *net, +const unsigned short port,int l3mdev, +const struct sock *sk); +static inline u32 inet_bhashfn (const struct net *net,const __u16 lport,const u32 bhash_size) {@@ -228,7 +282,7 @@ static inline u32 inet_bhashfn(const struct net *net, const __u16 lport, }void inet_bind_hash (struct sock *sk,struct inet_bind_bucket *tb, -const unsigned short snum); +struct inet_bind2_bucket *tb2,const unsigned short snum);/* These can have wildcards, don't try too hard. */ static inline u32 inet_lhashfn (const struct net *net,const unsigned short num)include/net/sock.h +14
-0@@ -351,6 +351,7 @@ struct sk_filter;* @sk_txtime_report_errors: set report errors mode for SO_TXTIME * @sk_txtime_unused: unused txtime flags * @ns_tracker: tracker for netns reference +* @sk_bind2_node: bind node in the bhash2 table */ struct sock {/* @@ -540,6 +541,7 @@ struct sock {#endif struct rcu_head sk_rcu;netns_tracker ns_tracker; +struct hlist_node sk_bind2_node; };enum sk_pacing {@@ -820,6 +822,16 @@ static inline void sk_add_bind_node(struct sock *sk,hlist_add_head (&sk->sk_bind_node, list); } +static inline void __sk_del_bind2_node (struct sock *sk) +{ +__hlist_del (&sk->sk_bind2_node); +} + +static inline void sk_add_bind2_node (struct sock *sk,struct hlist_head *list) +{ +hlist_add_head (&sk->sk_bind2_node, list); +} +#define sk_for_each (__sk, list) \ hlist_for_each_entry(__sk, list, sk_node)#define sk_for_each_rcu (__sk, list) \@@ -837,6 +849,8 @@ static inline void sk_add_bind_node(struct sock *sk, hlist_for_each_entry_safe(__sk, tmp, list, sk_node)#define sk_for_each_bound (__sk, list) \ hlist_for_each_entry(__sk, list, sk_bind_node) +#define sk_for_each_bound_bhash2 (__sk, list) \ + hlist_for_each_entry(__sk, list, sk_bind2_node)/** * sk_for_each_entry_offset_rcu - iterate over a list at a given struct offset net/dccp/proto.c +7
-1@@ -1121,6 +1121,12 @@ static int __init dccp_init(void) SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT, NULL);if (!dccp_hashinfo.bind_bucket_cachep) goto out_free_hashinfo2; + dccp_hashinfo.bind2_bucket_cachep = +kmem_cache_create ("dccp_bind2_bucket" , +sizeof (struct inet_bind2_bucket ),0 , +SLAB_HWCACHE_ALIGN |SLAB_ACCOUNT ,NULL ); +if (!dccp_hashinfo.bind2_bucket_cachep) + goto out_free_bind_bucket_cachep;/* * Size and allocate the main established and bind bucket @@ -1151,7 +1157,7 @@ static int __init dccp_init(void)if (!dccp_hashinfo.ehash) {DCCP_CRIT ("Failed to allocate DCCP established hash table" ); - goto out_free_bind_bucket_cachep; + goto out_free_bind2_bucket_cachep; }for (i =0 ; i <= dccp_hashinfo.ehash_mask; i++)
net/dccp/proto.c static int __init dccp_init(void) · line 1180
Quoted 5 lines
net/dccp/proto.c +2
-0@@ -1170,21 +1176,31 @@ static int __init dccp_init(void)continue ; dccp_hashinfo.bhash = (struct inet_bind_hashbucket *)__get_free_pages (GFP_ATOMIC |__GFP_NOWARN, bhash_order); + dccp_hashinfo.bhash2 = (struct inet_bind2_hashbucket *) +__get_free_pages (GFP_ATOMIC | __GFP_NOWARN, bhash_order);
I do not think you fixed the issue.
Imagine dccp_hashinfo.bhash allocation has failed, but bhash2 allocation succeeded.
We will run another time this code, because of the while (!dccp_hashinfo.bhash && --bhash_order >= 0);
So next allocation will overwrite dccp_hashinfo.bhash2 and memory is leaked.
I suggest adding :
if (dccp_hashinfo.bhash)
dccp_hashinfo.bhash2 = (struct inet_bind2_hashbucket *)
__get_free_pages(GFP_ATOMIC | __GFP_NOWARN, bhash_order);
net/dccp/proto.c static int __init dccp_init(void) · line 1186
Quoted 6 lines
net/dccp/proto.c +2
-0}while (!dccp_hashinfo.bhash && --bhash_order >=0 );if (!dccp_hashinfo.bhash) {DCCP_CRIT ("Failed to allocate DCCP bind hash table" ); +if (dccp_hashinfo.bhash2) +free_pages ((unsigned long )dccp_hashinfo.bhash2, bhash_order);
Then you can remove this part.
Quoted 9 lines
net/dccp/proto.c +5
-0goto out_free_dccp_locks; } +if (!dccp_hashinfo.bhash2) { +DCCP_CRIT ("Failed to allocate DCCP bind2 hash table" ); + goto out_free_dccp_bhash; + } +
Re: [PATCH net-next v3 1/2] net: Add a second bind table hashed by port and address
Quoted 251 lines · Eric Dumazet
Quoted 227 lines · Joanne Koong
We currently have one tcp bind table (bhash) which hashes by port number only. In the socket bind path, we check for bind conflicts by traversing the specified port's
inet_bind2_bucketwhile holding the bucket's spinlock (seeinet_csk_get_port()andinet_csk_bind_conflict()).In instances where there are tons of sockets hashed to the same port at different addresses, checking for a bind conflict is time-intensive and can cause softirq cpu lockups, as well as stops new tcp connections since
__inet_inherit_port()also contests for the spinlock.This patch proposes adding a second bind table, bhash2, that hashes by port and ip address. Searching the bhash2 table leads to significantly faster conflict resolution and less time holding the spinlock.
Signed-off-by Joanne Koong <[email protected]>--- include/net/inet_connection_sock.h | 3 + include/net/inet_hashtables.h | 56 ++++++- include/net/sock.h | 14 ++ net/dccp/proto.c | 34 ++++- net/ipv4/inet_connection_sock.c | 227 +++++++++++++++++++++-------- net/ipv4/inet_hashtables.c | 188 ++++++++++++++++++++++-- net/ipv4/tcp.c | 14 +- 7 files changed, 454 insertions(+), 82 deletions(-)include/net/inet_connection_sock.h +3
-0@@ -25,6 +25,7 @@#undef INET_CSK_CLEAR_TIMERSstruct inet_bind_bucket ; +struct inet_bind2_bucket ;struct tcp_congestion_ops ;/* @@ -57,6 +58,7 @@ struct inet_connection_sock_af_ops {* * @icsk_accept_queue: FIFO of established children * @icsk_bind_hash: Bind node +* @icsk_bind2_hash: Bind node in the bhash2 table * @icsk_timeout: Timeout * @icsk_retransmit_timer: Resend (no ack) * @icsk_rto: Retransmit timeout @@ -84,6 +86,7 @@ struct inet_connection_sock {struct inet_sock icsk_inet;struct request_sock_queue icsk_accept_queue;struct inet_bind_bucket *icsk_bind_hash; +struct inet_bind2_bucket *icsk_bind2_hash;unsigned long icsk_timeout;struct timer_list icsk_retransmit_timer;struct timer_list icsk_delack_timer;include/net/inet_hashtables.h +55
-1@@ -90,11 +90,30 @@ struct inet_bind_bucket {struct hlist_head owners; }; +struct inet_bind2_bucket { +possible_net_t ib_net; +int l3mdev; +unsigned short port; +union { +#if IS_ENABLED (CONFIG_IPV6 ) +struct in6_addr v6_rcv_saddr; +#endif +__be32 rcv_saddr; + }; +struct hlist_node node;/* Node in the inet2_bind_hashbucket chain */ +struct hlist_head owners;/* List of sockets hashed to this bucket */ +}; +static inline struct net *ib_net (struct inet_bind_bucket *ib) {return read_pnet (&ib->ib_net); } +static inline struct net *ib2_net (struct inet_bind2_bucket *ib) +{ +return read_pnet (&ib->ib_net); +} +#define inet_bind_bucket_for_each (tb, head) \ hlist_for_each_entry(tb, head, node)@@ -103,6 +122,15 @@ struct inet_bind_hashbucket {struct hlist_head chain; }; +/* This is synchronized using the inet_bind_hashbucket's spinlock. +* Instead of having separate spinlocks, the inet_bind2_hashbucket can share +* the inet_bind_hashbucket's given that in every case where the bhash2 table +* is useful, a lookup in the bhash table also occurs. +*/ +struct inet_bind2_hashbucket { +struct hlist_head chain; +}; +/* Sockets can be hashed in established or listening table. * We must use different 'nulls' end-of-chain value for all hash buckets : * A socket might transition from ESTABLISH to LISTEN state without @@ -138,6 +166,11 @@ struct inet_hashinfo {*/ struct kmem_cache *bind_bucket_cachep;struct inet_bind_hashbucket *bhash; +/* The 2nd binding table hashed by port and address. +* This is used primarily for expediting the resolution of bind conflicts. +*/ +struct kmem_cache *bind2_bucket_cachep; +struct inet_bind2_hashbucket *bhash2;unsigned int bhash_size;/* The 2nd listener table hashed by local port and address */ @@ -221,6 +254,27 @@ inet_bind_bucket_create(struct kmem_cache *cachep, struct net *net,void inet_bind_bucket_destroy (struct kmem_cache *cachep,struct inet_bind_bucket *tb); +static inline bool check_bind_bucket_match (struct inet_bind_bucket *tb,struct net *net, +const unsigned short port,int l3mdev) +{ +return net_eq (ib_net (tb), net) && tb->port == port && tb->l3mdev == l3mdev; +} + +struct inet_bind2_bucket * +inet_bind2_bucket_create (struct kmem_cache *cachep,struct net *net, +struct inet_bind2_hashbucket *head,const unsigned short port, +int l3mdev,const struct sock *sk); + +void inet_bind2_bucket_destroy (struct kmem_cache *cachep,struct inet_bind2_bucket *tb); + +struct inet_bind2_bucket * +inet_bind2_bucket_find (struct inet_hashinfo *hinfo,struct net *net,const unsigned short port, +int l3mdev,struct sock *sk,struct inet_bind2_hashbucket **head); + +bool check_bind2_bucket_match_nulladdr (struct inet_bind2_bucket *tb,struct net *net, +const unsigned short port,int l3mdev, +const struct sock *sk); +static inline u32 inet_bhashfn (const struct net *net,const __u16 lport,const u32 bhash_size) {@@ -228,7 +282,7 @@ static inline u32 inet_bhashfn(const struct net *net, const __u16 lport, }void inet_bind_hash (struct sock *sk,struct inet_bind_bucket *tb, -const unsigned short snum); +struct inet_bind2_bucket *tb2,const unsigned short snum);/* These can have wildcards, don't try too hard. */ static inline u32 inet_lhashfn (const struct net *net,const unsigned short num)include/net/sock.h +14
-0@@ -351,6 +351,7 @@ struct sk_filter;* @sk_txtime_report_errors: set report errors mode for SO_TXTIME * @sk_txtime_unused: unused txtime flags * @ns_tracker: tracker for netns reference +* @sk_bind2_node: bind node in the bhash2 table */ struct sock {/* @@ -540,6 +541,7 @@ struct sock {#endif struct rcu_head sk_rcu;netns_tracker ns_tracker; +struct hlist_node sk_bind2_node; };enum sk_pacing {@@ -820,6 +822,16 @@ static inline void sk_add_bind_node(struct sock *sk,hlist_add_head (&sk->sk_bind_node, list); } +static inline void __sk_del_bind2_node (struct sock *sk) +{ +__hlist_del (&sk->sk_bind2_node); +} + +static inline void sk_add_bind2_node (struct sock *sk,struct hlist_head *list) +{ +hlist_add_head (&sk->sk_bind2_node, list); +} +#define sk_for_each (__sk, list) \ hlist_for_each_entry(__sk, list, sk_node)#define sk_for_each_rcu (__sk, list) \@@ -837,6 +849,8 @@ static inline void sk_add_bind_node(struct sock *sk, hlist_for_each_entry_safe(__sk, tmp, list, sk_node)#define sk_for_each_bound (__sk, list) \ hlist_for_each_entry(__sk, list, sk_bind_node) +#define sk_for_each_bound_bhash2 (__sk, list) \ + hlist_for_each_entry(__sk, list, sk_bind2_node)/** * sk_for_each_entry_offset_rcu - iterate over a list at a given struct offset net/dccp/proto.c +7
-1@@ -1121,6 +1121,12 @@ static int __init dccp_init(void) SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT, NULL);if (!dccp_hashinfo.bind_bucket_cachep) goto out_free_hashinfo2; + dccp_hashinfo.bind2_bucket_cachep = +kmem_cache_create ("dccp_bind2_bucket" , +sizeof (struct inet_bind2_bucket ),0 , +SLAB_HWCACHE_ALIGN |SLAB_ACCOUNT ,NULL ); +if (!dccp_hashinfo.bind2_bucket_cachep) + goto out_free_bind_bucket_cachep;/* * Size and allocate the main established and bind bucket @@ -1151,7 +1157,7 @@ static int __init dccp_init(void)if (!dccp_hashinfo.ehash) {DCCP_CRIT ("Failed to allocate DCCP established hash table" ); - goto out_free_bind_bucket_cachep; + goto out_free_bind2_bucket_cachep; }for (i =0 ; i <= dccp_hashinfo.ehash_mask; i++)net/dccp/proto.c static int __init dccp_init(void) · line 1180
Quoted 5 lines
net/dccp/proto.c +2
-0@@ -1170,21 +1176,31 @@ static int __init dccp_init(void)continue ; dccp_hashinfo.bhash = (struct inet_bind_hashbucket *)__get_free_pages (GFP_ATOMIC |__GFP_NOWARN, bhash_order); + dccp_hashinfo.bhash2 = (struct inet_bind2_hashbucket *) +__get_free_pages (GFP_ATOMIC | __GFP_NOWARN, bhash_order);I do not think you fixed the issue.
Imagine dccp_hashinfo.bhash allocation has failed, but bhash2 allocation succeeded.
We will run another time this code, because of the while (!dccp_hashinfo.bhash && --bhash_order >= 0);
So next allocation will overwrite dccp_hashinfo.bhash2 and memory is leaked.
I suggest adding :
if (dccp_hashinfo.bhash)
dccp_hashinfo.bhash2 = (structinet_bind2_hashbucket*)
__get_free_pages(GFP_ATOMIC | __GFP_NOWARN, bhash_order);
Quoted 9 lines
net/dccp/proto.c static int __init dccp_init(void) · line 1186
Quoted 6 lines
net/dccp/proto.c +2
-0}while (!dccp_hashinfo.bhash && --bhash_order >=0 );if (!dccp_hashinfo.bhash) {DCCP_CRIT ("Failed to allocate DCCP bind hash table" ); +if (dccp_hashinfo.bhash2) +free_pages ((unsigned long )dccp_hashinfo.bhash2, bhash_order);Then you can remove this part.
Ah I see. Thanks for your feedback - I really like your suggestion. Do you want me to fix this and send another version or wait a bit?
Quoted 10 lines
Quoted 10 lines
net/dccp/proto.c +5
-0goto out_free_dccp_locks; } +if (!dccp_hashinfo.bhash2) { +DCCP_CRIT ("Failed to allocate DCCP bind2 hash table" ); + goto out_free_dccp_bhash; + } +
Re: [PATCH net-next v3 1/2] net: Add a second bind table hashed by port and address
Quoted 2 lines · Joanne Koong
Ah I see. Thanks for your feedback - I really like your suggestion. Do you want me to fix this and send another version or wait a bit?
You can send a new version at your convenience, thank you.
PATCH v3 2/2 selftests: Add test for timing a bind request to a port with a populated bhash entry
In reply to Joanne KoongThis test populates the bhash table for a given port with MAX_THREADS * MAX_CONNECTIONS sockets, and then times how long a bind request on the port takes.
When populating the bhash table, we create the sockets and then bind the sockets to the same address and port (SO_REUSEADDR and SO_REUSEPORT are set). When timing how long a bind on the port takes, we bind on a different address without SO_REUSEPORT set. We do not set SO_REUSEPORT because we are interested in the case where the bind request does not go through the tb->fastreuseport path, which is fragile (eg tb->fastreuseport path does not work if binding with a different uid).
To run the test locally, I did:
* ulimit -n 65535000
* ip addr add 2001:0db8:0:f101::1 dev eth0
* ./bind_bhash_test 443
--- tools/testing/selftests/net/.gitignore | 1 + tools/testing/selftests/net/Makefile | 2 + tools/testing/selftests/net/bind_bhash_test.c | 119 ++++++++++++++++++ 3 files changed, 122 insertions(+) create mode 100644 tools/testing/selftests/net/bind_bhash_test.c
tools/testing/selftests/net/.gitignore +1 -0
@@ -36,3 +36,4 @@ gro ioam6_parser toeplitz cmsg_sender +bind_bhash_test
tools/testing/selftests/net/Makefile +2 -0
@@ -56,6 +56,7 @@ TEST_GEN_PROGS += reuseport_dualstack reuseaddr_conflict tls TEST_GEN_FILES += toeplitz TEST_GEN_FILES += cmsg_sender TEST_PROGS += test_vxlan_vnifiltering.sh +TEST_GEN_FILES += bind_bhash_test TEST_FILES := settings
@@ -64,4 +65,5 @@ include ../lib.mk $(OUTPUT)/reuseport_bpf_numa: LDLIBS += -lnuma $(OUTPUT)/tcp_mmap: LDLIBS += -lpthread +$(OUTPUT)/bind_bhash_test: LDLIBS += -lpthread $(OUTPUT)/tcp_inq: LDLIBS += -lpthread
tools/testing/selftests/net/bind_bhash_test.c new file +119 -0
119 lines not on this page · raw message
Signature · 1 line
2.30.2