Add a bhash2 table hashed by port + address
Subsystems (5)
Jakub Kicinski
maintainerv5 cover Add a bhash2 table hashed by port + address
Joanne Koong says:
This patchset proposes adding a bhash2 table that hashes by port and address. The motivation behind bhash2 is to expedite bind requests in situations where the port has many sockets in its bhash table entry, which makes checking bind conflicts costly especially given that we acquire the table entry spinlock while doing so, which can cause softirq cpu lockups and can prevent new tcp connections.
We ran into this problem at Meta where the traffic team binds a large number of IPs to port 443 and the bind() call took a significant amount of time which led to cpu softirq lockups, which caused packet drops and other failures on the machine
The patches are as follows:
1/2 - Adds a second bhash table (bhash2) hashed by port and address 2/2 - Adds a test for timing how long an additional bind request takes when the bhash entry is populated
When experimentally testing this on a local server for ~24k sockets bound to the port, the results seen were:
ipv4:
before - 0.002317 seconds
with bhash2 - 0.000018 seconds
ipv6:
before - 0.002431 seconds
with bhash2 - 0.000021 seconds
v4 -> v5:
v4:
https://lore.kernel.org/netdev/[email protected]/
* Remove "inline" in check_bind2_bucket_match function
* Add line breaks to limit lengths to 80 columns
* Rebase to master and fix merge conflict
v3 -> v4:
v3:
https://lore.kernel.org/netdev/[email protected]/
* Fix the fix for the dccp bhash2 allocation
v2 -> v3:
v2:
https://lore.kernel.org/netdev/[email protected]/
* Fix bhash2 allocation error handling for dccp
* Rebase onto net-next/master
v1 -> v2:
v1:
https://lore.kernel.org/netdev/[email protected]/
* Attached test for timing bind request
Joanne Koong (2):
net: Add a second bind table hashed by port and address
selftests: Add test for timing a bind request to a port with a
populated bhash entry
include/net/inet_connection_sock.h | 3 + include/net/inet_hashtables.h | 68 ++++- include/net/sock.h | 14 + net/dccp/proto.c | 33 ++- net/ipv4/inet_connection_sock.c | 247 +++++++++++++----- net/ipv4/inet_hashtables.c | 193 +++++++++++++- net/ipv4/tcp.c | 14 +- tools/testing/selftests/net/.gitignore | 1 + tools/testing/selftests/net/Makefile | 2 + tools/testing/selftests/net/bind_bhash_test.c | 119 +++++++++ 10 files changed, 611 insertions(+), 83 deletions(-) create mode 100644 tools/testing/selftests/net/bind_bhash_test.c
Signature · 1 line
2.34.3
Jakub Kicinski
maintainerPATCH RESEND v5 1/2 net: Add a second bind table hashed by port and address
From: Joanne Koong <[email protected]>
We currently have one tcp bind table (bhash) which hashes by port number only. In the socket bind path, we check for bind conflicts by traversing the specified port's inet_bind2_bucket while holding the bucket's spinlock (see inet_csk_get_port() and inet_csk_bind_conflict()).
In instances where there are tons of sockets hashed to the same port at different addresses, checking for a bind conflict is time-intensive and can cause softirq cpu lockups, as well as stops new tcp connections since __inet_inherit_port() also contests for the spinlock.
This patch proposes adding a second bind table, bhash2, that hashes by port and ip address. Searching the bhash2 table leads to significantly faster conflict resolution and less time holding the spinlock.
--- include/net/inet_connection_sock.h | 3 + include/net/inet_hashtables.h | 68 +++++++- include/net/sock.h | 14 ++ net/dccp/proto.c | 33 +++- net/ipv4/inet_connection_sock.c | 247 +++++++++++++++++++++-------- net/ipv4/inet_hashtables.c | 193 ++++++++++++++++++++-- net/ipv4/tcp.c | 14 +- 7 files changed, 489 insertions(+), 83 deletions(-)
include/net/inet_connection_sock.h +3 -0
@@ -25,6 +25,7 @@#undef INET_CSK_CLEAR_TIMERSstruct inet_bind_bucket ; +struct inet_bind2_bucket ;struct tcp_congestion_ops ;/*
@@ -57,6 +58,7 @@ struct inet_connection_sock_af_ops {* * @icsk_accept_queue: FIFO of established children * @icsk_bind_hash: Bind node +* @icsk_bind2_hash: Bind node in the bhash2 table * @icsk_timeout: Timeout * @icsk_retransmit_timer: Resend (no ack) * @icsk_rto: Retransmit timeout
@@ -83,6 +85,7 @@ struct inet_connection_sock {struct inet_sock icsk_inet;struct request_sock_queue icsk_accept_queue;struct inet_bind_bucket *icsk_bind_hash; +struct inet_bind2_bucket *icsk_bind2_hash;unsigned long icsk_timeout;struct timer_list icsk_retransmit_timer;struct timer_list icsk_delack_timer;
include/net/inet_hashtables.h +67 -1
103 lines not on this page · raw message
include/net/sock.h +14 -0
@@ -348,6 +348,7 @@ struct sk_filter;* @sk_txtime_report_errors: set report errors mode for SO_TXTIME * @sk_txtime_unused: unused txtime flags * @ns_tracker: tracker for netns reference +* @sk_bind2_node: bind node in the bhash2 table */ struct sock {/*
@@ -537,6 +538,7 @@ struct sock {#endif struct rcu_head sk_rcu;netns_tracker ns_tracker; +struct hlist_node sk_bind2_node; };enum sk_pacing {
@@ -817,6 +819,16 @@ static inline void sk_add_bind_node(struct sock *sk,hlist_add_head (&sk->sk_bind_node, list); } +static inline void __sk_del_bind2_node (struct sock *sk) +{ +__hlist_del (&sk->sk_bind2_node); +} + +static inline void sk_add_bind2_node (struct sock *sk,struct hlist_head *list) +{ +hlist_add_head (&sk->sk_bind2_node, list); +} +#define sk_for_each (__sk, list) \ hlist_for_each_entry(__sk, list, sk_node)#define sk_for_each_rcu (__sk, list) \
@@ -834,6 +846,8 @@ static inline void sk_add_bind_node(struct sock *sk, hlist_for_each_entry_safe(__sk, tmp, list, sk_node)#define sk_for_each_bound (__sk, list) \ hlist_for_each_entry(__sk, list, sk_bind_node) +#define sk_for_each_bound_bhash2 (__sk, list) \ + hlist_for_each_entry(__sk, list, sk_bind2_node)/** * sk_for_each_entry_offset_rcu - iterate over a list at a given struct offset
net/dccp/proto.c +28 -5
@@ -1120,6 +1120,12 @@ static int __init dccp_init(void) SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT, NULL);if (!dccp_hashinfo.bind_bucket_cachep) goto out_free_hashinfo2; + dccp_hashinfo.bind2_bucket_cachep = +kmem_cache_create ("dccp_bind2_bucket" , +sizeof (struct inet_bind2_bucket ),0 , +SLAB_HWCACHE_ALIGN |SLAB_ACCOUNT ,NULL ); +if (!dccp_hashinfo.bind2_bucket_cachep) + goto out_free_bind_bucket_cachep;/* * Size and allocate the main established and bind bucket
@@ -1150,7 +1156,7 @@ static int __init dccp_init(void)if (!dccp_hashinfo.ehash) {DCCP_CRIT ("Failed to allocate DCCP established hash table" ); - goto out_free_bind_bucket_cachep; + goto out_free_bind2_bucket_cachep; }for (i =0 ; i <= dccp_hashinfo.ehash_mask; i++)
@@ -1176,14 +1182,23 @@ static int __init dccp_init(void) goto out_free_dccp_locks; } + dccp_hashinfo.bhash2 = (struct inet_bind2_hashbucket *) +__get_free_pages (GFP_ATOMIC | __GFP_NOWARN, bhash_order); + +if (!dccp_hashinfo.bhash2) { +DCCP_CRIT ("Failed to allocate DCCP bind2 hash table" ); + goto out_free_dccp_bhash; + } +for (i =0 ; i < dccp_hashinfo.bhash_size; i++) {spin_lock_init (&dccp_hashinfo.bhash[i].lock);INIT_HLIST_HEAD (&dccp_hashinfo.bhash[i].chain); +INIT_HLIST_HEAD (&dccp_hashinfo.bhash2[i].chain); } rc =dccp_mib_init ();if (rc) - goto out_free_dccp_bhash; + goto out_free_dccp_bhash2; rc =dccp_ackvec_init ();if (rc)
@@ -1207,30 +1222,38 @@ static int __init dccp_init(void)dccp_ackvec_exit (); out_free_dccp_mib:dccp_mib_exit (); +out_free_dccp_bhash2: +free_pages ((unsigned long )dccp_hashinfo.bhash2, bhash_order); out_free_dccp_bhash:free_pages ((unsigned long )dccp_hashinfo.bhash, bhash_order); out_free_dccp_locks:inet_ehash_locks_free (&dccp_hashinfo); out_free_dccp_ehash:free_pages ((unsigned long )dccp_hashinfo.ehash, ehash_order); +out_free_bind2_bucket_cachep: +kmem_cache_destroy (dccp_hashinfo.bind2_bucket_cachep); out_free_bind_bucket_cachep:kmem_cache_destroy (dccp_hashinfo.bind_bucket_cachep); out_free_hashinfo2:inet_hashinfo2_free_mod (&dccp_hashinfo); out_fail: dccp_hashinfo.bhash =NULL ; + dccp_hashinfo.bhash2 =NULL ; dccp_hashinfo.ehash =NULL ; dccp_hashinfo.bind_bucket_cachep =NULL ; + dccp_hashinfo.bind2_bucket_cachep =NULL ;return rc; } static void __exit dccp_fini(void) { +int bhash_order =get_order (dccp_hashinfo.bhash_size * +sizeof (struct inet_bind_hashbucket )); +ccid_cleanup_builtins ();dccp_mib_exit (); -free_pages ((unsigned long )dccp_hashinfo.bhash, -get_order (dccp_hashinfo.bhash_size * -sizeof (struct inet_bind_hashbucket ))); +free_pages ((unsigned long )dccp_hashinfo.bhash, bhash_order); +free_pages ((unsigned long )dccp_hashinfo.bhash2, bhash_order);free_pages ((unsigned long )dccp_hashinfo.ehash,get_order ((dccp_hashinfo.ehash_mask +1 ) *sizeof (struct inet_ehash_bucket )));
net/ipv4/inet_connection_sock.c +183 -64
337 lines not on this page · raw message
net/ipv4/inet_hashtables.c +182 -11
300 lines not on this page · raw message
net/ipv4/tcp.c +12 -2
@@ -4604,6 +4604,12 @@ void __init tcp_init(void)SLAB_HWCACHE_ALIGN |SLAB_PANIC |SLAB_ACCOUNT , NULL); + tcp_hashinfo.bind2_bucket_cachep = +kmem_cache_create ("tcp_bind2_bucket" , +sizeof (struct inet_bind2_bucket ),0 , +SLAB_HWCACHE_ALIGN |SLAB_PANIC | +SLAB_ACCOUNT , +NULL );/* Size and allocate the main established and bind bucket * hash tables.
@@ -4626,8 +4632,9 @@ void __init tcp_init(void)if (inet_ehash_locks_alloc (&tcp_hashinfo))panic ("TCP: failed to alloc ehash_locks" ); tcp_hashinfo.bhash = - alloc_large_system_hash("TCP bind", - sizeof(struct inet_bind_hashbucket), + alloc_large_system_hash("TCP bind bhash tables", + sizeof(struct inet_bind_hashbucket) + + sizeof(struct inet_bind2_hashbucket), tcp_hashinfo.ehash_mask + 1, 17, /* one slot per 128 KB of memory */ 0,
@@ -4636,9 +4643,12 @@ void __init tcp_init(void)0 , 64 * 1024); tcp_hashinfo.bhash_size =1U << tcp_hashinfo.bhash_size; + tcp_hashinfo.bhash2 = + (struct inet_bind2_hashbucket *)(tcp_hashinfo.bhash + tcp_hashinfo.bhash_size);for (i =0 ; i < tcp_hashinfo.bhash_size; i++) {spin_lock_init (&tcp_hashinfo.bhash[i].lock);INIT_HLIST_HEAD (&tcp_hashinfo.bhash[i].chain); +INIT_HLIST_HEAD (&tcp_hashinfo.bhash2[i].chain); }
Signature · 1 line
2.34.3
Jakub Kicinski
maintainerPATCH RESEND v5 2/2 selftests: Add test for timing a bind request to a port with a populated bhash entry
In reply to Jakub KicinskiFrom: Joanne Koong <[email protected]>
This test populates the bhash table for a given port with MAX_THREADS * MAX_CONNECTIONS sockets, and then times how long a bind request on the port takes.
When populating the bhash table, we create the sockets and then bind the sockets to the same address and port (SO_REUSEADDR and SO_REUSEPORT are set). When timing how long a bind on the port takes, we bind on a different address without SO_REUSEPORT set. We do not set SO_REUSEPORT because we are interested in the case where the bind request does not go through the tb->fastreuseport path, which is fragile (eg tb->fastreuseport path does not work if binding with a different uid).
To run the test locally, I did:
* ulimit -n 65535000
* ip addr add 2001:0db8:0:f101::1 dev eth0
* ./bind_bhash_test 443
--- tools/testing/selftests/net/.gitignore | 1 + tools/testing/selftests/net/Makefile | 2 + tools/testing/selftests/net/bind_bhash_test.c | 119 ++++++++++++++++++ 3 files changed, 122 insertions(+) create mode 100644 tools/testing/selftests/net/bind_bhash_test.c
tools/testing/selftests/net/.gitignore +1 -0
@@ -36,3 +36,4 @@ gro ioam6_parser toeplitz cmsg_sender +bind_bhash_test
tools/testing/selftests/net/Makefile +2 -0
@@ -59,6 +59,7 @@ TEST_GEN_FILES += toeplitz TEST_GEN_FILES += cmsg_sender TEST_GEN_FILES += stress_reuseport_listen TEST_PROGS += test_vxlan_vnifiltering.sh +TEST_GEN_FILES += bind_bhash_test TEST_FILES := settings
@@ -69,4 +70,5 @@ include bpf/Makefile $(OUTPUT)/reuseport_bpf_numa: LDLIBS += -lnuma $(OUTPUT)/tcp_mmap: LDLIBS += -lpthread +$(OUTPUT)/bind_bhash_test: LDLIBS += -lpthread $(OUTPUT)/tcp_inq: LDLIBS += -lpthread
tools/testing/selftests/net/bind_bhash_test.c new file +119 -0
119 lines not on this page · raw message
Signature · 1 line
2.34.3
This series was applied to netdev/net-next.git (master) by Jakub Kicinski <[email protected]>:
Hello:
This series was applied to netdev/net-next.git (master) by Jakub Kicinski <[email protected]>:
Quoted 10 lines · On Thu, 19 May 2022 17:18:32 -0700 you wrote:
Joanne Koong says:
This patchset proposes adding a bhash2 table that hashes by port and address. The motivation behind bhash2 is to expedite bind requests in situations where the port has many sockets in its bhash table entry, which makes checking bind conflicts costly especially given that we acquire the table entry spinlock while doing so, which can cause softirq cpu lockups and can prevent new tcp connections.
[...]
Here is the summary with links:
- [net-next,v5,resend,1/2] net: Add a second bind table hashed by port and address
https://git.kernel.org/netdev/net-next/c/d5a42de8bdbe - [net-next,v5,resend,2/2] selftests: Add test for timing a bind request to a port with a populated bhash entry https://git.kernel.org/netdev/net-next/c/538aaf9b2383
You are awesome, thank you!
Signature · 2 lines
Deet-doot-dot, I am a bot. https://korg.docs.kernel.org/patchwork/pwbot.html