bhash2 binding table fixups
Subsystems (5)
v3 cover bhash2 binding table fixups
There are two fix-ups related to the bhash2 binding table in this patchset:
1) Fixes an inconsistency in the bhash2 table in the case where a socket's rcv saddr changes after it is binded.
2) Fixes the case where the bhash2 hashbucket is accessed concurrently by sockets that hash to different bhash hashbuckets. To address this, the bhash2 hashbucket has its own lock.
--
v2 -> v3:
v2:
https://lore.kernel.org/netdev/[email protected]/
* Add a patch that adds bhash2 hashbucket locks
v1 -> v2:
v1:
https://lore.kernel.org/netdev/[email protected]/
* Mark __inet_bhash2_update_saddr as static
Joanne Koong (3):
net: Update bhash2 when socket's rcv saddr changes
net: Add bhash2 hashbucket locks selftests/net: Add sk_bind_sendto_listen test
include/net/inet_hashtables.h | 31 +++-- include/net/ipv6.h | 2 +- net/dccp/ipv4.c | 10 +- net/dccp/ipv6.c | 4 +- net/dccp/proto.c | 3 +- net/ipv4/af_inet.c | 7 +- net/ipv4/inet_connection_sock.c | 60 +++++---- net/ipv4/inet_hashtables.c | 115 +++++++++++++----- net/ipv4/tcp.c | 7 +- net/ipv4/tcp_ipv4.c | 8 +- net/ipv6/inet6_hashtables.c | 4 +- net/ipv6/tcp_ipv6.c | 4 +- tools/testing/selftests/net/.gitignore | 1 + tools/testing/selftests/net/Makefile | 1 + .../selftests/net/sk_bind_sendto_listen.c | 82 +++++++++++++ 15 files changed, 251 insertions(+), 88 deletions(-) create mode 100644 tools/testing/selftests/net/sk_bind_sendto_listen.c
Signature · 1 line
2.30.2
PATCH v3 1/3 net: Update bhash2 when socket's rcv saddr changes
Commit d5a42de8bdbe ("net: Add a second bind table hashed by port and address") added a second bind table, bhash2, that hashes by a socket's port and rcv address.
However, there are two cases where the socket's rcv saddr can change after it has been binded:
1) The case where there is a bind() call on "::" (IPADDR_ANY) and then a connect() call. The kernel will assign the socket an address when it handles the connect()
2) In inet_sk_reselect_saddr(), which is called when rerouting fails when rebuilding the sk header (invoked by inet_sk_rebuild_header)
In these two cases, we need to update the bhash2 table by removing the entry for the old address, and adding a new entry reflecting the updated address.
--- include/net/inet_hashtables.h | 6 ++- include/net/ipv6.h | 2 +- net/dccp/ipv4.c | 10 +++-- net/dccp/ipv6.c | 4 +- net/ipv4/af_inet.c | 7 +++- net/ipv4/inet_hashtables.c | 70 ++++++++++++++++++++++++++++++++--- net/ipv4/tcp_ipv4.c | 8 +++- net/ipv6/inet6_hashtables.c | 4 +- net/ipv6/tcp_ipv6.c | 4 +- 9 files changed, 97 insertions(+), 18 deletions(-)
include/net/inet_hashtables.h +4 -2
@@ -448,11 +448,13 @@ static inline void sk_rcv_saddr_set(struct sock *sk, __be32 addr) }int __inet_hash_connect (struct inet_timewait_death_row *death_row, -struct sock *sk,u64 port_offset, +struct sock *sk,u64 port_offset,bool prev_inaddr_any,int (*check_established)(struct inet_timewait_death_row *,struct sock *,__u16 ,struct inet_timewait_sock **));int inet_hash_connect (struct inet_timewait_death_row *death_row, -struct sock *sk); +struct sock *sk,bool prev_inaddr_any); + +int inet_bhash2_update_saddr (struct sock *sk);#endif /* _INET_HASHTABLES_H */
include/net/ipv6.h +1 -1
@@ -1187,7 +1187,7 @@ int inet6_compat_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg);int inet6_hash_connect (struct inet_timewait_death_row *death_row, -struct sock *sk); +struct sock *sk,bool prev_inaddr_any);int inet6_sendmsg (struct socket *sock,struct msghdr *msg,size_t size);int inet6_recvmsg (struct socket *sock,struct msghdr *msg,size_t size,int flags);
net/dccp/ipv4.c +7 -3
@@ -47,12 +47,13 @@ int dccp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)const struct sockaddr_in *usin = (struct sockaddr_in *)uaddr;struct inet_sock *inet =inet_sk (sk);struct dccp_sock *dp =dccp_sk (sk); +struct ip_options_rcu *inet_opt;__be16 orig_sport, orig_dport; +bool prev_inaddr_any = false;__be32 daddr, nexthop;struct flowi4 *fl4;struct rtable *rt;int err; -struct ip_options_rcu *inet_opt; dp->dccps_role =DCCP_ROLE_CLIENT ;
@@ -89,8 +90,11 @@ int dccp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)if (inet_opt ==NULL || !inet_opt->opt.srr) daddr = fl4->daddr; -if (inet->inet_saddr ==0 ) +if (inet->inet_saddr ==0 ) { inet->inet_saddr = fl4->saddr; + prev_inaddr_any = true; + } +sk_rcv_saddr_set (sk, inet->inet_saddr); inet->inet_dport = usin->sin_port;sk_daddr_set (sk, daddr);
@@ -105,7 +109,7 @@ int dccp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)* complete initialization after this. */ dccp_set_state (sk,DCCP_REQUESTING ); - err =inet_hash_connect (&dccp_death_row, sk); + err =inet_hash_connect (&dccp_death_row, sk, prev_inaddr_any);if (err !=0 ) goto failure;
net/dccp/ipv6.c +3 -1
@@ -824,6 +824,7 @@ static int dccp_v6_connect(struct sock *sk, struct sockaddr *uaddr,struct ipv6_pinfo *np =inet6_sk (sk);struct dccp_sock *dp =dccp_sk (sk);struct in6_addr *saddr =NULL , *final_p, final; +bool prev_inaddr_any = false;struct ipv6_txoptions *opt;struct flowi6 fl6;struct dst_entry *dst;
@@ -936,6 +937,7 @@ static int dccp_v6_connect(struct sock *sk, struct sockaddr *uaddr,if (saddr ==NULL ) { saddr = &fl6.saddr; sk->sk_v6_rcv_saddr = *saddr; + prev_inaddr_any = true; }/* set the source address */
@@ -951,7 +953,7 @@ static int dccp_v6_connect(struct sock *sk, struct sockaddr *uaddr, inet->inet_dport = usin->sin6_port;dccp_set_state (sk,DCCP_REQUESTING ); - err =inet6_hash_connect (&dccp_death_row, sk); + err =inet6_hash_connect (&dccp_death_row, sk, prev_inaddr_any);if (err) goto late_failure;
net/ipv4/af_inet.c +6 -1
@@ -1221,10 +1221,11 @@ static int inet_sk_reselect_saddr(struct sock *sk)struct inet_sock *inet =inet_sk (sk);__be32 old_saddr = inet->inet_saddr;__be32 daddr = inet->inet_daddr; +struct ip_options_rcu *inet_opt;struct flowi4 *fl4;struct rtable *rt;__be32 new_saddr; -struct ip_options_rcu *inet_opt; +int err; inet_opt =rcu_dereference_protected (inet->inet_opt,lockdep_sock_is_held (sk));
@@ -1253,6 +1254,10 @@ static int inet_sk_reselect_saddr(struct sock *sk) inet->inet_saddr = inet->inet_rcv_saddr = new_saddr; + err =inet_bhash2_update_saddr (sk); +if (err) +return err; +/* * XXX The only one ugly spot where we need to * XXX really change the sockets identity after
net/ipv4/inet_hashtables.c +65 -5
110 lines not on this page · raw message
net/ipv4/tcp_ipv4.c +6 -2
@@ -203,6 +203,7 @@ int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)struct inet_sock *inet =inet_sk (sk);struct tcp_sock *tp =tcp_sk (sk);__be16 orig_sport, orig_dport; +bool prev_inaddr_any = false;__be32 daddr, nexthop;struct flowi4 *fl4;struct rtable *rt;
@@ -246,8 +247,11 @@ int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)if (!inet_opt || !inet_opt->opt.srr) daddr = fl4->daddr; -if (!inet->inet_saddr) +if (!inet->inet_saddr) { inet->inet_saddr = fl4->saddr; + prev_inaddr_any = true; + } +sk_rcv_saddr_set (sk, inet->inet_saddr);if (tp->rx_opt.ts_recent_stamp && inet->inet_daddr != daddr) {
@@ -273,7 +277,7 @@ int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)* complete initialization after this. */ tcp_set_state (sk,TCP_SYN_SENT ); - err =inet_hash_connect (tcp_death_row, sk); + err =inet_hash_connect (tcp_death_row, sk, prev_inaddr_any);if (err) goto failure;
net/ipv6/inet6_hashtables.c +2 -2
@@ -317,13 +317,13 @@ static u64 inet6_sk_port_offset(const struct sock *sk) }int inet6_hash_connect (struct inet_timewait_death_row *death_row, -struct sock *sk) +struct sock *sk,bool prev_inaddr_any) {u64 port_offset =0 ;if (!inet_sk (sk)->inet_num) port_offset =inet6_sk_port_offset (sk); -return __inet_hash_connect (death_row, sk, port_offset, +return __inet_hash_connect (death_row, sk, port_offset, prev_inaddr_any, __inet6_check_established); }EXPORT_SYMBOL_GPL (inet6_hash_connect);
net/ipv6/tcp_ipv6.c +3 -1
@@ -152,6 +152,7 @@ static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr,struct ipv6_pinfo *np =tcp_inet6_sk (sk);struct tcp_sock *tp =tcp_sk (sk);struct in6_addr *saddr =NULL , *final_p, final; +bool prev_inaddr_any = false;struct ipv6_txoptions *opt;struct flowi6 fl6;struct dst_entry *dst;
@@ -289,6 +290,7 @@ static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr,if (!saddr) { saddr = &fl6.saddr; sk->sk_v6_rcv_saddr = *saddr; + prev_inaddr_any = true; }/* set the source address */
@@ -309,7 +311,7 @@ static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr,tcp_set_state (sk,TCP_SYN_SENT ); tcp_death_row =sock_net (sk)->ipv4.tcp_death_row; - err =inet6_hash_connect (tcp_death_row, sk); + err =inet6_hash_connect (tcp_death_row, sk, prev_inaddr_any);if (err) goto late_failure;
Signature · 1 line
2.30.2
Re: [PATCH net-next v3 1/3] net: Update bhash2 when socket's rcv saddr changes
From: Joanne Koong <[email protected]> Date: Fri, 10 Jun 2022 19:16:44 -0700
Quoted 204 lines
Commit
d5a42de8bdbe("net: Add a second bind table hashed by port and address") added a second bind table, bhash2, that hashes by a socket's port and rcv address.However, there are two cases where the socket's rcv saddr can change after it has been binded:
1) The case where there is a
bind()call on "::" (IPADDR_ANY) and then aconnect()call. The kernel will assign the socket an address when it handles theconnect()2) In
inet_sk_reselect_saddr(), which is called when rerouting fails when rebuilding the sk header (invoked byinet_sk_rebuild_header)In these two cases, we need to update the bhash2 table by removing the entry for the old address, and adding a new entry reflecting the updated address.
Reported-by [email protected]Fixes d5a42de8bdbe ("net: Add a second bind table hashed by port and address")Signed-off-by Joanne Koong <[email protected]>Reviewed-by Eric Dumazet <[email protected]>--- include/net/inet_hashtables.h | 6 ++- include/net/ipv6.h | 2 +- net/dccp/ipv4.c | 10 +++-- net/dccp/ipv6.c | 4 +- net/ipv4/af_inet.c | 7 +++- net/ipv4/inet_hashtables.c | 70 ++++++++++++++++++++++++++++++++--- net/ipv4/tcp_ipv4.c | 8 +++- net/ipv6/inet6_hashtables.c | 4 +- net/ipv6/tcp_ipv6.c | 4 +- 9 files changed, 97 insertions(+), 18 deletions(-)include/net/inet_hashtables.h +4
-2@@ -448,11 +448,13 @@ static inline void sk_rcv_saddr_set(struct sock *sk, __be32 addr) }int __inet_hash_connect (struct inet_timewait_death_row *death_row, -struct sock *sk,u64 port_offset, +struct sock *sk,u64 port_offset,bool prev_inaddr_any,int (*check_established)(struct inet_timewait_death_row *,struct sock *,__u16 ,struct inet_timewait_sock **));int inet_hash_connect (struct inet_timewait_death_row *death_row, -struct sock *sk); +struct sock *sk,bool prev_inaddr_any); + +int inet_bhash2_update_saddr (struct sock *sk);#endif /* _INET_HASHTABLES_H */ include/net/ipv6.h +1
-1@@ -1187,7 +1187,7 @@ int inet6_compat_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg);int inet6_hash_connect (struct inet_timewait_death_row *death_row, -struct sock *sk); +struct sock *sk,bool prev_inaddr_any);int inet6_sendmsg (struct socket *sock,struct msghdr *msg,size_t size);int inet6_recvmsg (struct socket *sock,struct msghdr *msg,size_t size,int flags);net/dccp/ipv4.c +7
-3@@ -47,12 +47,13 @@ int dccp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)const struct sockaddr_in *usin = (struct sockaddr_in *)uaddr;struct inet_sock *inet =inet_sk (sk);struct dccp_sock *dp =dccp_sk (sk); +struct ip_options_rcu *inet_opt;__be16 orig_sport, orig_dport; +bool prev_inaddr_any = false;__be32 daddr, nexthop;struct flowi4 *fl4;struct rtable *rt;int err; -struct ip_options_rcu *inet_opt; dp->dccps_role =DCCP_ROLE_CLIENT ;@@ -89,8 +90,11 @@ int dccp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)if (inet_opt ==NULL || !inet_opt->opt.srr) daddr = fl4->daddr; -if (inet->inet_saddr ==0 ) +if (inet->inet_saddr ==0 ) { inet->inet_saddr = fl4->saddr; + prev_inaddr_any = true; + } +sk_rcv_saddr_set (sk, inet->inet_saddr); inet->inet_dport = usin->sin_port;sk_daddr_set (sk, daddr);@@ -105,7 +109,7 @@ int dccp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)* complete initialization after this. */ dccp_set_state (sk,DCCP_REQUESTING ); - err =inet_hash_connect (&dccp_death_row, sk); + err =inet_hash_connect (&dccp_death_row, sk, prev_inaddr_any);if (err !=0 ) goto failure;net/dccp/ipv6.c +3
-1@@ -824,6 +824,7 @@ static int dccp_v6_connect(struct sock *sk, struct sockaddr *uaddr,struct ipv6_pinfo *np =inet6_sk (sk);struct dccp_sock *dp =dccp_sk (sk);struct in6_addr *saddr =NULL , *final_p, final; +bool prev_inaddr_any = false;struct ipv6_txoptions *opt;struct flowi6 fl6;struct dst_entry *dst;@@ -936,6 +937,7 @@ static int dccp_v6_connect(struct sock *sk, struct sockaddr *uaddr,if (saddr ==NULL ) { saddr = &fl6.saddr; sk->sk_v6_rcv_saddr = *saddr; + prev_inaddr_any = true; }/* set the source address */ @@ -951,7 +953,7 @@ static int dccp_v6_connect(struct sock *sk, struct sockaddr *uaddr, inet->inet_dport = usin->sin6_port;dccp_set_state (sk,DCCP_REQUESTING ); - err =inet6_hash_connect (&dccp_death_row, sk); + err =inet6_hash_connect (&dccp_death_row, sk, prev_inaddr_any);if (err) goto late_failure;net/ipv4/af_inet.c +6
-1@@ -1221,10 +1221,11 @@ static int inet_sk_reselect_saddr(struct sock *sk)struct inet_sock *inet =inet_sk (sk);__be32 old_saddr = inet->inet_saddr;__be32 daddr = inet->inet_daddr; +struct ip_options_rcu *inet_opt;struct flowi4 *fl4;struct rtable *rt;__be32 new_saddr; -struct ip_options_rcu *inet_opt; +int err; inet_opt =rcu_dereference_protected (inet->inet_opt,lockdep_sock_is_held (sk));@@ -1253,6 +1254,10 @@ static int inet_sk_reselect_saddr(struct sock *sk) inet->inet_saddr = inet->inet_rcv_saddr = new_saddr; + err =inet_bhash2_update_saddr (sk); +if (err) +return err; +/* * XXX The only one ugly spot where we need to * XXX really change the sockets identity after net/ipv4/inet_hashtables.c +31
-0@@ -826,6 +826,55 @@ inet_bind2_bucket_find(struct inet_hashinfo *hinfo, struct net *net,return bhash2; } +/* the lock for the socket's corresponding bhash entry must be held */ +static int __inet_bhash2_update_saddr (struct sock *sk, +struct inet_hashinfo *hinfo, +struct net *net,int port,int l3mdev) +{ +struct inet_bind2_hashbucket *head2; +struct inet_bind2_bucket *tb2; + + tb2 =inet_bind2_bucket_find (hinfo, net, port, l3mdev, sk, + &head2); +if (!tb2) { + tb2 =inet_bind2_bucket_create (hinfo->bind2_bucket_cachep, + net, head2, port, l3mdev, sk); +if (!tb2) +return -ENOMEM ; + } + +/* Remove the socket's old entry from bhash2 */ +__sk_del_bind2_node (sk); + +sk_add_bind2_node (sk, &tb2->owners); +inet_csk (sk)->icsk_bind2_hash = tb2; + +return 0 ; +} + +/* This should be called if/when a socket's rcv saddr changes after it has +* been binded. +*/ +int inet_bhash2_update_saddr (struct sock *sk) +{
net/ipv4/inet_hashtables.c inet_bind2_bucket_find(struct inet_hashinfo *hinfo, struct net *net, · line 869
Quoted 10 lines
net/ipv4/inet_hashtables.c +10
-0+struct inet_hashinfo *hinfo = sk->sk_prot->h.hashinfo; +int l3mdev =inet_sk_bound_l3mdev (sk); +struct inet_bind_hashbucket *head; +int port =inet_sk (sk)->inet_num; +struct net *net =sock_net (sk); +int err; + + head = &hinfo->bhash[inet_bhashfn (net, port, hinfo->bhash_size)]; + +spin_lock_bh (&head->lock);
I think this patch should be 2nd one in the series because we know this lock does not protect bhash2 and it makes diff smaller.
Also, the series should be applied to net instead of net-next.
Quoted 21 lines
net/ipv4/inet_hashtables.c +9
-1+ + err =__inet_bhash2_update_saddr (sk, hinfo, net, port, l3mdev); + +spin_unlock_bh (&head->lock); + +return err; +} +/* RFC 6056 3.3.4. Algorithm 4: Double-Hash Port Selection Algorithm * Note that we use 32bit integers (vs RFC 'short integers') * because 2^16 is not a multiple of num_ephemeral and this @@ -840,7 +889,7 @@ inet_bind2_bucket_find(struct inet_hashinfo *hinfo, struct net *net,static u32 *table_perturb;int __inet_hash_connect (struct inet_timewait_death_row *death_row, -struct sock *sk,u64 port_offset, +struct sock *sk,u64 port_offset,bool prev_inaddr_any,int (*check_established)(struct inet_timewait_death_row *,struct sock *,__u16 ,struct inet_timewait_sock **)) {
net/ipv4/inet_hashtables.c int __inet_hash_connect(struct inet_timewait_death_row *death_row, · line 910
Quoted 4 lines
net/ipv4/inet_hashtables.c +1
-0@@ -858,11 +907,24 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row,int l3mdev;u32 index; + l3mdev =inet_sk_bound_l3mdev (sk);
This add/remove dance can be saved.
Quoted 130 lines
net/ipv4/inet_hashtables.c +14
-4+if (port) { head = &hinfo->bhash[inet_bhashfn (net, port, hinfo->bhash_size)]; tb =inet_csk (sk)->icsk_bind_hash; +spin_lock_bh (&head->lock); + +if (prev_inaddr_any) { + ret =__inet_bhash2_update_saddr (sk, hinfo, net, port, + l3mdev); +if (ret) { +spin_unlock_bh (&head->lock); +return ret; + } + } +if (sk_head (&tb->owners) == sk && !sk->sk_bind_node.next) {inet_ehash_nolisten (sk,NULL ,NULL );spin_unlock_bh (&head->lock);@@ -875,8 +937,6 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row,return ret; } - l3mdev =inet_sk_bound_l3mdev (sk); -inet_get_local_port_range (net, &low, &high); high++;/* [32768, 60999] -> [32768, 61000[ */ remaining = high - low;@@ -987,13 +1047,13 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row,* Bind a port for a connect operation and hash it. */ int inet_hash_connect (struct inet_timewait_death_row *death_row, -struct sock *sk) +struct sock *sk,bool prev_inaddr_any) {u64 port_offset =0 ;if (!inet_sk (sk)->inet_num) port_offset =inet_sk_port_offset (sk); -return __inet_hash_connect (death_row, sk, port_offset, +return __inet_hash_connect (death_row, sk, port_offset, prev_inaddr_any, __inet_check_established); }EXPORT_SYMBOL_GPL (inet_hash_connect);net/ipv4/tcp_ipv4.c +6
-2@@ -203,6 +203,7 @@ int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)struct inet_sock *inet =inet_sk (sk);struct tcp_sock *tp =tcp_sk (sk);__be16 orig_sport, orig_dport; +bool prev_inaddr_any = false;__be32 daddr, nexthop;struct flowi4 *fl4;struct rtable *rt;@@ -246,8 +247,11 @@ int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)if (!inet_opt || !inet_opt->opt.srr) daddr = fl4->daddr; -if (!inet->inet_saddr) +if (!inet->inet_saddr) { inet->inet_saddr = fl4->saddr; + prev_inaddr_any = true; + } +sk_rcv_saddr_set (sk, inet->inet_saddr);if (tp->rx_opt.ts_recent_stamp && inet->inet_daddr != daddr) {@@ -273,7 +277,7 @@ int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)* complete initialization after this. */ tcp_set_state (sk,TCP_SYN_SENT ); - err =inet_hash_connect (tcp_death_row, sk); + err =inet_hash_connect (tcp_death_row, sk, prev_inaddr_any);if (err) goto failure;net/ipv6/inet6_hashtables.c +2
-2@@ -317,13 +317,13 @@ static u64 inet6_sk_port_offset(const struct sock *sk) }int inet6_hash_connect (struct inet_timewait_death_row *death_row, -struct sock *sk) +struct sock *sk,bool prev_inaddr_any) {u64 port_offset =0 ;if (!inet_sk (sk)->inet_num) port_offset =inet6_sk_port_offset (sk); -return __inet_hash_connect (death_row, sk, port_offset, +return __inet_hash_connect (death_row, sk, port_offset, prev_inaddr_any, __inet6_check_established); }EXPORT_SYMBOL_GPL (inet6_hash_connect);net/ipv6/tcp_ipv6.c +3
-1@@ -152,6 +152,7 @@ static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr,struct ipv6_pinfo *np =tcp_inet6_sk (sk);struct tcp_sock *tp =tcp_sk (sk);struct in6_addr *saddr =NULL , *final_p, final; +bool prev_inaddr_any = false;struct ipv6_txoptions *opt;struct flowi6 fl6;struct dst_entry *dst;@@ -289,6 +290,7 @@ static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr,if (!saddr) { saddr = &fl6.saddr; sk->sk_v6_rcv_saddr = *saddr; + prev_inaddr_any = true; }/* set the source address */ @@ -309,7 +311,7 @@ static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr,tcp_set_state (sk,TCP_SYN_SENT ); tcp_death_row =sock_net (sk)->ipv4.tcp_death_row; - err =inet6_hash_connect (tcp_death_row, sk); + err =inet6_hash_connect (tcp_death_row, sk, prev_inaddr_any);if (err) goto late_failure;Signature · 1 line
2.30.2
PATCH v3 2/3 net: Add bhash2 hashbucket locks
In reply to Joanne KoongCurrently, the bhash2 hashbucket uses its corresponding bhash hashbucket's lock for serializing concurrent accesses. There, however, can be the case where the bhash2 hashbucket is accessed concurrently by multiple processes that hash to different bhash hashbuckets but to the same bhash2 hashbucket.
As such, each bhash2 hashbucket will need to have its own lock instead of using its corresponding bhash hashbucket's lock.
--- include/net/inet_hashtables.h | 25 +++---- net/dccp/proto.c | 3 +- net/ipv4/inet_connection_sock.c | 60 +++++++++------- net/ipv4/inet_hashtables.c | 119 +++++++++++++++----------------- net/ipv4/tcp.c | 7 +- 5 files changed, 107 insertions(+), 107 deletions(-)
include/net/inet_hashtables.h +10 -15
@@ -124,15 +124,6 @@ struct inet_bind_hashbucket {struct hlist_head chain; }; -/* This is synchronized using the inet_bind_hashbucket's spinlock. -* Instead of having separate spinlocks, the inet_bind2_hashbucket can share -* the inet_bind_hashbucket's given that in every case where the bhash2 table -* is useful, a lookup in the bhash table also occurs. -*/ -struct inet_bind2_hashbucket { -struct hlist_head chain; -}; -/* Sockets can be hashed in established or listening table. * We must use different 'nulls' end-of-chain value for all hash buckets : * A socket might transition from ESTABLISH to LISTEN state without
@@ -169,7 +160,7 @@ struct inet_hashinfo {* conflicts. */ struct kmem_cache *bind2_bucket_cachep; -struct inet_bind2_hashbucket *bhash2; +struct inet_bind_hashbucket *bhash2;unsigned int bhash_size;/* The 2nd listener table hashed by local port and address */
@@ -240,7 +231,7 @@ static inline bool check_bind_bucket_match(struct inet_bind_bucket *tb,struct inet_bind2_bucket *inet_bind2_bucket_create (struct kmem_cache *cachep,struct net *net, -struct inet_bind2_hashbucket *head, +struct inet_bind_hashbucket *head,const unsigned short port,int l3mdev,const struct sock *sk);
@@ -248,12 +239,12 @@ void inet_bind2_bucket_destroy(struct kmem_cache *cachep, struct inet_bind2_bucket *tb);struct inet_bind2_bucket * -inet_bind2_bucket_find (struct inet_hashinfo *hinfo,struct net *net, +inet_bind2_bucket_find (struct inet_bind_hashbucket *head, +struct inet_hashinfo *hinfo,struct net *net,const unsigned short port,int l3mdev, -struct sock *sk, -struct inet_bind2_hashbucket **head); +struct sock *sk); -bool check_bind2_bucket_match_nulladdr(struct inet_bind2_bucket *tb, +bool check_bind2_bucket_match_addr_any(struct inet_bind2_bucket *tb, struct net *net, const unsigned short port, int l3mdev,
@@ -265,6 +256,10 @@ static inline u32 inet_bhashfn(const struct net *net, const __u16 lport,return (lport +net_hash_mix (net)) & (bhash_size -1 ); } +struct inet_bind_hashbucket * +inet_bhashfn_portaddr (struct inet_hashinfo *hinfo,const struct sock *sk, +const struct net *net,unsigned short port); +void inet_bind_hash (struct sock *sk,struct inet_bind_bucket *tb,struct inet_bind2_bucket *tb2,const unsigned short snum);
net/dccp/proto.c +2 -1
@@ -1182,7 +1182,7 @@ static int __init dccp_init(void) goto out_free_dccp_locks; } - dccp_hashinfo.bhash2 = (struct inet_bind2_hashbucket *) + dccp_hashinfo.bhash2 = (struct inet_bind_hashbucket *)__get_free_pages (GFP_ATOMIC | __GFP_NOWARN, bhash_order);if (!dccp_hashinfo.bhash2) {
@@ -1193,6 +1193,7 @@ static int __init dccp_init(void)for (i =0 ; i < dccp_hashinfo.bhash_size; i++) {spin_lock_init (&dccp_hashinfo.bhash[i].lock);INIT_HLIST_HEAD (&dccp_hashinfo.bhash[i].chain); +spin_lock_init (&dccp_hashinfo.bhash2[i].lock);INIT_HLIST_HEAD (&dccp_hashinfo.bhash2[i].chain); }
net/ipv4/inet_connection_sock.c +35 -25
155 lines not on this page · raw message
net/ipv4/inet_hashtables.c +57 -62
282 lines not on this page · raw message
net/ipv4/tcp.c +3 -4
@@ -4633,8 +4633,7 @@ void __init tcp_init(void)panic ("TCP: failed to alloc ehash_locks" ); tcp_hashinfo.bhash = alloc_large_system_hash("TCP bind bhash tables", - sizeof(struct inet_bind_hashbucket) + - sizeof(struct inet_bind2_hashbucket), + 2 * sizeof(struct inet_bind_hashbucket), tcp_hashinfo.ehash_mask + 1, 17, /* one slot per 128 KB of memory */ 0,
@@ -4643,11 +4642,11 @@ void __init tcp_init(void)0 , 64 * 1024); tcp_hashinfo.bhash_size =1U << tcp_hashinfo.bhash_size; - tcp_hashinfo.bhash2 = - (struct inet_bind2_hashbucket *)(tcp_hashinfo.bhash + tcp_hashinfo.bhash_size); + tcp_hashinfo.bhash2 = tcp_hashinfo.bhash + tcp_hashinfo.bhash_size;for (i =0 ; i < tcp_hashinfo.bhash_size; i++) {spin_lock_init (&tcp_hashinfo.bhash[i].lock);INIT_HLIST_HEAD (&tcp_hashinfo.bhash[i].chain); +spin_lock_init (&tcp_hashinfo.bhash2[i].lock);INIT_HLIST_HEAD (&tcp_hashinfo.bhash2[i].chain); }
Signature · 1 line
2.30.2
Re: [PATCH net-next v3 2/3] net: Add bhash2 hashbucket locks
Quoted 157 lines · Joanne Koong
Currently, the bhash2 hashbucket uses its corresponding bhash hashbucket's lock for serializing concurrent accesses. There, however, can be the case where the bhash2 hashbucket is accessed concurrently by multiple processes that hash to different bhash hashbuckets but to the same bhash2 hashbucket.
As such, each bhash2 hashbucket will need to have its own lock instead of using its corresponding bhash hashbucket's lock.
Fixes d5a42de8bdbe ("net: Add a second bind table hashed by port and address")Signed-off-by Joanne Koong <[email protected]>--- include/net/inet_hashtables.h | 25 +++---- net/dccp/proto.c | 3 +- net/ipv4/inet_connection_sock.c | 60 +++++++++------- net/ipv4/inet_hashtables.c | 119 +++++++++++++++----------------- net/ipv4/tcp.c | 7 +- 5 files changed, 107 insertions(+), 107 deletions(-)include/net/inet_hashtables.h +10
-15@@ -124,15 +124,6 @@ struct inet_bind_hashbucket {struct hlist_head chain; }; -/* This is synchronized using the inet_bind_hashbucket's spinlock. -* Instead of having separate spinlocks, the inet_bind2_hashbucket can share -* the inet_bind_hashbucket's given that in every case where the bhash2 table -* is useful, a lookup in the bhash table also occurs. -*/ -struct inet_bind2_hashbucket { -struct hlist_head chain; -}; -/* Sockets can be hashed in established or listening table. * We must use different 'nulls' end-of-chain value for all hash buckets : * A socket might transition from ESTABLISH to LISTEN state without @@ -169,7 +160,7 @@ struct inet_hashinfo {* conflicts. */ struct kmem_cache *bind2_bucket_cachep; -struct inet_bind2_hashbucket *bhash2; +struct inet_bind_hashbucket *bhash2;unsigned int bhash_size;/* The 2nd listener table hashed by local port and address */ @@ -240,7 +231,7 @@ static inline bool check_bind_bucket_match(struct inet_bind_bucket *tb,struct inet_bind2_bucket *inet_bind2_bucket_create (struct kmem_cache *cachep,struct net *net, -struct inet_bind2_hashbucket *head, +struct inet_bind_hashbucket *head,const unsigned short port,int l3mdev,const struct sock *sk);@@ -248,12 +239,12 @@ void inet_bind2_bucket_destroy(struct kmem_cache *cachep, struct inet_bind2_bucket *tb);struct inet_bind2_bucket * -inet_bind2_bucket_find (struct inet_hashinfo *hinfo,struct net *net, +inet_bind2_bucket_find (struct inet_bind_hashbucket *head, +struct inet_hashinfo *hinfo,struct net *net,const unsigned short port,int l3mdev, -struct sock *sk, -struct inet_bind2_hashbucket **head); +struct sock *sk); -bool check_bind2_bucket_match_nulladdr(struct inet_bind2_bucket *tb, +bool check_bind2_bucket_match_addr_any(struct inet_bind2_bucket *tb, struct net *net, const unsigned short port, int l3mdev,@@ -265,6 +256,10 @@ static inline u32 inet_bhashfn(const struct net *net, const __u16 lport,return (lport +net_hash_mix (net)) & (bhash_size -1 ); } +struct inet_bind_hashbucket * +inet_bhashfn_portaddr (struct inet_hashinfo *hinfo,const struct sock *sk, +const struct net *net,unsigned short port); +void inet_bind_hash (struct sock *sk,struct inet_bind_bucket *tb,struct inet_bind2_bucket *tb2,const unsigned short snum);net/dccp/proto.c +2
-1@@ -1182,7 +1182,7 @@ static int __init dccp_init(void) goto out_free_dccp_locks; } - dccp_hashinfo.bhash2 = (struct inet_bind2_hashbucket *) + dccp_hashinfo.bhash2 = (struct inet_bind_hashbucket *)__get_free_pages (GFP_ATOMIC | __GFP_NOWARN, bhash_order);if (!dccp_hashinfo.bhash2) {@@ -1193,6 +1193,7 @@ static int __init dccp_init(void)for (i =0 ; i < dccp_hashinfo.bhash_size; i++) {spin_lock_init (&dccp_hashinfo.bhash[i].lock);INIT_HLIST_HEAD (&dccp_hashinfo.bhash[i].chain); +spin_lock_init (&dccp_hashinfo.bhash2[i].lock);INIT_HLIST_HEAD (&dccp_hashinfo.bhash2[i].chain); }net/ipv4/inet_connection_sock.c +7
-8@@ -131,14 +131,14 @@ static bool use_bhash2_on_bind(const struct sock *sk)return sk->sk_rcv_saddr !=htonl (INADDR_ANY ); } -static u32 get_bhash2_nulladdr_hash (const struct sock *sk,struct net *net, +static u32 get_bhash2_addr_any_hash (const struct sock *sk,struct net *net,int port) {#if IS_ENABLED (CONFIG_IPV6 ) -struct in6_addr nulladdr = {}; +struct in6_addr addr_any = {};if (sk->sk_family ==AF_INET6 ) -return ipv6_portaddr_hash (net, &nulladdr, port); +return ipv6_portaddr_hash (net, &addr_any, port);#endif return ipv4_portaddr_hash (net,0 , port); }@@ -204,18 +204,18 @@ static bool check_bhash2_conflict(const struct sock *sk,return false; } -/* This should be called only when the corresponding inet_bind_bucket spinlock -* is held -*/ +/* This should be called only when the tb and tb2 hashbuckets' locks are held */ static int inet_csk_bind_conflict (const struct sock *sk,int port,struct inet_bind_bucket *tb,struct inet_bind2_bucket *tb2,/* may be null */ +struct inet_bind_hashbucket *head_tb2,bool relax,bool reuseport_ok) {struct inet_hashinfo *hinfo = sk->sk_prot->h.hashinfo;kuid_t uid =sock_i_uid ((struct sock *)sk);struct sock_reuseport *reuseport_cb; -struct inet_bind2_hashbucket *head2; +struct inet_bind_hashbucket *head_addr_any; +bool addr_any_conflict = false;bool reuseport_cb_ok;struct sock *sk2;struct net *net;@@ -254,33 +254,39 @@ static int inet_csk_bind_conflict(const struct sock *sk, int port,/* check there's no conflict with an existing IPV6_ADDR_ANY (if ipv6) or * INADDR_ANY (if ipv4) socket. */ - hash =get_bhash2_nulladdr_hash (sk, net, port);
net/ipv4/inet_connection_sock.c static int inet_csk_bind_conflict(const struct sock *sk, int port, · line 263
Quoted 10 lines
net/ipv4/inet_connection_sock.c +5
-3- head2 = &hinfo->bhash2[hash & (hinfo->bhash_size -1 )]; + hash =get_bhash2_addr_any_hash (sk, net, port); + head_addr_any = &hinfo->bhash2[hash & (hinfo->bhash_size -1 )]; l3mdev =inet_sk_bound_l3mdev (sk); -inet_bind_bucket_for_each (tb2, &head2->chain) -if (check_bind2_bucket_match_nulladdr (tb2, net, port, l3mdev, sk)) + +if (head_addr_any != head_tb2) +spin_lock_bh (&head_addr_any->lock);
I do not think you need the _bh. Look at my next comment for the rationale.
Quoted 36 lines
net/ipv4/inet_connection_sock.c +11
-6+ +inet_bind_bucket_for_each (tb2, &head_addr_any->chain) +if (check_bind2_bucket_match_addr_any (tb2, net, port, l3mdev, sk))break ;if (tb2 &&check_bhash2_conflict (sk, tb2, uid, relax, reuseport_cb_ok, reuseport_ok)) -return true; + addr_any_conflict = true; -return false; +if (head_addr_any != head_tb2) +spin_unlock_bh (&head_addr_any->lock); + +return addr_any_conflict; }/* * Find an open port number for the socket. Returns with the -* inet_bind_hashbucket lock held. +* inet_bind_hashbucket locks held if successful. */ static struct inet_bind_hashbucket *inet_csk_find_open_port (struct sock *sk,struct inet_bind_bucket **tb_ret,struct inet_bind2_bucket **tb2_ret, -struct inet_bind2_hashbucket **head2_ret,int *port_ret) +struct inet_bind_hashbucket **head2_ret,int *port_ret) {struct inet_hashinfo *hinfo = sk->sk_prot->h.hashinfo; -struct inet_bind2_hashbucket *head2; -struct inet_bind_hashbucket *head; +struct inet_bind_hashbucket *head, *head2;struct net *net =sock_net (sk);int i, low, high, attempt_half;struct inet_bind2_bucket *tb2;
net/ipv4/inet_connection_sock.c inet_csk_find_open_port(struct sock *sk, struct inet_bind_bucket **tb_ret, · line 336
Quoted 8 lines
net/ipv4/inet_connection_sock.c +2
-2@@ -325,19 +331,22 @@ inet_csk_find_open_port(struct sock *sk, struct inet_bind_bucket **tb_ret,continue ; head = &hinfo->bhash[inet_bhashfn (net, port, hinfo->bhash_size)]; + head2 =inet_bhashfn_portaddr (hinfo, sk, net, port);spin_lock_bh (&head->lock); - tb2 =inet_bind2_bucket_find (hinfo, net, port, l3mdev, sk, - &head2); +spin_lock_bh (&head2->lock);
Note: No need to disable BH twice.
spin_lock_bh(&lock1); spin_lock_bh(&lock2); spin_unlock_bh(&lock2); spin_unlock_bh(&lock1);
Can instead be:
spin_lock_bh(&lock1); spin_lock(&lock2); spin_unlock(&lock2); spin_unlock_bh(&lock1);
Quoted 361 lines
net/ipv4/inet_connection_sock.c +10
-6+ + tb2 =inet_bind2_bucket_find (head2, hinfo, net, port, l3mdev, sk);inet_bind_bucket_for_each (tb, &head->chain)if (check_bind_bucket_match (tb, net, port, l3mdev)) {if (!inet_csk_bind_conflict (sk, port, tb, tb2, - relax, false)) + head2, relax, false)) goto success; goto next_port; } tb =NULL ; goto success; next_port: +spin_unlock_bh (&head2->lock);spin_unlock_bh (&head->lock);cond_resched (); }@@ -459,10 +468,9 @@ int inet_csk_get_port(struct sock *sk, unsigned short snum)bool reuse = sk->sk_reuse && sk->sk_state !=TCP_LISTEN ;struct inet_hashinfo *hinfo = sk->sk_prot->h.hashinfo;bool bhash_created = false, bhash2_created = false; +struct inet_bind_hashbucket *head, *head2;struct inet_bind2_bucket *tb2 =NULL ; -struct inet_bind2_hashbucket *head2;struct inet_bind_bucket *tb =NULL ; -struct inet_bind_hashbucket *head;struct net *net =sock_net (sk);int ret =1 , port = snum;bool found_port = false;@@ -480,13 +488,14 @@ int inet_csk_get_port(struct sock *sk, unsigned short snum) }else { head = &hinfo->bhash[inet_bhashfn (net, port, hinfo->bhash_size)]; + head2 =inet_bhashfn_portaddr (hinfo, sk, net, port);spin_lock_bh (&head->lock);inet_bind_bucket_for_each (tb, &head->chain)if (check_bind_bucket_match (tb, net, port, l3mdev))break ; - tb2 =inet_bind2_bucket_find (hinfo, net, port, l3mdev, sk, - &head2); +spin_lock_bh (&head2->lock); + tb2 =inet_bind2_bucket_find (head2, hinfo, net, port, l3mdev, sk); }if (!tb) {@@ -513,7 +522,7 @@ int inet_csk_get_port(struct sock *sk, unsigned short snum)if ((tb->fastreuse >0 && reuse) ||sk_reuseport_match (tb, sk)) goto success; -if (inet_csk_bind_conflict (sk, port, tb, tb2, true, true)) +if (inet_csk_bind_conflict (sk, port, tb, tb2, head2, true, true)) goto fail_unlock; } success:@@ -533,6 +542,7 @@ int inet_csk_get_port(struct sock *sk, unsigned short snum)inet_bind2_bucket_destroy (hinfo->bind2_bucket_cachep, tb2); } +spin_unlock_bh (&head2->lock);spin_unlock_bh (&head->lock);return ret; }net/ipv4/inet_hashtables.c +56
-62@@ -83,7 +83,7 @@ struct inet_bind_bucket *inet_bind_bucket_create(struct kmem_cache *cachep,struct inet_bind2_bucket *inet_bind2_bucket_create (struct kmem_cache *cachep,struct net *net, -struct inet_bind2_hashbucket *head, +struct inet_bind_hashbucket *head,const unsigned short port,int l3mdev,const struct sock *sk)@@ -127,9 +127,7 @@ void inet_bind_bucket_destroy(struct kmem_cache *cachep, struct inet_bind_bucket } } -/* Caller must hold the lock for the corresponding hashbucket in the bhash table -* with local BH disabled -*/ +/* Caller must hold hashbucket lock for this tb with local BH disabled */ void inet_bind2_bucket_destroy (struct kmem_cache *cachep,struct inet_bind2_bucket *tb) {if (hlist_empty (&tb->owners)) {@@ -157,6 +155,9 @@ static void __inet_put_port(struct sock *sk)const int bhash =inet_bhashfn (sock_net (sk),inet_sk (sk)->inet_num, hashinfo->bhash_size);struct inet_bind_hashbucket *head = &hashinfo->bhash[bhash]; +struct inet_bind_hashbucket *head2 = +inet_bhashfn_portaddr (hashinfo, sk,sock_net (sk), +inet_sk (sk)->inet_num);struct inet_bind2_bucket *tb2;struct inet_bind_bucket *tb;@@ -167,12 +168,15 @@ static void __inet_put_port(struct sock *sk)inet_sk (sk)->inet_num =0 ;inet_bind_bucket_destroy (hashinfo->bind_bucket_cachep, tb); +spin_lock (&head2->lock);if (inet_csk (sk)->icsk_bind2_hash) { tb2 =inet_csk (sk)->icsk_bind2_hash;__sk_del_bind2_node (sk);inet_csk (sk)->icsk_bind2_hash =NULL ;inet_bind2_bucket_destroy (hashinfo->bind2_bucket_cachep, tb2); } +spin_unlock (&head2->lock); +spin_unlock (&head->lock); }@@ -191,7 +195,9 @@ int __inet_inherit_port(const struct sock *sk, struct sock *child)const int bhash =inet_bhashfn (sock_net (sk), port, table->bhash_size);struct inet_bind_hashbucket *head = &table->bhash[bhash]; -struct inet_bind2_hashbucket *head_bhash2; +struct inet_bind_hashbucket *head2 = +inet_bhashfn_portaddr (table, child,sock_net (sk), + port);bool created_inet_bind_bucket = false;struct net *net =sock_net (sk);struct inet_bind2_bucket *tb2;@@ -199,9 +205,11 @@ int __inet_inherit_port(const struct sock *sk, struct sock *child)int l3mdev;spin_lock (&head->lock); +spin_lock (&head2->lock); tb =inet_csk (sk)->icsk_bind_hash; tb2 =inet_csk (sk)->icsk_bind2_hash;if (unlikely (!tb || !tb2)) { +spin_unlock (&head2->lock);spin_unlock (&head->lock);return -ENOENT ; }@@ -221,6 +229,7 @@ int __inet_inherit_port(const struct sock *sk, struct sock *child) tb =inet_bind_bucket_create (table->bind_bucket_cachep, net, head, port, l3mdev);if (!tb) { +spin_unlock (&head2->lock);spin_unlock (&head->lock);return -ENOMEM ; }@@ -233,17 +242,17 @@ int __inet_inherit_port(const struct sock *sk, struct sock *child) l3mdev =inet_sk_bound_l3mdev (sk); bhash2_find: - tb2 =inet_bind2_bucket_find (table, net, port, l3mdev, child, - &head_bhash2); + tb2 =inet_bind2_bucket_find (head2, table, net, port, l3mdev, child);if (!tb2) { tb2 =inet_bind2_bucket_create (table->bind2_bucket_cachep, - net, head_bhash2, port, + net, head2, port, l3mdev, child);if (!tb2) goto error; } }inet_bind_hash (child, tb, tb2, port); +spin_unlock (&head2->lock);spin_unlock (&head->lock);return 0 ;@@ -251,6 +260,7 @@ int __inet_inherit_port(const struct sock *sk, struct sock *child) error:if (created_inet_bind_bucket)inet_bind_bucket_destroy (table->bind_bucket_cachep, tb); +spin_unlock (&head2->lock);spin_unlock (&head->lock);return -ENOMEM ; }@@ -771,24 +781,24 @@ static bool check_bind2_bucket_match(struct inet_bind2_bucket *tb, tb->l3mdev == l3mdev && tb->rcv_saddr == sk->sk_rcv_saddr; } -bool check_bind2_bucket_match_nulladdr (struct inet_bind2_bucket *tb, +bool check_bind2_bucket_match_addr_any (struct inet_bind2_bucket *tb,struct net *net,const unsigned short port,int l3mdev,const struct sock *sk) {#if IS_ENABLED (CONFIG_IPV6 ) -struct in6_addr nulladdr = {}; +struct in6_addr addr_any = {};if (sk->sk_family ==AF_INET6 )return net_eq (ib2_net (tb), net) && tb->port == port && tb->l3mdev == l3mdev && -ipv6_addr_equal (&tb->v6_rcv_saddr, &nulladdr); +ipv6_addr_equal (&tb->v6_rcv_saddr, &addr_any); else#endif return net_eq (ib2_net (tb), net) && tb->port == port && tb->l3mdev == l3mdev && tb->rcv_saddr ==0 ; } -static struct inet_bind2_hashbucket * +struct inet_bind_hashbucket *inet_bhashfn_portaddr (struct inet_hashinfo *hinfo,const struct sock *sk,const struct net *net,unsigned short port) {@@ -803,55 +813,21 @@ inet_bhashfn_portaddr(struct inet_hashinfo *hinfo, const struct sock *sk,return &hinfo->bhash2[hash & (hinfo->bhash_size -1 )]; } -/* This should only be called when the spinlock for the socket's corresponding -* bind_hashbucket is held -*/ +/* The socket's bhash2 hashbucket spinlock must be held when this is called */ struct inet_bind2_bucket * -inet_bind2_bucket_find (struct inet_hashinfo *hinfo,struct net *net, -const unsigned short port,int l3mdev,struct sock *sk, -struct inet_bind2_hashbucket **head) +inet_bind2_bucket_find (struct inet_bind_hashbucket *head, +struct inet_hashinfo *hinfo,struct net *net, +const unsigned short port,int l3mdev,struct sock *sk) {struct inet_bind2_bucket *bhash2 =NULL ; -struct inet_bind2_hashbucket *h; - h =inet_bhashfn_portaddr (hinfo, sk, net, port); -inet_bind_bucket_for_each (bhash2, &h->chain) { +inet_bind_bucket_for_each (bhash2, &head->chain)if (check_bind2_bucket_match (bhash2, net, port, l3mdev, sk))break ; - } - -if (head) - *head = h;return bhash2; } -/* the lock for the socket's corresponding bhash entry must be held */ -static int __inet_bhash2_update_saddr (struct sock *sk, -struct inet_hashinfo *hinfo, -struct net *net,int port,int l3mdev) -{ -struct inet_bind2_hashbucket *head2; -struct inet_bind2_bucket *tb2; - - tb2 =inet_bind2_bucket_find (hinfo, net, port, l3mdev, sk, - &head2); -if (!tb2) { - tb2 =inet_bind2_bucket_create (hinfo->bind2_bucket_cachep, - net, head2, port, l3mdev, sk); -if (!tb2) -return -ENOMEM ; - } - -/* Remove the socket's old entry from bhash2 */ -__sk_del_bind2_node (sk); - -sk_add_bind2_node (sk, &tb2->owners); -inet_csk (sk)->icsk_bind2_hash = tb2; - -return 0 ; -} -/* This should be called if/when a socket's rcv saddr changes after it has * been binded. */ @@ -862,17 +838,31 @@ int inet_bhash2_update_saddr(struct sock *sk)struct inet_bind_hashbucket *head;int port =inet_sk (sk)->inet_num;struct net *net =sock_net (sk); -int err; +struct inet_bind2_bucket *tb2; - head = &hinfo->bhash[inet_bhashfn (net, port, hinfo->bhash_size)]; + head =inet_bhashfn_portaddr (hinfo, sk, net, port);spin_lock_bh (&head->lock); - err =__inet_bhash2_update_saddr (sk, hinfo, net, port, l3mdev); + tb2 =inet_bind2_bucket_find (head, hinfo, net, port, l3mdev, sk); +if (!tb2) { + tb2 =inet_bind2_bucket_create (hinfo->bind2_bucket_cachep, + net, head, port, l3mdev, sk); +if (!tb2) { +spin_unlock_bh (&head->lock); +return -ENOMEM ; + } + } + +/* Remove the socket's old entry from bhash2 */ +__sk_del_bind2_node (sk); + +sk_add_bind2_node (sk, &tb2->owners); +inet_csk (sk)->icsk_bind2_hash = tb2;spin_unlock_bh (&head->lock); -return err; +return 0 ; }/* RFC 6056 3.3.4. Algorithm 4: Double-Hash Port Selection Algorithm @@ -894,9 +884,8 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row, struct sock *, __u16, struct inet_timewait_sock **)) { struct inet_hashinfo *hinfo = death_row->hashinfo; +struct inet_bind_hashbucket *head, *head2;struct inet_timewait_sock *tw =NULL ; -struct inet_bind2_hashbucket *head2; -struct inet_bind_hashbucket *head;int port =inet_sk (sk)->inet_num;struct net *net =sock_net (sk);struct inet_bind2_bucket *tb2;@@ -907,8 +896,6 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row,int l3mdev;u32 index; - l3mdev =inet_sk_bound_l3mdev (sk); -if (port) { head = &hinfo->bhash[inet_bhashfn (net, port, hinfo->bhash_size)];@@ -917,8 +904,7 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row,spin_lock_bh (&head->lock);if (prev_inaddr_any) { - ret =__inet_bhash2_update_saddr (sk, hinfo, net, port, - l3mdev); + ret =inet_bhash2_update_saddr (sk);if (ret) {spin_unlock_bh (&head->lock);return ret;@@ -937,6 +923,8 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row,return ret; } + l3mdev =inet_sk_bound_l3mdev (sk); +inet_get_local_port_range (net, &low, &high); high++;/* [32768, 60999] -> [32768, 61000[ */ remaining = high - low;@@ -1006,7 +994,10 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row,/* Find the corresponding tb2 bucket since we need to * add the socket to the bhash2 table as well */ - tb2 =inet_bind2_bucket_find (hinfo, net, port, l3mdev, sk, &head2); + head2 =inet_bhashfn_portaddr (hinfo, sk, net, port); +spin_lock (&head2->lock); + + tb2 =inet_bind2_bucket_find (head2, hinfo, net, port, l3mdev, sk);if (!tb2) { tb2 =inet_bind2_bucket_create (hinfo->bind2_bucket_cachep, net, head2, port, l3mdev, sk);@@ -1024,6 +1015,9 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row,/* Head lock still held and bh's disabled */ inet_bind_hash (sk, tb, tb2, port); + +spin_unlock (&head2->lock); +if (sk_unhashed (sk)) {inet_sk (sk)->inet_sport =htons (port);inet_ehash_nolisten (sk, (struct sock *)tw,NULL );
net/ipv4/inet_hashtables.c int __inet_hash_connect(struct inet_timewait_death_row *death_row, · line 1034
Quoted 4 lines
net/ipv4/inet_hashtables.c +1
-0@@ -1037,6 +1031,7 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row,return 0 ; error: +spin_unlock_bh (&head2->lock);
Are you sure _bh has been used at spin_lock() side ?
Quoted 34 lines
net/ipv4/inet_hashtables.c +0
-0if (tb_created)inet_bind_bucket_destroy (hinfo->bind_bucket_cachep, tb);spin_unlock_bh (&head->lock);net/ipv4/tcp.c +3
-5@@ -4633,8 +4633,7 @@ void __init tcp_init(void)panic ("TCP: failed to alloc ehash_locks" ); tcp_hashinfo.bhash = alloc_large_system_hash("TCP bind bhash tables", - sizeof(struct inet_bind_hashbucket) + - sizeof(struct inet_bind2_hashbucket), + 2 * sizeof(struct inet_bind_hashbucket), tcp_hashinfo.ehash_mask + 1, 17, /* one slot per 128 KB of memory */ 0,@@ -4643,11 +4642,11 @@ void __init tcp_init(void)0 , 64 * 1024); tcp_hashinfo.bhash_size =1U << tcp_hashinfo.bhash_size; - tcp_hashinfo.bhash2 = - (struct inet_bind2_hashbucket *)(tcp_hashinfo.bhash + tcp_hashinfo.bhash_size); + tcp_hashinfo.bhash2 = tcp_hashinfo.bhash + tcp_hashinfo.bhash_size;for (i =0 ; i < tcp_hashinfo.bhash_size; i++) {spin_lock_init (&tcp_hashinfo.bhash[i].lock);INIT_HLIST_HEAD (&tcp_hashinfo.bhash[i].chain); +spin_lock_init (&tcp_hashinfo.bhash2[i].lock);INIT_HLIST_HEAD (&tcp_hashinfo.bhash2[i].chain); } --2.30.2
Re: [PATCH net-next v3 2/3] net: Add bhash2 hashbucket locks
In reply to Joanne KoongFrom: Joanne Koong <[email protected]> Date: Fri, 10 Jun 2022 19:16:45 -0700
Quoted 137 lines
Currently, the bhash2 hashbucket uses its corresponding bhash hashbucket's lock for serializing concurrent accesses. There, however, can be the case where the bhash2 hashbucket is accessed concurrently by multiple processes that hash to different bhash hashbuckets but to the same bhash2 hashbucket.
As such, each bhash2 hashbucket will need to have its own lock instead of using its corresponding bhash hashbucket's lock.
Fixes d5a42de8bdbe ("net: Add a second bind table hashed by port and address")Signed-off-by Joanne Koong <[email protected]>--- include/net/inet_hashtables.h | 25 +++---- net/dccp/proto.c | 3 +- net/ipv4/inet_connection_sock.c | 60 +++++++++------- net/ipv4/inet_hashtables.c | 119 +++++++++++++++----------------- net/ipv4/tcp.c | 7 +- 5 files changed, 107 insertions(+), 107 deletions(-)include/net/inet_hashtables.h +10
-15@@ -124,15 +124,6 @@ struct inet_bind_hashbucket {struct hlist_head chain; }; -/* This is synchronized using the inet_bind_hashbucket's spinlock. -* Instead of having separate spinlocks, the inet_bind2_hashbucket can share -* the inet_bind_hashbucket's given that in every case where the bhash2 table -* is useful, a lookup in the bhash table also occurs. -*/ -struct inet_bind2_hashbucket { -struct hlist_head chain; -}; -/* Sockets can be hashed in established or listening table. * We must use different 'nulls' end-of-chain value for all hash buckets : * A socket might transition from ESTABLISH to LISTEN state without @@ -169,7 +160,7 @@ struct inet_hashinfo {* conflicts. */ struct kmem_cache *bind2_bucket_cachep; -struct inet_bind2_hashbucket *bhash2; +struct inet_bind_hashbucket *bhash2;unsigned int bhash_size;/* The 2nd listener table hashed by local port and address */ @@ -240,7 +231,7 @@ static inline bool check_bind_bucket_match(struct inet_bind_bucket *tb,struct inet_bind2_bucket *inet_bind2_bucket_create (struct kmem_cache *cachep,struct net *net, -struct inet_bind2_hashbucket *head, +struct inet_bind_hashbucket *head,const unsigned short port,int l3mdev,const struct sock *sk);@@ -248,12 +239,12 @@ void inet_bind2_bucket_destroy(struct kmem_cache *cachep, struct inet_bind2_bucket *tb);struct inet_bind2_bucket * -inet_bind2_bucket_find (struct inet_hashinfo *hinfo,struct net *net, +inet_bind2_bucket_find (struct inet_bind_hashbucket *head, +struct inet_hashinfo *hinfo,struct net *net,const unsigned short port,int l3mdev, -struct sock *sk, -struct inet_bind2_hashbucket **head); +struct sock *sk); -bool check_bind2_bucket_match_nulladdr(struct inet_bind2_bucket *tb, +bool check_bind2_bucket_match_addr_any(struct inet_bind2_bucket *tb, struct net *net, const unsigned short port, int l3mdev,@@ -265,6 +256,10 @@ static inline u32 inet_bhashfn(const struct net *net, const __u16 lport,return (lport +net_hash_mix (net)) & (bhash_size -1 ); } +struct inet_bind_hashbucket * +inet_bhashfn_portaddr (struct inet_hashinfo *hinfo,const struct sock *sk, +const struct net *net,unsigned short port); +void inet_bind_hash (struct sock *sk,struct inet_bind_bucket *tb,struct inet_bind2_bucket *tb2,const unsigned short snum);net/dccp/proto.c +2
-1@@ -1182,7 +1182,7 @@ static int __init dccp_init(void) goto out_free_dccp_locks; } - dccp_hashinfo.bhash2 = (struct inet_bind2_hashbucket *) + dccp_hashinfo.bhash2 = (struct inet_bind_hashbucket *)__get_free_pages (GFP_ATOMIC | __GFP_NOWARN, bhash_order);if (!dccp_hashinfo.bhash2) {@@ -1193,6 +1193,7 @@ static int __init dccp_init(void)for (i =0 ; i < dccp_hashinfo.bhash_size; i++) {spin_lock_init (&dccp_hashinfo.bhash[i].lock);INIT_HLIST_HEAD (&dccp_hashinfo.bhash[i].chain); +spin_lock_init (&dccp_hashinfo.bhash2[i].lock);INIT_HLIST_HEAD (&dccp_hashinfo.bhash2[i].chain); }net/ipv4/inet_connection_sock.c +4
-6@@ -131,14 +131,14 @@ static bool use_bhash2_on_bind(const struct sock *sk)return sk->sk_rcv_saddr !=htonl (INADDR_ANY ); } -static u32 get_bhash2_nulladdr_hash (const struct sock *sk,struct net *net, +static u32 get_bhash2_addr_any_hash (const struct sock *sk,struct net *net,int port) {#if IS_ENABLED (CONFIG_IPV6 ) -struct in6_addr nulladdr = {}; +struct in6_addr addr_any = {};if (sk->sk_family ==AF_INET6 ) -return ipv6_portaddr_hash (net, &nulladdr, port); +return ipv6_portaddr_hash (net, &addr_any, port);#endif return ipv4_portaddr_hash (net,0 , port); }@@ -204,18 +204,18 @@ static bool check_bhash2_conflict(const struct sock *sk,return false; } -/* This should be called only when the corresponding inet_bind_bucket spinlock -* is held -*/ +/* This should be called only when the tb and tb2 hashbuckets' locks are held */ static int inet_csk_bind_conflict(const struct sock *sk, int port,
net/ipv4/inet_connection_sock.c static bool check_bhash2_conflict(const struct sock *sk, · line 217
Quoted 10 lines
net/ipv4/inet_connection_sock.c +2
-1struct inet_bind_bucket *tb, struct inet_bind2_bucket *tb2, /* may be null */ + struct inet_bind_hashbucket *head_tb2, bool relax, bool reuseport_ok) { struct inet_hashinfo *hinfo = sk->sk_prot->h.hashinfo;kuid_t uid =sock_i_uid ((struct sock *)sk);struct sock_reuseport *reuseport_cb; -struct inet_bind2_hashbucket *head2; +struct inet_bind_hashbucket *head_addr_any;
nit: Keep reverse christmas order.
Quoted 461 lines
net/ipv4/inet_connection_sock.c +29
-18+bool addr_any_conflict = false;bool reuseport_cb_ok;struct sock *sk2;struct net *net;@@ -254,33 +254,39 @@ static int inet_csk_bind_conflict(const struct sock *sk, int port,/* check there's no conflict with an existing IPV6_ADDR_ANY (if ipv6) or * INADDR_ANY (if ipv4) socket. */ - hash =get_bhash2_nulladdr_hash (sk, net, port); - head2 = &hinfo->bhash2[hash & (hinfo->bhash_size -1 )]; + hash =get_bhash2_addr_any_hash (sk, net, port); + head_addr_any = &hinfo->bhash2[hash & (hinfo->bhash_size -1 )]; l3mdev =inet_sk_bound_l3mdev (sk); -inet_bind_bucket_for_each (tb2, &head2->chain) -if (check_bind2_bucket_match_nulladdr (tb2, net, port, l3mdev, sk)) + +if (head_addr_any != head_tb2) +spin_lock_bh (&head_addr_any->lock); + +inet_bind_bucket_for_each (tb2, &head_addr_any->chain) +if (check_bind2_bucket_match_addr_any (tb2, net, port, l3mdev, sk))break ;if (tb2 &&check_bhash2_conflict (sk, tb2, uid, relax, reuseport_cb_ok, reuseport_ok)) -return true; + addr_any_conflict = true; -return false; +if (head_addr_any != head_tb2) +spin_unlock_bh (&head_addr_any->lock); + +return addr_any_conflict; }/* * Find an open port number for the socket. Returns with the -* inet_bind_hashbucket lock held. +* inet_bind_hashbucket locks held if successful. */ static struct inet_bind_hashbucket *inet_csk_find_open_port (struct sock *sk,struct inet_bind_bucket **tb_ret,struct inet_bind2_bucket **tb2_ret, -struct inet_bind2_hashbucket **head2_ret,int *port_ret) +struct inet_bind_hashbucket **head2_ret,int *port_ret) {struct inet_hashinfo *hinfo = sk->sk_prot->h.hashinfo; -struct inet_bind2_hashbucket *head2; -struct inet_bind_hashbucket *head; +struct inet_bind_hashbucket *head, *head2;struct net *net =sock_net (sk);int i, low, high, attempt_half;struct inet_bind2_bucket *tb2;@@ -325,19 +331,22 @@ inet_csk_find_open_port(struct sock *sk, struct inet_bind_bucket **tb_ret,continue ; head = &hinfo->bhash[inet_bhashfn (net, port, hinfo->bhash_size)]; + head2 =inet_bhashfn_portaddr (hinfo, sk, net, port);spin_lock_bh (&head->lock); - tb2 =inet_bind2_bucket_find (hinfo, net, port, l3mdev, sk, - &head2); +spin_lock_bh (&head2->lock); + + tb2 =inet_bind2_bucket_find (head2, hinfo, net, port, l3mdev, sk);inet_bind_bucket_for_each (tb, &head->chain)if (check_bind_bucket_match (tb, net, port, l3mdev)) {if (!inet_csk_bind_conflict (sk, port, tb, tb2, - relax, false)) + head2, relax, false)) goto success; goto next_port; } tb =NULL ; goto success; next_port: +spin_unlock_bh (&head2->lock);spin_unlock_bh (&head->lock);cond_resched (); }@@ -459,10 +468,9 @@ int inet_csk_get_port(struct sock *sk, unsigned short snum)bool reuse = sk->sk_reuse && sk->sk_state !=TCP_LISTEN ;struct inet_hashinfo *hinfo = sk->sk_prot->h.hashinfo;bool bhash_created = false, bhash2_created = false; +struct inet_bind_hashbucket *head, *head2;struct inet_bind2_bucket *tb2 =NULL ; -struct inet_bind2_hashbucket *head2;struct inet_bind_bucket *tb =NULL ; -struct inet_bind_hashbucket *head;struct net *net =sock_net (sk);int ret =1 , port = snum;bool found_port = false;@@ -480,13 +488,14 @@ int inet_csk_get_port(struct sock *sk, unsigned short snum) }else { head = &hinfo->bhash[inet_bhashfn (net, port, hinfo->bhash_size)]; + head2 =inet_bhashfn_portaddr (hinfo, sk, net, port);spin_lock_bh (&head->lock);inet_bind_bucket_for_each (tb, &head->chain)if (check_bind_bucket_match (tb, net, port, l3mdev))break ; - tb2 =inet_bind2_bucket_find (hinfo, net, port, l3mdev, sk, - &head2); +spin_lock_bh (&head2->lock); + tb2 =inet_bind2_bucket_find (head2, hinfo, net, port, l3mdev, sk); }if (!tb) {@@ -513,7 +522,7 @@ int inet_csk_get_port(struct sock *sk, unsigned short snum)if ((tb->fastreuse >0 && reuse) ||sk_reuseport_match (tb, sk)) goto success; -if (inet_csk_bind_conflict (sk, port, tb, tb2, true, true)) +if (inet_csk_bind_conflict (sk, port, tb, tb2, head2, true, true)) goto fail_unlock; } success:@@ -533,6 +542,7 @@ int inet_csk_get_port(struct sock *sk, unsigned short snum)inet_bind2_bucket_destroy (hinfo->bind2_bucket_cachep, tb2); } +spin_unlock_bh (&head2->lock);spin_unlock_bh (&head->lock);return ret; }net/ipv4/inet_hashtables.c +57
-62@@ -83,7 +83,7 @@ struct inet_bind_bucket *inet_bind_bucket_create(struct kmem_cache *cachep,struct inet_bind2_bucket *inet_bind2_bucket_create (struct kmem_cache *cachep,struct net *net, -struct inet_bind2_hashbucket *head, +struct inet_bind_hashbucket *head,const unsigned short port,int l3mdev,const struct sock *sk)@@ -127,9 +127,7 @@ void inet_bind_bucket_destroy(struct kmem_cache *cachep, struct inet_bind_bucket } } -/* Caller must hold the lock for the corresponding hashbucket in the bhash table -* with local BH disabled -*/ +/* Caller must hold hashbucket lock for this tb with local BH disabled */ void inet_bind2_bucket_destroy (struct kmem_cache *cachep,struct inet_bind2_bucket *tb) {if (hlist_empty (&tb->owners)) {@@ -157,6 +155,9 @@ static void __inet_put_port(struct sock *sk)const int bhash =inet_bhashfn (sock_net (sk),inet_sk (sk)->inet_num, hashinfo->bhash_size);struct inet_bind_hashbucket *head = &hashinfo->bhash[bhash]; +struct inet_bind_hashbucket *head2 = +inet_bhashfn_portaddr (hashinfo, sk,sock_net (sk), +inet_sk (sk)->inet_num);struct inet_bind2_bucket *tb2;struct inet_bind_bucket *tb;@@ -167,12 +168,15 @@ static void __inet_put_port(struct sock *sk)inet_sk (sk)->inet_num =0 ;inet_bind_bucket_destroy (hashinfo->bind_bucket_cachep, tb); +spin_lock (&head2->lock);if (inet_csk (sk)->icsk_bind2_hash) { tb2 =inet_csk (sk)->icsk_bind2_hash;__sk_del_bind2_node (sk);inet_csk (sk)->icsk_bind2_hash =NULL ;inet_bind2_bucket_destroy (hashinfo->bind2_bucket_cachep, tb2); } +spin_unlock (&head2->lock); +spin_unlock (&head->lock); }@@ -191,7 +195,9 @@ int __inet_inherit_port(const struct sock *sk, struct sock *child)const int bhash =inet_bhashfn (sock_net (sk), port, table->bhash_size);struct inet_bind_hashbucket *head = &table->bhash[bhash]; -struct inet_bind2_hashbucket *head_bhash2; +struct inet_bind_hashbucket *head2 = +inet_bhashfn_portaddr (table, child,sock_net (sk), + port);bool created_inet_bind_bucket = false;struct net *net =sock_net (sk);struct inet_bind2_bucket *tb2;@@ -199,9 +205,11 @@ int __inet_inherit_port(const struct sock *sk, struct sock *child)int l3mdev;spin_lock (&head->lock); +spin_lock (&head2->lock); tb =inet_csk (sk)->icsk_bind_hash; tb2 =inet_csk (sk)->icsk_bind2_hash;if (unlikely (!tb || !tb2)) { +spin_unlock (&head2->lock);spin_unlock (&head->lock);return -ENOENT ; }@@ -221,6 +229,7 @@ int __inet_inherit_port(const struct sock *sk, struct sock *child) tb =inet_bind_bucket_create (table->bind_bucket_cachep, net, head, port, l3mdev);if (!tb) { +spin_unlock (&head2->lock);spin_unlock (&head->lock);return -ENOMEM ; }@@ -233,17 +242,17 @@ int __inet_inherit_port(const struct sock *sk, struct sock *child) l3mdev =inet_sk_bound_l3mdev (sk); bhash2_find: - tb2 =inet_bind2_bucket_find (table, net, port, l3mdev, child, - &head_bhash2); + tb2 =inet_bind2_bucket_find (head2, table, net, port, l3mdev, child);if (!tb2) { tb2 =inet_bind2_bucket_create (table->bind2_bucket_cachep, - net, head_bhash2, port, + net, head2, port, l3mdev, child);if (!tb2) goto error; } }inet_bind_hash (child, tb, tb2, port); +spin_unlock (&head2->lock);spin_unlock (&head->lock);return 0 ;@@ -251,6 +260,7 @@ int __inet_inherit_port(const struct sock *sk, struct sock *child) error:if (created_inet_bind_bucket)inet_bind_bucket_destroy (table->bind_bucket_cachep, tb); +spin_unlock (&head2->lock);spin_unlock (&head->lock);return -ENOMEM ; }@@ -771,24 +781,24 @@ static bool check_bind2_bucket_match(struct inet_bind2_bucket *tb, tb->l3mdev == l3mdev && tb->rcv_saddr == sk->sk_rcv_saddr; } -bool check_bind2_bucket_match_nulladdr (struct inet_bind2_bucket *tb, +bool check_bind2_bucket_match_addr_any (struct inet_bind2_bucket *tb,struct net *net,const unsigned short port,int l3mdev,const struct sock *sk) {#if IS_ENABLED (CONFIG_IPV6 ) -struct in6_addr nulladdr = {}; +struct in6_addr addr_any = {};if (sk->sk_family ==AF_INET6 )return net_eq (ib2_net (tb), net) && tb->port == port && tb->l3mdev == l3mdev && -ipv6_addr_equal (&tb->v6_rcv_saddr, &nulladdr); +ipv6_addr_equal (&tb->v6_rcv_saddr, &addr_any); else#endif return net_eq (ib2_net (tb), net) && tb->port == port && tb->l3mdev == l3mdev && tb->rcv_saddr ==0 ; } -static struct inet_bind2_hashbucket * +struct inet_bind_hashbucket *inet_bhashfn_portaddr (struct inet_hashinfo *hinfo,const struct sock *sk,const struct net *net,unsigned short port) {@@ -803,55 +813,21 @@ inet_bhashfn_portaddr(struct inet_hashinfo *hinfo, const struct sock *sk,return &hinfo->bhash2[hash & (hinfo->bhash_size -1 )]; } -/* This should only be called when the spinlock for the socket's corresponding -* bind_hashbucket is held -*/ +/* The socket's bhash2 hashbucket spinlock must be held when this is called */ struct inet_bind2_bucket * -inet_bind2_bucket_find (struct inet_hashinfo *hinfo,struct net *net, -const unsigned short port,int l3mdev,struct sock *sk, -struct inet_bind2_hashbucket **head) +inet_bind2_bucket_find (struct inet_bind_hashbucket *head, +struct inet_hashinfo *hinfo,struct net *net, +const unsigned short port,int l3mdev,struct sock *sk) {struct inet_bind2_bucket *bhash2 =NULL ; -struct inet_bind2_hashbucket *h; - h =inet_bhashfn_portaddr (hinfo, sk, net, port); -inet_bind_bucket_for_each (bhash2, &h->chain) { +inet_bind_bucket_for_each (bhash2, &head->chain)if (check_bind2_bucket_match (bhash2, net, port, l3mdev, sk))break ; - } - -if (head) - *head = h;return bhash2; } -/* the lock for the socket's corresponding bhash entry must be held */ -static int __inet_bhash2_update_saddr (struct sock *sk, -struct inet_hashinfo *hinfo, -struct net *net,int port,int l3mdev) -{ -struct inet_bind2_hashbucket *head2; -struct inet_bind2_bucket *tb2; - - tb2 =inet_bind2_bucket_find (hinfo, net, port, l3mdev, sk, - &head2); -if (!tb2) { - tb2 =inet_bind2_bucket_create (hinfo->bind2_bucket_cachep, - net, head2, port, l3mdev, sk); -if (!tb2) -return -ENOMEM ; - } - -/* Remove the socket's old entry from bhash2 */ -__sk_del_bind2_node (sk); - -sk_add_bind2_node (sk, &tb2->owners); -inet_csk (sk)->icsk_bind2_hash = tb2; - -return 0 ; -} -/* This should be called if/when a socket's rcv saddr changes after it has * been binded. */ @@ -862,17 +838,31 @@ int inet_bhash2_update_saddr(struct sock *sk)struct inet_bind_hashbucket *head;int port =inet_sk (sk)->inet_num;struct net *net =sock_net (sk); -int err; +struct inet_bind2_bucket *tb2; - head = &hinfo->bhash[inet_bhashfn (net, port, hinfo->bhash_size)]; + head =inet_bhashfn_portaddr (hinfo, sk, net, port);spin_lock_bh (&head->lock); - err =__inet_bhash2_update_saddr (sk, hinfo, net, port, l3mdev); + tb2 =inet_bind2_bucket_find (head, hinfo, net, port, l3mdev, sk); +if (!tb2) { + tb2 =inet_bind2_bucket_create (hinfo->bind2_bucket_cachep, + net, head, port, l3mdev, sk); +if (!tb2) { +spin_unlock_bh (&head->lock); +return -ENOMEM ; + } + } + +/* Remove the socket's old entry from bhash2 */ +__sk_del_bind2_node (sk); + +sk_add_bind2_node (sk, &tb2->owners); +inet_csk (sk)->icsk_bind2_hash = tb2;spin_unlock_bh (&head->lock); -return err; +return 0 ; }/* RFC 6056 3.3.4. Algorithm 4: Double-Hash Port Selection Algorithm @@ -894,9 +884,8 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row, struct sock *, __u16, struct inet_timewait_sock **)) { struct inet_hashinfo *hinfo = death_row->hashinfo; +struct inet_bind_hashbucket *head, *head2;struct inet_timewait_sock *tw =NULL ; -struct inet_bind2_hashbucket *head2; -struct inet_bind_hashbucket *head;int port =inet_sk (sk)->inet_num;struct net *net =sock_net (sk);struct inet_bind2_bucket *tb2;@@ -907,8 +896,6 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row,int l3mdev;u32 index; - l3mdev =inet_sk_bound_l3mdev (sk); -if (port) { head = &hinfo->bhash[inet_bhashfn (net, port, hinfo->bhash_size)];@@ -917,8 +904,7 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row,spin_lock_bh (&head->lock);if (prev_inaddr_any) { - ret =__inet_bhash2_update_saddr (sk, hinfo, net, port, - l3mdev); + ret =inet_bhash2_update_saddr (sk);if (ret) {spin_unlock_bh (&head->lock);return ret;@@ -937,6 +923,8 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row,return ret; } + l3mdev =inet_sk_bound_l3mdev (sk); +inet_get_local_port_range (net, &low, &high); high++;/* [32768, 60999] -> [32768, 61000[ */ remaining = high - low;@@ -1006,7 +994,10 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row,/* Find the corresponding tb2 bucket since we need to * add the socket to the bhash2 table as well */ - tb2 =inet_bind2_bucket_find (hinfo, net, port, l3mdev, sk, &head2); + head2 =inet_bhashfn_portaddr (hinfo, sk, net, port); +spin_lock (&head2->lock); + + tb2 =inet_bind2_bucket_find (head2, hinfo, net, port, l3mdev, sk);if (!tb2) { tb2 =inet_bind2_bucket_create (hinfo->bind2_bucket_cachep, net, head2, port, l3mdev, sk);@@ -1024,6 +1015,9 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row,/* Head lock still held and bh's disabled */ inet_bind_hash (sk, tb, tb2, port); + +spin_unlock (&head2->lock); +if (sk_unhashed (sk)) {inet_sk (sk)->inet_sport =htons (port);inet_ehash_nolisten (sk, (struct sock *)tw,NULL );@@ -1037,6 +1031,7 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row,return 0 ; error: +spin_unlock_bh (&head2->lock);if (tb_created)inet_bind_bucket_destroy (hinfo->bind_bucket_cachep, tb);spin_unlock_bh (&head->lock);net/ipv4/tcp.c +3
-4@@ -4633,8 +4633,7 @@ void __init tcp_init(void)panic ("TCP: failed to alloc ehash_locks" ); tcp_hashinfo.bhash = alloc_large_system_hash("TCP bind bhash tables", - sizeof(struct inet_bind_hashbucket) + - sizeof(struct inet_bind2_hashbucket), + 2 * sizeof(struct inet_bind_hashbucket), tcp_hashinfo.ehash_mask + 1, 17, /* one slot per 128 KB of memory */ 0,@@ -4643,11 +4642,11 @@ void __init tcp_init(void)0 , 64 * 1024); tcp_hashinfo.bhash_size =1U << tcp_hashinfo.bhash_size; - tcp_hashinfo.bhash2 = - (struct inet_bind2_hashbucket *)(tcp_hashinfo.bhash + tcp_hashinfo.bhash_size); + tcp_hashinfo.bhash2 = tcp_hashinfo.bhash + tcp_hashinfo.bhash_size;for (i =0 ; i < tcp_hashinfo.bhash_size; i++) {spin_lock_init (&tcp_hashinfo.bhash[i].lock);INIT_HLIST_HEAD (&tcp_hashinfo.bhash[i].chain); +spin_lock_init (&tcp_hashinfo.bhash2[i].lock);INIT_HLIST_HEAD (&tcp_hashinfo.bhash2[i].chain); }Signature · 1 line
2.30.2
Re: [PATCH net-next v3 2/3] net: Add bhash2 hashbucket locks
In reply to Joanne KoongQuoted 156 lines · Joanne Koong
Currently, the bhash2 hashbucket uses its corresponding bhash hashbucket's lock for serializing concurrent accesses. There, however, can be the case where the bhash2 hashbucket is accessed concurrently by multiple processes that hash to different bhash hashbuckets but to the same bhash2 hashbucket.
As such, each bhash2 hashbucket will need to have its own lock instead of using its corresponding bhash hashbucket's lock.
Fixes d5a42de8bdbe ("net: Add a second bind table hashed by port and address")Signed-off-by Joanne Koong <[email protected]>---
include/net/inet_hashtables.h | 25 +++---- net/dccp/proto.c | 3 +- net/ipv4/inet_connection_sock.c | 60 +++++++++------- net/ipv4/inet_hashtables.c | 119 +++++++++++++++----------------- net/ipv4/tcp.c | 7 +- 5 files changed, 107 insertions(+), 107 deletions(-)include/net/inet_hashtables.h +1
-10@@ -124,15 +124,6 @@ struct inet_bind_hashbucket {struct hlist_head chain; }; -/* This is synchronized using the inet_bind_hashbucket's spinlock. -* Instead of having separate spinlocks, the inet_bind2_hashbucket can share -* the inet_bind_hashbucket's given that in every case where the bhash2 table -* is useful, a lookup in the bhash table also occurs. -*/ -struct inet_bind2_hashbucket { -struct hlist_head chain; -}; -/* Sockets can be hashed in established or listening table. * We must use different 'nulls' end-of-chain value for all hash buckets : * A socket might transition from ESTABLISH to LISTEN state without @@ -169,7 +160,7 @@ struct inet_hashinfo {* conflicts. */ struct kmem_cache *bind2_bucket_cachep; -struct inet_bind2_hashbucket *bhash2; +struct inet_bind_hashbucket *bhash2;unsigned int bhash_size;/* The 2nd listener table hashed by local port and address */ @@ -240,7 +231,7 @@ static inline bool check_bind_bucket_match(struct inet_bind_bucket *tb,struct
inet_bind2_bucket* inet_bind2_bucket_create(structkmem_cache*cachep, struct net *net,include/net/inet_hashtables.h +1
-1- struct inet_bind2_hashbucket *head, + struct inet_bind_hashbucket *head,const unsigned short port, int l3mdev, const struct sock *sk);include/net/inet_hashtables.h +0
-0@@ -248,12 +239,12 @@ void inet_bind2_bucket_destroy(struct kmem_cache *cachep, struct inet_bind2_bucket *tb);struct
inet_bind2_bucket*include/net/inet_hashtables.h +2
-1-inet_bind2_bucket_find(struct inet_hashinfo *hinfo, struct net *net, +inet_bind2_bucket_find(struct inet_bind_hashbucket *head, + struct inet_hashinfo *hinfo, struct net *net,const unsigned short port, int l3mdev,
include/net/inet_hashtables.h +2
-3-struct sock *sk, - struct inet_bind2_hashbucket **head); + struct sock *sk); -bool check_bind2_bucket_match_nulladdr(struct inet_bind2_bucket *tb, +bool check_bind2_bucket_match_addr_any(struct inet_bind2_bucket *tb,struct net *net, const unsigned short port, int l3mdev,
include/net/inet_hashtables.h +4
-0@@ -265,6 +256,10 @@ static inline u32 inet_bhashfn(const struct net *net, const __u16 lport,return (lport +net_hash_mix (net)) & (bhash_size -1 ); } +struct inet_bind_hashbucket * +inet_bhashfn_portaddr (struct inet_hashinfo *hinfo,const struct sock *sk, +const struct net *net,unsigned short port); +void inet_bind_hash (struct sock *sk,struct inet_bind_bucket *tb,struct inet_bind2_bucket *tb2,const unsigned short snum);net/dccp/proto.c +2
-1@@ -1182,7 +1182,7 @@ static int __init dccp_init(void) goto out_free_dccp_locks; } - dccp_hashinfo.bhash2 = (struct inet_bind2_hashbucket *) + dccp_hashinfo.bhash2 = (struct inet_bind_hashbucket *)__get_free_pages (GFP_ATOMIC | __GFP_NOWARN, bhash_order);if (!dccp_hashinfo.bhash2) {@@ -1193,6 +1193,7 @@ static int __init dccp_init(void)for (i =0 ; i < dccp_hashinfo.bhash_size; i++) {spin_lock_init (&dccp_hashinfo.bhash[i].lock);INIT_HLIST_HEAD (&dccp_hashinfo.bhash[i].chain); +spin_lock_init (&dccp_hashinfo.bhash2[i].lock);INIT_HLIST_HEAD (&dccp_hashinfo.bhash2[i].chain); }net/ipv4/inet_connection_sock.c +7
-8@@ -131,14 +131,14 @@ static bool use_bhash2_on_bind(const struct sock *sk)return sk->sk_rcv_saddr !=htonl (INADDR_ANY ); } -static u32 get_bhash2_nulladdr_hash (const struct sock *sk,struct net *net, +static u32 get_bhash2_addr_any_hash (const struct sock *sk,struct net *net,int port) {#if IS_ENABLED (CONFIG_IPV6 ) -struct in6_addr nulladdr = {}; +struct in6_addr addr_any = {};if (sk->sk_family ==AF_INET6 ) -return ipv6_portaddr_hash (net, &nulladdr, port); +return ipv6_portaddr_hash (net, &addr_any, port);#endif return ipv4_portaddr_hash (net,0 , port); }@@ -204,18 +204,18 @@ static bool check_bhash2_conflict(const struct sock *sk,return false; } -/* This should be called only when the corresponding inet_bind_bucket spinlock -* is held -*/ +/* This should be called only when the tb and tb2 hashbuckets' locks are held */ static int inet_csk_bind_conflict (const struct sock *sk,int port,struct inet_bind_bucket *tb,struct inet_bind2_bucket *tb2,/* may be null */ +struct inet_bind_hashbucket *head_tb2,bool relax,bool reuseport_ok) {struct inet_hashinfo *hinfo = sk->sk_prot->h.hashinfo;kuid_t uid =sock_i_uid ((struct sock *)sk);struct sock_reuseport *reuseport_cb; -struct inet_bind2_hashbucket *head2; +struct inet_bind_hashbucket *head_addr_any; +bool addr_any_conflict = false;bool reuseport_cb_ok;struct sock *sk2;struct net *net;@@ -254,33 +254,39 @@ static int inet_csk_bind_conflict(const struct sock *sk, int port,/* check there's no conflict with an existing IPV6_ADDR_ANY (if ipv6) or * INADDR_ANY (if ipv4) socket. */ - hash =get_bhash2_nulladdr_hash (sk, net, port);
net/ipv4/inet_connection_sock.c static int inet_csk_bind_conflict(const struct sock *sk, int port, · line 263
Quoted 10 lines
net/ipv4/inet_connection_sock.c +5
-3- head2 = &hinfo->bhash2[hash & (hinfo->bhash_size -1 )]; + hash =get_bhash2_addr_any_hash (sk, net, port); + head_addr_any = &hinfo->bhash2[hash & (hinfo->bhash_size -1 )]; l3mdev =inet_sk_bound_l3mdev (sk); -inet_bind_bucket_for_each (tb2, &head2->chain) -if (check_bind2_bucket_match_nulladdr (tb2, net, port, l3mdev, sk)) + +if (head_addr_any != head_tb2) +spin_lock_bh (&head_addr_any->lock);
Hi Joanne -
syzkaller is consistently hitting a warning here (about 10x per minute):
============================================
WARNING: possible recursive locking detected
5.19.0-rc1-00382-g78347e8e15bf #1 Not tainted
--------------------------------------------
sshd/352 is trying to acquire lock:ffffc90000968640 (&tcp_hashinfo.bhash2[i].lock){+.-.}-{2:2}, at: spin_lock_bh include/linux/spinlock.h:354 [inline] ffffc90000968640 (&tcp_hashinfo.bhash2[i].lock){+.-.}-{2:2}, at: inet_csk_bind_conflict+0x4c4/0x8e0 net/ipv4/inet_connection_sock.c:263
but task is already holding lock:ffffc90000883d28 (&tcp_hashinfo.bhash2[i].lock){+.-.}-{2:2}, at: spin_lock_bh include/linux/spinlock.h:354 [inline] ffffc90000883d28 (&tcp_hashinfo.bhash2[i].lock){+.-.}-{2:2}, at: inet_csk_get_port+0x528/0xea0 net/ipv4/inet_connection_sock.c:497
other info that might help us debug this:
Possible unsafe locking scenario:
CPU0
---- lock(&tcp_hashinfo.bhash2[i].lock);
lock(&tcp_hashinfo.bhash2[i].lock);
*** DEADLOCK ***
May be due to missing lock nesting notation
3 locks held by sshd/352:
#0: ffff88810a500e70 (sk_lock-AF_INET6){+.+.}-{0:0}, at: lock_sock include/net/sock.h:1679 [inline]
#0: ffff88810a500e70 (sk_lock-AF_INET6){+.+.}-{0:0}, at: inet_listen+0x94/0x650 net/ipv4/af_inet.c:199
#1: ffffc90000720060 (&tcp_hashinfo.bhash[i].lock){+.-.}-{2:2}, at: spin_lock_bh include/linux/spinlock.h:354 [inline] #1: ffffc90000720060 (&tcp_hashinfo.bhash[i].lock){+.-.}-{2:2}, at: inet_csk_get_port+0x3d7/0xea0 net/ipv4/inet_connection_sock.c:492 #2: ffffc90000883d28 (&tcp_hashinfo.bhash2[i].lock){+.-.}-{2:2}, at: spin_lock_bh include/linux/spinlock.h:354 [inline] #2: ffffc90000883d28 (&tcp_hashinfo.bhash2[i].lock){+.-.}-{2:2}, at: inet_csk_get_port+0x528/0xea0 net/ipv4/inet_connection_sock.c:497
stack backtrace: CPU: 0 PID: 352 Comm: sshd Not tainted 5.19.0-rc1-00382-g78347e8e15bf #1 Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.15.0-1 04/01/2014 Call Trace: <TASK> __dump_stack lib/dump_stack.c:88 [inline] dump_stack_lvl+0xcd/0x134 lib/dump_stack.c:106 print_deadlock_bug kernel/locking/lockdep.c:2988 [inline] check_deadlock kernel/locking/lockdep.c:3031 [inline] validate_chain.cold+0x168/0x180 kernel/locking/lockdep.c:3816 __lock_acquire+0xadb/0x17f0 kernel/locking/lockdep.c:5053 lock_acquire kernel/locking/lockdep.c:5665 [inline] lock_acquire+0x1ab/0x570 kernel/locking/lockdep.c:5630 __raw_spin_lock_bh include/linux/spinlock_api_smp.h:126 [inline] _raw_spin_lock_bh+0x34/0x40 kernel/locking/spinlock.c:178 spin_lock_bh include/linux/spinlock.h:354 [inline] inet_csk_bind_conflict+0x4c4/0x8e0 net/ipv4/inet_connection_sock.c:263 inet_csk_get_port+0x75d/0xea0 net/ipv4/inet_connection_sock.c:525 inet_csk_listen_start+0x143/0x3d0 net/ipv4/inet_connection_sock.c:1188 inet_listen+0x22f/0x650 net/ipv4/af_inet.c:228 __sys_listen+0x189/0x260 net/socket.c:1810 __do_sys_listen net/socket.c:1819 [inline] __se_sys_listen net/socket.c:1817 [inline] __x64_sys_listen+0x54/0x80 net/socket.c:1817 do_syscall_x64 arch/x86/entry/common.c:50 [inline] do_syscall_64+0x38/0x90 arch/x86/entry/common.c:80 entry_SYSCALL_64_after_hwframe+0x46/0xb0 RIP: 0033:0x7f0d578c7027 Code: f0 ff ff 73 01 c3 48 8b 0d 66 fe 0b 00 f7 d8 64 89 01 48 83 c8 ff c3 66 2e 0f 1f 84 00 00 00 00 00 66 90 b8 32 00 00 00 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 8b 0d 39 fe 0b 00 f7 d8 64 89 01 48 RSP: 002b:00007fff6b837208 EFLAGS: 00000246 ORIG_RAX: 0000000000000032 RAX: ffffffffffffffda RBX: 000000000000000b RCX: 00007f0d578c7027 RDX: 000000000000001c RSI: 0000000000000080 RDI: 0000000000000008 RBP: 0000000000000008 R08: 0000000000000004 R09: 0000000000000003 R10: 00007fff6b8371f4 R11: 0000000000000246 R12: 00007fff6b837280 R13: 00007fff6b837770 R14: 00007fff6b8372a0 R15: 000055d977b7b260 </TASK>
CONFIG_LOCKDEP and CONFIG_PROVE_LOCKING will help reproduce this, it doesn't seem to be anything special that syzkaller is doing.
Regards,
Mat
Quoted 444 lines
net/ipv4/inet_connection_sock.c +9
-3+ +inet_bind_bucket_for_each (tb2, &head_addr_any->chain) +if (check_bind2_bucket_match_addr_any (tb2, net, port, l3mdev, sk))break ;if (tb2 &&check_bhash2_conflict (sk, tb2, uid, relax, reuseport_cb_ok, reuseport_ok)) -return true; + addr_any_conflict = true; -return false; +if (head_addr_any != head_tb2) +spin_unlock_bh (&head_addr_any->lock); + +return addr_any_conflict; }/* * Find an open port number for the socket. Returns with the -* inet_bind_hashbucket lock held. +* inet_bind_hashbucket locks held if successful. */ static struct
inet_bind_hashbucket* inet_csk_find_open_port(struct sock *sk, structinet_bind_bucket**tb_ret,
structinet_bind2_bucket**tb2_ret,net/ipv4/inet_connection_sock.c +1
-1- struct inet_bind2_hashbucket **head2_ret, int *port_ret) + struct inet_bind_hashbucket **head2_ret, int *port_ret){ struct inet_hashinfo *hinfo = sk->sk_prot->h.hashinfo;net/ipv4/inet_connection_sock.c +1
-2-struct inet_bind2_hashbucket *head2; -struct inet_bind_hashbucket *head; +struct inet_bind_hashbucket *head, *head2;struct net *net = sock_net(sk); int i, low, high, attempt_half; struct inet_bind2_bucket *tb2;net/ipv4/inet_connection_sock.c +5
-3@@ -325,19 +331,22 @@ inet_csk_find_open_port(struct sock *sk, struct inet_bind_bucket **tb_ret,continue ; head = &hinfo->bhash[inet_bhashfn (net, port, hinfo->bhash_size)]; + head2 =inet_bhashfn_portaddr (hinfo, sk, net, port);spin_lock_bh (&head->lock); - tb2 =inet_bind2_bucket_find (hinfo, net, port, l3mdev, sk, - &head2); +spin_lock_bh (&head2->lock); + + tb2 =inet_bind2_bucket_find (head2, hinfo, net, port, l3mdev, sk);inet_bind_bucket_for_each (tb, &head->chain)if (check_bind_bucket_match (tb, net, port, l3mdev)) {if (!inet_csk_bind_conflict (sk, port, tb, tb2, - relax, false)) + head2, relax, false)) goto success; goto next_port; } tb =NULL ; goto success;
next_port:net/ipv4/inet_connection_sock.c +1
-0+spin_unlock_bh (&head2->lock);spin_unlock_bh(&head->lock); cond_resched(); }net/ipv4/inet_connection_sock.c +5
-5@@ -459,10 +468,9 @@ int inet_csk_get_port(struct sock *sk, unsigned short snum)bool reuse = sk->sk_reuse && sk->sk_state !=TCP_LISTEN ;struct inet_hashinfo *hinfo = sk->sk_prot->h.hashinfo;bool bhash_created = false, bhash2_created = false; +struct inet_bind_hashbucket *head, *head2;struct inet_bind2_bucket *tb2 =NULL ; -struct inet_bind2_hashbucket *head2;struct inet_bind_bucket *tb =NULL ; -struct inet_bind_hashbucket *head;struct net *net =sock_net (sk);int ret =1 , port = snum;bool found_port = false;@@ -480,13 +488,14 @@ int inet_csk_get_port(struct sock *sk, unsigned short snum) }else { head = &hinfo->bhash[inet_bhashfn (net, port, hinfo->bhash_size)]; + head2 =inet_bhashfn_portaddr (hinfo, sk, net, port);spin_lock_bh (&head->lock);inet_bind_bucket_for_each (tb, &head->chain)if (check_bind_bucket_match (tb, net, port, l3mdev))break ; - tb2 =inet_bind2_bucket_find (hinfo, net, port, l3mdev, sk, - &head2); +spin_lock_bh (&head2->lock); + tb2 =inet_bind2_bucket_find (head2, hinfo, net, port, l3mdev, sk); }if (!tb) {@@ -513,7 +522,7 @@ int inet_csk_get_port(struct sock *sk, unsigned short snum)if ((tb->fastreuse >0 && reuse) ||sk_reuseport_match (tb, sk)) goto success; -if (inet_csk_bind_conflict (sk, port, tb, tb2, true, true)) +if (inet_csk_bind_conflict (sk, port, tb, tb2, head2, true, true)) goto fail_unlock; }success:
net/ipv4/inet_connection_sock.c +1
-0@@ -533,6 +542,7 @@ int inet_csk_get_port(struct sock *sk, unsigned short snum)inet_bind2_bucket_destroy (hinfo->bind2_bucket_cachep, tb2); } +spin_unlock_bh (&head2->lock);spin_unlock_bh (&head->lock);return ret; }net/ipv4/inet_hashtables.c +14
-5@@ -83,7 +83,7 @@ struct inet_bind_bucket *inet_bind_bucket_create(struct kmem_cache *cachep,struct inet_bind2_bucket *inet_bind2_bucket_create (struct kmem_cache *cachep,struct net *net, -struct inet_bind2_hashbucket *head, +struct inet_bind_hashbucket *head,const unsigned short port,int l3mdev,const struct sock *sk)@@ -127,9 +127,7 @@ void inet_bind_bucket_destroy(struct kmem_cache *cachep, struct inet_bind_bucket } } -/* Caller must hold the lock for the corresponding hashbucket in the bhash table -* with local BH disabled -*/ +/* Caller must hold hashbucket lock for this tb with local BH disabled */ void inet_bind2_bucket_destroy (struct kmem_cache *cachep,struct inet_bind2_bucket *tb) {if (hlist_empty (&tb->owners)) {@@ -157,6 +155,9 @@ static void __inet_put_port(struct sock *sk)const int bhash =inet_bhashfn (sock_net (sk),inet_sk (sk)->inet_num, hashinfo->bhash_size);struct inet_bind_hashbucket *head = &hashinfo->bhash[bhash]; +struct inet_bind_hashbucket *head2 = +inet_bhashfn_portaddr (hashinfo, sk,sock_net (sk), +inet_sk (sk)->inet_num);struct inet_bind2_bucket *tb2;struct inet_bind_bucket *tb;@@ -167,12 +168,15 @@ static void __inet_put_port(struct sock *sk)inet_sk (sk)->inet_num =0 ;inet_bind_bucket_destroy (hashinfo->bind_bucket_cachep, tb); +spin_lock (&head2->lock);if (inet_csk (sk)->icsk_bind2_hash) { tb2 =inet_csk (sk)->icsk_bind2_hash;__sk_del_bind2_node (sk);inet_csk (sk)->icsk_bind2_hash =NULL ;inet_bind2_bucket_destroy (hashinfo->bind2_bucket_cachep, tb2); } +spin_unlock (&head2->lock); +spin_unlock (&head->lock); }@@ -191,7 +195,9 @@ int __inet_inherit_port(const struct sock *sk, struct sock *child)const int bhash =inet_bhashfn (sock_net (sk), port, table->bhash_size);struct inet_bind_hashbucket *head = &table->bhash[bhash]; -struct inet_bind2_hashbucket *head_bhash2; +struct inet_bind_hashbucket *head2 = +inet_bhashfn_portaddr (table, child,sock_net (sk), + port);bool created_inet_bind_bucket = false;struct net *net =sock_net (sk);struct inet_bind2_bucket *tb2;@@ -199,9 +205,11 @@ int __inet_inherit_port(const struct sock *sk, struct sock *child)int l3mdev;spin_lock (&head->lock); +spin_lock (&head2->lock); tb =inet_csk (sk)->icsk_bind_hash; tb2 =inet_csk (sk)->icsk_bind2_hash;if (unlikely (!tb || !tb2)) { +spin_unlock (&head2->lock);spin_unlock (&head->lock);return -ENOENT ; }@@ -221,6 +229,7 @@ int __inet_inherit_port(const struct sock *sk, struct sock *child) tb =inet_bind_bucket_create (table->bind_bucket_cachep, net, head, port, l3mdev);if (!tb) { +spin_unlock (&head2->lock);spin_unlock (&head->lock);return -ENOMEM ; }@@ -233,17 +242,17 @@ int __inet_inherit_port(const struct sock *sk, struct sock *child) l3mdev =inet_sk_bound_l3mdev (sk);
bhash2_find:net/ipv4/inet_hashtables.c +1
-2- tb2 =inet_bind2_bucket_find (table, net, port, l3mdev, child, - &head_bhash2); + tb2 =inet_bind2_bucket_find (head2, table, net, port, l3mdev, child);if (!tb2) { tb2 = inet_bind2_bucket_create(table->bind2_bucket_cachep,net/ipv4/inet_hashtables.c +1
-1- net, head_bhash2, port, + net, head2, port,l3mdev, child); if (!tb2) goto error; } } inet_bind_hash(child, tb, tb2, port);net/ipv4/inet_hashtables.c +1
-0+spin_unlock (&head2->lock);spin_unlock(&head->lock); return 0;net/ipv4/inet_hashtables.c +0
-0@@ -251,6 +260,7 @@ int __inet_inherit_port(const struct sock *sk, struct sock *child)error:
if (created_inet_bind_bucket) inet_bind_bucket_destroy(table->bind_bucket_cachep, tb);net/ipv4/inet_hashtables.c +1
-0+spin_unlock (&head2->lock);spin_unlock(&head->lock); return -ENOMEM; }net/ipv4/inet_hashtables.c +5
-7@@ -771,24 +781,24 @@ static bool check_bind2_bucket_match(struct inet_bind2_bucket *tb, tb->l3mdev == l3mdev && tb->rcv_saddr == sk->sk_rcv_saddr; } -bool check_bind2_bucket_match_nulladdr (struct inet_bind2_bucket *tb, +bool check_bind2_bucket_match_addr_any (struct inet_bind2_bucket *tb,struct net *net,const unsigned short port,int l3mdev,const struct sock *sk) {#if IS_ENABLED (CONFIG_IPV6 ) -struct in6_addr nulladdr = {}; +struct in6_addr addr_any = {};if (sk->sk_family ==AF_INET6 )return net_eq (ib2_net (tb), net) && tb->port == port && tb->l3mdev == l3mdev && -ipv6_addr_equal (&tb->v6_rcv_saddr, &nulladdr); +ipv6_addr_equal (&tb->v6_rcv_saddr, &addr_any); else#endif return net_eq (ib2_net (tb), net) && tb->port == port && tb->l3mdev == l3mdev && tb->rcv_saddr ==0 ; } -static struct inet_bind2_hashbucket * +struct inet_bind_hashbucket *inet_bhashfn_portaddr (struct inet_hashinfo *hinfo,const struct sock *sk,const struct net *net,unsigned short port) {@@ -803,55 +813,21 @@ inet_bhashfn_portaddr(struct inet_hashinfo *hinfo, const struct sock *sk,return &hinfo->bhash2[hash & (hinfo->bhash_size -1 )]; } -/* This should only be called when the spinlock for the socket's corresponding -* bind_hashbucket is held -*/ +/* The socket's bhash2 hashbucket spinlock must be held when this is called */ struct
inet_bind2_bucket*net/ipv4/inet_hashtables.c +3
-3-inet_bind2_bucket_find(struct inet_hashinfo *hinfo, struct net *net, - const unsigned short port, int l3mdev, struct sock *sk, - struct inet_bind2_hashbucket **head) +inet_bind2_bucket_find(struct inet_bind_hashbucket *head, + struct inet_hashinfo *hinfo, struct net *net, + const unsigned short port, int l3mdev, struct sock *sk){ struct inet_bind2_bucket *bhash2 = NULL;net/ipv4/inet_hashtables.c +1
-3-struct inet_bind2_hashbucket *h; - h =inet_bhashfn_portaddr (hinfo, sk, net, port); -inet_bind_bucket_for_each (bhash2, &h->chain) { +inet_bind_bucket_for_each (bhash2, &head->chain)if (check_bind2_bucket_match(bhash2, net, port, l3mdev, sk)) break;net/ipv4/inet_hashtables.c +0
-4- } - -if (head) - *head = h;return bhash2; }net/ipv4/inet_hashtables.c +0
-26-/* the lock for the socket's corresponding bhash entry must be held */ -static int __inet_bhash2_update_saddr (struct sock *sk, -struct inet_hashinfo *hinfo, -struct net *net,int port,int l3mdev) -{ -struct inet_bind2_hashbucket *head2; -struct inet_bind2_bucket *tb2; - - tb2 =inet_bind2_bucket_find (hinfo, net, port, l3mdev, sk, - &head2); -if (!tb2) { - tb2 =inet_bind2_bucket_create (hinfo->bind2_bucket_cachep, - net, head2, port, l3mdev, sk); -if (!tb2) -return -ENOMEM ; - } - -/* Remove the socket's old entry from bhash2 */ -__sk_del_bind2_node (sk); - -sk_add_bind2_node (sk, &tb2->owners); -inet_csk (sk)->icsk_bind2_hash = tb2; - -return 0 ; -} -/* This should be called if/when a socket's rcv saddr changes after it has
* been binded.
*/net/ipv4/inet_hashtables.c +29
-11@@ -862,17 +838,31 @@ int inet_bhash2_update_saddr(struct sock *sk)struct inet_bind_hashbucket *head;int port =inet_sk (sk)->inet_num;struct net *net =sock_net (sk); -int err; +struct inet_bind2_bucket *tb2; - head = &hinfo->bhash[inet_bhashfn (net, port, hinfo->bhash_size)]; + head =inet_bhashfn_portaddr (hinfo, sk, net, port);spin_lock_bh (&head->lock); - err =__inet_bhash2_update_saddr (sk, hinfo, net, port, l3mdev); + tb2 =inet_bind2_bucket_find (head, hinfo, net, port, l3mdev, sk); +if (!tb2) { + tb2 =inet_bind2_bucket_create (hinfo->bind2_bucket_cachep, + net, head, port, l3mdev, sk); +if (!tb2) { +spin_unlock_bh (&head->lock); +return -ENOMEM ; + } + } + +/* Remove the socket's old entry from bhash2 */ +__sk_del_bind2_node (sk); + +sk_add_bind2_node (sk, &tb2->owners); +inet_csk (sk)->icsk_bind2_hash = tb2;spin_unlock_bh (&head->lock); -return err; +return 0 ; }/* RFC 6056 3.3.4. Algorithm 4: Double-Hash Port Selection Algorithm @@ -894,9 +884,8 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row, struct sock *, __u16, struct inet_timewait_sock **)) { struct inet_hashinfo *hinfo = death_row->hashinfo; +struct inet_bind_hashbucket *head, *head2;struct inet_timewait_sock *tw =NULL ; -struct inet_bind2_hashbucket *head2; -struct inet_bind_hashbucket *head;int port =inet_sk (sk)->inet_num;struct net *net =sock_net (sk);struct inet_bind2_bucket *tb2;@@ -907,8 +896,6 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row,int l3mdev;u32 index; - l3mdev =inet_sk_bound_l3mdev (sk); -if (port) { head = &hinfo->bhash[inet_bhashfn (net, port, hinfo->bhash_size)];@@ -917,8 +904,7 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row,spin_lock_bh (&head->lock);if (prev_inaddr_any) { - ret =__inet_bhash2_update_saddr (sk, hinfo, net, port, - l3mdev); + ret =inet_bhash2_update_saddr (sk);if (ret) {spin_unlock_bh (&head->lock);return ret;@@ -937,6 +923,8 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row,return ret; } + l3mdev =inet_sk_bound_l3mdev (sk); +inet_get_local_port_range (net, &low, &high); high++;/* [32768, 60999] -> [32768, 61000[ */ remaining = high - low;@@ -1006,7 +994,10 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row,/* Find the corresponding tb2 bucket since we need to * add the socket to the bhash2 table as well */ - tb2 =inet_bind2_bucket_find (hinfo, net, port, l3mdev, sk, &head2); + head2 =inet_bhashfn_portaddr (hinfo, sk, net, port); +spin_lock (&head2->lock); + + tb2 =inet_bind2_bucket_find (head2, hinfo, net, port, l3mdev, sk);if (!tb2) { tb2 =inet_bind2_bucket_create (hinfo->bind2_bucket_cachep, net, head2, port, l3mdev, sk);@@ -1024,6 +1015,9 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row,/* Head lock still held and bh's disabled */ inet_bind_hash (sk, tb, tb2, port); + +spin_unlock (&head2->lock); +if (sk_unhashed (sk)) {inet_sk (sk)->inet_sport =htons (port);inet_ehash_nolisten (sk, (struct sock *)tw,NULL );@@ -1037,6 +1031,7 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row,return 0 ;error:
net/ipv4/inet_hashtables.c +1
-0+spin_unlock_bh (&head2->lock);if (tb_created) inet_bind_bucket_destroy(hinfo->bind_bucket_cachep, tb); spin_unlock_bh(&head->lock);net/ipv4/tcp.c +3
-4@@ -4633,8 +4633,7 @@ void __init tcp_init(void)panic ("TCP: failed to alloc ehash_locks" ); tcp_hashinfo.bhash = alloc_large_system_hash("TCP bind bhash tables", - sizeof(struct inet_bind_hashbucket) + - sizeof(struct inet_bind2_hashbucket), + 2 * sizeof(struct inet_bind_hashbucket), tcp_hashinfo.ehash_mask + 1, 17, /* one slot per 128 KB of memory */ 0,@@ -4643,11 +4642,11 @@ void __init tcp_init(void)0 , 64 * 1024); tcp_hashinfo.bhash_size =1U << tcp_hashinfo.bhash_size; - tcp_hashinfo.bhash2 = - (struct inet_bind2_hashbucket *)(tcp_hashinfo.bhash + tcp_hashinfo.bhash_size); + tcp_hashinfo.bhash2 = tcp_hashinfo.bhash + tcp_hashinfo.bhash_size;for (i =0 ; i < tcp_hashinfo.bhash_size; i++) {spin_lock_init (&tcp_hashinfo.bhash[i].lock);INIT_HLIST_HEAD (&tcp_hashinfo.bhash[i].chain); +spin_lock_init (&tcp_hashinfo.bhash2[i].lock);INIT_HLIST_HEAD (&tcp_hashinfo.bhash2[i].chain); }Signature · 1 line
2.30.2
--
Mat Martineau
Intel
Paolo Abeni
maintainerRe: [PATCH net-next v3 2/3] net: Add bhash2 hashbucket locks
Quoted 188 lines · Mat Martineau
Quoted 156 lines · Joanne Koong
Currently, the bhash2 hashbucket uses its corresponding bhash hashbucket's lock for serializing concurrent accesses. There, however, can be the case where the bhash2 hashbucket is accessed concurrently by multiple processes that hash to different bhash hashbuckets but to the same bhash2 hashbucket.
As such, each bhash2 hashbucket will need to have its own lock instead of using its corresponding bhash hashbucket's lock.
Fixes d5a42de8bdbe ("net: Add a second bind table hashed by port and address")Signed-off-by Joanne Koong <[email protected]>---
include/net/inet_hashtables.h | 25 +++---- net/dccp/proto.c | 3 +- net/ipv4/inet_connection_sock.c | 60 +++++++++------- net/ipv4/inet_hashtables.c | 119 +++++++++++++++----------------- net/ipv4/tcp.c | 7 +- 5 files changed, 107 insertions(+), 107 deletions(-)include/net/inet_hashtables.h +1
-10@@ -124,15 +124,6 @@ struct inet_bind_hashbucket {struct hlist_head chain; }; -/* This is synchronized using the inet_bind_hashbucket's spinlock. -* Instead of having separate spinlocks, the inet_bind2_hashbucket can share -* the inet_bind_hashbucket's given that in every case where the bhash2 table -* is useful, a lookup in the bhash table also occurs. -*/ -struct inet_bind2_hashbucket { -struct hlist_head chain; -}; -/* Sockets can be hashed in established or listening table. * We must use different 'nulls' end-of-chain value for all hash buckets : * A socket might transition from ESTABLISH to LISTEN state without @@ -169,7 +160,7 @@ struct inet_hashinfo {* conflicts. */ struct kmem_cache *bind2_bucket_cachep; -struct inet_bind2_hashbucket *bhash2; +struct inet_bind_hashbucket *bhash2;unsigned int bhash_size;/* The 2nd listener table hashed by local port and address */ @@ -240,7 +231,7 @@ static inline bool check_bind_bucket_match(struct inet_bind_bucket *tb,struct
inet_bind2_bucket* inet_bind2_bucket_create(structkmem_cache*cachep, struct net *net,include/net/inet_hashtables.h +1
-1- struct inet_bind2_hashbucket *head, + struct inet_bind_hashbucket *head,const unsigned short port, int l3mdev, const struct sock *sk);include/net/inet_hashtables.h +0
-0@@ -248,12 +239,12 @@ void inet_bind2_bucket_destroy(struct kmem_cache *cachep, struct inet_bind2_bucket *tb);struct
inet_bind2_bucket*include/net/inet_hashtables.h +2
-1-inet_bind2_bucket_find(struct inet_hashinfo *hinfo, struct net *net, +inet_bind2_bucket_find(struct inet_bind_hashbucket *head, + struct inet_hashinfo *hinfo, struct net *net,const unsigned short port, int l3mdev,
include/net/inet_hashtables.h +2
-3-struct sock *sk, - struct inet_bind2_hashbucket **head); + struct sock *sk); -bool check_bind2_bucket_match_nulladdr(struct inet_bind2_bucket *tb, +bool check_bind2_bucket_match_addr_any(struct inet_bind2_bucket *tb,struct net *net, const unsigned short port, int l3mdev,
include/net/inet_hashtables.h +4
-0@@ -265,6 +256,10 @@ static inline u32 inet_bhashfn(const struct net *net, const __u16 lport,return (lport +net_hash_mix (net)) & (bhash_size -1 ); } +struct inet_bind_hashbucket * +inet_bhashfn_portaddr (struct inet_hashinfo *hinfo,const struct sock *sk, +const struct net *net,unsigned short port); +void inet_bind_hash (struct sock *sk,struct inet_bind_bucket *tb,struct inet_bind2_bucket *tb2,const unsigned short snum);net/dccp/proto.c +2
-1@@ -1182,7 +1182,7 @@ static int __init dccp_init(void) goto out_free_dccp_locks; } - dccp_hashinfo.bhash2 = (struct inet_bind2_hashbucket *) + dccp_hashinfo.bhash2 = (struct inet_bind_hashbucket *)__get_free_pages (GFP_ATOMIC | __GFP_NOWARN, bhash_order);if (!dccp_hashinfo.bhash2) {@@ -1193,6 +1193,7 @@ static int __init dccp_init(void)for (i =0 ; i < dccp_hashinfo.bhash_size; i++) {spin_lock_init (&dccp_hashinfo.bhash[i].lock);INIT_HLIST_HEAD (&dccp_hashinfo.bhash[i].chain); +spin_lock_init (&dccp_hashinfo.bhash2[i].lock);INIT_HLIST_HEAD (&dccp_hashinfo.bhash2[i].chain); }net/ipv4/inet_connection_sock.c +7
-8@@ -131,14 +131,14 @@ static bool use_bhash2_on_bind(const struct sock *sk)return sk->sk_rcv_saddr !=htonl (INADDR_ANY ); } -static u32 get_bhash2_nulladdr_hash (const struct sock *sk,struct net *net, +static u32 get_bhash2_addr_any_hash (const struct sock *sk,struct net *net,int port) {#if IS_ENABLED (CONFIG_IPV6 ) -struct in6_addr nulladdr = {}; +struct in6_addr addr_any = {};if (sk->sk_family ==AF_INET6 ) -return ipv6_portaddr_hash (net, &nulladdr, port); +return ipv6_portaddr_hash (net, &addr_any, port);#endif return ipv4_portaddr_hash (net,0 , port); }@@ -204,18 +204,18 @@ static bool check_bhash2_conflict(const struct sock *sk,return false; } -/* This should be called only when the corresponding inet_bind_bucket spinlock -* is held -*/ +/* This should be called only when the tb and tb2 hashbuckets' locks are held */ static int inet_csk_bind_conflict (const struct sock *sk,int port,struct inet_bind_bucket *tb,struct inet_bind2_bucket *tb2,/* may be null */ +struct inet_bind_hashbucket *head_tb2,bool relax,bool reuseport_ok) {struct inet_hashinfo *hinfo = sk->sk_prot->h.hashinfo;kuid_t uid =sock_i_uid ((struct sock *)sk);struct sock_reuseport *reuseport_cb; -struct inet_bind2_hashbucket *head2; +struct inet_bind_hashbucket *head_addr_any; +bool addr_any_conflict = false;bool reuseport_cb_ok;struct sock *sk2;struct net *net;@@ -254,33 +254,39 @@ static int inet_csk_bind_conflict(const struct sock *sk, int port,/* check there's no conflict with an existing IPV6_ADDR_ANY (if ipv6) or * INADDR_ANY (if ipv4) socket. */ - hash =get_bhash2_nulladdr_hash (sk, net, port);net/ipv4/inet_connection_sock.c static int inet_csk_bind_conflict(const struct sock *sk, int port, · line 263
Quoted 10 lines
net/ipv4/inet_connection_sock.c +5
-3- head2 = &hinfo->bhash2[hash & (hinfo->bhash_size -1 )]; + hash =get_bhash2_addr_any_hash (sk, net, port); + head_addr_any = &hinfo->bhash2[hash & (hinfo->bhash_size -1 )]; l3mdev =inet_sk_bound_l3mdev (sk); -inet_bind_bucket_for_each (tb2, &head2->chain) -if (check_bind2_bucket_match_nulladdr (tb2, net, port, l3mdev, sk)) + +if (head_addr_any != head_tb2) +spin_lock_bh (&head_addr_any->lock);Hi Joanne -
syzkaller is consistently hitting a warning here (about 10x per minute):
============================================
WARNING: possible recursive locking detected
5.19.0-rc1-00382-g78347e8e15bf #1 Not tainted
--------------------------------------------
sshd/352 is trying to acquire lock:ffffc90000968640(&tcp_hashinfo.bhash2[i].lock){+.-.}-{2:2}, at:spin_lock_bhinclude/linux/spinlock.h:354[inline]ffffc90000968640(&tcp_hashinfo.bhash2[i].lock){+.-.}-{2:2}, at:inet_csk_bind_conflict+0x4c4/0x8e0net/ipv4/inet_connection_sock.c:263but task is already holding lock:
ffffc90000883d28(&tcp_hashinfo.bhash2[i].lock){+.-.}-{2:2}, at:spin_lock_bhinclude/linux/spinlock.h:354[inline]ffffc90000883d28(&tcp_hashinfo.bhash2[i].lock){+.-.}-{2:2}, at:inet_csk_get_port+0x528/0xea0net/ipv4/inet_connection_sock.c:497other info that might help us debug this:
Possible unsafe locking scenario:
Quoted 8 lines
CPU0 ---- lock(&tcp_hashinfo.bhash2[i].lock); lock(&tcp_hashinfo.bhash2[i].lock);*** DEADLOCK ***
May be due to missing lock nesting notation
This looks like a real deadlock scenario.
One dumb way of solving it would be always acquiring the bhash2 lock for head_addr_any and for port/addr, in a fixed order - e.g. lower hash first.
Anyway this fix looks not trivial. I'm wondering if we should consider a revert of the feature until a better/more robust design is ready?
Thanks
Paolo
Re: [PATCH net-next v3 2/3] net: Add bhash2 hashbucket locks
Quoted 199 lines · Paolo Abeni
Quoted 188 lines · Mat Martineau
Quoted 156 lines · Joanne Koong
Currently, the bhash2 hashbucket uses its corresponding bhash hashbucket's lock for serializing concurrent accesses. There, however, can be the case where the bhash2 hashbucket is accessed concurrently by multiple processes that hash to different bhash hashbuckets but to the same bhash2 hashbucket.
As such, each bhash2 hashbucket will need to have its own lock instead of using its corresponding bhash hashbucket's lock.
Fixes d5a42de8bdbe ("net: Add a second bind table hashed by port and address")Signed-off-by Joanne Koong <[email protected]>---
include/net/inet_hashtables.h | 25 +++---- net/dccp/proto.c | 3 +- net/ipv4/inet_connection_sock.c | 60 +++++++++------- net/ipv4/inet_hashtables.c | 119 +++++++++++++++----------------- net/ipv4/tcp.c | 7 +- 5 files changed, 107 insertions(+), 107 deletions(-)include/net/inet_hashtables.h +1
-10@@ -124,15 +124,6 @@ struct inet_bind_hashbucket {struct hlist_head chain; }; -/* This is synchronized using the inet_bind_hashbucket's spinlock. -* Instead of having separate spinlocks, the inet_bind2_hashbucket can share -* the inet_bind_hashbucket's given that in every case where the bhash2 table -* is useful, a lookup in the bhash table also occurs. -*/ -struct inet_bind2_hashbucket { -struct hlist_head chain; -}; -/* Sockets can be hashed in established or listening table. * We must use different 'nulls' end-of-chain value for all hash buckets : * A socket might transition from ESTABLISH to LISTEN state without @@ -169,7 +160,7 @@ struct inet_hashinfo {* conflicts. */ struct kmem_cache *bind2_bucket_cachep; -struct inet_bind2_hashbucket *bhash2; +struct inet_bind_hashbucket *bhash2;unsigned int bhash_size;/* The 2nd listener table hashed by local port and address */ @@ -240,7 +231,7 @@ static inline bool check_bind_bucket_match(struct inet_bind_bucket *tb,struct
inet_bind2_bucket* inet_bind2_bucket_create(structkmem_cache*cachep, struct net *net,include/net/inet_hashtables.h +1
-1-struct inet_bind2_hashbucket *head, +struct inet_bind_hashbucket *head, const unsigned short port, int l3mdev, const struct sock *sk);@@ -248,12 +239,12 @@ void inet_bind2_bucket_destroy(struct kmem_cache *cachep, struct inet_bind2_bucket *tb);struct
inet_bind2_bucket*include/net/inet_hashtables.h +8
-4-inet_bind2_bucket_find(struct inet_hashinfo *hinfo, struct net *net, +inet_bind2_bucket_find(struct inet_bind_hashbucket *head, + struct inet_hashinfo *hinfo, struct net *net, const unsigned short port, int l3mdev, - struct sock *sk, - struct inet_bind2_hashbucket **head); + struct sock *sk); -bool check_bind2_bucket_match_nulladdr(struct inet_bind2_bucket *tb, +bool check_bind2_bucket_match_addr_any(struct inet_bind2_bucket *tb, struct net *net, const unsigned short port, int l3mdev,@@ -265,6 +256,10 @@ static inline u32 inet_bhashfn(const struct net *net, const __u16 lport,return (lport +net_hash_mix (net)) & (bhash_size -1 ); } +struct inet_bind_hashbucket * +inet_bhashfn_portaddr (struct inet_hashinfo *hinfo,const struct sock *sk, +const struct net *net,unsigned short port); +void inet_bind_hash (struct sock *sk,struct inet_bind_bucket *tb,struct inet_bind2_bucket *tb2,const unsigned short snum);net/dccp/proto.c +2
-1@@ -1182,7 +1182,7 @@ static int __init dccp_init(void) goto out_free_dccp_locks; } - dccp_hashinfo.bhash2 = (struct inet_bind2_hashbucket *) + dccp_hashinfo.bhash2 = (struct inet_bind_hashbucket *)__get_free_pages (GFP_ATOMIC | __GFP_NOWARN, bhash_order);if (!dccp_hashinfo.bhash2) {@@ -1193,6 +1193,7 @@ static int __init dccp_init(void)for (i =0 ; i < dccp_hashinfo.bhash_size; i++) {spin_lock_init (&dccp_hashinfo.bhash[i].lock);INIT_HLIST_HEAD (&dccp_hashinfo.bhash[i].chain); +spin_lock_init (&dccp_hashinfo.bhash2[i].lock);INIT_HLIST_HEAD (&dccp_hashinfo.bhash2[i].chain); }net/ipv4/inet_connection_sock.c +7
-8@@ -131,14 +131,14 @@ static bool use_bhash2_on_bind(const struct sock *sk)return sk->sk_rcv_saddr !=htonl (INADDR_ANY ); } -static u32 get_bhash2_nulladdr_hash (const struct sock *sk,struct net *net, +static u32 get_bhash2_addr_any_hash (const struct sock *sk,struct net *net,int port) {#if IS_ENABLED (CONFIG_IPV6 ) -struct in6_addr nulladdr = {}; +struct in6_addr addr_any = {};if (sk->sk_family ==AF_INET6 ) -return ipv6_portaddr_hash (net, &nulladdr, port); +return ipv6_portaddr_hash (net, &addr_any, port);#endif return ipv4_portaddr_hash (net,0 , port); }@@ -204,18 +204,18 @@ static bool check_bhash2_conflict(const struct sock *sk,return false; } -/* This should be called only when the corresponding inet_bind_bucket spinlock -* is held -*/ +/* This should be called only when the tb and tb2 hashbuckets' locks are held */ static int inet_csk_bind_conflict (const struct sock *sk,int port,struct inet_bind_bucket *tb,struct inet_bind2_bucket *tb2,/* may be null */ +struct inet_bind_hashbucket *head_tb2,bool relax,bool reuseport_ok) {struct inet_hashinfo *hinfo = sk->sk_prot->h.hashinfo;kuid_t uid =sock_i_uid ((struct sock *)sk);struct sock_reuseport *reuseport_cb; -struct inet_bind2_hashbucket *head2; +struct inet_bind_hashbucket *head_addr_any; +bool addr_any_conflict = false;bool reuseport_cb_ok;struct sock *sk2;struct net *net;@@ -254,33 +254,39 @@ static int inet_csk_bind_conflict(const struct sock *sk, int port,/* check there's no conflict with an existing IPV6_ADDR_ANY (if ipv6) or * INADDR_ANY (if ipv4) socket. */ - hash =get_bhash2_nulladdr_hash (sk, net, port);net/ipv4/inet_connection_sock.c static int inet_csk_bind_conflict(const struct sock *sk, int port, · line 263
Quoted 10 lines
net/ipv4/inet_connection_sock.c +5
-3- head2 = &hinfo->bhash2[hash & (hinfo->bhash_size -1 )]; + hash =get_bhash2_addr_any_hash (sk, net, port); + head_addr_any = &hinfo->bhash2[hash & (hinfo->bhash_size -1 )]; l3mdev =inet_sk_bound_l3mdev (sk); -inet_bind_bucket_for_each (tb2, &head2->chain) -if (check_bind2_bucket_match_nulladdr (tb2, net, port, l3mdev, sk)) + +if (head_addr_any != head_tb2) +spin_lock_bh (&head_addr_any->lock);Hi Joanne -
syzkaller is consistently hitting a warning here (about 10x per minute):
============================================
WARNING: possible recursive locking detected
5.19.0-rc1-00382-g78347e8e15bf #1 Not tainted
--------------------------------------------
sshd/352 is trying to acquire lock:ffffc90000968640(&tcp_hashinfo.bhash2[i].lock){+.-.}-{2:2}, at:spin_lock_bhinclude/linux/spinlock.h:354[inline]ffffc90000968640(&tcp_hashinfo.bhash2[i].lock){+.-.}-{2:2}, at:inet_csk_bind_conflict+0x4c4/0x8e0net/ipv4/inet_connection_sock.c:263but task is already holding lock:
ffffc90000883d28(&tcp_hashinfo.bhash2[i].lock){+.-.}-{2:2}, at:spin_lock_bhinclude/linux/spinlock.h:354[inline]ffffc90000883d28(&tcp_hashinfo.bhash2[i].lock){+.-.}-{2:2}, at:inet_csk_get_port+0x528/0xea0net/ipv4/inet_connection_sock.c:497other info that might help us debug this:
Possible unsafe locking scenario:Quoted 8 lines
CPU0 ---- lock(&tcp_hashinfo.bhash2[i].lock); lock(&tcp_hashinfo.bhash2[i].lock);*** DEADLOCK ***
May be due to missing lock nesting notation
Quoted 9 lines
This looks like a real deadlock scenario.
One dumb way of solving it would be always acquiring the bhash2 lock for
head_addr_anyand for port/addr, in a fixed order - e.g. lower hash first.Anyway this fix looks not trivial. I'm wondering if we should consider a revert of the feature until a better/more robust design is ready?
Thanks for your feedback, Eric, Kuniyuki, Mat, and Paolo.
I think it's a good idea to revert bhash2 and then I will resubmit once all the fixes are in place. Please let me know if there's anything I need to do on my side to revert this.
Sorry for the inconveniences.
Quoted 4 lines
Thanks
Paolo
Re: [PATCH net-next v3 2/3] net: Add bhash2 hashbucket locks
Quoted 11 lines · Joanne Koong
Quoted 3 lines · Paolo Abeni
Anyway this fix looks not trivial. I'm wondering if we should consider a revert of the feature until a better/more robust design is ready?
Thanks for your feedback, Eric, Kuniyuki, Mat, and Paolo.
I think it's a good idea to revert bhash2 and then I will resubmit once all the fixes are in place. Please let me know if there's anything I need to do on my side to revert this.
I did not want to pressure you, we still have some time (maybe two weeks...)
Reverting d5a42de8bdbe should be trivial.
Quoted 2 lines
Sorry for the inconveniences.
No worries :)
Jakub Kicinski
maintainerRe: [PATCH net-next v3 2/3] net: Add bhash2 hashbucket locks
In reply to Joanne KoongQuoted 3 lines · On Tue, 14 Jun 2022 11:08:41 -0700 Joanne Koong wrote:
I think it's a good idea to revert bhash2 and then I will resubmit once all the fixes are in place. Please let me know if there's anything I need to do on my side to revert this.
We'd appreciate if you could post a revert patch and gather some of the history into the commit message (with some Links: to reports and postings of the fixes if possible).
PATCH v3 3/3 selftests/net: Add sk_bind_sendto_listen test
In reply to Joanne KoongThis patch adds a new test called sk_bind_sendto_listen.
This test exercises the path where a socket's rcv saddr changes after it has been added to the binding tables, and then a listen() on the socket is invoked. The listen() should succeed.
This test is copied over from one of syzbot's tests:
https://syzkaller.appspot.com/x/repro.c?x=1673a38df00000
--- tools/testing/selftests/net/.gitignore | 1 + tools/testing/selftests/net/Makefile | 1 + .../selftests/net/sk_bind_sendto_listen.c | 82 +++++++++++++++++++ 3 files changed, 84 insertions(+) create mode 100644 tools/testing/selftests/net/sk_bind_sendto_listen.c
tools/testing/selftests/net/.gitignore +1 -0
@@ -38,3 +38,4 @@ ioam6_parser toeplitz cmsg_sender bind_bhash_test +sk_bind_sendto_listen
tools/testing/selftests/net/Makefile +1 -0
@@ -60,6 +60,7 @@ TEST_GEN_FILES += cmsg_sender TEST_GEN_FILES += stress_reuseport_listen TEST_PROGS += test_vxlan_vnifiltering.sh TEST_GEN_FILES += bind_bhash_test +TEST_GEN_FILES += sk_bind_sendto_listen TEST_FILES := settings
tools/testing/selftests/net/sk_bind_sendto_listen.c new file +82 -0
82 lines not on this page · raw message
Signature · 1 line
2.30.2