inet: add READ_ONCE(sk->sk_bound_dev_if) in INET_MATCH()
Subsystems (5)
PATCH v2 1/1 inet: add READ_ONCE(sk->sk_bound_dev_if) in INET_MATCH()
From: Eric Dumazet <[email protected]>
INET_MATCH() runs without holding a lock on the socket.
We probably need to annotate most reads.
This patch makes INET_MATCH() an inline function to ease our changes.
v2:
We remove the 32bit version of it, as modern compilers should generate the same code really, no need to try to be smarter.
Also make 'struct net *net' the first argument.
---
Sent as a standalone patch to not spam netdev@ list.
include/net/inet_hashtables.h | 33 +++++++++++++++------------------ include/net/sock.h | 3 --- net/ipv4/inet_hashtables.c | 15 +++++---------- net/ipv4/udp.c | 3 +-- 4 files changed, 21 insertions(+), 33 deletions(-)
include/net/inet_hashtables.h +15 -18
@@ -295,7 +295,6 @@ static inline struct sock *inet_lookup_listener(struct net *net, ((__force __portpair)(((__u32)(__dport) << 16) | (__force __u32)(__be16)(__sport)))#endif -#if (BITS_PER_LONG ==64 )#ifdef __BIG_ENDIAN#define INET_ADDR_COOKIE (__name, __saddr, __daddr) \ const __addrpair __name = (__force __addrpair) ( \
@@ -307,24 +306,22 @@ static inline struct sock *inet_lookup_listener(struct net *net, (((__force __u64)(__be32)(__daddr)) << 32) | \ ((__force __u64)(__be32)(__saddr)))#endif /* __BIG_ENDIAN */ -#define INET_MATCH (__sk, __net, __cookie, __saddr, __daddr, __ports, __dif, __sdif) \ - (((__sk)->sk_portpair == (__ports)) && \ - ((__sk)->sk_addrpair == (__cookie)) && \ - (((__sk)->sk_bound_dev_if == (__dif)) || \ - ((__sk)->sk_bound_dev_if == (__sdif))) && \ - net_eq(sock_net(__sk), (__net))) -#else /* 32-bit arch */ -#define INET_ADDR_COOKIE (__name, __saddr, __daddr) \ - const int __name __deprecated __attribute__((unused)) -#define INET_MATCH (__sk, __net, __cookie, __saddr, __daddr, __ports, __dif, __sdif) \ - (((__sk)->sk_portpair == (__ports)) && \ - ((__sk)->sk_daddr == (__saddr)) && \ - ((__sk)->sk_rcv_saddr == (__daddr)) && \ - (((__sk)->sk_bound_dev_if == (__dif)) || \ - ((__sk)->sk_bound_dev_if == (__sdif))) && \ - net_eq(sock_net(__sk), (__net))) -#endif /* 64-bit arch */ +static inline bool INET_MATCH (struct net *net,const struct sock *sk, +const __addrpair cookie,const __portpair ports, +int dif,int sdif) +{ +int bound_dev_if; + +if (!net_eq (sock_net (sk), net) || + sk->sk_portpair != ports || + sk->sk_addrpair != cookie) +return false; + +/* Paired with WRITE_ONCE() from sock_bindtoindex_locked() */ + bound_dev_if =READ_ONCE (sk->sk_bound_dev_if); +return bound_dev_if == dif || bound_dev_if == sdif; +}/* Sockets in TCP_CLOSE state are _always_ taken out of the hash, so we need * not check it for lookups anymore, thanks Alexey. -DaveM
include/net/sock.h +0 -3
@@ -161,9 +161,6 @@ typedef __u64 __bitwise __addrpair;* for struct sock and struct inet_timewait_sock. */ struct sock_common { - /* skc_daddr and skc_rcv_saddr must be grouped on a 8 bytes aligned - * address on 64bit arches : cf INET_MATCH() - */ union { __addrpair skc_addrpair; struct {
net/ipv4/inet_hashtables.c +5 -10
@@ -410,13 +410,11 @@ struct sock *__inet_lookup_established(struct net *net,sk_nulls_for_each_rcu (sk, node, &head->chain) {if (sk->sk_hash != hash)continue ; -if (likely (INET_MATCH (sk, net, acookie, - saddr, daddr, ports, dif, sdif))) { +if (likely (INET_MATCH (net, sk, acookie, ports, dif, sdif))) {if (unlikely (!refcount_inc_not_zero (&sk->sk_refcnt))) goto out; -if (unlikely (!INET_MATCH (sk, net, acookie, - saddr, daddr, ports, - dif, sdif))) { +if (unlikely (!INET_MATCH (net, sk, acookie, + ports, dif, sdif))) {sock_gen_put (sk); goto begin; }
@@ -465,8 +463,7 @@ static int __inet_check_established(struct inet_timewait_death_row *death_row,if (sk2->sk_hash != hash)continue ; -if (likely (INET_MATCH (sk2, net, acookie, - saddr, daddr, ports, dif, sdif))) { +if (likely (INET_MATCH (net, sk2, acookie, ports, dif, sdif))) {if (sk2->sk_state ==TCP_TIME_WAIT ) { tw =inet_twsk (sk2);if (twsk_unique (sk, sk2, twp))
@@ -532,9 +529,7 @@ static bool inet_ehash_lookup_by_sk(struct sock *sk,if (esk->sk_hash != sk->sk_hash)continue ;if (sk->sk_family ==AF_INET ) { -if (unlikely (INET_MATCH (esk, net, acookie, - sk->sk_daddr, - sk->sk_rcv_saddr, +if (unlikely (INET_MATCH (net, esk, acookie, ports, dif, sdif))) {return true; }
net/ipv4/udp.c +1 -2
@@ -2563,8 +2563,7 @@ static struct sock *__udp4_lib_demux_lookup(struct net *net,struct sock *sk;udp_portaddr_for_each_entry_rcu (sk, &hslot2->head) { -if (INET_MATCH (sk, net, acookie, rmt_addr, - loc_addr, ports, dif, sdif)) +if (INET_MATCH (net, sk, acookie, ports, dif, sdif))return sk;/* Only check first socket in chain */ break ;
Signature · 1 line
2.36.0.550.gb090851708-goog
Quoted 53 lines · Eric Dumazet
From: Eric Dumazet <[email protected]>
INET_MATCH()runs without holding a lock on the socket.We probably need to annotate most reads.
This patch makes
INET_MATCH()an inline function to ease our changes.v2:
We remove the 32bit version of it, as modern compilers should generate the same code really, no need to try to be smarter.
Also make 'struct net *net' the first argument.
Signed-off-by Eric Dumazet <[email protected]>---
Sent as a standalone patch to not spam netdev@ list.
include/net/inet_hashtables.h | 33 +++++++++++++++------------------ include/net/sock.h | 3 --- net/ipv4/inet_hashtables.c | 15 +++++---------- net/ipv4/udp.c | 3 +-- 4 files changed, 21 insertions(+), 33 deletions(-)include/net/inet_hashtables.h +0
-10@@ -295,7 +295,6 @@ static inline struct sock *inet_lookup_listener(struct net *net, ((__force __portpair)(((__u32)(__dport) << 16) | (__force __u32)(__be16)(__sport)))#endif -#if (BITS_PER_LONG ==64 )#ifdef __BIG_ENDIAN#define INET_ADDR_COOKIE (__name, __saddr, __daddr) \ const __addrpair __name = (__force __addrpair) ( \@@ -307,24 +306,22 @@ static inline struct sock *inet_lookup_listener(struct net *net, (((__force __u64)(__be32)(__daddr)) << 32) | \ ((__force __u64)(__be32)(__saddr)))#endif /* __BIG_ENDIAN */ -#define INET_MATCH (__sk, __net, __cookie, __saddr, __daddr, __ports, __dif, __sdif) \ - (((__sk)->sk_portpair == (__ports)) && \ - ((__sk)->sk_addrpair == (__cookie)) && \ - (((__sk)->sk_bound_dev_if == (__dif)) || \ - ((__sk)->sk_bound_dev_if == (__sdif))) && \ - net_eq(sock_net(__sk), (__net))) -#else /* 32-bit arch */ -#define INET_ADDR_COOKIE (__name, __saddr, __daddr) \ - const int __name __deprecated __attribute__((unused))
include/net/inet_hashtables.h static inline struct sock *inet_lookup_listener(struct net *net, · line 310
Quoted 10 lines
include/net/inet_hashtables.h +1
-8-#define INET_MATCH (__sk, __net, __cookie, __saddr, __daddr, __ports, __dif, __sdif) \ - (((__sk)->sk_portpair == (__ports)) && \ - ((__sk)->sk_daddr == (__saddr)) && \ - ((__sk)->sk_rcv_saddr == (__daddr)) && \ - (((__sk)->sk_bound_dev_if == (__dif)) || \ - ((__sk)->sk_bound_dev_if == (__sdif))) && \ - net_eq(sock_net(__sk), (__net))) -#endif /* 64-bit arch */ +static inline bool INET_MATCH(struct net *net, const struct sock *sk,
When you convert the #define into an inline function, wouldn't it be more natural to name it lower caps?
static inline bool inet_match(struct net *net, ... )
Best,
Oliver
Quoted 87 lines
include/net/inet_hashtables.h +14
-0+ const __addrpair cookie, const __portpair ports, + int dif, int sdif) +{ + int bound_dev_if; + +if (!net_eq (sock_net (sk), net) || + sk->sk_portpair != ports || + sk->sk_addrpair != cookie) +return false; + +/* Paired with WRITE_ONCE() from sock_bindtoindex_locked() */ + bound_dev_if =READ_ONCE (sk->sk_bound_dev_if); +return bound_dev_if == dif || bound_dev_if == sdif; +}/* Sockets in TCP_CLOSE state are _always_ taken out of the hash, so we need * not check it for lookups anymore, thanks Alexey. -DaveM include/net/sock.h +0
-3@@ -161,9 +161,6 @@ typedef __u64 __bitwise __addrpair;* for struct sock and struct inet_timewait_sock. */ struct sock_common { - /* skc_daddr and skc_rcv_saddr must be grouped on a 8 bytes aligned - * address on 64bit arches : cf INET_MATCH() - */ union { __addrpair skc_addrpair; struct {net/ipv4/inet_hashtables.c +5
-10@@ -410,13 +410,11 @@ struct sock *__inet_lookup_established(struct net *net,sk_nulls_for_each_rcu (sk, node, &head->chain) {if (sk->sk_hash != hash)continue ; -if (likely (INET_MATCH (sk, net, acookie, - saddr, daddr, ports, dif, sdif))) { +if (likely (INET_MATCH (net, sk, acookie, ports, dif, sdif))) {if (unlikely (!refcount_inc_not_zero (&sk->sk_refcnt))) goto out; -if (unlikely (!INET_MATCH (sk, net, acookie, - saddr, daddr, ports, - dif, sdif))) { +if (unlikely (!INET_MATCH (net, sk, acookie, + ports, dif, sdif))) {sock_gen_put (sk); goto begin; }@@ -465,8 +463,7 @@ static int __inet_check_established(struct inet_timewait_death_row *death_row,if (sk2->sk_hash != hash)continue ; -if (likely (INET_MATCH (sk2, net, acookie, - saddr, daddr, ports, dif, sdif))) { +if (likely (INET_MATCH (net, sk2, acookie, ports, dif, sdif))) {if (sk2->sk_state ==TCP_TIME_WAIT ) { tw =inet_twsk (sk2);if (twsk_unique (sk, sk2, twp))@@ -532,9 +529,7 @@ static bool inet_ehash_lookup_by_sk(struct sock *sk,if (esk->sk_hash != sk->sk_hash)continue ;if (sk->sk_family ==AF_INET ) { -if (unlikely (INET_MATCH (esk, net, acookie, - sk->sk_daddr, - sk->sk_rcv_saddr, +if (unlikely (INET_MATCH (net, esk, acookie, ports, dif, sdif))) {return true; }net/ipv4/udp.c +1
-2@@ -2563,8 +2563,7 @@ static struct sock *__udp4_lib_demux_lookup(struct net *net,struct sock *sk;udp_portaddr_for_each_entry_rcu (sk, &hslot2->head) { -if (INET_MATCH (sk, net, acookie, rmt_addr, - loc_addr, ports, dif, sdif)) +if (INET_MATCH (net, sk, acookie, ports, dif, sdif))return sk;/* Only check first socket in chain */ break ;
Quoted 4 lines · Oliver Hartkopp
When you convert the #define into an inline function, wouldn't it be more natural to name it lower caps?
static inline bool inet_match(struct net *net, ... )
Sure, it is only a matter for us to remember all the past/present names, based on implementation details, especially at backport times.
Jakub Kicinski
maintainerQuoted 9 lines · On Thu, 12 May 2022 10:14:23 -0700 Eric Dumazet wrote:
Quoted 4 lines · Oliver Hartkopp
When you convert the #define into an inline function, wouldn't it be more natural to name it lower caps?
static inline bool inet_match(struct net *net, ... )
Sure, it is only a matter for us to remember all the past/present names, based on implementation details, especially at backport times.
We can apply as is if you prefer, but I'm not sure I follow TBH. The prototype (arguments) of the function/macro have changed so there is nothing to be gained from not changing the name AFAICT, no?
Quoted 7 lines · Jakub Kicinski
Quoted 5 lines · On Thu, 12 May 2022 10:14:23 -0700 Eric Dumazet wrote:
Quoted 3 lines · Oliver Hartkopp
When you convert the #define into an inline function, wouldn't it be more natural to name it lower caps?
Quoted 8 lines
Quoted 4 lines
Quoted 1 line
static inline bool inet_match(struct net *net, ... )
Sure, it is only a matter for us to remember all the past/present names, based on implementation details, especially at backport times.
We can apply as is if you prefer, but I'm not sure I follow TBH. The prototype (arguments) of the function/macro have changed so there is nothing to be gained from not changing the name AFAICT, no?
This was applied by David, I will send the v2 of the series, and a patch renaming INET_MATCH().
Thanks.