raw: RCU conversion
Subsystems (2)
v2 cover raw: RCU conversion
From: Eric Dumazet <[email protected]>
Using rwlock in networking code is extremely risky. writers can starve if enough readers are constantly grabing the rwlock.
I thought rwlock were at fault and sent this patch:
https://lkml.org/lkml/2022/6/17/272
But Peter and Linus essentially told me rwlock had to be unfair.
We need to get rid of rwlock in networking stacks.
Without this conversion, following script triggers soft lockups:
for i in {1..48}
do
ping -f -n -q 127.0.0.1 &
sleep 0.1
done
Next step will be to convert ping sockets to RCU as well.
v2: small issue in first patch, detected by kernel bot
Polish second patch in net/ipv4/raw_diag.c
Eric Dumazet (2):
raw: use more conventional iterators raw: convert raw sockets to RCU
include/net/raw.h | 16 +++-- include/net/rawv6.h | 7 +- net/ipv4/af_inet.c | 2 + net/ipv4/raw.c | 162 +++++++++++++++++--------------------------- net/ipv4/raw_diag.c | 53 ++++++++------- net/ipv6/af_inet6.c | 3 + net/ipv6/raw.c | 119 ++++++++++++-------------------- 7 files changed, 157 insertions(+), 205 deletions(-)
Signature · 1 line
2.36.1.476.g0c4daa206d-goog
PATCH v2 1/2 raw: use more conventional iterators
From: Eric Dumazet <[email protected]>
In order to prepare the following patch,
I change raw v4 & v6 code to use more conventional iterators.
--- include/net/raw.h | 5 +-- include/net/rawv6.h | 6 +-- net/ipv4/raw.c | 91 +++++++++++++------------------------ net/ipv4/raw_diag.c | 33 +++++++------- net/ipv6/raw.c | 107 ++++++++++++++++---------------------------- 5 files changed, 92 insertions(+), 150 deletions(-)
include/net/raw.h +2 -3
@@ -20,9 +20,8 @@extern struct proto raw_prot;extern struct raw_hashinfo raw_v4_hashinfo; -struct sock *__raw_v4_lookup (struct net *net,struct sock *sk, -unsigned short num,__be32 raddr, -__be32 laddr,int dif,int sdif); +bool raw_v4_match (struct net *net,struct sock *sk,unsigned short num, +__be32 raddr,__be32 laddr,int dif,int sdif);int raw_abort (struct sock *sk,int err);void raw_icmp_error (struct sk_buff *,int ,u32 );
include/net/rawv6.h +3 -3
@@ -5,9 +5,9 @@#include <net/protocol.h> extern struct raw_hashinfo raw_v6_hashinfo; -struct sock *__raw_v6_lookup (struct net *net,struct sock *sk, -unsigned short num,const struct in6_addr *loc_addr, -const struct in6_addr *rmt_addr,int dif,int sdif); +bool raw_v6_match (struct net *net,struct sock *sk,unsigned short num, +const struct in6_addr *loc_addr, +const struct in6_addr *rmt_addr,int dif,int sdif);int raw_abort (struct sock *sk,int err);
net/ipv4/raw.c +32 -59
139 lines not on this page · raw message
net/ipv4/raw_diag.c +16 -17
@@ -34,31 +34,30 @@ raw_get_hashinfo(const struct inet_diag_req_v2 *r)* use helper to figure it out. */ -static struct sock *raw_lookup (struct net *net,struct sock *from, -const struct inet_diag_req_v2 *req) +static bool raw_lookup (struct net *net,struct sock *sk, +const struct inet_diag_req_v2 *req) {struct inet_diag_req_raw *r = (void *)req; -struct sock *sk =NULL ;if (r->sdiag_family ==AF_INET ) - sk =__raw_v4_lookup (net, from, r->sdiag_raw_protocol, - r->id.idiag_dst[0 ], - r->id.idiag_src[0 ], - r->id.idiag_if,0 ); +return raw_v4_match (net, sk, r->sdiag_raw_protocol, + r->id.idiag_dst[0 ], + r->id.idiag_src[0 ], + r->id.idiag_if,0 );#if IS_ENABLED (CONFIG_IPV6 ) else - sk =__raw_v6_lookup (net, from, r->sdiag_raw_protocol, - (const struct in6_addr *)r->id.idiag_src, - (const struct in6_addr *)r->id.idiag_dst, - r->id.idiag_if,0 ); + return raw_v6_match(net, sk, r->sdiag_raw_protocol, + (const struct in6_addr *)r->id.idiag_src, + (const struct in6_addr *)r->id.idiag_dst, + r->id.idiag_if,0 );#endif -return sk; +return false; }static struct sock *raw_sock_get (struct net *net,const struct inet_diag_req_v2 *r) {struct raw_hashinfo *hashinfo =raw_get_hashinfo (r); -struct sock *sk =NULL , *s; +struct sock *sk;int slot;if (IS_ERR (hashinfo))
@@ -66,9 +65,8 @@ static struct sock *raw_sock_get(struct net *net, const struct inet_diag_req_v2read_lock (&hashinfo->lock);for (slot =0 ; slot <RAW_HTABLE_SIZE ; slot++) { -sk_for_each (s, &hashinfo->ht[slot]) { - sk =raw_lookup (net, s, r); -if (sk) { +sk_for_each (sk, &hashinfo->ht[slot]) { +if (raw_lookup (net, sk, r)) {/* * Grab it and keep until we fill * diag meaage to be reported, so
@@ -81,10 +79,11 @@ static struct sock *raw_sock_get(struct net *net, const struct inet_diag_req_v2 } } } + sk =ERR_PTR (-ENOENT ); out_unlock:read_unlock (&hashinfo->lock); -return sk ? sk :ERR_PTR (-ENOENT ); +return sk; } static int raw_diag_dump_one(struct netlink_callback *cb,
net/ipv6/raw.c +39 -68
162 lines not on this page · raw message
Signature · 1 line
2.36.1.476.g0c4daa206d-goog
PATCH v2 2/2 raw: convert raw sockets to RCU
In reply to Eric DumazetFrom: Eric Dumazet <[email protected]>
Using rwlock in networking code is extremely risky. writers can starve if enough readers are constantly grabing the rwlock.
I thought rwlock were at fault and sent this patch:
https://lkml.org/lkml/2022/6/17/272
But Peter and Linus essentially told me rwlock had to be unfair.
We need to get rid of rwlock in networking code.
Without this fix, following script triggers soft lockups:
for i in {1..48}
do
ping -f -n -q 127.0.0.1 &
sleep 0.1
done
--- include/net/raw.h | 11 +++++- include/net/rawv6.h | 1 + net/ipv4/af_inet.c | 2 ++ net/ipv4/raw.c | 83 +++++++++++++++++++++------------------------ net/ipv4/raw_diag.c | 22 +++++++----- net/ipv6/af_inet6.c | 3 ++ net/ipv6/raw.c | 28 +++++++-------- 7 files changed, 80 insertions(+), 70 deletions(-)
include/net/raw.h +10 -1
@@ -33,9 +33,18 @@ int raw_rcv(struct sock *, struct sk_buff *);struct raw_hashinfo {rwlock_t lock; -struct hlist_head ht[RAW_HTABLE_SIZE ]; +struct hlist_nulls_head ht[RAW_HTABLE_SIZE ]; }; +static inline void raw_hashinfo_init (struct raw_hashinfo *hashinfo) +{ +int i; + +rwlock_init (&hashinfo->lock); +for (i =0 ; i <RAW_HTABLE_SIZE ; i++) +INIT_HLIST_NULLS_HEAD (&hashinfo->ht[i], i); +} +#ifdef CONFIG_PROC_FS int raw_proc_init (void );void raw_proc_exit (void );
include/net/rawv6.h +1 -0
@@ -3,6 +3,7 @@#define _NET_RAWV6_H#include <net/protocol.h> +#include <net/raw.h> extern struct raw_hashinfo raw_v6_hashinfo; bool raw_v6_match(struct net *net, struct sock *sk, unsigned short num,
net/ipv4/af_inet.c +2 -0
@@ -1929,6 +1929,8 @@ static int __init inet_init(void)sock_skb_cb_check_size (sizeof (struct inet_skb_parm )); +raw_hashinfo_init (&raw_v4_hashinfo); + rc =proto_register (&tcp_prot,1 );if (rc) goto out;
net/ipv4/raw.c +38 -45
193 lines not on this page · raw message
net/ipv4/raw_diag.c +13 -9
@@ -57,31 +57,32 @@ static bool raw_lookup(struct net *net, struct sock *sk,static struct sock *raw_sock_get (struct net *net,const struct inet_diag_req_v2 *r) {struct raw_hashinfo *hashinfo =raw_get_hashinfo (r); +struct hlist_nulls_head *hlist; +struct hlist_nulls_node *hnode;struct sock *sk;int slot;if (IS_ERR (hashinfo))return ERR_CAST (hashinfo); -read_lock (&hashinfo->lock); +rcu_read_lock ();for (slot =0 ; slot <RAW_HTABLE_SIZE ; slot++) { -sk_for_each (sk, &hashinfo->ht[slot]) { + hlist = &hashinfo->ht[slot]; +hlist_nulls_for_each_entry (sk, hnode, hlist, sk_nulls_node) {if (raw_lookup (net, sk, r)) {/* * Grab it and keep until we fill -* diag meaage to be reported, so +* diag message to be reported, so * caller should call sock_put then. -* We can do that because we're keeping -* hashinfo->lock here. */ -sock_hold (sk); - goto out_unlock; +if (refcount_inc_not_zero (&sk->sk_refcnt)) + goto out_unlock; } } } sk =ERR_PTR (-ENOENT ); out_unlock: -read_unlock (&hashinfo->lock); +rcu_read_unlock ();return sk; }
@@ -141,6 +142,8 @@ static void raw_diag_dump(struct sk_buff *skb, struct netlink_callback *cb,struct raw_hashinfo *hashinfo =raw_get_hashinfo (r);struct net *net =sock_net (skb->sk);struct inet_diag_dump_data *cb_data; +struct hlist_nulls_head *hlist; +struct hlist_nulls_node *hnode;int num, s_num, slot, s_slot;struct sock *sk =NULL ;struct nlattr *bc;
@@ -157,7 +160,8 @@ static void raw_diag_dump(struct sk_buff *skb, struct netlink_callback *cb,for (slot = s_slot; slot <RAW_HTABLE_SIZE ; s_num =0 , slot++) { num =0 ; -sk_for_each (sk, &hashinfo->ht[slot]) { + hlist = &hashinfo->ht[slot]; +hlist_nulls_for_each_entry (sk, hnode, hlist, sk_nulls_node) {struct inet_sock *inet =inet_sk (sk);if (!net_eq (sock_net (sk), net))
net/ipv6/af_inet6.c +3 -0
@@ -63,6 +63,7 @@#include <net/compat.h> #include <net/xfrm.h> #include <net/ioam6.h> +#include <net/rawv6.h> #include <linux/uaccess.h> #include <linux/mroute6.h>
@@ -1073,6 +1074,8 @@ static int __init inet6_init(void) goto out; } +raw_hashinfo_init (&raw_v6_hashinfo); + err =proto_register (&tcpv6_prot,1 );if (err) goto out;
net/ipv6/raw.c +13 -15
@@ -61,9 +61,7 @@#define ICMPV6_HDRLEN 4/* ICMPv6 header, RFC 4443 Section 2.1 */ -struct raw_hashinfo raw_v6_hashinfo = { - .lock =__RW_LOCK_UNLOCKED (raw_v6_hashinfo.lock), -}; +struct raw_hashinfo raw_v6_hashinfo;EXPORT_SYMBOL_GPL (raw_v6_hashinfo); bool raw_v6_match(struct net *net, struct sock *sk, unsigned short num,
@@ -143,9 +141,10 @@ EXPORT_SYMBOL(rawv6_mh_filter_unregister);static bool ipv6_raw_deliver (struct sk_buff *skb,int nexthdr) {struct net *net =dev_net (skb->dev); +struct hlist_nulls_head *hlist; +struct hlist_nulls_node *hnode;const struct in6_addr *saddr;const struct in6_addr *daddr; -struct hlist_head *head;struct sock *sk;bool delivered = false;__u8 hash;
@@ -154,11 +153,9 @@ static bool ipv6_raw_deliver(struct sk_buff *skb, int nexthdr) daddr = saddr +1 ; hash = nexthdr & (RAW_HTABLE_SIZE -1 ); - head = &raw_v6_hashinfo.ht[hash]; -if (hlist_empty (head)) -return false; -read_lock (&raw_v6_hashinfo.lock); - sk_for_each(sk, head) { + hlist = &raw_v6_hashinfo.ht[hash]; +rcu_read_lock (); + hlist_nulls_for_each_entry(sk, hnode, hlist, sk_nulls_node) { int filtered; if (!raw_v6_match(net, sk, nexthdr, daddr, saddr,
@@ -203,7 +200,7 @@ static bool ipv6_raw_deliver(struct sk_buff *skb, int nexthdr) } } } -read_unlock (&raw_v6_hashinfo.lock); +rcu_read_unlock ();return delivered; }
@@ -337,14 +334,15 @@ void raw6_icmp_error(struct sk_buff *skb, int nexthdr, {const struct in6_addr *saddr, *daddr;struct net *net =dev_net (skb->dev); -struct hlist_head *head; +struct hlist_nulls_head *hlist; +struct hlist_nulls_node *hnode;struct sock *sk;int hash; hash = nexthdr & (RAW_HTABLE_SIZE -1 ); - head = &raw_v6_hashinfo.ht[hash]; -read_lock (&raw_v6_hashinfo.lock); -sk_for_each (sk, head) { + hlist = &raw_v6_hashinfo.ht[hash]; +rcu_read_lock (); +hlist_nulls_for_each_entry (sk, hnode, hlist, sk_nulls_node) {/* Note: ipv6_hdr(skb) != skb->data */ const struct ipv6hdr *ip6h = (const struct ipv6hdr *)skb->data; saddr = &ip6h->saddr;
@@ -355,7 +353,7 @@ void raw6_icmp_error(struct sk_buff *skb, int nexthdr,continue ;rawv6_err (sk, skb,NULL , type, code, inner_offset, info); } -read_unlock (&raw_v6_hashinfo.lock); +rcu_read_unlock (); }static inline int rawv6_rcv_skb (struct sock *sk,struct sk_buff *skb)
Signature · 1 line
2.36.1.476.g0c4daa206d-goog
This series was applied to netdev/net-next.git (master) by David S. Miller <[email protected]>:
Hello:
This series was applied to netdev/net-next.git (master) by David S. Miller <[email protected]>:
Quoted 9 lines · On Fri, 17 Jun 2022 20:47:03 -0700 you wrote:
From: Eric Dumazet <[email protected]>
Using rwlock in networking code is extremely risky. writers can starve if enough readers are constantly grabing the rwlock.
I thought rwlock were at fault and sent this patch:
[...]
Here is the summary with links:
- [v2,net-next,1/2] raw: use more conventional iterators (no matching commit)
- [v2,net-next,2/2] raw: convert raw sockets to RCU
https://git.kernel.org/netdev/net-next/c/0daf07e52709
You are awesome, thank you!
Signature · 2 lines
Deet-doot-dot, I am a bot. https://korg.docs.kernel.org/patchwork/pwbot.html