inetpeer: reduce false sharing and atomic operations
Subsystems (3)
v2 cover inetpeer: reduce false sharing and atomic operations
After commit 8c2bd38b95f7 ("icmp: change the order of rate limits"), there is a risk that a host receiving packets from an unique source targeting closed ports is using a common inet_peer structure from many cpus.
All these cpus have to acquire/release a refcount and update the inet_peer timestamp (p->dtime)
Switch to pure RCU to avoid changing the refcount, and update p->dtime only once per jiffy.
Tested:
DUT : 128 cores, 32 hw rx queues.
receiving 8,400,000 UDP packets per second, targeting closed ports.
Before the series:
- napi poll can not keep up, NIC drops 1,200,000 packets per second.
- We use 20 % of cpu cycles
After this series:
- All packets are received (no more hw drops)
- We use 12 % of cpu cycles.
v2: addded Simon and Ido feedback from v1
v1: https://lore.kernel.org/netdev/[email protected]/T/#mc6b32422714235f8608580a7dbf464c203300578
Eric Dumazet (4):
inetpeer: remove create argument of inet_getpeer_v[46]()
inetpeer: remove create argument of inet_getpeer()
inetpeer: update inetpeer timestamp in inet_getpeer()
inetpeer: do not get a refcount in inet_getpeer()
include/net/inetpeer.h | 12 +++++------- net/ipv4/icmp.c | 9 ++++----- net/ipv4/inetpeer.c | 31 ++++++++----------------------- net/ipv4/ip_fragment.c | 15 ++++++++++----- net/ipv4/route.c | 17 +++++++++-------- net/ipv6/icmp.c | 6 +++--- net/ipv6/ip6_output.c | 6 +++--- net/ipv6/ndisc.c | 8 +++++--- 8 files changed, 47 insertions(+), 57 deletions(-)
Signature · 1 line
2.47.1.613.gc27f4b7a9f-goog
PATCH v2 1/4 inetpeer: remove create argument of inet_getpeer_v[46]()
All callers of inet_getpeer_v4() and inet_getpeer_v6() want to create an inetpeer.
--- include/net/inetpeer.h | 9 ++++----- net/ipv4/icmp.c | 2 +- net/ipv4/ip_fragment.c | 2 +- net/ipv4/route.c | 4 ++-- net/ipv6/icmp.c | 2 +- net/ipv6/ip6_output.c | 2 +- net/ipv6/ndisc.c | 2 +- 7 files changed, 11 insertions(+), 12 deletions(-)
include/net/inetpeer.h +4 -5
@@ -101,25 +101,24 @@ struct inet_peer *inet_getpeer(struct inet_peer_base *base,static inline struct inet_peer *inet_getpeer_v4 (struct inet_peer_base *base,__be32 v4daddr, -int vif,int create) +int vif) {struct inetpeer_addr daddr; daddr.a4.addr = v4daddr; daddr.a4.vif = vif; daddr.family =AF_INET ; -return inet_getpeer (base, &daddr, create); +return inet_getpeer (base, &daddr,1 ); }static inline struct inet_peer *inet_getpeer_v6 (struct inet_peer_base *base, -const struct in6_addr *v6daddr, -int create) +const struct in6_addr *v6daddr) {struct inetpeer_addr daddr; daddr.a6 = *v6daddr; daddr.family =AF_INET6 ; -return inet_getpeer (base, &daddr, create); +return inet_getpeer (base, &daddr,1 ); } static inline int inetpeer_addr_cmp(const struct inetpeer_addr *a,
net/ipv4/icmp.c +1 -1
@@ -322,7 +322,7 @@ static bool icmpv4_xrlim_allow(struct net *net, struct rtable *rt, goto out; vif =l3mdev_master_ifindex (dst->dev); - peer =inet_getpeer_v4 (net->ipv4.peers, fl4->daddr, vif,1 ); + peer =inet_getpeer_v4 (net->ipv4.peers, fl4->daddr, vif); rc =inet_peer_xrlim_allow (peer,READ_ONCE (net->ipv4.sysctl_icmp_ratelimit));if (peer)
net/ipv4/ip_fragment.c +1 -1
@@ -89,7 +89,7 @@ static void ip4_frag_init(struct inet_frag_queue *q, const void *a) q->key.v4 = *key; qp->ecn =0 ; qp->peer = q->fqdir->max_dist ? -inet_getpeer_v4 (net->ipv4.peers, key->saddr, key->vif,1 ) : +inet_getpeer_v4 (net->ipv4.peers, key->saddr, key->vif) :NULL ; }
net/ipv4/route.c +2 -2
@@ -873,7 +873,7 @@ void ip_rt_send_redirect(struct sk_buff *skb)rcu_read_unlock (); net =dev_net (rt->dst.dev); - peer =inet_getpeer_v4 (net->ipv4.peers,ip_hdr (skb)->saddr, vif,1 ); + peer =inet_getpeer_v4 (net->ipv4.peers,ip_hdr (skb)->saddr, vif);if (!peer) {icmp_send (skb,ICMP_REDIRECT ,ICMP_REDIR_HOST ,rt_nexthop (rt,ip_hdr (skb)->daddr));
@@ -976,7 +976,7 @@ static int ip_error(struct sk_buff *skb) } peer =inet_getpeer_v4 (net->ipv4.peers,ip_hdr (skb)->saddr, -l3mdev_master_ifindex (skb->dev),1 ); +l3mdev_master_ifindex (skb->dev)); send = true;if (peer) {
net/ipv6/icmp.c +1 -1
@@ -222,7 +222,7 @@ static bool icmpv6_xrlim_allow(struct sock *sk, u8 type,if (rt->rt6i_dst.plen <128 ) tmo >>= ((128 - rt->rt6i_dst.plen)>>5 ); - peer =inet_getpeer_v6 (net->ipv6.peers, &fl6->daddr,1 ); + peer =inet_getpeer_v6 (net->ipv6.peers, &fl6->daddr); res =inet_peer_xrlim_allow (peer, tmo);if (peer)inet_putpeer (peer);
net/ipv6/ip6_output.c +1 -1
@@ -613,7 +613,7 @@ int ip6_forward(struct sk_buff *skb)else target = &hdr->daddr; - peer =inet_getpeer_v6 (net->ipv6.peers, &hdr->daddr,1 ); + peer =inet_getpeer_v6 (net->ipv6.peers, &hdr->daddr);/* Limit redirects both by destination (here) and by source (inside ndisc_send_redirect)
net/ipv6/ndisc.c +1 -1
@@ -1731,7 +1731,7 @@ void ndisc_send_redirect(struct sk_buff *skb, const struct in6_addr *target) "Redirect: destination is not a neighbour\n"); goto release; } - peer =inet_getpeer_v6 (net->ipv6.peers, &ipv6_hdr (skb)->saddr,1 ); + peer =inet_getpeer_v6 (net->ipv6.peers, &ipv6_hdr (skb)->saddr); ret =inet_peer_xrlim_allow (peer,1 *HZ );if (peer)inet_putpeer (peer);
Signature · 1 line
2.47.1.613.gc27f4b7a9f-goog
PATCH v2 2/4 inetpeer: remove create argument of inet_getpeer()
In reply to Eric DumazetAll callers of inet_getpeer() want to create an inetpeer.
--- include/net/inetpeer.h | 7 +++---- net/ipv4/inetpeer.c | 11 ++--------- 2 files changed, 5 insertions(+), 13 deletions(-)
include/net/inetpeer.h +3 -4
@@ -96,8 +96,7 @@ static inline struct in6_addr *inetpeer_get_addr_v6(struct inetpeer_addr *iaddr)/* can be called with or without local BH being disabled */ struct inet_peer *inet_getpeer (struct inet_peer_base *base, -const struct inetpeer_addr *daddr, -int create); +const struct inetpeer_addr *daddr); static inline struct inet_peer *inet_getpeer_v4(struct inet_peer_base *base, __be32 v4daddr,
@@ -108,7 +107,7 @@ static inline struct inet_peer *inet_getpeer_v4(struct inet_peer_base *base, daddr.a4.addr = v4daddr; daddr.a4.vif = vif; daddr.family =AF_INET ; -return inet_getpeer (base, &daddr,1 ); +return inet_getpeer (base, &daddr); } static inline struct inet_peer *inet_getpeer_v6(struct inet_peer_base *base,
@@ -118,7 +117,7 @@ static inline struct inet_peer *inet_getpeer_v6(struct inet_peer_base *base, daddr.a6 = *v6daddr; daddr.family =AF_INET6 ; -return inet_getpeer (base, &daddr,1 ); +return inet_getpeer (base, &daddr); } static inline int inetpeer_addr_cmp(const struct inetpeer_addr *a,
net/ipv4/inetpeer.c +2 -9
@@ -169,13 +169,11 @@ static void inet_peer_gc(struct inet_peer_base *base, }struct inet_peer *inet_getpeer (struct inet_peer_base *base, -const struct inetpeer_addr *daddr, -int create) +const struct inetpeer_addr *daddr) {struct inet_peer *p, *gc_stack[PEER_MAX_GC ];struct rb_node **pp, *parent;unsigned int gc_cnt, seq; -int invalidated;/* Attempt a lockless lookup first. * Because of a concurrent writer, we might not find an existing entry.
@@ -183,16 +181,11 @@ struct inet_peer *inet_getpeer(struct inet_peer_base *base,rcu_read_lock (); seq =read_seqbegin (&base->lock); p =lookup (daddr, base, seq,NULL , &gc_cnt, &parent, &pp); - invalidated =read_seqretry (&base->lock, seq);rcu_read_unlock ();if (p)return p; -/* If no writer did a change during our lookup, we can return early. */ -if (!create && !invalidated) -return NULL ; -/* retry an exact lookup, taking the lock before. * At least, nodes should be hot in our cache. */
@@ -201,7 +194,7 @@ struct inet_peer *inet_getpeer(struct inet_peer_base *base, gc_cnt =0 ; p =lookup (daddr, base, seq, gc_stack, &gc_cnt, &parent, &pp); -if (!p && create) { +if (!p) { p =kmem_cache_alloc (peer_cachep,GFP_ATOMIC );if (p) { p->daddr = *daddr;
Signature · 1 line
2.47.1.613.gc27f4b7a9f-goog
PATCH v2 3/4 inetpeer: update inetpeer timestamp in inet_getpeer()
In reply to Eric Dumazetinet_putpeer() will be removed in the following patch, because we will no longer use refcounts.
Update inetpeer timestamp (p->dtime) at lookup time.
--- net/ipv4/inetpeer.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-)
net/ipv4/inetpeer.c +4 -8
@@ -95,6 +95,7 @@ static struct inet_peer *lookup(const struct inetpeer_addr *daddr, {struct rb_node **pp, *parent, *next;struct inet_peer *p; +u32 now; pp = &base->rb_root.rb_node; parent =NULL ;
@@ -110,6 +111,9 @@ static struct inet_peer *lookup(const struct inetpeer_addr *daddr,if (cmp ==0 ) {if (!refcount_inc_not_zero (&p->refcnt))break ; + now = jiffies; +if (READ_ONCE (p->dtime) != now) +WRITE_ONCE (p->dtime, now);return p; }if (gc_stack) {
@@ -150,9 +154,6 @@ static void inet_peer_gc(struct inet_peer_base *base,for (i =0 ; i < gc_cnt; i++) { p = gc_stack[i]; -/* The READ_ONCE() pairs with the WRITE_ONCE() -* in inet_putpeer() -*/ delta = (__u32 )jiffies -READ_ONCE (p->dtime);if (delta < ttl || !refcount_dec_if_one (&p->refcnt))
@@ -224,11 +225,6 @@ EXPORT_SYMBOL_GPL(inet_getpeer);void inet_putpeer (struct inet_peer *p) { -/* The WRITE_ONCE() pairs with itself (we run lockless) -* and the READ_ONCE() in inet_peer_gc() -*/ -WRITE_ONCE (p->dtime, (__u32 )jiffies); -if (refcount_dec_and_test (&p->refcnt))kfree_rcu (p, rcu); }
Signature · 1 line
2.47.1.613.gc27f4b7a9f-goog
PATCH v2 4/4 inetpeer: do not get a refcount in inet_getpeer()
In reply to Eric DumazetAll inet_getpeer() callers except ip4_frag_init() don't need to acquire a permanent refcount on the inetpeer.
They can switch to full RCU protection.
Move the refcount_inc_not_zero() into ip4_frag_init(), so that all the other callers no longer have to perform a pair of expensive atomic operations on a possibly contended cache line.
inet_putpeer() no longer needs to be exported.
After this patch, my DUT can receive 8,400,000 UDP packets per second targetting closed ports, using 50% less cpu cycles than before.
Also change two calls to l3mdev_master_ifindex() by l3mdev_master_ifindex_rcu() (Ido ideas)
--- net/ipv4/icmp.c | 9 ++++----- net/ipv4/inetpeer.c | 8 ++------ net/ipv4/ip_fragment.c | 15 ++++++++++----- net/ipv4/route.c | 15 ++++++++------- net/ipv6/icmp.c | 4 ++-- net/ipv6/ip6_output.c | 4 ++-- net/ipv6/ndisc.c | 6 ++++-- 7 files changed, 32 insertions(+), 29 deletions(-)
net/ipv4/icmp.c +4 -5
@@ -312,7 +312,6 @@ static bool icmpv4_xrlim_allow(struct net *net, struct rtable *rt,struct dst_entry *dst = &rt->dst;struct inet_peer *peer;bool rc = true; -int vif;if (!apply_ratelimit)return true;
@@ -321,12 +320,12 @@ static bool icmpv4_xrlim_allow(struct net *net, struct rtable *rt,if (dst->dev && (dst->dev->flags&IFF_LOOPBACK )) goto out; - vif =l3mdev_master_ifindex (dst->dev); - peer =inet_getpeer_v4 (net->ipv4.peers, fl4->daddr, vif); +rcu_read_lock (); + peer =inet_getpeer_v4 (net->ipv4.peers, fl4->daddr, +l3mdev_master_ifindex_rcu (dst->dev)); rc =inet_peer_xrlim_allow (peer,READ_ONCE (net->ipv4.sysctl_icmp_ratelimit)); -if (peer) -inet_putpeer (peer); +rcu_read_unlock (); out:if (!rc)__ICMP_INC_STATS (net,ICMP_MIB_RATELIMITHOST );
net/ipv4/inetpeer.c +2 -6
@@ -109,8 +109,6 @@ static struct inet_peer *lookup(const struct inetpeer_addr *daddr, p = rb_entry(parent, struct inet_peer, rb_node); cmp =inetpeer_addr_cmp (daddr, &p->daddr);if (cmp ==0 ) { -if (!refcount_inc_not_zero (&p->refcnt)) -break ; now = jiffies;if (READ_ONCE (p->dtime) != now)WRITE_ONCE (p->dtime, now);
@@ -169,6 +167,7 @@ static void inet_peer_gc(struct inet_peer_base *base, } } +/* Must be called under RCU : No refcount change is done here. */ struct inet_peer *inet_getpeer (struct inet_peer_base *base,const struct inetpeer_addr *daddr) {
@@ -179,10 +178,8 @@ struct inet_peer *inet_getpeer(struct inet_peer_base *base,/* Attempt a lockless lookup first. * Because of a concurrent writer, we might not find an existing entry. */ -rcu_read_lock (); seq =read_seqbegin (&base->lock); p =lookup (daddr, base, seq,NULL , &gc_cnt, &parent, &pp); -rcu_read_unlock ();if (p)return p;
@@ -200,7 +197,7 @@ struct inet_peer *inet_getpeer(struct inet_peer_base *base,if (p) { p->daddr = *daddr; p->dtime = (__u32 )jiffies; -refcount_set (&p->refcnt,2 ); +refcount_set (&p->refcnt,1 );atomic_set (&p->rid,0 ); p->metrics[RTAX_LOCK -1 ] =INETPEER_METRICS_NEW ; p->rate_tokens =0 ;
@@ -228,7 +225,6 @@ void inet_putpeer(struct inet_peer *p)if (refcount_dec_and_test (&p->refcnt))kfree_rcu (p, rcu); } -EXPORT_SYMBOL_GPL (inet_putpeer);/* * Check transmit rate limitation for given message.
net/ipv4/ip_fragment.c +10 -5
@@ -82,15 +82,20 @@ static int ip_frag_reasm(struct ipq *qp, struct sk_buff *skb,static void ip4_frag_init (struct inet_frag_queue *q,const void *a) { struct ipq *qp = container_of(q, struct ipq, q); -struct net *net = q->fqdir->net; -const struct frag_v4_compare_key *key = a; +struct net *net = q->fqdir->net; +struct inet_peer *p =NULL ; q->key.v4 = *key; qp->ecn =0 ; - qp->peer = q->fqdir->max_dist ? -inet_getpeer_v4 (net->ipv4.peers, key->saddr, key->vif) : -NULL ; +if (q->fqdir->max_dist) { +rcu_read_lock (); + p =inet_getpeer_v4 (net->ipv4.peers, key->saddr, key->vif); +if (p && !refcount_inc_not_zero (&p->refcnt)) + p =NULL ; +rcu_read_unlock (); + } + qp->peer = p; }static void ip4_frag_free (struct inet_frag_queue *q)
net/ipv4/route.c +8 -7
@@ -870,11 +870,11 @@ void ip_rt_send_redirect(struct sk_buff *skb) } log_martians =IN_DEV_LOG_MARTIANS (in_dev); vif =l3mdev_master_ifindex_rcu (rt->dst.dev); -rcu_read_unlock (); net =dev_net (rt->dst.dev); peer =inet_getpeer_v4 (net->ipv4.peers,ip_hdr (skb)->saddr, vif);if (!peer) { +rcu_read_unlock ();icmp_send (skb,ICMP_REDIRECT ,ICMP_REDIR_HOST ,rt_nexthop (rt,ip_hdr (skb)->daddr));return ;
@@ -893,7 +893,7 @@ void ip_rt_send_redirect(struct sk_buff *skb)*/ if (peer->n_redirects >= ip_rt_redirect_number) { peer->rate_last = jiffies; - goto out_put_peer; + goto out_unlock; }/* Check for load limit; set rate_last to the latest sent
@@ -914,8 +914,8 @@ void ip_rt_send_redirect(struct sk_buff *skb) &ip_hdr (skb)->saddr,inet_iif (skb), &ip_hdr(skb)->daddr, &gw); } -out_put_peer: -inet_putpeer (peer); +out_unlock: +rcu_read_unlock (); }static int ip_error (struct sk_buff *skb)
@@ -975,9 +975,9 @@ static int ip_error(struct sk_buff *skb)break ; } +rcu_read_lock (); peer =inet_getpeer_v4 (net->ipv4.peers,ip_hdr (skb)->saddr, -l3mdev_master_ifindex (skb->dev)); - +l3mdev_master_ifindex_rcu (skb->dev)); send = true;if (peer) { now = jiffies;
@@ -989,8 +989,9 @@ static int ip_error(struct sk_buff *skb) peer->rate_tokens -= ip_rt_error_cost; else send = false; -inet_putpeer (peer); } +rcu_read_unlock (); +if (send)icmp_send (skb,ICMP_DEST_UNREACH , code,0 );
net/ipv6/icmp.c +2 -2
@@ -222,10 +222,10 @@ static bool icmpv6_xrlim_allow(struct sock *sk, u8 type,if (rt->rt6i_dst.plen <128 ) tmo >>= ((128 - rt->rt6i_dst.plen)>>5 ); +rcu_read_lock (); peer =inet_getpeer_v6 (net->ipv6.peers, &fl6->daddr); res =inet_peer_xrlim_allow (peer, tmo); -if (peer) -inet_putpeer (peer); +rcu_read_unlock (); }if (!res) __ICMP6_INC_STATS(net, ip6_dst_idev(dst),
net/ipv6/ip6_output.c +2 -2
@@ -613,6 +613,7 @@ int ip6_forward(struct sk_buff *skb)else target = &hdr->daddr; +rcu_read_lock (); peer =inet_getpeer_v6 (net->ipv6.peers, &hdr->daddr);/* Limit redirects both by destination (here)
@@ -620,8 +621,7 @@ int ip6_forward(struct sk_buff *skb)*/ if (inet_peer_xrlim_allow (peer,1 *HZ ))ndisc_send_redirect (skb, target); -if (peer) -inet_putpeer (peer); +rcu_read_unlock (); }else {int addrtype =ipv6_addr_type (&hdr->saddr);
net/ipv6/ndisc.c +4 -2
@@ -1731,10 +1731,12 @@ void ndisc_send_redirect(struct sk_buff *skb, const struct in6_addr *target) "Redirect: destination is not a neighbour\n"); goto release; } + +rcu_read_lock (); peer =inet_getpeer_v6 (net->ipv6.peers, &ipv6_hdr (skb)->saddr); ret =inet_peer_xrlim_allow (peer,1 *HZ ); -if (peer) -inet_putpeer (peer); +rcu_read_unlock (); +if (!ret) goto release;
Signature · 1 line
2.47.1.613.gc27f4b7a9f-goog
This series was applied to netdev/net-next.git (main) by Jakub Kicinski <[email protected]>:
Hello:
This series was applied to netdev/net-next.git (main) by Jakub Kicinski <[email protected]>:
Quoted 9 lines · On Sun, 15 Dec 2024 17:56:25 +0000 you wrote:
After commit
8c2bd38b95f7("icmp: change the order of rate limits"), there is a risk that a host receiving packets from an unique source targeting closed ports is using a commoninet_peerstructure from many cpus.All these cpus have to acquire/release a refcount and update the
inet_peertimestamp (p->dtime)[...]
Here is the summary with links:
- [v2,net-next,1/4] inetpeer: remove create argument of inet_getpeer_v[46]()
https://git.kernel.org/netdev/net-next/c/661cd8fc8e90 - [v2,net-next,2/4] inetpeer: remove create argument of inet_getpeer() https://git.kernel.org/netdev/net-next/c/7a596a50c4a4 - [v2,net-next,3/4] inetpeer: update inetpeer timestamp in inet_getpeer() https://git.kernel.org/netdev/net-next/c/50b362f21d6c - [v2,net-next,4/4] inetpeer: do not get a refcount in inet_getpeer() https://git.kernel.org/netdev/net-next/c/a853c609504e
You are awesome, thank you!
Signature · 2 lines
Deet-doot-dot, I am a bot. https://korg.docs.kernel.org/patchwork/pwbot.html