net: ip: refactor SG checks
Subsystems (3)
Jakub Kicinski
maintainerPATCH v2 1/2 net: ip: refactor SG checks
There is a number of rt->dst.dev->features & NETIF_F_SG checks scattered throughout the code. Shorten the lines by caching the result of this check.
--- net/ipv4/ip_output.c | 13 ++++++------- net/ipv6/ip6_output.c | 13 ++++++------- 2 files changed, 12 insertions(+), 14 deletions(-)
net/ipv4/ip_output.c +6 -7
@@ -981,12 +981,14 @@ static int __ip_append_data(struct sock *sk,unsigned int maxfraglen, fragheaderlen, maxnonfragsize;int csummode =CHECKSUM_NONE ;struct rtable *rt = (struct rtable *)cork->dst; +bool has_sg, paged, extra_uref = false;unsigned int wmem_alloc_delta =0 ; -bool paged, extra_uref = false;u32 tskey =0 ; skb =skb_peek_tail (queue); + has_sg = rt->dst.dev->features &NETIF_F_SG ; + exthdrlen = !skb ? rt->dst.header_len :0 ; mtu = cork->gso_size ?IP_MAX_MTU : cork->fragsize; paged = !!cork->gso_size;
@@ -1023,8 +1025,7 @@ static int __ip_append_data(struct sock *sk,if (!uarg)return -ENOBUFS ; extra_uref = !skb_zcopy (skb);/* only ref on new uarg */ -if (rt->dst.dev->features &NETIF_F_SG && - csummode ==CHECKSUM_PARTIAL ) { +if (has_sg && csummode ==CHECKSUM_PARTIAL ) { paged = true; }else { uarg->zerocopy =0 ;
@@ -1074,8 +1075,7 @@ static int __ip_append_data(struct sock *sk, fraglen = datalen + fragheaderlen; pagedlen =0 ; -if ((flags &MSG_MORE ) && - !(rt->dst.dev->features&NETIF_F_SG )) +if ((flags &MSG_MORE ) && !has_sg) alloclen = mtu;else if (!paged) alloclen = fraglen;
@@ -1174,8 +1174,7 @@ static int __ip_append_data(struct sock *sk,if (copy > length) copy = length; -if (!(rt->dst.dev->features&NETIF_F_SG ) && -skb_tailroom (skb) >= copy) { +if (!has_sg &&skb_tailroom (skb) >= copy) {unsigned int off; off = skb->len;
net/ipv6/ip6_output.c +6 -7
@@ -1444,8 +1444,8 @@ static int __ip6_append_data(struct sock *sk,struct ipv6_txoptions *opt = v6_cork->opt;int csummode =CHECKSUM_NONE ;unsigned int maxnonfragsize, headersize; +bool has_sg, paged, extra_uref = false;unsigned int wmem_alloc_delta =0 ; -bool paged, extra_uref = false; skb =skb_peek_tail (queue);if (!skb) {
@@ -1453,6 +1453,8 @@ static int __ip6_append_data(struct sock *sk, dst_exthdrlen = rt->dst.header_len - rt->rt6i_nfheader_len; } + has_sg = rt->dst.dev->features &NETIF_F_SG ; + paged = !!cork->gso_size; mtu = cork->gso_size ?IP6_MAX_MTU : cork->fragsize; orig_mtu = mtu;
@@ -1515,8 +1517,7 @@ static int __ip6_append_data(struct sock *sk,if (!uarg)return -ENOBUFS ; extra_uref = !skb_zcopy (skb);/* only ref on new uarg */ -if (rt->dst.dev->features &NETIF_F_SG && - csummode ==CHECKSUM_PARTIAL ) { +if (has_sg && csummode ==CHECKSUM_PARTIAL ) { paged = true; }else { uarg->zerocopy =0 ;
@@ -1582,8 +1583,7 @@ static int __ip6_append_data(struct sock *sk, fraglen = datalen + fragheaderlen; pagedlen =0 ; -if ((flags &MSG_MORE ) && - !(rt->dst.dev->features&NETIF_F_SG )) +if ((flags &MSG_MORE ) && !has_sg) alloclen = mtu;else if (!paged) alloclen = fraglen;
@@ -1698,8 +1698,7 @@ static int __ip6_append_data(struct sock *sk,if (copy > length) copy = length; -if (!(rt->dst.dev->features&NETIF_F_SG ) && -skb_tailroom (skb) >= copy) { +if (!has_sg &&skb_tailroom (skb) >= copy) {unsigned int off; off = skb->len;
Signature · 1 line
2.31.1
Jakub Kicinski
maintainerPATCH v2 2/2 net: ip: avoid OOM kills with large UDP sends over loopback
Dave observed number of machines hitting OOM on the UDP send path. The workload seems to be sending large UDP packets over loopback. Since loopback has MTU of 64k kernel will try to allocate an skb with up to 64k of head space. This has a good chance of failing under memory pressure. What's worse if the message length is <32k the allocation may trigger an OOM killer.
This is entirely avoidable, we can use an skb with frags.
af_unix solves a similar problem by limiting the head length to SKB_MAX_ALLOC. This seems like a good and simple approach. It means that UDP messages > 16kB will now use fragments if underlying device supports SG, if extra allocator pressure causes regressions in real workloads we can switch to trying the large allocation first and falling back.
--- net/ipv4/ip_output.c | 2 +- net/ipv6/ip6_output.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-)
net/ipv4/ip_output.c +1 -1
@@ -1077,7 +1077,7 @@ static int __ip_append_data(struct sock *sk,if ((flags &MSG_MORE ) && !has_sg) alloclen = mtu; -else if (!paged) +else if (!paged && (fraglen <SKB_MAX_ALLOC || !has_sg)) alloclen = fraglen;else { alloclen =min_t (int, fraglen,MAX_HEADER );
net/ipv6/ip6_output.c +1 -1
@@ -1585,7 +1585,7 @@ static int __ip6_append_data(struct sock *sk,if ((flags &MSG_MORE ) && !has_sg) alloclen = mtu; -else if (!paged) +else if (!paged && (fraglen <SKB_MAX_ALLOC || !has_sg)) alloclen = fraglen;else { alloclen =min_t (int, fraglen,MAX_HEADER );
Signature · 1 line
2.31.1
Re: [PATCH net-next v2 2/2] net: ip: avoid OOM kills with large UDP sends over loopback
Quoted 9 lines · Jakub Kicinski
Dave observed number of machines hitting OOM on the UDP send path. The workload seems to be sending large UDP packets over loopback. Since loopback has MTU of 64k kernel will try to allocate an skb with up to 64k of head space. This has a good chance of failing under memory pressure. What's worse if the message length is <32k the allocation may trigger an OOM killer.
This is entirely avoidable, we can use an skb with frags.
Are you referring to IP fragments, or page frags ?
Quoted 21 lines
af_unixsolves a similar problem by limiting the head length toSKB_MAX_ALLOC. This seems like a good and simple approach. It means that UDP messages > 16kB will now use fragments if underlying device supports SG, if extra allocator pressure causes regressions in real workloads we can switch to trying the large allocation first and falling back.Reported-by Dave Jones <[email protected]>Signed-off-by Jakub Kicinski <[email protected]>--- net/ipv4/ip_output.c | 2 +- net/ipv6/ip6_output.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-)net/ipv4/ip_output.c +0
-0
net/ipv4/ip_output.c static int __ip_append_data(struct sock *sk, · line 1080
Quoted 5 lines
net/ipv4/ip_output.c +1
-1@@ -1077,7 +1077,7 @@ static int __ip_append_data(struct sock *sk,if ((flags &MSG_MORE ) && !has_sg) alloclen = mtu; -else if (!paged) +else if (!paged && (fraglen <SKB_MAX_ALLOC || !has_sg))
This looks indeed better, but there are some boundary conditions, caused by the fact that we add hh_len+15 later when allocating the skb.
(I expect hh_len+15 being 31)
You probably need
else if (!paged && (fraglen + hh_len + 15 < SKB_MAX_ALLOC || !has_sg))
Otherwise we might still attempt order-3 allocations ?
SKB_MAX_ALLOC is 16064 currently (skb_shinfo size being 320 on 64bit arches)
An UDP message with 16034 bytes of payload would translate to alloclen==16062. If we add 28 bytes for UDP+IP headers, plus 31 bytes for hh_len+31 this would go to 16413, thus asking for 32768 bytes (order-3 page)
(16062+320 = 16382, which is smaller than 16384)
Quoted 17 lines
net/ipv4/ip_output.c +0
-0alloclen = fraglen; else { alloclen =min_t (int, fraglen,MAX_HEADER );net/ipv6/ip6_output.c +1
-1@@ -1585,7 +1585,7 @@ static int __ip6_append_data(struct sock *sk,if ((flags &MSG_MORE ) && !has_sg) alloclen = mtu; -else if (!paged) +else if (!paged && (fraglen <SKB_MAX_ALLOC || !has_sg)) alloclen = fraglen;else { alloclen =min_t (int, fraglen,MAX_HEADER );
Jakub Kicinski
maintainerRe: [PATCH net-next v2 2/2] net: ip: avoid OOM kills with large UDP sends over loopback
Quoted 9 lines · On Wed, 23 Jun 2021 16:25:18 +0200 Eric Dumazet wrote:
Quoted 8 lines · Jakub Kicinski
Dave observed number of machines hitting OOM on the UDP send path. The workload seems to be sending large UDP packets over loopback. Since loopback has MTU of 64k kernel will try to allocate an skb with up to 64k of head space. This has a good chance of failing under memory pressure. What's worse if the message length is <32k the allocation may trigger an OOM killer.
Quoted 3 lines
Quoted 1 line
This is entirely avoidable, we can use an skb with frags.
Are you referring to IP fragments, or page frags ?
page frags, annoyingly overloaded term. I'll say paged, it's not the common term but at least won't be confusing.
Quoted 37 lines
Quoted 20 lines
af_unixsolves a similar problem by limiting the head length toSKB_MAX_ALLOC. This seems like a good and simple approach. It means that UDP messages > 16kB will now use fragments if underlying device supports SG, if extra allocator pressure causes regressions in real workloads we can switch to trying the large allocation first and falling back.Reported-by Dave Jones <[email protected]>Signed-off-by Jakub Kicinski <[email protected]>--- net/ipv4/ip_output.c | 2 +- net/ipv6/ip6_output.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-)net/ipv4/ip_output.c +0
-0net/ipv4/ip_output.c static int __ip_append_data(struct sock *sk, · line 1080
Quoted 5 lines
net/ipv4/ip_output.c +1
-1@@ -1077,7 +1077,7 @@ static int __ip_append_data(struct sock *sk,if ((flags &MSG_MORE ) && !has_sg) alloclen = mtu; -else if (!paged) +else if (!paged && (fraglen <SKB_MAX_ALLOC || !has_sg))This looks indeed better, but there are some boundary conditions, caused by the fact that we add hh_len+15 later when allocating the skb.
(I expect hh_len+15 being 31)
You probably need
else if (!paged && (fraglen + hh_len + 15 < SKB_MAX_ALLOC || !has_sg))Otherwise we might still attempt order-3 allocations ?
Quoted 7 lines
SKB_MAX_ALLOCis 16064 currently (skb_shinfosize being 320 on 64bit arches)An UDP message with 16034 bytes of payload would translate to alloclen==16062. If we add 28 bytes for UDP+IP headers, plus 31 bytes for hh_len+31 this would go to 16413, thus asking for 32768 bytes (order-3 page)
(16062+320 = 16382, which is smaller than 16384)
Will do, thanks!
Quoted 17 lines
Quoted 17 lines
net/ipv4/ip_output.c +0
-0alloclen = fraglen; else { alloclen =min_t (int, fraglen,MAX_HEADER );net/ipv6/ip6_output.c +1
-1@@ -1585,7 +1585,7 @@ static int __ip6_append_data(struct sock *sk,if ((flags &MSG_MORE ) && !has_sg) alloclen = mtu; -else if (!paged) +else if (!paged && (fraglen <SKB_MAX_ALLOC || !has_sg)) alloclen = fraglen;else { alloclen =min_t (int, fraglen,MAX_HEADER );