Skip to contentSkip to conversation

net: ip: refactor SG checks

PATCH In review Jakub Kicinski 2 participants 4 messages Jun 22, 2021 – Jun 23, 2021

Subsystems (3)
lore
Share & tools
lore

Jakub Kicinski

author maintainer
#0 lore raw

PATCH v2 1/2 net: ip: refactor SG checks

There is a number of rt->dst.dev->features & NETIF_F_SG checks scattered throughout the code. Shorten the lines by caching the result of this check.

---
 net/ipv4/ip_output.c  | 13 ++++++-------
 net/ipv6/ip6_output.c | 13 ++++++-------
 2 files changed, 12 insertions(+), 14 deletions(-)
net/ipv4/ip_output.c +6 -7
index c3efc7d658f6..90031f5446bd 100644
@@ -981,12 +981,14 @@ static int __ip_append_data(struct sock *sk,
 	unsigned int maxfraglen, fragheaderlen, maxnonfragsize;
 	int csummode = CHECKSUM_NONE;
 	struct rtable *rt = (struct rtable *)cork->dst;
+	bool has_sg, paged, extra_uref = false;
 	unsigned int wmem_alloc_delta = 0;
-	bool paged, extra_uref = false;
 	u32 tskey = 0;
 
 	skb = skb_peek_tail(queue);
 
+	has_sg = rt->dst.dev->features & NETIF_F_SG;
+
 	exthdrlen = !skb ? rt->dst.header_len : 0;
 	mtu = cork->gso_size ? IP_MAX_MTU : cork->fragsize;
 	paged = !!cork->gso_size;
@@ -1023,8 +1025,7 @@ static int __ip_append_data(struct sock *sk,
 		if (!uarg)
 			return -ENOBUFS;
 		extra_uref = !skb_zcopy(skb);	/* only ref on new uarg */
-		if (rt->dst.dev->features & NETIF_F_SG &&
-		    csummode == CHECKSUM_PARTIAL) {
+		if (has_sg && csummode == CHECKSUM_PARTIAL) {
 			paged = true;
 		} else {
 			uarg->zerocopy = 0;
@@ -1074,8 +1075,7 @@ static int __ip_append_data(struct sock *sk,
 			fraglen = datalen + fragheaderlen;
 			pagedlen = 0;
 
-			if ((flags & MSG_MORE) &&
-			    !(rt->dst.dev->features&NETIF_F_SG))
+			if ((flags & MSG_MORE) && !has_sg)
 				alloclen = mtu;
 			else if (!paged)
 				alloclen = fraglen;
@@ -1174,8 +1174,7 @@ static int __ip_append_data(struct sock *sk,
 		if (copy > length)
 			copy = length;
 
-		if (!(rt->dst.dev->features&NETIF_F_SG) &&
-		    skb_tailroom(skb) >= copy) {
+		if (!has_sg && skb_tailroom(skb) >= copy) {
 			unsigned int off;
 
 			off = skb->len;
net/ipv6/ip6_output.c +6 -7
index ff4f9ebcf7f6..c667b7e2856f 100644
@@ -1444,8 +1444,8 @@ static int __ip6_append_data(struct sock *sk,
 	struct ipv6_txoptions *opt = v6_cork->opt;
 	int csummode = CHECKSUM_NONE;
 	unsigned int maxnonfragsize, headersize;
+	bool has_sg, paged, extra_uref = false;
 	unsigned int wmem_alloc_delta = 0;
-	bool paged, extra_uref = false;
 
 	skb = skb_peek_tail(queue);
 	if (!skb) {
@@ -1453,6 +1453,8 @@ static int __ip6_append_data(struct sock *sk,
 		dst_exthdrlen = rt->dst.header_len - rt->rt6i_nfheader_len;
 	}
 
+	has_sg = rt->dst.dev->features & NETIF_F_SG;
+
 	paged = !!cork->gso_size;
 	mtu = cork->gso_size ? IP6_MAX_MTU : cork->fragsize;
 	orig_mtu = mtu;
@@ -1515,8 +1517,7 @@ static int __ip6_append_data(struct sock *sk,
 		if (!uarg)
 			return -ENOBUFS;
 		extra_uref = !skb_zcopy(skb);	/* only ref on new uarg */
-		if (rt->dst.dev->features & NETIF_F_SG &&
-		    csummode == CHECKSUM_PARTIAL) {
+		if (has_sg && csummode == CHECKSUM_PARTIAL) {
 			paged = true;
 		} else {
 			uarg->zerocopy = 0;
@@ -1582,8 +1583,7 @@ static int __ip6_append_data(struct sock *sk,
 			fraglen = datalen + fragheaderlen;
 			pagedlen = 0;
 
-			if ((flags & MSG_MORE) &&
-			    !(rt->dst.dev->features&NETIF_F_SG))
+			if ((flags & MSG_MORE) && !has_sg)
 				alloclen = mtu;
 			else if (!paged)
 				alloclen = fraglen;
@@ -1698,8 +1698,7 @@ static int __ip6_append_data(struct sock *sk,
 		if (copy > length)
 			copy = length;
 
-		if (!(rt->dst.dev->features&NETIF_F_SG) &&
-		    skb_tailroom(skb) >= copy) {
+		if (!has_sg && skb_tailroom(skb) >= copy) {
 			unsigned int off;
 
 			off = skb->len;
Signature · 1 line
2.31.1
author maintainer
#1 lore raw

PATCH v2 2/2 net: ip: avoid OOM kills with large UDP sends over loopback

Dave observed number of machines hitting OOM on the UDP send path. The workload seems to be sending large UDP packets over loopback. Since loopback has MTU of 64k kernel will try to allocate an skb with up to 64k of head space. This has a good chance of failing under memory pressure. What's worse if the message length is <32k the allocation may trigger an OOM killer.

This is entirely avoidable, we can use an skb with frags.

af_unix solves a similar problem by limiting the head length to SKB_MAX_ALLOC. This seems like a good and simple approach. It means that UDP messages > 16kB will now use fragments if underlying device supports SG, if extra allocator pressure causes regressions in real workloads we can switch to trying the large allocation first and falling back.

---
 net/ipv4/ip_output.c  | 2 +-
 net/ipv6/ip6_output.c | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
net/ipv4/ip_output.c +1 -1
index 90031f5446bd..1ab140c173d0 100644
@@ -1077,7 +1077,7 @@ static int __ip_append_data(struct sock *sk,
 
 			if ((flags & MSG_MORE) && !has_sg)
 				alloclen = mtu;
-			else if (!paged)
+			else if (!paged && (fraglen < SKB_MAX_ALLOC || !has_sg))
 				alloclen = fraglen;
 			else {
 				alloclen = min_t(int, fraglen, MAX_HEADER);
net/ipv6/ip6_output.c +1 -1
index c667b7e2856f..46d805097a79 100644
@@ -1585,7 +1585,7 @@ static int __ip6_append_data(struct sock *sk,
 
 			if ((flags & MSG_MORE) && !has_sg)
 				alloclen = mtu;
-			else if (!paged)
+			else if (!paged && (fraglen < SKB_MAX_ALLOC || !has_sg))
 				alloclen = fraglen;
 			else {
 				alloclen = min_t(int, fraglen, MAX_HEADER);
Signature · 1 line
2.31.1
#2 lore raw

Re: [PATCH net-next v2 2/2] net: ip: avoid OOM kills with large UDP sends over loopback

Quoted 9 lines · Jakub Kicinski

Dave observed number of machines hitting OOM on the UDP send path. The workload seems to be sending large UDP packets over loopback. Since loopback has MTU of 64k kernel will try to allocate an skb with up to 64k of head space. This has a good chance of failing under memory pressure. What's worse if the message length is <32k the allocation may trigger an OOM killer.

This is entirely avoidable, we can use an skb with frags.

Are you referring to IP fragments, or page frags ?

Quoted 21 lines

af_unix solves a similar problem by limiting the head length to SKB_MAX_ALLOC. This seems like a good and simple approach. It means that UDP messages > 16kB will now use fragments if underlying device supports SG, if extra allocator pressure causes regressions in real workloads we can switch to trying the large allocation first and falling back.

---
 net/ipv4/ip_output.c  | 2 +-
 net/ipv6/ip6_output.c | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
net/ipv4/ip_output.c +0 -0
index 90031f5446bd..1ab140c173d0 100644

net/ipv4/ip_output.c static int __ip_append_data(struct sock *sk, · line 1080

Quoted 5 lines
net/ipv4/ip_output.c +1 -1
@@ -1077,7 +1077,7 @@ static int __ip_append_data(struct sock *sk,
 
 			if ((flags & MSG_MORE) && !has_sg)
 				alloclen = mtu;
-			else if (!paged)
+			else if (!paged && (fraglen < SKB_MAX_ALLOC || !has_sg))

This looks indeed better, but there are some boundary conditions, caused by the fact that we add hh_len+15 later when allocating the skb.

(I expect hh_len+15 being 31)

You probably need

	else if (!paged && (fraglen + hh_len + 15 < SKB_MAX_ALLOC || !has_sg))

Otherwise we might still attempt order-3 allocations ?

SKB_MAX_ALLOC is 16064 currently (skb_shinfo size being 320 on 64bit arches)

An UDP message with 16034 bytes of payload would translate to alloclen==16062. If we add 28 bytes for UDP+IP headers, plus 31 bytes for hh_len+31 this would go to 16413, thus asking for 32768 bytes (order-3 page)

(16062+320 = 16382, which is smaller than 16384)

Quoted 17 lines
net/ipv4/ip_output.c +0 -0
 				alloclen = fraglen;
 			else {
 				alloclen = min_t(int, fraglen, MAX_HEADER);
net/ipv6/ip6_output.c +1 -1
index c667b7e2856f..46d805097a79 100644
@@ -1585,7 +1585,7 @@ static int __ip6_append_data(struct sock *sk,
 
 			if ((flags & MSG_MORE) && !has_sg)
 				alloclen = mtu;
-			else if (!paged)
+			else if (!paged && (fraglen < SKB_MAX_ALLOC || !has_sg))
 				alloclen = fraglen;
 			else {
 				alloclen = min_t(int, fraglen, MAX_HEADER);
author maintainer
#3 lore raw

Re: [PATCH net-next v2 2/2] net: ip: avoid OOM kills with large UDP sends over loopback

Quoted 9 lines · On Wed, 23 Jun 2021 16:25:18 +0200 Eric Dumazet wrote:
Quoted 8 lines · Jakub Kicinski

Dave observed number of machines hitting OOM on the UDP send path. The workload seems to be sending large UDP packets over loopback. Since loopback has MTU of 64k kernel will try to allocate an skb with up to 64k of head space. This has a good chance of failing under memory pressure. What's worse if the message length is <32k the allocation may trigger an OOM killer.

Quoted 3 lines
Quoted 1 line

This is entirely avoidable, we can use an skb with frags.

Are you referring to IP fragments, or page frags ?

page frags, annoyingly overloaded term. I'll say paged, it's not the common term but at least won't be confusing.

Quoted 37 lines
Quoted 20 lines

af_unix solves a similar problem by limiting the head length to SKB_MAX_ALLOC. This seems like a good and simple approach. It means that UDP messages > 16kB will now use fragments if underlying device supports SG, if extra allocator pressure causes regressions in real workloads we can switch to trying the large allocation first and falling back.

---
 net/ipv4/ip_output.c  | 2 +-
 net/ipv6/ip6_output.c | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
net/ipv4/ip_output.c +0 -0
index 90031f5446bd..1ab140c173d0 100644

net/ipv4/ip_output.c static int __ip_append_data(struct sock *sk, · line 1080

Quoted 5 lines
net/ipv4/ip_output.c +1 -1
@@ -1077,7 +1077,7 @@ static int __ip_append_data(struct sock *sk,
 
 			if ((flags & MSG_MORE) && !has_sg)
 				alloclen = mtu;
-			else if (!paged)
+			else if (!paged && (fraglen < SKB_MAX_ALLOC || !has_sg))  

This looks indeed better, but there are some boundary conditions, caused by the fact that we add hh_len+15 later when allocating the skb.

(I expect hh_len+15 being 31)

You probably need

	else if (!paged && (fraglen + hh_len + 15 < SKB_MAX_ALLOC || !has_sg))

Otherwise we might still attempt order-3 allocations ?

Quoted 7 lines

SKB_MAX_ALLOC is 16064 currently (skb_shinfo size being 320 on 64bit arches)

An UDP message with 16034 bytes of payload would translate to alloclen==16062. If we add 28 bytes for UDP+IP headers, plus 31 bytes for hh_len+31 this would go to 16413, thus asking for 32768 bytes (order-3 page)

(16062+320 = 16382, which is smaller than 16384)

Will do, thanks!

Quoted 17 lines
Quoted 17 lines
net/ipv4/ip_output.c +0 -0
 				alloclen = fraglen;
 			else {
 				alloclen = min_t(int, fraglen, MAX_HEADER);
net/ipv6/ip6_output.c +1 -1
index c667b7e2856f..46d805097a79 100644
@@ -1585,7 +1585,7 @@ static int __ip6_append_data(struct sock *sk,
 
 			if ((flags & MSG_MORE) && !has_sg)
 				alloclen = mtu;
-			else if (!paged)
+			else if (!paged && (fraglen < SKB_MAX_ALLOC || !has_sg))
 				alloclen = fraglen;
 			else {
 				alloclen = min_t(int, fraglen, MAX_HEADER);