Skip to contentSkip to conversation

net: add and use skb_unclone_keeptruesize() helper

PATCH Applied Eric Dumazet 3 participants 3 messages Nov 2, 2021 – Nov 3, 2021

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 net: add and use skb_unclone_keeptruesize() helper

From: Eric Dumazet <[email protected]>

While commit 097b9146c0e2 ("net: fix up truesize of cloned skb in skb_prepare_for_shift()") fixed immediate issues found when KFENCE was enabled/tested, there are still similar issues, when tcp_trim_head() hits KFENCE while the master skb is cloned.

This happens under heavy networking TX workloads, when the TX completion might be delayed after incoming ACK.

This patch fixes the WARNING in sk_stream_kill_queues when sk->sk_mem_queued/sk->sk_forward_alloc are not zero.

Fixes d3fb45f370d9 ("mm, kfence: insert KFENCE hooks for SLAB")
Cc Marco Elver <[email protected]>
---
 include/linux/skbuff.h | 16 ++++++++++++++++
 net/core/skbuff.c      | 14 +-------------
 net/ipv4/tcp_output.c  |  6 +++---
 3 files changed, 20 insertions(+), 16 deletions(-)
include/linux/skbuff.h +16 -0
index 841e2f0f5240ba9e210bb9a3fc1cbedc2162b2a8..b8c273af2910c780dcfbc8f18fc05e115089010b 100644
@@ -1671,6 +1671,22 @@ static inline int skb_unclone(struct sk_buff *skb, gfp_t pri)
 	return 0;
 }
 
+/* This variant of skb_unclone() makes sure skb->truesize is not changed */
+static inline int skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri)
+{
+	might_sleep_if(gfpflags_allow_blocking(pri));
+
+	if (skb_cloned(skb)) {
+		unsigned int save = skb->truesize;
+		int res;
+
+		res = pskb_expand_head(skb, 0, 0, pri);
+		skb->truesize = save;
+		return res;
+	}
+	return 0;
+}
+
 /**
  *	skb_header_cloned - is the header a clone
  *	@skb: buffer to check
net/core/skbuff.c +1 -13
index fe9358437380c826d6438efe939afc4b38135cff..38d7dee4bbe9e96a811ff9cfca33429b5f7dbff1 100644
@@ -3449,19 +3449,7 @@ EXPORT_SYMBOL(skb_split);
  */
 static int skb_prepare_for_shift(struct sk_buff *skb)
 {
-	int ret = 0;
-
-	if (skb_cloned(skb)) {
-		/* Save and restore truesize: pskb_expand_head() may reallocate
-		 * memory where ksize(kmalloc(S)) != ksize(kmalloc(S)), but we
-		 * cannot change truesize at this point.
-		 */
-		unsigned int save_truesize = skb->truesize;
-
-		ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
-		skb->truesize = save_truesize;
-	}
-	return ret;
+	return skb_unclone_keeptruesize(skb, GFP_ATOMIC);
 }
 
 /**
net/ipv4/tcp_output.c +3 -3
index 6d72f3ea48c4ef0d193ec804653e4d4321f3f20a..0492f6942778db21f855216bf4387682fb37091e 100644
@@ -1562,7 +1562,7 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue,
 		return -ENOMEM;
 	}
 
-	if (skb_unclone(skb, gfp))
+	if (skb_unclone_keeptruesize(skb, gfp))
 		return -ENOMEM;
 
 	/* Get a new skb... force flag on. */
@@ -1672,7 +1672,7 @@ int tcp_trim_head(struct sock *sk, struct sk_buff *skb, u32 len)
 {
 	u32 delta_truesize;
 
-	if (skb_unclone(skb, GFP_ATOMIC))
+	if (skb_unclone_keeptruesize(skb, GFP_ATOMIC))
 		return -ENOMEM;
 
 	delta_truesize = __pskb_trim_head(skb, len);
@@ -3184,7 +3184,7 @@ int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs)
 				 cur_mss, GFP_ATOMIC))
 			return -ENOMEM; /* We'll try again later. */
 	} else {
-		if (skb_unclone(skb, GFP_ATOMIC))
+		if (skb_unclone_keeptruesize(skb, GFP_ATOMIC))
 			return -ENOMEM;
 
 		diff = tcp_skb_pcount(skb);
Signature · 1 line
2.33.1.1089.g2158813163f-goog
Acked-by on v1 1/1
#1 lore raw
Quoted 12 lines · Eric Dumazet

From: Eric Dumazet <[email protected]>

While commit 097b9146c0e2 ("net: fix up truesize of cloned skb in skb_prepare_for_shift()") fixed immediate issues found when KFENCE was enabled/tested, there are still similar issues, when tcp_trim_head() hits KFENCE while the master skb is cloned.

This happens under heavy networking TX workloads, when the TX completion might be delayed after incoming ACK.

Quoted 6 lines

This patch fixes the WARNING in sk_stream_kill_queues when sk->sk_mem_queued/sk->sk_forward_alloc are not zero.

Fixes d3fb45f370d9 ("mm, kfence: insert KFENCE hooks for SLAB")
Cc Marco Elver <[email protected]>

Thanks.

Quoted 92 lines
---
 include/linux/skbuff.h | 16 ++++++++++++++++
 net/core/skbuff.c      | 14 +-------------
 net/ipv4/tcp_output.c  |  6 +++---
 3 files changed, 20 insertions(+), 16 deletions(-)
include/linux/skbuff.h +16 -0
index 841e2f0f5240ba9e210bb9a3fc1cbedc2162b2a8..b8c273af2910c780dcfbc8f18fc05e115089010b 100644
@@ -1671,6 +1671,22 @@ static inline int skb_unclone(struct sk_buff *skb, gfp_t pri)
        return 0;
 }
 
+/* This variant of skb_unclone() makes sure skb->truesize is not changed */
+static inline int skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri)
+{
+       might_sleep_if(gfpflags_allow_blocking(pri));
+
+       if (skb_cloned(skb)) {
+               unsigned int save = skb->truesize;
+               int res;
+
+               res = pskb_expand_head(skb, 0, 0, pri);
+               skb->truesize = save;
+               return res;
+       }
+       return 0;
+}
+
 /**
  *     skb_header_cloned - is the header a clone
  *     @skb: buffer to check
net/core/skbuff.c +1 -13
index fe9358437380c826d6438efe939afc4b38135cff..38d7dee4bbe9e96a811ff9cfca33429b5f7dbff1 100644
@@ -3449,19 +3449,7 @@ EXPORT_SYMBOL(skb_split);
  */
 static int skb_prepare_for_shift(struct sk_buff *skb)
 {
-       int ret = 0;
-
-       if (skb_cloned(skb)) {
-               /* Save and restore truesize: pskb_expand_head() may reallocate
-                * memory where ksize(kmalloc(S)) != ksize(kmalloc(S)), but we
-                * cannot change truesize at this point.
-                */
-               unsigned int save_truesize = skb->truesize;
-
-               ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
-               skb->truesize = save_truesize;
-       }
-       return ret;
+       return skb_unclone_keeptruesize(skb, GFP_ATOMIC);
 }
 
 /**
net/ipv4/tcp_output.c +3 -4
index 6d72f3ea48c4ef0d193ec804653e4d4321f3f20a..0492f6942778db21f855216bf4387682fb37091e 100644
@@ -1562,7 +1562,7 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue,
                return -ENOMEM;
        }
 
-       if (skb_unclone(skb, gfp))
+       if (skb_unclone_keeptruesize(skb, gfp))
                return -ENOMEM;
 
        /* Get a new skb... force flag on. */
@@ -1672,7 +1672,7 @@ int tcp_trim_head(struct sock *sk, struct sk_buff *skb, u32 len)
 {
        u32 delta_truesize;
 
-       if (skb_unclone(skb, GFP_ATOMIC))
+       if (skb_unclone_keeptruesize(skb, GFP_ATOMIC))
                return -ENOMEM;
 
        delta_truesize = __pskb_trim_head(skb, len);
@@ -3184,7 +3184,7 @@ int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs)
                                 cur_mss, GFP_ATOMIC))
                        return -ENOMEM; /* We'll try again later. */
        } else {
-               if (skb_unclone(skb, GFP_ATOMIC))
+               if (skb_unclone_keeptruesize(skb, GFP_ATOMIC))
                        return -ENOMEM;
 
                diff = tcp_skb_pcount(skb);
--

2.33.1.1089.g2158813163f-goog

#2 lore raw
In reply to Eric Dumazet
This patch was applied to netdev/net.git (master) by Jakub Kicinski <[email protected]>:

Hello:

This patch was applied to netdev/net.git (master)
by Jakub Kicinski <[email protected]>:

Quoted 9 lines · On Mon, 1 Nov 2021 17:45:55 -0700 you wrote:

From: Eric Dumazet <[email protected]>

While commit 097b9146c0e2 ("net: fix up truesize of cloned skb in skb_prepare_for_shift()") fixed immediate issues found when KFENCE was enabled/tested, there are still similar issues, when tcp_trim_head() hits KFENCE while the master skb is cloned.

[...]

Here is the summary with links:
- [net] net: add and use skb_unclone_keeptruesize() helper

    https://git.kernel.org/netdev/net/c/c4777efa751d

You are awesome, thank you!

Signature · 2 lines
Deet-doot-dot, I am a bot.
https://korg.docs.kernel.org/patchwork/pwbot.html