tcp: tx path fully headless
Subsystems (3)
cover tcp: tx path fully headless
This series completes transition of TCP stack tx path to headless packets : All payload now reside in page frags, never in skb->head.
Eric Dumazet (3):
tcp: let tcp_send_syn_data() build headless packets
tcp: remove some dead code
tcp: remove size parameter from tcp_stream_alloc_skb()
include/net/tcp.h | 3 +- net/ipv4/tcp.c | 8 ++--- net/ipv4/tcp_output.c | 77 +++++++++++++++++++------------------------ 3 files changed, 40 insertions(+), 48 deletions(-)
Signature · 1 line
2.41.0.162.gfafddb0af9-goog
PATCH 1/3 tcp: let tcp_send_syn_data() build headless packets
tcp_send_syn_data() is the last component in TCP transmit path to put payload in skb->head.
Switch it to use page frags, so that we can remove dead code later.
This allows to put more payload than previous implementation.
--- include/net/tcp.h | 1 + net/ipv4/tcp.c | 2 +- net/ipv4/tcp_output.c | 31 +++++++++++++++++++------------ 3 files changed, 21 insertions(+), 13 deletions(-)
include/net/tcp.h +1 -0
@@ -333,6 +333,7 @@ int tcp_sendpage(struct sock *sk, struct page *page, int offset, size_t size,int tcp_sendpage_locked (struct sock *sk,struct page *page,int offset,size_t size,int flags);int tcp_send_mss (struct sock *sk,int *size_goal,int flags); +int tcp_wmem_schedule (struct sock *sk,int copy);void tcp_push (struct sock *sk,int flags,int mss_now,int nonagle,int size_goal);void tcp_release_cb (struct sock *sk);
net/ipv4/tcp.c +1 -1
@@ -957,7 +957,7 @@ static int tcp_downgrade_zcopy_pure(struct sock *sk, struct sk_buff *skb) } -static int tcp_wmem_schedule (struct sock *sk,int copy) +int tcp_wmem_schedule (struct sock *sk,int copy) {int left;
net/ipv4/tcp_output.c +19 -12
@@ -3802,8 +3802,9 @@ static int tcp_send_syn_data(struct sock *sk, struct sk_buff *syn)struct inet_connection_sock *icsk =inet_csk (sk);struct tcp_sock *tp =tcp_sk (sk);struct tcp_fastopen_request *fo = tp->fastopen_req; -int space, err =0 ; +struct page_frag *pfrag =sk_page_frag (sk);struct sk_buff *syn_data; +int space, err =0 ; tp->rx_opt.mss_clamp = tp->advmss;/* If MSS is not cached */ if (!tcp_fastopen_cookie_check (sk, &tp->rx_opt.mss_clamp, &fo->cookie))
@@ -3822,25 +3823,31 @@ static int tcp_send_syn_data(struct sock *sk, struct sk_buff *syn) space =min_t (size_t, space, fo->size); -/* limit to order-0 allocations */ - space =min_t (size_t, space,SKB_MAX_HEAD (MAX_TCP_HEADER )); - - syn_data =tcp_stream_alloc_skb (sk, space, sk->sk_allocation, false); +if (space && + !skb_page_frag_refill (min_t (size_t, space,PAGE_SIZE ), + pfrag, sk->sk_allocation)) + goto fallback; + syn_data =tcp_stream_alloc_skb (sk,0 , sk->sk_allocation, false);if (!syn_data) goto fallback;memcpy (syn_data->cb, syn->cb,sizeof (syn->cb));if (space) { -int copied =copy_from_iter (skb_put (syn_data, space), space, - &fo->data->msg_iter); -if (unlikely (!copied)) { + space =min_t (size_t, space, pfrag->size - pfrag->offset); + space =tcp_wmem_schedule (sk, space); + } +if (space) { + space =copy_page_from_iter (pfrag->page, pfrag->offset, + space, &fo->data->msg_iter); +if (unlikely (!space)) {tcp_skb_tsorted_anchor_cleanup (syn_data);kfree_skb (syn_data); goto fallback; } -if (copied != space) { -skb_trim (syn_data, copied); - space = copied; - } +skb_fill_page_desc (syn_data,0 , pfrag->page, + pfrag->offset, space); +page_ref_inc (pfrag->page); + pfrag->offset += space; +skb_len_add (syn_data, space);skb_zcopy_set (syn_data, fo->uarg,NULL ); }/* No more data pending in inet_wait_for_connect() */
Signature · 1 line
2.41.0.162.gfafddb0af9-goog
PATCH 2/3 tcp: remove some dead code
In reply to Eric DumazetNow all skbs in write queue do not contain any payload in skb->head, we can remove some dead code.
--- net/ipv4/tcp_output.c | 40 ++++++++++++---------------------------- 1 file changed, 12 insertions(+), 28 deletions(-)
net/ipv4/tcp_output.c +12 -28
@@ -1530,7 +1530,7 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue, {struct tcp_sock *tp =tcp_sk (sk);struct sk_buff *buff; -int nsize, old_factor; +int old_factor;long limit;int nlen;u8 flags;
@@ -1538,9 +1538,7 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue,if (WARN_ON (len > skb->len))return -EINVAL ; - nsize =skb_headlen (skb) - len; -if (nsize <0 ) - nsize =0 ; +DEBUG_NET_WARN_ON_ONCE (skb_headlen (skb));/* tcp_sendmsg() can overshoot sk_wmem_queued by one full size skb. * We need some allowance to not penalize applications setting small
@@ -1560,7 +1558,7 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue,return -ENOMEM ;/* Get a new skb... force flag on. */ - buff =tcp_stream_alloc_skb (sk, nsize, gfp, true); + buff =tcp_stream_alloc_skb (sk,0 , gfp, true);if (!buff)return -ENOMEM ;/* We'll just try again later. */ skb_copy_decrypted (buff, skb);
@@ -1568,7 +1566,7 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue,sk_wmem_queued_add (sk, buff->truesize);sk_mem_charge (sk, buff->truesize); - nlen = skb->len - len - nsize; + nlen = skb->len - len; buff->truesize += nlen; skb->truesize -= nlen;
@@ -1626,13 +1624,7 @@ static int __pskb_trim_head(struct sk_buff *skb, int len)struct skb_shared_info *shinfo;int i, k, eat; - eat =min_t (int, len,skb_headlen (skb)); -if (eat) { -__skb_pull (skb, eat); - len -= eat; -if (!len) -return 0 ; - } +DEBUG_NET_WARN_ON_ONCE (skb_headlen (skb)); eat = len; k =0 ; shinfo =skb_shinfo (skb);
@@ -1671,12 +1663,10 @@ int tcp_trim_head(struct sock *sk, struct sk_buff *skb, u32 len)TCP_SKB_CB (skb)->seq += len; -if (delta_truesize) { - skb->truesize -= delta_truesize; -sk_wmem_queued_add (sk, -delta_truesize); -if (!skb_zcopy_pure (skb)) -sk_mem_uncharge (sk, delta_truesize); - } + skb->truesize -= delta_truesize; +sk_wmem_queued_add (sk, -delta_truesize); +if (!skb_zcopy_pure (skb)) +sk_mem_uncharge (sk, delta_truesize);/* Any change of skb->len requires recalculation of tso factor. */ if (tcp_skb_pcount (skb) >1 )
@@ -2126,9 +2116,7 @@ static int tso_fragment(struct sock *sk, struct sk_buff *skb, unsigned int len,u8 flags;/* All of a TSO frame must be composed of paged data. */ -if (skb->len != skb->data_len) -return tcp_fragment (sk,TCP_FRAG_IN_WRITE_QUEUE , - skb, len, mss_now, gfp); +DEBUG_NET_WARN_ON_ONCE (skb->len != skb->data_len); buff =tcp_stream_alloc_skb (sk,0 , gfp, true);if (unlikely (!buff))
@@ -2487,12 +2475,8 @@ static int tcp_mtu_probe(struct sock *sk) }else {TCP_SKB_CB (nskb)->tcp_flags |=TCP_SKB_CB (skb)->tcp_flags & ~(TCPHDR_FIN |TCPHDR_PSH ); -if (!skb_shinfo (skb)->nr_frags) { -skb_pull (skb, copy); - }else { -__pskb_trim_head (skb, copy); -tcp_set_skb_tso_segs (skb, mss_now); - } +__pskb_trim_head (skb, copy); +tcp_set_skb_tso_segs (skb, mss_now);TCP_SKB_CB (skb)->seq += copy; }
Signature · 1 line
2.41.0.162.gfafddb0af9-goog
PATCH 3/3 tcp: remove size parameter from tcp_stream_alloc_skb()
In reply to Eric DumazetNow all tcp_stream_alloc_skb() callers pass @size == 0, we can remove this parameter.
--- include/net/tcp.h | 2 +- net/ipv4/tcp.c | 6 +++--- net/ipv4/tcp_output.c | 10 +++++----- 3 files changed, 9 insertions(+), 9 deletions(-)
include/net/tcp.h +1 -1
@@ -350,7 +350,7 @@ void tcp_twsk_purge(struct list_head *net_exit_list, int family);ssize_t tcp_splice_read (struct socket *sk,loff_t *ppos,struct pipe_inode_info *pipe,size_t len,unsigned int flags); -struct sk_buff *tcp_stream_alloc_skb (struct sock *sk,int size,gfp_t gfp, +struct sk_buff *tcp_stream_alloc_skb (struct sock *sk,gfp_t gfp,bool force_schedule);void tcp_enter_quickack_mode (struct sock *sk,unsigned int max_quickacks);
net/ipv4/tcp.c +3 -3
@@ -858,12 +858,12 @@ ssize_t tcp_splice_read(struct socket *sock, loff_t *ppos, }EXPORT_SYMBOL (tcp_splice_read); -struct sk_buff *tcp_stream_alloc_skb (struct sock *sk,int size,gfp_t gfp, +struct sk_buff *tcp_stream_alloc_skb (struct sock *sk,gfp_t gfp,bool force_schedule) {struct sk_buff *skb; - skb =alloc_skb_fclone (size +MAX_TCP_HEADER , gfp); + skb =alloc_skb_fclone (MAX_TCP_HEADER , gfp);if (likely (skb)) {bool mem_scheduled;
@@ -1178,7 +1178,7 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size) goto restart; } first_skb =tcp_rtx_and_write_queues_empty (sk); - skb =tcp_stream_alloc_skb (sk,0 , sk->sk_allocation, + skb =tcp_stream_alloc_skb (sk, sk->sk_allocation, first_skb);if (!skb) goto wait_for_space;
net/ipv4/tcp_output.c +5 -5
@@ -1558,7 +1558,7 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue,return -ENOMEM ;/* Get a new skb... force flag on. */ - buff =tcp_stream_alloc_skb (sk,0 , gfp, true); + buff =tcp_stream_alloc_skb (sk, gfp, true);if (!buff)return -ENOMEM ;/* We'll just try again later. */ skb_copy_decrypted (buff, skb);
@@ -2118,7 +2118,7 @@ static int tso_fragment(struct sock *sk, struct sk_buff *skb, unsigned int len,/* All of a TSO frame must be composed of paged data. */ DEBUG_NET_WARN_ON_ONCE (skb->len != skb->data_len); - buff =tcp_stream_alloc_skb (sk,0 , gfp, true); + buff =tcp_stream_alloc_skb (sk, gfp, true);if (unlikely (!buff))return -ENOMEM ;skb_copy_decrypted (buff, skb);
@@ -2434,7 +2434,7 @@ static int tcp_mtu_probe(struct sock *sk)return -1 ;/* We're allowed to probe. Build it now. */ - nskb =tcp_stream_alloc_skb (sk,0 ,GFP_ATOMIC , false); + nskb =tcp_stream_alloc_skb (sk,GFP_ATOMIC , false);if (!nskb)return -1 ;
@@ -3811,7 +3811,7 @@ static int tcp_send_syn_data(struct sock *sk, struct sk_buff *syn) !skb_page_frag_refill(min_t(size_t, space, PAGE_SIZE), pfrag, sk->sk_allocation)) goto fallback; - syn_data =tcp_stream_alloc_skb (sk,0 , sk->sk_allocation, false); + syn_data =tcp_stream_alloc_skb (sk, sk->sk_allocation, false);if (!syn_data) goto fallback;memcpy (syn_data->cb, syn->cb,sizeof (syn->cb));
@@ -3896,7 +3896,7 @@ int tcp_connect(struct sock *sk)return 0 ; } - buff =tcp_stream_alloc_skb (sk,0 , sk->sk_allocation, true); + buff =tcp_stream_alloc_skb (sk, sk->sk_allocation, true);if (unlikely (!buff))return -ENOBUFS ;
Signature · 1 line
2.41.0.162.gfafddb0af9-goog
This series was applied to netdev/net-next.git (main) by David S. Miller <[email protected]>:
Hello:
This series was applied to netdev/net-next.git (main) by David S. Miller <[email protected]>:
Quoted 10 lines · On Fri, 9 Jun 2023 20:42:43 +0000 you wrote:
This series completes transition of TCP stack tx path to headless packets : All payload now reside in page frags, never in
skb->head.Eric Dumazet (3):
tcp: lettcp_send_syn_data()build headless packets
tcp: remove some dead code
tcp: remove size parameter fromtcp_stream_alloc_skb()[...]
Here is the summary with links:
- [net-next,1/3] tcp: let tcp_send_syn_data() build headless packets
https://git.kernel.org/netdev/net-next/c/fbf934068f6b - [net-next,2/3] tcp: remove some dead code https://git.kernel.org/netdev/net-next/c/b4a24397139c - [net-next,3/3] tcp: remove size parameter from tcp_stream_alloc_skb() https://git.kernel.org/netdev/net-next/c/5882efff88aa
You are awesome, thank you!
Signature · 2 lines
Deet-doot-dot, I am a bot. https://korg.docs.kernel.org/patchwork/pwbot.html