Skip to contentSkip to conversation

tcp: support preloading data on a listening socket

PATCH Superseded Jeremy Harris 2 participants 8 messages May 21, 2025 – May 22, 2025

Subsystems (6)

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#1 lore raw

PATCH v2 1/6 tcp: support writing to a socket in listening state

In reply to Jeremy Harris

In the tcp sendmsg handler, permit a write in LISTENING state if a MSG_PRELOAD flag is used. Copy from iovec to a linear sk_buff for placement on the socket write queue.

---
 include/linux/socket.h                        |   1 +
 net/ipv4/tcp.c                                | 115 ++++++++++++++++++
 .../perf/trace/beauty/include/linux/socket.h  |   1 +
 tools/perf/trace/beauty/msg_flags.c           |   3 +
 4 files changed, 120 insertions(+)
include/linux/socket.h +1 -0
index 3b262487ec06..b41f4cd4dc97 100644
@@ -330,6 +330,7 @@ struct ucred {
 #define MSG_SOCK_DEVMEM 0x2000000	/* Receive devmem skbs as cmsg */
 #define MSG_ZEROCOPY	0x4000000	/* Use user data in kernel path */
 #define MSG_SPLICE_PAGES 0x8000000	/* Splice the pages from the iterator in sendmsg() */
+#define MSG_PRELOAD	0x10000000	/* Preload tx data while listening */
 #define MSG_FASTOPEN	0x20000000	/* Send data in TCP SYN */
 #define MSG_CMSG_CLOEXEC 0x40000000	/* Set close_on_exec for file
 					   descriptor received through
net/ipv4/tcp.c +115 -0

127 lines not on this page · raw message

tools/perf/trace/beauty/include/linux/socket.h +1 -0
index c3322eb3d686..e9ea498169f3 100644
@@ -330,6 +330,7 @@ struct ucred {
 #define MSG_SOCK_DEVMEM 0x2000000	/* Receive devmem skbs as cmsg */
 #define MSG_ZEROCOPY	0x4000000	/* Use user data in kernel path */
 #define MSG_SPLICE_PAGES 0x8000000	/* Splice the pages from the iterator in sendmsg() */
+#define MSG_PRELOAD	0x10000000	/* Preload tx data while listening */
 #define MSG_FASTOPEN	0x20000000	/* Send data in TCP SYN */
 #define MSG_CMSG_CLOEXEC 0x40000000	/* Set close_on_exec for file
 					   descriptor received through
tools/perf/trace/beauty/msg_flags.c +3 -0
index 2da581ff0c80..27e40da9b02d 100644
@@ -20,6 +20,9 @@
 #ifndef MSG_SPLICE_PAGES
 #define MSG_SPLICE_PAGES	0x8000000
 #endif
+#ifndef MSG_PRELOAD
+#define MSG_PRELOAD		0x10000000
+#endif
 #ifndef MSG_FASTOPEN
 #define MSG_FASTOPEN		0x20000000
 #endif
Signature · 1 line
2.49.0
#7 lore raw

PATCH v2 6/6 tcp: fastopen: extend retransmit-queue trimming to handle linear sk_buff

In reply to Jeremy Harris

A corner-case for the 3rd-ack after a data-on-synack is for some but not all of the data to be acked. Support this by adding to the retransmit-queue trim routine to handle a linear sk_buff.

---
 net/ipv4/tcp_output.c | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)
net/ipv4/tcp_output.c +16 -2
index c50553c1c795..bff5934ff04b 100644
@@ -1708,8 +1708,22 @@ static int __pskb_trim_head(struct sk_buff *skb, int len)
 	struct skb_shared_info *shinfo;
 	int i, k, eat;
 
-	DEBUG_NET_WARN_ON_ONCE(skb_headlen(skb));
-	eat = len;
+	eat = skb_headlen(skb);
+	if (unlikely(eat)) {
+		if (len < eat)
+			eat = len;
+		skb->head += eat;
+		skb->len -= eat;
+		if (skb->data_len)
+			skb->data_len -= eat;
+
+		eat = len - eat;
+		if (eat == 0)
+			return len;
+	} else {
+		eat = len;
+	}
+
 	k = 0;
 	shinfo = skb_shinfo(skb);
 	for (i = 0; i < shinfo->nr_frags; i++) {
Signature · 1 line
2.49.0
#5 lore raw

PATCH v2 4/6 tcp: transmit any pending data on receipt of 3rd-ack

In reply to Jeremy Harris

For the non-fastopen case of prelaod, when the 3rd-ack arrives there will be data on the write queue. Transmit it immediately by allowing the SYN_SENT state to run the xmit-recovery code.

---
 net/ipv4/tcp_input.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
net/ipv4/tcp_input.c +2 -1
index 8ec92dec321a..345a08baaf02 100644
@@ -3900,7 +3900,8 @@ static void tcp_xmit_recovery(struct sock *sk, int rexmit)
 {
 	struct tcp_sock *tp = tcp_sk(sk);
 
-	if (rexmit == REXMIT_NONE || sk->sk_state == TCP_SYN_SENT)
+	if ((rexmit == REXMIT_NONE && sk->sk_state != TCP_SYN_RECV) ||
+	    sk->sk_state == TCP_SYN_SENT)
 		return;
 
 	if (unlikely(rexmit == REXMIT_NEW)) {
Signature · 1 line
2.49.0
#4 lore raw

PATCH v2 3/6 tcp: fastopen: add write-data to fastopen synack packet

In reply to Jeremy Harris

While building the synack packet, for a fastopen socket copy data from write queue to the packet.
Move the data from write queue to retransmit queue.

---
 net/ipv4/tcp_output.c | 32 ++++++++++++++++++++++++++++++--
 1 file changed, 30 insertions(+), 2 deletions(-)
net/ipv4/tcp_output.c +30 -2
index 3ac8d2d17e1f..c50553c1c795 100644
@@ -3702,7 +3702,7 @@ int tcp_send_synack(struct sock *sk)
 
 /**
  * tcp_make_synack - Allocate one skb and build a SYNACK packet.
- * @sk: listener socket
+ * @sk: listener socket (or child socket for fastopen)
  * @dst: dst entry attached to the SYNACK. It is consumed and caller
  *       should not use it again.
  * @req: request_sock pointer
@@ -3719,6 +3719,7 @@ struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst,
 	struct inet_request_sock *ireq = inet_rsk(req);
 	const struct tcp_sock *tp = tcp_sk(sk);
 	struct tcp_out_options opts;
+	struct sock *fastopen_sk = (struct sock *)sk;
 	struct tcp_key key = {};
 	struct sk_buff *skb;
 	int tcp_header_size;
@@ -3748,7 +3749,7 @@ struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst,
 		 * cpu might call us concurrently.
 		 * sk->sk_wmem_alloc in an atomic, we can promote to rw.
 		 */
-		skb_set_owner_w(skb, (struct sock *)sk);
+		skb_set_owner_w(skb, fastopen_sk);
 		break;
 	}
 	skb_dst_set(skb, dst);
@@ -3831,6 +3832,33 @@ struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst,
 	th->window = htons(min(req->rsk_rcv_wnd, 65535U));
 	tcp_options_write(th, NULL, tcp_rsk(req), &opts, &key);
 	th->doff = (tcp_header_size >> 2);
+
+	/* If this is a FASTOPEN, and there is write-data on the accept socket,
+	 * re-copy it to the synack segment. If not FASTOPEN. any data waits
+	 * until 3rd-ack arrival.
+	 */
+
+	if (synack_type == TCP_SYNACK_FASTOPEN &&
+	    !skb_queue_empty(&sk->sk_write_queue)) {
+		struct sk_buff *a_skb = tcp_write_queue_tail(sk);
+		int copy = min_t(int, a_skb->len, skb_tailroom(skb));
+
+		skb_put_data(skb, a_skb->data, copy);
+		TCP_SKB_CB(skb)->end_seq += copy;
+
+		tcp_skb_pcount_set(a_skb, 1);
+		WRITE_ONCE(tcp_sk(fastopen_sk)->write_seq,
+			   TCP_SKB_CB(a_skb)->end_seq);
+
+		skb_set_delivery_time(a_skb, now, SKB_CLOCK_MONOTONIC);
+
+		/* Move the data to the retransmit queue.
+		 * Code elsewhere implies this is a full child socket and
+		 * can be treated as writeable - permitting the cast.
+		 */
+		tcp_event_new_data_sent(fastopen_sk, a_skb);
+	}
+
 	TCP_INC_STATS(sock_net(sk), TCP_MIB_OUTSEGS);
 
 	/* Okay, we have all we need - do the md5 hash if needed */
Signature · 1 line
2.49.0
#6 lore raw

PATCH v2 5/6 tcp: fastopen: retransmit data when only the SYN of a synack-with-data is acked

In reply to Jeremy Harris

A corner-case for the 3rd-ack after a data-on-synack is for only the SYN to be acked. Handle this by, in ack processing, when in SYN_RECV state (the state is not yet updated to ESTABLISHED) marking the retransmit-queue sk_buff as having been lost.

---
 net/ipv4/tcp_input.c | 12 ++++++++++++
 1 file changed, 12 insertions(+)
net/ipv4/tcp_input.c +12 -0
index 345a08baaf02..a53021edddd5 100644
@@ -4069,6 +4069,18 @@ static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag)
 				      &rexmit);
 	}
 
+	/* On receiving a 3rd-ack, if we never sent a packet via
+	 * the normal means (which counts them), yet there is data
+	 * remaining for retransmit, it was data-on-synack not acked;
+	 * mark the skb for retransmission.
+	 */
+	if (sk->sk_state == TCP_SYN_RECV && tp->segs_out == 0) {
+		struct sk_buff *skb = tcp_rtx_queue_head(sk);
+
+		if (skb)
+			tcp_mark_skb_lost(sk, skb);
+	}
+
 	/* If needed, reset TLP/RTO timer when RACK doesn't set. */
 	if (flag & FLAG_SET_XMIT_TIMER)
 		tcp_set_xmit_timer(sk);
Signature · 1 line
2.49.0
#0 lore raw

v2 cover tcp: support preloading data on a listening socket

v2 changes:
- Split out the preload operation to a separate routine from tcp_sendmsg_locked() and restrict from looping over the supplied iovec

------
Support write to a listen TCP socket, for immediate transmission on all later passive connection establishments parented by the listen socket.

On a normal connection transmission of the data is triggered by the receipt of the 3rd-ack. On a fastopen (with accepted cookie) connection the data is sent in the synack packet.

The data preload is done using a sendmsg with a newly-defined flag (MSG_PRELOAD); the amount of data limited to a single linear sk_buff. Note that this definition is the last-but-two bit available if "int" is 32 bits.

Intent: lower latency for server-first protocols using TCP.
Known cases of this use are SMTP and MySQL.

Measurements:
Packet capture (laptop, loopback, TFO requeste) for initial SYN to first client data packet (5 samples):

- baseline TFO-C 1064 1470 1455 1547 1595 usec
- patched non-TFO 140 150 159 144 153 usec
- patched TFO-C 142 149 149 125 125 usec

Out of scope:
- Client-first protocols
- TLS-on-connect

Testing:

A) packetdrill scripts for
- normal non-TFO
- normal TFO
- synack lost
- 3rd-ack acks only the SYN
- 3rd-ack acks partial data
(NB: packetdrill can only check the data size, not actual content)

B) Application use, running the application testsuite and manual check of specific cases via packet capture

C) Daily-driver laptop use (not expected to trigger the feature; only regression-test)

D) KASAN/syzkaller
- enable_syscalls "socket$inet_tcp", "listen", "sendmsg", "accept",

      "read", "write", "close", "syz_emit_ethernet", "syz_extract_tcp_res"

- the coverage seems rather limited; the sendmsg onto a listen socket is there, but I am not convinced actual TCP connections are being excercised. tcp_minisocks.c is entirely uncovered.
- A need for limiting iteration in the above sendmesg was found (RCU timeouts), hence v2, but no hint of locking problems.
Eric: could you expand on your previous comment? If it referred to the listening socket, tcp_sendmsg_locked() is called with the sk locked.

Jeremy Harris (6):
tcp: support writing to a socket in listening state
tcp: copy write-data from listen socket to accept child socket
tcp: fastopen: add write-data to fastopen synack packet
tcp: transmit any pending data on receipt of 3rd-ack
tcp: fastopen: retransmit data when only the SYN of a synack-with-data
is acked
tcp: fastopen: extend retransmit-queue trimming to handle linear
sk_buff

 include/linux/socket.h                        |   1 +
 net/ipv4/tcp.c                                | 115 ++++++++++++++++++
 net/ipv4/tcp_fastopen.c                       |   3 +-
 net/ipv4/tcp_input.c                          |  15 ++-
 net/ipv4/tcp_ipv4.c                           |   4 +-
 net/ipv4/tcp_minisocks.c                      |  58 ++++++++-
 net/ipv4/tcp_output.c                         |  50 +++++++-
 .../perf/trace/beauty/include/linux/socket.h  |   1 +
 tools/perf/trace/beauty/msg_flags.c           |   3 +
 9 files changed, 237 insertions(+), 13 deletions(-)

base-commit: f685204c57e87d2a88b159c7525426d70ee745c9

Signature · 1 line
2.49.0
#3 lore raw

PATCH v2 2/6 tcp: copy write-data from listen socket to accept child socket

Set the request_sock flag for fastopen earlier, making it available to the af_ops SYN-handler function.

In that function copy data from the listen socket write queue into an sk_buff, allocating if needed and adding to the write queue of the newly-created child socket.
Set sequence number values depending on the fastopen status.

---
 net/ipv4/tcp_fastopen.c  |  3 ++-
 net/ipv4/tcp_ipv4.c      |  4 +--
 net/ipv4/tcp_minisocks.c | 58 ++++++++++++++++++++++++++++++++++++----
 3 files changed, 57 insertions(+), 8 deletions(-)
net/ipv4/tcp_fastopen.c +2 -1
index 9b83d639b5ac..03a86d0b87ba 100644
@@ -245,6 +245,8 @@ static struct sock *tcp_fastopen_create_child(struct sock *sk,
 	struct sock *child;
 	bool own_req;
 
+	tcp_rsk(req)->tfo_listener = true;
+
 	child = inet_csk(sk)->icsk_af_ops->syn_recv_sock(sk, skb, req, NULL,
 							 NULL, &own_req);
 	if (!child)
@@ -261,7 +263,6 @@ static struct sock *tcp_fastopen_create_child(struct sock *sk,
 	tp = tcp_sk(child);
 
 	rcu_assign_pointer(tp->fastopen_rsk, req);
-	tcp_rsk(req)->tfo_listener = true;
 
 	/* RFC1323: The window in SYN & SYN/ACK segments is never
 	 * scaled. So correct it appropriately.
net/ipv4/tcp_ipv4.c +2 -2
index 6a14f9e6fef6..e488effdbdb2 100644
@@ -1747,8 +1747,8 @@ EXPORT_IPV6_MOD(tcp_v4_conn_request);
 
 
 /*
- * The three way handshake has completed - we got a valid synack -
- * now create the new socket.
+ * The three way handshake has completed - we got a valid synack
+ * (or a FASTOPEN syn) - now create the new socket.
  */
 struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb,
 				  struct request_sock *req,
net/ipv4/tcp_minisocks.c +53 -5

76 lines not on this page · raw message

Signature · 1 line
2.49.0
#2 lore raw

Re: [PATCH net-next v2 1/6] tcp: support writing to a socket in listening state

In reply to Jeremy Harris
kernel test robot noticed the following build warnings:

Hi Jeremy,

kernel test robot noticed the following build warnings:

[auto build test WARNING on f685204c57e87d2a88b159c7525426d70ee745c9]

url:    https://github.com/intel-lab-lkp/linux/commits/Jeremy-Harris/tcp-support-writing-to-a-socket-in-listening-state/20250521-195234
base:   f685204c57e87d2a88b159c7525426d70ee745c9
patch link:    https://lore.kernel.org/r/d3f47c9b5b08237b6e76f7b0739d59089683c86e.1747826775.git.jgh%40exim.org
patch subject: [PATCH net-next v2 1/6]     tcp: support writing to a socket in listening state
config: i386-buildonly-randconfig-001-20250522 (https://download.01.org/0day-ci/archive/20250522/[email protected]/config)
compiler: clang version 20.1.2 (https://github.com/llvm/llvm-project 58df0ef89dd64126512e4ee27b4ac3fd8ddf6247)
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20250522/[email protected]/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/

All warnings (new ones prefixed by >>):

>> net/ipv4/tcp.c:1065:23: warning: variable 'sockc' set but not used [-Wunused-but-set-variable]
    1065 |         struct sockcm_cookie sockc;
         |                              ^

1 warning generated.

vim +/sockc +1065 net/ipv4/tcp.c

1059
1060 /* Cut-down version of tcp_sendmsg_locked(), for writing on a listen socket
1061 */
1062 static int tcp_sendmsg_preload(struct sock *sk, struct msghdr *msg)
1063 {
1064 struct sk_buff *skb;

> 1065		struct sockcm_cookie sockc;

1066 int flags, err, copied = 0;
1067 int size_goal;
1068 int process_backlog = 0;
1069 long timeo;
1070
1071 if (sk->sk_state != TCP_LISTEN)
1072 return -EINVAL;
1073
1074 flags = msg->msg_flags;
1075
1076 sockc = (struct sockcm_cookie){ .tsflags = READ_ONCE(sk->sk_tsflags) };
1077
1078 timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
1079
1080 /* Ok commence sending. */
1081 restart:
1082 /* Use a arbitrary "mss" value */
1083 size_goal = 1000;
1084
1085 err = -EPIPE;
1086 if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
1087 goto do_error;
1088
1089 while (msg_data_left(msg)) {
1090 ssize_t copy = 0;
1091
1092 skb = tcp_write_queue_tail(sk);
1093 if (skb)
1094 copy = size_goal - skb->len;
1095
1096 trace_tcp_sendmsg_locked(sk, msg, skb, size_goal);
1097
1098 if (copy <= 0 || !tcp_skb_can_collapse_to(skb)) {
1099 bool first_skb = !skb;
1100
1101 /* Limit to only one skb on the sk write queue */
1102
1103 if (!first_skb)
1104 goto out_nopush;
1105
1106 if (!sk_stream_memory_free(sk))
1107 goto wait_for_space;
1108
1109 if (unlikely(process_backlog >= 16)) {
1110 process_backlog = 0;
1111 if (sk_flush_backlog(sk))
1112 goto restart;
1113 }
1114
1115 skb = tcp_stream_alloc_skb(sk, sk->sk_allocation,
1116 first_skb);
1117 if (!skb)
1118 goto wait_for_space;
1119
1120 process_backlog++;
1121

Signature · 2 lines
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki