Skip to contentSkip to conversation

mptcp: track window announced to peer

PATCH In review Florian Westphal 4 participants 7 messages Oct 22, 2020 – Oct 27, 2020

Subsystems (3)
lore
Share & tools
lore

Florian Westphal

author
#0 lore raw

PATCH v3 1/1 mptcp: track window announced to peer

OoO handling attemtps to detect when packet is out-of-window by testing current ack sequence and remaining space vs. sequence number.

This doesn't work reliably. Store the highest allowed sequence number that we've announced and use it to detect oow packets.

Do this when mptcp options get written to the packet (wire format). For this to work we need to move the write_options call until after stack selected a new tcp window.

Signed-off-by Florian Westphal <fw(a)strlen.de>

---
Changes in v3:
- add a few line breaks
- update rcvwnd even for non-dss packets

 include/net/mptcp.h   |  3 ++-
 net/ipv4/tcp_output.c | 11 +++++++----
 net/mptcp/options.c   | 22 +++++++++++++++++++++-
 net/mptcp/protocol.c  | 12 +++++++-----
 net/mptcp/protocol.h  |  1 +
 5 files changed, 38 insertions(+), 11 deletions(-)
include/net/mptcp.h +2 -1
index 6e706d838e4e..b6cf07143a8a 100644
@@ -88,7 +88,8 @@ bool mptcp_established_options(struct sock *sk, struct sk_buff *skb,
 			       struct mptcp_out_options *opts);
 void mptcp_incoming_options(struct sock *sk, struct sk_buff *skb);
 
-void mptcp_write_options(__be32 *ptr, struct mptcp_out_options *opts);
+void mptcp_write_options(__be32 *ptr, const struct tcp_sock *tp,
+			 struct mptcp_out_options *opts);
 
 /* move the skb extension owership, with the assumption that 'to' is
  * newly allocated
net/ipv4/tcp_output.c +7 -4
index 9abf5a0358d5..a127856ab7df 100644
@@ -445,11 +445,12 @@ struct tcp_out_options {
 	struct mptcp_out_options mptcp;
 };
 
-static void mptcp_options_write(__be32 *ptr, struct tcp_out_options *opts)
+static void mptcp_options_write(__be32 *ptr, const struct tcp_sock *tp,
+				struct tcp_out_options *opts)
 {
 #if IS_ENABLED(CONFIG_MPTCP)
 	if (unlikely(OPTION_MPTCP & opts->options))
-		mptcp_write_options(ptr, &opts->mptcp);
+		mptcp_write_options(ptr, tp, &opts->mptcp);
 #endif
 }
 
@@ -701,7 +702,7 @@ static void tcp_options_write(__be32 *ptr, struct tcp_sock *tp,
 
 	smc_options_write(ptr, &options);
 
-	mptcp_options_write(ptr, opts);
+	mptcp_options_write(ptr, tp, opts);
 }
 
 static void smc_set_option(const struct tcp_sock *tp,
@@ -1348,7 +1349,6 @@ static int __tcp_transmit_skb(struct sock *sk, struct sk_buff *skb,
 		}
 	}
 
-	tcp_options_write((__be32 *)(th + 1), tp, &opts);
 	skb_shinfo(skb)->gso_type = sk->sk_gso_type;
 	if (likely(!(tcb->tcp_flags & TCPHDR_SYN))) {
 		th->window      = htons(tcp_select_window(sk));
@@ -1359,6 +1359,9 @@ static int __tcp_transmit_skb(struct sock *sk, struct sk_buff *skb,
 		 */
 		th->window	= htons(min(tp->rcv_wnd, 65535U));
 	}
+
+	tcp_options_write((__be32 *)(th + 1), tp, &opts);
+
 #ifdef CONFIG_TCP_MD5SIG
 	/* Calculate the MD5 hash, as we have all we need now */
 	if (md5) {
net/mptcp/options.c +21 -1
index f2d1e27a2bc1..2e9f1c4ea008 100644
@@ -991,7 +991,24 @@ void mptcp_incoming_options(struct sock *sk, struct sk_buff *skb)
 	}
 }
 
-void mptcp_write_options(__be32 *ptr, struct mptcp_out_options *opts)
+static void mptcp_set_rwin(const struct tcp_sock *tp)
+{
+	const struct sock *ssk = (const struct sock *)tp;
+	const struct mptcp_subflow_context *subflow;
+	struct mptcp_sock *msk;
+	u64 ack_seq;
+
+	subflow = mptcp_subflow_ctx(ssk);
+	msk = mptcp_sk(subflow->conn);
+
+	ack_seq = READ_ONCE(msk->ack_seq) + tp->rcv_wnd;
+
+	if (after64(ack_seq, READ_ONCE(msk->rcv_wnd_sent)))
+		WRITE_ONCE(msk->rcv_wnd_sent, ack_seq);
+}
+
+void mptcp_write_options(__be32 *ptr, const struct tcp_sock *tp,
+			 struct mptcp_out_options *opts)
 {
 	if ((OPTION_MPTCP_MPC_SYN | OPTION_MPTCP_MPC_SYNACK |
 	     OPTION_MPTCP_MPC_ACK) & opts->suboptions) {
@@ -1148,4 +1165,7 @@ void mptcp_write_options(__be32 *ptr, struct mptcp_out_options *opts)
 					   TCPOPT_NOP << 8 | TCPOPT_NOP, ptr);
 		}
 	}
+
+	if (tp)
+		mptcp_set_rwin(tp);
 }
net/mptcp/protocol.c +7 -5
index 6c131759f3e5..cd747696e7af 100644
@@ -168,19 +168,19 @@ static void mptcp_data_queue_ofo(struct mptcp_sock *msk, struct sk_buff *skb)
 	struct rb_node **p, *parent;
 	u64 seq, end_seq, max_seq;
 	struct sk_buff *skb1;
-	int space;
 
 	seq = MPTCP_SKB_CB(skb)->map_seq;
 	end_seq = MPTCP_SKB_CB(skb)->end_seq;
-	space = tcp_space(sk);
-	max_seq = space > 0 ? space + msk->ack_seq : msk->ack_seq;
+	max_seq = READ_ONCE(msk->rcv_wnd_sent);
 
 	pr_debug("msk=%p seq=%llx limit=%llx empty=%d", msk, seq, max_seq,
 		 RB_EMPTY_ROOT(&msk->out_of_order_queue));
-	if (after64(seq, max_seq)) {
+	if (after64(end_seq, max_seq)) {
 		/* out of window */
 		mptcp_drop(sk, skb);
-		pr_debug("oow by %ld", (unsigned long)seq - (unsigned long)max_seq);
+		pr_debug("oow by %lld, rcv_wnd_sent %llu\n",
+			 (unsigned long long)end_seq - (unsigned long)max_seq,
+			 (unsigned long long)msk->rcv_wnd_sent);
 		MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_NODSSWINDOW);
 		return;
 	}
@@ -2258,6 +2258,7 @@ struct sock *mptcp_sk_clone(const struct sock *sk,
 		mptcp_crypto_key_sha(msk->remote_key, NULL, &ack_seq);
 		ack_seq++;
 		WRITE_ONCE(msk->ack_seq, ack_seq);
+		WRITE_ONCE(msk->rcv_wnd_sent, ack_seq);
 	}
 
 #if !IS_ENABLED(CONFIG_KASAN)
@@ -2567,6 +2568,7 @@ void mptcp_finish_connect(struct sock *ssk)
 	WRITE_ONCE(msk->write_seq, subflow->idsn + 1);
 	WRITE_ONCE(msk->snd_nxt, msk->write_seq);
 	WRITE_ONCE(msk->ack_seq, ack_seq);
+	WRITE_ONCE(msk->rcv_wnd_sent, ack_seq);
 	WRITE_ONCE(msk->can_ack, 1);
 	atomic64_set(&msk->snd_una, msk->write_seq);
 
net/mptcp/protocol.h +1 -0
index b4c8dbe9236b..bf15c2fc0ba1 100644
@@ -211,6 +211,7 @@ struct mptcp_sock {
 	u64		write_seq;
 	u64		snd_nxt;
 	u64		ack_seq;
+	u64		rcv_wnd_sent;
 	u64		rcv_data_fin_seq;
 	struct sock	*last_snd;
 	int		snd_burst;
Signature · 1 line
2.26.2
#1 lore raw
Quoted 66 lines · Florian Westphal

OoO handling attemtps to detect when packet is out-of-window by testing current ack sequence and remaining space vs. sequence number.

This doesn't work reliably. Store the highest allowed sequence number that we've announced and use it to detect oow packets.

Do this when mptcp options get written to the packet (wire format). For this to work we need to move the write_options call until after stack selected a new tcp window.

Signed-off-by Florian Westphal <fw(a)strlen.de>

---
Changes in v3:
- add a few line breaks
- update rcvwnd even for non-dss packets

include/net/mptcp.h   |  3 ++-
net/ipv4/tcp_output.c | 11 +++++++----
net/mptcp/options.c   | 22 +++++++++++++++++++++-
net/mptcp/protocol.c  | 12 +++++++-----
net/mptcp/protocol.h  |  1 +
5 files changed, 38 insertions(+), 11 deletions(-)
include/net/mptcp.h +2 -1
index 6e706d838e4e..b6cf07143a8a 100644
@@ -88,7 +88,8 @@ bool mptcp_established_options(struct sock *sk, struct sk_buff *skb,
 			       struct mptcp_out_options *opts);
 void mptcp_incoming_options(struct sock *sk, struct sk_buff *skb);
 
-void mptcp_write_options(__be32 *ptr, struct mptcp_out_options *opts);
+void mptcp_write_options(__be32 *ptr, const struct tcp_sock *tp,
+			 struct mptcp_out_options *opts);
 
 /* move the skb extension owership, with the assumption that 'to' is
 * newly allocated
net/ipv4/tcp_output.c +4 -3
index 9abf5a0358d5..a127856ab7df 100644
@@ -445,11 +445,12 @@ struct tcp_out_options {
 	struct mptcp_out_options mptcp;
 };
 
-static void mptcp_options_write(__be32 *ptr, struct tcp_out_options *opts)
+static void mptcp_options_write(__be32 *ptr, const struct tcp_sock *tp,
+				struct tcp_out_options *opts)
 {
 #if IS_ENABLED(CONFIG_MPTCP)
 	if (unlikely(OPTION_MPTCP & opts->options))
-		mptcp_write_options(ptr, &opts->mptcp);
+		mptcp_write_options(ptr, tp, &opts->mptcp);
 #endif
 }
 
@@ -701,7 +702,7 @@ static void tcp_options_write(__be32 *ptr, struct tcp_sock *tp,
 
 	smc_options_write(ptr, &options);
 
-	mptcp_options_write(ptr, opts);
+	mptcp_options_write(ptr, tp, opts);
 }
 
 static void smc_set_option(const struct tcp_sock *tp,

net/ipv4/tcp_output.c static int __tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, · line 1354

Quoted 7 lines
net/ipv4/tcp_output.c +0 -1
@@ -1348,7 +1349,6 @@ static int __tcp_transmit_skb(struct sock *sk, struct sk_buff *skb,
 		}
 	}
 
-	tcp_options_write((__be32 *)(th + 1), tp, &opts);
 	skb_shinfo(skb)->gso_type = sk->sk_gso_type;
 	if (likely(!(tcb->tcp_flags & TCPHDR_SYN))) {
 		th->window      = htons(tcp_select_window(sk));

These lines of code between the old tcp_options_write() location and the new location were last touched by Eric, doing some cache miss optimizations. It doesn't seem like populating the TCP options just after th->window instead of just before should behave similarly with the data cache - are you concerned about this at all?

Quoted 23 lines
net/ipv4/tcp_output.c +3 -0
@@ -1359,6 +1359,9 @@ static int __tcp_transmit_skb(struct sock *sk, struct sk_buff *skb,
 		 */
 		th->window	= htons(min(tp->rcv_wnd, 65535U));
 	}
+
+	tcp_options_write((__be32 *)(th + 1), tp, &opts);
+
 #ifdef CONFIG_TCP_MD5SIG
 	/* Calculate the MD5 hash, as we have all we need now */
 	if (md5) {
net/mptcp/options.c +4 -1
index f2d1e27a2bc1..2e9f1c4ea008 100644
@@ -991,7 +991,24 @@ void mptcp_incoming_options(struct sock *sk, struct sk_buff *skb)
 	}
 }
 
-void mptcp_write_options(__be32 *ptr, struct mptcp_out_options *opts)
+static void mptcp_set_rwin(const struct tcp_sock *tp)
+{
+	const struct sock *ssk = (const struct sock *)tp;
+	const struct mptcp_subflow_context *subflow;

net/mptcp/options.c void mptcp_incoming_options(struct sock *sk, struct sk_buff *skb) · line 1007

Quoted 10 lines
net/mptcp/options.c +10 -0
+	struct mptcp_sock *msk;
+	u64 ack_seq;
+
+	subflow = mptcp_subflow_ctx(ssk);
+	msk = mptcp_sk(subflow->conn);
+
+	ack_seq = READ_ONCE(msk->ack_seq) + tp->rcv_wnd;
+
+	if (after64(ack_seq, READ_ONCE(msk->rcv_wnd_sent)))
+		WRITE_ONCE(msk->rcv_wnd_sent, ack_seq);

This looks racy if packet headers on multiple subflows are created
concurrently: msk->rcv_wnd_sent could end up with an earlier sequence number than it should have, leading to possible extra drops. It looks like a hard race to trigger, but do you think rcv_wnd_sent should use cmpxchg like wnd_end and snd_una?

Mat

Quoted 74 lines
net/mptcp/options.c +7 -0
+}
+
+void mptcp_write_options(__be32 *ptr, const struct tcp_sock *tp,
+			 struct mptcp_out_options *opts)
 {
 	if ((OPTION_MPTCP_MPC_SYN | OPTION_MPTCP_MPC_SYNACK |
 	     OPTION_MPTCP_MPC_ACK) & opts->suboptions) {
@@ -1148,4 +1165,7 @@ void mptcp_write_options(__be32 *ptr, struct mptcp_out_options *opts)
 					   TCPOPT_NOP << 8 | TCPOPT_NOP, ptr);
 		}
 	}
+
+	if (tp)
+		mptcp_set_rwin(tp);
 }
net/mptcp/protocol.c +7 -5
index 6c131759f3e5..cd747696e7af 100644
@@ -168,19 +168,19 @@ static void mptcp_data_queue_ofo(struct mptcp_sock *msk, struct sk_buff *skb)
 	struct rb_node **p, *parent;
 	u64 seq, end_seq, max_seq;
 	struct sk_buff *skb1;
-	int space;
 
 	seq = MPTCP_SKB_CB(skb)->map_seq;
 	end_seq = MPTCP_SKB_CB(skb)->end_seq;
-	space = tcp_space(sk);
-	max_seq = space > 0 ? space + msk->ack_seq : msk->ack_seq;
+	max_seq = READ_ONCE(msk->rcv_wnd_sent);
 
 	pr_debug("msk=%p seq=%llx limit=%llx empty=%d", msk, seq, max_seq,
 		 RB_EMPTY_ROOT(&msk->out_of_order_queue));
-	if (after64(seq, max_seq)) {
+	if (after64(end_seq, max_seq)) {
 		/* out of window */
 		mptcp_drop(sk, skb);
-		pr_debug("oow by %ld", (unsigned long)seq - (unsigned long)max_seq);
+		pr_debug("oow by %lld, rcv_wnd_sent %llu\n",
+			 (unsigned long long)end_seq - (unsigned long)max_seq,
+			 (unsigned long long)msk->rcv_wnd_sent);
 		MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_NODSSWINDOW);
 		return;
 	}
@@ -2258,6 +2258,7 @@ struct sock *mptcp_sk_clone(const struct sock *sk,
 		mptcp_crypto_key_sha(msk->remote_key, NULL, &ack_seq);
 		ack_seq++;
 		WRITE_ONCE(msk->ack_seq, ack_seq);
+		WRITE_ONCE(msk->rcv_wnd_sent, ack_seq);
 	}
 
 #if !IS_ENABLED(CONFIG_KASAN)
@@ -2567,6 +2568,7 @@ void mptcp_finish_connect(struct sock *ssk)
 	WRITE_ONCE(msk->write_seq, subflow->idsn + 1);
 	WRITE_ONCE(msk->snd_nxt, msk->write_seq);
 	WRITE_ONCE(msk->ack_seq, ack_seq);
+	WRITE_ONCE(msk->rcv_wnd_sent, ack_seq);
 	WRITE_ONCE(msk->can_ack, 1);
 	atomic64_set(&msk->snd_una, msk->write_seq);
 
net/mptcp/protocol.h +1 -0
index b4c8dbe9236b..bf15c2fc0ba1 100644
@@ -211,6 +211,7 @@ struct mptcp_sock {
 	u64		write_seq;
 	u64		snd_nxt;
 	u64		ack_seq;
+	u64		rcv_wnd_sent;
 	u64		rcv_data_fin_seq;
 	struct sock	*last_snd;
 	int		snd_burst;
Signature · 1 line
2.26.2

--
Mat Martineau
Intel

#2 lore raw
Quoted 5 lines · Mat Martineau

On Thu, 22 Oct 2020, Florian Westphal wrote:

(quoted diff) +0 -0
@@ -1348,7 +1349,6 @@ static int __tcp_transmit_skb(struct sock *sk, struct sk_buff *skb,
 > 		}
 > 	}
Quoted 1 line
Quoted 10 lines
Quoted 4 lines
net/ipv4/tcp_output.c +0 -1
-	tcp_options_write((__be32 *)(th + 1), tp, &opts);
	skb_shinfo(skb)->gso_type = sk->sk_gso_type;
	if (likely(!(tcb->tcp_flags & TCPHDR_SYN))) {
		th->window      = htons(tcp_select_window(sk));

These lines of code between the old tcp_options_write() location and the new location were last touched by Eric, doing some cache miss optimizations. It doesn't seem like populating the TCP options just after th->window instead of just before should behave similarly with the data cache - are you concerned about this at all?

Thank you for the carefully review!

commit 51466a7545b73b7ad7bcfb33410d2823ccfaa501

Author: Eric Dumazet <edumazet(a)google.com>
Date:   Thu Jun 11 09:15:16 2015 -0700
    tcp: fill shinfo->gso_type at last moment

moves 'skb_shinfo(skb)->gso_type' initialization just after the option write. Looking at the commit message, what actually makes a difference in such commit is moving the mentioned assigment after the last xmit check to avoid touching extra cache lines if not needed - and we preserve that fact.

I think the above is sane performace-wise, but I guess Eric could still teach me better ;)

Quoted 32 lines
Quoted 32 lines
net/ipv4/tcp_output.c +3 -0
@@ -1359,6 +1359,9 @@ static int __tcp_transmit_skb(struct sock *sk, struct sk_buff *skb,
 		 */
 		th->window	= htons(min(tp->rcv_wnd, 65535U));
 	}
+
+	tcp_options_write((__be32 *)(th + 1), tp, &opts);
+
 #ifdef CONFIG_TCP_MD5SIG
 	/* Calculate the MD5 hash, as we have all we need now */
 	if (md5) {
net/mptcp/options.c +12 -1
index f2d1e27a2bc1..2e9f1c4ea008 100644
@@ -991,7 +991,24 @@ void mptcp_incoming_options(struct sock *sk, struct sk_buff *skb)
 	}
 }
 
-void mptcp_write_options(__be32 *ptr, struct mptcp_out_options *opts)
+static void mptcp_set_rwin(const struct tcp_sock *tp)
+{
+	const struct sock *ssk = (const struct sock *)tp;
+	const struct mptcp_subflow_context *subflow;
+	struct mptcp_sock *msk;
+	u64 ack_seq;
+
+	subflow = mptcp_subflow_ctx(ssk);
+	msk = mptcp_sk(subflow->conn);
+
+	ack_seq = READ_ONCE(msk->ack_seq) + tp->rcv_wnd;
+
Quoted 8 lines

net/mptcp/options.c void mptcp_incoming_options(struct sock *sk, struct sk_buff *skb) · line 1007

Quoted 2 lines
net/mptcp/options.c +2 -0
+	if (after64(ack_seq, READ_ONCE(msk->rcv_wnd_sent)))
+		WRITE_ONCE(msk->rcv_wnd_sent, ack_seq);

This looks racy if packet headers on multiple subflows are created
concurrently: msk->rcv_wnd_sent could end up with an earlier sequence number than it should have, leading to possible extra drops. It looks like a hard race to trigger, but do you think rcv_wnd_sent should use cmpxchg like wnd_end and snd_una?

I personally don't have a strong opinion on this, I guess we can live with the potential race, we have MIBs reporting both MPTCP retransmissions and OOW events, which would be both triggered if this race happens and have any conseguence.

Possibly related: as some point we will probabaly need to consolidate the cmpxchg()s with some other locking mechanims to reduce the atomic operations needed.

Cheers,

/P

author
#3 lore raw
Quoted 25 lines · Paolo Abeni
Quoted 5 lines · Mat Martineau

On Thu, 22 Oct 2020, Florian Westphal wrote:

(quoted diff) +0 -0
@@ -1348,7 +1349,6 @@ static int __tcp_transmit_skb(struct sock *sk, struct sk_buff *skb,
 > 		}
 > 	}
Quoted 1 line
Quoted 10 lines
Quoted 4 lines
net/ipv4/tcp_output.c +0 -1
-	tcp_options_write((__be32 *)(th + 1), tp, &opts);
	skb_shinfo(skb)->gso_type = sk->sk_gso_type;
	if (likely(!(tcb->tcp_flags & TCPHDR_SYN))) {
		th->window      = htons(tcp_select_window(sk));

These lines of code between the old tcp_options_write() location and the new location were last touched by Eric, doing some cache miss optimizations. It doesn't seem like populating the TCP options just after th->window instead of just before should behave similarly with the data cache - are you concerned about this at all?

Thank you for the carefully review!

commit 51466a7545b73b7ad7bcfb33410d2823ccfaa501

Author: Eric Dumazet <edumazet(a)google.com>
Date:   Thu Jun 11 09:15:16 2015 -0700
    tcp: fill shinfo->gso_type at last moment
Quoted 8 lines

moves 'skb_shinfo(skb)->gso_type' initialization just after the option write. Looking at the commit message, what actually makes a difference in such commit is moving the mentioned assigment after the last xmit check to avoid touching extra cache lines if not needed - and we preserve that fact.

I think the above is sane performace-wise, but I guess Eric could still teach me better ;)

If neither additional hooking nor change in tcp stack is possible only choice is to relax OOW test:

max_seq = READ_ONCE(msk->ack_seq) + READ_ONCE(sk->sk_rcvbuf); if (after64(end_seq, max_seq)) ...

that means we will queue up OOW packets if those are still within current rcvbuf size but its fine from protocol pov.

#4 lore raw
Quoted 32 lines · Florian Westphal
Quoted 31 lines · Paolo Abeni
Quoted 5 lines · Mat Martineau

On Thu, 22 Oct 2020, Florian Westphal wrote:

(quoted diff) +0 -0
@@ -1348,7 +1349,6 @@ static int __tcp_transmit_skb(struct sock *sk, struct sk_buff *skb,
 > 		}
 > 	}
Quoted 1 line
Quoted 10 lines
Quoted 4 lines
net/ipv4/tcp_output.c +0 -1
-	tcp_options_write((__be32 *)(th + 1), tp, &opts);
	skb_shinfo(skb)->gso_type = sk->sk_gso_type;
	if (likely(!(tcb->tcp_flags & TCPHDR_SYN))) {
		th->window      = htons(tcp_select_window(sk));

These lines of code between the old tcp_options_write() location and the new location were last touched by Eric, doing some cache miss optimizations. It doesn't seem like populating the TCP options just after th->window instead of just before should behave similarly with the data cache - are you concerned about this at all?

Thank you for the carefully review!

commit 51466a7545b73b7ad7bcfb33410d2823ccfaa501

Author: Eric Dumazet <edumazet(a)google.com>
Date:   Thu Jun 11 09:15:16 2015 -0700
    tcp: fill shinfo->gso_type at last moment

moves 'skb_shinfo(skb)->gso_type' initialization just after the option write. Looking at the commit message, what actually makes a difference in such commit is moving the mentioned assigment after the last xmit check to avoid touching extra cache lines if not needed - and we preserve that fact.

Quoted 4 lines
Quoted 2 lines

I think the above is sane performace-wise, but I guess Eric could still teach me better ;)

If neither additional hooking nor change in tcp stack is possible only choice

I'm sorry, I was likely misleading. I mean: the patch looks fine and safe to me. Perhpas we can try send that upstream as RFC to explicitly ask for feedback ?!?

Thanks!

Paolo

#5 lore raw
Quoted 8 lines · Paolo Abeni
Quoted 7 lines · Florian Westphal
Quoted 6 lines · Paolo Abeni
Quoted 5 lines · Mat Martineau

On Thu, 22 Oct 2020, Florian Westphal wrote:

(quoted diff) +0 -0
@@ -1348,7 +1349,6 @@ static int __tcp_transmit_skb(struct sock *sk, struct sk_buff *skb,
 > 		}
 > 	}
Quoted 1 line
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 4 lines
net/ipv4/tcp_output.c +0 -1
-	tcp_options_write((__be32 *)(th + 1), tp, &opts);
	skb_shinfo(skb)->gso_type = sk->sk_gso_type;
	if (likely(!(tcb->tcp_flags & TCPHDR_SYN))) {
		th->window      = htons(tcp_select_window(sk));

These lines of code between the old tcp_options_write() location and the new location were last touched by Eric, doing some cache miss optimizations. It doesn't seem like populating the TCP options just after th->window instead of just before should behave similarly with the data cache - are you concerned about this at all?

I made a mistake re-editing my last sentence here - was trying to say it *does* seem like the cache behavior should be similar.

Quoted 15 lines
Quoted 15 lines
Quoted 15 lines

Thank you for the carefully review!

commit 51466a7545b73b7ad7bcfb33410d2823ccfaa501

Author: Eric Dumazet <edumazet(a)google.com>
Date:   Thu Jun 11 09:15:16 2015 -0700
    tcp: fill shinfo->gso_type at last moment

moves 'skb_shinfo(skb)->gso_type' initialization just after the option write. Looking at the commit message, what actually makes a difference in such commit is moving the mentioned assigment after the last xmit check to avoid touching extra cache lines if not needed - and we preserve that fact.

Quoted 9 lines
Quoted 4 lines
Quoted 2 lines

I think the above is sane performace-wise, but I guess Eric could still teach me better ;)

If neither additional hooking nor change in tcp stack is possible only choice

I'm sorry, I was likely misleading. I mean: the patch looks fine and safe to me. Perhpas we can try send that upstream as RFC to explicitly ask for feedback ?!?

Agreed!

--
Mat Martineau
Intel

#6 lore raw
In reply to Florian Westphal

Hi Florian, Mat, Paolo,

Quoted 9 lines · Florian Westphal

OoO handling attemtps to detect when packet is out-of-window by testing current ack sequence and remaining space vs. sequence number.

This doesn't work reliably. Store the highest allowed sequence number that we've announced and use it to detect oow packets.

Do this when mptcp options get written to the packet (wire format). For this to work we need to move the write_options call until after stack selected a new tcp window.

Thank you for the patch and reviews!

- 99873062750d: mptcp: track window announced to peer
- I added Mat's Reviewed-by and Paolo's Acked-by tags, please tell me
if you prefer when I don't add those if they are not explicit. It is just to avoid multiple messages on the netdev ML.
- (I fixed a small typo with 'attempts' in the commit message and
reported by codespell)
- Results: 9bceebfe0a58..c041aa4e115c

Tests + export are in progress!

Cheers,
Matt

Signature · 2 lines
Tessares | Belgium | Hybrid Access Solutions
www.tessares.net