Skip to contentSkip to conversation

mptcp: Use 32-bit DATA_ACK when possible

RFC In review Christoph Paasch 2 participants 3 messages May 13, 2020

Subsystems (2)
lore
Share & tools
lore

Christoph Paasch

author
#0 lore raw

RFC 1/1 mptcp: Use 32-bit DATA_ACK when possible

RFC8684 allows to send 32-bit DATA_ACKs as long as the peer is not sending 64-bit data-sequence numbers. The 64-bit DSN is only there for extreme scenarios when a very high throughput subflow is combined with a long-RTT subflow such that the high-throughput subflow wraps around the 32-bit sequence number space within an RTT of the high-RTT subflow.

It is thus a rare scenario and we should try to use the 32-bit DATA_ACK instead as long as possible. It allows to reduce the TCP-option overhead by 4 bytes, thus makes space for an additional SACK-block. It also makes tcpdumps much easier to read when the DSN and DATA_ACK are both either 32 or 64-bit.

Signed-off-by Christoph Paasch <cpaasch(a)apple.com>
---
 include/net/mptcp.h  |  1 +
 net/mptcp/options.c  | 33 ++++++++++++++++++++++++---------
 net/mptcp/protocol.h |  1 +
 net/mptcp/subflow.c  |  2 ++
 4 files changed, 28 insertions(+), 9 deletions(-)
include/net/mptcp.h +1 -0
index e60275659de6..25d8b36ed94f 100644
@@ -17,6 +17,7 @@ struct seq_file;
 /* MPTCP sk_buff extension data */
 struct mptcp_ext {
 	u64		data_ack;
+	u32		data_ack32;
 	u64		data_seq;
 	u32		subflow_seq;
 	u16		data_len;
net/mptcp/options.c +24 -9
index 45497af23906..ece6f92cf7d1 100644
@@ -516,7 +516,16 @@ static bool mptcp_established_options_dss(struct sock *sk, struct sk_buff *skb,
 		return ret;
 	}
 
-	ack_size = TCPOLEN_MPTCP_DSS_ACK64;
+	if (subflow->use_64bit_ack) {
+		ack_size = TCPOLEN_MPTCP_DSS_ACK64;
+		opts->ext_copy.data_ack = msk->ack_seq;
+		opts->ext_copy.ack64 = 1;
+	} else {
+		ack_size = TCPOLEN_MPTCP_DSS_ACK32;
+		opts->ext_copy.data_ack32 = (uint32_t)(msk->ack_seq);
+		opts->ext_copy.ack64 = 0;
+	}
+	opts->ext_copy.use_ack = 1;
 
 	/* Add kind/length/subtype/flag overhead if mapping is not populated */
 	if (dss_size == 0)
@@ -524,10 +533,6 @@ static bool mptcp_established_options_dss(struct sock *sk, struct sk_buff *skb,
 
 	dss_size += ack_size;
 
-	opts->ext_copy.data_ack = msk->ack_seq;
-	opts->ext_copy.ack64 = 1;
-	opts->ext_copy.use_ack = 1;
-
 	*size = ALIGN(dss_size, 4);
 	return true;
 }
@@ -986,8 +991,13 @@ void mptcp_write_options(__be32 *ptr, struct mptcp_out_options *opts)
 		u8 flags = 0;
 
 		if (mpext->use_ack) {
-			len += TCPOLEN_MPTCP_DSS_ACK64;
-			flags = MPTCP_DSS_HAS_ACK | MPTCP_DSS_ACK64;
+			flags = MPTCP_DSS_HAS_ACK;
+			if (mpext->ack64) {
+				len += TCPOLEN_MPTCP_DSS_ACK64;
+				flags |= MPTCP_DSS_ACK64;
+			} else {
+				len += TCPOLEN_MPTCP_DSS_ACK32;
+			}
 		}
 
 		if (mpext->use_map) {
@@ -1004,8 +1014,13 @@ void mptcp_write_options(__be32 *ptr, struct mptcp_out_options *opts)
 		*ptr++ = mptcp_option(MPTCPOPT_DSS, len, 0, flags);
 
 		if (mpext->use_ack) {
-			put_unaligned_be64(mpext->data_ack, ptr);
-			ptr += 2;
+			if (mpext->ack64) {
+				put_unaligned_be64(mpext->data_ack, ptr);
+				ptr += 2;
+			} else {
+				put_unaligned_be32(mpext->data_ack32, ptr);
+				ptr += 1;
+			}
 		}
 
 		if (mpext->use_map) {
net/mptcp/protocol.h +1 -0
index e4ca6320ce76..f5adca93e8fb 100644
@@ -290,6 +290,7 @@ struct mptcp_subflow_context {
 		data_avail : 1,
 		rx_eof : 1,
 		data_fin_tx_enable : 1,
+		use_64bit_ack : 1, /* Set when we received a 64-bit DSN */
 		can_ack : 1;	    /* only after processing the remote a key */
 	u64	data_fin_tx_seq;
 	u32	remote_nonce;
net/mptcp/subflow.c +2 -0
index 009d5c478062..f22dad482cd4 100644
@@ -661,9 +661,11 @@ static enum mapping_status get_mapping_status(struct sock *ssk)
 	if (!mpext->dsn64) {
 		map_seq = expand_seq(subflow->map_seq, subflow->map_data_len,
 				     mpext->data_seq);
+		subflow->use_64bit_ack = 0;
 		pr_debug("expanded seq=%llu", subflow->map_seq);
 	} else {
 		map_seq = mpext->data_seq;
+		subflow->use_64bit_ack = 1;
 	}
 
 	if (subflow->map_valid) {
Signature · 1 line
2.23.0
#1 lore raw
Quoted 25 lines · Christoph Paasch

RFC8684 allows to send 32-bit DATA_ACKs as long as the peer is not sending 64-bit data-sequence numbers. The 64-bit DSN is only there for extreme scenarios when a very high throughput subflow is combined with a long-RTT subflow such that the high-throughput subflow wraps around the 32-bit sequence number space within an RTT of the high-RTT subflow.

It is thus a rare scenario and we should try to use the 32-bit DATA_ACK instead as long as possible. It allows to reduce the TCP-option overhead by 4 bytes, thus makes space for an additional SACK-block. It also makes tcpdumps much easier to read when the DSN and DATA_ACK are both either 32 or 64-bit.

Signed-off-by Christoph Paasch <cpaasch(a)apple.com>
---
 include/net/mptcp.h  |  1 +
 net/mptcp/options.c  | 33 ++++++++++++++++++++++++---------
 net/mptcp/protocol.h |  1 +
 net/mptcp/subflow.c  |  2 ++
 4 files changed, 28 insertions(+), 9 deletions(-)
include/net/mptcp.h +0 -0
index e60275659de6..25d8b36ed94f 100644

include/net/mptcp.h struct seq_file; · line 20

Quoted 4 lines
include/net/mptcp.h +1 -0
@@ -17,6 +17,7 @@ struct seq_file;
 /* MPTCP sk_buff extension data */
 struct mptcp_ext {
 	u64		data_ack;
+	u32		data_ack32;

I think the above 2 fields could be "union-ed". I think it's preferrable to avoid growing mptcp_ext size.

Quoted 12 lines
include/net/mptcp.h +0 -0
 	u64		data_seq;
 	u32		subflow_seq;
 	u16		data_len;
net/mptcp/options.c +0 -1
index 45497af23906..ece6f92cf7d1 100644
@@ -516,7 +516,16 @@ static bool mptcp_established_options_dss(struct sock *sk, struct sk_buff *skb,
 		return ret;
 	}
 
-	ack_size = TCPOLEN_MPTCP_DSS_ACK64;

net/mptcp/options.c static bool mptcp_established_options_dss(struct sock *sk, struct sk_buff *skb, · line 528

Quoted 10 lines
net/mptcp/options.c +10 -0
+	if (subflow->use_64bit_ack) {
+		ack_size = TCPOLEN_MPTCP_DSS_ACK64;
+		opts->ext_copy.data_ack = msk->ack_seq;
+		opts->ext_copy.ack64 = 1;
+	} else {
+		ack_size = TCPOLEN_MPTCP_DSS_ACK32;
+		opts->ext_copy.data_ack32 = (uint32_t)(msk->ack_seq);
+		opts->ext_copy.ack64 = 0;
+	}
+	opts->ext_copy.use_ack = 1;

If I read correctly, 'use_64bit_ack' is initialized with 0 and only set if the peer sends 64bits sequence number. When interacting with impl using 32bits DSN, our stack will flip the behavior at least once for passive socket:

MPC+data is internally translated into DSS with 64bit DSN, so first passive socket ack will use 64 bits.

Later, if the client uses 32 bits acks, it will switch to 32 bit acks.

Is that ok?

Quoted 50 lines
net/mptcp/options.c +14 -8
 	/* Add kind/length/subtype/flag overhead if mapping is not populated */
 	if (dss_size == 0)
@@ -524,10 +533,6 @@ static bool mptcp_established_options_dss(struct sock *sk, struct sk_buff *skb,
 
 	dss_size += ack_size;
 
-	opts->ext_copy.data_ack = msk->ack_seq;
-	opts->ext_copy.ack64 = 1;
-	opts->ext_copy.use_ack = 1;
-
 	*size = ALIGN(dss_size, 4);
 	return true;
 }
@@ -986,8 +991,13 @@ void mptcp_write_options(__be32 *ptr, struct mptcp_out_options *opts)
 		u8 flags = 0;
 
 		if (mpext->use_ack) {
-			len += TCPOLEN_MPTCP_DSS_ACK64;
-			flags = MPTCP_DSS_HAS_ACK | MPTCP_DSS_ACK64;
+			flags = MPTCP_DSS_HAS_ACK;
+			if (mpext->ack64) {
+				len += TCPOLEN_MPTCP_DSS_ACK64;
+				flags |= MPTCP_DSS_ACK64;
+			} else {
+				len += TCPOLEN_MPTCP_DSS_ACK32;
+			}
 		}
 
 		if (mpext->use_map) {
@@ -1004,8 +1014,13 @@ void mptcp_write_options(__be32 *ptr, struct mptcp_out_options *opts)
 		*ptr++ = mptcp_option(MPTCPOPT_DSS, len, 0, flags);
 
 		if (mpext->use_ack) {
-			put_unaligned_be64(mpext->data_ack, ptr);
-			ptr += 2;
+			if (mpext->ack64) {
+				put_unaligned_be64(mpext->data_ack, ptr);
+				ptr += 2;
+			} else {
+				put_unaligned_be32(mpext->data_ack32, ptr);
+				ptr += 1;
+			}
 		}
 
 		if (mpext->use_map) {
net/mptcp/protocol.h +0 -0
index e4ca6320ce76..f5adca93e8fb 100644

net/mptcp/protocol.h struct mptcp_subflow_context { · line 296

Quoted 7 lines
net/mptcp/protocol.h +1 -0
@@ -290,6 +290,7 @@ struct mptcp_subflow_context {
 		data_avail : 1,
 		rx_eof : 1,
 		data_fin_tx_enable : 1,
+		use_64bit_ack : 1, /* Set when we received a 64-bit DSN */
 		can_ack : 1;	    /* only after processing the remote a key */
 	u64	data_fin_tx_seq;
 	u32	remote_nonce;

Possibly with a wider scope than this patch, but the above bitfields contains both fields that are rarely changed (data_fin_tx_enable, can_ack, request_mptcp, etc...) and fields that are set on every packet /every DSS (map_vaild, data_avail, the newly added use_64bit_ack).

At least 'data_avail' is accessed without holding the ssk socket lock by mptcp_subflow_recv_lookup(). Perhaps we could revisit the binary layout of this struct?

Other than the above LGTM!

Cheers,

Paolo

author
#2 lore raw
Quoted 21 lines · Paolo Abeni
Quoted 20 lines · Christoph Paasch

RFC8684 allows to send 32-bit DATA_ACKs as long as the peer is not sending 64-bit data-sequence numbers. The 64-bit DSN is only there for extreme scenarios when a very high throughput subflow is combined with a long-RTT subflow such that the high-throughput subflow wraps around the 32-bit sequence number space within an RTT of the high-RTT subflow.

It is thus a rare scenario and we should try to use the 32-bit DATA_ACK instead as long as possible. It allows to reduce the TCP-option overhead by 4 bytes, thus makes space for an additional SACK-block. It also makes tcpdumps much easier to read when the DSN and DATA_ACK are both either 32 or 64-bit.

Signed-off-by Christoph Paasch <cpaasch(a)apple.com>
---
 include/net/mptcp.h  |  1 +
 net/mptcp/options.c  | 33 ++++++++++++++++++++++++---------
 net/mptcp/protocol.h |  1 +
 net/mptcp/subflow.c  |  2 ++
 4 files changed, 28 insertions(+), 9 deletions(-)
Quoted 12 lines

include/net/mptcp.h struct seq_file; · line 20

Quoted 9 lines
include/net/mptcp.h +1 -0
index e60275659de6..25d8b36ed94f 100644
@@ -17,6 +17,7 @@ struct seq_file;
 /* MPTCP sk_buff extension data */
 struct mptcp_ext {
 	u64		data_ack;
+	u32		data_ack32;

I think the above 2 fields could be "union-ed". I think it's preferrable to avoid growing mptcp_ext size.

Sounds good!

Quoted 29 lines
Quoted 13 lines
include/net/mptcp.h +0 -0
 	u64		data_seq;
 	u32		subflow_seq;
 	u16		data_len;
net/mptcp/options.c +0 -1
index 45497af23906..ece6f92cf7d1 100644
@@ -516,7 +516,16 @@ static bool mptcp_established_options_dss(struct sock *sk, struct sk_buff *skb,
 		return ret;
 	}
 
-	ack_size = TCPOLEN_MPTCP_DSS_ACK64;

net/mptcp/options.c static bool mptcp_established_options_dss(struct sock *sk, struct sk_buff *skb, · line 528

Quoted 10 lines
net/mptcp/options.c +10 -0
+	if (subflow->use_64bit_ack) {
+		ack_size = TCPOLEN_MPTCP_DSS_ACK64;
+		opts->ext_copy.data_ack = msk->ack_seq;
+		opts->ext_copy.ack64 = 1;
+	} else {
+		ack_size = TCPOLEN_MPTCP_DSS_ACK32;
+		opts->ext_copy.data_ack32 = (uint32_t)(msk->ack_seq);
+		opts->ext_copy.ack64 = 0;
+	}
+	opts->ext_copy.use_ack = 1;

If I read correctly, 'use_64bit_ack' is initialized with 0 and only set if the peer sends 64bits sequence number. When interacting with impl using 32bits DSN, our stack will flip the behavior at least once for passive socket:

Quoted 6 lines

MPC+data is internally translated into DSS with 64bit DSN, so first passive socket ack will use 64 bits.

Later, if the client uses 32 bits acks, it will switch to 32 bit acks.

Is that ok?

Yes, I saw that behavior in my tests. I think it is ok. For the initial data there won't be a SACK-block, so nothing is lost there.

Quoted 63 lines
Quoted 50 lines
net/mptcp/options.c +14 -8
 	/* Add kind/length/subtype/flag overhead if mapping is not populated */
 	if (dss_size == 0)
@@ -524,10 +533,6 @@ static bool mptcp_established_options_dss(struct sock *sk, struct sk_buff *skb,
 
 	dss_size += ack_size;
 
-	opts->ext_copy.data_ack = msk->ack_seq;
-	opts->ext_copy.ack64 = 1;
-	opts->ext_copy.use_ack = 1;
-
 	*size = ALIGN(dss_size, 4);
 	return true;
 }
@@ -986,8 +991,13 @@ void mptcp_write_options(__be32 *ptr, struct mptcp_out_options *opts)
 		u8 flags = 0;
 
 		if (mpext->use_ack) {
-			len += TCPOLEN_MPTCP_DSS_ACK64;
-			flags = MPTCP_DSS_HAS_ACK | MPTCP_DSS_ACK64;
+			flags = MPTCP_DSS_HAS_ACK;
+			if (mpext->ack64) {
+				len += TCPOLEN_MPTCP_DSS_ACK64;
+				flags |= MPTCP_DSS_ACK64;
+			} else {
+				len += TCPOLEN_MPTCP_DSS_ACK32;
+			}
 		}
 
 		if (mpext->use_map) {
@@ -1004,8 +1014,13 @@ void mptcp_write_options(__be32 *ptr, struct mptcp_out_options *opts)
 		*ptr++ = mptcp_option(MPTCPOPT_DSS, len, 0, flags);
 
 		if (mpext->use_ack) {
-			put_unaligned_be64(mpext->data_ack, ptr);
-			ptr += 2;
+			if (mpext->ack64) {
+				put_unaligned_be64(mpext->data_ack, ptr);
+				ptr += 2;
+			} else {
+				put_unaligned_be32(mpext->data_ack32, ptr);
+				ptr += 1;
+			}
 		}
 
 		if (mpext->use_map) {
net/mptcp/protocol.h +0 -0
index e4ca6320ce76..f5adca93e8fb 100644

net/mptcp/protocol.h struct mptcp_subflow_context { · line 296

Quoted 7 lines
net/mptcp/protocol.h +1 -0
@@ -290,6 +290,7 @@ struct mptcp_subflow_context {
 		data_avail : 1,
 		rx_eof : 1,
 		data_fin_tx_enable : 1,
+		use_64bit_ack : 1, /* Set when we received a 64-bit DSN */
 		can_ack : 1;	    /* only after processing the remote a key */
 	u64	data_fin_tx_seq;
 	u32	remote_nonce;

Possibly with a wider scope than this patch, but the above bitfields contains both fields that are rarely changed (data_fin_tx_enable, can_ack, request_mptcp, etc...) and fields that are set on every packet /every DSS (map_vaild, data_avail, the newly added use_64bit_ack).

Quoted 3 lines

At least 'data_avail' is accessed without holding the ssk socket lock by mptcp_subflow_recv_lookup(). Perhaps we could revisit the binary layout of this struct?

Yes, that might be good: https://github.com/multipath-tcp/mptcp_net-next/issues/24

Quoted 1 line

Other than the above LGTM!

Cool! I will make the union and send to netnext if there are no other comments here on the list.

Thanks,
Christoph