Skip to contentSkip to conversation

Implement MPTCPv1 Handshake

PATCH In review Christoph Paasch 4 participants 9 messages Nov 7, 2019 – Nov 30, 2019

Subsystems (3)
lore
Share & tools
lore

Christoph Paasch

author
#0 lore raw

cover Implement MPTCPv1 Handshake

Implement the MP_CAPABLE exchange from RFC6824bis, allowing for MPTCP to work reliably with stateless servers (aka., those using TCP SYN-cookies). The trick is to reliably transmit the keys from the client to the server by combining them with data. As the data will be retransmitted, the keys will reach the server as well.

Christoph Paasch (2):
mptcp: parse and emit MP_CAPABLE option according to v1 spec.
mptcp: process MP_CAPABLE data option.

 include/linux/tcp.h   |   3 +-
 include/net/mptcp.h   |  11 +-
 net/ipv4/tcp_input.c  |   2 +-
 net/ipv4/tcp_output.c |   2 +-
 net/mptcp/options.c   | 246 ++++++++++++++++++++++++++++++++----------
 net/mptcp/protocol.c  |  24 +++--
 net/mptcp/protocol.h  |  14 ++-
 net/mptcp/subflow.c   |  42 +++++++-
 8 files changed, 261 insertions(+), 83 deletions(-)
Signature · 1 line
2.23.0
author
#1 lore raw

PATCH 1/2 mptcp: parse and emit MP_CAPABLE option according to v1 spec.

Local key is sent on syn/ack, and both keys are sent on 3rd ack. MP_CAPABLE messages len are updated accordingly.
We need the skbuff to emit correctly the above, so we push the skbuff struct as an argument all the way from tcp code to the relevant mptcp callbacks.

When processing incoming MP_CAPABLE + data, build a full blown DSS-like map info, to simplify later processing.
On child socket creation, we need to record the remote key, if available.

Signed-off-by Christoph Paasch <cpaasch(a)apple.com>
---
 include/linux/tcp.h   |   3 +-
 include/net/mptcp.h   |  11 +--
 net/ipv4/tcp_input.c  |   2 +-
 net/ipv4/tcp_output.c |   2 +-
 net/mptcp/options.c   | 186 +++++++++++++++++++++++++++++++-----------
 net/mptcp/protocol.c  |   2 +-
 net/mptcp/protocol.h  |   6 +-
 net/mptcp/subflow.c   |  12 ++-
 8 files changed, 163 insertions(+), 61 deletions(-)
include/linux/tcp.h +2 -1
index 41730d9bcb32..8c8d51d65adf 100644
@@ -120,7 +120,8 @@ struct tcp_options_received {
 			data_fin:1,
 			use_ack:1,
 			ack64:1,
-			__unused:3;
+			mp_capable_map:1,
+			__unused:2;
 		u8	add_addr : 1,
 			rm_addr : 1,
 			family : 4;
include/net/mptcp.h +6 -5
index eba39a881767..babff18e4ee8 100644
@@ -25,7 +25,8 @@ struct mptcp_ext {
 			data_fin:1,
 			use_ack:1,
 			ack64:1,
-			__unused:2;
+			mp_capable_map:1,
+			__unused:1;
 };
 
 struct mptcp_out_options {
@@ -64,10 +65,10 @@ static inline bool rsk_is_mptcp(const struct request_sock *req)
 	return tcp_rsk(req)->is_mptcp;
 }
 
-void mptcp_parse_option(const unsigned char *ptr, int opsize,
-			struct tcp_options_received *opt_rx);
-bool mptcp_syn_options(struct sock *sk, unsigned int *size,
-		       struct mptcp_out_options *opts);
+void mptcp_parse_option(const struct sk_buff *skb, const unsigned char *ptr,
+			int opsize, struct tcp_options_received *opt_rx);
+bool mptcp_syn_options(struct sock *sk, const struct sk_buff *skb,
+		       unsigned int *size, struct mptcp_out_options *opts);
 void mptcp_rcv_synsent(struct sock *sk);
 bool mptcp_synack_options(const struct request_sock *req, unsigned int *size,
 			  struct mptcp_out_options *opts);
net/ipv4/tcp_input.c +1 -1
index d1ab9edfb764..16bd46a097d5 100644
@@ -3922,7 +3922,7 @@ void tcp_parse_options(const struct net *net,
 				break;
 #endif
 			case TCPOPT_MPTCP:
-				mptcp_parse_option(ptr, opsize, opt_rx);
+				mptcp_parse_option(skb, ptr, opsize, opt_rx);
 				break;
 
 			case TCPOPT_FASTOPEN:
net/ipv4/tcp_output.c +1 -1
index f864749a1ec1..a9a0e61285b1 100644
@@ -682,7 +682,7 @@ static unsigned int tcp_syn_options(struct sock *sk, struct sk_buff *skb,
 	if (sk_is_mptcp(sk)) {
 		unsigned int size;
 
-		if (mptcp_syn_options(sk, &size, &opts->mptcp)) {
+		if (mptcp_syn_options(sk, skb, &size, &opts->mptcp)) {
 			opts->options |= OPTION_MPTCP;
 			remaining -= size;
 		}
net/mptcp/options.c +139 -47

294 lines not on this page · raw message

net/mptcp/protocol.c +1 -1
index 656e2fda0c7e..9ad735a62330 100644
@@ -1093,7 +1093,7 @@ static struct socket *mptcp_socket_create_get(struct mptcp_sock *msk)
 	msk->subflow = ssock;
 	subflow = mptcp_subflow_ctx(msk->subflow->sk);
 	subflow->request_mptcp = 1; /* @@ if MPTCP enabled */
-	subflow->request_version = 0; /* currently only v0 supported */
+	subflow->request_version = 1; /* only v1 supported */
 
 	sock_hold(ssock->sk);
 
net/mptcp/protocol.h +4 -2
index 791f2c19cfb8..5c96777f9b3c 100644
@@ -33,9 +33,10 @@
 #define MPTCPOPT_MP_FASTCLOSE	7
 
 /* MPTCP suboption lengths */
-#define TCPOLEN_MPTCP_MPC_SYN		12
-#define TCPOLEN_MPTCP_MPC_SYNACK	20
+#define TCPOLEN_MPTCP_MPC_SYN		4
+#define TCPOLEN_MPTCP_MPC_SYNACK	12
 #define TCPOLEN_MPTCP_MPC_ACK		20
+#define TCPOLEN_MPTCP_MPC_ACK_DATA	22
 #define TCPOLEN_MPTCP_MPJ_SYN		12
 #define TCPOLEN_MPTCP_MPJ_SYNACK	16
 #define TCPOLEN_MPTCP_MPJ_ACK		24
@@ -200,6 +201,7 @@ struct mptcp_subflow_context {
 	u64	remote_key;
 	u64	idsn;
 	u64	map_seq;
+	u32	snd_isn;
 	u32	token;
 	u32	rel_write_seq;
 	u32	map_subflow_seq;
net/mptcp/subflow.c +9 -3
index f08c4b713978..dce5bf137eea 100644
@@ -129,7 +129,6 @@ static void subflow_v4_init_req(struct request_sock *req,
 			subflow_req->version = listener->request_version;
 		else
 			subflow_req->version = rx_opt.mptcp.version;
-		subflow_req->remote_key = rx_opt.mptcp.sndr_key;
 		subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq;
 	} else if (rx_opt.mptcp.mp_join && listener->request_mptcp) {
 		subflow_req->mp_join = 1;
@@ -266,9 +265,16 @@ static struct sock *subflow_syn_recv_sock(const struct sock *sk,
 
 	pr_debug("listener=%p, req=%p, conn=%p", listener, req, listener->conn);
 
-	/* if the sk is MP_CAPABLE, we already received the client key */
+	/* if the sk is MP_CAPABLE, we need to fetch the client key */
 	subflow_req = mptcp_subflow_rsk(req);
-	if (!subflow_req->mp_capable && subflow_req->mp_join) {
+	if (subflow_req->mp_capable) {
+		opt_rx.mptcp.mp_capable = 0;
+		mptcp_get_options(skb, &opt_rx);
+		if (!opt_rx.mptcp.mp_capable)
+			subflow_req->mp_capable = 0;
+		else
+			subflow_req->remote_key = opt_rx.mptcp.sndr_key;
+	} else if (subflow_req->mp_join) {
 		opt_rx.mptcp.mp_join = 0;
 		mptcp_get_options(skb, &opt_rx);
 		if (!opt_rx.mptcp.mp_join ||
Signature · 1 line
2.23.0
#2 lore raw

[MPTCP] Re: [PATCH 1/2] mptcp: parse and emit MP_CAPABLE option according to v1 spec.

Hi,

Quoted 34 lines · Christoph Paasch
net/mptcp/options.c +16 -9
@@ -356,28 +387,66 @@ void mptcp_rcv_synsent(struct sock *sk)
 	}
 }
 
-static bool mptcp_established_options_mp(struct sock *sk, unsigned int *size,
+static bool mptcp_established_options_mp(struct sock *sk, struct sk_buff *skb,
+					 unsigned int *size,
 					 unsigned int remaining,
 					 struct mptcp_out_options *opts)
 {
 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
+	struct mptcp_ext *mpext;
+	unsigned int data_len;
+
+	pr_debug("subflow=%p fourth_ack=%d seq=%x:%x remaining=%d", subflow,
+		 subflow->fourth_ack, subflow->snd_isn,
+		 skb ? TCP_SKB_CB(skb)->seq : 0, remaining);
+
+	if (subflow->mp_capable && !subflow->fourth_ack && skb &&
+	    subflow->snd_isn == TCP_SKB_CB(skb)->seq) {
+		/* When skb is not available, we better over-estimate the
+		 * emitted options len. A full DSS option is longer than
+		 * TCPOLEN_MPTCP_MPC_ACK_DATA, so let's the caller try to fit
+		 * that.
+		 */
 
-	if (subflow->mp_capable && !subflow->fourth_ack &&
-	    remaining >= TCPOLEN_MPTCP_MPC_ACK) {
-		opts->suboptions = OPTION_MPTCP_MPC_ACK;
-		opts->sndr_key = subflow->local_key;
-		opts->rcvr_key = subflow->remote_key;
-		*size = TCPOLEN_MPTCP_MPC_ACK;
-		subflow->fourth_ack = 1;
-		pr_debug("subflow=%p, local_key=%llu, remote_key=%llu",

net/mptcp/options.c void mptcp_rcv_synsent(struct sock *sk) · line 418

Quoted 10 lines
net/mptcp/options.c +8 -2
-			 subflow, subflow->local_key, subflow->remote_key);
-		return true;
+		mpext = mptcp_get_ext(skb);
+		data_len = mpext ? mpext->data_len : 0;
+		/* Section 3.1.
+		 * The MP_CAPABLE option is carried on the SYN, SYN/ACK, and ACK
+		 * packets that start the first subflow of an MPTCP connection,
+		 * as well as the first packet that carries data
+		 */
+		if (data_len > 0 && remaining >= TCPOLEN_MPTCP_MPC_ACK_DATA) {

It's not clear to me which is the expected behavior when we can't fit MPTCP options inside the TCP header. As the maximum MP_CAPABLE/MP_JOIN opt len is 24 bytes, and we have at most 16 bytes in the previous opt (tstamp, md5) it looks like this can't ever happen, so could emit a WARN_ON_ONCE() here?

BTW we currently emit a warn if DSS exceeds the TCP option space, and that can actually happen (max DSS without csum is 26 bytes, with csum 28). Perhaps we could switch to 32 bits seq if we hit such condition?

Cheers,

Paolo

#3 lore raw

[MPTCP] Re: [PATCH 1/2] mptcp: parse and emit MP_CAPABLE option according to v1 spec.

In reply to Christoph Paasch

Hi Christoph,

Quoted 11 lines · Christoph Paasch

Local key is sent on syn/ack, and both keys are sent on 3rd ack. MP_CAPABLE messages len are updated accordingly.
We need the skbuff to emit correctly the above, so we push the skbuff struct as an argument all the way from tcp code to the relevant mptcp callbacks.

When processing incoming MP_CAPABLE + data, build a full blown DSS-like map info, to simplify later processing.
On child socket creation, we need to record the remote key, if available.

Signed-off-by Christoph Paasch <cpaasch(a)apple.com>

Thank you for sharing this! It looks very nice!

I just have a small question, please see below:

[...]

Quoted 4 lines
net/mptcp/options.c +0 -0
index 1b08e1193991..9cdbd617c49a 100644

[...]

Quoted 15 lines
net/mptcp/options.c +9 -2
@@ -24,12 +24,24 @@ void mptcp_parse_option(const unsigned char *ptr, int opsize,
  	 * 10-17: Receiver key (optional)
  	 */
  	case MPTCPOPT_MP_CAPABLE:
-		if (opsize != TCPOLEN_MPTCP_MPC_SYN &&
-		    opsize != TCPOLEN_MPTCP_MPC_SYNACK)
+		/* strict size checking */
+		if (!(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) {
+			if (skb->len > tcp_hdr(skb)->doff << 2)
+				expected_opsize = TCPOLEN_MPTCP_MPC_ACK_DATA;
+			else
+				expected_opsize = TCPOLEN_MPTCP_MPC_ACK;
+		} else {
+			if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_ACK)
+				expected_opsize = TCPOLEN_MPTCP_MPC_SYNACK;

net/mptcp/options.c void mptcp_parse_option(const unsigned char *ptr, int opsize, · line 44

Quoted 10 lines
net/mptcp/options.c +6 -1
+			else
+				expected_opsize = TCPOLEN_MPTCP_MPC_SYN;
+		}
+		if (opsize != expected_opsize)
  			break;
 
+		/* only support V1 of the protocol */
  		mp_opt->version = *ptr++ & MPTCP_VERSION_MASK;
-		if (mp_opt->version != 0)
+		if (mp_opt->version != 1)

Should we not only "break" here is the version is < 1? To be compatible with possible newer versions.

It seems confirmed by the RFC, section 3.1:

If a host supports multiple versions of MPTCP,
the sender of the MP_CAPABLE option SHOULD signal the highest version number it supports. In return, in its MP_CAPABLE option, the receiver will signal the version number it wishes to use, which MUST be equal to or lower than the version number indicated in the initial MP_CAPABLE.

detail: maybe we need a #define for this minimal version that we support?

Cheers,
Matt

Signature · 5 lines
Matthieu Baerts | R&D Engineer
matthieu.baerts(a)tessares.net
Tessares SA | Hybrid Access Solutions
www.tessares.net
1 Avenue Jean Monnet, 1348 Louvain-la-Neuve, Belgium
#4 lore raw

[MPTCP] Re: [PATCH 1/2] mptcp: parse and emit MP_CAPABLE option according to v1 spec.

In reply to Christoph Paasch

Review

Quoted 4 lines · Christoph Paasch
(quoted diff) +2 -1
-#define TCPOLEN_MPTCP_MPC_SYNACK       20
+#define TCPOLEN_MPTCP_MPC_SYN          4
+#define TCPOLEN_MPTCP_MPC_SYNACK       12
 #define TCPOLEN_MPTCP_MPC_ACK          20

hello Christoph and Paolo,

the length of the 'mp_capable' option in the SYN/ACK doesn't really change when moving from v0 to v1 version of the MPTCP protocol.

Current mptcp net-next echoes the client's key in the SYN/ACK but I think this is not intentional (at least, it does not seem to adhere to section 3.1 of RFC https://tools.ietf.org/html/rfc6824).

For the record, this is why tcpdump refuses to dissect captures of selftests:

 # ./mptcp_connect.sh -4 -c -e

IP 10.0.1.1.49752 > 10.0.1.1.10000: Flags [S], seq 2594718977, win 65535, options [mss 65495,sackOK,TS val 3944347269 ecr 0,nop,wscale 8,mptcp capable {0xf931b304deb39a42}], length 0 IP 10.0.1.1.10000 > 10.0.1.1.49752: Flags [S.], seq 3996234497, ack 2594718978, win 65535, options [mss 65495,sackOK,TS val 3944347270 ecr 3944347269,nop,wscale 8,mptcp capable[bad opt]>
^^^ bad option
IP 10.0.1.1.49752 > 10.0.1.1.10000: Flags [.], ack 1, win 256, options [nop,nop,TS val 3944347270 ecr 3944347270,mptcp capable {0xf931b304deb39a42,0x96614f185e633ce}], length 0

and also, it's the reason why packetdrill never stores the server key in its variables, see:

https://github.com/dcaratti/packetdrill/issues/1#issuecomment-559569544

is it possible to fix size of the syn/ack generated by v0 version of the protocol as a separate patch? Otherwise I will need to add a special handling for SYN/ACK packets generated by v0 net-next kernels packets.

(the code I tried is quite trivial, and it passes the smoke-test in tools/testing/selftests)

--- >8 ---

net/mptcp/options.c +0 -0
@@ -57,16 +57,8 @@ void mptcp_parse_option(const unsigned char *ptr, int

opsize,

                mp_opt->sndr_key = get_unaligned_be64(ptr);
                ptr += 8;

- if (opsize == TCPOLEN_MPTCP_MPC_SYNACK) {

-                       mp_opt->rcvr_key = get_unaligned_be64(ptr);
-                       ptr += 8;
-                       pr_debug("MP_CAPABLE flags=%x, sndr=%llu,

rcvr=%llu",
- mp_opt->flags, mp_opt->sndr_key,
- mp_opt->rcvr_key);
- } else {
- pr_debug("MP_CAPABLE flags=%x, sndr=%llu",
- mp_opt->flags, mp_opt->sndr_key);
- }
+ pr_debug("MP_CAPABLE flags=%x, sndr=%llu",

+                        mp_opt->flags, mp_opt->sndr_key);
                break;
 
        /* MPTCPOPT_MP_JOIN
@@ -650,8 +642,7 @@ void mptcp_write_options(__be32 *ptr, struct
mptcp_out_options *opts)
                                      MPTCP_CAP_HMAC_SHA1);
                put_unaligned_be64(opts->sndr_key, ptr);
                ptr += 2;
-               if ((OPTION_MPTCP_MPC_SYNACK |
-                    OPTION_MPTCP_MPC_ACK) & opts->suboptions) {
+               if (OPTION_MPTCP_MPC_ACK & opts->suboptions) {
                        put_unaligned_be64(opts->rcvr_key, ptr);
                        ptr += 2;
                }
net/mptcp/protocol.h +1 -1
index acad61c70de9..10347c9b4dff 100644
@@ -34,7 +34,7 @@
 
 /* MPTCP suboption lengths */
 #define TCPOLEN_MPTCP_MPC_SYN          12
-#define TCPOLEN_MPTCP_MPC_SYNACK       20
+#define TCPOLEN_MPTCP_MPC_SYNACK       12
 #define TCPOLEN_MPTCP_MPC_ACK          20
 #define TCPOLEN_MPTCP_MPJ_SYN          12
 #define TCPOLEN_MPTCP_MPJ_SYNACK       16

--- >8 ---

any feedback appreciated, thank you in advance!

Signature · 1 line
davide
#5 lore raw

[MPTCP] Re: [PATCH 1/2] mptcp: parse and emit MP_CAPABLE option according to v1 spec.

Hi Davide,

Quoted 8 lines · Davide Caratti

net/mptcp/protocol.h line 38

Quoted 4 lines · Christoph Paasch
net/mptcp/protocol.h +2 -1
-#define TCPOLEN_MPTCP_MPC_SYNACK       20
+#define TCPOLEN_MPTCP_MPC_SYN          4
+#define TCPOLEN_MPTCP_MPC_SYNACK       12
  #define TCPOLEN_MPTCP_MPC_ACK          20

hello Christoph and Paolo,

Quoted 6 lines

the length of the 'mp_capable' option in the SYN/ACK doesn't really change when moving from v0 to v1 version of the MPTCP protocol.

Current mptcp net-next echoes the client's key in the SYN/ACK but I think this is not intentional (at least, it does not seem to adhere to section 3.1 of RFC https://tools.ietf.org/html/rfc6824).

Indeed, good catch, MP_CAPABLE options are taking 12 bytes for both the SYN and SYN/ACK.

Quoted 10 lines

For the record, this is why tcpdump refuses to dissect captures of selftests:

  # ./mptcp_connect.sh -4 -c -e

IP 10.0.1.1.49752 > 10.0.1.1.10000: Flags [S], seq 2594718977, win 65535, options [mss 65495,sackOK,TS val 3944347269 ecr 0,nop,wscale 8,mptcp capable {0xf931b304deb39a42}], length 0 IP 10.0.1.1.10000 > 10.0.1.1.49752: Flags [S.], seq 3996234497, ack 2594718978, win 65535, options [mss 65495,sackOK,TS val 3944347270 ecr 3944347269,nop,wscale 8,mptcp capable[bad opt]>
^^^ bad option
IP 10.0.1.1.49752 > 10.0.1.1.10000: Flags [.], ack 1, win 256, options [nop,nop,TS val 3944347270 ecr 3944347270,mptcp capable {0xf931b304deb39a42,0x96614f185e633ce}], length 0

Quoted 4 lines

and also, it's the reason why packetdrill never stores the server key in its variables, see:

https://github.com/dcaratti/packetdrill/issues/1#issuecomment-559569544

The new version of packetdrill is already helping us finding issues \o/

Quoted 6 lines

is it possible to fix size of the syn/ack generated by v0 version of the protocol as a separate patch? Otherwise I will need to add a special handling for SYN/ACK packets generated by v0 net-next kernels packets.

(the code I tried is quite trivial, and it passes the smoke-test in tools/testing/selftests)

Thank you for the patch! It looks good to me.
I think it would be good to apply it before applying v1 patch.

May you send a "proper patch"? If there is no objection (and an additional review?), I can apply it.

Cheers,
Matt

Signature · 5 lines
Matthieu Baerts | R&D Engineer
matthieu.baerts(a)tessares.net
Tessares SA | Hybrid Access Solutions
www.tessares.net
1 Avenue Jean Monnet, 1348 Louvain-la-Neuve, Belgium
#6 lore raw

[MPTCP] Re: [PATCH 1/2] mptcp: parse and emit MP_CAPABLE option according to v1 spec.

hello Matthieu, thanks a lot for looking at this!

Quoted 6 lines · Matthieu Baerts

Hi Davide,

Quoted 1 line

https://github.com/dcaratti/packetdrill/issues/1#issuecomment-559569544

The new version of packetdrill is already helping us finding issues \o/

yay! :)

Quoted 4 lines
Quoted 4 lines

is it possible to fix size of the syn/ack generated by v0 version of the protocol as a separate patch? Otherwise I will need to add a special handling for SYN/ACK packets generated by v0 net-next kernels packets.

Quoted 8 lines
Quoted 2 lines

(the code I tried is quite trivial, and it passes the smoke-test in tools/testing/selftests)

Thank you for the patch! It looks good to me.
I think it would be good to apply it before applying v1 patch.

May you send a "proper patch"? If there is no objection (and an additional review?), I can apply it.

I'm doubtful of what should we do with the first hunk, i.e.

net/mptcp/options.c +0 -0
@@ -57,16 +57,8 @@ void mptcp_parse_option(const unsigned char *ptr, int

opsize,

                mp_opt->sndr_key = get_unaligned_be64(ptr);
                ptr += 8;

- if (opsize == TCPOLEN_MPTCP_MPC_SYNACK) {

-                       mp_opt->rcvr_key = get_unaligned_be64(ptr);
-                       ptr += 8;
-                       pr_debug("MP_CAPABLE flags=%x, sndr=%llu,

rcvr=%llu",
- mp_opt->flags, mp_opt->sndr_key,
- mp_opt->rcvr_key);
- } else {
- pr_debug("MP_CAPABLE flags=%x, sndr=%llu",
- mp_opt->flags, mp_opt->sndr_key);
- }
+ pr_debug("MP_CAPABLE flags=%x, sndr=%llu",

+                        mp_opt->flags, mp_opt->sndr_key);
                break;
 
        /* MPTCPOPT_MP_JOIN

here the kernel stores in mp_opt->rcvr_key the value received on the "longer" mp_capable option (that is, the 3rd packet in the 3-way handshake). I don't really get what the server should do with the echoed key received by the client, and a grep over the code didn't help:

$ grep -r rcvr_key net/* include/*
net/mptcp/options.c:            opts->rcvr_key = subflow->remote_key;
net/mptcp/options.c:            opts->rcvr_key = subflow_req->remote_key;
net/mptcp/options.c:                    put_unaligned_be64(opts->rcvr_key, ptr);
include/linux/tcp.h:            u64     rcvr_key;
include/net/mptcp.h:    u64 rcvr_key;

I take as a friday activity (when packetdrill will be in a decent shape... so, not not this friday :) ) to understand what happens to a mptcp-net-next server when the client echoes back a wrong key. To be as much conservative as possible (and preserve the debug printout), we could just replace the above hunk with the following one:

--- >8 ---

net/mptcp/options.c +1 -1
@@ -57,7 +57,7 @@ void mptcp_parse_option(const unsigned char *ptr, int opsize,
                mp_opt->sndr_key = get_unaligned_be64(ptr);
                ptr += 8;
 
-               if (opsize == TCPOLEN_MPTCP_MPC_SYNACK) {
+               if (opsize == TCPOLEN_MPTCP_MPC_ACK) {
                        mp_opt->rcvr_key = get_unaligned_be64(ptr);
                        ptr += 8;
                        pr_debug("MP_CAPABLE flags=%x, sndr=%llu, rcvr=%llu",

--- >8 ---

that will also create less troubles to Christian when he rebases his v1 mp_capable patch on top of this.

WDYT?

Signature · 1 line
davide 
#7 lore raw

[MPTCP] Re: [PATCH 1/2] mptcp: parse and emit MP_CAPABLE option according to v1 spec.

Hi Davide,

I prepared this email yesterday but it seems I put my laptop to sleep too quickly :)

Quoted 1 line · Davide Caratti

hello Matthieu, thanks a lot for looking at this!

That's the least I can do :-)

Quoted 1 line · Matthieu Baerts

[...]

Quoted 4 lines
Quoted 2 lines

May you send a "proper patch"? If there is no objection (and an additional review?), I can apply it.

I'm doubtful of what should we do with the first hunk, i.e.

Sorry, I misunderstood this. I guess here the code below is valid if you simply replace TCPOLEN_MPTCP_MPC_SYNACK by TCPOLEN_MPTCP_MPC_ACK.

And I see it is what you propose below.

When we parse the SYN. we should parse the sender key (first line) and in the 3rd ACK both key (the if-statement).

Quoted 22 lines
net/mptcp/options.c +2 -10
@@ -57,16 +57,8 @@ void mptcp_parse_option(const unsigned char *ptr, int
 opsize,
                 mp_opt->sndr_key = get_unaligned_be64(ptr);
                 ptr += 8;
 
-               if (opsize == TCPOLEN_MPTCP_MPC_SYNACK) {
-                       mp_opt->rcvr_key = get_unaligned_be64(ptr);
-                       ptr += 8;
-                       pr_debug("MP_CAPABLE flags=%x, sndr=%llu,
 rcvr=%llu",
-                                mp_opt->flags, mp_opt->sndr_key,
-                                mp_opt->rcvr_key);
-               } else {
-                       pr_debug("MP_CAPABLE flags=%x, sndr=%llu",
-                                mp_opt->flags, mp_opt->sndr_key);
-               }
+               pr_debug("MP_CAPABLE flags=%x, sndr=%llu",
+                        mp_opt->flags, mp_opt->sndr_key);
                 break;
Quoted 6 lines
         /* MPTCPOPT_MP_JOIN

here the kernel stores in mp_opt->rcvr_key the value received on the "longer" mp_capable option (that is, the 3rd packet in the 3-way handshake). I don't really get what the server should do with the echoed key received by the client, and a grep over the code didn't help:

I think you need it to send a MPTCP Fast Close which is not implemented yet. I don't think we really need it somewhere else. Could it be used with ADD_ADDR in MPTCPv1? I would need to re-read this.

Quoted 1 line
$ grep -r rcvr_key net/* include/*

We can see who has a SSD with NVME and can use "grep" instead of "git grep" or others :-þ

Quoted 26 lines
net/mptcp/options.c:            opts->rcvr_key = subflow->remote_key;
net/mptcp/options.c:            opts->rcvr_key = subflow_req->remote_key;
net/mptcp/options.c:                    put_unaligned_be64(opts->rcvr_key, ptr);
include/linux/tcp.h:            u64     rcvr_key;
include/net/mptcp.h:    u64 rcvr_key;

I take as a friday activity (when packetdrill will be in a decent shape... so, not not this friday :) ) to understand what happens to a mptcp-net-next server when the client echoes back a wrong key. To be as much conservative as possible (and preserve the debug printout), we could just replace the above hunk with the following one:

--- >8 ---

net/mptcp/options.c +1 -2
@@ -57,7 +57,7 @@ void mptcp_parse_option(const unsigned char *ptr, int opsize,
                 mp_opt->sndr_key = get_unaligned_be64(ptr);
                 ptr += 8;
 
-               if (opsize == TCPOLEN_MPTCP_MPC_SYNACK) {
+               if (opsize == TCPOLEN_MPTCP_MPC_ACK) {
                         mp_opt->rcvr_key = get_unaligned_be64(ptr);
                         ptr += 8;
                         pr_debug("MP_CAPABLE flags=%x, sndr=%llu, rcvr=%llu",
--- >8 ---
Quoted 4 lines

that will also create less troubles to Christian when he rebases his v1 mp_capable patch on top of this.

WDYT?

Indeed, it seems to be the good fix to do: we should replace the two TCPOLEN_MPTCP_MPC_SYNACK by TCPOLEN_MPTCP_MPC_ACK in mptcp_parse_option().

Thank you for looking at that!

Cheers,
Matt

Signature · 5 lines
Matthieu Baerts | R&D Engineer
matthieu.baerts(a)tessares.net
Tessares SA | Hybrid Access Solutions
www.tessares.net
1 Avenue Jean Monnet, 1348 Louvain-la-Neuve, Belgium
author
#8 lore raw

PATCH 2/2 mptcp: process MP_CAPABLE data option.

In reply to Christoph Paasch

On the server side we can receive the remote key after that the connection is established. We need to explicitly track the 'missing remote key' status and avoid emtting mptcp ack till we get such info.

When a late/retransmitted/OoO pkt carrying MP_CAPABLE[+data] option is received, we have to propagate the mptcp seq number info to the msk socket. To avoid ABBA locking issue, explicitly check for that in recvmsg(), where we own msk and subflow sock locks.

The above also means that an established mp_capable subflow - still waiting for the remote key - can be 'downgraded' to plain TCP.

Such change could potentially block forever reader waiting for new data - as they hook to msk, while later wake-up after the downgrade will be on subflow only.

The above issue is not handled here, we likely have to get rid of msk->fallback to handle that cleanly.

Signed-off-by Christoph Paasch <cpaasch(a)apple.com>
---
 net/mptcp/options.c  | 60 +++++++++++++++++++++++++++++++++++---------
 net/mptcp/protocol.c | 22 ++++++++++------
 net/mptcp/protocol.h |  8 ++++--
 net/mptcp/subflow.c  | 38 ++++++++++++++++++++++++----
 4 files changed, 102 insertions(+), 26 deletions(-)
net/mptcp/options.c +48 -12
index 9cdbd617c49a..ea7e96c96342 100644
@@ -375,11 +375,13 @@ void mptcp_rcv_synsent(struct sock *sk)
 
 	if (subflow->request_mptcp && tp->rx_opt.mptcp.mp_capable) {
 		subflow->mp_capable = 1;
+		subflow->can_ack = 1;
 		subflow->remote_key = tp->rx_opt.mptcp.sndr_key;
 		pr_debug("subflow=%p, remote_key=%llu", subflow,
 			 subflow->remote_key);
 	} else if (subflow->request_join && tp->rx_opt.mptcp.mp_join) {
 		subflow->mp_join = 1;
+		subflow->can_ack = 1;
 		subflow->thmac = tp->rx_opt.mptcp.thmac;
 		subflow->remote_nonce = tp->rx_opt.mptcp.nonce;
 		pr_debug("subflow=%p, thmac=%llu, remote_nonce=%u", subflow,
@@ -458,8 +460,10 @@ static bool mptcp_established_options_dss(struct sock *sk, struct sk_buff *skb,
 					  unsigned int remaining,
 					  struct mptcp_out_options *opts)
 {
+	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
 	unsigned int dss_size = 0;
 	struct mptcp_ext *mpext;
+	struct mptcp_sock *msk;
 	unsigned int ack_size;
 
 	mpext = skb ? mptcp_get_ext(skb) : NULL;
@@ -480,6 +484,11 @@ static bool mptcp_established_options_dss(struct sock *sk, struct sk_buff *skb,
 		}
 	}
 
+	opts->ext_copy.use_ack = 0;
+	msk = mptcp_sk(subflow->conn);
+	if (!msk || !READ_ONCE(msk->can_ack))
+		goto no_ack;
+
 	ack_size = TCPOLEN_MPTCP_DSS_ACK64;
 
 	/* Add kind/length/subtype/flag overhead if mapping is not populated */
@@ -487,26 +496,16 @@ static bool mptcp_established_options_dss(struct sock *sk, struct sk_buff *skb,
 		ack_size += TCPOLEN_MPTCP_DSS_BASE;
 
 	if (ack_size <= remaining) {
-		struct mptcp_sock *msk;
-
 		dss_size += ack_size;
 
-		msk = mptcp_sk(mptcp_subflow_ctx(sk)->conn);
-		if (msk) {
-			opts->ext_copy.data_ack = msk->ack_seq;
-		} else {
-			mptcp_crypto_key_sha1(mptcp_subflow_ctx(sk)->remote_key,
-					      NULL, &opts->ext_copy.data_ack);
-			opts->ext_copy.data_ack++;
-		}
-
+		opts->ext_copy.data_ack = msk->ack_seq;
 		opts->ext_copy.ack64 = 1;
 		opts->ext_copy.use_ack = 1;
 	} else {
-		opts->ext_copy.use_ack = 0;
 		WARN(1, "MPTCP: Ack dropped");
 	}
 
+no_ack:
 	if (!dss_size)
 		return false;
 
@@ -650,6 +649,41 @@ static void update_una(struct mptcp_sock *msk,
 	}
 }
 
+static bool check_fourth_ack(struct mptcp_subflow_context *subflow,
+			     struct sk_buff *skb,
+			     struct mptcp_options_received *mp_opt)
+{
+	/* here we can process OoO, in-window pkts, only in-sequence 4th ack
+	 * are relevant
+	 */
+	if (likely(subflow->fourth_ack ||
+		   TCP_SKB_CB(skb)->seq != subflow->ssn_offset + 1))
+		return true;
+
+	/* mp join is easier: any tcp ack will do for it */
+	if (subflow->mp_join) {
+		subflow->fourth_ack = 1;
+		return true;
+	}
+
+	if (mp_opt->use_ack)
+		subflow->fourth_ack = 1;
+
+	if (subflow->can_ack)
+		return true;
+
+	/* if still waiting for remote/client key, we exepct an
+	 * MP_CAPABLE + data as first establish packet
+	 */
+	if (!mp_opt->mp_capable) {
+		subflow->mp_capable = 0;
+		return false;
+	}
+	subflow->remote_key = mp_opt->sndr_key;
+	subflow->can_ack = 1;
+	return true;
+}
+
 void mptcp_incoming_options(struct sock *sk, struct sk_buff *skb,
 			    struct tcp_options_received *opt_rx)
 {
@@ -662,6 +696,8 @@ void mptcp_incoming_options(struct sock *sk, struct sk_buff *skb,
 		return;
 
 	mp_opt = &opt_rx->mptcp;
+	if (!check_fourth_ack(subflow, skb, mp_opt))
+		return;
 
 	if (msk && mp_opt->add_addr) {
 		if (mp_opt->family == MPTCP_ADDR_IPVERSION_4)
net/mptcp/protocol.c +15 -7
index 9ad735a62330..8159d74947a9 100644
@@ -517,6 +517,12 @@ static int mptcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
 
 		lock_sock(ssk);
 		while (mptcp_subflow_data_available(ssk) && !done) {
+			if (unlikely(!msk->can_ack)) {
+				msk->remote_key = subflow->remote_key;
+				msk->ack_seq = subflow->map_seq;
+				msk->can_ack = true;
+			}
+
 			/* try to read as much data as available */
 			map_remaining = subflow->map_data_len -
 					mptcp_subflow_get_map_offset(subflow);
@@ -823,7 +829,6 @@ static struct sock *mptcp_accept(struct sock *sk, int flags, int *err,
 		__mptcp_init_sock(new_mptcp_sock);
 
 		msk = mptcp_sk(new_mptcp_sock);
-		msk->remote_key = subflow->remote_key;
 		msk->local_key = subflow->local_key;
 		msk->token = subflow->token;
 
@@ -832,11 +837,15 @@ static struct sock *mptcp_accept(struct sock *sk, int flags, int *err,
 
 		mptcp_pm_new_connection(msk, 1);
 
-		mptcp_crypto_key_sha1(msk->remote_key, NULL, &ack_seq);
 		msk->write_seq = subflow->idsn + 1;
 		atomic64_set(&msk->snd_una, msk->write_seq);
-		ack_seq++;
-		msk->ack_seq = ack_seq;
+		if (subflow->can_ack) {
+			msk->can_ack = true;
+			msk->remote_key = subflow->remote_key;
+			mptcp_crypto_key_sha1(msk->remote_key, NULL, &ack_seq);
+			ack_seq++;
+			msk->ack_seq = ack_seq;
+		}
 		newsk = new_mptcp_sock;
 		list_add(&subflow->node, &msk->conn_list);
 		bh_unlock_sock(new_mptcp_sock);
@@ -850,8 +859,6 @@ static struct sock *mptcp_accept(struct sock *sk, int flags, int *err,
 		 * the receive path and process the pending ones
 		 */
 		lock_sock(new_sock->sk);
-		subflow->map_seq = ack_seq;
-		subflow->map_subflow_seq = 1;
 		subflow->rel_write_seq = 1;
 		subflow->tcp_sock = new_sock;
 		subflow->conn = new_mptcp_sock;
@@ -1005,11 +1012,12 @@ void mptcp_finish_connect(struct sock *sk, int mp_capable)
 
 		mptcp_pm_new_connection(msk, 0);
 
-		mptcp_crypto_key_sha1(msk->remote_key, NULL, &ack_seq);
 		msk->write_seq = subflow->idsn + 1;
 		atomic64_set(&msk->snd_una, msk->write_seq);
+		mptcp_crypto_key_sha1(msk->remote_key, NULL, &ack_seq);
 		ack_seq++;
 		msk->ack_seq = ack_seq;
+		msk->can_ack = 1;
 		subflow->map_seq = ack_seq;
 		subflow->map_subflow_seq = 1;
 		subflow->rel_write_seq = 1;
net/mptcp/protocol.h +6 -2
index 5c96777f9b3c..896e6acd0370 100644
@@ -134,6 +134,7 @@ struct mptcp_sock {
 	u32		token;
 	unsigned long	flags;
 	u16		dport;
+	bool		can_ack;
 	struct work_struct rtx_work;
 	struct list_head conn_list;
 	struct list_head rtx_queue;
@@ -172,9 +173,10 @@ static inline struct mptcp_data_frag *mptcp_rtx_head(const struct sock *sk)
 
 struct mptcp_subflow_request_sock {
 	struct	tcp_request_sock sk;
-	u8	mp_capable : 1,
+	u16	mp_capable : 1,
 		mp_join : 1,
 		backup : 1,
+		remote_key_valid : 1,
 		version : 4;
 	u8	local_id;
 	u8	remote_id;
@@ -216,9 +218,11 @@ struct mptcp_subflow_context {
 		fourth_ack : 1,	    /* send initial DSS */
 		conn_finished : 1,
 		map_valid : 1,
+		mp_capable_map : 1,
 		backup : 1,
 		data_avail : 1,
-		rx_eof : 1;
+		rx_eof : 1,
+		can_ack : 1;	    /* only after processing the remote a key */
 	u32	remote_nonce;
 	u64	thmac;
 	u32	local_nonce;
net/mptcp/subflow.c +33 -5
index dce5bf137eea..e9a499508363 100644
@@ -108,6 +108,7 @@ static void subflow_v4_init_req(struct request_sock *req,
 
 	subflow_req->mp_capable = 0;
 	subflow_req->mp_join = 0;
+	subflow_req->remote_key_valid = 0;
 
 	if (rx_opt.mptcp.mp_capable) {
 		SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MPCAPABLEPASSIVE);
@@ -265,15 +266,25 @@ static struct sock *subflow_syn_recv_sock(const struct sock *sk,
 
 	pr_debug("listener=%p, req=%p, conn=%p", listener, req, listener->conn);
 
-	/* if the sk is MP_CAPABLE, we need to fetch the client key */
+	/* if the sk is MP_CAPABLE, we try to fetch the client key */
 	subflow_req = mptcp_subflow_rsk(req);
 	if (subflow_req->mp_capable) {
+		if (TCP_SKB_CB(skb)->seq != subflow_req->ssn_offset + 1) {
+			/* here we can receive and accept an in-window,
+			 * out-of-order pkt, which will not carry the MP_CAPABLE
+			 * opt even on mptcp enabled paths
+			 */
+			goto create_child;
+		}
+
 		opt_rx.mptcp.mp_capable = 0;
 		mptcp_get_options(skb, &opt_rx);
-		if (!opt_rx.mptcp.mp_capable)
-			subflow_req->mp_capable = 0;
-		else
+		if (opt_rx.mptcp.mp_capable) {
 			subflow_req->remote_key = opt_rx.mptcp.sndr_key;
+			subflow_req->remote_key_valid = 1;
+		} else {
+			subflow_req->mp_capable = 0;
+		}
 	} else if (subflow_req->mp_join) {
 		opt_rx.mptcp.mp_join = 0;
 		mptcp_get_options(skb, &opt_rx);
@@ -282,6 +293,7 @@ static struct sock *subflow_syn_recv_sock(const struct sock *sk,
 			return NULL;
 	}
 
+create_child:
 	child = tcp_v4_syn_recv_sock(sk, skb, req, dst, req_unhash, own_req);
 
 	if (child && *own_req) {
@@ -457,6 +469,7 @@ static enum mapping_status get_mapping_status(struct sock *ssk)
 	subflow->map_subflow_seq = mpext->subflow_seq;
 	subflow->map_data_len = mpext->data_len;
 	subflow->map_valid = 1;
+	subflow->mp_capable_map = mpext->mp_capable_map;
 	pr_debug("new map seq=%llu subflow_seq=%u data_len=%u",
 		 subflow->map_seq, subflow->map_subflow_seq,
 		 subflow->map_data_len);
@@ -508,6 +521,16 @@ static bool subflow_check_data_avail(struct sock *ssk)
 		if (WARN_ON_ONCE(!skb))
 			return false;
 
+		/* if msk lacks the remote key, this subflow must provide an
+		 * MP_CAPABLE-based mapping
+		 */
+		if (unlikely(!READ_ONCE(msk->ack_seq))) {
+			if (subflow->mp_capable_map)
+				break;
+			ssk->sk_err = EBADMSG;
+			goto fatal;
+		}
+
 		old_ack = READ_ONCE(msk->ack_seq);
 		ack_seq = mptcp_subflow_get_mapped_dsn(subflow);
 		pr_debug("msk ack_seq=%llx subflow ack_seq=%llx", old_ack,
@@ -815,14 +838,19 @@ static void subflow_ulp_clone(const struct request_sock *req,
 	new_ctx->tcp_sk_data_ready = old_ctx->tcp_sk_data_ready;
 
 	if (subflow_req->mp_capable) {
+		/* see comments in subflow_syn_recv_sock(), MPTCP connection
+		 * is fully establishged only after we receive the remote key
+		 */
 		new_ctx->mp_capable = 1;
-		new_ctx->fourth_ack = 1;
+		new_ctx->fourth_ack = subflow_req->remote_key_valid;
+		new_ctx->can_ack = subflow_req->remote_key_valid;
 		new_ctx->remote_key = subflow_req->remote_key;
 		new_ctx->local_key = subflow_req->local_key;
 		new_ctx->token = subflow_req->token;
 		new_ctx->ssn_offset = subflow_req->ssn_offset;
 		new_ctx->idsn = subflow_req->idsn;
 	} else if (subflow_req->mp_join) {
+		new_ctx->can_ack = 1;
 		new_ctx->mp_join = 1;
 		new_ctx->fourth_ack = 1;
 		new_ctx->backup = subflow_req->backup;
Signature · 1 line
2.23.0