Skip to contentSkip to conversation

Forward-ports from mptcp_trunk to mptcp-net-next

PATCH In review Christoph Paasch 2 participants 20 messages May 3, 2018 – Jun 8, 2018

Subsystems (4)
lore
Share & tools
lore

Christoph Paasch

author
#0 lore raw

cover Forward-ports from mptcp_trunk to mptcp-net-next

Here are some forward-ports from mptcp_trunk to mptcp-net-next of the lockless subflows patches and the two bug-fixes.

Christoph Paasch (14):
mptcp: Cleanup
mptcp: Improved debugging
mptcp: Use correct tcp_queue when calling mptcp_fragment
mptcp: Use mptcp_can_new_subflow where possible
mptcp: Render mptcp_sub_inherit_sockopts lockless
mptcp: Make mptcp_set_new_pathindex lockless
mptcp: Remove cnt_subflows
mptcp: Remove cnt_established
mptcp: Rename tw_lock to mpcb_list_lock
mptcp: Make subflow-list an RCU-list
mptcp: Don't take meta-lock when receiving third ACK
mptcp: Make mptcp_do_join_short and mptcp_lookup_join lockless
mptcp: Reset tsorted_anchor after call to tcp_fragment
mptcp: Pick correct sk when iterating in mptcp_set_rto

 include/net/mptcp.h          | 127 ++++++++++++-----------------
 net/ipv4/af_inet.c           |   9 ++-
 net/ipv4/ip_sockglue.c       |   7 +-
 net/ipv4/tcp.c               |  31 +++++---
 net/ipv4/tcp_ipv4.c          |  41 +---------
 net/ipv4/tcp_minisocks.c     |   9 ++-
 net/ipv6/tcp_ipv6.c          |  41 +---------
 net/mptcp/mptcp_ctrl.c       | 186 ++++++++++++++++++++++++++++---------------
 net/mptcp/mptcp_fullmesh.c   |  30 ++++---
 net/mptcp/mptcp_input.c      | 150 ++++++++++++----------------------
 net/mptcp/mptcp_ipv4.c       |  34 +++-----
 net/mptcp/mptcp_ipv6.c       |  15 ++--
 net/mptcp/mptcp_ndiffports.c |   2 +-
 net/mptcp/mptcp_output.c     |  61 ++++++++++----
 net/mptcp/mptcp_sched.c      |  32 ++++----
 15 files changed, 371 insertions(+), 404 deletions(-)
Signature · 1 line
2.16.2
author
#1 lore raw

PATCH 1/14 mptcp: Cleanup

Just some basic cleanup, fixing some comments, renaming sk to meta_sk, and moving mptcp_finish_handshake out of mptcp_ipv4.c to mptcp_ctrl.c

Signed-off-by Christoph Paasch <cpaasch(a)apple.com>
Signed-off-by Matthieu Baerts <matthieu.baerts(a)tessares.net>
(cherry picked from commit 1b95146829844c1c98471194cafd5d4d99fc682a)
Signed-off-by Christoph Paasch <cpaasch(a)apple.com>
---
 include/net/mptcp.h        |  8 ++++----
 net/mptcp/mptcp_ctrl.c     | 25 +++++++++++++++++++++----
 net/mptcp/mptcp_fullmesh.c |  2 +-
 net/mptcp/mptcp_input.c    |  7 ++-----
 net/mptcp/mptcp_ipv4.c     | 26 ++++----------------------
 net/mptcp/mptcp_ipv6.c     |  8 ++++----
 6 files changed, 36 insertions(+), 40 deletions(-)
include/net/mptcp.h +4 -4
index 8abb6985d852..f6ca550d9e6c 100644
@@ -660,10 +660,10 @@ extern int sysctl_mptcp_syn_retries;
 
 extern struct workqueue_struct *mptcp_wq;
 
-#define mptcp_debug(fmt, args...)					\
-	do {								\
-		if (unlikely(sysctl_mptcp_debug))			\
-			pr_err(fmt, ##args);	\
+#define mptcp_debug(fmt, args...)						\
+	do {									\
+		if (unlikely(sysctl_mptcp_debug))				\
+			pr_err(fmt, ##args);					\
 	} while (0)
 
 /* Iterates over all subflows */
net/mptcp/mptcp_ctrl.c +21 -4
index 5b8320687122..001ba216db7f 100644
@@ -1315,8 +1315,8 @@ void mptcp_fallback_meta_sk(struct sock *meta_sk)
 int mptcp_add_sock(struct sock *meta_sk, struct sock *sk, u8 loc_id, u8 rem_id,
 		   gfp_t flags)
 {
-	struct mptcp_cb *mpcb	= tcp_sk(meta_sk)->mpcb;
-	struct tcp_sock *tp	= tcp_sk(sk);
+	struct mptcp_cb *mpcb = tcp_sk(meta_sk)->mpcb;
+	struct tcp_sock *tp = tcp_sk(sk);
 
 	tp->mptcp = kmem_cache_zalloc(mptcp_sock_cache, flags);
 	if (!tp->mptcp)
@@ -2119,9 +2119,9 @@ struct sock *mptcp_check_req_child(struct sock *meta_sk,
 				   struct sk_buff *skb,
 				   const struct mptcp_options_received *mopt)
 {
-	struct tcp_sock *child_tp = tcp_sk(child);
+	const struct mptcp_cb *mpcb = tcp_sk(meta_sk)->mpcb;
 	struct mptcp_request_sock *mtreq = mptcp_rsk(req);
-	struct mptcp_cb *mpcb = tcp_sk(meta_sk)->mpcb;
+	struct tcp_sock *child_tp = tcp_sk(child);
 	u8 hash_mac_check[20];
 
 	child_tp->inside_tk_table = 0;
@@ -2487,6 +2487,23 @@ int mptcp_conn_request(struct sock *sk, struct sk_buff *skb)
 	return 0;
 }
 
+int mptcp_finish_handshake(struct sock *child, struct sk_buff *skb)
+	__releases(&child->sk_lock.slock)
+{
+	int ret;
+
+	/* We don't call tcp_child_process here, because we hold
+	 * already the meta-sk-lock and are sure that it is not owned
+	 * by the user.
+	 */
+	tcp_sk(child)->segs_in += max_t(u16, 1, skb_shinfo(skb)->gso_segs);
+	ret = tcp_rcv_state_process(child, skb);
+	bh_unlock_sock(child);
+	sock_put(child);
+
+	return ret;
+}
+
 static void __mptcp_get_info(const struct sock *meta_sk,
 			     struct mptcp_meta_info *info)
 {
net/mptcp/mptcp_fullmesh.c +1 -1
index 9690478fae70..d29102a1dffb 100644
@@ -79,7 +79,7 @@ struct fullmesh_priv {
 	u8 rem4_bits;
 	u8 rem6_bits;
 
-	/* Are we established the additional subflows for primary pair? */
+	/* Have we established the additional subflows for primary pair? */
 	u8 first_pair:1;
 };
 
net/mptcp/mptcp_input.c +2 -5
index 12d7337b9b18..f3a4dbe45bc6 100644
@@ -89,10 +89,7 @@ static inline int mptcp_tso_acked_reinject(const struct sock *meta_sk,
 	return packets_acked;
 }
 
-/**
- * Cleans the meta-socket retransmission queue and the reinject-queue.
- * @sk must be the metasocket.
- */
+/* Cleans the meta-socket retransmission queue and the reinject-queue. */
 static void mptcp_clean_rtx_queue(struct sock *meta_sk, u32 prior_snd_una)
 {
 	struct sk_buff *skb, *tmp, *next;
@@ -350,7 +347,7 @@ static int mptcp_verif_dss_csum(struct sock *sk)
 
 	/* Now, checksum must be 0 */
 	if (unlikely(csum_fold(csum_tcp))) {
-		pr_err("%s csum is wrong: %#x data_seq %u dss_csum_added %d overflowed %d iterations %d\n",
+		pr_err("%s csum is wrong: %#x tcp-seq %u dss_csum_added %d overflowed %d iterations %d\n",
 		       __func__, csum_fold(csum_tcp), TCP_SKB_CB(last)->seq,
 		       dss_csum_added, overflowed, iter);
 
net/mptcp/mptcp_ipv4.c +4 -22
index 061377f79a18..f059117139be 100644
@@ -96,11 +96,11 @@ static u32 mptcp_v4_cookie_init_seq(struct request_sock *req, const struct sock
 }
 #endif
 
-static int mptcp_v4_join_init_req(struct request_sock *req, const struct sock *sk,
+static int mptcp_v4_join_init_req(struct request_sock *req, const struct sock *meta_sk,
 				  struct sk_buff *skb, bool want_cookie)
 {
 	struct mptcp_request_sock *mtreq = mptcp_rsk(req);
-	const struct mptcp_cb *mpcb = tcp_sk(sk)->mpcb;
+	const struct mptcp_cb *mpcb = tcp_sk(meta_sk)->mpcb;
 	union inet_addr addr;
 	int loc_id;
 	bool low_prio = false;
@@ -112,14 +112,14 @@ static int mptcp_v4_join_init_req(struct request_sock *req, const struct sock *s
 	 */
 	mtreq->hash_entry.pprev = NULL;
 
-	tcp_request_sock_ipv4_ops.init_req(req, sk, skb, want_cookie);
+	tcp_request_sock_ipv4_ops.init_req(req, meta_sk, skb, want_cookie);
 
 	mtreq->mptcp_loc_nonce = mptcp_v4_get_nonce(ip_hdr(skb)->saddr,
 						    ip_hdr(skb)->daddr,
 						    tcp_hdr(skb)->source,
 						    tcp_hdr(skb)->dest);
 	addr.ip = inet_rsk(req)->ir_loc_addr;
-	loc_id = mpcb->pm_ops->get_local_id(AF_INET, &addr, sock_net(sk), &low_prio);
+	loc_id = mpcb->pm_ops->get_local_id(AF_INET, &addr, sock_net(meta_sk), &low_prio);
 	if (loc_id == -1)
 		return -1;
 	mtreq->loc_id = loc_id;
@@ -149,24 +149,6 @@ static int mptcp_v4_join_request(struct sock *meta_sk, struct sk_buff *skb)
 				meta_sk, skb);
 }
 
-int mptcp_finish_handshake(struct sock *child, struct sk_buff *skb)
-	__releases(&child->sk_lock.slock)
-{
-	int ret;
-
-	/* We don't call tcp_child_process here, because we hold
-	 * already the meta-sk-lock and are sure that it is not owned
-	 * by the user.
-	 */
-	tcp_sk(child)->segs_in += max_t(u16, 1, skb_shinfo(skb)->gso_segs);
-	ret = tcp_rcv_state_process(child, skb);
-	bh_unlock_sock(child);
-	sock_put(child);
-
-	return ret;
-}
-
-
 /* Similar to: tcp_v4_do_rcv
  * We only process join requests here. (either the SYN or the final ACK)
  */
net/mptcp/mptcp_ipv6.c +4 -4
index c968de95709e..a367f103e81a 100644
@@ -125,11 +125,11 @@ static u32 mptcp_v6_cookie_init_seq(struct request_sock *req, const struct sock
 }
 #endif
 
-static int mptcp_v6_join_init_req(struct request_sock *req, const struct sock *sk,
+static int mptcp_v6_join_init_req(struct request_sock *req, const struct sock *meta_sk,
 				  struct sk_buff *skb, bool want_cookie)
 {
 	struct mptcp_request_sock *mtreq = mptcp_rsk(req);
-	const struct mptcp_cb *mpcb = tcp_sk(sk)->mpcb;
+	const struct mptcp_cb *mpcb = tcp_sk(meta_sk)->mpcb;
 	union inet_addr addr;
 	int loc_id;
 	bool low_prio = false;
@@ -141,14 +141,14 @@ static int mptcp_v6_join_init_req(struct request_sock *req, const struct sock *s
 	 */
 	mtreq->hash_entry.pprev = NULL;
 
-	tcp_request_sock_ipv6_ops.init_req(req, sk, skb, want_cookie);
+	tcp_request_sock_ipv6_ops.init_req(req, meta_sk, skb, want_cookie);
 
 	mtreq->mptcp_loc_nonce = mptcp_v6_get_nonce(ipv6_hdr(skb)->saddr.s6_addr32,
 						    ipv6_hdr(skb)->daddr.s6_addr32,
 						    tcp_hdr(skb)->source,
 						    tcp_hdr(skb)->dest);
 	addr.in6 = inet_rsk(req)->ir_v6_loc_addr;
-	loc_id = mpcb->pm_ops->get_local_id(AF_INET6, &addr, sock_net(sk), &low_prio);
+	loc_id = mpcb->pm_ops->get_local_id(AF_INET6, &addr, sock_net(meta_sk), &low_prio);
 	if (loc_id == -1)
 		return -1;
 	mtreq->loc_id = loc_id;
Signature · 1 line
2.16.2
author
#2 lore raw

PATCH 2/14 mptcp: Improved debugging

In reply to Christoph Paasch

Allow to print the value of ret when mptcp_add_sock fails.

Signed-off-by Christoph Paasch <cpaasch(a)apple.com>
Signed-off-by Matthieu Baerts <matthieu.baerts(a)tessares.net>
(cherry picked from commit d5e0d6ab54f1cf78f53a95d6b78514b5cf75ac9e)
Signed-off-by Christoph Paasch <cpaasch(a)apple.com>
---
 net/mptcp/mptcp_ipv4.c | 3 ++-
 net/mptcp/mptcp_ipv6.c | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)
net/mptcp/mptcp_ipv4.c +2 -1
index f059117139be..6fce833c6859 100644
@@ -281,7 +281,8 @@ int mptcp_init4_subsockets(struct sock *meta_sk, const struct mptcp_loc4 *loc,
 	lockdep_set_class_and_name(&(sk)->sk_lock.slock, &meta_slock_key, meta_slock_key_name);
 	lockdep_init_map(&(sk)->sk_lock.dep_map, meta_key_name, &meta_key, 0);
 
-	if (mptcp_add_sock(meta_sk, sk, loc->loc4_id, rem->rem4_id, GFP_KERNEL)) {
+	ret = mptcp_add_sock(meta_sk, sk, loc->loc4_id, rem->rem4_id, GFP_KERNEL);
+	if (ret) {
 		net_err_ratelimited("%s mptcp_add_sock failed ret: %d\n",
 				    __func__, ret);
 		goto error;
net/mptcp/mptcp_ipv6.c +2 -1
index a367f103e81a..bd6b16becc49 100644
@@ -307,7 +307,8 @@ int mptcp_init6_subsockets(struct sock *meta_sk, const struct mptcp_loc6 *loc,
 	lockdep_set_class_and_name(&(sk)->sk_lock.slock, &meta_slock_key, meta_slock_key_name);
 	lockdep_init_map(&(sk)->sk_lock.dep_map, meta_key_name, &meta_key, 0);
 
-	if (mptcp_add_sock(meta_sk, sk, loc->loc6_id, rem->rem6_id, GFP_KERNEL)) {
+	ret = mptcp_add_sock(meta_sk, sk, loc->loc6_id, rem->rem6_id, GFP_KERNEL);
+	if (ret) {
 		net_err_ratelimited("%s mptcp_add_sock failed ret: %d\n",
 				    __func__, ret);
 		goto error;
Signature · 1 line
2.16.2
author
#3 lore raw

PATCH 3/14 mptcp: Use correct tcp_queue when calling mptcp_fragment

In reply to Christoph Paasch

The skb may come from the write-queue, the rtx-queue or the reinject queue.

When it comes from the rtx-queue, we need to give mptcp_fragment the right tcp_queue.

To handle the case when it comes from the reinject-queue, I will post a follow-up patch as this seems to be a bug in the stable release as well.

Fixes 8a1051d977cf ("Merge tag 'v4.15' into mptcp_trunk")
Signed-off-by Christoph Paasch <cpaasch(a)apple.com>
Signed-off-by Matthieu Baerts <matthieu.baerts(a)tessares.net>
(cherry picked from commit edf4b8133e304238b1812c147455f65b2c15c436)
Signed-off-by Christoph Paasch <cpaasch(a)apple.com>
---
 net/mptcp/mptcp_output.c | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)
net/mptcp/mptcp_output.c +14 -3
index 3dd4c17c7681..3b99cef02c98 100644
@@ -575,6 +575,11 @@ static bool mptcp_skb_entail(struct sock *sk, struct sk_buff *skb, int reinject)
 
 /* Fragment an skb and update the mptcp meta-data. Due to reinject, we
  * might need to undo some operations done by tcp_fragment.
+ *
+ * Be careful, the skb may come from 3 different places:
+ * - The send-queue (tcp_queue == TCP_FRAG_IN_WRITE_QUEUE)
+ * - The retransmit-queue (tcp_queue == TCP_FRAG_IN_RTX_QUEUE)
+ * - The reinject-queue (reinject == -1)
  */
 static int mptcp_fragment(struct sock *meta_sk, enum tcp_queue tcp_queue,
 			  struct sk_buff *skb, u32 len,
@@ -599,7 +604,10 @@ static int mptcp_fragment(struct sock *meta_sk, enum tcp_queue tcp_queue,
 	if (ret)
 		return ret;
 
-	buff = skb->next;
+	if (tcp_queue == TCP_FRAG_IN_WRITE_QUEUE)
+		buff = skb->next;
+	else
+		buff = skb_rb_next(skb);
 
 	flags = TCP_SKB_CB(skb)->mptcp_flags;
 	TCP_SKB_CB(skb)->mptcp_flags = flags & ~(MPTCPHDR_FIN);
@@ -732,6 +740,7 @@ bool mptcp_write_xmit(struct sock *meta_sk, unsigned int mss_now, int nonagle,
 
 	while ((skb = mpcb->sched_ops->next_segment(meta_sk, &reinject, &subsk,
 						    &sublimit))) {
+		enum tcp_queue tcp_queue = TCP_FRAG_IN_WRITE_QUEUE;
 		unsigned int limit;
 
 		WARN(TCP_SKB_CB(skb)->sacked, "sacked: %u reinject: %u",
@@ -747,6 +756,8 @@ bool mptcp_write_xmit(struct sock *meta_sk, unsigned int mss_now, int nonagle,
 				__kfree_skb(skb);
 				continue;
 			}
+		} else if (reinject == -1) {
+			tcp_queue = TCP_FRAG_IN_RTX_QUEUE;
 		}
 
 		/* If the segment was cloned (e.g. a meta retransmission),
@@ -809,7 +820,7 @@ bool mptcp_write_xmit(struct sock *meta_sk, unsigned int mss_now, int nonagle,
 			limit = min(limit, sublimit);
 
 		if (skb->len > limit &&
-		    unlikely(mptcp_fragment(meta_sk, TCP_FRAG_IN_WRITE_QUEUE,
+		    unlikely(mptcp_fragment(meta_sk, tcp_queue,
 					    skb, limit, gfp, reinject)))
 			break;
 
@@ -1537,7 +1548,7 @@ int mptcp_retransmit_skb(struct sock *meta_sk, struct sk_buff *skb)
 					    TCP_NAGLE_OFF);
 
 	if (skb->len > limit &&
-	    unlikely(mptcp_fragment(meta_sk, TCP_FRAG_IN_WRITE_QUEUE, skb,
+	    unlikely(mptcp_fragment(meta_sk, TCP_FRAG_IN_RTX_QUEUE, skb,
 				    limit, GFP_ATOMIC, 0)))
 		goto failed;
 
Signature · 1 line
2.16.2
author
#4 lore raw

PATCH 4/14 mptcp: Use mptcp_can_new_subflow where possible

In reply to Christoph Paasch

Let's reuse the function instead of implementing the same checks again.

Signed-off-by Christoph Paasch <cpaasch(a)apple.com>
Signed-off-by Matthieu Baerts <matthieu.baerts(a)tessares.net>
(cherry picked from commit 068ba20e4137fc430004d740efda4978f8e26150)
Signed-off-by Christoph Paasch <cpaasch(a)apple.com>
---
 net/mptcp/mptcp_input.c | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)
net/mptcp/mptcp_input.c +2 -4
index f3a4dbe45bc6..ba9e3fe62824 100644
@@ -1171,7 +1171,7 @@ int mptcp_lookup_join(struct sk_buff *skb, struct inet_timewait_sock *tw)
 	}
 
 	mpcb = tcp_sk(meta_sk)->mpcb;
-	if (mpcb->infinite_mapping_rcv || mpcb->send_infinite_mapping) {
+	if (!mptcp_can_new_subflow(meta_sk)) {
 		/* We are in fallback-mode on the reception-side -
 		 * no new subflows!
 		 */
@@ -1255,9 +1255,7 @@ int mptcp_do_join_short(struct sk_buff *skb,
 	 * And the send-reset will try to take yet another one (ip_send_reply).
 	 * Thus, we propagate the reset up to tcp_rcv_state_process.
 	 */
-	if (tcp_sk(meta_sk)->mpcb->infinite_mapping_rcv ||
-	    tcp_sk(meta_sk)->mpcb->send_infinite_mapping ||
-	    meta_sk->sk_state == TCP_CLOSE || !tcp_sk(meta_sk)->inside_tk_table) {
+	if (!mptcp_can_new_subflow(meta_sk)) {
 		MPTCP_INC_STATS(sock_net(meta_sk), MPTCP_MIB_JOINFALLBACK);
 		bh_unlock_sock(meta_sk);
 		sock_put(meta_sk); /* Taken by mptcp_hash_find */
Signature · 1 line
2.16.2
author
#5 lore raw

PATCH 5/14 mptcp: Render mptcp_sub_inherit_sockopts lockless

In reply to Christoph Paasch

We are accessing the meta-socket's TOS field twice. It could change in-between as we do this in a lockless manner.

Thus, let's make sure we read it only once.

Signed-off-by Christoph Paasch <cpaasch(a)apple.com>
Signed-off-by Matthieu Baerts <matthieu.baerts(a)tessares.net>
(cherry picked from commit 343a132cbc92989d311c09ae5976a354eec3c9fd)
Signed-off-by Christoph Paasch <cpaasch(a)apple.com>
---
 net/mptcp/mptcp_ctrl.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)
net/mptcp/mptcp_ctrl.c +6 -2
index 001ba216db7f..1e67d5d6c87d 100644
@@ -945,11 +945,15 @@ static void mptcp_mpcb_inherit_sockopts(struct sock *meta_sk, struct sock *maste
 	inet_sk(master_sk)->recverr = 0;
 }
 
+/* Called without holding lock on meta_sk */
 static void mptcp_sub_inherit_sockopts(const struct sock *meta_sk, struct sock *sub_sk)
 {
+	__u8 meta_tos;
+
 	/* IP_TOS also goes to the subflow. */
-	if (inet_sk(sub_sk)->tos != inet_sk(meta_sk)->tos) {
-		inet_sk(sub_sk)->tos = inet_sk(meta_sk)->tos;
+	meta_tos = READ_ONCE(inet_sk(meta_sk)->tos);
+	if (inet_sk(sub_sk)->tos != meta_tos) {
+		inet_sk(sub_sk)->tos = meta_tos;
 		sub_sk->sk_priority = meta_sk->sk_priority;
 		sk_dst_reset(sub_sk);
 	}
Signature · 1 line
2.16.2
author
#6 lore raw

PATCH 6/14 mptcp: Make mptcp_set_new_pathindex lockless

In reply to Christoph Paasch

There is quite a complex logic just for setting the next path-index. It also prevents us from making the operation lockless.

Let's simplify this and make it lockless through a simple test_and_set_bit.

Signed-off-by Christoph Paasch <cpaasch(a)apple.com>
Signed-off-by Matthieu Baerts <matthieu.baerts(a)tessares.net>
(cherry picked from commit e426daa6e7bf3797aee5993258cecc5bc8a89c31)
Signed-off-by Christoph Paasch <cpaasch(a)apple.com>
---
 include/net/mptcp.h    | 34 +---------------------------------
 net/mptcp/mptcp_ctrl.c | 16 ++++++++++++++++
 2 files changed, 17 insertions(+), 33 deletions(-)
include/net/mptcp.h +1 -33
index f6ca550d9e6c..43f7f2ae1bad 100644
@@ -326,9 +326,7 @@ struct mptcp_cb {
 	u8 mptcp_pm[MPTCP_PM_SIZE] __aligned(8);
 	struct mptcp_pm_ops *pm_ops;
 
-	u32 path_index_bits;
-	/* Next pi to pick up in case a new path becomes available */
-	u8 next_path_index;
+	unsigned long path_index_bits;
 
 	__u8	mptcp_ver;
 
@@ -1247,36 +1245,6 @@ static inline bool mptcp_fallback_infinite(struct sock *sk, int flag)
 	return false;
 }
 
-/* Find the first index whose bit in the bit-field == 0 */
-static inline u8 mptcp_set_new_pathindex(struct mptcp_cb *mpcb)
-{
-	u8 base = mpcb->next_path_index;
-	int i;
-
-	/* Start at 1, because 0 is reserved for the meta-sk */
-	mptcp_for_each_bit_unset(mpcb->path_index_bits >> base, i) {
-		if (i + base < 1)
-			continue;
-		if (i + base >= sizeof(mpcb->path_index_bits) * 8)
-			break;
-		i += base;
-		mpcb->path_index_bits |= (1 << i);
-		mpcb->next_path_index = i + 1;
-		return i;
-	}
-	mptcp_for_each_bit_unset(mpcb->path_index_bits, i) {
-		if (i >= sizeof(mpcb->path_index_bits) * 8)
-			break;
-		if (i < 1)
-			continue;
-		mpcb->path_index_bits |= (1 << i);
-		mpcb->next_path_index = i + 1;
-		return i;
-	}
-
-	return 0;
-}
-
 static inline bool mptcp_v6_is_v4_mapped(const struct sock *sk)
 {
 	return sk->sk_family == AF_INET6 &&
net/mptcp/mptcp_ctrl.c +16 -0
index 1e67d5d6c87d..c2aae9020d91 100644
@@ -1316,6 +1316,22 @@ void mptcp_fallback_meta_sk(struct sock *meta_sk)
 	kmem_cache_free(mptcp_cb_cache, tcp_sk(meta_sk)->mpcb);
 }
 
+/*  Called without holding lock on mpcb */
+static u8 mptcp_set_new_pathindex(struct mptcp_cb *mpcb)
+{
+	int i;
+
+	/* Start at 1, because 0 is reserved for the meta-sk */
+	for (i = 1; i < sizeof(mpcb->path_index_bits) * 8; i++) {
+		if (!test_and_set_bit(i, &mpcb->path_index_bits))
+			break;
+	}
+
+	if (i == sizeof(mpcb->path_index_bits) * 8)
+		return 0;
+	return i;
+}
+
 int mptcp_add_sock(struct sock *meta_sk, struct sock *sk, u8 loc_id, u8 rem_id,
 		   gfp_t flags)
 {
Signature · 1 line
2.16.2
author
#7 lore raw

PATCH 7/14 mptcp: Remove cnt_subflows

In reply to Christoph Paasch

Our goal is to make subflow establishment lockless. cnt_subflows would need to either become an atomic variable, or we can remove it.

Well, let's remove it! In the general case it is not needed at all. Some exotic schedulers need cnt_subflows, but we don't care if we add complexity in these schedulers.

Something special in mptcp_verif_dss_csum(). There the access was actually wrong. Simply because cnt_subflows > 1, it doesn't mean we have a working subflow. So, we fix that. I don't think this issue is big enough to backport it.

Finally mptcp_sched.c was also using cnt_subflows but merely for speeding up things to avoid going through the whole scheduler code when there is only one subflow. So, we can remove it and trade performance for a better design.

Signed-off-by Christoph Paasch <cpaasch(a)apple.com>
Signed-off-by Matthieu Baerts <matthieu.baerts(a)tessares.net>
(cherry picked from commit b079a187260f0591ee292655f00526a5c8371cfe)
---
 include/net/mptcp.h          | 12 +++++++++++-
 net/mptcp/mptcp_ctrl.c       | 15 ++++++---------
 net/mptcp/mptcp_input.c      | 11 ++++++++++-
 net/mptcp/mptcp_ndiffports.c |  2 +-
 net/mptcp/mptcp_output.c     |  2 +-
 net/mptcp/mptcp_sched.c      | 11 -----------
 6 files changed, 29 insertions(+), 24 deletions(-)
include/net/mptcp.h +11 -1
index 43f7f2ae1bad..b1ea946ea898 100644
@@ -277,7 +277,6 @@ struct mptcp_cb {
 		rcv_hiseq_index:1; /* Index in rcv_high_order of rcv_nxt */
 
 	/* socket count in this connection */
-	u8 cnt_subflows;
 	u8 cnt_established;
 
 #define MPTCP_SCHED_DATA_SIZE 8
@@ -1266,6 +1265,17 @@ static inline bool mptcp_can_new_subflow(const struct sock *meta_sk)
 	       !tcp_sk(meta_sk)->mpcb->send_infinite_mapping;
 }
 
+static inline int mptcp_subflow_count(const struct mptcp_cb *mpcb)
+{
+	struct sock *sk;
+	int i = 0;
+
+	mptcp_for_each_sk(mpcb, sk)
+		i++;
+
+	return i;
+}
+
 /* TCP and MPTCP mpc flag-depending functions */
 u16 mptcp_select_window(struct sock *sk);
 void mptcp_tcp_set_rto(struct sock *sk);
net/mptcp/mptcp_ctrl.c +6 -9
index c2aae9020d91..60e778ac362a 100644
@@ -1379,7 +1379,6 @@ int mptcp_add_sock(struct sock *meta_sk, struct sock *sk, u8 loc_id, u8 rem_id,
 	mpcb->connection_list = tp;
 	tp->mptcp->attached = 1;
 
-	mpcb->cnt_subflows++;
 	atomic_add(atomic_read(&((struct sock *)tp)->sk_rmem_alloc),
 		   &meta_sk->sk_rmem_alloc);
 
@@ -1399,23 +1398,21 @@ int mptcp_add_sock(struct sock *meta_sk, struct sock *sk, u8 loc_id, u8 rem_id,
 	sk->sk_destruct = mptcp_sock_destruct;
 
 	if (sk->sk_family == AF_INET)
-		mptcp_debug("%s: token %#x pi %d, src_addr:%pI4:%d dst_addr:%pI4:%d, cnt_subflows now %d\n",
+		mptcp_debug("%s: token %#x pi %d, src_addr:%pI4:%d dst_addr:%pI4:%d\n",
 			    __func__ , mpcb->mptcp_loc_token,
 			    tp->mptcp->path_index,
 			    &((struct inet_sock *)tp)->inet_saddr,
 			    ntohs(((struct inet_sock *)tp)->inet_sport),
 			    &((struct inet_sock *)tp)->inet_daddr,
-			    ntohs(((struct inet_sock *)tp)->inet_dport),
-			    mpcb->cnt_subflows);
+			    ntohs(((struct inet_sock *)tp)->inet_dport));
 #if IS_ENABLED(CONFIG_IPV6)
 	else
-		mptcp_debug("%s: token %#x pi %d, src_addr:%pI6:%d dst_addr:%pI6:%d, cnt_subflows now %d\n",
+		mptcp_debug("%s: token %#x pi %d, src_addr:%pI6:%d dst_addr:%pI6:%d\n",
 			    __func__ , mpcb->mptcp_loc_token,
 			    tp->mptcp->path_index, &inet6_sk(sk)->saddr,
 			    ntohs(((struct inet_sock *)tp)->inet_sport),
 			    &sk->sk_v6_daddr,
-			    ntohs(((struct inet_sock *)tp)->inet_dport),
-			    mpcb->cnt_subflows);
+			    ntohs(((struct inet_sock *)tp)->inet_dport));
 #endif
 
 	return 0;
@@ -1449,7 +1446,6 @@ void mptcp_del_sock(struct sock *sk)
 			}
 		}
 	}
-	mpcb->cnt_subflows--;
 	if (tp->mptcp->establish_increased)
 		mpcb->cnt_established--;
 
@@ -2813,8 +2809,9 @@ static int mptcp_pm_seq_show(struct seq_file *seq, void *v)
 					   ntohs(isk->inet_dport));
 #endif
 			}
+
 			seq_printf(seq, " %02X %02X %08X:%08X %lu",
-				   meta_sk->sk_state, mpcb->cnt_subflows,
+				   meta_sk->sk_state, mptcp_subflow_count(mpcb),
 				   meta_tp->write_seq - meta_tp->snd_una,
 				   max_t(int, meta_tp->rcv_nxt -
 					 meta_tp->copied_seq, 0),
net/mptcp/mptcp_input.c +10 -1
index ba9e3fe62824..126c961ec55a 100644
@@ -347,6 +347,8 @@ static int mptcp_verif_dss_csum(struct sock *sk)
 
 	/* Now, checksum must be 0 */
 	if (unlikely(csum_fold(csum_tcp))) {
+		struct sock *sk_it = NULL;
+
 		pr_err("%s csum is wrong: %#x tcp-seq %u dss_csum_added %d overflowed %d iterations %d\n",
 		       __func__, csum_fold(csum_tcp), TCP_SKB_CB(last)->seq,
 		       dss_csum_added, overflowed, iter);
@@ -359,7 +361,14 @@ static int mptcp_verif_dss_csum(struct sock *sk)
 		 */
 		tp->mpcb->csum_cutoff_seq = tp->mptcp->map_data_seq;
 
-		if (tp->mpcb->cnt_subflows > 1) {
+		/* Search for another subflow that is fully established */
+		mptcp_for_each_sk(tp->mpcb, sk_it) {
+			if (sk_it != sk &&
+			    tcp_sk(sk_it)->mptcp->fully_established)
+				break;
+		}
+
+		if (sk_it) {
 			mptcp_send_reset(sk);
 			ans = -1;
 		} else {
net/mptcp/mptcp_ndiffports.c +1 -1
index 10147d5e3cca..58acf50df752 100644
@@ -53,7 +53,7 @@ static void create_subflow_worker(struct work_struct *work)
 	    !tcp_sk(mpcb->master_sk)->mptcp->fully_established)
 		goto exit;
 
-	if (num_subflows > iter && num_subflows > mpcb->cnt_subflows) {
+	if (num_subflows > iter && num_subflows > mptcp_subflow_count(mpcb)) {
 		if (meta_sk->sk_family == AF_INET ||
 		    mptcp_v6_is_v4_mapped(meta_sk)) {
 			struct mptcp_loc4 loc;
net/mptcp/mptcp_output.c +1 -1
index 3b99cef02c98..4b88b32d5954 100644
@@ -1350,7 +1350,7 @@ void mptcp_send_active_reset(struct sock *meta_sk, gfp_t priority)
 	struct mptcp_cb *mpcb = meta_tp->mpcb;
 	struct sock *sk;
 
-	if (!mpcb->cnt_subflows)
+	if (!mpcb->connection_list)
 		return;
 
 	WARN_ON(meta_tp->send_mp_fclose);
net/mptcp/mptcp_sched.c +0 -11
index 1881eaec4f3e..a2543c60bd31 100644
@@ -216,14 +216,6 @@ static struct sock *get_available_subflow(struct sock *meta_sk,
 	struct sock *sk;
 	bool force;
 
-	/* if there is only one subflow, bypass the scheduling function */
-	if (mpcb->cnt_subflows == 1) {
-		sk = (struct sock *)mpcb->connection_list;
-		if (!mptcp_is_available(sk, skb, zero_wnd_test))
-			sk = NULL;
-		return sk;
-	}
-
 	/* Answer data_fin on same subflow!!! */
 	if (meta_sk->sk_shutdown & RCV_SHUTDOWN &&
 	    skb && mptcp_is_data_fin(skb)) {
@@ -264,9 +256,6 @@ static struct sk_buff *mptcp_rcv_buf_optimization(struct sock *sk, int penal)
 	struct sk_buff *skb_head;
 	struct defsched_priv *dsp = defsched_get_priv(tp);
 
-	if (tp->mpcb->cnt_subflows == 1)
-		return NULL;
-
 	meta_sk = mptcp_meta_sk(sk);
 	skb_head = tcp_rtx_queue_head(meta_sk);
 
Signature · 1 line
2.16.2
author
#8 lore raw

PATCH 8/14 mptcp: Remove cnt_established

In reply to Christoph Paasch

We want to remove the number of up-calls into the meta-socket. cnt_established is one of those where we are modifying a field in the mpcb.

We can remove it fairly safely. It's only use is in some MPTCP-specific congestion controls to short-circuit some parts of it.

One particular use was in mptcp_should_expand_sndbuf(). There, we use it to see whether to expand the sndbuf when there are only backup subflows available.

One problem with this old code is that the decision whether or not to send on a backup subflow is steered by the modular scheduler. We cannot assume in core-MPTCP code that backup subflows are never being sent on.

By no more taking backup subflows into account, we greatly simplify mptcp_should_expand_sndbuf at the cost of potentially increasing the window even if there is no subflow ready to send because it's a backup subflow.

Signed-off-by Christoph Paasch <cpaasch(a)apple.com>
Signed-off-by Matthieu Baerts <matthieu.baerts(a)tessares.net>
(cherry picked from commit 773f0e58474e56dfdf4334b889beebd4a8062d51)
---
 include/net/mptcp.h     |  4 ----
 net/mptcp/mptcp_ctrl.c  |  8 --------
 net/mptcp/mptcp_input.c | 20 ++------------------
 3 files changed, 2 insertions(+), 30 deletions(-)
include/net/mptcp.h +0 -4
index b1ea946ea898..1174b0ae6d88 100644
@@ -166,7 +166,6 @@ struct mptcp_tcp_sock {
 	u16	map_data_len;
 	u16	slave_sk:1,
 		fully_established:1,
-		establish_increased:1,
 		second_packet:1,
 		attached:1,
 		send_mp_fail:1,
@@ -276,9 +275,6 @@ struct mptcp_cb {
 		snd_hiseq_index:1, /* Index in snd_high_order of snd_nxt */
 		rcv_hiseq_index:1; /* Index in rcv_high_order of rcv_nxt */
 
-	/* socket count in this connection */
-	u8 cnt_established;
-
 #define MPTCP_SCHED_DATA_SIZE 8
 	u8 mptcp_sched[MPTCP_SCHED_DATA_SIZE] __aligned(8);
 	struct mptcp_sched_ops *sched_ops;
net/mptcp/mptcp_ctrl.c +0 -8
index 60e778ac362a..02eaf2208db6 100644
@@ -737,11 +737,6 @@ static void mptcp_set_state(struct sock *sk)
 		tcp_sk(meta_sk)->lsndtime = tcp_jiffies32;
 	}
 
-	if (sk->sk_state == TCP_ESTABLISHED) {
-		tcp_sk(sk)->mptcp->establish_increased = 1;
-		tcp_sk(sk)->mpcb->cnt_established++;
-	}
-
 	if (sk->sk_state == TCP_CLOSE) {
 		if (!sock_flag(sk, SOCK_DEAD))
 			mptcp_sub_close(sk, 0);
@@ -1446,9 +1441,6 @@ void mptcp_del_sock(struct sock *sk)
 			}
 		}
 	}
-	if (tp->mptcp->establish_increased)
-		mpcb->cnt_established--;
-
 	tp->mptcp->next = NULL;
 	tp->mptcp->attached = 0;
 	mpcb->path_index_bits &= ~(1 << tp->mptcp->path_index);
net/mptcp/mptcp_input.c +2 -18
index 126c961ec55a..211605347f70 100644
@@ -2357,11 +2357,9 @@ int mptcp_rcv_synsent_state_process(struct sock *sk, struct sock **skptr,
 /* Similar to tcp_should_expand_sndbuf */
 bool mptcp_should_expand_sndbuf(const struct sock *sk)
 {
-	const struct sock *sk_it;
 	const struct sock *meta_sk = mptcp_meta_sk(sk);
 	const struct tcp_sock *meta_tp = tcp_sk(meta_sk);
-	int cnt_backups = 0;
-	int backup_available = 0;
+	const struct sock *sk_it;
 
 	/* We circumvent this check in tcp_check_space, because we want to
 	 * always call sk_write_space. So, we reproduce the check here.
@@ -2393,24 +2391,10 @@ bool mptcp_should_expand_sndbuf(const struct sock *sk)
 		if (!mptcp_sk_can_send(sk_it))
 			continue;
 
-		/* Backup-flows have to be counted - if there is no other
-		 * subflow we take the backup-flow into account.
-		 */
-		if (tp_it->mptcp->rcv_low_prio || tp_it->mptcp->low_prio)
-			cnt_backups++;
-
-		if (tcp_packets_in_flight(tp_it) < tp_it->snd_cwnd) {
-			if (tp_it->mptcp->rcv_low_prio || tp_it->mptcp->low_prio) {
-				backup_available = 1;
-				continue;
-			}
+		if (tcp_packets_in_flight(tp_it) < tp_it->snd_cwnd)
 			return true;
-		}
 	}
 
-	/* Backup-flow is available for sending - update send-buffer */
-	if (meta_tp->mpcb->cnt_established == cnt_backups && backup_available)
-		return true;
 	return false;
 }
 
Signature · 1 line
2.16.2
author
#9 lore raw

PATCH 9/14 mptcp: Rename tw_lock to mpcb_list_lock

In reply to Christoph Paasch

The lock is going to cover more than just the tw-list. Namely, the subflow-list stored in the mpcb.

Signed-off-by Christoph Paasch <cpaasch(a)apple.com>
Signed-off-by Matthieu Baerts <matthieu.baerts(a)tessares.net>
(cherry picked from commit 411ffb5c2b2a42d578f11b800c97f23db1cb77ee)
Signed-off-by Christoph Paasch <cpaasch(a)apple.com>
---
 include/net/mptcp.h    |  4 +++-
 net/mptcp/mptcp_ctrl.c | 14 +++++++-------
 2 files changed, 10 insertions(+), 8 deletions(-)
include/net/mptcp.h +3 -1
index 1174b0ae6d88..0f9d47b45895 100644
@@ -258,6 +258,9 @@ struct mptcp_cb {
 	/* list of sockets that need a call to release_cb */
 	struct hlist_head callback_list;
 
+	/* Lock used for protecting the different rcu-lists of mptcp_cb */
+	spinlock_t mpcb_list_lock;
+
 	/* High-order bits of 64-bit sequence numbers */
 	u32 snd_high_order[2];
 	u32 rcv_high_order[2];
@@ -289,7 +292,6 @@ struct mptcp_cb {
 	u64	infinite_rcv_seq;
 
 	/***** Start of fields, used for connection closure */
-	spinlock_t	 tw_lock;
 	unsigned char	 mptw_state;
 	u8		 dfin_path_index;
 
net/mptcp/mptcp_ctrl.c +7 -7
index 02eaf2208db6..052ae0e052b0 100644
@@ -665,14 +665,14 @@ void mptcp_sock_destruct(struct sock *sk)
 		 * update to the rcv_nxt of the time-wait-sock and remove
 		 * its reference to the mpcb.
 		 */
-		spin_lock_bh(&mpcb->tw_lock);
+		spin_lock_bh(&mpcb->mpcb_list_lock);
 		list_for_each_entry_rcu(mptw, &mpcb->tw_list, list) {
 			list_del_rcu(&mptw->list);
 			mptw->in_list = 0;
 			mptcp_mpcb_put(mpcb);
 			rcu_assign_pointer(mptw->mpcb, NULL);
 		}
-		spin_unlock_bh(&mpcb->tw_lock);
+		spin_unlock_bh(&mpcb->mpcb_list_lock);
 
 		mptcp_debug("%s destroying meta-sk token %#x\n", __func__,
 			    tcp_sk(sk)->mpcb->mptcp_loc_token);
@@ -1279,9 +1279,9 @@ static int mptcp_alloc_mpcb(struct sock *meta_sk, __u64 remote_key,
 
 	/* Init time-wait stuff */
 	INIT_LIST_HEAD(&mpcb->tw_list);
-	spin_lock_init(&mpcb->tw_lock);
 
 	INIT_HLIST_HEAD(&mpcb->callback_list);
+	spin_lock_init(&mpcb->mpcb_list_lock);
 
 	mptcp_mpcb_inherit_sockopts(meta_sk, master_sk);
 
@@ -2249,10 +2249,10 @@ int mptcp_init_tw_sock(struct sock *sk, struct tcp_timewait_sock *tw)
 		mptw->rcv_nxt++;
 	rcu_assign_pointer(mptw->mpcb, mpcb);
 
-	spin_lock_bh(&mpcb->tw_lock);
+	spin_lock_bh(&mpcb->mpcb_list_lock);
 	list_add_rcu(&mptw->list, &tp->mpcb->tw_list);
 	mptw->in_list = 1;
-	spin_unlock_bh(&mpcb->tw_lock);
+	spin_unlock_bh(&mpcb->mpcb_list_lock);
 
 	return 0;
 }
@@ -2268,12 +2268,12 @@ void mptcp_twsk_destructor(struct tcp_timewait_sock *tw)
 	 * from the list and drop the ref properly.
 	 */
 	if (mpcb && refcount_inc_not_zero(&mpcb->mpcb_refcnt)) {
-		spin_lock_bh(&mpcb->tw_lock);
+		spin_lock_bh(&mpcb->mpcb_list_lock);
 		if (tw->mptcp_tw->in_list) {
 			list_del_rcu(&tw->mptcp_tw->list);
 			tw->mptcp_tw->in_list = 0;
 		}
-		spin_unlock_bh(&mpcb->tw_lock);
+		spin_unlock_bh(&mpcb->mpcb_list_lock);
 
 		/* Twice, because we increased it above */
 		mptcp_mpcb_put(mpcb);
Signature · 1 line
2.16.2
author
#10 lore raw

PATCH 10/14 mptcp: Make subflow-list an RCU-list

In reply to Christoph Paasch

mptcp_add_sock() will now be called without holding the meta-level lock. However, mptcp_add_sock() wants to add the subflow to the meta-level list, thus we need to protect this by a lock. We use the mpcb_list_lock for that.

Now that we are locking during add/del, and want to allow lockless traversal of the list, this implies that we need to make it an RCU-list.

So, this patch transitions to the RCU hlist. The list-traversal macros (hlist_for_each_entry_rcu) require me to now pass the mptcp_tcp_sock to mptcp_for_each_*. So, I had to change all the places in the code where we call one of the list-traversal macros to adapt to this.

Signed-off-by Christoph Paasch <cpaasch(a)apple.com>
Signed-off-by Matthieu Baerts <matthieu.baerts(a)tessares.net>
(cherry picked from commit 7a662b690069642c138da16ff9396e6826fd0a95)
---
 include/net/mptcp.h        | 67 +++++++++++++++++-----------------
 net/ipv4/af_inet.c         |  9 ++---
 net/ipv4/ip_sockglue.c     |  7 ++--
 net/ipv4/tcp.c             | 31 +++++++++-------
 net/mptcp/mptcp_ctrl.c     | 89 ++++++++++++++++++++++++++++------------------
 net/mptcp/mptcp_fullmesh.c | 28 ++++++++++-----
 net/mptcp/mptcp_input.c    | 44 +++++++++++++++--------
 net/mptcp/mptcp_output.c   | 38 +++++++++++++-------
 net/mptcp/mptcp_sched.c    | 21 +++++++----
 9 files changed, 207 insertions(+), 127 deletions(-)
include/net/mptcp.h +34 -33
index 0f9d47b45895..091e56a7b334 100644
@@ -156,7 +156,7 @@ struct mptcp_options_received {
 };
 
 struct mptcp_tcp_sock {
-	struct tcp_sock	*next;		/* Next subflow socket */
+	struct hlist_node node;
 	struct hlist_node cb_list;
 	struct mptcp_options_received rx_opt;
 
@@ -254,7 +254,7 @@ struct mptcp_sched_ops {
 
 struct mptcp_cb {
 	/* list of sockets in this multipath connection */
-	struct tcp_sock *connection_list;
+	struct hlist_head conn_list;
 	/* list of sockets that need a call to release_cb */
 	struct hlist_head callback_list;
 
@@ -309,7 +309,7 @@ struct mptcp_cb {
 	/***** Start of fields, used for subflow establishment */
 	struct sock *meta_sk;
 
-	/* Master socket, also part of the connection_list, this
+	/* Master socket, also part of the conn_list, this
 	 * socket is the one that the application sees.
 	 */
 	struct sock *master_sk;
@@ -661,21 +661,17 @@ extern struct workqueue_struct *mptcp_wq;
 			pr_err(fmt, ##args);					\
 	} while (0)
 
-/* Iterates over all subflows */
-#define mptcp_for_each_tp(mpcb, tp)					\
-	for ((tp) = (mpcb)->connection_list; (tp); (tp) = (tp)->mptcp->next)
+static inline struct sock *mptcp_to_sock(const struct mptcp_tcp_sock *mptcp)
+{
+	return (struct sock *)mptcp->tp;
+}
 
-#define mptcp_for_each_sk(mpcb, sk)					\
-	for ((sk) = (struct sock *)(mpcb)->connection_list;		\
-	     sk;							\
-	     sk = (struct sock *)tcp_sk(sk)->mptcp->next)
+#define mptcp_for_each_sub(__mpcb, __mptcp)					\
+	hlist_for_each_entry_rcu(__mptcp, &((__mpcb)->conn_list), node)
 
-#define mptcp_for_each_sk_safe(__mpcb, __sk, __temp)			\
-	for (__sk = (struct sock *)(__mpcb)->connection_list,		\
-	     __temp = __sk ? (struct sock *)tcp_sk(__sk)->mptcp->next : NULL; \
-	     __sk;							\
-	     __sk = __temp,						\
-	     __temp = __sk ? (struct sock *)tcp_sk(__sk)->mptcp->next : NULL)
+/* Must be called with the appropriate lock held */
+#define mptcp_for_each_sub_safe(__mpcb, __mptcp, __tmp)				\
+	hlist_for_each_entry_safe(__mptcp, __tmp, &((__mpcb)->conn_list), node)
 
 /* Iterates over all bit set to 1 in a bitset */
 #define mptcp_for_each_bit_set(b, i)					\
@@ -922,12 +918,14 @@ struct request_sock *rev_mptcp_rsk(const struct mptcp_request_sock *req)
 
 static inline bool mptcp_can_sendpage(struct sock *sk)
 {
-	struct sock *sk_it;
+	struct mptcp_tcp_sock *mptcp;
 
 	if (tcp_sk(sk)->mpcb->dss_csum)
 		return false;
 
-	mptcp_for_each_sk(tcp_sk(sk)->mpcb, sk_it) {
+	mptcp_for_each_sub(tcp_sk(sk)->mpcb, mptcp) {
+		struct sock *sk_it = mptcp_to_sock(mptcp);
+
 		if (!(sk_it->sk_route_caps & NETIF_F_SG))
 			return false;
 	}
@@ -961,9 +959,12 @@ static inline void mptcp_send_reset(struct sock *sk)
 static inline void mptcp_sub_force_close_all(struct mptcp_cb *mpcb,
 					     struct sock *except)
 {
-	struct sock *sk_it, *tmp;
+	struct mptcp_tcp_sock *mptcp;
+	struct hlist_node *tmp;
+
+	mptcp_for_each_sub_safe(mpcb, mptcp, tmp) {
+		struct sock *sk_it = mptcp_to_sock(mptcp);
 
-	mptcp_for_each_sk_safe(mpcb, sk_it, tmp) {
 		if (sk_it != except)
 			mptcp_send_reset(sk_it);
 	}
@@ -1149,12 +1150,14 @@ static inline int mptcp_sk_can_send_ack(const struct sock *sk)
 
 static inline bool mptcp_can_sg(const struct sock *meta_sk)
 {
-	struct sock *sk;
+	struct mptcp_tcp_sock *mptcp;
 
 	if (tcp_sk(meta_sk)->mpcb->dss_csum)
 		return false;
 
-	mptcp_for_each_sk(tcp_sk(meta_sk)->mpcb, sk) {
+	mptcp_for_each_sub(tcp_sk(meta_sk)->mpcb, mptcp) {
+		struct sock *sk = mptcp_to_sock(mptcp);
+
 		if (!mptcp_sk_can_send(sk))
 			continue;
 		if (!(sk->sk_route_caps & NETIF_F_SG))
@@ -1165,9 +1168,9 @@ static inline bool mptcp_can_sg(const struct sock *meta_sk)
 
 static inline void mptcp_set_rto(struct sock *sk)
 {
-	struct tcp_sock *tp = tcp_sk(sk);
-	struct sock *sk_it;
 	struct inet_connection_sock *micsk = inet_csk(mptcp_meta_sk(sk));
+	struct tcp_sock *tp = tcp_sk(sk);
+	struct mptcp_tcp_sock *mptcp;
 	__u32 max_rto = 0;
 
 	/* We are in recovery-phase on the MPTCP-level. Do not update the
@@ -1176,7 +1179,9 @@ static inline void mptcp_set_rto(struct sock *sk)
 	if (micsk->icsk_retransmits)
 		return;
 
-	mptcp_for_each_sk(tp->mpcb, sk_it) {
+	mptcp_for_each_sub(tp->mpcb, mptcp) {
+		struct sock *sk_it = mptcp_to_sock(mptcp);
+
 		if ((mptcp_sk_can_send(sk_it) || sk->sk_state == TCP_SYN_RECV) &&
 		    inet_csk(sk_it)->icsk_rto > max_rto)
 			max_rto = inet_csk(sk_it)->icsk_rto;
@@ -1265,10 +1270,10 @@ static inline bool mptcp_can_new_subflow(const struct sock *meta_sk)
 
 static inline int mptcp_subflow_count(const struct mptcp_cb *mpcb)
 {
-	struct sock *sk;
+	struct mptcp_tcp_sock *mptcp;
 	int i = 0;
 
-	mptcp_for_each_sk(mpcb, sk)
+	mptcp_for_each_sub(mpcb, mptcp)
 		i++;
 
 	return i;
@@ -1286,12 +1291,8 @@ bool mptcp_prune_ofo_queue(struct sock *sk);
 	do {				\
 	} while (0)
 
-/* Without MPTCP, we just do one iteration
- * over the only socket available. This assumes that
- * the sk/tp arg is the socket in that case.
- */
-#define mptcp_for_each_sk(mpcb, sk)
-#define mptcp_for_each_sk_safe(__mpcb, __sk, __temp)
+#define mptcp_for_each_sub(__mpcb, __mptcp)					\
+	if (0)
 
 #define MPTCP_INC_STATS(net, field)	\
 	do {				\
net/ipv4/af_inet.c +5 -4
index a58a8fbd6192..4b44aa87af2d 100644
@@ -744,13 +744,14 @@ int inet_accept(struct socket *sock, struct socket *newsock, int flags,
 	sock_rps_record_flow(sk2);
 
 	if (sk2->sk_protocol == IPPROTO_TCP && mptcp(tcp_sk(sk2))) {
-		struct sock *sk_it = sk2;
+		struct mptcp_tcp_sock *mptcp;
 
-		mptcp_for_each_sk(tcp_sk(sk2)->mpcb, sk_it)
-			sock_rps_record_flow(sk_it);
+		mptcp_for_each_sub(tcp_sk(sk2)->mpcb, mptcp) {
+			sock_rps_record_flow(mptcp_to_sock(mptcp));
+		}
 
 		if (tcp_sk(sk2)->mpcb->master_sk) {
-			sk_it = tcp_sk(sk2)->mpcb->master_sk;
+			struct sock *sk_it = tcp_sk(sk2)->mpcb->master_sk;
 
 			write_lock_bh(&sk_it->sk_callback_lock);
 			sk_it->sk_wq = newsock->wq;
net/ipv4/ip_sockglue.c +5 -2
index 6a41981c276e..eb72cf990876 100644
@@ -753,8 +753,11 @@ static int do_ip_setsockopt(struct sock *sk, int level,
 			sk_dst_reset(sk);
 			/* Update TOS on mptcp subflow */
 			if (is_meta_sk(sk)) {
-				struct sock *sk_it;
-				mptcp_for_each_sk(tcp_sk(sk)->mpcb, sk_it) {
+				struct mptcp_tcp_sock *mptcp;
+
+				mptcp_for_each_sub(tcp_sk(sk)->mpcb, mptcp) {
+					struct sock *sk_it = mptcp_to_sock(mptcp);
+
 					if (inet_sk(sk_it)->tos != inet_sk(sk)->tos) {
 						inet_sk(sk_it)->tos = inet_sk(sk)->tos;
 						sk_it->sk_priority = sk->sk_priority;
net/ipv4/tcp.c +19 -12
index eeb98619b181..ab17bc222490 100644
@@ -813,9 +813,11 @@ ssize_t tcp_splice_read(struct socket *sock, loff_t *ppos,
 
 #ifdef CONFIG_MPTCP
 	if (mptcp(tcp_sk(sk))) {
-		struct sock *sk_it;
-		mptcp_for_each_sk(tcp_sk(sk)->mpcb, sk_it)
-			sock_rps_record_flow(sk_it);
+		struct mptcp_tcp_sock *mptcp;
+
+		mptcp_for_each_sub(tcp_sk(sk)->mpcb, mptcp) {
+			sock_rps_record_flow(mptcp_to_sock(mptcp));
+		}
 	}
 #endif
 	/*
@@ -993,7 +995,7 @@ ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset,
 	}
 
 	if (mptcp(tp)) {
-		struct sock *sk_it = sk;
+		struct mptcp_tcp_sock *mptcp;
 
 		/* We must check this with socket-lock hold because we iterate
 		 * over the subflows.
@@ -1008,8 +1010,9 @@ ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset,
 			return ret;
 		}
 
-		mptcp_for_each_sk(tp->mpcb, sk_it)
-			sock_rps_record_flow(sk_it);
+		mptcp_for_each_sub(tp->mpcb, mptcp) {
+			sock_rps_record_flow(mptcp_to_sock(mptcp));
+		}
 	}
 
 	sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
@@ -1287,9 +1290,11 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size)
 	}
 
 	if (mptcp(tp)) {
-		struct sock *sk_it = sk;
-		mptcp_for_each_sk(tp->mpcb, sk_it)
-			sock_rps_record_flow(sk_it);
+		struct mptcp_tcp_sock *mptcp;
+
+		mptcp_for_each_sub(tp->mpcb, mptcp) {
+			sock_rps_record_flow(mptcp_to_sock(mptcp));
+		}
 	}
 
 	if (unlikely(tp->repair)) {
@@ -1994,9 +1999,11 @@ int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int nonblock,
 
 #ifdef CONFIG_MPTCP
 	if (mptcp(tp)) {
-		struct sock *sk_it;
-		mptcp_for_each_sk(tp->mpcb, sk_it)
-			sock_rps_record_flow(sk_it);
+		struct mptcp_tcp_sock *mptcp;
+
+		mptcp_for_each_sub(tp->mpcb, mptcp) {
+			sock_rps_record_flow(mptcp_to_sock(mptcp));
+		}
 	}
 #endif
 
net/mptcp/mptcp_ctrl.c +55 -34

236 lines not on this page · raw message

net/mptcp/mptcp_fullmesh.c +20 -8
index d29102a1dffb..53c1ba7319f5 100644
@@ -903,8 +903,9 @@ static void mptcp_address_worker(struct work_struct *work)
 			}
 
 			if (event->code == MPTCP_EVENT_DEL) {
-				struct sock *sk, *tmpsk;
+				struct mptcp_tcp_sock *mptcp;
 				struct mptcp_loc_addr *mptcp_local;
+				struct hlist_node *tmp;
 				bool found = false;
 
 				mptcp_local = rcu_dereference_bh(fm_ns->local);
@@ -914,7 +915,9 @@ static void mptcp_address_worker(struct work_struct *work)
 					update_addr_bitfields(meta_sk, mptcp_local);
 
 				/* Look for the socket and remove him */
-				mptcp_for_each_sk_safe(mpcb, sk, tmpsk) {
+				mptcp_for_each_sub_safe(mpcb, mptcp, tmp) {
+					struct sock *sk = mptcp_to_sock(mptcp);
+
 					if ((event->family == AF_INET6 &&
 					     (sk->sk_family == AF_INET ||
 					      mptcp_v6_is_v4_mapped(sk))) ||
@@ -964,9 +967,10 @@ static void mptcp_address_worker(struct work_struct *work)
 			}
 
 			if (event->code == MPTCP_EVENT_MOD) {
-				struct sock *sk;
+				struct mptcp_tcp_sock *mptcp;
 
-				mptcp_for_each_sk(mpcb, sk) {
+				mptcp_for_each_sub(mpcb, mptcp) {
+					struct sock *sk = mptcp_to_sock(mptcp);
 					struct tcp_sock *tp = tcp_sk(sk);
 					if (event->family == AF_INET &&
 					    (sk->sk_family == AF_INET ||
@@ -1455,8 +1459,9 @@ static void full_mesh_release_sock(struct sock *meta_sk)
 	struct mptcp_cb *mpcb = tcp_sk(meta_sk)->mpcb;
 	struct fullmesh_priv *fmp = fullmesh_get_priv(mpcb);
 	const struct mptcp_fm_ns *fm_ns = fm_get_ns(sock_net(meta_sk));
-	struct sock *sk, *tmpsk;
 	bool meta_v4 = meta_sk->sk_family == AF_INET;
+	struct mptcp_tcp_sock *mptcp;
+	struct hlist_node *tmp;
 	int i;
 
 	rcu_read_lock_bh();
@@ -1470,7 +1475,8 @@ static void full_mesh_release_sock(struct sock *meta_sk)
 		struct in_addr ifa = mptcp_local->locaddr4[i].addr;
 		bool found = false;
 
-		mptcp_for_each_sk(mpcb, sk) {
+		mptcp_for_each_sub(mpcb, mptcp) {
+			struct sock *sk = mptcp_to_sock(mptcp);
 			struct tcp_sock *tp = tcp_sk(sk);
 
 			if (sk->sk_family == AF_INET6 &&
@@ -1491,6 +1497,8 @@ static void full_mesh_release_sock(struct sock *meta_sk)
 		}
 
 		if (!found) {
+			struct sock *sk;
+
 			fmp->add_addr++;
 			mpcb->addr_signal = 1;
 
@@ -1511,7 +1519,8 @@ static void full_mesh_release_sock(struct sock *meta_sk)
 		struct in6_addr ifa = mptcp_local->locaddr6[i].addr;
 		bool found = false;
 
-		mptcp_for_each_sk(mpcb, sk) {
+		mptcp_for_each_sub(mpcb, mptcp) {
+			struct sock *sk = mptcp_to_sock(mptcp);
 			struct tcp_sock *tp = tcp_sk(sk);
 
 			if (sk->sk_family == AF_INET ||
@@ -1532,6 +1541,8 @@ static void full_mesh_release_sock(struct sock *meta_sk)
 		}
 
 		if (!found) {
+			struct sock *sk;
+
 			fmp->add_addr++;
 			mpcb->addr_signal = 1;
 
@@ -1546,7 +1557,8 @@ static void full_mesh_release_sock(struct sock *meta_sk)
 #endif
 
 	/* Now, detect address-removals */
-	mptcp_for_each_sk_safe(mpcb, sk, tmpsk) {
+	mptcp_for_each_sub_safe(mpcb, mptcp, tmp) {
+		struct sock *sk = mptcp_to_sock(mptcp);
 		bool shall_remove = true;
 
 		if (sk->sk_family == AF_INET || mptcp_v6_is_v4_mapped(sk)) {
net/mptcp/mptcp_input.c +29 -15

112 lines not on this page · raw message

net/mptcp/mptcp_output.c +25 -13
index 4b88b32d5954..12733ba2c1e1 100644
@@ -644,7 +644,6 @@ int mptcp_write_wakeup(struct sock *meta_sk, int mib)
 {
 	struct tcp_sock *meta_tp = tcp_sk(meta_sk);
 	struct sk_buff *skb;
-	struct sock *sk_it;
 	int ans = 0;
 
 	if (meta_sk->sk_state == TCP_CLOSE)
@@ -701,17 +700,22 @@ int mptcp_write_wakeup(struct sock *meta_sk, int mib)
 
 		return 0;
 	} else {
+		struct mptcp_tcp_sock *mptcp;
+
 window_probe:
 		if (between(meta_tp->snd_up, meta_tp->snd_una + 1,
 			    meta_tp->snd_una + 0xFFFF)) {
-			mptcp_for_each_sk(meta_tp->mpcb, sk_it) {
+			mptcp_for_each_sub(meta_tp->mpcb, mptcp) {
+				struct sock *sk_it = mptcp_to_sock(mptcp);
+
 				if (mptcp_sk_can_send_ack(sk_it))
 					tcp_xmit_probe_skb(sk_it, 1, mib);
 			}
 		}
 
 		/* At least one of the tcp_xmit_probe_skb's has to succeed */
-		mptcp_for_each_sk(meta_tp->mpcb, sk_it) {
+		mptcp_for_each_sub(meta_tp->mpcb, mptcp) {
+			struct sock *sk_it = mptcp_to_sock(mptcp);
 			int ret;
 
 			if (!mptcp_sk_can_send_ack(sk_it))
@@ -729,6 +733,7 @@ bool mptcp_write_xmit(struct sock *meta_sk, unsigned int mss_now, int nonagle,
 		     int push_one, gfp_t gfp)
 {
 	struct tcp_sock *meta_tp = tcp_sk(meta_sk), *subtp;
+	struct mptcp_tcp_sock *mptcp;
 	struct sock *subsk = NULL;
 	struct mptcp_cb *mpcb = meta_tp->mpcb;
 	struct sk_buff *skb;
@@ -853,7 +858,8 @@ bool mptcp_write_xmit(struct sock *meta_sk, unsigned int mss_now, int nonagle,
 			break;
 	}
 
-	mptcp_for_each_sk(mpcb, subsk) {
+	mptcp_for_each_sub(mpcb, mptcp) {
+		subsk = mptcp_to_sock(mptcp);
 		subtp = tcp_sk(subsk);
 
 		if (!(path_mask & mptcp_pi_to_flag(subtp->mptcp->path_index)))
@@ -1350,7 +1356,7 @@ void mptcp_send_active_reset(struct sock *meta_sk, gfp_t priority)
 	struct mptcp_cb *mpcb = meta_tp->mpcb;
 	struct sock *sk;
 
-	if (!mpcb->connection_list)
+	if (hlist_empty(&mpcb->conn_list))
 		return;
 
 	WARN_ON(meta_tp->send_mp_fclose);
@@ -1725,10 +1731,11 @@ void mptcp_select_initial_window(const struct sock *sk, int __space, __u32 mss,
 static inline u64 mptcp_calc_rate(const struct sock *meta_sk, unsigned int mss,
 				  unsigned int (*mss_cb)(struct sock *sk))
 {
-	struct sock *sk;
+	struct mptcp_tcp_sock *mptcp;
 	u64 rate = 0;
 
-	mptcp_for_each_sk(tcp_sk(meta_sk)->mpcb, sk) {
+	mptcp_for_each_sub(tcp_sk(meta_sk)->mpcb, mptcp) {
+		struct sock *sk = mptcp_to_sock(mptcp);
 		struct tcp_sock *tp = tcp_sk(sk);
 		int this_mss;
 		u64 this_rate;
@@ -1780,11 +1787,12 @@ static inline u64 mptcp_calc_rate(const struct sock *meta_sk, unsigned int mss,
 static unsigned int __mptcp_current_mss(const struct sock *meta_sk,
 					unsigned int (*mss_cb)(struct sock *sk))
 {
+	struct mptcp_tcp_sock *mptcp;
 	unsigned int mss = 0;
 	u64 rate = 0;
-	struct sock *sk;
 
-	mptcp_for_each_sk(tcp_sk(meta_sk)->mpcb, sk) {
+	mptcp_for_each_sub(tcp_sk(meta_sk)->mpcb, mptcp) {
+		struct sock *sk = mptcp_to_sock(mptcp);
 		int this_mss;
 		u64 this_rate;
 
@@ -1849,14 +1857,16 @@ int mptcp_select_size(const struct sock *meta_sk, bool first_skb, bool zc)
 
 int mptcp_check_snd_buf(const struct tcp_sock *tp)
 {
-	const struct sock *sk;
+	const struct mptcp_tcp_sock *mptcp;
 	u32 rtt_max = tp->srtt_us;
 	u64 bw_est;
 
 	if (!tp->srtt_us)
 		return tp->reordering + 1;
 
-	mptcp_for_each_sk(tp->mpcb, sk) {
+	mptcp_for_each_sub(tp->mpcb, mptcp) {
+		const struct sock *sk = mptcp_to_sock(mptcp);
+
 		if (!mptcp_sk_can_send(sk))
 			continue;
 
@@ -1874,11 +1884,13 @@ int mptcp_check_snd_buf(const struct tcp_sock *tp)
 unsigned int mptcp_xmit_size_goal(const struct sock *meta_sk, u32 mss_now,
 				  int large_allowed)
 {
-	struct sock *sk;
 	u32 xmit_size_goal = 0;
 
 	if (large_allowed && !tcp_sk(meta_sk)->mpcb->dss_csum) {
-		mptcp_for_each_sk(tcp_sk(meta_sk)->mpcb, sk) {
+		struct mptcp_tcp_sock *mptcp;
+
+		mptcp_for_each_sub(tcp_sk(meta_sk)->mpcb, mptcp) {
+			struct sock *sk = mptcp_to_sock(mptcp);
 			int this_size_goal;
 
 			if (!mptcp_sk_can_send(sk))
net/mptcp/mptcp_sched.c +15 -6
index a2543c60bd31..b440df7aae71 100644
@@ -135,9 +135,10 @@ static struct sock
 	u32 min_srtt = 0xffffffff;
 	bool found_unused = false;
 	bool found_unused_una = false;
-	struct sock *sk;
+	struct mptcp_tcp_sock *mptcp;
 
-	mptcp_for_each_sk(mpcb, sk) {
+	mptcp_for_each_sub(mpcb, mptcp) {
+		struct sock *sk = mptcp_to_sock(mptcp);
 		struct tcp_sock *tp = tcp_sk(sk);
 		bool unused = false;
 
@@ -219,7 +220,11 @@ static struct sock *get_available_subflow(struct sock *meta_sk,
 	/* Answer data_fin on same subflow!!! */
 	if (meta_sk->sk_shutdown & RCV_SHUTDOWN &&
 	    skb && mptcp_is_data_fin(skb)) {
-		mptcp_for_each_sk(mpcb, sk) {
+		struct mptcp_tcp_sock *mptcp;
+
+		mptcp_for_each_sub(mpcb, mptcp) {
+			sk = mptcp_to_sock(mptcp);
+
 			if (tcp_sk(sk)->mptcp->path_index == mpcb->dfin_path_index &&
 			    mptcp_is_available(sk, skb, zero_wnd_test))
 				return sk;
@@ -252,7 +257,7 @@ static struct sk_buff *mptcp_rcv_buf_optimization(struct sock *sk, int penal)
 {
 	struct sock *meta_sk;
 	const struct tcp_sock *tp = tcp_sk(sk);
-	struct tcp_sock *tp_it;
+	struct mptcp_tcp_sock *mptcp;
 	struct sk_buff *skb_head;
 	struct defsched_priv *dsp = defsched_get_priv(tp);
 
@@ -275,7 +280,9 @@ static struct sk_buff *mptcp_rcv_buf_optimization(struct sock *sk, int penal)
 		goto retrans;
 
 	/* Half the cwnd of the slow flow */
-	mptcp_for_each_tp(tp->mpcb, tp_it) {
+	mptcp_for_each_sub(tp->mpcb, mptcp) {
+		struct tcp_sock *tp_it = mptcp->tp;
+
 		if (tp_it != tp &&
 		    TCP_SKB_CB(skb_head)->path_mask & mptcp_pi_to_flag(tp_it->mptcp->path_index)) {
 			if (tp->srtt_us < tp_it->srtt_us && inet_csk((struct sock *)tp_it)->icsk_ca_state == TCP_CA_Open) {
@@ -298,7 +305,9 @@ static struct sk_buff *mptcp_rcv_buf_optimization(struct sock *sk, int penal)
 	/* Segment not yet injected into this path? Take it!!! */
 	if (!(TCP_SKB_CB(skb_head)->path_mask & mptcp_pi_to_flag(tp->mptcp->path_index))) {
 		bool do_retrans = false;
-		mptcp_for_each_tp(tp->mpcb, tp_it) {
+		mptcp_for_each_sub(tp->mpcb, mptcp) {
+			struct tcp_sock *tp_it = mptcp->tp;
+
 			if (tp_it != tp &&
 			    TCP_SKB_CB(skb_head)->path_mask & mptcp_pi_to_flag(tp_it->mptcp->path_index)) {
 				if (tp_it->snd_cwnd <= 4) {
Signature · 1 line
2.16.2
#11 lore raw

Re: [MPTCP] [PATCH mptcp-net-next 10/14] mptcp: Make subflow-list an RCU-list

Christoph -

Quoted 350 lines · Christoph Paasch

mptcp_add_sock() will now be called without holding the meta-level lock. However, mptcp_add_sock() wants to add the subflow to the meta-level list, thus we need to protect this by a lock. We use the mpcb_list_lock for that.

Now that we are locking during add/del, and want to allow lockless traversal of the list, this implies that we need to make it an RCU-list.

So, this patch transitions to the RCU hlist. The list-traversal macros (hlist_for_each_entry_rcu) require me to now pass the mptcp_tcp_sock to mptcp_for_each_*. So, I had to change all the places in the code where we call one of the list-traversal macros to adapt to this.

Signed-off-by Christoph Paasch <cpaasch(a)apple.com>
Signed-off-by Matthieu Baerts <matthieu.baerts(a)tessares.net>
(cherry picked from commit 7a662b690069642c138da16ff9396e6826fd0a95)

---

include/net/mptcp.h        | 67 +++++++++++++++++-----------------
net/ipv4/af_inet.c         |  9 ++---
net/ipv4/ip_sockglue.c     |  7 ++--
net/ipv4/tcp.c             | 31 +++++++++-------
net/mptcp/mptcp_ctrl.c     | 89 ++++++++++++++++++++++++++++------------------
net/mptcp/mptcp_fullmesh.c | 28 ++++++++++-----
net/mptcp/mptcp_input.c    | 44 +++++++++++++++--------
net/mptcp/mptcp_output.c   | 38 +++++++++++++-------
net/mptcp/mptcp_sched.c    | 21 +++++++----
9 files changed, 207 insertions(+), 127 deletions(-)
include/net/mptcp.h +34 -33
index 0f9d47b45895..091e56a7b334 100644
@@ -156,7 +156,7 @@ struct mptcp_options_received {
 };
 
 struct mptcp_tcp_sock {
-	struct tcp_sock	*next;		/* Next subflow socket */
+	struct hlist_node node;
 	struct hlist_node cb_list;
 	struct mptcp_options_received rx_opt;
 
@@ -254,7 +254,7 @@ struct mptcp_sched_ops {
 
 struct mptcp_cb {
 	/* list of sockets in this multipath connection */
-	struct tcp_sock *connection_list;
+	struct hlist_head conn_list;
 	/* list of sockets that need a call to release_cb */
 	struct hlist_head callback_list;
 
@@ -309,7 +309,7 @@ struct mptcp_cb {
 	/***** Start of fields, used for subflow establishment */
 	struct sock *meta_sk;
 
-	/* Master socket, also part of the connection_list, this
+	/* Master socket, also part of the conn_list, this
 	 * socket is the one that the application sees.
 	 */
 	struct sock *master_sk;
@@ -661,21 +661,17 @@ extern struct workqueue_struct *mptcp_wq;
 			pr_err(fmt, ##args);					\
 	} while (0)
 
-/* Iterates over all subflows */
-#define mptcp_for_each_tp(mpcb, tp)					\
-	for ((tp) = (mpcb)->connection_list; (tp); (tp) = (tp)->mptcp->next)
+static inline struct sock *mptcp_to_sock(const struct mptcp_tcp_sock *mptcp)
+{
+	return (struct sock *)mptcp->tp;
+}
 
-#define mptcp_for_each_sk(mpcb, sk)					\
-	for ((sk) = (struct sock *)(mpcb)->connection_list;		\
-	     sk;							\
-	     sk = (struct sock *)tcp_sk(sk)->mptcp->next)
+#define mptcp_for_each_sub(__mpcb, __mptcp)					\
+	hlist_for_each_entry_rcu(__mptcp, &((__mpcb)->conn_list), node)
 
-#define mptcp_for_each_sk_safe(__mpcb, __sk, __temp)			\
-	for (__sk = (struct sock *)(__mpcb)->connection_list,		\
-	     __temp = __sk ? (struct sock *)tcp_sk(__sk)->mptcp->next : NULL; \
-	     __sk;							\
-	     __sk = __temp,						\
-	     __temp = __sk ? (struct sock *)tcp_sk(__sk)->mptcp->next : NULL)
+/* Must be called with the appropriate lock held */
+#define mptcp_for_each_sub_safe(__mpcb, __mptcp, __tmp)				\
+	hlist_for_each_entry_safe(__mptcp, __tmp, &((__mpcb)->conn_list), node)
 
 /* Iterates over all bit set to 1 in a bitset */
 #define mptcp_for_each_bit_set(b, i)					\
@@ -922,12 +918,14 @@ struct request_sock *rev_mptcp_rsk(const struct mptcp_request_sock *req)
 
 static inline bool mptcp_can_sendpage(struct sock *sk)
 {
-	struct sock *sk_it;
+	struct mptcp_tcp_sock *mptcp;
 
 	if (tcp_sk(sk)->mpcb->dss_csum)
 		return false;
 
-	mptcp_for_each_sk(tcp_sk(sk)->mpcb, sk_it) {
+	mptcp_for_each_sub(tcp_sk(sk)->mpcb, mptcp) {
+		struct sock *sk_it = mptcp_to_sock(mptcp);
+
 		if (!(sk_it->sk_route_caps & NETIF_F_SG))
 			return false;
 	}
@@ -961,9 +959,12 @@ static inline void mptcp_send_reset(struct sock *sk)
 static inline void mptcp_sub_force_close_all(struct mptcp_cb *mpcb,
 					     struct sock *except)
 {
-	struct sock *sk_it, *tmp;
+	struct mptcp_tcp_sock *mptcp;
+	struct hlist_node *tmp;
+
+	mptcp_for_each_sub_safe(mpcb, mptcp, tmp) {
+		struct sock *sk_it = mptcp_to_sock(mptcp);
 
-	mptcp_for_each_sk_safe(mpcb, sk_it, tmp) {
 		if (sk_it != except)
 			mptcp_send_reset(sk_it);
 	}
@@ -1149,12 +1150,14 @@ static inline int mptcp_sk_can_send_ack(const struct sock *sk)
 
 static inline bool mptcp_can_sg(const struct sock *meta_sk)
 {
-	struct sock *sk;
+	struct mptcp_tcp_sock *mptcp;
 
 	if (tcp_sk(meta_sk)->mpcb->dss_csum)
 		return false;
 
-	mptcp_for_each_sk(tcp_sk(meta_sk)->mpcb, sk) {
+	mptcp_for_each_sub(tcp_sk(meta_sk)->mpcb, mptcp) {
+		struct sock *sk = mptcp_to_sock(mptcp);
+
 		if (!mptcp_sk_can_send(sk))
 			continue;
 		if (!(sk->sk_route_caps & NETIF_F_SG))
@@ -1165,9 +1168,9 @@ static inline bool mptcp_can_sg(const struct sock *meta_sk)
 
 static inline void mptcp_set_rto(struct sock *sk)
 {
-	struct tcp_sock *tp = tcp_sk(sk);
-	struct sock *sk_it;
 	struct inet_connection_sock *micsk = inet_csk(mptcp_meta_sk(sk));
+	struct tcp_sock *tp = tcp_sk(sk);
+	struct mptcp_tcp_sock *mptcp;
 	__u32 max_rto = 0;
 
 	/* We are in recovery-phase on the MPTCP-level. Do not update the
@@ -1176,7 +1179,9 @@ static inline void mptcp_set_rto(struct sock *sk)
 	if (micsk->icsk_retransmits)
 		return;
 
-	mptcp_for_each_sk(tp->mpcb, sk_it) {
+	mptcp_for_each_sub(tp->mpcb, mptcp) {
+		struct sock *sk_it = mptcp_to_sock(mptcp);
+
 		if ((mptcp_sk_can_send(sk_it) || sk->sk_state == TCP_SYN_RECV) &&
 		    inet_csk(sk_it)->icsk_rto > max_rto)
 			max_rto = inet_csk(sk_it)->icsk_rto;
@@ -1265,10 +1270,10 @@ static inline bool mptcp_can_new_subflow(const struct sock *meta_sk)
 
 static inline int mptcp_subflow_count(const struct mptcp_cb *mpcb)
 {
-	struct sock *sk;
+	struct mptcp_tcp_sock *mptcp;
 	int i = 0;
 
-	mptcp_for_each_sk(mpcb, sk)
+	mptcp_for_each_sub(mpcb, mptcp)
 		i++;
 
 	return i;
@@ -1286,12 +1291,8 @@ bool mptcp_prune_ofo_queue(struct sock *sk);
 	do {				\
 	} while (0)
 
-/* Without MPTCP, we just do one iteration
- * over the only socket available. This assumes that
- * the sk/tp arg is the socket in that case.
- */
-#define mptcp_for_each_sk(mpcb, sk)
-#define mptcp_for_each_sk_safe(__mpcb, __sk, __temp)
+#define mptcp_for_each_sub(__mpcb, __mptcp)					\
+	if (0)
 
 #define MPTCP_INC_STATS(net, field)	\
 	do {				\
net/ipv4/af_inet.c +5 -4
index a58a8fbd6192..4b44aa87af2d 100644
@@ -744,13 +744,14 @@ int inet_accept(struct socket *sock, struct socket *newsock, int flags,
 	sock_rps_record_flow(sk2);
 
 	if (sk2->sk_protocol == IPPROTO_TCP && mptcp(tcp_sk(sk2))) {
-		struct sock *sk_it = sk2;
+		struct mptcp_tcp_sock *mptcp;
 
-		mptcp_for_each_sk(tcp_sk(sk2)->mpcb, sk_it)
-			sock_rps_record_flow(sk_it);
+		mptcp_for_each_sub(tcp_sk(sk2)->mpcb, mptcp) {
+			sock_rps_record_flow(mptcp_to_sock(mptcp));
+		}
 
 		if (tcp_sk(sk2)->mpcb->master_sk) {
-			sk_it = tcp_sk(sk2)->mpcb->master_sk;
+			struct sock *sk_it = tcp_sk(sk2)->mpcb->master_sk;
 
 			write_lock_bh(&sk_it->sk_callback_lock);
 			sk_it->sk_wq = newsock->wq;
net/ipv4/ip_sockglue.c +5 -2
index 6a41981c276e..eb72cf990876 100644
@@ -753,8 +753,11 @@ static int do_ip_setsockopt(struct sock *sk, int level,
 			sk_dst_reset(sk);
 			/* Update TOS on mptcp subflow */
 			if (is_meta_sk(sk)) {
-				struct sock *sk_it;
-				mptcp_for_each_sk(tcp_sk(sk)->mpcb, sk_it) {
+				struct mptcp_tcp_sock *mptcp;
+
+				mptcp_for_each_sub(tcp_sk(sk)->mpcb, mptcp) {
+					struct sock *sk_it = mptcp_to_sock(mptcp);
+
 					if (inet_sk(sk_it)->tos != inet_sk(sk)->tos) {
 						inet_sk(sk_it)->tos = inet_sk(sk)->tos;
 						sk_it->sk_priority = sk->sk_priority;
net/ipv4/tcp.c +19 -12
index eeb98619b181..ab17bc222490 100644
@@ -813,9 +813,11 @@ ssize_t tcp_splice_read(struct socket *sock, loff_t *ppos,
 
 #ifdef CONFIG_MPTCP
 	if (mptcp(tcp_sk(sk))) {
-		struct sock *sk_it;
-		mptcp_for_each_sk(tcp_sk(sk)->mpcb, sk_it)
-			sock_rps_record_flow(sk_it);
+		struct mptcp_tcp_sock *mptcp;
+
+		mptcp_for_each_sub(tcp_sk(sk)->mpcb, mptcp) {
+			sock_rps_record_flow(mptcp_to_sock(mptcp));
+		}
 	}
 #endif
 	/*
@@ -993,7 +995,7 @@ ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset,
 	}
 
 	if (mptcp(tp)) {
-		struct sock *sk_it = sk;
+		struct mptcp_tcp_sock *mptcp;
 
 		/* We must check this with socket-lock hold because we iterate
 		 * over the subflows.
@@ -1008,8 +1010,9 @@ ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset,
 			return ret;
 		}
 
-		mptcp_for_each_sk(tp->mpcb, sk_it)
-			sock_rps_record_flow(sk_it);
+		mptcp_for_each_sub(tp->mpcb, mptcp) {
+			sock_rps_record_flow(mptcp_to_sock(mptcp));
+		}
 	}
 
 	sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
@@ -1287,9 +1290,11 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size)
 	}
 
 	if (mptcp(tp)) {
-		struct sock *sk_it = sk;
-		mptcp_for_each_sk(tp->mpcb, sk_it)
-			sock_rps_record_flow(sk_it);
+		struct mptcp_tcp_sock *mptcp;
+
+		mptcp_for_each_sub(tp->mpcb, mptcp) {
+			sock_rps_record_flow(mptcp_to_sock(mptcp));
+		}
 	}
 
 	if (unlikely(tp->repair)) {
@@ -1994,9 +1999,11 @@ int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int nonblock,
 
 #ifdef CONFIG_MPTCP
 	if (mptcp(tp)) {
-		struct sock *sk_it;
-		mptcp_for_each_sk(tp->mpcb, sk_it)
-			sock_rps_record_flow(sk_it);
+		struct mptcp_tcp_sock *mptcp;
+
+		mptcp_for_each_sub(tp->mpcb, mptcp) {
+			sock_rps_record_flow(mptcp_to_sock(mptcp));
+		}
 	}
 #endif
 
net/mptcp/mptcp_ctrl.c +10 -4
index 052ae0e052b0..925d05d673d8 100644
@@ -553,10 +553,12 @@ void mptcp_hash_remove_bh(struct tcp_sock *meta_tp)
 struct sock *mptcp_select_ack_sock(const struct sock *meta_sk)
 {
 	const struct tcp_sock *meta_tp = tcp_sk(meta_sk);
-	struct sock *sk, *rttsk = NULL, *lastsk = NULL;
+	struct sock *rttsk = NULL, *lastsk = NULL;
 	u32 min_time = 0, last_active = 0;
+	struct mptcp_tcp_sock *mptcp;
 
-	mptcp_for_each_sk(meta_tp->mpcb, sk) {
+	mptcp_for_each_sub(meta_tp->mpcb, mptcp) {
+		struct sock *sk = mptcp_to_sock(mptcp);
 		struct tcp_sock *tp = tcp_sk(sk);
 		u32 elapsed;
 
@@ -689,7 +691,8 @@ void mptcp_sock_destruct(struct sock *sk)
 void mptcp_destroy_sock(struct sock *sk)
 {
 	if (is_meta_sk(sk)) {
-		struct sock *sk_it, *tmpsk;
+		struct mptcp_tcp_sock *mptcp;
+		struct hlist_node *tmp;
 
 		__skb_queue_purge(&tcp_sk(sk)->mpcb->reinject_queue);
 
@@ -699,7 +702,9 @@ void mptcp_destroy_sock(struct sock *sk)
 		 * not have been closed properly (as we are waiting for the
 		 * DATA_ACK of the DATA_FIN).
 		 */
-		mptcp_for_each_sk_safe(tcp_sk(sk)->mpcb, sk_it, tmpsk) {
+		mptcp_for_each_sub_safe(tcp_sk(sk)->mpcb, mptcp, tmp) {
+			struct sock *sk_it = mptcp_to_sock(mptcp);
+
 			/* Already did call tcp_close - waiting for graceful
 			 * closure, or if we are retransmitting fast-close on
 			 * the subflow. The reset (or timeout) will kill the
@@ -1281,6 +1286,7 @@ static int mptcp_alloc_mpcb(struct sock *meta_sk, __u64 remote_key,
 	INIT_LIST_HEAD(&mpcb->tw_list);
 
 	INIT_HLIST_HEAD(&mpcb->callback_list);
+	INIT_HLIST_HEAD(&mpcb->conn_list);
 	spin_lock_init(&mpcb->mpcb_list_lock);
 
 	mptcp_mpcb_inherit_sockopts(meta_sk, master_sk);

net/mptcp/mptcp_ctrl.c int mptcp_add_sock(struct sock *meta_sk, struct sock *sk, u8 loc_id, u8 rem_id, · line 1383

Quoted 10 lines
net/mptcp/mptcp_ctrl.c +5 -2
@@ -1370,8 +1376,12 @@ int mptcp_add_sock(struct sock *meta_sk, struct sock *sk, u8 loc_id, u8 rem_id,
 	sock_hold(meta_sk);
 	refcount_inc(&mpcb->mpcb_refcnt);
 
-	tp->mptcp->next = mpcb->connection_list;
-	mpcb->connection_list = tp;
+	local_bh_disable();
+	spin_lock(&mpcb->mpcb_list_lock);
+	hlist_add_head_rcu(&tp->mptcp->node, &mpcb->conn_list);
+	spin_unlock(&mpcb->mpcb_list_lock);
+	local_bh_enable();

I'm curious why the local_bh_disable is required here - I'm assuming it's possible to be called in softirq conext in this function but not where you delete from the list. A comment might help.

Quoted 28 lines
net/mptcp/mptcp_ctrl.c +2 -6
+
 	tp->mptcp->attached = 1;
 
 	atomic_add(atomic_read(&((struct sock *)tp)->sk_rmem_alloc),
@@ -1415,14 +1425,13 @@ int mptcp_add_sock(struct sock *meta_sk, struct sock *sk, u8 loc_id, u8 rem_id,
 
 void mptcp_del_sock(struct sock *sk)
 {
-	struct tcp_sock *tp = tcp_sk(sk), *tp_prev;
+	struct tcp_sock *tp = tcp_sk(sk);
 	struct mptcp_cb *mpcb;
 
 	if (!tp->mptcp || !tp->mptcp->attached)
 		return;
 
 	mpcb = tp->mpcb;
-	tp_prev = mpcb->connection_list;
 
 	if (mpcb->pm_ops->delete_subflow)
 		mpcb->pm_ops->delete_subflow(sk);
@@ -1431,17 +1440,10 @@ void mptcp_del_sock(struct sock *sk)
 		    __func__, mpcb->mptcp_loc_token, tp->mptcp->path_index,
 		    sk->sk_state, is_meta_sk(sk));
 
-	if (tp_prev == tp) {
-		mpcb->connection_list = tp->mptcp->next;
-	} else {
-		for (; tp_prev && tp_prev->mptcp->next; tp_prev = tp_prev->mptcp->next) {

net/mptcp/mptcp_ctrl.c void mptcp_del_sock(struct sock *sk) · line 1445

Quoted 10 lines
net/mptcp/mptcp_ctrl.c +3 -7
-			if (tp_prev->mptcp->next == tp) {
-				tp_prev->mptcp->next = tp->mptcp->next;
-				break;
-			}
-		}
-	}
-	tp->mptcp->next = NULL;
+	spin_lock(&mpcb->mpcb_list_lock);
+	hlist_del_init_rcu(&tp->mptcp->node);
+	spin_unlock(&mpcb->mpcb_list_lock);

Is it guaranteed that RCU synchronization happens between here and when the tcp_sock is freed? Do you need to set SOCK_RCU_FREE?

Mat

Quoted 579 lines
net/mptcp/mptcp_ctrl.c +35 -15
+
 	tp->mptcp->attached = 0;
 	mpcb->path_index_bits &= ~(1 << tp->mptcp->path_index);
 
@@ -1488,8 +1490,8 @@ void mptcp_update_metasocket(const struct sock *meta_sk)
 void mptcp_cleanup_rbuf(struct sock *meta_sk, int copied)
 {
 	struct tcp_sock *meta_tp = tcp_sk(meta_sk);
-	struct sock *sk;
 	bool recheck_rcv_window = false;
+	struct mptcp_tcp_sock *mptcp;
 	__u32 rcv_window_now = 0;
 
 	if (copied > 0 && !(meta_sk->sk_shutdown & RCV_SHUTDOWN)) {
@@ -1500,7 +1502,8 @@ void mptcp_cleanup_rbuf(struct sock *meta_sk, int copied)
 			recheck_rcv_window = true;
 	}
 
-	mptcp_for_each_sk(meta_tp->mpcb, sk) {
+	mptcp_for_each_sub(meta_tp->mpcb, mptcp) {
+		struct sock *sk = mptcp_to_sock(mptcp);
 		struct tcp_sock *tp = tcp_sk(sk);
 		const struct inet_connection_sock *icsk = inet_csk(sk);
 
@@ -1687,10 +1690,13 @@ EXPORT_SYMBOL(mptcp_sub_force_close);
 */
 void mptcp_update_sndbuf(const struct tcp_sock *tp)
 {
-	struct sock *meta_sk = tp->meta_sk, *sk;
+	struct sock *meta_sk = tp->meta_sk;
 	int new_sndbuf = 0, old_sndbuf = meta_sk->sk_sndbuf;
+	struct mptcp_tcp_sock *mptcp;
+
+	mptcp_for_each_sub(tp->mpcb, mptcp) {
+		struct sock *sk = mptcp_to_sock(mptcp);
 
-	mptcp_for_each_sk(tp->mpcb, sk) {
 		if (!mptcp_sk_can_send(sk))
 			continue;
 
@@ -1719,8 +1725,8 @@ void mptcp_update_sndbuf(const struct tcp_sock *tp)
 void mptcp_close(struct sock *meta_sk, long timeout)
 {
 	struct tcp_sock *meta_tp = tcp_sk(meta_sk);
-	struct sock *sk_it, *tmpsk;
 	struct mptcp_cb *mpcb = meta_tp->mpcb;
+	struct mptcp_tcp_sock *mptcp;
 	struct sk_buff *skb;
 	int data_was_unread = 0;
 	int state;
@@ -1753,7 +1759,12 @@ void mptcp_close(struct sock *meta_sk, long timeout)
 
 	/* If socket has been already reset (e.g. in tcp_reset()) - kill it. */
 	if (meta_sk->sk_state == TCP_CLOSE) {
-		mptcp_for_each_sk_safe(mpcb, sk_it, tmpsk) {
+		struct mptcp_tcp_sock *mptcp;
+		struct hlist_node *tmp;
+
+		mptcp_for_each_sub_safe(mpcb, mptcp, tmp) {
+			struct sock *sk_it = mptcp_to_sock(mptcp);
+
 			if (tcp_sk(sk_it)->send_mp_fclose)
 				continue;
 			mptcp_sub_close(sk_it, 0);
@@ -1774,10 +1785,14 @@ void mptcp_close(struct sock *meta_sk, long timeout)
 	} else if (tcp_close_state(meta_sk)) {
 		mptcp_send_fin(meta_sk);
 	} else if (meta_tp->snd_una == meta_tp->write_seq) {
+		struct mptcp_tcp_sock *mptcp;
+		struct hlist_node *tmp;
+
 		/* The DATA_FIN has been sent and acknowledged
 		 * (e.g., by sk_shutdown). Close all the other subflows
 		 */
-		mptcp_for_each_sk_safe(mpcb, sk_it, tmpsk) {
+		mptcp_for_each_sub_safe(mpcb, mptcp, tmp) {
+			struct sock *sk_it = mptcp_to_sock(mptcp);
 			unsigned long delay = 0;
 			/* If we are the passive closer, don't trigger
 			 * subflow-fin until the subflow has been finned
@@ -1801,7 +1816,9 @@ void mptcp_close(struct sock *meta_sk, long timeout)
 	/* socket will be freed after mptcp_close - we have to prevent
 	 * access from the subflows.
 	 */
-	mptcp_for_each_sk(mpcb, sk_it) {
+	mptcp_for_each_sub(mpcb, mptcp) {
+		struct sock *sk_it = mptcp_to_sock(mptcp);
+
 		/* Similar to sock_orphan, but we don't set it DEAD, because
 		 * the callbacks are still set and must be called.
 		 */
@@ -1886,8 +1903,9 @@ void mptcp_close(struct sock *meta_sk, long timeout)
 
 void mptcp_disconnect(struct sock *sk)
 {
-	struct sock *subsk, *tmpsk;
+	struct mptcp_tcp_sock *mptcp;
 	struct tcp_sock *tp = tcp_sk(sk);
+	struct hlist_node *tmp;
 
 	__skb_queue_purge(&tp->mpcb->reinject_queue);
 
@@ -1895,7 +1913,9 @@ void mptcp_disconnect(struct sock *sk)
 		mptcp_hash_remove_bh(tp);
 
 	local_bh_disable();
-	mptcp_for_each_sk_safe(tp->mpcb, subsk, tmpsk) {
+	mptcp_for_each_sub_safe(tp->mpcb, mptcp, tmp) {
+		struct sock *subsk = mptcp_to_sock(mptcp);
+
 		/* The socket will get removed from the subsocket-list
 		 * and made non-mptcp by setting mpc to 0.
 		 *
@@ -2579,7 +2599,6 @@ static void mptcp_get_sub_info(struct sock *sk, struct mptcp_sub_info *info)
 int mptcp_get_info(const struct sock *meta_sk, char __user *optval, int optlen)
 {
 	const struct tcp_sock *meta_tp = tcp_sk(meta_sk);
-	struct sock *sk;
 
 	struct mptcp_meta_info meta_info;
 	struct mptcp_info m_info;
@@ -2625,16 +2644,17 @@ int mptcp_get_info(const struct sock *meta_sk, char __user *optval, int optlen)
 
 	if (m_info.subflows) {
 		unsigned int len, sub_len = 0;
+		struct mptcp_tcp_sock *mptcp;
 		char __user *ptr;
 
 		ptr = (char __user *)m_info.subflows;
 		len = m_info.sub_len;
 
-		mptcp_for_each_sk(meta_tp->mpcb, sk) {
+		mptcp_for_each_sub(meta_tp->mpcb, mptcp) {
 			struct tcp_info t_info;
 			unsigned int tmp_len;
 
-			tcp_get_info(sk, &t_info);
+			tcp_get_info(mptcp_to_sock(mptcp), &t_info);
 
 			tmp_len = min_t(unsigned int, len, info_len);
 			len -= tmp_len;
@@ -2654,6 +2674,7 @@ int mptcp_get_info(const struct sock *meta_sk, char __user *optval, int optlen)
 
 	if (m_info.subflow_info) {
 		unsigned int len, sub_info_len, total_sub_info_len = 0;
+		struct mptcp_tcp_sock *mptcp;
 		char __user *ptr;
 
 		ptr = (char __user *)m_info.subflow_info;
@@ -2663,11 +2684,11 @@ int mptcp_get_info(const struct sock *meta_sk, char __user *optval, int optlen)
 				     sizeof(struct mptcp_sub_info));
 		m_info.sub_info_len = sub_info_len;
 
-		mptcp_for_each_sk(meta_tp->mpcb, sk) {
+		mptcp_for_each_sub(meta_tp->mpcb, mptcp) {
 			struct mptcp_sub_info m_sub_info;
 			unsigned int tmp_len;
 
-			mptcp_get_sub_info(sk, &m_sub_info);
+			mptcp_get_sub_info(mptcp_to_sock(mptcp), &m_sub_info);
 
 			tmp_len = min_t(unsigned int, len, sub_info_len);
 			len -= tmp_len;
net/mptcp/mptcp_fullmesh.c +20 -8
index d29102a1dffb..53c1ba7319f5 100644
@@ -903,8 +903,9 @@ static void mptcp_address_worker(struct work_struct *work)
 			}
 
 			if (event->code == MPTCP_EVENT_DEL) {
-				struct sock *sk, *tmpsk;
+				struct mptcp_tcp_sock *mptcp;
 				struct mptcp_loc_addr *mptcp_local;
+				struct hlist_node *tmp;
 				bool found = false;
 
 				mptcp_local = rcu_dereference_bh(fm_ns->local);
@@ -914,7 +915,9 @@ static void mptcp_address_worker(struct work_struct *work)
 					update_addr_bitfields(meta_sk, mptcp_local);
 
 				/* Look for the socket and remove him */
-				mptcp_for_each_sk_safe(mpcb, sk, tmpsk) {
+				mptcp_for_each_sub_safe(mpcb, mptcp, tmp) {
+					struct sock *sk = mptcp_to_sock(mptcp);
+
 					if ((event->family == AF_INET6 &&
 					     (sk->sk_family == AF_INET ||
 					      mptcp_v6_is_v4_mapped(sk))) ||
@@ -964,9 +967,10 @@ static void mptcp_address_worker(struct work_struct *work)
 			}
 
 			if (event->code == MPTCP_EVENT_MOD) {
-				struct sock *sk;
+				struct mptcp_tcp_sock *mptcp;
 
-				mptcp_for_each_sk(mpcb, sk) {
+				mptcp_for_each_sub(mpcb, mptcp) {
+					struct sock *sk = mptcp_to_sock(mptcp);
 					struct tcp_sock *tp = tcp_sk(sk);
 					if (event->family == AF_INET &&
 					    (sk->sk_family == AF_INET ||
@@ -1455,8 +1459,9 @@ static void full_mesh_release_sock(struct sock *meta_sk)
 	struct mptcp_cb *mpcb = tcp_sk(meta_sk)->mpcb;
 	struct fullmesh_priv *fmp = fullmesh_get_priv(mpcb);
 	const struct mptcp_fm_ns *fm_ns = fm_get_ns(sock_net(meta_sk));
-	struct sock *sk, *tmpsk;
 	bool meta_v4 = meta_sk->sk_family == AF_INET;
+	struct mptcp_tcp_sock *mptcp;
+	struct hlist_node *tmp;
 	int i;
 
 	rcu_read_lock_bh();
@@ -1470,7 +1475,8 @@ static void full_mesh_release_sock(struct sock *meta_sk)
 		struct in_addr ifa = mptcp_local->locaddr4[i].addr;
 		bool found = false;
 
-		mptcp_for_each_sk(mpcb, sk) {
+		mptcp_for_each_sub(mpcb, mptcp) {
+			struct sock *sk = mptcp_to_sock(mptcp);
 			struct tcp_sock *tp = tcp_sk(sk);
 
 			if (sk->sk_family == AF_INET6 &&
@@ -1491,6 +1497,8 @@ static void full_mesh_release_sock(struct sock *meta_sk)
 		}
 
 		if (!found) {
+			struct sock *sk;
+
 			fmp->add_addr++;
 			mpcb->addr_signal = 1;
 
@@ -1511,7 +1519,8 @@ static void full_mesh_release_sock(struct sock *meta_sk)
 		struct in6_addr ifa = mptcp_local->locaddr6[i].addr;
 		bool found = false;
 
-		mptcp_for_each_sk(mpcb, sk) {
+		mptcp_for_each_sub(mpcb, mptcp) {
+			struct sock *sk = mptcp_to_sock(mptcp);
 			struct tcp_sock *tp = tcp_sk(sk);
 
 			if (sk->sk_family == AF_INET ||
@@ -1532,6 +1541,8 @@ static void full_mesh_release_sock(struct sock *meta_sk)
 		}
 
 		if (!found) {
+			struct sock *sk;
+
 			fmp->add_addr++;
 			mpcb->addr_signal = 1;
 
@@ -1546,7 +1557,8 @@ static void full_mesh_release_sock(struct sock *meta_sk)
 #endif
 
 	/* Now, detect address-removals */
-	mptcp_for_each_sk_safe(mpcb, sk, tmpsk) {
+	mptcp_for_each_sub_safe(mpcb, mptcp, tmp) {
+		struct sock *sk = mptcp_to_sock(mptcp);
 		bool shall_remove = true;
 
 		if (sk->sk_family == AF_INET || mptcp_v6_is_v4_mapped(sk)) {
net/mptcp/mptcp_input.c +29 -15
index 211605347f70..64b0265bfd06 100644
@@ -126,12 +126,14 @@ static void mptcp_clean_rtx_queue(struct sock *meta_sk, u32 prior_snd_una)
 		tcp_rtx_queue_unlink(skb, meta_sk);
 
 		if (mptcp_is_data_fin(skb)) {
-			struct sock *sk_it, *sk_tmp;
+			struct mptcp_tcp_sock *mptcp;
+			struct hlist_node *tmp;
 
 			/* DATA_FIN has been acknowledged - now we can close
 			 * the subflows
 			 */
-			mptcp_for_each_sk_safe(mpcb, sk_it, sk_tmp) {
+			mptcp_for_each_sub_safe(mpcb, mptcp, tmp) {
+				struct sock *sk_it = mptcp_to_sock(mptcp);
 				unsigned long delay = 0;
 
 				/* If we are the passive closer, don't trigger
@@ -347,6 +349,7 @@ static int mptcp_verif_dss_csum(struct sock *sk)
 
 	/* Now, checksum must be 0 */
 	if (unlikely(csum_fold(csum_tcp))) {
+		struct mptcp_tcp_sock *mptcp;
 		struct sock *sk_it = NULL;
 
 		pr_err("%s csum is wrong: %#x tcp-seq %u dss_csum_added %d overflowed %d iterations %d\n",
@@ -362,7 +365,9 @@ static int mptcp_verif_dss_csum(struct sock *sk)
 		tp->mpcb->csum_cutoff_seq = tp->mptcp->map_data_seq;
 
 		/* Search for another subflow that is fully established */
-		mptcp_for_each_sk(tp->mpcb, sk_it) {
+		mptcp_for_each_sub(tp->mpcb, mptcp) {
+			sk_it = mptcp_to_sock(mptcp);
+
 			if (sk_it != sk &&
 			    tcp_sk(sk_it)->mptcp->fully_established)
 				break;
@@ -1308,12 +1313,15 @@ int mptcp_do_join_short(struct sk_buff *skb,
 */
 void mptcp_fin(struct sock *meta_sk)
 {
-	struct sock *sk = NULL, *sk_it;
+	struct sock *sk = NULL;
 	struct tcp_sock *meta_tp = tcp_sk(meta_sk);
 	struct mptcp_cb *mpcb = meta_tp->mpcb;
+	struct mptcp_tcp_sock *mptcp;
 	unsigned char state;
 
-	mptcp_for_each_sk(mpcb, sk_it) {
+	mptcp_for_each_sub(mpcb, mptcp) {
+		struct sock *sk_it = mptcp_to_sock(mptcp);
+
 		if (tcp_sk(sk_it)->mptcp->path_index == mpcb->dfin_path_index) {
 			sk = sk_it;
 			break;
@@ -1585,9 +1593,12 @@ void mptcp_clean_rtx_infinite(const struct sk_buff *skb, struct sock *sk)
 
 static void mptcp_send_reset_rem_id(const struct mptcp_cb *mpcb, u8 rem_id)
 {
-	struct sock *sk_it, *tmpsk;
+	struct mptcp_tcp_sock *mptcp;
+	struct hlist_node *tmp;
+
+	mptcp_for_each_sub_safe(mpcb, mptcp, tmp) {
+		struct sock *sk_it = mptcp_to_sock(mptcp);
 
-	mptcp_for_each_sk_safe(mpcb, sk_it, tmpsk) {
 		if (tcp_sk(sk_it)->mptcp->rem_id == rem_id) {
 			mptcp_reinject_data(sk_it, 0);
 			mptcp_send_reset(sk_it);
@@ -1892,13 +1903,15 @@ void tcp_parse_mptcp_options(const struct sk_buff *skb,
 bool mptcp_check_rtt(const struct tcp_sock *tp, int time)
 {
 	struct mptcp_cb *mpcb = tp->mpcb;
-	struct sock *sk;
+	struct mptcp_tcp_sock *mptcp;
 	u32 rtt_max = 0;
 
 	/* In MPTCP, we take the max delay across all flows,
 	 * in order to take into account meta-reordering buffers.
 	 */
-	mptcp_for_each_sk(mpcb, sk) {
+	mptcp_for_each_sub(mpcb, mptcp) {
+		struct sock *sk = mptcp_to_sock(mptcp);
+
 		if (!mptcp_sk_can_recv(sk))
 			continue;
 
@@ -2173,9 +2186,9 @@ bool mptcp_handle_options(struct sock *sk, const struct tcphdr *th,
 		if (mopt->saw_low_prio == 1) {
 			tp->mptcp->rcv_low_prio = mopt->low_prio;
 		} else {
-			struct sock *sk_it;
-			mptcp_for_each_sk(tp->mpcb, sk_it) {
-				struct mptcp_tcp_sock *mptcp = tcp_sk(sk_it)->mptcp;
+			struct mptcp_tcp_sock *mptcp;
+
+			mptcp_for_each_sub(tp->mpcb, mptcp) {
 				if (mptcp->rem_id == mopt->prio_addr_id)
 					mptcp->rcv_low_prio = mopt->low_prio;
 			}
@@ -2359,7 +2372,7 @@ bool mptcp_should_expand_sndbuf(const struct sock *sk)
 {
 	const struct sock *meta_sk = mptcp_meta_sk(sk);
 	const struct tcp_sock *meta_tp = tcp_sk(meta_sk);
-	const struct sock *sk_it;
+	const struct mptcp_tcp_sock *mptcp;
 
 	/* We circumvent this check in tcp_check_space, because we want to
 	 * always call sk_write_space. So, we reproduce the check here.
@@ -2385,8 +2398,9 @@ bool mptcp_should_expand_sndbuf(const struct sock *sk)
 	/* For MPTCP we look for a subsocket that could send data.
 	 * If we found one, then we update the send-buffer.
 	 */
-	mptcp_for_each_sk(meta_tp->mpcb, sk_it) {
-		struct tcp_sock *tp_it = tcp_sk(sk_it);
+	mptcp_for_each_sub(meta_tp->mpcb, mptcp) {
+		const struct sock *sk_it = mptcp_to_sock(mptcp);
+		const struct tcp_sock *tp_it = tcp_sk(sk_it);
 
 		if (!mptcp_sk_can_send(sk_it))
 			continue;
net/mptcp/mptcp_output.c +2 -1
index 4b88b32d5954..12733ba2c1e1 100644
@@ -644,7 +644,6 @@ int mptcp_write_wakeup(struct sock *meta_sk, int mib)
 {
 	struct tcp_sock *meta_tp = tcp_sk(meta_sk);
 	struct sk_buff *skb;
-	struct sock *sk_it;
 	int ans = 0;
 
 	if (meta_sk->sk_state == TCP_CLOSE)
@@ -701,17 +700,22 @@ int mptcp_write_wakeup(struct sock *meta_sk, int mib)
 
 		return 0;
 	} else {
+		struct mptcp_tcp_sock *mptcp;
+

window_probe:

		if (between(meta_tp->snd_up, meta_tp->snd_una + 1,
			    meta_tp->snd_una + 0xFFFF)) {
net/mptcp/mptcp_output.c +3 -1
-			mptcp_for_each_sk(meta_tp->mpcb, sk_it) {
+			mptcp_for_each_sub(meta_tp->mpcb, mptcp) {
+				struct sock *sk_it = mptcp_to_sock(mptcp);
+
				if (mptcp_sk_can_send_ack(sk_it))
					tcp_xmit_probe_skb(sk_it, 1, mib);
			}
		}

		/* At least one of the tcp_xmit_probe_skb's has to succeed */
net/mptcp/mptcp_output.c +2 -1
-		mptcp_for_each_sk(meta_tp->mpcb, sk_it) {
+		mptcp_for_each_sub(meta_tp->mpcb, mptcp) {
+			struct sock *sk_it = mptcp_to_sock(mptcp);
			int ret;

			if (!mptcp_sk_can_send_ack(sk_it))
net/mptcp/mptcp_output.c +18 -10
@@ -729,6 +733,7 @@ bool mptcp_write_xmit(struct sock *meta_sk, unsigned int mss_now, int nonagle,
 		     int push_one, gfp_t gfp)
 {
 	struct tcp_sock *meta_tp = tcp_sk(meta_sk), *subtp;
+	struct mptcp_tcp_sock *mptcp;
 	struct sock *subsk = NULL;
 	struct mptcp_cb *mpcb = meta_tp->mpcb;
 	struct sk_buff *skb;
@@ -853,7 +858,8 @@ bool mptcp_write_xmit(struct sock *meta_sk, unsigned int mss_now, int nonagle,
 			break;
 	}
 
-	mptcp_for_each_sk(mpcb, subsk) {
+	mptcp_for_each_sub(mpcb, mptcp) {
+		subsk = mptcp_to_sock(mptcp);
 		subtp = tcp_sk(subsk);
 
 		if (!(path_mask & mptcp_pi_to_flag(subtp->mptcp->path_index)))
@@ -1350,7 +1356,7 @@ void mptcp_send_active_reset(struct sock *meta_sk, gfp_t priority)
 	struct mptcp_cb *mpcb = meta_tp->mpcb;
 	struct sock *sk;
 
-	if (!mpcb->connection_list)
+	if (hlist_empty(&mpcb->conn_list))
 		return;
 
 	WARN_ON(meta_tp->send_mp_fclose);
@@ -1725,10 +1731,11 @@ void mptcp_select_initial_window(const struct sock *sk, int __space, __u32 mss,
 static inline u64 mptcp_calc_rate(const struct sock *meta_sk, unsigned int mss,
 				  unsigned int (*mss_cb)(struct sock *sk))
 {
-	struct sock *sk;
+	struct mptcp_tcp_sock *mptcp;
 	u64 rate = 0;
 
-	mptcp_for_each_sk(tcp_sk(meta_sk)->mpcb, sk) {
+	mptcp_for_each_sub(tcp_sk(meta_sk)->mpcb, mptcp) {
+		struct sock *sk = mptcp_to_sock(mptcp);
 		struct tcp_sock *tp = tcp_sk(sk);
 		int this_mss;
 		u64 this_rate;
@@ -1780,11 +1787,12 @@ static inline u64 mptcp_calc_rate(const struct sock *meta_sk, unsigned int mss,
 static unsigned int __mptcp_current_mss(const struct sock *meta_sk,
 					unsigned int (*mss_cb)(struct sock *sk))
 {
+	struct mptcp_tcp_sock *mptcp;
 	unsigned int mss = 0;
 	u64 rate = 0;
-	struct sock *sk;
 
-	mptcp_for_each_sk(tcp_sk(meta_sk)->mpcb, sk) {
+	mptcp_for_each_sub(tcp_sk(meta_sk)->mpcb, mptcp) {
+		struct sock *sk = mptcp_to_sock(mptcp);
 		int this_mss;
 		u64 this_rate;
 
@@ -1849,14 +1857,16 @@ int mptcp_select_size(const struct sock *meta_sk, bool first_skb, bool zc)
 
 int mptcp_check_snd_buf(const struct tcp_sock *tp)
 {
-	const struct sock *sk;
+	const struct mptcp_tcp_sock *mptcp;
 	u32 rtt_max = tp->srtt_us;
 	u64 bw_est;
 
 	if (!tp->srtt_us)
 		return tp->reordering + 1;
 
-	mptcp_for_each_sk(tp->mpcb, sk) {
+	mptcp_for_each_sub(tp->mpcb, mptcp) {
+		const struct sock *sk = mptcp_to_sock(mptcp);
+
 		if (!mptcp_sk_can_send(sk))
 			continue;
 
@@ -1874,11 +1884,13 @@ int mptcp_check_snd_buf(const struct tcp_sock *tp)
 unsigned int mptcp_xmit_size_goal(const struct sock *meta_sk, u32 mss_now,
 				  int large_allowed)
 {
-	struct sock *sk;
 	u32 xmit_size_goal = 0;
 
 	if (large_allowed && !tcp_sk(meta_sk)->mpcb->dss_csum) {
-		mptcp_for_each_sk(tcp_sk(meta_sk)->mpcb, sk) {
+		struct mptcp_tcp_sock *mptcp;
+
+		mptcp_for_each_sub(tcp_sk(meta_sk)->mpcb, mptcp) {
+			struct sock *sk = mptcp_to_sock(mptcp);
 			int this_size_goal;
 
 			if (!mptcp_sk_can_send(sk))
net/mptcp/mptcp_sched.c +15 -6
index a2543c60bd31..b440df7aae71 100644
@@ -135,9 +135,10 @@ static struct sock
 	u32 min_srtt = 0xffffffff;
 	bool found_unused = false;
 	bool found_unused_una = false;
-	struct sock *sk;
+	struct mptcp_tcp_sock *mptcp;
 
-	mptcp_for_each_sk(mpcb, sk) {
+	mptcp_for_each_sub(mpcb, mptcp) {
+		struct sock *sk = mptcp_to_sock(mptcp);
 		struct tcp_sock *tp = tcp_sk(sk);
 		bool unused = false;
 
@@ -219,7 +220,11 @@ static struct sock *get_available_subflow(struct sock *meta_sk,
 	/* Answer data_fin on same subflow!!! */
 	if (meta_sk->sk_shutdown & RCV_SHUTDOWN &&
 	    skb && mptcp_is_data_fin(skb)) {
-		mptcp_for_each_sk(mpcb, sk) {
+		struct mptcp_tcp_sock *mptcp;
+
+		mptcp_for_each_sub(mpcb, mptcp) {
+			sk = mptcp_to_sock(mptcp);
+
 			if (tcp_sk(sk)->mptcp->path_index == mpcb->dfin_path_index &&
 			    mptcp_is_available(sk, skb, zero_wnd_test))
 				return sk;
@@ -252,7 +257,7 @@ static struct sk_buff *mptcp_rcv_buf_optimization(struct sock *sk, int penal)
 {
 	struct sock *meta_sk;
 	const struct tcp_sock *tp = tcp_sk(sk);
-	struct tcp_sock *tp_it;
+	struct mptcp_tcp_sock *mptcp;
 	struct sk_buff *skb_head;
 	struct defsched_priv *dsp = defsched_get_priv(tp);
 
@@ -275,7 +280,9 @@ static struct sk_buff *mptcp_rcv_buf_optimization(struct sock *sk, int penal)
 		goto retrans;
 
 	/* Half the cwnd of the slow flow */
-	mptcp_for_each_tp(tp->mpcb, tp_it) {
+	mptcp_for_each_sub(tp->mpcb, mptcp) {
+		struct tcp_sock *tp_it = mptcp->tp;
+
 		if (tp_it != tp &&
 		    TCP_SKB_CB(skb_head)->path_mask & mptcp_pi_to_flag(tp_it->mptcp->path_index)) {
 			if (tp->srtt_us < tp_it->srtt_us && inet_csk((struct sock *)tp_it)->icsk_ca_state == TCP_CA_Open) {
@@ -298,7 +305,9 @@ static struct sk_buff *mptcp_rcv_buf_optimization(struct sock *sk, int penal)
 	/* Segment not yet injected into this path? Take it!!! */
 	if (!(TCP_SKB_CB(skb_head)->path_mask & mptcp_pi_to_flag(tp->mptcp->path_index))) {
 		bool do_retrans = false;
-		mptcp_for_each_tp(tp->mpcb, tp_it) {
+		mptcp_for_each_sub(tp->mpcb, mptcp) {
+			struct tcp_sock *tp_it = mptcp->tp;
+
 			if (tp_it != tp &&
 			    TCP_SKB_CB(skb_head)->path_mask & mptcp_pi_to_flag(tp_it->mptcp->path_index)) {
 				if (tp_it->snd_cwnd <= 4) {
Signature · 1 line
2.16.2

--
Mat Martineau
Intel OTC

author
#12 lore raw

Re: [MPTCP] [PATCH mptcp-net-next 10/14] mptcp: Make subflow-list an RCU-list

Hello Mat,

Quoted 358 lines · Mat Martineau

Christoph -

Quoted 353 lines · Christoph Paasch

mptcp_add_sock() will now be called without holding the meta-level lock. However, mptcp_add_sock() wants to add the subflow to the meta-level list, thus we need to protect this by a lock. We use the mpcb_list_lock for that.

Now that we are locking during add/del, and want to allow lockless traversal of the list, this implies that we need to make it an RCU-list.

So, this patch transitions to the RCU hlist. The list-traversal macros (hlist_for_each_entry_rcu) require me to now pass the mptcp_tcp_sock to mptcp_for_each_*. So, I had to change all the places in the code where we call one of the list-traversal macros to adapt to this.

Signed-off-by Christoph Paasch <cpaasch(a)apple.com>
Signed-off-by Matthieu Baerts <matthieu.baerts(a)tessares.net>
(cherry picked from commit 7a662b690069642c138da16ff9396e6826fd0a95)

---

include/net/mptcp.h        | 67 +++++++++++++++++-----------------
net/ipv4/af_inet.c         |  9 ++---
net/ipv4/ip_sockglue.c     |  7 ++--
net/ipv4/tcp.c             | 31 +++++++++-------
net/mptcp/mptcp_ctrl.c     | 89 ++++++++++++++++++++++++++++------------------
net/mptcp/mptcp_fullmesh.c | 28 ++++++++++-----
net/mptcp/mptcp_input.c    | 44 +++++++++++++++--------
net/mptcp/mptcp_output.c   | 38 +++++++++++++-------
net/mptcp/mptcp_sched.c    | 21 +++++++----
9 files changed, 207 insertions(+), 127 deletions(-)
include/net/mptcp.h +34 -33
index 0f9d47b45895..091e56a7b334 100644
@@ -156,7 +156,7 @@ struct mptcp_options_received {
 };
 
 struct mptcp_tcp_sock {
-	struct tcp_sock	*next;		/* Next subflow socket */
+	struct hlist_node node;
 	struct hlist_node cb_list;
 	struct mptcp_options_received rx_opt;
 
@@ -254,7 +254,7 @@ struct mptcp_sched_ops {
 
 struct mptcp_cb {
 	/* list of sockets in this multipath connection */
-	struct tcp_sock *connection_list;
+	struct hlist_head conn_list;
 	/* list of sockets that need a call to release_cb */
 	struct hlist_head callback_list;
 
@@ -309,7 +309,7 @@ struct mptcp_cb {
 	/***** Start of fields, used for subflow establishment */
 	struct sock *meta_sk;
 
-	/* Master socket, also part of the connection_list, this
+	/* Master socket, also part of the conn_list, this
 	 * socket is the one that the application sees.
 	 */
 	struct sock *master_sk;
@@ -661,21 +661,17 @@ extern struct workqueue_struct *mptcp_wq;
 			pr_err(fmt, ##args);					\
 	} while (0)
 
-/* Iterates over all subflows */
-#define mptcp_for_each_tp(mpcb, tp)					\
-	for ((tp) = (mpcb)->connection_list; (tp); (tp) = (tp)->mptcp->next)
+static inline struct sock *mptcp_to_sock(const struct mptcp_tcp_sock *mptcp)
+{
+	return (struct sock *)mptcp->tp;
+}
 
-#define mptcp_for_each_sk(mpcb, sk)					\
-	for ((sk) = (struct sock *)(mpcb)->connection_list;		\
-	     sk;							\
-	     sk = (struct sock *)tcp_sk(sk)->mptcp->next)
+#define mptcp_for_each_sub(__mpcb, __mptcp)					\
+	hlist_for_each_entry_rcu(__mptcp, &((__mpcb)->conn_list), node)
 
-#define mptcp_for_each_sk_safe(__mpcb, __sk, __temp)			\
-	for (__sk = (struct sock *)(__mpcb)->connection_list,		\
-	     __temp = __sk ? (struct sock *)tcp_sk(__sk)->mptcp->next : NULL; \
-	     __sk;							\
-	     __sk = __temp,						\
-	     __temp = __sk ? (struct sock *)tcp_sk(__sk)->mptcp->next : NULL)
+/* Must be called with the appropriate lock held */
+#define mptcp_for_each_sub_safe(__mpcb, __mptcp, __tmp)				\
+	hlist_for_each_entry_safe(__mptcp, __tmp, &((__mpcb)->conn_list), node)
 
 /* Iterates over all bit set to 1 in a bitset */
 #define mptcp_for_each_bit_set(b, i)					\
@@ -922,12 +918,14 @@ struct request_sock *rev_mptcp_rsk(const struct mptcp_request_sock *req)
 
 static inline bool mptcp_can_sendpage(struct sock *sk)
 {
-	struct sock *sk_it;
+	struct mptcp_tcp_sock *mptcp;
 
 	if (tcp_sk(sk)->mpcb->dss_csum)
 		return false;
 
-	mptcp_for_each_sk(tcp_sk(sk)->mpcb, sk_it) {
+	mptcp_for_each_sub(tcp_sk(sk)->mpcb, mptcp) {
+		struct sock *sk_it = mptcp_to_sock(mptcp);
+
 		if (!(sk_it->sk_route_caps & NETIF_F_SG))
 			return false;
 	}
@@ -961,9 +959,12 @@ static inline void mptcp_send_reset(struct sock *sk)
 static inline void mptcp_sub_force_close_all(struct mptcp_cb *mpcb,
 					     struct sock *except)
 {
-	struct sock *sk_it, *tmp;
+	struct mptcp_tcp_sock *mptcp;
+	struct hlist_node *tmp;
+
+	mptcp_for_each_sub_safe(mpcb, mptcp, tmp) {
+		struct sock *sk_it = mptcp_to_sock(mptcp);
 
-	mptcp_for_each_sk_safe(mpcb, sk_it, tmp) {
 		if (sk_it != except)
 			mptcp_send_reset(sk_it);
 	}
@@ -1149,12 +1150,14 @@ static inline int mptcp_sk_can_send_ack(const struct sock *sk)
 
 static inline bool mptcp_can_sg(const struct sock *meta_sk)
 {
-	struct sock *sk;
+	struct mptcp_tcp_sock *mptcp;
 
 	if (tcp_sk(meta_sk)->mpcb->dss_csum)
 		return false;
 
-	mptcp_for_each_sk(tcp_sk(meta_sk)->mpcb, sk) {
+	mptcp_for_each_sub(tcp_sk(meta_sk)->mpcb, mptcp) {
+		struct sock *sk = mptcp_to_sock(mptcp);
+
 		if (!mptcp_sk_can_send(sk))
 			continue;
 		if (!(sk->sk_route_caps & NETIF_F_SG))
@@ -1165,9 +1168,9 @@ static inline bool mptcp_can_sg(const struct sock *meta_sk)
 
 static inline void mptcp_set_rto(struct sock *sk)
 {
-	struct tcp_sock *tp = tcp_sk(sk);
-	struct sock *sk_it;
 	struct inet_connection_sock *micsk = inet_csk(mptcp_meta_sk(sk));
+	struct tcp_sock *tp = tcp_sk(sk);
+	struct mptcp_tcp_sock *mptcp;
 	__u32 max_rto = 0;
 
 	/* We are in recovery-phase on the MPTCP-level. Do not update the
@@ -1176,7 +1179,9 @@ static inline void mptcp_set_rto(struct sock *sk)
 	if (micsk->icsk_retransmits)
 		return;
 
-	mptcp_for_each_sk(tp->mpcb, sk_it) {
+	mptcp_for_each_sub(tp->mpcb, mptcp) {
+		struct sock *sk_it = mptcp_to_sock(mptcp);
+
 		if ((mptcp_sk_can_send(sk_it) || sk->sk_state == TCP_SYN_RECV) &&
 		    inet_csk(sk_it)->icsk_rto > max_rto)
 			max_rto = inet_csk(sk_it)->icsk_rto;
@@ -1265,10 +1270,10 @@ static inline bool mptcp_can_new_subflow(const struct sock *meta_sk)
 
 static inline int mptcp_subflow_count(const struct mptcp_cb *mpcb)
 {
-	struct sock *sk;
+	struct mptcp_tcp_sock *mptcp;
 	int i = 0;
 
-	mptcp_for_each_sk(mpcb, sk)
+	mptcp_for_each_sub(mpcb, mptcp)
 		i++;
 
 	return i;
@@ -1286,12 +1291,8 @@ bool mptcp_prune_ofo_queue(struct sock *sk);
 	do {				\
 	} while (0)
 
-/* Without MPTCP, we just do one iteration
- * over the only socket available. This assumes that
- * the sk/tp arg is the socket in that case.
- */
-#define mptcp_for_each_sk(mpcb, sk)
-#define mptcp_for_each_sk_safe(__mpcb, __sk, __temp)
+#define mptcp_for_each_sub(__mpcb, __mptcp)					\
+	if (0)
 
 #define MPTCP_INC_STATS(net, field)	\
 	do {				\
net/ipv4/af_inet.c +5 -4
index a58a8fbd6192..4b44aa87af2d 100644
@@ -744,13 +744,14 @@ int inet_accept(struct socket *sock, struct socket *newsock, int flags,
 	sock_rps_record_flow(sk2);
 
 	if (sk2->sk_protocol == IPPROTO_TCP && mptcp(tcp_sk(sk2))) {
-		struct sock *sk_it = sk2;
+		struct mptcp_tcp_sock *mptcp;
 
-		mptcp_for_each_sk(tcp_sk(sk2)->mpcb, sk_it)
-			sock_rps_record_flow(sk_it);
+		mptcp_for_each_sub(tcp_sk(sk2)->mpcb, mptcp) {
+			sock_rps_record_flow(mptcp_to_sock(mptcp));
+		}
 
 		if (tcp_sk(sk2)->mpcb->master_sk) {
-			sk_it = tcp_sk(sk2)->mpcb->master_sk;
+			struct sock *sk_it = tcp_sk(sk2)->mpcb->master_sk;
 
 			write_lock_bh(&sk_it->sk_callback_lock);
 			sk_it->sk_wq = newsock->wq;
net/ipv4/ip_sockglue.c +5 -2
index 6a41981c276e..eb72cf990876 100644
@@ -753,8 +753,11 @@ static int do_ip_setsockopt(struct sock *sk, int level,
 			sk_dst_reset(sk);
 			/* Update TOS on mptcp subflow */
 			if (is_meta_sk(sk)) {
-				struct sock *sk_it;
-				mptcp_for_each_sk(tcp_sk(sk)->mpcb, sk_it) {
+				struct mptcp_tcp_sock *mptcp;
+
+				mptcp_for_each_sub(tcp_sk(sk)->mpcb, mptcp) {
+					struct sock *sk_it = mptcp_to_sock(mptcp);
+
 					if (inet_sk(sk_it)->tos != inet_sk(sk)->tos) {
 						inet_sk(sk_it)->tos = inet_sk(sk)->tos;
 						sk_it->sk_priority = sk->sk_priority;
net/ipv4/tcp.c +19 -12
index eeb98619b181..ab17bc222490 100644
@@ -813,9 +813,11 @@ ssize_t tcp_splice_read(struct socket *sock, loff_t *ppos,
 
 #ifdef CONFIG_MPTCP
 	if (mptcp(tcp_sk(sk))) {
-		struct sock *sk_it;
-		mptcp_for_each_sk(tcp_sk(sk)->mpcb, sk_it)
-			sock_rps_record_flow(sk_it);
+		struct mptcp_tcp_sock *mptcp;
+
+		mptcp_for_each_sub(tcp_sk(sk)->mpcb, mptcp) {
+			sock_rps_record_flow(mptcp_to_sock(mptcp));
+		}
 	}
 #endif
 	/*
@@ -993,7 +995,7 @@ ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset,
 	}
 
 	if (mptcp(tp)) {
-		struct sock *sk_it = sk;
+		struct mptcp_tcp_sock *mptcp;
 
 		/* We must check this with socket-lock hold because we iterate
 		 * over the subflows.
@@ -1008,8 +1010,9 @@ ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset,
 			return ret;
 		}
 
-		mptcp_for_each_sk(tp->mpcb, sk_it)
-			sock_rps_record_flow(sk_it);
+		mptcp_for_each_sub(tp->mpcb, mptcp) {
+			sock_rps_record_flow(mptcp_to_sock(mptcp));
+		}
 	}
 
 	sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
@@ -1287,9 +1290,11 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size)
 	}
 
 	if (mptcp(tp)) {
-		struct sock *sk_it = sk;
-		mptcp_for_each_sk(tp->mpcb, sk_it)
-			sock_rps_record_flow(sk_it);
+		struct mptcp_tcp_sock *mptcp;
+
+		mptcp_for_each_sub(tp->mpcb, mptcp) {
+			sock_rps_record_flow(mptcp_to_sock(mptcp));
+		}
 	}
 
 	if (unlikely(tp->repair)) {
@@ -1994,9 +1999,11 @@ int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int nonblock,
 
 #ifdef CONFIG_MPTCP
 	if (mptcp(tp)) {
-		struct sock *sk_it;
-		mptcp_for_each_sk(tp->mpcb, sk_it)
-			sock_rps_record_flow(sk_it);
+		struct mptcp_tcp_sock *mptcp;
+
+		mptcp_for_each_sub(tp->mpcb, mptcp) {
+			sock_rps_record_flow(mptcp_to_sock(mptcp));
+		}
 	}
 #endif
 
net/mptcp/mptcp_ctrl.c +10 -4
index 052ae0e052b0..925d05d673d8 100644
@@ -553,10 +553,12 @@ void mptcp_hash_remove_bh(struct tcp_sock *meta_tp)
 struct sock *mptcp_select_ack_sock(const struct sock *meta_sk)
 {
 	const struct tcp_sock *meta_tp = tcp_sk(meta_sk);
-	struct sock *sk, *rttsk = NULL, *lastsk = NULL;
+	struct sock *rttsk = NULL, *lastsk = NULL;
 	u32 min_time = 0, last_active = 0;
+	struct mptcp_tcp_sock *mptcp;
 
-	mptcp_for_each_sk(meta_tp->mpcb, sk) {
+	mptcp_for_each_sub(meta_tp->mpcb, mptcp) {
+		struct sock *sk = mptcp_to_sock(mptcp);
 		struct tcp_sock *tp = tcp_sk(sk);
 		u32 elapsed;
 
@@ -689,7 +691,8 @@ void mptcp_sock_destruct(struct sock *sk)
 void mptcp_destroy_sock(struct sock *sk)
 {
 	if (is_meta_sk(sk)) {
-		struct sock *sk_it, *tmpsk;
+		struct mptcp_tcp_sock *mptcp;
+		struct hlist_node *tmp;
 
 		__skb_queue_purge(&tcp_sk(sk)->mpcb->reinject_queue);
 
@@ -699,7 +702,9 @@ void mptcp_destroy_sock(struct sock *sk)
 		 * not have been closed properly (as we are waiting for the
 		 * DATA_ACK of the DATA_FIN).
 		 */
-		mptcp_for_each_sk_safe(tcp_sk(sk)->mpcb, sk_it, tmpsk) {
+		mptcp_for_each_sub_safe(tcp_sk(sk)->mpcb, mptcp, tmp) {
+			struct sock *sk_it = mptcp_to_sock(mptcp);
+
 			/* Already did call tcp_close - waiting for graceful
 			 * closure, or if we are retransmitting fast-close on
 			 * the subflow. The reset (or timeout) will kill the
@@ -1281,6 +1286,7 @@ static int mptcp_alloc_mpcb(struct sock *meta_sk, __u64 remote_key,
 	INIT_LIST_HEAD(&mpcb->tw_list);
 
 	INIT_HLIST_HEAD(&mpcb->callback_list);
+	INIT_HLIST_HEAD(&mpcb->conn_list);
 	spin_lock_init(&mpcb->mpcb_list_lock);
 
 	mptcp_mpcb_inherit_sockopts(meta_sk, master_sk);
@@ -1370,8 +1376,12 @@ int mptcp_add_sock(struct sock *meta_sk, struct sock *sk, u8 loc_id, u8 rem_id,
 	sock_hold(meta_sk);
 	refcount_inc(&mpcb->mpcb_refcnt);
 
Quoted 11 lines

net/mptcp/mptcp_ctrl.c int mptcp_add_sock(struct sock *meta_sk, struct sock *sk, u8 loc_id, u8 rem_id, · line 1383

Quoted 7 lines
net/mptcp/mptcp_ctrl.c +5 -2
-	tp->mptcp->next = mpcb->connection_list;
-	mpcb->connection_list = tp;
+	local_bh_disable();
+	spin_lock(&mpcb->mpcb_list_lock);
+	hlist_add_head_rcu(&tp->mptcp->node, &mpcb->conn_list);
+	spin_unlock(&mpcb->mpcb_list_lock);
+	local_bh_enable();

I'm curious why the local_bh_disable is required here - I'm assuming it's possible to be called in softirq conext in this function but not where you delete from the list. A comment might help.

Yes, mptcp_add_sock() can be called from both contexts.

I just realize that mptcp_del_sock() can now also be called from both contexts. I need to disable bh there as well. spin_lock_bh() / spin_unlock_bh() would probably also be a better.

I will submit a separate patch to mptcp_trunk to fix and comment this. That way we have the same history on both.

Quoted 33 lines
Quoted 33 lines
net/mptcp/mptcp_ctrl.c +2 -11
+
 	tp->mptcp->attached = 1;
 
 	atomic_add(atomic_read(&((struct sock *)tp)->sk_rmem_alloc),
@@ -1415,14 +1425,13 @@ int mptcp_add_sock(struct sock *meta_sk, struct sock *sk, u8 loc_id, u8 rem_id,
 
 void mptcp_del_sock(struct sock *sk)
 {
-	struct tcp_sock *tp = tcp_sk(sk), *tp_prev;
+	struct tcp_sock *tp = tcp_sk(sk);
 	struct mptcp_cb *mpcb;
 
 	if (!tp->mptcp || !tp->mptcp->attached)
 		return;
 
 	mpcb = tp->mpcb;
-	tp_prev = mpcb->connection_list;
 
 	if (mpcb->pm_ops->delete_subflow)
 		mpcb->pm_ops->delete_subflow(sk);
@@ -1431,17 +1440,10 @@ void mptcp_del_sock(struct sock *sk)
 		    __func__, mpcb->mptcp_loc_token, tp->mptcp->path_index,
 		    sk->sk_state, is_meta_sk(sk));
 
-	if (tp_prev == tp) {
-		mpcb->connection_list = tp->mptcp->next;
-	} else {
-		for (; tp_prev && tp_prev->mptcp->next; tp_prev = tp_prev->mptcp->next) {
-			if (tp_prev->mptcp->next == tp) {
-				tp_prev->mptcp->next = tp->mptcp->next;
-				break;
-			}
-		}
Quoted 8 lines

net/mptcp/mptcp_ctrl.c void mptcp_del_sock(struct sock *sk) · line 1445

Quoted 5 lines
net/mptcp/mptcp_ctrl.c +3 -2
-	}
-	tp->mptcp->next = NULL;
+	spin_lock(&mpcb->mpcb_list_lock);
+	hlist_del_init_rcu(&tp->mptcp->node);
+	spin_unlock(&mpcb->mpcb_list_lock);

Is it guaranteed that RCU synchronization happens between here and when the tcp_sock is freed? Do you need to set SOCK_RCU_FREE?

Looking more through this, I think we are good here.

The trick is that traversal of the list actually always happens while holding the MPTCP-level lock.

I needed to convert this to RCU so that we can add to the list (without holding MPTCP-level lock), while traversing it. Using the hlist guarantees that the traversal is going forward.

I have some more patches coming now for mptcp_trunk to fix some of the fallout after this series here. How do you want to take these "forward-ports" ? Do you want me to submit everything (including this set here) at once to mptcp-net-next, or do you want to take the sets separately ?

Thanks,
Christoph

Quoted 588 lines

Mat

Quoted 579 lines
net/mptcp/mptcp_ctrl.c +35 -15
+
 	tp->mptcp->attached = 0;
 	mpcb->path_index_bits &= ~(1 << tp->mptcp->path_index);
 
@@ -1488,8 +1490,8 @@ void mptcp_update_metasocket(const struct sock *meta_sk)
 void mptcp_cleanup_rbuf(struct sock *meta_sk, int copied)
 {
 	struct tcp_sock *meta_tp = tcp_sk(meta_sk);
-	struct sock *sk;
 	bool recheck_rcv_window = false;
+	struct mptcp_tcp_sock *mptcp;
 	__u32 rcv_window_now = 0;
 
 	if (copied > 0 && !(meta_sk->sk_shutdown & RCV_SHUTDOWN)) {
@@ -1500,7 +1502,8 @@ void mptcp_cleanup_rbuf(struct sock *meta_sk, int copied)
 			recheck_rcv_window = true;
 	}
 
-	mptcp_for_each_sk(meta_tp->mpcb, sk) {
+	mptcp_for_each_sub(meta_tp->mpcb, mptcp) {
+		struct sock *sk = mptcp_to_sock(mptcp);
 		struct tcp_sock *tp = tcp_sk(sk);
 		const struct inet_connection_sock *icsk = inet_csk(sk);
 
@@ -1687,10 +1690,13 @@ EXPORT_SYMBOL(mptcp_sub_force_close);
 */
 void mptcp_update_sndbuf(const struct tcp_sock *tp)
 {
-	struct sock *meta_sk = tp->meta_sk, *sk;
+	struct sock *meta_sk = tp->meta_sk;
 	int new_sndbuf = 0, old_sndbuf = meta_sk->sk_sndbuf;
+	struct mptcp_tcp_sock *mptcp;
+
+	mptcp_for_each_sub(tp->mpcb, mptcp) {
+		struct sock *sk = mptcp_to_sock(mptcp);
 
-	mptcp_for_each_sk(tp->mpcb, sk) {
 		if (!mptcp_sk_can_send(sk))
 			continue;
 
@@ -1719,8 +1725,8 @@ void mptcp_update_sndbuf(const struct tcp_sock *tp)
 void mptcp_close(struct sock *meta_sk, long timeout)
 {
 	struct tcp_sock *meta_tp = tcp_sk(meta_sk);
-	struct sock *sk_it, *tmpsk;
 	struct mptcp_cb *mpcb = meta_tp->mpcb;
+	struct mptcp_tcp_sock *mptcp;
 	struct sk_buff *skb;
 	int data_was_unread = 0;
 	int state;
@@ -1753,7 +1759,12 @@ void mptcp_close(struct sock *meta_sk, long timeout)
 
 	/* If socket has been already reset (e.g. in tcp_reset()) - kill it. */
 	if (meta_sk->sk_state == TCP_CLOSE) {
-		mptcp_for_each_sk_safe(mpcb, sk_it, tmpsk) {
+		struct mptcp_tcp_sock *mptcp;
+		struct hlist_node *tmp;
+
+		mptcp_for_each_sub_safe(mpcb, mptcp, tmp) {
+			struct sock *sk_it = mptcp_to_sock(mptcp);
+
 			if (tcp_sk(sk_it)->send_mp_fclose)
 				continue;
 			mptcp_sub_close(sk_it, 0);
@@ -1774,10 +1785,14 @@ void mptcp_close(struct sock *meta_sk, long timeout)
 	} else if (tcp_close_state(meta_sk)) {
 		mptcp_send_fin(meta_sk);
 	} else if (meta_tp->snd_una == meta_tp->write_seq) {
+		struct mptcp_tcp_sock *mptcp;
+		struct hlist_node *tmp;
+
 		/* The DATA_FIN has been sent and acknowledged
 		 * (e.g., by sk_shutdown). Close all the other subflows
 		 */
-		mptcp_for_each_sk_safe(mpcb, sk_it, tmpsk) {
+		mptcp_for_each_sub_safe(mpcb, mptcp, tmp) {
+			struct sock *sk_it = mptcp_to_sock(mptcp);
 			unsigned long delay = 0;
 			/* If we are the passive closer, don't trigger
 			 * subflow-fin until the subflow has been finned
@@ -1801,7 +1816,9 @@ void mptcp_close(struct sock *meta_sk, long timeout)
 	/* socket will be freed after mptcp_close - we have to prevent
 	 * access from the subflows.
 	 */
-	mptcp_for_each_sk(mpcb, sk_it) {
+	mptcp_for_each_sub(mpcb, mptcp) {
+		struct sock *sk_it = mptcp_to_sock(mptcp);
+
 		/* Similar to sock_orphan, but we don't set it DEAD, because
 		 * the callbacks are still set and must be called.
 		 */
@@ -1886,8 +1903,9 @@ void mptcp_close(struct sock *meta_sk, long timeout)
 
 void mptcp_disconnect(struct sock *sk)
 {
-	struct sock *subsk, *tmpsk;
+	struct mptcp_tcp_sock *mptcp;
 	struct tcp_sock *tp = tcp_sk(sk);
+	struct hlist_node *tmp;
 
 	__skb_queue_purge(&tp->mpcb->reinject_queue);
 
@@ -1895,7 +1913,9 @@ void mptcp_disconnect(struct sock *sk)
 		mptcp_hash_remove_bh(tp);
 
 	local_bh_disable();
-	mptcp_for_each_sk_safe(tp->mpcb, subsk, tmpsk) {
+	mptcp_for_each_sub_safe(tp->mpcb, mptcp, tmp) {
+		struct sock *subsk = mptcp_to_sock(mptcp);
+
 		/* The socket will get removed from the subsocket-list
 		 * and made non-mptcp by setting mpc to 0.
 		 *
@@ -2579,7 +2599,6 @@ static void mptcp_get_sub_info(struct sock *sk, struct mptcp_sub_info *info)
 int mptcp_get_info(const struct sock *meta_sk, char __user *optval, int optlen)
 {
 	const struct tcp_sock *meta_tp = tcp_sk(meta_sk);
-	struct sock *sk;
 
 	struct mptcp_meta_info meta_info;
 	struct mptcp_info m_info;
@@ -2625,16 +2644,17 @@ int mptcp_get_info(const struct sock *meta_sk, char __user *optval, int optlen)
 
 	if (m_info.subflows) {
 		unsigned int len, sub_len = 0;
+		struct mptcp_tcp_sock *mptcp;
 		char __user *ptr;
 
 		ptr = (char __user *)m_info.subflows;
 		len = m_info.sub_len;
 
-		mptcp_for_each_sk(meta_tp->mpcb, sk) {
+		mptcp_for_each_sub(meta_tp->mpcb, mptcp) {
 			struct tcp_info t_info;
 			unsigned int tmp_len;
 
-			tcp_get_info(sk, &t_info);
+			tcp_get_info(mptcp_to_sock(mptcp), &t_info);
 
 			tmp_len = min_t(unsigned int, len, info_len);
 			len -= tmp_len;
@@ -2654,6 +2674,7 @@ int mptcp_get_info(const struct sock *meta_sk, char __user *optval, int optlen)
 
 	if (m_info.subflow_info) {
 		unsigned int len, sub_info_len, total_sub_info_len = 0;
+		struct mptcp_tcp_sock *mptcp;
 		char __user *ptr;
 
 		ptr = (char __user *)m_info.subflow_info;
@@ -2663,11 +2684,11 @@ int mptcp_get_info(const struct sock *meta_sk, char __user *optval, int optlen)
 				     sizeof(struct mptcp_sub_info));
 		m_info.sub_info_len = sub_info_len;
 
-		mptcp_for_each_sk(meta_tp->mpcb, sk) {
+		mptcp_for_each_sub(meta_tp->mpcb, mptcp) {
 			struct mptcp_sub_info m_sub_info;
 			unsigned int tmp_len;
 
-			mptcp_get_sub_info(sk, &m_sub_info);
+			mptcp_get_sub_info(mptcp_to_sock(mptcp), &m_sub_info);
 
 			tmp_len = min_t(unsigned int, len, sub_info_len);
 			len -= tmp_len;
net/mptcp/mptcp_fullmesh.c +20 -8
index d29102a1dffb..53c1ba7319f5 100644
@@ -903,8 +903,9 @@ static void mptcp_address_worker(struct work_struct *work)
 			}
 
 			if (event->code == MPTCP_EVENT_DEL) {
-				struct sock *sk, *tmpsk;
+				struct mptcp_tcp_sock *mptcp;
 				struct mptcp_loc_addr *mptcp_local;
+				struct hlist_node *tmp;
 				bool found = false;
 
 				mptcp_local = rcu_dereference_bh(fm_ns->local);
@@ -914,7 +915,9 @@ static void mptcp_address_worker(struct work_struct *work)
 					update_addr_bitfields(meta_sk, mptcp_local);
 
 				/* Look for the socket and remove him */
-				mptcp_for_each_sk_safe(mpcb, sk, tmpsk) {
+				mptcp_for_each_sub_safe(mpcb, mptcp, tmp) {
+					struct sock *sk = mptcp_to_sock(mptcp);
+
 					if ((event->family == AF_INET6 &&
 					     (sk->sk_family == AF_INET ||
 					      mptcp_v6_is_v4_mapped(sk))) ||
@@ -964,9 +967,10 @@ static void mptcp_address_worker(struct work_struct *work)
 			}
 
 			if (event->code == MPTCP_EVENT_MOD) {
-				struct sock *sk;
+				struct mptcp_tcp_sock *mptcp;
 
-				mptcp_for_each_sk(mpcb, sk) {
+				mptcp_for_each_sub(mpcb, mptcp) {
+					struct sock *sk = mptcp_to_sock(mptcp);
 					struct tcp_sock *tp = tcp_sk(sk);
 					if (event->family == AF_INET &&
 					    (sk->sk_family == AF_INET ||
@@ -1455,8 +1459,9 @@ static void full_mesh_release_sock(struct sock *meta_sk)
 	struct mptcp_cb *mpcb = tcp_sk(meta_sk)->mpcb;
 	struct fullmesh_priv *fmp = fullmesh_get_priv(mpcb);
 	const struct mptcp_fm_ns *fm_ns = fm_get_ns(sock_net(meta_sk));
-	struct sock *sk, *tmpsk;
 	bool meta_v4 = meta_sk->sk_family == AF_INET;
+	struct mptcp_tcp_sock *mptcp;
+	struct hlist_node *tmp;
 	int i;
 
 	rcu_read_lock_bh();
@@ -1470,7 +1475,8 @@ static void full_mesh_release_sock(struct sock *meta_sk)
 		struct in_addr ifa = mptcp_local->locaddr4[i].addr;
 		bool found = false;
 
-		mptcp_for_each_sk(mpcb, sk) {
+		mptcp_for_each_sub(mpcb, mptcp) {
+			struct sock *sk = mptcp_to_sock(mptcp);
 			struct tcp_sock *tp = tcp_sk(sk);
 
 			if (sk->sk_family == AF_INET6 &&
@@ -1491,6 +1497,8 @@ static void full_mesh_release_sock(struct sock *meta_sk)
 		}
 
 		if (!found) {
+			struct sock *sk;
+
 			fmp->add_addr++;
 			mpcb->addr_signal = 1;
 
@@ -1511,7 +1519,8 @@ static void full_mesh_release_sock(struct sock *meta_sk)
 		struct in6_addr ifa = mptcp_local->locaddr6[i].addr;
 		bool found = false;
 
-		mptcp_for_each_sk(mpcb, sk) {
+		mptcp_for_each_sub(mpcb, mptcp) {
+			struct sock *sk = mptcp_to_sock(mptcp);
 			struct tcp_sock *tp = tcp_sk(sk);
 
 			if (sk->sk_family == AF_INET ||
@@ -1532,6 +1541,8 @@ static void full_mesh_release_sock(struct sock *meta_sk)
 		}
 
 		if (!found) {
+			struct sock *sk;
+
 			fmp->add_addr++;
 			mpcb->addr_signal = 1;
 
@@ -1546,7 +1557,8 @@ static void full_mesh_release_sock(struct sock *meta_sk)
 #endif
 
 	/* Now, detect address-removals */
-	mptcp_for_each_sk_safe(mpcb, sk, tmpsk) {
+	mptcp_for_each_sub_safe(mpcb, mptcp, tmp) {
+		struct sock *sk = mptcp_to_sock(mptcp);
 		bool shall_remove = true;
 
 		if (sk->sk_family == AF_INET || mptcp_v6_is_v4_mapped(sk)) {
net/mptcp/mptcp_input.c +29 -15
index 211605347f70..64b0265bfd06 100644
@@ -126,12 +126,14 @@ static void mptcp_clean_rtx_queue(struct sock *meta_sk, u32 prior_snd_una)
 		tcp_rtx_queue_unlink(skb, meta_sk);
 
 		if (mptcp_is_data_fin(skb)) {
-			struct sock *sk_it, *sk_tmp;
+			struct mptcp_tcp_sock *mptcp;
+			struct hlist_node *tmp;
 
 			/* DATA_FIN has been acknowledged - now we can close
 			 * the subflows
 			 */
-			mptcp_for_each_sk_safe(mpcb, sk_it, sk_tmp) {
+			mptcp_for_each_sub_safe(mpcb, mptcp, tmp) {
+				struct sock *sk_it = mptcp_to_sock(mptcp);
 				unsigned long delay = 0;
 
 				/* If we are the passive closer, don't trigger
@@ -347,6 +349,7 @@ static int mptcp_verif_dss_csum(struct sock *sk)
 
 	/* Now, checksum must be 0 */
 	if (unlikely(csum_fold(csum_tcp))) {
+		struct mptcp_tcp_sock *mptcp;
 		struct sock *sk_it = NULL;
 
 		pr_err("%s csum is wrong: %#x tcp-seq %u dss_csum_added %d overflowed %d iterations %d\n",
@@ -362,7 +365,9 @@ static int mptcp_verif_dss_csum(struct sock *sk)
 		tp->mpcb->csum_cutoff_seq = tp->mptcp->map_data_seq;
 
 		/* Search for another subflow that is fully established */
-		mptcp_for_each_sk(tp->mpcb, sk_it) {
+		mptcp_for_each_sub(tp->mpcb, mptcp) {
+			sk_it = mptcp_to_sock(mptcp);
+
 			if (sk_it != sk &&
 			    tcp_sk(sk_it)->mptcp->fully_established)
 				break;
@@ -1308,12 +1313,15 @@ int mptcp_do_join_short(struct sk_buff *skb,
 */
 void mptcp_fin(struct sock *meta_sk)
 {
-	struct sock *sk = NULL, *sk_it;
+	struct sock *sk = NULL;
 	struct tcp_sock *meta_tp = tcp_sk(meta_sk);
 	struct mptcp_cb *mpcb = meta_tp->mpcb;
+	struct mptcp_tcp_sock *mptcp;
 	unsigned char state;
 
-	mptcp_for_each_sk(mpcb, sk_it) {
+	mptcp_for_each_sub(mpcb, mptcp) {
+		struct sock *sk_it = mptcp_to_sock(mptcp);
+
 		if (tcp_sk(sk_it)->mptcp->path_index == mpcb->dfin_path_index) {
 			sk = sk_it;
 			break;
@@ -1585,9 +1593,12 @@ void mptcp_clean_rtx_infinite(const struct sk_buff *skb, struct sock *sk)
 
 static void mptcp_send_reset_rem_id(const struct mptcp_cb *mpcb, u8 rem_id)
 {
-	struct sock *sk_it, *tmpsk;
+	struct mptcp_tcp_sock *mptcp;
+	struct hlist_node *tmp;
+
+	mptcp_for_each_sub_safe(mpcb, mptcp, tmp) {
+		struct sock *sk_it = mptcp_to_sock(mptcp);
 
-	mptcp_for_each_sk_safe(mpcb, sk_it, tmpsk) {
 		if (tcp_sk(sk_it)->mptcp->rem_id == rem_id) {
 			mptcp_reinject_data(sk_it, 0);
 			mptcp_send_reset(sk_it);
@@ -1892,13 +1903,15 @@ void tcp_parse_mptcp_options(const struct sk_buff *skb,
 bool mptcp_check_rtt(const struct tcp_sock *tp, int time)
 {
 	struct mptcp_cb *mpcb = tp->mpcb;
-	struct sock *sk;
+	struct mptcp_tcp_sock *mptcp;
 	u32 rtt_max = 0;
 
 	/* In MPTCP, we take the max delay across all flows,
 	 * in order to take into account meta-reordering buffers.
 	 */
-	mptcp_for_each_sk(mpcb, sk) {
+	mptcp_for_each_sub(mpcb, mptcp) {
+		struct sock *sk = mptcp_to_sock(mptcp);
+
 		if (!mptcp_sk_can_recv(sk))
 			continue;
 
@@ -2173,9 +2186,9 @@ bool mptcp_handle_options(struct sock *sk, const struct tcphdr *th,
 		if (mopt->saw_low_prio == 1) {
 			tp->mptcp->rcv_low_prio = mopt->low_prio;
 		} else {
-			struct sock *sk_it;
-			mptcp_for_each_sk(tp->mpcb, sk_it) {
-				struct mptcp_tcp_sock *mptcp = tcp_sk(sk_it)->mptcp;
+			struct mptcp_tcp_sock *mptcp;
+
+			mptcp_for_each_sub(tp->mpcb, mptcp) {
 				if (mptcp->rem_id == mopt->prio_addr_id)
 					mptcp->rcv_low_prio = mopt->low_prio;
 			}
@@ -2359,7 +2372,7 @@ bool mptcp_should_expand_sndbuf(const struct sock *sk)
 {
 	const struct sock *meta_sk = mptcp_meta_sk(sk);
 	const struct tcp_sock *meta_tp = tcp_sk(meta_sk);
-	const struct sock *sk_it;
+	const struct mptcp_tcp_sock *mptcp;
 
 	/* We circumvent this check in tcp_check_space, because we want to
 	 * always call sk_write_space. So, we reproduce the check here.
@@ -2385,8 +2398,9 @@ bool mptcp_should_expand_sndbuf(const struct sock *sk)
 	/* For MPTCP we look for a subsocket that could send data.
 	 * If we found one, then we update the send-buffer.
 	 */
-	mptcp_for_each_sk(meta_tp->mpcb, sk_it) {
-		struct tcp_sock *tp_it = tcp_sk(sk_it);
+	mptcp_for_each_sub(meta_tp->mpcb, mptcp) {
+		const struct sock *sk_it = mptcp_to_sock(mptcp);
+		const struct tcp_sock *tp_it = tcp_sk(sk_it);
 
 		if (!mptcp_sk_can_send(sk_it))
 			continue;
net/mptcp/mptcp_output.c +2 -1
index 4b88b32d5954..12733ba2c1e1 100644
@@ -644,7 +644,6 @@ int mptcp_write_wakeup(struct sock *meta_sk, int mib)
 {
 	struct tcp_sock *meta_tp = tcp_sk(meta_sk);
 	struct sk_buff *skb;
-	struct sock *sk_it;
 	int ans = 0;
 
 	if (meta_sk->sk_state == TCP_CLOSE)
@@ -701,17 +700,22 @@ int mptcp_write_wakeup(struct sock *meta_sk, int mib)
 
 		return 0;
 	} else {
+		struct mptcp_tcp_sock *mptcp;
+

window_probe:

		if (between(meta_tp->snd_up, meta_tp->snd_una + 1,
			    meta_tp->snd_una + 0xFFFF)) {
net/mptcp/mptcp_output.c +3 -1
-			mptcp_for_each_sk(meta_tp->mpcb, sk_it) {
+			mptcp_for_each_sub(meta_tp->mpcb, mptcp) {
+				struct sock *sk_it = mptcp_to_sock(mptcp);
+
				if (mptcp_sk_can_send_ack(sk_it))
					tcp_xmit_probe_skb(sk_it, 1, mib);
			}
		}

		/* At least one of the tcp_xmit_probe_skb's has to succeed */
net/mptcp/mptcp_output.c +2 -1
-		mptcp_for_each_sk(meta_tp->mpcb, sk_it) {
+		mptcp_for_each_sub(meta_tp->mpcb, mptcp) {
+			struct sock *sk_it = mptcp_to_sock(mptcp);
			int ret;

			if (!mptcp_sk_can_send_ack(sk_it))
net/mptcp/mptcp_output.c +18 -10
@@ -729,6 +733,7 @@ bool mptcp_write_xmit(struct sock *meta_sk, unsigned int mss_now, int nonagle,
 		     int push_one, gfp_t gfp)
 {
 	struct tcp_sock *meta_tp = tcp_sk(meta_sk), *subtp;
+	struct mptcp_tcp_sock *mptcp;
 	struct sock *subsk = NULL;
 	struct mptcp_cb *mpcb = meta_tp->mpcb;
 	struct sk_buff *skb;
@@ -853,7 +858,8 @@ bool mptcp_write_xmit(struct sock *meta_sk, unsigned int mss_now, int nonagle,
 			break;
 	}
 
-	mptcp_for_each_sk(mpcb, subsk) {
+	mptcp_for_each_sub(mpcb, mptcp) {
+		subsk = mptcp_to_sock(mptcp);
 		subtp = tcp_sk(subsk);
 
 		if (!(path_mask & mptcp_pi_to_flag(subtp->mptcp->path_index)))
@@ -1350,7 +1356,7 @@ void mptcp_send_active_reset(struct sock *meta_sk, gfp_t priority)
 	struct mptcp_cb *mpcb = meta_tp->mpcb;
 	struct sock *sk;
 
-	if (!mpcb->connection_list)
+	if (hlist_empty(&mpcb->conn_list))
 		return;
 
 	WARN_ON(meta_tp->send_mp_fclose);
@@ -1725,10 +1731,11 @@ void mptcp_select_initial_window(const struct sock *sk, int __space, __u32 mss,
 static inline u64 mptcp_calc_rate(const struct sock *meta_sk, unsigned int mss,
 				  unsigned int (*mss_cb)(struct sock *sk))
 {
-	struct sock *sk;
+	struct mptcp_tcp_sock *mptcp;
 	u64 rate = 0;
 
-	mptcp_for_each_sk(tcp_sk(meta_sk)->mpcb, sk) {
+	mptcp_for_each_sub(tcp_sk(meta_sk)->mpcb, mptcp) {
+		struct sock *sk = mptcp_to_sock(mptcp);
 		struct tcp_sock *tp = tcp_sk(sk);
 		int this_mss;
 		u64 this_rate;
@@ -1780,11 +1787,12 @@ static inline u64 mptcp_calc_rate(const struct sock *meta_sk, unsigned int mss,
 static unsigned int __mptcp_current_mss(const struct sock *meta_sk,
 					unsigned int (*mss_cb)(struct sock *sk))
 {
+	struct mptcp_tcp_sock *mptcp;
 	unsigned int mss = 0;
 	u64 rate = 0;
-	struct sock *sk;
 
-	mptcp_for_each_sk(tcp_sk(meta_sk)->mpcb, sk) {
+	mptcp_for_each_sub(tcp_sk(meta_sk)->mpcb, mptcp) {
+		struct sock *sk = mptcp_to_sock(mptcp);
 		int this_mss;
 		u64 this_rate;
 
@@ -1849,14 +1857,16 @@ int mptcp_select_size(const struct sock *meta_sk, bool first_skb, bool zc)
 
 int mptcp_check_snd_buf(const struct tcp_sock *tp)
 {
-	const struct sock *sk;
+	const struct mptcp_tcp_sock *mptcp;
 	u32 rtt_max = tp->srtt_us;
 	u64 bw_est;
 
 	if (!tp->srtt_us)
 		return tp->reordering + 1;
 
-	mptcp_for_each_sk(tp->mpcb, sk) {
+	mptcp_for_each_sub(tp->mpcb, mptcp) {
+		const struct sock *sk = mptcp_to_sock(mptcp);
+
 		if (!mptcp_sk_can_send(sk))
 			continue;
 
@@ -1874,11 +1884,13 @@ int mptcp_check_snd_buf(const struct tcp_sock *tp)
 unsigned int mptcp_xmit_size_goal(const struct sock *meta_sk, u32 mss_now,
 				  int large_allowed)
 {
-	struct sock *sk;
 	u32 xmit_size_goal = 0;
 
 	if (large_allowed && !tcp_sk(meta_sk)->mpcb->dss_csum) {
-		mptcp_for_each_sk(tcp_sk(meta_sk)->mpcb, sk) {
+		struct mptcp_tcp_sock *mptcp;
+
+		mptcp_for_each_sub(tcp_sk(meta_sk)->mpcb, mptcp) {
+			struct sock *sk = mptcp_to_sock(mptcp);
 			int this_size_goal;
 
 			if (!mptcp_sk_can_send(sk))
net/mptcp/mptcp_sched.c +15 -6
index a2543c60bd31..b440df7aae71 100644
@@ -135,9 +135,10 @@ static struct sock
 	u32 min_srtt = 0xffffffff;
 	bool found_unused = false;
 	bool found_unused_una = false;
-	struct sock *sk;
+	struct mptcp_tcp_sock *mptcp;
 
-	mptcp_for_each_sk(mpcb, sk) {
+	mptcp_for_each_sub(mpcb, mptcp) {
+		struct sock *sk = mptcp_to_sock(mptcp);
 		struct tcp_sock *tp = tcp_sk(sk);
 		bool unused = false;
 
@@ -219,7 +220,11 @@ static struct sock *get_available_subflow(struct sock *meta_sk,
 	/* Answer data_fin on same subflow!!! */
 	if (meta_sk->sk_shutdown & RCV_SHUTDOWN &&
 	    skb && mptcp_is_data_fin(skb)) {
-		mptcp_for_each_sk(mpcb, sk) {
+		struct mptcp_tcp_sock *mptcp;
+
+		mptcp_for_each_sub(mpcb, mptcp) {
+			sk = mptcp_to_sock(mptcp);
+
 			if (tcp_sk(sk)->mptcp->path_index == mpcb->dfin_path_index &&
 			    mptcp_is_available(sk, skb, zero_wnd_test))
 				return sk;
@@ -252,7 +257,7 @@ static struct sk_buff *mptcp_rcv_buf_optimization(struct sock *sk, int penal)
 {
 	struct sock *meta_sk;
 	const struct tcp_sock *tp = tcp_sk(sk);
-	struct tcp_sock *tp_it;
+	struct mptcp_tcp_sock *mptcp;
 	struct sk_buff *skb_head;
 	struct defsched_priv *dsp = defsched_get_priv(tp);
 
@@ -275,7 +280,9 @@ static struct sk_buff *mptcp_rcv_buf_optimization(struct sock *sk, int penal)
 		goto retrans;
 
 	/* Half the cwnd of the slow flow */
-	mptcp_for_each_tp(tp->mpcb, tp_it) {
+	mptcp_for_each_sub(tp->mpcb, mptcp) {
+		struct tcp_sock *tp_it = mptcp->tp;
+
 		if (tp_it != tp &&
 		    TCP_SKB_CB(skb_head)->path_mask & mptcp_pi_to_flag(tp_it->mptcp->path_index)) {
 			if (tp->srtt_us < tp_it->srtt_us && inet_csk((struct sock *)tp_it)->icsk_ca_state == TCP_CA_Open) {
@@ -298,7 +305,9 @@ static struct sk_buff *mptcp_rcv_buf_optimization(struct sock *sk, int penal)
 	/* Segment not yet injected into this path? Take it!!! */
 	if (!(TCP_SKB_CB(skb_head)->path_mask & mptcp_pi_to_flag(tp->mptcp->path_index))) {
 		bool do_retrans = false;
-		mptcp_for_each_tp(tp->mpcb, tp_it) {
+		mptcp_for_each_sub(tp->mpcb, mptcp) {
+			struct tcp_sock *tp_it = mptcp->tp;
+
 			if (tp_it != tp &&
 			    TCP_SKB_CB(skb_head)->path_mask & mptcp_pi_to_flag(tp_it->mptcp->path_index)) {
 				if (tp_it->snd_cwnd <= 4) {
Signature · 1 line
2.16.2

--
Mat Martineau
Intel OTC

#13 lore raw

Re: [MPTCP] [PATCH mptcp-net-next 10/14] mptcp: Make subflow-list an RCU-list

Hi Christoph -

Quoted 373 lines · Christoph Paasch

Hello Mat,

Quoted 358 lines · Mat Martineau

Christoph -

Quoted 353 lines · Christoph Paasch

mptcp_add_sock() will now be called without holding the meta-level lock. However, mptcp_add_sock() wants to add the subflow to the meta-level list, thus we need to protect this by a lock. We use the mpcb_list_lock for that.

Now that we are locking during add/del, and want to allow lockless traversal of the list, this implies that we need to make it an RCU-list.

So, this patch transitions to the RCU hlist. The list-traversal macros (hlist_for_each_entry_rcu) require me to now pass the mptcp_tcp_sock to mptcp_for_each_*. So, I had to change all the places in the code where we call one of the list-traversal macros to adapt to this.

Signed-off-by Christoph Paasch <cpaasch(a)apple.com>
Signed-off-by Matthieu Baerts <matthieu.baerts(a)tessares.net>
(cherry picked from commit 7a662b690069642c138da16ff9396e6826fd0a95)

---

include/net/mptcp.h        | 67 +++++++++++++++++-----------------
net/ipv4/af_inet.c         |  9 ++---
net/ipv4/ip_sockglue.c     |  7 ++--
net/ipv4/tcp.c             | 31 +++++++++-------
net/mptcp/mptcp_ctrl.c     | 89 ++++++++++++++++++++++++++++------------------
net/mptcp/mptcp_fullmesh.c | 28 ++++++++++-----
net/mptcp/mptcp_input.c    | 44 +++++++++++++++--------
net/mptcp/mptcp_output.c   | 38 +++++++++++++-------
net/mptcp/mptcp_sched.c    | 21 +++++++----
9 files changed, 207 insertions(+), 127 deletions(-)
include/net/mptcp.h +34 -33
index 0f9d47b45895..091e56a7b334 100644
@@ -156,7 +156,7 @@ struct mptcp_options_received {
 };
 
 struct mptcp_tcp_sock {
-	struct tcp_sock	*next;		/* Next subflow socket */
+	struct hlist_node node;
 	struct hlist_node cb_list;
 	struct mptcp_options_received rx_opt;
 
@@ -254,7 +254,7 @@ struct mptcp_sched_ops {
 
 struct mptcp_cb {
 	/* list of sockets in this multipath connection */
-	struct tcp_sock *connection_list;
+	struct hlist_head conn_list;
 	/* list of sockets that need a call to release_cb */
 	struct hlist_head callback_list;
 
@@ -309,7 +309,7 @@ struct mptcp_cb {
 	/***** Start of fields, used for subflow establishment */
 	struct sock *meta_sk;
 
-	/* Master socket, also part of the connection_list, this
+	/* Master socket, also part of the conn_list, this
 	 * socket is the one that the application sees.
 	 */
 	struct sock *master_sk;
@@ -661,21 +661,17 @@ extern struct workqueue_struct *mptcp_wq;
 			pr_err(fmt, ##args);					\
 	} while (0)
 
-/* Iterates over all subflows */
-#define mptcp_for_each_tp(mpcb, tp)					\
-	for ((tp) = (mpcb)->connection_list; (tp); (tp) = (tp)->mptcp->next)
+static inline struct sock *mptcp_to_sock(const struct mptcp_tcp_sock *mptcp)
+{
+	return (struct sock *)mptcp->tp;
+}
 
-#define mptcp_for_each_sk(mpcb, sk)					\
-	for ((sk) = (struct sock *)(mpcb)->connection_list;		\
-	     sk;							\
-	     sk = (struct sock *)tcp_sk(sk)->mptcp->next)
+#define mptcp_for_each_sub(__mpcb, __mptcp)					\
+	hlist_for_each_entry_rcu(__mptcp, &((__mpcb)->conn_list), node)
 
-#define mptcp_for_each_sk_safe(__mpcb, __sk, __temp)			\
-	for (__sk = (struct sock *)(__mpcb)->connection_list,		\
-	     __temp = __sk ? (struct sock *)tcp_sk(__sk)->mptcp->next : NULL; \
-	     __sk;							\
-	     __sk = __temp,						\
-	     __temp = __sk ? (struct sock *)tcp_sk(__sk)->mptcp->next : NULL)
+/* Must be called with the appropriate lock held */
+#define mptcp_for_each_sub_safe(__mpcb, __mptcp, __tmp)				\
+	hlist_for_each_entry_safe(__mptcp, __tmp, &((__mpcb)->conn_list), node)
 
 /* Iterates over all bit set to 1 in a bitset */
 #define mptcp_for_each_bit_set(b, i)					\
@@ -922,12 +918,14 @@ struct request_sock *rev_mptcp_rsk(const struct mptcp_request_sock *req)
 
 static inline bool mptcp_can_sendpage(struct sock *sk)
 {
-	struct sock *sk_it;
+	struct mptcp_tcp_sock *mptcp;
 
 	if (tcp_sk(sk)->mpcb->dss_csum)
 		return false;
 
-	mptcp_for_each_sk(tcp_sk(sk)->mpcb, sk_it) {
+	mptcp_for_each_sub(tcp_sk(sk)->mpcb, mptcp) {
+		struct sock *sk_it = mptcp_to_sock(mptcp);
+
 		if (!(sk_it->sk_route_caps & NETIF_F_SG))
 			return false;
 	}
@@ -961,9 +959,12 @@ static inline void mptcp_send_reset(struct sock *sk)
 static inline void mptcp_sub_force_close_all(struct mptcp_cb *mpcb,
 					     struct sock *except)
 {
-	struct sock *sk_it, *tmp;
+	struct mptcp_tcp_sock *mptcp;
+	struct hlist_node *tmp;
+
+	mptcp_for_each_sub_safe(mpcb, mptcp, tmp) {
+		struct sock *sk_it = mptcp_to_sock(mptcp);
 
-	mptcp_for_each_sk_safe(mpcb, sk_it, tmp) {
 		if (sk_it != except)
 			mptcp_send_reset(sk_it);
 	}
@@ -1149,12 +1150,14 @@ static inline int mptcp_sk_can_send_ack(const struct sock *sk)
 
 static inline bool mptcp_can_sg(const struct sock *meta_sk)
 {
-	struct sock *sk;
+	struct mptcp_tcp_sock *mptcp;
 
 	if (tcp_sk(meta_sk)->mpcb->dss_csum)
 		return false;
 
-	mptcp_for_each_sk(tcp_sk(meta_sk)->mpcb, sk) {
+	mptcp_for_each_sub(tcp_sk(meta_sk)->mpcb, mptcp) {
+		struct sock *sk = mptcp_to_sock(mptcp);
+
 		if (!mptcp_sk_can_send(sk))
 			continue;
 		if (!(sk->sk_route_caps & NETIF_F_SG))
@@ -1165,9 +1168,9 @@ static inline bool mptcp_can_sg(const struct sock *meta_sk)
 
 static inline void mptcp_set_rto(struct sock *sk)
 {
-	struct tcp_sock *tp = tcp_sk(sk);
-	struct sock *sk_it;
 	struct inet_connection_sock *micsk = inet_csk(mptcp_meta_sk(sk));
+	struct tcp_sock *tp = tcp_sk(sk);
+	struct mptcp_tcp_sock *mptcp;
 	__u32 max_rto = 0;
 
 	/* We are in recovery-phase on the MPTCP-level. Do not update the
@@ -1176,7 +1179,9 @@ static inline void mptcp_set_rto(struct sock *sk)
 	if (micsk->icsk_retransmits)
 		return;
 
-	mptcp_for_each_sk(tp->mpcb, sk_it) {
+	mptcp_for_each_sub(tp->mpcb, mptcp) {
+		struct sock *sk_it = mptcp_to_sock(mptcp);
+
 		if ((mptcp_sk_can_send(sk_it) || sk->sk_state == TCP_SYN_RECV) &&
 		    inet_csk(sk_it)->icsk_rto > max_rto)
 			max_rto = inet_csk(sk_it)->icsk_rto;
@@ -1265,10 +1270,10 @@ static inline bool mptcp_can_new_subflow(const struct sock *meta_sk)
 
 static inline int mptcp_subflow_count(const struct mptcp_cb *mpcb)
 {
-	struct sock *sk;
+	struct mptcp_tcp_sock *mptcp;
 	int i = 0;
 
-	mptcp_for_each_sk(mpcb, sk)
+	mptcp_for_each_sub(mpcb, mptcp)
 		i++;
 
 	return i;
@@ -1286,12 +1291,8 @@ bool mptcp_prune_ofo_queue(struct sock *sk);
 	do {				\
 	} while (0)
 
-/* Without MPTCP, we just do one iteration
- * over the only socket available. This assumes that
- * the sk/tp arg is the socket in that case.
- */
-#define mptcp_for_each_sk(mpcb, sk)
-#define mptcp_for_each_sk_safe(__mpcb, __sk, __temp)
+#define mptcp_for_each_sub(__mpcb, __mptcp)					\
+	if (0)
 
 #define MPTCP_INC_STATS(net, field)	\
 	do {				\
net/ipv4/af_inet.c +5 -4
index a58a8fbd6192..4b44aa87af2d 100644
@@ -744,13 +744,14 @@ int inet_accept(struct socket *sock, struct socket *newsock, int flags,
 	sock_rps_record_flow(sk2);
 
 	if (sk2->sk_protocol == IPPROTO_TCP && mptcp(tcp_sk(sk2))) {
-		struct sock *sk_it = sk2;
+		struct mptcp_tcp_sock *mptcp;
 
-		mptcp_for_each_sk(tcp_sk(sk2)->mpcb, sk_it)
-			sock_rps_record_flow(sk_it);
+		mptcp_for_each_sub(tcp_sk(sk2)->mpcb, mptcp) {
+			sock_rps_record_flow(mptcp_to_sock(mptcp));
+		}
 
 		if (tcp_sk(sk2)->mpcb->master_sk) {
-			sk_it = tcp_sk(sk2)->mpcb->master_sk;
+			struct sock *sk_it = tcp_sk(sk2)->mpcb->master_sk;
 
 			write_lock_bh(&sk_it->sk_callback_lock);
 			sk_it->sk_wq = newsock->wq;
net/ipv4/ip_sockglue.c +5 -2
index 6a41981c276e..eb72cf990876 100644
@@ -753,8 +753,11 @@ static int do_ip_setsockopt(struct sock *sk, int level,
 			sk_dst_reset(sk);
 			/* Update TOS on mptcp subflow */
 			if (is_meta_sk(sk)) {
-				struct sock *sk_it;
-				mptcp_for_each_sk(tcp_sk(sk)->mpcb, sk_it) {
+				struct mptcp_tcp_sock *mptcp;
+
+				mptcp_for_each_sub(tcp_sk(sk)->mpcb, mptcp) {
+					struct sock *sk_it = mptcp_to_sock(mptcp);
+
 					if (inet_sk(sk_it)->tos != inet_sk(sk)->tos) {
 						inet_sk(sk_it)->tos = inet_sk(sk)->tos;
 						sk_it->sk_priority = sk->sk_priority;
net/ipv4/tcp.c +19 -12
index eeb98619b181..ab17bc222490 100644
@@ -813,9 +813,11 @@ ssize_t tcp_splice_read(struct socket *sock, loff_t *ppos,
 
 #ifdef CONFIG_MPTCP
 	if (mptcp(tcp_sk(sk))) {
-		struct sock *sk_it;
-		mptcp_for_each_sk(tcp_sk(sk)->mpcb, sk_it)
-			sock_rps_record_flow(sk_it);
+		struct mptcp_tcp_sock *mptcp;
+
+		mptcp_for_each_sub(tcp_sk(sk)->mpcb, mptcp) {
+			sock_rps_record_flow(mptcp_to_sock(mptcp));
+		}
 	}
 #endif
 	/*
@@ -993,7 +995,7 @@ ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset,
 	}
 
 	if (mptcp(tp)) {
-		struct sock *sk_it = sk;
+		struct mptcp_tcp_sock *mptcp;
 
 		/* We must check this with socket-lock hold because we iterate
 		 * over the subflows.
@@ -1008,8 +1010,9 @@ ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset,
 			return ret;
 		}
 
-		mptcp_for_each_sk(tp->mpcb, sk_it)
-			sock_rps_record_flow(sk_it);
+		mptcp_for_each_sub(tp->mpcb, mptcp) {
+			sock_rps_record_flow(mptcp_to_sock(mptcp));
+		}
 	}
 
 	sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
@@ -1287,9 +1290,11 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size)
 	}
 
 	if (mptcp(tp)) {
-		struct sock *sk_it = sk;
-		mptcp_for_each_sk(tp->mpcb, sk_it)
-			sock_rps_record_flow(sk_it);
+		struct mptcp_tcp_sock *mptcp;
+
+		mptcp_for_each_sub(tp->mpcb, mptcp) {
+			sock_rps_record_flow(mptcp_to_sock(mptcp));
+		}
 	}
 
 	if (unlikely(tp->repair)) {
@@ -1994,9 +1999,11 @@ int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int nonblock,
 
 #ifdef CONFIG_MPTCP
 	if (mptcp(tp)) {
-		struct sock *sk_it;
-		mptcp_for_each_sk(tp->mpcb, sk_it)
-			sock_rps_record_flow(sk_it);
+		struct mptcp_tcp_sock *mptcp;
+
+		mptcp_for_each_sub(tp->mpcb, mptcp) {
+			sock_rps_record_flow(mptcp_to_sock(mptcp));
+		}
 	}
 #endif
 
net/mptcp/mptcp_ctrl.c +10 -4
index 052ae0e052b0..925d05d673d8 100644
@@ -553,10 +553,12 @@ void mptcp_hash_remove_bh(struct tcp_sock *meta_tp)
 struct sock *mptcp_select_ack_sock(const struct sock *meta_sk)
 {
 	const struct tcp_sock *meta_tp = tcp_sk(meta_sk);
-	struct sock *sk, *rttsk = NULL, *lastsk = NULL;
+	struct sock *rttsk = NULL, *lastsk = NULL;
 	u32 min_time = 0, last_active = 0;
+	struct mptcp_tcp_sock *mptcp;
 
-	mptcp_for_each_sk(meta_tp->mpcb, sk) {
+	mptcp_for_each_sub(meta_tp->mpcb, mptcp) {
+		struct sock *sk = mptcp_to_sock(mptcp);
 		struct tcp_sock *tp = tcp_sk(sk);
 		u32 elapsed;
 
@@ -689,7 +691,8 @@ void mptcp_sock_destruct(struct sock *sk)
 void mptcp_destroy_sock(struct sock *sk)
 {
 	if (is_meta_sk(sk)) {
-		struct sock *sk_it, *tmpsk;
+		struct mptcp_tcp_sock *mptcp;
+		struct hlist_node *tmp;
 
 		__skb_queue_purge(&tcp_sk(sk)->mpcb->reinject_queue);
 
@@ -699,7 +702,9 @@ void mptcp_destroy_sock(struct sock *sk)
 		 * not have been closed properly (as we are waiting for the
 		 * DATA_ACK of the DATA_FIN).
 		 */
-		mptcp_for_each_sk_safe(tcp_sk(sk)->mpcb, sk_it, tmpsk) {
+		mptcp_for_each_sub_safe(tcp_sk(sk)->mpcb, mptcp, tmp) {
+			struct sock *sk_it = mptcp_to_sock(mptcp);
+
 			/* Already did call tcp_close - waiting for graceful
 			 * closure, or if we are retransmitting fast-close on
 			 * the subflow. The reset (or timeout) will kill the
@@ -1281,6 +1286,7 @@ static int mptcp_alloc_mpcb(struct sock *meta_sk, __u64 remote_key,
 	INIT_LIST_HEAD(&mpcb->tw_list);
 
 	INIT_HLIST_HEAD(&mpcb->callback_list);
+	INIT_HLIST_HEAD(&mpcb->conn_list);
 	spin_lock_init(&mpcb->mpcb_list_lock);
 
 	mptcp_mpcb_inherit_sockopts(meta_sk, master_sk);
@@ -1370,8 +1376,12 @@ int mptcp_add_sock(struct sock *meta_sk, struct sock *sk, u8 loc_id, u8 rem_id,
 	sock_hold(meta_sk);
 	refcount_inc(&mpcb->mpcb_refcnt);
 
Quoted 11 lines

net/mptcp/mptcp_ctrl.c int mptcp_add_sock(struct sock *meta_sk, struct sock *sk, u8 loc_id, u8 rem_id, · line 1383

Quoted 7 lines
net/mptcp/mptcp_ctrl.c +5 -2
-	tp->mptcp->next = mpcb->connection_list;
-	mpcb->connection_list = tp;
+	local_bh_disable();
+	spin_lock(&mpcb->mpcb_list_lock);
+	hlist_add_head_rcu(&tp->mptcp->node, &mpcb->conn_list);
+	spin_unlock(&mpcb->mpcb_list_lock);
+	local_bh_enable();

I'm curious why the local_bh_disable is required here - I'm assuming it's possible to be called in softirq conext in this function but not where you delete from the list. A comment might help.

Quoted 8 lines

Yes, mptcp_add_sock() can be called from both contexts.

I just realize that mptcp_del_sock() can now also be called from both contexts. I need to disable bh there as well. spin_lock_bh() / spin_unlock_bh() would probably also be a better.

I will submit a separate patch to mptcp_trunk to fix and comment this. That way we have the same history on both.

Ok.

Quoted 43 lines
Quoted 34 lines
Quoted 34 lines
net/mptcp/mptcp_ctrl.c +2 -11
+
 	tp->mptcp->attached = 1;
 
 	atomic_add(atomic_read(&((struct sock *)tp)->sk_rmem_alloc),
@@ -1415,14 +1425,13 @@ int mptcp_add_sock(struct sock *meta_sk, struct sock *sk, u8 loc_id, u8 rem_id,
 
 void mptcp_del_sock(struct sock *sk)
 {
-	struct tcp_sock *tp = tcp_sk(sk), *tp_prev;
+	struct tcp_sock *tp = tcp_sk(sk);
 	struct mptcp_cb *mpcb;
 
 	if (!tp->mptcp || !tp->mptcp->attached)
 		return;
 
 	mpcb = tp->mpcb;
-	tp_prev = mpcb->connection_list;
 
 	if (mpcb->pm_ops->delete_subflow)
 		mpcb->pm_ops->delete_subflow(sk);
@@ -1431,17 +1440,10 @@ void mptcp_del_sock(struct sock *sk)
 		    __func__, mpcb->mptcp_loc_token, tp->mptcp->path_index,
 		    sk->sk_state, is_meta_sk(sk));
 
-	if (tp_prev == tp) {
-		mpcb->connection_list = tp->mptcp->next;
-	} else {
-		for (; tp_prev && tp_prev->mptcp->next; tp_prev = tp_prev->mptcp->next) {
-			if (tp_prev->mptcp->next == tp) {
-				tp_prev->mptcp->next = tp->mptcp->next;
-				break;
-			}
-		}
Quoted 8 lines

net/mptcp/mptcp_ctrl.c void mptcp_del_sock(struct sock *sk) · line 1445

Quoted 5 lines
net/mptcp/mptcp_ctrl.c +3 -2
-	}
-	tp->mptcp->next = NULL;
+	spin_lock(&mpcb->mpcb_list_lock);
+	hlist_del_init_rcu(&tp->mptcp->node);
+	spin_unlock(&mpcb->mpcb_list_lock);

Is it guaranteed that RCU synchronization happens between here and when the tcp_sock is freed? Do you need to set SOCK_RCU_FREE?

Quoted 8 lines

Looking more through this, I think we are good here.

The trick is that traversal of the list actually always happens while holding the MPTCP-level lock.

I needed to convert this to RCU so that we can add to the list (without holding MPTCP-level lock), while traversing it. Using the hlist guarantees that the traversal is going forward.

Thanks for that description - I was too focused on how RCU is often used to manage the lifetime of objects (deferring a free() until after the readers have released their read locks), but it's the thread-safe traversal aspect of RCU you're leveraging here. The lifetime of the subflow is managed as it was before the RCU changes.

Quoted 5 lines

I have some more patches coming now for mptcp_trunk to fix some of the fallout after this series here. How do you want to take these "forward-ports" ? Do you want me to submit everything (including this set here) at once to mptcp-net-next, or do you want to take the sets separately ?

I'll apply them all at the same time so mptcp-net-next can avoid any of its own fallout. A v2 patch set (these plus the new ones) rebased on the current mptcp-net-next would be best.

Mat

Quoted 595 lines

Thanks,
Christoph

Quoted 588 lines

Mat

Quoted 576 lines
net/mptcp/mptcp_ctrl.c +35 -15
+
 	tp->mptcp->attached = 0;
 	mpcb->path_index_bits &= ~(1 << tp->mptcp->path_index);
 
@@ -1488,8 +1490,8 @@ void mptcp_update_metasocket(const struct sock *meta_sk)
 void mptcp_cleanup_rbuf(struct sock *meta_sk, int copied)
 {
 	struct tcp_sock *meta_tp = tcp_sk(meta_sk);
-	struct sock *sk;
 	bool recheck_rcv_window = false;
+	struct mptcp_tcp_sock *mptcp;
 	__u32 rcv_window_now = 0;
 
 	if (copied > 0 && !(meta_sk->sk_shutdown & RCV_SHUTDOWN)) {
@@ -1500,7 +1502,8 @@ void mptcp_cleanup_rbuf(struct sock *meta_sk, int copied)
 			recheck_rcv_window = true;
 	}
 
-	mptcp_for_each_sk(meta_tp->mpcb, sk) {
+	mptcp_for_each_sub(meta_tp->mpcb, mptcp) {
+		struct sock *sk = mptcp_to_sock(mptcp);
 		struct tcp_sock *tp = tcp_sk(sk);
 		const struct inet_connection_sock *icsk = inet_csk(sk);
 
@@ -1687,10 +1690,13 @@ EXPORT_SYMBOL(mptcp_sub_force_close);
 */
 void mptcp_update_sndbuf(const struct tcp_sock *tp)
 {
-	struct sock *meta_sk = tp->meta_sk, *sk;
+	struct sock *meta_sk = tp->meta_sk;
 	int new_sndbuf = 0, old_sndbuf = meta_sk->sk_sndbuf;
+	struct mptcp_tcp_sock *mptcp;
+
+	mptcp_for_each_sub(tp->mpcb, mptcp) {
+		struct sock *sk = mptcp_to_sock(mptcp);
 
-	mptcp_for_each_sk(tp->mpcb, sk) {
 		if (!mptcp_sk_can_send(sk))
 			continue;
 
@@ -1719,8 +1725,8 @@ void mptcp_update_sndbuf(const struct tcp_sock *tp)
 void mptcp_close(struct sock *meta_sk, long timeout)
 {
 	struct tcp_sock *meta_tp = tcp_sk(meta_sk);
-	struct sock *sk_it, *tmpsk;
 	struct mptcp_cb *mpcb = meta_tp->mpcb;
+	struct mptcp_tcp_sock *mptcp;
 	struct sk_buff *skb;
 	int data_was_unread = 0;
 	int state;
@@ -1753,7 +1759,12 @@ void mptcp_close(struct sock *meta_sk, long timeout)
 
 	/* If socket has been already reset (e.g. in tcp_reset()) - kill it. */
 	if (meta_sk->sk_state == TCP_CLOSE) {
-		mptcp_for_each_sk_safe(mpcb, sk_it, tmpsk) {
+		struct mptcp_tcp_sock *mptcp;
+		struct hlist_node *tmp;
+
+		mptcp_for_each_sub_safe(mpcb, mptcp, tmp) {
+			struct sock *sk_it = mptcp_to_sock(mptcp);
+
 			if (tcp_sk(sk_it)->send_mp_fclose)
 				continue;
 			mptcp_sub_close(sk_it, 0);
@@ -1774,10 +1785,14 @@ void mptcp_close(struct sock *meta_sk, long timeout)
 	} else if (tcp_close_state(meta_sk)) {
 		mptcp_send_fin(meta_sk);
 	} else if (meta_tp->snd_una == meta_tp->write_seq) {
+		struct mptcp_tcp_sock *mptcp;
+		struct hlist_node *tmp;
+
 		/* The DATA_FIN has been sent and acknowledged
 		 * (e.g., by sk_shutdown). Close all the other subflows
 		 */
-		mptcp_for_each_sk_safe(mpcb, sk_it, tmpsk) {
+		mptcp_for_each_sub_safe(mpcb, mptcp, tmp) {
+			struct sock *sk_it = mptcp_to_sock(mptcp);
 			unsigned long delay = 0;
 			/* If we are the passive closer, don't trigger
 			 * subflow-fin until the subflow has been finned
@@ -1801,7 +1816,9 @@ void mptcp_close(struct sock *meta_sk, long timeout)
 	/* socket will be freed after mptcp_close - we have to prevent
 	 * access from the subflows.
 	 */
-	mptcp_for_each_sk(mpcb, sk_it) {
+	mptcp_for_each_sub(mpcb, mptcp) {
+		struct sock *sk_it = mptcp_to_sock(mptcp);
+
 		/* Similar to sock_orphan, but we don't set it DEAD, because
 		 * the callbacks are still set and must be called.
 		 */
@@ -1886,8 +1903,9 @@ void mptcp_close(struct sock *meta_sk, long timeout)
 
 void mptcp_disconnect(struct sock *sk)
 {
-	struct sock *subsk, *tmpsk;
+	struct mptcp_tcp_sock *mptcp;
 	struct tcp_sock *tp = tcp_sk(sk);
+	struct hlist_node *tmp;
 
 	__skb_queue_purge(&tp->mpcb->reinject_queue);
 
@@ -1895,7 +1913,9 @@ void mptcp_disconnect(struct sock *sk)
 		mptcp_hash_remove_bh(tp);
 
 	local_bh_disable();
-	mptcp_for_each_sk_safe(tp->mpcb, subsk, tmpsk) {
+	mptcp_for_each_sub_safe(tp->mpcb, mptcp, tmp) {
+		struct sock *subsk = mptcp_to_sock(mptcp);
+
 		/* The socket will get removed from the subsocket-list
 		 * and made non-mptcp by setting mpc to 0.
 		 *
@@ -2579,7 +2599,6 @@ static void mptcp_get_sub_info(struct sock *sk, struct mptcp_sub_info *info)
 int mptcp_get_info(const struct sock *meta_sk, char __user *optval, int optlen)
 {
 	const struct tcp_sock *meta_tp = tcp_sk(meta_sk);
-	struct sock *sk;
 
 	struct mptcp_meta_info meta_info;
 	struct mptcp_info m_info;
@@ -2625,16 +2644,17 @@ int mptcp_get_info(const struct sock *meta_sk, char __user *optval, int optlen)
 
 	if (m_info.subflows) {
 		unsigned int len, sub_len = 0;
+		struct mptcp_tcp_sock *mptcp;
 		char __user *ptr;
 
 		ptr = (char __user *)m_info.subflows;
 		len = m_info.sub_len;
 
-		mptcp_for_each_sk(meta_tp->mpcb, sk) {
+		mptcp_for_each_sub(meta_tp->mpcb, mptcp) {
 			struct tcp_info t_info;
 			unsigned int tmp_len;
 
-			tcp_get_info(sk, &t_info);
+			tcp_get_info(mptcp_to_sock(mptcp), &t_info);
 
 			tmp_len = min_t(unsigned int, len, info_len);
 			len -= tmp_len;
@@ -2654,6 +2674,7 @@ int mptcp_get_info(const struct sock *meta_sk, char __user *optval, int optlen)
 
 	if (m_info.subflow_info) {
 		unsigned int len, sub_info_len, total_sub_info_len = 0;
+		struct mptcp_tcp_sock *mptcp;
 		char __user *ptr;
 
 		ptr = (char __user *)m_info.subflow_info;
@@ -2663,11 +2684,11 @@ int mptcp_get_info(const struct sock *meta_sk, char __user *optval, int optlen)
 				     sizeof(struct mptcp_sub_info));
 		m_info.sub_info_len = sub_info_len;
 
-		mptcp_for_each_sk(meta_tp->mpcb, sk) {
+		mptcp_for_each_sub(meta_tp->mpcb, mptcp) {
 			struct mptcp_sub_info m_sub_info;
 			unsigned int tmp_len;
 
-			mptcp_get_sub_info(sk, &m_sub_info);
+			mptcp_get_sub_info(mptcp_to_sock(mptcp), &m_sub_info);
 
 			tmp_len = min_t(unsigned int, len, sub_info_len);
 			len -= tmp_len;
net/mptcp/mptcp_fullmesh.c +20 -8
index d29102a1dffb..53c1ba7319f5 100644
@@ -903,8 +903,9 @@ static void mptcp_address_worker(struct work_struct *work)
 			}
 
 			if (event->code == MPTCP_EVENT_DEL) {
-				struct sock *sk, *tmpsk;
+				struct mptcp_tcp_sock *mptcp;
 				struct mptcp_loc_addr *mptcp_local;
+				struct hlist_node *tmp;
 				bool found = false;
 
 				mptcp_local = rcu_dereference_bh(fm_ns->local);
@@ -914,7 +915,9 @@ static void mptcp_address_worker(struct work_struct *work)
 					update_addr_bitfields(meta_sk, mptcp_local);
 
 				/* Look for the socket and remove him */
-				mptcp_for_each_sk_safe(mpcb, sk, tmpsk) {
+				mptcp_for_each_sub_safe(mpcb, mptcp, tmp) {
+					struct sock *sk = mptcp_to_sock(mptcp);
+
 					if ((event->family == AF_INET6 &&
 					     (sk->sk_family == AF_INET ||
 					      mptcp_v6_is_v4_mapped(sk))) ||
@@ -964,9 +967,10 @@ static void mptcp_address_worker(struct work_struct *work)
 			}
 
 			if (event->code == MPTCP_EVENT_MOD) {
-				struct sock *sk;
+				struct mptcp_tcp_sock *mptcp;
 
-				mptcp_for_each_sk(mpcb, sk) {
+				mptcp_for_each_sub(mpcb, mptcp) {
+					struct sock *sk = mptcp_to_sock(mptcp);
 					struct tcp_sock *tp = tcp_sk(sk);
 					if (event->family == AF_INET &&
 					    (sk->sk_family == AF_INET ||
@@ -1455,8 +1459,9 @@ static void full_mesh_release_sock(struct sock *meta_sk)
 	struct mptcp_cb *mpcb = tcp_sk(meta_sk)->mpcb;
 	struct fullmesh_priv *fmp = fullmesh_get_priv(mpcb);
 	const struct mptcp_fm_ns *fm_ns = fm_get_ns(sock_net(meta_sk));
-	struct sock *sk, *tmpsk;
 	bool meta_v4 = meta_sk->sk_family == AF_INET;
+	struct mptcp_tcp_sock *mptcp;
+	struct hlist_node *tmp;
 	int i;
 
 	rcu_read_lock_bh();
@@ -1470,7 +1475,8 @@ static void full_mesh_release_sock(struct sock *meta_sk)
 		struct in_addr ifa = mptcp_local->locaddr4[i].addr;
 		bool found = false;
 
-		mptcp_for_each_sk(mpcb, sk) {
+		mptcp_for_each_sub(mpcb, mptcp) {
+			struct sock *sk = mptcp_to_sock(mptcp);
 			struct tcp_sock *tp = tcp_sk(sk);
 
 			if (sk->sk_family == AF_INET6 &&
@@ -1491,6 +1497,8 @@ static void full_mesh_release_sock(struct sock *meta_sk)
 		}
 
 		if (!found) {
+			struct sock *sk;
+
 			fmp->add_addr++;
 			mpcb->addr_signal = 1;
 
@@ -1511,7 +1519,8 @@ static void full_mesh_release_sock(struct sock *meta_sk)
 		struct in6_addr ifa = mptcp_local->locaddr6[i].addr;
 		bool found = false;
 
-		mptcp_for_each_sk(mpcb, sk) {
+		mptcp_for_each_sub(mpcb, mptcp) {
+			struct sock *sk = mptcp_to_sock(mptcp);
 			struct tcp_sock *tp = tcp_sk(sk);
 
 			if (sk->sk_family == AF_INET ||
@@ -1532,6 +1541,8 @@ static void full_mesh_release_sock(struct sock *meta_sk)
 		}
 
 		if (!found) {
+			struct sock *sk;
+
 			fmp->add_addr++;
 			mpcb->addr_signal = 1;
 
@@ -1546,7 +1557,8 @@ static void full_mesh_release_sock(struct sock *meta_sk)
 #endif
 
 	/* Now, detect address-removals */
-	mptcp_for_each_sk_safe(mpcb, sk, tmpsk) {
+	mptcp_for_each_sub_safe(mpcb, mptcp, tmp) {
+		struct sock *sk = mptcp_to_sock(mptcp);
 		bool shall_remove = true;
 
 		if (sk->sk_family == AF_INET || mptcp_v6_is_v4_mapped(sk)) {
net/mptcp/mptcp_input.c +29 -15
index 211605347f70..64b0265bfd06 100644
@@ -126,12 +126,14 @@ static void mptcp_clean_rtx_queue(struct sock *meta_sk, u32 prior_snd_una)
 		tcp_rtx_queue_unlink(skb, meta_sk);
 
 		if (mptcp_is_data_fin(skb)) {
-			struct sock *sk_it, *sk_tmp;
+			struct mptcp_tcp_sock *mptcp;
+			struct hlist_node *tmp;
 
 			/* DATA_FIN has been acknowledged - now we can close
 			 * the subflows
 			 */
-			mptcp_for_each_sk_safe(mpcb, sk_it, sk_tmp) {
+			mptcp_for_each_sub_safe(mpcb, mptcp, tmp) {
+				struct sock *sk_it = mptcp_to_sock(mptcp);
 				unsigned long delay = 0;
 
 				/* If we are the passive closer, don't trigger
@@ -347,6 +349,7 @@ static int mptcp_verif_dss_csum(struct sock *sk)
 
 	/* Now, checksum must be 0 */
 	if (unlikely(csum_fold(csum_tcp))) {
+		struct mptcp_tcp_sock *mptcp;
 		struct sock *sk_it = NULL;
 
 		pr_err("%s csum is wrong: %#x tcp-seq %u dss_csum_added %d overflowed %d iterations %d\n",
@@ -362,7 +365,9 @@ static int mptcp_verif_dss_csum(struct sock *sk)
 		tp->mpcb->csum_cutoff_seq = tp->mptcp->map_data_seq;
 
 		/* Search for another subflow that is fully established */
-		mptcp_for_each_sk(tp->mpcb, sk_it) {
+		mptcp_for_each_sub(tp->mpcb, mptcp) {
+			sk_it = mptcp_to_sock(mptcp);
+
 			if (sk_it != sk &&
 			    tcp_sk(sk_it)->mptcp->fully_established)
 				break;
@@ -1308,12 +1313,15 @@ int mptcp_do_join_short(struct sk_buff *skb,
 */
 void mptcp_fin(struct sock *meta_sk)
 {
-	struct sock *sk = NULL, *sk_it;
+	struct sock *sk = NULL;
 	struct tcp_sock *meta_tp = tcp_sk(meta_sk);
 	struct mptcp_cb *mpcb = meta_tp->mpcb;
+	struct mptcp_tcp_sock *mptcp;
 	unsigned char state;
 
-	mptcp_for_each_sk(mpcb, sk_it) {
+	mptcp_for_each_sub(mpcb, mptcp) {
+		struct sock *sk_it = mptcp_to_sock(mptcp);
+
 		if (tcp_sk(sk_it)->mptcp->path_index == mpcb->dfin_path_index) {
 			sk = sk_it;
 			break;
@@ -1585,9 +1593,12 @@ void mptcp_clean_rtx_infinite(const struct sk_buff *skb, struct sock *sk)
 
 static void mptcp_send_reset_rem_id(const struct mptcp_cb *mpcb, u8 rem_id)
 {
-	struct sock *sk_it, *tmpsk;
+	struct mptcp_tcp_sock *mptcp;
+	struct hlist_node *tmp;
+
+	mptcp_for_each_sub_safe(mpcb, mptcp, tmp) {
+		struct sock *sk_it = mptcp_to_sock(mptcp);
 
-	mptcp_for_each_sk_safe(mpcb, sk_it, tmpsk) {
 		if (tcp_sk(sk_it)->mptcp->rem_id == rem_id) {
 			mptcp_reinject_data(sk_it, 0);
 			mptcp_send_reset(sk_it);
@@ -1892,13 +1903,15 @@ void tcp_parse_mptcp_options(const struct sk_buff *skb,
 bool mptcp_check_rtt(const struct tcp_sock *tp, int time)
 {
 	struct mptcp_cb *mpcb = tp->mpcb;
-	struct sock *sk;
+	struct mptcp_tcp_sock *mptcp;
 	u32 rtt_max = 0;
 
 	/* In MPTCP, we take the max delay across all flows,
 	 * in order to take into account meta-reordering buffers.
 	 */
-	mptcp_for_each_sk(mpcb, sk) {
+	mptcp_for_each_sub(mpcb, mptcp) {
+		struct sock *sk = mptcp_to_sock(mptcp);
+
 		if (!mptcp_sk_can_recv(sk))
 			continue;
 
@@ -2173,9 +2186,9 @@ bool mptcp_handle_options(struct sock *sk, const struct tcphdr *th,
 		if (mopt->saw_low_prio == 1) {
 			tp->mptcp->rcv_low_prio = mopt->low_prio;
 		} else {
-			struct sock *sk_it;
-			mptcp_for_each_sk(tp->mpcb, sk_it) {
-				struct mptcp_tcp_sock *mptcp = tcp_sk(sk_it)->mptcp;
+			struct mptcp_tcp_sock *mptcp;
+
+			mptcp_for_each_sub(tp->mpcb, mptcp) {
 				if (mptcp->rem_id == mopt->prio_addr_id)
 					mptcp->rcv_low_prio = mopt->low_prio;
 			}
@@ -2359,7 +2372,7 @@ bool mptcp_should_expand_sndbuf(const struct sock *sk)
 {
 	const struct sock *meta_sk = mptcp_meta_sk(sk);
 	const struct tcp_sock *meta_tp = tcp_sk(meta_sk);
-	const struct sock *sk_it;
+	const struct mptcp_tcp_sock *mptcp;
 
 	/* We circumvent this check in tcp_check_space, because we want to
 	 * always call sk_write_space. So, we reproduce the check here.
@@ -2385,8 +2398,9 @@ bool mptcp_should_expand_sndbuf(const struct sock *sk)
 	/* For MPTCP we look for a subsocket that could send data.
 	 * If we found one, then we update the send-buffer.
 	 */
-	mptcp_for_each_sk(meta_tp->mpcb, sk_it) {
-		struct tcp_sock *tp_it = tcp_sk(sk_it);
+	mptcp_for_each_sub(meta_tp->mpcb, mptcp) {
+		const struct sock *sk_it = mptcp_to_sock(mptcp);
+		const struct tcp_sock *tp_it = tcp_sk(sk_it);
 
 		if (!mptcp_sk_can_send(sk_it))
 			continue;
net/mptcp/mptcp_output.c +2 -1
index 4b88b32d5954..12733ba2c1e1 100644
@@ -644,7 +644,6 @@ int mptcp_write_wakeup(struct sock *meta_sk, int mib)
 {
 	struct tcp_sock *meta_tp = tcp_sk(meta_sk);
 	struct sk_buff *skb;
-	struct sock *sk_it;
 	int ans = 0;
 
 	if (meta_sk->sk_state == TCP_CLOSE)
@@ -701,17 +700,22 @@ int mptcp_write_wakeup(struct sock *meta_sk, int mib)
 
 		return 0;
 	} else {
+		struct mptcp_tcp_sock *mptcp;
+

window_probe:

		if (between(meta_tp->snd_up, meta_tp->snd_una + 1,
			    meta_tp->snd_una + 0xFFFF)) {
net/mptcp/mptcp_output.c +3 -1
-			mptcp_for_each_sk(meta_tp->mpcb, sk_it) {
+			mptcp_for_each_sub(meta_tp->mpcb, mptcp) {
+				struct sock *sk_it = mptcp_to_sock(mptcp);
+
				if (mptcp_sk_can_send_ack(sk_it))
					tcp_xmit_probe_skb(sk_it, 1, mib);
			}
		}

		/* At least one of the tcp_xmit_probe_skb's has to succeed */
net/mptcp/mptcp_output.c +2 -1
-		mptcp_for_each_sk(meta_tp->mpcb, sk_it) {
+		mptcp_for_each_sub(meta_tp->mpcb, mptcp) {
+			struct sock *sk_it = mptcp_to_sock(mptcp);
			int ret;

			if (!mptcp_sk_can_send_ack(sk_it))
net/mptcp/mptcp_output.c +18 -10
@@ -729,6 +733,7 @@ bool mptcp_write_xmit(struct sock *meta_sk, unsigned int mss_now, int nonagle,
 		     int push_one, gfp_t gfp)
 {
 	struct tcp_sock *meta_tp = tcp_sk(meta_sk), *subtp;
+	struct mptcp_tcp_sock *mptcp;
 	struct sock *subsk = NULL;
 	struct mptcp_cb *mpcb = meta_tp->mpcb;
 	struct sk_buff *skb;
@@ -853,7 +858,8 @@ bool mptcp_write_xmit(struct sock *meta_sk, unsigned int mss_now, int nonagle,
 			break;
 	}
 
-	mptcp_for_each_sk(mpcb, subsk) {
+	mptcp_for_each_sub(mpcb, mptcp) {
+		subsk = mptcp_to_sock(mptcp);
 		subtp = tcp_sk(subsk);
 
 		if (!(path_mask & mptcp_pi_to_flag(subtp->mptcp->path_index)))
@@ -1350,7 +1356,7 @@ void mptcp_send_active_reset(struct sock *meta_sk, gfp_t priority)
 	struct mptcp_cb *mpcb = meta_tp->mpcb;
 	struct sock *sk;
 
-	if (!mpcb->connection_list)
+	if (hlist_empty(&mpcb->conn_list))
 		return;
 
 	WARN_ON(meta_tp->send_mp_fclose);
@@ -1725,10 +1731,11 @@ void mptcp_select_initial_window(const struct sock *sk, int __space, __u32 mss,
 static inline u64 mptcp_calc_rate(const struct sock *meta_sk, unsigned int mss,
 				  unsigned int (*mss_cb)(struct sock *sk))
 {
-	struct sock *sk;
+	struct mptcp_tcp_sock *mptcp;
 	u64 rate = 0;
 
-	mptcp_for_each_sk(tcp_sk(meta_sk)->mpcb, sk) {
+	mptcp_for_each_sub(tcp_sk(meta_sk)->mpcb, mptcp) {
+		struct sock *sk = mptcp_to_sock(mptcp);
 		struct tcp_sock *tp = tcp_sk(sk);
 		int this_mss;
 		u64 this_rate;
@@ -1780,11 +1787,12 @@ static inline u64 mptcp_calc_rate(const struct sock *meta_sk, unsigned int mss,
 static unsigned int __mptcp_current_mss(const struct sock *meta_sk,
 					unsigned int (*mss_cb)(struct sock *sk))
 {
+	struct mptcp_tcp_sock *mptcp;
 	unsigned int mss = 0;
 	u64 rate = 0;
-	struct sock *sk;
 
-	mptcp_for_each_sk(tcp_sk(meta_sk)->mpcb, sk) {
+	mptcp_for_each_sub(tcp_sk(meta_sk)->mpcb, mptcp) {
+		struct sock *sk = mptcp_to_sock(mptcp);
 		int this_mss;
 		u64 this_rate;
 
@@ -1849,14 +1857,16 @@ int mptcp_select_size(const struct sock *meta_sk, bool first_skb, bool zc)
 
 int mptcp_check_snd_buf(const struct tcp_sock *tp)
 {
-	const struct sock *sk;
+	const struct mptcp_tcp_sock *mptcp;
 	u32 rtt_max = tp->srtt_us;
 	u64 bw_est;
 
 	if (!tp->srtt_us)
 		return tp->reordering + 1;
 
-	mptcp_for_each_sk(tp->mpcb, sk) {
+	mptcp_for_each_sub(tp->mpcb, mptcp) {
+		const struct sock *sk = mptcp_to_sock(mptcp);
+
 		if (!mptcp_sk_can_send(sk))
 			continue;
 
@@ -1874,11 +1884,13 @@ int mptcp_check_snd_buf(const struct tcp_sock *tp)
 unsigned int mptcp_xmit_size_goal(const struct sock *meta_sk, u32 mss_now,
 				  int large_allowed)
 {
-	struct sock *sk;
 	u32 xmit_size_goal = 0;
 
 	if (large_allowed && !tcp_sk(meta_sk)->mpcb->dss_csum) {
-		mptcp_for_each_sk(tcp_sk(meta_sk)->mpcb, sk) {
+		struct mptcp_tcp_sock *mptcp;
+
+		mptcp_for_each_sub(tcp_sk(meta_sk)->mpcb, mptcp) {
+			struct sock *sk = mptcp_to_sock(mptcp);
 			int this_size_goal;
 
 			if (!mptcp_sk_can_send(sk))
net/mptcp/mptcp_sched.c +15 -7
index a2543c60bd31..b440df7aae71 100644
@@ -135,9 +135,10 @@ static struct sock
 	u32 min_srtt = 0xffffffff;
 	bool found_unused = false;
 	bool found_unused_una = false;
-	struct sock *sk;
+	struct mptcp_tcp_sock *mptcp;
 
-	mptcp_for_each_sk(mpcb, sk) {
+	mptcp_for_each_sub(mpcb, mptcp) {
+		struct sock *sk = mptcp_to_sock(mptcp);
 		struct tcp_sock *tp = tcp_sk(sk);
 		bool unused = false;
 
@@ -219,7 +220,11 @@ static struct sock *get_available_subflow(struct sock *meta_sk,
 	/* Answer data_fin on same subflow!!! */
 	if (meta_sk->sk_shutdown & RCV_SHUTDOWN &&
 	    skb && mptcp_is_data_fin(skb)) {
-		mptcp_for_each_sk(mpcb, sk) {
+		struct mptcp_tcp_sock *mptcp;
+
+		mptcp_for_each_sub(mpcb, mptcp) {
+			sk = mptcp_to_sock(mptcp);
+
 			if (tcp_sk(sk)->mptcp->path_index == mpcb->dfin_path_index &&
 			    mptcp_is_available(sk, skb, zero_wnd_test))
 				return sk;
@@ -252,7 +257,7 @@ static struct sk_buff *mptcp_rcv_buf_optimization(struct sock *sk, int penal)
 {
 	struct sock *meta_sk;
 	const struct tcp_sock *tp = tcp_sk(sk);
-	struct tcp_sock *tp_it;
+	struct mptcp_tcp_sock *mptcp;
 	struct sk_buff *skb_head;
 	struct defsched_priv *dsp = defsched_get_priv(tp);
 
@@ -275,7 +280,9 @@ static struct sk_buff *mptcp_rcv_buf_optimization(struct sock *sk, int penal)
 		goto retrans;
 
 	/* Half the cwnd of the slow flow */
-	mptcp_for_each_tp(tp->mpcb, tp_it) {
+	mptcp_for_each_sub(tp->mpcb, mptcp) {
+		struct tcp_sock *tp_it = mptcp->tp;
+
 		if (tp_it != tp &&
 		    TCP_SKB_CB(skb_head)->path_mask & mptcp_pi_to_flag(tp_it->mptcp->path_index)) {
 			if (tp->srtt_us < tp_it->srtt_us && inet_csk((struct sock *)tp_it)->icsk_ca_state == TCP_CA_Open) {
@@ -298,7 +305,9 @@ static struct sk_buff *mptcp_rcv_buf_optimization(struct sock *sk, int penal)
 	/* Segment not yet injected into this path? Take it!!! */
 	if (!(TCP_SKB_CB(skb_head)->path_mask & mptcp_pi_to_flag(tp->mptcp->path_index))) {
 		bool do_retrans = false;
-		mptcp_for_each_tp(tp->mpcb, tp_it) {
+		mptcp_for_each_sub(tp->mpcb, mptcp) {
+			struct tcp_sock *tp_it = mptcp->tp;
+
 			if (tp_it != tp &&
 			    TCP_SKB_CB(skb_head)->path_mask & mptcp_pi_to_flag(tp_it->mptcp->path_index)) {
 				if (tp_it->snd_cwnd <= 4) {
--
Quoted 3 lines

2.16.2

--
Mat Martineau
Intel OTC

--
Mat Martineau
Intel OTC

author
#14 lore raw

PATCH 11/14 mptcp: Don't take meta-lock when receiving third ACK

In reply to Christoph Paasch

Now everything is ready to do this in a lockless way.

Simply, in tcp_v4(6)_rcv, stop taking the meta-level lock. When creating the initial subflow, we currently are also holding the meta-level lock. We need to drop this one as soon as we handover control to the subflow (aka., when exiting mptcp_check_req_master()).

Data-reception however still requires the meta-level lock. Thus, we need to take it again in tcp_child_process().

Signed-off-by Christoph Paasch <cpaasch(a)apple.com>
Signed-off-by Matthieu Baerts <matthieu.baerts(a)tessares.net>
(cherry picked from commit 4966859c2ad3309437caedcc78c948c78eb343f3)
---
 net/ipv4/tcp_ipv4.c      | 41 ++++-------------------------------------
 net/ipv4/tcp_minisocks.c |  9 +++++++--
 net/ipv6/tcp_ipv6.c      | 41 ++++-------------------------------------
 net/mptcp/mptcp_ctrl.c   | 13 +++++++++++++
 net/mptcp/mptcp_ipv4.c   |  5 ++++-
 net/mptcp/mptcp_ipv6.c   |  4 ++++
 6 files changed, 36 insertions(+), 77 deletions(-)
net/ipv4/tcp_ipv4.c +4 -37
index 31c757f862c0..3bec4f0c2c32 100644
@@ -1739,45 +1739,16 @@ int tcp_v4_rcv(struct sk_buff *skb)
 			inet_csk_reqsk_queue_drop_and_put(sk, req);
 			goto lookup;
 		}
+		if (unlikely(is_meta_sk(sk) && !mptcp_can_new_subflow(sk))) {
+			inet_csk_reqsk_queue_drop_and_put(sk, req);
+			goto lookup;
+		}
 		/* We own a reference on the listener, increase it again
 		 * as we might lose it too soon.
 		 */
 		sock_hold(sk);
 		refcounted = true;
 
-		if (is_meta_sk(sk)) {
-			bh_lock_sock(sk);
-
-			if (!mptcp_can_new_subflow(sk)) {
-				inet_csk_reqsk_queue_drop_and_put(sk, req);
-				bh_unlock_sock(sk);
-
-				goto discard_and_relse;
-			}
-
-			if (sock_owned_by_user(sk)) {
-				th = (const struct tcphdr *)skb->data;
-				iph = ip_hdr(skb);
-				tcp_v4_fill_cb(skb, iph, th);
-
-				skb->sk = sk;
-				if (unlikely(sk_add_backlog(sk, skb,
-							    sk->sk_rcvbuf + sk->sk_sndbuf))) {
-					reqsk_put(req);
-
-					bh_unlock_sock(sk);
-					__NET_INC_STATS(net, LINUX_MIB_TCPBACKLOGDROP);
-					goto discard_and_relse;
-				}
-
-				reqsk_put(req);
-				bh_unlock_sock(sk);
-				sock_put(sk);
-
-				return 0;
-			}
-		}
-
 		nsk = NULL;
 		if (!tcp_filter(sk, skb)) {
 			th = (const struct tcphdr *)skb->data;
@@ -1787,8 +1758,6 @@ int tcp_v4_rcv(struct sk_buff *skb)
 		}
 		if (!nsk) {
 			reqsk_put(req);
-			if (is_meta_sk(sk))
-				bh_unlock_sock(sk);
 			if (req_stolen) {
 				/* Another cpu got exclusive access to req
 				 * and created a full blown socket.
@@ -1804,8 +1773,6 @@ int tcp_v4_rcv(struct sk_buff *skb)
 		if (nsk == sk) {
 			reqsk_put(req);
 			tcp_v4_restore_cb(skb);
-			if (is_meta_sk(sk))
-				bh_unlock_sock(sk);
 		} else if (tcp_child_process(sk, nsk, skb)) {
 			tcp_v4_send_reset(nsk, skb);
 			goto discard_and_relse;
net/ipv4/tcp_minisocks.c +7 -2
index 94e7bfd8343a..1581b1991733 100644
@@ -899,6 +899,11 @@ int tcp_child_process(struct sock *parent, struct sock *child,
 	sk_mark_napi_id(child, skb);
 
 	tcp_segs_in(tcp_sk(child), skb);
+	/* The following will be removed when we allow lockless data-reception
+	 * on the subflows.
+	 */
+	if (mptcp(tcp_sk(child)))
+		bh_lock_sock(meta_sk);
 	if (!sock_owned_by_user(meta_sk)) {
 		ret = tcp_rcv_state_process(child, skb);
 		/* Wakeup parent, send SIGIO */
@@ -914,9 +919,9 @@ int tcp_child_process(struct sock *parent, struct sock *child,
 		__sk_add_backlog(meta_sk, skb);
 	}
 
+	bh_unlock_sock(child);
 	if (mptcp(tcp_sk(child)))
-		bh_unlock_sock(child);
-	bh_unlock_sock(meta_sk);
+		bh_unlock_sock(meta_sk);
 	sock_put(child);
 	return ret;
 }
net/ipv6/tcp_ipv6.c +4 -37

62 lines not on this page · raw message

net/mptcp/mptcp_ctrl.c +13 -0
index 925d05d673d8..4be0e5c063ca 100644
@@ -1333,6 +1333,7 @@ static u8 mptcp_set_new_pathindex(struct mptcp_cb *mpcb)
 	return i;
 }
 
+/* May be called without holding the meta-level lock */
 int mptcp_add_sock(struct sock *meta_sk, struct sock *sk, u8 loc_id, u8 rem_id,
 		   gfp_t flags)
 {
@@ -2109,6 +2110,11 @@ int mptcp_check_req_fastopen(struct sock *child, struct request_sock *req)
 	/* Handled by the master_sk */
 	tcp_sk(meta_sk)->fastopen_rsk = NULL;
 
+	/* Subflow establishment is now lockless, drop the lock here it will
+	 * be taken again in tcp_child_process().
+	 */
+	bh_unlock_sock(meta_sk);
+
 	return 0;
 }
 
@@ -2138,9 +2144,15 @@ int mptcp_check_req_master(struct sock *sk, struct sock *child,
 		inet_csk_reqsk_queue_add(sk, req, meta_sk);
 	}
 
+	/* Subflow establishment is now lockless, drop the lock here it will
+	 * be taken again in tcp_child_process().
+	 */
+	bh_unlock_sock(meta_sk);
+
 	return 0;
 }
 
+/* May be called without holding the meta-level lock */
 struct sock *mptcp_check_req_child(struct sock *meta_sk,
 				   struct sock *child,
 				   struct request_sock *req,
@@ -2383,6 +2395,7 @@ void mptcp_tsq_sub_deferred(struct sock *meta_sk)
 	}
 }
 
+/* May be called without holding the meta-level lock */
 void mptcp_join_reqsk_init(const struct mptcp_cb *mpcb,
 			   const struct request_sock *req,
 			   struct sk_buff *skb)
net/mptcp/mptcp_ipv4.c +4 -1
index 6fce833c6859..c808d10205f5 100644
@@ -96,6 +96,7 @@ static u32 mptcp_v4_cookie_init_seq(struct request_sock *req, const struct sock
 }
 #endif
 
+/* May be called without holding the meta-level lock */
 static int mptcp_v4_join_init_req(struct request_sock *req, const struct sock *meta_sk,
 				  struct sk_buff *skb, bool want_cookie)
 {
@@ -141,7 +142,9 @@ struct request_sock_ops mptcp_request_sock_ops __read_mostly = {
 	.syn_ack_timeout =	tcp_syn_ack_timeout,
 };
 
-/* Similar to: tcp_v4_conn_request */
+/* Similar to: tcp_v4_conn_request
+ * May be called without holding the meta-level lock
+ */
 static int mptcp_v4_join_request(struct sock *meta_sk, struct sk_buff *skb)
 {
 	return tcp_conn_request(&mptcp_request_sock_ops,
net/mptcp/mptcp_ipv6.c +4 -0
index bd6b16becc49..2d730379d5ed 100644
@@ -125,6 +125,7 @@ static u32 mptcp_v6_cookie_init_seq(struct request_sock *req, const struct sock
 }
 #endif
 
+/* May be called without holding the meta-level lock */
 static int mptcp_v6_join_init_req(struct request_sock *req, const struct sock *meta_sk,
 				  struct sk_buff *skb, bool want_cookie)
 {
@@ -170,6 +171,9 @@ struct request_sock_ops mptcp6_request_sock_ops __read_mostly = {
 	.syn_ack_timeout =	tcp_syn_ack_timeout,
 };
 
+/* Similar to: tcp_v6_conn_request
+ * May be called without holding the meta-level lock
+ */
 static int mptcp_v6_join_request(struct sock *meta_sk, struct sk_buff *skb)
 {
 	return tcp_conn_request(&mptcp6_request_sock_ops,
Signature · 1 line
2.16.2
author
#15 lore raw

PATCH 12/14 mptcp: Make mptcp_do_join_short and mptcp_lookup_join lockless

In reply to Christoph Paasch

Now we can also make the subflow-SYN+MP_JOIN reception lockless.

In mptcp_lookup_join and mptcp_do_join_short, we were checking for mptcp_can_new_subflow(), because it is not possible to call tcp_v4_send_reset() while holding two locks (the listener lock and the meta-lock when coming from mptcp_do_join_short) when in mptcp_v4_do_rcv. Now that we remove the locking, we can remove the check for mptcp_can_new_subflow().

The remaining changes simply skip taking the meta-level lock which also implies that we don't have to check for sock_owned_by_user().

Signed-off-by Christoph Paasch <cpaasch(a)apple.com>
Signed-off-by Matthieu Baerts <matthieu.baerts(a)tessares.net>
(cherry picked from commit 005d56c200a227f04f41891e0a22b8fe4cff0889)
Signed-off-by Christoph Paasch <cpaasch(a)apple.com>
---
 net/mptcp/mptcp_input.c | 70 +++++++------------------------------------------
 1 file changed, 9 insertions(+), 61 deletions(-)
net/mptcp/mptcp_input.c +9 -61
index 64b0265bfd06..34cc613ee584 100644
@@ -1151,7 +1151,6 @@ struct mp_join *mptcp_find_join(const struct sk_buff *skb)
 
 int mptcp_lookup_join(struct sk_buff *skb, struct inet_timewait_sock *tw)
 {
-	const struct mptcp_cb *mpcb;
 	struct sock *meta_sk;
 	u32 token;
 	bool meta_v4;
@@ -1184,16 +1183,6 @@ int mptcp_lookup_join(struct sk_buff *skb, struct inet_timewait_sock *tw)
 		return -1;
 	}
 
-	mpcb = tcp_sk(meta_sk)->mpcb;
-	if (!mptcp_can_new_subflow(meta_sk)) {
-		/* We are in fallback-mode on the reception-side -
-		 * no new subflows!
-		 */
-		sock_put(meta_sk); /* Taken by mptcp_hash_find */
-		MPTCP_INC_STATS(sock_net(meta_sk), MPTCP_MIB_JOINFALLBACK);
-		return -1;
-	}
-
 	/* Coming from time-wait-sock processing in tcp_v4_rcv.
 	 * We have to deschedule it before continuing, because otherwise
 	 * mptcp_v4_do_rcv will hit again on it inside tcp_v4_hnd_req.
@@ -1204,26 +1193,13 @@ int mptcp_lookup_join(struct sk_buff *skb, struct inet_timewait_sock *tw)
 	/* OK, this is a new syn/join, let's create a new open request and
 	 * send syn+ack
 	 */
-	bh_lock_sock_nested(meta_sk);
-	if (sock_owned_by_user(meta_sk)) {
-		skb->sk = meta_sk;
-		if (unlikely(sk_add_backlog(meta_sk, skb,
-					    meta_sk->sk_rcvbuf + meta_sk->sk_sndbuf))) {
-			bh_unlock_sock(meta_sk);
-			__NET_INC_STATS(sock_net(meta_sk),
-					 LINUX_MIB_TCPBACKLOGDROP);
-			sock_put(meta_sk); /* Taken by mptcp_hash_find */
-			kfree_skb(skb);
-			return 1;
-		}
-	} else if (skb->protocol == htons(ETH_P_IP)) {
+	if (skb->protocol == htons(ETH_P_IP)) {
 		tcp_v4_do_rcv(meta_sk, skb);
 #if IS_ENABLED(CONFIG_IPV6)
 	} else {
 		tcp_v6_do_rcv(meta_sk, skb);
 #endif /* CONFIG_IPV6 */
 	}
-	bh_unlock_sock(meta_sk);
 	sock_put(meta_sk); /* Taken by mptcp_hash_find */
 	return 1;
 }
@@ -1260,48 +1236,20 @@ int mptcp_do_join_short(struct sk_buff *skb,
 	/* OK, this is a new syn/join, let's create a new open request and
 	 * send syn+ack
 	 */
-	bh_lock_sock(meta_sk);
 
-	/* This check is also done in mptcp_vX_do_rcv. But, there we cannot
-	 * call tcp_vX_send_reset, because we hold already two socket-locks.
-	 * (the listener and the meta from above)
-	 *
-	 * And the send-reset will try to take yet another one (ip_send_reply).
-	 * Thus, we propagate the reset up to tcp_rcv_state_process.
+	/* mptcp_v4_do_rcv tries to free the skb - we prevent this, as
+	 * the skb will finally be freed by tcp_v4_do_rcv (where we are
+	 * coming from)
 	 */
-	if (!mptcp_can_new_subflow(meta_sk)) {
-		MPTCP_INC_STATS(sock_net(meta_sk), MPTCP_MIB_JOINFALLBACK);
-		bh_unlock_sock(meta_sk);
-		sock_put(meta_sk); /* Taken by mptcp_hash_find */
-		return -1;
-	}
-
-	if (sock_owned_by_user(meta_sk)) {
-		skb->sk = meta_sk;
-		if (unlikely(sk_add_backlog(meta_sk, skb,
-					    meta_sk->sk_rcvbuf + meta_sk->sk_sndbuf)))
-			__NET_INC_STATS(net, LINUX_MIB_TCPBACKLOGDROP);
-		else
-			/* Must make sure that upper layers won't free the
-			 * skb if it is added to the backlog-queue.
-			 */
-			skb_get(skb);
-	} else {
-		/* mptcp_v4_do_rcv tries to free the skb - we prevent this, as
-		 * the skb will finally be freed by tcp_v4_do_rcv (where we are
-		 * coming from)
-		 */
-		skb_get(skb);
-		if (skb->protocol == htons(ETH_P_IP)) {
-			tcp_v4_do_rcv(meta_sk, skb);
+	skb_get(skb);
+	if (skb->protocol == htons(ETH_P_IP)) {
+		tcp_v4_do_rcv(meta_sk, skb);
 #if IS_ENABLED(CONFIG_IPV6)
-		} else { /* IPv6 */
-			tcp_v6_do_rcv(meta_sk, skb);
+	} else { /* IPv6 */
+		tcp_v6_do_rcv(meta_sk, skb);
 #endif /* CONFIG_IPV6 */
-		}
 	}
 
-	bh_unlock_sock(meta_sk);
 	sock_put(meta_sk); /* Taken by mptcp_hash_find */
 	return 0;
 }
Signature · 1 line
2.16.2
author
#16 lore raw

PATCH 13/14 mptcp: Reset tsorted_anchor after call to tcp_fragment

In reply to Christoph Paasch

The reinject-queue is special in the sense that its segments get kfree'd right after sending. We need to make sure that the dst-pointer is NULL as otherwise during the freeing we will access garbage.

Fixes 8a1051d977cf ("Merge tag 'v4.15' into mptcp_trunk")
Signed-off-by Christoph Paasch <cpaasch(a)apple.com>
Signed-off-by Matthieu Baerts <matthieu.baerts(a)tessares.net>
(cherry picked from commit e4eeb8266879fd69407381751ddcd232d266d0b8)
Signed-off-by Christoph Paasch <cpaasch(a)apple.com>
---
 net/mptcp/mptcp_output.c | 6 ++++++
 1 file changed, 6 insertions(+)
net/mptcp/mptcp_output.c +6 -0
index 12733ba2c1e1..b8c69aa17a55 100644
@@ -634,6 +634,12 @@ static int mptcp_fragment(struct sock *meta_sk, enum tcp_queue tcp_queue,
 			if (undo)
 				tcp_adjust_pcount(meta_sk, skb, -undo);
 		}
+
+		/* tcp_fragment's call to sk_stream_alloc_skb initializes the
+		 * tcp_tsorted_anchor. We need to revert this as it clashes
+		 * with the refdst pointer.
+		 */
+		tcp_skb_tsorted_anchor_cleanup(buff);
 	}
 
 	return 0;
Signature · 1 line
2.16.2
author
#17 lore raw

PATCH 14/14 mptcp: Pick correct sk when iterating in mptcp_set_rto

In reply to Christoph Paasch

We are iterating with sk_it as the iterator, so pick this one!

Fixes 57192ffda92b ("mptcp: Correct initial meta-RTO estimation after 3-way handshake")
Signed-off-by Christoph Paasch <cpaasch(a)apple.com>
Signed-off-by Matthieu Baerts <matthieu.baerts(a)tessares.net>
(cherry picked from commit 92a6e05825aa069d409d188f42ab62aa9f83c167)
Signed-off-by Christoph Paasch <cpaasch(a)apple.com>
---
 include/net/mptcp.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
include/net/mptcp.h +1 -1
index 091e56a7b334..e3694185c31f 100644
@@ -1182,7 +1182,7 @@ static inline void mptcp_set_rto(struct sock *sk)
 	mptcp_for_each_sub(tp->mpcb, mptcp) {
 		struct sock *sk_it = mptcp_to_sock(mptcp);
 
-		if ((mptcp_sk_can_send(sk_it) || sk->sk_state == TCP_SYN_RECV) &&
+		if ((mptcp_sk_can_send(sk_it) || sk_it->sk_state == TCP_SYN_RECV) &&
 		    inet_csk(sk_it)->icsk_rto > max_rto)
 			max_rto = inet_csk(sk_it)->icsk_rto;
 	}
Signature · 1 line
2.16.2
#18 lore raw
In reply to Christoph Paasch
Quoted 19 lines · Christoph Paasch

Here are some forward-ports from mptcp_trunk to mptcp-net-next of the lockless subflows patches and the two bug-fixes.

Christoph Paasch (14):
mptcp: Cleanup
mptcp: Improved debugging
mptcp: Use correct tcp_queue when calling mptcp_fragment
mptcp: Use mptcp_can_new_subflow where possible
mptcp: Render mptcp_sub_inherit_sockopts lockless
mptcp: Make mptcp_set_new_pathindex lockless
mptcp: Remove cnt_subflows
mptcp: Remove cnt_established
mptcp: Rename tw_lock to mpcb_list_lock
mptcp: Make subflow-list an RCU-list
mptcp: Don't take meta-lock when receiving third ACK
mptcp: Make mptcp_do_join_short and mptcp_lookup_join lockless
mptcp: Reset tsorted_anchor after call to tcp_fragment
mptcp: Pick correct sk when iterating in mptcp_set_rto

Quoted 16 lines
include/net/mptcp.h          | 127 ++++++++++++-----------------
net/ipv4/af_inet.c           |   9 ++-
net/ipv4/ip_sockglue.c       |   7 +-
net/ipv4/tcp.c               |  31 +++++---
net/ipv4/tcp_ipv4.c          |  41 +---------
net/ipv4/tcp_minisocks.c     |   9 ++-
net/ipv6/tcp_ipv6.c          |  41 +---------
net/mptcp/mptcp_ctrl.c       | 186 ++++++++++++++++++++++++++++---------------
net/mptcp/mptcp_fullmesh.c   |  30 ++++---
net/mptcp/mptcp_input.c      | 150 ++++++++++++----------------------
net/mptcp/mptcp_ipv4.c       |  34 +++-----
net/mptcp/mptcp_ipv6.c       |  15 ++--
net/mptcp/mptcp_ndiffports.c |   2 +-
net/mptcp/mptcp_output.c     |  61 ++++++++++----
net/mptcp/mptcp_sched.c      |  32 ++++----
15 files changed, 371 insertions(+), 404 deletions(-)

Thanks Christoph. This series applied cleanly on top of the merge I pushed earlier today. I'm taking a look at the RCU-related changes to better understand them before I push to the public repo.

--
Mat Martineau
Intel OTC

author
#19 lore raw
Quoted 39 lines · Mat Martineau
Quoted 19 lines · Christoph Paasch

Here are some forward-ports from mptcp_trunk to mptcp-net-next of the lockless subflows patches and the two bug-fixes.

Christoph Paasch (14):
mptcp: Cleanup
mptcp: Improved debugging
mptcp: Use correct tcp_queue when calling mptcp_fragment
mptcp: Use mptcp_can_new_subflow where possible
mptcp: Render mptcp_sub_inherit_sockopts lockless
mptcp: Make mptcp_set_new_pathindex lockless
mptcp: Remove cnt_subflows
mptcp: Remove cnt_established
mptcp: Rename tw_lock to mpcb_list_lock
mptcp: Make subflow-list an RCU-list
mptcp: Don't take meta-lock when receiving third ACK
mptcp: Make mptcp_do_join_short and mptcp_lookup_join lockless
mptcp: Reset tsorted_anchor after call to tcp_fragment
mptcp: Pick correct sk when iterating in mptcp_set_rto

Quoted 16 lines
include/net/mptcp.h          | 127 ++++++++++++-----------------
net/ipv4/af_inet.c           |   9 ++-
net/ipv4/ip_sockglue.c       |   7 +-
net/ipv4/tcp.c               |  31 +++++---
net/ipv4/tcp_ipv4.c          |  41 +---------
net/ipv4/tcp_minisocks.c     |   9 ++-
net/ipv6/tcp_ipv6.c          |  41 +---------
net/mptcp/mptcp_ctrl.c       | 186 ++++++++++++++++++++++++++++---------------
net/mptcp/mptcp_fullmesh.c   |  30 ++++---
net/mptcp/mptcp_input.c      | 150 ++++++++++++----------------------
net/mptcp/mptcp_ipv4.c       |  34 +++-----
net/mptcp/mptcp_ipv6.c       |  15 ++--
net/mptcp/mptcp_ndiffports.c |   2 +-
net/mptcp/mptcp_output.c     |  61 ++++++++++----
net/mptcp/mptcp_sched.c      |  32 ++++----
15 files changed, 371 insertions(+), 404 deletions(-)
Quoted 3 lines

Thanks Christoph. This series applied cleanly on top of the merge I pushed earlier today. I'm taking a look at the RCU-related changes to better understand them before I push to the public repo.

Sounds good!

I have some other minor bug-fixes that I am working on that will go through mptcp_trunk as well.

Christoph