Forward-ports of lockless subflow establishment
Subsystems (4)
Christoph Paasch
cover Forward-ports of lockless subflow establishment
As the fixes have now landed in mptcp_trunk, here are the forward-ports of the lockless subflow establishment.
Tested it a bit and it seems to work fine.
Christoph Paasch (20):
mptcp: Cleanup
mptcp: Improved debugging
mptcp: Use correct tcp_queue when calling mptcp_fragment
mptcp: Use mptcp_can_new_subflow where possible
mptcp: Render mptcp_sub_inherit_sockopts lockless
mptcp: Make mptcp_set_new_pathindex lockless
mptcp: Remove cnt_subflows
mptcp: Remove cnt_established
mptcp: Rename tw_lock to mpcb_list_lock
mptcp: Make subflow-list an RCU-list
mptcp: Don't take meta-lock when receiving third ACK
mptcp: Make mptcp_do_join_short and mptcp_lookup_join lockless
mptcp: Reset tsorted_anchor after call to tcp_fragment
mptcp: Pick correct sk when iterating in mptcp_set_rto
mptcp: Build-Fix
mptcp: Reset sk_it to NULL as we keep on iterating
mptcp: Close remaining subflows when falling back in
mptcp_verif_dss_csum
mptcp: Restart subflow-selection when we force a re-evaluation
mptcp: Don't free skb when tcp_transmit_skb without cloning
mptcp: Use spin_lock_bh where appropriate for mpcb_list_lock
include/net/mptcp.h | 132 ++++++++++++++----------------- net/ipv4/af_inet.c | 9 ++- net/ipv4/ip_sockglue.c | 7 +- net/ipv4/tcp.c | 31 +++++--- net/ipv4/tcp_ipv4.c | 41 +--------- net/ipv4/tcp_minisocks.c | 9 ++- net/ipv6/tcp_ipv6.c | 41 +--------- net/mptcp/mptcp_ctrl.c | 184 ++++++++++++++++++++++++++++--------------- net/mptcp/mptcp_fullmesh.c | 30 ++++--- net/mptcp/mptcp_input.c | 154 +++++++++++++----------------------- net/mptcp/mptcp_ipv4.c | 34 +++----- net/mptcp/mptcp_ipv6.c | 15 ++-- net/mptcp/mptcp_ndiffports.c | 2 +- net/mptcp/mptcp_output.c | 65 ++++++++++----- net/mptcp/mptcp_sched.c | 43 +++++----- 15 files changed, 388 insertions(+), 409 deletions(-)
Signature · 1 line
2.16.2
Christoph Paasch
PATCH 1/20 mptcp: Cleanup
Just some basic cleanup, fixing some comments, renaming sk to meta_sk, and moving mptcp_finish_handshake out of mptcp_ipv4.c to mptcp_ctrl.c
--- include/net/mptcp.h | 8 ++++---- net/mptcp/mptcp_ctrl.c | 25 +++++++++++++++++++++---- net/mptcp/mptcp_fullmesh.c | 2 +- net/mptcp/mptcp_input.c | 7 ++----- net/mptcp/mptcp_ipv4.c | 26 ++++---------------------- net/mptcp/mptcp_ipv6.c | 8 ++++---- 6 files changed, 36 insertions(+), 40 deletions(-)
include/net/mptcp.h +4 -4
@@ -660,10 +660,10 @@ extern int sysctl_mptcp_syn_retries;extern struct workqueue_struct *mptcp_wq; -#define mptcp_debug(fmt, args...) \ - do { \ - if (unlikely(sysctl_mptcp_debug)) \ - pr_err(fmt, ##args); \ +#define mptcp_debug(fmt, args...) \ + do { \ + if (unlikely(sysctl_mptcp_debug)) \ + pr_err(fmt, ##args); \ } while (0)/* Iterates over all subflows */
net/mptcp/mptcp_ctrl.c +21 -4
@@ -1337,8 +1337,8 @@ void mptcp_fallback_meta_sk(struct sock *meta_sk)int mptcp_add_sock (struct sock *meta_sk,struct sock *sk,u8 loc_id,u8 rem_id,gfp_t flags) { -struct mptcp_cb *mpcb =tcp_sk (meta_sk)->mpcb; -struct tcp_sock *tp =tcp_sk (sk); +struct mptcp_cb *mpcb =tcp_sk (meta_sk)->mpcb; +struct tcp_sock *tp =tcp_sk (sk); tp->mptcp =kmem_cache_zalloc (mptcp_sock_cache, flags);if (!tp->mptcp)
@@ -2146,9 +2146,9 @@ struct sock *mptcp_check_req_child(struct sock *meta_sk,struct sk_buff *skb, const struct mptcp_options_received *mopt) { - struct tcp_sock *child_tp = tcp_sk(child); + const struct mptcp_cb *mpcb = tcp_sk(meta_sk)->mpcb;struct mptcp_request_sock *mtreq =mptcp_rsk (req); -struct mptcp_cb *mpcb =tcp_sk (meta_sk)->mpcb; +struct tcp_sock *child_tp =tcp_sk (child);u8 hash_mac_check[20 ]; child_tp->inside_tk_table =0 ;
@@ -2514,6 +2514,23 @@ int mptcp_conn_request(struct sock *sk, struct sk_buff *skb)return 0 ; } +int mptcp_finish_handshake (struct sock *child,struct sk_buff *skb) + __releases(&child->sk_lock.slock) +{ +int ret; + +/* We don't call tcp_child_process here, because we hold +* already the meta-sk-lock and are sure that it is not owned +* by the user. +*/ +tcp_sk (child)->segs_in +=max_t (u16,1 ,skb_shinfo (skb)->gso_segs); + ret =tcp_rcv_state_process (child, skb); +bh_unlock_sock (child); +sock_put (child); + +return ret; +} +static void __mptcp_get_info (const struct sock *meta_sk,struct mptcp_meta_info *info) {
net/mptcp/mptcp_fullmesh.c +1 -1
@@ -79,7 +79,7 @@ struct fullmesh_priv {u8 rem4_bits;u8 rem6_bits; -/* Are we established the additional subflows for primary pair? */ +/* Have we established the additional subflows for primary pair? */ u8 first_pair:1; };
net/mptcp/mptcp_input.c +2 -5
@@ -89,10 +89,7 @@ static inline int mptcp_tso_acked_reinject(const struct sock *meta_sk,return packets_acked; } -/** -* Cleans the meta-socket retransmission queue and the reinject-queue. -* @sk must be the metasocket. -*/ +/* Cleans the meta-socket retransmission queue and the reinject-queue. */ static void mptcp_clean_rtx_queue (struct sock *meta_sk,u32 prior_snd_una) {struct sk_buff *skb, *tmp, *next;
@@ -350,7 +347,7 @@ static int mptcp_verif_dss_csum(struct sock *sk)/* Now, checksum must be 0 */ if (unlikely (csum_fold (csum_tcp))) { -pr_err ("%s csum is wrong: %#x data_seq %u dss_csum_added %d overflowed %d iterations %d\n" , +pr_err ("%s csum is wrong: %#x tcp-seq %u dss_csum_added %d overflowed %d iterations %d\n" , __func__,csum_fold (csum_tcp),TCP_SKB_CB (last)->seq, dss_csum_added, overflowed, iter);
net/mptcp/mptcp_ipv4.c +4 -22
@@ -96,11 +96,11 @@ static u32 mptcp_v4_cookie_init_seq(struct request_sock *req, const struct sock }#endif -static int mptcp_v4_join_init_req (struct request_sock *req,const struct sock *sk, +static int mptcp_v4_join_init_req (struct request_sock *req,const struct sock *meta_sk,struct sk_buff *skb,bool want_cookie) {struct mptcp_request_sock *mtreq =mptcp_rsk (req); -const struct mptcp_cb *mpcb =tcp_sk (sk)->mpcb; +const struct mptcp_cb *mpcb =tcp_sk (meta_sk)->mpcb;union inet_addr addr;int loc_id;bool low_prio = false;
@@ -112,14 +112,14 @@ static int mptcp_v4_join_init_req(struct request_sock *req, const struct sock *s*/ mtreq->hash_entry.pprev =NULL ; - tcp_request_sock_ipv4_ops.init_req (req, sk, skb, want_cookie); + tcp_request_sock_ipv4_ops.init_req (req, meta_sk, skb, want_cookie); mtreq->mptcp_loc_nonce =mptcp_v4_get_nonce (ip_hdr (skb)->saddr,ip_hdr (skb)->daddr,tcp_hdr (skb)->source,tcp_hdr (skb)->dest); addr.ip =inet_rsk (req)->ir_loc_addr; - loc_id = mpcb->pm_ops->get_local_id (AF_INET , &addr,sock_net (sk), &low_prio); + loc_id = mpcb->pm_ops->get_local_id (AF_INET , &addr,sock_net (meta_sk), &low_prio);if (loc_id ==-1 )return -1 ; mtreq->loc_id = loc_id;
@@ -149,24 +149,6 @@ static int mptcp_v4_join_request(struct sock *meta_sk, struct sk_buff *skb) meta_sk, skb); } -int mptcp_finish_handshake (struct sock *child,struct sk_buff *skb) - __releases(&child->sk_lock.slock) -{ -int ret; - -/* We don't call tcp_child_process here, because we hold -* already the meta-sk-lock and are sure that it is not owned -* by the user. -*/ -tcp_sk (child)->segs_in +=max_t (u16,1 ,skb_shinfo (skb)->gso_segs); - ret =tcp_rcv_state_process (child, skb); -bh_unlock_sock (child); -sock_put (child); - -return ret; -} - -/* Similar to: tcp_v4_do_rcv * We only process join requests here. (either the SYN or the final ACK) */
net/mptcp/mptcp_ipv6.c +4 -4
@@ -125,11 +125,11 @@ static u32 mptcp_v6_cookie_init_seq(struct request_sock *req, const struct sock }#endif -static int mptcp_v6_join_init_req (struct request_sock *req,const struct sock *sk, +static int mptcp_v6_join_init_req (struct request_sock *req,const struct sock *meta_sk,struct sk_buff *skb,bool want_cookie) {struct mptcp_request_sock *mtreq =mptcp_rsk (req); -const struct mptcp_cb *mpcb =tcp_sk (sk)->mpcb; +const struct mptcp_cb *mpcb =tcp_sk (meta_sk)->mpcb;union inet_addr addr;int loc_id;bool low_prio = false;
@@ -141,14 +141,14 @@ static int mptcp_v6_join_init_req(struct request_sock *req, const struct sock *s*/ mtreq->hash_entry.pprev =NULL ; - tcp_request_sock_ipv6_ops.init_req (req, sk, skb, want_cookie); + tcp_request_sock_ipv6_ops.init_req (req, meta_sk, skb, want_cookie); mtreq->mptcp_loc_nonce =mptcp_v6_get_nonce (ipv6_hdr (skb)->saddr.s6_addr32,ipv6_hdr (skb)->daddr.s6_addr32,tcp_hdr (skb)->source,tcp_hdr (skb)->dest); addr.in6 =inet_rsk (req)->ir_v6_loc_addr; - loc_id = mpcb->pm_ops->get_local_id (AF_INET6 , &addr,sock_net (sk), &low_prio); + loc_id = mpcb->pm_ops->get_local_id (AF_INET6 , &addr,sock_net (meta_sk), &low_prio);if (loc_id ==-1 )return -1 ; mtreq->loc_id = loc_id;
Signature · 1 line
2.16.2
Christoph Paasch
PATCH 1/2 mptcp: avoid removing useful skbs from the reinject queue.
In reply to Christoph PaaschFrom: Patrick Havelange <patrick.havelange(a)tessares.net>
After inserting the skbs in the reinject queue, the code was checking if it could remove potentially existing segments that are now no longer necessary. One condition for being able to remove such skbs was missing, resulting in potentially useful skbs being removed (i.e. they were NON redundant by others skbs !) from the reinject queue. In fact this logic is similar to the one in tcp_data_queue_ofo().
--- net/mptcp/mptcp_output.c | 3 +++ 1 file changed, 3 insertions(+)
net/mptcp/mptcp_output.c +3 -0
@@ -277,6 +277,9 @@ static void __mptcp_reinject_data(struct sk_buff *orig_skb, struct sock *meta_skif (!after (end_seq,TCP_SKB_CB (skb1)->seq))break ; +if (before (end_seq,TCP_SKB_CB (skb1)->end_seq)) +break ; +__skb_unlink (skb1, &mpcb->reinject_queue);__kfree_skb (skb1); }
Signature · 1 line
2.16.2
Christoph Paasch
PATCH 2/20 mptcp: Improved debugging
In reply to Christoph PaaschAllow to print the value of ret when mptcp_add_sock fails.
--- net/mptcp/mptcp_ipv4.c | 3 ++- net/mptcp/mptcp_ipv6.c | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-)
net/mptcp/mptcp_ipv4.c +2 -1
@@ -281,7 +281,8 @@ int mptcp_init4_subsockets(struct sock *meta_sk, const struct mptcp_loc4 *loc,lockdep_set_class_and_name (&(sk)->sk_lock.slock, &meta_slock_key, meta_slock_key_name);lockdep_init_map (&(sk)->sk_lock.dep_map, meta_key_name, &meta_key,0 ); -if (mptcp_add_sock (meta_sk, sk, loc->loc4_id, rem->rem4_id,GFP_KERNEL )) { + ret =mptcp_add_sock (meta_sk, sk, loc->loc4_id, rem->rem4_id,GFP_KERNEL ); +if (ret) {net_err_ratelimited ("%s mptcp_add_sock failed ret: %d\n" , __func__, ret); goto error;
net/mptcp/mptcp_ipv6.c +2 -1
@@ -307,7 +307,8 @@ int mptcp_init6_subsockets(struct sock *meta_sk, const struct mptcp_loc6 *loc,lockdep_set_class_and_name (&(sk)->sk_lock.slock, &meta_slock_key, meta_slock_key_name);lockdep_init_map (&(sk)->sk_lock.dep_map, meta_key_name, &meta_key,0 ); -if (mptcp_add_sock (meta_sk, sk, loc->loc6_id, rem->rem6_id,GFP_KERNEL )) { + ret =mptcp_add_sock (meta_sk, sk, loc->loc6_id, rem->rem6_id,GFP_KERNEL ); +if (ret) {net_err_ratelimited ("%s mptcp_add_sock failed ret: %d\n" , __func__, ret); goto error;
Signature · 1 line
2.16.2
Christoph Paasch
PATCH 2/2 mptcp: Iterate over subflow-list while holding the lock in tcp_splice_read
In reply to Christoph PaaschSubflows can get removed from under our feet, thus we might be iterating on garbage here.
That can panic like:
[52899.160112] BUG: unable to handle kernel NULL pointer dereference at (null) [52899.160157] IP: tcp_splice_read+0x225/0x330 [52899.160166] PGD 8000000164ff8067 P4D 8000000164ff8067 PUD 163d67067 PMD 0 [52899.160189] Oops: 0000 [#1] SMP PTI [52899.160198] Modules linked in: binfmt_misc xt_REDIRECT nf_nat_redirect xt_statistic xt_mark ipt_MASQUERADE nf_nat_masquerade_ipv4 xt_connmark xt_nat xt_comment xt_geoip(O) xt_conntrack iptable_mangle iptable_nat nf_nat_ipv4 nf_nat iptable_filter sch_fq_codel nf_conntrack_tftp nf_conntrack_ipv4 nf_defrag_ipv4 nf_conntrack_proto_gre nf_conntrack_irc nf_conntrack_ftp nf_conntrack pcspkr tun it87 hwmon_vid vfat fat x86_pkg_temp_thermal intel_powerclamp coretemp kvm_intel kvm irqbypass crct10dif_pclmul crc32_pclmul ghash_clmulni_intel pcbc aesni_intel crypto_simd glue_helper cryptd iTCO_wdt i2c_designware_platform iTCO_vendor_support i2c_designware_core intel_cstate intel_rapl_perf idma64 i2c_i801 sg virt_dma pinctrl_sunrisepoint wmi pinctrl_intel acpi_pad intel_lpss_pci intel_lpss mei_me pcc_cpufreq [52899.160410] mfd_core intel_pch_thermal mei shpchp ip_tables xfs libcrc32c sd_mod crc32c_intel igb ptp sdhci_pci pps_core sdhci dca i915 ahci i2c_algo_bit mmc_core libahci drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops libata drm video [last unloaded: pcspkr] [52899.160495] CPU: 0 PID: 21938 Comm: redsocks Tainted: G O 4.14.64+ #1 [52899.160506] Hardware name: Default string Default string/Default string, BIOS 5.12 07/01/2018 [52899.160518] task: ffff880164d45e00 task.stack: ffffc90002824000 [52899.160536] RIP: 0010:tcp_splice_read+0x225/0x330 [52899.160546] RSP: 0018:ffffc90002827dd8 EFLAGS: 00010286 [52899.160558] RAX: 0000000000000000 RBX: ffff88015b965280 RCX: 0000000000100000 [52899.160569] RDX: ffff88015ba30180 RSI: ffffc90002827ee8 RDI: ffff8801164d82c0 [52899.160579] RBP: ffffc90002827e50 R08: 00000000ffffffff R09: 0000000000000000 [52899.160589] R10: ffff880163940f00 R11: ffff88015ba30180 R12: ffff8801164d82c0 [52899.160599] R13: ffff88015ba30180 R14: ffffc90002827ee8 R15: 0000000000000003 [52899.160611] FS: 00007fae07922740(0000) GS:ffff88016ec00000(0000) knlGS:0000000000000000 [52899.160624] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [52899.160634] CR2: 0000000000000000 CR3: 0000000164c9a006 CR4: 00000000003606f0 [52899.160646] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [52899.160656] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [52899.160664] Call Trace: [52899.160684] ? kmem_cache_free+0x1aa/0x1c0 [52899.160702] sock_splice_read+0x25/0x30 [52899.160719] do_splice_to+0x76/0x90 [52899.160735] SyS_splice+0x6fd/0x750 [52899.160750] ? syscall_trace_enter+0x1cd/0x2b0 [52899.160766] do_syscall_64+0x79/0x1b0 [52899.160784] entry_SYSCALL_64_after_hwframe+0x3d/0xa2 [52899.160795] RIP: 0033:0x7fae07213493 [52899.160804] RSP: 002b:00007fff48ffdfe8 EFLAGS: 00000246 ORIG_RAX: 0000000000000113 [52899.160819] RAX: ffffffffffffffda RBX: 00007fff48ffe040 RCX: 00007fae07213493 [52899.160829] RDX: 000000000000008b RSI: 0000000000000000 RDI: 0000000000000081 [52899.160839] RBP: 0000000000000081 R08: 0000000000100000 R09: 0000000000000003 [52899.160849] R10: 0000000000000000 R11: 0000000000000246 R12: 0000000002214d40 [52899.160859] R13: 0000000000000081 R14: 0000000000000002 R15: 00000000022124f0 [52899.160870] Code: ff ff 48 8b 83 d0 07 00 00 48 8b 00 48 85 c0 0f 84 33 fe ff ff 44 8b 05 7a eb d3 00 41 f7 d0 0f 1f 44 00 00 48 8b 80 e8 07 00 00 <48> 8b 00 48 85 c0 75 ec e9 10 fe ff ff 0f b6 43 12 3c 01 0f 85 [52899.161029] RIP: tcp_splice_read+0x225/0x330 RSP: ffffc90002827dd8 [52899.161038] CR2: 0000000000000000
Github-issue: https://github.com/multipath-tcp/mptcp/issues/279
--- net/ipv4/tcp.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-)
net/ipv4/tcp.c +8 -7
@@ -811,13 +811,6 @@ ssize_t tcp_splice_read(struct socket *sock, loff_t *ppos,sock_rps_record_flow (sk); -#ifdef CONFIG_MPTCP -if (mptcp (tcp_sk (sk))) { -struct sock *sk_it; -mptcp_for_each_sk (tcp_sk (sk)->mpcb, sk_it) -sock_rps_record_flow (sk_it); - } -#endif /* * We can't seek on a socket input */
@@ -828,6 +821,14 @@ ssize_t tcp_splice_read(struct socket *sock, loff_t *ppos,lock_sock (sk); +#ifdef CONFIG_MPTCP +if (mptcp (tcp_sk (sk))) { +struct sock *sk_it; +mptcp_for_each_sk (tcp_sk (sk)->mpcb, sk_it) +sock_rps_record_flow (sk_it); + } +#endif + timeo =sock_rcvtimeo (sk, sock->file->f_flags &O_NONBLOCK );while (tss.len) { ret =__tcp_splice_read (sk, &tss);
Signature · 1 line
2.16.2
Christoph Paasch
PATCH 3/20 mptcp: Use correct tcp_queue when calling mptcp_fragment
In reply to Christoph PaaschThe skb may come from the write-queue, the rtx-queue or the reinject queue.
When it comes from the rtx-queue, we need to give mptcp_fragment the right tcp_queue.
To handle the case when it comes from the reinject-queue, I will post a follow-up patch as this seems to be a bug in the stable release as well.
--- net/mptcp/mptcp_output.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-)
net/mptcp/mptcp_output.c +14 -3
@@ -578,6 +578,11 @@ static bool mptcp_skb_entail(struct sock *sk, struct sk_buff *skb, int reinject)/* Fragment an skb and update the mptcp meta-data. Due to reinject, we * might need to undo some operations done by tcp_fragment. +* +* Be careful, the skb may come from 3 different places: +* - The send-queue (tcp_queue == TCP_FRAG_IN_WRITE_QUEUE) +* - The retransmit-queue (tcp_queue == TCP_FRAG_IN_RTX_QUEUE) +* - The reinject-queue (reinject == -1) */ static int mptcp_fragment(struct sock *meta_sk, enum tcp_queue tcp_queue, struct sk_buff *skb, u32 len,
@@ -602,7 +607,10 @@ static int mptcp_fragment(struct sock *meta_sk, enum tcp_queue tcp_queue,if (ret)return ret; - buff = skb->next; +if (tcp_queue ==TCP_FRAG_IN_WRITE_QUEUE ) + buff = skb->next; +else + buff =skb_rb_next (skb); flags =TCP_SKB_CB (skb)->mptcp_flags;TCP_SKB_CB (skb)->mptcp_flags = flags & ~(MPTCPHDR_FIN );
@@ -735,6 +743,7 @@ bool mptcp_write_xmit(struct sock *meta_sk, unsigned int mss_now, int nonagle,while ((skb = mpcb->sched_ops->next_segment (meta_sk, &reinject, &subsk, &sublimit))) { +enum tcp_queue tcp_queue =TCP_FRAG_IN_WRITE_QUEUE ;unsigned int limit; WARN(TCP_SKB_CB(skb)->sacked, "sacked: %u reinject: %u",
@@ -750,6 +759,8 @@ bool mptcp_write_xmit(struct sock *meta_sk, unsigned int mss_now, int nonagle,__kfree_skb (skb);continue ; } + }else if (reinject ==-1 ) { + tcp_queue =TCP_FRAG_IN_RTX_QUEUE ; }/* If the segment was cloned (e.g. a meta retransmission),
@@ -812,7 +823,7 @@ bool mptcp_write_xmit(struct sock *meta_sk, unsigned int mss_now, int nonagle, limit =min (limit, sublimit);if (skb->len > limit && -unlikely (mptcp_fragment (meta_sk,TCP_FRAG_IN_WRITE_QUEUE , +unlikely (mptcp_fragment (meta_sk, tcp_queue, skb, limit, gfp, reinject)))break ;
@@ -1540,7 +1551,7 @@ int mptcp_retransmit_skb(struct sock *meta_sk, struct sk_buff *skb) TCP_NAGLE_OFF);if (skb->len > limit && -unlikely (mptcp_fragment (meta_sk,TCP_FRAG_IN_WRITE_QUEUE , skb, +unlikely (mptcp_fragment (meta_sk,TCP_FRAG_IN_RTX_QUEUE , skb, limit,GFP_ATOMIC ,0 ))) goto failed;
Signature · 1 line
2.16.2
Christoph Paasch
PATCH 4/20 mptcp: Use mptcp_can_new_subflow where possible
In reply to Christoph PaaschLet's reuse the function instead of implementing the same checks again.
--- net/mptcp/mptcp_input.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-)
net/mptcp/mptcp_input.c +2 -4
@@ -1171,7 +1171,7 @@ int mptcp_lookup_join(struct sk_buff *skb, struct inet_timewait_sock *tw) } mpcb =tcp_sk (meta_sk)->mpcb; -if (mpcb->infinite_mapping_rcv || mpcb->send_infinite_mapping) { +if (!mptcp_can_new_subflow (meta_sk)) {/* We are in fallback-mode on the reception-side - * no new subflows! */
@@ -1255,9 +1255,7 @@ int mptcp_do_join_short(struct sk_buff *skb,* And the send-reset will try to take yet another one (ip_send_reply). * Thus, we propagate the reset up to tcp_rcv_state_process. */ -if (tcp_sk (meta_sk)->mpcb->infinite_mapping_rcv || -tcp_sk (meta_sk)->mpcb->send_infinite_mapping || - meta_sk->sk_state ==TCP_CLOSE || !tcp_sk (meta_sk)->inside_tk_table) { +if (!mptcp_can_new_subflow (meta_sk)) {MPTCP_INC_STATS (sock_net (meta_sk),MPTCP_MIB_JOINFALLBACK );bh_unlock_sock (meta_sk);sock_put (meta_sk);/* Taken by mptcp_hash_find */
Signature · 1 line
2.16.2
Christoph Paasch
PATCH 5/20 mptcp: Render mptcp_sub_inherit_sockopts lockless
In reply to Christoph PaaschWe are accessing the meta-socket's TOS field twice. It could change in-between as we do this in a lockless manner.
Thus, let's make sure we read it only once.
--- net/mptcp/mptcp_ctrl.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-)
net/mptcp/mptcp_ctrl.c +6 -2
@@ -953,11 +953,15 @@ static void mptcp_mpcb_inherit_sockopts(struct sock *meta_sk, struct sock *masteinet_sk (master_sk)->recverr =0 ; } +/* Called without holding lock on meta_sk */ static void mptcp_sub_inherit_sockopts (const struct sock *meta_sk,struct sock *sub_sk) { +__u8 meta_tos; +/* IP_TOS also goes to the subflow. */ -if (inet_sk (sub_sk)->tos !=inet_sk (meta_sk)->tos) { -inet_sk (sub_sk)->tos =inet_sk (meta_sk)->tos; + meta_tos =READ_ONCE (inet_sk (meta_sk)->tos); +if (inet_sk (sub_sk)->tos != meta_tos) { +inet_sk (sub_sk)->tos = meta_tos; sub_sk->sk_priority = meta_sk->sk_priority;sk_dst_reset (sub_sk); }
Signature · 1 line
2.16.2
Christoph Paasch
PATCH 6/20 mptcp: Make mptcp_set_new_pathindex lockless
In reply to Christoph PaaschThere is quite a complex logic just for setting the next path-index. It also prevents us from making the operation lockless.
Let's simplify this and make it lockless through a simple test_and_set_bit.
--- include/net/mptcp.h | 34 +--------------------------------- net/mptcp/mptcp_ctrl.c | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 33 deletions(-)
include/net/mptcp.h +1 -33
@@ -326,9 +326,7 @@ struct mptcp_cb {u8 mptcp_pm[MPTCP_PM_SIZE ]__aligned(8) ;struct mptcp_pm_ops *pm_ops; -u32 path_index_bits; -/* Next pi to pick up in case a new path becomes available */ -u8 next_path_index; +unsigned long path_index_bits;__u8 mptcp_ver;
@@ -1248,36 +1246,6 @@ static inline bool mptcp_fallback_infinite(struct sock *sk, int flag)return false; } -/* Find the first index whose bit in the bit-field == 0 */ -static inline u8 mptcp_set_new_pathindex (struct mptcp_cb *mpcb) -{ -u8 base = mpcb->next_path_index; -int i; - -/* Start at 1, because 0 is reserved for the meta-sk */ -mptcp_for_each_bit_unset (mpcb->path_index_bits >> base, i) { -if (i + base <1 ) -continue ; -if (i + base >=sizeof (mpcb->path_index_bits) *8 ) -break ; - i += base; - mpcb->path_index_bits |= (1 << i); - mpcb->next_path_index = i +1 ; -return i; - } -mptcp_for_each_bit_unset (mpcb->path_index_bits, i) { -if (i >=sizeof (mpcb->path_index_bits) *8 ) -break ; -if (i <1 ) -continue ; - mpcb->path_index_bits |= (1 << i); - mpcb->next_path_index = i +1 ; -return i; - } - -return 0 ; -} -static inline bool mptcp_v6_is_v4_mapped (const struct sock *sk) { return sk->sk_family == AF_INET6 &&
net/mptcp/mptcp_ctrl.c +16 -0
@@ -1338,6 +1338,22 @@ void mptcp_fallback_meta_sk(struct sock *meta_sk)kmem_cache_free (mptcp_cb_cache,tcp_sk (meta_sk)->mpcb); } +/* Called without holding lock on mpcb */ +static u8 mptcp_set_new_pathindex (struct mptcp_cb *mpcb) +{ +int i; + +/* Start at 1, because 0 is reserved for the meta-sk */ +for (i =1 ; i <sizeof (mpcb->path_index_bits) *8 ; i++) { +if (!test_and_set_bit (i, &mpcb->path_index_bits)) +break ; + } + +if (i ==sizeof (mpcb->path_index_bits) *8 ) +return 0 ; +return i; +} +int mptcp_add_sock (struct sock *meta_sk,struct sock *sk,u8 loc_id,u8 rem_id,gfp_t flags) {
Signature · 1 line
2.16.2
Christoph Paasch
PATCH 7/20 mptcp: Remove cnt_subflows
In reply to Christoph PaaschOur goal is to make subflow establishment lockless. cnt_subflows would need to either become an atomic variable, or we can remove it.
Well, let's remove it! In the general case it is not needed at all. Some exotic schedulers need cnt_subflows, but we don't care if we add complexity in these schedulers.
Something special in mptcp_verif_dss_csum(). There the access was actually wrong. Simply because cnt_subflows > 1, it doesn't mean we have a working subflow. So, we fix that. I don't think this issue is big enough to backport it.
Finally mptcp_sched.c was also using cnt_subflows but merely for speeding up things to avoid going through the whole scheduler code when there is only one subflow. So, we can remove it and trade performance for a better design.
--- include/net/mptcp.h | 12 +++++++++++- net/mptcp/mptcp_ctrl.c | 15 ++++++--------- net/mptcp/mptcp_input.c | 11 ++++++++++- net/mptcp/mptcp_ndiffports.c | 2 +- net/mptcp/mptcp_output.c | 2 +- net/mptcp/mptcp_sched.c | 11 ----------- 6 files changed, 29 insertions(+), 24 deletions(-)
include/net/mptcp.h +11 -1
@@ -277,7 +277,6 @@ struct mptcp_cb { rcv_hiseq_index:1 ;/* Index in rcv_high_order of rcv_nxt */ /* socket count in this connection */ -u8 cnt_subflows;u8 cnt_established;#define MPTCP_SCHED_DATA_SIZE 8
@@ -1267,6 +1266,17 @@ static inline bool mptcp_can_new_subflow(const struct sock *meta_sk) !tcp_sk (meta_sk)->mpcb->send_infinite_mapping; } +static inline int mptcp_subflow_count (const struct mptcp_cb *mpcb) +{ +struct sock *sk; +int i =0 ; + +mptcp_for_each_sk (mpcb, sk) + i++; + +return i; +} +/* TCP and MPTCP mpc flag-depending functions */ u16 mptcp_select_window (struct sock *sk);void mptcp_tcp_set_rto (struct sock *sk);
net/mptcp/mptcp_ctrl.c +6 -9
@@ -1401,7 +1401,6 @@ int mptcp_add_sock(struct sock *meta_sk, struct sock *sk, u8 loc_id, u8 rem_id, mpcb->connection_list = tp; tp->mptcp->attached =1 ; - mpcb->cnt_subflows++;atomic_add (atomic_read (&((struct sock *)tp)->sk_rmem_alloc), &meta_sk->sk_rmem_alloc);
@@ -1421,23 +1420,21 @@ int mptcp_add_sock(struct sock *meta_sk, struct sock *sk, u8 loc_id, u8 rem_id, sk->sk_destruct = mptcp_sock_destruct;if (sk->sk_family ==AF_INET ) -mptcp_debug ("%s: token %#x pi %d, src_addr:%pI4:%d dst_addr:%pI4:%d, cnt_subflows now %d\n" , +mptcp_debug ("%s: token %#x pi %d, src_addr:%pI4:%d dst_addr:%pI4:%d\n" , __func__ , mpcb->mptcp_loc_token, tp->mptcp->path_index, &((struct inet_sock *)tp)->inet_saddr,ntohs (((struct inet_sock *)tp)->inet_sport), &((struct inet_sock *)tp)->inet_daddr, -ntohs (((struct inet_sock *)tp)->inet_dport), - mpcb->cnt_subflows); +ntohs (((struct inet_sock *)tp)->inet_dport));#if IS_ENABLED (CONFIG_IPV6 ) else - mptcp_debug("%s: token %#x pi %d, src_addr:%pI6:%d dst_addr:%pI6:%d, cnt_subflows now %d\n", + mptcp_debug("%s: token %#x pi %d, src_addr:%pI6:%d dst_addr:%pI6:%d\n", __func__ , mpcb->mptcp_loc_token, tp->mptcp->path_index, &inet6_sk (sk)->saddr,ntohs (((struct inet_sock *)tp)->inet_sport), &sk->sk_v6_daddr, -ntohs (((struct inet_sock *)tp)->inet_dport), - mpcb->cnt_subflows); + ntohs(((struct inet_sock *)tp)->inet_dport));#endif return 0 ;
@@ -1471,7 +1468,6 @@ void mptcp_del_sock(struct sock *sk) } } } - mpcb->cnt_subflows--;if (tp->mptcp->establish_increased) mpcb->cnt_established--;
@@ -2840,8 +2836,9 @@ static int mptcp_pm_seq_show(struct seq_file *seq, void *v) ntohs(isk->inet_dport));#endif } + seq_printf(seq, " %02X %02X %08X:%08X %lu", - meta_sk->sk_state, mpcb->cnt_subflows, + meta_sk->sk_state, mptcp_subflow_count(mpcb), meta_tp->write_seq - meta_tp->snd_una, max_t(int, meta_tp->rcv_nxt - meta_tp->copied_seq, 0),
net/mptcp/mptcp_input.c +10 -1
@@ -347,6 +347,8 @@ static int mptcp_verif_dss_csum(struct sock *sk)/* Now, checksum must be 0 */ if (unlikely (csum_fold (csum_tcp))) { +struct sock *sk_it =NULL ; +pr_err ("%s csum is wrong: %#x tcp-seq %u dss_csum_added %d overflowed %d iterations %d\n" , __func__,csum_fold (csum_tcp),TCP_SKB_CB (last)->seq, dss_csum_added, overflowed, iter);
@@ -359,7 +361,14 @@ static int mptcp_verif_dss_csum(struct sock *sk)*/ tp->mpcb->csum_cutoff_seq = tp->mptcp->map_data_seq; -if (tp->mpcb->cnt_subflows >1 ) { +/* Search for another subflow that is fully established */ +mptcp_for_each_sk (tp->mpcb, sk_it) { +if (sk_it != sk && +tcp_sk (sk_it)->mptcp->fully_established) +break ; + } + +if (sk_it) {mptcp_send_reset (sk); ans =-1 ; }else {
net/mptcp/mptcp_ndiffports.c +1 -1
@@ -53,7 +53,7 @@ static void create_subflow_worker(struct work_struct *work) !tcp_sk(mpcb->master_sk)->mptcp->fully_established) goto exit; -if (num_subflows > iter && num_subflows > mpcb->cnt_subflows) { +if (num_subflows > iter && num_subflows >mptcp_subflow_count (mpcb)) {if (meta_sk->sk_family ==AF_INET ||mptcp_v6_is_v4_mapped (meta_sk)) {struct mptcp_loc4 loc;
net/mptcp/mptcp_output.c +1 -1
@@ -1353,7 +1353,7 @@ void mptcp_send_active_reset(struct sock *meta_sk, gfp_t priority)struct mptcp_cb *mpcb = meta_tp->mpcb;struct sock *sk; -if (!mpcb->cnt_subflows) +if (!mpcb->connection_list)return ;WARN_ON (meta_tp->send_mp_fclose);
net/mptcp/mptcp_sched.c +0 -11
@@ -216,14 +216,6 @@ static struct sock *get_available_subflow(struct sock *meta_sk,struct sock *sk;bool force; -/* if there is only one subflow, bypass the scheduling function */ -if (mpcb->cnt_subflows ==1 ) { - sk = (struct sock *)mpcb->connection_list; -if (!mptcp_is_available (sk, skb, zero_wnd_test)) - sk =NULL ; -return sk; - } -/* Answer data_fin on same subflow!!! */ if (meta_sk->sk_shutdown &RCV_SHUTDOWN && skb &&mptcp_is_data_fin (skb)) {
@@ -264,9 +256,6 @@ static struct sk_buff *mptcp_rcv_buf_optimization(struct sock *sk, int penal)struct sk_buff *skb_head;struct defsched_priv *dsp =defsched_get_priv (tp); -if (tp->mpcb->cnt_subflows ==1 ) -return NULL ; - meta_sk =mptcp_meta_sk (sk); skb_head =tcp_rtx_queue_head (meta_sk);
Signature · 1 line
2.16.2
Christoph Paasch
PATCH 8/20 mptcp: Remove cnt_established
In reply to Christoph PaaschWe want to remove the number of up-calls into the meta-socket. cnt_established is one of those where we are modifying a field in the mpcb.
We can remove it fairly safely. It's only use is in some MPTCP-specific congestion controls to short-circuit some parts of it.
One particular use was in mptcp_should_expand_sndbuf(). There, we use it to see whether to expand the sndbuf when there are only backup subflows available.
One problem with this old code is that the decision whether or not to send on a backup subflow is steered by the modular scheduler. We cannot assume in core-MPTCP code that backup subflows are never being sent on.
By no more taking backup subflows into account, we greatly simplify mptcp_should_expand_sndbuf at the cost of potentially increasing the window even if there is no subflow ready to send because it's a backup subflow.
--- include/net/mptcp.h | 4 ---- net/mptcp/mptcp_ctrl.c | 8 -------- net/mptcp/mptcp_input.c | 20 ++------------------ 3 files changed, 2 insertions(+), 30 deletions(-)
include/net/mptcp.h +0 -4
@@ -166,7 +166,6 @@ struct mptcp_tcp_sock {u16 map_data_len; u16 slave_sk:1, fully_established:1, - establish_increased:1, second_packet:1, attached:1, send_mp_fail:1,
@@ -276,9 +275,6 @@ struct mptcp_cb { snd_hiseq_index:1 ,/* Index in snd_high_order of snd_nxt */ rcv_hiseq_index:1; /* Index in rcv_high_order of rcv_nxt */ -/* socket count in this connection */ -u8 cnt_established; -#define MPTCP_SCHED_DATA_SIZE 8u8 mptcp_sched[MPTCP_SCHED_DATA_SIZE ]__aligned(8) ;struct mptcp_sched_ops *sched_ops;
net/mptcp/mptcp_ctrl.c +0 -8
@@ -745,11 +745,6 @@ static void mptcp_set_state(struct sock *sk)tcp_sk (meta_sk)->lsndtime = tcp_jiffies32; } -if (sk->sk_state ==TCP_ESTABLISHED ) { -tcp_sk (sk)->mptcp->establish_increased =1 ; -tcp_sk (sk)->mpcb->cnt_established++; - } -if (sk->sk_state ==TCP_CLOSE ) {if (!sock_flag (sk,SOCK_DEAD ))mptcp_sub_close (sk,0 );
@@ -1468,9 +1463,6 @@ void mptcp_del_sock(struct sock *sk) } } } -if (tp->mptcp->establish_increased) - mpcb->cnt_established--; - tp->mptcp->next =NULL ; tp->mptcp->attached =0 ; mpcb->path_index_bits &= ~(1 << tp->mptcp->path_index);
net/mptcp/mptcp_input.c +2 -18
@@ -2357,11 +2357,9 @@ int mptcp_rcv_synsent_state_process(struct sock *sk, struct sock **skptr,/* Similar to tcp_should_expand_sndbuf */ bool mptcp_should_expand_sndbuf (const struct sock *sk) { -const struct sock *sk_it;const struct sock *meta_sk =mptcp_meta_sk (sk);const struct tcp_sock *meta_tp =tcp_sk (meta_sk); -int cnt_backups =0 ; -int backup_available =0 ; +const struct sock *sk_it;/* We circumvent this check in tcp_check_space, because we want to * always call sk_write_space. So, we reproduce the check here.
@@ -2393,24 +2391,10 @@ bool mptcp_should_expand_sndbuf(const struct sock *sk)if (!mptcp_sk_can_send (sk_it))continue ; -/* Backup-flows have to be counted - if there is no other -* subflow we take the backup-flow into account. -*/ -if (tp_it->mptcp->rcv_low_prio || tp_it->mptcp->low_prio) - cnt_backups++; - -if (tcp_packets_in_flight (tp_it) < tp_it->snd_cwnd) { -if (tp_it->mptcp->rcv_low_prio || tp_it->mptcp->low_prio) { - backup_available =1 ; -continue ; - } +if (tcp_packets_in_flight (tp_it) < tp_it->snd_cwnd)return true; - } } -/* Backup-flow is available for sending - update send-buffer */ -if (meta_tp->mpcb->cnt_established == cnt_backups && backup_available) -return true;return false; }
Signature · 1 line
2.16.2
Christoph Paasch
PATCH 9/20 mptcp: Rename tw_lock to mpcb_list_lock
In reply to Christoph PaaschThe lock is going to cover more than just the tw-list. Namely, the subflow-list stored in the mpcb.
--- include/net/mptcp.h | 4 +++- net/mptcp/mptcp_ctrl.c | 14 +++++++------- 2 files changed, 10 insertions(+), 8 deletions(-)
include/net/mptcp.h +3 -1
@@ -258,6 +258,9 @@ struct mptcp_cb {/* list of sockets that need a call to release_cb */ struct hlist_head callback_list; +/* Lock used for protecting the different rcu-lists of mptcp_cb */ +spinlock_t mpcb_list_lock; +/* High-order bits of 64-bit sequence numbers */ u32 snd_high_order[2 ];u32 rcv_high_order[2 ];
@@ -289,7 +292,6 @@ struct mptcp_cb {u64 infinite_rcv_seq;/***** Start of fields, used for connection closure */ -spinlock_t tw_lock;unsigned char mptw_state;u8 dfin_path_index;
net/mptcp/mptcp_ctrl.c +7 -7
@@ -673,14 +673,14 @@ void mptcp_sock_destruct(struct sock *sk)* update to the rcv_nxt of the time-wait-sock and remove * its reference to the mpcb. */ -spin_lock_bh (&mpcb->tw_lock); +spin_lock_bh (&mpcb->mpcb_list_lock);list_for_each_entry_rcu (mptw, &mpcb->tw_list, list) {list_del_rcu (&mptw->list); mptw->in_list =0 ;mptcp_mpcb_put (mpcb);rcu_assign_pointer (mptw->mpcb,NULL ); } -spin_unlock_bh (&mpcb->tw_lock); +spin_unlock_bh (&mpcb->mpcb_list_lock);mptcp_debug ("%s destroying meta-sk token %#x\n" , __func__,tcp_sk (sk)->mpcb->mptcp_loc_token);
@@ -1301,9 +1301,9 @@ static int mptcp_alloc_mpcb(struct sock *meta_sk, __u64 remote_key,/* Init time-wait stuff */ INIT_LIST_HEAD (&mpcb->tw_list); -spin_lock_init (&mpcb->tw_lock);INIT_HLIST_HEAD (&mpcb->callback_list); +spin_lock_init (&mpcb->mpcb_list_lock);mptcp_mpcb_inherit_sockopts (meta_sk, master_sk);
@@ -2276,10 +2276,10 @@ int mptcp_init_tw_sock(struct sock *sk, struct tcp_timewait_sock *tw) mptw->rcv_nxt++;rcu_assign_pointer (mptw->mpcb, mpcb); -spin_lock_bh (&mpcb->tw_lock); +spin_lock_bh (&mpcb->mpcb_list_lock);list_add_rcu (&mptw->list, &tp->mpcb->tw_list); mptw->in_list =1 ; -spin_unlock_bh (&mpcb->tw_lock); +spin_unlock_bh (&mpcb->mpcb_list_lock);return 0 ; }
@@ -2295,12 +2295,12 @@ void mptcp_twsk_destructor(struct tcp_timewait_sock *tw)* from the list and drop the ref properly. */ if (mpcb &&refcount_inc_not_zero (&mpcb->mpcb_refcnt)) { -spin_lock_bh (&mpcb->tw_lock); +spin_lock_bh (&mpcb->mpcb_list_lock);if (tw->mptcp_tw->in_list) {list_del_rcu (&tw->mptcp_tw->list); tw->mptcp_tw->in_list =0 ; } -spin_unlock_bh (&mpcb->tw_lock); +spin_unlock_bh (&mpcb->mpcb_list_lock);/* Twice, because we increased it above */ mptcp_mpcb_put (mpcb);
Signature · 1 line
2.16.2
Mat Martineau
Re: [MPTCP] [PATCH 09/20] mptcp: Rename tw_lock to mpcb_list_lock
Quoted 12 lines · Christoph Paasch
The lock is going to cover more than just the tw-list. Namely, the subflow-list stored in the mpcb.
Signed-off-by Christoph Paasch <cpaasch(a)apple.com>Signed-off-by Matthieu Baerts <matthieu.baerts(a)tessares.net>(cherry picked from commit 411ffb5c2b2a42d578f11b800c97f23db1cb77ee)Signed-off-by Christoph Paasch <cpaasch(a)apple.com>---
include/net/mptcp.h| 4 +++-net/mptcp/mptcp_ctrl.c| 14 +++++++-------2 files changed, 10 insertions(+), 8 deletions(-)
I haven't found any issues with the patch set up to this point, and will continue reviewing on Monday.
--
Mat Martineau
Intel OTC
Christoph Paasch
PATCH 10/20 mptcp: Make subflow-list an RCU-list
In reply to Christoph Paaschmptcp_add_sock() will now be called without holding the meta-level lock. However, mptcp_add_sock() wants to add the subflow to the meta-level list, thus we need to protect this by a lock. We use the mpcb_list_lock for that.
Now that we are locking during add/del, and want to allow lockless traversal of the list, this implies that we need to make it an RCU-list.
So, this patch transitions to the RCU hlist. The list-traversal macros (hlist_for_each_entry_rcu) require me to now pass the mptcp_tcp_sock to mptcp_for_each_*. So, I had to change all the places in the code where we call one of the list-traversal macros to adapt to this.
--- include/net/mptcp.h | 67 +++++++++++++++++----------------- net/ipv4/af_inet.c | 9 ++--- net/ipv4/ip_sockglue.c | 7 ++-- net/ipv4/tcp.c | 31 +++++++++------- net/mptcp/mptcp_ctrl.c | 89 ++++++++++++++++++++++++++++------------------ net/mptcp/mptcp_fullmesh.c | 28 ++++++++++----- net/mptcp/mptcp_input.c | 44 +++++++++++++++-------- net/mptcp/mptcp_output.c | 38 +++++++++++++------- net/mptcp/mptcp_sched.c | 21 +++++++---- 9 files changed, 207 insertions(+), 127 deletions(-)
include/net/mptcp.h +34 -33
@@ -156,7 +156,7 @@ struct mptcp_options_received { };struct mptcp_tcp_sock { -struct tcp_sock *next;/* Next subflow socket */ +struct hlist_node node;struct hlist_node cb_list;struct mptcp_options_received rx_opt;
@@ -254,7 +254,7 @@ struct mptcp_sched_ops {struct mptcp_cb {/* list of sockets in this multipath connection */ -struct tcp_sock *connection_list; +struct hlist_head conn_list;/* list of sockets that need a call to release_cb */ struct hlist_head callback_list;
@@ -309,7 +309,7 @@ struct mptcp_cb {/***** Start of fields, used for subflow establishment */ struct sock *meta_sk; -/* Master socket, also part of the connection_list, this +/* Master socket, also part of the conn_list, this * socket is the one that the application sees. */ struct sock *master_sk;
@@ -661,21 +661,17 @@ extern struct workqueue_struct *mptcp_wq; pr_err(fmt, ##args); \ } while (0) -/* Iterates over all subflows */ -#define mptcp_for_each_tp (mpcb, tp) \ - for ((tp) = (mpcb)->connection_list; (tp); (tp) = (tp)->mptcp->next) +static inline struct sock *mptcp_to_sock (const struct mptcp_tcp_sock *mptcp) +{ +return (struct sock *)mptcp->tp; +} -#define mptcp_for_each_sk (mpcb, sk) \ - for ((sk) = (struct sock *)(mpcb)->connection_list; \ - sk; \ - sk = (struct sock *)tcp_sk(sk)->mptcp->next) +#define mptcp_for_each_sub (__mpcb, __mptcp) \ + hlist_for_each_entry_rcu(__mptcp, &((__mpcb)->conn_list), node) -#define mptcp_for_each_sk_safe (__mpcb, __sk, __temp) \ - for (__sk = (struct sock *)(__mpcb)->connection_list, \ - __temp = __sk ? (struct sock *)tcp_sk(__sk)->mptcp->next : NULL; \ - __sk; \ - __sk = __temp, \ - __temp = __sk ? (struct sock *)tcp_sk(__sk)->mptcp->next : NULL) +/* Must be called with the appropriate lock held */ +#define mptcp_for_each_sub_safe (__mpcb, __mptcp, __tmp) \ + hlist_for_each_entry_safe(__mptcp, __tmp, &((__mpcb)->conn_list), node)/* Iterates over all bit set to 1 in a bitset */ #define mptcp_for_each_bit_set (b, i) \
@@ -923,12 +919,14 @@ struct request_sock *rev_mptcp_rsk(const struct mptcp_request_sock *req)static inline bool mptcp_can_sendpage (struct sock *sk) { -struct sock *sk_it; +struct mptcp_tcp_sock *mptcp;if (tcp_sk (sk)->mpcb->dss_csum)return false; -mptcp_for_each_sk (tcp_sk (sk)->mpcb, sk_it) { +mptcp_for_each_sub (tcp_sk (sk)->mpcb, mptcp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +if (!(sk_it->sk_route_caps &NETIF_F_SG ))return false; }
@@ -962,9 +960,12 @@ static inline void mptcp_send_reset(struct sock *sk)static inline void mptcp_sub_force_close_all (struct mptcp_cb *mpcb,struct sock *except) { -struct sock *sk_it, *tmp; +struct mptcp_tcp_sock *mptcp; +struct hlist_node *tmp; + +mptcp_for_each_sub_safe (mpcb, mptcp, tmp) { +struct sock *sk_it =mptcp_to_sock (mptcp); -mptcp_for_each_sk_safe (mpcb, sk_it, tmp) {if (sk_it != except)mptcp_send_reset (sk_it); }
@@ -1150,12 +1151,14 @@ static inline int mptcp_sk_can_send_ack(const struct sock *sk)static inline bool mptcp_can_sg (const struct sock *meta_sk) { -struct sock *sk; +struct mptcp_tcp_sock *mptcp;if (tcp_sk (meta_sk)->mpcb->dss_csum)return false; -mptcp_for_each_sk (tcp_sk (meta_sk)->mpcb, sk) { +mptcp_for_each_sub (tcp_sk (meta_sk)->mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp); +if (!mptcp_sk_can_send (sk))continue ;if (!(sk->sk_route_caps &NETIF_F_SG ))
@@ -1166,9 +1169,9 @@ static inline bool mptcp_can_sg(const struct sock *meta_sk)static inline void mptcp_set_rto (struct sock *sk) { -struct tcp_sock *tp =tcp_sk (sk); -struct sock *sk_it;struct inet_connection_sock *micsk =inet_csk (mptcp_meta_sk (sk)); +struct tcp_sock *tp =tcp_sk (sk); +struct mptcp_tcp_sock *mptcp;__u32 max_rto =0 ;/* We are in recovery-phase on the MPTCP-level. Do not update the
@@ -1177,7 +1180,9 @@ static inline void mptcp_set_rto(struct sock *sk)if (micsk->icsk_retransmits)return ; -mptcp_for_each_sk (tp->mpcb, sk_it) { +mptcp_for_each_sub (tp->mpcb, mptcp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +if ((mptcp_sk_can_send (sk_it) || sk->sk_state ==TCP_SYN_RECV ) &&inet_csk (sk_it)->icsk_rto > max_rto) max_rto =inet_csk (sk_it)->icsk_rto;
@@ -1266,10 +1271,10 @@ static inline bool mptcp_can_new_subflow(const struct sock *meta_sk)static inline int mptcp_subflow_count (const struct mptcp_cb *mpcb) { -struct sock *sk; +struct mptcp_tcp_sock *mptcp;int i =0 ; -mptcp_for_each_sk (mpcb, sk) +mptcp_for_each_sub (mpcb, mptcp) i++;return i;
@@ -1287,12 +1292,8 @@ bool mptcp_prune_ofo_queue(struct sock *sk);do { \ } while (0) -/* Without MPTCP, we just do one iteration -* over the only socket available. This assumes that -* the sk/tp arg is the socket in that case. - */ -#define mptcp_for_each_sk (mpcb, sk) -#define mptcp_for_each_sk_safe (__mpcb, __sk, __temp) +#define mptcp_for_each_sub (__mpcb, __mptcp) \ + if (0)#define MPTCP_INC_STATS (net, field) \ do { \
net/ipv4/af_inet.c +5 -4
@@ -744,13 +744,14 @@ int inet_accept(struct socket *sock, struct socket *newsock, int flags,sock_rps_record_flow (sk2);if (sk2->sk_protocol ==IPPROTO_TCP &&mptcp (tcp_sk (sk2))) { -struct sock *sk_it = sk2; +struct mptcp_tcp_sock *mptcp; -mptcp_for_each_sk (tcp_sk (sk2)->mpcb, sk_it) -sock_rps_record_flow (sk_it); +mptcp_for_each_sub (tcp_sk (sk2)->mpcb, mptcp) { +sock_rps_record_flow (mptcp_to_sock (mptcp)); + }if (tcp_sk (sk2)->mpcb->master_sk) { - sk_it =tcp_sk (sk2)->mpcb->master_sk; +struct sock *sk_it =tcp_sk (sk2)->mpcb->master_sk;write_lock_bh (&sk_it->sk_callback_lock); sk_it->sk_wq = newsock->wq;
net/ipv4/ip_sockglue.c +5 -2
@@ -756,8 +756,11 @@ static int do_ip_setsockopt(struct sock *sk, int level,sk_dst_reset (sk);/* Update TOS on mptcp subflow */ if (is_meta_sk (sk)) { -struct sock *sk_it; -mptcp_for_each_sk (tcp_sk (sk)->mpcb, sk_it) { +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (tcp_sk (sk)->mpcb, mptcp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +if (inet_sk (sk_it)->tos !=inet_sk (sk)->tos) {inet_sk (sk_it)->tos =inet_sk (sk)->tos; sk_it->sk_priority = sk->sk_priority;
net/ipv4/tcp.c +19 -12
@@ -823,9 +823,11 @@ ssize_t tcp_splice_read(struct socket *sock, loff_t *ppos,#ifdef CONFIG_MPTCP if (mptcp (tcp_sk (sk))) { -struct sock *sk_it; -mptcp_for_each_sk (tcp_sk (sk)->mpcb, sk_it) -sock_rps_record_flow (sk_it); +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (tcp_sk (sk)->mpcb, mptcp) { +sock_rps_record_flow (mptcp_to_sock (mptcp)); + } }#endif
@@ -993,7 +995,7 @@ ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset, }if (mptcp (tp)) { -struct sock *sk_it = sk; +struct mptcp_tcp_sock *mptcp;/* We must check this with socket-lock hold because we iterate * over the subflows.
@@ -1008,8 +1010,9 @@ ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset,return ret; } -mptcp_for_each_sk (tp->mpcb, sk_it) -sock_rps_record_flow (sk_it); +mptcp_for_each_sub (tp->mpcb, mptcp) { +sock_rps_record_flow (mptcp_to_sock (mptcp)); + } }sk_clear_bit (SOCKWQ_ASYNC_NOSPACE , sk);
@@ -1288,9 +1291,11 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size) }if (mptcp (tp)) { -struct sock *sk_it = sk; -mptcp_for_each_sk (tp->mpcb, sk_it) -sock_rps_record_flow (sk_it); +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (tp->mpcb, mptcp) { +sock_rps_record_flow (mptcp_to_sock (mptcp)); + } }if (unlikely (tp->repair)) {
@@ -2006,9 +2011,11 @@ int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int nonblock,#ifdef CONFIG_MPTCP if (mptcp (tp)) { -struct sock *sk_it; -mptcp_for_each_sk (tp->mpcb, sk_it) -sock_rps_record_flow (sk_it); +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (tp->mpcb, mptcp) { +sock_rps_record_flow (mptcp_to_sock (mptcp)); + } }#endif
net/mptcp/mptcp_ctrl.c +55 -34
236 lines not on this page · raw message
net/mptcp/mptcp_fullmesh.c +20 -8
@@ -903,8 +903,9 @@ static void mptcp_address_worker(struct work_struct *work) }if (event->code ==MPTCP_EVENT_DEL ) { -struct sock *sk, *tmpsk; +struct mptcp_tcp_sock *mptcp;struct mptcp_loc_addr *mptcp_local; +struct hlist_node *tmp;bool found = false; mptcp_local =rcu_dereference_bh (fm_ns->local);
@@ -914,7 +915,9 @@ static void mptcp_address_worker(struct work_struct *work)update_addr_bitfields (meta_sk, mptcp_local);/* Look for the socket and remove him */ -mptcp_for_each_sk_safe (mpcb, sk, tmpsk) { +mptcp_for_each_sub_safe (mpcb, mptcp, tmp) { +struct sock *sk =mptcp_to_sock (mptcp); + if ((event->family == AF_INET6 && (sk->sk_family == AF_INET || mptcp_v6_is_v4_mapped(sk))) ||
@@ -964,9 +967,10 @@ static void mptcp_address_worker(struct work_struct *work) }if (event->code ==MPTCP_EVENT_MOD ) { -struct sock *sk; +struct mptcp_tcp_sock *mptcp; -mptcp_for_each_sk (mpcb, sk) { +mptcp_for_each_sub (mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);struct tcp_sock *tp =tcp_sk (sk); if (event->family == AF_INET && (sk->sk_family == AF_INET ||
@@ -1455,8 +1459,9 @@ static void full_mesh_release_sock(struct sock *meta_sk)struct mptcp_cb *mpcb =tcp_sk (meta_sk)->mpcb;struct fullmesh_priv *fmp =fullmesh_get_priv (mpcb);const struct mptcp_fm_ns *fm_ns =fm_get_ns (sock_net (meta_sk)); -struct sock *sk, *tmpsk;bool meta_v4 = meta_sk->sk_family ==AF_INET ; +struct mptcp_tcp_sock *mptcp; +struct hlist_node *tmp;int i;rcu_read_lock_bh ();
@@ -1470,7 +1475,8 @@ static void full_mesh_release_sock(struct sock *meta_sk)struct in_addr ifa = mptcp_local->locaddr4[i].addr;bool found = false; -mptcp_for_each_sk (mpcb, sk) { +mptcp_for_each_sub (mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);struct tcp_sock *tp =tcp_sk (sk); if (sk->sk_family == AF_INET6 &&
@@ -1491,6 +1497,8 @@ static void full_mesh_release_sock(struct sock *meta_sk) }if (!found) { +struct sock *sk; + fmp->add_addr++; mpcb->addr_signal =1 ;
@@ -1511,7 +1519,8 @@ static void full_mesh_release_sock(struct sock *meta_sk)struct in6_addr ifa = mptcp_local->locaddr6[i].addr;bool found = false; -mptcp_for_each_sk (mpcb, sk) { +mptcp_for_each_sub (mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);struct tcp_sock *tp =tcp_sk (sk); if (sk->sk_family == AF_INET ||
@@ -1532,6 +1541,8 @@ static void full_mesh_release_sock(struct sock *meta_sk) }if (!found) { +struct sock *sk; + fmp->add_addr++; mpcb->addr_signal =1 ;
@@ -1546,7 +1557,8 @@ static void full_mesh_release_sock(struct sock *meta_sk)#endif /* Now, detect address-removals */ -mptcp_for_each_sk_safe (mpcb, sk, tmpsk) { +mptcp_for_each_sub_safe (mpcb, mptcp, tmp) { +struct sock *sk =mptcp_to_sock (mptcp);bool shall_remove = true;if (sk->sk_family ==AF_INET ||mptcp_v6_is_v4_mapped (sk)) {
net/mptcp/mptcp_input.c +29 -15
112 lines not on this page · raw message
net/mptcp/mptcp_output.c +25 -13
@@ -647,7 +647,6 @@ int mptcp_write_wakeup(struct sock *meta_sk, int mib) {struct tcp_sock *meta_tp =tcp_sk (meta_sk);struct sk_buff *skb; -struct sock *sk_it;int ans =0 ;if (meta_sk->sk_state ==TCP_CLOSE )
@@ -704,17 +703,22 @@ int mptcp_write_wakeup(struct sock *meta_sk, int mib)return 0 ; }else { +struct mptcp_tcp_sock *mptcp; + window_probe:if (between (meta_tp->snd_up, meta_tp->snd_una +1 , meta_tp->snd_una +0xFFFF )) { -mptcp_for_each_sk (meta_tp->mpcb, sk_it) { +mptcp_for_each_sub (meta_tp->mpcb, mptcp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +if (mptcp_sk_can_send_ack (sk_it))tcp_xmit_probe_skb (sk_it,1 , mib); } }/* At least one of the tcp_xmit_probe_skb's has to succeed */ -mptcp_for_each_sk (meta_tp->mpcb, sk_it) { +mptcp_for_each_sub (meta_tp->mpcb, mptcp) { +struct sock *sk_it =mptcp_to_sock (mptcp);int ret;if (!mptcp_sk_can_send_ack (sk_it))
@@ -732,6 +736,7 @@ bool mptcp_write_xmit(struct sock *meta_sk, unsigned int mss_now, int nonagle, int push_one, gfp_t gfp) {struct tcp_sock *meta_tp =tcp_sk (meta_sk), *subtp; +struct mptcp_tcp_sock *mptcp;struct sock *subsk =NULL ;struct mptcp_cb *mpcb = meta_tp->mpcb;struct sk_buff *skb;
@@ -856,7 +861,8 @@ bool mptcp_write_xmit(struct sock *meta_sk, unsigned int mss_now, int nonagle,break ; } -mptcp_for_each_sk (mpcb, subsk) { +mptcp_for_each_sub (mpcb, mptcp) { + subsk =mptcp_to_sock (mptcp); subtp =tcp_sk (subsk);if (!(path_mask &mptcp_pi_to_flag (subtp->mptcp->path_index)))
@@ -1353,7 +1359,7 @@ void mptcp_send_active_reset(struct sock *meta_sk, gfp_t priority)struct mptcp_cb *mpcb = meta_tp->mpcb;struct sock *sk; -if (!mpcb->connection_list) +if (hlist_empty (&mpcb->conn_list))return ;WARN_ON (meta_tp->send_mp_fclose);
@@ -1728,10 +1734,11 @@ void mptcp_select_initial_window(const struct sock *sk, int __space, __u32 mss,static inline u64 mptcp_calc_rate (const struct sock *meta_sk,unsigned int mss,unsigned int (*mss_cb)(struct sock *sk)) { -struct sock *sk; +struct mptcp_tcp_sock *mptcp;u64 rate =0 ; -mptcp_for_each_sk (tcp_sk (meta_sk)->mpcb, sk) { +mptcp_for_each_sub (tcp_sk (meta_sk)->mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);struct tcp_sock *tp =tcp_sk (sk);int this_mss;u64 this_rate;
@@ -1783,11 +1790,12 @@ static inline u64 mptcp_calc_rate(const struct sock *meta_sk, unsigned int mss,static unsigned int __mptcp_current_mss (const struct sock *meta_sk,unsigned int (*mss_cb)(struct sock *sk)) { +struct mptcp_tcp_sock *mptcp;unsigned int mss =0 ;u64 rate =0 ; -struct sock *sk; -mptcp_for_each_sk (tcp_sk (meta_sk)->mpcb, sk) { +mptcp_for_each_sub (tcp_sk (meta_sk)->mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);int this_mss;u64 this_rate;
@@ -1852,14 +1860,16 @@ int mptcp_select_size(const struct sock *meta_sk, bool first_skb, bool zc)int mptcp_check_snd_buf (const struct tcp_sock *tp) { -const struct sock *sk; +const struct mptcp_tcp_sock *mptcp;u32 rtt_max = tp->srtt_us;u64 bw_est;if (!tp->srtt_us)return tp->reordering +1 ; -mptcp_for_each_sk (tp->mpcb, sk) { +mptcp_for_each_sub (tp->mpcb, mptcp) { +const struct sock *sk =mptcp_to_sock (mptcp); +if (!mptcp_sk_can_send (sk))continue ;
@@ -1877,11 +1887,13 @@ int mptcp_check_snd_buf(const struct tcp_sock *tp)unsigned int mptcp_xmit_size_goal (const struct sock *meta_sk,u32 mss_now,int large_allowed) { -struct sock *sk;u32 xmit_size_goal =0 ;if (large_allowed && !tcp_sk (meta_sk)->mpcb->dss_csum) { -mptcp_for_each_sk (tcp_sk (meta_sk)->mpcb, sk) { +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (tcp_sk (meta_sk)->mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);int this_size_goal;if (!mptcp_sk_can_send (sk))
net/mptcp/mptcp_sched.c +15 -6
@@ -135,9 +135,10 @@ static struct socku32 min_srtt =0xffffffff ;bool found_unused = false;bool found_unused_una = false; -struct sock *sk; +struct mptcp_tcp_sock *mptcp; -mptcp_for_each_sk (mpcb, sk) { +mptcp_for_each_sub (mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);struct tcp_sock *tp =tcp_sk (sk);bool unused = false;
@@ -219,7 +220,11 @@ static struct sock *get_available_subflow(struct sock *meta_sk,/* Answer data_fin on same subflow!!! */ if (meta_sk->sk_shutdown &RCV_SHUTDOWN && skb &&mptcp_is_data_fin (skb)) { -mptcp_for_each_sk (mpcb, sk) { +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (mpcb, mptcp) { + sk =mptcp_to_sock (mptcp); +if (tcp_sk (sk)->mptcp->path_index == mpcb->dfin_path_index &&mptcp_is_available (sk, skb, zero_wnd_test))return sk;
@@ -252,7 +257,7 @@ static struct sk_buff *mptcp_rcv_buf_optimization(struct sock *sk, int penal) {struct sock *meta_sk;const struct tcp_sock *tp =tcp_sk (sk); -struct tcp_sock *tp_it; +struct mptcp_tcp_sock *mptcp;struct sk_buff *skb_head;struct defsched_priv *dsp =defsched_get_priv (tp);
@@ -275,7 +280,9 @@ static struct sk_buff *mptcp_rcv_buf_optimization(struct sock *sk, int penal) goto retrans;/* Half the cwnd of the slow flow */ -mptcp_for_each_tp (tp->mpcb, tp_it) { +mptcp_for_each_sub (tp->mpcb, mptcp) { +struct tcp_sock *tp_it = mptcp->tp; +if (tp_it != tp &&TCP_SKB_CB (skb_head)->path_mask &mptcp_pi_to_flag (tp_it->mptcp->path_index)) {if (tp->srtt_us < tp_it->srtt_us &&inet_csk ((struct sock *)tp_it)->icsk_ca_state == TCP_CA_Open) {
@@ -298,7 +305,9 @@ static struct sk_buff *mptcp_rcv_buf_optimization(struct sock *sk, int penal)/* Segment not yet injected into this path? Take it!!! */ if (!(TCP_SKB_CB (skb_head)->path_mask &mptcp_pi_to_flag (tp->mptcp->path_index))) {bool do_retrans = false; -mptcp_for_each_tp (tp->mpcb, tp_it) { +mptcp_for_each_sub (tp->mpcb, mptcp) { +struct tcp_sock *tp_it = mptcp->tp; +if (tp_it != tp &&TCP_SKB_CB (skb_head)->path_mask &mptcp_pi_to_flag (tp_it->mptcp->path_index)) {if (tp_it->snd_cwnd <=4 ) {
Signature · 1 line
2.16.2
Mat Martineau
Re: [MPTCP] [PATCH 10/20] mptcp: Make subflow-list an RCU-list
Quoted 67 lines · Christoph Paasch
mptcp_add_sock()will now be called without holding the meta-level lock. However,mptcp_add_sock()wants to add the subflow to the meta-level list, thus we need to protect this by a lock. We use thempcb_list_lockfor that.Now that we are locking during add/del, and want to allow lockless traversal of the list, this implies that we need to make it an RCU-list.
So, this patch transitions to the RCU hlist. The list-traversal macros (
hlist_for_each_entry_rcu) require me to now pass themptcp_tcp_sockto mptcp_for_each_*. So, I had to change all the places in the code where we call one of the list-traversal macros to adapt to this.Signed-off-by Christoph Paasch <cpaasch(a)apple.com>Signed-off-by Matthieu Baerts <matthieu.baerts(a)tessares.net>(cherry picked from commit 7a662b690069642c138da16ff9396e6826fd0a95)---
include/net/mptcp.h | 67 +++++++++++++++++----------------- net/ipv4/af_inet.c | 9 ++--- net/ipv4/ip_sockglue.c | 7 ++-- net/ipv4/tcp.c | 31 +++++++++------- net/mptcp/mptcp_ctrl.c | 89 ++++++++++++++++++++++++++++------------------ net/mptcp/mptcp_fullmesh.c | 28 ++++++++++----- net/mptcp/mptcp_input.c | 44 +++++++++++++++-------- net/mptcp/mptcp_output.c | 38 +++++++++++++------- net/mptcp/mptcp_sched.c | 21 +++++++---- 9 files changed, 207 insertions(+), 127 deletions(-)include/net/mptcp.h +4
-6@@ -156,7 +156,7 @@ struct mptcp_options_received { };struct mptcp_tcp_sock { -struct tcp_sock *next;/* Next subflow socket */ +struct hlist_node node;struct hlist_node cb_list;struct mptcp_options_received rx_opt;@@ -254,7 +254,7 @@ struct mptcp_sched_ops {struct mptcp_cb {/* list of sockets in this multipath connection */ -struct tcp_sock *connection_list; +struct hlist_head conn_list;/* list of sockets that need a call to release_cb */ struct hlist_head callback_list;@@ -309,7 +309,7 @@ struct mptcp_cb {/***** Start of fields, used for subflow establishment */ struct sock *meta_sk; -/* Master socket, also part of the connection_list, this +/* Master socket, also part of the conn_list, this * socket is the one that the application sees. */ struct sock *master_sk;@@ -661,21 +661,17 @@ extern struct workqueue_struct *mptcp_wq; pr_err(fmt, ##args); \ } while (0) -/* Iterates over all subflows */ -#define mptcp_for_each_tp (mpcb, tp) \ - for ((tp) = (mpcb)->connection_list; (tp); (tp) = (tp)->mptcp->next) +static inline struct sock *mptcp_to_sock (const struct mptcp_tcp_sock *mptcp)
include/net/mptcp.h extern struct workqueue_struct *mptcp_wq; · line 670
Quoted 10 lines
include/net/mptcp.h +5
-4+{ +return (struct sock *)mptcp->tp; +} -#define mptcp_for_each_sk (mpcb, sk) \ - for ((sk) = (struct sock *)(mpcb)->connection_list; \ - sk; \ - sk = (struct sock *)tcp_sk(sk)->mptcp->next) +#define mptcp_for_each_sub (__mpcb, __mptcp) \ + hlist_for_each_entry_rcu(__mptcp, &((__mpcb)->conn_list), node)
When I asked about rcu synchronization with an earlier version of this patch set, you mentioned that it didn't look like an issue because the lifetime of the subflow sockets was still managed under a lock - the main reason to use rcu here is to allow adding to the list while the MPTCP-level lock is held. That seems to also imply that rcu_read_lock/rcu_read_unlock are not required when using these list iteration macros. This is an unconventional use of rcu list entries, so it would be helpful to explain the expected use of the conn_list. I think the main things are the lack of rcu_read_lock during iteration, and which locks to hold during iterate, add, and delete. Maybe the macro could check the state of the lock when built for debug?
include/net/mptcp.h extern struct workqueue_struct *mptcp_wq; · line 672
Quoted 8 lines
include/net/mptcp.h +1
-6-#define mptcp_for_each_sk_safe (__mpcb, __sk, __temp) \ - for (__sk = (struct sock *)(__mpcb)->connection_list, \ - __temp = __sk ? (struct sock *)tcp_sk(__sk)->mptcp->next : NULL; \ - __sk; \ - __sk = __temp, \ - __temp = __sk ? (struct sock *)tcp_sk(__sk)->mptcp->next : NULL) +/* Must be called with the appropriate lock held */
If MPTCP-level socket lock is the appropriate lock, can that be specified here?
include/net/mptcp.h extern struct workqueue_struct *mptcp_wq; · line 674
Quoted 2 lines
include/net/mptcp.h +2
-0+#define mptcp_for_each_sub_safe (__mpcb, __mptcp, __tmp) \ + hlist_for_each_entry_safe(__mptcp, __tmp, &((__mpcb)->conn_list), node)
The comment just before mptcp_for_each_sub_safe seems to imply that the difference between the two macros is that one needs a lock and one doesn't, but isn't the second (_safe) macro intended for use when subflows are being removed from the list while iterating? Could you add a comment to clarify?
Thanks,
Mat
Quoted 890 lines
include/net/mptcp.h +22
-17/* Iterates over all bit set to 1 in a bitset */ #define mptcp_for_each_bit_set (b, i) \@@ -923,12 +919,14 @@ struct request_sock *rev_mptcp_rsk(const struct mptcp_request_sock *req)static inline bool mptcp_can_sendpage (struct sock *sk) { -struct sock *sk_it; +struct mptcp_tcp_sock *mptcp;if (tcp_sk (sk)->mpcb->dss_csum)return false; -mptcp_for_each_sk (tcp_sk (sk)->mpcb, sk_it) { +mptcp_for_each_sub (tcp_sk (sk)->mpcb, mptcp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +if (!(sk_it->sk_route_caps &NETIF_F_SG ))return false; }@@ -962,9 +960,12 @@ static inline void mptcp_send_reset(struct sock *sk)static inline void mptcp_sub_force_close_all (struct mptcp_cb *mpcb,struct sock *except) { -struct sock *sk_it, *tmp; +struct mptcp_tcp_sock *mptcp; +struct hlist_node *tmp; + +mptcp_for_each_sub_safe (mpcb, mptcp, tmp) { +struct sock *sk_it =mptcp_to_sock (mptcp); -mptcp_for_each_sk_safe (mpcb, sk_it, tmp) {if (sk_it != except)mptcp_send_reset (sk_it); }@@ -1150,12 +1151,14 @@ static inline int mptcp_sk_can_send_ack(const struct sock *sk)static inline bool mptcp_can_sg (const struct sock *meta_sk) { -struct sock *sk; +struct mptcp_tcp_sock *mptcp;if (tcp_sk (meta_sk)->mpcb->dss_csum)return false; -mptcp_for_each_sk (tcp_sk (meta_sk)->mpcb, sk) { +mptcp_for_each_sub (tcp_sk (meta_sk)->mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp); +if (!mptcp_sk_can_send (sk))continue ;if (!(sk->sk_route_caps &NETIF_F_SG ))@@ -1166,9 +1169,9 @@ static inline bool mptcp_can_sg(const struct sock *meta_sk)static inline void mptcp_set_rto (struct sock *sk) { -struct tcp_sock *tp =tcp_sk (sk); -struct sock *sk_it;struct inet_connection_sock *micsk =inet_csk (mptcp_meta_sk (sk)); +struct tcp_sock *tp =tcp_sk (sk); +struct mptcp_tcp_sock *mptcp;__u32 max_rto =0 ;/* We are in recovery-phase on the MPTCP-level. Do not update the @@ -1177,7 +1180,9 @@ static inline void mptcp_set_rto(struct sock *sk)if (micsk->icsk_retransmits)return ; -mptcp_for_each_sk (tp->mpcb, sk_it) { +mptcp_for_each_sub (tp->mpcb, mptcp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +if ((mptcp_sk_can_send (sk_it) || sk->sk_state ==TCP_SYN_RECV ) &&inet_csk (sk_it)->icsk_rto > max_rto) max_rto =inet_csk (sk_it)->icsk_rto;@@ -1266,10 +1271,10 @@ static inline bool mptcp_can_new_subflow(const struct sock *meta_sk)static inline int mptcp_subflow_count (const struct mptcp_cb *mpcb) { -struct sock *sk; +struct mptcp_tcp_sock *mptcp;int i =0 ; -mptcp_for_each_sk (mpcb, sk) +mptcp_for_each_sub (mpcb, mptcp) i++;return i;@@ -1287,12 +1292,8 @@ bool mptcp_prune_ofo_queue(struct sock *sk);do { \ } while (0) -/* Without MPTCP, we just do one iteration -* over the only socket available. This assumes that -* the sk/tp arg is the socket in that case. - */ -#define mptcp_for_each_sk (mpcb, sk) -#define mptcp_for_each_sk_safe (__mpcb, __sk, __temp) +#define mptcp_for_each_sub (__mpcb, __mptcp) \ + if (0)#define MPTCP_INC_STATS (net, field) \ do { \net/ipv4/af_inet.c +5
-4@@ -744,13 +744,14 @@ int inet_accept(struct socket *sock, struct socket *newsock, int flags,sock_rps_record_flow (sk2);if (sk2->sk_protocol ==IPPROTO_TCP &&mptcp (tcp_sk (sk2))) { -struct sock *sk_it = sk2; +struct mptcp_tcp_sock *mptcp; -mptcp_for_each_sk (tcp_sk (sk2)->mpcb, sk_it) -sock_rps_record_flow (sk_it); +mptcp_for_each_sub (tcp_sk (sk2)->mpcb, mptcp) { +sock_rps_record_flow (mptcp_to_sock (mptcp)); + }if (tcp_sk (sk2)->mpcb->master_sk) { - sk_it =tcp_sk (sk2)->mpcb->master_sk; +struct sock *sk_it =tcp_sk (sk2)->mpcb->master_sk;write_lock_bh (&sk_it->sk_callback_lock); sk_it->sk_wq = newsock->wq;net/ipv4/ip_sockglue.c +5
-2@@ -756,8 +756,11 @@ static int do_ip_setsockopt(struct sock *sk, int level,sk_dst_reset (sk);/* Update TOS on mptcp subflow */ if (is_meta_sk (sk)) { -struct sock *sk_it; -mptcp_for_each_sk (tcp_sk (sk)->mpcb, sk_it) { +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (tcp_sk (sk)->mpcb, mptcp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +if (inet_sk (sk_it)->tos !=inet_sk (sk)->tos) {inet_sk (sk_it)->tos =inet_sk (sk)->tos; sk_it->sk_priority = sk->sk_priority;net/ipv4/tcp.c +19
-12@@ -823,9 +823,11 @@ ssize_t tcp_splice_read(struct socket *sock, loff_t *ppos,#ifdef CONFIG_MPTCP if (mptcp (tcp_sk (sk))) { -struct sock *sk_it; -mptcp_for_each_sk (tcp_sk (sk)->mpcb, sk_it) -sock_rps_record_flow (sk_it); +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (tcp_sk (sk)->mpcb, mptcp) { +sock_rps_record_flow (mptcp_to_sock (mptcp)); + } }#endif @@ -993,7 +995,7 @@ ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset, }if (mptcp (tp)) { -struct sock *sk_it = sk; +struct mptcp_tcp_sock *mptcp;/* We must check this with socket-lock hold because we iterate * over the subflows. @@ -1008,8 +1010,9 @@ ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset,return ret; } -mptcp_for_each_sk (tp->mpcb, sk_it) -sock_rps_record_flow (sk_it); +mptcp_for_each_sub (tp->mpcb, mptcp) { +sock_rps_record_flow (mptcp_to_sock (mptcp)); + } }sk_clear_bit (SOCKWQ_ASYNC_NOSPACE , sk);@@ -1288,9 +1291,11 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size) }if (mptcp (tp)) { -struct sock *sk_it = sk; -mptcp_for_each_sk (tp->mpcb, sk_it) -sock_rps_record_flow (sk_it); +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (tp->mpcb, mptcp) { +sock_rps_record_flow (mptcp_to_sock (mptcp)); + } }if (unlikely (tp->repair)) {@@ -2006,9 +2011,11 @@ int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int nonblock,#ifdef CONFIG_MPTCP if (mptcp (tp)) { -struct sock *sk_it; -mptcp_for_each_sk (tp->mpcb, sk_it) -sock_rps_record_flow (sk_it); +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (tp->mpcb, mptcp) { +sock_rps_record_flow (mptcp_to_sock (mptcp)); + } }#endif net/mptcp/mptcp_ctrl.c +55
-34@@ -561,10 +561,12 @@ void mptcp_hash_remove_bh(struct tcp_sock *meta_tp)struct sock *mptcp_select_ack_sock (const struct sock *meta_sk) {const struct tcp_sock *meta_tp =tcp_sk (meta_sk); -struct sock *sk, *rttsk =NULL , *lastsk =NULL ; +struct sock *rttsk =NULL , *lastsk =NULL ;u32 min_time =0 , last_active =0 ; +struct mptcp_tcp_sock *mptcp; -mptcp_for_each_sk (meta_tp->mpcb, sk) { +mptcp_for_each_sub (meta_tp->mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);struct tcp_sock *tp =tcp_sk (sk);u32 elapsed;@@ -697,7 +699,8 @@ void mptcp_sock_destruct(struct sock *sk)void mptcp_destroy_sock (struct sock *sk) {if (is_meta_sk (sk)) { -struct sock *sk_it, *tmpsk; +struct mptcp_tcp_sock *mptcp; +struct hlist_node *tmp;__skb_queue_purge (&tcp_sk (sk)->mpcb->reinject_queue);@@ -707,7 +710,9 @@ void mptcp_destroy_sock(struct sock *sk)* not have been closed properly (as we are waiting for the * DATA_ACK of the DATA_FIN). */ -mptcp_for_each_sk_safe (tcp_sk (sk)->mpcb, sk_it, tmpsk) { +mptcp_for_each_sub_safe (tcp_sk (sk)->mpcb, mptcp, tmp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +/* Already did call tcp_close - waiting for graceful * closure, or if we are retransmitting fast-close on * the subflow. The reset (or timeout) will kill the @@ -1303,6 +1308,7 @@ static int mptcp_alloc_mpcb(struct sock *meta_sk, __u64 remote_key,INIT_LIST_HEAD (&mpcb->tw_list);INIT_HLIST_HEAD (&mpcb->callback_list); +INIT_HLIST_HEAD (&mpcb->conn_list);spin_lock_init (&mpcb->mpcb_list_lock);mptcp_mpcb_inherit_sockopts (meta_sk, master_sk);@@ -1392,8 +1398,12 @@ int mptcp_add_sock(struct sock *meta_sk, struct sock *sk, u8 loc_id, u8 rem_id,sock_hold (meta_sk);refcount_inc (&mpcb->mpcb_refcnt); - tp->mptcp->next = mpcb->connection_list; - mpcb->connection_list = tp; +local_bh_disable (); +spin_lock (&mpcb->mpcb_list_lock); +hlist_add_head_rcu (&tp->mptcp->node, &mpcb->conn_list); +spin_unlock (&mpcb->mpcb_list_lock); +local_bh_enable (); + tp->mptcp->attached =1 ; atomic_add(atomic_read(&((struct sock *)tp)->sk_rmem_alloc),@@ -1437,14 +1447,13 @@ int mptcp_add_sock(struct sock *meta_sk, struct sock *sk, u8 loc_id, u8 rem_id,void mptcp_del_sock (struct sock *sk) { -struct tcp_sock *tp =tcp_sk (sk), *tp_prev; +struct tcp_sock *tp =tcp_sk (sk);struct mptcp_cb *mpcb;if (!tp->mptcp || !tp->mptcp->attached)return ; mpcb = tp->mpcb; - tp_prev = mpcb->connection_list;if (mpcb->pm_ops->delete_subflow) mpcb->pm_ops->delete_subflow (sk);@@ -1453,17 +1462,10 @@ void mptcp_del_sock(struct sock *sk) __func__, mpcb->mptcp_loc_token, tp->mptcp->path_index, sk->sk_state, is_meta_sk(sk)); -if (tp_prev == tp) { - mpcb->connection_list = tp->mptcp->next; - }else { -for (; tp_prev && tp_prev->mptcp->next; tp_prev = tp_prev->mptcp->next) { -if (tp_prev->mptcp->next == tp) { - tp_prev->mptcp->next = tp->mptcp->next; -break ; - } - } - } - tp->mptcp->next =NULL ; +spin_lock (&mpcb->mpcb_list_lock); +hlist_del_init_rcu (&tp->mptcp->node); +spin_unlock (&mpcb->mpcb_list_lock); + tp->mptcp->attached =0 ; mpcb->path_index_bits &= ~(1 << tp->mptcp->path_index);@@ -1510,8 +1512,8 @@ void mptcp_update_metasocket(const struct sock *meta_sk)void mptcp_cleanup_rbuf (struct sock *meta_sk,int copied) {struct tcp_sock *meta_tp =tcp_sk (meta_sk); -struct sock *sk;bool recheck_rcv_window = false; +struct mptcp_tcp_sock *mptcp;__u32 rcv_window_now =0 ;if (copied >0 && !(meta_sk->sk_shutdown &RCV_SHUTDOWN )) {@@ -1522,7 +1524,8 @@ void mptcp_cleanup_rbuf(struct sock *meta_sk, int copied) recheck_rcv_window = true; } -mptcp_for_each_sk (meta_tp->mpcb, sk) { +mptcp_for_each_sub (meta_tp->mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);struct tcp_sock *tp =tcp_sk (sk);const struct inet_connection_sock *icsk =inet_csk (sk);@@ -1709,10 +1712,13 @@ EXPORT_SYMBOL(mptcp_sub_force_close);*/ void mptcp_update_sndbuf (const struct tcp_sock *tp) { -struct sock *meta_sk = tp->meta_sk, *sk; +struct sock *meta_sk = tp->meta_sk;int new_sndbuf =0 , old_sndbuf = meta_sk->sk_sndbuf; +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (tp->mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp); -mptcp_for_each_sk (tp->mpcb, sk) {if (!mptcp_sk_can_send (sk))continue ;@@ -1741,8 +1747,8 @@ void mptcp_update_sndbuf(const struct tcp_sock *tp)void mptcp_close (struct sock *meta_sk,long timeout) {struct tcp_sock *meta_tp =tcp_sk (meta_sk); -struct sock *sk_it, *tmpsk;struct mptcp_cb *mpcb = meta_tp->mpcb; +struct mptcp_tcp_sock *mptcp;struct sk_buff *skb;int data_was_unread =0 ;int state;@@ -1775,7 +1781,12 @@ void mptcp_close(struct sock *meta_sk, long timeout)/* If socket has been already reset (e.g. in tcp_reset()) - kill it. */ if (meta_sk->sk_state ==TCP_CLOSE ) { -mptcp_for_each_sk_safe (mpcb, sk_it, tmpsk) { +struct mptcp_tcp_sock *mptcp; +struct hlist_node *tmp; + +mptcp_for_each_sub_safe (mpcb, mptcp, tmp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +if (tcp_sk (sk_it)->send_mp_fclose)continue ;mptcp_sub_close (sk_it,0 );@@ -1796,10 +1807,14 @@ void mptcp_close(struct sock *meta_sk, long timeout) }else if (tcp_close_state (meta_sk)) {mptcp_send_fin (meta_sk); }else if (meta_tp->snd_una == meta_tp->write_seq) { +struct mptcp_tcp_sock *mptcp; +struct hlist_node *tmp; +/* The DATA_FIN has been sent and acknowledged * (e.g., by sk_shutdown). Close all the other subflows */ -mptcp_for_each_sk_safe (mpcb, sk_it, tmpsk) { +mptcp_for_each_sub_safe (mpcb, mptcp, tmp) { +struct sock *sk_it =mptcp_to_sock (mptcp);unsigned long delay =0 ;/* If we are the passive closer, don't trigger * subflow-fin until the subflow has been finned @@ -1823,7 +1838,9 @@ void mptcp_close(struct sock *meta_sk, long timeout)/* socket will be freed after mptcp_close - we have to prevent * access from the subflows. */ -mptcp_for_each_sk (mpcb, sk_it) { +mptcp_for_each_sub (mpcb, mptcp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +/* Similar to sock_orphan, but we don't set it DEAD, because * the callbacks are still set and must be called. */ @@ -1908,8 +1925,9 @@ void mptcp_close(struct sock *meta_sk, long timeout)void mptcp_disconnect (struct sock *sk) { -struct sock *subsk, *tmpsk; +struct mptcp_tcp_sock *mptcp;struct tcp_sock *tp =tcp_sk (sk); +struct hlist_node *tmp;__skb_queue_purge (&tp->mpcb->reinject_queue);@@ -1917,7 +1935,9 @@ void mptcp_disconnect(struct sock *sk)mptcp_hash_remove_bh (tp);local_bh_disable (); -mptcp_for_each_sk_safe (tp->mpcb, subsk, tmpsk) { +mptcp_for_each_sub_safe (tp->mpcb, mptcp, tmp) { +struct sock *subsk =mptcp_to_sock (mptcp); +/* The socket will get removed from the subsocket-list * and made non-mptcp by setting mpc to 0. * @@ -2606,7 +2626,6 @@ static void mptcp_get_sub_info(struct sock *sk, struct mptcp_sub_info *info)int mptcp_get_info (const struct sock *meta_sk,char __user *optval,int optlen) {const struct tcp_sock *meta_tp =tcp_sk (meta_sk); -struct sock *sk;struct mptcp_meta_info meta_info;struct mptcp_info m_info;@@ -2652,16 +2671,17 @@ int mptcp_get_info(const struct sock *meta_sk, char __user *optval, int optlen)if (m_info.subflows) {unsigned int len, sub_len =0 ; +struct mptcp_tcp_sock *mptcp;char __user *ptr; ptr = (char __user *)m_info.subflows; len = m_info.sub_len; -mptcp_for_each_sk (meta_tp->mpcb, sk) { +mptcp_for_each_sub (meta_tp->mpcb, mptcp) {struct tcp_info t_info;unsigned int tmp_len; -tcp_get_info (sk, &t_info); +tcp_get_info (mptcp_to_sock (mptcp), &t_info); tmp_len = min_t(unsigned int, len, info_len); len -= tmp_len;@@ -2681,6 +2701,7 @@ int mptcp_get_info(const struct sock *meta_sk, char __user *optval, int optlen)if (m_info.subflow_info) {unsigned int len, sub_info_len, total_sub_info_len =0 ; +struct mptcp_tcp_sock *mptcp;char __user *ptr; ptr = (char __user *)m_info.subflow_info;@@ -2690,11 +2711,11 @@ int mptcp_get_info(const struct sock *meta_sk, char __user *optval, int optlen) sizeof(struct mptcp_sub_info)); m_info.sub_info_len = sub_info_len; -mptcp_for_each_sk (meta_tp->mpcb, sk) { +mptcp_for_each_sub (meta_tp->mpcb, mptcp) {struct mptcp_sub_info m_sub_info;unsigned int tmp_len; -mptcp_get_sub_info (sk, &m_sub_info); +mptcp_get_sub_info (mptcp_to_sock (mptcp), &m_sub_info); tmp_len = min_t(unsigned int, len, sub_info_len); len -= tmp_len;net/mptcp/mptcp_fullmesh.c +20
-8@@ -903,8 +903,9 @@ static void mptcp_address_worker(struct work_struct *work) }if (event->code ==MPTCP_EVENT_DEL ) { -struct sock *sk, *tmpsk; +struct mptcp_tcp_sock *mptcp;struct mptcp_loc_addr *mptcp_local; +struct hlist_node *tmp;bool found = false; mptcp_local =rcu_dereference_bh (fm_ns->local);@@ -914,7 +915,9 @@ static void mptcp_address_worker(struct work_struct *work)update_addr_bitfields (meta_sk, mptcp_local);/* Look for the socket and remove him */ -mptcp_for_each_sk_safe (mpcb, sk, tmpsk) { +mptcp_for_each_sub_safe (mpcb, mptcp, tmp) { +struct sock *sk =mptcp_to_sock (mptcp); + if ((event->family == AF_INET6 && (sk->sk_family == AF_INET || mptcp_v6_is_v4_mapped(sk))) ||@@ -964,9 +967,10 @@ static void mptcp_address_worker(struct work_struct *work) }if (event->code ==MPTCP_EVENT_MOD ) { -struct sock *sk; +struct mptcp_tcp_sock *mptcp; -mptcp_for_each_sk (mpcb, sk) { +mptcp_for_each_sub (mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);struct tcp_sock *tp =tcp_sk (sk); if (event->family == AF_INET && (sk->sk_family == AF_INET ||@@ -1455,8 +1459,9 @@ static void full_mesh_release_sock(struct sock *meta_sk)struct mptcp_cb *mpcb =tcp_sk (meta_sk)->mpcb;struct fullmesh_priv *fmp =fullmesh_get_priv (mpcb);const struct mptcp_fm_ns *fm_ns =fm_get_ns (sock_net (meta_sk)); -struct sock *sk, *tmpsk;bool meta_v4 = meta_sk->sk_family ==AF_INET ; +struct mptcp_tcp_sock *mptcp; +struct hlist_node *tmp;int i;rcu_read_lock_bh ();@@ -1470,7 +1475,8 @@ static void full_mesh_release_sock(struct sock *meta_sk)struct in_addr ifa = mptcp_local->locaddr4[i].addr;bool found = false; -mptcp_for_each_sk (mpcb, sk) { +mptcp_for_each_sub (mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);struct tcp_sock *tp =tcp_sk (sk); if (sk->sk_family == AF_INET6 &&@@ -1491,6 +1497,8 @@ static void full_mesh_release_sock(struct sock *meta_sk) }if (!found) { +struct sock *sk; + fmp->add_addr++; mpcb->addr_signal =1 ;@@ -1511,7 +1519,8 @@ static void full_mesh_release_sock(struct sock *meta_sk)struct in6_addr ifa = mptcp_local->locaddr6[i].addr;bool found = false; -mptcp_for_each_sk (mpcb, sk) { +mptcp_for_each_sub (mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);struct tcp_sock *tp =tcp_sk (sk); if (sk->sk_family == AF_INET ||@@ -1532,6 +1541,8 @@ static void full_mesh_release_sock(struct sock *meta_sk) }if (!found) { +struct sock *sk; + fmp->add_addr++; mpcb->addr_signal =1 ;@@ -1546,7 +1557,8 @@ static void full_mesh_release_sock(struct sock *meta_sk)#endif /* Now, detect address-removals */ -mptcp_for_each_sk_safe (mpcb, sk, tmpsk) { +mptcp_for_each_sub_safe (mpcb, mptcp, tmp) { +struct sock *sk =mptcp_to_sock (mptcp);bool shall_remove = true;if (sk->sk_family ==AF_INET ||mptcp_v6_is_v4_mapped (sk)) {net/mptcp/mptcp_input.c +29
-15@@ -126,12 +126,14 @@ static void mptcp_clean_rtx_queue(struct sock *meta_sk, u32 prior_snd_una)tcp_rtx_queue_unlink (skb, meta_sk);if (mptcp_is_data_fin (skb)) { -struct sock *sk_it, *sk_tmp; +struct mptcp_tcp_sock *mptcp; +struct hlist_node *tmp;/* DATA_FIN has been acknowledged - now we can close * the subflows */ -mptcp_for_each_sk_safe (mpcb, sk_it, sk_tmp) { +mptcp_for_each_sub_safe (mpcb, mptcp, tmp) { +struct sock *sk_it =mptcp_to_sock (mptcp);unsigned long delay =0 ;/* If we are the passive closer, don't trigger @@ -347,6 +349,7 @@ static int mptcp_verif_dss_csum(struct sock *sk)/* Now, checksum must be 0 */ if (unlikely (csum_fold (csum_tcp))) { +struct mptcp_tcp_sock *mptcp;struct sock *sk_it =NULL ; pr_err("%s csum is wrong: %#x tcp-seq %u dss_csum_added %d overflowed %d iterations %d\n",@@ -362,7 +365,9 @@ static int mptcp_verif_dss_csum(struct sock *sk) tp->mpcb->csum_cutoff_seq = tp->mptcp->map_data_seq;/* Search for another subflow that is fully established */ -mptcp_for_each_sk (tp->mpcb, sk_it) { +mptcp_for_each_sub (tp->mpcb, mptcp) { + sk_it =mptcp_to_sock (mptcp); +if (sk_it != sk &&tcp_sk (sk_it)->mptcp->fully_established)break ;@@ -1308,12 +1313,15 @@ int mptcp_do_join_short(struct sk_buff *skb,*/ void mptcp_fin (struct sock *meta_sk) { -struct sock *sk =NULL , *sk_it; +struct sock *sk =NULL ;struct tcp_sock *meta_tp =tcp_sk (meta_sk);struct mptcp_cb *mpcb = meta_tp->mpcb; +struct mptcp_tcp_sock *mptcp;unsigned char state; -mptcp_for_each_sk (mpcb, sk_it) { +mptcp_for_each_sub (mpcb, mptcp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +if (tcp_sk (sk_it)->mptcp->path_index == mpcb->dfin_path_index) { sk = sk_it;break ;@@ -1585,9 +1593,12 @@ void mptcp_clean_rtx_infinite(const struct sk_buff *skb, struct sock *sk)static void mptcp_send_reset_rem_id (const struct mptcp_cb *mpcb,u8 rem_id) { -struct sock *sk_it, *tmpsk; +struct mptcp_tcp_sock *mptcp; +struct hlist_node *tmp; + +mptcp_for_each_sub_safe (mpcb, mptcp, tmp) { +struct sock *sk_it =mptcp_to_sock (mptcp); -mptcp_for_each_sk_safe (mpcb, sk_it, tmpsk) {if (tcp_sk (sk_it)->mptcp->rem_id == rem_id) {mptcp_reinject_data (sk_it,0 );mptcp_send_reset (sk_it);@@ -1892,13 +1903,15 @@ void tcp_parse_mptcp_options(const struct sk_buff *skb,bool mptcp_check_rtt (const struct tcp_sock *tp,int time) {struct mptcp_cb *mpcb = tp->mpcb; -struct sock *sk; +struct mptcp_tcp_sock *mptcp;u32 rtt_max =0 ;/* In MPTCP, we take the max delay across all flows, * in order to take into account meta-reordering buffers. */ -mptcp_for_each_sk (mpcb, sk) { +mptcp_for_each_sub (mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp); +if (!mptcp_sk_can_recv (sk))continue ;@@ -2173,9 +2186,9 @@ bool mptcp_handle_options(struct sock *sk, const struct tcphdr *th,if (mopt->saw_low_prio ==1 ) { tp->mptcp->rcv_low_prio = mopt->low_prio; }else { -struct sock *sk_it; -mptcp_for_each_sk (tp->mpcb, sk_it) { -struct mptcp_tcp_sock *mptcp =tcp_sk (sk_it)->mptcp; +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (tp->mpcb, mptcp) {if (mptcp->rem_id == mopt->prio_addr_id) mptcp->rcv_low_prio = mopt->low_prio; }@@ -2359,7 +2372,7 @@ bool mptcp_should_expand_sndbuf(const struct sock *sk) {const struct sock *meta_sk =mptcp_meta_sk (sk);const struct tcp_sock *meta_tp =tcp_sk (meta_sk); -const struct sock *sk_it; +const struct mptcp_tcp_sock *mptcp;/* We circumvent this check in tcp_check_space, because we want to * always call sk_write_space. So, we reproduce the check here. @@ -2385,8 +2398,9 @@ bool mptcp_should_expand_sndbuf(const struct sock *sk)/* For MPTCP we look for a subsocket that could send data. * If we found one, then we update the send-buffer. */ -mptcp_for_each_sk (meta_tp->mpcb, sk_it) { -struct tcp_sock *tp_it =tcp_sk (sk_it); +mptcp_for_each_sub (meta_tp->mpcb, mptcp) { +const struct sock *sk_it =mptcp_to_sock (mptcp); +const struct tcp_sock *tp_it =tcp_sk (sk_it);if (!mptcp_sk_can_send (sk_it))continue ;net/mptcp/mptcp_output.c +2
-1@@ -647,7 +647,6 @@ int mptcp_write_wakeup(struct sock *meta_sk, int mib) {struct tcp_sock *meta_tp =tcp_sk (meta_sk);struct sk_buff *skb; -struct sock *sk_it;int ans =0 ;if (meta_sk->sk_state ==TCP_CLOSE )@@ -704,17 +703,22 @@ int mptcp_write_wakeup(struct sock *meta_sk, int mib)return 0 ; }else { +struct mptcp_tcp_sock *mptcp; +
window_probe:if (between(meta_tp->snd_up, meta_tp->snd_una + 1, meta_tp->snd_una + 0xFFFF)) {net/mptcp/mptcp_output.c +3
-1-mptcp_for_each_sk (meta_tp->mpcb, sk_it) { +mptcp_for_each_sub (meta_tp->mpcb, mptcp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +if (mptcp_sk_can_send_ack(sk_it)) tcp_xmit_probe_skb(sk_it, 1, mib); } } /* At least one of the tcp_xmit_probe_skb's has to succeed */net/mptcp/mptcp_output.c +2
-1-mptcp_for_each_sk (meta_tp->mpcb, sk_it) { +mptcp_for_each_sub (meta_tp->mpcb, mptcp) { +struct sock *sk_it =mptcp_to_sock (mptcp);int ret; if (!mptcp_sk_can_send_ack(sk_it))net/mptcp/mptcp_output.c +18
-10@@ -732,6 +736,7 @@ bool mptcp_write_xmit(struct sock *meta_sk, unsigned int mss_now, int nonagle, int push_one, gfp_t gfp) {struct tcp_sock *meta_tp =tcp_sk (meta_sk), *subtp; +struct mptcp_tcp_sock *mptcp;struct sock *subsk =NULL ;struct mptcp_cb *mpcb = meta_tp->mpcb;struct sk_buff *skb;@@ -856,7 +861,8 @@ bool mptcp_write_xmit(struct sock *meta_sk, unsigned int mss_now, int nonagle,break ; } -mptcp_for_each_sk (mpcb, subsk) { +mptcp_for_each_sub (mpcb, mptcp) { + subsk =mptcp_to_sock (mptcp); subtp =tcp_sk (subsk);if (!(path_mask &mptcp_pi_to_flag (subtp->mptcp->path_index)))@@ -1353,7 +1359,7 @@ void mptcp_send_active_reset(struct sock *meta_sk, gfp_t priority)struct mptcp_cb *mpcb = meta_tp->mpcb;struct sock *sk; -if (!mpcb->connection_list) +if (hlist_empty (&mpcb->conn_list))return ;WARN_ON (meta_tp->send_mp_fclose);@@ -1728,10 +1734,11 @@ void mptcp_select_initial_window(const struct sock *sk, int __space, __u32 mss,static inline u64 mptcp_calc_rate (const struct sock *meta_sk,unsigned int mss,unsigned int (*mss_cb)(struct sock *sk)) { -struct sock *sk; +struct mptcp_tcp_sock *mptcp;u64 rate =0 ; -mptcp_for_each_sk (tcp_sk (meta_sk)->mpcb, sk) { +mptcp_for_each_sub (tcp_sk (meta_sk)->mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);struct tcp_sock *tp =tcp_sk (sk);int this_mss;u64 this_rate;@@ -1783,11 +1790,12 @@ static inline u64 mptcp_calc_rate(const struct sock *meta_sk, unsigned int mss,static unsigned int __mptcp_current_mss (const struct sock *meta_sk,unsigned int (*mss_cb)(struct sock *sk)) { +struct mptcp_tcp_sock *mptcp;unsigned int mss =0 ;u64 rate =0 ; -struct sock *sk; -mptcp_for_each_sk (tcp_sk (meta_sk)->mpcb, sk) { +mptcp_for_each_sub (tcp_sk (meta_sk)->mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);int this_mss;u64 this_rate;@@ -1852,14 +1860,16 @@ int mptcp_select_size(const struct sock *meta_sk, bool first_skb, bool zc)int mptcp_check_snd_buf (const struct tcp_sock *tp) { -const struct sock *sk; +const struct mptcp_tcp_sock *mptcp;u32 rtt_max = tp->srtt_us;u64 bw_est;if (!tp->srtt_us)return tp->reordering +1 ; -mptcp_for_each_sk (tp->mpcb, sk) { +mptcp_for_each_sub (tp->mpcb, mptcp) { +const struct sock *sk =mptcp_to_sock (mptcp); +if (!mptcp_sk_can_send (sk))continue ;@@ -1877,11 +1887,13 @@ int mptcp_check_snd_buf(const struct tcp_sock *tp)unsigned int mptcp_xmit_size_goal (const struct sock *meta_sk,u32 mss_now,int large_allowed) { -struct sock *sk;u32 xmit_size_goal =0 ;if (large_allowed && !tcp_sk (meta_sk)->mpcb->dss_csum) { -mptcp_for_each_sk (tcp_sk (meta_sk)->mpcb, sk) { +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (tcp_sk (meta_sk)->mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);int this_size_goal;if (!mptcp_sk_can_send (sk))net/mptcp/mptcp_sched.c +15
-6@@ -135,9 +135,10 @@ static struct socku32 min_srtt =0xffffffff ;bool found_unused = false;bool found_unused_una = false; -struct sock *sk; +struct mptcp_tcp_sock *mptcp; -mptcp_for_each_sk (mpcb, sk) { +mptcp_for_each_sub (mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);struct tcp_sock *tp =tcp_sk (sk);bool unused = false;@@ -219,7 +220,11 @@ static struct sock *get_available_subflow(struct sock *meta_sk,/* Answer data_fin on same subflow!!! */ if (meta_sk->sk_shutdown &RCV_SHUTDOWN && skb &&mptcp_is_data_fin (skb)) { -mptcp_for_each_sk (mpcb, sk) { +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (mpcb, mptcp) { + sk =mptcp_to_sock (mptcp); +if (tcp_sk (sk)->mptcp->path_index == mpcb->dfin_path_index &&mptcp_is_available (sk, skb, zero_wnd_test))return sk;@@ -252,7 +257,7 @@ static struct sk_buff *mptcp_rcv_buf_optimization(struct sock *sk, int penal) {struct sock *meta_sk;const struct tcp_sock *tp =tcp_sk (sk); -struct tcp_sock *tp_it; +struct mptcp_tcp_sock *mptcp;struct sk_buff *skb_head;struct defsched_priv *dsp =defsched_get_priv (tp);@@ -275,7 +280,9 @@ static struct sk_buff *mptcp_rcv_buf_optimization(struct sock *sk, int penal) goto retrans;/* Half the cwnd of the slow flow */ -mptcp_for_each_tp (tp->mpcb, tp_it) { +mptcp_for_each_sub (tp->mpcb, mptcp) { +struct tcp_sock *tp_it = mptcp->tp; +if (tp_it != tp &&TCP_SKB_CB (skb_head)->path_mask &mptcp_pi_to_flag (tp_it->mptcp->path_index)) {if (tp->srtt_us < tp_it->srtt_us &&inet_csk ((struct sock *)tp_it)->icsk_ca_state == TCP_CA_Open) {@@ -298,7 +305,9 @@ static struct sk_buff *mptcp_rcv_buf_optimization(struct sock *sk, int penal)/* Segment not yet injected into this path? Take it!!! */ if (!(TCP_SKB_CB (skb_head)->path_mask &mptcp_pi_to_flag (tp->mptcp->path_index))) {bool do_retrans = false; -mptcp_for_each_tp (tp->mpcb, tp_it) { +mptcp_for_each_sub (tp->mpcb, mptcp) { +struct tcp_sock *tp_it = mptcp->tp; +if (tp_it != tp &&TCP_SKB_CB (skb_head)->path_mask &mptcp_pi_to_flag (tp_it->mptcp->path_index)) {if (tp_it->snd_cwnd <=4 ) {Signature · 1 line
2.16.2
--
Mat Martineau
Intel OTC
Christoph Paasch
Re: [MPTCP] [PATCH 10/20] mptcp: Make subflow-list an RCU-list
Quoted 73 lines · Mat Martineau
Quoted 71 lines · Christoph Paasch
mptcp_add_sock()will now be called without holding the meta-level lock. However,mptcp_add_sock()wants to add the subflow to the meta-level list, thus we need to protect this by a lock. We use thempcb_list_lockfor that.Now that we are locking during add/del, and want to allow lockless traversal of the list, this implies that we need to make it an RCU-list.
So, this patch transitions to the RCU hlist. The list-traversal macros (
hlist_for_each_entry_rcu) require me to now pass themptcp_tcp_sockto mptcp_for_each_*. So, I had to change all the places in the code where we call one of the list-traversal macros to adapt to this.Signed-off-by Christoph Paasch <cpaasch(a)apple.com>Signed-off-by Matthieu Baerts <matthieu.baerts(a)tessares.net>(cherry picked from commit 7a662b690069642c138da16ff9396e6826fd0a95)---
include/net/mptcp.h | 67 +++++++++++++++++----------------- net/ipv4/af_inet.c | 9 ++--- net/ipv4/ip_sockglue.c | 7 ++-- net/ipv4/tcp.c | 31 +++++++++------- net/mptcp/mptcp_ctrl.c | 89 ++++++++++++++++++++++++++++------------------ net/mptcp/mptcp_fullmesh.c | 28 ++++++++++----- net/mptcp/mptcp_input.c | 44 +++++++++++++++-------- net/mptcp/mptcp_output.c | 38 +++++++++++++------- net/mptcp/mptcp_sched.c | 21 +++++++---- 9 files changed, 207 insertions(+), 127 deletions(-)include/net/mptcp.h +7
-6@@ -156,7 +156,7 @@ struct mptcp_options_received { };struct mptcp_tcp_sock { -struct tcp_sock *next;/* Next subflow socket */ +struct hlist_node node;struct hlist_node cb_list;struct mptcp_options_received rx_opt;@@ -254,7 +254,7 @@ struct mptcp_sched_ops {struct mptcp_cb {/* list of sockets in this multipath connection */ -struct tcp_sock *connection_list; +struct hlist_head conn_list;/* list of sockets that need a call to release_cb */ struct hlist_head callback_list;@@ -309,7 +309,7 @@ struct mptcp_cb {/***** Start of fields, used for subflow establishment */ struct sock *meta_sk; -/* Master socket, also part of the connection_list, this +/* Master socket, also part of the conn_list, this * socket is the one that the application sees. */ struct sock *master_sk;@@ -661,21 +661,17 @@ extern struct workqueue_struct *mptcp_wq; pr_err(fmt, ##args); \ } while (0) -/* Iterates over all subflows */ -#define mptcp_for_each_tp (mpcb, tp) \ - for ((tp) = (mpcb)->connection_list; (tp); (tp) = (tp)->mptcp->next) +static inline struct sock *mptcp_to_sock (const struct mptcp_tcp_sock *mptcp) +{ +return (struct sock *)mptcp->tp; +}
Quoted 11 lines
include/net/mptcp.h extern struct workqueue_struct *mptcp_wq; · line 670
Quoted 6 lines
include/net/mptcp.h +2
-4-#define mptcp_for_each_sk (mpcb, sk) \ - for ((sk) = (struct sock *)(mpcb)->connection_list; \ - sk; \ - sk = (struct sock *)tcp_sk(sk)->mptcp->next) +#define mptcp_for_each_sub (__mpcb, __mptcp) \ + hlist_for_each_entry_rcu(__mptcp, &((__mpcb)->conn_list), node)When I asked about rcu synchronization with an earlier version of this patch set, you mentioned that it didn't look like an issue because the lifetime of the subflow sockets was still managed under a lock - the main reason to use rcu here is to allow adding to the list while the MPTCP-level lock is held.
And mostly, to be able to iterate over the list whithout the need to hold any lock. That way, we can do lockless subflow establishment.
I think that's a very typical use of RCU-lists :)
Quoted 6 lines
That seems to also imply that rcu_read_lock/rcu_read_unlock are not required when using these list iteration macros. This is an unconventional use of rcu list entries, so it would be helpful to explain the expected use of the
conn_list. I think the main things are the lack ofrcu_read_lockduring iteration, and which locks to hold during iterate, add, and delete. Maybe the macro could check the state of the lock when built for debug?
Currently, we don't have a "build-for-debug"-macro. But, definitely something we could add in the future.
Quoted 12 lines
include/net/mptcp.h extern struct workqueue_struct *mptcp_wq; · line 672
Quoted 9 lines
include/net/mptcp.h +1
-6-#define mptcp_for_each_sk_safe (__mpcb, __sk, __temp) \ - for (__sk = (struct sock *)(__mpcb)->connection_list, \ - __temp = __sk ? (struct sock *)tcp_sk(__sk)->mptcp->next : NULL; \ - __sk; \ - __sk = __temp, \ - __temp = __sk ? (struct sock *)tcp_sk(__sk)->mptcp->next : NULL) +/* Must be called with the appropriate lock held */ If MPTCP-level socket lock is the appropriate lock, can that be specified here?
The comment is kinda useless IMO. Seems to me to be rather a copy-paste issue as I was developing this part here.
Quoted 7 lines
include/net/mptcp.h extern struct workqueue_struct *mptcp_wq; · line 674
Quoted 2 lines
include/net/mptcp.h +2
-0+#define mptcp_for_each_sub_safe (__mpcb, __mptcp, __tmp) \ + hlist_for_each_entry_safe(__mptcp, __tmp, &((__mpcb)->conn_list), node)The comment just before
mptcp_for_each_sub_safeseems to imply that the difference between the two macros is that one needs a lock and one doesn't, but isn't the second (_safe) macro intended for use when subflows are being removed from the list while iterating? Could you add a comment to clarify?
Yes, with the "appropriate lock" comment and _safe in itself this is kinda confusing. I will clarify that, saying that the _safe macro is there to allow removal while iterating and entirely remove the "appropriate lock" comment.
I am going to do this through a separate patch to mptcp_trunk that I will forward-port to mptcp-net-next later on.
Christoph
Quoted 900 lines
Thanks,
Mat
Quoted 890 lines
include/net/mptcp.h +22
-17/* Iterates over all bit set to 1 in a bitset */ #define mptcp_for_each_bit_set (b, i) \@@ -923,12 +919,14 @@ struct request_sock *rev_mptcp_rsk(const struct mptcp_request_sock *req)static inline bool mptcp_can_sendpage (struct sock *sk) { -struct sock *sk_it; +struct mptcp_tcp_sock *mptcp;if (tcp_sk (sk)->mpcb->dss_csum)return false; -mptcp_for_each_sk (tcp_sk (sk)->mpcb, sk_it) { +mptcp_for_each_sub (tcp_sk (sk)->mpcb, mptcp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +if (!(sk_it->sk_route_caps &NETIF_F_SG ))return false; }@@ -962,9 +960,12 @@ static inline void mptcp_send_reset(struct sock *sk)static inline void mptcp_sub_force_close_all (struct mptcp_cb *mpcb,struct sock *except) { -struct sock *sk_it, *tmp; +struct mptcp_tcp_sock *mptcp; +struct hlist_node *tmp; + +mptcp_for_each_sub_safe (mpcb, mptcp, tmp) { +struct sock *sk_it =mptcp_to_sock (mptcp); -mptcp_for_each_sk_safe (mpcb, sk_it, tmp) {if (sk_it != except)mptcp_send_reset (sk_it); }@@ -1150,12 +1151,14 @@ static inline int mptcp_sk_can_send_ack(const struct sock *sk)static inline bool mptcp_can_sg (const struct sock *meta_sk) { -struct sock *sk; +struct mptcp_tcp_sock *mptcp;if (tcp_sk (meta_sk)->mpcb->dss_csum)return false; -mptcp_for_each_sk (tcp_sk (meta_sk)->mpcb, sk) { +mptcp_for_each_sub (tcp_sk (meta_sk)->mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp); +if (!mptcp_sk_can_send (sk))continue ;if (!(sk->sk_route_caps &NETIF_F_SG ))@@ -1166,9 +1169,9 @@ static inline bool mptcp_can_sg(const struct sock *meta_sk)static inline void mptcp_set_rto (struct sock *sk) { -struct tcp_sock *tp =tcp_sk (sk); -struct sock *sk_it;struct inet_connection_sock *micsk =inet_csk (mptcp_meta_sk (sk)); +struct tcp_sock *tp =tcp_sk (sk); +struct mptcp_tcp_sock *mptcp;__u32 max_rto =0 ;/* We are in recovery-phase on the MPTCP-level. Do not update the @@ -1177,7 +1180,9 @@ static inline void mptcp_set_rto(struct sock *sk)if (micsk->icsk_retransmits)return ; -mptcp_for_each_sk (tp->mpcb, sk_it) { +mptcp_for_each_sub (tp->mpcb, mptcp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +if ((mptcp_sk_can_send (sk_it) || sk->sk_state ==TCP_SYN_RECV ) &&inet_csk (sk_it)->icsk_rto > max_rto) max_rto =inet_csk (sk_it)->icsk_rto;@@ -1266,10 +1271,10 @@ static inline bool mptcp_can_new_subflow(const struct sock *meta_sk)static inline int mptcp_subflow_count (const struct mptcp_cb *mpcb) { -struct sock *sk; +struct mptcp_tcp_sock *mptcp;int i =0 ; -mptcp_for_each_sk (mpcb, sk) +mptcp_for_each_sub (mpcb, mptcp) i++;return i;@@ -1287,12 +1292,8 @@ bool mptcp_prune_ofo_queue(struct sock *sk);do { \ } while (0) -/* Without MPTCP, we just do one iteration -* over the only socket available. This assumes that -* the sk/tp arg is the socket in that case. - */ -#define mptcp_for_each_sk (mpcb, sk) -#define mptcp_for_each_sk_safe (__mpcb, __sk, __temp) +#define mptcp_for_each_sub (__mpcb, __mptcp) \ + if (0)#define MPTCP_INC_STATS (net, field) \ do { \net/ipv4/af_inet.c +5
-4@@ -744,13 +744,14 @@ int inet_accept(struct socket *sock, struct socket *newsock, int flags,sock_rps_record_flow (sk2);if (sk2->sk_protocol ==IPPROTO_TCP &&mptcp (tcp_sk (sk2))) { -struct sock *sk_it = sk2; +struct mptcp_tcp_sock *mptcp; -mptcp_for_each_sk (tcp_sk (sk2)->mpcb, sk_it) -sock_rps_record_flow (sk_it); +mptcp_for_each_sub (tcp_sk (sk2)->mpcb, mptcp) { +sock_rps_record_flow (mptcp_to_sock (mptcp)); + }if (tcp_sk (sk2)->mpcb->master_sk) { - sk_it =tcp_sk (sk2)->mpcb->master_sk; +struct sock *sk_it =tcp_sk (sk2)->mpcb->master_sk;write_lock_bh (&sk_it->sk_callback_lock); sk_it->sk_wq = newsock->wq;net/ipv4/ip_sockglue.c +5
-2@@ -756,8 +756,11 @@ static int do_ip_setsockopt(struct sock *sk, int level,sk_dst_reset (sk);/* Update TOS on mptcp subflow */ if (is_meta_sk (sk)) { -struct sock *sk_it; -mptcp_for_each_sk (tcp_sk (sk)->mpcb, sk_it) { +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (tcp_sk (sk)->mpcb, mptcp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +if (inet_sk (sk_it)->tos !=inet_sk (sk)->tos) {inet_sk (sk_it)->tos =inet_sk (sk)->tos; sk_it->sk_priority = sk->sk_priority;net/ipv4/tcp.c +19
-12@@ -823,9 +823,11 @@ ssize_t tcp_splice_read(struct socket *sock, loff_t *ppos,#ifdef CONFIG_MPTCP if (mptcp (tcp_sk (sk))) { -struct sock *sk_it; -mptcp_for_each_sk (tcp_sk (sk)->mpcb, sk_it) -sock_rps_record_flow (sk_it); +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (tcp_sk (sk)->mpcb, mptcp) { +sock_rps_record_flow (mptcp_to_sock (mptcp)); + } }#endif @@ -993,7 +995,7 @@ ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset, }if (mptcp (tp)) { -struct sock *sk_it = sk; +struct mptcp_tcp_sock *mptcp;/* We must check this with socket-lock hold because we iterate * over the subflows. @@ -1008,8 +1010,9 @@ ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset,return ret; } -mptcp_for_each_sk (tp->mpcb, sk_it) -sock_rps_record_flow (sk_it); +mptcp_for_each_sub (tp->mpcb, mptcp) { +sock_rps_record_flow (mptcp_to_sock (mptcp)); + } }sk_clear_bit (SOCKWQ_ASYNC_NOSPACE , sk);@@ -1288,9 +1291,11 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size) }if (mptcp (tp)) { -struct sock *sk_it = sk; -mptcp_for_each_sk (tp->mpcb, sk_it) -sock_rps_record_flow (sk_it); +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (tp->mpcb, mptcp) { +sock_rps_record_flow (mptcp_to_sock (mptcp)); + } }if (unlikely (tp->repair)) {@@ -2006,9 +2011,11 @@ int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int nonblock,#ifdef CONFIG_MPTCP if (mptcp (tp)) { -struct sock *sk_it; -mptcp_for_each_sk (tp->mpcb, sk_it) -sock_rps_record_flow (sk_it); +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (tp->mpcb, mptcp) { +sock_rps_record_flow (mptcp_to_sock (mptcp)); + } }#endif net/mptcp/mptcp_ctrl.c +55
-34@@ -561,10 +561,12 @@ void mptcp_hash_remove_bh(struct tcp_sock *meta_tp)struct sock *mptcp_select_ack_sock (const struct sock *meta_sk) {const struct tcp_sock *meta_tp =tcp_sk (meta_sk); -struct sock *sk, *rttsk =NULL , *lastsk =NULL ; +struct sock *rttsk =NULL , *lastsk =NULL ;u32 min_time =0 , last_active =0 ; +struct mptcp_tcp_sock *mptcp; -mptcp_for_each_sk (meta_tp->mpcb, sk) { +mptcp_for_each_sub (meta_tp->mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);struct tcp_sock *tp =tcp_sk (sk);u32 elapsed;@@ -697,7 +699,8 @@ void mptcp_sock_destruct(struct sock *sk)void mptcp_destroy_sock (struct sock *sk) {if (is_meta_sk (sk)) { -struct sock *sk_it, *tmpsk; +struct mptcp_tcp_sock *mptcp; +struct hlist_node *tmp;__skb_queue_purge (&tcp_sk (sk)->mpcb->reinject_queue);@@ -707,7 +710,9 @@ void mptcp_destroy_sock(struct sock *sk)* not have been closed properly (as we are waiting for the * DATA_ACK of the DATA_FIN). */ -mptcp_for_each_sk_safe (tcp_sk (sk)->mpcb, sk_it, tmpsk) { +mptcp_for_each_sub_safe (tcp_sk (sk)->mpcb, mptcp, tmp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +/* Already did call tcp_close - waiting for graceful * closure, or if we are retransmitting fast-close on * the subflow. The reset (or timeout) will kill the @@ -1303,6 +1308,7 @@ static int mptcp_alloc_mpcb(struct sock *meta_sk, __u64 remote_key,INIT_LIST_HEAD (&mpcb->tw_list);INIT_HLIST_HEAD (&mpcb->callback_list); +INIT_HLIST_HEAD (&mpcb->conn_list);spin_lock_init (&mpcb->mpcb_list_lock);mptcp_mpcb_inherit_sockopts (meta_sk, master_sk);@@ -1392,8 +1398,12 @@ int mptcp_add_sock(struct sock *meta_sk, struct sock *sk, u8 loc_id, u8 rem_id,sock_hold (meta_sk);refcount_inc (&mpcb->mpcb_refcnt); - tp->mptcp->next = mpcb->connection_list; - mpcb->connection_list = tp; +local_bh_disable (); +spin_lock (&mpcb->mpcb_list_lock); +hlist_add_head_rcu (&tp->mptcp->node, &mpcb->conn_list); +spin_unlock (&mpcb->mpcb_list_lock); +local_bh_enable (); + tp->mptcp->attached =1 ; atomic_add(atomic_read(&((struct sock *)tp)->sk_rmem_alloc),@@ -1437,14 +1447,13 @@ int mptcp_add_sock(struct sock *meta_sk, struct sock *sk, u8 loc_id, u8 rem_id,void mptcp_del_sock (struct sock *sk) { -struct tcp_sock *tp =tcp_sk (sk), *tp_prev; +struct tcp_sock *tp =tcp_sk (sk);struct mptcp_cb *mpcb;if (!tp->mptcp || !tp->mptcp->attached)return ; mpcb = tp->mpcb; - tp_prev = mpcb->connection_list;if (mpcb->pm_ops->delete_subflow) mpcb->pm_ops->delete_subflow (sk);@@ -1453,17 +1462,10 @@ void mptcp_del_sock(struct sock *sk) __func__, mpcb->mptcp_loc_token, tp->mptcp->path_index, sk->sk_state, is_meta_sk(sk)); -if (tp_prev == tp) { - mpcb->connection_list = tp->mptcp->next; - }else { -for (; tp_prev && tp_prev->mptcp->next; tp_prev = tp_prev->mptcp->next) { -if (tp_prev->mptcp->next == tp) { - tp_prev->mptcp->next = tp->mptcp->next; -break ; - } - } - } - tp->mptcp->next =NULL ; +spin_lock (&mpcb->mpcb_list_lock); +hlist_del_init_rcu (&tp->mptcp->node); +spin_unlock (&mpcb->mpcb_list_lock); + tp->mptcp->attached =0 ; mpcb->path_index_bits &= ~(1 << tp->mptcp->path_index);@@ -1510,8 +1512,8 @@ void mptcp_update_metasocket(const struct sock *meta_sk)void mptcp_cleanup_rbuf (struct sock *meta_sk,int copied) {struct tcp_sock *meta_tp =tcp_sk (meta_sk); -struct sock *sk;bool recheck_rcv_window = false; +struct mptcp_tcp_sock *mptcp;__u32 rcv_window_now =0 ;if (copied >0 && !(meta_sk->sk_shutdown &RCV_SHUTDOWN )) {@@ -1522,7 +1524,8 @@ void mptcp_cleanup_rbuf(struct sock *meta_sk, int copied) recheck_rcv_window = true; } -mptcp_for_each_sk (meta_tp->mpcb, sk) { +mptcp_for_each_sub (meta_tp->mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);struct tcp_sock *tp =tcp_sk (sk);const struct inet_connection_sock *icsk =inet_csk (sk);@@ -1709,10 +1712,13 @@ EXPORT_SYMBOL(mptcp_sub_force_close);*/ void mptcp_update_sndbuf (const struct tcp_sock *tp) { -struct sock *meta_sk = tp->meta_sk, *sk; +struct sock *meta_sk = tp->meta_sk;int new_sndbuf =0 , old_sndbuf = meta_sk->sk_sndbuf; +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (tp->mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp); -mptcp_for_each_sk (tp->mpcb, sk) {if (!mptcp_sk_can_send (sk))continue ;@@ -1741,8 +1747,8 @@ void mptcp_update_sndbuf(const struct tcp_sock *tp)void mptcp_close (struct sock *meta_sk,long timeout) {struct tcp_sock *meta_tp =tcp_sk (meta_sk); -struct sock *sk_it, *tmpsk;struct mptcp_cb *mpcb = meta_tp->mpcb; +struct mptcp_tcp_sock *mptcp;struct sk_buff *skb;int data_was_unread =0 ;int state;@@ -1775,7 +1781,12 @@ void mptcp_close(struct sock *meta_sk, long timeout)/* If socket has been already reset (e.g. in tcp_reset()) - kill it. */ if (meta_sk->sk_state ==TCP_CLOSE ) { -mptcp_for_each_sk_safe (mpcb, sk_it, tmpsk) { +struct mptcp_tcp_sock *mptcp; +struct hlist_node *tmp; + +mptcp_for_each_sub_safe (mpcb, mptcp, tmp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +if (tcp_sk (sk_it)->send_mp_fclose)continue ;mptcp_sub_close (sk_it,0 );@@ -1796,10 +1807,14 @@ void mptcp_close(struct sock *meta_sk, long timeout) }else if (tcp_close_state (meta_sk)) {mptcp_send_fin (meta_sk); }else if (meta_tp->snd_una == meta_tp->write_seq) { +struct mptcp_tcp_sock *mptcp; +struct hlist_node *tmp; +/* The DATA_FIN has been sent and acknowledged * (e.g., by sk_shutdown). Close all the other subflows */ -mptcp_for_each_sk_safe (mpcb, sk_it, tmpsk) { +mptcp_for_each_sub_safe (mpcb, mptcp, tmp) { +struct sock *sk_it =mptcp_to_sock (mptcp);unsigned long delay =0 ;/* If we are the passive closer, don't trigger * subflow-fin until the subflow has been finned @@ -1823,7 +1838,9 @@ void mptcp_close(struct sock *meta_sk, long timeout)/* socket will be freed after mptcp_close - we have to prevent * access from the subflows. */ -mptcp_for_each_sk (mpcb, sk_it) { +mptcp_for_each_sub (mpcb, mptcp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +/* Similar to sock_orphan, but we don't set it DEAD, because * the callbacks are still set and must be called. */ @@ -1908,8 +1925,9 @@ void mptcp_close(struct sock *meta_sk, long timeout)void mptcp_disconnect (struct sock *sk) { -struct sock *subsk, *tmpsk; +struct mptcp_tcp_sock *mptcp;struct tcp_sock *tp =tcp_sk (sk); +struct hlist_node *tmp;__skb_queue_purge (&tp->mpcb->reinject_queue);@@ -1917,7 +1935,9 @@ void mptcp_disconnect(struct sock *sk)mptcp_hash_remove_bh (tp);local_bh_disable (); -mptcp_for_each_sk_safe (tp->mpcb, subsk, tmpsk) { +mptcp_for_each_sub_safe (tp->mpcb, mptcp, tmp) { +struct sock *subsk =mptcp_to_sock (mptcp); +/* The socket will get removed from the subsocket-list * and made non-mptcp by setting mpc to 0. * @@ -2606,7 +2626,6 @@ static void mptcp_get_sub_info(struct sock *sk, struct mptcp_sub_info *info)int mptcp_get_info (const struct sock *meta_sk,char __user *optval,int optlen) {const struct tcp_sock *meta_tp =tcp_sk (meta_sk); -struct sock *sk;struct mptcp_meta_info meta_info;struct mptcp_info m_info;@@ -2652,16 +2671,17 @@ int mptcp_get_info(const struct sock *meta_sk, char __user *optval, int optlen)if (m_info.subflows) {unsigned int len, sub_len =0 ; +struct mptcp_tcp_sock *mptcp;char __user *ptr; ptr = (char __user *)m_info.subflows; len = m_info.sub_len; -mptcp_for_each_sk (meta_tp->mpcb, sk) { +mptcp_for_each_sub (meta_tp->mpcb, mptcp) {struct tcp_info t_info;unsigned int tmp_len; -tcp_get_info (sk, &t_info); +tcp_get_info (mptcp_to_sock (mptcp), &t_info); tmp_len = min_t(unsigned int, len, info_len); len -= tmp_len;@@ -2681,6 +2701,7 @@ int mptcp_get_info(const struct sock *meta_sk, char __user *optval, int optlen)if (m_info.subflow_info) {unsigned int len, sub_info_len, total_sub_info_len =0 ; +struct mptcp_tcp_sock *mptcp;char __user *ptr; ptr = (char __user *)m_info.subflow_info;@@ -2690,11 +2711,11 @@ int mptcp_get_info(const struct sock *meta_sk, char __user *optval, int optlen) sizeof(struct mptcp_sub_info)); m_info.sub_info_len = sub_info_len; -mptcp_for_each_sk (meta_tp->mpcb, sk) { +mptcp_for_each_sub (meta_tp->mpcb, mptcp) {struct mptcp_sub_info m_sub_info;unsigned int tmp_len; -mptcp_get_sub_info (sk, &m_sub_info); +mptcp_get_sub_info (mptcp_to_sock (mptcp), &m_sub_info); tmp_len = min_t(unsigned int, len, sub_info_len); len -= tmp_len;net/mptcp/mptcp_fullmesh.c +20
-8@@ -903,8 +903,9 @@ static void mptcp_address_worker(struct work_struct *work) }if (event->code ==MPTCP_EVENT_DEL ) { -struct sock *sk, *tmpsk; +struct mptcp_tcp_sock *mptcp;struct mptcp_loc_addr *mptcp_local; +struct hlist_node *tmp;bool found = false; mptcp_local =rcu_dereference_bh (fm_ns->local);@@ -914,7 +915,9 @@ static void mptcp_address_worker(struct work_struct *work)update_addr_bitfields (meta_sk, mptcp_local);/* Look for the socket and remove him */ -mptcp_for_each_sk_safe (mpcb, sk, tmpsk) { +mptcp_for_each_sub_safe (mpcb, mptcp, tmp) { +struct sock *sk =mptcp_to_sock (mptcp); + if ((event->family == AF_INET6 && (sk->sk_family == AF_INET || mptcp_v6_is_v4_mapped(sk))) ||@@ -964,9 +967,10 @@ static void mptcp_address_worker(struct work_struct *work) }if (event->code ==MPTCP_EVENT_MOD ) { -struct sock *sk; +struct mptcp_tcp_sock *mptcp; -mptcp_for_each_sk (mpcb, sk) { +mptcp_for_each_sub (mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);struct tcp_sock *tp =tcp_sk (sk); if (event->family == AF_INET && (sk->sk_family == AF_INET ||@@ -1455,8 +1459,9 @@ static void full_mesh_release_sock(struct sock *meta_sk)struct mptcp_cb *mpcb =tcp_sk (meta_sk)->mpcb;struct fullmesh_priv *fmp =fullmesh_get_priv (mpcb);const struct mptcp_fm_ns *fm_ns =fm_get_ns (sock_net (meta_sk)); -struct sock *sk, *tmpsk;bool meta_v4 = meta_sk->sk_family ==AF_INET ; +struct mptcp_tcp_sock *mptcp; +struct hlist_node *tmp;int i;rcu_read_lock_bh ();@@ -1470,7 +1475,8 @@ static void full_mesh_release_sock(struct sock *meta_sk)struct in_addr ifa = mptcp_local->locaddr4[i].addr;bool found = false; -mptcp_for_each_sk (mpcb, sk) { +mptcp_for_each_sub (mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);struct tcp_sock *tp =tcp_sk (sk); if (sk->sk_family == AF_INET6 &&@@ -1491,6 +1497,8 @@ static void full_mesh_release_sock(struct sock *meta_sk) }if (!found) { +struct sock *sk; + fmp->add_addr++; mpcb->addr_signal =1 ;@@ -1511,7 +1519,8 @@ static void full_mesh_release_sock(struct sock *meta_sk)struct in6_addr ifa = mptcp_local->locaddr6[i].addr;bool found = false; -mptcp_for_each_sk (mpcb, sk) { +mptcp_for_each_sub (mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);struct tcp_sock *tp =tcp_sk (sk); if (sk->sk_family == AF_INET ||@@ -1532,6 +1541,8 @@ static void full_mesh_release_sock(struct sock *meta_sk) }if (!found) { +struct sock *sk; + fmp->add_addr++; mpcb->addr_signal =1 ;@@ -1546,7 +1557,8 @@ static void full_mesh_release_sock(struct sock *meta_sk)#endif /* Now, detect address-removals */ -mptcp_for_each_sk_safe (mpcb, sk, tmpsk) { +mptcp_for_each_sub_safe (mpcb, mptcp, tmp) { +struct sock *sk =mptcp_to_sock (mptcp);bool shall_remove = true;if (sk->sk_family ==AF_INET ||mptcp_v6_is_v4_mapped (sk)) {net/mptcp/mptcp_input.c +29
-15@@ -126,12 +126,14 @@ static void mptcp_clean_rtx_queue(struct sock *meta_sk, u32 prior_snd_una)tcp_rtx_queue_unlink (skb, meta_sk);if (mptcp_is_data_fin (skb)) { -struct sock *sk_it, *sk_tmp; +struct mptcp_tcp_sock *mptcp; +struct hlist_node *tmp;/* DATA_FIN has been acknowledged - now we can close * the subflows */ -mptcp_for_each_sk_safe (mpcb, sk_it, sk_tmp) { +mptcp_for_each_sub_safe (mpcb, mptcp, tmp) { +struct sock *sk_it =mptcp_to_sock (mptcp);unsigned long delay =0 ;/* If we are the passive closer, don't trigger @@ -347,6 +349,7 @@ static int mptcp_verif_dss_csum(struct sock *sk)/* Now, checksum must be 0 */ if (unlikely (csum_fold (csum_tcp))) { +struct mptcp_tcp_sock *mptcp;struct sock *sk_it =NULL ; pr_err("%s csum is wrong: %#x tcp-seq %u dss_csum_added %d overflowed %d iterations %d\n",@@ -362,7 +365,9 @@ static int mptcp_verif_dss_csum(struct sock *sk) tp->mpcb->csum_cutoff_seq = tp->mptcp->map_data_seq;/* Search for another subflow that is fully established */ -mptcp_for_each_sk (tp->mpcb, sk_it) { +mptcp_for_each_sub (tp->mpcb, mptcp) { + sk_it =mptcp_to_sock (mptcp); +if (sk_it != sk &&tcp_sk (sk_it)->mptcp->fully_established)break ;@@ -1308,12 +1313,15 @@ int mptcp_do_join_short(struct sk_buff *skb,*/ void mptcp_fin (struct sock *meta_sk) { -struct sock *sk =NULL , *sk_it; +struct sock *sk =NULL ;struct tcp_sock *meta_tp =tcp_sk (meta_sk);struct mptcp_cb *mpcb = meta_tp->mpcb; +struct mptcp_tcp_sock *mptcp;unsigned char state; -mptcp_for_each_sk (mpcb, sk_it) { +mptcp_for_each_sub (mpcb, mptcp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +if (tcp_sk (sk_it)->mptcp->path_index == mpcb->dfin_path_index) { sk = sk_it;break ;@@ -1585,9 +1593,12 @@ void mptcp_clean_rtx_infinite(const struct sk_buff *skb, struct sock *sk)static void mptcp_send_reset_rem_id (const struct mptcp_cb *mpcb,u8 rem_id) { -struct sock *sk_it, *tmpsk; +struct mptcp_tcp_sock *mptcp; +struct hlist_node *tmp; + +mptcp_for_each_sub_safe (mpcb, mptcp, tmp) { +struct sock *sk_it =mptcp_to_sock (mptcp); -mptcp_for_each_sk_safe (mpcb, sk_it, tmpsk) {if (tcp_sk (sk_it)->mptcp->rem_id == rem_id) {mptcp_reinject_data (sk_it,0 );mptcp_send_reset (sk_it);@@ -1892,13 +1903,15 @@ void tcp_parse_mptcp_options(const struct sk_buff *skb,bool mptcp_check_rtt (const struct tcp_sock *tp,int time) {struct mptcp_cb *mpcb = tp->mpcb; -struct sock *sk; +struct mptcp_tcp_sock *mptcp;u32 rtt_max =0 ;/* In MPTCP, we take the max delay across all flows, * in order to take into account meta-reordering buffers. */ -mptcp_for_each_sk (mpcb, sk) { +mptcp_for_each_sub (mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp); +if (!mptcp_sk_can_recv (sk))continue ;@@ -2173,9 +2186,9 @@ bool mptcp_handle_options(struct sock *sk, const struct tcphdr *th,if (mopt->saw_low_prio ==1 ) { tp->mptcp->rcv_low_prio = mopt->low_prio; }else { -struct sock *sk_it; -mptcp_for_each_sk (tp->mpcb, sk_it) { -struct mptcp_tcp_sock *mptcp =tcp_sk (sk_it)->mptcp; +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (tp->mpcb, mptcp) {if (mptcp->rem_id == mopt->prio_addr_id) mptcp->rcv_low_prio = mopt->low_prio; }@@ -2359,7 +2372,7 @@ bool mptcp_should_expand_sndbuf(const struct sock *sk) {const struct sock *meta_sk =mptcp_meta_sk (sk);const struct tcp_sock *meta_tp =tcp_sk (meta_sk); -const struct sock *sk_it; +const struct mptcp_tcp_sock *mptcp;/* We circumvent this check in tcp_check_space, because we want to * always call sk_write_space. So, we reproduce the check here. @@ -2385,8 +2398,9 @@ bool mptcp_should_expand_sndbuf(const struct sock *sk)/* For MPTCP we look for a subsocket that could send data. * If we found one, then we update the send-buffer. */ -mptcp_for_each_sk (meta_tp->mpcb, sk_it) { -struct tcp_sock *tp_it =tcp_sk (sk_it); +mptcp_for_each_sub (meta_tp->mpcb, mptcp) { +const struct sock *sk_it =mptcp_to_sock (mptcp); +const struct tcp_sock *tp_it =tcp_sk (sk_it);if (!mptcp_sk_can_send (sk_it))continue ;net/mptcp/mptcp_output.c +2
-1@@ -647,7 +647,6 @@ int mptcp_write_wakeup(struct sock *meta_sk, int mib) {struct tcp_sock *meta_tp =tcp_sk (meta_sk);struct sk_buff *skb; -struct sock *sk_it;int ans =0 ;if (meta_sk->sk_state ==TCP_CLOSE )@@ -704,17 +703,22 @@ int mptcp_write_wakeup(struct sock *meta_sk, int mib)return 0 ; }else { +struct mptcp_tcp_sock *mptcp; +
window_probe:if (between(meta_tp->snd_up, meta_tp->snd_una + 1, meta_tp->snd_una + 0xFFFF)) {net/mptcp/mptcp_output.c +3
-1-mptcp_for_each_sk (meta_tp->mpcb, sk_it) { +mptcp_for_each_sub (meta_tp->mpcb, mptcp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +if (mptcp_sk_can_send_ack(sk_it)) tcp_xmit_probe_skb(sk_it, 1, mib); } } /* At least one of the tcp_xmit_probe_skb's has to succeed */net/mptcp/mptcp_output.c +2
-1-mptcp_for_each_sk (meta_tp->mpcb, sk_it) { +mptcp_for_each_sub (meta_tp->mpcb, mptcp) { +struct sock *sk_it =mptcp_to_sock (mptcp);int ret; if (!mptcp_sk_can_send_ack(sk_it))net/mptcp/mptcp_output.c +18
-10@@ -732,6 +736,7 @@ bool mptcp_write_xmit(struct sock *meta_sk, unsigned int mss_now, int nonagle, int push_one, gfp_t gfp) {struct tcp_sock *meta_tp =tcp_sk (meta_sk), *subtp; +struct mptcp_tcp_sock *mptcp;struct sock *subsk =NULL ;struct mptcp_cb *mpcb = meta_tp->mpcb;struct sk_buff *skb;@@ -856,7 +861,8 @@ bool mptcp_write_xmit(struct sock *meta_sk, unsigned int mss_now, int nonagle,break ; } -mptcp_for_each_sk (mpcb, subsk) { +mptcp_for_each_sub (mpcb, mptcp) { + subsk =mptcp_to_sock (mptcp); subtp =tcp_sk (subsk);if (!(path_mask &mptcp_pi_to_flag (subtp->mptcp->path_index)))@@ -1353,7 +1359,7 @@ void mptcp_send_active_reset(struct sock *meta_sk, gfp_t priority)struct mptcp_cb *mpcb = meta_tp->mpcb;struct sock *sk; -if (!mpcb->connection_list) +if (hlist_empty (&mpcb->conn_list))return ;WARN_ON (meta_tp->send_mp_fclose);@@ -1728,10 +1734,11 @@ void mptcp_select_initial_window(const struct sock *sk, int __space, __u32 mss,static inline u64 mptcp_calc_rate (const struct sock *meta_sk,unsigned int mss,unsigned int (*mss_cb)(struct sock *sk)) { -struct sock *sk; +struct mptcp_tcp_sock *mptcp;u64 rate =0 ; -mptcp_for_each_sk (tcp_sk (meta_sk)->mpcb, sk) { +mptcp_for_each_sub (tcp_sk (meta_sk)->mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);struct tcp_sock *tp =tcp_sk (sk);int this_mss;u64 this_rate;@@ -1783,11 +1790,12 @@ static inline u64 mptcp_calc_rate(const struct sock *meta_sk, unsigned int mss,static unsigned int __mptcp_current_mss (const struct sock *meta_sk,unsigned int (*mss_cb)(struct sock *sk)) { +struct mptcp_tcp_sock *mptcp;unsigned int mss =0 ;u64 rate =0 ; -struct sock *sk; -mptcp_for_each_sk (tcp_sk (meta_sk)->mpcb, sk) { +mptcp_for_each_sub (tcp_sk (meta_sk)->mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);int this_mss;u64 this_rate;@@ -1852,14 +1860,16 @@ int mptcp_select_size(const struct sock *meta_sk, bool first_skb, bool zc)int mptcp_check_snd_buf (const struct tcp_sock *tp) { -const struct sock *sk; +const struct mptcp_tcp_sock *mptcp;u32 rtt_max = tp->srtt_us;u64 bw_est;if (!tp->srtt_us)return tp->reordering +1 ; -mptcp_for_each_sk (tp->mpcb, sk) { +mptcp_for_each_sub (tp->mpcb, mptcp) { +const struct sock *sk =mptcp_to_sock (mptcp); +if (!mptcp_sk_can_send (sk))continue ;@@ -1877,11 +1887,13 @@ int mptcp_check_snd_buf(const struct tcp_sock *tp)unsigned int mptcp_xmit_size_goal (const struct sock *meta_sk,u32 mss_now,int large_allowed) { -struct sock *sk;u32 xmit_size_goal =0 ;if (large_allowed && !tcp_sk (meta_sk)->mpcb->dss_csum) { -mptcp_for_each_sk (tcp_sk (meta_sk)->mpcb, sk) { +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (tcp_sk (meta_sk)->mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);int this_size_goal;if (!mptcp_sk_can_send (sk))net/mptcp/mptcp_sched.c +15
-6@@ -135,9 +135,10 @@ static struct socku32 min_srtt =0xffffffff ;bool found_unused = false;bool found_unused_una = false; -struct sock *sk; +struct mptcp_tcp_sock *mptcp; -mptcp_for_each_sk (mpcb, sk) { +mptcp_for_each_sub (mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);struct tcp_sock *tp =tcp_sk (sk);bool unused = false;@@ -219,7 +220,11 @@ static struct sock *get_available_subflow(struct sock *meta_sk,/* Answer data_fin on same subflow!!! */ if (meta_sk->sk_shutdown &RCV_SHUTDOWN && skb &&mptcp_is_data_fin (skb)) { -mptcp_for_each_sk (mpcb, sk) { +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (mpcb, mptcp) { + sk =mptcp_to_sock (mptcp); +if (tcp_sk (sk)->mptcp->path_index == mpcb->dfin_path_index &&mptcp_is_available (sk, skb, zero_wnd_test))return sk;@@ -252,7 +257,7 @@ static struct sk_buff *mptcp_rcv_buf_optimization(struct sock *sk, int penal) {struct sock *meta_sk;const struct tcp_sock *tp =tcp_sk (sk); -struct tcp_sock *tp_it; +struct mptcp_tcp_sock *mptcp;struct sk_buff *skb_head;struct defsched_priv *dsp =defsched_get_priv (tp);@@ -275,7 +280,9 @@ static struct sk_buff *mptcp_rcv_buf_optimization(struct sock *sk, int penal) goto retrans;/* Half the cwnd of the slow flow */ -mptcp_for_each_tp (tp->mpcb, tp_it) { +mptcp_for_each_sub (tp->mpcb, mptcp) { +struct tcp_sock *tp_it = mptcp->tp; +if (tp_it != tp &&TCP_SKB_CB (skb_head)->path_mask &mptcp_pi_to_flag (tp_it->mptcp->path_index)) {if (tp->srtt_us < tp_it->srtt_us &&inet_csk ((struct sock *)tp_it)->icsk_ca_state == TCP_CA_Open) {@@ -298,7 +305,9 @@ static struct sk_buff *mptcp_rcv_buf_optimization(struct sock *sk, int penal)/* Segment not yet injected into this path? Take it!!! */ if (!(TCP_SKB_CB (skb_head)->path_mask &mptcp_pi_to_flag (tp->mptcp->path_index))) {bool do_retrans = false; -mptcp_for_each_tp (tp->mpcb, tp_it) { +mptcp_for_each_sub (tp->mpcb, mptcp) { +struct tcp_sock *tp_it = mptcp->tp; +if (tp_it != tp &&TCP_SKB_CB (skb_head)->path_mask &mptcp_pi_to_flag (tp_it->mptcp->path_index)) {if (tp_it->snd_cwnd <=4 ) {Signature · 1 line
2.16.2--
Mat Martineau
Intel OTC
Mat Martineau
Re: [MPTCP] [PATCH 10/20] mptcp: Make subflow-list an RCU-list
Quoted 86 lines · Christoph Paasch
Quoted 73 lines · Mat Martineau
Quoted 71 lines · Christoph Paasch
mptcp_add_sock()will now be called without holding the meta-level lock. However,mptcp_add_sock()wants to add the subflow to the meta-level list, thus we need to protect this by a lock. We use thempcb_list_lockfor that.Now that we are locking during add/del, and want to allow lockless traversal of the list, this implies that we need to make it an RCU-list.
So, this patch transitions to the RCU hlist. The list-traversal macros (
hlist_for_each_entry_rcu) require me to now pass themptcp_tcp_sockto mptcp_for_each_*. So, I had to change all the places in the code where we call one of the list-traversal macros to adapt to this.Signed-off-by Christoph Paasch <cpaasch(a)apple.com>Signed-off-by Matthieu Baerts <matthieu.baerts(a)tessares.net>(cherry picked from commit 7a662b690069642c138da16ff9396e6826fd0a95)---
include/net/mptcp.h | 67 +++++++++++++++++----------------- net/ipv4/af_inet.c | 9 ++--- net/ipv4/ip_sockglue.c | 7 ++-- net/ipv4/tcp.c | 31 +++++++++------- net/mptcp/mptcp_ctrl.c | 89 ++++++++++++++++++++++++++++------------------ net/mptcp/mptcp_fullmesh.c | 28 ++++++++++----- net/mptcp/mptcp_input.c | 44 +++++++++++++++-------- net/mptcp/mptcp_output.c | 38 +++++++++++++------- net/mptcp/mptcp_sched.c | 21 +++++++---- 9 files changed, 207 insertions(+), 127 deletions(-)include/net/mptcp.h +7
-6@@ -156,7 +156,7 @@ struct mptcp_options_received { };struct mptcp_tcp_sock { -struct tcp_sock *next;/* Next subflow socket */ +struct hlist_node node;struct hlist_node cb_list;struct mptcp_options_received rx_opt;@@ -254,7 +254,7 @@ struct mptcp_sched_ops {struct mptcp_cb {/* list of sockets in this multipath connection */ -struct tcp_sock *connection_list; +struct hlist_head conn_list;/* list of sockets that need a call to release_cb */ struct hlist_head callback_list;@@ -309,7 +309,7 @@ struct mptcp_cb {/***** Start of fields, used for subflow establishment */ struct sock *meta_sk; -/* Master socket, also part of the connection_list, this +/* Master socket, also part of the conn_list, this * socket is the one that the application sees. */ struct sock *master_sk;@@ -661,21 +661,17 @@ extern struct workqueue_struct *mptcp_wq; pr_err(fmt, ##args); \ } while (0) -/* Iterates over all subflows */ -#define mptcp_for_each_tp (mpcb, tp) \ - for ((tp) = (mpcb)->connection_list; (tp); (tp) = (tp)->mptcp->next) +static inline struct sock *mptcp_to_sock (const struct mptcp_tcp_sock *mptcp) +{ +return (struct sock *)mptcp->tp; +}Quoted 11 lines
include/net/mptcp.h extern struct workqueue_struct *mptcp_wq; · line 670
Quoted 6 lines
include/net/mptcp.h +2
-4-#define mptcp_for_each_sk (mpcb, sk) \ - for ((sk) = (struct sock *)(mpcb)->connection_list; \ - sk; \ - sk = (struct sock *)tcp_sk(sk)->mptcp->next) +#define mptcp_for_each_sub (__mpcb, __mptcp) \ + hlist_for_each_entry_rcu(__mptcp, &((__mpcb)->conn_list), node)When I asked about rcu synchronization with an earlier version of this patch set, you mentioned that it didn't look like an issue because the lifetime of the subflow sockets was still managed under a lock - the main reason to use rcu here is to allow adding to the list while the MPTCP-level lock is held.
Quoted 2 lines
And mostly, to be able to iterate over the list whithout the need to hold any lock. That way, we can do lockless subflow establishment.
There needs to be *some* lock to iterate over an RCU list, right? Even if it's just the non-blocking rcu_read_lock?
You had mentioned in a previous response that traversal of the list always happened with the MPTCP-level lock held
(https://lists.01.org/pipermail/mptcp/2018-June/000665.html) - which I bring up not as a "gotcha" but hopefully to either jog some memories or help us collectively understand the locking model in this patch. Without rcu_read_lock, I was depending on that MPTCP-level lock to prevent deletion during iteration.
Quoted 2 lines
I think that's a very typical use of RCU-lists :)
Idiomatic RCU code I've seen (note: that's far from all RCU code :) ) and documentation (like
https://www.kernel.org/doc/Documentation/RCU/checklist.txt item #2 rule-of-thumb) involve using rcu_read_lock/rcu_read_unlock while iterating and then deferring some operations until the end of the grace period. When you get to the "Unless..." sentence at the end of checklist #2 I think we may be covered by the MPTCP-level lock, but it's uncommon enough usage that I think it's helpful to explain.
Quoted 35 lines
Quoted 7 lines
That seems to also imply that rcu_read_lock/rcu_read_unlock are not required when using these list iteration macros. This is an unconventional use of rcu list entries, so it would be helpful to explain the expected use of the
conn_list. I think the main things are the lack ofrcu_read_lockduring iteration, and which locks to hold during iterate, add, and delete. Maybe the macro could check the state of the lock when built for debug?Currently, we don't have a "build-for-debug"-macro. But, definitely something we could add in the future.
Quoted 12 lines
include/net/mptcp.h extern struct workqueue_struct *mptcp_wq; · line 672
Quoted 9 lines
include/net/mptcp.h +1
-6-#define mptcp_for_each_sk_safe (__mpcb, __sk, __temp) \ - for (__sk = (struct sock *)(__mpcb)->connection_list, \ - __temp = __sk ? (struct sock *)tcp_sk(__sk)->mptcp->next : NULL; \ - __sk; \ - __sk = __temp, \ - __temp = __sk ? (struct sock *)tcp_sk(__sk)->mptcp->next : NULL) +/* Must be called with the appropriate lock held */ If MPTCP-level socket lock is the appropriate lock, can that be specified here?
The comment is kinda useless IMO. Seems to me to be rather a copy-paste issue as I was developing this part here.
Quoted 7 lines
include/net/mptcp.h extern struct workqueue_struct *mptcp_wq; · line 674
Quoted 2 lines
include/net/mptcp.h +2
-0+#define mptcp_for_each_sub_safe (__mpcb, __mptcp, __tmp) \ + hlist_for_each_entry_safe(__mptcp, __tmp, &((__mpcb)->conn_list), node)The comment just before
mptcp_for_each_sub_safeseems to imply that the difference between the two macros is that one needs a lock and one doesn't, but isn't the second (_safe) macro intended for use when subflows are being removed from the list while iterating? Could you add a comment to clarify?
Quoted 7 lines
Yes, with the "appropriate lock" comment and
_safein itself this is kinda confusing. I will clarify that, saying that the_safemacro is there to allow removal while iterating and entirely remove the "appropriate lock" comment.I am going to do this through a separate patch to
mptcp_trunkthat I will forward-port to mptcp-net-next later on.
Sure, no problem using a later patch to clarify the comments.
Mat
Quoted 906 lines
Christoph
Quoted 900 lines
Thanks,
Mat
Quoted 887 lines
include/net/mptcp.h +22
-17/* Iterates over all bit set to 1 in a bitset */ #define mptcp_for_each_bit_set (b, i) \@@ -923,12 +919,14 @@ struct request_sock *rev_mptcp_rsk(const struct mptcp_request_sock *req)static inline bool mptcp_can_sendpage (struct sock *sk) { -struct sock *sk_it; +struct mptcp_tcp_sock *mptcp;if (tcp_sk (sk)->mpcb->dss_csum)return false; -mptcp_for_each_sk (tcp_sk (sk)->mpcb, sk_it) { +mptcp_for_each_sub (tcp_sk (sk)->mpcb, mptcp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +if (!(sk_it->sk_route_caps &NETIF_F_SG ))return false; }@@ -962,9 +960,12 @@ static inline void mptcp_send_reset(struct sock *sk)static inline void mptcp_sub_force_close_all (struct mptcp_cb *mpcb,struct sock *except) { -struct sock *sk_it, *tmp; +struct mptcp_tcp_sock *mptcp; +struct hlist_node *tmp; + +mptcp_for_each_sub_safe (mpcb, mptcp, tmp) { +struct sock *sk_it =mptcp_to_sock (mptcp); -mptcp_for_each_sk_safe (mpcb, sk_it, tmp) {if (sk_it != except)mptcp_send_reset (sk_it); }@@ -1150,12 +1151,14 @@ static inline int mptcp_sk_can_send_ack(const struct sock *sk)static inline bool mptcp_can_sg (const struct sock *meta_sk) { -struct sock *sk; +struct mptcp_tcp_sock *mptcp;if (tcp_sk (meta_sk)->mpcb->dss_csum)return false; -mptcp_for_each_sk (tcp_sk (meta_sk)->mpcb, sk) { +mptcp_for_each_sub (tcp_sk (meta_sk)->mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp); +if (!mptcp_sk_can_send (sk))continue ;if (!(sk->sk_route_caps &NETIF_F_SG ))@@ -1166,9 +1169,9 @@ static inline bool mptcp_can_sg(const struct sock *meta_sk)static inline void mptcp_set_rto (struct sock *sk) { -struct tcp_sock *tp =tcp_sk (sk); -struct sock *sk_it;struct inet_connection_sock *micsk =inet_csk (mptcp_meta_sk (sk)); +struct tcp_sock *tp =tcp_sk (sk); +struct mptcp_tcp_sock *mptcp;__u32 max_rto =0 ;/* We are in recovery-phase on the MPTCP-level. Do not update the @@ -1177,7 +1180,9 @@ static inline void mptcp_set_rto(struct sock *sk)if (micsk->icsk_retransmits)return ; -mptcp_for_each_sk (tp->mpcb, sk_it) { +mptcp_for_each_sub (tp->mpcb, mptcp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +if ((mptcp_sk_can_send (sk_it) || sk->sk_state ==TCP_SYN_RECV ) &&inet_csk (sk_it)->icsk_rto > max_rto) max_rto =inet_csk (sk_it)->icsk_rto;@@ -1266,10 +1271,10 @@ static inline bool mptcp_can_new_subflow(const struct sock *meta_sk)static inline int mptcp_subflow_count (const struct mptcp_cb *mpcb) { -struct sock *sk; +struct mptcp_tcp_sock *mptcp;int i =0 ; -mptcp_for_each_sk (mpcb, sk) +mptcp_for_each_sub (mpcb, mptcp) i++;return i;@@ -1287,12 +1292,8 @@ bool mptcp_prune_ofo_queue(struct sock *sk);do { \ } while (0) -/* Without MPTCP, we just do one iteration -* over the only socket available. This assumes that -* the sk/tp arg is the socket in that case. - */ -#define mptcp_for_each_sk (mpcb, sk) -#define mptcp_for_each_sk_safe (__mpcb, __sk, __temp) +#define mptcp_for_each_sub (__mpcb, __mptcp) \ + if (0)#define MPTCP_INC_STATS (net, field) \ do { \net/ipv4/af_inet.c +5
-4@@ -744,13 +744,14 @@ int inet_accept(struct socket *sock, struct socket *newsock, int flags,sock_rps_record_flow (sk2);if (sk2->sk_protocol ==IPPROTO_TCP &&mptcp (tcp_sk (sk2))) { -struct sock *sk_it = sk2; +struct mptcp_tcp_sock *mptcp; -mptcp_for_each_sk (tcp_sk (sk2)->mpcb, sk_it) -sock_rps_record_flow (sk_it); +mptcp_for_each_sub (tcp_sk (sk2)->mpcb, mptcp) { +sock_rps_record_flow (mptcp_to_sock (mptcp)); + }if (tcp_sk (sk2)->mpcb->master_sk) { - sk_it =tcp_sk (sk2)->mpcb->master_sk; +struct sock *sk_it =tcp_sk (sk2)->mpcb->master_sk;write_lock_bh (&sk_it->sk_callback_lock); sk_it->sk_wq = newsock->wq;net/ipv4/ip_sockglue.c +5
-2@@ -756,8 +756,11 @@ static int do_ip_setsockopt(struct sock *sk, int level,sk_dst_reset (sk);/* Update TOS on mptcp subflow */ if (is_meta_sk (sk)) { -struct sock *sk_it; -mptcp_for_each_sk (tcp_sk (sk)->mpcb, sk_it) { +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (tcp_sk (sk)->mpcb, mptcp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +if (inet_sk (sk_it)->tos !=inet_sk (sk)->tos) {inet_sk (sk_it)->tos =inet_sk (sk)->tos; sk_it->sk_priority = sk->sk_priority;net/ipv4/tcp.c +19
-12@@ -823,9 +823,11 @@ ssize_t tcp_splice_read(struct socket *sock, loff_t *ppos,#ifdef CONFIG_MPTCP if (mptcp (tcp_sk (sk))) { -struct sock *sk_it; -mptcp_for_each_sk (tcp_sk (sk)->mpcb, sk_it) -sock_rps_record_flow (sk_it); +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (tcp_sk (sk)->mpcb, mptcp) { +sock_rps_record_flow (mptcp_to_sock (mptcp)); + } }#endif @@ -993,7 +995,7 @@ ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset, }if (mptcp (tp)) { -struct sock *sk_it = sk; +struct mptcp_tcp_sock *mptcp;/* We must check this with socket-lock hold because we iterate * over the subflows. @@ -1008,8 +1010,9 @@ ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset,return ret; } -mptcp_for_each_sk (tp->mpcb, sk_it) -sock_rps_record_flow (sk_it); +mptcp_for_each_sub (tp->mpcb, mptcp) { +sock_rps_record_flow (mptcp_to_sock (mptcp)); + } }sk_clear_bit (SOCKWQ_ASYNC_NOSPACE , sk);@@ -1288,9 +1291,11 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size) }if (mptcp (tp)) { -struct sock *sk_it = sk; -mptcp_for_each_sk (tp->mpcb, sk_it) -sock_rps_record_flow (sk_it); +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (tp->mpcb, mptcp) { +sock_rps_record_flow (mptcp_to_sock (mptcp)); + } }if (unlikely (tp->repair)) {@@ -2006,9 +2011,11 @@ int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int nonblock,#ifdef CONFIG_MPTCP if (mptcp (tp)) { -struct sock *sk_it; -mptcp_for_each_sk (tp->mpcb, sk_it) -sock_rps_record_flow (sk_it); +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (tp->mpcb, mptcp) { +sock_rps_record_flow (mptcp_to_sock (mptcp)); + } }#endif net/mptcp/mptcp_ctrl.c +55
-34@@ -561,10 +561,12 @@ void mptcp_hash_remove_bh(struct tcp_sock *meta_tp)struct sock *mptcp_select_ack_sock (const struct sock *meta_sk) {const struct tcp_sock *meta_tp =tcp_sk (meta_sk); -struct sock *sk, *rttsk =NULL , *lastsk =NULL ; +struct sock *rttsk =NULL , *lastsk =NULL ;u32 min_time =0 , last_active =0 ; +struct mptcp_tcp_sock *mptcp; -mptcp_for_each_sk (meta_tp->mpcb, sk) { +mptcp_for_each_sub (meta_tp->mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);struct tcp_sock *tp =tcp_sk (sk);u32 elapsed;@@ -697,7 +699,8 @@ void mptcp_sock_destruct(struct sock *sk)void mptcp_destroy_sock (struct sock *sk) {if (is_meta_sk (sk)) { -struct sock *sk_it, *tmpsk; +struct mptcp_tcp_sock *mptcp; +struct hlist_node *tmp;__skb_queue_purge (&tcp_sk (sk)->mpcb->reinject_queue);@@ -707,7 +710,9 @@ void mptcp_destroy_sock(struct sock *sk)* not have been closed properly (as we are waiting for the * DATA_ACK of the DATA_FIN). */ -mptcp_for_each_sk_safe (tcp_sk (sk)->mpcb, sk_it, tmpsk) { +mptcp_for_each_sub_safe (tcp_sk (sk)->mpcb, mptcp, tmp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +/* Already did call tcp_close - waiting for graceful * closure, or if we are retransmitting fast-close on * the subflow. The reset (or timeout) will kill the @@ -1303,6 +1308,7 @@ static int mptcp_alloc_mpcb(struct sock *meta_sk, __u64 remote_key,INIT_LIST_HEAD (&mpcb->tw_list);INIT_HLIST_HEAD (&mpcb->callback_list); +INIT_HLIST_HEAD (&mpcb->conn_list);spin_lock_init (&mpcb->mpcb_list_lock);mptcp_mpcb_inherit_sockopts (meta_sk, master_sk);@@ -1392,8 +1398,12 @@ int mptcp_add_sock(struct sock *meta_sk, struct sock *sk, u8 loc_id, u8 rem_id,sock_hold (meta_sk);refcount_inc (&mpcb->mpcb_refcnt); - tp->mptcp->next = mpcb->connection_list; - mpcb->connection_list = tp; +local_bh_disable (); +spin_lock (&mpcb->mpcb_list_lock); +hlist_add_head_rcu (&tp->mptcp->node, &mpcb->conn_list); +spin_unlock (&mpcb->mpcb_list_lock); +local_bh_enable (); + tp->mptcp->attached =1 ; atomic_add(atomic_read(&((struct sock *)tp)->sk_rmem_alloc),@@ -1437,14 +1447,13 @@ int mptcp_add_sock(struct sock *meta_sk, struct sock *sk, u8 loc_id, u8 rem_id,void mptcp_del_sock (struct sock *sk) { -struct tcp_sock *tp =tcp_sk (sk), *tp_prev; +struct tcp_sock *tp =tcp_sk (sk);struct mptcp_cb *mpcb;if (!tp->mptcp || !tp->mptcp->attached)return ; mpcb = tp->mpcb; - tp_prev = mpcb->connection_list;if (mpcb->pm_ops->delete_subflow) mpcb->pm_ops->delete_subflow (sk);@@ -1453,17 +1462,10 @@ void mptcp_del_sock(struct sock *sk) __func__, mpcb->mptcp_loc_token, tp->mptcp->path_index, sk->sk_state, is_meta_sk(sk)); -if (tp_prev == tp) { - mpcb->connection_list = tp->mptcp->next; - }else { -for (; tp_prev && tp_prev->mptcp->next; tp_prev = tp_prev->mptcp->next) { -if (tp_prev->mptcp->next == tp) { - tp_prev->mptcp->next = tp->mptcp->next; -break ; - } - } - } - tp->mptcp->next =NULL ; +spin_lock (&mpcb->mpcb_list_lock); +hlist_del_init_rcu (&tp->mptcp->node); +spin_unlock (&mpcb->mpcb_list_lock); + tp->mptcp->attached =0 ; mpcb->path_index_bits &= ~(1 << tp->mptcp->path_index);@@ -1510,8 +1512,8 @@ void mptcp_update_metasocket(const struct sock *meta_sk)void mptcp_cleanup_rbuf (struct sock *meta_sk,int copied) {struct tcp_sock *meta_tp =tcp_sk (meta_sk); -struct sock *sk;bool recheck_rcv_window = false; +struct mptcp_tcp_sock *mptcp;__u32 rcv_window_now =0 ;if (copied >0 && !(meta_sk->sk_shutdown &RCV_SHUTDOWN )) {@@ -1522,7 +1524,8 @@ void mptcp_cleanup_rbuf(struct sock *meta_sk, int copied) recheck_rcv_window = true; } -mptcp_for_each_sk (meta_tp->mpcb, sk) { +mptcp_for_each_sub (meta_tp->mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);struct tcp_sock *tp =tcp_sk (sk);const struct inet_connection_sock *icsk =inet_csk (sk);@@ -1709,10 +1712,13 @@ EXPORT_SYMBOL(mptcp_sub_force_close);*/ void mptcp_update_sndbuf (const struct tcp_sock *tp) { -struct sock *meta_sk = tp->meta_sk, *sk; +struct sock *meta_sk = tp->meta_sk;int new_sndbuf =0 , old_sndbuf = meta_sk->sk_sndbuf; +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (tp->mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp); -mptcp_for_each_sk (tp->mpcb, sk) {if (!mptcp_sk_can_send (sk))continue ;@@ -1741,8 +1747,8 @@ void mptcp_update_sndbuf(const struct tcp_sock *tp)void mptcp_close (struct sock *meta_sk,long timeout) {struct tcp_sock *meta_tp =tcp_sk (meta_sk); -struct sock *sk_it, *tmpsk;struct mptcp_cb *mpcb = meta_tp->mpcb; +struct mptcp_tcp_sock *mptcp;struct sk_buff *skb;int data_was_unread =0 ;int state;@@ -1775,7 +1781,12 @@ void mptcp_close(struct sock *meta_sk, long timeout)/* If socket has been already reset (e.g. in tcp_reset()) - kill it. */ if (meta_sk->sk_state ==TCP_CLOSE ) { -mptcp_for_each_sk_safe (mpcb, sk_it, tmpsk) { +struct mptcp_tcp_sock *mptcp; +struct hlist_node *tmp; + +mptcp_for_each_sub_safe (mpcb, mptcp, tmp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +if (tcp_sk (sk_it)->send_mp_fclose)continue ;mptcp_sub_close (sk_it,0 );@@ -1796,10 +1807,14 @@ void mptcp_close(struct sock *meta_sk, long timeout) }else if (tcp_close_state (meta_sk)) {mptcp_send_fin (meta_sk); }else if (meta_tp->snd_una == meta_tp->write_seq) { +struct mptcp_tcp_sock *mptcp; +struct hlist_node *tmp; +/* The DATA_FIN has been sent and acknowledged * (e.g., by sk_shutdown). Close all the other subflows */ -mptcp_for_each_sk_safe (mpcb, sk_it, tmpsk) { +mptcp_for_each_sub_safe (mpcb, mptcp, tmp) { +struct sock *sk_it =mptcp_to_sock (mptcp);unsigned long delay =0 ;/* If we are the passive closer, don't trigger * subflow-fin until the subflow has been finned @@ -1823,7 +1838,9 @@ void mptcp_close(struct sock *meta_sk, long timeout)/* socket will be freed after mptcp_close - we have to prevent * access from the subflows. */ -mptcp_for_each_sk (mpcb, sk_it) { +mptcp_for_each_sub (mpcb, mptcp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +/* Similar to sock_orphan, but we don't set it DEAD, because * the callbacks are still set and must be called. */ @@ -1908,8 +1925,9 @@ void mptcp_close(struct sock *meta_sk, long timeout)void mptcp_disconnect (struct sock *sk) { -struct sock *subsk, *tmpsk; +struct mptcp_tcp_sock *mptcp;struct tcp_sock *tp =tcp_sk (sk); +struct hlist_node *tmp;__skb_queue_purge (&tp->mpcb->reinject_queue);@@ -1917,7 +1935,9 @@ void mptcp_disconnect(struct sock *sk)mptcp_hash_remove_bh (tp);local_bh_disable (); -mptcp_for_each_sk_safe (tp->mpcb, subsk, tmpsk) { +mptcp_for_each_sub_safe (tp->mpcb, mptcp, tmp) { +struct sock *subsk =mptcp_to_sock (mptcp); +/* The socket will get removed from the subsocket-list * and made non-mptcp by setting mpc to 0. * @@ -2606,7 +2626,6 @@ static void mptcp_get_sub_info(struct sock *sk, struct mptcp_sub_info *info)int mptcp_get_info (const struct sock *meta_sk,char __user *optval,int optlen) {const struct tcp_sock *meta_tp =tcp_sk (meta_sk); -struct sock *sk;struct mptcp_meta_info meta_info;struct mptcp_info m_info;@@ -2652,16 +2671,17 @@ int mptcp_get_info(const struct sock *meta_sk, char __user *optval, int optlen)if (m_info.subflows) {unsigned int len, sub_len =0 ; +struct mptcp_tcp_sock *mptcp;char __user *ptr; ptr = (char __user *)m_info.subflows; len = m_info.sub_len; -mptcp_for_each_sk (meta_tp->mpcb, sk) { +mptcp_for_each_sub (meta_tp->mpcb, mptcp) {struct tcp_info t_info;unsigned int tmp_len; -tcp_get_info (sk, &t_info); +tcp_get_info (mptcp_to_sock (mptcp), &t_info); tmp_len = min_t(unsigned int, len, info_len); len -= tmp_len;@@ -2681,6 +2701,7 @@ int mptcp_get_info(const struct sock *meta_sk, char __user *optval, int optlen)if (m_info.subflow_info) {unsigned int len, sub_info_len, total_sub_info_len =0 ; +struct mptcp_tcp_sock *mptcp;char __user *ptr; ptr = (char __user *)m_info.subflow_info;@@ -2690,11 +2711,11 @@ int mptcp_get_info(const struct sock *meta_sk, char __user *optval, int optlen) sizeof(struct mptcp_sub_info)); m_info.sub_info_len = sub_info_len; -mptcp_for_each_sk (meta_tp->mpcb, sk) { +mptcp_for_each_sub (meta_tp->mpcb, mptcp) {struct mptcp_sub_info m_sub_info;unsigned int tmp_len; -mptcp_get_sub_info (sk, &m_sub_info); +mptcp_get_sub_info (mptcp_to_sock (mptcp), &m_sub_info); tmp_len = min_t(unsigned int, len, sub_info_len); len -= tmp_len;net/mptcp/mptcp_fullmesh.c +20
-8@@ -903,8 +903,9 @@ static void mptcp_address_worker(struct work_struct *work) }if (event->code ==MPTCP_EVENT_DEL ) { -struct sock *sk, *tmpsk; +struct mptcp_tcp_sock *mptcp;struct mptcp_loc_addr *mptcp_local; +struct hlist_node *tmp;bool found = false; mptcp_local =rcu_dereference_bh (fm_ns->local);@@ -914,7 +915,9 @@ static void mptcp_address_worker(struct work_struct *work)update_addr_bitfields (meta_sk, mptcp_local);/* Look for the socket and remove him */ -mptcp_for_each_sk_safe (mpcb, sk, tmpsk) { +mptcp_for_each_sub_safe (mpcb, mptcp, tmp) { +struct sock *sk =mptcp_to_sock (mptcp); + if ((event->family == AF_INET6 && (sk->sk_family == AF_INET || mptcp_v6_is_v4_mapped(sk))) ||@@ -964,9 +967,10 @@ static void mptcp_address_worker(struct work_struct *work) }if (event->code ==MPTCP_EVENT_MOD ) { -struct sock *sk; +struct mptcp_tcp_sock *mptcp; -mptcp_for_each_sk (mpcb, sk) { +mptcp_for_each_sub (mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);struct tcp_sock *tp =tcp_sk (sk); if (event->family == AF_INET && (sk->sk_family == AF_INET ||@@ -1455,8 +1459,9 @@ static void full_mesh_release_sock(struct sock *meta_sk)struct mptcp_cb *mpcb =tcp_sk (meta_sk)->mpcb;struct fullmesh_priv *fmp =fullmesh_get_priv (mpcb);const struct mptcp_fm_ns *fm_ns =fm_get_ns (sock_net (meta_sk)); -struct sock *sk, *tmpsk;bool meta_v4 = meta_sk->sk_family ==AF_INET ; +struct mptcp_tcp_sock *mptcp; +struct hlist_node *tmp;int i;rcu_read_lock_bh ();@@ -1470,7 +1475,8 @@ static void full_mesh_release_sock(struct sock *meta_sk)struct in_addr ifa = mptcp_local->locaddr4[i].addr;bool found = false; -mptcp_for_each_sk (mpcb, sk) { +mptcp_for_each_sub (mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);struct tcp_sock *tp =tcp_sk (sk); if (sk->sk_family == AF_INET6 &&@@ -1491,6 +1497,8 @@ static void full_mesh_release_sock(struct sock *meta_sk) }if (!found) { +struct sock *sk; + fmp->add_addr++; mpcb->addr_signal =1 ;@@ -1511,7 +1519,8 @@ static void full_mesh_release_sock(struct sock *meta_sk)struct in6_addr ifa = mptcp_local->locaddr6[i].addr;bool found = false; -mptcp_for_each_sk (mpcb, sk) { +mptcp_for_each_sub (mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);struct tcp_sock *tp =tcp_sk (sk); if (sk->sk_family == AF_INET ||@@ -1532,6 +1541,8 @@ static void full_mesh_release_sock(struct sock *meta_sk) }if (!found) { +struct sock *sk; + fmp->add_addr++; mpcb->addr_signal =1 ;@@ -1546,7 +1557,8 @@ static void full_mesh_release_sock(struct sock *meta_sk)#endif /* Now, detect address-removals */ -mptcp_for_each_sk_safe (mpcb, sk, tmpsk) { +mptcp_for_each_sub_safe (mpcb, mptcp, tmp) { +struct sock *sk =mptcp_to_sock (mptcp);bool shall_remove = true;if (sk->sk_family ==AF_INET ||mptcp_v6_is_v4_mapped (sk)) {net/mptcp/mptcp_input.c +29
-15@@ -126,12 +126,14 @@ static void mptcp_clean_rtx_queue(struct sock *meta_sk, u32 prior_snd_una)tcp_rtx_queue_unlink (skb, meta_sk);if (mptcp_is_data_fin (skb)) { -struct sock *sk_it, *sk_tmp; +struct mptcp_tcp_sock *mptcp; +struct hlist_node *tmp;/* DATA_FIN has been acknowledged - now we can close * the subflows */ -mptcp_for_each_sk_safe (mpcb, sk_it, sk_tmp) { +mptcp_for_each_sub_safe (mpcb, mptcp, tmp) { +struct sock *sk_it =mptcp_to_sock (mptcp);unsigned long delay =0 ;/* If we are the passive closer, don't trigger @@ -347,6 +349,7 @@ static int mptcp_verif_dss_csum(struct sock *sk)/* Now, checksum must be 0 */ if (unlikely (csum_fold (csum_tcp))) { +struct mptcp_tcp_sock *mptcp;struct sock *sk_it =NULL ; pr_err("%s csum is wrong: %#x tcp-seq %u dss_csum_added %d overflowed %d iterations %d\n",@@ -362,7 +365,9 @@ static int mptcp_verif_dss_csum(struct sock *sk) tp->mpcb->csum_cutoff_seq = tp->mptcp->map_data_seq;/* Search for another subflow that is fully established */ -mptcp_for_each_sk (tp->mpcb, sk_it) { +mptcp_for_each_sub (tp->mpcb, mptcp) { + sk_it =mptcp_to_sock (mptcp); +if (sk_it != sk &&tcp_sk (sk_it)->mptcp->fully_established)break ;@@ -1308,12 +1313,15 @@ int mptcp_do_join_short(struct sk_buff *skb,*/ void mptcp_fin (struct sock *meta_sk) { -struct sock *sk =NULL , *sk_it; +struct sock *sk =NULL ;struct tcp_sock *meta_tp =tcp_sk (meta_sk);struct mptcp_cb *mpcb = meta_tp->mpcb; +struct mptcp_tcp_sock *mptcp;unsigned char state; -mptcp_for_each_sk (mpcb, sk_it) { +mptcp_for_each_sub (mpcb, mptcp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +if (tcp_sk (sk_it)->mptcp->path_index == mpcb->dfin_path_index) { sk = sk_it;break ;@@ -1585,9 +1593,12 @@ void mptcp_clean_rtx_infinite(const struct sk_buff *skb, struct sock *sk)static void mptcp_send_reset_rem_id (const struct mptcp_cb *mpcb,u8 rem_id) { -struct sock *sk_it, *tmpsk; +struct mptcp_tcp_sock *mptcp; +struct hlist_node *tmp; + +mptcp_for_each_sub_safe (mpcb, mptcp, tmp) { +struct sock *sk_it =mptcp_to_sock (mptcp); -mptcp_for_each_sk_safe (mpcb, sk_it, tmpsk) {if (tcp_sk (sk_it)->mptcp->rem_id == rem_id) {mptcp_reinject_data (sk_it,0 );mptcp_send_reset (sk_it);@@ -1892,13 +1903,15 @@ void tcp_parse_mptcp_options(const struct sk_buff *skb,bool mptcp_check_rtt (const struct tcp_sock *tp,int time) {struct mptcp_cb *mpcb = tp->mpcb; -struct sock *sk; +struct mptcp_tcp_sock *mptcp;u32 rtt_max =0 ;/* In MPTCP, we take the max delay across all flows, * in order to take into account meta-reordering buffers. */ -mptcp_for_each_sk (mpcb, sk) { +mptcp_for_each_sub (mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp); +if (!mptcp_sk_can_recv (sk))continue ;@@ -2173,9 +2186,9 @@ bool mptcp_handle_options(struct sock *sk, const struct tcphdr *th,if (mopt->saw_low_prio ==1 ) { tp->mptcp->rcv_low_prio = mopt->low_prio; }else { -struct sock *sk_it; -mptcp_for_each_sk (tp->mpcb, sk_it) { -struct mptcp_tcp_sock *mptcp =tcp_sk (sk_it)->mptcp; +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (tp->mpcb, mptcp) {if (mptcp->rem_id == mopt->prio_addr_id) mptcp->rcv_low_prio = mopt->low_prio; }@@ -2359,7 +2372,7 @@ bool mptcp_should_expand_sndbuf(const struct sock *sk) {const struct sock *meta_sk =mptcp_meta_sk (sk);const struct tcp_sock *meta_tp =tcp_sk (meta_sk); -const struct sock *sk_it; +const struct mptcp_tcp_sock *mptcp;/* We circumvent this check in tcp_check_space, because we want to * always call sk_write_space. So, we reproduce the check here. @@ -2385,8 +2398,9 @@ bool mptcp_should_expand_sndbuf(const struct sock *sk)/* For MPTCP we look for a subsocket that could send data. * If we found one, then we update the send-buffer. */ -mptcp_for_each_sk (meta_tp->mpcb, sk_it) { -struct tcp_sock *tp_it =tcp_sk (sk_it); +mptcp_for_each_sub (meta_tp->mpcb, mptcp) { +const struct sock *sk_it =mptcp_to_sock (mptcp); +const struct tcp_sock *tp_it =tcp_sk (sk_it);if (!mptcp_sk_can_send (sk_it))continue ;net/mptcp/mptcp_output.c +2
-1@@ -647,7 +647,6 @@ int mptcp_write_wakeup(struct sock *meta_sk, int mib) {struct tcp_sock *meta_tp =tcp_sk (meta_sk);struct sk_buff *skb; -struct sock *sk_it;int ans =0 ;if (meta_sk->sk_state ==TCP_CLOSE )@@ -704,17 +703,22 @@ int mptcp_write_wakeup(struct sock *meta_sk, int mib)return 0 ; }else { +struct mptcp_tcp_sock *mptcp; +
window_probe:if (between(meta_tp->snd_up, meta_tp->snd_una + 1, meta_tp->snd_una + 0xFFFF)) {net/mptcp/mptcp_output.c +3
-1-mptcp_for_each_sk (meta_tp->mpcb, sk_it) { +mptcp_for_each_sub (meta_tp->mpcb, mptcp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +if (mptcp_sk_can_send_ack(sk_it)) tcp_xmit_probe_skb(sk_it, 1, mib); } } /* At least one of the tcp_xmit_probe_skb's has to succeed */net/mptcp/mptcp_output.c +2
-1-mptcp_for_each_sk (meta_tp->mpcb, sk_it) { +mptcp_for_each_sub (meta_tp->mpcb, mptcp) { +struct sock *sk_it =mptcp_to_sock (mptcp);int ret; if (!mptcp_sk_can_send_ack(sk_it))net/mptcp/mptcp_output.c +18
-10@@ -732,6 +736,7 @@ bool mptcp_write_xmit(struct sock *meta_sk, unsigned int mss_now, int nonagle, int push_one, gfp_t gfp) {struct tcp_sock *meta_tp =tcp_sk (meta_sk), *subtp; +struct mptcp_tcp_sock *mptcp;struct sock *subsk =NULL ;struct mptcp_cb *mpcb = meta_tp->mpcb;struct sk_buff *skb;@@ -856,7 +861,8 @@ bool mptcp_write_xmit(struct sock *meta_sk, unsigned int mss_now, int nonagle,break ; } -mptcp_for_each_sk (mpcb, subsk) { +mptcp_for_each_sub (mpcb, mptcp) { + subsk =mptcp_to_sock (mptcp); subtp =tcp_sk (subsk);if (!(path_mask &mptcp_pi_to_flag (subtp->mptcp->path_index)))@@ -1353,7 +1359,7 @@ void mptcp_send_active_reset(struct sock *meta_sk, gfp_t priority)struct mptcp_cb *mpcb = meta_tp->mpcb;struct sock *sk; -if (!mpcb->connection_list) +if (hlist_empty (&mpcb->conn_list))return ;WARN_ON (meta_tp->send_mp_fclose);@@ -1728,10 +1734,11 @@ void mptcp_select_initial_window(const struct sock *sk, int __space, __u32 mss,static inline u64 mptcp_calc_rate (const struct sock *meta_sk,unsigned int mss,unsigned int (*mss_cb)(struct sock *sk)) { -struct sock *sk; +struct mptcp_tcp_sock *mptcp;u64 rate =0 ; -mptcp_for_each_sk (tcp_sk (meta_sk)->mpcb, sk) { +mptcp_for_each_sub (tcp_sk (meta_sk)->mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);struct tcp_sock *tp =tcp_sk (sk);int this_mss;u64 this_rate;@@ -1783,11 +1790,12 @@ static inline u64 mptcp_calc_rate(const struct sock *meta_sk, unsigned int mss,static unsigned int __mptcp_current_mss (const struct sock *meta_sk,unsigned int (*mss_cb)(struct sock *sk)) { +struct mptcp_tcp_sock *mptcp;unsigned int mss =0 ;u64 rate =0 ; -struct sock *sk; -mptcp_for_each_sk (tcp_sk (meta_sk)->mpcb, sk) { +mptcp_for_each_sub (tcp_sk (meta_sk)->mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);int this_mss;u64 this_rate;@@ -1852,14 +1860,16 @@ int mptcp_select_size(const struct sock *meta_sk, bool first_skb, bool zc)int mptcp_check_snd_buf (const struct tcp_sock *tp) { -const struct sock *sk; +const struct mptcp_tcp_sock *mptcp;u32 rtt_max = tp->srtt_us;u64 bw_est;if (!tp->srtt_us)return tp->reordering +1 ; -mptcp_for_each_sk (tp->mpcb, sk) { +mptcp_for_each_sub (tp->mpcb, mptcp) { +const struct sock *sk =mptcp_to_sock (mptcp); +if (!mptcp_sk_can_send (sk))continue ;@@ -1877,11 +1887,13 @@ int mptcp_check_snd_buf(const struct tcp_sock *tp)unsigned int mptcp_xmit_size_goal (const struct sock *meta_sk,u32 mss_now,int large_allowed) { -struct sock *sk;u32 xmit_size_goal =0 ;if (large_allowed && !tcp_sk (meta_sk)->mpcb->dss_csum) { -mptcp_for_each_sk (tcp_sk (meta_sk)->mpcb, sk) { +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (tcp_sk (meta_sk)->mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);int this_size_goal;if (!mptcp_sk_can_send (sk))net/mptcp/mptcp_sched.c +15
-7@@ -135,9 +135,10 @@ static struct socku32 min_srtt =0xffffffff ;bool found_unused = false;bool found_unused_una = false; -struct sock *sk; +struct mptcp_tcp_sock *mptcp; -mptcp_for_each_sk (mpcb, sk) { +mptcp_for_each_sub (mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);struct tcp_sock *tp =tcp_sk (sk);bool unused = false;@@ -219,7 +220,11 @@ static struct sock *get_available_subflow(struct sock *meta_sk,/* Answer data_fin on same subflow!!! */ if (meta_sk->sk_shutdown &RCV_SHUTDOWN && skb &&mptcp_is_data_fin (skb)) { -mptcp_for_each_sk (mpcb, sk) { +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (mpcb, mptcp) { + sk =mptcp_to_sock (mptcp); +if (tcp_sk (sk)->mptcp->path_index == mpcb->dfin_path_index &&mptcp_is_available (sk, skb, zero_wnd_test))return sk;@@ -252,7 +257,7 @@ static struct sk_buff *mptcp_rcv_buf_optimization(struct sock *sk, int penal) {struct sock *meta_sk;const struct tcp_sock *tp =tcp_sk (sk); -struct tcp_sock *tp_it; +struct mptcp_tcp_sock *mptcp;struct sk_buff *skb_head;struct defsched_priv *dsp =defsched_get_priv (tp);@@ -275,7 +280,9 @@ static struct sk_buff *mptcp_rcv_buf_optimization(struct sock *sk, int penal) goto retrans;/* Half the cwnd of the slow flow */ -mptcp_for_each_tp (tp->mpcb, tp_it) { +mptcp_for_each_sub (tp->mpcb, mptcp) { +struct tcp_sock *tp_it = mptcp->tp; +if (tp_it != tp &&TCP_SKB_CB (skb_head)->path_mask &mptcp_pi_to_flag (tp_it->mptcp->path_index)) {if (tp->srtt_us < tp_it->srtt_us &&inet_csk ((struct sock *)tp_it)->icsk_ca_state == TCP_CA_Open) {@@ -298,7 +305,9 @@ static struct sk_buff *mptcp_rcv_buf_optimization(struct sock *sk, int penal)/* Segment not yet injected into this path? Take it!!! */ if (!(TCP_SKB_CB (skb_head)->path_mask &mptcp_pi_to_flag (tp->mptcp->path_index))) {bool do_retrans = false; -mptcp_for_each_tp (tp->mpcb, tp_it) { +mptcp_for_each_sub (tp->mpcb, mptcp) { +struct tcp_sock *tp_it = mptcp->tp; +if (tp_it != tp &&TCP_SKB_CB (skb_head)->path_mask &mptcp_pi_to_flag (tp_it->mptcp->path_index)) {if (tp_it->snd_cwnd <=4 ) { --Quoted 3 lines
2.16.2
--
Mat Martineau
Intel OTC
--
Mat Martineau
Intel OTC
Christoph Paasch
Re: [MPTCP] [PATCH 10/20] mptcp: Make subflow-list an RCU-list
Quoted 88 lines · Mat Martineau
Quoted 86 lines · Christoph Paasch
Quoted 84 lines · Mat Martineau
Quoted 67 lines · Christoph Paasch
mptcp_add_sock()will now be called without holding the meta-level lock. However,mptcp_add_sock()wants to add the subflow to the meta-level list, thus we need to protect this by a lock. We use thempcb_list_lockfor that.Now that we are locking during add/del, and want to allow lockless traversal of the list, this implies that we need to make it an RCU-list.
So, this patch transitions to the RCU hlist. The list-traversal macros (
hlist_for_each_entry_rcu) require me to now pass themptcp_tcp_sockto mptcp_for_each_*. So, I had to change all the places in the code where we call one of the list-traversal macros to adapt to this.Signed-off-by Christoph Paasch <cpaasch(a)apple.com>Signed-off-by Matthieu Baerts <matthieu.baerts(a)tessares.net>(cherry picked from commit 7a662b690069642c138da16ff9396e6826fd0a95)---
include/net/mptcp.h | 67 +++++++++++++++++----------------- net/ipv4/af_inet.c | 9 ++--- net/ipv4/ip_sockglue.c | 7 ++-- net/ipv4/tcp.c | 31 +++++++++------- net/mptcp/mptcp_ctrl.c | 89 ++++++++++++++++++++++++++++------------------ net/mptcp/mptcp_fullmesh.c | 28 ++++++++++----- net/mptcp/mptcp_input.c | 44 +++++++++++++++-------- net/mptcp/mptcp_output.c | 38 +++++++++++++------- net/mptcp/mptcp_sched.c | 21 +++++++---- 9 files changed, 207 insertions(+), 127 deletions(-)include/net/mptcp.h +4
-6@@ -156,7 +156,7 @@ struct mptcp_options_received { };struct mptcp_tcp_sock { -struct tcp_sock *next;/* Next subflow socket */ +struct hlist_node node;struct hlist_node cb_list;struct mptcp_options_received rx_opt;@@ -254,7 +254,7 @@ struct mptcp_sched_ops {struct mptcp_cb {/* list of sockets in this multipath connection */ -struct tcp_sock *connection_list; +struct hlist_head conn_list;/* list of sockets that need a call to release_cb */ struct hlist_head callback_list;@@ -309,7 +309,7 @@ struct mptcp_cb {/***** Start of fields, used for subflow establishment */ struct sock *meta_sk; -/* Master socket, also part of the connection_list, this +/* Master socket, also part of the conn_list, this * socket is the one that the application sees. */ struct sock *master_sk;@@ -661,21 +661,17 @@ extern struct workqueue_struct *mptcp_wq; pr_err(fmt, ##args); \ } while (0) -/* Iterates over all subflows */ -#define mptcp_for_each_tp (mpcb, tp) \ - for ((tp) = (mpcb)->connection_list; (tp); (tp) = (tp)->mptcp->next) +static inline struct sock *mptcp_to_sock (const struct mptcp_tcp_sock *mptcp)include/net/mptcp.h extern struct workqueue_struct *mptcp_wq; · line 670
Quoted 10 lines
include/net/mptcp.h +5
-4+{ +return (struct sock *)mptcp->tp; +} -#define mptcp_for_each_sk (mpcb, sk) \ - for ((sk) = (struct sock *)(mpcb)->connection_list; \ - sk; \ - sk = (struct sock *)tcp_sk(sk)->mptcp->next) +#define mptcp_for_each_sub (__mpcb, __mptcp) \ + hlist_for_each_entry_rcu(__mptcp, &((__mpcb)->conn_list), node)When I asked about rcu synchronization with an earlier version of this patch set, you mentioned that it didn't look like an issue because the lifetime of the subflow sockets was still managed under a lock - the main reason to use rcu here is to allow adding to the list while the MPTCP-level lock is held.
Quoted 5 lines
Quoted 2 lines
And mostly, to be able to iterate over the list whithout the need to hold any lock. That way, we can do lockless subflow establishment.
There needs to be *some* lock to iterate over an RCU list, right? Even if it's just the non-blocking
rcu_read_lock?
In the lockless subflow establishment case, we are already in an rcu_read_lock'd region.
You can check __inet_lookup_established(), which also iterates over an RCU-list without explicitly taking the read-lock.
Because, in ip_local_deliver_finish, in order to get access to the protocol-handler (tcp_v4_rcv()), we need to do the rcu_read_lock(). And thus, we are always inside tcp_v4_rcv() while holding the rcu_read_lock().
In all the other cases (where we are not coming from tcp_v4_rcv(), but either from a timer or from user-space), we are holding the meta-socket lock (either the spin-lock or the other lock_sock()).
It's the same behavior as for normal TCP sockets, where we are not doing an explicit rcu_read_lock() to dereference tp->md5sig_info (as can be seen through the rcu_dereference_check).
Quoted 7 lines
You had mentioned in a previous response that traversal of the list always happened with the MPTCP-level lock held
(https://lists.01.org/pipermail/mptcp/2018-June/000665.html) - which I bring up not as a "gotcha" but hopefully to either jog some memories or help us collectively understand the locking model in this patch. Withoutrcu_read_lock, I was depending on that MPTCP-level lock to prevent deletion during iteration.
Yes, it has been a long time and my memory might have been a bit outdated on this patch-set. :)
Next time, let's try to dive into this level of detail during the submission on mptcp-dev.
But yeah, as I describe above, we are always holding a lock, either the rcu_read_lock, the meta's spin-lock or mark the meta-socket as "owned-by-user" through lock_sock().
Quoted 10 lines
Quoted 1 line
I think that's a very typical use of RCU-lists :)
Idiomatic RCU code I've seen (note: that's far from all RCU code :) ) and documentation (like
https://www.kernel.org/doc/Documentation/RCU/checklist.txt item #2 rule-of-thumb) involve using rcu_read_lock/rcu_read_unlock while iterating and then deferring some operations until the end of the grace period. When you get to the "Unless..." sentence at the end of checklist #2 I think we may be covered by the MPTCP-level lock, but it's uncommon enough usage that I think it's helpful to explain.
The "Unless..." part seems to me to be common in the networking subsystem as we have this concept of sockets being in an RCU-protected hash-table while also having a lock and being reference-counted.
Quoted 39 lines
Quoted 39 lines
Quoted 6 lines
That seems to also imply that rcu_read_lock/rcu_read_unlock are not required when using these list iteration macros. This is an unconventional use of rcu list entries, so it would be helpful to explain the expected use of the
conn_list. I think the main things are the lack ofrcu_read_lockduring iteration, and which locks to hold during iterate, add, and delete. Maybe the macro could check the state of the lock when built for debug?Currently, we don't have a "build-for-debug"-macro. But, definitely something we could add in the future.
Quoted 12 lines
include/net/mptcp.h extern struct workqueue_struct *mptcp_wq; · line 672
Quoted 9 lines
include/net/mptcp.h +1
-6-#define mptcp_for_each_sk_safe (__mpcb, __sk, __temp) \ - for (__sk = (struct sock *)(__mpcb)->connection_list, \ - __temp = __sk ? (struct sock *)tcp_sk(__sk)->mptcp->next : NULL; \ - __sk; \ - __sk = __temp, \ - __temp = __sk ? (struct sock *)tcp_sk(__sk)->mptcp->next : NULL) +/* Must be called with the appropriate lock held */ If MPTCP-level socket lock is the appropriate lock, can that be specified here?
The comment is kinda useless IMO. Seems to me to be rather a copy-paste issue as I was developing this part here.
Quoted 7 lines
include/net/mptcp.h extern struct workqueue_struct *mptcp_wq; · line 674
Quoted 2 lines
include/net/mptcp.h +2
-0+#define mptcp_for_each_sub_safe (__mpcb, __mptcp, __tmp) \ + hlist_for_each_entry_safe(__mptcp, __tmp, &((__mpcb)->conn_list), node)The comment just before
mptcp_for_each_sub_safeseems to imply that the difference between the two macros is that one needs a lock and one doesn't, but isn't the second (_safe) macro intended for use when subflows are being removed from the list while iterating? Could you add a comment to clarify?Yes, with the "appropriate lock" comment and
_safein itself this is kinda confusing. I will clarify that, saying that the_safemacro is there to allow removal while iterating and entirely remove the "appropriate lock" comment.
Quoted 4 lines
Quoted 2 lines
I am going to do this through a separate patch to
mptcp_trunkthat I will forward-port to mptcp-net-next later on.Sure, no problem using a later patch to clarify the comments.
Sounds good!
Thanks,
Christoph
Quoted 914 lines
Mat
Quoted 906 lines
Christoph
Quoted 900 lines
Thanks,
Mat
Quoted 887 lines
include/net/mptcp.h +22
-17/* Iterates over all bit set to 1 in a bitset */ #define mptcp_for_each_bit_set (b, i) \@@ -923,12 +919,14 @@ struct request_sock *rev_mptcp_rsk(const struct mptcp_request_sock *req)static inline bool mptcp_can_sendpage (struct sock *sk) { -struct sock *sk_it; +struct mptcp_tcp_sock *mptcp;if (tcp_sk (sk)->mpcb->dss_csum)return false; -mptcp_for_each_sk (tcp_sk (sk)->mpcb, sk_it) { +mptcp_for_each_sub (tcp_sk (sk)->mpcb, mptcp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +if (!(sk_it->sk_route_caps &NETIF_F_SG ))return false; }@@ -962,9 +960,12 @@ static inline void mptcp_send_reset(struct sock *sk)static inline void mptcp_sub_force_close_all (struct mptcp_cb *mpcb,struct sock *except) { -struct sock *sk_it, *tmp; +struct mptcp_tcp_sock *mptcp; +struct hlist_node *tmp; + +mptcp_for_each_sub_safe (mpcb, mptcp, tmp) { +struct sock *sk_it =mptcp_to_sock (mptcp); -mptcp_for_each_sk_safe (mpcb, sk_it, tmp) {if (sk_it != except)mptcp_send_reset (sk_it); }@@ -1150,12 +1151,14 @@ static inline int mptcp_sk_can_send_ack(const struct sock *sk)static inline bool mptcp_can_sg (const struct sock *meta_sk) { -struct sock *sk; +struct mptcp_tcp_sock *mptcp;if (tcp_sk (meta_sk)->mpcb->dss_csum)return false; -mptcp_for_each_sk (tcp_sk (meta_sk)->mpcb, sk) { +mptcp_for_each_sub (tcp_sk (meta_sk)->mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp); +if (!mptcp_sk_can_send (sk))continue ;if (!(sk->sk_route_caps &NETIF_F_SG ))@@ -1166,9 +1169,9 @@ static inline bool mptcp_can_sg(const struct sock *meta_sk)static inline void mptcp_set_rto (struct sock *sk) { -struct tcp_sock *tp =tcp_sk (sk); -struct sock *sk_it;struct inet_connection_sock *micsk =inet_csk (mptcp_meta_sk (sk)); +struct tcp_sock *tp =tcp_sk (sk); +struct mptcp_tcp_sock *mptcp;__u32 max_rto =0 ;/* We are in recovery-phase on the MPTCP-level. Do not update the @@ -1177,7 +1180,9 @@ static inline void mptcp_set_rto(struct sock *sk)if (micsk->icsk_retransmits)return ; -mptcp_for_each_sk (tp->mpcb, sk_it) { +mptcp_for_each_sub (tp->mpcb, mptcp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +if ((mptcp_sk_can_send (sk_it) || sk->sk_state ==TCP_SYN_RECV ) &&inet_csk (sk_it)->icsk_rto > max_rto) max_rto =inet_csk (sk_it)->icsk_rto;@@ -1266,10 +1271,10 @@ static inline bool mptcp_can_new_subflow(const struct sock *meta_sk)static inline int mptcp_subflow_count (const struct mptcp_cb *mpcb) { -struct sock *sk; +struct mptcp_tcp_sock *mptcp;int i =0 ; -mptcp_for_each_sk (mpcb, sk) +mptcp_for_each_sub (mpcb, mptcp) i++;return i;@@ -1287,12 +1292,8 @@ bool mptcp_prune_ofo_queue(struct sock *sk);do { \ } while (0) -/* Without MPTCP, we just do one iteration -* over the only socket available. This assumes that -* the sk/tp arg is the socket in that case. - */ -#define mptcp_for_each_sk (mpcb, sk) -#define mptcp_for_each_sk_safe (__mpcb, __sk, __temp) +#define mptcp_for_each_sub (__mpcb, __mptcp) \ + if (0)#define MPTCP_INC_STATS (net, field) \ do { \net/ipv4/af_inet.c +5
-4@@ -744,13 +744,14 @@ int inet_accept(struct socket *sock, struct socket *newsock, int flags,sock_rps_record_flow (sk2);if (sk2->sk_protocol ==IPPROTO_TCP &&mptcp (tcp_sk (sk2))) { -struct sock *sk_it = sk2; +struct mptcp_tcp_sock *mptcp; -mptcp_for_each_sk (tcp_sk (sk2)->mpcb, sk_it) -sock_rps_record_flow (sk_it); +mptcp_for_each_sub (tcp_sk (sk2)->mpcb, mptcp) { +sock_rps_record_flow (mptcp_to_sock (mptcp)); + }if (tcp_sk (sk2)->mpcb->master_sk) { - sk_it =tcp_sk (sk2)->mpcb->master_sk; +struct sock *sk_it =tcp_sk (sk2)->mpcb->master_sk;write_lock_bh (&sk_it->sk_callback_lock); sk_it->sk_wq = newsock->wq;net/ipv4/ip_sockglue.c +5
-2@@ -756,8 +756,11 @@ static int do_ip_setsockopt(struct sock *sk, int level,sk_dst_reset (sk);/* Update TOS on mptcp subflow */ if (is_meta_sk (sk)) { -struct sock *sk_it; -mptcp_for_each_sk (tcp_sk (sk)->mpcb, sk_it) { +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (tcp_sk (sk)->mpcb, mptcp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +if (inet_sk (sk_it)->tos !=inet_sk (sk)->tos) {inet_sk (sk_it)->tos =inet_sk (sk)->tos; sk_it->sk_priority = sk->sk_priority;net/ipv4/tcp.c +19
-12@@ -823,9 +823,11 @@ ssize_t tcp_splice_read(struct socket *sock, loff_t *ppos,#ifdef CONFIG_MPTCP if (mptcp (tcp_sk (sk))) { -struct sock *sk_it; -mptcp_for_each_sk (tcp_sk (sk)->mpcb, sk_it) -sock_rps_record_flow (sk_it); +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (tcp_sk (sk)->mpcb, mptcp) { +sock_rps_record_flow (mptcp_to_sock (mptcp)); + } }#endif @@ -993,7 +995,7 @@ ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset, }if (mptcp (tp)) { -struct sock *sk_it = sk; +struct mptcp_tcp_sock *mptcp;/* We must check this with socket-lock hold because we iterate * over the subflows. @@ -1008,8 +1010,9 @@ ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset,return ret; } -mptcp_for_each_sk (tp->mpcb, sk_it) -sock_rps_record_flow (sk_it); +mptcp_for_each_sub (tp->mpcb, mptcp) { +sock_rps_record_flow (mptcp_to_sock (mptcp)); + } }sk_clear_bit (SOCKWQ_ASYNC_NOSPACE , sk);@@ -1288,9 +1291,11 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size) }if (mptcp (tp)) { -struct sock *sk_it = sk; -mptcp_for_each_sk (tp->mpcb, sk_it) -sock_rps_record_flow (sk_it); +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (tp->mpcb, mptcp) { +sock_rps_record_flow (mptcp_to_sock (mptcp)); + } }if (unlikely (tp->repair)) {@@ -2006,9 +2011,11 @@ int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int nonblock,#ifdef CONFIG_MPTCP if (mptcp (tp)) { -struct sock *sk_it; -mptcp_for_each_sk (tp->mpcb, sk_it) -sock_rps_record_flow (sk_it); +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (tp->mpcb, mptcp) { +sock_rps_record_flow (mptcp_to_sock (mptcp)); + } }#endif net/mptcp/mptcp_ctrl.c +55
-34@@ -561,10 +561,12 @@ void mptcp_hash_remove_bh(struct tcp_sock *meta_tp)struct sock *mptcp_select_ack_sock (const struct sock *meta_sk) {const struct tcp_sock *meta_tp =tcp_sk (meta_sk); -struct sock *sk, *rttsk =NULL , *lastsk =NULL ; +struct sock *rttsk =NULL , *lastsk =NULL ;u32 min_time =0 , last_active =0 ; +struct mptcp_tcp_sock *mptcp; -mptcp_for_each_sk (meta_tp->mpcb, sk) { +mptcp_for_each_sub (meta_tp->mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);struct tcp_sock *tp =tcp_sk (sk);u32 elapsed;@@ -697,7 +699,8 @@ void mptcp_sock_destruct(struct sock *sk)void mptcp_destroy_sock (struct sock *sk) {if (is_meta_sk (sk)) { -struct sock *sk_it, *tmpsk; +struct mptcp_tcp_sock *mptcp; +struct hlist_node *tmp;__skb_queue_purge (&tcp_sk (sk)->mpcb->reinject_queue);@@ -707,7 +710,9 @@ void mptcp_destroy_sock(struct sock *sk)* not have been closed properly (as we are waiting for the * DATA_ACK of the DATA_FIN). */ -mptcp_for_each_sk_safe (tcp_sk (sk)->mpcb, sk_it, tmpsk) { +mptcp_for_each_sub_safe (tcp_sk (sk)->mpcb, mptcp, tmp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +/* Already did call tcp_close - waiting for graceful * closure, or if we are retransmitting fast-close on * the subflow. The reset (or timeout) will kill the @@ -1303,6 +1308,7 @@ static int mptcp_alloc_mpcb(struct sock *meta_sk, __u64 remote_key,INIT_LIST_HEAD (&mpcb->tw_list);INIT_HLIST_HEAD (&mpcb->callback_list); +INIT_HLIST_HEAD (&mpcb->conn_list);spin_lock_init (&mpcb->mpcb_list_lock);mptcp_mpcb_inherit_sockopts (meta_sk, master_sk);@@ -1392,8 +1398,12 @@ int mptcp_add_sock(struct sock *meta_sk, struct sock *sk, u8 loc_id, u8 rem_id,sock_hold (meta_sk);refcount_inc (&mpcb->mpcb_refcnt); - tp->mptcp->next = mpcb->connection_list; - mpcb->connection_list = tp; +local_bh_disable (); +spin_lock (&mpcb->mpcb_list_lock); +hlist_add_head_rcu (&tp->mptcp->node, &mpcb->conn_list); +spin_unlock (&mpcb->mpcb_list_lock); +local_bh_enable (); + tp->mptcp->attached =1 ; atomic_add(atomic_read(&((struct sock *)tp)->sk_rmem_alloc),@@ -1437,14 +1447,13 @@ int mptcp_add_sock(struct sock *meta_sk, struct sock *sk, u8 loc_id, u8 rem_id,void mptcp_del_sock (struct sock *sk) { -struct tcp_sock *tp =tcp_sk (sk), *tp_prev; +struct tcp_sock *tp =tcp_sk (sk);struct mptcp_cb *mpcb;if (!tp->mptcp || !tp->mptcp->attached)return ; mpcb = tp->mpcb; - tp_prev = mpcb->connection_list;if (mpcb->pm_ops->delete_subflow) mpcb->pm_ops->delete_subflow (sk);@@ -1453,17 +1462,10 @@ void mptcp_del_sock(struct sock *sk) __func__, mpcb->mptcp_loc_token, tp->mptcp->path_index, sk->sk_state, is_meta_sk(sk)); -if (tp_prev == tp) { - mpcb->connection_list = tp->mptcp->next; - }else { -for (; tp_prev && tp_prev->mptcp->next; tp_prev = tp_prev->mptcp->next) { -if (tp_prev->mptcp->next == tp) { - tp_prev->mptcp->next = tp->mptcp->next; -break ; - } - } - } - tp->mptcp->next =NULL ; +spin_lock (&mpcb->mpcb_list_lock); +hlist_del_init_rcu (&tp->mptcp->node); +spin_unlock (&mpcb->mpcb_list_lock); + tp->mptcp->attached =0 ; mpcb->path_index_bits &= ~(1 << tp->mptcp->path_index);@@ -1510,8 +1512,8 @@ void mptcp_update_metasocket(const struct sock *meta_sk)void mptcp_cleanup_rbuf (struct sock *meta_sk,int copied) {struct tcp_sock *meta_tp =tcp_sk (meta_sk); -struct sock *sk;bool recheck_rcv_window = false; +struct mptcp_tcp_sock *mptcp;__u32 rcv_window_now =0 ;if (copied >0 && !(meta_sk->sk_shutdown &RCV_SHUTDOWN )) {@@ -1522,7 +1524,8 @@ void mptcp_cleanup_rbuf(struct sock *meta_sk, int copied) recheck_rcv_window = true; } -mptcp_for_each_sk (meta_tp->mpcb, sk) { +mptcp_for_each_sub (meta_tp->mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);struct tcp_sock *tp =tcp_sk (sk);const struct inet_connection_sock *icsk =inet_csk (sk);@@ -1709,10 +1712,13 @@ EXPORT_SYMBOL(mptcp_sub_force_close);*/ void mptcp_update_sndbuf (const struct tcp_sock *tp) { -struct sock *meta_sk = tp->meta_sk, *sk; +struct sock *meta_sk = tp->meta_sk;int new_sndbuf =0 , old_sndbuf = meta_sk->sk_sndbuf; +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (tp->mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp); -mptcp_for_each_sk (tp->mpcb, sk) {if (!mptcp_sk_can_send (sk))continue ;@@ -1741,8 +1747,8 @@ void mptcp_update_sndbuf(const struct tcp_sock *tp)void mptcp_close (struct sock *meta_sk,long timeout) {struct tcp_sock *meta_tp =tcp_sk (meta_sk); -struct sock *sk_it, *tmpsk;struct mptcp_cb *mpcb = meta_tp->mpcb; +struct mptcp_tcp_sock *mptcp;struct sk_buff *skb;int data_was_unread =0 ;int state;@@ -1775,7 +1781,12 @@ void mptcp_close(struct sock *meta_sk, long timeout)/* If socket has been already reset (e.g. in tcp_reset()) - kill it. */ if (meta_sk->sk_state ==TCP_CLOSE ) { -mptcp_for_each_sk_safe (mpcb, sk_it, tmpsk) { +struct mptcp_tcp_sock *mptcp; +struct hlist_node *tmp; + +mptcp_for_each_sub_safe (mpcb, mptcp, tmp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +if (tcp_sk (sk_it)->send_mp_fclose)continue ;mptcp_sub_close (sk_it,0 );@@ -1796,10 +1807,14 @@ void mptcp_close(struct sock *meta_sk, long timeout) }else if (tcp_close_state (meta_sk)) {mptcp_send_fin (meta_sk); }else if (meta_tp->snd_una == meta_tp->write_seq) { +struct mptcp_tcp_sock *mptcp; +struct hlist_node *tmp; +/* The DATA_FIN has been sent and acknowledged * (e.g., by sk_shutdown). Close all the other subflows */ -mptcp_for_each_sk_safe (mpcb, sk_it, tmpsk) { +mptcp_for_each_sub_safe (mpcb, mptcp, tmp) { +struct sock *sk_it =mptcp_to_sock (mptcp);unsigned long delay =0 ;/* If we are the passive closer, don't trigger * subflow-fin until the subflow has been finned @@ -1823,7 +1838,9 @@ void mptcp_close(struct sock *meta_sk, long timeout)/* socket will be freed after mptcp_close - we have to prevent * access from the subflows. */ -mptcp_for_each_sk (mpcb, sk_it) { +mptcp_for_each_sub (mpcb, mptcp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +/* Similar to sock_orphan, but we don't set it DEAD, because * the callbacks are still set and must be called. */ @@ -1908,8 +1925,9 @@ void mptcp_close(struct sock *meta_sk, long timeout)void mptcp_disconnect (struct sock *sk) { -struct sock *subsk, *tmpsk; +struct mptcp_tcp_sock *mptcp;struct tcp_sock *tp =tcp_sk (sk); +struct hlist_node *tmp;__skb_queue_purge (&tp->mpcb->reinject_queue);@@ -1917,7 +1935,9 @@ void mptcp_disconnect(struct sock *sk)mptcp_hash_remove_bh (tp);local_bh_disable (); -mptcp_for_each_sk_safe (tp->mpcb, subsk, tmpsk) { +mptcp_for_each_sub_safe (tp->mpcb, mptcp, tmp) { +struct sock *subsk =mptcp_to_sock (mptcp); +/* The socket will get removed from the subsocket-list * and made non-mptcp by setting mpc to 0. * @@ -2606,7 +2626,6 @@ static void mptcp_get_sub_info(struct sock *sk, struct mptcp_sub_info *info)int mptcp_get_info (const struct sock *meta_sk,char __user *optval,int optlen) {const struct tcp_sock *meta_tp =tcp_sk (meta_sk); -struct sock *sk;struct mptcp_meta_info meta_info;struct mptcp_info m_info;@@ -2652,16 +2671,17 @@ int mptcp_get_info(const struct sock *meta_sk, char __user *optval, int optlen)if (m_info.subflows) {unsigned int len, sub_len =0 ; +struct mptcp_tcp_sock *mptcp;char __user *ptr; ptr = (char __user *)m_info.subflows; len = m_info.sub_len; -mptcp_for_each_sk (meta_tp->mpcb, sk) { +mptcp_for_each_sub (meta_tp->mpcb, mptcp) {struct tcp_info t_info;unsigned int tmp_len; -tcp_get_info (sk, &t_info); +tcp_get_info (mptcp_to_sock (mptcp), &t_info); tmp_len = min_t(unsigned int, len, info_len); len -= tmp_len;@@ -2681,6 +2701,7 @@ int mptcp_get_info(const struct sock *meta_sk, char __user *optval, int optlen)if (m_info.subflow_info) {unsigned int len, sub_info_len, total_sub_info_len =0 ; +struct mptcp_tcp_sock *mptcp;char __user *ptr; ptr = (char __user *)m_info.subflow_info;@@ -2690,11 +2711,11 @@ int mptcp_get_info(const struct sock *meta_sk, char __user *optval, int optlen) sizeof(struct mptcp_sub_info)); m_info.sub_info_len = sub_info_len; -mptcp_for_each_sk (meta_tp->mpcb, sk) { +mptcp_for_each_sub (meta_tp->mpcb, mptcp) {struct mptcp_sub_info m_sub_info;unsigned int tmp_len; -mptcp_get_sub_info (sk, &m_sub_info); +mptcp_get_sub_info (mptcp_to_sock (mptcp), &m_sub_info); tmp_len = min_t(unsigned int, len, sub_info_len); len -= tmp_len;net/mptcp/mptcp_fullmesh.c +20
-8@@ -903,8 +903,9 @@ static void mptcp_address_worker(struct work_struct *work) }if (event->code ==MPTCP_EVENT_DEL ) { -struct sock *sk, *tmpsk; +struct mptcp_tcp_sock *mptcp;struct mptcp_loc_addr *mptcp_local; +struct hlist_node *tmp;bool found = false; mptcp_local =rcu_dereference_bh (fm_ns->local);@@ -914,7 +915,9 @@ static void mptcp_address_worker(struct work_struct *work)update_addr_bitfields (meta_sk, mptcp_local);/* Look for the socket and remove him */ -mptcp_for_each_sk_safe (mpcb, sk, tmpsk) { +mptcp_for_each_sub_safe (mpcb, mptcp, tmp) { +struct sock *sk =mptcp_to_sock (mptcp); + if ((event->family == AF_INET6 && (sk->sk_family == AF_INET || mptcp_v6_is_v4_mapped(sk))) ||@@ -964,9 +967,10 @@ static void mptcp_address_worker(struct work_struct *work) }if (event->code ==MPTCP_EVENT_MOD ) { -struct sock *sk; +struct mptcp_tcp_sock *mptcp; -mptcp_for_each_sk (mpcb, sk) { +mptcp_for_each_sub (mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);struct tcp_sock *tp =tcp_sk (sk); if (event->family == AF_INET && (sk->sk_family == AF_INET ||@@ -1455,8 +1459,9 @@ static void full_mesh_release_sock(struct sock *meta_sk)struct mptcp_cb *mpcb =tcp_sk (meta_sk)->mpcb;struct fullmesh_priv *fmp =fullmesh_get_priv (mpcb);const struct mptcp_fm_ns *fm_ns =fm_get_ns (sock_net (meta_sk)); -struct sock *sk, *tmpsk;bool meta_v4 = meta_sk->sk_family ==AF_INET ; +struct mptcp_tcp_sock *mptcp; +struct hlist_node *tmp;int i;rcu_read_lock_bh ();@@ -1470,7 +1475,8 @@ static void full_mesh_release_sock(struct sock *meta_sk)struct in_addr ifa = mptcp_local->locaddr4[i].addr;bool found = false; -mptcp_for_each_sk (mpcb, sk) { +mptcp_for_each_sub (mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);struct tcp_sock *tp =tcp_sk (sk); if (sk->sk_family == AF_INET6 &&@@ -1491,6 +1497,8 @@ static void full_mesh_release_sock(struct sock *meta_sk) }if (!found) { +struct sock *sk; + fmp->add_addr++; mpcb->addr_signal =1 ;@@ -1511,7 +1519,8 @@ static void full_mesh_release_sock(struct sock *meta_sk)struct in6_addr ifa = mptcp_local->locaddr6[i].addr;bool found = false; -mptcp_for_each_sk (mpcb, sk) { +mptcp_for_each_sub (mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);struct tcp_sock *tp =tcp_sk (sk); if (sk->sk_family == AF_INET ||@@ -1532,6 +1541,8 @@ static void full_mesh_release_sock(struct sock *meta_sk) }if (!found) { +struct sock *sk; + fmp->add_addr++; mpcb->addr_signal =1 ;@@ -1546,7 +1557,8 @@ static void full_mesh_release_sock(struct sock *meta_sk)#endif /* Now, detect address-removals */ -mptcp_for_each_sk_safe (mpcb, sk, tmpsk) { +mptcp_for_each_sub_safe (mpcb, mptcp, tmp) { +struct sock *sk =mptcp_to_sock (mptcp);bool shall_remove = true;if (sk->sk_family ==AF_INET ||mptcp_v6_is_v4_mapped (sk)) {net/mptcp/mptcp_input.c +29
-15@@ -126,12 +126,14 @@ static void mptcp_clean_rtx_queue(struct sock *meta_sk, u32 prior_snd_una)tcp_rtx_queue_unlink (skb, meta_sk);if (mptcp_is_data_fin (skb)) { -struct sock *sk_it, *sk_tmp; +struct mptcp_tcp_sock *mptcp; +struct hlist_node *tmp;/* DATA_FIN has been acknowledged - now we can close * the subflows */ -mptcp_for_each_sk_safe (mpcb, sk_it, sk_tmp) { +mptcp_for_each_sub_safe (mpcb, mptcp, tmp) { +struct sock *sk_it =mptcp_to_sock (mptcp);unsigned long delay =0 ;/* If we are the passive closer, don't trigger @@ -347,6 +349,7 @@ static int mptcp_verif_dss_csum(struct sock *sk)/* Now, checksum must be 0 */ if (unlikely (csum_fold (csum_tcp))) { +struct mptcp_tcp_sock *mptcp;struct sock *sk_it =NULL ; pr_err("%s csum is wrong: %#x tcp-seq %u dss_csum_added %d overflowed %d iterations %d\n",@@ -362,7 +365,9 @@ static int mptcp_verif_dss_csum(struct sock *sk) tp->mpcb->csum_cutoff_seq = tp->mptcp->map_data_seq;/* Search for another subflow that is fully established */ -mptcp_for_each_sk (tp->mpcb, sk_it) { +mptcp_for_each_sub (tp->mpcb, mptcp) { + sk_it =mptcp_to_sock (mptcp); +if (sk_it != sk &&tcp_sk (sk_it)->mptcp->fully_established)break ;@@ -1308,12 +1313,15 @@ int mptcp_do_join_short(struct sk_buff *skb,*/ void mptcp_fin (struct sock *meta_sk) { -struct sock *sk =NULL , *sk_it; +struct sock *sk =NULL ;struct tcp_sock *meta_tp =tcp_sk (meta_sk);struct mptcp_cb *mpcb = meta_tp->mpcb; +struct mptcp_tcp_sock *mptcp;unsigned char state; -mptcp_for_each_sk (mpcb, sk_it) { +mptcp_for_each_sub (mpcb, mptcp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +if (tcp_sk (sk_it)->mptcp->path_index == mpcb->dfin_path_index) { sk = sk_it;break ;@@ -1585,9 +1593,12 @@ void mptcp_clean_rtx_infinite(const struct sk_buff *skb, struct sock *sk)static void mptcp_send_reset_rem_id (const struct mptcp_cb *mpcb,u8 rem_id) { -struct sock *sk_it, *tmpsk; +struct mptcp_tcp_sock *mptcp; +struct hlist_node *tmp; + +mptcp_for_each_sub_safe (mpcb, mptcp, tmp) { +struct sock *sk_it =mptcp_to_sock (mptcp); -mptcp_for_each_sk_safe (mpcb, sk_it, tmpsk) {if (tcp_sk (sk_it)->mptcp->rem_id == rem_id) {mptcp_reinject_data (sk_it,0 );mptcp_send_reset (sk_it);@@ -1892,13 +1903,15 @@ void tcp_parse_mptcp_options(const struct sk_buff *skb,bool mptcp_check_rtt (const struct tcp_sock *tp,int time) {struct mptcp_cb *mpcb = tp->mpcb; -struct sock *sk; +struct mptcp_tcp_sock *mptcp;u32 rtt_max =0 ;/* In MPTCP, we take the max delay across all flows, * in order to take into account meta-reordering buffers. */ -mptcp_for_each_sk (mpcb, sk) { +mptcp_for_each_sub (mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp); +if (!mptcp_sk_can_recv (sk))continue ;@@ -2173,9 +2186,9 @@ bool mptcp_handle_options(struct sock *sk, const struct tcphdr *th,if (mopt->saw_low_prio ==1 ) { tp->mptcp->rcv_low_prio = mopt->low_prio; }else { -struct sock *sk_it; -mptcp_for_each_sk (tp->mpcb, sk_it) { -struct mptcp_tcp_sock *mptcp =tcp_sk (sk_it)->mptcp; +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (tp->mpcb, mptcp) {if (mptcp->rem_id == mopt->prio_addr_id) mptcp->rcv_low_prio = mopt->low_prio; }@@ -2359,7 +2372,7 @@ bool mptcp_should_expand_sndbuf(const struct sock *sk) {const struct sock *meta_sk =mptcp_meta_sk (sk);const struct tcp_sock *meta_tp =tcp_sk (meta_sk); -const struct sock *sk_it; +const struct mptcp_tcp_sock *mptcp;/* We circumvent this check in tcp_check_space, because we want to * always call sk_write_space. So, we reproduce the check here. @@ -2385,8 +2398,9 @@ bool mptcp_should_expand_sndbuf(const struct sock *sk)/* For MPTCP we look for a subsocket that could send data. * If we found one, then we update the send-buffer. */ -mptcp_for_each_sk (meta_tp->mpcb, sk_it) { -struct tcp_sock *tp_it =tcp_sk (sk_it); +mptcp_for_each_sub (meta_tp->mpcb, mptcp) { +const struct sock *sk_it =mptcp_to_sock (mptcp); +const struct tcp_sock *tp_it =tcp_sk (sk_it);if (!mptcp_sk_can_send (sk_it))continue ;net/mptcp/mptcp_output.c +2
-1@@ -647,7 +647,6 @@ int mptcp_write_wakeup(struct sock *meta_sk, int mib) {struct tcp_sock *meta_tp =tcp_sk (meta_sk);struct sk_buff *skb; -struct sock *sk_it;int ans =0 ;if (meta_sk->sk_state ==TCP_CLOSE )@@ -704,17 +703,22 @@ int mptcp_write_wakeup(struct sock *meta_sk, int mib)return 0 ; }else { +struct mptcp_tcp_sock *mptcp; +
window_probe:if (between(meta_tp->snd_up, meta_tp->snd_una + 1, meta_tp->snd_una + 0xFFFF)) {net/mptcp/mptcp_output.c +3
-1-mptcp_for_each_sk (meta_tp->mpcb, sk_it) { +mptcp_for_each_sub (meta_tp->mpcb, mptcp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +if (mptcp_sk_can_send_ack(sk_it)) tcp_xmit_probe_skb(sk_it, 1, mib); } } /* At least one of the tcp_xmit_probe_skb's has to succeed */net/mptcp/mptcp_output.c +2
-1-mptcp_for_each_sk (meta_tp->mpcb, sk_it) { +mptcp_for_each_sub (meta_tp->mpcb, mptcp) { +struct sock *sk_it =mptcp_to_sock (mptcp);int ret; if (!mptcp_sk_can_send_ack(sk_it))net/mptcp/mptcp_output.c +18
-10@@ -732,6 +736,7 @@ bool mptcp_write_xmit(struct sock *meta_sk, unsigned int mss_now, int nonagle, int push_one, gfp_t gfp) {struct tcp_sock *meta_tp =tcp_sk (meta_sk), *subtp; +struct mptcp_tcp_sock *mptcp;struct sock *subsk =NULL ;struct mptcp_cb *mpcb = meta_tp->mpcb;struct sk_buff *skb;@@ -856,7 +861,8 @@ bool mptcp_write_xmit(struct sock *meta_sk, unsigned int mss_now, int nonagle,break ; } -mptcp_for_each_sk (mpcb, subsk) { +mptcp_for_each_sub (mpcb, mptcp) { + subsk =mptcp_to_sock (mptcp); subtp =tcp_sk (subsk);if (!(path_mask &mptcp_pi_to_flag (subtp->mptcp->path_index)))@@ -1353,7 +1359,7 @@ void mptcp_send_active_reset(struct sock *meta_sk, gfp_t priority)struct mptcp_cb *mpcb = meta_tp->mpcb;struct sock *sk; -if (!mpcb->connection_list) +if (hlist_empty (&mpcb->conn_list))return ;WARN_ON (meta_tp->send_mp_fclose);@@ -1728,10 +1734,11 @@ void mptcp_select_initial_window(const struct sock *sk, int __space, __u32 mss,static inline u64 mptcp_calc_rate (const struct sock *meta_sk,unsigned int mss,unsigned int (*mss_cb)(struct sock *sk)) { -struct sock *sk; +struct mptcp_tcp_sock *mptcp;u64 rate =0 ; -mptcp_for_each_sk (tcp_sk (meta_sk)->mpcb, sk) { +mptcp_for_each_sub (tcp_sk (meta_sk)->mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);struct tcp_sock *tp =tcp_sk (sk);int this_mss;u64 this_rate;@@ -1783,11 +1790,12 @@ static inline u64 mptcp_calc_rate(const struct sock *meta_sk, unsigned int mss,static unsigned int __mptcp_current_mss (const struct sock *meta_sk,unsigned int (*mss_cb)(struct sock *sk)) { +struct mptcp_tcp_sock *mptcp;unsigned int mss =0 ;u64 rate =0 ; -struct sock *sk; -mptcp_for_each_sk (tcp_sk (meta_sk)->mpcb, sk) { +mptcp_for_each_sub (tcp_sk (meta_sk)->mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);int this_mss;u64 this_rate;@@ -1852,14 +1860,16 @@ int mptcp_select_size(const struct sock *meta_sk, bool first_skb, bool zc)int mptcp_check_snd_buf (const struct tcp_sock *tp) { -const struct sock *sk; +const struct mptcp_tcp_sock *mptcp;u32 rtt_max = tp->srtt_us;u64 bw_est;if (!tp->srtt_us)return tp->reordering +1 ; -mptcp_for_each_sk (tp->mpcb, sk) { +mptcp_for_each_sub (tp->mpcb, mptcp) { +const struct sock *sk =mptcp_to_sock (mptcp); +if (!mptcp_sk_can_send (sk))continue ;@@ -1877,11 +1887,13 @@ int mptcp_check_snd_buf(const struct tcp_sock *tp)unsigned int mptcp_xmit_size_goal (const struct sock *meta_sk,u32 mss_now,int large_allowed) { -struct sock *sk;u32 xmit_size_goal =0 ;if (large_allowed && !tcp_sk (meta_sk)->mpcb->dss_csum) { -mptcp_for_each_sk (tcp_sk (meta_sk)->mpcb, sk) { +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (tcp_sk (meta_sk)->mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);int this_size_goal;if (!mptcp_sk_can_send (sk))net/mptcp/mptcp_sched.c +15
-7@@ -135,9 +135,10 @@ static struct socku32 min_srtt =0xffffffff ;bool found_unused = false;bool found_unused_una = false; -struct sock *sk; +struct mptcp_tcp_sock *mptcp; -mptcp_for_each_sk (mpcb, sk) { +mptcp_for_each_sub (mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);struct tcp_sock *tp =tcp_sk (sk);bool unused = false;@@ -219,7 +220,11 @@ static struct sock *get_available_subflow(struct sock *meta_sk,/* Answer data_fin on same subflow!!! */ if (meta_sk->sk_shutdown &RCV_SHUTDOWN && skb &&mptcp_is_data_fin (skb)) { -mptcp_for_each_sk (mpcb, sk) { +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (mpcb, mptcp) { + sk =mptcp_to_sock (mptcp); +if (tcp_sk (sk)->mptcp->path_index == mpcb->dfin_path_index &&mptcp_is_available (sk, skb, zero_wnd_test))return sk;@@ -252,7 +257,7 @@ static struct sk_buff *mptcp_rcv_buf_optimization(struct sock *sk, int penal) {struct sock *meta_sk;const struct tcp_sock *tp =tcp_sk (sk); -struct tcp_sock *tp_it; +struct mptcp_tcp_sock *mptcp;struct sk_buff *skb_head;struct defsched_priv *dsp =defsched_get_priv (tp);@@ -275,7 +280,9 @@ static struct sk_buff *mptcp_rcv_buf_optimization(struct sock *sk, int penal) goto retrans;/* Half the cwnd of the slow flow */ -mptcp_for_each_tp (tp->mpcb, tp_it) { +mptcp_for_each_sub (tp->mpcb, mptcp) { +struct tcp_sock *tp_it = mptcp->tp; +if (tp_it != tp &&TCP_SKB_CB (skb_head)->path_mask &mptcp_pi_to_flag (tp_it->mptcp->path_index)) {if (tp->srtt_us < tp_it->srtt_us &&inet_csk ((struct sock *)tp_it)->icsk_ca_state == TCP_CA_Open) {@@ -298,7 +305,9 @@ static struct sk_buff *mptcp_rcv_buf_optimization(struct sock *sk, int penal)/* Segment not yet injected into this path? Take it!!! */ if (!(TCP_SKB_CB (skb_head)->path_mask &mptcp_pi_to_flag (tp->mptcp->path_index))) {bool do_retrans = false; -mptcp_for_each_tp (tp->mpcb, tp_it) { +mptcp_for_each_sub (tp->mpcb, mptcp) { +struct tcp_sock *tp_it = mptcp->tp; +if (tp_it != tp &&TCP_SKB_CB (skb_head)->path_mask &mptcp_pi_to_flag (tp_it->mptcp->path_index)) {if (tp_it->snd_cwnd <=4 ) { --Quoted 3 lines
2.16.2
--
Mat Martineau
Intel OTC--
Mat Martineau
Intel OTC
Mat Martineau
Re: [MPTCP] [PATCH 10/20] mptcp: Make subflow-list an RCU-list
Quoted 105 lines · Christoph Paasch
Quoted 88 lines · Mat Martineau
Quoted 86 lines · Christoph Paasch
Quoted 84 lines · Mat Martineau
Quoted 67 lines · Christoph Paasch
mptcp_add_sock()will now be called without holding the meta-level lock. However,mptcp_add_sock()wants to add the subflow to the meta-level list, thus we need to protect this by a lock. We use thempcb_list_lockfor that.Now that we are locking during add/del, and want to allow lockless traversal of the list, this implies that we need to make it an RCU-list.
So, this patch transitions to the RCU hlist. The list-traversal macros (
hlist_for_each_entry_rcu) require me to now pass themptcp_tcp_sockto mptcp_for_each_*. So, I had to change all the places in the code where we call one of the list-traversal macros to adapt to this.Signed-off-by Christoph Paasch <cpaasch(a)apple.com>Signed-off-by Matthieu Baerts <matthieu.baerts(a)tessares.net>(cherry picked from commit 7a662b690069642c138da16ff9396e6826fd0a95)---
include/net/mptcp.h | 67 +++++++++++++++++----------------- net/ipv4/af_inet.c | 9 ++--- net/ipv4/ip_sockglue.c | 7 ++-- net/ipv4/tcp.c | 31 +++++++++------- net/mptcp/mptcp_ctrl.c | 89 ++++++++++++++++++++++++++++------------------ net/mptcp/mptcp_fullmesh.c | 28 ++++++++++----- net/mptcp/mptcp_input.c | 44 +++++++++++++++-------- net/mptcp/mptcp_output.c | 38 +++++++++++++------- net/mptcp/mptcp_sched.c | 21 +++++++---- 9 files changed, 207 insertions(+), 127 deletions(-)include/net/mptcp.h +4
-6@@ -156,7 +156,7 @@ struct mptcp_options_received { };struct mptcp_tcp_sock { -struct tcp_sock *next;/* Next subflow socket */ +struct hlist_node node;struct hlist_node cb_list;struct mptcp_options_received rx_opt;@@ -254,7 +254,7 @@ struct mptcp_sched_ops {struct mptcp_cb {/* list of sockets in this multipath connection */ -struct tcp_sock *connection_list; +struct hlist_head conn_list;/* list of sockets that need a call to release_cb */ struct hlist_head callback_list;@@ -309,7 +309,7 @@ struct mptcp_cb {/***** Start of fields, used for subflow establishment */ struct sock *meta_sk; -/* Master socket, also part of the connection_list, this +/* Master socket, also part of the conn_list, this * socket is the one that the application sees. */ struct sock *master_sk;@@ -661,21 +661,17 @@ extern struct workqueue_struct *mptcp_wq; pr_err(fmt, ##args); \ } while (0) -/* Iterates over all subflows */ -#define mptcp_for_each_tp (mpcb, tp) \ - for ((tp) = (mpcb)->connection_list; (tp); (tp) = (tp)->mptcp->next) +static inline struct sock *mptcp_to_sock (const struct mptcp_tcp_sock *mptcp)include/net/mptcp.h extern struct workqueue_struct *mptcp_wq; · line 670
Quoted 10 lines
include/net/mptcp.h +5
-4+{ +return (struct sock *)mptcp->tp; +} -#define mptcp_for_each_sk (mpcb, sk) \ - for ((sk) = (struct sock *)(mpcb)->connection_list; \ - sk; \ - sk = (struct sock *)tcp_sk(sk)->mptcp->next) +#define mptcp_for_each_sub (__mpcb, __mptcp) \ + hlist_for_each_entry_rcu(__mptcp, &((__mpcb)->conn_list), node)When I asked about rcu synchronization with an earlier version of this patch set, you mentioned that it didn't look like an issue because the lifetime of the subflow sockets was still managed under a lock - the main reason to use rcu here is to allow adding to the list while the MPTCP-level lock is held.
Quoted 5 lines
Quoted 2 lines
And mostly, to be able to iterate over the list whithout the need to hold any lock. That way, we can do lockless subflow establishment.
There needs to be *some* lock to iterate over an RCU list, right? Even if it's just the non-blocking
rcu_read_lock?In the lockless subflow establishment case, we are already in an rcu_read_lock'd region.
You can check
__inet_lookup_established(), which also iterates over an RCU-list without explicitly taking the read-lock.
Because, inip_local_deliver_finish, in order to get access to the protocol-handler (tcp_v4_rcv()), we need to do thercu_read_lock(). And thus, we are always insidetcp_v4_rcv()while holding thercu_read_lock().
Quoted 7 lines
In all the other cases (where we are not coming from
tcp_v4_rcv(), but either from a timer or from user-space), we are holding the meta-socket lock (either the spin-lock or the otherlock_sock()).It's the same behavior as for normal TCP sockets, where we are not doing an explicit
rcu_read_lock()to dereferencetp->md5sig_info(as can be seen through thercu_dereference_check).
Ok, that lines up with what I understood from the code.
Quoted 9 lines
Quoted 8 lines
You had mentioned in a previous response that traversal of the list always happened with the MPTCP-level lock held
(https://lists.01.org/pipermail/mptcp/2018-June/000665.html) - which I bring up not as a "gotcha" but hopefully to either jog some memories or help us collectively understand the locking model in this patch. Withoutrcu_read_lock, I was depending on that MPTCP-level lock to prevent deletion during iteration.
Quoted 5 lines
Yes, it has been a long time and my memory might have been a bit outdated on this patch-set. :)
Next time, let's try to dive into this level of detail during the submission on mptcp-dev.
Yes, makes sense to have the discussion early on for changes that will end up on both branches.
Quoted 17 lines
But yeah, as I describe above, we are always holding a lock, either the
rcu_read_lock, the meta's spin-lock or mark the meta-socket as "owned-by-user" throughlock_sock().Quoted 10 lines
Quoted 1 line
I think that's a very typical use of RCU-lists :)
Idiomatic RCU code I've seen (note: that's far from all RCU code :) ) and documentation (like
https://www.kernel.org/doc/Documentation/RCU/checklist.txt item #2 rule-of-thumb) involve using rcu_read_lock/rcu_read_unlock while iterating and then deferring some operations until the end of the grace period. When you get to the "Unless..." sentence at the end of checklist #2 I think we may be covered by the MPTCP-level lock, but it's uncommon enough usage that I think it's helpful to explain.
Quoted 3 lines
The "Unless..." part seems to me to be common in the networking subsystem as we have this concept of sockets being in an RCU-protected hash-table while also having a lock and being reference-counted.
Thanks for the explanations in this thread - I'll try to get more context from the other RCU usage in the networking stack. At this point in the process it works to merge the series and confirm the locking behavior from there. If anything comes up we'll fix it.
Regards,
Mat
Quoted 43 lines
Quoted 43 lines
Quoted 42 lines
Quoted 7 lines
That seems to also imply that rcu_read_lock/rcu_read_unlock are not required when using these list iteration macros. This is an unconventional use of rcu list entries, so it would be helpful to explain the expected use of the
conn_list. I think the main things are the lack ofrcu_read_lockduring iteration, and which locks to hold during iterate, add, and delete. Maybe the macro could check the state of the lock when built for debug?Currently, we don't have a "build-for-debug"-macro. But, definitely something we could add in the future.
Quoted 12 lines
include/net/mptcp.h extern struct workqueue_struct *mptcp_wq; · line 672
Quoted 9 lines
include/net/mptcp.h +1
-6-#define mptcp_for_each_sk_safe (__mpcb, __sk, __temp) \ - for (__sk = (struct sock *)(__mpcb)->connection_list, \ - __temp = __sk ? (struct sock *)tcp_sk(__sk)->mptcp->next : NULL; \ - __sk; \ - __sk = __temp, \ - __temp = __sk ? (struct sock *)tcp_sk(__sk)->mptcp->next : NULL) +/* Must be called with the appropriate lock held */ If MPTCP-level socket lock is the appropriate lock, can that be specified here?
The comment is kinda useless IMO. Seems to me to be rather a copy-paste issue as I was developing this part here.
Quoted 7 lines
include/net/mptcp.h extern struct workqueue_struct *mptcp_wq; · line 674
Quoted 2 lines
include/net/mptcp.h +2
-0+#define mptcp_for_each_sub_safe (__mpcb, __mptcp, __tmp) \ + hlist_for_each_entry_safe(__mptcp, __tmp, &((__mpcb)->conn_list), node)The comment just before
mptcp_for_each_sub_safeseems to imply that the difference between the two macros is that one needs a lock and one doesn't, but isn't the second (_safe) macro intended for use when subflows are being removed from the list while iterating? Could you add a comment to clarify?Yes, with the "appropriate lock" comment and
_safein itself this is kinda confusing. I will clarify that, saying that the_safemacro is there to allow removal while iterating and entirely remove the "appropriate lock" comment.I am going to do this through a separate patch to
mptcp_trunkthat I will forward-port to mptcp-net-next later on.
Quoted 923 lines
Quoted 1 line
Sure, no problem using a later patch to clarify the comments.
Sounds good!
Thanks,
ChristophQuoted 914 lines
Mat
Quoted 906 lines
Christoph
Quoted 900 lines
Thanks,
Mat
Quoted 887 lines
include/net/mptcp.h +22
-17/* Iterates over all bit set to 1 in a bitset */ #define mptcp_for_each_bit_set (b, i) \@@ -923,12 +919,14 @@ struct request_sock *rev_mptcp_rsk(const struct mptcp_request_sock *req)static inline bool mptcp_can_sendpage (struct sock *sk) { -struct sock *sk_it; +struct mptcp_tcp_sock *mptcp;if (tcp_sk (sk)->mpcb->dss_csum)return false; -mptcp_for_each_sk (tcp_sk (sk)->mpcb, sk_it) { +mptcp_for_each_sub (tcp_sk (sk)->mpcb, mptcp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +if (!(sk_it->sk_route_caps &NETIF_F_SG ))return false; }@@ -962,9 +960,12 @@ static inline void mptcp_send_reset(struct sock *sk)static inline void mptcp_sub_force_close_all (struct mptcp_cb *mpcb,struct sock *except) { -struct sock *sk_it, *tmp; +struct mptcp_tcp_sock *mptcp; +struct hlist_node *tmp; + +mptcp_for_each_sub_safe (mpcb, mptcp, tmp) { +struct sock *sk_it =mptcp_to_sock (mptcp); -mptcp_for_each_sk_safe (mpcb, sk_it, tmp) {if (sk_it != except)mptcp_send_reset (sk_it); }@@ -1150,12 +1151,14 @@ static inline int mptcp_sk_can_send_ack(const struct sock *sk)static inline bool mptcp_can_sg (const struct sock *meta_sk) { -struct sock *sk; +struct mptcp_tcp_sock *mptcp;if (tcp_sk (meta_sk)->mpcb->dss_csum)return false; -mptcp_for_each_sk (tcp_sk (meta_sk)->mpcb, sk) { +mptcp_for_each_sub (tcp_sk (meta_sk)->mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp); +if (!mptcp_sk_can_send (sk))continue ;if (!(sk->sk_route_caps &NETIF_F_SG ))@@ -1166,9 +1169,9 @@ static inline bool mptcp_can_sg(const struct sock *meta_sk)static inline void mptcp_set_rto (struct sock *sk) { -struct tcp_sock *tp =tcp_sk (sk); -struct sock *sk_it;struct inet_connection_sock *micsk =inet_csk (mptcp_meta_sk (sk)); +struct tcp_sock *tp =tcp_sk (sk); +struct mptcp_tcp_sock *mptcp;__u32 max_rto =0 ;/* We are in recovery-phase on the MPTCP-level. Do not update the @@ -1177,7 +1180,9 @@ static inline void mptcp_set_rto(struct sock *sk)if (micsk->icsk_retransmits)return ; -mptcp_for_each_sk (tp->mpcb, sk_it) { +mptcp_for_each_sub (tp->mpcb, mptcp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +if ((mptcp_sk_can_send (sk_it) || sk->sk_state ==TCP_SYN_RECV ) &&inet_csk (sk_it)->icsk_rto > max_rto) max_rto =inet_csk (sk_it)->icsk_rto;@@ -1266,10 +1271,10 @@ static inline bool mptcp_can_new_subflow(const struct sock *meta_sk)static inline int mptcp_subflow_count (const struct mptcp_cb *mpcb) { -struct sock *sk; +struct mptcp_tcp_sock *mptcp;int i =0 ; -mptcp_for_each_sk (mpcb, sk) +mptcp_for_each_sub (mpcb, mptcp) i++;return i;@@ -1287,12 +1292,8 @@ bool mptcp_prune_ofo_queue(struct sock *sk);do { \ } while (0) -/* Without MPTCP, we just do one iteration -* over the only socket available. This assumes that -* the sk/tp arg is the socket in that case. - */ -#define mptcp_for_each_sk (mpcb, sk) -#define mptcp_for_each_sk_safe (__mpcb, __sk, __temp) +#define mptcp_for_each_sub (__mpcb, __mptcp) \ + if (0)#define MPTCP_INC_STATS (net, field) \ do { \net/ipv4/af_inet.c +5
-4@@ -744,13 +744,14 @@ int inet_accept(struct socket *sock, struct socket *newsock, int flags,sock_rps_record_flow (sk2);if (sk2->sk_protocol ==IPPROTO_TCP &&mptcp (tcp_sk (sk2))) { -struct sock *sk_it = sk2; +struct mptcp_tcp_sock *mptcp; -mptcp_for_each_sk (tcp_sk (sk2)->mpcb, sk_it) -sock_rps_record_flow (sk_it); +mptcp_for_each_sub (tcp_sk (sk2)->mpcb, mptcp) { +sock_rps_record_flow (mptcp_to_sock (mptcp)); + }if (tcp_sk (sk2)->mpcb->master_sk) { - sk_it =tcp_sk (sk2)->mpcb->master_sk; +struct sock *sk_it =tcp_sk (sk2)->mpcb->master_sk;write_lock_bh (&sk_it->sk_callback_lock); sk_it->sk_wq = newsock->wq;net/ipv4/ip_sockglue.c +5
-2@@ -756,8 +756,11 @@ static int do_ip_setsockopt(struct sock *sk, int level,sk_dst_reset (sk);/* Update TOS on mptcp subflow */ if (is_meta_sk (sk)) { -struct sock *sk_it; -mptcp_for_each_sk (tcp_sk (sk)->mpcb, sk_it) { +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (tcp_sk (sk)->mpcb, mptcp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +if (inet_sk (sk_it)->tos !=inet_sk (sk)->tos) {inet_sk (sk_it)->tos =inet_sk (sk)->tos; sk_it->sk_priority = sk->sk_priority;net/ipv4/tcp.c +19
-12@@ -823,9 +823,11 @@ ssize_t tcp_splice_read(struct socket *sock, loff_t *ppos,#ifdef CONFIG_MPTCP if (mptcp (tcp_sk (sk))) { -struct sock *sk_it; -mptcp_for_each_sk (tcp_sk (sk)->mpcb, sk_it) -sock_rps_record_flow (sk_it); +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (tcp_sk (sk)->mpcb, mptcp) { +sock_rps_record_flow (mptcp_to_sock (mptcp)); + } }#endif @@ -993,7 +995,7 @@ ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset, }if (mptcp (tp)) { -struct sock *sk_it = sk; +struct mptcp_tcp_sock *mptcp;/* We must check this with socket-lock hold because we iterate * over the subflows. @@ -1008,8 +1010,9 @@ ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset,return ret; } -mptcp_for_each_sk (tp->mpcb, sk_it) -sock_rps_record_flow (sk_it); +mptcp_for_each_sub (tp->mpcb, mptcp) { +sock_rps_record_flow (mptcp_to_sock (mptcp)); + } }sk_clear_bit (SOCKWQ_ASYNC_NOSPACE , sk);@@ -1288,9 +1291,11 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size) }if (mptcp (tp)) { -struct sock *sk_it = sk; -mptcp_for_each_sk (tp->mpcb, sk_it) -sock_rps_record_flow (sk_it); +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (tp->mpcb, mptcp) { +sock_rps_record_flow (mptcp_to_sock (mptcp)); + } }if (unlikely (tp->repair)) {@@ -2006,9 +2011,11 @@ int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int nonblock,#ifdef CONFIG_MPTCP if (mptcp (tp)) { -struct sock *sk_it; -mptcp_for_each_sk (tp->mpcb, sk_it) -sock_rps_record_flow (sk_it); +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (tp->mpcb, mptcp) { +sock_rps_record_flow (mptcp_to_sock (mptcp)); + } }#endif net/mptcp/mptcp_ctrl.c +55
-34@@ -561,10 +561,12 @@ void mptcp_hash_remove_bh(struct tcp_sock *meta_tp)struct sock *mptcp_select_ack_sock (const struct sock *meta_sk) {const struct tcp_sock *meta_tp =tcp_sk (meta_sk); -struct sock *sk, *rttsk =NULL , *lastsk =NULL ; +struct sock *rttsk =NULL , *lastsk =NULL ;u32 min_time =0 , last_active =0 ; +struct mptcp_tcp_sock *mptcp; -mptcp_for_each_sk (meta_tp->mpcb, sk) { +mptcp_for_each_sub (meta_tp->mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);struct tcp_sock *tp =tcp_sk (sk);u32 elapsed;@@ -697,7 +699,8 @@ void mptcp_sock_destruct(struct sock *sk)void mptcp_destroy_sock (struct sock *sk) {if (is_meta_sk (sk)) { -struct sock *sk_it, *tmpsk; +struct mptcp_tcp_sock *mptcp; +struct hlist_node *tmp;__skb_queue_purge (&tcp_sk (sk)->mpcb->reinject_queue);@@ -707,7 +710,9 @@ void mptcp_destroy_sock(struct sock *sk)* not have been closed properly (as we are waiting for the * DATA_ACK of the DATA_FIN). */ -mptcp_for_each_sk_safe (tcp_sk (sk)->mpcb, sk_it, tmpsk) { +mptcp_for_each_sub_safe (tcp_sk (sk)->mpcb, mptcp, tmp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +/* Already did call tcp_close - waiting for graceful * closure, or if we are retransmitting fast-close on * the subflow. The reset (or timeout) will kill the @@ -1303,6 +1308,7 @@ static int mptcp_alloc_mpcb(struct sock *meta_sk, __u64 remote_key,INIT_LIST_HEAD (&mpcb->tw_list);INIT_HLIST_HEAD (&mpcb->callback_list); +INIT_HLIST_HEAD (&mpcb->conn_list);spin_lock_init (&mpcb->mpcb_list_lock);mptcp_mpcb_inherit_sockopts (meta_sk, master_sk);@@ -1392,8 +1398,12 @@ int mptcp_add_sock(struct sock *meta_sk, struct sock *sk, u8 loc_id, u8 rem_id,sock_hold (meta_sk);refcount_inc (&mpcb->mpcb_refcnt); - tp->mptcp->next = mpcb->connection_list; - mpcb->connection_list = tp; +local_bh_disable (); +spin_lock (&mpcb->mpcb_list_lock); +hlist_add_head_rcu (&tp->mptcp->node, &mpcb->conn_list); +spin_unlock (&mpcb->mpcb_list_lock); +local_bh_enable (); + tp->mptcp->attached =1 ; atomic_add(atomic_read(&((struct sock *)tp)->sk_rmem_alloc),@@ -1437,14 +1447,13 @@ int mptcp_add_sock(struct sock *meta_sk, struct sock *sk, u8 loc_id, u8 rem_id,void mptcp_del_sock (struct sock *sk) { -struct tcp_sock *tp =tcp_sk (sk), *tp_prev; +struct tcp_sock *tp =tcp_sk (sk);struct mptcp_cb *mpcb;if (!tp->mptcp || !tp->mptcp->attached)return ; mpcb = tp->mpcb; - tp_prev = mpcb->connection_list;if (mpcb->pm_ops->delete_subflow) mpcb->pm_ops->delete_subflow (sk);@@ -1453,17 +1462,10 @@ void mptcp_del_sock(struct sock *sk) __func__, mpcb->mptcp_loc_token, tp->mptcp->path_index, sk->sk_state, is_meta_sk(sk)); -if (tp_prev == tp) { - mpcb->connection_list = tp->mptcp->next; - }else { -for (; tp_prev && tp_prev->mptcp->next; tp_prev = tp_prev->mptcp->next) { -if (tp_prev->mptcp->next == tp) { - tp_prev->mptcp->next = tp->mptcp->next; -break ; - } - } - } - tp->mptcp->next =NULL ; +spin_lock (&mpcb->mpcb_list_lock); +hlist_del_init_rcu (&tp->mptcp->node); +spin_unlock (&mpcb->mpcb_list_lock); + tp->mptcp->attached =0 ; mpcb->path_index_bits &= ~(1 << tp->mptcp->path_index);@@ -1510,8 +1512,8 @@ void mptcp_update_metasocket(const struct sock *meta_sk)void mptcp_cleanup_rbuf (struct sock *meta_sk,int copied) {struct tcp_sock *meta_tp =tcp_sk (meta_sk); -struct sock *sk;bool recheck_rcv_window = false; +struct mptcp_tcp_sock *mptcp;__u32 rcv_window_now =0 ;if (copied >0 && !(meta_sk->sk_shutdown &RCV_SHUTDOWN )) {@@ -1522,7 +1524,8 @@ void mptcp_cleanup_rbuf(struct sock *meta_sk, int copied) recheck_rcv_window = true; } -mptcp_for_each_sk (meta_tp->mpcb, sk) { +mptcp_for_each_sub (meta_tp->mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);struct tcp_sock *tp =tcp_sk (sk);const struct inet_connection_sock *icsk =inet_csk (sk);@@ -1709,10 +1712,13 @@ EXPORT_SYMBOL(mptcp_sub_force_close);*/ void mptcp_update_sndbuf (const struct tcp_sock *tp) { -struct sock *meta_sk = tp->meta_sk, *sk; +struct sock *meta_sk = tp->meta_sk;int new_sndbuf =0 , old_sndbuf = meta_sk->sk_sndbuf; +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (tp->mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp); -mptcp_for_each_sk (tp->mpcb, sk) {if (!mptcp_sk_can_send (sk))continue ;@@ -1741,8 +1747,8 @@ void mptcp_update_sndbuf(const struct tcp_sock *tp)void mptcp_close (struct sock *meta_sk,long timeout) {struct tcp_sock *meta_tp =tcp_sk (meta_sk); -struct sock *sk_it, *tmpsk;struct mptcp_cb *mpcb = meta_tp->mpcb; +struct mptcp_tcp_sock *mptcp;struct sk_buff *skb;int data_was_unread =0 ;int state;@@ -1775,7 +1781,12 @@ void mptcp_close(struct sock *meta_sk, long timeout)/* If socket has been already reset (e.g. in tcp_reset()) - kill it. */ if (meta_sk->sk_state ==TCP_CLOSE ) { -mptcp_for_each_sk_safe (mpcb, sk_it, tmpsk) { +struct mptcp_tcp_sock *mptcp; +struct hlist_node *tmp; + +mptcp_for_each_sub_safe (mpcb, mptcp, tmp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +if (tcp_sk (sk_it)->send_mp_fclose)continue ;mptcp_sub_close (sk_it,0 );@@ -1796,10 +1807,14 @@ void mptcp_close(struct sock *meta_sk, long timeout) }else if (tcp_close_state (meta_sk)) {mptcp_send_fin (meta_sk); }else if (meta_tp->snd_una == meta_tp->write_seq) { +struct mptcp_tcp_sock *mptcp; +struct hlist_node *tmp; +/* The DATA_FIN has been sent and acknowledged * (e.g., by sk_shutdown). Close all the other subflows */ -mptcp_for_each_sk_safe (mpcb, sk_it, tmpsk) { +mptcp_for_each_sub_safe (mpcb, mptcp, tmp) { +struct sock *sk_it =mptcp_to_sock (mptcp);unsigned long delay =0 ;/* If we are the passive closer, don't trigger * subflow-fin until the subflow has been finned @@ -1823,7 +1838,9 @@ void mptcp_close(struct sock *meta_sk, long timeout)/* socket will be freed after mptcp_close - we have to prevent * access from the subflows. */ -mptcp_for_each_sk (mpcb, sk_it) { +mptcp_for_each_sub (mpcb, mptcp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +/* Similar to sock_orphan, but we don't set it DEAD, because * the callbacks are still set and must be called. */ @@ -1908,8 +1925,9 @@ void mptcp_close(struct sock *meta_sk, long timeout)void mptcp_disconnect (struct sock *sk) { -struct sock *subsk, *tmpsk; +struct mptcp_tcp_sock *mptcp;struct tcp_sock *tp =tcp_sk (sk); +struct hlist_node *tmp;__skb_queue_purge (&tp->mpcb->reinject_queue);@@ -1917,7 +1935,9 @@ void mptcp_disconnect(struct sock *sk)mptcp_hash_remove_bh (tp);local_bh_disable (); -mptcp_for_each_sk_safe (tp->mpcb, subsk, tmpsk) { +mptcp_for_each_sub_safe (tp->mpcb, mptcp, tmp) { +struct sock *subsk =mptcp_to_sock (mptcp); +/* The socket will get removed from the subsocket-list * and made non-mptcp by setting mpc to 0. * @@ -2606,7 +2626,6 @@ static void mptcp_get_sub_info(struct sock *sk, struct mptcp_sub_info *info)int mptcp_get_info (const struct sock *meta_sk,char __user *optval,int optlen) {const struct tcp_sock *meta_tp =tcp_sk (meta_sk); -struct sock *sk;struct mptcp_meta_info meta_info;struct mptcp_info m_info;@@ -2652,16 +2671,17 @@ int mptcp_get_info(const struct sock *meta_sk, char __user *optval, int optlen)if (m_info.subflows) {unsigned int len, sub_len =0 ; +struct mptcp_tcp_sock *mptcp;char __user *ptr; ptr = (char __user *)m_info.subflows; len = m_info.sub_len; -mptcp_for_each_sk (meta_tp->mpcb, sk) { +mptcp_for_each_sub (meta_tp->mpcb, mptcp) {struct tcp_info t_info;unsigned int tmp_len; -tcp_get_info (sk, &t_info); +tcp_get_info (mptcp_to_sock (mptcp), &t_info); tmp_len = min_t(unsigned int, len, info_len); len -= tmp_len;@@ -2681,6 +2701,7 @@ int mptcp_get_info(const struct sock *meta_sk, char __user *optval, int optlen)if (m_info.subflow_info) {unsigned int len, sub_info_len, total_sub_info_len =0 ; +struct mptcp_tcp_sock *mptcp;char __user *ptr; ptr = (char __user *)m_info.subflow_info;@@ -2690,11 +2711,11 @@ int mptcp_get_info(const struct sock *meta_sk, char __user *optval, int optlen) sizeof(struct mptcp_sub_info)); m_info.sub_info_len = sub_info_len; -mptcp_for_each_sk (meta_tp->mpcb, sk) { +mptcp_for_each_sub (meta_tp->mpcb, mptcp) {struct mptcp_sub_info m_sub_info;unsigned int tmp_len; -mptcp_get_sub_info (sk, &m_sub_info); +mptcp_get_sub_info (mptcp_to_sock (mptcp), &m_sub_info); tmp_len = min_t(unsigned int, len, sub_info_len); len -= tmp_len;net/mptcp/mptcp_fullmesh.c +20
-8@@ -903,8 +903,9 @@ static void mptcp_address_worker(struct work_struct *work) }if (event->code ==MPTCP_EVENT_DEL ) { -struct sock *sk, *tmpsk; +struct mptcp_tcp_sock *mptcp;struct mptcp_loc_addr *mptcp_local; +struct hlist_node *tmp;bool found = false; mptcp_local =rcu_dereference_bh (fm_ns->local);@@ -914,7 +915,9 @@ static void mptcp_address_worker(struct work_struct *work)update_addr_bitfields (meta_sk, mptcp_local);/* Look for the socket and remove him */ -mptcp_for_each_sk_safe (mpcb, sk, tmpsk) { +mptcp_for_each_sub_safe (mpcb, mptcp, tmp) { +struct sock *sk =mptcp_to_sock (mptcp); + if ((event->family == AF_INET6 && (sk->sk_family == AF_INET || mptcp_v6_is_v4_mapped(sk))) ||@@ -964,9 +967,10 @@ static void mptcp_address_worker(struct work_struct *work) }if (event->code ==MPTCP_EVENT_MOD ) { -struct sock *sk; +struct mptcp_tcp_sock *mptcp; -mptcp_for_each_sk (mpcb, sk) { +mptcp_for_each_sub (mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);struct tcp_sock *tp =tcp_sk (sk); if (event->family == AF_INET && (sk->sk_family == AF_INET ||@@ -1455,8 +1459,9 @@ static void full_mesh_release_sock(struct sock *meta_sk)struct mptcp_cb *mpcb =tcp_sk (meta_sk)->mpcb;struct fullmesh_priv *fmp =fullmesh_get_priv (mpcb);const struct mptcp_fm_ns *fm_ns =fm_get_ns (sock_net (meta_sk)); -struct sock *sk, *tmpsk;bool meta_v4 = meta_sk->sk_family ==AF_INET ; +struct mptcp_tcp_sock *mptcp; +struct hlist_node *tmp;int i;rcu_read_lock_bh ();@@ -1470,7 +1475,8 @@ static void full_mesh_release_sock(struct sock *meta_sk)struct in_addr ifa = mptcp_local->locaddr4[i].addr;bool found = false; -mptcp_for_each_sk (mpcb, sk) { +mptcp_for_each_sub (mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);struct tcp_sock *tp =tcp_sk (sk); if (sk->sk_family == AF_INET6 &&@@ -1491,6 +1497,8 @@ static void full_mesh_release_sock(struct sock *meta_sk) }if (!found) { +struct sock *sk; + fmp->add_addr++; mpcb->addr_signal =1 ;@@ -1511,7 +1519,8 @@ static void full_mesh_release_sock(struct sock *meta_sk)struct in6_addr ifa = mptcp_local->locaddr6[i].addr;bool found = false; -mptcp_for_each_sk (mpcb, sk) { +mptcp_for_each_sub (mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);struct tcp_sock *tp =tcp_sk (sk); if (sk->sk_family == AF_INET ||@@ -1532,6 +1541,8 @@ static void full_mesh_release_sock(struct sock *meta_sk) }if (!found) { +struct sock *sk; + fmp->add_addr++; mpcb->addr_signal =1 ;@@ -1546,7 +1557,8 @@ static void full_mesh_release_sock(struct sock *meta_sk)#endif /* Now, detect address-removals */ -mptcp_for_each_sk_safe (mpcb, sk, tmpsk) { +mptcp_for_each_sub_safe (mpcb, mptcp, tmp) { +struct sock *sk =mptcp_to_sock (mptcp);bool shall_remove = true;if (sk->sk_family ==AF_INET ||mptcp_v6_is_v4_mapped (sk)) {net/mptcp/mptcp_input.c +29
-15@@ -126,12 +126,14 @@ static void mptcp_clean_rtx_queue(struct sock *meta_sk, u32 prior_snd_una)tcp_rtx_queue_unlink (skb, meta_sk);if (mptcp_is_data_fin (skb)) { -struct sock *sk_it, *sk_tmp; +struct mptcp_tcp_sock *mptcp; +struct hlist_node *tmp;/* DATA_FIN has been acknowledged - now we can close * the subflows */ -mptcp_for_each_sk_safe (mpcb, sk_it, sk_tmp) { +mptcp_for_each_sub_safe (mpcb, mptcp, tmp) { +struct sock *sk_it =mptcp_to_sock (mptcp);unsigned long delay =0 ;/* If we are the passive closer, don't trigger @@ -347,6 +349,7 @@ static int mptcp_verif_dss_csum(struct sock *sk)/* Now, checksum must be 0 */ if (unlikely (csum_fold (csum_tcp))) { +struct mptcp_tcp_sock *mptcp;struct sock *sk_it =NULL ; pr_err("%s csum is wrong: %#x tcp-seq %u dss_csum_added %d overflowed %d iterations %d\n",@@ -362,7 +365,9 @@ static int mptcp_verif_dss_csum(struct sock *sk) tp->mpcb->csum_cutoff_seq = tp->mptcp->map_data_seq;/* Search for another subflow that is fully established */ -mptcp_for_each_sk (tp->mpcb, sk_it) { +mptcp_for_each_sub (tp->mpcb, mptcp) { + sk_it =mptcp_to_sock (mptcp); +if (sk_it != sk &&tcp_sk (sk_it)->mptcp->fully_established)break ;@@ -1308,12 +1313,15 @@ int mptcp_do_join_short(struct sk_buff *skb,*/ void mptcp_fin (struct sock *meta_sk) { -struct sock *sk =NULL , *sk_it; +struct sock *sk =NULL ;struct tcp_sock *meta_tp =tcp_sk (meta_sk);struct mptcp_cb *mpcb = meta_tp->mpcb; +struct mptcp_tcp_sock *mptcp;unsigned char state; -mptcp_for_each_sk (mpcb, sk_it) { +mptcp_for_each_sub (mpcb, mptcp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +if (tcp_sk (sk_it)->mptcp->path_index == mpcb->dfin_path_index) { sk = sk_it;break ;@@ -1585,9 +1593,12 @@ void mptcp_clean_rtx_infinite(const struct sk_buff *skb, struct sock *sk)static void mptcp_send_reset_rem_id (const struct mptcp_cb *mpcb,u8 rem_id) { -struct sock *sk_it, *tmpsk; +struct mptcp_tcp_sock *mptcp; +struct hlist_node *tmp; + +mptcp_for_each_sub_safe (mpcb, mptcp, tmp) { +struct sock *sk_it =mptcp_to_sock (mptcp); -mptcp_for_each_sk_safe (mpcb, sk_it, tmpsk) {if (tcp_sk (sk_it)->mptcp->rem_id == rem_id) {mptcp_reinject_data (sk_it,0 );mptcp_send_reset (sk_it);@@ -1892,13 +1903,15 @@ void tcp_parse_mptcp_options(const struct sk_buff *skb,bool mptcp_check_rtt (const struct tcp_sock *tp,int time) {struct mptcp_cb *mpcb = tp->mpcb; -struct sock *sk; +struct mptcp_tcp_sock *mptcp;u32 rtt_max =0 ;/* In MPTCP, we take the max delay across all flows, * in order to take into account meta-reordering buffers. */ -mptcp_for_each_sk (mpcb, sk) { +mptcp_for_each_sub (mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp); +if (!mptcp_sk_can_recv (sk))continue ;@@ -2173,9 +2186,9 @@ bool mptcp_handle_options(struct sock *sk, const struct tcphdr *th,if (mopt->saw_low_prio ==1 ) { tp->mptcp->rcv_low_prio = mopt->low_prio; }else { -struct sock *sk_it; -mptcp_for_each_sk (tp->mpcb, sk_it) { -struct mptcp_tcp_sock *mptcp =tcp_sk (sk_it)->mptcp; +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (tp->mpcb, mptcp) {if (mptcp->rem_id == mopt->prio_addr_id) mptcp->rcv_low_prio = mopt->low_prio; }@@ -2359,7 +2372,7 @@ bool mptcp_should_expand_sndbuf(const struct sock *sk) {const struct sock *meta_sk =mptcp_meta_sk (sk);const struct tcp_sock *meta_tp =tcp_sk (meta_sk); -const struct sock *sk_it; +const struct mptcp_tcp_sock *mptcp;/* We circumvent this check in tcp_check_space, because we want to * always call sk_write_space. So, we reproduce the check here. @@ -2385,8 +2398,9 @@ bool mptcp_should_expand_sndbuf(const struct sock *sk)/* For MPTCP we look for a subsocket that could send data. * If we found one, then we update the send-buffer. */ -mptcp_for_each_sk (meta_tp->mpcb, sk_it) { -struct tcp_sock *tp_it =tcp_sk (sk_it); +mptcp_for_each_sub (meta_tp->mpcb, mptcp) { +const struct sock *sk_it =mptcp_to_sock (mptcp); +const struct tcp_sock *tp_it =tcp_sk (sk_it);if (!mptcp_sk_can_send (sk_it))continue ;net/mptcp/mptcp_output.c +2
-1@@ -647,7 +647,6 @@ int mptcp_write_wakeup(struct sock *meta_sk, int mib) {struct tcp_sock *meta_tp =tcp_sk (meta_sk);struct sk_buff *skb; -struct sock *sk_it;int ans =0 ;if (meta_sk->sk_state ==TCP_CLOSE )@@ -704,17 +703,22 @@ int mptcp_write_wakeup(struct sock *meta_sk, int mib)return 0 ; }else { +struct mptcp_tcp_sock *mptcp; +
window_probe:if (between(meta_tp->snd_up, meta_tp->snd_una + 1, meta_tp->snd_una + 0xFFFF)) {net/mptcp/mptcp_output.c +3
-1-mptcp_for_each_sk (meta_tp->mpcb, sk_it) { +mptcp_for_each_sub (meta_tp->mpcb, mptcp) { +struct sock *sk_it =mptcp_to_sock (mptcp); +if (mptcp_sk_can_send_ack(sk_it)) tcp_xmit_probe_skb(sk_it, 1, mib); } } /* At least one of the tcp_xmit_probe_skb's has to succeed */net/mptcp/mptcp_output.c +2
-1-mptcp_for_each_sk (meta_tp->mpcb, sk_it) { +mptcp_for_each_sub (meta_tp->mpcb, mptcp) { +struct sock *sk_it =mptcp_to_sock (mptcp);int ret; if (!mptcp_sk_can_send_ack(sk_it))net/mptcp/mptcp_output.c +18
-10@@ -732,6 +736,7 @@ bool mptcp_write_xmit(struct sock *meta_sk, unsigned int mss_now, int nonagle, int push_one, gfp_t gfp) {struct tcp_sock *meta_tp =tcp_sk (meta_sk), *subtp; +struct mptcp_tcp_sock *mptcp;struct sock *subsk =NULL ;struct mptcp_cb *mpcb = meta_tp->mpcb;struct sk_buff *skb;@@ -856,7 +861,8 @@ bool mptcp_write_xmit(struct sock *meta_sk, unsigned int mss_now, int nonagle,break ; } -mptcp_for_each_sk (mpcb, subsk) { +mptcp_for_each_sub (mpcb, mptcp) { + subsk =mptcp_to_sock (mptcp); subtp =tcp_sk (subsk);if (!(path_mask &mptcp_pi_to_flag (subtp->mptcp->path_index)))@@ -1353,7 +1359,7 @@ void mptcp_send_active_reset(struct sock *meta_sk, gfp_t priority)struct mptcp_cb *mpcb = meta_tp->mpcb;struct sock *sk; -if (!mpcb->connection_list) +if (hlist_empty (&mpcb->conn_list))return ;WARN_ON (meta_tp->send_mp_fclose);@@ -1728,10 +1734,11 @@ void mptcp_select_initial_window(const struct sock *sk, int __space, __u32 mss,static inline u64 mptcp_calc_rate (const struct sock *meta_sk,unsigned int mss,unsigned int (*mss_cb)(struct sock *sk)) { -struct sock *sk; +struct mptcp_tcp_sock *mptcp;u64 rate =0 ; -mptcp_for_each_sk (tcp_sk (meta_sk)->mpcb, sk) { +mptcp_for_each_sub (tcp_sk (meta_sk)->mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);struct tcp_sock *tp =tcp_sk (sk);int this_mss;u64 this_rate;@@ -1783,11 +1790,12 @@ static inline u64 mptcp_calc_rate(const struct sock *meta_sk, unsigned int mss,static unsigned int __mptcp_current_mss (const struct sock *meta_sk,unsigned int (*mss_cb)(struct sock *sk)) { +struct mptcp_tcp_sock *mptcp;unsigned int mss =0 ;u64 rate =0 ; -struct sock *sk; -mptcp_for_each_sk (tcp_sk (meta_sk)->mpcb, sk) { +mptcp_for_each_sub (tcp_sk (meta_sk)->mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);int this_mss;u64 this_rate;@@ -1852,14 +1860,16 @@ int mptcp_select_size(const struct sock *meta_sk, bool first_skb, bool zc)int mptcp_check_snd_buf (const struct tcp_sock *tp) { -const struct sock *sk; +const struct mptcp_tcp_sock *mptcp;u32 rtt_max = tp->srtt_us;u64 bw_est;if (!tp->srtt_us)return tp->reordering +1 ; -mptcp_for_each_sk (tp->mpcb, sk) { +mptcp_for_each_sub (tp->mpcb, mptcp) { +const struct sock *sk =mptcp_to_sock (mptcp); +if (!mptcp_sk_can_send (sk))continue ;@@ -1877,11 +1887,13 @@ int mptcp_check_snd_buf(const struct tcp_sock *tp)unsigned int mptcp_xmit_size_goal (const struct sock *meta_sk,u32 mss_now,int large_allowed) { -struct sock *sk;u32 xmit_size_goal =0 ;if (large_allowed && !tcp_sk (meta_sk)->mpcb->dss_csum) { -mptcp_for_each_sk (tcp_sk (meta_sk)->mpcb, sk) { +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (tcp_sk (meta_sk)->mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);int this_size_goal;if (!mptcp_sk_can_send (sk))net/mptcp/mptcp_sched.c +15
-7@@ -135,9 +135,10 @@ static struct socku32 min_srtt =0xffffffff ;bool found_unused = false;bool found_unused_una = false; -struct sock *sk; +struct mptcp_tcp_sock *mptcp; -mptcp_for_each_sk (mpcb, sk) { +mptcp_for_each_sub (mpcb, mptcp) { +struct sock *sk =mptcp_to_sock (mptcp);struct tcp_sock *tp =tcp_sk (sk);bool unused = false;@@ -219,7 +220,11 @@ static struct sock *get_available_subflow(struct sock *meta_sk,/* Answer data_fin on same subflow!!! */ if (meta_sk->sk_shutdown &RCV_SHUTDOWN && skb &&mptcp_is_data_fin (skb)) { -mptcp_for_each_sk (mpcb, sk) { +struct mptcp_tcp_sock *mptcp; + +mptcp_for_each_sub (mpcb, mptcp) { + sk =mptcp_to_sock (mptcp); +if (tcp_sk (sk)->mptcp->path_index == mpcb->dfin_path_index &&mptcp_is_available (sk, skb, zero_wnd_test))return sk;@@ -252,7 +257,7 @@ static struct sk_buff *mptcp_rcv_buf_optimization(struct sock *sk, int penal) {struct sock *meta_sk;const struct tcp_sock *tp =tcp_sk (sk); -struct tcp_sock *tp_it; +struct mptcp_tcp_sock *mptcp;struct sk_buff *skb_head;struct defsched_priv *dsp =defsched_get_priv (tp);@@ -275,7 +280,9 @@ static struct sk_buff *mptcp_rcv_buf_optimization(struct sock *sk, int penal) goto retrans;/* Half the cwnd of the slow flow */ -mptcp_for_each_tp (tp->mpcb, tp_it) { +mptcp_for_each_sub (tp->mpcb, mptcp) { +struct tcp_sock *tp_it = mptcp->tp; +if (tp_it != tp &&TCP_SKB_CB (skb_head)->path_mask &mptcp_pi_to_flag (tp_it->mptcp->path_index)) {if (tp->srtt_us < tp_it->srtt_us &&inet_csk ((struct sock *)tp_it)->icsk_ca_state == TCP_CA_Open) {@@ -298,7 +305,9 @@ static struct sk_buff *mptcp_rcv_buf_optimization(struct sock *sk, int penal)/* Segment not yet injected into this path? Take it!!! */ if (!(TCP_SKB_CB (skb_head)->path_mask &mptcp_pi_to_flag (tp->mptcp->path_index))) {bool do_retrans = false; -mptcp_for_each_tp (tp->mpcb, tp_it) { +mptcp_for_each_sub (tp->mpcb, mptcp) { +struct tcp_sock *tp_it = mptcp->tp; +if (tp_it != tp &&TCP_SKB_CB (skb_head)->path_mask &mptcp_pi_to_flag (tp_it->mptcp->path_index)) {if (tp_it->snd_cwnd <=4 ) { --Quoted 3 lines
2.16.2
--
Mat Martineau
Intel OTC--
Mat Martineau
Intel OTC
--
Mat Martineau
Intel OTC
Christoph Paasch
PATCH 11/20 mptcp: Don't take meta-lock when receiving third ACK
In reply to Christoph PaaschNow everything is ready to do this in a lockless way.
Simply, in tcp_v4(6)_rcv, stop taking the meta-level lock. When creating the initial subflow, we currently are also holding the meta-level lock. We need to drop this one as soon as we handover control to the subflow (aka., when exiting mptcp_check_req_master()).
Data-reception however still requires the meta-level lock. Thus, we need to take it again in tcp_child_process().
--- net/ipv4/tcp_ipv4.c | 41 ++++------------------------------------- net/ipv4/tcp_minisocks.c | 9 +++++++-- net/ipv6/tcp_ipv6.c | 41 ++++------------------------------------- net/mptcp/mptcp_ctrl.c | 13 +++++++++++++ net/mptcp/mptcp_ipv4.c | 5 ++++- net/mptcp/mptcp_ipv6.c | 4 ++++ 6 files changed, 36 insertions(+), 77 deletions(-)
net/ipv4/tcp_ipv4.c +4 -37
@@ -1798,45 +1798,16 @@ int tcp_v4_rcv(struct sk_buff *skb)inet_csk_reqsk_queue_drop_and_put (sk, req); goto lookup; } +if (unlikely (is_meta_sk (sk) && !mptcp_can_new_subflow (sk))) { +inet_csk_reqsk_queue_drop_and_put (sk, req); + goto lookup; + }/* We own a reference on the listener, increase it again * as we might lose it too soon. */ sock_hold (sk); refcounted = true; -if (is_meta_sk (sk)) { -bh_lock_sock (sk); - -if (!mptcp_can_new_subflow (sk)) { -inet_csk_reqsk_queue_drop_and_put (sk, req); -bh_unlock_sock (sk); - - goto discard_and_relse; - } - -if (sock_owned_by_user (sk)) { - th = (const struct tcphdr *)skb->data; - iph =ip_hdr (skb); -tcp_v4_fill_cb (skb, iph, th); - - skb->sk = sk; -if (unlikely (sk_add_backlog (sk, skb, - sk->sk_rcvbuf + sk->sk_sndbuf))) { -reqsk_put (req); - -bh_unlock_sock (sk); -__NET_INC_STATS (net,LINUX_MIB_TCPBACKLOGDROP ); - goto discard_and_relse; - } - -reqsk_put (req); -bh_unlock_sock (sk); -sock_put (sk); - -return 0 ; - } - } - nsk =NULL ;if (!tcp_filter (sk, skb)) { th = (const struct tcphdr *)skb->data;
@@ -1846,8 +1817,6 @@ int tcp_v4_rcv(struct sk_buff *skb) }if (!nsk) {reqsk_put (req); -if (is_meta_sk (sk)) -bh_unlock_sock (sk);if (req_stolen) {/* Another cpu got exclusive access to req * and created a full blown socket.
@@ -1863,8 +1832,6 @@ int tcp_v4_rcv(struct sk_buff *skb)if (nsk == sk) {reqsk_put (req);tcp_v4_restore_cb (skb); -if (is_meta_sk (sk)) -bh_unlock_sock (sk); }else if (tcp_child_process (sk, nsk, skb)) {tcp_v4_send_reset (nsk, skb); goto discard_and_relse;
net/ipv4/tcp_minisocks.c +7 -2
@@ -902,6 +902,11 @@ int tcp_child_process(struct sock *parent, struct sock *child,sk_mark_napi_id (child, skb);tcp_segs_in (tcp_sk (child), skb); +/* The following will be removed when we allow lockless data-reception +* on the subflows. +*/ +if (mptcp (tcp_sk (child))) +bh_lock_sock (meta_sk);if (!sock_owned_by_user (meta_sk)) { ret =tcp_rcv_state_process (child, skb);/* Wakeup parent, send SIGIO */
@@ -917,9 +922,9 @@ int tcp_child_process(struct sock *parent, struct sock *child,__sk_add_backlog (meta_sk, skb); } +bh_unlock_sock (child);if (mptcp (tcp_sk (child))) -bh_unlock_sock (child); -bh_unlock_sock (meta_sk); +bh_unlock_sock (meta_sk);sock_put (child);return ret; }
net/ipv6/tcp_ipv6.c +4 -37
62 lines not on this page · raw message
net/mptcp/mptcp_ctrl.c +13 -0
@@ -1355,6 +1355,7 @@ static u8 mptcp_set_new_pathindex(struct mptcp_cb *mpcb)return i; } +/* May be called without holding the meta-level lock */ int mptcp_add_sock (struct sock *meta_sk,struct sock *sk,u8 loc_id,u8 rem_id,gfp_t flags) {
@@ -2136,6 +2137,11 @@ int mptcp_check_req_fastopen(struct sock *child, struct request_sock *req)/* Handled by the master_sk */ tcp_sk (meta_sk)->fastopen_rsk =NULL ; +/* Subflow establishment is now lockless, drop the lock here it will +* be taken again in tcp_child_process(). +*/ +bh_unlock_sock (meta_sk); +return 0 ; }
@@ -2165,9 +2171,15 @@ int mptcp_check_req_master(struct sock *sk, struct sock *child,inet_csk_reqsk_queue_add (sk, req, meta_sk); } +/* Subflow establishment is now lockless, drop the lock here it will +* be taken again in tcp_child_process(). +*/ +bh_unlock_sock (meta_sk); +return 0 ; } +/* May be called without holding the meta-level lock */ struct sock *mptcp_check_req_child(struct sock *meta_sk, struct sock *child, struct request_sock *req,
@@ -2410,6 +2422,7 @@ void mptcp_tsq_sub_deferred(struct sock *meta_sk) } } +/* May be called without holding the meta-level lock */ void mptcp_join_reqsk_init (const struct mptcp_cb *mpcb,const struct request_sock *req,struct sk_buff *skb)
net/mptcp/mptcp_ipv4.c +4 -1
@@ -96,6 +96,7 @@ static u32 mptcp_v4_cookie_init_seq(struct request_sock *req, const struct sock }#endif +/* May be called without holding the meta-level lock */ static int mptcp_v4_join_init_req (struct request_sock *req,const struct sock *meta_sk,struct sk_buff *skb,bool want_cookie) {
@@ -141,7 +142,9 @@ struct request_sock_ops mptcp_request_sock_ops __read_mostly = { .syn_ack_timeout = tcp_syn_ack_timeout, }; -/* Similar to: tcp_v4_conn_request */ +/* Similar to: tcp_v4_conn_request +* May be called without holding the meta-level lock +*/ static int mptcp_v4_join_request (struct sock *meta_sk,struct sk_buff *skb) { return tcp_conn_request(&mptcp_request_sock_ops,
net/mptcp/mptcp_ipv6.c +4 -0
@@ -125,6 +125,7 @@ static u32 mptcp_v6_cookie_init_seq(struct request_sock *req, const struct sock }#endif +/* May be called without holding the meta-level lock */ static int mptcp_v6_join_init_req (struct request_sock *req,const struct sock *meta_sk,struct sk_buff *skb,bool want_cookie) {
@@ -170,6 +171,9 @@ struct request_sock_ops mptcp6_request_sock_ops __read_mostly = { .syn_ack_timeout = tcp_syn_ack_timeout, }; +/* Similar to: tcp_v6_conn_request +* May be called without holding the meta-level lock +*/ static int mptcp_v6_join_request (struct sock *meta_sk,struct sk_buff *skb) { return tcp_conn_request(&mptcp6_request_sock_ops,
Signature · 1 line
2.16.2
Christoph Paasch
PATCH 12/20 mptcp: Make mptcp_do_join_short and mptcp_lookup_join lockless
In reply to Christoph PaaschNow we can also make the subflow-SYN+MP_JOIN reception lockless.
In mptcp_lookup_join and mptcp_do_join_short, we were checking for mptcp_can_new_subflow(), because it is not possible to call tcp_v4_send_reset() while holding two locks (the listener lock and the meta-lock when coming from mptcp_do_join_short) when in mptcp_v4_do_rcv. Now that we remove the locking, we can remove the check for mptcp_can_new_subflow().
The remaining changes simply skip taking the meta-level lock which also implies that we don't have to check for sock_owned_by_user().
--- net/mptcp/mptcp_input.c | 70 +++++++------------------------------------------ 1 file changed, 9 insertions(+), 61 deletions(-)
net/mptcp/mptcp_input.c +9 -61
@@ -1151,7 +1151,6 @@ struct mp_join *mptcp_find_join(const struct sk_buff *skb)int mptcp_lookup_join (struct sk_buff *skb,struct inet_timewait_sock *tw) { -const struct mptcp_cb *mpcb;struct sock *meta_sk;u32 token;bool meta_v4;
@@ -1184,16 +1183,6 @@ int mptcp_lookup_join(struct sk_buff *skb, struct inet_timewait_sock *tw)return -1 ; } - mpcb =tcp_sk (meta_sk)->mpcb; -if (!mptcp_can_new_subflow (meta_sk)) { -/* We are in fallback-mode on the reception-side - -* no new subflows! -*/ -sock_put (meta_sk);/* Taken by mptcp_hash_find */ -MPTCP_INC_STATS (sock_net (meta_sk),MPTCP_MIB_JOINFALLBACK ); -return -1 ; - } -/* Coming from time-wait-sock processing in tcp_v4_rcv. * We have to deschedule it before continuing, because otherwise * mptcp_v4_do_rcv will hit again on it inside tcp_v4_hnd_req.
@@ -1204,26 +1193,13 @@ int mptcp_lookup_join(struct sk_buff *skb, struct inet_timewait_sock *tw)/* OK, this is a new syn/join, let's create a new open request and * send syn+ack */ -bh_lock_sock_nested (meta_sk); -if (sock_owned_by_user (meta_sk)) { - skb->sk = meta_sk; -if (unlikely (sk_add_backlog (meta_sk, skb, - meta_sk->sk_rcvbuf + meta_sk->sk_sndbuf))) { -bh_unlock_sock (meta_sk); -__NET_INC_STATS (sock_net (meta_sk), -LINUX_MIB_TCPBACKLOGDROP ); -sock_put (meta_sk);/* Taken by mptcp_hash_find */ -kfree_skb (skb); -return 1 ; - } - }else if (skb->protocol ==htons (ETH_P_IP )) { +if (skb->protocol ==htons (ETH_P_IP )) {tcp_v4_do_rcv (meta_sk, skb);#if IS_ENABLED (CONFIG_IPV6 ) } else {tcp_v6_do_rcv (meta_sk, skb);#endif /* CONFIG_IPV6 */ } -bh_unlock_sock (meta_sk);sock_put (meta_sk);/* Taken by mptcp_hash_find */ return 1 ; }
@@ -1260,48 +1236,20 @@ int mptcp_do_join_short(struct sk_buff *skb,/* OK, this is a new syn/join, let's create a new open request and * send syn+ack */ -bh_lock_sock (meta_sk); -/* This check is also done in mptcp_vX_do_rcv. But, there we cannot -* call tcp_vX_send_reset, because we hold already two socket-locks. -* (the listener and the meta from above) -* -* And the send-reset will try to take yet another one (ip_send_reply). -* Thus, we propagate the reset up to tcp_rcv_state_process. +/* mptcp_v4_do_rcv tries to free the skb - we prevent this, as +* the skb will finally be freed by tcp_v4_do_rcv (where we are +* coming from) */ -if (!mptcp_can_new_subflow (meta_sk)) { -MPTCP_INC_STATS (sock_net (meta_sk),MPTCP_MIB_JOINFALLBACK ); -bh_unlock_sock (meta_sk); -sock_put (meta_sk);/* Taken by mptcp_hash_find */ -return -1 ; - } - -if (sock_owned_by_user (meta_sk)) { - skb->sk = meta_sk; -if (unlikely (sk_add_backlog (meta_sk, skb, - meta_sk->sk_rcvbuf + meta_sk->sk_sndbuf))) -__NET_INC_STATS (net,LINUX_MIB_TCPBACKLOGDROP ); -else -/* Must make sure that upper layers won't free the -* skb if it is added to the backlog-queue. -*/ -skb_get (skb); - }else { -/* mptcp_v4_do_rcv tries to free the skb - we prevent this, as -* the skb will finally be freed by tcp_v4_do_rcv (where we are -* coming from) -*/ -skb_get (skb); -if (skb->protocol ==htons (ETH_P_IP )) { -tcp_v4_do_rcv (meta_sk, skb); +skb_get (skb); +if (skb->protocol ==htons (ETH_P_IP )) { +tcp_v4_do_rcv (meta_sk, skb);#if IS_ENABLED (CONFIG_IPV6 ) - } else { /* IPv6 */ -tcp_v6_do_rcv (meta_sk, skb); + } else { /* IPv6 */ +tcp_v6_do_rcv (meta_sk, skb);#endif /* CONFIG_IPV6 */ - } } -bh_unlock_sock (meta_sk);sock_put (meta_sk);/* Taken by mptcp_hash_find */ return 0 ; }
Signature · 1 line
2.16.2
Christoph Paasch
PATCH 13/20 mptcp: Reset tsorted_anchor after call to tcp_fragment
In reply to Christoph PaaschThe reinject-queue is special in the sense that its segments get kfree'd right after sending. We need to make sure that the dst-pointer is NULL as otherwise during the freeing we will access garbage.
--- net/mptcp/mptcp_output.c | 6 ++++++ 1 file changed, 6 insertions(+)
net/mptcp/mptcp_output.c +6 -0
@@ -637,6 +637,12 @@ static int mptcp_fragment(struct sock *meta_sk, enum tcp_queue tcp_queue,if (undo)tcp_adjust_pcount (meta_sk, skb, -undo); } + +/* tcp_fragment's call to sk_stream_alloc_skb initializes the +* tcp_tsorted_anchor. We need to revert this as it clashes +* with the refdst pointer. +*/ +tcp_skb_tsorted_anchor_cleanup (buff); }return 0 ;
Signature · 1 line
2.16.2
Christoph Paasch
PATCH 14/20 mptcp: Pick correct sk when iterating in mptcp_set_rto
In reply to Christoph PaaschWe are iterating with sk_it as the iterator, so pick this one!
--- include/net/mptcp.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
include/net/mptcp.h +1 -1
@@ -1183,7 +1183,7 @@ static inline void mptcp_set_rto(struct sock *sk)mptcp_for_each_sub (tp->mpcb, mptcp) {struct sock *sk_it =mptcp_to_sock (mptcp); -if ((mptcp_sk_can_send (sk_it) || sk->sk_state ==TCP_SYN_RECV ) && +if ((mptcp_sk_can_send (sk_it) || sk_it->sk_state ==TCP_SYN_RECV ) &&inet_csk (sk_it)->icsk_rto > max_rto) max_rto =inet_csk (sk_it)->icsk_rto; }
Signature · 1 line
2.16.2
Christoph Paasch
PATCH 15/20 mptcp: Build-Fix
In reply to Christoph PaaschNeed to declare a dummy-function when MPTCP is off.
--- include/net/mptcp.h | 5 +++++ 1 file changed, 5 insertions(+)
include/net/mptcp.h +5 -0
@@ -1292,6 +1292,11 @@ bool mptcp_prune_ofo_queue(struct sock *sk);do { \ } while (0) +static inline struct sock *mptcp_to_sock (const struct mptcp_tcp_sock *mptcp) +{ +return NULL ; +} +#define mptcp_for_each_sub (__mpcb, __mptcp) \ if (0)
Signature · 1 line
2.16.2
Christoph Paasch
PATCH 16/20 mptcp: Reset sk_it to NULL as we keep on iterating
In reply to Christoph PaaschThe loop relies on ending with sk_it set to NULL to decide whether or not to send a RST.
As we changed the loop-iteration with commit 7a662b690069 ("mptcp: Make subflow-list an RCU-list") to rather loop on the mptcp_tcp_sock, we need to make sure that at each
iteration sk_it is set back to NULL.
--- net/mptcp/mptcp_input.c | 2 ++ 1 file changed, 2 insertions(+)
net/mptcp/mptcp_input.c +2 -0
@@ -371,6 +371,8 @@ static int mptcp_verif_dss_csum(struct sock *sk)if (sk_it != sk &&tcp_sk (sk_it)->mptcp->fully_established)break ; + + sk_it =NULL ; }if (sk_it) {
Signature · 1 line
2.16.2
Christoph Paasch
PATCH 17/20 mptcp: Close remaining subflows when falling back in mptcp_verif_dss_csum
In reply to Christoph PaaschSince commit b079a187260f ("mptcp: Remove cnt_subflows") subflows can actually be in non-fully established and we will still send an MP_FAIL.
Thus, there can be multiple subflows and we need to make sure that the other ones are getting closed in that case.
Otherwise, we will hit the case where we have a TCP-subflow hanging around (potentially calling tcp_write_err()) when the meta is already closed and we will get a warning like:
[ 213.983095] ------------[ cut here ]------------ [ 213.984532] Meta already closed i_rcv 0 i_snd 0 send_i 1 flags 0x2000301 [ 213.986591] WARNING: CPU: 3 PID: 0 at /mnt/tmp/builder/builder_env/mptcp/net/mptcp/mptcp_ctrl.c:626 mptcp_sock_def_error_report+0xe0/0xf0
[...]
--- net/mptcp/mptcp_input.c | 2 ++ 1 file changed, 2 insertions(+)
net/mptcp/mptcp_input.c +2 -0
@@ -387,6 +387,8 @@ static int mptcp_verif_dss_csum(struct sock *sk)kfree_skb (tmp); } +mptcp_sub_force_close_all (tp->mpcb, sk); + ans =0 ; } }
Signature · 1 line
2.16.2
Christoph Paasch
PATCH 18/20 mptcp: Restart subflow-selection when we force a re-evaluation
In reply to Christoph PaaschAfter setting the path-mask to 0, all subflows become eligible again. At that point, we should re-evaluate the subflow selection.
--- net/mptcp/mptcp_sched.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-)
net/mptcp/mptcp_sched.c +9 -2
@@ -215,7 +215,7 @@ static struct sock *get_available_subflow(struct sock *meta_sk, {struct mptcp_cb *mpcb =tcp_sk (meta_sk)->mpcb;struct sock *sk; -bool force; +bool looping = false, force;/* Answer data_fin on same subflow!!! */ if (meta_sk->sk_shutdown & RCV_SHUTDOWN &&
@@ -232,6 +232,7 @@ static struct sock *get_available_subflow(struct sock *meta_sk, }/* Find the best subflow */ +restart: sk =get_subflow_from_selectors (mpcb, skb, &subflow_is_active, zero_wnd_test, &force);if (force)
@@ -242,7 +243,7 @@ static struct sock *get_available_subflow(struct sock *meta_sk, sk =get_subflow_from_selectors (mpcb, skb, &subflow_is_backup, zero_wnd_test, &force); -if (!force && skb) +if (!force && skb) {/* one used backup sk or one NULL sk where there is no one * temporally unavailable unused backup sk *
@@ -250,6 +251,12 @@ static struct sock *get_available_subflow(struct sock *meta_sk,* sks, so clean the path mask */ TCP_SKB_CB (skb)->path_mask =0 ; + +if (!looping) { + looping = true; + goto restart; + } + }return sk; }
Signature · 1 line
2.16.2
Christoph Paasch
PATCH 19/20 mptcp: Don't free skb when tcp_transmit_skb without cloning
In reply to Christoph PaaschBecause, tcp_transmit_skb will already take care of the free'ing.
--- net/mptcp/mptcp_output.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-)
net/mptcp/mptcp_output.c +1 -3
@@ -562,10 +562,8 @@ static bool mptcp_skb_entail(struct sock *sk, struct sk_buff *skb, int reinject)tcp_init_tso_segs (subskb,1 );/* Empty data-fins are sent immediatly on the subflow */ -if (tcp_transmit_skb (sk, subskb,0 ,GFP_ATOMIC ) !=0 ) { -kfree_skb (subskb); +if (tcp_transmit_skb (sk, subskb,0 ,GFP_ATOMIC ))return false; - } }if (!tp->mptcp->fully_established) {
Signature · 1 line
2.16.2
Christoph Paasch
PATCH 20/20 mptcp: Use spin_lock_bh where appropriate for mpcb_list_lock
In reply to Christoph Paaschmptcp_add_sock/mptcp_del_sock can both be called from user- and bh-context. Thus, we need to do a spin_lock_bh().
--- net/mptcp/mptcp_ctrl.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-)
net/mptcp/mptcp_ctrl.c +4 -6
@@ -1399,11 +1399,9 @@ int mptcp_add_sock(struct sock *meta_sk, struct sock *sk, u8 loc_id, u8 rem_id,sock_hold (meta_sk);refcount_inc (&mpcb->mpcb_refcnt); -local_bh_disable (); -spin_lock (&mpcb->mpcb_list_lock); +spin_lock_bh (&mpcb->mpcb_list_lock);hlist_add_head_rcu (&tp->mptcp->node, &mpcb->conn_list); -spin_unlock (&mpcb->mpcb_list_lock); -local_bh_enable (); +spin_unlock_bh (&mpcb->mpcb_list_lock); tp->mptcp->attached =1 ;
@@ -1463,9 +1461,9 @@ void mptcp_del_sock(struct sock *sk) __func__, mpcb->mptcp_loc_token, tp->mptcp->path_index, sk->sk_state, is_meta_sk(sk)); -spin_lock (&mpcb->mpcb_list_lock); +spin_lock_bh (&mpcb->mpcb_list_lock);hlist_del_init_rcu (&tp->mptcp->node); -spin_unlock (&mpcb->mpcb_list_lock); +spin_unlock_bh (&mpcb->mpcb_list_lock); tp->mptcp->attached =0 ; mpcb->path_index_bits &= ~(1 << tp->mptcp->path_index);
Signature · 1 line
2.16.2
Mat Martineau
Re: [MPTCP] [PATCH 20/20] mptcp: Use spin_lock_bh where appropriate for mpcb_list_lock
Hi Christoph -
I have no additional comments on patches 11-20, so it's only the email thread on patch 10 we need to resolve.
Mat
Quoted 46 lines · Christoph Paasch
mptcp_add_sock/mptcp_del_sock can both be called from user- and bh-context. Thus, we need to do a
spin_lock_bh().Fixes 7a662b690069 ("mptcp: Make subflow-list an RCU-list")Signed-off-by Christoph Paasch <cpaasch(a)apple.com>Signed-off-by Matthieu Baerts <matthieu.baerts(a)tessares.net>(cherry picked from commit c45c9521f503d8dcaac3f3ac5bc4628d83079891)Signed-off-by Christoph Paasch <cpaasch(a)apple.com>---
net/mptcp/mptcp_ctrl.c| 10 ++++------1 file changed, 4 insertions(+), 6 deletions(-)net/mptcp/mptcp_ctrl.c +4
-6@@ -1399,11 +1399,9 @@ int mptcp_add_sock(struct sock *meta_sk, struct sock *sk, u8 loc_id, u8 rem_id,sock_hold (meta_sk);refcount_inc (&mpcb->mpcb_refcnt); -local_bh_disable (); -spin_lock (&mpcb->mpcb_list_lock); +spin_lock_bh (&mpcb->mpcb_list_lock);hlist_add_head_rcu (&tp->mptcp->node, &mpcb->conn_list); -spin_unlock (&mpcb->mpcb_list_lock); -local_bh_enable (); +spin_unlock_bh (&mpcb->mpcb_list_lock); tp->mptcp->attached =1 ;@@ -1463,9 +1461,9 @@ void mptcp_del_sock(struct sock *sk) __func__, mpcb->mptcp_loc_token, tp->mptcp->path_index, sk->sk_state, is_meta_sk(sk)); -spin_lock (&mpcb->mpcb_list_lock); +spin_lock_bh (&mpcb->mpcb_list_lock);hlist_del_init_rcu (&tp->mptcp->node); -spin_unlock (&mpcb->mpcb_list_lock); +spin_unlock_bh (&mpcb->mpcb_list_lock); tp->mptcp->attached =0 ; mpcb->path_index_bits &= ~(1 << tp->mptcp->path_index);Signature · 1 line
2.16.2
--
Mat Martineau
Intel OTC