MP_FAIL echo and retrans
Subsystems (3)
v5 cover MP_FAIL echo and retrans
v5:
- re-check for TCP_CLOSE.
- add a new helper mptcp_check_mp_fail_response().
- add two timers cleanup patches.
v4:
- start and stop sk_timer instead of setting and clearing msk->flags.
- add a new patch.
v3:
- use msk->sk_timer and a msk->flags bit.
- use READ_ONCE/WRITE_ONCE for subflow->mp_fail_response_expect.
- update selftest.
v2:
- don't clear mp_fail_response_expect flag.
- add a helper mp_fail_response_expect_subflow to get the subflow, instead of using msk->first.
- add locks as Mat suggested.
- add a selftest.
Geliang Tang (6):
mptcp: use mptcp_stop_timer
mptcp: add data lock for sk timers
mptcp: add MP_FAIL response support
mptcp: reset subflow when MP_FAIL doesn't respond
selftests: mptcp: check MP_FAIL response mibs
selftests: mptcp: print extra msg in chk_csum_nr
net/mptcp/pm.c | 18 +++++- net/mptcp/protocol.c | 64 ++++++++++++++++++- net/mptcp/protocol.h | 2 + net/mptcp/subflow.c | 13 ++++ .../testing/selftests/net/mptcp/mptcp_join.sh | 49 ++++++++++++-- 5 files changed, 139 insertions(+), 7 deletions(-)
Signature · 1 line
2.34.1
PATCH v5 1/6 mptcp: use mptcp_stop_timer
Use the helper mptcp_stop_timer() instead of using sk_stop_timer() to stop icsk_retransmit_timer directly.
--- net/mptcp/protocol.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
net/mptcp/protocol.c +2 -2
@@ -2756,7 +2756,7 @@ static void __mptcp_destroy_sock(struct sock *sk)/* join list will be eventually flushed (with rst) at sock lock release time*/ list_splice_init (&msk->conn_list, &conn_list); -sk_stop_timer (sk, &msk->sk.icsk_retransmit_timer); +mptcp_stop_timer (sk);sk_stop_timer (sk, &sk->sk_timer); msk->pm.status =0 ;
@@ -2864,7 +2864,7 @@ static int mptcp_disconnect(struct sock *sk, int flags)__mptcp_close_ssk (sk, ssk, subflow,MPTCP_CF_FASTCLOSE ); } -sk_stop_timer (sk, &msk->sk.icsk_retransmit_timer); +mptcp_stop_timer (sk);sk_stop_timer (sk, &sk->sk_timer);if (mptcp_sk (sk)->token)
Signature · 1 line
2.34.1
PATCH v5 2/6 mptcp: add data lock for sk timers
In reply to Geliang Tangmptcp_data_lock() needs to be held when manipulating the msk retransmit_timer or the sk sk_timer. This patch adds the data lock for the both timers.
--- net/mptcp/protocol.c | 12 ++++++++++++ 1 file changed, 12 insertions(+)
net/mptcp/protocol.c +12 -0
@@ -1604,8 +1604,10 @@ void __mptcp_push_pending(struct sock *sk, unsigned int flags) out:/* ensure the rtx timer is running */ +mptcp_data_lock (sk);if (!mptcp_timer_pending (sk))mptcp_reset_timer (sk); +mptcp_data_unlock (sk);if (copied)__mptcp_check_send_data_fin (sk); }
@@ -2490,8 +2492,10 @@ static void __mptcp_retrans(struct sock *sk) reset_timer:mptcp_check_and_set_pending (sk); +mptcp_data_lock (sk);if (!mptcp_timer_pending (sk))mptcp_reset_timer (sk); +mptcp_data_unlock (sk); }static void mptcp_worker (struct work_struct *work)
@@ -2654,8 +2658,10 @@ void mptcp_subflow_shutdown(struct sock *sk, struct sock *ssk, int how) }else {pr_debug ("Sending DATA_FIN on subflow %p" , ssk);tcp_send_ack (ssk); +mptcp_data_lock (sk);if (!mptcp_timer_pending (sk))mptcp_reset_timer (sk); +mptcp_data_unlock (sk); }break ; }
@@ -2756,8 +2762,10 @@ static void __mptcp_destroy_sock(struct sock *sk)/* join list will be eventually flushed (with rst) at sock lock release time*/ list_splice_init (&msk->conn_list, &conn_list); +mptcp_data_lock (sk);mptcp_stop_timer (sk);sk_stop_timer (sk, &sk->sk_timer); +mptcp_data_unlock (sk); msk->pm.status =0 ;/* clears msk->subflow, allowing the following loop to close
@@ -2819,7 +2827,9 @@ static void mptcp_close(struct sock *sk, long timeout)__mptcp_destroy_sock (sk); do_cancel_work = true; }else { +mptcp_data_lock (sk);sk_reset_timer (sk, &sk->sk_timer, jiffies +TCP_TIMEWAIT_LEN ); +mptcp_data_unlock (sk); }release_sock (sk);if (do_cancel_work)
@@ -2864,8 +2874,10 @@ static int mptcp_disconnect(struct sock *sk, int flags)__mptcp_close_ssk (sk, ssk, subflow,MPTCP_CF_FASTCLOSE ); } +mptcp_data_lock (sk);mptcp_stop_timer (sk);sk_stop_timer (sk, &sk->sk_timer); +mptcp_data_unlock (sk);if (mptcp_sk (sk)->token)mptcp_event (MPTCP_EVENT_CLOSED ,mptcp_sk (sk),NULL ,GFP_KERNEL );
Signature · 1 line
2.34.1
PATCH v5 3/6 mptcp: add MP_FAIL response support
In reply to Geliang TangThis patch added a new struct member mp_fail_response_expect in struct mptcp_subflow_context to support MP_FAIL response. In the single subflow with checksum error and contiguous data special case, a MP_FAIL sent in response to another MP_FAIL.
--- net/mptcp/pm.c | 10 +++++++++- net/mptcp/protocol.h | 1 + net/mptcp/subflow.c | 2 ++ 3 files changed, 12 insertions(+), 1 deletion(-)
net/mptcp/pm.c +9 -1
@@ -279,8 +279,16 @@ void mptcp_pm_mp_fail_received(struct sock *sk, u64 fail_seq)pr_debug ("fail_seq=%llu" , fail_seq); -if (!mptcp_has_another_subflow (sk) &&READ_ONCE (msk->allow_infinite_fallback)) +if (mptcp_has_another_subflow (sk) || !READ_ONCE (msk->allow_infinite_fallback)) +return ; + +if (!READ_ONCE (subflow->mp_fail_response_expect)) { +pr_debug ("send MP_FAIL response and infinite map" ); + + subflow->send_mp_fail =1 ; +MPTCP_INC_STATS (sock_net (sk),MPTCP_MIB_MPFAILTX ); subflow->send_infinite_map =1 ; + } }/* path manager helpers */
net/mptcp/protocol.h +1 -0
@@ -456,6 +456,7 @@ struct mptcp_subflow_context { stale :1 ,/* unable to snd/rcv data, do not use for xmit */ local_id_valid : 1; /* local_id is correctly initialized */enum mptcp_data_avail data_avail; +bool mp_fail_response_expect;u32 remote_nonce;u64 thmac;u32 local_nonce;
net/mptcp/subflow.c +2 -0
@@ -1217,6 +1217,8 @@ static bool subflow_check_data_avail(struct sock *ssk)tcp_send_active_reset (ssk,GFP_ATOMIC );while ((skb =skb_peek (&ssk->sk_receive_queue)))sk_eat_skb (ssk, skb); + }else { +WRITE_ONCE (subflow->mp_fail_response_expect, true); }WRITE_ONCE (subflow->data_avail,MPTCP_SUBFLOW_NODATA );return true;
Signature · 1 line
2.34.1
PATCH v5 4/6 mptcp: reset subflow when MP_FAIL doesn't respond
In reply to Geliang TangThis patch added a new msk->flags bit MPTCP_FAIL_NO_RESPONSE, then reused sk_timer to trigger a check if we have not received a response from the peer after sending MP_FAIL. If the peer doesn't respond properly, reset the subflow.
--- net/mptcp/pm.c | 8 ++++++++ net/mptcp/protocol.c | 48 ++++++++++++++++++++++++++++++++++++++++++++ net/mptcp/protocol.h | 1 + net/mptcp/subflow.c | 11 ++++++++++ 4 files changed, 68 insertions(+)
net/mptcp/pm.c +8 -0
@@ -276,6 +276,7 @@ void mptcp_pm_mp_fail_received(struct sock *sk, u64 fail_seq) {struct mptcp_subflow_context *subflow =mptcp_subflow_ctx (sk);struct mptcp_sock *msk =mptcp_sk (subflow->conn); +struct sock *s = (struct sock *)msk;pr_debug ("fail_seq=%llu" , fail_seq);
@@ -288,6 +289,13 @@ void mptcp_pm_mp_fail_received(struct sock *sk, u64 fail_seq) subflow->send_mp_fail =1 ;MPTCP_INC_STATS (sock_net (sk),MPTCP_MIB_MPFAILTX ); subflow->send_infinite_map =1 ; + }else if (s &&inet_sk_state_load (s) !=TCP_CLOSE ) { +pr_debug ("MP_FAIL response received" ); + +mptcp_data_lock (s); +if (inet_sk_state_load (s) !=TCP_CLOSE ) +sk_stop_timer (s, &s->sk_timer); +mptcp_data_unlock (s); } }
net/mptcp/protocol.c +48 -0
70 lines not on this page · raw message
net/mptcp/protocol.h +1 -0
@@ -116,6 +116,7 @@#define MPTCP_WORK_EOF 3#define MPTCP_FALLBACK_DONE 4#define MPTCP_WORK_CLOSE_SUBFLOW 5 +#define MPTCP_FAIL_NO_RESPONSE 6/* MPTCP socket release cb flags */ #define MPTCP_PUSH_PENDING 1
net/mptcp/subflow.c +11 -0
@@ -968,6 +968,7 @@ static enum mapping_status get_mapping_status(struct sock *ssk, {struct mptcp_subflow_context *subflow =mptcp_subflow_ctx (ssk);bool csum_reqd =READ_ONCE (msk->csum_enabled); +struct sock *sk = (struct sock *)msk;struct mptcp_ext *mpext;struct sk_buff *skb;u16 data_len;
@@ -1009,6 +1010,12 @@ static enum mapping_status get_mapping_status(struct sock *ssk,pr_debug ("infinite mapping received" );MPTCP_INC_STATS (sock_net (ssk),MPTCP_MIB_INFINITEMAPRX ); subflow->map_data_len =0 ; +if (sk &&inet_sk_state_load (sk) !=TCP_CLOSE ) { +mptcp_data_lock (sk); +if (inet_sk_state_load (sk) !=TCP_CLOSE ) +sk_stop_timer (sk, &sk->sk_timer); +mptcp_data_unlock (sk); + }return MAPPING_INVALID ; }
@@ -1219,6 +1226,10 @@ static bool subflow_check_data_avail(struct sock *ssk)sk_eat_skb (ssk, skb); }else {WRITE_ONCE (subflow->mp_fail_response_expect, true); +/* The data lock is acquired in __mptcp_move_skbs() */ +sk_reset_timer ((struct sock *)msk, + &((struct sock *)msk)->sk_timer, + jiffies +TCP_RTO_MAX ); }WRITE_ONCE (subflow->data_avail,MPTCP_SUBFLOW_NODATA );return true;
Signature · 1 line
2.34.1
PATCH v5 5/6 selftests: mptcp: check MP_FAIL response mibs
In reply to Geliang TangThis patch extended chk_fail_nr to check the MP_FAIL response mibs.
Added a new argument invert for chk_fail_nr to allow it can check the MP_FAIL TX and RX mibs from the opposite direction.
When the infinite map was received before the MP_FAIL response, the response will be lost. A '-' can be added into fail_tx or fail_rx to represent that MP_FAIL response TX or RX can be lost when doing the checks.
--- .../testing/selftests/net/mptcp/mptcp_join.sh | 38 +++++++++++++++++-- 1 file changed, 35 insertions(+), 3 deletions(-)
tools/testing/selftests/net/mptcp/mptcp_join.sh +35 -3
@@ -1055,13 +1055,38 @@ chk_fail_nr() { local fail_tx=$1 local fail_rx=$2 + local ns_invert=${3:-""} local count local dump_stats + local ns_tx=$ns1 + local ns_rx=$ns2 + local extra_msg="" + local allow_tx_lost=0 + local allow_rx_lost=0 + + if [[ $ns_invert = "invert" ]]; then + ns_tx=$ns2 + ns_rx=$ns1 + extra_msg=" invert" + fi + + if [[ "${fail_tx}" = "-"* ]]; then + allow_tx_lost=1 + fail_tx=${fail_tx:1} + fi + if [[ "${fail_rx}" = "-"* ]]; then + allow_rx_lost=1 + fail_rx=${fail_rx:1} + fi printf "%-${nr_blank}s %s" " " "ftx" - count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMPFailTx | awk '{print $2}') + count=$(ip netns exec $ns_tx nstat -as | grep MPTcpExtMPFailTx | awk '{print $2}') [ -z "$count" ] && count=0 if [ "$count" != "$fail_tx" ]; then + extra_msg="$extra_msg,tx=$count" + fi + if { [ "$count" != "$fail_tx" ] && [ $allow_tx_lost -eq 0 ]; } || + { [ "$count" -gt "$fail_tx" ] && [ $allow_tx_lost -eq 1 ]; }; then echo "[fail] got $count MP_FAIL[s] TX expected $fail_tx" fail_test dump_stats=1
@@ -1070,17 +1095,23 @@ chk_fail_nr() fi echo -n " - failrx" - count=$(ip netns exec $ns2 nstat -as | grep MPTcpExtMPFailRx | awk '{print $2}') + count=$(ip netns exec $ns_rx nstat -as | grep MPTcpExtMPFailRx | awk '{print $2}') [ -z "$count" ] && count=0 if [ "$count" != "$fail_rx" ]; then + extra_msg="$extra_msg,rx=$count" + fi + if { [ "$count" != "$fail_rx" ] && [ $allow_rx_lost -eq 0 ]; } || + { [ "$count" -gt "$fail_rx" ] && [ $allow_rx_lost -eq 1 ]; }; then echo "[fail] got $count MP_FAIL[s] RX expected $fail_rx" fail_test dump_stats=1 else - echo "[ ok ]" + echo -n "[ ok ]" fi [ "${dump_stats}" = 1 ] && dump_stats + + echo "$extra_msg" } chk_fclose_nr()
@@ -2672,6 +2703,7 @@ fail_tests() if reset_with_fail "Infinite map" 1; then run_tests $ns1 $ns2 10.0.1.1 128 chk_join_nr 0 0 0 +1 +0 1 0 1 "$(pedit_action_pkts)" + chk_fail_nr 1 -1 invert fi }
Signature · 1 line
2.34.1
PATCH v5 6/6 selftests: mptcp: print extra msg in chk_csum_nr
In reply to Geliang TangWhen the multiple checksum errors occur in chk_csum_nr(), print the numbers of the errors as an extra message.
--- tools/testing/selftests/net/mptcp/mptcp_join.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-)
tools/testing/selftests/net/mptcp/mptcp_join.sh +10 -1
@@ -1014,6 +1014,7 @@ chk_csum_nr() local csum_ns2=${2:-0} local count local dump_stats + local extra_msg="" local allow_multi_errors_ns1=0 local allow_multi_errors_ns2=0
@@ -1029,6 +1030,9 @@ chk_csum_nr() printf "%-${nr_blank}s %s" " " "sum" count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtDataCsumErr | awk '{print $2}') [ -z "$count" ] && count=0 + if [ "$count" != "$csum_ns1" ]; then + extra_msg="$extra_msg ns1=$count" + fi if { [ "$count" != $csum_ns1 ] && [ $allow_multi_errors_ns1 -eq 0 ]; } || { [ "$count" -lt $csum_ns1 ] && [ $allow_multi_errors_ns1 -eq 1 ]; }; then echo "[fail] got $count data checksum error[s] expected $csum_ns1"
@@ -1040,15 +1044,20 @@ chk_csum_nr() echo -n " - csum " count=$(ip netns exec $ns2 nstat -as | grep MPTcpExtDataCsumErr | awk '{print $2}') [ -z "$count" ] && count=0 + if [ "$count" != "$csum_ns2" ]; then + extra_msg="$extra_msg ns2=$count" + fi if { [ "$count" != $csum_ns2 ] && [ $allow_multi_errors_ns2 -eq 0 ]; } || { [ "$count" -lt $csum_ns2 ] && [ $allow_multi_errors_ns2 -eq 1 ]; }; then echo "[fail] got $count data checksum error[s] expected $csum_ns2" fail_test dump_stats=1 else - echo "[ ok ]" + echo -n "[ ok ]" fi [ "${dump_stats}" = 1 ] && dump_stats + + echo "$extra_msg" } chk_fail_nr()
Signature · 1 line
2.34.1
Re: selftests: mptcp: print extra msg in chk_csum_nr: Tests Results
Hi Geliang,
Thank you for your modifications, that's great!
Our CI did some validations and here is its report:
- KVM Validation: normal:
- Success! ✅:
- Task: https://cirrus-ci.com/task/6750472991145984
- Summary: https://api.cirrus-ci.com/v1/artifact/task/6750472991145984/summary/summary.txt
- KVM Validation: debug:
- Unstable: 1 failed test(s): selftest_mptcp_join 🔴:
- Task: https://cirrus-ci.com/task/4511867316994048
- Summary: https://api.cirrus-ci.com/v1/artifact/task/4511867316994048/summary/summary.txt
Initiator: Patchew Applier
Commits: https://github.com/multipath-tcp/mptcp_net-next/commits/3d584c2de1fc
Please note that despite all the efforts that have been already done to have a stable tests suite when executed on a public CI like here, it is possible some reported issues are not due to your modifications. Still, do not hesitate to help us improve that ;-)
Cheers,
MPTCP GH Action bot
Bot operated by Matthieu Baerts (Tessares)
Quoted 5 lines · Geliang Tang
v5:
- re-check forTCP_CLOSE.
- add a new helpermptcp_check_mp_fail_response().
- add two timers cleanup patches.
Hi Geliang -
v5 is looking good to me so far, but I still need to run some tests.
Thanks,
Mat
Quoted 26 lines
v4:
- start and stopsk_timerinstead of setting and clearingmsk->flags.
- add a new patch.v3:
- usemsk->sk_timerand amsk->flagsbit.
- use READ_ONCE/WRITE_ONCE forsubflow->mp_fail_response_expect.
- update selftest.v2:
- don't clearmp_fail_response_expectflag.
- add a helpermp_fail_response_expect_subflowto get the subflow, instead of usingmsk->first.
- add locks as Mat suggested.
- add a selftest.Geliang Tang (6):
mptcp: usemptcp_stop_timer
mptcp: add data lock for sk timers
mptcp: addMP_FAILresponse support
mptcp: reset subflow whenMP_FAILdoesn't respond
selftests: mptcp: checkMP_FAILresponse mibs
selftests: mptcp: print extra msg inchk_csum_nr
Quoted 12 lines
net/mptcp/pm.c | 18 +++++- net/mptcp/protocol.c | 64 ++++++++++++++++++- net/mptcp/protocol.h | 2 + net/mptcp/subflow.c | 13 ++++ .../testing/selftests/net/mptcp/mptcp_join.sh | 49 ++++++++++++-- 5 files changed, 139 insertions(+), 7 deletions(-)Signature · 1 line
2.34.1
--
Mat Martineau
Intel
Mat Martineau
Reviewed-byQuoted 8 lines · Mat Martineau
Quoted 5 lines · Geliang Tang
v5:
- re-check forTCP_CLOSE.
- add a new helpermptcp_check_mp_fail_response().
- add two timers cleanup patches.
Quoted 4 lines
Hi Geliang -
v5 is looking good to me so far, but I still need to run some tests.
Ok, I got a chance to look at the packet traces from the self tests. I also tested with a shorter timeout and commented out the echo, and the subflow was reset as expected.
Looks good for the export branch:
It would be good to have a packetdrill test for this, even if it does take a couple of minutes to fail. We wouldn't have to run it in the normal CI.
Mat
Quoted 40 lines
Quoted 27 lines
v4:
- start and stopsk_timerinstead of setting and clearingmsk->flags.
- add a new patch.v3:
- usemsk->sk_timerand amsk->flagsbit.
- use READ_ONCE/WRITE_ONCE forsubflow->mp_fail_response_expect.
- update selftest.v2:
- don't clearmp_fail_response_expectflag.
- add a helpermp_fail_response_expect_subflowto get the subflow, instead of usingmsk->first.
- add locks as Mat suggested.
- add a selftest.Geliang Tang (6):
mptcp: usemptcp_stop_timer
mptcp: add data lock for sk timers
mptcp: addMP_FAILresponse support
mptcp: reset subflow whenMP_FAILdoesn't respond
selftests: mptcp: checkMP_FAILresponse mibs
selftests: mptcp: print extra msg inchk_csum_nrQuoted 12 lines
net/mptcp/pm.c | 18 +++++- net/mptcp/protocol.c | 64 ++++++++++++++++++- net/mptcp/protocol.h | 2 + net/mptcp/subflow.c | 13 ++++ .../testing/selftests/net/mptcp/mptcp_join.sh | 49 ++++++++++++-- 5 files changed, 139 insertions(+), 7 deletions(-)Signature · 1 line
2.34.1
Quoted 5 lines
--
Mat Martineau
Intel
--
Mat Martineau
Intel
Hi Geliang, Mat,
Quoted 15 lines · Mat Martineau
Quoted 8 lines · Mat Martineau
Quoted 5 lines · Geliang Tang
v5:
- re-check forTCP_CLOSE.
- add a new helpermptcp_check_mp_fail_response().
- add two timers cleanup patches.Quoted 4 lines
Hi Geliang -
v5 is looking good to me so far, but I still need to run some tests.
Quoted 7 lines
Ok, I got a chance to look at the packet traces from the self tests. I also tested with a shorter timeout and commented out the echo, and the subflow was reset as expected.
Looks good for the export branch:
Reviewed-by Mat Martineau <[email protected]>
With a bit of delay, sorry for that, these patches are now in our tree!
Thank you for the patches and the reviews!
New patches for t/upstream:
- cf0e4660d69e: mptcp: use mptcp_stop_timer
- 4afa5bc41699: mptcp: add data lock for sk timers
- d1919f72133d: mptcp: add MP_FAIL response support
- bb04e167b541: mptcp: reset subflow when MP_FAIL doesn't respond
- 735b7655b253: selftests: mptcp: check MP_FAIL response mibs
- 054951083391: selftests: mptcp: print extra msg in chk_csum_nr
- Results: b3fcb6331ff3..73b94fb27a8c (export)
Builds and tests are now in progress:
https://cirrus-ci.com/github/multipath-tcp/mptcp_net-next/export/20220323T170824
https://github.com/multipath-tcp/mptcp_net-next/actions/workflows/build-validation.yml?query=branch:export
Quoted 3 lines
It would be good to have a packetdrill test for this, even if it does take a couple of minutes to fail. We wouldn't have to run it in the normal CI.
I didn't close #261 for this reason then.
https://github.com/multipath-tcp/mptcp_net-next/issues/261
Cheers,
Matt
Signature · 2 lines
Tessares | Belgium | Hybrid Access Solutions www.tessares.net