Skip to contentSkip to conversation

tcp: timer restart for tail loss

RFC PATCH Superseded Per Hurtig 6 participants 17 messages Dec 7, 2015 – Dec 10, 2015

Subsystems (4)
lore
Share & tools
lore

Per Hurtig

author
#0 lore raw

cover tcp: timer restart for tail loss

This is a request for comments.

RTO and TLP restart is a modification to the restart process of the RTO and TLP timers in TCP. Currently, both timers are restarted with its corresponding timeout value when an acknowledgment (ACK) for correctly received data is received. In many situations, resetting the timers on incoming ACKs will add an implicit offset of at least RTT seconds to the loss recovery process.

The goal of the modified restart is to provide quicker loss recovery for segments lost in the end of a burst/connection, where the limited feedback from a receiver inhibits the use of fast/early retransmit. To accomplish this the algorithm adjusts the RTO and PTO (TLP's timer) values on each rearm of the timers to allow them to expire after exactly RTO and PTO ms, respectively.

The restart behavior is controlled by the new tcp_timer_restart sysctl. tcp_timer_restart==0; disables RTOR and TLPR.

		 ==1; enables RTOR.
		 ==2; enables TLPR.
		 ==3; enables both RTOR and TLPR. [DEFAULT]

The new restart behavior has been approved by the IETF for publication as an experimental RFC (for the RTO part of the mechanism) [1], and experiments have been conducted to show both the benefit of this strategy (in terms of reduced loss-recovery delays) and the limited negative impact (in terms of spurious retransmissions) [2]. More information regarding the mechanism can be found at [3].

Basic functionality tests, using packetdrill, are available at:
https://github.com/perhurt/packetdrill/tree/master/gtests/net/packetdrill/tests/linux/timer_restart

[1] https://datatracker.ietf.org/doc/draft-ietf-tcpm-rtorestart/
[2] http://www.sigcomm.org/sites/default/files/ccr/papers/2015/January/0000000-0000000.pdf
[3] http://riteproject.eu/resources/rto-restart/

Per Hurtig (2):
tcp: RTO Restart (RTOR)
tcp: TLP restart (TLPR)

 Documentation/networking/ip-sysctl.txt | 12 ++++++++++++
 include/net/tcp.h                      |  6 +++++-
 net/ipv4/sysctl_net_ipv4.c             | 10 ++++++++++
 net/ipv4/tcp_input.c                   | 26 +++++++++++++++++++++++++-
 net/ipv4/tcp_output.c                  | 12 ++++++++++--
 5 files changed, 62 insertions(+), 4 deletions(-)
Signature · 1 line
1.9.1
author
#1 lore raw

RFC PATCH 1/2 tcp: RTO Restart (RTOR)

This patch implements the RTO restart modification (RTOR). When data is ACKed, and the RTO timer is restarted, the time elapsed since the last outstanding segment was transmitted is subtracted from the calculated RTO value. This way, the RTO timer will expire after exactly RTO seconds, and not RTO + RTT [+ delACK] seconds.

This patch also implements a new sysctl (tcp_timer_restart) that is used to control the timer restart behavior.

---
 Documentation/networking/ip-sysctl.txt | 12 ++++++++++++
 include/net/tcp.h                      |  4 ++++
 net/ipv4/sysctl_net_ipv4.c             | 10 ++++++++++
 net/ipv4/tcp_input.c                   | 24 ++++++++++++++++++++++++
 4 files changed, 50 insertions(+)
Documentation/networking/ip-sysctl.txt +12 -0
index 2ea4c45..4094128 100644
@@ -591,6 +591,18 @@ tcp_syn_retries - INTEGER
 	with the current initial RTO of 1second. With this the final timeout
 	for an active TCP connection attempt will happen after 127seconds.
 
+tcp_timer_restart - INTEGER
+	Controls how the RTO and PTO timers are restarted (RTOR and TLPR).
+	If set (per timer or combined) the timers are restarted with
+	respect to the earliest outstanding segment, to not extend tail loss
+	latency unnecessarily.
+	Possible values:
+		0 disables RTOR and TLPR.
+		1 enables RTOR.
+		2 enables TLPR.
+		3 enables RTOR and TLPR.
+	Default: 3
+
 tcp_timestamps - BOOLEAN
 	Enable timestamps as defined in RFC1323.
 
include/net/tcp.h +4 -0
index f80e74c..bf98768 100644
@@ -76,6 +76,9 @@ void tcp_time_wait(struct sock *sk, int state, int timeo);
 /* After receiving this amount of duplicate ACKs fast retransmit starts. */
 #define TCP_FASTRETRANS_THRESH 3
 
+/* Disable RTO Restart if the number of outstanding segments is at least. */
+#define TCP_RTORESTART_THRESH	4
+
 /* Maximal number of ACKs sent quickly to accelerate slow-start. */
 #define TCP_MAX_QUICKACKS	16U
 
@@ -284,6 +287,7 @@ extern int sysctl_tcp_autocorking;
 extern int sysctl_tcp_invalid_ratelimit;
 extern int sysctl_tcp_pacing_ss_ratio;
 extern int sysctl_tcp_pacing_ca_ratio;
+extern int sysctl_tcp_timer_restart;
 
 extern atomic_long_t tcp_memory_allocated;
 extern struct percpu_counter tcp_sockets_allocated;
net/ipv4/sysctl_net_ipv4.c +10 -0
index a0bd7a5..dfb6968 100644
@@ -28,6 +28,7 @@
 
 static int zero;
 static int one = 1;
+static int three = 3;
 static int four = 4;
 static int thousand = 1000;
 static int gso_max_segs = GSO_MAX_SEGS;
@@ -745,6 +746,15 @@ static struct ctl_table ipv4_table[] = {
 		.extra2		= &thousand,
 	},
 	{
+		.procname	= "tcp_timer_restart",
+		.data		= &sysctl_tcp_timer_restart,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec_minmax,
+		.extra1		= &zero,
+		.extra2		= &three,
+	},
+	{
 		.procname	= "tcp_autocorking",
 		.data		= &sysctl_tcp_autocorking,
 		.maxlen		= sizeof(int),
net/ipv4/tcp_input.c +24 -0
index fdd88c3..66e0425 100644
@@ -101,6 +101,7 @@ int sysctl_tcp_thin_dupack __read_mostly;
 
 int sysctl_tcp_moderate_rcvbuf __read_mostly = 1;
 int sysctl_tcp_early_retrans __read_mostly = 3;
+int sysctl_tcp_timer_restart __read_mostly = 3;
 int sysctl_tcp_invalid_ratelimit __read_mostly = HZ/2;
 
 #define FLAG_DATA		0x01 /* Incoming frame contained data.		*/
@@ -2997,6 +2998,18 @@ static void tcp_cong_avoid(struct sock *sk, u32 ack, u32 acked)
 	tcp_sk(sk)->snd_cwnd_stamp = tcp_time_stamp;
 }
 
+static u32 tcp_unsent_pkts(const struct sock *sk)
+{
+	struct sk_buff *skb = tcp_send_head(sk);
+	u32 pkts = 0;
+
+	if (skb)
+		tcp_for_write_queue_from(skb, sk)
+			pkts += tcp_skb_pcount(skb);
+
+	return pkts;
+}
+
 /* Restart timer after forward progress on connection.
  * RFC2988 recommends to restart timer to now+rto.
  */
@@ -3027,6 +3040,17 @@ void tcp_rearm_rto(struct sock *sk)
 			 */
 			if (delta > 0)
 				rto = delta;
+		} else if (icsk->icsk_pending == ICSK_TIME_RETRANS &&
+			   (sysctl_tcp_timer_restart == 1 ||
+			    sysctl_tcp_timer_restart == 3) &&
+			   (tp->packets_out + tcp_unsent_pkts(sk) <
+			    TCP_RTORESTART_THRESH)) {
+			struct sk_buff *skb = tcp_write_queue_head(sk);
+			const u32 rto_time_stamp = tcp_skb_timestamp(skb);
+			s32 delta = (s32)(tcp_time_stamp - rto_time_stamp);
+
+			if (delta > 0 && rto > delta)
+				rto -= delta;
 		}
 		inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS, rto,
 					  TCP_RTO_MAX);
Signature · 1 line
1.9.1
#2 lore raw

Re: [RFC PATCH net-next 1/2] tcp: RTO Restart (RTOR)

Quoted 40 lines · Per Hurtig

This patch implements the RTO restart modification (RTOR). When data is ACKed, and the RTO timer is restarted, the time elapsed since the last outstanding segment was transmitted is subtracted from the calculated RTO value. This way, the RTO timer will expire after exactly RTO seconds, and not RTO + RTT [+ delACK] seconds.

This patch also implements a new sysctl (tcp_timer_restart) that is used to control the timer restart behavior.

---
 Documentation/networking/ip-sysctl.txt | 12 ++++++++++++
 include/net/tcp.h                      |  4 ++++
 net/ipv4/sysctl_net_ipv4.c             | 10 ++++++++++
 net/ipv4/tcp_input.c                   | 24 ++++++++++++++++++++++++
 4 files changed, 50 insertions(+)
Documentation/networking/ip-sysctl.txt +12 -0
index 2ea4c45..4094128 100644
@@ -591,6 +591,18 @@ tcp_syn_retries - INTEGER
 	with the current initial RTO of 1second. With this the final timeout
 	for an active TCP connection attempt will happen after 127seconds.
 
+tcp_timer_restart - INTEGER
+	Controls how the RTO and PTO timers are restarted (RTOR and TLPR).
+	If set (per timer or combined) the timers are restarted with
+	respect to the earliest outstanding segment, to not extend tail loss
+	latency unnecessarily.
+	Possible values:
+		0 disables RTOR and TLPR.
+		1 enables RTOR.
+		2 enables TLPR.
+		3 enables RTOR and TLPR.
+	Default: 3
+
 tcp_timestamps - BOOLEAN
 	Enable timestamps as defined in RFC1323.
 

[...snip...]

Quoted 4 lines
net/ipv4/tcp_input.c +0 -0
index fdd88c3..66e0425 100644

[...snip...]

net/ipv4/tcp_input.c void tcp_rearm_rto(struct sock *sk) · line 3045

Quoted 10 lines

/* Restart timer after forward progress on connection.
* RFC2988 recommends to restart timer to now+rto. */

net/ipv4/tcp_input.c +3 -0
@@ -3027,6 +3040,17 @@ void tcp_rearm_rto(struct sock *sk)
 			 */
 			if (delta > 0)
 				rto = delta;
+		} else if (icsk->icsk_pending == ICSK_TIME_RETRANS &&
+			   (sysctl_tcp_timer_restart == 1 ||
+			    sysctl_tcp_timer_restart == 3) &&

Use a bit operation here instead? Also I think that this sysctl would benefit from named constants rather than use of literals (similar comment applies to the other patch too).

Quoted 11 lines
net/ipv4/tcp_input.c +8 -0
+			   (tp->packets_out + tcp_unsent_pkts(sk) <
+			    TCP_RTORESTART_THRESH)) {
+			struct sk_buff *skb = tcp_write_queue_head(sk);
+			const u32 rto_time_stamp = tcp_skb_timestamp(skb);
+			s32 delta = (s32)(tcp_time_stamp - rto_time_stamp);
+
+			if (delta > 0 && rto > delta)
+				rto -= delta;
 		}
 		inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS, rto,
 					  TCP_RTO_MAX);
Signature · 1 line
 i.
#3 lore raw

Re: [RFC PATCH net-next 1/2] tcp: RTO Restart (RTOR)

In reply to Per Hurtig
Quoted 21 lines · Per Hurtig

This patch implements the RTO restart modification (RTOR). When data is ACKed, and the RTO timer is restarted, the time elapsed since the last outstanding segment was transmitted is subtracted from the calculated RTO value. This way, the RTO timer will expire after exactly RTO seconds, and not RTO + RTT [+ delACK] seconds.

This patch also implements a new sysctl (tcp_timer_restart) that is used to control the timer restart behavior.

---
 Documentation/networking/ip-sysctl.txt | 12 ++++++++++++
 include/net/tcp.h                      |  4 ++++
 net/ipv4/sysctl_net_ipv4.c             | 10 ++++++++++
 net/ipv4/tcp_input.c                   | 24 ++++++++++++++++++++++++
 4 files changed, 50 insertions(+)
Documentation/networking/ip-sysctl.txt +0 -0
index 2ea4c45..4094128 100644

(snip)

Quoted 20 lines
Documentation/networking/ip-sysctl.txt +12 -0
@@ -2997,6 +2998,18 @@ static void tcp_cong_avoid(struct sock *sk, u32 ack, u32 acked)
 	tcp_sk(sk)->snd_cwnd_stamp = tcp_time_stamp;
 }
 
+static u32 tcp_unsent_pkts(const struct sock *sk)
+{
+	struct sk_buff *skb = tcp_send_head(sk);
+	u32 pkts = 0;
+
+	if (skb)
+		tcp_for_write_queue_from(skb, sk)
+			pkts += tcp_skb_pcount(skb);
+
+	return pkts;
+}
+
 /* Restart timer after forward progress on connection.
  * RFC2988 recommends to restart timer to now+rto.
  */

Documentation/networking/ip-sysctl.txt void tcp_rearm_rto(struct sock *sk) · line 3047

Quoted 8 lines
Documentation/networking/ip-sysctl.txt +5 -0
@@ -3027,6 +3040,17 @@ void tcp_rearm_rto(struct sock *sk)
 			 */
 			if (delta > 0)
 				rto = delta;
+		} else if (icsk->icsk_pending == ICSK_TIME_RETRANS &&
+			   (sysctl_tcp_timer_restart == 1 ||
+			    sysctl_tcp_timer_restart == 3) &&
+			   (tp->packets_out + tcp_unsent_pkts(sk) <
+			    TCP_RTORESTART_THRESH)) {

(snip)

By when this gets hit, you could have a big write queue. What about wrapping at least this this condition tp->packets_out + tcp_unsent_pkts(sk) < TCP_RTORESTART_THRESH in its own check function? Like:

+static bool tcp_can_rtor(const struct sock *sk)
+{
+ struct sk_buff *skb = tcp_send_head(sk);
+ s32 target = TCP_RTORESTART_THRESH - tp->packets_out;
+
+ if (target <= 0)
+ return false;
+
+ if (skb) {
+ tcp_for_write_queue_from(skb, sk) {
+ target -= tcp_skb_pcount(skb);
+ if (target <= 0)
+ return false;
+ }
+ }
+
+ return true;
+}

This way it will only traverse what is needed for the check itself.

Marcelo

#4 lore raw

Re: [RFC PATCH net-next 1/2] tcp: RTO Restart (RTOR)

In reply to Per Hurtig

On Mon, 2015-12-07 at 10:00 +0100, Per Hurtig wrote:
\

net/ipv4/tcp_input.c static void tcp_cong_avoid(struct sock *sk, u32 ack, u32 acked) · line 3011

Quoted 12 lines
net/ipv4/tcp_input.c +11 -0
+static u32 tcp_unsent_pkts(const struct sock *sk)
+{
+	struct sk_buff *skb = tcp_send_head(sk);
+	u32 pkts = 0;
+
+	if (skb)
+		tcp_for_write_queue_from(skb, sk)
+			pkts += tcp_skb_pcount(skb);
+
+	return pkts;
+}

write queue can be very big (consider GSO/TSO being off for example)

Parsing it just to implement later :

(tp->packets_out + tcp_unsent_pkts(sk) <

Review

Quoted 1 line
(quoted diff) +1 -0
+			    TCP_RTORESTART_THRESH)

is probably not very efficient.

I would rather implement a different helper, aborting the loop as soon as the condition can not be met anymore.

Quoted 22 lines
net/ipv4/tcp_input.c +12 -0
+
 /* Restart timer after forward progress on connection.
  * RFC2988 recommends to restart timer to now+rto.
  */
@@ -3027,6 +3040,17 @@ void tcp_rearm_rto(struct sock *sk)
 			 */
 			if (delta > 0)
 				rto = delta;
+		} else if (icsk->icsk_pending == ICSK_TIME_RETRANS &&
+			   (sysctl_tcp_timer_restart == 1 ||
+			    sysctl_tcp_timer_restart == 3) &&
+			   (tp->packets_out + tcp_unsent_pkts(sk) <
+			    TCP_RTORESTART_THRESH)) {
+			struct sk_buff *skb = tcp_write_queue_head(sk);
+			const u32 rto_time_stamp = tcp_skb_timestamp(skb);
+			s32 delta = (s32)(tcp_time_stamp - rto_time_stamp);
+
+			if (delta > 0 && rto > delta)
+				rto -= delta;
 		}
 		inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS, rto,
 					  TCP_RTO_MAX);
#5 lore raw

Re: [RFC PATCH net-next 1/2] tcp: RTO Restart (RTOR)

In reply to Per Hurtig
Quoted 122 lines · Per Hurtig

This patch implements the RTO restart modification (RTOR). When data is ACKed, and the RTO timer is restarted, the time elapsed since the last outstanding segment was transmitted is subtracted from the calculated RTO value. This way, the RTO timer will expire after exactly RTO seconds, and not RTO + RTT [+ delACK] seconds.

This patch also implements a new sysctl (tcp_timer_restart) that is used to control the timer restart behavior.

---
 Documentation/networking/ip-sysctl.txt | 12 ++++++++++++
 include/net/tcp.h                      |  4 ++++
 net/ipv4/sysctl_net_ipv4.c             | 10 ++++++++++
 net/ipv4/tcp_input.c                   | 24 ++++++++++++++++++++++++
 4 files changed, 50 insertions(+)
Documentation/networking/ip-sysctl.txt +12 -0
index 2ea4c45..4094128 100644
@@ -591,6 +591,18 @@ tcp_syn_retries - INTEGER
        with the current initial RTO of 1second. With this the final timeout
        for an active TCP connection attempt will happen after 127seconds.
 
+tcp_timer_restart - INTEGER
+       Controls how the RTO and PTO timers are restarted (RTOR and TLPR).
+       If set (per timer or combined) the timers are restarted with
+       respect to the earliest outstanding segment, to not extend tail loss
+       latency unnecessarily.
+       Possible values:
+               0 disables RTOR and TLPR.
+               1 enables RTOR.
+               2 enables TLPR.
+               3 enables RTOR and TLPR.
+       Default: 3
+
 tcp_timestamps - BOOLEAN
        Enable timestamps as defined in RFC1323.
 
include/net/tcp.h +4 -0
index f80e74c..bf98768 100644
@@ -76,6 +76,9 @@ void tcp_time_wait(struct sock *sk, int state, int timeo);
 /* After receiving this amount of duplicate ACKs fast retransmit starts. */
 #define TCP_FASTRETRANS_THRESH 3
 
+/* Disable RTO Restart if the number of outstanding segments is at least. */
+#define TCP_RTORESTART_THRESH  4
+
 /* Maximal number of ACKs sent quickly to accelerate slow-start. */
 #define TCP_MAX_QUICKACKS      16U
 
@@ -284,6 +287,7 @@ extern int sysctl_tcp_autocorking;
 extern int sysctl_tcp_invalid_ratelimit;
 extern int sysctl_tcp_pacing_ss_ratio;
 extern int sysctl_tcp_pacing_ca_ratio;
+extern int sysctl_tcp_timer_restart;
 
 extern atomic_long_t tcp_memory_allocated;
 extern struct percpu_counter tcp_sockets_allocated;
net/ipv4/sysctl_net_ipv4.c +10 -0
index a0bd7a5..dfb6968 100644
@@ -28,6 +28,7 @@
 
 static int zero;
 static int one = 1;
+static int three = 3;
 static int four = 4;
 static int thousand = 1000;
 static int gso_max_segs = GSO_MAX_SEGS;
@@ -745,6 +746,15 @@ static struct ctl_table ipv4_table[] = {
                .extra2         = &thousand,
        },
        {
+               .procname       = "tcp_timer_restart",
+               .data           = &sysctl_tcp_timer_restart,
+               .maxlen         = sizeof(int),
+               .mode           = 0644,
+               .proc_handler   = proc_dointvec_minmax,
+               .extra1         = &zero,
+               .extra2         = &three,
+       },
+       {
                .procname       = "tcp_autocorking",
                .data           = &sysctl_tcp_autocorking,
                .maxlen         = sizeof(int),
net/ipv4/tcp_input.c +13 -0
index fdd88c3..66e0425 100644
@@ -101,6 +101,7 @@ int sysctl_tcp_thin_dupack __read_mostly;
 
 int sysctl_tcp_moderate_rcvbuf __read_mostly = 1;
 int sysctl_tcp_early_retrans __read_mostly = 3;
+int sysctl_tcp_timer_restart __read_mostly = 3;
 int sysctl_tcp_invalid_ratelimit __read_mostly = HZ/2;
 
 #define FLAG_DATA              0x01 /* Incoming frame contained data.          */
@@ -2997,6 +2998,18 @@ static void tcp_cong_avoid(struct sock *sk, u32 ack, u32 acked)
        tcp_sk(sk)->snd_cwnd_stamp = tcp_time_stamp;
 }
 
+static u32 tcp_unsent_pkts(const struct sock *sk)
+{
+       struct sk_buff *skb = tcp_send_head(sk);
+       u32 pkts = 0;
+
+       if (skb)
+               tcp_for_write_queue_from(skb, sk)
+                       pkts += tcp_skb_pcount(skb);
+
+       return pkts;
+}
+
 /* Restart timer after forward progress on connection.
  * RFC2988 recommends to restart timer to now+rto.
  */

net/ipv4/tcp_input.c void tcp_rearm_rto(struct sock *sk) · line 3046

Quoted 7 lines
net/ipv4/tcp_input.c +4 -0
@@ -3027,6 +3040,17 @@ void tcp_rearm_rto(struct sock *sk)
                         */
                        if (delta > 0)
                                rto = delta;
+               } else if (icsk->icsk_pending == ICSK_TIME_RETRANS &&
+                          (sysctl_tcp_timer_restart == 1 ||
+                           sysctl_tcp_timer_restart == 3) &&
+                          (tp->packets_out + tcp_unsent_pkts(sk) <

We re-arm RTO every ACK. Do we really need a loop here to improve short-transfer RTO performance?

Going one level up: what's the rationale to arm RTO differently depending on the #packets in the write queue? if the idea is to offset the elapsed of the first unacked packet in RTO, it should generally apply to any packet w/ any inflight, and we can forgo another packet counting heuristic.

While larger inflight can trigger more fast recoveries, tail drops can happen at any time. This is why TLP is always enabled of any inflight. Sounds like we can benefit if RTO-restart is always used too? I am planning to experiment your patch w and w/o the packet count checks.

Quoted 14 lines
net/ipv4/tcp_input.c +7 -1
+                           TCP_RTORESTART_THRESH)) {
+                       struct sk_buff *skb = tcp_write_queue_head(sk);
+                       const u32 rto_time_stamp = tcp_skb_timestamp(skb);
+                       s32 delta = (s32)(tcp_time_stamp - rto_time_stamp);
+
+                       if (delta > 0 && rto > delta)
+                               rto -= delta;
                }
                inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS, rto,
                                          TCP_RTO_MAX);
--

1.9.1

author
#6 lore raw

Re: [RFC PATCH net-next 1/2] tcp: RTO Restart (RTOR)

Quoted 136 lines · Yuchung Cheng
Quoted 122 lines · Per Hurtig

This patch implements the RTO restart modification (RTOR). When data is ACKed, and the RTO timer is restarted, the time elapsed since the last outstanding segment was transmitted is subtracted from the calculated RTO value. This way, the RTO timer will expire after exactly RTO seconds, and not RTO + RTT [+ delACK] seconds.

This patch also implements a new sysctl (tcp_timer_restart) that is used to control the timer restart behavior.

---

Documentation/networking/ip-sysctl.txt | 12 ++++++++++++
include/net/tcp.h                      |  4 ++++
net/ipv4/sysctl_net_ipv4.c             | 10 ++++++++++
net/ipv4/tcp_input.c                   | 24 ++++++++++++++++++++++++
4 files changed, 50 insertions(+)
Documentation/networking/ip-sysctl.txt +12 -0
index 2ea4c45..4094128 100644
@@ -591,6 +591,18 @@ tcp_syn_retries - INTEGER
       with the current initial RTO of 1second. With this the final timeout
       for an active TCP connection attempt will happen after 127seconds.
 
+tcp_timer_restart - INTEGER
+       Controls how the RTO and PTO timers are restarted (RTOR and TLPR).
+       If set (per timer or combined) the timers are restarted with
+       respect to the earliest outstanding segment, to not extend tail loss
+       latency unnecessarily.
+       Possible values:
+               0 disables RTOR and TLPR.
+               1 enables RTOR.
+               2 enables TLPR.
+               3 enables RTOR and TLPR.
+       Default: 3
+

tcp_timestamps - BOOLEAN
Enable timestamps as defined in RFC1323.

include/net/tcp.h +4 -0
index f80e74c..bf98768 100644
@@ -76,6 +76,9 @@ void tcp_time_wait(struct sock *sk, int state, int timeo);
 /* After receiving this amount of duplicate ACKs fast retransmit starts. */
 #define TCP_FASTRETRANS_THRESH 3
 
+/* Disable RTO Restart if the number of outstanding segments is at least. */
+#define TCP_RTORESTART_THRESH  4
+
 /* Maximal number of ACKs sent quickly to accelerate slow-start. */
 #define TCP_MAX_QUICKACKS      16U
 
@@ -284,6 +287,7 @@ extern int sysctl_tcp_autocorking;
 extern int sysctl_tcp_invalid_ratelimit;
 extern int sysctl_tcp_pacing_ss_ratio;
 extern int sysctl_tcp_pacing_ca_ratio;
+extern int sysctl_tcp_timer_restart;
 
 extern atomic_long_t tcp_memory_allocated;
 extern struct percpu_counter tcp_sockets_allocated;
net/ipv4/sysctl_net_ipv4.c +10 -0
index a0bd7a5..dfb6968 100644
@@ -28,6 +28,7 @@
 
 static int zero;
 static int one = 1;
+static int three = 3;
 static int four = 4;
 static int thousand = 1000;
 static int gso_max_segs = GSO_MAX_SEGS;
@@ -745,6 +746,15 @@ static struct ctl_table ipv4_table[] = {
               .extra2         = &thousand,
       },
       {
+               .procname       = "tcp_timer_restart",
+               .data           = &sysctl_tcp_timer_restart,
+               .maxlen         = sizeof(int),
+               .mode           = 0644,
+               .proc_handler   = proc_dointvec_minmax,
+               .extra1         = &zero,
+               .extra2         = &three,
+       },
+       {
               .procname       = "tcp_autocorking",
               .data           = &sysctl_tcp_autocorking,
               .maxlen         = sizeof(int),
net/ipv4/tcp_input.c +13 -0
index fdd88c3..66e0425 100644
@@ -101,6 +101,7 @@ int sysctl_tcp_thin_dupack __read_mostly;
 
 int sysctl_tcp_moderate_rcvbuf __read_mostly = 1;
 int sysctl_tcp_early_retrans __read_mostly = 3;
+int sysctl_tcp_timer_restart __read_mostly = 3;
 int sysctl_tcp_invalid_ratelimit __read_mostly = HZ/2;
 
 #define FLAG_DATA              0x01 /* Incoming frame contained data.          */
@@ -2997,6 +2998,18 @@ static void tcp_cong_avoid(struct sock *sk, u32 ack, u32 acked)
       tcp_sk(sk)->snd_cwnd_stamp = tcp_time_stamp;
 }
 
+static u32 tcp_unsent_pkts(const struct sock *sk)
+{
+       struct sk_buff *skb = tcp_send_head(sk);
+       u32 pkts = 0;
+
+       if (skb)
+               tcp_for_write_queue_from(skb, sk)
+                       pkts += tcp_skb_pcount(skb);
+
+       return pkts;
+}
+
 /* Restart timer after forward progress on connection.
 * RFC2988 recommends to restart timer to now+rto.
 */

net/ipv4/tcp_input.c void tcp_rearm_rto(struct sock *sk) · line 3046

Quoted 7 lines
net/ipv4/tcp_input.c +4 -0
@@ -3027,6 +3040,17 @@ void tcp_rearm_rto(struct sock *sk)
                        */
                       if (delta > 0)
                               rto = delta;
+               } else if (icsk->icsk_pending == ICSK_TIME_RETRANS &&
+                          (sysctl_tcp_timer_restart == 1 ||
+                           sysctl_tcp_timer_restart == 3) &&
+                          (tp->packets_out + tcp_unsent_pkts(sk) <

We re-arm RTO every ACK. Do we really need a loop here to improve short-transfer RTO performance?

Quoted 6 lines

Going one level up: what's the rationale to arm RTO differently depending on the #packets in the write queue? if the idea is to offset the elapsed of the first unacked packet in RTO, it should generally apply to any packet w/ any inflight, and we can forgo another packet counting heuristic.

The rationale is to not risk being overly aggressive when there is enough packets in the write queue to allow other recovery strategies than RTO to kick in. However, this version of the patch tightly follows the IETF internet-draft of the mechanism. If people like a more aggressive approach I’m all for that, it’ll at least be a cleaner solution in terms of code :)

Quoted 5 lines

While larger inflight can trigger more fast recoveries, tail drops can happen at any time. This is why TLP is always enabled of any inflight. Sounds like we can benefit if RTO-restart is always used too? I am planning to experiment your patch w and w/o the packet count checks.

This is of course possible. We welcome any experimentation with this piece of code to make it better!

Quoted 19 lines
Quoted 12 lines
net/ipv4/tcp_input.c +7 -1
+                           TCP_RTORESTART_THRESH)) {
+                       struct sk_buff *skb = tcp_write_queue_head(sk);
+                       const u32 rto_time_stamp = tcp_skb_timestamp(skb);
+                       s32 delta = (s32)(tcp_time_stamp - rto_time_stamp);
+
+                       if (delta > 0 && rto > delta)
+                               rto -= delta;
               }
               inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS, rto,
                                         TCP_RTO_MAX);
--
Quoted 3 lines

1.9.1

--
To unsubscribe from this list: send the line "unsubscribe netdev" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

author
#7 lore raw

RFC PATCH 2/2 tcp: TLP restart (TLPR)

In reply to Per Hurtig

This patch implements the TLP restart modification (TLPR). When data is ACKed, and TLP's PTO timer is restarted, the time elapsed since the last outstanding segment was transmitted is subtracted from the calculated RTO value to not unnecessarily delay loss probes.

---
 include/net/tcp.h     |  2 +-
 net/ipv4/tcp_input.c  |  2 +-
 net/ipv4/tcp_output.c | 12 ++++++++++--
 3 files changed, 12 insertions(+), 4 deletions(-)
include/net/tcp.h +1 -1
index bf98768..8ac4118 100644
@@ -566,7 +566,7 @@ void tcp_push_one(struct sock *, unsigned int mss_now);
 void tcp_send_ack(struct sock *sk);
 void tcp_send_delayed_ack(struct sock *sk);
 void tcp_send_loss_probe(struct sock *sk);
-bool tcp_schedule_loss_probe(struct sock *sk);
+bool tcp_schedule_loss_probe(struct sock *sk, bool restart);
 
 /* tcp_input.c */
 void tcp_resume_early_retransmit(struct sock *sk);
net/ipv4/tcp_input.c +1 -1
index 66e0425..28d3b21 100644
@@ -3655,7 +3655,7 @@ static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag)
 	}
 
 	if (icsk->icsk_pending == ICSK_TIME_RETRANS)
-		tcp_schedule_loss_probe(sk);
+		tcp_schedule_loss_probe(sk, true);
 	tcp_update_pacing_rate(sk);
 	return 1;
 
net/ipv4/tcp_output.c +10 -2
index cb7ca56..752db3d 100644
@@ -2135,7 +2135,7 @@ repair:
 
 		/* Send one loss probe per tail loss episode. */
 		if (push_one != 2)
-			tcp_schedule_loss_probe(sk);
+			tcp_schedule_loss_probe(sk, false);
 		is_cwnd_limited |= (tcp_packets_in_flight(tp) >= tp->snd_cwnd);
 		tcp_cwnd_validate(sk, is_cwnd_limited);
 		return false;
@@ -2143,10 +2143,11 @@ repair:
 	return !tp->packets_out && tcp_send_head(sk);
 }
 
-bool tcp_schedule_loss_probe(struct sock *sk)
+bool tcp_schedule_loss_probe(struct sock *sk, bool restart)
 {
 	struct inet_connection_sock *icsk = inet_csk(sk);
 	struct tcp_sock *tp = tcp_sk(sk);
+	struct sk_buff *skb = tcp_write_queue_head(sk);
 	u32 timeout, tlp_time_stamp, rto_time_stamp;
 	u32 rtt = usecs_to_jiffies(tp->srtt_us >> 3);
 
@@ -2186,6 +2187,13 @@ bool tcp_schedule_loss_probe(struct sock *sk)
 	if (tp->packets_out == 1)
 		timeout = max_t(u32, timeout,
 				(rtt + (rtt >> 1) + TCP_DELACK_MAX));
+	if (sysctl_tcp_timer_restart > 1 && restart && skb) {
+		const u32 rto_time_stamp = tcp_skb_timestamp(skb);
+		s32 delta = (s32)(tcp_time_stamp - rto_time_stamp);
+
+		if (delta > 0 && timeout > delta)
+			timeout -= delta;
+	}
 	timeout = max_t(u32, timeout, msecs_to_jiffies(10));
 
 	/* If RTO is shorter, just schedule TLP in its place. */
Signature · 1 line
1.9.1
author
#8 lore raw

v2 cover tcp: timer restart for tail loss

In reply to Per Hurtig

This is a request for comments.

RTO and TLP restart is a modification to the restart process of the RTO and TLP timers in TCP. Currently, both timers are restarted with its corresponding timeout value when an acknowledgment (ACK) for correctly received data is received. In many situations, resetting the timers on incoming ACKs will add an implicit offset of at least RTT seconds to the loss recovery process.

The goal of the modified restart is to provide quicker loss recovery for segments lost in the end of a burst/connection, where the limited feedback from a receiver inhibits the use of fast/early retransmit. To accomplish this the algorithm adjusts the RTO and PTO (TLP's timer) values on each rearm of the timers to allow them to expire after exactly RTO and PTO ms, respectively.

The restart behavior is controlled by the new tcp_timer_restart sysctl. tcp_timer_restart==0; disables RTOR and TLPR.

		 ==1; enables RTOR.
		 ==2; enables TLPR.
		 ==3; enables both RTOR and TLPR. [DEFAULT]

The new restart behavior has been approved by the IETF for publication as an experimental RFC (for the RTO part of the mechanism) [1], and experiments have been conducted to show both the benefit of this strategy (in terms of reduced loss-recovery delays) and the limited negative impact (in terms of spurious retransmissions) [2]. More information regarding the mechanism can be found at [3].

Basic functionality tests, using packetdrill, are available at:
https://github.com/perhurt/packetdrill/tree/master/gtests/net/packetdrill/tests/linux/timer_restart

[1] https://datatracker.ietf.org/doc/draft-ietf-tcpm-rtorestart/
[2] http://www.sigcomm.org/sites/default/files/ccr/papers/2015/January/0000000-0000000.pdf
[3] http://riteproject.eu/resources/rto-restart/

Per Hurtig (2):
tcp: RTO Restart (RTOR)
tcp: TLP restart (TLPR)

 Documentation/networking/ip-sysctl.txt | 12 ++++++++++++
 include/net/tcp.h                      |  8 +++++++-
 net/ipv4/sysctl_net_ipv4.c             | 10 ++++++++++
 net/ipv4/tcp_input.c                   | 31 ++++++++++++++++++++++++++++++-
 net/ipv4/tcp_output.c                  | 12 ++++++++++--
 5 files changed, 69 insertions(+), 4 deletions(-)
Signature · 1 line
1.9.1
author
#9 lore raw

RFC PATCH v2 1/2 tcp: RTO Restart (RTOR)

This patch implements the RTO restart modification (RTOR). When data is ACKed, and the RTO timer is restarted, the time elapsed since the last outstanding segment was transmitted is subtracted from the calculated RTO value. This way, the RTO timer will expire after exactly RTO seconds, and not RTO + RTT [+ delACK] seconds.

This patch also implements a new sysctl (tcp_timer_restart) that is used to control the timer restart behavior.

---
 Documentation/networking/ip-sysctl.txt | 12 ++++++++++++
 include/net/tcp.h                      |  6 ++++++
 net/ipv4/sysctl_net_ipv4.c             | 10 ++++++++++
 net/ipv4/tcp_input.c                   | 29 +++++++++++++++++++++++++++++
 4 files changed, 57 insertions(+)
Documentation/networking/ip-sysctl.txt +12 -0
index 2ea4c45..4094128 100644
@@ -591,6 +591,18 @@ tcp_syn_retries - INTEGER
 	with the current initial RTO of 1second. With this the final timeout
 	for an active TCP connection attempt will happen after 127seconds.
 
+tcp_timer_restart - INTEGER
+	Controls how the RTO and PTO timers are restarted (RTOR and TLPR).
+	If set (per timer or combined) the timers are restarted with
+	respect to the earliest outstanding segment, to not extend tail loss
+	latency unnecessarily.
+	Possible values:
+		0 disables RTOR and TLPR.
+		1 enables RTOR.
+		2 enables TLPR.
+		3 enables RTOR and TLPR.
+	Default: 3
+
 tcp_timestamps - BOOLEAN
 	Enable timestamps as defined in RFC1323.
 
include/net/tcp.h +6 -0
index f80e74c..833efb7 100644
@@ -76,6 +76,11 @@ void tcp_time_wait(struct sock *sk, int state, int timeo);
 /* After receiving this amount of duplicate ACKs fast retransmit starts. */
 #define TCP_FASTRETRANS_THRESH 3
 
+/* Disable RTO Restart if the number of outstanding segments is at least. */
+#define TCP_TIMER_RTORESTART	1
+#define TCP_TIMER_TLPRESTART	2
+#define TCP_RTORESTART_THRESH	4
+
 /* Maximal number of ACKs sent quickly to accelerate slow-start. */
 #define TCP_MAX_QUICKACKS	16U
 
@@ -284,6 +289,7 @@ extern int sysctl_tcp_autocorking;
 extern int sysctl_tcp_invalid_ratelimit;
 extern int sysctl_tcp_pacing_ss_ratio;
 extern int sysctl_tcp_pacing_ca_ratio;
+extern int sysctl_tcp_timer_restart;
 
 extern atomic_long_t tcp_memory_allocated;
 extern struct percpu_counter tcp_sockets_allocated;
net/ipv4/sysctl_net_ipv4.c +10 -0
index a0bd7a5..dfb6968 100644
@@ -28,6 +28,7 @@
 
 static int zero;
 static int one = 1;
+static int three = 3;
 static int four = 4;
 static int thousand = 1000;
 static int gso_max_segs = GSO_MAX_SEGS;
@@ -745,6 +746,15 @@ static struct ctl_table ipv4_table[] = {
 		.extra2		= &thousand,
 	},
 	{
+		.procname	= "tcp_timer_restart",
+		.data		= &sysctl_tcp_timer_restart,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec_minmax,
+		.extra1		= &zero,
+		.extra2		= &three,
+	},
+	{
 		.procname	= "tcp_autocorking",
 		.data		= &sysctl_tcp_autocorking,
 		.maxlen		= sizeof(int),
net/ipv4/tcp_input.c +29 -0
index 2d656ee..2870af3 100644
@@ -101,6 +101,8 @@ int sysctl_tcp_thin_dupack __read_mostly;
 
 int sysctl_tcp_moderate_rcvbuf __read_mostly = 1;
 int sysctl_tcp_early_retrans __read_mostly = 3;
+int sysctl_tcp_timer_restart __read_mostly = TCP_TIMER_RTORESTART |
+					     TCP_TIMER_TLPRESTART;
 int sysctl_tcp_invalid_ratelimit __read_mostly = HZ/2;
 
 #define FLAG_DATA		0x01 /* Incoming frame contained data.		*/
@@ -2997,6 +2999,22 @@ static void tcp_cong_avoid(struct sock *sk, u32 ack, u32 acked)
 	tcp_sk(sk)->snd_cwnd_stamp = tcp_time_stamp;
 }
 
+static u32 tcp_unsent_pkts(const struct sock *sk, u32 ulimit)
+{
+	struct sk_buff *skb = tcp_send_head(sk);
+	u32 pkts = 0;
+
+	if (skb)
+		tcp_for_write_queue_from(skb, sk) {
+			pkts += tcp_skb_pcount(skb);
+
+			if (ulimit && pkts >= ulimit)
+				return ulimit;
+		}
+
+	return pkts;
+}
+
 /* Restart timer after forward progress on connection.
  * RFC2988 recommends to restart timer to now+rto.
  */
@@ -3027,6 +3045,17 @@ void tcp_rearm_rto(struct sock *sk)
 			 */
 			if (delta > 0)
 				rto = delta;
+		} else if (icsk->icsk_pending == ICSK_TIME_RETRANS &&
+			   (sysctl_tcp_timer_restart & TCP_TIMER_RTORESTART) &&
+			   (tp->packets_out +
+			    tcp_unsent_pkts(sk, TCP_RTORESTART_THRESH) <
+			    TCP_RTORESTART_THRESH)) {
+			struct sk_buff *skb = tcp_write_queue_head(sk);
+			const u32 rto_time_stamp = tcp_skb_timestamp(skb);
+			s32 delta = (s32)(tcp_time_stamp - rto_time_stamp);
+
+			if (delta > 0 && rto > delta)
+				rto -= delta;
 		}
 		inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS, rto,
 					  TCP_RTO_MAX);
Signature · 1 line
1.9.1
#10 lore raw

Re: [RFC PATCHv2 net-next 1/2] tcp: RTO Restart (RTOR)

Quoted 45 lines · Per Hurtig

This patch implements the RTO restart modification (RTOR). When data is ACKed, and the RTO timer is restarted, the time elapsed since the last outstanding segment was transmitted is subtracted from the calculated RTO value. This way, the RTO timer will expire after exactly RTO seconds, and not RTO + RTT [+ delACK] seconds.

This patch also implements a new sysctl (tcp_timer_restart) that is used to control the timer restart behavior.

---
 Documentation/networking/ip-sysctl.txt | 12 ++++++++++++
 include/net/tcp.h                      |  6 ++++++
 net/ipv4/sysctl_net_ipv4.c             | 10 ++++++++++
 net/ipv4/tcp_input.c                   | 29 +++++++++++++++++++++++++++++
 4 files changed, 57 insertions(+)
Documentation/networking/ip-sysctl.txt +12 -0
index 2ea4c45..4094128 100644
@@ -591,6 +591,18 @@ tcp_syn_retries - INTEGER
 	with the current initial RTO of 1second. With this the final timeout
 	for an active TCP connection attempt will happen after 127seconds.
 
+tcp_timer_restart - INTEGER
+	Controls how the RTO and PTO timers are restarted (RTOR and TLPR).
+	If set (per timer or combined) the timers are restarted with
+	respect to the earliest outstanding segment, to not extend tail loss
+	latency unnecessarily.
+	Possible values:
+		0 disables RTOR and TLPR.
+		1 enables RTOR.
+		2 enables TLPR.
+		3 enables RTOR and TLPR.
+	Default: 3
+
 tcp_timestamps - BOOLEAN
 	Enable timestamps as defined in RFC1323.
 
include/net/tcp.h +0 -0
index f80e74c..833efb7 100644

include/net/tcp.h void tcp_time_wait(struct sock *sk, int state, int timeo); · line 82

Quoted 7 lines
include/net/tcp.h +4 -0
@@ -76,6 +76,11 @@ void tcp_time_wait(struct sock *sk, int state, int timeo);
 /* After receiving this amount of duplicate ACKs fast retransmit starts. */
 #define TCP_FASTRETRANS_THRESH 3
 
+/* Disable RTO Restart if the number of outstanding segments is at least. */
+#define TCP_TIMER_RTORESTART	1
+#define TCP_TIMER_TLPRESTART	2
+#define TCP_RTORESTART_THRESH	4

Unfortunately the comment got now separated from the actual define.

Signature · 1 line
 i.
author
#11 lore raw

Re: [RFC PATCHv2 net-next 1/2] tcp: RTO Restart (RTOR)

Quoted 61 lines · Ilpo Järvinen
Quoted 45 lines · Per Hurtig

This patch implements the RTO restart modification (RTOR). When data is ACKed, and the RTO timer is restarted, the time elapsed since the last outstanding segment was transmitted is subtracted from the calculated RTO value. This way, the RTO timer will expire after exactly RTO seconds, and not RTO + RTT [+ delACK] seconds.

This patch also implements a new sysctl (tcp_timer_restart) that is used to control the timer restart behavior.

---

Documentation/networking/ip-sysctl.txt | 12 ++++++++++++
include/net/tcp.h                      |  6 ++++++
net/ipv4/sysctl_net_ipv4.c             | 10 ++++++++++
net/ipv4/tcp_input.c                   | 29 +++++++++++++++++++++++++++++
4 files changed, 57 insertions(+)
Documentation/networking/ip-sysctl.txt +12 -0
index 2ea4c45..4094128 100644
@@ -591,6 +591,18 @@ tcp_syn_retries - INTEGER
 	with the current initial RTO of 1second. With this the final timeout
 	for an active TCP connection attempt will happen after 127seconds.
 
+tcp_timer_restart - INTEGER
+	Controls how the RTO and PTO timers are restarted (RTOR and TLPR).
+	If set (per timer or combined) the timers are restarted with
+	respect to the earliest outstanding segment, to not extend tail loss
+	latency unnecessarily.
+	Possible values:
+		0 disables RTOR and TLPR.
+		1 enables RTOR.
+		2 enables TLPR.
+		3 enables RTOR and TLPR.
+	Default: 3
+

tcp_timestamps - BOOLEAN
Enable timestamps as defined in RFC1323.

include/net/tcp.h +0 -0
index f80e74c..833efb7 100644

include/net/tcp.h void tcp_time_wait(struct sock *sk, int state, int timeo); · line 82

Quoted 7 lines
include/net/tcp.h +4 -0
@@ -76,6 +76,11 @@ void tcp_time_wait(struct sock *sk, int state, int timeo);
 /* After receiving this amount of duplicate ACKs fast retransmit starts. */
 #define TCP_FASTRETRANS_THRESH 3
 
+/* Disable RTO Restart if the number of outstanding segments is at least. */
+#define TCP_TIMER_RTORESTART	1
+#define TCP_TIMER_TLPRESTART	2
+#define TCP_RTORESTART_THRESH	4

Unfortunately the comment got now separated from the actual define.

Quoted 2 lines

--
i.

Darn, I knew I missed something. Well, I’ll fix that in the next round. Suppose there are more things that could be improved.

Per

#12 lore raw

Re: [RFC PATCHv2 net-next 1/2] tcp: RTO Restart (RTOR)

In reply to Per Hurtig
Quoted 5 lines · Per Hurtig
net/ipv4/tcp_input.c +5 -0
+static u32 tcp_unsent_pkts(const struct sock *sk, u32 ulimit)
+{
+	struct sk_buff *skb = tcp_send_head(sk);
+	u32 pkts = 0;
+

net/ipv4/tcp_input.c static void tcp_cong_avoid(struct sock *sk, u32 ack, u32 acked) · line 3016

Quoted 10 lines
net/ipv4/tcp_input.c +10 -0
+	if (skb)
+		tcp_for_write_queue_from(skb, sk) {
+			pkts += tcp_skb_pcount(skb);
+
+			if (ulimit && pkts >= ulimit)
+				return ulimit;
+		}
+
+	return pkts;
+}

Considering Yuchung feedback, have you looked at using an approximation instead ?

(ie using tp->write_seq - tp->snd_nxt)

author
#13 lore raw

Re: [RFC PATCHv2 net-next 1/2] tcp: RTO Restart (RTOR)

Quoted 25 lines · Eric Dumazet
Quoted 5 lines · Per Hurtig
net/ipv4/tcp_input.c +5 -0
+static u32 tcp_unsent_pkts(const struct sock *sk, u32 ulimit)
+{
+	struct sk_buff *skb = tcp_send_head(sk);
+	u32 pkts = 0;
+

net/ipv4/tcp_input.c static void tcp_cong_avoid(struct sock *sk, u32 ack, u32 acked) · line 3016

Quoted 10 lines
net/ipv4/tcp_input.c +10 -0
+	if (skb)
+		tcp_for_write_queue_from(skb, sk) {
+			pkts += tcp_skb_pcount(skb);
+
+			if (ulimit && pkts >= ulimit)
+				return ulimit;
+		}
+
+	return pkts;
+}

Considering Yuchung feedback, have you looked at using an approximation instead ?

Quoted 3 lines

(ie using tp->write_seq - tp->snd_nxt)

Well, an approximation is rather “dangerous” as missing a single packet could inhibit the desired behaviour. If looping is undesired, I think a better solution is to actually *not* do this check at all and instead rely solely on the

tp->packets_out < TCP_RTORESTART_THRESH

check instead. The reason why the number of unsent packets was included was only to fix a corner case where it should be possible to use the modified restart, but impossible due to the conditioning.

However, this corner case is likely to not occur very often and we may be better off with the simpler check.

The corner case (if I remember this correctly) is that the restart is not triggered when you have 2 segments in flight and (i) have a congestion window of exactly 3; or (ii) get a packet written to the socket just between previous data transmission and the arrival of the acknowledgment that triggers the restart.

— Per

maintainer
#14 lore raw

Re: [RFC PATCHv2 net-next 1/2] tcp: RTO Restart (RTOR)

Quoted 29 lines · Per Hurtig
Quoted 25 lines · Eric Dumazet
Quoted 5 lines · Per Hurtig
net/ipv4/tcp_input.c +5 -0
+static u32 tcp_unsent_pkts(const struct sock *sk, u32 ulimit)
+{
+    struct sk_buff *skb = tcp_send_head(sk);
+    u32 pkts = 0;
+

net/ipv4/tcp_input.c static void tcp_cong_avoid(struct sock *sk, u32 ack, u32 acked) · line 3016

Quoted 10 lines
net/ipv4/tcp_input.c +10 -0
+    if (skb)
+            tcp_for_write_queue_from(skb, sk) {
+                    pkts += tcp_skb_pcount(skb);
+
+                    if (ulimit && pkts >= ulimit)
+                            return ulimit;
+            }
+
+    return pkts;
+}

Considering Yuchung feedback, have you looked at using an approximation instead ?

Quoted 3 lines

(ie using tp->write_seq - tp->snd_nxt)

Quoted 6 lines

Well, an approximation is rather “dangerous” as missing a single packet could inhibit the desired behaviour. If looping is undesired, I think a better solution is to actually *not* do this check at all and instead rely solely on the

tp->packets_out < TCP_RTORESTART_THRESH

Yes, this simpler version seems very much preferable, IMHO. I agree that it does not seem worth the complexity to try to cover the kind of corner cases you outline.

I would also suggest a TCP_RTORESTART_THRESH value higher than 4.

In the ID at https://tools.ietf.org/html/draft-ietf-tcpm-rtorestart-10 it says:

The RECOMMENDED value of rrthresh is four, as this value will ensure that RTOR is only used when fast retransmit cannot be triggered.

But my sense is that fast retransmit is often not triggered at in-flight counts of much higher than 4, due to drop-tail queues, TSO bursts, the initial IW10 being unpaced, etc. It would be interesting to see A/B experiments for a few TCP_RTORESTART_THRESH values, say, 4 vs 10.

neal

author
#15 lore raw

Re: [RFC PATCHv2 net-next 1/2] tcp: RTO Restart (RTOR)

Quoted 56 lines

10 dec. 2015 kl. 16:37 skrev Neal Cardwell <[email protected]>:

Quoted 26 lines · Per Hurtig
Quoted 25 lines · Eric Dumazet
Quoted 5 lines · Per Hurtig
net/ipv4/tcp_input.c +5 -0
+static u32 tcp_unsent_pkts(const struct sock *sk, u32 ulimit)
+{
+    struct sk_buff *skb = tcp_send_head(sk);
+    u32 pkts = 0;
+

net/ipv4/tcp_input.c static void tcp_cong_avoid(struct sock *sk, u32 ack, u32 acked) · line 3016

Quoted 10 lines
net/ipv4/tcp_input.c +10 -0
+    if (skb)
+            tcp_for_write_queue_from(skb, sk) {
+                    pkts += tcp_skb_pcount(skb);
+
+                    if (ulimit && pkts >= ulimit)
+                            return ulimit;
+            }
+
+    return pkts;
+}

Considering Yuchung feedback, have you looked at using an approximation instead ?

Quoted 8 lines
Quoted 1 line

(ie using tp->write_seq - tp->snd_nxt)

Well, an approximation is rather “dangerous” as missing a single packet could inhibit the desired behaviour. If looping is undesired, I think a better solution is to actually *not* do this check at all and instead rely solely on the

tp->packets_out < TCP_RTORESTART_THRESH

Yes, this simpler version seems very much preferable, IMHO. I agree that it does not seem worth the complexity to try to cover the kind of corner cases you outline.

I would also suggest a TCP_RTORESTART_THRESH value higher than 4.

In the ID at https://tools.ietf.org/html/draft-ietf-tcpm-rtorestart-10 it says:

The RECOMMENDED value of rrthresh is four, as this value will ensure that RTOR is only used when fast retransmit cannot be triggered.

But my sense is that fast retransmit is often not triggered at in-flight counts of much higher than 4, due to drop-tail queues, TSO bursts, the initial IW10 being unpaced, etc. It would be interesting to see A/B experiments for a few TCP_RTORESTART_THRESH values, say, 4 vs 10.

Quoted 5 lines

neal
--
To unsubscribe from this list: send the line "unsubscribe netdev" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

Sure. One idea could also be to use the "reordering" value as a dynamic threshhold?

-- Per

author
#16 lore raw

RFC PATCH v2 2/2 tcp: TLP restart (TLPR)

In reply to Per Hurtig

This patch implements the TLP restart modification (TLPR). When data is ACKed, and TLP's PTO timer is restarted, the time elapsed since the last outstanding segment was transmitted is subtracted from the calculated RTO value to not unnecessarily delay loss probes.

---
 include/net/tcp.h     |  2 +-
 net/ipv4/tcp_input.c  |  2 +-
 net/ipv4/tcp_output.c | 12 ++++++++++--
 3 files changed, 12 insertions(+), 4 deletions(-)
include/net/tcp.h +1 -1
index 833efb7..14c0888 100644
@@ -568,7 +568,7 @@ void tcp_push_one(struct sock *, unsigned int mss_now);
 void tcp_send_ack(struct sock *sk);
 void tcp_send_delayed_ack(struct sock *sk);
 void tcp_send_loss_probe(struct sock *sk);
-bool tcp_schedule_loss_probe(struct sock *sk);
+bool tcp_schedule_loss_probe(struct sock *sk, bool restart);
 
 /* tcp_input.c */
 void tcp_resume_early_retransmit(struct sock *sk);
net/ipv4/tcp_input.c +1 -1
index 2870af3..952c4cd 100644
@@ -3660,7 +3660,7 @@ static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag)
 	}
 
 	if (icsk->icsk_pending == ICSK_TIME_RETRANS)
-		tcp_schedule_loss_probe(sk);
+		tcp_schedule_loss_probe(sk, true);
 	tcp_update_pacing_rate(sk);
 	return 1;
 
net/ipv4/tcp_output.c +10 -2
index a800cee..b89ba50 100644
@@ -2135,7 +2135,7 @@ repair:
 
 		/* Send one loss probe per tail loss episode. */
 		if (push_one != 2)
-			tcp_schedule_loss_probe(sk);
+			tcp_schedule_loss_probe(sk, false);
 		is_cwnd_limited |= (tcp_packets_in_flight(tp) >= tp->snd_cwnd);
 		tcp_cwnd_validate(sk, is_cwnd_limited);
 		return false;
@@ -2143,10 +2143,11 @@ repair:
 	return !tp->packets_out && tcp_send_head(sk);
 }
 
-bool tcp_schedule_loss_probe(struct sock *sk)
+bool tcp_schedule_loss_probe(struct sock *sk, bool restart)
 {
 	struct inet_connection_sock *icsk = inet_csk(sk);
 	struct tcp_sock *tp = tcp_sk(sk);
+	struct sk_buff *skb = tcp_write_queue_head(sk);
 	u32 timeout, tlp_time_stamp, rto_time_stamp;
 	u32 rtt = usecs_to_jiffies(tp->srtt_us >> 3);
 
@@ -2186,6 +2187,13 @@ bool tcp_schedule_loss_probe(struct sock *sk)
 	if (tp->packets_out == 1)
 		timeout = max_t(u32, timeout,
 				(rtt + (rtt >> 1) + TCP_DELACK_MAX));
+	if (sysctl_tcp_timer_restart & TCP_TIMER_TLPRESTART && restart && skb) {
+		const u32 rto_time_stamp = tcp_skb_timestamp(skb);
+		s32 delta = (s32)(tcp_time_stamp - rto_time_stamp);
+
+		if (delta > 0 && timeout > delta)
+			timeout -= delta;
+	}
 	timeout = max_t(u32, timeout, msecs_to_jiffies(10));
 
 	/* If RTO is shorter, just schedule TLP in its place. */
Signature · 1 line
1.9.1