tcp: exponential backoff in tcp_send_ack()
Subsystems (3)
cover tcp: exponential backoff in tcp_send_ack()
From: Eric Dumazet <[email protected]>
We had outages caused by repeated skb allocation failures in tcp_send_ack()
It is time to add exponential backoff to reduce number of attempts. Before doing so, first patch removes icsk_ack.blocked to make room for a new field (icsk_ack.retry)
Eric Dumazet (2):
inet: remove icsk_ack.blocked
tcp: add exponential backoff in __tcp_send_ack()
include/net/inet_connection_sock.h | 5 +++-- net/dccp/timer.c | 1 - net/ipv4/inet_connection_sock.c | 2 +- net/ipv4/tcp.c | 6 ++---- net/ipv4/tcp_output.c | 18 ++++++++++-------- net/ipv4/tcp_timer.c | 1 - 6 files changed, 16 insertions(+), 17 deletions(-)
Signature · 1 line
2.28.0.806.g8561365e88-goog
PATCH 1/2 inet: remove icsk_ack.blocked
From: Eric Dumazet <[email protected]>
TCP has been using it to work around the possibility of tcp_delack_timer() finding the socket owned by user.
After commit 6f458dfb4092 ("tcp: improve latencies of timer triggered events") we added TCP_DELACK_TIMER_DEFERRED atomic bit for more immediate recovery, so we can get rid of icsk_ack.blocked
This frees space that following patch will reuse.
--- include/net/inet_connection_sock.h | 4 ++-- net/dccp/timer.c | 1 - net/ipv4/inet_connection_sock.c | 2 +- net/ipv4/tcp.c | 6 ++---- net/ipv4/tcp_output.c | 7 ++----- net/ipv4/tcp_timer.c | 1 - 6 files changed, 7 insertions(+), 14 deletions(-)
include/net/inet_connection_sock.h +2 -2
@@ -110,7 +110,7 @@ struct inet_connection_sock {__u8 pending;/* ACK is pending */ __u8 quick;/* Scheduled number of quick acks */ __u8 pingpong;/* The session is interactive */ -__u8 blocked;/* Delayed ACK was blocked by socket lock */ +/* one byte hole. */ __u32 ato;/* Predicted tick of soft clock */ unsigned long timeout;/* Currently scheduled timeout */ __u32 lrcvtime;/* timestamp of last received data packet */
@@ -198,7 +198,7 @@ static inline void inet_csk_clear_xmit_timer(struct sock *sk, const int what)sk_stop_timer (sk, &icsk->icsk_retransmit_timer);#endif }else if (what ==ICSK_TIME_DACK ) { - icsk->icsk_ack.blocked = icsk->icsk_ack.pending =0 ; + icsk->icsk_ack.pending =0 ;#ifdef INET_CSK_CLEAR_TIMERS sk_stop_timer (sk, &icsk->icsk_delack_timer);#endif
net/dccp/timer.c +0 -1
@@ -176,7 +176,6 @@ static void dccp_delack_timer(struct timer_list *t)bh_lock_sock (sk);if (sock_owned_by_user (sk)) {/* Try again later. */ - icsk->icsk_ack.blocked =1 ;__NET_INC_STATS (sock_net (sk),LINUX_MIB_DELAYEDACKLOCKED );sk_reset_timer (sk, &icsk->icsk_delack_timer, jiffies +TCP_DELACK_MIN );
net/ipv4/inet_connection_sock.c +1 -1
@@ -564,7 +564,7 @@ void inet_csk_clear_xmit_timers(struct sock *sk) {struct inet_connection_sock *icsk =inet_csk (sk); - icsk->icsk_pending = icsk->icsk_ack.pending = icsk->icsk_ack.blocked =0 ; + icsk->icsk_pending = icsk->icsk_ack.pending =0 ;sk_stop_timer (sk, &icsk->icsk_retransmit_timer);sk_stop_timer (sk, &icsk->icsk_delack_timer);
net/ipv4/tcp.c +2 -4
@@ -1538,10 +1538,8 @@ void tcp_cleanup_rbuf(struct sock *sk, int copied)if (inet_csk_ack_scheduled (sk)) {const struct inet_connection_sock *icsk =inet_csk (sk); -/* Delayed ACKs frequently hit locked sockets during bulk -* receive. */ - if (icsk->icsk_ack.blocked || - /* Once-per-two-segments ACK was not sent by tcp_input.c */ + + if (/* Once-per-two-segments ACK was not sent by tcp_input.c */ tp->rcv_nxt - tp->rcv_wup > icsk->icsk_ack.rcv_mss ||/* * If this read emptied read buffer, we send ACK, if
net/ipv4/tcp_output.c +2 -5
@@ -3911,11 +3911,8 @@ void tcp_send_delayed_ack(struct sock *sk)/* Use new timeout only if there wasn't a older one earlier. */ if (icsk->icsk_ack.pending &ICSK_ACK_TIMER ) { -/* If delack timer was blocked or is about to expire, -* send ACK now. -*/ -if (icsk->icsk_ack.blocked || -time_before_eq (icsk->icsk_ack.timeout, jiffies + (ato >>2 ))) { +/* If delack timer is about to expire, send ACK now. */ +if (time_before_eq (icsk->icsk_ack.timeout, jiffies + (ato >>2 ))) {tcp_send_ack (sk);return ; }
net/ipv4/tcp_timer.c +0 -1
@@ -331,7 +331,6 @@ static void tcp_delack_timer(struct timer_list *t)if (!sock_owned_by_user (sk)) {tcp_delack_timer_handler (sk); }else { - icsk->icsk_ack.blocked =1 ;__NET_INC_STATS (sock_net (sk),LINUX_MIB_DELAYEDACKLOCKED );/* deleguate our work to tcp_release_cb() */ if (!test_and_set_bit (TCP_DELACK_TIMER_DEFERRED , &sk->sk_tsq_flags))
Signature · 1 line
2.28.0.806.g8561365e88-goog
Re: [PATCH net-next 1/2] inet: remove icsk_ack.blocked
Quoted 6 lines · Eric Dumazet
From: Eric Dumazet <[email protected]>
TCP has been using it to work around the possibility of
tcp_delack_timer()finding the socket owned by user.
Quoted 7 lines
After commit
6f458dfb4092("tcp: improve latencies of timer triggered events") we addedTCP_DELACK_TIMER_DEFERREDatomic bit for more immediate recovery, so we can get rid of icsk_ack.blockedThis frees space that following patch will reuse.
Signed-off-by Eric Dumazet <[email protected]>
Quoted 106 lines
--- include/net/inet_connection_sock.h | 4 ++-- net/dccp/timer.c | 1 - net/ipv4/inet_connection_sock.c | 2 +- net/ipv4/tcp.c | 6 ++---- net/ipv4/tcp_output.c | 7 ++----- net/ipv4/tcp_timer.c | 1 - 6 files changed, 7 insertions(+), 14 deletions(-)include/net/inet_connection_sock.h +2
-2@@ -110,7 +110,7 @@ struct inet_connection_sock {__u8 pending;/* ACK is pending */ __u8 quick;/* Scheduled number of quick acks */ __u8 pingpong;/* The session is interactive */ -__u8 blocked;/* Delayed ACK was blocked by socket lock */ +/* one byte hole. */ __u32 ato;/* Predicted tick of soft clock */ unsigned long timeout;/* Currently scheduled timeout */ __u32 lrcvtime;/* timestamp of last received data packet */ @@ -198,7 +198,7 @@ static inline void inet_csk_clear_xmit_timer(struct sock *sk, const int what)sk_stop_timer (sk, &icsk->icsk_retransmit_timer);#endif }else if (what ==ICSK_TIME_DACK ) { - icsk->icsk_ack.blocked = icsk->icsk_ack.pending =0 ; + icsk->icsk_ack.pending =0 ;#ifdef INET_CSK_CLEAR_TIMERS sk_stop_timer (sk, &icsk->icsk_delack_timer);#endif net/dccp/timer.c +0
-1@@ -176,7 +176,6 @@ static void dccp_delack_timer(struct timer_list *t)bh_lock_sock (sk);if (sock_owned_by_user (sk)) {/* Try again later. */ - icsk->icsk_ack.blocked =1 ;__NET_INC_STATS (sock_net (sk),LINUX_MIB_DELAYEDACKLOCKED );sk_reset_timer (sk, &icsk->icsk_delack_timer, jiffies +TCP_DELACK_MIN );net/ipv4/inet_connection_sock.c +1
-1@@ -564,7 +564,7 @@ void inet_csk_clear_xmit_timers(struct sock *sk) {struct inet_connection_sock *icsk =inet_csk (sk); - icsk->icsk_pending = icsk->icsk_ack.pending = icsk->icsk_ack.blocked =0 ; + icsk->icsk_pending = icsk->icsk_ack.pending =0 ;sk_stop_timer (sk, &icsk->icsk_retransmit_timer);sk_stop_timer (sk, &icsk->icsk_delack_timer);net/ipv4/tcp.c +2
-4@@ -1538,10 +1538,8 @@ void tcp_cleanup_rbuf(struct sock *sk, int copied)if (inet_csk_ack_scheduled (sk)) {const struct inet_connection_sock *icsk =inet_csk (sk); -/* Delayed ACKs frequently hit locked sockets during bulk -* receive. */ - if (icsk->icsk_ack.blocked || - /* Once-per-two-segments ACK was not sent by tcp_input.c */ + + if (/* Once-per-two-segments ACK was not sent by tcp_input.c */ tp->rcv_nxt - tp->rcv_wup > icsk->icsk_ack.rcv_mss ||/* * If this read emptied read buffer, we send ACK, if net/ipv4/tcp_output.c +2
-5@@ -3911,11 +3911,8 @@ void tcp_send_delayed_ack(struct sock *sk)/* Use new timeout only if there wasn't a older one earlier. */ if (icsk->icsk_ack.pending &ICSK_ACK_TIMER ) { -/* If delack timer was blocked or is about to expire, -* send ACK now. -*/ -if (icsk->icsk_ack.blocked || -time_before_eq (icsk->icsk_ack.timeout, jiffies + (ato >>2 ))) { +/* If delack timer is about to expire, send ACK now. */ +if (time_before_eq (icsk->icsk_ack.timeout, jiffies + (ato >>2 ))) {tcp_send_ack (sk);return ; }net/ipv4/tcp_timer.c +0
-2@@ -331,7 +331,6 @@ static void tcp_delack_timer(struct timer_list *t)if (!sock_owned_by_user (sk)) {tcp_delack_timer_handler (sk); }else { - icsk->icsk_ack.blocked =1 ;__NET_INC_STATS (sock_net (sk),LINUX_MIB_DELAYEDACKLOCKED );/* deleguate our work to tcp_release_cb() */ if (!test_and_set_bit (TCP_DELACK_TIMER_DEFERRED , &sk->sk_tsq_flags)) --2.28.0.806.g8561365e88-goog
PATCH 2/2 tcp: add exponential backoff in __tcp_send_ack()
In reply to Eric DumazetFrom: Eric Dumazet <[email protected]>
Whenever host is under very high memory pressure, __tcp_send_ack() skb allocation fails, and we setup a 200 ms (TCP_DELACK_MAX) timer before retrying.
On hosts with high number of TCP sockets, we can spend considerable amount of cpu cycles in these attempts, add high pressure on various spinlocks in mm-layer, ultimately blocking threads attempting to free space from making any progress.
This patch adds standard exponential backoff to avoid adding fuel to the fire.
--- include/net/inet_connection_sock.h | 3 ++- net/ipv4/tcp_output.c | 11 ++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-)
include/net/inet_connection_sock.h +2 -1
@@ -110,7 +110,7 @@ struct inet_connection_sock {__u8 pending;/* ACK is pending */ __u8 quick;/* Scheduled number of quick acks */ __u8 pingpong;/* The session is interactive */ -/* one byte hole. */ +__u8 retry;/* Number of attempts */ __u32 ato;/* Predicted tick of soft clock */ unsigned long timeout;/* Currently scheduled timeout */ __u32 lrcvtime;/* timestamp of last received data packet */
@@ -199,6 +199,7 @@ static inline void inet_csk_clear_xmit_timer(struct sock *sk, const int what)#endif }else if (what ==ICSK_TIME_DACK ) { icsk->icsk_ack.pending =0 ; + icsk->icsk_ack.retry =0 ;#ifdef INET_CSK_CLEAR_TIMERS sk_stop_timer (sk, &icsk->icsk_delack_timer);#endif
net/ipv4/tcp_output.c +8 -3
@@ -3941,10 +3941,15 @@ void __tcp_send_ack(struct sock *sk, u32 rcv_nxt) buff =alloc_skb (MAX_TCP_HEADER ,sk_gfp_mask (sk,GFP_ATOMIC | __GFP_NOWARN));if (unlikely (!buff)) { +struct inet_connection_sock *icsk =inet_csk (sk); +unsigned long delay; + + delay =TCP_DELACK_MAX << icsk->icsk_ack.retry; +if (delay <TCP_RTO_MAX ) + icsk->icsk_ack.retry++;inet_csk_schedule_ack (sk); -inet_csk (sk)->icsk_ack.ato =TCP_ATO_MIN ; -inet_csk_reset_xmit_timer (sk,ICSK_TIME_DACK , -TCP_DELACK_MAX ,TCP_RTO_MAX ); + icsk->icsk_ack.ato =TCP_ATO_MIN ; +inet_csk_reset_xmit_timer (sk,ICSK_TIME_DACK , delay,TCP_RTO_MAX );return ; }
Signature · 1 line
2.28.0.806.g8561365e88-goog
Re: [PATCH net-next 2/2] tcp: add exponential backoff in __tcp_send_ack()
Quoted 13 lines · Eric Dumazet
From: Eric Dumazet <[email protected]>
Whenever host is under very high memory pressure,
__tcp_send_ack()skb allocation fails, and we setup a 200 ms (TCP_DELACK_MAX) timer before retrying.On hosts with high number of TCP sockets, we can spend considerable amount of cpu cycles in these attempts, add high pressure on various spinlocks in mm-layer, ultimately blocking threads attempting to free space from making any progress.
Quoted 4 lines
This patch adds standard exponential backoff to avoid adding fuel to the fire.
Signed-off-by Eric Dumazet <[email protected]>
Quoted 52 lines
--- include/net/inet_connection_sock.h | 3 ++- net/ipv4/tcp_output.c | 11 ++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-)include/net/inet_connection_sock.h +2
-1@@ -110,7 +110,7 @@ struct inet_connection_sock {__u8 pending;/* ACK is pending */ __u8 quick;/* Scheduled number of quick acks */ __u8 pingpong;/* The session is interactive */ -/* one byte hole. */ +__u8 retry;/* Number of attempts */ __u32 ato;/* Predicted tick of soft clock */ unsigned long timeout;/* Currently scheduled timeout */ __u32 lrcvtime;/* timestamp of last received data packet */ @@ -199,6 +199,7 @@ static inline void inet_csk_clear_xmit_timer(struct sock *sk, const int what)#endif }else if (what ==ICSK_TIME_DACK ) { icsk->icsk_ack.pending =0 ; + icsk->icsk_ack.retry =0 ;#ifdef INET_CSK_CLEAR_TIMERS sk_stop_timer (sk, &icsk->icsk_delack_timer);#endif net/ipv4/tcp_output.c +8
-4@@ -3941,10 +3941,15 @@ void __tcp_send_ack(struct sock *sk, u32 rcv_nxt) buff =alloc_skb (MAX_TCP_HEADER ,sk_gfp_mask (sk,GFP_ATOMIC | __GFP_NOWARN));if (unlikely (!buff)) { +struct inet_connection_sock *icsk =inet_csk (sk); +unsigned long delay; + + delay =TCP_DELACK_MAX << icsk->icsk_ack.retry; +if (delay <TCP_RTO_MAX ) + icsk->icsk_ack.retry++;inet_csk_schedule_ack (sk); -inet_csk (sk)->icsk_ack.ato =TCP_ATO_MIN ; -inet_csk_reset_xmit_timer (sk,ICSK_TIME_DACK , -TCP_DELACK_MAX ,TCP_RTO_MAX ); + icsk->icsk_ack.ato =TCP_ATO_MIN ; +inet_csk_reset_xmit_timer (sk,ICSK_TIME_DACK , delay,TCP_RTO_MAX );return ; } --2.28.0.806.g8561365e88-goog
David Miller
maintainerFrom: Eric Dumazet <[email protected]>
Date: Wed, 30 Sep 2020 05:54:55 -0700
Quoted 5 lines
We had outages caused by repeated skb allocation failures in
tcp_send_ack()It is time to add exponential backoff to reduce number of attempts. Before doing so, first patch removes icsk_ack.blocked to make room for a new field (icsk_ack.retry)
Series applied, thanks Eric.