tcp: remove two indirect calls from xmit path
Subsystems (4)
cover tcp: remove two indirect calls from xmit path
__tcp_transmit_skb() does two indirect calls per packet, lets get rid of them.
Eric Dumazet (2):
tcp: remove indirect calls for icsk->icsk_af_ops->queue_xmit
tcp: remove indirect calls for icsk->icsk_af_ops->send_check
include/net/ip.h | 6 +----- include/net/ip6_checksum.h | 9 --------- include/net/tcp.h | 4 ++++ net/ipv4/ip_output.c | 6 ++++++ net/ipv4/tcp_output.c | 12 ++++++++++-- net/ipv6/tcp_ipv6.c | 7 +++++++ 6 files changed, 28 insertions(+), 16 deletions(-)
Signature · 1 line
2.27.0.111.gc72c7da667-goog
PATCH 1/2 tcp: remove indirect calls for icsk->icsk_af_ops->queue_xmit
Mitigate RETPOLINE costs in __tcp_transmit_skb()
by using INDIRECT_CALL_INET() wrapper.
--- include/net/ip.h | 6 +----- include/net/tcp.h | 1 + net/ipv4/ip_output.c | 6 ++++++ net/ipv4/tcp_output.c | 7 ++++++- 4 files changed, 14 insertions(+), 6 deletions(-)
include/net/ip.h +1 -5
@@ -231,11 +231,7 @@ struct sk_buff *ip_make_skb(struct sock *sk, struct flowi4 *fl4, struct ipcm_cookie *ipc, struct rtable **rtp, struct inet_cork *cork, unsigned int flags); -static inline int ip_queue_xmit (struct sock *sk,struct sk_buff *skb, -struct flowi *fl) -{ -return __ip_queue_xmit (sk, skb, fl,inet_sk (sk)->tos); -} +int ip_queue_xmit (struct sock *sk,struct sk_buff *skb,struct flowi *fl);static inline struct sk_buff *ip_finish_skb (struct sock *sk,struct flowi4 *fl4) {
include/net/tcp.h +1 -0
@@ -25,6 +25,7 @@#include <linux/skbuff.h> #include <linux/kref.h> #include <linux/ktime.h> +#include <linux/indirect_call_wrapper.h> #include <net/inet_connection_sock.h> #include <net/inet_timewait_sock.h>
net/ipv4/ip_output.c +6 -0
@@ -539,6 +539,12 @@ int __ip_queue_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl, }EXPORT_SYMBOL (__ip_queue_xmit); +int ip_queue_xmit (struct sock *sk,struct sk_buff *skb,struct flowi *fl) +{ +return __ip_queue_xmit (sk, skb, fl,inet_sk (sk)->tos); +} +EXPORT_SYMBOL (ip_queue_xmit); +static void ip_copy_metadata (struct sk_buff *to,struct sk_buff *from) { to->pkt_type = from->pkt_type;
net/ipv4/tcp_output.c +6 -1
@@ -1064,6 +1064,9 @@ static void tcp_update_skb_after_send(struct sock *sk, struct sk_buff *skb,list_move_tail (&skb->tcp_tsorted_anchor, &tp->tsorted_sent_queue); } +INDIRECT_CALLABLE_DECLARE(int ip_queue_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl)); +INDIRECT_CALLABLE_DECLARE(int inet6_csk_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl)); +/* This routine actually transmits TCP packets queued in by * tcp_do_sendmsg(). This is used by both the initial * transmission and possible later retransmissions.
@@ -1235,7 +1238,9 @@ static int __tcp_transmit_skb(struct sock *sk, struct sk_buff *skb,tcp_add_tx_delay (skb, tp); - err = icsk->icsk_af_ops->queue_xmit (sk, skb, &inet->cork.fl); + err =INDIRECT_CALL_INET (icsk->icsk_af_ops->queue_xmit, + inet6_csk_xmit, ip_queue_xmit, + sk, skb, &inet->cork.fl);if (unlikely (err >0 )) {tcp_enter_cwr (sk);
Signature · 1 line
2.27.0.111.gc72c7da667-goog
PATCH 2/2 tcp: remove indirect calls for icsk->icsk_af_ops->send_check
In reply to Eric DumazetMitigate RETPOLINE costs in __tcp_transmit_skb()
by using INDIRECT_CALL_INET() wrapper.
--- include/net/ip6_checksum.h | 9 --------- include/net/tcp.h | 3 +++ net/ipv4/tcp_output.c | 5 ++++- net/ipv6/tcp_ipv6.c | 7 +++++++ 4 files changed, 14 insertions(+), 10 deletions(-)
include/net/ip6_checksum.h +0 -9
@@ -85,15 +85,6 @@ static inline void tcp_v6_gso_csum_prep(struct sk_buff *skb) th->check = ~tcp_v6_check (0 , &ipv6h->saddr, &ipv6h->daddr,0 ); } -#if IS_ENABLED (CONFIG_IPV6 ) -static inline void tcp_v6_send_check (struct sock *sk,struct sk_buff *skb) -{ -struct ipv6_pinfo *np =inet6_sk (sk); - -__tcp_v6_send_check (skb, &np->saddr, &sk->sk_v6_daddr); -} -#endif - static inline __sum16 udp_v6_check(int len, const struct in6_addr *saddr, const struct in6_addr *daddr,
include/net/tcp.h +3 -0
@@ -932,6 +932,9 @@ static inline int tcp_v6_sdif(const struct sk_buff *skb)#endif return 0 ; } + +INDIRECT_CALLABLE_DECLARE(void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb)); +#endif static inline bool inet_exact_dif_match (struct net *net,struct sk_buff *skb)
net/ipv4/tcp_output.c +4 -1
@@ -1066,6 +1066,7 @@ static void tcp_update_skb_after_send(struct sock *sk, struct sk_buff *skb, INDIRECT_CALLABLE_DECLARE(int ip_queue_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl)); INDIRECT_CALLABLE_DECLARE(int inet6_csk_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl)); +INDIRECT_CALLABLE_DECLARE(void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb));/* This routine actually transmits TCP packets queued in by * tcp_do_sendmsg(). This is used by both the initial
@@ -1210,7 +1211,9 @@ static int __tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, }#endif - icsk->icsk_af_ops->send_check (sk, skb); +INDIRECT_CALL_INET (icsk->icsk_af_ops->send_check, + tcp_v6_send_check, tcp_v4_send_check, + sk, skb);if (likely (tcb->tcp_flags &TCPHDR_ACK ))tcp_event_ack_sent (sk,tcp_skb_pcount (skb), rcv_nxt);
net/ipv6/tcp_ipv6.c +7 -0
@@ -1811,6 +1811,13 @@ static struct timewait_sock_ops tcp6_timewait_sock_ops = { .twsk_destructor = tcp_twsk_destructor, }; +INDIRECT_CALLABLE_SCOPE void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb) +{ +struct ipv6_pinfo *np =inet6_sk (sk); + +__tcp_v6_send_check (skb, &np->saddr, &sk->sk_v6_daddr); +} +const struct inet_connection_sock_af_ops ipv6_specific = { .queue_xmit = inet6_csk_xmit, .send_check = tcp_v6_send_check,
Signature · 1 line
2.27.0.111.gc72c7da667-goog
David Miller
maintainerFrom: Eric Dumazet <[email protected]>
Date: Fri, 19 Jun 2020 12:12:33 -0700
Quoted 2 lines
__tcp_transmit_skb()does two indirect calls per packet, lets get rid of them.
Series applied, thanks.