tcp: avoid indirect call in __sk_stream_memory_free()
Subsystems (4)
cover tcp: avoid indirect call in __sk_stream_memory_free()
From: Eric Dumazet <[email protected]>
Small improvement for CONFIG_RETPOLINE=y, when dealing with TCP sockets.
Eric Dumazet (2):
tcp: uninline tcp_stream_memory_free()
tcp: avoid indirect call to tcp_stream_memory_free()
include/net/sock.h | 8 ++++++-- include/net/tcp.h | 13 +------------ net/ipv4/tcp_ipv4.c | 14 ++++++++++++++ 3 files changed, 21 insertions(+), 14 deletions(-)
Signature · 1 line
2.29.2.299.gdc1121823c-goog
PATCH 1/2 tcp: uninline tcp_stream_memory_free()
From: Eric Dumazet <[email protected]>
Both IPv4 and IPv6 needs it via a function pointer. Following patch will avoid the indirect call.
Signed-off-by Eric Dumazet <[email protected]>
--- include/net/tcp.h | 13 +------------ net/ipv4/tcp_ipv4.c | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 12 deletions(-)
include/net/tcp.h +1 -12
@@ -1965,18 +1965,7 @@ static inline u32 tcp_notsent_lowat(const struct tcp_sock *tp)return tp->notsent_lowat ?: net->ipv4.sysctl_tcp_notsent_lowat; } -/* @wake is one when sk_stream_write_space() calls us. -* This sends EPOLLOUT only if notsent_bytes is half the limit. -* This mimics the strategy used in sock_def_write_space(). -*/ -static inline bool tcp_stream_memory_free (const struct sock *sk,int wake) -{ -const struct tcp_sock *tp =tcp_sk (sk); -u32 notsent_bytes =READ_ONCE (tp->write_seq) - -READ_ONCE (tp->snd_nxt); - -return (notsent_bytes << wake) <tcp_notsent_lowat (tp); -} +bool tcp_stream_memory_free (const struct sock *sk,int wake);#ifdef CONFIG_PROC_FS int tcp4_proc_init (void );
net/ipv4/tcp_ipv4.c +14 -0
@@ -2740,6 +2740,20 @@ void tcp4_proc_exit(void) }#endif /* CONFIG_PROC_FS */ +/* @wake is one when sk_stream_write_space() calls us. +* This sends EPOLLOUT only if notsent_bytes is half the limit. +* This mimics the strategy used in sock_def_write_space(). +*/ +bool tcp_stream_memory_free (const struct sock *sk,int wake) +{ +const struct tcp_sock *tp =tcp_sk (sk); +u32 notsent_bytes =READ_ONCE (tp->write_seq) - +READ_ONCE (tp->snd_nxt); + +return (notsent_bytes << wake) <tcp_notsent_lowat (tp); +} +EXPORT_SYMBOL (tcp_stream_memory_free); +struct proto tcp_prot = { .name ="TCP" , .owner =THIS_MODULE ,
Signature · 1 line
2.29.2.299.gdc1121823c-goog
PATCH 2/2 tcp: avoid indirect call to tcp_stream_memory_free()
In reply to Eric DumazetFrom: Eric Dumazet <[email protected]>
Signed-off-by Eric Dumazet <[email protected]>
--- include/net/sock.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-)
include/net/sock.h +6 -2
@@ -60,7 +60,7 @@#include <linux/rculist_nulls.h> #include <linux/poll.h> #include <linux/sockptr.h> - +#include <linux/indirect_call_wrapper.h> #include <linux/atomic.h> #include <linux/refcount.h> #include <net/dst.h>
@@ -1264,13 +1264,17 @@ static inline void sk_refcnt_debug_release(const struct sock *sk)#define sk_refcnt_debug_release (sk) do { } while (0)#endif /* SOCK_REFCNT_DEBUG */ +INDIRECT_CALLABLE_DECLARE(bool tcp_stream_memory_free(const struct sock *sk, int wake)); +static inline bool __sk_stream_memory_free (const struct sock *sk,int wake) {if (READ_ONCE (sk->sk_wmem_queued) >=READ_ONCE (sk->sk_sndbuf))return false;return sk->sk_prot->stream_memory_free ? - sk->sk_prot->stream_memory_free (sk, wake) : true; +INDIRECT_CALL_1 (sk->sk_prot->stream_memory_free, + tcp_stream_memory_free, + sk, wake) : true; }static inline bool sk_stream_memory_free (const struct sock *sk)
Signature · 1 line
2.29.2.299.gdc1121823c-goog
Quoted 3 lines · On Fri, 13 Nov 2020 07:08:07 -0800 Eric Dumazet wrote:
From: Eric Dumazet <[email protected]>
Small improvement for
CONFIG_RETPOLINE=y, when dealing with TCP sockets.
Applied, thank you!