Skip to contentSkip to conversation

tcp: avoid indirect call in __sk_stream_memory_free()

PATCH Applied Eric Dumazet 2 participants 4 messages Nov 13, 2020 – Nov 15, 2020

Subsystems (4)
lore
Share & tools
lore
#0 lore raw

cover tcp: avoid indirect call in __sk_stream_memory_free()

From: Eric Dumazet <[email protected]>

Small improvement for CONFIG_RETPOLINE=y, when dealing with TCP sockets.

Eric Dumazet (2):
tcp: uninline tcp_stream_memory_free()
tcp: avoid indirect call to tcp_stream_memory_free()

 include/net/sock.h  |  8 ++++++--
 include/net/tcp.h   | 13 +------------
 net/ipv4/tcp_ipv4.c | 14 ++++++++++++++
 3 files changed, 21 insertions(+), 14 deletions(-)
Signature · 1 line
2.29.2.299.gdc1121823c-goog
#1 lore raw

PATCH 1/2 tcp: uninline tcp_stream_memory_free()

From: Eric Dumazet <[email protected]>

Both IPv4 and IPv6 needs it via a function pointer. Following patch will avoid the indirect call.

---
 include/net/tcp.h   | 13 +------------
 net/ipv4/tcp_ipv4.c | 14 ++++++++++++++
 2 files changed, 15 insertions(+), 12 deletions(-)
include/net/tcp.h +1 -12
index 4aba0f069b05d68b2595ec9a3bf8da7786437fde..d643ee4e42495f0b28c87e79d6192c23f8cdba7f 100644
@@ -1965,18 +1965,7 @@ static inline u32 tcp_notsent_lowat(const struct tcp_sock *tp)
 	return tp->notsent_lowat ?: net->ipv4.sysctl_tcp_notsent_lowat;
 }
 
-/* @wake is one when sk_stream_write_space() calls us.
- * This sends EPOLLOUT only if notsent_bytes is half the limit.
- * This mimics the strategy used in sock_def_write_space().
- */
-static inline bool tcp_stream_memory_free(const struct sock *sk, int wake)
-{
-	const struct tcp_sock *tp = tcp_sk(sk);
-	u32 notsent_bytes = READ_ONCE(tp->write_seq) -
-			    READ_ONCE(tp->snd_nxt);
-
-	return (notsent_bytes << wake) < tcp_notsent_lowat(tp);
-}
+bool tcp_stream_memory_free(const struct sock *sk, int wake);
 
 #ifdef CONFIG_PROC_FS
 int tcp4_proc_init(void);
net/ipv4/tcp_ipv4.c +14 -0
index 7352c097ae48f7df6caba1ae5b98dccfecd79d1a..c2d5132c523c8d4258e407c4cc5112840b07805e 100644
@@ -2740,6 +2740,20 @@ void tcp4_proc_exit(void)
 }
 #endif /* CONFIG_PROC_FS */
 
+/* @wake is one when sk_stream_write_space() calls us.
+ * This sends EPOLLOUT only if notsent_bytes is half the limit.
+ * This mimics the strategy used in sock_def_write_space().
+ */
+bool tcp_stream_memory_free(const struct sock *sk, int wake)
+{
+	const struct tcp_sock *tp = tcp_sk(sk);
+	u32 notsent_bytes = READ_ONCE(tp->write_seq) -
+			    READ_ONCE(tp->snd_nxt);
+
+	return (notsent_bytes << wake) < tcp_notsent_lowat(tp);
+}
+EXPORT_SYMBOL(tcp_stream_memory_free);
+
 struct proto tcp_prot = {
 	.name			= "TCP",
 	.owner			= THIS_MODULE,
Signature · 1 line
2.29.2.299.gdc1121823c-goog
#2 lore raw

PATCH 2/2 tcp: avoid indirect call to tcp_stream_memory_free()

In reply to Eric Dumazet

From: Eric Dumazet <[email protected]>

---
 include/net/sock.h | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)
include/net/sock.h +6 -2
index a5c6ae78df77d08e4b65e634e93040446486bdc2..1d29aeae74fdbbae3410def33ebc66758e034205 100644
@@ -60,7 +60,7 @@
 #include <linux/rculist_nulls.h>
 #include <linux/poll.h>
 #include <linux/sockptr.h>
-
+#include <linux/indirect_call_wrapper.h>
 #include <linux/atomic.h>
 #include <linux/refcount.h>
 #include <net/dst.h>
@@ -1264,13 +1264,17 @@ static inline void sk_refcnt_debug_release(const struct sock *sk)
 #define sk_refcnt_debug_release(sk) do { } while (0)
 #endif /* SOCK_REFCNT_DEBUG */
 
+INDIRECT_CALLABLE_DECLARE(bool tcp_stream_memory_free(const struct sock *sk, int wake));
+
 static inline bool __sk_stream_memory_free(const struct sock *sk, int wake)
 {
 	if (READ_ONCE(sk->sk_wmem_queued) >= READ_ONCE(sk->sk_sndbuf))
 		return false;
 
 	return sk->sk_prot->stream_memory_free ?
-		sk->sk_prot->stream_memory_free(sk, wake) : true;
+		INDIRECT_CALL_1(sk->sk_prot->stream_memory_free,
+			        tcp_stream_memory_free,
+				sk, wake) : true;
 }
 
 static inline bool sk_stream_memory_free(const struct sock *sk)
Signature · 1 line
2.29.2.299.gdc1121823c-goog