net: Add sysctl to toggle early demux for tcp and udp

PATCH Applied Subash Abhinov Kasiviswanathan 3 participants 7 messages Mar 23, 2017

Subsystems (6)

Networking Network drivers Documentation IPv4/IPv6 +2 more subsystems

lore

Share & tools

lore

Subash Abhinov Kasiviswanathan

author

2017-03-23 02:02 UTC #0 lore raw

PATCH v4 1/1 net: Add sysctl to toggle early demux for tcp and udp

Certain system process significant unconnected UDP workload. It would be preferrable to disable UDP early demux for those systems and enable it for TCP only.

By disabling UDP demux, we see these slight gains on an ARM64 system- 782 -> 788Mbps unconnected single stream UDPv4
633 -> 654Mbps unconnected UDPv4 different sources

The performance impact can change based on CPU architecure and cache sizes. There will not much difference seen if entire UDP hash table is in cache.

Both sysctls are enabled by default to preserve existing behavior.

v1->v2: Change function pointer instead of adding conditional as suggested by Stephen.

v2->v3: Read once in callers to avoid issues due to compiler optimizations. Also update commit message with the tests.

v3>v4: Store and use read once result instead of querying pointer again incorrectly.

Signed-off-by Subash Abhinov Kasiviswanathan <[email protected]>

Cc Stephen Hemminger <[email protected]>

Cc Tom Herbert <[email protected]>

Cc David Miller <[email protected]>

---
 Documentation/networking/ip-sysctl.txt | 11 +++++++-
 include/net/netns/ipv4.h               |  2 ++
 include/net/tcp.h                      |  2 ++
 include/net/udp.h                      |  3 +++
 net/ipv4/af_inet.c                     | 22 ++++++++++++++--
 net/ipv4/ip_input.c                    |  5 ++--
 net/ipv4/sysctl_net_ipv4.c             | 48 ++++++++++++++++++++++++++++++++++
 net/ipv6/ip6_input.c                   |  6 +++--
 net/ipv6/tcp_ipv6.c                    | 10 ++++++-
 net/ipv6/udp.c                         | 10 ++++++-
 10 files changed, 110 insertions(+), 9 deletions(-)

Documentation/networking/ip-sysctl.txt +10 -1

index eaee2c8..b1c6500 100644
@@ -856,12 +856,21 @@ ip_dynaddr - BOOLEAN
 ip_early_demux - BOOLEAN
 	Optimize input packet processing down to one demux for
 	certain kinds of local sockets.  Currently we only do this
-	for established TCP sockets.
+	for established TCP and connected UDP sockets.
 
 	It may add an additional cost for pure routing workloads that
 	reduces overall throughput, in such case you should disable it.
 	Default: 1
 
+tcp_early_demux - BOOLEAN
+	Enable early demux for established TCP sockets.
+	Default: 1
+
+udp_early_demux - BOOLEAN
+	Enable early demux for connected UDP sockets. Disable this if
+	your system could experience more unconnected load.
+	Default: 1
+
 icmp_echo_ignore_all - BOOLEAN
 	If set non-zero, then the kernel will ignore all ICMP ECHO
 	requests sent to it.

include/net/netns/ipv4.h +2 -0

index a0e8919..cd686c4 100644
@@ -95,6 +95,8 @@ struct netns_ipv4 {
 	/* Shall we try to damage output packets if routing dev changes? */
 	int sysctl_ip_dynaddr;
 	int sysctl_ip_early_demux;
+	int sysctl_tcp_early_demux;
+	int sysctl_udp_early_demux;
 
 	int sysctl_fwmark_reflect;
 	int sysctl_tcp_fwmark_accept;

include/net/tcp.h +2 -0

index e614ad4..edc1df4 100644
@@ -1932,4 +1932,6 @@ static inline void tcp_listendrop(const struct sock *sk)
 	__NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENDROPS);
 }
 
+void tcp_v4_early_demux_configure(int enable);
+void tcp_v6_early_demux_configure(int enable);
 #endif	/* _TCP_H */

include/net/udp.h +3 -0

index c9d8b8e..33198fa 100644
@@ -372,4 +372,7 @@ struct udp_iter_state {
 #if IS_ENABLED(CONFIG_IPV6)
 void udpv6_encap_enable(void);
 #endif
+
+void udp_v4_early_demux_configure(int enable);
+void udp_v6_early_demux_configure(int enable);
 #endif	/* _UDP_H */

net/ipv4/af_inet.c +20 -2

index 6b1fc6e..d286750 100644
@@ -1599,7 +1599,7 @@ u64 snmp_fold_field64(void __percpu *mib, int offt, size_t syncp_offset)
 };
 #endif
 
-static const struct net_protocol tcp_protocol = {
+static struct net_protocol tcp_protocol = {
 	.early_demux	=	tcp_v4_early_demux,
 	.handler	=	tcp_v4_rcv,
 	.err_handler	=	tcp_v4_err,

@@ -1608,7 +1608,7 @@ u64 snmp_fold_field64(void __percpu *mib, int offt, size_t syncp_offset)
 	.icmp_strict_tag_validation = 1,
 };
 
-static const struct net_protocol udp_protocol = {
+static struct net_protocol udp_protocol = {
 	.early_demux =	udp_v4_early_demux,
 	.handler =	udp_rcv,
 	.err_handler =	udp_err,

@@ -1616,6 +1616,22 @@ u64 snmp_fold_field64(void __percpu *mib, int offt, size_t syncp_offset)
 	.netns_ok =	1,
 };
 
+void tcp_v4_early_demux_configure(int enable)
+{
+	if (enable)
+		tcp_protocol.early_demux = tcp_v4_early_demux;
+	else
+		tcp_protocol.early_demux = NULL;
+}
+
+void udp_v4_early_demux_configure(int enable)
+{
+	if (enable)
+		udp_protocol.early_demux = udp_v4_early_demux;
+	else
+		udp_protocol.early_demux = NULL;
+}
+
 static const struct net_protocol icmp_protocol = {
 	.handler =	icmp_rcv,
 	.err_handler =	icmp_err,

@@ -1720,6 +1736,8 @@ static __net_init int inet_init_net(struct net *net)
 	net->ipv4.sysctl_ip_default_ttl = IPDEFTTL;
 	net->ipv4.sysctl_ip_dynaddr = 0;
 	net->ipv4.sysctl_ip_early_demux = 1;
+	net->ipv4.sysctl_udp_early_demux = 1;
+	net->ipv4.sysctl_tcp_early_demux = 1;
 #ifdef CONFIG_SYSCTL
 	net->ipv4.sysctl_ip_prot_sock = PROT_SOCK;
 #endif

net/ipv4/ip_input.c +3 -2

index d6feabb..fa2dc8f 100644
@@ -313,6 +313,7 @@ static int ip_rcv_finish(struct net *net, struct sock *sk, struct sk_buff *skb)
 	const struct iphdr *iph = ip_hdr(skb);
 	struct rtable *rt;
 	struct net_device *dev = skb->dev;
+	void (*edemux)(struct sk_buff *skb);
 
 	/* if ingress device is enslaved to an L3 master device pass the
 	 * skb to its handler for processing

@@ -329,8 +330,8 @@ static int ip_rcv_finish(struct net *net, struct sock *sk, struct sk_buff *skb)
 		int protocol = iph->protocol;
 
 		ipprot = rcu_dereference(inet_protos[protocol]);
-		if (ipprot && ipprot->early_demux) {
-			ipprot->early_demux(skb);
+		if (ipprot && (edemux = READ_ONCE(ipprot->early_demux))) {
+			edemux(skb);
 			/* must reload iph, skb->head might have changed */
 			iph = ip_hdr(skb);
 		}

net/ipv4/sysctl_net_ipv4.c +48 -0

60 lines not on this page · raw message

net/ipv6/ip6_input.c +4 -2

index aacfb4b..b04539d 100644
@@ -49,6 +49,8 @@
 
 int ip6_rcv_finish(struct net *net, struct sock *sk, struct sk_buff *skb)
 {
+	void (*edemux)(struct sk_buff *skb);
+
 	/* if ingress device is enslaved to an L3 master device pass the
 	 * skb to its handler for processing
 	 */

@@ -60,8 +62,8 @@ int ip6_rcv_finish(struct net *net, struct sock *sk, struct sk_buff *skb)
 		const struct inet6_protocol *ipprot;
 
 		ipprot = rcu_dereference(inet6_protos[ipv6_hdr(skb)->nexthdr]);
-		if (ipprot && ipprot->early_demux)
-			ipprot->early_demux(skb);
+		if (ipprot && (edemux = READ_ONCE(ipprot->early_demux)))
+			edemux(skb);
 	}
 	if (!skb_valid_dst(skb))
 		ip6_route_input(skb);

net/ipv6/tcp_ipv6.c +9 -1

index 0f08d71..e26622f 100644
@@ -1925,13 +1925,21 @@ struct proto tcpv6_prot = {
 	.diag_destroy		= tcp_abort,
 };
 
-static const struct inet6_protocol tcpv6_protocol = {
+static struct inet6_protocol tcpv6_protocol = {
 	.early_demux	=	tcp_v6_early_demux,
 	.handler	=	tcp_v6_rcv,
 	.err_handler	=	tcp_v6_err,
 	.flags		=	INET6_PROTO_NOPOLICY|INET6_PROTO_FINAL,
 };
 
+void tcp_v6_early_demux_configure(int enable)
+{
+	if (enable)
+		tcpv6_protocol.early_demux = tcp_v6_early_demux;
+	else
+		tcpv6_protocol.early_demux = NULL;
+}
+
 static struct inet_protosw tcpv6_protosw = {
 	.type		=	SOCK_STREAM,
 	.protocol	=	IPPROTO_TCP,

net/ipv6/udp.c +9 -1

index 08a188f..7178a18 100644
@@ -1436,13 +1436,21 @@ int compat_udpv6_getsockopt(struct sock *sk, int level, int optname,
 }
 #endif
 
-static const struct inet6_protocol udpv6_protocol = {
+static struct inet6_protocol udpv6_protocol = {
 	.early_demux	=	udp_v6_early_demux,
 	.handler	=	udpv6_rcv,
 	.err_handler	=	udpv6_err,
 	.flags		=	INET6_PROTO_NOPOLICY|INET6_PROTO_FINAL,
 };
 
+void udp_v6_early_demux_configure(int enable)
+{
+	if (enable)
+		udpv6_protocol.early_demux = udp_v6_early_demux;
+	else
+		udpv6_protocol.early_demux = NULL;
+}
+
 /* ------------------------------------------------------------------------ */
 #ifdef CONFIG_PROC_FS
 int udp6_seq_show(struct seq_file *seq, void *v)

Signature · 1 line

1.9.1

David Miller

maintainer Applied

2017-03-23 02:43 UTC #1 lore raw

From: Subash Abhinov Kasiviswanathan <[email protected]>
Date: Wed, 22 Mar 2017 20:00:41 -0600

Quoted 17 lines

Certain system process significant unconnected UDP workload. It would be preferrable to disable UDP early demux for those systems and enable it for TCP only.
By disabling UDP demux, we see these slight gains on an ARM64 system- 782 -> 788Mbps unconnected single stream UDPv4
633 -> 654Mbps unconnected UDPv4 different sources
The performance impact can change based on CPU architecure and cache sizes. There will not much difference seen if entire UDP hash table is in cache.
Both sysctls are enabled by default to preserve existing behavior.
v1->v2: Change function pointer instead of adding conditional as suggested by Stephen.

Quoted 8 lines

v2->v3: Read once in callers to avoid issues due to compiler optimizations. Also update commit message with the tests.
v3>v4: Store and use read once result instead of querying pointer again incorrectly.
Signed-off-by Subash Abhinov Kasiviswanathan <[email protected]>
Suggested-by Eric Dumazet <[email protected]>

Applied, thanks.

David Miller

maintainer

2017-03-23 03:36 UTC #2 lore raw

From: David Miller <[email protected]>
Date: Wed, 22 Mar 2017 19:43:57 -0700 (PDT)

Quoted 23 lines

From: Subash Abhinov Kasiviswanathan <[email protected]>
Date: Wed, 22 Mar 2017 20:00:41 -0600
Quoted 20 lines
Certain system process significant unconnected UDP workload. It would be preferrable to disable UDP early demux for those systems and enable it for TCP only.
By disabling UDP demux, we see these slight gains on an ARM64 system- 782 -> 788Mbps unconnected single stream UDPv4
633 -> 654Mbps unconnected UDPv4 different sources
The performance impact can change based on CPU architecure and cache sizes. There will not much difference seen if entire UDP hash table is in cache.
Both sysctls are enabled by default to preserve existing behavior.
v1->v2: Change function pointer instead of adding conditional as suggested by Stephen.
v2->v3: Read once in callers to avoid issues due to compiler optimizations. Also update commit message with the tests.

Quoted 7 lines

Quoted 5 lines
v3>v4: Store and use read once result instead of querying pointer again incorrectly.
Signed-off-by Subash Abhinov Kasiviswanathan <[email protected]>
Suggested-by Eric Dumazet <[email protected]>
Applied, thanks.

This doesn't build with ipv6 as a module:

net/built-in.o: In function `proc_tcp_early_demux':
sysctl_net_ipv4.c:(.text+0x145cfc): undefined reference to tcp_v6_early_demux_configure' net/built-in.o: In function proc_udp_early_demux':
sysctl_net_ipv4.c:(.text+0x145d4c): undefined reference to `udp_v6_early_demux_configure' Makefile:983: recipe for target 'vmlinux' failed

Subash Abhinov Kasiviswanathan

author

2017-03-23 03:55 UTC #3 lore raw

Quoted 9 lines

This doesn't build with ipv6 as a module:
net/built-in.o: In function `proc_tcp_early_demux':
sysctl_net_ipv4.c:(.text+0x145cfc): undefined reference to `tcp_v6_early_demux_configure'
net/built-in.o: In function `proc_udp_early_demux':
sysctl_net_ipv4.c:(.text+0x145d4c): undefined reference to `udp_v6_early_demux_configure'
Makefile:983: recipe for target 'vmlinux' failed

Hi David

Sorry about that (we always have IPV6 enabled). Do I have to send a v5 or separate patch for this?

David Miller

maintainer

2017-03-23 04:41 UTC #4 lore raw

From: Subash Abhinov Kasiviswanathan <[email protected]>
Date: Wed, 22 Mar 2017 21:55:04 -0600

Quoted 9 lines

Quoted 8 lines
This doesn't build with ipv6 as a module:
net/built-in.o: In function `proc_tcp_early_demux':
sysctl_net_ipv4.c:(.text+0x145cfc): undefined reference to `tcp_v6_early_demux_configure'
net/built-in.o: In function `proc_udp_early_demux':
sysctl_net_ipv4.c:(.text+0x145d4c): undefined reference to `udp_v6_early_demux_configure'
Makefile:983: recipe for target 'vmlinux' failed

Quoted 4 lines

Hi David
Sorry about that (we always have IPV6 enabled). Do I have to send a v5 or separate patch for this?

Please send a v5, thanks.

kbuild test robot

build bot

2017-03-23 17:12 UTC #5 lore raw

In reply to Subash Abhinov Kasiviswanathan

Hi Subash,

[auto build test ERROR on net-next/master]

        # save the attached .config to linux build tree
        make ARCH=x86_64

All errors (new ones prefixed by >>):

net/built-in.o: In function `proc_tcp_early_demux':

>> sysctl_net_ipv4.c:(.text+0x7fe04): undefined reference to `tcp_v6_early_demux_configure'
   net/built-in.o: In function `proc_udp_early_demux':
>> sysctl_net_ipv4.c:(.text+0x7fe3d): undefined reference to `udp_v6_early_demux_configure'

---
0-DAY kernel test infrastructure Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all Intel Corporation

kbuild test robot

build bot

2017-03-23 18:12 UTC #6 lore raw

In reply to Subash Abhinov Kasiviswanathan

url: https://github.com/0day-ci/linux/commits/Subash-Abhinov-Kasiviswanathan/net-Add-sysctl-to-toggle-early-demux-for-tcp-and-udp/20170323-205131 config: arm-allmodconfig (attached as .config)…

Hi Subash,

[auto build test ERROR on net-next/master]

url: https://github.com/0day-ci/linux/commits/Subash-Abhinov-Kasiviswanathan/net-Add-sysctl-to-toggle-early-demux-for-tcp-and-udp/20170323-205131
config: arm-allmodconfig (attached as .config)
compiler: arm-linux-gnueabi-gcc (Debian 6.1.1-9) 6.1.1 20160705
reproduce:
wget https://raw.githubusercontent.com/01org/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# save the attached .config to linux build tree
make.cross ARCH=arm

All errors (new ones prefixed by >>):

net/built-in.o: In function `proc_tcp_early_demux':

>> ncsi-manage.c:(.text+0xdffd4): undefined reference to `tcp_v6_early_demux_configure'
   net/built-in.o: In function `proc_udp_early_demux':
>> ncsi-manage.c:(.text+0xe0040): undefined reference to `udp_v6_early_demux_configure'

---
0-DAY kernel test infrastructure Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all Intel Corporation