net: Add sysctl to toggle early demux for tcp and udp
Subsystems (6)
PATCH v3 1/1 net: Add sysctl to toggle early demux for tcp and udp
Certain system process significant unconnected UDP workload. It would be preferrable to disable UDP early demux for those systems and enable it for TCP only.
By disabling UDP demux, we see these slight gains on an ARM64 system- 782 -> 788Mbps unconnected single stream UDPv4
633 -> 654Mbps unconnected UDPv4 different sources
The performance impact can change based on CPU architecure and cache sizes. There will not much difference seen if entire UDP hash table is in cache.
Both sysctls are enabled by default to preserve existing behavior.
v1->v2: Change function pointer instead of adding conditional as suggested by Stephen.
v2->v3: Read once in callers to avoid issues due to compiler optimizations. Also update commit message with the tests.
--- Documentation/networking/ip-sysctl.txt | 11 +++++++- include/net/netns/ipv4.h | 2 ++ include/net/tcp.h | 2 ++ include/net/udp.h | 3 +++ net/ipv4/af_inet.c | 22 ++++++++++++++-- net/ipv4/ip_input.c | 2 +- net/ipv4/sysctl_net_ipv4.c | 48 ++++++++++++++++++++++++++++++++++ net/ipv6/ip6_input.c | 2 +- net/ipv6/tcp_ipv6.c | 10 ++++++- net/ipv6/udp.c | 10 ++++++- 10 files changed, 105 insertions(+), 7 deletions(-)
Documentation/networking/ip-sysctl.txt +10 -1
@@ -848,12 +848,21 @@ ip_dynaddr - BOOLEAN ip_early_demux - BOOLEAN Optimize input packet processing down to one demux for certain kinds of local sockets. Currently we only do this - for established TCP sockets. + for established TCP and connected UDP sockets. It may add an additional cost for pure routing workloads that reduces overall throughput, in such case you should disable it. Default: 1 +tcp_early_demux - BOOLEAN + Enable early demux for established TCP sockets. + Default: 1 + +udp_early_demux - BOOLEAN + Enable early demux for connected UDP sockets. Disable this if + your system could experience more unconnected load. + Default: 1 + icmp_echo_ignore_all - BOOLEAN If set non-zero, then the kernel will ignore all ICMP ECHO requests sent to it.
include/net/netns/ipv4.h +2 -0
@@ -95,6 +95,8 @@ struct netns_ipv4 {/* Shall we try to damage output packets if routing dev changes? */ int sysctl_ip_dynaddr;int sysctl_ip_early_demux; +int sysctl_tcp_early_demux; +int sysctl_udp_early_demux;int sysctl_fwmark_reflect;int sysctl_tcp_fwmark_accept;
include/net/tcp.h +2 -0
@@ -1932,4 +1932,6 @@ static inline void tcp_listendrop(const struct sock *sk) __NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENDROPS); } +void tcp_v4_early_demux_configure (int enable); +void tcp_v6_early_demux_configure (int enable);#endif /* _TCP_H */
include/net/udp.h +3 -0
@@ -372,4 +372,7 @@ struct udp_iter_state {#if IS_ENABLED (CONFIG_IPV6 )void udpv6_encap_enable (void );#endif + +void udp_v4_early_demux_configure (int enable); +void udp_v6_early_demux_configure (int enable);#endif /* _UDP_H */
net/ipv4/af_inet.c +20 -2
@@ -1599,7 +1599,7 @@ u64 snmp_fold_field64(void __percpu *mib, int offt, size_t syncp_offset) };#endif -static const struct net_protocol tcp_protocol = { +static struct net_protocol tcp_protocol = { .early_demux = tcp_v4_early_demux, .handler = tcp_v4_rcv, .err_handler = tcp_v4_err,
@@ -1608,7 +1608,7 @@ u64 snmp_fold_field64(void __percpu *mib, int offt, size_t syncp_offset) .icmp_strict_tag_validation = 1, }; -static const struct net_protocol udp_protocol = { +static struct net_protocol udp_protocol = { .early_demux = udp_v4_early_demux, .handler = udp_rcv, .err_handler = udp_err,
@@ -1616,6 +1616,22 @@ u64 snmp_fold_field64(void __percpu *mib, int offt, size_t syncp_offset) .netns_ok = 1, }; +void tcp_v4_early_demux_configure (int enable) +{ +if (enable) + tcp_protocol.early_demux = tcp_v4_early_demux; +else + tcp_protocol.early_demux =NULL ; +} + +void udp_v4_early_demux_configure (int enable) +{ +if (enable) + udp_protocol.early_demux = udp_v4_early_demux; +else + udp_protocol.early_demux =NULL ; +} +static const struct net_protocol icmp_protocol = { .handler = icmp_rcv, .err_handler = icmp_err,
@@ -1720,6 +1736,8 @@ static __net_init int inet_init_net(struct net *net) net->ipv4.sysctl_ip_default_ttl =IPDEFTTL ; net->ipv4.sysctl_ip_dynaddr =0 ; net->ipv4.sysctl_ip_early_demux =1 ; + net->ipv4.sysctl_udp_early_demux =1 ; + net->ipv4.sysctl_tcp_early_demux =1 ;#ifdef CONFIG_SYSCTL net->ipv4.sysctl_ip_prot_sock =PROT_SOCK ;#endif
net/ipv4/ip_input.c +1 -1
@@ -329,7 +329,7 @@ static int ip_rcv_finish(struct net *net, struct sock *sk, struct sk_buff *skb)int protocol = iph->protocol; ipprot =rcu_dereference (inet_protos[protocol]); -if (ipprot && ipprot->early_demux) { +if (ipprot &&READ_ONCE (ipprot->early_demux)) { ipprot->early_demux (skb);/* must reload iph, skb->head might have changed */ iph =ip_hdr (skb);
net/ipv4/sysctl_net_ipv4.c +48 -0
60 lines not on this page · raw message
net/ipv6/ip6_input.c +1 -1
@@ -60,7 +60,7 @@ int ip6_rcv_finish(struct net *net, struct sock *sk, struct sk_buff *skb)const struct inet6_protocol *ipprot; ipprot =rcu_dereference (inet6_protos[ipv6_hdr (skb)->nexthdr]); -if (ipprot && ipprot->early_demux) +if (ipprot &&READ_ONCE (ipprot->early_demux)) ipprot->early_demux (skb); }if (!skb_valid_dst (skb))
net/ipv6/tcp_ipv6.c +9 -1
@@ -1925,13 +1925,21 @@ struct proto tcpv6_prot = { .diag_destroy = tcp_abort, }; -static const struct inet6_protocol tcpv6_protocol = { +static struct inet6_protocol tcpv6_protocol = { .early_demux = tcp_v6_early_demux, .handler = tcp_v6_rcv, .err_handler = tcp_v6_err, .flags =INET6_PROTO_NOPOLICY |INET6_PROTO_FINAL , }; +void tcp_v6_early_demux_configure (int enable) +{ +if (enable) + tcpv6_protocol.early_demux = tcp_v6_early_demux; +else + tcpv6_protocol.early_demux =NULL ; +} +static struct inet_protosw tcpv6_protosw = { .type =SOCK_STREAM , .protocol =IPPROTO_TCP ,
net/ipv6/udp.c +9 -1
@@ -1436,13 +1436,21 @@ int compat_udpv6_getsockopt(struct sock *sk, int level, int optname, }#endif -static const struct inet6_protocol udpv6_protocol = { +static struct inet6_protocol udpv6_protocol = { .early_demux = udp_v6_early_demux, .handler = udpv6_rcv, .err_handler = udpv6_err, .flags =INET6_PROTO_NOPOLICY |INET6_PROTO_FINAL , }; +void udp_v6_early_demux_configure (int enable) +{ +if (enable) + udpv6_protocol.early_demux = udp_v6_early_demux; +else + udpv6_protocol.early_demux =NULL ; +} +/* ------------------------------------------------------------------------ */ #ifdef CONFIG_PROC_FS int udp6_seq_show (struct seq_file *seq,void *v)
Signature · 1 line
1.9.1
David Miller
maintainerFrom: Subash Abhinov Kasiviswanathan <[email protected]>
Date: Tue, 21 Mar 2017 21:20:10 -0600
net/ipv4/ip_input.c static int ip_rcv_finish(struct net *net, struct sock *sk, struct sk_buff *skb) · line 333
Quoted 7 lines
net/ipv4/ip_input.c +1
-1@@ -329,7 +329,7 @@ static int ip_rcv_finish(struct net *net, struct sock *sk, struct sk_buff *skb)int protocol = iph->protocol; ipprot =rcu_dereference (inet_protos[protocol]); -if (ipprot && ipprot->early_demux) { +if (ipprot &&READ_ONCE (ipprot->early_demux)) { ipprot->early_demux (skb);
I think you need to use a local variable for the function pointer in conjunction with READ_ONCE() for this to work properly:
if (ipprot && (func = READ_ONCE(ipprot->early_demux))) {
func(skb);
...net/ipv6/ip6_input.c int ip6_rcv_finish(struct net *net, struct sock *sk, struct sk_buff *skb) · line 66
Quoted 13 lines
net/ipv6/ip6_input.c +1
-1@@ -60,7 +60,7 @@ int ip6_rcv_finish(struct net *net, struct sock *sk, struct sk_buff *skb)const struct inet6_protocol *ipprot; ipprot =rcu_dereference (inet6_protos[ipv6_hdr (skb)->nexthdr]); -if (ipprot && ipprot->early_demux) +if (ipprot &&READ_ONCE (ipprot->early_demux)) ipprot->early_demux (skb); }if (!skb_valid_dst (skb))
Likewise.
kbuild test robot
build boturl: https://github.com/0day-ci/linux/commits/Subash-Abhinov-Kasiviswanathan/net-Add-sysctl-to-toggle-early-demux-for-tcp-and-udp/20170323-182822 config: x86_64-kexec (attached as .config) compiler…
Hi Subash,
[auto build test ERROR on net-next/master]
url: https://github.com/0day-ci/linux/commits/Subash-Abhinov-Kasiviswanathan/net-Add-sysctl-to-toggle-early-demux-for-tcp-and-udp/20170323-182822
config: x86_64-kexec (attached as .config)
compiler: gcc-6 (Debian 6.2.0-3) 6.2.0 20160901
reproduce:
# save the attached .config to linux build tree
make ARCH=x86_64 All errors (new ones prefixed by >>):
net/built-in.o: In function `proc_tcp_early_demux':
sysctl_net_ipv4.c:(.text+0x7fe04): undefined reference to tcp_v6_early_demux_configure' net/built-in.o: In function proc_udp_early_demux':
>> sysctl_net_ipv4.c:(.text+0x7fe3d): undefined reference to `udp_v6_early_demux_configure'
---
0-DAY kernel test infrastructure Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all Intel Corporation
kbuild test robot
build boturl: https://github.com/0day-ci/linux/commits/Subash-Abhinov-Kasiviswanathan/net-Add-sysctl-to-toggle-early-demux-for-tcp-and-udp/20170323-182822 config: alpha-defconfig (attached as .config)…
Hi Subash,
[auto build test ERROR on net-next/master]
url: https://github.com/0day-ci/linux/commits/Subash-Abhinov-Kasiviswanathan/net-Add-sysctl-to-toggle-early-demux-for-tcp-and-udp/20170323-182822
config: alpha-defconfig (attached as .config)
compiler: alpha-linux-gnu-gcc (Debian 6.1.1-9) 6.1.1 20160705
reproduce:
wget https://raw.githubusercontent.com/01org/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# save the attached .config to linux build tree
make.cross ARCH=alpha
All errors (new ones prefixed by >>):
net/built-in.o: In function `proc_tcp_early_demux':
net/ipv4/sysctl_net_ipv4.c:308: undefined reference to tcp_v6_early_demux_configure' net/ipv4/sysctl_net_ipv4.c:308: undefined reference to tcp_v6_early_demux_configure' net/built-in.o: In function `proc_udp_early_demux':
>> net/ipv4/sysctl_net_ipv4.c:325: undefined reference to `udp_v6_early_demux_configure' >> net/ipv4/sysctl_net_ipv4.c:325: undefined reference to `udp_v6_early_demux_configure'
vim +325 net/ipv4/sysctl_net_ipv4.c
302 ret = proc_dointvec(table, write, buffer, lenp, ppos);
303
304 if (write && !ret) {
305 int enabled = init_net.ipv4.sysctl_tcp_early_demux;
306
307 tcp_v4_early_demux_configure(enabled);
> 308 tcp_v6_early_demux_configure(enabled);
309 }
310
311 return ret;
312 }
313
314 static int proc_udp_early_demux(struct ctl_table *table, int write,
315 void __user *buffer, size_t *lenp, loff_t *ppos)
316 {
317 int ret = 0;
318
319 ret = proc_dointvec(table, write, buffer, lenp, ppos);
320
321 if (write && !ret) {
322 int enabled = init_net.ipv4.sysctl_udp_early_demux;
323
324 udp_v4_early_demux_configure(enabled);
> 325 udp_v6_early_demux_configure(enabled); 326 }
327
328 return ret;
---
0-DAY kernel test infrastructure Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all Intel Corporation