ipv6: Implement optimized IPv6 masked address comparison for ARM64

PATCH In review Subash Abhinov Kasiviswanathan 3 participants 4 messages Mar 17, 2017

Subsystems (22)

Networking Network drivers Arm (32-bit) x86 +18 more subsystems

lore

Share & tools

lore

Subash Abhinov Kasiviswanathan

author

2017-03-17 04:43 UTC #0 lore raw

PATCH 1/1 ipv6: Implement optimized IPv6 masked address comparison for ARM64

Android devices use multiple ip[6]tables for statistics, UID matching and other functionality. Perf output indicated that ip6_do_table was taking a considerable amount of CPU and more that ip_do_table for an equivalent rate. ipv6_masked_addr_cmp was chosen for optimization as there are more instructions required than the equivalent operation in ip_packet_match.

Using 128 bit operations helps to reduce the number of instructions for the match on an ARM64 system. This helps to improve UDPv6 DL performance by 40Mbps (860Mbps -> 900Mbps) on a CPU limited system.

Tested on x86_64 UML to check if generic version is used and ARM64 to verify that ARM64 version is used.

Signed-off-by Subash Abhinov Kasiviswanathan <[email protected]>

---
 arch/alpha/include/asm/Kbuild      |  1 +
 arch/arc/include/asm/Kbuild        |  1 +
 arch/arm/include/asm/Kbuild        |  1 +
 arch/arm64/include/asm/ipv6.h      | 29 +++++++++++++++++++++++++++++
 arch/avr32/include/asm/Kbuild      |  1 +
 arch/blackfin/include/asm/Kbuild   |  1 +
 arch/c6x/include/asm/Kbuild        |  1 +
 arch/cris/include/asm/Kbuild       |  1 +
 arch/frv/include/asm/Kbuild        |  1 +
 arch/h8300/include/asm/Kbuild      |  1 +
 arch/hexagon/include/asm/Kbuild    |  1 +
 arch/ia64/include/asm/Kbuild       |  1 +
 arch/m32r/include/asm/Kbuild       |  1 +
 arch/m68k/include/asm/Kbuild       |  1 +
 arch/metag/include/asm/Kbuild      |  1 +
 arch/microblaze/include/asm/Kbuild |  1 +
 arch/mips/include/asm/Kbuild       |  1 +
 arch/mn10300/include/asm/Kbuild    |  1 +
 arch/nios2/include/asm/Kbuild      |  1 +
 arch/openrisc/include/asm/Kbuild   |  1 +
 arch/parisc/include/asm/Kbuild     |  1 +
 arch/powerpc/include/asm/Kbuild    |  1 +
 arch/s390/include/asm/Kbuild       |  1 +
 arch/score/include/asm/Kbuild      |  1 +
 arch/sh/include/asm/Kbuild         |  1 +
 arch/sparc/include/asm/Kbuild      |  1 +
 arch/tile/include/asm/Kbuild       |  1 +
 arch/um/include/asm/Kbuild         |  1 +
 arch/unicore32/include/asm/Kbuild  |  1 +
 arch/x86/include/asm/Kbuild        |  1 +
 arch/xtensa/include/asm/Kbuild     |  1 +
 include/asm-generic/ipv6.h         | 32 ++++++++++++++++++++++++++++++++
 include/net/ipv6.h                 | 20 +-------------------
 33 files changed, 92 insertions(+), 19 deletions(-)
 create mode 100644 arch/arm64/include/asm/ipv6.h
 create mode 100644 include/asm-generic/ipv6.h

arch/alpha/include/asm/Kbuild +1 -0

index d103db5..5b7e92b 100644
@@ -3,6 +3,7 @@
 generic-y += clkdev.h
 generic-y += exec.h
 generic-y += export.h
+generic-y += ipv6.h
 generic-y += irq_work.h
 generic-y += mcs_spinlock.h
 generic-y += mm-arch-hooks.h

arch/arc/include/asm/Kbuild +1 -0

index 63a0401..99f1456 100644
@@ -14,6 +14,7 @@ generic-y += hw_irq.h
 generic-y += ioctl.h
 generic-y += ioctls.h
 generic-y += ipcbuf.h
+generic-y += ipv6.h
 generic-y += irq_regs.h
 generic-y += irq_work.h
 generic-y += kmap_types.h

arch/arm/include/asm/Kbuild +1 -0

index b14e8c7..a0ba9ac 100644
@@ -9,6 +9,7 @@ generic-y += errno.h
 generic-y += exec.h
 generic-y += ioctl.h
 generic-y += ipcbuf.h
+generic-y += ipv6.h
 generic-y += irq_regs.h
 generic-y += kdebug.h
 generic-y += local.h

arch/arm64/include/asm/ipv6.h new file +29 -0

new file mode 100644
index 0000000..d49dec6
@@ -0,0 +1,29 @@
+/* Copyright (c) 2017, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef __ASM_IPV6_H
+#define __ASM_IPV6_H
+
+#include <linux/types.h>
+
+static inline bool
+ipv6_masked_addr_cmp(const struct in6_addr *a1, const struct in6_addr *m,
+		     const struct in6_addr *a2)
+{
+	const __uint128_t *ul1 = (const __uint128_t *)a1;
+	const __uint128_t *ulm = (const __uint128_t *)m;
+	const __uint128_t *ul2 = (const __uint128_t *)a1;
+
+	return !!((*ul1 ^ *ul2) & *ulm);
+}
+#define ipv6_masked_addr_cmp ipv6_masked_addr_cmp
+#endif /* __ASM_IPV6_H */

arch/avr32/include/asm/Kbuild +1 -0

index 3d7ef2c..fd6a964 100644
@@ -6,6 +6,7 @@ generic-y += div64.h
 generic-y += emergency-restart.h
 generic-y += exec.h
 generic-y += futex.h
+generic-y += ipv6.h
 generic-y += irq_regs.h
 generic-y += irq_work.h
 generic-y += local.h

arch/blackfin/include/asm/Kbuild +1 -0

index 625db8a..f713d85 100644
@@ -12,6 +12,7 @@ generic-y += futex.h
 generic-y += hw_irq.h
 generic-y += ioctl.h
 generic-y += ipcbuf.h
+generic-y += ipv6.h
 generic-y += irq_regs.h
 generic-y += irq_work.h
 generic-y += kdebug.h

arch/c6x/include/asm/Kbuild +1 -0

index 82619c3..ff8033f 100644
@@ -20,6 +20,7 @@ generic-y += io.h
 generic-y += ioctl.h
 generic-y += ioctls.h
 generic-y += ipcbuf.h
+generic-y += ipv6.h
 generic-y += irq_regs.h
 generic-y += irq_work.h
 generic-y += kdebug.h

arch/cris/include/asm/Kbuild +1 -0

index 0f5132b..c5b5fa0 100644
@@ -15,6 +15,7 @@ generic-y += futex.h
 generic-y += hardirq.h
 generic-y += ioctl.h
 generic-y += ipcbuf.h
+generic-y += ipv6.h
 generic-y += irq_regs.h
 generic-y += irq_work.h
 generic-y += kdebug.h

arch/frv/include/asm/Kbuild +1 -0

index c33b467..0717ffb 100644
@@ -1,6 +1,7 @@
 
 generic-y += clkdev.h
 generic-y += exec.h
+generic-y += ipv6.h
 generic-y += irq_work.h
 generic-y += mcs_spinlock.h
 generic-y += mm-arch-hooks.h

arch/h8300/include/asm/Kbuild +1 -0

index 341740c..0058275 100644
@@ -23,6 +23,7 @@ generic-y += hw_irq.h
 generic-y += ioctl.h
 generic-y += ioctls.h
 generic-y += ipcbuf.h
+generic-y += ipv6.h
 generic-y += irq_regs.h
 generic-y += irq_work.h
 generic-y += kdebug.h

arch/hexagon/include/asm/Kbuild +1 -0

index 797b64a..4985925 100644
@@ -20,6 +20,7 @@ generic-y += ioctl.h
 generic-y += ioctls.h
 generic-y += iomap.h
 generic-y += ipcbuf.h
+generic-y += ipv6.h
 generic-y += irq_regs.h
 generic-y += irq_work.h
 generic-y += kdebug.h

arch/ia64/include/asm/Kbuild +1 -0

index 502a91d..cacba4c 100644
@@ -1,6 +1,7 @@
 
 generic-y += clkdev.h
 generic-y += exec.h
+generic-y += ipv6.h
 generic-y += irq_work.h
 generic-y += kvm_para.h
 generic-y += mcs_spinlock.h

arch/m32r/include/asm/Kbuild +1 -0

index deb2987..8ed14f5 100644
@@ -2,6 +2,7 @@
 generic-y += clkdev.h
 generic-y += current.h
 generic-y += exec.h
+generic-y += ipv6.h
 generic-y += irq_work.h
 generic-y += kvm_para.h
 generic-y += mcs_spinlock.h

arch/m68k/include/asm/Kbuild +1 -0

index d4f9ccb..1ddfeac 100644
@@ -9,6 +9,7 @@ generic-y += futex.h
 generic-y += hw_irq.h
 generic-y += ioctl.h
 generic-y += ipcbuf.h
+generic-y += ipv6.h
 generic-y += irq_regs.h
 generic-y += irq_work.h
 generic-y += kdebug.h

arch/metag/include/asm/Kbuild +1 -0

index f9b9df5..2382a6e 100644
@@ -16,6 +16,7 @@ generic-y += hw_irq.h
 generic-y += ioctl.h
 generic-y += ioctls.h
 generic-y += ipcbuf.h
+generic-y += ipv6.h
 generic-y += irq_regs.h
 generic-y += irq_work.h
 generic-y += kdebug.h

arch/microblaze/include/asm/Kbuild +1 -0

index 1732ec1..66b7d8a 100644
@@ -3,6 +3,7 @@ generic-y += barrier.h
 generic-y += clkdev.h
 generic-y += device.h
 generic-y += exec.h
+generic-y += ipv6.h
 generic-y += irq_work.h
 generic-y += mcs_spinlock.h
 generic-y += mm-arch-hooks.h

arch/mips/include/asm/Kbuild +1 -0

index 2535c7b..4453f33 100644
@@ -5,6 +5,7 @@ generic-y += current.h
 generic-y += dma-contiguous.h
 generic-y += emergency-restart.h
 generic-y += export.h
+generic-y += ipv6.h
 generic-y += irq_work.h
 generic-y += local64.h
 generic-y += mcs_spinlock.h

arch/mn10300/include/asm/Kbuild +1 -0

index 97f64c7..df55c2b 100644
@@ -2,6 +2,7 @@
 generic-y += barrier.h
 generic-y += clkdev.h
 generic-y += exec.h
+generic-y += ipv6.h
 generic-y += irq_work.h
 generic-y += mcs_spinlock.h
 generic-y += mm-arch-hooks.h

arch/nios2/include/asm/Kbuild +1 -0

index aaa3c21..d6b13ee 100644
@@ -22,6 +22,7 @@ generic-y += hw_irq.h
 generic-y += ioctl.h
 generic-y += ioctls.h
 generic-y += ipcbuf.h
+generic-y += ipv6.h
 generic-y += irq_regs.h
 generic-y += irq_work.h
 generic-y += kdebug.h

arch/openrisc/include/asm/Kbuild +1 -0

index fb01873..0d9ad5a 100644
@@ -25,6 +25,7 @@ generic-y += ioctl.h
 generic-y += ioctls.h
 generic-y += ipcbuf.h
 generic-y += irq.h
+generic-y += ipv6.h
 generic-y += irq_regs.h
 generic-y += irq_work.h
 generic-y += kdebug.h

arch/parisc/include/asm/Kbuild +1 -0

index a9909c2..b0e156e 100644
@@ -8,6 +8,7 @@ generic-y += div64.h
 generic-y += emergency-restart.h
 generic-y += exec.h
 generic-y += hw_irq.h
+generic-y += ipv6.h
 generic-y += irq_regs.h
 generic-y += irq_work.h
 generic-y += kdebug.h

arch/powerpc/include/asm/Kbuild +1 -0

index 5c4fbc8..f675f7c 100644
@@ -1,6 +1,7 @@
 generic-y += clkdev.h
 generic-y += div64.h
 generic-y += export.h
+generic-y += ipv6.h
 generic-y += irq_regs.h
 generic-y += irq_work.h
 generic-y += local64.h

arch/s390/include/asm/Kbuild +1 -0

index 8aea32f..53a2335 100644
@@ -2,6 +2,7 @@ generic-y += asm-offsets.h
 generic-y += clkdev.h
 generic-y += dma-contiguous.h
 generic-y += export.h
+generic-y += ipv6.h
 generic-y += irq_work.h
 generic-y += mcs_spinlock.h
 generic-y += mm-arch-hooks.h

arch/score/include/asm/Kbuild +1 -0

index 926943a..9405456 100644
@@ -4,6 +4,7 @@ header-y +=
 generic-y += barrier.h
 generic-y += clkdev.h
 generic-y += current.h
+generic-y += ipv6.h
 generic-y += irq_work.h
 generic-y += mcs_spinlock.h
 generic-y += mm-arch-hooks.h

arch/sh/include/asm/Kbuild +1 -0

index cf2a750..3984834 100644
@@ -10,6 +10,7 @@ generic-y += exec.h
 generic-y += fcntl.h
 generic-y += ioctl.h
 generic-y += ipcbuf.h
+generic-y += ipv6.h
 generic-y += irq_regs.h
 generic-y += irq_work.h
 generic-y += kvm_para.h

arch/sparc/include/asm/Kbuild +1 -0

index e9e837b..d2acdaf 100644
@@ -6,6 +6,7 @@ generic-y += div64.h
 generic-y += emergency-restart.h
 generic-y += exec.h
 generic-y += export.h
+generic-y += ipv6.h
 generic-y += irq_regs.h
 generic-y += irq_work.h
 generic-y += linkage.h

arch/tile/include/asm/Kbuild +1 -0

index aa48b6e..9487778 100644
@@ -12,6 +12,7 @@ generic-y += fcntl.h
 generic-y += hw_irq.h
 generic-y += ioctl.h
 generic-y += ioctls.h
+generic-y += ipv6.h
 generic-y += ipcbuf.h
 generic-y += irq_regs.h
 generic-y += local.h

arch/um/include/asm/Kbuild +1 -0

index e9d42aa..b226353 100644
@@ -11,6 +11,7 @@ generic-y += futex.h
 generic-y += hardirq.h
 generic-y += hw_irq.h
 generic-y += io.h
+generic-y += ipv6.h
 generic-y += irq_regs.h
 generic-y += irq_work.h
 generic-y += kdebug.h

arch/unicore32/include/asm/Kbuild +1 -0

index 84205fe..d01d723 100644
@@ -19,6 +19,7 @@ generic-y += hw_irq.h
 generic-y += ioctl.h
 generic-y += ioctls.h
 generic-y += ipcbuf.h
+generic-y += ipv6.h
 generic-y += irq_regs.h
 generic-y += irq_work.h
 generic-y += kdebug.h

arch/x86/include/asm/Kbuild +1 -0

index 5d6a53f..8a68e12 100644
@@ -9,5 +9,6 @@ generated-y += xen-hypercalls.h
 generic-y += clkdev.h
 generic-y += dma-contiguous.h
 generic-y += early_ioremap.h
+generic-y += ipv6.h
 generic-y += mcs_spinlock.h
 generic-y += mm-arch-hooks.h

arch/xtensa/include/asm/Kbuild +1 -0

index f41408c..dc02075 100644
@@ -9,6 +9,7 @@ generic-y += exec.h
 generic-y += fcntl.h
 generic-y += hardirq.h
 generic-y += ioctl.h
+generic-y += ipv6.h
 generic-y += irq_regs.h
 generic-y += irq_work.h
 generic-y += kdebug.h

include/asm-generic/ipv6.h new file +32 -0

new file mode 100644
index 0000000..754adac
@@ -0,0 +1,32 @@
+/*	Linux INET6 implementation
+ *
+ *	Authors:
+ *	Pedro Roque		<[email protected]>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+
+#ifndef ipv6_masked_addr_cmp
+#define ipv6_masked_addr_cmp ipv6_masked_addr_cmp
+static inline bool
+ipv6_masked_addr_cmp(const struct in6_addr *a1, const struct in6_addr *m,
+		     const struct in6_addr *a2)
+{
+#if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && BITS_PER_LONG == 64
+	const unsigned long *ul1 = (const unsigned long *)a1;
+	const unsigned long *ulm = (const unsigned long *)m;
+	const unsigned long *ul2 = (const unsigned long *)a2;
+
+	return !!(((ul1[0] ^ ul2[0]) & ulm[0]) |
+		  ((ul1[1] ^ ul2[1]) & ulm[1]));
+#else
+	return !!(((a1->s6_addr32[0] ^ a2->s6_addr32[0]) & m->s6_addr32[0]) |
+		  ((a1->s6_addr32[1] ^ a2->s6_addr32[1]) & m->s6_addr32[1]) |
+		  ((a1->s6_addr32[2] ^ a2->s6_addr32[2]) & m->s6_addr32[2]) |
+		  ((a1->s6_addr32[3] ^ a2->s6_addr32[3]) & m->s6_addr32[3]));
+#endif
+}
+#endif

include/net/ipv6.h +1 ~~-19~~

index dbf0abb..08ad1a98 100644
@@ -21,6 +21,7 @@
 #include <net/flow.h>
 #include <net/flow_dissector.h>
 #include <net/snmp.h>
+#include <asm/ipv6.h>
 
 #define SIN6_LEN_RFC2133	24

@@ -385,25 +386,6 @@ static inline int ipv6_addr_cmp(const struct in6_addr *a1, const struct in6_addr
 	return memcmp(a1, a2, sizeof(struct in6_addr));
 }
 
-static inline bool
-ipv6_masked_addr_cmp(const struct in6_addr *a1, const struct in6_addr *m,
-		     const struct in6_addr *a2)
-{
-#if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && BITS_PER_LONG == 64
-	const unsigned long *ul1 = (const unsigned long *)a1;
-	const unsigned long *ulm = (const unsigned long *)m;
-	const unsigned long *ul2 = (const unsigned long *)a2;
-
-	return !!(((ul1[0] ^ ul2[0]) & ulm[0]) |
-		  ((ul1[1] ^ ul2[1]) & ulm[1]));
-#else
-	return !!(((a1->s6_addr32[0] ^ a2->s6_addr32[0]) & m->s6_addr32[0]) |
-		  ((a1->s6_addr32[1] ^ a2->s6_addr32[1]) & m->s6_addr32[1]) |
-		  ((a1->s6_addr32[2] ^ a2->s6_addr32[2]) & m->s6_addr32[2]) |
-		  ((a1->s6_addr32[3] ^ a2->s6_addr32[3]) & m->s6_addr32[3]));
-#endif
-}
-
 static inline void ipv6_addr_prefix(struct in6_addr *pfx, 
 				    const struct in6_addr *addr,
 				    int plen)

Signature · 1 line

1.9.1

Robin Murphy

2017-03-17 12:10 UTC #1 lore raw

Quoted 10 lines · Subash Abhinov Kasiviswanathan

Android devices use multiple ip[6]tables for statistics, UID matching and other functionality. Perf output indicated that ip6_do_table was taking a considerable amount of CPU and more that ip_do_table for an equivalent rate. ipv6_masked_addr_cmp was chosen for optimization as there are more instructions required than the equivalent operation in ip_packet_match.
Using 128 bit operations helps to reduce the number of instructions for the match on an ARM64 system. This helps to improve UDPv6 DL performance by 40Mbps (860Mbps -> 900Mbps) on a CPU limited system.

After trying to have a look at the codegen difference it makes, I think I may have found why it's faster ;)

----------
[root@space-channel-5 ~]# cat > ip.c
#include <stdbool.h>
#include <netinet/in.h>

bool
ipv6_masked_addr_cmp(const struct in6_addr *a1, const struct in6_addr *m,
const struct in6_addr *a2)

{
	const unsigned long *ul1 = (const unsigned long *)a1;
	const unsigned long *ulm = (const unsigned long *)m;
	const unsigned long *ul2 = (const unsigned long *)a2;

	return !!(((ul1[0] ^ ul2[0]) & ulm[0]) |
		  ((ul1[1] ^ ul2[1]) & ulm[1]));
}

bool
ipv6_masked_addr_cmp_new(const struct in6_addr *a1, const struct in6_addr *m,
const struct in6_addr *a2)

{
	const __uint128_t *ul1 = (const __uint128_t *)a1;
	const __uint128_t *ulm = (const __uint128_t *)m;
	const __uint128_t *ul2 = (const __uint128_t *)a1;

	return !!((*ul1 ^ *ul2) & *ulm);
}

[root@space-channel-5 ~]# gcc -c -O2 ip.c
[root@space-channel-5 ~]# objdump -d ip.o

ip.o: file format elf64-littleaarch64

Disassembly of section .text:

0000000000000000 <ipv6_masked_addr_cmp>:
0: a9401847 ldp x7, x6, [x2]
4: a9401003 ldp x3, x4, [x0]
8: f9400025 ldr x5, [x1]
c: f9400422 ldr x2, [x1, #8]
10: ca070060 eor x0, x3, x7
14: ca060081 eor x1, x4, x6
18: 8a050000 and x0, x0, x5
1c: 8a020021 and x1, x1, x2
20: aa010000 orr x0, x0, x1
24: f100001f cmp x0, #0x0
28: 1a9f07e0 cset w0, ne // ne = any
2c: d65f03c0 ret

0000000000000030 <ipv6_masked_addr_cmp_new>:
30: 52800000 mov w0, #0x0 // #0
34: d65f03c0 ret
[root@space-channel-5 ~]# gcc --version
gcc (GCC) 6.3.1 20170306
Copyright (C) 2016 Free Software Foundation, Inc.
This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.

----------

That's clearly not right - I'm not sure quite what undefined behaviour assumption convinces GCC to optimise the whole thing away, but I do note that the generic 64-bit version really isn't far off optimal already. Even if it happens to work out in practice due to inlining behaviour, I don't think that's something we'd want to rely on.

Robin.

Quoted 513 lines

Tested on x86_64 UML to check if generic version is used and ARM64 to verify that ARM64 version is used.

Signed-off-by Subash Abhinov Kasiviswanathan <[email protected]>

---
 arch/alpha/include/asm/Kbuild      |  1 +
 arch/arc/include/asm/Kbuild        |  1 +
 arch/arm/include/asm/Kbuild        |  1 +
 arch/arm64/include/asm/ipv6.h      | 29 +++++++++++++++++++++++++++++
 arch/avr32/include/asm/Kbuild      |  1 +
 arch/blackfin/include/asm/Kbuild   |  1 +
 arch/c6x/include/asm/Kbuild        |  1 +
 arch/cris/include/asm/Kbuild       |  1 +
 arch/frv/include/asm/Kbuild        |  1 +
 arch/h8300/include/asm/Kbuild      |  1 +
 arch/hexagon/include/asm/Kbuild    |  1 +
 arch/ia64/include/asm/Kbuild       |  1 +
 arch/m32r/include/asm/Kbuild       |  1 +
 arch/m68k/include/asm/Kbuild       |  1 +
 arch/metag/include/asm/Kbuild      |  1 +
 arch/microblaze/include/asm/Kbuild |  1 +
 arch/mips/include/asm/Kbuild       |  1 +
 arch/mn10300/include/asm/Kbuild    |  1 +
 arch/nios2/include/asm/Kbuild      |  1 +
 arch/openrisc/include/asm/Kbuild   |  1 +
 arch/parisc/include/asm/Kbuild     |  1 +
 arch/powerpc/include/asm/Kbuild    |  1 +
 arch/s390/include/asm/Kbuild       |  1 +
 arch/score/include/asm/Kbuild      |  1 +
 arch/sh/include/asm/Kbuild         |  1 +
 arch/sparc/include/asm/Kbuild      |  1 +
 arch/tile/include/asm/Kbuild       |  1 +
 arch/um/include/asm/Kbuild         |  1 +
 arch/unicore32/include/asm/Kbuild  |  1 +
 arch/x86/include/asm/Kbuild        |  1 +
 arch/xtensa/include/asm/Kbuild     |  1 +
 include/asm-generic/ipv6.h         | 32 ++++++++++++++++++++++++++++++++
 include/net/ipv6.h                 | 20 +-------------------
 33 files changed, 92 insertions(+), 19 deletions(-)
 create mode 100644 arch/arm64/include/asm/ipv6.h
 create mode 100644 include/asm-generic/ipv6.h

arch/alpha/include/asm/Kbuild +1 -0

index d103db5..5b7e92b 100644
@@ -3,6 +3,7 @@
 generic-y += clkdev.h
 generic-y += exec.h
 generic-y += export.h
+generic-y += ipv6.h
 generic-y += irq_work.h
 generic-y += mcs_spinlock.h
 generic-y += mm-arch-hooks.h

arch/arc/include/asm/Kbuild +1 -0

index 63a0401..99f1456 100644
@@ -14,6 +14,7 @@ generic-y += hw_irq.h
 generic-y += ioctl.h
 generic-y += ioctls.h
 generic-y += ipcbuf.h
+generic-y += ipv6.h
 generic-y += irq_regs.h
 generic-y += irq_work.h
 generic-y += kmap_types.h

arch/arm/include/asm/Kbuild +1 -0

index b14e8c7..a0ba9ac 100644
@@ -9,6 +9,7 @@ generic-y += errno.h
 generic-y += exec.h
 generic-y += ioctl.h
 generic-y += ipcbuf.h
+generic-y += ipv6.h
 generic-y += irq_regs.h
 generic-y += kdebug.h
 generic-y += local.h

arch/arm64/include/asm/ipv6.h new file +29 -0

new file mode 100644
index 0000000..d49dec6
@@ -0,0 +1,29 @@
+/* Copyright (c) 2017, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef __ASM_IPV6_H
+#define __ASM_IPV6_H
+
+#include <linux/types.h>
+
+static inline bool
+ipv6_masked_addr_cmp(const struct in6_addr *a1, const struct in6_addr *m,
+		     const struct in6_addr *a2)
+{
+	const __uint128_t *ul1 = (const __uint128_t *)a1;
+	const __uint128_t *ulm = (const __uint128_t *)m;
+	const __uint128_t *ul2 = (const __uint128_t *)a1;
+
+	return !!((*ul1 ^ *ul2) & *ulm);
+}
+#define ipv6_masked_addr_cmp ipv6_masked_addr_cmp
+#endif /* __ASM_IPV6_H */

arch/avr32/include/asm/Kbuild +1 -0

index 3d7ef2c..fd6a964 100644
@@ -6,6 +6,7 @@ generic-y += div64.h
 generic-y += emergency-restart.h
 generic-y += exec.h
 generic-y += futex.h
+generic-y += ipv6.h
 generic-y += irq_regs.h
 generic-y += irq_work.h
 generic-y += local.h

arch/blackfin/include/asm/Kbuild +1 -0

index 625db8a..f713d85 100644
@@ -12,6 +12,7 @@ generic-y += futex.h
 generic-y += hw_irq.h
 generic-y += ioctl.h
 generic-y += ipcbuf.h
+generic-y += ipv6.h
 generic-y += irq_regs.h
 generic-y += irq_work.h
 generic-y += kdebug.h

arch/c6x/include/asm/Kbuild +1 -0

index 82619c3..ff8033f 100644
@@ -20,6 +20,7 @@ generic-y += io.h
 generic-y += ioctl.h
 generic-y += ioctls.h
 generic-y += ipcbuf.h
+generic-y += ipv6.h
 generic-y += irq_regs.h
 generic-y += irq_work.h
 generic-y += kdebug.h

arch/cris/include/asm/Kbuild +1 -0

index 0f5132b..c5b5fa0 100644
@@ -15,6 +15,7 @@ generic-y += futex.h
 generic-y += hardirq.h
 generic-y += ioctl.h
 generic-y += ipcbuf.h
+generic-y += ipv6.h
 generic-y += irq_regs.h
 generic-y += irq_work.h
 generic-y += kdebug.h

arch/frv/include/asm/Kbuild +1 -0

index c33b467..0717ffb 100644
@@ -1,6 +1,7 @@
 
 generic-y += clkdev.h
 generic-y += exec.h
+generic-y += ipv6.h
 generic-y += irq_work.h
 generic-y += mcs_spinlock.h
 generic-y += mm-arch-hooks.h

arch/h8300/include/asm/Kbuild +1 -0

index 341740c..0058275 100644
@@ -23,6 +23,7 @@ generic-y += hw_irq.h
 generic-y += ioctl.h
 generic-y += ioctls.h
 generic-y += ipcbuf.h
+generic-y += ipv6.h
 generic-y += irq_regs.h
 generic-y += irq_work.h
 generic-y += kdebug.h

arch/hexagon/include/asm/Kbuild +1 -0

index 797b64a..4985925 100644
@@ -20,6 +20,7 @@ generic-y += ioctl.h
 generic-y += ioctls.h
 generic-y += iomap.h
 generic-y += ipcbuf.h
+generic-y += ipv6.h
 generic-y += irq_regs.h
 generic-y += irq_work.h
 generic-y += kdebug.h

arch/ia64/include/asm/Kbuild +1 -0

index 502a91d..cacba4c 100644
@@ -1,6 +1,7 @@
 
 generic-y += clkdev.h
 generic-y += exec.h
+generic-y += ipv6.h
 generic-y += irq_work.h
 generic-y += kvm_para.h
 generic-y += mcs_spinlock.h

arch/m32r/include/asm/Kbuild +1 -0

index deb2987..8ed14f5 100644
@@ -2,6 +2,7 @@
 generic-y += clkdev.h
 generic-y += current.h
 generic-y += exec.h
+generic-y += ipv6.h
 generic-y += irq_work.h
 generic-y += kvm_para.h
 generic-y += mcs_spinlock.h

arch/m68k/include/asm/Kbuild +1 -0

index d4f9ccb..1ddfeac 100644
@@ -9,6 +9,7 @@ generic-y += futex.h
 generic-y += hw_irq.h
 generic-y += ioctl.h
 generic-y += ipcbuf.h
+generic-y += ipv6.h
 generic-y += irq_regs.h
 generic-y += irq_work.h
 generic-y += kdebug.h

arch/metag/include/asm/Kbuild +1 -0

index f9b9df5..2382a6e 100644
@@ -16,6 +16,7 @@ generic-y += hw_irq.h
 generic-y += ioctl.h
 generic-y += ioctls.h
 generic-y += ipcbuf.h
+generic-y += ipv6.h
 generic-y += irq_regs.h
 generic-y += irq_work.h
 generic-y += kdebug.h

arch/microblaze/include/asm/Kbuild +1 -0

index 1732ec1..66b7d8a 100644
@@ -3,6 +3,7 @@ generic-y += barrier.h
 generic-y += clkdev.h
 generic-y += device.h
 generic-y += exec.h
+generic-y += ipv6.h
 generic-y += irq_work.h
 generic-y += mcs_spinlock.h
 generic-y += mm-arch-hooks.h

arch/mips/include/asm/Kbuild +1 -0

index 2535c7b..4453f33 100644
@@ -5,6 +5,7 @@ generic-y += current.h
 generic-y += dma-contiguous.h
 generic-y += emergency-restart.h
 generic-y += export.h
+generic-y += ipv6.h
 generic-y += irq_work.h
 generic-y += local64.h
 generic-y += mcs_spinlock.h

arch/mn10300/include/asm/Kbuild +1 -0

index 97f64c7..df55c2b 100644
@@ -2,6 +2,7 @@
 generic-y += barrier.h
 generic-y += clkdev.h
 generic-y += exec.h
+generic-y += ipv6.h
 generic-y += irq_work.h
 generic-y += mcs_spinlock.h
 generic-y += mm-arch-hooks.h

arch/nios2/include/asm/Kbuild +1 -0

index aaa3c21..d6b13ee 100644
@@ -22,6 +22,7 @@ generic-y += hw_irq.h
 generic-y += ioctl.h
 generic-y += ioctls.h
 generic-y += ipcbuf.h
+generic-y += ipv6.h
 generic-y += irq_regs.h
 generic-y += irq_work.h
 generic-y += kdebug.h

arch/openrisc/include/asm/Kbuild +1 -0

index fb01873..0d9ad5a 100644
@@ -25,6 +25,7 @@ generic-y += ioctl.h
 generic-y += ioctls.h
 generic-y += ipcbuf.h
 generic-y += irq.h
+generic-y += ipv6.h
 generic-y += irq_regs.h
 generic-y += irq_work.h
 generic-y += kdebug.h

arch/parisc/include/asm/Kbuild +1 -0

index a9909c2..b0e156e 100644
@@ -8,6 +8,7 @@ generic-y += div64.h
 generic-y += emergency-restart.h
 generic-y += exec.h
 generic-y += hw_irq.h
+generic-y += ipv6.h
 generic-y += irq_regs.h
 generic-y += irq_work.h
 generic-y += kdebug.h

arch/powerpc/include/asm/Kbuild +1 -0

index 5c4fbc8..f675f7c 100644
@@ -1,6 +1,7 @@
 generic-y += clkdev.h
 generic-y += div64.h
 generic-y += export.h
+generic-y += ipv6.h
 generic-y += irq_regs.h
 generic-y += irq_work.h
 generic-y += local64.h

arch/s390/include/asm/Kbuild +1 -0

index 8aea32f..53a2335 100644
@@ -2,6 +2,7 @@ generic-y += asm-offsets.h
 generic-y += clkdev.h
 generic-y += dma-contiguous.h
 generic-y += export.h
+generic-y += ipv6.h
 generic-y += irq_work.h
 generic-y += mcs_spinlock.h
 generic-y += mm-arch-hooks.h

arch/score/include/asm/Kbuild +1 -0

index 926943a..9405456 100644
@@ -4,6 +4,7 @@ header-y +=
 generic-y += barrier.h
 generic-y += clkdev.h
 generic-y += current.h
+generic-y += ipv6.h
 generic-y += irq_work.h
 generic-y += mcs_spinlock.h
 generic-y += mm-arch-hooks.h

arch/sh/include/asm/Kbuild +1 -0

index cf2a750..3984834 100644
@@ -10,6 +10,7 @@ generic-y += exec.h
 generic-y += fcntl.h
 generic-y += ioctl.h
 generic-y += ipcbuf.h
+generic-y += ipv6.h
 generic-y += irq_regs.h
 generic-y += irq_work.h
 generic-y += kvm_para.h

arch/sparc/include/asm/Kbuild +1 -0

index e9e837b..d2acdaf 100644
@@ -6,6 +6,7 @@ generic-y += div64.h
 generic-y += emergency-restart.h
 generic-y += exec.h
 generic-y += export.h
+generic-y += ipv6.h
 generic-y += irq_regs.h
 generic-y += irq_work.h
 generic-y += linkage.h

arch/tile/include/asm/Kbuild +1 -0

index aa48b6e..9487778 100644
@@ -12,6 +12,7 @@ generic-y += fcntl.h
 generic-y += hw_irq.h
 generic-y += ioctl.h
 generic-y += ioctls.h
+generic-y += ipv6.h
 generic-y += ipcbuf.h
 generic-y += irq_regs.h
 generic-y += local.h

arch/um/include/asm/Kbuild +1 -0

index e9d42aa..b226353 100644
@@ -11,6 +11,7 @@ generic-y += futex.h
 generic-y += hardirq.h
 generic-y += hw_irq.h
 generic-y += io.h
+generic-y += ipv6.h
 generic-y += irq_regs.h
 generic-y += irq_work.h
 generic-y += kdebug.h

arch/unicore32/include/asm/Kbuild +1 -0

index 84205fe..d01d723 100644
@@ -19,6 +19,7 @@ generic-y += hw_irq.h
 generic-y += ioctl.h
 generic-y += ioctls.h
 generic-y += ipcbuf.h
+generic-y += ipv6.h
 generic-y += irq_regs.h
 generic-y += irq_work.h
 generic-y += kdebug.h

arch/x86/include/asm/Kbuild +1 -0

index 5d6a53f..8a68e12 100644
@@ -9,5 +9,6 @@ generated-y += xen-hypercalls.h
 generic-y += clkdev.h
 generic-y += dma-contiguous.h
 generic-y += early_ioremap.h
+generic-y += ipv6.h
 generic-y += mcs_spinlock.h
 generic-y += mm-arch-hooks.h

arch/xtensa/include/asm/Kbuild +1 -0

index f41408c..dc02075 100644
@@ -9,6 +9,7 @@ generic-y += exec.h
 generic-y += fcntl.h
 generic-y += hardirq.h
 generic-y += ioctl.h
+generic-y += ipv6.h
 generic-y += irq_regs.h
 generic-y += irq_work.h
 generic-y += kdebug.h

include/asm-generic/ipv6.h new file +32 -0

new file mode 100644
index 0000000..754adac
@@ -0,0 +1,32 @@
+/*	Linux INET6 implementation
+ *
+ *	Authors:
+ *	Pedro Roque		<[email protected]>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+
+#ifndef ipv6_masked_addr_cmp
+#define ipv6_masked_addr_cmp ipv6_masked_addr_cmp
+static inline bool
+ipv6_masked_addr_cmp(const struct in6_addr *a1, const struct in6_addr *m,
+		     const struct in6_addr *a2)
+{
+#if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && BITS_PER_LONG == 64
+	const unsigned long *ul1 = (const unsigned long *)a1;
+	const unsigned long *ulm = (const unsigned long *)m;
+	const unsigned long *ul2 = (const unsigned long *)a2;
+
+	return !!(((ul1[0] ^ ul2[0]) & ulm[0]) |
+		  ((ul1[1] ^ ul2[1]) & ulm[1]));
+#else
+	return !!(((a1->s6_addr32[0] ^ a2->s6_addr32[0]) & m->s6_addr32[0]) |
+		  ((a1->s6_addr32[1] ^ a2->s6_addr32[1]) & m->s6_addr32[1]) |
+		  ((a1->s6_addr32[2] ^ a2->s6_addr32[2]) & m->s6_addr32[2]) |
+		  ((a1->s6_addr32[3] ^ a2->s6_addr32[3]) & m->s6_addr32[3]));
+#endif
+}
+#endif

include/net/ipv6.h +1 ~~-19~~

index dbf0abb..08ad1a98 100644
@@ -21,6 +21,7 @@
 #include <net/flow.h>
 #include <net/flow_dissector.h>
 #include <net/snmp.h>
+#include <asm/ipv6.h>
 
 #define SIN6_LEN_RFC2133	24

@@ -385,25 +386,6 @@ static inline int ipv6_addr_cmp(const struct in6_addr *a1, const struct in6_addr
 	return memcmp(a1, a2, sizeof(struct in6_addr));
 }
 
-static inline bool
-ipv6_masked_addr_cmp(const struct in6_addr *a1, const struct in6_addr *m,
-		     const struct in6_addr *a2)
-{
-#if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && BITS_PER_LONG == 64
-	const unsigned long *ul1 = (const unsigned long *)a1;
-	const unsigned long *ulm = (const unsigned long *)m;
-	const unsigned long *ul2 = (const unsigned long *)a2;
-
-	return !!(((ul1[0] ^ ul2[0]) & ulm[0]) |
-		  ((ul1[1] ^ ul2[1]) & ulm[1]));
-#else
-	return !!(((a1->s6_addr32[0] ^ a2->s6_addr32[0]) & m->s6_addr32[0]) |
-		  ((a1->s6_addr32[1] ^ a2->s6_addr32[1]) & m->s6_addr32[1]) |
-		  ((a1->s6_addr32[2] ^ a2->s6_addr32[2]) & m->s6_addr32[2]) |
-		  ((a1->s6_addr32[3] ^ a2->s6_addr32[3]) & m->s6_addr32[3]));
-#endif
-}
-
 static inline void ipv6_addr_prefix(struct in6_addr *pfx, 
 				    const struct in6_addr *addr,
 				    int plen)

James Greenhalgh

2017-03-17 12:23 UTC #2 lore raw

Quoted 43 lines · Robin Murphy

Quoted 10 lines · Subash Abhinov Kasiviswanathan
Android devices use multiple ip[6]tables for statistics, UID matching and other functionality. Perf output indicated that ip6_do_table was taking a considerable amount of CPU and more that ip_do_table for an equivalent rate. ipv6_masked_addr_cmp was chosen for optimization as there are more instructions required than the equivalent operation in ip_packet_match.
Using 128 bit operations helps to reduce the number of instructions for the match on an ARM64 system. This helps to improve UDPv6 DL performance by 40Mbps (860Mbps -> 900Mbps) on a CPU limited system.
After trying to have a look at the codegen difference it makes, I think I may have found why it's faster ;)
----------
[root@space-channel-5 ~]# cat > ip.c
#include <stdbool.h>
#include <netinet/in.h>
bool
ipv6_masked_addr_cmp(const struct in6_addr *a1, const struct in6_addr *m,
const struct in6_addr *a2)
{
	const unsigned long *ul1 = (const unsigned long *)a1;
	const unsigned long *ulm = (const unsigned long *)m;
	const unsigned long *ul2 = (const unsigned long *)a2;

	return !!(((ul1[0] ^ ul2[0]) & ulm[0]) |
		  ((ul1[1] ^ ul2[1]) & ulm[1]));
}
bool
ipv6_masked_addr_cmp_new(const struct in6_addr *a1, const struct in6_addr *m,
const struct in6_addr *a2)
{
	const __uint128_t *ul1 = (const __uint128_t *)a1;
	const __uint128_t *ulm = (const __uint128_t *)m;
	const __uint128_t *ul2 = (const __uint128_t *)a1;

	return !!((*ul1 ^ *ul2) & *ulm);
}

<snip>

Quoted 2 lines

That's clearly not right - I'm not sure quite what undefined behaviour assumption convinces GCC to optimise the whole thing away>

While the pointer casting is a bit ghastly, I don't actually think that GCC is taking advantage of undefined behaviour here, rather it looks like you have a simple typo on line 3:

Quoted 3 lines

	const __uint128_t *ul1 = (const __uint128_t *)a1;
	const __uint128_t *ulm = (const __uint128_t *)m;
	const __uint128_t *ul2 = (const __uint128_t *)a1;

ul2 = a2, surely?

As it is (stripping casts) you have a1 ^ a1, which will get you to 0 pretty quickly. Fixing that up for you;

  bool
  ipv6_masked_addr_cmp_new(const struct in6_addr *a1, const struct
  in6_addr *m,
  		     const struct in6_addr *a2)
  {
  	const __uint128_t *ul1 = (const __uint128_t *)a1;
  	const __uint128_t *ulm = (const __uint128_t *)m;
  	const __uint128_t *ul2 = (const __uint128_t *)a2;

  	return !!((*ul1 ^ *ul2) & *ulm);
  }

$ gcc -O2

ipv6_masked_addr_cmp_new:

	ldp	x4, x3, [x0]
	ldp	x5, x2, [x2]
	ldp	x0, x1, [x1]
	eor	x4, x4, x5
	eor	x2, x3, x2
	and	x0, x0, x4
	and	x1, x1, x2
	orr	x0, x0, x1
	cmp	x0, 0
	cset	w0, ne
	ret

Which at least looks like it might calculate something useful :-)

Cheers,
James

Subash Abhinov Kasiviswanathan

author

2017-03-17 21:20 UTC #3 lore raw

Quoted 44 lines

Quoted 3 lines
That's clearly not right - I'm not sure quite what undefined behaviour assumption convinces GCC to optimise the whole thing away>
While the pointer casting is a bit ghastly, I don't actually think that GCC is taking advantage of undefined behaviour here, rather it looks like you have a simple typo on line 3:
Quoted 3 lines
	const __uint128_t *ul1 = (const __uint128_t *)a1;
	const __uint128_t *ulm = (const __uint128_t *)m;
	const __uint128_t *ul2 = (const __uint128_t *)a1;
ul2 = a2, surely?
As it is (stripping casts) you have a1 ^ a1, which will get you to 0 pretty quickly. Fixing that up for you;
  bool
  ipv6_masked_addr_cmp_new(const struct in6_addr *a1, const struct
  in6_addr *m,
  		     const struct in6_addr *a2)
  {
  	const __uint128_t *ul1 = (const __uint128_t *)a1;
  	const __uint128_t *ulm = (const __uint128_t *)m;
  	const __uint128_t *ul2 = (const __uint128_t *)a2;

  	return !!((*ul1 ^ *ul2) & *ulm);
  }
$ gcc -O2
ipv6_masked_addr_cmp_new:
	ldp	x4, x3, [x0]
	ldp	x5, x2, [x2]
	ldp	x0, x1, [x1]
	eor	x4, x4, x5
	eor	x2, x3, x2
	and	x0, x0, x4
	and	x1, x1, x2
	orr	x0, x0, x1
	cmp	x0, 0
	cset	w0, ne
	ret

Quoted 2 lines

Which at least looks like it might calculate something useful :-)

Hi Robin / James

Thanks for checking and sorry for the confusion. I'll retest this.