arm64: use SHA3 instructions to speed up XOR
cover arm64: use SHA3 instructions to speed up XOR
If available, use the 3-way EOR3 instruction that is part of the SHA3 extension rather than the ordinary 2-way EOR NEON instruction. Doing so speeds up XOR processing by ~20% on Apple M1 when using the 5-way version.
This is also useful as a canary for spotting regressions in the static call API, as we export a static call from one module and consume it in another.
Ard Biesheuvel (2):
arm64/xor: use static calls for inner NEON helpers arm64/xor: use EOR3 instructions when available
arch/arm64/Kconfig | 3 + arch/arm64/include/asm/xor.h | 24 ++- arch/arm64/lib/xor-neon.c | 165 +++++++++++++++++++- 3 files changed, 179 insertions(+), 13 deletions(-)
Signature · 7 lines
2.30.2 _______________________________________________ linux-arm-kernel mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
PATCH 1/2 arm64/xor: use static calls for inner NEON helpers
Call the inner NEON helpers using static calls rather than loading their addresses from a struct. This will be used in a subsequent patch to switch between NEON and SHA3 based implementations of the XOR code.
--- arch/arm64/include/asm/xor.h | 24 ++++++++++++++++---- arch/arm64/lib/xor-neon.c | 20 +++++++++------- 2 files changed, 31 insertions(+), 13 deletions(-)
arch/arm64/include/asm/xor.h +19 -5
@@ -7,19 +7,33 @@*/ #include <linux/hardirq.h> +#include <linux/static_call.h> #include <asm-generic/xor.h> #include <asm/hwcap.h> #include <asm/neon.h> #ifdef CONFIG_KERNEL_MODE_NEON -extern struct xor_block_template const xor_block_inner_neon; +void xor_arm64_neon_2 (unsigned long bytes,unsigned long *p1, +unsigned long *p2); +void xor_arm64_neon_3 (unsigned long bytes,unsigned long *p1, +unsigned long *p2,unsigned long *p3); +void xor_arm64_neon_4 (unsigned long bytes,unsigned long *p1, +unsigned long *p2,unsigned long *p3, +unsigned long *p4); +void xor_arm64_neon_5 (unsigned long bytes,unsigned long *p1, +unsigned long *p2,unsigned long *p3, +unsigned long *p4,unsigned long *p5); + +DECLARE_STATIC_CALL (xor_arm64_3, xor_arm64_neon_3); +DECLARE_STATIC_CALL (xor_arm64_4, xor_arm64_neon_4); +DECLARE_STATIC_CALL (xor_arm64_5, xor_arm64_neon_5);static void xor_neon_2 (unsigned long bytes,unsigned long *p1,unsigned long *p2) {kernel_neon_begin (); - xor_block_inner_neon.do_2 (bytes, p1, p2); +xor_arm64_neon_2 (bytes, p1, p2);kernel_neon_end (); }
@@ -28,7 +42,7 @@ xor_neon_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, unsigned long *p3) {kernel_neon_begin (); - xor_block_inner_neon.do_3 (bytes, p1, p2, p3); +static_call (xor_arm64_3)(bytes, p1, p2, p3);kernel_neon_end (); }
@@ -37,7 +51,7 @@ xor_neon_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, unsigned long *p3, unsigned long *p4) {kernel_neon_begin (); - xor_block_inner_neon.do_4 (bytes, p1, p2, p3, p4); +static_call (xor_arm64_4)(bytes, p1, p2, p3, p4);kernel_neon_end (); }
@@ -46,7 +60,7 @@ xor_neon_5(unsigned long bytes, unsigned long *p1, unsigned long *p2, unsigned long *p3, unsigned long *p4, unsigned long *p5) {kernel_neon_begin (); - xor_block_inner_neon.do_5 (bytes, p1, p2, p3, p4, p5); +static_call (xor_arm64_5)(bytes, p1, p2, p3, p4, p5);kernel_neon_end (); }
arch/arm64/lib/xor-neon.c +12 -8
@@ -7,6 +7,7 @@*/ #include <linux/raid/xor.h> +#include <linux/static_call.h> #include <linux/module.h> #include <asm/neon-intrinsics.h>
@@ -36,6 +37,7 @@ void xor_arm64_neon_2(unsigned long bytes, unsigned long *p1, dp2 +=8 ; }while (--lines >0 ); } +EXPORT_SYMBOL (xor_arm64_neon_2);void xor_arm64_neon_3 (unsigned long bytes,unsigned long *p1,unsigned long *p2,unsigned long *p3)
@@ -71,6 +73,7 @@ void xor_arm64_neon_3(unsigned long bytes, unsigned long *p1, dp3 +=8 ; }while (--lines >0 ); } +EXPORT_SYMBOL (xor_arm64_neon_3);void xor_arm64_neon_4 (unsigned long bytes,unsigned long *p1,unsigned long *p2,unsigned long *p3,unsigned long *p4)
@@ -114,6 +117,7 @@ void xor_arm64_neon_4(unsigned long bytes, unsigned long *p1, dp4 +=8 ; }while (--lines >0 ); } +EXPORT_SYMBOL (xor_arm64_neon_4); void xor_arm64_neon_5(unsigned long bytes, unsigned long *p1, unsigned long *p2, unsigned long *p3,
@@ -166,15 +170,15 @@ void xor_arm64_neon_5(unsigned long bytes, unsigned long *p1, dp5 +=8 ; }while (--lines >0 ); } +EXPORT_SYMBOL (xor_arm64_neon_5); -struct xor_block_template const xor_block_inner_neon = { - .name ="__inner_neon__" , - .do_2 = xor_arm64_neon_2, - .do_3 = xor_arm64_neon_3, - .do_4 = xor_arm64_neon_4, - .do_5 = xor_arm64_neon_5, -}; -EXPORT_SYMBOL (xor_block_inner_neon); +DEFINE_STATIC_CALL (xor_arm64_3, xor_arm64_neon_3); +DEFINE_STATIC_CALL (xor_arm64_4, xor_arm64_neon_4); +DEFINE_STATIC_CALL (xor_arm64_5, xor_arm64_neon_5); + +EXPORT_STATIC_CALL (xor_arm64_3); +EXPORT_STATIC_CALL (xor_arm64_4); +EXPORT_STATIC_CALL (xor_arm64_5);MODULE_AUTHOR ("Jackie Liu <[email protected]>" );MODULE_DESCRIPTION ("ARMv8 XOR Extensions" );
Signature · 7 lines
2.30.2 _______________________________________________ linux-arm-kernel mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
PATCH 2/2 arm64/xor: use EOR3 instructions when available
In reply to Ard BiesheuvelUse the EOR3 instruction to implement xor_blocks() if the instruction is available, which is the case if the CPU implements the SHA-3 extension. This is about 20% faster on Apple M1 when using the 5-way version.
--- arch/arm64/Kconfig | 3 + arch/arm64/lib/xor-neon.c | 145 ++++++++++++++++++++ 2 files changed, 148 insertions(+)
arch/arm64/Kconfig +3 -0
@@ -2034,6 +2034,9 @@ config SYSVIPC_COMPAT def_bool y depends on COMPAT && SYSVIPC +config CC_HAVE_SHA3 + def_bool $(cc-option, -march=armv8.2-a+sha3) + menu "Power management options" source "kernel/power/Kconfig"
arch/arm64/lib/xor-neon.c +145 -0
157 lines not on this page · raw message
Signature · 7 lines
2.30.2 _______________________________________________ linux-arm-kernel mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
Catalin Marinas
maintainerRe: [PATCH 2/2] arm64/xor: use EOR3 instructions when available
Hi Ard,
I trust you on the algorithm but some minor issues below.
arch/arm64/Kconfig config SYSVIPC_COMPAT · line 2038
Quoted 10 lines · Ard Biesheuvel
arch/arm64/Kconfig +2
-0@@ -2034,6 +2034,9 @@ config SYSVIPC_COMPAT def_bool y depends on COMPAT && SYSVIPC +config CC_HAVE_SHA3 + def_bool $(cc-option, -march=armv8.2-a+sha3)
Is it the compiler or the assembler that we need to support this? I think it's sufficient to only check the latter.
I'd also move it to the ARMv8.2 section.
Quoted 11 lines
arch/arm64/Kconfig +1
-0+ menu "Power management options" source "kernel/power/Kconfig"arch/arm64/lib/xor-neon.c +0
-0@@ -172,6 +172,135 @@ void xor_arm64_neon_5(unsigned long bytes, unsigned long *p1, }EXPORT_SYMBOL (xor_arm64_neon_5);
arch/arm64/lib/xor-neon.c void xor_arm64_neon_5(unsigned long bytes, unsigned long *p1, · line 183
Quoted 10 lines
arch/arm64/lib/xor-neon.c +9
-0+static inline uint64x2_t eor3 (uint64x2_t p,uint64x2_t q,uint64x2_t r) +{ +uint64x2_t res; + + asm(".arch armv8.2-a+sha3 \n" +"eor3 %0.16b, %1.16b, %2.16b, %3.16b" + :"=w" (res) :"w" (p),"w" (q),"w" (r)); +return res; +}
The .arch here may confuse the compiler/assembler since it overrides any other .arch. I think this diff on top would do but I haven't extensively tested it. I can fold it in if you give it a try:
arch/arm64/Kconfig +6 -3
@@ -1545,6 +1545,12 @@ endmenu menu "ARMv8.2 architectural features" +config AS_HAS_ARMV8_2 + def_bool $(cc-option,-Wa$(comma)-march=armv8.2-a) + +config AS_HAS_SHA3 + def_bool $(as-instr,.arch armv8.2-a+sha3) + config ARM64_PMEM bool "Enable support for persistent memory" select ARCH_HAS_PMEM_API
@@ -2032,9 +2038,6 @@ config SYSVIPC_COMPAT def_bool y depends on COMPAT && SYSVIPC -config CC_HAVE_SHA3 - def_bool $(cc-option, -march=armv8.2-a+sha3) - menu "Power management options" source "kernel/power/Kconfig"
arch/arm64/Makefile +5 -0
@@ -58,6 +58,11 @@ stack_protector_prepare: prepare0 include/generated/asm-offsets.h)) endif +ifeq ($(CONFIG_AS_HAS_ARMV8_2), y) +# make sure to pass the newest target architecture to -march. +asm-arch := armv8.2-a +endif + # Ensure that if the compiler supports branch protection we default it # off, this will be overridden if we are using branch protection. branch-prot-flags-y += $(call cc-option,-mbranch-protection=none)
arch/arm64/lib/xor-neon.c +2 -2
@@ -176,7 +176,7 @@ static inline uint64x2_t eor3(uint64x2_t p, uint64x2_t q, uint64x2_t r) {uint64x2_t res; - asm(".arch armv8.2-a+sha3 \n" + asm(ARM64_ASM_PREAMBLE ".arch_extension sha3\n" "eor3 %0.16b, %1.16b, %2.16b, %3.16b" :"=w" (res) :"w" (p),"w" (q),"w" (r));return res;
@@ -311,7 +311,7 @@ EXPORT_STATIC_CALL(xor_arm64_5);static int __init xor_neon_init (void ) { -if (IS_ENABLED (CONFIG_CC_HAVE_SHA3 ) &&cpu_have_named_feature (SHA3 )) { +if (IS_ENABLED (CONFIG_AS_HAS_SHA3 ) &&cpu_have_named_feature (SHA3 )) {static_call_update (xor_arm64_3, xor_arm64_eor3_3);static_call_update (xor_arm64_4, xor_arm64_eor3_4);static_call_update (xor_arm64_5, xor_arm64_eor3_5);
Signature · 6 lines
Catalin _______________________________________________ linux-arm-kernel mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
Re: [PATCH 2/2] arm64/xor: use EOR3 instructions when available
Quoted 34 lines · Catalin Marinas
Hi Ard,
I trust you on the algorithm but some minor issues below.
arch/arm64/Kconfig config SYSVIPC_COMPAT · line 2038
Quoted 10 lines · Ard Biesheuvel
arch/arm64/Kconfig +2
-0@@ -2034,6 +2034,9 @@ config SYSVIPC_COMPAT def_bool y depends on COMPAT && SYSVIPC +config CC_HAVE_SHA3 + def_bool $(cc-option, -march=armv8.2-a+sha3)Is it the compiler or the assembler that we need to support this? I think it's sufficient to only check the latter.
I'd also move it to the ARMv8.2 section.
Quoted 12 lines
arch/arm64/Kconfig +1
-0+ menu "Power management options" source "kernel/power/Kconfig"arch/arm64/lib/xor-neon.c +0
-0@@ -172,6 +172,135 @@ void xor_arm64_neon_5(unsigned long bytes, unsigned long *p1, }EXPORT_SYMBOL (xor_arm64_neon_5);
Quoted 14 lines
arch/arm64/lib/xor-neon.c void xor_arm64_neon_5(unsigned long bytes, unsigned long *p1, · line 183
Quoted 9 lines
arch/arm64/lib/xor-neon.c +9
-0+static inline uint64x2_t eor3 (uint64x2_t p,uint64x2_t q,uint64x2_t r) +{ +uint64x2_t res; + + asm(".arch armv8.2-a+sha3 \n" +"eor3 %0.16b, %1.16b, %2.16b, %3.16b" + :"=w" (res) :"w" (p),"w" (q),"w" (r)); +return res; +}The .arch here may confuse the compiler/assembler since it overrides any other .arch. I think this diff on top would do but I haven't extensively tested it. I can fold it in if you give it a try:
I was going to respin this without the static_call changes, since those are not going to land anytime soon, and for this code, it doesn't really matter anyway. I'll fold in your diff and test it as well.
Quoted 67 lines
arch/arm64/Kconfig +6
-3@@ -1545,6 +1545,12 @@ endmenu menu "ARMv8.2 architectural features" +config AS_HAS_ARMV8_2 + def_bool $(cc-option,-Wa$(comma)-march=armv8.2-a) + +config AS_HAS_SHA3 + def_bool $(as-instr,.arch armv8.2-a+sha3) + config ARM64_PMEM bool "Enable support for persistent memory" select ARCH_HAS_PMEM_API@@ -2032,9 +2038,6 @@ config SYSVIPC_COMPAT def_bool y depends on COMPAT && SYSVIPC -config CC_HAVE_SHA3 - def_bool $(cc-option, -march=armv8.2-a+sha3) - menu "Power management options" source "kernel/power/Kconfig"arch/arm64/Makefile +5
-0@@ -58,6 +58,11 @@ stack_protector_prepare: prepare0 include/generated/asm-offsets.h)) endif +ifeq ($(CONFIG_AS_HAS_ARMV8_2), y) +# make sure to pass the newest target architecture to -march. +asm-arch := armv8.2-a +endif + # Ensure that if the compiler supports branch protection we default it # off, this will be overridden if we are using branch protection. branch-prot-flags-y += $(call cc-option,-mbranch-protection=none)arch/arm64/lib/xor-neon.c +2
-3@@ -176,7 +176,7 @@ static inline uint64x2_t eor3(uint64x2_t p, uint64x2_t q, uint64x2_t r) {uint64x2_t res; - asm(".arch armv8.2-a+sha3 \n" + asm(ARM64_ASM_PREAMBLE ".arch_extension sha3\n" "eor3 %0.16b, %1.16b, %2.16b, %3.16b" :"=w" (res) :"w" (p),"w" (q),"w" (r));return res;@@ -311,7 +311,7 @@ EXPORT_STATIC_CALL(xor_arm64_5);static int __init xor_neon_init (void ) { -if (IS_ENABLED (CONFIG_CC_HAVE_SHA3 ) &&cpu_have_named_feature (SHA3 )) { +if (IS_ENABLED (CONFIG_AS_HAS_SHA3 ) &&cpu_have_named_feature (SHA3 )) {static_call_update (xor_arm64_3, xor_arm64_eor3_3);static_call_update (xor_arm64_4, xor_arm64_eor3_4);static_call_update (xor_arm64_5, xor_arm64_eor3_5); --
Quoted 1 line
Catalin
_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
Catalin Marinas
maintainerRe: [PATCH 2/2] arm64/xor: use EOR3 instructions when available
Quoted 44 lines · Ard Biesheuvel
Quoted 29 lines · Catalin Marinas
arch/arm64/Kconfig config SYSVIPC_COMPAT · line 2038
Quoted 10 lines · Ard Biesheuvel
arch/arm64/Kconfig +2
-0@@ -2034,6 +2034,9 @@ config SYSVIPC_COMPAT def_bool y depends on COMPAT && SYSVIPC +config CC_HAVE_SHA3 + def_bool $(cc-option, -march=armv8.2-a+sha3)Is it the compiler or the assembler that we need to support this? I think it's sufficient to only check the latter.
I'd also move it to the ARMv8.2 section.
Quoted 12 lines
arch/arm64/Kconfig +1
-0+ menu "Power management options" source "kernel/power/Kconfig"arch/arm64/lib/xor-neon.c +0
-0@@ -172,6 +172,135 @@ void xor_arm64_neon_5(unsigned long bytes, unsigned long *p1, }EXPORT_SYMBOL (xor_arm64_neon_5);Quoted 13 lines
arch/arm64/lib/xor-neon.c void xor_arm64_neon_5(unsigned long bytes, unsigned long *p1, · line 183
Quoted 9 lines
arch/arm64/lib/xor-neon.c +9
-0+static inline uint64x2_t eor3 (uint64x2_t p,uint64x2_t q,uint64x2_t r) +{ +uint64x2_t res; + + asm(".arch armv8.2-a+sha3 \n" +"eor3 %0.16b, %1.16b, %2.16b, %3.16b" + :"=w" (res) :"w" (p),"w" (q),"w" (r)); +return res; +}The .arch here may confuse the compiler/assembler since it overrides any other .arch. I think this diff on top would do but I haven't extensively tested it. I can fold it in if you give it a try:
Quoted 2 lines
I was going to respin this without the
static_callchanges, since those are not going to land anytime soon,
I thought the generic implementation still works, though not the most efficient.
Quoted 3 lines
and for this code, it
doesn't really matter anyway. I'll fold in your diff and test it as well.
Sounds fine to me.
Signature · 6 lines
Catalin _______________________________________________ linux-arm-kernel mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
Re: [PATCH 2/2] arm64/xor: use EOR3 instructions when available
Quoted 46 lines · Catalin Marinas
Quoted 44 lines · Ard Biesheuvel
Quoted 42 lines · Catalin Marinas
arch/arm64/Kconfig config SYSVIPC_COMPAT · line 2038
Quoted 10 lines · Ard Biesheuvel
arch/arm64/Kconfig +2
-0@@ -2034,6 +2034,9 @@ config SYSVIPC_COMPAT def_bool y depends on COMPAT && SYSVIPC +config CC_HAVE_SHA3 + def_bool $(cc-option, -march=armv8.2-a+sha3)Is it the compiler or the assembler that we need to support this? I think it's sufficient to only check the latter.
I'd also move it to the ARMv8.2 section.
Quoted 11 lines
arch/arm64/Kconfig +1
-0+ menu "Power management options" source "kernel/power/Kconfig"arch/arm64/lib/xor-neon.c +0
-0@@ -172,6 +172,135 @@ void xor_arm64_neon_5(unsigned long bytes, unsigned long *p1, }EXPORT_SYMBOL (xor_arm64_neon_5);arch/arm64/lib/xor-neon.c void xor_arm64_neon_5(unsigned long bytes, unsigned long *p1, · line 183
Quoted 10 lines
arch/arm64/lib/xor-neon.c +9
-0+static inline uint64x2_t eor3 (uint64x2_t p,uint64x2_t q,uint64x2_t r) +{ +uint64x2_t res; + + asm(".arch armv8.2-a+sha3 \n" +"eor3 %0.16b, %1.16b, %2.16b, %3.16b" + :"=w" (res) :"w" (p),"w" (q),"w" (r)); +return res; +}The .arch here may confuse the compiler/assembler since it overrides any other .arch. I think this diff on top would do but I haven't extensively tested it. I can fold it in if you give it a try:
Quoted 6 lines
Quoted 2 lines
I was going to respin this without the
static_callchanges, since those are not going to land anytime soon,I thought the generic implementation still works, though not the most efficient.
It does work, but the existing code already uses function pointers, so at this point, it is just unneeded churn.
Quoted 8 lines
Quoted 3 lines
and for this code, it
doesn't really matter anyway. I'll fold in your diff and test it as well.Sounds fine to me.
--
Catalin
_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel