xor: enable auto-vectorization in Clang
Subsystems (10)
v4 cover xor: enable auto-vectorization in Clang
Update the xor_blocks() prototypes so that the compiler understands that the inputs always refer to distinct regions of memory. This is implied by the existing implementations, as they use different granularities for the load/xor/store loops.
With that, we can fix the ARM/Clang version, which refuses to SIMD vectorize otherwise, and throws a spurious warning related to the GCC version being incompatible.
Changes since v3:
- revert broken PPC argument rename - doing it fully results in too much pointless churn, and the 'inner' altivec routines are not strictly part of the xor_blocks API anyway
Changes since v2:
- fix arm64 build after rebase
- name PPC argument names consistently
- add Nick's acks and link tags
Changes since v1:
- fix PPC build
- add Nathan's Tested-by
Ard Biesheuvel (2):
lib/xor: make xor prototypes more friendly to compiler vectorization
crypto: arm/xor - make vectorized C code Clang-friendly
arch/alpha/include/asm/xor.h | 53 ++++++++---- arch/arm/include/asm/xor.h | 42 ++++++---- arch/arm/lib/xor-neon.c | 12 +-- arch/arm64/include/asm/xor.h | 21 +++-- arch/arm64/lib/xor-neon.c | 46 +++++++---- arch/ia64/include/asm/xor.h | 21 +++-- arch/powerpc/include/asm/xor_altivec.h | 25 +++--- arch/powerpc/lib/xor_vmx.c | 28 ++++--- arch/powerpc/lib/xor_vmx.h | 27 ++++--- arch/powerpc/lib/xor_vmx_glue.c | 32 ++++---- arch/s390/lib/xor.c | 21 +++-- arch/sparc/include/asm/xor_32.h | 21 +++-- arch/sparc/include/asm/xor_64.h | 42 ++++++---- arch/x86/include/asm/xor.h | 42 ++++++---- arch/x86/include/asm/xor_32.h | 42 ++++++---- arch/x86/include/asm/xor_avx.h | 21 +++-- include/asm-generic/xor.h | 84 +++++++++++++------- include/linux/raid/xor.h | 21 +++-- 18 files changed, 384 insertions(+), 217 deletions(-)
Signature · 1 line
2.30.2
PATCH v4 1/2 lib/xor: make xor prototypes more friendly to compiler vectorization
Modern compilers are perfectly capable of extracting parallelism from the XOR routines, provided that the prototypes reflect the nature of the input accurately, in particular, the fact that the input vectors are expected not to overlap. This is not documented explicitly, but is implied by the interchangeability of the various C routines, some of which use temporary variables while others don't: this means that these routines only behave identically for non-overlapping inputs.
So let's decorate these input vectors with the __restrict modifier, which informs the compiler that there is no overlap. While at it, make the input-only vectors pointer-to-const as well.
--- arch/alpha/include/asm/xor.h | 53 ++++++++---- arch/arm/include/asm/xor.h | 42 ++++++---- arch/arm64/include/asm/xor.h | 21 +++-- arch/arm64/lib/xor-neon.c | 46 +++++++---- arch/ia64/include/asm/xor.h | 21 +++-- arch/powerpc/include/asm/xor_altivec.h | 25 +++--- arch/powerpc/lib/xor_vmx.c | 28 ++++--- arch/powerpc/lib/xor_vmx.h | 27 ++++--- arch/powerpc/lib/xor_vmx_glue.c | 32 ++++---- arch/s390/lib/xor.c | 21 +++-- arch/sparc/include/asm/xor_32.h | 21 +++-- arch/sparc/include/asm/xor_64.h | 42 ++++++---- arch/x86/include/asm/xor.h | 42 ++++++---- arch/x86/include/asm/xor_32.h | 42 ++++++---- arch/x86/include/asm/xor_avx.h | 21 +++-- include/asm-generic/xor.h | 84 +++++++++++++------- include/linux/raid/xor.h | 21 +++-- 17 files changed, 381 insertions(+), 208 deletions(-)
arch/alpha/include/asm/xor.h +36 -17
@@ -5,24 +5,43 @@* Optimized RAID-5 checksumming functions for alpha EV5 and EV6 */ -extern void xor_alpha_2 (unsigned long ,unsigned long *,unsigned long *); -extern void xor_alpha_3 (unsigned long ,unsigned long *,unsigned long *, -unsigned long *); -extern void xor_alpha_4 (unsigned long ,unsigned long *,unsigned long *, -unsigned long *,unsigned long *); -extern void xor_alpha_5 (unsigned long ,unsigned long *,unsigned long *, -unsigned long *,unsigned long *,unsigned long *); +extern void +xor_alpha_2 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2); +extern void +xor_alpha_3 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3); +extern void +xor_alpha_4 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3, +const unsigned long * __restrict p4); +extern void +xor_alpha_5 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3, +const unsigned long * __restrict p4, +const unsigned long * __restrict p5); -extern void xor_alpha_prefetch_2 (unsigned long ,unsigned long *, -unsigned long *); -extern void xor_alpha_prefetch_3 (unsigned long ,unsigned long *, -unsigned long *,unsigned long *); -extern void xor_alpha_prefetch_4 (unsigned long ,unsigned long *, -unsigned long *,unsigned long *, -unsigned long *); -extern void xor_alpha_prefetch_5 (unsigned long ,unsigned long *, -unsigned long *,unsigned long *, -unsigned long *,unsigned long *); +extern void +xor_alpha_prefetch_2 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2); +extern void +xor_alpha_prefetch_3 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3); +extern void +xor_alpha_prefetch_4 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3, +const unsigned long * __restrict p4); +extern void +xor_alpha_prefetch_5 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3, +const unsigned long * __restrict p4, +const unsigned long * __restrict p5); asm(" \n\ .text \n\
arch/arm/include/asm/xor.h +28 -14
90 lines not on this page · raw message
arch/arm64/include/asm/xor.h +14 -7
@@ -16,7 +16,8 @@extern struct xor_block_template const xor_block_inner_neon;static void -xor_neon_2 (unsigned long bytes,unsigned long *p1,unsigned long *p2) +xor_neon_2 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2) {kernel_neon_begin (); xor_block_inner_neon.do_2 (bytes, p1, p2);
@@ -24,8 +25,9 @@ xor_neon_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) }static void -xor_neon_3 (unsigned long bytes,unsigned long *p1,unsigned long *p2, -unsigned long *p3) +xor_neon_3 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3) {kernel_neon_begin (); xor_block_inner_neon.do_3 (bytes, p1, p2, p3);
@@ -33,8 +35,10 @@ xor_neon_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, }static void -xor_neon_4 (unsigned long bytes,unsigned long *p1,unsigned long *p2, -unsigned long *p3,unsigned long *p4) +xor_neon_4 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3, +const unsigned long * __restrict p4) {kernel_neon_begin (); xor_block_inner_neon.do_4 (bytes, p1, p2, p3, p4);
@@ -42,8 +46,11 @@ xor_neon_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, }static void -xor_neon_5 (unsigned long bytes,unsigned long *p1,unsigned long *p2, -unsigned long *p3,unsigned long *p4,unsigned long *p5) +xor_neon_5 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3, +const unsigned long * __restrict p4, +const unsigned long * __restrict p5) {kernel_neon_begin (); xor_block_inner_neon.do_5 (bytes, p1, p2, p3, p4, p5);
arch/arm64/lib/xor-neon.c +29 -17
88 lines not on this page · raw message
arch/ia64/include/asm/xor.h +14 -7
@@ -4,13 +4,20 @@*/ -extern void xor_ia64_2 (unsigned long ,unsigned long *,unsigned long *); -extern void xor_ia64_3 (unsigned long ,unsigned long *,unsigned long *, -unsigned long *); -extern void xor_ia64_4 (unsigned long ,unsigned long *,unsigned long *, -unsigned long *,unsigned long *); -extern void xor_ia64_5 (unsigned long ,unsigned long *,unsigned long *, -unsigned long *,unsigned long *,unsigned long *); +extern void xor_ia64_2 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2); +extern void xor_ia64_3 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3); +extern void xor_ia64_4 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3, +const unsigned long * __restrict p4); +extern void xor_ia64_5 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3, +const unsigned long * __restrict p4, +const unsigned long * __restrict p5);static struct xor_block_template xor_block_ia64 = { .name ="ia64" ,
arch/powerpc/include/asm/xor_altivec.h +14 -11
@@ -3,17 +3,20 @@#define _ASM_POWERPC_XOR_ALTIVEC_H#ifdef CONFIG_ALTIVEC - -void xor_altivec_2 (unsigned long bytes,unsigned long *v1_in, -unsigned long *v2_in); -void xor_altivec_3 (unsigned long bytes,unsigned long *v1_in, -unsigned long *v2_in,unsigned long *v3_in); -void xor_altivec_4 (unsigned long bytes,unsigned long *v1_in, -unsigned long *v2_in,unsigned long *v3_in, -unsigned long *v4_in); -void xor_altivec_5 (unsigned long bytes,unsigned long *v1_in, -unsigned long *v2_in,unsigned long *v3_in, -unsigned long *v4_in,unsigned long *v5_in); +void xor_altivec_2 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2); +void xor_altivec_3 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3); +void xor_altivec_4 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3, +const unsigned long * __restrict p4); +void xor_altivec_5 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3, +const unsigned long * __restrict p4, +const unsigned long * __restrict p5);#endif #endif /* _ASM_POWERPC_XOR_ALTIVEC_H */
arch/powerpc/lib/xor_vmx.c +18 -10
@@ -49,8 +49,9 @@ typedef vector signed char unative_t; V1##_3 = vec_xor(V1##_3, V2##_3); \ }while (0 ) -void __xor_altivec_2(unsigned long bytes, unsigned long *v1_in, - unsigned long *v2_in) +void __xor_altivec_2(unsigned long bytes, + unsigned long * __restrict v1_in, + const unsigned long * __restrict v2_in) {DEFINE (v1);DEFINE (v2);
@@ -67,8 +68,10 @@ void __xor_altivec_2(unsigned long bytes, unsigned long *v1_in, }while (--lines >0 ); } -void __xor_altivec_3 (unsigned long bytes,unsigned long *v1_in, -unsigned long *v2_in,unsigned long *v3_in) +void __xor_altivec_3 (unsigned long bytes, +unsigned long * __restrict v1_in, +const unsigned long * __restrict v2_in, +const unsigned long * __restrict v3_in) {DEFINE (v1);DEFINE (v2);
@@ -89,9 +92,11 @@ void __xor_altivec_3(unsigned long bytes, unsigned long *v1_in, }while (--lines >0 ); } -void __xor_altivec_4 (unsigned long bytes,unsigned long *v1_in, -unsigned long *v2_in,unsigned long *v3_in, -unsigned long *v4_in) +void __xor_altivec_4 (unsigned long bytes, +unsigned long * __restrict v1_in, +const unsigned long * __restrict v2_in, +const unsigned long * __restrict v3_in, +const unsigned long * __restrict v4_in) {DEFINE (v1);DEFINE (v2);
@@ -116,9 +121,12 @@ void __xor_altivec_4(unsigned long bytes, unsigned long *v1_in, }while (--lines >0 ); } -void __xor_altivec_5 (unsigned long bytes,unsigned long *v1_in, -unsigned long *v2_in,unsigned long *v3_in, -unsigned long *v4_in,unsigned long *v5_in) +void __xor_altivec_5 (unsigned long bytes, +unsigned long * __restrict v1_in, +const unsigned long * __restrict v2_in, +const unsigned long * __restrict v3_in, +const unsigned long * __restrict v4_in, +const unsigned long * __restrict v5_in) {DEFINE (v1);DEFINE (v2);
arch/powerpc/lib/xor_vmx.h +14 -13
@@ -6,16 +6,17 @@* outside of the enable/disable altivec block. */ -void __xor_altivec_2 (unsigned long bytes,unsigned long *v1_in, -unsigned long *v2_in); - -void __xor_altivec_3 (unsigned long bytes,unsigned long *v1_in, -unsigned long *v2_in,unsigned long *v3_in); - -void __xor_altivec_4 (unsigned long bytes,unsigned long *v1_in, -unsigned long *v2_in,unsigned long *v3_in, -unsigned long *v4_in); - -void __xor_altivec_5 (unsigned long bytes,unsigned long *v1_in, -unsigned long *v2_in,unsigned long *v3_in, -unsigned long *v4_in,unsigned long *v5_in); +void __xor_altivec_2 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2); +void __xor_altivec_3 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3); +void __xor_altivec_4 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3, +const unsigned long * __restrict p4); +void __xor_altivec_5 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3, +const unsigned long * __restrict p4, +const unsigned long * __restrict p5);
arch/powerpc/lib/xor_vmx_glue.c +18 -14
@@ -12,47 +12,51 @@#include <asm/xor_altivec.h> #include "xor_vmx.h" -void xor_altivec_2 (unsigned long bytes,unsigned long *v1_in, -unsigned long *v2_in) +void xor_altivec_2 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2) {preempt_disable ();enable_kernel_altivec (); -__xor_altivec_2 (bytes, v1_in, v2_in); +__xor_altivec_2 (bytes, p1, p2);disable_kernel_altivec ();preempt_enable (); }EXPORT_SYMBOL (xor_altivec_2); -void xor_altivec_3 (unsigned long bytes,unsigned long *v1_in, -unsigned long *v2_in,unsigned long *v3_in) +void xor_altivec_3 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3) {preempt_disable ();enable_kernel_altivec (); -__xor_altivec_3 (bytes, v1_in, v2_in, v3_in); +__xor_altivec_3 (bytes, p1, p2, p3);disable_kernel_altivec ();preempt_enable (); }EXPORT_SYMBOL (xor_altivec_3); -void xor_altivec_4 (unsigned long bytes,unsigned long *v1_in, -unsigned long *v2_in,unsigned long *v3_in, -unsigned long *v4_in) +void xor_altivec_4 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3, +const unsigned long * __restrict p4) {preempt_disable ();enable_kernel_altivec (); -__xor_altivec_4 (bytes, v1_in, v2_in, v3_in, v4_in); +__xor_altivec_4 (bytes, p1, p2, p3, p4);disable_kernel_altivec ();preempt_enable (); }EXPORT_SYMBOL (xor_altivec_4); -void xor_altivec_5 (unsigned long bytes,unsigned long *v1_in, -unsigned long *v2_in,unsigned long *v3_in, -unsigned long *v4_in,unsigned long *v5_in) +void xor_altivec_5 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3, +const unsigned long * __restrict p4, +const unsigned long * __restrict p5) {preempt_disable ();enable_kernel_altivec (); -__xor_altivec_5 (bytes, v1_in, v2_in, v3_in, v4_in, v5_in); +__xor_altivec_5 (bytes, p1, p2, p3, p4, p5);disable_kernel_altivec ();preempt_enable (); }
arch/s390/lib/xor.c +14 -7
@@ -11,7 +11,8 @@#include <linux/raid/xor.h> #include <asm/xor.h> -static void xor_xc_2 (unsigned long bytes,unsigned long *p1,unsigned long *p2) +static void xor_xc_2 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2) { asmvolatile ( " larl 1,2f\n"
@@ -32,8 +33,9 @@ static void xor_xc_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) : "0", "1", "cc", "memory"); } -static void xor_xc_3 (unsigned long bytes,unsigned long *p1,unsigned long *p2, -unsigned long *p3) +static void xor_xc_3 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3) { asmvolatile ( " larl 1,2f\n"
@@ -58,8 +60,10 @@ static void xor_xc_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, : : "0", "1", "cc", "memory"); } -static void xor_xc_4 (unsigned long bytes,unsigned long *p1,unsigned long *p2, -unsigned long *p3,unsigned long *p4) +static void xor_xc_4 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3, +const unsigned long * __restrict p4) { asmvolatile ( " larl 1,2f\n"
@@ -88,8 +92,11 @@ static void xor_xc_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, : : "0", "1", "cc", "memory"); } -static void xor_xc_5 (unsigned long bytes,unsigned long *p1,unsigned long *p2, -unsigned long *p3,unsigned long *p4,unsigned long *p5) +static void xor_xc_5 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3, +const unsigned long * __restrict p4, +const unsigned long * __restrict p5) { asmvolatile ( " larl 1,2f\n"
arch/sparc/include/asm/xor_32.h +14 -7
@@ -13,7 +13,8 @@*/ static void -sparc_2 (unsigned long bytes,unsigned long *p1,unsigned long *p2) +sparc_2 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2) {int lines = bytes / (sizeof (long )) /8 ;
@@ -50,8 +51,9 @@ sparc_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) }static void -sparc_3 (unsigned long bytes,unsigned long *p1,unsigned long *p2, -unsigned long *p3) +sparc_3 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3) {int lines = bytes / (sizeof (long )) /8 ;
@@ -101,8 +103,10 @@ sparc_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, }static void -sparc_4 (unsigned long bytes,unsigned long *p1,unsigned long *p2, -unsigned long *p3,unsigned long *p4) +sparc_4 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3, +const unsigned long * __restrict p4) {int lines = bytes / (sizeof (long )) /8 ;
@@ -165,8 +169,11 @@ sparc_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, }static void -sparc_5 (unsigned long bytes,unsigned long *p1,unsigned long *p2, -unsigned long *p3,unsigned long *p4,unsigned long *p5) +sparc_5 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3, +const unsigned long * __restrict p4, +const unsigned long * __restrict p5) {int lines = bytes / (sizeof (long )) /8 ;
arch/sparc/include/asm/xor_64.h +28 -14
54 lines not on this page · raw message
arch/x86/include/asm/xor.h +28 -14
90 lines not on this page · raw message
arch/x86/include/asm/xor_32.h +28 -14
90 lines not on this page · raw message
arch/x86/include/asm/xor_avx.h +14 -7
@@ -26,7 +26,8 @@ BLOCK4(8) \ BLOCK4(12) -static void xor_avx_2 (unsigned long bytes,unsigned long *p0,unsigned long *p1) +static void xor_avx_2 (unsigned long bytes,unsigned long * __restrict p0, +const unsigned long * __restrict p1) {unsigned long lines = bytes >>9 ;
@@ -52,8 +53,9 @@ do { \kernel_fpu_end (); } -static void xor_avx_3 (unsigned long bytes,unsigned long *p0,unsigned long *p1, -unsigned long *p2) +static void xor_avx_3 (unsigned long bytes,unsigned long * __restrict p0, +const unsigned long * __restrict p1, +const unsigned long * __restrict p2) {unsigned long lines = bytes >>9 ;
@@ -82,8 +84,10 @@ do { \kernel_fpu_end (); } -static void xor_avx_4 (unsigned long bytes,unsigned long *p0,unsigned long *p1, -unsigned long *p2,unsigned long *p3) +static void xor_avx_4 (unsigned long bytes,unsigned long * __restrict p0, +const unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3) {unsigned long lines = bytes >>9 ;
@@ -115,8 +119,11 @@ do { \kernel_fpu_end (); } -static void xor_avx_5 (unsigned long bytes,unsigned long *p0,unsigned long *p1, -unsigned long *p2,unsigned long *p3,unsigned long *p4) +static void xor_avx_5 (unsigned long bytes,unsigned long * __restrict p0, +const unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3, +const unsigned long * __restrict p4) {unsigned long lines = bytes >>9 ;
include/asm-generic/xor.h +56 -28
180 lines not on this page · raw message
include/linux/raid/xor.h +14 -7
@@ -11,13 +11,20 @@ struct xor_block_template {struct xor_block_template *next;const char *name;int speed; -void (*do_2)(unsigned long ,unsigned long *,unsigned long *); -void (*do_3)(unsigned long ,unsigned long *,unsigned long *, -unsigned long *); -void (*do_4)(unsigned long ,unsigned long *,unsigned long *, -unsigned long *,unsigned long *); -void (*do_5)(unsigned long ,unsigned long *,unsigned long *, - unsigned long *, unsigned long *, unsigned long *); +void (*do_2)(unsigned long ,unsigned long * __restrict, +const unsigned long * __restrict); +void (*do_3)(unsigned long ,unsigned long * __restrict, +const unsigned long * __restrict, +const unsigned long * __restrict); +void (*do_4)(unsigned long ,unsigned long * __restrict, +const unsigned long * __restrict, +const unsigned long * __restrict, +const unsigned long * __restrict); +void (*do_5)(unsigned long ,unsigned long * __restrict, +const unsigned long * __restrict, +const unsigned long * __restrict, +const unsigned long * __restrict, + const unsigned long * __restrict); };#endif
Signature · 1 line
2.30.2
PATCH v4 2/2 crypto: arm/xor - make vectorized C code Clang-friendly
In reply to Ard BiesheuvelThe ARM version of the accelerated XOR routines are simply the 8-way C routines passed through the auto-vectorizer with SIMD codegen enabled. This used to require GCC version 4.6 at least, but given that 5.1 is now the baseline, this check is no longer necessary, and actually misidentifies Clang as GCC < 4.6 as Clang defines the GCC major/minor as well, but makes no attempt at doing this in a way that conveys feature parity with a certain version of GCC (which would not be a great idea in the first place).
So let's drop the version check, and make the auto-vectorize pragma (which is based on a GCC-specific command line option) GCC-only. Since Clang performs SIMD auto-vectorization by default at -O2, no pragma is necessary here.
--- arch/arm/lib/xor-neon.c | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-)
arch/arm/lib/xor-neon.c +3 -9
@@ -17,17 +17,11 @@ MODULE_LICENSE("GPL");/* * Pull in the reference implementations while instructing GCC (through * -ftree-vectorize) to attempt to exploit implicit parallelism and emit -* NEON instructions. +* NEON instructions. Clang does this by default at O2 so no pragma is +* needed. */ -#if __GNUC__ >4 || (__GNUC__ ==4 && __GNUC_MINOR__ >=6 ) +#ifdef CONFIG_CC_IS_GCC #pragma GCC optimize "tree-vectorize" -#else -/* -* While older versions of GCC do not generate incorrect code, they fail to -* recognize the parallel nature of these functions, and emit plain ARM code, -* which is known to be slower than the optimized ARM code in asm-arm/xor.h. -*/ -#warning This code requires at least version 4.6 of GCC#endif #pragma GCC diagnostic ignored "-Wunused-variable"
Signature · 1 line
2.30.2
Herbert Xu
maintainerQuoted 41 lines · Ard Biesheuvel
Update the
xor_blocks()prototypes so that the compiler understands that the inputs always refer to distinct regions of memory. This is implied by the existing implementations, as they use different granularities for the load/xor/store loops.With that, we can fix the ARM/Clang version, which refuses to SIMD vectorize otherwise, and throws a spurious warning related to the GCC version being incompatible.
Changes since v3:
- revert broken PPC argument rename - doing it fully results in too much pointless churn, and the 'inner' altivec routines are not strictly part of thexor_blocksAPI anywayChanges since v2:
- fix arm64 build after rebase
- name PPC argument names consistently
- add Nick's acks and link tagsChanges since v1:
- fix PPC build
- add Nathan's Tested-byCc Nick Desaulniers <[email protected]>Cc Nathan Chancellor <[email protected]>Ard Biesheuvel (2):
lib/xor: make xor prototypes more friendly to compiler vectorization
crypto: arm/xor - make vectorized C code Clang-friendlyarch/alpha/include/asm/xor.h | 53 ++++++++---- arch/arm/include/asm/xor.h | 42 ++++++---- arch/arm/lib/xor-neon.c | 12 +-- arch/arm64/include/asm/xor.h | 21 +++-- arch/arm64/lib/xor-neon.c | 46 +++++++---- arch/ia64/include/asm/xor.h | 21 +++-- arch/powerpc/include/asm/xor_altivec.h | 25 +++--- arch/powerpc/lib/xor_vmx.c | 28 ++++--- arch/powerpc/lib/xor_vmx.h | 27 ++++--- arch/powerpc/lib/xor_vmx_glue.c | 32 ++++---- arch/s390/lib/xor.c | 21 +++--
Quoted 8 lines
arch/sparc/include/asm/xor_32.h | 21 +++-- arch/sparc/include/asm/xor_64.h | 42 ++++++---- arch/x86/include/asm/xor.h | 42 ++++++---- arch/x86/include/asm/xor_32.h | 42 ++++++---- arch/x86/include/asm/xor_avx.h | 21 +++-- include/asm-generic/xor.h | 84 +++++++++++++------- include/linux/raid/xor.h | 21 +++-- 18 files changed, 384 insertions(+), 217 deletions(-)
All applied. Thanks.
Signature · 3 lines
Email: Herbert Xu <[email protected]> Home Page: http://gondor.apana.org.au/~herbert/ PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt