xor: enable auto-vectorization in Clang
Subsystems (10)
cover xor: enable auto-vectorization in Clang
Update the xor_blocks() prototypes so that the compiler understands that the inputs always refer to distinct regions of memory. This is implied by the existing implementations, as they use different granularities for the load/xor/store loops.
With that, we can fix the ARM/Clang version, which refuses to SIMD vectorize otherwise, and throws a spurious warning related to the GCC version being incompatible.
Ard Biesheuvel (2):
lib/xor: make xor prototypes more friendely to compiler vectorization
crypto: arm/xor - make vectorized C code Clang-friendly
arch/alpha/include/asm/xor.h | 53 ++++++++---- arch/arm/include/asm/xor.h | 42 ++++++---- arch/arm/lib/xor-neon.c | 12 +-- arch/arm64/include/asm/xor.h | 21 +++-- arch/arm64/lib/xor-neon.c | 23 +++--- arch/ia64/include/asm/xor.h | 21 +++-- arch/powerpc/include/asm/xor_altivec.h | 25 +++--- arch/powerpc/lib/xor_vmx.h | 27 ++++--- arch/powerpc/lib/xor_vmx_glue.c | 32 ++++---- arch/s390/lib/xor.c | 21 +++-- arch/sparc/include/asm/xor_32.h | 21 +++-- arch/sparc/include/asm/xor_64.h | 42 ++++++---- arch/x86/include/asm/xor.h | 42 ++++++---- arch/x86/include/asm/xor_32.h | 42 ++++++---- arch/x86/include/asm/xor_avx.h | 21 +++-- include/asm-generic/xor.h | 84 +++++++++++++------- include/linux/raid/xor.h | 21 +++-- 17 files changed, 351 insertions(+), 199 deletions(-)
base-commit: e783362eb54cd99b2cac8b3a9aeac942e6f6ac07
Signature · 1 line
2.30.2
PATCH 1/2 lib/xor: make xor prototypes more friendely to compiler vectorization
Modern compilers are perfectly capable of extracting parallelism from the XOR routines, provided that the prototypes reflect the nature of the input accurately, in particular, the fact that the input vectors are expected not to overlap. This is not documented explicitly, but is implied by the interchangeability of the various C routines, some of which use temporary variables while others don't: this means that these routines only behave identically for non-overlapping inputs.
So let's decorate these input vectors with the __restrict modifier, which informs the compiler that there is no overlap. While at it, make the input-only vectors pointer-to-const as well.
--- arch/alpha/include/asm/xor.h | 53 ++++++++---- arch/arm/include/asm/xor.h | 42 ++++++---- arch/arm64/include/asm/xor.h | 21 +++-- arch/arm64/lib/xor-neon.c | 23 +++--- arch/ia64/include/asm/xor.h | 21 +++-- arch/powerpc/include/asm/xor_altivec.h | 25 +++--- arch/powerpc/lib/xor_vmx.h | 27 ++++--- arch/powerpc/lib/xor_vmx_glue.c | 32 ++++---- arch/s390/lib/xor.c | 21 +++-- arch/sparc/include/asm/xor_32.h | 21 +++-- arch/sparc/include/asm/xor_64.h | 42 ++++++---- arch/x86/include/asm/xor.h | 42 ++++++---- arch/x86/include/asm/xor_32.h | 42 ++++++---- arch/x86/include/asm/xor_avx.h | 21 +++-- include/asm-generic/xor.h | 84 +++++++++++++------- include/linux/raid/xor.h | 21 +++-- 16 files changed, 348 insertions(+), 190 deletions(-)
arch/alpha/include/asm/xor.h +36 -17
@@ -5,24 +5,43 @@* Optimized RAID-5 checksumming functions for alpha EV5 and EV6 */ -extern void xor_alpha_2 (unsigned long ,unsigned long *,unsigned long *); -extern void xor_alpha_3 (unsigned long ,unsigned long *,unsigned long *, -unsigned long *); -extern void xor_alpha_4 (unsigned long ,unsigned long *,unsigned long *, -unsigned long *,unsigned long *); -extern void xor_alpha_5 (unsigned long ,unsigned long *,unsigned long *, -unsigned long *,unsigned long *,unsigned long *); +extern void +xor_alpha_2 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2); +extern void +xor_alpha_3 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3); +extern void +xor_alpha_4 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3, +const unsigned long * __restrict p4); +extern void +xor_alpha_5 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3, +const unsigned long * __restrict p4, +const unsigned long * __restrict p5); -extern void xor_alpha_prefetch_2 (unsigned long ,unsigned long *, -unsigned long *); -extern void xor_alpha_prefetch_3 (unsigned long ,unsigned long *, -unsigned long *,unsigned long *); -extern void xor_alpha_prefetch_4 (unsigned long ,unsigned long *, -unsigned long *,unsigned long *, -unsigned long *); -extern void xor_alpha_prefetch_5 (unsigned long ,unsigned long *, -unsigned long *,unsigned long *, -unsigned long *,unsigned long *); +extern void +xor_alpha_prefetch_2 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2); +extern void +xor_alpha_prefetch_3 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3); +extern void +xor_alpha_prefetch_4 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3, +const unsigned long * __restrict p4); +extern void +xor_alpha_prefetch_5 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3, +const unsigned long * __restrict p4, +const unsigned long * __restrict p5); asm(" \n\ .text \n\
arch/arm/include/asm/xor.h +28 -14
90 lines not on this page · raw message
arch/arm64/include/asm/xor.h +14 -7
@@ -16,7 +16,8 @@extern struct xor_block_template const xor_block_inner_neon;static void -xor_neon_2 (unsigned long bytes,unsigned long *p1,unsigned long *p2) +xor_neon_2 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2) {kernel_neon_begin (); xor_block_inner_neon.do_2 (bytes, p1, p2);
@@ -24,8 +25,9 @@ xor_neon_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) }static void -xor_neon_3 (unsigned long bytes,unsigned long *p1,unsigned long *p2, -unsigned long *p3) +xor_neon_3 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3) {kernel_neon_begin (); xor_block_inner_neon.do_3 (bytes, p1, p2, p3);
@@ -33,8 +35,10 @@ xor_neon_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, }static void -xor_neon_4 (unsigned long bytes,unsigned long *p1,unsigned long *p2, -unsigned long *p3,unsigned long *p4) +xor_neon_4 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3, +const unsigned long * __restrict p4) {kernel_neon_begin (); xor_block_inner_neon.do_4 (bytes, p1, p2, p3, p4);
@@ -42,8 +46,11 @@ xor_neon_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, }static void -xor_neon_5 (unsigned long bytes,unsigned long *p1,unsigned long *p2, -unsigned long *p3,unsigned long *p4,unsigned long *p5) +xor_neon_5 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3, +const unsigned long * __restrict p4, +const unsigned long * __restrict p5) {kernel_neon_begin (); xor_block_inner_neon.do_5 (bytes, p1, p2, p3, p4, p5);
arch/arm64/lib/xor-neon.c +14 -9
@@ -10,8 +10,8 @@#include <linux/module.h> #include <asm/neon-intrinsics.h> -void xor_arm64_neon_2 (unsigned long bytes,unsigned long *p1, -unsigned long *p2) +void xor_arm64_neon_2 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2) {uint64_t *dp1 = (uint64_t *)p1;uint64_t *dp2 = (uint64_t *)p2;
@@ -37,8 +37,9 @@ void xor_arm64_neon_2(unsigned long bytes, unsigned long *p1, }while (--lines >0 ); } -void xor_arm64_neon_3 (unsigned long bytes,unsigned long *p1, -unsigned long *p2,unsigned long *p3) +void xor_arm64_neon_3 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3) {uint64_t *dp1 = (uint64_t *)p1;uint64_t *dp2 = (uint64_t *)p2;
@@ -72,8 +73,10 @@ void xor_arm64_neon_3(unsigned long bytes, unsigned long *p1, }while (--lines >0 ); } -void xor_arm64_neon_4 (unsigned long bytes,unsigned long *p1, -unsigned long *p2,unsigned long *p3,unsigned long *p4) +void xor_arm64_neon_4 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3, +const unsigned long * __restrict p4) {uint64_t *dp1 = (uint64_t *)p1;uint64_t *dp2 = (uint64_t *)p2;
@@ -115,9 +118,11 @@ void xor_arm64_neon_4(unsigned long bytes, unsigned long *p1, }while (--lines >0 ); } -void xor_arm64_neon_5 (unsigned long bytes,unsigned long *p1, -unsigned long *p2,unsigned long *p3, -unsigned long *p4,unsigned long *p5) +void xor_arm64_neon_5 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3, +const unsigned long * __restrict p4, +const unsigned long * __restrict p5) {uint64_t *dp1 = (uint64_t *)p1;uint64_t *dp2 = (uint64_t *)p2;
arch/ia64/include/asm/xor.h +14 -7
@@ -4,13 +4,20 @@*/ -extern void xor_ia64_2 (unsigned long ,unsigned long *,unsigned long *); -extern void xor_ia64_3 (unsigned long ,unsigned long *,unsigned long *, -unsigned long *); -extern void xor_ia64_4 (unsigned long ,unsigned long *,unsigned long *, -unsigned long *,unsigned long *); -extern void xor_ia64_5 (unsigned long ,unsigned long *,unsigned long *, -unsigned long *,unsigned long *,unsigned long *); +extern void xor_ia64_2 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2); +extern void xor_ia64_3 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3); +extern void xor_ia64_4 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3, +const unsigned long * __restrict p4); +extern void xor_ia64_5 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3, +const unsigned long * __restrict p4, +const unsigned long * __restrict p5);static struct xor_block_template xor_block_ia64 = { .name ="ia64" ,
arch/powerpc/include/asm/xor_altivec.h +14 -11
@@ -3,17 +3,20 @@#define _ASM_POWERPC_XOR_ALTIVEC_H#ifdef CONFIG_ALTIVEC - -void xor_altivec_2 (unsigned long bytes,unsigned long *v1_in, -unsigned long *v2_in); -void xor_altivec_3 (unsigned long bytes,unsigned long *v1_in, -unsigned long *v2_in,unsigned long *v3_in); -void xor_altivec_4 (unsigned long bytes,unsigned long *v1_in, -unsigned long *v2_in,unsigned long *v3_in, -unsigned long *v4_in); -void xor_altivec_5 (unsigned long bytes,unsigned long *v1_in, -unsigned long *v2_in,unsigned long *v3_in, -unsigned long *v4_in,unsigned long *v5_in); +void xor_altivec_2 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2); +void xor_altivec_3 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3); +void xor_altivec_4 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3, +const unsigned long * __restrict p4); +void xor_altivec_5 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3, +const unsigned long * __restrict p4, +const unsigned long * __restrict p5);#endif #endif /* _ASM_POWERPC_XOR_ALTIVEC_H */
arch/powerpc/lib/xor_vmx.h +14 -13
@@ -6,16 +6,17 @@* outside of the enable/disable altivec block. */ -void __xor_altivec_2 (unsigned long bytes,unsigned long *v1_in, -unsigned long *v2_in); - -void __xor_altivec_3 (unsigned long bytes,unsigned long *v1_in, -unsigned long *v2_in,unsigned long *v3_in); - -void __xor_altivec_4 (unsigned long bytes,unsigned long *v1_in, -unsigned long *v2_in,unsigned long *v3_in, -unsigned long *v4_in); - -void __xor_altivec_5 (unsigned long bytes,unsigned long *v1_in, -unsigned long *v2_in,unsigned long *v3_in, -unsigned long *v4_in,unsigned long *v5_in); +void __xor_altivec_2 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2); +void __xor_altivec_3 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3); +void __xor_altivec_4 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3, +const unsigned long * __restrict p4); +void __xor_altivec_5 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3, +const unsigned long * __restrict p4, +const unsigned long * __restrict p5);
arch/powerpc/lib/xor_vmx_glue.c +18 -14
@@ -12,47 +12,51 @@#include <asm/xor_altivec.h> #include "xor_vmx.h" -void xor_altivec_2 (unsigned long bytes,unsigned long *v1_in, -unsigned long *v2_in) +void xor_altivec_2 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2) {preempt_disable ();enable_kernel_altivec (); -__xor_altivec_2 (bytes, v1_in, v2_in); +__xor_altivec_2 (bytes, p1, p2);disable_kernel_altivec ();preempt_enable (); }EXPORT_SYMBOL (xor_altivec_2); -void xor_altivec_3 (unsigned long bytes,unsigned long *v1_in, -unsigned long *v2_in,unsigned long *v3_in) +void xor_altivec_3 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3) {preempt_disable ();enable_kernel_altivec (); -__xor_altivec_3 (bytes, v1_in, v2_in, v3_in); +__xor_altivec_3 (bytes, p1, p2, p3);disable_kernel_altivec ();preempt_enable (); }EXPORT_SYMBOL (xor_altivec_3); -void xor_altivec_4 (unsigned long bytes,unsigned long *v1_in, -unsigned long *v2_in,unsigned long *v3_in, -unsigned long *v4_in) +void xor_altivec_4 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3, +const unsigned long * __restrict p4) {preempt_disable ();enable_kernel_altivec (); -__xor_altivec_4 (bytes, v1_in, v2_in, v3_in, v4_in); +__xor_altivec_4 (bytes, p1, p2, p3, p4);disable_kernel_altivec ();preempt_enable (); }EXPORT_SYMBOL (xor_altivec_4); -void xor_altivec_5 (unsigned long bytes,unsigned long *v1_in, -unsigned long *v2_in,unsigned long *v3_in, -unsigned long *v4_in,unsigned long *v5_in) +void xor_altivec_5 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3, +const unsigned long * __restrict p4, +const unsigned long * __restrict p5) {preempt_disable ();enable_kernel_altivec (); -__xor_altivec_5 (bytes, v1_in, v2_in, v3_in, v4_in, v5_in); +__xor_altivec_5 (bytes, p1, p2, p3, p4, p5);disable_kernel_altivec ();preempt_enable (); }
arch/s390/lib/xor.c +14 -7
@@ -11,7 +11,8 @@#include <linux/raid/xor.h> #include <asm/xor.h> -static void xor_xc_2 (unsigned long bytes,unsigned long *p1,unsigned long *p2) +static void xor_xc_2 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2) { asmvolatile ( " larl 1,2f\n"
@@ -32,8 +33,9 @@ static void xor_xc_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) : "0", "1", "cc", "memory"); } -static void xor_xc_3 (unsigned long bytes,unsigned long *p1,unsigned long *p2, -unsigned long *p3) +static void xor_xc_3 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3) { asmvolatile ( " larl 1,2f\n"
@@ -58,8 +60,10 @@ static void xor_xc_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, : : "0", "1", "cc", "memory"); } -static void xor_xc_4 (unsigned long bytes,unsigned long *p1,unsigned long *p2, -unsigned long *p3,unsigned long *p4) +static void xor_xc_4 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3, +const unsigned long * __restrict p4) { asmvolatile ( " larl 1,2f\n"
@@ -88,8 +92,11 @@ static void xor_xc_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, : : "0", "1", "cc", "memory"); } -static void xor_xc_5 (unsigned long bytes,unsigned long *p1,unsigned long *p2, -unsigned long *p3,unsigned long *p4,unsigned long *p5) +static void xor_xc_5 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3, +const unsigned long * __restrict p4, +const unsigned long * __restrict p5) { asmvolatile ( " larl 1,2f\n"
arch/sparc/include/asm/xor_32.h +14 -7
@@ -13,7 +13,8 @@*/ static void -sparc_2 (unsigned long bytes,unsigned long *p1,unsigned long *p2) +sparc_2 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2) {int lines = bytes / (sizeof (long )) /8 ;
@@ -50,8 +51,9 @@ sparc_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) }static void -sparc_3 (unsigned long bytes,unsigned long *p1,unsigned long *p2, -unsigned long *p3) +sparc_3 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3) {int lines = bytes / (sizeof (long )) /8 ;
@@ -101,8 +103,10 @@ sparc_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, }static void -sparc_4 (unsigned long bytes,unsigned long *p1,unsigned long *p2, -unsigned long *p3,unsigned long *p4) +sparc_4 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3, +const unsigned long * __restrict p4) {int lines = bytes / (sizeof (long )) /8 ;
@@ -165,8 +169,11 @@ sparc_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, }static void -sparc_5 (unsigned long bytes,unsigned long *p1,unsigned long *p2, -unsigned long *p3,unsigned long *p4,unsigned long *p5) +sparc_5 (unsigned long bytes,unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3, +const unsigned long * __restrict p4, +const unsigned long * __restrict p5) {int lines = bytes / (sizeof (long )) /8 ;
arch/sparc/include/asm/xor_64.h +28 -14
54 lines not on this page · raw message
arch/x86/include/asm/xor.h +28 -14
90 lines not on this page · raw message
arch/x86/include/asm/xor_32.h +28 -14
90 lines not on this page · raw message
arch/x86/include/asm/xor_avx.h +14 -7
@@ -26,7 +26,8 @@ BLOCK4(8) \ BLOCK4(12) -static void xor_avx_2 (unsigned long bytes,unsigned long *p0,unsigned long *p1) +static void xor_avx_2 (unsigned long bytes,unsigned long * __restrict p0, +const unsigned long * __restrict p1) {unsigned long lines = bytes >>9 ;
@@ -52,8 +53,9 @@ do { \kernel_fpu_end (); } -static void xor_avx_3 (unsigned long bytes,unsigned long *p0,unsigned long *p1, -unsigned long *p2) +static void xor_avx_3 (unsigned long bytes,unsigned long * __restrict p0, +const unsigned long * __restrict p1, +const unsigned long * __restrict p2) {unsigned long lines = bytes >>9 ;
@@ -82,8 +84,10 @@ do { \kernel_fpu_end (); } -static void xor_avx_4 (unsigned long bytes,unsigned long *p0,unsigned long *p1, -unsigned long *p2,unsigned long *p3) +static void xor_avx_4 (unsigned long bytes,unsigned long * __restrict p0, +const unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3) {unsigned long lines = bytes >>9 ;
@@ -115,8 +119,11 @@ do { \kernel_fpu_end (); } -static void xor_avx_5 (unsigned long bytes,unsigned long *p0,unsigned long *p1, -unsigned long *p2,unsigned long *p3,unsigned long *p4) +static void xor_avx_5 (unsigned long bytes,unsigned long * __restrict p0, +const unsigned long * __restrict p1, +const unsigned long * __restrict p2, +const unsigned long * __restrict p3, +const unsigned long * __restrict p4) {unsigned long lines = bytes >>9 ;
include/asm-generic/xor.h +56 -28
180 lines not on this page · raw message
include/linux/raid/xor.h +14 -7
@@ -11,13 +11,20 @@ struct xor_block_template {struct xor_block_template *next;const char *name;int speed; -void (*do_2)(unsigned long ,unsigned long *,unsigned long *); -void (*do_3)(unsigned long ,unsigned long *,unsigned long *, -unsigned long *); -void (*do_4)(unsigned long ,unsigned long *,unsigned long *, -unsigned long *,unsigned long *); -void (*do_5)(unsigned long ,unsigned long *,unsigned long *, - unsigned long *, unsigned long *, unsigned long *); +void (*do_2)(unsigned long ,unsigned long * __restrict, +const unsigned long * __restrict); +void (*do_3)(unsigned long ,unsigned long * __restrict, +const unsigned long * __restrict, +const unsigned long * __restrict); +void (*do_4)(unsigned long ,unsigned long * __restrict, +const unsigned long * __restrict, +const unsigned long * __restrict, +const unsigned long * __restrict); +void (*do_5)(unsigned long ,unsigned long * __restrict, +const unsigned long * __restrict, +const unsigned long * __restrict, +const unsigned long * __restrict, + const unsigned long * __restrict); };#endif
Signature · 1 line
2.30.2
kernel test robot
build botRe: [PATCH 1/2] lib/xor: make xor prototypes more friendely to compiler vectorization
I love your patch! Yet something to improve:
Hi Ard,
I love your patch! Yet something to improve:
[auto build test ERROR on e783362eb54cd99b2cac8b3a9aeac942e6f6ac07]
url: https://github.com/0day-ci/linux/commits/Ard-Biesheuvel/xor-enable-auto-vectorization-in-Clang/20220127-161449 base: e783362eb54cd99b2cac8b3a9aeac942e6f6ac07 config: arm64-randconfig-r015-20220127 (https://download.01.org/0day-ci/archive/20220128/202201280125.Q4T02G9Q-lkp(a)intel.com/config) compiler: aarch64-linux-gcc (GCC) 11.2.0
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# https://github.com/0day-ci/linux/commit/543912d52a64185845fd9bde2412680ef0156a7c
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Ard-Biesheuvel/xor-enable-auto-vectorization-in-Clang/20220127-161449 git checkout 543912d52a64185845fd9bde2412680ef0156a7c
# save the config file to linux build tree
mkdir build_dir
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-11.2.0 make.cross O=build_dir ARCH=arm64 SHELL=/bin/bash
If you fix the issue, kindly add following tag as appropriate
All errors (new ones prefixed by >>):
arch/arm64/lib/xor-neon.c:13:6: warning: no previous prototype for 'xor_arm64_neon_2' [-Wmissing-prototypes]
13 | void xor_arm64_neon_2(unsigned long bytes, unsigned long * __restrict p1,
| ^~~~~~~~~~~~~~~~
arch/arm64/lib/xor-neon.c:40:6: warning: no previous prototype for 'xor_arm64_neon_3' [-Wmissing-prototypes]
40 | void xor_arm64_neon_3(unsigned long bytes, unsigned long * __restrict p1,
| ^~~~~~~~~~~~~~~~
arch/arm64/lib/xor-neon.c:76:6: warning: no previous prototype for 'xor_arm64_neon_4' [-Wmissing-prototypes]
76 | void xor_arm64_neon_4(unsigned long bytes, unsigned long * __restrict p1,
| ^~~~~~~~~~~~~~~~
arch/arm64/lib/xor-neon.c:121:6: warning: no previous prototype for 'xor_arm64_neon_5' [-Wmissing-prototypes]
121 | void xor_arm64_neon_5(unsigned long bytes, unsigned long * __restrict p1,
| ^~~~~~~~~~~~~~~~
arch/arm64/lib/xor-neon.c: In function 'xor_neon_init':
>> arch/arm64/lib/xor-neon.c:316:43: error: assignment to 'void (*)(long unsigned int, long unsigned int * __restrict__, const long unsigned int * __restrict__, const long unsigned int * __restrict__)' from incompatible pointer type 'void (*)(long unsigned int, long unsigned int *, long unsigned int *, long unsigned int *)' [-Werror=incompatible-pointer-types]
316 | xor_block_inner_neon.do_3 = xor_arm64_eor3_3;
| ^
>> arch/arm64/lib/xor-neon.c:317:43: error: assignment to 'void (*)(long unsigned int, long unsigned int * __restrict__, const long unsigned int * __restrict__, const long unsigned int * __restrict__, const long unsigned int * __restrict__)' from incompatible pointer type 'void (*)(long unsigned int, long unsigned int *, long unsigned int *, long unsigned int *, long unsigned int *)' [-Werror=incompatible-pointer-types]
317 | xor_block_inner_neon.do_4 = xor_arm64_eor3_4;
| ^
arch/arm64/lib/xor-neon.c:318:43: error: assignment to 'void (*)(long unsigned int, long unsigned int * __restrict__, const long unsigned int * __restrict__, const long unsigned int * __restrict__, const long unsigned int * __restrict__, const long unsigned int * __restrict__)' from incompatible pointer type 'void (*)(long unsigned int, long unsigned int *, long unsigned int *, long unsigned int *, long unsigned int *, long unsigned int *)' [-Werror=incompatible-pointer-types]
318 | xor_block_inner_neon.do_5 = xor_arm64_eor3_5;
| ^cc1: some warnings being treated as errors
vim +316 arch/arm64/lib/xor-neon.c
2c54b423cf85ba Ard Biesheuvel 2021-12-13 312
2c54b423cf85ba Ard Biesheuvel 2021-12-13 313 static int __init xor_neon_init(void)
2c54b423cf85ba Ard Biesheuvel 2021-12-13 314 {
2c54b423cf85ba Ard Biesheuvel 2021-12-13 315 if (IS_ENABLED(CONFIG_AS_HAS_SHA3) && cpu_have_named_feature(SHA3)) {
2c54b423cf85ba Ard Biesheuvel 2021-12-13 @316 xor_block_inner_neon.do_3 = xor_arm64_eor3_3;
2c54b423cf85ba Ard Biesheuvel 2021-12-13 @317 xor_block_inner_neon.do_4 = xor_arm64_eor3_4;
2c54b423cf85ba Ard Biesheuvel 2021-12-13 318 xor_block_inner_neon.do_5 = xor_arm64_eor3_5;
2c54b423cf85ba Ard Biesheuvel 2021-12-13 319 }
2c54b423cf85ba Ard Biesheuvel 2021-12-13 320 return 0;
2c54b423cf85ba Ard Biesheuvel 2021-12-13 321 }
2c54b423cf85ba Ard Biesheuvel 2021-12-13 322 module_init(xor_neon_init);2c54b423cf85ba Ard Biesheuvel 2021-12-13 323
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org
Herbert Xu
maintainerRe: [PATCH 1/2] lib/xor: make xor prototypes more friendely to compiler vectorization
In reply to Ard Biesheuvelarch/arm64/lib/xor-neon.c
Quoted 5 lines · Ard Biesheuvel
arch/arm64/lib/xor-neon.c +0
-0
This fails to build:
../arch/arm64/lib/xor-neon.c:13:6: warning: no previous prototype for ‘xor_arm64_neon_2’ [-Wmissing-prototypes]
13 | void xor_arm64_neon_2(unsigned long bytes, unsigned long * __restrict p1,
| ^~~~~~~~~~~~~~~~
../arch/arm64/lib/xor-neon.c:40:6: warning: no previous prototype for ‘xor_arm64_neon_3’ [-Wmissing-prototypes]
40 | void xor_arm64_neon_3(unsigned long bytes, unsigned long * __restrict p1,
| ^~~~~~~~~~~~~~~~
../arch/arm64/lib/xor-neon.c:76:6: warning: no previous prototype for ‘xor_arm64_neon_4’ [-Wmissing-prototypes]
76 | void xor_arm64_neon_4(unsigned long bytes, unsigned long * __restrict p1,
| ^~~~~~~~~~~~~~~~
../arch/arm64/lib/xor-neon.c:121:6: warning: no previous prototype for ‘xor_arm64_neon_5’ [-Wmissing-prototypes]
121 | void xor_arm64_neon_5(unsigned long bytes, unsigned long * __restrict p1,
| ^~~~~~~~~~~~~~~~../arch/arm64/lib/xor-neon.c: In function ‘xor_neon_init’:../arch/arm64/lib/xor-neon.c:316:29: error: assignment to ‘void (*)(long unsigned int, long unsigned int * __restrict__, const long unsigned int * __restrict__, const long unsigned int * __restrict__)’ from incompatible pointer type ‘void (*)(long unsigned int, long unsigned int *, long unsigned int *, long unsigned int *)’ [-Werror=incompatible-pointer-types]
316 | xor_block_inner_neon.do_3 = xor_arm64_eor3_3;
| ^../arch/arm64/lib/xor-neon.c:317:29: error: assignment to ‘void (*)(long unsigned int, long unsigned int * __restrict__, const long unsigned int * __restrict__, const long unsigned int * __restrict__, const long unsigned int * __restrict__)’ from incompatible pointer type ‘void (*)(long unsigned int, long unsigned int *, long unsigned int *, long unsigned int *, long unsigned int *)’ [-Werror=incompatible-pointer-types]
317 | xor_block_inner_neon.do_4 = xor_arm64_eor3_4;
| ^../arch/arm64/lib/xor-neon.c:318:29: error: assignment to ‘void (*)(long unsigned int, long unsigned int * __restrict__, const long unsigned int * __restrict__, const long unsigned int * __restrict__, const long unsigned int * __restrict__, const long unsigned int * __restrict__)’ from incompatible pointer type ‘void (*)(long unsigned int, long unsigned int *, long unsigned int *, long unsigned int *, long unsigned int *, long unsigned int *)’ [-Werror=incompatible-pointer-types]
318 | xor_block_inner_neon.do_5 = xor_arm64_eor3_5;
| ^
cc1: some warnings being treated as errors
Cheers,
Signature · 3 lines
Email: Herbert Xu <[email protected]> Home Page: http://gondor.apana.org.au/~herbert/ PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
PATCH 2/2 crypto: arm/xor - make vectorized C code Clang-friendly
In reply to Ard BiesheuvelThe ARM version of the accelerated XOR routines are simply the 8-way C routines passed through the auto-vectorizer with SIMD codegen enabled. This used to require GCC version 4.6 at least, but given that 5.1 is now the baseline, this check is no longer necessary, and actually misidentifies Clang as GCC < 4.6 as Clang defines the GCC major/minor as well, but makes no attempt at doing this in a way that conveys feature parity with a certain version of GCC (which would not be a great idea in the first place).
So let's drop the version check, and make the auto-vectorize pragma (which is based on a GCC-specific command line option) GCC-only. Since Clang performs SIMD auto-vectorization by default at -O2, no pragma is necessary here.
--- arch/arm/lib/xor-neon.c | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-)
arch/arm/lib/xor-neon.c +3 -9
@@ -17,17 +17,11 @@ MODULE_LICENSE("GPL");/* * Pull in the reference implementations while instructing GCC (through * -ftree-vectorize) to attempt to exploit implicit parallelism and emit -* NEON instructions. +* NEON instructions. Clang does this by default at O2 so no pragma is +* needed. */ -#if __GNUC__ >4 || (__GNUC__ ==4 && __GNUC_MINOR__ >=6 ) +#ifdef CONFIG_CC_IS_GCC #pragma GCC optimize "tree-vectorize" -#else -/* -* While older versions of GCC do not generate incorrect code, they fail to -* recognize the parallel nature of these functions, and emit plain ARM code, -* which is known to be slower than the optimized ARM code in asm-arm/xor.h. -*/ -#warning This code requires at least version 4.6 of GCC#endif #pragma GCC diagnostic ignored "-Wunused-variable"
Signature · 1 line
2.30.2
Hi Ard,
Quoted 9 lines · Ard Biesheuvel
Update the
xor_blocks()prototypes so that the compiler understands that the inputs always refer to distinct regions of memory. This is implied by the existing implementations, as they use different granularities for the load/xor/store loops.With that, we can fix the ARM/Clang version, which refuses to SIMD vectorize otherwise, and throws a spurious warning related to the GCC version being incompatible.
Quoted 6 lines
Cc Nick Desaulniers <[email protected]>Cc Nathan Chancellor <[email protected]>Ard Biesheuvel (2):
lib/xor: make xor prototypes more friendely to compiler vectorization
crypto: arm/xor - make vectorized C code Clang-friendly
I tested multi_v7_defconfig + CONFIG_BTRFS=y (to get CONFIG_XOR_BLOCKS=y) in QEMU 6.2.0 (10 boots) and the xor neon code gets faster according to do_xor_speed():
mainline @ 626b2dda7651:
[ 2.591449] neon : 1166 MB/sec [ 2.579454] neon : 1118 MB/sec [ 2.589061] neon : 1163 MB/sec [ 2.581827] neon : 1167 MB/sec [ 2.599079] neon : 1166 MB/sec [ 2.579252] neon : 1147 MB/sec [ 2.582637] neon : 1168 MB/sec [ 2.582872] neon : 1164 MB/sec [ 2.570671] neon : 1167 MB/sec [ 2.571830] neon : 1166 MB/sec
mainline @ 626b2dda7651 with series:
[ 2.570227] neon : 1238 MB/sec [ 2.571642] neon : 1237 MB/sec [ 2.580370] neon : 1234 MB/sec [ 2.581966] neon : 1238 MB/sec [ 2.582313] neon : 1236 MB/sec [ 2.572291] neon : 1238 MB/sec [ 2.570625] neon : 1233 MB/sec [ 2.571897] neon : 1234 MB/sec [ 2.589616] neon : 1228 MB/sec [ 2.582449] neon : 1236 MB/sec
This series is currently broken for powerpc [1], as the functions in arch/powerpc/lib/xor_vmx.c were not updated.
arch/powerpc/lib/xor_vmx.c:52:6: error: conflicting types for '__xor_altivec_2' void __xor_altivec_2(unsigned long bytes, unsigned long *v1_in,
^
arch/powerpc/lib/xor_vmx.h:9:6: note: previous declaration is here
void __xor_altivec_2(unsigned long bytes, unsigned long * __restrict p1,
^
arch/powerpc/lib/xor_vmx.c:70:6: error: conflicting types for '__xor_altivec_3'
void __xor_altivec_3(unsigned long bytes, unsigned long *v1_in,
^
arch/powerpc/lib/xor_vmx.h:11:6: note: previous declaration is here
void __xor_altivec_3(unsigned long bytes, unsigned long * __restrict p1,
^
arch/powerpc/lib/xor_vmx.c:92:6: error: conflicting types for '__xor_altivec_4'
void __xor_altivec_4(unsigned long bytes, unsigned long *v1_in,
^
arch/powerpc/lib/xor_vmx.h:14:6: note: previous declaration is here
void __xor_altivec_4(unsigned long bytes, unsigned long * __restrict p1,
^
arch/powerpc/lib/xor_vmx.c:119:6: error: conflicting types for '__xor_altivec_5'
void __xor_altivec_5(unsigned long bytes, unsigned long *v1_in,
^
arch/powerpc/lib/xor_vmx.h:18:6: note: previous declaration is here
void __xor_altivec_5(unsigned long bytes, unsigned long * __restrict p1,
^4 errors generated.
If I fix that up [2], it builds and resolves an instance of -Wframe-larger-than= in the xor altivec code, as seen with pmac32_defconfig.
Before this series:
arch/powerpc/lib/xor_vmx.c:119:6: error: stack frame size (1232) exceeds limit (1024) in '__xor_altivec_5' [-Werror,-Wframe-larger-than] void __xor_altivec_5(unsigned long bytes, unsigned long *v1_in,
^
1 error generated.
After this patch (with CONFIG_FRAME_WARN=100 andCONFIG_PPC_DISABLE_WERROR=y):
arch/powerpc/lib/xor_vmx.c:52:6: warning: stack frame size (128) exceeds limit (100) in '__xor_altivec_2' [-Wframe-larger-than] void __xor_altivec_2(unsigned long bytes,
^
arch/powerpc/lib/xor_vmx.c:71:6: warning: stack frame size (160) exceeds limit (100) in '__xor_altivec_3' [-Wframe-larger-than]
void __xor_altivec_3(unsigned long bytes,
^
arch/powerpc/lib/xor_vmx.c:95:6: warning: stack frame size (144) exceeds limit (100) in '__xor_altivec_4' [-Wframe-larger-than]
void __xor_altivec_4(unsigned long bytes,
^
arch/powerpc/lib/xor_vmx.c:124:6: warning: stack frame size (160) exceeds limit (100) in '__xor_altivec_5' [-Wframe-larger-than]
void __xor_altivec_5(unsigned long bytes,
^4 warnings generated.
There is a similar performance gain as ARM according to do_xor_speed():
Before:
altivec : 222 MB/sec altivec : 222 MB/sec altivec : 222 MB/sec altivec : 219 MB/sec altivec : 222 MB/sec altivec : 222 MB/sec altivec : 222 MB/sec altivec : 222 MB/sec altivec : 222 MB/sec altivec : 222 MB/sec
After:
altivec : 278 MB/sec altivec : 276 MB/sec altivec : 278 MB/sec altivec : 278 MB/sec altivec : 278 MB/sec altivec : 278 MB/sec altivec : 278 MB/sec altivec : 278 MB/sec altivec : 278 MB/sec altivec : 278 MB/sec
I did also build test arm64 and x86_64 and saw no errors. I did runtime test arm64 for improvements and did not see any, which is good, since I take that as meaning it was working fine before and there is no regression.
Once the build error is fixed, consider this series:
[1]: https://lore.kernel.org/r/[email protected]/
[2]: https://github.com/ClangBuiltLinux/linux/issues/563#issuecomment-1005175153
Cheers,
Nathan