XOR implementation for ARMv8
PATCH 1/1 XOR implementation for ARMv8
Use the 128-bit SIMD registers and SIMD arithmetic instructions for XOR calculation in assembly language.
Experimental results show that LDP/STP is more effective than LD1/ST1 for loading/restoring the operand, and we get better performance when using 16 SIMD registers than 32 registers. The result of xor speed test (measured by do_xor_speed) are as follows:
32regs : 4352.000 MB/sec 8regs : 4435.200 MB/sec ARM64-LD1-regs32: 38886.400 MB/sec ARM64-LD1-regs16: 45280.000 MB/sec ARM64-LDP-regs32: 44608.000 MB/sec ARM64-LDP-regs16: 53625.600 MB/sec
Iozone tests on disk array of RAID 5 show that the speed of of write operation can be improved by 15%~30%. This patch is currently against a linux 4.0.5 kernel for the arm64 architecture.
Please review, any input welcome.
---
include/asm/xor.h | 34 +++++++ kernel/arm64ksyms.c | 13 ++ lib/Makefile | 2 lib/xor.S | 228 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 276 insertions(+), 1 deletion(-)
--------------------------------------------------------------------------------
arch/arm64/include/asm/xor.h +34 -0
@@ -0,0 +1,34 @@ +/* +* arch/arm64/include/asm/xor.h +* +* Copyright (C) Xiaodong Liu <[email protected]>, Changsha, P.R. China +* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License version 2 as +* published by the Free Software Foundation. +*/ + +#include <asm-generic/xor.h> +extern void xor_arm64ldpregs16_2 (unsigned long ,unsigned long *,unsigned long *); +extern void xor_arm64ldpregs16_3 (unsigned long ,unsigned long *,unsigned long *, +unsigned long *); +extern void xor_arm64ldpregs16_4 (unsigned long ,unsigned long *,unsigned long *, +unsigned long *,unsigned long *); +extern void xor_arm64ldpregs16_5 (unsigned long ,unsigned long *,unsigned long *, +unsigned long *,unsigned long *,unsigned long *); + +static struct xor_block_template xor_block_arm64ldpregs16 = { + .name ="ARM64LDPregs16" , + .do_2 = xor_arm64ldpregs16_2, + .do_3 = xor_arm64ldpregs16_3, + .do_4 = xor_arm64ldpregs16_4, + .do_5 = xor_arm64ldpregs16_5, +}; + +#undef XOR_TRY_TEMPLATES +#define XOR_TRY_TEMPLATES \ + do { \ + xor_speed(&xor_block_arm64ldpregs16); \ + xor_speed(&xor_block_32regs); \ + xor_speed(&xor_block_8regs); \ + } while (0)
arch/arm64/kernel/arm64ksyms.c +13 -0
@@ -65,3 +65,16 @@ EXPORT_SYMBOL(test_and_change_bit);#ifdef CONFIG_FUNCTION_TRACER EXPORT_SYMBOL (_mcount);#endif + +/* xor ops */ +extern void xor_arm64ldpregs16_2 (unsigned long ,unsigned long *,unsigned long *); +extern void xor_arm64ldpregs16_3 (unsigned long ,unsigned long *,unsigned long *, +unsigned long *); +extern void xor_arm64ldpregs16_4 (unsigned long ,unsigned long *,unsigned long *, +unsigned long *,unsigned long *); +extern void xor_arm64ldpregs16_5 (unsigned long ,unsigned long *,unsigned long *, +unsigned long *,unsigned long *,unsigned long *); +EXPORT_SYMBOL (xor_arm64ldpregs16_2); +EXPORT_SYMBOL (xor_arm64ldpregs16_3); +EXPORT_SYMBOL (xor_arm64ldpregs16_4); +EXPORT_SYMBOL (xor_arm64ldpregs16_5);
arch/arm64/lib/Makefile +1 -1
@@ -2,4 +2,4 @@ lib-y := bitops.o clear_user.o delay.o copy_to_user.o copy_in_user.o copy_page.o \ clear_page.o memchr.o memcpy.o memmove.o memset.o \ memcmp.o strcmp.o strncmp.o strlen.o strnlen.o \ - strchr.o strrchr.o + strchr.o strrchr.o xor.o
arch/arm64/lib/xor.S +228 -0
228 lines not on this page · raw message
PATCH 1/1 XOR implementation for ARMv8
Quoted 1 line · ???
Use the 128-bit SIMD registers and SIMD arithmetic instructions for XOR calculation in assembly language.
Don't you need kernel_neon_begin()/kernel_neon_end() somewhere? (see Documentation/arm/kernel_mode_neon.txt).
Signature · 1 line
Jerome
PATCH 1/1 XOR implementation for ARMv8
Quoted 8 lines · J?r?me Forissier
Quoted 1 line · ???
Use the 128-bit SIMD registers and SIMD arithmetic instructions for XOR calculation in assembly language.
Don't you need kernel_neon_begin()/kernel_neon_end() somewhere? (see
Documentation/arm/kernel_mode_neon.txt).
Jerome is right: use of this driver will corrupt the FP/SIMD state of arbitrary userland tasks if you don't explicitly claim the NEON for in-kernel use by calling kernel_neon_begin)_ and end()
Since XOR may be called in interrupt context, this could add a fixed overhead to each call, even if you are calling the function many times in a row. This means you may be better off using even fewer registers, and use kernel_neon_begin_partial() instead.
May I ask what kind of core you tested this on?
Signature · 1 line
Ard.
PATCH 1/1 XOR implementation for ARMv8
In reply to 刘晓东Quoted 37 lines · ???
diff -pruN -X dontdiff
linux-4.0.5-orig/arch/arm64/lib/xor.Slinux-4.0.5-mod/arch/arm64/lib/xor.Sarch/arm64/lib/xor.S 2015-06-24 09:25:49.969256540 +0800 renamed from arch/arm64/lib/xor.S 1970-01-01 08:00:00.000000000 +0800 +33
-0@@ -0,0 +1,228 @@ +/* + * arch/arm64/lib/xor.S + * + * Copyright (C) Xiaodong Liu <[email protected]>, Changsha, P.R. China + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/linkage.h> +#include <asm/assembler.h> +.macro xor_vectorregs16 + eor v24.16b, v24.16b, v16.16b + eor v25.16b, v25.16b, v17.16b + eor v26.16b, v26.16b, v18.16b + eor v27.16b, v27.16b, v19.16b + eor v28.16b, v28.16b, v20.16b + eor v29.16b, v29.16b, v21.16b + eor v30.16b, v30.16b, v22.16b + eor v31.16b, v31.16b, v23.16b +.endm + +.align 4 + +/* + * void xor_arm64ldpregs16_2(unsigned long size, unsigned long * dst, unsigned long *src); + * + * Parameters: + * x0 - size + * x1 - dst + * x2 - src + */
arch/arm64/lib/xor.S 2015-06-24 09:25:49.969256540 +0800 line 43
Quoted 10 lines
arch/arm64/lib/xor.S 2015-06-24 09:25:49.969256540 +0800 renamed from arch/arm64/lib/xor.S 1970-01-01 08:00:00.000000000 +0800 +10
-0+ENTRY(xor_arm64ldpregs16_2) + + lsr x0, x0, #10 + +.p2align 4 +Loop23: + ldp q16, q17, [x2], #32 + ldp q18, q19, [x2], #32 + ldp q20, q21, [x2], #32 + ldp q22, q23, [x2], #32
Have you tried using immediate offsets instead of post-index addressing? E.g.
ldp q16, q17, [x2] ldp q18, q19, [x2, #32], #32 ldp q20, q21, [x2, #64], #32 ldp q22, q23, [x2, #96] add x2, x2, #128
Will
PATCH 1/1 XOR implementation for ARMv8
Quoted 52 lines · Will Deacon
Quoted 37 lines · ???
diff -pruN -X dontdiff
linux-4.0.5-orig/arch/arm64/lib/xor.Slinux-4.0.5-mod/arch/arm64/lib/xor.Sarch/arm64/lib/xor.S 2015-06-24 09:25:49.969256540 +0800 renamed from arch/arm64/lib/xor.S 1970-01-01 08:00:00.000000000 +0800 +33
-0@@ -0,0 +1,228 @@ +/* + * arch/arm64/lib/xor.S + * + * Copyright (C) Xiaodong Liu <[email protected]>, Changsha, P.R. China + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/linkage.h> +#include <asm/assembler.h> +.macro xor_vectorregs16 + eor v24.16b, v24.16b, v16.16b + eor v25.16b, v25.16b, v17.16b + eor v26.16b, v26.16b, v18.16b + eor v27.16b, v27.16b, v19.16b + eor v28.16b, v28.16b, v20.16b + eor v29.16b, v29.16b, v21.16b + eor v30.16b, v30.16b, v22.16b + eor v31.16b, v31.16b, v23.16b +.endm + +.align 4 + +/* + * void xor_arm64ldpregs16_2(unsigned long size, unsigned long * dst, unsigned long *src); + * + * Parameters: + * x0 - size + * x1 - dst + * x2 - src + */arch/arm64/lib/xor.S 2015-06-24 09:25:49.969256540 +0800 line 43
Quoted 10 lines
arch/arm64/lib/xor.S 2015-06-24 09:25:49.969256540 +0800 renamed from arch/arm64/lib/xor.S 1970-01-01 08:00:00.000000000 +0800 +10
-0+ENTRY(xor_arm64ldpregs16_2) + + lsr x0, x0, #10 + +.p2align 4 +Loop23: + ldp q16, q17, [x2], #32 + ldp q18, q19, [x2], #32 + ldp q20, q21, [x2], #32 + ldp q22, q23, [x2], #32Have you tried using immediate offsets instead of post-index addressing? E.g.
Quoted 3 lines
ldp q16, q17, [x2] ldp q18, q19, [x2, #32], #32 ldp q20, q21, [x2, #64], #32
Without the post-index offsets, of course ;)
Will