arm64:lib: the optimized string library routines for armv8 processors
cover arm64:lib: the optimized string library routines for armv8 processors
From: "zhichang.yuan" <[email protected]>
In current aarch64 kernel,there are a few string library routines implemented in arm64/lib,such as memcpy,memset, memmove,strchr. Most string routines frequently used are provided by the architecture-independent string library.Those routines are not efficient.
This patch focus on improving the sting routines' performance in ARMv8. It contains eight optimized functions.The work is based on the cortex-string project in Linaro toolchain.
The cortex-string code can be found in this website:
https://code.launchpad.net/cortex-strings
To obtain better performance,several ideas were utilized:
1) memory burst access;
For the long memory data operation,adopted the armv8 instruction pairs, ldp/stp,to transfer the bulk data.Try best to use continuous ldp/stp to trigger the burst access.
2) parallel processing
The current string routines mostly processed per-byte. This patch processes the data in parallel.Such as strlen, it will process eight string bytes each time.
3) aligned memory access
Classfy the process into several categories according to the input memory address parameters.For the non-alignment memory address,firstly process the begginning short-length data to make the memory address aligned,then start the remain processing on alignment address.
After the optimization,those routines have better performance than the current ones. Please refer to this website to get the test results:
https://wiki.linaro.org/WorkingGroups/Kernel/ARMv8/cortex-strings
--
zhichang.yuan (6):
arm64: lib: Implement optimized memcpy routine
arm64: lib: Implement optimized memmove routine
arm64: lib: Implement optimized memset routine
arm64: lib: Implement optimized memcmp routine
arm64: lib: Implement optimized string compare routines
arm64: lib: Implement optimized string length routines
arch/arm64/include/asm/string.h | 15 ++ arch/arm64/kernel/arm64ksyms.c | 5 + arch/arm64/lib/Makefile | 5 +- arch/arm64/lib/memcmp.S | 258 +++++++++++++++++++++++++++++ arch/arm64/lib/memcpy.S | 182 ++++++++++++++++++--- arch/arm64/lib/memmove.S | 195 ++++++++++++++++++---- arch/arm64/lib/memset.S | 227 +++++++++++++++++++++++--- arch/arm64/lib/strcmp.S | 256 +++++++++++++++++++++++++++++ arch/arm64/lib/strlen.S | 131 +++++++++++++++ arch/arm64/lib/strncmp.S | 340 +++++++++++++++++++++++++++++++++++++++ arch/arm64/lib/strnlen.S | 179 +++++++++++++++++++++ 11 files changed, 1714 insertions(+), 79 deletions(-) create mode 100644 arch/arm64/lib/memcmp.S create mode 100644 arch/arm64/lib/strcmp.S create mode 100644 arch/arm64/lib/strlen.S create mode 100644 arch/arm64/lib/strncmp.S create mode 100644 arch/arm64/lib/strnlen.S
Signature · 1 line
1.7.9.5
PATCH 1/6 arm64: lib: Implement optimized memcpy routine
From: "zhichang.yuan" <[email protected]>
This patch, based on Linaro's Cortex Strings library, improves the performance of the assembly optimized memcpy() function.
--- arch/arm64/lib/memcpy.S | 182 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 160 insertions(+), 22 deletions(-)
arch/arm64/lib/memcpy.S +160 -22
@@ -1,5 +1,13 @@ /* * Copyright (C) 2013 ARM Ltd. + * Copyright (C) 2013 Linaro. + * + * This code is based on glibc cortex strings work originally authored by Linaro + * and re-licensed under GPLv2 for the Linux kernel. The original code can + * be found @ + * + * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/ + * files/head:/src/aarch64/ * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as
@@ -27,27 +35,157 @@ * Returns: * x0 - dest */ +#define dstin x0 +#define src x1 +#define count x2 +#define tmp1 x3 +#define tmp1w w3 +#define tmp2 x4 +#define tmp2w w4 +#define tmp3 x5 +#define tmp3w w5 +#define dst x6 + +#define A_l x7 +#define A_h x8 +#define B_l x9 +#define B_h x10 +#define C_l x11 +#define C_h x12 +#define D_l x13 +#define D_h x14 + ENTRY(memcpy) - mov x4, x0 - subs x2, x2, #8 - b.mi 2f -1: ldr x3, [x1], #8 - subs x2, x2, #8 - str x3, [x4], #8 - b.pl 1b -2: adds x2, x2, #4 - b.mi 3f - ldr w3, [x1], #4 - sub x2, x2, #4 - str w3, [x4], #4 -3: adds x2, x2, #2 - b.mi 4f - ldrh w3, [x1], #2 - sub x2, x2, #2 - strh w3, [x4], #2 -4: adds x2, x2, #1 - b.mi 5f - ldrb w3, [x1] - strb w3, [x4] -5: ret + mov dst, dstin + cmp count, #16 + /*If memory length is less than 16, stp or ldp can not be used.*/ + b.lo .Ltiny15 +.Lover16: + neg tmp2, src + ands tmp2, tmp2, #15/* Bytes to reach alignment. */ + b.eq .LSrcAligned + sub count, count, tmp2 + /* + * Use ldp and sdp to copy 16 bytes,then backward the src to + * aligned address.This way is more efficient. + * But the risk overwriting the source area exists.Here,prefer to + * access memory forward straight,no backward.It will need a bit + * more instructions, but on the same time,the accesses are aligned. + */ + tbz tmp2, #0, 1f + ldrb tmp1w, [src], #1 + strb tmp1w, [dst], #1 +1: + tbz tmp2, #1, 1f + ldrh tmp1w, [src], #2 + strh tmp1w, [dst], #2 +1: + tbz tmp2, #2, 1f + ldr tmp1w, [src], #4 + str tmp1w, [dst], #4 +1: + tbz tmp2, #3, .LSrcAligned + ldr tmp1, [src],#8 + str tmp1, [dst],#8 + +.LSrcAligned: + cmp count, #64 + b.ge .Lcpy_over64 + /* + * Deal with small copies quickly by dropping straight into the + * exit block. + */ +.Ltail63: + /* + * Copy up to 48 bytes of data. At this point we only need the + * bottom 6 bits of count to be accurate. + */ + ands tmp1, count, #0x30 + b.eq .Ltiny15 + cmp tmp1w, #0x20 + b.eq 1f + b.lt 2f + ldp A_l, A_h, [src], #16 + stp A_l, A_h, [dst], #16 +1: + ldp A_l, A_h, [src], #16 + stp A_l, A_h, [dst], #16 +2: + ldp A_l, A_h, [src], #16 + stp A_l, A_h, [dst], #16 +.Ltiny15: + /* + * To make memmove simpler, here don't make src backwards. + * since backwards will probably overwrite the src area where src + * data for nex copy located,if dst is not so far from src. + */ + tbz count, #3, 1f + ldr tmp1, [src], #8 + str tmp1, [dst], #8 +1: + tbz count, #2, 1f + ldr tmp1w, [src], #4 + str tmp1w, [dst], #4 +1: + tbz count, #1, 1f + ldrh tmp1w, [src], #2 + strh tmp1w, [dst], #2 +1: + tbz count, #0, .Lexitfunc + ldrb tmp1w, [src] + strb tmp1w, [dst] + +.Lexitfunc: + ret + +.Lcpy_over64: + subs count, count, #128 + b.ge .Lcpy_body_large + /* + * Less than 128 bytes to copy, so handle 64 here and then jump + * to the tail. + */ + ldp A_l, A_h, [src],#16 + stp A_l, A_h, [dst],#16 + ldp B_l, B_h, [src],#16 + ldp C_l, C_h, [src],#16 + stp B_l, B_h, [dst],#16 + stp C_l, C_h, [dst],#16 + ldp D_l, D_h, [src],#16 + stp D_l, D_h, [dst],#16 + + tst count, #0x3f + b.ne .Ltail63 + ret + + /* + * Critical loop. Start at a new cache line boundary. Assuming + * 64 bytes per line this ensures the entire loop is in one line. + */ + .p2align 6 +.Lcpy_body_large: + /* There are at least 128 bytes to copy. */ + ldp A_l, A_h, [src],#16 + ldp B_l, B_h, [src],#16 + ldp C_l, C_h, [src],#16 + ldp D_l, D_h, [src],#16 +1: + stp A_l, A_h, [dst],#16 + ldp A_l, A_h, [src],#16 + stp B_l, B_h, [dst],#16 + ldp B_l, B_h, [src],#16 + stp C_l, C_h, [dst],#16 + ldp C_l, C_h, [src],#16 + stp D_l, D_h, [dst],#16 + ldp D_l, D_h, [src],#16 + subs count, count, #64 + b.ge 1b + stp A_l, A_h, [dst],#16 + stp B_l, B_h, [dst],#16 + stp C_l, C_h, [dst],#16 + stp D_l, D_h, [dst],#16 + + tst count, #0x3f + b.ne .Ltail63 + ret ENDPROC(memcpy)
Signature · 1 line
1.7.9.5
PATCH 1/6 arm64: lib: Implement optimized memcpy routine
Quoted 42 lines · zhichang.yuan at linaro.org
From: "zhichang.yuan" <[email protected]>
This patch, based on Linaro's Cortex Strings library, improves the performance of the assembly optimized
memcpy()function.Signed-off-by Zhichang Yuan <[email protected]>Signed-off-by Deepak Saxena <[email protected]>--- arch/arm64/lib/memcpy.S | 182 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 160 insertions(+), 22 deletions(-)arch/arm64/lib/memcpy.S +17
-0@@ -1,5 +1,13 @@ /* * Copyright (C) 2013 ARM Ltd. + * Copyright (C) 2013 Linaro. + * + * This code is based on glibc cortex strings work originally authored by Linaro + * and re-licensed under GPLv2 for the Linux kernel. The original code can + * be found @ + * + * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/ + * files/head:/src/aarch64/ * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as@@ -27,27 +35,157 @@ * Returns: * x0 - dest */ +#define dstin x0 +#define src x1 +#define count x2 +#define tmp1 x3 +#define tmp1w w3 +#define tmp2 x4 +#define tmp2w w4 +#define tmp3 x5 +#define tmp3w w5
arch/arm64/lib/memcpy.S line 56
Quoted 10 lines
arch/arm64/lib/memcpy.S +10
-0+#define dst x6 + +#define A_l x7 +#define A_h x8 +#define B_l x9 +#define B_h x10 +#define C_l x11 +#define C_h x12 +#define D_l x13 +#define D_h x14
Use .req instead of #define?
Quoted 29 lines
arch/arm64/lib/memcpy.S +6
-22ENTRY(memcpy) - mov x4, x0 - subs x2, x2, #8 - b.mi 2f -1: ldr x3, [x1], #8 - subs x2, x2, #8 - str x3, [x4], #8 - b.pl 1b -2: adds x2, x2, #4 - b.mi 3f - ldr w3, [x1], #4 - sub x2, x2, #4 - str w3, [x4], #4 -3: adds x2, x2, #2 - b.mi 4f - ldrh w3, [x1], #2 - sub x2, x2, #2 - strh w3, [x4], #2 -4: adds x2, x2, #1 - b.mi 5f - ldrb w3, [x1] - strb w3, [x4] -5: ret + mov dst, dstin + cmp count, #16 + /*If memory length is less than 16, stp or ldp can not be used.*/ + b.lo .Ltiny15 +.Lover16: + neg tmp2, src
arch/arm64/lib/memcpy.S line 74
Quoted 10 lines
arch/arm64/lib/memcpy.S +10
-0+ ands tmp2, tmp2, #15/* Bytes to reach alignment. */ + b.eq .LSrcAligned + sub count, count, tmp2 + /* + * Use ldp and sdp to copy 16 bytes,then backward the src to + * aligned address.This way is more efficient. + * But the risk overwriting the source area exists.Here,prefer to + * access memory forward straight,no backward.It will need a bit + * more instructions, but on the same time,the accesses are aligned. + */
This comment reads very badly:
- sdp doesn't exist
- `more efficient' than what? How is this measured?
- `access memory forward straight,no backward' What?
Please re-write it in a clearer fashion, so that reviewers have some understanding of your optimisations when potentially trying to change the code later on.
arch/arm64/lib/memcpy.S line 86
Quoted 12 lines
arch/arm64/lib/memcpy.S +12
-0+ tbz tmp2, #0, 1f + ldrb tmp1w, [src], #1 + strb tmp1w, [dst], #1 +1: + tbz tmp2, #1, 1f + ldrh tmp1w, [src], #2 + strh tmp1w, [dst], #2 +1: + tbz tmp2, #2, 1f + ldr tmp1w, [src], #4 + str tmp1w, [dst], #4 +1:
Three labels called '1:' ? Can you make them unique please (the old code just incremented a counter).
Quoted 25 lines
arch/arm64/lib/memcpy.S +25
-0+ tbz tmp2, #3, .LSrcAligned + ldr tmp1, [src],#8 + str tmp1, [dst],#8 + +.LSrcAligned: + cmp count, #64 + b.ge .Lcpy_over64 + /* + * Deal with small copies quickly by dropping straight into the + * exit block. + */ +.Ltail63: + /* + * Copy up to 48 bytes of data. At this point we only need the + * bottom 6 bits of count to be accurate. + */ + ands tmp1, count, #0x30 + b.eq .Ltiny15 + cmp tmp1w, #0x20 + b.eq 1f + b.lt 2f + ldp A_l, A_h, [src], #16 + stp A_l, A_h, [dst], #16 +1: + ldp A_l, A_h, [src], #16
arch/arm64/lib/memcpy.S line 121
Quoted 10 lines
arch/arm64/lib/memcpy.S +10
-0+ stp A_l, A_h, [dst], #16 +2: + ldp A_l, A_h, [src], #16 + stp A_l, A_h, [dst], #16 +.Ltiny15: + /* + * To make memmove simpler, here don't make src backwards. + * since backwards will probably overwrite the src area where src + * data for nex copy located,if dst is not so far from src. + */
Another awful comment...
arch/arm64/lib/memcpy.S line 133
Quoted 12 lines
arch/arm64/lib/memcpy.S +12
-0+ tbz count, #3, 1f + ldr tmp1, [src], #8 + str tmp1, [dst], #8 +1: + tbz count, #2, 1f + ldr tmp1w, [src], #4 + str tmp1w, [dst], #4 +1: + tbz count, #1, 1f + ldrh tmp1w, [src], #2 + strh tmp1w, [dst], #2 +1:
... and more of these labels.
Quoted 22 lines
arch/arm64/lib/memcpy.S +22
-0+ tbz count, #0, .Lexitfunc + ldrb tmp1w, [src] + strb tmp1w, [dst] + +.Lexitfunc: + ret + +.Lcpy_over64: + subs count, count, #128 + b.ge .Lcpy_body_large + /* + * Less than 128 bytes to copy, so handle 64 here and then jump + * to the tail. + */ + ldp A_l, A_h, [src],#16 + stp A_l, A_h, [dst],#16 + ldp B_l, B_h, [src],#16 + ldp C_l, C_h, [src],#16 + stp B_l, B_h, [dst],#16 + stp C_l, C_h, [dst],#16 + ldp D_l, D_h, [src],#16 + stp D_l, D_h, [dst],#16
arch/arm64/lib/memcpy.S line 165
Quoted 10 lines
arch/arm64/lib/memcpy.S +10
-0+ + tst count, #0x3f + b.ne .Ltail63 + ret + + /* + * Critical loop. Start at a new cache line boundary. Assuming + * 64 bytes per line this ensures the entire loop is in one line. + */ + .p2align 6
Can you parameterise this with L1_CACHE_SHIFT instead?
Will
PATCH 1/6 arm64: lib: Implement optimized memcpy routine
Hi Will
Thanks for your reply.
I think your comments are ok, i will modify the patches involved those questions. After those fixes are ready, the patch v2 will be submitted.
Thanks again!
Zhichang
Quoted 212 lines · Will Deacon
Quoted 42 lines · zhichang.yuan at linaro.org
From: "zhichang.yuan" <[email protected]>
This patch, based on Linaro's Cortex Strings library, improves the performance of the assembly optimized
memcpy()function.Signed-off-by Zhichang Yuan <[email protected]>Signed-off-by Deepak Saxena <[email protected]>--- arch/arm64/lib/memcpy.S | 182 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 160 insertions(+), 22 deletions(-)arch/arm64/lib/memcpy.S +17
-0@@ -1,5 +1,13 @@ /* * Copyright (C) 2013 ARM Ltd. + * Copyright (C) 2013 Linaro. + * + * This code is based on glibc cortex strings work originally authored by Linaro + * and re-licensed under GPLv2 for the Linux kernel. The original code can + * be found @ + * + * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/ + * files/head:/src/aarch64/ * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as@@ -27,27 +35,157 @@ * Returns: * x0 - dest */ +#define dstin x0 +#define src x1 +#define count x2 +#define tmp1 x3 +#define tmp1w w3 +#define tmp2 x4 +#define tmp2w w4 +#define tmp3 x5 +#define tmp3w w5arch/arm64/lib/memcpy.S line 56
Quoted 10 lines
arch/arm64/lib/memcpy.S +10
-0+#define dst x6 + +#define A_l x7 +#define A_h x8 +#define B_l x9 +#define B_h x10 +#define C_l x11 +#define C_h x12 +#define D_l x13 +#define D_h x14Use .req instead of #define?
Quoted 29 lines
arch/arm64/lib/memcpy.S +6
-22ENTRY(memcpy) - mov x4, x0 - subs x2, x2, #8 - b.mi 2f -1: ldr x3, [x1], #8 - subs x2, x2, #8 - str x3, [x4], #8 - b.pl 1b -2: adds x2, x2, #4 - b.mi 3f - ldr w3, [x1], #4 - sub x2, x2, #4 - str w3, [x4], #4 -3: adds x2, x2, #2 - b.mi 4f - ldrh w3, [x1], #2 - sub x2, x2, #2 - strh w3, [x4], #2 -4: adds x2, x2, #1 - b.mi 5f - ldrb w3, [x1] - strb w3, [x4] -5: ret + mov dst, dstin + cmp count, #16 + /*If memory length is less than 16, stp or ldp can not be used.*/ + b.lo .Ltiny15 +.Lover16: + neg tmp2, srcarch/arm64/lib/memcpy.S line 74
Quoted 10 lines
arch/arm64/lib/memcpy.S +10
-0+ ands tmp2, tmp2, #15/* Bytes to reach alignment. */ + b.eq .LSrcAligned + sub count, count, tmp2 + /* + * Use ldp and sdp to copy 16 bytes,then backward the src to + * aligned address.This way is more efficient. + * But the risk overwriting the source area exists.Here,prefer to + * access memory forward straight,no backward.It will need a bit + * more instructions, but on the same time,the accesses are aligned. + */This comment reads very badly:
- sdp doesn't exist
- `more efficient' than what? How is this measured?
- `access memory forward straight,no backward' What?Please re-write it in a clearer fashion, so that reviewers have some understanding of your optimisations when potentially trying to change the code later on.
arch/arm64/lib/memcpy.S line 86
Quoted 12 lines
arch/arm64/lib/memcpy.S +12
-0+ tbz tmp2, #0, 1f + ldrb tmp1w, [src], #1 + strb tmp1w, [dst], #1 +1: + tbz tmp2, #1, 1f + ldrh tmp1w, [src], #2 + strh tmp1w, [dst], #2 +1: + tbz tmp2, #2, 1f + ldr tmp1w, [src], #4 + str tmp1w, [dst], #4 +1:Three labels called '1:' ? Can you make them unique please (the old code just incremented a counter).
Quoted 25 lines
arch/arm64/lib/memcpy.S +25
-0+ tbz tmp2, #3, .LSrcAligned + ldr tmp1, [src],#8 + str tmp1, [dst],#8 + +.LSrcAligned: + cmp count, #64 + b.ge .Lcpy_over64 + /* + * Deal with small copies quickly by dropping straight into the + * exit block. + */ +.Ltail63: + /* + * Copy up to 48 bytes of data. At this point we only need the + * bottom 6 bits of count to be accurate. + */ + ands tmp1, count, #0x30 + b.eq .Ltiny15 + cmp tmp1w, #0x20 + b.eq 1f + b.lt 2f + ldp A_l, A_h, [src], #16 + stp A_l, A_h, [dst], #16 +1: + ldp A_l, A_h, [src], #16arch/arm64/lib/memcpy.S line 121
Quoted 10 lines
arch/arm64/lib/memcpy.S +10
-0+ stp A_l, A_h, [dst], #16 +2: + ldp A_l, A_h, [src], #16 + stp A_l, A_h, [dst], #16 +.Ltiny15: + /* + * To make memmove simpler, here don't make src backwards. + * since backwards will probably overwrite the src area where src + * data for nex copy located,if dst is not so far from src. + */Another awful comment...
arch/arm64/lib/memcpy.S line 133
Quoted 12 lines
arch/arm64/lib/memcpy.S +12
-0+ tbz count, #3, 1f + ldr tmp1, [src], #8 + str tmp1, [dst], #8 +1: + tbz count, #2, 1f + ldr tmp1w, [src], #4 + str tmp1w, [dst], #4 +1: + tbz count, #1, 1f + ldrh tmp1w, [src], #2 + strh tmp1w, [dst], #2 +1:... and more of these labels.
Quoted 22 lines
arch/arm64/lib/memcpy.S +22
-0+ tbz count, #0, .Lexitfunc + ldrb tmp1w, [src] + strb tmp1w, [dst] + +.Lexitfunc: + ret + +.Lcpy_over64: + subs count, count, #128 + b.ge .Lcpy_body_large + /* + * Less than 128 bytes to copy, so handle 64 here and then jump + * to the tail. + */ + ldp A_l, A_h, [src],#16 + stp A_l, A_h, [dst],#16 + ldp B_l, B_h, [src],#16 + ldp C_l, C_h, [src],#16 + stp B_l, B_h, [dst],#16 + stp C_l, C_h, [dst],#16 + ldp D_l, D_h, [src],#16 + stp D_l, D_h, [dst],#16arch/arm64/lib/memcpy.S line 165
Quoted 10 lines
arch/arm64/lib/memcpy.S +10
-0+ + tst count, #0x3f + b.ne .Ltail63 + ret + + /* + * Critical loop. Start at a new cache line boundary. Assuming + * 64 bytes per line this ensures the entire loop is in one line. + */ + .p2align 6Can you parameterise this with
L1_CACHE_SHIFTinstead?Will
PATCH 2/6 arm64: lib: Implement optimized memmove routine
In reply to zhichang.yuan at linaro.orgFrom: "zhichang.yuan" <[email protected]>
This patch, based on Linaro's Cortex Strings library, improves the performance of the assembly optimized memmove() function.
--- arch/arm64/lib/memmove.S | 195 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 166 insertions(+), 29 deletions(-)
arch/arm64/lib/memmove.S +166 -29
@@ -1,13 +1,21 @@ /* * Copyright (C) 2013 ARM Ltd. + * Copyright (C) 2013 Linaro. + * + * This code is based on glibc cortex strings work originally authored by Linaro + * and re-licensed under GPLv2 for the Linux kernel. The original code can + * be found @ + * + * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/ + * files/head:/src/aarch64/ * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * This program is distributed "as is" WITHOUT ANY WARRANTY of any + * kind, whether express or implied; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License
@@ -18,7 +26,8 @@ #include <asm/assembler.h> /* - * Move a buffer from src to test (alignment handled by the hardware). + * Move a buffer from src to test + * (alignment handled by the hardware for part cases). * If dest <= src, call memcpy, otherwise copy in reverse order. * * Parameters:
@@ -28,30 +37,158 @@ * Returns: * x0 - dest */ +#define dstin x0 +#define src x1 +#define count x2 +#define tmp1 x3 +#define tmp1w w3 +#define tmp2 x4 +#define tmp2w w4 +#define tmp3 x5 +#define tmp3w w5 +#define dst x6 + +#define A_l x7 +#define A_h x8 +#define B_l x9 +#define B_h x10 +#define C_l x11 +#define C_h x12 +#define D_l x13 +#define D_h x14 + ENTRY(memmove) - cmp x0, x1 - b.ls memcpy - add x4, x0, x2 - add x1, x1, x2 - subs x2, x2, #8 - b.mi 2f -1: ldr x3, [x1, #-8]! - subs x2, x2, #8 - str x3, [x4, #-8]! - b.pl 1b -2: adds x2, x2, #4 - b.mi 3f - ldr w3, [x1, #-4]! - sub x2, x2, #4 - str w3, [x4, #-4]! -3: adds x2, x2, #2 - b.mi 4f - ldrh w3, [x1, #-2]! - sub x2, x2, #2 - strh w3, [x4, #-2]! -4: adds x2, x2, #1 - b.mi 5f - ldrb w3, [x1, #-1] - strb w3, [x4, #-1] -5: ret + cmp dstin, src + /*b.eq .Lexitfunc*/ + b.lo memcpy + add tmp1, src, count + cmp dstin, tmp1 + b.hs memcpy /* No overlap. */ + + add dst, dstin, count + add src, src, count + cmp count, #16 + b.lo .Ltail15 + +.Lover16: + ands tmp2, src, #15 /* Bytes to reach alignment. */ + b.eq .LSrcAligned + sub count, count, tmp2 + /* + * process the aligned offset length to make the src aligned firstly. + * those extra instructions' cost is acceptable. It also make the + * coming accesses are based on aligned address. + */ + tbz tmp2, #0, 1f + ldrb tmp1w, [src, #-1]! + strb tmp1w, [dst, #-1]! +1: + tbz tmp2, #1, 1f + ldrh tmp1w, [src, #-2]! + strh tmp1w, [dst, #-2]! +1: + tbz tmp2, #2, 1f + ldr tmp1w, [src, #-4]! + str tmp1w, [dst, #-4]! +1: + tbz tmp2, #3, .LSrcAligned + ldr tmp1, [src, #-8]! + str tmp1, [dst, #-8]! + +.LSrcAligned: + cmp count, #64 + b.ge .Lcpy_over64 + + /* + * Deal with small copies quickly by dropping straight into the + * exit block.*/ +.Ltail63: + /* + * Copy up to 48 bytes of data. At this point we only need the + * bottom 6 bits of count to be accurate. + */ + ands tmp1, count, #0x30 + b.eq .Ltail15 + cmp tmp1w, #0x20 + b.eq 1f + b.lt 2f + ldp A_l, A_h, [src, #-16]! + stp A_l, A_h, [dst, #-16]! +1: + ldp A_l, A_h, [src, #-16]! + stp A_l, A_h, [dst, #-16]! +2: + ldp A_l, A_h, [src, #-16]! + stp A_l, A_h, [dst, #-16]! + +.Ltail15: + tbz count, #3, 1f + ldr tmp1, [src, #-8]! + str tmp1, [dst, #-8]! +1: + tbz count, #2, 1f + ldr tmp1w, [src, #-4]! + str tmp1w, [dst, #-4]! +1: + tbz count, #1, 1f + ldrh tmp1w, [src, #-2]! + strh tmp1w, [dst, #-2]! +1: + tbz count, #0, .Lexitfunc + ldrb tmp1w, [src, #-1] + strb tmp1w, [dst, #-1] + +.Lexitfunc: + ret + +.Lcpy_over64: + subs count, count, #128 + b.ge .Lcpy_body_large + /* + * Less than 128 bytes to copy, so handle 64 here and then jump + * to the tail. + */ + ldp A_l, A_h, [src, #-16] + stp A_l, A_h, [dst, #-16] + ldp B_l, B_h, [src, #-32] + ldp C_l, C_h, [src, #-48] + stp B_l, B_h, [dst, #-32] + stp C_l, C_h, [dst, #-48] + ldp D_l, D_h, [src, #-64]! + stp D_l, D_h, [dst, #-64]! + + tst count, #0x3f + b.ne .Ltail63 + ret + + /* + * Critical loop. Start at a new cache line boundary. Assuming + * 64 bytes per line this ensures the entire loop is in one line. + */ + .p2align 6 +.Lcpy_body_large: + /* There are@least 128 bytes to copy. */ + ldp A_l, A_h, [src, #-16] + ldp B_l, B_h, [src, #-32] + ldp C_l, C_h, [src, #-48] + ldp D_l, D_h, [src, #-64]! +1: + stp A_l, A_h, [dst, #-16] + ldp A_l, A_h, [src, #-16] + stp B_l, B_h, [dst, #-32] + ldp B_l, B_h, [src, #-32] + stp C_l, C_h, [dst, #-48] + ldp C_l, C_h, [src, #-48] + stp D_l, D_h, [dst, #-64]! + ldp D_l, D_h, [src, #-64]! + subs count, count, #64 + b.ge 1b + stp A_l, A_h, [dst, #-16] + stp B_l, B_h, [dst, #-32] + stp C_l, C_h, [dst, #-48] + stp D_l, D_h, [dst, #-64]! + + tst count, #0x3f + b.ne .Ltail63 + ret ENDPROC(memmove)
Signature · 1 line
1.7.9.5
PATCH 3/6 arm64: lib: Implement optimized memset routine
In reply to zhichang.yuan at linaro.orgFrom: "zhichang.yuan" <[email protected]>
This patch, based on Linaro's Cortex Strings library, improves the performance of the assembly optimized memset() function.
--- arch/arm64/lib/memset.S | 227 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 201 insertions(+), 26 deletions(-)
arch/arm64/lib/memset.S +201 -26
@@ -1,13 +1,21 @@ /* * Copyright (C) 2013 ARM Ltd. + * Copyright (C) 2013 Linaro. + * + * This code is based on glibc cortex strings work originally authored by Linaro + * and re-licensed under GPLv2 for the Linux kernel. The original code can + * be found @ + * + * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/ + * files/head:/src/aarch64/ * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * This program is distributed "as is" WITHOUT ANY WARRANTY of any + * kind, whether express or implied; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License
@@ -18,7 +26,7 @@ #include <asm/assembler.h> /* - * Fill in the buffer with character c (alignment handled by the hardware) + * Fill in the buffer with character c * * Parameters: * x0 - buf
@@ -27,27 +35,194 @@ * Returns: * x0 - buf */ + +/* By default we assume that the DC instruction can be used to zero +* data blocks more efficiently. In some circumstances this might be +* unsafe, for example in an asymmetric multiprocessor environment with +* different DC clear lengths (neither the upper nor lower lengths are +* safe to use). The feature can be disabled by defining DONT_USE_DC. +*/ + +#define dstin x0 +#define val w1 +#define count x2 +#define tmp1 x3 +#define tmp1w w3 +#define tmp2 x4 +#define tmp2w w4 +#define zva_len_x x5 +#define zva_len w5 +#define zva_bits_x x6 + +#define A_l x7 +#define A_lw w7 +#define dst x8 +#define tmp3w w9 +#define tmp3 x9 + ENTRY(memset) - mov x4, x0 - and w1, w1, #0xff - orr w1, w1, w1, lsl #8 - orr w1, w1, w1, lsl #16 - orr x1, x1, x1, lsl #32 - subs x2, x2, #8 - b.mi 2f -1: str x1, [x4], #8 - subs x2, x2, #8 - b.pl 1b -2: adds x2, x2, #4 - b.mi 3f - sub x2, x2, #4 - str w1, [x4], #4 -3: adds x2, x2, #2 - b.mi 4f - sub x2, x2, #2 - strh w1, [x4], #2 -4: adds x2, x2, #1 - b.mi 5f - strb w1, [x4] -5: ret + mov dst, dstin /* Preserve return value. */ + and A_lw, val, #255 + orr A_lw, A_lw, A_lw, lsl #8 + orr A_lw, A_lw, A_lw, lsl #16 + orr A_l, A_l, A_l, lsl #32 + + /*first align dst with 16...*/ + neg tmp2, dst + ands tmp2, tmp2, #15 + b.eq .Laligned + + cmp count, #15 + b.le .Ltail15tiny + /* + * The count is not less than 16, we can use stp to set 16 bytes + * once. This way is more efficient but the access is non-aligned. + */ + stp A_l, A_l, [dst] + /*make the dst aligned..*/ + sub count, count, tmp2 + add dst, dst, tmp2 + + /*Here, dst is aligned 16 now...*/ +.Laligned: +#ifndef DONT_USE_DC + cbz A_l, .Lzero_mem +#endif + +.Ltail_maybe_long: + cmp count, #64 + b.ge .Lnot_short +.Ltail63: + ands tmp1, count, #0x30 + b.eq .Ltail15tiny + cmp tmp1w, #0x20 + b.eq 1f + b.lt 2f + stp A_l, A_l, [dst], #16 +1: + stp A_l, A_l, [dst], #16 +2: + stp A_l, A_l, [dst], #16 +/* +* The following process is non-aligned access. But it is more efficient than +* .Ltail15tiny. Of-course, we can delete this code, but have a bit +* performance cost. +*/ + ands count, count, #15 + cbz count, 1f + add dst, dst, count + stp A_l, A_l, [dst, #-16] /* Repeat some/all of last store. */ +1: + ret + +.Ltail15tiny: + /* Set up to 15 bytes. Does not assume earlier memory + being set. */ + tbz count, #3, 1f + str A_l, [dst], #8 +1: + tbz count, #2, 1f + str A_lw, [dst], #4 +1: + tbz count, #1, 1f + strh A_lw, [dst], #2 +1: + tbz count, #0, 1f + strb A_lw, [dst] +1: + ret + + /* + * Critical loop. Start at a new cache line boundary. Assuming + * 64 bytes per line, this ensures the entire loop is in one line. + */ + .p2align 6 +.Lnot_short: /*count must be not less than 64*/ + sub dst, dst, #16/* Pre-bias. */ + sub count, count, #64 +1: + stp A_l, A_l, [dst, #16] + stp A_l, A_l, [dst, #32] + stp A_l, A_l, [dst, #48] + stp A_l, A_l, [dst, #64]! + subs count, count, #64 + b.ge 1b + tst count, #0x3f + add dst, dst, #16 + b.ne .Ltail63 +.Lexitfunc: + ret + +#ifndef DONT_USE_DC + /* + * For zeroing memory, check to see if we can use the ZVA feature to + * zero entire 'cache' lines. + */ +.Lzero_mem: + cmp count, #63 + b.le .Ltail63 + /* + * For zeroing small amounts of memory, it's not worth setting up + * the line-clear code. + */ + cmp count, #128 + b.lt .Lnot_short /*count is at least 128 bytes*/ + + mrs tmp1, dczid_el0 + tbnz tmp1, #4, .Lnot_short + mov tmp3w, #4 + and zva_len, tmp1w, #15 /* Safety: other bits reserved. */ + lsl zva_len, tmp3w, zva_len + + ands tmp3w, zva_len, #63 + /* + * ensure the zva_len is not less than 64. + * It is not meaningful to use ZVA if the block size is less than 64. + */ + b.ne .Lnot_short +.Lzero_by_line: + /* + * Compute how far we need to go to become suitably aligned. We're + * already at quad-word alignment. + */ + cmp count, zva_len_x + b.lt .Lnot_short /* Not enough to reach alignment. */ + sub zva_bits_x, zva_len_x, #1 + neg tmp2, dst + ands tmp2, tmp2, zva_bits_x + b.eq 1f /* Already aligned. */ + /* Not aligned, check that there's enough to copy after alignment.*/ + sub tmp1, count, tmp2 + /* + * grantee the remain length to be ZVA is bigger than 64, + * avoid to make the 2f's process over mem range.*/ + cmp tmp1, #64 + ccmp tmp1, zva_len_x, #8, ge /* NZCV=0b1000 */ + b.lt .Lnot_short + /* + * We know that there's at least 64 bytes to zero and that it's safe + * to overrun by 64 bytes. + */ + mov count, tmp1 +2: + stp A_l, A_l, [dst] + stp A_l, A_l, [dst, #16] + stp A_l, A_l, [dst, #32] + subs tmp2, tmp2, #64 + stp A_l, A_l, [dst, #48] + add dst, dst, #64 + b.ge 2b + /* We've overrun a bit, so adjust dst downwards.*/ + add dst, dst, tmp2 +1: + sub count, count, zva_len_x +3: + dc zva, dst + add dst, dst, zva_len_x + subs count, count, zva_len_x + b.ge 3b + ands count, count, zva_bits_x + b.ne .Ltail_maybe_long + ret +#endif /* DONT_USE_DC */ ENDPROC(memset)
Signature · 1 line
1.7.9.5
PATCH 3/6 arm64: lib: Implement optimized memset routine
Quoted 27 lines · zhichang.yuan at linaro.org
From: "zhichang.yuan" <[email protected]>
This patch, based on Linaro's Cortex Strings library, improves the performance of the assembly optimized
memset()function.Signed-off-by Zhichang Yuan <[email protected]>Signed-off-by Deepak Saxena <[email protected]>--- arch/arm64/lib/memset.S | 227 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 201 insertions(+), 26 deletions(-)arch/arm64/lib/memset.S +8
-0@@ -1,13 +1,21 @@ /* * Copyright (C) 2013 ARM Ltd. + * Copyright (C) 2013 Linaro. + * + * This code is based on glibc cortex strings work originally authored by Linaro + * and re-licensed under GPLv2 for the Linux kernel. The original code can + * be found @ + * + * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/ + * files/head:/src/aarch64/ *
arch/arm64/lib/memset.S line 18
Quoted 10 lines
arch/arm64/lib/memset.S +3
-3* This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * This program is distributed "as is" WITHOUT ANY WARRANTY of any + * kind, whether express or implied; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
Why are you changing this?
Quoted 13 lines
arch/arm64/lib/memset.S +1
-1* GNU General Public License for more details. * * You should have received a copy of the GNU General Public License@@ -18,7 +26,7 @@ #include <asm/assembler.h> /* - * Fill in the buffer with character c (alignment handled by the hardware) + * Fill in the buffer with character c * * Parameters: * x0 - buf
arch/arm64/lib/memset.S line 44
Quoted 10 lines
arch/arm64/lib/memset.S +7
-0@@ -27,27 +35,194 @@ * Returns: * x0 - buf */ + +/* By default we assume that the DC instruction can be used to zero +* data blocks more efficiently. In some circumstances this might be +* unsafe, for example in an asymmetric multiprocessor environment with +* different DC clear lengths (neither the upper nor lower lengths are +* safe to use). The feature can be disabled by defining DONT_USE_DC. +*/
We already use DC ZVA for clear_page, so I think we should start off using it unconditionally. If we need to revisit this later, we can, but adding a random #ifdef doesn't feel like something we need initially.
For the benefit of anybody else reviewing this; the DC ZVA instruction still works for normal, non-cacheable memory.
The comments I made on the earlier patch wrt quality of comments and labels seem to apply to all of the patches in this series.
Will
PATCH 3/6 arm64: lib: Implement optimized memset routine
Quoted 56 lines · Will Deacon
Quoted 27 lines · zhichang.yuan at linaro.org
From: "zhichang.yuan" <[email protected]>
This patch, based on Linaro's Cortex Strings library, improves the performance of the assembly optimized
memset()function.Signed-off-by Zhichang Yuan <[email protected]>Signed-off-by Deepak Saxena <[email protected]>--- arch/arm64/lib/memset.S | 227 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 201 insertions(+), 26 deletions(-)arch/arm64/lib/memset.S +8
-0@@ -1,13 +1,21 @@ /* * Copyright (C) 2013 ARM Ltd. + * Copyright (C) 2013 Linaro. + * + * This code is based on glibc cortex strings work originally authored by Linaro + * and re-licensed under GPLv2 for the Linux kernel. The original code can + * be found @ + * + * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/ + * files/head:/src/aarch64/ *arch/arm64/lib/memset.S line 18
Quoted 10 lines
arch/arm64/lib/memset.S +3
-3* This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * This program is distributed "as is" WITHOUT ANY WARRANTY of any + * kind, whether express or implied; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See theWhy are you changing this?
Quoted 15 lines
arch/arm64/lib/memset.S +1
-1* GNU General Public License for more details. * * You should have received a copy of the GNU General Public License@@ -18,7 +26,7 @@ #include <asm/assembler.h> /* - * Fill in the buffer with character c (alignment handled by the hardware) + * Fill in the buffer with character c * * Parameters: * x0 - buf@@ -27,27 +35,194 @@ * Returns: * x0 - buf
Quoted 8 lines
arch/arm64/lib/memset.S line 44
Quoted 8 lines
arch/arm64/lib/memset.S +7
-0*/ + +/* By default we assume that the DC instruction can be used to zero +* data blocks more efficiently. In some circumstances this might be +* unsafe, for example in an asymmetric multiprocessor environment with +* different DC clear lengths (neither the upper nor lower lengths are +* safe to use). The feature can be disabled by defining DONT_USE_DC. +*/
This comments is not so correct, for the AMP,i think the DONT_USE_DC also is not necessary. Since this memset will read the dczid_el0 each time,it will get the current correct value from the system register.
Quoted 4 lines
We already use DC ZVA for
clear_page, so I think we should start off using it unconditionally. If we need to revisit this later, we can, but adding a random #ifdef doesn't feel like something we need initially.
As for the macro DONT_USE_DC,i am no sure whether default configure of some systems do not permit the DC ZVA. In this case,the try to use DC ZVA will be ended and turn back to the normal memory setting process. It will not cause any error,only need more instructions executed.
I initially think using the DONT_USE_DC is a bit efficient. Actually,even if the system doesn't support DC ZVA,the cost of reading the dczid_el0 register is small. It is not worthy to introduce a kernel macro.
I will modify it.
Zhichang
Quoted 9 lines
For the benefit of anybody else reviewing this; the DC ZVA instruction still works for normal, non-cacheable memory.
The comments I made on the earlier patch wrt quality of comments and labels seem to apply to all of the patches in this series.
Will
PATCH 4/6 arm64: lib: Implement optimized memcmp routine
In reply to zhichang.yuan at linaro.orgFrom: "zhichang.yuan" <[email protected]>
This patch, based on Linaro's Cortex Strings library, adds an assembly optimized memcmp() function.
--- arch/arm64/include/asm/string.h | 3 + arch/arm64/kernel/arm64ksyms.c | 1 + arch/arm64/lib/Makefile | 2 +- arch/arm64/lib/memcmp.S | 258 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 263 insertions(+), 1 deletion(-) create mode 100644 arch/arm64/lib/memcmp.S
arch/arm64/include/asm/string.h +3 -0
@@ -34,4 +34,7 @@ extern void *memchr(const void *, int, __kernel_size_t);#define __HAVE_ARCH_MEMSETextern void *memset (void *,int ,__kernel_size_t ); +#define __HAVE_ARCH_MEMCMP +extern int memcmp (const void *,const void *,size_t ); +#endif
arch/arm64/kernel/arm64ksyms.c +1 -0
@@ -51,6 +51,7 @@ EXPORT_SYMBOL(memset);EXPORT_SYMBOL (memcpy);EXPORT_SYMBOL (memmove);EXPORT_SYMBOL (memchr); +EXPORT_SYMBOL (memcmp);/* atomic bitops */ EXPORT_SYMBOL (set_bit);
arch/arm64/lib/Makefile +1 -1
@@ -2,5 +2,5 @@ lib-y := bitops.o delay.o \ strncpy_from_user.o strnlen_user.o clear_user.o \ copy_from_user.o copy_to_user.o copy_in_user.o \ copy_page.o clear_page.o \ - memchr.o memcpy.o memmove.o memset.o \ + memchr.o memcpy.o memmove.o memset.o memcmp.o \ strchr.o strrchr.o
arch/arm64/lib/memcmp.S new file +258 -0
258 lines not on this page · raw message
Signature · 1 line
1.7.9.5
PATCH 4/6 arm64: lib: Implement optimized memcmp routine
Quoted 4 lines · zhichang.yuan at linaro.org
From: "zhichang.yuan" <[email protected]>
This patch, based on Linaro's Cortex Strings library, adds an assembly optimized
memcmp()function.
[...]
arch/arm64/lib/memcmp.S line 221
Quoted 5 lines
arch/arm64/lib/memcmp.S new file +5
-0+#ifndef __ARM64EB__ + rev diff, diff + rev data1, data1 + rev data2, data2 +#endif
Given that I can't see you defining __ARM64EB__, I'd take a guess at you never having tested this code on a big-endian machine.
Please can you fix that?
Will
PATCH 4/6 arm64: lib: Implement optimized memcmp routine
Quoted 8 lines · Will Deacon
Quoted 4 lines · zhichang.yuan at linaro.org
From: "zhichang.yuan" <[email protected]>
This patch, based on Linaro's Cortex Strings library, adds an assembly optimized
memcmp()function.[...]
Quoted 8 lines
arch/arm64/lib/memcmp.S line 221
Quoted 5 lines
arch/arm64/lib/memcmp.S new file +5
-0+#ifndef __ARM64EB__ + rev diff, diff + rev data1, data1 + rev data2, data2 +#endifGiven that I can't see you defining
__ARM64EB__, I'd take a guess at you never having tested this code on a big-endian machine.
We'll test it, but I believe __ARM64BE__ is a compiler constant that is automagically present when building big endian, similar to __ARMBE__
Quoted 4 lines
Please can you fix that?
Will
Signature · 3 lines
Deepak Saxena - Kernel Working Group Lead Linaro.org | Open source software for ARM SoCs Follow Linaro: http://www.facebook.com/pages/Linaro
Catalin Marinas
maintainerPATCH 4/6 arm64: lib: Implement optimized memcmp routine
Quoted 18 lines · Deepak Saxena
Quoted 8 lines · Will Deacon
Quoted 4 lines · zhichang.yuan at linaro.org
From: "zhichang.yuan" <[email protected]>
This patch, based on Linaro's Cortex Strings library, adds an assembly optimized
memcmp()function.[...]
Quoted 8 lines
arch/arm64/lib/memcmp.S line 221
Quoted 5 lines
arch/arm64/lib/memcmp.S new file +5
-0+#ifndef __ARM64EB__ + rev diff, diff + rev data1, data1 + rev data2, data2 +#endifGiven that I can't see you defining
__ARM64EB__, I'd take a guess at you never having tested this code on a big-endian machine.
Quoted 3 lines
We'll test it, but I believe
__ARM64BE__is a compiler constant that is automagically present when building big endian, similar to__ARMBE__
It's __AARCH64EB__ (that's how we spot people not testing on big-endian ;)).
Signature · 1 line
Catalin
PATCH 5/6 arm64: lib: Implement optimized string compare routines
In reply to zhichang.yuan at linaro.orgFrom: "zhichang.yuan" <[email protected]>
This patch, based on Linaro's Cortex Strings library, adds an assembly optimized strcmp() and strncmp() functions.
--- arch/arm64/include/asm/string.h | 6 + arch/arm64/kernel/arm64ksyms.c | 2 + arch/arm64/lib/Makefile | 2 +- arch/arm64/lib/strcmp.S | 256 +++++++++++++++++++++++++++++ arch/arm64/lib/strncmp.S | 340 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 605 insertions(+), 1 deletion(-) create mode 100644 arch/arm64/lib/strcmp.S create mode 100644 arch/arm64/lib/strncmp.S
arch/arm64/include/asm/string.h +6 -0
@@ -22,6 +22,12 @@ extern char *strrchr(const char *, int c);#define __HAVE_ARCH_STRCHRextern char *strchr (const char *,int c); +#define __HAVE_ARCH_STRCMP +extern int strcmp (const char *,const char *); + +#define __HAVE_ARCH_STRNCMP +extern int strncmp (const char *,const char *,__kernel_size_t ); +#define __HAVE_ARCH_MEMCPYextern void *memcpy (void *,const void *,__kernel_size_t );
arch/arm64/kernel/arm64ksyms.c +2 -0
@@ -47,6 +47,8 @@ EXPORT_SYMBOL(memstart_addr);/* string / mem functions */ EXPORT_SYMBOL (strchr);EXPORT_SYMBOL (strrchr); +EXPORT_SYMBOL (strcmp); +EXPORT_SYMBOL (strncmp);EXPORT_SYMBOL (memset);EXPORT_SYMBOL (memcpy);EXPORT_SYMBOL (memmove);
arch/arm64/lib/Makefile +1 -1
@@ -3,4 +3,4 @@ lib-y := bitops.o delay.o \ copy_from_user.o copy_to_user.o copy_in_user.o \ copy_page.o clear_page.o \ memchr.o memcpy.o memmove.o memset.o memcmp.o \ - strchr.o strrchr.o + strchr.o strrchr.o strcmp.o strncmp.o
arch/arm64/lib/strcmp.S new file +256 -0
256 lines not on this page · raw message
arch/arm64/lib/strncmp.S new file +340 -0
340 lines not on this page · raw message
Signature · 1 line
1.7.9.5
PATCH 6/6 arm64: lib: Implement optimized string length routines
In reply to zhichang.yuan at linaro.orgFrom: "zhichang.yuan" <[email protected]>
This patch, based on Linaro's Cortex Strings library, adds an assembly optimized strlen() and strnlen() functions.
--- arch/arm64/include/asm/string.h | 6 ++ arch/arm64/kernel/arm64ksyms.c | 2 + arch/arm64/lib/Makefile | 3 +- arch/arm64/lib/strlen.S | 131 ++++++++++++++++++++++++++++ arch/arm64/lib/strnlen.S | 179 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 320 insertions(+), 1 deletion(-) create mode 100644 arch/arm64/lib/strlen.S create mode 100644 arch/arm64/lib/strnlen.S
arch/arm64/include/asm/string.h +6 -0
@@ -28,6 +28,12 @@ extern int strcmp(const char *, const char *);#define __HAVE_ARCH_STRNCMPextern int strncmp (const char *,const char *,__kernel_size_t ); +#define __HAVE_ARCH_STRLEN +extern __kernel_size_t strlen (const char *); + +#define __HAVE_ARCH_STRNLEN +extern __kernel_size_t strnlen (const char *,__kernel_size_t ); +#define __HAVE_ARCH_MEMCPYextern void *memcpy (void *,const void *,__kernel_size_t );
arch/arm64/kernel/arm64ksyms.c +2 -0
@@ -49,6 +49,8 @@ EXPORT_SYMBOL(strchr);EXPORT_SYMBOL (strrchr);EXPORT_SYMBOL (strcmp);EXPORT_SYMBOL (strncmp); +EXPORT_SYMBOL (strlen); +EXPORT_SYMBOL (strnlen);EXPORT_SYMBOL (memset);EXPORT_SYMBOL (memcpy);EXPORT_SYMBOL (memmove);
arch/arm64/lib/Makefile +2 -1
@@ -3,4 +3,5 @@ lib-y := bitops.o delay.o \ copy_from_user.o copy_to_user.o copy_in_user.o \ copy_page.o clear_page.o \ memchr.o memcpy.o memmove.o memset.o memcmp.o \ - strchr.o strrchr.o strcmp.o strncmp.o + strchr.o strrchr.o strcmp.o strncmp.o \ + strlen.o strnlen.o
arch/arm64/lib/strlen.S new file +131 -0
131 lines not on this page · raw message
arch/arm64/lib/strnlen.S new file +179 -0
179 lines not on this page · raw message
Signature · 1 line
1.7.9.5