Skip to contentSkip to conversation

arm64:lib: the optimized string library routines for armv8 processors

PATCH Superseded zhichang.yuan at linaro.org 4 participants 14 messages Dec 11, 2013 – Dec 19, 2013 arm64

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

cover arm64:lib: the optimized string library routines for armv8 processors

From: "zhichang.yuan" <[email protected]>

In current aarch64 kernel,there are a few string library routines implemented in arm64/lib,such as memcpy,memset, memmove,strchr. Most string routines frequently used are provided by the architecture-independent string library.Those routines are not efficient.

This patch focus on improving the sting routines' performance in ARMv8. It contains eight optimized functions.The work is based on the cortex-string project in Linaro toolchain.

The cortex-string code can be found in this website:
https://code.launchpad.net/cortex-strings
To obtain better performance,several ideas were utilized:
1) memory burst access;
For the long memory data operation,adopted the armv8 instruction pairs, ldp/stp,to transfer the bulk data.Try best to use continuous ldp/stp to trigger the burst access.
2) parallel processing
The current string routines mostly processed per-byte. This patch processes the data in parallel.Such as strlen, it will process eight string bytes each time.
3) aligned memory access
Classfy the process into several categories according to the input memory address parameters.For the non-alignment memory address,firstly process the begginning short-length data to make the memory address aligned,then start the remain processing on alignment address.

After the optimization,those routines have better performance than the current ones. Please refer to this website to get the test results:
https://wiki.linaro.org/WorkingGroups/Kernel/ARMv8/cortex-strings

--

zhichang.yuan (6):
arm64: lib: Implement optimized memcpy routine
arm64: lib: Implement optimized memmove routine
arm64: lib: Implement optimized memset routine
arm64: lib: Implement optimized memcmp routine
arm64: lib: Implement optimized string compare routines
arm64: lib: Implement optimized string length routines

 arch/arm64/include/asm/string.h |   15 ++
 arch/arm64/kernel/arm64ksyms.c  |    5 +
 arch/arm64/lib/Makefile         |    5 +-
 arch/arm64/lib/memcmp.S         |  258 +++++++++++++++++++++++++++++
 arch/arm64/lib/memcpy.S         |  182 ++++++++++++++++++---
 arch/arm64/lib/memmove.S        |  195 ++++++++++++++++++----
 arch/arm64/lib/memset.S         |  227 +++++++++++++++++++++++---
 arch/arm64/lib/strcmp.S         |  256 +++++++++++++++++++++++++++++
 arch/arm64/lib/strlen.S         |  131 +++++++++++++++
 arch/arm64/lib/strncmp.S        |  340 +++++++++++++++++++++++++++++++++++++++
 arch/arm64/lib/strnlen.S        |  179 +++++++++++++++++++++
 11 files changed, 1714 insertions(+), 79 deletions(-)
 create mode 100644 arch/arm64/lib/memcmp.S
 create mode 100644 arch/arm64/lib/strcmp.S
 create mode 100644 arch/arm64/lib/strlen.S
 create mode 100644 arch/arm64/lib/strncmp.S
 create mode 100644 arch/arm64/lib/strnlen.S
Signature · 1 line
1.7.9.5
#1 lore raw

PATCH 1/6 arm64: lib: Implement optimized memcpy routine

From: "zhichang.yuan" <[email protected]>

This patch, based on Linaro's Cortex Strings library, improves the performance of the assembly optimized memcpy() function.

---
 arch/arm64/lib/memcpy.S |  182 +++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 160 insertions(+), 22 deletions(-)
arch/arm64/lib/memcpy.S +160 -22
index 27b5003..e3bab71 100644
@@ -1,5 +1,13 @@
 /*
  * Copyright (C) 2013 ARM Ltd.
+ * Copyright (C) 2013 Linaro.
+ *
+ * This code is based on glibc cortex strings work originally authored by Linaro
+ * and re-licensed under GPLv2 for the Linux kernel. The original code can
+ * be found @
+ *
+ * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/
+ * files/head:/src/aarch64/
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
@@ -27,27 +35,157 @@
  * Returns:
  *	x0 - dest
  */
+#define dstin	x0
+#define src	x1
+#define count	x2
+#define tmp1	x3
+#define tmp1w	w3
+#define tmp2	x4
+#define tmp2w	w4
+#define tmp3	x5
+#define tmp3w	w5
+#define dst	x6
+
+#define A_l	x7
+#define A_h	x8
+#define B_l	x9
+#define B_h	x10
+#define C_l	x11
+#define C_h	x12
+#define D_l	x13
+#define D_h	x14
+
 ENTRY(memcpy)
-	mov	x4, x0
-	subs	x2, x2, #8
-	b.mi	2f
-1:	ldr	x3, [x1], #8
-	subs	x2, x2, #8
-	str	x3, [x4], #8
-	b.pl	1b
-2:	adds	x2, x2, #4
-	b.mi	3f
-	ldr	w3, [x1], #4
-	sub	x2, x2, #4
-	str	w3, [x4], #4
-3:	adds	x2, x2, #2
-	b.mi	4f
-	ldrh	w3, [x1], #2
-	sub	x2, x2, #2
-	strh	w3, [x4], #2
-4:	adds	x2, x2, #1
-	b.mi	5f
-	ldrb	w3, [x1]
-	strb	w3, [x4]
-5:	ret
+	mov	dst, dstin
+	cmp	count, #16
+	/*If memory length is less than 16, stp or ldp can not be used.*/
+	b.lo	.Ltiny15
+.Lover16:
+	neg	tmp2, src
+	ands	tmp2, tmp2, #15/* Bytes to reach alignment. */
+	b.eq	.LSrcAligned
+	sub	count, count, tmp2
+	/*
+	* Use ldp and sdp to copy 16 bytes,then backward the src to
+	* aligned address.This way is more efficient.
+	* But the risk overwriting the source area exists.Here,prefer to
+	* access memory forward straight,no backward.It will need a bit
+	* more instructions, but on the same time,the accesses are aligned.
+	*/
+	tbz	tmp2, #0, 1f
+	ldrb	tmp1w, [src], #1
+	strb	tmp1w, [dst], #1
+1:
+	tbz	tmp2, #1, 1f
+	ldrh	tmp1w, [src], #2
+	strh	tmp1w, [dst], #2
+1:
+	tbz	tmp2, #2, 1f
+	ldr	tmp1w, [src], #4
+	str	tmp1w, [dst], #4
+1:
+	tbz	tmp2, #3, .LSrcAligned
+	ldr	tmp1, [src],#8
+	str	tmp1, [dst],#8
+
+.LSrcAligned:
+	cmp	count, #64
+	b.ge	.Lcpy_over64
+	/*
+	* Deal with small copies quickly by dropping straight into the
+	* exit block.
+	*/
+.Ltail63:
+	/*
+	* Copy up to 48 bytes of data. At this point we only need the
+	* bottom 6 bits of count to be accurate.
+	*/
+	ands	tmp1, count, #0x30
+	b.eq	.Ltiny15
+	cmp	tmp1w, #0x20
+	b.eq	1f
+	b.lt	2f
+	ldp	A_l, A_h, [src], #16
+	stp	A_l, A_h, [dst], #16
+1:
+	ldp	A_l, A_h, [src], #16
+	stp	A_l, A_h, [dst], #16
+2:
+	ldp	A_l, A_h, [src], #16
+	stp	A_l, A_h, [dst], #16
+.Ltiny15:
+	/*
+	* To make memmove simpler, here don't make src backwards.
+	* since backwards will probably overwrite the src area where src
+	* data for nex copy located,if dst is not so far from src.
+	*/
+	tbz	count, #3, 1f
+	ldr	tmp1, [src], #8
+	str	tmp1, [dst], #8
+1:
+	tbz	count, #2, 1f
+	ldr	tmp1w, [src], #4
+	str	tmp1w, [dst], #4
+1:
+	tbz	count, #1, 1f
+	ldrh	tmp1w, [src], #2
+	strh	tmp1w, [dst], #2
+1:
+	tbz	count, #0, .Lexitfunc
+	ldrb	tmp1w, [src]
+	strb	tmp1w, [dst]
+
+.Lexitfunc:
+	ret
+
+.Lcpy_over64:
+	subs	count, count, #128
+	b.ge	.Lcpy_body_large
+	/*
+	* Less than 128 bytes to copy, so handle 64 here and then jump
+	* to the tail.
+	*/
+	ldp	A_l, A_h, [src],#16
+	stp	A_l, A_h, [dst],#16
+	ldp	B_l, B_h, [src],#16
+	ldp	C_l, C_h, [src],#16
+	stp	B_l, B_h, [dst],#16
+	stp	C_l, C_h, [dst],#16
+	ldp	D_l, D_h, [src],#16
+	stp	D_l, D_h, [dst],#16
+
+	tst	count, #0x3f
+	b.ne	.Ltail63
+	ret
+
+	/*
+	* Critical loop.  Start at a new cache line boundary.  Assuming
+	* 64 bytes per line this ensures the entire loop is in one line.
+	*/
+	.p2align	6
+.Lcpy_body_large:
+	/* There are at least 128 bytes to copy.  */
+	ldp	A_l, A_h, [src],#16
+	ldp	B_l, B_h, [src],#16
+	ldp	C_l, C_h, [src],#16
+	ldp	D_l, D_h, [src],#16
+1:
+	stp	A_l, A_h, [dst],#16
+	ldp	A_l, A_h, [src],#16
+	stp	B_l, B_h, [dst],#16
+	ldp	B_l, B_h, [src],#16
+	stp	C_l, C_h, [dst],#16
+	ldp	C_l, C_h, [src],#16
+	stp	D_l, D_h, [dst],#16
+	ldp	D_l, D_h, [src],#16
+	subs	count, count, #64
+	b.ge	1b
+	stp	A_l, A_h, [dst],#16
+	stp	B_l, B_h, [dst],#16
+	stp	C_l, C_h, [dst],#16
+	stp	D_l, D_h, [dst],#16
+
+	tst	count, #0x3f
+	b.ne	.Ltail63
+	ret
 ENDPROC(memcpy)
Signature · 1 line
1.7.9.5
#2 lore raw

PATCH 1/6 arm64: lib: Implement optimized memcpy routine

Quoted 42 lines · zhichang.yuan at linaro.org

From: "zhichang.yuan" <[email protected]>

This patch, based on Linaro's Cortex Strings library, improves the performance of the assembly optimized memcpy() function.

---
 arch/arm64/lib/memcpy.S |  182 +++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 160 insertions(+), 22 deletions(-)
arch/arm64/lib/memcpy.S +17 -0
index 27b5003..e3bab71 100644
@@ -1,5 +1,13 @@
 /*
  * Copyright (C) 2013 ARM Ltd.
+ * Copyright (C) 2013 Linaro.
+ *
+ * This code is based on glibc cortex strings work originally authored by Linaro
+ * and re-licensed under GPLv2 for the Linux kernel. The original code can
+ * be found @
+ *
+ * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/
+ * files/head:/src/aarch64/
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
@@ -27,27 +35,157 @@
  * Returns:
  *	x0 - dest
  */
+#define dstin	x0
+#define src	x1
+#define count	x2
+#define tmp1	x3
+#define tmp1w	w3
+#define tmp2	x4
+#define tmp2w	w4
+#define tmp3	x5
+#define tmp3w	w5

arch/arm64/lib/memcpy.S line 56

Quoted 10 lines
arch/arm64/lib/memcpy.S +10 -0
+#define dst	x6
+
+#define A_l	x7
+#define A_h	x8
+#define B_l	x9
+#define B_h	x10
+#define C_l	x11
+#define C_h	x12
+#define D_l	x13
+#define D_h	x14

Use .req instead of #define?

Quoted 29 lines
arch/arm64/lib/memcpy.S +6 -22
 ENTRY(memcpy)
-	mov	x4, x0
-	subs	x2, x2, #8
-	b.mi	2f
-1:	ldr	x3, [x1], #8
-	subs	x2, x2, #8
-	str	x3, [x4], #8
-	b.pl	1b
-2:	adds	x2, x2, #4
-	b.mi	3f
-	ldr	w3, [x1], #4
-	sub	x2, x2, #4
-	str	w3, [x4], #4
-3:	adds	x2, x2, #2
-	b.mi	4f
-	ldrh	w3, [x1], #2
-	sub	x2, x2, #2
-	strh	w3, [x4], #2
-4:	adds	x2, x2, #1
-	b.mi	5f
-	ldrb	w3, [x1]
-	strb	w3, [x4]
-5:	ret
+	mov	dst, dstin
+	cmp	count, #16
+	/*If memory length is less than 16, stp or ldp can not be used.*/
+	b.lo	.Ltiny15
+.Lover16:
+	neg	tmp2, src

arch/arm64/lib/memcpy.S line 74

Quoted 10 lines
arch/arm64/lib/memcpy.S +10 -0
+	ands	tmp2, tmp2, #15/* Bytes to reach alignment. */
+	b.eq	.LSrcAligned
+	sub	count, count, tmp2
+	/*
+	* Use ldp and sdp to copy 16 bytes,then backward the src to
+	* aligned address.This way is more efficient.
+	* But the risk overwriting the source area exists.Here,prefer to
+	* access memory forward straight,no backward.It will need a bit
+	* more instructions, but on the same time,the accesses are aligned.
+	*/

This comment reads very badly:

- sdp doesn't exist
- `more efficient' than what? How is this measured?
- `access memory forward straight,no backward' What?

Please re-write it in a clearer fashion, so that reviewers have some understanding of your optimisations when potentially trying to change the code later on.

arch/arm64/lib/memcpy.S line 86

Quoted 12 lines
arch/arm64/lib/memcpy.S +12 -0
+	tbz	tmp2, #0, 1f
+	ldrb	tmp1w, [src], #1
+	strb	tmp1w, [dst], #1
+1:
+	tbz	tmp2, #1, 1f
+	ldrh	tmp1w, [src], #2
+	strh	tmp1w, [dst], #2
+1:
+	tbz	tmp2, #2, 1f
+	ldr	tmp1w, [src], #4
+	str	tmp1w, [dst], #4
+1:

Three labels called '1:' ? Can you make them unique please (the old code just incremented a counter).

Quoted 25 lines
arch/arm64/lib/memcpy.S +25 -0
+	tbz	tmp2, #3, .LSrcAligned
+	ldr	tmp1, [src],#8
+	str	tmp1, [dst],#8
+
+.LSrcAligned:
+	cmp	count, #64
+	b.ge	.Lcpy_over64
+	/*
+	* Deal with small copies quickly by dropping straight into the
+	* exit block.
+	*/
+.Ltail63:
+	/*
+	* Copy up to 48 bytes of data. At this point we only need the
+	* bottom 6 bits of count to be accurate.
+	*/
+	ands	tmp1, count, #0x30
+	b.eq	.Ltiny15
+	cmp	tmp1w, #0x20
+	b.eq	1f
+	b.lt	2f
+	ldp	A_l, A_h, [src], #16
+	stp	A_l, A_h, [dst], #16
+1:
+	ldp	A_l, A_h, [src], #16

arch/arm64/lib/memcpy.S line 121

Quoted 10 lines
arch/arm64/lib/memcpy.S +10 -0
+	stp	A_l, A_h, [dst], #16
+2:
+	ldp	A_l, A_h, [src], #16
+	stp	A_l, A_h, [dst], #16
+.Ltiny15:
+	/*
+	* To make memmove simpler, here don't make src backwards.
+	* since backwards will probably overwrite the src area where src
+	* data for nex copy located,if dst is not so far from src.
+	*/

Another awful comment...

arch/arm64/lib/memcpy.S line 133

Quoted 12 lines
arch/arm64/lib/memcpy.S +12 -0
+	tbz	count, #3, 1f
+	ldr	tmp1, [src], #8
+	str	tmp1, [dst], #8
+1:
+	tbz	count, #2, 1f
+	ldr	tmp1w, [src], #4
+	str	tmp1w, [dst], #4
+1:
+	tbz	count, #1, 1f
+	ldrh	tmp1w, [src], #2
+	strh	tmp1w, [dst], #2
+1:

... and more of these labels.

Quoted 22 lines
arch/arm64/lib/memcpy.S +22 -0
+	tbz	count, #0, .Lexitfunc
+	ldrb	tmp1w, [src]
+	strb	tmp1w, [dst]
+
+.Lexitfunc:
+	ret
+
+.Lcpy_over64:
+	subs	count, count, #128
+	b.ge	.Lcpy_body_large
+	/*
+	* Less than 128 bytes to copy, so handle 64 here and then jump
+	* to the tail.
+	*/
+	ldp	A_l, A_h, [src],#16
+	stp	A_l, A_h, [dst],#16
+	ldp	B_l, B_h, [src],#16
+	ldp	C_l, C_h, [src],#16
+	stp	B_l, B_h, [dst],#16
+	stp	C_l, C_h, [dst],#16
+	ldp	D_l, D_h, [src],#16
+	stp	D_l, D_h, [dst],#16

arch/arm64/lib/memcpy.S line 165

Quoted 10 lines
arch/arm64/lib/memcpy.S +10 -0
+
+	tst	count, #0x3f
+	b.ne	.Ltail63
+	ret
+
+	/*
+	* Critical loop.  Start at a new cache line boundary.  Assuming
+	* 64 bytes per line this ensures the entire loop is in one line.
+	*/
+	.p2align	6

Can you parameterise this with L1_CACHE_SHIFT instead?

Will

#3 lore raw

PATCH 1/6 arm64: lib: Implement optimized memcpy routine

Hi Will

Thanks for your reply.
I think your comments are ok, i will modify the patches involved those questions. After those fixes are ready, the patch v2 will be submitted.

Thanks again!
Zhichang

Quoted 212 lines · Will Deacon
Quoted 42 lines · zhichang.yuan at linaro.org

From: "zhichang.yuan" <[email protected]>

This patch, based on Linaro's Cortex Strings library, improves the performance of the assembly optimized memcpy() function.

---
 arch/arm64/lib/memcpy.S |  182 +++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 160 insertions(+), 22 deletions(-)
arch/arm64/lib/memcpy.S +17 -0
index 27b5003..e3bab71 100644
@@ -1,5 +1,13 @@
 /*
  * Copyright (C) 2013 ARM Ltd.
+ * Copyright (C) 2013 Linaro.
+ *
+ * This code is based on glibc cortex strings work originally authored by Linaro
+ * and re-licensed under GPLv2 for the Linux kernel. The original code can
+ * be found @
+ *
+ * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/
+ * files/head:/src/aarch64/
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
@@ -27,27 +35,157 @@
  * Returns:
  *	x0 - dest
  */
+#define dstin	x0
+#define src	x1
+#define count	x2
+#define tmp1	x3
+#define tmp1w	w3
+#define tmp2	x4
+#define tmp2w	w4
+#define tmp3	x5
+#define tmp3w	w5

arch/arm64/lib/memcpy.S line 56

Quoted 10 lines
arch/arm64/lib/memcpy.S +10 -0
+#define dst	x6
+
+#define A_l	x7
+#define A_h	x8
+#define B_l	x9
+#define B_h	x10
+#define C_l	x11
+#define C_h	x12
+#define D_l	x13
+#define D_h	x14

Use .req instead of #define?

Quoted 29 lines
arch/arm64/lib/memcpy.S +6 -22
 ENTRY(memcpy)
-	mov	x4, x0
-	subs	x2, x2, #8
-	b.mi	2f
-1:	ldr	x3, [x1], #8
-	subs	x2, x2, #8
-	str	x3, [x4], #8
-	b.pl	1b
-2:	adds	x2, x2, #4
-	b.mi	3f
-	ldr	w3, [x1], #4
-	sub	x2, x2, #4
-	str	w3, [x4], #4
-3:	adds	x2, x2, #2
-	b.mi	4f
-	ldrh	w3, [x1], #2
-	sub	x2, x2, #2
-	strh	w3, [x4], #2
-4:	adds	x2, x2, #1
-	b.mi	5f
-	ldrb	w3, [x1]
-	strb	w3, [x4]
-5:	ret
+	mov	dst, dstin
+	cmp	count, #16
+	/*If memory length is less than 16, stp or ldp can not be used.*/
+	b.lo	.Ltiny15
+.Lover16:
+	neg	tmp2, src

arch/arm64/lib/memcpy.S line 74

Quoted 10 lines
arch/arm64/lib/memcpy.S +10 -0
+	ands	tmp2, tmp2, #15/* Bytes to reach alignment. */
+	b.eq	.LSrcAligned
+	sub	count, count, tmp2
+	/*
+	* Use ldp and sdp to copy 16 bytes,then backward the src to
+	* aligned address.This way is more efficient.
+	* But the risk overwriting the source area exists.Here,prefer to
+	* access memory forward straight,no backward.It will need a bit
+	* more instructions, but on the same time,the accesses are aligned.
+	*/

This comment reads very badly:

- sdp doesn't exist
- `more efficient' than what? How is this measured?
- `access memory forward straight,no backward' What?

Please re-write it in a clearer fashion, so that reviewers have some understanding of your optimisations when potentially trying to change the code later on.

arch/arm64/lib/memcpy.S line 86

Quoted 12 lines
arch/arm64/lib/memcpy.S +12 -0
+	tbz	tmp2, #0, 1f
+	ldrb	tmp1w, [src], #1
+	strb	tmp1w, [dst], #1
+1:
+	tbz	tmp2, #1, 1f
+	ldrh	tmp1w, [src], #2
+	strh	tmp1w, [dst], #2
+1:
+	tbz	tmp2, #2, 1f
+	ldr	tmp1w, [src], #4
+	str	tmp1w, [dst], #4
+1:

Three labels called '1:' ? Can you make them unique please (the old code just incremented a counter).

Quoted 25 lines
arch/arm64/lib/memcpy.S +25 -0
+	tbz	tmp2, #3, .LSrcAligned
+	ldr	tmp1, [src],#8
+	str	tmp1, [dst],#8
+
+.LSrcAligned:
+	cmp	count, #64
+	b.ge	.Lcpy_over64
+	/*
+	* Deal with small copies quickly by dropping straight into the
+	* exit block.
+	*/
+.Ltail63:
+	/*
+	* Copy up to 48 bytes of data. At this point we only need the
+	* bottom 6 bits of count to be accurate.
+	*/
+	ands	tmp1, count, #0x30
+	b.eq	.Ltiny15
+	cmp	tmp1w, #0x20
+	b.eq	1f
+	b.lt	2f
+	ldp	A_l, A_h, [src], #16
+	stp	A_l, A_h, [dst], #16
+1:
+	ldp	A_l, A_h, [src], #16

arch/arm64/lib/memcpy.S line 121

Quoted 10 lines
arch/arm64/lib/memcpy.S +10 -0
+	stp	A_l, A_h, [dst], #16
+2:
+	ldp	A_l, A_h, [src], #16
+	stp	A_l, A_h, [dst], #16
+.Ltiny15:
+	/*
+	* To make memmove simpler, here don't make src backwards.
+	* since backwards will probably overwrite the src area where src
+	* data for nex copy located,if dst is not so far from src.
+	*/

Another awful comment...

arch/arm64/lib/memcpy.S line 133

Quoted 12 lines
arch/arm64/lib/memcpy.S +12 -0
+	tbz	count, #3, 1f
+	ldr	tmp1, [src], #8
+	str	tmp1, [dst], #8
+1:
+	tbz	count, #2, 1f
+	ldr	tmp1w, [src], #4
+	str	tmp1w, [dst], #4
+1:
+	tbz	count, #1, 1f
+	ldrh	tmp1w, [src], #2
+	strh	tmp1w, [dst], #2
+1:

... and more of these labels.

Quoted 22 lines
arch/arm64/lib/memcpy.S +22 -0
+	tbz	count, #0, .Lexitfunc
+	ldrb	tmp1w, [src]
+	strb	tmp1w, [dst]
+
+.Lexitfunc:
+	ret
+
+.Lcpy_over64:
+	subs	count, count, #128
+	b.ge	.Lcpy_body_large
+	/*
+	* Less than 128 bytes to copy, so handle 64 here and then jump
+	* to the tail.
+	*/
+	ldp	A_l, A_h, [src],#16
+	stp	A_l, A_h, [dst],#16
+	ldp	B_l, B_h, [src],#16
+	ldp	C_l, C_h, [src],#16
+	stp	B_l, B_h, [dst],#16
+	stp	C_l, C_h, [dst],#16
+	ldp	D_l, D_h, [src],#16
+	stp	D_l, D_h, [dst],#16

arch/arm64/lib/memcpy.S line 165

Quoted 10 lines
arch/arm64/lib/memcpy.S +10 -0
+
+	tst	count, #0x3f
+	b.ne	.Ltail63
+	ret
+
+	/*
+	* Critical loop.  Start at a new cache line boundary.  Assuming
+	* 64 bytes per line this ensures the entire loop is in one line.
+	*/
+	.p2align	6

Can you parameterise this with L1_CACHE_SHIFT instead?

Will

#4 lore raw

PATCH 2/6 arm64: lib: Implement optimized memmove routine

In reply to zhichang.yuan at linaro.org

From: "zhichang.yuan" <[email protected]>

This patch, based on Linaro's Cortex Strings library, improves the performance of the assembly optimized memmove() function.

---
 arch/arm64/lib/memmove.S |  195 +++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 166 insertions(+), 29 deletions(-)
arch/arm64/lib/memmove.S +166 -29
index b79fdfa..61ee8d9 100644
@@ -1,13 +1,21 @@
 /*
  * Copyright (C) 2013 ARM Ltd.
+ * Copyright (C) 2013 Linaro.
+ *
+ * This code is based on glibc cortex strings work originally authored by Linaro
+ * and re-licensed under GPLv2 for the Linux kernel. The original code can
+ * be found @
+ *
+ * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/
+ * files/head:/src/aarch64/
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * This program is distributed "as is" WITHOUT ANY WARRANTY of any
+ * kind, whether express or implied; without even the implied warranty
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
@@ -18,7 +26,8 @@
 #include <asm/assembler.h>
 
 /*
- * Move a buffer from src to test (alignment handled by the hardware).
+ * Move a buffer from src to test
+ * (alignment handled by the hardware for part cases).
  * If dest <= src, call memcpy, otherwise copy in reverse order.
  *
  * Parameters:
@@ -28,30 +37,158 @@
  * Returns:
  *	x0 - dest
  */
+#define dstin	x0
+#define src	x1
+#define count	x2
+#define tmp1	x3
+#define tmp1w	w3
+#define tmp2	x4
+#define tmp2w	w4
+#define tmp3	x5
+#define tmp3w	w5
+#define dst	x6
+
+#define A_l	x7
+#define A_h	x8
+#define B_l	x9
+#define B_h	x10
+#define C_l	x11
+#define C_h	x12
+#define D_l	x13
+#define D_h	x14
+
 ENTRY(memmove)
-	cmp	x0, x1
-	b.ls	memcpy
-	add	x4, x0, x2
-	add	x1, x1, x2
-	subs	x2, x2, #8
-	b.mi	2f
-1:	ldr	x3, [x1, #-8]!
-	subs	x2, x2, #8
-	str	x3, [x4, #-8]!
-	b.pl	1b
-2:	adds	x2, x2, #4
-	b.mi	3f
-	ldr	w3, [x1, #-4]!
-	sub	x2, x2, #4
-	str	w3, [x4, #-4]!
-3:	adds	x2, x2, #2
-	b.mi	4f
-	ldrh	w3, [x1, #-2]!
-	sub	x2, x2, #2
-	strh	w3, [x4, #-2]!
-4:	adds	x2, x2, #1
-	b.mi	5f
-	ldrb	w3, [x1, #-1]
-	strb	w3, [x4, #-1]
-5:	ret
+	cmp	dstin, src
+	/*b.eq	.Lexitfunc*/
+	b.lo	memcpy
+	add	tmp1, src, count
+	cmp	dstin, tmp1
+	b.hs	memcpy		/* No overlap.  */
+
+	add	dst, dstin, count
+	add	src, src, count
+	cmp	count, #16
+	b.lo	.Ltail15
+
+.Lover16:
+	ands	tmp2, src, #15     /* Bytes to reach alignment.  */
+	b.eq	.LSrcAligned
+	sub	count, count, tmp2
+	/*
+	* process the aligned offset length to make the src aligned firstly.
+	* those extra instructions' cost is acceptable. It also make the
+	* coming accesses are based on aligned address.
+	*/
+	tbz	tmp2, #0, 1f
+	ldrb	tmp1w, [src, #-1]!
+	strb	tmp1w, [dst, #-1]!
+1:
+	tbz	tmp2, #1, 1f
+	ldrh	tmp1w, [src, #-2]!
+	strh	tmp1w, [dst, #-2]!
+1:
+	tbz	tmp2, #2, 1f
+	ldr	tmp1w, [src, #-4]!
+	str	tmp1w, [dst, #-4]!
+1:
+	tbz	tmp2, #3, .LSrcAligned
+	ldr	tmp1, [src, #-8]!
+	str	tmp1, [dst, #-8]!
+
+.LSrcAligned:
+	cmp	count, #64
+	b.ge	.Lcpy_over64
+
+	/*
+	* Deal with small copies quickly by dropping straight into the
+	* exit block.*/
+.Ltail63:
+	/*
+	* Copy up to 48 bytes of data. At this point we only need the
+	* bottom 6 bits of count to be accurate.
+	*/
+	ands	tmp1, count, #0x30
+	b.eq	.Ltail15
+	cmp	tmp1w, #0x20
+	b.eq	1f
+	b.lt	2f
+	ldp	A_l, A_h, [src, #-16]!
+	stp	A_l, A_h, [dst, #-16]!
+1:
+	ldp	A_l, A_h, [src, #-16]!
+	stp	A_l, A_h, [dst, #-16]!
+2:
+	ldp	A_l, A_h, [src, #-16]!
+	stp	A_l, A_h, [dst, #-16]!
+
+.Ltail15:
+	tbz	count, #3, 1f
+	ldr	tmp1, [src, #-8]!
+	str	tmp1, [dst, #-8]!
+1:
+	tbz	count, #2, 1f
+	ldr	tmp1w, [src, #-4]!
+	str	tmp1w, [dst, #-4]!
+1:
+	tbz	count, #1, 1f
+	ldrh	tmp1w, [src, #-2]!
+	strh	tmp1w, [dst, #-2]!
+1:
+	tbz	count, #0, .Lexitfunc
+	ldrb	tmp1w, [src, #-1]
+	strb	tmp1w, [dst, #-1]
+
+.Lexitfunc:
+	ret
+
+.Lcpy_over64:
+	subs	count, count, #128
+	b.ge	.Lcpy_body_large
+	/*
+	* Less than 128 bytes to copy, so handle 64 here and then jump
+	* to the tail.
+	*/
+	ldp	A_l, A_h, [src, #-16]
+	stp	A_l, A_h, [dst, #-16]
+	ldp	B_l, B_h, [src, #-32]
+	ldp	C_l, C_h, [src, #-48]
+	stp	B_l, B_h, [dst, #-32]
+	stp	C_l, C_h, [dst, #-48]
+	ldp	D_l, D_h, [src, #-64]!
+	stp	D_l, D_h, [dst, #-64]!
+
+	tst	count, #0x3f
+	b.ne	.Ltail63
+	ret
+
+	/*
+	* Critical loop. Start at a new cache line boundary. Assuming
+	* 64 bytes per line this ensures the entire loop is in one line.
+	*/
+	.p2align	6
+.Lcpy_body_large:
+	/* There are@least 128 bytes to copy.  */
+	ldp	A_l, A_h, [src, #-16]
+	ldp	B_l, B_h, [src, #-32]
+	ldp	C_l, C_h, [src, #-48]
+	ldp	D_l, D_h, [src, #-64]!
+1:
+	stp	A_l, A_h, [dst, #-16]
+	ldp	A_l, A_h, [src, #-16]
+	stp	B_l, B_h, [dst, #-32]
+	ldp	B_l, B_h, [src, #-32]
+	stp	C_l, C_h, [dst, #-48]
+	ldp	C_l, C_h, [src, #-48]
+	stp	D_l, D_h, [dst, #-64]!
+	ldp	D_l, D_h, [src, #-64]!
+	subs	count, count, #64
+	b.ge	1b
+	stp	A_l, A_h, [dst, #-16]
+	stp	B_l, B_h, [dst, #-32]
+	stp	C_l, C_h, [dst, #-48]
+	stp	D_l, D_h, [dst, #-64]!
+
+	tst	count, #0x3f
+	b.ne	.Ltail63
+	ret
 ENDPROC(memmove)
Signature · 1 line
1.7.9.5
#5 lore raw

PATCH 3/6 arm64: lib: Implement optimized memset routine

In reply to zhichang.yuan at linaro.org

From: "zhichang.yuan" <[email protected]>

This patch, based on Linaro's Cortex Strings library, improves the performance of the assembly optimized memset() function.

---
 arch/arm64/lib/memset.S |  227 +++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 201 insertions(+), 26 deletions(-)
arch/arm64/lib/memset.S +201 -26
index 87e4a68..90b973e 100644
@@ -1,13 +1,21 @@
 /*
  * Copyright (C) 2013 ARM Ltd.
+ * Copyright (C) 2013 Linaro.
+ *
+ * This code is based on glibc cortex strings work originally authored by Linaro
+ * and re-licensed under GPLv2 for the Linux kernel. The original code can
+ * be found @
+ *
+ * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/
+ * files/head:/src/aarch64/
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * This program is distributed "as is" WITHOUT ANY WARRANTY of any
+ * kind, whether express or implied; without even the implied warranty
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
@@ -18,7 +26,7 @@
 #include <asm/assembler.h>
 
 /*
- * Fill in the buffer with character c (alignment handled by the hardware)
+ * Fill in the buffer with character c
  *
  * Parameters:
  *	x0 - buf
@@ -27,27 +35,194 @@
  * Returns:
  *	x0 - buf
  */
+
+/* By default we assume that the DC instruction can be used to zero
+*  data blocks more efficiently.  In some circumstances this might be
+*  unsafe, for example in an asymmetric multiprocessor environment with
+*  different DC clear lengths (neither the upper nor lower lengths are
+*  safe to use).  The feature can be disabled by defining DONT_USE_DC.
+*/
+
+#define dstin		x0
+#define val		w1
+#define count		x2
+#define tmp1		x3
+#define tmp1w		w3
+#define tmp2		x4
+#define tmp2w		w4
+#define zva_len_x	x5
+#define zva_len	w5
+#define zva_bits_x	x6
+
+#define A_l		x7
+#define A_lw		w7
+#define dst		x8
+#define tmp3w		w9
+#define tmp3		x9
+
 ENTRY(memset)
-	mov	x4, x0
-	and	w1, w1, #0xff
-	orr	w1, w1, w1, lsl #8
-	orr	w1, w1, w1, lsl #16
-	orr	x1, x1, x1, lsl #32
-	subs	x2, x2, #8
-	b.mi	2f
-1:	str	x1, [x4], #8
-	subs	x2, x2, #8
-	b.pl	1b
-2:	adds	x2, x2, #4
-	b.mi	3f
-	sub	x2, x2, #4
-	str	w1, [x4], #4
-3:	adds	x2, x2, #2
-	b.mi	4f
-	sub	x2, x2, #2
-	strh	w1, [x4], #2
-4:	adds	x2, x2, #1
-	b.mi	5f
-	strb	w1, [x4]
-5:	ret
+	mov	dst, dstin	/* Preserve return value.  */
+	and	A_lw, val, #255
+	orr	A_lw, A_lw, A_lw, lsl #8
+	orr	A_lw, A_lw, A_lw, lsl #16
+	orr	A_l, A_l, A_l, lsl #32
+
+	/*first align dst with 16...*/
+	neg	tmp2, dst
+	ands	tmp2, tmp2, #15
+	b.eq	.Laligned
+
+	cmp	count, #15
+	b.le	.Ltail15tiny
+	/*
+	* The count is not less than 16, we can use stp to set 16 bytes
+	* once. This way is more efficient but the access is non-aligned.
+	*/
+	stp	A_l, A_l, [dst]
+	/*make the dst aligned..*/
+	sub	count, count, tmp2
+	add	dst, dst, tmp2
+
+	/*Here, dst is aligned 16 now...*/
+.Laligned:
+#ifndef DONT_USE_DC
+	cbz	A_l, .Lzero_mem
+#endif
+
+.Ltail_maybe_long:
+	cmp	count, #64
+	b.ge	.Lnot_short
+.Ltail63:
+	ands	tmp1, count, #0x30
+	b.eq	.Ltail15tiny
+	cmp	tmp1w, #0x20
+	b.eq	1f
+	b.lt	2f
+	stp	A_l, A_l, [dst], #16
+1:
+	stp	A_l, A_l, [dst], #16
+2:
+	stp	A_l, A_l, [dst], #16
+/*
+* The following process is non-aligned access. But it is more efficient than
+* .Ltail15tiny. Of-course, we can delete this code, but have a bit
+* performance cost.
+*/
+	ands	count, count, #15
+	cbz	count, 1f
+	add	dst, dst, count
+	stp	A_l, A_l, [dst, #-16]	/* Repeat some/all of last store. */
+1:
+	ret
+
+.Ltail15tiny:
+	/* Set up to 15 bytes.  Does not assume earlier memory
+	being set.  */
+	tbz	count, #3, 1f
+	str	A_l, [dst], #8
+1:
+	tbz	count, #2, 1f
+	str	A_lw, [dst], #4
+1:
+	tbz	count, #1, 1f
+	strh	A_lw, [dst], #2
+1:
+	tbz	count, #0, 1f
+	strb	A_lw, [dst]
+1:
+	ret
+
+	/*
+	* Critical loop. Start at a new cache line boundary. Assuming
+	* 64 bytes per line, this ensures the entire loop is in one line.
+	*/
+	.p2align	6
+.Lnot_short: /*count must be not less than 64*/
+	sub	dst, dst, #16/* Pre-bias.  */
+	sub	count, count, #64
+1:
+	stp	A_l, A_l, [dst, #16]
+	stp	A_l, A_l, [dst, #32]
+	stp	A_l, A_l, [dst, #48]
+	stp	A_l, A_l, [dst, #64]!
+	subs	count, count, #64
+	b.ge	1b
+	tst	count, #0x3f
+	add	dst, dst, #16
+	b.ne	.Ltail63
+.Lexitfunc:
+	ret
+
+#ifndef DONT_USE_DC
+	/*
+	* For zeroing memory, check to see if we can use the ZVA feature to
+	* zero entire 'cache' lines.
+	*/
+.Lzero_mem:
+	cmp	count, #63
+	b.le	.Ltail63
+	/*
+	* For zeroing small amounts of memory, it's not worth setting up
+	* the line-clear code.
+	*/
+	cmp	count, #128
+	b.lt	.Lnot_short /*count is at least  128 bytes*/
+
+	mrs	tmp1, dczid_el0
+	tbnz	tmp1, #4, .Lnot_short
+	mov	tmp3w, #4
+	and	zva_len, tmp1w, #15	/* Safety: other bits reserved.  */
+	lsl	zva_len, tmp3w, zva_len
+
+	ands	tmp3w, zva_len, #63
+	/*
+	* ensure the zva_len is not less than 64.
+	* It is not meaningful to use ZVA if the block size is less than 64.
+	*/
+	b.ne	.Lnot_short
+.Lzero_by_line:
+	/*
+	* Compute how far we need to go to become suitably aligned. We're
+	* already at quad-word alignment.
+	*/
+	cmp	count, zva_len_x
+	b.lt	.Lnot_short		/* Not enough to reach alignment.  */
+	sub	zva_bits_x, zva_len_x, #1
+	neg	tmp2, dst
+	ands	tmp2, tmp2, zva_bits_x
+	b.eq	1f			/* Already aligned.  */
+	/* Not aligned, check that there's enough to copy after alignment.*/
+	sub	tmp1, count, tmp2
+	/*
+	* grantee the remain length to be ZVA is bigger than 64,
+	* avoid to make the 2f's process over mem range.*/
+	cmp	tmp1, #64
+	ccmp	tmp1, zva_len_x, #8, ge	/* NZCV=0b1000 */
+	b.lt	.Lnot_short
+	/*
+	* We know that there's at least 64 bytes to zero and that it's safe
+	* to overrun by 64 bytes.
+	*/
+	mov	count, tmp1
+2:
+	stp	A_l, A_l, [dst]
+	stp	A_l, A_l, [dst, #16]
+	stp	A_l, A_l, [dst, #32]
+	subs	tmp2, tmp2, #64
+	stp	A_l, A_l, [dst, #48]
+	add	dst, dst, #64
+	b.ge	2b
+	/* We've overrun a bit, so adjust dst downwards.*/
+	add	dst, dst, tmp2
+1:
+	sub	count, count, zva_len_x
+3:
+	dc	zva, dst
+	add	dst, dst, zva_len_x
+	subs	count, count, zva_len_x
+	b.ge	3b
+	ands	count, count, zva_bits_x
+	b.ne	.Ltail_maybe_long
+	ret
+#endif /* DONT_USE_DC */
 ENDPROC(memset)
Signature · 1 line
1.7.9.5
#6 lore raw

PATCH 3/6 arm64: lib: Implement optimized memset routine

Quoted 27 lines · zhichang.yuan at linaro.org

From: "zhichang.yuan" <[email protected]>

This patch, based on Linaro's Cortex Strings library, improves the performance of the assembly optimized memset() function.

---
 arch/arm64/lib/memset.S |  227 +++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 201 insertions(+), 26 deletions(-)
arch/arm64/lib/memset.S +8 -0
index 87e4a68..90b973e 100644
@@ -1,13 +1,21 @@
 /*
  * Copyright (C) 2013 ARM Ltd.
+ * Copyright (C) 2013 Linaro.
+ *
+ * This code is based on glibc cortex strings work originally authored by Linaro
+ * and re-licensed under GPLv2 for the Linux kernel. The original code can
+ * be found @
+ *
+ * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/
+ * files/head:/src/aarch64/
  *

arch/arm64/lib/memset.S line 18

Quoted 10 lines
arch/arm64/lib/memset.S +3 -3
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * This program is distributed "as is" WITHOUT ANY WARRANTY of any
+ * kind, whether express or implied; without even the implied warranty
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

Why are you changing this?

Quoted 13 lines
arch/arm64/lib/memset.S +1 -1
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
@@ -18,7 +26,7 @@
 #include <asm/assembler.h>
 
 /*
- * Fill in the buffer with character c (alignment handled by the hardware)
+ * Fill in the buffer with character c
  *
  * Parameters:
  *	x0 - buf

arch/arm64/lib/memset.S line 44

Quoted 10 lines
arch/arm64/lib/memset.S +7 -0
@@ -27,27 +35,194 @@
  * Returns:
  *	x0 - buf
  */
+
+/* By default we assume that the DC instruction can be used to zero
+*  data blocks more efficiently.  In some circumstances this might be
+*  unsafe, for example in an asymmetric multiprocessor environment with
+*  different DC clear lengths (neither the upper nor lower lengths are
+*  safe to use).  The feature can be disabled by defining DONT_USE_DC.
+*/

We already use DC ZVA for clear_page, so I think we should start off using it unconditionally. If we need to revisit this later, we can, but adding a random #ifdef doesn't feel like something we need initially.

For the benefit of anybody else reviewing this; the DC ZVA instruction still works for normal, non-cacheable memory.

The comments I made on the earlier patch wrt quality of comments and labels seem to apply to all of the patches in this series.

Will

#7 lore raw

PATCH 3/6 arm64: lib: Implement optimized memset routine

Quoted 56 lines · Will Deacon
Quoted 27 lines · zhichang.yuan at linaro.org

From: "zhichang.yuan" <[email protected]>

This patch, based on Linaro's Cortex Strings library, improves the performance of the assembly optimized memset() function.

---
 arch/arm64/lib/memset.S |  227 +++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 201 insertions(+), 26 deletions(-)
arch/arm64/lib/memset.S +8 -0
index 87e4a68..90b973e 100644
@@ -1,13 +1,21 @@
 /*
  * Copyright (C) 2013 ARM Ltd.
+ * Copyright (C) 2013 Linaro.
+ *
+ * This code is based on glibc cortex strings work originally authored by Linaro
+ * and re-licensed under GPLv2 for the Linux kernel. The original code can
+ * be found @
+ *
+ * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/
+ * files/head:/src/aarch64/
  *

arch/arm64/lib/memset.S line 18

Quoted 10 lines
arch/arm64/lib/memset.S +3 -3
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * This program is distributed "as is" WITHOUT ANY WARRANTY of any
+ * kind, whether express or implied; without even the implied warranty
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

Why are you changing this?

Quoted 15 lines
arch/arm64/lib/memset.S +1 -1
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
@@ -18,7 +26,7 @@
 #include <asm/assembler.h>
 
 /*
- * Fill in the buffer with character c (alignment handled by the hardware)
+ * Fill in the buffer with character c
  *
  * Parameters:
  *	x0 - buf
@@ -27,27 +35,194 @@
  * Returns:
  *	x0 - buf
Quoted 8 lines

arch/arm64/lib/memset.S line 44

Quoted 8 lines
arch/arm64/lib/memset.S +7 -0
  */
+
+/* By default we assume that the DC instruction can be used to zero
+*  data blocks more efficiently.  In some circumstances this might be
+*  unsafe, for example in an asymmetric multiprocessor environment with
+*  different DC clear lengths (neither the upper nor lower lengths are
+*  safe to use).  The feature can be disabled by defining DONT_USE_DC.
+*/

This comments is not so correct, for the AMP,i think the DONT_USE_DC also is not necessary. Since this memset will read the dczid_el0 each time,it will get the current correct value from the system register.

Quoted 4 lines

We already use DC ZVA for clear_page, so I think we should start off using it unconditionally. If we need to revisit this later, we can, but adding a random #ifdef doesn't feel like something we need initially.

As for the macro DONT_USE_DC,i am no sure whether default configure of some systems do not permit the DC ZVA. In this case,the try to use DC ZVA will be ended and turn back to the normal memory setting process. It will not cause any error,only need more instructions executed.
I initially think using the DONT_USE_DC is a bit efficient. Actually,even if the system doesn't support DC ZVA,the cost of reading the dczid_el0 register is small. It is not worthy to introduce a kernel macro.
I will modify it.

Zhichang

Quoted 9 lines

For the benefit of anybody else reviewing this; the DC ZVA instruction still works for normal, non-cacheable memory.

The comments I made on the earlier patch wrt quality of comments and labels seem to apply to all of the patches in this series.

Will

#8 lore raw

PATCH 4/6 arm64: lib: Implement optimized memcmp routine

In reply to zhichang.yuan at linaro.org

From: "zhichang.yuan" <[email protected]>

This patch, based on Linaro's Cortex Strings library, adds an assembly optimized memcmp() function.

---
 arch/arm64/include/asm/string.h |    3 +
 arch/arm64/kernel/arm64ksyms.c  |    1 +
 arch/arm64/lib/Makefile         |    2 +-
 arch/arm64/lib/memcmp.S         |  258 +++++++++++++++++++++++++++++++++++++++
 4 files changed, 263 insertions(+), 1 deletion(-)
 create mode 100644 arch/arm64/lib/memcmp.S
arch/arm64/include/asm/string.h +3 -0
index 3ee8b30..3a43305 100644
@@ -34,4 +34,7 @@ extern void *memchr(const void *, int, __kernel_size_t);
 #define __HAVE_ARCH_MEMSET
 extern void *memset(void *, int, __kernel_size_t);
 
+#define __HAVE_ARCH_MEMCMP
+extern int memcmp(const void *, const void *, size_t);
+
 #endif
arch/arm64/kernel/arm64ksyms.c +1 -0
index e7ee770..af02a25 100644
@@ -51,6 +51,7 @@ EXPORT_SYMBOL(memset);
 EXPORT_SYMBOL(memcpy);
 EXPORT_SYMBOL(memmove);
 EXPORT_SYMBOL(memchr);
+EXPORT_SYMBOL(memcmp);
 
 	/* atomic bitops */
 EXPORT_SYMBOL(set_bit);
arch/arm64/lib/Makefile +1 -1
index 59acc0e..a6a8d3d 100644
@@ -2,5 +2,5 @@ lib-y		:= bitops.o delay.o					\
 		   strncpy_from_user.o strnlen_user.o clear_user.o	\
 		   copy_from_user.o copy_to_user.o copy_in_user.o	\
 		   copy_page.o clear_page.o				\
-		   memchr.o memcpy.o memmove.o memset.o			\
+		   memchr.o memcpy.o memmove.o memset.o memcmp.o	\
 		   strchr.o strrchr.o
arch/arm64/lib/memcmp.S new file +258 -0

258 lines not on this page · raw message

Signature · 1 line
1.7.9.5
#9 lore raw

PATCH 4/6 arm64: lib: Implement optimized memcmp routine

Quoted 4 lines · zhichang.yuan at linaro.org

From: "zhichang.yuan" <[email protected]>

This patch, based on Linaro's Cortex Strings library, adds an assembly optimized memcmp() function.

[...]

arch/arm64/lib/memcmp.S line 221

Quoted 5 lines
arch/arm64/lib/memcmp.S new file +5 -0
+#ifndef	__ARM64EB__
+	rev	diff, diff
+	rev	data1, data1
+	rev	data2, data2
+#endif

Given that I can't see you defining __ARM64EB__, I'd take a guess at you never having tested this code on a big-endian machine.

Please can you fix that?

Will

#10 lore raw

PATCH 4/6 arm64: lib: Implement optimized memcmp routine

Quoted 8 lines · Will Deacon
Quoted 4 lines · zhichang.yuan at linaro.org

From: "zhichang.yuan" <[email protected]>

This patch, based on Linaro's Cortex Strings library, adds an assembly optimized memcmp() function.

[...]

Quoted 8 lines

arch/arm64/lib/memcmp.S line 221

Quoted 5 lines
arch/arm64/lib/memcmp.S new file +5 -0
+#ifndef      __ARM64EB__
+     rev     diff, diff
+     rev     data1, data1
+     rev     data2, data2
+#endif

Given that I can't see you defining __ARM64EB__, I'd take a guess at you never having tested this code on a big-endian machine.

We'll test it, but I believe __ARM64BE__ is a compiler constant that is automagically present when building big endian, similar to __ARMBE__

Quoted 4 lines

Please can you fix that?

Will

Signature · 3 lines
Deepak Saxena - Kernel Working Group Lead
Linaro.org | Open source software for ARM SoCs
Follow Linaro: http://www.facebook.com/pages/Linaro
maintainer
#11 lore raw

PATCH 4/6 arm64: lib: Implement optimized memcmp routine

Quoted 18 lines · Deepak Saxena
Quoted 8 lines · Will Deacon
Quoted 4 lines · zhichang.yuan at linaro.org

From: "zhichang.yuan" <[email protected]>

This patch, based on Linaro's Cortex Strings library, adds an assembly optimized memcmp() function.

[...]

Quoted 8 lines

arch/arm64/lib/memcmp.S line 221

Quoted 5 lines
arch/arm64/lib/memcmp.S new file +5 -0
+#ifndef      __ARM64EB__
+     rev     diff, diff
+     rev     data1, data1
+     rev     data2, data2
+#endif

Given that I can't see you defining __ARM64EB__, I'd take a guess at you never having tested this code on a big-endian machine.

Quoted 3 lines

We'll test it, but I believe __ARM64BE__ is a compiler constant that is automagically present when building big endian, similar to __ARMBE__

It's __AARCH64EB__ (that's how we spot people not testing on big-endian ;)).

Signature · 1 line
Catalin
#12 lore raw

PATCH 5/6 arm64: lib: Implement optimized string compare routines

In reply to zhichang.yuan at linaro.org

From: "zhichang.yuan" <[email protected]>

This patch, based on Linaro's Cortex Strings library, adds an assembly optimized strcmp() and strncmp() functions.

---
 arch/arm64/include/asm/string.h |    6 +
 arch/arm64/kernel/arm64ksyms.c  |    2 +
 arch/arm64/lib/Makefile         |    2 +-
 arch/arm64/lib/strcmp.S         |  256 +++++++++++++++++++++++++++++
 arch/arm64/lib/strncmp.S        |  340 +++++++++++++++++++++++++++++++++++++++
 5 files changed, 605 insertions(+), 1 deletion(-)
 create mode 100644 arch/arm64/lib/strcmp.S
 create mode 100644 arch/arm64/lib/strncmp.S
arch/arm64/include/asm/string.h +6 -0
index 3a43305..6133f49 100644
@@ -22,6 +22,12 @@ extern char *strrchr(const char *, int c);
 #define __HAVE_ARCH_STRCHR
 extern char *strchr(const char *, int c);
 
+#define __HAVE_ARCH_STRCMP
+extern int strcmp(const char *, const char *);
+
+#define __HAVE_ARCH_STRNCMP
+extern int strncmp(const char *, const char *, __kernel_size_t);
+
 #define __HAVE_ARCH_MEMCPY
 extern void *memcpy(void *, const void *, __kernel_size_t);
 
arch/arm64/kernel/arm64ksyms.c +2 -0
index af02a25..d61fa39 100644
@@ -47,6 +47,8 @@ EXPORT_SYMBOL(memstart_addr);
 	/* string / mem functions */
 EXPORT_SYMBOL(strchr);
 EXPORT_SYMBOL(strrchr);
+EXPORT_SYMBOL(strcmp);
+EXPORT_SYMBOL(strncmp);
 EXPORT_SYMBOL(memset);
 EXPORT_SYMBOL(memcpy);
 EXPORT_SYMBOL(memmove);
arch/arm64/lib/Makefile +1 -1
index a6a8d3d..5ded21f 100644
@@ -3,4 +3,4 @@ lib-y		:= bitops.o delay.o					\
 		   copy_from_user.o copy_to_user.o copy_in_user.o	\
 		   copy_page.o clear_page.o				\
 		   memchr.o memcpy.o memmove.o memset.o memcmp.o	\
-		   strchr.o strrchr.o
+		   strchr.o strrchr.o strcmp.o strncmp.o
arch/arm64/lib/strcmp.S new file +256 -0

256 lines not on this page · raw message

arch/arm64/lib/strncmp.S new file +340 -0

340 lines not on this page · raw message

Signature · 1 line
1.7.9.5
#13 lore raw

PATCH 6/6 arm64: lib: Implement optimized string length routines

In reply to zhichang.yuan at linaro.org

From: "zhichang.yuan" <[email protected]>

This patch, based on Linaro's Cortex Strings library, adds an assembly optimized strlen() and strnlen() functions.

---
 arch/arm64/include/asm/string.h |    6 ++
 arch/arm64/kernel/arm64ksyms.c  |    2 +
 arch/arm64/lib/Makefile         |    3 +-
 arch/arm64/lib/strlen.S         |  131 ++++++++++++++++++++++++++++
 arch/arm64/lib/strnlen.S        |  179 +++++++++++++++++++++++++++++++++++++++
 5 files changed, 320 insertions(+), 1 deletion(-)
 create mode 100644 arch/arm64/lib/strlen.S
 create mode 100644 arch/arm64/lib/strnlen.S
arch/arm64/include/asm/string.h +6 -0
index 6133f49..64d2d48 100644
@@ -28,6 +28,12 @@ extern int strcmp(const char *, const char *);
 #define __HAVE_ARCH_STRNCMP
 extern int strncmp(const char *, const char *, __kernel_size_t);
 
+#define __HAVE_ARCH_STRLEN
+extern __kernel_size_t strlen(const char *);
+
+#define __HAVE_ARCH_STRNLEN
+extern __kernel_size_t strnlen(const char *, __kernel_size_t);
+
 #define __HAVE_ARCH_MEMCPY
 extern void *memcpy(void *, const void *, __kernel_size_t);
 
arch/arm64/kernel/arm64ksyms.c +2 -0
index d61fa39..c71b435 100644
@@ -49,6 +49,8 @@ EXPORT_SYMBOL(strchr);
 EXPORT_SYMBOL(strrchr);
 EXPORT_SYMBOL(strcmp);
 EXPORT_SYMBOL(strncmp);
+EXPORT_SYMBOL(strlen);
+EXPORT_SYMBOL(strnlen);
 EXPORT_SYMBOL(memset);
 EXPORT_SYMBOL(memcpy);
 EXPORT_SYMBOL(memmove);
arch/arm64/lib/Makefile +2 -1
index 5ded21f..e64a94c 100644
@@ -3,4 +3,5 @@ lib-y		:= bitops.o delay.o					\
 		   copy_from_user.o copy_to_user.o copy_in_user.o	\
 		   copy_page.o clear_page.o				\
 		   memchr.o memcpy.o memmove.o memset.o memcmp.o	\
-		   strchr.o strrchr.o strcmp.o strncmp.o
+		   strchr.o strrchr.o strcmp.o strncmp.o		\
+		   strlen.o strnlen.o
arch/arm64/lib/strlen.S new file +131 -0

131 lines not on this page · raw message

arch/arm64/lib/strnlen.S new file +179 -0

179 lines not on this page · raw message

Signature · 1 line
1.7.9.5