Skip to contentSkip to conversation

Handle unaligned case by avoiding store crossing cache line

RFC PATCH In review [email protected] 3 participants 3 messages Oct 12, 2010 x86

lore
Share & tools
lore
#0 lore raw

RFC PATCH 1/1 Handle unaligned case by avoiding store crossing cache line

From: Ma Ling <[email protected]>

In this patch we mannage to reduce penalty from crossing cache line on some CPU archs. There are two crossing-cache-line cases:
read and write, but write is more expensive because of no cache-way predication and read-for-ownership operations on some archs, here we avoid sotre unaligned cases, another reason is shift register will cause more penalty on decode stages, so tolerate read.

Bellow tests cases are on Atom 64bit based on Xie's patch(value is 1).

Len src/dst improvement speedup(x)

    1, 	 0/ 0:	1
    1, 	 4/ 0:	1
    1, 	 0/ 4:	1
    1, 	 4/ 4:	1
    2, 	 0/ 0:	1
    2, 	 4/ 0:	1
    2, 	 0/ 4:	1
    2, 	 4/ 4:	1
    4, 	 0/ 0:	1
    4, 	 4/ 0:	1
    4, 	 0/ 4:	1
    4, 	 4/ 4:	1
    8, 	 0/ 0:	1
    8, 	 4/ 0:	1
    8, 	 0/ 4:	1
    8, 	 4/ 4:	1

16, 0/ 0: 1
16, 4/ 0: 1
16, 0/ 4: 1
16, 4/ 4: 1
32, 0/ 0: 1
32, 4/ 0: 1
32, 0/ 4: 1
32, 4/ 4: 1
64, 0/ 0: 1
64, 4/ 0: 0.7
64, 0/ 4: 1
64, 4/ 4: 0.8
128, 0/ 0: 1.3
128, 4/ 0: 1.7
128, 0/ 4: 1
128, 4/ 4: 1.1
256, 0/ 0: 1.1
256, 4/ 0: 1.4
256, 0/ 4: 1
256, 4/ 4: 1.1
512, 0/ 0: 1
512, 4/ 0: 1.5
512, 0/ 4: 1
512, 4/ 4: 1
1024, 0/ 0: 1
1024, 4/ 0: 1.5
1024, 0/ 4: 1
1024, 4/ 4: 1
2048, 0/ 0: 1
2048, 4/ 0: 1.6
2048, 0/ 4: 1
2048, 4/ 4: 1
4096, 0/ 0: 1
4096, 4/ 0: 1.6
4096, 0/ 4: 1
4096, 4/ 4: 1

---
 arch/x86/lib/memcpy_64.S |   59 ++++++++++++++++++++++++++++++++++++++++-----
 1 files changed, 52 insertions(+), 7 deletions(-)
arch/x86/lib/memcpy_64.S +52 -7
index 75ef61e..7545b08 100644
@@ -45,7 +45,7 @@ ENTRY(memcpy)
 	/*
 	 * Use 32bit CMP here to avoid long NOP padding.
 	 */
-	cmp  $0x20, %edx
+	cmp  $0x28, %rdx
 	jb .Lhandle_tail
 
 	/*
@@ -54,7 +54,20 @@ ENTRY(memcpy)
 	 */
 	cmp  %dil, %sil
 	jl .Lcopy_backward
-	subl $0x20, %edx
+
+	/*
+	 * We append data to avoid store crossing cache.
+	 */
+	movq (%rsi), %rcx
+	movq %rdi, %r8
+	addq $8, %rdi
+	andq $-8, %rdi
+	movq %rcx, (%r8)
+	subq %rdi, %r8
+	addq %r8, %rdx
+	subq %r8, %rsi
+	
+	subq $0x20, %rdx
 .Lcopy_forward_loop:
 	subq $0x20,	%rdx
 
@@ -74,20 +87,28 @@ ENTRY(memcpy)
 	leaq 4*8(%rdi),	%rdi
 	jae  .Lcopy_forward_loop
 	addq $0x20,	%rdx
-	jmp  .Lhandle_tail
+	jmp   .Lless_32bytes
 
+	.p2align 4
 .Lcopy_backward:
 	/*
 	 * Calculate copy position to tail.
 	 */
 	addq %rdx,	%rsi
 	addq %rdx,	%rdi
-	subq $0x20,	%rdx
+
 	/*
-	 * At most 3 ALU operations in one cycle,
-	 * so append NOPS in the same 16bytes trunk.
+	 * We append data to avoid store crossing cache.
 	 */
-	.p2align 4
+	movq -8(%rsi), %rcx
+	movq %rdi, %r8
+	andq $-8, %rdi
+	movq %rcx, -8(%r8)
+	subq %rdi, %r8
+	subq %r8, %rdx
+	subq %r8, %rsi
+
+	subq $0x20,	%rdx
 .Lcopy_backward_loop:
 	subq $0x20,	%rdx
 	movq -1*8(%rsi),	%r8
@@ -108,7 +129,31 @@ ENTRY(memcpy)
 	addq $0x20,	%rdx
 	subq %rdx,	%rsi
 	subq %rdx,	%rdi
+	jmp   .Lless_32bytes
+
+	.p2align 4
 .Lhandle_tail:
+
+	cmpq $32,	%rdx
+	jb   .Lless_32bytes
+
+	/*
+	 * Move data from 32 bytes to 39 bytes.
+	 */
+	movq 0*8(%rsi), %rcx
+	movq 1*8(%rsi),	%r8
+	movq -3*8(%rsi, %rdx),	%r9
+	movq -2*8(%rsi, %rdx),	%r10
+	movq -1*8(%rsi, %rdx),	%r11
+	movq %rcx,	0*8(%rdi)
+	movq %r8,	1*8(%rdi)
+	movq %r9,	-3*8(%rdi, %rdx)
+	movq %r10,	-2*8(%rdi, %rdx)
+	movq %r11,	-1*8(%rdi, %rdx)
+	retq
+
+	.p2align 4
+.Lless_32bytes:
 	cmpq $16,	%rdx
 	jb   .Lless_16bytes
 
Signature · 1 line
1.6.5.2
#1 lore raw
Quoted 9 lines · On Tue, Oct 12, 2010 at 10:48 PM, <[email protected]> wrote:

From: Ma Ling <[email protected]>

In this patch we mannage to reduce penalty from crossing cache line on some CPU archs. There are two crossing-cache-line cases:
read and write, but write is more expensive because of no cache-way predication and read-for-ownership operations on some archs, here we avoid sotre unaligned cases, another reason is shift register will cause more penalty on decode stages, so tolerate read.

...

Quoted 10 lines

---
 arch/x86/lib/memcpy_64.S |   59 ++++++++++++++++++++++++++++++++++++++++-----

 1 files changed, 52 insertions(+), 7 deletions(-)
arch/x86/lib/memcpy_64.S +0 -0
index 75ef61e..7545b08 100644

arch/x86/lib/memcpy_64.S ENTRY(memcpy) · line 48

Quoted 5 lines
arch/x86/lib/memcpy_64.S +1 -1
@@ -45,7 +45,7 @@ ENTRY(memcpy)
        /*
         * Use 32bit CMP here to avoid long NOP padding.
         */
-       cmp  $0x20, %edx
+       cmp  $0x28, %rdx

Well, look above your change. The comment says "Use 32bit CMP". If you really want to go to 64-bit one, then change comment too.

arch/x86/lib/memcpy_64.S ENTRY(memcpy) · line 68

Quoted 11 lines
arch/x86/lib/memcpy_64.S +11 -0
+       /*
+        * We append data to avoid store crossing cache.
+        */
+       movq (%rsi), %rcx
+       movq %rdi, %r8
+       addq $8, %rdi
+       andq $-8, %rdi
+       movq %rcx, (%r8)
+       subq %rdi, %r8
+       addq %r8, %rdx
+       subq %r8, %rsi

The comment doesn't really help to understand what you are doing here. Maybe "Align store location to 32 bytes to avoid crossing cachelines"?

Quoted 5 lines

       /*
-        * At most 3 ALU operations in one cycle,
-        * so append NOPS in the same 16bytes trunk.
+        * We append data to avoid store crossing cache. */

Same here.

Signature · 1 line
vda