x86_64/lib: improve the performance of memmove
Subsystems (2)
PATCH 1/1 x86_64/lib: improve the performance of memmove
When the dest and the src do overlap and the memory area is large, memmove of x86_64 is very inefficient, and it led to bad performance, such as btrfs's file deletion performance. This patch improved the performance of memmove on x86_64 by using __memcpy_bwd() instead of byte copy when doing large memory area copy (len > 64).
I have tested this patchset by doing 500 bytes memory copy for 50000 times with various alignments and buffer sizes on my x86_64 box:
Len Src Unalign Dest Unalign Without Patch Patch applied
--- ----------- ------------ ------------- -------------
256 0 0 0s 815158us 0s 249647us
256 0 4 0s 816059us 0s 324210us
256 0 7 0s 815192us 0s 324254us
256 3 0 0s 815179us 0s 325991us
256 3 1 0s 815161us 0s 378462us
256 3 4 0s 815154us 0s 779306us
256 3 7 0s 815151us 0s 782924us
256 7 0 0s 815839us 0s 325524us
256 7 4 0s 815149us 0s 375658us
256 7 7 0s 815160us 0s 374488us
1024 0 0 3s 125891us 0s 437662us
1024 0 1 3s 125940us 0s 777524us
1024 0 4 3s 159788us 0s 778850us
1024 0 7 3s 155177us 0s 733927us
1024 4 0 3s 118323us 0s 830167us
1024 4 4 3s 129124us 0s 962505us
1024 4 7 3s 123456us 2s 600326us
After appling this patchset, the performance of the file creation and deletion on some filesystem become better. I have tested it with the following benchmark tool on my x86_64 box.
http://marc.info/?l=linux-btrfs&m=128212635122920&q=p3
Test steps:
# ./creat_unlink 50000
The result(Total time):
Ext4:
2.6.36-rc4 2.6.36-rc4 + patch
file creation 0.737007 0.701888 4.8%UP
file deletion 0.422226 0.413457 2.1%UP
Btrfs:
2.6.36-rc4 2.6.36-rc4 + patch
file creation 0.977638 0.935208 4.3%UP
file deletion 1.327140 1.221073 8%UP
--- arch/x86/include/asm/string_64.h | 1 + arch/x86/lib/Makefile | 2 +- arch/x86/lib/memcpy_bwd_64.S | 137 ++++++++++++++++++++++++++++++++++++++ arch/x86/lib/memmove_64.c | 10 ++- 4 files changed, 145 insertions(+), 5 deletions(-) create mode 100644 arch/x86/lib/memcpy_bwd_64.S
arch/x86/include/asm/string_64.h +1 -0
@@ -55,6 +55,7 @@ extern void *__memcpy(void *to, const void *from, size_t len);void *memset (void *s,int c,size_t n);#define __HAVE_ARCH_MEMMOVE +extern void *__memcpy_bwd (void *dest,const void *src,size_t count);void *memmove (void *dest,const void *src,size_t count);int memcmp (const void *cs,const void *ct,size_t count);
arch/x86/lib/Makefile +1 -1
@@ -19,7 +19,7 @@ obj-$(CONFIG_SMP) += msr-smp.o cache-smp.o lib-y := delay.o lib-y += thunk_$(BITS).o lib-y += usercopy_$(BITS).o getuser.o putuser.o -lib-y += memcpy_$(BITS).o +lib-y += memcpy_$(BITS).o memcpy_bwd_$(BITS).o lib-$(CONFIG_INSTRUCTION_DECODER) += insn.o inat.o obj-y += msr.o msr-reg.o msr-reg-export.o
arch/x86/lib/memcpy_bwd_64.S new file +137 -0
137 lines not on this page · raw message
arch/x86/lib/memmove_64.c +6 -4
@@ -8,14 +8,16 @@#undef memmovevoid *memmove (void *dest,const void *src,size_t count) { -if (dest < src) { +if (dest < src || dest - src >= count)return memcpy (dest, src, count); - }else { +else if (count <=64 ) {char *p = dest + count;const char *s = src + count;while (count--) *--p = *--s; - } -return dest; + +return dest; + }else +return __memcpy_bwd (dest, src, count); }EXPORT_SYMBOL (memmove);
Signature · 1 line
1.7.0.1