Fix potential issue on memmove
PATCH v3 1/1 Fix potential issue on memmove
From: Ma Ling <[email protected]>
memmove allow source and destination address to be overlap, but no limitation for memcpy. So memmove use forward or backward copy mode to handle src > dest and dest > src cases respectively. However memcpy has not address overlap, it may use any copy mode theoretically. Our original memmove will call memcpy and assume it must use forward copy mode, otherwise the system will crash, it is potential issue. The patch based on tip/x86/mem avoids this assumption.
---
In this version we optimized original 32 and 64bit memmove.
arch/x86/lib/memcpy_32.c | 199 +++++++++++++++++++++++++++++++++++++++++---- arch/x86/lib/memmove_64.c | 189 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 362 insertions(+), 26 deletions(-)
arch/x86/lib/memcpy_32.c +182 -17
@@ -22,22 +22,187 @@ EXPORT_SYMBOL(memset);void *memmove (void *dest,const void *src,size_t n) { -int d0, d1, d2; - -if (dest < src) { -memcpy (dest, src, n); - }else { - __asm__ __volatile__( -"std\n\t" -"rep\n\t" -"movsb\n\t" -"cld" - :"=&c" (d0),"=&S" (d1),"=&D" (d2) - :"0" (n), -"1" (n-1 +src), -"2" (n-1 +dest) - :"memory" ); - } -return dest; + +int d0,d1,d2,d3,d4,d5; +char *ret = dest; + + __asm__ __volatile__( +/* Handle more 16bytes in loop */ +"cmp $0x10, %0\n\t" +"jb 1f\n\t" + +/* Decide forward/backward copy mode */ +"cmp %2, %1\n\t" +"jb 2f\n\t" + +/* +* movs instruction have many startup latency +* so we handle small size by general register. +*/ +"cmp $680, %0\n\t" +"jb 3f\n\t" +/* +* movs instruction is only good for aligned case. +*/ +"mov %1, %3\n\t" +"xor %2, %3\n\t" +"and $0xff, %3\n\t" +"jz 4f\n\t" +"3:\n\t" +"sub $0x10, %0\n\t" + +/* +* We gobble 16byts forward in each loop. +*/ +"3:\n\t" +"sub $0x10, %0\n\t" +"mov 0*4(%1), %3\n\t" +"mov 1*4(%1), %4\n\t" +"mov %3, 0*4(%2)\n\t" +"mov %4, 1*4(%2)\n\t" +"mov 2*4(%1), %3\n\t" +"mov 3*4(%1), %4\n\t" +"mov %3, 2*4(%2)\n\t" +"mov %4, 3*4(%2)\n\t" +"lea 0x10(%1), %1\n\t" +"lea 0x10(%2), %2\n\t" +"jae 3b\n\t" +"add $0x10, %0\n\t" +"jmp 1f\n\t" + +/* +* Handle data forward by movs. +*/ +".p2align 4\n\t" +"4:\n\t" +"mov -4(%1, %0), %3\n\t" +"lea -4(%2, %0), %4\n\t" +"shr $2, %0\n\t" +"rep movsl\n\t" +"mov %3, (%4)\n\t" +"jmp 11f\n\t" +/* +* Handle data backward by movs. +*/ +".p2align 4\n\t" +"6:\n\t" +"mov (%1), %3\n\t" +"mov %2, %4\n\t" +"lea -4(%1, %0), %1\n\t" +"lea -4(%2, %0), %2\n\t" +"shr $2, %0\n\t" +"std\n\t" +"rep movsl\n\t" +"mov %3,(%4)\n\t" +"cld\n\t" +"jmp 11f\n\t" + +/* +* Start to prepare for backward copy. +*/ +".p2align 4\n\t" +"2:\n\t" +"cmp $680, %0\n\t" +"jb 5f\n\t" +"mov %1, %3\n\t" +"xor %2, %3\n\t" +"and $0xff, %3\n\t" +"jz 6b\n\t" + +/* +* Calculate copy position to tail. +*/ +"5:\n\t" +"add %0, %1\n\t" +"add %0, %2\n\t" +"sub $0x10, %0\n\t" + +/* +* We gobble 16byts backward in each loop. +*/ +"7:\n\t" +"sub $0x10, %0\n\t" + +"mov -1*4(%1), %3\n\t" +"mov -2*4(%1), %4\n\t" +"mov %3, -1*4(%2)\n\t" +"mov %4, -2*4(%2)\n\t" +"mov -3*4(%1), %3\n\t" +"mov -4*4(%1), %4\n\t" +"mov %3, -3*4(%2)\n\t" +"mov %4, -4*4(%2)\n\t" +"lea -0x10(%1), %1\n\t" +"lea -0x10(%2), %2\n\t" +"jae 7b\n\t" +/* +* Calculate copy position to head. +*/ +"add $0x10, %0\n\t" +"sub %0, %1\n\t" +"sub %0, %2\n\t" + +/* +* Move data from 8 bytes to 15 bytes. +*/ +".p2align 4\n\t" +"1:\n\t" +"cmp $8, %0\n\t" +"jb 8f\n\t" +"mov 0*4(%1), %3\n\t" +"mov 1*4(%1), %4\n\t" +"mov -2*4(%1, %0), %5\n\t" +"mov -1*4(%1, %0), %1\n\t" + +"mov %3, 0*4(%2)\n\t" +"mov %4, 1*4(%2)\n\t" +"mov %5, -2*4(%2, %0)\n\t" +"mov %1, -1*4(%2, %0)\n\t" +"jmp 11f\n\t" + +/* +* Move data from 4 bytes to 7 bytes. +*/ +".p2align 4\n\t" +"8:\n\t" +"cmp $4, %0\n\t" +"jb 9f\n\t" +"mov 0*4(%1), %3\n\t" +"mov -1*4(%1, %0), %4\n\t" +"mov %3, 0*4(%2)\n\t" +"mov %4, -1*4(%2, %0)\n\t" +"jmp 11f\n\t" + +/* +* Move data from 2 bytes to 3 bytes. +*/ +".p2align 4\n\t" +"9:\n\t" +"cmp $2, %0\n\t" +"jb 10f\n\t" +"movw 0*2(%1), %%dx\n\t" +"movw -1*2(%1, %0), %%bx\n\t" +"movw %%dx, 0*2(%2)\n\t" +"movw %%bx, -1*2(%2, %0)\n\t" +"jmp 11f\n\t" + +/* +* Move data for 1 byte. +*/ +".p2align 4\n\t" +"10:\n\t" +"cmp $1, %0\n\t" +"jb 11f\n\t" +"movb (%1), %%cl\n\t" +"movb %%cl, (%2)\n\t" +".p2align 4\n\t" +"11:" + :"=&c" (d0),"=&S" (d1),"=&D" (d2), +"=r" (d3),"=r" (d4),"=r" (d5) + :"0" (n), +"1" (src), +"2" (dest) + :"memory" ); + + }EXPORT_SYMBOL (memmove);
arch/x86/lib/memmove_64.c +180 -9
194 lines not on this page · raw message
Signature · 1 line
1.6.5.2
H. Peter Anvin
reviewerQuoted 11 lines · On 09/14/2010 06:50 PM, [email protected] wrote:
From: Ma Ling <[email protected]>
memmove allow source and destination address to be overlap, but no limitation for memcpy. So memmove use forward or backward copy mode to handle src > dest and dest > src cases respectively. However memcpy has not address overlap, it may use any copy mode theoretically. Our original memmove will call memcpy and assume it must use forward copy mode, otherwise the system will crash, it is potential issue. The patch based on tip/x86/mem avoids this assumption.
Quoted 1 line
Signed-off-by Ma Ling <[email protected]>
This patch is certainly not based on the current tip:x86/mem. Could you check out the current tip:x86/mem (which has a previous patch of yours in it) and submit an incremental patch with a description about what updates the previous patch and why?
-hpa