parisc: Fix access fault handling in pa_memcpy()
Subsystems (2)
Helge Deller
maintainerPATCH 1/1 parisc: Fix access fault handling in pa_memcpy()
pa_memcpy() is the major memcpy implementation in the parisc kernel which is used to do any kind of userspace/kernel memory copies.
Al Viro noticed various bugs in the implementation of pa_mempcy(), most notably that in case of faults it may report back to have copied more bytes than it actually did.
Fixing those bugs is quite hard in the C-implementation, because the compiler is messing around with the registers and we are not guaranteed that specific variables are always in the same processor registers. This makes proper fault handling complicated.
This patch implements pa_memcpy() in assembler. That way we have correct fault handling and adding a 64-bit copy routine was quite easy.
Runtime tested with 32- and 64bit kernels.
arch/parisc/lib/lusercopy.S +281 -0
@@ -5,6 +5,7 @@ * Copyright (C) 2000 Richard Hirst <rhirst with parisc-linux.org> * Copyright (C) 2001 Matthieu Delahaye <delahaym at esiee.fr> * Copyright (C) 2003 Randolph Chung <tausq with parisc-linux.org> + * Copyright (C) 2017 Helge Deller <[email protected]> * * * This program is free software; you can redistribute it and/or modify
@@ -132,4 +133,284 @@ ENDPROC_CFI(lstrnlen_user) .procend + + + /* + * unsigned long pa_memcpy(void *dstp, const void *srcp, + * unsigned long len) + * + * Inputs: + * - sr1 contains space of source region + * - sr2 contains space of destination region + * + * Returns number of bytes not transferred. + * If bytes were not copied, this means either a read- or write fault + * happened. + */ + + dst = arg0 + src = arg1 + len = arg2 + t1 = r19 + t2 = r20 + t3 = r21 + t4 = r22 + srcspc = sr1 + dstspc = sr2 + + t0 = r1 + a1 = t1 + a2 = t2 + a3 = t3 + a0 = t4 + + save_src = ret0 + save_dst = ret1 + save_len = arg3 + +ENTRY_CFI(pa_memcpy) + .proc + .callinfo NO_CALLS + .entry + + /* short copy with less than 16bytes? */ + cmpib,>>=,n 15,len,byte_loop + + /* same alignment? */ + xor src,dst,t0 + extru t0,31,2,t1 + cmpib,<>,n 0,t1,unaligned_copy + +#ifdef CONFIG_64BIT + /* only do 64bit copies if we can get aligned. */ + extru t0,31,3,t1 + cmpib,<>,n 0,t1,align_loop32 + + /* align until we are 64bit-word-aligned */ +align_loop64: + extru dst,31,3,t1 + cmpib,=,n 0,t1,copy_loop_16 +20: ldb,ma 1(srcspc,src),t1 +21: stb,ma t1,1(dstspc,dst) + b align_loop64 + ldo -1(len),len + + ASM_EXCEPTIONTABLE_ENTRY(20b,copy_done) + ASM_EXCEPTIONTABLE_ENTRY(21b,copy_done) + + ldi 31,t0 +copy_loop_16: + cmpb,COND(>>=),n t0,len,word_loop + +10: ldd 0(srcspc,src),t1 +11: ldd 8(srcspc,src),t2 +12: std t1,0(dstspc,dst) +13: std t2,8(dstspc,dst) + ldo 16(src),src + ldo 16(dst),dst + ldo -16(len),len +14: ldd 0(srcspc,src),t3 +15: ldd 8(srcspc,src),t4 +16: std t3,0(dstspc,dst) +17: std t4,8(dstspc,dst) + + ASM_EXCEPTIONTABLE_ENTRY(10b,byte_loop_run) + ASM_EXCEPTIONTABLE_ENTRY(11b,byte_loop_run) + ASM_EXCEPTIONTABLE_ENTRY(12b,byte_loop_run) + ASM_EXCEPTIONTABLE_ENTRY(13b,byte_loop_run) + ASM_EXCEPTIONTABLE_ENTRY(14b,byte_loop_run) + ASM_EXCEPTIONTABLE_ENTRY(15b,byte_loop_run) + ASM_EXCEPTIONTABLE_ENTRY(16b,byte_loop_run) + ASM_EXCEPTIONTABLE_ENTRY(17b,byte_loop_run) + + ldo 16(src),src + ldo 16(dst),dst + b copy_loop_16 + ldo -16(len),len + +word_loop: + cmpib,COND(>>=),n 3,len,byte_loop +20: ldw,ma 4(srcspc,src),t1 +21: stw,ma t1,4(dstspc,dst) + b word_loop + ldo -4(len),len + + ASM_EXCEPTIONTABLE_ENTRY(20b,byte_loop_run) + ASM_EXCEPTIONTABLE_ENTRY(21b,byte_loop_run) + +#endif /* CONFIG_64BIT */ + + /* align until we are 32bit-word-aligned */ +align_loop32: + extru dst,31,2,t1 + cmpib,=,n 0,t1,copy_loop_4 +20: ldb,ma 1(srcspc,src),t1 +21: stb,ma t1,1(dstspc,dst) + b align_loop32 + ldo -1(len),len + + ASM_EXCEPTIONTABLE_ENTRY(20b,copy_done) + ASM_EXCEPTIONTABLE_ENTRY(21b,copy_done) + + +copy_loop_4: + cmpib,COND(>>=),n 15,len,byte_loop + +10: ldw 0(srcspc,src),t1 +11: ldw 4(srcspc,src),t2 +12: ldw 8(srcspc,src),t3 +13: ldw 12(srcspc,src),t4 +14: stw t1,0(dstspc,dst) +15: stw t2,4(dstspc,dst) +16: stw t3,8(dstspc,dst) +17: stw t4,12(dstspc,dst) + + ASM_EXCEPTIONTABLE_ENTRY(10b,byte_loop_run) + ASM_EXCEPTIONTABLE_ENTRY(11b,byte_loop_run) + ASM_EXCEPTIONTABLE_ENTRY(12b,byte_loop_run) + ASM_EXCEPTIONTABLE_ENTRY(13b,byte_loop_run) + ASM_EXCEPTIONTABLE_ENTRY(14b,byte_loop_run) + ASM_EXCEPTIONTABLE_ENTRY(15b,byte_loop_run) + ASM_EXCEPTIONTABLE_ENTRY(16b,byte_loop_run) + ASM_EXCEPTIONTABLE_ENTRY(17b,byte_loop_run) + + ldo 16(src),src + ldo 16(dst),dst + b copy_loop_4 + ldo -16(len),len + +byte_loop: + cmpclr,COND(<>) len,%r0,%r0 + b,n copy_done +byte_loop_run: +20: ldb,ma 1(srcspc,src),t1 +21: stb,ma t1,1(dstspc,dst) + b byte_loop + ldo -1(len),len + + ASM_EXCEPTIONTABLE_ENTRY(20b,copy_done) + ASM_EXCEPTIONTABLE_ENTRY(21b,copy_done) + +copy_done: + bv %r0(%r2) + copy len,ret0 + + + /* src and dst are not aligned the same way. */ + /* need to go the hard way */ +unaligned_copy: + /* align until dst is 32bit-word-aligned */ + extru dst,31,2,t1 + cmpib,COND(=),n 0,t1,copy_dstaligned +20: ldb,ma 1(srcspc,src),t1 +21: stb,ma t1,1(dstspc,dst) + b unaligned_copy + ldo -1(len),len + + ASM_EXCEPTIONTABLE_ENTRY(20b,copy_done) + ASM_EXCEPTIONTABLE_ENTRY(21b,copy_done) + +copy_dstaligned: + + /* store src, dst and len in safe place */ + copy src,save_src + copy dst,save_dst + copy len,save_len + + /* len now needs give number of words to copy */ + SHRREG len,2,len + + /* + * Copy from a not-aligned src to an aligned dst using shifts. + * Handles 4 words per loop. + * This code is derived from glibc. + */ + + depw,z src,28,2,t0 + subi 32,t0,t0 + mtsar t0 + extru len,31,2,t0 + cmpib,= 2,t0,.Lcase2 + /* Make src aligned by rounding it down. */ + depi 0,31,2,src + + cmpiclr,<> 3,t0,%r0 + b,n .Lcase3 + cmpiclr,<> 1,t0,%r0 + b,n .Lcase1 +.Lcase0: + cmpb,= %r0,len,Lcda_finish + nop + +1: ldw,ma 4(srcspc,src), a3 + ASM_EXCEPTIONTABLE_ENTRY(1b,Lcda_finish) +1: ldw,ma 4(srcspc,src), a0 + ASM_EXCEPTIONTABLE_ENTRY(1b,Lcda_finish) + b,n .Ldo3 +.Lcase1: +1: ldw,ma 4(srcspc,src), a2 + ASM_EXCEPTIONTABLE_ENTRY(1b,Lcda_finish) +1: ldw,ma 4(srcspc,src), a3 + ASM_EXCEPTIONTABLE_ENTRY(1b,Lcda_finish) + ldo -1(len),len + cmpb,=,n %r0,len,.Ldo0 +.Ldo4: +1: ldw,ma 4(srcspc,src), a0 + ASM_EXCEPTIONTABLE_ENTRY(1b,Lcda_finish) + shrpw a2, a3, %sar, t0 +1: stw,ma t0, 4(dstspc,dst) + ASM_EXCEPTIONTABLE_ENTRY(1b,Lcda_finish) +.Ldo3: +1: ldw,ma 4(srcspc,src), a1 + ASM_EXCEPTIONTABLE_ENTRY(1b,Lcda_finish) + shrpw a3, a0, %sar, t0 +1: stw,ma t0, 4(dstspc,dst) + ASM_EXCEPTIONTABLE_ENTRY(1b,Lcda_finish) +.Ldo2: +1: ldw,ma 4(srcspc,src), a2 + ASM_EXCEPTIONTABLE_ENTRY(1b,Lcda_finish) + shrpw a0, a1, %sar, t0 +1: stw,ma t0, 4(dstspc,dst) + ASM_EXCEPTIONTABLE_ENTRY(1b,Lcda_finish) +.Ldo1: +1: ldw,ma 4(srcspc,src), a3 + ASM_EXCEPTIONTABLE_ENTRY(1b,Lcda_finish) + shrpw a1, a2, %sar, t0 +1: stw,ma t0, 4(dstspc,dst) + ASM_EXCEPTIONTABLE_ENTRY(1b,Lcda_finish) + ldo -4(len),len + cmpb,<> %r0,len,.Ldo4 + nop +.Ldo0: + shrpw a2, a3, %sar, t0 +1: stw,ma t0, 4(dstspc,dst) + ASM_EXCEPTIONTABLE_ENTRY(1b,Lcda_finish) + +Lcda_finish: + + /* calculate new src, dst and len and jump to byte-copy */ + sub dst,save_dst,t0 + add save_src,t0,src + b byte_loop + sub save_len,t0,len + +.Lcase3: +1: ldw,ma 4(srcspc,src), a0 + ASM_EXCEPTIONTABLE_ENTRY(1b,Lcda_finish) +1: ldw,ma 4(srcspc,src), a1 + ASM_EXCEPTIONTABLE_ENTRY(1b,Lcda_finish) + b .Ldo2 + ldo 1(len),len +.Lcase2: +1: ldw,ma 4(srcspc,src), a1 + ASM_EXCEPTIONTABLE_ENTRY(1b,Lcda_finish) +1: ldw,ma 4(srcspc,src), a2 + ASM_EXCEPTIONTABLE_ENTRY(1b,Lcda_finish) + b .Ldo1 + ldo 2(len),len + + .exit +ENDPROC_CFI(pa_memcpy) + .procend + .end
arch/parisc/lib/memcpy.c +1 -449
471 lines not on this page · raw message
Quoted 7 lines · Helge Deller
pa_memcpy()is the major memcpy implementation in the parisc kernel which is used to do any kind of userspace/kernel memory copies.Al Viro noticed various bugs in the implementation of
pa_mempcy(), most notably that in case of faults it may report back to have copied more bytes than it actually did.
Quoted 7 lines
Fixing those bugs is quite hard in the C-implementation, because the compiler is messing around with the registers and we are not guaranteed that specific variables are always in the same processor registers. This makes proper fault handling complicated.
This patch implements
pa_memcpy()in assembler. That way we have correct fault handling and adding a 64-bit copy routine was quite easy.
Hmm... Is there any point trying to fall back from store fault to byte copy? Note that in copy_dstaligned() dst *is* aligned, so if the word store fails, there's no realistic chance of having a byte store succeed.
Helge Deller
maintainerQuoted 13 lines · Al Viro
Quoted 12 lines · Helge Deller
pa_memcpy()is the major memcpy implementation in the parisc kernel which is used to do any kind of userspace/kernel memory copies.Al Viro noticed various bugs in the implementation of
pa_mempcy(), most notably that in case of faults it may report back to have copied more bytes than it actually did.Fixing those bugs is quite hard in the C-implementation, because the compiler is messing around with the registers and we are not guaranteed that specific variables are always in the same processor registers. This makes proper fault handling complicated.
Quoted 6 lines
Quoted 2 lines
This patch implements
pa_memcpy()in assembler. That way we have correct fault handling and adding a 64-bit copy routine was quite easy.Hmm... Is there any point trying to fall back from store fault to byte copy? Note that in
copy_dstaligned()dst *is* aligned, so if the word store fails, there's no realistic chance of having a byte store succeed.
Good point. I'll fix that.
Thanks,
Helge
Helge Deller
maintainerPATCH v2 1/1 parisc: Fix access fault handling in pa_memcpy()
In reply to Helge Dellerpa_memcpy() is the major memcpy implementation in the parisc kernel which is used to do any kind of userspace/kernel memory copies.
Al Viro noticed various bugs in the implementation of pa_mempcy(), most notably that in case of faults it may report back to have copied more bytes than it actually did.
Fixing those bugs is quite hard in the C-implementation, because the compiler is messing around with the registers and we are not guaranteed that specific variables are always in the same processor registers. This makes proper fault handling complicated.
This patch implements pa_memcpy() in assembler. That way we have correct fault handling and adding a 64-bit copy routine was quite easy.
Runtime tested with 32- and 64bit kernels.
---
Changes to v1:
- do not fall back to byte store from store fault (noticed by Al)
- better wording in comments (suggestions by Dave)
- keep notices from old implementation from Randolph (suggested by Dave)
- use local jump labels to avoid them showing up in vmlinux
- lots of smaller cleanups
arch/parisc/lib/lusercopy.S +319 -0
@@ -5,6 +5,7 @@ * Copyright (C) 2000 Richard Hirst <rhirst with parisc-linux.org> * Copyright (C) 2001 Matthieu Delahaye <delahaym at esiee.fr> * Copyright (C) 2003 Randolph Chung <tausq with parisc-linux.org> + * Copyright (C) 2017 Helge Deller <[email protected]> * * * This program is free software; you can redistribute it and/or modify
@@ -132,4 +133,322 @@ ENDPROC_CFI(lstrnlen_user) .procend + + +/* + * unsigned long pa_memcpy(void *dstp, const void *srcp, unsigned long len) + * + * Inputs: + * - sr1 already contains space of source region + * - sr2 already contains space of destination region + * + * Returns: + * - number of bytes not transferred. If some bytes were not copied, this + * means either a read or write fault happened. + * + * This code is based on a C-implementation of a copy routine written by + * Randolph Chung, which in turn was derived from the glibc. + * + * Several strategies are tried to try to get the best performance for various + * conditions. In the optimal case, we copy by loops that copy 32- or 16-bytes + * at a time using general registers. Unaligned copies are handled either by + * aligning the destination and then using shift-and-write method, or in a few + * cases by falling back to a byte-at-a-time copy. + * + * Testing with various alignments and buffer sizes shows that this code is + * often >10x faster than a simple byte-at-a-time copy, even for strangely + * aligned operands. It is interesting to note that the glibc version of memcpy + * (written in C) is actually quite fast already. This routine is able to beat + * it by 30-40% for aligned copies because of the loop unrolling, but in some + * cases the glibc version is still slightly faster. This lends more + * credibility that gcc can generate very good code as long as we are careful. + * + * Possible optimizations: + * - add cache prefetching + * - try not to use the post-increment address modifiers; they may create + * additional interlocks. Assumption is that those were only efficient on old + * machines (pre PA8000 processors) + */ + + dst = arg0 + src = arg1 + len = arg2 + t1 = r19 + t2 = r20 + t3 = r21 + t4 = r22 + srcspc = sr1 + dstspc = sr2 + + t0 = r1 + a1 = t1 + a2 = t2 + a3 = t3 + a0 = t4 + + save_src = ret0 + save_dst = ret1 + save_len = arg3 + +ENTRY_CFI(pa_memcpy) + .proc + .callinfo NO_CALLS + .entry + + /* short copy with less than 16 bytes? */ + cmpib,>>=,n 15,len,.Lbyte_loop + + /* same alignment? */ + xor src,dst,t0 + extru t0,31,2,t1 + cmpib,<>,n 0,t1,.Lunaligned_copy + +#ifdef CONFIG_64BIT + /* only do 64-bit copies if we can get aligned. */ + extru t0,31,3,t1 + cmpib,<>,n 0,t1,.Lalign_loop32 + + /* loop until we are 64-bit aligned */ +.Lalign_loop64: + extru dst,31,3,t1 + cmpib,=,n 0,t1,.Lcopy_loop_16 +20: ldb,ma 1(srcspc,src),t1 +21: stb,ma t1,1(dstspc,dst) + b .Lalign_loop64 + ldo -1(len),len + + ASM_EXCEPTIONTABLE_ENTRY(20b,.Lcopy_done) + ASM_EXCEPTIONTABLE_ENTRY(21b,.Lcopy_done) + + ldi 31,t0 +.Lcopy_loop_16: + cmpb,COND(>>=),n t0,len,.Lword_loop + +10: ldd,ma 8(srcspc,src),t1 +11: ldd,ma 8(srcspc,src),t2 +12: std,ma t1,8(dstspc,dst) +13: std,ma t2,8(dstspc,dst) + ldo -16(len),len +14: ldd,ma 8(srcspc,src),t1 +15: ldd,ma 8(srcspc,src),t2 +16: std,ma t1,8(dstspc,dst) +17: std,ma t2,8(dstspc,dst) + + ASM_EXCEPTIONTABLE_ENTRY(10b,.Lcopy_done) + ASM_EXCEPTIONTABLE_ENTRY(11b,.Lcopy16_fault0) + ASM_EXCEPTIONTABLE_ENTRY(12b,.Lcopy_done) + ASM_EXCEPTIONTABLE_ENTRY(13b,.Lcopy16_fault8) + ASM_EXCEPTIONTABLE_ENTRY(14b,.Lcopy_done) + ASM_EXCEPTIONTABLE_ENTRY(15b,.Lcopy16_fault0) + ASM_EXCEPTIONTABLE_ENTRY(16b,.Lcopy_done) + ASM_EXCEPTIONTABLE_ENTRY(17b,.Lcopy16_fault8) + + b .Lcopy_loop_16 + ldo -16(len),len + +.Lcopy16_fault0: +10: std t1,0(dstspc,dst) + ASM_EXCEPTIONTABLE_ENTRY(10b,.Lcopy_done) +.Lcopy16_fault8: + b .Lcopy_done + ldo -8(len),len + +.Lword_loop: + cmpib,COND(>>=),n 3,len,.Lbyte_loop +20: ldw,ma 4(srcspc,src),t1 +21: stw,ma t1,4(dstspc,dst) + b .Lword_loop + ldo -4(len),len + + ASM_EXCEPTIONTABLE_ENTRY(20b,.Lcopy_done) + ASM_EXCEPTIONTABLE_ENTRY(21b,.Lcopy_done) + +#endif /* CONFIG_64BIT */ + + /* loop until we are 32bit aligned */ +.Lalign_loop32: + extru dst,31,2,t1 + cmpib,=,n 0,t1,.Lcopy_loop_4 +20: ldb,ma 1(srcspc,src),t1 +21: stb,ma t1,1(dstspc,dst) + b .Lalign_loop32 + ldo -1(len),len + + ASM_EXCEPTIONTABLE_ENTRY(20b,.Lcopy_done) + ASM_EXCEPTIONTABLE_ENTRY(21b,.Lcopy_done) + + +.Lcopy_loop_4: + cmpib,COND(>>=),n 15,len,.Lbyte_loop + +10: ldw 0(srcspc,src),t1 +11: ldw 4(srcspc,src),t2 +12: ldw 8(srcspc,src),t3 +13: ldw 12(srcspc,src),t4 +14: stw,ma t1,4(dstspc,dst) +15: stw,ma t2,4(dstspc,dst) +16: stw,ma t3,4(dstspc,dst) +17: stw,ma t4,4(dstspc,dst) + + ASM_EXCEPTIONTABLE_ENTRY(10b,.Lcopy_done) + ASM_EXCEPTIONTABLE_ENTRY(11b,.Lbyte_loop_run) + ASM_EXCEPTIONTABLE_ENTRY(12b,.Lbyte_loop_run) + ASM_EXCEPTIONTABLE_ENTRY(13b,.Lbyte_loop_run) + ASM_EXCEPTIONTABLE_ENTRY(14b,.Lcopy_done) + ASM_EXCEPTIONTABLE_ENTRY(15b,.Lcopy_fault4) + ASM_EXCEPTIONTABLE_ENTRY(16b,.Lcopy_fault8) + ASM_EXCEPTIONTABLE_ENTRY(17b,.Lcopy_fault12) + + ldo 16(src),src + b .Lcopy_loop_4 + ldo -16(len),len + +.Lcopy_fault12: + ldo -4(len),len +.Lcopy_fault8: + ldo -4(len),len +.Lcopy_fault4: + b .Lcopy_done + ldo -4(len),len + +.Lbyte_loop: + cmpclr,COND(<>) len,%r0,%r0 + b,n .Lcopy_done +.Lbyte_loop_run: +20: ldb,ma 1(srcspc,src),t1 +21: stb,ma t1,1(dstspc,dst) + b .Lbyte_loop + ldo -1(len),len + + ASM_EXCEPTIONTABLE_ENTRY(20b,.Lcopy_done) + ASM_EXCEPTIONTABLE_ENTRY(21b,.Lcopy_done) + +.Lcopy_done: + bv %r0(%r2) + copy len,ret0 + + + /* src and dst are not aligned the same way. */ + /* need to go the hard way */ +.Lunaligned_copy: + /* align until dst is 32bit-word-aligned */ + extru dst,31,2,t1 + cmpib,COND(=),n 0,t1,copy_dstaligned +20: ldb,ma 1(srcspc,src),t1 +21: stb,ma t1,1(dstspc,dst) + b .Lunaligned_copy + ldo -1(len),len + + ASM_EXCEPTIONTABLE_ENTRY(20b,.Lcopy_done) + ASM_EXCEPTIONTABLE_ENTRY(21b,.Lcopy_done) + +copy_dstaligned: + + /* store src, dst and len in safe place */ + copy src,save_src + copy dst,save_dst + copy len,save_len + + /* len now needs give number of words to copy */ + SHRREG len,2,len + + /* + * Copy from a not-aligned src to an aligned dst using shifts. + * Handles 4 words per loop. + */ + + depw,z src,28,2,t0 + subi 32,t0,t0 + mtsar t0 + extru len,31,2,t0 + cmpib,= 2,t0,.Lcase2 + /* Make src aligned by rounding it down. */ + depi 0,31,2,src + + cmpiclr,<> 3,t0,%r0 + b,n .Lcase3 + cmpiclr,<> 1,t0,%r0 + b,n .Lcase1 +.Lcase0: + cmpb,= %r0,len,.Lcda_finish + nop + +1: ldw,ma 4(srcspc,src), a3 + ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault) +1: ldw,ma 4(srcspc,src), a0 + ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault) + b,n .Ldo3 +.Lcase1: +1: ldw,ma 4(srcspc,src), a2 + ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault) +1: ldw,ma 4(srcspc,src), a3 + ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault) + ldo -1(len),len + cmpb,=,n %r0,len,.Ldo0 +.Ldo4: +1: ldw,ma 4(srcspc,src), a0 + ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault) + shrpw a2, a3, %sar, t0 +1: stw,ma t0, 4(dstspc,dst) + ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_wrfault) +.Ldo3: +1: ldw,ma 4(srcspc,src), a1 + ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault) + shrpw a3, a0, %sar, t0 +1: stw,ma t0, 4(dstspc,dst) + ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_wrfault) +.Ldo2: +1: ldw,ma 4(srcspc,src), a2 + ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault) + shrpw a0, a1, %sar, t0 +1: stw,ma t0, 4(dstspc,dst) + ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_wrfault) +.Ldo1: +1: ldw,ma 4(srcspc,src), a3 + ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault) + shrpw a1, a2, %sar, t0 +1: stw,ma t0, 4(dstspc,dst) + ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_wrfault) + ldo -4(len),len + cmpb,<> %r0,len,.Ldo4 + nop +.Ldo0: + shrpw a2, a3, %sar, t0 +1: stw,ma t0, 4(dstspc,dst) + ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_wrfault) + +.Lcda_rdfault: +.Lcda_finish: + /* calculate new src, dst and len and jump to byte-copy loop */ + sub dst,save_dst,t0 + add save_src,t0,src + b .Lbyte_loop + sub save_len,t0,len + + +.Lcase3: +1: ldw,ma 4(srcspc,src), a0 + ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault) +1: ldw,ma 4(srcspc,src), a1 + ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault) + b .Ldo2 + ldo 1(len),len +.Lcase2: +1: ldw,ma 4(srcspc,src), a1 + ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault) +1: ldw,ma 4(srcspc,src), a2 + ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault) + b .Ldo1 + ldo 2(len),len + +.Lcda_wrfault: + /* write fault: calculate written len and exit */ + sub dst,save_dst,t0 + b .Lcopy_done + sub save_len,t0,len + + .exit +ENDPROC_CFI(pa_memcpy) + .procend + .end
arch/parisc/lib/memcpy.c +3 -456
487 lines not on this page · raw message
Helge Deller
maintainerRe: [PATCH v2] parisc: Fix access fault handling in pa_memcpy() - testcase
Just for the mailing list archive:
Attached is the testcase, sent to me by Al Viro, which I used to check the copy_from_user() changes.
Originally written by Andrey Ryabinin <[email protected]>.
Helge
lib/Makefile +2 -0
@@ -242,3 +242,5 @@ UBSAN_SANITIZE_ubsan.o := n obj-$(CONFIG_SBITMAP) += sbitmap.o obj-$(CONFIG_PARMAN) += parman.o + +obj-m += test-cfu.o
lib/test-cfu.c new file +107 -0
107 lines not on this page · raw message