arm64: use generic strnlen_user and strncpy_from_user functions
PATCH 1/4 arm64: use generic strnlen_user and strncpy_from_user functions
This patch implements the word-at-a-time interface for arm64 using the same algorithm as ARM. We use the fls64 macro, which expands to a clz instruction via a compiler builtin. Big-endian configurations make use of the implementation from asm-generic.
With this implemented, we can replace our byte-at-a-time strnlen_user and strncpy_from_user functions with the optimised generic versions.
--- arch/arm64/Kconfig | 2 ++ arch/arm64/include/asm/uaccess.h | 25 +++------------ arch/arm64/include/asm/word-at-a-time.h | 54 +++++++++++++++++++++++++++++++++ arch/arm64/kernel/arm64ksyms.c | 5 +-- arch/arm64/lib/Makefile | 8 ++--- arch/arm64/lib/strncpy_from_user.S | 50 ------------------------------ arch/arm64/lib/strnlen_user.S | 47 ---------------------------- 7 files changed, 64 insertions(+), 127 deletions(-) create mode 100644 arch/arm64/include/asm/word-at-a-time.h delete mode 100644 arch/arm64/lib/strncpy_from_user.S delete mode 100644 arch/arm64/lib/strnlen_user.S
arch/arm64/Kconfig +2 -0
@@ -17,6 +17,8 @@ config ARM64 select GENERIC_IRQ_SHOW select GENERIC_SCHED_CLOCK select GENERIC_SMP_IDLE_THREAD + select GENERIC_STRNCPY_FROM_USER + select GENERIC_STRNLEN_USER select GENERIC_TIME_VSYSCALL select HARDIRQS_SW_RESEND select HAVE_ARCH_TRACEHOOK
arch/arm64/include/asm/uaccess.h +4 -21
@@ -100,6 +100,7 @@ static inline void set_fs(mm_segment_t fs) })#define access_ok (type, addr, size) __range_ok(addr, size) +#define user_addr_max get_fs/* * The "__xxx" versions of the user access functions do not verify the address
@@ -240,9 +241,6 @@ extern unsigned long __must_check __copy_to_user(void __user *to, const void *frextern unsigned long __must_check __copy_in_user (void __user *to,const void __user *from,unsigned long n);extern unsigned long __must_check __clear_user (void __user *addr,unsigned long n); -extern unsigned long __must_check __strncpy_from_user (char *to,const char __user *from,unsigned long count); -extern unsigned long __must_check __strnlen_user (const char __user *s,long n); -static inline unsigned long __must_check copy_from_user (void *to,const void __user *from,unsigned long n) {if (access_ok (VERIFY_READ , from, n))
@@ -276,24 +274,9 @@ static inline unsigned long __must_check clear_user(void __user *to, unsigned lo return n; } -static inline long __must_check strncpy_from_user (char *dst,const char __user *src,long count) -{ -long res = -EFAULT ; -if (access_ok (VERIFY_READ , src,1 )) - res =__strncpy_from_user (dst, src, count); -return res; -} - -#define strlen_user (s) strnlen_user(s, ~0UL >> 1) +extern long strncpy_from_user (char *dest,const char __user *src,long count); -static inline long __must_check strnlen_user (const char __user *s,long n) -{ -unsigned long res =0 ; - -if (__addr_ok (s)) - res =__strnlen_user (s, n); - -return res; -} +extern __must_check long strlen_user (const char __user *str); +extern __must_check long strnlen_user (const char __user *str,long n);#endif /* __ASM_UACCESS_H */
arch/arm64/include/asm/word-at-a-time.h new file +54 -0
54 lines not on this page · raw message
arch/arm64/kernel/arm64ksyms.c +1 -4
@@ -29,13 +29,10 @@#include <asm/checksum.h> -/* user mem (segment) */ -EXPORT_SYMBOL (__strnlen_user); -EXPORT_SYMBOL (__strncpy_from_user); -EXPORT_SYMBOL (copy_page);EXPORT_SYMBOL (clear_page); +/* user mem (segment) */ EXPORT_SYMBOL (__copy_from_user);EXPORT_SYMBOL (__copy_to_user);EXPORT_SYMBOL (__clear_user);
arch/arm64/lib/Makefile +3 -5
@@ -1,6 +1,4 @@ -lib-y := bitops.o delay.o \ - strncpy_from_user.o strnlen_user.o clear_user.o \ - copy_from_user.o copy_to_user.o copy_in_user.o \ - copy_page.o clear_page.o \ - memchr.o memcpy.o memmove.o memset.o \ +lib-y := bitops.o clear_user.o delay.o copy_from_user.o \ + copy_to_user.o copy_in_user.o copy_page.o \ + clear_page.o memchr.o memcpy.o memmove.o memset.o \ strchr.o strrchr.o
arch/arm64/lib/strncpy_from_user.S deleted +0 -50
50 lines not on this page · raw message
arch/arm64/lib/strnlen_user.S deleted +0 -47
47 lines not on this page · raw message
Signature · 1 line
1.8.2.2
PATCH 2/4 arm64: futex: ensure .fixup entries are sufficiently aligned
AArch64 instructions must be 4-byte aligned, so make sure this is true for the futex .fixup section.
--- arch/arm64/include/asm/futex.h | 1 + 1 file changed, 1 insertion(+)
arch/arm64/include/asm/futex.h +1 -0
@@ -30,6 +30,7 @@" cbnz %w3, 1b\n" \"3:\n" \" .pushsection .fixup,\"ax\"\n" \ +" .align 2\n" \"4: mov %w0, %w5\n" \" b 3b\n" \" .popsection\n" \
Signature · 1 line
1.8.2.2
PATCH 3/4 arm64: dcache: select DCACHE_WORD_ACCESS for little-endian CPUs
In reply to Will DeaconDCACHE_WORD_ACCESS uses the word-at-a-time API for optimised string comparisons in the vfs layer.
This patch implements support for load_unaligned_zeropad in much the same way as has been done for ARM, although big-endian systems are also supported.
--- arch/arm64/Kconfig | 1 + arch/arm64/include/asm/word-at-a-time.h | 40 +++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+)
arch/arm64/Kconfig +1 -0
@@ -11,6 +11,7 @@ config ARM64 select BUILDTIME_EXTABLE_SORT select CLONE_BACKWARDS select COMMON_CLK + select DCACHE_WORD_ACCESS select GENERIC_CLOCKEVENTS select GENERIC_IOMAP select GENERIC_IRQ_PROBE
arch/arm64/include/asm/word-at-a-time.h +40 -0
@@ -47,8 +47,48 @@ static inline unsigned long find_zero(unsigned long mask)return fls64 (mask) >>3 ; } +#define zero_bytemask (mask) (mask) +#else /* __AARCH64EB__ */ #include <asm-generic/word-at-a-time.h> #endif +/* +* Load an unaligned word from kernel space. +* +* In the (very unlikely) case of the word being a page-crosser +* and the next page not being mapped, take the exception and +* return zeroes in the non-existing part. +*/ +static inline unsigned long load_unaligned_zeropad (const void *addr) +{ +unsigned long ret, offset; + +/* Load word from unaligned pointer addr */ + asm( + "1: ldr %0, %3\n" + "2:\n" + " .pushsection .fixup,\"ax\"\n" + " .align 2\n" + "3: and %1, %2, #0x7\n" + " bic %2, %2, #0x7\n" + " ldr %0, [%2]\n" + " lsl %1, %1, #0x3\n" +#ifndef __AARCH64EB__ + " lsr %0, %0, %1\n" +#else +" lsl %0, %0, %1\n" +#endif +" b 2b\n" +" .popsection\n" +" .pushsection __ex_table,\"a\"\n" +" .align 3\n" +" .quad 1b, 3b\n" +" .popsection" + :"=&r" (ret),"=&r" (offset) + :"r" (addr),"Q" (*(unsigned long *)addr)); + +return ret; +} +#endif /* __ASM_WORD_AT_A_TIME_H */
Signature · 1 line
1.8.2.2
PATCH 4/4 arm64: kconfig: select HAVE_EFFICIENT_UNALIGNED_ACCESS
In reply to Will DeaconARMv8 CPUs can perform efficient unaligned memory accesses in hardware and this feature is relied up on by code such as the dcache word-at-a-time name hashing.
This patch selects HAVE_EFFICIENT_UNALIGNED_ACCESS for arm64.
--- arch/arm64/Kconfig | 1 + 1 file changed, 1 insertion(+)
arch/arm64/Kconfig +1 -0
@@ -27,6 +27,7 @@ config ARM64 select HAVE_DEBUG_KMEMLEAK select HAVE_DMA_API_DEBUG select HAVE_DMA_ATTRS + select HAVE_EFFICIENT_UNALIGNED_ACCESS select HAVE_GENERIC_DMA_COHERENT select HAVE_HW_BREAKPOINT if PERF_EVENTS select HAVE_MEMBLOCK
Signature · 1 line
1.8.2.2