riscv: optimized mem* functions
Subsystems (2)
v2 cover riscv: optimized mem* functions
From: Matteo Croce <[email protected]>
Replace the assembly mem{cpy,move,set} with C equivalent.
Try to access RAM with the largest bit width possible, but without doing unaligned accesses.
Tested on a BeagleV Starlight with a SiFive U74 core, where the improvement is noticeable.
v1 -> v2:
- reduce the threshold from 64 to 16 bytes
- fix KASAN build
- optimize memset
Matteo Croce (3):
riscv: optimized memcpy
riscv: optimized memmove
riscv: optimized memset
arch/riscv/include/asm/string.h | 18 ++-- arch/riscv/kernel/Makefile | 1 - arch/riscv/kernel/riscv_ksyms.c | 17 ---- arch/riscv/lib/Makefile | 4 +- arch/riscv/lib/memcpy.S | 108 --------------------- arch/riscv/lib/memmove.S | 64 ------------- arch/riscv/lib/memset.S | 113 ---------------------- arch/riscv/lib/string.c | 162 ++++++++++++++++++++++++++++++++ 8 files changed, 172 insertions(+), 315 deletions(-) delete mode 100644 arch/riscv/kernel/riscv_ksyms.c delete mode 100644 arch/riscv/lib/memcpy.S delete mode 100644 arch/riscv/lib/memmove.S delete mode 100644 arch/riscv/lib/memset.S create mode 100644 arch/riscv/lib/string.c
Signature · 1 line
2.31.1
PATCH v2 1/3 riscv: optimized memcpy
From: Matteo Croce <[email protected]>
Write a C version of memcpy() which uses the biggest data size allowed, without generating unaligned accesses.
The procedure is made of three steps:
First copy data one byte at time until the destination buffer is aligned to a long boundary.
Then copy the data one long at time shifting the current and the next u8 to compose a long at every cycle.
Finally, copy the remainder one byte at time.
On a BeagleV, the TCP RX throughput increased by 45%:
before:
$ iperf3 -c beaglev Connecting to host beaglev, port 5201 [ 5] local 192.168.85.6 port 44840 connected to 192.168.85.48 port 5201 [ ID] Interval Transfer Bitrate Retr Cwnd [ 5] 0.00-1.00 sec 76.4 MBytes 641 Mbits/sec 27 624 KBytes [ 5] 1.00-2.00 sec 72.5 MBytes 608 Mbits/sec 0 708 KBytes [ 5] 2.00-3.00 sec 73.8 MBytes 619 Mbits/sec 10 451 KBytes [ 5] 3.00-4.00 sec 72.5 MBytes 608 Mbits/sec 0 564 KBytes [ 5] 4.00-5.00 sec 73.8 MBytes 619 Mbits/sec 0 658 KBytes [ 5] 5.00-6.00 sec 73.8 MBytes 619 Mbits/sec 14 522 KBytes [ 5] 6.00-7.00 sec 73.8 MBytes 619 Mbits/sec 0 621 KBytes [ 5] 7.00-8.00 sec 72.5 MBytes 608 Mbits/sec 0 706 KBytes [ 5] 8.00-9.00 sec 73.8 MBytes 619 Mbits/sec 20 580 KBytes [ 5] 9.00-10.00 sec 73.8 MBytes 619 Mbits/sec 0 672 KBytes - - - - - - - - - - - - - - - - - - - - - - - - - [ ID] Interval Transfer Bitrate Retr [ 5] 0.00-10.00 sec 736 MBytes 618 Mbits/sec 71 sender [ 5] 0.00-10.01 sec 733 MBytes 615 Mbits/sec receiver
after:
$ iperf3 -c beaglev Connecting to host beaglev, port 5201 [ 5] local 192.168.85.6 port 44864 connected to 192.168.85.48 port 5201 [ ID] Interval Transfer Bitrate Retr Cwnd [ 5] 0.00-1.00 sec 109 MBytes 912 Mbits/sec 48 559 KBytes [ 5] 1.00-2.00 sec 108 MBytes 902 Mbits/sec 0 690 KBytes [ 5] 2.00-3.00 sec 106 MBytes 891 Mbits/sec 36 396 KBytes [ 5] 3.00-4.00 sec 108 MBytes 902 Mbits/sec 0 567 KBytes [ 5] 4.00-5.00 sec 106 MBytes 891 Mbits/sec 0 699 KBytes [ 5] 5.00-6.00 sec 106 MBytes 891 Mbits/sec 32 414 KBytes [ 5] 6.00-7.00 sec 106 MBytes 891 Mbits/sec 0 583 KBytes [ 5] 7.00-8.00 sec 106 MBytes 891 Mbits/sec 0 708 KBytes [ 5] 8.00-9.00 sec 106 MBytes 891 Mbits/sec 28 433 KBytes [ 5] 9.00-10.00 sec 108 MBytes 902 Mbits/sec 0 591 KBytes - - - - - - - - - - - - - - - - - - - - - - - - - [ ID] Interval Transfer Bitrate Retr [ 5] 0.00-10.00 sec 1.04 GBytes 897 Mbits/sec 144 sender [ 5] 0.00-10.01 sec 1.04 GBytes 894 Mbits/sec receiver
And the decreased CPU time of the memcpy() is observable with perf top. This is the perf top -Ue task-clock output when doing the test:
before:
Overhead Shared O Symbol 42.22% [kernel] [k] memcpy 35.00% [kernel] [k] __asm_copy_to_user 3.50% [kernel] [k] sifive_l2_flush64_range 2.30% [kernel] [k] stmmac_napi_poll_rx 1.11% [kernel] [k] memset
after:
Overhead Shared O Symbol 45.69% [kernel] [k] __asm_copy_to_user 29.06% [kernel] [k] memcpy 4.09% [kernel] [k] sifive_l2_flush64_range 2.77% [kernel] [k] stmmac_napi_poll_rx 1.24% [kernel] [k] memset
Signed-off-by Matteo Croce <[email protected]>
--- arch/riscv/include/asm/string.h | 8 ++- arch/riscv/kernel/riscv_ksyms.c | 2 - arch/riscv/lib/Makefile | 2 +- arch/riscv/lib/memcpy.S | 108 -------------------------------- arch/riscv/lib/string.c | 94 +++++++++++++++++++++++++++ 5 files changed, 101 insertions(+), 113 deletions(-) delete mode 100644 arch/riscv/lib/memcpy.S create mode 100644 arch/riscv/lib/string.c
arch/riscv/include/asm/string.h +6 -2
@@ -12,9 +12,13 @@#define __HAVE_ARCH_MEMSET extern asmlinkage void *memset(void *, int, size_t); extern asmlinkage void *__memset(void *, int, size_t); + +#ifdef CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE #define __HAVE_ARCH_MEMCPY -extern asmlinkage void *memcpy(void *, const void *, size_t); -extern asmlinkage void *__memcpy(void *, const void *, size_t); +extern void *memcpy (void *dest,const void *src,size_t count); +extern void *__memcpy (void *dest,const void *src,size_t count); +#endif +#define __HAVE_ARCH_MEMMOVE extern asmlinkage void *memmove(void *, const void *, size_t); extern asmlinkage void *__memmove(void *, const void *, size_t);
arch/riscv/kernel/riscv_ksyms.c +0 -2
@@ -10,8 +10,6 @@* Assembly functions that may be used (directly or indirectly) by modules */ EXPORT_SYMBOL (memset); -EXPORT_SYMBOL (memcpy);EXPORT_SYMBOL (memmove);EXPORT_SYMBOL (__memset); -EXPORT_SYMBOL (__memcpy);EXPORT_SYMBOL (__memmove);
arch/riscv/lib/Makefile +1 -1
@@ -1,9 +1,9 @@ # SPDX-License-Identifier: GPL-2.0-only lib-y += delay.o -lib-y += memcpy.o lib-y += memset.o lib-y += memmove.o lib-$(CONFIG_MMU) += uaccess.o lib-$(CONFIG_64BIT) += tishift.o +lib-$(CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE) += string.o obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
arch/riscv/lib/memcpy.S deleted +0 -108
108 lines not on this page · raw message
arch/riscv/lib/string.c new file +94 -0
94 lines not on this page · raw message
Signature · 1 line
2.31.1
PATCH v2 2/3 riscv: optimized memmove
In reply to Matteo CroceFrom: Matteo Croce <[email protected]>
When the destination buffer is before the source one, or when the buffers doesn't overlap, it's safe to use memcpy() instead, which is optimized to use the bigger data size possible.
Signed-off-by Matteo Croce <[email protected]>
--- arch/riscv/include/asm/string.h | 6 ++-- arch/riscv/kernel/riscv_ksyms.c | 2 -- arch/riscv/lib/Makefile | 1 - arch/riscv/lib/memmove.S | 64 --------------------------------- arch/riscv/lib/string.c | 26 ++++++++++++++ 5 files changed, 29 insertions(+), 70 deletions(-) delete mode 100644 arch/riscv/lib/memmove.S
arch/riscv/include/asm/string.h +3 -3
@@ -17,11 +17,11 @@ extern asmlinkage void *__memset(void *, int, size_t);#define __HAVE_ARCH_MEMCPYextern void *memcpy (void *dest,const void *src,size_t count);extern void *__memcpy (void *dest,const void *src,size_t count); +#define __HAVE_ARCH_MEMMOVE +extern void *memmove (void *dest,const void *src,size_t count); +extern void *__memmove (void *dest,const void *src,size_t count);#endif -#define __HAVE_ARCH_MEMMOVE -extern asmlinkage void *memmove(void *, const void *, size_t); -extern asmlinkage void *__memmove(void *, const void *, size_t);/* For those files which don't want to check by kasan. */ #if defined(CONFIG_KASAN ) && !defined(__SANITIZE_ADDRESS__)#define memcpy (dst, src, len) __memcpy(dst, src, len)
arch/riscv/kernel/riscv_ksyms.c +0 -2
@@ -10,6 +10,4 @@* Assembly functions that may be used (directly or indirectly) by modules */ EXPORT_SYMBOL (memset); -EXPORT_SYMBOL (memmove);EXPORT_SYMBOL (__memset); -EXPORT_SYMBOL (__memmove);
arch/riscv/lib/Makefile +0 -1
@@ -1,7 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only lib-y += delay.o lib-y += memset.o -lib-y += memmove.o lib-$(CONFIG_MMU) += uaccess.o lib-$(CONFIG_64BIT) += tishift.o lib-$(CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE) += string.o
arch/riscv/lib/memmove.S deleted +0 -64
64 lines not on this page · raw message
arch/riscv/lib/string.c +26 -0
@@ -92,3 +92,29 @@ void *__memcpy(void *dest, const void *src, size_t count)return memcpy (dest, src, count); }EXPORT_SYMBOL (__memcpy); + +/* +* Simply check if the buffer overlaps an call memcpy() in case, +* otherwise do a simple one byte at time backward copy. +*/ +void __weak *memmove (void *dest,const void *src,size_t count) +{ +if (dest < src || src + count <= dest) +return memcpy (dest, src, count); + +if (dest > src) { +const char *s = src + count; +char *tmp = dest + count; + +while (count--) + *--tmp = *--s; + } +return dest; +} +EXPORT_SYMBOL (memmove); + +void *__memmove (void *dest,const void *src,size_t count) +{ +return memmove (dest, src, count); +} +EXPORT_SYMBOL (__memmove);
Signature · 1 line
2.31.1
PATCH v2 3/3 riscv: optimized memset
In reply to Matteo CroceFrom: Matteo Croce <[email protected]>
The generic memset is defined as a byte at time write. This is always safe, but it's slower than a 4 byte or even 8 byte write.
Write a generic memset which fills the data one byte at time until the destination is aligned, then fills using the largest size allowed, and finally fills the remaining data one byte at time.
Signed-off-by Matteo Croce <[email protected]>
--- arch/riscv/include/asm/string.h | 10 +-- arch/riscv/kernel/Makefile | 1 - arch/riscv/kernel/riscv_ksyms.c | 13 ---- arch/riscv/lib/Makefile | 1 - arch/riscv/lib/memset.S | 113 -------------------------------- arch/riscv/lib/string.c | 42 ++++++++++++ 6 files changed, 45 insertions(+), 135 deletions(-) delete mode 100644 arch/riscv/kernel/riscv_ksyms.c delete mode 100644 arch/riscv/lib/memset.S
arch/riscv/include/asm/string.h +3 -7
@@ -6,14 +6,10 @@#ifndef _ASM_RISCV_STRING_H#define _ASM_RISCV_STRING_H -#include <linux/types.h> -#include <linux/linkage.h> - -#define __HAVE_ARCH_MEMSET -extern asmlinkage void *memset(void *, int, size_t); -extern asmlinkage void *__memset(void *, int, size_t); -#ifdef CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE +#define __HAVE_ARCH_MEMSET +extern void *memset (void *s,int c,size_t count); +extern void *__memset (void *s,int c,size_t count);#define __HAVE_ARCH_MEMCPYextern void *memcpy (void *dest,const void *src,size_t count);extern void *__memcpy (void *dest,const void *src,size_t count);
arch/riscv/kernel/Makefile +0 -1
@@ -31,7 +31,6 @@ obj-y += syscall_table.o obj-y += sys_riscv.o obj-y += time.o obj-y += traps.o -obj-y += riscv_ksyms.o obj-y += stacktrace.o obj-y += cacheinfo.o obj-y += patch.o
arch/riscv/kernel/riscv_ksyms.c deleted +0 -13
@@ -1,13 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* -* Copyright (C) 2017 Zihao Yu -*/ - -#include <linux/export.h> -#include <linux/uaccess.h> - -/* -* Assembly functions that may be used (directly or indirectly) by modules -*/ -EXPORT_SYMBOL (memset); -EXPORT_SYMBOL (__memset);
arch/riscv/lib/Makefile +0 -1
@@ -1,6 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only lib-y += delay.o -lib-y += memset.o lib-$(CONFIG_MMU) += uaccess.o lib-$(CONFIG_64BIT) += tishift.o lib-$(CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE) += string.o
arch/riscv/lib/memset.S deleted +0 -113
113 lines not on this page · raw message
arch/riscv/lib/string.c +42 -0
45 lines not on this page · raw message
Signature · 1 line
2.31.1