riscv: use the generic string routines
Subsystems (2)
PATCH 1/1 riscv: use the generic string routines
From: Matteo Croce <[email protected]>
Use the generic routines which handle alignment properly.
These are the performances measured on a BeagleV machine for a 32 mbyte buffer:
memcpy:
original aligned: 75 Mb/s
original unaligned: 75 Mb/s
new aligned: 114 Mb/s
new unaligned: 107 Mb/s
memset:
original aligned: 140 Mb/s
original unaligned: 140 Mb/s
new aligned: 241 Mb/s
new unaligned: 241 Mb/s
TCP throughput with iperf3 gives a similar improvement as well.
This is the binary size increase according to bloat-o-meter:
add/remove: 0/0 grow/shrink: 4/2 up/down: 432/-36 (396)
Function old new delta memcpy 36 324 +288 memset 32 148 +116 strlcpy 116 132 +16 strscpy_pad 84 96 +12 strlcat 176 164 -12 memmove 76 52 -24
Total: Before=1225371, After=1225767, chg +0.03%
--- arch/riscv/include/asm/Kbuild | 1 + arch/riscv/include/asm/string.h | 32 --------- arch/riscv/kernel/Makefile | 1 - arch/riscv/kernel/riscv_ksyms.c | 17 ----- arch/riscv/lib/Makefile | 3 - arch/riscv/lib/memcpy.S | 108 ------------------------------ arch/riscv/lib/memmove.S | 64 ------------------ arch/riscv/lib/memset.S | 113 -------------------------------- 8 files changed, 1 insertion(+), 338 deletions(-) delete mode 100644 arch/riscv/include/asm/string.h delete mode 100644 arch/riscv/kernel/riscv_ksyms.c delete mode 100644 arch/riscv/lib/memcpy.S delete mode 100644 arch/riscv/lib/memmove.S delete mode 100644 arch/riscv/lib/memset.S
arch/riscv/include/asm/Kbuild +1 -0
@@ -3,5 +3,6 @@ generic-y += early_ioremap.h generic-y += extable.h generic-y += flat.h generic-y += kvm_para.h +generic-y += string.h generic-y += user.h generic-y += vmlinux.lds.h
arch/riscv/include/asm/string.h deleted +0 -32
@@ -1,32 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* -* Copyright (C) 2013 Regents of the University of California -*/ - -#ifndef _ASM_RISCV_STRING_H -#define _ASM_RISCV_STRING_H - -#include <linux/types.h> -#include <linux/linkage.h> - -#define __HAVE_ARCH_MEMSET -extern asmlinkage void *memset(void *, int, size_t); -extern asmlinkage void *__memset(void *, int, size_t); -#define __HAVE_ARCH_MEMCPY -extern asmlinkage void *memcpy(void *, const void *, size_t); -extern asmlinkage void *__memcpy(void *, const void *, size_t); -#define __HAVE_ARCH_MEMMOVE -extern asmlinkage void *memmove(void *, const void *, size_t); -extern asmlinkage void *__memmove(void *, const void *, size_t); -/* For those files which don't want to check by kasan. */ -#if defined(CONFIG_KASAN ) && !defined(__SANITIZE_ADDRESS__) -#define memcpy (dst, src, len) __memcpy(dst, src, len) -#define memset (s, c, n) __memset(s, c, n) -#define memmove (dst, src, len) __memmove(dst, src, len) - -#ifndef __NO_FORTIFY -#define __NO_FORTIFY/* FORTIFY_SOURCE uses __builtin_memcpy, etc. */ -#endif - -#endif -#endif /* _ASM_RISCV_STRING_H */
arch/riscv/kernel/Makefile +0 -1
@@ -31,7 +31,6 @@ obj-y += syscall_table.o obj-y += sys_riscv.o obj-y += time.o obj-y += traps.o -obj-y += riscv_ksyms.o obj-y += stacktrace.o obj-y += cacheinfo.o obj-y += patch.o
arch/riscv/kernel/riscv_ksyms.c deleted +0 -17
@@ -1,17 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* -* Copyright (C) 2017 Zihao Yu -*/ - -#include <linux/export.h> -#include <linux/uaccess.h> - -/* -* Assembly functions that may be used (directly or indirectly) by modules -*/ -EXPORT_SYMBOL (memset); -EXPORT_SYMBOL (memcpy); -EXPORT_SYMBOL (memmove); -EXPORT_SYMBOL (__memset); -EXPORT_SYMBOL (__memcpy); -EXPORT_SYMBOL (__memmove);
arch/riscv/lib/Makefile +0 -3
@@ -1,8 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only lib-y += delay.o -lib-y += memcpy.o -lib-y += memset.o -lib-y += memmove.o lib-$(CONFIG_MMU) += uaccess.o lib-$(CONFIG_64BIT) += tishift.o
arch/riscv/lib/memcpy.S deleted +0 -108
108 lines not on this page · raw message
arch/riscv/lib/memmove.S deleted +0 -64
64 lines not on this page · raw message
arch/riscv/lib/memset.S deleted +0 -113
113 lines not on this page · raw message
Signature · 1 line
2.31.1
Quoted 32 lines · Matteo Croce
From: Matteo Croce <[email protected]>
Use the generic routines which handle alignment properly.
These are the performances measured on a BeagleV machine for a 32 mbyte buffer:
memcpy:
original aligned: 75 Mb/s original unaligned: 75 Mb/s new aligned: 114 Mb/s new unaligned: 107 Mb/smemset:
original aligned: 140 Mb/s original unaligned: 140 Mb/s new aligned: 241 Mb/s new unaligned: 241 Mb/sTCP throughput with iperf3 gives a similar improvement as well.
This is the binary size increase according to bloat-o-meter:
add/remove: 0/0 grow/shrink: 4/2 up/down: 432/-36 (396)
Function old new delta memcpy 36 324 +288 memset 32 148 +116 strlcpy 116 132 +16 strscpy_pad 84 96 +12 strlcat 176 164 -12 memmove 76 52 -24
Quoted 5 lines
Total: Before=1225371, After=1225767, chg +0.03%
Signed-off-by Matteo Croce <[email protected]>Signed-off-by Emil Renner Berthing <[email protected]>---
Hi,
can someone have a look at this change and share opinions?
Regards,
Signature · 1 line
per aspera ad upstream
Palmer Dabbelt
maintainerQuoted 35 lines · On Tue, 03 Aug 2021 09:54:34 PDT (-0700), [email protected] wrote:
Quoted 34 lines · Matteo Croce
From: Matteo Croce <[email protected]>
Use the generic routines which handle alignment properly.
These are the performances measured on a BeagleV machine for a 32 mbyte buffer:
memcpy:
original aligned: 75 Mb/s original unaligned: 75 Mb/s new aligned: 114 Mb/s new unaligned: 107 Mb/smemset:
original aligned: 140 Mb/s original unaligned: 140 Mb/s new aligned: 241 Mb/s new unaligned: 241 Mb/sTCP throughput with iperf3 gives a similar improvement as well.
This is the binary size increase according to bloat-o-meter:
add/remove: 0/0 grow/shrink: 4/2 up/down: 432/-36 (396)
Function old new delta memcpy 36 324 +288 memset 32 148 +116 strlcpy 116 132 +16 strscpy_pad 84 96 +12 strlcat 176 164 -12 memmove 76 52 -24Total: Before=1225371, After=1225767, chg +0.03%
Quoted 7 lines
Quoted 3 lines
Signed-off-by Matteo Croce <[email protected]>Signed-off-by Emil Renner Berthing <[email protected]>---
Hi,
can someone have a look at this change and share opinions?
This LGTM. How are the generic string routines landing? I'm happy to take this into my for-next, but IIUC we need the optimized generic versions first so we don't have a performance regression falling back to the trivial ones for a bit. Is there a shared tag I can pull in?
From: Palmer Dabbelt
Quoted 42 lines
Sent: 04 August 2021 21:40
Quoted 39 lines · On Tue, 03 Aug 2021 09:54:34 PDT (-0700), [email protected] wrote:
Quoted 37 lines · Matteo Croce
From: Matteo Croce <[email protected]>
Use the generic routines which handle alignment properly.
These are the performances measured on a BeagleV machine for a 32 mbyte buffer:
memcpy:
original aligned: 75 Mb/s original unaligned: 75 Mb/s new aligned: 114 Mb/s new unaligned: 107 Mb/smemset:
original aligned: 140 Mb/s original unaligned: 140 Mb/s new aligned: 241 Mb/s new unaligned: 241 Mb/sTCP throughput with iperf3 gives a similar improvement as well.
This is the binary size increase according to bloat-o-meter:
add/remove: 0/0 grow/shrink: 4/2 up/down: 432/-36 (396)
Function old new delta memcpy 36 324 +288 memset 32 148 +116 strlcpy 116 132 +16 strscpy_pad 84 96 +12 strlcat 176 164 -12 memmove 76 52 -24Total: Before=1225371, After=1225767, chg +0.03%
Signed-off-by Matteo Croce <[email protected]>Signed-off-by Emil Renner Berthing <[email protected]>---
Quoted 8 lines
Quoted 3 lines
Hi,
can someone have a look at this change and share opinions?
This LGTM. How are the generic string routines landing? I'm happy to take this into my for-next, but IIUC we need the optimized generic versions first so we don't have a performance regression falling back to the trivial ones for a bit. Is there a shared tag I can pull in?
I thought the actual problem was that the asm copy functions were doing misaligned transfers and faulting.
There is no way that the simple C loop should be as fast as the asm function given the delay cycles reading from memory.
You definitely need to test much smaller copies where the buffers are resident in the L1 data cache.
Anything else is completely dominated by the cache line fills/spills.
You also need to test on the much faster riscv implementations not just on the beaglev board.
David
-
Registered Address Lakeside, Bramley Road, Mount Farm, Milton Keynes, MK1 1PT, UK Registration No: 1397386 (Wales)
Quoted 41 lines · Palmer Dabbelt
Quoted 39 lines · On Tue, 03 Aug 2021 09:54:34 PDT (-0700), [email protected] wrote:
Quoted 37 lines · Matteo Croce
From: Matteo Croce <[email protected]>
Use the generic routines which handle alignment properly.
These are the performances measured on a BeagleV machine for a 32 mbyte buffer:
memcpy:
original aligned: 75 Mb/s original unaligned: 75 Mb/s new aligned: 114 Mb/s new unaligned: 107 Mb/smemset:
original aligned: 140 Mb/s original unaligned: 140 Mb/s new aligned: 241 Mb/s new unaligned: 241 Mb/sTCP throughput with iperf3 gives a similar improvement as well.
This is the binary size increase according to bloat-o-meter:
add/remove: 0/0 grow/shrink: 4/2 up/down: 432/-36 (396)
Function old new delta memcpy 36 324 +288 memset 32 148 +116 strlcpy 116 132 +16 strscpy_pad 84 96 +12 strlcat 176 164 -12 memmove 76 52 -24Total: Before=1225371, After=1225767, chg +0.03%
Signed-off-by Matteo Croce <[email protected]>Signed-off-by Emil Renner Berthing <[email protected]>---
Quoted 8 lines
Quoted 3 lines
Hi,
can someone have a look at this change and share opinions?
This LGTM. How are the generic string routines landing? I'm happy to take this into my for-next, but IIUC we need the optimized generic versions first so we don't have a performance regression falling back to the trivial ones for a bit. Is there a shared tag I can pull in?
Hi,
I see them only in linux-next by now.
Signature · 1 line
per aspera ad upstream
Palmer Dabbelt
maintainerQuoted 46 lines · On Thu, 05 Aug 2021 03:31:04 PDT (-0700), [email protected] wrote:
Quoted 45 lines · Palmer Dabbelt
Quoted 42 lines · On Tue, 03 Aug 2021 09:54:34 PDT (-0700), [email protected] wrote:
Quoted 32 lines · Matteo Croce
From: Matteo Croce <[email protected]>
Use the generic routines which handle alignment properly.
These are the performances measured on a BeagleV machine for a 32 mbyte buffer:
memcpy:
original aligned: 75 Mb/s original unaligned: 75 Mb/s new aligned: 114 Mb/s new unaligned: 107 Mb/smemset:
original aligned: 140 Mb/s original unaligned: 140 Mb/s new aligned: 241 Mb/s new unaligned: 241 Mb/sTCP throughput with iperf3 gives a similar improvement as well.
This is the binary size increase according to bloat-o-meter:
add/remove: 0/0 grow/shrink: 4/2 up/down: 432/-36 (396)
Function old new delta memcpy 36 324 +288 memset 32 148 +116 strlcpy 116 132 +16 strscpy_pad 84 96 +12 strlcat 176 164 -12 memmove 76 52 -24Quoted 5 lines
Total: Before=1225371, After=1225767, chg +0.03%
Signed-off-by Matteo Croce <[email protected]>Signed-off-by Emil Renner Berthing <[email protected]>---
Hi,
can someone have a look at this change and share opinions?
Quoted 8 lines
Quoted 4 lines
This LGTM. How are the generic string routines landing? I'm happy to take this into my for-next, but IIUC we need the optimized generic versions first so we don't have a performance regression falling back to the trivial ones for a bit. Is there a shared tag I can pull in?
Hi,
I see them only in linux-next by now.
These ended up getting rejected by Linus, so I'm going to hold off on this for now. If they're really out of lib/ then I'll take the C routines in arch/riscv, but either way it's an issue for the next release.
..
Quoted 4 lines
These ended up getting rejected by Linus, so I'm going to hold off on this for now. If they're really out of
lib/then I'll take the C routines inarch/riscv, but either way it's an issue for the next release.
I've been half following this.
I've not seen any comparisons between the C functions proposed here and the riscv asm ones that had the fix for misaligned transfers applied.
IIRC there is a comment in the asm ones that the unrolled 'read lots' - 'write lots' loop is faster than the older (asm) read-write loop.
But I've not seen any archictural discussions at all.
A simple in-order single-issue cpu will execute the unrolled loop faster just because it has fewer instructions. The read-lots - write-lots almost certainly helps avoid read-latency delaying things if multiple reads can be pipelined.
The writes are almost certainly 'posted' and pipelined, But a simple cpu could easily require all writes finish before doing a read.
A super-scaler (multi-issue) cpu gives you the ability to get the loop control instructions 'for free' with carefully written assembler.
At which point a copy for 'life cache' data should be limited only by the cpu's cache memory bandwidth.
If reads and writes can interleave then a loop that alternates reads and writes (read each register just after writing it) may mean that you always keep the cpu-cache interface busy.
This would be especially true if the cpu can execute both a cache read and write in the same cycle. (Which many moderate performance cpu can.)
None of the requires out-of-order execution, just execution to continue while a read is in progress.
I'm also guessing that any performance testing has been done with the (relatively) cheap boards that are readily available.
But I've also seen references in the press to much faster riscv cpu that are definitely multi-issue and may have some simple out-of-order execution.
Any changes ought to be tested on these faster systems.
I also recall that some of the performance measurements were made with long buffers - they will be dominated by the cache to DRAM (and maybe TLB lookup) timings, not the copy loop.
For a simple cpu you ought to be able to measure the number of cpu cycles used for a copy - and account for all of them.
For something like x86 you can show that the copy is being limited by the cpu-cache bandwidth. (FWIW measurements of the inet checksum code on x86 show it runs at half the expected speed on a lot of Intel cpu - no one ever measured it.)
David
-
Registered Address Lakeside, Bramley Road, Mount Farm, Milton Keynes, MK1 1PT, UK Registration No: 1397386 (Wales)
Quoted 53 lines · Palmer Dabbelt
Quoted 51 lines · On Thu, 05 Aug 2021 03:31:04 PDT (-0700), [email protected] wrote:
Quoted 49 lines · Palmer Dabbelt
Quoted 35 lines · On Tue, 03 Aug 2021 09:54:34 PDT (-0700), [email protected] wrote:
Quoted 34 lines · Matteo Croce
From: Matteo Croce <[email protected]>
Use the generic routines which handle alignment properly.
These are the performances measured on a BeagleV machine for a 32 mbyte buffer:
memcpy:
original aligned: 75 Mb/s original unaligned: 75 Mb/s new aligned: 114 Mb/s new unaligned: 107 Mb/smemset:
original aligned: 140 Mb/s original unaligned: 140 Mb/s new aligned: 241 Mb/s new unaligned: 241 Mb/sTCP throughput with iperf3 gives a similar improvement as well.
This is the binary size increase according to bloat-o-meter:
add/remove: 0/0 grow/shrink: 4/2 up/down: 432/-36 (396)
Function old new delta memcpy 36 324 +288 memset 32 148 +116 strlcpy 116 132 +16 strscpy_pad 84 96 +12 strlcat 176 164 -12 memmove 76 52 -24Total: Before=1225371, After=1225767, chg +0.03%
Quoted 7 lines
Quoted 3 lines
Signed-off-by Matteo Croce <[email protected]>Signed-off-by Emil Renner Berthing <[email protected]>---
Hi,
can someone have a look at this change and share opinions?
This LGTM. How are the generic string routines landing? I'm happy to take this into my for-next, but IIUC we need the optimized generic versions first so we don't have a performance regression falling back to the trivial ones for a bit. Is there a shared tag I can pull in?
Quoted 8 lines
Quoted 3 lines
Hi,
I see them only in linux-next by now.
These ended up getting rejected by Linus, so I'm going to hold off on this for now. If they're really out of
lib/then I'll take the C routines inarch/riscv, but either way it's an issue for the next release.
Agree, we should take the C routine in arch/riscv for common implementation. If any vendor what custom implementation they could use the alternative framework in errata for string operations.
Signature · 4 lines
Best Regards Guo Ren ML: https://lore.kernel.org/linux-csky/
Quoted 7 lines
Quoted 4 lines
These ended up getting rejected by Linus, so I'm going to hold off on this for now. If they're really out of
lib/then I'll take the C routines inarch/riscv, but either way it's an issue for the next release.Agree, we should take the C routine in
arch/riscvfor common implementation. If any vendor what custom implementation they could use the alternative framework in errata for string operations.
I though the asm ones were significantly faster because they were less affected by read latency.
(But they were horribly broken for misaligned transfers.)
David
-
Registered Address Lakeside, Bramley Road, Mount Farm, Milton Keynes, MK1 1PT, UK Registration No: 1397386 (Wales)
Quoted 5 lines · David Laight
Quoted 5 lines
Quoted 5 lines
These ended up getting rejected by Linus, so I'm going to hold off on this for now. If they're really out of
lib/then I'll take the C routines inarch/riscv, but either way it's an issue for the next release.
Quoted 9 lines
Quoted 3 lines
Agree, we should take the C routine in
arch/riscvfor common implementation. If any vendor what custom implementation they could use the alternative framework in errata for string operations.I though the asm ones were significantly faster because they were less affected by read latency.
(But they were horribly broken for misaligned transfers.)
I can get the same exact performance (and a very similar machine code) in C with this on top of the C memset implementation:
arch/riscv/lib/string.c +0 -0
@@ -112,9 +112,12 @@ EXPORT_SYMBOL(__memmove);void *memmove (void *dest,const void *src,size_t count)__weak
__alias(__memmove);
EXPORT_SYMBOL(memmove);
+#define BATCH 4
+
void *__memset(void *s, int c, size_t count)
{
union types dest = { .as_u8 = s };
+ int i;
if (count >= MIN_THRESHOLD) {
unsigned long cu = (unsigned long)c;
@@ -138,8 +141,12 @@ void *__memset(void *s, int c, size_t count)
}
/* Copy using the largest size allowed */
- for (; count >= BYTES_LONG; count -= BYTES_LONG)
- *dest.as_ulong++ = cu;
+ for (; count >= BYTES_LONG * BATCH; count -= BYTES_LONG * BATCH) { +#pragma GCC unroll 4
+ for (i = 0; i < BATCH; i++)
+ dest.as_ulong[i] = cu;
+ dest.as_ulong += BATCH;
+ }
}
On the BeagleV the memset speed with the different batch size are:
1 (stock): 267 Mb/s
2: 272 Mb/s
4: 276 Mb/s
8: 276 Mb/s
The problem with biggest batch size is that it will fallback to a single byte copy if the buffers are too small.
Regards,
Signature · 1 line
per aspera ad upstream