Zbb string optimizations and call support in alternatives
v4 cover Zbb string optimizations and call support in alternatives
From: Heiko Stuebner <[email protected]>
This series still tries to allow optimized string functions for specific extensions. The last approach of using an inline base function to hold the alternative calls did cause some issues in a number of places
So instead of that we're now just using an alternative j at the beginning of the generic function to jump to a separate place inside the function itself.
This of course needs a fixup for "j" instructions in alternative blocks, so that is provided here as well.
Technically patch4 got a review from Andrew, but that was still with the inline approach, so I didn't bring it over to v4.
changes since v3:
- rebase on top of 6.2-rc1 + the applied alternative-call series
- add alternative fixup for jal instructions
- drop the inline functions and instead just jump
changes since v2:
- add patch fixing the c.jalr funct4 value
- reword some commit messages
- fix position of auipc addition patch (earlier)
- fix compile errors from patch-reordering gone wrong (worked at the end of v2, but compiling individual patches caused issues) - patches are now tested individually
- limit Zbb variants for GNU as for now
(LLVM support for .option arch is still under review)
- prevent str-functions from getting optimized to builtin-variants
changes since v1:
- a number of generalizations/cleanups for instruction parsing
- use accessor function to access instructions (Emil)
- actually patch the correct location when having more than one instruction in an alternative block
- string function cleanups (comments etc) (Conor)
- move zbb extension above s* extensions in cpu.c lists
changes since rfc:
- make Zbb code actually work
- drop some unneeded patches
- a lot of cleanups
Heiko Stuebner (5):
RISC-V: move some stray __RISCV_INSN_FUNCS definitions from kprobes
RISC-V: add helpers for J-type immediate handling
RISC-V: fix jal addresses in patched alternatives
RISC-V: add infrastructure to allow different str* implementations
RISC-V: add zbb support to string functions
arch/riscv/Kconfig | 24 ++++ arch/riscv/include/asm/errata_list.h | 3 +- arch/riscv/include/asm/hwcap.h | 1 + arch/riscv/include/asm/insn.h | 36 ++++++ arch/riscv/include/asm/string.h | 12 ++ arch/riscv/kernel/alternative.c | 27 ++++ arch/riscv/kernel/cpu.c | 1 + arch/riscv/kernel/cpufeature.c | 18 +++ arch/riscv/kernel/probes/simulate-insn.h | 3 - arch/riscv/kernel/riscv_ksyms.c | 3 + arch/riscv/lib/Makefile | 3 + arch/riscv/lib/strcmp.S | 131 ++++++++++++++++++++ arch/riscv/lib/strlen.S | 142 +++++++++++++++++++++ arch/riscv/lib/strncmp.S | 151 +++++++++++++++++++++++ arch/riscv/purgatory/Makefile | 13 ++ 15 files changed, 564 insertions(+), 4 deletions(-) create mode 100644 arch/riscv/lib/strcmp.S create mode 100644 arch/riscv/lib/strlen.S create mode 100644 arch/riscv/lib/strncmp.S
Signature · 7 lines
2.35.1 _______________________________________________ linux-riscv mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-riscv
PATCH v4 1/5 RISC-V: move some stray __RISCV_INSN_FUNCS definitions from kprobes
From: Heiko Stuebner <[email protected]>
The __RISCV_INSN_FUNCS originally declared riscv_insn_is_* functions inside the kprobes implementation. This got moved into a central header in commit ec5f90877516 ("RISC-V: Move riscv_insn_is_* macros into a common header").
Though it looks like I overlooked two of them, so fix that. FENCE itself is an instruction defined directly by its own opcode, while the created riscv_isn_is_system function covers all instructions defined under the SYSTEM opcode.
--- arch/riscv/include/asm/insn.h | 10 ++++++++++ arch/riscv/kernel/probes/simulate-insn.h | 3 --- 2 files changed, 10 insertions(+), 3 deletions(-)
arch/riscv/include/asm/insn.h +10 -0
@@ -128,6 +128,7 @@#define RVC_C2_RD_OPOFF 7/* parts of opcode for RVG*/ +#define RVG_OPCODE_FENCE 0x0f#define RVG_OPCODE_AUIPC 0x17#define RVG_OPCODE_BRANCH 0x63#define RVG_OPCODE_JALR 0x67
@@ -163,6 +164,7 @@#define RVG_MATCH_AUIPC (RVG_OPCODE_AUIPC)#define RVG_MATCH_JALR (RV_ENCODE_FUNCT3(JALR) | RVG_OPCODE_JALR)#define RVG_MATCH_JAL (RVG_OPCODE_JAL) +#define RVG_MATCH_FENCE (RVG_OPCODE_FENCE)#define RVG_MATCH_BEQ (RV_ENCODE_FUNCT3(BEQ) | RVG_OPCODE_BRANCH)#define RVG_MATCH_BNE (RV_ENCODE_FUNCT3(BNE) | RVG_OPCODE_BRANCH)#define RVG_MATCH_BLT (RV_ENCODE_FUNCT3(BLT) | RVG_OPCODE_BRANCH)
@@ -182,6 +184,7 @@#define RVG_MASK_AUIPC (RV_INSN_OPCODE_MASK)#define RVG_MASK_JALR (RV_INSN_FUNCT3_MASK | RV_INSN_OPCODE_MASK)#define RVG_MASK_JAL (RV_INSN_OPCODE_MASK) +#define RVG_MASK_FENCE (RV_INSN_OPCODE_MASK)#define RVC_MASK_C_JALR (RVC_INSN_FUNCT4_MASK | RVC_INSN_J_RS2_MASK | RVC_INSN_OPCODE_MASK)#define RVC_MASK_C_JR (RVC_INSN_FUNCT4_MASK | RVC_INSN_J_RS2_MASK | RVC_INSN_OPCODE_MASK)#define RVC_MASK_C_JAL (RVC_INSN_FUNCT3_MASK | RVC_INSN_OPCODE_MASK)
@@ -233,6 +236,13 @@ __RISCV_INSN_FUNCS(c_bnez, RVC_MASK_C_BNEZ, RVC_MATCH_C_BNEZ) __RISCV_INSN_FUNCS(c_ebreak, RVC_MASK_C_EBREAK, RVC_MATCH_C_EBREAK) __RISCV_INSN_FUNCS(ebreak, RVG_MASK_EBREAK, RVG_MATCH_EBREAK) __RISCV_INSN_FUNCS(sret, RVG_MASK_SRET, RVG_MATCH_SRET) +__RISCV_INSN_FUNCS (fence,RVG_MASK_FENCE ,RVG_MATCH_FENCE ); + +/* special case to catch _any_ system instruction */ +static __always_inline bool riscv_insn_is_system(u32 code) +{ +return (code &RV_INSN_OPCODE_MASK ) ==RVG_OPCODE_SYSTEM ; +}/* special case to catch _any_ branch instruction */ static __always_inline bool riscv_insn_is_branch(u32 code)
arch/riscv/kernel/probes/simulate-insn.h +0 -3
@@ -12,9 +12,6 @@ } \ } while (0) -__RISCV_INSN_FUNCS (system,0x7f ,0x73 ); -__RISCV_INSN_FUNCS (fence,0x7f ,0x0f ); -#define RISCV_INSN_SET_SIMULATE (name, code) \ do { \ if (riscv_insn_is_##name(code)) { \
Signature · 7 lines
2.35.1 _______________________________________________ linux-riscv mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-riscv
Re: [PATCH v4 1/5] RISC-V: move some stray __RISCV_INSN_FUNCS definitions from kprobes
Quoted 6 lines · Heiko Stuebner
From: Heiko Stuebner <[email protected]>
The
__RISCV_INSN_FUNCSoriginally declared riscv_insn_is_* functions inside the kprobes implementation. This got moved into a central header in commitec5f90877516("RISC-V: Move riscv_insn_is_* macros into a common header").
Quoted 6 lines
Though it looks like I overlooked two of them, so fix that. FENCE itself is an instruction defined directly by its own opcode, while the created
riscv_isn_is_systemfunction covers all instructions defined under the SYSTEM opcode.Fixes ec5f90877516 ("RISC-V: Move riscv_insn_is_* macros into a common header")
Not quite sure why it needs a fixes tag, nothing was actually broken previously, right?
Not that I really care, but Sasha is likely gonna pick it up for backporting as a result.
Either way, looks grand.
Quoted 65 lines
Signed-off-by Heiko Stuebner <[email protected]>--- arch/riscv/include/asm/insn.h | 10 ++++++++++ arch/riscv/kernel/probes/simulate-insn.h | 3 --- 2 files changed, 10 insertions(+), 3 deletions(-)arch/riscv/include/asm/insn.h +10
-0@@ -128,6 +128,7 @@#define RVC_C2_RD_OPOFF 7/* parts of opcode for RVG*/ +#define RVG_OPCODE_FENCE 0x0f#define RVG_OPCODE_AUIPC 0x17#define RVG_OPCODE_BRANCH 0x63#define RVG_OPCODE_JALR 0x67@@ -163,6 +164,7 @@#define RVG_MATCH_AUIPC (RVG_OPCODE_AUIPC)#define RVG_MATCH_JALR (RV_ENCODE_FUNCT3(JALR) | RVG_OPCODE_JALR)#define RVG_MATCH_JAL (RVG_OPCODE_JAL) +#define RVG_MATCH_FENCE (RVG_OPCODE_FENCE)#define RVG_MATCH_BEQ (RV_ENCODE_FUNCT3(BEQ) | RVG_OPCODE_BRANCH)#define RVG_MATCH_BNE (RV_ENCODE_FUNCT3(BNE) | RVG_OPCODE_BRANCH)#define RVG_MATCH_BLT (RV_ENCODE_FUNCT3(BLT) | RVG_OPCODE_BRANCH)@@ -182,6 +184,7 @@#define RVG_MASK_AUIPC (RV_INSN_OPCODE_MASK)#define RVG_MASK_JALR (RV_INSN_FUNCT3_MASK | RV_INSN_OPCODE_MASK)#define RVG_MASK_JAL (RV_INSN_OPCODE_MASK) +#define RVG_MASK_FENCE (RV_INSN_OPCODE_MASK)#define RVC_MASK_C_JALR (RVC_INSN_FUNCT4_MASK | RVC_INSN_J_RS2_MASK | RVC_INSN_OPCODE_MASK)#define RVC_MASK_C_JR (RVC_INSN_FUNCT4_MASK | RVC_INSN_J_RS2_MASK | RVC_INSN_OPCODE_MASK)#define RVC_MASK_C_JAL (RVC_INSN_FUNCT3_MASK | RVC_INSN_OPCODE_MASK)@@ -233,6 +236,13 @@ __RISCV_INSN_FUNCS(c_bnez, RVC_MASK_C_BNEZ, RVC_MATCH_C_BNEZ) __RISCV_INSN_FUNCS(c_ebreak, RVC_MASK_C_EBREAK, RVC_MATCH_C_EBREAK) __RISCV_INSN_FUNCS(ebreak, RVG_MASK_EBREAK, RVG_MATCH_EBREAK) __RISCV_INSN_FUNCS(sret, RVG_MASK_SRET, RVG_MATCH_SRET) +__RISCV_INSN_FUNCS (fence,RVG_MASK_FENCE ,RVG_MATCH_FENCE ); + +/* special case to catch _any_ system instruction */ +static __always_inline bool riscv_insn_is_system(u32 code) +{ +return (code &RV_INSN_OPCODE_MASK ) ==RVG_OPCODE_SYSTEM ; +}/* special case to catch _any_ branch instruction */ static __always_inline bool riscv_insn_is_branch(u32 code)arch/riscv/kernel/probes/simulate-insn.h +0
-3@@ -12,9 +12,6 @@ } \ } while (0) -__RISCV_INSN_FUNCS (system,0x7f ,0x73 ); -__RISCV_INSN_FUNCS (fence,0x7f ,0x0f ); -#define RISCV_INSN_SET_SIMULATE (name, code) \ do { \ if (riscv_insn_is_##name(code)) { \Signature · 1 line
2.35.1
_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv
Re: [PATCH v4 1/5] RISC-V: move some stray __RISCV_INSN_FUNCS definitions from kprobes
Am Montag, 9. Januar 2023, 21:53:27 CET schrieb Conor Dooley:
Quoted 12 lines · Heiko Stuebner
Quoted 11 lines
From: Heiko Stuebner <[email protected]>
The
__RISCV_INSN_FUNCSoriginally declared riscv_insn_is_* functions inside the kprobes implementation. This got moved into a central header in commitec5f90877516("RISC-V: Move riscv_insn_is_* macros into a common header").Though it looks like I overlooked two of them, so fix that. FENCE itself is an instruction defined directly by its own opcode, while the created
riscv_isn_is_systemfunction covers all instructions defined under the SYSTEM opcode.
Quoted 4 lines
Quoted 1 line
Fixes ec5f90877516 ("RISC-V: Move riscv_insn_is_* macros into a common header")Not quite sure why it needs a fixes tag, nothing was actually broken previously, right?
I was going with the thought of "this should've been in the original patch already" / "the original patch does not fully do what it says it's doing" hence the Fixes, but you're right - it's just cosmetics at this point and would just incur more work for the stable people.
So I've dropped the Fixes
Heiko
_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv
Re: [PATCH v4 1/5] RISC-V: move some stray __RISCV_INSN_FUNCS definitions from kprobes
In reply to Heiko StuebnerQuoted 11 lines · Heiko Stuebner
From: Heiko Stuebner <[email protected]>
The
__RISCV_INSN_FUNCSoriginally declared riscv_insn_is_* functions inside the kprobes implementation. This got moved into a central header in commitec5f90877516("RISC-V: Move riscv_insn_is_* macros into a common header").Though it looks like I overlooked two of them, so fix that. FENCE itself is an instruction defined directly by its own opcode, while the created
riscv_isn_is_systemfunction covers all instructions defined under the SYSTEM opcode.
Quoted 7 lines
Fixes ec5f90877516 ("RISC-V: Move riscv_insn_is_* macros into a common header")Signed-off-by Heiko Stuebner <[email protected]>--- arch/riscv/include/asm/insn.h | 10 ++++++++++ arch/riscv/kernel/probes/simulate-insn.h | 3 --- 2 files changed, 10 insertions(+), 3 deletions(-)
_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv
PATCH v4 2/5 RISC-V: add helpers for J-type immediate handling
In reply to Heiko StuebnerFrom: Heiko Stuebner <[email protected]>
Similar to the helper for the u-type + i-type imm handling, add helper- functions for j-type immediates.
While it also would be possible to open-code that bit of imm wiggling using the macros already provided in insn.h, it's way better to have consistency on how we handle this across all function encoding/decoding.
--- arch/riscv/include/asm/insn.h | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+)
arch/riscv/include/asm/insn.h +26 -0
@@ -301,6 +301,32 @@ static __always_inline bool riscv_insn_is_branch(u32 code) (RVC_X (x_,RVC_B_IMM_7_6_OPOFF ,RVC_B_IMM_7_6_MASK ) <<RVC_B_IMM_7_6_OFF ) | \ (RVC_IMM_SIGN(x_) << RVC_B_IMM_SIGN_OFF); }) +/* +* Get the immediate from a J-type instruction. +* +* @insn: instruction to process +* Return: immediate +*/ +static inline s32 riscv_insn_extract_jtype_imm (u32 insn) +{ +return RV_EXTRACT_JTYPE_IMM (insn); +} + +/* +* Update a J-type instruction with an immediate value. +* +* @insn: pointer to the jtype instruction +* @imm: the immediate to insert into the instruction +*/ +static inline void riscv_insn_insert_jtype_imm (u32 *insn,s32 imm) +{ + *insn &= ~GENMASK (31 ,12 ); + *insn |= (((imm & (RV_J_IMM_10_1_MASK <<RV_J_IMM_10_1_OFF )) <<RV_I_IMM_11_0_OPOFF ) | + ((imm & (RV_J_IMM_11_MASK <<RV_J_IMM_11_OFF )) <<RV_J_IMM_11_OPOFF ) | + ((imm & (RV_J_IMM_19_12_OPOFF <<RV_J_IMM_19_12_OFF )) <<RV_J_IMM_19_12_OPOFF ) | + ((imm & (1 <<RV_J_IMM_SIGN_OFF )) <<RV_J_IMM_SIGN_OPOFF )); +} +/* * Put together one immediate from a U-type and I-type instruction pair. *
Signature · 7 lines
2.35.1 _______________________________________________ linux-riscv mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-riscv
Re: [PATCH v4 2/5] RISC-V: add helpers for J-type immediate handling
Quoted 33 lines · Heiko Stuebner
From: Heiko Stuebner <[email protected]>
Similar to the helper for the u-type + i-type imm handling, add helper- functions for j-type immediates.
While it also would be possible to open-code that bit of imm wiggling using the macros already provided in
insn.h, it's way better to have consistency on how we handle this across all function encoding/decoding.Signed-off-by Heiko Stuebner <[email protected]>--- arch/riscv/include/asm/insn.h | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+)arch/riscv/include/asm/insn.h +11
-0@@ -301,6 +301,32 @@ static __always_inline bool riscv_insn_is_branch(u32 code) (RVC_X (x_,RVC_B_IMM_7_6_OPOFF ,RVC_B_IMM_7_6_MASK ) <<RVC_B_IMM_7_6_OFF ) | \ (RVC_IMM_SIGN(x_) << RVC_B_IMM_SIGN_OFF); }) +/* +* Get the immediate from a J-type instruction. +* +* @insn: instruction to process +* Return: immediate +*/ +static inline s32 riscv_insn_extract_jtype_imm (u32 insn) +{ +return RV_EXTRACT_JTYPE_IMM (insn); +} +
arch/riscv/include/asm/insn.h static __always_inline bool riscv_insn_is_branch(u32 code) · line 324
Quoted 10 lines
arch/riscv/include/asm/insn.h +10
-0+/* +* Update a J-type instruction with an immediate value. +* +* @insn: pointer to the jtype instruction +* @imm: the immediate to insert into the instruction +*/ +static inline void riscv_insn_insert_jtype_imm (u32 *insn,s32 imm) +{ + *insn &= ~GENMASK (31 ,12 ); + *insn |= (((imm & (RV_J_IMM_10_1_MASK << RV_J_IMM_10_1_OFF)) << RV_I_IMM_11_0_OPOFF) |
^
RV_I? That should be RV_J_IMM_10_1_OPOFF, no?
arch/riscv/include/asm/insn.h static __always_inline bool riscv_insn_is_branch(u32 code) · line 328
Quoted 4 lines
arch/riscv/include/asm/insn.h +4
-0+ ((imm & (RV_J_IMM_11_MASK <<RV_J_IMM_11_OFF )) <<RV_J_IMM_11_OPOFF ) | + ((imm & (RV_J_IMM_19_12_OPOFF <<RV_J_IMM_19_12_OFF )) <<RV_J_IMM_19_12_OPOFF ) | + ((imm & (1 << RV_J_IMM_SIGN_OFF)) << RV_J_IMM_SIGN_OPOFF)); +}
Otherwise, seems fine?
Quoted 7 lines
arch/riscv/include/asm/insn.h +1
-0+/* * Put together one immediate from a U-type and I-type instruction pair. * Signature · 1 line
2.35.1
_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv
Re: [PATCH v4 2/5] RISC-V: add helpers for J-type immediate handling
In reply to Heiko StuebnerQuoted 33 lines · Heiko Stuebner
From: Heiko Stuebner <[email protected]>
Similar to the helper for the u-type + i-type imm handling, add helper- functions for j-type immediates.
While it also would be possible to open-code that bit of imm wiggling using the macros already provided in
insn.h, it's way better to have consistency on how we handle this across all function encoding/decoding.Signed-off-by Heiko Stuebner <[email protected]>--- arch/riscv/include/asm/insn.h | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+)arch/riscv/include/asm/insn.h +11
-0@@ -301,6 +301,32 @@ static __always_inline bool riscv_insn_is_branch(u32 code) (RVC_X (x_,RVC_B_IMM_7_6_OPOFF ,RVC_B_IMM_7_6_MASK ) <<RVC_B_IMM_7_6_OFF ) | \ (RVC_IMM_SIGN(x_) << RVC_B_IMM_SIGN_OFF); }) +/* +* Get the immediate from a J-type instruction. +* +* @insn: instruction to process +* Return: immediate +*/ +static inline s32 riscv_insn_extract_jtype_imm (u32 insn) +{ +return RV_EXTRACT_JTYPE_IMM (insn); +} +
arch/riscv/include/asm/insn.h static __always_inline bool riscv_insn_is_branch(u32 code) · line 324
Quoted 10 lines
arch/riscv/include/asm/insn.h +10
-0+/* +* Update a J-type instruction with an immediate value. +* +* @insn: pointer to the jtype instruction +* @imm: the immediate to insert into the instruction +*/ +static inline void riscv_insn_insert_jtype_imm (u32 *insn,s32 imm) +{ + *insn &= ~GENMASK (31 ,12 ); + *insn |= (((imm & (RV_J_IMM_10_1_MASK << RV_J_IMM_10_1_OFF)) << RV_I_IMM_11_0_OPOFF) |
^ RV_J_IMM_10_1_OPOFF (as pointed out by Conor)
arch/riscv/include/asm/insn.h static __always_inline bool riscv_insn_is_branch(u32 code) · line 326
Quoted 2 lines
arch/riscv/include/asm/insn.h +2
-0+ ((imm & (RV_J_IMM_11_MASK <<RV_J_IMM_11_OFF )) <<RV_J_IMM_11_OPOFF ) | + ((imm & (RV_J_IMM_19_12_OPOFF << RV_J_IMM_19_12_OFF)) << RV_J_IMM_19_12_OPOFF) |
^ RV_J_IMM_19_12_MASK
arch/riscv/include/asm/insn.h static __always_inline bool riscv_insn_is_branch(u32 code) · line 327
Quoted 1 line
arch/riscv/include/asm/insn.h +1
-0+ ((imm & (1 << RV_J_IMM_SIGN_OFF)) << RV_J_IMM_SIGN_OPOFF));
nit: can drop the outermost ()
Quoted 8 lines
arch/riscv/include/asm/insn.h +2
-0+} +/* * Put together one immediate from a U-type and I-type instruction pair. * Signature · 1 line
2.35.1
Thanks,
drew
_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv
Re: [PATCH v4 2/5] RISC-V: add helpers for J-type immediate handling
Quoted 7 lines · Andrew Jones
arch/riscv/include/asm/insn.h static __always_inline bool riscv_insn_is_branch(u32 code) · line 324
Quoted 1 line
arch/riscv/include/asm/insn.h +1
-0+ *insn |= (((imm & (RV_J_IMM_10_1_MASK << RV_J_IMM_10_1_OFF)) << RV_I_IMM_11_0_OPOFF) |^
RV_J_IMM_10_1_OPOFF(as pointed out by Conor)arch/riscv/include/asm/insn.h static __always_inline bool riscv_insn_is_branch(u32 code) · line 326
Quoted 2 lines
arch/riscv/include/asm/insn.h +2
-0+ ((imm & (RV_J_IMM_11_MASK <<RV_J_IMM_11_OFF )) <<RV_J_IMM_11_OPOFF ) | + ((imm & (RV_J_IMM_19_12_OPOFF << RV_J_IMM_19_12_OFF)) << RV_J_IMM_19_12_OPOFF) |^ RV_J_IMM_19_12_MASK
You'd think that having seen one I'd have picked up on the other one being wrong too...
_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv
Re: [PATCH v4 2/5] RISC-V: add helpers for J-type immediate handling
Quoted 5 lines · Conor Dooley
Quoted 3 lines · Andrew Jones
arch/riscv/include/asm/insn.h static __always_inline bool riscv_insn_is_branch(u32 code) · line 324
Quoted 1 line
arch/riscv/include/asm/insn.h +1
-0+ *insn |= (((imm & (RV_J_IMM_10_1_MASK << RV_J_IMM_10_1_OFF)) << RV_I_IMM_11_0_OPOFF) |^
RV_J_IMM_10_1_OPOFF(as pointed out by Conor)
Quoted 7 lines
Quoted 3 lines
arch/riscv/include/asm/insn.h static __always_inline bool riscv_insn_is_branch(u32 code) · line 326
Quoted 2 lines
arch/riscv/include/asm/insn.h +2
-0+ ((imm & (RV_J_IMM_11_MASK <<RV_J_IMM_11_OFF )) <<RV_J_IMM_11_OPOFF ) | + ((imm & (RV_J_IMM_19_12_OPOFF << RV_J_IMM_19_12_OFF)) << RV_J_IMM_19_12_OPOFF) |^ RV_J_IMM_19_12_MASKYou'd think that having seen one I'd have picked up on the other one being wrong too...
we also need RV_X() usage or do similar ;)
[1] https://lore.kernel.org/linux-riscv/[email protected]/
_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv
PATCH v4 3/5 RISC-V: fix jal addresses in patched alternatives
In reply to Heiko StuebnerFrom: Heiko Stuebner <[email protected]>
Alternatives live in a different section, so addresses used by jal instructions will point to wrong locations after the patch got applied.
Similar to arm64, adjust the location to consider that offset.
--- arch/riscv/kernel/alternative.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+)
arch/riscv/kernel/alternative.c +27 -0
@@ -79,6 +79,21 @@ static void riscv_alternative_fix_auipc_jalr(void *ptr, u32 auipc_insn,patch_text_nosync (ptr, call,sizeof (u32) *2 ); } +static void riscv_alternative_fix_jal (void *ptr,u32 jal_insn,int patch_offset) +{ +s32 imm; + +/* get and adjust new target address */ + imm =riscv_insn_extract_jtype_imm (jal_insn); + imm -= patch_offset; + +/* update instructions */ +riscv_insn_insert_jtype_imm (&jal_insn, imm); + +/* patch the call place again */ +patch_text_nosync (ptr, &jal_insn,sizeof (u32)); +} +void riscv_alternative_fix_offsets (void *alt_ptr,unsigned int len,int patch_offset) {
@@ -106,6 +121,18 @@ void riscv_alternative_fix_offsets(void *alt_ptr, unsigned int len,riscv_alternative_fix_auipc_jalr (alt_ptr + i *sizeof (u32), insn, insn2, patch_offset); } + +if (riscv_insn_is_jal (insn)) { +s32 imm =riscv_insn_extract_jtype_imm (insn); + +/* don't modify jumps inside the alternative block */ +if ((alt_ptr + i *sizeof (u32) + imm) >= alt_ptr && + (alt_ptr + i *sizeof (u32) + imm) < (alt_ptr + len)) +continue ; + +riscv_alternative_fix_jal (alt_ptr + i *sizeof (u32), + insn, patch_offset); + } } }
Signature · 7 lines
2.35.1 _______________________________________________ linux-riscv mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-riscv
Re: [PATCH v4 3/5] RISC-V: fix jal addresses in patched alternatives
Quoted 19 lines · Heiko Stuebner
From: Heiko Stuebner <[email protected]>
Alternatives live in a different section, so addresses used by jal instructions will point to wrong locations after the patch got applied.
Similar to arm64, adjust the location to consider that offset.
Signed-off-by Heiko Stuebner <[email protected]>--- arch/riscv/kernel/alternative.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+)arch/riscv/kernel/alternative.c +0
-0@@ -79,6 +79,21 @@ static void riscv_alternative_fix_auipc_jalr(void *ptr, u32 auipc_insn,patch_text_nosync (ptr, call,sizeof (u32) *2 ); }
arch/riscv/kernel/alternative.c static void riscv_alternative_fix_auipc_jalr(void *ptr, u32 auipc_insn, · line 90
Quoted 10 lines
arch/riscv/kernel/alternative.c +9
-0+static void riscv_alternative_fix_jal (void *ptr,u32 jal_insn,int patch_offset) +{ +s32 imm; + +/* get and adjust new target address */ + imm =riscv_insn_extract_jtype_imm (jal_insn); + imm -= patch_offset; + +/* update instructions */
instruction
arch/riscv/kernel/alternative.c void riscv_alternative_fix_offsets(void *alt_ptr, unsigned int len, · line 122
Quoted 12 lines
arch/riscv/kernel/alternative.c +6
-0+riscv_insn_insert_jtype_imm (&jal_insn, imm); + +/* patch the call place again */ +patch_text_nosync (ptr, &jal_insn,sizeof (u32)); +} +void riscv_alternative_fix_offsets (void *alt_ptr,unsigned int len,int patch_offset) {@@ -106,6 +121,18 @@ void riscv_alternative_fix_offsets(void *alt_ptr, unsigned int len,riscv_alternative_fix_auipc_jalr (alt_ptr + i *sizeof (u32), insn, insn2, patch_offset);
I think we should add an i++ here. There's not a problem now, since we're only adding a fixup for jal, not jalr, but we should future-proof this and there's no reason to revisit an already fixed-up instruction anyway.
arch/riscv/kernel/alternative.c void riscv_alternative_fix_offsets(void *alt_ptr, unsigned int len, · line 128
Quoted 6 lines
arch/riscv/kernel/alternative.c +5
-0} + +if (riscv_insn_is_jal (insn)) { +s32 imm =riscv_insn_extract_jtype_imm (insn); + +/* don't modify jumps inside the alternative block */
Don't
Quoted 13 lines
arch/riscv/kernel/alternative.c +7
-0+if ((alt_ptr + i *sizeof (u32) + imm) >= alt_ptr && + (alt_ptr + i *sizeof (u32) + imm) < (alt_ptr + len)) +continue ; + +riscv_alternative_fix_jal (alt_ptr + i *sizeof (u32), + insn, patch_offset); + } } }Signature · 1 line
2.35.1
Otherwise
Thanks,
drew
_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv
Re: [PATCH v4 3/5] RISC-V: fix jal addresses in patched alternatives
Quoted 33 lines · Andrew Jones
Quoted 19 lines · Heiko Stuebner
From: Heiko Stuebner <[email protected]>
Alternatives live in a different section, so addresses used by jal instructions will point to wrong locations after the patch got applied.
Similar to arm64, adjust the location to consider that offset.
Signed-off-by Heiko Stuebner <[email protected]>--- arch/riscv/kernel/alternative.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+)arch/riscv/kernel/alternative.c +0
-0@@ -79,6 +79,21 @@ static void riscv_alternative_fix_auipc_jalr(void *ptr, u32 auipc_insn,patch_text_nosync (ptr, call,sizeof (u32) *2 ); }arch/riscv/kernel/alternative.c static void riscv_alternative_fix_auipc_jalr(void *ptr, u32 auipc_insn, · line 90
Quoted 10 lines
arch/riscv/kernel/alternative.c +9
-0+static void riscv_alternative_fix_jal (void *ptr,u32 jal_insn,int patch_offset) +{ +s32 imm; + +/* get and adjust new target address */ + imm =riscv_insn_extract_jtype_imm (jal_insn); + imm -= patch_offset; + +/* update instructions */ instruction
Quoted 16 lines
arch/riscv/kernel/alternative.c void riscv_alternative_fix_offsets(void *alt_ptr, unsigned int len, · line 122
Quoted 12 lines
arch/riscv/kernel/alternative.c +6
-0+riscv_insn_insert_jtype_imm (&jal_insn, imm); + +/* patch the call place again */ +patch_text_nosync (ptr, &jal_insn,sizeof (u32)); +} +void riscv_alternative_fix_offsets (void *alt_ptr,unsigned int len,int patch_offset) {@@ -106,6 +121,18 @@ void riscv_alternative_fix_offsets(void *alt_ptr, unsigned int len,riscv_alternative_fix_auipc_jalr (alt_ptr + i *sizeof (u32), insn, insn2, patch_offset);I think we should add an i++ here. There's not a problem now, since we're only adding a fixup for jal, not jalr, but we should future-proof this and there's no reason to revisit an already fixed-up instruction anyway.
Hi Andrew,
IMHO, adding the i++ here belongs to the aupic jalr imm patching commit, since the patch has been merged to riscv-next, I think it's better to add a seperate patch to do this improvement, thus I didn't update as commented here in my v3 series.
Thanks
Quoted 26 lines
arch/riscv/kernel/alternative.c void riscv_alternative_fix_offsets(void *alt_ptr, unsigned int len, · line 128
Quoted 7 lines
arch/riscv/kernel/alternative.c +5
-0} + +if (riscv_insn_is_jal (insn)) { +s32 imm =riscv_insn_extract_jtype_imm (insn); + +/* don't modify jumps inside the alternative block */ Don't
Quoted 13 lines
arch/riscv/kernel/alternative.c +7
-0+if ((alt_ptr + i *sizeof (u32) + imm) >= alt_ptr && + (alt_ptr + i *sizeof (u32) + imm) < (alt_ptr + len)) +continue ; + +riscv_alternative_fix_jal (alt_ptr + i *sizeof (u32), + insn, patch_offset); + } } }Signature · 1 line
2.35.1Otherwise
Quoted 4 lines
Reviewed-by Andrew Jones <[email protected]>Thanks,
drew
_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv
Re: [PATCH v4 3/5] RISC-V: fix jal addresses in patched alternatives
In reply to Heiko StuebnerQuoted 6 lines · Heiko Stuebner
From: Heiko Stuebner <[email protected]>
Alternatives live in a different section, so addresses used by jal instructions will point to wrong locations after the patch got applied.
Similar to arm64, adjust the location to consider that offset.
I already submitted this in a month ago.
https://lore.kernel.org/linux-riscv/[email protected]/
Quoted 54 lines
Signed-off-by Heiko Stuebner <[email protected]>--- arch/riscv/kernel/alternative.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+)arch/riscv/kernel/alternative.c +27
-0@@ -79,6 +79,21 @@ static void riscv_alternative_fix_auipc_jalr(void *ptr, u32 auipc_insn,patch_text_nosync (ptr, call,sizeof (u32) *2 ); } +static void riscv_alternative_fix_jal (void *ptr,u32 jal_insn,int patch_offset) +{ +s32 imm; + +/* get and adjust new target address */ + imm =riscv_insn_extract_jtype_imm (jal_insn); + imm -= patch_offset; + +/* update instructions */ +riscv_insn_insert_jtype_imm (&jal_insn, imm); + +/* patch the call place again */ +patch_text_nosync (ptr, &jal_insn,sizeof (u32)); +} +void riscv_alternative_fix_offsets (void *alt_ptr,unsigned int len,int patch_offset) {@@ -106,6 +121,18 @@ void riscv_alternative_fix_offsets(void *alt_ptr, unsigned int len,riscv_alternative_fix_auipc_jalr (alt_ptr + i *sizeof (u32), insn, insn2, patch_offset); } + +if (riscv_insn_is_jal (insn)) { +s32 imm =riscv_insn_extract_jtype_imm (insn); + +/* don't modify jumps inside the alternative block */ +if ((alt_ptr + i *sizeof (u32) + imm) >= alt_ptr && + (alt_ptr + i *sizeof (u32) + imm) < (alt_ptr + len)) +continue ; + +riscv_alternative_fix_jal (alt_ptr + i *sizeof (u32), + insn, patch_offset); + } } }Signature · 1 line
2.35.1
_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv
Re: [PATCH v4 3/5] RISC-V: fix jal addresses in patched alternatives
Hi,
Am Mittwoch, 11. Januar 2023, 14:18:14 CET schrieb Jisheng Zhang:
Quoted 10 lines · Heiko Stuebner
Quoted 6 lines
From: Heiko Stuebner <[email protected]>
Alternatives live in a different section, so addresses used by jal instructions will point to wrong locations after the patch got applied.
Similar to arm64, adjust the location to consider that offset.
I already submitted this in a month ago.
https://lore.kernel.org/linux-riscv/[email protected]/
Yep, though the now merged base series for the call fixups changed a lot since that. With changes needed as we discussed in your series.
As the change for the jal fixer itself was this small and I needed it for my work, I did just go ahead and implemented it so I could move forward.
I'm not that attached to the authorship, so if you want we can set you as author or Co-Developed-by or so that is just fine.
Heiko
Quoted 43 lines
Quoted 43 lines
Signed-off-by Heiko Stuebner <[email protected]>--- arch/riscv/kernel/alternative.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+)arch/riscv/kernel/alternative.c +22
-0@@ -79,6 +79,21 @@ static void riscv_alternative_fix_auipc_jalr(void *ptr, u32 auipc_insn,patch_text_nosync (ptr, call,sizeof (u32) *2 ); } +static void riscv_alternative_fix_jal (void *ptr,u32 jal_insn,int patch_offset) +{ +s32 imm; + +/* get and adjust new target address */ + imm =riscv_insn_extract_jtype_imm (jal_insn); + imm -= patch_offset; + +/* update instructions */ +riscv_insn_insert_jtype_imm (&jal_insn, imm); + +/* patch the call place again */ +patch_text_nosync (ptr, &jal_insn,sizeof (u32)); +} +void riscv_alternative_fix_offsets (void *alt_ptr,unsigned int len,int patch_offset) {@@ -106,6 +121,18 @@ void riscv_alternative_fix_offsets(void *alt_ptr, unsigned int len,riscv_alternative_fix_auipc_jalr (alt_ptr + i *sizeof (u32), insn, insn2, patch_offset); } + +if (riscv_insn_is_jal (insn)) { +s32 imm =riscv_insn_extract_jtype_imm (insn); + +/* don't modify jumps inside the alternative block */ +if ((alt_ptr + i *sizeof (u32) + imm) >= alt_ptr && + (alt_ptr + i *sizeof (u32) + imm) < (alt_ptr + len))
Quoted 9 lines
arch/riscv/kernel/alternative.c void riscv_alternative_fix_offsets(void *alt_ptr, unsigned int len, · line 138
Quoted 8 lines
arch/riscv/kernel/alternative.c +5
-0+continue ; + +riscv_alternative_fix_jal (alt_ptr + i *sizeof (u32), + insn, patch_offset); + } } }
_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv
Re: [PATCH v4 3/5] RISC-V: fix jal addresses in patched alternatives
In reply to Jisheng ZhangQuoted 11 lines · Jisheng Zhang
Quoted 6 lines · Heiko Stuebner
From: Heiko Stuebner <[email protected]>
Alternatives live in a different section, so addresses used by jal instructions will point to wrong locations after the patch got applied.
Similar to arm64, adjust the location to consider that offset.
I already submitted this in a month ago.
https://lore.kernel.org/linux-riscv/[email protected]/
Hi Jisheng,
It'd be great to see a v3 of your series on the list, since I'd like to base zicboz work on it.
Thanks,
drew
_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv
Re: [PATCH v4 3/5] RISC-V: fix jal addresses in patched alternatives
Quoted 9 lines · Andrew Jones
Quoted 8 lines · Jisheng Zhang
Quoted 6 lines · Heiko Stuebner
From: Heiko Stuebner <[email protected]>
Alternatives live in a different section, so addresses used by jal instructions will point to wrong locations after the patch got applied.
Similar to arm64, adjust the location to consider that offset.
Quoted 8 lines
Quoted 3 lines
I already submitted this in a month ago.
https://lore.kernel.org/linux-riscv/[email protected]/Hi Jisheng,
It'd be great to see a v3 of your series on the list, since I'd like to base zicboz work on it.
Hi Andrew,
Sorry for being late. I'm cooking the patches, will send out today.
Best Regards
_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv
PATCH v4 4/5 RISC-V: add infrastructure to allow different str* implementations
In reply to Heiko StuebnerFrom: Heiko Stuebner <[email protected]>
Depending on supported extensions on specific RISC-V cores, optimized str* functions might make sense.
This adds basic infrastructure to allow patching the function calls via alternatives later on.
The Linux kernel provides standard implementations for string functions but when architectures want to extend them, they need to provide their own.
The added generic string functions are done in assembler (taken from disassembling the main-kernel functions for now) to allow us to control the used registers and extend them with optimized variants.
--- arch/riscv/include/asm/string.h | 10 +++++++++ arch/riscv/kernel/riscv_ksyms.c | 3 +++ arch/riscv/lib/Makefile | 3 +++ arch/riscv/lib/strcmp.S | 37 ++++++++++++++++++++++++++++++ arch/riscv/lib/strlen.S | 28 +++++++++++++++++++++++ arch/riscv/lib/strncmp.S | 40 +++++++++++++++++++++++++++++++++ arch/riscv/purgatory/Makefile | 13 +++++++++++ 7 files changed, 134 insertions(+) create mode 100644 arch/riscv/lib/strcmp.S create mode 100644 arch/riscv/lib/strlen.S create mode 100644 arch/riscv/lib/strncmp.S
arch/riscv/include/asm/string.h +10 -0
@@ -18,6 +18,16 @@ extern asmlinkage void *__memcpy(void *, const void *, size_t);#define __HAVE_ARCH_MEMMOVE extern asmlinkage void *memmove(void *, const void *, size_t); extern asmlinkage void *__memmove(void *, const void *, size_t); + +#define __HAVE_ARCH_STRCMP +extern asmlinkage int strcmp(const char *cs, const char *ct); + +#define __HAVE_ARCH_STRLEN +extern asmlinkage __kernel_size_t strlen(const char *); + +#define __HAVE_ARCH_STRNCMP +extern asmlinkage int strncmp(const char *cs, const char *ct, size_t count); +/* For those files which don't want to check by kasan. */ #if defined(CONFIG_KASAN ) && !defined(__SANITIZE_ADDRESS__)#define memcpy (dst, src, len) __memcpy(dst, src, len)
arch/riscv/kernel/riscv_ksyms.c +3 -0
@@ -12,6 +12,9 @@EXPORT_SYMBOL (memset);EXPORT_SYMBOL (memcpy);EXPORT_SYMBOL (memmove); +EXPORT_SYMBOL (strcmp); +EXPORT_SYMBOL (strlen); +EXPORT_SYMBOL (strncmp);EXPORT_SYMBOL (__memset);EXPORT_SYMBOL (__memcpy);EXPORT_SYMBOL (__memmove);
arch/riscv/lib/Makefile +3 -0
@@ -3,6 +3,9 @@ lib-y += delay.o lib-y += memcpy.o lib-y += memset.o lib-y += memmove.o +lib-y += strcmp.o +lib-y += strlen.o +lib-y += strncmp.o lib-$(CONFIG_MMU) += uaccess.o lib-$(CONFIG_64BIT) += tishift.o
arch/riscv/lib/strcmp.S new file +37 -0
@@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include <linux/linkage.h> +#include <asm/asm.h> +#include <asm-generic/export.h> + +/* int strcmp(const char *cs, const char *ct) */ +SYM_FUNC_START(strcmp) + /* + * Returns + * a0 - comparison result, value like strcmp + * + * Parameters + * a0 - string1 + * a1 - string2 + * + * Clobbers + * t0, t1, t2 + */ + mv t2, a1 +1: + lbu t1, 0(a0) + lbu t0, 0(a1) + addi a0, a0, 1 + addi a1, a1, 1 + beq t1, t0, 3f + li a0, 1 + bgeu t1, t0, 2f + li a0, -1 +2: + mv a1, t2 + ret +3: + bnez t1, 1b + li a0, 0 + j 2b +SYM_FUNC_END(strcmp)
arch/riscv/lib/strlen.S new file +28 -0
@@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include <linux/linkage.h> +#include <asm/asm.h> +#include <asm-generic/export.h> + +/* int strlen(const char *s) */ +SYM_FUNC_START(strlen) + /* + * Returns + * a0 - string length + * + * Parameters + * a0 - String to measure + * + * Clobbers: + * t0, t1 + */ + mv t1, a0 +1: + lbu t0, 0(t1) + bnez t0, 2f + sub a0, t1, a0 + ret +2: + addi t1, t1, 1 + j 1b +SYM_FUNC_END(strlen)
arch/riscv/lib/strncmp.S new file +40 -0
@@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include <linux/linkage.h> +#include <asm/asm.h> +#include <asm-generic/export.h> + +/* int strncmp(const char *cs, const char *ct, size_t count) */ +SYM_FUNC_START(strncmp) + /* + * Returns + * a0 - comparison result, value like strncmp + * + * Parameters + * a0 - string1 + * a1 - string2 + * a2 - number of characters to compare + * + * Clobbers + * t0, t1, t2 + */ + li t0, 0 +1: + beq a2, t0, 4f + add t1, a0, t0 + add t2, a1, t0 + lbu t1, 0(t1) + lbu t2, 0(t2) + beq t1, t2, 3f + li a0, 1 + bgeu t1, t2, 2f + li a0, -1 +2: + ret +3: + addi t0, t0, 1 + bnez t1, 1b +4: + li a0, 0 + j 2b +SYM_FUNC_END(strncmp)
arch/riscv/purgatory/Makefile +13 -0
@@ -2,6 +2,7 @@ OBJECT_FILES_NON_STANDARD := y purgatory-y := purgatory.o sha256.o entry.o string.o ctype.o memcpy.o memset.o +purgatory-y += strcmp.o strlen.o strncmp.o targets += $(purgatory-y) PURGATORY_OBJS = $(addprefix $(obj)/,$(purgatory-y))
@@ -18,6 +19,15 @@ $(obj)/memcpy.o: $(srctree)/arch/riscv/lib/memcpy.S FORCE $(obj)/memset.o: $(srctree)/arch/riscv/lib/memset.S FORCE $(call if_changed_rule,as_o_S) +$(obj)/strcmp.o: $(srctree)/arch/riscv/lib/strcmp.S FORCE + $(call if_changed_rule,as_o_S) + +$(obj)/strlen.o: $(srctree)/arch/riscv/lib/strlen.S FORCE + $(call if_changed_rule,as_o_S) + +$(obj)/strncmp.o: $(srctree)/arch/riscv/lib/strncmp.S FORCE + $(call if_changed_rule,as_o_S) + $(obj)/sha256.o: $(srctree)/lib/crypto/sha256.c FORCE $(call if_changed_rule,cc_o_c)
@@ -77,6 +87,9 @@ CFLAGS_ctype.o += $(PURGATORY_CFLAGS) AFLAGS_REMOVE_entry.o += -Wa,-gdwarf-2 AFLAGS_REMOVE_memcpy.o += -Wa,-gdwarf-2 AFLAGS_REMOVE_memset.o += -Wa,-gdwarf-2 +AFLAGS_REMOVE_strcmp.o += -Wa,-gdwarf-2 +AFLAGS_REMOVE_strlen.o += -Wa,-gdwarf-2 +AFLAGS_REMOVE_strncmp.o += -Wa,-gdwarf-2 $(obj)/purgatory.ro: $(PURGATORY_OBJS) FORCE $(call if_changed,ld)
Signature · 7 lines
2.35.1 _______________________________________________ linux-riscv mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-riscv
Re: [PATCH v4 4/5] RISC-V: add infrastructure to allow different str* implementations
Quoted 12 lines · Heiko Stuebner
From: Heiko Stuebner <[email protected]>
Depending on supported extensions on specific RISC-V cores, optimized str* functions might make sense.
This adds basic infrastructure to allow patching the function calls via alternatives later on.
The Linux kernel provides standard implementations for string functions but when architectures want to extend them, they need to provide their own.
Quoted 5 lines
The added generic string functions are done in assembler (taken from disassembling the main-kernel functions for now) to allow us to control the used registers and extend them with optimized variants.
Signed-off-by Heiko Stuebner <[email protected]>
asm looks the same as what I previously provided a review for, but I assume you dropped cos of the EFI stub stuff? You can have it back I suppose..
Quoted 226 lines
--- arch/riscv/include/asm/string.h | 10 +++++++++ arch/riscv/kernel/riscv_ksyms.c | 3 +++ arch/riscv/lib/Makefile | 3 +++ arch/riscv/lib/strcmp.S | 37 ++++++++++++++++++++++++++++++ arch/riscv/lib/strlen.S | 28 +++++++++++++++++++++++ arch/riscv/lib/strncmp.S | 40 +++++++++++++++++++++++++++++++++ arch/riscv/purgatory/Makefile | 13 +++++++++++ 7 files changed, 134 insertions(+) create mode 100644 arch/riscv/lib/strcmp.S create mode 100644 arch/riscv/lib/strlen.S create mode 100644 arch/riscv/lib/strncmp.Sarch/riscv/include/asm/string.h +10
-0@@ -18,6 +18,16 @@ extern asmlinkage void *__memcpy(void *, const void *, size_t);#define __HAVE_ARCH_MEMMOVE extern asmlinkage void *memmove(void *, const void *, size_t); extern asmlinkage void *__memmove(void *, const void *, size_t); + +#define __HAVE_ARCH_STRCMP +extern asmlinkage int strcmp(const char *cs, const char *ct); + +#define __HAVE_ARCH_STRLEN +extern asmlinkage __kernel_size_t strlen(const char *); + +#define __HAVE_ARCH_STRNCMP +extern asmlinkage int strncmp(const char *cs, const char *ct, size_t count); +/* For those files which don't want to check by kasan. */ #if defined(CONFIG_KASAN ) && !defined(__SANITIZE_ADDRESS__)#define memcpy (dst, src, len) __memcpy(dst, src, len)arch/riscv/kernel/riscv_ksyms.c +3
-0@@ -12,6 +12,9 @@EXPORT_SYMBOL (memset);EXPORT_SYMBOL (memcpy);EXPORT_SYMBOL (memmove); +EXPORT_SYMBOL (strcmp); +EXPORT_SYMBOL (strlen); +EXPORT_SYMBOL (strncmp);EXPORT_SYMBOL (__memset);EXPORT_SYMBOL (__memcpy);EXPORT_SYMBOL (__memmove);arch/riscv/lib/Makefile +3
-0@@ -3,6 +3,9 @@ lib-y += delay.o lib-y += memcpy.o lib-y += memset.o lib-y += memmove.o +lib-y += strcmp.o +lib-y += strlen.o +lib-y += strncmp.o lib-$(CONFIG_MMU) += uaccess.o lib-$(CONFIG_64BIT) += tishift.oarch/riscv/lib/strcmp.S new file +37
-0@@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include <linux/linkage.h> +#include <asm/asm.h> +#include <asm-generic/export.h> + +/* int strcmp(const char *cs, const char *ct) */ +SYM_FUNC_START(strcmp) + /* + * Returns + * a0 - comparison result, value like strcmp + * + * Parameters + * a0 - string1 + * a1 - string2 + * + * Clobbers + * t0, t1, t2 + */ + mv t2, a1 +1: + lbu t1, 0(a0) + lbu t0, 0(a1) + addi a0, a0, 1 + addi a1, a1, 1 + beq t1, t0, 3f + li a0, 1 + bgeu t1, t0, 2f + li a0, -1 +2: + mv a1, t2 + ret +3: + bnez t1, 1b + li a0, 0 + j 2b +SYM_FUNC_END(strcmp)arch/riscv/lib/strlen.S new file +28
-0@@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include <linux/linkage.h> +#include <asm/asm.h> +#include <asm-generic/export.h> + +/* int strlen(const char *s) */ +SYM_FUNC_START(strlen) + /* + * Returns + * a0 - string length + * + * Parameters + * a0 - String to measure + * + * Clobbers: + * t0, t1 + */ + mv t1, a0 +1: + lbu t0, 0(t1) + bnez t0, 2f + sub a0, t1, a0 + ret +2: + addi t1, t1, 1 + j 1b +SYM_FUNC_END(strlen)arch/riscv/lib/strncmp.S new file +40
-0@@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include <linux/linkage.h> +#include <asm/asm.h> +#include <asm-generic/export.h> + +/* int strncmp(const char *cs, const char *ct, size_t count) */ +SYM_FUNC_START(strncmp) + /* + * Returns + * a0 - comparison result, value like strncmp + * + * Parameters + * a0 - string1 + * a1 - string2 + * a2 - number of characters to compare + * + * Clobbers + * t0, t1, t2 + */ + li t0, 0 +1: + beq a2, t0, 4f + add t1, a0, t0 + add t2, a1, t0 + lbu t1, 0(t1) + lbu t2, 0(t2) + beq t1, t2, 3f + li a0, 1 + bgeu t1, t2, 2f + li a0, -1 +2: + ret +3: + addi t0, t0, 1 + bnez t1, 1b +4: + li a0, 0 + j 2b +SYM_FUNC_END(strncmp)arch/riscv/purgatory/Makefile +13
-0@@ -2,6 +2,7 @@ OBJECT_FILES_NON_STANDARD := y purgatory-y := purgatory.o sha256.o entry.o string.o ctype.o memcpy.o memset.o +purgatory-y += strcmp.o strlen.o strncmp.o targets += $(purgatory-y) PURGATORY_OBJS = $(addprefix $(obj)/,$(purgatory-y))@@ -18,6 +19,15 @@ $(obj)/memcpy.o: $(srctree)/arch/riscv/lib/memcpy.S FORCE $(obj)/memset.o: $(srctree)/arch/riscv/lib/memset.S FORCE $(call if_changed_rule,as_o_S) +$(obj)/strcmp.o: $(srctree)/arch/riscv/lib/strcmp.S FORCE + $(call if_changed_rule,as_o_S) + +$(obj)/strlen.o: $(srctree)/arch/riscv/lib/strlen.S FORCE + $(call if_changed_rule,as_o_S) + +$(obj)/strncmp.o: $(srctree)/arch/riscv/lib/strncmp.S FORCE + $(call if_changed_rule,as_o_S) + $(obj)/sha256.o: $(srctree)/lib/crypto/sha256.c FORCE $(call if_changed_rule,cc_o_c)@@ -77,6 +87,9 @@ CFLAGS_ctype.o += $(PURGATORY_CFLAGS) AFLAGS_REMOVE_entry.o += -Wa,-gdwarf-2 AFLAGS_REMOVE_memcpy.o += -Wa,-gdwarf-2 AFLAGS_REMOVE_memset.o += -Wa,-gdwarf-2 +AFLAGS_REMOVE_strcmp.o += -Wa,-gdwarf-2 +AFLAGS_REMOVE_strlen.o += -Wa,-gdwarf-2 +AFLAGS_REMOVE_strncmp.o += -Wa,-gdwarf-2 $(obj)/purgatory.ro: $(PURGATORY_OBJS) FORCE $(call if_changed,ld)Signature · 1 line
2.35.1
_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv
Re: [PATCH v4 4/5] RISC-V: add infrastructure to allow different str* implementations
Am Montag, 9. Januar 2023, 23:37:45 CET schrieb Conor Dooley:
Quoted 17 lines · Heiko Stuebner
Quoted 16 lines
From: Heiko Stuebner <[email protected]>
Depending on supported extensions on specific RISC-V cores, optimized str* functions might make sense.
This adds basic infrastructure to allow patching the function calls via alternatives later on.
The Linux kernel provides standard implementations for string functions but when architectures want to extend them, they need to provide their own.
The added generic string functions are done in assembler (taken from disassembling the main-kernel functions for now) to allow us to control the used registers and extend them with optimized variants.
Quoted 6 lines
Quoted 1 line
Signed-off-by Heiko Stuebner <[email protected]>asm looks the same as what I previously provided a review for, but I assume you dropped cos of the EFI stub stuff? You can have it back I suppose..
Reviewed-by Conor Dooley <[email protected]>
That was the
| Technically patch4 got a review from Andrew, but that was still with | the inline approach, so I didn't bring it over to v4.
part in the cover-letter, thanks for bringing the rb back :-)
Heiko
Quoted 221 lines
Quoted 221 lines
--- arch/riscv/include/asm/string.h | 10 +++++++++ arch/riscv/kernel/riscv_ksyms.c | 3 +++ arch/riscv/lib/Makefile | 3 +++ arch/riscv/lib/strcmp.S | 37 ++++++++++++++++++++++++++++++ arch/riscv/lib/strlen.S | 28 +++++++++++++++++++++++ arch/riscv/lib/strncmp.S | 40 +++++++++++++++++++++++++++++++++ arch/riscv/purgatory/Makefile | 13 +++++++++++ 7 files changed, 134 insertions(+) create mode 100644 arch/riscv/lib/strcmp.S create mode 100644 arch/riscv/lib/strlen.S create mode 100644 arch/riscv/lib/strncmp.Sarch/riscv/include/asm/string.h +10
-0@@ -18,6 +18,16 @@ extern asmlinkage void *__memcpy(void *, const void *, size_t);#define __HAVE_ARCH_MEMMOVE extern asmlinkage void *memmove(void *, const void *, size_t); extern asmlinkage void *__memmove(void *, const void *, size_t); + +#define __HAVE_ARCH_STRCMP +extern asmlinkage int strcmp(const char *cs, const char *ct); + +#define __HAVE_ARCH_STRLEN +extern asmlinkage __kernel_size_t strlen(const char *); + +#define __HAVE_ARCH_STRNCMP +extern asmlinkage int strncmp(const char *cs, const char *ct, size_t count); +/* For those files which don't want to check by kasan. */ #if defined(CONFIG_KASAN ) && !defined(__SANITIZE_ADDRESS__)#define memcpy (dst, src, len) __memcpy(dst, src, len)arch/riscv/kernel/riscv_ksyms.c +3
-0@@ -12,6 +12,9 @@EXPORT_SYMBOL (memset);EXPORT_SYMBOL (memcpy);EXPORT_SYMBOL (memmove); +EXPORT_SYMBOL (strcmp); +EXPORT_SYMBOL (strlen); +EXPORT_SYMBOL (strncmp);EXPORT_SYMBOL (__memset);EXPORT_SYMBOL (__memcpy);EXPORT_SYMBOL (__memmove);arch/riscv/lib/Makefile +3
-0@@ -3,6 +3,9 @@ lib-y += delay.o lib-y += memcpy.o lib-y += memset.o lib-y += memmove.o +lib-y += strcmp.o +lib-y += strlen.o +lib-y += strncmp.o lib-$(CONFIG_MMU) += uaccess.o lib-$(CONFIG_64BIT) += tishift.oarch/riscv/lib/strcmp.S new file +37
-0@@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include <linux/linkage.h> +#include <asm/asm.h> +#include <asm-generic/export.h> + +/* int strcmp(const char *cs, const char *ct) */ +SYM_FUNC_START(strcmp) + /* + * Returns + * a0 - comparison result, value like strcmp + * + * Parameters + * a0 - string1 + * a1 - string2 + * + * Clobbers + * t0, t1, t2 + */ + mv t2, a1 +1: + lbu t1, 0(a0) + lbu t0, 0(a1) + addi a0, a0, 1 + addi a1, a1, 1 + beq t1, t0, 3f + li a0, 1 + bgeu t1, t0, 2f + li a0, -1 +2: + mv a1, t2 + ret +3: + bnez t1, 1b + li a0, 0 + j 2b +SYM_FUNC_END(strcmp)arch/riscv/lib/strlen.S new file +28
-0@@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include <linux/linkage.h> +#include <asm/asm.h> +#include <asm-generic/export.h> + +/* int strlen(const char *s) */ +SYM_FUNC_START(strlen) + /* + * Returns + * a0 - string length + * + * Parameters + * a0 - String to measure + * + * Clobbers: + * t0, t1 + */ + mv t1, a0 +1: + lbu t0, 0(t1) + bnez t0, 2f + sub a0, t1, a0 + ret +2: + addi t1, t1, 1 + j 1b +SYM_FUNC_END(strlen)arch/riscv/lib/strncmp.S new file +40
-0@@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include <linux/linkage.h> +#include <asm/asm.h> +#include <asm-generic/export.h> + +/* int strncmp(const char *cs, const char *ct, size_t count) */ +SYM_FUNC_START(strncmp) + /* + * Returns + * a0 - comparison result, value like strncmp + * + * Parameters + * a0 - string1 + * a1 - string2 + * a2 - number of characters to compare + * + * Clobbers + * t0, t1, t2 + */ + li t0, 0 +1: + beq a2, t0, 4f + add t1, a0, t0 + add t2, a1, t0 + lbu t1, 0(t1) + lbu t2, 0(t2) + beq t1, t2, 3f + li a0, 1 + bgeu t1, t2, 2f + li a0, -1 +2: + ret +3: + addi t0, t0, 1 + bnez t1, 1b +4: + li a0, 0 + j 2b +SYM_FUNC_END(strncmp)arch/riscv/purgatory/Makefile +13
-0@@ -2,6 +2,7 @@ OBJECT_FILES_NON_STANDARD := y purgatory-y := purgatory.o sha256.o entry.o string.o ctype.o memcpy.o memset.o +purgatory-y += strcmp.o strlen.o strncmp.o targets += $(purgatory-y) PURGATORY_OBJS = $(addprefix $(obj)/,$(purgatory-y))@@ -18,6 +19,15 @@ $(obj)/memcpy.o: $(srctree)/arch/riscv/lib/memcpy.S FORCE $(obj)/memset.o: $(srctree)/arch/riscv/lib/memset.S FORCE $(call if_changed_rule,as_o_S) +$(obj)/strcmp.o: $(srctree)/arch/riscv/lib/strcmp.S FORCE + $(call if_changed_rule,as_o_S) + +$(obj)/strlen.o: $(srctree)/arch/riscv/lib/strlen.S FORCE + $(call if_changed_rule,as_o_S) + +$(obj)/strncmp.o: $(srctree)/arch/riscv/lib/strncmp.S FORCE + $(call if_changed_rule,as_o_S) + $(obj)/sha256.o: $(srctree)/lib/crypto/sha256.c FORCE $(call if_changed_rule,cc_o_c)@@ -77,6 +87,9 @@ CFLAGS_ctype.o += $(PURGATORY_CFLAGS) AFLAGS_REMOVE_entry.o += -Wa,-gdwarf-2 AFLAGS_REMOVE_memcpy.o += -Wa,-gdwarf-2 AFLAGS_REMOVE_memset.o += -Wa,-gdwarf-2 +AFLAGS_REMOVE_strcmp.o += -Wa,-gdwarf-2 +AFLAGS_REMOVE_strlen.o += -Wa,-gdwarf-2 +AFLAGS_REMOVE_strncmp.o += -Wa,-gdwarf-2
Quoted 3 lines
arch/riscv/purgatory/Makefile CFLAGS_ctype.o += $(PURGATORY_CFLAGS) · line 95
Quoted 2 lines
arch/riscv/purgatory/Makefile +0
-0$(obj)/purgatory.ro: $(PURGATORY_OBJS) FORCE $(call if_changed,ld)
_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv
Re: [PATCH v4 4/5] RISC-V: add infrastructure to allow different str* implementations
In reply to Heiko StuebnerQuoted 11 lines · Heiko Stuebner
From: Heiko Stuebner <[email protected]>
Depending on supported extensions on specific RISC-V cores, optimized str* functions might make sense.
This adds basic infrastructure to allow patching the function calls via alternatives later on.
The Linux kernel provides standard implementations for string functions but when architectures want to extend them, they need to provide their own.
And the compiler provides builtins. In the previous series it appeared to be a bad idea to compile the kernel with the compiler's builtins disabled. How will the optimized string functions which will be based on this patch be selected?
Thanks,
drew
_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv
Re: [PATCH v4 4/5] RISC-V: add infrastructure to allow different str* implementations
Hi Andrew,
Am Dienstag, 10. Januar 2023, 10:39:36 CET schrieb Andrew Jones:
Quoted 9 lines · Heiko Stuebner
Quoted 8 lines
From: Heiko Stuebner <[email protected]>
Depending on supported extensions on specific RISC-V cores, optimized str* functions might make sense.
This adds basic infrastructure to allow patching the function calls via alternatives later on.
Quoted 8 lines
Quoted 3 lines
The Linux kernel provides standard implementations for string functions but when architectures want to extend them, they need to provide their own.
And the compiler provides builtins. In the previous series it appeared to be a bad idea to compile the kernel with the compiler's builtins disabled. How will the optimized string functions which will be based on this patch be selected?
yep, the consensus seemingly was that the compiler knows best when to use builtins for some cases (which is probably correct), hence the move away from the inline bases.
So I guess the first decision is the compiler's wether to use a builtin or the kernel string function (same as for mem*) .
In my tests, I did see both getting used - so it's definitly not lost work :-) .
After that when landing in these here, we want to select the best variant for the host-system the kernel runs on.
I.e. this one as baseline or for example using zbb as an "alternative".
As for the "more" variants, I currently have more patches on top, that then use an ALTERNATIVE_2
ALTERNATIVE_2("nop",
"j variant_zbb_unaligned", 0, CPUFEATURE_ZBB | CPUFEATURE_FAST_UNALIGNED, 0, CONFIG_RISCV_ISA_ZBB, "j variant_zbb", 0, CPUFEATURE_ZBB, CPUFEATURE_FAST_UNALIGNED, CONFIG_RISCV_ISA_ZBB)
with the "errata_id" being used as a bitfield to use extension combinations. And a "not"-field, so I can do a has-zbb + has-not-fast-unaligned
Heiko
_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv
Re: [PATCH v4 4/5] RISC-V: add infrastructure to allow different str* implementations
Quoted 38 lines · Heiko Stübner
Hi Andrew,
Am Dienstag, 10. Januar 2023, 10:39:36 CET schrieb Andrew Jones:
Quoted 9 lines · Heiko Stuebner
Quoted 8 lines
From: Heiko Stuebner <[email protected]>
Depending on supported extensions on specific RISC-V cores, optimized str* functions might make sense.
This adds basic infrastructure to allow patching the function calls via alternatives later on.
Quoted 8 lines
Quoted 3 lines
The Linux kernel provides standard implementations for string functions but when architectures want to extend them, they need to provide their own.
And the compiler provides builtins. In the previous series it appeared to be a bad idea to compile the kernel with the compiler's builtins disabled. How will the optimized string functions which will be based on this patch be selected?
yep, the consensus seemingly was that the compiler knows best when to use builtins for some cases (which is probably correct), hence the move away from the inline bases.
So I guess the first decision is the compiler's wether to use a builtin or the kernel string function (same as for mem*) .
In my tests, I did see both getting used - so it's definitly not lost work :-) .
After that when landing in these here, we want to select the best variant for the host-system the kernel runs on.
I.e. this one as baseline or for example using zbb as an "alternative".
As for the "more" variants, I currently have more patches on top, that then use an
ALTERNATIVE_2
Quoted 6 lines
ALTERNATIVE_2("nop",
"jvariant_zbb_unaligned", 0,CPUFEATURE_ZBB|CPUFEATURE_FAST_UNALIGNED, 0,CONFIG_RISCV_ISA_ZBB, "jvariant_zbb", 0,CPUFEATURE_ZBB,CPUFEATURE_FAST_UNALIGNED,CONFIG_RISCV_ISA_ZBB)with the "
errata_id" being used as a bitfield to use extension combinations. And a "not"-field, so I can do a has-zbb + has-not-fast-unaligned
Thanks, Heiko. This was the info I needed. It might be nice to put some of it in the commit message too.
drew
_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv
Re: [PATCH v4 4/5] RISC-V: add infrastructure to allow different str* implementations
Quoted 22 lines · Andrew Jones
Quoted 21 lines · Heiko Stübner
Hi Andrew,
Am Dienstag, 10. Januar 2023, 10:39:36 CET schrieb Andrew Jones:
Quoted 17 lines · Heiko Stuebner
Quoted 11 lines
From: Heiko Stuebner <[email protected]>
Depending on supported extensions on specific RISC-V cores, optimized str* functions might make sense.
This adds basic infrastructure to allow patching the function calls via alternatives later on.
The Linux kernel provides standard implementations for string functions but when architectures want to extend them, they need to provide their own.
And the compiler provides builtins. In the previous series it appeared to be a bad idea to compile the kernel with the compiler's builtins disabled. How will the optimized string functions which will be based on this patch be selected?
Quoted 6 lines
Quoted 6 lines
yep, the consensus seemingly was that the compiler knows best when to use builtins for some cases (which is probably correct), hence the move away from the inline bases.
So I guess the first decision is the compiler's wether to use a builtin or the kernel string function (same as for mem*) .
Hi Heiko,
Thinking about this some more, I'd still like to understand how/when the compiler makes this choice. Are compiler flags involved? Or do some heuristics dictate the choice?
Thanks,
drew
_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv
A message that is not in the archive (sent off-list or not yet received): CAEg0e7gJgpoiGjfLeedba0-r=dCE1Z_qkU53w_+-cVjsuqaC3A@mail.gmail.com
Re: [PATCH v4 4/5] RISC-V: add infrastructure to allow different str* implementations
Drew,
the reason why alignment is critical for correctness of optimisations in userspace applications is that exception semantics must be retained when crossing page boundaries.
Philipp.
Quoted 47 lines · Christoph Müllner
Quoted 28 lines · Andrew Jones
Quoted 26 lines · Andrew Jones
Quoted 25 lines · Heiko Stübner
Hi Andrew,
Am Dienstag, 10. Januar 2023, 10:39:36 CET schrieb Andrew Jones:
Quoted 9 lines · Heiko Stuebner
Quoted 8 lines
From: Heiko Stuebner <[email protected]>
Depending on supported extensions on specific RISC-V cores, optimized str* functions might make sense.
This adds basic infrastructure to allow patching the function calls via alternatives later on.
Quoted 8 lines
Quoted 3 lines
The Linux kernel provides standard implementations for string functions but when architectures want to extend them, they need to provide their own.
And the compiler provides builtins. In the previous series it appeared to be a bad idea to compile the kernel with the compiler's builtins disabled. How will the optimized string functions which will be based on this patch be selected?
yep, the consensus seemingly was that the compiler knows best when to use builtins for some cases (which is probably correct), hence the move away from the inline bases.
Quoted 8 lines
Quoted 2 lines
Quoted 2 lines
So I guess the first decision is the compiler's wether to use a builtin or the kernel string function (same as for mem*) .
Hi Heiko,
Thinking about this some more, I'd still like to understand how/when the compiler makes this choice. Are compiler flags involved? Or do some heuristics dictate the choice?
1) Arch-independent expansion is used e.g. for calculating the result for statically known strings.
2) Arch-specific expansion (not mainline, but my series can be found here: [1]) can do whatever is reasonable. My series emits a bounded unrolled comparison loop for str(n)cmp with a bound that can be adjusted via flag. But it only triggers if both strings are known to be aligned.
Quoted 8 lines
[1] https://gcc.gnu.org/pipermail/gcc-patches/2022-November/605993.html
BR
ChristophQuoted 3 lines
Thanks,
drew
_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv
Re: [PATCH v4 4/5] RISC-V: add infrastructure to allow different str* implementations
In reply to a message not in the archiveQuoted 53 lines · Christoph Müllner
Quoted 32 lines · Andrew Jones
Quoted 7 lines · Andrew Jones
Quoted 6 lines · Heiko Stübner
Hi Andrew,
Am Dienstag, 10. Januar 2023, 10:39:36 CET schrieb Andrew Jones:
Quoted 3 lines · Heiko Stuebner
Quoted 2 lines
From: Heiko Stuebner <[email protected]>
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Depending on supported extensions on specific RISC-V cores, optimized str* functions might make sense.
This adds basic infrastructure to allow patching the function calls via alternatives later on.
The Linux kernel provides standard implementations for string
functions
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
but when architectures want to extend them, they need to provide
their
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 1 line
own.
And the compiler provides builtins. In the previous series it
appeared
Quoted 6 lines
Quoted 6 lines
Quoted 3 lines
to be a bad idea to compile the kernel with the compiler's builtins disabled. How will the optimized string functions which will be based on this patch be selected?
yep, the consensus seemingly was that the compiler knows best when to use builtins for some cases (which is probably correct), hence the
move
Quoted 3 lines
Quoted 3 lines
away from the inline bases.
So I guess the first decision is the compiler's wether to use a
Quoted 9 lines
builtin or
Quoted 1 line
Quoted 1 line
the kernel string function (same as for mem*) .
Hi Heiko,
Thinking about this some more, I'd still like to understand how/when the compiler makes this choice. Are compiler flags involved? Or do some heuristics dictate the choice?
1) Arch-independent expansion is used e.g. for calculating the result for statically known strings.
2) Arch-specific expansion (not mainline, but my series can be found here:
[1]) can do whatever is reasonable.
My series emits a bounded unrolled comparison loop for str(n)cmp with a bound that can be adjusted via flag.
But it only triggers if both strings are known to be aligned.
Quoted 2 lines
[1] https://gcc.gnu.org/pipermail/gcc-patches/2022-November/605993.html
Thanks Christoph!
drew
_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv
Re: [PATCH v4 4/5] RISC-V: add infrastructure to allow different str* implementations
In reply to Heiko StuebnerQuoted 95 lines · Heiko Stuebner
From: Heiko Stuebner <[email protected]>
Depending on supported extensions on specific RISC-V cores, optimized str* functions might make sense.
This adds basic infrastructure to allow patching the function calls via alternatives later on.
The Linux kernel provides standard implementations for string functions but when architectures want to extend them, they need to provide their own.
The added generic string functions are done in assembler (taken from disassembling the main-kernel functions for now) to allow us to control the used registers and extend them with optimized variants.
Signed-off-by Heiko Stuebner <[email protected]>--- arch/riscv/include/asm/string.h | 10 +++++++++ arch/riscv/kernel/riscv_ksyms.c | 3 +++ arch/riscv/lib/Makefile | 3 +++ arch/riscv/lib/strcmp.S | 37 ++++++++++++++++++++++++++++++ arch/riscv/lib/strlen.S | 28 +++++++++++++++++++++++ arch/riscv/lib/strncmp.S | 40 +++++++++++++++++++++++++++++++++ arch/riscv/purgatory/Makefile | 13 +++++++++++ 7 files changed, 134 insertions(+) create mode 100644 arch/riscv/lib/strcmp.S create mode 100644 arch/riscv/lib/strlen.S create mode 100644 arch/riscv/lib/strncmp.Sarch/riscv/include/asm/string.h +10
-0@@ -18,6 +18,16 @@ extern asmlinkage void *__memcpy(void *, const void *, size_t);#define __HAVE_ARCH_MEMMOVE extern asmlinkage void *memmove(void *, const void *, size_t); extern asmlinkage void *__memmove(void *, const void *, size_t); + +#define __HAVE_ARCH_STRCMP +extern asmlinkage int strcmp(const char *cs, const char *ct); + +#define __HAVE_ARCH_STRLEN +extern asmlinkage __kernel_size_t strlen(const char *); + +#define __HAVE_ARCH_STRNCMP +extern asmlinkage int strncmp(const char *cs, const char *ct, size_t count); +/* For those files which don't want to check by kasan. */ #if defined(CONFIG_KASAN ) && !defined(__SANITIZE_ADDRESS__)#define memcpy (dst, src, len) __memcpy(dst, src, len)arch/riscv/kernel/riscv_ksyms.c +3
-0@@ -12,6 +12,9 @@EXPORT_SYMBOL (memset);EXPORT_SYMBOL (memcpy);EXPORT_SYMBOL (memmove); +EXPORT_SYMBOL (strcmp); +EXPORT_SYMBOL (strlen); +EXPORT_SYMBOL (strncmp);EXPORT_SYMBOL (__memset);EXPORT_SYMBOL (__memcpy);EXPORT_SYMBOL (__memmove);arch/riscv/lib/Makefile +3
-0@@ -3,6 +3,9 @@ lib-y += delay.o lib-y += memcpy.o lib-y += memset.o lib-y += memmove.o +lib-y += strcmp.o +lib-y += strlen.o +lib-y += strncmp.o lib-$(CONFIG_MMU) += uaccess.o lib-$(CONFIG_64BIT) += tishift.oarch/riscv/lib/strcmp.S new file +10
-0@@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include <linux/linkage.h> +#include <asm/asm.h> +#include <asm-generic/export.h> + +/* int strcmp(const char *cs, const char *ct) */ +SYM_FUNC_START(strcmp) + /* + * Returns
arch/riscv/lib/strcmp.S line 20
Quoted 10 lines
arch/riscv/lib/strcmp.S new file +10
-0+ * a0 - comparison result, value like strcmp + * + * Parameters + * a0 - string1 + * a1 - string2 + * + * Clobbers + * t0, t1, t2 + */ + mv t2, a1
The above instruction and the 'mv a1, t2' below appear to be attempting to preserve a1, but that shouldn't be necessary.
arch/riscv/lib/strcmp.S line 23
Quoted 3 lines
arch/riscv/lib/strcmp.S new file +3
-0+1: + lbu t1, 0(a0) + lbu t0, 0(a1)
I'd rather have t0 be 0(a0) and t1 be 0(a1)
arch/riscv/lib/strcmp.S line 36
Quoted 13 lines
arch/riscv/lib/strcmp.S new file +13
-0+ addi a0, a0, 1 + addi a1, a1, 1 + beq t1, t0, 3f + li a0, 1 + bgeu t1, t0, 2f + li a0, -1 +2: + mv a1, t2 + ret +3: + bnez t1, 1b + li a0, 0 + j 2b
For fun I removed one conditional and one unconditional branch (untested)
1:
lbu t0, 0(a0)
lbu t1, 0(a1)
addi a0, a0, 1
addi a1, a1, 1
bne t0, t1, 2f
bnez t0, 1b
li a0, 0
ret
2:
slt a1, t1, t0
slli a1, a1, 1
li a0, -1
add a0, a0, a1ret
Quoted 24 lines
arch/riscv/lib/strcmp.S new file +1
-0+SYM_FUNC_END(strcmp)arch/riscv/lib/strlen.S new file +17
-0@@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include <linux/linkage.h> +#include <asm/asm.h> +#include <asm-generic/export.h> + +/* int strlen(const char *s) */ +SYM_FUNC_START(strlen) + /* + * Returns + * a0 - string length + * + * Parameters + * a0 - String to measure + * + * Clobbers: + * t0, t1
arch/riscv/lib/strlen.S line 27
Quoted 10 lines
arch/riscv/lib/strlen.S new file +10
-0+ */ + mv t1, a0 +1: + lbu t0, 0(t1) + bnez t0, 2f + sub a0, t1, a0 + ret +2: + addi t1, t1, 1 + j 1b
Slightly reorganizing looks better (to me)
mv t1, a0
1:
lbu t0, 0(t1) beqz t0, 2f addi t1, t1, 1 j 1b
2:
sub a0, t1, a0
ret
Quoted 36 lines
arch/riscv/lib/strlen.S new file +1
-0+SYM_FUNC_END(strlen)arch/riscv/lib/strncmp.S new file +29
-0@@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include <linux/linkage.h> +#include <asm/asm.h> +#include <asm-generic/export.h> + +/* int strncmp(const char *cs, const char *ct, size_t count) */ +SYM_FUNC_START(strncmp) + /* + * Returns + * a0 - comparison result, value like strncmp + * + * Parameters + * a0 - string1 + * a1 - string2 + * a2 - number of characters to compare + * + * Clobbers + * t0, t1, t2 + */ + li t0, 0 +1: + beq a2, t0, 4f + add t1, a0, t0 + add t2, a1, t0 + lbu t1, 0(t1) + lbu t2, 0(t2) + beq t1, t2, 3f + li a0, 1
arch/riscv/lib/strncmp.S line 39
Quoted 10 lines
arch/riscv/lib/strncmp.S new file +10
-0+ bgeu t1, t2, 2f + li a0, -1 +2: + ret +3: + addi t0, t0, 1 + bnez t1, 1b +4: + li a0, 0 + j 2b
(untested)
li t2, 0
1:
beq a2, t2, 2f
lbu t0, 0(a0)
lbu t1, 0(a1)
addi a0, a0, 1
addi a1, a1, 1
bne t0, t1, 3f
addi t2, t2, 1
bnez t0, 1b2:
li a0, 0
ret
3:
slt a1, t1, t0
slli a1, a1, 1
li a0, -1
add a0, a0, a1ret
Quoted 42 lines
arch/riscv/lib/strncmp.S new file +1
-0+SYM_FUNC_END(strncmp)arch/riscv/purgatory/Makefile +13
-0@@ -2,6 +2,7 @@ OBJECT_FILES_NON_STANDARD := y purgatory-y := purgatory.o sha256.o entry.o string.o ctype.o memcpy.o memset.o +purgatory-y += strcmp.o strlen.o strncmp.o targets += $(purgatory-y) PURGATORY_OBJS = $(addprefix $(obj)/,$(purgatory-y))@@ -18,6 +19,15 @@ $(obj)/memcpy.o: $(srctree)/arch/riscv/lib/memcpy.S FORCE $(obj)/memset.o: $(srctree)/arch/riscv/lib/memset.S FORCE $(call if_changed_rule,as_o_S) +$(obj)/strcmp.o: $(srctree)/arch/riscv/lib/strcmp.S FORCE + $(call if_changed_rule,as_o_S) + +$(obj)/strlen.o: $(srctree)/arch/riscv/lib/strlen.S FORCE + $(call if_changed_rule,as_o_S) + +$(obj)/strncmp.o: $(srctree)/arch/riscv/lib/strncmp.S FORCE + $(call if_changed_rule,as_o_S) + $(obj)/sha256.o: $(srctree)/lib/crypto/sha256.c FORCE $(call if_changed_rule,cc_o_c)@@ -77,6 +87,9 @@ CFLAGS_ctype.o += $(PURGATORY_CFLAGS) AFLAGS_REMOVE_entry.o += -Wa,-gdwarf-2 AFLAGS_REMOVE_memcpy.o += -Wa,-gdwarf-2 AFLAGS_REMOVE_memset.o += -Wa,-gdwarf-2 +AFLAGS_REMOVE_strcmp.o += -Wa,-gdwarf-2 +AFLAGS_REMOVE_strlen.o += -Wa,-gdwarf-2 +AFLAGS_REMOVE_strncmp.o += -Wa,-gdwarf-2 $(obj)/purgatory.ro: $(PURGATORY_OBJS) FORCE $(call if_changed,ld)Signature · 1 line
2.35.1
With at least the removal of the unnecessary preserving of a1 in strcmp, then it looks correct to me, so
But I think there's room for making it more readable, and maybe even optimized, as I've tried to do.
Thanks,
drew
_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv
Re: [PATCH v4 4/5] RISC-V: add infrastructure to allow different str* implementations
Quoted 1 line · Andrew Jones
On Mon, Jan 09, 2023 at 07:17:54PM +0100, Heiko Stuebner wrote:
...
Quoted 42 lines
Quoted 5 lines
arch/riscv/lib/strcmp.S new file +5
-0+ +/* int strcmp(const char *cs, const char *ct) */ +SYM_FUNC_START(strcmp) + /* + * Returnsarch/riscv/lib/strcmp.S line 20
Quoted 10 lines
arch/riscv/lib/strcmp.S new file +10
-0+ * a0 - comparison result, value like strcmp + * + * Parameters + * a0 - string1 + * a1 - string2 + * + * Clobbers + * t0, t1, t2 + */ + mv t2, a1The above instruction and the 'mv a1, t2' below appear to be attempting to preserve a1, but that shouldn't be necessary.
arch/riscv/lib/strcmp.S line 23
Quoted 3 lines
+1:
arch/riscv/lib/strcmp.S new file +2
-0+ lbu t1, 0(a0) + lbu t0, 0(a1)I'd rather have t0 be 0(a0) and t1 be 0(a1)
arch/riscv/lib/strcmp.S line 36
Quoted 13 lines
arch/riscv/lib/strcmp.S new file +13
-0+ addi a0, a0, 1 + addi a1, a1, 1 + beq t1, t0, 3f + li a0, 1 + bgeu t1, t0, 2f + li a0, -1 +2: + mv a1, t2 + ret +3: + bnez t1, 1b + li a0, 0 + j 2bFor fun I removed one conditional and one unconditional branch (untested)
1:
Quoted 13 lines
lbu t0, 0(a0)
lbu t1, 0(a1)
addi a0, a0, 1
addi a1, a1, 1
bne t0, t1, 2f
bnez t0, 1b
li a0, 0
ret
2:
slt a1, t1, t0
slli a1, a1, 1
li a0, -1
add a0, a0, a1If we just need < 0 and > 0 then the above four instructions can become
sub a0, t0, t1
Quoted 2 lines
ret
Similar comment for strncmp.
Thanks,
drew
_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv
Re: [PATCH v4 4/5] RISC-V: add infrastructure to allow different str* implementations
In reply to Andrew JonesHi Andrew,
thanks a lot for taking the time to provide these extensive and really helpful comments.
Am Dienstag, 10. Januar 2023, 13:13:20 CET schrieb Andrew Jones:
Quoted 101 lines · Heiko Stuebner
Quoted 100 lines
From: Heiko Stuebner <[email protected]>
Depending on supported extensions on specific RISC-V cores, optimized str* functions might make sense.
This adds basic infrastructure to allow patching the function calls via alternatives later on.
The Linux kernel provides standard implementations for string functions but when architectures want to extend them, they need to provide their own.
The added generic string functions are done in assembler (taken from disassembling the main-kernel functions for now) to allow us to control the used registers and extend them with optimized variants.
Signed-off-by Heiko Stuebner <[email protected]>--- arch/riscv/include/asm/string.h | 10 +++++++++ arch/riscv/kernel/riscv_ksyms.c | 3 +++ arch/riscv/lib/Makefile | 3 +++ arch/riscv/lib/strcmp.S | 37 ++++++++++++++++++++++++++++++ arch/riscv/lib/strlen.S | 28 +++++++++++++++++++++++ arch/riscv/lib/strncmp.S | 40 +++++++++++++++++++++++++++++++++ arch/riscv/purgatory/Makefile | 13 +++++++++++ 7 files changed, 134 insertions(+) create mode 100644 arch/riscv/lib/strcmp.S create mode 100644 arch/riscv/lib/strlen.S create mode 100644 arch/riscv/lib/strncmp.Sarch/riscv/include/asm/string.h +10
-0@@ -18,6 +18,16 @@ extern asmlinkage void *__memcpy(void *, const void *, size_t);#define __HAVE_ARCH_MEMMOVE extern asmlinkage void *memmove(void *, const void *, size_t); extern asmlinkage void *__memmove(void *, const void *, size_t); + +#define __HAVE_ARCH_STRCMP +extern asmlinkage int strcmp(const char *cs, const char *ct); + +#define __HAVE_ARCH_STRLEN +extern asmlinkage __kernel_size_t strlen(const char *); + +#define __HAVE_ARCH_STRNCMP +extern asmlinkage int strncmp(const char *cs, const char *ct, size_t count); +/* For those files which don't want to check by kasan. */ #if defined(CONFIG_KASAN ) && !defined(__SANITIZE_ADDRESS__)#define memcpy (dst, src, len) __memcpy(dst, src, len)arch/riscv/kernel/riscv_ksyms.c +3
-0@@ -12,6 +12,9 @@EXPORT_SYMBOL (memset);EXPORT_SYMBOL (memcpy);EXPORT_SYMBOL (memmove); +EXPORT_SYMBOL (strcmp); +EXPORT_SYMBOL (strlen); +EXPORT_SYMBOL (strncmp);EXPORT_SYMBOL (__memset);EXPORT_SYMBOL (__memcpy);EXPORT_SYMBOL (__memmove);arch/riscv/lib/Makefile +3
-0@@ -3,6 +3,9 @@ lib-y += delay.o lib-y += memcpy.o lib-y += memset.o lib-y += memmove.o +lib-y += strcmp.o +lib-y += strlen.o +lib-y += strncmp.o lib-$(CONFIG_MMU) += uaccess.o lib-$(CONFIG_64BIT) += tishift.oarch/riscv/lib/strcmp.S new file +15
-0@@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include <linux/linkage.h> +#include <asm/asm.h> +#include <asm-generic/export.h> + +/* int strcmp(const char *cs, const char *ct) */ +SYM_FUNC_START(strcmp) + /* + * Returns + * a0 - comparison result, value like strcmp + * + * Parameters + * a0 - string1 + * a1 - string2
Quoted 8 lines
arch/riscv/lib/strcmp.S line 20
Quoted 5 lines
arch/riscv/lib/strcmp.S new file +5
-0+ * + * Clobbers + * t0, t1, t2 + */ + mv t2, a1The above instruction and the 'mv a1, t2' below appear to be attempting to preserve a1, but that shouldn't be necessary.
correct and gone now
Quoted 5 lines
arch/riscv/lib/strcmp.S line 23
Quoted 3 lines
arch/riscv/lib/strcmp.S new file +3
-0+1: + lbu t1, 0(a0) + lbu t0, 0(a1)I'd rather have t0 be 0(a0) and t1 be 0(a1)
ok
Quoted 30 lines
arch/riscv/lib/strcmp.S line 36
Quoted 13 lines
arch/riscv/lib/strcmp.S new file +13
-0+ addi a0, a0, 1 + addi a1, a1, 1 + beq t1, t0, 3f + li a0, 1 + bgeu t1, t0, 2f + li a0, -1 +2: + mv a1, t2 + ret +3: + bnez t1, 1b + li a0, 0 + j 2bFor fun I removed one conditional and one unconditional branch (untested)
1:
lbu t0, 0(a0) lbu t1, 0(a1) addi a0, a0, 1 addi a1, a1, 1 bne t0, t1, 2f bnez t0, 1b li a0, 0 ret 2: slt a1, t1, t0 slli a1, a1, 1 li a0, -1 add a0, a0, a1
Quoted 1 line
ret
yep that
- looks correct
- and also seems to produce correct results
also including your
sub a0, t0, t1
comment from the followup that then produces the same result als the zbb-variant.
And I've also verified the
- 0, if the s1 and s2 are equal;
- a negative value if s1 is less than s2;
- a positive value if s1 is greater than s2. return value calling convention with documentation
And added a comment above it, pointing out this fact for the next person stumbling over this :-)
Quoted 39 lines
Quoted 24 lines
arch/riscv/lib/strcmp.S new file +1
-0+SYM_FUNC_END(strcmp)arch/riscv/lib/strlen.S new file +17
-0@@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include <linux/linkage.h> +#include <asm/asm.h> +#include <asm-generic/export.h> + +/* int strlen(const char *s) */ +SYM_FUNC_START(strlen) + /* + * Returns + * a0 - string length + * + * Parameters + * a0 - String to measure + * + * Clobbers: + * t0, t1arch/riscv/lib/strlen.S line 27
Quoted 10 lines
arch/riscv/lib/strlen.S new file +10
-0+ */ + mv t1, a0 +1: + lbu t0, 0(t1) + bnez t0, 2f + sub a0, t1, a0 + ret +2: + addi t1, t1, 1 + j 1bSlightly reorganizing looks better (to me)
mv t1, a0
1:
Quoted 7 lines
lbu t0, 0(t1) beqz t0, 2f addi t1, t1, 1 j 1b 2: sub a0, t1, a0 ret
ok
Quoted 63 lines
Quoted 36 lines
arch/riscv/lib/strlen.S new file +1
-0+SYM_FUNC_END(strlen)arch/riscv/lib/strncmp.S new file +29
-0@@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include <linux/linkage.h> +#include <asm/asm.h> +#include <asm-generic/export.h> + +/* int strncmp(const char *cs, const char *ct, size_t count) */ +SYM_FUNC_START(strncmp) + /* + * Returns + * a0 - comparison result, value like strncmp + * + * Parameters + * a0 - string1 + * a1 - string2 + * a2 - number of characters to compare + * + * Clobbers + * t0, t1, t2 + */ + li t0, 0 +1: + beq a2, t0, 4f + add t1, a0, t0 + add t2, a1, t0 + lbu t1, 0(t1) + lbu t2, 0(t2) + beq t1, t2, 3f + li a0, 1arch/riscv/lib/strncmp.S line 39
Quoted 10 lines
arch/riscv/lib/strncmp.S new file +10
-0+ bgeu t1, t2, 2f + li a0, -1 +2: + ret +3: + addi t0, t0, 1 + bnez t1, 1b +4: + li a0, 0 + j 2b(untested)
li t2, 0
1:beq a2, t2, 2f lbu t0, 0(a0) lbu t1, 0(a1) addi a0, a0, 1 addi a1, a1, 1 bne t0, t1, 3f addi t2, t2, 1 bnez t0, 1b2:
li a0, 0
ret
3:
Quoted 5 lines
slt a1, t1, t0 slli a1, a1, 1 li a0, -1 add a0, a0, a1ret
same here, I did go over the changed assembly and verified that it produces the same results as the original and then did a second pass to also add the
sub a0, t0, t1
replacment.
Quoted 44 lines
Quoted 31 lines
arch/riscv/lib/strncmp.S new file +1
-0+SYM_FUNC_END(strncmp)arch/riscv/purgatory/Makefile +10
-0@@ -2,6 +2,7 @@ OBJECT_FILES_NON_STANDARD := y purgatory-y := purgatory.o sha256.o entry.o string.o ctype.o memcpy.o memset.o +purgatory-y += strcmp.o strlen.o strncmp.o targets += $(purgatory-y) PURGATORY_OBJS = $(addprefix $(obj)/,$(purgatory-y))@@ -18,6 +19,15 @@ $(obj)/memcpy.o: $(srctree)/arch/riscv/lib/memcpy.S FORCE $(obj)/memset.o: $(srctree)/arch/riscv/lib/memset.S FORCE $(call if_changed_rule,as_o_S) +$(obj)/strcmp.o: $(srctree)/arch/riscv/lib/strcmp.S FORCE + $(call if_changed_rule,as_o_S) + +$(obj)/strlen.o: $(srctree)/arch/riscv/lib/strlen.S FORCE + $(call if_changed_rule,as_o_S) + +$(obj)/strncmp.o: $(srctree)/arch/riscv/lib/strncmp.S FORCE + $(call if_changed_rule,as_o_S) + $(obj)/sha256.o: $(srctree)/lib/crypto/sha256.c FORCE $(call if_changed_rule,cc_o_c)arch/riscv/purgatory/Makefile CFLAGS_ctype.o += $(PURGATORY_CFLAGS) · line 95
Quoted 9 lines
arch/riscv/purgatory/Makefile +3
-0@@ -77,6 +87,9 @@ CFLAGS_ctype.o += $(PURGATORY_CFLAGS) AFLAGS_REMOVE_entry.o += -Wa,-gdwarf-2 AFLAGS_REMOVE_memcpy.o += -Wa,-gdwarf-2 AFLAGS_REMOVE_memset.o += -Wa,-gdwarf-2 +AFLAGS_REMOVE_strcmp.o += -Wa,-gdwarf-2 +AFLAGS_REMOVE_strlen.o += -Wa,-gdwarf-2 +AFLAGS_REMOVE_strncmp.o += -Wa,-gdwarf-2 $(obj)/purgatory.ro: $(PURGATORY_OBJS) FORCE $(call if_changed,ld)With at least the removal of the unnecessary preserving of a1 in strcmp, then it looks correct to me, so
Quoted 4 lines
Reviewed-by Andrew Jones <[email protected]>But I think there's room for making it more readable, and maybe even optimized, as I've tried to do.
Thanks a lot
Heiko
_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv
PATCH v4 5/5 RISC-V: add zbb support to string functions
In reply to Heiko StuebnerFrom: Heiko Stuebner <[email protected]>
Add handling for ZBB extension and add support for using it as a variant for optimized string functions.
Support for the Zbb-str-variants is limited to the GNU-assembler for now, as LLVM has not yet acquired the functionality to selectively change the arch option in assembler code. This is still under review at
https://reviews.llvm.org/D123515
--- arch/riscv/Kconfig | 24 ++++++ arch/riscv/include/asm/errata_list.h | 3 +- arch/riscv/include/asm/hwcap.h | 1 + arch/riscv/include/asm/string.h | 2 + arch/riscv/kernel/cpu.c | 1 + arch/riscv/kernel/cpufeature.c | 18 +++++ arch/riscv/lib/strcmp.S | 94 ++++++++++++++++++++++ arch/riscv/lib/strlen.S | 114 +++++++++++++++++++++++++++ arch/riscv/lib/strncmp.S | 111 ++++++++++++++++++++++++++ 9 files changed, 367 insertions(+), 1 deletion(-)
arch/riscv/Kconfig +24 -0
@@ -416,6 +416,30 @@ config RISCV_ISA_SVPBMT If you don't know what to do here, say Y. +config TOOLCHAIN_HAS_ZBB + bool + default y + depends on !64BIT || $(cc-option,-mabi=lp64 -march=rv64ima_zbb) + depends on !32BIT || $(cc-option,-mabi=ilp32 -march=rv32ima_zbb) + depends on LLD_VERSION >= 150000 || LD_VERSION >= 23900 + depends on AS_IS_GNU + +config RISCV_ISA_ZBB + bool "Zbb extension support for bit manipulation instructions" + depends on TOOLCHAIN_HAS_ZBB + depends on !XIP_KERNEL && MMU + select RISCV_ALTERNATIVE + default y + help + Adds support to dynamically detect the presence of the ZBB + extension (basic bit manipulation) and enable its usage. + + The Zbb extension provides instructions to accelerate a number + of bit-specific operations (count bit population, sign extending, + bitrotation, etc). + + If you don't know what to do here, say Y. + config TOOLCHAIN_HAS_ZICBOM bool default y
arch/riscv/include/asm/errata_list.h +2 -1
@@ -24,7 +24,8 @@#define CPUFEATURE_SVPBMT 0#define CPUFEATURE_ZICBOM 1 -#define CPUFEATURE_NUMBER 2 +#define CPUFEATURE_ZBB 2 +#define CPUFEATURE_NUMBER 3#ifdef __ASSEMBLY__
arch/riscv/include/asm/hwcap.h +1 -0
@@ -59,6 +59,7 @@ enum riscv_isa_ext_id {RISCV_ISA_EXT_ZIHINTPAUSE ,RISCV_ISA_EXT_SSTC ,RISCV_ISA_EXT_SVINVAL , +RISCV_ISA_EXT_ZBB , RISCV_ISA_EXT_ID_MAX };static_assert (RISCV_ISA_EXT_ID_MAX <=RISCV_ISA_EXT_MAX );
arch/riscv/include/asm/string.h +2 -0
@@ -6,6 +6,8 @@#ifndef _ASM_RISCV_STRING_H#define _ASM_RISCV_STRING_H +#include <asm/alternative-macros.h> +#include <asm/errata_list.h> #include <linux/types.h> #include <linux/linkage.h>
arch/riscv/kernel/cpu.c +1 -0
@@ -162,6 +162,7 @@ arch_initcall(riscv_cpuinfo_init);* extensions by an underscore. */ static struct riscv_isa_ext_data isa_ext_arr[] = { +__RISCV_ISA_EXT_DATA (zbb,RISCV_ISA_EXT_ZBB ),__RISCV_ISA_EXT_DATA (sscofpmf,RISCV_ISA_EXT_SSCOFPMF ),__RISCV_ISA_EXT_DATA (sstc,RISCV_ISA_EXT_SSTC ),__RISCV_ISA_EXT_DATA (svinval,RISCV_ISA_EXT_SVINVAL ),
arch/riscv/kernel/cpufeature.c +18 -0
@@ -222,6 +222,7 @@ void __init riscv_fill_hwcap(void)set_bit (nr, this_isa); } }else { +SET_ISA_EXT_MAP ("zbb" ,RISCV_ISA_EXT_ZBB );SET_ISA_EXT_MAP ("sscofpmf" ,RISCV_ISA_EXT_SSCOFPMF );SET_ISA_EXT_MAP ("svpbmt" ,RISCV_ISA_EXT_SVPBMT );SET_ISA_EXT_MAP ("zicbom" ,RISCV_ISA_EXT_ZICBOM );
@@ -301,6 +302,20 @@ static bool __init_or_module cpufeature_probe_zicbom(unsigned int stage)return true; } +static bool __init_or_module cpufeature_probe_zbb(unsigned int stage) +{ +if (!IS_ENABLED (CONFIG_RISCV_ISA_ZBB )) +return false; + +if (stage ==RISCV_ALTERNATIVES_EARLY_BOOT ) +return false; + +if (!riscv_isa_extension_available (NULL ,ZBB )) +return false; + +return true; +} +/* * Probe presence of individual extensions. *
@@ -318,6 +333,9 @@ static u32 __init_or_module cpufeature_probe(unsigned int stage)if (cpufeature_probe_zicbom (stage)) cpu_req_feature |=BIT (CPUFEATURE_ZICBOM ); +if (cpufeature_probe_zbb (stage)) + cpu_req_feature |=BIT (CPUFEATURE_ZBB ); +return cpu_req_feature; }
arch/riscv/lib/strcmp.S +94 -0
107 lines not on this page · raw message
arch/riscv/lib/strlen.S +114 -0
127 lines not on this page · raw message
arch/riscv/lib/strncmp.S +111 -0
124 lines not on this page · raw message
Signature · 7 lines
2.35.1 _______________________________________________ linux-riscv mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-riscv
Re: [PATCH v4 5/5] RISC-V: add zbb support to string functions
Quoted 10 lines · Heiko Stuebner
From: Heiko Stuebner <[email protected]>
Add handling for ZBB extension and add support for using it as a variant for optimized string functions.
Support for the Zbb-str-variants is limited to the GNU-assembler for now, as LLVM has not yet acquired the functionality to selectively change the arch option in assembler code. This is still under review at
https://reviews.llvm.org/D123515
Shame, soon hopefully!
Quoted 3 lines
Co-developed-by Christoph Muellner <[email protected]>Signed-off-by Christoph Muellner <[email protected]>Signed-off-by Heiko Stuebner <[email protected]>
If you've not changed the asm bits since vN-1, then:
Quoted 520 lines
--- arch/riscv/Kconfig | 24 ++++++ arch/riscv/include/asm/errata_list.h | 3 +- arch/riscv/include/asm/hwcap.h | 1 + arch/riscv/include/asm/string.h | 2 + arch/riscv/kernel/cpu.c | 1 + arch/riscv/kernel/cpufeature.c | 18 +++++ arch/riscv/lib/strcmp.S | 94 ++++++++++++++++++++++ arch/riscv/lib/strlen.S | 114 +++++++++++++++++++++++++++ arch/riscv/lib/strncmp.S | 111 ++++++++++++++++++++++++++ 9 files changed, 367 insertions(+), 1 deletion(-)arch/riscv/Kconfig +24
-0@@ -416,6 +416,30 @@ config RISCV_ISA_SVPBMT If you don't know what to do here, say Y. +config TOOLCHAIN_HAS_ZBB + bool + default y + depends on !64BIT || $(cc-option,-mabi=lp64 -march=rv64ima_zbb) + depends on !32BIT || $(cc-option,-mabi=ilp32 -march=rv32ima_zbb) + depends on LLD_VERSION >= 150000 || LD_VERSION >= 23900 + depends on AS_IS_GNU + +config RISCV_ISA_ZBB + bool "Zbb extension support for bit manipulation instructions" + depends on TOOLCHAIN_HAS_ZBB + depends on !XIP_KERNEL && MMU + select RISCV_ALTERNATIVE + default y + help + Adds support to dynamically detect the presence of the ZBB + extension (basic bit manipulation) and enable its usage. + + The Zbb extension provides instructions to accelerate a number + of bit-specific operations (count bit population, sign extending, + bitrotation, etc). + + If you don't know what to do here, say Y. + config TOOLCHAIN_HAS_ZICBOM bool default yarch/riscv/include/asm/errata_list.h +2
-1@@ -24,7 +24,8 @@#define CPUFEATURE_SVPBMT 0#define CPUFEATURE_ZICBOM 1 -#define CPUFEATURE_NUMBER 2 +#define CPUFEATURE_ZBB 2 +#define CPUFEATURE_NUMBER 3#ifdef __ASSEMBLY__arch/riscv/include/asm/hwcap.h +1
-0@@ -59,6 +59,7 @@ enum riscv_isa_ext_id {RISCV_ISA_EXT_ZIHINTPAUSE ,RISCV_ISA_EXT_SSTC ,RISCV_ISA_EXT_SVINVAL , +RISCV_ISA_EXT_ZBB , RISCV_ISA_EXT_ID_MAX };static_assert (RISCV_ISA_EXT_ID_MAX <=RISCV_ISA_EXT_MAX );arch/riscv/include/asm/string.h +2
-0@@ -6,6 +6,8 @@#ifndef _ASM_RISCV_STRING_H#define _ASM_RISCV_STRING_H +#include <asm/alternative-macros.h> +#include <asm/errata_list.h> #include <linux/types.h> #include <linux/linkage.h> arch/riscv/kernel/cpu.c +1
-0@@ -162,6 +162,7 @@ arch_initcall(riscv_cpuinfo_init);* extensions by an underscore. */ static struct riscv_isa_ext_data isa_ext_arr[] = { +__RISCV_ISA_EXT_DATA (zbb,RISCV_ISA_EXT_ZBB ),__RISCV_ISA_EXT_DATA (sscofpmf,RISCV_ISA_EXT_SSCOFPMF ),__RISCV_ISA_EXT_DATA (sstc,RISCV_ISA_EXT_SSTC ),__RISCV_ISA_EXT_DATA (svinval,RISCV_ISA_EXT_SVINVAL ),arch/riscv/kernel/cpufeature.c +18
-0@@ -222,6 +222,7 @@ void __init riscv_fill_hwcap(void)set_bit (nr, this_isa); } }else { +SET_ISA_EXT_MAP ("zbb" ,RISCV_ISA_EXT_ZBB );SET_ISA_EXT_MAP ("sscofpmf" ,RISCV_ISA_EXT_SSCOFPMF );SET_ISA_EXT_MAP ("svpbmt" ,RISCV_ISA_EXT_SVPBMT );SET_ISA_EXT_MAP ("zicbom" ,RISCV_ISA_EXT_ZICBOM );@@ -301,6 +302,20 @@ static bool __init_or_module cpufeature_probe_zicbom(unsigned int stage)return true; } +static bool __init_or_module cpufeature_probe_zbb(unsigned int stage) +{ +if (!IS_ENABLED (CONFIG_RISCV_ISA_ZBB )) +return false; + +if (stage ==RISCV_ALTERNATIVES_EARLY_BOOT ) +return false; + +if (!riscv_isa_extension_available (NULL ,ZBB )) +return false; + +return true; +} +/* * Probe presence of individual extensions. * @@ -318,6 +333,9 @@ static u32 __init_or_module cpufeature_probe(unsigned int stage)if (cpufeature_probe_zicbom (stage)) cpu_req_feature |=BIT (CPUFEATURE_ZICBOM ); +if (cpufeature_probe_zbb (stage)) + cpu_req_feature |=BIT (CPUFEATURE_ZBB ); +return cpu_req_feature; }arch/riscv/lib/strcmp.S +94
-0@@ -3,9 +3,14 @@ #include <linux/linkage.h> #include <asm/asm.h> #include <asm-generic/export.h> +#include <asm/alternative-macros.h> +#include <asm/errata_list.h> /* int strcmp(const char *cs, const char *ct) */ SYM_FUNC_START(strcmp) + + ALTERNATIVE("nop", "j variant_zbb", 0, CPUFEATURE_ZBB, CONFIG_RISCV_ISA_ZBB) + /* * Returns * a0 - comparison result, value like strcmp@@ -34,4 +39,93 @@ SYM_FUNC_START(strcmp) bnez t1, 1b li a0, 0 j 2b + +/* + * Variant of strcmp using the ZBB extension if available + */ +#ifdef CONFIG_RISCV_ISA_ZBB +variant_zbb: +#define src1 a0 +#define result a0 +#define src2 t5 +#define data1 t0 +#define data2 t1 +#define align t2 +#define data1_orcb t3 +#define m1 t4 + +.option push +.option arch,+zbb + + /* + * Returns + * a0 - comparison result, value like strcmp + * + * Parameters + * a0 - string1 + * a1 - string2 + * + * Clobbers + * t0, t1, t2, t3, t4, t5 + */ + mv src2, a1 + + or align, src1, src2 + li m1, -1 + and align, align, SZREG-1 + bnez align, 3f + + /* Main loop for aligned string. */ + .p2align 3 +1: + REG_L data1, 0(src1) + REG_L data2, 0(src2) + orc.b data1_orcb, data1 + bne data1_orcb, m1, 2f + addi src1, src1, SZREG + addi src2, src2, SZREG + beq data1, data2, 1b + + /* + * Words don't match, and no null byte in the first + * word. Get bytes in big-endian order and compare. + */ +#ifndef CONFIG_CPU_BIG_ENDIAN + rev8 data1, data1 + rev8 data2, data2 +#endif + + /* Synthesize (data1 >= data2) ? 1 : -1 in a branchless sequence. */ + sltu result, data1, data2 + neg result, result + ori result, result, 1 + ret + +2: + /* + * Found a null byte. + * If words don't match, fall back to simple loop. + */ + bne data1, data2, 3f + + /* Otherwise, strings are equal. */ + li result, 0 + ret + + /* Simple loop for misaligned strings. */ + .p2align 3 +3: + lbu data1, 0(src1) + lbu data2, 0(src2) + addi src1, src1, 1 + addi src2, src2, 1 + bne data1, data2, 4f + bnez data1, 3b + +4: + sub result, data1, data2 + ret + +.option pop +#endif SYM_FUNC_END(strcmp)arch/riscv/lib/strlen.S +114
-0@@ -3,9 +3,14 @@ #include <linux/linkage.h> #include <asm/asm.h> #include <asm-generic/export.h> +#include <asm/alternative-macros.h> +#include <asm/errata_list.h> /* int strlen(const char *s) */ SYM_FUNC_START(strlen) + + ALTERNATIVE("nop", "j variant_zbb", 0, CPUFEATURE_ZBB, CONFIG_RISCV_ISA_ZBB) + /* * Returns * a0 - string length@@ -25,4 +30,113 @@ SYM_FUNC_START(strlen) 2: addi t1, t1, 1 j 1b + +/* + * Variant of strlen using the ZBB extension if available + */ +#ifdef CONFIG_RISCV_ISA_ZBB +variant_zbb: + +#define src a0 +#define result a0 +#define addr t0 +#define data t1 +#define offset t2 +#define offset_bits t2 +#define valid_bytes t3 +#define m1 t3 + +#ifdef CONFIG_CPU_BIG_ENDIAN +# define CZ clz +# define SHIFT sll +#else +# define CZ ctz +# define SHIFT srl +#endif + +.option push +.option arch,+zbb + + /* + * Returns + * a0 - string length + * + * Parameters + * a0 - String to measure + * + * Clobbers + * t0, t1, t2, t3 + */ + + /* Number of irrelevant bytes in the first word. */ + andi offset, src, SZREG-1 + + /* Align pointer. */ + andi addr, src, -SZREG + + li valid_bytes, SZREG + sub valid_bytes, valid_bytes, offset + slli offset_bits, offset, RISCV_LGPTR + + /* Get the first word. */ + REG_L data, 0(addr) + + /* + * Shift away the partial data we loaded to remove the irrelevant bytes + * preceding the string with the effect of adding NUL bytes at the + * end of the string. + */ + SHIFT data, data, offset_bits + + /* Convert non-NUL into 0xff and NUL into 0x00. */ + orc.b data, data + + /* Convert non-NUL into 0x00 and NUL into 0xff. */ + not data, data + + /* + * Search for the first set bit (corresponding to a NUL byte in the + * original chunk). + */ + CZ data, data + + /* + * The first chunk is special: commpare against the number + * of valid bytes in this chunk. + */ + srli result, data, 3 + bgtu valid_bytes, result, 3f + + /* Prepare for the word comparison loop. */ + addi offset, addr, SZREG + li m1, -1 + + /* + * Our critical loop is 4 instructions and processes data in + * 4 byte or 8 byte chunks. + */ + .p2align 3 +1: + REG_L data, SZREG(addr) + addi addr, addr, SZREG + orc.b data, data + beq data, m1, 1b +2: + not data, data + CZ data, data + + /* Get number of processed words. */ + sub offset, addr, offset + + /* Add number of characters in the first word. */ + add result, result, offset + srli data, data, 3 + + /* Add number of characters in the last word. */ + add result, result, data +3: + ret + +.option pop +#endif SYM_FUNC_END(strlen)arch/riscv/lib/strncmp.S +111
-0@@ -3,9 +3,14 @@ #include <linux/linkage.h> #include <asm/asm.h> #include <asm-generic/export.h> +#include <asm/alternative-macros.h> +#include <asm/errata_list.h> /* int strncmp(const char *cs, const char *ct, size_t count) */ SYM_FUNC_START(strncmp) + + ALTERNATIVE("nop", "j variant_zbb", 0, CPUFEATURE_ZBB, CONFIG_RISCV_ISA_ZBB) + /* * Returns * a0 - comparison result, value like strncmp@@ -37,4 +42,110 @@ SYM_FUNC_START(strncmp) 4: li a0, 0 j 2b + +/* + * Variant of strncmp using the ZBB extension if available + */ +#ifdef CONFIG_RISCV_ISA_ZBB +variant_zbb: + +#define src1 a0 +#define result a0 +#define src2 t6 +#define len a2 +#define data1 t0 +#define data2 t1 +#define align t2 +#define data1_orcb t3 +#define limit t4 +#define m1 t5 + +.option push +.option arch,+zbb + + /* + * Returns + * a0 - comparison result, like strncmp + * + * Parameters + * a0 - string1 + * a1 - string2 + * a2 - number of characters to compare + * + * Clobbers + * t0, t1, t2, t3, t4, t5, t6 + */ + mv src2, a1 + + or align, src1, src2 + li m1, -1 + and align, align, SZREG-1 + add limit, src1, len + bnez align, 4f + + /* Adjust limit for fast-path. */ + addi limit, limit, -SZREG + + /* Main loop for aligned string. */ + .p2align 3 +1: + bgt src1, limit, 3f + REG_L data1, 0(src1) + REG_L data2, 0(src2) + orc.b data1_orcb, data1 + bne data1_orcb, m1, 2f + addi src1, src1, SZREG + addi src2, src2, SZREG + beq data1, data2, 1b + + /* + * Words don't match, and no null byte in the first + * word. Get bytes in big-endian order and compare. + */ +#ifndef CONFIG_CPU_BIG_ENDIAN + rev8 data1, data1 + rev8 data2, data2 +#endif + + /* Synthesize (data1 >= data2) ? 1 : -1 in a branchless sequence. */ + sltu result, data1, data2 + neg result, result + ori result, result, 1 + ret + +2: + /* + * Found a null byte. + * If words don't match, fall back to simple loop. + */ + bne data1, data2, 3f + + /* Otherwise, strings are equal. */ + li result, 0 + ret + + /* Simple loop for misaligned strings. */ +3: + /* Restore limit for slow-path. */ + addi limit, limit, SZREG + .p2align 3 +4: + bge src1, limit, 6f + lbu data1, 0(src1) + lbu data2, 0(src2) + addi src1, src1, 1 + addi src2, src2, 1 + bne data1, data2, 5f + bnez data1, 4b + +5: + sub result, data1, data2 + ret + +6: + li result, 0 + ret + +.option pop +#endif SYM_FUNC_END(strncmp)Signature · 1 line
2.35.1
_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv
Re: [PATCH v4 5/5] RISC-V: add zbb support to string functions
In reply to Heiko StuebnerQuoted 92 lines · Heiko Stuebner
From: Heiko Stuebner <[email protected]>
Add handling for ZBB extension and add support for using it as a variant for optimized string functions.
Support for the Zbb-str-variants is limited to the GNU-assembler for now, as LLVM has not yet acquired the functionality to selectively change the arch option in assembler code. This is still under review at
https://reviews.llvm.org/D123515Co-developed-by Christoph Muellner <[email protected]>Signed-off-by Christoph Muellner <[email protected]>Signed-off-by Heiko Stuebner <[email protected]>--- arch/riscv/Kconfig | 24 ++++++ arch/riscv/include/asm/errata_list.h | 3 +- arch/riscv/include/asm/hwcap.h | 1 + arch/riscv/include/asm/string.h | 2 + arch/riscv/kernel/cpu.c | 1 + arch/riscv/kernel/cpufeature.c | 18 +++++ arch/riscv/lib/strcmp.S | 94 ++++++++++++++++++++++ arch/riscv/lib/strlen.S | 114 +++++++++++++++++++++++++++ arch/riscv/lib/strncmp.S | 111 ++++++++++++++++++++++++++ 9 files changed, 367 insertions(+), 1 deletion(-)arch/riscv/Kconfig +24
-0@@ -416,6 +416,30 @@ config RISCV_ISA_SVPBMT If you don't know what to do here, say Y. +config TOOLCHAIN_HAS_ZBB + bool + default y + depends on !64BIT || $(cc-option,-mabi=lp64 -march=rv64ima_zbb) + depends on !32BIT || $(cc-option,-mabi=ilp32 -march=rv32ima_zbb) + depends on LLD_VERSION >= 150000 || LD_VERSION >= 23900 + depends on AS_IS_GNU + +config RISCV_ISA_ZBB + bool "Zbb extension support for bit manipulation instructions" + depends on TOOLCHAIN_HAS_ZBB + depends on !XIP_KERNEL && MMU + select RISCV_ALTERNATIVE + default y + help + Adds support to dynamically detect the presence of the ZBB + extension (basic bit manipulation) and enable its usage. + + The Zbb extension provides instructions to accelerate a number + of bit-specific operations (count bit population, sign extending, + bitrotation, etc). + + If you don't know what to do here, say Y. + config TOOLCHAIN_HAS_ZICBOM bool default yarch/riscv/include/asm/errata_list.h +2
-1@@ -24,7 +24,8 @@#define CPUFEATURE_SVPBMT 0#define CPUFEATURE_ZICBOM 1 -#define CPUFEATURE_NUMBER 2 +#define CPUFEATURE_ZBB 2 +#define CPUFEATURE_NUMBER 3#ifdef __ASSEMBLY__arch/riscv/include/asm/hwcap.h +1
-0@@ -59,6 +59,7 @@ enum riscv_isa_ext_id {RISCV_ISA_EXT_ZIHINTPAUSE ,RISCV_ISA_EXT_SSTC ,RISCV_ISA_EXT_SVINVAL , +RISCV_ISA_EXT_ZBB , RISCV_ISA_EXT_ID_MAX };static_assert (RISCV_ISA_EXT_ID_MAX <=RISCV_ISA_EXT_MAX );arch/riscv/include/asm/string.h +0
-0
arch/riscv/include/asm/string.h line 10
Quoted 5 lines
arch/riscv/include/asm/string.h +2
-0@@ -6,6 +6,8 @@#ifndef _ASM_RISCV_STRING_H#define _ASM_RISCV_STRING_H +#include <asm/alternative-macros.h> +#include <asm/errata_list.h>
Why are these includes getting added? Shouldn't they be getting directly added to whatever is including asm/string.h instead?
Quoted 8 lines
arch/riscv/include/asm/string.h +0
-0#include <linux/types.h> #include <linux/linkage.h> arch/riscv/kernel/cpu.c +0
-0
arch/riscv/kernel/cpu.c arch_initcall(riscv_cpuinfo_init); · line 168
Quoted 7 lines
arch/riscv/kernel/cpu.c +1
-0@@ -162,6 +162,7 @@ arch_initcall(riscv_cpuinfo_init);* extensions by an underscore. */ static struct riscv_isa_ext_data isa_ext_arr[] = { +__RISCV_ISA_EXT_DATA (zbb,RISCV_ISA_EXT_ZBB ),__RISCV_ISA_EXT_DATA (sscofpmf,RISCV_ISA_EXT_SSCOFPMF ),__RISCV_ISA_EXT_DATA (sstc,RISCV_ISA_EXT_SSTC ),__RISCV_ISA_EXT_DATA (svinval,RISCV_ISA_EXT_SVINVAL ),
Huh, this array still doesn't appear to be in order... Zbb should be getting inserted between the Zi* extensions (which should be first) and the S* extensions and each of those three categories should be in alphabetical order.
arch/riscv/kernel/cpufeature.c void __init riscv_fill_hwcap(void) · line 228
Quoted 12 lines
arch/riscv/kernel/cpufeature.c +1
-0@@ -222,6 +222,7 @@ void __init riscv_fill_hwcap(void)set_bit (nr, this_isa); } }else { +SET_ISA_EXT_MAP ("zbb" ,RISCV_ISA_EXT_ZBB );SET_ISA_EXT_MAP ("sscofpmf" ,RISCV_ISA_EXT_SSCOFPMF );SET_ISA_EXT_MAP ("svpbmt" ,RISCV_ISA_EXT_SVPBMT );SET_ISA_EXT_MAP ("zicbom" ,RISCV_ISA_EXT_ZICBOM );
I think we wanted this one in alphabetical order...
Quoted 22 lines
arch/riscv/kernel/cpufeature.c +14
-0@@ -301,6 +302,20 @@ static bool __init_or_module cpufeature_probe_zicbom(unsigned int stage)return true; } +static bool __init_or_module cpufeature_probe_zbb(unsigned int stage) +{ +if (!IS_ENABLED (CONFIG_RISCV_ISA_ZBB )) +return false; + +if (stage ==RISCV_ALTERNATIVES_EARLY_BOOT ) +return false; + +if (!riscv_isa_extension_available (NULL ,ZBB )) +return false; + +return true; +} +/* * Probe presence of individual extensions. *
arch/riscv/kernel/cpufeature.c static u32 __init_or_module cpufeature_probe(unsigned int stage) · line 341
Quoted 9 lines
arch/riscv/kernel/cpufeature.c +3
-0@@ -318,6 +333,9 @@ static u32 __init_or_module cpufeature_probe(unsigned int stage)if (cpufeature_probe_zicbom (stage)) cpu_req_feature |=BIT (CPUFEATURE_ZICBOM ); +if (cpufeature_probe_zbb (stage)) + cpu_req_feature |=BIT (CPUFEATURE_ZBB ); +return cpu_req_feature; }
I'm skipping the asm review for now since I'm still not clear on the plan for non-optimized string functions vs. optimized vs. compiler builtins.
Thanks,
drew
_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv
Re: [PATCH v4 5/5] RISC-V: add zbb support to string functions
Quoted 11 lines · Andrew Jones
Quoted 9 lines · Heiko Stuebner
arch/riscv/kernel/cpu.c +1
-0@@ -162,6 +162,7 @@ arch_initcall(riscv_cpuinfo_init);* extensions by an underscore. */ static struct riscv_isa_ext_data isa_ext_arr[] = { +__RISCV_ISA_EXT_DATA (zbb,RISCV_ISA_EXT_ZBB ),
Quoted 8 lines
arch/riscv/kernel/cpu.c arch_initcall(riscv_cpuinfo_init); · line 168
Quoted 3 lines
arch/riscv/kernel/cpu.c +0
-0__RISCV_ISA_EXT_DATA (sscofpmf,RISCV_ISA_EXT_SSCOFPMF ),__RISCV_ISA_EXT_DATA (sstc,RISCV_ISA_EXT_SSTC ), __RISCV_ISA_EXT_DATA(svinval, RISCV_ISA_EXT_SVINVAL),Huh, this array still doesn't appear to be in order... Zbb should be getting inserted between the Zi* extensions (which should be first) and the S* extensions and each of those three categories should be in alphabetical order.
Correct. The new entry was at least added in the right place, reordering existing entries aside.
Quoted 6 lines
Quoted 6 lines
arch/riscv/kernel/cpufeature.c +0
-0@@ -222,6 +222,7 @@ void __init riscv_fill_hwcap(void)set_bit (nr, this_isa);
Quoted 8 lines
arch/riscv/kernel/cpufeature.c void __init riscv_fill_hwcap(void) · line 228
Quoted 6 lines
arch/riscv/kernel/cpufeature.c +1
-0} }else { +SET_ISA_EXT_MAP ("zbb" ,RISCV_ISA_EXT_ZBB );SET_ISA_EXT_MAP ("sscofpmf" ,RISCV_ISA_EXT_SSCOFPMF );SET_ISA_EXT_MAP ("svpbmt" ,RISCV_ISA_EXT_SVPBMT );SET_ISA_EXT_MAP ("zicbom" ,RISCV_ISA_EXT_ZICBOM );I think we wanted this one in alphabetical order...
Correct again.
I've been avoiding mentioning this stuff in reviews though as Palmer has not yet picked up the patches [0] putting these arrays into those orders in the first place.
I tried to harass him about them last night, but he didn't get around to them. Perhaps worth mentioning tomorrow if we're gonna keep having to discussing these lists in reviews?
Thanks,
Conor.
0 - https://lore.kernel.org/all/[email protected]/
(I tried applying it yesterday, worked with git am -3)
_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv
Re: [PATCH v4 5/5] RISC-V: add zbb support to string functions
Am Dienstag, 10. Januar 2023, 11:14:37 CET schrieb Conor Dooley:
Quoted 49 lines · Andrew Jones
Quoted 11 lines · Heiko Stuebner
Quoted 10 lines
arch/riscv/kernel/cpu.c +1
-0@@ -162,6 +162,7 @@ arch_initcall(riscv_cpuinfo_init);* extensions by an underscore. */ static struct riscv_isa_ext_data isa_ext_arr[] = { +__RISCV_ISA_EXT_DATA (zbb,RISCV_ISA_EXT_ZBB ),Quoted 8 lines
arch/riscv/kernel/cpu.c arch_initcall(riscv_cpuinfo_init); · line 168
Quoted 3 lines
arch/riscv/kernel/cpu.c +0
-0__RISCV_ISA_EXT_DATA (sscofpmf,RISCV_ISA_EXT_SSCOFPMF ),__RISCV_ISA_EXT_DATA (sstc,RISCV_ISA_EXT_SSTC ), __RISCV_ISA_EXT_DATA(svinval, RISCV_ISA_EXT_SVINVAL),Huh, this array still doesn't appear to be in order... Zbb should be getting inserted between the Zi* extensions (which should be first) and the S* extensions and each of those three categories should be in alphabetical order.
Correct. The new entry was at least added in the right place, reordering existing entries aside.
Quoted 6 lines
Quoted 6 lines
arch/riscv/kernel/cpufeature.c +0
-0@@ -222,6 +222,7 @@ void __init riscv_fill_hwcap(void)set_bit (nr, this_isa);Quoted 8 lines
arch/riscv/kernel/cpufeature.c void __init riscv_fill_hwcap(void) · line 228
Quoted 6 lines
arch/riscv/kernel/cpufeature.c +1
-0} }else { +SET_ISA_EXT_MAP ("zbb" ,RISCV_ISA_EXT_ZBB );SET_ISA_EXT_MAP ("sscofpmf" ,RISCV_ISA_EXT_SSCOFPMF );SET_ISA_EXT_MAP ("svpbmt" ,RISCV_ISA_EXT_SVPBMT );SET_ISA_EXT_MAP ("zicbom" ,RISCV_ISA_EXT_ZICBOM );I think we wanted this one in alphabetical order...
Correct again.
I've been avoiding mentioning this stuff in reviews though as Palmer has not yet picked up the patches [0] putting these arrays into those orders in the first place.
I tried to harass him about them last night, but he didn't get around to them. Perhaps worth mentioning tomorrow if we're gonna keep having to discussing these lists in reviews?
Quoted 5 lines
Thanks,
Conor.0 - https://lore.kernel.org/all/[email protected]/
(I tried applying it yesterday, worked withgit am -3)
I've gone forward and imported that series as a dependency in my tree so that the ordering becomes correct :-) .
Though of course I got a conflict with Andrew's
commit e923f4625ed3 ("riscv: Apply a static assert to riscv_isa_ext_id") but fixed that up - though maybe a rebased v3 may be in order?
Heiko
_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv
Re: [PATCH v4 5/5] RISC-V: add zbb support to string functions
Quoted 7 lines · Heiko Stübner
I've gone forward and imported that series as a dependency in my tree so that the ordering becomes correct :-) .
Though of course I got a conflict with Andrew's
commite923f4625ed3("riscv: Apply a static assert toriscv_isa_ext_id") but fixed that up - though maybe a rebased v3 may be in order?
To be honest, I "strategically" didn't resend a v3 despite that trivial conflict since it's at the bottom of the list on patchwork. I was hoping that it'd get merged earlier in the window so any of the various bits of extension support stuff could go on top of the re-order, given it conflicts with, give or take, every other extension related patch. That's not come to pass (yet?) though.
I can easily respin, but if I do & some other extension bit gets merged before it, it'll conflict again, so I didn't in the hopes that the conflict could be resolved...
What you think?
_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv
Re: [PATCH v4 5/5] RISC-V: add zbb support to string functions
Am Donnerstag, 12. Januar 2023, 13:06:01 CET schrieb Conor Dooley:
Quoted 16 lines · Heiko Stübner
Quoted 7 lines
I've gone forward and imported that series as a dependency in my tree so that the ordering becomes correct :-) .
Though of course I got a conflict with Andrew's
commite923f4625ed3("riscv: Apply a static assert toriscv_isa_ext_id") but fixed that up - though maybe a rebased v3 may be in order?To be honest, I "strategically" didn't resend a v3 despite that trivial conflict since it's at the bottom of the list on patchwork. I was hoping that it'd get merged earlier in the window so any of the various bits of extension support stuff could go on top of the re-order, given it conflicts with, give or take, every other extension related patch. That's not come to pass (yet?) though.
Quoted 3 lines
I can easily respin, but if I do & some other extension bit gets merged before it, it'll conflict again, so I didn't in the hopes that the conflict could be resolved...
I guess it mainly comes down to what is easier for Palmer.
From personal experience, the less you have to think before applying a patch, the easier it is to just do that that in a free minute. Though on the other hand, the conflict in your series is pretty minimal.
So both ways have their advantages somehow.
_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv
Re: [PATCH v4 5/5] RISC-V: add zbb support to string functions
In reply to Heiko StuebnerQuoted 160 lines · Heiko Stuebner
From: Heiko Stuebner <[email protected]>
Add handling for ZBB extension and add support for using it as a variant for optimized string functions.
Support for the Zbb-str-variants is limited to the GNU-assembler for now, as LLVM has not yet acquired the functionality to selectively change the arch option in assembler code. This is still under review at
https://reviews.llvm.org/D123515Co-developed-by Christoph Muellner <[email protected]>Signed-off-by Christoph Muellner <[email protected]>Signed-off-by Heiko Stuebner <[email protected]>--- arch/riscv/Kconfig | 24 ++++++ arch/riscv/include/asm/errata_list.h | 3 +- arch/riscv/include/asm/hwcap.h | 1 + arch/riscv/include/asm/string.h | 2 + arch/riscv/kernel/cpu.c | 1 + arch/riscv/kernel/cpufeature.c | 18 +++++ arch/riscv/lib/strcmp.S | 94 ++++++++++++++++++++++ arch/riscv/lib/strlen.S | 114 +++++++++++++++++++++++++++ arch/riscv/lib/strncmp.S | 111 ++++++++++++++++++++++++++ 9 files changed, 367 insertions(+), 1 deletion(-)arch/riscv/Kconfig +24
-0@@ -416,6 +416,30 @@ config RISCV_ISA_SVPBMT If you don't know what to do here, say Y. +config TOOLCHAIN_HAS_ZBB + bool + default y + depends on !64BIT || $(cc-option,-mabi=lp64 -march=rv64ima_zbb) + depends on !32BIT || $(cc-option,-mabi=ilp32 -march=rv32ima_zbb) + depends on LLD_VERSION >= 150000 || LD_VERSION >= 23900 + depends on AS_IS_GNU + +config RISCV_ISA_ZBB + bool "Zbb extension support for bit manipulation instructions" + depends on TOOLCHAIN_HAS_ZBB + depends on !XIP_KERNEL && MMU + select RISCV_ALTERNATIVE + default y + help + Adds support to dynamically detect the presence of the ZBB + extension (basic bit manipulation) and enable its usage. + + The Zbb extension provides instructions to accelerate a number + of bit-specific operations (count bit population, sign extending, + bitrotation, etc). + + If you don't know what to do here, say Y. + config TOOLCHAIN_HAS_ZICBOM bool default yarch/riscv/include/asm/errata_list.h +2
-1@@ -24,7 +24,8 @@#define CPUFEATURE_SVPBMT 0#define CPUFEATURE_ZICBOM 1 -#define CPUFEATURE_NUMBER 2 +#define CPUFEATURE_ZBB 2 +#define CPUFEATURE_NUMBER 3#ifdef __ASSEMBLY__arch/riscv/include/asm/hwcap.h +1
-0@@ -59,6 +59,7 @@ enum riscv_isa_ext_id {RISCV_ISA_EXT_ZIHINTPAUSE ,RISCV_ISA_EXT_SSTC ,RISCV_ISA_EXT_SVINVAL , +RISCV_ISA_EXT_ZBB , RISCV_ISA_EXT_ID_MAX };static_assert (RISCV_ISA_EXT_ID_MAX <=RISCV_ISA_EXT_MAX );arch/riscv/include/asm/string.h +2
-0@@ -6,6 +6,8 @@#ifndef _ASM_RISCV_STRING_H#define _ASM_RISCV_STRING_H +#include <asm/alternative-macros.h> +#include <asm/errata_list.h> #include <linux/types.h> #include <linux/linkage.h> arch/riscv/kernel/cpu.c +1
-0@@ -162,6 +162,7 @@ arch_initcall(riscv_cpuinfo_init);* extensions by an underscore. */ static struct riscv_isa_ext_data isa_ext_arr[] = { +__RISCV_ISA_EXT_DATA (zbb,RISCV_ISA_EXT_ZBB ),__RISCV_ISA_EXT_DATA (sscofpmf,RISCV_ISA_EXT_SSCOFPMF ),__RISCV_ISA_EXT_DATA (sstc,RISCV_ISA_EXT_SSTC ),__RISCV_ISA_EXT_DATA (svinval,RISCV_ISA_EXT_SVINVAL ),arch/riscv/kernel/cpufeature.c +18
-0@@ -222,6 +222,7 @@ void __init riscv_fill_hwcap(void)set_bit (nr, this_isa); } }else { +SET_ISA_EXT_MAP ("zbb" ,RISCV_ISA_EXT_ZBB );SET_ISA_EXT_MAP ("sscofpmf" ,RISCV_ISA_EXT_SSCOFPMF );SET_ISA_EXT_MAP ("svpbmt" ,RISCV_ISA_EXT_SVPBMT );SET_ISA_EXT_MAP ("zicbom" ,RISCV_ISA_EXT_ZICBOM );@@ -301,6 +302,20 @@ static bool __init_or_module cpufeature_probe_zicbom(unsigned int stage)return true; } +static bool __init_or_module cpufeature_probe_zbb(unsigned int stage) +{ +if (!IS_ENABLED (CONFIG_RISCV_ISA_ZBB )) +return false; + +if (stage ==RISCV_ALTERNATIVES_EARLY_BOOT ) +return false; + +if (!riscv_isa_extension_available (NULL ,ZBB )) +return false; + +return true; +} +/* * Probe presence of individual extensions. * @@ -318,6 +333,9 @@ static u32 __init_or_module cpufeature_probe(unsigned int stage)if (cpufeature_probe_zicbom (stage)) cpu_req_feature |=BIT (CPUFEATURE_ZICBOM ); +if (cpufeature_probe_zbb (stage)) + cpu_req_feature |=BIT (CPUFEATURE_ZBB ); +return cpu_req_feature; }arch/riscv/lib/strcmp.S +0
-0
arch/riscv/lib/strcmp.S line 12
Quoted 10 lines
arch/riscv/lib/strcmp.S +4
-0@@ -3,9 +3,14 @@ #include <linux/linkage.h> #include <asm/asm.h> #include <asm-generic/export.h> +#include <asm/alternative-macros.h> +#include <asm/errata_list.h> /* int strcmp(const char *cs, const char *ct) */ SYM_FUNC_START(strcmp) + + ALTERNATIVE("nop", "j variant_zbb", 0, CPUFEATURE_ZBB, CONFIG_RISCV_ISA_ZBB)
How about strcmp_zbb and similar for the other functions for the labels?
Quoted 12 lines
arch/riscv/lib/strcmp.S +5
-0+ /* * Returns * a0 - comparison result, value like strcmp@@ -34,4 +39,93 @@ SYM_FUNC_START(strcmp) bnez t1, 1b li a0, 0 j 2b + +/* + * Variant of strcmp using the ZBB extension if available + */
arch/riscv/lib/strcmp.S SYM_FUNC_START(strcmp) · line 55
Quoted 10 lines
arch/riscv/lib/strcmp.S +10
-0+#ifdef CONFIG_RISCV_ISA_ZBB +variant_zbb: +#define src1 a0 +#define result a0 +#define src2 t5 +#define data1 t0 +#define data2 t1 +#define align t2 +#define data1_orcb t3 +#define m1 t4
nit: It's probably just me, but I'm not a big fan of creating defines for registers, particularly when the meaning of the registers' contents gets changed, because then the name, at least temporarily, no longer accurately describes the content.
And, in this case, it looks like the code matches Appendix A, except the register and label naming. I'd prefer to have those match too so I could "review" with the diff tool.
Quoted 14 lines
arch/riscv/lib/strcmp.S +14
-0+ +.option push +.option arch,+zbb + + /* + * Returns + * a0 - comparison result, value like strcmp + * + * Parameters + * a0 - string1 + * a1 - string2 + * + * Clobbers + * t0, t1, t2, t3, t4, t5
arch/riscv/lib/strcmp.S SYM_FUNC_START(strcmp) · line 79
Quoted 10 lines
arch/riscv/lib/strcmp.S +10
-0+ */ + mv src2, a1 + + or align, src1, src2 + li m1, -1 + and align, align, SZREG-1 + bnez align, 3f + + /* Main loop for aligned string. */ + .p2align 3
Why are the starts of the loops aligned to 8 byte boundaries?
Quoted 37 lines
arch/riscv/lib/strcmp.S +37
-0+1: + REG_L data1, 0(src1) + REG_L data2, 0(src2) + orc.b data1_orcb, data1 + bne data1_orcb, m1, 2f + addi src1, src1, SZREG + addi src2, src2, SZREG + beq data1, data2, 1b + + /* + * Words don't match, and no null byte in the first + * word. Get bytes in big-endian order and compare. + */ +#ifndef CONFIG_CPU_BIG_ENDIAN + rev8 data1, data1 + rev8 data2, data2 +#endif + + /* Synthesize (data1 >= data2) ? 1 : -1 in a branchless sequence. */ + sltu result, data1, data2 + neg result, result + ori result, result, 1 + ret + +2: + /* + * Found a null byte. + * If words don't match, fall back to simple loop. + */ + bne data1, data2, 3f + + /* Otherwise, strings are equal. */ + li result, 0 + ret + + /* Simple loop for misaligned strings. */ + .p2align 3
arch/riscv/lib/strcmp.S SYM_FUNC_START(strcmp) · line 126
Quoted 10 lines
arch/riscv/lib/strcmp.S +10
-0+3: + lbu data1, 0(src1) + lbu data2, 0(src2) + addi src1, src1, 1 + addi src2, src2, 1 + bne data1, data2, 4f + bnez data1, 3b + +4: + sub result, data1, data2
The non-optimized version returns explicitly -1, 0, 1, whereas this returns < 0, 0, > 0. Assuming we don't need the explicit -1 / 1 then we can change the non-optimized version to do this subtraction for its return value too, saving several instructions.
Quoted 57 lines
arch/riscv/lib/strcmp.S +4
-0+ ret + +.option pop +#endif SYM_FUNC_END(strcmp)arch/riscv/lib/strlen.S +34
-0@@ -3,9 +3,14 @@ #include <linux/linkage.h> #include <asm/asm.h> #include <asm-generic/export.h> +#include <asm/alternative-macros.h> +#include <asm/errata_list.h> /* int strlen(const char *s) */ SYM_FUNC_START(strlen) + + ALTERNATIVE("nop", "j variant_zbb", 0, CPUFEATURE_ZBB, CONFIG_RISCV_ISA_ZBB) + /* * Returns * a0 - string length@@ -25,4 +30,113 @@ SYM_FUNC_START(strlen) 2: addi t1, t1, 1 j 1b + +/* + * Variant of strlen using the ZBB extension if available + */ +#ifdef CONFIG_RISCV_ISA_ZBB +variant_zbb: + +#define src a0 +#define result a0 +#define addr t0 +#define data t1 +#define offset t2 +#define offset_bits t2 +#define valid_bytes t3 +#define m1 t3 + +#ifdef CONFIG_CPU_BIG_ENDIAN +# define CZ clz +# define SHIFT sll +#else +# define CZ ctz +# define SHIFT srl +#endif + +.option push +.option arch,+zbb + + /* + * Returns
arch/riscv/lib/strlen.S SYM_FUNC_START(strlen) · line 71
Quoted 10 lines
arch/riscv/lib/strlen.S +10
-0+ * a0 - string length + * + * Parameters + * a0 - String to measure + * + * Clobbers + * t0, t1, t2, t3 + */ + + /* Number of irrelevant bytes in the first word. */
I see that this comment doesn't just apply to the next instruction, but to several instructions. It threw me off at first. Again, I think I'd prefer we just copy+paste the Appendix, comments an all.
arch/riscv/lib/strlen.S SYM_FUNC_START(strlen) · line 79
Quoted 8 lines
arch/riscv/lib/strlen.S +8
-0+ andi offset, src, SZREG-1 + + /* Align pointer. */ + andi addr, src, -SZREG + + li valid_bytes, SZREG + sub valid_bytes, valid_bytes, offset + slli offset_bits, offset, RISCV_LGPTR
The shift immediate should be 3, even for rv32, since we want bits-per-byte, not bytes-per-word. The spec example uses PTRLOG, which does imply bytes-per-word to me as well, so that seems like a spec bug.
arch/riscv/lib/strlen.S SYM_FUNC_START(strlen) · line 87
Quoted 8 lines
arch/riscv/lib/strlen.S +8
-0+ + /* Get the first word. */ + REG_L data, 0(addr) + + /* + * Shift away the partial data we loaded to remove the irrelevant bytes + * preceding the string with the effect of adding NUL bytes at the + * end of the string.
the end of the string's first word.
Quoted 7 lines
arch/riscv/lib/strlen.S +7
-0+ */ + SHIFT data, data, offset_bits + + /* Convert non-NUL into 0xff and NUL into 0x00. */ + orc.b data, data + + /* Convert non-NUL into 0x00 and NUL into 0xff. */
arch/riscv/lib/strlen.S SYM_FUNC_START(strlen) · line 104
Quoted 10 lines
arch/riscv/lib/strlen.S +10
-0+ not data, data + + /* + * Search for the first set bit (corresponding to a NUL byte in the + * original chunk). + */ + CZ data, data + + /* + * The first chunk is special: commpare against the number
compare
Quoted 94 lines
arch/riscv/lib/strlen.S +37
-0+ * of valid bytes in this chunk. + */ + srli result, data, 3 + bgtu valid_bytes, result, 3f + + /* Prepare for the word comparison loop. */ + addi offset, addr, SZREG + li m1, -1 + + /* + * Our critical loop is 4 instructions and processes data in + * 4 byte or 8 byte chunks. + */ + .p2align 3 +1: + REG_L data, SZREG(addr) + addi addr, addr, SZREG + orc.b data, data + beq data, m1, 1b +2: + not data, data + CZ data, data + + /* Get number of processed words. */ + sub offset, addr, offset + + /* Add number of characters in the first word. */ + add result, result, offset + srli data, data, 3 + + /* Add number of characters in the last word. */ + add result, result, data +3: + ret + +.option pop +#endif SYM_FUNC_END(strlen)arch/riscv/lib/strncmp.S +38
-0@@ -3,9 +3,14 @@ #include <linux/linkage.h> #include <asm/asm.h> #include <asm-generic/export.h> +#include <asm/alternative-macros.h> +#include <asm/errata_list.h> /* int strncmp(const char *cs, const char *ct, size_t count) */ SYM_FUNC_START(strncmp) + + ALTERNATIVE("nop", "j variant_zbb", 0, CPUFEATURE_ZBB, CONFIG_RISCV_ISA_ZBB) + /* * Returns * a0 - comparison result, value like strncmp@@ -37,4 +42,110 @@ SYM_FUNC_START(strncmp) 4: li a0, 0 j 2b + +/* + * Variant of strncmp using the ZBB extension if available + */ +#ifdef CONFIG_RISCV_ISA_ZBB +variant_zbb: + +#define src1 a0 +#define result a0 +#define src2 t6 +#define len a2 +#define data1 t0 +#define data2 t1 +#define align t2 +#define data1_orcb t3 +#define limit t4 +#define m1 t5 + +.option push +.option arch,+zbb + + /* + * Returns + * a0 - comparison result, like strncmp + * + * Parameters + * a0 - string1 + * a1 - string2 + * a2 - number of characters to compare + * + * Clobbers + * t0, t1, t2, t3, t4, t5, t6 + */
arch/riscv/lib/strncmp.S SYM_FUNC_START(strncmp) · line 87
Quoted 10 lines
arch/riscv/lib/strncmp.S +10
-0+ mv src2, a1 + + or align, src1, src2 + li m1, -1 + and align, align, SZREG-1 + add limit, src1, len + bnez align, 4f + + /* Adjust limit for fast-path. */ + addi limit, limit, -SZREG
To avoid unnecessarily writing the last SZREG bytes one at a time when 'len' is SZREG aligned, we should be using the explicitly aligned limit here (limit & ~(SZREG - 1)) rather than just subtracting SZREG. Then, down in the slow-path we use the original limit to finish off if len wasn't aligned. When it was aligned, we'll just immediately branch to the ret. (I suspect many times 'len' will be SZREG aligned, since string buffers are typically allocated with power-of-two sizes).
Quoted 50 lines
arch/riscv/lib/strncmp.S +50
-0+ + /* Main loop for aligned string. */ + .p2align 3 +1: + bgt src1, limit, 3f + REG_L data1, 0(src1) + REG_L data2, 0(src2) + orc.b data1_orcb, data1 + bne data1_orcb, m1, 2f + addi src1, src1, SZREG + addi src2, src2, SZREG + beq data1, data2, 1b + + /* + * Words don't match, and no null byte in the first + * word. Get bytes in big-endian order and compare. + */ +#ifndef CONFIG_CPU_BIG_ENDIAN + rev8 data1, data1 + rev8 data2, data2 +#endif + + /* Synthesize (data1 >= data2) ? 1 : -1 in a branchless sequence. */ + sltu result, data1, data2 + neg result, result + ori result, result, 1 + ret + +2: + /* + * Found a null byte. + * If words don't match, fall back to simple loop. + */ + bne data1, data2, 3f + + /* Otherwise, strings are equal. */ + li result, 0 + ret + + /* Simple loop for misaligned strings. */ +3: + /* Restore limit for slow-path. */ + addi limit, limit, SZREG + .p2align 3 +4: + bge src1, limit, 6f + lbu data1, 0(src1) + lbu data2, 0(src2) + addi src1, src1, 1 + addi src2, src2, 1
arch/riscv/lib/strncmp.S SYM_FUNC_START(strncmp) · line 147
Quoted 10 lines
arch/riscv/lib/strncmp.S +10
-0+ bne data1, data2, 5f + bnez data1, 4b + +5: + sub result, data1, data2 + ret + +6: + li result, 0 + ret
As strcmp and strncmp have so much in common it's tempting to try and merge them somehow. Maybe with a handful of macros shared between them?
Quoted 7 lines
arch/riscv/lib/strncmp.S +3
-0+ +.option pop +#endif SYM_FUNC_END(strncmp)Signature · 1 line
2.35.1
Thanks,
drew
_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv
Re: [PATCH v4 5/5] RISC-V: add zbb support to string functions
Quoted 168 lines · Andrew Jones
Quoted 166 lines · Heiko Stuebner
From: Heiko Stuebner <[email protected]>
Add handling for ZBB extension and add support for using it as a variant for optimized string functions.
Support for the Zbb-str-variants is limited to the GNU-assembler for now, as LLVM has not yet acquired the functionality to selectively change the arch option in assembler code. This is still under review at
https://reviews.llvm.org/D123515Co-developed-by Christoph Muellner <[email protected]>Signed-off-by Christoph Muellner <[email protected]>Signed-off-by Heiko Stuebner <[email protected]>--- arch/riscv/Kconfig | 24 ++++++ arch/riscv/include/asm/errata_list.h | 3 +- arch/riscv/include/asm/hwcap.h | 1 + arch/riscv/include/asm/string.h | 2 + arch/riscv/kernel/cpu.c | 1 + arch/riscv/kernel/cpufeature.c | 18 +++++ arch/riscv/lib/strcmp.S | 94 ++++++++++++++++++++++ arch/riscv/lib/strlen.S | 114 +++++++++++++++++++++++++++ arch/riscv/lib/strncmp.S | 111 ++++++++++++++++++++++++++ 9 files changed, 367 insertions(+), 1 deletion(-)arch/riscv/Kconfig +24
-0@@ -416,6 +416,30 @@ config RISCV_ISA_SVPBMT If you don't know what to do here, say Y. +config TOOLCHAIN_HAS_ZBB + bool + default y + depends on !64BIT || $(cc-option,-mabi=lp64 -march=rv64ima_zbb) + depends on !32BIT || $(cc-option,-mabi=ilp32 -march=rv32ima_zbb) + depends on LLD_VERSION >= 150000 || LD_VERSION >= 23900 + depends on AS_IS_GNU + +config RISCV_ISA_ZBB + bool "Zbb extension support for bit manipulation instructions" + depends on TOOLCHAIN_HAS_ZBB + depends on !XIP_KERNEL && MMU + select RISCV_ALTERNATIVE + default y + help + Adds support to dynamically detect the presence of the ZBB + extension (basic bit manipulation) and enable its usage. + + The Zbb extension provides instructions to accelerate a number + of bit-specific operations (count bit population, sign extending, + bitrotation, etc). + + If you don't know what to do here, say Y. + config TOOLCHAIN_HAS_ZICBOM bool default yarch/riscv/include/asm/errata_list.h +2
-1@@ -24,7 +24,8 @@#define CPUFEATURE_SVPBMT 0#define CPUFEATURE_ZICBOM 1 -#define CPUFEATURE_NUMBER 2 +#define CPUFEATURE_ZBB 2 +#define CPUFEATURE_NUMBER 3#ifdef __ASSEMBLY__arch/riscv/include/asm/hwcap.h +1
-0@@ -59,6 +59,7 @@ enum riscv_isa_ext_id {RISCV_ISA_EXT_ZIHINTPAUSE ,RISCV_ISA_EXT_SSTC ,RISCV_ISA_EXT_SVINVAL , +RISCV_ISA_EXT_ZBB , RISCV_ISA_EXT_ID_MAX };static_assert (RISCV_ISA_EXT_ID_MAX <=RISCV_ISA_EXT_MAX );arch/riscv/include/asm/string.h +2
-0@@ -6,6 +6,8 @@#ifndef _ASM_RISCV_STRING_H#define _ASM_RISCV_STRING_H +#include <asm/alternative-macros.h> +#include <asm/errata_list.h> #include <linux/types.h> #include <linux/linkage.h> arch/riscv/kernel/cpu.c +1
-0@@ -162,6 +162,7 @@ arch_initcall(riscv_cpuinfo_init);* extensions by an underscore. */ static struct riscv_isa_ext_data isa_ext_arr[] = { +__RISCV_ISA_EXT_DATA (zbb,RISCV_ISA_EXT_ZBB ),__RISCV_ISA_EXT_DATA (sscofpmf,RISCV_ISA_EXT_SSCOFPMF ),__RISCV_ISA_EXT_DATA (sstc,RISCV_ISA_EXT_SSTC ),__RISCV_ISA_EXT_DATA (svinval,RISCV_ISA_EXT_SVINVAL ),arch/riscv/kernel/cpufeature.c +18
-0@@ -222,6 +222,7 @@ void __init riscv_fill_hwcap(void)set_bit (nr, this_isa); } }else { +SET_ISA_EXT_MAP ("zbb" ,RISCV_ISA_EXT_ZBB );SET_ISA_EXT_MAP ("sscofpmf" ,RISCV_ISA_EXT_SSCOFPMF );SET_ISA_EXT_MAP ("svpbmt" ,RISCV_ISA_EXT_SVPBMT );SET_ISA_EXT_MAP ("zicbom" ,RISCV_ISA_EXT_ZICBOM );@@ -301,6 +302,20 @@ static bool __init_or_module cpufeature_probe_zicbom(unsigned int stage)return true; } +static bool __init_or_module cpufeature_probe_zbb(unsigned int stage) +{ +if (!IS_ENABLED (CONFIG_RISCV_ISA_ZBB )) +return false; + +if (stage ==RISCV_ALTERNATIVES_EARLY_BOOT ) +return false; + +if (!riscv_isa_extension_available (NULL ,ZBB )) +return false; + +return true; +} +/* * Probe presence of individual extensions. * @@ -318,6 +333,9 @@ static u32 __init_or_module cpufeature_probe(unsigned int stage)if (cpufeature_probe_zicbom (stage)) cpu_req_feature |=BIT (CPUFEATURE_ZICBOM ); +if (cpufeature_probe_zbb (stage)) + cpu_req_feature |=BIT (CPUFEATURE_ZBB ); +return cpu_req_feature; }arch/riscv/lib/strcmp.S +2
-0@@ -3,9 +3,14 @@ #include <linux/linkage.h> #include <asm/asm.h> #include <asm-generic/export.h> +#include <asm/alternative-macros.h> +#include <asm/errata_list.h>
Quoted 6 lines
arch/riscv/lib/strcmp.S line 12
Quoted 4 lines
arch/riscv/lib/strcmp.S +2
-0/* int strcmp(const char *cs, const char *ct) */ SYM_FUNC_START(strcmp) + + ALTERNATIVE("nop", "j variant_zbb", 0, CPUFEATURE_ZBB, CONFIG_RISCV_ISA_ZBB)How about
strcmp_zbband similar for the other functions for the labels?
Fully agree!
Quoted 20 lines
Quoted 20 lines
arch/riscv/lib/strcmp.S +12
-0+ /* * Returns * a0 - comparison result, value like strcmp@@ -34,4 +39,93 @@ SYM_FUNC_START(strcmp) bnez t1, 1b li a0, 0 j 2b + +/* + * Variant of strcmp using the ZBB extension if available + */ +#ifdef CONFIG_RISCV_ISA_ZBB +variant_zbb: +#define src1 a0 +#define result a0 +#define src2 t5 +#define data1 t0 +#define data2 t1
Quoted 8 lines
arch/riscv/lib/strcmp.S SYM_FUNC_START(strcmp) · line 55
Quoted 3 lines
arch/riscv/lib/strcmp.S +3
-0+#define align t2 +#define data1_orcb t3 +#define m1 t4nit: It's probably just me, but I'm not a big fan of creating defines for registers, particularly when the meaning of the registers' contents gets changed, because then the name, at least temporarily, no longer accurately describes the content.
Without any strong feelings about this: I think this is common practice when writing asm code (helps to write and understand the code).
See random other sources where this is also done:
https://sourceware.org/git/?p=glibc.git;a=blob;f=sysdeps/aarch64/strcmp.S;h=054ce15ef8273073de4c821e3f415f1f70541554;hb=HEAD#l35
https://github.com/openssl/openssl/blob/master/crypto/aes/asm/aes-x86_64.pl#L58
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/arch/arm64/lib/memcmp.S?h=v6.2-rc3#n20
Quoted 3 lines
And, in this case, it looks like the code matches Appendix A, except the register and label naming. I'd prefer to have those match too so I could "review" with the diff tool.
Good that you did not "review" the code with diff. If so, you would not have found the LGPTR issue (below).
Quoted 20 lines
Quoted 20 lines
arch/riscv/lib/strcmp.S +18
-0+ +.option push +.option arch,+zbb + + /* + * Returns + * a0 - comparison result, value like strcmp + * + * Parameters + * a0 - string1 + * a1 - string2 + * + * Clobbers + * t0, t1, t2, t3, t4, t5 + */ + mv src2, a1 + + or align, src1, src2
Quoted 8 lines
arch/riscv/lib/strcmp.S SYM_FUNC_START(strcmp) · line 79
Quoted 6 lines
arch/riscv/lib/strcmp.S +6
-0+ li m1, -1 + and align, align, SZREG-1 + bnez align, 3f + + /* Main loop for aligned string. */ + .p2align 3Why are the starts of the loops aligned to 8 byte boundaries?
Loop alignment seems to be a valid optimization method for other architectures. I don't see anything specific in RISC-V that would make loop alignment irrelevant.
AArch64 code seems to align loops (or other jump targets) to 8 or 16 bytes. This was considered as not harmful, but potentially beneficial.
Quoted 46 lines
Quoted 46 lines
arch/riscv/lib/strcmp.S +44
-0+1: + REG_L data1, 0(src1) + REG_L data2, 0(src2) + orc.b data1_orcb, data1 + bne data1_orcb, m1, 2f + addi src1, src1, SZREG + addi src2, src2, SZREG + beq data1, data2, 1b + + /* + * Words don't match, and no null byte in the first + * word. Get bytes in big-endian order and compare. + */ +#ifndef CONFIG_CPU_BIG_ENDIAN + rev8 data1, data1 + rev8 data2, data2 +#endif + + /* Synthesize (data1 >= data2) ? 1 : -1 in a branchless sequence. */ + sltu result, data1, data2 + neg result, result + ori result, result, 1 + ret + +2: + /* + * Found a null byte. + * If words don't match, fall back to simple loop. + */ + bne data1, data2, 3f + + /* Otherwise, strings are equal. */ + li result, 0 + ret + + /* Simple loop for misaligned strings. */ + .p2align 3 +3: + lbu data1, 0(src1) + lbu data2, 0(src2) + addi src1, src1, 1 + addi src2, src2, 1 + bne data1, data2, 4f + bnez data1, 3b
Quoted 8 lines
arch/riscv/lib/strcmp.S SYM_FUNC_START(strcmp) · line 126
Quoted 3 lines
arch/riscv/lib/strcmp.S +3
-0+ +4: + sub result, data1, data2The non-optimized version returns explicitly -1, 0, 1, whereas this returns < 0, 0, > 0. Assuming we don't need the explicit -1 / 1 then we can change the non-optimized version to do this subtraction for its return value too, saving several instructions.
Regarding "we don't need":
"The strcmp() and strncmp() functions return an integer less than, equal
to, or greater than zero if s1 (or the first n bytes thereof) is found, respectively, to be less than, to match, or be greater than s2."
So, yes, the values -1/+1 are just some of the allowed return values.
Quoted 65 lines
Quoted 65 lines
arch/riscv/lib/strcmp.S +4
-0+ ret + +.option pop +#endif SYM_FUNC_END(strcmp)arch/riscv/lib/strlen.S +40
-0@@ -3,9 +3,14 @@ #include <linux/linkage.h> #include <asm/asm.h> #include <asm-generic/export.h> +#include <asm/alternative-macros.h> +#include <asm/errata_list.h> /* int strlen(const char *s) */ SYM_FUNC_START(strlen) + + ALTERNATIVE("nop", "j variant_zbb", 0, CPUFEATURE_ZBB, CONFIG_RISCV_ISA_ZBB) + /* * Returns * a0 - string length@@ -25,4 +30,113 @@ SYM_FUNC_START(strlen) 2: addi t1, t1, 1 j 1b + +/* + * Variant of strlen using the ZBB extension if available + */ +#ifdef CONFIG_RISCV_ISA_ZBB +variant_zbb: + +#define src a0 +#define result a0 +#define addr t0 +#define data t1 +#define offset t2 +#define offset_bits t2 +#define valid_bytes t3 +#define m1 t3 + +#ifdef CONFIG_CPU_BIG_ENDIAN +# define CZ clz +# define SHIFT sll +#else +# define CZ ctz +# define SHIFT srl +#endif + +.option push +.option arch,+zbb + + /* + * Returns + * a0 - string length + * + * Parameters + * a0 - String to measure + * + * Clobbers
Quoted 8 lines
arch/riscv/lib/strlen.S SYM_FUNC_START(strlen) · line 71
Quoted 4 lines
arch/riscv/lib/strlen.S +4
-0+ * t0, t1, t2, t3 + */ + + /* Number of irrelevant bytes in the first word. */I see that this comment doesn't just apply to the next instruction, but to several instructions. It threw me off at first. Again, I think I'd prefer we just copy+paste the Appendix, comments an all.
I think this comment only applies to this line.
The loop processes on word-aliged chunks and only the first chunk can be misaligned.
The andi instruction calculates the number of irrelevant bytes in this first word.
The Appendix states "// offset" here, so I think this comment is better.
But I don't mind if people want to stick with the comments from the Appendix.
Quoted 14 lines
arch/riscv/lib/strlen.S SYM_FUNC_START(strlen) · line 79
Quoted 10 lines
arch/riscv/lib/strlen.S +8
-0+ andi offset, src, SZREG-1 + + /* Align pointer. */ + andi addr, src, -SZREG + + li valid_bytes, SZREG + sub valid_bytes, valid_bytes, offset + slli offset_bits, offset, RISCV_LGPTRThe shift immediate should be 3, even for rv32, since we want bits-per-byte, not bytes-per-word. The spec example uses PTRLOG, which does imply bytes-per-word to me as well, so that seems like a spec bug.
Correct, that's an issue in the Appendix.
Quoted 120 lines
arch/riscv/lib/strlen.S SYM_FUNC_START(strlen) · line 87
Quoted 10 lines
arch/riscv/lib/strlen.S +8
-0+ + /* Get the first word. */ + REG_L data, 0(addr) + + /* + * Shift away the partial data we loaded to remove the irrelevant bytes + * preceding the string with the effect of adding NUL bytes at the + * end of the string.the end of the string's first word.
Quoted 7 lines
arch/riscv/lib/strlen.S +7
-0+ */ + SHIFT data, data, offset_bits + + /* Convert non-NUL into 0xff and NUL into 0x00. */ + orc.b data, data + + /* Convert non-NUL into 0x00 and NUL into 0xff. */arch/riscv/lib/strlen.S SYM_FUNC_START(strlen) · line 104
Quoted 10 lines
arch/riscv/lib/strlen.S +10
-0+ not data, data + + /* + * Search for the first set bit (corresponding to a NUL byte in the + * original chunk). + */ + CZ data, data + + /* + * The first chunk is special: commpare against the numbercompare
Quoted 87 lines
arch/riscv/lib/strlen.S +37
-0+ * of valid bytes in this chunk. + */ + srli result, data, 3 + bgtu valid_bytes, result, 3f + + /* Prepare for the word comparison loop. */ + addi offset, addr, SZREG + li m1, -1 + + /* + * Our critical loop is 4 instructions and processes data in + * 4 byte or 8 byte chunks. + */ + .p2align 3 +1: + REG_L data, SZREG(addr) + addi addr, addr, SZREG + orc.b data, data + beq data, m1, 1b +2: + not data, data + CZ data, data + + /* Get number of processed words. */ + sub offset, addr, offset + + /* Add number of characters in the first word. */ + add result, result, offset + srli data, data, 3 + + /* Add number of characters in the last word. */ + add result, result, data +3: + ret + +.option pop +#endif SYM_FUNC_END(strlen)arch/riscv/lib/strncmp.S +31
-0@@ -3,9 +3,14 @@ #include <linux/linkage.h> #include <asm/asm.h> #include <asm-generic/export.h> +#include <asm/alternative-macros.h> +#include <asm/errata_list.h> /* int strncmp(const char *cs, const char *ct, size_t count) */ SYM_FUNC_START(strncmp) + + ALTERNATIVE("nop", "j variant_zbb", 0, CPUFEATURE_ZBB, CONFIG_RISCV_ISA_ZBB) + /* * Returns * a0 - comparison result, value like strncmp@@ -37,4 +42,110 @@ SYM_FUNC_START(strncmp) 4: li a0, 0 j 2b + +/* + * Variant of strncmp using the ZBB extension if available + */ +#ifdef CONFIG_RISCV_ISA_ZBB +variant_zbb: + +#define src1 a0 +#define result a0 +#define src2 t6 +#define len a2 +#define data1 t0 +#define data2 t1 +#define align t2 +#define data1_orcb t3 +#define limit t4 +#define m1 t5 + +.option push +.option arch,+zbb + + /* + * Returns + * a0 - comparison result, like strncmp + * + * Parameters
Quoted 8 lines
arch/riscv/lib/strncmp.S SYM_FUNC_START(strncmp) · line 78
Quoted 8 lines
arch/riscv/lib/strncmp.S +8
-0+ * a0 - string1 + * a1 - string2 + * a2 - number of characters to compare + * + * Clobbers + * t0, t1, t2, t3, t4, t5, t6 + */ + mv src2, a1
Why is this needed (i.e. why is src2 not a1, but t6)?
Quoted 10 lines
arch/riscv/lib/strncmp.S SYM_FUNC_START(strncmp) · line 87
Quoted 9 lines
arch/riscv/lib/strncmp.S +9
-0+ + or align, src1, src2 + li m1, -1 + and align, align, SZREG-1 + add limit, src1, len + bnez align, 4f + + /* Adjust limit for fast-path. */ + addi limit, limit, -SZREG
Quoted 7 lines
To avoid unnecessarily writing the last SZREG bytes one at a time when 'len' is SZREG aligned, we should be using the explicitly aligned limit here (limit & ~(SZREG - 1)) rather than just subtracting SZREG. Then, down in the slow-path we use the original limit to finish off if len wasn't aligned. When it was aligned, we'll just immediately branch to the ret. (I suspect many times 'len' will be SZREG aligned, since string buffers are typically allocated with power-of-two sizes).
Yes, that's a good idea!
So I propose to keep the (original) limit and introduce a new register here:
andi fast_limit, limit, -SZREG // ~(SZREG - 1) == -SZREG
...and use that in the loop:
bgt src1, fast_limit, 3f
...and remove the limit restore ("addi limit, limit, SZREG").
Quoted 65 lines
Quoted 51 lines
arch/riscv/lib/strncmp.S +50
-0+ + /* Main loop for aligned string. */ + .p2align 3 +1: + bgt src1, limit, 3f + REG_L data1, 0(src1) + REG_L data2, 0(src2) + orc.b data1_orcb, data1 + bne data1_orcb, m1, 2f + addi src1, src1, SZREG + addi src2, src2, SZREG + beq data1, data2, 1b + + /* + * Words don't match, and no null byte in the first + * word. Get bytes in big-endian order and compare. + */ +#ifndef CONFIG_CPU_BIG_ENDIAN + rev8 data1, data1 + rev8 data2, data2 +#endif + + /* Synthesize (data1 >= data2) ? 1 : -1 in a branchless sequence. */ + sltu result, data1, data2 + neg result, result + ori result, result, 1 + ret + +2: + /* + * Found a null byte. + * If words don't match, fall back to simple loop. + */ + bne data1, data2, 3f + + /* Otherwise, strings are equal. */ + li result, 0 + ret + + /* Simple loop for misaligned strings. */ +3: + /* Restore limit for slow-path. */ + addi limit, limit, SZREG + .p2align 3 +4: + bge src1, limit, 6f + lbu data1, 0(src1) + lbu data2, 0(src2) + addi src1, src1, 1 + addi src2, src2, 1arch/riscv/lib/strncmp.S SYM_FUNC_START(strncmp) · line 147
Quoted 10 lines
arch/riscv/lib/strncmp.S +10
-0+ bne data1, data2, 5f + bnez data1, 4b + +5: + sub result, data1, data2 + ret + +6: + li result, 0 + retAs strcmp and strncmp have so much in common it's tempting to try and merge them somehow. Maybe with a handful of macros shared between them?
Quoted 10 lines
Quoted 7 lines
arch/riscv/lib/strncmp.S +3
-1+ +.option pop +#endif SYM_FUNC_END(strncmp) --2.35.1
Thanks,
drew
_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv
Re: [PATCH v4 5/5] RISC-V: add zbb support to string functions
Quoted 3 lines · Christoph Müllner
Quoted 2 lines · Andrew Jones
On Mon, Jan 09, 2023 at 07:17:55PM +0100, Heiko Stuebner wrote:
...
Quoted 10 lines
Quoted 7 lines
arch/riscv/lib/strlen.S SYM_FUNC_START(strlen) · line 79
Quoted 3 lines
arch/riscv/lib/strlen.S +3
-0+ li valid_bytes, SZREG + sub valid_bytes, valid_bytes, offset + slli offset_bits, offset, RISCV_LGPTRThe shift immediate should be 3, even for rv32, since we want bits-per-byte, not bytes-per-word. The spec example uses PTRLOG, which does imply bytes-per-word to me as well, so that seems like a spec bug.
Correct, that's an issue in the Appendix.
Thanks for the confirmation. I've sent a PR,
https://github.com/riscv/riscv-bitmanip/pull/182
drew
_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv
Re: [PATCH v4 5/5] RISC-V: add zbb support to string functions
In reply to Christoph MüllnerQuoted 27 lines · Christoph Müllner
Quoted 18 lines
Quoted 18 lines
arch/riscv/lib/strcmp.S +18
-0+ +.option push +.option arch,+zbb + + /* + * Returns + * a0 - comparison result, value like strcmp + * + * Parameters + * a0 - string1 + * a1 - string2 + * + * Clobbers + * t0, t1, t2, t3, t4, t5 + */ + mv src2, a1 + + or align, src1, src2Quoted 8 lines
arch/riscv/lib/strcmp.S SYM_FUNC_START(strcmp) · line 79
Quoted 6 lines
arch/riscv/lib/strcmp.S +6
-0+ li m1, -1 + and align, align, SZREG-1 + bnez align, 3f + + /* Main loop for aligned string. */ + .p2align 3Why are the starts of the loops aligned to 8 byte boundaries?
Quoted 3 lines
Loop alignment seems to be a valid optimization method for other architectures. I don't see anything specific in RISC-V that would make loop alignment irrelevant.
Right. It's pretty common to see small, but observable performance improvements on many uarchs due to code alignments. There's typically three knobs to tune in this space. First is function alignment, second is loop alignment, then finally other jump target alignments.
Drew, happy to sync on this if you want -- I've got a low priority TODO to come up with some sensible guidelines for our v1 chip. I'm not too inclined to push hard on it yet as it'll be a lot easier to find the sweet spot once real hardware is available.
jeff
_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv
Re: [PATCH v4 5/5] RISC-V: add zbb support to string functions
In reply to Andrew JonesHi Andrew,
Am Mittwoch, 11. Januar 2023, 13:24:52 CET schrieb Andrew Jones:
Quoted 167 lines · Heiko Stuebner
Quoted 166 lines
From: Heiko Stuebner <[email protected]>
Add handling for ZBB extension and add support for using it as a variant for optimized string functions.
Support for the Zbb-str-variants is limited to the GNU-assembler for now, as LLVM has not yet acquired the functionality to selectively change the arch option in assembler code. This is still under review at
https://reviews.llvm.org/D123515Co-developed-by Christoph Muellner <[email protected]>Signed-off-by Christoph Muellner <[email protected]>Signed-off-by Heiko Stuebner <[email protected]>--- arch/riscv/Kconfig | 24 ++++++ arch/riscv/include/asm/errata_list.h | 3 +- arch/riscv/include/asm/hwcap.h | 1 + arch/riscv/include/asm/string.h | 2 + arch/riscv/kernel/cpu.c | 1 + arch/riscv/kernel/cpufeature.c | 18 +++++ arch/riscv/lib/strcmp.S | 94 ++++++++++++++++++++++ arch/riscv/lib/strlen.S | 114 +++++++++++++++++++++++++++ arch/riscv/lib/strncmp.S | 111 ++++++++++++++++++++++++++ 9 files changed, 367 insertions(+), 1 deletion(-)arch/riscv/Kconfig +24
-0@@ -416,6 +416,30 @@ config RISCV_ISA_SVPBMT If you don't know what to do here, say Y. +config TOOLCHAIN_HAS_ZBB + bool + default y + depends on !64BIT || $(cc-option,-mabi=lp64 -march=rv64ima_zbb) + depends on !32BIT || $(cc-option,-mabi=ilp32 -march=rv32ima_zbb) + depends on LLD_VERSION >= 150000 || LD_VERSION >= 23900 + depends on AS_IS_GNU + +config RISCV_ISA_ZBB + bool "Zbb extension support for bit manipulation instructions" + depends on TOOLCHAIN_HAS_ZBB + depends on !XIP_KERNEL && MMU + select RISCV_ALTERNATIVE + default y + help + Adds support to dynamically detect the presence of the ZBB + extension (basic bit manipulation) and enable its usage. + + The Zbb extension provides instructions to accelerate a number + of bit-specific operations (count bit population, sign extending, + bitrotation, etc). + + If you don't know what to do here, say Y. + config TOOLCHAIN_HAS_ZICBOM bool default yarch/riscv/include/asm/errata_list.h +2
-1@@ -24,7 +24,8 @@#define CPUFEATURE_SVPBMT 0#define CPUFEATURE_ZICBOM 1 -#define CPUFEATURE_NUMBER 2 +#define CPUFEATURE_ZBB 2 +#define CPUFEATURE_NUMBER 3#ifdef __ASSEMBLY__arch/riscv/include/asm/hwcap.h +1
-0@@ -59,6 +59,7 @@ enum riscv_isa_ext_id {RISCV_ISA_EXT_ZIHINTPAUSE ,RISCV_ISA_EXT_SSTC ,RISCV_ISA_EXT_SVINVAL , +RISCV_ISA_EXT_ZBB , RISCV_ISA_EXT_ID_MAX };static_assert (RISCV_ISA_EXT_ID_MAX <=RISCV_ISA_EXT_MAX );arch/riscv/include/asm/string.h +2
-0@@ -6,6 +6,8 @@#ifndef _ASM_RISCV_STRING_H#define _ASM_RISCV_STRING_H +#include <asm/alternative-macros.h> +#include <asm/errata_list.h> #include <linux/types.h> #include <linux/linkage.h> arch/riscv/kernel/cpu.c +1
-0@@ -162,6 +162,7 @@ arch_initcall(riscv_cpuinfo_init);* extensions by an underscore. */ static struct riscv_isa_ext_data isa_ext_arr[] = { +__RISCV_ISA_EXT_DATA (zbb,RISCV_ISA_EXT_ZBB ),__RISCV_ISA_EXT_DATA (sscofpmf,RISCV_ISA_EXT_SSCOFPMF ),__RISCV_ISA_EXT_DATA (sstc,RISCV_ISA_EXT_SSTC ),__RISCV_ISA_EXT_DATA (svinval,RISCV_ISA_EXT_SVINVAL ),arch/riscv/kernel/cpufeature.c +18
-0@@ -222,6 +222,7 @@ void __init riscv_fill_hwcap(void)set_bit (nr, this_isa); } }else { +SET_ISA_EXT_MAP ("zbb" ,RISCV_ISA_EXT_ZBB );SET_ISA_EXT_MAP ("sscofpmf" ,RISCV_ISA_EXT_SSCOFPMF );SET_ISA_EXT_MAP ("svpbmt" ,RISCV_ISA_EXT_SVPBMT );SET_ISA_EXT_MAP ("zicbom" ,RISCV_ISA_EXT_ZICBOM );@@ -301,6 +302,20 @@ static bool __init_or_module cpufeature_probe_zicbom(unsigned int stage)return true; } +static bool __init_or_module cpufeature_probe_zbb(unsigned int stage) +{ +if (!IS_ENABLED (CONFIG_RISCV_ISA_ZBB )) +return false; + +if (stage ==RISCV_ALTERNATIVES_EARLY_BOOT ) +return false; + +if (!riscv_isa_extension_available (NULL ,ZBB )) +return false; + +return true; +} +/* * Probe presence of individual extensions. * @@ -318,6 +333,9 @@ static u32 __init_or_module cpufeature_probe(unsigned int stage)if (cpufeature_probe_zicbom (stage)) cpu_req_feature |=BIT (CPUFEATURE_ZICBOM ); +if (cpufeature_probe_zbb (stage)) + cpu_req_feature |=BIT (CPUFEATURE_ZBB ); +return cpu_req_feature; }arch/riscv/lib/strcmp.S +2
-0@@ -3,9 +3,14 @@ #include <linux/linkage.h> #include <asm/asm.h> #include <asm-generic/export.h> +#include <asm/alternative-macros.h> +#include <asm/errata_list.h>
Quoted 6 lines
arch/riscv/lib/strcmp.S line 12
Quoted 4 lines
arch/riscv/lib/strcmp.S +2
-0/* int strcmp(const char *cs, const char *ct) */ SYM_FUNC_START(strcmp) + + ALTERNATIVE("nop", "j variant_zbb", 0, CPUFEATURE_ZBB, CONFIG_RISCV_ISA_ZBB)How about
strcmp_zbband similar for the other functions for the labels?
agreed, that is a way more sensible naming
Quoted 19 lines
Quoted 19 lines
arch/riscv/lib/strcmp.S +12
-0+ /* * Returns * a0 - comparison result, value like strcmp@@ -34,4 +39,93 @@ SYM_FUNC_START(strcmp) bnez t1, 1b li a0, 0 j 2b + +/* + * Variant of strcmp using the ZBB extension if available + */ +#ifdef CONFIG_RISCV_ISA_ZBB +variant_zbb: +#define src1 a0 +#define result a0 +#define src2 t5 +#define data1 t0 +#define data2 t1
Quoted 8 lines
arch/riscv/lib/strcmp.S SYM_FUNC_START(strcmp) · line 55
Quoted 3 lines
arch/riscv/lib/strcmp.S +3
-0+#define align t2 +#define data1_orcb t3 +#define m1 t4nit: It's probably just me, but I'm not a big fan of creating defines for registers, particularly when the meaning of the registers' contents gets changed, because then the name, at least temporarily, no longer accurately describes the content.
I just had to scroll up and down a lot between these defines and the parts using it, cause I couldn't really remember which register that was :-)
Also these somewhat generic names the produce conflicts in my zbb+fast-unaligned variant, so would need more prefixes.
And I actually find it harder to read the assembly when the names are these different lengths instead of the usual 2 characters and things are neatly aligned in the assembly.
So in sum, I decided to follow that suggestion and dropped the defines.
Quoted 147 lines
And, in this case, it looks like the code matches Appendix A, except the register and label naming. I'd prefer to have those match too so I could "review" with the diff tool.
Quoted 14 lines
arch/riscv/lib/strcmp.S +14
-0+ +.option push +.option arch,+zbb + + /* + * Returns + * a0 - comparison result, value like strcmp + * + * Parameters + * a0 - string1 + * a1 - string2 + * + * Clobbers + * t0, t1, t2, t3, t4, t5arch/riscv/lib/strcmp.S SYM_FUNC_START(strcmp) · line 79
Quoted 10 lines
arch/riscv/lib/strcmp.S +10
-0+ */ + mv src2, a1 + + or align, src1, src2 + li m1, -1 + and align, align, SZREG-1 + bnez align, 3f + + /* Main loop for aligned string. */ + .p2align 3Why are the starts of the loops aligned to 8 byte boundaries?
Quoted 37 lines
arch/riscv/lib/strcmp.S +37
-0+1: + REG_L data1, 0(src1) + REG_L data2, 0(src2) + orc.b data1_orcb, data1 + bne data1_orcb, m1, 2f + addi src1, src1, SZREG + addi src2, src2, SZREG + beq data1, data2, 1b + + /* + * Words don't match, and no null byte in the first + * word. Get bytes in big-endian order and compare. + */ +#ifndef CONFIG_CPU_BIG_ENDIAN + rev8 data1, data1 + rev8 data2, data2 +#endif + + /* Synthesize (data1 >= data2) ? 1 : -1 in a branchless sequence. */ + sltu result, data1, data2 + neg result, result + ori result, result, 1 + ret + +2: + /* + * Found a null byte. + * If words don't match, fall back to simple loop. + */ + bne data1, data2, 3f + + /* Otherwise, strings are equal. */ + li result, 0 + ret + + /* Simple loop for misaligned strings. */ + .p2align 3arch/riscv/lib/strcmp.S SYM_FUNC_START(strcmp) · line 126
Quoted 10 lines
arch/riscv/lib/strcmp.S +10
-0+3: + lbu data1, 0(src1) + lbu data2, 0(src2) + addi src1, src1, 1 + addi src2, src2, 1 + bne data1, data2, 4f + bnez data1, 3b + +4: + sub result, data1, data2The non-optimized version returns explicitly -1, 0, 1, whereas this returns < 0, 0, > 0. Assuming we don't need the explicit -1 / 1 then we can change the non-optimized version to do this subtraction for its return value too, saving several instructions.
Quoted 63 lines
arch/riscv/lib/strcmp.S +4
-0+ ret + +.option pop +#endif SYM_FUNC_END(strcmp)arch/riscv/lib/strlen.S +40
-0@@ -3,9 +3,14 @@ #include <linux/linkage.h> #include <asm/asm.h> #include <asm-generic/export.h> +#include <asm/alternative-macros.h> +#include <asm/errata_list.h> /* int strlen(const char *s) */ SYM_FUNC_START(strlen) + + ALTERNATIVE("nop", "j variant_zbb", 0, CPUFEATURE_ZBB, CONFIG_RISCV_ISA_ZBB) + /* * Returns * a0 - string length@@ -25,4 +30,113 @@ SYM_FUNC_START(strlen) 2: addi t1, t1, 1 j 1b + +/* + * Variant of strlen using the ZBB extension if available + */ +#ifdef CONFIG_RISCV_ISA_ZBB +variant_zbb: + +#define src a0 +#define result a0 +#define addr t0 +#define data t1 +#define offset t2 +#define offset_bits t2 +#define valid_bytes t3 +#define m1 t3 + +#ifdef CONFIG_CPU_BIG_ENDIAN +# define CZ clz +# define SHIFT sll +#else +# define CZ ctz +# define SHIFT srl +#endif + +.option push +.option arch,+zbb + + /* + * Returns + * a0 - string length + * + * Parameters + * a0 - String to measure + * + * Clobbers
Quoted 8 lines
arch/riscv/lib/strlen.S SYM_FUNC_START(strlen) · line 71
Quoted 4 lines
arch/riscv/lib/strlen.S +4
-0+ * t0, t1, t2, t3 + */ + + /* Number of irrelevant bytes in the first word. */I see that this comment doesn't just apply to the next instruction, but to several instructions. It threw me off at first. Again, I think I'd prefer we just copy+paste the Appendix, comments an all.
Hmm, somehow for me as non-native-speaker in assembly [ ;-) ], using temporary registers for temporary variables feels somehow more intuitive.
Quoted 4 lines
Quoted 4 lines
arch/riscv/lib/strlen.S +4
-0+ andi offset, src, SZREG-1 + + /* Align pointer. */ + andi addr, src, -SZREG
Quoted 8 lines
arch/riscv/lib/strlen.S SYM_FUNC_START(strlen) · line 79
Quoted 4 lines
arch/riscv/lib/strlen.S +4
-0+ + li valid_bytes, SZREG + sub valid_bytes, valid_bytes, offset + slli offset_bits, offset, RISCV_LGPTRThe shift immediate should be 3, even for rv32, since we want bits-per-byte, not bytes-per-word. The spec example uses PTRLOG, which does imply bytes-per-word to me as well, so that seems like a spec bug.
fixed this according to your's and Christoph's comment
Quoted 10 lines
arch/riscv/lib/strlen.S SYM_FUNC_START(strlen) · line 87
Quoted 8 lines
arch/riscv/lib/strlen.S +8
-0+ + /* Get the first word. */ + REG_L data, 0(addr) + + /* + * Shift away the partial data we loaded to remove the irrelevant bytes + * preceding the string with the effect of adding NUL bytes at the + * end of the string.the end of the string's first word.
ok
Quoted 11 lines
Quoted 11 lines
arch/riscv/lib/strlen.S +11
-0+ */ + SHIFT data, data, offset_bits + + /* Convert non-NUL into 0xff and NUL into 0x00. */ + orc.b data, data + + /* Convert non-NUL into 0x00 and NUL into 0xff. */ + not data, data + + /* + * Search for the first set bit (corresponding to a NUL byte in the
Quoted 8 lines
arch/riscv/lib/strlen.S SYM_FUNC_START(strlen) · line 104
Quoted 6 lines
arch/riscv/lib/strlen.S +6
-0+ * original chunk). + */ + CZ data, data + + /* + * The first chunk is special: commpare against the numbercompare
ok
Quoted 105 lines
Quoted 94 lines
arch/riscv/lib/strlen.S +37
-0+ * of valid bytes in this chunk. + */ + srli result, data, 3 + bgtu valid_bytes, result, 3f + + /* Prepare for the word comparison loop. */ + addi offset, addr, SZREG + li m1, -1 + + /* + * Our critical loop is 4 instructions and processes data in + * 4 byte or 8 byte chunks. + */ + .p2align 3 +1: + REG_L data, SZREG(addr) + addi addr, addr, SZREG + orc.b data, data + beq data, m1, 1b +2: + not data, data + CZ data, data + + /* Get number of processed words. */ + sub offset, addr, offset + + /* Add number of characters in the first word. */ + add result, result, offset + srli data, data, 3 + + /* Add number of characters in the last word. */ + add result, result, data +3: + ret + +.option pop +#endif SYM_FUNC_END(strlen)arch/riscv/lib/strncmp.S +38
-0@@ -3,9 +3,14 @@ #include <linux/linkage.h> #include <asm/asm.h> #include <asm-generic/export.h> +#include <asm/alternative-macros.h> +#include <asm/errata_list.h> /* int strncmp(const char *cs, const char *ct, size_t count) */ SYM_FUNC_START(strncmp) + + ALTERNATIVE("nop", "j variant_zbb", 0, CPUFEATURE_ZBB, CONFIG_RISCV_ISA_ZBB) + /* * Returns * a0 - comparison result, value like strncmp@@ -37,4 +42,110 @@ SYM_FUNC_START(strncmp) 4: li a0, 0 j 2b + +/* + * Variant of strncmp using the ZBB extension if available + */ +#ifdef CONFIG_RISCV_ISA_ZBB +variant_zbb: + +#define src1 a0 +#define result a0 +#define src2 t6 +#define len a2 +#define data1 t0 +#define data2 t1 +#define align t2 +#define data1_orcb t3 +#define limit t4 +#define m1 t5 + +.option push +.option arch,+zbb + + /* + * Returns + * a0 - comparison result, like strncmp + * + * Parameters + * a0 - string1 + * a1 - string2 + * a2 - number of characters to compare + * + * Clobbers + * t0, t1, t2, t3, t4, t5, t6 + */arch/riscv/lib/strncmp.S SYM_FUNC_START(strncmp) · line 87
Quoted 10 lines
arch/riscv/lib/strncmp.S +10
-0+ mv src2, a1 + + or align, src1, src2 + li m1, -1 + and align, align, SZREG-1 + add limit, src1, len + bnez align, 4f + + /* Adjust limit for fast-path. */ + addi limit, limit, -SZREG
Quoted 7 lines
To avoid unnecessarily writing the last SZREG bytes one at a time when 'len' is SZREG aligned, we should be using the explicitly aligned limit here (limit & ~(SZREG - 1)) rather than just subtracting SZREG. Then, down in the slow-path we use the original limit to finish off if len wasn't aligned. When it was aligned, we'll just immediately branch to the ret. (I suspect many times 'len' will be SZREG aligned, since string buffers are typically allocated with power-of-two sizes).
I followed Christoph's suggestion from his follow-up to your mail to address this.
Quoted 55 lines
Quoted 55 lines
arch/riscv/lib/strncmp.S +55
-0+ + /* Main loop for aligned string. */ + .p2align 3 +1: + bgt src1, limit, 3f + REG_L data1, 0(src1) + REG_L data2, 0(src2) + orc.b data1_orcb, data1 + bne data1_orcb, m1, 2f + addi src1, src1, SZREG + addi src2, src2, SZREG + beq data1, data2, 1b + + /* + * Words don't match, and no null byte in the first + * word. Get bytes in big-endian order and compare. + */ +#ifndef CONFIG_CPU_BIG_ENDIAN + rev8 data1, data1 + rev8 data2, data2 +#endif + + /* Synthesize (data1 >= data2) ? 1 : -1 in a branchless sequence. */ + sltu result, data1, data2 + neg result, result + ori result, result, 1 + ret + +2: + /* + * Found a null byte. + * If words don't match, fall back to simple loop. + */ + bne data1, data2, 3f + + /* Otherwise, strings are equal. */ + li result, 0 + ret + + /* Simple loop for misaligned strings. */ +3: + /* Restore limit for slow-path. */ + addi limit, limit, SZREG + .p2align 3 +4: + bge src1, limit, 6f + lbu data1, 0(src1) + lbu data2, 0(src2) + addi src1, src1, 1 + addi src2, src2, 1 + bne data1, data2, 5f + bnez data1, 4b + +5: + sub result, data1, data2
Quoted 8 lines
arch/riscv/lib/strncmp.S SYM_FUNC_START(strncmp) · line 147
Quoted 5 lines
arch/riscv/lib/strncmp.S +5
-0+ ret + +6: + li result, 0 + retAs strcmp and strncmp have so much in common it's tempting to try and merge them somehow. Maybe with a handful of macros shared between them?
I guess the question is does this make things easier to read or harder.
As mentioned above, going between register defines and code was hard already. With at least a strcmp-zbb-unaligned variant in my follow-up and possible more future variants, my feeling is to leave strcmp, strncmp in their own .S files for readability (and how everyone else is doing it) and thus sharing macros between them would require a shared header then.
Without comments strncmp_zbb is like 46 lines long, so right now it feels like splitting things up would make it harder to follow, but I could be wrong :-) .
Heiko
_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv
Quoted 13 lines · Heiko Stuebner
From: Heiko Stuebner <[email protected]>
This series still tries to allow optimized string functions for specific extensions. The last approach of using an inline base function to hold the alternative calls did cause some issues in a number of places
So instead of that we're now just using an alternative j at the beginning of the generic function to jump to a separate place inside the function itself.
This of course needs a fixup for "j" instructions in alternative blocks, so that is provided here as well.
Quoted 8 lines
Technically patch4 got a review from Andrew, but that was still with the inline approach, so I didn't bring it over to v4.
changes since v3:
- rebase on top of 6.2-rc1 + the applied alternative-call series
- add alternative fixup for jal instructions
- drop the inline functions and instead just jump
Hi Heiko,
I think I have sent out the jal fixup a month ago. And similar as V3, Zbb series need to be split from the general alternative improvement.
Thanks
Quoted 43 lines
changes since v2:
- add patch fixing the c.jalr funct4 value
- reword some commit messages
- fix position of auipc addition patch (earlier)
- fix compile errors from patch-reordering gone wrong (worked at the end of v2, but compiling individual patches caused issues) - patches are now tested individually
- limit Zbb variants for GNU as for now
(LLVM support for .option arch is still under review)
- prevent str-functions from getting optimized to builtin-variantschanges since v1:
- a number of generalizations/cleanups for instruction parsing
- use accessor function to access instructions (Emil)
- actually patch the correct location when having more than one instruction in an alternative block
- string function cleanups (comments etc) (Conor)
- move zbb extension above s* extensions incpu.clistschanges since rfc:
- make Zbb code actually work
- drop some unneeded patches
- a lot of cleanupsHeiko Stuebner (5):
RISC-V: move some stray__RISCV_INSN_FUNCSdefinitions from kprobes
RISC-V: add helpers for J-type immediate handling
RISC-V: fix jal addresses in patched alternatives
RISC-V: add infrastructure to allow different str* implementations
RISC-V: add zbb support to string functionsarch/riscv/Kconfig | 24 ++++ arch/riscv/include/asm/errata_list.h | 3 +- arch/riscv/include/asm/hwcap.h | 1 + arch/riscv/include/asm/insn.h | 36 ++++++ arch/riscv/include/asm/string.h | 12 ++ arch/riscv/kernel/alternative.c | 27 ++++ arch/riscv/kernel/cpu.c | 1 + arch/riscv/kernel/cpufeature.c | 18 +++ arch/riscv/kernel/probes/simulate-insn.h | 3 - arch/riscv/kernel/riscv_ksyms.c | 3 + arch/riscv/lib/Makefile | 3 +
Quoted 12 lines
arch/riscv/lib/strcmp.S | 131 ++++++++++++++++++++ arch/riscv/lib/strlen.S | 142 +++++++++++++++++++++ arch/riscv/lib/strncmp.S | 151 +++++++++++++++++++++++ arch/riscv/purgatory/Makefile | 13 ++ 15 files changed, 564 insertions(+), 4 deletions(-) create mode 100644 arch/riscv/lib/strcmp.S create mode 100644 arch/riscv/lib/strlen.S create mode 100644 arch/riscv/lib/strncmp.SSignature · 1 line
2.35.1
_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv