riscv: convert bottom half of exception handling to C
Subsystems (2)
v2 cover riscv: convert bottom half of exception handling to C
For readability, maintainability and future scalability, convert the bottom half of the exception handling to C.
During the conversion, I found Anton fixed a performance issue[1] and my patches will touch the same exception asm code, so this series is applied against Anton's patch.
patch1 and patch2 are cleanup patches. patch3~patch7 do the conversion and some cleanup/optimization after conversion.
Mostly the assembly code is converted to C in a relatively straightforward manner.
However, there are two modifications I need to mention:
1. the cause I.E the CSR_CAUSE value is passed to do_traps() via. 2nd param, do_traps() doesn't get it from pt_regs because this way an extra memory load is needed, the exception handling sits in hot code path, every instruction matters.
2.To cope with SIFIVE_CIP_453 errata, it looks like we don't need alternative mechanism any more after the asm->c conversion. Just replace the excp_vect_table two entries.
since v1:
- add two clean up patches
- remove the two patches which fix imbalance in RAS, instead, explicitly mark this series is applied against one of them.
- save cause in pt_regs prior to calling do_traps()
- do more clean up and optimization after the asm->c conversion:
staticalize and mark do_irq() as __always_inline.
Jisheng Zhang (7):
riscv: traps: staticalize handle_break()
riscv: traps: remove __visible annotation
riscv: convert bottom half of exception handling to C
riscv: errata: remove ALT_INSN_FAULT and ALT_PAGE_FAULT
riscv: errata: sifive: remove NOMMU handling
riscv: staticalize and remove asmlinkage from updated functions
riscv: traps: mark do_irq() as __always_inline
arch/riscv/errata/sifive/errata.c | 25 +++++++--- arch/riscv/errata/sifive/errata_cip_453.S | 4 -- arch/riscv/include/asm/asm-prototypes.h | 20 +------- arch/riscv/include/asm/entry-common.h | 1 - arch/riscv/include/asm/errata_list.h | 21 ++------- arch/riscv/kernel/entry.S | 57 +---------------------- arch/riscv/kernel/kernel_mode_vector.c | 2 +- arch/riscv/kernel/traps.c | 55 ++++++++++++++++++---- 8 files changed, 73 insertions(+), 112 deletions(-)
Signature · 1 line
2.43.0
PATCH v2 1/7 riscv: traps: staticalize handle_break()
handle_break() is only called in traps.c, make it static.
--- arch/riscv/include/asm/entry-common.h | 1 - arch/riscv/kernel/traps.c | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-)
arch/riscv/include/asm/entry-common.h +0 -1
@@ -23,7 +23,6 @@ static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs,#define arch_exit_to_user_mode_prepare arch_exit_to_user_mode_preparevoid handle_page_fault (struct pt_regs *regs); -void handle_break (struct pt_regs *regs);#ifdef CONFIG_RISCV_MISALIGNED int handle_misaligned_load (struct pt_regs *regs);
arch/riscv/kernel/traps.c +1 -1
@@ -270,7 +270,7 @@ static bool probe_breakpoint_handler(struct pt_regs *regs)return user ?uprobe_breakpoint_handler (regs) :kprobe_breakpoint_handler (regs); } -void handle_break (struct pt_regs *regs) +static void handle_break (struct pt_regs *regs) {if (probe_single_step_handler (regs))return ;
Signature · 1 line
2.43.0
PATCH v2 2/7 riscv: traps: remove __visible annotation
In reply to Jisheng ZhangCommit f307307992bf ("riscv: for C functions called only from assembly, mark with __visible") resolve sparse warnings for C functions called only by assembly code by adding __visible annotations instead of adding prototypes. But after commit 030f1dfa8550 ("riscv: traps: Fix no prototype warnings"), prototypes were added. So the __visible annotations are not needed any more, remove them.
--- arch/riscv/kernel/traps.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-)
arch/riscv/kernel/traps.c +7 -7
@@ -147,7 +147,7 @@ static void do_trap_error(struct pt_regs *regs, int signo, int code,#define __trap_section noinstr#endif #define DO_ERROR_INFO (name, signo, code, str) \ -asmlinkage __visible __trap_section void name(struct pt_regs *regs) \ +asmlinkage __trap_section void name(struct pt_regs *regs) \ { \ if (user_mode(regs)) { \ irqentry_enter_from_user_mode(regs); \
@@ -167,7 +167,7 @@ DO_ERROR_INFO(do_trap_insn_misaligned,DO_ERROR_INFO (do_trap_insn_fault,SIGSEGV ,SEGV_ACCERR ,"instruction access fault" ); -asmlinkage __visible __trap_section void do_trap_insn_illegal(struct pt_regs *regs) +asmlinkage __trap_section void do_trap_insn_illegal(struct pt_regs *regs) {bool handled;
@@ -198,7 +198,7 @@ asmlinkage __visible __trap_section void do_trap_insn_illegal(struct pt_regs *reDO_ERROR_INFO (do_trap_load_fault,SIGSEGV ,SEGV_ACCERR ,"load access fault" ); -asmlinkage __visible __trap_section void do_trap_load_misaligned(struct pt_regs *regs) +asmlinkage __trap_section void do_trap_load_misaligned(struct pt_regs *regs) {if (user_mode (regs)) {irqentry_enter_from_user_mode (regs);
@@ -219,7 +219,7 @@ asmlinkage __visible __trap_section void do_trap_load_misaligned(struct pt_regs } } -asmlinkage __visible __trap_section void do_trap_store_misaligned(struct pt_regs *regs) +asmlinkage __trap_section void do_trap_store_misaligned(struct pt_regs *regs) {if (user_mode (regs)) {irqentry_enter_from_user_mode (regs);
@@ -294,7 +294,7 @@ static void handle_break(struct pt_regs *regs)die (regs,"Kernel BUG" ); } -asmlinkage __visible __trap_section void do_trap_break(struct pt_regs *regs) +asmlinkage __trap_section void do_trap_break(struct pt_regs *regs) {if (user_mode (regs)) {irqentry_enter_from_user_mode (regs);
@@ -311,7 +311,7 @@ asmlinkage __visible __trap_section void do_trap_break(struct pt_regs *regs) } } -asmlinkage __visible __trap_section __no_stack_protector +asmlinkage __trap_section __no_stack_protectorvoid do_trap_ecall_u (struct pt_regs *regs) {if (user_mode (regs)) {
@@ -355,7 +355,7 @@ void do_trap_ecall_u(struct pt_regs *regs) }#ifdef CONFIG_MMU -asmlinkage __visible noinstr void do_page_fault(struct pt_regs *regs) +asmlinkage noinstr void do_page_fault(struct pt_regs *regs) {irqentry_state_t state =irqentry_enter (regs);
Signature · 1 line
2.43.0
PATCH v2 3/7 riscv: convert bottom half of exception handling to C
In reply to Jisheng ZhangFor readability, maintainability and future scalability, convert the bottom half of the exception handling to C.
Mostly the assembly code is converted to C in a relatively straightforward manner.
However, there are two modifications I need to mention:
1. the cause I.E the CSR_CAUSE value is passed to do_traps() via. 2nd param, do_traps() doesn't get it from pt_regs because this way an extra memory load is needed, the exception handling sits in hot code path, every instruction matters.
2.To cope with SIFIVE_CIP_453 errata, it looks like we don't need alternative mechanism any more after the asm->c conversion. Just replace the excp_vect_table two entries.
--- arch/riscv/errata/sifive/errata.c | 25 ++++++++--- arch/riscv/include/asm/asm-prototypes.h | 1 + arch/riscv/include/asm/errata_list.h | 5 +-- arch/riscv/kernel/entry.S | 57 +------------------------ arch/riscv/kernel/traps.c | 37 ++++++++++++++++ 5 files changed, 61 insertions(+), 64 deletions(-)
arch/riscv/errata/sifive/errata.c +19 -6
@@ -10,9 +10,14 @@#include <linux/bug.h> #include <asm/patch.h> #include <asm/alternative.h> +#include <asm/csr.h> #include <asm/vendorid_list.h> #include <asm/errata_list.h> +extern void (*excp_vect_table[])(struct pt_regs *regs); +extern void sifive_cip_453_insn_fault_trp (struct pt_regs *regs); +extern void sifive_cip_453_page_fault_trp (struct pt_regs *regs); +struct errata_info_t {char name[32 ];bool (*check_func)(unsigned long arch_id,unsigned long impid);
@@ -20,6 +25,9 @@ struct errata_info_t {static bool errata_cip_453_check_func (unsigned long arch_id,unsigned long impid) { +if (!IS_ENABLED (CONFIG_ERRATA_SIFIVE_CIP_453 )) +return false; +/* * Affected cores: * Architecture ID: 0x8000000000000007
@@ -51,10 +59,6 @@ static bool errata_cip_1200_check_func(unsigned long arch_id, unsigned long imp }static struct errata_info_t errata_list[ERRATA_SIFIVE_NUMBER ] = { - { - .name ="cip-453" , - .check_func = errata_cip_453_check_func - }, { .name ="cip-1200" , .check_func = errata_cip_1200_check_func
@@ -62,11 +66,20 @@ static struct errata_info_t errata_list[ERRATA_SIFIVE_NUMBER] = { }; static u32 __init_or_module sifive_errata_probe(unsigned long archid, -unsigned long impid) +unsigned long impid, +unsigned int stage) {int idx;u32 cpu_req_errata =0 ; +if (stage ==RISCV_ALTERNATIVES_BOOT ) { +if (IS_ENABLED (CONFIG_MMU ) && +errata_cip_453_check_func (archid, impid)) { + excp_vect_table[EXC_INST_ACCESS ] = sifive_cip_453_insn_fault_trp; + excp_vect_table[EXC_INST_PAGE_FAULT ] = sifive_cip_453_page_fault_trp; + } + } +for (idx =0 ; idx <ERRATA_SIFIVE_NUMBER ; idx++)if (errata_list[idx].check_func (archid, impid)) cpu_req_errata |= (1U << idx);
@@ -99,7 +112,7 @@ void sifive_errata_patch_func(struct alt_entry *begin, struct alt_entry *end,if (stage ==RISCV_ALTERNATIVES_EARLY_BOOT )return ; - cpu_req_errata =sifive_errata_probe (archid, impid); + cpu_req_errata =sifive_errata_probe (archid, impid, stage);for (alt = begin; alt < end; alt++) {if (alt->vendor_id !=SIFIVE_VENDOR_ID )
arch/riscv/include/asm/asm-prototypes.h +1 -0
@@ -55,5 +55,6 @@ DECLARE_DO_ERROR_INFO(do_trap_break); asmlinkage void handle_bad_stack(struct pt_regs *regs); asmlinkage void do_page_fault(struct pt_regs *regs); asmlinkage void do_irq(struct pt_regs *regs); +asmlinkage void do_traps(struct pt_regs *regs, unsigned long cause);#endif /* _ASM_RISCV_PROTOTYPES_H */
arch/riscv/include/asm/errata_list.h +2 -3
@@ -17,9 +17,8 @@#endif #ifdef CONFIG_ERRATA_SIFIVE -#define ERRATA_SIFIVE_CIP_453 0 -#define ERRATA_SIFIVE_CIP_1200 1 -#define ERRATA_SIFIVE_NUMBER 2 +#define ERRATA_SIFIVE_CIP_1200 0 +#define ERRATA_SIFIVE_NUMBER 1#endif #ifdef CONFIG_ERRATA_THEAD
arch/riscv/kernel/entry.S +2 -55
71 lines not on this page · raw message
arch/riscv/kernel/traps.c +37 -0
@@ -390,6 +390,43 @@ asmlinkage void noinstr do_irq(struct pt_regs *regs)irqentry_exit (regs, state); } +void (*excp_vect_table[])(struct pt_regs *regs)__ro_after_init = { + do_trap_insn_misaligned,/* 0 Instruction address misaligned */ + do_trap_insn_fault,/* 1 Instruction access fault */ + do_trap_insn_illegal,/* 2 Illegal instruction */ + do_trap_break,/* 3 Breakpoint */ + do_trap_load_misaligned,/* 4 Load address misaligned */ + do_trap_load_fault,/* 5 Load access fault */ + do_trap_store_misaligned,/* 6 Store/AMO address misaligned */ + do_trap_store_fault,/* 7 Store/AMO access fault */ + do_trap_ecall_u,/* 8 Environment call from U-mode */ + do_trap_ecall_s,/* 9 Environment call from S-mode */ + do_trap_unknown,/* 10 Reserved */ + do_trap_ecall_m,/* 11 Environment call from M-mode */ +#ifdef CONFIG_MMU + do_page_fault,/* 12 Instruciton page fault */ + do_page_fault,/* 13 Load page fault */ + do_trap_unknown,/* 14 Reserved */ + do_page_fault,/* 15 Store/AMO page fault */ +#endif +}; + +asmlinkage void noinstr do_traps(struct pt_regs *regs, unsigned long cause) +{ +#ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE +riscv_v_context_nesting_start (regs); +#endif +if (cause &CAUSE_IRQ_FLAG ) { +do_irq (regs); + }else { +if (cause >=ARRAY_SIZE (excp_vect_table)) { +do_trap_unknown (regs); +return ; + } + excp_vect_table[cause](regs); + } +} +#ifdef CONFIG_GENERIC_BUG int is_valid_bugaddr (unsigned long pc) {
Signature · 1 line
2.43.0
Quoted 17 lines · Jisheng Zhang
For readability, maintainability and future scalability, convert the bottom half of the exception handling to C.
Mostly the assembly code is converted to C in a relatively straightforward manner.
However, there are two modifications I need to mention:
1. the cause I.E the
CSR_CAUSEvalue is passed todo_traps()via. 2nd param,do_traps()doesn't get it frompt_regsbecause this way an extra memory load is needed, the exception handling sits in hot code path, every instruction matters.2.To cope with
SIFIVE_CIP_453errata, it looks like we don't need alternative mechanism any more after theasm->cconversion. Just replace theexcp_vect_tabletwo entries.
Quoted 9 lines
Signed-off-by Jisheng Zhang <[email protected]>---
arch/riscv/errata/sifive/errata.c | 25 ++++++++--- arch/riscv/include/asm/asm-prototypes.h | 1 + arch/riscv/include/asm/errata_list.h | 5 +-- arch/riscv/kernel/entry.S | 57 +------------------------ arch/riscv/kernel/traps.c | 37 ++++++++++++++++ 5 files changed, 61 insertions(+), 64 deletions(-)
Sorry for noticing it so late. And thanks to Bjorn for pointing it out. lgtm.
PATCH v2 4/7 riscv: errata: remove ALT_INSN_FAULT and ALT_PAGE_FAULT
In reply to Jisheng ZhangThey are used for SIFIVE_CIP_453 errata, which has been solved by replacing the excp_vect_table[] two entries in last commit. So these two macros are useless now, remove them.
--- arch/riscv/include/asm/errata_list.h | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-)
arch/riscv/include/asm/errata_list.h +1 -15
@@ -27,21 +27,7 @@#define ERRATA_THEAD_NUMBER 2#endif -#ifdef __ASSEMBLY__ - -#define ALT_INSN_FAULT (x) \ -ALTERNATIVE(__stringify(RISCV_PTR do_trap_insn_fault), \ - __stringify(RISCV_PTR sifive_cip_453_insn_fault_trp), \ - SIFIVE_VENDOR_ID, ERRATA_SIFIVE_CIP_453, \ - CONFIG_ERRATA_SIFIVE_CIP_453) - -#define ALT_PAGE_FAULT (x) \ -ALTERNATIVE(__stringify(RISCV_PTR do_page_fault), \ - __stringify(RISCV_PTR sifive_cip_453_page_fault_trp), \ - SIFIVE_VENDOR_ID, ERRATA_SIFIVE_CIP_453, \ - CONFIG_ERRATA_SIFIVE_CIP_453) -#else /* !__ASSEMBLY__ */ - +#ifndef __ASSEMBLY__#define ALT_SFENCE_VMA_ASID (asid) \ asm(ALTERNATIVE("sfence.vma x0, %0", "sfence.vma", SIFIVE_VENDOR_ID, \ ERRATA_SIFIVE_CIP_1200, CONFIG_ERRATA_SIFIVE_CIP_1200) \
Signature · 1 line
2.43.0
PATCH v2 5/7 riscv: errata: sifive: remove NOMMU handling
In reply to Jisheng ZhangSince NOMMU is now properly handling in generic do_traps() which will call do_trap_unknown() for instruciton page fault for NOMMU.
--- arch/riscv/errata/sifive/errata_cip_453.S | 4 ---- 1 file changed, 4 deletions(-)
arch/riscv/errata/sifive/errata_cip_453.S +0 -4
@@ -23,11 +23,7 @@ SYM_FUNC_START(sifive_cip_453_page_fault_trp) ADD_SIGN_EXT a0, t0, t1 -#ifdef CONFIG_MMU la t0, do_page_fault -#else - la t0, do_trap_unknown -#endif jr t0 SYM_FUNC_END(sifive_cip_453_page_fault_trp)
Signature · 1 line
2.43.0
PATCH v2 6/7 riscv: staticalize and remove asmlinkage from updated functions
In reply to Jisheng ZhangNow that the callers of these functions have moved into C, they are only called in trap.c and no longer need the asmlinkage annotation. So make them static and remove asmlinkage from them.
--- arch/riscv/include/asm/asm-prototypes.h | 19 +------------------ arch/riscv/kernel/kernel_mode_vector.c | 2 +- arch/riscv/kernel/traps.c | 16 ++++++++-------- 3 files changed, 10 insertions(+), 27 deletions(-)
arch/riscv/include/asm/asm-prototypes.h +1 -18
@@ -31,30 +31,13 @@ void xor_regs_5_(unsigned long bytes, unsigned long *__restrict p1, const unsigned long *__restrict p5);#ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE -asmlinkage void riscv_v_context_nesting_start(struct pt_regs *regs); +void riscv_v_context_nesting_start (struct pt_regs *regs); asmlinkage void riscv_v_context_nesting_end(struct pt_regs *regs);#endif /* CONFIG_RISCV_ISA_V_PREEMPTIVE */ #endif /* CONFIG_RISCV_ISA_V */ -#define DECLARE_DO_ERROR_INFO (name) asmlinkage void name(struct pt_regs *regs) - -DECLARE_DO_ERROR_INFO (do_trap_unknown); -DECLARE_DO_ERROR_INFO (do_trap_insn_misaligned); -DECLARE_DO_ERROR_INFO (do_trap_insn_fault); -DECLARE_DO_ERROR_INFO (do_trap_insn_illegal); -DECLARE_DO_ERROR_INFO (do_trap_load_fault); -DECLARE_DO_ERROR_INFO (do_trap_load_misaligned); -DECLARE_DO_ERROR_INFO (do_trap_store_misaligned); -DECLARE_DO_ERROR_INFO (do_trap_store_fault); -DECLARE_DO_ERROR_INFO (do_trap_ecall_u); -DECLARE_DO_ERROR_INFO (do_trap_ecall_s); -DECLARE_DO_ERROR_INFO (do_trap_ecall_m); -DECLARE_DO_ERROR_INFO (do_trap_break); - asmlinkage void handle_bad_stack(struct pt_regs *regs); -asmlinkage void do_page_fault(struct pt_regs *regs); -asmlinkage void do_irq(struct pt_regs *regs); asmlinkage void do_traps(struct pt_regs *regs, unsigned long cause);#endif /* _ASM_RISCV_PROTOTYPES_H */
arch/riscv/kernel/kernel_mode_vector.c +1 -1
@@ -152,7 +152,7 @@ static int riscv_v_start_kernel_context(bool *is_nested) }/* low-level V context handling code, called with irq disabled */ -asmlinkage void riscv_v_context_nesting_start(struct pt_regs *regs) +void riscv_v_context_nesting_start (struct pt_regs *regs) {int depth;
arch/riscv/kernel/traps.c +8 -8
@@ -147,7 +147,7 @@ static void do_trap_error(struct pt_regs *regs, int signo, int code,#define __trap_section noinstr#endif #define DO_ERROR_INFO (name, signo, code, str) \ -asmlinkage __trap_section void name(struct pt_regs *regs) \ +static __trap_section void name(struct pt_regs *regs) \ { \ if (user_mode(regs)) { \ irqentry_enter_from_user_mode(regs); \
@@ -167,7 +167,7 @@ DO_ERROR_INFO(do_trap_insn_misaligned,DO_ERROR_INFO (do_trap_insn_fault,SIGSEGV ,SEGV_ACCERR ,"instruction access fault" ); -asmlinkage __trap_section void do_trap_insn_illegal(struct pt_regs *regs) +static __trap_section void do_trap_insn_illegal(struct pt_regs *regs) {bool handled;
@@ -198,7 +198,7 @@ asmlinkage __trap_section void do_trap_insn_illegal(struct pt_regs *regs)DO_ERROR_INFO (do_trap_load_fault,SIGSEGV ,SEGV_ACCERR ,"load access fault" ); -asmlinkage __trap_section void do_trap_load_misaligned(struct pt_regs *regs) +static __trap_section void do_trap_load_misaligned(struct pt_regs *regs) {if (user_mode (regs)) {irqentry_enter_from_user_mode (regs);
@@ -219,7 +219,7 @@ asmlinkage __trap_section void do_trap_load_misaligned(struct pt_regs *regs) } } -asmlinkage __trap_section void do_trap_store_misaligned(struct pt_regs *regs) +static __trap_section void do_trap_store_misaligned(struct pt_regs *regs) {if (user_mode (regs)) {irqentry_enter_from_user_mode (regs);
@@ -294,7 +294,7 @@ static void handle_break(struct pt_regs *regs)die (regs,"Kernel BUG" ); } -asmlinkage __trap_section void do_trap_break(struct pt_regs *regs) +static __trap_section void do_trap_break(struct pt_regs *regs) {if (user_mode (regs)) {irqentry_enter_from_user_mode (regs);
@@ -311,7 +311,7 @@ asmlinkage __trap_section void do_trap_break(struct pt_regs *regs) } } -asmlinkage __trap_section __no_stack_protector +static __trap_section __no_stack_protectorvoid do_trap_ecall_u (struct pt_regs *regs) {if (user_mode (regs)) {
@@ -355,7 +355,7 @@ void do_trap_ecall_u(struct pt_regs *regs) }#ifdef CONFIG_MMU -asmlinkage noinstr void do_page_fault(struct pt_regs *regs) +static noinstr void do_page_fault(struct pt_regs *regs) {irqentry_state_t state =irqentry_enter (regs);
@@ -378,7 +378,7 @@ static void noinstr handle_riscv_irq(struct pt_regs *regs)irq_exit_rcu (); } -asmlinkage void noinstr do_irq(struct pt_regs *regs) +static void noinstr do_irq(struct pt_regs *regs) {irqentry_state_t state =irqentry_enter (regs);
Signature · 1 line
2.43.0
PATCH v2 7/7 riscv: traps: mark do_irq() as __always_inline
In reply to Jisheng ZhangSince do_irq() is only called in traps.c, so mark it as __always_inline this will allow the compiler to get rid of the stack setup/tear down code and eliminate a handful of instructions.
--- arch/riscv/kernel/traps.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/riscv/kernel/traps.c +1 -1
@@ -378,7 +378,7 @@ static void noinstr handle_riscv_irq(struct pt_regs *regs)irq_exit_rcu (); } -static void noinstr do_irq(struct pt_regs *regs) +static __always_inline void do_irq(struct pt_regs *regs) {irqentry_state_t state =irqentry_enter (regs);
Signature · 1 line
2.43.0