xtensa: add load/store exception handler
Subsystems (2)
Max Filippov
maintainercover xtensa: add load/store exception handler
Hello,
this series rearranges unaligned exception handler and adds load/store exception handler that allows transparent 1- and 2-byte wide reads from memory attached to an instruction bus of an xtensa core.
Max Filippov (4):
xtensa: move early_trap_init from kasan_early_init to init_arch
xtensa: always install slow handler for unaligned access exception
xtensa: rearrange unaligned exception handler
xtensa: add load/store exception handler
arch/xtensa/Kconfig | 12 ++ arch/xtensa/include/asm/traps.h | 7 + arch/xtensa/kernel/align.S | 256 ++++++++++++++++++++++---------- arch/xtensa/kernel/setup.c | 7 + arch/xtensa/kernel/traps.c | 27 +++- arch/xtensa/mm/kasan_init.c | 2 - 6 files changed, 221 insertions(+), 90 deletions(-)
Signature · 1 line
2.30.2
Max Filippov
maintainerPATCH 1/4 xtensa: move early_trap_init from kasan_early_init to init_arch
There may be other users for the early traps besides KASAN. Move call to the early_trap_init from kasan_early_init. Protect init_exc_table initializer with ifdef to make sure it builds on noMMU configurations.
--- arch/xtensa/include/asm/traps.h | 2 ++ arch/xtensa/kernel/setup.c | 6 ++++++ arch/xtensa/mm/kasan_init.c | 2 -- 3 files changed, 8 insertions(+), 2 deletions(-)
arch/xtensa/include/asm/traps.h +2 -0
@@ -64,8 +64,10 @@ void do_unhandled(struct pt_regs *regs);static inline void __init early_trap_init (void ) {static struct exc_table init_exc_table__initdata = { +#ifdef CONFIG_MMU .fast_kernel_handler[EXCCAUSE_DTLB_MISS ] = fast_second_level_miss, +#endif };xtensa_set_sr (&init_exc_table, excsave1); }
arch/xtensa/kernel/setup.c +6 -0
@@ -47,6 +47,7 @@#include <asm/smp.h> #include <asm/sysmem.h> #include <asm/timex.h> +#include <asm/traps.h> #if defined(CONFIG_VGA_CONSOLE ) || defined(CONFIG_DUMMY_CONSOLE ) struct screen_info screen_info = {
@@ -242,6 +243,11 @@ void __init early_init_devtree(void *params)void __init init_arch (bp_tag_t *bp_start) { +/* Initialize basic exception handling if configuration may need it */ + +if (IS_ENABLED (CONFIG_KASAN )) +early_trap_init (); +/* Initialize MMU. */ init_mmu ();
arch/xtensa/mm/kasan_init.c +0 -2
@@ -14,7 +14,6 @@#include <linux/kernel.h> #include <asm/initialize_mmu.h> #include <asm/tlbflush.h> -#include <asm/traps.h> void __init kasan_early_init (void ) {
@@ -31,7 +30,6 @@ void __init kasan_early_init(void)BUG_ON (!pmd_none (*pmd));set_pmd (pmd,__pmd ((unsigned long )kasan_early_shadow_pte)); } -early_trap_init (); }static void __init populate (void *start,void *end)
Signature · 1 line
2.30.2
Max Filippov
maintainerPATCH 2/4 xtensa: always install slow handler for unaligned access exception
In reply to Max FilippovCurrently slow handler for unaligned access exception is not installed when CPU has hardware support for unaligned access. However some opcodes (e.g. l32ai, s32ri, s32c1i) would still raise unaligned access exception even on such CPUs. In that case instead of SIGBUS and a diagnostic entry in the kernel log the faulting process would receive SIGILL. Always install slow handler for unaligned access exception to fix that.
--- arch/xtensa/kernel/traps.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-)
arch/xtensa/kernel/traps.c +1 -5
@@ -54,9 +54,7 @@ static void do_interrupt(struct pt_regs *regs);#if XTENSA_FAKE_NMI static void do_nmi (struct pt_regs *regs);#endif -#if XCHAL_UNALIGNED_LOAD_EXCEPTION ||XCHAL_UNALIGNED_STORE_EXCEPTION static void do_unaligned_user (struct pt_regs *regs); -#endif static void do_multihit (struct pt_regs *regs);#if XTENSA_HAVE_COPROCESSORS static void do_coprocessor (struct pt_regs *regs);
@@ -102,9 +100,9 @@ static dispatch_init_table_t __initdata dispatch_init_table[] = { #ifdef CONFIG_XTENSA_UNALIGNED_USER { EXCCAUSE_UNALIGNED, USER, fast_unaligned }, #endif -{ EXCCAUSE_UNALIGNED, 0, do_unaligned_user }, { EXCCAUSE_UNALIGNED, KRNL, fast_unaligned }, #endif +{ EXCCAUSE_UNALIGNED, 0, do_unaligned_user }, #ifdef CONFIG_MMU { EXCCAUSE_ITLB_MISS, 0, do_page_fault }, { EXCCAUSE_ITLB_MISS, USER|KRNL, fast_second_level_miss},
@@ -363,7 +361,6 @@ static void do_div0(struct pt_regs *regs)* accesses causes from user space. */ -#if XCHAL_UNALIGNED_LOAD_EXCEPTION ||XCHAL_UNALIGNED_STORE_EXCEPTION static void do_unaligned_user (struct pt_regs *regs) { __die_if_kernel("Unhandled unaligned exception in kernel",
@@ -375,7 +372,6 @@ static void do_unaligned_user(struct pt_regs *regs) task_pid_nr(current), regs->pc);force_sig_fault (SIGBUS ,BUS_ADRALN , (void *) regs->excvaddr); } -#endif #if XTENSA_HAVE_COPROCESSORS static void do_coprocessor (struct pt_regs *regs)
Signature · 1 line
2.30.2
Max Filippov
maintainerPATCH 3/4 xtensa: rearrange unaligned exception handler
In reply to Max Filippov- extract initialization part of the exception handler into a separate function.
- use single label for invalid instruction instead of two labels, one for load and one for store, at one place.
- use sext instruction for sign extension when available.
- store SAR on the stack instead of in a0.
- replace numeric labels for load and store writeback with .Lload_w and .Lstore_w respectively.
--- arch/xtensa/kernel/align.S | 171 +++++++++++++++++++------------------ 1 file changed, 89 insertions(+), 82 deletions(-)
arch/xtensa/kernel/align.S +89 -82
@@ -157,58 +157,7 @@ .literal_position ENTRY(fast_unaligned) - /* Note: We don't expect the address to be aligned on a word - * boundary. After all, the processor generated that exception - * and it would be a hardware fault. - */ - - /* Save some working register */ - - s32i a4, a2, PT_AREG4 - s32i a5, a2, PT_AREG5 - s32i a6, a2, PT_AREG6 - s32i a7, a2, PT_AREG7 - s32i a8, a2, PT_AREG8 - - rsr a0, depc - s32i a0, a2, PT_AREG2 - s32i a3, a2, PT_AREG3 - - rsr a3, excsave1 - movi a4, fast_unaligned_fixup - s32i a4, a3, EXC_TABLE_FIXUP - - /* Keep value of SAR in a0 */ - - rsr a0, sar - rsr a8, excvaddr # load unaligned memory address - - /* Now, identify one of the following load/store instructions. - * - * The only possible danger of a double exception on the - * following l32i instructions is kernel code in vmalloc - * memory. The processor was just executing at the EPC_1 - * address, and indeed, already fetched the instruction. That - * guarantees a TLB mapping, which hasn't been replaced by - * this unaligned exception handler that uses only static TLB - * mappings. However, high-level interrupt handlers might - * modify TLB entries, so for the generic case, we register a - * TABLE_FIXUP handler here, too. - */ - - /* a3...a6 saved on stack, a2 = SP */ - - /* Extract the instruction that caused the unaligned access. */ - - rsr a7, epc1 # load exception address - movi a3, ~3 - and a3, a3, a7 # mask lower bits - - l32i a4, a3, 0 # load 2 words - l32i a5, a3, 4 - - __ssa8 a7 - __src_b a4, a4, a5 # a4 has the instruction + call0 .Lsave_and_load_instruction /* Analyze the instruction (load or store?). */
@@ -249,7 +198,7 @@ ENTRY(fast_unaligned) addi a7, a7, 2 # increment PC (assume 16-bit insn) extui a5, a4, INSN_OP0, 4 - _beqi a5, OP0_L32I_N, 1f # l32i.n: jump + _beqi a5, OP0_L32I_N, .Lload_w# l32i.n: jump addi a7, a7, 1 #else
@@ -257,21 +206,24 @@ ENTRY(fast_unaligned) #endif extui a5, a4, INSN_OP1, 4 - _beqi a5, OP1_L32I, 1f # l32i: jump + _beqi a5, OP1_L32I, .Lload_w # l32i: jump extui a3, a3, 0, 16 # extract lower 16 bits - _beqi a5, OP1_L16UI, 1f + _beqi a5, OP1_L16UI, .Lload_w addi a5, a5, -OP1_L16SI - _bnez a5, .Linvalid_instruction_load + _bnez a5, .Linvalid_instruction /* sign extend value */ - +#if XCHAL_HAVE_SEXT + sext a3, a3, 15 +#else slli a3, a3, 16 srai a3, a3, 16 +#endif /* Set target register. */ -1: +.Lload_w: extui a4, a4, INSN_T, 4 # extract target register movi a5, .Lload_table addx8 a4, a4, a5
@@ -297,28 +249,27 @@ ENTRY(fast_unaligned) mov a15, a3 ; _j .Lexit; .align 8 .Lstore_table: - l32i a3, a2, PT_AREG0; _j 1f; .align 8 - mov a3, a1; _j 1f; .align 8 # fishy?? - l32i a3, a2, PT_AREG2; _j 1f; .align 8 - l32i a3, a2, PT_AREG3; _j 1f; .align 8 - l32i a3, a2, PT_AREG4; _j 1f; .align 8 - l32i a3, a2, PT_AREG5; _j 1f; .align 8 - l32i a3, a2, PT_AREG6; _j 1f; .align 8 - l32i a3, a2, PT_AREG7; _j 1f; .align 8 - l32i a3, a2, PT_AREG8; _j 1f; .align 8 - mov a3, a9 ; _j 1f; .align 8 - mov a3, a10 ; _j 1f; .align 8 - mov a3, a11 ; _j 1f; .align 8 - mov a3, a12 ; _j 1f; .align 8 - mov a3, a13 ; _j 1f; .align 8 - mov a3, a14 ; _j 1f; .align 8 - mov a3, a15 ; _j 1f; .align 8 + l32i a3, a2, PT_AREG0; _j .Lstore_w; .align 8 + mov a3, a1; _j .Lstore_w; .align 8 # fishy?? + l32i a3, a2, PT_AREG2; _j .Lstore_w; .align 8 + l32i a3, a2, PT_AREG3; _j .Lstore_w; .align 8 + l32i a3, a2, PT_AREG4; _j .Lstore_w; .align 8 + l32i a3, a2, PT_AREG5; _j .Lstore_w; .align 8 + l32i a3, a2, PT_AREG6; _j .Lstore_w; .align 8 + l32i a3, a2, PT_AREG7; _j .Lstore_w; .align 8 + l32i a3, a2, PT_AREG8; _j .Lstore_w; .align 8 + mov a3, a9 ; _j .Lstore_w; .align 8 + mov a3, a10 ; _j .Lstore_w; .align 8 + mov a3, a11 ; _j .Lstore_w; .align 8 + mov a3, a12 ; _j .Lstore_w; .align 8 + mov a3, a13 ; _j .Lstore_w; .align 8 + mov a3, a14 ; _j .Lstore_w; .align 8 + mov a3, a15 ; _j .Lstore_w; .align 8 /* We cannot handle this exception. */ .extern _kernel_exception -.Linvalid_instruction_load: -.Linvalid_instruction_store: +.Linvalid_instruction: movi a4, 0 rsr a3, excsave1
@@ -326,6 +277,7 @@ ENTRY(fast_unaligned) /* Restore a4...a8 and SAR, set SP, and jump to default exception. */ + l32i a0, a2, PT_SAR l32i a8, a2, PT_AREG8 l32i a7, a2, PT_AREG7 l32i a6, a2, PT_AREG6
@@ -343,8 +295,8 @@ ENTRY(fast_unaligned) 2: movi a0, _user_exception jx a0 -1: # a7: instruction pointer, a4: instruction, a3: value - + # a7: instruction pointer, a4: instruction, a3: value +.Lstore_w: movi a6, 0 # mask: ffffffff:00000000 #if XCHAL_HAVE_DENSITY
@@ -361,7 +313,7 @@ ENTRY(fast_unaligned) extui a5, a4, INSN_OP1, 4 # extract OP1 _beqi a5, OP1_S32I, 1f # jump if 32 bit store - _bnei a5, OP1_S16I, .Linvalid_instruction_store + _bnei a5, OP1_S16I, .Linvalid_instruction movi a5, -1 __extl a3, a3 # get 16-bit value
@@ -434,6 +386,7 @@ ENTRY(fast_unaligned) /* Restore working register */ + l32i a0, a2, PT_SAR l32i a8, a2, PT_AREG8 l32i a7, a2, PT_AREG7 l32i a6, a2, PT_AREG6
@@ -448,6 +401,59 @@ ENTRY(fast_unaligned) l32i a2, a2, PT_AREG2 rfe + .align 4 +.Lsave_and_load_instruction: + + /* Save some working register */ + + s32i a3, a2, PT_AREG3 + s32i a4, a2, PT_AREG4 + s32i a5, a2, PT_AREG5 + s32i a6, a2, PT_AREG6 + s32i a7, a2, PT_AREG7 + s32i a8, a2, PT_AREG8 + + rsr a4, depc + s32i a4, a2, PT_AREG2 + + rsr a5, sar + s32i a5, a2, PT_SAR + + rsr a3, excsave1 + movi a4, fast_unaligned_fixup + s32i a4, a3, EXC_TABLE_FIXUP + + rsr a8, excvaddr # load unaligned memory address + + /* Now, identify one of the following load/store instructions. + * + * The only possible danger of a double exception on the + * following l32i instructions is kernel code in vmalloc + * memory. The processor was just executing at the EPC_1 + * address, and indeed, already fetched the instruction. That + * guarantees a TLB mapping, which hasn't been replaced by + * this unaligned exception handler that uses only static TLB + * mappings. However, high-level interrupt handlers might + * modify TLB entries, so for the generic case, we register a + * TABLE_FIXUP handler here, too. + */ + + /* a3...a6 saved on stack, a2 = SP */ + + /* Extract the instruction that caused the unaligned access. */ + + rsr a7, epc1 # load exception address + movi a3, ~3 + and a3, a3, a7 # mask lower bits + + l32i a4, a3, 0 # load 2 words + l32i a5, a3, 4 + + __ssa8 a7 + __src_b a4, a4, a5 # a4 has the instruction + + ret + ENDPROC(fast_unaligned) ENTRY(fast_unaligned_fixup)
@@ -459,10 +465,11 @@ ENTRY(fast_unaligned_fixup) l32i a7, a2, PT_AREG7 l32i a6, a2, PT_AREG6 l32i a5, a2, PT_AREG5 - l32i a4, a2, PT_AREG4 + l32i a4, a2, PT_SAR l32i a0, a2, PT_AREG2 - xsr a0, depc # restore depc and a0 - wsr a0, sar + wsr a4, sar + wsr a0, depc # restore depc and a0 + l32i a4, a2, PT_AREG4 rsr a0, exccause s32i a0, a2, PT_DEPC # mark as a regular exception
Signature · 1 line
2.30.2
Max Filippov
maintainerPATCH 4/4 xtensa: add load/store exception handler
In reply to Max FilippovMemory attached to instruction bus of the xtensa CPU is only accessible for a limited subset of opcodes. Other opcodes generate an exception with the load/store error cause code. This property complicates use of such systems. Provide a handler that recognizes and transparently fixes such exceptions. The following opcodes are recognized when used outside of FLIX bundles: l32i, l32i.n, l16ui, l16si, l8ui.
--- arch/xtensa/Kconfig | 12 ++++ arch/xtensa/include/asm/traps.h | 5 ++ arch/xtensa/kernel/align.S | 109 ++++++++++++++++++++++++++++---- arch/xtensa/kernel/setup.c | 3 +- arch/xtensa/kernel/traps.c | 21 +++++- 5 files changed, 136 insertions(+), 14 deletions(-)
arch/xtensa/Kconfig +12 -0
@@ -209,6 +209,18 @@ config XTENSA_UNALIGNED_USER Say Y here to enable unaligned memory access in user space. +config XTENSA_LOAD_STORE + bool "Load/store exception handler for memory only readable with l32" + help + The Xtensa architecture only allows reading memory attached to its + instruction bus with l32r and l32i instructions, all other + instructions raise an exception with the LoadStoreErrorCause code. + This makes it hard to use some configurations, e.g. store string + literals in FLASH memory attached to the instruction bus. + + Say Y here to enable exception handler that allows transparent + byte and 2-byte access to memory attached to instruction bus. + config HAVE_SMP bool "System Supports SMP (MX)" depends on XTENSA_VARIANT_CUSTOM
arch/xtensa/include/asm/traps.h +5 -0
@@ -47,6 +47,7 @@ __init trap_set_handler(int cause, xtensa_exception_handler *handler); asmlinkage void fast_illegal_instruction_user(void); asmlinkage void fast_syscall_user(void); asmlinkage void fast_alloca(void); +asmlinkage void fast_load_store(void); asmlinkage void fast_unaligned(void); asmlinkage void fast_second_level_miss(void); asmlinkage void fast_store_prohibited(void);
@@ -64,6 +65,10 @@ void do_unhandled(struct pt_regs *regs);static inline void __init early_trap_init (void ) {static struct exc_table init_exc_table__initdata = { +#ifdef CONFIG_XTENSA_LOAD_STORE + .fast_kernel_handler[EXCCAUSE_LOAD_STORE_ERROR ] = + fast_load_store, +#endif #ifdef CONFIG_MMU .fast_kernel_handler[EXCCAUSE_DTLB_MISS ] = fast_second_level_miss,
arch/xtensa/kernel/align.S +97 -12
196 lines not on this page · raw message
arch/xtensa/kernel/setup.c +2 -1
@@ -245,7 +245,8 @@ void __init init_arch(bp_tag_t *bp_start) {/* Initialize basic exception handling if configuration may need it */ -if (IS_ENABLED (CONFIG_KASAN )) +if (IS_ENABLED (CONFIG_KASAN ) || +IS_ENABLED (CONFIG_XTENSA_LOAD_STORE ))early_trap_init ();/* Initialize MMU. */
arch/xtensa/kernel/traps.c +20 -1
@@ -54,6 +54,9 @@ static void do_interrupt(struct pt_regs *regs);#if XTENSA_FAKE_NMI static void do_nmi (struct pt_regs *regs);#endif +#ifdef CONFIG_XTENSA_LOAD_STORE +static void do_load_store (struct pt_regs *regs); +#endif static void do_unaligned_user (struct pt_regs *regs);static void do_multihit (struct pt_regs *regs);#if XTENSA_HAVE_COPROCESSORS
@@ -89,7 +92,10 @@ static dispatch_init_table_t __initdata dispatch_init_table[] = { { EXCCAUSE_SYSTEM_CALL, USER, fast_syscall_user }, { EXCCAUSE_SYSTEM_CALL, 0, system_call },/* EXCCAUSE_INSTRUCTION_FETCH unhandled */ -/* EXCCAUSE_LOAD_STORE_ERROR unhandled*/ +#ifdef CONFIG_XTENSA_LOAD_STORE +{ EXCCAUSE_LOAD_STORE_ERROR, USER|KRNL, fast_load_store }, +{ EXCCAUSE_LOAD_STORE_ERROR, 0, do_load_store }, +#endif { EXCCAUSE_LEVEL1_INTERRUPT, 0, do_interrupt }, #ifdef SUPPORT_WINDOWED { EXCCAUSE_ALLOCA, USER|KRNL, fast_alloca },
@@ -354,6 +360,19 @@ static void do_div0(struct pt_regs *regs)force_sig_fault (SIGFPE ,FPE_INTDIV , (void __user *)regs->pc); } +#ifdef CONFIG_XTENSA_LOAD_STORE +static void do_load_store (struct pt_regs *regs) +{ +__die_if_kernel ("Unhandled load/store exception in kernel" , + regs,SIGKILL ); + +pr_info_ratelimited ("Load/store error to %08lx in '%s' (pid = %d, pc = %#010lx)\n" , + regs->excvaddr, current->comm, +task_pid_nr (current), regs->pc); +force_sig_fault (SIGBUS ,BUS_ADRERR , (void *)regs->excvaddr); +} +#endif +/* * Handle unaligned memory accesses from user space. Kill task. *
Signature · 1 line
2.30.2