riscv: ftrace: support fastcc in Clang for WITH_ARGS
Subsystems (4)
PATCH v4 1/12 riscv: ftrace: support fastcc in Clang for WITH_ARGS
From: Andy Chiu <[email protected]>
Some caller-saved registers which are not defined as function arguments in the ABI can still be passed as arguments when the kernel is compiled with Clang. As a result, we must save and restore those registers to prevent ftrace from clobbering them.
- [1]: https://reviews.llvm.org/D68559
---
Changelogs v4:
- Add a fix tag (Björn, Evgenii)
--- arch/riscv/include/asm/ftrace.h | 7 +++++++ arch/riscv/kernel/asm-offsets.c | 7 +++++++ arch/riscv/kernel/mcount-dyn.S | 16 ++++++++++++++-- 3 files changed, 28 insertions(+), 2 deletions(-)
arch/riscv/include/asm/ftrace.h +7 -0
@@ -146,6 +146,13 @@ struct __arch_ftrace_regs {unsigned long a5;unsigned long a6;unsigned long a7; +#ifdef CONFIG_CC_IS_CLANG +unsigned long t2; +unsigned long t3; +unsigned long t4; +unsigned long t5; +unsigned long t6; +#endif }; }; };
arch/riscv/kernel/asm-offsets.c +7 -0
@@ -501,6 +501,13 @@ void asm_offsets(void)DEFINE (FREGS_SP , offsetof(struct __arch_ftrace_regs , sp));DEFINE (FREGS_S0 , offsetof(struct __arch_ftrace_regs , s0));DEFINE (FREGS_T1 , offsetof(struct __arch_ftrace_regs , t1)); +#ifdef CONFIG_CC_IS_CLANG +DEFINE (FREGS_T2 , offsetof(struct __arch_ftrace_regs , t2)); +DEFINE (FREGS_T3 , offsetof(struct __arch_ftrace_regs , t3)); +DEFINE (FREGS_T4 , offsetof(struct __arch_ftrace_regs , t4)); +DEFINE (FREGS_T5 , offsetof(struct __arch_ftrace_regs , t5)); +DEFINE (FREGS_T6 , offsetof(struct __arch_ftrace_regs , t6)); +#endif DEFINE (FREGS_A0 , offsetof(struct __arch_ftrace_regs , a0));DEFINE (FREGS_A1 , offsetof(struct __arch_ftrace_regs , a1));DEFINE (FREGS_A2 , offsetof(struct __arch_ftrace_regs , a2));
arch/riscv/kernel/mcount-dyn.S +14 -2
@@ -96,7 +96,13 @@ REG_S x8, FREGS_S0(sp) #endif REG_S x6, FREGS_T1(sp) - +#ifdef CONFIG_CC_IS_CLANG + REG_S x7, FREGS_T2(sp) + REG_S x28, FREGS_T3(sp) + REG_S x29, FREGS_T4(sp) + REG_S x30, FREGS_T5(sp) + REG_S x31, FREGS_T6(sp) +#endif // save the arguments REG_S x10, FREGS_A0(sp) REG_S x11, FREGS_A1(sp)
@@ -115,7 +121,13 @@ REG_L x8, FREGS_S0(sp) #endif REG_L x6, FREGS_T1(sp) - +#ifdef CONFIG_CC_IS_CLANG + REG_L x7, FREGS_T2(sp) + REG_L x28, FREGS_T3(sp) + REG_L x29, FREGS_T4(sp) + REG_L x30, FREGS_T5(sp) + REG_L x31, FREGS_T6(sp) +#endif // restore the arguments REG_L x10, FREGS_A0(sp) REG_L x11, FREGS_A1(sp)
Signature · 1 line
2.39.3 (Apple Git-145)
PATCH v4 2/12 riscv: ftrace factor out code defined by !WITH_ARG
DYNAMIC_FTRACE selects DYNAMIC_FTRACE_WITH_ARGS and mcount-dyn.S in riscv, so we can remove ifdef jargons of WITH_ARG when it is known that DYNAMIC_FTRACE is true.
---
Changelog v4:
- Add anew patch that makes ftrace and its asm easier to maintain
--- arch/riscv/kernel/ftrace.c | 15 --------------- arch/riscv/kernel/mcount-dyn.S | 34 ---------------------------------- 2 files changed, 49 deletions(-)
arch/riscv/kernel/ftrace.c +0 -15
@@ -210,7 +210,6 @@ void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, }#ifdef CONFIG_DYNAMIC_FTRACE -#ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGSvoid ftrace_graph_func (unsigned long ip,unsigned long parent_ip,struct ftrace_ops *op,struct ftrace_regs *fregs) {
@@ -231,19 +230,5 @@ void ftrace_graph_func(unsigned long ip, unsigned long parent_ip,if (!function_graph_enter_regs (old, ip, frame_pointer, parent, fregs)) *parent = return_hooker; } -#else /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */ -extern void ftrace_graph_call (void ); -int ftrace_enable_ftrace_graph_caller (void ) -{ -return __ftrace_modify_call ((unsigned long )&ftrace_graph_call, - (unsigned long )&prepare_ftrace_return, true, true); -} - -int ftrace_disable_ftrace_graph_caller (void ) -{ -return __ftrace_modify_call ((unsigned long )&ftrace_graph_call, - (unsigned long )&prepare_ftrace_return, false, true); -} -#endif /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */ #endif /* CONFIG_DYNAMIC_FTRACE */ #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
arch/riscv/kernel/mcount-dyn.S +0 -34
@@ -56,8 +56,6 @@ addi sp, sp, ABI_SIZE_ON_STACK .endm -#ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS - /** * SAVE_ABI_REGS - save regs against the ftrace_regs struct *
@@ -149,36 +147,6 @@ mv a3, sp .endm -#endif /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */ - -#ifndef CONFIG_DYNAMIC_FTRACE_WITH_ARGS -SYM_FUNC_START(ftrace_caller) - SAVE_ABI - - addi a0, t0, -FENTRY_RA_OFFSET - la a1, function_trace_op - REG_L a2, 0(a1) - mv a1, ra - mv a3, sp - -SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL) - call ftrace_stub - -#ifdef CONFIG_FUNCTION_GRAPH_TRACER - addi a0, sp, ABI_RA - REG_L a1, ABI_T0(sp) - addi a1, a1, -FENTRY_RA_OFFSET -#ifdef HAVE_FUNCTION_GRAPH_FP_TEST - mv a2, s0 -#endif -SYM_INNER_LABEL(ftrace_graph_call, SYM_L_GLOBAL) - call ftrace_stub -#endif - RESTORE_ABI - jr t0 -SYM_FUNC_END(ftrace_caller) - -#else /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */ SYM_FUNC_START(ftrace_caller) mv t1, zero SAVE_ABI_REGS
@@ -194,8 +162,6 @@ SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL) jr t1 SYM_FUNC_END(ftrace_caller) -#endif /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */ - #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS SYM_CODE_START(ftrace_stub_direct_tramp) jr t0
Signature · 1 line
2.39.3 (Apple Git-145)
PATCH v4 3/12 riscv: ftrace: align patchable functions to 4 Byte boundary
In reply to Andy ChiuFrom: Andy Chiu <[email protected]>
We are changing ftrace code patching in order to remove dependency from stop_machine() and enable kernel preemption. This requires us to align functions entry at a 4-B align address.
However, -falign-functions on older versions of GCC alone was not strong enoungh to align all functions. In fact, cold functions are not aligned after turning on optimizations. We consider this is a bug in GCC and turn off guess-branch-probility as a workaround to align all functions.
GCC bug id: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=88345
The option -fmin-function-alignment is able to align all functions properly on newer versions of gcc. So, we add a cc-option to test if the toolchain supports it.
--- arch/riscv/Kconfig | 2 ++ 1 file changed, 2 insertions(+)
arch/riscv/Kconfig +2 -0
@@ -150,6 +150,7 @@ config RISCV select HAVE_DEBUG_KMEMLEAK select HAVE_DMA_CONTIGUOUS if MMU select HAVE_DYNAMIC_FTRACE if !XIP_KERNEL && MMU && (CLANG_SUPPORTS_DYNAMIC_FTRACE || GCC_SUPPORTS_DYNAMIC_FTRACE) + select FUNCTION_ALIGNMENT_4B if HAVE_DYNAMIC_FTRACE && RISCV_ISA_C select HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS select HAVE_DYNAMIC_FTRACE_WITH_ARGS if HAVE_DYNAMIC_FTRACE select HAVE_FTRACE_GRAPH_FUNC
@@ -236,6 +237,7 @@ config CLANG_SUPPORTS_DYNAMIC_FTRACE config GCC_SUPPORTS_DYNAMIC_FTRACE def_bool CC_IS_GCC depends on $(cc-option,-fpatchable-function-entry=8) + depends on CC_HAS_MIN_FUNCTION_ALIGNMENT || !RISCV_ISA_C config HAVE_SHADOW_CALL_STACK def_bool $(cc-option,-fsanitize=shadow-call-stack)
Signature · 1 line
2.39.3 (Apple Git-145)
PATCH v4 4/12 kernel: ftrace: export ftrace_sync_ipi
In reply to Andy ChiuThe following ftrace patch for riscv uses a data store to update ftrace function. Therefore, a romote fence is required to order it against function_trace_op updates. The mechanism is similar to the fence between function_trace_op and update_ftrace_func in the generic ftrace, so we leverage the same ftrace_sync_ipi function.
--- include/linux/ftrace.h | 1 + kernel/trace/ftrace.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-)
include/linux/ftrace.h +1 -0
@@ -807,6 +807,7 @@ extern void ftrace_call(void);extern void ftrace_regs_call (void );extern void mcount_call (void ); +void ftrace_sync_ipi (void *data);void ftrace_modify_all_code (int command);#ifndef FTRACE_ADDR
kernel/trace/ftrace.c +1 -1
@@ -188,7 +188,7 @@ static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip, op->saved_func (ip, parent_ip, op, fregs); } -static void ftrace_sync_ipi (void *data) +void ftrace_sync_ipi (void *data) {/* Probably not needed, but do it anyway */ smp_rmb ();
Signature · 1 line
2.39.3 (Apple Git-145)
kernel test robot
build botRe: [PATCH v4 04/12] kernel: ftrace: export ftrace_sync_ipi
kernel test robot noticed the following build warnings:
Hi Andy,
kernel test robot noticed the following build warnings:
[auto build test WARNING on linus/master]
[also build test WARNING on v6.15-rc1 next-20250408]
[cannot apply to trace/for-next]
[If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Andy-Chiu/riscv-ftrace-factor-out-code-defined-by-WITH_ARG/20250408-025114 base: linus/master patch link: https://lore.kernel.org/r/20250407180838.42877-4-andybnac%40gmail.com patch subject: [PATCH v4 04/12] kernel: ftrace: export ftrace_sync_ipi config: xtensa-randconfig-001-20250409 (https://download.01.org/0day-ci/archive/20250409/[email protected]/config) compiler: xtensa-linux-gcc (GCC) 11.5.0 reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20250409/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All warnings (new ones prefixed by >>):
>> kernel/trace/ftrace.c:191:6: warning: no previous prototype for 'ftrace_sync_ipi' [-Wmissing-prototypes]
191 | void ftrace_sync_ipi(void *data)
| ^~~~~~~~~~~~~~~
vim +/ftrace_sync_ipi +191 kernel/trace/ftrace.c
190
> 191 void ftrace_sync_ipi(void *data)
192 {
193 /* Probably not needed, but do it anyway */
194 smp_rmb();
195 }
196
Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki
Alexandre Ghiti
reviewerRe: [PATCH v4 04/12] kernel: ftrace: export ftrace_sync_ipi
Hi Andy,
Quoted 33 lines · kernel test robot
Hi Andy,
kernel test robot noticed the following build warnings:
[auto build test WARNING on linus/master]
[also build test WARNING on v6.15-rc1 next-20250408]
[cannot apply to trace/for-next]
[If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]url: https://github.com/intel-lab-lkp/linux/commits/Andy-Chiu/riscv-ftrace-factor-out-code-defined-by-WITH_ARG/20250408-025114 base: linus/master patch link: https://lore.kernel.org/r/20250407180838.42877-4-andybnac%40gmail.com patch subject: [PATCH v4 04/12] kernel: ftrace: export ftrace_sync_ipi config: xtensa-randconfig-001-20250409 (https://download.01.org/0day-ci/archive/20250409/[email protected]/config) compiler: xtensa-linux-gcc (GCC) 11.5.0 reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20250409/[email protected]/reproduce)If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/All warnings (new ones prefixed by >>):
Quoted 1 line
Quoted 1 line
kernel/trace/ftrace.c:191:6: warning: no previous prototype for 'ftrace_sync_ipi' [-Wmissing-prototypes]191 | void
ftrace_sync_ipi(void *data)
| ^~~~~~~~~~~~~~~vim +/ftrace_sync_ipi +191
kernel/trace/ftrace.c
Quoted 8 lines
190Quoted 1 line
191 void
ftrace_sync_ipi(void *data)192 { 193 /* Probably not needed, but do it anyway */ 194 smp_rmb(); 195 } 196
I fixed this with the following that I squashed into your patch, no need to resend a new series for this.
include/linux/ftrace.h +0 -0
@@ -635,6 +635,8 @@ enum {
#define ftrace_get_symaddr(fentry_ip) (0)
#endif
+void ftrace_sync_ipi(void *data);
+
#ifdef CONFIG_DYNAMIC_FTRACE
void ftrace_arch_code_modify_prepare(void); @@ -807,7 +809,6 @@ extern void ftrace_call(void); extern void ftrace_regs_call(void); extern void mcount_call(void);
-void ftrace_sync_ipi(void *data);
void ftrace_modify_all_code(int command);
#ifndef FTRACE_ADDR
Thanks,
Alex
PATCH v4 5/12 riscv: ftrace: prepare ftrace for atomic code patching
In reply to Andy ChiuFrom: Andy Chiu <[email protected]>
We use an AUIPC+JALR pair to jump into a ftrace trampoline. Since instruction fetch can break down to 4 byte at a time, it is impossible to update two instructions without a race. In order to mitigate it, we initialize the patchable entry to AUIPC + NOP4. Then, the run-time code patching can change NOP4 to JALR to eable/disable ftrcae from a function. This limits the reach of each ftrace entry to +-2KB displacing from ftrace_caller.
Starting from the trampoline, we add a level of indirection for it to reach ftrace caller target. Now, it loads the target address from a memory location, then perform the jump. This enable the kernel to update the target atomically.
The new don't-stop-the-world text patching on change only one RISC-V instruction:
| -8: &ftrace_ops of the associated tracer function. | <ftrace enable>: | 0: auipc t0, hi(ftrace_caller) | 4: jalr t0, lo(ftrace_caller) | | -8: &ftrace_nop_ops | <ftrace disable>: | 0: auipc t0, hi(ftrace_caller) | 4: nop
This means that f+0x0 is fixed, and should not be claimed by ftrace, e.g. kprobe should be able to put a probe in f+0x0. Thus, we adjust the offset and MCOUNT_INSN_SIZE accordingly.
---
Changelog v4:
- Include Björn's fix for kprobe
- Refactor code for better reading (Robbin, Björn)
- Remove make_call_ra and friedns (Björn)
- Update comments to match reality (Björn)
- Drop code defined by !WITH_ARG
- Add a synchronization point when updating ftrace_call_dest (Björn)
--- arch/riscv/include/asm/ftrace.h | 49 ++++++------ arch/riscv/kernel/ftrace.c | 130 ++++++++++++++++---------------- arch/riscv/kernel/mcount-dyn.S | 9 +-- 3 files changed, 92 insertions(+), 96 deletions(-)
arch/riscv/include/asm/ftrace.h +22 -27
@@ -20,10 +20,9 @@ extern void *return_address(unsigned int level);#define ftrace_return_address (n) return_address(n)void _mcount (void ); -static inline unsigned long ftrace_call_adjust (unsigned long addr) -{ -return addr; -} +unsigned long ftrace_call_adjust (unsigned long addr); +unsigned long arch_ftrace_get_symaddr (unsigned long fentry_ip); +#define ftrace_get_symaddr (fentry_ip) arch_ftrace_get_symaddr(fentry_ip)/* * Let's do like x86/arm64 and ignore the compat syscalls.
@@ -57,12 +56,21 @@ struct dyn_arch_ftrace {* 2) jalr: setting low-12 offset to ra, jump to ra, and set ra to * return address (original pc + 4) * +* The first 2 instructions for each tracable function is compiled to 2 nop +* instructions. Then, the kernel initializes the first instruction to auipc at +* boot time (<ftrace disable>). The second instruction is patched to jalr to +* start the trace. +* +*<Image>: +* 0: nop +* 4: nop +* *<ftrace enable>: -* 0: auipc t0/ra, 0x? -* 4: jalr t0/ra, ?(t0/ra) +* 0: auipc t0, 0x? +* 4: jalr t0, ?(t0) * *<ftrace disable>: -* 0: nop +* 0: auipc t0, 0x? * 4: nop * * Dynamic ftrace generates probes to call sites, so we must deal with
@@ -75,10 +83,9 @@ struct dyn_arch_ftrace {#define AUIPC_OFFSET_MASK (0xfffff000)#define AUIPC_PAD (0x00001000)#define JALR_SHIFT 20 -#define JALR_RA (0x000080e7) -#define AUIPC_RA (0x00000097)#define JALR_T0 (0x000282e7)#define AUIPC_T0 (0x00000297) +#define JALR_RANGE (JALR_SIGN_MASK - 1)#define to_jalr_t0 (offset) \ (((offset & JALR_OFFSET_MASK) << JALR_SHIFT) | JALR_T0)
@@ -96,26 +103,14 @@ do { \ call[1 ] =to_jalr_t0 (offset); \ }while (0 ) -#define to_jalr_ra (offset) \ - (((offset & JALR_OFFSET_MASK) << JALR_SHIFT) | JALR_RA) - -#define to_auipc_ra (offset) \ - ((offset & JALR_SIGN_MASK) ? \ - (((offset & AUIPC_OFFSET_MASK) + AUIPC_PAD) | AUIPC_RA) : \ - ((offset & AUIPC_OFFSET_MASK) | AUIPC_RA)) - -#define make_call_ra (caller, callee, call) \ -do { \ - unsigned int offset = \ - (unsigned long) (callee) - (unsigned long) (caller); \ - call[0] = to_auipc_ra(offset); \ - call[1] = to_jalr_ra(offset); \ -} while (0) -/* -* Let auipc+jalr be the basic *mcount unit*, so we make it 8 bytes here. +* Only the jalr insn in the auipc+jalr is patched, so we make it 4 +* bytes here. */ -#define MCOUNT_INSN_SIZE 8 +#define MCOUNT_INSN_SIZE 4 +#define MCOUNT_AUIPC_SIZE 4 +#define MCOUNT_JALR_SIZE 4 +#define MCOUNT_NOP4_SIZE 4#ifndef __ASSEMBLY__struct dyn_ftrace ;
arch/riscv/kernel/ftrace.c +66 -64
187 lines not on this page · raw message
arch/riscv/kernel/mcount-dyn.S +4 -5
@@ -13,7 +13,6 @@ .text -#define FENTRY_RA_OFFSET 8 #define ABI_SIZE_ON_STACK 80 #define ABI_A0 0 #define ABI_A1 8
@@ -62,8 +61,7 @@ * After the stack is established, * * 0(sp) stores the PC of the traced function which can be accessed -* by &(fregs)->epc in tracing function. Note that the real -* function entry address should be computed with -FENTRY_RA_OFFSET. +* by &(fregs)->epc in tracing function. * * 8(sp) stores the function return address (i.e. parent IP) that * can be accessed by &(fregs)->ra in tracing function.
@@ -140,7 +138,7 @@ .endm .macro PREPARE_ARGS - addi a0, t0, -FENTRY_RA_OFFSET + addi a0, t0, -MCOUNT_JALR_SIZE // ip (callsite's jalr insn) la a1, function_trace_op REG_L a2, 0(a1) mv a1, ra
@@ -153,7 +151,8 @@ SYM_FUNC_START(ftrace_caller) PREPARE_ARGS SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL) - call ftrace_stub + REG_L ra, ftrace_call_dest + jalr ra, 0(ra) RESTORE_ABI_REGS bnez t1, .Ldirect
Signature · 1 line
2.39.3 (Apple Git-145)
Re: [PATCH v4 05/12] riscv: ftrace: prepare ftrace for atomic code patching
Hi, thanks,
Quoted 175 lines · Andy Chiu
From: Andy Chiu <[email protected]>
We use an AUIPC+JALR pair to jump into a ftrace trampoline. Since instruction fetch can break down to 4 byte at a time, it is impossible to update two instructions without a race. In order to mitigate it, we initialize the patchable entry to AUIPC + NOP4. Then, the run-time code patching can change NOP4 to JALR to eable/disable ftrcae from a function. This limits the reach of each ftrace entry to +-2KB displacing from
ftrace_caller.Starting from the trampoline, we add a level of indirection for it to reach ftrace caller target. Now, it loads the target address from a memory location, then perform the jump. This enable the kernel to update the target atomically.
The new don't-stop-the-world text patching on change only one RISC-V instruction:
| -8: &ftrace_ops of the associated tracer function. | <ftrace enable>: | 0: auipc t0, hi(ftrace_caller) | 4: jalr t0, lo(ftrace_caller) | | -8: &ftrace_nop_ops | <ftrace disable>: | 0: auipc t0, hi(ftrace_caller) | 4: nopThis means that
f+0x0is fixed, and should not be claimed by ftrace, e.g. kprobe should be able to put a probe inf+0x0. Thus, we adjust the offset andMCOUNT_INSN_SIZEaccordingly.Co-developed-by Björn Töpel <[email protected]>Signed-off-by Björn Töpel <[email protected]>Signed-off-by Andy Chiu <[email protected]>---
Changelog v4:
- Include Björn's fix for kprobe
- Refactor code for better reading (Robbin, Björn)
- Removemake_call_raand friedns (Björn)
- Update comments to match reality (Björn)
- Drop code defined by !WITH_ARG
- Add a synchronization point when updatingftrace_call_dest(Björn)
---
arch/riscv/include/asm/ftrace.h | 49 ++++++------ arch/riscv/kernel/ftrace.c | 130 ++++++++++++++++---------------- arch/riscv/kernel/mcount-dyn.S | 9 +--3 files changed, 92 insertions(+), 96 deletions(-)arch/riscv/include/asm/ftrace.h +0
-0@@ -20,10 +20,9 @@ extern void *return_address(unsigned int level); #define ftrace_return_address(n) return_address(n)
void_mcount(void);arch/riscv/include/asm/ftrace.h +3
-4-static inline unsigned long ftrace_call_adjust (unsigned long addr) -{ -return addr; -} +unsigned long ftrace_call_adjust (unsigned long addr); +unsigned long arch_ftrace_get_symaddr (unsigned long fentry_ip); +#define ftrace_get_symaddr (fentry_ip) arch_ftrace_get_symaddr(fentry_ip)
/*
* Let's do like x86/arm64 and ignore the compat syscalls.arch/riscv/include/asm/ftrace.h +2
-0@@ -57,12 +56,21 @@ struct dyn_arch_ftrace { * 2) jalr: setting low-12 offset to ra, jump to ra, and set ra to * return address (original pc + 4) * + * The first 2 instructions for each tracable function is compiled to 2 nop + * instructions. Then, the kernel initializes the first instruction to auipcat
+ * boot time (<ftrace disable>). The second instruction is patched to jalr to
+ * start the trace.
+ *
+ *<Image>:
+ * 0: nop
+ * 4: nop
+ *
*<ftrace enable>:
- * 0: auipc t0/ra, 0x?
- * 4: jalr t0/ra, ?(t0/ra)
+ * 0: auipc t0, 0x?
+ * 4: jalr t0, ?(t0)
*
*<ftrace disable>:
- * 0: nop
+ * 0: auipc t0, 0x?
* 4: nop
*
* Dynamic ftrace generates probes to call sites, so we must deal witharch/riscv/include/asm/ftrace.h +1
-2@@ -75,10 +83,9 @@ struct dyn_arch_ftrace { #define AUIPC_OFFSET_MASK (0xfffff000) #define AUIPC_PAD (0x00001000) #define JALR_SHIFT 20 -#define JALR_RA (0x000080e7) -#define AUIPC_RA (0x00000097) #define JALR_T0 (0x000282e7) #define AUIPC_T0 (0x00000297) +#define JALR_RANGE (JALR_SIGN_MASK - 1)
#defineto_jalr_t0(offset)\
(((offset &JALR_OFFSET_MASK) <<JALR_SHIFT) |JALR_T0)arch/riscv/include/asm/ftrace.h +0
-0@@ -96,26 +103,14 @@ do { \ call[1] = to_jalr_t0(offset); \ } while (0)arch/riscv/include/asm/ftrace.h +0
-16-#define to_jalr_ra (offset) \ - (((offset & JALR_OFFSET_MASK) << JALR_SHIFT) | JALR_RA) - -#define to_auipc_ra (offset) \ - ((offset & JALR_SIGN_MASK) ? \ - (((offset & AUIPC_OFFSET_MASK) + AUIPC_PAD) | AUIPC_RA) : \ - ((offset & AUIPC_OFFSET_MASK) | AUIPC_RA)) - -#define make_call_ra (caller, callee, call) \ -do { \ - unsigned int offset = \ - (unsigned long) (callee) - (unsigned long) (caller); \ - call[0] = to_auipc_ra(offset); \ - call[1] = to_jalr_ra(offset); \ -} while (0) -/*
- * Let auipc+jalr be the basic *mcount unit*, so we make it 8 bytes here.
+ * Only the jalr insn in the auipc+jalr is patched, so we make it 4
+ * bytes here.
*/arch/riscv/include/asm/ftrace.h +4
-1-#define MCOUNT_INSN_SIZE 8 +#define MCOUNT_INSN_SIZE 4 +#define MCOUNT_AUIPC_SIZE 4 +#define MCOUNT_JALR_SIZE 4 +#define MCOUNT_NOP4_SIZE 4
#ifndef__ASSEMBLY__
structdyn_ftrace;arch/riscv/kernel/ftrace.c +1
-0@@ -8,10 +8,21 @@ #include <linux/ftrace.h> #include <linux/uaccess.h> #include <linux/memory.h> +#include <linux/irqflags.h> #include <linux/stop_machine.h> #include <asm/cacheflush.h> #include <asm/text-patching.h>arch/riscv/kernel/ftrace.c +10
-0+unsigned long ftrace_call_adjust (unsigned long addr) +{ +return addr +MCOUNT_AUIPC_SIZE ; +} + +unsigned long arch_ftrace_get_symaddr (unsigned long fentry_ip) +{ +return fentry_ip -MCOUNT_AUIPC_SIZE ; +} +#ifdef
CONFIG_DYNAMIC_FTRACE
voidftrace_arch_code_modify_prepare(void)__acquires(&text_mutex){arch/riscv/kernel/ftrace.c +0
-0@@ -32,51 +43,32 @@ void ftrace_arch_code_modify_post_process(void) __releases(&text_mutex) mutex_unlock(&text_mutex); }arch/riscv/kernel/ftrace.c +1
-2-static int ftrace_check_current_call (unsigned long hook_pos, - unsigned int *expected) +static int __ftrace_modify_call(unsigned long source, unsigned long target,bool validate)
{
arch/riscv/kernel/ftrace.c void ftrace_arch_code_modify_post_process(void) __releases(&text_mutex) · line 48
Quoted 1 line
arch/riscv/kernel/ftrace.c +1
-0+unsigned int call[2 ], offset;
There is a mix of unsigned int and u32.
It doesn't seem to serve any purpose?
Quoted 136 lines
unsigned int replaced[2];
arch/riscv/kernel/ftrace.c +0
-1-unsigned int nops[2 ] = {RISCV_INSN_NOP4 ,RISCV_INSN_NOP4 };arch/riscv/kernel/ftrace.c +2
-3-/* we expect nops at the hook position */ -if (!expected) - expected = nops; + offset = target - source; + call[1 ] =to_jalr_t0 (offset);arch/riscv/kernel/ftrace.c +0
-15-/* -* Read the text we want to modify; -* return must be -EFAULT on read error -*/ -if (copy_from_kernel_nofault (replaced, (void *)hook_pos, -MCOUNT_INSN_SIZE )) -return -EFAULT ; - -/* -* Make sure it is what we expect it to be; -* return must be -EINVAL on failed comparison -*/ -if (memcmp (expected, replaced,sizeof (replaced))) { - pr_err("%p: expected (%08x %08x) but got (%08x %08x)\n", - (void *)hook_pos, expected[0], expected[1],replaced[0],
arch/riscv/kernel/ftrace.c +7
-2- replaced[1]); -return -EINVAL ; +if (validate) { + call[0 ] =to_auipc_t0 (offset); +/* +* Read the text we want to modify; +* return must be -EFAULT on read error +*/ + if (copy_from_kernel_nofault(replaced, (void *)source, 2 *
MCOUNT_INSN_SIZE))arch/riscv/kernel/ftrace.c +7
-0+return -EFAULT ; + +if (replaced[0 ] != call[0 ]) { +pr_err ("%p: expected (%08x) but got (%08x)\n" , + (void *)source, call[0], replaced[0]); +return -EINVAL ; + }}
arch/riscv/kernel/ftrace.c +0
-15-return 0 ; -} - -static int __ftrace_modify_call (unsigned long hook_pos,unsigned long target, -bool enable,bool ra) -{ -unsigned int call[2 ]; -unsigned int nops[2 ] = {RISCV_INSN_NOP4 ,RISCV_INSN_NOP4 }; - -if (ra) -make_call_ra (hook_pos, target, call); -else -make_call_t0 (hook_pos, target, call); - -/* Replace the auipc-jalr pair at once. Return -EPERM on write error. */
arch/riscv/kernel/ftrace.c +0
-1- if (patch_insn_write((void *)hook_pos, enable ? call : nops,
MCOUNT_INSN_SIZE))arch/riscv/kernel/ftrace.c +2
-0+/* Replace the jalr at once. Return -EPERM on write error. */ + if (patch_insn_write((void *)(source + MCOUNT_AUIPC_SIZE), call + 1,
MCOUNT_JALR_SIZE))
return-EPERM;
return 0;arch/riscv/kernel/ftrace.c +0
-0@@ -84,22 +76,21 @@ static int __ftrace_modify_call(unsigned long hook_pos, unsigned long target,
intftrace_make_call(struct dyn_ftrace *rec, unsigned long addr){arch/riscv/kernel/ftrace.c +1
-1-unsigned int call[2 ]; +unsigned long distance, orig_addr, pc = rec->ip -MCOUNT_AUIPC_SIZE ;arch/riscv/kernel/ftrace.c +4
-4-make_call_t0 (rec->ip, addr, call); - -if (patch_insn_write ((void *)rec->ip, call,MCOUNT_INSN_SIZE )) -return -EPERM ; + orig_addr = (unsigned long )&ftrace_caller; + distance = addr > orig_addr ? addr - orig_addr : orig_addr - addr; +if (distance >JALR_RANGE ) +return -EINVAL ;arch/riscv/kernel/ftrace.c +1
-1-return 0 ; +return __ftrace_modify_call (pc, addr, false);}
arch/riscv/kernel/ftrace.c +1
-2-int ftrace_make_nop (struct module *mod,struct dyn_ftrace *rec, - unsigned long addr) +int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, unsigned longaddr)
{arch/riscv/kernel/ftrace.c +1
-1-unsigned int nops[2 ] = {RISCV_INSN_NOP4 ,RISCV_INSN_NOP4 }; +u32 nop4 =RISCV_INSN_NOP4 ;arch/riscv/kernel/ftrace.c +1
-1-if (patch_insn_write ((void *)rec->ip, nops,MCOUNT_INSN_SIZE )) +if (patch_insn_write ((void *)rec->ip, &nop4,MCOUNT_NOP4_SIZE ))return
-EPERM;
return 0;arch/riscv/kernel/ftrace.c +7
-1@@ -114,21 +105,38 @@ int ftrace_make_nop(struct module *mod, structdyn_ftrace *rec, */ int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec) { -int out; +unsigned long pc = rec->ip -MCOUNT_AUIPC_SIZE ; +unsigned int nops[2 ], offset; +int ret; + + offset = (unsigned long ) &ftrace_caller - pc; + nops[0 ] =to_auipc_t0 (offset); + nops[1 ] =RISCV_INSN_NOP4 ;
mutex_lock(&text_mutex);arch/riscv/kernel/ftrace.c +1
-1- out =ftrace_make_nop (mod, rec,MCOUNT_ADDR ); + ret =patch_insn_write ((void *)pc, nops,2 *MCOUNT_INSN_SIZE );
mutex_unlock(&text_mutex);arch/riscv/kernel/ftrace.c +1
-1-return out; +return ret;}
+ftrace_func_tftrace_call_dest=ftrace_stub; intftrace_update_ftrace_func(ftrace_func_t func){arch/riscv/kernel/ftrace.c +4
-4-int ret =__ftrace_modify_call ((unsigned long )&ftrace_call, - (unsigned long)func, true, true); - -return ret; +WRITE_ONCE (ftrace_call_dest, func); +/* +* The data fence ensure that the update to ftrace_call_dest happens +* before the write to function_trace_op later in the generic ftrace.
arch/riscv/kernel/ftrace.c int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, · line 139
Quoted 10 lines
arch/riscv/kernel/ftrace.c +10
-0+* If the sequence is not enforced, then an old ftrace_call_dest may +* race loading a new function_trace_op set in ftrace_modify_all_code +* +* If we are in stop_machine, then we don't need to call remote fence +* as there is no concurrent read-side of ftrace_call_dest. +*/ +smp_wmb (); +if (!irqs_disabled ()) +smp_call_function (ftrace_sync_ipi,NULL ,1 ); +return 0 ;
Here we allow the read of cpu_online_mask being reordered with the store to ftrace_call_dest.
Can't this happen then?
Hart X | Hart new
load cpu_online_mask |
| store cpu_online_mask
| tlb shootdown/icache flush
store ftrace_call_dest |
ftrace_sync_ipi | // not hit
| sees old value ?Thanks, Robbin
Quoted 64 lines
}
structftrace_modify_param{arch/riscv/kernel/ftrace.c +1
-2@@ -172,17 +180,11 @@ void arch_ftrace_update_code(int command) int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, unsigned long addr) { +unsigned long caller = rec->ip -MCOUNT_AUIPC_SIZE ; unsigned int call[2]; -unsigned long caller = rec->ip; -int ret;
make_call_t0(caller, old_addr, call);arch/riscv/kernel/ftrace.c +1
-6- ret =ftrace_check_current_call (caller, call); - -if (ret) -return ret; - -return __ftrace_modify_call (caller, addr, true, false); +return __ftrace_modify_call (caller, addr, true);}
#endifarch/riscv/kernel/mcount-dyn.S +0
-0@@ -13,7 +13,6 @@
.text
-#defineFENTRY_RA_OFFSET8
#defineABI_SIZE_ON_STACK80
#defineABI_A00
#defineABI_A18arch/riscv/kernel/mcount-dyn.S +1
-2@@ -62,8 +61,7 @@ * After the stack is established, * * 0(sp) stores the PC of the traced function which can be accessed -* by &(fregs)->epc in tracing function. Note that the real -* function entry address should be computed with -FENTRY_RA_OFFSET. +* by &(fregs)->epc in tracing function. * * 8(sp) stores the function return address (i.e. parent IP) that * can be accessed by &(fregs)->ra in tracing function.@@ -140,7 +138,7 @@.endm
.macroPREPARE_ARGSarch/riscv/kernel/mcount-dyn.S +1
-1- addi a0, t0, -FENTRY_RA_OFFSET + addi a0, t0, -MCOUNT_JALR_SIZE // ip (callsite's jalr insn)la a1,
function_trace_op
REG_La2, 0(a1)
mv a1, raarch/riscv/kernel/mcount-dyn.S +0
-0@@ -153,7 +151,8 @@ SYM_FUNC_START(ftrace_caller)
PREPARE_ARGS
SYM_INNER_LABEL(ftrace_call,SYM_L_GLOBAL)arch/riscv/kernel/mcount-dyn.S +2
-1- call ftrace_stub + REG_L ra, ftrace_call_dest + jalr ra, 0(ra)
RESTORE_ABI_REGS
bnez t1, .Ldirect
Alexandre Ghiti
reviewerRe: [PATCH v4 05/12] riscv: ftrace: prepare ftrace for atomic code patching
In reply to Andy ChiuHi Andy,
Quoted 150 lines · Andy Chiu
From: Andy Chiu <[email protected]>
We use an AUIPC+JALR pair to jump into a ftrace trampoline. Since instruction fetch can break down to 4 byte at a time, it is impossible to update two instructions without a race. In order to mitigate it, we initialize the patchable entry to AUIPC + NOP4. Then, the run-time code patching can change NOP4 to JALR to eable/disable ftrcae from a function. This limits the reach of each ftrace entry to +-2KB displacing from
ftrace_caller.Starting from the trampoline, we add a level of indirection for it to reach ftrace caller target. Now, it loads the target address from a memory location, then perform the jump. This enable the kernel to update the target atomically.
The new don't-stop-the-world text patching on change only one RISC-V instruction:
| -8: &ftrace_ops of the associated tracer function. | <ftrace enable>: | 0: auipc t0, hi(ftrace_caller) | 4: jalr t0, lo(ftrace_caller) | | -8: &ftrace_nop_ops | <ftrace disable>: | 0: auipc t0, hi(ftrace_caller) | 4: nopThis means that
f+0x0is fixed, and should not be claimed by ftrace, e.g. kprobe should be able to put a probe inf+0x0. Thus, we adjust the offset andMCOUNT_INSN_SIZEaccordingly.Co-developed-by Björn Töpel <[email protected]>Signed-off-by Björn Töpel <[email protected]>Signed-off-by Andy Chiu <[email protected]>---
Changelog v4:
- Include Björn's fix for kprobe
- Refactor code for better reading (Robbin, Björn)
- Removemake_call_raand friedns (Björn)
- Update comments to match reality (Björn)
- Drop code defined by !WITH_ARG
- Add a synchronization point when updatingftrace_call_dest(Björn)--- arch/riscv/include/asm/ftrace.h | 49 ++++++------ arch/riscv/kernel/ftrace.c | 130 ++++++++++++++++---------------- arch/riscv/kernel/mcount-dyn.S | 9 +-- 3 files changed, 92 insertions(+), 96 deletions(-)arch/riscv/include/asm/ftrace.h +22
-27@@ -20,10 +20,9 @@ extern void *return_address(unsigned int level);#define ftrace_return_address (n) return_address(n)void _mcount (void ); -static inline unsigned long ftrace_call_adjust (unsigned long addr) -{ -return addr; -} +unsigned long ftrace_call_adjust (unsigned long addr); +unsigned long arch_ftrace_get_symaddr (unsigned long fentry_ip); +#define ftrace_get_symaddr (fentry_ip) arch_ftrace_get_symaddr(fentry_ip)/* * Let's do like x86/arm64 and ignore the compat syscalls. @@ -57,12 +56,21 @@ struct dyn_arch_ftrace {* 2) jalr: setting low-12 offset to ra, jump to ra, and set ra to * return address (original pc + 4) * +* The first 2 instructions for each tracable function is compiled to 2 nop +* instructions. Then, the kernel initializes the first instruction to auipc at +* boot time (<ftrace disable>). The second instruction is patched to jalr to +* start the trace. +* +*<Image>: +* 0: nop +* 4: nop +* *<ftrace enable>: -* 0: auipc t0/ra, 0x? -* 4: jalr t0/ra, ?(t0/ra) +* 0: auipc t0, 0x? +* 4: jalr t0, ?(t0) * *<ftrace disable>: -* 0: nop +* 0: auipc t0, 0x? * 4: nop * * Dynamic ftrace generates probes to call sites, so we must deal with @@ -75,10 +83,9 @@ struct dyn_arch_ftrace {#define AUIPC_OFFSET_MASK (0xfffff000)#define AUIPC_PAD (0x00001000)#define JALR_SHIFT 20 -#define JALR_RA (0x000080e7) -#define AUIPC_RA (0x00000097)#define JALR_T0 (0x000282e7)#define AUIPC_T0 (0x00000297) +#define JALR_RANGE (JALR_SIGN_MASK - 1)#define to_jalr_t0 (offset) \ (((offset & JALR_OFFSET_MASK) << JALR_SHIFT) | JALR_T0)@@ -96,26 +103,14 @@ do { \ call[1 ] =to_jalr_t0 (offset); \ }while (0 ) -#define to_jalr_ra (offset) \ - (((offset & JALR_OFFSET_MASK) << JALR_SHIFT) | JALR_RA) - -#define to_auipc_ra (offset) \ - ((offset & JALR_SIGN_MASK) ? \ - (((offset & AUIPC_OFFSET_MASK) + AUIPC_PAD) | AUIPC_RA) : \ - ((offset & AUIPC_OFFSET_MASK) | AUIPC_RA)) - -#define make_call_ra (caller, callee, call) \ -do { \ - unsigned int offset = \ - (unsigned long) (callee) - (unsigned long) (caller); \ - call[0] = to_auipc_ra(offset); \ - call[1] = to_jalr_ra(offset); \ -} while (0) -/* -* Let auipc+jalr be the basic *mcount unit*, so we make it 8 bytes here. +* Only the jalr insn in the auipc+jalr is patched, so we make it 4 +* bytes here. */ -#define MCOUNT_INSN_SIZE 8 +#define MCOUNT_INSN_SIZE 4 +#define MCOUNT_AUIPC_SIZE 4 +#define MCOUNT_JALR_SIZE 4 +#define MCOUNT_NOP4_SIZE 4#ifndef __ASSEMBLY__struct dyn_ftrace ;arch/riscv/kernel/ftrace.c +1
-0@@ -8,10 +8,21 @@#include <linux/ftrace.h> #include <linux/uaccess.h> #include <linux/memory.h> +#include <linux/irqflags.h> #include <linux/stop_machine.h> #include <asm/cacheflush.h> #include <asm/text-patching.h>
arch/riscv/kernel/ftrace.c line 25
Quoted 10 lines
arch/riscv/kernel/ftrace.c +10
-0+unsigned long ftrace_call_adjust (unsigned long addr) +{ +return addr +MCOUNT_AUIPC_SIZE ; +} + +unsigned long arch_ftrace_get_symaddr (unsigned long fentry_ip) +{ +return fentry_ip -MCOUNT_AUIPC_SIZE ; +} +
Those functions cause the following errors when building with !CONFIG_DYNAMIC_FTRACE, but I'm not sure how to fix this:
../arch/riscv/kernel/ftrace.c: In function 'ftrace_call_adjust':../arch/riscv/kernel/ftrace.c:19:35: error: 'MCOUNT_AUIPC_SIZE' undeclared (first use in this function) 19 | return addr + 8 + MCOUNT_AUIPC_SIZE;
| ^~~~~~~~~~~~~~~~~../arch/riscv/kernel/ftrace.c:19:35: note: each undeclared identifier is reported only once for each function it appears in CC fs/9p/vfs_dir.o../arch/riscv/kernel/ftrace.c: In function 'arch_ftrace_get_symaddr':../arch/riscv/kernel/ftrace.c:26:28: error: 'MCOUNT_AUIPC_SIZE' undeclared (first use in this function) 26 | return fentry_ip - MCOUNT_AUIPC_SIZE;
| ^~~~~~~~~~~~~~~~~
CC drivers/pci/pcie/pme.o../arch/riscv/kernel/ftrace.c: In function 'ftrace_call_adjust':../arch/riscv/kernel/ftrace.c:22:1: error: control reaches end of non-void function [-Werror=return-type] 22 | }
| ^../arch/riscv/kernel/ftrace.c: In function 'arch_ftrace_get_symaddr':../arch/riscv/kernel/ftrace.c:27:1: error: control reaches end of non-void function [-Werror=return-type] 27 | }
| ^
cc1: some warnings being treated as errors
make[5]: *** [../scripts/Makefile.build:203: arch/riscv/kernel/ftrace.o] Error 1
Quoted 214 lines
arch/riscv/kernel/ftrace.c +55
-64#ifdef CONFIG_DYNAMIC_FTRACE void ftrace_arch_code_modify_prepare(void) __acquires(&text_mutex) {@@ -32,51 +43,32 @@ void ftrace_arch_code_modify_post_process(void) __releases(&text_mutex)mutex_unlock (&text_mutex); } -static int ftrace_check_current_call (unsigned long hook_pos, -unsigned int *expected) +static int __ftrace_modify_call (unsigned long source,unsigned long target,bool validate) { +unsigned int call[2 ], offset;unsigned int replaced[2 ]; -unsigned int nops[2 ] = {RISCV_INSN_NOP4 ,RISCV_INSN_NOP4 }; -/* we expect nops at the hook position */ -if (!expected) - expected = nops; + offset = target - source; + call[1 ] =to_jalr_t0 (offset); -/* -* Read the text we want to modify; -* return must be -EFAULT on read error -*/ -if (copy_from_kernel_nofault (replaced, (void *)hook_pos, -MCOUNT_INSN_SIZE )) -return -EFAULT ; - -/* -* Make sure it is what we expect it to be; -* return must be -EINVAL on failed comparison -*/ -if (memcmp (expected, replaced,sizeof (replaced))) { -pr_err ("%p: expected (%08x %08x) but got (%08x %08x)\n" , - (void *)hook_pos, expected[0 ], expected[1 ], replaced[0 ], - replaced[1 ]); -return -EINVAL ; +if (validate) { + call[0 ] =to_auipc_t0 (offset); +/* +* Read the text we want to modify; +* return must be -EFAULT on read error +*/ +if (copy_from_kernel_nofault (replaced, (void *)source,2 *MCOUNT_INSN_SIZE )) +return -EFAULT ; + +if (replaced[0 ] != call[0 ]) { +pr_err ("%p: expected (%08x) but got (%08x)\n" , + (void *)source, call[0 ], replaced[0 ]); +return -EINVAL ; + } } -return 0 ; -} - -static int __ftrace_modify_call (unsigned long hook_pos,unsigned long target, -bool enable,bool ra) -{ -unsigned int call[2 ]; -unsigned int nops[2 ] = {RISCV_INSN_NOP4 ,RISCV_INSN_NOP4 }; - -if (ra) -make_call_ra (hook_pos, target, call); -else -make_call_t0 (hook_pos, target, call); - -/* Replace the auipc-jalr pair at once. Return -EPERM on write error. */ -if (patch_insn_write ((void *)hook_pos, enable ? call : nops,MCOUNT_INSN_SIZE )) +/* Replace the jalr at once. Return -EPERM on write error. */ +if (patch_insn_write ((void *)(source +MCOUNT_AUIPC_SIZE ), call +1 ,MCOUNT_JALR_SIZE ))return -EPERM ;return 0 ;@@ -84,22 +76,21 @@ static int __ftrace_modify_call(unsigned long hook_pos, unsigned long target,int ftrace_make_call (struct dyn_ftrace *rec,unsigned long addr) { -unsigned int call[2 ]; +unsigned long distance, orig_addr, pc = rec->ip -MCOUNT_AUIPC_SIZE ; -make_call_t0 (rec->ip, addr, call); - -if (patch_insn_write ((void *)rec->ip, call,MCOUNT_INSN_SIZE )) -return -EPERM ; + orig_addr = (unsigned long )&ftrace_caller; + distance = addr > orig_addr ? addr - orig_addr : orig_addr - addr; +if (distance >JALR_RANGE ) +return -EINVAL ; -return 0 ; +return __ftrace_modify_call (pc, addr, false); } -int ftrace_make_nop (struct module *mod,struct dyn_ftrace *rec, -unsigned long addr) +int ftrace_make_nop (struct module *mod,struct dyn_ftrace *rec,unsigned long addr) { -unsigned int nops[2 ] = {RISCV_INSN_NOP4 ,RISCV_INSN_NOP4 }; +u32 nop4 =RISCV_INSN_NOP4 ; -if (patch_insn_write ((void *)rec->ip, nops,MCOUNT_INSN_SIZE )) +if (patch_insn_write ((void *)rec->ip, &nop4,MCOUNT_NOP4_SIZE ))return -EPERM ;return 0 ;@@ -114,21 +105,38 @@ int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec,*/ int ftrace_init_nop (struct module *mod,struct dyn_ftrace *rec) { -int out; +unsigned long pc = rec->ip -MCOUNT_AUIPC_SIZE ; +unsigned int nops[2 ], offset; +int ret; + + offset = (unsigned long ) &ftrace_caller - pc; + nops[0 ] =to_auipc_t0 (offset); + nops[1 ] =RISCV_INSN_NOP4 ;mutex_lock (&text_mutex); - out =ftrace_make_nop (mod, rec,MCOUNT_ADDR ); + ret =patch_insn_write ((void *)pc, nops,2 *MCOUNT_INSN_SIZE );mutex_unlock (&text_mutex); -return out; +return ret; } +ftrace_func_t ftrace_call_dest = ftrace_stub;int ftrace_update_ftrace_func (ftrace_func_t func) { -int ret =__ftrace_modify_call ((unsigned long )&ftrace_call, - (unsigned long )func, true, true); - -return ret; +WRITE_ONCE (ftrace_call_dest, func); +/* +* The data fence ensure that the update to ftrace_call_dest happens +* before the write to function_trace_op later in the generic ftrace. +* If the sequence is not enforced, then an old ftrace_call_dest may +* race loading a new function_trace_op set in ftrace_modify_all_code +* +* If we are in stop_machine, then we don't need to call remote fence +* as there is no concurrent read-side of ftrace_call_dest. +*/ +smp_wmb (); +if (!irqs_disabled ()) +smp_call_function (ftrace_sync_ipi,NULL ,1 ); +return 0 ; }struct ftrace_modify_param {@@ -172,17 +180,11 @@ void arch_ftrace_update_code(int command)int ftrace_modify_call (struct dyn_ftrace *rec,unsigned long old_addr,unsigned long addr) { +unsigned long caller = rec->ip -MCOUNT_AUIPC_SIZE ;unsigned int call[2 ]; -unsigned long caller = rec->ip; -int ret;make_call_t0 (caller, old_addr, call); - ret =ftrace_check_current_call (caller, call); - -if (ret) -return ret; - -return __ftrace_modify_call (caller, addr, true, false); +return __ftrace_modify_call (caller, addr, true); }#endif arch/riscv/kernel/mcount-dyn.S +4
-5@@ -13,7 +13,6 @@ .text -#define FENTRY_RA_OFFSET 8 #define ABI_SIZE_ON_STACK 80 #define ABI_A0 0 #define ABI_A1 8@@ -62,8 +61,7 @@ * After the stack is established, * * 0(sp) stores the PC of the traced function which can be accessed -* by &(fregs)->epc in tracing function. Note that the real -* function entry address should be computed with -FENTRY_RA_OFFSET. +* by &(fregs)->epc in tracing function. * * 8(sp) stores the function return address (i.e. parent IP) that * can be accessed by &(fregs)->ra in tracing function.@@ -140,7 +138,7 @@ .endm .macro PREPARE_ARGS - addi a0, t0, -FENTRY_RA_OFFSET + addi a0, t0, -MCOUNT_JALR_SIZE // ip (callsite's jalr insn) la a1, function_trace_op REG_L a2, 0(a1) mv a1, ra@@ -153,7 +151,8 @@ SYM_FUNC_START(ftrace_caller) PREPARE_ARGS SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL) - call ftrace_stub + REG_L ra, ftrace_call_dest + jalr ra, 0(ra) RESTORE_ABI_REGS bnez t1, .Ldirect
Alexandre Ghiti
reviewerRe: [PATCH v4 05/12] riscv: ftrace: prepare ftrace for atomic code patching
Quoted 179 lines · Alexandre Ghiti
Hi Andy,
Quoted 149 lines · Andy Chiu
From: Andy Chiu <[email protected]>
We use an AUIPC+JALR pair to jump into a ftrace trampoline. Since instruction fetch can break down to 4 byte at a time, it is impossible to update two instructions without a race. In order to mitigate it, we initialize the patchable entry to AUIPC + NOP4. Then, the run-time code patching can change NOP4 to JALR to eable/disable ftrcae from a function. This limits the reach of each ftrace entry to +-2KB displacing from
ftrace_caller.Starting from the trampoline, we add a level of indirection for it to reach ftrace caller target. Now, it loads the target address from a memory location, then perform the jump. This enable the kernel to update the target atomically.
The new don't-stop-the-world text patching on change only one RISC-V instruction:
| -8: &ftrace_ops of the associated tracer function. | <ftrace enable>: | 0: auipc t0, hi(ftrace_caller) | 4: jalr t0, lo(ftrace_caller) | | -8: &ftrace_nop_ops | <ftrace disable>: | 0: auipc t0, hi(ftrace_caller) | 4: nopThis means that
f+0x0is fixed, and should not be claimed by ftrace, e.g. kprobe should be able to put a probe inf+0x0. Thus, we adjust the offset andMCOUNT_INSN_SIZEaccordingly.Co-developed-by Björn Töpel <[email protected]>Signed-off-by Björn Töpel <[email protected]>Signed-off-by Andy Chiu <[email protected]>---
Changelog v4:
- Include Björn's fix for kprobe
- Refactor code for better reading (Robbin, Björn)
- Removemake_call_raand friedns (Björn)
- Update comments to match reality (Björn)
- Drop code defined by !WITH_ARG
- Add a synchronization point when updatingftrace_call_dest(Björn)
---
arch/riscv/include/asm/ftrace.h| 49 ++++++------ arch/riscv/kernel/ftrace.c | 130 ++++++++++++++++---------------- arch/riscv/kernel/mcount-dyn.S | 9 +--3 files changed, 92 insertions(+), 96 deletions(-)arch/riscv/include/asm/ftrace.h +4
-4@@ -20,10 +20,9 @@ extern void *return_address(unsigned int level); #define ftrace_return_address(n) return_address(n) void _mcount(void); -static inline unsigned long ftrace_call_adjust (unsigned long addr) -{ - return addr; -} +unsigned long ftrace_call_adjust (unsigned long addr); +unsigned long arch_ftrace_get_symaddr (unsigned long fentry_ip); +#define ftrace_get_symaddr (fentry_ip) arch_ftrace_get_symaddr(fentry_ip) /** Let's do like x86/arm64 and ignore the compat syscalls. @@ -57,12 +56,21 @@ struct dyn_arch_ftrace { * 2) jalr: setting low-12 offset to ra, jump to ra, and set ra to * return address (original pc + 4) * + * The first 2 instructions for each tracable function is compiledto 2 nop
+ * instructions. Then, the kernel initializes the first instruction to auipc at
+ * boot time (<ftrace disable>). The second instruction is patched to jalr to
+ * start the trace.
+ *
+ *<Image>:
+ * 0: nop
+ * 4: nop
+ *
*<ftrace enable>:
- * 0: auipc t0/ra, 0x?
- * 4: jalr t0/ra, ?(t0/ra)
+ * 0: auipc t0, 0x?
+ * 4: jalr t0, ?(t0)
*
*<ftrace disable>:
- * 0: nop
+ * 0: auipc t0, 0x?
* 4: nop
*
* Dynamic ftrace generates probes to call sites, so we must deal witharch/riscv/include/asm/ftrace.h +1
-18@@ -75,10 +83,9 @@ struct dyn_arch_ftrace { #define AUIPC_OFFSET_MASK (0xfffff000) #define AUIPC_PAD (0x00001000) #define JALR_SHIFT 20 -#define JALR_RA (0x000080e7) -#define AUIPC_RA (0x00000097) #define JALR_T0 (0x000282e7) #define AUIPC_T0 (0x00000297) +#define JALR_RANGE (JALR_SIGN_MASK - 1) #define to_jalr_t0(offset) \ (((offset & JALR_OFFSET_MASK) << JALR_SHIFT) | JALR_T0)@@ -96,26 +103,14 @@ do { \ call[1] = to_jalr_t0(offset); \ } while (0) -#define to_jalr_ra(offset) \ - (((offset & JALR_OFFSET_MASK) << JALR_SHIFT) | JALR_RA) - -#define to_auipc_ra (offset) \ - ((offset & JALR_SIGN_MASK) ? \ - (((offset & AUIPC_OFFSET_MASK) + AUIPC_PAD) | AUIPC_RA) : \ - ((offset & AUIPC_OFFSET_MASK) | AUIPC_RA)) - -#define make_call_ra (caller, callee, call) \ -do { \ - unsigned int offset = \ - (unsigned long) (callee) - (unsigned long) (caller); \ - call[0] = to_auipc_ra(offset); \ - call[1] = to_jalr_ra(offset); \ -} while (0) -/* -* Let auipc+jalr be the basic *mcount unit*, so we make it 8 bytes here.
+ * Only the jalr insn in the auipc+jalr is patched, so we make it 4
+ * bytes here.
*/arch/riscv/include/asm/ftrace.h +4
-1-#define MCOUNT_INSN_SIZE 8 +#define MCOUNT_INSN_SIZE 4 +#define MCOUNT_AUIPC_SIZE 4 +#define MCOUNT_JALR_SIZE 4 +#define MCOUNT_NOP4_SIZE 4#ifndef
__ASSEMBLY__
structdyn_ftrace;arch/riscv/kernel/ftrace.c +1
-0@@ -8,10 +8,21 @@ #include <linux/ftrace.h> #include <linux/uaccess.h> #include <linux/memory.h> +#include <linux/irqflags.h> #include <linux/stop_machine.h> #include <asm/cacheflush.h> #include <asm/text-patching.h>arch/riscv/kernel/ftrace.c line 24
Quoted 10 lines
arch/riscv/kernel/ftrace.c +9
-0+unsigned long ftrace_call_adjust(unsigned long addr) +{ + return addr + MCOUNT_AUIPC_SIZE; +} + +unsigned long arch_ftrace_get_symaddr (unsigned long fentry_ip) +{ + return fentry_ip - MCOUNT_AUIPC_SIZE; +} +Those functions cause the following errors when building with !CONFIG_DYNAMIC_FTRACE, but I'm not sure how to fix this:
../arch/riscv/kernel/ftrace.c: In function 'ftrace_call_adjust':../arch/riscv/kernel/ftrace.c:19:35: error: 'MCOUNT_AUIPC_SIZE' undeclared (first use in this function) 19 | return addr + 8 +MCOUNT_AUIPC_SIZE;
| ^~~~~~~~~~~~~~~~~../arch/riscv/kernel/ftrace.c:19:35: note: each undeclared identifier is reported only once for each function it appears in CCfs/9p/vfs_dir.o../arch/riscv/kernel/ftrace.c: In function 'arch_ftrace_get_symaddr':../arch/riscv/kernel/ftrace.c:26:28: error: 'MCOUNT_AUIPC_SIZE' undeclared (first use in this function) 26 | returnfentry_ip-MCOUNT_AUIPC_SIZE;
| ^~~~~~~~~~~~~~~~~
CCdrivers/pci/pcie/pme.o../arch/riscv/kernel/ftrace.c: In function 'ftrace_call_adjust':../arch/riscv/kernel/ftrace.c:22:1: error: control reaches end of non-void function [-Werror=return-type]
Quoted 9 lines
22 | }
| ^../arch/riscv/kernel/ftrace.c: In function 'arch_ftrace_get_symaddr':../arch/riscv/kernel/ftrace.c:27:1: error: control reaches end of non-void function [-Werror=return-type] 27 | }
| ^
cc1: some warnings being treated as errors
make[5]: *** [../scripts/Makefile.build:203:arch/riscv/kernel/ftrace.o] Error 1
So I fixed those errors with the following, let me know if that's not correct:
arch/riscv/kernel/ftrace.c +0 -0
@@ -13,6 +13,7 @@
#include <asm/cacheflush.h>
#include <asm/text-patching.h>
+#ifdef CONFIG_DYNAMIC_FTRACE
unsigned long ftrace_call_adjust(unsigned long addr)
{
if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS))
@@ -26,7 +27,6 @@ unsigned long arch_ftrace_get_symaddr(unsigned long fentry_ip) return fentry_ip - MCOUNT_AUIPC_SIZE;
}
-#ifdef CONFIG_DYNAMIC_FTRACE
void arch_ftrace_update_code(int command)
{
mutex_lock(&text_mutex);
@@ -191,7 +191,12 @@ int ftrace_update_ftrace_func(ftrace_func_t func) return 0;
}
-#endif
+#else /* CONFIG_DYNAMIC_FTRACE */
+unsigned long ftrace_call_adjust(unsigned long addr) +{
+ return addr;
+}
+#endif /* CONFIG_DYNAMIC_FTRACE */
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
Thanks,
Alex
Quoted 214 lines
Quoted 202 lines
arch/riscv/kernel/ftrace.c +2
-2#ifdef CONFIG_DYNAMIC_FTRACE void ftrace_arch_code_modify_prepare(void) __acquires(&text_mutex) {@@ -32,51 +43,32 @@ void ftrace_arch_code_modify_post_process(void) __releases(&text_mutex) mutex_unlock(&text_mutex); } -static int ftrace_check_current_call(unsigned long hook_pos, - unsigned int *expected) +static int __ftrace_modify_call (unsigned long source,unsigned long target,bool validate) { + unsigned int call[2], offset; unsigned int replaced[2]; - unsigned int nops[2] = {RISCV_INSN_NOP4, RISCV_INSN_NOP4};- /* we expect nops at the hook position */
- if (!expected)
- expected = nops;
+ offset = target - source;
+ call[1] =to_jalr_t0(offset);
- /*
- * Read the text we want to modify;
- * return must be-EFAULTon read error
- */
- if (copy_from_kernel_nofault(replaced, (void *)hook_pos, - MCOUNT_INSN_SIZE))
- return-EFAULT;
-
- /*
- * Make sure it is what we expect it to be;
- * return must be-EINVALon failed comparison
- */
- if (memcmp(expected, replaced, sizeof(replaced))) {
- pr_err("%p: expected (%08x %08x) but got (%08x %08x)\n", - (void *)hook_pos, expected[0], expected[1], replaced[0], - replaced[1]);
- return-EINVAL;
+ if (validate) {
+ call[0] =to_auipc_t0(offset);
+ /*
+ * Read the text we want to modify;
+ * return must be-EFAULTon read error
+ */
+ if (copy_from_kernel_nofault(replaced, (void *)source, 2 * MCOUNT_INSN_SIZE))
+ return-EFAULT;
+
+ if (replaced[0] != call[0]) {
+ pr_err("%p: expected (%08x) but got (%08x)\n", + (void *)source, call[0], replaced[0]);
+ return-EINVAL;
+ }
}
- return 0;arch/riscv/kernel/ftrace.c +0
-3-} - -static int __ftrace_modify_call(unsigned long hook_pos, unsignedlong target,
- bool enable, bool ra)arch/riscv/kernel/ftrace.c +0
-10-{ - unsigned int call[2]; - unsigned int nops[2] = {RISCV_INSN_NOP4, RISCV_INSN_NOP4}; - - if (ra) - make_call_ra(hook_pos, target, call); - else - make_call_t0(hook_pos, target, call); - - /* Replace the auipc-jalr pair at once. Return -EPERM on writeerror. */
- if (patch_insn_write((void *)hook_pos, enable ? call : nops, MCOUNT_INSN_SIZE))
+ /* Replace the jalr at once. Return-EPERMon write error. */ + if (patch_insn_write((void *)(source +MCOUNT_AUIPC_SIZE), call
+ 1,MCOUNT_JALR_SIZE))
return-EPERM;
return 0;arch/riscv/kernel/ftrace.c +9
-6@@ -84,22 +76,21 @@ static int __ftrace_modify_call(unsigned long hook_pos, unsigned long target, int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) { - unsigned int call[2]; + unsigned long distance, orig_addr, pc = rec->ip -MCOUNT_AUIPC_SIZE ; - make_call_t0(rec->ip, addr, call); - - if (patch_insn_write((void *)rec->ip, call, MCOUNT_INSN_SIZE)) - return -EPERM; + orig_addr = (unsigned long)&ftrace_caller; + distance = addr > orig_addr ? addr - orig_addr : orig_addr - addr; + if (distance > JALR_RANGE) + return -EINVAL; - return 0; + return __ftrace_modify_call(pc, addr, false); } -int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, - unsigned long addr) +int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec,unsigned long addr) { - unsigned int nops[2] = {RISCV_INSN_NOP4, RISCV_INSN_NOP4}; + u32 nop4 = RISCV_INSN_NOP4; - if (patch_insn_write((void *)rec->ip, nops, MCOUNT_INSN_SIZE)) + if (patch_insn_write((void *)rec->ip, &nop4, MCOUNT_NOP4_SIZE)) return -EPERM;return 0;
arch/riscv/kernel/ftrace.c +12
-6@@ -114,21 +105,38 @@ int ftrace_make_nop(struct module *mod, structdyn_ftrace *rec, */ int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec) { - int out; + unsigned long pc = rec->ip - MCOUNT_AUIPC_SIZE; + unsigned int nops[2], offset; + int ret; + + offset = (unsigned long) &ftrace_caller - pc; + nops[0] = to_auipc_t0(offset); + nops[1] = RISCV_INSN_NOP4; mutex_lock(&text_mutex); - out = ftrace_make_nop(mod, rec, MCOUNT_ADDR); + ret = patch_insn_write((void *)pc, nops, 2 * MCOUNT_INSN_SIZE); mutex_unlock(&text_mutex); - return out; + return ret; } +ftrace_func_t ftrace_call_dest = ftrace_stub; int ftrace_update_ftrace_func(ftrace_func_t func) { - int ret = __ftrace_modify_call((unsigned long)&ftrace_call, - (unsigned long)func, true, true); - - return ret; + WRITE_ONCE(ftrace_call_dest, func); + /* +* The data fence ensure that the update to ftrace_call_dest happens
+ * before the write tofunction_trace_oplater in the generic ftrace.
+ * If the sequence is not enforced, then an oldftrace_call_destmay
+ * race loading a newfunction_trace_opset inftrace_modify_all_code
+ *
+ * If we are instop_machine, then we don't need to call remote fence
+ * as there is no concurrent read-side offtrace_call_dest. + */
+smp_wmb();
+ if (!irqs_disabled())
+smp_call_function(ftrace_sync_ipi, NULL, 1);
+ return 0;
}
structftrace_modify_param{arch/riscv/kernel/ftrace.c +2
-8@@ -172,17 +180,11 @@ void arch_ftrace_update_code(int command) int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, unsigned long addr) { + unsigned long caller = rec->ip - MCOUNT_AUIPC_SIZE; unsigned int call[2]; - unsigned long caller = rec->ip; - int ret; make_call_t0(caller, old_addr, call); - ret = ftrace_check_current_call(caller, call); - - if (ret) - return ret; - - return __ftrace_modify_call(caller, addr, true, false); + return __ftrace_modify_call(caller, addr, true); } #endifdiff --git
a/arch/riscv/kernel/mcount-dyn.Sb/arch/riscv/kernel/mcount-dyn.Sindex3f06b40bb6c8..8aa554d56096100644arch/riscv/kernel/mcount-dyn.S +0
-0@@ -13,7 +13,6 @@.text
-#define FENTRY_RA_OFFSET 8
#define ABI_SIZE_ON_STACK 80
#define ABI_A0 0
#define ABI_A1 8arch/riscv/kernel/mcount-dyn.S +1
-2@@ -62,8 +61,7 @@ * After the stack is established, * * 0(sp) stores the PC of the traced function which can be accessed -* by &(fregs)->epc in tracing function. Note that the real -* function entry address should be computed with -FENTRY_RA_OFFSET. +* by &(fregs)->epc in tracing function. * * 8(sp) stores the function return address (i.e. parent IP) that * can be accessed by &(fregs)->ra in tracing function.@@ -140,7 +138,7 @@.endm
.macroPREPARE_ARGS
- addi a0, t0, -FENTRY_RA_OFFSET
+ addi a0, t0, -MCOUNT_JALR_SIZE // ip (callsite's jalr insn) la a1,function_trace_op
REG_L a2, 0(a1)
mv a1, raarch/riscv/kernel/mcount-dyn.S +0
-0@@ -153,7 +151,8 @@ SYM_FUNC_START(ftrace_caller)Quoted 7 lines
PREPARE_ARGS
SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL)
- callftrace_stub
+ REG_L ra,ftrace_call_dest
+ jalr ra, 0(ra)
RESTORE_ABI_REGS
bnez t1, .Ldirect_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv
Re: [PATCH v4 05/12] riscv: ftrace: prepare ftrace for atomic code patching
Quoted 228 lines · Alexandre Ghiti
Quoted 187 lines · Alexandre Ghiti
Hi Andy,
Quoted 150 lines · Andy Chiu
From: Andy Chiu <[email protected]>
We use an AUIPC+JALR pair to jump into a ftrace trampoline. Since instruction fetch can break down to 4 byte at a time, it is impossible to update two instructions without a race. In order to mitigate it, we initialize the patchable entry to AUIPC + NOP4. Then, the run-time code patching can change NOP4 to JALR to eable/disable ftrcae from a function. This limits the reach of each ftrace entry to +-2KB displacing from
ftrace_caller.Starting from the trampoline, we add a level of indirection for it to reach ftrace caller target. Now, it loads the target address from a memory location, then perform the jump. This enable the kernel to update the target atomically.
The new don't-stop-the-world text patching on change only one RISC-V instruction:
| -8: &ftrace_ops of the associated tracer function. | <ftrace enable>: | 0: auipc t0, hi(ftrace_caller) | 4: jalr t0, lo(ftrace_caller) | | -8: &ftrace_nop_ops | <ftrace disable>: | 0: auipc t0, hi(ftrace_caller) | 4: nopThis means that
f+0x0is fixed, and should not be claimed by ftrace, e.g. kprobe should be able to put a probe inf+0x0. Thus, we adjust the offset andMCOUNT_INSN_SIZEaccordingly.Co-developed-by Björn Töpel <[email protected]>Signed-off-by Björn Töpel <[email protected]>Signed-off-by Andy Chiu <[email protected]>---
Changelog v4:
- Include Björn's fix for kprobe
- Refactor code for better reading (Robbin, Björn)
- Removemake_call_raand friedns (Björn)
- Update comments to match reality (Björn)
- Drop code defined by !WITH_ARG
- Add a synchronization point when updatingftrace_call_dest(Björn)--- arch/riscv/include/asm/ftrace.h | 49 ++++++------ arch/riscv/kernel/ftrace.c | 130 ++++++++++++++++---------------- arch/riscv/kernel/mcount-dyn.S | 9 +-- 3 files changed, 92 insertions(+), 96 deletions(-)arch/riscv/include/asm/ftrace.h +4
-4@@ -20,10 +20,9 @@ extern void *return_address(unsigned int level);#define ftrace_return_address (n) return_address(n)void _mcount (void ); -static inline unsigned long ftrace_call_adjust (unsigned long addr) -{ -return addr; -} +unsigned long ftrace_call_adjust (unsigned long addr); +unsigned long arch_ftrace_get_symaddr (unsigned long fentry_ip); +#define ftrace_get_symaddr (fentry_ip)arch_ftrace_get_symaddr (fentry_ip)/* * Let's do like x86/arm64 and ignore the compat syscalls. @@ -57,12 +56,21 @@ struct dyn_arch_ftrace {* 2) jalr: setting low-12 offset to ra, jump to ra, and set ra to * return address (original pc + 4) * +* The first 2 instructions for each tracable function is compiled to 2 nop
+ * instructions. Then, the kernel initializes the first instruction to auipc at
+ * boot time (<ftrace disable>). The second instruction is patched to jalr to
+ * start the trace.
+ *
+ *<Image>:
+ * 0: nop
+ * 4: nop
+ *
*<ftrace enable>:- * 0: auipc t0/ra, 0x? - * 4: jalr t0/ra, ?(t0/ra) + * 0: auipc t0, 0x? + * 4: jalr t0, ?(t0)*
*<ftrace disable>:
- * 0: nop
+ * 0: auipc t0, 0x?
* 4: nop
*
* Dynamic ftrace generates probes to call sites, so we must deal witharch/riscv/include/asm/ftrace.h +1
-18@@ -75,10 +83,9 @@ struct dyn_arch_ftrace {#define AUIPC_OFFSET_MASK (0xfffff000)#define AUIPC_PAD (0x00001000)#define JALR_SHIFT 20 -#define JALR_RA (0x000080e7) -#define AUIPC_RA (0x00000097)#define JALR_T0 (0x000282e7)#define AUIPC_T0 (0x00000297) +#define JALR_RANGE (JALR_SIGN_MASK - 1)#define to_jalr_t0 (offset) \ (((offset & JALR_OFFSET_MASK) << JALR_SHIFT) | JALR_T0)@@ -96,26 +103,14 @@ do { \ call[1 ] =to_jalr_t0 (offset); \ }while (0 ) -#define to_jalr_ra(offset) \ - (((offset & JALR_OFFSET_MASK) << JALR_SHIFT) | JALR_RA) - -#define to_auipc_ra (offset) \ - ((offset & JALR_SIGN_MASK) ? \ - (((offset & AUIPC_OFFSET_MASK) + AUIPC_PAD) | AUIPC_RA) : \ - ((offset & AUIPC_OFFSET_MASK) | AUIPC_RA)) - -#define make_call_ra (caller, callee, call) \ -do { \ - unsigned int offset = \ - (unsigned long) (callee) - (unsigned long) (caller); \ - call[0] = to_auipc_ra(offset); \ - call[1] = to_jalr_ra(offset); \ -} while (0) -/* -* Let auipc+jalr be the basic *mcount unit*, so we make it 8 bytes here.
+ * Only the jalr insn in the auipc+jalr is patched, so we make it 4
+ * bytes here.
*/arch/riscv/include/asm/ftrace.h +4
-1-#define MCOUNT_INSN_SIZE 8 +#define MCOUNT_INSN_SIZE 4 +#define MCOUNT_AUIPC_SIZE 4 +#define MCOUNT_JALR_SIZE 4 +#define MCOUNT_NOP4_SIZE 4#ifndef __ASSEMBLY__struct dyn_ftrace ;arch/riscv/kernel/ftrace.c +1
-0@@ -8,10 +8,21 @@#include <linux/ftrace.h> #include <linux/uaccess.h> #include <linux/memory.h> +#include <linux/irqflags.h> #include <linux/stop_machine.h> #include <asm/cacheflush.h> #include <asm/text-patching.h> arch/riscv/kernel/ftrace.c line 24
Quoted 10 lines
arch/riscv/kernel/ftrace.c +9
-0+unsigned long ftrace_call_adjust(unsigned long addr) +{ +return addr +MCOUNT_AUIPC_SIZE ; +} + +unsigned long arch_ftrace_get_symaddr (unsigned long fentry_ip) +{ +return fentry_ip -MCOUNT_AUIPC_SIZE ; +} +Those functions cause the following errors when building with !CONFIG_DYNAMIC_FTRACE, but I'm not sure how to fix this:
../arch/riscv/kernel/ftrace.c: In function 'ftrace_call_adjust':../arch/riscv/kernel/ftrace.c:19:35: error: 'MCOUNT_AUIPC_SIZE' undeclared (first use in this function)19 | return addr + 8 + MCOUNT_AUIPC_SIZE; | ^~~~~~~~~~~~~~~~~
../arch/riscv/kernel/ftrace.c:19:35: note: each undeclared identifier is reported only once for each function it appears in
CCfs/9p/vfs_dir.o../arch/riscv/kernel/ftrace.c: In function 'arch_ftrace_get_symaddr':../arch/riscv/kernel/ftrace.c:26:28: error: 'MCOUNT_AUIPC_SIZE' undeclared (first use in this function)26 | return fentry_ip - MCOUNT_AUIPC_SIZE; | ^~~~~~~~~~~~~~~~~CC
drivers/pci/pcie/pme.o../arch/riscv/kernel/ftrace.c: In function 'ftrace_call_adjust':../arch/riscv/kernel/ftrace.c:22:1: error: control reaches end of non-void function [-Werror=return-type]
22 | }| ^Quoted 9 lines
../arch/riscv/kernel/ftrace.c: In function 'arch_ftrace_get_symaddr':../arch/riscv/kernel/ftrace.c:27:1: error: control reaches end of non-void function [-Werror=return-type]
27 | }| ^cc1: some warnings being treated as errors
make[5]: *** [../scripts/Makefile.build:203:arch/riscv/kernel/ftrace.o] Error 1So I fixed those errors with the following, let me know if that's not correct:
arch/riscv/kernel/ftrace.c +1
-1@@ -13,6 +13,7 @@#include <asm/cacheflush.h> #include <asm/text-patching.h> +#ifdef CONFIG_DYNAMIC_FTRACE unsigned long ftrace_call_adjust (unsigned long addr) {if (IS_ENABLED (CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS ))@@ -26,7 +27,6 @@ unsigned long arch_ftrace_get_symaddr(unsigned long fentry_ip)return fentry_ip -MCOUNT_AUIPC_SIZE ; } -#ifdef CONFIG_DYNAMIC_FTRACE void arch_ftrace_update_code (int command) {mutex_lock(&text_mutex);arch/riscv/kernel/ftrace.c +0
-0@@ -191,7 +191,12 @@ int ftrace_update_ftrace_func(ftrace_func_t func)return 0 ; }
arch/riscv/kernel/ftrace.c int ftrace_update_ftrace_func(ftrace_func_t func) · line 202
Quoted 10 lines
arch/riscv/kernel/ftrace.c +6
-1-#endif +#else /* CONFIG_DYNAMIC_FTRACE */ +unsigned long ftrace_call_adjust (unsigned long addr) +{ +return addr; +} +#endif /* CONFIG_DYNAMIC_FTRACE */ #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
Hi Alex,
Yes, this is valid, thanks for noticing and fixing the error. I would personally prefer leaving the #else /* CONFIG_DYNAMIC_FTRACE */ part in ftrace.h, but it can also come later as a refactor. Or, I can respin the series, with fixes on this and the previous patch, along with some typos and variable declarations that Robin mentioned.
Thanks,
Andy
Alexandre Ghiti
reviewerRe: [PATCH v4 05/12] riscv: ftrace: prepare ftrace for atomic code patching
Hi Andy,
Quoted 238 lines · Andy Chiu
Quoted 225 lines · Alexandre Ghiti
Quoted 191 lines · Alexandre Ghiti
Hi Andy,
Quoted 150 lines · Andy Chiu
From: Andy Chiu <[email protected]>
We use an AUIPC+JALR pair to jump into a ftrace trampoline. Since instruction fetch can break down to 4 byte at a time, it is impossible to update two instructions without a race. In order to mitigate it, we initialize the patchable entry to AUIPC + NOP4. Then, the run-time code patching can change NOP4 to JALR to eable/disable ftrcae from a function. This limits the reach of each ftrace entry to +-2KB displacing from
ftrace_caller.Starting from the trampoline, we add a level of indirection for it to reach ftrace caller target. Now, it loads the target address from a memory location, then perform the jump. This enable the kernel to update the target atomically.
The new don't-stop-the-world text patching on change only one RISC-V instruction:
| -8: &ftrace_ops of the associated tracer function. | <ftrace enable>: | 0: auipc t0, hi(ftrace_caller) | 4: jalr t0, lo(ftrace_caller) | | -8: &ftrace_nop_ops | <ftrace disable>: | 0: auipc t0, hi(ftrace_caller) | 4: nopThis means that
f+0x0is fixed, and should not be claimed by ftrace, e.g. kprobe should be able to put a probe inf+0x0. Thus, we adjust the offset andMCOUNT_INSN_SIZEaccordingly.Co-developed-by Björn Töpel <[email protected]>Signed-off-by Björn Töpel <[email protected]>Signed-off-by Andy Chiu <[email protected]>---
Changelog v4:
- Include Björn's fix for kprobe
- Refactor code for better reading (Robbin, Björn)
- Removemake_call_raand friedns (Björn)
- Update comments to match reality (Björn)
- Drop code defined by !WITH_ARG
- Add a synchronization point when updatingftrace_call_dest(Björn)--- arch/riscv/include/asm/ftrace.h | 49 ++++++------ arch/riscv/kernel/ftrace.c | 130 ++++++++++++++++---------------- arch/riscv/kernel/mcount-dyn.S | 9 +-- 3 files changed, 92 insertions(+), 96 deletions(-)arch/riscv/include/asm/ftrace.h +4
-4@@ -20,10 +20,9 @@ extern void *return_address(unsigned int level);#define ftrace_return_address (n) return_address(n)void _mcount (void ); -static inline unsigned long ftrace_call_adjust (unsigned long addr) -{ -return addr; -} +unsigned long ftrace_call_adjust (unsigned long addr); +unsigned long arch_ftrace_get_symaddr (unsigned long fentry_ip); +#define ftrace_get_symaddr (fentry_ip)arch_ftrace_get_symaddr (fentry_ip)/* * Let's do like x86/arm64 and ignore the compat syscalls. @@ -57,12 +56,21 @@ struct dyn_arch_ftrace {* 2) jalr: setting low-12 offset to ra, jump to ra, and set ra to * return address (original pc + 4) * +* The first 2 instructions for each tracable function is compiled to 2 nop
+ * instructions. Then, the kernel initializes the first instruction to auipc at
+ * boot time (<ftrace disable>). The second instruction is patched to jalr to
+ * start the trace.
+ *
+ *<Image>:
+ * 0: nop
+ * 4: nop
+ *
*<ftrace enable>:- * 0: auipc t0/ra, 0x? - * 4: jalr t0/ra, ?(t0/ra) + * 0: auipc t0, 0x? + * 4: jalr t0, ?(t0) * *<ftrace disable>:- * 0: nop
+ * 0: auipc t0, 0x?
* 4: nop
*
* Dynamic ftrace generates probes to call sites, so we must deal witharch/riscv/include/asm/ftrace.h +1
-18@@ -75,10 +83,9 @@ struct dyn_arch_ftrace {#define AUIPC_OFFSET_MASK (0xfffff000)#define AUIPC_PAD (0x00001000)#define JALR_SHIFT 20 -#define JALR_RA (0x000080e7) -#define AUIPC_RA (0x00000097)#define JALR_T0 (0x000282e7)#define AUIPC_T0 (0x00000297) +#define JALR_RANGE (JALR_SIGN_MASK - 1)#define to_jalr_t0 (offset) \ (((offset & JALR_OFFSET_MASK) << JALR_SHIFT) | JALR_T0)@@ -96,26 +103,14 @@ do { \ call[1 ] =to_jalr_t0 (offset); \ }while (0 ) -#define to_jalr_ra(offset) \ - (((offset & JALR_OFFSET_MASK) << JALR_SHIFT) | JALR_RA) - -#define to_auipc_ra (offset) \ - ((offset & JALR_SIGN_MASK) ? \ - (((offset & AUIPC_OFFSET_MASK) + AUIPC_PAD) | AUIPC_RA) : \ - ((offset & AUIPC_OFFSET_MASK) | AUIPC_RA)) - -#define make_call_ra (caller, callee, call) \ -do { \ - unsigned int offset = \ - (unsigned long) (callee) - (unsigned long) (caller); \ - call[0] = to_auipc_ra(offset); \ - call[1] = to_jalr_ra(offset); \ -} while (0) -/* -* Let auipc+jalr be the basic *mcount unit*, so we make it 8 bytes here.
+ * Only the jalr insn in the auipc+jalr is patched, so we make it 4
+ * bytes here.*/arch/riscv/include/asm/ftrace.h +4
-1-#define MCOUNT_INSN_SIZE 8 +#define MCOUNT_INSN_SIZE 4 +#define MCOUNT_AUIPC_SIZE 4 +#define MCOUNT_JALR_SIZE 4 +#define MCOUNT_NOP4_SIZE 4#ifndef __ASSEMBLY__struct dyn_ftrace ;arch/riscv/kernel/ftrace.c +1
-0@@ -8,10 +8,21 @@#include <linux/ftrace.h> #include <linux/uaccess.h> #include <linux/memory.h> +#include <linux/irqflags.h> #include <linux/stop_machine.h> #include <asm/cacheflush.h> #include <asm/text-patching.h> arch/riscv/kernel/ftrace.c line 24
Quoted 10 lines
arch/riscv/kernel/ftrace.c +9
-0+unsigned long ftrace_call_adjust(unsigned long addr) +{ +return addr +MCOUNT_AUIPC_SIZE ; +} + +unsigned long arch_ftrace_get_symaddr (unsigned long fentry_ip) +{ +return fentry_ip -MCOUNT_AUIPC_SIZE ; +} +Those functions cause the following errors when building with !CONFIG_DYNAMIC_FTRACE, but I'm not sure how to fix this:
../arch/riscv/kernel/ftrace.c: In function 'ftrace_call_adjust':../arch/riscv/kernel/ftrace.c:19:35: error: 'MCOUNT_AUIPC_SIZE' undeclared (first use in this function)19 | return addr + 8 + MCOUNT_AUIPC_SIZE; | ^~~~~~~~~~~~~~~~~
../arch/riscv/kernel/ftrace.c:19:35: note: each undeclared identifier is reported only once for each function it appears in
CCfs/9p/vfs_dir.o../arch/riscv/kernel/ftrace.c: In function 'arch_ftrace_get_symaddr':../arch/riscv/kernel/ftrace.c:26:28: error: 'MCOUNT_AUIPC_SIZE' undeclared (first use in this function)26 | return fentry_ip - MCOUNT_AUIPC_SIZE; | ^~~~~~~~~~~~~~~~~CC
drivers/pci/pcie/pme.o../arch/riscv/kernel/ftrace.c: In function 'ftrace_call_adjust':../arch/riscv/kernel/ftrace.c:22:1: error: control reaches end of non-void function [-Werror=return-type]22 | } | ^ ../arch/riscv/kernel/ftrace.c: In function 'arch_ftrace_get_symaddr': ../arch/riscv/kernel/ftrace.c:27:1: error: control reaches end of non-void function [-Werror=return-type] 27 | } | ^Quoted 4 lines
cc1: some warnings being treated as errors
make[5]: *** [../scripts/Makefile.build:203:arch/riscv/kernel/ftrace.o] Error 1So I fixed those errors with the following, let me know if that's not correct:
arch/riscv/kernel/ftrace.c +1
-1@@ -13,6 +13,7 @@#include <asm/cacheflush.h> #include <asm/text-patching.h> +#ifdef CONFIG_DYNAMIC_FTRACE unsigned long ftrace_call_adjust (unsigned long addr) {if (IS_ENABLED (CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS ))@@ -26,7 +27,6 @@ unsigned long arch_ftrace_get_symaddr(unsigned long fentry_ip)return fentry_ip -MCOUNT_AUIPC_SIZE ; } -#ifdef CONFIG_DYNAMIC_FTRACE void arch_ftrace_update_code (int command) {mutex_lock(&text_mutex);arch/riscv/kernel/ftrace.c +0
-0@@ -191,7 +191,12 @@ int ftrace_update_ftrace_func(ftrace_func_t func)return 0 ; }arch/riscv/kernel/ftrace.c int ftrace_update_ftrace_func(ftrace_func_t func) · line 202
Quoted 10 lines
arch/riscv/kernel/ftrace.c +6
-1-#endif +#else /* CONFIG_DYNAMIC_FTRACE */ +unsigned long ftrace_call_adjust (unsigned long addr) +{ +return addr; +} +#endif /* CONFIG_DYNAMIC_FTRACE */ #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,Hi Alex,
Quoted 5 lines
Yes, this is valid, thanks for noticing and fixing the error. I would personally prefer leaving the #else /*
CONFIG_DYNAMIC_FTRACE*/ part inftrace.h, but it can also come later as a refactor. Or, I can respin the series, with fixes on this and the previous patch, along with some typos and variable declarations that Robin mentioned.
If you can respin a new patchset soon, that's nice, in the meantime I keep this version + my fixes (without Robbin remarks though) in my for-next branch as we definitely want this in 6.16 :)
So it's up to you!
Thanks,
Alex
Quoted 3 lines
Thanks,
Andy
PATCH v4 6/12 riscv: ftrace: do not use stop_machine to update code
In reply to Andy ChiuFrom: Andy Chiu <[email protected]>
Now it is safe to remove dependency from stop_machine() for us to patch code in ftrace.
---
Changelog v4:
- assume ftrace_update_ftrace_func is always called with irqs enabled
--- arch/riscv/kernel/ftrace.c | 64 ++++++-------------------------------- 1 file changed, 10 insertions(+), 54 deletions(-)
arch/riscv/kernel/ftrace.c +10 -54
@@ -24,23 +24,13 @@ unsigned long arch_ftrace_get_symaddr(unsigned long fentry_ip) }#ifdef CONFIG_DYNAMIC_FTRACE -void ftrace_arch_code_modify_prepare(void) __acquires(&text_mutex) +void arch_ftrace_update_code (int command) {mutex_lock (&text_mutex); - -/* -* The code sequences we use for ftrace can't be patched while the -* kernel is running, so we need to use stop_machine() to modify them -* for now. This doesn't play nice with text_mutex, we use this flag -* to elide the check. -*/ - riscv_patch_in_stop_machine = true; -} - -void ftrace_arch_code_modify_post_process(void) __releases(&text_mutex) -{ - riscv_patch_in_stop_machine = false; + command |=FTRACE_MAY_SLEEP ; +ftrace_modify_all_code (command);mutex_unlock (&text_mutex); +flush_icache_all (); }static int __ftrace_modify_call (unsigned long source,unsigned long target,bool validate)
@@ -129,51 +119,17 @@ int ftrace_update_ftrace_func(ftrace_func_t func)* before the write to function_trace_op later in the generic ftrace. * If the sequence is not enforced, then an old ftrace_call_dest may * race loading a new function_trace_op set in ftrace_modify_all_code -* -* If we are in stop_machine, then we don't need to call remote fence -* as there is no concurrent read-side of ftrace_call_dest. */ smp_wmb (); -if (!irqs_disabled ()) -smp_call_function (ftrace_sync_ipi,NULL ,1 ); - return 0; -} - -struct ftrace_modify_param { -int command; -atomic_t cpu_count; -}; - -static int __ftrace_modify_code (void *data) -{ -struct ftrace_modify_param *param = data; - -if (atomic_inc_return (¶m->cpu_count) ==num_online_cpus ()) { -ftrace_modify_all_code (param->command); -/* -* Make sure the patching store is effective *before* we -* increment the counter which releases all waiting CPUs -* by using the release variant of atomic increment. The -* release pairs with the call to local_flush_icache_all() -* on the waiting CPU. -*/ -atomic_inc_return_release (¶m->cpu_count); - }else { -while (atomic_read (¶m->cpu_count) <=num_online_cpus ()) -cpu_relax (); - -local_flush_icache_all (); - } - +/* +* Updating ftrace dpes not take stop_machine path, so irqs should not +* be disabled. +*/ +WARN_ON (irqs_disabled ()); +smp_call_function (ftrace_sync_ipi,NULL ,1 ); return 0; } -void arch_ftrace_update_code (int command) -{ -struct ftrace_modify_param param = { command,ATOMIC_INIT (0 ) }; - -stop_machine (__ftrace_modify_code, ¶m, cpu_online_mask); -}#endif #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
Signature · 1 line
2.39.3 (Apple Git-145)
PATCH v4 7/12 riscv: vector: Support calling schedule() for preemptible Vector
In reply to Andy ChiuFrom: Andy Chiu <[email protected]>
Each function entry implies a call to ftrace infrastructure. And it may call into schedule in some cases. So, it is possible for preemptible kernel-mode Vector to implicitly call into schedule. Since all V-regs are caller-saved, it is possible to drop all V context when a thread voluntarily call schedule(). Besides, we currently don't pass argument through vector register, so we don't have to save/restore V-regs in ftrace trampoline.
--- arch/riscv/include/asm/processor.h | 5 +++++ arch/riscv/include/asm/vector.h | 22 +++++++++++++++++++--- 2 files changed, 24 insertions(+), 3 deletions(-)
arch/riscv/include/asm/processor.h +5 -0
@@ -79,6 +79,10 @@ struct pt_regs;* Thus, the task does not own preempt_v. Any use of Vector will have to * save preempt_v, if dirty, and fallback to non-preemptible kernel-mode * Vector. +* - bit 29: The thread voluntarily calls schedule() while holding an active +* preempt_v. All preempt_v context should be dropped in such case because +* V-regs are caller-saved. Only sstatus.VS=ON is persisted across a +* schedule() call. * - bit 30: The in-kernel preempt_v context is saved, and requries to be * restored when returning to the context that owns the preempt_v. * - bit 31: The in-kernel preempt_v context is dirty, as signaled by the
@@ -93,6 +97,7 @@ struct pt_regs;#define RISCV_PREEMPT_V 0x00000100#define RISCV_PREEMPT_V_DIRTY 0x80000000#define RISCV_PREEMPT_V_NEED_RESTORE 0x40000000 +#define RISCV_PREEMPT_V_IN_SCHEDULE 0x20000000/* CPU-specific state of a task */ struct thread_struct {
arch/riscv/include/asm/vector.h +19 -3
@@ -120,6 +120,11 @@ static __always_inline void riscv_v_disable(void)csr_clear (CSR_SSTATUS ,SR_VS ); } +static __always_inline bool riscv_v_is_on(void) +{ +return !!(csr_read (CSR_SSTATUS ) &SR_VS ); +} + static __always_inline void __vstate_csr_save(struct __riscv_v_ext_state *dest) { asm volatile (
@@ -366,6 +371,11 @@ static inline void __switch_to_vector(struct task_struct *prev,struct pt_regs *regs;if (riscv_preempt_v_started (prev)) { +if (riscv_v_is_on ()) { +WARN_ON (prev->thread.riscv_v_flags &RISCV_V_CTX_DEPTH_MASK ); +riscv_v_disable (); + prev->thread.riscv_v_flags |=RISCV_PREEMPT_V_IN_SCHEDULE ; + }if (riscv_preempt_v_dirty (prev)) {__riscv_v_vstate_save (&prev->thread.kernel_vstate, prev->thread.kernel_vstate.datap);
@@ -376,10 +386,16 @@ static inline void __switch_to_vector(struct task_struct *prev,riscv_v_vstate_save (&prev->thread.vstate, regs); } -if (riscv_preempt_v_started (next)) -riscv_preempt_v_set_restore (next); -else +if (riscv_preempt_v_started (next)) { +if (next->thread.riscv_v_flags &RISCV_PREEMPT_V_IN_SCHEDULE ) { + next->thread.riscv_v_flags &= ~RISCV_PREEMPT_V_IN_SCHEDULE ; +riscv_v_enable (); + }else { +riscv_preempt_v_set_restore (next); + } + }else {riscv_v_vstate_set_restore (next,task_pt_regs (next)); + } }void riscv_v_vstate_ctrl_init (struct task_struct *tsk);
Signature · 1 line
2.39.3 (Apple Git-145)
PATCH v4 8/12 riscv: add a data fence for CMODX in the kernel mode
In reply to Andy ChiuRISC-V spec explicitly calls out that a local fence.i is not enough for the code modification to be visble from a remote hart. In fact, it states:
To make a store to instruction memory visible to all RISC-V harts, the writing hart also has to execute a data FENCE before requesting that all remote RISC-V harts execute a FENCE.I.
Although current riscv drivers for IPI use ordered MMIO when sending IPIs in order to synchronize the action between previous csd writes, riscv does not restrict itself to any particular flavor of IPI. Any driver or firmware implementation that does not order data writes before the IPI may pose a risk for code-modifying race.
Thus, add a fence here to order data writes before making the IPI.
---
Changelog v4:
- Explain more in commit msg and add Björn's R-b
--- arch/riscv/mm/cacheflush.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-)
arch/riscv/mm/cacheflush.c +14 -1
@@ -24,7 +24,20 @@ void flush_icache_all(void)if (num_online_cpus () <2 )return ; -else if (riscv_use_sbi_for_rfence ()) + +/* +* Make sure all previous writes to the D$ are ordered before making +* the IPI. The RISC-V spec states that a hart must execute a data fence +* before triggering a remote fence.i in order to make the modification +* visable for remote harts. +* +* IPIs on RISC-V are triggered by MMIO writes to either CLINT or +* S-IMSIC, so the fence ensures previous data writes "happen before" +* the MMIO. +*/ +RISCV_FENCE (w, o); + +if (riscv_use_sbi_for_rfence ())sbi_remote_fence_i (NULL );else on_each_cpu (ipi_remote_fence_i,NULL ,1 );
Signature · 1 line
2.39.3 (Apple Git-145)
PATCH v4 9/12 riscv: ftrace: support PREEMPT
In reply to Andy ChiuFrom: Andy Chiu <[email protected]>
Now, we can safely enable dynamic ftrace with kernel preemption.
--- arch/riscv/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/riscv/Kconfig +1 -1
@@ -157,7 +157,7 @@ config RISCV select HAVE_FTRACE_MCOUNT_RECORD if !XIP_KERNEL select HAVE_FUNCTION_GRAPH_TRACER if HAVE_DYNAMIC_FTRACE_WITH_ARGS select HAVE_FUNCTION_GRAPH_FREGS - select HAVE_FUNCTION_TRACER if !XIP_KERNEL && !PREEMPTION + select HAVE_FUNCTION_TRACER if !XIP_KERNEL select HAVE_EBPF_JIT if MMU select HAVE_GUP_FAST if MMU select HAVE_FUNCTION_ARG_ACCESS_API
Signature · 1 line
2.39.3 (Apple Git-145)
PATCH v4 10/12 riscv: Implement HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS
In reply to Andy ChiuFrom: Puranjay Mohan <[email protected]>
This patch enables support for DYNAMIC_FTRACE_WITH_CALL_OPS on RISC-V. This allows each ftrace callsite to provide an ftrace_ops to the common ftrace trampoline, allowing each callsite to invoke distinct tracer functions without the need to fall back to list processing or to allocate custom trampolines for each callsite. This significantly speeds up cases where multiple distinct trace functions are used and callsites are mostly traced by a single tracer.
The idea and most of the implementation is taken from the ARM64's implementation of the same feature. The idea is to place a pointer to the ftrace_ops as a literal at a fixed offset from the function entry point, which can be recovered by the common ftrace trampoline.
We use -fpatchable-function-entry to reserve 8 bytes above the function entry by emitting 2 4 byte or 4 2 byte nops depending on the presence of CONFIG_RISCV_ISA_C. These 8 bytes are patched at runtime with a pointer to the associated ftrace_ops for that callsite. Functions are aligned to 8 bytes to make sure that the accesses to this literal are atomic.
This approach allows for directly invoking ftrace_ops::func even for ftrace_ops which are dynamically-allocated (or part of a module), without going via ftrace_ops_list_func.
We've benchamrked this with the ftrace_ops sample module on Spacemit K1 Jupiter:
Without this patch:
baseline (Linux rivos 6.14.0-09584-g7d06015d936c #3 SMP Sat Mar 29
+-----------------------+-----------------+----------------------------+ | Number of tracers | Total time (ns) | Per-call average time | |-----------------------+-----------------+----------------------------| | Relevant | Irrelevant | 100000 calls | Total (ns) | Overhead (ns) | |----------+------------+-----------------+------------+---------------| | 0 | 0 | 1357958 | 13 | - | | 0 | 1 | 1302375 | 13 | - | | 0 | 2 | 1302375 | 13 | - | | 0 | 10 | 1379084 | 13 | - | | 0 | 100 | 1302458 | 13 | - | | 0 | 200 | 1302333 | 13 | - | |----------+------------+-----------------+------------+---------------| | 1 | 0 | 13677833 | 136 | 123 | | 1 | 1 | 18500916 | 185 | 172 | | 1 | 2 | 22856459 | 228 | 215 | | 1 | 10 | 58824709 | 588 | 575 | | 1 | 100 | 505141584 | 5051 | 5038 | | 1 | 200 | 1580473126 | 15804 | 15791 | |----------+------------+-----------------+------------+---------------| | 1 | 0 | 13561000 | 135 | 122 | | 2 | 0 | 19707292 | 197 | 184 | | 10 | 0 | 67774750 | 677 | 664 | | 100 | 0 | 714123125 | 7141 | 7128 | | 200 | 0 | 1918065668 | 19180 | 19167 | +----------+------------+-----------------+------------+---------------+
Note: per-call overhead is estimated relative to the baseline case with 0 relevant tracers and 0 irrelevant tracers.
With this patch:
v4-rc4 (Linux rivos 6.14.0-09598-gd75747611c93 #4 SMP Sat Mar 29
+-----------------------+-----------------+----------------------------+ | Number of tracers | Total time (ns) | Per-call average time | |-----------------------+-----------------+----------------------------| | Relevant | Irrelevant | 100000 calls | Total (ns) | Overhead (ns) | |----------+------------+-----------------+------------+---------------| | 0 | 0 | 1459917 | 14 | - | | 0 | 1 | 1408000 | 14 | - | | 0 | 2 | 1383792 | 13 | - | | 0 | 10 | 1430709 | 14 | - | | 0 | 100 | 1383791 | 13 | - | | 0 | 200 | 1383750 | 13 | - | |----------+------------+-----------------+------------+---------------| | 1 | 0 | 5238041 | 52 | 38 | | 1 | 1 | 5228542 | 52 | 38 | | 1 | 2 | 5325917 | 53 | 40 | | 1 | 10 | 5299667 | 52 | 38 | | 1 | 100 | 5245250 | 52 | 39 | | 1 | 200 | 5238459 | 52 | 39 | |----------+------------+-----------------+------------+---------------| | 1 | 0 | 5239083 | 52 | 38 | | 2 | 0 | 19449417 | 194 | 181 | | 10 | 0 | 67718584 | 677 | 663 | | 100 | 0 | 709840708 | 7098 | 7085 | | 200 | 0 | 2203580626 | 22035 | 22022 | +----------+------------+-----------------+------------+---------------+
Note: per-call overhead is estimated relative to the baseline case with 0 relevant tracers and 0 irrelevant tracers.
As can be seen from the above:
a) Whenever there is a single relevant tracer function associated with a tracee, the overhead of invoking the tracer is constant, and does not scale with the number of tracers which are *not* associated with that tracee.
b) The overhead for a single relevant tracer has dropped to ~1/3 of the overhead prior to this series (from 122ns to 38ns). This is largely due to permitting calls to dynamically-allocated ftrace_ops without going through ftrace_ops_list_func.
[update kconfig, asm, refactor]
---
Changelog v4:
- include benchmark result from a real hardware, shout-out to Björn!
- new patch copy from Puranjay's RFC implementation
- Drop code related to !WITH_ARG && DYNAMIC_FTRACE
--- arch/riscv/Kconfig | 2 + arch/riscv/Makefile | 4 +- arch/riscv/kernel/asm-offsets.c | 3 ++ arch/riscv/kernel/ftrace.c | 67 +++++++++++++++++++++++++++++++++ arch/riscv/kernel/mcount-dyn.S | 35 +++++++++++++++-- 5 files changed, 105 insertions(+), 6 deletions(-)
arch/riscv/Kconfig +2 -0
@@ -99,6 +99,7 @@ config RISCV select EDAC_SUPPORT select FRAME_POINTER if PERF_EVENTS || (FUNCTION_TRACER && !DYNAMIC_FTRACE) select FTRACE_MCOUNT_USE_PATCHABLE_FUNCTION_ENTRY if DYNAMIC_FTRACE + select FUNCTION_ALIGNMENT_8B if DYNAMIC_FTRACE_WITH_CALL_OPS select GENERIC_ARCH_TOPOLOGY select GENERIC_ATOMIC64 if !64BIT select GENERIC_CLOCKEVENTS_BROADCAST if SMP
@@ -152,6 +153,7 @@ config RISCV select HAVE_DYNAMIC_FTRACE if !XIP_KERNEL && MMU && (CLANG_SUPPORTS_DYNAMIC_FTRACE || GCC_SUPPORTS_DYNAMIC_FTRACE) select FUNCTION_ALIGNMENT_4B if HAVE_DYNAMIC_FTRACE && RISCV_ISA_C select HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS + select HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS if (DYNAMIC_FTRACE_WITH_ARGS && !CFI_CLANG) select HAVE_DYNAMIC_FTRACE_WITH_ARGS if HAVE_DYNAMIC_FTRACE select HAVE_FTRACE_GRAPH_FUNC select HAVE_FTRACE_MCOUNT_RECORD if !XIP_KERNEL
arch/riscv/Makefile +2 -2
@@ -15,9 +15,9 @@ ifeq ($(CONFIG_DYNAMIC_FTRACE),y) LDFLAGS_vmlinux += --no-relax KBUILD_CPPFLAGS += -DCC_USING_PATCHABLE_FUNCTION_ENTRY ifeq ($(CONFIG_RISCV_ISA_C),y) - CC_FLAGS_FTRACE := -fpatchable-function-entry=4 + CC_FLAGS_FTRACE := -fpatchable-function-entry=8,4 else - CC_FLAGS_FTRACE := -fpatchable-function-entry=2 + CC_FLAGS_FTRACE := -fpatchable-function-entry=4,2 endif endif
arch/riscv/kernel/asm-offsets.c +3 -0
@@ -493,6 +493,9 @@ void asm_offsets(void)DEFINE (STACKFRAME_SIZE_ON_STACK ,ALIGN (sizeof (struct stackframe ),STACK_ALIGN ));OFFSET (STACKFRAME_FP , stackframe, fp);OFFSET (STACKFRAME_RA , stackframe, ra); +#ifdef CONFIG_FUNCTION_TRACER +DEFINE (FTRACE_OPS_FUNC , offsetof(struct ftrace_ops , func)); +#endif #ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS DEFINE (FREGS_SIZE_ON_STACK ,ALIGN (sizeof (struct __arch_ftrace_regs ),STACK_ALIGN ));
arch/riscv/kernel/ftrace.c +67 -0
108 lines not on this page · raw message
arch/riscv/kernel/mcount-dyn.S +31 -4
@@ -139,10 +139,34 @@ .macro PREPARE_ARGS addi a0, t0, -MCOUNT_JALR_SIZE // ip (callsite's jalr insn) +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS + /* + * When CALL_OPS is enabled (2 or 4) nops [8B] are placed before the + * function entry, these are later overwritten with the pointer to the + * associated struct ftrace_ops. + * + * -8: &ftrace_ops of the associated tracer function. + *<ftrace enable>: + * 0: auipc t0/ra, 0x? + * 4: jalr t0/ra, ?(t0/ra) + * + * -8: &ftrace_nop_ops + *<ftrace disable>: + * 0: nop + * 4: nop + * + * t0 is set to ip+8 after the jalr is executed at the callsite, + * so we find the associated op at t0-16. + */ + mv a1, ra // parent_ip + REG_L a2, -16(t0) // op + REG_L ra, FTRACE_OPS_FUNC(a2) // op->func +#else la a1, function_trace_op - REG_L a2, 0(a1) - mv a1, ra - mv a3, sp + REG_L a2, 0(a1) // op + mv a1, ra // parent_ip +#endif + mv a3, sp // regs .endm SYM_FUNC_START(ftrace_caller)
@@ -150,10 +174,13 @@ SYM_FUNC_START(ftrace_caller) SAVE_ABI_REGS PREPARE_ARGS +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS + jalr ra +#else SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL) REG_L ra, ftrace_call_dest jalr ra, 0(ra) - +#endif RESTORE_ABI_REGS bnez t1, .Ldirect jr t0
Signature · 1 line
2.39.3 (Apple Git-145)
Re: [PATCH v4 10/12] riscv: Implement HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS
Hey,
Quoted 104 lines · Andy Chiu
From: Puranjay Mohan <[email protected]>
This patch enables support for
DYNAMIC_FTRACE_WITH_CALL_OPSon RISC-V. This allows each ftrace callsite to provide anftrace_opsto the common ftrace trampoline, allowing each callsite to invoke distinct tracer functions without the need to fall back to list processing or to allocate custom trampolines for each callsite. This significantly speeds up cases where multiple distinct trace functions are used and callsites are mostly traced by a single tracer.The idea and most of the implementation is taken from the ARM64's implementation of the same feature. The idea is to place a pointer to the
ftrace_opsas a literal at a fixed offset from the function entry point, which can be recovered by the common ftrace trampoline.We use -fpatchable-function-entry to reserve 8 bytes above the function entry by emitting 2 4 byte or 4 2 byte nops depending on the presence of
CONFIG_RISCV_ISA_C. These 8 bytes are patched at runtime with a pointer to the associatedftrace_opsfor that callsite. Functions are aligned to 8 bytes to make sure that the accesses to this literal are atomic.This approach allows for directly invoking
ftrace_ops::funceven forftrace_opswhich are dynamically-allocated (or part of a module), without going viaftrace_ops_list_func.We've benchamrked this with the
ftrace_opssample module on Spacemit K1 Jupiter:Without this patch:
baseline (Linux rivos 6.14.0-09584-g7d06015d936c #3 SMP Sat Mar 29
+-----------------------+-----------------+----------------------------+ | Number of tracers | Total time (ns) | Per-call average time | |-----------------------+-----------------+----------------------------| | Relevant | Irrelevant | 100000 calls | Total (ns) | Overhead (ns) | |----------+------------+-----------------+------------+---------------| | 0 | 0 | 1357958 | 13 | - | | 0 | 1 | 1302375 | 13 | - | | 0 | 2 | 1302375 | 13 | - | | 0 | 10 | 1379084 | 13 | - | | 0 | 100 | 1302458 | 13 | - | | 0 | 200 | 1302333 | 13 | - | |----------+------------+-----------------+------------+---------------| | 1 | 0 | 13677833 | 136 | 123 | | 1 | 1 | 18500916 | 185 | 172 | | 1 | 2 | 22856459 | 228 | 215 | | 1 | 10 | 58824709 | 588 | 575 | | 1 | 100 | 505141584 | 5051 | 5038 | | 1 | 200 | 1580473126 | 15804 | 15791 | |----------+------------+-----------------+------------+---------------| | 1 | 0 | 13561000 | 135 | 122 | | 2 | 0 | 19707292 | 197 | 184 | | 10 | 0 | 67774750 | 677 | 664 | | 100 | 0 | 714123125 | 7141 | 7128 | | 200 | 0 | 1918065668 | 19180 | 19167 | +----------+------------+-----------------+------------+---------------+Note: per-call overhead is estimated relative to the baseline case with 0 relevant tracers and 0 irrelevant tracers.
With this patch:
v4-rc4 (Linux rivos 6.14.0-09598-gd75747611c93 #4 SMP Sat Mar 29
+-----------------------+-----------------+----------------------------+ | Number of tracers | Total time (ns) | Per-call average time | |-----------------------+-----------------+----------------------------| | Relevant | Irrelevant | 100000 calls | Total (ns) | Overhead (ns) | |----------+------------+-----------------+------------+---------------| | 0 | 0 | 1459917 | 14 | - | | 0 | 1 | 1408000 | 14 | - | | 0 | 2 | 1383792 | 13 | - | | 0 | 10 | 1430709 | 14 | - | | 0 | 100 | 1383791 | 13 | - | | 0 | 200 | 1383750 | 13 | - | |----------+------------+-----------------+------------+---------------| | 1 | 0 | 5238041 | 52 | 38 | | 1 | 1 | 5228542 | 52 | 38 | | 1 | 2 | 5325917 | 53 | 40 | | 1 | 10 | 5299667 | 52 | 38 | | 1 | 100 | 5245250 | 52 | 39 | | 1 | 200 | 5238459 | 52 | 39 | |----------+------------+-----------------+------------+---------------| | 1 | 0 | 5239083 | 52 | 38 | | 2 | 0 | 19449417 | 194 | 181 | | 10 | 0 | 67718584 | 677 | 663 | | 100 | 0 | 709840708 | 7098 | 7085 | | 200 | 0 | 2203580626 | 22035 | 22022 | +----------+------------+-----------------+------------+---------------+Note: per-call overhead is estimated relative to the baseline case with 0 relevant tracers and 0 irrelevant tracers.
As can be seen from the above:
a) Whenever there is a single relevant tracer function associated with a tracee, the overhead of invoking the tracer is constant, and does not scale with the number of tracers which are *not* associated with that tracee.
b) The overhead for a single relevant tracer has dropped to ~1/3 of the overhead prior to this series (from 122ns to 38ns). This is largely due to permitting calls to dynamically-allocated
ftrace_opswithout going throughftrace_ops_list_func.
Quoted 6 lines
Signed-off-by Puranjay Mohan <[email protected]>[update kconfig, asm, refactor]
Signed-off-by Andy Chiu <[email protected]>Tested-by Björn Töpel <[email protected]>
I bisected a boot failure to this commit [c217157bcd1df ("riscv:
Implement HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS")] yesterday, that appears to be affecting all LLVM versions that I currently have installed. From some initial testing of Kconfig options, it looks like the issue is CFI_CLANG related because when I disable CFI_CLANG things work once more. Since this option depends on !CFI_CLANG, but is def_bool y, I modified Kconfig to force disable it at all times and tested !DYNAMIC_FTRACE_WITH_CALL_OPS && !CFG_CLANG, which did boot.
I dunno anything about what's going on in this patch, but so little in it relates to having DYNAMIC_FTRACE_WITH_CALL_OPS, that I was able to figure out that the problem is -fpatchable-function-entry=8,4
FWIW, if anyone checks out this commit directly, you'll need to cherry-pick commit e9d86b8e17e72 ("scripts: Do not strip .rela.dyn section"), as the base of the branch that c217157bcd1df is on is v6.15-rc3, which is in itself broken in turn by the issue fixed by e9d86b8e17e72. Probably not someone anyone will do, but made for an awful time trying to figure out what commit was at fault!
Cheers,
Conor.
Re: [PATCH v4 10/12] riscv: Implement HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS
Quoted 124 lines · Conor Dooley
Hey,
Quoted 104 lines · Andy Chiu
From: Puranjay Mohan <[email protected]>
This patch enables support for
DYNAMIC_FTRACE_WITH_CALL_OPSon RISC-V. This allows each ftrace callsite to provide anftrace_opsto the common ftrace trampoline, allowing each callsite to invoke distinct tracer functions without the need to fall back to list processing or to allocate custom trampolines for each callsite. This significantly speeds up cases where multiple distinct trace functions are used and callsites are mostly traced by a single tracer.The idea and most of the implementation is taken from the ARM64's implementation of the same feature. The idea is to place a pointer to the
ftrace_opsas a literal at a fixed offset from the function entry point, which can be recovered by the common ftrace trampoline.We use -fpatchable-function-entry to reserve 8 bytes above the function entry by emitting 2 4 byte or 4 2 byte nops depending on the presence of
CONFIG_RISCV_ISA_C. These 8 bytes are patched at runtime with a pointer to the associatedftrace_opsfor that callsite. Functions are aligned to 8 bytes to make sure that the accesses to this literal are atomic.This approach allows for directly invoking
ftrace_ops::funceven forftrace_opswhich are dynamically-allocated (or part of a module), without going viaftrace_ops_list_func.We've benchamrked this with the
ftrace_opssample module on Spacemit K1 Jupiter:Without this patch:
baseline (Linux rivos 6.14.0-09584-g7d06015d936c #3 SMP Sat Mar 29
+-----------------------+-----------------+----------------------------+ | Number of tracers | Total time (ns) | Per-call average time | |-----------------------+-----------------+----------------------------| | Relevant | Irrelevant | 100000 calls | Total (ns) | Overhead (ns) | |----------+------------+-----------------+------------+---------------| | 0 | 0 | 1357958 | 13 | - | | 0 | 1 | 1302375 | 13 | - | | 0 | 2 | 1302375 | 13 | - | | 0 | 10 | 1379084 | 13 | - | | 0 | 100 | 1302458 | 13 | - | | 0 | 200 | 1302333 | 13 | - | |----------+------------+-----------------+------------+---------------| | 1 | 0 | 13677833 | 136 | 123 | | 1 | 1 | 18500916 | 185 | 172 | | 1 | 2 | 22856459 | 228 | 215 | | 1 | 10 | 58824709 | 588 | 575 | | 1 | 100 | 505141584 | 5051 | 5038 | | 1 | 200 | 1580473126 | 15804 | 15791 | |----------+------------+-----------------+------------+---------------| | 1 | 0 | 13561000 | 135 | 122 | | 2 | 0 | 19707292 | 197 | 184 | | 10 | 0 | 67774750 | 677 | 664 | | 100 | 0 | 714123125 | 7141 | 7128 | | 200 | 0 | 1918065668 | 19180 | 19167 | +----------+------------+-----------------+------------+---------------+Note: per-call overhead is estimated relative to the baseline case with 0 relevant tracers and 0 irrelevant tracers.
With this patch:
v4-rc4 (Linux rivos 6.14.0-09598-gd75747611c93 #4 SMP Sat Mar 29
+-----------------------+-----------------+----------------------------+ | Number of tracers | Total time (ns) | Per-call average time | |-----------------------+-----------------+----------------------------| | Relevant | Irrelevant | 100000 calls | Total (ns) | Overhead (ns) | |----------+------------+-----------------+------------+---------------| | 0 | 0 | 1459917 | 14 | - | | 0 | 1 | 1408000 | 14 | - | | 0 | 2 | 1383792 | 13 | - | | 0 | 10 | 1430709 | 14 | - | | 0 | 100 | 1383791 | 13 | - | | 0 | 200 | 1383750 | 13 | - | |----------+------------+-----------------+------------+---------------| | 1 | 0 | 5238041 | 52 | 38 | | 1 | 1 | 5228542 | 52 | 38 | | 1 | 2 | 5325917 | 53 | 40 | | 1 | 10 | 5299667 | 52 | 38 | | 1 | 100 | 5245250 | 52 | 39 | | 1 | 200 | 5238459 | 52 | 39 | |----------+------------+-----------------+------------+---------------| | 1 | 0 | 5239083 | 52 | 38 | | 2 | 0 | 19449417 | 194 | 181 | | 10 | 0 | 67718584 | 677 | 663 | | 100 | 0 | 709840708 | 7098 | 7085 | | 200 | 0 | 2203580626 | 22035 | 22022 | +----------+------------+-----------------+------------+---------------+Note: per-call overhead is estimated relative to the baseline case with 0 relevant tracers and 0 irrelevant tracers.
As can be seen from the above:
a) Whenever there is a single relevant tracer function associated with a tracee, the overhead of invoking the tracer is constant, and does not scale with the number of tracers which are *not* associated with that tracee.
b) The overhead for a single relevant tracer has dropped to ~1/3 of the overhead prior to this series (from 122ns to 38ns). This is largely due to permitting calls to dynamically-allocated
ftrace_opswithout going throughftrace_ops_list_func.Quoted 6 lines
Signed-off-by Puranjay Mohan <[email protected]>[update kconfig, asm, refactor]
Signed-off-by Andy Chiu <[email protected]>Tested-by Björn Töpel <[email protected]>I bisected a boot failure to this commit [
c217157bcd1df("riscv:
ImplementHAVE_DYNAMIC_FTRACE_WITH_CALL_OPS")] yesterday, that appears to be affecting all LLVM versions that I currently have installed. From some initial testing ofKconfigoptions, it looks like the issue isCFI_CLANGrelated because when I disableCFI_CLANGthings work once more. Since this option depends on !CFI_CLANG, but isdef_booly, I modifiedKconfigto force disable it at all times and tested !DYNAMIC_FTRACE_WITH_CALL_OPS && !CFG_CLANG, which did boot.
Quoted 4 lines
I dunno anything about what's going on in this patch, but so little in it relates to having
DYNAMIC_FTRACE_WITH_CALL_OPS, that I was able to figure out that the problem is -fpatchable-function-entry=8,4
DYNAMIC_FTRACE_WITH_CALL_OPS can't work together with CFI_CLANG.
arm64 has:
select HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS \
if (DYNAMIC_FTRACE_WITH_ARGS && !CFI && \
(CC_IS_CLANG || !CC_OPTIMIZE_FOR_SIZE))
would need something similar for riscv if not already done.
Thanks,
Puranjay
Re: [PATCH v4 10/12] riscv: Implement HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS
Quoted 132 lines · Puranjay Mohan
Quoted 124 lines · Conor Dooley
Hey,
Quoted 104 lines · Andy Chiu
From: Puranjay Mohan <[email protected]>
This patch enables support for
DYNAMIC_FTRACE_WITH_CALL_OPSon RISC-V. This allows each ftrace callsite to provide anftrace_opsto the common ftrace trampoline, allowing each callsite to invoke distinct tracer functions without the need to fall back to list processing or to allocate custom trampolines for each callsite. This significantly speeds up cases where multiple distinct trace functions are used and callsites are mostly traced by a single tracer.The idea and most of the implementation is taken from the ARM64's implementation of the same feature. The idea is to place a pointer to the
ftrace_opsas a literal at a fixed offset from the function entry point, which can be recovered by the common ftrace trampoline.We use -fpatchable-function-entry to reserve 8 bytes above the function entry by emitting 2 4 byte or 4 2 byte nops depending on the presence of
CONFIG_RISCV_ISA_C. These 8 bytes are patched at runtime with a pointer to the associatedftrace_opsfor that callsite. Functions are aligned to 8 bytes to make sure that the accesses to this literal are atomic.This approach allows for directly invoking
ftrace_ops::funceven forftrace_opswhich are dynamically-allocated (or part of a module), without going viaftrace_ops_list_func.We've benchamrked this with the
ftrace_opssample module on Spacemit K1 Jupiter:Without this patch:
baseline (Linux rivos 6.14.0-09584-g7d06015d936c #3 SMP Sat Mar 29
+-----------------------+-----------------+----------------------------+ | Number of tracers | Total time (ns) | Per-call average time | |-----------------------+-----------------+----------------------------| | Relevant | Irrelevant | 100000 calls | Total (ns) | Overhead (ns) | |----------+------------+-----------------+------------+---------------| | 0 | 0 | 1357958 | 13 | - | | 0 | 1 | 1302375 | 13 | - | | 0 | 2 | 1302375 | 13 | - | | 0 | 10 | 1379084 | 13 | - | | 0 | 100 | 1302458 | 13 | - | | 0 | 200 | 1302333 | 13 | - | |----------+------------+-----------------+------------+---------------| | 1 | 0 | 13677833 | 136 | 123 | | 1 | 1 | 18500916 | 185 | 172 | | 1 | 2 | 22856459 | 228 | 215 | | 1 | 10 | 58824709 | 588 | 575 | | 1 | 100 | 505141584 | 5051 | 5038 | | 1 | 200 | 1580473126 | 15804 | 15791 | |----------+------------+-----------------+------------+---------------| | 1 | 0 | 13561000 | 135 | 122 | | 2 | 0 | 19707292 | 197 | 184 | | 10 | 0 | 67774750 | 677 | 664 | | 100 | 0 | 714123125 | 7141 | 7128 | | 200 | 0 | 1918065668 | 19180 | 19167 | +----------+------------+-----------------+------------+---------------+Note: per-call overhead is estimated relative to the baseline case with 0 relevant tracers and 0 irrelevant tracers.
With this patch:
v4-rc4 (Linux rivos 6.14.0-09598-gd75747611c93 #4 SMP Sat Mar 29
+-----------------------+-----------------+----------------------------+ | Number of tracers | Total time (ns) | Per-call average time | |-----------------------+-----------------+----------------------------| | Relevant | Irrelevant | 100000 calls | Total (ns) | Overhead (ns) | |----------+------------+-----------------+------------+---------------| | 0 | 0 | 1459917 | 14 | - | | 0 | 1 | 1408000 | 14 | - | | 0 | 2 | 1383792 | 13 | - | | 0 | 10 | 1430709 | 14 | - | | 0 | 100 | 1383791 | 13 | - | | 0 | 200 | 1383750 | 13 | - | |----------+------------+-----------------+------------+---------------| | 1 | 0 | 5238041 | 52 | 38 | | 1 | 1 | 5228542 | 52 | 38 | | 1 | 2 | 5325917 | 53 | 40 | | 1 | 10 | 5299667 | 52 | 38 | | 1 | 100 | 5245250 | 52 | 39 | | 1 | 200 | 5238459 | 52 | 39 | |----------+------------+-----------------+------------+---------------| | 1 | 0 | 5239083 | 52 | 38 | | 2 | 0 | 19449417 | 194 | 181 | | 10 | 0 | 67718584 | 677 | 663 | | 100 | 0 | 709840708 | 7098 | 7085 | | 200 | 0 | 2203580626 | 22035 | 22022 | +----------+------------+-----------------+------------+---------------+Note: per-call overhead is estimated relative to the baseline case with 0 relevant tracers and 0 irrelevant tracers.
As can be seen from the above:
a) Whenever there is a single relevant tracer function associated with a tracee, the overhead of invoking the tracer is constant, and does not scale with the number of tracers which are *not* associated with that tracee.
b) The overhead for a single relevant tracer has dropped to ~1/3 of the overhead prior to this series (from 122ns to 38ns). This is largely due to permitting calls to dynamically-allocated
ftrace_opswithout going throughftrace_ops_list_func.Quoted 6 lines
Signed-off-by Puranjay Mohan <[email protected]>[update kconfig, asm, refactor]
Signed-off-by Andy Chiu <[email protected]>Tested-by Björn Töpel <[email protected]>I bisected a boot failure to this commit [
c217157bcd1df("riscv:
ImplementHAVE_DYNAMIC_FTRACE_WITH_CALL_OPS")] yesterday, that appears to be affecting all LLVM versions that I currently have installed. From some initial testing ofKconfigoptions, it looks like the issue isCFI_CLANGrelated because when I disableCFI_CLANGthings work once more. Since this option depends on !CFI_CLANG, but isdef_booly, I modifiedKconfigto force disable it at all times and tested !DYNAMIC_FTRACE_WITH_CALL_OPS && !CFG_CLANG, which did boot.Quoted 4 lines
I dunno anything about what's going on in this patch, but so little in it relates to having
DYNAMIC_FTRACE_WITH_CALL_OPS, that I was able to figure out that the problem is -fpatchable-function-entry=8,4
DYNAMIC_FTRACE_WITH_CALL_OPScan't work together withCFI_CLANG.
Quoted 7 lines
arm64 has:
select
HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS\if (DYNAMIC_FTRACE_WITH_ARGS && !CFI && \ (CC_IS_CLANG || !CC_OPTIMIZE_FOR_SIZE))would need something similar for riscv if not already done.
I think you've misunderstood my email. We already have:
select HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS if (DYNAMIC_FTRACE_WITH_ARGS && !CFI)
The problem is that the patch broke using CFI_CLANG, due to the fpatchable-function-entry change.
Cheers,
Conor.
Re: [PATCH v4 10/12] riscv: Implement HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS
Quoted 143 lines · Conor Dooley
Quoted 132 lines · Puranjay Mohan
Quoted 124 lines · Conor Dooley
Hey,
Quoted 104 lines · Andy Chiu
From: Puranjay Mohan <[email protected]>
This patch enables support for
DYNAMIC_FTRACE_WITH_CALL_OPSon RISC-V. This allows each ftrace callsite to provide anftrace_opsto the common ftrace trampoline, allowing each callsite to invoke distinct tracer functions without the need to fall back to list processing or to allocate custom trampolines for each callsite. This significantly speeds up cases where multiple distinct trace functions are used and callsites are mostly traced by a single tracer.The idea and most of the implementation is taken from the ARM64's implementation of the same feature. The idea is to place a pointer to the
ftrace_opsas a literal at a fixed offset from the function entry point, which can be recovered by the common ftrace trampoline.We use -fpatchable-function-entry to reserve 8 bytes above the function entry by emitting 2 4 byte or 4 2 byte nops depending on the presence of
CONFIG_RISCV_ISA_C. These 8 bytes are patched at runtime with a pointer to the associatedftrace_opsfor that callsite. Functions are aligned to 8 bytes to make sure that the accesses to this literal are atomic.This approach allows for directly invoking
ftrace_ops::funceven forftrace_opswhich are dynamically-allocated (or part of a module), without going viaftrace_ops_list_func.We've benchamrked this with the
ftrace_opssample module on Spacemit K1 Jupiter:Without this patch:
baseline (Linux rivos 6.14.0-09584-g7d06015d936c #3 SMP Sat Mar 29
+-----------------------+-----------------+----------------------------+ | Number of tracers | Total time (ns) | Per-call average time | |-----------------------+-----------------+----------------------------| | Relevant | Irrelevant | 100000 calls | Total (ns) | Overhead (ns) | |----------+------------+-----------------+------------+---------------| | 0 | 0 | 1357958 | 13 | - | | 0 | 1 | 1302375 | 13 | - | | 0 | 2 | 1302375 | 13 | - | | 0 | 10 | 1379084 | 13 | - | | 0 | 100 | 1302458 | 13 | - | | 0 | 200 | 1302333 | 13 | - | |----------+------------+-----------------+------------+---------------| | 1 | 0 | 13677833 | 136 | 123 | | 1 | 1 | 18500916 | 185 | 172 | | 1 | 2 | 22856459 | 228 | 215 | | 1 | 10 | 58824709 | 588 | 575 | | 1 | 100 | 505141584 | 5051 | 5038 | | 1 | 200 | 1580473126 | 15804 | 15791 | |----------+------------+-----------------+------------+---------------| | 1 | 0 | 13561000 | 135 | 122 | | 2 | 0 | 19707292 | 197 | 184 | | 10 | 0 | 67774750 | 677 | 664 | | 100 | 0 | 714123125 | 7141 | 7128 | | 200 | 0 | 1918065668 | 19180 | 19167 | +----------+------------+-----------------+------------+---------------+Note: per-call overhead is estimated relative to the baseline case with 0 relevant tracers and 0 irrelevant tracers.
With this patch:
v4-rc4 (Linux rivos 6.14.0-09598-gd75747611c93 #4 SMP Sat Mar 29
+-----------------------+-----------------+----------------------------+ | Number of tracers | Total time (ns) | Per-call average time | |-----------------------+-----------------+----------------------------| | Relevant | Irrelevant | 100000 calls | Total (ns) | Overhead (ns) | |----------+------------+-----------------+------------+---------------| | 0 | 0 | 1459917 | 14 | - | | 0 | 1 | 1408000 | 14 | - | | 0 | 2 | 1383792 | 13 | - | | 0 | 10 | 1430709 | 14 | - | | 0 | 100 | 1383791 | 13 | - | | 0 | 200 | 1383750 | 13 | - | |----------+------------+-----------------+------------+---------------| | 1 | 0 | 5238041 | 52 | 38 | | 1 | 1 | 5228542 | 52 | 38 | | 1 | 2 | 5325917 | 53 | 40 | | 1 | 10 | 5299667 | 52 | 38 | | 1 | 100 | 5245250 | 52 | 39 | | 1 | 200 | 5238459 | 52 | 39 | |----------+------------+-----------------+------------+---------------| | 1 | 0 | 5239083 | 52 | 38 | | 2 | 0 | 19449417 | 194 | 181 | | 10 | 0 | 67718584 | 677 | 663 | | 100 | 0 | 709840708 | 7098 | 7085 | | 200 | 0 | 2203580626 | 22035 | 22022 | +----------+------------+-----------------+------------+---------------+Note: per-call overhead is estimated relative to the baseline case with 0 relevant tracers and 0 irrelevant tracers.
As can be seen from the above:
a) Whenever there is a single relevant tracer function associated with a tracee, the overhead of invoking the tracer is constant, and does not scale with the number of tracers which are *not* associated with that tracee.
b) The overhead for a single relevant tracer has dropped to ~1/3 of the overhead prior to this series (from 122ns to 38ns). This is largely due to permitting calls to dynamically-allocated
ftrace_opswithout going throughftrace_ops_list_func.Quoted 6 lines
Signed-off-by Puranjay Mohan <[email protected]>[update kconfig, asm, refactor]
Signed-off-by Andy Chiu <[email protected]>Tested-by Björn Töpel <[email protected]>I bisected a boot failure to this commit [
c217157bcd1df("riscv:
ImplementHAVE_DYNAMIC_FTRACE_WITH_CALL_OPS")] yesterday, that appears to be affecting all LLVM versions that I currently have installed. From some initial testing ofKconfigoptions, it looks like the issue isCFI_CLANGrelated because when I disableCFI_CLANGthings work once more. Since this option depends on !CFI_CLANG, but isdef_booly, I modifiedKconfigto force disable it at all times and tested !DYNAMIC_FTRACE_WITH_CALL_OPS && !CFG_CLANG, which did boot.Quoted 4 lines
I dunno anything about what's going on in this patch, but so little in it relates to having
DYNAMIC_FTRACE_WITH_CALL_OPS, that I was able to figure out that the problem is -fpatchable-function-entry=8,4
DYNAMIC_FTRACE_WITH_CALL_OPScan't work together withCFI_CLANG.Quoted 7 lines
arm64 has:
select
HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS\if (DYNAMIC_FTRACE_WITH_ARGS && !CFI && \ (CC_IS_CLANG || !CC_OPTIMIZE_FOR_SIZE))would need something similar for riscv if not already done.
Quoted 6 lines
I think you've misunderstood my email. We already have:
select HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS if (DYNAMIC_FTRACE_WITH_ARGS && !CFI)The problem is that the patch broke using
CFI_CLANG, due to the fpatchable-function-entry change.
Yeah, sorry I did not see the patch,
the original one I sent had:
+ifeq ($(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS), y) +ifeq ($(CONFIG_RISCV_ISA_C),y)
+ CC_FLAGS_FTRACE := -fpatchable-function-entry=8,4 +else
+ CC_FLAGS_FTRACE := -fpatchable-function-entry=4,2 +endif
+else
The basic Idea is that we can't put nops before the function entry when using CFI_CLANG, because they both interfere with each other.
the fix should be something like:
-- >8 --
arch/riscv/Makefile +8 -0
@@ -14,11 +14,19 @@ endif ifeq ($(CONFIG_DYNAMIC_FTRACE),y) LDFLAGS_vmlinux += --no-relax KBUILD_CPPFLAGS += -DCC_USING_PATCHABLE_FUNCTION_ENTRY +ifeq ($(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS),y) ifeq ($(CONFIG_RISCV_ISA_C),y) CC_FLAGS_FTRACE := -fpatchable-function-entry=8,4 else CC_FLAGS_FTRACE := -fpatchable-function-entry=4,2 endif +else +ifeq ($(CONFIG_RISCV_ISA_C),y) + CC_FLAGS_FTRACE := -fpatchable-function-entry=4 +else + CC_FLAGS_FTRACE := -fpatchable-function-entry=2 +endif +endif endif ifeq ($(CONFIG_CMODEL_MEDLOW),y)
-- 8< --
Thanks,
Puranjay
Re: [PATCH v4 10/12] riscv: Implement HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS
Quoted 71 lines · Puranjay Mohan
Quoted 50 lines · Conor Dooley
Quoted 39 lines · Puranjay Mohan
Quoted 31 lines · Conor Dooley
Hey,
Quoted 11 lines · Andy Chiu
From: Puranjay Mohan <[email protected]>
This patch enables support for
DYNAMIC_FTRACE_WITH_CALL_OPSon RISC-V. This allows each ftrace callsite to provide anftrace_opsto the common ftrace trampoline, allowing each callsite to invoke distinct tracer functions without the need to fall back to list processing or to allocate custom trampolines for each callsite. This significantly speeds up cases where multiple distinct trace functions are used and callsites are mostly traced by a single tracer.Quoted 6 lines
Signed-off-by Puranjay Mohan <[email protected]>[update kconfig, asm, refactor]
Signed-off-by Andy Chiu <[email protected]>Tested-by Björn Töpel <[email protected]>I bisected a boot failure to this commit [
c217157bcd1df("riscv:
ImplementHAVE_DYNAMIC_FTRACE_WITH_CALL_OPS")] yesterday, that appears to be affecting all LLVM versions that I currently have installed. From some initial testing ofKconfigoptions, it looks like the issue isCFI_CLANGrelated because when I disableCFI_CLANGthings work once more. Since this option depends on !CFI_CLANG, but isdef_booly, I modifiedKconfigto force disable it at all times and tested !DYNAMIC_FTRACE_WITH_CALL_OPS && !CFG_CLANG, which did boot.Quoted 4 lines
I dunno anything about what's going on in this patch, but so little in it relates to having
DYNAMIC_FTRACE_WITH_CALL_OPS, that I was able to figure out that the problem is -fpatchable-function-entry=8,4
DYNAMIC_FTRACE_WITH_CALL_OPScan't work together withCFI_CLANG.Quoted 7 lines
arm64 has:
select
HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS\if (DYNAMIC_FTRACE_WITH_ARGS && !CFI && \ (CC_IS_CLANG || !CC_OPTIMIZE_FOR_SIZE))would need something similar for riscv if not already done.
Quoted 6 lines
I think you've misunderstood my email. We already have:
select HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS if (DYNAMIC_FTRACE_WITH_ARGS && !CFI)The problem is that the patch broke using
CFI_CLANG, due to the fpatchable-function-entry change.Yeah, sorry I did not see the patch,
the original one I sent had:(quoted diff) +7
-0+ifeq ($(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS), y) +ifeq ($(CONFIG_RISCV_ISA_C),y) + CC_FLAGS_FTRACE := -fpatchable-function-entry=8,4 +else + CC_FLAGS_FTRACE := -fpatchable-function-entry=4,2 +endif +else
Quoted 4 lines
The basic Idea is that we can't put nops before the function entry when using
CFI_CLANG, because they both interfere with each other.the fix should be something like:
Ye, this is what Nathan and I both did locally, give or take. I just wasn't sure if this was actually correct to do or if it was just papering over an issue with our CFI support. Do you want to send this as a patch?
Quoted 33 lines
-- >8 --
arch/riscv/Makefile +8
-1@@ -14,11 +14,19 @@ endif ifeq ($(CONFIG_DYNAMIC_FTRACE),y) LDFLAGS_vmlinux += --no-relax KBUILD_CPPFLAGS += -DCC_USING_PATCHABLE_FUNCTION_ENTRY +ifeq ($(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS),y) ifeq ($(CONFIG_RISCV_ISA_C),y) CC_FLAGS_FTRACE := -fpatchable-function-entry=8,4 else CC_FLAGS_FTRACE := -fpatchable-function-entry=4,2 endif +else +ifeq ($(CONFIG_RISCV_ISA_C),y) + CC_FLAGS_FTRACE := -fpatchable-function-entry=4 +else + CC_FLAGS_FTRACE := -fpatchable-function-entry=2 +endif +endif endif ifeq ($(CONFIG_CMODEL_MEDLOW),y) -- 8< --Thanks,
Puranjay
Re: [PATCH v4 10/12] riscv: Implement HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS
Quoted 76 lines · Conor Dooley
Quoted 74 lines · Puranjay Mohan
Quoted 50 lines · Conor Dooley
Quoted 39 lines · Puranjay Mohan
Quoted 31 lines · Conor Dooley
Hey,
Quoted 11 lines · Andy Chiu
From: Puranjay Mohan <[email protected]>
This patch enables support for
DYNAMIC_FTRACE_WITH_CALL_OPSon RISC-V. This allows each ftrace callsite to provide anftrace_opsto the common ftrace trampoline, allowing each callsite to invoke distinct tracer functions without the need to fall back to list processing or to allocate custom trampolines for each callsite. This significantly speeds up cases where multiple distinct trace functions are used and callsites are mostly traced by a single tracer.Quoted 6 lines
Signed-off-by Puranjay Mohan <[email protected]>[update kconfig, asm, refactor]
Signed-off-by Andy Chiu <[email protected]>Tested-by Björn Töpel <[email protected]>I bisected a boot failure to this commit [
c217157bcd1df("riscv:
ImplementHAVE_DYNAMIC_FTRACE_WITH_CALL_OPS")] yesterday, that appears to be affecting all LLVM versions that I currently have installed. From some initial testing ofKconfigoptions, it looks like the issue isCFI_CLANGrelated because when I disableCFI_CLANGthings work once more. Since this option depends on !CFI_CLANG, but isdef_booly, I modifiedKconfigto force disable it at all times and tested !DYNAMIC_FTRACE_WITH_CALL_OPS && !CFG_CLANG, which did boot.Quoted 4 lines
I dunno anything about what's going on in this patch, but so little in it relates to having
DYNAMIC_FTRACE_WITH_CALL_OPS, that I was able to figure out that the problem is -fpatchable-function-entry=8,4
DYNAMIC_FTRACE_WITH_CALL_OPScan't work together withCFI_CLANG.Quoted 7 lines
arm64 has:
select
HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS\if (DYNAMIC_FTRACE_WITH_ARGS && !CFI && \ (CC_IS_CLANG || !CC_OPTIMIZE_FOR_SIZE))would need something similar for riscv if not already done.
Quoted 6 lines
I think you've misunderstood my email. We already have:
select HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS if (DYNAMIC_FTRACE_WITH_ARGS && !CFI)The problem is that the patch broke using
CFI_CLANG, due to the fpatchable-function-entry change.Yeah, sorry I did not see the patch,
the original one I sent had:(quoted diff) +7
-0+ifeq ($(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS), y) +ifeq ($(CONFIG_RISCV_ISA_C),y) + CC_FLAGS_FTRACE := -fpatchable-function-entry=8,4 +else + CC_FLAGS_FTRACE := -fpatchable-function-entry=4,2 +endif +elseThe basic Idea is that we can't put nops before the function entry when using
CFI_CLANG, because they both interfere with each other.
Quoted 6 lines
Quoted 1 line
the fix should be something like:
Ye, this is what Nathan and I both did locally, give or take. I just wasn't sure if this was actually correct to do or if it was just papering over an issue with our CFI support. Do you want to send this as a patch?
Yes, I will send a patch with fixes tag.
Thanks,
Puranjay
Re: [PATCH v4 10/12] riscv: Implement HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS
Quoted 6 lines · Puranjay Mohan
Quoted 4 lines
Ye, this is what Nathan and I both did locally, give or take. I just wasn't sure if this was actually correct to do or if it was just papering over an issue with our CFI support. Do you want to send this as a patch?
Yes, I will send a patch with fixes tag.
Great, thanks! Add a cc: stable too, while you're at it ;)
PATCH v4 11/12 riscv: ftrace: support direct call using call_ops
In reply to Andy Chiujump to FTRACE_ADDR if distance is out of reach
---
Changelog v4:
- New patch since v4
- Include Björn's fix for kprobe (adjusting ftrace address with MCOUNT_INSN_SIZE)
- Clean out an unused variable
--- arch/riscv/Kconfig | 2 +- arch/riscv/include/asm/ftrace.h | 6 ++++ arch/riscv/kernel/asm-offsets.c | 3 ++ arch/riscv/kernel/ftrace.c | 13 ++++----- arch/riscv/kernel/mcount-dyn.S | 51 +++++++++++++++++++++------------ 5 files changed, 48 insertions(+), 27 deletions(-)
arch/riscv/Kconfig +1 -1
@@ -152,7 +152,7 @@ config RISCV select HAVE_DMA_CONTIGUOUS if MMU select HAVE_DYNAMIC_FTRACE if !XIP_KERNEL && MMU && (CLANG_SUPPORTS_DYNAMIC_FTRACE || GCC_SUPPORTS_DYNAMIC_FTRACE) select FUNCTION_ALIGNMENT_4B if HAVE_DYNAMIC_FTRACE && RISCV_ISA_C - select HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS + select HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS if HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS select HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS if (DYNAMIC_FTRACE_WITH_ARGS && !CFI_CLANG) select HAVE_DYNAMIC_FTRACE_WITH_ARGS if HAVE_DYNAMIC_FTRACE select HAVE_FTRACE_GRAPH_FUNC
arch/riscv/include/asm/ftrace.h +6 -0
@@ -130,6 +130,9 @@ struct __arch_ftrace_regs {unsigned long sp;unsigned long s0;unsigned long t1; +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS +unsigned long direct_tramp; +#endif union { unsigned long args[8]; struct {
@@ -223,10 +226,13 @@ void ftrace_graph_func(unsigned long ip, unsigned long parent_ip, struct ftrace_ops *op, struct ftrace_regs *fregs);#define ftrace_graph_func ftrace_graph_func +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS static inline void arch_ftrace_set_direct_caller (struct ftrace_regs *fregs,unsigned long addr) {arch_ftrace_regs (fregs)->t1 = addr; } +#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ +#endif /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */ #endif /* __ASSEMBLY__ */
arch/riscv/kernel/asm-offsets.c +3 -0
@@ -495,6 +495,9 @@ void asm_offsets(void)OFFSET (STACKFRAME_RA , stackframe, ra);#ifdef CONFIG_FUNCTION_TRACER DEFINE (FTRACE_OPS_FUNC , offsetof(struct ftrace_ops , func)); +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS +DEFINE (FTRACE_OPS_DIRECT_CALL , offsetof(struct ftrace_ops , direct_call)); +#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ #endif #ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS
arch/riscv/kernel/ftrace.c +5 -8
@@ -16,7 +16,7 @@unsigned long ftrace_call_adjust (unsigned long addr) {if (IS_ENABLED (CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS )) -return addr +8 ; +return addr +8 +MCOUNT_AUIPC_SIZE ;return addr +MCOUNT_AUIPC_SIZE ; }
@@ -83,10 +83,9 @@ static const struct ftrace_ops *riscv64_rec_get_ops(struct dyn_ftrace *rec)return ops; } -static int ftrace_rec_set_ops (const struct dyn_ftrace *rec, -const struct ftrace_ops *ops) +static int ftrace_rec_set_ops (const struct dyn_ftrace *rec,const struct ftrace_ops *ops) { -unsigned long literal = rec->ip -8 ; +unsigned long literal =ALIGN_DOWN (rec->ip -12 ,8 );return patch_text_nosync ((void *)literal, &ops,sizeof (ops)); }
@@ -117,7 +116,7 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) orig_addr = (unsigned long )&ftrace_caller; distance = addr > orig_addr ? addr - orig_addr : orig_addr - addr;if (distance >JALR_RANGE ) -return -EINVAL ; + addr =FTRACE_ADDR ;return __ftrace_modify_call (pc, addr, false); }
@@ -199,15 +198,13 @@ int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, unsigned long addr) {unsigned long caller = rec->ip -MCOUNT_AUIPC_SIZE ; -unsigned int call[2 ];int ret; -make_call_t0 (caller, old_addr, call); ret =ftrace_rec_update_ops (rec);if (ret)return ret; -return __ftrace_modify_call (caller, addr, true); +return __ftrace_modify_call (caller,FTRACE_ADDR , true); }#endif
arch/riscv/kernel/mcount-dyn.S +33 -18
89 lines not on this page · raw message
Signature · 1 line
2.39.3 (Apple Git-145)
PATCH v4 12/12 riscv: Documentation: add a description about dynamic ftrace
In reply to Andy ChiuAdd a section in cmodx to describe how dynamic ftrace works on riscv, limitations, and assumptions.
--- Documentation/arch/riscv/cmodx.rst | 46 +++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 7 deletions(-)
Documentation/arch/riscv/cmodx.rst +39 -7
@@ -10,13 +10,45 @@ modified by the program itself. Instruction storage and the instruction cache program must enforce its own synchronization with the unprivileged fence.i instruction. -However, the default Linux ABI prohibits the use of fence.i in userspace -applications. At any point the scheduler may migrate a task onto a new hart. If -migration occurs after the userspace synchronized the icache and instruction -storage with fence.i, the icache on the new hart will no longer be clean. This -is due to the behavior of fence.i only affecting the hart that it is called on. -Thus, the hart that the task has been migrated to may not have synchronized -instruction storage and icache. +CMODX in the Kernel Space +--------------------- + +Dynamic ftrace +--------------------- + +Essentially, dynamic ftrace directs the control flow by inserting a function +call at each patchable function entry, and patches it dynamically at runtime to +enable or disable the redirection. In the case of RISC-V, 2 instructions, +AUIPC + JALR, are required to compose a function call. However, it is impossible +to patch 2 instructions and expect that a concurrent read-side executes them +without a race condition. This series makes atmoic code patching possible in +RISC-V ftrace. Kernel preemption makes things even worse as it allows the old +state to persist across the patching process with stop_machine(). + +In order to get rid of stop_machine() and run dynamic ftrace with full kernel +preemption, we partially initialize each patchable function entry at boot-time, +setting the first instruction to AUIPC, and the second to NOP. Now, atmoic +patching is possible because the kernel only has to update one instruction. +According to Ziccif, as long as an instruction is naturally aligned, the ISA +guarantee an atomic update. + +By fixing down the first instruction, AUIPC, the range of the ftrace trampoline +is limited to +-2K from the predetermined target, ftrace_caller, due to the lack +of immediate encoding space in RISC-V. To address the issue, we introduce +CALL_OPS, where an 8B naturally align metadata is added in front of each +pacthable function. The metadata is resolved at the first trampoline, then the +execution can be derect to another custom trampoline. + +CMODX in the User Space +--------------------- + +Though fence.i is an unprivileged instruction, the default Linux ABI prohibits +the use of fence.i in userspace applications. At any point the scheduler may +migrate a task onto a new hart. If migration occurs after the userspace +synchronized the icache and instruction storage with fence.i, the icache on the +new hart will no longer be clean. This is due to the behavior of fence.i only +affecting the hart that it is called on. Thus, the hart that the task has been +migrated to may not have synchronized instruction storage and icache. There are two ways to solve this problem: use the riscv_flush_icache() syscall, or use the ``PR_RISCV_SET_ICACHE_FLUSH_CTX`` prctl() and emit fence.i in
Signature · 1 line
2.39.3 (Apple Git-145)
Re: [PATCH v4 12/12] riscv: Documentation: add a description about dynamic ftrace
Hi, thanks,
s/atmoic/atomic
Quoted 1 line · Andy Chiu
+without a race condition. This series makes atmoic code patching possible in
...
Quoted 1 line
+setting the first instruction to AUIPC, and the second to NOP. Now, atmoic
/Robbin
Quoted 31 lines
Documentation/arch/riscv/cmodx.rst +5
-0+patching is possible because the kernel only has to update one instruction. +According to Ziccif, as long as an instruction is naturally aligned, the ISA +guarantee an atomic update. + +By fixing down the first instruction, AUIPC, the range of the ftracetrampoline
+is limited to +-2K from the predetermined target,ftrace_caller, due to the lackDocumentation/arch/riscv/cmodx.rst +3
-0+of immediate encoding space in RISC-V. To address the issue, we introduce +CALL_OPS, where an 8B naturally align metadata is added in front of each +pacthable function. The metadata is resolved at the first trampoline, thenthe
Documentation/arch/riscv/cmodx.rst +6
-0+execution can be derect to another custom trampoline. + +CMODX in the User Space +--------------------- + +Though fence.i is an unprivileged instruction, the default Linux ABIprohibits
Documentation/arch/riscv/cmodx.rst +3
-0+the use of fence.i in userspace applications. At any point the scheduler may +migrate a task onto a new hart. If migration occurs after the userspace +synchronized the icache and instruction storage with fence.i, the icache onthe
(quoted diff) +2
-0+new hart will no longer be clean. This is due to the behavior of fence.i only +affecting the hart that it is called on. Thus, the hart that the task hasbeen
+migrated to may not have synchronized instruction storage and icache.
There are two ways to solve this problem: use theriscv_flush_icache()syscall,
or use the ``PR_RISCV_SET_ICACHE_FLUSH_CTX``prctl()and emit fence.i in
(Trimming the Cc:-list a bit)
Hi Andy!
Quoted 1 line · Andy Chiu
From: Andy Chiu <[email protected]>
...
(Oh, if you send another revision, please add a cover, so it's easier to comment around the full series.)
Thanks a lot for doing the lion part of this series, Andy! With this in place, *finally* stop machine way of text patching is past us, and we can move RISC-V out from the 20th century. ;-)
I applied your series, and Steven's series [1] to [2], and ran that on QEMU (riscv64, ~RVA23), and on Milk-V Jupiter (Spacemit K1) with:
* CONFIG_FTRACE_STARTUP_TEST
* ftrace kselftest
No visible regressions, and now the ftrace kselftest can actually complete! For the series:
[1] https://lore.kernel.org/all/[email protected]/
[2] commit 3b07108ada81 ("Merge tag 'linux_kselftest-fixes-6.15-rc2' of git://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest")
Björn
Quoted 5 lines · Björn Töpel
(Trimming the Cc:-list a bit)
Hi Andy!
Quoted 7 lines
Quoted 1 line · Andy Chiu
From: Andy Chiu <[email protected]>
...
(Oh, if you send another revision, please add a cover, so it's easier to comment around the full series.)
I am so sorry about this! I wrote a cover letter but apparently I didn't send it. I am attaching the original cover letter below. Please let me know if there is any better way to address this trouble.
Quoted 13 lines
Thanks a lot for doing the lion part of this series, Andy! With this in place, *finally* stop machine way of text patching is past us, and we can move RISC-V out from the 20th century. ;-)
I applied your series, and Steven's series [1] to [2], and ran that on QEMU (riscv64, ~RVA23), and on Milk-V Jupiter (Spacemit K1) with:
*CONFIG_FTRACE_STARTUP_TEST
* ftrace kselftestNo visible regressions, and now the ftrace kselftest can actually complete! For the series:
Quoted 7 lines
Tested-by Björn Töpel <[email protected]>[1] https://lore.kernel.org/all/[email protected]/
[2] commit3b07108ada81("Merge tag 'linux_kselftest-fixes-6.15-rc2' ofgit://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest")Björn
This series makes atomic code patching in ftrace possible and eliminates the need of the stop_machine dance. The major difference of this version is that we merge the CALL_OPS support from Puranjay [1] and make direct calls available for practical uses such as BPF. Thanks for the time reviewing the series and suggestions, we hope this version gets a step closer to happening in the upstream.
Please reference the link to v3 below for more introductory view of the implementation
Added patch: 2, 4, 10, 11, 12
Modified patch: 5, 6
Unchanged patch: 1, 3, 7, 8, 9
(1, 8 has commit msg modified)
Special thanks to Björn for his efforts on testing and guiding the series!
[1]: https://lore.kernel.org/lkml/[email protected]/
Changes in v4:
- Rebase on top of v6.15-rc1
- Add a fix tag and R-b from Björn (1)
- Remove unused code enclosed by !WITH_ARG and unused macros (2, 5)
- Export ftrace_sync_ipi for its use in riscv (3, 4)
- Fix a bug with kprobe after probing at the start of symbol is allowed, by correcting ftrace_call_adjust (5, 11)
- Synchronize update of ftrace destination and the data passed to it
(5)
- Include Puranjay's patch for CALL_OPS (10)
- Support direct calls based on CALL_OPS (11)
- Add a documentation that breifly explain CMODX for dynamic ftrace (12)
- Link to v3: https://lore.kernel.org/r/[email protected]
Changes in v3:
- Add a fix tag for patch 1
- Add a data fence before sending out remote fence.i (6)
- Link to v2: https://lore.kernel.org/all/[email protected]/
Changes in v2:
- Drop patch 1 as it is merged through fixes.
- Drop patch 2, which converts kernel_text_address into notrace. As
users can prevent tracing it by configuring the tracefs.
- Use a more generic way in kconfig to align functions.
- Link to v1: https://lore.kernel.org/r/[email protected]
Andy Chiu (11):
riscv: ftrace: support fastcc in Clang for WITH_ARGS
riscv: ftrace factor out code defined by !WITH_ARG
riscv: ftrace: align patchable functions to 4 Byte boundary
kernel: ftrace: export ftrace_sync_ipi
riscv: ftrace: prepare ftrace for atomic code patching
riscv: ftrace: do not use stop_machine to update code
riscv: vector: Support calling schedule() for preemptible Vector
riscv: add a data fence for CMODX in the kernel mode
riscv: ftrace: support PREEMPT
riscv: ftrace: support direct call using call_ops
riscv: Documentation: add a description about dynamic ftrace
Puranjay Mohan (1):
riscv: Implement HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS
Documentation/arch/riscv/cmodx.rst | 46 +++++- arch/riscv/Kconfig | 8 +- arch/riscv/Makefile | 4 +- arch/riscv/include/asm/ftrace.h | 62 ++++---- arch/riscv/include/asm/processor.h | 5 + arch/riscv/include/asm/vector.h | 22 ++- arch/riscv/kernel/asm-offsets.c | 13 ++ arch/riscv/kernel/ftrace.c | 241 +++++++++++++++-------------- arch/riscv/kernel/mcount-dyn.S | 117 ++++++++------ arch/riscv/mm/cacheflush.c | 15 +- include/linux/ftrace.h | 1 + kernel/trace/ftrace.c | 2 +- 12 files changed, 327 insertions(+), 209 deletions(-)
--
2.39.3 (Apple Git-145)
This series was applied to riscv/linux.git (for-next) by Alexandre Ghiti <[email protected]>:
Hello:
This series was applied to riscv/linux.git (for-next) by Alexandre Ghiti <[email protected]>:
Quoted 8 lines · On Tue, 8 Apr 2025 02:08:25 +0800 you wrote:
From: Andy Chiu <[email protected]>
Some caller-saved registers which are not defined as function arguments in the ABI can still be passed as arguments when the kernel is compiled with Clang. As a result, we must save and restore those registers to prevent ftrace from clobbering them.
[...]
Here is the summary with links:
- [v4,01/12] riscv: ftrace: support fastcc in Clang for WITH_ARGS
https://git.kernel.org/riscv/c/7cecf4f30c33 - [v4,02/12] riscv: ftrace factor out code defined by !WITH_ARG https://git.kernel.org/riscv/c/2efa234f5e0c - [v4,03/12] riscv: ftrace: align patchable functions to 4 Byte boundary https://git.kernel.org/riscv/c/cced570c2c0c - [v4,04/12] kernel: ftrace: export ftrace_sync_ipi (no matching commit) - [v4,05/12] riscv: ftrace: prepare ftrace for atomic code patching (no matching commit) - [v4,06/12] riscv: ftrace: do not use stop_machine to update code (no matching commit) - [v4,07/12] riscv: vector: Support calling schedule() for preemptible Vector https://git.kernel.org/riscv/c/e2a8cbdbe932 - [v4,08/12] riscv: add a data fence for CMODX in the kernel mode https://git.kernel.org/riscv/c/29b59e3bbb6e - [v4,09/12] riscv: ftrace: support PREEMPT https://git.kernel.org/riscv/c/f48ba55bb8a8 - [v4,10/12] riscv: Implement HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS (no matching commit) - [v4,11/12] riscv: ftrace: support direct call using call_ops https://git.kernel.org/riscv/c/7ef9ae7457c0 - [v4,12/12] riscv: Documentation: add a description about dynamic ftrace https://git.kernel.org/riscv/c/0e07200b2af6
You are awesome, thank you!
Signature · 2 lines
Deet-doot-dot, I am a bot. https://korg.docs.kernel.org/patchwork/pwbot.html