ftrace: Have callbacks handle their own recursion
Subsystems (10)
Steven Rostedt
maintainerPATCH v2 5/11 kprobes/ftrace: Add recursion protection to the ftrace callback
In reply to Steven RostedtFrom: "Steven Rostedt (VMware)" <[email protected]>
If a ftrace callback does not supply its own recursion protection and does not set the RECURSION_SAFE flag in its ftrace_ops, then ftrace will make a helper trampoline to do so before calling the callback instead of just calling the callback directly.
The default for ftrace_ops is going to change. It will expect that handlers provide their own recursion protection, unless its ftrace_ops states otherwise.
--- arch/csky/kernel/probes/ftrace.c | 12 ++++++++++-- arch/parisc/kernel/ftrace.c | 13 +++++++++++-- arch/powerpc/kernel/kprobes-ftrace.c | 11 ++++++++++- arch/s390/kernel/ftrace.c | 13 +++++++++++-- arch/x86/kernel/kprobes/ftrace.c | 12 ++++++++++-- 5 files changed, 52 insertions(+), 9 deletions(-)
arch/csky/kernel/probes/ftrace.c +10 -2
@@ -13,16 +13,21 @@ int arch_check_ftrace_location(struct kprobe *p)void kprobe_ftrace_handler (unsigned long ip,unsigned long parent_ip,struct ftrace_ops *ops,struct pt_regs *regs) { +int bit;bool lr_saver = false;struct kprobe *p;struct kprobe_ctlblk *kcb; -/* Preempt is disabled by ftrace */ + bit =ftrace_test_recursion_trylock (); +if (bit <0 ) +return ; + +preempt_disable_notrace (); p =get_kprobe ((kprobe_opcode_t *)ip);if (!p) { p =get_kprobe ((kprobe_opcode_t *)(ip -MCOUNT_INSN_SIZE ));if (unlikely (!p) ||kprobe_disabled (p)) -return ; + goto out; lr_saver = true; }
@@ -56,6 +61,9 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,*/ __this_cpu_write (current_kprobe,NULL ); } +out: +preempt_enable_notrace (); +ftrace_test_recursion_unlock (bit); }NOKPROBE_SYMBOL (kprobe_ftrace_handler);
arch/parisc/kernel/ftrace.c +11 -2
@@ -208,13 +208,19 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, {struct kprobe_ctlblk *kcb;struct kprobe *p =get_kprobe ((kprobe_opcode_t *)ip); +int bit; -if (unlikely (!p) ||kprobe_disabled (p)) + bit =ftrace_test_recursion_trylock (); +if (bit <0 )return ; +preempt_disable_notrace (); +if (unlikely (!p) ||kprobe_disabled (p)) + goto out; +if (kprobe_running ()) {kprobes_inc_nmissed_count (p); -return ; + goto out; }__this_cpu_write (current_kprobe, p);
@@ -235,6 +241,9 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, } }__this_cpu_write (current_kprobe,NULL ); +out: +preempt_enable_notrace (); +ftrace_test_recursion_unlock (bit); }NOKPROBE_SYMBOL (kprobe_ftrace_handler);
arch/powerpc/kernel/kprobes-ftrace.c +10 -1
@@ -18,10 +18,16 @@ void kprobe_ftrace_handler(unsigned long nip, unsigned long parent_nip, {struct kprobe *p;struct kprobe_ctlblk *kcb; +int bit; + bit =ftrace_test_recursion_trylock (); +if (bit <0 ) +return ; + +preempt_disable_notrace (); p =get_kprobe ((kprobe_opcode_t *)nip);if (unlikely (!p) ||kprobe_disabled (p)) -return ; + goto out; kcb =get_kprobe_ctlblk ();if (kprobe_running ()) {
@@ -52,6 +58,9 @@ void kprobe_ftrace_handler(unsigned long nip, unsigned long parent_nip,*/ __this_cpu_write (current_kprobe,NULL ); } +out: +preempt_enable_notrace (); +ftrace_test_recursion_unlock (bit); }NOKPROBE_SYMBOL (kprobe_ftrace_handler);
arch/s390/kernel/ftrace.c +11 -2
@@ -202,13 +202,19 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, {struct kprobe_ctlblk *kcb;struct kprobe *p =get_kprobe ((kprobe_opcode_t *)ip); +int bit; -if (unlikely (!p) ||kprobe_disabled (p)) + bit =ftrace_test_recursion_trylock (); +if (bit <0 )return ; +preempt_disable_notrace (); +if (unlikely (!p) ||kprobe_disabled (p)) + goto out; +if (kprobe_running ()) {kprobes_inc_nmissed_count (p); -return ; + goto out; }__this_cpu_write (current_kprobe, p);
@@ -228,6 +234,9 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, } }__this_cpu_write (current_kprobe,NULL ); +out: +preempt_enable_notrace (); +ftrace_test_recursion_unlock (bit); }NOKPROBE_SYMBOL (kprobe_ftrace_handler);
arch/x86/kernel/kprobes/ftrace.c +10 -2
@@ -18,11 +18,16 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, {struct kprobe *p;struct kprobe_ctlblk *kcb; +int bit; -/* Preempt is disabled by ftrace */ + bit =ftrace_test_recursion_trylock (); +if (bit <0 ) +return ; + +preempt_disable_notrace (); p =get_kprobe ((kprobe_opcode_t *)ip);if (unlikely (!p) ||kprobe_disabled (p)) -return ; + goto out; kcb =get_kprobe_ctlblk ();if (kprobe_running ()) {
@@ -52,6 +57,9 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,*/ __this_cpu_write (current_kprobe,NULL ); } +out: +preempt_enable_notrace (); +ftrace_test_recursion_unlock (bit); }NOKPROBE_SYMBOL (kprobe_ftrace_handler);
Signature · 1 line
2.28.0
Steven Rostedt
maintainerPATCH v2 2/11 ftrace: Add ftrace_test_recursion_trylock() helper function
In reply to Steven RostedtFrom: "Steven Rostedt (VMware)" <[email protected]>
To make it easier for ftrace callbacks to have recursion protection, provide a ftrace_test_recursion_trylock() and ftrace_test_recursion_unlock() helper that tests for recursion.
--- include/linux/trace_recursion.h | 25 +++++++++++++++++++++++++ kernel/trace/trace_functions.c | 12 +++++------- 2 files changed, 30 insertions(+), 7 deletions(-)
include/linux/trace_recursion.h +25 -0
@@ -183,5 +183,30 @@ static __always_inline void trace_clear_recursion(int bit) current->trace_recursion = val; } +/** +* ftrace_test_recursion_trylock - tests for recursion in same context +* +* Use this for ftrace callbacks. This will detect if the function +* tracing recursed in the same context (normal vs interrupt), +* +* Returns: -1 if a recursion happened. +* >= 0 if no recursion +*/ +static __always_inline int ftrace_test_recursion_trylock(void) +{ +return trace_test_and_set_recursion (TRACE_FTRACE_START ,TRACE_FTRACE_MAX ); +} + +/** +* ftrace_test_recursion_unlock - called when function callback is complete +* @bit: The return of a successful ftrace_test_recursion_trylock() +* +* This is used at the end of a ftrace callback. +*/ +static __always_inline void ftrace_test_recursion_unlock(int bit) +{ +trace_clear_recursion (bit); +} +#endif /* CONFIG_TRACING */ #endif /* _LINUX_TRACE_RECURSION_H */
kernel/trace/trace_functions.c +5 -7
@@ -141,22 +141,20 @@ function_trace_call(unsigned long ip, unsigned long parent_ip,if (unlikely (!tr->function_enabled))return ; + bit =ftrace_test_recursion_trylock (); +if (bit <0 ) +return ; + pc =preempt_count ();preempt_disable_notrace (); - bit =trace_test_and_set_recursion (TRACE_FTRACE_START ,TRACE_FTRACE_MAX ); -if (bit <0 ) - goto out; - cpu =smp_processor_id (); data =per_cpu_ptr (tr->array_buffer.data, cpu);if (!atomic_read (&data->disabled)) {local_save_flags (flags);trace_function (tr, ip, parent_ip, flags, pc); } -trace_clear_recursion (bit); - - out: +ftrace_test_recursion_unlock (bit);preempt_enable_notrace (); }
Signature · 1 line
2.28.0
Steven Rostedt
maintainerPATCH v2 10/11 ftrace: Reverse what the RECURSION flag means in the ftrace_ops
In reply to Steven RostedtFrom: "Steven Rostedt (VMware)" <[email protected]>
Now that all callbacks are recursion safe, reverse the meaning of the RECURSION flag and rename it from RECURSION_SAFE to simply RECURSION. Now only callbacks that request to have recursion protecting it will have the added trampoline to do so.
Also remove the outdated comment about "PER_CPU" when determining to use the ftrace_ops_assist_func.
--- Documentation/trace/ftrace-uses.rst | 82 +++++++++++++++++++++-------- include/linux/ftrace.h | 12 ++--- kernel/trace/fgraph.c | 3 +- kernel/trace/ftrace.c | 20 ++++--- kernel/trace/trace_events.c | 1 - kernel/trace/trace_functions.c | 2 +- kernel/trace/trace_selftest.c | 7 +-- kernel/trace/trace_stack.c | 1 - 8 files changed, 79 insertions(+), 49 deletions(-)
Documentation/trace/ftrace-uses.rst +60 -22
@@ -30,8 +30,8 @@ The ftrace context This requires extra care to what can be done inside a callback. A callback can be called outside the protective scope of RCU. -The ftrace infrastructure has some protections against recursions and RCU -but one must still be very careful how they use the callbacks. +There are helper functions to help against recursion, and making sure +RCU is watching. These are explained below. The ftrace_ops structure
@@ -108,6 +108,50 @@ The prototype of the callback function is as follows (as of v4.14): at the start of the function where ftrace was tracing. Otherwise it either contains garbage, or NULL. +Protect your callback +===================== + +As functions can be called from anywhere, and it is possible that a function +called by a callback may also be traced, and call that same callback, +recursion protection must be used. There are two helper functions that +can help in this regard. If you start your code with: + + int bit; + + bit = ftrace_test_recursion_trylock(); + if (bit < 0) + return; + +and end it with: + + ftrace_test_recursion_unlock(bit); + +The code in between will be safe to use, even if it ends up calling a +function that the callback is tracing. Note, on success, +ftrace_test_recursion_trylock() will disable preemption, and the +ftrace_test_recursion_unlock() will enable it again (if it was previously +enabled). + +Alternatively, if the FTRACE_OPS_FL_RECURSION flag is set on the ftrace_ops +(as explained below), then a helper trampoline will be used to test +for recursion for the callback and no recursion test needs to be done. +But this is at the expense of a slightly more overhead from an extra +function call. + +If your callback accesses any data or critical section that requires RCU +protection, it is best to make sure that RCU is "watching", otherwise +that data or critical section will not be protected as expected. In this +case add: + + if (!rcu_is_watching()) + return; + +Alternatively, if the FTRACE_OPS_FL_RCU flag is set on the ftrace_ops +(as explained below), then a helper trampoline will be used to test +for rcu_is_watching for the callback and no other test needs to be done. +But this is at the expense of a slightly more overhead from an extra +function call. + The ftrace FLAGS ================
@@ -128,26 +172,20 @@ FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED will not fail with this flag set. But the callback must check if regs is NULL or not to determine if the architecture supports it. -FTRACE_OPS_FL_RECURSION_SAFE - By default, a wrapper is added around the callback to - make sure that recursion of the function does not occur. That is, - if a function that is called as a result of the callback's execution - is also traced, ftrace will prevent the callback from being called - again. But this wrapper adds some overhead, and if the callback is - safe from recursion, it can set this flag to disable the ftrace - protection. - - Note, if this flag is set, and recursion does occur, it could cause - the system to crash, and possibly reboot via a triple fault. - - It is OK if another callback traces a function that is called by a - callback that is marked recursion safe. Recursion safe callbacks - must never trace any function that are called by the callback - itself or any nested functions that those functions call. - - If this flag is set, it is possible that the callback will also - be called with preemption enabled (when CONFIG_PREEMPTION is set), - but this is not guaranteed. +FTRACE_OPS_FL_RECURSION + By default, it is expected that the callback can handle recursion. + But if the callback is not that worried about overehead, then + setting this bit will add the recursion protection around the + callback by calling a helper function that will do the recursion + protection and only call the callback if it did not recurse. + + Note, if this flag is not set, and recursion does occur, it could + cause the system to crash, and possibly reboot via a triple fault. + + Not, if this flag is set, then the callback will always be called + with preemption disabled. If it is not set, then it is possible + (but not guaranteed) that the callback will be called in + preemptable context. FTRACE_OPS_FL_IPMODIFY Requires FTRACE_OPS_FL_SAVE_REGS set. If the callback is to "hijack"
include/linux/ftrace.h +6 -6
@@ -98,7 +98,7 @@ ftrace_func_t ftrace_ops_get_func(struct ftrace_ops *ops);/* * FTRACE_OPS_FL_* bits denote the state of ftrace_ops struct and are * set in the flags member. -* CONTROL, SAVE_REGS, SAVE_REGS_IF_SUPPORTED, RECURSION_SAFE, STUB and +* CONTROL, SAVE_REGS, SAVE_REGS_IF_SUPPORTED, RECURSION, STUB and * IPMODIFY are a kind of attribute flags which can be set only before * registering the ftrace_ops, and can not be modified while registered. * Changing those attribute flags after registering ftrace_ops will
@@ -121,10 +121,10 @@ ftrace_func_t ftrace_ops_get_func(struct ftrace_ops *ops);* passing regs to the handler. * Note, if this flag is set, the SAVE_REGS flag will automatically * get set upon registering the ftrace_ops, if the arch supports it. -* RECURSION_SAFE - The ftrace_ops can set this to tell the ftrace infrastructure -* that the call back has its own recursion protection. If it does -* not set this, then the ftrace infrastructure will add recursion -* protection for the caller. +* RECURSION - The ftrace_ops can set this to tell the ftrace infrastructure +* that the call back needs recursion protection. If it does +* not set this, then the ftrace infrastructure will assume +* that the callback can handle recursion on its own. * STUB - The ftrace_ops is just a place holder. * INITIALIZED - The ftrace_ops has already been initialized (first use time * register_ftrace_function() is called, it will initialized the ops)
@@ -156,7 +156,7 @@ enum {FTRACE_OPS_FL_DYNAMIC =BIT (1 ),FTRACE_OPS_FL_SAVE_REGS =BIT (2 ),FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED =BIT (3 ), -FTRACE_OPS_FL_RECURSION_SAFE =BIT (4 ), +FTRACE_OPS_FL_RECURSION =BIT (4 ),FTRACE_OPS_FL_STUB =BIT (5 ),FTRACE_OPS_FL_INITIALIZED =BIT (6 ), FTRACE_OPS_FL_DELETED = BIT(7),
kernel/trace/fgraph.c +1 -2
@@ -334,8 +334,7 @@ unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx,static struct ftrace_ops graph_ops = { .func = ftrace_stub, - .flags =FTRACE_OPS_FL_RECURSION_SAFE | -FTRACE_OPS_FL_INITIALIZED | + .flags =FTRACE_OPS_FL_INITIALIZED |FTRACE_OPS_FL_PID |FTRACE_OPS_FL_STUB , #ifdef FTRACE_GRAPH_TRAMP_ADDR
kernel/trace/ftrace.c +9 -11
@@ -80,7 +80,7 @@ enum {struct ftrace_ops ftrace_list_end__read_mostly = { .func = ftrace_stub, - .flags =FTRACE_OPS_FL_RECURSION_SAFE |FTRACE_OPS_FL_STUB , + .flags =FTRACE_OPS_FL_STUB ,INIT_OPS_HASH (ftrace_list_end) };
@@ -866,7 +866,7 @@ static void unregister_ftrace_profiler(void)#else static struct ftrace_ops ftrace_profile_ops__read_mostly = { .func = function_profile_call, - .flags =FTRACE_OPS_FL_RECURSION_SAFE |FTRACE_OPS_FL_INITIALIZED , + .flags =FTRACE_OPS_FL_INITIALIZED ,INIT_OPS_HASH (ftrace_profile_ops) };
@@ -1040,8 +1040,7 @@ struct ftrace_ops global_ops = { .local_hash.notrace_hash = EMPTY_HASH, .local_hash.filter_hash = EMPTY_HASH,INIT_OPS_HASH (global_ops) - .flags =FTRACE_OPS_FL_RECURSION_SAFE | -FTRACE_OPS_FL_INITIALIZED | + .flags =FTRACE_OPS_FL_INITIALIZED | FTRACE_OPS_FL_PID, };
@@ -2382,7 +2381,7 @@ static void call_direct_funcs(unsigned long ip, unsigned long pip,struct ftrace_ops direct_ops = { .func = call_direct_funcs, - .flags =FTRACE_OPS_FL_IPMODIFY |FTRACE_OPS_FL_RECURSION_SAFE + .flags =FTRACE_OPS_FL_IPMODIFY |FTRACE_OPS_FL_DIRECT |FTRACE_OPS_FL_SAVE_REGS |FTRACE_OPS_FL_PERMANENT ,/*
@@ -6864,8 +6863,7 @@ void ftrace_init_trace_array(struct trace_array *tr)struct ftrace_ops global_ops = { .func = ftrace_stub, - .flags =FTRACE_OPS_FL_RECURSION_SAFE | -FTRACE_OPS_FL_INITIALIZED | + .flags =FTRACE_OPS_FL_INITIALIZED |FTRACE_OPS_FL_PID , };
@@ -7023,11 +7021,11 @@ NOKPROBE_SYMBOL(ftrace_ops_assist_func);ftrace_func_t ftrace_ops_get_func (struct ftrace_ops *ops) {/* -* If the function does not handle recursion, needs to be RCU safe, -* or does per cpu logic, then we need to call the assist handler. +* If the function does not handle recursion or needs to be RCU safe, +* then we need to call the assist handler. */ -if (!(ops->flags &FTRACE_OPS_FL_RECURSION_SAFE ) || - ops->flags &FTRACE_OPS_FL_RCU ) +if (ops->flags & (FTRACE_OPS_FL_RECURSION | +FTRACE_OPS_FL_RCU ))return ftrace_ops_assist_func;return ops->func;
kernel/trace/trace_events.c +0 -1
@@ -3712,7 +3712,6 @@ function_test_events_call(unsigned long ip, unsigned long parent_ip,static struct ftrace_ops trace_ops__initdata = { .func = function_test_events_call, - .flags =FTRACE_OPS_FL_RECURSION_SAFE , };static __init void event_trace_self_test_with_function (void )
kernel/trace/trace_functions.c +1 -1
@@ -48,7 +48,7 @@ int ftrace_allocate_ftrace_ops(struct trace_array *tr)/* Currently only the non stack version is supported */ ops->func = function_trace_call; - ops->flags =FTRACE_OPS_FL_RECURSION_SAFE |FTRACE_OPS_FL_PID ; + ops->flags =FTRACE_OPS_FL_PID ; tr->ops = ops; ops->private = tr;
kernel/trace/trace_selftest.c +2 -5
@@ -150,17 +150,14 @@ static void trace_selftest_test_dyn_func(unsigned long ip,static struct ftrace_ops test_probe1 = { .func = trace_selftest_test_probe1_func, - .flags =FTRACE_OPS_FL_RECURSION_SAFE , };static struct ftrace_ops test_probe2 = { .func = trace_selftest_test_probe2_func, - .flags =FTRACE_OPS_FL_RECURSION_SAFE , };static struct ftrace_ops test_probe3 = { .func = trace_selftest_test_probe3_func, - .flags =FTRACE_OPS_FL_RECURSION_SAFE , };static void print_counts (void )
@@ -448,11 +445,11 @@ static void trace_selftest_test_recursion_safe_func(unsigned long ip,static struct ftrace_ops test_rec_probe = { .func = trace_selftest_test_recursion_func, + .flags =FTRACE_OPS_FL_RECURSION , };static struct ftrace_ops test_recsafe_probe = { .func = trace_selftest_test_recursion_safe_func, - .flags =FTRACE_OPS_FL_RECURSION_SAFE , }; static int
@@ -561,7 +558,7 @@ static void trace_selftest_test_regs_func(unsigned long ip,static struct ftrace_ops test_regs_probe = { .func = trace_selftest_test_regs_func, - .flags =FTRACE_OPS_FL_RECURSION_SAFE |FTRACE_OPS_FL_SAVE_REGS , + .flags =FTRACE_OPS_FL_SAVE_REGS , }; static int
kernel/trace/trace_stack.c +0 -1
@@ -318,7 +318,6 @@ stack_trace_call(unsigned long ip, unsigned long parent_ip,static struct ftrace_ops trace_ops__read_mostly = { .func = stack_trace_call, - .flags =FTRACE_OPS_FL_RECURSION_SAFE , }; static ssize_t
Signature · 1 line
2.28.0
Steven Rostedt
maintainerPATCH v2 11/11 ftrace: Add recording of functions that caused recursion
In reply to Steven RostedtFrom: "Steven Rostedt (VMware)" <[email protected]>
This adds CONFIG_FTRACE_RECORD_RECURSION that will record to a file "recursed_functions" all the functions that caused recursion while a callback to the function tracer was running.
--- Documentation/trace/ftrace-uses.rst | 6 +- arch/csky/kernel/probes/ftrace.c | 2 +- arch/parisc/kernel/ftrace.c | 2 +- arch/powerpc/kernel/kprobes-ftrace.c | 2 +- arch/s390/kernel/ftrace.c | 2 +- arch/x86/kernel/kprobes/ftrace.c | 2 +- fs/pstore/ftrace.c | 2 +- include/linux/trace_recursion.h | 32 +++- kernel/livepatch/patch.c | 2 +- kernel/trace/Kconfig | 25 +++ kernel/trace/Makefile | 1 + kernel/trace/ftrace.c | 4 +- kernel/trace/trace_event_perf.c | 2 +- kernel/trace/trace_functions.c | 2 +- kernel/trace/trace_output.c | 6 +- kernel/trace/trace_output.h | 1 + kernel/trace/trace_recursion_record.c | 220 ++++++++++++++++++++++++++ 17 files changed, 293 insertions(+), 20 deletions(-) create mode 100644 kernel/trace/trace_recursion_record.c
Documentation/trace/ftrace-uses.rst +4 -2
@@ -118,7 +118,7 @@ can help in this regard. If you start your code with: int bit; - bit = ftrace_test_recursion_trylock(); + bit = ftrace_test_recursion_trylock(ip, parent_ip); if (bit < 0) return;
@@ -130,7 +130,9 @@ The code in between will be safe to use, even if it ends up calling a function that the callback is tracing. Note, on success, ftrace_test_recursion_trylock() will disable preemption, and the ftrace_test_recursion_unlock() will enable it again (if it was previously -enabled). +enabled). The instruction pointer (ip) and its parent (parent_ip) is passed to +ftrace_test_recursion_trylock() to record where the recursion happened +(if CONFIG_FTRACE_RECORD_RECURSION is set). Alternatively, if the FTRACE_OPS_FL_RECURSION flag is set on the ftrace_ops (as explained below), then a helper trampoline will be used to test
arch/csky/kernel/probes/ftrace.c +1 -1
@@ -18,7 +18,7 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,struct kprobe *p;struct kprobe_ctlblk *kcb; - bit =ftrace_test_recursion_trylock (); + bit =ftrace_test_recursion_trylock (ip, parent_ip);if (bit <0 )return ;
arch/parisc/kernel/ftrace.c +1 -1
@@ -210,7 +210,7 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,struct kprobe *p =get_kprobe ((kprobe_opcode_t *)ip);int bit; - bit =ftrace_test_recursion_trylock (); + bit =ftrace_test_recursion_trylock (ip, parent_ip);if (bit <0 )return ;
arch/powerpc/kernel/kprobes-ftrace.c +1 -1
@@ -20,7 +20,7 @@ void kprobe_ftrace_handler(unsigned long nip, unsigned long parent_nip,struct kprobe_ctlblk *kcb;int bit; - bit =ftrace_test_recursion_trylock (); + bit =ftrace_test_recursion_trylock (nip, parent_nip);if (bit <0 )return ;
arch/s390/kernel/ftrace.c +1 -1
@@ -204,7 +204,7 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,struct kprobe *p =get_kprobe ((kprobe_opcode_t *)ip);int bit; - bit =ftrace_test_recursion_trylock (); + bit =ftrace_test_recursion_trylock (ip, parent_ip);if (bit <0 )return ;
arch/x86/kernel/kprobes/ftrace.c +1 -1
@@ -20,7 +20,7 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,struct kprobe_ctlblk *kcb;int bit; - bit =ftrace_test_recursion_trylock (); + bit =ftrace_test_recursion_trylock (ip, parent_ip);if (bit <0 )return ;
fs/pstore/ftrace.c +1 -1
@@ -41,7 +41,7 @@ static void notrace pstore_ftrace_call(unsigned long ip,if (unlikely (oops_in_progress))return ; - bit =ftrace_test_recursion_trylock (); + bit =ftrace_test_recursion_trylock (ip, parent_ip);if (bit <0 )return ;
include/linux/trace_recursion.h +28 -4
@@ -142,7 +142,28 @@ static __always_inline int trace_get_context_bit(void) pc &HARDIRQ_MASK ?TRACE_CTX_IRQ :TRACE_CTX_SOFTIRQ ; } -static __always_inline int trace_test_and_set_recursion(int start, int max) +#ifdef CONFIG_FTRACE_RECORD_RECURSION +extern void ftrace_record_recursion (unsigned long ip,unsigned long parent_ip); +/* +* The paranoid_test check can cause dropped reports (unlikely), but +* if the recursion is common, it will likely still be recorded later. +* But the paranoid_test is needed to make sure we don't crash. +*/ +# define do_ftrace_record_recursion (ip, pip) \ + do { \ + static atomic_t paranoid_test; \ + if (!atomic_read(¶noid_test)) { \ + atomic_inc(¶noid_test); \ + ftrace_record_recursion(ip, pip); \ + atomic_dec(¶noid_test); \ + } \ + } while (0) +#else +# define do_ftrace_record_recursion (ip, pip) do { } while (0) +#endif + +static __always_inline int trace_test_and_set_recursion(unsigned long ip, unsigned long pip, +int start,int max) {unsigned int val = current->trace_recursion;int bit;
@@ -158,8 +179,10 @@ static __always_inline int trace_test_and_set_recursion(int start, int max)* a switch between contexts. Allow for a single recursion. */ bit =TRACE_TRANSITION_BIT ; -if (trace_recursion_test (bit)) +if (trace_recursion_test (bit)) { +do_ftrace_record_recursion (ip, pip);return -1 ; + }trace_recursion_set (bit);barrier ();return bit +1 ;
@@ -199,9 +222,10 @@ static __always_inline void trace_clear_recursion(int bit)* Returns: -1 if a recursion happened. * >= 0 if no recursion */ -static __always_inline int ftrace_test_recursion_trylock(void) +static __always_inline int ftrace_test_recursion_trylock(unsigned long ip, +unsigned long parent_ip) { -return trace_test_and_set_recursion (TRACE_FTRACE_START ,TRACE_FTRACE_MAX ); +return trace_test_and_set_recursion (ip, parent_ip,TRACE_FTRACE_START ,TRACE_FTRACE_MAX ); }/**
kernel/livepatch/patch.c +1 -1
@@ -49,7 +49,7 @@ static void notrace klp_ftrace_handler(unsigned long ip, ops = container_of(fops, struct klp_ops, fops); - bit =ftrace_test_recursion_trylock (); + bit =ftrace_test_recursion_trylock (ip, parent_ip);if (WARN_ON_ONCE (bit <0 ))return ;/*
kernel/trace/Kconfig +25 -0
@@ -727,6 +727,31 @@ config TRACE_EVAL_MAP_FILE If unsure, say N. +config FTRACE_RECORD_RECURSION + bool "Record functions that recurse in function tracing" + depends on FUNCTION_TRACER + help + All callbacks that attach to the function tracing have some sort + of protection against recursion. Even though the protection exists, + it adds overhead. This option will create a file in the tracefs + file system called "recursed_functions" that will list the functions + that triggered a recursion. + + This will add more overhead to cases that have recursion. + + If unsure, say N + +config FTRACE_RECORD_RECURSION_SIZE + int "Max number of recursed functions to record" + default 128 + depends on FTRACE_RECORD_RECURSION + help + This defines the limit of number of functions that can be + listed in the "recursed_functions" file, that lists all + the functions that caused a recursion to happen. + This file can be reset, but the limit can not change in + size at runtime. + config GCOV_PROFILE_FTRACE bool "Enable GCOV profiling on ftrace subsystem" depends on GCOV_KERNEL
kernel/trace/Makefile +1 -0
@@ -92,6 +92,7 @@ obj-$(CONFIG_DYNAMIC_EVENTS) += trace_dynevent.o obj-$(CONFIG_PROBE_EVENTS) += trace_probe.o obj-$(CONFIG_UPROBE_EVENTS) += trace_uprobe.o obj-$(CONFIG_BOOTTIME_TRACING) += trace_boot.o +obj-$(CONFIG_FTRACE_RECORD_RECURSION) += trace_recursion_record.o obj-$(CONFIG_TRACEPOINT_BENCHMARK) += trace_benchmark.o
kernel/trace/ftrace.c +2 -2
@@ -6918,7 +6918,7 @@ __ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,struct ftrace_ops *op;int bit; - bit =trace_test_and_set_recursion (TRACE_LIST_START ,TRACE_LIST_MAX ); + bit =trace_test_and_set_recursion (ip, parent_ip,TRACE_LIST_START ,TRACE_LIST_MAX );if (bit <0 )return ;
@@ -6993,7 +6993,7 @@ static void ftrace_ops_assist_func(unsigned long ip, unsigned long parent_ip, {int bit; - bit =trace_test_and_set_recursion (TRACE_LIST_START ,TRACE_LIST_MAX ); + bit =trace_test_and_set_recursion (ip, parent_ip,TRACE_LIST_START ,TRACE_LIST_MAX );if (bit <0 )return ;
kernel/trace/trace_event_perf.c +1 -1
@@ -447,7 +447,7 @@ perf_ftrace_function_call(unsigned long ip, unsigned long parent_ip,if ((unsigned long )ops->private !=smp_processor_id ())return ; - bit =ftrace_test_recursion_trylock (); + bit =ftrace_test_recursion_trylock (ip, parent_ip);if (bit <0 )return ;
kernel/trace/trace_functions.c +1 -1
@@ -141,7 +141,7 @@ function_trace_call(unsigned long ip, unsigned long parent_ip,if (unlikely (!tr->function_enabled))return ; - bit =ftrace_test_recursion_trylock (); + bit =ftrace_test_recursion_trylock (ip, parent_ip);if (bit <0 )return ;
kernel/trace/trace_output.c +3 -3
@@ -353,8 +353,8 @@ static inline const char *kretprobed(const char *name) }#endif /* CONFIG_KRETPROBES */ -static void -seq_print_sym (struct trace_seq *s,unsigned long address,bool offset) +void +trace_seq_print_sym (struct trace_seq *s,unsigned long address,bool offset) {#ifdef CONFIG_KALLSYMS char str[KSYM_SYMBOL_LEN ];
@@ -420,7 +420,7 @@ seq_print_ip_sym(struct trace_seq *s, unsigned long ip, unsigned long sym_flags) goto out; } -seq_print_sym (s, ip, sym_flags &TRACE_ITER_SYM_OFFSET ); +trace_seq_print_sym (s, ip, sym_flags &TRACE_ITER_SYM_OFFSET );if (sym_flags &TRACE_ITER_SYM_ADDR )trace_seq_printf (s," <" IP_FMT ">" , ip);
kernel/trace/trace_output.h +1 -0
@@ -16,6 +16,7 @@ extern int seq_print_ip_sym(struct trace_seq *s, unsigned long ip, unsigned long sym_flags); +extern void trace_seq_print_sym (struct trace_seq *s,unsigned long address,bool offset);extern int trace_print_context (struct trace_iterator *iter);extern int trace_print_lat_context (struct trace_iterator *iter);
kernel/trace/trace_recursion_record.c new file +220 -0
220 lines not on this page · raw message
Signature · 1 line
2.28.0
Steven Rostedt
maintainerPATCH v2 7/11 livepatch: Trigger WARNING if livepatch function fails due to recursion
In reply to Steven RostedtFrom: "Steven Rostedt (VMware)" <[email protected]>
If for some reason a function is called that triggers the recursion detection of live patching, trigger a warning. By not executing the live patch code, it is possible that the old unpatched function will be called placing the system into an unknown state.
--- kernel/livepatch/patch.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
kernel/livepatch/patch.c +1 -1
@@ -50,7 +50,7 @@ static void notrace klp_ftrace_handler(unsigned long ip, ops = container_of(fops, struct klp_ops, fops); bit =ftrace_test_recursion_trylock (); -if (bit <0 ) +if (WARN_ON_ONCE (bit <0 ))return ;/* * A variant of synchronize_rcu() is used to allow patching functions
Signature · 1 line
2.28.0
Steven Rostedt
maintainerPATCH v2 9/11 perf/ftrace: Check for rcu_is_watching() in callback function
In reply to Steven RostedtFrom: "Steven Rostedt (VMware)" <[email protected]>
If a ftrace callback requires "rcu_is_watching", then it adds the FTRACE_OPS_FL_RCU flag and it will not be called if RCU is not "watching". But this means that it will use a trampoline when called, and this slows down the function tracing a tad. By checking rcu_is_watching() from within the callback, it no longer needs the RCU flag set in the ftrace_ops and it can be safely called directly.
--- kernel/trace/trace_event_perf.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
kernel/trace/trace_event_perf.c +3 -1
@@ -441,6 +441,9 @@ perf_ftrace_function_call(unsigned long ip, unsigned long parent_ip,int rctx;int bit; +if (!rcu_is_watching ()) +return ; +if ((unsigned long )ops->private !=smp_processor_id ())return ;
@@ -484,7 +487,6 @@ static int perf_ftrace_function_register(struct perf_event *event) {struct ftrace_ops *ops = &event->ftrace_ops; - ops->flags =FTRACE_OPS_FL_RCU ; ops->func = perf_ftrace_function_call; ops->private = (void *)(unsigned long )nr_cpu_ids;
Signature · 1 line
2.28.0
Steven Rostedt
maintainerv2 cover ftrace: Have callbacks handle their own recursion
I found that having the ftrace infrastructure use its own trampoline to handle recursion and RCU by defaulte unless the ftrace_ops set the appropriate flags, was an issue that nobody set those flags. But then their callbacks would suffer from an unnecessary overhead instead of simply handling the recursion itself.
This series makes it mandatory that ftrace callbacks handle recursion or set a flag asking ftrace to do it for it. It also creates helper functions to help these callbacks to have recursion protection.
Changes since v1:
- Reworded the paragraph that talks about the ftrace_ops default recursion in the change logs so that it makes more sense (recommend by Petr Mladek)
- Rebased on fixes that will be sent upstream as soon as my testing is complete, that fixes the recursion logic. NMIs no longer trigger false recursion positives, and also logic to handle transitions between context that also causes false positives. These fixes are not part of this series, but will be added before these patches.
- Added code to create a "recursed_functions" file in tracefs, that lists the functions that triggered recursion. This was how I detected the bugs from before. If you enable CONFIG_FTRACE_RECORD_RECURSION it will add this logic. If you also enable FTRACE_STARTUP_TEST, one of the tests triggers a recursion, and this will be shown in the file after boot up (to test if it actually works).
# cat recursed_functions ftrace_ops_assist_func+0x84/0x160: trace_selftest_dynamic_test_func+0x0/0x10
Steven Rostedt (VMware) (11):
ftrace: Move the recursion testing into global headers
ftrace: Add ftrace_test_recursion_trylock() helper function
ftrace: Optimize testing what context current is in pstore/ftrace: Add recursion protection to the ftrace callback kprobes/ftrace: Add recursion protection to the ftrace callback livepatch/ftrace: Add recursion protection to the ftrace callback
livepatch: Trigger WARNING if livepatch function fails due to recursion perf/ftrace: Add recursion protection to the ftrace callback perf/ftrace: Check for rcu_is_watching() in callback function
ftrace: Reverse what the RECURSION flag means in the ftrace_ops
ftrace: Add recording of functions that caused recursion
----
Documentation/trace/ftrace-uses.rst | 84 +++++++++--- arch/csky/kernel/probes/ftrace.c | 12 +- arch/parisc/kernel/ftrace.c | 13 +- arch/powerpc/kernel/kprobes-ftrace.c | 11 +- arch/s390/kernel/ftrace.c | 13 +- arch/x86/kernel/kprobes/ftrace.c | 12 +- fs/pstore/ftrace.c | 6 + include/linux/ftrace.h | 13 +- include/linux/trace_recursion.h | 243 ++++++++++++++++++++++++++++++++++ kernel/livepatch/patch.c | 5 + kernel/trace/Kconfig | 25 ++++ kernel/trace/Makefile | 1 + kernel/trace/fgraph.c | 3 +- kernel/trace/ftrace.c | 24 ++-- kernel/trace/trace.h | 177 ------------------------- kernel/trace/trace_event_perf.c | 13 +- kernel/trace/trace_events.c | 1 - kernel/trace/trace_functions.c | 14 +- kernel/trace/trace_output.c | 6 +- kernel/trace/trace_output.h | 1 + kernel/trace/trace_recursion_record.c | 220 ++++++++++++++++++++++++++++++ kernel/trace/trace_selftest.c | 7 +- kernel/trace/trace_stack.c | 1 - 23 files changed, 656 insertions(+), 249 deletions(-) create mode 100644 include/linux/trace_recursion.h create mode 100644 kernel/trace/trace_recursion_record.c
Steven Rostedt
maintainerPATCH v2 6/11 livepatch/ftrace: Add recursion protection to the ftrace callback
From: "Steven Rostedt (VMware)" <[email protected]>
If a ftrace callback does not supply its own recursion protection and does not set the RECURSION_SAFE flag in its ftrace_ops, then ftrace will make a helper trampoline to do so before calling the callback instead of just calling the callback directly.
The default for ftrace_ops is going to change. It will expect that handlers provide their own recursion protection, unless its ftrace_ops states otherwise.
--- kernel/livepatch/patch.c | 5 +++++ 1 file changed, 5 insertions(+)
kernel/livepatch/patch.c +5 -0
@@ -45,9 +45,13 @@ static void notrace klp_ftrace_handler(unsigned long ip,struct klp_ops *ops;struct klp_func *func;int patch_state; +int bit; ops = container_of(fops, struct klp_ops, fops); + bit =ftrace_test_recursion_trylock (); +if (bit <0 ) +return ;/* * A variant of synchronize_rcu() is used to allow patching functions * where RCU is not watching, see klp_synchronize_transition().
@@ -117,6 +121,7 @@ static void notrace klp_ftrace_handler(unsigned long ip, unlock:preempt_enable_notrace (); +ftrace_test_recursion_unlock (bit); }/*
Signature · 1 line
2.28.0
Steven Rostedt
maintainerPATCH v2 8/11 perf/ftrace: Add recursion protection to the ftrace callback
In reply to Steven RostedtFrom: "Steven Rostedt (VMware)" <[email protected]>
If a ftrace callback does not supply its own recursion protection and does not set the RECURSION_SAFE flag in its ftrace_ops, then ftrace will make a helper trampoline to do so before calling the callback instead of just calling the callback directly.
The default for ftrace_ops is going to change. It will expect that handlers provide their own recursion protection, unless its ftrace_ops states otherwise.
--- kernel/trace/trace_event_perf.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-)
kernel/trace/trace_event_perf.c +8 -1
@@ -439,10 +439,15 @@ perf_ftrace_function_call(unsigned long ip, unsigned long parent_ip,struct hlist_head head;struct pt_regs regs;int rctx; +int bit;if ((unsigned long )ops->private !=smp_processor_id ())return ; + bit =ftrace_test_recursion_trylock (); +if (bit <0 ) +return ; + event = container_of(ops, struct perf_event, ftrace_ops);/*
@@ -463,13 +468,15 @@ perf_ftrace_function_call(unsigned long ip, unsigned long parent_ip, entry =perf_trace_buf_alloc (ENTRY_SIZE ,NULL , &rctx);if (!entry) -return ; + goto out; entry->ip = ip; entry->parent_ip = parent_ip;perf_trace_buf_submit (entry,ENTRY_SIZE , rctx,TRACE_FN ,1 , ®s, &head,NULL ); +out: +ftrace_test_recursion_unlock (bit);#undef ENTRY_SIZE }
Signature · 1 line
2.28.0
Steven Rostedt
maintainerPATCH v2 1/11 ftrace: Move the recursion testing into global headers
In reply to Steven RostedtFrom: "Steven Rostedt (VMware)" <[email protected]>
Currently, if a callback is registered to a ftrace function and its ftrace_ops does not have the RECURSION flag set, it is encapsulated in a helper function that does the recursion for it.
Really, all the callbacks should have their own recursion protection for performance reasons. But they should not all implement their own. Move the recursion helpers to global headers, so that all callbacks can use them.
--- include/linux/ftrace.h | 1 + include/linux/trace_recursion.h | 187 ++++++++++++++++++++++++++++++++ kernel/trace/trace.h | 177 ------------------------------ 3 files changed, 188 insertions(+), 177 deletions(-) create mode 100644 include/linux/trace_recursion.h
include/linux/ftrace.h +1 -0
@@ -7,6 +7,7 @@#ifndef _LINUX_FTRACE_H#define _LINUX_FTRACE_H +#include <linux/trace_recursion.h> #include <linux/trace_clock.h> #include <linux/kallsyms.h> #include <linux/linkage.h>
include/linux/trace_recursion.h new file +187 -0
187 lines not on this page · raw message
kernel/trace/trace.h +0 -177
183 lines not on this page · raw message
Signature · 1 line
2.28.0
Steven Rostedt
maintainerPATCH v2 3/11 ftrace: Optimize testing what context current is in
In reply to Steven RostedtFrom: "Steven Rostedt (VMware)" <[email protected]>
The preempt_count() is not a simple location in memory, it could be part of per_cpu code or more. Each access to preempt_count(), or one of its accessor functions (like in_interrupt()) takes several cycles. By reading preempt_count() once, and then doing tests to find the context against the value return is slightly faster than using in_nmi() and in_interrupt().
--- include/linux/trace_recursion.h | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-)
include/linux/trace_recursion.h +20 -13
@@ -117,22 +117,29 @@ enum {#define TRACE_CONTEXT_MASK TRACE_LIST_MAX +/* +* Used for setting context +* NMI = 0 +* IRQ = 1 +* SOFTIRQ = 2 +* NORMAL = 3 +*/ +enum { +TRACE_CTX_NMI , +TRACE_CTX_IRQ , +TRACE_CTX_SOFTIRQ , +TRACE_CTX_NORMAL , +}; + static __always_inline int trace_get_context_bit(void) { -int bit; - -if (in_interrupt ()) { -if (in_nmi ()) - bit =0 ; - -else if (in_irq ()) - bit =1 ; -else - bit =2 ; - }else - bit =3 ; +unsigned long pc =preempt_count (); -return bit; +if (!(pc & (NMI_MASK |HARDIRQ_MASK |SOFTIRQ_OFFSET ))) +return TRACE_CTX_NORMAL ; +else +return pc &NMI_MASK ?TRACE_CTX_NMI : + pc &HARDIRQ_MASK ?TRACE_CTX_IRQ :TRACE_CTX_SOFTIRQ ; } static __always_inline int trace_test_and_set_recursion(int start, int max)
Signature · 1 line
2.28.0
Steven Rostedt
maintainerPATCH v2 4/11 pstore/ftrace: Add recursion protection to the ftrace callback
In reply to Steven RostedtFrom: "Steven Rostedt (VMware)" <[email protected]>
If a ftrace callback does not supply its own recursion protection and does not set the RECURSION_SAFE flag in its ftrace_ops, then ftrace will make a helper trampoline to do so before calling the callback instead of just calling the callback directly.
The default for ftrace_ops is going to change. It will expect that handlers provide their own recursion protection, unless its ftrace_ops states otherwise.
--- fs/pstore/ftrace.c | 6 ++++++ 1 file changed, 6 insertions(+)
fs/pstore/ftrace.c +6 -0
@@ -28,6 +28,7 @@ static void notrace pstore_ftrace_call(unsigned long ip,struct ftrace_ops *op, struct pt_regs *regs) { + int bit;unsigned long flags;struct pstore_ftrace_record rec = {}; struct pstore_record record = {
@@ -40,6 +41,10 @@ static void notrace pstore_ftrace_call(unsigned long ip,if (unlikely (oops_in_progress))return ; + bit =ftrace_test_recursion_trylock (); +if (bit <0 ) +return ; +local_irq_save (flags); rec.ip = ip;
@@ -49,6 +54,7 @@ static void notrace pstore_ftrace_call(unsigned long ip, psinfo->write (&record);local_irq_restore (flags); +ftrace_test_recursion_unlock (bit); }static struct ftrace_ops pstore_ftrace_ops__read_mostly = {
Signature · 1 line
2.28.0
Re: [PATCH 07/11 v2] livepatch: Trigger WARNING if livepatch function fails due to recursion
In reply to Steven RostedtQuoted 7 lines · Steven Rostedt
From: "Steven Rostedt (VMware)" <[email protected]>
If for some reason a function is called that triggers the recursion detection of live patching, trigger a warning. By not executing the live patch code, it is possible that the old unpatched function will be called placing the system into an unknown state.
Quoted 8 lines
Cc Josh Poimboeuf <[email protected]>Cc Jiri Kosina <[email protected]>Cc Miroslav Benes <[email protected]>Cc Joe Lawrence <[email protected]>Suggested-by Petr Mladek <[email protected]>
It has actually been first suggested by Miroslav. He might want to take the fame and eventual shame ;-)
Quoted 1 line
Signed-off-by Steven Rostedt (VMware) <[email protected]>
Best Regards,
Petr
Petr Mladek
maintainerRe: [PATCH 11/11 v2] ftrace: Add recording of functions that caused recursion
In reply to Steven RostedtQuoted 16 lines · Steven Rostedt
From: "Steven Rostedt (VMware)" <[email protected]>
This adds
CONFIG_FTRACE_RECORD_RECURSIONthat will record to a file "recursed_functions" all the functions that caused recursion while a callback to the function tracer was running.kernel/trace/trace_recursion_record.c new file +6
-0@@ -0,0 +1,220 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/seq_file.h> +#include <linux/kallsyms.h> +#include <linux/module.h> +#include <linux/ftrace.h>
kernel/trace/trace_recursion_record.c line 16
Quoted 10 lines
kernel/trace/trace_recursion_record.c new file +10
-0+#include <linux/fs.h> + +#include "trace_output.h" + +struct recursed_functions { +unsigned long ip; +unsigned long parent_ip; +}; + +static struct recursed_functions recursed_functions[CONFIG_FTRACE_RECORD_RECURSION_SIZE ];
The code tries to be lockless safe as much as possible. It would make sense to allign the array.
Quoted 18 lines
kernel/trace/trace_recursion_record.c new file +18
-0+static atomic_t nr_records; + +/* +* Cache the last found function. Yes, updates to this is racey, but +* so is memory cache ;-) +*/ +static unsigned long cached_function; + +void ftrace_record_recursion (unsigned long ip,unsigned long parent_ip) +{ +int index; +int i =0 ; +unsigned long old; + + again: +/* First check the last one recorded */ +if (ip == cached_function) +return ;
kernel/trace/trace_recursion_record.c line 44
Quoted 10 lines
kernel/trace/trace_recursion_record.c new file +10
-0+ + index =atomic_read (&nr_records); +/* nr_records is -1 when clearing records */ +smp_mb__after_atomic (); +if (index <0 ) +return ; + +/* See below */ +if (i > index) + index = i;
This looks like a complicated way to do index++ via "i" variable. I guess that it was needed only in some older variant of the code. See below.
Quoted 15 lines
kernel/trace/trace_recursion_record.c new file +15
-0+if (index >=CONFIG_FTRACE_RECORD_RECURSION_SIZE ) +return ; + +for (i = index -1 ; i >=0 ; i--) { +if (recursed_functions[i].ip == ip) { + cached_function = ip; +return ; + } + } + + cached_function = ip; + +/* +* We only want to add a function if it hasn't been added before. +* Add to the current location before incrementing the count.
kernel/trace/trace_recursion_record.c line 69
Quoted 10 lines
kernel/trace/trace_recursion_record.c new file +10
-0+* If it fails to add, then increment the index (save in i) +* and try again. +*/ + old =cmpxchg (&recursed_functions[index].ip,0 , ip); +if (old !=0 ) { +/* Did something else already added this for us? */ +if (old == ip) +return ; +/* Try the next location (use i for the next index) */ + i = index +1 ;
What about
index++;
We basically want to run the code again with index + 1 limit.
Maybe, it even does not make sense to check the array again and we should just try to store the value into the next slot.
kernel/trace/trace_recursion_record.c line 73
Quoted 4 lines
kernel/trace/trace_recursion_record.c new file +4
-0+ goto again; + } + + recursed_functions[index].parent_ip = parent_ip;
WRITE_ONCE() ?
Quoted 16 lines
kernel/trace/trace_recursion_record.c new file +16
-0+ +/* +* It's still possible that we could race with the clearing +* CPU0 CPU1 +* ---- ---- +* ip = func +* nr_records = -1; +* recursed_functions[0] = 0; +* i = -1 +* if (i < 0) +* nr_records = 0; +* (new recursion detected) +* recursed_functions[0] = func +* cmpxchg(recursed_functions[0], +* func, 0) +*
kernel/trace/trace_recursion_record.c line 99
Quoted 10 lines
kernel/trace/trace_recursion_record.c new file +10
-0+* But the worse that could happen is that we get a zero in +* the recursed_functions array, and it's likely that "func" will +* be recorded again. +*/ + i =atomic_read (&nr_records); +smp_mb__after_atomic (); +if (i <0 ) +cmpxchg (&recursed_functions[index].ip, ip,0 ); +else if (i <= index) +atomic_cmpxchg (&nr_records, i, index +1 );
This looks weird. It would shift nr_records past the record added in this call. It might skip many slots that were zeroed when clearing. Also we do not know if our entry was not zeroed as well.
I would suggest to do it some other way (not even compile tested):
void ftrace_record_recursion(unsigned long ip, unsigned long parent_ip)
{
int index, old_index;
int i = 0;
unsigned long old_ip;again:
/* First check the last one recorded. */
if (ip == READ_ONCE(cached_function))
return;
index = atomic_read(&nr_records);
/* nr_records is -1 when clearing records. */
smp_mb__after_atomic();
if (index < 0)
return;
/* Already cached? */
for (i = index - 1; i >= 0; i--) {
if (recursed_functions[i].ip == ip) {
WRITE_ONCE(cached_function, ip);
return;
}
}
if (index >= CONFIG_FTRACE_RECORD_RECURSION_SIZE)
return;
/*
* Try to reserve the slot. It might be already taken
* or the entire cache cleared.
*/
old_index = atomic_cmpxchg(&nr_records, index, index + 1);
if (old_index != index)
goto again;
/*
* Be careful. The entire cache might have been cleared and reused in
* the meantime. Replace only empty slot.
*/
old_ip = cmpxchg(&recursed_functions[index].ip, 0, ip);
if (old_ip != 0)
goto again;
old_ip = cmpxchg(&recursed_functions[index].parent_ip, 0, parrent_ip);
if (old_ip != 0)
goto again;
/*
* No ip is better than non-consistent one. The race with
* clearing should be rare and not worth a perfect solution.
*/
if (READ_ONCE(recursed_functions[index].ip) != ip) {
cmpxchg(&recursed_functions[index].ip, ip, 0UL)
goto again;
}
}The last check probably is not needed. Inconsistent entries should be prevented by the way how this func is called:
static atomic_t paranoid_test; \
if (!atomic_read(¶noid_test)) { \
atomic_inc(¶noid_test); \
ftrace_record_recursion(ip, pip); \
atomic_dec(¶noid_test); \
} \The rest of the patchset looks fine. I do not feel comfortable to give it Reviewed-by because I did not review it in depth.
I spent more time with the above lockless code. I took it is a training. I need to improve this skill to feel more comfortable with the lockless printk ring buffer ;-)
Best Regards,
Petr
Steven Rostedt
maintainerRe: [PATCH 07/11 v2] livepatch: Trigger WARNING if livepatch function fails due to recursion
In reply to Petr MladekQuoted 17 lines · On Mon, 2 Nov 2020 15:41:09 +0100 Petr Mladek <[email protected]> wrote:
Quoted 7 lines · Steven Rostedt
From: "Steven Rostedt (VMware)" <[email protected]>
If for some reason a function is called that triggers the recursion detection of live patching, trigger a warning. By not executing the live patch code, it is possible that the old unpatched function will be called placing the system into an unknown state.
Quoted 8 lines
Cc Josh Poimboeuf <[email protected]>Cc Jiri Kosina <[email protected]>Cc Miroslav Benes <[email protected]>Cc Joe Lawrence <[email protected]>Suggested-by Petr Mladek <[email protected]>
Quoted 2 lines
It has actually been first suggested by Miroslav. He might want to take the fame and eventual shame ;-)
I'll switch suggested by to Miroslav.
Quoted 4 lines
Quoted 2 lines
Signed-off-by Steven Rostedt (VMware) <[email protected]>Reviewed-by Petr Mladek <[email protected]>
Thanks!
-- Steve
Steven Rostedt
maintainerRe: [PATCH 11/11 v2] ftrace: Add recording of functions that caused recursion
In reply to Petr MladekQuoted 22 lines · On Mon, 2 Nov 2020 17:41:47 +0100 Petr Mladek <[email protected]> wrote:
Quoted 21 lines · Steven Rostedt
From: "Steven Rostedt (VMware)" <[email protected]>
This adds
CONFIG_FTRACE_RECORD_RECURSIONthat will record to a file "recursed_functions" all the functions that caused recursion while a callback to the function tracer was running.kernel/trace/trace_recursion_record.c new file +11
-0@@ -0,0 +1,220 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/seq_file.h> +#include <linux/kallsyms.h> +#include <linux/module.h> +#include <linux/ftrace.h> +#include <linux/fs.h> + +#include "trace_output.h" + +struct recursed_functions {
Quoted 8 lines
kernel/trace/trace_recursion_record.c line 16
Quoted 5 lines
kernel/trace/trace_recursion_record.c new file +5
-0+unsigned long ip; +unsigned long parent_ip; +}; + +static struct recursed_functions recursed_functions[CONFIG_FTRACE_RECORD_RECURSION_SIZE ];The code tries to be lockless safe as much as possible. It would make sense to allign the array.
Hmm, is there an arch where the compiler would put an array of structures with two unsigned long, misaligned?
Quoted 26 lines
Quoted 26 lines
kernel/trace/trace_recursion_record.c new file +24
-0+static atomic_t nr_records; + +/* +* Cache the last found function. Yes, updates to this is racey, but +* so is memory cache ;-) +*/ +static unsigned long cached_function; + +void ftrace_record_recursion (unsigned long ip,unsigned long parent_ip) +{ +int index; +int i =0 ; +unsigned long old; + + again: +/* First check the last one recorded */ +if (ip == cached_function) +return ; + + index =atomic_read (&nr_records); +/* nr_records is -1 when clearing records */ +smp_mb__after_atomic (); +if (index <0 ) +return ;
Quoted 8 lines
kernel/trace/trace_recursion_record.c line 44
Quoted 4 lines
kernel/trace/trace_recursion_record.c new file +4
-0+ +/* See below */ +if (i > index) + index = i;This looks like a complicated way to do index++ via "i" variable. I guess that it was needed only in some older variant of the code. See below.
Because we reread the index above, and index could be bigger than i (more than index + 1).
Quoted 29 lines
Quoted 16 lines
kernel/trace/trace_recursion_record.c new file +15
-0+if (index >=CONFIG_FTRACE_RECORD_RECURSION_SIZE ) +return ; + +for (i = index -1 ; i >=0 ; i--) { +if (recursed_functions[i].ip == ip) { + cached_function = ip; +return ; + } + } + + cached_function = ip; + +/* +* We only want to add a function if it hasn't been added before. +* Add to the current location before incrementing the count. kernel/trace/trace_recursion_record.c line 69
Quoted 10 lines
kernel/trace/trace_recursion_record.c new file +10
-0+* If it fails to add, then increment the index (save in i) +* and try again. +*/ + old =cmpxchg (&recursed_functions[index].ip,0 , ip); +if (old !=0 ) { +/* Did something else already added this for us? */ +if (old == ip) +return ; +/* Try the next location (use i for the next index) */ + i = index +1 ;What about
Quoted 3 lines
index++;We basically want to run the code again with index + 1 limit.
But something else could update nr_records, and we want to use that if nr_records is greater than i.
Now, we could swap the use case, and have
int index = 0; [..] i = atomic_read(&nr_records); if (i > index) index = i; [..] index++; goto again;
Quoted 3 lines
Maybe, it even does not make sense to check the array again and we should just try to store the value into the next slot.
We do this dance to prevent duplicates.
But you are correct, that this went through a few iterations. And the first ones didn't have the cmpxchg on the ip itself, and that could make it so that we don't need this index = i dance.
Quoted 7 lines
kernel/trace/trace_recursion_record.c line 73
Quoted 5 lines
kernel/trace/trace_recursion_record.c new file +4
-0+ goto again; + } + + recursed_functions[index].parent_ip = parent_ip;
WRITE_ONCE()?
Does it really matter?
Quoted 83 lines
Quoted 17 lines
kernel/trace/trace_recursion_record.c new file +16
-0+ +/* +* It's still possible that we could race with the clearing +* CPU0 CPU1 +* ---- ---- +* ip = func +* nr_records = -1; +* recursed_functions[0] = 0; +* i = -1 +* if (i < 0) +* nr_records = 0; +* (new recursion detected) +* recursed_functions[0] = func +* cmpxchg(recursed_functions[0], +* func, 0) +* kernel/trace/trace_recursion_record.c line 99
Quoted 10 lines
kernel/trace/trace_recursion_record.c new file +10
-0+* But the worse that could happen is that we get a zero in +* the recursed_functions array, and it's likely that "func" will +* be recorded again. +*/ + i =atomic_read (&nr_records); +smp_mb__after_atomic (); +if (i <0 ) +cmpxchg (&recursed_functions[index].ip, ip,0 ); +else if (i <= index) +atomic_cmpxchg (&nr_records, i, index +1 );This looks weird. It would shift
nr_recordspast the record added in this call. It might skip many slots that were zeroed when clearing. Also we do not know if our entry was not zeroed as well.I would suggest to do it some other way (not even compile tested):
void
ftrace_record_recursion(unsigned long ip, unsigned long parent_ip){ int index, old_index; int i = 0; unsigned long old_ip;again:
/* First check the last one recorded. */ if (ip == READ_ONCE(cached_function)) return; index = atomic_read(&nr_records); /* nr_records is -1 when clearing records. */ smp_mb__after_atomic(); if (index < 0) return; /* Already cached? */ for (i = index - 1; i >= 0; i--) { if (recursed_functions[i].ip == ip) { WRITE_ONCE(cached_function, ip); return; } } if (index >= CONFIG_FTRACE_RECORD_RECURSION_SIZE) return; /* * Try to reserve the slot. It might be already taken * or the entire cache cleared. */ old_index = atomic_cmpxchg(&nr_records, index, index + 1); if (old_index != index) goto again; /* * Be careful. The entire cache might have been cleared and reused in * the meantime. Replace only empty slot. */ old_ip = cmpxchg(&recursed_functions[index].ip, 0, ip); if (old_ip != 0) goto again; old_ip = cmpxchg(&recursed_functions[index].parent_ip, 0, parrent_ip); if (old_ip != 0) goto again; /*
Quoted 8 lines
* No ip is better than non-consistent one. The race with
* clearing should be rare and not worth a perfect solution.
*/
if (READ_ONCE(recursed_functions[index].ip) != ip) {
cmpxchg(&recursed_functions[index].ip, ip, 0UL)
goto again;
}
}Let me go and rewrite it, this time considering the cmpxchg in the ip update code. I may end up with what you have above ;-)
Quoted 14 lines
The last check probably is not needed. Inconsistent entries should be prevented by the way how this func is called:
static atomic_t paranoid_test; \ if (!atomic_read(¶noid_test)) { \ atomic_inc(¶noid_test); \ ftrace_record_recursion(ip, pip); \ atomic_dec(¶noid_test); \ } \
Quoted 6 lines
The rest of the patchset looks fine. I do not feel comfortable to give it Reviewed-by because I did not review it in depth.
I spent more time with the above lockless code. I took it is a training. I need to improve this skill to feel more comfortable with the lockless printk ring buffer ;-)
Yeah, everything becomes exponentially complex when you make it lockless with multiple concurrent writers.
-- Steve
Steven Rostedt
maintainerRe: [PATCH 11/11 v2] ftrace: Add recording of functions that caused recursion
Quoted 80 lines · On Mon, 2 Nov 2020 12:09:07 -0500 Steven Rostedt <[email protected]> wrote:
Quoted 16 lines
Quoted 16 lines
kernel/trace/trace_recursion_record.c new file +16
-0+void ftrace_record_recursion (unsigned long ip,unsigned long parent_ip) +{ +int index; +int i =0 ; +unsigned long old; + + again: +/* First check the last one recorded */ +if (ip == cached_function) +return ; + + index =atomic_read (&nr_records); +/* nr_records is -1 when clearing records */ +smp_mb__after_atomic (); +if (index <0 ) +return ;Quoted 8 lines
kernel/trace/trace_recursion_record.c line 44
Quoted 4 lines
kernel/trace/trace_recursion_record.c new file +4
-0+ +/* See below */ +if (i > index) + index = i;This looks like a complicated way to do index++ via "i" variable. I guess that it was needed only in some older variant of the code. See below.
Because we reread the index above, and index could be bigger than i (more than index + 1).
Quoted 29 lines
Quoted 16 lines
kernel/trace/trace_recursion_record.c new file +15
-0+if (index >=CONFIG_FTRACE_RECORD_RECURSION_SIZE ) +return ; + +for (i = index -1 ; i >=0 ; i--) { +if (recursed_functions[i].ip == ip) { + cached_function = ip; +return ; + } + } + + cached_function = ip; + +/* +* We only want to add a function if it hasn't been added before. +* Add to the current location before incrementing the count. kernel/trace/trace_recursion_record.c line 69
Quoted 10 lines
kernel/trace/trace_recursion_record.c new file +10
-0+* If it fails to add, then increment the index (save in i) +* and try again. +*/ + old =cmpxchg (&recursed_functions[index].ip,0 , ip); +if (old !=0 ) { +/* Did something else already added this for us? */ +if (old == ip) +return ; +/* Try the next location (use i for the next index) */ + i = index +1 ;What about
Quoted 3 lines
index++;We basically want to run the code again with index + 1 limit.
But something else could update
nr_records, and we want to use that ifnr_recordsis greater than i.Now, we could swap the use case, and have
int index = 0; [..] i = atomic_read(&nr_records); if (i > index) index = i; [..] index++; goto again;Quoted 1 line
Quoted 8 lines
Quoted 2 lines
Maybe, it even does not make sense to check the array again and we should just try to store the value into the next slot.
We do this dance to prevent duplicates.
But you are correct, that this went through a few iterations. And the first ones didn't have the cmpxchg on the ip itself, and that could make it so that we don't need this index = i dance.
Playing with this more, I remember why I did this song and dance.
If we have two or more writers, and one beats the other in updating the ip (with a different function). This one will go and try again. The reason to look at one passed nr_records, is because of the race between the multiple writers. This one may loop before the other can update nr_records, and it will fail to apply it again.
You could just say, "hey we'll just keep looping until the other writer eventually updates nr_records". But this is where my paranoia gets in. What happens if that other writer takes an interrupt (interrupts are not disabled), and then deadlocks, or does something bad? This CPU will not get locked up spinning.
Unlikely scenario, and it would require a bug someplace else. But I don't want a bug report stating that it found this recursion locking locking up the CPU and hide the real culprit.
I'll add a comment to explain this in the code. And also swap the i and index around to make a little more sense.
-- Steve
Steven Rostedt
maintainerRe: [PATCH 11/11 v2] ftrace: Add recording of functions that caused recursion
In reply to Petr MladekQuoted 10 lines · On Mon, 2 Nov 2020 17:41:47 +0100 Petr Mladek <[email protected]> wrote:
kernel/trace/trace_recursion_record.c line 99
Quoted 6 lines
kernel/trace/trace_recursion_record.c new file +6
-0+ i =atomic_read (&nr_records); +smp_mb__after_atomic (); +if (i <0 ) +cmpxchg (&recursed_functions[index].ip, ip,0 ); +else if (i <= index) +atomic_cmpxchg (&nr_records, i, index +1 );This looks weird. It would shift
nr_recordspast the record added in this call. It might skip many slots that were zeroed when clearing. Also we do not know if our entry was not zeroed as well.
nr_records always holds the next position to write to.
index = nr_records; recursed_functions[index].ip = ip; nr_records++;
Before clearing, we have:
nr_records = -1; smp_mb(); memset(recursed_functions, 0); smp_wmb(); nr_records = 0;
When we enter this function:
i = nr_records; smp_mb(); if (i < 0) return;
Thus, we just stopped all new updates while clearing the records.
But what about if something is currently updating?
i = nr_records; smp_mb(); if (i < 0) cmpxchg(recursed_functions, ip, 0);
The above shows that if the current updating process notices that the clearing happens, it will clear the function it added.
else if (i <= index) cmpxchg(nr_records, i, index + 1);
This makes sure that nr_records only grows if it is greater or equal to zero.
The only race that I see that can happen, is the one in the comment I showed. And that is after enabling the recursed functions again after clearing, one CPU could add a function while another CPU that just added that same function could be just exiting this routine, notice that a clearing of the array happened, and remove its function (which was the same as the one just happened). So we get a "zero" in the array. If this happens, it is likely that that function will recurse again and will be added later.
-- Steve
Steven Rostedt
maintainerRe: [PATCH 11/11 v2] ftrace: Add recording of functions that caused recursion
Quoted 9 lines · On Mon, 2 Nov 2020 12:37:21 -0500 Steven Rostedt <[email protected]> wrote:
The only race that I see that can happen, is the one in the comment I showed. And that is after enabling the recursed functions again after clearing, one CPU could add a function while another CPU that just added that same function could be just exiting this routine, notice that a clearing of the array happened, and remove its function (which was the same as the one just happened). So we get a "zero" in the array. If this happens, it is likely that that function will recurse again and will be added later.
Updated version of this function:
-- Steve
void ftrace_record_recursion(unsigned long ip, unsigned long parent_ip)
{
int index = 0;
int i;
unsigned long old;again:
/* First check the last one recorded */
if (ip == cached_function)
return;
i = atomic_read(&nr_records);
/* nr_records is -1 when clearing records */
smp_mb__after_atomic();
if (i < 0)
return;
/*
* If there's two writers and this writer comes in second,
* the cmpxchg() below to update the ip will fail. Then this
* writer will try again. It is possible that index will now
* be greater than nr_records. This is because the writer
* that succeeded has not updated the nr_records yet.
* This writer could keep trying again until the other writer
* updates nr_records. But if the other writer takes an
* interrupt, and that interrupt locks up that CPU, we do
* not want this CPU to lock up due to the recursion protection,
* and have a bug report showing this CPU as the cause of
* locking up the computer. To not lose this record, this
* writer will simply use the next position to update the
* recursed_functions, and it will update the nr_records
* accordingly.
*/
if (index < i)
index = i;
if (index >= CONFIG_FTRACE_RECORD_RECURSION_SIZE)
return;
for (i = index - 1; i >= 0; i--) {
if (recursed_functions[i].ip == ip) {
cached_function = ip;
return;
}
}
cached_function = ip;
/*
* We only want to add a function if it hasn't been added before.
* Add to the current location before incrementing the count.
* If it fails to add, then increment the index (save in i)
* and try again.
*/
old = cmpxchg(&recursed_functions[index].ip, 0, ip);
if (old != 0) {
/* Did something else already added this for us? */
if (old == ip)
return;
/* Try the next location (use i for the next index) */
index++;
goto again;
}
recursed_functions[index].parent_ip = parent_ip;
/*
* It's still possible that we could race with the clearing
* CPU0 CPU1
* ---- ----
* ip = func
* nr_records = -1;
* recursed_functions[0] = 0;
* i = -1
* if (i < 0)
* nr_records = 0;
* (new recursion detected)
* recursed_functions[0] = func
* cmpxchg(recursed_functions[0],
* func, 0)
*
* But the worse that could happen is that we get a zero in
* the recursed_functions array, and it's likely that "func" will
* be recorded again.
*/
i = atomic_read(&nr_records);
smp_mb__after_atomic();
if (i < 0)
cmpxchg(&recursed_functions[index].ip, ip, 0);
else if (i <= index)
atomic_cmpxchg(&nr_records, i, index + 1);
}Steven Rostedt
maintainerPATCH v2 11/11 ftrace: Add recording of functions that caused recursion
In reply to Steven RostedtFrom: "Steven Rostedt (VMware)" <[email protected]>
This adds CONFIG_FTRACE_RECORD_RECURSION that will record to a file "recursed_functions" all the functions that caused recursion while a callback to the function tracer was running.
--- Documentation/trace/ftrace-uses.rst | 6 +- arch/csky/kernel/probes/ftrace.c | 2 +- arch/parisc/kernel/ftrace.c | 2 +- arch/powerpc/kernel/kprobes-ftrace.c | 2 +- arch/s390/kernel/ftrace.c | 2 +- arch/x86/kernel/kprobes/ftrace.c | 2 +- fs/pstore/ftrace.c | 2 +- include/linux/trace_recursion.h | 32 +++- kernel/livepatch/patch.c | 2 +- kernel/trace/Kconfig | 25 +++ kernel/trace/Makefile | 1 + kernel/trace/ftrace.c | 4 +- kernel/trace/trace_event_perf.c | 2 +- kernel/trace/trace_functions.c | 2 +- kernel/trace/trace_output.c | 6 +- kernel/trace/trace_output.h | 1 + kernel/trace/trace_recursion_record.c | 235 ++++++++++++++++++++++++++ 17 files changed, 308 insertions(+), 20 deletions(-) create mode 100644 kernel/trace/trace_recursion_record.c
Documentation/trace/ftrace-uses.rst +4 -2
@@ -118,7 +118,7 @@ can help in this regard. If you start your code with: int bit; - bit = ftrace_test_recursion_trylock(); + bit = ftrace_test_recursion_trylock(ip, parent_ip); if (bit < 0) return;
@@ -130,7 +130,9 @@ The code in between will be safe to use, even if it ends up calling a function that the callback is tracing. Note, on success, ftrace_test_recursion_trylock() will disable preemption, and the ftrace_test_recursion_unlock() will enable it again (if it was previously -enabled). +enabled). The instruction pointer (ip) and its parent (parent_ip) is passed to +ftrace_test_recursion_trylock() to record where the recursion happened +(if CONFIG_FTRACE_RECORD_RECURSION is set). Alternatively, if the FTRACE_OPS_FL_RECURSION flag is set on the ftrace_ops (as explained below), then a helper trampoline will be used to test
arch/csky/kernel/probes/ftrace.c +1 -1
@@ -18,7 +18,7 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,struct kprobe *p;struct kprobe_ctlblk *kcb; - bit =ftrace_test_recursion_trylock (); + bit =ftrace_test_recursion_trylock (ip, parent_ip);if (bit <0 )return ;
arch/parisc/kernel/ftrace.c +1 -1
@@ -210,7 +210,7 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,struct kprobe *p =get_kprobe ((kprobe_opcode_t *)ip);int bit; - bit =ftrace_test_recursion_trylock (); + bit =ftrace_test_recursion_trylock (ip, parent_ip);if (bit <0 )return ;
arch/powerpc/kernel/kprobes-ftrace.c +1 -1
@@ -20,7 +20,7 @@ void kprobe_ftrace_handler(unsigned long nip, unsigned long parent_nip,struct kprobe_ctlblk *kcb;int bit; - bit =ftrace_test_recursion_trylock (); + bit =ftrace_test_recursion_trylock (nip, parent_nip);if (bit <0 )return ;
arch/s390/kernel/ftrace.c +1 -1
@@ -204,7 +204,7 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,struct kprobe *p =get_kprobe ((kprobe_opcode_t *)ip);int bit; - bit =ftrace_test_recursion_trylock (); + bit =ftrace_test_recursion_trylock (ip, parent_ip);if (bit <0 )return ;
arch/x86/kernel/kprobes/ftrace.c +1 -1
@@ -20,7 +20,7 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,struct kprobe_ctlblk *kcb;int bit; - bit =ftrace_test_recursion_trylock (); + bit =ftrace_test_recursion_trylock (ip, parent_ip);if (bit <0 )return ;
fs/pstore/ftrace.c +1 -1
@@ -41,7 +41,7 @@ static void notrace pstore_ftrace_call(unsigned long ip,if (unlikely (oops_in_progress))return ; - bit =ftrace_test_recursion_trylock (); + bit =ftrace_test_recursion_trylock (ip, parent_ip);if (bit <0 )return ;
include/linux/trace_recursion.h +28 -4
@@ -142,7 +142,28 @@ static __always_inline int trace_get_context_bit(void) pc &HARDIRQ_MASK ?TRACE_CTX_IRQ :TRACE_CTX_SOFTIRQ ; } -static __always_inline int trace_test_and_set_recursion(int start, int max) +#ifdef CONFIG_FTRACE_RECORD_RECURSION +extern void ftrace_record_recursion (unsigned long ip,unsigned long parent_ip); +/* +* The paranoid_test check can cause dropped reports (unlikely), but +* if the recursion is common, it will likely still be recorded later. +* But the paranoid_test is needed to make sure we don't crash. +*/ +# define do_ftrace_record_recursion (ip, pip) \ + do { \ + static atomic_t paranoid_test; \ + if (!atomic_read(¶noid_test)) { \ + atomic_inc(¶noid_test); \ + ftrace_record_recursion(ip, pip); \ + atomic_dec(¶noid_test); \ + } \ + } while (0) +#else +# define do_ftrace_record_recursion (ip, pip) do { } while (0) +#endif + +static __always_inline int trace_test_and_set_recursion(unsigned long ip, unsigned long pip, +int start,int max) {unsigned int val = current->trace_recursion;int bit;
@@ -158,8 +179,10 @@ static __always_inline int trace_test_and_set_recursion(int start, int max)* a switch between contexts. Allow for a single recursion. */ bit =TRACE_TRANSITION_BIT ; -if (trace_recursion_test (bit)) +if (trace_recursion_test (bit)) { +do_ftrace_record_recursion (ip, pip);return -1 ; + }trace_recursion_set (bit);barrier ();return bit +1 ;
@@ -199,9 +222,10 @@ static __always_inline void trace_clear_recursion(int bit)* Returns: -1 if a recursion happened. * >= 0 if no recursion */ -static __always_inline int ftrace_test_recursion_trylock(void) +static __always_inline int ftrace_test_recursion_trylock(unsigned long ip, +unsigned long parent_ip) { -return trace_test_and_set_recursion (TRACE_FTRACE_START ,TRACE_FTRACE_MAX ); +return trace_test_and_set_recursion (ip, parent_ip,TRACE_FTRACE_START ,TRACE_FTRACE_MAX ); }/**
kernel/livepatch/patch.c +1 -1
@@ -49,7 +49,7 @@ static void notrace klp_ftrace_handler(unsigned long ip, ops = container_of(fops, struct klp_ops, fops); - bit =ftrace_test_recursion_trylock (); + bit =ftrace_test_recursion_trylock (ip, parent_ip);if (WARN_ON_ONCE (bit <0 ))return ;/*
kernel/trace/Kconfig +25 -0
@@ -727,6 +727,31 @@ config TRACE_EVAL_MAP_FILE If unsure, say N. +config FTRACE_RECORD_RECURSION + bool "Record functions that recurse in function tracing" + depends on FUNCTION_TRACER + help + All callbacks that attach to the function tracing have some sort + of protection against recursion. Even though the protection exists, + it adds overhead. This option will create a file in the tracefs + file system called "recursed_functions" that will list the functions + that triggered a recursion. + + This will add more overhead to cases that have recursion. + + If unsure, say N + +config FTRACE_RECORD_RECURSION_SIZE + int "Max number of recursed functions to record" + default 128 + depends on FTRACE_RECORD_RECURSION + help + This defines the limit of number of functions that can be + listed in the "recursed_functions" file, that lists all + the functions that caused a recursion to happen. + This file can be reset, but the limit can not change in + size at runtime. + config GCOV_PROFILE_FTRACE bool "Enable GCOV profiling on ftrace subsystem" depends on GCOV_KERNEL
kernel/trace/Makefile +1 -0
@@ -92,6 +92,7 @@ obj-$(CONFIG_DYNAMIC_EVENTS) += trace_dynevent.o obj-$(CONFIG_PROBE_EVENTS) += trace_probe.o obj-$(CONFIG_UPROBE_EVENTS) += trace_uprobe.o obj-$(CONFIG_BOOTTIME_TRACING) += trace_boot.o +obj-$(CONFIG_FTRACE_RECORD_RECURSION) += trace_recursion_record.o obj-$(CONFIG_TRACEPOINT_BENCHMARK) += trace_benchmark.o
kernel/trace/ftrace.c +2 -2
@@ -6918,7 +6918,7 @@ __ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,struct ftrace_ops *op;int bit; - bit =trace_test_and_set_recursion (TRACE_LIST_START ,TRACE_LIST_MAX ); + bit =trace_test_and_set_recursion (ip, parent_ip,TRACE_LIST_START ,TRACE_LIST_MAX );if (bit <0 )return ;
@@ -6993,7 +6993,7 @@ static void ftrace_ops_assist_func(unsigned long ip, unsigned long parent_ip, {int bit; - bit =trace_test_and_set_recursion (TRACE_LIST_START ,TRACE_LIST_MAX ); + bit =trace_test_and_set_recursion (ip, parent_ip,TRACE_LIST_START ,TRACE_LIST_MAX );if (bit <0 )return ;
kernel/trace/trace_event_perf.c +1 -1
@@ -447,7 +447,7 @@ perf_ftrace_function_call(unsigned long ip, unsigned long parent_ip,if ((unsigned long )ops->private !=smp_processor_id ())return ; - bit =ftrace_test_recursion_trylock (); + bit =ftrace_test_recursion_trylock (ip, parent_ip);if (bit <0 )return ;
kernel/trace/trace_functions.c +1 -1
@@ -141,7 +141,7 @@ function_trace_call(unsigned long ip, unsigned long parent_ip,if (unlikely (!tr->function_enabled))return ; - bit =ftrace_test_recursion_trylock (); + bit =ftrace_test_recursion_trylock (ip, parent_ip);if (bit <0 )return ;
kernel/trace/trace_output.c +3 -3
@@ -353,8 +353,8 @@ static inline const char *kretprobed(const char *name) }#endif /* CONFIG_KRETPROBES */ -static void -seq_print_sym (struct trace_seq *s,unsigned long address,bool offset) +void +trace_seq_print_sym (struct trace_seq *s,unsigned long address,bool offset) {#ifdef CONFIG_KALLSYMS char str[KSYM_SYMBOL_LEN ];
@@ -420,7 +420,7 @@ seq_print_ip_sym(struct trace_seq *s, unsigned long ip, unsigned long sym_flags) goto out; } -seq_print_sym (s, ip, sym_flags &TRACE_ITER_SYM_OFFSET ); +trace_seq_print_sym (s, ip, sym_flags &TRACE_ITER_SYM_OFFSET );if (sym_flags &TRACE_ITER_SYM_ADDR )trace_seq_printf (s," <" IP_FMT ">" , ip);
kernel/trace/trace_output.h +1 -0
@@ -16,6 +16,7 @@ extern int seq_print_ip_sym(struct trace_seq *s, unsigned long ip, unsigned long sym_flags); +extern void trace_seq_print_sym (struct trace_seq *s,unsigned long address,bool offset);extern int trace_print_context (struct trace_iterator *iter);extern int trace_print_lat_context (struct trace_iterator *iter);
kernel/trace/trace_recursion_record.c new file +235 -0
235 lines not on this page · raw message
Signature · 1 line
2.25.4
Steven Rostedt
maintainerPATCH v2 11/11 ftrace: Add recording of functions that caused recursion
In reply to Steven RostedtFrom c532ff6b048dd4a12943b05c7b8ce30666c587c8 Mon Sep 17 00:00:00 2001
From: "Steven Rostedt (VMware)" <[email protected]>
Date: Thu, 29 Oct 2020 15:27:06 -0400
Subject: [PATCH] ftrace: Add recording of functions that caused recursion
This adds CONFIG_FTRACE_RECORD_RECURSION that will record to a file "recursed_functions" all the functions that caused recursion while a callback to the function tracer was running.
---
Changes since v2.1:
Added EXPORT_SYMBOL_GPL() to ftrace_record_recursion() function
Documentation/trace/ftrace-uses.rst | 6 +- arch/csky/kernel/probes/ftrace.c | 2 +- arch/parisc/kernel/ftrace.c | 2 +- arch/powerpc/kernel/kprobes-ftrace.c | 2 +- arch/s390/kernel/ftrace.c | 2 +- arch/x86/kernel/kprobes/ftrace.c | 2 +- fs/pstore/ftrace.c | 2 +- include/linux/trace_recursion.h | 32 +++- kernel/livepatch/patch.c | 2 +- kernel/trace/Kconfig | 25 +++ kernel/trace/Makefile | 1 + kernel/trace/ftrace.c | 4 +- kernel/trace/trace_event_perf.c | 2 +- kernel/trace/trace_functions.c | 2 +- kernel/trace/trace_output.c | 6 +- kernel/trace/trace_output.h | 1 + kernel/trace/trace_recursion_record.c | 236 ++++++++++++++++++++++++++ 17 files changed, 309 insertions(+), 20 deletions(-) create mode 100644 kernel/trace/trace_recursion_record.c
Documentation/trace/ftrace-uses.rst +4 -2
@@ -118,7 +118,7 @@ can help in this regard. If you start your code with: int bit; - bit = ftrace_test_recursion_trylock(); + bit = ftrace_test_recursion_trylock(ip, parent_ip); if (bit < 0) return;
@@ -130,7 +130,9 @@ The code in between will be safe to use, even if it ends up calling a function that the callback is tracing. Note, on success, ftrace_test_recursion_trylock() will disable preemption, and the ftrace_test_recursion_unlock() will enable it again (if it was previously -enabled). +enabled). The instruction pointer (ip) and its parent (parent_ip) is passed to +ftrace_test_recursion_trylock() to record where the recursion happened +(if CONFIG_FTRACE_RECORD_RECURSION is set). Alternatively, if the FTRACE_OPS_FL_RECURSION flag is set on the ftrace_ops (as explained below), then a helper trampoline will be used to test
arch/csky/kernel/probes/ftrace.c +1 -1
@@ -18,7 +18,7 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,struct kprobe *p;struct kprobe_ctlblk *kcb; - bit =ftrace_test_recursion_trylock (); + bit =ftrace_test_recursion_trylock (ip, parent_ip);if (bit <0 )return ;
arch/parisc/kernel/ftrace.c +1 -1
@@ -210,7 +210,7 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,struct kprobe *p =get_kprobe ((kprobe_opcode_t *)ip);int bit; - bit =ftrace_test_recursion_trylock (); + bit =ftrace_test_recursion_trylock (ip, parent_ip);if (bit <0 )return ;
arch/powerpc/kernel/kprobes-ftrace.c +1 -1
@@ -20,7 +20,7 @@ void kprobe_ftrace_handler(unsigned long nip, unsigned long parent_nip,struct kprobe_ctlblk *kcb;int bit; - bit =ftrace_test_recursion_trylock (); + bit =ftrace_test_recursion_trylock (nip, parent_nip);if (bit <0 )return ;
arch/s390/kernel/ftrace.c +1 -1
@@ -204,7 +204,7 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,struct kprobe *p =get_kprobe ((kprobe_opcode_t *)ip);int bit; - bit =ftrace_test_recursion_trylock (); + bit =ftrace_test_recursion_trylock (ip, parent_ip);if (bit <0 )return ;
arch/x86/kernel/kprobes/ftrace.c +1 -1
@@ -20,7 +20,7 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,struct kprobe_ctlblk *kcb;int bit; - bit =ftrace_test_recursion_trylock (); + bit =ftrace_test_recursion_trylock (ip, parent_ip);if (bit <0 )return ;
fs/pstore/ftrace.c +1 -1
@@ -41,7 +41,7 @@ static void notrace pstore_ftrace_call(unsigned long ip,if (unlikely (oops_in_progress))return ; - bit =ftrace_test_recursion_trylock (); + bit =ftrace_test_recursion_trylock (ip, parent_ip);if (bit <0 )return ;
include/linux/trace_recursion.h +28 -4
@@ -142,7 +142,28 @@ static __always_inline int trace_get_context_bit(void) pc &HARDIRQ_MASK ?TRACE_CTX_IRQ :TRACE_CTX_SOFTIRQ ; } -static __always_inline int trace_test_and_set_recursion(int start, int max) +#ifdef CONFIG_FTRACE_RECORD_RECURSION +extern void ftrace_record_recursion (unsigned long ip,unsigned long parent_ip); +/* +* The paranoid_test check can cause dropped reports (unlikely), but +* if the recursion is common, it will likely still be recorded later. +* But the paranoid_test is needed to make sure we don't crash. +*/ +# define do_ftrace_record_recursion (ip, pip) \ + do { \ + static atomic_t paranoid_test; \ + if (!atomic_read(¶noid_test)) { \ + atomic_inc(¶noid_test); \ + ftrace_record_recursion(ip, pip); \ + atomic_dec(¶noid_test); \ + } \ + } while (0) +#else +# define do_ftrace_record_recursion (ip, pip) do { } while (0) +#endif + +static __always_inline int trace_test_and_set_recursion(unsigned long ip, unsigned long pip, +int start,int max) {unsigned int val = current->trace_recursion;int bit;
@@ -158,8 +179,10 @@ static __always_inline int trace_test_and_set_recursion(int start, int max)* a switch between contexts. Allow for a single recursion. */ bit =TRACE_TRANSITION_BIT ; -if (trace_recursion_test (bit)) +if (trace_recursion_test (bit)) { +do_ftrace_record_recursion (ip, pip);return -1 ; + }trace_recursion_set (bit);barrier ();return bit +1 ;
@@ -199,9 +222,10 @@ static __always_inline void trace_clear_recursion(int bit)* Returns: -1 if a recursion happened. * >= 0 if no recursion */ -static __always_inline int ftrace_test_recursion_trylock(void) +static __always_inline int ftrace_test_recursion_trylock(unsigned long ip, +unsigned long parent_ip) { -return trace_test_and_set_recursion (TRACE_FTRACE_START ,TRACE_FTRACE_MAX ); +return trace_test_and_set_recursion (ip, parent_ip,TRACE_FTRACE_START ,TRACE_FTRACE_MAX ); }/**
kernel/livepatch/patch.c +1 -1
@@ -49,7 +49,7 @@ static void notrace klp_ftrace_handler(unsigned long ip, ops = container_of(fops, struct klp_ops, fops); - bit =ftrace_test_recursion_trylock (); + bit =ftrace_test_recursion_trylock (ip, parent_ip);if (WARN_ON_ONCE (bit <0 ))return ;/*
kernel/trace/Kconfig +25 -0
@@ -727,6 +727,31 @@ config TRACE_EVAL_MAP_FILE If unsure, say N. +config FTRACE_RECORD_RECURSION + bool "Record functions that recurse in function tracing" + depends on FUNCTION_TRACER + help + All callbacks that attach to the function tracing have some sort + of protection against recursion. Even though the protection exists, + it adds overhead. This option will create a file in the tracefs + file system called "recursed_functions" that will list the functions + that triggered a recursion. + + This will add more overhead to cases that have recursion. + + If unsure, say N + +config FTRACE_RECORD_RECURSION_SIZE + int "Max number of recursed functions to record" + default 128 + depends on FTRACE_RECORD_RECURSION + help + This defines the limit of number of functions that can be + listed in the "recursed_functions" file, that lists all + the functions that caused a recursion to happen. + This file can be reset, but the limit can not change in + size at runtime. + config GCOV_PROFILE_FTRACE bool "Enable GCOV profiling on ftrace subsystem" depends on GCOV_KERNEL
kernel/trace/Makefile +1 -0
@@ -92,6 +92,7 @@ obj-$(CONFIG_DYNAMIC_EVENTS) += trace_dynevent.o obj-$(CONFIG_PROBE_EVENTS) += trace_probe.o obj-$(CONFIG_UPROBE_EVENTS) += trace_uprobe.o obj-$(CONFIG_BOOTTIME_TRACING) += trace_boot.o +obj-$(CONFIG_FTRACE_RECORD_RECURSION) += trace_recursion_record.o obj-$(CONFIG_TRACEPOINT_BENCHMARK) += trace_benchmark.o
kernel/trace/ftrace.c +2 -2
@@ -6918,7 +6918,7 @@ __ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,struct ftrace_ops *op;int bit; - bit =trace_test_and_set_recursion (TRACE_LIST_START ,TRACE_LIST_MAX ); + bit =trace_test_and_set_recursion (ip, parent_ip,TRACE_LIST_START ,TRACE_LIST_MAX );if (bit <0 )return ;
@@ -6993,7 +6993,7 @@ static void ftrace_ops_assist_func(unsigned long ip, unsigned long parent_ip, {int bit; - bit =trace_test_and_set_recursion (TRACE_LIST_START ,TRACE_LIST_MAX ); + bit =trace_test_and_set_recursion (ip, parent_ip,TRACE_LIST_START ,TRACE_LIST_MAX );if (bit <0 )return ;
kernel/trace/trace_event_perf.c +1 -1
@@ -447,7 +447,7 @@ perf_ftrace_function_call(unsigned long ip, unsigned long parent_ip,if ((unsigned long )ops->private !=smp_processor_id ())return ; - bit =ftrace_test_recursion_trylock (); + bit =ftrace_test_recursion_trylock (ip, parent_ip);if (bit <0 )return ;
kernel/trace/trace_functions.c +1 -1
@@ -141,7 +141,7 @@ function_trace_call(unsigned long ip, unsigned long parent_ip,if (unlikely (!tr->function_enabled))return ; - bit =ftrace_test_recursion_trylock (); + bit =ftrace_test_recursion_trylock (ip, parent_ip);if (bit <0 )return ;
kernel/trace/trace_output.c +3 -3
@@ -353,8 +353,8 @@ static inline const char *kretprobed(const char *name) }#endif /* CONFIG_KRETPROBES */ -static void -seq_print_sym (struct trace_seq *s,unsigned long address,bool offset) +void +trace_seq_print_sym (struct trace_seq *s,unsigned long address,bool offset) {#ifdef CONFIG_KALLSYMS char str[KSYM_SYMBOL_LEN ];
@@ -420,7 +420,7 @@ seq_print_ip_sym(struct trace_seq *s, unsigned long ip, unsigned long sym_flags) goto out; } -seq_print_sym (s, ip, sym_flags &TRACE_ITER_SYM_OFFSET ); +trace_seq_print_sym (s, ip, sym_flags &TRACE_ITER_SYM_OFFSET );if (sym_flags &TRACE_ITER_SYM_ADDR )trace_seq_printf (s," <" IP_FMT ">" , ip);
kernel/trace/trace_output.h +1 -0
@@ -16,6 +16,7 @@ extern int seq_print_ip_sym(struct trace_seq *s, unsigned long ip, unsigned long sym_flags); +extern void trace_seq_print_sym (struct trace_seq *s,unsigned long address,bool offset);extern int trace_print_context (struct trace_iterator *iter);extern int trace_print_lat_context (struct trace_iterator *iter);
kernel/trace/trace_recursion_record.c new file +236 -0
236 lines not on this page · raw message
Signature · 1 line
2.25.4
Petr Mladek
maintainerRe: [PATCH 11/11 v2] ftrace: Add recording of functions that caused recursion
In reply to Steven RostedtQuoted 64 lines · Steven Rostedt
Quoted 38 lines · On Mon, 2 Nov 2020 17:41:47 +0100 Petr Mladek <[email protected]> wrote:
Quoted 24 lines · Steven Rostedt
From: "Steven Rostedt (VMware)" <[email protected]>
This adds
CONFIG_FTRACE_RECORD_RECURSIONthat will record to a file "recursed_functions" all the functions that caused recursion while a callback to the function tracer was running.kernel/trace/trace_recursion_record.c new file +15
-0+if (index >=CONFIG_FTRACE_RECORD_RECURSION_SIZE ) +return ; + +for (i = index -1 ; i >=0 ; i--) { +if (recursed_functions[i].ip == ip) { + cached_function = ip; +return ; + } + } + + cached_function = ip; + +/* +* We only want to add a function if it hasn't been added before. +* Add to the current location before incrementing the count. kernel/trace/trace_recursion_record.c line 69
Quoted 10 lines
kernel/trace/trace_recursion_record.c new file +10
-0+* If it fails to add, then increment the index (save in i) +* and try again. +*/ + old =cmpxchg (&recursed_functions[index].ip,0 , ip); +if (old !=0 ) { +/* Did something else already added this for us? */ +if (old == ip) +return ; +/* Try the next location (use i for the next index) */ + i = index +1 ;What about
Quoted 3 lines
index++;We basically want to run the code again with index + 1 limit.
But something else could update
nr_records, and we want to use that ifnr_recordsis greater than i.Now, we could swap the use case, and have
int index = 0; [..] i = atomic_read(&nr_records); if (i > index) index = i; [..] index++; goto again;Quoted 1 line
Quoted 4 lines
Quoted 2 lines
Maybe, it even does not make sense to check the array again and we should just try to store the value into the next slot.
We do this dance to prevent duplicates.
I see.
My code was wrong. It reserved slot for the new "ip" by cmpxchg on nr_records. The "ip" was stored later so that any parallel call need not see that it is a dumplicate.
Your code reserves the slot by cmpxchg of "ip". Any parallel call would fail to take the slot and see the "ip" in the next iteration.
Best Regards,
Petr
Masami Hiramatsu
maintainerRe: [PATCH 05/11 v2] kprobes/ftrace: Add recursion protection to the ftrace callback
In reply to Steven RostedtQuoted 88 lines · On Fri, 30 Oct 2020 17:31:47 -0400 Steven Rostedt <[email protected]> wrote:
From: "Steven Rostedt (VMware)" <[email protected]>
If a ftrace callback does not supply its own recursion protection and does not set the
RECURSION_SAFEflag in itsftrace_ops, then ftrace will make a helper trampoline to do so before calling the callback instead of just calling the callback directly.The default for
ftrace_opsis going to change. It will expect that handlers provide their own recursion protection, unless itsftrace_opsstates otherwise.Cc Andrew Morton <[email protected]>Cc Masami Hiramatsu <[email protected]>Cc Guo Ren <[email protected]>Cc "James E.J. Bottomley" <[email protected]>Cc Helge Deller <[email protected]>Cc Michael Ellerman <[email protected]>Cc Benjamin Herrenschmidt <[email protected]>Cc Paul Mackerras <[email protected]>Cc Heiko Carstens <[email protected]>Cc Vasily Gorbik <[email protected]>Cc Christian Borntraeger <[email protected]>Cc Thomas Gleixner <[email protected]>Cc Borislav Petkov <[email protected]>Cc "H. Peter Anvin" <[email protected]>Cc "Naveen N. Rao" <[email protected]>Cc Anil S Keshavamurthy <[email protected]>Cc "David S. Miller" <[email protected]>Signed-off-by Steven Rostedt (VMware) <[email protected]>--- arch/csky/kernel/probes/ftrace.c | 12 ++++++++++-- arch/parisc/kernel/ftrace.c | 13 +++++++++++-- arch/powerpc/kernel/kprobes-ftrace.c | 11 ++++++++++- arch/s390/kernel/ftrace.c | 13 +++++++++++-- arch/x86/kernel/kprobes/ftrace.c | 12 ++++++++++-- 5 files changed, 52 insertions(+), 9 deletions(-)arch/csky/kernel/probes/ftrace.c +10
-2@@ -13,16 +13,21 @@ int arch_check_ftrace_location(struct kprobe *p)void kprobe_ftrace_handler (unsigned long ip,unsigned long parent_ip,struct ftrace_ops *ops,struct pt_regs *regs) { +int bit;bool lr_saver = false;struct kprobe *p;struct kprobe_ctlblk *kcb; -/* Preempt is disabled by ftrace */ + bit =ftrace_test_recursion_trylock (); +if (bit <0 ) +return ; + +preempt_disable_notrace (); p =get_kprobe ((kprobe_opcode_t *)ip);if (!p) { p =get_kprobe ((kprobe_opcode_t *)(ip -MCOUNT_INSN_SIZE ));if (unlikely (!p) ||kprobe_disabled (p)) -return ; + goto out; lr_saver = true; }@@ -56,6 +61,9 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,*/ __this_cpu_write (current_kprobe,NULL ); } +out: +preempt_enable_notrace (); +ftrace_test_recursion_unlock (bit); }NOKPROBE_SYMBOL (kprobe_ftrace_handler);arch/parisc/kernel/ftrace.c +0
-0@@ -208,13 +208,19 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, {
arch/parisc/kernel/ftrace.c void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, · line 217
Quoted 10 lines
arch/parisc/kernel/ftrace.c +4
-1struct kprobe_ctlblk *kcb;struct kprobe *p =get_kprobe ((kprobe_opcode_t *)ip); +int bit; -if (unlikely (!p) ||kprobe_disabled (p)) + bit =ftrace_test_recursion_trylock (); +if (bit <0 )return ; +preempt_disable_notrace ();
If we disable preempt here, we also move the get_kprobe() here as below. (get_kprobe() accesses percpu variable)
p = get_kprobe((kprobe_opcode_t *)ip);
Quoted 58 lines
arch/parisc/kernel/ftrace.c +7
-1+if (unlikely (!p) ||kprobe_disabled (p)) + goto out; +if (kprobe_running ()) {kprobes_inc_nmissed_count (p); -return ; + goto out; }__this_cpu_write (current_kprobe, p);@@ -235,6 +241,9 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, } }__this_cpu_write (current_kprobe,NULL ); +out: +preempt_enable_notrace (); +ftrace_test_recursion_unlock (bit); }NOKPROBE_SYMBOL (kprobe_ftrace_handler);arch/powerpc/kernel/kprobes-ftrace.c +10
-1@@ -18,10 +18,16 @@ void kprobe_ftrace_handler(unsigned long nip, unsigned long parent_nip, {struct kprobe *p;struct kprobe_ctlblk *kcb; +int bit; + bit =ftrace_test_recursion_trylock (); +if (bit <0 ) +return ; + +preempt_disable_notrace (); p =get_kprobe ((kprobe_opcode_t *)nip);if (unlikely (!p) ||kprobe_disabled (p)) -return ; + goto out; kcb =get_kprobe_ctlblk ();if (kprobe_running ()) {@@ -52,6 +58,9 @@ void kprobe_ftrace_handler(unsigned long nip, unsigned long parent_nip,*/ __this_cpu_write (current_kprobe,NULL ); } +out: +preempt_enable_notrace (); +ftrace_test_recursion_unlock (bit); }NOKPROBE_SYMBOL (kprobe_ftrace_handler);arch/s390/kernel/ftrace.c +0
-0@@ -202,13 +202,19 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, {
arch/s390/kernel/ftrace.c void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, · line 211
Quoted 10 lines
arch/s390/kernel/ftrace.c +4
-1struct kprobe_ctlblk *kcb;struct kprobe *p =get_kprobe ((kprobe_opcode_t *)ip); +int bit; -if (unlikely (!p) ||kprobe_disabled (p)) + bit =ftrace_test_recursion_trylock (); +if (bit <0 )return ; +preempt_disable_notrace ();
Ditto.
Others look good to me.
Thank you,
Quoted 57 lines
arch/s390/kernel/ftrace.c +7
-1+if (unlikely (!p) ||kprobe_disabled (p)) + goto out; +if (kprobe_running ()) {kprobes_inc_nmissed_count (p); -return ; + goto out; }__this_cpu_write (current_kprobe, p);@@ -228,6 +234,9 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, } }__this_cpu_write (current_kprobe,NULL ); +out: +preempt_enable_notrace (); +ftrace_test_recursion_unlock (bit); }NOKPROBE_SYMBOL (kprobe_ftrace_handler);arch/x86/kernel/kprobes/ftrace.c +10
-2@@ -18,11 +18,16 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, {struct kprobe *p;struct kprobe_ctlblk *kcb; +int bit; -/* Preempt is disabled by ftrace */ + bit =ftrace_test_recursion_trylock (); +if (bit <0 ) +return ; + +preempt_disable_notrace (); p =get_kprobe ((kprobe_opcode_t *)ip);if (unlikely (!p) ||kprobe_disabled (p)) -return ; + goto out; kcb =get_kprobe_ctlblk ();if (kprobe_running ()) {@@ -52,6 +57,9 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,*/ __this_cpu_write (current_kprobe,NULL ); } +out: +preempt_enable_notrace (); +ftrace_test_recursion_unlock (bit); }NOKPROBE_SYMBOL (kprobe_ftrace_handler);Signature · 1 line
2.28.0
Signature · 1 line
Masami Hiramatsu <[email protected]>
Petr Mladek
maintainerRe: [PATCH 11/11 v2.2] ftrace: Add recording of functions that caused recursion
In reply to Steven RostedtQuoted 22 lines · Steven Rostedt
From
c532ff6b048dd4a12943b05c7b8ce30666c587c8Mon Sep 17 00:00:00 2001
From: "Steven Rostedt (VMware)" <[email protected]>
Date: Thu, 29 Oct 2020 15:27:06 -0400
Subject: [PATCH] ftrace: Add recording of functions that caused recursionThis adds
CONFIG_FTRACE_RECORD_RECURSIONthat will record to a file "recursed_functions" all the functions that caused recursion while a callback to the function tracer was running.include/linux/trace_recursion.h +4
-1@@ -142,7 +142,28 @@ static __always_inline int trace_get_context_bit(void) pc &HARDIRQ_MASK ?TRACE_CTX_IRQ :TRACE_CTX_SOFTIRQ ; } -static __always_inline int trace_test_and_set_recursion(int start, int max) +#ifdef CONFIG_FTRACE_RECORD_RECURSION +extern void ftrace_record_recursion (unsigned long ip,unsigned long parent_ip); +/* +* The paranoid_test check can cause dropped reports (unlikely), but
include/linux/trace_recursion.h static __always_inline int trace_get_context_bit(void) · line 158
Quoted 10 lines
include/linux/trace_recursion.h +10
-0+* if the recursion is common, it will likely still be recorded later. +* But the paranoid_test is needed to make sure we don't crash. +*/ +# define do_ftrace_record_recursion (ip, pip) \ + do { \ + static atomic_t paranoid_test; \ + if (!atomic_read(¶noid_test)) { \ + atomic_inc(¶noid_test); \ + ftrace_record_recursion(ip, pip); \ + atomic_dec(¶noid_test); \
BTW: What is actually the purpose of paranoid_test, please?
It prevents nested ftrace_record_recursion() calls on the same CPU (recursion, nesting from IRQ, NMI context).
Parallel calls from different CPUs are still possible:
CPU0 CPU1
if (!atomic_read(¶noid_test)) if (!atomic_read(¶noid_test))
// passes // passes
atomic_inc(¶noid_test); atomic_inc(¶noid_test);
I do not see how a nested call could cause crash while a parallel one would be OK.
Quoted 107 lines
kernel/trace/trace_recursion_record.c new file +104
-0@@ -0,0 +1,236 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/seq_file.h> +#include <linux/kallsyms.h> +#include <linux/module.h> +#include <linux/ftrace.h> +#include <linux/fs.h> + +#include "trace_output.h" + +struct recursed_functions { +unsigned long ip; +unsigned long parent_ip; +}; + +static struct recursed_functions recursed_functions[CONFIG_FTRACE_RECORD_RECURSION_SIZE ]; +static atomic_t nr_records; + +/* +* Cache the last found function. Yes, updates to this is racey, but +* so is memory cache ;-) +*/ +static unsigned long cached_function; + +void ftrace_record_recursion (unsigned long ip,unsigned long parent_ip) +{ +int index =0 ; +int i; +unsigned long old; + + again: +/* First check the last one recorded */ +if (ip == cached_function) +return ; + + i =atomic_read (&nr_records); +/* nr_records is -1 when clearing records */ +smp_mb__after_atomic (); +if (i <0 ) +return ; + +/* +* If there's two writers and this writer comes in second, +* the cmpxchg() below to update the ip will fail. Then this +* writer will try again. It is possible that index will now +* be greater than nr_records. This is because the writer +* that succeeded has not updated the nr_records yet. +* This writer could keep trying again until the other writer +* updates nr_records. But if the other writer takes an +* interrupt, and that interrupt locks up that CPU, we do +* not want this CPU to lock up due to the recursion protection, +* and have a bug report showing this CPU as the cause of +* locking up the computer. To not lose this record, this +* writer will simply use the next position to update the +* recursed_functions, and it will update the nr_records +* accordingly. +*/ +if (index < i) + index = i; +if (index >=CONFIG_FTRACE_RECORD_RECURSION_SIZE ) +return ; + +for (i = index -1 ; i >=0 ; i--) { +if (recursed_functions[i].ip == ip) { + cached_function = ip; +return ; + } + } + + cached_function = ip; + +/* +* We only want to add a function if it hasn't been added before. +* Add to the current location before incrementing the count. +* If it fails to add, then increment the index (save in i) +* and try again. +*/ + old =cmpxchg (&recursed_functions[index].ip,0 , ip); +if (old !=0 ) { +/* Did something else already added this for us? */ +if (old == ip) +return ; +/* Try the next location (use i for the next index) */ + index++; + goto again; + } + + recursed_functions[index].parent_ip = parent_ip; + +/* +* It's still possible that we could race with the clearing +* CPU0 CPU1 +* ---- ---- +* ip = func +* nr_records = -1; +* recursed_functions[0] = 0; +* i = -1 +* if (i < 0) +* nr_records = 0; +* (new recursion detected) +* recursed_functions[0] = func +* cmpxchg(recursed_functions[0], +* func, 0) +*
kernel/trace/trace_recursion_record.c line 114
Quoted 10 lines
kernel/trace/trace_recursion_record.c new file +10
-0+* But the worse that could happen is that we get a zero in +* the recursed_functions array, and it's likely that "func" will +* be recorded again. +*/ + i =atomic_read (&nr_records); +smp_mb__after_atomic (); +if (i <0 ) +cmpxchg (&recursed_functions[index].ip, ip,0 ); +else if (i <= index) +atomic_cmpxchg (&nr_records, i, index +1 );
Are you aware of the following race, please?
CPU0 CPU1
ftrace_record_recursion()
i = atomic_read(&nr_records);
// i = 20 (for example)
if (i < index)
index = i;
// index = 20;recursed_function_open() atomic_set(&nr_records, -1); memset(recursed_functions, 0, ); atomic_set(&nr_records, 0);
// successfully store ip at index == 20 cmpxchg(&recursed_functions[index].ip, 0, ip); recursed_functions[index].parent_ip = parent_ip;
// check race with clearing
i = atomic_read(&nr_records);
// i == 0
if (i < 0)
// no else atomic_cmpxchg(&nr_records, i, index + 1);
RESULT:
+ nr_records == 21
+ and slots 0..19 are zeroed
I played with the code and ended with head entangled by chicken & egg like problems.
I believe that a solution might be a combined atomic variable from nr_records + cleanup_count.
ftrace_record_recursion() would be allowed to increase nr_records only when cleanup_count is still the same. cleanup_count would be incremented together with clearing nr_records.
Well, I am not sure if it is worth the effort. The race is rather theoretical. In the worst case, the cache might contain many zero values.
Anyway, it is yet another experience for me that lockless algorithms are more tricky that one would expect.
Best Regards,
Petr
Steven Rostedt
maintainerRe: [PATCH 11/11 v2.2] ftrace: Add recording of functions that caused recursion
Quoted 7 lines · On Tue, 3 Nov 2020 15:10:43 +0100 Petr Mladek <[email protected]> wrote:
BTW: What is actually the purpose of
paranoid_test, please?It prevents nested
ftrace_record_recursion()calls on the same CPU (recursion, nesting from IRQ, NMI context).Parallel calls from different CPUs are still possible:
Quoted 9 lines
CPU0 CPU1
if (!atomic_read(¶noid_test)) if (!atomic_read(¶noid_test))
// passes // passes
atomic_inc(¶noid_test);atomic_inc(¶noid_test);I do not see how a nested call could cause crash while a parallel one would be OK.
Yeah, I should make that per cpu, but was lazy. ;-)
It was added at the end.
I'll update that to a per cpu, and local inc operations.
-- Steve
Steven Rostedt
maintainerPATCH v2 5/11 kprobes/ftrace: Add recursion protection to the ftrace callback
In reply to Masami HiramatsuFrom: "Steven Rostedt (VMware)" <[email protected]>
If a ftrace callback does not supply its own recursion protection and does not set the RECURSION_SAFE flag in its ftrace_ops, then ftrace will make a helper trampoline to do so before calling the callback instead of just calling the callback directly.
The default for ftrace_ops is going to change. It will expect that handlers provide their own recursion protection, unless its ftrace_ops states otherwise.
--- arch/csky/kernel/probes/ftrace.c | 12 ++++++++++-- arch/parisc/kernel/ftrace.c | 16 +++++++++++++--- arch/powerpc/kernel/kprobes-ftrace.c | 11 ++++++++++- arch/s390/kernel/ftrace.c | 16 +++++++++++++--- arch/x86/kernel/kprobes/ftrace.c | 12 ++++++++++-- 5 files changed, 56 insertions(+), 11 deletions(-)
arch/csky/kernel/probes/ftrace.c +10 -2
@@ -13,16 +13,21 @@ int arch_check_ftrace_location(struct kprobe *p)void kprobe_ftrace_handler (unsigned long ip,unsigned long parent_ip,struct ftrace_ops *ops,struct pt_regs *regs) { +int bit;bool lr_saver = false;struct kprobe *p;struct kprobe_ctlblk *kcb; -/* Preempt is disabled by ftrace */ + bit =ftrace_test_recursion_trylock (); +if (bit <0 ) +return ; + +preempt_disable_notrace (); p =get_kprobe ((kprobe_opcode_t *)ip);if (!p) { p =get_kprobe ((kprobe_opcode_t *)(ip -MCOUNT_INSN_SIZE ));if (unlikely (!p) ||kprobe_disabled (p)) -return ; + goto out; lr_saver = true; }
@@ -56,6 +61,9 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,*/ __this_cpu_write (current_kprobe,NULL ); } +out: +preempt_enable_notrace (); +ftrace_test_recursion_unlock (bit); }NOKPROBE_SYMBOL (kprobe_ftrace_handler);
arch/parisc/kernel/ftrace.c +13 -3
@@ -207,14 +207,21 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, struct ftrace_ops *ops, struct pt_regs *regs) { struct kprobe_ctlblk *kcb; -struct kprobe *p =get_kprobe ((kprobe_opcode_t *)ip); +struct kprobe *p; +int bit; -if (unlikely (!p) ||kprobe_disabled (p)) + bit =ftrace_test_recursion_trylock (); +if (bit <0 )return ; +preempt_disable_notrace (); + p =get_kprobe ((kprobe_opcode_t *)ip); +if (unlikely (!p) ||kprobe_disabled (p)) + goto out; +if (kprobe_running ()) {kprobes_inc_nmissed_count (p); -return ; + goto out; }__this_cpu_write (current_kprobe, p);
@@ -235,6 +242,9 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, } }__this_cpu_write (current_kprobe,NULL ); +out: +preempt_enable_notrace (); +ftrace_test_recursion_unlock (bit); }NOKPROBE_SYMBOL (kprobe_ftrace_handler);
arch/powerpc/kernel/kprobes-ftrace.c +10 -1
@@ -18,10 +18,16 @@ void kprobe_ftrace_handler(unsigned long nip, unsigned long parent_nip, {struct kprobe *p;struct kprobe_ctlblk *kcb; +int bit; + bit =ftrace_test_recursion_trylock (); +if (bit <0 ) +return ; + +preempt_disable_notrace (); p =get_kprobe ((kprobe_opcode_t *)nip);if (unlikely (!p) ||kprobe_disabled (p)) -return ; + goto out; kcb =get_kprobe_ctlblk ();if (kprobe_running ()) {
@@ -52,6 +58,9 @@ void kprobe_ftrace_handler(unsigned long nip, unsigned long parent_nip,*/ __this_cpu_write (current_kprobe,NULL ); } +out: +preempt_enable_notrace (); +ftrace_test_recursion_unlock (bit); }NOKPROBE_SYMBOL (kprobe_ftrace_handler);
arch/s390/kernel/ftrace.c +13 -3
@@ -201,14 +201,21 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, struct ftrace_ops *ops, struct pt_regs *regs) { struct kprobe_ctlblk *kcb; -struct kprobe *p =get_kprobe ((kprobe_opcode_t *)ip); +struct kprobe *p; +int bit; -if (unlikely (!p) ||kprobe_disabled (p)) + bit =ftrace_test_recursion_trylock (); +if (bit <0 )return ; +preempt_disable_notrace (); + p =get_kprobe ((kprobe_opcode_t *)ip); +if (unlikely (!p) ||kprobe_disabled (p)) + goto out; +if (kprobe_running ()) {kprobes_inc_nmissed_count (p); -return ; + goto out; }__this_cpu_write (current_kprobe, p);
@@ -228,6 +235,9 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, } }__this_cpu_write (current_kprobe,NULL ); +out: +preempt_enable_notrace (); +ftrace_test_recursion_unlock (bit); }NOKPROBE_SYMBOL (kprobe_ftrace_handler);
arch/x86/kernel/kprobes/ftrace.c +10 -2
@@ -18,11 +18,16 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, {struct kprobe *p;struct kprobe_ctlblk *kcb; +int bit; -/* Preempt is disabled by ftrace */ + bit =ftrace_test_recursion_trylock (); +if (bit <0 ) +return ; + +preempt_disable_notrace (); p =get_kprobe ((kprobe_opcode_t *)ip);if (unlikely (!p) ||kprobe_disabled (p)) -return ; + goto out; kcb =get_kprobe_ctlblk ();if (kprobe_running ()) {
@@ -52,6 +57,9 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,*/ __this_cpu_write (current_kprobe,NULL ); } +out: +preempt_enable_notrace (); +ftrace_test_recursion_unlock (bit); }NOKPROBE_SYMBOL (kprobe_ftrace_handler);
Signature · 1 line
2.25.4
Steven Rostedt
maintainerRe: [PATCH 11/11 v2.2] ftrace: Add recording of functions that caused recursion
In reply to Petr MladekQuoted 7 lines · On Tue, 3 Nov 2020 15:10:43 +0100 Petr Mladek <[email protected]> wrote:
BTW: What is actually the purpose of
paranoid_test, please?It prevents nested
ftrace_record_recursion()calls on the same CPU (recursion, nesting from IRQ, NMI context).Parallel calls from different CPUs are still possible:
Quoted 8 lines
CPU0 CPU1
if (!atomic_read(¶noid_test)) if (!atomic_read(¶noid_test))
// passes // passes
atomic_inc(¶noid_test);atomic_inc(¶noid_test);I do not see how a nested call could cause crash while a parallel one would be OK.
The problem was that I don't want something in the recording to get traced, and that detect the recursion, which would then call the recording, which would also be traced and detect the recursion and that would call the recording, which would also be traced and detect the recursion and that would call the recording, which .... (BOOM!)
Anyway, I decided to simply use another bit to protect against that.
New patch:
From a1f2aae996506169f2561986656f898d067d398b Mon Sep 17 00:00:00 2001
From: "Steven Rostedt (VMware)" <[email protected]>
Date: Thu, 29 Oct 2020 15:27:06 -0400
Subject: [PATCH] ftrace: Add recording of functions that caused recursion
This adds CONFIG_FTRACE_RECORD_RECURSION that will record to a file "recursed_functions" all the functions that caused recursion while a callback to the function tracer was running.
--- Documentation/trace/ftrace-uses.rst | 6 +- arch/csky/kernel/probes/ftrace.c | 2 +- arch/parisc/kernel/ftrace.c | 2 +- arch/powerpc/kernel/kprobes-ftrace.c | 2 +- arch/s390/kernel/ftrace.c | 2 +- arch/x86/kernel/kprobes/ftrace.c | 2 +- fs/pstore/ftrace.c | 2 +- include/linux/trace_recursion.h | 29 +++- kernel/livepatch/patch.c | 2 +- kernel/trace/Kconfig | 25 +++ kernel/trace/Makefile | 1 + kernel/trace/ftrace.c | 4 +- kernel/trace/trace_event_perf.c | 2 +- kernel/trace/trace_functions.c | 2 +- kernel/trace/trace_output.c | 6 +- kernel/trace/trace_output.h | 1 + kernel/trace/trace_recursion_record.c | 236 ++++++++++++++++++++++++++ 17 files changed, 306 insertions(+), 20 deletions(-) create mode 100644 kernel/trace/trace_recursion_record.c
Documentation/trace/ftrace-uses.rst +4 -2
@@ -118,7 +118,7 @@ can help in this regard. If you start your code with: int bit; - bit = ftrace_test_recursion_trylock(); + bit = ftrace_test_recursion_trylock(ip, parent_ip); if (bit < 0) return;
@@ -130,7 +130,9 @@ The code in between will be safe to use, even if it ends up calling a function that the callback is tracing. Note, on success, ftrace_test_recursion_trylock() will disable preemption, and the ftrace_test_recursion_unlock() will enable it again (if it was previously -enabled). +enabled). The instruction pointer (ip) and its parent (parent_ip) is passed to +ftrace_test_recursion_trylock() to record where the recursion happened +(if CONFIG_FTRACE_RECORD_RECURSION is set). Alternatively, if the FTRACE_OPS_FL_RECURSION flag is set on the ftrace_ops (as explained below), then a helper trampoline will be used to test
arch/csky/kernel/probes/ftrace.c +1 -1
@@ -18,7 +18,7 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,struct kprobe *p;struct kprobe_ctlblk *kcb; - bit =ftrace_test_recursion_trylock (); + bit =ftrace_test_recursion_trylock (ip, parent_ip);if (bit <0 )return ;
arch/parisc/kernel/ftrace.c +1 -1
@@ -210,7 +210,7 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,struct kprobe *p;int bit; - bit =ftrace_test_recursion_trylock (); + bit =ftrace_test_recursion_trylock (ip, parent_ip);if (bit <0 )return ;
arch/powerpc/kernel/kprobes-ftrace.c +1 -1
@@ -20,7 +20,7 @@ void kprobe_ftrace_handler(unsigned long nip, unsigned long parent_nip,struct kprobe_ctlblk *kcb;int bit; - bit =ftrace_test_recursion_trylock (); + bit =ftrace_test_recursion_trylock (nip, parent_nip);if (bit <0 )return ;
arch/s390/kernel/ftrace.c +1 -1
@@ -204,7 +204,7 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,struct kprobe *p;int bit; - bit =ftrace_test_recursion_trylock (); + bit =ftrace_test_recursion_trylock (ip, parent_ip);if (bit <0 )return ;
arch/x86/kernel/kprobes/ftrace.c +1 -1
@@ -20,7 +20,7 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,struct kprobe_ctlblk *kcb;int bit; - bit =ftrace_test_recursion_trylock (); + bit =ftrace_test_recursion_trylock (ip, parent_ip);if (bit <0 )return ;
fs/pstore/ftrace.c +1 -1
@@ -41,7 +41,7 @@ static void notrace pstore_ftrace_call(unsigned long ip,if (unlikely (oops_in_progress))return ; - bit =ftrace_test_recursion_trylock (); + bit =ftrace_test_recursion_trylock (ip, parent_ip);if (bit <0 )return ;
include/linux/trace_recursion.h +25 -4
@@ -91,6 +91,9 @@ enum {* not be correct. Allow for a single recursion to cover this case. */ TRACE_TRANSITION_BIT , + +/* Used to prevent recursion recording from recursing. */ + TRACE_RECORD_RECURSION_BIT, };#define trace_recursion_set (bit) do { (current)->trace_recursion |= (1<<(bit)); } while (0)
@@ -142,7 +145,22 @@ static __always_inline int trace_get_context_bit(void) pc &HARDIRQ_MASK ?TRACE_CTX_IRQ :TRACE_CTX_SOFTIRQ ; } -static __always_inline int trace_test_and_set_recursion(int start, int max) +#ifdef CONFIG_FTRACE_RECORD_RECURSION +extern void ftrace_record_recursion (unsigned long ip,unsigned long parent_ip); +# define do_ftrace_record_recursion (ip, pip) \ + do { \ + if (!trace_recursion_test(TRACE_RECORD_RECURSION_BIT)) { \ + trace_recursion_set(TRACE_RECORD_RECURSION_BIT); \ + ftrace_record_recursion(ip, pip); \ + trace_recursion_clear(TRACE_RECORD_RECURSION_BIT); \ + } \ + } while (0) +#else +# define do_ftrace_record_recursion (ip, pip) do { } while (0) +#endif + +static __always_inline int trace_test_and_set_recursion(unsigned long ip, unsigned long pip, +int start,int max) {unsigned int val = current->trace_recursion;int bit;
@@ -158,8 +176,10 @@ static __always_inline int trace_test_and_set_recursion(int start, int max)* a switch between contexts. Allow for a single recursion. */ bit =TRACE_TRANSITION_BIT ; -if (trace_recursion_test (bit)) +if (trace_recursion_test (bit)) { +do_ftrace_record_recursion (ip, pip);return -1 ; + }trace_recursion_set (bit);barrier ();return bit +1 ;
@@ -199,9 +219,10 @@ static __always_inline void trace_clear_recursion(int bit)* Returns: -1 if a recursion happened. * >= 0 if no recursion */ -static __always_inline int ftrace_test_recursion_trylock(void) +static __always_inline int ftrace_test_recursion_trylock(unsigned long ip, +unsigned long parent_ip) { -return trace_test_and_set_recursion (TRACE_FTRACE_START ,TRACE_FTRACE_MAX ); +return trace_test_and_set_recursion (ip, parent_ip,TRACE_FTRACE_START ,TRACE_FTRACE_MAX ); }/**
kernel/livepatch/patch.c +1 -1
@@ -49,7 +49,7 @@ static void notrace klp_ftrace_handler(unsigned long ip, ops = container_of(fops, struct klp_ops, fops); - bit =ftrace_test_recursion_trylock (); + bit =ftrace_test_recursion_trylock (ip, parent_ip);if (WARN_ON_ONCE (bit <0 ))return ;/*
kernel/trace/Kconfig +25 -0
@@ -727,6 +727,31 @@ config TRACE_EVAL_MAP_FILE If unsure, say N. +config FTRACE_RECORD_RECURSION + bool "Record functions that recurse in function tracing" + depends on FUNCTION_TRACER + help + All callbacks that attach to the function tracing have some sort + of protection against recursion. Even though the protection exists, + it adds overhead. This option will create a file in the tracefs + file system called "recursed_functions" that will list the functions + that triggered a recursion. + + This will add more overhead to cases that have recursion. + + If unsure, say N + +config FTRACE_RECORD_RECURSION_SIZE + int "Max number of recursed functions to record" + default 128 + depends on FTRACE_RECORD_RECURSION + help + This defines the limit of number of functions that can be + listed in the "recursed_functions" file, that lists all + the functions that caused a recursion to happen. + This file can be reset, but the limit can not change in + size at runtime. + config GCOV_PROFILE_FTRACE bool "Enable GCOV profiling on ftrace subsystem" depends on GCOV_KERNEL
kernel/trace/Makefile +1 -0
@@ -92,6 +92,7 @@ obj-$(CONFIG_DYNAMIC_EVENTS) += trace_dynevent.o obj-$(CONFIG_PROBE_EVENTS) += trace_probe.o obj-$(CONFIG_UPROBE_EVENTS) += trace_uprobe.o obj-$(CONFIG_BOOTTIME_TRACING) += trace_boot.o +obj-$(CONFIG_FTRACE_RECORD_RECURSION) += trace_recursion_record.o obj-$(CONFIG_TRACEPOINT_BENCHMARK) += trace_benchmark.o
kernel/trace/ftrace.c +2 -2
@@ -6918,7 +6918,7 @@ __ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,struct ftrace_ops *op;int bit; - bit =trace_test_and_set_recursion (TRACE_LIST_START ,TRACE_LIST_MAX ); + bit =trace_test_and_set_recursion (ip, parent_ip,TRACE_LIST_START ,TRACE_LIST_MAX );if (bit <0 )return ;
@@ -6993,7 +6993,7 @@ static void ftrace_ops_assist_func(unsigned long ip, unsigned long parent_ip, {int bit; - bit =trace_test_and_set_recursion (TRACE_LIST_START ,TRACE_LIST_MAX ); + bit =trace_test_and_set_recursion (ip, parent_ip,TRACE_LIST_START ,TRACE_LIST_MAX );if (bit <0 )return ;
kernel/trace/trace_event_perf.c +1 -1
@@ -447,7 +447,7 @@ perf_ftrace_function_call(unsigned long ip, unsigned long parent_ip,if ((unsigned long )ops->private !=smp_processor_id ())return ; - bit =ftrace_test_recursion_trylock (); + bit =ftrace_test_recursion_trylock (ip, parent_ip);if (bit <0 )return ;
kernel/trace/trace_functions.c +1 -1
@@ -141,7 +141,7 @@ function_trace_call(unsigned long ip, unsigned long parent_ip,if (unlikely (!tr->function_enabled))return ; - bit =ftrace_test_recursion_trylock (); + bit =ftrace_test_recursion_trylock (ip, parent_ip);if (bit <0 )return ;
kernel/trace/trace_output.c +3 -3
@@ -353,8 +353,8 @@ static inline const char *kretprobed(const char *name) }#endif /* CONFIG_KRETPROBES */ -static void -seq_print_sym (struct trace_seq *s,unsigned long address,bool offset) +void +trace_seq_print_sym (struct trace_seq *s,unsigned long address,bool offset) {#ifdef CONFIG_KALLSYMS char str[KSYM_SYMBOL_LEN ];
@@ -420,7 +420,7 @@ seq_print_ip_sym(struct trace_seq *s, unsigned long ip, unsigned long sym_flags) goto out; } -seq_print_sym (s, ip, sym_flags &TRACE_ITER_SYM_OFFSET ); +trace_seq_print_sym (s, ip, sym_flags &TRACE_ITER_SYM_OFFSET );if (sym_flags &TRACE_ITER_SYM_ADDR )trace_seq_printf (s," <" IP_FMT ">" , ip);
kernel/trace/trace_output.h +1 -0
@@ -16,6 +16,7 @@ extern int seq_print_ip_sym(struct trace_seq *s, unsigned long ip, unsigned long sym_flags); +extern void trace_seq_print_sym (struct trace_seq *s,unsigned long address,bool offset);extern int trace_print_context (struct trace_iterator *iter);extern int trace_print_lat_context (struct trace_iterator *iter);
kernel/trace/trace_recursion_record.c new file +236 -0
236 lines not on this page · raw message
Signature · 1 line
2.25.4