fix & prevent the missing preemption disabling
v4 cover fix & prevent the missing preemption disabling
The testing show that perf_ftrace_function_call() are using smp_processor_id() with preemption enabled, all the checking on CPU could be wrong after preemption.
As Peter point out, the section between ftrace_test_recursion_trylock/unlock() pair require the preemption to be disabled as 'Documentation/trace/ftrace-uses.rst' explained, but currently the work is done outside of the helpers.
And since the internal using of trace_test_and_set_recursion() and trace_clear_recursion() also require preemption to be disabled, we can just merge the logical together.
Patch 1/2 will make sure preemption disabled when recursion lock succeed, patch 2/2 will do smp_processor_id() checking after trylock() to address the issue.
v1: https://lore.kernel.org/all/[email protected]/
v2: https://lore.kernel.org/all/[email protected]/
v3: https://lore.kernel.org/all/[email protected]/
Michael Wang (2):
ftrace: disable preemption when recursion locked
ftrace: do CPU checking after preemption disabled
arch/csky/kernel/probes/ftrace.c | 2 -- arch/parisc/kernel/ftrace.c | 2 -- arch/powerpc/kernel/kprobes-ftrace.c | 2 -- arch/riscv/kernel/probes/ftrace.c | 2 -- arch/x86/kernel/kprobes/ftrace.c | 2 -- include/linux/trace_recursion.h | 20 +++++++++++++++++++- kernel/livepatch/patch.c | 13 +++++++------ kernel/trace/ftrace.c | 15 +++++---------- kernel/trace/trace_event_perf.c | 6 +++--- kernel/trace/trace_functions.c | 5 ----- 10 files changed, 34 insertions(+), 35 deletions(-)
Signature · 1 line
1.8.3.1
PATCH v4 1/2 ftrace: disable preemption when recursion locked
As the documentation explained, ftrace_test_recursion_trylock() and ftrace_test_recursion_unlock() were supposed to disable and enable preemption properly, however currently this work is done outside of the function, which could be missing by mistake.
And since the internal using of trace_test_and_set_recursion() and trace_clear_recursion() also require preemption disabled, we can just merge the logical.
This patch will make sure the preemption has been disabled when trace_test_and_set_recursion() return bit >= 0, and trace_clear_recursion() will enable the preemption if previously enabled.
--- arch/csky/kernel/probes/ftrace.c | 2 -- arch/parisc/kernel/ftrace.c | 2 -- arch/powerpc/kernel/kprobes-ftrace.c | 2 -- arch/riscv/kernel/probes/ftrace.c | 2 -- arch/x86/kernel/kprobes/ftrace.c | 2 -- include/linux/trace_recursion.h | 20 +++++++++++++++++++- kernel/livepatch/patch.c | 13 +++++++------ kernel/trace/ftrace.c | 15 +++++---------- kernel/trace/trace_functions.c | 5 ----- 9 files changed, 31 insertions(+), 32 deletions(-)
arch/csky/kernel/probes/ftrace.c +0 -2
@@ -17,7 +17,6 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,return ; regs =ftrace_get_regs (fregs); -preempt_disable_notrace (); p =get_kprobe ((kprobe_opcode_t *)ip);if (!p) { p =get_kprobe ((kprobe_opcode_t *)(ip -MCOUNT_INSN_SIZE ));
@@ -57,7 +56,6 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,__this_cpu_write (current_kprobe,NULL ); } out: -preempt_enable_notrace ();ftrace_test_recursion_unlock (bit); }NOKPROBE_SYMBOL (kprobe_ftrace_handler);
arch/parisc/kernel/ftrace.c +0 -2
@@ -216,7 +216,6 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,return ; regs =ftrace_get_regs (fregs); -preempt_disable_notrace (); p =get_kprobe ((kprobe_opcode_t *)ip);if (unlikely (!p) ||kprobe_disabled (p)) goto out;
@@ -245,7 +244,6 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, }__this_cpu_write (current_kprobe,NULL ); out: -preempt_enable_notrace ();ftrace_test_recursion_unlock (bit); }NOKPROBE_SYMBOL (kprobe_ftrace_handler);
arch/powerpc/kernel/kprobes-ftrace.c +0 -2
@@ -26,7 +26,6 @@ void kprobe_ftrace_handler(unsigned long nip, unsigned long parent_nip,return ; regs =ftrace_get_regs (fregs); -preempt_disable_notrace (); p =get_kprobe ((kprobe_opcode_t *)nip);if (unlikely (!p) ||kprobe_disabled (p)) goto out;
@@ -61,7 +60,6 @@ void kprobe_ftrace_handler(unsigned long nip, unsigned long parent_nip,__this_cpu_write (current_kprobe,NULL ); } out: -preempt_enable_notrace ();ftrace_test_recursion_unlock (bit); }NOKPROBE_SYMBOL (kprobe_ftrace_handler);
arch/riscv/kernel/probes/ftrace.c +0 -2
@@ -15,7 +15,6 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,if (bit <0 )return ; -preempt_disable_notrace (); p =get_kprobe ((kprobe_opcode_t *)ip);if (unlikely (!p) ||kprobe_disabled (p)) goto out;
@@ -52,7 +51,6 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,__this_cpu_write (current_kprobe,NULL ); } out: -preempt_enable_notrace ();ftrace_test_recursion_unlock (bit); }NOKPROBE_SYMBOL (kprobe_ftrace_handler);
arch/x86/kernel/kprobes/ftrace.c +0 -2
@@ -25,7 +25,6 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,if (bit <0 )return ; -preempt_disable_notrace (); p =get_kprobe ((kprobe_opcode_t *)ip);if (unlikely (!p) ||kprobe_disabled (p)) goto out;
@@ -59,7 +58,6 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,__this_cpu_write (current_kprobe,NULL ); } out: -preempt_enable_notrace ();ftrace_test_recursion_unlock (bit); }NOKPROBE_SYMBOL (kprobe_ftrace_handler);
include/linux/trace_recursion.h +19 -1
@@ -159,6 +159,9 @@ static __always_inline int trace_get_context_bit(void)# define do_ftrace_record_recursion (ip, pip) do { } while (0)#endif +/* +* Preemption is promised to be disabled when return bit >= 0. +*/ static __always_inline int trace_test_and_set_recursion(unsigned long ip, unsigned long pip,int start,int max) {
@@ -189,14 +192,29 @@ static __always_inline int trace_test_and_set_recursion(unsigned long ip, unsign current->trace_recursion = val;barrier (); +/* +* Disable preemption to fulfill the promise. +* +* Don't worry about the bit 0 cases, they indicate +* the disabling behaviour has already been done by +* internal call previously. +*/ +preempt_disable_notrace (); +return bit +1 ; } +/* +* Preemption will be enabled (if it was previously enabled). +*/ static __always_inline void trace_clear_recursion(int bit) {if (!bit)return ; +if (bit >0 ) +preempt_enable_notrace (); +barrier (); bit--;trace_recursion_clear (bit);
@@ -209,7 +227,7 @@ static __always_inline void trace_clear_recursion(int bit)* tracing recursed in the same context (normal vs interrupt), * * Returns: -1 if a recursion happened. -* >= 0 if no recursion +* >= 0 if no recursion. */ static __always_inline int ftrace_test_recursion_trylock(unsigned long ip,unsigned long parent_ip)
kernel/livepatch/patch.c +7 -6
@@ -49,14 +49,16 @@ static void notrace klp_ftrace_handler(unsigned long ip, ops = container_of(fops, struct klp_ops, fops); +/* +* +* The ftrace_test_recursion_trylock() will disable preemption, +* which is required for the variant of synchronize_rcu() that is +* used to allow patching functions where RCU is not watching. +* See klp_synchronize_transition() for more details. +*/ bit =ftrace_test_recursion_trylock (ip, parent_ip);if (WARN_ON_ONCE (bit <0 ))return ; -/* -* A variant of synchronize_rcu() is used to allow patching functions -* where RCU is not watching, see klp_synchronize_transition(). -*/ -preempt_disable_notrace (); func = list_first_or_null_rcu(&ops->func_stack, struct klp_func, stack_node);
@@ -120,7 +122,6 @@ static void notrace klp_ftrace_handler(unsigned long ip,klp_arch_set_pc (fregs, (unsigned long )func->new_func); unlock: -preempt_enable_notrace ();ftrace_test_recursion_unlock (bit); }
kernel/trace/ftrace.c +5 -10
@@ -6977,16 +6977,15 @@ void ftrace_reset_array_ops(struct trace_array *tr)struct ftrace_ops *op;int bit; +/* +* The ftrace_test_and_set_recursion() will disable preemption, +* which is required since some of the ops may be dynamically +* allocated, they must be freed after a synchronize_rcu(). +*/ bit =trace_test_and_set_recursion (ip, parent_ip,TRACE_LIST_START ,TRACE_LIST_MAX );if (bit <0 )return ; -/* -* Some of the ops may be dynamically allocated, -* they must be freed after a synchronize_rcu(). -*/ -preempt_disable_notrace (); -do_for_each_ftrace_op (op, ftrace_ops_list) {/* Stub functions don't need to be called nor tested */ if (op->flags &FTRACE_OPS_FL_STUB )
@@ -7010,7 +7009,6 @@ void ftrace_reset_array_ops(struct trace_array *tr) } }while_for_each_ftrace_op (op); out: -preempt_enable_notrace ();trace_clear_recursion (bit); }
@@ -7056,12 +7054,9 @@ static void ftrace_ops_assist_func(unsigned long ip, unsigned long parent_ip,if (bit <0 )return ; -preempt_disable_notrace (); -if (!(op->flags &FTRACE_OPS_FL_RCU ) ||rcu_is_watching ()) op->func (ip, parent_ip, op, fregs); -preempt_enable_notrace ();trace_clear_recursion (bit); }NOKPROBE_SYMBOL (ftrace_ops_assist_func);
kernel/trace/trace_functions.c +0 -5
@@ -186,7 +186,6 @@ static void function_trace_start(struct trace_array *tr)return ; trace_ctx =tracing_gen_ctx (); -preempt_disable_notrace (); cpu =smp_processor_id (); data =per_cpu_ptr (tr->array_buffer.data, cpu);
@@ -194,7 +193,6 @@ static void function_trace_start(struct trace_array *tr)trace_function (tr, ip, parent_ip, trace_ctx);ftrace_test_recursion_unlock (bit); -preempt_enable_notrace (); }#ifdef CONFIG_UNWINDER_ORC
@@ -298,8 +296,6 @@ static inline void process_repeats(struct trace_array *tr,if (bit <0 )return ; -preempt_disable_notrace (); - cpu =smp_processor_id (); data =per_cpu_ptr (tr->array_buffer.data, cpu);if (atomic_read (&data->disabled))
@@ -324,7 +320,6 @@ static inline void process_repeats(struct trace_array *tr, out:ftrace_test_recursion_unlock (bit); -preempt_enable_notrace (); } static void
Signature · 1 line
1.8.3.1
PATCH v4 2/2 ftrace: do CPU checking after preemption disabled
In reply to 王贇With CONFIG_DEBUG_PREEMPT we observed reports like:
BUG: using smp_processor_id() in preemptible caller is perf_ftrace_function_call+0x6f/0x2e0 CPU: 1 PID: 680 Comm: a.out Not tainted Call Trace: <TASK> dump_stack_lvl+0x8d/0xcf check_preemption_disabled+0x104/0x110 ? optimize_nops.isra.7+0x230/0x230 ? text_poke_bp_batch+0x9f/0x310 perf_ftrace_function_call+0x6f/0x2e0 ... __text_poke+0x5/0x620 text_poke_bp_batch+0x9f/0x310
This telling us the CPU could be changed after task is preempted, and the checking on CPU before preemption will be invalid.
Since now ftrace_test_recursion_trylock() will help to disable the preemption, this patch just do the checking after trylock() to address the issue.
--- kernel/trace/trace_event_perf.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
kernel/trace/trace_event_perf.c +3 -3
@@ -441,13 +441,13 @@ void perf_trace_buf_update(void *record, u16 type)if (!rcu_is_watching ())return ; -if ((unsigned long )ops->private !=smp_processor_id ()) -return ; - bit =ftrace_test_recursion_trylock (ip, parent_ip);if (bit <0 )return ; +if ((unsigned long )ops->private !=smp_processor_id ()) + goto out; + event = container_of(ops, struct perf_event, ftrace_ops);/*
Signature · 1 line
1.8.3.1
Just a ping, to see if there are any more comments :-P
Regards,
Michael Wang
Quoted 35 lines · 王贇
The testing show that
perf_ftrace_function_call()are usingsmp_processor_id()with preemption enabled, all the checking on CPU could be wrong after preemption.As Peter point out, the section between ftrace_test_recursion_trylock/unlock() pair require the preemption to be disabled as '
Documentation/trace/ftrace-uses.rst' explained, but currently the work is done outside of the helpers.And since the internal using of
trace_test_and_set_recursion()andtrace_clear_recursion()also require preemption to be disabled, we can just merge the logical together.Patch 1/2 will make sure preemption disabled when recursion lock succeed, patch 2/2 will do
smp_processor_id()checking aftertrylock()to address the issue.v1: https://lore.kernel.org/all/[email protected]/
v2: https://lore.kernel.org/all/[email protected]/
v3: https://lore.kernel.org/all/[email protected]/Michael Wang (2):
ftrace: disable preemption when recursion locked
ftrace: do CPU checking after preemption disabledarch/csky/kernel/probes/ftrace.c | 2 -- arch/parisc/kernel/ftrace.c | 2 -- arch/powerpc/kernel/kprobes-ftrace.c | 2 -- arch/riscv/kernel/probes/ftrace.c | 2 -- arch/x86/kernel/kprobes/ftrace.c | 2 -- include/linux/trace_recursion.h | 20 +++++++++++++++++++- kernel/livepatch/patch.c | 13 +++++++------ kernel/trace/ftrace.c | 15 +++++---------- kernel/trace/trace_event_perf.c | 6 +++--- kernel/trace/trace_functions.c | 5 ----- 10 files changed, 34 insertions(+), 35 deletions(-)
Steven Rostedt
maintainerQuoted 1 line · On Tue, 26 Oct 2021 10:09:12 +0800 王贇 <[email protected]> wrote:
Just a ping, to see if there are any more comments :-P
I guess you missed what went into mainline (and your name found a bug in my perl script for importing patches ;-)
https://lore.kernel.org/all/[email protected]/
Which means patch 1 needs to change:
Quoted 13 lines
(quoted diff) +10
-0+ /* + * Disable preemption to fulfill the promise. + * + * Don't worry about the bit 0 cases, they indicate + * the disabling behaviour has already been done by + * internal call previously. + */ + preempt_disable_notrace(); + return bit + 1; } +/*
Review
Quoted 10 lines
(quoted diff) +5
-0+ * Preemption will be enabled (if it was previously enabled). + */ static __always_inline void trace_clear_recursion(int bit) { if (!bit) return; + if (bit > 0) + preempt_enable_notrace(); +
Where this wont work anymore.
Need to preempt disable and enable always.
-- Steve
Quoted 6 lines
(quoted diff) +0
-0barrier(); bit--; trace_recursion_clear(bit);@@ -209,7 +227,7 @@ static __always_inline void trace_clear_recursion(int bit) * tracing recursed in the same context (normal vs interrupt), *
Quoted 9 lines · Steven Rostedt
Quoted 1 line · On Tue, 26 Oct 2021 10:09:12 +0800 王贇 <[email protected]> wrote:
Just a ping, to see if there are any more comments :-P
I guess you missed what went into mainline (and your name found a bug in my perl script for importing patches ;-)
Cool~ Missing some chinese font maybe, that's fine :-)
Quoted 22 lines
Which means patch 1 needs to change:
Quoted 20 lines
(quoted diff) +12
-0+ /* + * Disable preemption to fulfill the promise. + * + * Don't worry about the bit 0 cases, they indicate + * the disabling behaviour has already been done by + * internal call previously. + */ + preempt_disable_notrace(); + return bit + 1; } +/* + * Preemption will be enabled (if it was previously enabled). + */ static __always_inline void trace_clear_recursion(int bit) { if (!bit) return;
Quoted 7 lines
Review
Quoted 3 lines
(quoted diff) +3
-0+ if (bit > 0) + preempt_enable_notrace(); +Where this wont work anymore.
Need to preempt disable and enable always.
Yup, will rebase on the latest changes~
Regards,
Michael Wang
Quoted 10 lines
-- Steve
Quoted 6 lines
(quoted diff) +0
-0barrier(); bit--; trace_recursion_clear(bit);@@ -209,7 +227,7 @@ static __always_inline void trace_clear_recursion(int bit) * tracing recursed in the same context (normal vs interrupt), *