tracing: Updates for v6.20
Subsystems (5)
cover tracing: Updates for v6.20
git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git
trace/for-next
Head SHA1: a46023d5616ed3ed781e56ca93400eb9490e3646
Paul E. McKenney (1):
srcu: Fix warning to permit SRCU-fast readers in NMI handlers
Steven Rostedt (3):
tracing: perf: Have perf tracepoint callbacks always disable preemption
bpf: Have __bpf_trace_run() use rcu_read_lock_dont_migrate()
tracing: Guard __DECLARE_TRACE() use of __DO_TRACE_CALL() with SRCU-fast
----
include/linux/tracepoint.h | 9 +++++---- include/trace/perf.h | 4 ++-- include/trace/trace_events.h | 4 ++-- kernel/rcu/srcutree.c | 3 ++- kernel/trace/bpf_trace.c | 5 ++--- kernel/tracepoint.c | 18 ++++++++++++++---- 6 files changed, 27 insertions(+), 16 deletions(-)
PATCH 1/4 tracing: perf: Have perf tracepoint callbacks always disable preemption
From: Steven Rostedt <[email protected]>
In preparation to convert protection of tracepoints from being protected by a preempt disabled section to being protected by SRCU, have all the perf callbacks disable preemption as perf expects preemption to be disabled when processing tracepoints.
While at it, convert the perf system call callback preempt_disable() to a guard(preempt).
--- include/trace/perf.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
include/trace/perf.h +2 -2
@@ -71,6 +71,7 @@ perf_trace_##call(void *__data, proto) \ u64 __count __attribute__((unused)); \ struct task_struct *__task __attribute__((unused)); \ \ +guard (preempt_notrace)(); \ do_perf_trace_##call(__data, args); \ }
@@ -85,9 +86,8 @@ perf_trace_##call(void *__data, proto) \ struct task_struct *__task __attribute__((unused)); \ \might_fault (); \ -preempt_disable_notrace (); \ +guard (preempt_notrace)(); \ do_perf_trace_##call(__data, args); \ - preempt_enable_notrace(); \ }/*
Signature · 1 line
2.51.0
PATCH 2/4 bpf: Have __bpf_trace_run() use rcu_read_lock_dont_migrate()
In reply to Steven RostedtFrom: Steven Rostedt <[email protected]>
In order to switch the protection of tracepoint callbacks from preempt_disable() to srcu_read_lock_fast() the BPF callback from tracepoints needs to have migration prevention as the BPF programs expect to stay on the same CPU as they execute. Put together the RCU protection with migration prevention and use rcu_read_lock_dont_migrate() in __bpf_trace_run(). This will allow tracepoints callbacks to be preemptible.
--- kernel/trace/bpf_trace.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-)
kernel/trace/bpf_trace.c +2 -3
@@ -2062,7 +2062,7 @@ void __bpf_trace_run(struct bpf_raw_tp_link *link, u64 *args)struct bpf_run_ctx *old_run_ctx;struct bpf_trace_run_ctx run_ctx; -cant_sleep (); +rcu_read_lock_dont_migrate ();if (unlikely (this_cpu_inc_return (*(prog->active)) !=1 )) {bpf_prog_inc_misses_counter (prog); goto out;
@@ -2071,13 +2071,12 @@ void __bpf_trace_run(struct bpf_raw_tp_link *link, u64 *args) run_ctx.bpf_cookie = link->cookie; old_run_ctx =bpf_set_run_ctx (&run_ctx.run_ctx); -rcu_read_lock (); (void )bpf_prog_run (prog, args); -rcu_read_unlock ();bpf_reset_run_ctx (old_run_ctx); out:this_cpu_dec (*(prog->active)); +rcu_read_unlock_migrate (); }#define UNPACK (...) __VA_ARGS__
Signature · 1 line
2.51.0
PATCH 3/4 srcu: Fix warning to permit SRCU-fast readers in NMI handlers
In reply to Steven RostedtFrom: "Paul E. McKenney" <[email protected]>
SRCU-fast is designed to be used in NMI handlers, even going so far as to use atomic operations for architectures supporting NMIs but not providing NMI-safe per-CPU atomic operations. However, the WARN_ON_ONCE() in __srcu_check_read_flavor() complains if SRCU-fast is used in an NMI handler. This commit therefore modifies that WARN_ON_ONCE() to avoid such complaints.
--- kernel/rcu/srcutree.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
kernel/rcu/srcutree.c +2 -1
@@ -789,7 +789,8 @@ void __srcu_check_read_flavor(struct srcu_struct *ssp, int read_flavor)struct srcu_data *sdp;/* NMI-unsafe use in NMI is a bad sign, as is multi-bit read_flavor values. */ -WARN_ON_ONCE ((read_flavor !=SRCU_READ_FLAVOR_NMI ) &&in_nmi ()); +WARN_ON_ONCE (read_flavor !=SRCU_READ_FLAVOR_NMI && + read_flavor !=SRCU_READ_FLAVOR_FAST &&in_nmi ());WARN_ON_ONCE (read_flavor & (read_flavor -1 )); sdp =raw_cpu_ptr (ssp->sda);
Signature · 1 line
2.51.0
Re: [for-next][PATCH 3/4] srcu: Fix warning to permit SRCU-fast readers in NMI handlers
Quoted 9 lines · Steven Rostedt
From: "Paul E. McKenney" <[email protected]>
SRCU-fast is designed to be used in NMI handlers, even going so far as to use atomic operations for architectures supporting NMIs but not providing NMI-safe per-CPU atomic operations. However, the
WARN_ON_ONCE()in__srcu_check_read_flavor()complains if SRCU-fast is used in an NMI handler. This commit therefore modifies thatWARN_ON_ONCE()to avoid such complaints.
Quoted 10 lines
Reported-by Steven Rostedt <[email protected]>Signed-off-by Paul E. McKenney <[email protected]>Tested-by Steven Rostedt <[email protected]>Cc Andrii Nakryiko <[email protected]>Cc Boqun Feng <[email protected]>Cc Alexei Starovoitov <[email protected]>Cc Peter Zijlstra <[email protected]>Signed-off-by Steven Rostedt (Google) <[email protected]>
Thank you, Steven! I will drop this in the RCU PR since you already plan to carry it via tracing.
Regards,
Boqun
Quoted 22 lines
--- kernel/rcu/srcutree.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)kernel/rcu/srcutree.c +2
-1@@ -789,7 +789,8 @@ void __srcu_check_read_flavor(struct srcu_struct *ssp, int read_flavor)struct srcu_data *sdp;/* NMI-unsafe use in NMI is a bad sign, as is multi-bit read_flavor values. */ -WARN_ON_ONCE ((read_flavor !=SRCU_READ_FLAVOR_NMI ) &&in_nmi ()); +WARN_ON_ONCE (read_flavor !=SRCU_READ_FLAVOR_NMI && + read_flavor !=SRCU_READ_FLAVOR_FAST &&in_nmi ());WARN_ON_ONCE (read_flavor & (read_flavor -1 )); sdp =raw_cpu_ptr (ssp->sda);Signature · 1 line
2.51.0
Paul E. McKenney
maintainerRe: [for-next][PATCH 3/4] srcu: Fix warning to permit SRCU-fast readers in NMI handlers
Quoted 21 lines · Boqun Feng
Quoted 9 lines · Steven Rostedt
From: "Paul E. McKenney" <[email protected]>
SRCU-fast is designed to be used in NMI handlers, even going so far as to use atomic operations for architectures supporting NMIs but not providing NMI-safe per-CPU atomic operations. However, the
WARN_ON_ONCE()in__srcu_check_read_flavor()complains if SRCU-fast is used in an NMI handler. This commit therefore modifies thatWARN_ON_ONCE()to avoid such complaints.Quoted 10 lines
Reported-by Steven Rostedt <[email protected]>Signed-off-by Paul E. McKenney <[email protected]>Tested-by Steven Rostedt <[email protected]>Cc Andrii Nakryiko <[email protected]>Cc Boqun Feng <[email protected]>Cc Alexei Starovoitov <[email protected]>Cc Peter Zijlstra <[email protected]>Signed-off-by Steven Rostedt (Google) <[email protected]>
Quoted 2 lines
Thank you, Steven! I will drop this in the RCU PR since you already plan to carry it via tracing.
Thank you both!
Thanx, Paul
Quoted 25 lines
Regards,
BoqunQuoted 22 lines
--- kernel/rcu/srcutree.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)kernel/rcu/srcutree.c +2
-1@@ -789,7 +789,8 @@ void __srcu_check_read_flavor(struct srcu_struct *ssp, int read_flavor)struct srcu_data *sdp;/* NMI-unsafe use in NMI is a bad sign, as is multi-bit read_flavor values. */ -WARN_ON_ONCE ((read_flavor !=SRCU_READ_FLAVOR_NMI ) &&in_nmi ()); +WARN_ON_ONCE (read_flavor !=SRCU_READ_FLAVOR_NMI && + read_flavor !=SRCU_READ_FLAVOR_FAST &&in_nmi ());WARN_ON_ONCE (read_flavor & (read_flavor -1 )); sdp =raw_cpu_ptr (ssp->sda);Signature · 1 line
2.51.0
Re: [for-next][PATCH 3/4] srcu: Fix warning to permit SRCU-fast readers in NMI handlers
In reply to Boqun FengQuoted 2 lines · On Fri, 30 Jan 2026 07:51:16 -0800 Boqun Feng <[email protected]> wrote:
Thank you, Steven! I will drop this in the RCU PR since you already plan to carry it via tracing.
Thanks. It was a needed dependency for the last patch.
-- Steve
PATCH 4/4 tracing: Guard __DECLARE_TRACE() use of __DO_TRACE_CALL() with SRCU-fast
In reply to Steven RostedtFrom: Steven Rostedt <[email protected]>
The current use of guard(preempt_notrace)() within __DECLARE_TRACE() to protect invocation of __DO_TRACE_CALL() means that BPF programs attached to tracepoints are non-preemptible. This is unhelpful in real-time systems, whose users apparently wish to use BPF while also achieving low latencies. (Who knew?)
One option would be to use preemptible RCU, but this introduces many opportunities for infinite recursion, which many consider to be counterproductive, especially given the relatively small stacks provided by the Linux kernel. These opportunities could be shut down by sufficiently energetic duplication of code, but this sort of thing is considered impolite in some circles.
Therefore, use the shiny new SRCU-fast API, which provides somewhat faster readers than those of preemptible RCU, at least on Paul E. McKenney's laptop, where task_struct access is more expensive than access to per-CPU variables. And SRCU-fast provides way faster readers than does SRCU, courtesy of being able to avoid the read-side use of smp_mb(). Also, it is quite straightforward to create srcu_read_{,un}lock_fast_notrace() functions.
--- include/linux/tracepoint.h | 9 +++++---- include/trace/trace_events.h | 4 ++-- kernel/tracepoint.c | 18 ++++++++++++++---- 3 files changed, 21 insertions(+), 10 deletions(-)
include/linux/tracepoint.h +5 -4
@@ -108,14 +108,15 @@ void for_each_tracepoint_in_module(struct module *mod,* An alternative is to use the following for batch reclaim associated * with a given tracepoint: * -* - tracepoint_is_faultable() == false: call_rcu() +* - tracepoint_is_faultable() == false: call_srcu() * - tracepoint_is_faultable() == true: call_rcu_tasks_trace() */ #ifdef CONFIG_TRACEPOINTS +extern struct srcu_struct tracepoint_srcu;static inline void tracepoint_synchronize_unregister (void ) {synchronize_rcu_tasks_trace (); -synchronize_rcu (); +synchronize_srcu (&tracepoint_srcu); }static inline bool tracepoint_is_faultable (struct tracepoint *tp) {
@@ -275,13 +276,13 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p) return static_branch_unlikely(&__tracepoint_##name.key);\ } -#define __DECLARE_TRACE (name, proto, args, cond, data_proto) \ +#define __DECLARE_TRACE (name, proto, args, cond, data_proto) \ __DECLARE_TRACE_COMMON(name, PARAMS(proto), PARAMS(args), PARAMS(data_proto)) \ static inline void __do_trace_##name(proto) \ { \ TRACEPOINT_CHECK(name) \ if (cond) { \ - guard(preempt_notrace)(); \ + guard(srcu_fast_notrace)(&tracepoint_srcu); \ __DO_TRACE_CALL(name, TP_ARGS(args)); \ } \ } \
include/trace/trace_events.h +2 -2
@@ -436,6 +436,7 @@ __DECLARE_EVENT_CLASS(call, PARAMS(proto), PARAMS(args), PARAMS(tstruct), \ static notrace void \ trace_event_raw_event_##call(void *__data, proto) \ { \ + guard(preempt_notrace)(); \ do_trace_event_raw_event_##call(__data, args); \ }
@@ -447,9 +448,8 @@ static notrace void \ trace_event_raw_event_##call(void *__data, proto) \ { \ might_fault(); \ - preempt_disable_notrace(); \ + guard(preempt_notrace)(); \ do_trace_event_raw_event_##call(__data, args); \ - preempt_enable_notrace(); \ }/*
kernel/tracepoint.c +14 -4
@@ -34,9 +34,13 @@ enum tp_transition_sync {struct tp_transition_snapshot {unsigned long rcu; +unsigned long srcu_gp;bool ongoing; }; +DEFINE_SRCU_FAST (tracepoint_srcu); +EXPORT_SYMBOL_GPL (tracepoint_srcu); +/* Protected by tracepoints_mutex */ static struct tp_transition_snapshot tp_transition_snapshot[_NR_TP_TRANSITION_SYNC];
@@ -46,6 +50,7 @@ static void tp_rcu_get_state(enum tp_transition_sync sync)/* Keep the latest get_state snapshot. */ snapshot->rcu =get_state_synchronize_rcu (); + snapshot->srcu_gp =start_poll_synchronize_srcu (&tracepoint_srcu); snapshot->ongoing = true; }
@@ -56,6 +61,8 @@ static void tp_rcu_cond_sync(enum tp_transition_sync sync)if (!snapshot->ongoing)return ;cond_synchronize_rcu (snapshot->rcu); +if (!poll_state_synchronize_srcu (&tracepoint_srcu, snapshot->srcu_gp)) +synchronize_srcu (&tracepoint_srcu); snapshot->ongoing = false; }
@@ -112,10 +119,13 @@ static inline void release_probes(struct tracepoint *tp, struct tracepoint_funcstruct tp_probes *tp_probes =container_of (old, struct tp_probes, probes[0]); -if (tracepoint_is_faultable (tp)) -call_rcu_tasks_trace (&tp_probes->rcu, rcu_free_old_probes); -else -call_rcu (&tp_probes->rcu, rcu_free_old_probes); +if (tracepoint_is_faultable (tp)) { +call_rcu_tasks_trace (&tp_probes->rcu, + rcu_free_old_probes); + }else { +call_srcu (&tracepoint_srcu, &tp_probes->rcu, + rcu_free_old_probes); + } } }
Signature · 1 line
2.51.0