tracing: Protect the buffer from recursion in perf
PATCH 1/1 tracing: Protect the buffer from recursion in perf
While tracing using events with perf, if one enables the lockdep:lock_acquire event, it will infect every other perf trace events.
Basically, you can enable whatever set of trace events through perf but if this event is part of the set, the only result we can get is a long list of lock_acquire events of rcu read lock, and only that.
This is because of a recursion inside perf.
1) When a trace event is triggered, it will fill a per cpu buffer and submit it to perf.
2) Perf will commit this event but will also protect some data using rcu_read_lock
3) A recursion appears: rcu_read_lock triggers a lock_acquire event that will fill the per cpu event and then submit the buffer to perf.
4) Perf detects a recursion and ignore it
5) Perf continue its work on the previous event, but its buffer has been overwritten by the lock_acquire event, it has then been turned into a lock_acquire event of rcu read lock
Such scenario also happens with lock_release with rcu_read_unlock().
We could turn the rcu_read_lock() into __rcu_read_lock() to drop the lock debugging from perf fast path, but that would make us lose the rcu debugging and that doesn't prevent from other possible kind of recursion from perf in the future.
This patch adds a recursion protection on the perf trace per cpu buffers to solve the problem.
--- include/linux/ftrace_event.h | 9 +++++- include/trace/ftrace.h | 39 +++++++++++++++++++++------ kernel/trace/trace_event_profile.c | 41 +++++++++++++---------------- kernel/trace/trace_kprobe.c | 50 ++++++++++++++++++++++++++++++------ kernel/trace/trace_syscalls.c | 44 ++++++++++++++++++++++++++------ 5 files changed, 133 insertions(+), 50 deletions(-)
include/linux/ftrace_event.h +7 -2
@@ -137,8 +137,13 @@ struct ftrace_event_call {#define FTRACE_MAX_PROFILE_SIZE 2048 -extern char *trace_profile_buf; -extern char *trace_profile_buf_nmi; +struct perf_trace_buf { +char buf[FTRACE_MAX_PROFILE_SIZE ]; +int recursion; +}; + +extern struct perf_trace_buf *perf_trace_buf; +extern struct perf_trace_buf *perf_trace_buf_nmi;#define MAX_FILTER_PRED 32#define MAX_FILTER_STR_VAL 256/* Should handle KSYM_SYMBOL_LEN */
include/trace/ftrace.h +30 -9
@@ -649,6 +649,7 @@ __attribute__((section("_ftrace_events"))) event_##call = { \* struct ftrace_event_call *event_call = &event_<call>; * extern void perf_tp_event(int, u64, u64, void *, int); * struct ftrace_raw_##call *entry; +* struct perf_trace_buf *trace_buf; * u64 __addr = 0, __count = 1; * unsigned long irq_flags; * struct trace_entry *ent;
@@ -673,14 +674,25 @@ __attribute__((section("_ftrace_events"))) event_##call = { \* __cpu = smp_processor_id(); * * if (in_nmi()) -* raw_data = rcu_dereference(trace_profile_buf_nmi); +* trace_buf = rcu_dereference(perf_trace_buf_nmi); * else -* raw_data = rcu_dereference(trace_profile_buf); +* trace_buf = rcu_dereference(perf_trace_buf); * -* if (!raw_data) +* if (!trace_buf) * goto end; * -* raw_data = per_cpu_ptr(raw_data, __cpu); +* trace_buf = per_cpu_ptr(trace_buf, __cpu); +* +* // Avoid recursion from perf that could mess up the buffer +* if (trace_buf->recursion++) +* goto end_recursion; +* +* raw_data = trace_buf->buf; +* +* // Make recursion update visible before entering perf_tp_event +* // so that we protect from perf recursions. +* +* barrier(); * * //zero dead bytes from alignment to avoid stack leak to userspace: * *(u64 *)(&raw_data[__entry_size - sizeof(u64)]) = 0ULL;
@@ -713,8 +725,9 @@ static void ftrace_profile_##call(proto) \ { \ struct ftrace_data_offsets_##call __maybe_unused __data_offsets;\ struct ftrace_event_call *event_call = &event_##call; \ - extern void perf_tp_event(int, u64, u64, void *, int); \ + extern void perf_tp_event(int, u64, u64, void *, int); \ struct ftrace_raw_##call *entry; \ + struct perf_trace_buf *trace_buf; \ u64 __addr = 0, __count = 1; \ unsigned long irq_flags; \ struct trace_entry *ent; \
@@ -739,14 +752,20 @@ static void ftrace_profile_##call(proto) \ __cpu =smp_processor_id (); \ \if (in_nmi ()) \ - raw_data =rcu_dereference (trace_profile_buf_nmi); \ + trace_buf =rcu_dereference (perf_trace_buf_nmi); \else \ - raw_data =rcu_dereference (trace_profile_buf); \ + trace_buf =rcu_dereference (perf_trace_buf); \ \ -if (!raw_data) \ +if (!trace_buf) \ goto end; \ \ - raw_data =per_cpu_ptr (raw_data, __cpu); \ + trace_buf =per_cpu_ptr (trace_buf, __cpu); \ +if (trace_buf->recursion++) \ + goto end_recursion; \ + \ +barrier (); \ + \ + raw_data = trace_buf->buf; \ \ *(u64 *)(&raw_data[__entry_size -sizeof (u64)]) =0ULL ; \ entry = (struct ftrace_raw_##call *)raw_data; \
@@ -761,6 +780,8 @@ static void ftrace_profile_##call(proto) \perf_tp_event (event_call->id, __addr, __count, entry, \ __entry_size); \ \ +end_recursion: \ + trace_buf->recursion--; \ end: \local_irq_restore (irq_flags); \ \
kernel/trace/trace_event_profile.c +18 -23
87 lines not on this page · raw message
kernel/trace/trace_kprobe.c +42 -8
98 lines not on this page · raw message
kernel/trace/trace_syscalls.c +36 -8
88 lines not on this page · raw message
Signature · 1 line
1.6.2.3
Quoted 23 lines · Frederic Weisbecker
While tracing using events with perf, if one enables the lockdep:lock_acquire event, it will infect every other perf trace events.
Basically, you can enable whatever set of trace events through perf but if this event is part of the set, the only result we can get is a long list of
lock_acquireevents of rcu read lock, and only that.This is because of a recursion inside perf.
1) When a trace event is triggered, it will fill a per cpu buffer and submit it to perf.
2) Perf will commit this event but will also protect some data usingrcu_read_lock
3) A recursion appears:rcu_read_locktriggers alock_acquireevent that will fill the per cpu event and then submit the buffer to perf.
4) Perf detects a recursion and ignore it
5) Perf continue its work on the previous event, but its buffer has been overwritten by thelock_acquireevent, it has then been turned into alock_acquireevent of rcu read lockSuch scenario also happens with
lock_releasewithrcu_read_unlock().
Quoted 7 lines
We could turn the
rcu_read_lock()into__rcu_read_lock()to drop the lock debugging from perf fast path, but that would make us lose the rcu debugging and that doesn't prevent from other possible kind of recursion from perf in the future.This patch adds a recursion protection on the perf trace per cpu buffers to solve the problem.
OK, so you added a counter to find recursion.
I found a style issue, see below.
include/trace/ftrace.h __attribute__((section("_ftrace_events"))) event_##call = { \ · line 652
Quoted 9 lines
include/trace/ftrace.h +1
-0@@ -649,6 +649,7 @@ __attribute__((section("_ftrace_events"))) event_##call = { \* struct ftrace_event_call *event_call =&event_<call>; * extern void perf_tp_event(int, u64, u64, void *, int); * struct ftrace_raw_##call *entry; +* struct perf_trace_buf *trace_buf;
^^ Here, we need a space :-)
Others looks good to me, Thanks!
Signature · 7 lines
Masami Hiramatsu Software Engineer Hitachi Computer Products (America), Inc. Software Solutions Division e-mail: [email protected]
PATCH v2 1/1 tracing: Protect the buffer from recursion in perf
While tracing using events with perf, if one enables the lockdep:lock_acquire event, it will infect every other perf trace events.
Basically, you can enable whatever set of trace events through perf but if this event is part of the set, the only result we can get is a long list of lock_acquire events of rcu read lock, and only that.
This is because of a recursion inside perf.
1) When a trace event is triggered, it will fill a per cpu buffer and submit it to perf.
2) Perf will commit this event but will also protect some data using rcu_read_lock
3) A recursion appears: rcu_read_lock triggers a lock_acquire event that will fill the per cpu event and then submit the buffer to perf.
4) Perf detects a recursion and ignore it
5) Perf continue its work on the previous event, but its buffer has been overwritten by the lock_acquire event, it has then been turned into a lock_acquire event of rcu read lock
Such scenario also happens with lock_release with rcu_read_unlock().
We could turn the rcu_read_lock() into __rcu_read_lock() to drop the lock debugging from perf fast path, but that would make us lose the rcu debugging and that doesn't prevent from other possible kind of recursion from perf in the future.
This patch adds a recursion protection based on a counter on the perf trace per cpu buffers to solve the problem.
v2: Fixed lost whitespace, added reviewed-by tag
--- include/linux/ftrace_event.h | 9 +++++- include/trace/ftrace.h | 39 +++++++++++++++++++++------ kernel/trace/trace_event_profile.c | 41 +++++++++++++---------------- kernel/trace/trace_kprobe.c | 50 ++++++++++++++++++++++++++++++------ kernel/trace/trace_syscalls.c | 44 ++++++++++++++++++++++++++------ 5 files changed, 133 insertions(+), 50 deletions(-)
include/linux/ftrace_event.h +7 -2
@@ -137,8 +137,13 @@ struct ftrace_event_call {#define FTRACE_MAX_PROFILE_SIZE 2048 -extern char *trace_profile_buf; -extern char *trace_profile_buf_nmi; +struct perf_trace_buf { +char buf[FTRACE_MAX_PROFILE_SIZE ]; +int recursion; +}; + +extern struct perf_trace_buf *perf_trace_buf; +extern struct perf_trace_buf *perf_trace_buf_nmi;#define MAX_FILTER_PRED 32#define MAX_FILTER_STR_VAL 256/* Should handle KSYM_SYMBOL_LEN */
include/trace/ftrace.h +30 -9
@@ -649,6 +649,7 @@ __attribute__((section("_ftrace_events"))) event_##call = { \* struct ftrace_event_call *event_call = &event_<call>; * extern void perf_tp_event(int, u64, u64, void *, int); * struct ftrace_raw_##call *entry; +* struct perf_trace_buf *trace_buf; * u64 __addr = 0, __count = 1; * unsigned long irq_flags; * struct trace_entry *ent;
@@ -673,14 +674,25 @@ __attribute__((section("_ftrace_events"))) event_##call = { \* __cpu = smp_processor_id(); * * if (in_nmi()) -* raw_data = rcu_dereference(trace_profile_buf_nmi); +* trace_buf = rcu_dereference(perf_trace_buf_nmi); * else -* raw_data = rcu_dereference(trace_profile_buf); +* trace_buf = rcu_dereference(perf_trace_buf); * -* if (!raw_data) +* if (!trace_buf) * goto end; * -* raw_data = per_cpu_ptr(raw_data, __cpu); +* trace_buf = per_cpu_ptr(trace_buf, __cpu); +* +* // Avoid recursion from perf that could mess up the buffer +* if (trace_buf->recursion++) +* goto end_recursion; +* +* raw_data = trace_buf->buf; +* +* // Make recursion update visible before entering perf_tp_event +* // so that we protect from perf recursions. +* +* barrier(); * * //zero dead bytes from alignment to avoid stack leak to userspace: * *(u64 *)(&raw_data[__entry_size - sizeof(u64)]) = 0ULL;
@@ -713,8 +725,9 @@ static void ftrace_profile_##call(proto) \ { \ struct ftrace_data_offsets_##call __maybe_unused __data_offsets;\ struct ftrace_event_call *event_call = &event_##call; \ - extern void perf_tp_event(int, u64, u64, void *, int); \ + extern void perf_tp_event(int, u64, u64, void *, int); \ struct ftrace_raw_##call *entry; \ + struct perf_trace_buf *trace_buf; \ u64 __addr = 0, __count = 1; \ unsigned long irq_flags; \ struct trace_entry *ent; \
@@ -739,14 +752,20 @@ static void ftrace_profile_##call(proto) \ __cpu =smp_processor_id (); \ \if (in_nmi ()) \ - raw_data =rcu_dereference (trace_profile_buf_nmi); \ + trace_buf =rcu_dereference (perf_trace_buf_nmi); \else \ - raw_data =rcu_dereference (trace_profile_buf); \ + trace_buf =rcu_dereference (perf_trace_buf); \ \ -if (!raw_data) \ +if (!trace_buf) \ goto end; \ \ - raw_data =per_cpu_ptr (raw_data, __cpu); \ + trace_buf =per_cpu_ptr (trace_buf, __cpu); \ +if (trace_buf->recursion++) \ + goto end_recursion; \ + \ +barrier (); \ + \ + raw_data = trace_buf->buf; \ \ *(u64 *)(&raw_data[__entry_size -sizeof (u64)]) =0ULL ; \ entry = (struct ftrace_raw_##call *)raw_data; \
@@ -761,6 +780,8 @@ static void ftrace_profile_##call(proto) \perf_tp_event (event_call->id, __addr, __count, entry, \ __entry_size); \ \ +end_recursion: \ + trace_buf->recursion--; \ end: \local_irq_restore (irq_flags); \ \
kernel/trace/trace_event_profile.c +18 -23
87 lines not on this page · raw message
kernel/trace/trace_kprobe.c +42 -8
98 lines not on this page · raw message
kernel/trace/trace_syscalls.c +36 -8
88 lines not on this page · raw message
Signature · 1 line
1.6.2.3
[tip:perf/probes] tracing, perf_events: Protect the buffer from recursion in perf
Commit-ID: 444a2a3bcd6d5bed5c823136f68fcc93c0fe283f Gitweb: http://git.kernel.org/tip/444a2a3bcd6d5bed5c823136f68fcc93c0fe283f Author: Frederic Weisbecker <[email protected]> AuthorDate: Fri, 6 Nov 2009 04:13:05 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Sun, 8 Nov 2009 10:31:42 +0100
tracing, perf_events: Protect the buffer from recursion in perf
While tracing using events with perf, if one enables the lockdep:lock_acquire event, it will infect every other perf trace events.
Basically, you can enable whatever set of trace events through perf but if this event is part of the set, the only result we can get is a long list of lock_acquire events of rcu read lock, and only that.
This is because of a recursion inside perf.
1) When a trace event is triggered, it will fill a per cpu buffer and submit it to perf.
2) Perf will commit this event but will also protect some data using rcu_read_lock
3) A recursion appears: rcu_read_lock triggers a lock_acquire event that will fill the per cpu event and then submit the buffer to perf.
4) Perf detects a recursion and ignores it
5) Perf continues its work on the previous event, but its buffer has been overwritten by the lock_acquire event, it has then been turned into a lock_acquire event of rcu read lock
Such scenario also happens with lock_release withrcu_read_unlock().
We could turn the rcu_read_lock() into __rcu_read_lock() to drop the lock debugging from perf fast path, but that would make us lose the rcu debugging and that doesn't prevent from other possible kind of recursion from perf in the future.
This patch adds a recursion protection based on a counter on the perf trace per cpu buffers to solve the problem.
-v2: Fixed lost whitespace, added reviewed-by tag
LKML-Reference: <[email protected]>
--- include/linux/ftrace_event.h | 9 +++++- include/trace/ftrace.h | 39 +++++++++++++++++++++------ kernel/trace/trace_event_profile.c | 41 +++++++++++++---------------- kernel/trace/trace_kprobe.c | 50 ++++++++++++++++++++++++++++++------ kernel/trace/trace_syscalls.c | 44 ++++++++++++++++++++++++++------ 5 files changed, 133 insertions(+), 50 deletions(-)
include/linux/ftrace_event.h +7 -2
@@ -137,8 +137,13 @@ struct ftrace_event_call {#define FTRACE_MAX_PROFILE_SIZE 2048 -extern char *trace_profile_buf; -extern char *trace_profile_buf_nmi; +struct perf_trace_buf { +char buf[FTRACE_MAX_PROFILE_SIZE ]; +int recursion; +}; + +extern struct perf_trace_buf *perf_trace_buf; +extern struct perf_trace_buf *perf_trace_buf_nmi;#define MAX_FILTER_PRED 32#define MAX_FILTER_STR_VAL 256/* Should handle KSYM_SYMBOL_LEN */
include/trace/ftrace.h +30 -9
@@ -649,6 +649,7 @@ __attribute__((section("_ftrace_events"))) event_##call = { \* struct ftrace_event_call *event_call = &event_<call>; * extern void perf_tp_event(int, u64, u64, void *, int); * struct ftrace_raw_##call *entry; +* struct perf_trace_buf *trace_buf; * u64 __addr = 0, __count = 1; * unsigned long irq_flags; * struct trace_entry *ent;
@@ -673,14 +674,25 @@ __attribute__((section("_ftrace_events"))) event_##call = { \* __cpu = smp_processor_id(); * * if (in_nmi()) -* raw_data = rcu_dereference(trace_profile_buf_nmi); +* trace_buf = rcu_dereference(perf_trace_buf_nmi); * else -* raw_data = rcu_dereference(trace_profile_buf); +* trace_buf = rcu_dereference(perf_trace_buf); * -* if (!raw_data) +* if (!trace_buf) * goto end; * -* raw_data = per_cpu_ptr(raw_data, __cpu); +* trace_buf = per_cpu_ptr(trace_buf, __cpu); +* +* // Avoid recursion from perf that could mess up the buffer +* if (trace_buf->recursion++) +* goto end_recursion; +* +* raw_data = trace_buf->buf; +* +* // Make recursion update visible before entering perf_tp_event +* // so that we protect from perf recursions. +* +* barrier(); * * //zero dead bytes from alignment to avoid stack leak to userspace: * *(u64 *)(&raw_data[__entry_size - sizeof(u64)]) = 0ULL;
@@ -713,8 +725,9 @@ static void ftrace_profile_##call(proto) \ { \ struct ftrace_data_offsets_##call __maybe_unused __data_offsets;\ struct ftrace_event_call *event_call = &event_##call; \ - extern void perf_tp_event(int, u64, u64, void *, int); \ + extern void perf_tp_event(int, u64, u64, void *, int); \ struct ftrace_raw_##call *entry; \ + struct perf_trace_buf *trace_buf; \ u64 __addr = 0, __count = 1; \ unsigned long irq_flags; \ struct trace_entry *ent; \
@@ -739,14 +752,20 @@ static void ftrace_profile_##call(proto) \ __cpu =smp_processor_id (); \ \if (in_nmi ()) \ - raw_data =rcu_dereference (trace_profile_buf_nmi); \ + trace_buf =rcu_dereference (perf_trace_buf_nmi); \else \ - raw_data =rcu_dereference (trace_profile_buf); \ + trace_buf =rcu_dereference (perf_trace_buf); \ \ -if (!raw_data) \ +if (!trace_buf) \ goto end; \ \ - raw_data =per_cpu_ptr (raw_data, __cpu); \ + trace_buf =per_cpu_ptr (trace_buf, __cpu); \ +if (trace_buf->recursion++) \ + goto end_recursion; \ + \ +barrier (); \ + \ + raw_data = trace_buf->buf; \ \ *(u64 *)(&raw_data[__entry_size -sizeof (u64)]) =0ULL ; \ entry = (struct ftrace_raw_##call *)raw_data; \
@@ -761,6 +780,8 @@ static void ftrace_profile_##call(proto) \perf_tp_event (event_call->id, __addr, __count, entry, \ __entry_size); \ \ +end_recursion: \ + trace_buf->recursion--; \ end: \local_irq_restore (irq_flags); \ \
kernel/trace/trace_event_profile.c +18 -23
87 lines not on this page · raw message
kernel/trace/trace_kprobe.c +42 -8
98 lines not on this page · raw message
kernel/trace/trace_syscalls.c +36 -8
88 lines not on this page · raw message
Quoted 23 lines · Frederic Weisbecker
While tracing using events with perf, if one enables the lockdep:lock_acquire event, it will infect every other perf trace events.
Basically, you can enable whatever set of trace events through perf but if this event is part of the set, the only result we can get is a long list of
lock_acquireevents of rcu read lock, and only that.This is because of a recursion inside perf.
1) When a trace event is triggered, it will fill a per cpu buffer and submit it to perf.
2) Perf will commit this event but will also protect some data usingrcu_read_lock
3) A recursion appears:rcu_read_locktriggers alock_acquireevent that will fill the per cpu event and then submit the buffer to perf.
4) Perf detects a recursion and ignore it
5) Perf continue its work on the previous event, but its buffer has been overwritten by thelock_acquireevent, it has then been turned into alock_acquireevent of rcu read lockSuch scenario also happens with
lock_releasewithrcu_read_unlock().
Quoted 7 lines
We could turn the
rcu_read_lock()into__rcu_read_lock()to drop the lock debugging from perf fast path, but that would make us lose the rcu debugging and that doesn't prevent from other possible kind of recursion from perf in the future.This patch adds a recursion protection based on a counter on the perf trace per cpu buffers to solve the problem.
There already is recursion protection in kernel/perf_event.c:perf_swevent_recursion_context() and thereabouts. Could you not fix this by widening its scope?
Quoted 3 lines · Peter Zijlstra
There already is recursion protection in kernel/perf_event.c:perf_swevent_recursion_context() and thereabouts. Could you not fix this by widening its scope?
Hmm, indeed.
I could probably use perf_swevent_recursion_context() directly from the tracing fill path. And then split up a bit do_perf_sw_event() so that it continues to check the recursion for normal software events but not for trace events (for which we would already explicitly call perf_swevent_recursion_context()).
I'll try that, thanks!