Skip to contentSkip to conversation

tracing: Tracing event profiling updates

PATCH Superseded Frederic Weisbecker 4 participants 15 messages Sep 18, 2009 – Sep 19, 2009

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

cover tracing: Tracing event profiling updates

Ingo,

Please pull the latest tracing/core branch which can be found at:

git://git.kernel.org/pub/scm/linux/kernel/git/frederic/random-tracing.git
tracing/core

Thanks,

Frederic.

Frederic Weisbecker (2):
tracing: Factorize the events profile accounting
tracing: Allocate the ftrace event profile buffer dynamically

 include/linux/ftrace_event.h       |    9 ++-
 include/linux/syscalls.h           |   24 +++-----
 include/trace/ftrace.h             |  111 ++++++++++++++++++++---------------
 kernel/trace/trace_event_profile.c |   68 +++++++++++++++++++++-
 kernel/trace/trace_syscalls.c      |  103 +++++++++++++++++++++++++++------
 5 files changed, 228 insertions(+), 87 deletions(-)
#1 lore raw

PATCH 1/2 tracing: Factorize the events profile accounting

Factorize the events enabling accounting in a common tracing core helper. This reduces the size of the profile_enable() and profile_disable() callbacks for each trace events.

Cc Steven Rostedt <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Li Zefan <[email protected]>
Cc Jason Baron <[email protected]>
Cc Masami Hiramatsu <[email protected]>
Cc Ingo Molnar <[email protected]>
---
 include/linux/ftrace_event.h       |    4 ++--
 include/linux/syscalls.h           |   24 ++++++++----------------
 include/trace/ftrace.h             |   28 ++++++++--------------------
 kernel/trace/trace_event_profile.c |   20 ++++++++++++++++++--
 4 files changed, 36 insertions(+), 40 deletions(-)
include/linux/ftrace_event.h +2 -2
index bd099ba..bc103d7 100644
@@ -130,8 +130,8 @@ struct ftrace_event_call {
 	void			*data;
 
 	atomic_t		profile_count;
-	int			(*profile_enable)(struct ftrace_event_call *);
-	void			(*profile_disable)(struct ftrace_event_call *);
+	int			(*profile_enable)(void);
+	void			(*profile_disable)(void);
 };
 
 #define MAX_FILTER_PRED		32
include/linux/syscalls.h +8 -16
index a8e3782..7d9803c 100644
@@ -100,33 +100,25 @@ struct perf_counter_attr;
 
 #ifdef CONFIG_EVENT_PROFILE
 #define TRACE_SYS_ENTER_PROFILE(sname)					       \
-static int prof_sysenter_enable_##sname(struct ftrace_event_call *event_call)  \
+static int prof_sysenter_enable_##sname(void)				       \
 {									       \
-	int ret = 0;							       \
-	if (!atomic_inc_return(&event_enter_##sname.profile_count))	       \
-		ret = reg_prof_syscall_enter("sys"#sname);		       \
-	return ret;							       \
+	return reg_prof_syscall_enter("sys"#sname);			       \
 }									       \
 									       \
-static void prof_sysenter_disable_##sname(struct ftrace_event_call *event_call)\
+static void prof_sysenter_disable_##sname(void)				       \
 {									       \
-	if (atomic_add_negative(-1, &event_enter_##sname.profile_count))       \
-		unreg_prof_syscall_enter("sys"#sname);			       \
+	unreg_prof_syscall_enter("sys"#sname);				       \
 }
 
 #define TRACE_SYS_EXIT_PROFILE(sname)					       \
-static int prof_sysexit_enable_##sname(struct ftrace_event_call *event_call)   \
+static int prof_sysexit_enable_##sname(void)				       \
 {									       \
-	int ret = 0;							       \
-	if (!atomic_inc_return(&event_exit_##sname.profile_count))	       \
-		ret = reg_prof_syscall_exit("sys"#sname);		       \
-	return ret;							       \
+	return reg_prof_syscall_exit("sys"#sname);			       \
 }									       \
 									       \
-static void prof_sysexit_disable_##sname(struct ftrace_event_call *event_call) \
+static void prof_sysexit_disable_##sname(void)				       \
 {                                                                              \
-	if (atomic_add_negative(-1, &event_exit_##sname.profile_count))	       \
-		unreg_prof_syscall_exit("sys"#sname);			       \
+	unreg_prof_syscall_exit("sys"#sname);				       \
 }
 
 #define TRACE_SYS_ENTER_PROFILE_INIT(sname)				       \
include/trace/ftrace.h +8 -20
index 72a3b43..a822087 100644
@@ -382,20 +382,14 @@ static inline int ftrace_get_offsets_##call(				\
  *
  * NOTE: The insertion profile callback (ftrace_profile_<call>) is defined later
  *
- * static int ftrace_profile_enable_<call>(struct ftrace_event_call *event_call)
+ * static int ftrace_profile_enable_<call>(void)
  * {
- * 	int ret = 0;
- *
- * 	if (!atomic_inc_return(&event_call->profile_count))
- * 		ret = register_trace_<call>(ftrace_profile_<call>);
- *
- * 	return ret;
+ * 	return register_trace_<call>(ftrace_profile_<call>);
  * }
  *
- * static void ftrace_profile_disable_<call>(struct ftrace_event_call *event_call)
+ * static void ftrace_profile_disable_<call>(void)
  * {
- * 	if (atomic_add_negative(-1, &event->call->profile_count))
- * 		unregister_trace_<call>(ftrace_profile_<call>);
+ * 	unregister_trace_<call>(ftrace_profile_<call>);
  * }
  *
  */
@@ -405,20 +399,14 @@ static inline int ftrace_get_offsets_##call(				\
 									\
 static void ftrace_profile_##call(proto);				\
 									\
-static int ftrace_profile_enable_##call(struct ftrace_event_call *event_call) \
+static int ftrace_profile_enable_##call(void)				\
 {									\
-	int ret = 0;							\
-									\
-	if (!atomic_inc_return(&event_call->profile_count))		\
-		ret = register_trace_##call(ftrace_profile_##call);	\
-									\
-	return ret;							\
+	return register_trace_##call(ftrace_profile_##call);		\
 }									\
 									\
-static void ftrace_profile_disable_##call(struct ftrace_event_call *event_call)\
+static void ftrace_profile_disable_##call(void)				\
 {									\
-	if (atomic_add_negative(-1, &event_call->profile_count))	\
-		unregister_trace_##call(ftrace_profile_##call);		\
+	unregister_trace_##call(ftrace_profile_##call);			\
 }
 
 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
kernel/trace/trace_event_profile.c +18 -2
index 55a25c9..df4a74e 100644
@@ -8,6 +8,14 @@
 #include <linux/module.h>
 #include "trace.h"
 
+static int ftrace_profile_enable_event(struct ftrace_event_call *event)
+{
+	if (atomic_inc_return(&event->profile_count))
+		return 0;
+
+	return event->profile_enable();
+}
+
 int ftrace_profile_enable(int event_id)
 {
 	struct ftrace_event_call *event;
@@ -17,7 +25,7 @@ int ftrace_profile_enable(int event_id)
 	list_for_each_entry(event, &ftrace_events, list) {
 		if (event->id == event_id && event->profile_enable &&
 		    try_module_get(event->mod)) {
-			ret = event->profile_enable(event);
+			ret = ftrace_profile_enable_event(event);
 			break;
 		}
 	}
@@ -26,6 +34,14 @@ int ftrace_profile_enable(int event_id)
 	return ret;
 }
 
+static void ftrace_profile_disable_event(struct ftrace_event_call *event)
+{
+	if (!atomic_add_negative(-1, &event->profile_count))
+		return;
+
+	event->profile_disable();
+}
+
 void ftrace_profile_disable(int event_id)
 {
 	struct ftrace_event_call *event;
@@ -33,7 +49,7 @@ void ftrace_profile_disable(int event_id)
 	mutex_lock(&event_mutex);
 	list_for_each_entry(event, &ftrace_events, list) {
 		if (event->id == event_id) {
-			event->profile_disable(event);
+			ftrace_profile_disable_event(event);
 			module_put(event->mod);
 			break;
 		}
Signature · 1 line
1.6.2.3
maintainer Acked-by on v1 1/2
#2 lore raw

Re: [PATCH 1/2] tracing: Factorize the events profile accounting

Quoted 5 lines · Frederic Weisbecker

Factorize the events enabling accounting in a common tracing core helper. This reduces the size of the profile_enable() and profile_disable() callbacks for each trace events.

-- Steve

Quoted 11 lines
Cc Peter Zijlstra <[email protected]>
Cc Li Zefan <[email protected]>
Cc Jason Baron <[email protected]>
Cc Masami Hiramatsu <[email protected]>
Cc Ingo Molnar <[email protected]>
---
 include/linux/ftrace_event.h       |    4 ++--
 include/linux/syscalls.h           |   24 ++++++++----------------
 include/trace/ftrace.h             |   28 ++++++++--------------------
 kernel/trace/trace_event_profile.c |   20 ++++++++++++++++++--
 4 files changed, 36 insertions(+), 40 deletions(-)
#4 lore raw

PATCH 1/2 tracing: Factorize the events profile accounting

In reply to Frederic Weisbecker

Factorize the events enabling accounting in a common tracing core helper. This reduces the size of the profile_enable() and profile_disable() callbacks for each trace events.

---
 include/linux/ftrace_event.h       |    4 ++--
 include/linux/syscalls.h           |   24 ++++++++----------------
 include/trace/ftrace.h             |   28 ++++++++--------------------
 kernel/trace/trace_event_profile.c |   20 ++++++++++++++++++--
 4 files changed, 36 insertions(+), 40 deletions(-)
include/linux/ftrace_event.h +2 -2
index bd099ba..bc103d7 100644
@@ -130,8 +130,8 @@ struct ftrace_event_call {
 	void			*data;
 
 	atomic_t		profile_count;
-	int			(*profile_enable)(struct ftrace_event_call *);
-	void			(*profile_disable)(struct ftrace_event_call *);
+	int			(*profile_enable)(void);
+	void			(*profile_disable)(void);
 };
 
 #define MAX_FILTER_PRED		32
include/linux/syscalls.h +8 -16
index a8e3782..7d9803c 100644
@@ -100,33 +100,25 @@ struct perf_counter_attr;
 
 #ifdef CONFIG_EVENT_PROFILE
 #define TRACE_SYS_ENTER_PROFILE(sname)					       \
-static int prof_sysenter_enable_##sname(struct ftrace_event_call *event_call)  \
+static int prof_sysenter_enable_##sname(void)				       \
 {									       \
-	int ret = 0;							       \
-	if (!atomic_inc_return(&event_enter_##sname.profile_count))	       \
-		ret = reg_prof_syscall_enter("sys"#sname);		       \
-	return ret;							       \
+	return reg_prof_syscall_enter("sys"#sname);			       \
 }									       \
 									       \
-static void prof_sysenter_disable_##sname(struct ftrace_event_call *event_call)\
+static void prof_sysenter_disable_##sname(void)				       \
 {									       \
-	if (atomic_add_negative(-1, &event_enter_##sname.profile_count))       \
-		unreg_prof_syscall_enter("sys"#sname);			       \
+	unreg_prof_syscall_enter("sys"#sname);				       \
 }
 
 #define TRACE_SYS_EXIT_PROFILE(sname)					       \
-static int prof_sysexit_enable_##sname(struct ftrace_event_call *event_call)   \
+static int prof_sysexit_enable_##sname(void)				       \
 {									       \
-	int ret = 0;							       \
-	if (!atomic_inc_return(&event_exit_##sname.profile_count))	       \
-		ret = reg_prof_syscall_exit("sys"#sname);		       \
-	return ret;							       \
+	return reg_prof_syscall_exit("sys"#sname);			       \
 }									       \
 									       \
-static void prof_sysexit_disable_##sname(struct ftrace_event_call *event_call) \
+static void prof_sysexit_disable_##sname(void)				       \
 {                                                                              \
-	if (atomic_add_negative(-1, &event_exit_##sname.profile_count))	       \
-		unreg_prof_syscall_exit("sys"#sname);			       \
+	unreg_prof_syscall_exit("sys"#sname);				       \
 }
 
 #define TRACE_SYS_ENTER_PROFILE_INIT(sname)				       \
include/trace/ftrace.h +8 -20
index 72a3b43..a822087 100644
@@ -382,20 +382,14 @@ static inline int ftrace_get_offsets_##call(				\
  *
  * NOTE: The insertion profile callback (ftrace_profile_<call>) is defined later
  *
- * static int ftrace_profile_enable_<call>(struct ftrace_event_call *event_call)
+ * static int ftrace_profile_enable_<call>(void)
  * {
- * 	int ret = 0;
- *
- * 	if (!atomic_inc_return(&event_call->profile_count))
- * 		ret = register_trace_<call>(ftrace_profile_<call>);
- *
- * 	return ret;
+ * 	return register_trace_<call>(ftrace_profile_<call>);
  * }
  *
- * static void ftrace_profile_disable_<call>(struct ftrace_event_call *event_call)
+ * static void ftrace_profile_disable_<call>(void)
  * {
- * 	if (atomic_add_negative(-1, &event->call->profile_count))
- * 		unregister_trace_<call>(ftrace_profile_<call>);
+ * 	unregister_trace_<call>(ftrace_profile_<call>);
  * }
  *
  */
@@ -405,20 +399,14 @@ static inline int ftrace_get_offsets_##call(				\
 									\
 static void ftrace_profile_##call(proto);				\
 									\
-static int ftrace_profile_enable_##call(struct ftrace_event_call *event_call) \
+static int ftrace_profile_enable_##call(void)				\
 {									\
-	int ret = 0;							\
-									\
-	if (!atomic_inc_return(&event_call->profile_count))		\
-		ret = register_trace_##call(ftrace_profile_##call);	\
-									\
-	return ret;							\
+	return register_trace_##call(ftrace_profile_##call);		\
 }									\
 									\
-static void ftrace_profile_disable_##call(struct ftrace_event_call *event_call)\
+static void ftrace_profile_disable_##call(void)				\
 {									\
-	if (atomic_add_negative(-1, &event_call->profile_count))	\
-		unregister_trace_##call(ftrace_profile_##call);		\
+	unregister_trace_##call(ftrace_profile_##call);			\
 }
 
 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
kernel/trace/trace_event_profile.c +18 -2
index 55a25c9..df4a74e 100644
@@ -8,6 +8,14 @@
 #include <linux/module.h>
 #include "trace.h"
 
+static int ftrace_profile_enable_event(struct ftrace_event_call *event)
+{
+	if (atomic_inc_return(&event->profile_count))
+		return 0;
+
+	return event->profile_enable();
+}
+
 int ftrace_profile_enable(int event_id)
 {
 	struct ftrace_event_call *event;
@@ -17,7 +25,7 @@ int ftrace_profile_enable(int event_id)
 	list_for_each_entry(event, &ftrace_events, list) {
 		if (event->id == event_id && event->profile_enable &&
 		    try_module_get(event->mod)) {
-			ret = event->profile_enable(event);
+			ret = ftrace_profile_enable_event(event);
 			break;
 		}
 	}
@@ -26,6 +34,14 @@ int ftrace_profile_enable(int event_id)
 	return ret;
 }
 
+static void ftrace_profile_disable_event(struct ftrace_event_call *event)
+{
+	if (!atomic_add_negative(-1, &event->profile_count))
+		return;
+
+	event->profile_disable();
+}
+
 void ftrace_profile_disable(int event_id)
 {
 	struct ftrace_event_call *event;
@@ -33,7 +49,7 @@ void ftrace_profile_disable(int event_id)
 	mutex_lock(&event_mutex);
 	list_for_each_entry(event, &ftrace_events, list) {
 		if (event->id == event_id) {
-			event->profile_disable(event);
+			ftrace_profile_disable_event(event);
 			module_put(event->mod);
 			break;
 		}
Signature · 1 line
1.6.2.3
#5 lore raw

v2 cover tracing: Tracing event profiling updates

In reply to Frederic Weisbecker

Ingo,

Please forget the previous pull request. This new one addresses Steve's review.

Thanks,
Frederic.

git://git.kernel.org/pub/scm/linux/kernel/git/frederic/random-tracing.git
tracing/core-v2

Frederic Weisbecker (2):
tracing: Factorize the events profile accounting
tracing: Allocate the ftrace event profile buffer dynamically

 include/linux/ftrace_event.h       |   10 +++-
 include/linux/syscalls.h           |   24 +++-----
 include/trace/ftrace.h             |  111 ++++++++++++++++++++---------------
 kernel/trace/trace_event_profile.c |   79 +++++++++++++++++++++++++-
 kernel/trace/trace_syscalls.c      |   97 +++++++++++++++++++++++++------
 5 files changed, 234 insertions(+), 87 deletions(-)
#6 lore raw

v3 cover tracing: Tracing event profiling updates

Ingo,

Hopefully this is my last attempt.
This new iteration fixes the syscalls events to correctly handle the buffer. In the previous version, they did not care about interrupts.

I only resend the second patch as only this one has changed since the v2.

The new branch is in:
git://git.kernel.org/pub/scm/linux/kernel/git/frederic/random-tracing.git
tracing/core-v3

Thanks,
Frederic.

Frederic Weisbecker (2):
tracing: Factorize the events profile accounting
tracing: Allocate the ftrace event profile buffer dynamically

 include/linux/ftrace_event.h       |   10 +++-
 include/linux/syscalls.h           |   24 +++-----
 include/trace/ftrace.h             |  111 ++++++++++++++++++++---------------
 kernel/trace/trace_event_profile.c |   79 +++++++++++++++++++++++++-
 kernel/trace/trace_syscalls.c      |   97 +++++++++++++++++++++++++------
 5 files changed, 234 insertions(+), 87 deletions(-)
#7 lore raw
Quoted 20 lines · Frederic Weisbecker

Ingo,

Hopefully this is my last attempt.
This new iteration fixes the syscalls events to correctly handle the buffer. In the previous version, they did not care about interrupts.

I only resend the second patch as only this one has changed since the v2.

The new branch is in:
git://git.kernel.org/pub/scm/linux/kernel/git/frederic/random-tracing.git
tracing/core-v3

Thanks,
Frederic.

Frederic Weisbecker (2):
tracing: Factorize the events profile accounting
tracing: Allocate the ftrace event profile buffer dynamically

Quoted 6 lines
 include/linux/ftrace_event.h       |   10 +++-
 include/linux/syscalls.h           |   24 +++-----
 include/trace/ftrace.h             |  111 ++++++++++++++++++++---------------
 kernel/trace/trace_event_profile.c |   79 +++++++++++++++++++++++++-
 kernel/trace/trace_syscalls.c      |   97 +++++++++++++++++++++++++------
 5 files changed, 234 insertions(+), 87 deletions(-)

Hm, the naming is quite confusing here i think:

  -132,8 +133,12 @@ struct ftrace_event_call {
         atomic_t                profile_count;
         int                     (*profile_enable)(void);
         void                    (*profile_disable)(void);
 +       char                    *profile_buf;
 +       char                    *profile_buf_nmi;

These are generic events, not just 'profiling' histograms.

Generic events can have _many_ output modi:

 - SVGs                   (perf timeline)
 - histograms             (perf report)
 - traces                 (perf trace)
 - summaries / maximums   (perf sched lat)
 - maps                   (perf sched map)
 - graphs                 (perf report --call-graph)

So it's quite a misnomer to talk just about profiling here. This is an event record buffer.

Also, what is the currently maximum possible size of ->profile_buf? The max size of an event record? The new codepath looks a bit heavy with rcu-lock/unlock and other bits put inbetween - and this is now in the event sending critical path. Cannot we do a permanent buffer that needs no extra locking/reference protection?

Is the whole thing even justified? I mean, we keep the size of records low anyway. It's a _lot_ easier to handle on-stack records, they are the ideal (and very fast) dynamic allocator which is NMI and IRQ safe, etc.

Ingo

#8 lore raw
Quoted 50 lines · Ingo Molnar
Quoted 20 lines · Frederic Weisbecker

Ingo,

Hopefully this is my last attempt.
This new iteration fixes the syscalls events to correctly handle the buffer. In the previous version, they did not care about interrupts.

I only resend the second patch as only this one has changed since the v2.

The new branch is in:
git://git.kernel.org/pub/scm/linux/kernel/git/frederic/random-tracing.git
tracing/core-v3

Thanks,
Frederic.

Frederic Weisbecker (2):
tracing: Factorize the events profile accounting
tracing: Allocate the ftrace event profile buffer dynamically

Quoted 6 lines
 include/linux/ftrace_event.h       |   10 +++-
 include/linux/syscalls.h           |   24 +++-----
 include/trace/ftrace.h             |  111 ++++++++++++++++++++---------------
 kernel/trace/trace_event_profile.c |   79 +++++++++++++++++++++++++-
 kernel/trace/trace_syscalls.c      |   97 +++++++++++++++++++++++++------
 5 files changed, 234 insertions(+), 87 deletions(-)

Hm, the naming is quite confusing here i think:

  -132,8 +133,12 @@ struct ftrace_event_call {
         atomic_t                profile_count;
         int                     (*profile_enable)(void);
         void                    (*profile_disable)(void);
 +       char                    *profile_buf;
 +       char                    *profile_buf_nmi;

These are generic events, not just 'profiling' histograms.

Generic events can have _many_ output modi:

 - SVGs                   (perf timeline)
 - histograms             (perf report)
 - traces                 (perf trace)
 - summaries / maximums   (perf sched lat)
 - maps                   (perf sched map)
 - graphs                 (perf report --call-graph)
Quoted 2 lines

So it's quite a misnomer to talk just about profiling here. This is an event record buffer.

Agreed, I guess we can call the perf_event_buf/perf_event_nmi. Also may be the profile_enable/profile_disable should follow the renaming logic.

Quoted 11 lines

Also, what is the currently maximum possible size of ->profile_buf? The max size of an event record? The new codepath looks a bit heavy with rcu-lock/unlock and other bits put inbetween - and this is now in the event sending critical path. Cannot we do a permanent buffer that needs no extra locking/reference protection?

Is the whole thing even justified? I mean, we keep the size of records low anyway. It's a _lot_ easier to handle on-stack records, they are the ideal (and very fast) dynamic allocator which is NMI and IRQ safe, etc.

Ingo

The max size of an event is undefined once it uses either a __dynamic_array() or __string() field. (The latter is a subset of the former anyway).

Those are very special fields that can handle dynamic size arrays. That makes such events having an unpredictable size each time they are triggered.

That said, we are currently using a stack based buffer. That coupled with the unpredicatble event size must really be fixed. I mean, we don't want to run out of the stack boundaries once an event get a long string, once a new large event is added, or once an event is randomly triggered in a path where the stack is already deeply dug.

I've done this easy stack based as a first shot to support ftrace raw events by perf, but now this becomes something that needs to be fixed IMO.

I've made this rcu based thing to avoid wasting the buffer in memory (for each cpu) while we are not profiling the raw events.

I could drop that and keep these buffers static, but this seems wasteful wrt memory footprint while profiling/tracing is inactive.

#9 lore raw

PATCH v3 2/2 tracing: Allocate the ftrace event profile buffer dynamically

In reply to Frederic Weisbecker

Currently the trace event profile buffer is allocated in the stack. But this may be too much for the stack, as the events can have large statically defined field size and can also grow with dynamic arrays.

Allocate two per cpu buffer for all profiled events. The first cpu buffer is used to host every non-nmi context traces. It is protected by disabling the interrupts while writing and committing the trace.

The second buffer is reserved for nmi. So that there is no race between them and the first buffer.

The whole write/commit section is rcu protected because we release these buffers while deactivating the last profiling trace event.

v2: Move the buffers from trace_event to be global, as pointed by
Steven Rostedt.

v3: Fix the syscall events to handle the profiling buffer races
by disabling interrupts, now that the buffers are globals.

Cc Steven Rostedt <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Li Zefan <[email protected]>
Cc Jason Baron <[email protected]>
Cc Masami Hiramatsu <[email protected]>
Cc Ingo Molnar <[email protected]>
---
 include/linux/ftrace_event.h       |    6 ++
 include/trace/ftrace.h             |   83 ++++++++++++++++++++----------
 kernel/trace/trace_event_profile.c |   61 ++++++++++++++++++++++-
 kernel/trace/trace_syscalls.c      |   97 +++++++++++++++++++++++++++++-------
 4 files changed, 199 insertions(+), 48 deletions(-)
include/linux/ftrace_event.h +6 -0
index bc103d7..4ec5e67 100644
@@ -4,6 +4,7 @@
 #include <linux/ring_buffer.h>
 #include <linux/trace_seq.h>
 #include <linux/percpu.h>
+#include <linux/hardirq.h>
 
 struct trace_array;
 struct tracer;
@@ -134,6 +135,11 @@ struct ftrace_event_call {
 	void			(*profile_disable)(void);
 };
 
+#define FTRACE_MAX_PROFILE_SIZE	2048
+
+extern char			*trace_profile_buf;
+extern char			*trace_profile_buf_nmi;
+
 #define MAX_FILTER_PRED		32
 #define MAX_FILTER_STR_VAL	256	/* Should handle KSYM_SYMBOL_LEN */
 
include/trace/ftrace.h +55 -28

125 lines not on this page · raw message

kernel/trace/trace_event_profile.c +60 -1

82 lines not on this page · raw message

kernel/trace/trace_syscalls.c +78 -19

128 lines not on this page · raw message

Signature · 1 line
1.6.2.3
#10 lore raw

PATCH 2/2 tracing: Allocate the ftrace event profile buffer dynamically

In reply to Frederic Weisbecker

Currently the trace event profile buffer is allocated in the stack. But this may be too much for the stack, as the events can have large statically defined field size and can also grow with dynamic arrays.

Allocate two per cpu buffer for each profiled events. The first cpu buffer is used to host every non-nmi context traces. It is protected by disabling the interrupts while writing and committing the trace.

The second buffer is reserved for nmi. So that there is no race between them and the first buffer.

The whole write/commit section is rcu protected because we release these buffers while deactivating a trace event for profiling.

Cc Steven Rostedt <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Li Zefan <[email protected]>
Cc Jason Baron <[email protected]>
Cc Masami Hiramatsu <[email protected]>
Cc Ingo Molnar <[email protected]>
---
 include/linux/ftrace_event.h       |    5 ++
 include/trace/ftrace.h             |   83 +++++++++++++++++++----------
 kernel/trace/trace_event_profile.c |   50 +++++++++++++++++-
 kernel/trace/trace_syscalls.c      |  103 +++++++++++++++++++++++++++++-------
 4 files changed, 193 insertions(+), 48 deletions(-)
include/linux/ftrace_event.h +5 -0
index bc103d7..497eb37 100644
@@ -4,6 +4,7 @@
 #include <linux/ring_buffer.h>
 #include <linux/trace_seq.h>
 #include <linux/percpu.h>
+#include <linux/hardirq.h>
 
 struct trace_array;
 struct tracer;
@@ -132,8 +133,12 @@ struct ftrace_event_call {
 	atomic_t		profile_count;
 	int			(*profile_enable)(void);
 	void			(*profile_disable)(void);
+	char			*profile_buf;
+	char			*profile_buf_nmi;
 };
 
+#define FTRACE_MAX_PROFILE_SIZE	2048
+
 #define MAX_FILTER_PRED		32
 #define MAX_FILTER_STR_VAL	256	/* Should handle KSYM_SYMBOL_LEN */
 
include/trace/ftrace.h +55 -28

125 lines not on this page · raw message

kernel/trace/trace_event_profile.c +49 -1

71 lines not on this page · raw message

kernel/trace/trace_syscalls.c +84 -19

139 lines not on this page · raw message

Signature · 1 line
1.6.2.3
maintainer
#11 lore raw

Re: [PATCH 2/2] tracing: Allocate the ftrace event profile buffer dynamically

Quoted 201 lines · Frederic Weisbecker

Currently the trace event profile buffer is allocated in the stack. But this may be too much for the stack, as the events can have large statically defined field size and can also grow with dynamic arrays.

Allocate two per cpu buffer for each profiled events. The first cpu buffer is used to host every non-nmi context traces. It is protected by disabling the interrupts while writing and committing the trace.

The second buffer is reserved for nmi. So that there is no race between them and the first buffer.

The whole write/commit section is rcu protected because we release these buffers while deactivating a trace event for profiling.

Cc Steven Rostedt <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Li Zefan <[email protected]>
Cc Jason Baron <[email protected]>
Cc Masami Hiramatsu <[email protected]>
Cc Ingo Molnar <[email protected]>
---
 include/linux/ftrace_event.h       |    5 ++
 include/trace/ftrace.h             |   83 +++++++++++++++++++----------
 kernel/trace/trace_event_profile.c |   50 +++++++++++++++++-
 kernel/trace/trace_syscalls.c      |  103 +++++++++++++++++++++++++++++-------
 4 files changed, 193 insertions(+), 48 deletions(-)
include/linux/ftrace_event.h +5 -0
index bc103d7..497eb37 100644
@@ -4,6 +4,7 @@
 #include <linux/ring_buffer.h>
 #include <linux/trace_seq.h>
 #include <linux/percpu.h>
+#include <linux/hardirq.h>
 
 struct trace_array;
 struct tracer;
@@ -132,8 +133,12 @@ struct ftrace_event_call {
 	atomic_t		profile_count;
 	int			(*profile_enable)(void);
 	void			(*profile_disable)(void);
+	char			*profile_buf;
+	char			*profile_buf_nmi;
 };
 
+#define FTRACE_MAX_PROFILE_SIZE	2048
+
 #define MAX_FILTER_PRED		32
 #define MAX_FILTER_STR_VAL	256	/* Should handle KSYM_SYMBOL_LEN */
 
include/trace/ftrace.h +55 -28
index a822087..80864cf 100644
@@ -648,11 +648,12 @@ __attribute__((section("_ftrace_events"))) event_##call = {		\
  *	struct ftrace_raw_##call *entry;
  *	u64 __addr = 0, __count = 1;
  *	unsigned long irq_flags;
+ *	struct trace_entry *ent;
  *	int __entry_size;
  *	int __data_size;
+ *	int __cpu
  *	int pc;
  *
- *	local_save_flags(irq_flags);
  *	pc = preempt_count();
  *
  *	__data_size = ftrace_get_offsets_<call>(&__data_offsets, args);
@@ -663,25 +664,34 @@ __attribute__((section("_ftrace_events"))) event_##call = {		\
  *			     sizeof(u64));
  *	__entry_size -= sizeof(u32);
  *
- *	do {
- *		char raw_data[__entry_size]; <- allocate our sample in the stack
- *		struct trace_entry *ent;
+ *	// Protect the non nmi buffer
+ *	// This also protects the rcu read side
+ *	local_irq_save(irq_flags);
+ *	__cpu = smp_processor_id();
+ *
+ *	if (in_nmi())
+ *		raw_data = rcu_dereference(event_call->profile_buf_nmi);
+ *	else
+ *		raw_data = rcu_dereference(event_call->profile_buf);
+ *
+ *	if (!raw_data)
+ *		goto end;
  *
- *		zero dead bytes from alignment to avoid stack leak to userspace:
+ *	raw_data = per_cpu_ptr(raw_data, __cpu);
  *
- *		*(u64 *)(&raw_data[__entry_size - sizeof(u64)]) = 0ULL;
- *		entry = (struct ftrace_raw_<call> *)raw_data;
- *		ent = &entry->ent;
- *		tracing_generic_entry_update(ent, irq_flags, pc);
- *		ent->type = event_call->id;
+ *	//zero dead bytes from alignment to avoid stack leak to userspace:
+ *	*(u64 *)(&raw_data[__entry_size - sizeof(u64)]) = 0ULL;
+ *	entry = (struct ftrace_raw_<call> *)raw_data;
+ *	ent = &entry->ent;
+ *	tracing_generic_entry_update(ent, irq_flags, pc);
+ *	ent->type = event_call->id;
  *
- *		<tstruct> <- do some jobs with dynamic arrays
+ *	<tstruct> <- do some jobs with dynamic arrays
  *
- *		<assign>  <- affect our values
+ *	<assign>  <- affect our values
  *
- *		perf_tpcounter_event(event_call->id, __addr, __count, entry,
- *			     __entry_size);  <- submit them to perf counter
- *	} while (0);
+ *	perf_tpcounter_event(event_call->id, __addr, __count, entry,
+ *		     __entry_size);  <- submit them to perf counter
  *
  * }
  */
@@ -704,11 +714,13 @@ static void ftrace_profile_##call(proto)				\
 	struct ftrace_raw_##call *entry;				\
 	u64 __addr = 0, __count = 1;					\
 	unsigned long irq_flags;					\
+	struct trace_entry *ent;					\
 	int __entry_size;						\
 	int __data_size;						\
+	char *raw_data;							\
+	int __cpu;							\
 	int pc;								\
 									\
-	local_save_flags(irq_flags);					\
 	pc = preempt_count();						\
 									\
 	__data_size = ftrace_get_offsets_##call(&__data_offsets, args); \
@@ -716,23 +728,38 @@ static void ftrace_profile_##call(proto)				\
 			     sizeof(u64));				\
 	__entry_size -= sizeof(u32);					\
 									\
-	do {								\
-		char raw_data[__entry_size];				\
-		struct trace_entry *ent;				\
+	if (WARN_ONCE(__entry_size > FTRACE_MAX_PROFILE_SIZE,		\
+		      "profile buffer not large enough"))		\
+		return;							\
+									\
+	local_irq_save(irq_flags);					\
+	__cpu = smp_processor_id();					\
 									\
-		*(u64 *)(&raw_data[__entry_size - sizeof(u64)]) = 0ULL;	\
-		entry = (struct ftrace_raw_##call *)raw_data;		\
-		ent = &entry->ent;					\
-		tracing_generic_entry_update(ent, irq_flags, pc);	\
-		ent->type = event_call->id;				\
+	if (in_nmi())							\
+		raw_data = rcu_dereference(event_call->profile_buf_nmi);\
+	else								\
+		raw_data = rcu_dereference(event_call->profile_buf);	\
 									\
-		tstruct							\
+	if (!raw_data)							\
+		goto end;						\
 									\
-		{ assign; }						\
+	raw_data = per_cpu_ptr(raw_data, __cpu);			\
 									\
-		perf_tpcounter_event(event_call->id, __addr, __count, entry,\
+	*(u64 *)(&raw_data[__entry_size - sizeof(u64)]) = 0ULL;		\
+	entry = (struct ftrace_raw_##call *)raw_data;			\
+	ent = &entry->ent;						\
+	tracing_generic_entry_update(ent, irq_flags, pc);		\
+	ent->type = event_call->id;					\
+									\
+	tstruct								\
+									\
+	{ assign; }							\
+									\
+	perf_tpcounter_event(event_call->id, __addr, __count, entry,	\
 			     __entry_size);				\
-	} while (0);							\
+									\
+end:									\
+	local_irq_restore(irq_flags);					\
 									\
 }
 
kernel/trace/trace_event_profile.c +6 -0
index df4a74e..48d9aec 100644
@@ -8,12 +8,43 @@
 #include <linux/module.h>
 #include "trace.h"
 
+/*
+ * We can't use a size but a type in alloc_percpu()
+ * So let's create a dummy type that matches the desired size
+ */
+typedef struct {char buf[FTRACE_MAX_PROFILE_SIZE];} profile_buf_t;
+

kernel/trace/trace_event_profile.c line 25

Quoted 10 lines
kernel/trace/trace_event_profile.c +4 -1
 static int ftrace_profile_enable_event(struct ftrace_event_call *event)
 {
+	char *buf;
+	int ret;
+
 	if (atomic_inc_return(&event->profile_count))
 		return 0;
 
-	return event->profile_enable();
+	buf = (char *)alloc_percpu(profile_buf_t);

Is this allocating a buffer for every event? Why not just have one? All events can share the same buffer. It's only a temp buffer, and can only be used by one event at a time per cpu.

-- Steve

Quoted 200 lines
kernel/trace/trace_event_profile.c +39 -0
+	if (!buf)
+		return -ENOMEM;
+
+	rcu_assign_pointer(event->profile_buf, buf);
+
+	buf = (char *)alloc_percpu(profile_buf_t);
+	if (!buf) {
+		ret = -ENOMEM;
+		goto fail_nmi;
+	}
+
+	rcu_assign_pointer(event->profile_buf_nmi, buf);
+
+	ret = event->profile_enable();
+	if (!ret)
+		return 0;
+
+	kfree(event->profile_buf_nmi);
+fail_nmi:
+	kfree(event->profile_buf);
+
+	return ret;
 }
 
 int ftrace_profile_enable(int event_id)
@@ -36,10 +67,27 @@ int ftrace_profile_enable(int event_id)
 
 static void ftrace_profile_disable_event(struct ftrace_event_call *event)
 {
+	char *buf, *nmi_buf;
+
 	if (!atomic_add_negative(-1, &event->profile_count))
 		return;
 
 	event->profile_disable();
+
+	buf = event->profile_buf;
+	rcu_assign_pointer(event->profile_buf, NULL);
+
+	nmi_buf = event->profile_buf_nmi;
+	rcu_assign_pointer(event->profile_buf_nmi, NULL);
+
+	/*
+	 * Ensure every events in profiling have finished before
+	 * releasing the buffers
+	 */
+	synchronize_sched();
+
+	free_percpu(buf);
+	free_percpu(nmi_buf);
 }
 
 void ftrace_profile_disable(int event_id)
kernel/trace/trace_syscalls.c +84 -19
index 8712ce3..a7ecc34 100644
@@ -384,10 +384,13 @@ static int sys_prof_refcount_exit;
 
 static void prof_syscall_enter(struct pt_regs *regs, long id)
 {
-	struct syscall_trace_enter *rec;
 	struct syscall_metadata *sys_data;
+	struct ftrace_event_call *event;
+	struct syscall_trace_enter *rec;
+	char *raw_data;
 	int syscall_nr;
 	int size;
+	int cpu;
 
 	syscall_nr = syscall_get_nr(current, regs);
 	if (!test_bit(syscall_nr, enabled_prof_enter_syscalls))
@@ -397,25 +400,46 @@ static void prof_syscall_enter(struct pt_regs *regs, long id)
 	if (!sys_data)
 		return;
 
+	event = sys_data->enter_event;
+
 	/* get the size after alignment with the u32 buffer size field */
 	size = sizeof(unsigned long) * sys_data->nb_args + sizeof(*rec);
 	size = ALIGN(size + sizeof(u32), sizeof(u64));
 	size -= sizeof(u32);
 
-	do {
-		char raw_data[size];
+	if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE,
+		      "profile buffer not large enough"))
+		return;
 
-		/* zero the dead bytes from align to not leak stack to user */
-		*(u64 *)(&raw_data[size - sizeof(u64)]) = 0ULL;
+	/*
+	 * We are not in nmi. Also we can't be preempted by a sysenter event
+	 * from interrupt, so we can safely take this buffer without
+	 * masking interrupts. But we still need rcu_read_lock to protect
+	 * against concurrent buffer release.
+	 */
+	rcu_read_lock();
+
+	cpu = smp_processor_id();
+	raw_data = rcu_dereference(event->profile_buf);
+
+	if (!raw_data)
+		goto end;
 
-		rec = (struct syscall_trace_enter *) raw_data;
-		tracing_generic_entry_update(&rec->ent, 0, 0);
-		rec->ent.type = sys_data->enter_id;
-		rec->nr = syscall_nr;
-		syscall_get_arguments(current, regs, 0, sys_data->nb_args,
-				       (unsigned long *)&rec->args);
-		perf_tpcounter_event(sys_data->enter_id, 0, 1, rec, size);
-	} while(0);
+	raw_data = per_cpu_ptr(raw_data, cpu);
+
+	/* zero the dead bytes from align to not leak stack to user */
+	*(u64 *)(&raw_data[size - sizeof(u64)]) = 0ULL;
+
+	rec = (struct syscall_trace_enter *) raw_data;
+	tracing_generic_entry_update(&rec->ent, 0, 0);
+	rec->ent.type = sys_data->enter_id;
+	rec->nr = syscall_nr;
+	syscall_get_arguments(current, regs, 0, sys_data->nb_args,
+			       (unsigned long *)&rec->args);
+	perf_tpcounter_event(sys_data->enter_id, 0, 1, rec, size);
+
+end:
+	rcu_read_unlock();
 }
 
 int reg_prof_syscall_enter(char *name)
@@ -460,8 +484,12 @@ void unreg_prof_syscall_enter(char *name)
 static void prof_syscall_exit(struct pt_regs *regs, long ret)
 {
 	struct syscall_metadata *sys_data;
-	struct syscall_trace_exit rec;
+	struct ftrace_event_call *event;
+	struct syscall_trace_exit *rec;
 	int syscall_nr;
+	char *raw_data;
+	int size;
+	int cpu;
 
 	syscall_nr = syscall_get_nr(current, regs);
 	if (!test_bit(syscall_nr, enabled_prof_exit_syscalls))
@@ -471,12 +499,49 @@ static void prof_syscall_exit(struct pt_regs *regs, long ret)
 	if (!sys_data)
 		return;
 
-	tracing_generic_entry_update(&rec.ent, 0, 0);
-	rec.ent.type = sys_data->exit_id;
-	rec.nr = syscall_nr;
-	rec.ret = syscall_get_return_value(current, regs);
+	event = sys_data->exit_event;
+
+	/* We can probably do that at build time */
+	size = ALIGN(sizeof(*rec) + sizeof(u32), sizeof(u64));
+	size -= sizeof(u32);
+
+	/*
+	 * Impossible, but be paranoid with the future
+	 * How to put this check outside runtime?
+	 */
+	if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE,
+		"exit event has grown above profile buffer size"))
+		return;
+
+	/*
+	 * This buffer is safe against interrupt.
+	 * But we still need rcu_read_lock to protect
+	 * against concurrent buffer release.
+	 */
+	rcu_read_lock();
+
+	cpu = smp_processor_id();
+	raw_data = rcu_dereference(event->profile_buf);
+
+	if (!raw_data)
+		goto end;
 
-	perf_tpcounter_event(sys_data->exit_id, 0, 1, &rec, sizeof(rec));
+	raw_data = per_cpu_ptr(raw_data, cpu);
+
+	/* zero the dead bytes from align to not leak stack to user */
+	*(u64 *)(&raw_data[size - sizeof(u64)]) = 0ULL;
+
+	rec = (struct syscall_trace_exit *)raw_data;
+
+	tracing_generic_entry_update(&rec->ent, 0, 0);
+	rec->ent.type = sys_data->exit_id;
+	rec->nr = syscall_nr;
+	rec->ret = syscall_get_return_value(current, regs);
+
+	perf_tpcounter_event(sys_data->exit_id, 0, 1, rec, size);
+
+end:
+	rcu_read_unlock();
 }
 
 int reg_prof_syscall_exit(char *name)
#12 lore raw

Re: [PATCH 2/2] tracing: Allocate the ftrace event profile buffer dynamically

Quoted 7 lines · Steven Rostedt

Is this allocating a buffer for every event? Why not just have one? All events can share the same buffer. It's only a temp buffer, and can only be used by one event at a time per cpu.

-- Steve

Oh you're right. That was so obvious though...

The v2 is coming soon.

Thanks.

#13 lore raw

PATCH 2/2 tracing: Allocate the ftrace event profile buffer dynamically

In reply to Frederic Weisbecker

Currently the trace event profile buffer is allocated in the stack. But this may be too much for the stack, as the events can have large statically defined field size and can also grow with dynamic arrays.

Allocate two per cpu buffer for all profiled events. The first cpu buffer is used to host every non-nmi context traces. It is protected by disabling the interrupts while writing and committing the trace.

The second buffer is reserved for nmi. So that there is no race between them and the first buffer.

The whole write/commit section is rcu protected because we release these buffers while deactivating the last profiling trace event.

v2: Move the buffers from trace_event to be global, as pointed by
Steven Rostedt.

Cc Steven Rostedt <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Li Zefan <[email protected]>
Cc Jason Baron <[email protected]>
Cc Masami Hiramatsu <[email protected]>
Cc Ingo Molnar <[email protected]>
---
 include/linux/ftrace_event.h       |    6 ++
 include/trace/ftrace.h             |   83 ++++++++++++++++++++----------
 kernel/trace/trace_event_profile.c |   61 ++++++++++++++++++++++-
 kernel/trace/trace_syscalls.c      |   97 +++++++++++++++++++++++++++++-------
 4 files changed, 199 insertions(+), 48 deletions(-)
include/linux/ftrace_event.h +6 -0
index bc103d7..4ec5e67 100644
@@ -4,6 +4,7 @@
 #include <linux/ring_buffer.h>
 #include <linux/trace_seq.h>
 #include <linux/percpu.h>
+#include <linux/hardirq.h>
 
 struct trace_array;
 struct tracer;
@@ -134,6 +135,11 @@ struct ftrace_event_call {
 	void			(*profile_disable)(void);
 };
 
+#define FTRACE_MAX_PROFILE_SIZE	2048
+
+extern char			*trace_profile_buf;
+extern char			*trace_profile_buf_nmi;
+
 #define MAX_FILTER_PRED		32
 #define MAX_FILTER_STR_VAL	256	/* Should handle KSYM_SYMBOL_LEN */
 
include/trace/ftrace.h +55 -28

125 lines not on this page · raw message

kernel/trace/trace_event_profile.c +60 -1

82 lines not on this page · raw message

kernel/trace/trace_syscalls.c +78 -19

128 lines not on this page · raw message

Signature · 1 line
1.6.2.3
#14 lore raw

Re: [PATCH 2/2] tracing: Allocate the ftrace event profile buffer dynamically

Quoted 19 lines · Frederic Weisbecker

static void prof_syscall_enter(struct pt_regs *regs, long id) {

kernel/trace/trace_syscalls.c +4 -3
-	struct syscall_trace_enter *rec;
 	struct syscall_metadata *sys_data;
+	struct syscall_trace_enter *rec;
+	char *raw_data;
 	int syscall_nr;
 	int size;
+	int cpu;
 
 	syscall_nr = syscall_get_nr(current, regs);
 	if (!test_bit(syscall_nr, enabled_prof_enter_syscalls))
@@ -402,20 +404,39 @@ static void prof_syscall_enter(struct pt_regs *regs, long id)
 	size = ALIGN(size + sizeof(u32), sizeof(u64));
 	size -= sizeof(u32);
 
-	do {
-		char raw_data[size];
+	if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE,

kernel/trace/trace_syscalls.c static void prof_syscall_enter(struct pt_regs *regs, long id) · line 417

Quoted 10 lines
kernel/trace/trace_syscalls.c +10 -0
+		      "profile buffer not large enough"))
+		return;
+
+	/*
+	 * We are not in nmi. Also we can't be preempted by a sysenter event
+	 * from interrupt, so we can safely take this buffer without
+	 * masking interrupts. But we still need rcu_read_lock to protect
+	 * against concurrent buffer release.
+	 */
+	rcu_read_lock();

Oh...now that we use a global buffer, the buffer is not safe anymore against interrupts or NMIs.

Looks like I will need a third iteration.

Sorry for the noise...