tracing/triggers: A couple minor variable name changes

This changes __tt to tt to for the recently submitted kprobes trigger patch, and does the same for the syscall trigger invocations, which uses the same unnecessary naming convention pointed out by Steve Rostedt.

The kprobes patch also had trivial changes resulting from the rebase to current tracing/for-next.

The following changes since commit e0d18fe063464cb3f1a6d1939e4fcf47d92d8386:

tracing/probes: Fix build break on !CONFIG_KPROBE_EVENT (2014-01-03 15:27:18 -0500)

are available in the git repository at:

git://git.yoctoproject.org/linux-yocto-contrib.git tzanussi/kprobes-event-triggers-v2
http://git.yoctoproject.org/cgit/cgit.cgi/linux-yocto-contrib/log/?h=tzanussi/kprobes-event-triggers-v2

Tom Zanussi (2):
tracing/kprobes: Add trace event trigger invocations
tracing: Remove double-underscore naming in syscall trigger
invocations

 kernel/trace/trace_kprobe.c   | 42 ++++++++++++++++++++++++++++++++++++------
 kernel/trace/trace_syscalls.c | 16 ++++++++--------
 2 files changed, 44 insertions(+), 14 deletions(-)

1.8.3.1

Add code to the kprobe/kretprobe event functions that will invoke any event triggers associated with a probe's ftrace_event_file.

The code to do this is very similar to the invocation code already used to invoke the triggers associated with static events and essentially replaces the existing soft-disable checks with a superset that preserves the original behavior but adds the bits needed to support event triggers.

---
 kernel/trace/trace_kprobe.c | 42 ++++++++++++++++++++++++++++++++++++------
 1 file changed, 36 insertions(+), 6 deletions(-)

index ce0ed8a..3afa716 100644
@@ -929,12 +929,20 @@ __kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs,
 	struct ring_buffer *buffer;
 	int size, dsize, pc;
 	unsigned long irq_flags;
+	unsigned long eflags;
+	enum event_trigger_type tt = ETT_NONE;
 	struct ftrace_event_call *call = &tk->tp.call;
 
 	WARN_ON(call != ftrace_file->event_call);
 
-	if (test_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT, &ftrace_file->flags))
-		return;
+	eflags = ftrace_file->flags;
+
+	if (!(eflags & FTRACE_EVENT_FL_TRIGGER_COND)) {
+		if (eflags & FTRACE_EVENT_FL_TRIGGER_MODE)
+			event_triggers_call(ftrace_file, NULL);
+		if (eflags & FTRACE_EVENT_FL_SOFT_DISABLED)
+			return;
+	}
 
 	local_save_flags(irq_flags);
 	pc = preempt_count();

@@ -952,9 +960,16 @@ __kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs,
 	entry->ip = (unsigned long)tk->rp.kp.addr;
 	store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);
 
-	if (!filter_check_discard(ftrace_file, entry, buffer, event))
+	if (eflags & FTRACE_EVENT_FL_TRIGGER_COND)
+		tt = event_triggers_call(ftrace_file, entry);
+
+	if (test_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT, &ftrace_file->flags))
+		ring_buffer_discard_commit(buffer, event);
+	else if (!filter_check_discard(ftrace_file, entry, buffer, event))
 		trace_buffer_unlock_commit_regs(buffer, event,
 						irq_flags, pc, regs);
+	if (tt)
+		event_triggers_post_call(ftrace_file, tt);
 }
 
 static __kprobes void

@@ -977,12 +992,20 @@ __kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
 	struct ring_buffer *buffer;
 	int size, pc, dsize;
 	unsigned long irq_flags;
+	unsigned long eflags;
+	enum event_trigger_type tt = ETT_NONE;
 	struct ftrace_event_call *call = &tk->tp.call;
 
 	WARN_ON(call != ftrace_file->event_call);
 
-	if (test_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT, &ftrace_file->flags))
-		return;
+	eflags = ftrace_file->flags;
+
+	if (!(eflags & FTRACE_EVENT_FL_TRIGGER_COND)) {
+		if (eflags & FTRACE_EVENT_FL_TRIGGER_MODE)
+			event_triggers_call(ftrace_file, NULL);
+		if (eflags & FTRACE_EVENT_FL_SOFT_DISABLED)
+			return;
+	}
 
 	local_save_flags(irq_flags);
 	pc = preempt_count();

@@ -1001,9 +1024,16 @@ __kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
 	entry->ret_ip = (unsigned long)ri->ret_addr;
 	store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);
 
-	if (!filter_check_discard(ftrace_file, entry, buffer, event))
+	if (eflags & FTRACE_EVENT_FL_TRIGGER_COND)
+		tt = event_triggers_call(ftrace_file, entry);
+
+	if (test_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT, &ftrace_file->flags))
+		ring_buffer_discard_commit(buffer, event);
+	else if (!filter_check_discard(ftrace_file, entry, buffer, event))
 		trace_buffer_unlock_commit_regs(buffer, event,
 						irq_flags, pc, regs);
+	if (tt)
+		event_triggers_post_call(ftrace_file, tt);
 }
 
 static __kprobes void

1.8.3.1

Looks good for me :) BTW, we'll need similar code for trace_uprobe.c too.

Thank you,

Masami HIRAMATSU
IT Management Research Dept. Linux Technology Center
Hitachi, Ltd., Yokohama Research Laboratory
E-mail: [email protected]

There's no reason to use double-underscores for any variable name in ftrace_syscall_enter()/exit(), since those functions aren't generated and there's no need to avoid namespace collisions as with the event macros, which is where the original invocation code came from.

---
 kernel/trace/trace_syscalls.c | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

index fdd955f..a4acf9b 100644
@@ -306,7 +306,7 @@ static void ftrace_syscall_enter(void *data, struct pt_regs *regs, long id)
 	struct syscall_trace_enter *entry;
 	struct syscall_metadata *sys_data;
 	struct ring_buffer_event *event;
-	enum event_trigger_type __tt = ETT_NONE;
+	enum event_trigger_type tt = ETT_NONE;
 	struct ring_buffer *buffer;
 	unsigned long irq_flags;
 	unsigned long eflags;

@@ -352,15 +352,15 @@ static void ftrace_syscall_enter(void *data, struct pt_regs *regs, long id)
 	syscall_get_arguments(current, regs, 0, sys_data->nb_args, entry->args);
 
 	if (eflags & FTRACE_EVENT_FL_TRIGGER_COND)
-		__tt = event_triggers_call(ftrace_file, entry);
+		tt = event_triggers_call(ftrace_file, entry);
 
 	if (test_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT, &ftrace_file->flags))
 		ring_buffer_discard_commit(buffer, event);
 	else if (!filter_check_discard(ftrace_file, entry, buffer, event))
 		trace_current_buffer_unlock_commit(buffer, event,
 						   irq_flags, pc);
-	if (__tt)
-		event_triggers_post_call(ftrace_file, __tt);
+	if (tt)
+		event_triggers_post_call(ftrace_file, tt);
 }
 
 static void ftrace_syscall_exit(void *data, struct pt_regs *regs, long ret)

@@ -370,7 +370,7 @@ static void ftrace_syscall_exit(void *data, struct pt_regs *regs, long ret)
 	struct syscall_trace_exit *entry;
 	struct syscall_metadata *sys_data;
 	struct ring_buffer_event *event;
-	enum event_trigger_type __tt = ETT_NONE;
+	enum event_trigger_type tt = ETT_NONE;
 	struct ring_buffer *buffer;
 	unsigned long irq_flags;
 	unsigned long eflags;

@@ -414,15 +414,15 @@ static void ftrace_syscall_exit(void *data, struct pt_regs *regs, long ret)
 	entry->ret = syscall_get_return_value(current, regs);
 
 	if (eflags & FTRACE_EVENT_FL_TRIGGER_COND)
-		__tt = event_triggers_call(ftrace_file, entry);
+		tt = event_triggers_call(ftrace_file, entry);
 
 	if (test_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT, &ftrace_file->flags))
 		ring_buffer_discard_commit(buffer, event);
 	else if (!filter_check_discard(ftrace_file, entry, buffer, event))
 		trace_current_buffer_unlock_commit(buffer, event,
 						   irq_flags, pc);
-	if (__tt)
-		event_triggers_post_call(ftrace_file, __tt);
+	if (tt)
+		event_triggers_post_call(ftrace_file, tt);
 }
 
 static int reg_event_syscall_enter(struct ftrace_event_file *file,

1.8.3.1

I applied these locally, but I'm also going to apply this patch to get rid of all that duplicate code. Any objections? (or do you see any mistakes? I only compiled and boot tested it, nothing more strenuous) I will do that before pushing it though.

-- Steve

index 03d2db2..ccda66f 100644
@@ -370,6 +370,75 @@ extern enum event_trigger_type event_triggers_call(struct ftrace_event_file *fil
 extern void event_triggers_post_call(struct ftrace_event_file *file,
 				     enum event_trigger_type tt);
 
+static inline int
+ftrace_trigger_call_disabled(struct ftrace_event_file *file)
+{
+	unsigned long eflags = file->flags;
+
+	if (!(eflags & FTRACE_EVENT_FL_TRIGGER_COND)) {
+		if (eflags & FTRACE_EVENT_FL_TRIGGER_MODE)
+			event_triggers_call(file, NULL);
+		if (eflags & FTRACE_EVENT_FL_SOFT_DISABLED)
+			return 1;
+	}
+	return 0;
+}
+
+static inline int
+__event_trigger_test_discard(struct ftrace_event_file *file,
+			     struct ring_buffer *buffer,
+			     struct ring_buffer_event *event,
+			     void *entry,
+			     enum event_trigger_type *tt)
+{
+	unsigned long eflags = file->flags;
+
+	if (eflags & FTRACE_EVENT_FL_TRIGGER_COND)
+		*tt = event_triggers_call(file, entry);
+
+	if (test_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT, &file->flags))
+		ring_buffer_discard_commit(buffer, event);
+	else if (!filter_check_discard(file, entry, buffer, event))
+		return 1;
+
+	return 0;
+}
+
+static inline void
+event_trigger_unlock_commit(struct ftrace_event_file *file,
+			    struct ring_buffer *buffer,
+			    struct ring_buffer_event *event,
+			    void *entry, unsigned long irq_flags, int pc)
+{
+	unsigned long eflags = file->flags;
+	enum event_trigger_type tt = ETT_NONE;
+
+	if (__event_trigger_test_discard(file, buffer, event, entry, &tt))
+		trace_buffer_unlock_commit(buffer, event, irq_flags, pc);
+
+	if (tt)
+		event_triggers_post_call(file, tt);
+}
+
+
+static inline void
+event_trigger_unlock_commit_regs(struct ftrace_event_file *file,
+				 struct ring_buffer *buffer,
+				 struct ring_buffer_event *event,
+				 void *entry, unsigned long irq_flags, int pc,
+				 struct pt_regs *regs)
+{
+	unsigned long eflags = file->flags;
+	enum event_trigger_type tt = ETT_NONE;
+
+	if (__event_trigger_test_discard(file, buffer, event, entry, &tt))
+		trace_buffer_unlock_commit_regs(buffer, event,
+						irq_flags, pc, regs);
+
+	if (tt)
+		event_triggers_post_call(file, tt);
+}
+
 enum {
 	FILTER_OTHER = 0,
 	FILTER_STATIC_STRING,

index 0962968..7a43935 100644
@@ -546,8 +546,6 @@ ftrace_raw_event_##call(void *__data, proto)				\
 	struct ftrace_event_file *ftrace_file = __data;			\
 	struct ftrace_event_call *event_call = ftrace_file->event_call;	\
 	struct ftrace_data_offsets_##call __maybe_unused __data_offsets;\
-	unsigned long eflags = ftrace_file->flags;			\
-	enum event_trigger_type __tt = ETT_NONE;			\
 	struct ring_buffer_event *event;				\
 	struct ftrace_raw_##call *entry;				\
 	struct ring_buffer *buffer;					\

@@ -555,12 +553,8 @@ ftrace_raw_event_##call(void *__data, proto)				\
 	int __data_size;						\
 	int pc;								\
 									\
-	if (!(eflags & FTRACE_EVENT_FL_TRIGGER_COND)) {			\
-		if (eflags & FTRACE_EVENT_FL_TRIGGER_MODE)		\
-			event_triggers_call(ftrace_file, NULL);		\
-		if (eflags & FTRACE_EVENT_FL_SOFT_DISABLED)		\
-			return;						\
-	}								\
+	if (ftrace_trigger_call_disabled(ftrace_file))			\
+		return;							\
 									\
 	local_save_flags(irq_flags);					\
 	pc = preempt_count();						\

@@ -579,17 +573,8 @@ ftrace_raw_event_##call(void *__data, proto)				\
 									\
 	{ assign; }							\
 									\
-	if (eflags & FTRACE_EVENT_FL_TRIGGER_COND)			\
-		__tt = event_triggers_call(ftrace_file, entry);		\
-									\
-	if (test_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT,                 \
-		     &ftrace_file->flags))                              \
-		ring_buffer_discard_commit(buffer, event);              \
-	else if (!filter_check_discard(ftrace_file, entry, buffer, event)) \
-		trace_buffer_unlock_commit(buffer, event, irq_flags, pc); \
-									\
-	if (__tt)							\
-		event_triggers_post_call(ftrace_file, __tt);		\
+	event_trigger_unlock_commit(ftrace_file, buffer, event, entry, \
+				    irq_flags, pc);		       \
 }
 /*
  * The ftrace_test_probe is compiled out, it is only here as a build time check

Hi Steve,

It's a nice cleanup, and from my reading of it, the logic is all correct, but there are a couple things that I think are a bit misleading in the naming, see below..

On Mon, 06 Jan 2014 18:47:53 -0600

Honestly, I just whipped those names up out of the blue. Yeah, ftrace_trigger_soft_disabled() sounds better. Will switch.

Hmm, this was a ugly helper function (hence the "__" ;-) I really couldn't think of a good name. Actually, it's called "test_discard" which means it's just a test to know if it should discard or not.

How about if I switch it to ".._discard_test" and just comment what it returns? I was going to add comments regardless.

Hah, thanks. It originally was before I created the __event_trigger_test_discard() helper function. My extensive tests would have caught that too, but thanks. That saved me from having to fix it before running the tests again.

Hmm, OK, I think you may have convinced me. I'll swap the return values.

I need to write up some trigger tests to make sure they work the same before and after this patch.

This one was cut and pasted added.

Thanks!

-- Steve

Sounds great, thanks!

I'll run the next version through my 'testsuite', which unfortunately is still manual (one of the many things on my todo list is automate it)..

Tom

New patch.

-- Steve

tracing: Consolidate event trigger code

The event trigger code that checks for callback triggers before and after recording of an event has lots of flags checks. This code is duplicated throughout the ftrace events, kprobes and system calls. They all do the exact same checks against the event flags.

Added helper functions ftrace_trigger_soft_disabled(), event_trigger_unlock_commit() and event_trigger_unlock_commit_regs() that consolidated the code and these are used instead.

---
 include/linux/ftrace_event.h  | 117 ++++++++++++++++++++++++++++++++++++++++++
 include/trace/ftrace.h        |  23 ++-------
 kernel/trace/trace_kprobe.c   |  48 +++--------------
 kernel/trace/trace_syscalls.c |  48 +++--------------
 4 files changed, 137 insertions(+), 99 deletions(-)

index 03d2db2..4e4cc28 100644
@@ -370,6 +370,123 @@ extern enum event_trigger_type event_triggers_call(struct ftrace_event_file *fil
 extern void event_triggers_post_call(struct ftrace_event_file *file,
 				     enum event_trigger_type tt);
 
+/**
+ * ftrace_trigger_soft_disabled - do triggers and test if soft disabled
+ * @file: The file pointer of the event to test
+ *
+ * If any triggers without filters are attached to this event, they
+ * will be called here. If the event is soft disabled and has no
+ * triggers that require testing the fields, it will return true,
+ * otherwise false.
+ */
+static inline bool
+ftrace_trigger_soft_disabled(struct ftrace_event_file *file)
+{
+	unsigned long eflags = file->flags;
+
+	if (!(eflags & FTRACE_EVENT_FL_TRIGGER_COND)) {
+		if (eflags & FTRACE_EVENT_FL_TRIGGER_MODE)
+			event_triggers_call(file, NULL);
+		if (eflags & FTRACE_EVENT_FL_SOFT_DISABLED)
+			return true;
+	}
+	return false;
+}
+
+/*
+ * Helper function for event_trigger_unlock_commit{_regs}().
+ * If there are event triggers attached to this event that requires
+ * filtering against its fields, then they wil be called as the
+ * entry already holds the field information of the current event.
+ *
+ * It also checks if the event should be discarded or not.
+ * It is to be discarded if the event is soft disabled and the
+ * event was only recorded to process triggers, or if the event
+ * filter is active and this event did not match the filters.
+ *
+ * Returns true if the event is discarded, false otherwise.
+ */
+static inline bool
+__event_trigger_test_discard(struct ftrace_event_file *file,
+			     struct ring_buffer *buffer,
+			     struct ring_buffer_event *event,
+			     void *entry,
+			     enum event_trigger_type *tt)
+{
+	unsigned long eflags = file->flags;
+
+	if (eflags & FTRACE_EVENT_FL_TRIGGER_COND)
+		*tt = event_triggers_call(file, entry);
+
+	if (test_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT, &file->flags))
+		ring_buffer_discard_commit(buffer, event);
+	else if (!filter_check_discard(file, entry, buffer, event))
+		return false;
+
+	return true;
+}
+
+/**
+ * event_trigger_unlock_commit - handle triggers and finish event commit
+ * @file: The file pointer assoctiated to the event
+ * @buffer: The ring buffer that the event is being written to
+ * @event: The event meta data in the ring buffer
+ * @entry: The event itself
+ * @irq_flags: The state of the interrupts at the start of the event
+ * @pc: The state of the preempt count at the start of the event.
+ *
+ * This is a helper function to handle triggers that require data
+ * from the event itself. It also tests the event against filters and
+ * if the event is soft disabled and should be discarded.
+ */
+static inline void
+event_trigger_unlock_commit(struct ftrace_event_file *file,
+			    struct ring_buffer *buffer,
+			    struct ring_buffer_event *event,
+			    void *entry, unsigned long irq_flags, int pc)
+{
+	enum event_trigger_type tt = ETT_NONE;
+
+	if (!__event_trigger_test_discard(file, buffer, event, entry, &tt))
+		trace_buffer_unlock_commit(buffer, event, irq_flags, pc);
+
+	if (tt)
+		event_triggers_post_call(file, tt);
+}
+
+/**
+ * event_trigger_unlock_commit_regs - handle triggers and finish event commit
+ * @file: The file pointer assoctiated to the event
+ * @buffer: The ring buffer that the event is being written to
+ * @event: The event meta data in the ring buffer
+ * @entry: The event itself
+ * @irq_flags: The state of the interrupts at the start of the event
+ * @pc: The state of the preempt count at the start of the event.
+ *
+ * This is a helper function to handle triggers that require data
+ * from the event itself. It also tests the event against filters and
+ * if the event is soft disabled and should be discarded.
+ *
+ * Same as event_trigger_unlock_commit() but calls
+ * trace_buffer_unlock_commit_regs() instead of trace_buffer_unlock_commit().
+ */
+static inline void
+event_trigger_unlock_commit_regs(struct ftrace_event_file *file,
+				 struct ring_buffer *buffer,
+				 struct ring_buffer_event *event,
+				 void *entry, unsigned long irq_flags, int pc,
+				 struct pt_regs *regs)
+{
+	enum event_trigger_type tt = ETT_NONE;
+
+	if (!__event_trigger_test_discard(file, buffer, event, entry, &tt))
+		trace_buffer_unlock_commit_regs(buffer, event,
+						irq_flags, pc, regs);
+
+	if (tt)
+		event_triggers_post_call(file, tt);
+}
+
 enum {
 	FILTER_OTHER = 0,
 	FILTER_STATIC_STRING,

index 0962968..1a8b28d 100644
@@ -546,8 +546,6 @@ ftrace_raw_event_##call(void *__data, proto)				\
 	struct ftrace_event_file *ftrace_file = __data;			\
 	struct ftrace_event_call *event_call = ftrace_file->event_call;	\
 	struct ftrace_data_offsets_##call __maybe_unused __data_offsets;\
-	unsigned long eflags = ftrace_file->flags;			\
-	enum event_trigger_type __tt = ETT_NONE;			\
 	struct ring_buffer_event *event;				\
 	struct ftrace_raw_##call *entry;				\
 	struct ring_buffer *buffer;					\

@@ -555,12 +553,8 @@ ftrace_raw_event_##call(void *__data, proto)				\
 	int __data_size;						\
 	int pc;								\
 									\
-	if (!(eflags & FTRACE_EVENT_FL_TRIGGER_COND)) {			\
-		if (eflags & FTRACE_EVENT_FL_TRIGGER_MODE)		\
-			event_triggers_call(ftrace_file, NULL);		\
-		if (eflags & FTRACE_EVENT_FL_SOFT_DISABLED)		\
-			return;						\
-	}								\
+	if (ftrace_trigger_soft_disabled(ftrace_file))			\
+		return;							\
 									\
 	local_save_flags(irq_flags);					\
 	pc = preempt_count();						\

@@ -579,17 +573,8 @@ ftrace_raw_event_##call(void *__data, proto)				\
 									\
 	{ assign; }							\
 									\
-	if (eflags & FTRACE_EVENT_FL_TRIGGER_COND)			\
-		__tt = event_triggers_call(ftrace_file, entry);		\
-									\
-	if (test_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT,                 \
-		     &ftrace_file->flags))                              \
-		ring_buffer_discard_commit(buffer, event);              \
-	else if (!filter_check_discard(ftrace_file, entry, buffer, event)) \
-		trace_buffer_unlock_commit(buffer, event, irq_flags, pc); \
-									\
-	if (__tt)							\
-		event_triggers_post_call(ftrace_file, __tt);		\
+	event_trigger_unlock_commit(ftrace_file, buffer, event, entry, \
+				    irq_flags, pc);		       \
 }
 /*
  * The ftrace_test_probe is compiled out, it is only here as a build time check

1.8.1.4

Looks good to me and passes my normal testing.

Just so you know what encompasses my 'normal testing', below is what I normally run through - if everything checks out, I'm satisfied things are basically good, though of course it didn't catch your traceoff bug so it's not complete.

As soon as I get the chance, I'll automate it and add more tests for completeness - as it is it's grown into something that already takes too long to do manually, so I need to find the time....

---- basic high-level check
- quick enable/disable_event, capture limited number of events
- both counted and not
- verify flags
- verify removal

# echo 'enable_event:kmem:kmalloc:1' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_read/trigger;cat ~/junk.txt > /dev/null;echo 'disable_event:kmem:kmalloc' > /sys/kernel/debug/tracing/events/syscalls/sys_exit_read/trigger

# cat /sys/kernel/debug/tracing/events/kmem/kmalloc/enable
0*

# cat /sys/kernel/debug/tracing/events/syscalls/sys_enter_read/trigger
enable_event:kmem:kmalloc:count=0

# cat /sys/kernel/debug/tracing/events/syscalls/sys_enter_read/enable
0*

# cat /sys/kernel/debug/tracing/events/syscalls/sys_exit_read/trigger
disable_event:kmem:kmalloc:unlimited

# cat /sys/kernel/debug/tracing/events/syscalls/sys_exit_read/enable
0*

# cat /sys/kernel/debug/tracing/trace

# echo '!enable_event:kmem:kmalloc:1' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_read/trigger
# echo '!disable_event:kmem:kmalloc' > /sys/kernel/debug/tracing/events/syscalls/sys_exit_read/trigger

# cat /sys/kernel/debug/tracing/events/syscalls/sys_enter_read/trigger
# cat /sys/kernel/debug/tracing/events/syscalls/sys_enter_read/enable
0
# cat /sys/kernel/debug/tracing/events/kmem/kmalloc/enable
0

---- basic stacktrace checks
- both counted and not
- trigger filter

# echo 'stacktrace' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger

# cat /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
stacktrace:unlimited
# cat /sys/kernel/debug/tracing/trace

# echo '!stacktrace' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
# echo > /sys/kernel/debug/tracing/trace

# echo 'stacktrace:5 if bytes_req >= 512' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
# cat /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
stacktrace:count=0 if bytes_req >= 512

# cat /sys/kernel/debug/tracing/trace

# echo '!stacktrace:5 if bytes_req >= 512' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
# cat /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
# echo 'stacktrace:5 if bytes_req >= 65536' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
# cat /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
stacktrace:count=5 if bytes_req >= 65536

# echo '!stacktrace:5' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger

---- basic snapshot
- enable all block events, capture leadup to event
- snapshot with filter

# echo 1 > /sys/kernel/debug/tracing/events/block/enable
# echo 'snapshot:1 if nr_rq > 1' > /sys/kernel/debug/tracing/events/block/block_unplug/trigger

# cat /sys/kernel/debug/tracing/snapshot

# cat /sys/kernel/debug/tracing/events/block/block_unplug/trigger
# cat /sys/kernel/debug/tracing/events/block/block_unplug/enable
1*
# echo '!snapshot:1 if nr_rq > 1' > /sys/kernel/debug/tracing/events/block/block_unplug/trigger
# cat /sys/kernel/debug/tracing/events/block/block_unplug/trigger
# cat /sys/kernel/debug/tracing/events/block/block_unplug/enable
1

---- basic traceon/traceoff
- enable all block events, capture leadup to event
- traceoff with filter

# echo 'traceoff:1 if nr_rq > 1' > /sys/kernel/debug/tracing/events/block/block_unplug/trigger
# cat /sys/kernel/debug/tracing/events/block/block_unplug/trigger
# cat /sys/kernel/debug/tracing/events/block/block_unplug/enable

# cat /sys/kernel/debug/tracing/trace

# cat /sys/kernel/debug/tracing/tracing_on
0
# echo '!traceoff:1 if nr_rq > 1' > /sys/kernel/debug/tracing/events/block/block_unplug/trigger
# cat /sys/kernel/debug/tracing/events/block/block_unplug/enable
1
# echo 0 > /sys/kernel/debug/tracing/events/block/enable
# cat /sys/kernel/debug/tracing/events/block/block_unplug/enable

# echo 1 > /sys/kernel/debug/tracing/tracing_on

---- multi buffer non-syscall filter test

echo > /sys/kernel/debug/tracing/trace

echo 'bytes_alloc > 8192' > /sys/kernel/debug/tracing/events/kmem/kmalloc/filter cat /sys/kernel/debug/tracing/events/kmem/kmalloc/filter
bytes_alloc > 8192
mkdir /sys/kernel/debug/tracing/instances/test1
echo 'bytes_alloc > 2048' > /sys/kernel/debug/tracing/instances/test1/events/kmem/kmalloc/filter cat /sys/kernel/debug/tracing/events/kmem/kmalloc/filter
bytes_alloc > 8192
cat /sys/kernel/debug/tracing/instances/test1/events/kmem/kmalloc/filter bytes_alloc > 2048

echo 1 > /sys/kernel/debug/tracing/events/kmem/kmalloc/enable cat /sys/kernel/debug/tracing/trace

echo 1 > /sys/kernel/debug/tracing/instances/test1/events/kmem/kmalloc/enable cat /sys/kernel/debug/tracing/instances/test1/trace

echo 0 > /sys/kernel/debug/tracing/events/kmem/kmalloc/enable echo 0 > /sys/kernel/debug/tracing/instances/test1/events/kmem/kmalloc/enable echo 0 > /sys/kernel/debug/tracing/events/kmem/kmalloc/filter echo 0 > /sys/kernel/debug/tracing/instances/test1/events/kmem/kmalloc/filter

rmdir /sys/kernel/debug/tracing/instances/test1

--- multi buffer filter test

echo > /sys/kernel/debug/tracing/trace

echo 'count > 65536' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_read/filter echo 1 > /sys/kernel/debug/tracing/events/syscalls/sys_enter_read/enable cat /sys/kernel/debug/tracing/events/syscalls/sys_enter_read/filter
cat /sys/kernel/debug/tracing/trace

mkdir /sys/kernel/debug/tracing/instances/test1

echo 'count > 4096' > /sys/kernel/debug/tracing/instances/test1/events/syscalls/sys_enter_read/filter echo 1 > /sys/kernel/debug/tracing/instances/test1/events/syscalls/sys_enter_read/enable cat /sys/kernel/debug/tracing/instances/test1/events/syscalls/sys_enter_read/filter cat /sys/kernel/debug/tracing/events/syscalls/sys_enter_read/filter
cat /sys/kernel/debug/tracing/instances/test1/trace
cat /sys/kernel/debug/tracing/trace

echo 0 > /sys/kernel/debug/tracing/events/syscalls/sys_enter_read/filter echo 0 > /sys/kernel/debug/tracing/events/syscalls/sys_enter_read/enable

rmdir /sys/kernel/debug/tracing/instances/test1

---- multi-buffer trigger enable

# mkdir /sys/kernel/debug/tracing/instances/hoge
# echo 'enable_event:mce:mce_record' > /sys/kernel/debug/tracing/instances/hoge/events/syscalls/sys_enter_symlink/trigger
# cat /sys/kernel/debug/tracing/instances/hoge/events/syscalls/sys_enter_symlink/enable
0*
# cat /sys/kernel/debug/tracing/instances/hoge/events/mce/mce_record/enable
0*
# cat /sys/kernel/debug/tracing/events/mce/mce_record/enable
0
# ln -sf /dev/null /tmp
# cat /sys/kernel/debug/tracing/instances/hoge/events/mce/mce_record/enable
1*
# cat /sys/kernel/debug/tracing/events/mce/mce_record/enable
0

# rmdir /sys/kernel/debug/tracing/instances/hoge

---- enabled plus trigger test
- verify triggers work on enabled event

echo > /sys/kernel/debug/tracing/trace

echo 1 > /sys/kernel/debug/tracing/events/syscalls/sys_enter_fsync/enable echo 'stacktrace' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_fsync/trigger cat /sys/kernel/debug/tracing/events/syscalls/sys_enter_fsync/trigger

# do an emacs save to test fsync

cat /sys/kernel/debug/tracing/trace

echo 0 > /sys/kernel/debug/tracing/events/syscalls/sys_enter_fsync/enable echo '!stacktrace' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_fsync/trigger

---- multi-event trigger test, stack + enable
- verify multiple triggers on single event

echo > /sys/kernel/debug/tracing/trace

# registers syscall tracepoint
echo 'stacktrace' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_perf_event_open/trigger
echo 'enable_event:kmem:kmalloc:1' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_perf_event_open/trigger
cat /sys/kernel/debug/tracing/events/syscalls/sys_enter_perf_event_open/trigger

# do a perf stat to test

cat /sys/kernel/debug/tracing/trace

cat /sys/kernel/debug/tracing/events/kmem/kmalloc/enable

echo 0 > /sys/kernel/debug/tracing/events/kmem/kmalloc/enable

echo '!stacktrace' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_perf_event_open/trigger cat /sys/kernel/debug/tracing/events/syscalls/sys_enter_perf_event_open/trigger
echo '!enable_event:kmem:kmalloc:1' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_perf_event_open/trigger cat /sys/kernel/debug/tracing/events/syscalls/sys_enter_perf_event_open/trigger

---- multi-event trigger test, two enables
- verify multiple enable_event triggers on single event

echo > /sys/kernel/debug/tracing/trace

# registers syscall tracepoint
echo 'enable_event:kmem:kmalloc:1' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_perf_event_open/trigger
echo 'enable_event:kmem:kfree:1' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_perf_event_open/trigger
cat /sys/kernel/debug/tracing/events/syscalls/sys_enter_perf_event_open/trigger

# do a perf stat to test

cat /sys/kernel/debug/tracing/trace

cat /sys/kernel/debug/tracing/events/kmem/kmalloc/enable cat /sys/kernel/debug/tracing/events/kmem/kfree/enable

echo 0 > /sys/kernel/debug/tracing/events/kmem/kmalloc/enable echo 0 > /sys/kernel/debug/tracing/events/kmem/kfree/enable

echo '!enable_event:kmem:kmalloc:1' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_perf_event_open/trigger cat /sys/kernel/debug/tracing/events/syscalls/sys_enter_perf_event_open/trigger
echo '!enable_event:kmem:kfree:1' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_perf_event_open/trigger cat /sys/kernel/debug/tracing/events/syscalls/sys_enter_perf_event_open/trigger

---- multi-event trigger test, two enables, one with filter
- verify multiple enable_event triggers with filters
- also verify when only one triggers, other filtered out
- also verify no multiple same-type events

echo > /sys/kernel/debug/tracing/trace

# registers syscall tracepoint
echo 'enable_event:kmem:kmalloc:1 if common_preempt_count == 2' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_perf_event_open/trigger
# just to test that same event set with different filter or same event with different count can't be set
echo 'enable_event:kmem:kmalloc:1 if common_preempt_count == 1' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_perf_event_open/trigger
echo 'enable_event:kmem:kmalloc if common_preempt_count == 1' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_perf_event_open/trigger
  echo: write error: File exists
cat /sys/kernel/debug/tracing/events/syscalls/sys_enter_perf_event_open/trigger
cat /sys/kernel/debug/tracing/events/syscalls/sys_enter_perf_event_open/enable
cat /sys/kernel/debug/tracing/events/kmem/kmalloc/enable
echo 'enable_event:kmem:kfree:1' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_perf_event_open/trigger
cat /sys/kernel/debug/tracing/events/syscalls/sys_enter_perf_event_open/trigger

# do a perf stat to test
cat /sys/kernel/debug/tracing/trace

echo '!enable_event:kmem:kmalloc:1 if common_preempt_count == 2' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_perf_event_open/trigger echo 'enable_event:kmem:kmalloc:1 if common_preempt_count == 1' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_perf_event_open/trigger cat /sys/kernel/debug/tracing/events/syscalls/sys_enter_perf_event_open/trigger

# do a perf stat to test
cat /sys/kernel/debug/tracing/trace

echo '!enable_event:kmem:kmalloc:1 if common_preempt_count == 1' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_perf_event_open/trigger echo 'enable_event:kmem:kmalloc:1 if common_preempt_count <= 1' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_perf_event_open/trigger cat /sys/kernel/debug/tracing/events/syscalls/sys_enter_perf_event_open/trigger

# do a perf stat to test
cat /sys/kernel/debug/tracing/trace

echo '!enable_event:kmem:kmalloc:1' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_perf_event_open/trigger cat /sys/kernel/debug/tracing/events/syscalls/sys_enter_perf_event_open/trigger
echo '!enable_event:kmem:kfree:1' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_perf_event_open/trigger cat /sys/kernel/debug/tracing/events/syscalls/sys_enter_perf_event_open/trigger

cat /sys/kernel/debug/tracing/events/kmem/kmalloc/enable cat /sys/kernel/debug/tracing/events/kmem/kfree/enable

echo 0 > /sys/kernel/debug/tracing/events/kmem/kmalloc/enable echo 0 > /sys/kernel/debug/tracing/events/kmem/kfree/enable

---- multi-event trigger test, two non-enable
- multiple non-enable/disable triggers

echo > /sys/kernel/debug/tracing/trace

# registers syscall tracepoint
echo 'stacktrace' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_perf_event_open/trigger
echo 'snapshot' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_perf_event_open/trigger
cat /sys/kernel/debug/tracing/events/syscalls/sys_enter_perf_event_open/trigger

# do a perf stat to test
cat /sys/kernel/debug/tracing/trace
cat /sys/kernel/debug/tracing/snapshot

echo '!stacktrace' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_perf_event_open/trigger cat /sys/kernel/debug/tracing/events/syscalls/sys_enter_perf_event_open/trigger echo '!snapshot' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_perf_event_open/trigger cat /sys/kernel/debug/tracing/events/syscalls/sys_enter_perf_event_open/trigger

---- multi-event trigger test, two non-enable on non-syscall, one with filter
- verify multiple triggers on non-syscall event
- one with filter
- general removal syntax on counted

echo > /sys/kernel/debug/tracing/trace

# registers syscall tracepoint
echo 'stacktrace:5' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
echo 'snapshot:5 if bytes_req >= 4096' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger

cat /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger

cat /sys/kernel/debug/tracing/trace
cat /sys/kernel/debug/tracing/snapshot

echo '!stacktrace' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger cat /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
echo '!snapshot' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger cat /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger

---- kprobes - basic check
- stacktrace triggered by kprobe

echo > /sys/kernel/debug/tracing/trace
perf probe -a in_kmalloc=__kmalloc
echo 'stacktrace' > /sys/kernel/debug/tracing/events/probe/in_kmalloc/trigger perf record -e probe:in_kmalloc -aR sleep 10&
perf report
cat /sys/kernel/debug/tracing/trace
cat /sys/kernel/debug/tracing/events/probe/in_kmalloc/trigger echo '!stacktrace' > /sys/kernel/debug/tracing/events/probe/in_kmalloc/trigger perf probe -d in_kmalloc

---- kprobes - enable normal event from kprobe event

echo > /sys/kernel/debug/tracing/trace
perf probe -a in_kmalloc=__kmalloc
echo 'enable_event:kmem:kfree:1' > /sys/kernel/debug/tracing/events/probe/in_kmalloc/trigger cat /sys/kernel/debug/tracing/trace
cat /sys/kernel/debug/tracing/events/probe/in_kmalloc/trigger
echo '!enable_event:kmem:kfree:1' > /sys/kernel/debug/tracing/events/probe/in_kmalloc/trigger cat /sys/kernel/debug/tracing/events/kmem/kfree/enable
echo 0 > /sys/kernel/debug/tracing/events/kmem/kfree/enable
perf probe -d in_kmalloc

---- kprobes - enable kprobe event from normal event

echo > /sys/kernel/debug/tracing/trace
perf probe -a in_kmalloc=__kmalloc
echo 'enable_event:probe:in_kmalloc:1' > /sys/kernel/debug/tracing/events/kmem/kfree/trigger cat /sys/kernel/debug/tracing/trace
cat /sys/kernel/debug/tracing/events/kmem/kfree/trigger
cat /sys/kernel/debug/tracing/events/probe/in_kmalloc/enable
echo 0 > /sys/kernel/debug/tracing/events/probe/in_kmalloc/enable
echo '!enable_event:probe:in_kmalloc:1' > /sys/kernel/debug/tracing/events/kmem/kfree/trigger perf probe -d in_kmalloc

Thanks, I may look at these in the future to update my tests. I've attached some trigger tests that I just wrote up.

ftrace-event-traceon-off : tests the traceon and traceoff triggers.
- enables tracing, sets traceoff trigger, checks if it disables
- disables tracing, sets traceon trigger, checks if it enables
- does a countdown from 3 to make sure the tracing is disabled 3 times.

ftrace-event-stacktrace : tests the event stack trace. It
- checks enabling of events (non trigger)
- checks enabling only the stack trace
- checks enabling both the stack trace and events
- checks if the count works from 1 2 or 3 times

ftrace-event-trigger-event : tests enable/disable_event trigger
- checks if the event enables another event (several times)
- checks if the event disables another event (several times)
- checks if it enables an event count times "1 2 3"
- then it tests the filter part of the event trigger (enable only)

Feel free to add onto these.

-- Steve

I have some basic tests, attached is one. Which failed. Here's the patch that fixes it:

-- Steve

tracing: Fix counter for traceon/off event triggers

The counters for the traceon and traceoff are only suppose to decrement when the trigger enables or disables tracing. It is not suppose to decrement every time the event is hit.

Only decrement the counter if the trigger actually did something.

---
 kernel/trace/trace_events_trigger.c | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

index f6dd115..a53e0da 100644
@@ -742,13 +742,16 @@ traceon_trigger(struct event_trigger_data *data)
 static void
 traceon_count_trigger(struct event_trigger_data *data)
 {
+	if (tracing_is_on())
+		return;
+
 	if (!data->count)
 		return;
 
 	if (data->count != -1)
 		(data->count)--;
 
-	traceon_trigger(data);
+	tracing_on();
 }
 
 static void

@@ -763,13 +766,16 @@ traceoff_trigger(struct event_trigger_data *data)
 static void
 traceoff_count_trigger(struct event_trigger_data *data)
 {
+	if (!tracing_is_on())
+		return;
+
 	if (!data->count)
 		return;
 
 	if (data->count != -1)
 		(data->count)--;
 
-	traceoff_trigger(data);
+	tracing_off();
 }
 
 static int

1.8.1.4