Skip to contentSkip to conversation

tracing: add per-event filtering

RFC PATCH In review Tom Zanussi 2 participants 3 messages Mar 17, 2009 – Mar 18, 2009 Tracing

lore
Share & tools
lore
#0 lore raw

RFC PATCH 3/4 tracing: add per-event filtering

This patch adds per-event filtering to the event tracing subsystem.

It adds a 'filter' debugfs file to each event directory. This file can be written to to set filters; reading from it will display the current set of filters set for that event.

Basically, any field listed in the 'format' file for an event can be filtered on (including strings, but not yet other array types) using either matching ('==') or non-matching ('!=') 'predicates'. A 'predicate' can be either a single expression:

# echo pid != 0 > filter
# cat filter
pid != 0

or a compound expression of up to 8 sub-expressions combined using 'and' or 'or':

# echo comm == Xorg > filter
# echo and sig != 29 > filter
# cat filter
comm == Xorg
and sig != 29

Only events having field values matching an expression will be available in the trace output; non-matching events are discarded.

Note that a compound expression is built up by echoing each sub-expression separately - it's not the most efficient way to do things, but it keeps the parser simple and assumes that compound expressions will be relatively uncommon. In any case, a subsequent patch introducing a way to set filters for entire subsystems should mitigate any need to do this for lots of events.

Setting a filter without an 'and' or 'or' clears the previous filter completely and sets the filter to the new expression:

# cat filter
comm == Xorg
and sig != 29
# echo comm != Xorg

#cat filter
comm != Xorg

To clear a filter, echo 0 to the filter file:

# echo 0 > filter
# cat filter
none

The limit of 8 predicates for a compound expression is arbitrary - for efficiency, it's implemented as an array of pointers to predicates, and 8 seemed more than enough for any filter...

NOTE: As mentioned in patch 1, to filter on a common field i.e. the first 5 fields listed in the format file, you currently need to prepend an underscore to the field name, since there can be duplicate field names in a format file.

---
 kernel/trace/Makefile               |    1 +
 kernel/trace/trace.h                |   28 +++
 kernel/trace/trace_events.c         |   77 +++++++++
 kernel/trace/trace_events_filter.c  |  320 +++++++++++++++++++++++++++++++++++
 kernel/trace/trace_events_stage_3.h |    4 +
 5 files changed, 430 insertions(+), 0 deletions(-)
 create mode 100644 kernel/trace/trace_events_filter.c
kernel/trace/Makefile +1 -0
index c3feea0..34d70f9 100644
@@ -44,5 +44,6 @@ obj-$(CONFIG_EVENT_TRACER) += trace_events.o
 obj-$(CONFIG_EVENT_TRACER) += events.o
 obj-$(CONFIG_EVENT_TRACER) += trace_export.o
 obj-$(CONFIG_FTRACE_SYSCALLS) += trace_syscalls.o
+obj-$(CONFIG_EVENT_TRACER) += trace_events_filter.o
 
 libftrace-y := ftrace.o
kernel/trace/trace.h +28 -0
index 6981e28..3081df5 100644
@@ -796,10 +796,38 @@ struct ftrace_event_call {
 	int			(*show_format)(struct trace_seq *s);
 	int			(*define_fields)(void);
 	struct list_head	fields;
+	struct filter_pred	**preds;
+};
+
+#define MAX_FILTER_PRED 8
+
+struct filter_pred;
+
+typedef int (*filter_pred_fn_t) (struct filter_pred *pred, void *event);
+
+struct filter_pred {
+	filter_pred_fn_t fn;
+	u64 val;
+	char *str_val;
+	int str_len;
+	char *field_name;
+	int offset;
+	int not;
+	int or;
+	int compound;
+	int clear;
 };
 
 int trace_define_field(struct ftrace_event_call *call, char *type,
 		       char *name, int offset, int size);
+extern void filter_free_pred(struct filter_pred *pred);
+extern int filter_print_preds(struct filter_pred **preds, char *buf);
+extern int filter_parse(char **pbuf, struct filter_pred *pred);
+extern int filter_add_pred(struct ftrace_event_call *call,
+			   struct filter_pred *pred);
+extern void filter_free_preds(struct ftrace_event_call *call);
+extern int filter_match_preds(struct ftrace_event_call *call, void *rec);
+
 void event_trace_printk(unsigned long ip, const char *fmt, ...);
 extern struct ftrace_event_call __start_ftrace_events[];
 extern struct ftrace_event_call __stop_ftrace_events[];
kernel/trace/trace_events.c +77 -0

95 lines not on this page · raw message

kernel/trace/trace_events_filter.c new file +320 -0

320 lines not on this page · raw message

kernel/trace/trace_events_stage_3.h +4 -0
index 30627d8..d556274 100644
@@ -154,6 +154,7 @@ static struct ftrace_event_call event_##call;				\
 									\
 static void ftrace_raw_event_##call(proto)				\
 {									\
+	struct ftrace_event_call *call = &event_##call;			\
 	struct ring_buffer_event *event;				\
 	struct ftrace_raw_##call *entry;				\
 	unsigned long irq_flags;					\
@@ -172,6 +173,9 @@ static void ftrace_raw_event_##call(proto)				\
 	assign;								\
 									\
 	trace_current_buffer_unlock_commit(event, irq_flags, pc);	\
+									\
+	if (call->preds && !filter_match_preds(call, entry))		\
+		rb_event_discard(event);				\
 }									\
 									\
 static int ftrace_raw_reg_event_##call(void)				\
Signature · 1 line
1.5.6.3
maintainer
#1 lore raw
Quoted 13 lines · Tom Zanussi
kernel/trace/trace_events_stage_3.h +1 -0
index 30627d8..d556274 100644
@@ -154,6 +154,7 @@ static struct ftrace_event_call event_##call;				\
 									\
 static void ftrace_raw_event_##call(proto)				\
 {									\
+	struct ftrace_event_call *call = &event_##call;			\
 	struct ring_buffer_event *event;				\
 	struct ftrace_raw_##call *entry;				\
 	unsigned long irq_flags;					\

kernel/trace/trace_events_stage_3.h static void ftrace_raw_event_##call(proto) \ · line 175

Quoted 3 lines
kernel/trace/trace_events_stage_3.h +0 -0
@@ -172,6 +173,9 @@ static void ftrace_raw_event_##call(proto)				\
 	assign;								\
 									\
 	trace_current_buffer_unlock_commit(event, irq_flags, pc);	\

I have a concern hear about races. Once the commit happens, there's nothing protecting us from being preempted. The reader could in fact come in and consume the commit, and a new write may happen, and this discard will corrupt the data in the buffer. Also in overwrite mode, the writer itself can over write this.

Would you be able to move the above unlock_commit after this. I'll make the necessary changes in the commit to check for discarded data.

Thanks,

-- Steve

kernel/trace/trace_events_stage_3.h static void ftrace_raw_event_##call(proto) \ · line 176

Quoted 1 line
kernel/trace/trace_events_stage_3.h +1 -0
+									

\ > + if (call->preds && !filter_match_preds(call, entry)) \

Quoted 13 lines
kernel/trace/trace_events_stage_3.h +1 -0
+		rb_event_discard(event);				\
 }									\
 									\
 static int ftrace_raw_reg_event_##call(void)				\
Signature · 1 line
1.5.6.3
#2 lore raw
Quoted 26 lines · Steven Rostedt
Quoted 13 lines · Tom Zanussi
kernel/trace/trace_events_stage_3.h +1 -0
index 30627d8..d556274 100644
@@ -154,6 +154,7 @@ static struct ftrace_event_call event_##call;				\
 									\
 static void ftrace_raw_event_##call(proto)				\
 {									\
+	struct ftrace_event_call *call = &event_##call;			\
 	struct ring_buffer_event *event;				\
 	struct ftrace_raw_##call *entry;				\
 	unsigned long irq_flags;					\

kernel/trace/trace_events_stage_3.h static void ftrace_raw_event_##call(proto) \ · line 175

Quoted 3 lines
kernel/trace/trace_events_stage_3.h +0 -0
@@ -172,6 +173,9 @@ static void ftrace_raw_event_##call(proto)				\
 	assign;								\
 									\
 	trace_current_buffer_unlock_commit(event, irq_flags, pc);	\

I have a concern hear about races. Once the commit happens, there's nothing protecting us from being preempted. The reader could in fact come in and consume the commit, and a new write may happen, and this discard will corrupt the data in the buffer. Also in overwrite mode, the writer itself can over write this.

Quoted 3 lines

Would you be able to move the above unlock_commit after this. I'll make the necessary changes in the commit to check for discarded data.

Yeah, I had meant to do that, but was having enough trouble with the ring-buffer stuff that I wanted to avoid wondering what might happen if I discarded an uncommitted event at that point.

I'll make that change in the next version.

Tom

Quoted 21 lines

Thanks,

-- Steve

kernel/trace/trace_events_stage_3.h static void ftrace_raw_event_##call(proto) \ · line 176

Quoted 1 line
kernel/trace/trace_events_stage_3.h +1 -0
+									

\ > + if (call->preds && !filter_match_preds(call, entry)) \

Quoted 13 lines
kernel/trace/trace_events_stage_3.h +1 -0
+		rb_event_discard(event);				\
 }									\
 									\
 static int ftrace_raw_reg_event_##call(void)				\
Signature · 1 line
1.5.6.3