tracing: add per-event filtering
RFC PATCH 3/4 tracing: add per-event filtering
This patch adds per-event filtering to the event tracing subsystem.
It adds a 'filter' debugfs file to each event directory. This file can be written to to set filters; reading from it will display the current set of filters set for that event.
Basically, any field listed in the 'format' file for an event can be filtered on (including strings, but not yet other array types) using either matching ('==') or non-matching ('!=') 'predicates'. A 'predicate' can be either a single expression:
# echo pid != 0 > filter
# cat filter pid != 0
or a compound expression of up to 8 sub-expressions combined using 'and' or 'or':
# echo comm == Xorg > filter # echo and sig != 29 > filter
# cat filter comm == Xorg and sig != 29
Only events having field values matching an expression will be available in the trace output; non-matching events are discarded.
Note that a compound expression is built up by echoing each sub-expression separately - it's not the most efficient way to do things, but it keeps the parser simple and assumes that compound expressions will be relatively uncommon. In any case, a subsequent patch introducing a way to set filters for entire subsystems should mitigate any need to do this for lots of events.
Setting a filter without an 'and' or 'or' clears the previous filter completely and sets the filter to the new expression:
# cat filter comm == Xorg and sig != 29
# echo comm != Xorg
#cat filter
comm != Xorg
To clear a filter, echo 0 to the filter file:
# echo 0 > filter # cat filter none
The limit of 8 predicates for a compound expression is arbitrary - for efficiency, it's implemented as an array of pointers to predicates, and 8 seemed more than enough for any filter...
NOTE: As mentioned in patch 1, to filter on a common field i.e. the first 5 fields listed in the format file, you currently need to prepend an underscore to the field name, since there can be duplicate field names in a format file.
--- kernel/trace/Makefile | 1 + kernel/trace/trace.h | 28 +++ kernel/trace/trace_events.c | 77 +++++++++ kernel/trace/trace_events_filter.c | 320 +++++++++++++++++++++++++++++++++++ kernel/trace/trace_events_stage_3.h | 4 + 5 files changed, 430 insertions(+), 0 deletions(-) create mode 100644 kernel/trace/trace_events_filter.c
kernel/trace/Makefile +1 -0
@@ -44,5 +44,6 @@ obj-$(CONFIG_EVENT_TRACER) += trace_events.o obj-$(CONFIG_EVENT_TRACER) += events.o obj-$(CONFIG_EVENT_TRACER) += trace_export.o obj-$(CONFIG_FTRACE_SYSCALLS) += trace_syscalls.o +obj-$(CONFIG_EVENT_TRACER) += trace_events_filter.o libftrace-y := ftrace.o
kernel/trace/trace.h +28 -0
@@ -796,10 +796,38 @@ struct ftrace_event_call {int (*show_format)(struct trace_seq *s);int (*define_fields)(void );struct list_head fields; +struct filter_pred **preds; +}; + +#define MAX_FILTER_PRED 8 + +struct filter_pred ; + +typedef int (*filter_pred_fn_t ) (struct filter_pred *pred,void *event); + +struct filter_pred { +filter_pred_fn_t fn; +u64 val; +char *str_val; +int str_len; +char *field_name; +int offset; +int not; +int or; +int compound; +int clear; };int trace_define_field (struct ftrace_event_call *call,char *type,char *name,int offset,int size); +extern void filter_free_pred (struct filter_pred *pred); +extern int filter_print_preds (struct filter_pred **preds,char *buf); +extern int filter_parse (char **pbuf,struct filter_pred *pred); +extern int filter_add_pred (struct ftrace_event_call *call, +struct filter_pred *pred); +extern void filter_free_preds (struct ftrace_event_call *call); +extern int filter_match_preds (struct ftrace_event_call *call,void *rec); +void event_trace_printk (unsigned long ip,const char *fmt, ...);extern struct ftrace_event_call __start_ftrace_events[];extern struct ftrace_event_call __stop_ftrace_events[];
kernel/trace/trace_events.c +77 -0
95 lines not on this page · raw message
kernel/trace/trace_events_filter.c new file +320 -0
320 lines not on this page · raw message
kernel/trace/trace_events_stage_3.h +4 -0
@@ -154,6 +154,7 @@ static struct ftrace_event_call event_##call; \ \ static void ftrace_raw_event_##call(proto) \ { \ + struct ftrace_event_call *call = &event_##call; \ struct ring_buffer_event *event; \ struct ftrace_raw_##call *entry; \ unsigned long irq_flags; \
@@ -172,6 +173,9 @@ static void ftrace_raw_event_##call(proto) \ assign; \ \trace_current_buffer_unlock_commit (event, irq_flags, pc); \ + \ +if (call->preds && !filter_match_preds (call, entry)) \ +rb_event_discard (event); \ } \ \ static int ftrace_raw_reg_event_##call(void) \
Signature · 1 line
1.5.6.3
Steven Rostedt
maintainerQuoted 13 lines · Tom Zanussi
kernel/trace/trace_events_stage_3.h +1
-0@@ -154,6 +154,7 @@ static struct ftrace_event_call event_##call; \ \ static void ftrace_raw_event_##call(proto) \ { \ + struct ftrace_event_call *call = &event_##call; \ struct ring_buffer_event *event; \ struct ftrace_raw_##call *entry; \ unsigned long irq_flags; \
kernel/trace/trace_events_stage_3.h static void ftrace_raw_event_##call(proto) \ · line 175
Quoted 3 lines
kernel/trace/trace_events_stage_3.h +0
-0@@ -172,6 +173,9 @@ static void ftrace_raw_event_##call(proto) \ assign; \ \trace_current_buffer_unlock_commit (event, irq_flags, pc); \
I have a concern hear about races. Once the commit happens, there's nothing protecting us from being preempted. The reader could in fact come in and consume the commit, and a new write may happen, and this discard will corrupt the data in the buffer. Also in overwrite mode, the writer itself can over write this.
Would you be able to move the above unlock_commit after this. I'll make the necessary changes in the commit to check for discarded data.
Thanks,
-- Steve
kernel/trace/trace_events_stage_3.h static void ftrace_raw_event_##call(proto) \ · line 176
Quoted 1 line
kernel/trace/trace_events_stage_3.h +1
-0+
\ > + if (call->preds && !filter_match_preds(call, entry)) \
Quoted 13 lines
kernel/trace/trace_events_stage_3.h +1
-0+rb_event_discard (event); \ } \ \ static int ftrace_raw_reg_event_##call(void) \Signature · 1 line
1.5.6.3
Quoted 26 lines · Steven Rostedt
Quoted 13 lines · Tom Zanussi
kernel/trace/trace_events_stage_3.h +1
-0@@ -154,6 +154,7 @@ static struct ftrace_event_call event_##call; \ \ static void ftrace_raw_event_##call(proto) \ { \ + struct ftrace_event_call *call = &event_##call; \ struct ring_buffer_event *event; \ struct ftrace_raw_##call *entry; \ unsigned long irq_flags; \kernel/trace/trace_events_stage_3.h static void ftrace_raw_event_##call(proto) \ · line 175
Quoted 3 lines
kernel/trace/trace_events_stage_3.h +0
-0@@ -172,6 +173,9 @@ static void ftrace_raw_event_##call(proto) \ assign; \ \trace_current_buffer_unlock_commit (event, irq_flags, pc); \I have a concern hear about races. Once the commit happens, there's nothing protecting us from being preempted. The reader could in fact come in and consume the commit, and a new write may happen, and this discard will corrupt the data in the buffer. Also in overwrite mode, the writer itself can over write this.
Quoted 3 lines
Would you be able to move the above
unlock_commitafter this. I'll make the necessary changes in the commit to check for discarded data.
Yeah, I had meant to do that, but was having enough trouble with the ring-buffer stuff that I wanted to avoid wondering what might happen if I discarded an uncommitted event at that point.
I'll make that change in the next version.
Tom
Quoted 21 lines
Thanks,
-- Steve
kernel/trace/trace_events_stage_3.h static void ftrace_raw_event_##call(proto) \ · line 176
Quoted 1 line
kernel/trace/trace_events_stage_3.h +1
-0+\ > + if (
call->preds&& !filter_match_preds(call, entry)) \Quoted 13 lines
kernel/trace/trace_events_stage_3.h +1
-0+rb_event_discard (event); \ } \ \ static int ftrace_raw_reg_event_##call(void) \Signature · 1 line
1.5.6.3