trace: Add trace any kernel object
RFC PATCH v4 1/2 trace: Add trace any kernel object
Introduce a method based on function tracer to trace any object and get the value of the object dynamically. the object can from the dynamic event (kprobe_event/uprobe_event) or the static event(tracepoint).
Usage:
When using the kprobe event, only need to set the objtrace(a new trigger), we can get the value of the object. The object is from the setting of the kprobe event.
For example:
For the function bio_add_page():
int bio_add_page(struct bio *bio, struct page *page,
unsigned int len, unsigned int offset)
Firstly, we can set the base of the object, thus the first string "arg1" stands for the value of the first parameter of this function bio_add_gage(),
echo 'p bio_add_page arg1=$arg1' >> ./kprobe_events
Secondly, we can get the value dynamically base the above object.
find the offset of the bi_size in struct bio:
$ gdb vmlinux (gdb) p &(((struct bio *)0)->bi_iter.bi_size) $1 = (unsigned int *) 0x28
echo 'objtrace:0x28(arg1):u32:1 if comm == "cat"' > ./events/kprobes/ \
p_bio_add_page_0/trigger
The best way to use this is that we can set the entrance event and exit event, for example, the following example is to set the read_papes as the entrance event, and set the blk_account_io_start as the exit event.
dd if=/dev/zero of=/test.txt bs=4k count=3
cd /sys/kernel/debug/tracing/
echo 0 > ./tracing_on
echo 'p read_pages' >> ./kprobe_events
echo 'p blk_account_io_start' >> ./kprobe_events
echo 'traceon if comm == "cat"' > ./events/kprobes/p_read_pages_0/trigger
echo 'traceoff if comm == "cat"' > ./events/kprobes/p_blk_account_io_start_0/trigger echo 'p bio_add_page arg1=$arg1' >> ./kprobe_events
echo 'objtrace:0x28(arg1):u32:1 if comm == "cat"' > ./events/kprobes/p_bio_add_page_0/trigger cat /test.txt > /dev/null
cat ./trace
# tracer: nop
#
# entries-in-buffer/entries-written: 50/50 #P:1
#
# _-----=> irqs-off
# / _----=> need-resched
# | / _---=> hardirq/softirq
# || / _--=> preempt-depth
# ||| / _-=> migrate-disable
# |||| / delay
# TASK-PID CPU# ||||| TIMESTAMP FUNCTION
# | | | ||||| | |
cat-95 [000] ..... 1.412065: _raw_spin_unlock_irqrestore <-event_triggers_call object:0xffff888108af6328 value:0x0
cat-95 [000] ..... 1.412066: __bio_try_merge_page <-bio_add_page object:0xffff888108af6328 value:0x0
cat-95 [000] ..... 1.412066: __bio_add_page <-bio_add_page object:0xffff888108af6328 value:0x0
cat-95 [000] ..... 1.412066: rcu_read_unlock_strict <-xa_load object:0xffff888108af6328 value:0x1000
cat-95 [000] ..... 1.412066: bio_add_page <-ext4_mpage_readpages object:0xffff888108af6328 value:0x1000
cat-95 [000] ..... 1.412066: kprobe_ftrace_handler <-ftrace_ops_list_func object:0xffff888108af6328 value:0x1000
cat-95 [000] ..... 1.412067: get_kprobe <-kprobe_ftrace_handler object:0xffff888108af6328 value:0x1000
cat-95 [000] ..... 1.412067: __bio_try_merge_page <-bio_add_page object:0xffff888108af6328 value:0x1000
cat-95 [000] ..... 1.412067: __bio_add_page <-bio_add_page object:0xffff888108af6328 value:0x1000
cat-95 [000] ..... 1.412067: rcu_read_unlock_strict <-xa_load object:0xffff888108af6328 value:0x2000
cat-95 [000] ..... 1.412067: bio_add_page <-ext4_mpage_readpages object:0xffff888108af6328 value:0x2000
cat-95 [000] ..... 1.412067: kprobe_ftrace_handler <-ftrace_ops_list_func object:0xffff888108af6328 value:0x2000
cat-95 [000] ..... 1.412067: get_kprobe <-kprobe_ftrace_handler object:0xffff888108af6328 value:0x2000
cat-95 [000] ..... 1.412067: __bio_try_merge_page <-bio_add_page object:0xffff888108af6328 value:0x2000
cat-95 [000] ..... 1.412068: submit_bio <-ext4_mpage_readpages object:0xffff888108af6328 value:0x3000
cat-95 [000] ..... 1.412068: submit_bio_noacct <-ext4_mpage_readpages object:0xffff888108af6328 value:0x3000
cat-95 [000] ..... 1.412068: __submit_bio <-submit_bio_noacct object:0xffff888108af6328 value:0x3000
cat-95 [000] ..... 1.412068: blk_try_enter_queue <-__submit_bio object:0xffff888108af6328 value:0x3000
cat-95 [000] ..... 1.412068: rcu_read_unlock_strict <-blk_try_enter_queue object:0xffff888108af6328 value:0x3000
cat-95 [000] ..... 1.412068: rcu_read_unlock_strict <-blk_try_enter_queue object:0xffff888108af6328 value:0x3000
cat-95 [000] ..... 1.412068: submit_bio_checks <-__submit_bio object:0xffff888108af6328 value:0x3000
cat-95 [000] ..... 1.412068: __cond_resched <-submit_bio_checks object:0xffff888108af6328 value:0x3000
cat-95 [000] ..... 1.412068: rcu_all_qs <-__cond_resched object:0xffff888108af6328 value:0x3000
cat-95 [000] ..... 1.412068: should_fail_bio <-submit_bio_checks object:0xffff888108af6328 value:0x3000
cat-95 [000] ..... 1.412069: create_task_io_context <-submit_bio_checks object:0xffff888108af6328 value:0x3000
cat-95 [000] ..... 1.412069: kmem_cache_alloc_node <-create_task_io_context object:0xffff888108af6328 value:0x3000
cat-95 [000] ..... 1.412069: should_failslab <-kmem_cache_alloc_node object:0xffff888108af6328 value:0x3000
cat-95 [000] ..... 1.412069: _raw_spin_lock <-create_task_io_context object:0xffff888108af6328 value:0x3000
cat-95 [000] ..... 1.412069: blk_mq_submit_bio <-__submit_bio object:0xffff888108af6328 value:0x3000
cat-95 [000] ..... 1.412069: __blk_queue_split <-blk_mq_submit_bio object:0xffff888108af6328 value:0x3000
cat-95 [000] ..... 1.412069: bvec_split_segs <-__blk_queue_split object:0xffff888108af6328 value:0x3000
cat-95 [000] ..... 1.412069: blk_attempt_plug_merge <-blk_mq_submit_bio object:0xffff888108af6328 value:0x3000
cat-95 [000] ..... 1.412070: __blk_mq_sched_bio_merge <-blk_mq_submit_bio object:0xffff888108af6328 value:0x3000
cat-95 [000] ..... 1.412070: dd_bio_merge <-blk_mq_submit_bio object:0xffff888108af6328 value:0x3000
cat-95 [000] ..... 1.412070: _raw_spin_lock <-dd_bio_merge object:0xffff888108af6328 value:0x3000
cat-95 [000] ..... 1.412070: blk_mq_sched_try_merge <-dd_bio_merge object:0xffff888108af6328 value:0x3000
cat-95 [000] ..... 1.412070: elv_merge <-blk_mq_sched_try_merge object:0xffff888108af6328 value:0x3000
cat-95 [000] ..... 1.412070: elv_rqhash_find <-elv_merge object:0xffff888108af6328 value:0x3000
cat-95 [000] ..... 1.412070: dd_request_merge <-blk_mq_sched_try_merge object:0xffff888108af6328 value:0x3000
cat-95 [000] ..... 1.412070: elv_rb_find <-dd_request_merge object:0xffff888108af6328 value:0x3000
cat-95 [000] ..... 1.412070: __blk_mq_alloc_request <-blk_mq_submit_bio object:0xffff888108af6328 value:0x3000
cat-95 [000] ..... 1.412071: dd_limit_depth <-__blk_mq_alloc_request object:0xffff888108af6328 value:0x3000
cat-95 [000] ..... 1.412071: blk_mq_get_tag <-__blk_mq_alloc_request object:0xffff888108af6328 value:0x3000
cat-95 [000] ..... 1.412071: __blk_mq_get_tag <-blk_mq_get_tag object:0xffff888108af6328 value:0x3000
cat-95 [000] ..... 1.412071: blk_mq_rq_ctx_init.isra.0 <-blk_mq_submit_bio object:0xffff888108af6328 value:0x3000
cat-95 [000] ..... 1.412071: ktime_get <-blk_mq_rq_ctx_init.isra.0 object:0xffff888108af6328 value:0x3000
cat-95 [000] ..... 1.412071: dd_prepare_request <-blk_mq_rq_ctx_init.isra.0 object:0xffff888108af6328 value:0x3000
cat-95 [000] ..... 1.412071: blk_account_io_start <-blk_mq_submit_bio object:0xffff888108af6328 value:0x3000
cat-95 [000] ..... 1.412071: kprobe_ftrace_handler <-ftrace_ops_list_func object:0xffff888108af6328 value:0x3000
cat-95 [000] ..... 1.412071: get_kprobe <-kprobe_ftrace_handler object:0xffff888108af6328 value:0x3000--- include/linux/trace_events.h | 1 + kernel/trace/Kconfig | 9 + kernel/trace/Makefile | 1 + kernel/trace/trace.h | 8 + kernel/trace/trace_entries.h | 18 + kernel/trace/trace_events_trigger.c | 1 + kernel/trace/trace_object.c | 705 ++++++++++++++++++++++++++++ kernel/trace/trace_output.c | 40 ++ 8 files changed, 783 insertions(+) create mode 100644 kernel/trace/trace_object.c
include/linux/trace_events.h +1 -0
@@ -684,6 +684,7 @@ enum event_trigger_type {ETT_EVENT_HIST = (1 <<4 ),ETT_HIST_ENABLE = (1 <<5 ),ETT_EVENT_EPROBE = (1 <<6 ), + ETT_TRACE_OBJECT = (1 << 7), };extern int filter_match_preds (struct event_filter *filter,void *rec);
kernel/trace/Kconfig +9 -0
@@ -237,6 +237,15 @@ config FUNCTION_PROFILER If in doubt, say N. +config TRACE_OBJECT + bool "Trace kernel object" + depends on FUNCTION_TRACER + default y + help + This help kernel developer to trace any kernel object, and get the value + of the object dynamically. the object can from the dynamic event(kprobe_event/ + uprobe_event) or the static event(tracepoint). + config STACK_TRACER bool "Trace max stack" depends on HAVE_FUNCTION_TRACER
kernel/trace/Makefile +1 -0
@@ -66,6 +66,7 @@ obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += trace_functions_graph.o obj-$(CONFIG_TRACE_BRANCH_PROFILING) += trace_branch.o obj-$(CONFIG_BLK_DEV_IO_TRACE) += blktrace.o obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += fgraph.o +obj-$(CONFIG_TRACE_OBJECT) += trace_object.o ifeq ($(CONFIG_BLOCK),y) obj-$(CONFIG_EVENT_TRACING) += blktrace.o endif
kernel/trace/trace.h +8 -0
@@ -49,6 +49,7 @@ enum trace_type {TRACE_TIMERLAT ,TRACE_RAW_DATA ,TRACE_FUNC_REPEATS , +TRACE_OBJECT , __TRACE_LAST_TYPE, };
@@ -460,6 +461,7 @@ extern void __ftrace_bad_type(void); TRACE_GRAPH_RET); \ IF_ASSIGN(var, ent, struct func_repeats_entry, \TRACE_FUNC_REPEATS ); \ + IF_ASSIGN(var, ent, struct trace_object_entry, TRACE_OBJECT);\__ftrace_bad_type (); \ }while (0 )
@@ -1510,6 +1512,12 @@ static inline int register_trigger_hist_cmd(void) { return 0; }static inline int register_trigger_hist_enable_disable_cmds (void ) {return 0 ; }#endif +#ifdef CONFIG_TRACE_OBJECT +extern int register_trigger_object_cmd (void ); +#else +static inline int register_trigger_object_cmd (void ) {return 0 ; } +#endif +extern int register_trigger_cmds (void );extern void clear_event_triggers (struct trace_array *tr);
kernel/trace/trace_entries.h +18 -0
@@ -401,3 +401,21 @@ FTRACE_ENTRY(timerlat, timerlat_entry, __entry->context, __entry->timer_latency) ); + +/* +* trace object entry: +*/ +FTRACE_ENTRY (object, trace_object_entry, + +TRACE_OBJECT , + +F_STRUCT ( + __field( unsigned long, ip ) + __field( unsigned long, parent_ip ) + __field( unsigned long, object ) + __field( unsigned long, value ) + ), + +F_printk (" %ps <-- %ps object:%lx value=%lx\n" , (void *)__entry->ip, + (void *)__entry->parent_ip, __entry->object, __entry->value) +);
kernel/trace/trace_events_trigger.c +1 -0
@@ -1687,6 +1687,7 @@ __init int register_trigger_cmds(void)register_trigger_enable_disable_cmds ();register_trigger_hist_enable_disable_cmds ();register_trigger_hist_cmd (); +register_trigger_object_cmd ();return 0 ; }
kernel/trace/trace_object.c new file +705 -0
705 lines not on this page · raw message
kernel/trace/trace_output.c +40 -0
@@ -1552,6 +1552,45 @@ static struct trace_event trace_func_repeats_event = { .funcs = &trace_func_repeats_funcs, }; +/* TRACE_OBJECT */ +static enum print_line_t trace_object_print (struct trace_iterator *iter,int flags, +struct trace_event *event) +{ +struct trace_object_entry *field; +struct trace_seq *s = &iter->seq; + +trace_assign_type (field, iter->ent); +print_fn_trace (s, field->ip, field->parent_ip, flags); +trace_seq_printf (s," object:0x%lx" , field->object); +trace_seq_printf (s," value:0x%lx" , field->value); +trace_seq_putc (s,'\n' ); + +return trace_handle_return (s); +} + +static enum print_line_t trace_object_raw (struct trace_iterator *iter,int flags, +struct trace_event *event) +{ +struct trace_object_entry *field; + +trace_assign_type (field, iter->ent); + +trace_seq_printf (&iter->seq,"%lx %lx %lx\n" , + field->ip, field->parent_ip, field->value); + +return trace_handle_return (&iter->seq); +} + +static struct trace_event_functions trace_object_funcs = { + .trace = trace_object_print, + .raw = trace_object_raw, +}; + +static struct trace_event trace_object_event = { + .type =TRACE_OBJECT , + .funcs = &trace_object_funcs, +}; +static struct trace_event *events[]__initdata = { &trace_fn_event, &trace_ctx_event,
@@ -1566,6 +1605,7 @@ static struct trace_event *events[] __initdata = { &trace_timerlat_event, &trace_raw_data_event, &trace_func_repeats_event, + &trace_object_event, NULL };
Signature · 1 line
2.25.1
Steven Rostedt
maintainerQuoted 8 lines · On Mon, 1 Nov 2021 23:07:59 +0800 Jeff Xie <[email protected]> wrote:
Introduce a method based on function tracer to trace any object and get the value of the object dynamically. the object can from the dynamic event (kprobe_event/uprobe_event) or the static event(tracepoint).
Usage:
When using the kprobe event, only need to set the objtrace(a new trigger), we can get the value of the object. The object is from the setting of the kprobe event.
Hi Jeff,
I just wanted to let you know that because the merge window just opened (Linus released 5.15 and is now accepting changes for 5.16), I most likely wont have any time to review your changes while the merge window is open (for the next two weeks).
Please do not take my lack of responses during this time as ignoring you. I'm just going to be overwhelmed with getting things right, and this merge window is going to be a bit tougher for me than other merge windows.
Thanks,
-- Steve
Quoted 20 lines · Steven Rostedt
Quoted 8 lines · On Mon, 1 Nov 2021 23:07:59 +0800 Jeff Xie <[email protected]> wrote:
Introduce a method based on function tracer to trace any object and get the value of the object dynamically. the object can from the dynamic event (kprobe_event/uprobe_event) or the static event(tracepoint).
Usage:
When using the kprobe event, only need to set the objtrace(a new trigger), we can get the value of the object. The object is from the setting of the kprobe event.Hi Jeff,
I just wanted to let you know that because the merge window just opened (Linus released 5.15 and is now accepting changes for 5.16), I most likely wont have any time to review your changes while the merge window is open (for the next two weeks).
Quoted 7 lines
Please do not take my lack of responses during this time as ignoring you. I'm just going to be overwhelmed with getting things right, and this merge window is going to be a bit tougher for me than other merge windows.
Thanks,
-- Steve
I’m very glad you can explain this and let me know. because the previous patch[PATCH V3] lacks some functions.
Just update it.
Thanks.
---
JeffXie
kernel test robot
build bot[If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in https://git-scm.com/docs/git-format-patch]
Hi Jeff,
[FYI, it's a private test report for your RFC patch.]
[auto build test ERROR on linux/master]
[also build test ERROR on linus/master v5.15 next-20211104] [cannot apply to tip/perf/core]
[If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]
url: https://github.com/0day-ci/linux/commits/Jeff-Xie/trace-Add-trace-any-kernel-object/20211101-231428 base: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git 2f111a6fd5b5297b4e92f53798ca086f7c7d33a4 config: i386-buildonly-randconfig-r002-20211101 (attached as .config) compiler: clang version 14.0.0 (https://github.com/llvm/llvm-project 264d3b6d4e08401c5b50a85bd76e80b3461d77e6)
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# https://github.com/0day-ci/linux/commit/5c8589956ecad7f8aeec51f42e6b9012d38be7f7
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Jeff-Xie/trace-Add-trace-any-kernel-object/20211101-231428 git checkout 5c8589956ecad7f8aeec51f42e6b9012d38be7f7
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross W=1 ARCH=i386
If you fix the issue, kindly add following tag as appropriate
All errors (new ones prefixed by >>):
>> kernel/trace/trace_object.c:445:6: error: no previous prototype for function 'unregister_object_trigger' [-Werror,-Wmissing-prototypes]
void unregister_object_trigger(char *glob, struct event_trigger_ops *ops,
^
kernel/trace/trace_object.c:445:1: note: declare 'static' if the function is not intended to be used outside of this translation unit
void unregister_object_trigger(char *glob, struct event_trigger_ops *ops,
^
static
>> kernel/trace/trace_object.c:689:6: error: variable 'ret' is used uninitialized whenever 'if' condition is true [-Werror,-Wsometimes-uninitialized]
if (WARN_ON_ONCE(atomic_read(&trace_object_ref) <= 0))
^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
include/asm-generic/bug.h:103:33: note: expanded from macro 'WARN_ON_ONCE'
#define WARN_ON_ONCE(condition) ({ \
^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
kernel/trace/trace_object.c:704:9: note: uninitialized use occurs here
return ret;
^~~
kernel/trace/trace_object.c:689:2: note: remove the 'if' if its condition is always false
if (WARN_ON_ONCE(atomic_read(&trace_object_ref) <= 0))
^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
kernel/trace/trace_object.c:687:9: note: initialize the variable 'ret' to silence this warning
int ret;
^
= 02 errors generated.
vim +/unregister_object_trigger +445 kernel/trace/trace_object.c
444
> 445 void unregister_object_trigger(char *glob, struct event_trigger_ops *ops,
446 struct event_trigger_data *test,
447 struct trace_event_file *file)
448 {
449 struct event_trigger_data *data;
450 bool unregistered = false;
451
452 lockdep_assert_held(&event_mutex);
453
454 list_for_each_entry(data, &file->triggers, list) {
455 if (data->cmd_ops->trigger_type == test->cmd_ops->trigger_type) {
456 unregistered = true;
457 list_del_rcu(&data->list);
458 trace_event_trigger_enable_disable(file, 0);
459 update_cond_flag(file);
460 break;
461 }
462 }
463
464 if (unregistered && data->ops->free) {
465 data->ops->free(data->ops, data);
466 exit_trace_object();
467 }
468 }
469
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org