Skip to contentSkip to conversation

trace: Add trace any kernel object

RFC PATCH Superseded Jeff Xie 3 participants 4 messages Nov 1, 2021 – Nov 4, 2021 Tracing

v4 of 15 Superseded Latest is v15 →

lore
Share & tools
lore

Jeff Xie

author
#0 lore raw

RFC PATCH v4 1/2 trace: Add trace any kernel object

Introduce a method based on function tracer to trace any object and get the value of the object dynamically. the object can from the dynamic event (kprobe_event/uprobe_event) or the static event(tracepoint).

Usage:
When using the kprobe event, only need to set the objtrace(a new trigger), we can get the value of the object. The object is from the setting of the kprobe event.

For example:
For the function bio_add_page():

int bio_add_page(struct bio *bio, struct page *page,
unsigned int len, unsigned int offset)

Firstly, we can set the base of the object, thus the first string "arg1" stands for the value of the first parameter of this function bio_add_gage(),

echo 'p bio_add_page arg1=$arg1' >> ./kprobe_events

Secondly, we can get the value dynamically base the above object.

find the offset of the bi_size in struct bio:

$ gdb vmlinux
(gdb) p &(((struct bio *)0)->bi_iter.bi_size)
$1 = (unsigned int *) 0x28

echo 'objtrace:0x28(arg1):u32:1 if comm == "cat"' > ./events/kprobes/ \
p_bio_add_page_0/trigger

The best way to use this is that we can set the entrance event and exit event, for example, the following example is to set the read_papes as the entrance event, and set the blk_account_io_start as the exit event.

dd if=/dev/zero of=/test.txt bs=4k count=3
cd /sys/kernel/debug/tracing/
echo 0 > ./tracing_on
echo 'p read_pages' >> ./kprobe_events
echo 'p blk_account_io_start' >> ./kprobe_events
echo 'traceon if comm == "cat"' > ./events/kprobes/p_read_pages_0/trigger
echo 'traceoff if comm == "cat"' > ./events/kprobes/p_blk_account_io_start_0/trigger echo 'p bio_add_page arg1=$arg1' >> ./kprobe_events
echo 'objtrace:0x28(arg1):u32:1 if comm == "cat"' > ./events/kprobes/p_bio_add_page_0/trigger cat /test.txt > /dev/null
cat ./trace

# tracer: nop
#
# entries-in-buffer/entries-written: 50/50 #P:1
#
# _-----=> irqs-off
# / _----=> need-resched

#                              | / _---=> hardirq/softirq
#                              || / _--=> preempt-depth
#                              ||| / _-=> migrate-disable
#                              |||| /     delay
#           TASK-PID     CPU#  |||||  TIMESTAMP  FUNCTION
#              | |         |   |||||     |         |
             cat-95      [000] .....     1.412065: _raw_spin_unlock_irqrestore <-event_triggers_call object:0xffff888108af6328 value:0x0
             cat-95      [000] .....     1.412066: __bio_try_merge_page <-bio_add_page object:0xffff888108af6328 value:0x0
             cat-95      [000] .....     1.412066: __bio_add_page <-bio_add_page object:0xffff888108af6328 value:0x0
             cat-95      [000] .....     1.412066: rcu_read_unlock_strict <-xa_load object:0xffff888108af6328 value:0x1000
             cat-95      [000] .....     1.412066: bio_add_page <-ext4_mpage_readpages object:0xffff888108af6328 value:0x1000
             cat-95      [000] .....     1.412066: kprobe_ftrace_handler <-ftrace_ops_list_func object:0xffff888108af6328 value:0x1000
             cat-95      [000] .....     1.412067: get_kprobe <-kprobe_ftrace_handler object:0xffff888108af6328 value:0x1000
             cat-95      [000] .....     1.412067: __bio_try_merge_page <-bio_add_page object:0xffff888108af6328 value:0x1000
             cat-95      [000] .....     1.412067: __bio_add_page <-bio_add_page object:0xffff888108af6328 value:0x1000
             cat-95      [000] .....     1.412067: rcu_read_unlock_strict <-xa_load object:0xffff888108af6328 value:0x2000
             cat-95      [000] .....     1.412067: bio_add_page <-ext4_mpage_readpages object:0xffff888108af6328 value:0x2000
             cat-95      [000] .....     1.412067: kprobe_ftrace_handler <-ftrace_ops_list_func object:0xffff888108af6328 value:0x2000
             cat-95      [000] .....     1.412067: get_kprobe <-kprobe_ftrace_handler object:0xffff888108af6328 value:0x2000
             cat-95      [000] .....     1.412067: __bio_try_merge_page <-bio_add_page object:0xffff888108af6328 value:0x2000
             cat-95      [000] .....     1.412068: submit_bio <-ext4_mpage_readpages object:0xffff888108af6328 value:0x3000
             cat-95      [000] .....     1.412068: submit_bio_noacct <-ext4_mpage_readpages object:0xffff888108af6328 value:0x3000
             cat-95      [000] .....     1.412068: __submit_bio <-submit_bio_noacct object:0xffff888108af6328 value:0x3000
             cat-95      [000] .....     1.412068: blk_try_enter_queue <-__submit_bio object:0xffff888108af6328 value:0x3000
             cat-95      [000] .....     1.412068: rcu_read_unlock_strict <-blk_try_enter_queue object:0xffff888108af6328 value:0x3000
             cat-95      [000] .....     1.412068: rcu_read_unlock_strict <-blk_try_enter_queue object:0xffff888108af6328 value:0x3000
             cat-95      [000] .....     1.412068: submit_bio_checks <-__submit_bio object:0xffff888108af6328 value:0x3000
             cat-95      [000] .....     1.412068: __cond_resched <-submit_bio_checks object:0xffff888108af6328 value:0x3000
             cat-95      [000] .....     1.412068: rcu_all_qs <-__cond_resched object:0xffff888108af6328 value:0x3000
             cat-95      [000] .....     1.412068: should_fail_bio <-submit_bio_checks object:0xffff888108af6328 value:0x3000
             cat-95      [000] .....     1.412069: create_task_io_context <-submit_bio_checks object:0xffff888108af6328 value:0x3000
             cat-95      [000] .....     1.412069: kmem_cache_alloc_node <-create_task_io_context object:0xffff888108af6328 value:0x3000
             cat-95      [000] .....     1.412069: should_failslab <-kmem_cache_alloc_node object:0xffff888108af6328 value:0x3000
             cat-95      [000] .....     1.412069: _raw_spin_lock <-create_task_io_context object:0xffff888108af6328 value:0x3000
             cat-95      [000] .....     1.412069: blk_mq_submit_bio <-__submit_bio object:0xffff888108af6328 value:0x3000
             cat-95      [000] .....     1.412069: __blk_queue_split <-blk_mq_submit_bio object:0xffff888108af6328 value:0x3000
             cat-95      [000] .....     1.412069: bvec_split_segs <-__blk_queue_split object:0xffff888108af6328 value:0x3000
             cat-95      [000] .....     1.412069: blk_attempt_plug_merge <-blk_mq_submit_bio object:0xffff888108af6328 value:0x3000
             cat-95      [000] .....     1.412070: __blk_mq_sched_bio_merge <-blk_mq_submit_bio object:0xffff888108af6328 value:0x3000
             cat-95      [000] .....     1.412070: dd_bio_merge <-blk_mq_submit_bio object:0xffff888108af6328 value:0x3000
             cat-95      [000] .....     1.412070: _raw_spin_lock <-dd_bio_merge object:0xffff888108af6328 value:0x3000
             cat-95      [000] .....     1.412070: blk_mq_sched_try_merge <-dd_bio_merge object:0xffff888108af6328 value:0x3000
             cat-95      [000] .....     1.412070: elv_merge <-blk_mq_sched_try_merge object:0xffff888108af6328 value:0x3000
             cat-95      [000] .....     1.412070: elv_rqhash_find <-elv_merge object:0xffff888108af6328 value:0x3000
             cat-95      [000] .....     1.412070: dd_request_merge <-blk_mq_sched_try_merge object:0xffff888108af6328 value:0x3000
             cat-95      [000] .....     1.412070: elv_rb_find <-dd_request_merge object:0xffff888108af6328 value:0x3000
             cat-95      [000] .....     1.412070: __blk_mq_alloc_request <-blk_mq_submit_bio object:0xffff888108af6328 value:0x3000
             cat-95      [000] .....     1.412071: dd_limit_depth <-__blk_mq_alloc_request object:0xffff888108af6328 value:0x3000
             cat-95      [000] .....     1.412071: blk_mq_get_tag <-__blk_mq_alloc_request object:0xffff888108af6328 value:0x3000
             cat-95      [000] .....     1.412071: __blk_mq_get_tag <-blk_mq_get_tag object:0xffff888108af6328 value:0x3000
             cat-95      [000] .....     1.412071: blk_mq_rq_ctx_init.isra.0 <-blk_mq_submit_bio object:0xffff888108af6328 value:0x3000
             cat-95      [000] .....     1.412071: ktime_get <-blk_mq_rq_ctx_init.isra.0 object:0xffff888108af6328 value:0x3000
             cat-95      [000] .....     1.412071: dd_prepare_request <-blk_mq_rq_ctx_init.isra.0 object:0xffff888108af6328 value:0x3000
             cat-95      [000] .....     1.412071: blk_account_io_start <-blk_mq_submit_bio object:0xffff888108af6328 value:0x3000
             cat-95      [000] .....     1.412071: kprobe_ftrace_handler <-ftrace_ops_list_func object:0xffff888108af6328 value:0x3000
             cat-95      [000] .....     1.412071: get_kprobe <-kprobe_ftrace_handler object:0xffff888108af6328 value:0x3000
---
 include/linux/trace_events.h        |   1 +
 kernel/trace/Kconfig                |   9 +
 kernel/trace/Makefile               |   1 +
 kernel/trace/trace.h                |   8 +
 kernel/trace/trace_entries.h        |  18 +
 kernel/trace/trace_events_trigger.c |   1 +
 kernel/trace/trace_object.c         | 705 ++++++++++++++++++++++++++++
 kernel/trace/trace_output.c         |  40 ++
 8 files changed, 783 insertions(+)
 create mode 100644 kernel/trace/trace_object.c
include/linux/trace_events.h +1 -0
index 3e475eeb5a99..84b5c12c7f44 100644
@@ -684,6 +684,7 @@ enum event_trigger_type {
 	ETT_EVENT_HIST		= (1 << 4),
 	ETT_HIST_ENABLE		= (1 << 5),
 	ETT_EVENT_EPROBE	= (1 << 6),
+	ETT_TRACE_OBJECT	= (1 << 7),
 };
 
 extern int filter_match_preds(struct event_filter *filter, void *rec);
kernel/trace/Kconfig +9 -0
index 420ff4bc67fd..419ec048686a 100644
@@ -237,6 +237,15 @@ config FUNCTION_PROFILER
 
 	  If in doubt, say N.
 
+config TRACE_OBJECT
+	bool "Trace kernel object"
+	depends on FUNCTION_TRACER
+	default y
+	help
+	 This help kernel developer to trace any kernel object, and get the value
+	 of the object dynamically. the object can from the dynamic event(kprobe_event/
+	 uprobe_event) or the static event(tracepoint).
+
 config STACK_TRACER
 	bool "Trace max stack"
 	depends on HAVE_FUNCTION_TRACER
kernel/trace/Makefile +1 -0
index 6de5d4d63165..6d9e78a488aa 100644
@@ -66,6 +66,7 @@ obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += trace_functions_graph.o
 obj-$(CONFIG_TRACE_BRANCH_PROFILING) += trace_branch.o
 obj-$(CONFIG_BLK_DEV_IO_TRACE) += blktrace.o
 obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += fgraph.o
+obj-$(CONFIG_TRACE_OBJECT) += trace_object.o
 ifeq ($(CONFIG_BLOCK),y)
 obj-$(CONFIG_EVENT_TRACING) += blktrace.o
 endif
kernel/trace/trace.h +8 -0
index b7c0f8e160fb..d2508f977fe6 100644
@@ -49,6 +49,7 @@ enum trace_type {
 	TRACE_TIMERLAT,
 	TRACE_RAW_DATA,
 	TRACE_FUNC_REPEATS,
+	TRACE_OBJECT,
 
 	__TRACE_LAST_TYPE,
 };
@@ -460,6 +461,7 @@ extern void __ftrace_bad_type(void);
 			  TRACE_GRAPH_RET);		\
 		IF_ASSIGN(var, ent, struct func_repeats_entry,		\
 			  TRACE_FUNC_REPEATS);				\
+		IF_ASSIGN(var, ent, struct trace_object_entry, TRACE_OBJECT);\
 		__ftrace_bad_type();					\
 	} while (0)
 
@@ -1510,6 +1512,12 @@ static inline int register_trigger_hist_cmd(void) { return 0; }
 static inline int register_trigger_hist_enable_disable_cmds(void) { return 0; }
 #endif
 
+#ifdef CONFIG_TRACE_OBJECT
+extern int register_trigger_object_cmd(void);
+#else
+static inline int register_trigger_object_cmd(void) { return 0; }
+#endif
+
 extern int register_trigger_cmds(void);
 extern void clear_event_triggers(struct trace_array *tr);
 
kernel/trace/trace_entries.h +18 -0
index cd41e863b51c..b98c8d048bc3 100644
@@ -401,3 +401,21 @@ FTRACE_ENTRY(timerlat, timerlat_entry,
 		 __entry->context,
 		 __entry->timer_latency)
 );
+
+/*
+ * trace object entry:
+ */
+FTRACE_ENTRY(object, trace_object_entry,
+
+	TRACE_OBJECT,
+
+	F_STRUCT(
+		__field(	unsigned long,		ip		)
+		__field(	unsigned long,		parent_ip	)
+		__field(	unsigned long,		object		)
+		__field(	unsigned long,		value		)
+	),
+
+	F_printk(" %ps <-- %ps object:%lx value=%lx\n", (void *)__entry->ip,
+		(void *)__entry->parent_ip, __entry->object, __entry->value)
+);
kernel/trace/trace_events_trigger.c +1 -0
index 3d5c07239a2a..da2d871fa809 100644
@@ -1687,6 +1687,7 @@ __init int register_trigger_cmds(void)
 	register_trigger_enable_disable_cmds();
 	register_trigger_hist_enable_disable_cmds();
 	register_trigger_hist_cmd();
+	register_trigger_object_cmd();
 
 	return 0;
 }
kernel/trace/trace_object.c new file +705 -0

705 lines not on this page · raw message

kernel/trace/trace_output.c +40 -0
index c2ca40e8595b..99bc09f31e0c 100644
@@ -1552,6 +1552,45 @@ static struct trace_event trace_func_repeats_event = {
 	.funcs		= &trace_func_repeats_funcs,
 };
 
+/* TRACE_OBJECT */
+static enum print_line_t trace_object_print(struct trace_iterator *iter, int flags,
+					struct trace_event *event)
+{
+	struct trace_object_entry *field;
+	struct trace_seq *s = &iter->seq;
+
+	trace_assign_type(field, iter->ent);
+	print_fn_trace(s, field->ip, field->parent_ip, flags);
+	trace_seq_printf(s, " object:0x%lx", field->object);
+	trace_seq_printf(s, " value:0x%lx", field->value);
+	trace_seq_putc(s, '\n');
+
+	return trace_handle_return(s);
+}
+
+static enum print_line_t trace_object_raw(struct trace_iterator *iter, int flags,
+				      struct trace_event *event)
+{
+	struct trace_object_entry *field;
+
+	trace_assign_type(field, iter->ent);
+
+	trace_seq_printf(&iter->seq, "%lx %lx %lx\n",
+			 field->ip, field->parent_ip, field->value);
+
+	return trace_handle_return(&iter->seq);
+}
+
+static struct trace_event_functions trace_object_funcs = {
+	.trace		= trace_object_print,
+	.raw		= trace_object_raw,
+};
+
+static struct trace_event trace_object_event = {
+	.type		= TRACE_OBJECT,
+	.funcs		= &trace_object_funcs,
+};
+
 static struct trace_event *events[] __initdata = {
 	&trace_fn_event,
 	&trace_ctx_event,
@@ -1566,6 +1605,7 @@ static struct trace_event *events[] __initdata = {
 	&trace_timerlat_event,
 	&trace_raw_data_event,
 	&trace_func_repeats_event,
+	&trace_object_event,
 	NULL
 };
 
Signature · 1 line
2.25.1
maintainer
#1 lore raw
Quoted 8 lines · On Mon, 1 Nov 2021 23:07:59 +0800 Jeff Xie <[email protected]> wrote:

Introduce a method based on function tracer to trace any object and get the value of the object dynamically. the object can from the dynamic event (kprobe_event/uprobe_event) or the static event(tracepoint).

Usage:
When using the kprobe event, only need to set the objtrace(a new trigger), we can get the value of the object. The object is from the setting of the kprobe event.

Hi Jeff,

I just wanted to let you know that because the merge window just opened (Linus released 5.15 and is now accepting changes for 5.16), I most likely wont have any time to review your changes while the merge window is open (for the next two weeks).

Please do not take my lack of responses during this time as ignoring you. I'm just going to be overwhelmed with getting things right, and this merge window is going to be a bit tougher for me than other merge windows.

Thanks,

-- Steve

author
#2 lore raw
Quoted 20 lines · Steven Rostedt
Quoted 8 lines · On Mon, 1 Nov 2021 23:07:59 +0800 Jeff Xie <[email protected]> wrote:

Introduce a method based on function tracer to trace any object and get the value of the object dynamically. the object can from the dynamic event (kprobe_event/uprobe_event) or the static event(tracepoint).

Usage:
When using the kprobe event, only need to set the objtrace(a new trigger), we can get the value of the object. The object is from the setting of the kprobe event.

Hi Jeff,

I just wanted to let you know that because the merge window just opened (Linus released 5.15 and is now accepting changes for 5.16), I most likely wont have any time to review your changes while the merge window is open (for the next two weeks).

Quoted 7 lines

Please do not take my lack of responses during this time as ignoring you. I'm just going to be overwhelmed with getting things right, and this merge window is going to be a bit tougher for me than other merge windows.

Thanks,

-- Steve

I’m very glad you can explain this and let me know. because the previous patch[PATCH V3] lacks some functions.
Just update it.

Thanks.
---
JeffXie

#3 lore raw
In reply to Jeff Xie
[If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in https://git-scm.com/docs/git-format-patch]

Hi Jeff,

[FYI, it's a private test report for your RFC patch.]
[auto build test ERROR on linux/master]
[also build test ERROR on linus/master v5.15 next-20211104] [cannot apply to tip/perf/core]
[If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]

url:    https://github.com/0day-ci/linux/commits/Jeff-Xie/trace-Add-trace-any-kernel-object/20211101-231428
base:   https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git 2f111a6fd5b5297b4e92f53798ca086f7c7d33a4
config: i386-buildonly-randconfig-r002-20211101 (attached as .config)
compiler: clang version 14.0.0 (https://github.com/llvm/llvm-project 264d3b6d4e08401c5b50a85bd76e80b3461d77e6)

reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# https://github.com/0day-ci/linux/commit/5c8589956ecad7f8aeec51f42e6b9012d38be7f7
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Jeff-Xie/trace-Add-trace-any-kernel-object/20211101-231428 git checkout 5c8589956ecad7f8aeec51f42e6b9012d38be7f7
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross W=1 ARCH=i386

If you fix the issue, kindly add following tag as appropriate

All errors (new ones prefixed by >>):

>> kernel/trace/trace_object.c:445:6: error: no previous prototype for function 'unregister_object_trigger' [-Werror,-Wmissing-prototypes]
   void unregister_object_trigger(char *glob, struct event_trigger_ops *ops,
        ^
   kernel/trace/trace_object.c:445:1: note: declare 'static' if the function is not intended to be used outside of this translation unit
   void unregister_object_trigger(char *glob, struct event_trigger_ops *ops,
   ^
   static 
>> kernel/trace/trace_object.c:689:6: error: variable 'ret' is used uninitialized whenever 'if' condition is true [-Werror,-Wsometimes-uninitialized]
           if (WARN_ON_ONCE(atomic_read(&trace_object_ref) <= 0))
               ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   include/asm-generic/bug.h:103:33: note: expanded from macro 'WARN_ON_ONCE'
   #define WARN_ON_ONCE(condition) ({                              \
                                   ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   kernel/trace/trace_object.c:704:9: note: uninitialized use occurs here
           return ret;
                  ^~~
   kernel/trace/trace_object.c:689:2: note: remove the 'if' if its condition is always false
           if (WARN_ON_ONCE(atomic_read(&trace_object_ref) <= 0))
           ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   kernel/trace/trace_object.c:687:9: note: initialize the variable 'ret' to silence this warning
           int ret;
                  ^
                   = 0

2 errors generated.

vim +/unregister_object_trigger +445 kernel/trace/trace_object.c

444

 > 445	void unregister_object_trigger(char *glob, struct event_trigger_ops *ops,

446 struct event_trigger_data *test,
447 struct trace_event_file *file)
448 {
449 struct event_trigger_data *data;
450 bool unregistered = false;
451
452 lockdep_assert_held(&event_mutex);
453
454 list_for_each_entry(data, &file->triggers, list) {
455 if (data->cmd_ops->trigger_type == test->cmd_ops->trigger_type) {
456 unregistered = true;
457 list_del_rcu(&data->list);
458 trace_event_trigger_enable_disable(file, 0);
459 update_cond_flag(file);
460 break;
461 }
462 }
463
464 if (unregistered && data->ops->free) {
465 data->ops->free(data->ops, data);
466 exit_trace_object();
467 }
468 }
469

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org