Skip to contentSkip to conversation

tracing: kprobe-tracer plugin core

RFC PATCH Superseded Masami Hiramatsu 2 participants 3 messages Mar 21, 2009

Subsystems (2)

v2 of 4 Superseded Latest is v4 →

lore
Share & tools
lore
#0 lore raw

RFC PATCH v2 1/5 tracing: kprobe-tracer plugin core

Add kprobes based event tracer on ftrace.

This tracer is similar to the events tracer which is based on Tracepoint infrastructure. Instead of Tracepoint, this tracer is based on kprobes(kprobe and kretprobe). It probes anywhere where kprobes can probe(this means, all functions body except for __kprobes functions).

changes from v1:
- fix a bug in offset parsing.
- use __trace_bprintk() instead of __trace_printk().

Cc Steven Rostedt <[email protected]>
Cc Ananth N Mavinakayanahalli <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Frederic Weisbecker <[email protected]>

---

 Documentation/ftrace.txt    |   55 ++++++
 kernel/trace/Kconfig        |    9 +
 kernel/trace/Makefile       |    1
 kernel/trace/trace_kprobe.c |  413 +++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 478 insertions(+), 0 deletions(-)
 create mode 100644 kernel/trace/trace_kprobe.c
Documentation/ftrace.txt +55 -0
index fd9a3e6..a957adf 100644
@@ -1310,6 +1310,61 @@ dereference in a kernel module:
 [...]
 
 
+kprobe-based event tracer
+---------------------------
+
+This tracer is similar to the events tracer which is based on Tracepoint
+infrastructure. Instead of Tracepoint, this tracer is based on kprobes(kprobe
+and kretprobe). It probes anywhere where kprobes can probe(this means, all
+functions body except for __kprobes functions).
+
+Unlike the function tracer, this tracer can probe instructions inside of
+kernel functions. It allows you to check which instruction has been executed.
+
+Unlike the Tracepoint based events tracer, this tracer can add new probe points
+on the fly.
+
+Similar to the events tracer, this tracer doesn't need to be activated via
+current_tracer, instead of that, just set probe points via
+/debug/tracing/kprobe_probes.
+
+Synopsis of kprobe_probes:
+  p SYMBOL[+offs|-offs]|MEMADDR	: set a probe
+  r SYMBOL[+0]				: set a return probe
+
+E.g.
+  echo p sys_open > /debug/tracing/kprobe_probes
+
+ This sets a kprobe on the top of sys_open() function.
+
+  echo r sys_open >> /debug/tracing/kprobe_probes
+
+ This sets a kretprobe on the return point of sys_open() function.
+
+  echo > /debug/tracing/kprobe_probes
+
+ This clears all probe points. and you can see the traced information via
+/debug/tracing/trace.
+
+  echo /debug/tracing/trace
+# tracer: nop
+#
+#           TASK-PID    CPU#    TIMESTAMP  FUNCTION
+#              | |       |          |         |
+           <...>-5117  [003]   416.481638: sys_open: @sys_open+0
+           <...>-5117  [003]   416.481662: syscall_call: <-sys_open+0
+           <...>-5117  [003]   416.481739: sys_open: @sys_open+0
+           <...>-5117  [003]   416.481762: sysenter_do_call: <-sys_open+0
+           <...>-5117  [003]   416.481818: sys_open: @sys_open+0
+           <...>-5117  [003]   416.481842: sysenter_do_call: <-sys_open+0
+           <...>-5117  [003]   416.481882: sys_open: @sys_open+0
+           <...>-5117  [003]   416.481905: sysenter_do_call: <-sys_open+0
+
+ @SYMBOL means that kernel hits a probe, and <-SYMBOL means kernel returns
+from SYMBOL(e.g. "sysenter_do_call: <-sys_open+0" means kernel returns from
+sys_open to sysenter_do_call).
+
+
 function graph tracer
 ---------------------------
 
kernel/trace/Kconfig +9 -0
index b0a46f8..1980703 100644
@@ -353,6 +353,15 @@ config BLK_DEV_IO_TRACE
 
 	  If unsure, say N.
 
+config KPROBE_TRACER
+	depends on KPROBES
+	depends on X86
+	bool "Trace kprobes"
+	select TRACING
+	help
+	  This tracer probes everywhere where kprobes can probe it, and
+	  records various registers and memories specified by user.
+
 config DYNAMIC_FTRACE
 	bool "enable/disable ftrace tracepoints dynamically"
 	depends on FUNCTION_TRACER
kernel/trace/Makefile +1 -0
index 0e45c20..de2cdfb 100644
@@ -45,5 +45,6 @@ obj-$(CONFIG_EVENT_TRACER) += events.o
 obj-$(CONFIG_EVENT_TRACER) += trace_export.o
 obj-$(CONFIG_FTRACE_SYSCALLS) += trace_syscalls.o
 obj-$(CONFIG_EVENT_PROFILE) += trace_event_profile.o
+obj-$(CONFIG_KPROBE_TRACER) += trace_kprobe.o
 
 libftrace-y := ftrace.o
kernel/trace/trace_kprobe.c new file +413 -0

413 lines not on this page · raw message

Signature · 7 lines
Masami Hiramatsu

Software Engineer
Hitachi Computer Products (America) Inc.
Software Solutions Division

e-mail: [email protected]
#1 lore raw
Quoted 17 lines · Masami Hiramatsu

Add kprobes based event tracer on ftrace.

This tracer is similar to the events tracer which is based on Tracepoint infrastructure. Instead of Tracepoint, this tracer is based on kprobes(kprobe and kretprobe). It probes anywhere where kprobes can probe(this means, all functions body except for __kprobes functions).

changes from v1:
- fix a bug in offset parsing.
- use __trace_bprintk() instead of __trace_printk().

Cc Steven Rostedt <[email protected]>
Cc Ananth N Mavinakayanahalli <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Frederic Weisbecker <[email protected]>

---

[ ... ]

Quoted 16 lines
kernel/trace/trace_kprobe.c new file +16 -0
+/* Probes listing interfaces */
+static void *probes_seq_start(struct seq_file *m, loff_t *pos)
+{
+	struct trace_probe *probe;
+	loff_t n = *pos;
+
+	mutex_lock(&probe_lock);
+	if (!list_empty(&probe_list)) {
+		list_for_each_entry(probe, &probe_list, list)
+			if (0 == n--)
+				return probe;
+	}
+	return NULL;
+}
+
+static void *probes_seq_next(struct seq_file *m, void *v, loff_t *pos)

kernel/trace/trace_kprobe.c line 274

Quoted 10 lines
kernel/trace/trace_kprobe.c new file +10 -0
+{
+	struct trace_probe *tp = v;
+
+	(*pos)++;
+	if (tp->list.next == &probe_list)
+		tp = NULL;
+	else
+		tp = list_entry(tp->list.next, struct trace_probe, list);
+	return tp;
+}

Hm, I think seq_list_start/next will be sufficient for your needs.

Quoted 27 lines
kernel/trace/trace_kprobe.c new file +27 -0
+static void probes_seq_stop(struct seq_file *m, void *v)
+{
+	mutex_unlock(&probe_lock);
+}
+
+static int probes_seq_show(struct seq_file *m, void *v)
+{
+	struct trace_probe *tp = v;
+
+	if (tp == NULL)
+		return 0;
+
+	if (tp->symbol)
+		seq_printf(m, "%c %s%+ld\n",
+			probe_is_return(tp) ? 'r' : 'p',
+			probe_symbol(tp), probe_offset(tp));
+	else
+		seq_printf(m, "%c 0x%p\n",
+			probe_is_return(tp) ? 'r' : 'p',
+			probe_address(tp));
+	return 0;
+}
+
+static const struct seq_operations probes_seq_op = {
+	.start  = probes_seq_start,
+	.next   = probes_seq_next,
+	.stop   = probes_seq_stop,

kernel/trace/trace_kprobe.c line 312

Quoted 10 lines
kernel/trace/trace_kprobe.c new file +10 -0
+	.show   = probes_seq_show
+};
+
+static int probes_open(struct inode *inode, struct file *file)
+{
+	if ((file->f_mode & FMODE_WRITE) &&
+	    !(file->f_flags & O_APPEND))
+		cleanup_all_probes();
+
+	return seq_open(file, &probes_seq_op);

This seq_open is only for read case. No?

Quoted 82 lines
kernel/trace/trace_kprobe.c new file +82 -0
+}
+
+
+#define WRITE_BUFSIZE 128
+
+ssize_t probes_write(struct file *file, const char __user *buffer,
+		     size_t count, loff_t *ppos)
+{
+	char *kbuf, *tmp;
+	char **argv = NULL;
+	int argc = 0;
+	int ret;
+	size_t done;
+	size_t size;
+
+	if (!count || count < 0)
+		return 0;
+
+	kbuf = kmalloc(WRITE_BUFSIZE, GFP_KERNEL);
+	if (!kbuf)
+		return -ENOMEM;
+
+	ret = done = 0;
+	do {
+		size = count - done;
+		if (size > WRITE_BUFSIZE)
+			size = WRITE_BUFSIZE;
+		if (copy_from_user(kbuf, buffer + done, size)) {
+			ret = -EFAULT;
+			goto out;
+		}
+		kbuf[size] = '\0';
+		tmp = strchr(kbuf, '\n');
+		if (!tmp) {
+			pr_warning("Line length is too long: "
+				   "Should be less than %d.", WRITE_BUFSIZE);
+			ret = -EINVAL;
+			goto out;
+		}
+		*tmp = '\0';
+		size = tmp - kbuf + 1;
+		done += size;
+
+		argv = argv_split(GFP_KERNEL, kbuf, &argc);
+		if (!argv) {
+			ret = -ENOMEM;
+			goto out;
+		}
+
+		if (argc)
+			ret = create_trace_probe(argc, argv);
+
+		argv_free(argv);
+		if (ret < 0)
+			goto out;
+
+	} while (done < count);
+	ret = done;
+out:
+	kfree(kbuf);
+	return ret;
+}
+
+static const struct file_operations kprobe_points_ops = {
+	.owner          = THIS_MODULE,
+	.open           = probes_open,
+	.read           = seq_read,
+	.llseek         = seq_lseek,
+	.release        = seq_release,
+	.write		= probes_write,
+};
+
+/* event recording functions */
+static void kprobe_trace_record(unsigned long ip, struct trace_probe *tp,
+				struct pt_regs *regs)
+{
+	__trace_bprintk(ip, "%s%s%+ld\n",
+			probe_is_return(tp) ? "<-" : "@",
+			probe_symbol(tp), probe_offset(tp));
+}
+
+/* Make a debugfs interface for controling probe points */

kernel/trace/trace_kprobe.c line 404

Quoted 10 lines
kernel/trace/trace_kprobe.c new file +10 -0
+static __init int init_kprobe_trace(void)
+{
+	struct dentry *d_tracer;
+	struct dentry *entry;
+
+	d_tracer = tracing_init_dentry();
+	if (!d_tracer)
+		return 0;
+
+	entry = debugfs_create_file("kprobe_probes", 0444, d_tracer,

Shouldn't it be 0644 ?
Since its first intend, and the most important one, is to receive commands from user?

Thanks,
Frederic.

Quoted 19 lines
kernel/trace/trace_kprobe.c new file +9 -0
+				    NULL, &kprobe_points_ops);
+
+	if (!entry)
+		pr_warning("Could not create debugfs "
+			   "'kprobe_probess' entry\n");
+	return 0;
+}
+fs_initcall(init_kprobe_trace);
+
Signature · 7 lines
Masami Hiramatsu

Software Engineer
Hitachi Computer Products (America) Inc.
Software Solutions Division

e-mail: [email protected]
#2 lore raw
Quoted 44 lines · Frederic Weisbecker
Quoted 17 lines · Masami Hiramatsu

Add kprobes based event tracer on ftrace.

This tracer is similar to the events tracer which is based on Tracepoint infrastructure. Instead of Tracepoint, this tracer is based on kprobes(kprobe and kretprobe). It probes anywhere where kprobes can probe(this means, all functions body except for __kprobes functions).

changes from v1:
- fix a bug in offset parsing.
- use __trace_bprintk() instead of __trace_printk().

Cc Steven Rostedt <[email protected]>
Cc Ananth N Mavinakayanahalli <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Frederic Weisbecker <[email protected]>

---

[ ... ]

Quoted 22 lines
kernel/trace/trace_kprobe.c new file +22 -0
+/* Probes listing interfaces */
+static void *probes_seq_start(struct seq_file *m, loff_t *pos)
+{
+	struct trace_probe *probe;
+	loff_t n = *pos;
+
+	mutex_lock(&probe_lock);
+	if (!list_empty(&probe_list)) {
+		list_for_each_entry(probe, &probe_list, list)
+			if (0 == n--)
+				return probe;
+	}
+	return NULL;
+}
+
+static void *probes_seq_next(struct seq_file *m, void *v, loff_t *pos)
+{
+	struct trace_probe *tp = v;
+
+	(*pos)++;
+	if (tp->list.next == &probe_list)
+		tp = NULL;
Quoted 8 lines

kernel/trace/trace_kprobe.c line 274

Quoted 4 lines
kernel/trace/trace_kprobe.c new file +4 -0
+	else
+		tp = list_entry(tp->list.next, struct trace_probe, list);
+	return tp;
+}

Hm, I think seq_list_start/next will be sufficient for your needs.

Oh, those look very useful, thanks!

Quoted 32 lines
Quoted 32 lines
kernel/trace/trace_kprobe.c new file +32 -0
+static void probes_seq_stop(struct seq_file *m, void *v)
+{
+	mutex_unlock(&probe_lock);
+}
+
+static int probes_seq_show(struct seq_file *m, void *v)
+{
+	struct trace_probe *tp = v;
+
+	if (tp == NULL)
+		return 0;
+
+	if (tp->symbol)
+		seq_printf(m, "%c %s%+ld\n",
+			probe_is_return(tp) ? 'r' : 'p',
+			probe_symbol(tp), probe_offset(tp));
+	else
+		seq_printf(m, "%c 0x%p\n",
+			probe_is_return(tp) ? 'r' : 'p',
+			probe_address(tp));
+	return 0;
+}
+
+static const struct seq_operations probes_seq_op = {
+	.start  = probes_seq_start,
+	.next   = probes_seq_next,
+	.stop   = probes_seq_stop,
+	.show   = probes_seq_show
+};
+
+static int probes_open(struct inode *inode, struct file *file)
+{
Quoted 8 lines

kernel/trace/trace_kprobe.c line 312

Quoted 5 lines
kernel/trace/trace_kprobe.c new file +5 -0
+	if ((file->f_mode & FMODE_WRITE) &&
+	    !(file->f_flags & O_APPEND))
+		cleanup_all_probes();
+
+	return seq_open(file, &probes_seq_op);

This seq_open is only for read case. No?

Yes, but seq_release is called when closing. This seq_open is for that. (you can see the similar coding in trace_events.c)

Quoted 89 lines
Quoted 89 lines
kernel/trace/trace_kprobe.c new file +89 -0
+}
+
+
+#define WRITE_BUFSIZE 128
+
+ssize_t probes_write(struct file *file, const char __user *buffer,
+		     size_t count, loff_t *ppos)
+{
+	char *kbuf, *tmp;
+	char **argv = NULL;
+	int argc = 0;
+	int ret;
+	size_t done;
+	size_t size;
+
+	if (!count || count < 0)
+		return 0;
+
+	kbuf = kmalloc(WRITE_BUFSIZE, GFP_KERNEL);
+	if (!kbuf)
+		return -ENOMEM;
+
+	ret = done = 0;
+	do {
+		size = count - done;
+		if (size > WRITE_BUFSIZE)
+			size = WRITE_BUFSIZE;
+		if (copy_from_user(kbuf, buffer + done, size)) {
+			ret = -EFAULT;
+			goto out;
+		}
+		kbuf[size] = '\0';
+		tmp = strchr(kbuf, '\n');
+		if (!tmp) {
+			pr_warning("Line length is too long: "
+				   "Should be less than %d.", WRITE_BUFSIZE);
+			ret = -EINVAL;
+			goto out;
+		}
+		*tmp = '\0';
+		size = tmp - kbuf + 1;
+		done += size;
+
+		argv = argv_split(GFP_KERNEL, kbuf, &argc);
+		if (!argv) {
+			ret = -ENOMEM;
+			goto out;
+		}
+
+		if (argc)
+			ret = create_trace_probe(argc, argv);
+
+		argv_free(argv);
+		if (ret < 0)
+			goto out;
+
+	} while (done < count);
+	ret = done;
+out:
+	kfree(kbuf);
+	return ret;
+}
+
+static const struct file_operations kprobe_points_ops = {
+	.owner          = THIS_MODULE,
+	.open           = probes_open,
+	.read           = seq_read,
+	.llseek         = seq_lseek,
+	.release        = seq_release,
+	.write		= probes_write,
+};
+
+/* event recording functions */
+static void kprobe_trace_record(unsigned long ip, struct trace_probe *tp,
+				struct pt_regs *regs)
+{
+	__trace_bprintk(ip, "%s%s%+ld\n",
+			probe_is_return(tp) ? "<-" : "@",
+			probe_symbol(tp), probe_offset(tp));
+}
+
+/* Make a debugfs interface for controling probe points */
+static __init int init_kprobe_trace(void)
+{
+	struct dentry *d_tracer;
+	struct dentry *entry;
+
+	d_tracer = tracing_init_dentry();
+	if (!d_tracer)
Quoted 8 lines

kernel/trace/trace_kprobe.c line 404

Quoted 3 lines
kernel/trace/trace_kprobe.c new file +3 -0
+		return 0;
+
+	entry = debugfs_create_file("kprobe_probes", 0444, d_tracer,

Shouldn't it be 0644 ?
Since its first intend, and the most important one, is to receive commands from user?

Yes, right...

Thank you for review!

Signature · 7 lines
Masami Hiramatsu

Software Engineer
Hitachi Computer Products (America) Inc.
Software Solutions Division

e-mail: [email protected]