Skip to contentSkip to conversation

tracing/kprobes: Add method to display private kprobes in tracefs

PATCH In review Jianlin Lv 3 participants 5 messages Jul 25, 2022 – Sep 9, 2022

Subsystems (3)
lore
Share & tools
lore

Jianlin Lv

author
#0 lore raw

PATCH v2 1/1 tracing/kprobes: Add method to display private kprobes in tracefs

The private kprobes are not added to the global list dyn_event_list, so there is a missing interface to show probe hit and probe miss. This patch adds a profiling interface to check the number of hits or misses for private kprobes.

---
v2: update commit message

---
 Documentation/trace/kprobetrace.rst |  6 +++-
 kernel/trace/trace_dynevent.c       | 20 +++++++++++
 kernel/trace/trace_dynevent.h       | 37 ++++++++++++++++++++
 kernel/trace/trace_kprobe.c         | 54 +++++++++++++++++++++++++++++
 4 files changed, 116 insertions(+), 1 deletion(-)
Documentation/trace/kprobetrace.rst +5 -1
index b175d88f31eb..8815d64dd8a6 100644
@@ -146,7 +146,11 @@ trigger:
 Event Profiling
 ---------------
 You can check the total number of probe hits and probe miss-hits via
-/sys/kernel/debug/tracing/kprobe_profile.
+/sys/kernel/debug/tracing/kprobe_profile or
+/sys/kernel/debug/tracing/kprobe_local_profile.
+All kprobe events created by kprobe_events will be added to the global
+list, you can get their profiling via kprobe_profile; kprobe_local_profile
+shows profiling for private kprobe events created by perf_kprobe pmu.
 The first column is event name, the second is the number of probe hits,
 the third is the number of probe miss-hits.
 
kernel/trace/trace_dynevent.c +20 -0
index 076b447a1b88..70ec99cd9c53 100644
@@ -181,6 +181,26 @@ static const struct seq_operations dyn_event_seq_op = {
 	.show	= dyn_event_seq_show
 };
 
+#ifdef CONFIG_KPROBE_EVENTS
+LIST_HEAD(local_event_list);
+
+void *local_event_seq_start(struct seq_file *m, loff_t *pos)
+{
+	mutex_lock(&event_mutex);
+	return seq_list_start(&local_event_list, *pos);
+}
+
+void *local_event_seq_next(struct seq_file *m, void *v, loff_t *pos)
+{
+	return seq_list_next(v, &local_event_list, pos);
+}
+
+void local_event_seq_stop(struct seq_file *m, void *v)
+{
+	mutex_unlock(&event_mutex);
+}
+#endif /* CONFIG_KPROBE_EVENTS */
+
 /*
  * dyn_events_release_all - Release all specific events
  * @type:	the dyn_event_operations * which filters releasing events
kernel/trace/trace_dynevent.h +37 -0
index 936477a111d3..e30193470295 100644
@@ -101,6 +101,43 @@ void dyn_event_seq_stop(struct seq_file *m, void *v);
 int dyn_events_release_all(struct dyn_event_operations *type);
 int dyn_event_release(const char *raw_command, struct dyn_event_operations *type);
 
+#ifdef CONFIG_KPROBE_EVENTS
+extern struct list_head local_event_list;
+
+static inline
+int local_event_init(struct dyn_event *ev, struct dyn_event_operations *ops)
+{
+	if (!ev || !ops)
+		return -EINVAL;
+
+	INIT_LIST_HEAD(&ev->list);
+	ev->ops = ops;
+	return 0;
+}
+
+static inline int local_event_add(struct dyn_event *ev)
+{
+	lockdep_assert_held(&event_mutex);
+
+	if (!ev || !ev->ops)
+		return -EINVAL;
+
+	list_add_tail(&ev->list, &local_event_list);
+	return 0;
+}
+
+static inline void local_event_remove(struct dyn_event *ev)
+{
+	lockdep_assert_held(&event_mutex);
+	list_del_init(&ev->list);
+}
+
+void *local_event_seq_start(struct seq_file *m, loff_t *pos);
+void *local_event_seq_next(struct seq_file *m, void *v, loff_t *pos);
+void local_event_seq_stop(struct seq_file *m, void *v);
+
+#endif /* CONFIG_KPROBE_EVENTS */
+
 /*
  * for_each_dyn_event	-	iterate over the dyn_event list
  * @pos:	the struct dyn_event * to use as a loop cursor
kernel/trace/trace_kprobe.c +54 -0

78 lines not on this page · raw message

Signature · 1 line
2.25.1
maintainer
#1 lore raw

[ Adding Masami and Tom ]

Quoted 4 lines · On Mon, 25 Jul 2022 06:23:34 +0000 Jianlin Lv <[email protected]> wrote:

The private kprobes are not added to the global list dyn_event_list, so there is a missing interface to show probe hit and probe miss. This patch adds a profiling interface to check the number of hits or misses for private kprobes.

Masami, what do you think of this patch?

-- Steve

Quoted 193 lines

---
v2: update commit message

---
 Documentation/trace/kprobetrace.rst |  6 +++-
 kernel/trace/trace_dynevent.c       | 20 +++++++++++
 kernel/trace/trace_dynevent.h       | 37 ++++++++++++++++++++
 kernel/trace/trace_kprobe.c         | 54 +++++++++++++++++++++++++++++
 4 files changed, 116 insertions(+), 1 deletion(-)
Documentation/trace/kprobetrace.rst +5 -1
index b175d88f31eb..8815d64dd8a6 100644
@@ -146,7 +146,11 @@ trigger:
 Event Profiling
 ---------------
 You can check the total number of probe hits and probe miss-hits via
-/sys/kernel/debug/tracing/kprobe_profile.
+/sys/kernel/debug/tracing/kprobe_profile or
+/sys/kernel/debug/tracing/kprobe_local_profile.
+All kprobe events created by kprobe_events will be added to the global
+list, you can get their profiling via kprobe_profile; kprobe_local_profile
+shows profiling for private kprobe events created by perf_kprobe pmu.
 The first column is event name, the second is the number of probe hits,
 the third is the number of probe miss-hits.
 
kernel/trace/trace_dynevent.c +20 -0
index 076b447a1b88..70ec99cd9c53 100644
@@ -181,6 +181,26 @@ static const struct seq_operations dyn_event_seq_op = {
 	.show	= dyn_event_seq_show
 };
 
+#ifdef CONFIG_KPROBE_EVENTS
+LIST_HEAD(local_event_list);
+
+void *local_event_seq_start(struct seq_file *m, loff_t *pos)
+{
+	mutex_lock(&event_mutex);
+	return seq_list_start(&local_event_list, *pos);
+}
+
+void *local_event_seq_next(struct seq_file *m, void *v, loff_t *pos)
+{
+	return seq_list_next(v, &local_event_list, pos);
+}
+
+void local_event_seq_stop(struct seq_file *m, void *v)
+{
+	mutex_unlock(&event_mutex);
+}
+#endif /* CONFIG_KPROBE_EVENTS */
+
 /*
  * dyn_events_release_all - Release all specific events
  * @type:	the dyn_event_operations * which filters releasing events
kernel/trace/trace_dynevent.h +37 -0
index 936477a111d3..e30193470295 100644
@@ -101,6 +101,43 @@ void dyn_event_seq_stop(struct seq_file *m, void *v);
 int dyn_events_release_all(struct dyn_event_operations *type);
 int dyn_event_release(const char *raw_command, struct dyn_event_operations *type);
 
+#ifdef CONFIG_KPROBE_EVENTS
+extern struct list_head local_event_list;
+
+static inline
+int local_event_init(struct dyn_event *ev, struct dyn_event_operations *ops)
+{
+	if (!ev || !ops)
+		return -EINVAL;
+
+	INIT_LIST_HEAD(&ev->list);
+	ev->ops = ops;
+	return 0;
+}
+
+static inline int local_event_add(struct dyn_event *ev)
+{
+	lockdep_assert_held(&event_mutex);
+
+	if (!ev || !ev->ops)
+		return -EINVAL;
+
+	list_add_tail(&ev->list, &local_event_list);
+	return 0;
+}
+
+static inline void local_event_remove(struct dyn_event *ev)
+{
+	lockdep_assert_held(&event_mutex);
+	list_del_init(&ev->list);
+}
+
+void *local_event_seq_start(struct seq_file *m, loff_t *pos);
+void *local_event_seq_next(struct seq_file *m, void *v, loff_t *pos);
+void local_event_seq_stop(struct seq_file *m, void *v);
+
+#endif /* CONFIG_KPROBE_EVENTS */
+
 /*
  * for_each_dyn_event	-	iterate over the dyn_event list
  * @pos:	the struct dyn_event * to use as a loop cursor
kernel/trace/trace_kprobe.c +54 -0
index a245ea673715..76f500b17b46 100644
@@ -1213,6 +1213,52 @@ static const struct file_operations kprobe_profile_ops = {
 	.release        = seq_release,
 };
 
+#ifdef CONFIG_KPROBE_EVENTS
+/* kprobe Local profile  */
+static int local_probes_profile_seq_show(struct seq_file *m, void *v)
+{
+	struct dyn_event *ev = v;
+	struct trace_kprobe *tk;
+
+	if (!is_trace_kprobe(ev))
+		return 0;
+
+	tk = to_trace_kprobe(ev);
+	seq_printf(m, "  %-44s %15lu %15lu\n",
+		trace_probe_name(&tk->tp),
+		trace_kprobe_nhit(tk),
+		tk->rp.kp.nmissed);
+
+	return 0;
+}
+
+static const struct seq_operations local_profile_seq_op = {
+	.start  = local_event_seq_start,
+	.next   = local_event_seq_next,
+	.stop   = local_event_seq_stop,
+	.show   = local_probes_profile_seq_show
+};
+
+static int local_profile_open(struct inode *inode, struct file *file)
+{
+	int ret;
+
+	ret = security_locked_down(LOCKDOWN_TRACEFS);
+	if (ret)
+		return ret;
+
+	return seq_open(file, &local_profile_seq_op);
+}
+
+static const struct file_operations kprobe_local_profile_ops = {
+	.owner          = THIS_MODULE,
+	.open           = local_profile_open,
+	.read           = seq_read,
+	.llseek         = seq_lseek,
+	.release        = seq_release,
+};
+#endif /* CONFIG_KPROBE_EVENTS */
+
 /* Kprobe specific fetch functions */
 
 /* Return the length of string -- including null terminal byte */
@@ -1830,6 +1876,7 @@ create_local_trace_kprobe(char *func, void *addr, unsigned long offs,
 	if (ret < 0)
 		goto error;
 
+	local_event_add(&tk->devent);
 	return trace_probe_event_call(&tk->tp);
 error:
 	free_trace_kprobe(tk);
@@ -1849,6 +1896,7 @@ void destroy_local_trace_kprobe(struct trace_event_call *event_call)
 		return;
 	}
 
+	local_event_remove(&tk->devent);
 	__unregister_trace_kprobe(tk);
 
 	free_trace_kprobe(tk);
@@ -1929,6 +1977,12 @@ static __init int init_kprobe_trace(void)
 	trace_create_file("kprobe_profile", TRACE_MODE_READ,
 			  NULL, NULL, &kprobe_profile_ops);
 
+#ifdef CONFIG_KPROBE_EVENTS
+	/* kprobe Local profile */
+	tracefs_create_file("kprobe_local_profile", TRACE_MODE_READ,
+			  NULL, NULL, &kprobe_local_profile_ops);
+#endif /* CONFIG_KPROBE_EVENTS */
+
 	setup_boot_kprobe_events();
 
 	return 0;
#2 lore raw

[Adding bpf ML]

Quoted 6 lines · On Tue, 6 Sep 2022 11:12:14 -0400 Steven Rostedt <[email protected]> wrote:

[ Adding Masami and Tom ]

On Mon, 25 Jul 2022 06:23:34 +0000
Jianlin Lv <[email protected]> wrote:

Quoted 6 lines
Quoted 4 lines

The private kprobes are not added to the global list dyn_event_list, so there is a missing interface to show probe hit and probe miss. This patch adds a profiling interface to check the number of hits or misses for private kprobes.

Masami, what do you think of this patch?

I discussed it with BPF people when it was introduced and they didn't want to show up it on tracefs because it is a private one. I agreed that.

So I think this kind of interface must be managed by BPF subsystem. Is there any API to manage the BPF probe points in BPF subsystem?

Thank you,

Quoted 197 lines

-- Steve

Quoted 193 lines

---
v2: update commit message

---
 Documentation/trace/kprobetrace.rst |  6 +++-
 kernel/trace/trace_dynevent.c       | 20 +++++++++++
 kernel/trace/trace_dynevent.h       | 37 ++++++++++++++++++++
 kernel/trace/trace_kprobe.c         | 54 +++++++++++++++++++++++++++++
 4 files changed, 116 insertions(+), 1 deletion(-)
Documentation/trace/kprobetrace.rst +5 -1
index b175d88f31eb..8815d64dd8a6 100644
@@ -146,7 +146,11 @@ trigger:
 Event Profiling
 ---------------
 You can check the total number of probe hits and probe miss-hits via
-/sys/kernel/debug/tracing/kprobe_profile.
+/sys/kernel/debug/tracing/kprobe_profile or
+/sys/kernel/debug/tracing/kprobe_local_profile.
+All kprobe events created by kprobe_events will be added to the global
+list, you can get their profiling via kprobe_profile; kprobe_local_profile
+shows profiling for private kprobe events created by perf_kprobe pmu.
 The first column is event name, the second is the number of probe hits,
 the third is the number of probe miss-hits.
 
kernel/trace/trace_dynevent.c +20 -0
index 076b447a1b88..70ec99cd9c53 100644
@@ -181,6 +181,26 @@ static const struct seq_operations dyn_event_seq_op = {
 	.show	= dyn_event_seq_show
 };
 
+#ifdef CONFIG_KPROBE_EVENTS
+LIST_HEAD(local_event_list);
+
+void *local_event_seq_start(struct seq_file *m, loff_t *pos)
+{
+	mutex_lock(&event_mutex);
+	return seq_list_start(&local_event_list, *pos);
+}
+
+void *local_event_seq_next(struct seq_file *m, void *v, loff_t *pos)
+{
+	return seq_list_next(v, &local_event_list, pos);
+}
+
+void local_event_seq_stop(struct seq_file *m, void *v)
+{
+	mutex_unlock(&event_mutex);
+}
+#endif /* CONFIG_KPROBE_EVENTS */
+
 /*
  * dyn_events_release_all - Release all specific events
  * @type:	the dyn_event_operations * which filters releasing events
kernel/trace/trace_dynevent.h +37 -0
index 936477a111d3..e30193470295 100644
@@ -101,6 +101,43 @@ void dyn_event_seq_stop(struct seq_file *m, void *v);
 int dyn_events_release_all(struct dyn_event_operations *type);
 int dyn_event_release(const char *raw_command, struct dyn_event_operations *type);
 
+#ifdef CONFIG_KPROBE_EVENTS
+extern struct list_head local_event_list;
+
+static inline
+int local_event_init(struct dyn_event *ev, struct dyn_event_operations *ops)
+{
+	if (!ev || !ops)
+		return -EINVAL;
+
+	INIT_LIST_HEAD(&ev->list);
+	ev->ops = ops;
+	return 0;
+}
+
+static inline int local_event_add(struct dyn_event *ev)
+{
+	lockdep_assert_held(&event_mutex);
+
+	if (!ev || !ev->ops)
+		return -EINVAL;
+
+	list_add_tail(&ev->list, &local_event_list);
+	return 0;
+}
+
+static inline void local_event_remove(struct dyn_event *ev)
+{
+	lockdep_assert_held(&event_mutex);
+	list_del_init(&ev->list);
+}
+
+void *local_event_seq_start(struct seq_file *m, loff_t *pos);
+void *local_event_seq_next(struct seq_file *m, void *v, loff_t *pos);
+void local_event_seq_stop(struct seq_file *m, void *v);
+
+#endif /* CONFIG_KPROBE_EVENTS */
+
 /*
  * for_each_dyn_event	-	iterate over the dyn_event list
  * @pos:	the struct dyn_event * to use as a loop cursor
kernel/trace/trace_kprobe.c +54 -0
index a245ea673715..76f500b17b46 100644
@@ -1213,6 +1213,52 @@ static const struct file_operations kprobe_profile_ops = {
 	.release        = seq_release,
 };
 
+#ifdef CONFIG_KPROBE_EVENTS
+/* kprobe Local profile  */
+static int local_probes_profile_seq_show(struct seq_file *m, void *v)
+{
+	struct dyn_event *ev = v;
+	struct trace_kprobe *tk;
+
+	if (!is_trace_kprobe(ev))
+		return 0;
+
+	tk = to_trace_kprobe(ev);
+	seq_printf(m, "  %-44s %15lu %15lu\n",
+		trace_probe_name(&tk->tp),
+		trace_kprobe_nhit(tk),
+		tk->rp.kp.nmissed);
+
+	return 0;
+}
+
+static const struct seq_operations local_profile_seq_op = {
+	.start  = local_event_seq_start,
+	.next   = local_event_seq_next,
+	.stop   = local_event_seq_stop,
+	.show   = local_probes_profile_seq_show
+};
+
+static int local_profile_open(struct inode *inode, struct file *file)
+{
+	int ret;
+
+	ret = security_locked_down(LOCKDOWN_TRACEFS);
+	if (ret)
+		return ret;
+
+	return seq_open(file, &local_profile_seq_op);
+}
+
+static const struct file_operations kprobe_local_profile_ops = {
+	.owner          = THIS_MODULE,
+	.open           = local_profile_open,
+	.read           = seq_read,
+	.llseek         = seq_lseek,
+	.release        = seq_release,
+};
+#endif /* CONFIG_KPROBE_EVENTS */
+
 /* Kprobe specific fetch functions */
 
 /* Return the length of string -- including null terminal byte */
@@ -1830,6 +1876,7 @@ create_local_trace_kprobe(char *func, void *addr, unsigned long offs,
 	if (ret < 0)
 		goto error;
 
+	local_event_add(&tk->devent);
 	return trace_probe_event_call(&tk->tp);
 error:
 	free_trace_kprobe(tk);
@@ -1849,6 +1896,7 @@ void destroy_local_trace_kprobe(struct trace_event_call *event_call)
 		return;
 	}
 
+	local_event_remove(&tk->devent);
 	__unregister_trace_kprobe(tk);
 
 	free_trace_kprobe(tk);
@@ -1929,6 +1977,12 @@ static __init int init_kprobe_trace(void)
 	trace_create_file("kprobe_profile", TRACE_MODE_READ,
 			  NULL, NULL, &kprobe_profile_ops);
 
+#ifdef CONFIG_KPROBE_EVENTS
+	/* kprobe Local profile */
+	tracefs_create_file("kprobe_local_profile", TRACE_MODE_READ,
+			  NULL, NULL, &kprobe_local_profile_ops);
+#endif /* CONFIG_KPROBE_EVENTS */
+
 	setup_boot_kprobe_events();
 
 	return 0;
Signature · 1 line
Masami Hiramatsu (Google) <[email protected]>
author
#3 lore raw
Quoted 19 lines · Masami Hiramatsu

[Adding bpf ML]

Quoted 6 lines · On Tue, 6 Sep 2022 11:12:14 -0400 Steven Rostedt <[email protected]> wrote:

[ Adding Masami and Tom ]

On Mon, 25 Jul 2022 06:23:34 +0000
Jianlin Lv <[email protected]> wrote:

Quoted 6 lines
Quoted 4 lines

The private kprobes are not added to the global list dyn_event_list, so there is a missing interface to show probe hit and probe miss. This patch adds a profiling interface to check the number of hits or misses for private kprobes.

Masami, what do you think of this patch?

Quoted 7 lines

I discussed it with BPF people when it was introduced and they didn't want to show up it on tracefs because it is a private one. I agreed that.

So I think this kind of interface must be managed by BPF subsystem. Is there any API to manage the BPF probe points in BPF subsystem?

Thank you,

As far as I know, there is no API to manage private kprobes in the BPF subsystem, so I added a tracefs interface to display private info.

Could you give me some hints about how to implement the current functionality in the BPF subsystem?

Regards,
Jianlin

Quoted 202 lines
Quoted 196 lines

-- Steve

Quoted 192 lines

---
v2: update commit message

---
 Documentation/trace/kprobetrace.rst |  6 +++-
 kernel/trace/trace_dynevent.c       | 20 +++++++++++
 kernel/trace/trace_dynevent.h       | 37 ++++++++++++++++++++
 kernel/trace/trace_kprobe.c         | 54 +++++++++++++++++++++++++++++
 4 files changed, 116 insertions(+), 1 deletion(-)
Documentation/trace/kprobetrace.rst +5 -1
index b175d88f31eb..8815d64dd8a6 100644
@@ -146,7 +146,11 @@ trigger:
 Event Profiling
 ---------------
 You can check the total number of probe hits and probe miss-hits via
-/sys/kernel/debug/tracing/kprobe_profile.
+/sys/kernel/debug/tracing/kprobe_profile or
+/sys/kernel/debug/tracing/kprobe_local_profile.
+All kprobe events created by kprobe_events will be added to the global
+list, you can get their profiling via kprobe_profile; kprobe_local_profile
+shows profiling for private kprobe events created by perf_kprobe pmu.
 The first column is event name, the second is the number of probe hits,
 the third is the number of probe miss-hits.
 
kernel/trace/trace_dynevent.c +20 -0
index 076b447a1b88..70ec99cd9c53 100644
@@ -181,6 +181,26 @@ static const struct seq_operations dyn_event_seq_op = {
    .show   = dyn_event_seq_show
 };
 
+#ifdef CONFIG_KPROBE_EVENTS
+LIST_HEAD(local_event_list);
+
+void *local_event_seq_start(struct seq_file *m, loff_t *pos)
+{
+   mutex_lock(&event_mutex);
+   return seq_list_start(&local_event_list, *pos);
+}
+
+void *local_event_seq_next(struct seq_file *m, void *v, loff_t *pos)
+{
+   return seq_list_next(v, &local_event_list, pos);
+}
+
+void local_event_seq_stop(struct seq_file *m, void *v)
+{
+   mutex_unlock(&event_mutex);
+}
+#endif /* CONFIG_KPROBE_EVENTS */
+
 /*
  * dyn_events_release_all - Release all specific events
  * @type:  the dyn_event_operations * which filters releasing events
kernel/trace/trace_dynevent.h +37 -0
index 936477a111d3..e30193470295 100644
@@ -101,6 +101,43 @@ void dyn_event_seq_stop(struct seq_file *m, void *v);
 int dyn_events_release_all(struct dyn_event_operations *type);
 int dyn_event_release(const char *raw_command, struct dyn_event_operations *type);
 
+#ifdef CONFIG_KPROBE_EVENTS
+extern struct list_head local_event_list;
+
+static inline
+int local_event_init(struct dyn_event *ev, struct dyn_event_operations *ops)
+{
+   if (!ev || !ops)
+           return -EINVAL;
+
+   INIT_LIST_HEAD(&ev->list);
+   ev->ops = ops;
+   return 0;
+}
+
+static inline int local_event_add(struct dyn_event *ev)
+{
+   lockdep_assert_held(&event_mutex);
+
+   if (!ev || !ev->ops)
+           return -EINVAL;
+
+   list_add_tail(&ev->list, &local_event_list);
+   return 0;
+}
+
+static inline void local_event_remove(struct dyn_event *ev)
+{
+   lockdep_assert_held(&event_mutex);
+   list_del_init(&ev->list);
+}
+
+void *local_event_seq_start(struct seq_file *m, loff_t *pos);
+void *local_event_seq_next(struct seq_file *m, void *v, loff_t *pos);
+void local_event_seq_stop(struct seq_file *m, void *v);
+
+#endif /* CONFIG_KPROBE_EVENTS */
+
 /*
  * for_each_dyn_event      -       iterate over the dyn_event list
  * @pos:   the struct dyn_event * to use as a loop cursor
kernel/trace/trace_kprobe.c +54 -0
index a245ea673715..76f500b17b46 100644
@@ -1213,6 +1213,52 @@ static const struct file_operations kprobe_profile_ops = {
    .release        = seq_release,
 };
 
+#ifdef CONFIG_KPROBE_EVENTS
+/* kprobe Local profile  */
+static int local_probes_profile_seq_show(struct seq_file *m, void *v)
+{
+   struct dyn_event *ev = v;
+   struct trace_kprobe *tk;
+
+   if (!is_trace_kprobe(ev))
+           return 0;
+
+   tk = to_trace_kprobe(ev);
+   seq_printf(m, "  %-44s %15lu %15lu\n",
+           trace_probe_name(&tk->tp),
+           trace_kprobe_nhit(tk),
+           tk->rp.kp.nmissed);
+
+   return 0;
+}
+
+static const struct seq_operations local_profile_seq_op = {
+   .start  = local_event_seq_start,
+   .next   = local_event_seq_next,
+   .stop   = local_event_seq_stop,
+   .show   = local_probes_profile_seq_show
+};
+
+static int local_profile_open(struct inode *inode, struct file *file)
+{
+   int ret;
+
+   ret = security_locked_down(LOCKDOWN_TRACEFS);
+   if (ret)
+           return ret;
+
+   return seq_open(file, &local_profile_seq_op);
+}
+
+static const struct file_operations kprobe_local_profile_ops = {
+   .owner          = THIS_MODULE,
+   .open           = local_profile_open,
+   .read           = seq_read,
+   .llseek         = seq_lseek,
+   .release        = seq_release,
+};
+#endif /* CONFIG_KPROBE_EVENTS */
+
 /* Kprobe specific fetch functions */
 
 /* Return the length of string -- including null terminal byte */
@@ -1830,6 +1876,7 @@ create_local_trace_kprobe(char *func, void *addr, unsigned long offs,
    if (ret < 0)
            goto error;
 
+   local_event_add(&tk->devent);
    return trace_probe_event_call(&tk->tp);
 error:
    free_trace_kprobe(tk);
@@ -1849,6 +1896,7 @@ void destroy_local_trace_kprobe(struct trace_event_call *event_call)
            return;
    }
 
+   local_event_remove(&tk->devent);
    __unregister_trace_kprobe(tk);
 
    free_trace_kprobe(tk);
@@ -1929,6 +1977,12 @@ static __init int init_kprobe_trace(void)
    trace_create_file("kprobe_profile", TRACE_MODE_READ,
                      NULL, NULL, &kprobe_profile_ops);
 
+#ifdef CONFIG_KPROBE_EVENTS
+   /* kprobe Local profile */
+   tracefs_create_file("kprobe_local_profile", TRACE_MODE_READ,
+                     NULL, NULL, &kprobe_local_profile_ops);
+#endif /* CONFIG_KPROBE_EVENTS */
+
    setup_boot_kprobe_events();
 
Quoted 2 lines

kernel/trace/trace_kprobe.c static __init int init_kprobe_trace(void) · line 1988

Quoted 1 line
kernel/trace/trace_kprobe.c +0 -0
    return 0;

--
Masami Hiramatsu (Google) <[email protected]>

#4 lore raw
Quoted 26 lines · On Fri, 9 Sep 2022 09:26:23 +0800 Jianlin Lv <[email protected]> wrote:
Quoted 25 lines · Masami Hiramatsu

[Adding bpf ML]

Quoted 6 lines · On Tue, 6 Sep 2022 11:12:14 -0400 Steven Rostedt <[email protected]> wrote:

[ Adding Masami and Tom ]

On Mon, 25 Jul 2022 06:23:34 +0000
Jianlin Lv <[email protected]> wrote:

Quoted 6 lines
Quoted 4 lines

The private kprobes are not added to the global list dyn_event_list, so there is a missing interface to show probe hit and probe miss. This patch adds a profiling interface to check the number of hits or misses for private kprobes.

Masami, what do you think of this patch?

I discussed it with BPF people when it was introduced and they didn't want to show up it on tracefs because it is a private one. I agreed that.

So I think this kind of interface must be managed by BPF subsystem. Is there any API to manage the BPF probe points in BPF subsystem?

Quoted 7 lines
Quoted 1 line

Thank you,

As far as I know, there is no API to manage private kprobes in the BPF subsystem, so I added a tracefs interface to display private info.

Could you give me some hints about how to implement the current functionality in the BPF subsystem?

There is tools/bpf/bpftool, I think that is the best tool to implement such feature. If hitcount is shown with the BPF program, it should be more useful.

Thank you,

Quoted 206 lines

Regards,
Jianlin

Quoted 202 lines
Quoted 196 lines

-- Steve

Quoted 192 lines

---
v2: update commit message

---
 Documentation/trace/kprobetrace.rst |  6 +++-
 kernel/trace/trace_dynevent.c       | 20 +++++++++++
 kernel/trace/trace_dynevent.h       | 37 ++++++++++++++++++++
 kernel/trace/trace_kprobe.c         | 54 +++++++++++++++++++++++++++++
 4 files changed, 116 insertions(+), 1 deletion(-)
Documentation/trace/kprobetrace.rst +5 -1
index b175d88f31eb..8815d64dd8a6 100644
@@ -146,7 +146,11 @@ trigger:
 Event Profiling
 ---------------
 You can check the total number of probe hits and probe miss-hits via
-/sys/kernel/debug/tracing/kprobe_profile.
+/sys/kernel/debug/tracing/kprobe_profile or
+/sys/kernel/debug/tracing/kprobe_local_profile.
+All kprobe events created by kprobe_events will be added to the global
+list, you can get their profiling via kprobe_profile; kprobe_local_profile
+shows profiling for private kprobe events created by perf_kprobe pmu.
 The first column is event name, the second is the number of probe hits,
 the third is the number of probe miss-hits.
 
kernel/trace/trace_dynevent.c +20 -0
index 076b447a1b88..70ec99cd9c53 100644
@@ -181,6 +181,26 @@ static const struct seq_operations dyn_event_seq_op = {
    .show   = dyn_event_seq_show
 };
 
+#ifdef CONFIG_KPROBE_EVENTS
+LIST_HEAD(local_event_list);
+
+void *local_event_seq_start(struct seq_file *m, loff_t *pos)
+{
+   mutex_lock(&event_mutex);
+   return seq_list_start(&local_event_list, *pos);
+}
+
+void *local_event_seq_next(struct seq_file *m, void *v, loff_t *pos)
+{
+   return seq_list_next(v, &local_event_list, pos);
+}
+
+void local_event_seq_stop(struct seq_file *m, void *v)
+{
+   mutex_unlock(&event_mutex);
+}
+#endif /* CONFIG_KPROBE_EVENTS */
+
 /*
  * dyn_events_release_all - Release all specific events
  * @type:  the dyn_event_operations * which filters releasing events
kernel/trace/trace_dynevent.h +37 -0
index 936477a111d3..e30193470295 100644
@@ -101,6 +101,43 @@ void dyn_event_seq_stop(struct seq_file *m, void *v);
 int dyn_events_release_all(struct dyn_event_operations *type);
 int dyn_event_release(const char *raw_command, struct dyn_event_operations *type);
 
+#ifdef CONFIG_KPROBE_EVENTS
+extern struct list_head local_event_list;
+
+static inline
+int local_event_init(struct dyn_event *ev, struct dyn_event_operations *ops)
+{
+   if (!ev || !ops)
+           return -EINVAL;
+
+   INIT_LIST_HEAD(&ev->list);
+   ev->ops = ops;
+   return 0;
+}
+
+static inline int local_event_add(struct dyn_event *ev)
+{
+   lockdep_assert_held(&event_mutex);
+
+   if (!ev || !ev->ops)
+           return -EINVAL;
+
+   list_add_tail(&ev->list, &local_event_list);
+   return 0;
+}
+
+static inline void local_event_remove(struct dyn_event *ev)
+{
+   lockdep_assert_held(&event_mutex);
+   list_del_init(&ev->list);
+}
+
+void *local_event_seq_start(struct seq_file *m, loff_t *pos);
+void *local_event_seq_next(struct seq_file *m, void *v, loff_t *pos);
+void local_event_seq_stop(struct seq_file *m, void *v);
+
+#endif /* CONFIG_KPROBE_EVENTS */
+
 /*
  * for_each_dyn_event      -       iterate over the dyn_event list
  * @pos:   the struct dyn_event * to use as a loop cursor
kernel/trace/trace_kprobe.c +54 -0
index a245ea673715..76f500b17b46 100644
@@ -1213,6 +1213,52 @@ static const struct file_operations kprobe_profile_ops = {
    .release        = seq_release,
 };
 
+#ifdef CONFIG_KPROBE_EVENTS
+/* kprobe Local profile  */
+static int local_probes_profile_seq_show(struct seq_file *m, void *v)
+{
+   struct dyn_event *ev = v;
+   struct trace_kprobe *tk;
+
+   if (!is_trace_kprobe(ev))
+           return 0;
+
+   tk = to_trace_kprobe(ev);
+   seq_printf(m, "  %-44s %15lu %15lu\n",
+           trace_probe_name(&tk->tp),
+           trace_kprobe_nhit(tk),
+           tk->rp.kp.nmissed);
+
+   return 0;
+}
+
+static const struct seq_operations local_profile_seq_op = {
+   .start  = local_event_seq_start,
+   .next   = local_event_seq_next,
+   .stop   = local_event_seq_stop,
+   .show   = local_probes_profile_seq_show
+};
+
+static int local_profile_open(struct inode *inode, struct file *file)
+{
+   int ret;
+
+   ret = security_locked_down(LOCKDOWN_TRACEFS);
+   if (ret)
+           return ret;
+
+   return seq_open(file, &local_profile_seq_op);
+}
+
+static const struct file_operations kprobe_local_profile_ops = {
+   .owner          = THIS_MODULE,
+   .open           = local_profile_open,
+   .read           = seq_read,
+   .llseek         = seq_lseek,
+   .release        = seq_release,
+};
+#endif /* CONFIG_KPROBE_EVENTS */
+
 /* Kprobe specific fetch functions */
 
 /* Return the length of string -- including null terminal byte */
@@ -1830,6 +1876,7 @@ create_local_trace_kprobe(char *func, void *addr, unsigned long offs,
    if (ret < 0)
            goto error;
 
+   local_event_add(&tk->devent);
    return trace_probe_event_call(&tk->tp);
 error:
    free_trace_kprobe(tk);
@@ -1849,6 +1896,7 @@ void destroy_local_trace_kprobe(struct trace_event_call *event_call)
            return;
    }
 
+   local_event_remove(&tk->devent);
    __unregister_trace_kprobe(tk);
 
    free_trace_kprobe(tk);
@@ -1929,6 +1977,12 @@ static __init int init_kprobe_trace(void)
    trace_create_file("kprobe_profile", TRACE_MODE_READ,
                      NULL, NULL, &kprobe_profile_ops);
 
+#ifdef CONFIG_KPROBE_EVENTS
+   /* kprobe Local profile */
+   tracefs_create_file("kprobe_local_profile", TRACE_MODE_READ,
+                     NULL, NULL, &kprobe_local_profile_ops);
+#endif /* CONFIG_KPROBE_EVENTS */
+
    setup_boot_kprobe_events();
 
Quoted 2 lines

kernel/trace/trace_kprobe.c static __init int init_kprobe_trace(void) · line 1988

Quoted 1 line
kernel/trace/trace_kprobe.c +0 -0
    return 0;

--
Masami Hiramatsu (Google) <[email protected]>

Signature · 1 line
Masami Hiramatsu (Google) <[email protected]>