tracing/uprobes: Support ftrace_event_file base multibuffer

[v3->v4]:
1. use "bool enabled = is_trace_uprobe_enabled(tu);", suggested by Oleg.
2. use list_add_tail_rcu instead of list_add_rcu, suggested by Masami.
3. fix the error handling in probe_event_enable, found by Srikar.

-------------------

Support multi-buffer on uprobe-based dynamic events by using ftrace_event_file.

This patch is based kprobe-based dynamic events multibuffer support work initially, commited by Masami(commit 41a7dd420c), but revised as below:

Oleg changed the kprobe-based multibuffer design from array-pointers of ftrace_event_file into simple list, so this patch also change to the list degisn.

rcu_read_lock/unlock added into uprobe_trace_func/uretprobe_trace_func, to synchronize with ftrace_event_file list add and delete.

Even though we allow multi-uprobes instances now, but TP_FLAG_PROFILE/TP_FLAG_TRACE are still mutually exclusive in probe_event_enable currently, this means we cannot allow one user is using uprobe-tracer, and another user is using perf-probe on same uprobe concurrently.
(Perhaps this will be fix in future, kprobe dont't have this limitation now)

Signed-off-by zhangwei(Jovi) <[email protected]>

Cc Masami Hiramatsu <[email protected]>

Cc Frederic Weisbecker <[email protected]>

Cc Oleg Nesterov <[email protected]>

Cc Srikar Dronamraju <[email protected]>

---
 kernel/trace/trace_uprobe.c |  123 +++++++++++++++++++++++++++++++++++--------
 1 file changed, 101 insertions(+), 22 deletions(-)

kernel/trace/trace_uprobe.c +101 -22

index 32494fb0..760d4ed 100644
@@ -53,6 +53,7 @@ struct trace_uprobe {
 	struct list_head		list;
 	struct ftrace_event_class	class;
 	struct ftrace_event_call	call;
+	struct list_head		files;
 	struct trace_uprobe_filter	filter;
 	struct uprobe_consumer		consumer;
 	struct inode			*inode;

@@ -65,6 +66,11 @@ struct trace_uprobe {
 	struct probe_arg		args[];
 };
 
+struct event_file_link {
+	struct ftrace_event_file	*file;
+	struct list_head		list;
+};
+
 #define SIZEOF_TRACE_UPROBE(n)			\
 	(offsetof(struct trace_uprobe, args) +	\
 	(sizeof(struct probe_arg) * (n)))

@@ -124,6 +130,7 @@ alloc_trace_uprobe(const char *group, const char *event, int nargs, bool is_ret)
 		goto error;
 
 	INIT_LIST_HEAD(&tu->list);
+	INIT_LIST_HEAD(&tu->files);
 	tu->consumer.handler = uprobe_dispatcher;
 	if (is_ret)
 		tu->consumer.ret_handler = uretprobe_dispatcher;

@@ -511,7 +518,8 @@ static const struct file_operations uprobe_profile_ops = {
 };
 
 static void uprobe_trace_print(struct trace_uprobe *tu,
-				unsigned long func, struct pt_regs *regs)
+				unsigned long func, struct pt_regs *regs,
+				struct ftrace_event_file *ftrace_file)
 {
 	struct uprobe_trace_entry_head *entry;
 	struct ring_buffer_event *event;

@@ -520,9 +528,12 @@ static void uprobe_trace_print(struct trace_uprobe *tu,
 	int size, i;
 	struct ftrace_event_call *call = &tu->call;
 
+	WARN_ON(call != ftrace_file->event_call);
+
 	size = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
-	event = trace_current_buffer_lock_reserve(&buffer, call->event.type,
-						  size + tu->size, 0, 0);
+	event = trace_event_buffer_lock_reserve(&buffer, ftrace_file,
+						call->event.type,
+						size + tu->size, 0, 0);
 	if (!event)
 		return;
 

@@ -546,15 +557,28 @@ static void uprobe_trace_print(struct trace_uprobe *tu,
 /* uprobe handler */
 static int uprobe_trace_func(struct trace_uprobe *tu, struct pt_regs *regs)
 {
-	if (!is_ret_probe(tu))
-		uprobe_trace_print(tu, 0, regs);
+	struct event_file_link *link;
+
+	if (is_ret_probe(tu))
+		return 0;
+
+	rcu_read_lock();
+	list_for_each_entry(link, &tu->files, list)
+		uprobe_trace_print(tu, 0, regs, link->file);
+	rcu_read_unlock();
+
 	return 0;
 }
 
 static void uretprobe_trace_func(struct trace_uprobe *tu, unsigned long func,
 				struct pt_regs *regs)
 {
-	uprobe_trace_print(tu, func, regs);
+	struct event_file_link *link;
+
+	rcu_read_lock();
+	list_for_each_entry(link, &tu->files, list)
+		uprobe_trace_print(tu, func, regs, link->file);
+	rcu_read_unlock();
 }
 
 /* Event entry printers */

@@ -605,33 +629,87 @@ typedef bool (*filter_func_t)(struct uprobe_consumer *self,
 				struct mm_struct *mm);
 
 static int
-probe_event_enable(struct trace_uprobe *tu, int flag, filter_func_t filter)
+probe_event_enable(struct trace_uprobe *tu, struct ftrace_event_file *file,
+		   filter_func_t filter)
 {
+	bool enabled = is_trace_uprobe_enabled(tu);
+	struct event_file_link *link;
 	int ret = 0;
 
-	if (is_trace_uprobe_enabled(tu))
-		return -EINTR;
+	if (file) {
+		if (tu->flags & TP_FLAG_PROFILE)
+			return -EINTR;
+
+		link = kmalloc(sizeof(*link), GFP_KERNEL);
+		if (!link)
+			return -ENOMEM;
+
+		link->file = file;
+		list_add_tail_rcu(&link->list, &tu->files);
+
+		tu->flags |= TP_FLAG_TRACE;
+	} else {
+		if (tu->flags & TP_FLAG_TRACE)
+			return -EINTR;
+
+		tu->flags |= TP_FLAG_PROFILE;
+	}
 
 	WARN_ON(!uprobe_filter_is_empty(&tu->filter));
 
-	tu->flags |= flag;
-	tu->consumer.filter = filter;
-	ret = uprobe_register(tu->inode, tu->offset, &tu->consumer);
-	if (ret)
-		tu->flags &= ~flag;
+	/* we cannot call uprobe_register twice for same tu */
+	if (!enabled) {
+		tu->consumer.filter = filter;
+		ret = uprobe_register(tu->inode, tu->offset, &tu->consumer);
+	}
+
+	if (ret) {
+		if (file) {
+			list_del_rcu(&link->list);
+			kfree(link);
+			tu->flags &= ~TP_FLAG_TRACE;
+		} else
+			tu->flags &= ~TP_FLAG_PROFILE;
+	}
 
 	return ret;
 }
 
-static void probe_event_disable(struct trace_uprobe *tu, int flag)
+static struct event_file_link *
+find_event_file_link(struct trace_uprobe *tu, struct ftrace_event_file *file)
 {
-	if (!is_trace_uprobe_enabled(tu))
-		return;
+	struct event_file_link *link;
+
+	list_for_each_entry(link, &tu->files, list)
+		if (link->file == file)
+			return link;
+
+	return NULL;
+}
+
+static void
+probe_event_disable(struct trace_uprobe *tu, struct ftrace_event_file *file)
+{
+	if (file) {
+		struct event_file_link *link;
+
+		link = find_event_file_link(tu, file);
+		if (!link)
+			return;
+
+		list_del_rcu(&link->list);
+		/* synchronize with uprobe_trace_func/uretprobe_trace_func */
+		synchronize_sched();
+		kfree(link);
+
+		if (!list_empty(&tu->files))
+			return;
+	}
 
 	WARN_ON(!uprobe_filter_is_empty(&tu->filter));
 
 	uprobe_unregister(tu->inode, tu->offset, &tu->consumer);
-	tu->flags &= ~flag;
+	tu->flags &= file ? ~TP_FLAG_TRACE : ~TP_FLAG_PROFILE;
 }
 
 static int uprobe_event_define_fields(struct ftrace_event_call *event_call)

@@ -867,21 +945,22 @@ static
 int trace_uprobe_register(struct ftrace_event_call *event, enum trace_reg type, void *data)
 {
 	struct trace_uprobe *tu = event->data;
+	struct ftrace_event_file *file = data;
 
 	switch (type) {
 	case TRACE_REG_REGISTER:
-		return probe_event_enable(tu, TP_FLAG_TRACE, NULL);
+		return probe_event_enable(tu, file, NULL);
 
 	case TRACE_REG_UNREGISTER:
-		probe_event_disable(tu, TP_FLAG_TRACE);
+		probe_event_disable(tu, file);
 		return 0;
 
 #ifdef CONFIG_PERF_EVENTS
 	case TRACE_REG_PERF_REGISTER:
-		return probe_event_enable(tu, TP_FLAG_PROFILE, uprobe_perf_filter);
+		return probe_event_enable(tu, NULL, uprobe_perf_filter);
 
 	case TRACE_REG_PERF_UNREGISTER:
-		probe_event_disable(tu, TP_FLAG_PROFILE);
+		probe_event_disable(tu, NULL);
 		return 0;
 
 	case TRACE_REG_PERF_OPEN:

[v3->v4]:

kernel/trace/trace_uprobe.c +17 -2

-	if (is_trace_uprobe_enabled(tu))
-		return -EINTR;
+	if (file) {
+		if (tu->flags & TP_FLAG_PROFILE)
+			return -EINTR;
+
+		link = kmalloc(sizeof(*link), GFP_KERNEL);
+		if (!link)
+			return -ENOMEM;
+
+		link->file = file;
+		list_add_tail_rcu(&link->list, &tu->files);
+
+		tu->flags |= TP_FLAG_TRACE;
+	} else {
+		if (tu->flags & TP_FLAG_TRACE)
+			return -EINTR;
+
+		tu->flags |= TP_FLAG_PROFILE;

Quoted 2 lines · zhangwei(Jovi)

[v3->v4]:

I am wondering how much you will hate me if I suggest to make v5 ;)

But look, imho probe_event_enable() looks a bit more confusing than it needs.

Quoted 9 lines · Oleg Nesterov

Quoted 2 lines · zhangwei(Jovi)

[v3->v4]:

I am wondering how much you will hate me if I suggest to make v5 ;)

But look, imho probe_event_enable() looks a bit more confusing than it needs.

And I am a bit worried this patch removes the is_trace_uprobe_enabled() check from probe_event_enable()...

Yes I think it was never needed, afaics TRACE_REG_*_UNREGISTER can't come without successfull _REGISTER. And the bogus uprobe_unregister() is harmless in this particular case.

So I think this is fine, but perhaps the changelog should mention this "offtopic" change.

Oleg.

Quoted 2 lines · zhangwei(Jovi)

[v3->v4]:

I am wondering how much you will hate me if I suggest to make v5 ;)

But look, imho probe_event_enable() looks a bit more confusing than it needs.

kernel/trace/trace_uprobe.c typedef bool (*filter_func_t)(struct uprobe_consumer *self, · line 637

Quoted 7 lines

kernel/trace/trace_uprobe.c +4 -1

-probe_event_enable(struct trace_uprobe *tu, int flag, filter_func_t filter)
+probe_event_enable(struct trace_uprobe *tu, struct ftrace_event_file *file,
+		   filter_func_t filter)
 {
+	bool enabled = is_trace_uprobe_enabled(tu);
+	struct event_file_link *link;
 	int ret = 0;

Unnecessary initialization.

Quoted 19 lines

kernel/trace/trace_uprobe.c +17 -2

-	if (is_trace_uprobe_enabled(tu))
-		return -EINTR;
+	if (file) {
+		if (tu->flags & TP_FLAG_PROFILE)
+			return -EINTR;
+
+		link = kmalloc(sizeof(*link), GFP_KERNEL);
+		if (!link)
+			return -ENOMEM;
+
+		link->file = file;
+		list_add_tail_rcu(&link->list, &tu->files);
+
+		tu->flags |= TP_FLAG_TRACE;
+	} else {
+		if (tu->flags & TP_FLAG_TRACE)
+			return -EINTR;
+
+		tu->flags |= TP_FLAG_PROFILE;

kernel/trace/trace_uprobe.c typedef bool (*filter_func_t)(struct uprobe_consumer *self, · line 660

Quoted 10 lines

kernel/trace/trace_uprobe.c +2 -5

+	}
 
 	WARN_ON(!uprobe_filter_is_empty(&tu->filter));
 
-	tu->flags |= flag;
-	tu->consumer.filter = filter;
-	ret = uprobe_register(tu->inode, tu->offset, &tu->consumer);
-	if (ret)
-		tu->flags &= ~flag;
+	/* we cannot call uprobe_register twice for same tu */

The comment is confusing, I'd suggest to simply remove it.

Yes, we can't do uprobe_register() twice as we already discussed. But it is not that we "can't", we simply do not need this if uprobe was already created.

kernel/trace/trace_uprobe.c typedef bool (*filter_func_t)(struct uprobe_consumer *self, · line 668

Quoted 8 lines

kernel/trace/trace_uprobe.c +8 -0

+	if (!enabled) {
+		tu->consumer.filter = filter;
+		ret = uprobe_register(tu->inode, tu->offset, &tu->consumer);
+	}
+
+	if (ret) {
+		if (file) {
+			list_del_rcu(&link->list);

I won't insist, but _rcu is not needed in this case. Again, this looks a bit confusing, as if we expect that some rcu reader can ever see this entry. But this is not true and we are going to just kfree it without synchronize_rcu().

kernel/trace/trace_uprobe.c typedef bool (*filter_func_t)(struct uprobe_consumer *self, · line 673

Quoted 5 lines

kernel/trace/trace_uprobe.c +5 -0

+			kfree(link);
+			tu->flags &= ~TP_FLAG_TRACE;
+		} else
+			tu->flags &= ~TP_FLAG_PROFILE;
+	}

This is correct, but again, this is not immediately obvious.

Why it is correct to correct to clear TP_FLAG_TRACE? Because we know that "enabled" was false and thus we remove the single list entry.

So, perhaps,

	if (enabled)
		return 0;

	ret = uprobe_register();
	if (ret) {
		...;
	}

	return ret;

will be a bit more clean.

Oleg.

.

kernel/trace/trace_uprobe.c +22 -5

@@ -605,33 +629,87 @@ typedef bool (*filter_func_t)(struct uprobe_consumer *self,
 				struct mm_struct *mm);
 
 static int
-probe_event_enable(struct trace_uprobe *tu, int flag, filter_func_t filter)
+probe_event_enable(struct trace_uprobe *tu, struct ftrace_event_file *file,
+		   filter_func_t filter)
 {
+	bool enabled = is_trace_uprobe_enabled(tu);
+	struct event_file_link *link;
 	int ret = 0;
 
-	if (is_trace_uprobe_enabled(tu))
-		return -EINTR;
+	if (file) {
+		if (tu->flags & TP_FLAG_PROFILE)
+			return -EINTR;
+
+		link = kmalloc(sizeof(*link), GFP_KERNEL);
+		if (!link)
+			return -ENOMEM;
+
+		link->file = file;
+		list_add_tail_rcu(&link->list, &tu->files);
+
+		tu->flags |= TP_FLAG_TRACE;
+	} else {
+		if (tu->flags & TP_FLAG_TRACE)
+			return -EINTR;
+
+		tu->flags |= TP_FLAG_PROFILE;
+	}
 
 	WARN_ON(!uprobe_filter_is_empty(&tu->filter));
 
-	tu->flags |= flag;
-	tu->consumer.filter = filter;