Skip to contentSkip to conversation

bpf: Make BPF trampoline friendly to ftrace

PATCH In review Alexei Starovoitov 4 participants 9 messages Dec 9, 2019 – Dec 10, 2019

Subsystems (9)
lore
Share & tools
lore

Alexei Starovoitov

author maintainer
#0 lore raw

cover bpf: Make BPF trampoline friendly to ftrace

Patch 1 - fix crash function_graph tracer encounters BPF trampoline Patch 2 - use register_ftrace_direct() API to attach BPF trampoline Patch 3 - simple test

Alexei Starovoitov (3):
ftrace: Fix function_graph tracer interaction with BPF trampoline
bpf: Make BPF trampoline use register_ftrace_direct() API selftests/bpf: test function_graph tracer and bpf trampoline together

 arch/x86/kernel/ftrace.c                   | 14 -----
 include/linux/bpf.h                        |  1 +
 include/linux/ftrace.h                     |  5 ++
 kernel/bpf/trampoline.c                    | 64 ++++++++++++++++++++--
 kernel/trace/fgraph.c                      |  9 +++
 kernel/trace/ftrace.c                      | 19 +++----
 tools/testing/selftests/bpf/test_ftrace.sh | 39 +++++++++++++
 7 files changed, 119 insertions(+), 32 deletions(-)
 create mode 100755 tools/testing/selftests/bpf/test_ftrace.sh
Signature · 1 line
2.23.0
author maintainer
#1 lore raw

PATCH 1/3 ftrace: Fix function_graph tracer interaction with BPF trampoline

Depending on type of BPF programs served by BPF trampoline it can call original function. In such case the trampoline will skip one stack frame while returning. That will confuse function_graph tracer and will cause crashes with bad RIP. Teach graph tracer to skip functions that have BPF trampoline attached.

---
 arch/x86/kernel/ftrace.c | 14 --------------
 include/linux/ftrace.h   |  5 +++++
 kernel/trace/fgraph.c    |  9 +++++++++
 kernel/trace/ftrace.c    | 19 +++++++------------
 4 files changed, 21 insertions(+), 26 deletions(-)
arch/x86/kernel/ftrace.c +0 -14
index 060a361d9d11..024c3053dbba 100644
@@ -1042,20 +1042,6 @@ void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent,
 	if (unlikely(atomic_read(&current->tracing_graph_pause)))
 		return;
 
-	/*
-	 * If the return location is actually pointing directly to
-	 * the start of a direct trampoline (if we trace the trampoline
-	 * it will still be offset by MCOUNT_INSN_SIZE), then the
-	 * return address is actually off by one word, and we
-	 * need to adjust for that.
-	 */
-	if (ftrace_direct_func_count) {
-		if (ftrace_find_direct_func(self_addr + MCOUNT_INSN_SIZE)) {
-			self_addr = *parent;
-			parent++;
-		}
-	}
-
 	/*
 	 * Protect against fault, even if it shouldn't
 	 * happen. This tool is too much intrusive to
include/linux/ftrace.h +5 -0
index 7247d35c3d16..db95244a62d4 100644
@@ -264,6 +264,7 @@ int ftrace_modify_direct_caller(struct ftrace_func_entry *entry,
 				struct dyn_ftrace *rec,
 				unsigned long old_addr,
 				unsigned long new_addr);
+unsigned long ftrace_find_rec_direct(unsigned long ip);
 #else
 # define ftrace_direct_func_count 0
 static inline int register_ftrace_direct(unsigned long ip, unsigned long addr)
@@ -290,6 +291,10 @@ static inline int ftrace_modify_direct_caller(struct ftrace_func_entry *entry,
 {
 	return -ENODEV;
 }
+static inline unsigned long ftrace_find_rec_direct(unsigned long ip)
+{
+	return 0;
+}
 #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
 
 #ifndef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
kernel/trace/fgraph.c +9 -0
index 67e0c462b059..a2659735db73 100644
@@ -101,6 +101,15 @@ int function_graph_enter(unsigned long ret, unsigned long func,
 {
 	struct ftrace_graph_ent trace;
 
+	/*
+	 * Skip graph tracing if the return location is served by direct trampoline,
+	 * since call sequence and return addresses is unpredicatable anymore.
+	 * Ex: BPF trampoline may call original function and may skip frame
+	 * depending on type of BPF programs attached.
+	 */
+	if (ftrace_direct_func_count &&
+	    ftrace_find_rec_direct(ret - MCOUNT_INSN_SIZE))
+		return -EBUSY;
 	trace.func = func;
 	trace.depth = ++current->curr_ret_depth;
 
kernel/trace/ftrace.c +7 -12
index 74439ab5c2b6..ac99a3500076 100644
@@ -2364,7 +2364,7 @@ int ftrace_direct_func_count;
  * Search the direct_functions hash to see if the given instruction pointer
  * has a direct caller attached to it.
  */
-static unsigned long find_rec_direct(unsigned long ip)
+unsigned long ftrace_find_rec_direct(unsigned long ip)
 {
 	struct ftrace_func_entry *entry;
 
@@ -2380,7 +2380,7 @@ static void call_direct_funcs(unsigned long ip, unsigned long pip,
 {
 	unsigned long addr;
 
-	addr = find_rec_direct(ip);
+	addr = ftrace_find_rec_direct(ip);
 	if (!addr)
 		return;
 
@@ -2393,11 +2393,6 @@ struct ftrace_ops direct_ops = {
 			  | FTRACE_OPS_FL_DIRECT | FTRACE_OPS_FL_SAVE_REGS
 			  | FTRACE_OPS_FL_PERMANENT,
 };
-#else
-static inline unsigned long find_rec_direct(unsigned long ip)
-{
-	return 0;
-}
 #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
 
 /**
@@ -2417,7 +2412,7 @@ unsigned long ftrace_get_addr_new(struct dyn_ftrace *rec)
 
 	if ((rec->flags & FTRACE_FL_DIRECT) &&
 	    (ftrace_rec_count(rec) == 1)) {
-		addr = find_rec_direct(rec->ip);
+		addr = ftrace_find_rec_direct(rec->ip);
 		if (addr)
 			return addr;
 		WARN_ON_ONCE(1);
@@ -2458,7 +2453,7 @@ unsigned long ftrace_get_addr_curr(struct dyn_ftrace *rec)
 
 	/* Direct calls take precedence over trampolines */
 	if (rec->flags & FTRACE_FL_DIRECT_EN) {
-		addr = find_rec_direct(rec->ip);
+		addr = ftrace_find_rec_direct(rec->ip);
 		if (addr)
 			return addr;
 		WARN_ON_ONCE(1);
@@ -3604,7 +3599,7 @@ static int t_show(struct seq_file *m, void *v)
 		if (rec->flags & FTRACE_FL_DIRECT) {
 			unsigned long direct;
 
-			direct = find_rec_direct(rec->ip);
+			direct = ftrace_find_rec_direct(rec->ip);
 			if (direct)
 				seq_printf(m, "\n\tdirect-->%pS", (void *)direct);
 		}
@@ -5008,7 +5003,7 @@ int register_ftrace_direct(unsigned long ip, unsigned long addr)
 	mutex_lock(&direct_mutex);
 
 	/* See if there's a direct function at @ip already */
-	if (find_rec_direct(ip))
+	if (ftrace_find_rec_direct(ip))
 		goto out_unlock;
 
 	ret = -ENODEV;
@@ -5027,7 +5022,7 @@ int register_ftrace_direct(unsigned long ip, unsigned long addr)
 	if (ip != rec->ip) {
 		ip = rec->ip;
 		/* Need to check this ip for a direct. */
-		if (find_rec_direct(ip))
+		if (ftrace_find_rec_direct(ip))
 			goto out_unlock;
 	}
 
Signature · 1 line
2.23.0
#2 lore raw

Re: [PATCH bpf 1/3] ftrace: Fix function_graph tracer interaction with BPF trampoline

Quoted 7 lines · Alexei Starovoitov

Depending on type of BPF programs served by BPF trampoline it can call original function. In such case the trampoline will skip one stack frame while returning. That will confuse function_graph tracer and will cause crashes with bad RIP. Teach graph tracer to skip functions that have BPF trampoline attached.

Steven, please take a look.

maintainer
#3 lore raw

Re: [PATCH bpf 1/3] ftrace: Fix function_graph tracer interaction with BPF trampoline

Quoted 10 lines · On Tue, 10 Dec 2019 08:19:42 -0800 Alexei Starovoitov…
Quoted 7 lines · Alexei Starovoitov

Depending on type of BPF programs served by BPF trampoline it can call original function. In such case the trampoline will skip one stack frame while returning. That will confuse function_graph tracer and will cause crashes with bad RIP. Teach graph tracer to skip functions that have BPF trampoline attached.

Steven, please take a look.

I'll try to get to it today or tomorrow. I have some other work to get done that my job requires I do ;-)

-- Steve

maintainer
#4 lore raw

Re: [PATCH bpf 1/3] ftrace: Fix function_graph tracer interaction with BPF trampoline

In reply to Alexei Starovoitov
Quoted 7 lines · On Sun, 8 Dec 2019 16:01:12 -0800 Alexei Starovoitov <[email protected]> wrote:

#ifndef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS

kernel/trace/fgraph.c +0 -0
index 67e0c462b059..a2659735db73 100644
@@ -101,6 +101,15 @@ int function_graph_enter(unsigned long ret, unsigned long func,
 {

kernel/trace/fgraph.c int function_graph_enter(unsigned long ret, unsigned long func, · line 111

Quoted 10 lines
kernel/trace/fgraph.c +8 -0
 	struct ftrace_graph_ent trace;
 
+	/*
+	 * Skip graph tracing if the return location is served by direct trampoline,
+	 * since call sequence and return addresses is unpredicatable anymore.
+	 * Ex: BPF trampoline may call original function and may skip frame
+	 * depending on type of BPF programs attached.
+	 */
+	if (ftrace_direct_func_count &&
+	    ftrace_find_rec_direct(ret - MCOUNT_INSN_SIZE))

My only worry is that this may not work for all archs that implement it. But I figure we can cross that bridge when we get to it.

kernel/trace/fgraph.c int function_graph_enter(unsigned long ret, unsigned long func, · line 115

Quoted 4 lines
kernel/trace/fgraph.c +1 -0
+		return -EBUSY;
 	trace.func = func;
 	trace.depth = ++current->curr_ret_depth;
 

I added this patch to my queue and it's about 70% done going through my test suite (takes around 10 - 13 hours).

As I'm about to send a pull request to Linus tomorrow, I could include this patch (as it will be fully tested), and then you could apply the other two when it hits Linus's tree.

Would that work for you?

-- Steve

#5 lore raw

Re: [PATCH bpf 1/3] ftrace: Fix function_graph tracer interaction with BPF trampoline

Quoted 12 lines · Steven Rostedt
Quoted 9 lines · On Sun, 8 Dec 2019 16:01:12 -0800 Alexei Starovoitov <[email protected]> wrote:

#ifndef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS

kernel/trace/fgraph.c +0 -0
index 67e0c462b059..a2659735db73 100644
@@ -101,6 +101,15 @@ int function_graph_enter(unsigned long ret, unsigned long func,
 {
 	struct ftrace_graph_ent trace;
 
Quoted 11 lines

kernel/trace/fgraph.c int function_graph_enter(unsigned long ret, unsigned long func, · line 111

Quoted 8 lines
kernel/trace/fgraph.c +8 -0
+	/*
+	 * Skip graph tracing if the return location is served by direct trampoline,
+	 * since call sequence and return addresses is unpredicatable anymore.
+	 * Ex: BPF trampoline may call original function and may skip frame
+	 * depending on type of BPF programs attached.
+	 */
+	if (ftrace_direct_func_count &&
+	    ftrace_find_rec_direct(ret - MCOUNT_INSN_SIZE))

My only worry is that this may not work for all archs that implement it. But I figure we can cross that bridge when we get to it.

Right. Since bpf trampoline is going to be the only user in short term it's not an issue, since trampoline is x86-64 only so far.

Quoted 8 lines

kernel/trace/fgraph.c int function_graph_enter(unsigned long ret, unsigned long func, · line 115

Quoted 4 lines
kernel/trace/fgraph.c +1 -0
+		return -EBUSY;
 	trace.func = func;
 	trace.depth = ++current->curr_ret_depth;
 

I added this patch to my queue and it's about 70% done going through my test suite (takes around 10 - 13 hours).

Quoted 5 lines

As I'm about to send a pull request to Linus tomorrow, I could include this patch (as it will be fully tested), and then you could apply the other two when it hits Linus's tree.

Would that work for you?

Awesome. Much appreciate additional testing. I can certainly wait another day. I was hoping to get patch 2 all the way to Linus's tree before rc2 to make sure register_ftrace_direct() API is used for real in this kernel cycle. When everything will land I'll backport to our production kernel and then the actual stress testing begins :)

author maintainer
#6 lore raw

PATCH 2/3 bpf: Make BPF trampoline use register_ftrace_direct() API

In reply to Alexei Starovoitov

Make BPF trampoline attach its generated assembly code to kernel functions via register_ftrace_direct() API. It helps ftrace-based tracers co-exist with BPF trampoline on the same kernel function. It also switches attaching logic from arch specific text_poke to generic ftrace that is available on many architectures. text_poke is still necessary for bpf-to-bpf attach and for bpf_tail_call optimization.

---
 include/linux/bpf.h     |  1 +
 kernel/bpf/trampoline.c | 64 +++++++++++++++++++++++++++++++++++++----
 2 files changed, 59 insertions(+), 6 deletions(-)
include/linux/bpf.h +1 -0
index 35903f148be5..ac7de5291509 100644
@@ -461,6 +461,7 @@ struct bpf_trampoline {
 	struct {
 		struct btf_func_model model;
 		void *addr;
+		bool ftrace_managed;
 	} func;
 	/* list of BPF programs using this trampoline */
 	struct hlist_head progs_hlist[BPF_TRAMP_MAX];
kernel/bpf/trampoline.c +58 -6

90 lines not on this page · raw message

Signature · 1 line
2.23.0
author maintainer
#7 lore raw

PATCH 3/3 selftests/bpf: test function_graph tracer and bpf trampoline together

In reply to Alexei Starovoitov

Add simple test script to execute funciton graph tracer while BPF trampoline attaches and detaches from the functions being graph traced.

---
 tools/testing/selftests/bpf/test_ftrace.sh | 39 ++++++++++++++++++++++
 1 file changed, 39 insertions(+)
 create mode 100755 tools/testing/selftests/bpf/test_ftrace.sh
tools/testing/selftests/bpf/test_ftrace.sh new file +39 -0
new file mode 100755
index 000000000000..20de7bb873bc
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+TR=/sys/kernel/debug/tracing/
+clear_trace() { # reset trace output
+    echo > $TR/trace
+}
+
+disable_tracing() { # stop trace recording
+    echo 0 > $TR/tracing_on
+}
+
+enable_tracing() { # start trace recording
+    echo 1 > $TR/tracing_on
+}
+
+reset_tracer() { # reset the current tracer
+    echo nop > $TR/current_tracer
+}
+
+disable_tracing
+clear_trace
+
+echo "" > $TR/set_ftrace_filter
+echo '*printk* *console* *wake* *serial* *lock*' > $TR/set_ftrace_notrace
+
+echo "bpf_prog_test*" > $TR/set_graph_function
+echo "" > $TR/set_graph_notrace
+
+echo function_graph > $TR/current_tracer
+
+enable_tracing
+./test_progs -t fentry
+./test_progs -t fexit
+disable_tracing
+clear_trace
+
+reset_tracer
+
+exit 0
Signature · 1 line
2.23.0
Acked-by on the v1 series
#8 lore raw
In reply to Alexei Starovoitov
Quoted 8 lines · Alexei Starovoitov

Patch 1 - fix crash function_graph tracer encounters BPF trampoline Patch 2 - use register_ftrace_direct() API to attach BPF trampoline Patch 3 - simple test

Alexei Starovoitov (3):
ftrace: Fix function_graph tracer interaction with BPF trampoline
bpf: Make BPF trampoline use register_ftrace_direct() API selftests/bpf: test function_graph tracer and bpf trampoline together