Skip to contentSkip to conversation

tracing: Fix syscall tracepoint use-after-free

RFC PATCH Superseded Mathieu Desnoyers 3 participants 6 messages Oct 25, 2024 – Oct 26, 2024

Subsystems (3)

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore

Mathieu Desnoyers

author reviewer
#0 lore raw

RFC PATCH 1/1 tracing: Fix syscall tracepoint use-after-free

The grace period used internally within tracepoint.c:release_probes() uses call_rcu() to batch waiting for quiescence of old probe arrays, rather than using the tracepoint_synchronize_unregister() which blocks while waiting for quiescence.

With the introduction of faultable syscall tracepoints, this causes use-after-free issues reproduced with syzkaller.

Fix this by introducing tracepoint_call_rcu(), which uses the appropriate call_rcu() or call_rcu_tasks_trace() before invoking the rcu_free_old_probes callback.

Use tracepoint_call_rcu() in bpf_link_free() for raw tracepoints as well, which has the same problem for syscall tracepoints.

Reported-by [email protected]
Fixes a363d27cdbc2 ("tracing: Allow system call tracepoints to handle page faults")
Cc Michael Jeanson <[email protected]>
Cc Steven Rostedt <[email protected]>
Cc Masami Hiramatsu <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Alexei Starovoitov <[email protected]>
Cc Yonghong Song <[email protected]>
Cc Paul E. McKenney <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Arnaldo Carvalho de Melo <[email protected]>
Cc Mark Rutland <[email protected]>
Cc Alexander Shishkin <[email protected]>
Cc Namhyung Kim <[email protected]>
Cc Andrii Nakryiko <[email protected]>
Cc Joel Fernandes <[email protected]>
Cc Jordan Rife <[email protected]>

---
Changes since v0:
- Introduce tracepoint_call_rcu(),
- Fix bpf_link_free() use of call_rcu as well.

---
 include/linux/tracepoint.h |  9 +++++++++
 kernel/bpf/syscall.c       | 23 +++++++++++++++--------
 kernel/tracepoint.c        | 22 ++++++++++++++++++----
 3 files changed, 42 insertions(+), 12 deletions(-)
include/linux/tracepoint.h +9 -0
index 0dc67fad706c..45025d6b2dd6 100644
@@ -104,6 +104,8 @@ void for_each_tracepoint_in_module(struct module *mod,
  * tracepoint_synchronize_unregister must be called between the last tracepoint
  * probe unregistration and the end of module exit to make sure there is no
  * caller executing a probe when it is freed.
+ * An alternative to tracepoint_synchronize_unregister() is to use
+ * tracepoint_call_rcu() for batched reclaim.
  */
 #ifdef CONFIG_TRACEPOINTS
 static inline void tracepoint_synchronize_unregister(void)
@@ -111,9 +113,16 @@ static inline void tracepoint_synchronize_unregister(void)
 	synchronize_rcu_tasks_trace();
 	synchronize_rcu();
 }
+
+void tracepoint_call_rcu(struct tracepoint *tp, struct rcu_head *head,
+			 void (*callback)(struct rcu_head *head));
+
 #else
 static inline void tracepoint_synchronize_unregister(void)
 { }
+static inline void tracepoint_call_rcu(struct tracepoint *tp, struct rcu_head *head,
+				       void (*callback)(struct rcu_head *head))
+{ }
 #endif
 
 #ifdef CONFIG_HAVE_SYSCALL_TRACEPOINTS
kernel/bpf/syscall.c +15 -8
index 59de664e580d..1191dc1d4206 100644
@@ -3006,14 +3006,21 @@ static void bpf_link_free(struct bpf_link *link)
 		bpf_prog_put(link->prog);
 	}
 	if (ops->dealloc_deferred) {
-		/* schedule BPF link deallocation; if underlying BPF program
-		 * is sleepable, we need to first wait for RCU tasks trace
-		 * sync, then go through "classic" RCU grace period
-		 */
-		if (sleepable)
-			call_rcu_tasks_trace(&link->rcu, bpf_link_defer_dealloc_mult_rcu_gp);
-		else
-			call_rcu(&link->rcu, bpf_link_defer_dealloc_rcu_gp);
+		if (link->type == BPF_LINK_TYPE_RAW_TRACEPOINT) {
+		        struct bpf_raw_tp_link *raw_tp =
+				container_of(link, struct bpf_raw_tp_link, link);
+
+			tracepoint_call_rcu(raw_tp->btp->tp, &link->rcu, bpf_link_defer_dealloc_rcu_gp);
+		} else {
+			/* schedule BPF link deallocation; if underlying BPF program
+			 * is sleepable, we need to first wait for RCU tasks trace
+			 * sync, then go through "classic" RCU grace period
+			 */
+			if (sleepable)
+				call_rcu_tasks_trace(&link->rcu, bpf_link_defer_dealloc_mult_rcu_gp);
+			else
+				call_rcu(&link->rcu, bpf_link_defer_dealloc_rcu_gp);
+		}
 	} else if (ops->dealloc)
 		ops->dealloc(link);
 }
kernel/tracepoint.c +18 -4
index 6474e2cf22c9..ef60c5484eda 100644
@@ -106,13 +106,27 @@ static void rcu_free_old_probes(struct rcu_head *head)
 	kfree(container_of(head, struct tp_probes, rcu));
 }
 
-static inline void release_probes(struct tracepoint_func *old)
+static bool tracepoint_is_syscall(struct tracepoint *tp)
+{
+	return !strcmp(tp->name, "sys_enter") || !strcmp(tp->name, "sys_exit");
+}
+
+void tracepoint_call_rcu(struct tracepoint *tp, struct rcu_head *head,
+			 void (*callback)(struct rcu_head *head))
+{
+	if (tracepoint_is_syscall(tp))
+		call_rcu_tasks_trace(head, callback);
+	else
+		call_rcu(head, callback);
+}
+
+static inline void release_probes(struct tracepoint *tp, struct tracepoint_func *old)
 {
 	if (old) {
 		struct tp_probes *tp_probes = container_of(old,
 			struct tp_probes, probes[0]);
 
-		call_rcu(&tp_probes->rcu, rcu_free_old_probes);
+		tracepoint_call_rcu(tp, &tp_probes->rcu, rcu_free_old_probes);
 	}
 }
 
@@ -334,7 +348,7 @@ static int tracepoint_add_func(struct tracepoint *tp,
 		break;
 	}
 
-	release_probes(old);
+	release_probes(tp, old);
 	return 0;
 }
 
@@ -406,7 +420,7 @@ static int tracepoint_remove_func(struct tracepoint *tp,
 		WARN_ON_ONCE(1);
 		break;
 	}
-	release_probes(old);
+	release_probes(tp, old);
 	return 0;
 }
 
Signature · 1 line
2.39.5
#1 lore raw

kernel/bpf/syscall.c static void bpf_link_free(struct bpf_link *link) · line 3006

Quoted 6 lines
kernel/bpf/syscall.c +0 -0
index 59de664e580d..1191dc1d4206 100644
@@ -3006,14 +3006,21 @@ static void bpf_link_free(struct bpf_link *link)
                bpf_prog_put(link->prog);

I think we would need the same treatment with bpf_prog_put here. Something like,

tracepoint_call_rcu(raw_tp->btp->tp, &link->prog->aux->rcu,

		    bpf_link_defer_bpf_prog_put);

static void bpf_link_defer_bpf_prog_put(struct rcu_head *rcu)

{
	struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu);
	bpf_prog_put(aux->prox);
}

Alternatively, some context would need to be passed down to __bpf_prog_put_noref via the call to bpf_prog_put so it can choose whether or not to use call_rcu or call_rcu_tasks_trace.

kernel/bpf/syscall.c static void bpf_link_free(struct bpf_link *link) · line 3010

Quoted 5 lines
kernel/bpf/syscall.c +4 -1
-static inline void release_probes(struct tracepoint_func *old)
+static bool tracepoint_is_syscall(struct tracepoint *tp)
+{
+       return !strcmp(tp->name, "sys_enter") || !strcmp(tp->name, "sys_exit");
+}

I'm curious if it might be better to add some field to struct tracepoint like "sleepable" rather than adding a special case here based on the name? Of course, if it's only ever going to be these two cases then maybe adding a new field doesn't make sense.

-Jordan

author reviewer
#2 lore raw
Quoted 10 lines · Jordan Rife

kernel/bpf/syscall.c static void bpf_link_free(struct bpf_link *link) · line 3006

Quoted 6 lines
kernel/bpf/syscall.c +0 -0
index 59de664e580d..1191dc1d4206 100644
@@ -3006,14 +3006,21 @@ static void bpf_link_free(struct bpf_link *link)
                 bpf_prog_put(link->prog);

I think we would need the same treatment with bpf_prog_put here. Something like,

Quoted 8 lines

tracepoint_call_rcu(raw_tp->btp->tp, &link->prog->aux->rcu,

		    bpf_link_defer_bpf_prog_put);

static void bpf_link_defer_bpf_prog_put(struct rcu_head *rcu)

{
	struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu);
	bpf_prog_put(aux->prox);
}

Sure, I'll add this in a v2.

Quoted 4 lines

Alternatively, some context would need to be passed down to __bpf_prog_put_noref via the call to bpf_prog_put so it can choose whether or not to use call_rcu or call_rcu_tasks_trace.

Also possible, but more cumbersome.

Quoted 11 lines

kernel/bpf/syscall.c static void bpf_link_free(struct bpf_link *link) · line 3010

Quoted 6 lines
kernel/bpf/syscall.c +4 -1
-static inline void release_probes(struct tracepoint_func *old)
+static bool tracepoint_is_syscall(struct tracepoint *tp)
+{
+       return !strcmp(tp->name, "sys_enter") || !strcmp(tp->name, "sys_exit");
+}

I'm curious if it might be better to add some field to struct tracepoint like "sleepable" rather than adding a special case here based on the name? Of course, if it's only ever going to be these two cases then maybe adding a new field doesn't make sense.

I know Steven is reluctant to bloat the tracepoint struct because there are lots of tracepoint instances (thousands). So for now I thought that just comparing the name would be a good start.

We can eventually go a different route as well: introduce a section just to put the syscall tracepoints, and compare the struct tracepoint pointers to the section begin/end range. But it's rather complex for what should remain a simple fix.

Thanks,

Mathieu

Quoted 2 lines

-Jordan

Signature · 3 lines
Mathieu Desnoyers
EfficiOS Inc.
https://www.efficios.com
maintainer
#3 lore raw
Quoted 8 lines · On Fri, 25 Oct 2024 15:38:48 -0400 Mathieu Desnoyers…
Quoted 4 lines

I'm curious if it might be better to add some field to struct tracepoint like "sleepable" rather than adding a special case here based on the name? Of course, if it's only ever going to be these two cases then maybe adding a new field doesn't make sense.

I know Steven is reluctant to bloat the tracepoint struct because there are lots of tracepoint instances (thousands). So for now I thought that just comparing the name would be a good start.

You are correct. I really trying to keep the footprint of tracepoints/events down.

Quoted 5 lines

We can eventually go a different route as well: introduce a section just to put the syscall tracepoints, and compare the struct tracepoint pointers to the section begin/end range. But it's rather complex for what should remain a simple fix.

A separate section could work.

-- Steve

author reviewer
#4 lore raw
Quoted 16 lines · Steven Rostedt
Quoted 8 lines · On Fri, 25 Oct 2024 15:38:48 -0400 Mathieu Desnoyers…
Quoted 4 lines

I'm curious if it might be better to add some field to struct tracepoint like "sleepable" rather than adding a special case here based on the name? Of course, if it's only ever going to be these two cases then maybe adding a new field doesn't make sense.

I know Steven is reluctant to bloat the tracepoint struct because there are lots of tracepoint instances (thousands). So for now I thought that just comparing the name would be a good start.

You are correct. I really trying to keep the footprint of tracepoints/events down.

Quoted 1 line
Quoted 6 lines
Quoted 4 lines

We can eventually go a different route as well: introduce a section just to put the syscall tracepoints, and compare the struct tracepoint pointers to the section begin/end range. But it's rather complex for what should remain a simple fix.

A separate section could work.

I have another approach to suggest: it shrinks the size of struct tracepoint from 80 bytes down to 72 bytes on x86-64, we don't have to do any section/linker script trickery, and it's extensible for future flags:

struct static_key {
        int enabled;
        void *p;
};
struct static_key_false {
        struct static_key key;
};
struct static_call_key {
        void *func;
        void *p;
};
struct tracepoint {
        const char *name;               /* Tracepoint name */
        struct static_key_false key;
        struct static_call_key *static_call_key;
        void *static_call_tramp;
        void *iterator;
        void *probestub;
        void *funcs;
        /* Flags. */
        unsigned int regfunc:1,
                     syscall:1;
};
struct tracepoint_regfunc {
        struct tracepoint tp;
        int (*regfunc)(void);
        void (*unregfunc)(void);
};

Basically, a tracepoint with regfunc would define a struct tracepoint_regfunc rather than a struct tracepoint. So we remove both regfunc and unregfunc NULL pointers in the common case, which gives us plenty of room for flags.

When we want to access the regfunc/unregfunc from a struct tracepoint, we check the regfunc flag, and if set, we can use container_of() to get the struct tracepoint_regfunc.

Thoughts ?

Thanks,

Mathieu

Signature · 3 lines
Mathieu Desnoyers
EfficiOS Inc.
https://www.efficios.com
author reviewer
#5 lore raw
Quoted 67 lines · Mathieu Desnoyers
Quoted 16 lines · Steven Rostedt
Quoted 8 lines · On Fri, 25 Oct 2024 15:38:48 -0400 Mathieu Desnoyers…
Quoted 4 lines

I'm curious if it might be better to add some field to struct tracepoint like "sleepable" rather than adding a special case here based on the name? Of course, if it's only ever going to be these two cases then maybe adding a new field doesn't make sense.

I know Steven is reluctant to bloat the tracepoint struct because there are lots of tracepoint instances (thousands). So for now I thought that just comparing the name would be a good start.

You are correct. I really trying to keep the footprint of tracepoints/events down.

Quoted 1 line
Quoted 6 lines
Quoted 4 lines

We can eventually go a different route as well: introduce a section just to put the syscall tracepoints, and compare the struct tracepoint pointers to the section begin/end range. But it's rather complex for what should remain a simple fix.

A separate section could work.

I have another approach to suggest: it shrinks the size of struct tracepoint from 80 bytes down to 72 bytes on x86-64, we don't have to do any section/linker script trickery, and it's extensible for future flags:

struct static_key {
        int enabled;
        void *p;
};

struct static_key_false {
        struct static_key key;
};

struct static_call_key {
        void *func;
        void *p;
};

struct tracepoint {
        const char *name;               /* Tracepoint name */ struct static_key_false key;
        struct static_call_key *static_call_key; void *static_call_tramp;
        void *iterator;
        void *probestub;
        void *funcs;
        /* Flags. */
        unsigned int regfunc:1,
                     syscall:1;
};

struct tracepoint_regfunc {
        struct tracepoint tp;
        int (*regfunc)(void);
        void (*unregfunc)(void);
};

Basically, a tracepoint with regfunc would define a struct tracepoint_regfunc rather than a struct tracepoint. So we remove both regfunc and unregfunc NULL pointers in the common case, which gives us plenty of room for flags.

Quoted 4 lines

When we want to access the regfunc/unregfunc from a struct tracepoint, we check the regfunc flag, and if set, we can use container_of() to get the struct tracepoint_regfunc.

Actually I can achieve the same space saving with fewer changes like this:

struct tracepoint_ext {
    void *regfunc;
    void *unregfunc;
    /* Flags. */
    unsigned int syscall:1;
}
struct tracepoint {
        const char *name;               /* Tracepoint name */
        struct static_key_false key;
        struct static_call_key *static_call_key;
        void *static_call_tramp;
        void *iterator;
        void *probestub;
        void *funcs;
        struct tracepoint_ext *ext;
};

Thanks,

Mathieu

Quoted 7 lines

Thoughts ?

Thanks,

Mathieu

Signature · 3 lines
Mathieu Desnoyers
EfficiOS Inc.
https://www.efficios.com