Skip to contentSkip to conversation

bpf: Disable preemption in stack map code

PATCH In review Jiri Olsa 5 participants 18 messages Aug 5

Subsystems (3)
lore
Share & tools
lore

Jiri Olsa

author reviewer
#0 lore raw

v4 cover bpf: Disable preemption in stack map code

hi,
we need to disable preemption for get_perf_callchain and keep it disabled as long as we are accessing its returned trace entries buffer.

This patchset refactors both bpf_get_stack and bpf_get_stackid helpers as suggested by Andrii [1] before applying the actual preemption fix.

Note the initial fix was sent by Tao Chen [2], but there was no follow up on this since February, hence this post.

thanks,
jirka

v1: https://lore.kernel.org/bpf/[email protected]/
v2: https://lore.kernel.org/bpf/[email protected]/
v3: https://lore.kernel.org/bpf/[email protected]/

v4 changes:
- fixed changelogs [sashiko]
- nr_trace variable rename [sashiko]
- fixed Daniel's email

v3 changes:
- pass flags rgument to callchain_finalize [sashiko]
- renamed err label to out in bpf_get_stackid_pe [sashiko]
- used scoped_guard in bpf_get_stackid [Leon]
- replaced trace->nr modification from *_pe helpers with length argument (new patches 11,12) [Andrii]

v2 changes:
- removed several unused functions arguments [sashiko]
- restore trace-nr value fix in bpf_get_stack_pe [sashiko]
- kept rcu locking together with preemption disable in __bpf_get_stack [sashiko]
- clear buf on error paths in __bpf_get_task_stack [sashiko]

[1] https://lore.kernel.org/bpf/CAEf4BzZwvAUgLwz-M0Y_NJLTmedyY9U6s7LrSmn751hQdTP4Uw@mail.gmail.com/
[2] https://lore.kernel.org/bpf/[email protected]/
---
Daniel Borkmann (1):
bpf: Disable preemption in __bpf_get_stack

Jiri Olsa (11):
bpf: Factor stackid_init function from __bpf_get_stackid
bpf: Factor stackid_fastpath function from __bpf_get_stackid
bpf: Factor stackid_new_bucket from __bpf_get_stackid
bpf: Use stack id functions instead of __bpf_get_stackid
bpf: Disable preemption in bpf_get_stackid
bpf: Factor callchain_store function from __bpf_get_stack
bpf: Factor callchain_finalize function from __bpf_get_stack
bpf: Remove trace_in argument from __bpf_get_stack
bpf: Clear buf on error in __bpf_get_task_stack
bpf: Avoid changing callchain in bpf_get_stack_pe
bpf: Avoid changing callchain in bpf_get_stackid_pe

 kernel/bpf/stackmap.c | 314 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------------------------------------
 1 file changed, 205 insertions(+), 109 deletions(-)
author reviewer
#1 lore raw

PATCH v4 1/12 bpf: Factor stackid_init function from __bpf_get_stackid

The new stackid_init function stores all the necessary bits for stackid trace and it will be used by other functions in following changes.

---
 kernel/bpf/stackmap.c | 95 +++++++++++++++++++++++++++----------------
 1 file changed, 59 insertions(+), 36 deletions(-)
kernel/bpf/stackmap.c +59 -36
index 463f94ba1cc4..19f9ea605a3e 100644
@@ -504,33 +504,54 @@ get_callchain_entry_for_task(struct task_struct *task, u32 max_depth)
 #endif
 }
 
-static long __bpf_get_stackid(struct bpf_map *map,
-			      struct perf_callchain_entry *trace, u64 flags)
+struct stackid {
+	struct stack_map_bucket *bucket;
+	u64 *ips;
+	u32  nr;
+	u32  len;
+	u32  hash;
+	u32  id;
+};
+
+static int stackid_init(struct stackid *stackid, struct bpf_map *map,
+			struct perf_callchain_entry *trace, u64 flags)
 {
 	struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
-	struct stack_map_bucket *bucket, *new_bucket, *old_bucket;
-	u32 hash, id, trace_nr, trace_len, i, max_depth;
 	u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
-	bool user = flags & BPF_F_USER_STACK;
-	u64 *ips;
-	bool hash_matches;
+	u32 max_depth;
 
 	if (trace->nr <= skip)
 		/* skipping more than usable stack trace */
 		return -EFAULT;
 
 	max_depth = stack_map_calculate_max_depth(map->value_size, stack_map_data_size(map), flags);
-	trace_nr = min_t(u32, trace->nr - skip, max_depth - skip);
-	trace_len = trace_nr * sizeof(u64);
-	ips = trace->ip + skip;
-	hash = jhash2((u32 *)ips, trace_len / sizeof(u32), 0);
-	id = hash & (smap->n_buckets - 1);
-	bucket = READ_ONCE(smap->buckets[id]);
+	stackid->nr = min_t(u32, trace->nr - skip, max_depth - skip);
+	stackid->len = stackid->nr * sizeof(u64);
+	stackid->ips = trace->ip + skip;
+	stackid->hash = jhash2((u32 *)stackid->ips, stackid->len / sizeof(u32), 0);
+	stackid->id = stackid->hash & (smap->n_buckets - 1);
+	stackid->bucket = READ_ONCE(smap->buckets[stackid->id]);
+	return 0;
+}
 
-	hash_matches = bucket && bucket->hash == hash;
+static long __bpf_get_stackid(struct stackid *stackid, struct bpf_map *map,
+			      struct perf_callchain_entry *trace, u64 flags)
+{
+	struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
+	struct stack_map_bucket *new_bucket, *old_bucket;
+	bool user = flags & BPF_F_USER_STACK;
+	bool hash_matches;
+	u32 trace_len, i;
+	int err;
+
+	err = stackid_init(stackid, map, trace, flags);
+	if (err)
+		return err;
+
+	hash_matches = stackid->bucket && stackid->bucket->hash == stackid->hash;
 	/* fast cmp */
 	if (hash_matches && flags & BPF_F_FAST_STACK_CMP)
-		return id;
+		return stackid->id;
 
 	if (stack_map_use_build_id(map)) {
 		struct bpf_stack_build_id *id_offs;
@@ -540,42 +561,42 @@ static long __bpf_get_stackid(struct bpf_map *map,
 			pcpu_freelist_pop(&smap->freelist);
 		if (unlikely(!new_bucket))
 			return -ENOMEM;
-		new_bucket->nr = trace_nr;
+		new_bucket->nr = stackid->nr;
 		id_offs = (struct bpf_stack_build_id *)new_bucket->data;
-		for (i = 0; i < trace_nr; i++)
-			id_offs[i].ip = ips[i];
-		stack_map_get_build_id_offset(id_offs, trace_nr, user, false /* !may_fault */);
-		trace_len = trace_nr * sizeof(struct bpf_stack_build_id);
-		if (hash_matches && bucket->nr == trace_nr &&
-		    memcmp(bucket->data, new_bucket->data, trace_len) == 0) {
+		for (i = 0; i < stackid->nr; i++)
+			id_offs[i].ip = stackid->ips[i];
+		stack_map_get_build_id_offset(id_offs, stackid->nr, user, false /* !may_fault */);
+		trace_len = stackid->nr * sizeof(struct bpf_stack_build_id);
+		if (hash_matches && stackid->bucket->nr == stackid->nr &&
+		    memcmp(stackid->bucket->data, new_bucket->data, trace_len) == 0) {
 			pcpu_freelist_push(&smap->freelist, &new_bucket->fnode);
-			return id;
+			return stackid->id;
 		}
-		if (bucket && !(flags & BPF_F_REUSE_STACKID)) {
+		if (stackid->bucket && !(flags & BPF_F_REUSE_STACKID)) {
 			pcpu_freelist_push(&smap->freelist, &new_bucket->fnode);
 			return -EEXIST;
 		}
 	} else {
-		if (hash_matches && bucket->nr == trace_nr &&
-		    memcmp(bucket->data, ips, trace_len) == 0)
-			return id;
-		if (bucket && !(flags & BPF_F_REUSE_STACKID))
+		if (hash_matches && stackid->bucket->nr == stackid->nr &&
+		    memcmp(stackid->bucket->data, stackid->ips, stackid->len) == 0)
+			return stackid->id;
+		if (stackid->bucket && !(flags & BPF_F_REUSE_STACKID))
 			return -EEXIST;
 
 		new_bucket = (struct stack_map_bucket *)
 			pcpu_freelist_pop(&smap->freelist);
 		if (unlikely(!new_bucket))
 			return -ENOMEM;
-		memcpy(new_bucket->data, ips, trace_len);
+		memcpy(new_bucket->data, stackid->ips, stackid->len);
 	}
 
-	new_bucket->hash = hash;
-	new_bucket->nr = trace_nr;
+	new_bucket->hash = stackid->hash;
+	new_bucket->nr = stackid->nr;
 
-	old_bucket = xchg(&smap->buckets[id], new_bucket);
+	old_bucket = xchg(&smap->buckets[stackid->id], new_bucket);
 	if (old_bucket)
 		pcpu_freelist_push(&smap->freelist, &old_bucket->fnode);
-	return id;
+	return stackid->id;
 }
 
 BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
@@ -584,6 +605,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
 	u32 elem_size = stack_map_data_size(map);
 	bool user = flags & BPF_F_USER_STACK;
 	struct perf_callchain_entry *trace;
+	struct stackid stackid;
 	bool kernel = !user;
 	u32 max_depth;
 
@@ -599,7 +621,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
 		/* couldn't fetch the stack trace */
 		return -EFAULT;
 
-	return __bpf_get_stackid(map, trace, flags);
+	return __bpf_get_stackid(&stackid, map, trace, flags);
 }
 
 const struct bpf_func_proto bpf_get_stackid_proto = {
@@ -628,6 +650,7 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx,
 {
 	struct perf_event *event = ctx->event;
 	struct perf_callchain_entry *trace;
+	struct stackid stackid;
 	bool kernel, user;
 	__u64 nr_kernel;
 	int ret;
@@ -653,7 +676,7 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx,
 
 	if (kernel) {
 		trace->nr = nr_kernel;
-		ret = __bpf_get_stackid(map, trace, flags);
+		ret = __bpf_get_stackid(&stackid, map, trace, flags);
 	} else { /* user */
 		u64 skip = flags & BPF_F_SKIP_FIELD_MASK;
 
@@ -662,7 +685,7 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx,
 			return -EFAULT;
 
 		flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip;
-		ret = __bpf_get_stackid(map, trace, flags);
+		ret = __bpf_get_stackid(&stackid, map, trace, flags);
 	}
 
 	/* restore nr */
Signature · 1 line
2.54.0
#2 lore raw

Re: [PATCHv4 bpf-next 01/12] bpf: Factor stackid_init function from __bpf_get_stackid

Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:

Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:

Pre-existing issues:
- [High] Integer underflow in stack_map_calculate_max_depth causes a heap buffer overflow when sysctl_perf_event_max_stack is concurrently modified. --

--- Patch [1]: [PATCHv4 bpf-next 01/12] bpf: Factor stackid_init function from __bpf_get_stackid --- commit 036b334d8e7b51a2339ee19596a116e90ed56c01
Author: Jiri Olsa <[email protected]>

bpf: Factor stackid_init function from __bpf_get_stackid

The new stackid_init function stores all the necessary bits for stackid trace and it will be used by other functions in following changes.

Quoted 3 lines
kernel/bpf/stackmap.c +0 -0

[ ... ]

kernel/bpf/stackmap.c get_callchain_entry_for_task(struct task_struct *task, u32 max_depth) · line 528

Quoted 13 lines
kernel/bpf/stackmap.c +4 -0
+static int stackid_init(struct stackid *stackid, struct bpf_map *map,
+			struct perf_callchain_entry *trace, u64 flags)
 {
 	struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
 	u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
+	u32 max_depth;
 
 	if (trace->nr <= skip)
 		/* skipping more than usable stack trace */
 		return -EFAULT;
 
 	max_depth = stack_map_calculate_max_depth(map->value_size, stack_map_data_size(map), flags);
+	stackid->nr = min_t(u32, trace->nr - skip, max_depth - skip);

[Severity: High]
This is a pre-existing issue, but can this max_depth - skip calculation underflow?

If sysctl_perf_event_max_stack is decreased concurrently, stack_map_calculate_max_depth() can return a max_depth smaller than skip:

kernel/bpf/stackmap.c:stack_map_calculate_max_depth() {
	...
	max_depth += skip;
	if (max_depth > curr_sysctl_max_stack)
		return curr_sysctl_max_stack;
	...
}

This would cause max_depth - skip to underflow to a large integer. If trace->nr (which may have been captured with the older, larger sysctl) is greater than skip, it passes the early return check.

The min_t() call would then select trace->nr - skip, which can exceed the stack map's element capacity and lead to a heap buffer overflow later when copying entries into the bucket buffer in __bpf_get_stackid():

kernel/bpf/stackmap.c:__bpf_get_stackid() {
	...
		memcpy(new_bucket->data, stackid->ips, stackid->len);
	...
}
Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=1
author reviewer
#3 lore raw

PATCH v4 2/12 bpf: Factor stackid_fastpath function from __bpf_get_stackid

In reply to Jiri Olsa

The new stackid_fastpath does the fast stack hash and trace check, that does not need new bucket allocation. It covers both just-ip and buildid code paths. The -ENOENT error is the fall through to the slow path.

---
 kernel/bpf/stackmap.c | 43 ++++++++++++++++++++++++++++---------------
 1 file changed, 28 insertions(+), 15 deletions(-)
kernel/bpf/stackmap.c +28 -15
index 19f9ea605a3e..27210b5d16fc 100644
@@ -511,6 +511,7 @@ struct stackid {
 	u32  len;
 	u32  hash;
 	u32  id;
+	bool hash_matches;
 };
 
 static int stackid_init(struct stackid *stackid, struct bpf_map *map,
@@ -531,28 +532,46 @@ static int stackid_init(struct stackid *stackid, struct bpf_map *map,
 	stackid->hash = jhash2((u32 *)stackid->ips, stackid->len / sizeof(u32), 0);
 	stackid->id = stackid->hash & (smap->n_buckets - 1);
 	stackid->bucket = READ_ONCE(smap->buckets[stackid->id]);
+	stackid->hash_matches = stackid->bucket && stackid->bucket->hash == stackid->hash;
 	return 0;
 }
 
+static int stackid_fastpath(struct stackid *stackid, struct bpf_map *map,
+			    struct perf_callchain_entry *trace, u64 flags)
+{
+	int err;
+
+	err = stackid_init(stackid, map, trace, flags);
+	if (err)
+		return err;
+
+	/* fast cmp */
+	if (stackid->hash_matches && flags & BPF_F_FAST_STACK_CMP)
+		return stackid->id;
+
+	if (stack_map_use_build_id(map))
+		return -ENOENT;
+	if (stackid->hash_matches && stackid->bucket->nr == stackid->nr &&
+	    memcmp(stackid->bucket->data, stackid->ips, stackid->len) == 0)
+		return stackid->id;
+	if (stackid->bucket && !(flags & BPF_F_REUSE_STACKID))
+		return -EEXIST;
+	return -ENOENT;
+}
+
 static long __bpf_get_stackid(struct stackid *stackid, struct bpf_map *map,
 			      struct perf_callchain_entry *trace, u64 flags)
 {
 	struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
 	struct stack_map_bucket *new_bucket, *old_bucket;
 	bool user = flags & BPF_F_USER_STACK;
-	bool hash_matches;
 	u32 trace_len, i;
 	int err;
 
-	err = stackid_init(stackid, map, trace, flags);
-	if (err)
+	err = stackid_fastpath(stackid, map, trace, flags);
+	if (err != -ENOENT)
 		return err;
 
-	hash_matches = stackid->bucket && stackid->bucket->hash == stackid->hash;
-	/* fast cmp */
-	if (hash_matches && flags & BPF_F_FAST_STACK_CMP)
-		return stackid->id;
-
 	if (stack_map_use_build_id(map)) {
 		struct bpf_stack_build_id *id_offs;
 
@@ -567,7 +586,7 @@ static long __bpf_get_stackid(struct stackid *stackid, struct bpf_map *map,
 			id_offs[i].ip = stackid->ips[i];
 		stack_map_get_build_id_offset(id_offs, stackid->nr, user, false /* !may_fault */);
 		trace_len = stackid->nr * sizeof(struct bpf_stack_build_id);
-		if (hash_matches && stackid->bucket->nr == stackid->nr &&
+		if (stackid->hash_matches && stackid->bucket->nr == stackid->nr &&
 		    memcmp(stackid->bucket->data, new_bucket->data, trace_len) == 0) {
 			pcpu_freelist_push(&smap->freelist, &new_bucket->fnode);
 			return stackid->id;
@@ -577,12 +596,6 @@ static long __bpf_get_stackid(struct stackid *stackid, struct bpf_map *map,
 			return -EEXIST;
 		}
 	} else {
-		if (hash_matches && stackid->bucket->nr == stackid->nr &&
-		    memcmp(stackid->bucket->data, stackid->ips, stackid->len) == 0)
-			return stackid->id;
-		if (stackid->bucket && !(flags & BPF_F_REUSE_STACKID))
-			return -EEXIST;
-
 		new_bucket = (struct stack_map_bucket *)
 			pcpu_freelist_pop(&smap->freelist);
 		if (unlikely(!new_bucket))
Signature · 1 line
2.54.0
author reviewer
#4 lore raw

PATCH v4 3/12 bpf: Factor stackid_new_bucket from __bpf_get_stackid

In reply to Jiri Olsa

The new stackid_new_bucket allocates the new bucket and initializes it with the trace data.

---
 kernel/bpf/stackmap.c | 48 +++++++++++++++++++++++++++----------------
 1 file changed, 30 insertions(+), 18 deletions(-)
kernel/bpf/stackmap.c +30 -18
index 27210b5d16fc..d930d9754d7d 100644
@@ -559,31 +559,52 @@ static int stackid_fastpath(struct stackid *stackid, struct bpf_map *map,
 	return -ENOENT;
 }
 
+static struct stack_map_bucket *
+stackid_new_bucket(struct stackid *stackid, struct bpf_map *map)
+{
+	struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
+	struct bpf_stack_build_id *id_offs;
+	struct stack_map_bucket *bucket;
+	u32 i;
+
+	bucket = (struct stack_map_bucket *) pcpu_freelist_pop(&smap->freelist);
+	if (unlikely(!bucket))
+		return NULL;
+
+	if (stack_map_use_build_id(map)) {
+		id_offs = (struct bpf_stack_build_id *)bucket->data;
+		for (i = 0; i < stackid->nr; i++)
+			id_offs[i].ip = stackid->ips[i];
+	} else {
+		memcpy(bucket->data, stackid->ips, stackid->len);
+	}
+
+	bucket->hash = stackid->hash;
+	bucket->nr = stackid->nr;
+	return bucket;
+}
+
 static long __bpf_get_stackid(struct stackid *stackid, struct bpf_map *map,
 			      struct perf_callchain_entry *trace, u64 flags)
 {
 	struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
 	struct stack_map_bucket *new_bucket, *old_bucket;
 	bool user = flags & BPF_F_USER_STACK;
-	u32 trace_len, i;
+	u32 trace_len;
 	int err;
 
 	err = stackid_fastpath(stackid, map, trace, flags);
 	if (err != -ENOENT)
 		return err;
 
+	new_bucket = stackid_new_bucket(stackid, map);
+	if (!new_bucket)
+		return -ENOMEM;
+
 	if (stack_map_use_build_id(map)) {
 		struct bpf_stack_build_id *id_offs;
 
-		/* for build_id+offset, pop a bucket before slow cmp */
-		new_bucket = (struct stack_map_bucket *)
-			pcpu_freelist_pop(&smap->freelist);
-		if (unlikely(!new_bucket))
-			return -ENOMEM;
-		new_bucket->nr = stackid->nr;
 		id_offs = (struct bpf_stack_build_id *)new_bucket->data;
-		for (i = 0; i < stackid->nr; i++)
-			id_offs[i].ip = stackid->ips[i];
 		stack_map_get_build_id_offset(id_offs, stackid->nr, user, false /* !may_fault */);
 		trace_len = stackid->nr * sizeof(struct bpf_stack_build_id);
 		if (stackid->hash_matches && stackid->bucket->nr == stackid->nr &&
@@ -595,17 +616,8 @@ static long __bpf_get_stackid(struct stackid *stackid, struct bpf_map *map,
 			pcpu_freelist_push(&smap->freelist, &new_bucket->fnode);
 			return -EEXIST;
 		}
-	} else {
-		new_bucket = (struct stack_map_bucket *)
-			pcpu_freelist_pop(&smap->freelist);
-		if (unlikely(!new_bucket))
-			return -ENOMEM;
-		memcpy(new_bucket->data, stackid->ips, stackid->len);
 	}
 
-	new_bucket->hash = stackid->hash;
-	new_bucket->nr = stackid->nr;
-
 	old_bucket = xchg(&smap->buckets[stackid->id], new_bucket);
 	if (old_bucket)
 		pcpu_freelist_push(&smap->freelist, &old_bucket->fnode);
Signature · 1 line
2.54.0
author reviewer
#5 lore raw

PATCH v4 4/12 bpf: Use stack id functions instead of __bpf_get_stackid

In reply to Jiri Olsa

Replacing __bpf_get_stackid calls with sequence of following functions:

stackid_fastpath
stackid_new_bucket
stackid_install

This makes code more structured and allows us to easily disable preemption only in bpf_get_stackid in following changes.

---
 kernel/bpf/stackmap.c | 43 +++++++++++++++++++++++++++----------------
 1 file changed, 27 insertions(+), 16 deletions(-)
kernel/bpf/stackmap.c +27 -16
index d930d9754d7d..3ee0034daf52 100644
@@ -584,22 +584,13 @@ stackid_new_bucket(struct stackid *stackid, struct bpf_map *map)
 	return bucket;
 }
 
-static long __bpf_get_stackid(struct stackid *stackid, struct bpf_map *map,
-			      struct perf_callchain_entry *trace, u64 flags)
+static long stackid_install(struct stackid *stackid, struct bpf_map *map,
+			    struct stack_map_bucket *new_bucket, u64 flags)
 {
 	struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
-	struct stack_map_bucket *new_bucket, *old_bucket;
 	bool user = flags & BPF_F_USER_STACK;
+	struct stack_map_bucket *old_bucket;
 	u32 trace_len;
-	int err;
-
-	err = stackid_fastpath(stackid, map, trace, flags);
-	if (err != -ENOENT)
-		return err;
-
-	new_bucket = stackid_new_bucket(stackid, map);
-	if (!new_bucket)
-		return -ENOMEM;
 
 	if (stack_map_use_build_id(map)) {
 		struct bpf_stack_build_id *id_offs;
@@ -629,10 +620,12 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
 {
 	u32 elem_size = stack_map_data_size(map);
 	bool user = flags & BPF_F_USER_STACK;
+	struct stack_map_bucket *new_bucket;
 	struct perf_callchain_entry *trace;
 	struct stackid stackid;
 	bool kernel = !user;
 	u32 max_depth;
+	int err;
 
 	if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
 			       BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID)))
@@ -646,7 +639,15 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
 		/* couldn't fetch the stack trace */
 		return -EFAULT;
 
-	return __bpf_get_stackid(&stackid, map, trace, flags);
+	err = stackid_fastpath(&stackid, map, trace, flags);
+	if (err != -ENOENT)
+		return err;
+
+	new_bucket = stackid_new_bucket(&stackid, map);
+	if (!new_bucket)
+		return -ENOMEM;
+
+	return stackid_install(&stackid, map, new_bucket, flags);
 }
 
 const struct bpf_func_proto bpf_get_stackid_proto = {
@@ -674,6 +675,7 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx,
 	   struct bpf_map *, map, u64, flags)
 {
 	struct perf_event *event = ctx->event;
+	struct stack_map_bucket *new_bucket;
 	struct perf_callchain_entry *trace;
 	struct stackid stackid;
 	bool kernel, user;
@@ -701,7 +703,6 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx,
 
 	if (kernel) {
 		trace->nr = nr_kernel;
-		ret = __bpf_get_stackid(&stackid, map, trace, flags);
 	} else { /* user */
 		u64 skip = flags & BPF_F_SKIP_FIELD_MASK;
 
@@ -710,12 +711,22 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx,
 			return -EFAULT;
 
 		flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip;
-		ret = __bpf_get_stackid(&stackid, map, trace, flags);
 	}
 
+	ret = stackid_fastpath(&stackid, map, trace, flags);
+	if (ret != -ENOENT)
+		goto out;
+
+	new_bucket = stackid_new_bucket(&stackid, map);
+	if (new_bucket) {
+		trace->nr = nr;
+		return stackid_install(&stackid, map, new_bucket, flags);
+	}
+	ret = -ENOMEM;
+
+out:
 	/* restore nr */
 	trace->nr = nr;
-
 	return ret;
 }
 
Signature · 1 line
2.54.0
author reviewer
#6 lore raw

PATCH v4 5/12 bpf: Disable preemption in bpf_get_stackid

In reply to Jiri Olsa

The get_perf_callchain call needs disabled preemption plus we need it disabled as long as we access its returned trace entries buffer.

Note the bpf_get_stackid_pe function is executed already with preemption disabled.

Fixes d5a3b1f69186 ("bpf: introduce BPF_MAP_TYPE_STACK_TRACE")
---
 kernel/bpf/stackmap.c | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)
kernel/bpf/stackmap.c +13 -11
index 3ee0034daf52..5b18d728f4b8 100644
@@ -632,20 +632,22 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
 		return -EINVAL;
 
 	max_depth = stack_map_calculate_max_depth(map->value_size, elem_size, flags);
-	trace = get_perf_callchain(regs, kernel, user, max_depth,
-				   false, false, 0);
 
-	if (unlikely(!trace))
-		/* couldn't fetch the stack trace */
-		return -EFAULT;
+	scoped_guard(preempt) {
+		trace = get_perf_callchain(regs, kernel, user, max_depth,
+					   false, false, 0);
+		if (unlikely(!trace))
+			/* couldn't fetch the stack trace */
+			return -EFAULT;
 
-	err = stackid_fastpath(&stackid, map, trace, flags);
-	if (err != -ENOENT)
-		return err;
+		err = stackid_fastpath(&stackid, map, trace, flags);
+		if (err != -ENOENT)
+			return err;
 
-	new_bucket = stackid_new_bucket(&stackid, map);
-	if (!new_bucket)
-		return -ENOMEM;
+		new_bucket = stackid_new_bucket(&stackid, map);
+		if (!new_bucket)
+			return -ENOMEM;
+	}
 
 	return stackid_install(&stackid, map, new_bucket, flags);
 }
Signature · 1 line
2.54.0
author reviewer
#7 lore raw

PATCH v4 6/12 bpf: Factor callchain_store function from __bpf_get_stack

In reply to Jiri Olsa

The new callchain_store function stores trace entries buffer into user supplied buffer. It covers both just-ip and buildid data.

---
 kernel/bpf/stackmap.c | 39 +++++++++++++++++++++++++--------------
 1 file changed, 25 insertions(+), 14 deletions(-)
kernel/bpf/stackmap.c +25 -14
index 5b18d728f4b8..ee9905d67b8e 100644
@@ -741,6 +741,29 @@ const struct bpf_func_proto bpf_get_stackid_proto_pe = {
 	.arg3_type	= ARG_ANYTHING,
 };
 
+static u32 callchain_store(struct perf_callchain_entry *trace, void *buf,
+			   u32 elem_size, u64 flags)
+{
+	bool user_build_id = flags & BPF_F_USER_BUILD_ID;
+	u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
+	u32 trace_nr, copy_len;
+	u64 *ips;
+
+	trace_nr = trace->nr - skip;
+	copy_len = trace_nr * elem_size;
+
+	ips = trace->ip + skip;
+	if (user_build_id) {
+		struct bpf_stack_build_id *id_offs = buf;
+
+		for (u32 i = 0; i < trace_nr; i++)
+			id_offs[i].ip = ips[i];
+	} else {
+		memcpy(buf, ips, copy_len);
+	}
+	return trace_nr;
+}
+
 static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
 			    struct perf_callchain_entry *trace_in,
 			    void *buf, u32 size, u64 flags, bool may_fault)
@@ -753,7 +776,6 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
 	struct perf_callchain_entry *trace;
 	bool kernel = !user;
 	int err = -EINVAL;
-	u64 *ips;
 
 	if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
 			       BPF_F_USER_BUILD_ID)))
@@ -798,21 +820,10 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
 		goto err_fault;
 	}
 
-	trace_nr = trace->nr - skip;
+	trace_nr = callchain_store(trace, buf, elem_size, flags);
 	copy_len = trace_nr * elem_size;
 
-	ips = trace->ip + skip;
-	if (user_build_id) {
-		struct bpf_stack_build_id *id_offs = buf;
-		u32 i;
-
-		for (i = 0; i < trace_nr; i++)
-			id_offs[i].ip = ips[i];
-	} else {
-		memcpy(buf, ips, copy_len);
-	}
-
-	/* trace/ips should not be dereferenced after this point */
+	/* trace should not be dereferenced after this point */
 	if (may_fault)
 		rcu_read_unlock();
 
Signature · 1 line
2.54.0
author reviewer
#8 lore raw

PATCH v4 7/12 bpf: Factor callchain_finalize function from __bpf_get_stack

In reply to Jiri Olsa

The new callchain_finalize function calls the build-id retrieval (if needed) and zeroes the buffer. This makes things easier for preemption fix in following change.

---
 kernel/bpf/stackmap.c | 25 +++++++++++++++++--------
 1 file changed, 17 insertions(+), 8 deletions(-)
kernel/bpf/stackmap.c +17 -8
index ee9905d67b8e..cdeb6c2e50da 100644
@@ -764,16 +764,31 @@ static u32 callchain_store(struct perf_callchain_entry *trace, void *buf,
 	return trace_nr;
 }
 
+static long callchain_finalize(void *buf, u32 size, u32 trace_nr, u32 elem_size,
+			       u64 flags, bool may_fault)
+{
+	bool user_build_id = flags & BPF_F_USER_BUILD_ID;
+	bool user = flags & BPF_F_USER_STACK;
+	u32 copy_len = trace_nr * elem_size;
+
+	if (user_build_id)
+		stack_map_get_build_id_offset(buf, trace_nr, user, may_fault);
+
+	if (size > copy_len)
+		memset(buf + copy_len, 0, size - copy_len);
+	return copy_len;
+}
+
 static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
 			    struct perf_callchain_entry *trace_in,
 			    void *buf, u32 size, u64 flags, bool may_fault)
 {
-	u32 trace_nr, copy_len, elem_size, max_depth;
 	bool user_build_id = flags & BPF_F_USER_BUILD_ID;
 	bool crosstask = task && task != current;
 	u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
 	bool user = flags & BPF_F_USER_STACK;
 	struct perf_callchain_entry *trace;
+	u32 trace_nr, elem_size, max_depth;
 	bool kernel = !user;
 	int err = -EINVAL;
 
@@ -821,18 +836,12 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
 	}
 
 	trace_nr = callchain_store(trace, buf, elem_size, flags);
-	copy_len = trace_nr * elem_size;
 
 	/* trace should not be dereferenced after this point */
 	if (may_fault)
 		rcu_read_unlock();
 
-	if (user_build_id)
-		stack_map_get_build_id_offset(buf, trace_nr, user, may_fault);
-
-	if (size > copy_len)
-		memset(buf + copy_len, 0, size - copy_len);
-	return copy_len;
+	return callchain_finalize(buf, size, trace_nr, elem_size, flags, may_fault);
 
 err_fault:
 	err = -EFAULT;
Signature · 1 line
2.54.0
author reviewer
#9 lore raw

PATCH v4 8/12 bpf: Remove trace_in argument from __bpf_get_stack

In reply to Jiri Olsa

Now with the new callchain_* helper functions we can process trace_in case directly in bpf_get_stack_pe function and remove it from __bpf_get_stack which makes things easier for preemption fix in following change.

---
 kernel/bpf/stackmap.c | 50 ++++++++++++++++++++++++++++++-------------
 1 file changed, 35 insertions(+), 15 deletions(-)
kernel/bpf/stackmap.c +35 -15
index cdeb6c2e50da..b0e38e5b3738 100644
@@ -780,7 +780,6 @@ static long callchain_finalize(void *buf, u32 size, u32 trace_nr, u32 elem_size,
 }
 
 static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
-			    struct perf_callchain_entry *trace_in,
 			    void *buf, u32 size, u64 flags, bool may_fault)
 {
 	bool user_build_id = flags & BPF_F_USER_BUILD_ID;
@@ -819,10 +818,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
 	if (may_fault)
 		rcu_read_lock(); /* need RCU for perf's callchain below */
 
-	if (trace_in) {
-		trace = trace_in;
-		trace->nr = min_t(u32, trace->nr, max_depth);
-	} else if (kernel && task) {
+	if (kernel && task) {
 		trace = get_callchain_entry_for_task(task, max_depth);
 	} else {
 		trace = get_perf_callchain(regs, kernel, user, max_depth,
@@ -853,7 +849,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
 BPF_CALL_4(bpf_get_stack, struct pt_regs *, regs, void *, buf, u32, size,
 	   u64, flags)
 {
-	return __bpf_get_stack(regs, NULL, NULL, buf, size, flags, false /* !may_fault */);
+	return __bpf_get_stack(regs, NULL, buf, size, flags, false /* !may_fault */);
 }
 
 const struct bpf_func_proto bpf_get_stack_proto = {
@@ -869,7 +865,7 @@ const struct bpf_func_proto bpf_get_stack_proto = {
 BPF_CALL_4(bpf_get_stack_sleepable, struct pt_regs *, regs, void *, buf, u32, size,
 	   u64, flags)
 {
-	return __bpf_get_stack(regs, NULL, NULL, buf, size, flags, true /* may_fault */);
+	return __bpf_get_stack(regs, NULL, buf, size, flags, true /* may_fault */);
 }
 
 const struct bpf_func_proto bpf_get_stack_sleepable_proto = {
@@ -893,7 +889,7 @@ static long __bpf_get_task_stack(struct task_struct *task, void *buf, u32 size,
 
 	regs = task_pt_regs(task);
 	if (regs)
-		res = __bpf_get_stack(regs, task, NULL, buf, size, flags, may_fault);
+		res = __bpf_get_stack(regs, task, buf, size, flags, may_fault);
 	put_task_stack(task);
 
 	return res;
@@ -933,6 +929,32 @@ const struct bpf_func_proto bpf_get_task_stack_sleepable_proto = {
 	.arg4_type	= ARG_ANYTHING,
 };
 
+static int __bpf_get_stack_pe(struct perf_callchain_entry *trace, void *buf, u32 size,
+			      u64 flags)
+{
+	bool user_build_id = flags & BPF_F_USER_BUILD_ID;
+	u64 skip = flags & BPF_F_SKIP_FIELD_MASK;
+	bool user = flags & BPF_F_USER_STACK;
+	u32 elem_size, max_depth, trace_nr;
+	bool kernel = !user;
+
+	if (kernel && user_build_id)
+		return -EINVAL;
+
+	elem_size = user_build_id ? sizeof(struct bpf_stack_build_id) : sizeof(u64);
+	if (unlikely(size % elem_size))
+		return -EINVAL;
+
+	max_depth = stack_map_calculate_max_depth(size, elem_size, flags);
+	trace->nr = min_t(u32, trace->nr, max_depth);
+
+	if (trace->nr < skip)
+		return -EFAULT;
+
+	trace_nr = callchain_store(trace, buf, elem_size, flags);
+	return callchain_finalize(buf, size, trace_nr, elem_size, flags, false /* !may_fault */);
+}
+
 BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx,
 	   void *, buf, u32, size, u64, flags)
 {
@@ -944,7 +966,7 @@ BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx,
 	__u64 nr_kernel;
 
 	if (!(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN))
-		return __bpf_get_stack(regs, NULL, NULL, buf, size, flags, false /* !may_fault */);
+		return __bpf_get_stack(regs, NULL, buf, size, flags, false /* !may_fault */);
 
 	if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
 			       BPF_F_USER_BUILD_ID)))
@@ -964,7 +986,7 @@ BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx,
 		__u64 nr = trace->nr;
 
 		trace->nr = nr_kernel;
-		err = __bpf_get_stack(regs, NULL, trace, buf, size, flags, false /* !may_fault */);
+		err = __bpf_get_stack_pe(trace, buf, size, flags);
 
 		/* restore nr */
 		trace->nr = nr;
@@ -974,16 +996,14 @@ BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx,
 		skip += nr_kernel;
 		if (skip > BPF_F_SKIP_FIELD_MASK)
 			goto clear;
-
 		flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip;
-		err = __bpf_get_stack(regs, NULL, trace, buf, size, flags, false /* !may_fault */);
+		err = __bpf_get_stack_pe(trace, buf, size, flags);
 	}
-	return err;
 
 clear:
-	memset(buf, 0, size);
+	if (err < 0)
+		memset(buf, 0, size);
 	return err;
-
 }
 
 const struct bpf_func_proto bpf_get_stack_proto_pe = {
Signature · 1 line
2.54.0
author reviewer
#10 lore raw

PATCH v4 9/12 bpf: Clear buf on error in __bpf_get_task_stack

In reply to Jiri Olsa

Both bpf_get_task_stack and bpf_get_task_stack_sleepable helpers that use __bpf_get_task_stack have buf defined as ARG_PTR_TO_UNINIT_MEM argument and we should initialize the buf on every return path.

Adding missing buf memset for __bpf_get_task_stack fail paths. This provides deterministic buffer contents, which is useful when the buffer is used directly as a map key.

Fixes 06ab134ce8ec ("bpf: Refcount task stack in bpf_get_task_stack")
Fixes b992f01e6615 ("bpf: Guard against accessing NULL pt_regs in bpf_get_task_stack()")
---
 kernel/bpf/stackmap.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)
kernel/bpf/stackmap.c +5 -2
index b0e38e5b3738..eabeaef31b63 100644
@@ -884,14 +884,17 @@ static long __bpf_get_task_stack(struct task_struct *task, void *buf, u32 size,
 	struct pt_regs *regs;
 	long res = -EINVAL;
 
-	if (!try_get_task_stack(task))
+	if (!try_get_task_stack(task)) {
+		memset(buf, 0, size);
 		return -EFAULT;
+	}
 
 	regs = task_pt_regs(task);
 	if (regs)
 		res = __bpf_get_stack(regs, task, buf, size, flags, may_fault);
+	else
+		memset(buf, 0, size);
 	put_task_stack(task);
-
 	return res;
 }
 
Signature · 1 line
2.54.0
author reviewer
#11 lore raw

PATCH v4 10/12 bpf: Disable preemption in __bpf_get_stack

In reply to Jiri Olsa

From: Daniel Borkmann <[email protected]>

get_perf_callchain() returns a per-CPU perf_callchain_entry buffer and releases its recursion slot via put_callchain_entry() before returning, so nothing keeps the entry reserved while __bpf_get_stack() consumes it below.

A preemptible BPF program (e.g. a non-sleepable raw tracepoint program on a PREEMPT kernel, which runs under migrate_disable() but not preempt_disable()) can be scheduled out between obtaining the entry and the copy. Another task scheduled on the same CPU then reuses the same per-CPU buffer and overwrites trace->nr with a larger value. copy_len is then computed from the inflated trace->nr and can exceed the caller's buffer, causing an out-of-bounds write in the memcpy() and in the build_id path.

The rcu_read_lock() taken here alone does not prevent this. It is only taken on the may_fault path, and under CONFIG_PREEMPT_RCU it does not disable preemption; it merely keeps perf's callchain buffer array alive (freed via call_rcu()) and does nothing to stop another task from reusing the entry.

Disable preemption around obtaining the callchain entry and copying it into the caller's buffer, so the entry cannot be reused underneath us and trace->nr stays bounded by max_depth. Build ID resolution may fault and is therefore deferred until after preemption is re-enabled; by then the instruction pointers have already been copied into buf, so it operates only on that private copy. Note, preempt_disable() also subsumes the buffer-lifetime guarantee the rcu_read_lock() provided, since a preempt-disabled section is an RCU read-side critical section for the callchain buffers' call_rcu() reclaim.

Fixes c195651e565a ("bpf: add bpf_get_stack helper")
[ changed Fixes: commit ]
---
 kernel/bpf/stackmap.c | 3 +++
 1 file changed, 3 insertions(+)
kernel/bpf/stackmap.c +3 -0
index eabeaef31b63..789fe35b893a 100644
@@ -817,6 +817,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
 
 	if (may_fault)
 		rcu_read_lock(); /* need RCU for perf's callchain below */
+	preempt_disable();
 
 	if (kernel && task) {
 		trace = get_callchain_entry_for_task(task, max_depth);
@@ -828,6 +829,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
 	if (unlikely(!trace) || trace->nr < skip) {
 		if (may_fault)
 			rcu_read_unlock();
+		preempt_enable();
 		goto err_fault;
 	}
 
@@ -836,6 +838,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
 	/* trace should not be dereferenced after this point */
 	if (may_fault)
 		rcu_read_unlock();
+	preempt_enable();
 
 	return callchain_finalize(buf, size, trace_nr, elem_size, flags, may_fault);
 
Signature · 1 line
2.54.0
#12 lore raw

Re: [PATCHv4 bpf-next 10/12] bpf: Disable preemption in __bpf_get_stack

Quoted 50 lines · Jiri Olsa

From: Daniel Borkmann <[email protected]>

get_perf_callchain() returns a per-CPU perf_callchain_entry buffer and releases its recursion slot via put_callchain_entry() before returning, so nothing keeps the entry reserved while __bpf_get_stack() consumes it below.

A preemptible BPF program (e.g. a non-sleepable raw tracepoint program on a PREEMPT kernel, which runs under migrate_disable() but not preempt_disable()) can be scheduled out between obtaining the entry and the copy. Another task scheduled on the same CPU then reuses the same per-CPU buffer and overwrites trace->nr with a larger value. copy_len is then computed from the inflated trace->nr and can exceed the caller's buffer, causing an out-of-bounds write in the memcpy() and in the build_id path.

The rcu_read_lock() taken here alone does not prevent this. It is only taken on the may_fault path, and under CONFIG_PREEMPT_RCU it does not disable preemption; it merely keeps perf's callchain buffer array alive (freed via call_rcu()) and does nothing to stop another task from reusing the entry.

Disable preemption around obtaining the callchain entry and copying it into the caller's buffer, so the entry cannot be reused underneath us and trace->nr stays bounded by max_depth. Build ID resolution may fault and is therefore deferred until after preemption is re-enabled; by then the instruction pointers have already been copied into buf, so it operates only on that private copy. Note, preempt_disable() also subsumes the buffer-lifetime guarantee the rcu_read_lock() provided, since a preempt-disabled section is an RCU read-side critical section for the callchain buffers' call_rcu() reclaim.

Fixes c195651e565a ("bpf: add bpf_get_stack helper")
[ changed Fixes: commit ]
---
 kernel/bpf/stackmap.c | 3 +++
 1 file changed, 3 insertions(+)
kernel/bpf/stackmap.c +0 -0
index eabeaef31b63..789fe35b893a 100644

kernel/bpf/stackmap.c static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, · line 820

Quoted 4 lines
kernel/bpf/stackmap.c +1 -0
@@ -817,6 +817,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
 
        if (may_fault)
                rcu_read_lock(); /* need RCU for perf's callchain below */
+       preempt_disable();

nit: asymmetrical to preempt_enable, I'll move it to before rcu_read_lock, so we can have proper nesting

Quoted 22 lines
kernel/bpf/stackmap.c +2 -1
        if (kernel && task) {
                trace = get_callchain_entry_for_task(task, max_depth);
@@ -828,6 +829,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
        if (unlikely(!trace) || trace->nr < skip) {
                if (may_fault)
                        rcu_read_unlock();
+               preempt_enable();
                goto err_fault;
        }
 
@@ -836,6 +838,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
        /* trace should not be dereferenced after this point */
        if (may_fault)
                rcu_read_unlock();
+       preempt_enable();
 
        return callchain_finalize(buf, size, trace_nr, elem_size, flags, may_fault);
 
--

2.54.0

#13 lore raw

Re: [PATCHv4 bpf-next 10/12] bpf: Disable preemption in __bpf_get_stack

Quoted 52 lines · Andrii Nakryiko
Quoted 51 lines · Jiri Olsa

From: Daniel Borkmann <[email protected]>

get_perf_callchain() returns a per-CPU perf_callchain_entry buffer and releases its recursion slot via put_callchain_entry() before returning, so nothing keeps the entry reserved while __bpf_get_stack() consumes it below.

A preemptible BPF program (e.g. a non-sleepable raw tracepoint program on a PREEMPT kernel, which runs under migrate_disable() but not preempt_disable()) can be scheduled out between obtaining the entry and the copy. Another task scheduled on the same CPU then reuses the same per-CPU buffer and overwrites trace->nr with a larger value. copy_len is then computed from the inflated trace->nr and can exceed the caller's buffer, causing an out-of-bounds write in the memcpy() and in the build_id path.

The rcu_read_lock() taken here alone does not prevent this. It is only taken on the may_fault path, and under CONFIG_PREEMPT_RCU it does not disable preemption; it merely keeps perf's callchain buffer array alive (freed via call_rcu()) and does nothing to stop another task from reusing the entry.

Disable preemption around obtaining the callchain entry and copying it into the caller's buffer, so the entry cannot be reused underneath us and trace->nr stays bounded by max_depth. Build ID resolution may fault and is therefore deferred until after preemption is re-enabled; by then the instruction pointers have already been copied into buf, so it operates only on that private copy. Note, preempt_disable() also subsumes the buffer-lifetime guarantee the rcu_read_lock() provided, since a preempt-disabled section is an RCU read-side critical section for the callchain buffers' call_rcu() reclaim.

Fixes c195651e565a ("bpf: add bpf_get_stack helper")
[ changed Fixes: commit ]
---
 kernel/bpf/stackmap.c | 3 +++
 1 file changed, 3 insertions(+)
kernel/bpf/stackmap.c +0 -0
index eabeaef31b63..789fe35b893a 100644
@@ -817,6 +817,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
 
Quoted 6 lines

kernel/bpf/stackmap.c static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, · line 820

Quoted 3 lines
kernel/bpf/stackmap.c +1 -0
        if (may_fault)
                rcu_read_lock(); /* need RCU for perf's callchain below */
+       preempt_disable();

nit: asymmetrical to preempt_enable, I'll move it to before rcu_read_lock, so we can have proper nesting

ugh, nice.. thanks

jirka

Quoted 25 lines
Quoted 25 lines
kernel/bpf/stackmap.c +2 -1
        if (kernel && task) {
                trace = get_callchain_entry_for_task(task, max_depth);
@@ -828,6 +829,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
        if (unlikely(!trace) || trace->nr < skip) {
                if (may_fault)
                        rcu_read_unlock();
+               preempt_enable();
                goto err_fault;
        }
 
@@ -836,6 +838,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
        /* trace should not be dereferenced after this point */
        if (may_fault)
                rcu_read_unlock();
+       preempt_enable();
 
        return callchain_finalize(buf, size, trace_nr, elem_size, flags, may_fault);
 
--

2.54.0

author reviewer
#14 lore raw

PATCH v4 11/12 bpf: Avoid changing callchain in bpf_get_stack_pe

In reply to Jiri Olsa

There's no need to modify the trace object bpf_get_stack_pe, we just need to pass the needed callchain length in separate argument.

This way we can have callchain pointers const and remove the trace->nr modification and restoration.

Assisted-by Codex:GPT-5.5
---
 kernel/bpf/stackmap.c | 38 ++++++++++++++++----------------------
 1 file changed, 16 insertions(+), 22 deletions(-)
kernel/bpf/stackmap.c +16 -22
index 789fe35b893a..531c043d1aaa 100644
@@ -661,7 +661,7 @@ const struct bpf_func_proto bpf_get_stackid_proto = {
 	.arg3_type	= ARG_ANYTHING,
 };
 
-static __u64 count_kernel_ip(struct perf_callchain_entry *trace)
+static __u64 count_kernel_ip(const struct perf_callchain_entry *trace)
 {
 	__u64 nr_kernel = 0;
 
@@ -741,15 +741,15 @@ const struct bpf_func_proto bpf_get_stackid_proto_pe = {
 	.arg3_type	= ARG_ANYTHING,
 };
 
-static u32 callchain_store(struct perf_callchain_entry *trace, void *buf,
-			   u32 elem_size, u64 flags)
+static u32 callchain_store(const struct perf_callchain_entry *trace, u32 trace_nr,
+			   void *buf, u32 elem_size, u64 flags)
 {
 	bool user_build_id = flags & BPF_F_USER_BUILD_ID;
 	u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
-	u32 trace_nr, copy_len;
-	u64 *ips;
+	const u64 *ips;
+	u32 copy_len;
 
-	trace_nr = trace->nr - skip;
+	trace_nr = trace_nr - skip;
 	copy_len = trace_nr * elem_size;
 
 	ips = trace->ip + skip;
@@ -833,7 +833,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
 		goto err_fault;
 	}
 
-	trace_nr = callchain_store(trace, buf, elem_size, flags);
+	trace_nr = callchain_store(trace, trace->nr, buf, elem_size, flags);
 
 	/* trace should not be dereferenced after this point */
 	if (may_fault)
@@ -935,13 +935,13 @@ const struct bpf_func_proto bpf_get_task_stack_sleepable_proto = {
 	.arg4_type	= ARG_ANYTHING,
 };
 
-static int __bpf_get_stack_pe(struct perf_callchain_entry *trace, void *buf, u32 size,
-			      u64 flags)
+static int __bpf_get_stack_pe(const struct perf_callchain_entry *trace, u32 trace_nr,
+			      void *buf, u32 size, u64 flags)
 {
 	bool user_build_id = flags & BPF_F_USER_BUILD_ID;
 	u64 skip = flags & BPF_F_SKIP_FIELD_MASK;
 	bool user = flags & BPF_F_USER_STACK;
-	u32 elem_size, max_depth, trace_nr;
+	u32 elem_size, max_depth;
 	bool kernel = !user;
 
 	if (kernel && user_build_id)
@@ -952,12 +952,12 @@ static int __bpf_get_stack_pe(struct perf_callchain_entry *trace, void *buf, u32
 		return -EINVAL;
 
 	max_depth = stack_map_calculate_max_depth(size, elem_size, flags);
-	trace->nr = min_t(u32, trace->nr, max_depth);
+	trace_nr = min_t(u32, trace_nr, max_depth);
 
-	if (trace->nr < skip)
+	if (trace_nr < skip)
 		return -EFAULT;
 
-	trace_nr = callchain_store(trace, buf, elem_size, flags);
+	trace_nr = callchain_store(trace, trace_nr, buf, elem_size, flags);
 	return callchain_finalize(buf, size, trace_nr, elem_size, flags, false /* !may_fault */);
 }
 
@@ -965,8 +965,8 @@ BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx,
 	   void *, buf, u32, size, u64, flags)
 {
 	struct pt_regs *regs = (struct pt_regs *)(ctx->regs);
+	const struct perf_callchain_entry *trace;
 	struct perf_event *event = ctx->event;
-	struct perf_callchain_entry *trace;
 	bool kernel, user;
 	int err = -EINVAL;
 	__u64 nr_kernel;
@@ -989,13 +989,7 @@ BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx,
 	nr_kernel = count_kernel_ip(trace);
 
 	if (kernel) {
-		__u64 nr = trace->nr;
-
-		trace->nr = nr_kernel;
-		err = __bpf_get_stack_pe(trace, buf, size, flags);
-
-		/* restore nr */
-		trace->nr = nr;
+		err = __bpf_get_stack_pe(trace, nr_kernel, buf, size, flags);
 	} else { /* user */
 		u64 skip = flags & BPF_F_SKIP_FIELD_MASK;
 
@@ -1003,7 +997,7 @@ BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx,
 		if (skip > BPF_F_SKIP_FIELD_MASK)
 			goto clear;
 		flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip;
-		err = __bpf_get_stack_pe(trace, buf, size, flags);
+		err = __bpf_get_stack_pe(trace, trace->nr, buf, size, flags);
 	}
 
 clear:
Signature · 1 line
2.54.0
author reviewer
#15 lore raw

PATCH v4 12/12 bpf: Avoid changing callchain in bpf_get_stackid_pe

In reply to Jiri Olsa

There's no need to modify the trace object in bpf_get_stackid_pe, we just need to pass the needed callchain length in separate argument.

This way we can have callchain pointers const and remove the trace->nr modification and restoration.

Assisted-by Codex:GPT-5.5
---
 kernel/bpf/stackmap.c | 39 +++++++++++++++++----------------------
 1 file changed, 17 insertions(+), 22 deletions(-)
kernel/bpf/stackmap.c +17 -22
index 531c043d1aaa..65c25fee394a 100644
@@ -506,7 +506,7 @@ get_callchain_entry_for_task(struct task_struct *task, u32 max_depth)
 
 struct stackid {
 	struct stack_map_bucket *bucket;
-	u64 *ips;
+	const u64 *ips;
 	u32  nr;
 	u32  len;
 	u32  hash;
@@ -515,21 +515,21 @@ struct stackid {
 };
 
 static int stackid_init(struct stackid *stackid, struct bpf_map *map,
-			struct perf_callchain_entry *trace, u64 flags)
+			const struct perf_callchain_entry *trace, u32 trace_nr, u64 flags)
 {
 	struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
 	u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
 	u32 max_depth;
 
-	if (trace->nr <= skip)
+	if (trace_nr <= skip)
 		/* skipping more than usable stack trace */
 		return -EFAULT;
 
 	max_depth = stack_map_calculate_max_depth(map->value_size, stack_map_data_size(map), flags);
-	stackid->nr = min_t(u32, trace->nr - skip, max_depth - skip);
+	stackid->nr = min_t(u32, trace_nr - skip, max_depth - skip);
 	stackid->len = stackid->nr * sizeof(u64);
 	stackid->ips = trace->ip + skip;
-	stackid->hash = jhash2((u32 *)stackid->ips, stackid->len / sizeof(u32), 0);
+	stackid->hash = jhash2((const u32 *)stackid->ips, stackid->len / sizeof(u32), 0);
 	stackid->id = stackid->hash & (smap->n_buckets - 1);
 	stackid->bucket = READ_ONCE(smap->buckets[stackid->id]);
 	stackid->hash_matches = stackid->bucket && stackid->bucket->hash == stackid->hash;
@@ -537,11 +537,12 @@ static int stackid_init(struct stackid *stackid, struct bpf_map *map,
 }
 
 static int stackid_fastpath(struct stackid *stackid, struct bpf_map *map,
-			    struct perf_callchain_entry *trace, u64 flags)
+			    const struct perf_callchain_entry *trace, u32 trace_nr,
+			    u64 flags)
 {
 	int err;
 
-	err = stackid_init(stackid, map, trace, flags);
+	err = stackid_init(stackid, map, trace, trace_nr, flags);
 	if (err)
 		return err;
 
@@ -640,7 +641,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
 			/* couldn't fetch the stack trace */
 			return -EFAULT;
 
-		err = stackid_fastpath(&stackid, map, trace, flags);
+		err = stackid_fastpath(&stackid, map, trace, trace->nr, flags);
 		if (err != -ENOENT)
 			return err;
 
@@ -676,12 +677,13 @@ static __u64 count_kernel_ip(const struct perf_callchain_entry *trace)
 BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx,
 	   struct bpf_map *, map, u64, flags)
 {
+	const struct perf_callchain_entry *trace;
 	struct perf_event *event = ctx->event;
 	struct stack_map_bucket *new_bucket;
-	struct perf_callchain_entry *trace;
 	struct stackid stackid;
 	bool kernel, user;
 	__u64 nr_kernel;
+	u32 trace_nr;
 	int ret;
 
 	/* perf_sample_data doesn't have callchain, use bpf_get_stackid */
@@ -701,13 +703,13 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx,
 		return -EFAULT;
 
 	nr_kernel = count_kernel_ip(trace);
-	__u64 nr = trace->nr; /* save original */
 
 	if (kernel) {
-		trace->nr = nr_kernel;
+		trace_nr = nr_kernel;
 	} else { /* user */
 		u64 skip = flags & BPF_F_SKIP_FIELD_MASK;
 
+		trace_nr = trace->nr;
 		skip += nr_kernel;
 		if (skip > BPF_F_SKIP_FIELD_MASK)
 			return -EFAULT;
@@ -715,21 +717,14 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx,
 		flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip;
 	}
 
-	ret = stackid_fastpath(&stackid, map, trace, flags);
+	ret = stackid_fastpath(&stackid, map, trace, trace_nr, flags);
 	if (ret != -ENOENT)
-		goto out;
+		return ret;
 
 	new_bucket = stackid_new_bucket(&stackid, map);
-	if (new_bucket) {
-		trace->nr = nr;
+	if (new_bucket)
 		return stackid_install(&stackid, map, new_bucket, flags);
-	}
-	ret = -ENOMEM;
-
-out:
-	/* restore nr */
-	trace->nr = nr;
-	return ret;
+	return -ENOMEM;
 }
 
 const struct bpf_func_proto bpf_get_stackid_proto_pe = {
Signature · 1 line
2.54.0
#16 lore raw
In reply to Jiri Olsa
Quoted 44 lines · Jiri Olsa

hi,
we need to disable preemption for get_perf_callchain and keep it disabled as long as we are accessing its returned trace entries buffer.

This patchset refactors both bpf_get_stack and bpf_get_stackid helpers as suggested by Andrii [1] before applying the actual preemption fix.

Note the initial fix was sent by Tao Chen [2], but there was no follow up on this since February, hence this post.

thanks,
jirka

v1: https://lore.kernel.org/bpf/[email protected]/
v2: https://lore.kernel.org/bpf/[email protected]/
v3: https://lore.kernel.org/bpf/[email protected]/

v4 changes:
- fixed changelogs [sashiko]
- nr_trace variable rename [sashiko]
- fixed Daniel's email

v3 changes:
- pass flags rgument to callchain_finalize [sashiko]
- renamed err label to out in bpf_get_stackid_pe [sashiko]
- used scoped_guard in bpf_get_stackid [Leon]
- replaced trace->nr modification from *_pe helpers with length argument (new patches 11,12) [Andrii]

v2 changes:
- removed several unused functions arguments [sashiko]
- restore trace-nr value fix in bpf_get_stack_pe [sashiko]
- kept rcu locking together with preemption disable in __bpf_get_stack [sashiko]
- clear buf on error paths in __bpf_get_task_stack [sashiko]

[1] https://lore.kernel.org/bpf/CAEf4BzZwvAUgLwz-M0Y_NJLTmedyY9U6s7LrSmn751hQdTP4Uw@mail.gmail.com/
[2] https://lore.kernel.org/bpf/[email protected]/
---
Daniel Borkmann (1):
bpf: Disable preemption in __bpf_get_stack

Quoted 13 lines

Jiri Olsa (11):
bpf: Factor stackid_init function from __bpf_get_stackid
bpf: Factor stackid_fastpath function from __bpf_get_stackid
bpf: Factor stackid_new_bucket from __bpf_get_stackid
bpf: Use stack id functions instead of __bpf_get_stackid
bpf: Disable preemption in bpf_get_stackid
bpf: Factor callchain_store function from __bpf_get_stack
bpf: Factor callchain_finalize function from __bpf_get_stack
bpf: Remove trace_in argument from __bpf_get_stack
bpf: Clear buf on error in __bpf_get_task_stack
bpf: Avoid changing callchain in bpf_get_stack_pe
bpf: Avoid changing callchain in bpf_get_stackid_pe

The amount of repetitive but subtly different code for all the different program types to capture stack trace is a bit mind-numbing, it all looks good to me (except that preempt_enable ordering which I fixed up), I hope we didn't introduce any new subtle bugs, but oh well. Applied to bpf-next.

Quoted 2 lines
 kernel/bpf/stackmap.c | 314 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------------------------------------
 1 file changed, 205 insertions(+), 109 deletions(-)
#17 lore raw
In reply to Jiri Olsa
This series was applied to bpf/bpf-next.git (master) by Andrii Nakryiko <[email protected]>:

Hello:

This series was applied to bpf/bpf-next.git (master) by Andrii Nakryiko <[email protected]>:

Quoted 8 lines · On Wed, 5 Aug 2026 11:28:31 +0200 you wrote:

hi,
we need to disable preemption for get_perf_callchain and keep it disabled as long as we are accessing its returned trace entries buffer.

This patchset refactors both bpf_get_stack and bpf_get_stackid helpers as suggested by Andrii [1] before applying the actual preemption fix.

[...]

Here is the summary with links:
- [PATCHv4,bpf-next,01/12] bpf: Factor stackid_init function from __bpf_get_stackid

    https://git.kernel.org/bpf/bpf-next/c/15b837759a97
  - [PATCHv4,bpf-next,02/12] bpf: Factor stackid_fastpath function from __bpf_get_stackid
    https://git.kernel.org/bpf/bpf-next/c/0ca56befcffe
  - [PATCHv4,bpf-next,03/12] bpf: Factor stackid_new_bucket from __bpf_get_stackid
    https://git.kernel.org/bpf/bpf-next/c/bb4e6f4e1b68
  - [PATCHv4,bpf-next,04/12] bpf: Use stack id functions instead of __bpf_get_stackid
    https://git.kernel.org/bpf/bpf-next/c/09b3fd6caa0b
  - [PATCHv4,bpf-next,05/12] bpf: Disable preemption in bpf_get_stackid
    https://git.kernel.org/bpf/bpf-next/c/15f1bd857466
  - [PATCHv4,bpf-next,06/12] bpf: Factor callchain_store function from __bpf_get_stack
    https://git.kernel.org/bpf/bpf-next/c/cbb99938e793
  - [PATCHv4,bpf-next,07/12] bpf: Factor callchain_finalize function from __bpf_get_stack
    https://git.kernel.org/bpf/bpf-next/c/014fbe5902dc
  - [PATCHv4,bpf-next,08/12] bpf: Remove trace_in argument from __bpf_get_stack
    (no matching commit)
  - [PATCHv4,bpf-next,09/12] bpf: Clear buf on error in __bpf_get_task_stack
    https://git.kernel.org/bpf/bpf-next/c/f5d242825ca4
  - [PATCHv4,bpf-next,10/12] bpf: Disable preemption in __bpf_get_stack
    (no matching commit)
  - [PATCHv4,bpf-next,11/12] bpf: Avoid changing callchain in bpf_get_stack_pe
    (no matching commit)
  - [PATCHv4,bpf-next,12/12] bpf: Avoid changing callchain in bpf_get_stackid_pe
    https://git.kernel.org/bpf/bpf-next/c/a74594607a0b

You are awesome, thank you!

Signature · 2 lines
Deet-doot-dot, I am a bot.
https://korg.docs.kernel.org/patchwork/pwbot.html