Skip to contentSkip to conversation

Pass external callchain entry to get_perf_callchain

RFC PATCH Superseded Tao Chen 5 participants 12 messages Oct 14, 2025 – Oct 23, 2025

Subsystems (3)

v2 of 8 Superseded Latest is v8 →

lore
Share & tools
lore

Tao Chen

author
#0 lore raw

v2 cover Pass external callchain entry to get_perf_callchain

Background
==========
Alexei noted we should use preempt_disable to protect get_perf_callchain in bpf stackmap.
https://lore.kernel.org/bpf/CAADnVQ+s8B7-fvR1TNO-bniSyKv57cH_ihRszmZV7pQDyV=VDQ@mail.gmail.com

A previous patch was submitted to attempt fixing this issue. And Andrii suggested teach get_perf_callchain to let us pass that buffer directly to avoid that unnecessary copy.
https://lore.kernel.org/bpf/[email protected]

Proposed Solution
=================
Add external perf_callchain_entry parameter for get_perf_callchain to allow us to use external buffer from BPF side. The biggest advantage is that it can reduce unnecessary copies.

Todo
====
If the above changes are reasonable, it seems that get_callchain_entry_for_task could also use an external perf_callchain_entry.

But I'm not sure if this modification is appropriate. After all, the implementation of get_callchain_entry in the perf subsystem seems much more complex than directly using an external buffer.

Comments and suggestions are always welcome.

Change list:
- v1 -> v2
From Jiri
- rebase code, fix confict
- v1: https://lore.kernel.org/bpf/[email protected]

Tao Chen (2):
perf: Use extern perf_callchain_entry for get_perf_callchain
bpf: Pass external callchain entry to get_perf_callchain

 include/linux/perf_event.h |  4 ++--
 kernel/bpf/stackmap.c      | 19 +++++++++++--------
 kernel/events/callchain.c  | 13 +++++++++----
 kernel/events/core.c       |  2 +-
 4 files changed, 23 insertions(+), 15 deletions(-)
Signature · 1 line
2.48.1
author
#1 lore raw

RFC PATCH v2 1/2 perf: Use extern perf_callchain_entry for get_perf_callchain

From bpf stack map, we want to use our own buffers to avoid unnecessary copy, so let us pass it directly. BPF will use this in the next patch.

---
 include/linux/perf_event.h |  4 ++--
 kernel/bpf/stackmap.c      |  4 ++--
 kernel/events/callchain.c  | 13 +++++++++----
 kernel/events/core.c       |  2 +-
 4 files changed, 14 insertions(+), 9 deletions(-)
include/linux/perf_event.h +2 -2
index fd1d91017b9..b144da7d803 100644
@@ -1719,8 +1719,8 @@ DECLARE_PER_CPU(struct perf_callchain_entry, perf_callchain_entry);
 extern void perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs);
 extern void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs);
 extern struct perf_callchain_entry *
-get_perf_callchain(struct pt_regs *regs, bool kernel, bool user,
-		   u32 max_stack, bool crosstask, bool add_mark);
+get_perf_callchain(struct pt_regs *regs, struct perf_callchain_entry *external_entry,
+		   bool kernel, bool user, u32 max_stack, bool crosstask, bool add_mark);
 extern int get_callchain_buffers(int max_stack);
 extern void put_callchain_buffers(void);
 extern struct perf_callchain_entry *get_callchain_entry(int *rctx);
kernel/bpf/stackmap.c +2 -2
index 4d53cdd1374..94e46b7f340 100644
@@ -314,7 +314,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
 	if (max_depth > sysctl_perf_event_max_stack)
 		max_depth = sysctl_perf_event_max_stack;
 
-	trace = get_perf_callchain(regs, kernel, user, max_depth,
+	trace = get_perf_callchain(regs, NULL, kernel, user, max_depth,
 				   false, false);
 
 	if (unlikely(!trace))
@@ -451,7 +451,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
 	else if (kernel && task)
 		trace = get_callchain_entry_for_task(task, max_depth);
 	else
-		trace = get_perf_callchain(regs, kernel, user, max_depth,
+		trace = get_perf_callchain(regs, NULL, kernel, user, max_depth,
 					   crosstask, false);
 
 	if (unlikely(!trace) || trace->nr < skip) {
kernel/events/callchain.c +9 -4
index 808c0d7a31f..851e8f9d026 100644
@@ -217,8 +217,8 @@ static void fixup_uretprobe_trampoline_entries(struct perf_callchain_entry *entr
 }
 
 struct perf_callchain_entry *
-get_perf_callchain(struct pt_regs *regs, bool kernel, bool user,
-		   u32 max_stack, bool crosstask, bool add_mark)
+get_perf_callchain(struct pt_regs *regs, struct perf_callchain_entry *external_entry,
+		   bool kernel, bool user, u32 max_stack, bool crosstask, bool add_mark)
 {
 	struct perf_callchain_entry *entry;
 	struct perf_callchain_entry_ctx ctx;
@@ -228,7 +228,11 @@ get_perf_callchain(struct pt_regs *regs, bool kernel, bool user,
 	if (crosstask && user && !kernel)
 		return NULL;
 
-	entry = get_callchain_entry(&rctx);
+	if (external_entry)
+		entry = external_entry;
+	else
+		entry = get_callchain_entry(&rctx);
+
 	if (!entry)
 		return NULL;
 
@@ -260,7 +264,8 @@ get_perf_callchain(struct pt_regs *regs, bool kernel, bool user,
 	}
 
 exit_put:
-	put_callchain_entry(rctx);
+	if (!external_entry)
+		put_callchain_entry(rctx);
 
 	return entry;
 }
kernel/events/core.c +1 -1
index 7541f6f85fc..5d8e146003a 100644
@@ -8217,7 +8217,7 @@ perf_callchain(struct perf_event *event, struct pt_regs *regs)
 	if (!kernel && !user)
 		return &__empty_callchain;
 
-	callchain = get_perf_callchain(regs, kernel, user,
+	callchain = get_perf_callchain(regs, NULL, kernel, user,
 				       max_stack, crosstask, true);
 	return callchain ?: &__empty_callchain;
 }
Signature · 1 line
2.48.1
author
#2 lore raw

RFC PATCH v2 2/2 bpf: Pass external callchain entry to get_perf_callchain

In reply to Tao Chen

As Alexei noted, get_perf_callchain() return values may be reused if a task is preempted after the BPF program enters migrate disable mode. Drawing on the per-cpu design of bpf_perf_callchain_entries, stack-allocated memory of bpf_perf_callchain_entry is used here.

---
 kernel/bpf/stackmap.c | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)
kernel/bpf/stackmap.c +11 -8
index 94e46b7f340..acd72c021c0 100644
@@ -31,6 +31,11 @@ struct bpf_stack_map {
 	struct stack_map_bucket *buckets[] __counted_by(n_buckets);
 };
 
+struct bpf_perf_callchain_entry {
+	u64 nr;
+	u64 ip[PERF_MAX_STACK_DEPTH];
+};
+
 static inline bool stack_map_use_build_id(struct bpf_map *map)
 {
 	return (map->map_flags & BPF_F_STACK_BUILD_ID);
@@ -305,6 +310,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
 	bool user = flags & BPF_F_USER_STACK;
 	struct perf_callchain_entry *trace;
 	bool kernel = !user;
+	struct bpf_perf_callchain_entry entry = { 0 };
 
 	if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
 			       BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID)))
@@ -314,12 +320,8 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
 	if (max_depth > sysctl_perf_event_max_stack)
 		max_depth = sysctl_perf_event_max_stack;
 
-	trace = get_perf_callchain(regs, NULL, kernel, user, max_depth,
-				   false, false);
-
-	if (unlikely(!trace))
-		/* couldn't fetch the stack trace */
-		return -EFAULT;
+	trace = get_perf_callchain(regs, (struct perf_callchain_entry *)&entry,
+				   kernel, user, max_depth, false, false);
 
 	return __bpf_get_stackid(map, trace, flags);
 }
@@ -412,6 +414,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
 	u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
 	bool user = flags & BPF_F_USER_STACK;
 	struct perf_callchain_entry *trace;
+	struct bpf_perf_callchain_entry entry = { 0 };
 	bool kernel = !user;
 	int err = -EINVAL;
 	u64 *ips;
@@ -451,8 +454,8 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
 	else if (kernel && task)
 		trace = get_callchain_entry_for_task(task, max_depth);
 	else
-		trace = get_perf_callchain(regs, NULL, kernel, user, max_depth,
-					   crosstask, false);
+		trace = get_perf_callchain(regs, (struct perf_callchain_entry *)&entry,
+					   kernel, user, max_depth, crosstask, false);
 
 	if (unlikely(!trace) || trace->nr < skip) {
 		if (may_fault)
Signature · 1 line
2.48.1
#3 lore raw

Re: [RFC PATCH bpf-next v2 2/2] bpf: Pass external callchain entry to get_perf_callchain

Quoted 27 lines · Tao Chen

As Alexei noted, get_perf_callchain() return values may be reused if a task is preempted after the BPF program enters migrate disable mode. Drawing on the per-cpu design of bpf_perf_callchain_entries, stack-allocated memory of bpf_perf_callchain_entry is used here.

---
 kernel/bpf/stackmap.c | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)
kernel/bpf/stackmap.c +5 -0
index 94e46b7f340..acd72c021c0 100644
@@ -31,6 +31,11 @@ struct bpf_stack_map {
 	struct stack_map_bucket *buckets[] __counted_by(n_buckets);
 };
 
+struct bpf_perf_callchain_entry {
+	u64 nr;
+	u64 ip[PERF_MAX_STACK_DEPTH];
+};
+
 static inline bool stack_map_use_build_id(struct bpf_map *map)
 {
 	return (map->map_flags & BPF_F_STACK_BUILD_ID);

kernel/bpf/stackmap.c BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, · line 313

Quoted 4 lines
kernel/bpf/stackmap.c +1 -0
@@ -305,6 +310,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
 	bool user = flags & BPF_F_USER_STACK;
 	struct perf_callchain_entry *trace;
 	bool kernel = !user;
+	struct bpf_perf_callchain_entry entry = { 0 };

so IIUC having entries on stack we do not need to do preempt_disable you had in the previous version, right?

I saw Andrii's justification to have this on the stack, I think it's fine, but does it have to be initialized? it seems that only used entries are copied to map

jirka

Quoted 40 lines
kernel/bpf/stackmap.c +5 -8
 	if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
 			       BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID)))
@@ -314,12 +320,8 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
 	if (max_depth > sysctl_perf_event_max_stack)
 		max_depth = sysctl_perf_event_max_stack;
 
-	trace = get_perf_callchain(regs, NULL, kernel, user, max_depth,
-				   false, false);
-
-	if (unlikely(!trace))
-		/* couldn't fetch the stack trace */
-		return -EFAULT;
+	trace = get_perf_callchain(regs, (struct perf_callchain_entry *)&entry,
+				   kernel, user, max_depth, false, false);
 
 	return __bpf_get_stackid(map, trace, flags);
 }
@@ -412,6 +414,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
 	u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
 	bool user = flags & BPF_F_USER_STACK;
 	struct perf_callchain_entry *trace;
+	struct bpf_perf_callchain_entry entry = { 0 };
 	bool kernel = !user;
 	int err = -EINVAL;
 	u64 *ips;
@@ -451,8 +454,8 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
 	else if (kernel && task)
 		trace = get_callchain_entry_for_task(task, max_depth);
 	else
-		trace = get_perf_callchain(regs, NULL, kernel, user, max_depth,
-					   crosstask, false);
+		trace = get_perf_callchain(regs, (struct perf_callchain_entry *)&entry,
+					   kernel, user, max_depth, crosstask, false);
 
 	if (unlikely(!trace) || trace->nr < skip) {
 		if (may_fault)
Signature · 1 line
2.48.1
author
#4 lore raw

Re: [RFC PATCH bpf-next v2 2/2] bpf: Pass external callchain entry to get_perf_callchain

Quoted 4 lines · Jiri Olsa

Review

Quoted 1 line
(quoted diff) +1 -0
+	struct bpf_perf_callchain_entry entry = { 0 };

so IIUC having entries on stack we do not need to do preempt_disable you had in the previous version, right?

Yes, i think so, preempt_disable seems unnecessary.

Quoted 3 lines

I saw Andrii's justification to have this on the stack, I think it's fine, but does it have to be initialized? it seems that only used entries are copied to map

That makes sense. Removing it definitely looks better.

Signature · 2 lines
Best Regards
Tao Chen
#5 lore raw

Re: [RFC PATCH bpf-next v2 2/2] bpf: Pass external callchain entry to get_perf_callchain

In reply to Jiri Olsa
Quoted 37 lines · Jiri Olsa
Quoted 27 lines · Tao Chen

As Alexei noted, get_perf_callchain() return values may be reused if a task is preempted after the BPF program enters migrate disable mode. Drawing on the per-cpu design of bpf_perf_callchain_entries, stack-allocated memory of bpf_perf_callchain_entry is used here.

---
 kernel/bpf/stackmap.c | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)
kernel/bpf/stackmap.c +5 -0
index 94e46b7f340..acd72c021c0 100644
@@ -31,6 +31,11 @@ struct bpf_stack_map {
      struct stack_map_bucket *buckets[] __counted_by(n_buckets);
 };
 
+struct bpf_perf_callchain_entry {
+     u64 nr;
+     u64 ip[PERF_MAX_STACK_DEPTH];
+};
+
 static inline bool stack_map_use_build_id(struct bpf_map *map)
 {
      return (map->map_flags & BPF_F_STACK_BUILD_ID);

kernel/bpf/stackmap.c BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, · line 313

Quoted 4 lines
kernel/bpf/stackmap.c +1 -0
@@ -305,6 +310,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
      bool user = flags & BPF_F_USER_STACK;
      struct perf_callchain_entry *trace;
      bool kernel = !user;
+     struct bpf_perf_callchain_entry entry = { 0 };

so IIUC having entries on stack we do not need to do preempt_disable you had in the previous version, right?

Quoted 3 lines

I saw Andrii's justification to have this on the stack, I think it's fine, but does it have to be initialized? it seems that only used entries are copied to map

No. We're not adding 1k stack consumption.

pw-bot: cr

#6 lore raw

Re: [RFC PATCH bpf-next v2 2/2] bpf: Pass external callchain entry to get_perf_callchain

Quoted 22 lines · Alexei Starovoitov
Quoted 20 lines · Jiri Olsa
Quoted 18 lines · Tao Chen

As Alexei noted, get_perf_callchain() return values may be reused if a task is preempted after the BPF program enters migrate disable mode. Drawing on the per-cpu design of bpf_perf_callchain_entries, stack-allocated memory of bpf_perf_callchain_entry is used here.

---
 kernel/bpf/stackmap.c | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)
kernel/bpf/stackmap.c +0 -0
index 94e46b7f340..acd72c021c0 100644
@@ -31,6 +31,11 @@ struct bpf_stack_map {
      struct stack_map_bucket *buckets[] __counted_by(n_buckets);
 };
 
Quoted 5 lines
Quoted 5 lines

kernel/bpf/stackmap.c struct bpf_stack_map { · line 38

Quoted 5 lines
kernel/bpf/stackmap.c +5 -0
+struct bpf_perf_callchain_entry {
+     u64 nr;
+     u64 ip[PERF_MAX_STACK_DEPTH];
+};
+

we shouldn't introduce another type, there is perf_callchain_entry in linux/perf_event.h, what's the problem with using that?

Quoted 12 lines
Quoted 12 lines

kernel/bpf/stackmap.c BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, · line 313

Quoted 8 lines
kernel/bpf/stackmap.c +1 -0
 static inline bool stack_map_use_build_id(struct bpf_map *map)
 {
      return (map->map_flags & BPF_F_STACK_BUILD_ID);
@@ -305,6 +310,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
      bool user = flags & BPF_F_USER_STACK;
      struct perf_callchain_entry *trace;
      bool kernel = !user;
+     struct bpf_perf_callchain_entry entry = { 0 };

so IIUC having entries on stack we do not need to do preempt_disable you had in the previous version, right?

Quoted 5 lines
Quoted 3 lines

I saw Andrii's justification to have this on the stack, I think it's fine, but does it have to be initialized? it seems that only used entries are copied to map

No. We're not adding 1k stack consumption.

Right, and I thought we concluded as much last time, so it's a bit surprising to see this in this patch.

Tao, you should go with 3 entries per CPU used in a stack-like fashion. And then passing that entry into get_perf_callchain() (to avoid one extra copy).

Quoted 2 lines

pw-bot: cr

author
#7 lore raw

Re: [RFC PATCH bpf-next v2 2/2] bpf: Pass external callchain entry to get_perf_callchain

Quoted 24 lines · Andrii Nakryiko
Quoted 22 lines · Alexei Starovoitov
Quoted 20 lines · Jiri Olsa
Quoted 18 lines · Tao Chen

As Alexei noted, get_perf_callchain() return values may be reused if a task is preempted after the BPF program enters migrate disable mode. Drawing on the per-cpu design of bpf_perf_callchain_entries, stack-allocated memory of bpf_perf_callchain_entry is used here.

---
  kernel/bpf/stackmap.c | 19 +++++++++++--------
  1 file changed, 11 insertions(+), 8 deletions(-)
kernel/bpf/stackmap.c +0 -0
index 94e46b7f340..acd72c021c0 100644
@@ -31,6 +31,11 @@ struct bpf_stack_map {
       struct stack_map_bucket *buckets[] __counted_by(n_buckets);
  };
 
Quoted 8 lines
Quoted 5 lines
Quoted 5 lines

kernel/bpf/stackmap.c struct bpf_stack_map { · line 38

Quoted 5 lines
kernel/bpf/stackmap.c +5 -0
+struct bpf_perf_callchain_entry {
+     u64 nr;
+     u64 ip[PERF_MAX_STACK_DEPTH];
+};
+

we shouldn't introduce another type, there is perf_callchain_entry in linux/perf_event.h, what's the problem with using that?

perf_callchain_entry uses flexible array, DEFINE_PER_CPU seems do not
create buffer for this, for ease of use, the size of the ip array has been explicitly defined.

struct perf_callchain_entry {
        u64                             nr;
        u64                             ip[]; /* 

/proc/sys/kernel/perf_event_max_stack */
};

Quoted 13 lines
Quoted 13 lines
Quoted 13 lines

kernel/bpf/stackmap.c BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, · line 313

Quoted 9 lines
kernel/bpf/stackmap.c +1 -0
  static inline bool stack_map_use_build_id(struct bpf_map *map)
  {
       return (map->map_flags & BPF_F_STACK_BUILD_ID);
@@ -305,6 +310,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
       bool user = flags & BPF_F_USER_STACK;
       struct perf_callchain_entry *trace;
       bool kernel = !user;
+     struct bpf_perf_callchain_entry entry = { 0 };

so IIUC having entries on stack we do not need to do preempt_disable you had in the previous version, right?

Quoted 9 lines
Quoted 5 lines
Quoted 3 lines

I saw Andrii's justification to have this on the stack, I think it's fine, but does it have to be initialized? it seems that only used entries are copied to map

No. We're not adding 1k stack consumption.

Right, and I thought we concluded as much last time, so it's a bit surprising to see this in this patch.

Ok, I feel like I'm missing some context from our previous exchange.

Quoted 4 lines

Tao, you should go with 3 entries per CPU used in a stack-like fashion. And then passing that entry into get_perf_callchain() (to avoid one extra copy).

Got it. It is more clearer, will change it in v3.

Quoted 2 lines
Quoted 2 lines

pw-bot: cr

Signature · 2 lines
Best Regards
Tao Chen
#8 lore raw

Re: [RFC PATCH bpf-next v2 2/2] bpf: Pass external callchain entry to get_perf_callchain

Quoted 39 lines · Tao Chen
Quoted 24 lines · Andrii Nakryiko
Quoted 22 lines · Alexei Starovoitov
Quoted 20 lines · Jiri Olsa
Quoted 18 lines · Tao Chen

As Alexei noted, get_perf_callchain() return values may be reused if a task is preempted after the BPF program enters migrate disable mode. Drawing on the per-cpu design of bpf_perf_callchain_entries, stack-allocated memory of bpf_perf_callchain_entry is used here.

---
  kernel/bpf/stackmap.c | 19 +++++++++++--------
  1 file changed, 11 insertions(+), 8 deletions(-)
kernel/bpf/stackmap.c +0 -0
index 94e46b7f340..acd72c021c0 100644
@@ -31,6 +31,11 @@ struct bpf_stack_map {
       struct stack_map_bucket *buckets[] __counted_by(n_buckets);
  };
 
Quoted 8 lines
Quoted 5 lines
Quoted 5 lines

kernel/bpf/stackmap.c struct bpf_stack_map { · line 38

Quoted 5 lines
kernel/bpf/stackmap.c +5 -0
+struct bpf_perf_callchain_entry {
+     u64 nr;
+     u64 ip[PERF_MAX_STACK_DEPTH];
+};
+

we shouldn't introduce another type, there is perf_callchain_entry in linux/perf_event.h, what's the problem with using that?

perf_callchain_entry uses flexible array, DEFINE_PER_CPU seems do not create buffer for this, for ease of use, the size of the ip array has been explicitly defined.

Quoted 6 lines
struct perf_callchain_entry {
         u64                             nr;
         u64                             ip[]; /*

/proc/sys/kernel/perf_event_max_stack */
};

Ok, fair enough, but instead of casting between perf_callchain_entry and bpf_perf_callchain_entry, why not put perf_callchain_entry inside bpf_perf_callchain_entry as a first struct and pass a pointer to it. That looks a bit more appropriate? Though I'm not sure if compiler will complain about that flex array...

But on related note, I looked briefly at how perf gets those perf_callchain_entries, and it does seem like it also has a small stack of entries, so maybe we don't really need to invent anything here. See PERF_NR_CONTEXTS and how that's used.

If instead of disabling preemption we disable migration, then I think we should be good with relying on perf's callchain management, or am I missing something?

Quoted 39 lines
Quoted 13 lines
Quoted 13 lines
Quoted 13 lines

kernel/bpf/stackmap.c BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, · line 313

Quoted 9 lines
kernel/bpf/stackmap.c +1 -0
  static inline bool stack_map_use_build_id(struct bpf_map *map)
  {
       return (map->map_flags & BPF_F_STACK_BUILD_ID);
@@ -305,6 +310,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
       bool user = flags & BPF_F_USER_STACK;
       struct perf_callchain_entry *trace;
       bool kernel = !user;
+     struct bpf_perf_callchain_entry entry = { 0 };

so IIUC having entries on stack we do not need to do preempt_disable you had in the previous version, right?

Quoted 9 lines
Quoted 5 lines
Quoted 3 lines

I saw Andrii's justification to have this on the stack, I think it's fine, but does it have to be initialized? it seems that only used entries are copied to map

No. We're not adding 1k stack consumption.

Right, and I thought we concluded as much last time, so it's a bit surprising to see this in this patch.

Ok, I feel like I'm missing some context from our previous exchange.

Quoted 4 lines

Tao, you should go with 3 entries per CPU used in a stack-like fashion. And then passing that entry into get_perf_callchain() (to avoid one extra copy).

Got it. It is more clearer, will change it in v3.

Quoted 2 lines
Quoted 2 lines

pw-bot: cr

--
Best Regards
Tao Chen

author
#9 lore raw

Re: [RFC PATCH bpf-next v2 2/2] bpf: Pass external callchain entry to get_perf_callchain

Quoted 53 lines · Andrii Nakryiko
Quoted 39 lines · Tao Chen
Quoted 24 lines · Andrii Nakryiko
Quoted 22 lines · Alexei Starovoitov
Quoted 20 lines · Jiri Olsa
Quoted 18 lines · Tao Chen

As Alexei noted, get_perf_callchain() return values may be reused if a task is preempted after the BPF program enters migrate disable mode. Drawing on the per-cpu design of bpf_perf_callchain_entries, stack-allocated memory of bpf_perf_callchain_entry is used here.

---
   kernel/bpf/stackmap.c | 19 +++++++++++--------
   1 file changed, 11 insertions(+), 8 deletions(-)
kernel/bpf/stackmap.c +0 -0
index 94e46b7f340..acd72c021c0 100644
@@ -31,6 +31,11 @@ struct bpf_stack_map {
        struct stack_map_bucket *buckets[] __counted_by(n_buckets);
   };
 
Quoted 8 lines
Quoted 5 lines
Quoted 5 lines

kernel/bpf/stackmap.c struct bpf_stack_map { · line 38

Quoted 5 lines
kernel/bpf/stackmap.c +5 -0
+struct bpf_perf_callchain_entry {
+     u64 nr;
+     u64 ip[PERF_MAX_STACK_DEPTH];
+};
+

we shouldn't introduce another type, there is perf_callchain_entry in linux/perf_event.h, what's the problem with using that?

perf_callchain_entry uses flexible array, DEFINE_PER_CPU seems do not create buffer for this, for ease of use, the size of the ip array has been explicitly defined.

Quoted 6 lines
struct perf_callchain_entry {
          u64                             nr;
          u64                             ip[]; /*

/proc/sys/kernel/perf_event_max_stack */
};

Ok, fair enough, but instead of casting between perf_callchain_entry and bpf_perf_callchain_entry, why not put perf_callchain_entry inside bpf_perf_callchain_entry as a first struct and pass a pointer to it. That looks a bit more appropriate? Though I'm not sure if compiler will complain about that flex array...

Quoted 5 lines

But on related note, I looked briefly at how perf gets those perf_callchain_entries, and it does seem like it also has a small stack of entries, so maybe we don't really need to invent anything here. See PERF_NR_CONTEXTS and how that's used.

It seems so. The implementation of get_callchain_entry and put_callchain_entry is similar to a simple stack management.

struct perf_callchain_entry *get_callchain_entry(int *rctx)

{
        int cpu;
        struct callchain_cpus_entries *entries;

        *rctx = get_recursion_context(this_cpu_ptr(callchain_recursion));
        if (*rctx == -1)
                return NULL;

        entries = rcu_dereference(callchain_cpus_entries);
        if (!entries) {
put_recursion_context(this_cpu_ptr(callchain_recursion), *rctx);
                return NULL;
        }

        cpu = smp_processor_id();

        return (((void *)entries->cpu_entries[cpu]) +
                (*rctx * perf_callchain_entry__sizeof()));
}

void
put_callchain_entry(int rctx)

{
        put_recursion_context(this_cpu_ptr(callchain_recursion), rctx);
}
Quoted 4 lines

If instead of disabling preemption we disable migration, then I think we should be good with relying on perf's callchain management, or am I missing something?

Peter proposed refactoring the get_perf_callchain interface in v3, i think after that we can still use perf callchain, but with the refactored API, it can prevent being overwritten by preemption.

BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map

{
	...
	entry = bpf_get_perf_callchain;
	__bpf_get_stackid(map, entry, flags);
	bpf_put_callchain_entry(entry);
	...
}

A simple specific implementation is as follows:

include/linux/perf_event.h +1 -0
index fd1d91017b9..1c7573f085f 100644
@@ -67,6 +67,7 @@ struct perf_callchain_entry_ctx {
 	u32				nr;
 	short				contexts;
 	bool				contexts_maxed;
+	bool				add_mark;
 };
 
 typedef unsigned long (*perf_copy_f)(void *dst, const void *src,
@@ -1718,9 +1719,18 @@ DECLARE_PER_CPU(struct perf_callchain_entry, perf_callchain_entry);
 
 extern void perf_callchain_user(struct perf_callchain_entry_ctx 

*entry, struct pt_regs *regs);
extern void perf_callchain_kernel(struct perf_callchain_entry_ctx
*entry, struct pt_regs *regs);
+
+extern void __init_perf_callchain_ctx(struct perf_callchain_entry_ctx *ctx,
+ struct perf_callchain_entry *entry,
+ u32 max_stack, bool add_mark);
+
+extern void __get_perf_callchain_kernel(struct perf_callchain_entry_ctx *ctx, struct pt_regs *regs); +extern void __get_perf_callchain_user(struct perf_callchain_entry_ctx *ctx, struct pt_regs *regs); +
+
extern struct perf_callchain_entry *
get_perf_callchain(struct pt_regs *regs, bool kernel, bool user,
- u32 max_stack, bool crosstask, bool add_mark);
+ u32 max_stack, bool crosstask);
extern int get_callchain_buffers(int max_stack);
extern void put_callchain_buffers(void);
extern struct perf_callchain_entry *get_callchain_entry(int *rctx);

kernel/bpf/stackmap.c +32 -8
index 4d53cdd1374..3f2543e5244 100644
@@ -297,14 +297,38 @@ static long __bpf_get_stackid(struct bpf_map *map,
 	return id;
 }
 
+static struct perf_callchain_entry *entry
+bpf_get_perf_callchain(int *rctx, struct pt_regs *regs, bool kernel, bool user, int max_stack)
+{
+	struct perf_callchain_entry_ctx ctx;
+	struct perf_callchain_entry *entry;
+
+	entry = get_callchain_entry(&rctx);
+	if (unlikely(!entry))
+		return NULL;
+
+	__init_perf_callchain_ctx(&ctx, entry)
+	if (kernel)
+		__get_perf_callchain_kernel(&ctx, regs);
+	if (user)
+		__get_perf_callchain_user(&ctx, regs);
+	return entry;
+}
+
+static bpf_put_perf_callchain(int rctx)
+{
+	put_callchain_entry(rctx);
+}
+
 BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
 	   u64, flags)
 {
 	u32 max_depth = map->value_size / stack_map_data_size(map);
 	u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
 	bool user = flags & BPF_F_USER_STACK;
-	struct perf_callchain_entry *trace;
+	struct perf_callchain_entry *entry;
 	bool kernel = !user;
+	int rctx, ret;
 
 	if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
 			       BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID)))
@@ -314,14 +338,14 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
 	if (max_depth > sysctl_perf_event_max_stack)
 		max_depth = sysctl_perf_event_max_stack;
 
-	trace = get_perf_callchain(regs, kernel, user, max_depth,
-				   false, false);
-
-	if (unlikely(!trace))
-		/* couldn't fetch the stack trace */
+	entry = bpf_get_perf_callchain(&rctx, regs, kernel, user, max_depth);
+	if (unlikely(!entry))
 		return -EFAULT;
 
-	return __bpf_get_stackid(map, trace, flags);
+	ret = __bpf_get_stackid(map, entry, flags);
+	bpf_put_callchain_entry(rctx);
+
+	return ret;
 }
 
 const struct bpf_func_proto bpf_get_stackid_proto = {
@@ -452,7 +476,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
 		trace = get_callchain_entry_for_task(task, max_depth);
 	else
 		trace = get_perf_callchain(regs, kernel, user, max_depth,
-					   crosstask, false);
+					   crosstask);
 
 	if (unlikely(!trace) || trace->nr < skip) {
 		if (may_fault)
kernel/events/callchain.c +48 -27

95 lines not on this page · raw message

kernel/events/core.c +1 -1
index 7541f6f85fc..eb0f110593d 100644
@@ -8218,7 +8218,7 @@ perf_callchain(struct perf_event *event, struct pt_regs *regs)
 		return &__empty_callchain;
 
 	callchain = get_perf_callchain(regs, kernel, user,
-				       max_stack, crosstask, true);
+				       max_stack, crosstask);
 	return callchain ?: &__empty_callchain;
 }
 
Quoted 39 lines
Quoted 39 lines
Quoted 13 lines
Quoted 13 lines
Quoted 13 lines

kernel/bpf/stackmap.c BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, · line 313

Quoted 9 lines
kernel/bpf/stackmap.c +1 -0
   static inline bool stack_map_use_build_id(struct bpf_map *map)
   {
        return (map->map_flags & BPF_F_STACK_BUILD_ID);
@@ -305,6 +310,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
        bool user = flags & BPF_F_USER_STACK;
        struct perf_callchain_entry *trace;
        bool kernel = !user;
+     struct bpf_perf_callchain_entry entry = { 0 };

so IIUC having entries on stack we do not need to do preempt_disable you had in the previous version, right?

Quoted 9 lines
Quoted 5 lines
Quoted 3 lines

I saw Andrii's justification to have this on the stack, I think it's fine, but does it have to be initialized? it seems that only used entries are copied to map

No. We're not adding 1k stack consumption.

Right, and I thought we concluded as much last time, so it's a bit surprising to see this in this patch.

Ok, I feel like I'm missing some context from our previous exchange.

Quoted 4 lines

Tao, you should go with 3 entries per CPU used in a stack-like fashion. And then passing that entry into get_perf_callchain() (to avoid one extra copy).

Got it. It is more clearer, will change it in v3.

Quoted 2 lines
Quoted 2 lines

pw-bot: cr

--
Best Regards
Tao Chen

Signature · 2 lines
Best Regards
Tao Chen
#10 lore raw

Re: [RFC PATCH bpf-next v2 2/2] bpf: Pass external callchain entry to get_perf_callchain

In reply to Tao Chen
kernel test robot noticed the following build warnings:

Hi Tao,

[This is a private test report for your RFC patch.] kernel test robot noticed the following build warnings:

[auto build test WARNING on bpf-next/master]

url:    https://github.com/intel-lab-lkp/linux/commits/Tao-Chen/perf-Use-extern-perf_callchain_entry-for-get_perf_callchain/20251014-181245
base:   https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git master
patch link:    https://lore.kernel.org/r/20251014100128.2721104-3-chen.dylane%40linux.dev
patch subject: [RFC PATCH bpf-next v2 2/2] bpf: Pass external callchain entry to get_perf_callchain
config: sh-allyesconfig (https://download.01.org/0day-ci/archive/20251016/[email protected]/config)
compiler: sh4-linux-gcc (GCC) 15.1.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20251016/[email protected]/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/

All warnings (new ones prefixed by >>):

In file included from kernel/bpf/stackmap.c:6:
kernel/bpf/stackmap.c: In function 'bpf_get_stackid':

>> include/linux/filter.h:618:9: warning: the frame size of 1036 bytes is larger than 1024 bytes [-Wframe-larger-than=]
     618 |         }                                                                      \
         |         ^
   include/linux/filter.h:626:33: note: in expansion of macro 'BPF_CALL_x'
     626 | #define BPF_CALL_3(name, ...)   BPF_CALL_x(3, __NOATTR, name, __VA_ARGS__)
         |                                 ^~~~~~~~~~
   kernel/bpf/stackmap.c:305:1: note: in expansion of macro 'BPF_CALL_3'
     305 | BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
         | ^~~~~~~~~~
   kernel/bpf/stackmap.c: In function '__bpf_get_stack':
>> kernel/bpf/stackmap.c:497:1: warning: the frame size of 1080 bytes is larger than 1024 bytes [-Wframe-larger-than=]
     497 | }
         | ^

vim +497 kernel/bpf/stackmap.c

d5a3b1f691865b Alexei Starovoitov 2016-02-17 304

7b04d6d60fcfb5 Song Liu           2020-07-23 @305  BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
7b04d6d60fcfb5 Song Liu           2020-07-23  306  	   u64, flags)
7b04d6d60fcfb5 Song Liu           2020-07-23  307  {
7b04d6d60fcfb5 Song Liu           2020-07-23  308  	u32 max_depth = map->value_size / stack_map_data_size(map);
ee2a098851bfbe Namhyung Kim       2022-03-14  309  	u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
7b04d6d60fcfb5 Song Liu           2020-07-23  310  	bool user = flags & BPF_F_USER_STACK;
7b04d6d60fcfb5 Song Liu           2020-07-23  311  	struct perf_callchain_entry *trace;
7b04d6d60fcfb5 Song Liu           2020-07-23  312  	bool kernel = !user;
8d90c803bbb115 Tao Chen           2025-10-14  313  	struct bpf_perf_callchain_entry entry = { 0 };
7b04d6d60fcfb5 Song Liu           2020-07-23  314  
7b04d6d60fcfb5 Song Liu           2020-07-23  315  	if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
7b04d6d60fcfb5 Song Liu           2020-07-23  316  			       BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID)))
7b04d6d60fcfb5 Song Liu           2020-07-23  317  		return -EINVAL;
7b04d6d60fcfb5 Song Liu           2020-07-23  318  
ee2a098851bfbe Namhyung Kim       2022-03-14  319  	max_depth += skip;
ee2a098851bfbe Namhyung Kim       2022-03-14  320  	if (max_depth > sysctl_perf_event_max_stack)
ee2a098851bfbe Namhyung Kim       2022-03-14  321  		max_depth = sysctl_perf_event_max_stack;
ee2a098851bfbe Namhyung Kim       2022-03-14  322  
8d90c803bbb115 Tao Chen           2025-10-14  323  	trace = get_perf_callchain(regs, (struct perf_callchain_entry *)&entry,
8d90c803bbb115 Tao Chen           2025-10-14  324  				   kernel, user, max_depth, false, false);
7b04d6d60fcfb5 Song Liu           2020-07-23  325  
7b04d6d60fcfb5 Song Liu           2020-07-23  326  	return __bpf_get_stackid(map, trace, flags);
7b04d6d60fcfb5 Song Liu           2020-07-23  327  }
7b04d6d60fcfb5 Song Liu           2020-07-23  328  
d5a3b1f691865b Alexei Starovoitov 2016-02-17  329  const struct bpf_func_proto bpf_get_stackid_proto = {
d5a3b1f691865b Alexei Starovoitov 2016-02-17  330  	.func		= bpf_get_stackid,
d5a3b1f691865b Alexei Starovoitov 2016-02-17  331  	.gpl_only	= true,
d5a3b1f691865b Alexei Starovoitov 2016-02-17  332  	.ret_type	= RET_INTEGER,
d5a3b1f691865b Alexei Starovoitov 2016-02-17  333  	.arg1_type	= ARG_PTR_TO_CTX,
d5a3b1f691865b Alexei Starovoitov 2016-02-17  334  	.arg2_type	= ARG_CONST_MAP_PTR,
d5a3b1f691865b Alexei Starovoitov 2016-02-17  335  	.arg3_type	= ARG_ANYTHING,
d5a3b1f691865b Alexei Starovoitov 2016-02-17  336  };
d5a3b1f691865b Alexei Starovoitov 2016-02-17  337  
7b04d6d60fcfb5 Song Liu           2020-07-23  338  static __u64 count_kernel_ip(struct perf_callchain_entry *trace)
7b04d6d60fcfb5 Song Liu           2020-07-23  339  {
7b04d6d60fcfb5 Song Liu           2020-07-23  340  	__u64 nr_kernel = 0;
7b04d6d60fcfb5 Song Liu           2020-07-23  341  
7b04d6d60fcfb5 Song Liu           2020-07-23  342  	while (nr_kernel < trace->nr) {
7b04d6d60fcfb5 Song Liu           2020-07-23  343  		if (trace->ip[nr_kernel] == PERF_CONTEXT_USER)
7b04d6d60fcfb5 Song Liu           2020-07-23  344  			break;
7b04d6d60fcfb5 Song Liu           2020-07-23  345  		nr_kernel++;
7b04d6d60fcfb5 Song Liu           2020-07-23  346  	}
7b04d6d60fcfb5 Song Liu           2020-07-23  347  	return nr_kernel;
7b04d6d60fcfb5 Song Liu           2020-07-23  348  }
7b04d6d60fcfb5 Song Liu           2020-07-23  349  
7b04d6d60fcfb5 Song Liu           2020-07-23  350  BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx,
7b04d6d60fcfb5 Song Liu           2020-07-23  351  	   struct bpf_map *, map, u64, flags)
7b04d6d60fcfb5 Song Liu           2020-07-23  352  {
7b04d6d60fcfb5 Song Liu           2020-07-23  353  	struct perf_event *event = ctx->event;
7b04d6d60fcfb5 Song Liu           2020-07-23  354  	struct perf_callchain_entry *trace;
7b04d6d60fcfb5 Song Liu           2020-07-23  355  	bool kernel, user;
7b04d6d60fcfb5 Song Liu           2020-07-23  356  	__u64 nr_kernel;
7b04d6d60fcfb5 Song Liu           2020-07-23  357  	int ret;
7b04d6d60fcfb5 Song Liu           2020-07-23  358  
7b04d6d60fcfb5 Song Liu           2020-07-23  359  	/* perf_sample_data doesn't have callchain, use bpf_get_stackid */
16817ad7e8b317 Namhyung Kim       2022-09-08  360  	if (!(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN))
7b04d6d60fcfb5 Song Liu           2020-07-23  361  		return bpf_get_stackid((unsigned long)(ctx->regs),
7b04d6d60fcfb5 Song Liu           2020-07-23  362  				       (unsigned long) map, flags, 0, 0);
7b04d6d60fcfb5 Song Liu           2020-07-23  363  
7b04d6d60fcfb5 Song Liu           2020-07-23  364  	if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
7b04d6d60fcfb5 Song Liu           2020-07-23  365  			       BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID)))
7b04d6d60fcfb5 Song Liu           2020-07-23  366  		return -EINVAL;
7b04d6d60fcfb5 Song Liu           2020-07-23  367  
7b04d6d60fcfb5 Song Liu           2020-07-23  368  	user = flags & BPF_F_USER_STACK;
7b04d6d60fcfb5 Song Liu           2020-07-23  369  	kernel = !user;
7b04d6d60fcfb5 Song Liu           2020-07-23  370  
7b04d6d60fcfb5 Song Liu           2020-07-23  371  	trace = ctx->data->callchain;
7b04d6d60fcfb5 Song Liu           2020-07-23  372  	if (unlikely(!trace))
7b04d6d60fcfb5 Song Liu           2020-07-23  373  		return -EFAULT;
7b04d6d60fcfb5 Song Liu           2020-07-23  374  
7b04d6d60fcfb5 Song Liu           2020-07-23  375  	nr_kernel = count_kernel_ip(trace);
7b04d6d60fcfb5 Song Liu           2020-07-23  376  
7b04d6d60fcfb5 Song Liu           2020-07-23  377  	if (kernel) {
7b04d6d60fcfb5 Song Liu           2020-07-23  378  		__u64 nr = trace->nr;
7b04d6d60fcfb5 Song Liu           2020-07-23  379  
7b04d6d60fcfb5 Song Liu           2020-07-23  380  		trace->nr = nr_kernel;
7b04d6d60fcfb5 Song Liu           2020-07-23  381  		ret = __bpf_get_stackid(map, trace, flags);
7b04d6d60fcfb5 Song Liu           2020-07-23  382  
7b04d6d60fcfb5 Song Liu           2020-07-23  383  		/* restore nr */
7b04d6d60fcfb5 Song Liu           2020-07-23  384  		trace->nr = nr;
7b04d6d60fcfb5 Song Liu           2020-07-23  385  	} else { /* user */
7b04d6d60fcfb5 Song Liu           2020-07-23  386  		u64 skip = flags & BPF_F_SKIP_FIELD_MASK;
7b04d6d60fcfb5 Song Liu           2020-07-23  387  
7b04d6d60fcfb5 Song Liu           2020-07-23  388  		skip += nr_kernel;
7b04d6d60fcfb5 Song Liu           2020-07-23  389  		if (skip > BPF_F_SKIP_FIELD_MASK)
7b04d6d60fcfb5 Song Liu           2020-07-23  390  			return -EFAULT;
7b04d6d60fcfb5 Song Liu           2020-07-23  391  
7b04d6d60fcfb5 Song Liu           2020-07-23  392  		flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip;
7b04d6d60fcfb5 Song Liu           2020-07-23  393  		ret = __bpf_get_stackid(map, trace, flags);
7b04d6d60fcfb5 Song Liu           2020-07-23  394  	}
7b04d6d60fcfb5 Song Liu           2020-07-23  395  	return ret;
7b04d6d60fcfb5 Song Liu           2020-07-23  396  }
7b04d6d60fcfb5 Song Liu           2020-07-23  397  
7b04d6d60fcfb5 Song Liu           2020-07-23  398  const struct bpf_func_proto bpf_get_stackid_proto_pe = {
7b04d6d60fcfb5 Song Liu           2020-07-23  399  	.func		= bpf_get_stackid_pe,
7b04d6d60fcfb5 Song Liu           2020-07-23  400  	.gpl_only	= false,
7b04d6d60fcfb5 Song Liu           2020-07-23  401  	.ret_type	= RET_INTEGER,
7b04d6d60fcfb5 Song Liu           2020-07-23  402  	.arg1_type	= ARG_PTR_TO_CTX,
7b04d6d60fcfb5 Song Liu           2020-07-23  403  	.arg2_type	= ARG_CONST_MAP_PTR,
7b04d6d60fcfb5 Song Liu           2020-07-23  404  	.arg3_type	= ARG_ANYTHING,
7b04d6d60fcfb5 Song Liu           2020-07-23  405  };
7b04d6d60fcfb5 Song Liu           2020-07-23  406  
fa28dcb82a38f8 Song Liu           2020-06-29  407  static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
7b04d6d60fcfb5 Song Liu           2020-07-23  408  			    struct perf_callchain_entry *trace_in,
d4dd9775ec2424 Andrii Nakryiko    2024-08-29  409  			    void *buf, u32 size, u64 flags, bool may_fault)
c195651e565ae7 Yonghong Song      2018-04-28  410  {
ee2a098851bfbe Namhyung Kim       2022-03-14  411  	u32 trace_nr, copy_len, elem_size, num_elem, max_depth;
c195651e565ae7 Yonghong Song      2018-04-28  412  	bool user_build_id = flags & BPF_F_USER_BUILD_ID;
b8e3a87a627b57 Jordan Rome        2023-11-08  413  	bool crosstask = task && task != current;
c195651e565ae7 Yonghong Song      2018-04-28  414  	u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
c195651e565ae7 Yonghong Song      2018-04-28  415  	bool user = flags & BPF_F_USER_STACK;
c195651e565ae7 Yonghong Song      2018-04-28  416  	struct perf_callchain_entry *trace;
8d90c803bbb115 Tao Chen           2025-10-14  417  	struct bpf_perf_callchain_entry entry = { 0 };
c195651e565ae7 Yonghong Song      2018-04-28  418  	bool kernel = !user;
c195651e565ae7 Yonghong Song      2018-04-28  419  	int err = -EINVAL;
c195651e565ae7 Yonghong Song      2018-04-28  420  	u64 *ips;
c195651e565ae7 Yonghong Song      2018-04-28  421  
c195651e565ae7 Yonghong Song      2018-04-28  422  	if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
c195651e565ae7 Yonghong Song      2018-04-28  423  			       BPF_F_USER_BUILD_ID)))
c195651e565ae7 Yonghong Song      2018-04-28  424  		goto clear;
c195651e565ae7 Yonghong Song      2018-04-28  425  	if (kernel && user_build_id)
c195651e565ae7 Yonghong Song      2018-04-28  426  		goto clear;
c195651e565ae7 Yonghong Song      2018-04-28  427  
d4dd9775ec2424 Andrii Nakryiko    2024-08-29  428  	elem_size = user_build_id ? sizeof(struct bpf_stack_build_id) : sizeof(u64);
c195651e565ae7 Yonghong Song      2018-04-28  429  	if (unlikely(size % elem_size))
c195651e565ae7 Yonghong Song      2018-04-28  430  		goto clear;
c195651e565ae7 Yonghong Song      2018-04-28  431  
fa28dcb82a38f8 Song Liu           2020-06-29  432  	/* cannot get valid user stack for task without user_mode regs */
fa28dcb82a38f8 Song Liu           2020-06-29  433  	if (task && user && !user_mode(regs))
fa28dcb82a38f8 Song Liu           2020-06-29  434  		goto err_fault;
fa28dcb82a38f8 Song Liu           2020-06-29  435  
b8e3a87a627b57 Jordan Rome        2023-11-08  436  	/* get_perf_callchain does not support crosstask user stack walking
b8e3a87a627b57 Jordan Rome        2023-11-08  437  	 * but returns an empty stack instead of NULL.
b8e3a87a627b57 Jordan Rome        2023-11-08  438  	 */
b8e3a87a627b57 Jordan Rome        2023-11-08  439  	if (crosstask && user) {
b8e3a87a627b57 Jordan Rome        2023-11-08  440  		err = -EOPNOTSUPP;
b8e3a87a627b57 Jordan Rome        2023-11-08  441  		goto clear;
b8e3a87a627b57 Jordan Rome        2023-11-08  442  	}
b8e3a87a627b57 Jordan Rome        2023-11-08  443  
c195651e565ae7 Yonghong Song      2018-04-28  444  	num_elem = size / elem_size;
ee2a098851bfbe Namhyung Kim       2022-03-14  445  	max_depth = num_elem + skip;
ee2a098851bfbe Namhyung Kim       2022-03-14  446  	if (sysctl_perf_event_max_stack < max_depth)
ee2a098851bfbe Namhyung Kim       2022-03-14  447  		max_depth = sysctl_perf_event_max_stack;
fa28dcb82a38f8 Song Liu           2020-06-29  448  
d4dd9775ec2424 Andrii Nakryiko    2024-08-29  449  	if (may_fault)
d4dd9775ec2424 Andrii Nakryiko    2024-08-29  450  		rcu_read_lock(); /* need RCU for perf's callchain below */
d4dd9775ec2424 Andrii Nakryiko    2024-08-29  451  
7b04d6d60fcfb5 Song Liu           2020-07-23  452  	if (trace_in)
7b04d6d60fcfb5 Song Liu           2020-07-23  453  		trace = trace_in;
7b04d6d60fcfb5 Song Liu           2020-07-23  454  	else if (kernel && task)
ee2a098851bfbe Namhyung Kim       2022-03-14  455  		trace = get_callchain_entry_for_task(task, max_depth);
fa28dcb82a38f8 Song Liu           2020-06-29  456  	else
8d90c803bbb115 Tao Chen           2025-10-14  457  		trace = get_perf_callchain(regs, (struct perf_callchain_entry *)&entry,
8d90c803bbb115 Tao Chen           2025-10-14  458  					   kernel, user, max_depth, crosstask, false);
c195651e565ae7 Yonghong Song      2018-04-28  459  
d4dd9775ec2424 Andrii Nakryiko    2024-08-29  460  	if (unlikely(!trace) || trace->nr < skip) {
d4dd9775ec2424 Andrii Nakryiko    2024-08-29  461  		if (may_fault)
d4dd9775ec2424 Andrii Nakryiko    2024-08-29  462  			rcu_read_unlock();
c195651e565ae7 Yonghong Song      2018-04-28  463  		goto err_fault;
d4dd9775ec2424 Andrii Nakryiko    2024-08-29  464  	}
c195651e565ae7 Yonghong Song      2018-04-28  465  
ee2a098851bfbe Namhyung Kim       2022-03-14  466  	trace_nr = trace->nr - skip;
c195651e565ae7 Yonghong Song      2018-04-28  467  	trace_nr = (trace_nr <= num_elem) ? trace_nr : num_elem;
c195651e565ae7 Yonghong Song      2018-04-28  468  	copy_len = trace_nr * elem_size;
ee2a098851bfbe Namhyung Kim       2022-03-14  469  
ee2a098851bfbe Namhyung Kim       2022-03-14  470  	ips = trace->ip + skip;
d4dd9775ec2424 Andrii Nakryiko    2024-08-29  471  	if (user_build_id) {
4f4c4fc0153fb1 Andrii Nakryiko    2024-08-29  472  		struct bpf_stack_build_id *id_offs = buf;
4f4c4fc0153fb1 Andrii Nakryiko    2024-08-29  473  		u32 i;
4f4c4fc0153fb1 Andrii Nakryiko    2024-08-29  474  
4f4c4fc0153fb1 Andrii Nakryiko    2024-08-29  475  		for (i = 0; i < trace_nr; i++)
4f4c4fc0153fb1 Andrii Nakryiko    2024-08-29  476  			id_offs[i].ip = ips[i];
4f4c4fc0153fb1 Andrii Nakryiko    2024-08-29  477  	} else {
c195651e565ae7 Yonghong Song      2018-04-28  478  		memcpy(buf, ips, copy_len);
4f4c4fc0153fb1 Andrii Nakryiko    2024-08-29  479  	}
c195651e565ae7 Yonghong Song      2018-04-28  480  
d4dd9775ec2424 Andrii Nakryiko    2024-08-29  481  	/* trace/ips should not be dereferenced after this point */
d4dd9775ec2424 Andrii Nakryiko    2024-08-29  482  	if (may_fault)
d4dd9775ec2424 Andrii Nakryiko    2024-08-29  483  		rcu_read_unlock();
d4dd9775ec2424 Andrii Nakryiko    2024-08-29  484  
d4dd9775ec2424 Andrii Nakryiko    2024-08-29  485  	if (user_build_id)
d4dd9775ec2424 Andrii Nakryiko    2024-08-29  486  		stack_map_get_build_id_offset(buf, trace_nr, user, may_fault);
d4dd9775ec2424 Andrii Nakryiko    2024-08-29  487  
c195651e565ae7 Yonghong Song      2018-04-28  488  	if (size > copy_len)
c195651e565ae7 Yonghong Song      2018-04-28  489  		memset(buf + copy_len, 0, size - copy_len);
c195651e565ae7 Yonghong Song      2018-04-28  490  	return copy_len;
c195651e565ae7 Yonghong Song      2018-04-28  491  
c195651e565ae7 Yonghong Song      2018-04-28  492  err_fault:
c195651e565ae7 Yonghong Song      2018-04-28  493  	err = -EFAULT;
c195651e565ae7 Yonghong Song      2018-04-28  494  clear:
c195651e565ae7 Yonghong Song      2018-04-28  495  	memset(buf, 0, size);
c195651e565ae7 Yonghong Song      2018-04-28  496  	return err;

c195651e565ae7 Yonghong Song 2018-04-28 @497 }
c195651e565ae7 Yonghong Song 2018-04-28 498

Signature · 2 lines
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
#11 lore raw

Re: [RFC PATCH bpf-next v2 2/2] bpf: Pass external callchain entry to get_perf_callchain

In reply to Tao Chen
kernel test robot noticed the following build warnings:

Hi Tao,

[This is a private test report for your RFC patch.] kernel test robot noticed the following build warnings:

[auto build test WARNING on bpf-next/master]

url:    https://github.com/intel-lab-lkp/linux/commits/Tao-Chen/perf-Use-extern-perf_callchain_entry-for-get_perf_callchain/20251014-181245
base:   https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git master
patch link:    https://lore.kernel.org/r/20251014100128.2721104-3-chen.dylane%40linux.dev
patch subject: [RFC PATCH bpf-next v2 2/2] bpf: Pass external callchain entry to get_perf_callchain
config: hexagon-allmodconfig (https://download.01.org/0day-ci/archive/20251017/[email protected]/config)
compiler: clang version 17.0.6 (https://github.com/llvm/llvm-project 6009708b4367171ccdbf4b5905cb6a803753fe18)
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20251017/[email protected]/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/

All warnings (new ones prefixed by >>):

>> kernel/bpf/stackmap.c:305:12: warning: stack frame size (1064) exceeds limit (1024) in 'bpf_get_stackid' [-Wframe-larger-than]
     305 | BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
         |            ^
>> kernel/bpf/stackmap.c:350:12: warning: stack frame size (1064) exceeds limit (1024) in 'bpf_get_stackid_pe' [-Wframe-larger-than]
     350 | BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx,
         |            ^
>> kernel/bpf/stackmap.c:407:13: warning: stack frame size (1120) exceeds limit (1024) in '__bpf_get_stack' [-Wframe-larger-than]

407 | static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
| ^
3 warnings generated.

vim +/bpf_get_stackid +305 kernel/bpf/stackmap.c

d5a3b1f691865be Alexei Starovoitov 2016-02-17 304

7b04d6d60fcfb5b Song Liu           2020-07-23 @305  BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
7b04d6d60fcfb5b Song Liu           2020-07-23  306  	   u64, flags)
7b04d6d60fcfb5b Song Liu           2020-07-23  307  {
7b04d6d60fcfb5b Song Liu           2020-07-23  308  	u32 max_depth = map->value_size / stack_map_data_size(map);
ee2a098851bfbe8 Namhyung Kim       2022-03-14  309  	u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
7b04d6d60fcfb5b Song Liu           2020-07-23  310  	bool user = flags & BPF_F_USER_STACK;
7b04d6d60fcfb5b Song Liu           2020-07-23  311  	struct perf_callchain_entry *trace;
7b04d6d60fcfb5b Song Liu           2020-07-23  312  	bool kernel = !user;
8d90c803bbb1151 Tao Chen           2025-10-14  313  	struct bpf_perf_callchain_entry entry = { 0 };
7b04d6d60fcfb5b Song Liu           2020-07-23  314  
7b04d6d60fcfb5b Song Liu           2020-07-23  315  	if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
7b04d6d60fcfb5b Song Liu           2020-07-23  316  			       BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID)))
7b04d6d60fcfb5b Song Liu           2020-07-23  317  		return -EINVAL;
7b04d6d60fcfb5b Song Liu           2020-07-23  318  
ee2a098851bfbe8 Namhyung Kim       2022-03-14  319  	max_depth += skip;
ee2a098851bfbe8 Namhyung Kim       2022-03-14  320  	if (max_depth > sysctl_perf_event_max_stack)
ee2a098851bfbe8 Namhyung Kim       2022-03-14  321  		max_depth = sysctl_perf_event_max_stack;
ee2a098851bfbe8 Namhyung Kim       2022-03-14  322  
8d90c803bbb1151 Tao Chen           2025-10-14  323  	trace = get_perf_callchain(regs, (struct perf_callchain_entry *)&entry,
8d90c803bbb1151 Tao Chen           2025-10-14  324  				   kernel, user, max_depth, false, false);
7b04d6d60fcfb5b Song Liu           2020-07-23  325  
7b04d6d60fcfb5b Song Liu           2020-07-23  326  	return __bpf_get_stackid(map, trace, flags);
7b04d6d60fcfb5b Song Liu           2020-07-23  327  }
7b04d6d60fcfb5b Song Liu           2020-07-23  328  
d5a3b1f691865be Alexei Starovoitov 2016-02-17  329  const struct bpf_func_proto bpf_get_stackid_proto = {
d5a3b1f691865be Alexei Starovoitov 2016-02-17  330  	.func		= bpf_get_stackid,
d5a3b1f691865be Alexei Starovoitov 2016-02-17  331  	.gpl_only	= true,
d5a3b1f691865be Alexei Starovoitov 2016-02-17  332  	.ret_type	= RET_INTEGER,
d5a3b1f691865be Alexei Starovoitov 2016-02-17  333  	.arg1_type	= ARG_PTR_TO_CTX,
d5a3b1f691865be Alexei Starovoitov 2016-02-17  334  	.arg2_type	= ARG_CONST_MAP_PTR,
d5a3b1f691865be Alexei Starovoitov 2016-02-17  335  	.arg3_type	= ARG_ANYTHING,
d5a3b1f691865be Alexei Starovoitov 2016-02-17  336  };
d5a3b1f691865be Alexei Starovoitov 2016-02-17  337  
7b04d6d60fcfb5b Song Liu           2020-07-23  338  static __u64 count_kernel_ip(struct perf_callchain_entry *trace)
7b04d6d60fcfb5b Song Liu           2020-07-23  339  {
7b04d6d60fcfb5b Song Liu           2020-07-23  340  	__u64 nr_kernel = 0;
7b04d6d60fcfb5b Song Liu           2020-07-23  341  
7b04d6d60fcfb5b Song Liu           2020-07-23  342  	while (nr_kernel < trace->nr) {
7b04d6d60fcfb5b Song Liu           2020-07-23  343  		if (trace->ip[nr_kernel] == PERF_CONTEXT_USER)
7b04d6d60fcfb5b Song Liu           2020-07-23  344  			break;
7b04d6d60fcfb5b Song Liu           2020-07-23  345  		nr_kernel++;
7b04d6d60fcfb5b Song Liu           2020-07-23  346  	}
7b04d6d60fcfb5b Song Liu           2020-07-23  347  	return nr_kernel;
7b04d6d60fcfb5b Song Liu           2020-07-23  348  }
7b04d6d60fcfb5b Song Liu           2020-07-23  349  
7b04d6d60fcfb5b Song Liu           2020-07-23 @350  BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx,
7b04d6d60fcfb5b Song Liu           2020-07-23  351  	   struct bpf_map *, map, u64, flags)
7b04d6d60fcfb5b Song Liu           2020-07-23  352  {
7b04d6d60fcfb5b Song Liu           2020-07-23  353  	struct perf_event *event = ctx->event;
7b04d6d60fcfb5b Song Liu           2020-07-23  354  	struct perf_callchain_entry *trace;
7b04d6d60fcfb5b Song Liu           2020-07-23  355  	bool kernel, user;
7b04d6d60fcfb5b Song Liu           2020-07-23  356  	__u64 nr_kernel;
7b04d6d60fcfb5b Song Liu           2020-07-23  357  	int ret;
7b04d6d60fcfb5b Song Liu           2020-07-23  358  
7b04d6d60fcfb5b Song Liu           2020-07-23  359  	/* perf_sample_data doesn't have callchain, use bpf_get_stackid */
16817ad7e8b3172 Namhyung Kim       2022-09-08  360  	if (!(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN))
7b04d6d60fcfb5b Song Liu           2020-07-23  361  		return bpf_get_stackid((unsigned long)(ctx->regs),
7b04d6d60fcfb5b Song Liu           2020-07-23  362  				       (unsigned long) map, flags, 0, 0);
7b04d6d60fcfb5b Song Liu           2020-07-23  363  
7b04d6d60fcfb5b Song Liu           2020-07-23  364  	if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
7b04d6d60fcfb5b Song Liu           2020-07-23  365  			       BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID)))
7b04d6d60fcfb5b Song Liu           2020-07-23  366  		return -EINVAL;
7b04d6d60fcfb5b Song Liu           2020-07-23  367  
7b04d6d60fcfb5b Song Liu           2020-07-23  368  	user = flags & BPF_F_USER_STACK;
7b04d6d60fcfb5b Song Liu           2020-07-23  369  	kernel = !user;
7b04d6d60fcfb5b Song Liu           2020-07-23  370  
7b04d6d60fcfb5b Song Liu           2020-07-23  371  	trace = ctx->data->callchain;
7b04d6d60fcfb5b Song Liu           2020-07-23  372  	if (unlikely(!trace))
7b04d6d60fcfb5b Song Liu           2020-07-23  373  		return -EFAULT;
7b04d6d60fcfb5b Song Liu           2020-07-23  374  
7b04d6d60fcfb5b Song Liu           2020-07-23  375  	nr_kernel = count_kernel_ip(trace);
7b04d6d60fcfb5b Song Liu           2020-07-23  376  
7b04d6d60fcfb5b Song Liu           2020-07-23  377  	if (kernel) {
7b04d6d60fcfb5b Song Liu           2020-07-23  378  		__u64 nr = trace->nr;
7b04d6d60fcfb5b Song Liu           2020-07-23  379  
7b04d6d60fcfb5b Song Liu           2020-07-23  380  		trace->nr = nr_kernel;
7b04d6d60fcfb5b Song Liu           2020-07-23  381  		ret = __bpf_get_stackid(map, trace, flags);
7b04d6d60fcfb5b Song Liu           2020-07-23  382  
7b04d6d60fcfb5b Song Liu           2020-07-23  383  		/* restore nr */
7b04d6d60fcfb5b Song Liu           2020-07-23  384  		trace->nr = nr;
7b04d6d60fcfb5b Song Liu           2020-07-23  385  	} else { /* user */
7b04d6d60fcfb5b Song Liu           2020-07-23  386  		u64 skip = flags & BPF_F_SKIP_FIELD_MASK;
7b04d6d60fcfb5b Song Liu           2020-07-23  387  
7b04d6d60fcfb5b Song Liu           2020-07-23  388  		skip += nr_kernel;
7b04d6d60fcfb5b Song Liu           2020-07-23  389  		if (skip > BPF_F_SKIP_FIELD_MASK)
7b04d6d60fcfb5b Song Liu           2020-07-23  390  			return -EFAULT;
7b04d6d60fcfb5b Song Liu           2020-07-23  391  
7b04d6d60fcfb5b Song Liu           2020-07-23  392  		flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip;
7b04d6d60fcfb5b Song Liu           2020-07-23  393  		ret = __bpf_get_stackid(map, trace, flags);
7b04d6d60fcfb5b Song Liu           2020-07-23  394  	}
7b04d6d60fcfb5b Song Liu           2020-07-23  395  	return ret;
7b04d6d60fcfb5b Song Liu           2020-07-23  396  }
7b04d6d60fcfb5b Song Liu           2020-07-23  397  
7b04d6d60fcfb5b Song Liu           2020-07-23  398  const struct bpf_func_proto bpf_get_stackid_proto_pe = {
7b04d6d60fcfb5b Song Liu           2020-07-23  399  	.func		= bpf_get_stackid_pe,
7b04d6d60fcfb5b Song Liu           2020-07-23  400  	.gpl_only	= false,
7b04d6d60fcfb5b Song Liu           2020-07-23  401  	.ret_type	= RET_INTEGER,
7b04d6d60fcfb5b Song Liu           2020-07-23  402  	.arg1_type	= ARG_PTR_TO_CTX,
7b04d6d60fcfb5b Song Liu           2020-07-23  403  	.arg2_type	= ARG_CONST_MAP_PTR,
7b04d6d60fcfb5b Song Liu           2020-07-23  404  	.arg3_type	= ARG_ANYTHING,
7b04d6d60fcfb5b Song Liu           2020-07-23  405  };
7b04d6d60fcfb5b Song Liu           2020-07-23  406  
fa28dcb82a38f8e Song Liu           2020-06-29 @407  static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
7b04d6d60fcfb5b Song Liu           2020-07-23  408  			    struct perf_callchain_entry *trace_in,
d4dd9775ec24242 Andrii Nakryiko    2024-08-29  409  			    void *buf, u32 size, u64 flags, bool may_fault)
c195651e565ae7f Yonghong Song      2018-04-28  410  {
ee2a098851bfbe8 Namhyung Kim       2022-03-14  411  	u32 trace_nr, copy_len, elem_size, num_elem, max_depth;
c195651e565ae7f Yonghong Song      2018-04-28  412  	bool user_build_id = flags & BPF_F_USER_BUILD_ID;
b8e3a87a627b575 Jordan Rome        2023-11-08  413  	bool crosstask = task && task != current;
c195651e565ae7f Yonghong Song      2018-04-28  414  	u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
c195651e565ae7f Yonghong Song      2018-04-28  415  	bool user = flags & BPF_F_USER_STACK;
c195651e565ae7f Yonghong Song      2018-04-28  416  	struct perf_callchain_entry *trace;
8d90c803bbb1151 Tao Chen           2025-10-14  417  	struct bpf_perf_callchain_entry entry = { 0 };
c195651e565ae7f Yonghong Song      2018-04-28  418  	bool kernel = !user;
c195651e565ae7f Yonghong Song      2018-04-28  419  	int err = -EINVAL;
c195651e565ae7f Yonghong Song      2018-04-28  420  	u64 *ips;
c195651e565ae7f Yonghong Song      2018-04-28  421  
c195651e565ae7f Yonghong Song      2018-04-28  422  	if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
c195651e565ae7f Yonghong Song      2018-04-28  423  			       BPF_F_USER_BUILD_ID)))
c195651e565ae7f Yonghong Song      2018-04-28  424  		goto clear;
c195651e565ae7f Yonghong Song      2018-04-28  425  	if (kernel && user_build_id)
c195651e565ae7f Yonghong Song      2018-04-28  426  		goto clear;
c195651e565ae7f Yonghong Song      2018-04-28  427  
d4dd9775ec24242 Andrii Nakryiko    2024-08-29  428  	elem_size = user_build_id ? sizeof(struct bpf_stack_build_id) : sizeof(u64);
c195651e565ae7f Yonghong Song      2018-04-28  429  	if (unlikely(size % elem_size))
c195651e565ae7f Yonghong Song      2018-04-28  430  		goto clear;
c195651e565ae7f Yonghong Song      2018-04-28  431  
fa28dcb82a38f8e Song Liu           2020-06-29  432  	/* cannot get valid user stack for task without user_mode regs */
fa28dcb82a38f8e Song Liu           2020-06-29  433  	if (task && user && !user_mode(regs))
fa28dcb82a38f8e Song Liu           2020-06-29  434  		goto err_fault;
fa28dcb82a38f8e Song Liu           2020-06-29  435  
b8e3a87a627b575 Jordan Rome        2023-11-08  436  	/* get_perf_callchain does not support crosstask user stack walking
b8e3a87a627b575 Jordan Rome        2023-11-08  437  	 * but returns an empty stack instead of NULL.
b8e3a87a627b575 Jordan Rome        2023-11-08  438  	 */
b8e3a87a627b575 Jordan Rome        2023-11-08  439  	if (crosstask && user) {
b8e3a87a627b575 Jordan Rome        2023-11-08  440  		err = -EOPNOTSUPP;
b8e3a87a627b575 Jordan Rome        2023-11-08  441  		goto clear;
b8e3a87a627b575 Jordan Rome        2023-11-08  442  	}
b8e3a87a627b575 Jordan Rome        2023-11-08  443  
c195651e565ae7f Yonghong Song      2018-04-28  444  	num_elem = size / elem_size;
ee2a098851bfbe8 Namhyung Kim       2022-03-14  445  	max_depth = num_elem + skip;
ee2a098851bfbe8 Namhyung Kim       2022-03-14  446  	if (sysctl_perf_event_max_stack < max_depth)
ee2a098851bfbe8 Namhyung Kim       2022-03-14  447  		max_depth = sysctl_perf_event_max_stack;
fa28dcb82a38f8e Song Liu           2020-06-29  448  
d4dd9775ec24242 Andrii Nakryiko    2024-08-29  449  	if (may_fault)
d4dd9775ec24242 Andrii Nakryiko    2024-08-29  450  		rcu_read_lock(); /* need RCU for perf's callchain below */
d4dd9775ec24242 Andrii Nakryiko    2024-08-29  451  
7b04d6d60fcfb5b Song Liu           2020-07-23  452  	if (trace_in)
7b04d6d60fcfb5b Song Liu           2020-07-23  453  		trace = trace_in;
7b04d6d60fcfb5b Song Liu           2020-07-23  454  	else if (kernel && task)
ee2a098851bfbe8 Namhyung Kim       2022-03-14  455  		trace = get_callchain_entry_for_task(task, max_depth);
fa28dcb82a38f8e Song Liu           2020-06-29  456  	else
8d90c803bbb1151 Tao Chen           2025-10-14  457  		trace = get_perf_callchain(regs, (struct perf_callchain_entry *)&entry,
8d90c803bbb1151 Tao Chen           2025-10-14  458  					   kernel, user, max_depth, crosstask, false);
c195651e565ae7f Yonghong Song      2018-04-28  459  
d4dd9775ec24242 Andrii Nakryiko    2024-08-29  460  	if (unlikely(!trace) || trace->nr < skip) {
d4dd9775ec24242 Andrii Nakryiko    2024-08-29  461  		if (may_fault)
d4dd9775ec24242 Andrii Nakryiko    2024-08-29  462  			rcu_read_unlock();
c195651e565ae7f Yonghong Song      2018-04-28  463  		goto err_fault;
d4dd9775ec24242 Andrii Nakryiko    2024-08-29  464  	}
c195651e565ae7f Yonghong Song      2018-04-28  465  
ee2a098851bfbe8 Namhyung Kim       2022-03-14  466  	trace_nr = trace->nr - skip;
c195651e565ae7f Yonghong Song      2018-04-28  467  	trace_nr = (trace_nr <= num_elem) ? trace_nr : num_elem;
c195651e565ae7f Yonghong Song      2018-04-28  468  	copy_len = trace_nr * elem_size;
ee2a098851bfbe8 Namhyung Kim       2022-03-14  469  
ee2a098851bfbe8 Namhyung Kim       2022-03-14  470  	ips = trace->ip + skip;
d4dd9775ec24242 Andrii Nakryiko    2024-08-29  471  	if (user_build_id) {
4f4c4fc0153fb11 Andrii Nakryiko    2024-08-29  472  		struct bpf_stack_build_id *id_offs = buf;
4f4c4fc0153fb11 Andrii Nakryiko    2024-08-29  473  		u32 i;
4f4c4fc0153fb11 Andrii Nakryiko    2024-08-29  474  
4f4c4fc0153fb11 Andrii Nakryiko    2024-08-29  475  		for (i = 0; i < trace_nr; i++)
4f4c4fc0153fb11 Andrii Nakryiko    2024-08-29  476  			id_offs[i].ip = ips[i];
4f4c4fc0153fb11 Andrii Nakryiko    2024-08-29  477  	} else {
c195651e565ae7f Yonghong Song      2018-04-28  478  		memcpy(buf, ips, copy_len);
4f4c4fc0153fb11 Andrii Nakryiko    2024-08-29  479  	}
c195651e565ae7f Yonghong Song      2018-04-28  480  
d4dd9775ec24242 Andrii Nakryiko    2024-08-29  481  	/* trace/ips should not be dereferenced after this point */
d4dd9775ec24242 Andrii Nakryiko    2024-08-29  482  	if (may_fault)
d4dd9775ec24242 Andrii Nakryiko    2024-08-29  483  		rcu_read_unlock();
d4dd9775ec24242 Andrii Nakryiko    2024-08-29  484  
d4dd9775ec24242 Andrii Nakryiko    2024-08-29  485  	if (user_build_id)
d4dd9775ec24242 Andrii Nakryiko    2024-08-29  486  		stack_map_get_build_id_offset(buf, trace_nr, user, may_fault);
d4dd9775ec24242 Andrii Nakryiko    2024-08-29  487  
c195651e565ae7f Yonghong Song      2018-04-28  488  	if (size > copy_len)
c195651e565ae7f Yonghong Song      2018-04-28  489  		memset(buf + copy_len, 0, size - copy_len);
c195651e565ae7f Yonghong Song      2018-04-28  490  	return copy_len;
c195651e565ae7f Yonghong Song      2018-04-28  491  
c195651e565ae7f Yonghong Song      2018-04-28  492  err_fault:
c195651e565ae7f Yonghong Song      2018-04-28  493  	err = -EFAULT;
c195651e565ae7f Yonghong Song      2018-04-28  494  clear:
c195651e565ae7f Yonghong Song      2018-04-28  495  	memset(buf, 0, size);
c195651e565ae7f Yonghong Song      2018-04-28  496  	return err;
c195651e565ae7f Yonghong Song      2018-04-28  497  }
c195651e565ae7f Yonghong Song      2018-04-28  498  
Signature · 2 lines
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki