Pass external callchain entry to get_perf_callchain
Subsystems (3)
v2 cover Pass external callchain entry to get_perf_callchain
Background
==========
Alexei noted we should use preempt_disable to protect get_perf_callchain in bpf stackmap.
https://lore.kernel.org/bpf/CAADnVQ+s8B7-fvR1TNO-bniSyKv57cH_ihRszmZV7pQDyV=VDQ@mail.gmail.com
A previous patch was submitted to attempt fixing this issue. And Andrii suggested teach get_perf_callchain to let us pass that buffer directly to avoid that unnecessary copy.
https://lore.kernel.org/bpf/[email protected]
Proposed Solution
=================
Add external perf_callchain_entry parameter for get_perf_callchain to allow us to use external buffer from BPF side. The biggest advantage is that it can reduce unnecessary copies.
Todo
====
If the above changes are reasonable, it seems that get_callchain_entry_for_task could also use an external perf_callchain_entry.
But I'm not sure if this modification is appropriate. After all, the implementation of get_callchain_entry in the perf subsystem seems much more complex than directly using an external buffer.
Comments and suggestions are always welcome.
Change list:
- v1 -> v2
From Jiri
- rebase code, fix confict
- v1: https://lore.kernel.org/bpf/[email protected]
Tao Chen (2):
perf: Use extern perf_callchain_entry for get_perf_callchain
bpf: Pass external callchain entry to get_perf_callchain
include/linux/perf_event.h | 4 ++-- kernel/bpf/stackmap.c | 19 +++++++++++-------- kernel/events/callchain.c | 13 +++++++++---- kernel/events/core.c | 2 +- 4 files changed, 23 insertions(+), 15 deletions(-)
Signature · 1 line
2.48.1
RFC PATCH v2 1/2 perf: Use extern perf_callchain_entry for get_perf_callchain
From bpf stack map, we want to use our own buffers to avoid unnecessary copy, so let us pass it directly. BPF will use this in the next patch.
--- include/linux/perf_event.h | 4 ++-- kernel/bpf/stackmap.c | 4 ++-- kernel/events/callchain.c | 13 +++++++++---- kernel/events/core.c | 2 +- 4 files changed, 14 insertions(+), 9 deletions(-)
include/linux/perf_event.h +2 -2
@@ -1719,8 +1719,8 @@ DECLARE_PER_CPU(struct perf_callchain_entry, perf_callchain_entry);extern void perf_callchain_user (struct perf_callchain_entry_ctx *entry,struct pt_regs *regs);extern void perf_callchain_kernel (struct perf_callchain_entry_ctx *entry,struct pt_regs *regs);extern struct perf_callchain_entry * -get_perf_callchain (struct pt_regs *regs,bool kernel,bool user, -u32 max_stack,bool crosstask,bool add_mark); +get_perf_callchain (struct pt_regs *regs,struct perf_callchain_entry *external_entry, +bool kernel,bool user,u32 max_stack,bool crosstask,bool add_mark);extern int get_callchain_buffers (int max_stack);extern void put_callchain_buffers (void );extern struct perf_callchain_entry *get_callchain_entry (int *rctx);
kernel/bpf/stackmap.c +2 -2
@@ -314,7 +314,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,if (max_depth > sysctl_perf_event_max_stack) max_depth = sysctl_perf_event_max_stack; - trace =get_perf_callchain (regs, kernel, user, max_depth, + trace =get_perf_callchain (regs,NULL , kernel, user, max_depth, false, false);if (unlikely (!trace))
@@ -451,7 +451,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,else if (kernel && task) trace =get_callchain_entry_for_task (task, max_depth);else - trace =get_perf_callchain (regs, kernel, user, max_depth, + trace =get_perf_callchain (regs,NULL , kernel, user, max_depth, crosstask, false);if (unlikely (!trace) || trace->nr < skip) {
kernel/events/callchain.c +9 -4
@@ -217,8 +217,8 @@ static void fixup_uretprobe_trampoline_entries(struct perf_callchain_entry *entr }struct perf_callchain_entry * -get_perf_callchain (struct pt_regs *regs,bool kernel,bool user, -u32 max_stack,bool crosstask,bool add_mark) +get_perf_callchain (struct pt_regs *regs,struct perf_callchain_entry *external_entry, +bool kernel,bool user,u32 max_stack,bool crosstask,bool add_mark) {struct perf_callchain_entry *entry;struct perf_callchain_entry_ctx ctx;
@@ -228,7 +228,11 @@ get_perf_callchain(struct pt_regs *regs, bool kernel, bool user,if (crosstask && user && !kernel)return NULL ; - entry =get_callchain_entry (&rctx); +if (external_entry) + entry = external_entry; +else + entry =get_callchain_entry (&rctx); +if (!entry)return NULL ;
@@ -260,7 +264,8 @@ get_perf_callchain(struct pt_regs *regs, bool kernel, bool user, } exit_put: -put_callchain_entry (rctx); +if (!external_entry) +put_callchain_entry (rctx);return entry; }
kernel/events/core.c +1 -1
@@ -8217,7 +8217,7 @@ perf_callchain(struct perf_event *event, struct pt_regs *regs)if (!kernel && !user)return &__empty_callchain; - callchain =get_perf_callchain (regs, kernel, user, + callchain =get_perf_callchain (regs,NULL , kernel, user, max_stack, crosstask, true);return callchain ?: &__empty_callchain; }
Signature · 1 line
2.48.1
RFC PATCH v2 2/2 bpf: Pass external callchain entry to get_perf_callchain
In reply to Tao ChenAs Alexei noted, get_perf_callchain() return values may be reused if a task is preempted after the BPF program enters migrate disable mode. Drawing on the per-cpu design of bpf_perf_callchain_entries, stack-allocated memory of bpf_perf_callchain_entry is used here.
--- kernel/bpf/stackmap.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-)
kernel/bpf/stackmap.c +11 -8
@@ -31,6 +31,11 @@ struct bpf_stack_map { struct stack_map_bucket *buckets[] __counted_by(n_buckets); }; +struct bpf_perf_callchain_entry { +u64 nr; +u64 ip[PERF_MAX_STACK_DEPTH ]; +}; +static inline bool stack_map_use_build_id (struct bpf_map *map) {return (map->map_flags &BPF_F_STACK_BUILD_ID );
@@ -305,6 +310,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,bool user = flags &BPF_F_USER_STACK ;struct perf_callchain_entry *trace;bool kernel = !user; +struct bpf_perf_callchain_entry entry = {0 };if (unlikely (flags & ~(BPF_F_SKIP_FIELD_MASK |BPF_F_USER_STACK |BPF_F_FAST_STACK_CMP |BPF_F_REUSE_STACKID )))
@@ -314,12 +320,8 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,if (max_depth > sysctl_perf_event_max_stack) max_depth = sysctl_perf_event_max_stack; - trace =get_perf_callchain (regs,NULL , kernel, user, max_depth, - false, false); - -if (unlikely (!trace)) -/* couldn't fetch the stack trace */ -return -EFAULT ; + trace =get_perf_callchain (regs, (struct perf_callchain_entry *)&entry, + kernel, user, max_depth, false, false);return __bpf_get_stackid (map, trace, flags); }
@@ -412,6 +414,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,u32 skip = flags &BPF_F_SKIP_FIELD_MASK ;bool user = flags &BPF_F_USER_STACK ;struct perf_callchain_entry *trace; +struct bpf_perf_callchain_entry entry = {0 };bool kernel = !user;int err = -EINVAL ;u64 *ips;
@@ -451,8 +454,8 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,else if (kernel && task) trace =get_callchain_entry_for_task (task, max_depth);else - trace =get_perf_callchain (regs,NULL , kernel, user, max_depth, - crosstask, false); + trace =get_perf_callchain (regs, (struct perf_callchain_entry *)&entry, + kernel, user, max_depth, crosstask, false);if (unlikely (!trace) || trace->nr < skip) {if (may_fault)
Signature · 1 line
2.48.1
Re: [RFC PATCH bpf-next v2 2/2] bpf: Pass external callchain entry to get_perf_callchain
Quoted 27 lines · Tao Chen
As Alexei noted,
get_perf_callchain()return values may be reused if a task is preempted after the BPF program enters migrate disable mode. Drawing on the per-cpu design ofbpf_perf_callchain_entries, stack-allocated memory ofbpf_perf_callchain_entryis used here.Signed-off-by Tao Chen <[email protected]>--- kernel/bpf/stackmap.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-)kernel/bpf/stackmap.c +5
-0@@ -31,6 +31,11 @@ struct bpf_stack_map { struct stack_map_bucket *buckets[] __counted_by(n_buckets); }; +struct bpf_perf_callchain_entry { +u64 nr; +u64 ip[PERF_MAX_STACK_DEPTH ]; +}; +static inline bool stack_map_use_build_id (struct bpf_map *map) {return (map->map_flags &BPF_F_STACK_BUILD_ID );
kernel/bpf/stackmap.c BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, · line 313
Quoted 4 lines
kernel/bpf/stackmap.c +1
-0@@ -305,6 +310,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,bool user = flags &BPF_F_USER_STACK ;struct perf_callchain_entry *trace;bool kernel = !user; +struct bpf_perf_callchain_entry entry = {0 };
so IIUC having entries on stack we do not need to do preempt_disable you had in the previous version, right?
I saw Andrii's justification to have this on the stack, I think it's fine, but does it have to be initialized? it seems that only used entries are copied to map
jirka
Quoted 40 lines
kernel/bpf/stackmap.c +5
-8if (unlikely (flags & ~(BPF_F_SKIP_FIELD_MASK |BPF_F_USER_STACK |BPF_F_FAST_STACK_CMP |BPF_F_REUSE_STACKID )))@@ -314,12 +320,8 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,if (max_depth > sysctl_perf_event_max_stack) max_depth = sysctl_perf_event_max_stack; - trace =get_perf_callchain (regs,NULL , kernel, user, max_depth, - false, false); - -if (unlikely (!trace)) -/* couldn't fetch the stack trace */ -return -EFAULT ; + trace =get_perf_callchain (regs, (struct perf_callchain_entry *)&entry, + kernel, user, max_depth, false, false);return __bpf_get_stackid (map, trace, flags); }@@ -412,6 +414,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,u32 skip = flags &BPF_F_SKIP_FIELD_MASK ;bool user = flags &BPF_F_USER_STACK ;struct perf_callchain_entry *trace; +struct bpf_perf_callchain_entry entry = {0 };bool kernel = !user;int err = -EINVAL ;u64 *ips;@@ -451,8 +454,8 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,else if (kernel && task) trace =get_callchain_entry_for_task (task, max_depth);else - trace =get_perf_callchain (regs,NULL , kernel, user, max_depth, - crosstask, false); + trace =get_perf_callchain (regs, (struct perf_callchain_entry *)&entry, + kernel, user, max_depth, crosstask, false);if (unlikely (!trace) || trace->nr < skip) {if (may_fault)Signature · 1 line
2.48.1
Re: [RFC PATCH bpf-next v2 2/2] bpf: Pass external callchain entry to get_perf_callchain
Quoted 4 lines · Jiri Olsa
Review
Quoted 1 line
(quoted diff) +1
-0+ struct bpf_perf_callchain_entry entry = { 0 };so IIUC having entries on stack we do not need to do
preempt_disableyou had in the previous version, right?
Yes, i think so, preempt_disable seems unnecessary.
Quoted 3 lines
I saw Andrii's justification to have this on the stack, I think it's fine, but does it have to be initialized? it seems that only used entries are copied to map
That makes sense. Removing it definitely looks better.
Signature · 2 lines
Best Regards Tao Chen
Re: [RFC PATCH bpf-next v2 2/2] bpf: Pass external callchain entry to get_perf_callchain
In reply to Jiri OlsaQuoted 37 lines · Jiri Olsa
Quoted 27 lines · Tao Chen
As Alexei noted,
get_perf_callchain()return values may be reused if a task is preempted after the BPF program enters migrate disable mode. Drawing on the per-cpu design ofbpf_perf_callchain_entries, stack-allocated memory ofbpf_perf_callchain_entryis used here.Signed-off-by Tao Chen <[email protected]>--- kernel/bpf/stackmap.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-)kernel/bpf/stackmap.c +5
-0@@ -31,6 +31,11 @@ struct bpf_stack_map { struct stack_map_bucket *buckets[] __counted_by(n_buckets); }; +struct bpf_perf_callchain_entry { +u64 nr; +u64 ip[PERF_MAX_STACK_DEPTH ]; +}; +static inline bool stack_map_use_build_id (struct bpf_map *map) {return (map->map_flags &BPF_F_STACK_BUILD_ID );kernel/bpf/stackmap.c BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, · line 313
Quoted 4 lines
kernel/bpf/stackmap.c +1
-0@@ -305,6 +310,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,bool user = flags &BPF_F_USER_STACK ;struct perf_callchain_entry *trace;bool kernel = !user; +struct bpf_perf_callchain_entry entry = {0 };so IIUC having entries on stack we do not need to do
preempt_disableyou had in the previous version, right?
Quoted 3 lines
I saw Andrii's justification to have this on the stack, I think it's fine, but does it have to be initialized? it seems that only used entries are copied to map
No. We're not adding 1k stack consumption.
pw-bot: cr
kernel test robot
build botRe: [RFC PATCH bpf-next v2 2/2] bpf: Pass external callchain entry to get_perf_callchain
In reply to Tao Chenkernel test robot noticed the following build warnings:
Hi Tao,
[This is a private test report for your RFC patch.] kernel test robot noticed the following build warnings:
[auto build test WARNING on bpf-next/master]
url: https://github.com/intel-lab-lkp/linux/commits/Tao-Chen/perf-Use-extern-perf_callchain_entry-for-get_perf_callchain/20251014-181245 base: https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git master patch link: https://lore.kernel.org/r/20251014100128.2721104-3-chen.dylane%40linux.dev patch subject: [RFC PATCH bpf-next v2 2/2] bpf: Pass external callchain entry to get_perf_callchain config: sh-allyesconfig (https://download.01.org/0day-ci/archive/20251016/[email protected]/config) compiler: sh4-linux-gcc (GCC) 15.1.0 reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20251016/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All warnings (new ones prefixed by >>):
In file included from kernel/bpf/stackmap.c:6:
kernel/bpf/stackmap.c: In function 'bpf_get_stackid':
>> include/linux/filter.h:618:9: warning: the frame size of 1036 bytes is larger than 1024 bytes [-Wframe-larger-than=]
618 | } \
| ^
include/linux/filter.h:626:33: note: in expansion of macro 'BPF_CALL_x'
626 | #define BPF_CALL_3(name, ...) BPF_CALL_x(3, __NOATTR, name, __VA_ARGS__)
| ^~~~~~~~~~
kernel/bpf/stackmap.c:305:1: note: in expansion of macro 'BPF_CALL_3'
305 | BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
| ^~~~~~~~~~
kernel/bpf/stackmap.c: In function '__bpf_get_stack':
>> kernel/bpf/stackmap.c:497:1: warning: the frame size of 1080 bytes is larger than 1024 bytes [-Wframe-larger-than=]
497 | }
| ^vim +497 kernel/bpf/stackmap.c
d5a3b1f691865b Alexei Starovoitov 2016-02-17 304
7b04d6d60fcfb5 Song Liu 2020-07-23 @305 BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
7b04d6d60fcfb5 Song Liu 2020-07-23 306 u64, flags)
7b04d6d60fcfb5 Song Liu 2020-07-23 307 {
7b04d6d60fcfb5 Song Liu 2020-07-23 308 u32 max_depth = map->value_size / stack_map_data_size(map);
ee2a098851bfbe Namhyung Kim 2022-03-14 309 u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
7b04d6d60fcfb5 Song Liu 2020-07-23 310 bool user = flags & BPF_F_USER_STACK;
7b04d6d60fcfb5 Song Liu 2020-07-23 311 struct perf_callchain_entry *trace;
7b04d6d60fcfb5 Song Liu 2020-07-23 312 bool kernel = !user;
8d90c803bbb115 Tao Chen 2025-10-14 313 struct bpf_perf_callchain_entry entry = { 0 };
7b04d6d60fcfb5 Song Liu 2020-07-23 314
7b04d6d60fcfb5 Song Liu 2020-07-23 315 if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
7b04d6d60fcfb5 Song Liu 2020-07-23 316 BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID)))
7b04d6d60fcfb5 Song Liu 2020-07-23 317 return -EINVAL;
7b04d6d60fcfb5 Song Liu 2020-07-23 318
ee2a098851bfbe Namhyung Kim 2022-03-14 319 max_depth += skip;
ee2a098851bfbe Namhyung Kim 2022-03-14 320 if (max_depth > sysctl_perf_event_max_stack)
ee2a098851bfbe Namhyung Kim 2022-03-14 321 max_depth = sysctl_perf_event_max_stack;
ee2a098851bfbe Namhyung Kim 2022-03-14 322
8d90c803bbb115 Tao Chen 2025-10-14 323 trace = get_perf_callchain(regs, (struct perf_callchain_entry *)&entry,
8d90c803bbb115 Tao Chen 2025-10-14 324 kernel, user, max_depth, false, false);
7b04d6d60fcfb5 Song Liu 2020-07-23 325
7b04d6d60fcfb5 Song Liu 2020-07-23 326 return __bpf_get_stackid(map, trace, flags);
7b04d6d60fcfb5 Song Liu 2020-07-23 327 }
7b04d6d60fcfb5 Song Liu 2020-07-23 328
d5a3b1f691865b Alexei Starovoitov 2016-02-17 329 const struct bpf_func_proto bpf_get_stackid_proto = {
d5a3b1f691865b Alexei Starovoitov 2016-02-17 330 .func = bpf_get_stackid,
d5a3b1f691865b Alexei Starovoitov 2016-02-17 331 .gpl_only = true,
d5a3b1f691865b Alexei Starovoitov 2016-02-17 332 .ret_type = RET_INTEGER,
d5a3b1f691865b Alexei Starovoitov 2016-02-17 333 .arg1_type = ARG_PTR_TO_CTX,
d5a3b1f691865b Alexei Starovoitov 2016-02-17 334 .arg2_type = ARG_CONST_MAP_PTR,
d5a3b1f691865b Alexei Starovoitov 2016-02-17 335 .arg3_type = ARG_ANYTHING,
d5a3b1f691865b Alexei Starovoitov 2016-02-17 336 };
d5a3b1f691865b Alexei Starovoitov 2016-02-17 337
7b04d6d60fcfb5 Song Liu 2020-07-23 338 static __u64 count_kernel_ip(struct perf_callchain_entry *trace)
7b04d6d60fcfb5 Song Liu 2020-07-23 339 {
7b04d6d60fcfb5 Song Liu 2020-07-23 340 __u64 nr_kernel = 0;
7b04d6d60fcfb5 Song Liu 2020-07-23 341
7b04d6d60fcfb5 Song Liu 2020-07-23 342 while (nr_kernel < trace->nr) {
7b04d6d60fcfb5 Song Liu 2020-07-23 343 if (trace->ip[nr_kernel] == PERF_CONTEXT_USER)
7b04d6d60fcfb5 Song Liu 2020-07-23 344 break;
7b04d6d60fcfb5 Song Liu 2020-07-23 345 nr_kernel++;
7b04d6d60fcfb5 Song Liu 2020-07-23 346 }
7b04d6d60fcfb5 Song Liu 2020-07-23 347 return nr_kernel;
7b04d6d60fcfb5 Song Liu 2020-07-23 348 }
7b04d6d60fcfb5 Song Liu 2020-07-23 349
7b04d6d60fcfb5 Song Liu 2020-07-23 350 BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx,
7b04d6d60fcfb5 Song Liu 2020-07-23 351 struct bpf_map *, map, u64, flags)
7b04d6d60fcfb5 Song Liu 2020-07-23 352 {
7b04d6d60fcfb5 Song Liu 2020-07-23 353 struct perf_event *event = ctx->event;
7b04d6d60fcfb5 Song Liu 2020-07-23 354 struct perf_callchain_entry *trace;
7b04d6d60fcfb5 Song Liu 2020-07-23 355 bool kernel, user;
7b04d6d60fcfb5 Song Liu 2020-07-23 356 __u64 nr_kernel;
7b04d6d60fcfb5 Song Liu 2020-07-23 357 int ret;
7b04d6d60fcfb5 Song Liu 2020-07-23 358
7b04d6d60fcfb5 Song Liu 2020-07-23 359 /* perf_sample_data doesn't have callchain, use bpf_get_stackid */
16817ad7e8b317 Namhyung Kim 2022-09-08 360 if (!(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN))
7b04d6d60fcfb5 Song Liu 2020-07-23 361 return bpf_get_stackid((unsigned long)(ctx->regs),
7b04d6d60fcfb5 Song Liu 2020-07-23 362 (unsigned long) map, flags, 0, 0);
7b04d6d60fcfb5 Song Liu 2020-07-23 363
7b04d6d60fcfb5 Song Liu 2020-07-23 364 if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
7b04d6d60fcfb5 Song Liu 2020-07-23 365 BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID)))
7b04d6d60fcfb5 Song Liu 2020-07-23 366 return -EINVAL;
7b04d6d60fcfb5 Song Liu 2020-07-23 367
7b04d6d60fcfb5 Song Liu 2020-07-23 368 user = flags & BPF_F_USER_STACK;
7b04d6d60fcfb5 Song Liu 2020-07-23 369 kernel = !user;
7b04d6d60fcfb5 Song Liu 2020-07-23 370
7b04d6d60fcfb5 Song Liu 2020-07-23 371 trace = ctx->data->callchain;
7b04d6d60fcfb5 Song Liu 2020-07-23 372 if (unlikely(!trace))
7b04d6d60fcfb5 Song Liu 2020-07-23 373 return -EFAULT;
7b04d6d60fcfb5 Song Liu 2020-07-23 374
7b04d6d60fcfb5 Song Liu 2020-07-23 375 nr_kernel = count_kernel_ip(trace);
7b04d6d60fcfb5 Song Liu 2020-07-23 376
7b04d6d60fcfb5 Song Liu 2020-07-23 377 if (kernel) {
7b04d6d60fcfb5 Song Liu 2020-07-23 378 __u64 nr = trace->nr;
7b04d6d60fcfb5 Song Liu 2020-07-23 379
7b04d6d60fcfb5 Song Liu 2020-07-23 380 trace->nr = nr_kernel;
7b04d6d60fcfb5 Song Liu 2020-07-23 381 ret = __bpf_get_stackid(map, trace, flags);
7b04d6d60fcfb5 Song Liu 2020-07-23 382
7b04d6d60fcfb5 Song Liu 2020-07-23 383 /* restore nr */
7b04d6d60fcfb5 Song Liu 2020-07-23 384 trace->nr = nr;
7b04d6d60fcfb5 Song Liu 2020-07-23 385 } else { /* user */
7b04d6d60fcfb5 Song Liu 2020-07-23 386 u64 skip = flags & BPF_F_SKIP_FIELD_MASK;
7b04d6d60fcfb5 Song Liu 2020-07-23 387
7b04d6d60fcfb5 Song Liu 2020-07-23 388 skip += nr_kernel;
7b04d6d60fcfb5 Song Liu 2020-07-23 389 if (skip > BPF_F_SKIP_FIELD_MASK)
7b04d6d60fcfb5 Song Liu 2020-07-23 390 return -EFAULT;
7b04d6d60fcfb5 Song Liu 2020-07-23 391
7b04d6d60fcfb5 Song Liu 2020-07-23 392 flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip;
7b04d6d60fcfb5 Song Liu 2020-07-23 393 ret = __bpf_get_stackid(map, trace, flags);
7b04d6d60fcfb5 Song Liu 2020-07-23 394 }
7b04d6d60fcfb5 Song Liu 2020-07-23 395 return ret;
7b04d6d60fcfb5 Song Liu 2020-07-23 396 }
7b04d6d60fcfb5 Song Liu 2020-07-23 397
7b04d6d60fcfb5 Song Liu 2020-07-23 398 const struct bpf_func_proto bpf_get_stackid_proto_pe = {
7b04d6d60fcfb5 Song Liu 2020-07-23 399 .func = bpf_get_stackid_pe,
7b04d6d60fcfb5 Song Liu 2020-07-23 400 .gpl_only = false,
7b04d6d60fcfb5 Song Liu 2020-07-23 401 .ret_type = RET_INTEGER,
7b04d6d60fcfb5 Song Liu 2020-07-23 402 .arg1_type = ARG_PTR_TO_CTX,
7b04d6d60fcfb5 Song Liu 2020-07-23 403 .arg2_type = ARG_CONST_MAP_PTR,
7b04d6d60fcfb5 Song Liu 2020-07-23 404 .arg3_type = ARG_ANYTHING,
7b04d6d60fcfb5 Song Liu 2020-07-23 405 };
7b04d6d60fcfb5 Song Liu 2020-07-23 406
fa28dcb82a38f8 Song Liu 2020-06-29 407 static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
7b04d6d60fcfb5 Song Liu 2020-07-23 408 struct perf_callchain_entry *trace_in,
d4dd9775ec2424 Andrii Nakryiko 2024-08-29 409 void *buf, u32 size, u64 flags, bool may_fault)
c195651e565ae7 Yonghong Song 2018-04-28 410 {
ee2a098851bfbe Namhyung Kim 2022-03-14 411 u32 trace_nr, copy_len, elem_size, num_elem, max_depth;
c195651e565ae7 Yonghong Song 2018-04-28 412 bool user_build_id = flags & BPF_F_USER_BUILD_ID;
b8e3a87a627b57 Jordan Rome 2023-11-08 413 bool crosstask = task && task != current;
c195651e565ae7 Yonghong Song 2018-04-28 414 u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
c195651e565ae7 Yonghong Song 2018-04-28 415 bool user = flags & BPF_F_USER_STACK;
c195651e565ae7 Yonghong Song 2018-04-28 416 struct perf_callchain_entry *trace;
8d90c803bbb115 Tao Chen 2025-10-14 417 struct bpf_perf_callchain_entry entry = { 0 };
c195651e565ae7 Yonghong Song 2018-04-28 418 bool kernel = !user;
c195651e565ae7 Yonghong Song 2018-04-28 419 int err = -EINVAL;
c195651e565ae7 Yonghong Song 2018-04-28 420 u64 *ips;
c195651e565ae7 Yonghong Song 2018-04-28 421
c195651e565ae7 Yonghong Song 2018-04-28 422 if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
c195651e565ae7 Yonghong Song 2018-04-28 423 BPF_F_USER_BUILD_ID)))
c195651e565ae7 Yonghong Song 2018-04-28 424 goto clear;
c195651e565ae7 Yonghong Song 2018-04-28 425 if (kernel && user_build_id)
c195651e565ae7 Yonghong Song 2018-04-28 426 goto clear;
c195651e565ae7 Yonghong Song 2018-04-28 427
d4dd9775ec2424 Andrii Nakryiko 2024-08-29 428 elem_size = user_build_id ? sizeof(struct bpf_stack_build_id) : sizeof(u64);
c195651e565ae7 Yonghong Song 2018-04-28 429 if (unlikely(size % elem_size))
c195651e565ae7 Yonghong Song 2018-04-28 430 goto clear;
c195651e565ae7 Yonghong Song 2018-04-28 431
fa28dcb82a38f8 Song Liu 2020-06-29 432 /* cannot get valid user stack for task without user_mode regs */
fa28dcb82a38f8 Song Liu 2020-06-29 433 if (task && user && !user_mode(regs))
fa28dcb82a38f8 Song Liu 2020-06-29 434 goto err_fault;
fa28dcb82a38f8 Song Liu 2020-06-29 435
b8e3a87a627b57 Jordan Rome 2023-11-08 436 /* get_perf_callchain does not support crosstask user stack walking
b8e3a87a627b57 Jordan Rome 2023-11-08 437 * but returns an empty stack instead of NULL.
b8e3a87a627b57 Jordan Rome 2023-11-08 438 */
b8e3a87a627b57 Jordan Rome 2023-11-08 439 if (crosstask && user) {
b8e3a87a627b57 Jordan Rome 2023-11-08 440 err = -EOPNOTSUPP;
b8e3a87a627b57 Jordan Rome 2023-11-08 441 goto clear;
b8e3a87a627b57 Jordan Rome 2023-11-08 442 }
b8e3a87a627b57 Jordan Rome 2023-11-08 443
c195651e565ae7 Yonghong Song 2018-04-28 444 num_elem = size / elem_size;
ee2a098851bfbe Namhyung Kim 2022-03-14 445 max_depth = num_elem + skip;
ee2a098851bfbe Namhyung Kim 2022-03-14 446 if (sysctl_perf_event_max_stack < max_depth)
ee2a098851bfbe Namhyung Kim 2022-03-14 447 max_depth = sysctl_perf_event_max_stack;
fa28dcb82a38f8 Song Liu 2020-06-29 448
d4dd9775ec2424 Andrii Nakryiko 2024-08-29 449 if (may_fault)
d4dd9775ec2424 Andrii Nakryiko 2024-08-29 450 rcu_read_lock(); /* need RCU for perf's callchain below */
d4dd9775ec2424 Andrii Nakryiko 2024-08-29 451
7b04d6d60fcfb5 Song Liu 2020-07-23 452 if (trace_in)
7b04d6d60fcfb5 Song Liu 2020-07-23 453 trace = trace_in;
7b04d6d60fcfb5 Song Liu 2020-07-23 454 else if (kernel && task)
ee2a098851bfbe Namhyung Kim 2022-03-14 455 trace = get_callchain_entry_for_task(task, max_depth);
fa28dcb82a38f8 Song Liu 2020-06-29 456 else
8d90c803bbb115 Tao Chen 2025-10-14 457 trace = get_perf_callchain(regs, (struct perf_callchain_entry *)&entry,
8d90c803bbb115 Tao Chen 2025-10-14 458 kernel, user, max_depth, crosstask, false);
c195651e565ae7 Yonghong Song 2018-04-28 459
d4dd9775ec2424 Andrii Nakryiko 2024-08-29 460 if (unlikely(!trace) || trace->nr < skip) {
d4dd9775ec2424 Andrii Nakryiko 2024-08-29 461 if (may_fault)
d4dd9775ec2424 Andrii Nakryiko 2024-08-29 462 rcu_read_unlock();
c195651e565ae7 Yonghong Song 2018-04-28 463 goto err_fault;
d4dd9775ec2424 Andrii Nakryiko 2024-08-29 464 }
c195651e565ae7 Yonghong Song 2018-04-28 465
ee2a098851bfbe Namhyung Kim 2022-03-14 466 trace_nr = trace->nr - skip;
c195651e565ae7 Yonghong Song 2018-04-28 467 trace_nr = (trace_nr <= num_elem) ? trace_nr : num_elem;
c195651e565ae7 Yonghong Song 2018-04-28 468 copy_len = trace_nr * elem_size;
ee2a098851bfbe Namhyung Kim 2022-03-14 469
ee2a098851bfbe Namhyung Kim 2022-03-14 470 ips = trace->ip + skip;
d4dd9775ec2424 Andrii Nakryiko 2024-08-29 471 if (user_build_id) {
4f4c4fc0153fb1 Andrii Nakryiko 2024-08-29 472 struct bpf_stack_build_id *id_offs = buf;
4f4c4fc0153fb1 Andrii Nakryiko 2024-08-29 473 u32 i;
4f4c4fc0153fb1 Andrii Nakryiko 2024-08-29 474
4f4c4fc0153fb1 Andrii Nakryiko 2024-08-29 475 for (i = 0; i < trace_nr; i++)
4f4c4fc0153fb1 Andrii Nakryiko 2024-08-29 476 id_offs[i].ip = ips[i];
4f4c4fc0153fb1 Andrii Nakryiko 2024-08-29 477 } else {
c195651e565ae7 Yonghong Song 2018-04-28 478 memcpy(buf, ips, copy_len);
4f4c4fc0153fb1 Andrii Nakryiko 2024-08-29 479 }
c195651e565ae7 Yonghong Song 2018-04-28 480
d4dd9775ec2424 Andrii Nakryiko 2024-08-29 481 /* trace/ips should not be dereferenced after this point */
d4dd9775ec2424 Andrii Nakryiko 2024-08-29 482 if (may_fault)
d4dd9775ec2424 Andrii Nakryiko 2024-08-29 483 rcu_read_unlock();
d4dd9775ec2424 Andrii Nakryiko 2024-08-29 484
d4dd9775ec2424 Andrii Nakryiko 2024-08-29 485 if (user_build_id)
d4dd9775ec2424 Andrii Nakryiko 2024-08-29 486 stack_map_get_build_id_offset(buf, trace_nr, user, may_fault);
d4dd9775ec2424 Andrii Nakryiko 2024-08-29 487
c195651e565ae7 Yonghong Song 2018-04-28 488 if (size > copy_len)
c195651e565ae7 Yonghong Song 2018-04-28 489 memset(buf + copy_len, 0, size - copy_len);
c195651e565ae7 Yonghong Song 2018-04-28 490 return copy_len;
c195651e565ae7 Yonghong Song 2018-04-28 491
c195651e565ae7 Yonghong Song 2018-04-28 492 err_fault:
c195651e565ae7 Yonghong Song 2018-04-28 493 err = -EFAULT;
c195651e565ae7 Yonghong Song 2018-04-28 494 clear:
c195651e565ae7 Yonghong Song 2018-04-28 495 memset(buf, 0, size);
c195651e565ae7 Yonghong Song 2018-04-28 496 return err;c195651e565ae7 Yonghong Song 2018-04-28 @497 }c195651e565ae7 Yonghong Song 2018-04-28 498
Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki
kernel test robot
build botRe: [RFC PATCH bpf-next v2 2/2] bpf: Pass external callchain entry to get_perf_callchain
In reply to Tao Chenkernel test robot noticed the following build warnings:
Hi Tao,
[This is a private test report for your RFC patch.] kernel test robot noticed the following build warnings:
[auto build test WARNING on bpf-next/master]
url: https://github.com/intel-lab-lkp/linux/commits/Tao-Chen/perf-Use-extern-perf_callchain_entry-for-get_perf_callchain/20251014-181245 base: https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git master patch link: https://lore.kernel.org/r/20251014100128.2721104-3-chen.dylane%40linux.dev patch subject: [RFC PATCH bpf-next v2 2/2] bpf: Pass external callchain entry to get_perf_callchain config: hexagon-allmodconfig (https://download.01.org/0day-ci/archive/20251017/[email protected]/config) compiler: clang version 17.0.6 (https://github.com/llvm/llvm-project 6009708b4367171ccdbf4b5905cb6a803753fe18) reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20251017/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All warnings (new ones prefixed by >>):
>> kernel/bpf/stackmap.c:305:12: warning: stack frame size (1064) exceeds limit (1024) in 'bpf_get_stackid' [-Wframe-larger-than]
305 | BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
| ^
>> kernel/bpf/stackmap.c:350:12: warning: stack frame size (1064) exceeds limit (1024) in 'bpf_get_stackid_pe' [-Wframe-larger-than]
350 | BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx,
| ^
>> kernel/bpf/stackmap.c:407:13: warning: stack frame size (1120) exceeds limit (1024) in '__bpf_get_stack' [-Wframe-larger-than] 407 | static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
| ^
3 warnings generated.
vim +/bpf_get_stackid +305 kernel/bpf/stackmap.c
d5a3b1f691865be Alexei Starovoitov 2016-02-17 304
7b04d6d60fcfb5b Song Liu 2020-07-23 @305 BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
7b04d6d60fcfb5b Song Liu 2020-07-23 306 u64, flags)
7b04d6d60fcfb5b Song Liu 2020-07-23 307 {
7b04d6d60fcfb5b Song Liu 2020-07-23 308 u32 max_depth = map->value_size / stack_map_data_size(map);
ee2a098851bfbe8 Namhyung Kim 2022-03-14 309 u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
7b04d6d60fcfb5b Song Liu 2020-07-23 310 bool user = flags & BPF_F_USER_STACK;
7b04d6d60fcfb5b Song Liu 2020-07-23 311 struct perf_callchain_entry *trace;
7b04d6d60fcfb5b Song Liu 2020-07-23 312 bool kernel = !user;
8d90c803bbb1151 Tao Chen 2025-10-14 313 struct bpf_perf_callchain_entry entry = { 0 };
7b04d6d60fcfb5b Song Liu 2020-07-23 314
7b04d6d60fcfb5b Song Liu 2020-07-23 315 if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
7b04d6d60fcfb5b Song Liu 2020-07-23 316 BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID)))
7b04d6d60fcfb5b Song Liu 2020-07-23 317 return -EINVAL;
7b04d6d60fcfb5b Song Liu 2020-07-23 318
ee2a098851bfbe8 Namhyung Kim 2022-03-14 319 max_depth += skip;
ee2a098851bfbe8 Namhyung Kim 2022-03-14 320 if (max_depth > sysctl_perf_event_max_stack)
ee2a098851bfbe8 Namhyung Kim 2022-03-14 321 max_depth = sysctl_perf_event_max_stack;
ee2a098851bfbe8 Namhyung Kim 2022-03-14 322
8d90c803bbb1151 Tao Chen 2025-10-14 323 trace = get_perf_callchain(regs, (struct perf_callchain_entry *)&entry,
8d90c803bbb1151 Tao Chen 2025-10-14 324 kernel, user, max_depth, false, false);
7b04d6d60fcfb5b Song Liu 2020-07-23 325
7b04d6d60fcfb5b Song Liu 2020-07-23 326 return __bpf_get_stackid(map, trace, flags);
7b04d6d60fcfb5b Song Liu 2020-07-23 327 }
7b04d6d60fcfb5b Song Liu 2020-07-23 328
d5a3b1f691865be Alexei Starovoitov 2016-02-17 329 const struct bpf_func_proto bpf_get_stackid_proto = {
d5a3b1f691865be Alexei Starovoitov 2016-02-17 330 .func = bpf_get_stackid,
d5a3b1f691865be Alexei Starovoitov 2016-02-17 331 .gpl_only = true,
d5a3b1f691865be Alexei Starovoitov 2016-02-17 332 .ret_type = RET_INTEGER,
d5a3b1f691865be Alexei Starovoitov 2016-02-17 333 .arg1_type = ARG_PTR_TO_CTX,
d5a3b1f691865be Alexei Starovoitov 2016-02-17 334 .arg2_type = ARG_CONST_MAP_PTR,
d5a3b1f691865be Alexei Starovoitov 2016-02-17 335 .arg3_type = ARG_ANYTHING,
d5a3b1f691865be Alexei Starovoitov 2016-02-17 336 };
d5a3b1f691865be Alexei Starovoitov 2016-02-17 337
7b04d6d60fcfb5b Song Liu 2020-07-23 338 static __u64 count_kernel_ip(struct perf_callchain_entry *trace)
7b04d6d60fcfb5b Song Liu 2020-07-23 339 {
7b04d6d60fcfb5b Song Liu 2020-07-23 340 __u64 nr_kernel = 0;
7b04d6d60fcfb5b Song Liu 2020-07-23 341
7b04d6d60fcfb5b Song Liu 2020-07-23 342 while (nr_kernel < trace->nr) {
7b04d6d60fcfb5b Song Liu 2020-07-23 343 if (trace->ip[nr_kernel] == PERF_CONTEXT_USER)
7b04d6d60fcfb5b Song Liu 2020-07-23 344 break;
7b04d6d60fcfb5b Song Liu 2020-07-23 345 nr_kernel++;
7b04d6d60fcfb5b Song Liu 2020-07-23 346 }
7b04d6d60fcfb5b Song Liu 2020-07-23 347 return nr_kernel;
7b04d6d60fcfb5b Song Liu 2020-07-23 348 }
7b04d6d60fcfb5b Song Liu 2020-07-23 349
7b04d6d60fcfb5b Song Liu 2020-07-23 @350 BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx,
7b04d6d60fcfb5b Song Liu 2020-07-23 351 struct bpf_map *, map, u64, flags)
7b04d6d60fcfb5b Song Liu 2020-07-23 352 {
7b04d6d60fcfb5b Song Liu 2020-07-23 353 struct perf_event *event = ctx->event;
7b04d6d60fcfb5b Song Liu 2020-07-23 354 struct perf_callchain_entry *trace;
7b04d6d60fcfb5b Song Liu 2020-07-23 355 bool kernel, user;
7b04d6d60fcfb5b Song Liu 2020-07-23 356 __u64 nr_kernel;
7b04d6d60fcfb5b Song Liu 2020-07-23 357 int ret;
7b04d6d60fcfb5b Song Liu 2020-07-23 358
7b04d6d60fcfb5b Song Liu 2020-07-23 359 /* perf_sample_data doesn't have callchain, use bpf_get_stackid */
16817ad7e8b3172 Namhyung Kim 2022-09-08 360 if (!(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN))
7b04d6d60fcfb5b Song Liu 2020-07-23 361 return bpf_get_stackid((unsigned long)(ctx->regs),
7b04d6d60fcfb5b Song Liu 2020-07-23 362 (unsigned long) map, flags, 0, 0);
7b04d6d60fcfb5b Song Liu 2020-07-23 363
7b04d6d60fcfb5b Song Liu 2020-07-23 364 if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
7b04d6d60fcfb5b Song Liu 2020-07-23 365 BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID)))
7b04d6d60fcfb5b Song Liu 2020-07-23 366 return -EINVAL;
7b04d6d60fcfb5b Song Liu 2020-07-23 367
7b04d6d60fcfb5b Song Liu 2020-07-23 368 user = flags & BPF_F_USER_STACK;
7b04d6d60fcfb5b Song Liu 2020-07-23 369 kernel = !user;
7b04d6d60fcfb5b Song Liu 2020-07-23 370
7b04d6d60fcfb5b Song Liu 2020-07-23 371 trace = ctx->data->callchain;
7b04d6d60fcfb5b Song Liu 2020-07-23 372 if (unlikely(!trace))
7b04d6d60fcfb5b Song Liu 2020-07-23 373 return -EFAULT;
7b04d6d60fcfb5b Song Liu 2020-07-23 374
7b04d6d60fcfb5b Song Liu 2020-07-23 375 nr_kernel = count_kernel_ip(trace);
7b04d6d60fcfb5b Song Liu 2020-07-23 376
7b04d6d60fcfb5b Song Liu 2020-07-23 377 if (kernel) {
7b04d6d60fcfb5b Song Liu 2020-07-23 378 __u64 nr = trace->nr;
7b04d6d60fcfb5b Song Liu 2020-07-23 379
7b04d6d60fcfb5b Song Liu 2020-07-23 380 trace->nr = nr_kernel;
7b04d6d60fcfb5b Song Liu 2020-07-23 381 ret = __bpf_get_stackid(map, trace, flags);
7b04d6d60fcfb5b Song Liu 2020-07-23 382
7b04d6d60fcfb5b Song Liu 2020-07-23 383 /* restore nr */
7b04d6d60fcfb5b Song Liu 2020-07-23 384 trace->nr = nr;
7b04d6d60fcfb5b Song Liu 2020-07-23 385 } else { /* user */
7b04d6d60fcfb5b Song Liu 2020-07-23 386 u64 skip = flags & BPF_F_SKIP_FIELD_MASK;
7b04d6d60fcfb5b Song Liu 2020-07-23 387
7b04d6d60fcfb5b Song Liu 2020-07-23 388 skip += nr_kernel;
7b04d6d60fcfb5b Song Liu 2020-07-23 389 if (skip > BPF_F_SKIP_FIELD_MASK)
7b04d6d60fcfb5b Song Liu 2020-07-23 390 return -EFAULT;
7b04d6d60fcfb5b Song Liu 2020-07-23 391
7b04d6d60fcfb5b Song Liu 2020-07-23 392 flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip;
7b04d6d60fcfb5b Song Liu 2020-07-23 393 ret = __bpf_get_stackid(map, trace, flags);
7b04d6d60fcfb5b Song Liu 2020-07-23 394 }
7b04d6d60fcfb5b Song Liu 2020-07-23 395 return ret;
7b04d6d60fcfb5b Song Liu 2020-07-23 396 }
7b04d6d60fcfb5b Song Liu 2020-07-23 397
7b04d6d60fcfb5b Song Liu 2020-07-23 398 const struct bpf_func_proto bpf_get_stackid_proto_pe = {
7b04d6d60fcfb5b Song Liu 2020-07-23 399 .func = bpf_get_stackid_pe,
7b04d6d60fcfb5b Song Liu 2020-07-23 400 .gpl_only = false,
7b04d6d60fcfb5b Song Liu 2020-07-23 401 .ret_type = RET_INTEGER,
7b04d6d60fcfb5b Song Liu 2020-07-23 402 .arg1_type = ARG_PTR_TO_CTX,
7b04d6d60fcfb5b Song Liu 2020-07-23 403 .arg2_type = ARG_CONST_MAP_PTR,
7b04d6d60fcfb5b Song Liu 2020-07-23 404 .arg3_type = ARG_ANYTHING,
7b04d6d60fcfb5b Song Liu 2020-07-23 405 };
7b04d6d60fcfb5b Song Liu 2020-07-23 406
fa28dcb82a38f8e Song Liu 2020-06-29 @407 static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
7b04d6d60fcfb5b Song Liu 2020-07-23 408 struct perf_callchain_entry *trace_in,
d4dd9775ec24242 Andrii Nakryiko 2024-08-29 409 void *buf, u32 size, u64 flags, bool may_fault)
c195651e565ae7f Yonghong Song 2018-04-28 410 {
ee2a098851bfbe8 Namhyung Kim 2022-03-14 411 u32 trace_nr, copy_len, elem_size, num_elem, max_depth;
c195651e565ae7f Yonghong Song 2018-04-28 412 bool user_build_id = flags & BPF_F_USER_BUILD_ID;
b8e3a87a627b575 Jordan Rome 2023-11-08 413 bool crosstask = task && task != current;
c195651e565ae7f Yonghong Song 2018-04-28 414 u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
c195651e565ae7f Yonghong Song 2018-04-28 415 bool user = flags & BPF_F_USER_STACK;
c195651e565ae7f Yonghong Song 2018-04-28 416 struct perf_callchain_entry *trace;
8d90c803bbb1151 Tao Chen 2025-10-14 417 struct bpf_perf_callchain_entry entry = { 0 };
c195651e565ae7f Yonghong Song 2018-04-28 418 bool kernel = !user;
c195651e565ae7f Yonghong Song 2018-04-28 419 int err = -EINVAL;
c195651e565ae7f Yonghong Song 2018-04-28 420 u64 *ips;
c195651e565ae7f Yonghong Song 2018-04-28 421
c195651e565ae7f Yonghong Song 2018-04-28 422 if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
c195651e565ae7f Yonghong Song 2018-04-28 423 BPF_F_USER_BUILD_ID)))
c195651e565ae7f Yonghong Song 2018-04-28 424 goto clear;
c195651e565ae7f Yonghong Song 2018-04-28 425 if (kernel && user_build_id)
c195651e565ae7f Yonghong Song 2018-04-28 426 goto clear;
c195651e565ae7f Yonghong Song 2018-04-28 427
d4dd9775ec24242 Andrii Nakryiko 2024-08-29 428 elem_size = user_build_id ? sizeof(struct bpf_stack_build_id) : sizeof(u64);
c195651e565ae7f Yonghong Song 2018-04-28 429 if (unlikely(size % elem_size))
c195651e565ae7f Yonghong Song 2018-04-28 430 goto clear;
c195651e565ae7f Yonghong Song 2018-04-28 431
fa28dcb82a38f8e Song Liu 2020-06-29 432 /* cannot get valid user stack for task without user_mode regs */
fa28dcb82a38f8e Song Liu 2020-06-29 433 if (task && user && !user_mode(regs))
fa28dcb82a38f8e Song Liu 2020-06-29 434 goto err_fault;
fa28dcb82a38f8e Song Liu 2020-06-29 435
b8e3a87a627b575 Jordan Rome 2023-11-08 436 /* get_perf_callchain does not support crosstask user stack walking
b8e3a87a627b575 Jordan Rome 2023-11-08 437 * but returns an empty stack instead of NULL.
b8e3a87a627b575 Jordan Rome 2023-11-08 438 */
b8e3a87a627b575 Jordan Rome 2023-11-08 439 if (crosstask && user) {
b8e3a87a627b575 Jordan Rome 2023-11-08 440 err = -EOPNOTSUPP;
b8e3a87a627b575 Jordan Rome 2023-11-08 441 goto clear;
b8e3a87a627b575 Jordan Rome 2023-11-08 442 }
b8e3a87a627b575 Jordan Rome 2023-11-08 443
c195651e565ae7f Yonghong Song 2018-04-28 444 num_elem = size / elem_size;
ee2a098851bfbe8 Namhyung Kim 2022-03-14 445 max_depth = num_elem + skip;
ee2a098851bfbe8 Namhyung Kim 2022-03-14 446 if (sysctl_perf_event_max_stack < max_depth)
ee2a098851bfbe8 Namhyung Kim 2022-03-14 447 max_depth = sysctl_perf_event_max_stack;
fa28dcb82a38f8e Song Liu 2020-06-29 448
d4dd9775ec24242 Andrii Nakryiko 2024-08-29 449 if (may_fault)
d4dd9775ec24242 Andrii Nakryiko 2024-08-29 450 rcu_read_lock(); /* need RCU for perf's callchain below */
d4dd9775ec24242 Andrii Nakryiko 2024-08-29 451
7b04d6d60fcfb5b Song Liu 2020-07-23 452 if (trace_in)
7b04d6d60fcfb5b Song Liu 2020-07-23 453 trace = trace_in;
7b04d6d60fcfb5b Song Liu 2020-07-23 454 else if (kernel && task)
ee2a098851bfbe8 Namhyung Kim 2022-03-14 455 trace = get_callchain_entry_for_task(task, max_depth);
fa28dcb82a38f8e Song Liu 2020-06-29 456 else
8d90c803bbb1151 Tao Chen 2025-10-14 457 trace = get_perf_callchain(regs, (struct perf_callchain_entry *)&entry,
8d90c803bbb1151 Tao Chen 2025-10-14 458 kernel, user, max_depth, crosstask, false);
c195651e565ae7f Yonghong Song 2018-04-28 459
d4dd9775ec24242 Andrii Nakryiko 2024-08-29 460 if (unlikely(!trace) || trace->nr < skip) {
d4dd9775ec24242 Andrii Nakryiko 2024-08-29 461 if (may_fault)
d4dd9775ec24242 Andrii Nakryiko 2024-08-29 462 rcu_read_unlock();
c195651e565ae7f Yonghong Song 2018-04-28 463 goto err_fault;
d4dd9775ec24242 Andrii Nakryiko 2024-08-29 464 }
c195651e565ae7f Yonghong Song 2018-04-28 465
ee2a098851bfbe8 Namhyung Kim 2022-03-14 466 trace_nr = trace->nr - skip;
c195651e565ae7f Yonghong Song 2018-04-28 467 trace_nr = (trace_nr <= num_elem) ? trace_nr : num_elem;
c195651e565ae7f Yonghong Song 2018-04-28 468 copy_len = trace_nr * elem_size;
ee2a098851bfbe8 Namhyung Kim 2022-03-14 469
ee2a098851bfbe8 Namhyung Kim 2022-03-14 470 ips = trace->ip + skip;
d4dd9775ec24242 Andrii Nakryiko 2024-08-29 471 if (user_build_id) {
4f4c4fc0153fb11 Andrii Nakryiko 2024-08-29 472 struct bpf_stack_build_id *id_offs = buf;
4f4c4fc0153fb11 Andrii Nakryiko 2024-08-29 473 u32 i;
4f4c4fc0153fb11 Andrii Nakryiko 2024-08-29 474
4f4c4fc0153fb11 Andrii Nakryiko 2024-08-29 475 for (i = 0; i < trace_nr; i++)
4f4c4fc0153fb11 Andrii Nakryiko 2024-08-29 476 id_offs[i].ip = ips[i];
4f4c4fc0153fb11 Andrii Nakryiko 2024-08-29 477 } else {
c195651e565ae7f Yonghong Song 2018-04-28 478 memcpy(buf, ips, copy_len);
4f4c4fc0153fb11 Andrii Nakryiko 2024-08-29 479 }
c195651e565ae7f Yonghong Song 2018-04-28 480
d4dd9775ec24242 Andrii Nakryiko 2024-08-29 481 /* trace/ips should not be dereferenced after this point */
d4dd9775ec24242 Andrii Nakryiko 2024-08-29 482 if (may_fault)
d4dd9775ec24242 Andrii Nakryiko 2024-08-29 483 rcu_read_unlock();
d4dd9775ec24242 Andrii Nakryiko 2024-08-29 484
d4dd9775ec24242 Andrii Nakryiko 2024-08-29 485 if (user_build_id)
d4dd9775ec24242 Andrii Nakryiko 2024-08-29 486 stack_map_get_build_id_offset(buf, trace_nr, user, may_fault);
d4dd9775ec24242 Andrii Nakryiko 2024-08-29 487
c195651e565ae7f Yonghong Song 2018-04-28 488 if (size > copy_len)
c195651e565ae7f Yonghong Song 2018-04-28 489 memset(buf + copy_len, 0, size - copy_len);
c195651e565ae7f Yonghong Song 2018-04-28 490 return copy_len;
c195651e565ae7f Yonghong Song 2018-04-28 491
c195651e565ae7f Yonghong Song 2018-04-28 492 err_fault:
c195651e565ae7f Yonghong Song 2018-04-28 493 err = -EFAULT;
c195651e565ae7f Yonghong Song 2018-04-28 494 clear:
c195651e565ae7f Yonghong Song 2018-04-28 495 memset(buf, 0, size);
c195651e565ae7f Yonghong Song 2018-04-28 496 return err;
c195651e565ae7f Yonghong Song 2018-04-28 497 }
c195651e565ae7f Yonghong Song 2018-04-28 498 Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki
Re: [RFC PATCH bpf-next v2 2/2] bpf: Pass external callchain entry to get_perf_callchain
In reply to Alexei StarovoitovQuoted 22 lines · Alexei Starovoitov
Quoted 20 lines · Jiri Olsa
Quoted 18 lines · Tao Chen
As Alexei noted,
get_perf_callchain()return values may be reused if a task is preempted after the BPF program enters migrate disable mode. Drawing on the per-cpu design ofbpf_perf_callchain_entries, stack-allocated memory ofbpf_perf_callchain_entryis used here.Signed-off-by Tao Chen <[email protected]>--- kernel/bpf/stackmap.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-)kernel/bpf/stackmap.c +0
-0@@ -31,6 +31,11 @@ struct bpf_stack_map { struct stack_map_bucket *buckets[] __counted_by(n_buckets); };
Quoted 5 lines
Quoted 5 lines
kernel/bpf/stackmap.c struct bpf_stack_map { · line 38
Quoted 5 lines
kernel/bpf/stackmap.c +5
-0+struct bpf_perf_callchain_entry { +u64 nr; +u64 ip[PERF_MAX_STACK_DEPTH ]; +}; +
we shouldn't introduce another type, there is perf_callchain_entry in linux/perf_event.h, what's the problem with using that?
Quoted 12 lines
Quoted 12 lines
kernel/bpf/stackmap.c BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, · line 313
Quoted 8 lines
kernel/bpf/stackmap.c +1
-0static inline bool stack_map_use_build_id (struct bpf_map *map) {return (map->map_flags &BPF_F_STACK_BUILD_ID );@@ -305,6 +310,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,bool user = flags &BPF_F_USER_STACK ;struct perf_callchain_entry *trace;bool kernel = !user; +struct bpf_perf_callchain_entry entry = {0 };so IIUC having entries on stack we do not need to do
preempt_disableyou had in the previous version, right?
Quoted 5 lines
Quoted 3 lines
I saw Andrii's justification to have this on the stack, I think it's fine, but does it have to be initialized? it seems that only used entries are copied to map
No. We're not adding 1k stack consumption.
Right, and I thought we concluded as much last time, so it's a bit surprising to see this in this patch.
Tao, you should go with 3 entries per CPU used in a stack-like fashion. And then passing that entry into get_perf_callchain() (to avoid one extra copy).
Quoted 2 lines
pw-bot: cr
Re: [RFC PATCH bpf-next v2 2/2] bpf: Pass external callchain entry to get_perf_callchain
Quoted 24 lines · Andrii Nakryiko
Quoted 22 lines · Alexei Starovoitov
Quoted 20 lines · Jiri Olsa
Quoted 18 lines · Tao Chen
As Alexei noted,
get_perf_callchain()return values may be reused if a task is preempted after the BPF program enters migrate disable mode. Drawing on the per-cpu design ofbpf_perf_callchain_entries, stack-allocated memory ofbpf_perf_callchain_entryis used here.Signed-off-by Tao Chen <[email protected]>--- kernel/bpf/stackmap.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-)kernel/bpf/stackmap.c +0
-0@@ -31,6 +31,11 @@ struct bpf_stack_map { struct stack_map_bucket *buckets[] __counted_by(n_buckets); };
Quoted 8 lines
Quoted 5 lines
Quoted 5 lines
kernel/bpf/stackmap.c struct bpf_stack_map { · line 38
Quoted 5 lines
kernel/bpf/stackmap.c +5
-0+struct bpf_perf_callchain_entry { +u64 nr; +u64 ip[PERF_MAX_STACK_DEPTH ]; +}; +we shouldn't introduce another type, there is
perf_callchain_entryinlinux/perf_event.h, what's the problem with using that?
perf_callchain_entry uses flexible array, DEFINE_PER_CPU seems do not
create buffer for this, for ease of use, the size of the ip array has been explicitly defined.
struct perf_callchain_entry {
u64 nr;
u64 ip[]; /* /proc/sys/kernel/perf_event_max_stack */
};
Quoted 13 lines
Quoted 13 lines
Quoted 13 lines
kernel/bpf/stackmap.c BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, · line 313
Quoted 9 lines
kernel/bpf/stackmap.c +1
-0static inline bool stack_map_use_build_id (struct bpf_map *map) {return (map->map_flags &BPF_F_STACK_BUILD_ID );@@ -305,6 +310,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,bool user = flags &BPF_F_USER_STACK ;struct perf_callchain_entry *trace;bool kernel = !user; +struct bpf_perf_callchain_entry entry = {0 };so IIUC having entries on stack we do not need to do
preempt_disableyou had in the previous version, right?
Quoted 9 lines
Quoted 5 lines
Quoted 3 lines
I saw Andrii's justification to have this on the stack, I think it's fine, but does it have to be initialized? it seems that only used entries are copied to map
No. We're not adding 1k stack consumption.
Right, and I thought we concluded as much last time, so it's a bit surprising to see this in this patch.
Ok, I feel like I'm missing some context from our previous exchange.
Quoted 4 lines
Tao, you should go with 3 entries per CPU used in a stack-like fashion. And then passing that entry into
get_perf_callchain()(to avoid one extra copy).
Got it. It is more clearer, will change it in v3.
Quoted 2 lines
Quoted 2 lines
pw-bot: cr
Signature · 2 lines
Best Regards Tao Chen
Re: [RFC PATCH bpf-next v2 2/2] bpf: Pass external callchain entry to get_perf_callchain
Quoted 39 lines · Tao Chen
Quoted 24 lines · Andrii Nakryiko
Quoted 22 lines · Alexei Starovoitov
Quoted 20 lines · Jiri Olsa
Quoted 18 lines · Tao Chen
As Alexei noted,
get_perf_callchain()return values may be reused if a task is preempted after the BPF program enters migrate disable mode. Drawing on the per-cpu design ofbpf_perf_callchain_entries, stack-allocated memory ofbpf_perf_callchain_entryis used here.Signed-off-by Tao Chen <[email protected]>--- kernel/bpf/stackmap.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-)kernel/bpf/stackmap.c +0
-0@@ -31,6 +31,11 @@ struct bpf_stack_map { struct stack_map_bucket *buckets[] __counted_by(n_buckets); };Quoted 8 lines
Quoted 5 lines
Quoted 5 lines
kernel/bpf/stackmap.c struct bpf_stack_map { · line 38
Quoted 5 lines
kernel/bpf/stackmap.c +5
-0+struct bpf_perf_callchain_entry { +u64 nr; +u64 ip[PERF_MAX_STACK_DEPTH ]; +}; +we shouldn't introduce another type, there is
perf_callchain_entryinlinux/perf_event.h, what's the problem with using that?
perf_callchain_entryuses flexible array,DEFINE_PER_CPUseems do not create buffer for this, for ease of use, the size of the ip array has been explicitly defined.
Quoted 6 lines
struct perf_callchain_entry { u64 nr; u64 ip[]; /*
/proc/sys/kernel/perf_event_max_stack*/
};
Ok, fair enough, but instead of casting between perf_callchain_entry and bpf_perf_callchain_entry, why not put perf_callchain_entry inside bpf_perf_callchain_entry as a first struct and pass a pointer to it. That looks a bit more appropriate? Though I'm not sure if compiler will complain about that flex array...
But on related note, I looked briefly at how perf gets those perf_callchain_entries, and it does seem like it also has a small stack of entries, so maybe we don't really need to invent anything here. See PERF_NR_CONTEXTS and how that's used.
If instead of disabling preemption we disable migration, then I think we should be good with relying on perf's callchain management, or am I missing something?
Quoted 39 lines
Quoted 13 lines
Quoted 13 lines
Quoted 13 lines
kernel/bpf/stackmap.c BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, · line 313
Quoted 9 lines
kernel/bpf/stackmap.c +1
-0static inline bool stack_map_use_build_id (struct bpf_map *map) {return (map->map_flags &BPF_F_STACK_BUILD_ID );@@ -305,6 +310,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,bool user = flags &BPF_F_USER_STACK ;struct perf_callchain_entry *trace;bool kernel = !user; +struct bpf_perf_callchain_entry entry = {0 };so IIUC having entries on stack we do not need to do
preempt_disableyou had in the previous version, right?Quoted 9 lines
Quoted 5 lines
Quoted 3 lines
I saw Andrii's justification to have this on the stack, I think it's fine, but does it have to be initialized? it seems that only used entries are copied to map
No. We're not adding 1k stack consumption.
Right, and I thought we concluded as much last time, so it's a bit surprising to see this in this patch.
Ok, I feel like I'm missing some context from our previous exchange.
Quoted 4 lines
Tao, you should go with 3 entries per CPU used in a stack-like fashion. And then passing that entry into
get_perf_callchain()(to avoid one extra copy).Got it. It is more clearer, will change it in v3.
Quoted 2 lines
Quoted 2 lines
pw-bot: cr
--
Best Regards
Tao Chen
Re: [RFC PATCH bpf-next v2 2/2] bpf: Pass external callchain entry to get_perf_callchain
Quoted 53 lines · Andrii Nakryiko
Quoted 39 lines · Tao Chen
Quoted 24 lines · Andrii Nakryiko
Quoted 22 lines · Alexei Starovoitov
Quoted 20 lines · Jiri Olsa
Quoted 18 lines · Tao Chen
As Alexei noted,
get_perf_callchain()return values may be reused if a task is preempted after the BPF program enters migrate disable mode. Drawing on the per-cpu design ofbpf_perf_callchain_entries, stack-allocated memory ofbpf_perf_callchain_entryis used here.Signed-off-by Tao Chen <[email protected]>--- kernel/bpf/stackmap.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-)kernel/bpf/stackmap.c +0
-0@@ -31,6 +31,11 @@ struct bpf_stack_map { struct stack_map_bucket *buckets[] __counted_by(n_buckets); };Quoted 8 lines
Quoted 5 lines
Quoted 5 lines
kernel/bpf/stackmap.c struct bpf_stack_map { · line 38
Quoted 5 lines
kernel/bpf/stackmap.c +5
-0+struct bpf_perf_callchain_entry { +u64 nr; +u64 ip[PERF_MAX_STACK_DEPTH ]; +}; +we shouldn't introduce another type, there is
perf_callchain_entryinlinux/perf_event.h, what's the problem with using that?
perf_callchain_entryuses flexible array,DEFINE_PER_CPUseems do not create buffer for this, for ease of use, the size of the ip array has been explicitly defined.Quoted 6 lines
struct perf_callchain_entry { u64 nr; u64 ip[]; /*
/proc/sys/kernel/perf_event_max_stack*/
};Ok, fair enough, but instead of casting between
perf_callchain_entryandbpf_perf_callchain_entry, why not putperf_callchain_entryinsidebpf_perf_callchain_entryas a first struct and pass a pointer to it. That looks a bit more appropriate? Though I'm not sure if compiler will complain about that flex array...
Quoted 5 lines
But on related note, I looked briefly at how perf gets those
perf_callchain_entries, and it does seem like it also has a small stack of entries, so maybe we don't really need to invent anything here. SeePERF_NR_CONTEXTSand how that's used.
It seems so. The implementation of get_callchain_entry and put_callchain_entry is similar to a simple stack management.
struct perf_callchain_entry *get_callchain_entry(int *rctx)
{
int cpu;
struct callchain_cpus_entries *entries;
*rctx = get_recursion_context(this_cpu_ptr(callchain_recursion));
if (*rctx == -1)
return NULL;
entries = rcu_dereference(callchain_cpus_entries);
if (!entries) {put_recursion_context(this_cpu_ptr(callchain_recursion), *rctx);
return NULL;
}
cpu = smp_processor_id();
return (((void *)entries->cpu_entries[cpu]) +
(*rctx * perf_callchain_entry__sizeof()));
}voidput_callchain_entry(int rctx)
{
put_recursion_context(this_cpu_ptr(callchain_recursion), rctx);
}Quoted 4 lines
If instead of disabling preemption we disable migration, then I think we should be good with relying on perf's callchain management, or am I missing something?
Peter proposed refactoring the get_perf_callchain interface in v3, i think after that we can still use perf callchain, but with the refactored API, it can prevent being overwritten by preemption.
BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map
{
...
entry = bpf_get_perf_callchain;
__bpf_get_stackid(map, entry, flags);
bpf_put_callchain_entry(entry);
...
}A simple specific implementation is as follows:
include/linux/perf_event.h +1 -0
@@ -67,6 +67,7 @@ struct perf_callchain_entry_ctx {u32 nr;short contexts;bool contexts_maxed; +bool add_mark; }; typedef unsigned long (*perf_copy_f)(void *dst, const void *src,
@@ -1718,9 +1719,18 @@ DECLARE_PER_CPU(struct perf_callchain_entry, perf_callchain_entry); extern void perf_callchain_user(struct perf_callchain_entry_ctx
*entry, struct pt_regs *regs);
extern void perf_callchain_kernel(struct perf_callchain_entry_ctx
*entry, struct pt_regs *regs);
+
+extern void __init_perf_callchain_ctx(struct perf_callchain_entry_ctx *ctx,
+ struct perf_callchain_entry *entry,
+ u32 max_stack, bool add_mark);
+
+extern void __get_perf_callchain_kernel(struct perf_callchain_entry_ctx *ctx, struct pt_regs *regs); +extern void __get_perf_callchain_user(struct perf_callchain_entry_ctx *ctx, struct pt_regs *regs); +
+
extern struct perf_callchain_entry *
get_perf_callchain(struct pt_regs *regs, bool kernel, bool user,
- u32 max_stack, bool crosstask, bool add_mark);
+ u32 max_stack, bool crosstask);
extern int get_callchain_buffers(int max_stack);
extern void put_callchain_buffers(void);
extern struct perf_callchain_entry *get_callchain_entry(int *rctx);
kernel/bpf/stackmap.c +32 -8
@@ -297,14 +297,38 @@ static long __bpf_get_stackid(struct bpf_map *map,return id; } +static struct perf_callchain_entry *entry +bpf_get_perf_callchain (int *rctx,struct pt_regs *regs,bool kernel,bool user,int max_stack) +{ +struct perf_callchain_entry_ctx ctx; +struct perf_callchain_entry *entry; + + entry =get_callchain_entry (&rctx); +if (unlikely (!entry)) +return NULL ; + + __init_perf_callchain_ctx(&ctx, entry) +if (kernel) +__get_perf_callchain_kernel (&ctx, regs); +if (user) +__get_perf_callchain_user (&ctx, regs); +return entry; +} + +static bpf_put_perf_callchain(int rctx) +{ +put_callchain_entry (rctx); +} + BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, u64, flags) {u32 max_depth = map->value_size /stack_map_data_size (map);u32 skip = flags &BPF_F_SKIP_FIELD_MASK ;bool user = flags &BPF_F_USER_STACK ; -struct perf_callchain_entry *trace; +struct perf_callchain_entry *entry;bool kernel = !user; +int rctx, ret;if (unlikely (flags & ~(BPF_F_SKIP_FIELD_MASK |BPF_F_USER_STACK |BPF_F_FAST_STACK_CMP |BPF_F_REUSE_STACKID )))
@@ -314,14 +338,14 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,if (max_depth > sysctl_perf_event_max_stack) max_depth = sysctl_perf_event_max_stack; - trace =get_perf_callchain (regs, kernel, user, max_depth, - false, false); - -if (unlikely (!trace)) -/* couldn't fetch the stack trace */ + entry =bpf_get_perf_callchain (&rctx, regs, kernel, user, max_depth); +if (unlikely (!entry))return -EFAULT ; -return __bpf_get_stackid (map, trace, flags); + ret =__bpf_get_stackid (map, entry, flags); +bpf_put_callchain_entry (rctx); + +return ret; }const struct bpf_func_proto bpf_get_stackid_proto = {
@@ -452,7 +476,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, trace =get_callchain_entry_for_task (task, max_depth); else trace =get_perf_callchain (regs, kernel, user, max_depth, - crosstask, false); + crosstask);if (unlikely (!trace) || trace->nr < skip) {if (may_fault)
kernel/events/callchain.c +48 -27
95 lines not on this page · raw message
kernel/events/core.c +1 -1
@@ -8218,7 +8218,7 @@ perf_callchain(struct perf_event *event, struct pt_regs *regs)return &__empty_callchain; callchain =get_perf_callchain (regs, kernel, user, - max_stack, crosstask, true); + max_stack, crosstask);return callchain ?: &__empty_callchain; }
Quoted 39 lines
Quoted 39 lines
Quoted 13 lines
Quoted 13 lines
Quoted 13 lines
kernel/bpf/stackmap.c BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, · line 313
Quoted 9 lines
kernel/bpf/stackmap.c +1
-0static inline bool stack_map_use_build_id (struct bpf_map *map) {return (map->map_flags &BPF_F_STACK_BUILD_ID );@@ -305,6 +310,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,bool user = flags &BPF_F_USER_STACK ;struct perf_callchain_entry *trace;bool kernel = !user; +struct bpf_perf_callchain_entry entry = {0 };so IIUC having entries on stack we do not need to do
preempt_disableyou had in the previous version, right?Quoted 9 lines
Quoted 5 lines
Quoted 3 lines
I saw Andrii's justification to have this on the stack, I think it's fine, but does it have to be initialized? it seems that only used entries are copied to map
No. We're not adding 1k stack consumption.
Right, and I thought we concluded as much last time, so it's a bit surprising to see this in this patch.
Ok, I feel like I'm missing some context from our previous exchange.
Quoted 4 lines
Tao, you should go with 3 entries per CPU used in a stack-like fashion. And then passing that entry into
get_perf_callchain()(to avoid one extra copy).Got it. It is more clearer, will change it in v3.
Quoted 2 lines
Quoted 2 lines
pw-bot: cr
--
Best Regards
Tao Chen
Signature · 2 lines
Best Regards Tao Chen