perf: Use sample_flags for callchain
Subsystems (4)
Namhyung Kim
maintainerPATCH 1/3 perf: Use sample_flags for callchain
So that it can call perf_callchain() only if needed. Historically it used __PERF_SAMPLE_CALLCHAIN_EARLY but we can do that with sample_flags in the struct perf_sample_data.
--- arch/x86/events/amd/ibs.c | 4 +++- arch/x86/events/intel/ds.c | 8 ++++++-- kernel/events/core.c | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-)
arch/x86/events/amd/ibs.c +3 -1
@@ -798,8 +798,10 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs)* recorded as part of interrupt regs. Thus we need to use rip from * interrupt regs while unwinding call stack. */ -if (event->attr.sample_type &PERF_SAMPLE_CALLCHAIN ) +if (event->attr.sample_type &PERF_SAMPLE_CALLCHAIN ) { data.callchain =perf_callchain (event, iregs); + data.sample_flags |=PERF_SAMPLE_CALLCHAIN ; + } throttle =perf_event_overflow (event, &data, ®s); out:
arch/x86/events/intel/ds.c +6 -2
@@ -1546,8 +1546,10 @@ static void setup_pebs_fixed_sample_data(struct perf_event *event,* previous PMI context or an (I)RET happened between the record and * PMI. */ -if (sample_type &PERF_SAMPLE_CALLCHAIN ) +if (sample_type &PERF_SAMPLE_CALLCHAIN ) { data->callchain =perf_callchain (event, iregs); + data->sample_flags |=PERF_SAMPLE_CALLCHAIN ; + }/* * We use the interrupt regs as a base because the PEBS record does not
@@ -1719,8 +1721,10 @@ static void setup_pebs_adaptive_sample_data(struct perf_event *event,* previous PMI context or an (I)RET happened between the record and * PMI. */ -if (sample_type &PERF_SAMPLE_CALLCHAIN ) +if (sample_type &PERF_SAMPLE_CALLCHAIN ) { data->callchain =perf_callchain (event, iregs); + data->sample_flags |=PERF_SAMPLE_CALLCHAIN ; + } *regs = *iregs;/* The ip in basic is EventingIP */
kernel/events/core.c +1 -1
@@ -7323,7 +7323,7 @@ void perf_prepare_sample(struct perf_event_header *header,if (sample_type &PERF_SAMPLE_CALLCHAIN ) {int size =1 ; -if (!(sample_type & __PERF_SAMPLE_CALLCHAIN_EARLY)) +if (filtered_sample_type &PERF_SAMPLE_CALLCHAIN ) data->callchain =perf_callchain (event, regs); size += data->callchain->nr;
Signature · 1 line
2.37.2.789.g6183377224-goog
Namhyung Kim
maintainerPATCH 2/3 perf/bpf: Always use perf callchains if exist
If the perf_event has PERF_SAMPLE_CALLCHAIN, BPF can use it for stack trace. The problematic cases like PEBS and IBS already handled in the PMU driver and they filled the callchain info in the sample data. For others, we can call perf_callchain() before the BPF handler.
--- kernel/bpf/stackmap.c | 4 ++-- kernel/events/core.c | 12 ++++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-)
kernel/bpf/stackmap.c +2 -2
@@ -338,7 +338,7 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx,int ret;/* perf_sample_data doesn't have callchain, use bpf_get_stackid */ -if (!(event->attr.sample_type & __PERF_SAMPLE_CALLCHAIN_EARLY)) +if (!(event->attr.sample_type &PERF_SAMPLE_CALLCHAIN ))return bpf_get_stackid ((unsigned long )(ctx->regs), (unsigned long ) map, flags,0 ,0 );
@@ -506,7 +506,7 @@ BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx,int err = -EINVAL ;__u64 nr_kernel; -if (!(event->attr.sample_type & __PERF_SAMPLE_CALLCHAIN_EARLY)) +if (!(event->attr.sample_type &PERF_SAMPLE_CALLCHAIN ))return __bpf_get_stack (regs,NULL ,NULL , buf, size, flags); if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
kernel/events/core.c +10 -2
@@ -10003,8 +10003,16 @@ static void bpf_overflow_handler(struct perf_event *event, goto out;rcu_read_lock (); prog =READ_ONCE (event->prog); -if (prog) +if (prog) { +if (prog->call_get_stack && + (event->attr.sample_type &PERF_SAMPLE_CALLCHAIN ) && + !(data->sample_flags &PERF_SAMPLE_CALLCHAIN )) { + data->callchain =perf_callchain (event, regs); + data->sample_flags |=PERF_SAMPLE_CALLCHAIN ; + } + ret =bpf_prog_run (prog, &ctx); + }rcu_read_unlock (); out:__this_cpu_dec (bpf_prog_active);
@@ -10030,7 +10038,7 @@ static int perf_event_set_bpf_handler(struct perf_event *event,if (event->attr.precise_ip && prog->call_get_stack && - (!(event->attr.sample_type & __PERF_SAMPLE_CALLCHAIN_EARLY) || + (!(event->attr.sample_type &PERF_SAMPLE_CALLCHAIN ) || event->attr.exclude_callchain_kernel || event->attr.exclude_callchain_user)) {/*
Signature · 1 line
2.37.2.789.g6183377224-goog
Re: [PATCH 2/3] perf/bpf: Always use perf callchains if exist
Quoted 6 lines · Namhyung Kim
If the
perf_eventhasPERF_SAMPLE_CALLCHAIN, BPF can use it for stack trace. The problematic cases like PEBS and IBS already handled in the PMU driver and they filled the callchain info in the sample data. For others, we can callperf_callchain()before the BPF handler.Signed-off-by Namhyung Kim <[email protected]>
At least from the description it make sense. We're filling a callchain when it's been requested by the event, but it's missing on the sample data (aka, software fallback?). perf_callchain also seems to always fallback to &__empty_callchain in case of an error, so seems safe.
Quoted 64 lines
--- kernel/bpf/stackmap.c | 4 ++-- kernel/events/core.c | 12 ++++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-)kernel/bpf/stackmap.c +2
-2@@ -338,7 +338,7 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx,int ret;/* perf_sample_data doesn't have callchain, use bpf_get_stackid */ -if (!(event->attr.sample_type & __PERF_SAMPLE_CALLCHAIN_EARLY)) +if (!(event->attr.sample_type &PERF_SAMPLE_CALLCHAIN ))return bpf_get_stackid ((unsigned long )(ctx->regs), (unsigned long ) map, flags,0 ,0 );@@ -506,7 +506,7 @@ BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx,int err = -EINVAL ;__u64 nr_kernel; -if (!(event->attr.sample_type & __PERF_SAMPLE_CALLCHAIN_EARLY)) +if (!(event->attr.sample_type &PERF_SAMPLE_CALLCHAIN ))return __bpf_get_stack (regs,NULL ,NULL , buf, size, flags);if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK |
BPF_F_USER_STACK|kernel/events/core.c +9
-1@@ -10003,8 +10003,16 @@ static void bpf_overflow_handler(structperf_event *event, goto out;rcu_read_lock (); prog =READ_ONCE (event->prog); -if (prog) +if (prog) { +if (prog->call_get_stack && + (event->attr.sample_type &PERF_SAMPLE_CALLCHAIN ) && + !(data->sample_flags &PERF_SAMPLE_CALLCHAIN )) { + data->callchain =perf_callchain (event, regs); + data->sample_flags |=PERF_SAMPLE_CALLCHAIN ; + } + ret =bpf_prog_run (prog, &ctx); + }rcu_read_unlock (); out:__this_cpu_dec(bpf_prog_active);kernel/events/core.c +1
-1@@ -10030,7 +10038,7 @@ static int perf_event_set_bpf_handler(struct perf_event *event, if (event->attr.precise_ip && prog->call_get_stack && - (!(event->attr.sample_type & __PERF_SAMPLE_CALLCHAIN_EARLY) || + (!(event->attr.sample_type &PERF_SAMPLE_CALLCHAIN ) || event->attr.exclude_callchain_kernel || event->attr.exclude_callchain_user)) {/*
--
2.37.2.789.g6183377224-goog
[tip: perf/core] perf/bpf: Always use perf callchains if exist
In reply to Namhyung KimThe following commit has been merged into the perf/core branch of tip:
The following commit has been merged into the perf/core branch of tip:
Commit-ID: 16817ad7e8b31728b44ff9f17d8d894ed8a450d0 Gitweb: https://git.kernel.org/tip/16817ad7e8b31728b44ff9f17d8d894ed8a450d0 Author: Namhyung Kim <[email protected]> AuthorDate: Thu, 08 Sep 2022 14:41:03 -07:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 13 Sep 2022 15:03:22 +02:00
perf/bpf: Always use perf callchains if exist
If the perf_event has PERF_SAMPLE_CALLCHAIN, BPF can use it for stack trace. The problematic cases like PEBS and IBS already handled in the PMU driver and they filled the callchain info in the sample data. For others, we can call perf_callchain() before the BPF handler.
--- kernel/bpf/stackmap.c | 4 ++-- kernel/events/core.c | 12 ++++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-)
kernel/bpf/stackmap.c +2 -2
@@ -338,7 +338,7 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx,int ret;/* perf_sample_data doesn't have callchain, use bpf_get_stackid */ -if (!(event->attr.sample_type & __PERF_SAMPLE_CALLCHAIN_EARLY)) +if (!(event->attr.sample_type &PERF_SAMPLE_CALLCHAIN ))return bpf_get_stackid ((unsigned long )(ctx->regs), (unsigned long ) map, flags,0 ,0 );
@@ -506,7 +506,7 @@ BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx,int err = -EINVAL ;__u64 nr_kernel; -if (!(event->attr.sample_type & __PERF_SAMPLE_CALLCHAIN_EARLY)) +if (!(event->attr.sample_type &PERF_SAMPLE_CALLCHAIN ))return __bpf_get_stack (regs,NULL ,NULL , buf, size, flags); if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
kernel/events/core.c +10 -2
@@ -10000,8 +10000,16 @@ static void bpf_overflow_handler(struct perf_event *event, goto out;rcu_read_lock (); prog =READ_ONCE (event->prog); -if (prog) +if (prog) { +if (prog->call_get_stack && + (event->attr.sample_type &PERF_SAMPLE_CALLCHAIN ) && + !(data->sample_flags &PERF_SAMPLE_CALLCHAIN )) { + data->callchain =perf_callchain (event, regs); + data->sample_flags |=PERF_SAMPLE_CALLCHAIN ; + } + ret =bpf_prog_run (prog, &ctx); + }rcu_read_unlock (); out:__this_cpu_dec (bpf_prog_active);
@@ -10027,7 +10035,7 @@ static int perf_event_set_bpf_handler(struct perf_event *event,if (event->attr.precise_ip && prog->call_get_stack && - (!(event->attr.sample_type & __PERF_SAMPLE_CALLCHAIN_EARLY) || + (!(event->attr.sample_type &PERF_SAMPLE_CALLCHAIN ) || event->attr.exclude_callchain_kernel || event->attr.exclude_callchain_user)) {/*
Namhyung Kim
maintainerPATCH 3/3 perf: Kill __PERF_SAMPLE_CALLCHAIN_EARLY
In reply to Namhyung KimThere's no in-tree user anymore. Let's get rid of it.
--- arch/x86/events/amd/ibs.c | 10 ---------- arch/x86/events/intel/core.c | 3 --- include/uapi/linux/perf_event.h | 2 -- 3 files changed, 15 deletions(-)
arch/x86/events/amd/ibs.c +0 -10
@@ -300,16 +300,6 @@ static int perf_ibs_init(struct perf_event *event) hwc->config_base = perf_ibs->msr; hwc->config = config; -/* -* rip recorded by IbsOpRip will not be consistent with rsp and rbp -* recorded as part of interrupt regs. Thus we need to use rip from -* interrupt regs while unwinding call stack. Setting _EARLY flag -* makes sure we unwind call-stack before perf sample rip is set to -* IbsOpRip. -*/ -if (event->attr.sample_type &PERF_SAMPLE_CALLCHAIN ) - event->attr.sample_type |= __PERF_SAMPLE_CALLCHAIN_EARLY; -return 0 ; }
arch/x86/events/intel/core.c +0 -3
@@ -3846,9 +3846,6 @@ static int intel_pmu_hw_config(struct perf_event *event) }if (x86_pmu.pebs_aliases) x86_pmu.pebs_aliases (event); - -if (event->attr.sample_type &PERF_SAMPLE_CALLCHAIN ) - event->attr.sample_type |= __PERF_SAMPLE_CALLCHAIN_EARLY; }if (needs_branch_stack (event)) {
include/uapi/linux/perf_event.h +0 -2
@@ -164,8 +164,6 @@ enum perf_event_sample_format {PERF_SAMPLE_WEIGHT_STRUCT =1U <<24 , PERF_SAMPLE_MAX = 1U << 25, /* non-ABI */ - - __PERF_SAMPLE_CALLCHAIN_EARLY = 1ULL << 63, /* non-ABI; internal use */ };#define PERF_SAMPLE_WEIGHT_TYPE (PERF_SAMPLE_WEIGHT | PERF_SAMPLE_WEIGHT_STRUCT)
Signature · 1 line
2.37.2.789.g6183377224-goog
[tip: perf/core] perf: Kill __PERF_SAMPLE_CALLCHAIN_EARLY
The following commit has been merged into the perf/core branch of tip:
The following commit has been merged into the perf/core branch of tip:
Commit-ID: b4e12b2d70fd9eccdb3cef8015dc1788ca38e3fd Gitweb: https://git.kernel.org/tip/b4e12b2d70fd9eccdb3cef8015dc1788ca38e3fd Author: Namhyung Kim <[email protected]> AuthorDate: Thu, 08 Sep 2022 14:41:04 -07:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 13 Sep 2022 15:03:23 +02:00
perf: Kill __PERF_SAMPLE_CALLCHAIN_EARLY
There's no in-tree user anymore. Let's get rid of it.
--- arch/x86/events/amd/ibs.c | 10 ---------- arch/x86/events/intel/core.c | 3 --- include/uapi/linux/perf_event.h | 2 -- 3 files changed, 15 deletions(-)
arch/x86/events/amd/ibs.c +0 -10
@@ -300,16 +300,6 @@ static int perf_ibs_init(struct perf_event *event) hwc->config_base = perf_ibs->msr; hwc->config = config; -/* -* rip recorded by IbsOpRip will not be consistent with rsp and rbp -* recorded as part of interrupt regs. Thus we need to use rip from -* interrupt regs while unwinding call stack. Setting _EARLY flag -* makes sure we unwind call-stack before perf sample rip is set to -* IbsOpRip. -*/ -if (event->attr.sample_type &PERF_SAMPLE_CALLCHAIN ) - event->attr.sample_type |= __PERF_SAMPLE_CALLCHAIN_EARLY; -return 0 ; }
arch/x86/events/intel/core.c +0 -3
@@ -3868,9 +3868,6 @@ static int intel_pmu_hw_config(struct perf_event *event) }if (x86_pmu.pebs_aliases) x86_pmu.pebs_aliases (event); - -if (event->attr.sample_type &PERF_SAMPLE_CALLCHAIN ) - event->attr.sample_type |= __PERF_SAMPLE_CALLCHAIN_EARLY; }if (needs_branch_stack (event)) {
include/uapi/linux/perf_event.h +0 -2
@@ -164,8 +164,6 @@ enum perf_event_sample_format {PERF_SAMPLE_WEIGHT_STRUCT =1U <<24 , PERF_SAMPLE_MAX = 1U << 25, /* non-ABI */ - - __PERF_SAMPLE_CALLCHAIN_EARLY = 1ULL << 63, /* non-ABI; internal use */ };#define PERF_SAMPLE_WEIGHT_TYPE (PERF_SAMPLE_WEIGHT | PERF_SAMPLE_WEIGHT_STRUCT)
Quoted 5 lines · Namhyung Kim
So that it can call
perf_callchain()only if needed. Historically it used__PERF_SAMPLE_CALLCHAIN_EARLYbut we can do that withsample_flagsin the structperf_sample_data.Signed-off-by Namhyung Kim <[email protected]>
The series look good to me.
Thanks,
Kan
Quoted 63 lines
--- arch/x86/events/amd/ibs.c | 4 +++- arch/x86/events/intel/ds.c | 8 ++++++-- kernel/events/core.c | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-)arch/x86/events/amd/ibs.c +3
-1@@ -798,8 +798,10 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs)* recorded as part of interrupt regs. Thus we need to use rip from * interrupt regs while unwinding call stack. */ -if (event->attr.sample_type &PERF_SAMPLE_CALLCHAIN ) +if (event->attr.sample_type &PERF_SAMPLE_CALLCHAIN ) { data.callchain =perf_callchain (event, iregs); + data.sample_flags |=PERF_SAMPLE_CALLCHAIN ; + } throttle =perf_event_overflow (event, &data, ®s); out:arch/x86/events/intel/ds.c +6
-2@@ -1546,8 +1546,10 @@ static void setup_pebs_fixed_sample_data(struct perf_event *event,* previous PMI context or an (I)RET happened between the record and * PMI. */ -if (sample_type &PERF_SAMPLE_CALLCHAIN ) +if (sample_type &PERF_SAMPLE_CALLCHAIN ) { data->callchain =perf_callchain (event, iregs); + data->sample_flags |=PERF_SAMPLE_CALLCHAIN ; + }/* * We use the interrupt regs as a base because the PEBS record does not @@ -1719,8 +1721,10 @@ static void setup_pebs_adaptive_sample_data(struct perf_event *event,* previous PMI context or an (I)RET happened between the record and * PMI. */ -if (sample_type &PERF_SAMPLE_CALLCHAIN ) +if (sample_type &PERF_SAMPLE_CALLCHAIN ) { data->callchain =perf_callchain (event, iregs); + data->sample_flags |=PERF_SAMPLE_CALLCHAIN ; + } *regs = *iregs;/* The ip in basic is EventingIP */ kernel/events/core.c +1
-1@@ -7323,7 +7323,7 @@ void perf_prepare_sample(struct perf_event_header *header,if (sample_type &PERF_SAMPLE_CALLCHAIN ) {int size =1 ; -if (!(sample_type & __PERF_SAMPLE_CALLCHAIN_EARLY)) +if (filtered_sample_type &PERF_SAMPLE_CALLCHAIN ) data->callchain =perf_callchain (event, regs); size += data->callchain->nr;
The following commit has been merged into the perf/core branch of tip:
The following commit has been merged into the perf/core branch of tip:
Commit-ID: 3749d33e510c3dc695b3a5886b706310890d7ebd Gitweb: https://git.kernel.org/tip/3749d33e510c3dc695b3a5886b706310890d7ebd Author: Namhyung Kim <[email protected]> AuthorDate: Thu, 08 Sep 2022 14:41:02 -07:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 13 Sep 2022 15:03:22 +02:00
perf: Use sample_flags for callchain
So that it can call perf_callchain() only if needed. Historically it used __PERF_SAMPLE_CALLCHAIN_EARLY but we can do that with sample_flags in the struct perf_sample_data.
--- arch/x86/events/amd/ibs.c | 4 +++- arch/x86/events/intel/ds.c | 8 ++++++-- kernel/events/core.c | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-)
arch/x86/events/amd/ibs.c +3 -1
@@ -798,8 +798,10 @@ fail:* recorded as part of interrupt regs. Thus we need to use rip from * interrupt regs while unwinding call stack. */ -if (event->attr.sample_type &PERF_SAMPLE_CALLCHAIN ) +if (event->attr.sample_type &PERF_SAMPLE_CALLCHAIN ) { data.callchain =perf_callchain (event, iregs); + data.sample_flags |=PERF_SAMPLE_CALLCHAIN ; + } throttle =perf_event_overflow (event, &data, ®s); out:
arch/x86/events/intel/ds.c +6 -2
@@ -1546,8 +1546,10 @@ static void setup_pebs_fixed_sample_data(struct perf_event *event,* previous PMI context or an (I)RET happened between the record and * PMI. */ -if (sample_type &PERF_SAMPLE_CALLCHAIN ) +if (sample_type &PERF_SAMPLE_CALLCHAIN ) { data->callchain =perf_callchain (event, iregs); + data->sample_flags |=PERF_SAMPLE_CALLCHAIN ; + }/* * We use the interrupt regs as a base because the PEBS record does not
@@ -1719,8 +1721,10 @@ static void setup_pebs_adaptive_sample_data(struct perf_event *event,* previous PMI context or an (I)RET happened between the record and * PMI. */ -if (sample_type &PERF_SAMPLE_CALLCHAIN ) +if (sample_type &PERF_SAMPLE_CALLCHAIN ) { data->callchain =perf_callchain (event, iregs); + data->sample_flags |=PERF_SAMPLE_CALLCHAIN ; + } *regs = *iregs;/* The ip in basic is EventingIP */
kernel/events/core.c +1 -1
@@ -7320,7 +7320,7 @@ void perf_prepare_sample(struct perf_event_header *header,if (sample_type &PERF_SAMPLE_CALLCHAIN ) {int size =1 ; -if (!(sample_type & __PERF_SAMPLE_CALLCHAIN_EARLY)) +if (filtered_sample_type &PERF_SAMPLE_CALLCHAIN ) data->callchain =perf_callchain (event, regs); size += data->callchain->nr;