perf: Support the deferred unwinding infrastructure
Subsystems (4)
v14 cover perf: Support the deferred unwinding infrastructure
This is based on top of the deferred unwind core patch series:
https://lore.kernel.org/linux-trace-kernel/[email protected]/
git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git
unwind/core
This series implements the perf interface to use deferred user space stack tracing.
The first 5 patches are clean ups and simplifications. There's a standalone series with these patches here:
https://lore.kernel.org/linux-trace-kernel/[email protected]/
Patch 6 implements a task deferred tracing that works with events following a specific task (per thread).
Patch 7 implements a per CPU deferred tracing that requires the application (perf user space) to have a per CPU event buffer for every CPU where a task may migrate to from the time a deferred request is made to when the stack trace occurs, as a task may migrate to a different CPU after the request and before it goes back to user space.
The rest of the patches implement the tool side of perf.
KNOWN ISSUES:
- The marker that adds the USER_DEFERRED when the request was made, should also add the cookie. As the cookie can be used to figure out if dropped events missed a stack trace and not to attach a stack trace to the wrong events.
- The writing of the stack trace should probably be changed to act more like get_perf_callchain() where it does fixups to uprobes.
The code for this series is located here:
git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git
unwind/perf
Head SHA1: 5753b61c16f61e50f35bf0f3dfbf8a00b8de2d51
Changes since v13: https://lore.kernel.org/linux-trace-kernel/[email protected]/
- Missed one location to replace the current->mm == NULL check that still only checked PF_KTHREAD. It must also check PF_USER_WORKER.
- Need to copy the trace.entries[] one a at a time as the perf entry in the ring buffer has 64 bit entries, but trace.entries[] are size long.
- Added back the cookie field in perf_callchain_deferred_event structure (Note, it was a timestamp before) (Namhyung Kim)
- Add the cookie to the comment explaining perf_callchain_deferred_event.
- Fixed deferred_unwind_request() to return 1 if the request was already queued or was already executed to not incorrectly increment nr_no_switch_fast.
- Display the cookie in the -D output
Josh Poimboeuf (5):
perf: Remove get_perf_callchain() init_nr argument
perf: Have get_perf_callchain() return NULL if crosstask and user are set
perf: Simplify get_perf_callchain() user logic
perf: Skip user unwind if the task is a kernel thread
perf: Support deferred user callchains
Namhyung Kim (4):
perf tools: Minimal CALLCHAIN_DEFERRED support perf record: Enable defer_callchain for user callchains perf script: Display PERF_RECORD_CALLCHAIN_DEFERRED perf tools: Merge deferred user callchains
Steven Rostedt (2):
perf: Use current->flags & PF_KTHREAD|PF_USER_WORKER instead of current->mm == NULL
perf: Support deferred user callchains for per CPU events
----
include/linux/perf_event.h | 13 +- include/uapi/linux/perf_event.h | 20 +- kernel/bpf/stackmap.c | 8 +- kernel/events/callchain.c | 49 ++-- kernel/events/core.c | 424 +++++++++++++++++++++++++++++- tools/include/uapi/linux/perf_event.h | 19 +- tools/lib/perf/include/perf/event.h | 8 + tools/perf/Documentation/perf-script.txt | 5 + tools/perf/builtin-script.c | 92 +++++++ tools/perf/util/callchain.c | 24 ++ tools/perf/util/callchain.h | 3 + tools/perf/util/event.c | 1 + tools/perf/util/evlist.c | 1 + tools/perf/util/evlist.h | 1 + tools/perf/util/evsel.c | 39 +++ tools/perf/util/evsel.h | 1 + tools/perf/util/machine.c | 1 + tools/perf/util/perf_event_attr_fprintf.c | 1 + tools/perf/util/sample.h | 3 +- tools/perf/util/session.c | 79 ++++++ tools/perf/util/tool.c | 2 + tools/perf/util/tool.h | 4 +- 22 files changed, 762 insertions(+), 36 deletions(-)
PATCH v14 1/11 perf: Remove get_perf_callchain() init_nr argument
From: Josh Poimboeuf <[email protected]>
The 'init_nr' argument has double duty: it's used to initialize both the number of contexts and the number of stack entries. That's confusing and the callers always pass zero anyway. Hard code the zero.
--- include/linux/perf_event.h | 2 +- kernel/bpf/stackmap.c | 4 ++-- kernel/events/callchain.c | 12 ++++++------ kernel/events/core.c | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-)
include/linux/perf_event.h +1 -1
@@ -1719,7 +1719,7 @@ DECLARE_PER_CPU(struct perf_callchain_entry, perf_callchain_entry);extern void perf_callchain_user (struct perf_callchain_entry_ctx *entry,struct pt_regs *regs);extern void perf_callchain_kernel (struct perf_callchain_entry_ctx *entry,struct pt_regs *regs);extern struct perf_callchain_entry * -get_perf_callchain (struct pt_regs *regs,u32 init_nr,bool kernel,bool user, +get_perf_callchain (struct pt_regs *regs,bool kernel,bool user,u32 max_stack,bool crosstask,bool add_mark);extern int get_callchain_buffers (int max_stack);extern void put_callchain_buffers (void );
kernel/bpf/stackmap.c +2 -2
@@ -314,7 +314,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,if (max_depth > sysctl_perf_event_max_stack) max_depth = sysctl_perf_event_max_stack; - trace =get_perf_callchain (regs,0 , kernel, user, max_depth, + trace =get_perf_callchain (regs, kernel, user, max_depth, false, false);if (unlikely (!trace))
@@ -451,7 +451,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,else if (kernel && task) trace =get_callchain_entry_for_task (task, max_depth);else - trace =get_perf_callchain (regs,0 , kernel, user, max_depth, + trace =get_perf_callchain (regs, kernel, user, max_depth, crosstask, false);if (unlikely (!trace) || trace->nr < skip) {
kernel/events/callchain.c +6 -6
@@ -217,7 +217,7 @@ static void fixup_uretprobe_trampoline_entries(struct perf_callchain_entry *entr }struct perf_callchain_entry * -get_perf_callchain (struct pt_regs *regs,u32 init_nr,bool kernel,bool user, +get_perf_callchain (struct pt_regs *regs,bool kernel,bool user,u32 max_stack,bool crosstask,bool add_mark) {struct perf_callchain_entry *entry;
@@ -228,11 +228,11 @@ get_perf_callchain(struct pt_regs *regs, u32 init_nr, bool kernel, bool user,if (!entry)return NULL ; - ctx.entry = entry; - ctx.max_stack = max_stack; - ctx.nr = entry->nr = init_nr; - ctx.contexts =0 ; - ctx.contexts_maxed = false; + ctx.entry = entry; + ctx.max_stack = max_stack; + ctx.nr = entry->nr =0 ; + ctx.contexts =0 ; + ctx.contexts_maxed = false;if (kernel && !user_mode (regs)) {if (add_mark)
kernel/events/core.c +1 -1
@@ -8178,7 +8178,7 @@ perf_callchain(struct perf_event *event, struct pt_regs *regs)if (!kernel && !user)return &__empty_callchain; - callchain =get_perf_callchain (regs,0 , kernel, user, + callchain =get_perf_callchain (regs, kernel, user, max_stack, crosstask, true);return callchain ?: &__empty_callchain; }
Signature · 1 line
2.47.2
PATCH v14 2/11 perf: Have get_perf_callchain() return NULL if crosstask and user are set
In reply to Steven RostedtFrom: Josh Poimboeuf <[email protected]>
get_perf_callchain() doesn't support cross-task unwinding for user space stacks, have it return NULL if both the crosstask and user arguments are set.
--- kernel/events/callchain.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-)
kernel/events/callchain.c +5 -5
@@ -224,6 +224,10 @@ get_perf_callchain(struct pt_regs *regs, bool kernel, bool user,struct perf_callchain_entry_ctx ctx;int rctx, start_entry_idx; +/* crosstask is not supported for user stacks */ +if (crosstask && user && !kernel) +return NULL ; + entry =get_callchain_entry (&rctx);if (!entry)return NULL ;
@@ -240,7 +244,7 @@ get_perf_callchain(struct pt_regs *regs, bool kernel, bool user,perf_callchain_kernel (&ctx, regs); } -if (user) { +if (user && !crosstask) {if (!user_mode (regs)) {if (current->mm) regs =task_pt_regs (current);
@@ -249,9 +253,6 @@ get_perf_callchain(struct pt_regs *regs, bool kernel, bool user, }if (regs) { -if (crosstask) - goto exit_put; -if (add_mark)perf_callchain_store_context (&ctx,PERF_CONTEXT_USER );
@@ -261,7 +262,6 @@ get_perf_callchain(struct pt_regs *regs, bool kernel, bool user, } } -exit_put:put_callchain_entry (rctx);return entry;
Signature · 1 line
2.47.2
PATCH v14 3/11 perf: Use current->flags & PF_KTHREAD|PF_USER_WORKER instead of current->mm == NULL
In reply to Steven RostedtFrom: Steven Rostedt <[email protected]>
To determine if a task is a kernel thread or not, it is more reliable to use (current->flags & (PF_KTHREAD|PF_USER_WORKERi)) than to rely on current->mm being NULL. That is because some kernel tasks (io_uring helpers) may have a mm field.
---
Changes since v13: https://lore.kernel.org/[email protected]
- Missed one location that still only checked PF_KTHREAD
kernel/events/callchain.c | 6 +++--- kernel/events/core.c | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-)
kernel/events/callchain.c +3 -3
@@ -246,10 +246,10 @@ get_perf_callchain(struct pt_regs *regs, bool kernel, bool user,if (user && !crosstask) {if (!user_mode (regs)) { -if (current->mm) - regs =task_pt_regs (current); -else +if (current->flags & (PF_KTHREAD |PF_USER_WORKER )) regs =NULL ; +else + regs =task_pt_regs (current); }if (regs) {
kernel/events/core.c +2 -2
@@ -7414,7 +7414,7 @@ static void perf_sample_regs_user(struct perf_regs *regs_user,if (user_mode (regs)) { regs_user->abi =perf_reg_abi (current); regs_user->regs = regs; - }else if (!(current->flags &PF_KTHREAD )) { + }else if (!(current->flags & (PF_KTHREAD |PF_USER_WORKER ))) {perf_get_regs_user (regs_user, regs); }else { regs_user->abi =PERF_SAMPLE_REGS_ABI_NONE ;
@@ -8054,7 +8054,7 @@ static u64 perf_virt_to_phys(u64 virt)* Try IRQ-safe get_user_page_fast_only first. * If failed, leave phys_addr as 0. */ -if (current->mm !=NULL ) { +if (!(current->flags & (PF_KTHREAD |PF_USER_WORKER ))) {struct page *p;pagefault_disable ();
Signature · 1 line
2.47.2
PATCH v14 4/11 perf: Simplify get_perf_callchain() user logic
In reply to Steven RostedtFrom: Josh Poimboeuf <[email protected]>
Simplify the get_perf_callchain() user logic a bit. task_pt_regs() should never be NULL.
--- kernel/events/callchain.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-)
kernel/events/callchain.c +8 -10
@@ -247,21 +247,19 @@ get_perf_callchain(struct pt_regs *regs, bool kernel, bool user,if (user && !crosstask) {if (!user_mode (regs)) {if (current->flags & (PF_KTHREAD |PF_USER_WORKER )) - regs =NULL ; -else - regs =task_pt_regs (current); + goto exit_put; + regs =task_pt_regs (current); } -if (regs) { -if (add_mark) -perf_callchain_store_context (&ctx,PERF_CONTEXT_USER ); +if (add_mark) +perf_callchain_store_context (&ctx,PERF_CONTEXT_USER ); - start_entry_idx = entry->nr; -perf_callchain_user (&ctx, regs); -fixup_uretprobe_trampoline_entries (entry, start_entry_idx); - } + start_entry_idx = entry->nr; +perf_callchain_user (&ctx, regs); +fixup_uretprobe_trampoline_entries (entry, start_entry_idx); } +exit_put:put_callchain_entry (rctx);return entry;
Signature · 1 line
2.47.2
PATCH v14 5/11 perf: Skip user unwind if the task is a kernel thread
In reply to Steven RostedtFrom: Josh Poimboeuf <[email protected]>
If the task is not a user thread, there's no user stack to unwind.
--- kernel/events/core.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
kernel/events/core.c +2 -1
@@ -8166,7 +8166,8 @@ struct perf_callchain_entry * perf_callchain(struct perf_event *event, struct pt_regs *regs) { bool kernel = !event->attr.exclude_callchain_kernel; -bool user = !event->attr.exclude_callchain_user; +bool user = !event->attr.exclude_callchain_user && + !(current->flags & (PF_KTHREAD |PF_USER_WORKER ));/* Disallow cross-task user callchains. */ bool crosstask = event->ctx->task && event->ctx->task != current;const u32 max_stack = event->attr.sample_max_stack;
Signature · 1 line
2.47.2
PATCH v14 6/11 perf: Support deferred user callchains
In reply to Steven RostedtFrom: Josh Poimboeuf <[email protected]>
Use the new unwind_deferred_trace() interface (if available) to defer unwinds to task context. This will allow the use of .sframe (when it becomes available) and also prevents duplicate userspace unwinds.
As the struct unwind_stacktrace has its entries as "unsigned long", and it is used to copy directly into struct perf_callchain_entry which its "ip" field is defined as u64, only allow deferred callchains for 64bit architectures.
---
Changse since v13: https://lore.kernel.org/[email protected]
- Rename unwind_deferred_trace() to unwind_user_faultable(). The function's name was changed, but wasn't caught because the code was moved around in the next patch which also did the rename. Compiling this patch as a standalone caught the issue.
- Need to copy the trace.entries[] one a at a time as the perf entry in the ring buffer has 64 bit entries, but trace.entries[] are size long.
include/linux/perf_event.h | 7 +- include/uapi/linux/perf_event.h | 19 +++- kernel/bpf/stackmap.c | 4 +- kernel/events/callchain.c | 11 +- kernel/events/core.c | 154 +++++++++++++++++++++++++- tools/include/uapi/linux/perf_event.h | 19 +++- 6 files changed, 206 insertions(+), 8 deletions(-)
include/linux/perf_event.h +6 -1
@@ -53,6 +53,7 @@#include <linux/security.h> #include <linux/static_call.h> #include <linux/lockdep.h> +#include <linux/unwind_deferred.h> #include <asm/local.h>
@@ -880,6 +881,10 @@ struct perf_event {struct callback_head pending_task;unsigned int pending_work; +unsigned int pending_unwind_callback; +struct callback_head pending_unwind_work; +struct rcuwait pending_unwind_wait; +atomic_t event_limit;/* address range filters */
@@ -1720,7 +1725,7 @@ extern void perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pextern void perf_callchain_kernel (struct perf_callchain_entry_ctx *entry,struct pt_regs *regs);extern struct perf_callchain_entry *get_perf_callchain (struct pt_regs *regs,bool kernel,bool user, -u32 max_stack,bool crosstask,bool add_mark); +u32 max_stack,bool crosstask,bool add_mark,bool defer_user);extern int get_callchain_buffers (int max_stack);extern void put_callchain_buffers (void );extern struct perf_callchain_entry *get_callchain_entry (int *rctx);
include/uapi/linux/perf_event.h +18 -1
@@ -463,7 +463,8 @@ struct perf_event_attr { inherit_thread : 1, /* children only inherit if cloned with CLONE_THREAD */ remove_on_exec : 1, /* event is removed from task on exec */ sigtrap : 1, /* send synchronous SIGTRAP on event */ - __reserved_1 : 26; + defer_callchain: 1, /* generate PERF_RECORD_CALLCHAIN_DEFERRED records */ + __reserved_1 : 25;union {__u32 wakeup_events;/* wake up every n events */
@@ -1239,6 +1240,21 @@ enum perf_event_type {*/ PERF_RECORD_AUX_OUTPUT_HW_ID =21 , +/* +* This user callchain capture was deferred until shortly before +* returning to user space. Previous samples would have kernel +* callchains only and they need to be stitched with this to make full +* callchains. +* +* struct { +* struct perf_event_header header; +* u64 nr; +* u64 ips[nr]; +* struct sample_id sample_id; +* }; +*/ +PERF_RECORD_CALLCHAIN_DEFERRED =22 , + PERF_RECORD_MAX, /* non-ABI */ };
@@ -1269,6 +1285,7 @@ enum perf_callchain_context {PERF_CONTEXT_HV = (__u64)-32 ,PERF_CONTEXT_KERNEL = (__u64)-128 ,PERF_CONTEXT_USER = (__u64)-512 , +PERF_CONTEXT_USER_DEFERRED = (__u64)-640 ,PERF_CONTEXT_GUEST = (__u64)-2048 , PERF_CONTEXT_GUEST_KERNEL = (__u64)-2176,
kernel/bpf/stackmap.c +2 -2
@@ -315,7 +315,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, max_depth = sysctl_perf_event_max_stack; trace =get_perf_callchain (regs, kernel, user, max_depth, - false, false); + false, false, false);if (unlikely (!trace))/* couldn't fetch the stack trace */
@@ -452,7 +452,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, trace =get_callchain_entry_for_task (task, max_depth); else trace =get_perf_callchain (regs, kernel, user, max_depth, - crosstask, false); + crosstask, false, false);if (unlikely (!trace) || trace->nr < skip) {if (may_fault)
kernel/events/callchain.c +10 -1
@@ -218,7 +218,7 @@ static void fixup_uretprobe_trampoline_entries(struct perf_callchain_entry *entrstruct perf_callchain_entry *get_perf_callchain (struct pt_regs *regs,bool kernel,bool user, -u32 max_stack,bool crosstask,bool add_mark) +u32 max_stack,bool crosstask,bool add_mark,bool defer_user) {struct perf_callchain_entry *entry;struct perf_callchain_entry_ctx ctx;
@@ -251,6 +251,15 @@ get_perf_callchain(struct pt_regs *regs, bool kernel, bool user, regs =task_pt_regs (current); } +if (defer_user) { +/* +* Foretell the coming of PERF_RECORD_CALLCHAIN_DEFERRED +* which can be stitched to this one. +*/ +perf_callchain_store_context (&ctx,PERF_CONTEXT_USER_DEFERRED ); + goto exit_put; + } +if (add_mark)perf_callchain_store_context (&ctx,PERF_CONTEXT_USER );
kernel/events/core.c +152 -2
196 lines not on this page · raw message
tools/include/uapi/linux/perf_event.h +18 -1
@@ -463,7 +463,8 @@ struct perf_event_attr { inherit_thread : 1, /* children only inherit if cloned with CLONE_THREAD */ remove_on_exec : 1, /* event is removed from task on exec */ sigtrap : 1, /* send synchronous SIGTRAP on event */ - __reserved_1 : 26; + defer_callchain: 1, /* generate PERF_RECORD_CALLCHAIN_DEFERRED records */ + __reserved_1 : 25;union {__u32 wakeup_events;/* wake up every n events */
@@ -1239,6 +1240,21 @@ enum perf_event_type {*/ PERF_RECORD_AUX_OUTPUT_HW_ID =21 , +/* +* This user callchain capture was deferred until shortly before +* returning to user space. Previous samples would have kernel +* callchains only and they need to be stitched with this to make full +* callchains. +* +* struct { +* struct perf_event_header header; +* u64 nr; +* u64 ips[nr]; +* struct sample_id sample_id; +* }; +*/ +PERF_RECORD_CALLCHAIN_DEFERRED =22 , + PERF_RECORD_MAX, /* non-ABI */ };
@@ -1269,6 +1285,7 @@ enum perf_callchain_context {PERF_CONTEXT_HV = (__u64)-32 ,PERF_CONTEXT_KERNEL = (__u64)-128 ,PERF_CONTEXT_USER = (__u64)-512 , +PERF_CONTEXT_USER_DEFERRED = (__u64)-640 ,PERF_CONTEXT_GUEST = (__u64)-2048 , PERF_CONTEXT_GUEST_KERNEL = (__u64)-2176,
Signature · 1 line
2.47.2
PATCH v14 7/11 perf: Support deferred user callchains for per CPU events
In reply to Steven RostedtFrom: Steven Rostedt <[email protected]>
The deferred unwinder works fine for task events (events that trace only a specific task), as it can use a task_work from an interrupt or NMI and when the task goes back to user space it will call the event's callback to do the deferred unwinding.
But for per CPU events things are not so simple. When a per CPU event wants a deferred unwinding to occur, it cannot simply use a task_work as there's a many to many relationship. If the task migrates and another task is scheduled in where the per CPU event wants a deferred unwinding to occur on that task as well, and the task that migrated to another CPU has that CPU's event want to unwind it too, each CPU may need unwinding from more than one task, and each task may have requests from many CPUs.
To solve this, when a per CPU event is created that has defer_callchain attribute set, it will do a lookup from a global list (unwind_deferred_list), for a perf_unwind_deferred descriptor that has the id that matches the PID of the current task's group_leader.
If it is not found, then it will create one and add it to the global list. This descriptor contains an array of all possible CPUs, where each element is a perf_unwind_cpu descriptor.
The perf_unwind_cpu descriptor has a list of all the per CPU events that is tracing the matching CPU that corresponds to its index in the array, where the events belong to a task that has the same group_leader. It also has a processing bit and rcuwait to handle removal.
For each occupied perf_unwind_cpu descriptor in the array, the perf_deferred_unwind descriptor increments its nr_cpu_events. When a perf_unwind_cpu descriptor is empty, the nr_cpu_events is decremented. This is used to know when to free the perf_deferred_unwind descriptor, as when it becomes empty, it is no longer referenced.
Finally, the perf_deferred_unwind descriptor has an id that holds the PID of the group_leader for the tasks that the events were created by.
When a second (or more) per CPU event is created where the perf_deferred_unwind descriptor is already created, it just adds itself to the perf_unwind_cpu array of that descriptor. Updating the necessary counter. This is used to map different per CPU events to each other based on their group leader PID.
Each of these perf_deferred_unwind descriptors have a unwind_work that registers with the deferred unwind infrastructure via unwind_deferred_init(), where it also registers a callback to perf_event_deferred_cpu().
Now when a per CPU event requests a deferred unwinding, it calls unwind_deferred_request() with the associated perf_deferred_unwind descriptor. It is expected that the program that uses this has events on all CPUs, as the deferred trace may not be called on the CPU event that requested it. That is, the task may migrate and its user stack trace will be recorded on the CPU event of the CPU that it exits back to user space on.
---
Changes since v13: https://lore.kernel.org/[email protected]
- Added back the cookie field in perf_callchain_deferred_event structure (Note, it was a timestamp before) (Namhyung Kim)
- Add the cookie to the comment explaining perf_callchain_deferred_event.
- Fixed deferred_unwind_request() to return 1 if the request was already queued or was already executed to not incorrectly increment nr_no_switch_fast.
include/linux/perf_event.h | 4 + include/uapi/linux/perf_event.h | 1 + kernel/events/core.c | 319 +++++++++++++++++++++++++++++--- 3 files changed, 296 insertions(+), 28 deletions(-)
include/linux/perf_event.h +4 -0
@@ -733,6 +733,7 @@ struct swevent_hlist {struct bpf_prog ;struct perf_cgroup ;struct perf_buffer ; +struct perf_unwind_deferred ;struct pmu_event_list {raw_spinlock_t lock;
@@ -885,6 +886,9 @@ struct perf_event {struct callback_head pending_unwind_work;struct rcuwait pending_unwind_wait; +struct perf_unwind_deferred *unwind_deferred; +struct list_head unwind_list; +atomic_t event_limit;/* address range filters */
include/uapi/linux/perf_event.h +1 -0
@@ -1248,6 +1248,7 @@ enum perf_event_type {* * struct { * struct perf_event_header header; +* u64 cookie; * u64 nr; * u64 ips[nr]; * struct sample_id sample_id;
kernel/events/core.c +291 -28
403 lines not on this page · raw message
Signature · 1 line
2.47.2
PATCH v14 8/11 perf tools: Minimal CALLCHAIN_DEFERRED support
In reply to Steven RostedtFrom: Namhyung Kim <[email protected]>
Add a new event type for deferred callchains and a new callback for the struct perf_tool. For now it doesn't actually handle the deferred callchains but it just marks the sample if it has the PERF_CONTEXT_ USER_DEFFERED in the callchain array.
At least, perf report can dump the raw data with this change. Actually this requires the next commit to enable attr.defer_callchain, but if you already have a data file, it'll show the following result.
$ perf report -D ... [email protected] [0x40]: event: 22 . . ... raw event: size 64 bytes . 0000: 16 00 00 00 02 00 40 00 02 00 00 00 00 00 00 00 ......@......... . 0010: 00 fe ff ff ff ff ff ff 4b d3 3f 25 45 7f 00 00 ........K.?%E... . 0020: 21 03 00 00 21 03 00 00 43 02 12 ab 05 00 00 00 !...!...C....... . 0030: 00 00 00 00 00 00 00 00 09 00 00 00 00 00 00 00 ................
0 24344920643 0x5fe0 [0x40]: PERF_RECORD_CALLCHAIN_DEFERRED(IP, 0x2): 801/801: 0 [300000001] ... FP chain: nr:2
..... 0: fffffffffffffe00
..... 1: 00007f45253fd34b
: unhandled!
---
Changes since v13: https://lore.kernel.org/[email protected]
- Added back the cookie in the deferred event
- Display the cookie in the -D output
tools/lib/perf/include/perf/event.h | 8 ++++++++ tools/perf/util/event.c | 1 + tools/perf/util/evsel.c | 15 +++++++++++++++ tools/perf/util/machine.c | 1 + tools/perf/util/perf_event_attr_fprintf.c | 1 + tools/perf/util/sample.h | 3 ++- tools/perf/util/session.c | 18 ++++++++++++++++++ tools/perf/util/tool.c | 1 + tools/perf/util/tool.h | 3 ++- 9 files changed, 49 insertions(+), 2 deletions(-)
tools/lib/perf/include/perf/event.h +8 -0
@@ -151,6 +151,13 @@ struct perf_record_switch {__u32 next_prev_tid; }; +struct perf_record_callchain_deferred { +struct perf_event_header header; +__u64 cookie; +__u64 nr; +__u64 ips[]; +}; +struct perf_record_header_attr {struct perf_event_header header;struct perf_event_attr attr;
@@ -505,6 +512,7 @@ union perf_event {struct perf_record_read read;struct perf_record_throttle throttle;struct perf_record_sample sample; +struct perf_record_callchain_deferred callchain_deferred;struct perf_record_bpf_event bpf;struct perf_record_ksymbol ksymbol;struct perf_record_text_poke_event text_poke;
tools/perf/util/event.c +1 -0
@@ -58,6 +58,7 @@ static const char *perf_event__names[] = { [PERF_RECORD_CGROUP] = "CGROUP", [PERF_RECORD_TEXT_POKE] = "TEXT_POKE", [PERF_RECORD_AUX_OUTPUT_HW_ID] = "AUX_OUTPUT_HW_ID", + [PERF_RECORD_CALLCHAIN_DEFERRED] = "CALLCHAIN_DEFERRED", [PERF_RECORD_HEADER_ATTR] = "ATTR", [PERF_RECORD_HEADER_EVENT_TYPE] = "EVENT_TYPE", [PERF_RECORD_HEADER_TRACING_DATA] = "TRACING_DATA",
tools/perf/util/evsel.c +15 -0
@@ -2978,6 +2978,18 @@ int evsel__parse_sample(struct evsel *evsel, union perf_event *event, data->data_src =PERF_MEM_DATA_SRC_NONE ; data->vcpu =-1 ; +if (event->header.type ==PERF_RECORD_CALLCHAIN_DEFERRED ) { +const u64 max_callchain_nr =UINT64_MAX /sizeof (u64); + + data->callchain = (struct ip_callchain *)&event->callchain_deferred.nr; +if (data->callchain->nr > max_callchain_nr) +return -EFAULT ; + +if (evsel->core.attr.sample_id_all) +perf_evsel__parse_id_sample (evsel, event, data); +return 0 ; + } +if (event->header.type !=PERF_RECORD_SAMPLE ) {if (!evsel->core.attr.sample_id_all)return 0 ;
@@ -3108,6 +3120,9 @@ int evsel__parse_sample(struct evsel *evsel, union perf_event *event,if (data->callchain->nr > max_callchain_nr)return -EFAULT ; sz = data->callchain->nr *sizeof (u64); +if (evsel->core.attr.defer_callchain && data->callchain->nr >=1 && + data->callchain->ips[data->callchain->nr -1 ] ==PERF_CONTEXT_USER_DEFERRED ) + data->deferred_callchain = true;OVERFLOW_CHECK (array, sz, max_size); array = (void *)array + sz; }
tools/perf/util/machine.c +1 -0
@@ -2124,6 +2124,7 @@ static int add_callchain_ip(struct thread *thread, *cpumode =PERF_RECORD_MISC_KERNEL ;break ;case PERF_CONTEXT_USER : +case PERF_CONTEXT_USER_DEFERRED : *cpumode =PERF_RECORD_MISC_USER ;break ;default :
tools/perf/util/perf_event_attr_fprintf.c +1 -0
@@ -343,6 +343,7 @@ int perf_event_attr__fprintf(FILE *fp, struct perf_event_attr *attr,PRINT_ATTRf (inherit_thread, p_unsigned);PRINT_ATTRf (remove_on_exec, p_unsigned);PRINT_ATTRf (sigtrap, p_unsigned); +PRINT_ATTRf (defer_callchain, p_unsigned);PRINT_ATTRn ("{ wakeup_events, wakeup_watermark }" , wakeup_events, p_unsigned, false);PRINT_ATTRf (bp_type, p_unsigned);
tools/perf/util/sample.h +2 -1
@@ -108,7 +108,8 @@ struct perf_sample {u16 p_stage_cyc;u16 retire_lat; }; -bool no_hw_idx;/* No hw_idx collected in branch_stack */ +bool no_hw_idx;/* No hw_idx collected in branch_stack */ +bool deferred_callchain;/* Has deferred user callchains */ char insn[MAX_INSN ];void *raw_data;struct ip_callchain *callchain;
tools/perf/util/session.c +18 -0
@@ -715,6 +715,7 @@ static perf_event__swap_op perf_event__swap_ops[] = { [PERF_RECORD_CGROUP] = perf_event__cgroup_swap, [PERF_RECORD_TEXT_POKE] = perf_event__text_poke_swap, [PERF_RECORD_AUX_OUTPUT_HW_ID] = perf_event__all64_swap, + [PERF_RECORD_CALLCHAIN_DEFERRED] = perf_event__all64_swap, [PERF_RECORD_HEADER_ATTR] = perf_event__hdr_attr_swap, [PERF_RECORD_HEADER_EVENT_TYPE] = perf_event__event_type_swap, [PERF_RECORD_HEADER_TRACING_DATA] = perf_event__tracing_data_swap,
@@ -1118,6 +1119,20 @@ static void dump_sample(struct evsel *evsel, union perf_event *event,sample_read__printf (sample, evsel->core.attr.read_format); } +static void dump_deferred_callchain (struct evsel *evsel,union perf_event *event, +struct perf_sample *sample) +{ +if (!dump_trace) +return ; + +printf ("(IP, 0x%x): %d/%d: %#" PRIx64" [%llx]\n" , + event->header.misc, sample->pid, sample->tid, sample->ip, + event->callchain_deferred.cookie); + +if (evsel__has_callchain (evsel)) +callchain__printf (evsel, sample); +} +static void dump_read (struct evsel *evsel,union perf_event *event) {struct perf_record_read *read_event = &event->read;
@@ -1348,6 +1363,9 @@ static int machines__deliver_event(struct machines *machines,return tool->text_poke (tool, event, sample, machine);case PERF_RECORD_AUX_OUTPUT_HW_ID :return tool->aux_output_hw_id (tool, event, sample, machine); +case PERF_RECORD_CALLCHAIN_DEFERRED : +dump_deferred_callchain (evsel, event, sample); +return tool->callchain_deferred (tool, event, sample, evsel, machine);default : ++evlist->stats.nr_unknown_events;return -1 ;
tools/perf/util/tool.c +1 -0
@@ -266,6 +266,7 @@ void perf_tool__init(struct perf_tool *tool, bool ordered_events) tool->read = process_event_sample_stub; tool->throttle = process_event_stub; tool->unthrottle = process_event_stub; + tool->callchain_deferred = process_event_sample_stub; tool->attr = process_event_synth_attr_stub; tool->event_update = process_event_synth_event_update_stub; tool->tracing_data = process_event_synth_tracing_data_stub;
tools/perf/util/tool.h +2 -1
@@ -42,7 +42,8 @@ enum show_feature_header {struct perf_tool {event_sample sample, - read; + read, + callchain_deferred;event_op mmap, mmap2, comm,
Signature · 1 line
2.47.2
PATCH v14 9/11 perf record: Enable defer_callchain for user callchains
In reply to Steven RostedtFrom: Namhyung Kim <[email protected]>
And add the missing feature detection logic to clear the flag on old kernels.
$ perf record -g -vv true
...
------------------------------------------------------------
perf_event_attr:
type 0 (PERF_TYPE_HARDWARE)
size 136
config 0 (PERF_COUNT_HW_CPU_CYCLES)
{ sample_period, sample_freq } 4000
sample_type IP|TID|TIME|CALLCHAIN|PERIOD
read_format ID|LOST
disabled 1
inherit 1
mmap 1
comm 1
freq 1
enable_on_exec 1
task 1
sample_id_all 1
mmap2 1
comm_exec 1
ksymbol 1
bpf_event 1
defer_callchain 1
------------------------------------------------------------
sys_perf_event_open: pid 162755 cpu 0 group_fd -1 flags 0x8
sys_perf_event_open failed, error -22
switching off deferred callchain support--- tools/perf/util/evsel.c | 24 ++++++++++++++++++++++++ tools/perf/util/evsel.h | 1 + 2 files changed, 25 insertions(+)
tools/perf/util/evsel.c +24 -0
@@ -1076,6 +1076,14 @@ static void __evsel__config_callchain(struct evsel *evsel, struct record_opts *o } } +if (param->record_mode ==CALLCHAIN_FP && !attr->exclude_callchain_user) { +/* +* Enable deferred callchains optimistically. It'll be switched +* off later if the kernel doesn't support it. +*/ + attr->defer_callchain =1 ; + } +if (function) {pr_info ("Disabling user space callchains for function trace event.\n" ); attr->exclude_callchain_user =1 ;
@@ -2124,6 +2132,8 @@ static int __evsel__prepare_open(struct evsel *evsel, struct perf_cpu_map *cpus,static void evsel__disable_missing_features (struct evsel *evsel) { +if (perf_missing_features.defer_callchain) + evsel->core.attr.defer_callchain =0 ;if (perf_missing_features.inherit_sample_read && evsel->core.attr.inherit && (evsel->core.attr.sample_type &PERF_SAMPLE_READ )) evsel->core.attr.inherit =0 ;
@@ -2398,6 +2408,15 @@ static bool evsel__detect_missing_features(struct evsel *evsel, struct perf_cpu/* Please add new feature detection here. */ + attr.defer_callchain = true; + attr.sample_type =PERF_SAMPLE_CALLCHAIN ; +if (has_attr_feature (&attr,/*flags=*/ 0 )) + goto found; + perf_missing_features.defer_callchain = true; +pr_debug2 ("switching off deferred callchain support\n" ); + attr.defer_callchain = false; + attr.sample_type =0 ; + attr.inherit = true; attr.sample_type =PERF_SAMPLE_READ ;if (has_attr_feature (&attr,/*flags=*/ 0 ))
@@ -2509,6 +2528,11 @@ static bool evsel__detect_missing_features(struct evsel *evsel, struct perf_cpu errno = old_errno; check: +if (evsel->core.attr.defer_callchain && + evsel->core.attr.sample_type &PERF_SAMPLE_CALLCHAIN && + perf_missing_features.defer_callchain) +return true; +if (evsel->core.attr.inherit && (evsel->core.attr.sample_type &PERF_SAMPLE_READ ) && perf_missing_features.inherit_sample_read)
tools/perf/util/evsel.h +1 -0
@@ -221,6 +221,7 @@ struct perf_missing_features {bool branch_counters;bool aux_action;bool inherit_sample_read; +bool defer_callchain; };extern struct perf_missing_features perf_missing_features;
Signature · 1 line
2.47.2
PATCH v14 10/11 perf script: Display PERF_RECORD_CALLCHAIN_DEFERRED
In reply to Steven RostedtFrom: Namhyung Kim <[email protected]>
Handle the deferred callchains in the script output.
$ perf script
perf 801 [000] 18.031793: 1 cycles:P:
ffffffff91a14c36 __intel_pmu_enable_all.isra.0+0x56 ([kernel.kallsyms])
ffffffff91d373e9 perf_ctx_enable+0x39 ([kernel.kallsyms])
ffffffff91d36af7 event_function+0xd7 ([kernel.kallsyms])
ffffffff91d34222 remote_function+0x42 ([kernel.kallsyms])
ffffffff91c1ebe1 generic_exec_single+0x61 ([kernel.kallsyms])
ffffffff91c1edac smp_call_function_single+0xec ([kernel.kallsyms])
ffffffff91d37a9d event_function_call+0x10d ([kernel.kallsyms])
ffffffff91d33557 perf_event_for_each_child+0x37 ([kernel.kallsyms])
ffffffff91d47324 _perf_ioctl+0x204 ([kernel.kallsyms])
ffffffff91d47c43 perf_ioctl+0x33 ([kernel.kallsyms])
ffffffff91e2f216 __x64_sys_ioctl+0x96 ([kernel.kallsyms])
ffffffff9265f1ae do_syscall_64+0x9e ([kernel.kallsyms])
ffffffff92800130 entry_SYSCALL_64+0xb0 ([kernel.kallsyms])perf 801 [000] 18.031814: DEFERRED CALLCHAIN
7fb5fc22034b __GI___ioctl+0x3b (/usr/lib/x86_64-linux-gnu/libc.so.6)
--- tools/perf/builtin-script.c | 89 +++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+)
tools/perf/builtin-script.c +89 -0
@@ -2483,6 +2483,93 @@ static int process_sample_event(const struct perf_tool *tool,return ret; } +static int process_deferred_sample_event (const struct perf_tool *tool, +union perf_event *event, +struct perf_sample *sample, +struct evsel *evsel, +struct machine *machine) +{ + struct perf_script *scr = container_of(tool, struct perf_script, tool); +struct perf_event_attr *attr = &evsel->core.attr; +struct evsel_script *es = evsel->priv; +unsigned int type =output_type (attr->type); +struct addr_location al; +FILE *fp = es->fp; +int ret =0 ; + +if (output[type].fields ==0 ) +return 0 ; + +/* Set thread to NULL to indicate addr_al and al are not initialized */ +addr_location__init (&al); + +if (perf_time__ranges_skip_sample (scr->ptime_range, scr->range_num, + sample->time)) { + goto out_put; + } + +if (debug_mode) { +if (sample->time < last_timestamp) { +pr_err ("Samples misordered, previous: %" PRIu64 +" this: %" PRIu64"\n" , last_timestamp, + sample->time); + nr_unordered++; + } + last_timestamp = sample->time; + goto out_put; + } + +if (filter_cpu (sample)) + goto out_put; + +if (machine__resolve (machine, &al, sample) <0 ) { +pr_err ("problem processing %d event, skipping it.\n" , + event->header.type); + ret =-1 ; + goto out_put; + } + +if (al.filtered) + goto out_put; + +if (!show_event (sample, evsel, al.thread, &al,NULL )) + goto out_put; + +if (evswitch__discard (&scr->evswitch, evsel)) + goto out_put; + +perf_sample__fprintf_start (scr, sample, al.thread, evsel, +PERF_RECORD_CALLCHAIN_DEFERRED , fp); +fprintf (fp,"DEFERRED CALLCHAIN" ); + +if (PRINT_FIELD (IP )) { +struct callchain_cursor *cursor =NULL ; + +if (symbol_conf.use_callchain && sample->callchain) { + cursor =get_tls_callchain_cursor (); +if (thread__resolve_callchain (al.thread, cursor, evsel, + sample,NULL ,NULL , + scripting_max_stack)) { +pr_info ("cannot resolve deferred callchains\n" ); + cursor =NULL ; + } + } + +fputc (cursor ?'\n' :' ' , fp); +sample__fprintf_sym (sample, &al,0 , output[type].print_ip_opts, + cursor, symbol_conf.bt_stop_list, fp); + } + +fprintf (fp,"\n" ); + +if (verbose >0 ) +fflush (fp); + +out_put: +addr_location__exit (&al); +return ret; +} +// Used when scr->per_event_dump is not set static struct evsel_script es_stdout;
@@ -4069,6 +4156,7 @@ int cmd_script(int argc, const char **argv)perf_tool__init (&script.tool, !unsorted_dump); script.tool.sample = process_sample_event; + script.tool.callchain_deferred = process_deferred_sample_event; script.tool.mmap = perf_event__process_mmap; script.tool.mmap2 = perf_event__process_mmap2; script.tool.comm = perf_event__process_comm;
@@ -4095,6 +4183,7 @@ int cmd_script(int argc, const char **argv) script.tool.throttle = process_throttle_event; script.tool.unthrottle = process_throttle_event; script.tool.ordering_requires_timestamps = true; + script.tool.merge_deferred_callchains = false; session =perf_session__new (&data, &script.tool);if (IS_ERR (session))return PTR_ERR (session);
Signature · 1 line
2.47.2
PATCH v14 11/11 perf tools: Merge deferred user callchains
In reply to Steven RostedtFrom: Namhyung Kim <[email protected]>
Save samples with deferred callchains in a separate list and deliver them after merging the user callchains. If users don't want to merge they can set tool->merge_deferred_callchains to false to prevent the behavior.
With previous result, now perf script will show the merged callchains.
$ perf script
perf 801 [000] 18.031793: 1 cycles:P:
ffffffff91a14c36 __intel_pmu_enable_all.isra.0+0x56 ([kernel.kallsyms])
ffffffff91d373e9 perf_ctx_enable+0x39 ([kernel.kallsyms])
ffffffff91d36af7 event_function+0xd7 ([kernel.kallsyms])
ffffffff91d34222 remote_function+0x42 ([kernel.kallsyms])
ffffffff91c1ebe1 generic_exec_single+0x61 ([kernel.kallsyms])
ffffffff91c1edac smp_call_function_single+0xec ([kernel.kallsyms])
ffffffff91d37a9d event_function_call+0x10d ([kernel.kallsyms])
ffffffff91d33557 perf_event_for_each_child+0x37 ([kernel.kallsyms])
ffffffff91d47324 _perf_ioctl+0x204 ([kernel.kallsyms])
ffffffff91d47c43 perf_ioctl+0x33 ([kernel.kallsyms])
ffffffff91e2f216 __x64_sys_ioctl+0x96 ([kernel.kallsyms])
ffffffff9265f1ae do_syscall_64+0x9e ([kernel.kallsyms])
ffffffff92800130 entry_SYSCALL_64+0xb0 ([kernel.kallsyms])
7fb5fc22034b __GI___ioctl+0x3b (/usr/lib/x86_64-linux-gnu/libc.so.6)
...The old output can be get using --no-merge-callchain option. Also perf report can get the user callchain entry at the end.
$ perf report --no-children --percent-limit=0 --stdio -q -S __intel_pmu_enable_all.isra.0
# symbol: __intel_pmu_enable_all.isra.0
0.00% perf [kernel.kallsyms]
|
---__intel_pmu_enable_all.isra.0
perf_ctx_enable
event_function
remote_function
generic_exec_single
smp_call_function_single
event_function_call
perf_event_for_each_child
_perf_ioctl
perf_ioctl
__x64_sys_ioctl
do_syscall_64
entry_SYSCALL_64
__GI___ioctl--- tools/perf/Documentation/perf-script.txt | 5 ++ tools/perf/builtin-script.c | 5 +- tools/perf/util/callchain.c | 24 +++++++++ tools/perf/util/callchain.h | 3 ++ tools/perf/util/evlist.c | 1 + tools/perf/util/evlist.h | 1 + tools/perf/util/session.c | 63 +++++++++++++++++++++++- tools/perf/util/tool.c | 1 + tools/perf/util/tool.h | 1 + 9 files changed, 102 insertions(+), 2 deletions(-)
tools/perf/Documentation/perf-script.txt +5 -0
@@ -527,6 +527,11 @@ include::itrace.txt[] The known limitations include exception handing such as setjmp/longjmp will have calls/returns not match. +--merge-callchains:: + Enable merging deferred user callchains if available. This is the + default behavior. If you want to see separate CALLCHAIN_DEFERRED + records for some reason, use --no-merge-callchains explicitly. + :GMEXAMPLECMD: script :GMEXAMPLESUBCMD: include::guest-files.txt[]
tools/perf/builtin-script.c +4 -1
@@ -3775,6 +3775,7 @@ int cmd_script(int argc, const char **argv)bool header_only = false;bool script_started = false;bool unsorted_dump = false; +bool merge_deferred_callchains = true;char *rec_script_path =NULL ;char *rep_script_path =NULL ;struct perf_session *session;
@@ -3928,6 +3929,8 @@ int cmd_script(int argc, const char **argv) "Guest code can be found in hypervisor process"),OPT_BOOLEAN ('\0' ,"stitch-lbr" , &script.stitch_lbr,"Enable LBR callgraph stitching approach" ), +OPT_BOOLEAN ('\0' ,"merge-callchains" , &merge_deferred_callchains, +"Enable merge deferred user callchains" ),OPTS_EVSWITCH (&script.evswitch), OPT_END() };
@@ -4183,7 +4186,7 @@ int cmd_script(int argc, const char **argv) script.tool.throttle = process_throttle_event; script.tool.unthrottle = process_throttle_event; script.tool.ordering_requires_timestamps = true; - script.tool.merge_deferred_callchains = false; + script.tool.merge_deferred_callchains = merge_deferred_callchains; session =perf_session__new (&data, &script.tool);if (IS_ERR (session))return PTR_ERR (session);
tools/perf/util/callchain.c +24 -0
@@ -1828,3 +1828,27 @@ int sample__for_each_callchain_node(struct thread *thread, struct evsel *evsel, }return 0 ; } + +int sample__merge_deferred_callchain (struct perf_sample *sample_orig, +struct perf_sample *sample_callchain) +{ +u64 nr_orig = sample_orig->callchain->nr -1 ; +u64 nr_deferred = sample_callchain->callchain->nr; +struct ip_callchain *callchain; + + callchain =calloc (1 + nr_orig + nr_deferred,sizeof (u64)); +if (callchain ==NULL ) { + sample_orig->deferred_callchain = false; +return -ENOMEM ; + } + + callchain->nr = nr_orig + nr_deferred; +/* copy except for the last PERF_CONTEXT_USER_DEFERRED */ +memcpy (callchain->ips, sample_orig->callchain->ips, nr_orig *sizeof (u64)); +/* copy deferred use callchains */ +memcpy (&callchain->ips[nr_orig], sample_callchain->callchain->ips, + nr_deferred *sizeof (u64)); + + sample_orig->callchain = callchain; +return 0 ; +}
tools/perf/util/callchain.h +3 -0
@@ -317,4 +317,7 @@ int sample__for_each_callchain_node(struct thread *thread, struct evsel *evsel, struct perf_sample *sample, int max_stack, bool symbols, callchain_iter_fn cb, void *data); +int sample__merge_deferred_callchain (struct perf_sample *sample_orig, +struct perf_sample *sample_callchain); +#endif /* __PERF_CALLCHAIN_H */
tools/perf/util/evlist.c +1 -0
@@ -83,6 +83,7 @@ void evlist__init(struct evlist *evlist, struct perf_cpu_map *cpus, evlist->ctl_fd.ack =-1 ; evlist->ctl_fd.pos =-1 ; evlist->nr_br_cntr =-1 ; +INIT_LIST_HEAD (&evlist->deferred_samples); }struct evlist *evlist__new (void )
tools/perf/util/evlist.h +1 -0
@@ -86,6 +86,7 @@ struct evlist {int pos;/* index at evlist core object to check signals */ } ctl_fd;struct event_enable_timer *eet; +struct list_head deferred_samples; };struct evsel_str_handler {
tools/perf/util/session.c +62 -1
81 lines not on this page · raw message
tools/perf/util/tool.c +1 -0
@@ -245,6 +245,7 @@ void perf_tool__init(struct perf_tool *tool, bool ordered_events) tool->cgroup_events = false; tool->no_warn = false; tool->show_feat_hdr =SHOW_FEAT_NO_HEADER ; + tool->merge_deferred_callchains = true; tool->sample = process_event_sample_stub; tool->mmap = process_event_stub;
tools/perf/util/tool.h +1 -0
@@ -87,6 +87,7 @@ struct perf_tool {bool cgroup_events;bool no_warn;bool dont_split_sample_group; +bool merge_deferred_callchains;enum show_feature_header show_feat_hdr; };
Signature · 1 line
2.47.2