perf: add logic to collect off-cpu samples
cover perf: add logic to collect off-cpu samples
Add --off-cpu-kernel option to capture off-cpu sample alongwith on-cpu samples.
off-cpu samples represent time spent by task when it was on wait queue (schedule out to waiting for events, blocked on I/O, locks, timers, paging/swapping, etc)
Refer following links for more details:
https://lpc.events/event/17/contributions/1556/
https://www.youtube.com/watch?v=sF2faKGRnjs
Ajay Kaher (3):
perf/core: add logic to collect off-cpu sample
perf/record: add options --off-cpu-kernel
perf/report: add off-cpu samples
include/linux/perf_event.h | 16 ++++++++++++++ include/uapi/linux/perf_event.h | 3 ++- kernel/events/core.c | 27 ++++++++++++++++++----- tools/include/uapi/linux/perf_event.h | 3 ++- tools/perf/builtin-record.c | 2 ++ tools/perf/util/events_stats.h | 2 ++ tools/perf/util/evsel.c | 4 ++++ tools/perf/util/hist.c | 31 ++++++++++++++++++++++++--- tools/perf/util/hist.h | 1 + tools/perf/util/record.h | 1 + tools/perf/util/sample.h | 1 + 11 files changed, 81 insertions(+), 10 deletions(-)
Signature · 1 line
2.39.0
RFC PATCH 1/3 perf/core: add logic to collect off-cpu sample
following logics has been added to collect the off-cpu sample:
- 'task_pt_regs(current)' has been used to capture registers status off-cpu sample.
- off-cpu time represent the time period for which the target process not occupying the cpu cycles. And calculate as:
off-cpu time = swap-in time - swap-out time
--- include/linux/perf_event.h | 16 ++++++++++++++++ include/uapi/linux/perf_event.h | 3 ++- kernel/events/core.c | 27 ++++++++++++++++++++++----- tools/include/uapi/linux/perf_event.h | 3 ++- 4 files changed, 42 insertions(+), 7 deletions(-)
include/linux/perf_event.h +16 -0
@@ -599,6 +599,11 @@ enum perf_event_state { PERF_EVENT_STATE_ACTIVE = 1, }; +enum perf_sample_cpu { +PERF_SAMPLE_ON_CPU , +PERF_SAMPLE_OFF_CPU , +}; +struct file ;struct perf_sample_data ;
@@ -828,6 +833,7 @@ struct perf_event {void *security;#endif struct list_head sb_list; +u64 stop_time;/* * Certain events gets forwarded to another pmu internally by over-
@@ -1301,6 +1307,16 @@ static inline u32 perf_sample_data_size(struct perf_sample_data *data,return size; } +static inline void perf_sample_set_off_cpu (struct perf_sample_data *data) +{ + data->cpu_entry.reserved =PERF_SAMPLE_OFF_CPU ; +} + +static inline void perf_sample_set_on_cpu (struct perf_sample_data *data) +{ + data->cpu_entry.reserved =PERF_SAMPLE_ON_CPU ; +} +/* * Clear all bitfields in the perf_branch_entry. * The to and from fields are not cleared because they are
include/uapi/linux/perf_event.h +2 -1
@@ -460,7 +460,8 @@ struct perf_event_attr { inherit_thread : 1, /* children only inherit if cloned with CLONE_THREAD */ remove_on_exec : 1, /* event is removed from task on exec */ sigtrap : 1, /* send synchronous SIGTRAP on event */ - __reserved_1 : 26; + off_cpu : 1, /* include off_cpu sample */ + __reserved_1 : 25;union {__u32 wakeup_events;/* wakeup every n events */
kernel/events/core.c +22 -5
@@ -7168,10 +7168,8 @@ static void __perf_event_header__init_id(struct perf_sample_data *data,if (sample_type &PERF_SAMPLE_STREAM_ID ) data->stream_id = event->id; -if (sample_type &PERF_SAMPLE_CPU ) { - data->cpu_entry.cpu =raw_smp_processor_id (); - data->cpu_entry.reserved =0 ; - } +if (sample_type &PERF_SAMPLE_CPU ) + data->cpu_entry.cpu =raw_smp_processor_id (); } void perf_event_header__init_id(struct perf_event_header *header,
@@ -11083,8 +11081,8 @@ static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)return HRTIMER_NORESTART ; event->pmu->read (event); -perf_sample_data_init (&data,0 , event->hw.last_period); +perf_sample_set_on_cpu (&data); regs =get_irq_regs ();if (regs && !perf_exclude_event (event, regs)) {
@@ -11099,6 +11097,18 @@ static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)return ret; } +static void perf_swevent_offCPUSample (struct perf_event *event,u64 period) +{ +struct perf_sample_data data; +struct pt_regs *regs; + + event->pmu->read (event); +perf_sample_data_init (&data,0 , period); +perf_sample_set_off_cpu (&data); + regs =task_pt_regs (current); +__perf_event_overflow (event,1 , &data, regs); +} +static void perf_swevent_start_hrtimer (struct perf_event *event) {struct hw_perf_event *hwc = &event->hw;
@@ -11107,6 +11117,11 @@ static void perf_swevent_start_hrtimer(struct perf_event *event)if (!is_sampling_event (event))return ; +if (event->attr.off_cpu && event->stop_time && hwc->sample_period) { +perf_swevent_offCPUSample (event,perf_clock () - event->stop_time); + event->stop_time =0 ; + } + period =local64_read (&hwc->period_left);if (period) {if (period <0 )
@@ -11128,6 +11143,7 @@ static void perf_swevent_cancel_hrtimer(struct perf_event *event)ktime_t remaining =hrtimer_get_remaining (&hwc->hrtimer);local64_set (&hwc->period_left,ktime_to_ns (remaining)); + event->stop_time =perf_clock ();hrtimer_cancel (&hwc->hrtimer); } }
@@ -11139,6 +11155,7 @@ static void perf_swevent_init_hrtimer(struct perf_event *event)if (!is_sampling_event (event))return ; + event->stop_time =0 ;hrtimer_init (&hwc->hrtimer,CLOCK_MONOTONIC ,HRTIMER_MODE_REL_HARD ); hwc->hrtimer.function = perf_swevent_hrtimer;
tools/include/uapi/linux/perf_event.h +2 -1
@@ -460,7 +460,8 @@ struct perf_event_attr { inherit_thread : 1, /* children only inherit if cloned with CLONE_THREAD */ remove_on_exec : 1, /* event is removed from task on exec */ sigtrap : 1, /* send synchronous SIGTRAP on event */ - __reserved_1 : 26; + off_cpu : 1, /* include off_cpu sample */ + __reserved_1 : 25;union {__u32 wakeup_events;/* wakeup every n events */
Signature · 1 line
2.39.0
Peter Zijlstra
maintainerRe: [PATCH RFC 1/3] perf/core: add logic to collect off-cpu sample
Quoted 10 lines · Ajay Kaher
following logics has been added to collect the off-cpu sample:
- '
task_pt_regs(current)' has been used to capture registers status off-cpu sample.- off-cpu time represent the time period for which the target process not occupying the cpu cycles. And calculate as:
off-cpu time = swap-in time - swap-out time
I have absolutely no idea what you're trying to do :/ The above does not constitute a comprehensible Changelog.
Re: [PATCH RFC 1/3] perf/core: add logic to collect off-cpu sample
Quoted 7 lines · Peter Zijlstra
Quoted 5 lines · Ajay Kaher
following logics has been added to collect the off-cpu sample:
- '
task_pt_regs(current)' has been used to capture registers status off-cpu sample.
Quoted 8 lines
Quoted 5 lines
- off-cpu time represent the time period for which the target process not occupying the cpu cycles. And calculate as:
off-cpu time = swap-in time - swap-out time
I have absolutely no idea what you're trying to do :/ The above does not constitute a comprehensible Changelog.
Sorry Peter, it’s sched-in/out (not swap-in/out).
'Perf record' captures on-cpu samples at frequency which is specified by the user ( i.e. time period to collect sample is NSEC_PER_SEC / freq).
This patch is to collect the off_cpu sample and time period of off_cpu sample is calculated based upon the time when target task was sched_out to sched_in, as:
off-cpu time period = sched_in time - sched-out time
-Ajay
Peter Zijlstra
maintainerRe: [PATCH RFC 1/3] perf/core: add logic to collect off-cpu sample
Quoted 19 lines · Ajay Kaher
Quoted 7 lines · Peter Zijlstra
Quoted 5 lines · Ajay Kaher
following logics has been added to collect the off-cpu sample:
- '
task_pt_regs(current)' has been used to capture registers status off-cpu sample.Quoted 8 lines
Quoted 5 lines
- off-cpu time represent the time period for which the target process not occupying the cpu cycles. And calculate as:
off-cpu time = swap-in time - swap-out time
I have absolutely no idea what you're trying to do :/ The above does not constitute a comprehensible Changelog.
Sorry Peter, it’s sched-in/out (not swap-in/out).
Quoted 8 lines
'Perf record' captures on-cpu samples at frequency which is specified by the user ( i.e. time period to collect sample is
NSEC_PER_SEC/ freq).This patch is to collect the
off_cpusample and time period ofoff_cpusample is calculated based upon the time when target task wassched_outtosched_in, as:off-cpu time period =
sched_intime - sched-out time
But but but... you don't need anything new for that. The sched_wakeup tracepoint should generate an event in both the task/cpu that does the wakeup *AND* the task being woken.
So sched_switch + sched_wakeup should get you all this already. What am I missing?
RFC PATCH 2/3 perf/record: add options --off-cpu-kernel
In reply to Ajay Kaher--off-cpu-kernel to collect off cpu samples using on/off cpu time.
--- tools/perf/builtin-record.c | 2 ++ tools/perf/util/evsel.c | 2 ++ tools/perf/util/record.h | 1 + 3 files changed, 5 insertions(+)
tools/perf/builtin-record.c +2 -0
@@ -3571,6 +3571,8 @@ static struct option __record_options[] = { "write collected trace data into several data files using parallel threads", record__parse_threads),OPT_BOOLEAN (0 ,"off-cpu" , &record.off_cpu,"Enable off-cpu analysis" ), +OPT_BOOLEAN (0 ,"off-cpu-kernel" , &record.opts.off_cpu_kernel, +"Enable kernel based off-cpu analysis" ), OPT_END() };
tools/perf/util/evsel.c +2 -0
@@ -862,6 +862,8 @@ static void __evsel__config_callchain(struct evsel *evsel, struct record_opts *o attr->exclude_callchain_user =1 ;if (opts->user_callchains) attr->exclude_callchain_kernel =1 ; +if (opts->off_cpu_kernel) + attr->off_cpu =1 ;if (param->record_mode ==CALLCHAIN_LBR ) {if (!opts->branch_stack) {if (attr->exclude_user) {
tools/perf/util/record.h +1 -0
@@ -52,6 +52,7 @@ struct record_opts {bool kcore;bool text_poke;bool build_id; +bool off_cpu_kernel;unsigned int freq;unsigned int mmap_pages;unsigned int auxtrace_mmap_pages;
Signature · 1 line
2.39.0
RFC PATCH 3/3 perf/report: add off-cpu samples
In reply to Ajay Kaheroff-cpu samples represent the time period for which the target process not occupying the cpu cycles.
In following example, perf has collected 15 off-cpu samples and program was running on cpu for 27%:
Samples: 24 of 'task-clock:ppp', 15 of 'offcpu', Event count: ~9150831908 (73% offcpu)
+73.77% 73.77% a.out libc.so.6 [.] clock_nanosleep <-- off-cpu sample +24.04% 24.04% a.out [vdso] [.] __vdso_gettimeofday <-- on-cpu sample
--- tools/perf/util/events_stats.h | 2 ++ tools/perf/util/evsel.c | 2 ++ tools/perf/util/hist.c | 31 ++++++++++++++++++++++++++++--- tools/perf/util/hist.h | 1 + tools/perf/util/sample.h | 1 + 5 files changed, 34 insertions(+), 3 deletions(-)
tools/perf/util/events_stats.h +2 -0
@@ -44,8 +44,10 @@ struct events_stats {struct hists_stats {u64 total_period; +u64 total_period_off_cpu;u64 total_non_filtered_period;u32 nr_samples; +u64 nr_samples_off_cpu;u32 nr_non_filtered_samples;u32 nr_lost_samples; };
tools/perf/util/evsel.c +2 -0
@@ -1146,6 +1146,7 @@ void evsel__config(struct evsel *evsel, struct record_opts *opts, attr->write_backward = opts->overwrite ?1 :0 ; attr->read_format =PERF_FORMAT_LOST ; +evsel__set_sample_bit (evsel,CPU );evsel__set_sample_bit (evsel,IP );evsel__set_sample_bit (evsel,TID );
@@ -2438,6 +2439,7 @@ int evsel__parse_sample(struct evsel *evsel, union perf_event *event, u.val32[0 ] =bswap_32 (u.val32[0 ]); } + data->off_cpu = u.val32[1 ]; data->cpu = u.val32[0 ]; array++; }
tools/perf/util/hist.c +28 -3
@@ -23,6 +23,7 @@#include "thread.h" #include "block-info.h" #include "ui/progress.h" +#include "ui/util.h" #include <errno.h> #include <math.h> #include <inttypes.h>
@@ -725,6 +726,7 @@ __hists__add_entry(struct hists *hists, .socket = al->socket, .cpu = al->cpu, .cpumode = al->cpumode, + .off_cpu = sample->off_cpu, .ip = al->addr, .level = al->level, .code_page_size = sample->code_page_size,
@@ -1076,6 +1078,8 @@ iter_add_single_cumulative_entry(struct hist_entry_iter *iter,callchain_cursor_commit (get_tls_callchain_cursor ());hists__inc_nr_samples (hists, he->filtered); +if (sample->off_cpu) + ++hists->stats.nr_samples_off_cpu;return err; }
@@ -1740,6 +1744,7 @@ void hists__reset_stats(struct hists *hists) { hists->nr_entries =0 ; hists->stats.total_period =0 ; + hists->stats.total_period_off_cpu =0 ;hists__reset_filter_stats (hists); }
@@ -1757,6 +1762,9 @@ void hists__inc_stats(struct hists *hists, struct hist_entry *h) hists->nr_entries++; hists->stats.total_period += h->stat.period; + +if (h->off_cpu) + hists->stats.total_period_off_cpu += h->stat.period; }static void hierarchy_recalc_total_periods (struct hists *hists)
@@ -2745,14 +2753,20 @@ int __hists__scnprintf_title(struct hists *hists, char *bf, size_t size, bool shstruct thread *thread = hists->thread_filter;int socket_id = hists->socket_filter;unsigned long nr_samples = hists->stats.nr_samples; +unsigned long nr_samples_off_cpu = hists->stats.nr_samples_off_cpu;u64 nr_events = hists->stats.total_period; +int nr_events_off_cpu_percentage = (hists->stats.total_period_off_cpu *100 ) / nr_events;struct evsel *evsel =hists_to_evsel (hists);const char *ev_name =evsel__name (evsel);char buf[512 ], sample_freq_str[64 ] ="" ; +char oncpu_str[128 ] ="" ; +char offcpu_str[128 ] ="" ; +char offcpu_percentage_str[128 ] ="" ;size_t buflen =sizeof (buf);char ref[30 ] =" show reference callgraph, " ;bool enable_ref = false; +if (symbol_conf.filter_relative) { nr_samples = hists->stats.nr_non_filtered_samples; nr_events = hists->stats.total_non_filtered_period;
@@ -2785,10 +2799,21 @@ int __hists__scnprintf_title(struct hists *hists, char *bf, size_t size, bool shscnprintf (sample_freq_str,sizeof (sample_freq_str)," %d Hz," , evsel->core.attr.sample_freq); nr_samples =convert_unit (nr_samples, &unit); + +scnprintf (oncpu_str,sizeof (oncpu_str),"%lu%c of '%s'," , + nr_samples - nr_samples_off_cpu, unit, ev_name); + +if (evsel->core.attr.off_cpu) { +scnprintf (offcpu_str,sizeof (offcpu_str),"%lu%c of '%s'," , + nr_samples_off_cpu, unit,"offcpu" ); +scnprintf (offcpu_percentage_str,sizeof (offcpu_percentage_str), +"(%d%% offcpu)" , nr_events_off_cpu_percentage); + } + printed =scnprintf (bf, size, -"Samples: %lu%c of event%s '%s',%s%sEvent count (approx.): %" PRIu64, - nr_samples, unit, evsel->core.nr_members >1 ?"s" :"" , - ev_name, sample_freq_str, enable_ref ? ref :" " , nr_events); +"Samples: %s %s %s%sEvent count: ~%" PRIu64" %s" , + oncpu_str, offcpu_str, sample_freq_str, enable_ref ? ref :" " , + nr_events, offcpu_percentage_str);if (hists->uid_filter_str)
tools/perf/util/hist.h +1 -0
@@ -236,6 +236,7 @@ struct hist_entry {/* We are added by hists__add_dummy_entry. */ bool dummy;bool leaf; +bool off_cpu;char level;u8 filtered;
tools/perf/util/sample.h +1 -0
@@ -109,6 +109,7 @@ struct perf_sample {u16 retire_lat; };bool no_hw_idx;/* No hw_idx collected in branch_stack */ +bool off_cpu;char insn[MAX_INSN ];void *raw_data;struct ip_callchain *callchain;
Signature · 1 line
2.39.0
Ian Rogers
reviewerQuoted 11 lines · Ajay Kaher
Add --off-cpu-kernel option to capture off-cpu sample alongwith on-cpu samples.
off-cpu samples represent time spent by task when it was on wait queue (schedule out to waiting for events, blocked on I/O, locks, timers, paging/swapping, etc)
Refer following links for more details:
https://lpc.events/event/17/contributions/1556/
https://www.youtube.com/watch?v=sF2faKGRnjs
Hi Ajay,
I wonder if Howard's improvements (not landed) for perf record --off-cpu would solve this problem for you?
https://lore.kernel.org/lkml/[email protected]/
Or is that approach problematic due to the use of BPF?
Thanks,
Ian
Quoted 21 lines
Ajay Kaher (3):
perf/core: add logic to collect off-cpu sample
perf/record: add options --off-cpu-kernel
perf/report: add off-cpu samplesinclude/linux/perf_event.h | 16 ++++++++++++++ include/uapi/linux/perf_event.h | 3 ++- kernel/events/core.c | 27 ++++++++++++++++++----- tools/include/uapi/linux/perf_event.h | 3 ++- tools/perf/builtin-record.c | 2 ++ tools/perf/util/events_stats.h | 2 ++ tools/perf/util/evsel.c | 4 ++++ tools/perf/util/hist.c | 31 ++++++++++++++++++++++++--- tools/perf/util/hist.h | 1 + tools/perf/util/record.h | 1 + tools/perf/util/sample.h | 1 + 11 files changed, 81 insertions(+), 10 deletions(-)--
2.39.0
Quoted 14 lines · Ian Rogers
Quoted 11 lines · Ajay Kaher
Add --off-cpu-kernel option to capture off-cpu sample alongwith on-cpu samples.
off-cpu samples represent time spent by task when it was on wait queue (schedule out to waiting for events, blocked on I/O, locks, timers, paging/swapping, etc)
Refer following links for more details:
https://lpc.events/event/17/contributions/1556/
https://www.youtube.com/watch?v=sF2faKGRnjs
Quoted 7 lines
Hi Ajay,
I wonder if Howard's improvements (not landed) for
perf record --off-cpuwould solve this problem for you?
https://lore.kernel.org/lkml/[email protected]/
Or is that approach problematic due to the use of BPF?
Thanks Ian for your response and sharing Howard's improvements.
Yes, perf --off-cpu is based upon BPF and having following restrictions:
- target binary should be compiled with frame pointer, same mentioned in tools/perf/Documentation/perf-record.txt:
Note that BPF can collect stack traces using frame pointer ("fp") only, as of now. So the applications built without the frame pointer might see bogus addresses.
- perf should be complied with BUILD_BPF_SKEL=1:
Warning: option `off-cpu' is being ignored because no BUILD_BPF_SKEL=1
- off-cpu, on-cpu samples are not on the same result page. (I guess Howard has improve this, not tried his patches)
I have tried to collect the off-cpu sample same as on-cpu sample with the help of kernel/events/core.c. We will get one off-cpu sample from the target task sched-out to sched-in. Or we can say off-cpu samples are not dependent on frequency provided by the user to perf record.
I am also worried about having so many samples if sched-in/out frequency is high.
Thinking to merge samples if attributes are the same (i.e. stacktrace) and add the
off-cpu period to previous samples with the same attribute.
-Ajay
Quoted 24 lines
Thanks,
IanQuoted 21 lines
Ajay Kaher (3):
perf/core: add logic to collect off-cpu sample
perf/record: add options --off-cpu-kernel
perf/report: add off-cpu samplesinclude/linux/perf_event.h | 16 ++++++++++++++ include/uapi/linux/perf_event.h | 3 ++- kernel/events/core.c | 27 ++++++++++++++++++----- tools/include/uapi/linux/perf_event.h | 3 ++- tools/perf/builtin-record.c | 2 ++ tools/perf/util/events_stats.h | 2 ++ tools/perf/util/evsel.c | 4 ++++ tools/perf/util/hist.c | 31 ++++++++++++++++++++++++--- tools/perf/util/hist.h | 1 + tools/perf/util/record.h | 1 + tools/perf/util/sample.h | 1 + 11 files changed, 81 insertions(+), 10 deletions(-)--
2.39.0
Ian Rogers
reviewerQuoted 26 lines · Ajay Kaher
Quoted 14 lines · Ian Rogers
Quoted 11 lines · Ajay Kaher
Add --off-cpu-kernel option to capture off-cpu sample alongwith on-cpu samples.
off-cpu samples represent time spent by task when it was on wait queue (schedule out to waiting for events, blocked on I/O, locks, timers, paging/swapping, etc)
Refer following links for more details:
https://lpc.events/event/17/contributions/1556/
https://www.youtube.com/watch?v=sF2faKGRnjsQuoted 7 lines
Hi Ajay,
I wonder if Howard's improvements (not landed) for
perf record --off-cpuwould solve this problem for you?
https://lore.kernel.org/lkml/[email protected]/
Or is that approach problematic due to the use of BPF?Thanks Ian for your response and sharing Howard's improvements.
Quoted 7 lines
Yes, perf --off-cpu is based upon BPF and having following restrictions:
- target binary should be compiled with frame pointer, same mentioned in
tools/perf/Documentation/perf-record.txt:
Note that BPF can collect stack traces using frame pointer ("fp") only, as of now. So the applications built without the frame pointer might see bogus addresses.
Agreed, if you want more than the leaf function. There's some evidence that frame pointers may become the default:
https://www.brendangregg.com/blog/2024-03-17/the-return-of-the-frame-pointers.html
Quoted 2 lines
- perf should be complied with BUILD_BPF_SKEL=1:
Warning: option `off-cpu' is being ignored because no BUILD_BPF_SKEL=1
We've made this the default build behavior but probably a bigger issue is that generally to have permissions you need to run as root.
Quoted 8 lines
- off-cpu, on-cpu samples are not on the same result page. (I guess Howard has improve this, not tried his patches)
I have tried to collect the off-cpu sample same as on-cpu sample with the help of
kernel/events/core.c. We will get one off-cpu sample from the target task sched-out to sched-in. Or we can say off-cpu samples are not dependent on frequency provided by the user to perf record.
Quoted 5 lines
I am also worried about having so many samples if sched-in/out frequency is high.
Thinking to merge samples if attributes are the same (i.e. stacktrace) and add the
off-cpu period to previous samples with the same attribute.
Right, this would be useful to look at in the context of Howard's patches. The proposal in his changes are that short off-cpu times are aggregated in a BPF map and dumped at the end of perf record - this matches the existing perf record off-CPU behavior. Longer off-CPU times, where long is open for debate and will be a parameter (perhaps 100 microseconds) and cause a sample to be created. BPF programs create BPF output events, Howard's last patch series would rewrite the events in perf record to be off-CPU samples. At the last office hours it was discussed that we should dump the BPF output events directly, to keep perf record overhead minimal, and add the ability to rewrite into samples in tools like perf report, etc.
Thanks,
Ian
Quoted 26 lines
-Ajay
Quoted 24 lines
Thanks,
IanQuoted 21 lines
Ajay Kaher (3):
perf/core: add logic to collect off-cpu sample
perf/record: add options --off-cpu-kernel
perf/report: add off-cpu samplesinclude/linux/perf_event.h | 16 ++++++++++++++ include/uapi/linux/perf_event.h | 3 ++- kernel/events/core.c | 27 ++++++++++++++++++----- tools/include/uapi/linux/perf_event.h | 3 ++- tools/perf/builtin-record.c | 2 ++ tools/perf/util/events_stats.h | 2 ++ tools/perf/util/evsel.c | 4 ++++ tools/perf/util/hist.c | 31 ++++++++++++++++++++++++--- tools/perf/util/hist.h | 1 + tools/perf/util/record.h | 1 + tools/perf/util/sample.h | 1 + 11 files changed, 81 insertions(+), 10 deletions(-)--
2.39.0