perf stat: Unbreak perf stat with ARMv8 PMU events
cover perf stat: Unbreak perf stat with ARMv8 PMU events
Currently, perf-stat with armv8_pmu events with a workload is broken. This patch set just fixes that.
Before the patch set:
[root@localhost hulk]# tools/perf/perf stat -e armv8_pmuv3_0/ll_cache_rd/,armv8_pmuv3_0/ll_cache_miss_rd/ ls > /dev/null Segmentation fault
After the patch set:
[root@localhost hulk]# tools/perf/perf stat -e armv8_pmuv3_0/ll_cache_rd/,armv8_pmuv3_0/ll_cache_miss_rd/ ls > /dev/null
Performance counter stats for 'ls':
39,882 armv8_pmuv3_0/ll_cache_rd/
9,639 armv8_pmuv3_0/ll_cache_miss_rd/
0.001416690 seconds time elapsed
0.001469000 seconds user
0.000000000 seconds sysWei Li (2):
perf stat: Fix segfault when counting armv8 PMU events perf stat: Unbreak perf stat with armv8 PMU events
tools/lib/perf/include/internal/evlist.h | 1 + tools/perf/builtin-stat.c | 37 ++++++++++++++++-------- tools/perf/util/evlist.c | 23 ++++++++++++++- 3 files changed, 48 insertions(+), 13 deletions(-)
Signature · 1 line
2.17.1
PATCH 1/2 perf stat: Fix segfault when counting armv8_pmu events
When executing perf stat with armv8_pmu events with a workload, it will report a segfault as result.
(gdb) bt
#0 0x0000000000603fc8 in perf_evsel__close_fd_cpu (evsel=<optimized out>,
cpu=<optimized out>) at evsel.c:122
#1 perf_evsel__close_cpu (evsel=evsel@entry=0x716e950, cpu=7) at evsel.c:156
#2 0x00000000004d4718 in evlist__close (evlist=0x70a7cb0) at util/evlist.c:1242
#3 0x0000000000453404 in __run_perf_stat (argc=3, argc@entry=1, argv=0x30,
argv@entry=0xfffffaea2f90, run_idx=119, run_idx@entry=1701998435)
at builtin-stat.c:929
#4 0x0000000000455058 in run_perf_stat (run_idx=1701998435, argv=0xfffffaea2f90,
argc=1) at builtin-stat.c:947
#5 cmd_stat (argc=1, argv=0xfffffaea2f90) at builtin-stat.c:2357
#6 0x00000000004bb888 in run_builtin (p=p@entry=0x9764b8 <commands+288>,
argc=argc@entry=4, argv=argv@entry=0xfffffaea2f90) at perf.c:312
#7 0x00000000004bbb54 in handle_internal_command (argc=argc@entry=4,
argv=argv@entry=0xfffffaea2f90) at perf.c:364#8 0x0000000000435378 in run_argv (argcp=<synthetic pointer>,
argv=<synthetic pointer>) at perf.c:408
#9 main (argc=4, argv=0xfffffaea2f90) at perf.c:538
After debugging, i found the root reason is that the xyarray fd is created by evsel__open_per_thread() ignoring the cpu passed in create_perf_stat_counter(), while the evsel' cpumap is assigned as the corresponding PMU's cpumap in __add_event(). Thus, the xyarray fd is created with ncpus of dummy cpumap and an out of bounds 'cpu' index will be used in perf_evsel__close_fd_cpu().
To address this, add a flag to mark this situation and avoid using the affinity technique when closing/enabling/disabling events.
--- tools/lib/perf/include/internal/evlist.h | 1 + tools/perf/builtin-stat.c | 3 +++ tools/perf/util/evlist.c | 23 ++++++++++++++++++++++- 3 files changed, 26 insertions(+), 1 deletion(-)
tools/lib/perf/include/internal/evlist.h +1 -0
@@ -17,6 +17,7 @@ struct perf_evlist {struct list_head entries;int nr_entries;bool has_user_cpus; +bool open_per_thread;struct perf_cpu_map *cpus;struct perf_cpu_map *all_cpus;struct perf_thread_map *threads;
tools/perf/builtin-stat.c +3 -0
@@ -725,6 +725,9 @@ static int __run_perf_stat(int argc, const char **argv, int run_idx)if (group)perf_evlist__set_leader (evsel_list); +if (!(target__has_cpu (&target) && !target__has_per_thread (&target))) + evsel_list->core.open_per_thread = true; +if (affinity__setup (&affinity) <0 )return -1 ;
tools/perf/util/evlist.c +22 -1
@@ -383,6 +383,15 @@ void evlist__disable(struct evlist *evlist)int cpu, i, imm =0 ;bool has_imm = false; +if (evlist->core.open_per_thread) { +evlist__for_each_entry (evlist, pos) { +if (pos->disabled || !evsel__is_group_leader (pos) || !pos->core.fd) +continue ; +evsel__disable (pos); + } + goto out; + } +if (affinity__setup (&affinity) <0 )return ;
@@ -414,6 +423,7 @@ void evlist__disable(struct evlist *evlist) pos->disabled = true; } +out: evlist->enabled = false; }
@@ -423,6 +433,15 @@ void evlist__enable(struct evlist *evlist)struct affinity affinity;int cpu, i; +if (evlist->core.open_per_thread) { +evlist__for_each_entry (evlist, pos) { +if (!evsel__is_group_leader (pos) || !pos->core.fd) +continue ; +evsel__enable (pos); + } + goto out; + } +if (affinity__setup (&affinity) <0 )return ;
@@ -444,6 +463,7 @@ void evlist__enable(struct evlist *evlist) pos->disabled = false; } +out: evlist->enabled = true; }
@@ -1223,9 +1243,10 @@ void evlist__close(struct evlist *evlist)/* * With perf record core.cpus is usually NULL. +* Or perf stat may open events per-thread. * Use the old method to handle this for now. */ -if (!evlist->core.cpus) { +if (evlist->core.open_per_thread || !evlist->core.cpus) {evlist__for_each_entry_reverse (evlist, evsel)evsel__close (evsel);return ;
Signature · 1 line
2.17.1
Re: [PATCH 1/2] perf stat: Fix segfault when counting armv8_pmu events
Quoted 9 lines
After debugging, i found the root reason is that the xyarray fd is created by
evsel__open_per_thread()ignoring the cpu passed increate_perf_stat_counter(), while the evsel' cpumap is assigned as the corresponding PMU's cpumap in__add_event(). Thus, the xyarray fd is created with ncpus of dummy cpumap and an out of bounds 'cpu' index will be used inperf_evsel__close_fd_cpu().To address this, add a flag to mark this situation and avoid using the affinity technique when closing/enabling/disabling events.
The flag seems like a hack. How about figuring out the correct number of CPUs and using that?
-Andi
Re: [PATCH 1/2] perf stat: Fix segfault when counting armv8_pmu events
Quoted 7 lines · Andi Kleen
Quoted 7 lines
After debugging, i found the root reason is that the xyarray fd is created by
evsel__open_per_thread()ignoring the cpu passed increate_perf_stat_counter(), while the evsel' cpumap is assigned as the corresponding PMU's cpumap in__add_event(). Thus, the xyarray fd is created with ncpus of dummy cpumap and an out of bounds 'cpu' index will be used inperf_evsel__close_fd_cpu().
Quoted 5 lines
Quoted 2 lines
To address this, add a flag to mark this situation and avoid using the affinity technique when closing/enabling/disabling events.
The flag seems like a hack. How about figuring out the correct number of CPUs and using that?
Also would like to understand what's different on ARM64 than other architectures. Or could this happen on x86 too?
-Andi
Re: [PATCH 1/2] perf stat: Fix segfault when counting armv8_pmu events
Hi Andi,
Quoted 8 lines · Andi Kleen
Quoted 7 lines · Andi Kleen
Quoted 7 lines
After debugging, i found the root reason is that the xyarray fd is created by
evsel__open_per_thread()ignoring the cpu passed increate_perf_stat_counter(), while the evsel' cpumap is assigned as the corresponding PMU's cpumap in__add_event(). Thus, the xyarray fd is created with ncpus of dummy cpumap and an out of bounds 'cpu' index will be used inperf_evsel__close_fd_cpu().
Quoted 9 lines
Quoted 5 lines
Quoted 2 lines
To address this, add a flag to mark this situation and avoid using the affinity technique when closing/enabling/disabling events.
The flag seems like a hack. How about figuring out the correct number of CPUs and using that?
Also would like to understand what's different on ARM64 than other architectures. Or could this happen on x86 too?
The problem is that when the user requests per-task events, the cpumask is expected as NULL(dummy), while the armv8_pmu do has a cpumask which inherited by evsel. The armv8_pmu's cpumask was added for heterogeneous systems. So this issue can not happen on x86.
In fact, the cpumask is correct indeed, but it should't be used when we requesting per-task events. As these events should be install on all cores, i doubt how much we can benefit from the affinity technique, so i choosed to add a flag.
I also did a test on hisilicon arm64 d06 board, with 2 sockets 128 cores. Testing the following command 3 times, with/without the affinity technique:
time tools/perf/perf stat -ddd -C 0-127 --per-core --timeout=5000 2> /dev/null
* (HEAD detached at 7074674e7338) perf cpumap: Maintain cpumaps ordered and without dups
real 0m8.039s
user 0m0.402s
sys 0m2.582s
real 0m7.939s
user 0m0.360s
sys 0m2.560s
real 0m7.997s
user 0m0.358s
sys 0m2.586s
* (HEAD detached at 704e2f5b700d) perf stat: Use affinity for enabling/disabling events
real 0m7.954s
user 0m0.308s
sys 0m2.590s
real 0m12.959s
user 0m0.332s
sys 0m2.582s
real 0m18.009s
user 0m0.346s
sys 0m2.562s
The offcpu time is much longer when using affinity, i think that's what migration costs, could you please share me your test case?
Thanks,
Wei
Re: [PATCH 1/2] perf stat: Fix segfault when counting armv8_pmu events
In reply to Wei LiQuoted 2 lines · Wei Li
When executing perf stat with
armv8_pmuevents with a workload, it will report a segfault as result.
please share the perf stat command line you see that segfault for
thanks,
jirka
Quoted 132 lines
(gdb) bt
#00x0000000000603fc8inperf_evsel__close_fd_cpu(evsel=<optimized out>,cpu=<optimized out>) at evsel.c:122 #1 perf_evsel__close_cpu (evsel=evsel@entry=0x716e950, cpu=7) at evsel.c:156 #2 0x00000000004d4718 in evlist__close (evlist=0x70a7cb0) at util/evlist.c:1242 #3 0x0000000000453404 in __run_perf_stat (argc=3, argc@entry=1, argv=0x30, argv@entry=0xfffffaea2f90, run_idx=119, run_idx@entry=1701998435) at builtin-stat.c:929 #4 0x0000000000455058 in run_perf_stat (run_idx=1701998435, argv=0xfffffaea2f90, argc=1) at builtin-stat.c:947 #5 cmd_stat (argc=1, argv=0xfffffaea2f90) at builtin-stat.c:2357 #6 0x00000000004bb888 in run_builtin (p=p@entry=0x9764b8 <commands+288>, argc=argc@entry=4, argv=argv@entry=0xfffffaea2f90) at perf.c:312 #7 0x00000000004bbb54 in handle_internal_command (argc=argc@entry=4, argv=argv@entry=0xfffffaea2f90) at perf.c:364#8
0x0000000000435378inrun_argv(argcp=<synthetic pointer>,
argv=<synthetic pointer>) at perf.c:408
#9 main (argc=4, argv=0xfffffaea2f90) at perf.c:538After debugging, i found the root reason is that the xyarray fd is created by
evsel__open_per_thread()ignoring the cpu passed increate_perf_stat_counter(), while the evsel' cpumap is assigned as the corresponding PMU's cpumap in__add_event(). Thus, the xyarray fd is created with ncpus of dummy cpumap and an out of bounds 'cpu' index will be used inperf_evsel__close_fd_cpu().To address this, add a flag to mark this situation and avoid using the affinity technique when closing/enabling/disabling events.
Fixes 7736627b865d ("perf stat: Use affinity for closing file descriptors")Fixes 704e2f5b700d ("perf stat: Use affinity for enabling/disabling events")Signed-off-by Wei Li <[email protected]>--- tools/lib/perf/include/internal/evlist.h | 1 + tools/perf/builtin-stat.c | 3 +++ tools/perf/util/evlist.c | 23 ++++++++++++++++++++++- 3 files changed, 26 insertions(+), 1 deletion(-)tools/lib/perf/include/internal/evlist.h +1
-0@@ -17,6 +17,7 @@ struct perf_evlist {struct list_head entries;int nr_entries;bool has_user_cpus; +bool open_per_thread;struct perf_cpu_map *cpus;struct perf_cpu_map *all_cpus;struct perf_thread_map *threads;tools/perf/builtin-stat.c +3
-0@@ -725,6 +725,9 @@ static int __run_perf_stat(int argc, const char **argv, int run_idx)if (group)perf_evlist__set_leader (evsel_list); +if (!(target__has_cpu (&target) && !target__has_per_thread (&target))) + evsel_list->core.open_per_thread = true; +if (affinity__setup (&affinity) <0 )return -1 ;tools/perf/util/evlist.c +22
-1@@ -383,6 +383,15 @@ void evlist__disable(struct evlist *evlist)int cpu, i, imm =0 ;bool has_imm = false; +if (evlist->core.open_per_thread) { +evlist__for_each_entry (evlist, pos) { +if (pos->disabled || !evsel__is_group_leader (pos) || !pos->core.fd) +continue ; +evsel__disable (pos); + } + goto out; + } +if (affinity__setup (&affinity) <0 )return ;@@ -414,6 +423,7 @@ void evlist__disable(struct evlist *evlist) pos->disabled = true; } +out: evlist->enabled = false; }@@ -423,6 +433,15 @@ void evlist__enable(struct evlist *evlist)struct affinity affinity;int cpu, i; +if (evlist->core.open_per_thread) { +evlist__for_each_entry (evlist, pos) { +if (!evsel__is_group_leader (pos) || !pos->core.fd) +continue ; +evsel__enable (pos); + } + goto out; + } +if (affinity__setup (&affinity) <0 )return ;@@ -444,6 +463,7 @@ void evlist__enable(struct evlist *evlist) pos->disabled = false; } +out: evlist->enabled = true; }@@ -1223,9 +1243,10 @@ void evlist__close(struct evlist *evlist)/* * With perf record core.cpus is usually NULL. +* Or perf stat may open events per-thread. * Use the old method to handle this for now. */ -if (!evlist->core.cpus) { +if (evlist->core.open_per_thread || !evlist->core.cpus) {evlist__for_each_entry_reverse (evlist, evsel)evsel__close (evsel);return ;Signature · 1 line
2.17.1
Namhyung Kim
maintainerRe: [PATCH 1/2] perf stat: Fix segfault when counting armv8_pmu events
Quoted 6 lines · Jiri Olsa
Quoted 2 lines · Wei Li
When executing perf stat with
armv8_pmuevents with a workload, it will report a segfault as result.please share the perf stat command line you see that segfault for
It seems the description in the patch 0/2 already has it:
[root@localhost hulk]# tools/perf/perf stat -e
armv8_pmuv3_0/ll_cache_rd/,armv8_pmuv3_0/ll_cache_miss_rd/ ls > /dev/null
Segmentation fault
Thanks
Namhyun
Quoted 38 lines
thanks,
jirkaQuoted 34 lines
(gdb) bt
#00x0000000000603fc8inperf_evsel__close_fd_cpu(evsel=<optimized out>,cpu=<optimized out>) at evsel.c:122 #1 perf_evsel__close_cpu (evsel=evsel@entry=0x716e950, cpu=7) at evsel.c:156 #2 0x00000000004d4718 in evlist__close (evlist=0x70a7cb0) at util/evlist.c:1242 #3 0x0000000000453404 in __run_perf_stat (argc=3, argc@entry=1, argv=0x30, argv@entry=0xfffffaea2f90, run_idx=119, run_idx@entry=1701998435) at builtin-stat.c:929 #4 0x0000000000455058 in run_perf_stat (run_idx=1701998435, argv=0xfffffaea2f90, argc=1) at builtin-stat.c:947 #5 cmd_stat (argc=1, argv=0xfffffaea2f90) at builtin-stat.c:2357 #6 0x00000000004bb888 in run_builtin (p=p@entry=0x9764b8 <commands+288>, argc=argc@entry=4, argv=argv@entry=0xfffffaea2f90) at perf.c:312 #7 0x00000000004bbb54 in handle_internal_command (argc=argc@entry=4, argv=argv@entry=0xfffffaea2f90) at perf.c:364#8
0x0000000000435378inrun_argv(argcp=<synthetic pointer>,
argv=<synthetic pointer>) at perf.c:408
#9 main (argc=4, argv=0xfffffaea2f90) at perf.c:538After debugging, i found the root reason is that the xyarray fd is created by
evsel__open_per_thread()ignoring the cpu passed increate_perf_stat_counter(), while the evsel' cpumap is assigned as the corresponding PMU's cpumap in__add_event(). Thus, the xyarray fd is created with ncpus of dummy cpumap and an out of bounds 'cpu' index will be used inperf_evsel__close_fd_cpu().To address this, add a flag to mark this situation and avoid using the affinity technique when closing/enabling/disabling events.
Fixes 7736627b865d ("perf stat: Use affinity for closing file descriptors")Fixes 704e2f5b700d ("perf stat: Use affinity for enabling/disabling events")Signed-off-by Wei Li <[email protected]>---
Re: [PATCH 1/2] perf stat: Fix segfault when counting armv8_pmu events
Quoted 6 lines · Namhyung Kim
Quoted 5 lines · Jiri Olsa
Quoted 2 lines · Wei Li
When executing perf stat with
armv8_pmuevents with a workload, it will report a segfault as result.
Quoted 8 lines
Quoted 1 line
please share the perf stat command line you see that segfault for
It seems the description in the patch 0/2 already has it:
[root@localhost hulk]#
tools/perf/perfstat -e
armv8_pmuv3_0/ll_cache_rd/,armv8_pmuv3_0/ll_cache_miss_rd/ ls >/dev/null
Segmentation fault
yea I found it, but can't reproduce it.. I see the issue from patch 2, but not sure what's the problem so far
jirka
Quoted 44 lines
Thanks
NamhyunQuoted 38 lines
thanks,
jirkaQuoted 34 lines
(gdb) bt
#00x0000000000603fc8inperf_evsel__close_fd_cpu(evsel=<optimized out>,cpu=<optimized out>) at evsel.c:122 #1 perf_evsel__close_cpu (evsel=evsel@entry=0x716e950, cpu=7) at evsel.c:156 #2 0x00000000004d4718 in evlist__close (evlist=0x70a7cb0) at util/evlist.c:1242 #3 0x0000000000453404 in __run_perf_stat (argc=3, argc@entry=1, argv=0x30, argv@entry=0xfffffaea2f90, run_idx=119, run_idx@entry=1701998435) at builtin-stat.c:929 #4 0x0000000000455058 in run_perf_stat (run_idx=1701998435, argv=0xfffffaea2f90, argc=1) at builtin-stat.c:947 #5 cmd_stat (argc=1, argv=0xfffffaea2f90) at builtin-stat.c:2357 #6 0x00000000004bb888 in run_builtin (p=p@entry=0x9764b8 <commands+288>, argc=argc@entry=4, argv=argv@entry=0xfffffaea2f90) at perf.c:312 #7 0x00000000004bbb54 in handle_internal_command (argc=argc@entry=4, argv=argv@entry=0xfffffaea2f90) at perf.c:364#8
0x0000000000435378inrun_argv(argcp=<synthetic pointer>,
argv=<synthetic pointer>) at perf.c:408
#9 main (argc=4, argv=0xfffffaea2f90) at perf.c:538After debugging, i found the root reason is that the xyarray fd is created by
evsel__open_per_thread()ignoring the cpu passed increate_perf_stat_counter(), while the evsel' cpumap is assigned as the corresponding PMU's cpumap in__add_event(). Thus, the xyarray fd is created with ncpus of dummy cpumap and an out of bounds 'cpu' index will be used inperf_evsel__close_fd_cpu().To address this, add a flag to mark this situation and avoid using the affinity technique when closing/enabling/disabling events.
Fixes 7736627b865d ("perf stat: Use affinity for closing file descriptors")Fixes 704e2f5b700d ("perf stat: Use affinity for enabling/disabling events")Signed-off-by Wei Li <[email protected]>---
Namhyung Kim
maintainerRe: [PATCH 1/2] perf stat: Fix segfault when counting armv8_pmu events
Quoted 12 lines · Jiri Olsa
Quoted 10 lines · Namhyung Kim
Quoted 6 lines · Jiri Olsa
Quoted 2 lines · Wei Li
When executing perf stat with
armv8_pmuevents with a workload, it will report a segfault as result.please share the perf stat command line you see that segfault for
It seems the description in the patch 0/2 already has it:
Quoted 7 lines
Quoted 4 lines
[root@localhost hulk]#
tools/perf/perfstat -e
armv8_pmuv3_0/ll_cache_rd/,armv8_pmuv3_0/ll_cache_miss_rd/ ls >/dev/null
Segmentation faultyea I found it, but can't reproduce it.. I see the issue from patch 2, but not sure what's the problem so far
I think the problem is that armv8_pmu has a cpumask, and the user requested per-task events.
The code tried to open the event with a dummy cpu map since it's not a cpu event, but the pmu has cpu map and it's passed to evsel. So there's confusion somewhere whether it should use evsel->cpus or a dummy map.
Thanks
Namhyung
Re: [PATCH 1/2] perf stat: Fix segfault when counting armv8_pmu events
Quoted 21 lines · Namhyung Kim
Quoted 12 lines · Jiri Olsa
Quoted 10 lines · Namhyung Kim
Quoted 6 lines · Jiri Olsa
Quoted 2 lines · Wei Li
When executing perf stat with
armv8_pmuevents with a workload, it will report a segfault as result.please share the perf stat command line you see that segfault for
It seems the description in the patch 0/2 already has it:
Quoted 7 lines
Quoted 4 lines
[root@localhost hulk]#
tools/perf/perfstat -e
armv8_pmuv3_0/ll_cache_rd/,armv8_pmuv3_0/ll_cache_miss_rd/ ls >/dev/null
Segmentation faultyea I found it, but can't reproduce it.. I see the issue from patch 2, but not sure what's the problem so far
Quoted 7 lines
I think the problem is that
armv8_pmuhas a cpumask, and the user requested per-task events.The code tried to open the event with a dummy cpu map since it's not a cpu event, but the pmu has cpu map and it's passed to evsel. So there's confusion somewhere whether it should use
evsel->cpusor a dummy map.
you're right, I have following cpus file in pmu:
# cat /sys/devices/armv8_pmuv3_0/cpus 0-3
covering all the cpus.. and once you have cpumask/cpus file, you're system wide by default in current code, but we should not crash ;-)
I tried to cover this case in patch below and I probably broke some other use cases, but perhaps we could allow to open counters per cpus for given workload
I'll try to look at this more tomorrow
jirka
---
tools/perf/builtin-stat.c +0 -5
@@ -379,12 +379,7 @@ static int read_affinity_counters(struct timespec *rs)if (affinity__setup (&affinity) <0 )return -1 ; - ncpus =perf_cpu_map__nr (evsel_list->core.all_cpus); -if (!target__has_cpu (&target) ||target__has_per_thread (&target)) - ncpus =1 ;evlist__for_each_cpu (evsel_list, i, cpu) { -if (i >= ncpus) -break ;affinity__set (&affinity, cpu);evlist__for_each_entry (evsel_list, counter) {
tools/perf/util/evsel.c +10 -0
@@ -1861,6 +1861,16 @@ void evsel__close(struct evsel *evsel)perf_evsel__free_id (&evsel->core); } +int evsel__open_threads_per_cpu (struct evsel *evsel,struct perf_thread_map *threads, +struct perf_cpu_map *cpus,int cpu) +{ +if (cpu ==-1 ) +return evsel__open_cpu (evsel, cpus, threads,0 , + cpus ? cpus->nr :1 ); + +return evsel__open_cpu (evsel, cpus, threads, cpu, cpu +1 ); +} +int evsel__open_per_cpu (struct evsel *evsel,struct perf_cpu_map *cpus,int cpu) {if (cpu ==-1 )
tools/perf/util/evsel.h +2 -0
@@ -231,6 +231,8 @@ int evsel__enable(struct evsel *evsel);int evsel__disable (struct evsel *evsel);int evsel__disable_cpu (struct evsel *evsel,int cpu); +int evsel__open_threads_per_cpu (struct evsel *evsel,struct perf_thread_map *threads, +struct perf_cpu_map *cpus,int cpu);int evsel__open_per_cpu (struct evsel *evsel,struct perf_cpu_map *cpus,int cpu);int evsel__open_per_thread (struct evsel *evsel,struct perf_thread_map *threads); int evsel__open(struct evsel *evsel, struct perf_cpu_map *cpus,
tools/perf/util/stat.c +5 -0
@@ -560,6 +560,11 @@ int create_perf_stat_counter(struct evsel *evsel, attr->enable_on_exec =1 ; } +if (evsel->core.own_cpus && evsel->core.threads) { +return evsel__open_threads_per_cpu (evsel, evsel->core.threads, +evsel__cpus (evsel), cpu); + } +if (target__has_cpu (target) && !target__has_per_thread (target))return evsel__open_per_cpu (evsel,evsel__cpus (evsel), cpu);
Namhyung Kim
maintainerRe: [PATCH 1/2] perf stat: Fix segfault when counting armv8_pmu events
Quoted 18 lines · Jiri Olsa
Quoted 7 lines · Namhyung Kim
I think the problem is that
armv8_pmuhas a cpumask, and the user requested per-task events.The code tried to open the event with a dummy cpu map since it's not a cpu event, but the pmu has cpu map and it's passed to evsel. So there's confusion somewhere whether it should use
evsel->cpusor a dummy map.you're right, I have following cpus file in pmu:
# cat /sys/devices/armv8_pmuv3_0/cpus 0-3covering all the cpus.. and once you have cpumask/cpus file, you're system wide by default in current code, but we should not crash ;-)
Quoted 5 lines
I tried to cover this case in patch below and I probably broke some other use cases, but perhaps we could allow to open counters per cpus for given workload
I'll try to look at this more tomorrow
I'm thinking about a different approach, we can ignore cpu map for the ARM cpu PMU and use the dummy, not tested ;-)
Thanks
Namhyung
tools/lib/perf/evlist.c +3 -0
@@ -45,6 +45,9 @@ static void __perf_evlist__propagate_maps(struct perf_evlist *evlist,if (!evsel->own_cpus || evlist->has_user_cpus) {perf_cpu_map__put (evsel->cpus); evsel->cpus =perf_cpu_map__get (evlist->cpus); + }else if (!evsel->system_wide &&perf_cpu_map__empty (evlist->cpus)) { +perf_cpu_map__put (evsel->cpus); + evsel->cpus =perf_cpu_map__get (evlist->cpus); }else if (evsel->cpus != evsel->own_cpus) {perf_cpu_map__put (evsel->cpus); evsel->cpus =perf_cpu_map__get (evsel->own_cpus);
Re: [PATCH 1/2] perf stat: Fix segfault when counting armv8_pmu events
Quoted 37 lines · Namhyung Kim
Quoted 18 lines · Jiri Olsa
Quoted 7 lines · Namhyung Kim
I think the problem is that
armv8_pmuhas a cpumask, and the user requested per-task events.The code tried to open the event with a dummy cpu map since it's not a cpu event, but the pmu has cpu map and it's passed to evsel. So there's confusion somewhere whether it should use
evsel->cpusor a dummy map.you're right, I have following cpus file in pmu:
# cat /sys/devices/armv8_pmuv3_0/cpus 0-3covering all the cpus.. and once you have cpumask/cpus file, you're system wide by default in current code, but we should not crash ;-)
Quoted 5 lines
I tried to cover this case in patch below and I probably broke some other use cases, but perhaps we could allow to open counters per cpus for given workload
I'll try to look at this more tomorrow
I'm thinking about a different approach, we can ignore cpu map for the ARM cpu PMU and use the dummy, not tested ;-)
Thanks
Namhyungtools/lib/perf/evlist.c +0
-0
tools/lib/perf/evlist.c static void __perf_evlist__propagate_maps(struct perf_evlist *evlist, · line 50
Quoted 6 lines
tools/lib/perf/evlist.c +3
-0@@ -45,6 +45,9 @@ static void __perf_evlist__propagate_maps(struct perf_evlist *evlist,if (!evsel->own_cpus || evlist->has_user_cpus) {perf_cpu_map__put (evsel->cpus); evsel->cpus =perf_cpu_map__get (evlist->cpus); + }else if (!evsel->system_wide &&perf_cpu_map__empty (evlist->cpus)) { +perf_cpu_map__put (evsel->cpus); + evsel->cpus =perf_cpu_map__get (evlist->cpus);
but I like this fix, because mine messes up scaling ;-)
tools/lib/perf/evlist.c static void __perf_evlist__propagate_maps(struct perf_evlist *evlist, · line 53
Quoted 3 lines
tools/lib/perf/evlist.c +0
-0}else if (evsel->cpus != evsel->own_cpus) {perf_cpu_map__put (evsel->cpus); evsel->cpus =perf_cpu_map__get (evsel->own_cpus);
it looks like that cpus (/sys/devices/armv8_pmuv3_0/cpus) file can have some cpus switched off.. from brief scan of:
drivers/perf/arm_pmu_platform.c (search for supported_cpus)
but it looks like it's just check for interrupt, so counting might still work..?
thanks,
jirka
Re: [PATCH 1/2] perf stat: Fix segfault when counting armv8_pmu events
In reply to Namhyung KimQuoted 38 lines · Namhyung Kim
Quoted 18 lines · Jiri Olsa
Quoted 7 lines · Namhyung Kim
I think the problem is that
armv8_pmuhas a cpumask, and the user requested per-task events.The code tried to open the event with a dummy cpu map since it's not a cpu event, but the pmu has cpu map and it's passed to evsel. So there's confusion somewhere whether it should use
evsel->cpusor a dummy map.you're right, I have following cpus file in pmu:
# cat /sys/devices/armv8_pmuv3_0/cpus 0-3covering all the cpus.. and once you have cpumask/cpus file, you're system wide by default in current code, but we should not crash ;-)
Quoted 5 lines
I tried to cover this case in patch below and I probably broke some other use cases, but perhaps we could allow to open counters per cpus for given workload
I'll try to look at this more tomorrow
I'm thinking about a different approach, we can ignore cpu map for the ARM cpu PMU and use the dummy, not tested ;-)
Thanks
Namhyungtools/lib/perf/evlist.c +0
-0
tools/lib/perf/evlist.c static void __perf_evlist__propagate_maps(struct perf_evlist *evlist, · line 53
Quoted 9 lines
tools/lib/perf/evlist.c +3
-0@@ -45,6 +45,9 @@ static void __perf_evlist__propagate_maps(struct perf_evlist *evlist,if (!evsel->own_cpus || evlist->has_user_cpus) {perf_cpu_map__put (evsel->cpus); evsel->cpus =perf_cpu_map__get (evlist->cpus); + }else if (!evsel->system_wide &&perf_cpu_map__empty (evlist->cpus)) { +perf_cpu_map__put (evsel->cpus); + evsel->cpus =perf_cpu_map__get (evlist->cpus); }else if (evsel->cpus != evsel->own_cpus) {perf_cpu_map__put (evsel->cpus); evsel->cpus =perf_cpu_map__get (evsel->own_cpus);
Wei Li,
is this fixing your problem?
thanks,
jirka
Song Bao Hua (Barry Song)
Tested-byRE: [PATCH 1/2] perf stat: Fix segfault when counting armv8_pmu events
Quoted 69 lines
-----Original Message-----
From: Jiri Olsa [mailto:[email protected]]
Sent: Friday, October 2, 2020 10:00 PM
To: Namhyung Kim <[email protected]>; liwei (GF) <[email protected]>
Cc: Mark Rutland <[email protected]>; Andi Kleen <[email protected]>; Alexander Shishkin <[email protected]>; Alexey Budankov <[email protected]>; Adrian Hunter <[email protected]>; Arnaldo Carvalho de Melo <[email protected]>; linux-kernel <[email protected]>; Peter Zijlstra <[email protected]>; Andi Kleen <[email protected]>; Libin (Huawei) <[email protected]>; Ingo Molnar <[email protected]>; [email protected]
Subject: Re: [PATCH 1/2] perf stat: Fix segfault when countingarmv8_pmueventstools/lib/perf/evlist.c static void __perf_evlist__propagate_maps(struct
Quoted 37 lines · Namhyung Kim
Quoted 18 lines · Jiri Olsa
Quoted 7 lines · Namhyung Kim
I think the problem is that
armv8_pmuhas a cpumask, and the user requested per-task events.The code tried to open the event with a dummy cpu map since it's not a cpu event, but the pmu has cpu map and it's passed to evsel. So there's confusion somewhere whether it should use
evsel->cpusor a dummy map.you're right, I have following cpus file in pmu:
# cat /sys/devices/armv8_pmuv3_0/cpus 0-3covering all the cpus.. and once you have cpumask/cpus file, you're system wide by default in current code, but we should not crash ;-)
Quoted 5 lines
I tried to cover this case in patch below and I probably broke some other use cases, but perhaps we could allow to open counters per cpus for given workload
I'll try to look at this more tomorrow
I'm thinking about a different approach, we can ignore cpu map for the ARM cpu PMU and use the dummy, not tested ;-)
Thanks
Namhyungtools/lib/perf/evlist.c +0
-0@@ -45,6 +45,9 @@ static void __perf_evlist__propagate_maps(struct
perf_evlist*evlist,tools/lib/perf/evlist.c static void __perf_evlist__propagate_maps(struct perf_evlist *evlist,
Quoted 4 lines
tools/lib/perf/evlist.c +1
-0if (!evsel->own_cpus || evlist->has_user_cpus) {perf_cpu_map__put (evsel->cpus); evsel->cpus =perf_cpu_map__get (evlist->cpus); + } else if (!evsel->system_wide &&
perf_cpu_map__empty(evlist->cpus)) {tools/lib/perf/evlist.c static void __perf_evlist__propagate_maps(struct perf_evlist *evlist, · line 53
Quoted 6 lines
tools/lib/perf/evlist.c +2
-0+perf_cpu_map__put (evsel->cpus); + evsel->cpus =perf_cpu_map__get (evlist->cpus); }else if (evsel->cpus != evsel->own_cpus) {perf_cpu_map__put (evsel->cpus); evsel->cpus =perf_cpu_map__get (evsel->own_cpus);
Quoted 2 lines
Wei Li,
is this fixing your problem?
As I have seen the same crash and suggested another patch:
[PATCH] perf evlist: fix memory corruption for Kernel PMU event
https://lore.kernel.org/lkml/[email protected]/
Also, I have tested Namhyung Kim's patch on ARM64. It does fix the crash for me. So:
Please put the below fixes-tag in commit log:
Thanks
Barry
PATCH 2/2 perf stat: Unbreak perf stat with armv8_pmu events
In reply to Wei LiAfter the segfault is fixed, perf-stat with armv8_pmu events with a workload is still broken:
[root@localhost hulk]# tools/perf/perf stat -e armv8_pmuv3_0/ll_cache_rd/,armv8_pmuv3_0/ll_cache_miss_rd/ ls > /dev/null
Performance counter stats for 'ls':
<not counted> armv8_pmuv3_0/ll_cache_rd/ (0.00%)
<not counted> armv8_pmuv3_0/ll_cache_miss_rd/ (0.00%)
0.002052670 seconds time elapsed
0.000000000 seconds user
0.002086000 seconds sysIn fact, while the event will be opened per-thread, create_perf_stat_counter() is called as many times as the count of cpu in the evlist's cpumap, and lost all the file descriptors except the last one. If this counter is not scheduled during the period of time, it will be "not counted".
Add the process to don't open the needless events in such situation.
--- tools/perf/builtin-stat.c | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-)
tools/perf/builtin-stat.c +23 -13
@@ -712,6 +712,7 @@ static int __run_perf_stat(int argc, const char **argv, int run_idx)struct affinity affinity;int i, cpu;bool second_pass = false; +bool open_per_thread = false;if (forks) { if (perf_evlist__prepare_workload(evsel_list, &target, argv, is_pipe,
@@ -726,16 +727,17 @@ static int __run_perf_stat(int argc, const char **argv, int run_idx)perf_evlist__set_leader (evsel_list);if (!(target__has_cpu (&target) && !target__has_per_thread (&target))) - evsel_list->core.open_per_thread = true; + evsel_list->core.open_per_thread = open_per_thread = true;if (affinity__setup (&affinity) <0 )return -1 ; evlist__for_each_cpu (evsel_list, i, cpu) { -affinity__set (&affinity, cpu); +if (!open_per_thread) +affinity__set (&affinity, cpu);evlist__for_each_entry (evsel_list, counter) { -if (evsel__cpu_iter_skip (counter, cpu)) +if (!open_per_thread &&evsel__cpu_iter_skip (counter, cpu))continue ;if (counter->reset_group || counter->errored)continue ;
@@ -753,7 +755,8 @@ static int __run_perf_stat(int argc, const char **argv, int run_idx)if ((errno ==EINVAL || errno ==EBADF ) && counter->leader != counter && counter->weak_group) { -perf_evlist__reset_weak_group (evsel_list, counter, false); +perf_evlist__reset_weak_group (evsel_list, counter, + open_per_thread);assert (counter->reset_group); second_pass = true;continue ;
@@ -773,6 +776,9 @@ static int __run_perf_stat(int argc, const char **argv, int run_idx) } counter->supported = true; } + +if (open_per_thread) +break ; }if (second_pass) {
@@ -782,20 +788,22 @@ static int __run_perf_stat(int argc, const char **argv, int run_idx)*/ evlist__for_each_cpu (evsel_list, i, cpu) { -affinity__set (&affinity, cpu); -/* First close errored or weak retry */ -evlist__for_each_entry (evsel_list, counter) { -if (!counter->reset_group && !counter->errored) -continue ; -if (evsel__cpu_iter_skip_no_inc (counter, cpu)) -continue ; -perf_evsel__close_cpu (&counter->core, counter->cpu_iter); +if (!open_per_thread) { +affinity__set (&affinity, cpu); +/* First close errored or weak retry */ +evlist__for_each_entry (evsel_list, counter) { +if (!counter->reset_group && !counter->errored) +continue ; +if (evsel__cpu_iter_skip_no_inc (counter, cpu)) +continue ; +perf_evsel__close_cpu (&counter->core, counter->cpu_iter); + } }/* Now reopen weak */ evlist__for_each_entry (evsel_list, counter) {if (!counter->reset_group && !counter->errored)continue ; -if (evsel__cpu_iter_skip (counter, cpu)) +if (!open_per_thread &&evsel__cpu_iter_skip (counter, cpu))continue ;if (!counter->reset_group)continue ;
@@ -817,6 +825,8 @@ static int __run_perf_stat(int argc, const char **argv, int run_idx) } counter->supported = true; } +if (open_per_thread) +break ; } }affinity__cleanup (&affinity);
Signature · 1 line
2.17.1