perf stat: Ignore error thread when enabling system-wide --per-thread
PATCH 1/1 perf stat: Ignore error thread when enabling system-wide --per-thread
If we execute 'perf stat --per-thread' with non-root account (even set kernel.perf_event_paranoid = -1 yet), it reports the error:
jinyao@skl:~$ perf stat --per-thread Error: You may not have permission to collect system-wide stats.
Consider tweaking /proc/sys/kernel/perf_event_paranoid, which controls use of the performance events system by unprivileged users (without CAP_SYS_ADMIN).
The current value is 2:
-1: Allow use of (almost) all events by all users
Ignore mlock limit after perf_event_mlock_kb without CAP_IPC_LOCK
Quoted 1 line
= 0: Disallow ftrace function tracepoint by users without
CAP_SYS_ADMIN
Disallow raw tracepoint access by users without CAP_SYS_ADMIN
Quoted 2 lines
= 1: Disallow CPU event access by users without
CAP_SYS_ADMIN= 2: Disallow kernel profiling by users withoutCAP_SYS_ADMIN
To make this setting permanent, edit /etc/sysctl.conf too, e.g.:
kernel.perf_event_paranoid = -1
Perhaps the ptrace rule doesn't allow to trace some processes. But anyway the global --per-thread mode had better ignore such errors and continue working on other threads.
This patch will record the index of error thread in perf_evsel__open() and remove this thread before retrying.
For example (run with non-root, kernel.perf_event_paranoid isn't set):
jinyao@skl:~$ perf stat --per-thread ^C Performance counter stats for 'system wide':
vmstat-3458 6.171984 cpu-clock:u (msec) # 0.000 CPUs utilized
perf-3670 0.515599 cpu-clock:u (msec) # 0.000 CPUs utilized
vmstat-3458 1,163,643 cycles:u # 0.189 GHz
perf-3670 40,881 cycles:u # 0.079 GHz
vmstat-3458 1,410,238 instructions:u # 1.21 insn per cycle
perf-3670 3,536 instructions:u # 0.09 insn per cycle
vmstat-3458 288,937 branches:u # 46.814 M/sec
perf-3670 936 branches:u # 1.815 M/sec
vmstat-3458 15,195 branch-misses:u # 5.26% of all branches
perf-3670 76 branch-misses:u # 8.12% of all branches12.651675247 seconds time elapsed
--- tools/perf/builtin-stat.c | 14 +++++++++++++- tools/perf/util/evsel.c | 3 +++ tools/perf/util/thread_map.c | 1 + tools/perf/util/thread_map.h | 1 + 4 files changed, 18 insertions(+), 1 deletion(-)
tools/perf/builtin-stat.c +13 -1
@@ -632,7 +632,19 @@ static int __run_perf_stat(int argc, const char **argv)if (verbose >0 )ui__warning ("%s\n" , msg); goto try_again; - } + }else if (target__has_per_thread (&target) && + evsel_list->threads && + evsel_list->threads->err_thread !=-1 ) { +/* +* For global --per-thread case, skip current +* error thread. +*/ +if (!thread_map__remove (evsel_list->threads, + evsel_list->threads->err_thread)) { + evsel_list->threads->err_thread =-1 ; + goto try_again; + } + }perf_evsel__open_strerror (counter, &target, errno, msg,sizeof (msg));
tools/perf/util/evsel.c +3 -0
@@ -1899,6 +1899,9 @@ int perf_evsel__open(struct perf_evsel *evsel, struct cpu_map *cpus, goto fallback_missing_features; } out_close: +if (err) + threads->err_thread = thread; + do { while (--thread >= 0) { close(FD(evsel, cpu, thread));
tools/perf/util/thread_map.c +1 -0
@@ -32,6 +32,7 @@ static void thread_map__reset(struct thread_map *map, int start, int nr)size_t size = (nr - start) *sizeof (map->map[0 ]);memset (&map->map[start],0 , size); + map->err_thread =-1 ; }static struct thread_map *thread_map__realloc (struct thread_map *map,int nr)
tools/perf/util/thread_map.h +1 -0
@@ -14,6 +14,7 @@ struct thread_map_data {struct thread_map {refcount_t refcnt;int nr; +int err_thread;struct thread_map_data map[]; };
Signature · 1 line
2.7.4
Quoted 48 lines · Jin Yao
If we execute 'perf stat --per-thread' with non-root account (even set kernel.perf_event_paranoid = -1 yet), it reports the error:
jinyao@skl:~$ perf stat --per-thread Error: You may not have permission to collect system-wide stats.Consider tweaking
/proc/sys/kernel/perf_event_paranoid, which controls use of the performance events system by unprivileged users (withoutCAP_SYS_ADMIN).The current value is 2:
-1: Allow use of (almost) all events by all users
Ignore mlock limit afterperf_event_mlock_kbwithoutCAP_IPC_LOCKQuoted 1 line
= 0: Disallow ftrace function tracepoint by users without
CAP_SYS_ADMINDisallow raw tracepoint access by users without
CAP_SYS_ADMINQuoted 2 lines
= 1: Disallow CPU event access by users without
CAP_SYS_ADMIN= 2: Disallow kernel profiling by users withoutCAP_SYS_ADMINTo make this setting permanent, edit
/etc/sysctl.conftoo, e.g.:kernel.perf_event_paranoid = -1
Perhaps the ptrace rule doesn't allow to trace some processes. But anyway the global --per-thread mode had better ignore such errors and continue working on other threads.
This patch will record the index of error thread in
perf_evsel__open()and remove this thread before retrying.For example (run with non-root, kernel.perf_event_paranoid isn't set):
jinyao@skl:~$ perf stat --per-thread ^C Performance counter stats for 'system wide':vmstat-3458 6.171984 cpu-clock:u (msec) # 0.000 CPUs utilized perf-3670 0.515599 cpu-clock:u (msec) # 0.000 CPUs utilized vmstat-3458 1,163,643 cycles:u # 0.189 GHz perf-3670 40,881 cycles:u # 0.079 GHz vmstat-3458 1,410,238 instructions:u # 1.21 insn per cycle perf-3670 3,536 instructions:u # 0.09 insn per cycle vmstat-3458 288,937 branches:u # 46.814 M/sec perf-3670 936 branches:u # 1.815 M/sec vmstat-3458 15,195 branch-misses:u # 5.26% of all branches perf-3670 76 branch-misses:u # 8.12% of all branches
Quoted 1 line
12.651675247 seconds time elapsed
could we use the existing code like in attached patch?
we might also swap following warning for some generic one:
WARNING: Ignored open failure for pid 28392
jirka
---
tools/perf/builtin-stat.c +3 -0
@@ -276,6 +276,9 @@ static int create_perf_stat_counter(struct perf_evsel *evsel) attr->enable_on_exec =1 ; } +if (stat_config.aggr_mode ==AGGR_THREAD ) + evsel->ignore_missing_thread = true; +if (target__has_cpu (&target) && !target__has_per_thread (&target))return perf_evsel__open_per_cpu (evsel,perf_evsel__cpus (evsel));
tools/perf/util/evsel.c +1 -1
@@ -1656,7 +1656,7 @@ static bool ignore_missing_thread(struct perf_evsel *evsel,return false;/* The -ESRCH is perf event syscall errno for pid's not found. */ -if (err != -ESRCH ) +if (err != -ESRCH && err != -EACCES )return false;/* If there's only one thread, let it fail. */
Just tested. But looks it's not OK for '--per-thread' case.
jinyao@skl:~/$ ./perf stat --per-thread WARNING: Ignored open failure for pid 1 WARNING: Ignored open failure for pid 2 WARNING: Ignored open failure for pid 4 WARNING: Ignored open failure for pid 6 WARNING: Ignored open failure for pid 7 WARNING: Ignored open failure for pid 8 WARNING: Ignored open failure for pid 9 WARNING: Ignored open failure for pid 10 ...... WARNING: Ignored open failure for pid 11783 WARNING: Ignored open failure for pid 12275 WARNING: Ignored open failure for pid 31733 WARNING: Ignored open failure for pid 31737 Error: You may not have permission to collect system-wide stats.
Consider tweaking /proc/sys/kernel/perf_event_paranoid, which controls use of the performance events system by unprivileged users (without CAP_SYS_ADMIN).
The current value is 2:
-1: Allow use of (almost) all events by all users
Ignore mlock limit after perf_event_mlock_kb without CAP_IPC_LOCK
Quoted 1 line
= 0: Disallow ftrace function tracepoint by users without
CAP_SYS_ADMIN
Disallow raw tracepoint access by users without CAP_SYS_ADMIN
Quoted 2 lines
= 1: Disallow CPU event access by users without
CAP_SYS_ADMIN= 2: Disallow kernel profiling by users withoutCAP_SYS_ADMIN
To make this setting permanent, edit /etc/sysctl.conf too, e.g.:
kernel.perf_event_paranoid = -1
Thanks
Jin Yao
Quoted 88 lines · Jiri Olsa
Quoted 48 lines · Jin Yao
If we execute 'perf stat --per-thread' with non-root account (even set kernel.perf_event_paranoid = -1 yet), it reports the error:
jinyao@skl:~$ perf stat --per-thread Error: You may not have permission to collect system-wide stats.Consider tweaking
/proc/sys/kernel/perf_event_paranoid, which controls use of the performance events system by unprivileged users (withoutCAP_SYS_ADMIN).The current value is 2:
-1: Allow use of (almost) all events by all users
Ignore mlock limit afterperf_event_mlock_kbwithoutCAP_IPC_LOCKQuoted 1 line
= 0: Disallow ftrace function tracepoint by users without
CAP_SYS_ADMINDisallow raw tracepoint access by users without
CAP_SYS_ADMINQuoted 2 lines
= 1: Disallow CPU event access by users without
CAP_SYS_ADMIN= 2: Disallow kernel profiling by users withoutCAP_SYS_ADMINTo make this setting permanent, edit
/etc/sysctl.conftoo, e.g.:kernel.perf_event_paranoid = -1
Perhaps the ptrace rule doesn't allow to trace some processes. But anyway the global --per-thread mode had better ignore such errors and continue working on other threads.
This patch will record the index of error thread in
perf_evsel__open()and remove this thread before retrying.For example (run with non-root, kernel.perf_event_paranoid isn't set):
jinyao@skl:~$ perf stat --per-thread ^C Performance counter stats for 'system wide':vmstat-3458 6.171984 cpu-clock:u (msec) # 0.000 CPUs utilized perf-3670 0.515599 cpu-clock:u (msec) # 0.000 CPUs utilized vmstat-3458 1,163,643 cycles:u # 0.189 GHz perf-3670 40,881 cycles:u # 0.079 GHz vmstat-3458 1,410,238 instructions:u # 1.21 insn per cycle perf-3670 3,536 instructions:u # 0.09 insn per cycle vmstat-3458 288,937 branches:u # 46.814 M/sec perf-3670 936 branches:u # 1.815 M/sec vmstat-3458 15,195 branch-misses:u # 5.26% of all branches perf-3670 76 branch-misses:u # 8.12% of all branchesQuoted 1 line
12.651675247 seconds time elapsedcould we use the existing code like in attached patch?
we might also swap following warning for some generic one:
WARNING: Ignored open failure for pid 28392jirka
---
tools/perf/builtin-stat.c +3
-0@@ -276,6 +276,9 @@ static int create_perf_stat_counter(struct perf_evsel *evsel) attr->enable_on_exec =1 ; } +if (stat_config.aggr_mode ==AGGR_THREAD ) + evsel->ignore_missing_thread = true; +if (target__has_cpu (&target) && !target__has_per_thread (&target))return perf_evsel__open_per_cpu (evsel,perf_evsel__cpus (evsel));tools/perf/util/evsel.c +1
-1@@ -1656,7 +1656,7 @@ static bool ignore_missing_thread(struct perf_evsel *evsel,return false;/* The -ESRCH is perf event syscall errno for pid's not found. */ -if (err != -ESRCH ) +if (err != -ESRCH && err != -EACCES )return false;/* If there's only one thread, let it fail. */
Quoted 1 line · Yao
Just tested. But looks it's not OK for '--per-thread' case.
yea, I haven't tested much.. might need soem tweaking, but my point was that it could be doable on one place instead of introducing another if possible
jirka
Quoted 9 lines · Jiri Olsa
Quoted 1 line · Yao
Just tested. But looks it's not OK for '--per-thread' case.
yea, I haven't tested much.. might need soem tweaking, but my point was that it could be doable on one place instead of introducing another if possible
jirka
Yes, I understand. It'd better we can put the code to more common places.
Actually I used the similar patch as yours at first. While I found it didn't work for the case of system-wide '--per-thread' then I developed current one.
Thanks
Jin Yao
Quoted 9 lines · Jiri Olsa
Quoted 1 line · Yao
Just tested. But looks it's not OK for '--per-thread' case.
yea, I haven't tested much.. might need soem tweaking, but my point was that it could be doable on one place instead of introducing another if possible
jirka
Hi Jiri,
I ever considered to move the operation of removing error thread to perf_evsel__fallback(). The perf_evsel__fallback() is common code and it's shared by perf report, perf stat and perf top.
While finally I think it'd better let the caller decide to remove error thread and try again, or just return the warning message. perf_evsel__fallback() probably doesn't know what the caller want to do.
That's my current thinking. Maybe there will be a better fix...
Thanks
Jin Yao
Jiri Olsa
Acked-byQuoted 19 lines · Yao
Quoted 9 lines · Jiri Olsa
Quoted 1 line · Yao
Just tested. But looks it's not OK for '--per-thread' case.
yea, I haven't tested much.. might need soem tweaking, but my point was that it could be doable on one place instead of introducing another if possible
jirka
Hi Jiri,
I ever considered to move the operation of removing error thread to
perf_evsel__fallback(). Theperf_evsel__fallback()is common code and it's shared by perf report, perf stat and perf top.
Quoted 5 lines
While finally I think it'd better let the caller decide to remove error thread and try again, or just return the warning message.
perf_evsel__fallback()probably doesn't know what the caller want to do.That's my current thinking. Maybe there will be a better fix...
ok, can't think of better fix atm.. looks good ;-)
thanks,
jirka
Quoted 24 lines · Jiri Olsa
Quoted 23 lines · Yao
Quoted 9 lines · Jiri Olsa
Quoted 1 line · Yao
Just tested. But looks it's not OK for '--per-thread' case.
yea, I haven't tested much.. might need soem tweaking, but my point was that it could be doable on one place instead of introducing another if possible
jirka
Hi Jiri,
I ever considered to move the operation of removing error thread to
perf_evsel__fallback(). Theperf_evsel__fallback()is common code and it's shared by perf report, perf stat and perf top.While finally I think it'd better let the caller decide to remove error thread and try again, or just return the warning message.
perf_evsel__fallback()probably doesn't know what the caller want to do.
Quoted 9 lines
Quoted 1 line
That's my current thinking. Maybe there will be a better fix...
ok, can't think of better fix atm.. looks good ;-)
Acked-by Jiri Olsa <[email protected]>thanks,
jirka
Hi Arnaldo,
Could this fix be accepted?
Thanks
Jin Yao
Arnaldo Carvalho de Melo
maintainerEm Tue, Feb 27, 2018 at 08:53:13AM +0800, Jin, Yao escreveu:
Quoted 33 lines · Jiri Olsa
Quoted 30 lines · Yao
Quoted 19 lines · Jiri Olsa
Quoted 9 lines · Yao
Quoted 1 line
Just tested. But looks it's not OK for '--per-thread' case.
yea, I haven't tested much.. might need soem tweaking, but my point was that it could be doable on one place instead of introducing another if possible
jirka
Hi Jiri,
I ever considered to move the operation of removing error thread to
perf_evsel__fallback(). Theperf_evsel__fallback()is common code and it's shared by perf report, perf stat and perf top.Quoted 5 lines
While finally I think it'd better let the caller decide to remove error thread and try again, or just return the warning message.
perf_evsel__fallback()probably doesn't know what the caller want to do.That's my current thinking. Maybe there will be a better fix...
ok, can't think of better fix atm.. looks good ;-)
Acked-by Jiri Olsa <[email protected]>
Quoted 7 lines
Quoted 3 lines
thanks,
jirkaHi Arnaldo,
Could this fix be accepted?
yeah, its just strange that that err_thread is not used, applied.
- Arnaldo
tip-bot for Jin Yao
merge botperf stat: Ignore error thread when enabling system-wide --per-thread
Commit-ID: ab6c79b819f5a50cf41a11ebec17bef63b530333 Gitweb: https://git.kernel.org/tip/ab6c79b819f5a50cf41a11ebec17bef63b530333 Author: Jin Yao <[email protected]> AuthorDate: Tue, 16 Jan 2018 23:43:08 +0800 Committer: Arnaldo Carvalho de Melo <[email protected]> CommitDate: Tue, 27 Feb 2018 11:29:21 -0300
perf stat: Ignore error thread when enabling system-wide --per-thread
If we execute 'perf stat --per-thread' with non-root account (even set kernel.perf_event_paranoid = -1 yet), it reports the error:
jinyao@skl:~$ perf stat --per-thread Error: You may not have permission to collect system-wide stats.
Consider tweaking /proc/sys/kernel/perf_event_paranoid, which controls use of the performance events system by unprivileged users (without CAP_SYS_ADMIN).
The current value is 2:
-1: Allow use of (almost) all events by all users
Ignore mlock limit after perf_event_mlock_kb without CAP_IPC_LOCK
Quoted 1 line
= 0: Disallow ftrace function tracepoint by users without
CAP_SYS_ADMIN
Disallow raw tracepoint access by users without CAP_SYS_ADMIN
Quoted 2 lines
= 1: Disallow CPU event access by users without
CAP_SYS_ADMIN= 2: Disallow kernel profiling by users withoutCAP_SYS_ADMIN
To make this setting permanent, edit /etc/sysctl.conf too, e.g.:
kernel.perf_event_paranoid = -1
Perhaps the ptrace rule doesn't allow to trace some processes. But anyway the global --per-thread mode had better ignore such errors and continue working on other threads.
This patch will record the index of error thread in perf_evsel__open() and remove this thread before retrying.
For example (run with non-root, kernel.perf_event_paranoid isn't set):
jinyao@skl:~$ perf stat --per-thread ^C Performance counter stats for 'system wide':
vmstat-3458 6.171984 cpu-clock:u (msec) # 0.000 CPUs utilized
perf-3670 0.515599 cpu-clock:u (msec) # 0.000 CPUs utilized
vmstat-3458 1,163,643 cycles:u # 0.189 GHz
perf-3670 40,881 cycles:u # 0.079 GHz
vmstat-3458 1,410,238 instructions:u # 1.21 insn per cycle
perf-3670 3,536 instructions:u # 0.09 insn per cycle
vmstat-3458 288,937 branches:u # 46.814 M/sec
perf-3670 936 branches:u # 1.815 M/sec
vmstat-3458 15,195 branch-misses:u # 5.26% of all branches
perf-3670 76 branch-misses:u # 8.12% of all branches12.651675247 seconds time elapsed
--- tools/perf/builtin-stat.c | 14 +++++++++++++- tools/perf/util/evsel.c | 3 +++ tools/perf/util/thread_map.c | 1 + tools/perf/util/thread_map.h | 1 + 4 files changed, 18 insertions(+), 1 deletion(-)
tools/perf/builtin-stat.c +13 -1
@@ -637,7 +637,19 @@ try_again:if (verbose >0 )ui__warning ("%s\n" , msg); goto try_again; - } + }else if (target__has_per_thread (&target) && + evsel_list->threads && + evsel_list->threads->err_thread !=-1 ) { +/* +* For global --per-thread case, skip current +* error thread. +*/ +if (!thread_map__remove (evsel_list->threads, + evsel_list->threads->err_thread)) { + evsel_list->threads->err_thread =-1 ; + goto try_again; + } + }perf_evsel__open_strerror (counter, &target, errno, msg,sizeof (msg));
tools/perf/util/evsel.c +3 -0
@@ -1915,6 +1915,9 @@ try_fallback: goto fallback_missing_features; } out_close: +if (err) + threads->err_thread = thread; + do { while (--thread >= 0) { close(FD(evsel, cpu, thread));
tools/perf/util/thread_map.c +1 -0
@@ -32,6 +32,7 @@ static void thread_map__reset(struct thread_map *map, int start, int nr)size_t size = (nr - start) *sizeof (map->map[0 ]);memset (&map->map[start],0 , size); + map->err_thread =-1 ; }static struct thread_map *thread_map__realloc (struct thread_map *map,int nr)
tools/perf/util/thread_map.h +1 -0
@@ -14,6 +14,7 @@ struct thread_map_data {struct thread_map {refcount_t refcnt;int nr; +int err_thread;struct thread_map_data map[]; };