Skip to contentSkip to conversation

perf stat: Ignore error thread when enabling system-wide --per-thread

PATCH In review Jin Yao 4 participants 10 messages Jan 16, 2018 – Mar 6, 2018 perf events

lore
Share & tools
lore

Jin Yao

author
#0 lore raw

PATCH 1/1 perf stat: Ignore error thread when enabling system-wide --per-thread

If we execute 'perf stat --per-thread' with non-root account (even set kernel.perf_event_paranoid = -1 yet), it reports the error:

jinyao@skl:~$ perf stat --per-thread
Error:
You may not have permission to collect system-wide stats.

Consider tweaking /proc/sys/kernel/perf_event_paranoid, which controls use of the performance events system by unprivileged users (without CAP_SYS_ADMIN).

The current value is 2:

-1: Allow use of (almost) all events by all users
Ignore mlock limit after perf_event_mlock_kb without CAP_IPC_LOCK

Quoted 1 line

= 0: Disallow ftrace function tracepoint by users without CAP_SYS_ADMIN

Disallow raw tracepoint access by users without CAP_SYS_ADMIN

Quoted 2 lines

= 1: Disallow CPU event access by users without CAP_SYS_ADMIN = 2: Disallow kernel profiling by users without CAP_SYS_ADMIN

To make this setting permanent, edit /etc/sysctl.conf too, e.g.:

kernel.perf_event_paranoid = -1

Perhaps the ptrace rule doesn't allow to trace some processes. But anyway the global --per-thread mode had better ignore such errors and continue working on other threads.

This patch will record the index of error thread in perf_evsel__open() and remove this thread before retrying.

For example (run with non-root, kernel.perf_event_paranoid isn't set):

jinyao@skl:~$ perf stat --per-thread
^C
 Performance counter stats for 'system wide':
          vmstat-3458               6.171984      cpu-clock:u (msec)        #    0.000 CPUs utilized
            perf-3670               0.515599      cpu-clock:u (msec)        #    0.000 CPUs utilized
          vmstat-3458              1,163,643      cycles:u                  #    0.189 GHz
            perf-3670                 40,881      cycles:u                  #    0.079 GHz
          vmstat-3458              1,410,238      instructions:u            #    1.21  insn per cycle
            perf-3670                  3,536      instructions:u            #    0.09  insn per cycle
          vmstat-3458                288,937      branches:u                #   46.814 M/sec
            perf-3670                    936      branches:u                #    1.815 M/sec
          vmstat-3458                 15,195      branch-misses:u           #    5.26% of all branches
            perf-3670                     76      branch-misses:u           #    8.12% of all branches
      12.651675247 seconds time elapsed
---
 tools/perf/builtin-stat.c    | 14 +++++++++++++-
 tools/perf/util/evsel.c      |  3 +++
 tools/perf/util/thread_map.c |  1 +
 tools/perf/util/thread_map.h |  1 +
 4 files changed, 18 insertions(+), 1 deletion(-)
tools/perf/builtin-stat.c +13 -1
index 98bf9d3..bcdb47c 100644
@@ -632,7 +632,19 @@ static int __run_perf_stat(int argc, const char **argv)
                                 if (verbose > 0)
                                         ui__warning("%s\n", msg);
                                 goto try_again;
-                        }
+			} else if (target__has_per_thread(&target) &&
+				   evsel_list->threads &&
+				   evsel_list->threads->err_thread != -1) {
+				/*
+				 * For global --per-thread case, skip current
+				 * error thread.
+				 */
+				if (!thread_map__remove(evsel_list->threads,
+							evsel_list->threads->err_thread)) {
+					evsel_list->threads->err_thread = -1;
+					goto try_again;
+				}
+			}
 
 			perf_evsel__open_strerror(counter, &target,
 						  errno, msg, sizeof(msg));
tools/perf/util/evsel.c +3 -0
index a4d256e..12f8733 100644
@@ -1899,6 +1899,9 @@ int perf_evsel__open(struct perf_evsel *evsel, struct cpu_map *cpus,
 		goto fallback_missing_features;
 	}
 out_close:
+	if (err)
+		threads->err_thread = thread;
+
 	do {
 		while (--thread >= 0) {
 			close(FD(evsel, cpu, thread));
tools/perf/util/thread_map.c +1 -0
index 3e1038f..870cb0c 100644
@@ -32,6 +32,7 @@ static void thread_map__reset(struct thread_map *map, int start, int nr)
 	size_t size = (nr - start) * sizeof(map->map[0]);
 
 	memset(&map->map[start], 0, size);
+	map->err_thread = -1;
 }
 
 static struct thread_map *thread_map__realloc(struct thread_map *map, int nr)
tools/perf/util/thread_map.h +1 -0
index 0a806b9..ac6baf1 100644
@@ -14,6 +14,7 @@ struct thread_map_data {
 struct thread_map {
 	refcount_t refcnt;
 	int nr;
+	int err_thread;
 	struct thread_map_data map[];
 };
 
Signature · 1 line
2.7.4
#1 lore raw
Quoted 48 lines · Jin Yao

If we execute 'perf stat --per-thread' with non-root account (even set kernel.perf_event_paranoid = -1 yet), it reports the error:

jinyao@skl:~$ perf stat --per-thread
Error:
You may not have permission to collect system-wide stats.

Consider tweaking /proc/sys/kernel/perf_event_paranoid, which controls use of the performance events system by unprivileged users (without CAP_SYS_ADMIN).

The current value is 2:

-1: Allow use of (almost) all events by all users
Ignore mlock limit after perf_event_mlock_kb without CAP_IPC_LOCK

Quoted 1 line

= 0: Disallow ftrace function tracepoint by users without CAP_SYS_ADMIN

Disallow raw tracepoint access by users without CAP_SYS_ADMIN

Quoted 2 lines

= 1: Disallow CPU event access by users without CAP_SYS_ADMIN = 2: Disallow kernel profiling by users without CAP_SYS_ADMIN

To make this setting permanent, edit /etc/sysctl.conf too, e.g.:

kernel.perf_event_paranoid = -1

Perhaps the ptrace rule doesn't allow to trace some processes. But anyway the global --per-thread mode had better ignore such errors and continue working on other threads.

This patch will record the index of error thread in perf_evsel__open() and remove this thread before retrying.

For example (run with non-root, kernel.perf_event_paranoid isn't set):

jinyao@skl:~$ perf stat --per-thread
^C
 Performance counter stats for 'system wide':
          vmstat-3458               6.171984      cpu-clock:u (msec)        #    0.000 CPUs utilized
            perf-3670               0.515599      cpu-clock:u (msec)        #    0.000 CPUs utilized
          vmstat-3458              1,163,643      cycles:u                  #    0.189 GHz
            perf-3670                 40,881      cycles:u                  #    0.079 GHz
          vmstat-3458              1,410,238      instructions:u            #    1.21  insn per cycle
            perf-3670                  3,536      instructions:u            #    0.09  insn per cycle
          vmstat-3458                288,937      branches:u                #   46.814 M/sec
            perf-3670                    936      branches:u                #    1.815 M/sec
          vmstat-3458                 15,195      branch-misses:u           #    5.26% of all branches
            perf-3670                     76      branch-misses:u           #    8.12% of all branches
Quoted 1 line
      12.651675247 seconds time elapsed

could we use the existing code like in attached patch?

we might also swap following warning for some generic one:
WARNING: Ignored open failure for pid 28392

jirka

---

tools/perf/builtin-stat.c +3 -0
index 98bf9d32f222..f9d4d3ad6fc5 100644
@@ -276,6 +276,9 @@ static int create_perf_stat_counter(struct perf_evsel *evsel)
 			attr->enable_on_exec = 1;
 	}
 
+	if (stat_config.aggr_mode == AGGR_THREAD)
+		evsel->ignore_missing_thread = true;
+
 	if (target__has_cpu(&target) && !target__has_per_thread(&target))
 		return perf_evsel__open_per_cpu(evsel, perf_evsel__cpus(evsel));
 
tools/perf/util/evsel.c +1 -1
index 8f971a2301d1..509ee175bc97 100644
@@ -1656,7 +1656,7 @@ static bool ignore_missing_thread(struct perf_evsel *evsel,
 		return false;
 
 	/* The -ESRCH is perf event syscall errno for pid's not found. */
-	if (err != -ESRCH)
+	if (err != -ESRCH && err != -EACCES)
 		return false;
 
 	/* If there's only one thread, let it fail. */
author
#2 lore raw

Just tested. But looks it's not OK for '--per-thread' case.

jinyao@skl:~/$ ./perf stat --per-thread
WARNING: Ignored open failure for pid 1
WARNING: Ignored open failure for pid 2
WARNING: Ignored open failure for pid 4
WARNING: Ignored open failure for pid 6
WARNING: Ignored open failure for pid 7
WARNING: Ignored open failure for pid 8
WARNING: Ignored open failure for pid 9
WARNING: Ignored open failure for pid 10
......
WARNING: Ignored open failure for pid 11783
WARNING: Ignored open failure for pid 12275
WARNING: Ignored open failure for pid 31733
WARNING: Ignored open failure for pid 31737
Error:
You may not have permission to collect system-wide stats.

Consider tweaking /proc/sys/kernel/perf_event_paranoid, which controls use of the performance events system by unprivileged users (without CAP_SYS_ADMIN).

The current value is 2:

-1: Allow use of (almost) all events by all users
Ignore mlock limit after perf_event_mlock_kb without CAP_IPC_LOCK

Quoted 1 line

= 0: Disallow ftrace function tracepoint by users without CAP_SYS_ADMIN

Disallow raw tracepoint access by users without CAP_SYS_ADMIN

Quoted 2 lines

= 1: Disallow CPU event access by users without CAP_SYS_ADMIN = 2: Disallow kernel profiling by users without CAP_SYS_ADMIN

To make this setting permanent, edit /etc/sysctl.conf too, e.g.:

kernel.perf_event_paranoid = -1

Thanks
Jin Yao

Quoted 88 lines · Jiri Olsa
Quoted 48 lines · Jin Yao

If we execute 'perf stat --per-thread' with non-root account (even set kernel.perf_event_paranoid = -1 yet), it reports the error:

jinyao@skl:~$ perf stat --per-thread
Error:
You may not have permission to collect system-wide stats.

Consider tweaking /proc/sys/kernel/perf_event_paranoid, which controls use of the performance events system by unprivileged users (without CAP_SYS_ADMIN).

The current value is 2:

-1: Allow use of (almost) all events by all users
Ignore mlock limit after perf_event_mlock_kb without CAP_IPC_LOCK

Quoted 1 line

= 0: Disallow ftrace function tracepoint by users without CAP_SYS_ADMIN

Disallow raw tracepoint access by users without CAP_SYS_ADMIN

Quoted 2 lines

= 1: Disallow CPU event access by users without CAP_SYS_ADMIN = 2: Disallow kernel profiling by users without CAP_SYS_ADMIN

To make this setting permanent, edit /etc/sysctl.conf too, e.g.:

kernel.perf_event_paranoid = -1

Perhaps the ptrace rule doesn't allow to trace some processes. But anyway the global --per-thread mode had better ignore such errors and continue working on other threads.

This patch will record the index of error thread in perf_evsel__open() and remove this thread before retrying.

For example (run with non-root, kernel.perf_event_paranoid isn't set):

jinyao@skl:~$ perf stat --per-thread
^C
  Performance counter stats for 'system wide':
           vmstat-3458               6.171984      cpu-clock:u (msec)        #    0.000 CPUs utilized
             perf-3670               0.515599      cpu-clock:u (msec)        #    0.000 CPUs utilized
           vmstat-3458              1,163,643      cycles:u                  #    0.189 GHz
             perf-3670                 40,881      cycles:u                  #    0.079 GHz
           vmstat-3458              1,410,238      instructions:u            #    1.21  insn per cycle
             perf-3670                  3,536      instructions:u            #    0.09  insn per cycle
           vmstat-3458                288,937      branches:u                #   46.814 M/sec
             perf-3670                    936      branches:u                #    1.815 M/sec
           vmstat-3458                 15,195      branch-misses:u           #    5.26% of all branches
             perf-3670                     76      branch-misses:u           #    8.12% of all branches
Quoted 1 line
       12.651675247 seconds time elapsed

could we use the existing code like in attached patch?

we might also swap following warning for some generic one:
WARNING: Ignored open failure for pid 28392

jirka

---

tools/perf/builtin-stat.c +3 -0
index 98bf9d32f222..f9d4d3ad6fc5 100644
@@ -276,6 +276,9 @@ static int create_perf_stat_counter(struct perf_evsel *evsel)
  			attr->enable_on_exec = 1;
  	}
 
+	if (stat_config.aggr_mode == AGGR_THREAD)
+		evsel->ignore_missing_thread = true;
+
  	if (target__has_cpu(&target) && !target__has_per_thread(&target))
  		return perf_evsel__open_per_cpu(evsel, perf_evsel__cpus(evsel));
 
tools/perf/util/evsel.c +1 -1
index 8f971a2301d1..509ee175bc97 100644
@@ -1656,7 +1656,7 @@ static bool ignore_missing_thread(struct perf_evsel *evsel,
  		return false;
 
  	/* The -ESRCH is perf event syscall errno for pid's not found. */
-	if (err != -ESRCH)
+	if (err != -ESRCH && err != -EACCES)
  		return false;
 
  	/* If there's only one thread, let it fail. */
#3 lore raw
Quoted 1 line · Yao

Just tested. But looks it's not OK for '--per-thread' case.

yea, I haven't tested much.. might need soem tweaking, but my point was that it could be doable on one place instead of introducing another if possible

jirka

author
#4 lore raw
Quoted 9 lines · Jiri Olsa
Quoted 1 line · Yao

Just tested. But looks it's not OK for '--per-thread' case.

yea, I haven't tested much.. might need soem tweaking, but my point was that it could be doable on one place instead of introducing another if possible

jirka

Yes, I understand. It'd better we can put the code to more common places.

Actually I used the similar patch as yours at first. While I found it didn't work for the case of system-wide '--per-thread' then I developed current one.

Thanks
Jin Yao

author
#5 lore raw
In reply to Jiri Olsa
Quoted 9 lines · Jiri Olsa
Quoted 1 line · Yao

Just tested. But looks it's not OK for '--per-thread' case.

yea, I haven't tested much.. might need soem tweaking, but my point was that it could be doable on one place instead of introducing another if possible

jirka

Hi Jiri,

I ever considered to move the operation of removing error thread to perf_evsel__fallback(). The perf_evsel__fallback() is common code and it's shared by perf report, perf stat and perf top.

While finally I think it'd better let the caller decide to remove error thread and try again, or just return the warning message. perf_evsel__fallback() probably doesn't know what the caller want to do.

That's my current thinking. Maybe there will be a better fix...

Thanks
Jin Yao

Acked-by
#6 lore raw
Quoted 19 lines · Yao
Quoted 9 lines · Jiri Olsa
Quoted 1 line · Yao

Just tested. But looks it's not OK for '--per-thread' case.

yea, I haven't tested much.. might need soem tweaking, but my point was that it could be doable on one place instead of introducing another if possible

jirka

Hi Jiri,

I ever considered to move the operation of removing error thread to perf_evsel__fallback(). The perf_evsel__fallback() is common code and it's shared by perf report, perf stat and perf top.

Quoted 5 lines

While finally I think it'd better let the caller decide to remove error thread and try again, or just return the warning message. perf_evsel__fallback() probably doesn't know what the caller want to do.

That's my current thinking. Maybe there will be a better fix...

ok, can't think of better fix atm.. looks good ;-)

thanks,
jirka

author
#7 lore raw
Quoted 24 lines · Jiri Olsa
Quoted 23 lines · Yao
Quoted 9 lines · Jiri Olsa
Quoted 1 line · Yao

Just tested. But looks it's not OK for '--per-thread' case.

yea, I haven't tested much.. might need soem tweaking, but my point was that it could be doable on one place instead of introducing another if possible

jirka

Hi Jiri,

I ever considered to move the operation of removing error thread to perf_evsel__fallback(). The perf_evsel__fallback() is common code and it's shared by perf report, perf stat and perf top.

While finally I think it'd better let the caller decide to remove error thread and try again, or just return the warning message. perf_evsel__fallback() probably doesn't know what the caller want to do.

Quoted 9 lines
Quoted 1 line

That's my current thinking. Maybe there will be a better fix...

ok, can't think of better fix atm.. looks good ;-)

thanks,
jirka

Hi Arnaldo,

Could this fix be accepted?

Thanks
Jin Yao

#8 lore raw

Em Tue, Feb 27, 2018 at 08:53:13AM +0800, Jin, Yao escreveu:

Quoted 33 lines · Jiri Olsa
Quoted 30 lines · Yao
Quoted 19 lines · Jiri Olsa
Quoted 9 lines · Yao
Quoted 1 line

Just tested. But looks it's not OK for '--per-thread' case.

yea, I haven't tested much.. might need soem tweaking, but my point was that it could be doable on one place instead of introducing another if possible

jirka

Hi Jiri,

I ever considered to move the operation of removing error thread to perf_evsel__fallback(). The perf_evsel__fallback() is common code and it's shared by perf report, perf stat and perf top.

Quoted 5 lines

While finally I think it'd better let the caller decide to remove error thread and try again, or just return the warning message. perf_evsel__fallback() probably doesn't know what the caller want to do.

That's my current thinking. Maybe there will be a better fix...

ok, can't think of better fix atm.. looks good ;-)

Quoted 7 lines
Quoted 3 lines

thanks,
jirka

Hi Arnaldo,

Could this fix be accepted?

yeah, its just strange that that err_thread is not used, applied.

- Arnaldo

#9 lore raw
In reply to Jin Yao
perf stat: Ignore error thread when enabling system-wide --per-thread
Commit-ID:  ab6c79b819f5a50cf41a11ebec17bef63b530333
Gitweb:     https://git.kernel.org/tip/ab6c79b819f5a50cf41a11ebec17bef63b530333
Author:     Jin Yao <[email protected]>
AuthorDate: Tue, 16 Jan 2018 23:43:08 +0800
Committer:  Arnaldo Carvalho de Melo <[email protected]>
CommitDate: Tue, 27 Feb 2018 11:29:21 -0300

perf stat: Ignore error thread when enabling system-wide --per-thread

If we execute 'perf stat --per-thread' with non-root account (even set kernel.perf_event_paranoid = -1 yet), it reports the error:

  jinyao@skl:~$ perf stat --per-thread
  Error:
  You may not have permission to collect system-wide stats.

Consider tweaking /proc/sys/kernel/perf_event_paranoid, which controls use of the performance events system by unprivileged users (without CAP_SYS_ADMIN).

The current value is 2:

-1: Allow use of (almost) all events by all users
Ignore mlock limit after perf_event_mlock_kb without CAP_IPC_LOCK

Quoted 1 line

= 0: Disallow ftrace function tracepoint by users without CAP_SYS_ADMIN

Disallow raw tracepoint access by users without CAP_SYS_ADMIN

Quoted 2 lines

= 1: Disallow CPU event access by users without CAP_SYS_ADMIN = 2: Disallow kernel profiling by users without CAP_SYS_ADMIN

To make this setting permanent, edit /etc/sysctl.conf too, e.g.:

kernel.perf_event_paranoid = -1

Perhaps the ptrace rule doesn't allow to trace some processes. But anyway the global --per-thread mode had better ignore such errors and continue working on other threads.

This patch will record the index of error thread in perf_evsel__open() and remove this thread before retrying.

For example (run with non-root, kernel.perf_event_paranoid isn't set):

  jinyao@skl:~$ perf stat --per-thread
  ^C
   Performance counter stats for 'system wide':
         vmstat-3458    6.171984   cpu-clock:u (msec) #  0.000 CPUs utilized
           perf-3670    0.515599   cpu-clock:u (msec) #  0.000 CPUs utilized
         vmstat-3458   1,163,643   cycles:u           #  0.189 GHz
           perf-3670      40,881   cycles:u           #  0.079 GHz
         vmstat-3458   1,410,238   instructions:u     #  1.21  insn per cycle
           perf-3670       3,536   instructions:u     #  0.09  insn per cycle
         vmstat-3458     288,937   branches:u         # 46.814 M/sec
           perf-3670         936   branches:u         #  1.815 M/sec
         vmstat-3458      15,195   branch-misses:u    #  5.26% of all branches
           perf-3670          76   branch-misses:u    #  8.12% of all branches
        12.651675247 seconds time elapsed
---
 tools/perf/builtin-stat.c    | 14 +++++++++++++-
 tools/perf/util/evsel.c      |  3 +++
 tools/perf/util/thread_map.c |  1 +
 tools/perf/util/thread_map.h |  1 +
 4 files changed, 18 insertions(+), 1 deletion(-)
tools/perf/builtin-stat.c +13 -1
index fadcff52cd09..6214d2b220b2 100644
@@ -637,7 +637,19 @@ try_again:
                                 if (verbose > 0)
                                         ui__warning("%s\n", msg);
                                 goto try_again;
-                        }
+			} else if (target__has_per_thread(&target) &&
+				   evsel_list->threads &&
+				   evsel_list->threads->err_thread != -1) {
+				/*
+				 * For global --per-thread case, skip current
+				 * error thread.
+				 */
+				if (!thread_map__remove(evsel_list->threads,
+							evsel_list->threads->err_thread)) {
+					evsel_list->threads->err_thread = -1;
+					goto try_again;
+				}
+			}
 
 			perf_evsel__open_strerror(counter, &target,
 						  errno, msg, sizeof(msg));
tools/perf/util/evsel.c +3 -0
index ef351688b797..b56e1c2ddaee 100644
@@ -1915,6 +1915,9 @@ try_fallback:
 		goto fallback_missing_features;
 	}
 out_close:
+	if (err)
+		threads->err_thread = thread;
+
 	do {
 		while (--thread >= 0) {
 			close(FD(evsel, cpu, thread));
tools/perf/util/thread_map.c +1 -0
index 729dad8f412d..5d467d8ae9ab 100644
@@ -32,6 +32,7 @@ static void thread_map__reset(struct thread_map *map, int start, int nr)
 	size_t size = (nr - start) * sizeof(map->map[0]);
 
 	memset(&map->map[start], 0, size);
+	map->err_thread = -1;
 }
 
 static struct thread_map *thread_map__realloc(struct thread_map *map, int nr)
tools/perf/util/thread_map.h +1 -0
index 5ec91cfd1869..2f689c90a8c6 100644
@@ -14,6 +14,7 @@ struct thread_map_data {
 struct thread_map {
 	refcount_t refcnt;
 	int nr;
+	int err_thread;
 	struct thread_map_data map[];
 };