perf tools: Fix seg fault with Intel PT
Adrian Hunter
reviewercover perf tools: Fix seg fault with Intel PT
Hi
Here are a couple of small fixes for problems noticed on v4.5-rc1, although the first goes back to v4.4 and is cc stable.
Adrian Hunter (2):
perf tools: Fix seg fault with Intel PT perf tools: Fix another seg fault using Intel PT
tools/perf/util/parse-events.c | 3 +++ tools/perf/util/thread.c | 2 ++ tools/perf/util/thread.h | 6 ++---- 3 files changed, 7 insertions(+), 4 deletions(-)
Regards
Adrian
Adrian Hunter
reviewerPATCH 1/2 perf tools: Fix seg fault with Intel PT
Fix segmentation fault using:
perf record -e intel_pt//u uname
Back trace:
0 tracepoint_error (err=13, err@entry=-13, sys=sys@entry=0x18706c0 "sched", name=name@entry=0x1871c70 "sched_switch", e=<optimized out>, e=<optimized out>) at util/parse-events.c:416 1 0x000000000049791c in add_tracepoint (head_config=0x0, err=0x0, evt_name=0x1871c70 "sched_switch", sys_name=0x18706c0 "sched", idx=<optimized out>, list=<optimized out>)
at util/parse-events.c:433
2 add_tracepoint_event (list=<optimized out>, idx=<optimized out>, sys_name=0x18706c0 "sched", evt_name=0x1871c70 "sched_switch", err=0x0, head_config=0x0) at util/parse-events.c:498
3 0x0000000000498ec4 in parse_events_add_tracepoint (list=list@entry=0x1871c90, idx=idx@entry=0x7fffffffba30, sys=0x18706c0 "sched", event=0x1871c70 "sched_switch", err=err@entry=0x0,
head_config=head_config@entry=0x0) at util/parse-events.c:936
4 0x00000000004ccff2 in parse_events_parse (_data=_data@entry=0x7fffffffba20, scanner=0x18705f0) at util/parse-events.y:391
5 0x000000000049a567 in parse_events__scanner (start_token=258, data=0x7fffffffba20, str=0x0) at util/parse-events.c:1361
6 parse_events (evlist=evlist@entry=0x1871150, str=str@entry=0x55ef6c "sched:sched_switch", err=err@entry=0x0) at util/parse-events.c:1401
7 0x00000000004eb4b6 in perf_evlist__can_select_event (evlist=evlist@entry=0x186fb90, str=str@entry=0x55ef6c "sched:sched_switch") at util/record.c:253
8 0x000000000051e1d3 in intel_pt_track_switches (evlist=0x186fb90) at arch/x86/util/intel-pt.c:362
9 intel_pt_recording_options (itr=0x186ef70, evlist=0x186fb90, opts=0x7c3e08 <record+232>) at arch/x86/util/intel-pt.c:662
10 0x000000000042e244 in cmd_record (argc=1, argv=0x7fffffffe270, prefix=<optimized out>) at builtin-record.c:1260
11 0x000000000047cbd3 in run_builtin (p=p@entry=0x7cf2e8 <commands+168>, argc=argc@entry=4, argv=argv@entry=0x7fffffffe270) at perf.c:390
12 0x00000000004216a7 in handle_internal_command (argv=0x7fffffffe270, argc=4) at perf.c:451
13 run_argv (argv=0x7fffffffdff0, argcp=0x7fffffffdffc) at perf.c:495
14 main (argc=4, argv=0x7fffffffe270) at perf.c:618Intel PT attempts to find the sched:sched_switch tracepoint but that seg faults if tracefs is not readable, because the error reporting structure is null, as errors are not reported when automatically adding tracepoints. Fix by checking before using.
--- tools/perf/util/parse-events.c | 3 +++ 1 file changed, 3 insertions(+)
tools/perf/util/parse-events.c +3 -0
@@ -399,6 +399,9 @@ static void tracepoint_error(struct parse_events_error *e, int err, {char help[BUFSIZ ]; +if (!e) +return ; +/* * We get error directly from syscall errno ( > 0), * or from encoded pointer's error ( < 0).
Signature · 1 line
1.9.1
Arnaldo Carvalho de Melo
maintainerEm Tue, Jan 26, 2016 at 02:05:20PM +0200, Adrian Hunter escreveu:
Quoted 24 lines
Fix segmentation fault using:
perf record -e intel_pt//u uname
Back trace:
0
tracepoint_error(err=13, err@entry=-13, sys=sys@entry=0x18706c0 "sched", name=name@entry=0x1871c70 "sched_switch", e=<optimized out>, e=<optimized out>) atutil/parse-events.c:41610x000000000049791cinadd_tracepoint(head_config=0x0, err=0x0, evt_name=0x1871c70 "sched_switch", sys_name=0x18706c0 "sched", idx=<optimized out>, list=<optimized out>)at util/parse-events.c:433 2 add_tracepoint_event (list=<optimized out>, idx=<optimized out>, sys_name=0x18706c0 "sched", evt_name=0x1871c70 "sched_switch", err=0x0, head_config=0x0) at util/parse-events.c:498 3 0x0000000000498ec4 in parse_events_add_tracepoint (list=list@entry=0x1871c90, idx=idx@entry=0x7fffffffba30, sys=0x18706c0 "sched", event=0x1871c70 "sched_switch", err=err@entry=0x0, head_config=head_config@entry=0x0) at util/parse-events.c:936 4 0x00000000004ccff2 in parse_events_parse (_data=_data@entry=0x7fffffffba20, scanner=0x18705f0) at util/parse-events.y:391 5 0x000000000049a567 in parse_events__scanner (start_token=258, data=0x7fffffffba20, str=0x0) at util/parse-events.c:1361 6 parse_events (evlist=evlist@entry=0x1871150, str=str@entry=0x55ef6c "sched:sched_switch", err=err@entry=0x0) at util/parse-events.c:1401 7 0x00000000004eb4b6 in perf_evlist__can_select_event (evlist=evlist@entry=0x186fb90, str=str@entry=0x55ef6c "sched:sched_switch") at util/record.c:253 8 0x000000000051e1d3 in intel_pt_track_switches (evlist=0x186fb90) at arch/x86/util/intel-pt.c:362 9 intel_pt_recording_options (itr=0x186ef70, evlist=0x186fb90, opts=0x7c3e08 <record+232>) at arch/x86/util/intel-pt.c:662 10 0x000000000042e244 in cmd_record (argc=1, argv=0x7fffffffe270, prefix=<optimized out>) at builtin-record.c:1260 11 0x000000000047cbd3 in run_builtin (p=p@entry=0x7cf2e8 <commands+168>, argc=argc@entry=4, argv=argv@entry=0x7fffffffe270) at perf.c:390 12 0x00000000004216a7 in handle_internal_command (argv=0x7fffffffe270, argc=4) at perf.c:451 13 run_argv (argv=0x7fffffffdff0, argcp=0x7fffffffdffc) at perf.c:495 14 main (argc=4, argv=0x7fffffffe270) at perf.c:618
Quoted 2 lines
Intel PT attempts to find the sched:sched_switch tracepoint but that seg faults if tracefs is not readable, because the error reporting
How can I reproduce this problem? I tried your command but that doesn't cause any problem:
[acme@jouet linux]$ ls -la /sys/kernel/debug/tracing/
ls: cannot access /sys/kernel/debug/tracing/: Permission denied [acme@jouet linux]$ ls -la /sys/kernel/debug/
ls: cannot open directory /sys/kernel/debug/: Permission denied [acme@jouet linux]$ perf record -e intel_pt//u uname Linux
[ perf record: Woken up 1 times to write data ]
[ perf record: Captured and wrote 0.041 MB perf.data ] [acme@jouet linux]$ perf list intel
List of pre-defined events (to be used in -e):
intel_bts// [Kernel PMU event]
intel_pt// [Kernel PMU event]
[acme@jouet linux]$ perf record -e intel_pt//u uname Linux
[ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.041 MB perf.data ] [acme@jouet linux]$
[acme@jouet linux]$ perf evlist
intel_pt//u
dummy:u
[acme@jouet linux]$
[acme@jouet linux]$ perf evlist -v
intel_pt//u: type: 6, size: 112, config: 0x400, { sample_period, sample_freq }: 1, sample_type: IP|TID|TIME|CPU|IDENTIFIER, read_format: ID, disabled: 1, inherit: 1, exclude_kernel: 1, exclude_hv: 1, enable_on_exec: 1, sample_id_all: 1
dummy:u: type: 1, size: 112, config: 0x9, { sample_period, sample_freq }: 1, sample_type: IP|TID|TIME|CPU|IDENTIFIER, read_format: ID, disabled: 1, inherit: 1, exclude_kernel: 1, exclude_hv: 1, mmap: 1, comm: 1, enable_on_exec: 1, task: 1, sample_id_all: 1, mmap2: 1, comm_exec: 1, context_switch: 1 [acme@jouet linux]$
Humm, ok, so this happens only if tracefs is not readable _and_ this is an older kernel where perf_event_attr.context_switch is not available?
/me tries booting another machine, with an older kernel...
Quoted 26 lines
structure is null, as errors are not reported when automatically adding tracepoints. Fix by checking before using.
Signed-off-by Adrian Hunter <[email protected]>Fixes 196581717d85 ("perf tools: Enhance parsing events tracepoint error output")Cc [email protected] # v4.4+--- tools/perf/util/parse-events.c | 3 +++ 1 file changed, 3 insertions(+)tools/perf/util/parse-events.c +3
-0@@ -399,6 +399,9 @@ static void tracepoint_error(struct parse_events_error *e, int err, {char help[BUFSIZ ]; +if (!e) +return ; +/* * We get error directly from syscall errno ( > 0), * or from encoded pointer's error ( < 0). Signature · 1 line
1.9.1
Adrian Hunter
reviewerQuoted 57 lines · Arnaldo Carvalho de Melo
Em Tue, Jan 26, 2016 at 02:05:20PM +0200, Adrian Hunter escreveu:
Quoted 24 lines
Fix segmentation fault using:
perf record -e intel_pt//u uname
Back trace:
0
tracepoint_error(err=13, err@entry=-13, sys=sys@entry=0x18706c0 "sched", name=name@entry=0x1871c70 "sched_switch", e=<optimized out>, e=<optimized out>) atutil/parse-events.c:41610x000000000049791cinadd_tracepoint(head_config=0x0, err=0x0, evt_name=0x1871c70 "sched_switch", sys_name=0x18706c0 "sched", idx=<optimized out>, list=<optimized out>)at util/parse-events.c:433 2 add_tracepoint_event (list=<optimized out>, idx=<optimized out>, sys_name=0x18706c0 "sched", evt_name=0x1871c70 "sched_switch", err=0x0, head_config=0x0) at util/parse-events.c:498 3 0x0000000000498ec4 in parse_events_add_tracepoint (list=list@entry=0x1871c90, idx=idx@entry=0x7fffffffba30, sys=0x18706c0 "sched", event=0x1871c70 "sched_switch", err=err@entry=0x0, head_config=head_config@entry=0x0) at util/parse-events.c:936 4 0x00000000004ccff2 in parse_events_parse (_data=_data@entry=0x7fffffffba20, scanner=0x18705f0) at util/parse-events.y:391 5 0x000000000049a567 in parse_events__scanner (start_token=258, data=0x7fffffffba20, str=0x0) at util/parse-events.c:1361 6 parse_events (evlist=evlist@entry=0x1871150, str=str@entry=0x55ef6c "sched:sched_switch", err=err@entry=0x0) at util/parse-events.c:1401 7 0x00000000004eb4b6 in perf_evlist__can_select_event (evlist=evlist@entry=0x186fb90, str=str@entry=0x55ef6c "sched:sched_switch") at util/record.c:253 8 0x000000000051e1d3 in intel_pt_track_switches (evlist=0x186fb90) at arch/x86/util/intel-pt.c:362 9 intel_pt_recording_options (itr=0x186ef70, evlist=0x186fb90, opts=0x7c3e08 <record+232>) at arch/x86/util/intel-pt.c:662 10 0x000000000042e244 in cmd_record (argc=1, argv=0x7fffffffe270, prefix=<optimized out>) at builtin-record.c:1260 11 0x000000000047cbd3 in run_builtin (p=p@entry=0x7cf2e8 <commands+168>, argc=argc@entry=4, argv=argv@entry=0x7fffffffe270) at perf.c:390 12 0x00000000004216a7 in handle_internal_command (argv=0x7fffffffe270, argc=4) at perf.c:451 13 run_argv (argv=0x7fffffffdff0, argcp=0x7fffffffdffc) at perf.c:495 14 main (argc=4, argv=0x7fffffffe270) at perf.c:618Quoted 2 lines
Intel PT attempts to find the sched:sched_switch tracepoint but that seg faults if tracefs is not readable, because the error reporting
How can I reproduce this problem? I tried your command but that doesn't cause any problem:
[acme@jouet linux]$ ls -la
/sys/kernel/debug/tracing/
ls: cannot access/sys/kernel/debug/tracing/: Permission denied [acme@jouet linux]$ ls -la/sys/kernel/debug/
ls: cannot open directory/sys/kernel/debug/: Permission denied [acme@jouet linux]$ perf record -e intel_pt//u uname Linux
[ perf record: Woken up 1 times to write data ]
[ perf record: Captured and wrote 0.041 MB perf.data ] [acme@jouet linux]$ perf list intelList of pre-defined events (to be used in -e):
intel_bts// [Kernel PMU event]
intel_pt// [Kernel PMU event][acme@jouet linux]$ perf record -e intel_pt//u uname Linux
[ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.041 MB perf.data ] [acme@jouet linux]$[acme@jouet linux]$ perf evlist
intel_pt//u
dummy:u
[acme@jouet linux]$
Quoted 7 lines
[acme@jouet linux]$ perf evlist -v
intel_pt//u: type: 6, size: 112, config:0x400, {sample_period,sample_freq}: 1,sample_type: IP|TID|TIME|CPU|IDENTIFIER,read_format: ID, disabled: 1, inherit: 1,exclude_kernel: 1,exclude_hv: 1,enable_on_exec: 1,sample_id_all: 1
dummy:u: type: 1, size: 112, config:0x9, {sample_period,sample_freq}: 1,sample_type: IP|TID|TIME|CPU|IDENTIFIER,read_format: ID, disabled: 1, inherit: 1,exclude_kernel: 1,exclude_hv: 1, mmap: 1, comm: 1,enable_on_exec: 1, task: 1,sample_id_all: 1, mmap2: 1,comm_exec: 1,context_switch: 1 [acme@jouet linux]$Humm, ok, so this happens only if tracefs is not readable _and_ this is an older kernel where perf_event_attr.context_switch is not available?
Yes, I should have thought of that, sorry.
Quoted 30 lines
/me tries booting another machine, with an older kernel...
Quoted 26 lines
structure is null, as errors are not reported when automatically adding tracepoints. Fix by checking before using.
Signed-off-by Adrian Hunter <[email protected]>Fixes 196581717d85 ("perf tools: Enhance parsing events tracepoint error output")Cc [email protected] # v4.4+--- tools/perf/util/parse-events.c | 3 +++ 1 file changed, 3 insertions(+)tools/perf/util/parse-events.c +3
-0@@ -399,6 +399,9 @@ static void tracepoint_error(struct parse_events_error *e, int err, {char help[BUFSIZ ]; +if (!e) +return ; +/* * We get error directly from syscall errno ( > 0), * or from encoded pointer's error ( < 0). Signature · 1 line
1.9.1
Arnaldo Carvalho de Melo
maintainerEm Tue, Jan 26, 2016 at 03:34:15PM +0200, Adrian Hunter escreveu:
Quoted 63 lines · Arnaldo Carvalho de Melo
Quoted 62 lines
Em Tue, Jan 26, 2016 at 02:05:20PM +0200, Adrian Hunter escreveu:
Quoted 24 lines
Fix segmentation fault using:
perf record -e intel_pt//u uname
Back trace:
0
tracepoint_error(err=13, err@entry=-13, sys=sys@entry=0x18706c0 "sched", name=name@entry=0x1871c70 "sched_switch", e=<optimized out>, e=<optimized out>) atutil/parse-events.c:41610x000000000049791cinadd_tracepoint(head_config=0x0, err=0x0, evt_name=0x1871c70 "sched_switch", sys_name=0x18706c0 "sched", idx=<optimized out>, list=<optimized out>)at util/parse-events.c:433 2 add_tracepoint_event (list=<optimized out>, idx=<optimized out>, sys_name=0x18706c0 "sched", evt_name=0x1871c70 "sched_switch", err=0x0, head_config=0x0) at util/parse-events.c:498 3 0x0000000000498ec4 in parse_events_add_tracepoint (list=list@entry=0x1871c90, idx=idx@entry=0x7fffffffba30, sys=0x18706c0 "sched", event=0x1871c70 "sched_switch", err=err@entry=0x0, head_config=head_config@entry=0x0) at util/parse-events.c:936 4 0x00000000004ccff2 in parse_events_parse (_data=_data@entry=0x7fffffffba20, scanner=0x18705f0) at util/parse-events.y:391 5 0x000000000049a567 in parse_events__scanner (start_token=258, data=0x7fffffffba20, str=0x0) at util/parse-events.c:1361 6 parse_events (evlist=evlist@entry=0x1871150, str=str@entry=0x55ef6c "sched:sched_switch", err=err@entry=0x0) at util/parse-events.c:1401 7 0x00000000004eb4b6 in perf_evlist__can_select_event (evlist=evlist@entry=0x186fb90, str=str@entry=0x55ef6c "sched:sched_switch") at util/record.c:253 8 0x000000000051e1d3 in intel_pt_track_switches (evlist=0x186fb90) at arch/x86/util/intel-pt.c:362 9 intel_pt_recording_options (itr=0x186ef70, evlist=0x186fb90, opts=0x7c3e08 <record+232>) at arch/x86/util/intel-pt.c:662 10 0x000000000042e244 in cmd_record (argc=1, argv=0x7fffffffe270, prefix=<optimized out>) at builtin-record.c:1260 11 0x000000000047cbd3 in run_builtin (p=p@entry=0x7cf2e8 <commands+168>, argc=argc@entry=4, argv=argv@entry=0x7fffffffe270) at perf.c:390 12 0x00000000004216a7 in handle_internal_command (argv=0x7fffffffe270, argc=4) at perf.c:451 13 run_argv (argv=0x7fffffffdff0, argcp=0x7fffffffdffc) at perf.c:495 14 main (argc=4, argv=0x7fffffffe270) at perf.c:618Quoted 2 lines
Intel PT attempts to find the sched:sched_switch tracepoint but that seg faults if tracefs is not readable, because the error reporting
How can I reproduce this problem? I tried your command but that doesn't cause any problem:
[acme@jouet linux]$ ls -la
/sys/kernel/debug/tracing/
ls: cannot access/sys/kernel/debug/tracing/: Permission denied [acme@jouet linux]$ ls -la/sys/kernel/debug/
ls: cannot open directory/sys/kernel/debug/: Permission denied [acme@jouet linux]$ perf record -e intel_pt//u uname Linux
[ perf record: Woken up 1 times to write data ]
[ perf record: Captured and wrote 0.041 MB perf.data ] [acme@jouet linux]$ perf list intelList of pre-defined events (to be used in -e):
intel_bts// [Kernel PMU event]
intel_pt// [Kernel PMU event][acme@jouet linux]$ perf record -e intel_pt//u uname Linux
[ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.041 MB perf.data ] [acme@jouet linux]$[acme@jouet linux]$ perf evlist
intel_pt//u
dummy:u
[acme@jouet linux]$[acme@jouet linux]$ perf evlist -v
intel_pt//u: type: 6, size: 112, config:0x400, {sample_period,sample_freq}: 1,sample_type: IP|TID|TIME|CPU|IDENTIFIER,read_format: ID, disabled: 1, inherit: 1,exclude_kernel: 1,exclude_hv: 1,enable_on_exec: 1,sample_id_all: 1
dummy:u: type: 1, size: 112, config:0x9, {sample_period,sample_freq}: 1,sample_type: IP|TID|TIME|CPU|IDENTIFIER,read_format: ID, disabled: 1, inherit: 1,exclude_kernel: 1,exclude_hv: 1, mmap: 1, comm: 1,enable_on_exec: 1, task: 1,sample_id_all: 1, mmap2: 1,comm_exec: 1,context_switch: 1 [acme@jouet linux]$
Quoted 4 lines
Quoted 2 lines
Humm, ok, so this happens only if tracefs is not readable _and_ this is an older kernel where perf_event_attr.context_switch is not available?
Yes, I should have thought of that, sorry.
So, I just tried on another machine, with an older kernel
[acme@ssdandy linux]$ uname -r 3.10.0-229.14.1.el7.x86_64
[acme@ssdandy linux]$ ls -la /sys/kernel/debug/
ls: cannot open directory /sys/kernel/debug/: Permission denied [acme@ssdandy linux]$
[acme@ssdandy linux]$ grep tracefs /proc/filesystems [acme@ssdandy linux]$ grep debugfs /proc/filesystems
nodev debugfs
[acme@ssdandy linux]$ ls -la /sys/kernel/debug/
ls: cannot open directory /sys/kernel/debug/: Permission denied [acme@ssdandy linux]$ perf record -e intel_pt//u uname invalid or unsupported event: 'intel_pt//u'
Run 'perf list' for a list of valid events
Usage: perf record [<options>] [<command>]
or: perf record [<options>] -- <command> [<options>]
-e, --event <event> event selector. use 'perf list' to list available events
[acme@ssdandy linux]$
So, this needs:
1) A machine that supports Intel PT
2) A kernel that supports Intel PT
3) A kernel that doesn't have perf_event_attr.context_switch, so that it tries to use sched:sched_switch
4) an unreadable tracefs
What was the kernel where you stumbled on this problem? What machine?
- Arnaldo
Quoted 29 lines
Quoted 29 lines
/me tries booting another machine, with an older kernel...
Quoted 26 lines
structure is null, as errors are not reported when automatically adding tracepoints. Fix by checking before using.
Signed-off-by Adrian Hunter <[email protected]>Fixes 196581717d85 ("perf tools: Enhance parsing events tracepoint error output")Cc [email protected] # v4.4+--- tools/perf/util/parse-events.c | 3 +++ 1 file changed, 3 insertions(+)tools/perf/util/parse-events.c +3
-0@@ -399,6 +399,9 @@ static void tracepoint_error(struct parse_events_error *e, int err, {char help[BUFSIZ ]; +if (!e) +return ; +/* * We get error directly from syscall errno ( > 0), * or from encoded pointer's error ( < 0). Signature · 1 line
1.9.1
Adrian Hunter
reviewerQuoted 93 lines · Arnaldo Carvalho de Melo
Em Tue, Jan 26, 2016 at 03:34:15PM +0200, Adrian Hunter escreveu:
Quoted 63 lines · Arnaldo Carvalho de Melo
Quoted 62 lines
Em Tue, Jan 26, 2016 at 02:05:20PM +0200, Adrian Hunter escreveu:
Quoted 24 lines
Fix segmentation fault using:
perf record -e intel_pt//u uname
Back trace:
0
tracepoint_error(err=13, err@entry=-13, sys=sys@entry=0x18706c0 "sched", name=name@entry=0x1871c70 "sched_switch", e=<optimized out>, e=<optimized out>) atutil/parse-events.c:41610x000000000049791cinadd_tracepoint(head_config=0x0, err=0x0, evt_name=0x1871c70 "sched_switch", sys_name=0x18706c0 "sched", idx=<optimized out>, list=<optimized out>)at util/parse-events.c:433 2 add_tracepoint_event (list=<optimized out>, idx=<optimized out>, sys_name=0x18706c0 "sched", evt_name=0x1871c70 "sched_switch", err=0x0, head_config=0x0) at util/parse-events.c:498 3 0x0000000000498ec4 in parse_events_add_tracepoint (list=list@entry=0x1871c90, idx=idx@entry=0x7fffffffba30, sys=0x18706c0 "sched", event=0x1871c70 "sched_switch", err=err@entry=0x0, head_config=head_config@entry=0x0) at util/parse-events.c:936 4 0x00000000004ccff2 in parse_events_parse (_data=_data@entry=0x7fffffffba20, scanner=0x18705f0) at util/parse-events.y:391 5 0x000000000049a567 in parse_events__scanner (start_token=258, data=0x7fffffffba20, str=0x0) at util/parse-events.c:1361 6 parse_events (evlist=evlist@entry=0x1871150, str=str@entry=0x55ef6c "sched:sched_switch", err=err@entry=0x0) at util/parse-events.c:1401 7 0x00000000004eb4b6 in perf_evlist__can_select_event (evlist=evlist@entry=0x186fb90, str=str@entry=0x55ef6c "sched:sched_switch") at util/record.c:253 8 0x000000000051e1d3 in intel_pt_track_switches (evlist=0x186fb90) at arch/x86/util/intel-pt.c:362 9 intel_pt_recording_options (itr=0x186ef70, evlist=0x186fb90, opts=0x7c3e08 <record+232>) at arch/x86/util/intel-pt.c:662 10 0x000000000042e244 in cmd_record (argc=1, argv=0x7fffffffe270, prefix=<optimized out>) at builtin-record.c:1260 11 0x000000000047cbd3 in run_builtin (p=p@entry=0x7cf2e8 <commands+168>, argc=argc@entry=4, argv=argv@entry=0x7fffffffe270) at perf.c:390 12 0x00000000004216a7 in handle_internal_command (argv=0x7fffffffe270, argc=4) at perf.c:451 13 run_argv (argv=0x7fffffffdff0, argcp=0x7fffffffdffc) at perf.c:495 14 main (argc=4, argv=0x7fffffffe270) at perf.c:618Quoted 2 lines
Intel PT attempts to find the sched:sched_switch tracepoint but that seg faults if tracefs is not readable, because the error reporting
How can I reproduce this problem? I tried your command but that doesn't cause any problem:
[acme@jouet linux]$ ls -la
/sys/kernel/debug/tracing/
ls: cannot access/sys/kernel/debug/tracing/: Permission denied [acme@jouet linux]$ ls -la/sys/kernel/debug/
ls: cannot open directory/sys/kernel/debug/: Permission denied [acme@jouet linux]$ perf record -e intel_pt//u uname Linux
[ perf record: Woken up 1 times to write data ]
[ perf record: Captured and wrote 0.041 MB perf.data ] [acme@jouet linux]$ perf list intelList of pre-defined events (to be used in -e):
intel_bts// [Kernel PMU event]
intel_pt// [Kernel PMU event][acme@jouet linux]$ perf record -e intel_pt//u uname Linux
[ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.041 MB perf.data ] [acme@jouet linux]$[acme@jouet linux]$ perf evlist
intel_pt//u
dummy:u
[acme@jouet linux]$[acme@jouet linux]$ perf evlist -v
intel_pt//u: type: 6, size: 112, config:0x400, {sample_period,sample_freq}: 1,sample_type: IP|TID|TIME|CPU|IDENTIFIER,read_format: ID, disabled: 1, inherit: 1,exclude_kernel: 1,exclude_hv: 1,enable_on_exec: 1,sample_id_all: 1
dummy:u: type: 1, size: 112, config:0x9, {sample_period,sample_freq}: 1,sample_type: IP|TID|TIME|CPU|IDENTIFIER,read_format: ID, disabled: 1, inherit: 1,exclude_kernel: 1,exclude_hv: 1, mmap: 1, comm: 1,enable_on_exec: 1, task: 1,sample_id_all: 1, mmap2: 1,comm_exec: 1,context_switch: 1 [acme@jouet linux]$Quoted 4 lines
Quoted 2 lines
Humm, ok, so this happens only if tracefs is not readable _and_ this is an older kernel where perf_event_attr.context_switch is not available?
Yes, I should have thought of that, sorry.
So, I just tried on another machine, with an older kernel
[acme@ssdandy linux]$ uname -r 3.10.0-229.14.1.el7.x86_64
[acme@ssdandy linux]$ ls -la/sys/kernel/debug/
ls: cannot open directory/sys/kernel/debug/: Permission denied [acme@ssdandy linux]$
[acme@ssdandy linux]$ grep tracefs/proc/filesystems[acme@ssdandy linux]$ grep debugfs/proc/filesystems
nodev debugfs
[acme@ssdandy linux]$ ls -la/sys/kernel/debug/
ls: cannot open directory/sys/kernel/debug/: Permission denied [acme@ssdandy linux]$ perf record -e intel_pt//u uname invalid or unsupported event: 'intel_pt//u'
Run 'perf list' for a list of valid eventsUsage: perf record [<options>] [<command>]
or: perf record [<options>] -- <command> [<options>]-e, --event <event> event selector. use 'perf list' to list available events
[acme@ssdandy linux]$So, this needs:
Quoted 7 lines
1) A machine that supports Intel PT
2) A kernel that supports Intel PT
3) A kernel that doesn't have perf_event_attr.context_switch, so that it tries to use sched:sched_switch
4) an unreadable tracefsWhat was the kernel where you stumbled on this problem? What machine?
Kernel was v4.2, machine was Broadwell.
context_switch was added in v4.3 I think, and PT was v4.1 so I guess v4.1 and v4.2 are affected.
Quoted 33 lines
- Arnaldo
Quoted 29 lines
Quoted 29 lines
/me tries booting another machine, with an older kernel...
Quoted 26 lines
structure is null, as errors are not reported when automatically adding tracepoints. Fix by checking before using.
Signed-off-by Adrian Hunter <[email protected]>Fixes 196581717d85 ("perf tools: Enhance parsing events tracepoint error output")Cc [email protected] # v4.4+--- tools/perf/util/parse-events.c | 3 +++ 1 file changed, 3 insertions(+)tools/perf/util/parse-events.c +3
-0@@ -399,6 +399,9 @@ static void tracepoint_error(struct parse_events_error *e, int err, {char help[BUFSIZ ]; +if (!e) +return ; +/* * We get error directly from syscall errno ( > 0), * or from encoded pointer's error ( < 0). Signature · 1 line
1.9.1
Arnaldo Carvalho de Melo
maintainerEm Tue, Jan 26, 2016 at 04:00:56PM +0200, Adrian Hunter escreveu:
Quoted 11 lines · Arnaldo Carvalho de Melo
Quoted 9 lines
[acme@ssdandy linux]$
So, this needs:
1) A machine that supports Intel PT
2) A kernel that supports Intel PT
3) A kernel that doesn't have perf_event_attr.context_switch, so that it tries to use sched:sched_switch
4) an unreadable tracefs
Quoted 6 lines
Quoted 1 line
What was the kernel where you stumbled on this problem? What machine?
Kernel was v4.2, machine was Broadwell.
context_switchwas added in v4.3 I think, and PT was v4.1 so I guess v4.1 and v4.2 are affected.
Thanks for the info, my new main machine is a t450s, so Broadwell, will boot with 4.1/4.2.
- Arnaldo
tip-bot for Adrian Hunter
merge bot[tip:perf/urgent] perf tools: tracepoint_error() can receive e= NULL, robustify it
In reply to Adrian Hunterperf tools: tracepoint_error() can receive e=NULL, robustify it
Commit-ID: ec183d22cc284a7a1e17f0341219d8ec8ca070cc Gitweb: http://git.kernel.org/tip/ec183d22cc284a7a1e17f0341219d8ec8ca070cc Author: Adrian Hunter <[email protected]> AuthorDate: Tue, 26 Jan 2016 14:05:20 +0200 Committer: Arnaldo Carvalho de Melo <[email protected]> CommitDate: Mon, 1 Feb 2016 11:51:15 -0300
perf tools: tracepoint_error() can receive e=NULL, robustify it
Fixes segmentation fault using, for instance:
(gdb) run record -I -e intel_pt/tsc=1,noretcomp=1/u /bin/ls
Starting program: /home/acme/bin/perf record -I -e intel_pt/tsc=1,noretcomp=1/u /bin/ls Missing separate debuginfos, use: dnf debuginfo-install glibc-2.22-7.fc23.x86_64 [Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
Program received signal SIGSEGV, Segmentation fault.
0 x00000000004b9ea5 in tracepoint_error (e=0x0, err=13, sys=0x19b1370 "sched", name=0x19a5d00 "sched_switch") at util/parse-events.c:410 (gdb) bt
#0 0x00000000004b9ea5 in tracepoint_error (e=0x0, err=13, sys=0x19b1370 "sched", name=0x19a5d00 "sched_switch") at util/parse-events.c:410 #1 0x00000000004b9fc5 in add_tracepoint (list=0x19a5d20, idx=0x7fffffffb8c0, sys_name=0x19b1370 "sched", evt_name=0x19a5d00 "sched_switch", err=0x0, head_config=0x0)
at util/parse-events.c:433
#2 0x00000000004ba334 in add_tracepoint_event (list=0x19a5d20, idx=0x7fffffffb8c0, sys_name=0x19b1370 "sched", evt_name=0x19a5d00 "sched_switch", err=0x0, head_config=0x0)
at util/parse-events.c:498
#3 0x00000000004bb699 in parse_events_add_tracepoint (list=0x19a5d20, idx=0x7fffffffb8c0, sys=0x19b1370 "sched", event=0x19a5d00 "sched_switch", err=0x0, head_config=0x0)
at util/parse-events.c:936
#4 0x00000000004f6eda in parse_events_parse (_data=0x7fffffffb8b0, scanner=0x19a49d0) at util/parse-events.y:391
#5 0x00000000004bc8e5 in parse_events__scanner (str=0x663ff2 "sched:sched_switch", data=0x7fffffffb8b0, start_token=258) at util/parse-events.c:1361
#6 0x00000000004bca57 in parse_events (evlist=0x19a5220, str=0x663ff2 "sched:sched_switch", err=0x0) at util/parse-events.c:1401
#7 0x0000000000518d5f in perf_evlist__can_select_event (evlist=0x19a3b90, str=0x663ff2 "sched:sched_switch") at util/record.c:253
#8 0x0000000000553c42 in intel_pt_track_switches (evlist=0x19a3b90) at arch/x86/util/intel-pt.c:364
#9 0x00000000005549d1 in intel_pt_recording_options (itr=0x19a2c40, evlist=0x19a3b90, opts=0x8edf68 <record+232>) at arch/x86/util/intel-pt.c:664
#10 0x000000000051e076 in auxtrace_record__options (itr=0x19a2c40, evlist=0x19a3b90, opts=0x8edf68 <record+232>) at util/auxtrace.c:539
#11 0x0000000000433368 in cmd_record (argc=1, argv=0x7fffffffde60, prefix=0x0) at builtin-record.c:1264
#12 0x000000000049bec2 in run_builtin (p=0x8fa2a8 <commands+168>, argc=5, argv=0x7fffffffde60) at perf.c:390
#13 0x000000000049c12a in handle_internal_command (argc=5, argv=0x7fffffffde60) at perf.c:451
#14 0x000000000049c278 in run_argv (argcp=0x7fffffffdcbc, argv=0x7fffffffdcb0) at perf.c:495
#15 0x000000000049c60a in main (argc=5, argv=0x7fffffffde60) at perf.c:618(gdb)
Intel PT attempts to find the sched:sched_switch tracepoint but that seg faults if tracefs is not readable, because the error reporting structure is null, as errors are not reported when automatically adding tracepoints. Fix by checking before using.
Committer note:
This doesn't take place in a kernel that supports perf_event_attr.context_switch, that is the default way that will be used for tracking context switches, only in older kernels, like 4.2, in a machine with Intel PT (e.g. Broadwell) for non-priviledged users.
Further info from a similar patch by Wang:
The error is in tracepoint_error: it assumes the 'e' parameter is valid.
However, there are many situation a parse_event() can be called without parse_events_error. See result of
$ grep 'parse_events(.*NULL)' ./tools/perf/ -r'
--- tools/perf/util/parse-events.c | 3 +++ 1 file changed, 3 insertions(+)
tools/perf/util/parse-events.c +3 -0
@@ -399,6 +399,9 @@ static void tracepoint_error(struct parse_events_error *e, int err, {char help[BUFSIZ ]; +if (!e) +return ; +/* * We get error directly from syscall errno ( > 0), * or from encoded pointer's error ( < 0).
Adrian Hunter
reviewerPATCH 2/2 perf tools: Fix another seg fault using Intel PT
In reply to Adrian HunterFix segmentation fault using:
perf record -e intel_pt//u uname
perf script
Back trace:
0 __list_del (next=0x1880710, prev=0x0) at /home/ahunter/git/linux/tools/include/linux/list.h:89 1 __list_del_entry (entry=0x1880710) at /home/ahunter/git/linux/tools/include/linux/list.h:101 2 list_del_init (entry=0x1880710) at /home/ahunter/git/linux/tools/include/linux/list.h:144 3 thread__put (thread=0x1880710) at util/thread.c:104 4 0x00000000004fd699 in intel_pt_free (session=0x186fb90) at util/intel-pt.c:1747 5 0x00000000004c23cc in auxtrace__free (session=0x186fb90) at util/auxtrace.h:511 6 perf_session__delete (session=session@entry=0x186fb90) at util/session.c:181 7 0x0000000000443398 in cmd_script (argc=<optimized out>, argv=<optimized out>, prefix=<optimized out>) at builtin-script.c:2232 8 0x000000000047cbd3 in run_builtin (p=p@entry=0x7cf3a8 <commands+360>, argc=argc@entry=1, argv=argv@entry=0x7fffffffe210) at perf.c:390 9 0x00000000004216a7 in handle_internal_command (argv=0x7fffffffe210, argc=1) at perf.c:451
10 run_argv (argv=0x7fffffffdf90, argcp=0x7fffffffdf9c) at perf.c:495 11 main (argc=1, argv=0x7fffffffe210) at perf.c:618
The seg fault happens when Intel PT "puts" a "struct thread" that has been created as a placeholder for unknown threads. thread__put() assumes that a thread's list node can be deleted, which is not true in the case above because of:
commit fdce6a4edaad ("perf tools: Remove redundant initialization of thread linkage members")
which removed the list node initialization.
Expecting the list node to be re-initialized whenever removing a thread from an rb-tree seems fragile, so fix by taking the list node out of union, so that list_del_init() can be used on it with impunity.
--- tools/perf/util/thread.c | 2 ++ tools/perf/util/thread.h | 6 ++---- 2 files changed, 4 insertions(+), 4 deletions(-)
tools/perf/util/thread.c +2 -0
@@ -56,6 +56,7 @@ struct thread *thread__new(pid_t pid, pid_t tid)list_add (&comm->list, &thread->comm_list);atomic_set (&thread->refcnt,1 ); +INIT_LIST_HEAD (&thread->node);RB_CLEAR_NODE (&thread->rb_node); }
@@ -71,6 +72,7 @@ void thread__delete(struct thread *thread)struct comm *comm, *tmp;BUG_ON (!RB_EMPTY_NODE (&thread->rb_node)); +BUG_ON (!list_empty (&thread->node));thread_stack__free (thread);
tools/perf/util/thread.h +2 -4
@@ -13,10 +13,8 @@struct thread_stack ;struct thread { -union { -struct rb_node rb_node; -struct list_head node; - }; +struct rb_node rb_node; +struct list_head node;struct map_groups *mg;pid_t pid_;/* Not all tools update this */ pid_t tid;
Signature · 1 line
1.9.1