perf trace fixes
cover perf trace fixes
Hi Arnaldo:
As I mentioned on IRC perf-trace fails on older kernels -- like RHEL6. This set of patches makes it at least usable - though still some problems I am hoping you can fix.
Build perf with these patches and run:
perf trace -- dd if=/dev/zero of=/tmp/zero bs=4096 count=16
you see something like this which is just wrong:
3.684 ( 0.007 ms): write(buf: 2, count: 140737077958816 ) = 27
there is no fd (should be 1 for the write) and all the values are wrong. Perhaps it is an artifact of the older way of doing system call tracing, but I see something goofy with the 3.12 kernel as well:
5.633 ( 0.004 ms): write(fd: 2, buf: 0x7fff9177fee0, count: 24 ) = 24
According to strace that should be a return of 4096.
David Ahern (4):
perf trace: Add support for syscalls vs raw_syscalls perf trace: Fix crash on RHEL6
perf trace: Fix summary percentage when processing files perf trace: Add option to specify machine type
tools/perf/builtin-trace.c | 48 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 4 deletions(-)
Signature · 1 line
1.8.3.4 (Apple Git-47)
PATCH 1/4 perf trace: Add support for syscalls vs raw_syscalls
Older kernels (e.g., RHEL6) do system call tracing via syscalls:sys_{enter,exit} rather than raw_syscalls. Update perf-trace to detect lack of raw_syscalls support and try syscalls.
--- tools/perf/builtin-trace.c | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-)
tools/perf/builtin-trace.c +26 -2
@@ -12,6 +12,7 @@#include "util/thread_map.h" #include "util/stat.h" #include "trace-event.h" +#include "util/parse-events.h" #include <libaudit.h> #include <stdlib.h>
@@ -173,6 +174,10 @@ static struct perf_evsel *perf_evsel__syscall_newtp(const char *direction, void {struct perf_evsel *evsel =perf_evsel__newtp ("raw_syscalls" , direction); +/* older kernel (e.g., RHEL6) use syscalls:{enter,exit} */ +if (evsel ==NULL ) + evsel =perf_evsel__newtp ("syscalls" , direction); +if (evsel) {if (perf_evsel__init_syscall_tp (evsel, handler)) goto out_delete;
@@ -1801,10 +1806,11 @@ static int trace__record(int argc, const char **argv)"-R" ,"-m" ,"1024" ,"-c" ,"1" , -"-e" ,"raw_syscalls:sys_enter,raw_syscalls:sys_exit" , + "-e", }; - rec_argc =ARRAY_SIZE (record_args) + argc; +/* +1 is for the event string below */ + rec_argc =ARRAY_SIZE (record_args) +1 + argc; rec_argv =calloc (rec_argc +1 ,sizeof (char *));if (rec_argv ==NULL )
@@ -1813,6 +1819,17 @@ static int trace__record(int argc, const char **argv)for (i =0 ; i <ARRAY_SIZE (record_args); i++) rec_argv[i] = record_args[i]; +/* event string may be different for older kernels - e.g., RHEL6 */ +if (is_valid_tracepoint ("raw_syscalls:sys_enter" )) + rec_argv[i] ="raw_syscalls:sys_enter,raw_syscalls:sys_exit" ; +else if (is_valid_tracepoint ("syscalls:sys_enter" )) + rec_argv[i] ="syscalls:sys_enter,syscalls:sys_exit" ; +else { +pr_err ("Neither raw_syscalls nor syscalls events exist.\n" ); +return -1 ; + } + i++; +for (j =0 ; j < (unsigned int )argc; j++, i++) rec_argv[i] = argv[j];
@@ -2048,6 +2065,10 @@ static int trace__replay(struct trace *trace) evsel =perf_evlist__find_tracepoint_by_name (session->evlist,"raw_syscalls:sys_enter" ); +/* older kernels have syscalls tp versus raw_syscalls */ +if (evsel ==NULL ) + evsel =perf_evlist__find_tracepoint_by_name (session->evlist, +"syscalls:sys_enter" );if (evsel ==NULL ) {pr_err ("Data file does not have raw_syscalls:sys_enter event\n" ); goto out;
@@ -2061,6 +2082,9 @@ static int trace__replay(struct trace *trace) evsel =perf_evlist__find_tracepoint_by_name (session->evlist,"raw_syscalls:sys_exit" ); +if (evsel ==NULL ) + evsel =perf_evlist__find_tracepoint_by_name (session->evlist, +"syscalls:sys_exit" );if (evsel ==NULL ) {pr_err ("Data file does not have raw_syscalls:sys_exit event\n" ); goto out;
Signature · 1 line
1.8.3.4 (Apple Git-47)
tip-bot for David Ahern
merge bot[tip:perf/core] perf trace: Add support for syscalls vs raw_syscalls
perf trace: Add support for syscalls vs raw_syscalls
Commit-ID: 9aca7f1792c5d2d5d367bbe5cfe204fe40517929 Gitweb: http://git.kernel.org/tip/9aca7f1792c5d2d5d367bbe5cfe204fe40517929 Author: David Ahern <[email protected]> AuthorDate: Wed, 4 Dec 2013 19:41:39 -0700 Committer: Arnaldo Carvalho de Melo <[email protected]> CommitDate: Thu, 5 Dec 2013 10:05:45 -0300
perf trace: Add support for syscalls vs raw_syscalls
Older kernels (e.g., RHEL6) do system call tracing via syscalls:sys_{enter,exit} rather than raw_syscalls. Update perf-trace to detect lack of raw_syscalls support and try syscalls.
--- tools/perf/builtin-trace.c | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-)
tools/perf/builtin-trace.c +26 -2
@@ -12,6 +12,7 @@#include "util/thread_map.h" #include "util/stat.h" #include "trace-event.h" +#include "util/parse-events.h" #include <libaudit.h> #include <stdlib.h>
@@ -173,6 +174,10 @@ static struct perf_evsel *perf_evsel__syscall_newtp(const char *direction, void {struct perf_evsel *evsel =perf_evsel__newtp ("raw_syscalls" , direction); +/* older kernel (e.g., RHEL6) use syscalls:{enter,exit} */ +if (evsel ==NULL ) + evsel =perf_evsel__newtp ("syscalls" , direction); +if (evsel) {if (perf_evsel__init_syscall_tp (evsel, handler)) goto out_delete;
@@ -1801,10 +1806,11 @@ static int trace__record(int argc, const char **argv)"-R" ,"-m" ,"1024" ,"-c" ,"1" , -"-e" ,"raw_syscalls:sys_enter,raw_syscalls:sys_exit" , + "-e", }; - rec_argc =ARRAY_SIZE (record_args) + argc; +/* +1 is for the event string below */ + rec_argc =ARRAY_SIZE (record_args) +1 + argc; rec_argv =calloc (rec_argc +1 ,sizeof (char *));if (rec_argv ==NULL )
@@ -1813,6 +1819,17 @@ static int trace__record(int argc, const char **argv)for (i =0 ; i <ARRAY_SIZE (record_args); i++) rec_argv[i] = record_args[i]; +/* event string may be different for older kernels - e.g., RHEL6 */ +if (is_valid_tracepoint ("raw_syscalls:sys_enter" )) + rec_argv[i] ="raw_syscalls:sys_enter,raw_syscalls:sys_exit" ; +else if (is_valid_tracepoint ("syscalls:sys_enter" )) + rec_argv[i] ="syscalls:sys_enter,syscalls:sys_exit" ; +else { +pr_err ("Neither raw_syscalls nor syscalls events exist.\n" ); +return -1 ; + } + i++; +for (j =0 ; j < (unsigned int )argc; j++, i++) rec_argv[i] = argv[j];
@@ -2048,6 +2065,10 @@ static int trace__replay(struct trace *trace) evsel =perf_evlist__find_tracepoint_by_name (session->evlist,"raw_syscalls:sys_enter" ); +/* older kernels have syscalls tp versus raw_syscalls */ +if (evsel ==NULL ) + evsel =perf_evlist__find_tracepoint_by_name (session->evlist, +"syscalls:sys_enter" );if (evsel ==NULL ) {pr_err ("Data file does not have raw_syscalls:sys_enter event\n" ); goto out;
@@ -2061,6 +2082,9 @@ static int trace__replay(struct trace *trace) evsel =perf_evlist__find_tracepoint_by_name (session->evlist,"raw_syscalls:sys_exit" ); +if (evsel ==NULL ) + evsel =perf_evlist__find_tracepoint_by_name (session->evlist, +"syscalls:sys_exit" );if (evsel ==NULL ) {pr_err ("Data file does not have raw_syscalls:sys_exit event\n" ); goto out;
PATCH 2/4 perf trace: Fix crash on RHEL6
In reply to David Ahern--- tools/perf/builtin-trace.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
tools/perf/builtin-trace.c +1 -1
@@ -1455,7 +1455,7 @@ static size_t syscall__scnprintf_args(struct syscall *sc, char *bf, size_t size, {size_t printed =0 ; -if (sc->tp_format !=NULL ) { +if ((sc->tp_format !=NULL ) && (sc->tp_format->format.fields !=NULL )) {struct format_field *field;u8 bit =1 ; struct syscall_arg arg = {
Signature · 1 line
1.8.3.4 (Apple Git-47)
Re: [PATCH 2/4] perf trace: Fix crash on RHEL6
Em Wed, Dec 04, 2013 at 07:41:40PM -0700, David Ahern escreveu:
Quoted 1 line
Signed-off-by David Ahern <[email protected]>
What would make no fields to be present if tp_format is not NULL, i.e. if the /format file was successfully parsed?
- Arnaldo
Quoted 19 lines
--- tools/perf/builtin-trace.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)tools/perf/builtin-trace.c +1
-1@@ -1455,7 +1455,7 @@ static size_t syscall__scnprintf_args(struct syscall *sc, char *bf, size_t size, {size_t printed =0 ; -if (sc->tp_format !=NULL ) { +if ((sc->tp_format !=NULL ) && (sc->tp_format->format.fields !=NULL )) {struct format_field *field;u8 bit =1 ; struct syscall_arg arg = {Signature · 1 line
1.8.3.4 (Apple Git-47)
Re: [PATCH 2/4] perf trace: Fix crash on RHEL6
Quoted 5 lines · Arnaldo Carvalho de Melo
Em Wed, Dec 04, 2013 at 07:41:40PM -0700, David Ahern escreveu:
Quoted 1 line
Signed-off-by David Ahern <[email protected]>What would make no fields to be present if
tp_formatis not NULL, i.e. if the /format file was successfully parsed?
Sometimes it is not successfully parsed. I see failed to parse messages, so most likely the root cause. I was posting this one and the first one hoping you could take a look at RHEL6 problems. I think it is mostly an old kernel thing.
David
Re: [PATCH 2/4] perf trace: Fix crash on RHEL6
Em Thu, Dec 05, 2013 at 07:27:52AM -0700, David Ahern escreveu:
Quoted 11 lines · Arnaldo Carvalho de Melo
Quoted 5 lines
Em Wed, Dec 04, 2013 at 07:41:40PM -0700, David Ahern escreveu:
Quoted 1 line
Signed-off-by David Ahern <[email protected]>What would make no fields to be present if
tp_formatis not NULL, i.e. if the /format file was successfully parsed?Sometimes it is not successfully parsed. I see failed to parse messages, so most likely the root cause. I was posting this one and the first one hoping you could take a look at RHEL6 problems. I think it is mostly an old kernel thing.
OK, just questioning because perhaps you had some insight and had just not posted it as the changeset log msg.
- Arnaldo
PATCH 3/4 perf trace: Fix summary percentage when processing files
In reply to David AhernGetting a divide by 0 when events are processed from a file:
perf trace -i perf.data -s
...
dnsmasq (1684), 10 events, inf%, 0.000 msec
The problem is that the event count is not incremented as events are processed. With this patch:
perf trace -i perf.data -s
...
dnsmasq (1684), 10 events, 8.9%, 0.000 msec
--- tools/perf/builtin-trace.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
tools/perf/builtin-trace.c +3 -1
@@ -1770,8 +1770,10 @@ static int trace__process_sample(struct perf_tool *tool,if (!trace->full_time && trace->base_time ==0 ) trace->base_time = sample->time; -if (handler) +if (handler) { + ++trace->nr_events;handler (trace, evsel, sample); + }return err; }
Signature · 1 line
1.8.3.4 (Apple Git-47)
tip-bot for David Ahern
merge bot[tip:perf/core] perf trace: Fix summary percentage when processing files
perf trace: Fix summary percentage when processing files
Commit-ID: 3160565f0e005d2ec736ae25cf0a79988c0cbe71 Gitweb: http://git.kernel.org/tip/3160565f0e005d2ec736ae25cf0a79988c0cbe71 Author: David Ahern <[email protected]> AuthorDate: Wed, 4 Dec 2013 19:41:41 -0700 Committer: Arnaldo Carvalho de Melo <[email protected]> CommitDate: Thu, 5 Dec 2013 10:09:58 -0300
perf trace: Fix summary percentage when processing files
Getting a divide by 0 when events are processed from a file:
perf trace -i perf.data -s
...
dnsmasq (1684), 10 events, inf%, 0.000 msec
The problem is that the event count is not incremented as events are processed. With this patch:
perf trace -i perf.data -s
...
dnsmasq (1684), 10 events, 8.9%, 0.000 msec
--- tools/perf/builtin-trace.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
tools/perf/builtin-trace.c +3 -1
@@ -1770,8 +1770,10 @@ static int trace__process_sample(struct perf_tool *tool,if (!trace->full_time && trace->base_time ==0 ) trace->base_time = sample->time; -if (handler) +if (handler) { + ++trace->nr_events;handler (trace, evsel, sample); + }return err; }
PATCH 4/4 perf trace: Add option to specify machine type
In reply to David AhernPerhaps there is a better way to do this; I could not think of one and I don't see any field in the tracepoint that can be leveraged. So ...
perf-trace autodetects the machine type (e.g., i386, x86_64, etc) via libaudit. When running 32-bit apps on a 64-bit kernel the wrong machine type is used to convert syscall numbers to names leading to wrong information getting displayed to the user. This option allows the user to override the machine type to use.
--- tools/perf/builtin-trace.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+)
tools/perf/builtin-trace.c +14 -0
@@ -2274,6 +2274,7 @@ int cmd_trace(int argc, const char **argv, const char *prefix __maybe_unused) };const char *output_name =NULL ;const char *ev_qualifier_str =NULL ; +const char *machine_str =NULL ;const struct option trace_options[] = {OPT_BOOLEAN (0 ,"comm" , &trace.show_comm,"show the thread COMM next to its id" ),
@@ -2308,6 +2309,8 @@ int cmd_trace(int argc, const char **argv, const char *prefix __maybe_unused) "Show only syscall summary with statistics"),OPT_BOOLEAN ('S' ,"with-summary" , &trace.summary,"Show all syscalls and summary with statistics" ), +OPT_STRING ('M' ,NULL , &machine_str,"x86|x86_64" , +"Advanced: machine type for converting system calls: x86, x86_64" ), OPT_END() };int err;
@@ -2318,6 +2321,17 @@ int cmd_trace(int argc, const char **argv, const char *prefix __maybe_unused) argc =parse_options (argc, argv, trace_options, trace_usage,0 ); +if (machine_str) { +if (strcmp (machine_str,"x86" ) ==0 ) + trace.audit.machine =MACH_X86 ; +else if (strcmp (machine_str,"x86_64" ) ==0 ) + trace.audit.machine =MACH_86_64 ; +else { +pr_err ("Invalid machine type\n" ); +return -EINVAL ; + } + } +/* summary_only implies summary option, but don't overwrite summary if set */ if (trace.summary_only) trace.summary = trace.summary_only;
Signature · 1 line
1.8.3.4 (Apple Git-47)
Re: [PATCH 4/4] perf trace: Add option to specify machine type
Em Wed, Dec 04, 2013 at 07:41:42PM -0700, David Ahern escreveu:
Quoted 5 lines
Perhaps there is a better way to do this; I could not think of one and I don't see any field in the tracepoint that can be leveraged. So ...
perf-trace autodetects the machine type (e.g., i386,
x86_64, etc) via libaudit.
And that means that using perf.data files from another machine, say, ARM, will produce completely bogus results :-\
We need a way to store this info in the perf.data header, i.e. use the same algorithm that libaudit uses in audit_detect_machine() (and set the open_id as well, btw) at 'perf record' time and store it somewhere.
What we have now that could be used?
Lets see:
[acme@zoo linux]$ perf report | grep 'arch'
# arch : x86_64 [acme@zoo linux]$
Would that be enough? Stephane?
- Arnaldo
Quoted 56 lines
When running 32-bit apps on a 64-bit kernel the wrong machine type is used to convert syscall numbers to names leading to wrong information getting displayed to the user. This option allows the user to override the machine type to use.
Signed-off-by David Ahern <[email protected]>Cc Ingo Molnar <[email protected]>Cc Peter Zijlstra <[email protected]>Cc Frederic Weisbecker <[email protected]>Cc Jiri Olsa <[email protected]>Cc Namhyung Kim <[email protected]>--- tools/perf/builtin-trace.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+)tools/perf/builtin-trace.c +14
-0@@ -2274,6 +2274,7 @@ int cmd_trace(int argc, const char **argv, const char *prefix __maybe_unused) };const char *output_name =NULL ;const char *ev_qualifier_str =NULL ; +const char *machine_str =NULL ;const struct option trace_options[] = {OPT_BOOLEAN (0 ,"comm" , &trace.show_comm,"show the thread COMM next to its id" ),@@ -2308,6 +2309,8 @@ int cmd_trace(int argc, const char **argv, const char *prefix __maybe_unused) "Show only syscall summary with statistics"),OPT_BOOLEAN ('S' ,"with-summary" , &trace.summary,"Show all syscalls and summary with statistics" ), +OPT_STRING ('M' ,NULL , &machine_str,"x86|x86_64" , +"Advanced: machine type for converting system calls: x86, x86_64" ), OPT_END() };int err;@@ -2318,6 +2321,17 @@ int cmd_trace(int argc, const char **argv, const char *prefix __maybe_unused) argc =parse_options (argc, argv, trace_options, trace_usage,0 ); +if (machine_str) { +if (strcmp (machine_str,"x86" ) ==0 ) + trace.audit.machine =MACH_X86 ; +else if (strcmp (machine_str,"x86_64" ) ==0 ) + trace.audit.machine =MACH_86_64 ; +else { +pr_err ("Invalid machine type\n" ); +return -EINVAL ; + } + } +/* summary_only implies summary option, but don't overwrite summary if set */ if (trace.summary_only) trace.summary = trace.summary_only;Signature · 1 line
1.8.3.4 (Apple Git-47)
Re: [PATCH 4/4] perf trace: Add option to specify machine type
Quoted 9 lines · Arnaldo Carvalho de Melo
Em Wed, Dec 04, 2013 at 07:41:42PM -0700, David Ahern escreveu:
Quoted 5 lines
Perhaps there is a better way to do this; I could not think of one and I don't see any field in the tracepoint that can be leveraged. So ...
perf-trace autodetects the machine type (e.g., i386,
x86_64, etc) via libaudit.And that means that using perf.data files from another machine, say, ARM, will produce completely bogus results :-\
Indeed all offline analysis of trace data is wrong because it uses the arch of the machine running perf. I am not hitting that problem, but analysis on box.
Quoted 7 lines
We need a way to store this info in the perf.data header, i.e. use the same algorithm that libaudit uses in
audit_detect_machine()(and set theopen_idas well, btw) at 'perf record' time and store it somewhere.What we have now that could be used?
Quoted 7 lines
Lets see:
[acme@zoo linux]$ perf report | grep 'arch'
# arch : x86_64 [acme@zoo linux]$Would that be enough? Stephane?
That fixes one part -- using the arch in the file to translate system calls in the file.
But it is a per process problem: Some are 32-bit; others are 64-bit. I was hoping there was some option at the syscall entry level to tag the bitness. And then there is the problem of needing a solution that works from RHEL6 forward. This patch at least allows me to have sensible data until something more clever is thought of.
David
Re: [PATCH 4/4] perf trace: Add option to specify machine type
Em Thu, Dec 05, 2013 at 07:33:20AM -0700, David Ahern escreveu:
Quoted 28 lines · Arnaldo Carvalho de Melo
Quoted 9 lines
Em Wed, Dec 04, 2013 at 07:41:42PM -0700, David Ahern escreveu:
Quoted 5 lines
Perhaps there is a better way to do this; I could not think of one and I don't see any field in the tracepoint that can be leveraged. So ...
perf-trace autodetects the machine type (e.g., i386,
x86_64, etc) via libaudit.And that means that using perf.data files from another machine, say, ARM, will produce completely bogus results :-\
Indeed all offline analysis of trace data is wrong because it uses the arch of the machine running perf. I am not hitting that problem, but analysis on box.
Quoted 13 lines
We need a way to store this info in the perf.data header, i.e. use the same algorithm that libaudit uses in
audit_detect_machine()(and set theopen_idas well, btw) at 'perf record' time and store it somewhere.What we have now that could be used?
Lets see:
[acme@zoo linux]$ perf report | grep 'arch'
# arch : x86_64 [acme@zoo linux]$
Quoted 8 lines
Quoted 1 line
Would that be enough? Stephane?
That fixes one part -- using the arch in the file to translate system calls in the file.
But it is a per process problem: Some are 32-bit; others are 64-bit. I was hoping there was some option at the syscall entry level to tag the bitness. And then there is the problem of needing a solution
We need to look at how compat syscalls work and how that translates to our raw_syscalls tracing needs :-\
Quoted 4 lines
that works from RHEL6 forward. This patch at least allows me to have sensible data until something more clever is thought of.
David
Quoted 10 lines · David Ahern
Hi Arnaldo:
As I mentioned on IRC perf-trace fails on older kernels -- like RHEL6. This set of patches makes it at least usable - though still some problems I am hoping you can fix.
Build perf with these patches and run:
perf trace -- dd if=/dev/zero of=/tmp/zero bs=4096 count=16
Quoted 8 lines
you see something like this which is just wrong:
3.684 ( 0.007 ms): write(buf: 2, count: 140737077958816 ) = 27there is no fd (should be 1 for the write) and all the values are wrong. Perhaps it is an artifact of the older way of doing system call tracing, but I see something goofy with the 3.12 kernel as well:
5.633 ( 0.004 ms): write(fd: 2, buf: 0x7fff9177fee0, count: 24 ) = 24
forget this last comment about 3.12; it works fine. That write entry is the final write by dd:
write(2, "65536 bytes (66 kB) copied", 2665536 bytes (66 kB) copied) = 26
That one is fine and looking up I see the 4096 lines as expected.
For RHEL6, the problem is there for the 4096 lines:
32.641 ( 0.005 ms): read(buf: 0, count: 27258880) = 4096 32.655 ( 0.009 ms): write(buf: 1, count: 27258880) = 4096
buf and count are wrong. Adding -e write to the perf-trace I get:
33.775 ( 0.031 ms): write(buf: 1</tmp/zero>, count: 37437440) = 4096
which suggests an off-by-1 error with parsing syscalls versus raw_syscalls. I am hoping you have an idea on how to fix that.
David