perf tools: Add symfs option for off-box analysis using specified tree

The symfs argument allows analysis of perf.data file using a locally accessible filesystem tree with debug symbols - e.g., tree created during image builds, sshfs mount, loop mounted KVM disk images, USB keys, initrds, etc. Anything with an OS tree can be analyzed from anywhere without the need to populate a local data store with build-ids.

v2 --> v3: handle symfs="/"

v1 --> v2: removed symfs changes from builtin-annotate.c
fixed tab/space inconsistencies changed new strlen() to symbol_conf.symfs[0] added symfs option to Documentation files

---
 tools/perf/Documentation/perf-diff.txt      |    2 +
 tools/perf/Documentation/perf-report.txt    |    3 +
 tools/perf/Documentation/perf-timechart.txt |    2 +
 tools/perf/builtin-diff.c                   |    2 +
 tools/perf/builtin-report.c                 |    2 +
 tools/perf/builtin-timechart.c              |    2 +
 tools/perf/util/hist.c                      |   14 +++++-
 tools/perf/util/symbol.c                    |   57 ++++++++++++++++++++-------
 tools/perf/util/symbol.h                    |    1 +
 9 files changed, 67 insertions(+), 18 deletions(-)

index 6a9ec2b..74d7481 100644
@@ -66,6 +66,8 @@ OPTIONS
 --force::
        Don't complain, do it.
 
+--symfs=<directory>::
+        Look for files with symbols relative to this directory.
 
 SEE ALSO
 --------

index 59a1f57..9ed6079 100644
@@ -113,6 +113,9 @@ OPTIONS
 --force::
         Don't complain, do it.
 
+--symfs=<directory>::
+        Look for files with symbols relative to this directory.
+
 SEE ALSO
 --------
 linkperf:perf-stat[1]

index 4b17883..d7b79e2 100644
@@ -38,6 +38,8 @@ OPTIONS
 --process::
         Select the processes to display, by name or PID
 
+--symfs=<directory>::
+        Look for files with symbols relative to this directory.
 
 SEE ALSO
 --------

index 5e1a043..71126a6 100644
@@ -192,6 +192,8 @@ static const struct option options[] = {
 	OPT_STRING('t', "field-separator", &symbol_conf.field_sep, "separator",
 		   "separator for columns, no spaces will be added between "
 		   "columns '.' is reserved."),
+	OPT_STRING(0, "symfs", &symbol_conf.symfs, "directory",
+		    "Look for files with symbols relative to this directory"),
 	OPT_END()
 };
 

index 904519f..57b0f49 100644
@@ -479,6 +479,8 @@ static const struct option options[] = {
 		   "columns '.' is reserved."),
 	OPT_BOOLEAN('U', "hide-unresolved", &hide_unresolved,
 		    "Only display entries resolved to a symbol"),
+	OPT_STRING(0, "symfs", &symbol_conf.symfs, "directory",
+		    "Look for files with symbols relative to this directory"),
 	OPT_END()
 };
 

index d2fc461..965aa4e 100644
@@ -1021,6 +1021,8 @@ static const struct option options[] = {
 	OPT_CALLBACK('p', "process", NULL, "process",
 		      "process selector. Pass a pid or process name.",
 		       parse_process),
+	OPT_STRING(0, "symfs", &symbol_conf.symfs, "directory",
+		    "Look for files with symbols relative to this directory"),
 	OPT_END()
 };
 

index 2022e87..b6f9f3b 100644
@@ -1092,6 +1092,12 @@ int hist_entry__annotate(struct hist_entry *self, struct list_head *head,
 	FILE *file;
 	int err = 0;
 	u64 len;
+	char symfs_filename[PATH_MAX];
+
+	if (filename) {
+		snprintf(symfs_filename, sizeof(symfs_filename), "%s%s",
+			 symbol_conf.symfs, filename);
+	}
 
 	if (filename == NULL) {
 		if (dso->has_build_id) {

@@ -1100,9 +1106,9 @@ int hist_entry__annotate(struct hist_entry *self, struct list_head *head,
 			return -ENOMEM;
 		}
 		goto fallback;
-	} else if (readlink(filename, command, sizeof(command)) < 0 ||
+	} else if (readlink(symfs_filename, command, sizeof(command)) < 0 ||
 		   strstr(command, "[kernel.kallsyms]") ||
-		   access(filename, R_OK)) {
+		   access(symfs_filename, R_OK)) {
 		free(filename);
 fallback:
 		/*

@@ -1111,6 +1117,8 @@ fallback:
 		 * DSO is the same as when 'perf record' ran.
 		 */
 		filename = dso->long_name;
+		snprintf(symfs_filename, sizeof(symfs_filename), "%s%s",
+			 symbol_conf.symfs, filename);
 		free_filename = false;
 	}
 

@@ -1137,7 +1145,7 @@ fallback:
 		 "objdump --start-address=0x%016Lx --stop-address=0x%016Lx -dS -C %s|grep -v %s|expand",
 		 map__rip_2objdump(map, sym->start),
 		 map__rip_2objdump(map, sym->end),
-		 filename, filename);
+		 symfs_filename, filename);
 
 	pr_debug("Executing: %s\n", command);
 

index 038f220..bbe90d1 100644
@@ -85,6 +85,7 @@ struct symbol_conf {
        struct strlist	*dso_list,
 			*comm_list,
 			*sym_list;
+	const char	*symfs;
 };
 
 extern struct symbol_conf symbol_conf;

1.7.2.3

Em Thu, Dec 09, 2010 at 01:27:07PM -0700, David Ahern escreveu:

{
	char *resolved = realpath(argv[1], NULL);

	puts(resolved);
}

Em Tue, Dec 14, 2010 at 10:24:28AM -0200, Arnaldo Carvalho de Melo escreveu:

Ok, there are still some problems:

It is escaping the symfs jail for /home/acme/bin/perf, /lib64/libpthread-2.5.so and kernel modules, investigating...

- Arnaldo

[acme@mica linux]$ find /tmp/bla
/tmp/bla
/tmp/bla/lib64
/tmp/bla/lib64/libc-2.5.so
[acme@mica linux]$ perf report -v -g none --stdio --symfs /tmp/bla > /tmp/out
Looking at the vmlinux_path (3 entries long)
Failed to open [kernel.kallsyms], continuing without symbols
dso__synthesize_plt_symbols: problems reading /home/acme/bin/perf PLT info.
dso__synthesize_plt_symbols: problems reading /lib64/libpthread-2.5.so PLT info. dso__synthesize_plt_symbols: problems reading /lib/modules/2.6.37-rc3-00084-gd937616/kernel/drivers/net/ixgbe/ixgbe.ko PLT info. [acme@mica linux]$ cat /tmp/out
# Events: 503 cycles
#
# Overhead Command Shared Object Symbol

# ........  ...........  ........................................................................  .............................
#
    86.67%         perf  [kernel.kallsyms]                                                         0xffffffff8106f697 ! [k] 0xffffffff8106f697
     2.56%      swapper  [kernel.kallsyms]                                                         0xffffffff81089f50 ! [k] 0xffffffff81089f50
     0.84%         perf  /lib64/libc-2.5.so                                                        0x798ed          d [.] __memchr
     0.84%         perf  /home/acme/bin/perf                                                       0x26bf1          G [.] die_builtin
     0.82%         perf  /lib64/libc-2.5.so                                                        0x728ea          d [.] _int_malloc
     0.63%         perf  /lib64/libc-2.5.so                                                        0xe6077          d [.] ___snprintf_chk
     0.63%         perf  /lib64/libc-2.5.so                                                        0x7b528          d [.] __GI_memcpy
     0.63%         perf  /lib64/libc-2.5.so                                                        0x786c0          d [.] index
     0.51%         perf  /lib64/libc-2.5.so                                                        0x625b0          d [.] _IO_getline_internal
     0.42%         perf  /lib64/libc-2.5.so                                                        0x6265a          d [.] _IO_getline_info
     0.42%         perf  /lib64/libc-2.5.so                                                        0x6cbc4          d [.] _IO_new_file_underflow
     0.42%         perf  /lib64/libc-2.5.so                                                        0x6d148          d [.] _IO_un_link_internal
     0.34%         perf  /lib64/libc-2.5.so                                                        0x60c49          d [.] _IO_new_fclose
     0.21%         perf  /lib64/libc-2.5.so                                                        0x5ff40          d [.] __find_specmb
     0.21%         perf  /lib64/libc-2.5.so                                                        0x6c35b          d [.] _IO_new_file_fopen
     0.21%         perf  /home/acme/bin/perf                                                       0x1db4b          G [.] event__synthesize_kernel_mmap
     0.21%         perf  /lib64/libc-2.5.so                                                        0x6d3c5          d [.] _IO_link_in_internal
     0.21%         perf  /lib64/libc-2.5.so                                                        0x34524          d [.] __GI_____strtoll_l_internal
     0.21%         perf  /home/acme/bin/perf                                                       0x4700           G [.] _init
     0.21%         perf  /lib64/libc-2.5.so                                                        0x41907          d [.] _itoa_word
     0.21%         perf  /lib64/libc-2.5.so                                                        0x6148f          d [.] fgets
     0.21%         perf  /home/acme/bin/perf                                                       0x1deca          G [.] event__synthesize_comm
     0.21%         perf  /lib64/libc-2.5.so                                                        0xcfe7d          d [.] __munmap
     0.21%         perf  /lib64/libc-2.5.so                                                        0x79513          d [.] __GI_strstr
     0.21%         perf  /lib64/libc-2.5.so                                                        0x6e060          d [.] _IO_setb_internal
     0.21%         perf  /lib64/libc-2.5.so                                                        0x6bac3          d [.] _IO_file_close_internal
     0.21%         perf  /lib64/libc-2.5.so                                                        0x95dc3          d [.] __getdents64
     0.21%         perf  /lib64/libc-2.5.so                                                        0x6f1b2          d [.] _IO_str_init_static_internal
     0.21%         perf  /lib64/libpthread-2.5.so                                                  0xd210           G [.] __write_nocancel
     0.21%         perf  /lib64/libc-2.5.so                                                        0x9595a          d [.] __readdir_r
     0.21%         perf  /home/acme/bin/perf                                                       0xddbd           G [.] atexit_header
     0.21%         perf  /lib64/libc-2.5.so                                                        0xc4de7          d [.] __GI___open
     0.12%        pcscd  [unknown]                                                                 0x3f44a79110     ! [.] 0x3f44a79110    
     0.09%        pcscd  [kernel.kallsyms]                                                         0xffffffff810e59ac ! [k] 0xffffffff810e59ac
     0.03%         sshd  [kernel.kallsyms]                                                         0xffffffff810f45f9 ! [k] 0xffffffff810f45f9
     0.03%  kworker/0:1  /lib/modules/2.6.37-rc3-00084-gd937616/kernel/drivers/net/ixgbe/ixgbe.ko  0x605            G [k] ixgbe_update_stats
     0.02%         sshd  [unknown]                                                                 0x7f78ed5f8ccb   ! [.] 0x7f78ed5f8ccb  

#
# (For a higher level overview, try: perf report --sort comm,dso) #
[acme@mica linux]$

Em Tue, Dec 14, 2010 at 10:52:45AM -0200, Arnaldo Carvalho de Melo escreveu:

Those 'G' signals means DSO__ORIG_GUEST_KMODULE, since you were not prefixing that case with symfs, it was the jail leak point, fixed, doing some more testing before merging.

yuk. I did not touch the hypervisor paths.

So the 'guest' route is tried all the time because of the origin list -- even when it's not applicable.

David

Em Tue, Dec 14, 2010 at 07:33:27AM -0700, David S. Ahern escreveu:

Right, I don't like the way kvm is supported, will revisit this at some point.

If build-ids were first class citizens, all this would be moot :-\

- Arnaldo

I was not aware of realpath. Simple change to symbol__init?

strace -e trace=open /tmp/build-perf/perf report -v --symfs /tmp/f14-mnt -i /tmp/perf-cs.data

does not show any attempts to open a file not starting with the symfs path.

At the top of dso__synthesize_plt_symbols() is:

	snprintf(name, sizeof(name), "%s%s",
		 symbol_conf.symfs, self->long_name);
	fd = open(name, O_RDONLY);

so the open only gets the relative path.

David

Em Tue, Dec 14, 2010 at 07:27:42AM -0700, David S. Ahern escreveu:

I made symbol__init use realpath to reduce the arg to --symfs.

Probably because all the files you need are in the directories you passed to --symfs, remove one and it will leak.

Anyway, fixed in my perf/core branch, still not propagated from master.k.o to git.k.o :-\

- Arnaldo

Commit-ID:  ec5761eab318e50e69fcf8e63e9edaef5949c067
Gitweb:     http://git.kernel.org/tip/ec5761eab318e50e69fcf8e63e9edaef5949c067
Author:     David Ahern <[email protected]>
AuthorDate: Thu, 9 Dec 2010 13:27:07 -0700
Committer:  Arnaldo Carvalho de Melo <[email protected]>
CommitDate: Tue, 21 Dec 2010 20:17:51 -0200

perf symbols: Add symfs option for off-box analysis using specified tree

The symfs argument allows analysis of perf.data file using a locally accessible filesystem tree with debug symbols - e.g., tree created during image builds, sshfs mount, loop mounted KVM disk images, USB keys, initrds, etc. Anything with an OS tree can be analyzed from anywhere without the need to populate a local data store with build-ids.

Commiter notes:

o Fixed up symfs="/" variants handling.

o prefixed DSO__ORIG_GUEST_KMODULE case with symfs too, avoiding use of files outside the symfs directory.

LKML-Reference: <[email protected]>

---
 tools/perf/Documentation/perf-diff.txt      |    2 +
 tools/perf/Documentation/perf-report.txt    |    3 +
 tools/perf/Documentation/perf-timechart.txt |    2 +
 tools/perf/builtin-diff.c                   |    2 +
 tools/perf/builtin-report.c                 |    2 +
 tools/perf/builtin-timechart.c              |    2 +
 tools/perf/util/hist.c                      |   14 ++++-
 tools/perf/util/symbol.c                    |   72 +++++++++++++++++++++------
 tools/perf/util/symbol.h                    |    1 +
 9 files changed, 81 insertions(+), 19 deletions(-)

index 6a9ec2b..74d7481 100644
@@ -66,6 +66,8 @@ OPTIONS
 --force::
        Don't complain, do it.
 
+--symfs=<directory>::
+        Look for files with symbols relative to this directory.
 
 SEE ALSO
 --------

index fefea77..8ba03d6 100644
@@ -116,6 +116,9 @@ OPTIONS
 --force::
         Don't complain, do it.
 
+--symfs=<directory>::
+        Look for files with symbols relative to this directory.
+
 SEE ALSO
 --------
 linkperf:perf-stat[1]

index 4b17883..d7b79e2 100644
@@ -38,6 +38,8 @@ OPTIONS
 --process::
         Select the processes to display, by name or PID
 
+--symfs=<directory>::
+        Look for files with symbols relative to this directory.
 
 SEE ALSO
 --------

index 97846dc..3153e49 100644
@@ -194,6 +194,8 @@ static const struct option options[] = {
 	OPT_STRING('t', "field-separator", &symbol_conf.field_sep, "separator",
 		   "separator for columns, no spaces will be added between "
 		   "columns '.' is reserved."),
+	OPT_STRING(0, "symfs", &symbol_conf.symfs, "directory",
+		    "Look for files with symbols relative to this directory"),
 	OPT_END()
 };
 

index 4af7ce6..75183a4 100644
@@ -483,6 +483,8 @@ static const struct option options[] = {
 		   "columns '.' is reserved."),
 	OPT_BOOLEAN('U', "hide-unresolved", &hide_unresolved,
 		    "Only display entries resolved to a symbol"),
+	OPT_STRING(0, "symfs", &symbol_conf.symfs, "directory",
+		    "Look for files with symbols relative to this directory"),
 	OPT_END()
 };
 

index 459b5e3..d75084b 100644
@@ -1022,6 +1022,8 @@ static const struct option options[] = {
 	OPT_CALLBACK('p', "process", NULL, "process",
 		      "process selector. Pass a pid or process name.",
 		       parse_process),
+	OPT_STRING(0, "symfs", &symbol_conf.symfs, "directory",
+		    "Look for files with symbols relative to this directory"),
 	OPT_END()
 };
 

index a3b8416..d503670 100644
@@ -1092,6 +1092,12 @@ int hist_entry__annotate(struct hist_entry *self, struct list_head *head,
 	FILE *file;
 	int err = 0;
 	u64 len;
+	char symfs_filename[PATH_MAX];
+
+	if (filename) {
+		snprintf(symfs_filename, sizeof(symfs_filename), "%s%s",
+			 symbol_conf.symfs, filename);
+	}
 
 	if (filename == NULL) {
 		if (dso->has_build_id) {

@@ -1100,9 +1106,9 @@ int hist_entry__annotate(struct hist_entry *self, struct list_head *head,
 			return -ENOMEM;
 		}
 		goto fallback;
-	} else if (readlink(filename, command, sizeof(command)) < 0 ||
+	} else if (readlink(symfs_filename, command, sizeof(command)) < 0 ||
 		   strstr(command, "[kernel.kallsyms]") ||
-		   access(filename, R_OK)) {
+		   access(symfs_filename, R_OK)) {
 		free(filename);
 fallback:
 		/*

@@ -1111,6 +1117,8 @@ fallback:
 		 * DSO is the same as when 'perf record' ran.
 		 */
 		filename = dso->long_name;
+		snprintf(symfs_filename, sizeof(symfs_filename), "%s%s",
+			 symbol_conf.symfs, filename);
 		free_filename = false;
 	}
 

@@ -1137,7 +1145,7 @@ fallback:
 		 "objdump --start-address=0x%016Lx --stop-address=0x%016Lx -dS -C %s|grep -v %s|expand",
 		 map__rip_2objdump(map, sym->start),
 		 map__rip_2objdump(map, sym->end),
-		 filename, filename);
+		 symfs_filename, filename);
 
 	pr_debug("Executing: %s\n", command);
 

index 12defbe..bcd2f98 100644
@@ -86,6 +86,7 @@ struct symbol_conf {
        struct strlist	*dso_list,
 			*comm_list,
 			*sym_list;
+	const char	*symfs;
 };
 
 extern struct symbol_conf symbol_conf;