Skip to contentSkip to conversation

libdw dwarf unwind issue

DISCUSSION Jan Kratochvil 2 participants 6 messages Nov 22, 2020 – Dec 2, 2020 1 message not in the archive perf events

lore
Share & tools
lore
#1 lore raw
In reply to a message not in the archive
Quoted 27 lines · Jiri Olsa

When you compiled perf with libunwind, you get:

  $ make VF=1
  $ perf report -i perf.data --stdio

# Children Self Trace output

  # ........  ........  ..............
  #
     100.00%   100.00%  (5640789cdf50)
              |
              |--98.85%--0x7fc8eca0f554
              |          main
              |          main_loop_wait
              |          qemu_poll_ns
              |
               --1.15%--0x7fc8ecdc2ea4
                         iothread_run
                         aio_poll
                         qemu_poll_ns

When you compile perf with libdw, you get:

  $ make VF=1 NO_LIBUNWIND=1
  $ perf report -i perf.data --stdio

# Children Self Trace output

Quoted 8 lines
  # ........  ........  ..............
  #
     100.00%   100.00%  (5640789cdf50)
              |
              |--98.85%--0x45260879
              |          qemu_poll_ns
              |
               --1.15%--qemu_poll_ns

I get after the attached fix:

# Children Self Trace output

# ........  ........  ..............
#
   100.00%   100.00%  (5640789cdf50)
            |          
            |--98.85%--main
            |          main_loop_wait
            |          qemu_poll_ns
            |          
             --1.15%--iothread_run
                       aio_poll
                       qemu_poll_ns
Quoted 1 line

any idea why libdw is doing that? ;-)

Because libdw was reporting separate debuginfo file as the main file:

(lldb) p thread->process->dwfl->modulelist->main.name
(char *) $1 = 0x0000000000b52490 "/home/jkratoch/.debug/.build-id/fc/d0111443729957fa8ef9511ff5dc3e9a5ccf9d/debug" (lldb) p thread->process->dwfl->modulelist->debug.name
(char *) $2 = 0x0000000000000000

The correct way is to report the main ELF file as the main file:

(lldb) p thread->process->dwfl->modulelist->main.name
(char *) $1 = 0x0000000000b52490 "/home/jkratoch/.debug/.build-id/fc/d0111443729957fa8ef9511ff5dc3e9a5ccf9d/elf" (lldb) p thread->process->dwfl->modulelist->debug.name
(char *) $2 = 0x0000000000000000

In some cases it may later fill-in even the ->debug.name with "/debug" filename.

Jan Kratochvil

tools/perf/util/unwind-libdw.c +29 -6
index 7a3dbc259cec..5d66cafccde5 100644
@@ -20,10 +20,28 @@
 
 static char *debuginfo_path;
 
+static int __find_debuginfo(Dwfl_Module *mod __maybe_unused, void **userdata,
+			    const char *modname __maybe_unused, Dwarf_Addr base __maybe_unused,
+			    const char *file_name __maybe_unused,
+			    const char *debuglink_file __maybe_unused,
+			    GElf_Word debuglink_crc __maybe_unused, char **debuginfo_file_name)
+{
+	const struct dso *dso = *userdata;
+	char filename[PATH_MAX];
+
+	assert(dso);
+
+	// We could also use dso->symsrc_filename but it may not be set yet.
+	if (dso__build_id_filename(dso, filename, sizeof(filename), true))
+		*debuginfo_file_name = strdup(filename);
+	return -1;
+}
+
 static const Dwfl_Callbacks offline_callbacks = {
-	.find_debuginfo		= dwfl_standard_find_debuginfo,
+	.find_debuginfo		= __find_debuginfo,
 	.debuginfo_path		= &debuginfo_path,
 	.section_address	= dwfl_offline_section_address,
+	// .find_elf is not set as we use dwfl_report_elf() instead.
 };
 
 static int __report_module(struct addr_location *al, u64 ip,
@@ -46,16 +64,21 @@ static int __report_module(struct addr_location *al, u64 ip,
 	mod = dwfl_addrmodule(ui->dwfl, ip);
 	if (mod) {
 		Dwarf_Addr s;
+		void **userdatap;
 
-		dwfl_module_info(mod, NULL, &s, NULL, NULL, NULL, NULL, NULL);
+		dwfl_module_info(mod, &userdatap, &s, NULL, NULL, NULL, NULL, NULL);
+		*userdatap = dso;
 		if (s != al->map->start - al->map->pgoff)
 			mod = 0;
 	}
 
-	if (!mod)
-		mod = dwfl_report_elf(ui->dwfl, dso->short_name,
-				      (dso->symsrc_filename ? dso->symsrc_filename : dso->long_name), -1, al->map->start - al->map->pgoff,
-				      false);
+	if (!mod) {
+		char filename[PATH_MAX];
+
+		if (dso__build_id_filename(dso, filename, sizeof(filename), false))
+			mod = dwfl_report_elf(ui->dwfl, dso->short_name, filename, -1,
+					      al->map->start - al->map->pgoff, false);
+	}
 
 	return mod && dwfl_addrmodule(ui->dwfl, ip) == mod ? 0 : -1;
 }
#2 lore raw

On Sun, Nov 22, 2020 at 11:27:51PM +0100, Jan Kratochvil wrote:

SNIP

Quoted 14 lines
             --1.15%--iothread_run
                       aio_poll
                       qemu_poll_ns
Quoted 1 line

any idea why libdw is doing that? ;-)

Because libdw was reporting separate debuginfo file as the main file:

(lldb) p thread->process->dwfl->modulelist->main.name
(char *) $1 = 0x0000000000b52490 "/home/jkratoch/.debug/.build-id/fc/d0111443729957fa8ef9511ff5dc3e9a5ccf9d/debug" (lldb) p thread->process->dwfl->modulelist->debug.name
(char *) $2 = 0x0000000000000000

Quoted 8 lines

The correct way is to report the main ELF file as the main file:

(lldb) p thread->process->dwfl->modulelist->main.name
(char *) $1 = 0x0000000000b52490 "/home/jkratoch/.debug/.build-id/fc/d0111443729957fa8ef9511ff5dc3e9a5ccf9d/elf" (lldb) p thread->process->dwfl->modulelist->debug.name
(char *) $2 = 0x0000000000000000

In some cases it may later fill-in even the ->debug.name with "/debug" filename.

hi,
it fixes the issue for me, but it breaks the dwarf test:

72: DWARF unwind : FAILED!
[jolsa@krava perf]$ sudo ./perf test dwarf -v
72: DWARF unwind :
--- start ---
test child forked, pid 2084484
unwind: failed with 'No such file or directory' got wrong number of stack entries 0 != 8 test child finished with -1
---- end ----
DWARF unwind: FAILED!

thanks,
jirka

#3 lore raw
Quoted 12 lines · Jiri Olsa

it fixes the issue for me, but it breaks the dwarf test:

72: DWARF unwind : FAILED!
[jolsa@krava perf]$ sudo ./perf test dwarf -v
72: DWARF unwind :
--- start ---
test child forked, pid 2084484
unwind: failed with 'No such file or directory' got wrong number of stack entries 0 != 8 test child finished with -1
---- end ----
DWARF unwind: FAILED!

The previous patch required ~/.debug/ build-id symlink, fixed.

I hope dso->symsrc_filename is always already set if a separate debug info is present.

Jan

tools/perf/util/unwind-libdw.c +28 -5
index 7a3dbc259cec..400e4832b50e 100644
@@ -20,10 +20,25 @@
 
 static char *debuginfo_path;
 
+static int __find_debuginfo(Dwfl_Module *mod __maybe_unused, void **userdata,
+			    const char *modname __maybe_unused, Dwarf_Addr base __maybe_unused,
+			    const char *file_name __maybe_unused,
+			    const char *debuglink_file __maybe_unused,
+			    GElf_Word debuglink_crc __maybe_unused, char **debuginfo_file_name)
+{
+	const struct dso *dso = *userdata;
+
+	assert(dso);
+	if (dso->symsrc_filename)
+		*debuginfo_file_name = strdup(dso->symsrc_filename);
+	return -1;
+}
+
 static const Dwfl_Callbacks offline_callbacks = {
-	.find_debuginfo		= dwfl_standard_find_debuginfo,
+	.find_debuginfo		= __find_debuginfo,
 	.debuginfo_path		= &debuginfo_path,
 	.section_address	= dwfl_offline_section_address,
+	// .find_elf is not set as we use dwfl_report_elf() instead.
 };
 
 static int __report_module(struct addr_location *al, u64 ip,
@@ -46,16 +61,24 @@ static int __report_module(struct addr_location *al, u64 ip,
 	mod = dwfl_addrmodule(ui->dwfl, ip);
 	if (mod) {
 		Dwarf_Addr s;
+		void **userdatap;
 
-		dwfl_module_info(mod, NULL, &s, NULL, NULL, NULL, NULL, NULL);
+		dwfl_module_info(mod, &userdatap, &s, NULL, NULL, NULL, NULL, NULL);
+		*userdatap = dso;
 		if (s != al->map->start - al->map->pgoff)
 			mod = 0;
 	}
 
 	if (!mod)
-		mod = dwfl_report_elf(ui->dwfl, dso->short_name,
-				      (dso->symsrc_filename ? dso->symsrc_filename : dso->long_name), -1, al->map->start - al->map->pgoff,
-				      false);
+		mod = dwfl_report_elf(ui->dwfl, dso->short_name, dso->long_name, -1,
+				      al->map->start - al->map->pgoff, false);
+	if (!mod) {
+		char filename[PATH_MAX];
+
+		if (dso__build_id_filename(dso, filename, sizeof(filename), false))
+			mod = dwfl_report_elf(ui->dwfl, dso->short_name, filename, -1,
+					      al->map->start - al->map->pgoff, false);
+	}
 
 	return mod && dwfl_addrmodule(ui->dwfl, ip) == mod ? 0 : -1;
 }
#4 lore raw
Quoted 14 lines · Jan Kratochvil
Quoted 12 lines · Jiri Olsa

it fixes the issue for me, but it breaks the dwarf test:

72: DWARF unwind : FAILED!
[jolsa@krava perf]$ sudo ./perf test dwarf -v
72: DWARF unwind :
--- start ---
test child forked, pid 2084484
unwind: failed with 'No such file or directory' got wrong number of stack entries 0 != 8 test child finished with -1
---- end ----
DWARF unwind: FAILED!

Quoted 4 lines

The previous patch required ~/.debug/ build-id symlink, fixed.

I hope dso->symsrc_filename is always already set if a separate debug info is present.

it should be the one with .symtab section

the fix is working for me, would you mind send full patch? or just provide me with changelog and I can do it for you

thanks,
jirka

Quoted 64 lines

Jan

tools/perf/util/unwind-libdw.c +28 -5
index 7a3dbc259cec..400e4832b50e 100644
@@ -20,10 +20,25 @@
 
 static char *debuginfo_path;
 
+static int __find_debuginfo(Dwfl_Module *mod __maybe_unused, void **userdata,
+			    const char *modname __maybe_unused, Dwarf_Addr base __maybe_unused,
+			    const char *file_name __maybe_unused,
+			    const char *debuglink_file __maybe_unused,
+			    GElf_Word debuglink_crc __maybe_unused, char **debuginfo_file_name)
+{
+	const struct dso *dso = *userdata;
+
+	assert(dso);
+	if (dso->symsrc_filename)
+		*debuginfo_file_name = strdup(dso->symsrc_filename);
+	return -1;
+}
+
 static const Dwfl_Callbacks offline_callbacks = {
-	.find_debuginfo		= dwfl_standard_find_debuginfo,
+	.find_debuginfo		= __find_debuginfo,
 	.debuginfo_path		= &debuginfo_path,
 	.section_address	= dwfl_offline_section_address,
+	// .find_elf is not set as we use dwfl_report_elf() instead.
 };
 
 static int __report_module(struct addr_location *al, u64 ip,
@@ -46,16 +61,24 @@ static int __report_module(struct addr_location *al, u64 ip,
 	mod = dwfl_addrmodule(ui->dwfl, ip);
 	if (mod) {
 		Dwarf_Addr s;
+		void **userdatap;
 
-		dwfl_module_info(mod, NULL, &s, NULL, NULL, NULL, NULL, NULL);
+		dwfl_module_info(mod, &userdatap, &s, NULL, NULL, NULL, NULL, NULL);
+		*userdatap = dso;
 		if (s != al->map->start - al->map->pgoff)
 			mod = 0;
 	}
 
 	if (!mod)
-		mod = dwfl_report_elf(ui->dwfl, dso->short_name,
-				      (dso->symsrc_filename ? dso->symsrc_filename : dso->long_name), -1, al->map->start - al->map->pgoff,
-				      false);
+		mod = dwfl_report_elf(ui->dwfl, dso->short_name, dso->long_name, -1,
+				      al->map->start - al->map->pgoff, false);
+	if (!mod) {
+		char filename[PATH_MAX];
+
+		if (dso__build_id_filename(dso, filename, sizeof(filename), false))
+			mod = dwfl_report_elf(ui->dwfl, dso->short_name, filename, -1,
+					      al->map->start - al->map->pgoff, false);
+	}
 
 	return mod && dwfl_addrmodule(ui->dwfl, ip) == mod ? 0 : -1;
 }
#5 lore raw
Quoted 2 lines · Jiri Olsa

the fix is working for me, would you mind send full patch? or just provide me with changelog and I can do it for you

I am not proficient with all the meta-tags there, I make no copyright claims to this patch.

Regards,
Jan

perf libdw: Fix separte debug info files

elfutils needs to be provided main binary and separate debug info file respectively. Providing separate debug info file instead of the main binary is not sufficient.

One needs to try both supplied filename and its possible cache by its build-id depending on the use case.

tools/perf/util/unwind-libdw.c +27 -5
index 7a3dbc259cec..0ada907c60d4 100644
@@ -20,10 +20,24 @@
 
 static char *debuginfo_path;
 
+static int __find_debuginfo(Dwfl_Module *mod __maybe_unused, void **userdata,
+			    const char *modname __maybe_unused, Dwarf_Addr base __maybe_unused,
+			    const char *file_name, const char *debuglink_file __maybe_unused,
+			    GElf_Word debuglink_crc __maybe_unused, char **debuginfo_file_name)
+{
+	const struct dso *dso = *userdata;
+
+	assert(dso);
+	if (dso->symsrc_filename && strcmp (file_name, dso->symsrc_filename))
+		*debuginfo_file_name = strdup(dso->symsrc_filename);
+	return -1;
+}
+
 static const Dwfl_Callbacks offline_callbacks = {
-	.find_debuginfo		= dwfl_standard_find_debuginfo,
+	.find_debuginfo		= __find_debuginfo,
 	.debuginfo_path		= &debuginfo_path,
 	.section_address	= dwfl_offline_section_address,
+	// .find_elf is not set as we use dwfl_report_elf() instead.
 };
 
 static int __report_module(struct addr_location *al, u64 ip,
@@ -46,16 +60,24 @@ static int __report_module(struct addr_location *al, u64 ip,
 	mod = dwfl_addrmodule(ui->dwfl, ip);
 	if (mod) {
 		Dwarf_Addr s;
+		void **userdatap;
 
-		dwfl_module_info(mod, NULL, &s, NULL, NULL, NULL, NULL, NULL);
+		dwfl_module_info(mod, &userdatap, &s, NULL, NULL, NULL, NULL, NULL);
+		*userdatap = dso;
 		if (s != al->map->start - al->map->pgoff)
 			mod = 0;
 	}
 
 	if (!mod)
-		mod = dwfl_report_elf(ui->dwfl, dso->short_name,
-				      (dso->symsrc_filename ? dso->symsrc_filename : dso->long_name), -1, al->map->start - al->map->pgoff,
-				      false);
+		mod = dwfl_report_elf(ui->dwfl, dso->short_name, dso->long_name, -1,
+				      al->map->start - al->map->pgoff, false);
+	if (!mod) {
+		char filename[PATH_MAX];
+
+		if (dso__build_id_filename(dso, filename, sizeof(filename), false))
+			mod = dwfl_report_elf(ui->dwfl, dso->short_name, filename, -1,
+					      al->map->start - al->map->pgoff, false);
+	}
 
 	return mod && dwfl_addrmodule(ui->dwfl, ip) == mod ? 0 : -1;
 }
#6 lore raw
Quoted 11 lines · Jan Kratochvil
Quoted 2 lines · Jiri Olsa

the fix is working for me, would you mind send full patch? or just provide me with changelog and I can do it for you

I am not proficient with all the meta-tags there, I make no copyright claims to this patch.

Regards,
Jan

Quoted 8 lines

perf libdw: Fix separte debug info files

elfutils needs to be provided main binary and separate debug info file respectively. Providing separate debug info file instead of the main binary is not sufficient.

One needs to try both supplied filename and its possible cache by its build-id depending on the use case.

it's great, I'll just add your's SOB if that's ok:

thanks,
jirka

Quoted 60 lines
tools/perf/util/unwind-libdw.c +27 -5
index 7a3dbc259cec..0ada907c60d4 100644
@@ -20,10 +20,24 @@
 
 static char *debuginfo_path;
 
+static int __find_debuginfo(Dwfl_Module *mod __maybe_unused, void **userdata,
+			    const char *modname __maybe_unused, Dwarf_Addr base __maybe_unused,
+			    const char *file_name, const char *debuglink_file __maybe_unused,
+			    GElf_Word debuglink_crc __maybe_unused, char **debuginfo_file_name)
+{
+	const struct dso *dso = *userdata;
+
+	assert(dso);
+	if (dso->symsrc_filename && strcmp (file_name, dso->symsrc_filename))
+		*debuginfo_file_name = strdup(dso->symsrc_filename);
+	return -1;
+}
+
 static const Dwfl_Callbacks offline_callbacks = {
-	.find_debuginfo		= dwfl_standard_find_debuginfo,
+	.find_debuginfo		= __find_debuginfo,
 	.debuginfo_path		= &debuginfo_path,
 	.section_address	= dwfl_offline_section_address,
+	// .find_elf is not set as we use dwfl_report_elf() instead.
 };
 
 static int __report_module(struct addr_location *al, u64 ip,
@@ -46,16 +60,24 @@ static int __report_module(struct addr_location *al, u64 ip,
 	mod = dwfl_addrmodule(ui->dwfl, ip);
 	if (mod) {
 		Dwarf_Addr s;
+		void **userdatap;
 
-		dwfl_module_info(mod, NULL, &s, NULL, NULL, NULL, NULL, NULL);
+		dwfl_module_info(mod, &userdatap, &s, NULL, NULL, NULL, NULL, NULL);
+		*userdatap = dso;
 		if (s != al->map->start - al->map->pgoff)
 			mod = 0;
 	}
 
 	if (!mod)
-		mod = dwfl_report_elf(ui->dwfl, dso->short_name,
-				      (dso->symsrc_filename ? dso->symsrc_filename : dso->long_name), -1, al->map->start - al->map->pgoff,
-				      false);
+		mod = dwfl_report_elf(ui->dwfl, dso->short_name, dso->long_name, -1,
+				      al->map->start - al->map->pgoff, false);
+	if (!mod) {
+		char filename[PATH_MAX];
+
+		if (dso__build_id_filename(dso, filename, sizeof(filename), false))
+			mod = dwfl_report_elf(ui->dwfl, dso->short_name, filename, -1,
+					      al->map->start - al->map->pgoff, false);
+	}
 
 	return mod && dwfl_addrmodule(ui->dwfl, ip) == mod ? 0 : -1;
 }