perf unwind-libdw: Fix unwinding of multi-threaded processes
PATCH 1/1 perf unwind-libdw: Fix unwinding of multi-threaded processes
The libdw callback API has two levels: dwfl_getthread_frames() first finds the requested thread using the next_thread() or get_thread() callback and then walks its stack.
Since perf only has a snapshot of the stack of a single thread, it provides a stubbed-out API that always returns the pid the Dwfl was attached with (i.e. whatever was passed to dwfl_attach_state()), rather than the actual sampled thread's TID.
Commit 6b2658b3f36a ("perf unwind-libdw: Don't discard loaded ELF/DWARF after every unwind") changed libdw unwinding from recreating the Dwfl object for each sample to caching it in struct maps, which is shared by every thread in the process. It left next_thread() unchanged.
Since the pid passed to dwfl_attach_state() is only set at creation, only the thread of the first sample is ever found. As a result, dwfl_getthread_frames() fails with ESRCH when asked to unwind a sample from another thread.
Make next_thread() return the current sample's TID, provide get_thread() so libdw can find it directly, and pass the process PID expected by dwfl_attach_state(). This allows libdw to unwind samples from every thread in a multi-threaded process.
---
A standalone reproducer is available at:
https://github.com/kryggird/perf-bug-repro/tree/019c8ae97f9e4370ef2265d7badc3269e2a5044f
Run it with the perf binary being tested:
./reproduce.sh /path/to/perf
Results:
perf 7.1.3-201.fc44.x86_64:
1517 of 1522 samples have no userspace callchain
patched perf 7.2.rc4.g1590cf032971:
15 of 1449 samples have no userspace callchain
tools/perf/util/unwind-libdw.c | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-)
tools/perf/util/unwind-libdw.c +20 -3
@@ -173,14 +173,30 @@ static int entry(u64 ip, struct unwind_info *ui)return 0 ; } -static pid_t next_thread (Dwfl *dwfl,void *arg,void **thread_argp) +static pid_t next_thread (Dwfl *dwfl__maybe_unused ,void *arg,void **thread_argp) { +struct dwfl_ui_thread_info *dwfl_ui_ti = arg; +/* We want only single thread to be processed. */ if (*thread_argp !=NULL )return 0 ; +assert (dwfl_ui_ti->ui !=NULL ); *thread_argp = arg; -return dwfl_pid (dwfl); +return thread__tid (dwfl_ui_ti->ui->thread); +} + +static bool get_thread (Dwfl *dwfl__maybe_unused ,pid_t tid,void *arg, +void **thread_argp) +{ +struct dwfl_ui_thread_info *dwfl_ui_ti = arg; + +assert (dwfl_ui_ti->ui !=NULL ); +if (tid !=thread__tid (dwfl_ui_ti->ui->thread)) +return false; + + *thread_argp = arg; +return true; } static int access_dso_mem(struct unwind_info *ui, Dwarf_Addr addr,
@@ -306,6 +322,7 @@ static bool libdw_set_initial_registers(Dwfl_Thread *thread, void *arg)static const Dwfl_Thread_Callbacks callbacks = { .next_thread = next_thread, + .get_thread = get_thread, .memory_read = memory_read, .set_initial_registers = libdw_set_initial_registers, };
@@ -400,7 +417,7 @@ int libdw__get_entries(unwind_entry_cb_t cb, void *arg,if (err) goto out; -dwfl_attach_state (dwfl,/*elf=*/ NULL ,thread__tid (thread), &callbacks, +dwfl_attach_state (dwfl,/*elf=*/ NULL ,thread__pid (thread), &callbacks,/* Dwfl thread function argument*/ dwfl_ui_ti);// Ignore thread already attached error.
base-commit: b95f03f04d475aa6719d15a636ddf32222d55657
Signature · 1 line
2.55.0
[email protected]
AI reviewThank you for your contribution! Sashiko AI review found 3 potential issue(s) to consider:
Thank you for your contribution! Sashiko AI review found 3 potential issue(s) to consider:
New issues:
- [Low] Missing explicit POSIX standard header inclusion for <assert.h>, breaking compatibility with musl libc.
Pre-existing issues:
- [High] Missing NULL check on zalloc allocation for dwfl_ui_ti leads to a NULL pointer dereference, and unchecked dwfl_begin() failure leads to a memory leak.
- [High] The local pointer ui is declared as a static variable but is dynamically allocated and freed within the function, creating a severe data race. --
--- Patch [1]: [PATCH] perf unwind-libdw: Fix unwinding of multi-threaded processes --- commit cd2051f3de0d807edd3e4fcc7ed115a6364be27d
Author: Alessio Podda <[email protected]>
perf unwind-libdw: Fix unwinding of multi-threaded processes
This commit fixes unwinding of multi-threaded processes in perf. It updates next_thread() to return the current sample's TID and provides get_thread() so libdw can find the thread directly.
Quoted 8 lines
tools/perf/util/unwind-libdw.c +0
-0@@ -173,14 +173,30 @@ static int entry(u64 ip, struct unwind_info *ui)return 0 ; }
tools/perf/util/unwind-libdw.c static int entry(u64 ip, struct unwind_info *ui) · line 184
Quoted 10 lines
tools/perf/util/unwind-libdw.c +4
-1-static pid_t next_thread (Dwfl *dwfl,void *arg,void **thread_argp) +static pid_t next_thread (Dwfl *dwfl__maybe_unused ,void *arg,void **thread_argp) { +struct dwfl_ui_thread_info *dwfl_ui_ti = arg; +/* We want only single thread to be processed. */ if (*thread_argp !=NULL )return 0 ; +assert (dwfl_ui_ti->ui !=NULL );
[Severity: Low]
Does this file need an explicit include for <assert.h>? The perf subsystem guidelines require direct POSIX header inclusions for compatibility with musl libc, and it appears the assert macro is being introduced here without it.
Quoted 4 lines
tools/perf/util/unwind-libdw.c +2
-1+ *thread_argp = arg; -return dwfl_pid (dwfl); +return thread__tid (dwfl_ui_ti->ui->thread); }
[ ... ]
tools/perf/util/unwind-libdw.c int libdw__get_entries(unwind_entry_cb_t cb, void *arg, · line 422
Quoted 8 lines
tools/perf/util/unwind-libdw.c +1
-1@@ -400,7 +417,7 @@ int libdw__get_entries(unwind_entry_cb_t cb, void *arg,if (err) goto out; -dwfl_attach_state (dwfl,/*elf=*/ NULL ,thread__tid (thread), &callbacks, +dwfl_attach_state (dwfl,/*elf=*/ NULL ,thread__pid (thread), &callbacks,/* Dwfl thread function argument*/ dwfl_ui_ti);// Ignore thread already attached error.
[Severity: High]
This is a pre-existing issue, but can the allocation for dwfl_ui_ti fail and lead to a NULL pointer dereference?
Looking at earlier code in libdw__get_entries():
dwfl_ui_ti = zalloc(sizeof(*dwfl_ui_ti)); dwfl = dwfl_begin(&offline_callbacks); if (!dwfl) goto out; dwfl_ui_ti->dwfl = dwfl;
If zalloc fails, it seems we would dereference the NULL dwfl_ui_ti right away. Also, if dwfl_begin fails, we jump to out:
out:
...
dwfl_ui_ti->ui = NULL;
which would also dereference NULL, or leak the allocated dwfl_ui_ti if zalloc had succeeded. Does this error path need a NULL check and proper cleanup?
[Severity: High]
This is a pre-existing issue, but does the static declaration of the ui pointer cause data races?
In libdw__get_entries(), ui is declared as static but dynamically allocated:
static struct unwind_info *ui; ... ui = zalloc(sizeof(*ui) + sizeof(ui->entries[0]) * max_stack);
And then freed at the end of the same function:
free(ui);
If multiple threads process events concurrently, could this static pointer be overwritten, leading to memory leaks, use-after-free, or double-free issues?
Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=1
Ian Rogers
reviewerQuoted 20 lines · Alessio Podda
The libdw callback API has two levels:
dwfl_getthread_frames()first finds the requested thread using thenext_thread()orget_thread()callback and then walks its stack.Since perf only has a snapshot of the stack of a single thread, it provides a stubbed-out API that always returns the pid the Dwfl was attached with (i.e. whatever was passed to
dwfl_attach_state()), rather than the actual sampled thread's TID.Commit
6b2658b3f36a("perf unwind-libdw: Don't discard loaded ELF/DWARF after every unwind") changed libdw unwinding from recreating the Dwfl object for each sample to caching it in struct maps, which is shared by every thread in the process. It leftnext_thread()unchanged.Since the pid passed to
dwfl_attach_state()is only set at creation, only the thread of the first sample is ever found. As a result,dwfl_getthread_frames()fails with ESRCH when asked to unwind a sample from another thread.
Quoted 8 lines
Make
next_thread()return the current sample's TID, provideget_thread()so libdw can find it directly, and pass the process PID expected bydwfl_attach_state(). This allows libdw to unwind samples from every thread in a multi-threaded process.Fixes 6b2658b3f36a ("perf unwind-libdw: Don't discard loaded ELF/DWARF after every unwind")Assisted-by Codex:gpt-5.6-solSigned-off-by Alessio Podda <[email protected]>
This is great, thank you! Did you have a test that would reproduce this problem? Could we add it to tools/perf/tests ?
Thanks,
Ian
Quoted 79 lines
---
A standalone reproducer is available at:
https://github.com/kryggird/perf-bug-repro/tree/019c8ae97f9e4370ef2265d7badc3269e2a5044f
Run it with the perf binary being tested:
./reproduce.sh /path/to/perf
Results:
perf 7.1.3-201.fc44.x86_64:
1517 of 1522 samples have no userspace callchainpatched perf 7.2.rc4.g1590cf032971:
15 of 1449 samples have no userspace callchaintools/perf/util/unwind-libdw.c | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-)tools/perf/util/unwind-libdw.c +20
-3@@ -173,14 +173,30 @@ static int entry(u64 ip, struct unwind_info *ui)return 0 ; } -static pid_t next_thread (Dwfl *dwfl,void *arg,void **thread_argp) +static pid_t next_thread (Dwfl *dwfl__maybe_unused ,void *arg,void **thread_argp) { +struct dwfl_ui_thread_info *dwfl_ui_ti = arg; +/* We want only single thread to be processed. */ if (*thread_argp !=NULL )return 0 ; +assert (dwfl_ui_ti->ui !=NULL ); *thread_argp = arg; -return dwfl_pid (dwfl); +return thread__tid (dwfl_ui_ti->ui->thread); +} + +static bool get_thread (Dwfl *dwfl__maybe_unused ,pid_t tid,void *arg, +void **thread_argp) +{ +struct dwfl_ui_thread_info *dwfl_ui_ti = arg; + +assert (dwfl_ui_ti->ui !=NULL ); +if (tid !=thread__tid (dwfl_ui_ti->ui->thread)) +return false; + + *thread_argp = arg; +return true; } static int access_dso_mem(struct unwind_info *ui, Dwarf_Addr addr,@@ -306,6 +322,7 @@ static bool libdw_set_initial_registers(Dwfl_Thread *thread, void *arg)static const Dwfl_Thread_Callbacks callbacks = { .next_thread = next_thread, + .get_thread = get_thread, .memory_read = memory_read, .set_initial_registers = libdw_set_initial_registers, };@@ -400,7 +417,7 @@ int libdw__get_entries(unwind_entry_cb_t cb, void *arg,if (err) goto out; -dwfl_attach_state (dwfl,/*elf=*/ NULL ,thread__tid (thread), &callbacks, +dwfl_attach_state (dwfl,/*elf=*/ NULL ,thread__pid (thread), &callbacks,/* Dwfl thread function argument*/ dwfl_ui_ti);// Ignore thread already attached error. base-commit:
b95f03f04d475aa6719d15a636ddf32222d55657--
2.55.0
PATCH v2 1/1 perf unwind-libdw: Fix unwinding of multi-threaded processes
The libdw callback API has two levels: dwfl_getthread_frames() first finds the requested thread using the next_thread() or get_thread() callback and then walks its stack.
Since perf only has a snapshot of the stack of a single thread, it provides a stubbed-out API that always returns the pid the Dwfl was attached with (i.e. whatever was passed to dwfl_attach_state()), rather than the actual sampled thread's TID.
Commit 6b2658b3f36a ("perf unwind-libdw: Don't discard loaded ELF/DWARF after every unwind") changed libdw unwinding from recreating the Dwfl object for each sample to caching it in struct maps, which is shared by every thread in the process. It left next_thread() unchanged.
Since the pid passed to dwfl_attach_state() is only set at creation, only the thread of the first sample is ever found. As a result, dwfl_getthread_frames() fails with ESRCH when asked to unwind a sample from another thread.
Make next_thread() return the current sample's TID, provide get_thread() so libdw can find it directly, and pass the process PID expected by dwfl_attach_state(). This allows libdw to unwind samples from every thread in a multi-threaded process.
Add a shell regression test that records a four-thread workload and verifies that libdw recovers the worker callchain for every worker TID.
---
Changes in v2:
- Add a shell regression test for multi-threaded libdw unwinding.
- Include <assert.h> directly instead of relying on a transitive include.
.../shell/test_dwarf_unwind_multithreaded.sh | 65 +++++++++++++++++++ tools/perf/util/unwind-libdw.c | 24 ++++++- 2 files changed, 86 insertions(+), 3 deletions(-) create mode 100755 tools/perf/tests/shell/test_dwarf_unwind_multithreaded.sh
tools/perf/tests/shell/test_dwarf_unwind_multithreaded.sh new file +65 -0
@@ -0,0 +1,65 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# Test libdw unwinding of multi-threaded processes (exclusive) + +set -e + +if ! perf check feature -q libdw-dwarf-unwind; then + echo "Skip: libdw DWARF unwinding is not available" + exit 2 +fi + +tmpdir=$(mktemp -d /tmp/perf-test-dwarf-unwind-multithreaded.XXXXXX) +perf_data="$tmpdir/perf.data" +perf_script="$tmpdir/perf-script.txt" +nr_threads=4 +nr_worker_threads=$((nr_threads - 1)) + +cleanup() +{ + trap - EXIT TERM INT + rm -rf "$tmpdir" +} + +trap cleanup EXIT TERM INT + +if ! perf record -q -e task-clock:u -F 99 --call-graph dwarf,8192 \ + -o "$perf_data" -- perf test -w thloop 2 "$nr_threads" +then + echo "Skip: failed to record task-clock:u" + exit 2 +fi + +if ! perf script --unwind-style=libdw \ + -F comm,pid,tid,event,ip,sym -i "$perf_data" > "$perf_script" +then + echo "Error: failed to process the recording with libdw" >&2 + exit 1 +fi + +nr_unwound_threads=$( + awk ' + BEGIN { RS = "" } + + # thfunc is the worker-only caller of test_loop. Finding it proves + # that libdw unwound beyond the sampled leaf for this worker TID. + /thfunc/ { + split($2, id, "/") + seen[id[2]] = 1 + } + + END { + for (tid in seen) + nr_tids++ + print nr_tids + 0 + } + ' "$perf_script" +) + +if [ "$nr_unwound_threads" -ne "$nr_worker_threads" ]; then + echo "Error: expected callchains for $nr_worker_threads worker TIDs," \ + "found $nr_unwound_threads" >&2 + exit 1 +fi + +exit 0
tools/perf/util/unwind-libdw.c +21 -3
@@ -1,4 +1,5 @@// SPDX-License-Identifier: GPL-2.0 +#include <assert.h> #include <linux/compiler.h> #include <elfutils/libdw.h> #include <elfutils/libdwfl.h>
@@ -173,14 +174,30 @@ static int entry(u64 ip, struct unwind_info *ui)return 0 ; } -static pid_t next_thread (Dwfl *dwfl,void *arg,void **thread_argp) +static pid_t next_thread (Dwfl *dwfl__maybe_unused ,void *arg,void **thread_argp) { +struct dwfl_ui_thread_info *dwfl_ui_ti = arg; +/* We want only single thread to be processed. */ if (*thread_argp !=NULL )return 0 ; +assert (dwfl_ui_ti->ui !=NULL ); *thread_argp = arg; -return dwfl_pid (dwfl); +return thread__tid (dwfl_ui_ti->ui->thread); +} + +static bool get_thread (Dwfl *dwfl__maybe_unused ,pid_t tid,void *arg, +void **thread_argp) +{ +struct dwfl_ui_thread_info *dwfl_ui_ti = arg; + +assert (dwfl_ui_ti->ui !=NULL ); +if (tid !=thread__tid (dwfl_ui_ti->ui->thread)) +return false; + + *thread_argp = arg; +return true; } static int access_dso_mem(struct unwind_info *ui, Dwarf_Addr addr,
@@ -306,6 +323,7 @@ static bool libdw_set_initial_registers(Dwfl_Thread *thread, void *arg)static const Dwfl_Thread_Callbacks callbacks = { .next_thread = next_thread, + .get_thread = get_thread, .memory_read = memory_read, .set_initial_registers = libdw_set_initial_registers, };
@@ -400,7 +418,7 @@ int libdw__get_entries(unwind_entry_cb_t cb, void *arg,if (err) goto out; -dwfl_attach_state (dwfl,/*elf=*/ NULL ,thread__tid (thread), &callbacks, +dwfl_attach_state (dwfl,/*elf=*/ NULL ,thread__pid (thread), &callbacks,/* Dwfl thread function argument*/ dwfl_ui_ti);// Ignore thread already attached error.
base-commit: b95f03f04d475aa6719d15a636ddf32222d55657
Signature · 1 line
2.55.0
Namhyung Kim
maintainerQuoted 27 lines · Alessio Podda
The libdw callback API has two levels:
dwfl_getthread_frames()first finds the requested thread using thenext_thread()orget_thread()callback and then walks its stack.Since perf only has a snapshot of the stack of a single thread, it provides a stubbed-out API that always returns the pid the Dwfl was attached with (i.e. whatever was passed to
dwfl_attach_state()), rather than the actual sampled thread's TID.Commit
6b2658b3f36a("perf unwind-libdw: Don't discard loaded ELF/DWARF after every unwind") changed libdw unwinding from recreating the Dwfl object for each sample to caching it in struct maps, which is shared by every thread in the process. It leftnext_thread()unchanged.Since the pid passed to
dwfl_attach_state()is only set at creation, only the thread of the first sample is ever found. As a result,dwfl_getthread_frames()fails with ESRCH when asked to unwind a sample from another thread.Make
next_thread()return the current sample's TID, provideget_thread()so libdw can find it directly, and pass the process PID expected bydwfl_attach_state(). This allows libdw to unwind samples from every thread in a multi-threaded process.Add a shell regression test that records a four-thread workload and verifies that libdw recovers the worker callchain for every worker TID.
Quoted 7 lines
Fixes 6b2658b3f36a ("perf unwind-libdw: Don't discard loaded ELF/DWARF after every unwind")Assisted-by Codex:gpt-5.6-solSigned-off-by Alessio Podda <[email protected]>---
Changes in v2:
- Add a shell regression test for multi-threaded libdw unwinding.
- Include <assert.h> directly instead of relying on a transitive include.
Ian, are you ok with this now?
Thanks,
Namhyung
Quoted 141 lines
.../shell/test_dwarf_unwind_multithreaded.sh | 65 +++++++++++++++++++ tools/perf/util/unwind-libdw.c | 24 ++++++- 2 files changed, 86 insertions(+), 3 deletions(-) create mode 100755 tools/perf/tests/shell/test_dwarf_unwind_multithreaded.shtools/perf/tests/shell/test_dwarf_unwind_multithreaded.sh new file +65
-0@@ -0,0 +1,65 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# Test libdw unwinding of multi-threaded processes (exclusive) + +set -e + +if ! perf check feature -q libdw-dwarf-unwind; then + echo "Skip: libdw DWARF unwinding is not available" + exit 2 +fi + +tmpdir=$(mktemp -d /tmp/perf-test-dwarf-unwind-multithreaded.XXXXXX) +perf_data="$tmpdir/perf.data" +perf_script="$tmpdir/perf-script.txt" +nr_threads=4 +nr_worker_threads=$((nr_threads - 1)) + +cleanup() +{ + trap - EXIT TERM INT + rm -rf "$tmpdir" +} + +trap cleanup EXIT TERM INT + +if ! perf record -q -e task-clock:u -F 99 --call-graph dwarf,8192 \ + -o "$perf_data" -- perf test -w thloop 2 "$nr_threads" +then + echo "Skip: failed to record task-clock:u" + exit 2 +fi + +if ! perf script --unwind-style=libdw \ + -F comm,pid,tid,event,ip,sym -i "$perf_data" > "$perf_script" +then + echo "Error: failed to process the recording with libdw" >&2 + exit 1 +fi + +nr_unwound_threads=$( + awk ' + BEGIN { RS = "" } + + # thfunc is the worker-only caller of test_loop. Finding it proves + # that libdw unwound beyond the sampled leaf for this worker TID. + /thfunc/ { + split($2, id, "/") + seen[id[2]] = 1 + } + + END { + for (tid in seen) + nr_tids++ + print nr_tids + 0 + } + ' "$perf_script" +) + +if [ "$nr_unwound_threads" -ne "$nr_worker_threads" ]; then + echo "Error: expected callchains for $nr_worker_threads worker TIDs," \ + "found $nr_unwound_threads" >&2 + exit 1 +fi + +exit 0tools/perf/util/unwind-libdw.c +21
-3@@ -1,4 +1,5 @@// SPDX-License-Identifier: GPL-2.0 +#include <assert.h> #include <linux/compiler.h> #include <elfutils/libdw.h> #include <elfutils/libdwfl.h> @@ -173,14 +174,30 @@ static int entry(u64 ip, struct unwind_info *ui)return 0 ; } -static pid_t next_thread (Dwfl *dwfl,void *arg,void **thread_argp) +static pid_t next_thread (Dwfl *dwfl__maybe_unused ,void *arg,void **thread_argp) { +struct dwfl_ui_thread_info *dwfl_ui_ti = arg; +/* We want only single thread to be processed. */ if (*thread_argp !=NULL )return 0 ; +assert (dwfl_ui_ti->ui !=NULL ); *thread_argp = arg; -return dwfl_pid (dwfl); +return thread__tid (dwfl_ui_ti->ui->thread); +} + +static bool get_thread (Dwfl *dwfl__maybe_unused ,pid_t tid,void *arg, +void **thread_argp) +{ +struct dwfl_ui_thread_info *dwfl_ui_ti = arg; + +assert (dwfl_ui_ti->ui !=NULL ); +if (tid !=thread__tid (dwfl_ui_ti->ui->thread)) +return false; + + *thread_argp = arg; +return true; } static int access_dso_mem(struct unwind_info *ui, Dwarf_Addr addr,@@ -306,6 +323,7 @@ static bool libdw_set_initial_registers(Dwfl_Thread *thread, void *arg)static const Dwfl_Thread_Callbacks callbacks = { .next_thread = next_thread, + .get_thread = get_thread, .memory_read = memory_read, .set_initial_registers = libdw_set_initial_registers, };@@ -400,7 +418,7 @@ int libdw__get_entries(unwind_entry_cb_t cb, void *arg,if (err) goto out; -dwfl_attach_state (dwfl,/*elf=*/ NULL ,thread__tid (thread), &callbacks, +dwfl_attach_state (dwfl,/*elf=*/ NULL ,thread__pid (thread), &callbacks,/* Dwfl thread function argument*/ dwfl_ui_ti);// Ignore thread already attached error. base-commit:
b95f03f04d475aa6719d15a636ddf32222d55657Signature · 1 line
2.55.0
Quoted 10 lines · Alessio Podda
The libdw callback API has two levels:
dwfl_getthread_frames()first finds the requested thread using thenext_thread()orget_thread()callback and then walks its stack.Since perf only has a snapshot of the stack of a single thread, it provides a stubbed-out API that always returns the pid the Dwfl was attached with (i.e. whatever was passed to
dwfl_attach_state()), rather than the actual sampled thread's TID.[...]
Applied to perf-tools-next, thanks!
Best regards,
Namhyung