perf machine: Factor creating a "live" machine out of dwarf-unwind
Ian Rogers
reviewerPATCH v2 1/2 perf machine: Factor creating a "live" machine out of dwarf-unwind
Factor out for use in places other than the dwarf unwinding tests for libunwind.
--- tools/perf/tests/dwarf-unwind.c | 35 ++++------------------ tools/perf/util/machine.c | 53 +++++++++++++++++++++++++++------ tools/perf/util/machine.h | 1 + 3 files changed, 51 insertions(+), 38 deletions(-)
tools/perf/tests/dwarf-unwind.c +6 -29
@@ -15,7 +15,6 @@#include "symbol.h" #include "thread.h" #include "callchain.h" -#include "util/synthetic-events.h" /* For bsearch. We try to unwind functions in shared object. */ #include <stdlib.h>
@@ -37,24 +36,6 @@#define NO_TAIL_CALL_BARRIER __asm__ __volatile__("" : : : "memory");#endif -static int mmap_handler (const struct perf_tool *tool__maybe_unused , -union perf_event *event, -struct perf_sample *sample, -struct machine *machine) -{ -return machine__process_mmap2_event (machine, event, sample); -} - -static int init_live_machine (struct machine *machine) -{ -union perf_event event; -pid_t pid =getpid (); - -memset (&event,0 ,sizeof (event)); -return perf_event__synthesize_mmap_events (NULL , &event, pid, pid, - mmap_handler, machine, true); -} -/* * We need to keep these functions global, despite the * fact that they are used only locally in this object,
@@ -202,8 +183,12 @@ noinline int test__dwarf_unwind(struct test_suite *test __maybe_unused,struct machine *machine;struct thread *thread;int err =-1 ; +pid_t pid =getpid (); - machine =machine__new_host (); + callchain_param.record_mode =CALLCHAIN_DWARF ; + dwarf_callchain_users = true; + + machine =machine__new_live (/*kernel_maps=*/ true, pid);if (!machine) {pr_err ("Could not get machine\n" );return -1 ;
@@ -214,18 +199,10 @@ noinline int test__dwarf_unwind(struct test_suite *test __maybe_unused,return -1 ; } - callchain_param.record_mode =CALLCHAIN_DWARF ; - dwarf_callchain_users = true; - -if (init_live_machine (machine)) { -pr_err ("Could not init machine\n" ); - goto out; - } -if (verbose >1 )machine__fprintf (machine, stderr); - thread =machine__find_thread (machine,getpid (),getpid ()); + thread =machine__find_thread (machine, pid, pid);if (!thread) {pr_err ("Could not get thread\n" ); goto out;
tools/perf/util/machine.c +44 -9
73 lines not on this page · raw message
tools/perf/util/machine.h +1 -0
@@ -171,6 +171,7 @@ void machines__set_comm_exec(struct machines *machines, bool comm_exec);struct machine *machine__new_host (void );struct machine *machine__new_kallsyms (void ); +struct machine *machine__new_live (bool kernel_maps,pid_t pid);int machine__init (struct machine *machine,const char *root_dir,pid_t pid);void machine__exit (struct machine *machine);void machine__delete_threads (struct machine *machine);
Signature · 1 line
2.49.0.1204.g71687c7c1d-goog
Ian Rogers
reviewerPATCH v2 2/2 perf debug: Add function symbols to dump_stack
Symbolize stack traces by creating a live machine. Add this functionality to dump_stack and switch dump_stack users to use it. Switch TUI to use it. Add stack traces to the child test function which can be useful to diagnose blocked code.
Example output:
```
8: PERF_RECORD_* events & perf_sample fields : Running (1 active)
^C
Signal (2) while running tests.
Terminating tests with the same signal
Internal test harness failure. Completing any started tests:
: 8: PERF_RECORD_* events & perf_sample fields:
____ unexpected signal (2) ____
#0 0x5590fb6209b6 in child_test_sig_handler builtin-test.c:243
#1 0x7f4a91e49e20 in __restore_rt libc_sigaction.c:0
#2 0x7f4a91ee4f33 in clock_nanosleep@GLIBC_2.2.5 clock_nanosleep.c:71
#3 0x7f4a91ef0333 in __nanosleep nanosleep.c:26
#4 0x7f4a91f01f68 in __sleep sleep.c:55
#5 0x5590fb638c63 in test__PERF_RECORD perf-record.c:295
#6 0x5590fb620b43 in run_test_child builtin-test.c:269
#7 0x5590fb5b83ab in start_command run-command.c:127
#8 0x5590fb621572 in start_test builtin-test.c:467
#9 0x5590fb621a47 in __cmd_test builtin-test.c:573
#10 0x5590fb6225ea in cmd_test builtin-test.c:775
#11 0x5590fb5a9099 in run_builtin perf.c:351
#12 0x5590fb5a9340 in handle_internal_command perf.c:404
#13 0x5590fb5a9499 in run_argv perf.c:451
#14 0x5590fb5a97e2 in main perf.c:558
#15 0x7f4a91e33d68 in __libc_start_call_main libc_start_call_main.h:74
#16 0x7f4a91e33e25 in __libc_start_main@@GLIBC_2.34 libc-start.c:128
#17 0x5590fb4fd6d1 in _start perf[436d1]
```---
v2: Fix NO_BACKTRACE=1 build (Arnaldo)
--- tools/perf/tests/builtin-test.c | 15 +++++++- tools/perf/ui/tui/setup.c | 2 +- tools/perf/util/debug.c | 68 +++++++++++++++++++++++++++------ tools/perf/util/debug.h | 1 + 4 files changed, 73 insertions(+), 13 deletions(-)
tools/perf/tests/builtin-test.c +14 -1
@@ -6,6 +6,9 @@*/ #include <fcntl.h> #include <errno.h> +#ifdef HAVE_BACKTRACE_SUPPORT +#include <execinfo.h> +#endif #include <poll.h> #include <unistd.h> #include <setjmp.h>
@@ -231,6 +234,16 @@ static jmp_buf run_test_jmp_buf;static void child_test_sig_handler (int sig) { +#ifdef HAVE_BACKTRACE_SUPPORT +void *stackdump[32 ]; +size_t stackdump_size; +#endif + +fprintf (stderr,"\n---- unexpected signal (%d) ----\n" , sig); +#ifdef HAVE_BACKTRACE_SUPPORT + stackdump_size =backtrace (stackdump,ARRAY_SIZE (stackdump)); +__dump_stack (stderr, stackdump, stackdump_size); +#endif siglongjmp (run_test_jmp_buf, sig); }
@@ -244,7 +257,7 @@ static int run_test_child(struct child_process *process) err =sigsetjmp (run_test_jmp_buf,1 );if (err) { -fprintf (stderr,"\n---- unexpected signal (%d) ----\n" , err); +/* Received signal. */ err = err >0 ? -err :-1 ; goto err_out; }
tools/perf/ui/tui/setup.c +1 -1
@@ -108,7 +108,7 @@ static void ui__signal_backtrace(int sig)printf ("-------- backtrace --------\n" ); size =backtrace (stackdump,ARRAY_SIZE (stackdump)); -backtrace_symbols_fd (stackdump, size,STDOUT_FILENO ); +__dump_stack (stdout, stackdump, size);exit (0 ); }
tools/perf/util/debug.c +57 -11
89 lines not on this page · raw message
tools/perf/util/debug.h +1 -0
@@ -85,6 +85,7 @@ void debug_set_display_time(bool set);void perf_debug_setup (void );int perf_quiet_option (void ); +void __dump_stack (FILE *file,void **stackdump,size_t stackdump_size);void dump_stack (void );void sighandler_dump_stack (int sig);
Signature · 1 line
2.49.0.1204.g71687c7c1d-goog
Arnaldo Carvalho de Melo
maintainerRe: [PATCH v2 2/2] perf debug: Add function symbols to dump_stack
Quoted 5 lines · Ian Rogers
Symbolize stack traces by creating a live machine. Add this functionality to
dump_stackand switchdump_stackusers to use it. Switch TUI to use it. Add stack traces to the child test function which can be useful to diagnose blocked code.
Quoted 8 lines
Example output:
```
8: PERF_RECORD_* events &perf_samplefields : Running (1 active)
^C
Signal (2) while running tests.
Terminating tests with the same signal
Internal test harness failure. Completing any started tests:
: 8: PERF_RECORD_* events &perf_samplefields:
So you are testing it with:
root@number:~# perf test PERF_RECORD 8: PERF_RECORD_* events & perf_sample fields : Running (1 active) ^C Signal (2) while running tests. Terminating tests with the same signal Internal test harness failure. Completing any started tests: 8: PERF_RECORD_* events & perf_sample fields : Skip (permissions) root@number:~#
?
I built it with DEBUG=1 and without, and with your patch, following your example output, I'm not being able to reproduce.
Tried it as well with:
⬢ [acme@toolbx perf-tools-next]$ cat segv.patch
tools/perf/util/symbol.c +2 -0
@@ -402,6 +402,8 @@ static struct symbol *symbols__find(struct rb_root_cached *symbols, u64 ip) {struct rb_node *n; + *(int *)NULL =0 ; +if (symbols ==NULL )return NULL ;
⬢ [acme@toolbx perf-tools-next]$ patch -p1 < segv.patch
patching file tools/perf/util/symbol.c
⬢ [acme@toolbx perf-tools-next]$ m
rm: cannot remove '/home/acme/libexec/perf-core/scripts/python/Perf-Trace-Util/lib/Perf/Trace/__pycache__/Core.cpython-313.pyc': Permission denied
make: Entering directory '/home/acme/git/perf-tools-next/tools/perf'
BUILD: Doing 'make -j32' parallel build
Warning: Kernel ABI header differences:
diff -u tools/arch/arm64/include/asm/cputype.h arch/arm64/include/asm/cputype.h
Auto-detecting system features:
... libdw: [ on ] ... glibc: [ on ] ... libelf: [ on ] ... libnuma: [ on ] ... numa_num_possible_cpus: [ on ] ... libperl: [ on ] ... libpython: [ on ] ... libcrypto: [ on ] ... libcapstone: [ on ] ... llvm-perf: [ on ] ... zlib: [ on ] ... lzma: [ on ] ... get_cpuid: [ on ] ... bpf: [ on ] ... libaio: [ on ] ... libzstd: [ on ]
INSTALL libsubcmd_headers
INSTALL libperf_headers
INSTALL libapi_headers
INSTALL libsymbol_headers
INSTALL libbpf_headers
CC /tmp/build/perf-tools-next/util/symbol.o LD /tmp/build/perf-tools-next/util/perf-util-in.o LD /tmp/build/perf-tools-next/perf-util-in.o AR /tmp/build/perf-tools-next/libperf-util.a LINK /tmp/build/perf-tools-next/perf GEN /tmp/build/perf-tools-next/python/perf.cpython-313-x86_64-linux-gnu.so
INSTALL binaries
INSTALL tests
INSTALL libperf-jvmti.so
INSTALL libexec
INSTALL perf-archive
INSTALL perf-iostat
INSTALL perl-scripts
INSTALL python-scripts
INSTALL dlfilters
INSTALL perf_completion-script
INSTALL perf-tip
make: Leaving directory '/home/acme/git/perf-tools-next/tools/perf'
18: 'import perf' in python : Ok
⬢ [acme@toolbx perf-tools-next]$
root@number:~# perf top perf: Segmentation fault -------- backtrace -------- Segmentation fault (core dumped) root@number:~#
Tried also with this, but probably something else is at play:
root@number:~# perf probe -x ~/bin/perf dump_stack Added new event: probe_perf:dump_stack (on dump_stack in /home/acme/bin/perf)
You can now use it in all perf tools, such as:
perf record -e probe_perf:dump_stack -aR sleep 1
root@number:~#
root@number:~# perf trace -e probe_perf:dump_stack/max-stack=16/ perf top perf: Segmentation fault -------- backtrace -------- root@number:~#
Running perf trace on a separate terminal it also doesn't catch dump_stack being called (that probe point).
root@number:~# perf top --stdio PerfTop: 0 irqs/sec kernel: 0.0% exact: 0.0% lost: 0/0 drop: 0/0 [4000Hz cycles:P], (all, 32 CPUs) ------------------------------------------------------------------------------------------------------------------------------------------------------------------------
perf: Segmentation fault
Segmentation fault (core dumped)
root@number:~#
root@number:~# nm ~/bin/perf | grep dump_stack 0000000000637eaa T __dump_stack 00000000006380fa T dump_stack 000000000063816b T sighandler_dump_stack root@number:~#
⬢ [acme@toolbx perf-tools-next]$ rpm -q glibc
glibc-2.41-5.fc42.x86_64
root@number:~# uname -a Linux number 6.14.8-300.fc42.x86_64 #1 SMP PREEMPT_DYNAMIC Thu May 22 19:26:02 UTC 2025 x86_64 GNU/Linux root@number:~#
What am I missing?
- Arnaldo
Quoted 194 lines
____ unexpected signal (2) ____
#0 0x5590fb6209b6 in child_test_sig_handler builtin-test.c:243 #1 0x7f4a91e49e20 in __restore_rt libc_sigaction.c:0 #2 0x7f4a91ee4f33 in clock_nanosleep@GLIBC_2.2.5 clock_nanosleep.c:71 #3 0x7f4a91ef0333 in __nanosleep nanosleep.c:26 #4 0x7f4a91f01f68 in __sleep sleep.c:55 #5 0x5590fb638c63 in test__PERF_RECORD perf-record.c:295 #6 0x5590fb620b43 in run_test_child builtin-test.c:269 #7 0x5590fb5b83ab in start_command run-command.c:127 #8 0x5590fb621572 in start_test builtin-test.c:467 #9 0x5590fb621a47 in __cmd_test builtin-test.c:573 #10 0x5590fb6225ea in cmd_test builtin-test.c:775 #11 0x5590fb5a9099 in run_builtin perf.c:351 #12 0x5590fb5a9340 in handle_internal_command perf.c:404 #13 0x5590fb5a9499 in run_argv perf.c:451 #14 0x5590fb5a97e2 in main perf.c:558 #15 0x7f4a91e33d68 in __libc_start_call_main libc_start_call_main.h:74 #16 0x7f4a91e33e25 in __libc_start_main@@GLIBC_2.34 libc-start.c:128 #17 0x5590fb4fd6d1 in _start perf[436d1]```
Signed-off-by Ian Rogers <[email protected]>---
v2: Fix NO_BACKTRACE=1 build (Arnaldo)--- tools/perf/tests/builtin-test.c | 15 +++++++- tools/perf/ui/tui/setup.c | 2 +- tools/perf/util/debug.c | 68 +++++++++++++++++++++++++++------ tools/perf/util/debug.h | 1 + 4 files changed, 73 insertions(+), 13 deletions(-)tools/perf/tests/builtin-test.c +14
-1@@ -6,6 +6,9 @@*/ #include <fcntl.h> #include <errno.h> +#ifdef HAVE_BACKTRACE_SUPPORT +#include <execinfo.h> +#endif #include <poll.h> #include <unistd.h> #include <setjmp.h> @@ -231,6 +234,16 @@ static jmp_buf run_test_jmp_buf;static void child_test_sig_handler (int sig) { +#ifdef HAVE_BACKTRACE_SUPPORT +void *stackdump[32 ]; +size_t stackdump_size; +#endif + +fprintf (stderr,"\n---- unexpected signal (%d) ----\n" , sig); +#ifdef HAVE_BACKTRACE_SUPPORT + stackdump_size =backtrace (stackdump,ARRAY_SIZE (stackdump)); +__dump_stack (stderr, stackdump, stackdump_size); +#endif siglongjmp (run_test_jmp_buf, sig); }@@ -244,7 +257,7 @@ static int run_test_child(struct child_process *process) err =sigsetjmp (run_test_jmp_buf,1 );if (err) { -fprintf (stderr,"\n---- unexpected signal (%d) ----\n" , err); +/* Received signal. */ err = err >0 ? -err :-1 ; goto err_out; }tools/perf/ui/tui/setup.c +1
-1@@ -108,7 +108,7 @@ static void ui__signal_backtrace(int sig)printf ("-------- backtrace --------\n" ); size =backtrace (stackdump,ARRAY_SIZE (stackdump)); -backtrace_symbols_fd (stackdump, size,STDOUT_FILENO ); +__dump_stack (stdout, stackdump, size);exit (0 ); }tools/perf/util/debug.c +57
-11@@ -14,11 +14,18 @@#ifdef HAVE_BACKTRACE_SUPPORT #include <execinfo.h> #endif +#include "addr_location.h" #include "color.h" -#include "event.h" #include "debug.h" +#include "event.h" +#include "machine.h" +#include "map.h" #include "print_binary.h" +#include "srcline.h" +#include "symbol.h" +#include "synthetic-events.h" #include "target.h" +#include "thread.h" #include "trace-event.h" #include "ui/helpline.h" #include "ui/ui.h" @@ -298,21 +305,60 @@ void perf_debug_setup(void)libapi_set_print (pr_warning_wrapper, pr_warning_wrapper, pr_debug_wrapper); } +void __dump_stack (FILE *file,void **stackdump,size_t stackdump_size) +{ +/* TODO: async safety. printf, malloc, etc. aren't safe inside a signal handler. */ +pid_t pid =getpid (); +struct machine *machine =machine__new_live (/*kernel_maps=*/ false, pid); +struct thread *thread =NULL ; + +if (machine) + thread =machine__find_thread (machine, pid, pid); + +#ifdef HAVE_BACKTRACE_SUPPORT +if (!machine || !thread) { +/* +* Backtrace functions are async signal safe. Fall back on them +* if machine/thread creation fails. +*/ +backtrace_symbols_fd (stackdump, stackdump_size,fileno (file)); +machine__delete (machine); +return ; + } +#endif + +for (size_t i =0 ; i < stackdump_size; i++) { +struct addr_location al; +u64 addr = (u64 )stackdump[i]; +bool printed = false; + +addr_location__init (&al); +if (thread &&thread__find_map (thread,PERF_RECORD_MISC_USER , addr, &al)) { + al.sym =map__find_symbol (al.map, al.addr); +if (al.sym) { +fprintf (file," #%zd %p in %s " , i, stackdump[i], al.sym->name); + printed = true; + } + } +if (!printed) +fprintf (file," #%zd %p " , i, stackdump[i]); + +map__fprintf_srcline (al.map, al.addr,"" , file); +fprintf (file,"\n" ); +addr_location__exit (&al); + } +thread__put (thread); +machine__delete (machine); +} +/* Obtain a backtrace and print it to stdout. */ #ifdef HAVE_BACKTRACE_SUPPORT void dump_stack (void ) { -void *array[16 ]; -size_t size =backtrace (array,ARRAY_SIZE (array)); -char **strings =backtrace_symbols (array, size); -size_t i; - -printf ("Obtained %zd stack frames.\n" , size); - -for (i =0 ; i < size; i++) -printf ("%s\n" , strings[i]); +void *stackdump[32 ]; +size_t size =backtrace (stackdump,ARRAY_SIZE (stackdump)); -free (strings); +__dump_stack (stdout, stackdump, size); }#else void dump_stack (void ) {}tools/perf/util/debug.h +1
-0@@ -85,6 +85,7 @@ void debug_set_display_time(bool set);void perf_debug_setup (void );int perf_quiet_option (void ); +void __dump_stack (FILE *file,void **stackdump,size_t stackdump_size);void dump_stack (void );void sighandler_dump_stack (int sig);Signature · 1 line
2.49.0.1204.g71687c7c1d-goog
Ian Rogers
reviewerRe: [PATCH v2 2/2] perf debug: Add function symbols to dump_stack
Quoted 27 lines · Arnaldo Carvalho de Melo
Quoted 5 lines · Ian Rogers
Symbolize stack traces by creating a live machine. Add this functionality to
dump_stackand switchdump_stackusers to use it. Switch TUI to use it. Add stack traces to the child test function which can be useful to diagnose blocked code.Quoted 8 lines
Example output:
```
8: PERF_RECORD_* events &perf_samplefields : Running (1 active)
^C
Signal (2) while running tests.
Terminating tests with the same signal
Internal test harness failure. Completing any started tests:
: 8: PERF_RECORD_* events &perf_samplefields:So you are testing it with:
root@number:~# perf test PERF_RECORD 8: PERF_RECORD_* events & perf_sample fields : Running (1 active) ^C Signal (2) while running tests. Terminating tests with the same signal Internal test harness failure. Completing any started tests: 8: PERF_RECORD_* events & perf_sample fields : Skip (permissions) root@number:~#
Quoted 1 line
?
You are running the test as a child and by default the child test output is only displayed if verbose is enabled. I use verbose=2 below as verbose=1 won't display anything unless the child test gives a fail exit value:
``` $ git log --oneline b9ac06abfde9 perf debug: Add function symbols to dump_stack e561806265ed perf machine: Factor creating a "live" machine out of dwarf-unwind 628e124404b3 perf tests switch-tracking: Fix timestamp comparison $ make -C tools/perf O=/tmp/perf ... $ sudo /tmp/perf/perf test -vv PERF_RECORD 8: PERF_RECORD_* events & perf_sample fields: 8: PERF_RECORD_* events & perf_sample fields : Running (1 active) ^C Signal (2) while running tests. Terminating tests with the same signal Internal test harness failure. Completing any started tests: : 8: PERF_RECORD_* events & perf_sample fields: ---- unexpected signal (2) ---- #0 0x5617ec45e5e3 in child_test_sig_handler builtin-test.c:0 #1 0x7f855c649df0 in __restore_rt libc_sigaction.c:0 #2 0x7f855c699687 in __internal_syscall_cancel cancellation.c:64 #3 0x7f855c6e5f7a in clock_nanosleep@GLIBC_2.2.5 clock_nanosleep.c:72 #4 0x7f855c6f1393 in __nanosleep nanosleep.c:26 #5 0x7f855c702d68 in __sleep sleep.c:55 #6 0x5617ec46ebfb in test__PERF_RECORD perf-record.c:0 #7 0x5617ec45e4f0 in run_test_child builtin-test.c:0 #8 0x5617ec3faf0d in start_command run-command.c:127 #9 0x5617ec45f433 in __cmd_test builtin-test.c:0 #10 0x5617ec45faff in cmd_test perf[147aff] #11 0x5617ec3ed960 in run_builtin perf.c:0 #12 0x5617ec3edc7b in handle_internal_command perf.c:0 #13 0x5617ec368d33 in main perf[50d33] #14 0x7f855c633ca8 in __libc_start_call_main libc_start_call_main.h:74 #15 0x7f855c633d65 in __libc_start_main@@GLIBC_2.34 libc-start.c:128 #16 0x5617ec369381 in _start perf[51381] ---- unexpected signal (2) ---- #0 0x5617ec45e5e3 in child_test_sig_handler builtin-test.c:0 #1 0x7f855c649df0 in __restore_rt libc_sigaction.c:0 #2 0x7f855c6a3a14 in pthread_sigmask@GLIBC_2.2.5 pthread_sigmask.c:45 #3 0x7f855c649fd9 in __GI___sigprocmask sigprocmask.c:26 #4 0x7f855c72601b in __longjmp_chk longjmp.c:36 #5 0x5617ec45e600 in print_test_result.isra.0 builtin-test.c:0 #6 0x7f855c649df0 in __restore_rt libc_sigaction.c:0 #7 0x7f855c699687 in __internal_syscall_cancel cancellation.c:64 #8 0x7f855c6e5f7a in clock_nanosleep@GLIBC_2.2.5 clock_nanosleep.c:72 #9 0x7f855c6f1393 in __nanosleep nanosleep.c:26 #10 0x7f855c702d68 in __sleep sleep.c:55 #11 0x5617ec46ebfb in test__PERF_RECORD perf-record.c:0 #12 0x5617ec45e4f0 in run_test_child builtin-test.c:0 #13 0x5617ec3faf0d in start_command run-command.c:127 #14 0x5617ec45f433 in __cmd_test builtin-test.c:0 #15 0x5617ec45faff in cmd_test perf[147aff] #16 0x5617ec3ed960 in run_builtin perf.c:0 #17 0x5617ec3edc7b in handle_internal_command perf.c:0 #18 0x5617ec368d33 in main perf[50d33] #19 0x7f855c633ca8 in __libc_start_call_main libc_start_call_main.h:74 #20 0x7f855c633d65 in __libc_start_main@@GLIBC_2.34 libc-start.c:128 #21 0x5617ec369381 in _start perf[51381] 8: PERF_RECORD_* events & perf_sample fields : Skip (permissions) ```
Quoted 111 lines
I built it with DEBUG=1 and without, and with your patch, following your example output, I'm not being able to reproduce.
Tried it as well with:
⬢ [acme@toolbx perf-tools-next]$ cat segv.patch
tools/perf/util/symbol.c +2
-0@@ -402,6 +402,8 @@ static struct symbol *symbols__find(struct rb_root_cached *symbols, u64 ip) {struct rb_node *n; + *(int *)NULL =0 ; +if (symbols ==NULL )return NULL ;⬢ [acme@toolbx perf-tools-next]$ patch -p1 < segv.patch
patching filetools/perf/util/symbol.c
⬢ [acme@toolbx perf-tools-next]$ m
rm: cannot remove '/home/acme/libexec/perf-core/scripts/python/Perf-Trace-Util/lib/Perf/Trace/__pycache__/Core.cpython-313.pyc': Permission denied
make: Entering directory '/home/acme/git/perf-tools-next/tools/perf'
BUILD: Doing 'make -j32' parallel build
Warning: Kernel ABI header differences:
diff -utools/arch/arm64/include/asm/cputype.harch/arm64/include/asm/cputype.hAuto-detecting system features:
... libdw: [ on ] ... glibc: [ on ] ... libelf: [ on ] ... libnuma: [ on ] ... numa_num_possible_cpus: [ on ] ... libperl: [ on ] ... libpython: [ on ] ... libcrypto: [ on ] ... libcapstone: [ on ] ... llvm-perf: [ on ] ... zlib: [ on ] ... lzma: [ on ] ... get_cpuid: [ on ] ... bpf: [ on ] ... libaio: [ on ] ... libzstd: [ on ]INSTALL
libsubcmd_headers
INSTALLlibperf_headers
INSTALLlibapi_headers
INSTALLlibsymbol_headers
INSTALLlibbpf_headersCC /tmp/build/perf-tools-next/util/symbol.o LD /tmp/build/perf-tools-next/util/perf-util-in.o LD /tmp/build/perf-tools-next/perf-util-in.o AR /tmp/build/perf-tools-next/libperf-util.a LINK /tmp/build/perf-tools-next/perf GEN /tmp/build/perf-tools-next/python/perf.cpython-313-x86_64-linux-gnu.soINSTALL binaries
INSTALL tests
INSTALL libperf-jvmti.so
INSTALL libexec
INSTALL perf-archive
INSTALL perf-iostat
INSTALL perl-scripts
INSTALL python-scripts
INSTALL dlfilters
INSTALL perf_completion-script
INSTALL perf-tip
make: Leaving directory '/home/acme/git/perf-tools-next/tools/perf'
18: 'import perf' in python : Ok
⬢ [acme@toolbx perf-tools-next]$root@number:~# perf top perf: Segmentation fault -------- backtrace -------- Segmentation fault (core dumped) root@number:~#Tried also with this, but probably something else is at play:
root@number:~# perf probe -x ~/bin/perf dump_stack Added new event: probe_perf:dump_stack (on dump_stack in /home/acme/bin/perf)You can now use it in all perf tools, such as:
perf record -e probe_perf:dump_stack -aR sleep 1
root@number:~#
root@number:~# perf trace -e probe_perf:dump_stack/max-stack=16/ perf top perf: Segmentation fault -------- backtrace -------- root@number:~#Running perf trace on a separate terminal it also doesn't catch
dump_stackbeing called (that probe point).root@number:~# perf top --stdio PerfTop: 0 irqs/sec kernel: 0.0% exact: 0.0% lost: 0/0 drop: 0/0 [4000Hz cycles:P], (all, 32 CPUs) ------------------------------------------------------------------------------------------------------------------------------------------------------------------------perf: Segmentation fault
Segmentation fault (core dumped)
root@number:~#root@number:~# nm ~/bin/perf | grep dump_stack 0000000000637eaa T __dump_stack 00000000006380fa T dump_stack 000000000063816b T sighandler_dump_stack root@number:~#
Quoted 7 lines
⬢ [acme@toolbx perf-tools-next]$ rpm -q glibc
glibc-2.41-5.fc42.x86_64root@number:~# uname -a Linux number 6.14.8-300.fc42.x86_64 #1 SMP PREEMPT_DYNAMIC Thu May 22 19:26:02 UTC 2025 x86_64 GNU/Linux root@number:~#What am I missing?
For perf top I can do:
``` $ sudo /tmp/perf/perf top ``` in a different terminal: ``` $ sudo killall -11 perf ``` then back in the first terminal: ``` perf: Segmentation fault -------- backtrace -------- #0 0x55ee68647918 in ui__signal_backtrace setup.c:110 #1 0x7fe2d3849df0 in __restore_rt libc_sigaction.c:0 #2 0x7fe2d38a49ee in __syscall_cancel_arch syscall_cancel.S:56 #3 0x7fe2d3899668 in __internal_syscall_cancel cancellation.c:54 #4 0x7fe2d38996ad in __syscall_cancel cancellation.c:79 #5 0x7fe2d390d9c6 in __poll poll.c:43 #6 0x55ee6855dfcb in fdarray__poll array.c:139 #7 0x55ee68565a05 in perf_evlist__poll evlist.c:391 #8 0x55ee68673b5a in evlist__poll evlist.c:594 #9 0x55ee684e7a61 in __cmd_top builtin-top.c:1369 #10 0x55ee684e9a6c in cmd_top builtin-top.c:1856 #11 0x55ee6855d1f0 in run_builtin perf.c:351 #12 0x55ee6855d497 in handle_internal_command perf.c:404 #13 0x55ee6855d5f0 in run_argv perf.c:451 #14 0x55ee6855d939 in main perf.c:558 #15 0x7fe2d3833ca8 in __libc_start_call_main libc_start_call_main.h:74 #16 0x7fe2d3833d65 in __libc_start_main@@GLIBC_2.34 libc-start.c:128 #17 0x55ee684b0a91 in _start perf[4ca91] ```
As the same symbol code is used for the backtrace then adding the segv there is likely causing a segv in the signal handler in __dump_stack.
Thanks,
Ian
Quoted 196 lines
- Arnaldo
Quoted 194 lines
____ unexpected signal (2) ____
#0 0x5590fb6209b6 in child_test_sig_handler builtin-test.c:243 #1 0x7f4a91e49e20 in __restore_rt libc_sigaction.c:0 #2 0x7f4a91ee4f33 in clock_nanosleep@GLIBC_2.2.5 clock_nanosleep.c:71 #3 0x7f4a91ef0333 in __nanosleep nanosleep.c:26 #4 0x7f4a91f01f68 in __sleep sleep.c:55 #5 0x5590fb638c63 in test__PERF_RECORD perf-record.c:295 #6 0x5590fb620b43 in run_test_child builtin-test.c:269 #7 0x5590fb5b83ab in start_command run-command.c:127 #8 0x5590fb621572 in start_test builtin-test.c:467 #9 0x5590fb621a47 in __cmd_test builtin-test.c:573 #10 0x5590fb6225ea in cmd_test builtin-test.c:775 #11 0x5590fb5a9099 in run_builtin perf.c:351 #12 0x5590fb5a9340 in handle_internal_command perf.c:404 #13 0x5590fb5a9499 in run_argv perf.c:451 #14 0x5590fb5a97e2 in main perf.c:558 #15 0x7f4a91e33d68 in __libc_start_call_main libc_start_call_main.h:74 #16 0x7f4a91e33e25 in __libc_start_main@@GLIBC_2.34 libc-start.c:128 #17 0x5590fb4fd6d1 in _start perf[436d1]```
Signed-off-by Ian Rogers <[email protected]>---
v2: Fix NO_BACKTRACE=1 build (Arnaldo)--- tools/perf/tests/builtin-test.c | 15 +++++++- tools/perf/ui/tui/setup.c | 2 +- tools/perf/util/debug.c | 68 +++++++++++++++++++++++++++------ tools/perf/util/debug.h | 1 + 4 files changed, 73 insertions(+), 13 deletions(-)tools/perf/tests/builtin-test.c +14
-1@@ -6,6 +6,9 @@*/ #include <fcntl.h> #include <errno.h> +#ifdef HAVE_BACKTRACE_SUPPORT +#include <execinfo.h> +#endif #include <poll.h> #include <unistd.h> #include <setjmp.h> @@ -231,6 +234,16 @@ static jmp_buf run_test_jmp_buf;static void child_test_sig_handler (int sig) { +#ifdef HAVE_BACKTRACE_SUPPORT +void *stackdump[32 ]; +size_t stackdump_size; +#endif + +fprintf (stderr,"\n---- unexpected signal (%d) ----\n" , sig); +#ifdef HAVE_BACKTRACE_SUPPORT + stackdump_size =backtrace (stackdump,ARRAY_SIZE (stackdump)); +__dump_stack (stderr, stackdump, stackdump_size); +#endif siglongjmp (run_test_jmp_buf, sig); }@@ -244,7 +257,7 @@ static int run_test_child(struct child_process *process) err =sigsetjmp (run_test_jmp_buf,1 );if (err) { -fprintf (stderr,"\n---- unexpected signal (%d) ----\n" , err); +/* Received signal. */ err = err >0 ? -err :-1 ; goto err_out; }tools/perf/ui/tui/setup.c +1
-1@@ -108,7 +108,7 @@ static void ui__signal_backtrace(int sig)printf ("-------- backtrace --------\n" ); size =backtrace (stackdump,ARRAY_SIZE (stackdump)); -backtrace_symbols_fd (stackdump, size,STDOUT_FILENO ); +__dump_stack (stdout, stackdump, size);exit (0 ); }tools/perf/util/debug.c +57
-11@@ -14,11 +14,18 @@#ifdef HAVE_BACKTRACE_SUPPORT #include <execinfo.h> #endif +#include "addr_location.h" #include "color.h" -#include "event.h" #include "debug.h" +#include "event.h" +#include "machine.h" +#include "map.h" #include "print_binary.h" +#include "srcline.h" +#include "symbol.h" +#include "synthetic-events.h" #include "target.h" +#include "thread.h" #include "trace-event.h" #include "ui/helpline.h" #include "ui/ui.h" @@ -298,21 +305,60 @@ void perf_debug_setup(void)libapi_set_print (pr_warning_wrapper, pr_warning_wrapper, pr_debug_wrapper); } +void __dump_stack (FILE *file,void **stackdump,size_t stackdump_size) +{ +/* TODO: async safety. printf, malloc, etc. aren't safe inside a signal handler. */ +pid_t pid =getpid (); +struct machine *machine =machine__new_live (/*kernel_maps=*/ false, pid); +struct thread *thread =NULL ; + +if (machine) + thread =machine__find_thread (machine, pid, pid); + +#ifdef HAVE_BACKTRACE_SUPPORT +if (!machine || !thread) { +/* +* Backtrace functions are async signal safe. Fall back on them +* if machine/thread creation fails. +*/ +backtrace_symbols_fd (stackdump, stackdump_size,fileno (file)); +machine__delete (machine); +return ; + } +#endif + +for (size_t i =0 ; i < stackdump_size; i++) { +struct addr_location al; +u64 addr = (u64 )stackdump[i]; +bool printed = false; + +addr_location__init (&al); +if (thread &&thread__find_map (thread,PERF_RECORD_MISC_USER , addr, &al)) { + al.sym =map__find_symbol (al.map, al.addr); +if (al.sym) { +fprintf (file," #%zd %p in %s " , i, stackdump[i], al.sym->name); + printed = true; + } + } +if (!printed) +fprintf (file," #%zd %p " , i, stackdump[i]); + +map__fprintf_srcline (al.map, al.addr,"" , file); +fprintf (file,"\n" ); +addr_location__exit (&al); + } +thread__put (thread); +machine__delete (machine); +} +/* Obtain a backtrace and print it to stdout. */ #ifdef HAVE_BACKTRACE_SUPPORT void dump_stack (void ) { -void *array[16 ]; -size_t size =backtrace (array,ARRAY_SIZE (array)); -char **strings =backtrace_symbols (array, size); -size_t i; - -printf ("Obtained %zd stack frames.\n" , size); - -for (i =0 ; i < size; i++) -printf ("%s\n" , strings[i]); +void *stackdump[32 ]; +size_t size =backtrace (stackdump,ARRAY_SIZE (stackdump)); -free (strings); +__dump_stack (stdout, stackdump, size); }#else void dump_stack (void ) {}tools/perf/util/debug.h +1
-1@@ -85,6 +85,7 @@ void debug_set_display_time(bool set);void perf_debug_setup (void );int perf_quiet_option (void ); +void __dump_stack (FILE *file,void **stackdump,size_t stackdump_size);void dump_stack (void );void sighandler_dump_stack (int sig); --2.49.0.1204.g71687c7c1d-goog
Ian Rogers
reviewerRe: [PATCH v2 2/2] perf debug: Add function symbols to dump_stack
Quoted 250 lines · Ian Rogers
Quoted 27 lines · Arnaldo Carvalho de Melo
Quoted 5 lines · Ian Rogers
Symbolize stack traces by creating a live machine. Add this functionality to
dump_stackand switchdump_stackusers to use it. Switch TUI to use it. Add stack traces to the child test function which can be useful to diagnose blocked code.Quoted 8 lines
Example output:
```
8: PERF_RECORD_* events &perf_samplefields : Running (1 active)
^C
Signal (2) while running tests.
Terminating tests with the same signal
Internal test harness failure. Completing any started tests:
: 8: PERF_RECORD_* events &perf_samplefields:So you are testing it with:
root@number:~# perf test PERF_RECORD 8: PERF_RECORD_* events & perf_sample fields : Running (1 active) ^C Signal (2) while running tests. Terminating tests with the same signal Internal test harness failure. Completing any started tests: 8: PERF_RECORD_* events & perf_sample fields : Skip (permissions) root@number:~#Quoted 1 line
?
You are running the test as a child and by default the child test output is only displayed if verbose is enabled. I use verbose=2 below as verbose=1 won't display anything unless the child test gives a fail exit value:
``` $ git log --oneline b9ac06abfde9 perf debug: Add function symbols to dump_stack e561806265ed perf machine: Factor creating a "live" machine out of dwarf-unwind 628e124404b3 perf tests switch-tracking: Fix timestamp comparison $ make -C tools/perf O=/tmp/perf ... $ sudo /tmp/perf/perf test -vv PERF_RECORD 8: PERF_RECORD_* events & perf_sample fields: 8: PERF_RECORD_* events & perf_sample fields : Running (1 active) ^C Signal (2) while running tests. Terminating tests with the same signal Internal test harness failure. Completing any started tests: : 8: PERF_RECORD_* events & perf_sample fields: ---- unexpected signal (2) ---- #0 0x5617ec45e5e3 in child_test_sig_handler builtin-test.c:0 #1 0x7f855c649df0 in __restore_rt libc_sigaction.c:0 #2 0x7f855c699687 in __internal_syscall_cancel cancellation.c:64 #3 0x7f855c6e5f7a in clock_nanosleep@GLIBC_2.2.5 clock_nanosleep.c:72 #4 0x7f855c6f1393 in __nanosleep nanosleep.c:26 #5 0x7f855c702d68 in __sleep sleep.c:55 #6 0x5617ec46ebfb in test__PERF_RECORD perf-record.c:0 #7 0x5617ec45e4f0 in run_test_child builtin-test.c:0 #8 0x5617ec3faf0d in start_command run-command.c:127 #9 0x5617ec45f433 in __cmd_test builtin-test.c:0 #10 0x5617ec45faff in cmd_test perf[147aff] #11 0x5617ec3ed960 in run_builtin perf.c:0 #12 0x5617ec3edc7b in handle_internal_command perf.c:0 #13 0x5617ec368d33 in main perf[50d33] #14 0x7f855c633ca8 in __libc_start_call_main libc_start_call_main.h:74 #15 0x7f855c633d65 in __libc_start_main@@GLIBC_2.34 libc-start.c:128 #16 0x5617ec369381 in _start perf[51381] ---- unexpected signal (2) ---- #0 0x5617ec45e5e3 in child_test_sig_handler builtin-test.c:0 #1 0x7f855c649df0 in __restore_rt libc_sigaction.c:0 #2 0x7f855c6a3a14 in pthread_sigmask@GLIBC_2.2.5 pthread_sigmask.c:45 #3 0x7f855c649fd9 in __GI___sigprocmask sigprocmask.c:26 #4 0x7f855c72601b in __longjmp_chk longjmp.c:36 #5 0x5617ec45e600 in print_test_result.isra.0 builtin-test.c:0 #6 0x7f855c649df0 in __restore_rt libc_sigaction.c:0 #7 0x7f855c699687 in __internal_syscall_cancel cancellation.c:64 #8 0x7f855c6e5f7a in clock_nanosleep@GLIBC_2.2.5 clock_nanosleep.c:72 #9 0x7f855c6f1393 in __nanosleep nanosleep.c:26 #10 0x7f855c702d68 in __sleep sleep.c:55 #11 0x5617ec46ebfb in test__PERF_RECORD perf-record.c:0 #12 0x5617ec45e4f0 in run_test_child builtin-test.c:0 #13 0x5617ec3faf0d in start_command run-command.c:127 #14 0x5617ec45f433 in __cmd_test builtin-test.c:0 #15 0x5617ec45faff in cmd_test perf[147aff] #16 0x5617ec3ed960 in run_builtin perf.c:0 #17 0x5617ec3edc7b in handle_internal_command perf.c:0 #18 0x5617ec368d33 in main perf[50d33] #19 0x7f855c633ca8 in __libc_start_call_main libc_start_call_main.h:74 #20 0x7f855c633d65 in __libc_start_main@@GLIBC_2.34 libc-start.c:128 #21 0x5617ec369381 in _start perf[51381] 8: PERF_RECORD_* events & perf_sample fields : Skip (permissions) ```Quoted 111 lines
I built it with DEBUG=1 and without, and with your patch, following your example output, I'm not being able to reproduce.
Tried it as well with:
⬢ [acme@toolbx perf-tools-next]$ cat segv.patch
tools/perf/util/symbol.c +2
-0@@ -402,6 +402,8 @@ static struct symbol *symbols__find(struct rb_root_cached *symbols, u64 ip) {struct rb_node *n; + *(int *)NULL =0 ; +if (symbols ==NULL )return NULL ;⬢ [acme@toolbx perf-tools-next]$ patch -p1 < segv.patch
patching filetools/perf/util/symbol.c
⬢ [acme@toolbx perf-tools-next]$ m
rm: cannot remove '/home/acme/libexec/perf-core/scripts/python/Perf-Trace-Util/lib/Perf/Trace/__pycache__/Core.cpython-313.pyc': Permission denied
make: Entering directory '/home/acme/git/perf-tools-next/tools/perf'
BUILD: Doing 'make -j32' parallel build
Warning: Kernel ABI header differences:
diff -utools/arch/arm64/include/asm/cputype.harch/arm64/include/asm/cputype.hAuto-detecting system features:
... libdw: [ on ] ... glibc: [ on ] ... libelf: [ on ] ... libnuma: [ on ] ... numa_num_possible_cpus: [ on ] ... libperl: [ on ] ... libpython: [ on ] ... libcrypto: [ on ] ... libcapstone: [ on ] ... llvm-perf: [ on ] ... zlib: [ on ] ... lzma: [ on ] ... get_cpuid: [ on ] ... bpf: [ on ] ... libaio: [ on ] ... libzstd: [ on ]INSTALL
libsubcmd_headers
INSTALLlibperf_headers
INSTALLlibapi_headers
INSTALLlibsymbol_headers
INSTALLlibbpf_headersCC /tmp/build/perf-tools-next/util/symbol.o LD /tmp/build/perf-tools-next/util/perf-util-in.o LD /tmp/build/perf-tools-next/perf-util-in.o AR /tmp/build/perf-tools-next/libperf-util.a LINK /tmp/build/perf-tools-next/perf GEN /tmp/build/perf-tools-next/python/perf.cpython-313-x86_64-linux-gnu.soINSTALL binaries
INSTALL tests
INSTALL libperf-jvmti.so
INSTALL libexec
INSTALL perf-archive
INSTALL perf-iostat
INSTALL perl-scripts
INSTALL python-scripts
INSTALL dlfilters
INSTALL perf_completion-script
INSTALL perf-tip
make: Leaving directory '/home/acme/git/perf-tools-next/tools/perf'
18: 'import perf' in python : Ok
⬢ [acme@toolbx perf-tools-next]$root@number:~# perf top perf: Segmentation fault -------- backtrace -------- Segmentation fault (core dumped) root@number:~#Tried also with this, but probably something else is at play:
root@number:~# perf probe -x ~/bin/perf dump_stack Added new event: probe_perf:dump_stack (on dump_stack in /home/acme/bin/perf)You can now use it in all perf tools, such as:
perf record -e probe_perf:dump_stack -aR sleep 1
root@number:~#
root@number:~# perf trace -e probe_perf:dump_stack/max-stack=16/ perf top perf: Segmentation fault -------- backtrace -------- root@number:~#Running perf trace on a separate terminal it also doesn't catch
dump_stackbeing called (that probe point).root@number:~# perf top --stdio PerfTop: 0 irqs/sec kernel: 0.0% exact: 0.0% lost: 0/0 drop: 0/0 [4000Hz cycles:P], (all, 32 CPUs) ------------------------------------------------------------------------------------------------------------------------------------------------------------------------perf: Segmentation fault
Segmentation fault (core dumped)
root@number:~#root@number:~# nm ~/bin/perf | grep dump_stack 0000000000637eaa T __dump_stack 00000000006380fa T dump_stack 000000000063816b T sighandler_dump_stack root@number:~#Quoted 7 lines
⬢ [acme@toolbx perf-tools-next]$ rpm -q glibc
glibc-2.41-5.fc42.x86_64root@number:~# uname -a Linux number 6.14.8-300.fc42.x86_64 #1 SMP PREEMPT_DYNAMIC Thu May 22 19:26:02 UTC 2025 x86_64 GNU/Linux root@number:~#What am I missing?
For perf top I can do:
``` $ sudo /tmp/perf/perf top ``` in a different terminal: ``` $ sudo killall -11 perf ``` then back in the first terminal: ``` perf: Segmentation fault -------- backtrace -------- #0 0x55ee68647918 in ui__signal_backtrace setup.c:110 #1 0x7fe2d3849df0 in __restore_rt libc_sigaction.c:0 #2 0x7fe2d38a49ee in __syscall_cancel_arch syscall_cancel.S:56 #3 0x7fe2d3899668 in __internal_syscall_cancel cancellation.c:54 #4 0x7fe2d38996ad in __syscall_cancel cancellation.c:79 #5 0x7fe2d390d9c6 in __poll poll.c:43 #6 0x55ee6855dfcb in fdarray__poll array.c:139 #7 0x55ee68565a05 in perf_evlist__poll evlist.c:391 #8 0x55ee68673b5a in evlist__poll evlist.c:594 #9 0x55ee684e7a61 in __cmd_top builtin-top.c:1369 #10 0x55ee684e9a6c in cmd_top builtin-top.c:1856 #11 0x55ee6855d1f0 in run_builtin perf.c:351 #12 0x55ee6855d497 in handle_internal_command perf.c:404 #13 0x55ee6855d5f0 in run_argv perf.c:451 #14 0x55ee6855d939 in main perf.c:558 #15 0x7fe2d3833ca8 in __libc_start_call_main libc_start_call_main.h:74 #16 0x7fe2d3833d65 in __libc_start_main@@GLIBC_2.34 libc-start.c:128 #17 0x55ee684b0a91 in _start perf[4ca91] ```
Quoted 2 lines
As the same symbol code is used for the backtrace then adding the segv there is likely causing a segv in the signal handler in
__dump_stack.
Just to note, it would be nice for this to land as I need to rebase:
https://lore.kernel.org/lkml/[email protected]/
due to the machine changes. I believe the patches are WAI as shown in the output.
Thanks,
Ian
Quoted 199 lines
Thanks,
IanQuoted 196 lines
- Arnaldo
Quoted 194 lines
____ unexpected signal (2) ____
#0 0x5590fb6209b6 in child_test_sig_handler builtin-test.c:243 #1 0x7f4a91e49e20 in __restore_rt libc_sigaction.c:0 #2 0x7f4a91ee4f33 in clock_nanosleep@GLIBC_2.2.5 clock_nanosleep.c:71 #3 0x7f4a91ef0333 in __nanosleep nanosleep.c:26 #4 0x7f4a91f01f68 in __sleep sleep.c:55 #5 0x5590fb638c63 in test__PERF_RECORD perf-record.c:295 #6 0x5590fb620b43 in run_test_child builtin-test.c:269 #7 0x5590fb5b83ab in start_command run-command.c:127 #8 0x5590fb621572 in start_test builtin-test.c:467 #9 0x5590fb621a47 in __cmd_test builtin-test.c:573 #10 0x5590fb6225ea in cmd_test builtin-test.c:775 #11 0x5590fb5a9099 in run_builtin perf.c:351 #12 0x5590fb5a9340 in handle_internal_command perf.c:404 #13 0x5590fb5a9499 in run_argv perf.c:451 #14 0x5590fb5a97e2 in main perf.c:558 #15 0x7f4a91e33d68 in __libc_start_call_main libc_start_call_main.h:74 #16 0x7f4a91e33e25 in __libc_start_main@@GLIBC_2.34 libc-start.c:128 #17 0x5590fb4fd6d1 in _start perf[436d1]```
Signed-off-by Ian Rogers <[email protected]>---
v2: Fix NO_BACKTRACE=1 build (Arnaldo)--- tools/perf/tests/builtin-test.c | 15 +++++++- tools/perf/ui/tui/setup.c | 2 +- tools/perf/util/debug.c | 68 +++++++++++++++++++++++++++------ tools/perf/util/debug.h | 1 + 4 files changed, 73 insertions(+), 13 deletions(-)tools/perf/tests/builtin-test.c +14
-1@@ -6,6 +6,9 @@*/ #include <fcntl.h> #include <errno.h> +#ifdef HAVE_BACKTRACE_SUPPORT +#include <execinfo.h> +#endif #include <poll.h> #include <unistd.h> #include <setjmp.h> @@ -231,6 +234,16 @@ static jmp_buf run_test_jmp_buf;static void child_test_sig_handler (int sig) { +#ifdef HAVE_BACKTRACE_SUPPORT +void *stackdump[32 ]; +size_t stackdump_size; +#endif + +fprintf (stderr,"\n---- unexpected signal (%d) ----\n" , sig); +#ifdef HAVE_BACKTRACE_SUPPORT + stackdump_size =backtrace (stackdump,ARRAY_SIZE (stackdump)); +__dump_stack (stderr, stackdump, stackdump_size); +#endif siglongjmp (run_test_jmp_buf, sig); }@@ -244,7 +257,7 @@ static int run_test_child(struct child_process *process) err =sigsetjmp (run_test_jmp_buf,1 );if (err) { -fprintf (stderr,"\n---- unexpected signal (%d) ----\n" , err); +/* Received signal. */ err = err >0 ? -err :-1 ; goto err_out; }tools/perf/ui/tui/setup.c +1
-1@@ -108,7 +108,7 @@ static void ui__signal_backtrace(int sig)printf ("-------- backtrace --------\n" ); size =backtrace (stackdump,ARRAY_SIZE (stackdump)); -backtrace_symbols_fd (stackdump, size,STDOUT_FILENO ); +__dump_stack (stdout, stackdump, size);exit (0 ); }tools/perf/util/debug.c +57
-11@@ -14,11 +14,18 @@#ifdef HAVE_BACKTRACE_SUPPORT #include <execinfo.h> #endif +#include "addr_location.h" #include "color.h" -#include "event.h" #include "debug.h" +#include "event.h" +#include "machine.h" +#include "map.h" #include "print_binary.h" +#include "srcline.h" +#include "symbol.h" +#include "synthetic-events.h" #include "target.h" +#include "thread.h" #include "trace-event.h" #include "ui/helpline.h" #include "ui/ui.h" @@ -298,21 +305,60 @@ void perf_debug_setup(void)libapi_set_print (pr_warning_wrapper, pr_warning_wrapper, pr_debug_wrapper); } +void __dump_stack (FILE *file,void **stackdump,size_t stackdump_size) +{ +/* TODO: async safety. printf, malloc, etc. aren't safe inside a signal handler. */ +pid_t pid =getpid (); +struct machine *machine =machine__new_live (/*kernel_maps=*/ false, pid); +struct thread *thread =NULL ; + +if (machine) + thread =machine__find_thread (machine, pid, pid); + +#ifdef HAVE_BACKTRACE_SUPPORT +if (!machine || !thread) { +/* +* Backtrace functions are async signal safe. Fall back on them +* if machine/thread creation fails. +*/ +backtrace_symbols_fd (stackdump, stackdump_size,fileno (file)); +machine__delete (machine); +return ; + } +#endif + +for (size_t i =0 ; i < stackdump_size; i++) { +struct addr_location al; +u64 addr = (u64 )stackdump[i]; +bool printed = false; + +addr_location__init (&al); +if (thread &&thread__find_map (thread,PERF_RECORD_MISC_USER , addr, &al)) { + al.sym =map__find_symbol (al.map, al.addr); +if (al.sym) { +fprintf (file," #%zd %p in %s " , i, stackdump[i], al.sym->name); + printed = true; + } + } +if (!printed) +fprintf (file," #%zd %p " , i, stackdump[i]); + +map__fprintf_srcline (al.map, al.addr,"" , file); +fprintf (file,"\n" ); +addr_location__exit (&al); + } +thread__put (thread); +machine__delete (machine); +} +/* Obtain a backtrace and print it to stdout. */ #ifdef HAVE_BACKTRACE_SUPPORT void dump_stack (void ) { -void *array[16 ]; -size_t size =backtrace (array,ARRAY_SIZE (array)); -char **strings =backtrace_symbols (array, size); -size_t i; - -printf ("Obtained %zd stack frames.\n" , size); - -for (i =0 ; i < size; i++) -printf ("%s\n" , strings[i]); +void *stackdump[32 ]; +size_t size =backtrace (stackdump,ARRAY_SIZE (stackdump)); -free (strings); +__dump_stack (stdout, stackdump, size); }#else void dump_stack (void ) {}tools/perf/util/debug.h +1
-1@@ -85,6 +85,7 @@ void debug_set_display_time(bool set);void perf_debug_setup (void );int perf_quiet_option (void ); +void __dump_stack (FILE *file,void **stackdump,size_t stackdump_size);void dump_stack (void );void sighandler_dump_stack (int sig); --2.49.0.1204.g71687c7c1d-goog
Ian Rogers
reviewerRe: [PATCH v2 2/2] perf debug: Add function symbols to dump_stack
Quoted 255 lines · Ian Rogers
Quoted 250 lines · Ian Rogers
Quoted 27 lines · Arnaldo Carvalho de Melo
Quoted 5 lines · Ian Rogers
Symbolize stack traces by creating a live machine. Add this functionality to
dump_stackand switchdump_stackusers to use it. Switch TUI to use it. Add stack traces to the child test function which can be useful to diagnose blocked code.Quoted 8 lines
Example output:
```
8: PERF_RECORD_* events &perf_samplefields : Running (1 active)
^C
Signal (2) while running tests.
Terminating tests with the same signal
Internal test harness failure. Completing any started tests:
: 8: PERF_RECORD_* events &perf_samplefields:So you are testing it with:
root@number:~# perf test PERF_RECORD 8: PERF_RECORD_* events & perf_sample fields : Running (1 active) ^C Signal (2) while running tests. Terminating tests with the same signal Internal test harness failure. Completing any started tests: 8: PERF_RECORD_* events & perf_sample fields : Skip (permissions) root@number:~#Quoted 1 line
?
You are running the test as a child and by default the child test output is only displayed if verbose is enabled. I use verbose=2 below as verbose=1 won't display anything unless the child test gives a fail exit value:
``` $ git log --oneline b9ac06abfde9 perf debug: Add function symbols to dump_stack e561806265ed perf machine: Factor creating a "live" machine out of dwarf-unwind 628e124404b3 perf tests switch-tracking: Fix timestamp comparison $ make -C tools/perf O=/tmp/perf ... $ sudo /tmp/perf/perf test -vv PERF_RECORD 8: PERF_RECORD_* events & perf_sample fields: 8: PERF_RECORD_* events & perf_sample fields : Running (1 active) ^C Signal (2) while running tests. Terminating tests with the same signal Internal test harness failure. Completing any started tests: : 8: PERF_RECORD_* events & perf_sample fields: ---- unexpected signal (2) ---- #0 0x5617ec45e5e3 in child_test_sig_handler builtin-test.c:0 #1 0x7f855c649df0 in __restore_rt libc_sigaction.c:0 #2 0x7f855c699687 in __internal_syscall_cancel cancellation.c:64 #3 0x7f855c6e5f7a in clock_nanosleep@GLIBC_2.2.5 clock_nanosleep.c:72 #4 0x7f855c6f1393 in __nanosleep nanosleep.c:26 #5 0x7f855c702d68 in __sleep sleep.c:55 #6 0x5617ec46ebfb in test__PERF_RECORD perf-record.c:0 #7 0x5617ec45e4f0 in run_test_child builtin-test.c:0 #8 0x5617ec3faf0d in start_command run-command.c:127 #9 0x5617ec45f433 in __cmd_test builtin-test.c:0 #10 0x5617ec45faff in cmd_test perf[147aff] #11 0x5617ec3ed960 in run_builtin perf.c:0 #12 0x5617ec3edc7b in handle_internal_command perf.c:0 #13 0x5617ec368d33 in main perf[50d33] #14 0x7f855c633ca8 in __libc_start_call_main libc_start_call_main.h:74 #15 0x7f855c633d65 in __libc_start_main@@GLIBC_2.34 libc-start.c:128 #16 0x5617ec369381 in _start perf[51381] ---- unexpected signal (2) ---- #0 0x5617ec45e5e3 in child_test_sig_handler builtin-test.c:0 #1 0x7f855c649df0 in __restore_rt libc_sigaction.c:0 #2 0x7f855c6a3a14 in pthread_sigmask@GLIBC_2.2.5 pthread_sigmask.c:45 #3 0x7f855c649fd9 in __GI___sigprocmask sigprocmask.c:26 #4 0x7f855c72601b in __longjmp_chk longjmp.c:36 #5 0x5617ec45e600 in print_test_result.isra.0 builtin-test.c:0 #6 0x7f855c649df0 in __restore_rt libc_sigaction.c:0 #7 0x7f855c699687 in __internal_syscall_cancel cancellation.c:64 #8 0x7f855c6e5f7a in clock_nanosleep@GLIBC_2.2.5 clock_nanosleep.c:72 #9 0x7f855c6f1393 in __nanosleep nanosleep.c:26 #10 0x7f855c702d68 in __sleep sleep.c:55 #11 0x5617ec46ebfb in test__PERF_RECORD perf-record.c:0 #12 0x5617ec45e4f0 in run_test_child builtin-test.c:0 #13 0x5617ec3faf0d in start_command run-command.c:127 #14 0x5617ec45f433 in __cmd_test builtin-test.c:0 #15 0x5617ec45faff in cmd_test perf[147aff] #16 0x5617ec3ed960 in run_builtin perf.c:0 #17 0x5617ec3edc7b in handle_internal_command perf.c:0 #18 0x5617ec368d33 in main perf[50d33] #19 0x7f855c633ca8 in __libc_start_call_main libc_start_call_main.h:74 #20 0x7f855c633d65 in __libc_start_main@@GLIBC_2.34 libc-start.c:128 #21 0x5617ec369381 in _start perf[51381] 8: PERF_RECORD_* events & perf_sample fields : Skip (permissions) ```Quoted 111 lines
I built it with DEBUG=1 and without, and with your patch, following your example output, I'm not being able to reproduce.
Tried it as well with:
⬢ [acme@toolbx perf-tools-next]$ cat segv.patch
tools/perf/util/symbol.c +2
-0@@ -402,6 +402,8 @@ static struct symbol *symbols__find(struct rb_root_cached *symbols, u64 ip) {struct rb_node *n; + *(int *)NULL =0 ; +if (symbols ==NULL )return NULL ;⬢ [acme@toolbx perf-tools-next]$ patch -p1 < segv.patch
patching filetools/perf/util/symbol.c
⬢ [acme@toolbx perf-tools-next]$ m
rm: cannot remove '/home/acme/libexec/perf-core/scripts/python/Perf-Trace-Util/lib/Perf/Trace/__pycache__/Core.cpython-313.pyc': Permission denied
make: Entering directory '/home/acme/git/perf-tools-next/tools/perf'
BUILD: Doing 'make -j32' parallel build
Warning: Kernel ABI header differences:
diff -utools/arch/arm64/include/asm/cputype.harch/arm64/include/asm/cputype.hAuto-detecting system features:
... libdw: [ on ] ... glibc: [ on ] ... libelf: [ on ] ... libnuma: [ on ] ... numa_num_possible_cpus: [ on ] ... libperl: [ on ] ... libpython: [ on ] ... libcrypto: [ on ] ... libcapstone: [ on ] ... llvm-perf: [ on ] ... zlib: [ on ] ... lzma: [ on ] ... get_cpuid: [ on ] ... bpf: [ on ] ... libaio: [ on ] ... libzstd: [ on ]INSTALL
libsubcmd_headers
INSTALLlibperf_headers
INSTALLlibapi_headers
INSTALLlibsymbol_headers
INSTALLlibbpf_headersCC /tmp/build/perf-tools-next/util/symbol.o LD /tmp/build/perf-tools-next/util/perf-util-in.o LD /tmp/build/perf-tools-next/perf-util-in.o AR /tmp/build/perf-tools-next/libperf-util.a LINK /tmp/build/perf-tools-next/perf GEN /tmp/build/perf-tools-next/python/perf.cpython-313-x86_64-linux-gnu.soINSTALL binaries
INSTALL tests
INSTALL libperf-jvmti.so
INSTALL libexec
INSTALL perf-archive
INSTALL perf-iostat
INSTALL perl-scripts
INSTALL python-scripts
INSTALL dlfilters
INSTALL perf_completion-script
INSTALL perf-tip
make: Leaving directory '/home/acme/git/perf-tools-next/tools/perf'
18: 'import perf' in python : Ok
⬢ [acme@toolbx perf-tools-next]$root@number:~# perf top perf: Segmentation fault -------- backtrace -------- Segmentation fault (core dumped) root@number:~#Tried also with this, but probably something else is at play:
root@number:~# perf probe -x ~/bin/perf dump_stack Added new event: probe_perf:dump_stack (on dump_stack in /home/acme/bin/perf)You can now use it in all perf tools, such as:
perf record -e probe_perf:dump_stack -aR sleep 1
root@number:~#
root@number:~# perf trace -e probe_perf:dump_stack/max-stack=16/ perf top perf: Segmentation fault -------- backtrace -------- root@number:~#Running perf trace on a separate terminal it also doesn't catch
dump_stackbeing called (that probe point).root@number:~# perf top --stdio PerfTop: 0 irqs/sec kernel: 0.0% exact: 0.0% lost: 0/0 drop: 0/0 [4000Hz cycles:P], (all, 32 CPUs) ------------------------------------------------------------------------------------------------------------------------------------------------------------------------perf: Segmentation fault
Segmentation fault (core dumped)
root@number:~#root@number:~# nm ~/bin/perf | grep dump_stack 0000000000637eaa T __dump_stack 00000000006380fa T dump_stack 000000000063816b T sighandler_dump_stack root@number:~#Quoted 7 lines
⬢ [acme@toolbx perf-tools-next]$ rpm -q glibc
glibc-2.41-5.fc42.x86_64root@number:~# uname -a Linux number 6.14.8-300.fc42.x86_64 #1 SMP PREEMPT_DYNAMIC Thu May 22 19:26:02 UTC 2025 x86_64 GNU/Linux root@number:~#What am I missing?
For perf top I can do:
``` $ sudo /tmp/perf/perf top ``` in a different terminal: ``` $ sudo killall -11 perf ``` then back in the first terminal: ``` perf: Segmentation fault -------- backtrace -------- #0 0x55ee68647918 in ui__signal_backtrace setup.c:110 #1 0x7fe2d3849df0 in __restore_rt libc_sigaction.c:0 #2 0x7fe2d38a49ee in __syscall_cancel_arch syscall_cancel.S:56 #3 0x7fe2d3899668 in __internal_syscall_cancel cancellation.c:54 #4 0x7fe2d38996ad in __syscall_cancel cancellation.c:79 #5 0x7fe2d390d9c6 in __poll poll.c:43 #6 0x55ee6855dfcb in fdarray__poll array.c:139 #7 0x55ee68565a05 in perf_evlist__poll evlist.c:391 #8 0x55ee68673b5a in evlist__poll evlist.c:594 #9 0x55ee684e7a61 in __cmd_top builtin-top.c:1369 #10 0x55ee684e9a6c in cmd_top builtin-top.c:1856 #11 0x55ee6855d1f0 in run_builtin perf.c:351 #12 0x55ee6855d497 in handle_internal_command perf.c:404 #13 0x55ee6855d5f0 in run_argv perf.c:451 #14 0x55ee6855d939 in main perf.c:558 #15 0x7fe2d3833ca8 in __libc_start_call_main libc_start_call_main.h:74 #16 0x7fe2d3833d65 in __libc_start_main@@GLIBC_2.34 libc-start.c:128 #17 0x55ee684b0a91 in _start perf[4ca91] ```Quoted 2 lines
As the same symbol code is used for the backtrace then adding the segv there is likely causing a segv in the signal handler in
__dump_stack.
Quoted 4 lines
Just to note, it would be nice for this to land as I need to rebase:
https://lore.kernel.org/lkml/[email protected]/
due to the machine changes. I believe the patches are WAI as shown in the output.
Ok, I see this made it into v6.16 PR:
https://lore.kernel.org/lkml/[email protected]/
Thanks Arnaldo!
Ian
Ian Rogers
reviewerRe: [PATCH v2 2/2] perf debug: Add function symbols to dump_stack
Quoted 257 lines · Ian Rogers
Quoted 255 lines · Ian Rogers
Quoted 252 lines · Ian Rogers
Quoted 27 lines · Arnaldo Carvalho de Melo
Quoted 5 lines · Ian Rogers
Symbolize stack traces by creating a live machine. Add this functionality to
dump_stackand switchdump_stackusers to use it. Switch TUI to use it. Add stack traces to the child test function which can be useful to diagnose blocked code.Quoted 8 lines
Example output:
```
8: PERF_RECORD_* events &perf_samplefields : Running (1 active)
^C
Signal (2) while running tests.
Terminating tests with the same signal
Internal test harness failure. Completing any started tests:
: 8: PERF_RECORD_* events &perf_samplefields:So you are testing it with:
root@number:~# perf test PERF_RECORD 8: PERF_RECORD_* events & perf_sample fields : Running (1 active) ^C Signal (2) while running tests. Terminating tests with the same signal Internal test harness failure. Completing any started tests: 8: PERF_RECORD_* events & perf_sample fields : Skip (permissions) root@number:~#Quoted 1 line
?
You are running the test as a child and by default the child test output is only displayed if verbose is enabled. I use verbose=2 below as verbose=1 won't display anything unless the child test gives a fail exit value:
``` $ git log --oneline b9ac06abfde9 perf debug: Add function symbols to dump_stack e561806265ed perf machine: Factor creating a "live" machine out of dwarf-unwind 628e124404b3 perf tests switch-tracking: Fix timestamp comparison $ make -C tools/perf O=/tmp/perf ... $ sudo /tmp/perf/perf test -vv PERF_RECORD 8: PERF_RECORD_* events & perf_sample fields: 8: PERF_RECORD_* events & perf_sample fields : Running (1 active) ^C Signal (2) while running tests. Terminating tests with the same signal Internal test harness failure. Completing any started tests: : 8: PERF_RECORD_* events & perf_sample fields: ---- unexpected signal (2) ---- #0 0x5617ec45e5e3 in child_test_sig_handler builtin-test.c:0 #1 0x7f855c649df0 in __restore_rt libc_sigaction.c:0 #2 0x7f855c699687 in __internal_syscall_cancel cancellation.c:64 #3 0x7f855c6e5f7a in clock_nanosleep@GLIBC_2.2.5 clock_nanosleep.c:72 #4 0x7f855c6f1393 in __nanosleep nanosleep.c:26 #5 0x7f855c702d68 in __sleep sleep.c:55 #6 0x5617ec46ebfb in test__PERF_RECORD perf-record.c:0 #7 0x5617ec45e4f0 in run_test_child builtin-test.c:0 #8 0x5617ec3faf0d in start_command run-command.c:127 #9 0x5617ec45f433 in __cmd_test builtin-test.c:0 #10 0x5617ec45faff in cmd_test perf[147aff] #11 0x5617ec3ed960 in run_builtin perf.c:0 #12 0x5617ec3edc7b in handle_internal_command perf.c:0 #13 0x5617ec368d33 in main perf[50d33] #14 0x7f855c633ca8 in __libc_start_call_main libc_start_call_main.h:74 #15 0x7f855c633d65 in __libc_start_main@@GLIBC_2.34 libc-start.c:128 #16 0x5617ec369381 in _start perf[51381] ---- unexpected signal (2) ---- #0 0x5617ec45e5e3 in child_test_sig_handler builtin-test.c:0 #1 0x7f855c649df0 in __restore_rt libc_sigaction.c:0 #2 0x7f855c6a3a14 in pthread_sigmask@GLIBC_2.2.5 pthread_sigmask.c:45 #3 0x7f855c649fd9 in __GI___sigprocmask sigprocmask.c:26 #4 0x7f855c72601b in __longjmp_chk longjmp.c:36 #5 0x5617ec45e600 in print_test_result.isra.0 builtin-test.c:0 #6 0x7f855c649df0 in __restore_rt libc_sigaction.c:0 #7 0x7f855c699687 in __internal_syscall_cancel cancellation.c:64 #8 0x7f855c6e5f7a in clock_nanosleep@GLIBC_2.2.5 clock_nanosleep.c:72 #9 0x7f855c6f1393 in __nanosleep nanosleep.c:26 #10 0x7f855c702d68 in __sleep sleep.c:55 #11 0x5617ec46ebfb in test__PERF_RECORD perf-record.c:0 #12 0x5617ec45e4f0 in run_test_child builtin-test.c:0 #13 0x5617ec3faf0d in start_command run-command.c:127 #14 0x5617ec45f433 in __cmd_test builtin-test.c:0 #15 0x5617ec45faff in cmd_test perf[147aff] #16 0x5617ec3ed960 in run_builtin perf.c:0 #17 0x5617ec3edc7b in handle_internal_command perf.c:0 #18 0x5617ec368d33 in main perf[50d33] #19 0x7f855c633ca8 in __libc_start_call_main libc_start_call_main.h:74 #20 0x7f855c633d65 in __libc_start_main@@GLIBC_2.34 libc-start.c:128 #21 0x5617ec369381 in _start perf[51381] 8: PERF_RECORD_* events & perf_sample fields : Skip (permissions) ```Quoted 111 lines
I built it with DEBUG=1 and without, and with your patch, following your example output, I'm not being able to reproduce.
Tried it as well with:
⬢ [acme@toolbx perf-tools-next]$ cat segv.patch
tools/perf/util/symbol.c +2
-0@@ -402,6 +402,8 @@ static struct symbol *symbols__find(struct rb_root_cached *symbols, u64 ip) {struct rb_node *n; + *(int *)NULL =0 ; +if (symbols ==NULL )return NULL ;⬢ [acme@toolbx perf-tools-next]$ patch -p1 < segv.patch
patching filetools/perf/util/symbol.c
⬢ [acme@toolbx perf-tools-next]$ m
rm: cannot remove '/home/acme/libexec/perf-core/scripts/python/Perf-Trace-Util/lib/Perf/Trace/__pycache__/Core.cpython-313.pyc': Permission denied
make: Entering directory '/home/acme/git/perf-tools-next/tools/perf'
BUILD: Doing 'make -j32' parallel build
Warning: Kernel ABI header differences:
diff -utools/arch/arm64/include/asm/cputype.harch/arm64/include/asm/cputype.hAuto-detecting system features:
... libdw: [ on ] ... glibc: [ on ] ... libelf: [ on ] ... libnuma: [ on ] ... numa_num_possible_cpus: [ on ] ... libperl: [ on ] ... libpython: [ on ] ... libcrypto: [ on ] ... libcapstone: [ on ] ... llvm-perf: [ on ] ... zlib: [ on ] ... lzma: [ on ] ... get_cpuid: [ on ] ... bpf: [ on ] ... libaio: [ on ] ... libzstd: [ on ]INSTALL
libsubcmd_headers
INSTALLlibperf_headers
INSTALLlibapi_headers
INSTALLlibsymbol_headers
INSTALLlibbpf_headersCC /tmp/build/perf-tools-next/util/symbol.o LD /tmp/build/perf-tools-next/util/perf-util-in.o LD /tmp/build/perf-tools-next/perf-util-in.o AR /tmp/build/perf-tools-next/libperf-util.a LINK /tmp/build/perf-tools-next/perf GEN /tmp/build/perf-tools-next/python/perf.cpython-313-x86_64-linux-gnu.soINSTALL binaries
INSTALL tests
INSTALL libperf-jvmti.so
INSTALL libexec
INSTALL perf-archive
INSTALL perf-iostat
INSTALL perl-scripts
INSTALL python-scripts
INSTALL dlfilters
INSTALL perf_completion-script
INSTALL perf-tip
make: Leaving directory '/home/acme/git/perf-tools-next/tools/perf'
18: 'import perf' in python : Ok
⬢ [acme@toolbx perf-tools-next]$root@number:~# perf top perf: Segmentation fault -------- backtrace -------- Segmentation fault (core dumped) root@number:~#Tried also with this, but probably something else is at play:
root@number:~# perf probe -x ~/bin/perf dump_stack Added new event: probe_perf:dump_stack (on dump_stack in /home/acme/bin/perf)You can now use it in all perf tools, such as:
perf record -e probe_perf:dump_stack -aR sleep 1
root@number:~#
root@number:~# perf trace -e probe_perf:dump_stack/max-stack=16/ perf top perf: Segmentation fault -------- backtrace -------- root@number:~#Running perf trace on a separate terminal it also doesn't catch
dump_stackbeing called (that probe point).root@number:~# perf top --stdio PerfTop: 0 irqs/sec kernel: 0.0% exact: 0.0% lost: 0/0 drop: 0/0 [4000Hz cycles:P], (all, 32 CPUs) ------------------------------------------------------------------------------------------------------------------------------------------------------------------------perf: Segmentation fault
Segmentation fault (core dumped)
root@number:~#root@number:~# nm ~/bin/perf | grep dump_stack 0000000000637eaa T __dump_stack 00000000006380fa T dump_stack 000000000063816b T sighandler_dump_stack root@number:~#Quoted 7 lines
⬢ [acme@toolbx perf-tools-next]$ rpm -q glibc
glibc-2.41-5.fc42.x86_64root@number:~# uname -a Linux number 6.14.8-300.fc42.x86_64 #1 SMP PREEMPT_DYNAMIC Thu May 22 19:26:02 UTC 2025 x86_64 GNU/Linux root@number:~#What am I missing?
For perf top I can do:
``` $ sudo /tmp/perf/perf top ``` in a different terminal: ``` $ sudo killall -11 perf ``` then back in the first terminal: ``` perf: Segmentation fault -------- backtrace -------- #0 0x55ee68647918 in ui__signal_backtrace setup.c:110 #1 0x7fe2d3849df0 in __restore_rt libc_sigaction.c:0 #2 0x7fe2d38a49ee in __syscall_cancel_arch syscall_cancel.S:56 #3 0x7fe2d3899668 in __internal_syscall_cancel cancellation.c:54 #4 0x7fe2d38996ad in __syscall_cancel cancellation.c:79 #5 0x7fe2d390d9c6 in __poll poll.c:43 #6 0x55ee6855dfcb in fdarray__poll array.c:139 #7 0x55ee68565a05 in perf_evlist__poll evlist.c:391 #8 0x55ee68673b5a in evlist__poll evlist.c:594 #9 0x55ee684e7a61 in __cmd_top builtin-top.c:1369 #10 0x55ee684e9a6c in cmd_top builtin-top.c:1856 #11 0x55ee6855d1f0 in run_builtin perf.c:351 #12 0x55ee6855d497 in handle_internal_command perf.c:404 #13 0x55ee6855d5f0 in run_argv perf.c:451 #14 0x55ee6855d939 in main perf.c:558 #15 0x7fe2d3833ca8 in __libc_start_call_main libc_start_call_main.h:74 #16 0x7fe2d3833d65 in __libc_start_main@@GLIBC_2.34 libc-start.c:128 #17 0x55ee684b0a91 in _start perf[4ca91] ```As the same symbol code is used for the backtrace then adding the segv there is likely causing a segv in the signal handler in
__dump_stack.
Quoted 7 lines
Quoted 4 lines
Just to note, it would be nice for this to land as I need to rebase:
https://lore.kernel.org/lkml/[email protected]/
due to the machine changes. I believe the patches are WAI as shown in the output.Ok, I see this made it into v6.16 PR:
https://lore.kernel.org/lkml/[email protected]/
Actually just the first patch. I'll rebase and resend just patch 2 as v3.
Thanks,
Ian