util/error.c: Print backtrace on error
RFC PATCH 1/1 util/error.c: Print backtrace on error
Add a backtrace_on_error meson feature (enabled with --enable-backtrace-on-error) that compiles system binaries with -rdynamic option and prints a function backtrace on error to stderr.
Example output by adding an unconditional error_setg on error_abort in hw/arm/boot.c:
./qemu-system-aarch64(+0x13b4a2c) [0x55d015406a2c] ./qemu-system-aarch64(+0x13b4abd) [0x55d015406abd] ./qemu-system-aarch64(+0x13b4d49) [0x55d015406d49] ./qemu-system-aarch64(error_setg_internal+0xe7) [0x55d015406f62] ./qemu-system-aarch64(arm_load_dtb+0xbf) [0x55d014d7686f] ./qemu-system-aarch64(+0xd2f1d8) [0x55d014d811d8] ./qemu-system-aarch64(notifier_list_notify+0x44) [0x55d01540a282] ./qemu-system-aarch64(qdev_machine_creation_done+0xa0) [0x55d01476ae17] ./qemu-system-aarch64(+0xaa691e) [0x55d014af891e] ./qemu-system-aarch64(qmp_x_exit_preconfig+0x72) [0x55d014af8a5d] ./qemu-system-aarch64(qemu_init+0x2a89) [0x55d014afb657] ./qemu-system-aarch64(main+0x2f) [0x55d01521e836] /lib/x86_64-linux-gnu/libc.so.6(+0x29ca8) [0x7f3033d67ca8] /lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0x85) [0x7f3033d67d65] ./qemu-system-aarch64(_start+0x21) [0x55d0146814f1]
Unexpected error in arm_load_dtb() at ../hw/arm/boot.c:529:
For qemu-system-aarch64, this adds an additional 2MB to the binary size so this is not enabled by default. I also only tested this on Linux, it might not be portable.
--- meson.build | 3 +++ meson_options.txt | 2 ++ scripts/meson-buildoptions.sh | 4 ++++ util/error.c | 17 +++++++++++++++++ 4 files changed, 26 insertions(+)
meson.build +3 -0
@@ -2649,6 +2649,7 @@ config_host_data.set('CONFIG_DEBUG_STACK_USAGE', get_option('debug_stack_usage') config_host_data.set('CONFIG_DEBUG_TCG', get_option('debug_tcg')) config_host_data.set('CONFIG_DEBUG_REMAP', get_option('debug_remap')) config_host_data.set('CONFIG_QOM_CAST_DEBUG', get_option('qom_cast_debug')) +config_host_data.set('CONFIG_BACKTRACE', get_option('backtrace_on_error')) config_host_data.set('CONFIG_REPLICATION', get_option('replication').allowed()) config_host_data.set('CONFIG_FSFREEZE', qga_fsfreeze) config_host_data.set('CONFIG_FSTRIM', qga_fstrim)
@@ -4454,6 +4455,7 @@ foreach target : target_dirs endif emulator = executable(exe_name, exe['sources'], + export_dynamic: get_option('backtrace_on_error'), install: true, c_args: c_args, dependencies: arch_deps + exe['dependencies'],
@@ -4714,6 +4716,7 @@ if 'simple' in get_option('trace_backends') summary_info += {'Trace output file': get_option('trace_file') + '-<pid>'} endif summary_info += {'QOM debugging': get_option('qom_cast_debug')} +summary_info += {'Backtrace support': get_option('backtrace_on_error')} summary_info += {'Relocatable install': get_option('relocatable')} summary_info += {'vhost-kernel support': have_vhost_kernel} summary_info += {'vhost-net support': have_vhost_net}
meson_options.txt +2 -0
@@ -368,6 +368,8 @@ option('debug_stack_usage', type: 'boolean', value: false, description: 'measure coroutine stack usage') option('qom_cast_debug', type: 'boolean', value: true, description: 'cast debugging support') +option('backtrace_on_error', type: 'boolean', value: false, + description: 'print backtrace on error') option('slirp_smbd', type : 'feature', value : 'auto', description: 'use smbd (at path --smbd=*) in slirp networking')
scripts/meson-buildoptions.sh +4 -0
@@ -22,6 +22,8 @@ meson_options_help() { printf "%s\n" ' --docdir=VALUE Base directory for documentation installation' printf "%s\n" ' (can be empty) [share/doc]' printf "%s\n" ' --enable-asan enable address sanitizer' + printf "%s\n" ' --enable-backtrace-on-error' + printf "%s\n" ' print backtrace on error' printf "%s\n" ' --enable-block-drv-whitelist-in-tools' printf "%s\n" ' use block whitelist also in tools instead of only' printf "%s\n" ' QEMU'
@@ -251,6 +253,8 @@ _meson_option_parse() { --disable-gcov) printf "%s" -Db_coverage=false ;; --enable-lto) printf "%s" -Db_lto=true ;; --disable-lto) printf "%s" -Db_lto=false ;; + --enable-backtrace-on-error) printf "%s" -Dbacktrace_on_error=true ;; + --disable-backtrace-on-error) printf "%s" -Dbacktrace_on_error=false ;; --bindir=*) quote_sh "-Dbindir=$2" ;; --enable-blkio) printf "%s" -Dblkio=enabled ;; --disable-blkio) printf "%s" -Dblkio=disabled ;;
util/error.c +17 -0
@@ -17,13 +17,28 @@#include "qemu/error-report.h" #include "qapi/error-internal.h" +#ifdef CONFIG_BACKTRACE +#include <execinfo.h> +#endif /* CONFIG_BACKTRACE */ +Error *error_abort;Error *error_fatal;Error *error_warn; +static inline void dump_backtrace (void ) +{ +#ifdef CONFIG_BACKTRACE +void *buffer[255 ] = {0 }; +const int calls = +backtrace (buffer,sizeof (buffer) /sizeof (void *)); +backtrace_symbols_fd (buffer, calls,2 ); +#endif /* CONFIG_BACKTRACE */ +} +static void error_handle (Error **errp,Error *err) {if (errp == &error_abort) { +dump_backtrace ();if (err->func) {fprintf (stderr,"Unexpected error in %s() at %.*s:%d:\n" , err->func, err->src_len, err->src, err->line);
@@ -38,10 +53,12 @@ static void error_handle(Error **errp, Error *err)abort (); }if (errp == &error_fatal) { +dump_backtrace ();error_report_err (err);exit (1 ); }if (errp == &error_warn) { +dump_backtrace ();warn_report_err (err); }else if (errp && !*errp) { *errp = err;
---
base-commit: a41280fd5b94c49089f7631c6fa8bb9c308b7962
change-id: 20250805-backtrace-e8e4c2490b46
--
γαῖα πυρί μιχθήτω
Quoted 22 lines · Manos Pitsidianakis
Add a
backtrace_on_errormeson feature (enabled with --enable-backtrace-on-error) that compiles system binaries with -rdynamic option and prints a function backtrace on error to stderr.Example output by adding an unconditional
error_setgonerror_abortinhw/arm/boot.c:./qemu-system-aarch64(+0x13b4a2c) [
0x55d015406a2c] ./qemu-system-aarch64(+0x13b4abd) [0x55d015406abd] ./qemu-system-aarch64(+0x13b4d49) [0x55d015406d49] ./qemu-system-aarch64(error_setg_internal+0xe7) [0x55d015406f62] ./qemu-system-aarch64(arm_load_dtb+0xbf) [0x55d014d7686f] ./qemu-system-aarch64(+0xd2f1d8) [0x55d014d811d8] ./qemu-system-aarch64(notifier_list_notify+0x44) [0x55d01540a282] ./qemu-system-aarch64(qdev_machine_creation_done+0xa0) [0x55d01476ae17] ./qemu-system-aarch64(+0xaa691e) [0x55d014af891e] ./qemu-system-aarch64(qmp_x_exit_preconfig+0x72) [0x55d014af8a5d] ./qemu-system-aarch64(qemu_init+0x2a89) [0x55d014afb657] ./qemu-system-aarch64(main+0x2f) [0x55d01521e836] /lib/x86_64-linux-gnu/libc.so.6(+0x29ca8) [0x7f3033d67ca8] /lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0x85) [0x7f3033d67d65] ./qemu-system-aarch64(_start+0x21) [0x55d0146814f1]
Quoted 1 line
Unexpected error in
arm_load_dtb()at../hw/arm/boot.c:529:
From an end-user POV, IMHO the error messages need to be good enough that such backtraces aren't needed to understand the problem. For developers, GDB can give much better backtraces (file+line numbers, plus parameters plus local variables) in the ideally rare cases that the error message alone has insufficient info. So I'm not really convinced that programs (in general, not just QEMU) should try to create backtraces themselves.
With regards,
Daniel
Signature · 3 lines
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
Quoted 26 lines · Daniel P. Berrangé
Quoted 22 lines · Manos Pitsidianakis
Add a
backtrace_on_errormeson feature (enabled with --enable-backtrace-on-error) that compiles system binaries with -rdynamic option and prints a function backtrace on error to stderr.Example output by adding an unconditional
error_setgonerror_abortinhw/arm/boot.c:./qemu-system-aarch64(+0x13b4a2c) [
0x55d015406a2c] ./qemu-system-aarch64(+0x13b4abd) [0x55d015406abd] ./qemu-system-aarch64(+0x13b4d49) [0x55d015406d49] ./qemu-system-aarch64(error_setg_internal+0xe7) [0x55d015406f62] ./qemu-system-aarch64(arm_load_dtb+0xbf) [0x55d014d7686f] ./qemu-system-aarch64(+0xd2f1d8) [0x55d014d811d8] ./qemu-system-aarch64(notifier_list_notify+0x44) [0x55d01540a282] ./qemu-system-aarch64(qdev_machine_creation_done+0xa0) [0x55d01476ae17] ./qemu-system-aarch64(+0xaa691e) [0x55d014af891e] ./qemu-system-aarch64(qmp_x_exit_preconfig+0x72) [0x55d014af8a5d] ./qemu-system-aarch64(qemu_init+0x2a89) [0x55d014afb657] ./qemu-system-aarch64(main+0x2f) [0x55d01521e836] /lib/x86_64-linux-gnu/libc.so.6(+0x29ca8) [0x7f3033d67ca8] /lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0x85) [0x7f3033d67d65] ./qemu-system-aarch64(_start+0x21) [0x55d0146814f1]Quoted 1 line
Unexpected error in
arm_load_dtb()at../hw/arm/boot.c:529:
Quoted 7 lines
From an end-user POV, IMHO the error messages need to be good enough that such backtraces aren't needed to understand the problem. For developers, GDB can give much better backtraces (file+line numbers, plus parameters plus local variables) in the ideally rare cases that the error message alone has insufficient info. So I'm not really convinced that programs (in general, not just QEMU) should try to create backtraces themselves.
Hi Daniel,
I don't think there's value in replacing gdb debugging with this, I agree. I think it has value for "fire and forget" uses, when errors happen unexpectedly and are hard to replicate and you only end up with log entries and no easy way to debug it.
Signature · 2 lines
Manos Pitsidianakis Emulation and Virtualization Engineer at Linaro Ltd
Quoted 35 lines · Manos Pitsidianakis
Quoted 26 lines · Daniel P. Berrangé
Quoted 22 lines · Manos Pitsidianakis
Add a
backtrace_on_errormeson feature (enabled with --enable-backtrace-on-error) that compiles system binaries with -rdynamic option and prints a function backtrace on error to stderr.Example output by adding an unconditional
error_setgonerror_abortinhw/arm/boot.c:./qemu-system-aarch64(+0x13b4a2c) [
0x55d015406a2c] ./qemu-system-aarch64(+0x13b4abd) [0x55d015406abd] ./qemu-system-aarch64(+0x13b4d49) [0x55d015406d49] ./qemu-system-aarch64(error_setg_internal+0xe7) [0x55d015406f62] ./qemu-system-aarch64(arm_load_dtb+0xbf) [0x55d014d7686f] ./qemu-system-aarch64(+0xd2f1d8) [0x55d014d811d8] ./qemu-system-aarch64(notifier_list_notify+0x44) [0x55d01540a282] ./qemu-system-aarch64(qdev_machine_creation_done+0xa0) [0x55d01476ae17] ./qemu-system-aarch64(+0xaa691e) [0x55d014af891e] ./qemu-system-aarch64(qmp_x_exit_preconfig+0x72) [0x55d014af8a5d] ./qemu-system-aarch64(qemu_init+0x2a89) [0x55d014afb657] ./qemu-system-aarch64(main+0x2f) [0x55d01521e836] /lib/x86_64-linux-gnu/libc.so.6(+0x29ca8) [0x7f3033d67ca8] /lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0x85) [0x7f3033d67d65] ./qemu-system-aarch64(_start+0x21) [0x55d0146814f1]Quoted 1 line
Unexpected error in
arm_load_dtb()at../hw/arm/boot.c:529:Quoted 7 lines
From an end-user POV, IMHO the error messages need to be good enough that such backtraces aren't needed to understand the problem. For developers, GDB can give much better backtraces (file+line numbers, plus parameters plus local variables) in the ideally rare cases that the error message alone has insufficient info. So I'm not really convinced that programs (in general, not just QEMU) should try to create backtraces themselves.
Quoted 4 lines
I don't think there's value in replacing gdb debugging with this, I agree. I think it has value for "fire and forget" uses, when errors happen unexpectedly and are hard to replicate and you only end up with log entries and no easy way to debug it.
If the log entry with the error message is useless for devs, then it is even worse for end users... who will be copying that message into bug reports anyway. This patch doesn't feel like something we could enable in formal builds in the distro, so we still need better error reporting without it, such that user bug reports are actionable.
Was there a specific place where you found things hard to debug from the error message alone ? I'm sure we have plenty of examples of errors that can be improved, but wondering if there are some general patterns we're doing badly that would be a good win to improve ?
With regards,
Daniel
Signature · 3 lines
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
Quoted 48 lines · Daniel P. Berrangé
Quoted 35 lines · Manos Pitsidianakis
Quoted 26 lines · Daniel P. Berrangé
Quoted 22 lines · Manos Pitsidianakis
Add a
backtrace_on_errormeson feature (enabled with --enable-backtrace-on-error) that compiles system binaries with -rdynamic option and prints a function backtrace on error to stderr.Example output by adding an unconditional
error_setgonerror_abortinhw/arm/boot.c:./qemu-system-aarch64(+0x13b4a2c) [
0x55d015406a2c] ./qemu-system-aarch64(+0x13b4abd) [0x55d015406abd] ./qemu-system-aarch64(+0x13b4d49) [0x55d015406d49] ./qemu-system-aarch64(error_setg_internal+0xe7) [0x55d015406f62] ./qemu-system-aarch64(arm_load_dtb+0xbf) [0x55d014d7686f] ./qemu-system-aarch64(+0xd2f1d8) [0x55d014d811d8] ./qemu-system-aarch64(notifier_list_notify+0x44) [0x55d01540a282] ./qemu-system-aarch64(qdev_machine_creation_done+0xa0) [0x55d01476ae17] ./qemu-system-aarch64(+0xaa691e) [0x55d014af891e] ./qemu-system-aarch64(qmp_x_exit_preconfig+0x72) [0x55d014af8a5d] ./qemu-system-aarch64(qemu_init+0x2a89) [0x55d014afb657] ./qemu-system-aarch64(main+0x2f) [0x55d01521e836] /lib/x86_64-linux-gnu/libc.so.6(+0x29ca8) [0x7f3033d67ca8] /lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0x85) [0x7f3033d67d65] ./qemu-system-aarch64(_start+0x21) [0x55d0146814f1]Quoted 1 line
Unexpected error in
arm_load_dtb()at../hw/arm/boot.c:529:Quoted 7 lines
From an end-user POV, IMHO the error messages need to be good enough that such backtraces aren't needed to understand the problem. For developers, GDB can give much better backtraces (file+line numbers, plus parameters plus local variables) in the ideally rare cases that the error message alone has insufficient info. So I'm not really convinced that programs (in general, not just QEMU) should try to create backtraces themselves.
Quoted 4 lines
I don't think there's value in replacing gdb debugging with this, I agree. I think it has value for "fire and forget" uses, when errors happen unexpectedly and are hard to replicate and you only end up with log entries and no easy way to debug it.
If the log entry with the error message is useless for devs, then it is even worse for end users... who will be copying that message into bug reports anyway. This patch doesn't feel like something we could enable in formal builds in the distro, so we still need better error reporting without it, such that user bug reports are actionable.
Quoted 5 lines
Was there a specific place where you found things hard to debug from the error message alone ? I'm sure we have plenty of examples of errors that can be improved, but wondering if there are some general patterns we're doing badly that would be a good win to improve ?
Some months ago I was debugging a MemoryRegion use-after-free and used this code to figure out that the free was called from RCU context instead of the main thread.
For problems where the error can happen from multiple contexts and places in the code-base, a backtrace can provide additional insight that might be helpful in a few cases. Again, the intented usecase is not developers with gdb that can reproduce a bug. People ask on IRC about bugs they have that happen rarely over the timespan of a few months and they only have logs to go with. Considering that this feature can be off by default (as it is in this RFC) I don't think there's potential for distro end users to be confused.
Thanks,
Signature · 2 lines
Manos Pitsidianakis Emulation and Virtualization Engineer at Linaro Ltd
Quoted 55 lines · Manos Pitsidianakis
Quoted 48 lines · Daniel P. Berrangé
Quoted 35 lines · Manos Pitsidianakis
Quoted 26 lines · Daniel P. Berrangé
Quoted 22 lines · Manos Pitsidianakis
Add a
backtrace_on_errormeson feature (enabled with --enable-backtrace-on-error) that compiles system binaries with -rdynamic option and prints a function backtrace on error to stderr.Example output by adding an unconditional
error_setgonerror_abortinhw/arm/boot.c:./qemu-system-aarch64(+0x13b4a2c) [
0x55d015406a2c] ./qemu-system-aarch64(+0x13b4abd) [0x55d015406abd] ./qemu-system-aarch64(+0x13b4d49) [0x55d015406d49] ./qemu-system-aarch64(error_setg_internal+0xe7) [0x55d015406f62] ./qemu-system-aarch64(arm_load_dtb+0xbf) [0x55d014d7686f] ./qemu-system-aarch64(+0xd2f1d8) [0x55d014d811d8] ./qemu-system-aarch64(notifier_list_notify+0x44) [0x55d01540a282] ./qemu-system-aarch64(qdev_machine_creation_done+0xa0) [0x55d01476ae17] ./qemu-system-aarch64(+0xaa691e) [0x55d014af891e] ./qemu-system-aarch64(qmp_x_exit_preconfig+0x72) [0x55d014af8a5d] ./qemu-system-aarch64(qemu_init+0x2a89) [0x55d014afb657] ./qemu-system-aarch64(main+0x2f) [0x55d01521e836] /lib/x86_64-linux-gnu/libc.so.6(+0x29ca8) [0x7f3033d67ca8] /lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0x85) [0x7f3033d67d65] ./qemu-system-aarch64(_start+0x21) [0x55d0146814f1]Quoted 1 line
Unexpected error in
arm_load_dtb()at../hw/arm/boot.c:529:Quoted 7 lines
From an end-user POV, IMHO the error messages need to be good enough that such backtraces aren't needed to understand the problem. For developers, GDB can give much better backtraces (file+line numbers, plus parameters plus local variables) in the ideally rare cases that the error message alone has insufficient info. So I'm not really convinced that programs (in general, not just QEMU) should try to create backtraces themselves.
Quoted 4 lines
I don't think there's value in replacing gdb debugging with this, I agree. I think it has value for "fire and forget" uses, when errors happen unexpectedly and are hard to replicate and you only end up with log entries and no easy way to debug it.
If the log entry with the error message is useless for devs, then it is even worse for end users... who will be copying that message into bug reports anyway. This patch doesn't feel like something we could enable in formal builds in the distro, so we still need better error reporting without it, such that user bug reports are actionable.
Quoted 5 lines
Was there a specific place where you found things hard to debug from the error message alone ? I'm sure we have plenty of examples of errors that can be improved, but wondering if there are some general patterns we're doing badly that would be a good win to improve ?
Quoted 3 lines
Some months ago I was debugging a MemoryRegion use-after-free and used this code to figure out that the free was called from RCU context instead of the main thread.
We give useful names to many (but not neccessarily all) threads that we spawn. Perhaps we should call pthread_getname_np() to fetch the current thread name, and used that as a prefix on the error message we print out, as a bit of extra context ?
Obviously not as much info as a full stack trace, but that is something we could likely enable unconditionally without any overheads to worry about, so a likely incremental wni.
With regards,
Daniel
Signature · 3 lines
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
Quoted 56 lines · Daniel P. Berrangé
Quoted 55 lines · Manos Pitsidianakis
Quoted 53 lines · Daniel P. Berrangé
Quoted 35 lines · Manos Pitsidianakis
Quoted 26 lines · Daniel P. Berrangé
Quoted 22 lines · Manos Pitsidianakis
Add a
backtrace_on_errormeson feature (enabled with --enable-backtrace-on-error) that compiles system binaries with -rdynamic option and prints a function backtrace on error to stderr.Example output by adding an unconditional
error_setgonerror_abortinhw/arm/boot.c:./qemu-system-aarch64(+0x13b4a2c) [
0x55d015406a2c] ./qemu-system-aarch64(+0x13b4abd) [0x55d015406abd] ./qemu-system-aarch64(+0x13b4d49) [0x55d015406d49] ./qemu-system-aarch64(error_setg_internal+0xe7) [0x55d015406f62] ./qemu-system-aarch64(arm_load_dtb+0xbf) [0x55d014d7686f] ./qemu-system-aarch64(+0xd2f1d8) [0x55d014d811d8] ./qemu-system-aarch64(notifier_list_notify+0x44) [0x55d01540a282] ./qemu-system-aarch64(qdev_machine_creation_done+0xa0) [0x55d01476ae17] ./qemu-system-aarch64(+0xaa691e) [0x55d014af891e] ./qemu-system-aarch64(qmp_x_exit_preconfig+0x72) [0x55d014af8a5d] ./qemu-system-aarch64(qemu_init+0x2a89) [0x55d014afb657] ./qemu-system-aarch64(main+0x2f) [0x55d01521e836] /lib/x86_64-linux-gnu/libc.so.6(+0x29ca8) [0x7f3033d67ca8] /lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0x85) [0x7f3033d67d65] ./qemu-system-aarch64(_start+0x21) [0x55d0146814f1]Quoted 1 line
Unexpected error in
arm_load_dtb()at../hw/arm/boot.c:529:Quoted 7 lines
From an end-user POV, IMHO the error messages need to be good enough that such backtraces aren't needed to understand the problem. For developers, GDB can give much better backtraces (file+line numbers, plus parameters plus local variables) in the ideally rare cases that the error message alone has insufficient info. So I'm not really convinced that programs (in general, not just QEMU) should try to create backtraces themselves.
Quoted 4 lines
I don't think there's value in replacing gdb debugging with this, I agree. I think it has value for "fire and forget" uses, when errors happen unexpectedly and are hard to replicate and you only end up with log entries and no easy way to debug it.
If the log entry with the error message is useless for devs, then it is even worse for end users... who will be copying that message into bug reports anyway. This patch doesn't feel like something we could enable in formal builds in the distro, so we still need better error reporting without it, such that user bug reports are actionable.
Was there a specific place where you found things hard to debug from the error message alone ? I'm sure we have plenty of examples of errors that can be improved, but wondering if there are some general patterns we're doing badly that would be a good win to improve ?
Quoted 8 lines
Quoted 3 lines
Some months ago I was debugging a MemoryRegion use-after-free and used this code to figure out that the free was called from RCU context instead of the main thread.
We give useful names to many (but not neccessarily all) threads that we spawn. Perhaps we should call
pthread_getname_np()to fetch the current thread name, and used that as a prefix on the error message we print out, as a bit of extra context ?
Do we always have sensible names for threads or only if we enable the option?
Quoted 3 lines
Obviously not as much info as a full stack trace, but that is something we could likely enable unconditionally without any overheads to worry about, so a likely incremental wni.
The place where it comes in useful is when we get bug reports from users who have crashed QEMU in a embedded docker container and can't give us a reasonable reproducer. If we can encourage such users to enable this option (or maybe make it part of --enable-debug-info) then we could get a slightly more useful backtrace for those bugs.
I agree most sane configurations (i.e. a distro) would just attach gdb and use whatever symbol resolution the distro provides.
Quoted 3 lines
With regards,
Daniel
Signature · 2 lines
Alex Bennée Virtualisation Tech Lead @ Linaro
Quoted 16 lines · Alex Bennée
Quoted 14 lines · Daniel P. Berrangé
Quoted 12 lines · Manos Pitsidianakis
Quoted 7 lines · Daniel P. Berrangé
Was there a specific place where you found things hard to debug from the error message alone ? I'm sure we have plenty of examples of errors that can be improved, but wondering if there are some general patterns we're doing badly that would be a good win to improve ?
Some months ago I was debugging a MemoryRegion use-after-free and used this code to figure out that the free was called from RCU context instead of the main thread.
Quoted 7 lines
Quoted 4 lines
We give useful names to many (but not neccessarily all) threads that we spawn. Perhaps we should call
pthread_getname_np()to fetch the current thread name, and used that as a prefix on the error message we print out, as a bit of extra context ?Do we always have sensible names for threads or only if we enable the option?
I was surprised to discover we don't name threads by default, only if we add '-name debug-threads=yes'. I'm struggling to understand why we would ever want thread naming disabled, if an OS supports it ?
I'm inclined to deprecate 'debug-threads' and always set the names when available.
Quoted 9 lines
Quoted 3 lines
Obviously not as much info as a full stack trace, but that is something we could likely enable unconditionally without any overheads to worry about, so a likely incremental wni.
The place where it comes in useful is when we get bug reports from users who have crashed QEMU in a embedded docker container and can't give us a reasonable reproducer. If we can encourage such users to enable this option (or maybe make it part of --enable-debug-info) then we could get a slightly more useful backtrace for those bugs.
The challenge is whether this build option would be enabled widely enough to make a significant difference ?
I don't think we could really enable this in any distro builds, as this is way too noisy to have turned on unconditionally at build time for all users. Most containers are going to be consuming distro builds, with relatively few building custom QEMU themselves IME. We might have better luck if this was a runtime option to the -msg arg.
With regards,
Daniel
Signature · 3 lines
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
Quoted 13 lines · Daniel P. Berrangé
Quoted 12 lines · Alex Bennée
Quoted 10 lines · Daniel P. Berrangé
Quoted 9 lines · Manos Pitsidianakis
Quoted 7 lines · Daniel P. Berrangé
Was there a specific place where you found things hard to debug from the error message alone ? I'm sure we have plenty of examples of errors that can be improved, but wondering if there are some general patterns we're doing badly that would be a good win to improve ?
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 3 lines
Some months ago I was debugging a MemoryRegion use-after-free and used this code to figure out that the free was called from RCU context instead of the main thread.
I ran into something similar recently [1], and it was a pain to reproduce it. Luckily, I caught it using rr and could debug it, but it would have been much easier to just get a backtrace of the crash.
In this case, it was a segmentation fault, which is not covered by current patch. Which brings me the thought I share at the end of this email.
[1] https://lore.kernel.org/qemu-devel/[email protected]/T/#
Quoted 19 lines
Quoted 7 lines
Quoted 4 lines
We give useful names to many (but not neccessarily all) threads that we spawn. Perhaps we should call
pthread_getname_np()to fetch the current thread name, and used that as a prefix on the error message we print out, as a bit of extra context ?Do we always have sensible names for threads or only if we enable the option?
I was surprised to discover we don't name threads by default, only if we add '-name debug-threads=yes'. I'm struggling to understand why we would ever want thread naming disabled, if an OS supports it ?
I'm inclined to deprecate 'debug-threads' and always set the names when available.
Quoted 4 lines
Quoted 3 lines
Obviously not as much info as a full stack trace, but that is something we could likely enable unconditionally without any overheads to worry about, so a likely incremental wni.
Quoted 9 lines
Quoted 5 lines
The place where it comes in useful is when we get bug reports from users who have crashed QEMU in a embedded docker container and can't give us a reasonable reproducer. If we can encourage such users to enable this option (or maybe make it part of --enable-debug-info) then we could get a slightly more useful backtrace for those bugs.
The challenge is whether this build option would be enabled widely enough to make a significant difference ?
For developers working on crashes/bug fix, it's definitely a good addition (could come with --enable-debug for sure). It's something we could enable in CI by default too. Usually, with sanitizers, the reported stacktrace is enough to get a rough idea of what the problem is, without having to use any debugger.
Quoted 7 lines
I don't think we could really enable this in any distro builds, as this is way too noisy to have turned on unconditionally at build time for all users. Most containers are going to be consuming distro builds, with relatively few building custom QEMU themselves IME. We might have better luck if this was a runtime option to the -msg arg.
Regarding the outside world and users, I share Daniel's opinion that it would be too verbose if a backtrace is emitted with every fatal error message.
However, I think it could have *incredible* value if we reported this backtrace when QEMU segfaults, which is always something exceptional. In this case, we could always enable this.
It's not covered by the current patch, maybe it could be a great addition?
Regarding binary size increase due to -rdynamic, I already know some people won't like it, so I'm not sure how we can ensure to have useful symbols in distributed binaries, which is a harder debate than enabling backtraces on segfaults or not.
Quoted 2 lines
With regards,
Daniel
Quoted 19 lines · Pierrick Bouvier
Quoted 13 lines · Daniel P. Berrangé
Quoted 12 lines · Alex Bennée
Quoted 10 lines · Daniel P. Berrangé
Quoted 9 lines · Manos Pitsidianakis
Quoted 7 lines · Daniel P. Berrangé
Was there a specific place where you found things hard to debug from the error message alone ? I'm sure we have plenty of examples of errors that can be improved, but wondering if there are some general patterns we're doing badly that would be a good win to improve ?
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 3 lines
Some months ago I was debugging a MemoryRegion use-after-free and used this code to figure out that the free was called from RCU context instead of the main thread.
Quoted 19 lines
I ran into something similar recently [1], and it was a pain to reproduce it. Luckily, I caught it using rr and could debug it, but it would have been much easier to just get a backtrace of the crash.
In this case, it was a segmentation fault, which is not covered by current patch. Which brings me the thought I share at the end of this email.
[1] https://lore.kernel.org/qemu-devel/[email protected]/T/#
Quoted 13 lines
Quoted 7 lines
Quoted 4 lines
We give useful names to many (but not neccessarily all) threads that we spawn. Perhaps we should call
pthread_getname_np()to fetch the current thread name, and used that as a prefix on the error message we print out, as a bit of extra context ?Do we always have sensible names for threads or only if we enable the option?
I was surprised to discover we don't name threads by default, only if we add '-name debug-threads=yes'. I'm struggling to understand why we would ever want thread naming disabled, if an OS supports it ?
I'm inclined to deprecate 'debug-threads' and always set the names when available.
On POSIX, thread naming uses pthread_setname_np(), which is a GNU extension. Can't see drawbacks; just use it when available.
On Windows, thread naming appears to use a dynamically loaded SetThreadDescription(). Any drawbacks? I'm a Windows ignoramus...
Quoted 23 lines
Quoted 4 lines
Quoted 4 lines
Quoted 3 lines
Obviously not as much info as a full stack trace, but that is something we could likely enable unconditionally without any overheads to worry about, so a likely incremental wni.
Quoted 8 lines
Quoted 5 lines
The place where it comes in useful is when we get bug reports from users who have crashed QEMU in a embedded docker container and can't give us a reasonable reproducer. If we can encourage such users to enable this option (or maybe make it part of --enable-debug-info) then we could get a slightly more useful backtrace for those bugs.
The challenge is whether this build option would be enabled widely enough to make a significant difference ?
For developers working on crashes/bug fix, it's definitely a good addition (could come with --enable-debug for sure). It's something we could enable in CI by default too. Usually, with sanitizers, the reported stacktrace is enough to get a rough idea of what the problem is, without having to use any debugger.
Quoted 7 lines
I don't think we could really enable this in any distro builds, as this is way too noisy to have turned on unconditionally at build time for all users. Most containers are going to be consuming distro builds, with relatively few building custom QEMU themselves IME. We might have better luck if this was a runtime option to the -msg arg.
Quoted 1 line
Regarding the outside world and users, I share Daniel's opinion that it would be too verbose if a backtrace is emitted with every fatal error message.
Yes, that's out of the question. We can debate backtrace on internal errors, such as hitting &error_abort, or more generally abort(). Need to demonstrate it adds value to simply dumping core, which we get for free.
Quoted 1 line
However, I think it could have *incredible* value if we reported this backtrace when QEMU segfaults, which is always something exceptional.
This would be a best effort. The program is already out of order, and printing may or may not work. Avoiding printf() and memory allocation would improve the odds.
Quoted 4 lines
In this case, we could always enable this.
It's not covered by the current patch, maybe it could be a great addition?Regarding binary size increase due to -rdynamic, I already know some people won't like it, so I'm not sure how we can ensure to have useful symbols in distributed binaries, which is a harder debate than enabling backtraces on segfaults or not.
1. Core dumps may take disk space! Let's disable them.
2. My programs crash! I need to know why.
3. I know! Let's make all the program bigger!
SCNR ;)
Quoted 77 lines · Markus Armbruster
Quoted 19 lines · Pierrick Bouvier
Quoted 13 lines · Daniel P. Berrangé
Quoted 12 lines · Alex Bennée
Quoted 10 lines · Daniel P. Berrangé
Quoted 9 lines · Manos Pitsidianakis
Quoted 7 lines · Daniel P. Berrangé
Was there a specific place where you found things hard to debug from the error message alone ? I'm sure we have plenty of examples of errors that can be improved, but wondering if there are some general patterns we're doing badly that would be a good win to improve ?
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 3 lines
Some months ago I was debugging a MemoryRegion use-after-free and used this code to figure out that the free was called from RCU context instead of the main thread.
Quoted 19 lines
I ran into something similar recently [1], and it was a pain to reproduce it. Luckily, I caught it using rr and could debug it, but it would have been much easier to just get a backtrace of the crash.
In this case, it was a segmentation fault, which is not covered by current patch. Which brings me the thought I share at the end of this email.
[1] https://lore.kernel.org/qemu-devel/[email protected]/T/#
Quoted 13 lines
Quoted 7 lines
Quoted 4 lines
We give useful names to many (but not neccessarily all) threads that we spawn. Perhaps we should call
pthread_getname_np()to fetch the current thread name, and used that as a prefix on the error message we print out, as a bit of extra context ?Do we always have sensible names for threads or only if we enable the option?
I was surprised to discover we don't name threads by default, only if we add '-name debug-threads=yes'. I'm struggling to understand why we would ever want thread naming disabled, if an OS supports it ?
I'm inclined to deprecate 'debug-threads' and always set the names when available.On POSIX, thread naming uses
pthread_setname_np(), which is a GNU extension. Can't see drawbacks; just use it when available.On Windows, thread naming appears to use a dynamically loaded
SetThreadDescription(). Any drawbacks? I'm a Windows ignoramus...Quoted 23 lines
Quoted 4 lines
Quoted 4 lines
Quoted 3 lines
Obviously not as much info as a full stack trace, but that is something we could likely enable unconditionally without any overheads to worry about, so a likely incremental wni.
Quoted 8 lines
Quoted 5 lines
The place where it comes in useful is when we get bug reports from users who have crashed QEMU in a embedded docker container and can't give us a reasonable reproducer. If we can encourage such users to enable this option (or maybe make it part of --enable-debug-info) then we could get a slightly more useful backtrace for those bugs.
The challenge is whether this build option would be enabled widely enough to make a significant difference ?
For developers working on crashes/bug fix, it's definitely a good addition (could come with --enable-debug for sure). It's something we could enable in CI by default too. Usually, with sanitizers, the reported stacktrace is enough to get a rough idea of what the problem is, without having to use any debugger.
Quoted 7 lines
I don't think we could really enable this in any distro builds, as this is way too noisy to have turned on unconditionally at build time for all users. Most containers are going to be consuming distro builds, with relatively few building custom QEMU themselves IME. We might have better luck if this was a runtime option to the -msg arg.
Quoted 1 line
Regarding the outside world and users, I share Daniel's opinion that it would be too verbose if a backtrace is emitted with every fatal error message.
Yes, that's out of the question. We can debate backtrace on internal errors, such as hitting
&error_abort, or more generallyabort(). Need to demonstrate it adds value to simply dumping core, which we get for free.
Quoted 5 lines
Quoted 1 line
However, I think it could have *incredible* value if we reported this backtrace when QEMU segfaults, which is always something exceptional.
This would be a best effort. The program is already out of order, and printing may or may not work. Avoiding
printf()and memory allocation would improve the odds.
The risk of doing this on segvs in particular is that the act of generating the stack trace corrupts further state, which then makes debugging the original problem harder.
Quoted 5 lines
Quoted 4 lines
In this case, we could always enable this.
It's not covered by the current patch, maybe it could be a great addition?Regarding binary size increase due to -rdynamic, I already know some people won't like it, so I'm not sure how we can ensure to have useful symbols in distributed binaries, which is a harder debate than enabling backtraces on segfaults or not.
Quoted 7 lines
1. Core dumps may take disk space! Let's disable them.
2. My programs crash! I need to know why.
3. I know! Let's make all the program bigger!
SCNR ;)
FWIW, in systemd enabled distros, core dump stack traces can be captured for everything without needing to store the full dumps. It will capture this even from stuff running inside containers. Fedora at least has this enabled out of the box, not sure about defaults of other systemd based distros.
Also if you are capturing full core dumps, QEMU can (and generally should) be told to tell the kernel to omit guest RAM from dumps with
-machine dump-guest-core=off
which maps to madvise(MADV_DONTDUMP).
So disk space concerns shouldn't be a reason for loosing important debugging information from QEMU crashes in many cases
eg this example
$ qemu-system-x86_64 &
$ killall -SEGV qemu-system-x86_64
$ coredumpctl | tail -1
Mon 2025-08-18 17:35:45 BST 174809 1000 1000 SIGSEGV present /usr/bin/qemu-system-x86_64 4.1M
$ coredumpctl info 174809
PID: 174809 (qemu-system-x86)
UID: 1000 (berrange)
GID: 1000 (berrange)
Signal: 11 (SEGV)
Timestamp: Mon 2025-08-18 17:35:44 BST (2min 5s ago)
Command Line: qemu-system-x86_64
Executable: /usr/bin/qemu-system-x86_64
Control Group: /user.slice/user-1000.slice/[email protected]/user.slice/libpod-f697dc69d8bf78044d0a3964d6e9b7cc4644a66f07045796bfe5320780ffb0f0.scope/co>
Unit: [email protected]
User Unit: libpod-f697dc69d8bf78044d0a3964d6e9b7cc4644a66f07045796bfe5320780ffb0f0.scope
Slice: user-1000.slice
Owner UID: 1000 (berrange)
Boot ID: dab4d69ed89d444ba265e1b3a2ba3ccd
Machine ID: 6fd7abbbb7b447e3968dd84ca07ab101
Hostname: toolbx
Storage: /var/lib/systemd/coredump/core.qemu-system-x86.1000.dab4d69ed89d444ba265e1b3a2ba3ccd.174809.1755534944000000.zst (present)
Size on Disk: 4.1M
Message: Process 174809 (qemu-system-x86) of user 1000 dumped core. Module /usr/lib64/libfdt.so.1.7.2 without build-id.
Module /usr/lib64/libfdt.so.1.7.2
Module /usr/lib64/libcapstone.so.5 without build-id.
Module /usr/lib64/libcapstone.so.5
Module libltdl.so.7 from rpm libtool-2.5.4-4.fc42.x86_64
....snip...
Module libpixman-1.so.0 from rpm pixman-0.46.2-1.fc42.x86_64
Module libgnutls.so.30 from rpm gnutls-3.8.10-1.fc42.x86_64
Stack trace of thread 174809:
#0 0x00007f6900045642 n/a (/usr/lib64/libc.so.6 + 0x79642)
#1 0x00007f69000399a4 n/a (/usr/lib64/libc.so.6 + 0x6d9a4)
#2 0x00007f69000b3136 n/a (/usr/lib64/libc.so.6 + 0xe7136)
#3 0x000055f0d2b85e15 n/a (/usr/bin/qemu-system-x86_64 + 0x5eee15)
#4 0x000055f0d2b836e5 n/a (/usr/bin/qemu-system-x86_64 + 0x5ec6e5)
#5 0x000055f0d27d156e n/a (/usr/bin/qemu-system-x86_64 + 0x23a56e)
#6 0x000055f0d2ae6301 n/a (/usr/bin/qemu-system-x86_64 + 0x54f301)
#7 0x00007f68fffcf575 n/a (/usr/lib64/libc.so.6 + 0x3575)
#8 0x00007f68fffcf628 n/a (/usr/lib64/libc.so.6 + 0x3628)
#9 0x000055f0d25d4395 n/a (/usr/bin/qemu-system-x86_64 + 0x3d395)
ELF object binary architecture: AMD x86-64NB, it can't resolve symbols in this example, as my dev env is inside a toolbox container. If running outside a container, the stack trace would have shown all symbols too.
With regards,
Daniel
Signature · 3 lines
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
Quoted 25 lines · Daniel P. Berrangé
Quoted 16 lines · Alex Bennée
Quoted 14 lines · Daniel P. Berrangé
Quoted 12 lines · Manos Pitsidianakis
Quoted 7 lines · Daniel P. Berrangé
Was there a specific place where you found things hard to debug from the error message alone ? I'm sure we have plenty of examples of errors that can be improved, but wondering if there are some general patterns we're doing badly that would be a good win to improve ?
Some months ago I was debugging a MemoryRegion use-after-free and used this code to figure out that the free was called from RCU context instead of the main thread.
Quoted 7 lines
Quoted 4 lines
We give useful names to many (but not neccessarily all) threads that we spawn. Perhaps we should call
pthread_getname_np()to fetch the current thread name, and used that as a prefix on the error message we print out, as a bit of extra context ?Do we always have sensible names for threads or only if we enable the option?
Quoted 6 lines
I was surprised to discover we don't name threads by default, only if we add '-name debug-threads=yes'. I'm struggling to understand why we would ever want thread naming disabled, if an OS supports it ?
I'm inclined to deprecate 'debug-threads' and always set the names when available.
FYI, I'm working on a small series that will enable thread names and IDs to be printed by default with errors, and should post it sometime this week.
With regards,
Daniel
Signature · 3 lines
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
Quoted 35 lines · Manos Pitsidianakis
Quoted 26 lines · Daniel P. Berrangé
Quoted 22 lines · Manos Pitsidianakis
Add a
backtrace_on_errormeson feature (enabled with --enable-backtrace-on-error) that compiles system binaries with -rdynamic option and prints a function backtrace on error to stderr.Example output by adding an unconditional
error_setgonerror_abortinhw/arm/boot.c:./qemu-system-aarch64(+0x13b4a2c) [
0x55d015406a2c] ./qemu-system-aarch64(+0x13b4abd) [0x55d015406abd] ./qemu-system-aarch64(+0x13b4d49) [0x55d015406d49] ./qemu-system-aarch64(error_setg_internal+0xe7) [0x55d015406f62] ./qemu-system-aarch64(arm_load_dtb+0xbf) [0x55d014d7686f] ./qemu-system-aarch64(+0xd2f1d8) [0x55d014d811d8] ./qemu-system-aarch64(notifier_list_notify+0x44) [0x55d01540a282] ./qemu-system-aarch64(qdev_machine_creation_done+0xa0) [0x55d01476ae17] ./qemu-system-aarch64(+0xaa691e) [0x55d014af891e] ./qemu-system-aarch64(qmp_x_exit_preconfig+0x72) [0x55d014af8a5d] ./qemu-system-aarch64(qemu_init+0x2a89) [0x55d014afb657] ./qemu-system-aarch64(main+0x2f) [0x55d01521e836] /lib/x86_64-linux-gnu/libc.so.6(+0x29ca8) [0x7f3033d67ca8] /lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0x85) [0x7f3033d67d65] ./qemu-system-aarch64(_start+0x21) [0x55d0146814f1]Quoted 1 line
Unexpected error in
arm_load_dtb()at../hw/arm/boot.c:529:Quoted 7 lines
From an end-user POV, IMHO the error messages need to be good enough that such backtraces aren't needed to understand the problem. For developers, GDB can give much better backtraces (file+line numbers, plus parameters plus local variables) in the ideally rare cases that the error message alone has insufficient info. So I'm not really convinced that programs (in general, not just QEMU) should try to create backtraces themselves.
Quoted 6 lines
Hi Daniel,
I don't think there's value in replacing gdb debugging with this, I agree. I think it has value for "fire and forget" uses, when errors happen unexpectedly and are hard to replicate and you only end up with log entries and no easy way to debug it.
Enable core dumps. I doubt that's harder than recompiling and redeploying QEMU with backtraces enabled.