panic: Add options to dump system info on panic
Subsystems (2)
v3 cover panic: Add options to dump system info on panic
Kernel panic issues are always painful to debug, partially
because it's not easy to get enough information of the context when panic happens.
And we have ramoops and kdump for that, while this commit tries to provide a easier way to show the system info by adding a cmdline parameter, referring some idea from sysrq handler.
The patches have been used by us and did help on solving some nasty panic cases.
Please help to review, thanks!
- Feng
Changelog:
v3:
- Add one patch to add the "panic_print" into sysctl as suggested by Steven Rostedt
v2:
- change text "dump/DUMP" to "print/PRINT" which is more accurate, suggested by Andrew Morton
- add code to print ftrace buffer
Feng Tang (2):
panic: Add options to print system info when panic happens kernel/sysctl: Add panic_print into sysctl
Documentation/admin-guide/kernel-parameters.txt | 8 +++++++ Documentation/sysctl/kernel.txt | 17 +++++++++++++++ include/linux/kernel.h | 1 + include/uapi/linux/sysctl.h | 1 + kernel/panic.c | 28 +++++++++++++++++++++++++ kernel/sysctl.c | 7 +++++++ kernel/sysctl_binary.c | 1 + 7 files changed, 63 insertions(+)
Signature · 1 line
2.7.4
PATCH v3 1/2 panic: Add options to print system info when panic happens
Kernel panic issues are always painful to debug, partially
because it's not easy to get enough information of the context when panic happens.
And we have ramoops and kdump for that, while this commit tries to provide a easier way to show the system info by adding a cmdline parameter, referring some idea from sysrq handler.
--- Documentation/admin-guide/kernel-parameters.txt | 8 +++++++ kernel/panic.c | 28 +++++++++++++++++++++++++ 2 files changed, 36 insertions(+)
Documentation/admin-guide/kernel-parameters.txt +8 -0
@@ -3081,6 +3081,14 @@ timeout < 0: reboot immediately Format: <timeout> + panic_print= Bitmask for printing system info when panic happens. + User can chose combination of the following bits: + bit 0: print all tasks info + bit 1: print system memory info + bit 2: print timer info + bit 3: print locks info if CONFIG_LOCKDEP is on + bit 4: print ftrace buffer + panic_on_warn panic() instead of WARN(). Useful to cause kdump on a WARN().
kernel/panic.c +28 -0
@@ -45,6 +45,13 @@ int panic_on_warn __read_mostly;int panic_timeout =CONFIG_PANIC_TIMEOUT ;EXPORT_SYMBOL_GPL (panic_timeout); +#define PANIC_PRINT_TASK_INFO 0x00000001 +#define PANIC_PRINT_MEM_INFO 0x00000002 +#define PANIC_PRINT_TIMER_INFO 0x00000004 +#define PANIC_PRINT_LOCK_INFO 0x00000008 +#define PANIC_PRINT_FTRACE_INFO 0x00000010 +static unsigned long panic_print; +ATOMIC_NOTIFIER_HEAD (panic_notifier_list);EXPORT_SYMBOL (panic_notifier_list);
@@ -124,6 +131,24 @@ void nmi_panic(struct pt_regs *regs, const char *msg) }EXPORT_SYMBOL (nmi_panic); +static void panic_print_sys_info (void ) +{ +if (panic_print &PANIC_PRINT_TASK_INFO ) +show_state (); + +if (panic_print &PANIC_PRINT_MEM_INFO ) +show_mem (0 ,NULL ); + +if (panic_print &PANIC_PRINT_TIMER_INFO ) +sysrq_timer_list_show (); + +if (panic_print &PANIC_PRINT_LOCK_INFO ) +debug_show_all_locks (); + +if (panic_print &PANIC_PRINT_FTRACE_INFO ) +ftrace_dump (DUMP_ALL ); +} +/** * panic - halt the system * @fmt: The text string to print
@@ -250,6 +275,8 @@ void panic(const char *fmt, ...)debug_locks_off ();console_flush_on_panic (); +panic_print_sys_info (); +if (!panic_blink) panic_blink = no_blink;
@@ -654,6 +681,7 @@ void refcount_error_report(struct pt_regs *regs, const char *err)#endif core_param (panic, panic_timeout, int,0644 ); +core_param (panic_print, panic_print, ulong,0644 );core_param (pause_on_oops, pause_on_oops, int,0644 );core_param (panic_on_warn, panic_on_warn, int,0644 );core_param (crash_kexec_post_notifiers, crash_kexec_post_notifiers, bool,0644 );
Signature · 1 line
2.7.4
PATCH v3 2/2 kernel/sysctl: Add panic_print into sysctl
In reply to Feng TangSo that we can also runtime chose to print out the needed system info for panic, other than setting the kernel cmdline.
--- Documentation/sysctl/kernel.txt | 17 +++++++++++++++++ include/linux/kernel.h | 1 + include/uapi/linux/sysctl.h | 1 + kernel/panic.c | 2 +- kernel/sysctl.c | 7 +++++++ kernel/sysctl_binary.c | 1 + 6 files changed, 28 insertions(+), 1 deletion(-)
Documentation/sysctl/kernel.txt +17 -0
@@ -60,6 +60,7 @@ show up in /proc/sys/kernel: - panic_on_stackoverflow - panic_on_unrecovered_nmi - panic_on_warn +- panic_print - panic_on_rcu_stall - perf_cpu_time_max_percent - perf_event_paranoid
@@ -654,6 +655,22 @@ a kernel rebuild when attempting to kdump at the location of a WARN(). ============================================================== +panic_print: + +Bitmask for printing system info when panic happens. User can chose +combination of the following bits: + +bit 0: print all tasks info +bit 1: print system memory info +bit 2: print timer info +bit 3: print locks info if CONFIG_LOCKDEP is on +bit 4: print ftrace buffer + +So for example to print tasks and memory info on panic, user can: + echo 3 > /proc/sys/kernel/panic_print + +============================================================== + panic_on_rcu_stall: When set to 1, calls panic() after RCU stall detection messages. This
include/linux/kernel.h +1 -0
@@ -527,6 +527,7 @@ static inline u32 int_sqrt64(u64 x)extern void bust_spinlocks (int yes);extern int oops_in_progress;/* If set, an oops, panic(), BUG() or die() is in progress */ extern int panic_timeout; +extern unsigned long panic_print;extern int panic_on_oops;extern int panic_on_unrecovered_nmi;extern int panic_on_io_nmi;
include/uapi/linux/sysctl.h +1 -0
@@ -153,6 +153,7 @@ enumKERN_NMI_WATCHDOG =75 ,/* int: enable/disable nmi watchdog */ KERN_PANIC_ON_NMI =76 ,/* int: whether we will panic on an unrecovered */ KERN_PANIC_ON_WARN =77 ,/* int: call panic() in WARN() functions */ + KERN_PANIC_PRINT=78, /* ulong: bitmask to print system info on panic */ };
kernel/panic.c +1 -1
@@ -50,7 +50,7 @@ EXPORT_SYMBOL_GPL(panic_timeout);#define PANIC_PRINT_TIMER_INFO 0x00000004#define PANIC_PRINT_LOCK_INFO 0x00000008#define PANIC_PRINT_FTRACE_INFO 0x00000010 -static unsigned long panic_print; +unsigned long panic_print;ATOMIC_NOTIFIER_HEAD (panic_notifier_list);
kernel/sysctl.c +7 -0
@@ -807,6 +807,13 @@ static struct ctl_table kern_table[] = { .mode = 0644, .proc_handler = proc_dointvec, }, + { + .procname = "panic_print", + .data = &panic_print, + .maxlen = sizeof(unsigned long), + .mode = 0644, + .proc_handler = proc_doulongvec_minmax, + }, #if defined CONFIG_PRINTK { .procname = "printk",
kernel/sysctl_binary.c +1 -0
@@ -140,6 +140,7 @@ static const struct bin_table bin_kern_table[] = { { CTL_INT, KERN_MAX_LOCK_DEPTH, "max_lock_depth" }, { CTL_INT, KERN_PANIC_ON_NMI, "panic_on_unrecovered_nmi" }, { CTL_INT, KERN_PANIC_ON_WARN, "panic_on_warn" }, + { CTL_ULONG, KERN_PANIC_PRINT, "panic_print" }, {} };
Signature · 1 line
2.7.4
Quoted 4 lines · On Wed, 28 Nov 2018 17:54:00 +0800 Feng Tang <[email protected]> wrote:
Kernel panic issues are always painful to debug, partiallybecause it's not easy to get enough information of the context when panic happens.
Quoted 9 lines
And we have ramoops and kdump for that, while this commit tries to provide a easier way to show the system info by adding a cmdline parameter, referring some idea from sysrq handler.
The patches have been used by us and did help on solving some nasty panic cases.
Please help to review, thanks!
Looks fine to me.
Although, I still wonder if we should add a comment stating that bit 4 is the same as ftrace_dump_on_oops.
-- Steve