Change softlockup trigger limit using a kernel parameter
Subsystems (3)
PATCH 1/1 Change softlockup trigger limit using a kernel parameter
Kernel warns of softlockups if the softlockup thread is not able to run on a CPU for 10s. It is useful to lower the softlockup warning threshold in testing environments to catch potential lockups early. Following patch adds a kernel parameter 'softlockup_lim' to control the softlockup threshold.
Thanks,
Kiran
Control the trigger limit for softlockup warnings. This is useful for debugging softlockups, by lowering the softlockup_lim to identify possible softlockups earlier.
Documentation/kernel-parameters.txt +4 -0
@@ -1781,6 +1781,10 @@ and is between 256 and 4096 characters. snd-ymfpci= [HW,ALSA] + softlockup_lim= + [KNL] Soft lock up tolerance limit in seconds + { 1 - 10 } + sonycd535= [HW,CD] Format: <io>[,<irq>]
kernel/softlockup.c +16 -1
@@ -21,6 +21,7 @@ static DEFINE_PER_CPU(unsigned long, pri static DEFINE_PER_CPU(struct task_struct *, watchdog_task);static int did_panic =0 ; +static int softlockup_lim =10 ;static int softlock_panic (struct notifier_block *this,unsigned long event,void *ptr)
@@ -34,6 +35,20 @@ static struct notifier_block panic_block .notifier_call = softlock_panic, }; +static int __init softlockup_lim_setup (char *str) +{ +int lim =simple_strtol (str,NULL ,0 ); +if (lim >0 && lim <=10 ) { + softlockup_lim = lim; + }else { +printk ("%s: Invalid softlockup_lim parameter. " , __func__); +printk ("Using defaults.\n" ); + } +return 1 ; +} + +__setup ("softlockup_lim=" , softlockup_lim_setup); +/* * Returns seconds, approximately. We don't need nanosecond * resolution, and we don't need to waste time with a big divide when
@@ -97,7 +112,7 @@ void softlockup_tick(void)wake_up_process (per_cpu (watchdog_task, this_cpu));/* Warn about unreasonable 10+ seconds delays: */ -if (now > (touch_timestamp +10 )) { +if (now > (touch_timestamp + softlockup_lim)) {per_cpu (print_timestamp, this_cpu) = touch_timestamp;spin_lock (&print_lock);
Quoted 6 lines · On Mon, 16 Jul 2007 15:26:50 -0700 Ravikiran G Thirumalai <[email protected]>…
Kernel warns of softlockups if the softlockup thread is not able to run on a CPU for 10s. It is useful to lower the softlockup warning threshold in testing environments to catch potential lockups early. Following patch adds a kernel parameter '
softlockup_lim' to control the softlockup threshold.
Why not make it tunable at runtime?
Quoted 4 lines
Control the trigger limit for softlockup warnings. This is useful for debugging softlockups, by lowering the
softlockup_limto identify possible softlockups earlier.
Please check your patches with scripts/checkpatch.pl.
Quoted 11 lines · Andrew Morton
Quoted 6 lines · On Mon, 16 Jul 2007 15:26:50 -0700 Ravikiran G Thirumalai <[email protected]>…
Kernel warns of softlockups if the softlockup thread is not able to run on a CPU for 10s. It is useful to lower the softlockup warning threshold in testing environments to catch potential lockups early. Following patch adds a kernel parameter '
softlockup_lim' to control the softlockup threshold.Why not make it tunable at runtime?
Sure! Like a sysctl?
Here's a patch that does that (On top of Ingo's
softlockup-improve-debug-output.patch)
Quoted 7 lines
Quoted 5 lines
Control the trigger limit for softlockup warnings. This is useful for debugging softlockups, by lowering the
softlockup_limto identify possible softlockups earlier.Please check your patches with
scripts/checkpatch.pl.
Yep will-do.
(checkpatch emitted one warning for the patch below, but that was because of a 'stylo' that already exists in include/linux/sysctl.h -- which probably needs a style change patch by itself)
---
Control the trigger limit for softlockup warnings. This is useful for debugging softlockups, by lowering the softlockup_thresh sysctl, to identify possible softlockups earlier.
Patch also changes the softlockup printk to print the cpu softlockup time.
kernel/softlockup.c +5 -3
@@ -23,6 +23,7 @@ static DEFINE_PER_CPU(unsigned long, pri static DEFINE_PER_CPU(struct task_struct *, watchdog_task);static int did_panic; +int softlockup_thresh =10 ;static int softlock_panic (struct notifier_block *this,unsigned long event,void *ptr)
@@ -101,7 +102,7 @@ void softlockup_tick(void)wake_up_process (per_cpu (watchdog_task, this_cpu));/* Warn about unreasonable 10+ seconds delays: */ -if (now <= (touch_timestamp +10 )) +if (now <= (touch_timestamp + softlockup_thresh))return ; regs =get_irq_regs ();
@@ -109,8 +110,9 @@ void softlockup_tick(void)per_cpu (print_timestamp, this_cpu) = touch_timestamp;spin_lock (&print_lock); -printk (KERN_ERR "BUG: soft lockup detected on CPU#%d! [%s:%d]\n" , - this_cpu, current->comm, current->pid); +printk (KERN_ERR "BUG: soft lockup - CPU#%d stuck for %lus! [%s:%d]\n" , + this_cpu, now - touch_timestamp, + current->comm, current->pid);if (regs)show_regs (regs);else
kernel/sysctl.c +18 -0
@@ -78,6 +78,7 @@ extern int percpu_pagelist_fraction;extern int compat_log;extern int maps_protect;extern int sysctl_stat_interval; +extern int softlockup_thresh;/* this is needed for the proc_dointvec_minmax for [fs_]overflow UID and GID */ static int maxolduid =65535 ;
@@ -206,6 +207,10 @@ static ctl_table root_table[] = { { .ctl_name = 0 } }; +/* Constants for kernel table minimum and maximum */ +static int one =1 ; +static int ten =10 ; +static ctl_table kern_table[] = { { .ctl_name =KERN_PANIC ,
@@ -615,6 +620,19 @@ static ctl_table kern_table[] = { .proc_handler = &proc_dointvec, },#endif +#ifdef CONFIG_DETECT_SOFTLOCKUP + { + .ctl_name = KERN_SOFTLOCKUP_THRESHOLD, + .procname = "softlockup_thresh", + .data = &softlockup_thresh, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = &proc_dointvec_minmax, + .strategy = &sysctl_intvec, + .extra1 = &one, + .extra2 = &ten, + }, +#endif { .ctl_name = 0 } };
include/linux/sysctl.h +1 -0
@@ -165,6 +165,7 @@ enumKERN_MAX_LOCK_DEPTH =74 ,KERN_NMI_WATCHDOG =75 ,/* int: enable/disable nmi watchdog */ KERN_PANIC_ON_NMI =76 ,/* int: whether we will panic on an unrecovered */ + KERN_SOFTLOCKUP_THRESHOLD=77, /* int: softlockup tolerance threshold */ };
Documentation/sysctl/kernel.txt +8 -0
@@ -320,6 +320,14 @@ kernel. This value defaults to SHMMAX. ============================================================== +softlockup_thresh: + +This value can be used to lower the softlockup tolerance +threshold. The default threshold is 10s. If a cpu is locked up +for 10s, the kernel complains. Valid values are 1-10s. + +============================================================== + tainted: Non-zero if the kernel has been tainted. Numeric values, which
Quoted 23 lines · On Wed, 18 Jul 2007 22:41:21 -0700 Ravikiran G Thirumalai <[email protected]>…
Quoted 11 lines · Andrew Morton
Quoted 6 lines · On Mon, 16 Jul 2007 15:26:50 -0700 Ravikiran G Thirumalai <[email protected]>…
Kernel warns of softlockups if the softlockup thread is not able to run on a CPU for 10s. It is useful to lower the softlockup warning threshold in testing environments to catch potential lockups early. Following patch adds a kernel parameter '
softlockup_lim' to control the softlockup threshold.Why not make it tunable at runtime?
Sure! Like a sysctl?
Here's a patch that does that (On top of Ingo's
softlockup-improve-debug-output.patch)...
kernel/sysctl.c extern int percpu_pagelist_fraction; · line 81
Quoted 4 lines
kernel/sysctl.c +1
-0@@ -78,6 +78,7 @@ extern int percpu_pagelist_fraction;extern int compat_log;extern int maps_protect;extern int sysctl_stat_interval; +extern int softlockup_thresh;
Just because sysctl.c does this all over the place doesn't make it right ;) Please, if poss, find a header file for it.
kernel/sysctl.c static ctl_table root_table[] = { · line 212
Quoted 9 lines
kernel/sysctl.c +3
-0/* this is needed for the proc_dointvec_minmax for [fs_]overflow UID and GID */ static int maxolduid =65535 ;@@ -206,6 +207,10 @@ static ctl_table root_table[] = { { .ctl_name = 0 } }; +/* Constants for kernel table minimum and maximum */ +static int one =1 ; +static int ten =10 ;
I'd suggest that these go next to "zero", "two" and "one_hundred". Move 'em all to top-of-file where they should always have been.
Quoted 10 lines
kernel/sysctl.c +3
-0static ctl_table kern_table[] = { { .ctl_name =KERN_PANIC ,@@ -615,6 +620,19 @@ static ctl_table kern_table[] = { .proc_handler = &proc_dointvec, },#endif +#ifdef CONFIG_DETECT_SOFTLOCKUP + { + .ctl_name = KERN_SOFTLOCKUP_THRESHOLD,
kernel/sysctl.c static ctl_table kern_table[] = { · line 635
Quoted 10 lines
kernel/sysctl.c +10
-0+ .procname = "softlockup_thresh", + .data = &softlockup_thresh, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = &proc_dointvec_minmax, + .strategy = &sysctl_intvec, + .extra1 = &one, + .extra2 = &ten, + }, +#endif
argh. There's supposed to be a big comment right here:
/*
* NOTE: do not add new entries to this table unless you have read
* Documentation/sysctl/ctl_unnumbered.txt
*/
I'll fix that up. Please use CTL_UNNUMBERED.
include/linux/sysctl.h enum · line 169
Quoted 11 lines
kernel/sysctl.c +0
-0};Index:
linux-2.6.22/include/linux/sysctl.h
===================================================================include/linux/sysctl.h +1
-0@@ -165,6 +165,7 @@ enumKERN_MAX_LOCK_DEPTH =74 ,KERN_NMI_WATCHDOG =75 ,/* int: enable/disable nmi watchdog */ KERN_PANIC_ON_NMI =76 ,/* int: whether we will panic on an unrecovered */ + KERN_SOFTLOCKUP_THRESHOLD=77, /* int: softlockup tolerance threshold */ };
and zap this
Quoted 5 lines
Index:
linux-2.6.22/Documentation/sysctl/kernel.txt
===================================================================
Documentation/sysctl/kernel.txt kernel. This value defaults to SHMMAX. · line 328
Quoted 9 lines
Documentation/sysctl/kernel.txt +6
-0@@ -320,6 +320,14 @@ kernel. This value defaults to SHMMAX. ============================================================== +softlockup_thresh: + +This value can be used to lower the softlockup tolerance +threshold. The default threshold is 10s. If a cpu is locked up +for 10s, the kernel complains. Valid values are 1-10s. +
neato.
Quoted 8 lines · Andrew Morton
Documentation/sysctl/kernel.txt kernel. This value defaults to SHMMAX. · line 328
Quoted 6 lines
Documentation/sysctl/kernel.txt +6
-0+softlockup_thresh: + +This value can be used to lower the softlockup tolerance +threshold. The default threshold is 10s. If a cpu is locked up +for 10s, the kernel complains. Valid values are 1-10s. +neato.
please make sure this is applied after the softlockup watchdog patches i already did. (in particular the cleanup patch, which this could interact with)
also, i think the valid threshold should be between 1 and 60 seconds i think.
Ingo
Quoted 12 lines · Ingo Molnar
Quoted 8 lines · Andrew Morton
Documentation/sysctl/kernel.txt kernel. This value defaults to SHMMAX. · line 328
Quoted 6 lines
Documentation/sysctl/kernel.txt +6
-0+softlockup_thresh: + +This value can be used to lower the softlockup tolerance +threshold. The default threshold is 10s. If a cpu is locked up +for 10s, the kernel complains. Valid values are 1-10s. +neato.
Quoted 6 lines
please make sure this is applied after the softlockup watchdog patches i already did. (in particular the cleanup patch, which this could interact with)
also, i think the valid threshold should be between 1 and 60 seconds i think.
60 seconds! Is that not a pretty high threshold? The reason for lowering the tolerance threshold from 10s is to catch bugs early in lab environments, but why do we need to raise the tolerance thresh beyond 10s?
Thanks,
Kiran
Quoted 7 lines · Ravikiran G Thirumalai
Quoted 2 lines
also, i think the valid threshold should be between 1 and 60 seconds i think.
60 seconds! Is that not a pretty high threshold? The reason for lowering the tolerance threshold from 10s is to catch bugs early in lab environments, but why do we need to raise the tolerance thresh beyond 10s?
just in case someone sees false positives and wants to turn it off.
Ingo
Quoted 1 line · Ingo Molnar
just in case someone sees false positives and wants to turn it off.
Why not make 0=off?
J
Quoted 4 lines · Jeremy Fitzhardinge
Quoted 1 line · Ingo Molnar
just in case someone sees false positives and wants to turn it off.
Why not make 0=off?
A patch to disable softlockup during boot already went in.
http://git.kernel.org/?p=linux/kernel/git/torvalds/linux-2.6.git;a=commit;h=97842216b8400fe9d1a20468959e2989180f8f79
It uses kernel boot parameter to disable softlockup, not exactly disabling softlockup at run time though. Do we still need 0=off?
Thanks,
Kiran
Quoted 13 lines · Andrew Morton
Quoted 11 lines · On Wed, 18 Jul 2007 22:41:21 -0700 Ravikiran G Thirumalai <[email protected]>…
Quoted 10 lines · Andrew Morton
Quoted 6 lines · On Mon, 16 Jul 2007 15:26:50 -0700 Ravikiran G Thirumalai <[email protected]>…
Kernel warns of softlockups if the softlockup thread is not able to run on a CPU for 10s. It is useful to lower the softlockup warning threshold in testing environments to catch potential lockups early. Following patch adds a kernel parameter '
softlockup_lim' to control the softlockup threshold.
Quoted 4 lines
Quoted 4 lines
Quoted 1 line
Why not make it tunable at runtime?
Sure! Like a sysctl?
Hi Andrew,
Here's another take, incorporating all of your comments.
Thanks,
Kiran
Control the trigger limit for softlockup warnings. This is useful for debugging softlockups, by lowering the softlockup_thresh to identify possible softlockups earlier.
This patch:
1. Adds a sysctl softlockup_thresh with valid values of 1-60s (Higher value to disable false positives)
2. Changes the softlockup printk to print the cpu softlockup time
---
Patch applies on top of Ingo's softlockup patches
Documentation/sysctl/kernel.txt | 8 ++++++++ include/linux/sched.h | 1 + kernel/softlockup.c | 8 +++++--- kernel/sysctl.c | 25 +++++++++++++++++++------ 4 files changed, 33 insertions(+), 9 deletions(-)
kernel/softlockup.c +5 -3
@@ -23,6 +23,7 @@ static DEFINE_PER_CPU(unsigned long, pri static DEFINE_PER_CPU(struct task_struct *, watchdog_task);static int did_panic; +int softlockup_thresh =10 ;static int softlock_panic (struct notifier_block *this,unsigned long event,void *ptr)
@@ -101,7 +102,7 @@ void softlockup_tick(void)wake_up_process (per_cpu (watchdog_task, this_cpu));/* Warn about unreasonable 10+ seconds delays: */ -if (now <= (touch_timestamp +10 )) +if (now <= (touch_timestamp + softlockup_thresh))return ; regs =get_irq_regs ();
@@ -109,8 +110,9 @@ void softlockup_tick(void)per_cpu (print_timestamp, this_cpu) = touch_timestamp;spin_lock (&print_lock); -printk (KERN_ERR "BUG: soft lockup detected on CPU#%d! [%s:%d]\n" , - this_cpu, current->comm, current->pid); +printk (KERN_ERR "BUG: soft lockup - CPU#%d stuck for %lus! [%s:%d]\n" , + this_cpu, now - touch_timestamp, + current->comm, current->pid);if (regs)show_regs (regs);else
kernel/sysctl.c +19 -6
@@ -79,6 +79,12 @@ extern int compat_log;extern int maps_protect;extern int sysctl_stat_interval; +/* Constants used for minimum and maximum */ +static int one =1 ; +static int zero; +static int sixty =60 ; +static int one_hundred =100 ; +/* this is needed for the proc_dointvec_minmax for [fs_]overflow UID and GID */ static int maxolduid =65535 ;static int minolduid;
@@ -615,16 +621,23 @@ static ctl_table kern_table[] = { .proc_handler = &proc_dointvec, },#endif +#ifdef CONFIG_DETECT_SOFTLOCKUP + { + .ctl_name = CTL_UNNUMBERED, + .procname = "softlockup_thresh", + .data = &softlockup_thresh, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = &proc_dointvec_minmax, + .strategy = &sysctl_intvec, + .extra1 = &one, + .extra2 = &sixty, + }, +#endif { .ctl_name = 0 } }; -/* Constants for minimum and maximum testing in vm_table. -We use these as one-element integer vectors. */ -static int zero; -static int one_hundred =100 ; - - static ctl_table vm_table[] = { { .ctl_name = VM_OVERCOMMIT_MEMORY,
Documentation/sysctl/kernel.txt +8 -0
@@ -320,6 +320,14 @@ kernel. This value defaults to SHMMAX. ============================================================== +softlockup_thresh: + +This value can be used to lower the softlockup tolerance +threshold. The default threshold is 10s. If a cpu is locked up +for 10s, the kernel complains. Valid values are 1-60s. + +============================================================== + tainted: Non-zero if the kernel has been tainted. Numeric values, which
include/linux/sched.h +1 -0
@@ -236,6 +236,7 @@ extern void softlockup_tick(void);extern void spawn_softlockup_task (void );extern void touch_softlockup_watchdog (void );extern void touch_all_softlockup_watchdogs (void ); +extern int softlockup_thresh;#else static inline void softlockup_tick (void ) {