Skip to contentSkip to conversation

Change softlockup trigger limit using a kernel parameter

PATCH In review Ravikiran G Thirumalai 4 participants 10 messages Jul 16, 2007 – Jul 20, 2007

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 Change softlockup trigger limit using a kernel parameter

Kernel warns of softlockups if the softlockup thread is not able to run on a CPU for 10s. It is useful to lower the softlockup warning threshold in testing environments to catch potential lockups early. Following patch adds a kernel parameter 'softlockup_lim' to control the softlockup threshold.

Thanks,
Kiran

Control the trigger limit for softlockup warnings. This is useful for debugging softlockups, by lowering the softlockup_lim to identify possible softlockups earlier.

Documentation/kernel-parameters.txt +4 -0
Index: linux-2.6.22/Documentation/kernel-parameters.txt
===================================================================
@@ -1781,6 +1781,10 @@ and is between 256 and 4096 characters. 
 
 	snd-ymfpci=	[HW,ALSA]
 
+	softlockup_lim=
+			[KNL] Soft lock up tolerance limit in seconds
+			{ 1 - 10 }
+
 	sonycd535=	[HW,CD]
 			Format: <io>[,<irq>]
 
kernel/softlockup.c +16 -1
Index: linux-2.6.22/kernel/softlockup.c
===================================================================
@@ -21,6 +21,7 @@ static DEFINE_PER_CPU(unsigned long, pri
 static DEFINE_PER_CPU(struct task_struct *, watchdog_task);
 
 static int did_panic = 0;
+static int softlockup_lim = 10;
 
 static int
 softlock_panic(struct notifier_block *this, unsigned long event, void *ptr)
@@ -34,6 +35,20 @@ static struct notifier_block panic_block
 	.notifier_call = softlock_panic,
 };
 
+static int __init softlockup_lim_setup(char *str)
+{
+	int lim = simple_strtol(str, NULL, 0);
+	if(lim > 0 && lim <= 10) {
+		softlockup_lim = lim;
+	} else {
+		printk("%s: Invalid softlockup_lim parameter.  ", __func__);
+		printk("Using defaults.\n");
+	}
+	return 1;
+}
+
+__setup("softlockup_lim=", softlockup_lim_setup);
+
 /*
  * Returns seconds, approximately.  We don't need nanosecond
  * resolution, and we don't need to waste time with a big divide when
@@ -97,7 +112,7 @@ void softlockup_tick(void)
 		wake_up_process(per_cpu(watchdog_task, this_cpu));
 
 	/* Warn about unreasonable 10+ seconds delays: */
-	if (now > (touch_timestamp + 10)) {
+	if (now > (touch_timestamp + softlockup_lim)) {
 		per_cpu(print_timestamp, this_cpu) = touch_timestamp;
 
 		spin_lock(&print_lock);
#1 lore raw
Quoted 6 lines · On Mon, 16 Jul 2007 15:26:50 -0700 Ravikiran G Thirumalai <[email protected]>…

Kernel warns of softlockups if the softlockup thread is not able to run on a CPU for 10s. It is useful to lower the softlockup warning threshold in testing environments to catch potential lockups early. Following patch adds a kernel parameter 'softlockup_lim' to control the softlockup threshold.

Why not make it tunable at runtime?

Quoted 4 lines

Control the trigger limit for softlockup warnings. This is useful for debugging softlockups, by lowering the softlockup_lim to identify possible softlockups earlier.

Please check your patches with scripts/checkpatch.pl.

#2 lore raw
Quoted 11 lines · Andrew Morton
Quoted 6 lines · On Mon, 16 Jul 2007 15:26:50 -0700 Ravikiran G Thirumalai <[email protected]>…

Kernel warns of softlockups if the softlockup thread is not able to run on a CPU for 10s. It is useful to lower the softlockup warning threshold in testing environments to catch potential lockups early. Following patch adds a kernel parameter 'softlockup_lim' to control the softlockup threshold.

Why not make it tunable at runtime?

Sure! Like a sysctl?

Here's a patch that does that (On top of Ingo's
softlockup-improve-debug-output.patch)

Quoted 7 lines
Quoted 5 lines

Control the trigger limit for softlockup warnings. This is useful for debugging softlockups, by lowering the softlockup_lim to identify possible softlockups earlier.

Please check your patches with scripts/checkpatch.pl.

Yep will-do.
(checkpatch emitted one warning for the patch below, but that was because of a 'stylo' that already exists in include/linux/sysctl.h -- which probably needs a style change patch by itself)

---

Control the trigger limit for softlockup warnings. This is useful for debugging softlockups, by lowering the softlockup_thresh sysctl, to identify possible softlockups earlier.

Patch also changes the softlockup printk to print the cpu softlockup time.

kernel/softlockup.c +5 -3
Index: linux-2.6.22/kernel/softlockup.c
===================================================================
@@ -23,6 +23,7 @@ static DEFINE_PER_CPU(unsigned long, pri
 static DEFINE_PER_CPU(struct task_struct *, watchdog_task);
 
 static int did_panic;
+int softlockup_thresh = 10;
 
 static int
 softlock_panic(struct notifier_block *this, unsigned long event, void *ptr)
@@ -101,7 +102,7 @@ void softlockup_tick(void)
 		wake_up_process(per_cpu(watchdog_task, this_cpu));
 
 	/* Warn about unreasonable 10+ seconds delays: */
-	if (now <= (touch_timestamp + 10))
+	if (now <= (touch_timestamp + softlockup_thresh))
 		return;
 
 	regs = get_irq_regs();
@@ -109,8 +110,9 @@ void softlockup_tick(void)
 	per_cpu(print_timestamp, this_cpu) = touch_timestamp;
 
 	spin_lock(&print_lock);
-	printk(KERN_ERR "BUG: soft lockup detected on CPU#%d! [%s:%d]\n",
-			this_cpu, current->comm, current->pid);
+	printk(KERN_ERR "BUG: soft lockup - CPU#%d stuck for %lus! [%s:%d]\n",
+			this_cpu, now - touch_timestamp,
+				current->comm, current->pid);
 	if (regs)
 		show_regs(regs);
 	else
kernel/sysctl.c +18 -0
Index: linux-2.6.22/kernel/sysctl.c
===================================================================
@@ -78,6 +78,7 @@ extern int percpu_pagelist_fraction;
 extern int compat_log;
 extern int maps_protect;
 extern int sysctl_stat_interval;
+extern int softlockup_thresh;
 
 /* this is needed for the proc_dointvec_minmax for [fs_]overflow UID and GID */
 static int maxolduid = 65535;
@@ -206,6 +207,10 @@ static ctl_table root_table[] = {
 	{ .ctl_name = 0 }
 };
 
+/* Constants for kernel table minimum and  maximum */
+static int one = 1;
+static int ten = 10;
+
 static ctl_table kern_table[] = {
 	{
 		.ctl_name	= KERN_PANIC,
@@ -615,6 +620,19 @@ static ctl_table kern_table[] = {
 		.proc_handler   = &proc_dointvec,
 	},
 #endif
+#ifdef CONFIG_DETECT_SOFTLOCKUP
+	{
+		.ctl_name	= KERN_SOFTLOCKUP_THRESHOLD,
+		.procname	= "softlockup_thresh",
+		.data		= &softlockup_thresh,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= &proc_dointvec_minmax,
+		.strategy	= &sysctl_intvec,
+		.extra1		= &one,
+		.extra2		= &ten,
+	},
+#endif
 
 	{ .ctl_name = 0 }
 };
include/linux/sysctl.h +1 -0
Index: linux-2.6.22/include/linux/sysctl.h
===================================================================
@@ -165,6 +165,7 @@ enum
 	KERN_MAX_LOCK_DEPTH=74,
 	KERN_NMI_WATCHDOG=75, /* int: enable/disable nmi watchdog */
 	KERN_PANIC_ON_NMI=76, /* int: whether we will panic on an unrecovered */
+	KERN_SOFTLOCKUP_THRESHOLD=77, /* int: softlockup tolerance threshold */
 };
 
 
Documentation/sysctl/kernel.txt +8 -0
Index: linux-2.6.22/Documentation/sysctl/kernel.txt
===================================================================
@@ -320,6 +320,14 @@ kernel.  This value defaults to SHMMAX.
 
 ==============================================================
 
+softlockup_thresh:
+
+This value can be used to lower the softlockup tolerance
+threshold. The default threshold is 10s.  If a cpu is locked up
+for 10s, the kernel complains.  Valid values are 1-10s.
+
+==============================================================
+
 tainted: 
 
 Non-zero if the kernel has been tainted.  Numeric values, which
#3 lore raw
Quoted 23 lines · On Wed, 18 Jul 2007 22:41:21 -0700 Ravikiran G Thirumalai <[email protected]>…
Quoted 11 lines · Andrew Morton
Quoted 6 lines · On Mon, 16 Jul 2007 15:26:50 -0700 Ravikiran G Thirumalai <[email protected]>…

Kernel warns of softlockups if the softlockup thread is not able to run on a CPU for 10s. It is useful to lower the softlockup warning threshold in testing environments to catch potential lockups early. Following patch adds a kernel parameter 'softlockup_lim' to control the softlockup threshold.

Why not make it tunable at runtime?

Sure! Like a sysctl?

Here's a patch that does that (On top of Ingo's
softlockup-improve-debug-output.patch)

...

kernel/sysctl.c extern int percpu_pagelist_fraction; · line 81

Quoted 4 lines
kernel/sysctl.c +1 -0
@@ -78,6 +78,7 @@ extern int percpu_pagelist_fraction;
 extern int compat_log;
 extern int maps_protect;
 extern int sysctl_stat_interval;
+extern int softlockup_thresh;

Just because sysctl.c does this all over the place doesn't make it right ;) Please, if poss, find a header file for it.

kernel/sysctl.c static ctl_table root_table[] = { · line 212

Quoted 9 lines
kernel/sysctl.c +3 -0
 /* this is needed for the proc_dointvec_minmax for [fs_]overflow UID and GID */
 static int maxolduid = 65535;
@@ -206,6 +207,10 @@ static ctl_table root_table[] = {
 	{ .ctl_name = 0 }
 };
 
+/* Constants for kernel table minimum and  maximum */
+static int one = 1;
+static int ten = 10;

I'd suggest that these go next to "zero", "two" and "one_hundred". Move 'em all to top-of-file where they should always have been.

Quoted 10 lines
kernel/sysctl.c +3 -0
 static ctl_table kern_table[] = {
 	{
 		.ctl_name	= KERN_PANIC,
@@ -615,6 +620,19 @@ static ctl_table kern_table[] = {
 		.proc_handler   = &proc_dointvec,
 	},
 #endif
+#ifdef CONFIG_DETECT_SOFTLOCKUP
+	{
+		.ctl_name	= KERN_SOFTLOCKUP_THRESHOLD,

kernel/sysctl.c static ctl_table kern_table[] = { · line 635

Quoted 10 lines
kernel/sysctl.c +10 -0
+		.procname	= "softlockup_thresh",
+		.data		= &softlockup_thresh,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= &proc_dointvec_minmax,
+		.strategy	= &sysctl_intvec,
+		.extra1		= &one,
+		.extra2		= &ten,
+	},
+#endif

argh. There's supposed to be a big comment right here:

/*
* NOTE: do not add new entries to this table unless you have read
* Documentation/sysctl/ctl_unnumbered.txt
*/

I'll fix that up. Please use CTL_UNNUMBERED.

include/linux/sysctl.h enum · line 169

Quoted 11 lines
kernel/sysctl.c +0 -0
 };

Index: linux-2.6.22/include/linux/sysctl.h
===================================================================

include/linux/sysctl.h +1 -0
@@ -165,6 +165,7 @@ enum
 	KERN_MAX_LOCK_DEPTH=74,
 	KERN_NMI_WATCHDOG=75, /* int: enable/disable nmi watchdog */
 	KERN_PANIC_ON_NMI=76, /* int: whether we will panic on an unrecovered */
+	KERN_SOFTLOCKUP_THRESHOLD=77, /* int: softlockup tolerance threshold */
 };

and zap this

Quoted 5 lines

Index: linux-2.6.22/Documentation/sysctl/kernel.txt
===================================================================

Documentation/sysctl/kernel.txt kernel. This value defaults to SHMMAX. · line 328

Quoted 9 lines
Documentation/sysctl/kernel.txt +6 -0
@@ -320,6 +320,14 @@ kernel.  This value defaults to SHMMAX.
 
 ==============================================================
 
+softlockup_thresh:
+
+This value can be used to lower the softlockup tolerance
+threshold. The default threshold is 10s.  If a cpu is locked up
+for 10s, the kernel complains.  Valid values are 1-10s.
+

neato.

#4 lore raw
Quoted 8 lines · Andrew Morton

Documentation/sysctl/kernel.txt kernel. This value defaults to SHMMAX. · line 328

Quoted 6 lines
Documentation/sysctl/kernel.txt +6 -0
+softlockup_thresh:
+
+This value can be used to lower the softlockup tolerance
+threshold. The default threshold is 10s.  If a cpu is locked up
+for 10s, the kernel complains.  Valid values are 1-10s.
+

neato.

please make sure this is applied after the softlockup watchdog patches i already did. (in particular the cleanup patch, which this could interact with)

also, i think the valid threshold should be between 1 and 60 seconds i think.

Ingo

#5 lore raw
Quoted 12 lines · Ingo Molnar
Quoted 8 lines · Andrew Morton

Documentation/sysctl/kernel.txt kernel. This value defaults to SHMMAX. · line 328

Quoted 6 lines
Documentation/sysctl/kernel.txt +6 -0
+softlockup_thresh:
+
+This value can be used to lower the softlockup tolerance
+threshold. The default threshold is 10s.  If a cpu is locked up
+for 10s, the kernel complains.  Valid values are 1-10s.
+

neato.

Quoted 6 lines

please make sure this is applied after the softlockup watchdog patches i already did. (in particular the cleanup patch, which this could interact with)

also, i think the valid threshold should be between 1 and 60 seconds i think.

60 seconds! Is that not a pretty high threshold? The reason for lowering the tolerance threshold from 10s is to catch bugs early in lab environments, but why do we need to raise the tolerance thresh beyond 10s?

Thanks,
Kiran

#6 lore raw
Quoted 7 lines · Ravikiran G Thirumalai
Quoted 2 lines

also, i think the valid threshold should be between 1 and 60 seconds i think.

60 seconds! Is that not a pretty high threshold? The reason for lowering the tolerance threshold from 10s is to catch bugs early in lab environments, but why do we need to raise the tolerance thresh beyond 10s?

just in case someone sees false positives and wants to turn it off.

Ingo

#7 lore raw
Quoted 1 line · Ingo Molnar

just in case someone sees false positives and wants to turn it off.

Why not make 0=off?

J

#8 lore raw
Quoted 4 lines · Jeremy Fitzhardinge
Quoted 1 line · Ingo Molnar

just in case someone sees false positives and wants to turn it off.

Why not make 0=off?

A patch to disable softlockup during boot already went in.
http://git.kernel.org/?p=linux/kernel/git/torvalds/linux-2.6.git;a=commit;h=97842216b8400fe9d1a20468959e2989180f8f79

It uses kernel boot parameter to disable softlockup, not exactly disabling softlockup at run time though. Do we still need 0=off?

Thanks,
Kiran

#9 lore raw
In reply to Andrew Morton
Quoted 13 lines · Andrew Morton
Quoted 11 lines · On Wed, 18 Jul 2007 22:41:21 -0700 Ravikiran G Thirumalai <[email protected]>…
Quoted 10 lines · Andrew Morton
Quoted 6 lines · On Mon, 16 Jul 2007 15:26:50 -0700 Ravikiran G Thirumalai <[email protected]>…

Kernel warns of softlockups if the softlockup thread is not able to run on a CPU for 10s. It is useful to lower the softlockup warning threshold in testing environments to catch potential lockups early. Following patch adds a kernel parameter 'softlockup_lim' to control the softlockup threshold.

Quoted 4 lines
Quoted 4 lines
Quoted 1 line

Why not make it tunable at runtime?

Sure! Like a sysctl?

Hi Andrew,
Here's another take, incorporating all of your comments.

Thanks,
Kiran

Control the trigger limit for softlockup warnings. This is useful for debugging softlockups, by lowering the softlockup_thresh to identify possible softlockups earlier.

This patch:
1. Adds a sysctl softlockup_thresh with valid values of 1-60s (Higher value to disable false positives)
2. Changes the softlockup printk to print the cpu softlockup time

---
Patch applies on top of Ingo's softlockup patches

 Documentation/sysctl/kernel.txt |    8 ++++++++
 include/linux/sched.h           |    1 +
 kernel/softlockup.c             |    8 +++++---
 kernel/sysctl.c                 |   25 +++++++++++++++++++------
 4 files changed, 33 insertions(+), 9 deletions(-)
kernel/softlockup.c +5 -3
Index: linux-2.6.22/kernel/softlockup.c
===================================================================
@@ -23,6 +23,7 @@ static DEFINE_PER_CPU(unsigned long, pri
 static DEFINE_PER_CPU(struct task_struct *, watchdog_task);
 
 static int did_panic;
+int softlockup_thresh = 10;
 
 static int
 softlock_panic(struct notifier_block *this, unsigned long event, void *ptr)
@@ -101,7 +102,7 @@ void softlockup_tick(void)
 		wake_up_process(per_cpu(watchdog_task, this_cpu));
 
 	/* Warn about unreasonable 10+ seconds delays: */
-	if (now <= (touch_timestamp + 10))
+	if (now <= (touch_timestamp + softlockup_thresh))
 		return;
 
 	regs = get_irq_regs();
@@ -109,8 +110,9 @@ void softlockup_tick(void)
 	per_cpu(print_timestamp, this_cpu) = touch_timestamp;
 
 	spin_lock(&print_lock);
-	printk(KERN_ERR "BUG: soft lockup detected on CPU#%d! [%s:%d]\n",
-			this_cpu, current->comm, current->pid);
+	printk(KERN_ERR "BUG: soft lockup - CPU#%d stuck for %lus! [%s:%d]\n",
+			this_cpu, now - touch_timestamp,
+				current->comm, current->pid);
 	if (regs)
 		show_regs(regs);
 	else
kernel/sysctl.c +19 -6
Index: linux-2.6.22/kernel/sysctl.c
===================================================================
@@ -79,6 +79,12 @@ extern int compat_log;
 extern int maps_protect;
 extern int sysctl_stat_interval;
 
+/* Constants used for minimum and  maximum */
+static int one = 1;
+static int zero;
+static int sixty = 60;
+static int one_hundred = 100;
+
 /* this is needed for the proc_dointvec_minmax for [fs_]overflow UID and GID */
 static int maxolduid = 65535;
 static int minolduid;
@@ -615,16 +621,23 @@ static ctl_table kern_table[] = {
 		.proc_handler   = &proc_dointvec,
 	},
 #endif
+#ifdef CONFIG_DETECT_SOFTLOCKUP
+	{
+		.ctl_name	= CTL_UNNUMBERED,
+		.procname	= "softlockup_thresh",
+		.data		= &softlockup_thresh,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= &proc_dointvec_minmax,
+		.strategy	= &sysctl_intvec,
+		.extra1		= &one,
+		.extra2		= &sixty,
+	},
+#endif
 
 	{ .ctl_name = 0 }
 };
 
-/* Constants for minimum and maximum testing in vm_table.
-   We use these as one-element integer vectors. */
-static int zero;
-static int one_hundred = 100;
-
-
 static ctl_table vm_table[] = {
 	{
 		.ctl_name	= VM_OVERCOMMIT_MEMORY,
Documentation/sysctl/kernel.txt +8 -0
Index: linux-2.6.22/Documentation/sysctl/kernel.txt
===================================================================
@@ -320,6 +320,14 @@ kernel.  This value defaults to SHMMAX.
 
 ==============================================================
 
+softlockup_thresh:
+
+This value can be used to lower the softlockup tolerance
+threshold. The default threshold is 10s.  If a cpu is locked up
+for 10s, the kernel complains.  Valid values are 1-60s.
+
+==============================================================
+
 tainted: 
 
 Non-zero if the kernel has been tainted.  Numeric values, which
include/linux/sched.h +1 -0
Index: linux-2.6.22/include/linux/sched.h
===================================================================
@@ -236,6 +236,7 @@ extern void softlockup_tick(void);
 extern void spawn_softlockup_task(void);
 extern void touch_softlockup_watchdog(void);
 extern void touch_all_softlockup_watchdogs(void);
+extern int softlockup_thresh;
 #else
 static inline void softlockup_tick(void)
 {