Skip to contentSkip to conversation

softlockup: decouple hung tasks check from softlockup detection

PATCH Superseded Mandeep Singh Baines 4 participants 11 messages Jan 12, 2009 – Jan 17, 2009

Subsystems (4)
lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 softlockup: decouple hung tasks check from softlockup detection

Decoupling allows:

* hung tasks check to happen at very low priority
* hung tasks check and softlockup to be enabled/disabled independently at compile and/or run-time
* individual panic settings to be enabled disabled independently at compile and/or run-time
* softlockup threshold to be reduced without increasing hung tasks poll frequency (hung task check is expensive relative to softlock watchdog)
* hung task check to be zero over-head when disabled at run-time

---
 include/linux/sched.h |   14 +++++--
 kernel/Makefile       |    1 +
 kernel/softlockup.c   |  102 -------------------------------------------------
 kernel/sysctl.c       |   15 +++++++-
 lib/Kconfig.debug     |   38 ++++++++++++++++++
 5 files changed, 63 insertions(+), 107 deletions(-)
include/linux/sched.h +10 -4
index 55e30d1..9e6a39b 100644
@@ -296,9 +296,6 @@ extern void softlockup_tick(void);
 extern void touch_softlockup_watchdog(void);
 extern void touch_all_softlockup_watchdogs(void);
 extern unsigned int  softlockup_panic;
-extern unsigned long sysctl_hung_task_check_count;
-extern unsigned long sysctl_hung_task_timeout_secs;
-extern unsigned long sysctl_hung_task_warnings;
 extern int softlockup_thresh;
 #else
 static inline void softlockup_tick(void)
@@ -315,6 +312,15 @@ static inline void touch_all_softlockup_watchdogs(void)
 }
 #endif
 
+#ifdef CONFIG_DETECT_HUNG_TASK
+extern unsigned int  sysctl_hung_task_panic;
+extern unsigned long sysctl_hung_task_check_count;
+extern unsigned long sysctl_hung_task_timeout_secs;
+extern unsigned long sysctl_hung_task_warnings;
+extern int proc_dohung_task_timeout_secs(struct ctl_table *table, int write,
+					 struct file *filp, void __user *buffer,
+					 size_t *lenp, loff_t *ppos);
+#endif
 
 /* Attach to any functions which should be ignored in wchan output. */
 #define __sched		__attribute__((__section__(".sched.text")))
@@ -1207,7 +1213,7 @@ struct task_struct {
 /* ipc stuff */
 	struct sysv_sem sysvsem;
 #endif
-#ifdef CONFIG_DETECT_SOFTLOCKUP
+#ifdef CONFIG_DETECT_HUNG_TASK
 /* hung task detection */
 	unsigned long last_switch_timestamp;
 	unsigned long last_switch_count;
kernel/Makefile +1 -0
index 19fad00..2809bc6 100644
@@ -70,6 +70,7 @@ obj-$(CONFIG_AUDIT_TREE) += audit_tree.o
 obj-$(CONFIG_KPROBES) += kprobes.o
 obj-$(CONFIG_KGDB) += kgdb.o
 obj-$(CONFIG_DETECT_SOFTLOCKUP) += softlockup.o
+obj-$(CONFIG_DETECT_HUNG_TASK) += hung_task.o
 obj-$(CONFIG_GENERIC_HARDIRQS) += irq/
 obj-$(CONFIG_SECCOMP) += seccomp.o
 obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o
kernel/softlockup.c +0 -102

128 lines not on this page · raw message

kernel/sysctl.c +14 -1
index 3d56fe7..51a4748 100644
@@ -771,6 +771,19 @@ static struct ctl_table kern_table[] = {
 		.extra1		= &neg_one,
 		.extra2		= &sixty,
 	},
+#endif
+#ifdef CONFIG_HUNG_TASK
+	{
+		.ctl_name	= CTL_UNNUMBERED,
+		.procname	= "hung_task_panic",
+		.data		= &sysctl_hung_task_panic,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= &proc_dointvec_minmax,
+		.strategy	= &sysctl_intvec,
+		.extra1		= &zero,
+		.extra2		= &one,
+	},
 	{
 		.ctl_name	= CTL_UNNUMBERED,
 		.procname	= "hung_task_check_count",
@@ -786,7 +799,7 @@ static struct ctl_table kern_table[] = {
 		.data		= &sysctl_hung_task_timeout_secs,
 		.maxlen		= sizeof(unsigned long),
 		.mode		= 0644,
-		.proc_handler	= &proc_doulongvec_minmax,
+		.proc_handler	= &proc_dohung_task_timeout_secs,
 		.strategy	= &sysctl_intvec,
 	},
 	{
lib/Kconfig.debug +38 -0
index b0f239e..a52aa38 100644
@@ -186,6 +186,44 @@ config BOOTPARAM_SOFTLOCKUP_PANIC_VALUE
 	default 0 if !BOOTPARAM_SOFTLOCKUP_PANIC
 	default 1 if BOOTPARAM_SOFTLOCKUP_PANIC
 
+config DETECT_HUNG_TASK
+	bool "Detect Hung Tasks"
+	depends on DEBUG_KERNEL
+	default y
+	help
+	  Say Y here to enable the kernel to detect "hung tasks",
+	  which are bugs that cause the task to be stuck in
+	  uninterruptible "D" state indefinitiley.
+
+	  When a hung task is detected, the kernel will print the
+	  current stack trace (which you should report), but the
+	  task will stay in uninterruptible state. If lockdep is
+	  enabled then all held locks will also be reported. This
+	  feature has negligible overhead.
+
+config BOOTPARAM_HUNG_TASK_PANIC
+	bool "Panic (Reboot) On Hung Tasks"
+	depends on DETECT_HUNG_TASK
+	help
+	  Say Y here to enable the kernel to panic on "hung tasks",
+	  which are bugs that cause the kernel to leave a task stuck
+	  in uninterruptible "D" state.
+
+	  The panic can be used in combination with panic_timeout,
+	  to cause the system to reboot automatically after a
+	  hung task has been detected. This feature is useful for
+	  high-availability systems that have uptime guarantees and
+	  where a hung tasks must be resolved ASAP.
+
+	  Say N if unsure.
+
+config BOOTPARAM_HUNG_TASK_PANIC_VALUE
+	int
+	depends on DETECT_HUNG_TASK
+	range 0 1
+	default 0 if !BOOTPARAM_HUNG_TASK_PANIC
+	default 1 if BOOTPARAM_HUNG_TASK_PANIC
+
 config SCHED_DEBUG
 	bool "Collect scheduler debugging info"
 	depends on DEBUG_KERNEL && PROC_FS
Signature · 1 line
1.5.4.5
#1 lore raw
Quoted 10 lines · Mandeep Singh Baines

Decoupling allows:

* hung tasks check to happen at very low priority
* hung tasks check and softlockup to be enabled/disabled independently at compile and/or run-time
* individual panic settings to be enabled disabled independently at compile and/or run-time
* softlockup threshold to be reduced without increasing hung tasks poll frequency (hung task check is expensive relative to softlock watchdog)
* hung task check to be zero over-head when disabled at run-time

Quoted 7 lines
---
 include/linux/sched.h |   14 +++++--
 kernel/Makefile       |    1 +
 kernel/softlockup.c   |  102 -------------------------------------------------
 kernel/sysctl.c       |   15 +++++++-
 lib/Kconfig.debug     |   38 ++++++++++++++++++
 5 files changed, 63 insertions(+), 107 deletions(-)

Looks good in principle, but i suspect your patch submission should include the new kernel/hung_task.c file too ;-)

Ingo

#2 lore raw

PATCH v3 1/1 softlockup: decouple hung tasks check from softlockup detection

Quoted 4 lines · Ingo Molnar

Looks good in principle, but i suspect your patch submission should include the new kernel/hung_task.c file too ;-)

Ingo

Doh. Thanks for reviewing. Here is the same patch with hung_task.c included.

---

Decoupling allows:

* hung tasks check to happen at very low priority
* hung tasks check and softlockup to be enabled/disabled independently at compile and/or run-time
* individual panic settings to be enabled disabled independently at compile and/or run-time
* softlockup threshold to be reduced without increasing hung tasks poll frequency (hung task check is expensive relative to softlock watchdog)
* hung task check to be zero over-head when disabled at run-time

---
 include/linux/sched.h |   14 +++-
 kernel/Makefile       |    1 +
 kernel/hung_task.c    |  198 +++++++++++++++++++++++++++++++++++++++++++++++++
 kernel/softlockup.c   |  102 -------------------------
 kernel/sysctl.c       |   15 ++++-
 lib/Kconfig.debug     |   38 ++++++++++
 6 files changed, 261 insertions(+), 107 deletions(-)
 create mode 100644 kernel/hung_task.c
include/linux/sched.h +10 -4
index 55e30d1..9e6a39b 100644
@@ -296,9 +296,6 @@ extern void softlockup_tick(void);
 extern void touch_softlockup_watchdog(void);
 extern void touch_all_softlockup_watchdogs(void);
 extern unsigned int  softlockup_panic;
-extern unsigned long sysctl_hung_task_check_count;
-extern unsigned long sysctl_hung_task_timeout_secs;
-extern unsigned long sysctl_hung_task_warnings;
 extern int softlockup_thresh;
 #else
 static inline void softlockup_tick(void)
@@ -315,6 +312,15 @@ static inline void touch_all_softlockup_watchdogs(void)
 }
 #endif
 
+#ifdef CONFIG_DETECT_HUNG_TASK
+extern unsigned int  sysctl_hung_task_panic;
+extern unsigned long sysctl_hung_task_check_count;
+extern unsigned long sysctl_hung_task_timeout_secs;
+extern unsigned long sysctl_hung_task_warnings;
+extern int proc_dohung_task_timeout_secs(struct ctl_table *table, int write,
+					 struct file *filp, void __user *buffer,
+					 size_t *lenp, loff_t *ppos);
+#endif
 
 /* Attach to any functions which should be ignored in wchan output. */
 #define __sched		__attribute__((__section__(".sched.text")))
@@ -1207,7 +1213,7 @@ struct task_struct {
 /* ipc stuff */
 	struct sysv_sem sysvsem;
 #endif
-#ifdef CONFIG_DETECT_SOFTLOCKUP
+#ifdef CONFIG_DETECT_HUNG_TASK
 /* hung task detection */
 	unsigned long last_switch_timestamp;
 	unsigned long last_switch_count;
kernel/Makefile +1 -0
index 19fad00..2809bc6 100644
@@ -70,6 +70,7 @@ obj-$(CONFIG_AUDIT_TREE) += audit_tree.o
 obj-$(CONFIG_KPROBES) += kprobes.o
 obj-$(CONFIG_KGDB) += kgdb.o
 obj-$(CONFIG_DETECT_SOFTLOCKUP) += softlockup.o
+obj-$(CONFIG_DETECT_HUNG_TASK) += hung_task.o
 obj-$(CONFIG_GENERIC_HARDIRQS) += irq/
 obj-$(CONFIG_SECCOMP) += seccomp.o
 obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o
kernel/hung_task.c new file +198 -0

198 lines not on this page · raw message

kernel/softlockup.c +0 -102

128 lines not on this page · raw message

kernel/sysctl.c +14 -1
index 3d56fe7..51a4748 100644
@@ -771,6 +771,19 @@ static struct ctl_table kern_table[] = {
 		.extra1		= &neg_one,
 		.extra2		= &sixty,
 	},
+#endif
+#ifdef CONFIG_HUNG_TASK
+	{
+		.ctl_name	= CTL_UNNUMBERED,
+		.procname	= "hung_task_panic",
+		.data		= &sysctl_hung_task_panic,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= &proc_dointvec_minmax,
+		.strategy	= &sysctl_intvec,
+		.extra1		= &zero,
+		.extra2		= &one,
+	},
 	{
 		.ctl_name	= CTL_UNNUMBERED,
 		.procname	= "hung_task_check_count",
@@ -786,7 +799,7 @@ static struct ctl_table kern_table[] = {
 		.data		= &sysctl_hung_task_timeout_secs,
 		.maxlen		= sizeof(unsigned long),
 		.mode		= 0644,
-		.proc_handler	= &proc_doulongvec_minmax,
+		.proc_handler	= &proc_dohung_task_timeout_secs,
 		.strategy	= &sysctl_intvec,
 	},
 	{
lib/Kconfig.debug +38 -0
index b0f239e..a52aa38 100644
@@ -186,6 +186,44 @@ config BOOTPARAM_SOFTLOCKUP_PANIC_VALUE
 	default 0 if !BOOTPARAM_SOFTLOCKUP_PANIC
 	default 1 if BOOTPARAM_SOFTLOCKUP_PANIC
 
+config DETECT_HUNG_TASK
+	bool "Detect Hung Tasks"
+	depends on DEBUG_KERNEL
+	default y
+	help
+	  Say Y here to enable the kernel to detect "hung tasks",
+	  which are bugs that cause the task to be stuck in
+	  uninterruptible "D" state indefinitiley.
+
+	  When a hung task is detected, the kernel will print the
+	  current stack trace (which you should report), but the
+	  task will stay in uninterruptible state. If lockdep is
+	  enabled then all held locks will also be reported. This
+	  feature has negligible overhead.
+
+config BOOTPARAM_HUNG_TASK_PANIC
+	bool "Panic (Reboot) On Hung Tasks"
+	depends on DETECT_HUNG_TASK
+	help
+	  Say Y here to enable the kernel to panic on "hung tasks",
+	  which are bugs that cause the kernel to leave a task stuck
+	  in uninterruptible "D" state.
+
+	  The panic can be used in combination with panic_timeout,
+	  to cause the system to reboot automatically after a
+	  hung task has been detected. This feature is useful for
+	  high-availability systems that have uptime guarantees and
+	  where a hung tasks must be resolved ASAP.
+
+	  Say N if unsure.
+
+config BOOTPARAM_HUNG_TASK_PANIC_VALUE
+	int
+	depends on DETECT_HUNG_TASK
+	range 0 1
+	default 0 if !BOOTPARAM_HUNG_TASK_PANIC
+	default 1 if BOOTPARAM_HUNG_TASK_PANIC
+
 config SCHED_DEBUG
 	bool "Collect scheduler debugging info"
 	depends on DEBUG_KERNEL && PROC_FS
Signature · 1 line
1.5.4.5
#3 lore raw
Quoted 7 lines · Mandeep Singh Baines
Quoted 4 lines · Ingo Molnar

Looks good in principle, but i suspect your patch submission should include the new kernel/hung_task.c file too ;-)

Ingo

Doh. Thanks for reviewing. Here is the same patch with hung_task.c included.

this now conflicts with the softlockup-panic fix you posted - and that fix has to go first as it's v2.6.29 material. (while the decoupling patch is 2.6.30 material)

So could you please send a patch against tip/master:

http://people.redhat.com/mingo/tip.git/README

Thanks,

Ingo

#4 lore raw

PATCH v4 1/1 softlockup: decouple hung tasks check from softlockup detection

Quoted 5 lines · Ingo Molnar

this now conflicts with the softlockup-panic fix you posted - and that fix has to go first as it's v2.6.29 material. (while the decoupling patch is 2.6.30 material)

Quoted 7 lines

So could you please send a patch against tip/master:

http://people.redhat.com/mingo/tip.git/README

Thanks,

Ingo

No problem. Rebased to tip/master.

---

Decoupling allows:

* hung tasks check to happen at very low priority
* hung tasks check and softlockup to be enabled/disabled independently at compile and/or run-time
* individual panic settings to be enabled disabled independently at compile and/or run-time
* softlockup threshold to be reduced without increasing hung tasks poll frequency (hung task check is expensive relative to softlock watchdog)
* hung task check to be zero over-head when disabled at run-time

---
 include/linux/sched.h |   14 +++-
 kernel/Makefile       |    1 +
 kernel/hung_task.c    |  198 +++++++++++++++++++++++++++++++++++++++++++++++++
 kernel/softlockup.c   |  100 -------------------------
 kernel/sysctl.c       |   15 ++++-
 lib/Kconfig.debug     |   38 ++++++++++
 6 files changed, 261 insertions(+), 105 deletions(-)
 create mode 100644 kernel/hung_task.c
include/linux/sched.h +10 -4
index 5a0b9b6..bda891e 100644
@@ -298,9 +298,6 @@ extern int proc_dosoftlockup_thresh(struct ctl_table *table, int write,
 				    struct file *filp, void __user *buffer,
 				    size_t *lenp, loff_t *ppos);
 extern unsigned int  softlockup_panic;
-extern unsigned long sysctl_hung_task_check_count;
-extern unsigned long sysctl_hung_task_timeout_secs;
-extern unsigned long sysctl_hung_task_warnings;
 extern int softlockup_thresh;
 #else
 static inline void softlockup_tick(void)
@@ -317,6 +314,15 @@ static inline void touch_all_softlockup_watchdogs(void)
 }
 #endif
 
+#ifdef CONFIG_DETECT_HUNG_TASK
+extern unsigned int  sysctl_hung_task_panic;
+extern unsigned long sysctl_hung_task_check_count;
+extern unsigned long sysctl_hung_task_timeout_secs;
+extern unsigned long sysctl_hung_task_warnings;
+extern int proc_dohung_task_timeout_secs(struct ctl_table *table, int write,
+					 struct file *filp, void __user *buffer,
+					 size_t *lenp, loff_t *ppos);
+#endif
 
 /* Attach to any functions which should be ignored in wchan output. */
 #define __sched		__attribute__((__section__(".sched.text")))
@@ -1240,7 +1246,7 @@ struct task_struct {
 /* ipc stuff */
 	struct sysv_sem sysvsem;
 #endif
-#ifdef CONFIG_DETECT_SOFTLOCKUP
+#ifdef CONFIG_DETECT_HUNG_TASK
 /* hung task detection */
 	unsigned long last_switch_timestamp;
 	unsigned long last_switch_count;
kernel/Makefile +1 -0
index e411592..46cd2b6 100644
@@ -74,6 +74,7 @@ obj-$(CONFIG_AUDIT_TREE) += audit_tree.o
 obj-$(CONFIG_KPROBES) += kprobes.o
 obj-$(CONFIG_KGDB) += kgdb.o
 obj-$(CONFIG_DETECT_SOFTLOCKUP) += softlockup.o
+obj-$(CONFIG_DETECT_HUNG_TASK) += hung_task.o
 obj-$(CONFIG_GENERIC_HARDIRQS) += irq/
 obj-$(CONFIG_SECCOMP) += seccomp.o
 obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o
kernel/hung_task.c new file +198 -0

198 lines not on this page · raw message

kernel/softlockup.c +0 -100

126 lines not on this page · raw message

kernel/sysctl.c +14 -1
index 80849e0..d7971f6 100644
@@ -806,6 +806,19 @@ static struct ctl_table kern_table[] = {
 		.extra1		= &neg_one,
 		.extra2		= &sixty,
 	},
+#endif
+#ifdef CONFIG_DETECT_HUNG_TASK
+	{
+		.ctl_name	= CTL_UNNUMBERED,
+		.procname	= "hung_task_panic",
+		.data		= &sysctl_hung_task_panic,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= &proc_dointvec_minmax,
+		.strategy	= &sysctl_intvec,
+		.extra1		= &zero,
+		.extra2		= &one,
+	},
 	{
 		.ctl_name	= CTL_UNNUMBERED,
 		.procname	= "hung_task_check_count",
@@ -821,7 +834,7 @@ static struct ctl_table kern_table[] = {
 		.data		= &sysctl_hung_task_timeout_secs,
 		.maxlen		= sizeof(unsigned long),
 		.mode		= 0644,
-		.proc_handler	= &proc_doulongvec_minmax,
+		.proc_handler	= &proc_dohung_task_timeout_secs,
 		.strategy	= &sysctl_intvec,
 	},
 	{
lib/Kconfig.debug +38 -0
index 393eb8c..8ccaf97 100644
@@ -199,6 +199,44 @@ config BOOTPARAM_SOFTLOCKUP_PANIC_VALUE
 	default 0 if !BOOTPARAM_SOFTLOCKUP_PANIC
 	default 1 if BOOTPARAM_SOFTLOCKUP_PANIC
 
+config DETECT_HUNG_TASK
+	bool "Detect Hung Tasks"
+	depends on DEBUG_KERNEL
+	default y
+	help
+	  Say Y here to enable the kernel to detect "hung tasks",
+	  which are bugs that cause the task to be stuck in
+	  uninterruptible "D" state indefinitiley.
+
+	  When a hung task is detected, the kernel will print the
+	  current stack trace (which you should report), but the
+	  task will stay in uninterruptible state. If lockdep is
+	  enabled then all held locks will also be reported. This
+	  feature has negligible overhead.
+
+config BOOTPARAM_HUNG_TASK_PANIC
+	bool "Panic (Reboot) On Hung Tasks"
+	depends on DETECT_HUNG_TASK
+	help
+	  Say Y here to enable the kernel to panic on "hung tasks",
+	  which are bugs that cause the kernel to leave a task stuck
+	  in uninterruptible "D" state.
+
+	  The panic can be used in combination with panic_timeout,
+	  to cause the system to reboot automatically after a
+	  hung task has been detected. This feature is useful for
+	  high-availability systems that have uptime guarantees and
+	  where a hung tasks must be resolved ASAP.
+
+	  Say N if unsure.
+
+config BOOTPARAM_HUNG_TASK_PANIC_VALUE
+	int
+	depends on DETECT_HUNG_TASK
+	range 0 1
+	default 0 if !BOOTPARAM_HUNG_TASK_PANIC
+	default 1 if BOOTPARAM_HUNG_TASK_PANIC
+
 config SCHED_DEBUG
 	bool "Collect scheduler debugging info"
 	depends on DEBUG_KERNEL && PROC_FS
Signature · 1 line
1.5.4.5
Applied
#5 lore raw
Quoted 8 lines · Mandeep Singh Baines
Quoted 7 lines · Ingo Molnar

this now conflicts with the softlockup-panic fix you posted - and that fix has to go first as it's v2.6.29 material. (while the decoupling patch is 2.6.30 material)

So could you please send a patch against tip/master:

Quoted 7 lines
Quoted 5 lines

http://people.redhat.com/mingo/tip.git/README

Thanks,

Ingo

No problem. Rebased to tip/master.

applied to tip/core/softlockup - thanks Mandeep!

Ingo

#6 lore raw
Quoted 3 lines · Ingo Molnar
Quoted 1 line

No problem. Rebased to tip/master.

applied to tip/core/softlockup - thanks Mandeep!

doesnt build with !SOFTLOCKUP:

kernel/fork.c:1049: error: 'struct task_struct' has no member named 'last_switch_count' kernel/fork.c:1050: error: 'struct task_struct' has no member named 'last_switch_timestamp'

Could you send a delta fix patch relative to -v4 please? Thanks,

Ingo

#7 lore raw
In reply to Mandeep Singh Baines
Quoted 10 lines · Mandeep Singh Baines

Decoupling allows:

* hung tasks check to happen at very low priority
* hung tasks check and softlockup to be enabled/disabled independently at compile and/or run-time
* individual panic settings to be enabled disabled independently at compile and/or run-time
* softlockup threshold to be reduced without increasing hung tasks poll frequency (hung task check is expensive relative to softlock watchdog)
* hung task check to be zero over-head when disabled at run-time

Quoted 7 lines
---
 include/linux/sched.h |   14 +++++--
 kernel/Makefile       |    1 +
 kernel/softlockup.c   |  102 -------------------------------------------------
 kernel/sysctl.c       |   15 +++++++-
 lib/Kconfig.debug     |   38 ++++++++++++++++++
 5 files changed, 63 insertions(+), 107 deletions(-)

kernel/hung_task.c missing?

Hannes

#8 lore raw
In reply to Mandeep Singh Baines

Hi Mandeep,

2009/1/12 Mandeep Singh Baines <[email protected]>:

Quoted 12 lines
kernel/softlockup.c +0 -12
-/*
- * Check whether a TASK_UNINTERRUPTIBLE does not get woken up for
- * a really long time (120 seconds). If that happens, print out
- * a warning.
- */
-static void check_hung_uninterruptible_tasks(int this_cpu)
-{
-       int max_count = sysctl_hung_task_check_count;
-       unsigned long now = get_timestamp(this_cpu);
-       struct task_struct *g, *t;
-
-       /*

kernel/softlockup.c void softlockup_tick(void) · line 235

Quoted 10 lines
kernel/softlockup.c +0 -10
-        * If the system crashed already then all bets are off,
-        * do not report extra hung tasks:
-        */
-       if (test_taint(TAINT_DIE) || did_panic)
-               return;
-
-       read_lock(&tasklist_lock);
-       do_each_thread(g, t) {
-               if (!--max_count)
-                       goto unlock;

Instead of having this arbitrary limit of tasks, why not just lurk the need_resched() and then schedule if it needs too.

I know that sounds a bit racy, because you will have to release the tasklist_lock and
a lot of things can happen in the task list until you become resched. But you can do a get_task_struct() on g and t before your thread is going to sleep and then put them
when it is awaken.
Perhaps some tasks will disappear or be appended in the list before g and t, but that doesn't really matter:
if they disappear, they didn't lockup, and if they were appended, they are not enough cold to be analyzed :-)

This way you can drop the arbitrary limit of task number given by the user....

Frederic.

Quoted 142 lines
kernel/softlockup.c +0 -26
-               /* use "==" to skip the TASK_KILLABLE tasks waiting on NFS */
-               if (t->state == TASK_UNINTERRUPTIBLE)
-                       check_hung_task(t, now);
-       } while_each_thread(g, t);
- unlock:
-       read_unlock(&tasklist_lock);
-}
-
-/*
 * The watchdog thread - runs every second and touches the timestamp.
 */
 static int watchdog(void *__bind_cpu)
 {
       struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
-       int this_cpu = (long)__bind_cpu;
 
       sched_setscheduler(current, SCHED_FIFO, &param);
@@ -267,11 +181,6 @@ static int watchdog(void *__bind_cpu)
               if (kthread_should_stop())
                       break;
 
-               if (this_cpu == check_cpu) {
-                       if (sysctl_hung_task_timeout_secs)
-                               check_hung_uninterruptible_tasks(this_cpu);
-               }
-
               set_current_state(TASK_INTERRUPTIBLE);
       }
       __set_current_state(TASK_RUNNING);
@@ -303,20 +212,9 @@ cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
               break;
       case CPU_ONLINE:
       case CPU_ONLINE_FROZEN:
-               check_cpu = any_online_cpu(cpu_online_map);
               wake_up_process(per_cpu(watchdog_task, hotcpu));
               break;
 #ifdef CONFIG_HOTPLUG_CPU
-       case CPU_DOWN_PREPARE:
-       case CPU_DOWN_PREPARE_FROZEN:
-               if (hotcpu == check_cpu) {
-                       cpumask_t temp_cpu_online_map = cpu_online_map;
-
-                       cpu_clear(hotcpu, temp_cpu_online_map);
-                       check_cpu = any_online_cpu(temp_cpu_online_map);
-               }
-               break;
-
       case CPU_UP_CANCELED:
       case CPU_UP_CANCELED_FROZEN:
               if (!per_cpu(watchdog_task, hotcpu))
kernel/sysctl.c +14 -1
index 3d56fe7..51a4748 100644
@@ -771,6 +771,19 @@ static struct ctl_table kern_table[] = {
               .extra1         = &neg_one,
               .extra2         = &sixty,
       },
+#endif
+#ifdef CONFIG_HUNG_TASK
+       {
+               .ctl_name       = CTL_UNNUMBERED,
+               .procname       = "hung_task_panic",
+               .data           = &sysctl_hung_task_panic,
+               .maxlen         = sizeof(int),
+               .mode           = 0644,
+               .proc_handler   = &proc_dointvec_minmax,
+               .strategy       = &sysctl_intvec,
+               .extra1         = &zero,
+               .extra2         = &one,
+       },
       {
               .ctl_name       = CTL_UNNUMBERED,
               .procname       = "hung_task_check_count",
@@ -786,7 +799,7 @@ static struct ctl_table kern_table[] = {
               .data           = &sysctl_hung_task_timeout_secs,
               .maxlen         = sizeof(unsigned long),
               .mode           = 0644,
-               .proc_handler   = &proc_doulongvec_minmax,
+               .proc_handler   = &proc_dohung_task_timeout_secs,
               .strategy       = &sysctl_intvec,
       },
       {
lib/Kconfig.debug +38 -1
index b0f239e..a52aa38 100644
@@ -186,6 +186,44 @@ config BOOTPARAM_SOFTLOCKUP_PANIC_VALUE
       default 0 if !BOOTPARAM_SOFTLOCKUP_PANIC
       default 1 if BOOTPARAM_SOFTLOCKUP_PANIC
 
+config DETECT_HUNG_TASK
+       bool "Detect Hung Tasks"
+       depends on DEBUG_KERNEL
+       default y
+       help
+         Say Y here to enable the kernel to detect "hung tasks",
+         which are bugs that cause the task to be stuck in
+         uninterruptible "D" state indefinitiley.
+
+         When a hung task is detected, the kernel will print the
+         current stack trace (which you should report), but the
+         task will stay in uninterruptible state. If lockdep is
+         enabled then all held locks will also be reported. This
+         feature has negligible overhead.
+
+config BOOTPARAM_HUNG_TASK_PANIC
+       bool "Panic (Reboot) On Hung Tasks"
+       depends on DETECT_HUNG_TASK
+       help
+         Say Y here to enable the kernel to panic on "hung tasks",
+         which are bugs that cause the kernel to leave a task stuck
+         in uninterruptible "D" state.
+
+         The panic can be used in combination with panic_timeout,
+         to cause the system to reboot automatically after a
+         hung task has been detected. This feature is useful for
+         high-availability systems that have uptime guarantees and
+         where a hung tasks must be resolved ASAP.
+
+         Say N if unsure.
+
+config BOOTPARAM_HUNG_TASK_PANIC_VALUE
+       int
+       depends on DETECT_HUNG_TASK
+       range 0 1
+       default 0 if !BOOTPARAM_HUNG_TASK_PANIC
+       default 1 if BOOTPARAM_HUNG_TASK_PANIC
+
 config SCHED_DEBUG
       bool "Collect scheduler debugging info"
       depends on DEBUG_KERNEL && PROC_FS
--

1.5.4.5

--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/

#9 lore raw

Hi Frédéric,

Quoted 20 lines · Frédéric Weisbecker

kernel/softlockup.c void softlockup_tick(void) · line 235

Quoted 4 lines
kernel/softlockup.c +0 -4
-       read_lock(&tasklist_lock);
-       do_each_thread(g, t) {
-               if (!--max_count)
-                       goto unlock;

Instead of having this arbitrary limit of tasks, why not just lurk the need_resched() and then schedule if it needs too.

I know that sounds a bit racy, because you will have to release the tasklist_lock and
a lot of things can happen in the task list until you become resched. But you can do a get_task_struct() on g and t before your thread is going to sleep and then put them
when it is awaken.
Perhaps some tasks will disappear or be appended in the list before g and t, but that doesn't really matter:
if they disappear, they didn't lockup, and if they were appended, they are not enough cold to be analyzed :-)

Quoted 4 lines

This way you can drop the arbitrary limit of task number given by the user....

Frederic.

Would be nice to remove the limit. But I don't think get_task_struct() can be used to prevent a task from being unlinked from the task list. It only prevents the task_struct from being freed. So hung_task could end up holding a reference to an unlinked task after it returns from schedule().

That doesn't mean what you are suggesting can't be implemented. Just means that the case of the held task being unlinked needs to be handled.

Regards,
Mandeep

#10 lore raw
Quoted 33 lines · Mandeep Singh Baines

Hi Frédéric,

Quoted 20 lines · Frédéric Weisbecker

kernel/softlockup.c void softlockup_tick(void) · line 235

Quoted 4 lines
kernel/softlockup.c +0 -4
-       read_lock(&tasklist_lock);
-       do_each_thread(g, t) {
-               if (!--max_count)
-                       goto unlock;

Instead of having this arbitrary limit of tasks, why not just lurk the need_resched() and then schedule if it needs too.

I know that sounds a bit racy, because you will have to release the tasklist_lock and
a lot of things can happen in the task list until you become resched. But you can do a get_task_struct() on g and t before your thread is going to sleep and then put them
when it is awaken.
Perhaps some tasks will disappear or be appended in the list before g and t, but that doesn't really matter:
if they disappear, they didn't lockup, and if they were appended, they are not enough cold to be analyzed :-)

Quoted 4 lines

This way you can drop the arbitrary limit of task number given by the user....

Frederic.

Would be nice to remove the limit. But I don't think get_task_struct() can be used to prevent a task from being unlinked from the task list. It only prevents the task_struct from being freed. So hung_task could end up holding a reference to an unlinked task after it returns from schedule().

Quoted 5 lines

That doesn't mean what you are suggesting can't be implemented. Just means that the case of the held task being unlinked needs to be handled.

Regards,
Mandeep

Hmm, you're right.
Why not testing 1024 tasks, then check need_resched and if you sleep and the task becomes unlinked (there are few chances) so... that's not a big deal actually, you will have better chances on the next check :-)

I think that's a bit important since you are more likely to see a soft-lockup if you have a lot of tasks.