softlockup: decouple hung tasks check from softlockup detection
Subsystems (4)
PATCH v2 1/1 softlockup: decouple hung tasks check from softlockup detection
Decoupling allows:
* hung tasks check to happen at very low priority
* hung tasks check and softlockup to be enabled/disabled independently at compile and/or run-time
* individual panic settings to be enabled disabled independently at compile and/or run-time
* softlockup threshold to be reduced without increasing hung tasks poll frequency (hung task check is expensive relative to softlock watchdog)
* hung task check to be zero over-head when disabled at run-time
--- include/linux/sched.h | 14 +++++-- kernel/Makefile | 1 + kernel/softlockup.c | 102 ------------------------------------------------- kernel/sysctl.c | 15 +++++++- lib/Kconfig.debug | 38 ++++++++++++++++++ 5 files changed, 63 insertions(+), 107 deletions(-)
include/linux/sched.h +10 -4
@@ -296,9 +296,6 @@ extern void softlockup_tick(void);extern void touch_softlockup_watchdog (void );extern void touch_all_softlockup_watchdogs (void );extern unsigned int softlockup_panic; -extern unsigned long sysctl_hung_task_check_count; -extern unsigned long sysctl_hung_task_timeout_secs; -extern unsigned long sysctl_hung_task_warnings;extern int softlockup_thresh;#else static inline void softlockup_tick (void )
@@ -315,6 +312,15 @@ static inline void touch_all_softlockup_watchdogs(void) }#endif +#ifdef CONFIG_DETECT_HUNG_TASK +extern unsigned int sysctl_hung_task_panic; +extern unsigned long sysctl_hung_task_check_count; +extern unsigned long sysctl_hung_task_timeout_secs; +extern unsigned long sysctl_hung_task_warnings; +extern int proc_dohung_task_timeout_secs (struct ctl_table *table,int write, +struct file *filp,void __user *buffer, +size_t *lenp,loff_t *ppos); +#endif /* Attach to any functions which should be ignored in wchan output. */ #define __sched __attribute__((__section__(".sched.text")))
@@ -1207,7 +1213,7 @@ struct task_struct {/* ipc stuff */ struct sysv_sem sysvsem;#endif -#ifdef CONFIG_DETECT_SOFTLOCKUP +#ifdef CONFIG_DETECT_HUNG_TASK /* hung task detection */ unsigned long last_switch_timestamp;unsigned long last_switch_count;
kernel/Makefile +1 -0
@@ -70,6 +70,7 @@ obj-$(CONFIG_AUDIT_TREE) += audit_tree.o obj-$(CONFIG_KPROBES) += kprobes.o obj-$(CONFIG_KGDB) += kgdb.o obj-$(CONFIG_DETECT_SOFTLOCKUP) += softlockup.o +obj-$(CONFIG_DETECT_HUNG_TASK) += hung_task.o obj-$(CONFIG_GENERIC_HARDIRQS) += irq/ obj-$(CONFIG_SECCOMP) += seccomp.o obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o
kernel/softlockup.c +0 -102
128 lines not on this page · raw message
kernel/sysctl.c +14 -1
@@ -771,6 +771,19 @@ static struct ctl_table kern_table[] = { .extra1 = &neg_one, .extra2 = &sixty, }, +#endif +#ifdef CONFIG_HUNG_TASK + { + .ctl_name = CTL_UNNUMBERED, + .procname = "hung_task_panic", + .data = &sysctl_hung_task_panic, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = &proc_dointvec_minmax, + .strategy = &sysctl_intvec, + .extra1 = &zero, + .extra2 = &one, + }, { .ctl_name = CTL_UNNUMBERED, .procname = "hung_task_check_count",
@@ -786,7 +799,7 @@ static struct ctl_table kern_table[] = { .data = &sysctl_hung_task_timeout_secs, .maxlen = sizeof(unsigned long), .mode = 0644, - .proc_handler = &proc_doulongvec_minmax, + .proc_handler = &proc_dohung_task_timeout_secs, .strategy = &sysctl_intvec, }, {
lib/Kconfig.debug +38 -0
@@ -186,6 +186,44 @@ config BOOTPARAM_SOFTLOCKUP_PANIC_VALUE default 0 if !BOOTPARAM_SOFTLOCKUP_PANIC default 1 if BOOTPARAM_SOFTLOCKUP_PANIC +config DETECT_HUNG_TASK + bool "Detect Hung Tasks" + depends on DEBUG_KERNEL + default y + help + Say Y here to enable the kernel to detect "hung tasks", + which are bugs that cause the task to be stuck in + uninterruptible "D" state indefinitiley. + + When a hung task is detected, the kernel will print the + current stack trace (which you should report), but the + task will stay in uninterruptible state. If lockdep is + enabled then all held locks will also be reported. This + feature has negligible overhead. + +config BOOTPARAM_HUNG_TASK_PANIC + bool "Panic (Reboot) On Hung Tasks" + depends on DETECT_HUNG_TASK + help + Say Y here to enable the kernel to panic on "hung tasks", + which are bugs that cause the kernel to leave a task stuck + in uninterruptible "D" state. + + The panic can be used in combination with panic_timeout, + to cause the system to reboot automatically after a + hung task has been detected. This feature is useful for + high-availability systems that have uptime guarantees and + where a hung tasks must be resolved ASAP. + + Say N if unsure. + +config BOOTPARAM_HUNG_TASK_PANIC_VALUE + int + depends on DETECT_HUNG_TASK + range 0 1 + default 0 if !BOOTPARAM_HUNG_TASK_PANIC + default 1 if BOOTPARAM_HUNG_TASK_PANIC + config SCHED_DEBUG bool "Collect scheduler debugging info" depends on DEBUG_KERNEL && PROC_FS
Signature · 1 line
1.5.4.5
Quoted 10 lines · Mandeep Singh Baines
Decoupling allows:
* hung tasks check to happen at very low priority
* hung tasks check and softlockup to be enabled/disabled independently at compile and/or run-time
* individual panic settings to be enabled disabled independently at compile and/or run-time
* softlockup threshold to be reduced without increasing hung tasks poll frequency (hung task check is expensive relative to softlock watchdog)
* hung task check to be zero over-head when disabled at run-time
Quoted 7 lines
--- include/linux/sched.h | 14 +++++-- kernel/Makefile | 1 + kernel/softlockup.c | 102 ------------------------------------------------- kernel/sysctl.c | 15 +++++++- lib/Kconfig.debug | 38 ++++++++++++++++++ 5 files changed, 63 insertions(+), 107 deletions(-)
Looks good in principle, but i suspect your patch submission should include the new kernel/hung_task.c file too ;-)
Ingo
PATCH v3 1/1 softlockup: decouple hung tasks check from softlockup detection
Quoted 4 lines · Ingo Molnar
Looks good in principle, but i suspect your patch submission should include the new
kernel/hung_task.cfile too ;-)Ingo
Doh. Thanks for reviewing. Here is the same patch with hung_task.c included.
---
Decoupling allows:
* hung tasks check to happen at very low priority
* hung tasks check and softlockup to be enabled/disabled independently at compile and/or run-time
* individual panic settings to be enabled disabled independently at compile and/or run-time
* softlockup threshold to be reduced without increasing hung tasks poll frequency (hung task check is expensive relative to softlock watchdog)
* hung task check to be zero over-head when disabled at run-time
--- include/linux/sched.h | 14 +++- kernel/Makefile | 1 + kernel/hung_task.c | 198 +++++++++++++++++++++++++++++++++++++++++++++++++ kernel/softlockup.c | 102 ------------------------- kernel/sysctl.c | 15 ++++- lib/Kconfig.debug | 38 ++++++++++ 6 files changed, 261 insertions(+), 107 deletions(-) create mode 100644 kernel/hung_task.c
include/linux/sched.h +10 -4
@@ -296,9 +296,6 @@ extern void softlockup_tick(void);extern void touch_softlockup_watchdog (void );extern void touch_all_softlockup_watchdogs (void );extern unsigned int softlockup_panic; -extern unsigned long sysctl_hung_task_check_count; -extern unsigned long sysctl_hung_task_timeout_secs; -extern unsigned long sysctl_hung_task_warnings;extern int softlockup_thresh;#else static inline void softlockup_tick (void )
@@ -315,6 +312,15 @@ static inline void touch_all_softlockup_watchdogs(void) }#endif +#ifdef CONFIG_DETECT_HUNG_TASK +extern unsigned int sysctl_hung_task_panic; +extern unsigned long sysctl_hung_task_check_count; +extern unsigned long sysctl_hung_task_timeout_secs; +extern unsigned long sysctl_hung_task_warnings; +extern int proc_dohung_task_timeout_secs (struct ctl_table *table,int write, +struct file *filp,void __user *buffer, +size_t *lenp,loff_t *ppos); +#endif /* Attach to any functions which should be ignored in wchan output. */ #define __sched __attribute__((__section__(".sched.text")))
@@ -1207,7 +1213,7 @@ struct task_struct {/* ipc stuff */ struct sysv_sem sysvsem;#endif -#ifdef CONFIG_DETECT_SOFTLOCKUP +#ifdef CONFIG_DETECT_HUNG_TASK /* hung task detection */ unsigned long last_switch_timestamp;unsigned long last_switch_count;
kernel/Makefile +1 -0
@@ -70,6 +70,7 @@ obj-$(CONFIG_AUDIT_TREE) += audit_tree.o obj-$(CONFIG_KPROBES) += kprobes.o obj-$(CONFIG_KGDB) += kgdb.o obj-$(CONFIG_DETECT_SOFTLOCKUP) += softlockup.o +obj-$(CONFIG_DETECT_HUNG_TASK) += hung_task.o obj-$(CONFIG_GENERIC_HARDIRQS) += irq/ obj-$(CONFIG_SECCOMP) += seccomp.o obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o
kernel/hung_task.c new file +198 -0
198 lines not on this page · raw message
kernel/softlockup.c +0 -102
128 lines not on this page · raw message
kernel/sysctl.c +14 -1
@@ -771,6 +771,19 @@ static struct ctl_table kern_table[] = { .extra1 = &neg_one, .extra2 = &sixty, }, +#endif +#ifdef CONFIG_HUNG_TASK + { + .ctl_name = CTL_UNNUMBERED, + .procname = "hung_task_panic", + .data = &sysctl_hung_task_panic, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = &proc_dointvec_minmax, + .strategy = &sysctl_intvec, + .extra1 = &zero, + .extra2 = &one, + }, { .ctl_name = CTL_UNNUMBERED, .procname = "hung_task_check_count",
@@ -786,7 +799,7 @@ static struct ctl_table kern_table[] = { .data = &sysctl_hung_task_timeout_secs, .maxlen = sizeof(unsigned long), .mode = 0644, - .proc_handler = &proc_doulongvec_minmax, + .proc_handler = &proc_dohung_task_timeout_secs, .strategy = &sysctl_intvec, }, {
lib/Kconfig.debug +38 -0
@@ -186,6 +186,44 @@ config BOOTPARAM_SOFTLOCKUP_PANIC_VALUE default 0 if !BOOTPARAM_SOFTLOCKUP_PANIC default 1 if BOOTPARAM_SOFTLOCKUP_PANIC +config DETECT_HUNG_TASK + bool "Detect Hung Tasks" + depends on DEBUG_KERNEL + default y + help + Say Y here to enable the kernel to detect "hung tasks", + which are bugs that cause the task to be stuck in + uninterruptible "D" state indefinitiley. + + When a hung task is detected, the kernel will print the + current stack trace (which you should report), but the + task will stay in uninterruptible state. If lockdep is + enabled then all held locks will also be reported. This + feature has negligible overhead. + +config BOOTPARAM_HUNG_TASK_PANIC + bool "Panic (Reboot) On Hung Tasks" + depends on DETECT_HUNG_TASK + help + Say Y here to enable the kernel to panic on "hung tasks", + which are bugs that cause the kernel to leave a task stuck + in uninterruptible "D" state. + + The panic can be used in combination with panic_timeout, + to cause the system to reboot automatically after a + hung task has been detected. This feature is useful for + high-availability systems that have uptime guarantees and + where a hung tasks must be resolved ASAP. + + Say N if unsure. + +config BOOTPARAM_HUNG_TASK_PANIC_VALUE + int + depends on DETECT_HUNG_TASK + range 0 1 + default 0 if !BOOTPARAM_HUNG_TASK_PANIC + default 1 if BOOTPARAM_HUNG_TASK_PANIC + config SCHED_DEBUG bool "Collect scheduler debugging info" depends on DEBUG_KERNEL && PROC_FS
Signature · 1 line
1.5.4.5
Quoted 7 lines · Mandeep Singh Baines
Quoted 4 lines · Ingo Molnar
Looks good in principle, but i suspect your patch submission should include the new
kernel/hung_task.cfile too ;-)Ingo
Doh. Thanks for reviewing. Here is the same patch with
hung_task.cincluded.
this now conflicts with the softlockup-panic fix you posted - and that fix has to go first as it's v2.6.29 material. (while the decoupling patch is 2.6.30 material)
So could you please send a patch against tip/master:
http://people.redhat.com/mingo/tip.git/README
Thanks,
Ingo
PATCH v4 1/1 softlockup: decouple hung tasks check from softlockup detection
Quoted 5 lines · Ingo Molnar
this now conflicts with the softlockup-panic fix you posted - and that fix has to go first as it's v2.6.29 material. (while the decoupling patch is 2.6.30 material)
Quoted 7 lines
So could you please send a patch against tip/master:
http://people.redhat.com/mingo/tip.git/README
Thanks,
Ingo
No problem. Rebased to tip/master.
---
Decoupling allows:
* hung tasks check to happen at very low priority
* hung tasks check and softlockup to be enabled/disabled independently at compile and/or run-time
* individual panic settings to be enabled disabled independently at compile and/or run-time
* softlockup threshold to be reduced without increasing hung tasks poll frequency (hung task check is expensive relative to softlock watchdog)
* hung task check to be zero over-head when disabled at run-time
--- include/linux/sched.h | 14 +++- kernel/Makefile | 1 + kernel/hung_task.c | 198 +++++++++++++++++++++++++++++++++++++++++++++++++ kernel/softlockup.c | 100 ------------------------- kernel/sysctl.c | 15 ++++- lib/Kconfig.debug | 38 ++++++++++ 6 files changed, 261 insertions(+), 105 deletions(-) create mode 100644 kernel/hung_task.c
include/linux/sched.h +10 -4
@@ -298,9 +298,6 @@ extern int proc_dosoftlockup_thresh(struct ctl_table *table, int write, struct file *filp, void __user *buffer, size_t *lenp, loff_t *ppos);extern unsigned int softlockup_panic; -extern unsigned long sysctl_hung_task_check_count; -extern unsigned long sysctl_hung_task_timeout_secs; -extern unsigned long sysctl_hung_task_warnings;extern int softlockup_thresh;#else static inline void softlockup_tick (void )
@@ -317,6 +314,15 @@ static inline void touch_all_softlockup_watchdogs(void) }#endif +#ifdef CONFIG_DETECT_HUNG_TASK +extern unsigned int sysctl_hung_task_panic; +extern unsigned long sysctl_hung_task_check_count; +extern unsigned long sysctl_hung_task_timeout_secs; +extern unsigned long sysctl_hung_task_warnings; +extern int proc_dohung_task_timeout_secs (struct ctl_table *table,int write, +struct file *filp,void __user *buffer, +size_t *lenp,loff_t *ppos); +#endif /* Attach to any functions which should be ignored in wchan output. */ #define __sched __attribute__((__section__(".sched.text")))
@@ -1240,7 +1246,7 @@ struct task_struct {/* ipc stuff */ struct sysv_sem sysvsem;#endif -#ifdef CONFIG_DETECT_SOFTLOCKUP +#ifdef CONFIG_DETECT_HUNG_TASK /* hung task detection */ unsigned long last_switch_timestamp;unsigned long last_switch_count;
kernel/Makefile +1 -0
@@ -74,6 +74,7 @@ obj-$(CONFIG_AUDIT_TREE) += audit_tree.o obj-$(CONFIG_KPROBES) += kprobes.o obj-$(CONFIG_KGDB) += kgdb.o obj-$(CONFIG_DETECT_SOFTLOCKUP) += softlockup.o +obj-$(CONFIG_DETECT_HUNG_TASK) += hung_task.o obj-$(CONFIG_GENERIC_HARDIRQS) += irq/ obj-$(CONFIG_SECCOMP) += seccomp.o obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o
kernel/hung_task.c new file +198 -0
198 lines not on this page · raw message
kernel/softlockup.c +0 -100
126 lines not on this page · raw message
kernel/sysctl.c +14 -1
@@ -806,6 +806,19 @@ static struct ctl_table kern_table[] = { .extra1 = &neg_one, .extra2 = &sixty, }, +#endif +#ifdef CONFIG_DETECT_HUNG_TASK + { + .ctl_name = CTL_UNNUMBERED, + .procname = "hung_task_panic", + .data = &sysctl_hung_task_panic, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = &proc_dointvec_minmax, + .strategy = &sysctl_intvec, + .extra1 = &zero, + .extra2 = &one, + }, { .ctl_name = CTL_UNNUMBERED, .procname = "hung_task_check_count",
@@ -821,7 +834,7 @@ static struct ctl_table kern_table[] = { .data = &sysctl_hung_task_timeout_secs, .maxlen = sizeof(unsigned long), .mode = 0644, - .proc_handler = &proc_doulongvec_minmax, + .proc_handler = &proc_dohung_task_timeout_secs, .strategy = &sysctl_intvec, }, {
lib/Kconfig.debug +38 -0
@@ -199,6 +199,44 @@ config BOOTPARAM_SOFTLOCKUP_PANIC_VALUE default 0 if !BOOTPARAM_SOFTLOCKUP_PANIC default 1 if BOOTPARAM_SOFTLOCKUP_PANIC +config DETECT_HUNG_TASK + bool "Detect Hung Tasks" + depends on DEBUG_KERNEL + default y + help + Say Y here to enable the kernel to detect "hung tasks", + which are bugs that cause the task to be stuck in + uninterruptible "D" state indefinitiley. + + When a hung task is detected, the kernel will print the + current stack trace (which you should report), but the + task will stay in uninterruptible state. If lockdep is + enabled then all held locks will also be reported. This + feature has negligible overhead. + +config BOOTPARAM_HUNG_TASK_PANIC + bool "Panic (Reboot) On Hung Tasks" + depends on DETECT_HUNG_TASK + help + Say Y here to enable the kernel to panic on "hung tasks", + which are bugs that cause the kernel to leave a task stuck + in uninterruptible "D" state. + + The panic can be used in combination with panic_timeout, + to cause the system to reboot automatically after a + hung task has been detected. This feature is useful for + high-availability systems that have uptime guarantees and + where a hung tasks must be resolved ASAP. + + Say N if unsure. + +config BOOTPARAM_HUNG_TASK_PANIC_VALUE + int + depends on DETECT_HUNG_TASK + range 0 1 + default 0 if !BOOTPARAM_HUNG_TASK_PANIC + default 1 if BOOTPARAM_HUNG_TASK_PANIC + config SCHED_DEBUG bool "Collect scheduler debugging info" depends on DEBUG_KERNEL && PROC_FS
Signature · 1 line
1.5.4.5
Ingo Molnar
AppliedQuoted 8 lines · Mandeep Singh Baines
Quoted 7 lines · Ingo Molnar
this now conflicts with the softlockup-panic fix you posted - and that fix has to go first as it's v2.6.29 material. (while the decoupling patch is 2.6.30 material)
So could you please send a patch against tip/master:
Quoted 7 lines
Quoted 5 lines
http://people.redhat.com/mingo/tip.git/README
Thanks,
Ingo
No problem. Rebased to tip/master.
applied to tip/core/softlockup - thanks Mandeep!
Ingo
Quoted 3 lines · Ingo Molnar
Quoted 1 line
No problem. Rebased to tip/master.
applied to tip/core/softlockup - thanks Mandeep!
doesnt build with !SOFTLOCKUP:
kernel/fork.c:1049: error: 'struct task_struct' has no member named 'last_switch_count' kernel/fork.c:1050: error: 'struct task_struct' has no member named 'last_switch_timestamp'
Could you send a delta fix patch relative to -v4 please? Thanks,
Ingo
Quoted 10 lines · Mandeep Singh Baines
Decoupling allows:
* hung tasks check to happen at very low priority
* hung tasks check and softlockup to be enabled/disabled independently at compile and/or run-time
* individual panic settings to be enabled disabled independently at compile and/or run-time
* softlockup threshold to be reduced without increasing hung tasks poll frequency (hung task check is expensive relative to softlock watchdog)
* hung task check to be zero over-head when disabled at run-time
Quoted 7 lines
--- include/linux/sched.h | 14 +++++-- kernel/Makefile | 1 + kernel/softlockup.c | 102 ------------------------------------------------- kernel/sysctl.c | 15 +++++++- lib/Kconfig.debug | 38 ++++++++++++++++++ 5 files changed, 63 insertions(+), 107 deletions(-)
kernel/hung_task.c missing?
Hannes
Hi Mandeep,
2009/1/12 Mandeep Singh Baines <[email protected]>:
Quoted 12 lines
kernel/softlockup.c +0
-12-/* -* Check whether a TASK_UNINTERRUPTIBLE does not get woken up for -* a really long time (120 seconds). If that happens, print out -* a warning. -*/ -static void check_hung_uninterruptible_tasks (int this_cpu) -{ -int max_count = sysctl_hung_task_check_count; -unsigned long now =get_timestamp (this_cpu); -struct task_struct *g, *t; - -/*
kernel/softlockup.c void softlockup_tick(void) · line 235
Quoted 10 lines
kernel/softlockup.c +0
-10-* If the system crashed already then all bets are off, -* do not report extra hung tasks: -*/ -if (test_taint (TAINT_DIE ) || did_panic) -return ; - -read_lock (&tasklist_lock); - do_each_thread(g, t) { -if (!--max_count) - goto unlock;
Instead of having this arbitrary limit of tasks, why not just lurk the need_resched() and then schedule if it needs too.
I know that sounds a bit racy, because you will have to release the tasklist_lock and
a lot of things can happen in the task list until you become resched. But you can do a get_task_struct() on g and t before your thread is going to sleep and then put them
when it is awaken.
Perhaps some tasks will disappear or be appended in the list before g and t, but that doesn't really matter:
if they disappear, they didn't lockup, and if they were appended, they are not enough cold to be analyzed :-)
This way you can drop the arbitrary limit of task number given by the user....
Frederic.
Quoted 142 lines
kernel/softlockup.c +0
-26-/* use "==" to skip the TASK_KILLABLE tasks waiting on NFS */ -if (t->state ==TASK_UNINTERRUPTIBLE ) -check_hung_task (t, now); - }while_each_thread (g, t); - unlock: -read_unlock (&tasklist_lock); -} - -/* * The watchdog thread - runs every second and touches the timestamp. */ static int watchdog (void *__bind_cpu) {struct sched_param param = { .sched_priority =MAX_RT_PRIO -1 }; -int this_cpu = (long )__bind_cpu;sched_setscheduler (current,SCHED_FIFO , ¶m);@@ -267,11 +181,6 @@ static int watchdog(void *__bind_cpu)if (kthread_should_stop ())break ; -if (this_cpu == check_cpu) { -if (sysctl_hung_task_timeout_secs) -check_hung_uninterruptible_tasks (this_cpu); - } -set_current_state (TASK_INTERRUPTIBLE ); }__set_current_state (TASK_RUNNING );@@ -303,20 +212,9 @@ cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)break ;case CPU_ONLINE :case CPU_ONLINE_FROZEN : - check_cpu =any_online_cpu (cpu_online_map);wake_up_process (per_cpu (watchdog_task, hotcpu));break ;#ifdef CONFIG_HOTPLUG_CPU -case CPU_DOWN_PREPARE : -case CPU_DOWN_PREPARE_FROZEN : -if (hotcpu == check_cpu) { -cpumask_t temp_cpu_online_map = cpu_online_map; - -cpu_clear (hotcpu, temp_cpu_online_map); - check_cpu =any_online_cpu (temp_cpu_online_map); - } -break ; -case CPU_UP_CANCELED :case CPU_UP_CANCELED_FROZEN :if (!per_cpu (watchdog_task, hotcpu))kernel/sysctl.c +14
-1@@ -771,6 +771,19 @@ static struct ctl_table kern_table[] = { .extra1 = &neg_one, .extra2 = &sixty, }, +#endif +#ifdef CONFIG_HUNG_TASK + { + .ctl_name = CTL_UNNUMBERED, + .procname = "hung_task_panic", + .data = &sysctl_hung_task_panic, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = &proc_dointvec_minmax, + .strategy = &sysctl_intvec, + .extra1 = &zero, + .extra2 = &one, + }, { .ctl_name = CTL_UNNUMBERED, .procname = "hung_task_check_count",@@ -786,7 +799,7 @@ static struct ctl_table kern_table[] = { .data = &sysctl_hung_task_timeout_secs, .maxlen = sizeof(unsigned long), .mode = 0644, - .proc_handler = &proc_doulongvec_minmax, + .proc_handler = &proc_dohung_task_timeout_secs, .strategy = &sysctl_intvec, }, {lib/Kconfig.debug +38
-1@@ -186,6 +186,44 @@ config BOOTPARAM_SOFTLOCKUP_PANIC_VALUE default 0 if !BOOTPARAM_SOFTLOCKUP_PANIC default 1 if BOOTPARAM_SOFTLOCKUP_PANIC +config DETECT_HUNG_TASK + bool "Detect Hung Tasks" + depends on DEBUG_KERNEL + default y + help + Say Y here to enable the kernel to detect "hung tasks", + which are bugs that cause the task to be stuck in + uninterruptible "D" state indefinitiley. + + When a hung task is detected, the kernel will print the + current stack trace (which you should report), but the + task will stay in uninterruptible state. If lockdep is + enabled then all held locks will also be reported. This + feature has negligible overhead. + +config BOOTPARAM_HUNG_TASK_PANIC + bool "Panic (Reboot) On Hung Tasks" + depends on DETECT_HUNG_TASK + help + Say Y here to enable the kernel to panic on "hung tasks", + which are bugs that cause the kernel to leave a task stuck + in uninterruptible "D" state. + + The panic can be used in combination with panic_timeout, + to cause the system to reboot automatically after a + hung task has been detected. This feature is useful for + high-availability systems that have uptime guarantees and + where a hung tasks must be resolved ASAP. + + Say N if unsure. + +config BOOTPARAM_HUNG_TASK_PANIC_VALUE + int + depends on DETECT_HUNG_TASK + range 0 1 + default 0 if !BOOTPARAM_HUNG_TASK_PANIC + default 1 if BOOTPARAM_HUNG_TASK_PANIC + config SCHED_DEBUG bool "Collect scheduler debugging info" depends on DEBUG_KERNEL && PROC_FS --1.5.4.5
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
Hi Frédéric,
Quoted 20 lines · Frédéric Weisbecker
kernel/softlockup.c void softlockup_tick(void) · line 235
Quoted 4 lines
kernel/softlockup.c +0
-4-read_lock (&tasklist_lock); - do_each_thread(g, t) { -if (!--max_count) - goto unlock;Instead of having this arbitrary limit of tasks, why not just lurk the
need_resched()and then schedule if it needs too.I know that sounds a bit racy, because you will have to release the
tasklist_lockand
a lot of things can happen in the task list until you become resched. But you can do aget_task_struct()on g and t before your thread is going to sleep and then put them
when it is awaken.
Perhaps some tasks will disappear or be appended in the list before g and t, but that doesn't really matter:
if they disappear, they didn't lockup, and if they were appended, they are not enough cold to be analyzed :-)
Quoted 4 lines
This way you can drop the arbitrary limit of task number given by the user....
Frederic.
Would be nice to remove the limit. But I don't think get_task_struct() can be used to prevent a task from being unlinked from the task list. It only prevents the task_struct from being freed. So hung_task could end up holding a reference to an unlinked task after it returns from schedule().
That doesn't mean what you are suggesting can't be implemented. Just means that the case of the held task being unlinked needs to be handled.
Regards,
Mandeep
Quoted 33 lines · Mandeep Singh Baines
Hi Frédéric,
Quoted 20 lines · Frédéric Weisbecker
kernel/softlockup.c void softlockup_tick(void) · line 235
Quoted 4 lines
kernel/softlockup.c +0
-4-read_lock (&tasklist_lock); - do_each_thread(g, t) { -if (!--max_count) - goto unlock;Instead of having this arbitrary limit of tasks, why not just lurk the
need_resched()and then schedule if it needs too.I know that sounds a bit racy, because you will have to release the
tasklist_lockand
a lot of things can happen in the task list until you become resched. But you can do aget_task_struct()on g and t before your thread is going to sleep and then put them
when it is awaken.
Perhaps some tasks will disappear or be appended in the list before g and t, but that doesn't really matter:
if they disappear, they didn't lockup, and if they were appended, they are not enough cold to be analyzed :-)Quoted 4 lines
This way you can drop the arbitrary limit of task number given by the user....
Frederic.
Would be nice to remove the limit. But I don't think
get_task_struct()can be used to prevent a task from being unlinked from the task list. It only prevents thetask_structfrom being freed. Sohung_taskcould end up holding a reference to an unlinked task after it returns fromschedule().
Quoted 5 lines
That doesn't mean what you are suggesting can't be implemented. Just means that the case of the held task being unlinked needs to be handled.
Regards,
Mandeep
Hmm, you're right.
Why not testing 1024 tasks, then check need_resched and if you sleep and the task becomes unlinked (there are few chances) so... that's not a big deal actually, you will have better chances on the next check :-)
I think that's a bit important since you are more likely to see a soft-lockup if you have a lot of tasks.