kernel/hung_task.c: allow to set checking interval separately from timeout
Subsystems (6)
PATCH v2 1/1 kernel/hung_task.c: allow to set checking interval separately from timeout
Currently task hung checking interval is equal to timeout, as the result hung is detected anywhere between timeout and 2*timeout. This is fine for most interactive environments, but this hurts automated testing setups (syzbot). In an automated setup we need to strictly order CPU lockup < RCU stall < workqueue lockup < task hung < silent loss, so that RCU stall is not detected as task hung and task hung is not detected as silent machine loss. The large variance in task hung detection timeout requires setting silent machine loss timeout to a very large value (e.g. if task hung is 3 mins, then silent loss need to be set to ~7 mins). The additional 3 minutes significantly reduce testing efficiency because usually we crash kernel within a minute, and this can add hours to bug localization process as it needs to do dozens of tests.
Allow setting checking interval separately from timeout. This allows to set timeout to, say, 3 minutes, but checking interval to 10 secs.
The interval is controlled via a new hung_task_check_interval_secs sysctl, similar to the existing hung_task_timeout_secs sysctl. The default value of 0 results in the current behavior:
checking interval is equal to timeout.
---
Changes since v1:
- add entry to Documentation/sysctl/kernel.txt
- rename hung_task_period_secs sysctl to hung_task_check_interval_secs
--- Documentation/sysctl/kernel.txt | 15 ++++++++++++++- include/linux/sched.h | 1 + include/linux/sched/sysctl.h | 1 + kernel/fork.c | 1 + kernel/hung_task.c | 15 ++++++++++++++- kernel/sysctl.c | 8 ++++++++ 6 files changed, 39 insertions(+), 2 deletions(-)
Documentation/sysctl/kernel.txt +14 -1
@@ -38,6 +38,7 @@ show up in /proc/sys/kernel: - hung_task_panic - hung_task_check_count - hung_task_timeout_secs +- hung_task_check_interval_secs - hung_task_warnings - kexec_load_disabled - kptr_restrict
@@ -354,7 +355,7 @@ This file shows up if CONFIG_DETECT_HUNG_TASK is enabled. hung_task_timeout_secs: -Check interval. When a task in D state did not get scheduled +When a task in D state did not get scheduled for more than this value report a warning. This file shows up if CONFIG_DETECT_HUNG_TASK is enabled.
@@ -363,6 +364,18 @@ Possible values to set are in range {0..LONG_MAX/HZ}. ============================================================== +hung_task_check_interval_secs: + +Hung task check interval. If hung task checking is enabled +(see hung_task_timeout_secs), the check is done every +hung_task_check_interval_secs seconds. +This file shows up if CONFIG_DETECT_HUNG_TASK is enabled. + +0 (default): means use hung_task_timeout_secs as checking interval. +Possible values to set are in range {0..LONG_MAX/HZ}. + +============================================================== + hung_task_warnings: The maximum number of warnings to report. During a check interval
include/linux/sched.h +1 -0
@@ -849,6 +849,7 @@ struct task_struct {#endif #ifdef CONFIG_DETECT_HUNG_TASK unsigned long last_switch_count; +unsigned long last_switch_time;#endif /* Filesystem information: */ struct fs_struct *fs;
include/linux/sched/sysctl.h +1 -0
@@ -10,6 +10,7 @@ struct ctl_table;extern int sysctl_hung_task_check_count;extern unsigned int sysctl_hung_task_panic;extern unsigned long sysctl_hung_task_timeout_secs; +extern unsigned long sysctl_hung_task_check_interval_secs;extern int sysctl_hung_task_warnings; extern int proc_dohung_task_timeout_secs(struct ctl_table *table, int write, void __user *buffer,
kernel/fork.c +1 -0
@@ -1262,6 +1262,7 @@ static int copy_mm(unsigned long clone_flags, struct task_struct *tsk) tsk->nvcsw = tsk->nivcsw =0 ;#ifdef CONFIG_DETECT_HUNG_TASK tsk->last_switch_count = tsk->nvcsw + tsk->nivcsw; + tsk->last_switch_time =0 ;#endif tsk->mm =NULL ;
kernel/hung_task.c +14 -1
@@ -40,6 +40,11 @@ int __read_mostly sysctl_hung_task_check_count = PID_MAX_LIMIT;*/ unsigned long __read_mostly sysctl_hung_task_timeout_secs =CONFIG_DEFAULT_HUNG_TASK_TIMEOUT ; +/* +* Zero (default value) means use sysctl_hung_task_timeout_secs: +*/ +unsigned long __read_mostly sysctl_hung_task_check_interval_secs; +int __read_mostly sysctl_hung_task_warnings =10 ;static int __read_mostly did_panic;
@@ -98,8 +103,11 @@ static void check_hung_task(struct task_struct *t, unsigned long timeout)if (switch_count != t->last_switch_count) { t->last_switch_count = switch_count; + t->last_switch_time = jiffies;return ; } +if (time_is_after_jiffies (t->last_switch_time + timeout *HZ )) +return ;trace_sched_process_hang (t);
@@ -245,8 +253,13 @@ static int watchdog(void *dummy)for ( ; ; ) {unsigned long timeout = sysctl_hung_task_timeout_secs; -long t =hung_timeout_jiffies (hung_last_checked, timeout); +unsigned long interval = sysctl_hung_task_check_interval_secs; +long t; +if (interval ==0 ) + interval = timeout; + interval = min_t(unsigned long, interval, timeout); + t =hung_timeout_jiffies (hung_last_checked, interval);if (t <=0 ) {if (!atomic_xchg (&reset_hung_task,0 ))check_hung_uninterruptible_tasks (timeout);
kernel/sysctl.c +8 -0
@@ -1098,6 +1098,14 @@ static struct ctl_table kern_table[] = { .proc_handler = proc_dohung_task_timeout_secs, .extra2 = &hung_task_timeout_max, }, + { + .procname = "hung_task_check_interval_secs", + .data = &sysctl_hung_task_check_interval_secs, + .maxlen = sizeof(unsigned long), + .mode = 0644, + .proc_handler = proc_dohung_task_timeout_secs, + .extra2 = &hung_task_timeout_max, + }, { .procname = "hung_task_warnings", .data = &sysctl_hung_task_warnings,
Signature · 1 line
2.18.0.rc1.242.g61856ae69a-goog
Andrew Morton
maintainerQuoted 14 lines · On Mon, 11 Jun 2018 13:10:04 +0200 Dmitry Vyukov <[email protected]> wrote:
Currently task hung checking interval is equal to timeout, as the result hung is detected anywhere between timeout and 2*timeout. This is fine for most interactive environments, but this hurts automated testing setups (syzbot). In an automated setup we need to strictly order CPU lockup < RCU stall < workqueue lockup < task hung < silent loss, so that RCU stall is not detected as task hung and task hung is not detected as silent machine loss. The large variance in task hung detection timeout requires setting silent machine loss timeout to a very large value (e.g. if task hung is 3 mins, then silent loss need to be set to ~7 mins). The additional 3 minutes significantly reduce testing efficiency because usually we crash kernel within a minute, and this can add hours to bug localization process as it needs to do dozens of tests.
Quoted 8 lines
Allow setting checking interval separately from timeout. This allows to set timeout to, say, 3 minutes, but checking interval to 10 secs.
The interval is controlled via a new
hung_task_check_interval_secssysctl, similar to the existinghung_task_timeout_secssysctl. The default value of 0 results in the current behavior:
checking interval is equal to timeout.
I suppose we shoold do this:
kernel/sysctl.c renamed from kernel/sysctl.c~kernel-hung_taskc-allow-to-set-checking-interval-separately-from-timeout-fix +4 -1
@@ -145,7 +145,10 @@ static int minolduid;static int ngroups_max =NGROUPS_MAX ;static const int cap_last_cap =CAP_LAST_CAP ; -/*this is needed for proc_doulongvec_minmax of sysctl_hung_task_timeout_secs */ +/* +* This is needed for proc_doulongvec_minmax of sysctl_hung_task_timeout_secs +* and hung_task_check_interval_secs +*/ #ifdef CONFIG_DETECT_HUNG_TASK static unsigned long hung_task_timeout_max = (LONG_MAX /HZ );#endif
Quoted 20 lines · Andrew Morton
Quoted 18 lines · On Mon, 11 Jun 2018 13:10:04 +0200 Dmitry Vyukov <[email protected]> wrote:
Currently task hung checking interval is equal to timeout, as the result hung is detected anywhere between timeout and 2*timeout. This is fine for most interactive environments, but this hurts automated testing setups (syzbot). In an automated setup we need to strictly order CPU lockup < RCU stall < workqueue lockup < task hung < silent loss, so that RCU stall is not detected as task hung and task hung is not detected as silent machine loss. The large variance in task hung detection timeout requires setting silent machine loss timeout to a very large value (e.g. if task hung is 3 mins, then silent loss need to be set to ~7 mins). The additional 3 minutes significantly reduce testing efficiency because usually we crash kernel within a minute, and this can add hours to bug localization process as it needs to do dozens of tests.
Allow setting checking interval separately from timeout. This allows to set timeout to, say, 3 minutes, but checking interval to 10 secs.
Quoted 6 lines
Quoted 4 lines
The interval is controlled via a new
hung_task_check_interval_secssysctl, similar to the existinghung_task_timeout_secssysctl. The default value of 0 results in the current behavior:
checking interval is equal to timeout.I suppose we shoold do this:
Hi Andrew,
I see you added the patch and fixup to mm tree. Do you want me to resend v3 with the fixup included, or how does this work?
Thanks
Quoted 14 lines
kernel/sysctl.c renamed from kernel/sysctl.c~kernel-hung_taskc-allow-to-set-checking-interval-separately-from-timeout-fix +4
-1@@ -145,7 +145,10 @@ static int minolduid;static int ngroups_max =NGROUPS_MAX ;static const int cap_last_cap =CAP_LAST_CAP ; -/*this is needed for proc_doulongvec_minmax of sysctl_hung_task_timeout_secs */ +/* +* This is needed for proc_doulongvec_minmax of sysctl_hung_task_timeout_secs +* and hung_task_check_interval_secs +*/ #ifdef CONFIG_DETECT_HUNG_TASK static unsigned long hung_task_timeout_max = (LONG_MAX /HZ );#endif
Andrew Morton
maintainerQuoted 7 lines · On Wed, 20 Jun 2018 07:23:49 +0200 Dmitry Vyukov <[email protected]> wrote:
Quoted 1 line
I suppose we shoold do this:
Hi Andrew,
I see you added the patch and fixup to mm tree. Do you want me to resend v3 with the fixup included, or how does this work?
Is OK. I queued both patches and shall consolidate them into a single one prior to sending upstream to Linus.