kernel/hung_task.c: disable on suspend
RFC PATCH 1/1 kernel/hung_task.c: disable on suspend
It is possible to observe hung_task complaints when system goes to suspend-to-idle state:
PM: Syncing filesystems ... done.
Freezing user space processes ... (elapsed 0.001 seconds) done. OOM killer disabled.
Freezing remaining freezable tasks ... (elapsed 0.002 seconds) done. sd 0:0:0:0: [sda] Synchronizing SCSI cache
INFO: task bash:1569 blocked for more than 120 seconds.
Not tainted 4.19.0-rc3_+ #687
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
bash D 0 1569 604 0x00000000
Call Trace:
? __schedule+0x1fe/0x7e0
schedule+0x28/0x80
suspend_devices_and_enter+0x4ac/0x750
pm_suspend+0x2c0/0x310Register a PM notifier to disable the detector on suspend and re-enable back on wakeup.
---
RFC: It really makes me wonder why nobody reported this before, makes
me think I'm missing something.
--- kernel/hung_task.c | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-)
kernel/hung_task.c +25 -1
@@ -15,6 +15,7 @@#include <linux/lockdep.h> #include <linux/export.h> #include <linux/sysctl.h> +#include <linux/suspend.h> #include <linux/utsname.h> #include <linux/sched/signal.h> #include <linux/sched/debug.h>
@@ -242,6 +243,24 @@ void reset_hung_task_detector(void) }EXPORT_SYMBOL_GPL (reset_hung_task_detector); +static bool hung_detector_suspended; + +static int hungtask_pm_notify (struct notifier_block *self, +unsigned long action,void *hcpu) +{ +switch (action) { +case PM_SUSPEND_PREPARE : + hung_detector_suspended = true; +break ; +case PM_POST_SUSPEND : + hung_detector_suspended = false; +break ; +default : +break ; + } +return NOTIFY_OK ; +} +/* * kthread which checks for tasks stuck in D state */
@@ -261,7 +280,8 @@ static int watchdog(void *dummy) interval = min_t(unsigned long, interval, timeout); t =hung_timeout_jiffies (hung_last_checked, interval);if (t <=0 ) { -if (!atomic_xchg (&reset_hung_task,0 )) +if (!atomic_xchg (&reset_hung_task,0 ) && + !hung_detector_suspended)check_hung_uninterruptible_tasks (timeout); hung_last_checked = jiffies;continue ;
@@ -275,6 +295,10 @@ static int watchdog(void *dummy)static int __init hung_task_init (void ) {atomic_notifier_chain_register (&panic_notifier_list, &panic_block); + +/* Disable hung task detector on suspend */ +pm_notifier (hungtask_pm_notify,0 ); + watchdog_task =kthread_run (watchdog,NULL ,"khungtaskd" );return 0 ;
Signature · 1 line
2.14.4
Quoted 9 lines · Vitaly Kuznetsov
It is possible to observe
hung_taskcomplaints when system goes to suspend-to-idle state:PM: Syncing filesystems ... done.
Freezing user space processes ... (elapsed 0.001 seconds) done. OOM killer disabled.
Freezing remaining freezable tasks ... (elapsed 0.002 seconds) done. sd 0:0:0:0: [sda] Synchronizing SCSI cache
Quoted 9 lines
INFO: task bash:1569 blocked for more than 120 seconds.
Not tainted 4.19.0-rc3_+ #687
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
bash D 0 1569 604 0x00000000
Call Trace:
? __schedule+0x1fe/0x7e0
schedule+0x28/0x80
suspend_devices_and_enter+0x4ac/0x750
pm_suspend+0x2c0/0x310This actually is a good catch, but the problem is related to what happens to the monotonic clock during suspend to idle.
The clock issue needs to be addressed anyway IMO and then this problem will go away automatically.
Quoted 24 lines
Register a PM notifier to disable the detector on suspend and re-enable back on wakeup.
Signed-off-by Vitaly Kuznetsov <[email protected]>---
RFC: It really makes me wonder why nobody reported this before, makes
me think I'm missing something.--- kernel/hung_task.c | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-)kernel/hung_task.c +1
-0@@ -15,6 +15,7 @@#include <linux/lockdep.h> #include <linux/export.h> #include <linux/sysctl.h> +#include <linux/suspend.h> #include <linux/utsname.h> #include <linux/sched/signal.h> #include <linux/sched/debug.h>
kernel/hung_task.c void reset_hung_task_detector(void) · line 252
Quoted 10 lines
kernel/hung_task.c +7
-0@@ -242,6 +243,24 @@ void reset_hung_task_detector(void) }EXPORT_SYMBOL_GPL (reset_hung_task_detector); +static bool hung_detector_suspended; + +static int hungtask_pm_notify (struct notifier_block *self, +unsigned long action,void *hcpu) +{ +switch (action) { +case PM_SUSPEND_PREPARE :
You'd want PM_HIBERNATION_PREPARE here too I think.
kernel/hung_task.c void reset_hung_task_detector(void) · line 255
Quoted 3 lines
kernel/hung_task.c +3
-0+ hung_detector_suspended = true; +break ; +case PM_POST_SUSPEND :
And PM_POST_HIBERNATION here for consistency.
Quoted 35 lines
kernel/hung_task.c +14
-2+ hung_detector_suspended = false; +break ; +default : +break ; + } +return NOTIFY_OK ; +} +/* * kthread which checks for tasks stuck in D state */ @@ -261,7 +280,8 @@ static int watchdog(void *dummy) interval = min_t(unsigned long, interval, timeout); t =hung_timeout_jiffies (hung_last_checked, interval);if (t <=0 ) { -if (!atomic_xchg (&reset_hung_task,0 )) +if (!atomic_xchg (&reset_hung_task,0 ) && + !hung_detector_suspended)check_hung_uninterruptible_tasks (timeout); hung_last_checked = jiffies;continue ;@@ -275,6 +295,10 @@ static int watchdog(void *dummy)static int __init hung_task_init (void ) {atomic_notifier_chain_register (&panic_notifier_list, &panic_block); + +/* Disable hung task detector on suspend */ +pm_notifier (hungtask_pm_notify,0 ); + watchdog_task =kthread_run (watchdog,NULL ,"khungtaskd" );return 0 ; --2.14.4
Quoted 20 lines · Rafael J. Wysocki
Quoted 9 lines · Vitaly Kuznetsov
It is possible to observe
hung_taskcomplaints when system goes to suspend-to-idle state:PM: Syncing filesystems ... done.
Freezing user space processes ... (elapsed 0.001 seconds) done. OOM killer disabled.
Freezing remaining freezable tasks ... (elapsed 0.002 seconds) done. sd 0:0:0:0: [sda] Synchronizing SCSI cacheQuoted 9 lines
INFO: task bash:1569 blocked for more than 120 seconds. Not tainted 4.19.0-rc3_+ #687 "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. bash D 0 1569 604 0x00000000 Call Trace: ? __schedule+0x1fe/0x7e0 schedule+0x28/0x80 suspend_devices_and_enter+0x4ac/0x750 pm_suspend+0x2c0/0x310
Quoted 5 lines
This actually is a good catch, but the problem is related to what happens to the monotonic clock during suspend to idle.
The clock issue needs to be addressed anyway IMO and then this problem will go away automatically.
Do I understand it correctly that the suggestion is to fully suspend monothonic clock in s2idle (and don't advance it after resume)?
Quoted 38 lines
Quoted 25 lines
Register a PM notifier to disable the detector on suspend and re-enable back on wakeup.
Signed-off-by Vitaly Kuznetsov <[email protected]>---
RFC: It really makes me wonder why nobody reported this before, makes
me think I'm missing something.--- kernel/hung_task.c | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-)kernel/hung_task.c +1
-0@@ -15,6 +15,7 @@#include <linux/lockdep.h> #include <linux/export.h> #include <linux/sysctl.h> +#include <linux/suspend.h> #include <linux/utsname.h> #include <linux/sched/signal.h> #include <linux/sched/debug.h> kernel/hung_task.c void reset_hung_task_detector(void) · line 252
Quoted 10 lines
kernel/hung_task.c +7
-0@@ -242,6 +243,24 @@ void reset_hung_task_detector(void) }EXPORT_SYMBOL_GPL (reset_hung_task_detector); +static bool hung_detector_suspended; + +static int hungtask_pm_notify (struct notifier_block *self, +unsigned long action,void *hcpu) +{ +switch (action) { +case PM_SUSPEND_PREPARE :You'd want
PM_HIBERNATION_PREPAREhere too I think.
Quoted 6 lines
kernel/hung_task.c void reset_hung_task_detector(void) · line 255
Quoted 3 lines
kernel/hung_task.c +3
-0+ hung_detector_suspended = true; +break ; +case PM_POST_SUSPEND :And
PM_POST_HIBERNATIONhere for consistency.
Sure, will do in v1.
Quoted 35 lines
Quoted 35 lines
kernel/hung_task.c +14
-2+ hung_detector_suspended = false; +break ; +default : +break ; + } +return NOTIFY_OK ; +} +/* * kthread which checks for tasks stuck in D state */ @@ -261,7 +280,8 @@ static int watchdog(void *dummy) interval = min_t(unsigned long, interval, timeout); t =hung_timeout_jiffies (hung_last_checked, interval);if (t <=0 ) { -if (!atomic_xchg (&reset_hung_task,0 )) +if (!atomic_xchg (&reset_hung_task,0 ) && + !hung_detector_suspended)check_hung_uninterruptible_tasks (timeout); hung_last_checked = jiffies;continue ;@@ -275,6 +295,10 @@ static int watchdog(void *dummy)static int __init hung_task_init (void ) {atomic_notifier_chain_register (&panic_notifier_list, &panic_block); + +/* Disable hung task detector on suspend */ +pm_notifier (hungtask_pm_notify,0 ); + watchdog_task =kthread_run (watchdog,NULL ,"khungtaskd" );return 0 ; --2.14.4
Signature · 1 line
Vitaly
Quoted 23 lines · Vitaly Kuznetsov
Quoted 20 lines · Rafael J. Wysocki
Quoted 18 lines · Vitaly Kuznetsov
It is possible to observe
hung_taskcomplaints when system goes to suspend-to-idle state:PM: Syncing filesystems ... done.
Freezing user space processes ... (elapsed 0.001 seconds) done. OOM killer disabled.
Freezing remaining freezable tasks ... (elapsed 0.002 seconds) done. sd 0:0:0:0: [sda] Synchronizing SCSI cacheINFO: task bash:1569 blocked for more than 120 seconds. Not tainted 4.19.0-rc3_+ #687 "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. bash D 0 1569 604 0x00000000 Call Trace: ? __schedule+0x1fe/0x7e0 schedule+0x28/0x80 suspend_devices_and_enter+0x4ac/0x750 pm_suspend+0x2c0/0x310
Quoted 8 lines
Quoted 5 lines
This actually is a good catch, but the problem is related to what happens to the monotonic clock during suspend to idle.
The clock issue needs to be addressed anyway IMO and then this problem will go away automatically.
Do I understand it correctly that the suggestion is to fully suspend monothonic clock in s2idle (and don't advance it after resume)?
Something like that.
We already do that to some extent, but kind of with the assumption that it will not grow too much in s2idle_loop(), but it may actually grow too much in there, so we need to compensate for that to make the s2idle behavior reflect the suspend-to-RAM one.
I have a patch for that, will post it shortly.
Thanks,
Rafael
Quoted 21 lines · Rafael J. Wysocki
Quoted 9 lines · Vitaly Kuznetsov
It is possible to observe
hung_taskcomplaints when system goes to suspend-to-idle state:PM: Syncing filesystems ... done.
Freezing user space processes ... (elapsed 0.001 seconds) done. OOM killer disabled.
Freezing remaining freezable tasks ... (elapsed 0.002 seconds) done. sd 0:0:0:0: [sda] Synchronizing SCSI cacheQuoted 9 lines
INFO: task bash:1569 blocked for more than 120 seconds. Not tainted 4.19.0-rc3_+ #687 "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. bash D 0 1569 604 0x00000000 Call Trace: ? __schedule+0x1fe/0x7e0 schedule+0x28/0x80 suspend_devices_and_enter+0x4ac/0x750 pm_suspend+0x2c0/0x310
Quoted 5 lines
This actually is a good catch, but the problem is related to what happens to the monotonic clock during suspend to idle.
The clock issue needs to be addressed anyway IMO and then this problem will go away automatically.
I don't understand your discussion with Vitaly, but shouldn't we make khungtaskd thread freezable anyway?
Oleg.
kernel/hung_task.c +3 -2
@@ -185,7 +185,7 @@ static void check_hung_uninterruptible_t hung_task_show_lock = false;rcu_read_lock ();for_each_process_thread (g, t) { -if (!max_count--) +if (!max_count-- ||freezing (current)) goto unlock;if (!--batch_count) { batch_count =HUNG_TASK_BATCHING ;
@@ -249,6 +249,7 @@ static int watchdog(void *dummy) {unsigned long hung_last_checked = jiffies; +set_freezable ();set_user_nice (current,0 );for ( ; ; ) {
@@ -266,7 +267,7 @@ static int watchdog(void *dummy) hung_last_checked = jiffies;continue ; } -schedule_timeout_interruptible (t); +freezable_schedule_timeout_interruptible (t); }return 0 ;
Quoted 23 lines · Oleg Nesterov
Quoted 21 lines · Rafael J. Wysocki
Quoted 18 lines · Vitaly Kuznetsov
It is possible to observe
hung_taskcomplaints when system goes to suspend-to-idle state:PM: Syncing filesystems ... done.
Freezing user space processes ... (elapsed 0.001 seconds) done. OOM killer disabled.
Freezing remaining freezable tasks ... (elapsed 0.002 seconds) done. sd 0:0:0:0: [sda] Synchronizing SCSI cacheINFO: task bash:1569 blocked for more than 120 seconds. Not tainted 4.19.0-rc3_+ #687 "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. bash D 0 1569 604 0x00000000 Call Trace: ? __schedule+0x1fe/0x7e0 schedule+0x28/0x80 suspend_devices_and_enter+0x4ac/0x750 pm_suspend+0x2c0/0x310
Quoted 8 lines
Quoted 5 lines
This actually is a good catch, but the problem is related to what happens to the monotonic clock during suspend to idle.
The clock issue needs to be addressed anyway IMO and then this problem will go away automatically.
I don't understand your discussion with Vitaly, but shouldn't we make khungtaskd thread freezable anyway?
Well, if the Vitaly's patch is applied, that shouldn't be necessary and I wouldn't like to add more freezable kernel threads if that's avoidable TBH.
Cheers,
Rafael