Skip to contentSkip to conversation

kernel/hung_task.c: disable on suspend

RFC PATCH In review Vitaly Kuznetsov 3 participants 6 messages Sep 12, 2018 – Sep 13, 2018 Hung task detector

lore
Share & tools
lore
#0 lore raw

RFC PATCH 1/1 kernel/hung_task.c: disable on suspend

It is possible to observe hung_task complaints when system goes to suspend-to-idle state:

PM: Syncing filesystems ... done.
Freezing user space processes ... (elapsed 0.001 seconds) done. OOM killer disabled.
Freezing remaining freezable tasks ... (elapsed 0.002 seconds) done. sd 0:0:0:0: [sda] Synchronizing SCSI cache

 INFO: task bash:1569 blocked for more than 120 seconds.
       Not tainted 4.19.0-rc3_+ #687
 "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
 bash            D    0  1569    604 0x00000000
 Call Trace:
  ? __schedule+0x1fe/0x7e0
  schedule+0x28/0x80
  suspend_devices_and_enter+0x4ac/0x750
  pm_suspend+0x2c0/0x310

Register a PM notifier to disable the detector on suspend and re-enable back on wakeup.

---
RFC: It really makes me wonder why nobody reported this before, makes
me think I'm missing something.

---
 kernel/hung_task.c | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)
kernel/hung_task.c +25 -1
index b9132d1269ef..d75f288c016f 100644
@@ -15,6 +15,7 @@
 #include <linux/lockdep.h>
 #include <linux/export.h>
 #include <linux/sysctl.h>
+#include <linux/suspend.h>
 #include <linux/utsname.h>
 #include <linux/sched/signal.h>
 #include <linux/sched/debug.h>
@@ -242,6 +243,24 @@ void reset_hung_task_detector(void)
 }
 EXPORT_SYMBOL_GPL(reset_hung_task_detector);
 
+static bool hung_detector_suspended;
+
+static int hungtask_pm_notify(struct notifier_block *self,
+			      unsigned long action, void *hcpu)
+{
+	switch (action) {
+	case PM_SUSPEND_PREPARE:
+		hung_detector_suspended = true;
+		break;
+	case PM_POST_SUSPEND:
+		hung_detector_suspended = false;
+		break;
+	default:
+		break;
+	}
+	return NOTIFY_OK;
+}
+
 /*
  * kthread which checks for tasks stuck in D state
  */
@@ -261,7 +280,8 @@ static int watchdog(void *dummy)
 		interval = min_t(unsigned long, interval, timeout);
 		t = hung_timeout_jiffies(hung_last_checked, interval);
 		if (t <= 0) {
-			if (!atomic_xchg(&reset_hung_task, 0))
+			if (!atomic_xchg(&reset_hung_task, 0) &&
+			    !hung_detector_suspended)
 				check_hung_uninterruptible_tasks(timeout);
 			hung_last_checked = jiffies;
 			continue;
@@ -275,6 +295,10 @@ static int watchdog(void *dummy)
 static int __init hung_task_init(void)
 {
 	atomic_notifier_chain_register(&panic_notifier_list, &panic_block);
+
+	/* Disable hung task detector on suspend */
+	pm_notifier(hungtask_pm_notify, 0);
+
 	watchdog_task = kthread_run(watchdog, NULL, "khungtaskd");
 
 	return 0;
Signature · 1 line
2.14.4
#1 lore raw
Quoted 9 lines · Vitaly Kuznetsov

It is possible to observe hung_task complaints when system goes to suspend-to-idle state:

PM: Syncing filesystems ... done.
Freezing user space processes ... (elapsed 0.001 seconds) done. OOM killer disabled.
Freezing remaining freezable tasks ... (elapsed 0.002 seconds) done. sd 0:0:0:0: [sda] Synchronizing SCSI cache

Quoted 9 lines
 INFO: task bash:1569 blocked for more than 120 seconds.
       Not tainted 4.19.0-rc3_+ #687
 "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
 bash            D    0  1569    604 0x00000000
 Call Trace:
  ? __schedule+0x1fe/0x7e0
  schedule+0x28/0x80
  suspend_devices_and_enter+0x4ac/0x750
  pm_suspend+0x2c0/0x310

This actually is a good catch, but the problem is related to what happens to the monotonic clock during suspend to idle.

The clock issue needs to be addressed anyway IMO and then this problem will go away automatically.

Quoted 24 lines

Register a PM notifier to disable the detector on suspend and re-enable back on wakeup.

---
RFC: It really makes me wonder why nobody reported this before, makes
me think I'm missing something.

---
 kernel/hung_task.c | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)
kernel/hung_task.c +1 -0
index b9132d1269ef..d75f288c016f 100644
@@ -15,6 +15,7 @@
 #include <linux/lockdep.h>
 #include <linux/export.h>
 #include <linux/sysctl.h>
+#include <linux/suspend.h>
 #include <linux/utsname.h>
 #include <linux/sched/signal.h>
 #include <linux/sched/debug.h>

kernel/hung_task.c void reset_hung_task_detector(void) · line 252

Quoted 10 lines
kernel/hung_task.c +7 -0
@@ -242,6 +243,24 @@ void reset_hung_task_detector(void)
 }
 EXPORT_SYMBOL_GPL(reset_hung_task_detector);
 
+static bool hung_detector_suspended;
+
+static int hungtask_pm_notify(struct notifier_block *self,
+                             unsigned long action, void *hcpu)
+{
+       switch (action) {
+       case PM_SUSPEND_PREPARE:

You'd want PM_HIBERNATION_PREPARE here too I think.

kernel/hung_task.c void reset_hung_task_detector(void) · line 255

Quoted 3 lines
kernel/hung_task.c +3 -0
+               hung_detector_suspended = true;
+               break;
+       case PM_POST_SUSPEND:

And PM_POST_HIBERNATION here for consistency.

Quoted 35 lines
kernel/hung_task.c +14 -2
+               hung_detector_suspended = false;
+               break;
+       default:
+               break;
+       }
+       return NOTIFY_OK;
+}
+
 /*
  * kthread which checks for tasks stuck in D state
  */
@@ -261,7 +280,8 @@ static int watchdog(void *dummy)
                interval = min_t(unsigned long, interval, timeout);
                t = hung_timeout_jiffies(hung_last_checked, interval);
                if (t <= 0) {
-                       if (!atomic_xchg(&reset_hung_task, 0))
+                       if (!atomic_xchg(&reset_hung_task, 0) &&
+                           !hung_detector_suspended)
                                check_hung_uninterruptible_tasks(timeout);
                        hung_last_checked = jiffies;
                        continue;
@@ -275,6 +295,10 @@ static int watchdog(void *dummy)
 static int __init hung_task_init(void)
 {
        atomic_notifier_chain_register(&panic_notifier_list, &panic_block);
+
+       /* Disable hung task detector on suspend */
+       pm_notifier(hungtask_pm_notify, 0);
+
        watchdog_task = kthread_run(watchdog, NULL, "khungtaskd");
 
        return 0;
--

2.14.4

#2 lore raw
Quoted 20 lines · Rafael J. Wysocki
Quoted 9 lines · Vitaly Kuznetsov

It is possible to observe hung_task complaints when system goes to suspend-to-idle state:

PM: Syncing filesystems ... done.
Freezing user space processes ... (elapsed 0.001 seconds) done. OOM killer disabled.
Freezing remaining freezable tasks ... (elapsed 0.002 seconds) done. sd 0:0:0:0: [sda] Synchronizing SCSI cache

Quoted 9 lines
 INFO: task bash:1569 blocked for more than 120 seconds.
       Not tainted 4.19.0-rc3_+ #687
 "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
 bash            D    0  1569    604 0x00000000
 Call Trace:
  ? __schedule+0x1fe/0x7e0
  schedule+0x28/0x80
  suspend_devices_and_enter+0x4ac/0x750
  pm_suspend+0x2c0/0x310
Quoted 5 lines

This actually is a good catch, but the problem is related to what happens to the monotonic clock during suspend to idle.

The clock issue needs to be addressed anyway IMO and then this problem will go away automatically.

Do I understand it correctly that the suggestion is to fully suspend monothonic clock in s2idle (and don't advance it after resume)?

Quoted 38 lines
Quoted 25 lines

Register a PM notifier to disable the detector on suspend and re-enable back on wakeup.

---
RFC: It really makes me wonder why nobody reported this before, makes
me think I'm missing something.

---
 kernel/hung_task.c | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)
kernel/hung_task.c +1 -0
index b9132d1269ef..d75f288c016f 100644
@@ -15,6 +15,7 @@
 #include <linux/lockdep.h>
 #include <linux/export.h>
 #include <linux/sysctl.h>
+#include <linux/suspend.h>
 #include <linux/utsname.h>
 #include <linux/sched/signal.h>
 #include <linux/sched/debug.h>

kernel/hung_task.c void reset_hung_task_detector(void) · line 252

Quoted 10 lines
kernel/hung_task.c +7 -0
@@ -242,6 +243,24 @@ void reset_hung_task_detector(void)
 }
 EXPORT_SYMBOL_GPL(reset_hung_task_detector);
 
+static bool hung_detector_suspended;
+
+static int hungtask_pm_notify(struct notifier_block *self,
+                             unsigned long action, void *hcpu)
+{
+       switch (action) {
+       case PM_SUSPEND_PREPARE:

You'd want PM_HIBERNATION_PREPARE here too I think.

Quoted 6 lines

kernel/hung_task.c void reset_hung_task_detector(void) · line 255

Quoted 3 lines
kernel/hung_task.c +3 -0
+               hung_detector_suspended = true;
+               break;
+       case PM_POST_SUSPEND:

And PM_POST_HIBERNATION here for consistency.

Sure, will do in v1.

Quoted 35 lines
Quoted 35 lines
kernel/hung_task.c +14 -2
+               hung_detector_suspended = false;
+               break;
+       default:
+               break;
+       }
+       return NOTIFY_OK;
+}
+
 /*
  * kthread which checks for tasks stuck in D state
  */
@@ -261,7 +280,8 @@ static int watchdog(void *dummy)
                interval = min_t(unsigned long, interval, timeout);
                t = hung_timeout_jiffies(hung_last_checked, interval);
                if (t <= 0) {
-                       if (!atomic_xchg(&reset_hung_task, 0))
+                       if (!atomic_xchg(&reset_hung_task, 0) &&
+                           !hung_detector_suspended)
                                check_hung_uninterruptible_tasks(timeout);
                        hung_last_checked = jiffies;
                        continue;
@@ -275,6 +295,10 @@ static int watchdog(void *dummy)
 static int __init hung_task_init(void)
 {
        atomic_notifier_chain_register(&panic_notifier_list, &panic_block);
+
+       /* Disable hung task detector on suspend */
+       pm_notifier(hungtask_pm_notify, 0);
+
        watchdog_task = kthread_run(watchdog, NULL, "khungtaskd");
 
        return 0;
--

2.14.4

Signature · 1 line
  Vitaly
#3 lore raw
Quoted 23 lines · Vitaly Kuznetsov
Quoted 20 lines · Rafael J. Wysocki
Quoted 18 lines · Vitaly Kuznetsov

It is possible to observe hung_task complaints when system goes to suspend-to-idle state:

PM: Syncing filesystems ... done.
Freezing user space processes ... (elapsed 0.001 seconds) done. OOM killer disabled.
Freezing remaining freezable tasks ... (elapsed 0.002 seconds) done. sd 0:0:0:0: [sda] Synchronizing SCSI cache

 INFO: task bash:1569 blocked for more than 120 seconds.
       Not tainted 4.19.0-rc3_+ #687
 "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
 bash            D    0  1569    604 0x00000000
 Call Trace:
  ? __schedule+0x1fe/0x7e0
  schedule+0x28/0x80
  suspend_devices_and_enter+0x4ac/0x750
  pm_suspend+0x2c0/0x310
Quoted 8 lines
Quoted 5 lines

This actually is a good catch, but the problem is related to what happens to the monotonic clock during suspend to idle.

The clock issue needs to be addressed anyway IMO and then this problem will go away automatically.

Do I understand it correctly that the suggestion is to fully suspend monothonic clock in s2idle (and don't advance it after resume)?

Something like that.

We already do that to some extent, but kind of with the assumption that it will not grow too much in s2idle_loop(), but it may actually grow too much in there, so we need to compensate for that to make the s2idle behavior reflect the suspend-to-RAM one.

I have a patch for that, will post it shortly.

Thanks,
Rafael

#4 lore raw
In reply to Rafael J. Wysocki
Quoted 21 lines · Rafael J. Wysocki
Quoted 9 lines · Vitaly Kuznetsov

It is possible to observe hung_task complaints when system goes to suspend-to-idle state:

PM: Syncing filesystems ... done.
Freezing user space processes ... (elapsed 0.001 seconds) done. OOM killer disabled.
Freezing remaining freezable tasks ... (elapsed 0.002 seconds) done. sd 0:0:0:0: [sda] Synchronizing SCSI cache

Quoted 9 lines
 INFO: task bash:1569 blocked for more than 120 seconds.
       Not tainted 4.19.0-rc3_+ #687
 "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
 bash            D    0  1569    604 0x00000000
 Call Trace:
  ? __schedule+0x1fe/0x7e0
  schedule+0x28/0x80
  suspend_devices_and_enter+0x4ac/0x750
  pm_suspend+0x2c0/0x310
Quoted 5 lines

This actually is a good catch, but the problem is related to what happens to the monotonic clock during suspend to idle.

The clock issue needs to be addressed anyway IMO and then this problem will go away automatically.

I don't understand your discussion with Vitaly, but shouldn't we make khungtaskd thread freezable anyway?

Oleg.

kernel/hung_task.c +3 -2
@@ -185,7 +185,7 @@ static void check_hung_uninterruptible_t
 	hung_task_show_lock = false;
 	rcu_read_lock();
 	for_each_process_thread(g, t) {
-		if (!max_count--)
+		if (!max_count-- || freezing(current))
 			goto unlock;
 		if (!--batch_count) {
 			batch_count = HUNG_TASK_BATCHING;
@@ -249,6 +249,7 @@ static int watchdog(void *dummy)
 {
 	unsigned long hung_last_checked = jiffies;
 
+	set_freezable();
 	set_user_nice(current, 0);
 
 	for ( ; ; ) {
@@ -266,7 +267,7 @@ static int watchdog(void *dummy)
 			hung_last_checked = jiffies;
 			continue;
 		}
-		schedule_timeout_interruptible(t);
+		freezable_schedule_timeout_interruptible(t);
 	}
 
 	return 0;
#5 lore raw
Quoted 23 lines · Oleg Nesterov
Quoted 21 lines · Rafael J. Wysocki
Quoted 18 lines · Vitaly Kuznetsov

It is possible to observe hung_task complaints when system goes to suspend-to-idle state:

PM: Syncing filesystems ... done.
Freezing user space processes ... (elapsed 0.001 seconds) done. OOM killer disabled.
Freezing remaining freezable tasks ... (elapsed 0.002 seconds) done. sd 0:0:0:0: [sda] Synchronizing SCSI cache

 INFO: task bash:1569 blocked for more than 120 seconds.
       Not tainted 4.19.0-rc3_+ #687
 "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
 bash            D    0  1569    604 0x00000000
 Call Trace:
  ? __schedule+0x1fe/0x7e0
  schedule+0x28/0x80
  suspend_devices_and_enter+0x4ac/0x750
  pm_suspend+0x2c0/0x310
Quoted 8 lines
Quoted 5 lines

This actually is a good catch, but the problem is related to what happens to the monotonic clock during suspend to idle.

The clock issue needs to be addressed anyway IMO and then this problem will go away automatically.

I don't understand your discussion with Vitaly, but shouldn't we make khungtaskd thread freezable anyway?

Well, if the Vitaly's patch is applied, that shouldn't be necessary and I wouldn't like to add more freezable kernel threads if that's avoidable TBH.

Cheers,
Rafael