Skip to contentSkip to conversation

task blocked for more than 120 seconds

DISCUSSION Sergey Senozhatsky 3 participants 5 messages Nov 4, 2010 – Nov 5, 2010

lore
Share & tools
lore
#0 lore raw

Hello,
Got the following traces:

[42001.449295] INFO: task openbox:17761 blocked for more than 120 seconds.
[42001.449303] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[42001.449311] openbox       D 0000000000000003     0 17761   3723 0x00000000
[42001.449321]  ffff88004ce8dc38 0000000000000046 0000000000000000 ffff88004ce8c000
[42001.449333]  ffff88004ce8dfd8 0000000000012040 ffff880133010000 ffff88004ce8dfd8
[42001.449343]  0000000000012040 0000000000012040 ffff88004ce8dfd8 0000000000012040
[42001.449354] Call Trace:
[42001.449369]  [<ffffffff814212ff>] schedule_timeout+0x38/0x220
[42001.449381]  [<ffffffff81066f21>] ? mark_held_locks+0x50/0x72
[42001.449389]  [<ffffffff81423dae>] ? _raw_spin_unlock_irq+0x2b/0x59
[42001.449398]  [<ffffffff81036070>] ? get_parent_ip+0x11/0x41
[42001.449406]  [<ffffffff81036132>] ? sub_preempt_count+0x92/0xa5
[42001.449413]  [<ffffffff8142057e>] wait_for_common+0xca/0x144
[42001.449421]  [<ffffffff81038b13>] ? default_wake_function+0x0/0xf
[42001.449429]  [<ffffffff81423e4b>] ? _raw_spin_unlock_irqrestore+0x6f/0x74
[42001.449436]  [<ffffffff81420692>] wait_for_completion+0x18/0x1a
[42001.449445]  [<ffffffff81084808>] stop_one_cpu+0x8c/0xba
[42001.449453]  [<ffffffff810355cb>] ? migration_cpu_stop+0x0/0x3d
[42001.449461]  [<ffffffff81035aef>] sched_exec+0xc3/0xdc
[42001.449470]  [<ffffffff810e7dad>] do_execve+0xaa/0x267
[42001.449479]  [<ffffffff8100a58b>] sys_execve+0x3e/0x55
[42001.449488]  [<ffffffff8100265c>] stub_execve+0x6c/0xc0
[42001.449494] 1 lock held by openbox/17761:
[42001.449498]  #0:  (&sig->cred_guard_mutex){+.+.+.}, at: [<ffffffff810e7b1d>] prepare_bprm_creds+0x28/0x64
[47761.448699] INFO: task make:29123 blocked for more than 120 seconds.
[47761.448704] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[47761.448710] make          D 0000000000000001     0 29123  24208 0x00000004
[47761.448719]  ffff880115dadcf0 0000000000000046 ffff880100000000 ffff880115dac000
[47761.448731]  ffff880115dadfd8 0000000000012040 ffff880133290000 ffff880115dadfd8
[47761.448741]  0000000000012040 0000000000012040 ffff880115dadfd8 0000000000012040
[47761.448752] Call Trace:
[47761.448766]  [<ffffffff814212ff>] schedule_timeout+0x38/0x220
[47761.448774]  [<ffffffff814204da>] ? wait_for_common+0x26/0x144
[47761.448784]  [<ffffffff81207b53>] ? do_raw_spin_lock+0x6b/0x122
[47761.448792]  [<ffffffff81423dae>] ? _raw_spin_unlock_irq+0x2b/0x59
[47761.448801]  [<ffffffff81036070>] ? get_parent_ip+0x11/0x41
[47761.448808]  [<ffffffff81036132>] ? sub_preempt_count+0x92/0xa5
[47761.448816]  [<ffffffff8142057e>] wait_for_common+0xca/0x144
[47761.448823]  [<ffffffff81038b13>] ? default_wake_function+0x0/0xf
[47761.448831]  [<ffffffff81036132>] ? sub_preempt_count+0x92/0xa5
[47761.448838]  [<ffffffff81420692>] wait_for_completion+0x18/0x1a
[47761.448845]  [<ffffffff8103cec0>] do_fork+0x2b0/0x32e
[47761.448856]  [<ffffffff810021ba>] ? sysret_check+0x2e/0x69
[47761.448864]  [<ffffffff8100a526>] sys_vfork+0x20/0x22
[47761.448871]  [<ffffffff81002563>] stub_vfork+0x13/0x20
[47761.448878]  [<ffffffff81002182>] ? system_call_fastpath+0x16/0x1b
[47761.448884] no locks held by make/29123.
[47761.448891] INFO: task make:31380 blocked for more than 120 seconds.
[47761.448895] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[47761.448899] make          D 0000000000000000     0 31380  29123 0x00000000
[47761.448908]  ffff880157f01c38 0000000000000046 0000000000000000 ffff880157f00000
[47761.448918]  ffff880157f01fd8 0000000000012040 ffff8800773abea0 ffff880157f01fd8
[47761.448929]  0000000000012040 0000000000012040 ffff880157f01fd8 0000000000012040
[47761.448939] Call Trace:
[47761.448947]  [<ffffffff814212ff>] schedule_timeout+0x38/0x220
[47761.448957]  [<ffffffff81066f21>] ? mark_held_locks+0x50/0x72
[47761.448965]  [<ffffffff81423dae>] ? _raw_spin_unlock_irq+0x2b/0x59
[47761.448972]  [<ffffffff81036070>] ? get_parent_ip+0x11/0x41
[47761.448979]  [<ffffffff81036132>] ? sub_preempt_count+0x92/0xa5
[47761.448986]  [<ffffffff8142057e>] wait_for_common+0xca/0x144
[47761.448994]  [<ffffffff81038b13>] ? default_wake_function+0x0/0xf
[47761.449001]  [<ffffffff81423e4b>] ? _raw_spin_unlock_irqrestore+0x6f/0x74
[47761.449008]  [<ffffffff81420692>] wait_for_completion+0x18/0x1a
[47761.449017]  [<ffffffff81084808>] stop_one_cpu+0x8c/0xba
[47761.449025]  [<ffffffff810355cb>] ? migration_cpu_stop+0x0/0x3d
[47761.449033]  [<ffffffff81035aef>] sched_exec+0xc3/0xdc
[47761.449041]  [<ffffffff810e7dad>] do_execve+0xaa/0x267
[47761.449049]  [<ffffffff8100a58b>] sys_execve+0x3e/0x55
[47761.449056]  [<ffffffff8100265c>] stub_execve+0x6c/0xc0
[47761.449062] 1 lock held by make/31380:
[47761.449065]  #0:  (&sig->cred_guard_mutex){+.+.+.}, at: [<ffffffff810e7b1d>] prepare_bprm_creds+0x28/0x64

Sergey

#1 lore raw
Quoted 12 lines · Sergey Senozhatsky

Hello,
Got the following traces:

[42001.449295] INFO: task openbox:17761 blocked for more than 120 seconds.
[42001.449303] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[42001.449311] openbox       D 0000000000000003     0 17761   3723 0x00000000
[42001.449321]  ffff88004ce8dc38 0000000000000046 0000000000000000 ffff88004ce8c000
[42001.449333]  ffff88004ce8dfd8 0000000000012040 ffff880133010000 ffff88004ce8dfd8
[42001.449343]  0000000000012040 0000000000012040 ffff88004ce8dfd8 0000000000012040
[42001.449354] Call Trace:
[42001.449369]  [<ffffffff814212ff>] schedule_timeout+0x38/0x220
[42001.449381]  [<ffffffff81066f21>] ? mark_held_locks+0x50/0x72
Quoted 8 lines
[42001.449389]  [<ffffffff81423dae>] ? _raw_spin_unlock_irq+0x2b/0x59
[42001.449398]  [<ffffffff81036070>] ? get_parent_ip+0x11/0x41
[42001.449406]  [<ffffffff81036132>] ? sub_preempt_count+0x92/0xa5
[42001.449413]  [<ffffffff8142057e>] wait_for_common+0xca/0x144
[42001.449421]  [<ffffffff81038b13>] ? default_wake_function+0x0/0xf
[42001.449429]  [<ffffffff81423e4b>] ? _raw_spin_unlock_irqrestore+0x6f/0x74
[42001.449436]  [<ffffffff81420692>] wait_for_completion+0x18/0x1a
[42001.449445]  [<ffffffff81084808>] stop_one_cpu+0x8c/0xba

These traces look related to:
http://article.gmane.org/gmane.linux.kernel/1055100

Does reverting 34f971f6f7988be4d014eec3e3526bee6d007ffa help in your case?

Signature · 1 line
Markus
Tested-by Marcelo Tosatti
#2 lore raw
Quoted 22 lines · Markus Trippelsdorf
Quoted 12 lines · Sergey Senozhatsky

Hello,
Got the following traces:

[42001.449295] INFO: task openbox:17761 blocked for more than 120 seconds.
[42001.449303] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[42001.449311] openbox       D 0000000000000003     0 17761   3723 0x00000000
[42001.449321]  ffff88004ce8dc38 0000000000000046 0000000000000000 ffff88004ce8c000
[42001.449333]  ffff88004ce8dfd8 0000000000012040 ffff880133010000 ffff88004ce8dfd8
[42001.449343]  0000000000012040 0000000000012040 ffff88004ce8dfd8 0000000000012040
[42001.449354] Call Trace:
[42001.449369]  [<ffffffff814212ff>] schedule_timeout+0x38/0x220
[42001.449381]  [<ffffffff81066f21>] ? mark_held_locks+0x50/0x72
Quoted 8 lines
[42001.449389]  [<ffffffff81423dae>] ? _raw_spin_unlock_irq+0x2b/0x59
[42001.449398]  [<ffffffff81036070>] ? get_parent_ip+0x11/0x41
[42001.449406]  [<ffffffff81036132>] ? sub_preempt_count+0x92/0xa5
[42001.449413]  [<ffffffff8142057e>] wait_for_common+0xca/0x144
[42001.449421]  [<ffffffff81038b13>] ? default_wake_function+0x0/0xf
[42001.449429]  [<ffffffff81423e4b>] ? _raw_spin_unlock_irqrestore+0x6f/0x74
[42001.449436]  [<ffffffff81420692>] wait_for_completion+0x18/0x1a
[42001.449445]  [<ffffffff81084808>] stop_one_cpu+0x8c/0xba
Quoted 5 lines

These traces look related to:
http://article.gmane.org/gmane.linux.kernel/1055100

Does reverting 34f971f6f7988be4d014eec3e3526bee6d007ffa help in your case?

---
Subject: sched: Fixup cross sched_class wakeup preemption
From: Peter Zijlstra <[email protected]>
Date: Sun Oct 31 12:37:04 CET 2010

Instead of dealing with sched classes inside each check_preempt_curr() implementation, pull out this logic into the generic wakeup preemption path.

This fixes a hang in KVM (and others) where we are waiting for the stop machine thread to run..

LKML-Reference: <new-submission>

---
 kernel/sched.c          |   39 ++++++++++++++++++++++++++++-----------
 kernel/sched_fair.c     |    6 ------
 kernel/sched_stoptask.c |    2 +-
 3 files changed, 29 insertions(+), 18 deletions(-)
kernel/sched.c +28 -11
Index: linux-2.6/kernel/sched.c
===================================================================
@@ -560,18 +560,9 @@ struct rq {
 
 static DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
 
-static inline
-void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags)
-{
-	rq->curr->sched_class->check_preempt_curr(rq, p, flags);
 
-	/*
-	 * A queue event has occurred, and we're going to schedule.  In
-	 * this case, we can save a useless back to back clock update.
-	 */
-	if (test_tsk_need_resched(p))
-		rq->skip_clock_update = 1;
-}
+static inline
+void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags);
 
 static inline int cpu_of(struct rq *rq)
 {
@@ -9400,4 +9391,30 @@ void synchronize_sched_expedited(void)
 }
 EXPORT_SYMBOL_GPL(synchronize_sched_expedited);
 
+static inline
+void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags)
+{
+	const struct sched_class *class;
+
+	if (p->sched_class == rq->curr->sched_class)
+		rq->curr->sched_class->check_preempt_curr(rq, p, flags);
+
+	for_each_class(class) {
+		if (class == rq->curr->sched_class)
+			break;
+		if (class == p->sched_class) {
+			resched_task(rq->curr);
+			break;
+		}
+	}
+
+	/*
+	 * A queue event has occurred, and we're going to schedule.  In
+	 * this case, we can save a useless back to back clock update.
+	 */
+	if (test_tsk_need_resched(rq->curr))
+		rq->skip_clock_update = 1;
+}
+
+
 #endif /* #else #ifndef CONFIG_SMP */
kernel/sched_fair.c +0 -6
Index: linux-2.6/kernel/sched_fair.c
===================================================================
@@ -1654,12 +1654,6 @@ static void check_preempt_wakeup(struct
 	struct cfs_rq *cfs_rq = task_cfs_rq(curr);
 	int scale = cfs_rq->nr_running >= sched_nr_latency;
 
-	if (unlikely(rt_prio(p->prio)))
-		goto preempt;
-
-	if (unlikely(p->sched_class != &fair_sched_class))
-		return;
-
 	if (unlikely(se == pse))
 		return;
 
kernel/sched_stoptask.c +1 -1
Index: linux-2.6/kernel/sched_stoptask.c
===================================================================
@@ -19,7 +19,7 @@ select_task_rq_stop(struct rq *rq, struc
 static void
 check_preempt_curr_stop(struct rq *rq, struct task_struct *p, int flags)
 {
-	resched_task(rq->curr); /* we preempt everything */
+	/* we're never preempted */
 }
 
 static struct task_struct *pick_next_task_stop(struct rq *rq)
#3 lore raw
Quoted 23 lines · Peter Zijlstra
Quoted 22 lines · Markus Trippelsdorf
Quoted 20 lines · Sergey Senozhatsky

Hello,
Got the following traces:

[42001.449295] INFO: task openbox:17761 blocked for more than 120 seconds.
[42001.449303] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[42001.449311] openbox       D 0000000000000003     0 17761   3723 0x00000000
[42001.449321]  ffff88004ce8dc38 0000000000000046 0000000000000000 ffff88004ce8c000
[42001.449333]  ffff88004ce8dfd8 0000000000012040 ffff880133010000 ffff88004ce8dfd8
[42001.449343]  0000000000012040 0000000000012040 ffff88004ce8dfd8 0000000000012040
[42001.449354] Call Trace:
[42001.449369]  [<ffffffff814212ff>] schedule_timeout+0x38/0x220
[42001.449381]  [<ffffffff81066f21>] ? mark_held_locks+0x50/0x72
[42001.449389]  [<ffffffff81423dae>] ? _raw_spin_unlock_irq+0x2b/0x59
[42001.449398]  [<ffffffff81036070>] ? get_parent_ip+0x11/0x41
[42001.449406]  [<ffffffff81036132>] ? sub_preempt_count+0x92/0xa5
[42001.449413]  [<ffffffff8142057e>] wait_for_common+0xca/0x144
[42001.449421]  [<ffffffff81038b13>] ? default_wake_function+0x0/0xf
[42001.449429]  [<ffffffff81423e4b>] ? _raw_spin_unlock_irqrestore+0x6f/0x74
[42001.449436]  [<ffffffff81420692>] wait_for_completion+0x18/0x1a
[42001.449445]  [<ffffffff81084808>] stop_one_cpu+0x8c/0xba
Quoted 7 lines
Quoted 5 lines

These traces look related to:
http://article.gmane.org/gmane.linux.kernel/1055100

Does reverting 34f971f6f7988be4d014eec3e3526bee6d007ffa help in your case?

Hello,
I'll test "sched: Fixup cross sched_class wakeup preemption" first and then (if it fails) I'll test 34f971f6f7988be4d014eec3e3526bee6d007ffa revert.

Sergey

Quoted 108 lines

---
Subject: sched: Fixup cross sched_class wakeup preemption
From: Peter Zijlstra <[email protected]>
Date: Sun Oct 31 12:37:04 CET 2010

Instead of dealing with sched classes inside each check_preempt_curr() implementation, pull out this logic into the generic wakeup preemption path.

This fixes a hang in KVM (and others) where we are waiting for the stop machine thread to run..

LKML-Reference: <new-submission>

---
 kernel/sched.c          |   39 ++++++++++++++++++++++++++++-----------
 kernel/sched_fair.c     |    6 ------
 kernel/sched_stoptask.c |    2 +-
 3 files changed, 29 insertions(+), 18 deletions(-)

Index: linux-2.6/kernel/sched.c
===================================================================

kernel/sched.c +28 -11
@@ -560,18 +560,9 @@ struct rq {
 
 static DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
 
-static inline
-void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags)
-{
-	rq->curr->sched_class->check_preempt_curr(rq, p, flags);
 
-	/*
-	 * A queue event has occurred, and we're going to schedule.  In
-	 * this case, we can save a useless back to back clock update.
-	 */
-	if (test_tsk_need_resched(p))
-		rq->skip_clock_update = 1;
-}
+static inline
+void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags);
 
 static inline int cpu_of(struct rq *rq)
 {
@@ -9400,4 +9391,30 @@ void synchronize_sched_expedited(void)
 }
 EXPORT_SYMBOL_GPL(synchronize_sched_expedited);
 
+static inline
+void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags)
+{
+	const struct sched_class *class;
+
+	if (p->sched_class == rq->curr->sched_class)
+		rq->curr->sched_class->check_preempt_curr(rq, p, flags);
+
+	for_each_class(class) {
+		if (class == rq->curr->sched_class)
+			break;
+		if (class == p->sched_class) {
+			resched_task(rq->curr);
+			break;
+		}
+	}
+
+	/*
+	 * A queue event has occurred, and we're going to schedule.  In
+	 * this case, we can save a useless back to back clock update.
+	 */
+	if (test_tsk_need_resched(rq->curr))
+		rq->skip_clock_update = 1;
+}
+
+
 #endif /* #else #ifndef CONFIG_SMP */

Index: linux-2.6/kernel/sched_fair.c
===================================================================

kernel/sched_fair.c +0 -6
@@ -1654,12 +1654,6 @@ static void check_preempt_wakeup(struct
 	struct cfs_rq *cfs_rq = task_cfs_rq(curr);
 	int scale = cfs_rq->nr_running >= sched_nr_latency;
 
-	if (unlikely(rt_prio(p->prio)))
-		goto preempt;
-
-	if (unlikely(p->sched_class != &fair_sched_class))
-		return;
-
 	if (unlikely(se == pse))
 		return;
 

Index: linux-2.6/kernel/sched_stoptask.c
===================================================================

kernel/sched_stoptask.c +1 -1
@@ -19,7 +19,7 @@ select_task_rq_stop(struct rq *rq, struc
 static void
 check_preempt_curr_stop(struct rq *rq, struct task_struct *p, int flags)
 {
-	resched_task(rq->curr); /* we preempt everything */
+	/* we're never preempted */
 }
 
 static struct task_struct *pick_next_task_stop(struct rq *rq)
author Tested-by
#4 lore raw
In reply to Peter Zijlstra
Quoted 11 lines · Peter Zijlstra

Subject: sched: Fixup cross sched_class wakeup preemption
From: Peter Zijlstra <[email protected]>
Date: Sun Oct 31 12:37:04 CET 2010

Instead of dealing with sched classes inside each check_preempt_curr() implementation, pull out this logic into the generic wakeup preemption path.

This fixes a hang in KVM (and others) where we are waiting for the stop machine thread to run..

Hello,

Peter, patch seems to work.

Sergey