task blocked for more than 120 seconds
Hello,
Got the following traces:
[42001.449295] INFO: task openbox:17761 blocked for more than 120 seconds.
[42001.449303] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[42001.449311] openbox D 0000000000000003 0 17761 3723 0x00000000
[42001.449321] ffff88004ce8dc38 0000000000000046 0000000000000000 ffff88004ce8c000
[42001.449333] ffff88004ce8dfd8 0000000000012040 ffff880133010000 ffff88004ce8dfd8
[42001.449343] 0000000000012040 0000000000012040 ffff88004ce8dfd8 0000000000012040
[42001.449354] Call Trace:
[42001.449369] [<ffffffff814212ff>] schedule_timeout+0x38/0x220
[42001.449381] [<ffffffff81066f21>] ? mark_held_locks+0x50/0x72
[42001.449389] [<ffffffff81423dae>] ? _raw_spin_unlock_irq+0x2b/0x59
[42001.449398] [<ffffffff81036070>] ? get_parent_ip+0x11/0x41
[42001.449406] [<ffffffff81036132>] ? sub_preempt_count+0x92/0xa5
[42001.449413] [<ffffffff8142057e>] wait_for_common+0xca/0x144
[42001.449421] [<ffffffff81038b13>] ? default_wake_function+0x0/0xf
[42001.449429] [<ffffffff81423e4b>] ? _raw_spin_unlock_irqrestore+0x6f/0x74
[42001.449436] [<ffffffff81420692>] wait_for_completion+0x18/0x1a
[42001.449445] [<ffffffff81084808>] stop_one_cpu+0x8c/0xba
[42001.449453] [<ffffffff810355cb>] ? migration_cpu_stop+0x0/0x3d
[42001.449461] [<ffffffff81035aef>] sched_exec+0xc3/0xdc
[42001.449470] [<ffffffff810e7dad>] do_execve+0xaa/0x267
[42001.449479] [<ffffffff8100a58b>] sys_execve+0x3e/0x55
[42001.449488] [<ffffffff8100265c>] stub_execve+0x6c/0xc0
[42001.449494] 1 lock held by openbox/17761:
[42001.449498] #0: (&sig->cred_guard_mutex){+.+.+.}, at: [<ffffffff810e7b1d>] prepare_bprm_creds+0x28/0x64[47761.448699] INFO: task make:29123 blocked for more than 120 seconds.
[47761.448704] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[47761.448710] make D 0000000000000001 0 29123 24208 0x00000004
[47761.448719] ffff880115dadcf0 0000000000000046 ffff880100000000 ffff880115dac000
[47761.448731] ffff880115dadfd8 0000000000012040 ffff880133290000 ffff880115dadfd8
[47761.448741] 0000000000012040 0000000000012040 ffff880115dadfd8 0000000000012040
[47761.448752] Call Trace:
[47761.448766] [<ffffffff814212ff>] schedule_timeout+0x38/0x220
[47761.448774] [<ffffffff814204da>] ? wait_for_common+0x26/0x144
[47761.448784] [<ffffffff81207b53>] ? do_raw_spin_lock+0x6b/0x122
[47761.448792] [<ffffffff81423dae>] ? _raw_spin_unlock_irq+0x2b/0x59
[47761.448801] [<ffffffff81036070>] ? get_parent_ip+0x11/0x41
[47761.448808] [<ffffffff81036132>] ? sub_preempt_count+0x92/0xa5
[47761.448816] [<ffffffff8142057e>] wait_for_common+0xca/0x144
[47761.448823] [<ffffffff81038b13>] ? default_wake_function+0x0/0xf
[47761.448831] [<ffffffff81036132>] ? sub_preempt_count+0x92/0xa5
[47761.448838] [<ffffffff81420692>] wait_for_completion+0x18/0x1a
[47761.448845] [<ffffffff8103cec0>] do_fork+0x2b0/0x32e
[47761.448856] [<ffffffff810021ba>] ? sysret_check+0x2e/0x69
[47761.448864] [<ffffffff8100a526>] sys_vfork+0x20/0x22
[47761.448871] [<ffffffff81002563>] stub_vfork+0x13/0x20
[47761.448878] [<ffffffff81002182>] ? system_call_fastpath+0x16/0x1b
[47761.448884] no locks held by make/29123.
[47761.448891] INFO: task make:31380 blocked for more than 120 seconds.
[47761.448895] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[47761.448899] make D 0000000000000000 0 31380 29123 0x00000000
[47761.448908] ffff880157f01c38 0000000000000046 0000000000000000 ffff880157f00000
[47761.448918] ffff880157f01fd8 0000000000012040 ffff8800773abea0 ffff880157f01fd8
[47761.448929] 0000000000012040 0000000000012040 ffff880157f01fd8 0000000000012040
[47761.448939] Call Trace:
[47761.448947] [<ffffffff814212ff>] schedule_timeout+0x38/0x220
[47761.448957] [<ffffffff81066f21>] ? mark_held_locks+0x50/0x72
[47761.448965] [<ffffffff81423dae>] ? _raw_spin_unlock_irq+0x2b/0x59
[47761.448972] [<ffffffff81036070>] ? get_parent_ip+0x11/0x41
[47761.448979] [<ffffffff81036132>] ? sub_preempt_count+0x92/0xa5
[47761.448986] [<ffffffff8142057e>] wait_for_common+0xca/0x144
[47761.448994] [<ffffffff81038b13>] ? default_wake_function+0x0/0xf
[47761.449001] [<ffffffff81423e4b>] ? _raw_spin_unlock_irqrestore+0x6f/0x74
[47761.449008] [<ffffffff81420692>] wait_for_completion+0x18/0x1a
[47761.449017] [<ffffffff81084808>] stop_one_cpu+0x8c/0xba
[47761.449025] [<ffffffff810355cb>] ? migration_cpu_stop+0x0/0x3d
[47761.449033] [<ffffffff81035aef>] sched_exec+0xc3/0xdc
[47761.449041] [<ffffffff810e7dad>] do_execve+0xaa/0x267
[47761.449049] [<ffffffff8100a58b>] sys_execve+0x3e/0x55
[47761.449056] [<ffffffff8100265c>] stub_execve+0x6c/0xc0
[47761.449062] 1 lock held by make/31380:
[47761.449065] #0: (&sig->cred_guard_mutex){+.+.+.}, at: [<ffffffff810e7b1d>] prepare_bprm_creds+0x28/0x64Sergey
Quoted 12 lines · Sergey Senozhatsky
Hello,
Got the following traces:[42001.449295] INFO: task openbox:17761 blocked for more than 120 seconds. [42001.449303] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [42001.449311] openbox D 0000000000000003 0 17761 3723 0x00000000 [42001.449321] ffff88004ce8dc38 0000000000000046 0000000000000000 ffff88004ce8c000 [42001.449333] ffff88004ce8dfd8 0000000000012040 ffff880133010000 ffff88004ce8dfd8 [42001.449343] 0000000000012040 0000000000012040 ffff88004ce8dfd8 0000000000012040 [42001.449354] Call Trace: [42001.449369] [<ffffffff814212ff>] schedule_timeout+0x38/0x220 [42001.449381] [<ffffffff81066f21>] ? mark_held_locks+0x50/0x72
Quoted 8 lines
[42001.449389] [<ffffffff81423dae>] ? _raw_spin_unlock_irq+0x2b/0x59 [42001.449398] [<ffffffff81036070>] ? get_parent_ip+0x11/0x41 [42001.449406] [<ffffffff81036132>] ? sub_preempt_count+0x92/0xa5 [42001.449413] [<ffffffff8142057e>] wait_for_common+0xca/0x144 [42001.449421] [<ffffffff81038b13>] ? default_wake_function+0x0/0xf [42001.449429] [<ffffffff81423e4b>] ? _raw_spin_unlock_irqrestore+0x6f/0x74 [42001.449436] [<ffffffff81420692>] wait_for_completion+0x18/0x1a [42001.449445] [<ffffffff81084808>] stop_one_cpu+0x8c/0xba
These traces look related to:
http://article.gmane.org/gmane.linux.kernel/1055100
Does reverting 34f971f6f7988be4d014eec3e3526bee6d007ffa help in your case?
Signature · 1 line
Markus
Peter Zijlstra
Tested-by Marcelo TosattiQuoted 22 lines · Markus Trippelsdorf
Quoted 12 lines · Sergey Senozhatsky
Hello,
Got the following traces:[42001.449295] INFO: task openbox:17761 blocked for more than 120 seconds. [42001.449303] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [42001.449311] openbox D 0000000000000003 0 17761 3723 0x00000000 [42001.449321] ffff88004ce8dc38 0000000000000046 0000000000000000 ffff88004ce8c000 [42001.449333] ffff88004ce8dfd8 0000000000012040 ffff880133010000 ffff88004ce8dfd8 [42001.449343] 0000000000012040 0000000000012040 ffff88004ce8dfd8 0000000000012040 [42001.449354] Call Trace: [42001.449369] [<ffffffff814212ff>] schedule_timeout+0x38/0x220 [42001.449381] [<ffffffff81066f21>] ? mark_held_locks+0x50/0x72Quoted 8 lines
[42001.449389] [<ffffffff81423dae>] ? _raw_spin_unlock_irq+0x2b/0x59 [42001.449398] [<ffffffff81036070>] ? get_parent_ip+0x11/0x41 [42001.449406] [<ffffffff81036132>] ? sub_preempt_count+0x92/0xa5 [42001.449413] [<ffffffff8142057e>] wait_for_common+0xca/0x144 [42001.449421] [<ffffffff81038b13>] ? default_wake_function+0x0/0xf [42001.449429] [<ffffffff81423e4b>] ? _raw_spin_unlock_irqrestore+0x6f/0x74 [42001.449436] [<ffffffff81420692>] wait_for_completion+0x18/0x1a [42001.449445] [<ffffffff81084808>] stop_one_cpu+0x8c/0xba
Quoted 5 lines
These traces look related to:
http://article.gmane.org/gmane.linux.kernel/1055100Does reverting
34f971f6f7988be4d014eec3e3526bee6d007ffahelp in your case?
---
Subject: sched: Fixup cross sched_class wakeup preemption
From: Peter Zijlstra <[email protected]>
Date: Sun Oct 31 12:37:04 CET 2010
Instead of dealing with sched classes inside each check_preempt_curr() implementation, pull out this logic into the generic wakeup preemption path.
This fixes a hang in KVM (and others) where we are waiting for the stop machine thread to run..
LKML-Reference: <new-submission>
--- kernel/sched.c | 39 ++++++++++++++++++++++++++++----------- kernel/sched_fair.c | 6 ------ kernel/sched_stoptask.c | 2 +- 3 files changed, 29 insertions(+), 18 deletions(-)
kernel/sched.c +28 -11
@@ -560,18 +560,9 @@ struct rq { static DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); -static inline -void check_preempt_curr (struct rq *rq,struct task_struct *p,int flags) -{ - rq->curr->sched_class->check_preempt_curr (rq, p, flags); -/* -* A queue event has occurred, and we're going to schedule. In -* this case, we can save a useless back to back clock update. -*/ -if (test_tsk_need_resched (p)) - rq->skip_clock_update =1 ; -} +static inline +void check_preempt_curr (struct rq *rq,struct task_struct *p,int flags);static inline int cpu_of (struct rq *rq) {
@@ -9400,4 +9391,30 @@ void synchronize_sched_expedited(void) }EXPORT_SYMBOL_GPL (synchronize_sched_expedited); +static inline +void check_preempt_curr (struct rq *rq,struct task_struct *p,int flags) +{ +const struct sched_class *class; + +if (p->sched_class == rq->curr->sched_class) + rq->curr->sched_class->check_preempt_curr (rq, p, flags); + +for_each_class (class) { +if (class == rq->curr->sched_class) +break ; +if (class == p->sched_class) { +resched_task (rq->curr); +break ; + } + } + +/* +* A queue event has occurred, and we're going to schedule. In +* this case, we can save a useless back to back clock update. +*/ +if (test_tsk_need_resched (rq->curr)) + rq->skip_clock_update =1 ; +} + +#endif /* #else #ifndef CONFIG_SMP */
kernel/sched_fair.c +0 -6
@@ -1654,12 +1654,6 @@ static void check_preempt_wakeup(structstruct cfs_rq *cfs_rq =task_cfs_rq (curr);int scale = cfs_rq->nr_running >= sched_nr_latency; -if (unlikely (rt_prio (p->prio))) - goto preempt; - -if (unlikely (p->sched_class != &fair_sched_class)) -return ; -if (unlikely (se == pse))return ;
kernel/sched_stoptask.c +1 -1
@@ -19,7 +19,7 @@ select_task_rq_stop(struct rq *rq, strucstatic void check_preempt_curr_stop (struct rq *rq,struct task_struct *p,int flags) { -resched_task (rq->curr);/* we preempt everything */ +/* we're never preempted */ }static struct task_struct *pick_next_task_stop (struct rq *rq)
Quoted 23 lines · Peter Zijlstra
Quoted 22 lines · Markus Trippelsdorf
Quoted 20 lines · Sergey Senozhatsky
Hello,
Got the following traces:[42001.449295] INFO: task openbox:17761 blocked for more than 120 seconds. [42001.449303] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [42001.449311] openbox D 0000000000000003 0 17761 3723 0x00000000 [42001.449321] ffff88004ce8dc38 0000000000000046 0000000000000000 ffff88004ce8c000 [42001.449333] ffff88004ce8dfd8 0000000000012040 ffff880133010000 ffff88004ce8dfd8 [42001.449343] 0000000000012040 0000000000012040 ffff88004ce8dfd8 0000000000012040 [42001.449354] Call Trace: [42001.449369] [<ffffffff814212ff>] schedule_timeout+0x38/0x220 [42001.449381] [<ffffffff81066f21>] ? mark_held_locks+0x50/0x72 [42001.449389] [<ffffffff81423dae>] ? _raw_spin_unlock_irq+0x2b/0x59 [42001.449398] [<ffffffff81036070>] ? get_parent_ip+0x11/0x41 [42001.449406] [<ffffffff81036132>] ? sub_preempt_count+0x92/0xa5 [42001.449413] [<ffffffff8142057e>] wait_for_common+0xca/0x144 [42001.449421] [<ffffffff81038b13>] ? default_wake_function+0x0/0xf [42001.449429] [<ffffffff81423e4b>] ? _raw_spin_unlock_irqrestore+0x6f/0x74 [42001.449436] [<ffffffff81420692>] wait_for_completion+0x18/0x1a [42001.449445] [<ffffffff81084808>] stop_one_cpu+0x8c/0xba
Quoted 7 lines
Quoted 5 lines
These traces look related to:
http://article.gmane.org/gmane.linux.kernel/1055100Does reverting
34f971f6f7988be4d014eec3e3526bee6d007ffahelp in your case?
Hello,
I'll test "sched: Fixup cross sched_class wakeup preemption" first and then (if it fails) I'll test 34f971f6f7988be4d014eec3e3526bee6d007ffa revert.
Sergey
Quoted 108 lines
---
Subject: sched: Fixup crosssched_classwakeup preemption
From: Peter Zijlstra <[email protected]>
Date: Sun Oct 31 12:37:04 CET 2010Instead of dealing with sched classes inside each
check_preempt_curr()implementation, pull out this logic into the generic wakeup preemption path.This fixes a hang in KVM (and others) where we are waiting for the stop machine thread to run..
Tested-by Marcelo Tosatti <[email protected]>Signed-off-by Peter Zijlstra <[email protected]>LKML-Reference: <new-submission>
--- kernel/sched.c | 39 ++++++++++++++++++++++++++++----------- kernel/sched_fair.c | 6 ------ kernel/sched_stoptask.c | 2 +- 3 files changed, 29 insertions(+), 18 deletions(-)Index:
linux-2.6/kernel/sched.c
===================================================================kernel/sched.c +28
-11@@ -560,18 +560,9 @@ struct rq { static DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); -static inline -void check_preempt_curr (struct rq *rq,struct task_struct *p,int flags) -{ - rq->curr->sched_class->check_preempt_curr (rq, p, flags); -/* -* A queue event has occurred, and we're going to schedule. In -* this case, we can save a useless back to back clock update. -*/ -if (test_tsk_need_resched (p)) - rq->skip_clock_update =1 ; -} +static inline +void check_preempt_curr (struct rq *rq,struct task_struct *p,int flags);static inline int cpu_of (struct rq *rq) {@@ -9400,4 +9391,30 @@ void synchronize_sched_expedited(void) }EXPORT_SYMBOL_GPL (synchronize_sched_expedited); +static inline +void check_preempt_curr (struct rq *rq,struct task_struct *p,int flags) +{ +const struct sched_class *class; + +if (p->sched_class == rq->curr->sched_class) + rq->curr->sched_class->check_preempt_curr (rq, p, flags); + +for_each_class (class) { +if (class == rq->curr->sched_class) +break ; +if (class == p->sched_class) { +resched_task (rq->curr); +break ; + } + } + +/* +* A queue event has occurred, and we're going to schedule. In +* this case, we can save a useless back to back clock update. +*/ +if (test_tsk_need_resched (rq->curr)) + rq->skip_clock_update =1 ; +} + +#endif /* #else #ifndef CONFIG_SMP */ Index:
linux-2.6/kernel/sched_fair.c
===================================================================kernel/sched_fair.c +0
-6@@ -1654,12 +1654,6 @@ static void check_preempt_wakeup(structstruct cfs_rq *cfs_rq =task_cfs_rq (curr);int scale = cfs_rq->nr_running >= sched_nr_latency; -if (unlikely (rt_prio (p->prio))) - goto preempt; - -if (unlikely (p->sched_class != &fair_sched_class)) -return ; -if (unlikely (se == pse))return ;Index:
linux-2.6/kernel/sched_stoptask.c
===================================================================kernel/sched_stoptask.c +1
-1@@ -19,7 +19,7 @@ select_task_rq_stop(struct rq *rq, strucstatic void check_preempt_curr_stop (struct rq *rq,struct task_struct *p,int flags) { -resched_task (rq->curr);/* we preempt everything */ +/* we're never preempted */ }static struct task_struct *pick_next_task_stop (struct rq *rq)
Sergey Senozhatsky
Tested-byQuoted 11 lines · Peter Zijlstra
Subject: sched: Fixup cross
sched_classwakeup preemption
From: Peter Zijlstra <[email protected]>
Date: Sun Oct 31 12:37:04 CET 2010Instead of dealing with sched classes inside each
check_preempt_curr()implementation, pull out this logic into the generic wakeup preemption path.This fixes a hang in KVM (and others) where we are waiting for the stop machine thread to run..
Hello,
Peter, patch seems to work.
Sergey