sched: fixes for rt-migration-test failures
Steven Rostedt
reviewerIngo,
We've been experiencing rt-migration-test failures
http://rostedt.homelinux.com/code/rt-migrate-test.c
Gregory Haskins and myself have been debugging this for several days. We found three bugs so far in the code that causes the failure of this test. The failure is that an RT task may not migrate properly and may wait behind a lower RT prio task for the migration.
These two patches solve two of the bugs. There's also a bug that is triggered with RT_GROUP scheduling that we will look at later.
The test can still fail after a long time running and we are still looking into other issues. But for now this solves two of those issues.
Please pull the latest update tree, which can be found at:
git://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-2.6-trace.git
update
Steven Rostedt (2):
sched: check for pushing rt tasks after all scheduling
sched: add new prio to cpupri before removing old prio
----
kernel/sched.c | 38 +++++++++++++++++++++++++++----------- kernel/sched_cpupri.c | 30 ++++++++++++++++-------------- 2 files changed, 43 insertions(+), 25 deletions(-)
Steven Rostedt
reviewerPATCH 1/2 sched: check for pushing rt tasks after all scheduling
From: Steven Rostedt <[email protected]>
The current method for pushing RT tasks after scheduling only happens after a context switch. But we found cases where a task is set up on a run queue to be pushed but the push never happens because the schedule chooses the same task.
This bug was found with the help of Gregory Haskins and the use of ftrace (trace_printk). It tooks several days for both of us analyzing the code and the trace output to find this.
--- kernel/sched.c | 38 +++++++++++++++++++++++++++----------- 1 files changed, 27 insertions(+), 11 deletions(-)
kernel/sched.c +27 -11
@@ -2791,14 +2791,14 @@ prepare_task_switch(struct rq *rq, struct task_struct *prev,* with the lock held can cause deadlocks; see schedule() for * details.) */ -static void finish_task_switch (struct rq *rq,struct task_struct *prev) +static int finish_task_switch (struct rq *rq,struct task_struct *prev) __releases(rq->lock) {struct mm_struct *mm = rq->prev_mm;long prev_state; -#ifdef CONFIG_SMP int post_schedule =0 ; +#ifdef CONFIG_SMP if (current->sched_class->needs_post_schedule) post_schedule = current->sched_class->needs_post_schedule (rq);#endif
@@ -2820,10 +2820,6 @@ static void finish_task_switch(struct rq *rq, struct task_struct *prev)finish_arch_switch (prev);perf_counter_task_sched_in (current,cpu_of (rq));finish_lock_switch (rq, prev); -#ifdef CONFIG_SMP -if (post_schedule) - current->sched_class->post_schedule (rq); -#endif fire_sched_in_preempt_notifiers (current);if (mm)
@@ -2836,6 +2832,8 @@ static void finish_task_switch(struct rq *rq, struct task_struct *prev)kprobe_flush_task (prev);put_task_struct (prev); } + +return post_schedule; }/**
@@ -2846,8 +2844,15 @@ asmlinkage void schedule_tail(struct task_struct *prev) __releases(rq->lock) {struct rq *rq =this_rq (); +int post_schedule; + + post_schedule =finish_task_switch (rq, prev); + +#ifdef CONFIG_SMP +if (post_schedule) + current->sched_class->post_schedule (rq); +#endif -finish_task_switch (rq, prev);#ifdef __ARCH_WANT_UNLOCKED_CTXSW/* In this case, finish_task_switch does not reenable preemption */ preempt_enable ();
@@ -2860,7 +2865,7 @@ asmlinkage void schedule_tail(struct task_struct *prev)* context_switch - switch to the new MM and the new * thread's register state. */ -static inline void +static inline int context_switch (struct rq *rq,struct task_struct *prev,struct task_struct *next) {
@@ -2907,7 +2912,7 @@ context_switch(struct rq *rq, struct task_struct *prev,* CPUs since it called schedule(), thus the 'rq' on its stack * frame will be invalid. */ -finish_task_switch (this_rq (), prev); +return finish_task_switch (this_rq (), prev); }/*
@@ -5318,6 +5323,7 @@ asmlinkage void __sched schedule(void) {struct task_struct *prev, *next;unsigned long *switch_count; +int post_schedule =0 ;struct rq *rq;int cpu;
@@ -5368,15 +5374,25 @@ need_resched_nonpreemptible: rq->curr = next; ++*switch_count; -context_switch (rq, prev, next);/* unlocks the rq */ + post_schedule =context_switch (rq, prev, next);/* unlocks the rq */ /* * the context switch might have flipped the stack from under * us, hence refresh the local variables. */ cpu =smp_processor_id (); rq =cpu_rq (cpu); - }else + }else { +#ifdef CONFIG_SMP +if (current->sched_class->needs_post_schedule) + post_schedule = current->sched_class->needs_post_schedule (rq); +#endif spin_unlock_irq (&rq->lock); + } + +#ifdef CONFIG_SMP +if (post_schedule) + current->sched_class->post_schedule (rq); +#endif if (unlikely (reacquire_kernel_lock (current) <0 )) goto need_resched_nonpreemptible;
Signature · 3 lines
1.6.3.3 --
Peter Zijlstra
maintainerRe: [PATCH 1/2] sched: check for pushing rt tasks after all scheduling
Quoted 14 lines · Steven Rostedt
plain text document attachment (0001-sched-check-for-pushing-rt-tasks-after-all-schedulin.patch)
From: Steven Rostedt <[email protected]>The current method for pushing RT tasks after scheduling only happens after a context switch. But we found cases where a task is set up on a run queue to be pushed but the push never happens because the schedule chooses the same task.
This bug was found with the help of Gregory Haskins and the use of ftrace (
trace_printk). It tooks several days for both of us analyzing the code and the trace output to find this.
kernel/sched.c need_resched_nonpreemptible:
Quoted 6 lines
kernel/sched.c +4
-0+if (current->sched_class->needs_post_schedule) + post_schedule = current->sched_class->needs_post_schedule (rq); +if (post_schedule) + current->sched_class->post_schedule (rq);
Why can't we omit that first call, and do the second unconditionally, using storage in the class rq to save state?
Re: [PATCH 1/2] sched: check for pushing rt tasks after all scheduling
Quoted 19 lines · Peter Zijlstra
Quoted 16 lines · Steven Rostedt
plain text document attachment (0001-sched-check-for-pushing-rt-tasks-after-all-schedulin.patch)
From: Steven Rostedt <[email protected]>The current method for pushing RT tasks after scheduling only happens after a context switch. But we found cases where a task is set up on a run queue to be pushed but the push never happens because the schedule chooses the same task.
This bug was found with the help of Gregory Haskins and the use of ftrace (
trace_printk). It tooks several days for both of us analyzing the code and the trace output to find this.kernel/sched.c +2
-0+if (current->sched_class->needs_post_schedule) + post_schedule = current->sched_class->needs_post_schedule (rq);
Quoted 6 lines
Review
Quoted 2 lines
(quoted diff) +2
-0+ if (post_schedule) + current->sched_class->post_schedule(rq);Why can't we omit that first call, and do the second unconditionally, using storage in the class rq to save state?
Yeah, that is a good idea. Plus I see another bug that Steven and I overlooked. Steve is out on holiday today, so I will put together a v2.
Regards,
-Greg
PATCH 1/1 sched: enhance the pre/post scheduling logic
In reply to Peter Zijlstra(Applies to Steve's linux-2.6-trace.git/update:c8cab1dd)
This patch applies on top of Steven's recent submission to address the feedback from Peter, as well as fix a different race that was not previously noticed.
It should be noted that all three of these patches should be considered for any kernel that includes our push/pull scheduler design, which I believe includes 24-rt and newer, as well as roughly 2.6.27 mainline.
Kind Regards,
-Greg
---------------------------
From: Gregory Haskins <[email protected]>
sched: enhance the pre/post scheduling logic
We currently have an explicit "needs_post" vtable method which returns a stack variable for whether we should later run post-schedule. This leads to an awkward exchange of the variable as it bubbles back up out of the context switch. Peter Zijlstra observed that this information could be stored in the run-queue itself instead of handled on the stack.
Therefore, we revert to the method of having context_switch return void, and update an internal rq->post_schedule variable when we require further processing.
In addition, we fix a race condition where we try to access current->sched_class without holding the rq->lock. This is technically racy, as the sched-class could change out from under us. Instead, we reference the per-rq post_schedule variable with the runqueue unlocked, but with preemption disabled to see if we need to reacquire the rq->lock.
Finally, we clean the code up slightly by removing the #ifdef CONFIG_SMP conditionals from the schedule() call, and implement some inline helper functions instead.
This patch passes checkpatch, and rt-migrate
---
include/linux/sched.h | 1 - kernel/sched.c | 82 ++++++++++++++++++++++++++++++------------------- kernel/sched_rt.c | 31 +++++++------------ 3 files changed, 61 insertions(+), 53 deletions(-)
include/linux/sched.h +0 -1
@@ -1045,7 +1045,6 @@ struct sched_class { struct rq *busiest, struct sched_domain *sd, enum cpu_idle_type idle);void (*pre_schedule) (struct rq *this_rq,struct task_struct *task); -int (*needs_post_schedule) (struct rq *this_rq);void (*post_schedule) (struct rq *this_rq);void (*task_wake_up) (struct rq *this_rq,struct task_struct *task);
kernel/sched.c +50 -32
159 lines not on this page · raw message
kernel/sched_rt.c +11 -20
@@ -1056,6 +1056,11 @@ static struct task_struct *_pick_next_task_rt(struct rq *rq)return p; } +static inline int has_pushable_tasks (struct rq *rq) +{ +return !plist_head_empty (&rq->rt.pushable_tasks); +} +static struct task_struct *pick_next_task_rt (struct rq *rq) {struct task_struct *p =_pick_next_task_rt (rq);
@@ -1064,6 +1069,12 @@ static struct task_struct *pick_next_task_rt(struct rq *rq)if (p)dequeue_pushable_task (rq, p); +/* +* We detect this state here so that we can avoid taking the RQ +* lock again later if there is no need to push +*/ + rq->post_schedule =has_pushable_tasks (rq); +return p; }
@@ -1262,11 +1273,6 @@ static struct rq *find_lock_lowest_rq(struct task_struct *task, struct rq *rq)return lowest_rq; } -static inline int has_pushable_tasks (struct rq *rq) -{ -return !plist_head_empty (&rq->rt.pushable_tasks); -} -static struct task_struct *pick_next_pushable_task (struct rq *rq) {struct task_struct *p;
@@ -1466,23 +1472,9 @@ static void pre_schedule_rt(struct rq *rq, struct task_struct *prev)pull_rt_task (rq); } -/* -* assumes rq->lock is held -*/ -static int needs_post_schedule_rt (struct rq *rq) -{ -return has_pushable_tasks (rq); -} -static void post_schedule_rt (struct rq *rq) { -/* -* This is only called if needs_post_schedule_rt() indicates that -* we need to push tasks away -*/ -spin_lock_irq (&rq->lock);push_rt_tasks (rq); -spin_unlock_irq (&rq->lock); }/*
@@ -1758,7 +1750,6 @@ static const struct sched_class rt_sched_class = { .rq_online = rq_online_rt, .rq_offline = rq_offline_rt, .pre_schedule = pre_schedule_rt, - .needs_post_schedule = needs_post_schedule_rt, .post_schedule = post_schedule_rt, .task_wake_up = task_wake_up_rt, .switched_from = switched_from_rt,
Peter Zijlstra
maintainerRe: [PATCH] sched: enhance the pre/post scheduling logic
include/linux/sched.h struct sched_class { · line 1049
Quoted 11 lines · Gregory Haskins
include/linux/sched.h +0
-1@@ -1045,7 +1045,6 @@ struct sched_class { struct rq *busiest, struct sched_domain *sd, enum cpu_idle_type idle);void (*pre_schedule) (struct rq *this_rq,struct task_struct *task); -int (*needs_post_schedule) (struct rq *this_rq);void (*post_schedule) (struct rq *this_rq);void (*task_wake_up) (struct rq *this_rq,struct task_struct *task);
awesome, one method less ;-)
Quoted 26 lines
include/linux/sched.h +25
-0+#ifdef CONFIG_SMP + +/* assumes rq->lock is held */ +static inline void pre_schedule (struct rq *rq,struct task_struct *prev) +{ +if (prev->sched_class->pre_schedule) + prev->sched_class->pre_schedule (rq, prev); +} + +/* rq->lock is NOT held, but preemption is disabled */ +static inline void post_schedule (struct rq *rq) +{ +if (rq->post_schedule) { +unsigned long flags; + +spin_lock_irqsave (&rq->lock, flags); +if (rq->curr->sched_class->post_schedule) + rq->curr->sched_class->post_schedule (rq); +spin_unlock_irqrestore (&rq->lock, flags); + + rq->post_schedule =0 ; + } +} + +#else
include/linux/sched.h struct sched_class {
Quoted 10 lines
include/linux/sched.h +7
-1-return post_schedule; +static inline void pre_schedule (struct rq *rq,struct task_struct *p) +{ +} + +static inline void post_schedule (struct rq *rq) +{ } +#endif
Wouldn't you sleep much safer at night if both versions were to check those assumptions under SCHED_DEBUG? :-)
Quoted 11 lines
include/linux/sched.h +1
-4@@ -2844,14 +2873,14 @@ asmlinkage void schedule_tail(struct task_struct *prev) __releases(rq->lock) {struct rq *rq =this_rq (); -int post_schedule; - post_schedule =finish_task_switch (rq, prev); +finish_task_switch (rq, prev); -#ifdef CONFIG_SMP -if (post_schedule)
include/linux/sched.h asmlinkage void schedule_tail(struct task_struct *prev) · line 2886
Quoted 10 lines
include/linux/sched.h +5
-2- current->sched_class->post_schedule (rq); -#endif +/* +* FIXME: do we need to worry about rq being invalidated by the +* task_switch? +*/ +post_schedule (rq);#ifdef __ARCH_WANT_UNLOCKED_CTXSW/* In this case, finish_task_switch does not reenable preemption */
You know I really can't take patches with FIXME's in ;-)
I think only switch_to() messes with your stacks, finish_task_switch() should be safe, but double check me.
OK, so I stuck the patch in anyway..
[tip:sched/core] sched: Enhance the pre/post scheduling logic
In reply to Gregory HaskinsCommit-ID: 3f029d3c6d62068d59301d90c18dbde8ee402107 Gitweb: http://git.kernel.org/tip/3f029d3c6d62068d59301d90c18dbde8ee402107 Author: Gregory Haskins <[email protected]> AuthorDate: Wed, 29 Jul 2009 11:08:47 -0400 Committer: Ingo Molnar <[email protected]> CommitDate: Sun, 2 Aug 2009 14:26:10 +0200
sched: Enhance the pre/post scheduling logic
We currently have an explicit "needs_post" vtable method which returns a stack variable for whether we should later run post-schedule. This leads to an awkward exchange of the variable as it bubbles back up out of the context switch. Peter Zijlstra observed that this information could be stored in the run-queue itself instead of handled on the stack.
Therefore, we revert to the method of having context_switch return void, and update an internal rq->post_schedule variable when we require further processing.
In addition, we fix a race condition where we try to access current->sched_class without holding the rq->lock. This is technically racy, as the sched-class could change out from under us. Instead, we reference the per-rq post_schedule variable with the runqueue unlocked, but with preemption disabled to see if we need to reacquire the rq->lock.
Finally, we clean the code up slightly by removing the #ifdef CONFIG_SMP conditionals from the schedule() call, and implement some inline helper functions instead.
This patch passes checkpatch, and rt-migrate.
LKML-Reference: <[email protected]>
--- include/linux/sched.h | 1 - kernel/sched.c | 82 ++++++++++++++++++++++++++++++------------------- kernel/sched_rt.c | 31 ++++++------------ 3 files changed, 61 insertions(+), 53 deletions(-)
include/linux/sched.h +0 -1
@@ -1047,7 +1047,6 @@ struct sched_class { struct rq *busiest, struct sched_domain *sd, enum cpu_idle_type idle);void (*pre_schedule) (struct rq *this_rq,struct task_struct *task); -int (*needs_post_schedule) (struct rq *this_rq);void (*post_schedule) (struct rq *this_rq);void (*task_wake_up) (struct rq *this_rq,struct task_struct *task);
kernel/sched.c +50 -32
159 lines not on this page · raw message
kernel/sched_rt.c +11 -20
@@ -1056,6 +1056,11 @@ static struct task_struct *_pick_next_task_rt(struct rq *rq)return p; } +static inline int has_pushable_tasks (struct rq *rq) +{ +return !plist_head_empty (&rq->rt.pushable_tasks); +} +static struct task_struct *pick_next_task_rt (struct rq *rq) {struct task_struct *p =_pick_next_task_rt (rq);
@@ -1064,6 +1069,12 @@ static struct task_struct *pick_next_task_rt(struct rq *rq)if (p)dequeue_pushable_task (rq, p); +/* +* We detect this state here so that we can avoid taking the RQ +* lock again later if there is no need to push +*/ + rq->post_schedule =has_pushable_tasks (rq); +return p; }
@@ -1262,11 +1273,6 @@ static struct rq *find_lock_lowest_rq(struct task_struct *task, struct rq *rq)return lowest_rq; } -static inline int has_pushable_tasks (struct rq *rq) -{ -return !plist_head_empty (&rq->rt.pushable_tasks); -} -static struct task_struct *pick_next_pushable_task (struct rq *rq) {struct task_struct *p;
@@ -1466,23 +1472,9 @@ static void pre_schedule_rt(struct rq *rq, struct task_struct *prev)pull_rt_task (rq); } -/* -* assumes rq->lock is held -*/ -static int needs_post_schedule_rt (struct rq *rq) -{ -return has_pushable_tasks (rq); -} -static void post_schedule_rt (struct rq *rq) { -/* -* This is only called if needs_post_schedule_rt() indicates that -* we need to push tasks away -*/ -spin_lock_irq (&rq->lock);push_rt_tasks (rq); -spin_unlock_irq (&rq->lock); }/*
@@ -1758,7 +1750,6 @@ static const struct sched_class rt_sched_class = { .rq_online = rq_online_rt, .rq_offline = rq_offline_rt, .pre_schedule = pre_schedule_rt, - .needs_post_schedule = needs_post_schedule_rt, .post_schedule = post_schedule_rt, .task_wake_up = task_wake_up_rt, .switched_from = switched_from_rt,
[tip:sched/core] sched: Check for pushing rt tasks after all scheduling
In reply to Steven RostedtCommit-ID: da19ab510343c6496fe8b8f890091296032025c9 Gitweb: http://git.kernel.org/tip/da19ab510343c6496fe8b8f890091296032025c9 Author: Steven Rostedt <[email protected]> AuthorDate: Wed, 29 Jul 2009 00:21:22 -0400 Committer: Ingo Molnar <[email protected]> CommitDate: Sun, 2 Aug 2009 14:26:08 +0200
sched: Check for pushing rt tasks after all scheduling
The current method for pushing RT tasks after scheduling only happens after a context switch. But we found cases where a task is set up on a run queue to be pushed but the push never happens because the schedule chooses the same task.
This bug was found with the help of Gregory Haskins and the use of ftrace (trace_printk). It tooks several days for both of us analyzing the code and the trace output to find this.
LKML-Reference: <[email protected]>
--- kernel/sched.c | 38 +++++++++++++++++++++++++++----------- 1 files changed, 27 insertions(+), 11 deletions(-)
kernel/sched.c +27 -11
@@ -2839,14 +2839,14 @@ prepare_task_switch(struct rq *rq, struct task_struct *prev,* with the lock held can cause deadlocks; see schedule() for * details.) */ -static void finish_task_switch (struct rq *rq,struct task_struct *prev) +static int finish_task_switch (struct rq *rq,struct task_struct *prev) __releases(rq->lock) {struct mm_struct *mm = rq->prev_mm;long prev_state; -#ifdef CONFIG_SMP int post_schedule =0 ; +#ifdef CONFIG_SMP if (current->sched_class->needs_post_schedule) post_schedule = current->sched_class->needs_post_schedule (rq);#endif
@@ -2868,10 +2868,6 @@ static void finish_task_switch(struct rq *rq, struct task_struct *prev)finish_arch_switch (prev);perf_counter_task_sched_in (current,cpu_of (rq));finish_lock_switch (rq, prev); -#ifdef CONFIG_SMP -if (post_schedule) - current->sched_class->post_schedule (rq); -#endif fire_sched_in_preempt_notifiers (current);if (mm)
@@ -2884,6 +2880,8 @@ static void finish_task_switch(struct rq *rq, struct task_struct *prev)kprobe_flush_task (prev);put_task_struct (prev); } + +return post_schedule; }/**
@@ -2894,8 +2892,15 @@ asmlinkage void schedule_tail(struct task_struct *prev) __releases(rq->lock) {struct rq *rq =this_rq (); +int post_schedule; + + post_schedule =finish_task_switch (rq, prev); + +#ifdef CONFIG_SMP +if (post_schedule) + current->sched_class->post_schedule (rq); +#endif -finish_task_switch (rq, prev);#ifdef __ARCH_WANT_UNLOCKED_CTXSW/* In this case, finish_task_switch does not reenable preemption */ preempt_enable ();
@@ -2908,7 +2913,7 @@ asmlinkage void schedule_tail(struct task_struct *prev)* context_switch - switch to the new MM and the new * thread's register state. */ -static inline void +static inline int context_switch (struct rq *rq,struct task_struct *prev,struct task_struct *next) {
@@ -2955,7 +2960,7 @@ context_switch(struct rq *rq, struct task_struct *prev,* CPUs since it called schedule(), thus the 'rq' on its stack * frame will be invalid. */ -finish_task_switch (this_rq (), prev); +return finish_task_switch (this_rq (), prev); }/*
@@ -5366,6 +5371,7 @@ asmlinkage void __sched schedule(void) {struct task_struct *prev, *next;unsigned long *switch_count; +int post_schedule =0 ;struct rq *rq;int cpu;
@@ -5416,15 +5422,25 @@ need_resched_nonpreemptible: rq->curr = next; ++*switch_count; -context_switch (rq, prev, next);/* unlocks the rq */ + post_schedule =context_switch (rq, prev, next);/* unlocks the rq */ /* * the context switch might have flipped the stack from under * us, hence refresh the local variables. */ cpu =smp_processor_id (); rq =cpu_rq (cpu); - }else + }else { +#ifdef CONFIG_SMP +if (current->sched_class->needs_post_schedule) + post_schedule = current->sched_class->needs_post_schedule (rq); +#endif spin_unlock_irq (&rq->lock); + } + +#ifdef CONFIG_SMP +if (post_schedule) + current->sched_class->post_schedule (rq); +#endif if (unlikely (reacquire_kernel_lock (current) <0 )) goto need_resched_nonpreemptible;
Steven Rostedt
reviewerPATCH 2/2 sched: add new prio to cpupri before removing old prio
In reply to Steven RostedtFrom: Steven Rostedt <[email protected]>
We need to add the new prio to the cpupri accounting before removing the old prio. This is because removing the old prio first will open a race window where the cpu will be removed from pri_active. In this case the cpu will not be visible for RT push and pulls. This could cause a RT task to not migrate appropriately, and create a very large latency.
This bug was found with the use of ftrace sched events and trace_printk.
--- kernel/sched_cpupri.c | 30 ++++++++++++++++-------------- 1 files changed, 16 insertions(+), 14 deletions(-)
kernel/sched_cpupri.c +16 -14
@@ -114,21 +114,11 @@ void cpupri_set(struct cpupri *cp, int cpu, int newpri)/* * If the cpu was currently mapped to a different value, we -* first need to unmap the old value +* need to map it to the new value then remove the old value. +* Note, we must add the new value first, otherwise we risk the +* cpu being cleared from pri_active, and this cpu could be +* missed for a push or pull. */ -if (likely (oldpri !=CPUPRI_INVALID )) { -struct cpupri_vec *vec = &cp->pri_to_cpu[oldpri]; - -spin_lock_irqsave (&vec->lock, flags); - - vec->count--; -if (!vec->count) -clear_bit (oldpri, cp->pri_active); -cpumask_clear_cpu (cpu, vec->mask); - -spin_unlock_irqrestore (&vec->lock, flags); - } -if (likely (newpri !=CPUPRI_INVALID )) {struct cpupri_vec *vec = &cp->pri_to_cpu[newpri];
@@ -141,6 +131,18 @@ void cpupri_set(struct cpupri *cp, int cpu, int newpri)spin_unlock_irqrestore (&vec->lock, flags); } +if (likely (oldpri !=CPUPRI_INVALID )) { +struct cpupri_vec *vec = &cp->pri_to_cpu[oldpri]; + +spin_lock_irqsave (&vec->lock, flags); + + vec->count--; +if (!vec->count) +clear_bit (oldpri, cp->pri_active); +cpumask_clear_cpu (cpu, vec->mask); + +spin_unlock_irqrestore (&vec->lock, flags); + } *currpri = newpri; }
Signature · 3 lines
1.6.3.3 --
[tip:sched/core] sched: Add new prio to cpupri before removing old prio
Commit-ID: c3a2ae3d93c0f10d29c071f599764d00b8de00cb Gitweb: http://git.kernel.org/tip/c3a2ae3d93c0f10d29c071f599764d00b8de00cb Author: Steven Rostedt <[email protected]> AuthorDate: Wed, 29 Jul 2009 00:21:23 -0400 Committer: Ingo Molnar <[email protected]> CommitDate: Sun, 2 Aug 2009 14:26:09 +0200
sched: Add new prio to cpupri before removing old prio
We need to add the new prio to the cpupri accounting before removing the old prio. This is because removing the old prio first will open a race window where the cpu will be removed from pri_active. In this case the cpu will not be visible for RT push and pulls. This could cause a RT task to not migrate appropriately, and create a very large latency.
This bug was found with the use of ftrace sched events and trace_printk.
LKML-Reference: <[email protected]>
--- kernel/sched_cpupri.c | 30 ++++++++++++++++-------------- 1 files changed, 16 insertions(+), 14 deletions(-)
kernel/sched_cpupri.c +16 -14
@@ -127,21 +127,11 @@ void cpupri_set(struct cpupri *cp, int cpu, int newpri)/* * If the cpu was currently mapped to a different value, we -* first need to unmap the old value +* need to map it to the new value then remove the old value. +* Note, we must add the new value first, otherwise we risk the +* cpu being cleared from pri_active, and this cpu could be +* missed for a push or pull. */ -if (likely (oldpri !=CPUPRI_INVALID )) { -struct cpupri_vec *vec = &cp->pri_to_cpu[oldpri]; - -spin_lock_irqsave (&vec->lock, flags); - - vec->count--; -if (!vec->count) -clear_bit (oldpri, cp->pri_active); -cpumask_clear_cpu (cpu, vec->mask); - -spin_unlock_irqrestore (&vec->lock, flags); - } -if (likely (newpri !=CPUPRI_INVALID )) {struct cpupri_vec *vec = &cp->pri_to_cpu[newpri];
@@ -154,6 +144,18 @@ void cpupri_set(struct cpupri *cp, int cpu, int newpri)spin_unlock_irqrestore (&vec->lock, flags); } +if (likely (oldpri !=CPUPRI_INVALID )) { +struct cpupri_vec *vec = &cp->pri_to_cpu[oldpri]; + +spin_lock_irqsave (&vec->lock, flags); + + vec->count--; +if (!vec->count) +clear_bit (oldpri, cp->pri_active); +cpumask_clear_cpu (cpu, vec->mask); + +spin_unlock_irqrestore (&vec->lock, flags); + } *currpri = newpri; }