sched: Fix missing preemption opportunity
Ingo,
Please pull the sched/urgent branch that can be found at:
git://git.kernel.org/pub/scm/linux/kernel/git/frederic/linux-dynticks.git
sched/urgent
HEAD: 4ccbe99852951957419bc616f888e297a478e50b
It's part of the preempt/schedule cleanups series suggested by Linus. I'm reworking these but this one commit is an exception because it's a bugfix. So I'm sending it now. I believe it's not a regression so it's perhaps too late for -rc5. I let you judge. The commit is based on -rc5 so it can be merged on sched/core otherwise.
Thanks,
Frederic
---
Frederic Weisbecker (1):
sched: Fix missing preemption check in cond_resched()
kernel/sched/core.c | 40 +++++++++++++++++++--------------------- 1 file changed, 19 insertions(+), 21 deletions(-)
---
commit 4ccbe99852951957419bc616f888e297a478e50b
Author: Frederic Weisbecker <[email protected]> Date: Sun Dec 14 22:04:47 2014 +0100
sched: Fix missing preemption check in cond_resched()
If an interrupt fires in cond_resched(), between the call to __schedule() and the PREEMPT_ACTIVE count decrementation, and that interrupt sets TIF_NEED_RESCHED, the call to preempt_schedule_irq() will be ignored due to the PREEMPT_ACTIVE count. This kind of scenario, with irq preemption being delayed because it's interrupting a preempt-disabled area, is usually fixed up after preemption is re-enabled back with an explicit call to preempt_schedule().
This is what preempt_enable() does but a raw preempt count decrement as performed by __preempt_count_sub(PREEMPT_ACTIVE) doesn't handle delayed preemption check. Therefore when such a race happens, the rescheduling is going to be delayed until the next scheduler or preemption entrypoint. This can be a problem for scheduler latency sensitive workloads.
Lets fix that by consolidating cond_resched() with preempt_schedule() internals.
kernel/sched/core.c +19 -21
@@ -2877,6 +2877,21 @@ void __sched schedule_preempt_disabled(void)preempt_disable (); } +static void preempt_schedule_common (void ) +{ +do { +__preempt_count_add (PREEMPT_ACTIVE ); +__schedule (); +__preempt_count_sub (PREEMPT_ACTIVE ); + +/* +* Check again in case we missed a preemption opportunity +* between schedule and now. +*/ +barrier (); + }while (need_resched ()); +} +#ifdef CONFIG_PREEMPT /* * this is the entry point to schedule() from in-kernel preemption
@@ -2892,17 +2907,7 @@ asmlinkage __visible void __sched notrace preempt_schedule(void)if (likely (!preemptible ()))return ; -do { -__preempt_count_add (PREEMPT_ACTIVE ); -__schedule (); -__preempt_count_sub (PREEMPT_ACTIVE ); - -/* -* Check again in case we missed a preemption opportunity -* between schedule and now. -*/ -barrier (); - }while (need_resched ()); +preempt_schedule_common (); }NOKPROBE_SYMBOL (preempt_schedule);EXPORT_SYMBOL (preempt_schedule);
@@ -4202,17 +4207,10 @@ SYSCALL_DEFINE0(sched_yield)return 0 ; } -static void __cond_resched (void ) -{ -__preempt_count_add (PREEMPT_ACTIVE ); -__schedule (); -__preempt_count_sub (PREEMPT_ACTIVE ); -} - int __sched _cond_resched(void) {if (should_resched ()) { -__cond_resched (); +preempt_schedule_common ();return 1 ; }return 0 ;
@@ -4237,7 +4235,7 @@ int __cond_resched_lock(spinlock_t *lock)if (spin_needbreak (lock) || resched) {spin_unlock (lock);if (resched) -__cond_resched (); +preempt_schedule_common ();else cpu_relax (); ret =1 ;
@@ -4253,7 +4251,7 @@ int __sched __cond_resched_softirq(void)if (should_resched ()) {local_bh_enable (); -__cond_resched (); +preempt_schedule_common ();local_bh_disable ();return 1 ; }
Peter Zijlstra
maintainerI picked up the patch; will drop it if Ingo also does ;-)
Quoted 7 lines · Frederic Weisbecker
+++
b/kernel/sched/core.ckernel/sched/core.c +2
-0@@ -2877,6 +2877,21 @@ void __sched schedule_preempt_disabled(void)preempt_disable (); } +static void preempt_schedule_common (void ) +{
kernel/sched/core.c void __sched schedule_preempt_disabled(void) · line 2891
Quoted 10 lines
kernel/sched/core.c +10
-0+ do { + __preempt_count_add(PREEMPT_ACTIVE); + __schedule(); + __preempt_count_sub(PREEMPT_ACTIVE); + + /* + * Check again in case we missed a preemption opportunity + * between schedule and now. + */ + barrier();
I do however wonder about this barrier() here; why do we think we need it?
Is that because test_bit() it 'broken'? The bitops are typically atomic ops and atomic reads should be through a volatile cast (x86 constant_test_bit doesn't seem to do this).
Should we go audit and fix that?
Quoted 2 lines
kernel/sched/core.c +2
-0+ }while (need_resched ()); +}
Quoted 25 lines · Peter Zijlstra
I picked up the patch; will drop it if Ingo also does ;-)
Quoted 7 lines · Frederic Weisbecker
+++
b/kernel/sched/core.ckernel/sched/core.c +2
-0@@ -2877,6 +2877,21 @@ void __sched schedule_preempt_disabled(void)preempt_disable (); } +static void preempt_schedule_common (void ) +{kernel/sched/core.c void __sched schedule_preempt_disabled(void) · line 2891
Quoted 10 lines
kernel/sched/core.c +10
-0+ do { + __preempt_count_add(PREEMPT_ACTIVE); + __schedule(); + __preempt_count_sub(PREEMPT_ACTIVE); + + /* + * Check again in case we missed a preemption opportunity + * between schedule and now. + */ + barrier();I do however wonder about this
barrier()here; why do we think we need it?
Quoted 5 lines
Is that because
test_bit()it 'broken'? The bitops are typically atomic ops and atomic reads should be through a volatile cast (x86constant_test_bitdoesn't seem to do this).Should we go audit and fix that?
I looked up with git blame and this was already there prior the first git commit v2.6.12 without appropriate explanation.
We must make sure that the PREEMPT_ACTIVE decrement is visible before we do the NEED_RESCHED test or an interrupt could spuriously miss a preempt_schedule_irq() opportunity.
__preempt_count_sub() in asm-generic is an inline, so an implicit barrier(). Only x86 overwrites it yet and it does so through an inline as well.
And __preempt_count_ops() must really imply a barrier() anyway, anything else would be insane (probably we should specify that in a comment somewhere). Although I see that responsability is taken from non-underscored callers...
tip-bot for Frederic Weisbecker
merge botsched: Fix missing preemption opportunity
Commit-ID: a18b5d01819235629289212ad428a5ee2b40f0d9 Gitweb: http://git.kernel.org/tip/a18b5d01819235629289212ad428a5ee2b40f0d9 Author: Frederic Weisbecker <[email protected]> AuthorDate: Thu, 22 Jan 2015 18:08:04 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Fri, 30 Jan 2015 19:38:51 +0100
sched: Fix missing preemption opportunity
If an interrupt fires in cond_resched(), between the call to __schedule() and the PREEMPT_ACTIVE count decrementation, and that interrupt sets TIF_NEED_RESCHED, the call to preempt_schedule_irq() will be ignored due to the PREEMPT_ACTIVE count. This kind of scenario, with irq preemption being delayed because it's interrupting a preempt-disabled area, is usually fixed up after preemption is re-enabled back with an explicit call to preempt_schedule().
This is what preempt_enable() does but a raw preempt count decrement as performed by __preempt_count_sub(PREEMPT_ACTIVE) doesn't handle delayed preemption check. Therefore when such a race happens, the rescheduling is going to be delayed until the next scheduler or preemption entrypoint. This can be a problem for scheduler latency sensitive workloads.
Lets fix that by consolidating cond_resched() with preempt_schedule() internals.
--- kernel/sched/core.c | 40 +++++++++++++++++++--------------------- 1 file changed, 19 insertions(+), 21 deletions(-)
kernel/sched/core.c +19 -21
@@ -2884,6 +2884,21 @@ void __sched schedule_preempt_disabled(void)preempt_disable (); } +static void preempt_schedule_common (void ) +{ +do { +__preempt_count_add (PREEMPT_ACTIVE ); +__schedule (); +__preempt_count_sub (PREEMPT_ACTIVE ); + +/* +* Check again in case we missed a preemption opportunity +* between schedule and now. +*/ +barrier (); + }while (need_resched ()); +} +#ifdef CONFIG_PREEMPT /* * this is the entry point to schedule() from in-kernel preemption
@@ -2899,17 +2914,7 @@ asmlinkage __visible void __sched notrace preempt_schedule(void)if (likely (!preemptible ()))return ; -do { -__preempt_count_add (PREEMPT_ACTIVE ); -__schedule (); -__preempt_count_sub (PREEMPT_ACTIVE ); - -/* -* Check again in case we missed a preemption opportunity -* between schedule and now. -*/ -barrier (); - }while (need_resched ()); +preempt_schedule_common (); }NOKPROBE_SYMBOL (preempt_schedule);EXPORT_SYMBOL (preempt_schedule);
@@ -4209,17 +4214,10 @@ SYSCALL_DEFINE0(sched_yield)return 0 ; } -static void __cond_resched (void ) -{ -__preempt_count_add (PREEMPT_ACTIVE ); -__schedule (); -__preempt_count_sub (PREEMPT_ACTIVE ); -} - int __sched _cond_resched(void) {if (should_resched ()) { -__cond_resched (); +preempt_schedule_common ();return 1 ; }return 0 ;
@@ -4244,7 +4242,7 @@ int __cond_resched_lock(spinlock_t *lock)if (spin_needbreak (lock) || resched) {spin_unlock (lock);if (resched) -__cond_resched (); +preempt_schedule_common ();else cpu_relax (); ret =1 ;
@@ -4260,7 +4258,7 @@ int __sched __cond_resched_softirq(void)if (should_resched ()) {local_bh_enable (); -__cond_resched (); +preempt_schedule_common ();local_bh_disable ();return 1 ; }