migrate disable fixes and performance
Subsystems (3)
v2 cover migrate disable fixes and performance
These are the unapplied patches from v1, minus the sched deadline patch, and with stop_one_cpu_nowait() in place of clobbering current->state.
Scott Wood (3):
sched: migrate_enable: Use select_fallback_rq()
sched: Lazy migrate_disable processing
sched: migrate_enable: Use stop_one_cpu_nowait()
include/linux/cpu.h | 4 - include/linux/sched.h | 11 +-- include/linux/stop_machine.h | 2 + init/init_task.c | 4 + kernel/cpu.c | 103 +++++++++-------------- kernel/sched/core.c | 195 ++++++++++++++++++------------------------- kernel/sched/sched.h | 4 + kernel/stop_machine.c | 7 +- lib/smp_processor_id.c | 3 + 9 files changed, 142 insertions(+), 191 deletions(-)
Signature · 1 line
1.8.3.1
PATCH v2 1/3 sched: migrate_enable: Use select_fallback_rq()
migrate_enable() currently open-codes a variant of select_fallback_rq(). However, it does not have the "No more Mr. Nice Guy" fallback and thus it will pass an invalid CPU to the migration thread if cpus_mask only contains a CPU that is !active.
---
This scenario will be more likely after the next patch, since the migrate_disable_update check goes away. However, it could happen anyway if cpus_mask was updated to a CPU other than the one we were pinned to, and that CPU subsequently became inactive.
--- kernel/sched/core.c | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-)
kernel/sched/core.c +10 -15
@@ -7405,6 +7405,7 @@ void migrate_enable(void)if (p->migrate_disable_update) {struct rq *rq;struct rq_flags rf; +int cpu =task_cpu (p); rq =task_rq_lock (p, &rf);update_rq_clock (rq);
@@ -7414,21 +7415,15 @@ void migrate_enable(void) p->migrate_disable_update =0 ; -WARN_ON (smp_processor_id () !=task_cpu (p)); -if (!cpumask_test_cpu (task_cpu (p), &p->cpus_mask)) { -const struct cpumask *cpu_valid_mask = cpu_active_mask; -struct migration_arg arg; -unsigned int dest_cpu; - -if (p->flags &PF_KTHREAD ) { -/* -* Kernel threads are allowed on online && !active CPUs -*/ - cpu_valid_mask = cpu_online_mask; - } - dest_cpu =cpumask_any_and (cpu_valid_mask, &p->cpus_mask); - arg.task = p; - arg.dest_cpu = dest_cpu; +WARN_ON (smp_processor_id () != cpu); +if (!cpumask_test_cpu (cpu, &p->cpus_mask)) { +struct migration_arg arg = { p }; +struct rq_flags rf; + + rq =task_rq_lock (p, &rf); +update_rq_clock (rq); + arg.dest_cpu =select_fallback_rq (cpu, p); +task_rq_unlock (rq, p, &rf);unpin_current_cpu ();preempt_lazy_enable ();
Signature · 1 line
1.8.3.1
PATCH v2 2/3 sched: Lazy migrate_disable processing
In reply to Scott WoodAvoid overhead on the majority of migrate disable/enable sequences by only manipulating scheduler data (and grabbing the relevant locks) when the task actually schedules while migrate-disabled. A kernel build showed around a 10% reduction in system time (with CONFIG_NR_CPUS=512).
Instead of cpuhp_pin_lock, CPU hotplug is handled by keeping a per-CPU count of the number of pinned tasks (including tasks which have not scheduled in the migrate-disabled section); takedown_cpu() will wait until that reaches zero (confirmed by take_cpu_down() in stop machine context to deal with races) before migrating tasks off of the cpu.
To simplify synchronization, updating cpus_mask is no longer deferred until migrate_enable(). This lets us not have to worry about migrate_enable() missing the update if it's on the fast path (didn't schedule during the migrate disabled section). It also makes the code a bit simpler and reduces deviation from mainline.
While the main motivation for this is the performance benefit, lazy migrate disable also eliminates the restriction on calling migrate_disable() while atomic but leaving the atomic region prior to calling migrate_enable() -- though this won't help with local_bh_disable() (and thus rcutorture) unless something similar is done with the recently added local_lock.
---
The speedup is smaller than before, due to commit 659252061477862f ("lib/smp_processor_id: Don't use cpumask_equal()") achieving an overlapping speedup.
--- include/linux/cpu.h | 4 -- include/linux/sched.h | 11 +-- init/init_task.c | 4 ++ kernel/cpu.c | 103 +++++++++++----------------- kernel/sched/core.c | 180 ++++++++++++++++++++----------------------------- kernel/sched/sched.h | 4 ++ lib/smp_processor_id.c | 3 + 7 files changed, 128 insertions(+), 181 deletions(-)
include/linux/cpu.h +0 -4
@@ -113,8 +113,6 @@ static inline void cpu_maps_update_done(void)extern void cpu_hotplug_enable (void );void clear_tasks_mm_cpumask (int cpu);int cpu_down (unsigned int cpu); -extern void pin_current_cpu (void ); -extern void unpin_current_cpu (void );#else /* CONFIG_HOTPLUG_CPU */
@@ -126,8 +124,6 @@ static inline void cpus_read_unlock(void) { }static inline void lockdep_assert_cpus_held (void ) { }static inline void cpu_hotplug_disable (void ) { }static inline void cpu_hotplug_enable (void ) { } -static inline void pin_current_cpu (void ) { } -static inline void unpin_current_cpu (void ) { }#endif /* !CONFIG_HOTPLUG_CPU */
include/linux/sched.h +6 -5
@@ -229,6 +229,8 @@extern long io_schedule_timeout (long timeout);extern void io_schedule (void ); +int cpu_nr_pinned (int cpu); +/** * struct prev_cputime - snapshot of system and user cputime * @utime: time spent in user mode
@@ -661,16 +663,13 @@ struct task_struct {cpumask_t cpus_mask;#if defined(CONFIG_SMP ) && defined(CONFIG_PREEMPT_RT_BASE )int migrate_disable; -int migrate_disable_update; -int pinned_on_cpu; +bool migrate_disable_scheduled;# ifdef CONFIG_SCHED_DEBUG -int migrate_disable_atomic; +int pinned_on_cpu;# endif -#elif !defined(CONFIG_SMP ) && defined(CONFIG_PREEMPT_RT_BASE )# ifdef CONFIG_SCHED_DEBUG int migrate_disable; -int migrate_disable_atomic;# endif #endif #ifdef CONFIG_PREEMPT_RT_FULL
@@ -2074,4 +2073,6 @@ static inline void rseq_syscall(struct pt_regs *regs)#endif +extern struct task_struct *takedown_cpu_task; +#endif
init/init_task.c +4 -0
@@ -81,6 +81,10 @@ struct task_struct init_task .cpus_ptr = &init_task.cpus_mask, .cpus_mask = CPU_MASK_ALL, .nr_cpus_allowed= NR_CPUS, +#if defined(CONFIG_SMP ) && defined(CONFIG_PREEMPT_RT_BASE ) && \ + defined(CONFIG_SCHED_DEBUG ) + .pinned_on_cpu = -1, +#endif .mm = NULL, .active_mm = &init_mm, .restart_block = {
kernel/cpu.c +38 -65
148 lines not on this page · raw message
kernel/sched/core.c +73 -107
286 lines not on this page · raw message
kernel/sched/sched.h +4 -0
@@ -952,6 +952,10 @@ struct rq {/* Must be inspected within a rcu lock section */ struct cpuidle_state *idle_state;#endif + +#if defined(CONFIG_PREEMPT_RT_BASE ) && defined(CONFIG_SMP ) +int nr_pinned; +#endif };#ifdef CONFIG_FAIR_GROUP_SCHED
lib/smp_processor_id.c +3 -0
@@ -23,6 +23,9 @@ unsigned int check_preemption_disabled(const char *what1, const char *what2)* Kernel threads bound to a single CPU can safely use * smp_processor_id(): */ +if (current->migrate_disable) + goto out; +if (current->nr_cpus_allowed ==1 ) goto out;
Signature · 1 line
1.8.3.1
Re: [PATCH RT v2 2/3] sched: Lazy migrate_disable processing
Quoted 221 lines · Scott Wood
Avoid overhead on the majority of migrate disable/enable sequences by only manipulating scheduler data (and grabbing the relevant locks) when the task actually schedules while migrate-disabled. A kernel build showed around a 10% reduction in system time (with
CONFIG_NR_CPUS=512).Instead of
cpuhp_pin_lock, CPU hotplug is handled by keeping a per-CPU count of the number of pinned tasks (including tasks which have not scheduled in the migrate-disabled section);takedown_cpu()will wait until that reaches zero (confirmed bytake_cpu_down()in stop machine context to deal with races) before migrating tasks off of the cpu.To simplify synchronization, updating
cpus_maskis no longer deferred untilmigrate_enable(). This lets us not have to worry aboutmigrate_enable()missing the update if it's on the fast path (didn't schedule during the migrate disabled section). It also makes the code a bit simpler and reduces deviation from mainline.While the main motivation for this is the performance benefit, lazy migrate disable also eliminates the restriction on calling
migrate_disable()while atomic but leaving the atomic region prior to callingmigrate_enable()-- though this won't help withlocal_bh_disable()(and thus rcutorture) unless something similar is done with the recently addedlocal_lock.Signed-off-by Scott Wood <[email protected]>---
The speedup is smaller than before, due to commit659252061477862f("lib/smp_processor_id: Don't usecpumask_equal()") achieving an overlapping speedup.--- include/linux/cpu.h | 4 -- include/linux/sched.h | 11 +-- init/init_task.c | 4 ++ kernel/cpu.c | 103 +++++++++++----------------- kernel/sched/core.c | 180 ++++++++++++++++++++----------------------------- kernel/sched/sched.h | 4 ++ lib/smp_processor_id.c | 3 + 7 files changed, 128 insertions(+), 181 deletions(-)include/linux/cpu.h +0
-4@@ -113,8 +113,6 @@ static inline void cpu_maps_update_done(void)extern void cpu_hotplug_enable (void );void clear_tasks_mm_cpumask (int cpu);int cpu_down (unsigned int cpu); -extern void pin_current_cpu (void ); -extern void unpin_current_cpu (void );#else /* CONFIG_HOTPLUG_CPU */ @@ -126,8 +124,6 @@ static inline void cpus_read_unlock(void) { }static inline void lockdep_assert_cpus_held (void ) { }static inline void cpu_hotplug_disable (void ) { }static inline void cpu_hotplug_enable (void ) { } -static inline void pin_current_cpu (void ) { } -static inline void unpin_current_cpu (void ) { }#endif /* !CONFIG_HOTPLUG_CPU */ include/linux/sched.h +6
-5@@ -229,6 +229,8 @@extern long io_schedule_timeout (long timeout);extern void io_schedule (void ); +int cpu_nr_pinned (int cpu); +/** * struct prev_cputime - snapshot of system and user cputime * @utime: time spent in user mode @@ -661,16 +663,13 @@ struct task_struct {cpumask_t cpus_mask;#if defined(CONFIG_SMP ) && defined(CONFIG_PREEMPT_RT_BASE )int migrate_disable; -int migrate_disable_update; -int pinned_on_cpu; +bool migrate_disable_scheduled;# ifdef CONFIG_SCHED_DEBUG -int migrate_disable_atomic; +int pinned_on_cpu;# endif -#elif !defined(CONFIG_SMP ) && defined(CONFIG_PREEMPT_RT_BASE )# ifdef CONFIG_SCHED_DEBUG int migrate_disable; -int migrate_disable_atomic;# endif #endif #ifdef CONFIG_PREEMPT_RT_FULL @@ -2074,4 +2073,6 @@ static inline void rseq_syscall(struct pt_regs *regs)#endif +extern struct task_struct *takedown_cpu_task; +#endif init/init_task.c +4
-0@@ -81,6 +81,10 @@ struct task_struct init_task .cpus_ptr = &init_task.cpus_mask, .cpus_mask = CPU_MASK_ALL, .nr_cpus_allowed= NR_CPUS, +#if defined(CONFIG_SMP ) && defined(CONFIG_PREEMPT_RT_BASE ) && \ + defined(CONFIG_SCHED_DEBUG ) + .pinned_on_cpu = -1, +#endif .mm = NULL, .active_mm = &init_mm, .restart_block = {kernel/cpu.c +11
-59@@ -76,11 +76,6 @@ static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state) = { .fail = CPUHP_INVALID, }; -#if defined(CONFIG_HOTPLUG_CPU ) && defined(CONFIG_PREEMPT_RT_FULL ) -static DEFINE_PER_CPU(struct rt_rw_lock, cpuhp_pin_lock) = \ -__RWLOCK_RT_INITIALIZER (cpuhp_pin_lock); -#endif -#if defined(CONFIG_LOCKDEP ) && defined(CONFIG_SMP )static struct lockdep_map cpuhp_state_up_map =STATIC_LOCKDEP_MAP_INIT ("cpuhp_state-up" , &cpuhp_state_up_map);@@ -287,57 +282,6 @@ void cpu_maps_update_done(void)#ifdef CONFIG_HOTPLUG_CPU -/** -* pin_current_cpu - Prevent the current cpu from being unplugged -*/ -void pin_current_cpu (void ) -{ -#ifdef CONFIG_PREEMPT_RT_FULL -struct rt_rw_lock *cpuhp_pin; -unsigned int cpu; -int ret; - -again: - cpuhp_pin =this_cpu_ptr (&cpuhp_pin_lock); - ret =__read_rt_trylock (cpuhp_pin); -if (ret) { - current->pinned_on_cpu =smp_processor_id (); -return ; - } - cpu =smp_processor_id (); -preempt_lazy_enable (); -preempt_enable (); - -sleeping_lock_inc (); -__read_rt_lock (cpuhp_pin); -sleeping_lock_dec (); - -preempt_disable (); -preempt_lazy_disable (); -if (cpu !=smp_processor_id ()) { -__read_rt_unlock (cpuhp_pin); - goto again; - } - current->pinned_on_cpu = cpu; -#endif -} - -/** -* unpin_current_cpu - Allow unplug of current cpu -*/ -void unpin_current_cpu (void ) -{ -#ifdef CONFIG_PREEMPT_RT_FULL -struct rt_rw_lock *cpuhp_pin =this_cpu_ptr (&cpuhp_pin_lock); - -if (WARN_ON (current->pinned_on_cpu !=smp_processor_id ())) - cpuhp_pin =per_cpu_ptr (&cpuhp_pin_lock, current->pinned_on_cpu); - - current->pinned_on_cpu =-1 ; -__read_rt_unlock (cpuhp_pin); -#endif -} -DEFINE_STATIC_PERCPU_RWSEM (cpu_hotplug_lock);void cpus_read_lock (void )@@ -895,6 +839,15 @@ static int take_cpu_down(void *_param)int err, cpu =smp_processor_id ();int ret; +#ifdef CONFIG_PREEMPT_RT_BASE +/* +* If any tasks disabled migration before we got here, +* go back and sleep again. +*/ +if (cpu_nr_pinned (cpu)) +return -EAGAIN ; +#endif +/* Ensure this CPU doesn't handle any more interrupts. */ err =__cpu_disable ();if (err <0 )@@ -924,11 +877,10 @@ static int take_cpu_down(void *_param)return 0 ; } +struct task_struct *takedown_cpu_task; +static int takedown_cpu (unsigned int cpu) { -#ifdef CONFIG_PREEMPT_RT_FULL -struct rt_rw_lock *cpuhp_pin =per_cpu_ptr (&cpuhp_pin_lock, cpu); -#endif struct cpuhp_cpu_state *st =per_cpu_ptr (&cpuhp_state, cpu);int err;
kernel/cpu.c static int takedown_cpu(unsigned int cpu) · line 898
Quoted 8 lines
kernel/cpu.c +3
-2@@ -941,17 +893,38 @@ static int takedown_cpu(unsigned int cpu)*/ irq_lock_sparse (); -#ifdef CONFIG_PREEMPT_RT_FULL -__write_rt_lock (cpuhp_pin); +#ifdef CONFIG_PREEMPT_RT_BASE +WARN_ON_ONCE (takedown_cpu_task); + takedown_cpu_task = current;
I don't know how likely it is for more than one task calling takedown_cpu() concurrently. But if that can happen, there is a possibility of missed wakeup.
The current code is fragile. How about something like
while (cmpxchg(&takedown_cpu_task, NULL, current) { WARN_ON_ONCE(1);
schedule();
}
Cheers,
Longman
Re: [PATCH RT v2 2/3] sched: Lazy migrate_disable processing
Quoted 230 lines · Waiman Long
Quoted 229 lines · Scott Wood
Avoid overhead on the majority of migrate disable/enable sequences by only manipulating scheduler data (and grabbing the relevant locks) when the task actually schedules while migrate-disabled. A kernel build showed around a 10% reduction in system time (with
CONFIG_NR_CPUS=512).Instead of
cpuhp_pin_lock, CPU hotplug is handled by keeping a per-CPU count of the number of pinned tasks (including tasks which have not scheduled in the migrate-disabled section);takedown_cpu()will wait until that reaches zero (confirmed bytake_cpu_down()in stop machine context to deal with races) before migrating tasks off of the cpu.To simplify synchronization, updating
cpus_maskis no longer deferred untilmigrate_enable(). This lets us not have to worry aboutmigrate_enable()missing the update if it's on the fast path (didn't schedule during the migrate disabled section). It also makes the code a bit simpler and reduces deviation from mainline.While the main motivation for this is the performance benefit, lazy migrate disable also eliminates the restriction on calling
migrate_disable()while atomic but leaving the atomic region prior to callingmigrate_enable()-- though this won't help withlocal_bh_disable()
(and thus rcutorture) unless something similar is done with the recently addedlocal_lock.Signed-off-by Scott Wood <[email protected]>---
The speedup is smaller than before, due to commit659252061477862f("lib/smp_processor_id: Don't usecpumask_equal()") achieving an overlapping speedup.--- include/linux/cpu.h | 4 -- include/linux/sched.h | 11 +-- init/init_task.c | 4 ++ kernel/cpu.c | 103 +++++++++++----------------- kernel/sched/core.c | 180 ++++++++++++++++++++-------------------- --------- kernel/sched/sched.h | 4 ++ lib/smp_processor_id.c | 3 + 7 files changed, 128 insertions(+), 181 deletions(-)include/linux/cpu.h +0
-4@@ -113,8 +113,6 @@ static inline void cpu_maps_update_done(void)extern void cpu_hotplug_enable (void );void clear_tasks_mm_cpumask (int cpu);int cpu_down (unsigned int cpu); -extern void pin_current_cpu (void ); -extern void unpin_current_cpu (void );#else /* CONFIG_HOTPLUG_CPU */ @@ -126,8 +124,6 @@ static inline void cpus_read_unlock(void) { }static inline void lockdep_assert_cpus_held (void ) { }static inline void cpu_hotplug_disable (void ) { }static inline void cpu_hotplug_enable (void ) { } -static inline void pin_current_cpu (void ) { } -static inline void unpin_current_cpu (void ) { }#endif /* !CONFIG_HOTPLUG_CPU */ include/linux/sched.h +6
-5@@ -229,6 +229,8 @@extern long io_schedule_timeout (long timeout);extern void io_schedule (void ); +int cpu_nr_pinned (int cpu); +/** * struct prev_cputime - snapshot of system and user cputime * @utime: time spent in user mode @@ -661,16 +663,13 @@ struct task_struct {cpumask_t cpus_mask;#if defined(CONFIG_SMP ) && defined(CONFIG_PREEMPT_RT_BASE )int migrate_disable; -int migrate_disable_update; -int pinned_on_cpu; +bool migrate_disable_scheduled;# ifdef CONFIG_SCHED_DEBUG -int migrate_disable_atomic; +int pinned_on_cpu;# endif -#elif !defined(CONFIG_SMP ) && defined(CONFIG_PREEMPT_RT_BASE )# ifdef CONFIG_SCHED_DEBUG int migrate_disable; -int migrate_disable_atomic;# endif #endif #ifdef CONFIG_PREEMPT_RT_FULL @@ -2074,4 +2073,6 @@ static inline void rseq_syscall(struct pt_regs *regs)#endif +extern struct task_struct *takedown_cpu_task; +#endif
init/init_task.c +4
-0@@ -81,6 +81,10 @@ struct task_struct init_task .cpus_ptr = &init_task.cpus_mask, .cpus_mask = CPU_MASK_ALL, .nr_cpus_allowed= NR_CPUS, +#if defined(CONFIG_SMP ) && defined(CONFIG_PREEMPT_RT_BASE ) && \ + defined(CONFIG_SCHED_DEBUG ) + .pinned_on_cpu = -1, +#endif .mm = NULL, .active_mm = &init_mm, .restart_block = {kernel/cpu.c +0
-5@@ -76,11 +76,6 @@ static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state) = { .fail = CPUHP_INVALID, }; -#if defined(CONFIG_HOTPLUG_CPU ) && defined(CONFIG_PREEMPT_RT_FULL ) -static DEFINE_PER_CPU(struct rt_rw_lock, cpuhp_pin_lock) = \ -__RWLOCK_RT_INITIALIZER (cpuhp_pin_lock); -#endif -#if defined(CONFIG_LOCKDEP ) && defined(CONFIG_SMP ) static struct lockdep_map cpuhp_state_up_map =kernel/cpu.c +0
-51STATIC_LOCKDEP_MAP_INIT ("cpuhp_state-up" , &cpuhp_state_up_map);@@ -287,57 +282,6 @@ void cpu_maps_update_done(void)#ifdef CONFIG_HOTPLUG_CPU -/** -* pin_current_cpu - Prevent the current cpu from being unplugged -*/ -void pin_current_cpu (void ) -{ -#ifdef CONFIG_PREEMPT_RT_FULL -struct rt_rw_lock *cpuhp_pin; -unsigned int cpu; -int ret; - -again: - cpuhp_pin =this_cpu_ptr (&cpuhp_pin_lock); - ret =__read_rt_trylock (cpuhp_pin); -if (ret) { - current->pinned_on_cpu =smp_processor_id (); -return ; - } - cpu =smp_processor_id (); -preempt_lazy_enable (); -preempt_enable (); - -sleeping_lock_inc (); -__read_rt_lock (cpuhp_pin); -sleeping_lock_dec (); - -preempt_disable (); -preempt_lazy_disable (); -if (cpu !=smp_processor_id ()) { -__read_rt_unlock (cpuhp_pin); - goto again; - } - current->pinned_on_cpu = cpu; -#endif -} - -/** -* unpin_current_cpu - Allow unplug of current cpu -*/ -void unpin_current_cpu (void ) -{ -#ifdef CONFIG_PREEMPT_RT_FULL -struct rt_rw_lock *cpuhp_pin =this_cpu_ptr (&cpuhp_pin_lock); - -if (WARN_ON (current->pinned_on_cpu !=smp_processor_id ())) - cpuhp_pin =per_cpu_ptr (&cpuhp_pin_lock, current- >pinned_on_cpu); - - current->pinned_on_cpu =-1 ; -__read_rt_unlock (cpuhp_pin); -#endif -} -DEFINE_STATIC_PERCPU_RWSEM (cpu_hotplug_lock);void
cpus_read_lock(void)kernel/cpu.c +11
-3@@ -895,6 +839,15 @@ static int take_cpu_down(void *_param)int err, cpu =smp_processor_id ();int ret; +#ifdef CONFIG_PREEMPT_RT_BASE +/* +* If any tasks disabled migration before we got here, +* go back and sleep again. +*/ +if (cpu_nr_pinned (cpu)) +return -EAGAIN ; +#endif +/* Ensure this CPU doesn't handle any more interrupts. */ err =__cpu_disable ();if (err <0 )@@ -924,11 +877,10 @@ static int take_cpu_down(void *_param)return 0 ; } +struct task_struct *takedown_cpu_task; +static int takedown_cpu (unsigned int cpu) { -#ifdef CONFIG_PREEMPT_RT_FULL -struct rt_rw_lock *cpuhp_pin =per_cpu_ptr (&cpuhp_pin_lock, cpu); -#endif struct cpuhp_cpu_state *st =per_cpu_ptr (&cpuhp_state, cpu);int err;@@ -941,17 +893,38 @@ static int takedown_cpu(unsigned int cpu)*/ irq_lock_sparse ();
Quoted 9 lines
kernel/cpu.c static int takedown_cpu(unsigned int cpu) · line 898
Quoted 5 lines
kernel/cpu.c +3
-2-#ifdef CONFIG_PREEMPT_RT_FULL -__write_rt_lock (cpuhp_pin); +#ifdef CONFIG_PREEMPT_RT_BASE +WARN_ON_ONCE (takedown_cpu_task); + takedown_cpu_task = current;I don't know how likely it is for more than one task calling
takedown_cpu()concurrently. But if that can happen, there is a possibility of missed wakeup.
cpus_write_lock() in _cpu_down() prevents this, as does cpu_add_remove_lock.
Quoted 6 lines
The current code is fragile. How about something like
while (cmpxchg(&takedown_cpu_task, NULL, current) { WARN_ON_ONCE(1); schedule(); }
It's a debug check to try to let us know if the code ever changes such that multiple cpus can be taken down in parallel, not an attempt to actually make that work.
That said, looking at this again, it seems there is a possibility for a different race -- if __cpu_disable() returns an error, takedown_cpu_task will be NULLed out without flushing tasks from the cpu. This could happen between reads of takedown_cpu_task in migrate_enable(), leading to wake_up_process(NULL). Even if we use READ_ONCE(), if we get delayed by an interrupt long enough, the task pointed to might go away (or, less seriously, receive a spurious wakeup). A fix could be to stick a raw lock around the if+wakeup and around takedown_cpu_task = NULL in the error path.
-Scott
PATCH v2 3/3 sched: migrate_enable: Use stop_one_cpu_nowait()
In reply to Scott Woodmigrate_enable() can be called with current->state != TASK_RUNNING. Avoid clobbering the existing state by using stop_one_cpu_nowait(). Since we're stopping the current cpu, we know that we won't get past __schedule() until migration_cpu_stop() has run (at least up to the point of migrating us to another cpu).
--- include/linux/stop_machine.h | 2 ++ kernel/sched/core.c | 22 +++++++++++++--------- kernel/stop_machine.c | 7 +++++-- 3 files changed, 20 insertions(+), 11 deletions(-)
include/linux/stop_machine.h +2 -0
@@ -26,6 +26,8 @@ struct cpu_stop_work {cpu_stop_fn_t fn;void *arg;struct cpu_stop_done *done; +/* Did not run due to disabled stopper; for nowait debug checks */ +bool disabled; };int stop_one_cpu (unsigned int cpu,cpu_stop_fn_t fn,void *arg);
kernel/sched/core.c +13 -9
@@ -1051,6 +1051,7 @@ static struct rq *move_queued_task(struct rq *rq, struct rq_flags *rf,struct migration_arg {struct task_struct *task;int dest_cpu; +bool done; };/*
@@ -1086,6 +1087,11 @@ static int migration_cpu_stop(void *data)struct task_struct *p = arg->task;struct rq *rq =this_rq ();struct rq_flags rf; +int dest_cpu = arg->dest_cpu; + +/* We don't look at arg after this point. */ +smp_mb (); + arg->done = true;/* * The original target CPU might have gone down and we might
@@ -1108,9 +1114,9 @@ static int migration_cpu_stop(void *data)*/ if (task_rq (p) == rq) {if (task_on_rq_queued (p)) - rq =__migrate_task (rq, &rf, p, arg->dest_cpu); + rq =__migrate_task (rq, &rf, p, dest_cpu);else - p->wake_cpu = arg->dest_cpu; + p->wake_cpu = dest_cpu; }rq_unlock (rq, &rf);raw_spin_unlock (&p->pi_lock);
@@ -7392,6 +7398,7 @@ void migrate_enable(void)WARN_ON (smp_processor_id () != cpu);if (!is_cpu_allowed (p, cpu)) {struct migration_arg arg = { p }; +struct cpu_stop_work work;struct rq_flags rf; rq =task_rq_lock (p, &rf);
@@ -7399,13 +7406,10 @@ void migrate_enable(void) arg.dest_cpu =select_fallback_rq (cpu, p);task_rq_unlock (rq, p, &rf); -preempt_lazy_enable (); -preempt_enable (); - -sleeping_lock_inc (); -stop_one_cpu (task_cpu (p), migration_cpu_stop, &arg); -sleeping_lock_dec (); -return ; +stop_one_cpu_nowait (task_cpu (p), migration_cpu_stop, + &arg, &work); +__schedule (true); +WARN_ON_ONCE (!arg.done && !work.disabled); } out:
kernel/stop_machine.c +5 -2
@@ -85,8 +85,11 @@ static bool cpu_stop_queue_work(unsigned int cpu, struct cpu_stop_work *work) enabled = stopper->enabled;if (enabled)__cpu_stop_queue_work (stopper, work, &wakeq); -else if (work->done) -cpu_stop_signal_done (work->done); +else { + work->disabled = true; +if (work->done) +cpu_stop_signal_done (work->done); + }raw_spin_unlock_irqrestore (&stopper->lock, flags);wake_up_q (&wakeq);
Signature · 1 line
1.8.3.1
Quoted 3 lines · Scott Wood
These are the unapplied patches from v1, minus the sched deadline patch, and with
stop_one_cpu_nowait()in place of clobberingcurrent->state.
Thank you. They will be part of the next release.
Sebastian