_cpu_down: don't play with current->cpus_allowed
Subsystems (2)
PATCH 5/6 _cpu_down: don't play with current->cpus_allowed
_cpu_down() changes the current task's affinity and then recovers it at the end. The problems are well known: we can't restore old_allowed if it was bound to the now-dead-cpu, and we can race with the userspace which can change cpu-affinity during unplug.
_cpu_down() should not play with current->cpus_allowed at all. Instead, take_cpu_down() can migrate the caller of _cpu_down() after __cpu_disable() removes the dying cpu from cpu_online_mask.
---
include/linux/sched.h | 1 + kernel/sched.c | 2 +- kernel/cpu.c | 18 ++++++------------ 3 files changed, 8 insertions(+), 13 deletions(-)
include/linux/sched.h renamed from include/linux/sched.h~4_CPU_DOWN_AFFINITY +1 -0
@@ -1843,6 +1843,7 @@ extern void sched_clock_idle_sleep_eventextern void sched_clock_idle_wakeup_event (u64 delta_ns);#ifdef CONFIG_HOTPLUG_CPU +extern void move_task_off_dead_cpu (int dead_cpu,struct task_struct *p);extern void idle_task_exit (void );#else static inline void idle_task_exit (void ) {}
kernel/sched.c renamed from kernel/sched.c~4_CPU_DOWN_AFFINITY +1 -1
@@ -5503,7 +5503,7 @@ static int migration_thread(void *data)/* * Figure out where task on dead CPU should go, use force if necessary. */ -static void move_task_off_dead_cpu (int dead_cpu,struct task_struct *p) +void move_task_off_dead_cpu (int dead_cpu,struct task_struct *p) {struct rq *rq =cpu_rq (dead_cpu);int needs_cpu, dest_cpu;
kernel/cpu.c renamed from kernel/cpu.c~4_CPU_DOWN_AFFINITY +6 -12
@@ -163,6 +163,7 @@ static inline void check_for_tasks(int c }struct take_cpu_down_param { +struct task_struct *caller;unsigned long mod;void *hcpu; };
@@ -171,6 +172,7 @@ struct take_cpu_down_param {static int __ref take_cpu_down (void *_param) {struct take_cpu_down_param *param = _param; +unsigned int cpu = (unsigned long )param->hcpu;int err;/* Ensure this CPU doesn't handle any more interrupts. */
@@ -181,6 +183,8 @@ static int __ref take_cpu_down(void *_paraw_notifier_call_chain (&cpu_chain,CPU_DYING | param->mod, param->hcpu); +if (task_cpu (param->caller) == cpu) +move_task_off_dead_cpu (cpu, param->caller);/* Force idle task to run as soon as we yield: it should immediately notice cpu is offline and die quickly. */ sched_idle_next ();
@@ -191,10 +195,10 @@ static int __ref take_cpu_down(void *_pastatic int __ref _cpu_down (unsigned int cpu,int tasks_frozen) {int err, nr_calls =0 ; -cpumask_var_t old_allowed;void *hcpu = (void *)(long )cpu;unsigned long mod = tasks_frozen ?CPU_TASKS_FROZEN :0 ;struct take_cpu_down_param tcd_param = { + .caller = current, .mod = mod, .hcpu = hcpu, };
@@ -205,9 +209,6 @@ static int __ref _cpu_down(unsigned intif (!cpu_online (cpu))return -EINVAL ; -if (!alloc_cpumask_var (&old_allowed,GFP_KERNEL )) -return -ENOMEM ; -cpu_hotplug_begin ();set_cpu_active (cpu, false); err = __raw_notifier_call_chain(&cpu_chain, CPU_DOWN_PREPARE | mod,
@@ -224,10 +225,6 @@ static int __ref _cpu_down(unsigned int goto out_release; } -/* Ensure that we are not runnable on dying cpu */ -cpumask_copy (old_allowed, ¤t->cpus_allowed); -set_cpus_allowed_ptr (current, cpu_active_mask); - err =__stop_machine (take_cpu_down, &tcd_param,cpumask_of (cpu));if (err) {set_cpu_active (cpu, true);
@@ -236,7 +233,7 @@ static int __ref _cpu_down(unsigned int hcpu) == NOTIFY_BAD)BUG (); - goto out_allowed; + goto out_release; }BUG_ON (cpu_online (cpu));
@@ -254,8 +251,6 @@ static int __ref _cpu_down(unsigned intcheck_for_tasks (cpu); -out_allowed: -set_cpus_allowed_ptr (current, old_allowed); out_release:cpu_hotplug_done ();if (!err) {
@@ -263,7 +258,6 @@ out_release: hcpu) == NOTIFY_BAD)BUG (); } -free_cpumask_var (old_allowed);return err; }
Quoted 10 lines · Oleg Nesterov
_cpu_down()changes the current task's affinity and then recovers it at the end. The problems are well known: we can't restoreold_allowedif it was bound to the now-dead-cpu, and we can race with the userspace which can change cpu-affinity during unplug.
_cpu_down()should not play withcurrent->cpus_allowedat all. Instead,take_cpu_down()can migrate the caller of_cpu_down()after__cpu_disable()removes the dying cpu fromcpu_online_mask.Signed-off-by Oleg Nesterov <[email protected]>
Quoted 113 lines
---
include/linux/sched.h | 1 + kernel/sched.c | 2 +- kernel/cpu.c | 18 ++++++------------ 3 files changed, 8 insertions(+), 13 deletions(-)include/linux/sched.h renamed from include/linux/sched.h~4_CPU_DOWN_AFFINITY +1
-0@@ -1843,6 +1843,7 @@ extern void sched_clock_idle_sleep_eventextern void sched_clock_idle_wakeup_event (u64 delta_ns);#ifdef CONFIG_HOTPLUG_CPU +extern void move_task_off_dead_cpu (int dead_cpu,struct task_struct *p);extern void idle_task_exit (void );#else static inline void idle_task_exit (void ) {}kernel/sched.c renamed from kernel/sched.c~4_CPU_DOWN_AFFINITY +1
-1@@ -5503,7 +5503,7 @@ static int migration_thread(void *data)/* * Figure out where task on dead CPU should go, use force if necessary. */ -static void move_task_off_dead_cpu (int dead_cpu,struct task_struct *p) +void move_task_off_dead_cpu (int dead_cpu,struct task_struct *p) {struct rq *rq =cpu_rq (dead_cpu);int needs_cpu, dest_cpu;kernel/cpu.c renamed from kernel/cpu.c~4_CPU_DOWN_AFFINITY +6
-12@@ -163,6 +163,7 @@ static inline void check_for_tasks(int c }struct take_cpu_down_param { +struct task_struct *caller;unsigned long mod;void *hcpu; };@@ -171,6 +172,7 @@ struct take_cpu_down_param {static int __ref take_cpu_down (void *_param) {struct take_cpu_down_param *param = _param; +unsigned int cpu = (unsigned long )param->hcpu;int err;/* Ensure this CPU doesn't handle any more interrupts. */ @@ -181,6 +183,8 @@ static int __ref take_cpu_down(void *_paraw_notifier_call_chain (&cpu_chain,CPU_DYING | param->mod, param->hcpu); +if (task_cpu (param->caller) == cpu) +move_task_off_dead_cpu (cpu, param->caller);/* Force idle task to run as soon as we yield: it should immediately notice cpu is offline and die quickly. */ sched_idle_next ();@@ -191,10 +195,10 @@ static int __ref take_cpu_down(void *_pastatic int __ref _cpu_down (unsigned int cpu,int tasks_frozen) {int err, nr_calls =0 ; -cpumask_var_t old_allowed;void *hcpu = (void *)(long )cpu;unsigned long mod = tasks_frozen ?CPU_TASKS_FROZEN :0 ;struct take_cpu_down_param tcd_param = { + .caller = current, .mod = mod, .hcpu = hcpu, };@@ -205,9 +209,6 @@ static int __ref _cpu_down(unsigned intif (!cpu_online (cpu))return -EINVAL ; -if (!alloc_cpumask_var (&old_allowed,GFP_KERNEL )) -return -ENOMEM ; -cpu_hotplug_begin ();set_cpu_active (cpu, false); err = __raw_notifier_call_chain(&cpu_chain, CPU_DOWN_PREPARE | mod,@@ -224,10 +225,6 @@ static int __ref _cpu_down(unsigned int goto out_release; } -/* Ensure that we are not runnable on dying cpu */ -cpumask_copy (old_allowed, ¤t->cpus_allowed); -set_cpus_allowed_ptr (current, cpu_active_mask); - err =__stop_machine (take_cpu_down, &tcd_param,cpumask_of (cpu));if (err) {set_cpu_active (cpu, true);@@ -236,7 +233,7 @@ static int __ref _cpu_down(unsigned int hcpu) == NOTIFY_BAD)BUG (); - goto out_allowed; + goto out_release; }BUG_ON (cpu_online (cpu));@@ -254,8 +251,6 @@ static int __ref _cpu_down(unsigned intcheck_for_tasks (cpu); -out_allowed: -set_cpus_allowed_ptr (current, old_allowed); out_release:cpu_hotplug_done ();if (!err) {@@ -263,7 +258,6 @@ out_release: hcpu) == NOTIFY_BAD)BUG (); } -free_cpumask_var (old_allowed);return err; }
[tip:sched/core] sched: _cpu_down(): Don't play with current->cpus_allowed
In reply to Oleg NesterovCommit-ID: 6a1bdc1b577ebcb65f6603c57f8347309bc4ab13 Gitweb: http://git.kernel.org/tip/6a1bdc1b577ebcb65f6603c57f8347309bc4ab13 Author: Oleg Nesterov <[email protected]> AuthorDate: Mon, 15 Mar 2010 10:10:23 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Fri, 2 Apr 2010 20:12:03 +0200
sched: _cpu_down(): Don't play with current->cpus_allowed
_cpu_down() changes the current task's affinity and then recovers it at the end. The problems are well known: we can't restore old_allowed if it was bound to the now-dead-cpu, and we can race with the userspace which can change cpu-affinity during unplug.
_cpu_down() should not play with current->cpus_allowed at all. Instead, take_cpu_down() can migrate the caller of _cpu_down() after __cpu_disable() removes the dying cpu from cpu_online_mask.
LKML-Reference: <[email protected]>
--- include/linux/sched.h | 1 + kernel/cpu.c | 18 ++++++------------ kernel/sched.c | 2 +- 3 files changed, 8 insertions(+), 13 deletions(-)
include/linux/sched.h +1 -0
@@ -1843,6 +1843,7 @@ extern void sched_clock_idle_sleep_event(void);extern void sched_clock_idle_wakeup_event (u64 delta_ns);#ifdef CONFIG_HOTPLUG_CPU +extern void move_task_off_dead_cpu (int dead_cpu,struct task_struct *p);extern void idle_task_exit (void );#else static inline void idle_task_exit (void ) {}
kernel/cpu.c +6 -12
@@ -163,6 +163,7 @@ static inline void check_for_tasks(int cpu) }struct take_cpu_down_param { +struct task_struct *caller;unsigned long mod;void *hcpu; };
@@ -171,6 +172,7 @@ struct take_cpu_down_param {static int __ref take_cpu_down (void *_param) {struct take_cpu_down_param *param = _param; +unsigned int cpu = (unsigned long )param->hcpu;int err;/* Ensure this CPU doesn't handle any more interrupts. */
@@ -181,6 +183,8 @@ static int __ref take_cpu_down(void *_param)raw_notifier_call_chain (&cpu_chain,CPU_DYING | param->mod, param->hcpu); +if (task_cpu (param->caller) == cpu) +move_task_off_dead_cpu (cpu, param->caller);/* Force idle task to run as soon as we yield: it should immediately notice cpu is offline and die quickly. */ sched_idle_next ();
@@ -191,10 +195,10 @@ static int __ref take_cpu_down(void *_param)static int __ref _cpu_down (unsigned int cpu,int tasks_frozen) {int err, nr_calls =0 ; -cpumask_var_t old_allowed;void *hcpu = (void *)(long )cpu;unsigned long mod = tasks_frozen ?CPU_TASKS_FROZEN :0 ;struct take_cpu_down_param tcd_param = { + .caller = current, .mod = mod, .hcpu = hcpu, };
@@ -205,9 +209,6 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen)if (!cpu_online (cpu))return -EINVAL ; -if (!alloc_cpumask_var (&old_allowed,GFP_KERNEL )) -return -ENOMEM ; -cpu_hotplug_begin ();set_cpu_active (cpu, false); err = __raw_notifier_call_chain(&cpu_chain, CPU_DOWN_PREPARE | mod,
@@ -224,10 +225,6 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen) goto out_release; } -/* Ensure that we are not runnable on dying cpu */ -cpumask_copy (old_allowed, ¤t->cpus_allowed); -set_cpus_allowed_ptr (current, cpu_active_mask); - err =__stop_machine (take_cpu_down, &tcd_param,cpumask_of (cpu));if (err) {set_cpu_active (cpu, true);
@@ -236,7 +233,7 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen) hcpu) == NOTIFY_BAD)BUG (); - goto out_allowed; + goto out_release; }BUG_ON (cpu_online (cpu));
@@ -254,8 +251,6 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen)check_for_tasks (cpu); -out_allowed: -set_cpus_allowed_ptr (current, old_allowed); out_release:cpu_hotplug_done ();if (!err) {
@@ -263,7 +258,6 @@ out_release: hcpu) == NOTIFY_BAD)BUG (); } -free_cpumask_var (old_allowed);return err; }
kernel/sched.c +1 -1
@@ -5442,7 +5442,7 @@ static int migration_thread(void *data)/* * Figure out where task on dead CPU should go, use force if necessary. */ -static void move_task_off_dead_cpu (int dead_cpu,struct task_struct *p) +void move_task_off_dead_cpu (int dead_cpu,struct task_struct *p) {struct rq *rq =cpu_rq (dead_cpu);int needs_cpu,uninitialized_var (dest_cpu );