cpu: Speedup disable_nonboot_cpus()
PATCH 1/1 cpu: Speedup disable_nonboot_cpus()
The routine disable_nonboot_cpus() shuts down cpus sequentially using for_each_online_cpu(cpu) to call cpu_down() one cpu at a time. cpu_down() calls __stop_machine() which stops all the cpus while it disables one. Then it re-enables the remaining cpus, only to do it all over again for the next cpu. The result is that it takes 16 minutes on a 1024 cpu system to disable 1023 cpus.
This patch changes disable_nonboot_cpus() to pass a bitmask of cpus to cpu_down() and modifies cpu_down() to only call __stop_machine() once.
On a 1024 cpu system this reduces the time it takes to disable all but one cpu from 16 minutes down to 4 minutes.
To: Andrew Morton <[email protected]>
Cc: "H. Peter Anvin" <[email protected]>
Cc: Ingo Molnar <[email protected]>
Cc: Robin Holt <[email protected]>
Cc: Russ Anderson <[email protected]>
--- kernel/cpu.c | 104 ++++++++++++++++++++++++++++++++++------------------------- 1 file changed, 61 insertions(+), 43 deletions(-)
kernel/cpu.c +61 -43
@@ -241,13 +241,13 @@ static inline void check_for_tasks(int cstruct take_cpu_down_param {unsigned long mod; -void *hcpu; };/* Take this CPU down. */ static int __ref take_cpu_down (void *_param) {struct take_cpu_down_param *param = _param; +void *hcpu = (void *)(long )smp_processor_id ();int err;/* Ensure this CPU doesn't handle any more interrupts. */
@@ -255,21 +255,21 @@ static int __ref take_cpu_down(void *_paif (err <0 )return err; -cpu_notify (CPU_DYING | param->mod, param->hcpu); +cpu_notify (CPU_DYING | param->mod, hcpu);/* Park the stopper thread */ kthread_park (current);return 0 ; }/* Requires cpu_add_remove_lock to be held */ -static int __ref _cpu_down (unsigned int cpu,int tasks_frozen) +static int __ref _cpu_down (const cpumask_t *cpus_to_offline,int tasks_frozen) { -int err, nr_calls =0 ; +int err =0 , cpu =0 , nr_calls =0 ;void *hcpu = (void *)(long )cpu; +cpumask_var_t cpus_offlined;unsigned long mod = tasks_frozen ?CPU_TASKS_FROZEN :0 ;struct take_cpu_down_param tcd_param = { .mod = mod, - .hcpu = hcpu, };if (num_online_cpus () ==1 )
@@ -278,46 +278,67 @@ static int __ref _cpu_down(unsigned intif (!cpu_online (cpu))return -EINVAL ; +if (!alloc_cpumask_var (&cpus_offlined,GFP_KERNEL )) +return -ENOMEM ; +cpu_hotplug_begin (); +cpumask_copy (cpus_offlined, cpus_to_offline); - err =__cpu_notify (CPU_DOWN_PREPARE | mod, hcpu,-1 , &nr_calls); -if (err) { - nr_calls--; -__cpu_notify (CPU_DOWN_FAILED | mod, hcpu, nr_calls,NULL ); -printk ("%s: attempt to take down CPU %u failed\n" , +for_each_cpu_mask (cpu, *cpus_to_offline) { +if (!cpu_online (cpu)) +continue ; + hcpu = (void *)(long )cpu; + err =__cpu_notify (CPU_DOWN_PREPARE | mod, hcpu,-1 , &nr_calls); +if (err) { + nr_calls--; +__cpu_notify (CPU_DOWN_FAILED | mod, hcpu, nr_calls,NULL ); +pr_err ("%s: attempt to take down CPU %u failed\n" , __func__, cpu); - goto out_release; + goto out_release; + } +smpboot_park_threads (cpu); } -smpboot_park_threads (cpu); - err =__stop_machine (take_cpu_down, &tcd_param,cpumask_of (cpu)); + err =__stop_machine (take_cpu_down, &tcd_param, cpus_to_offline);if (err) {/* CPU didn't die: tell everyone. Can't complain. */ -smpboot_unpark_threads (cpu); -cpu_notify_nofail (CPU_DOWN_FAILED | mod, hcpu); +for_each_cpu_mask (cpu, *cpus_to_offline) { + hcpu = (void *)(long )cpu; +smpboot_unpark_threads (cpu); +cpu_notify_nofail (CPU_DOWN_FAILED | mod, hcpu); + } goto out_release; } -BUG_ON (cpu_online (cpu));/* * The migration_call() CPU_DYING callback will have removed all * runnable tasks from the cpu, there's only the idle task left now * that the migration thread is done doing the stop_machine thing. -* -* Wait for the stop thread to go away. */ -while (!idle_cpu (cpu)) -cpu_relax (); - -/* This actually kills the CPU. */ -__cpu_die (cpu); +for_each_cpu_mask (cpu, *cpus_offlined) { +BUG_ON (cpu_online (cpu)); -/* CPU is completely dead: tell everyone. Too late to complain. */ -cpu_notify_nofail (CPU_DEAD | mod, hcpu); - -check_for_tasks (cpu); +/* +* Wait for the stop thread to go away. +*/ +while (!idle_cpu (cpu)) +cpu_relax (); + +/* +* This actually kills the CPU. +*/ +__cpu_die (cpu); + +/* +* CPU is completely dead: tell everyone. Too late to complain. +*/ + hcpu = (void *)(long )cpu; +cpu_notify_nofail (CPU_DEAD | mod, hcpu); +check_for_tasks (cpu); + } out_release: +free_cpumask_var (cpus_offlined);cpu_hotplug_done ();if (!err)cpu_notify_nofail (CPU_POST_DEAD | mod, hcpu);
@@ -327,6 +348,7 @@ out_release:int __ref cpu_down (unsigned int cpu) {int err; +cpumask_var_t cpumask;cpu_maps_update_begin ();
@@ -335,7 +357,11 @@ int __ref cpu_down(unsigned int cpu) goto out; } - err =_cpu_down (cpu,0 ); +if (!alloc_cpumask_var (&cpumask,GFP_KERNEL )) +return -ENOMEM ; +cpumask_set_cpu (cpu, cpumask); + err =_cpu_down (cpumask,0 ); +free_cpumask_var (cpumask); out:cpu_maps_update_done ();
@@ -459,7 +485,7 @@ static cpumask_var_t frozen_cpus;int disable_nonboot_cpus (void ) { -int cpu, first_cpu, error =0 ; +int first_cpu, error =0 ;cpu_maps_update_begin (); first_cpu =cpumask_first (cpu_online_mask);
@@ -467,21 +493,13 @@ int disable_nonboot_cpus(void)* We take down all of the non-boot CPUs in one shot to avoid races * with the userspace trying to use the CPU hotplug at the same time */ -cpumask_clear (frozen_cpus); -printk ("Disabling non-boot CPUs ...\n" ); -for_each_online_cpu (cpu) { -if (cpu == first_cpu) -continue ; - error =_cpu_down (cpu,1 ); -if (!error) -cpumask_set_cpu (cpu, frozen_cpus); -else { -printk (KERN_ERR "Error taking CPU%d down: %d\n" , - cpu, error); -break ; - } - } +cpumask_copy (frozen_cpus, cpu_online_mask); +cpumask_clear_cpu (first_cpu, frozen_cpus);/* all but one cpu*/ + + error =_cpu_down (frozen_cpus,1 ); +if (error) +pr_err ("Error %d stopping cpus\n" , error);if (!error) {BUG_ON (num_online_cpus () >1 );
Signature · 2 lines
Russ Anderson, OS RAS/Partitioning Project Lead SGI - Silicon Graphics Inc [email protected]
Quoted 8 lines · On Fri, 3 May 2013 17:35:44 -0500 Russ Anderson <[email protected]> wrote:
The routine
disable_nonboot_cpus()shuts down cpus sequentially usingfor_each_online_cpu(cpu)to callcpu_down()one cpu at a time.cpu_down()calls__stop_machine()which stops all the cpus while it disables one. Then it re-enables the remaining cpus, only to do it all over again for the next cpu. The result is that it takes 16 minutes on a 1024 cpu system to disable 1023 cpus.
Quoted 6 lines
This patch changes
disable_nonboot_cpus()to pass a bitmask of cpus tocpu_down()and modifiescpu_down()to only call__stop_machine()once.On a 1024 cpu system this reduces the time it takes to disable all but one cpu from 16 minutes down to 4 minutes.
That's still a helluva long time. What's the kernel *doing* for all that time?
Quoted 9 lines
kernel/cpu.c +0
-0@@ -255,21 +255,21 @@ static int __ref take_cpu_down(void *_paif (err <0 )return err;
kernel/cpu.c static int __ref take_cpu_down(void *_pa · line 265
Quoted 10 lines
kernel/cpu.c +2
-2-cpu_notify (CPU_DYING | param->mod, param->hcpu); +cpu_notify (CPU_DYING | param->mod, hcpu);/* Park the stopper thread */ kthread_park (current);return 0 ; }/* Requires cpu_add_remove_lock to be held */ -static int __ref _cpu_down (unsigned int cpu,int tasks_frozen) +static int __ref _cpu_down (const cpumask_t *cpus_to_offline,int tasks_frozen)
_cpu_down() is now misnamed - it downs multiple CPUs.
Quoted 38 lines
kernel/cpu.c +15
-8{ -int err, nr_calls =0 ; +int err =0 , cpu =0 , nr_calls =0 ;void *hcpu = (void *)(long )cpu; +cpumask_var_t cpus_offlined;unsigned long mod = tasks_frozen ?CPU_TASKS_FROZEN :0 ;struct take_cpu_down_param tcd_param = { .mod = mod, - .hcpu = hcpu, };if (num_online_cpus () ==1 )@@ -278,46 +278,67 @@ static int __ref _cpu_down(unsigned intif (!cpu_online (cpu))return -EINVAL ; +if (!alloc_cpumask_var (&cpus_offlined,GFP_KERNEL )) +return -ENOMEM ; +cpu_hotplug_begin (); +cpumask_copy (cpus_offlined, cpus_to_offline); - err =__cpu_notify (CPU_DOWN_PREPARE | mod, hcpu,-1 , &nr_calls); -if (err) { - nr_calls--; -__cpu_notify (CPU_DOWN_FAILED | mod, hcpu, nr_calls,NULL ); -printk ("%s: attempt to take down CPU %u failed\n" , +for_each_cpu_mask (cpu, *cpus_to_offline) { +if (!cpu_online (cpu)) +continue ; + hcpu = (void *)(long )cpu; + err =__cpu_notify (CPU_DOWN_PREPARE | mod, hcpu,-1 , &nr_calls); +if (err) { + nr_calls--; +__cpu_notify (CPU_DOWN_FAILED | mod, hcpu, nr_calls,NULL ); +pr_err ("%s: attempt to take down CPU %u failed\n" , __func__, cpu); - goto out_release;
kernel/cpu.c static int __ref _cpu_down(unsigned int · line 304
Quoted 10 lines
kernel/cpu.c +4
-2+ goto out_release; + } +smpboot_park_threads (cpu); } -smpboot_park_threads (cpu); - err =__stop_machine (take_cpu_down, &tcd_param,cpumask_of (cpu)); + err =__stop_machine (take_cpu_down, &tcd_param, cpus_to_offline);if (err) {/* CPU didn't die: tell everyone. Can't complain. */
This comment is now inaccurate. "One or more of the CPUs didn't die"?.
I'm not sure what "Can't complain" means. Perhaps expand on this while you're there?
kernel/cpu.c static int __ref _cpu_down(unsigned int · line 308
Quoted 6 lines
kernel/cpu.c +4
-2-smpboot_unpark_threads (cpu); -cpu_notify_nofail (CPU_DOWN_FAILED | mod, hcpu); +for_each_cpu_mask (cpu, *cpus_to_offline) { + hcpu = (void *)(long )cpu; +smpboot_unpark_threads (cpu); +cpu_notify_nofail (CPU_DOWN_FAILED | mod, hcpu);
Is this accurate? What happens if we asked stop_machine() to down 100 CPUs but it failed at CPU #50? We now tell listeners that we failed to down all 100 CPUs? That's not true.
Quoted 36 lines
kernel/cpu.c +15
-12+ } goto out_release; } -BUG_ON (cpu_online (cpu));/* * The migration_call() CPU_DYING callback will have removed all * runnable tasks from the cpu, there's only the idle task left now * that the migration thread is done doing the stop_machine thing. -* -* Wait for the stop thread to go away. */ -while (!idle_cpu (cpu)) -cpu_relax (); - -/* This actually kills the CPU. */ -__cpu_die (cpu); +for_each_cpu_mask (cpu, *cpus_offlined) { +BUG_ON (cpu_online (cpu)); -/* CPU is completely dead: tell everyone. Too late to complain. */ -cpu_notify_nofail (CPU_DEAD | mod, hcpu); - -check_for_tasks (cpu); +/* +* Wait for the stop thread to go away. +*/ +while (!idle_cpu (cpu)) +cpu_relax (); + +/* +* This actually kills the CPU. +*/ +__cpu_die (cpu); + +/*
kernel/cpu.c static int __ref _cpu_down(unsigned int · line 342
Quoted 10 lines
kernel/cpu.c +7
-0+* CPU is completely dead: tell everyone. Too late to complain. +*/ + hcpu = (void *)(long )cpu; +cpu_notify_nofail (CPU_DEAD | mod, hcpu); +check_for_tasks (cpu); + } out_release: +free_cpumask_var (cpus_offlined);cpu_hotplug_done ();
Swap the above two lines and we reduced the locked region by an unmeasurable amount!
Quoted 5 lines
kernel/cpu.c +0
-0if (!err)cpu_notify_nofail (CPU_POST_DEAD | mod, hcpu);
Quoted 14 lines · Andrew Morton
Quoted 12 lines · On Fri, 3 May 2013 17:35:44 -0500 Russ Anderson <[email protected]> wrote:
The routine
disable_nonboot_cpus()shuts down cpus sequentially usingfor_each_online_cpu(cpu)to callcpu_down()one cpu at a time.cpu_down()calls__stop_machine()which stops all the cpus while it disables one. Then it re-enables the remaining cpus, only to do it all over again for the next cpu. The result is that it takes 16 minutes on a 1024 cpu system to disable 1023 cpus.This patch changes
disable_nonboot_cpus()to pass a bitmask of cpus tocpu_down()and modifiescpu_down()to only call__stop_machine()once.
Quoted 5 lines
Quoted 2 lines
On a 1024 cpu system this reduces the time it takes to disable all but one cpu from 16 minutes down to 4 minutes.
That's still a helluva long time. What's the kernel *doing* for all that time?
Since __raw_notifier_call_chain() isn't thread safe, it is still called one cpu at a time. See "__cpu_notify(CPU_DOWN_PREPARE" below. So we still go sequentially for each cpu through the call chain.
These are the two heaviest hitters on the call chain.
sched_init_smp ->
cpuset_cpu_inactive ->
cpuset_update_active_cpus ->
partition_sched_domains(1, NULL, NULL)
schedule_work(&cpuset_hotplug_work);
cpuset_hotplug_workfn
mutex_lock(&cpuset_mutex);
partition_sched_domains()
partition_sched_domains ->
unregister_sched_domain_sysctl() ->
unregister_sysctl_table
register_sched_domain_sysctl
for_each_possible_cpu()
register_sysctl_table
uncore_cpu_nb -> uncore_cpu_notifier
uncore_event_exit_cpu
for_each_online_cpu()The partition_sched_domains() code gets called for each cpu, each time destroying and building new domains. register_sched_domain_sysctl() calls for_each_possible_cpu() each time.
Likewise uncore_event_exit_cpu() calls for_each_online_cpu() for each cpu.
Neither of those scale well.
Quoted 16 lines
Quoted 16 lines
kernel/cpu.c +1
-1@@ -255,21 +255,21 @@ static int __ref take_cpu_down(void *_paif (err <0 )return err; -cpu_notify (CPU_DYING | param->mod, param->hcpu); +cpu_notify (CPU_DYING | param->mod, hcpu);/* Park the stopper thread */ kthread_park (current);return 0 ; }
Quoted 5 lines
kernel/cpu.c static int __ref take_cpu_down(void *_pa · line 265
Quoted 3 lines
kernel/cpu.c +1
-1/* Requires cpu_add_remove_lock to be held */ -static int __ref _cpu_down (unsigned int cpu,int tasks_frozen) +static int __ref _cpu_down (const cpumask_t *cpus_to_offline,int tasks_frozen)
_cpu_down()is now misnamed - it downs multiple CPUs.
OK, I'll change it.
Quoted 51 lines
Quoted 38 lines
kernel/cpu.c +15
-8{ -int err, nr_calls =0 ; +int err =0 , cpu =0 , nr_calls =0 ;void *hcpu = (void *)(long )cpu; +cpumask_var_t cpus_offlined;unsigned long mod = tasks_frozen ?CPU_TASKS_FROZEN :0 ;struct take_cpu_down_param tcd_param = { .mod = mod, - .hcpu = hcpu, };if (num_online_cpus () ==1 )@@ -278,46 +278,67 @@ static int __ref _cpu_down(unsigned intif (!cpu_online (cpu))return -EINVAL ; +if (!alloc_cpumask_var (&cpus_offlined,GFP_KERNEL )) +return -ENOMEM ; +cpu_hotplug_begin (); +cpumask_copy (cpus_offlined, cpus_to_offline); - err =__cpu_notify (CPU_DOWN_PREPARE | mod, hcpu,-1 , &nr_calls); -if (err) { - nr_calls--; -__cpu_notify (CPU_DOWN_FAILED | mod, hcpu, nr_calls,NULL ); -printk ("%s: attempt to take down CPU %u failed\n" , +for_each_cpu_mask (cpu, *cpus_to_offline) { +if (!cpu_online (cpu)) +continue ; + hcpu = (void *)(long )cpu; + err =__cpu_notify (CPU_DOWN_PREPARE | mod, hcpu,-1 , &nr_calls); +if (err) { + nr_calls--; +__cpu_notify (CPU_DOWN_FAILED | mod, hcpu, nr_calls,NULL ); +pr_err ("%s: attempt to take down CPU %u failed\n" , __func__, cpu); - goto out_release;kernel/cpu.c static int __ref _cpu_down(unsigned int · line 304
Quoted 10 lines
kernel/cpu.c +4
-2+ goto out_release; + } +smpboot_park_threads (cpu); } -smpboot_park_threads (cpu); - err =__stop_machine (take_cpu_down, &tcd_param,cpumask_of (cpu)); + err =__stop_machine (take_cpu_down, &tcd_param, cpus_to_offline);if (err) {/* CPU didn't die: tell everyone. Can't complain. */ This comment is now inaccurate. "One or more of the CPUs didn't die"?.
Quoted 2 lines
I'm not sure what "Can't complain" means. Perhaps expand on this while you're there?
OK.
Quoted 10 lines
kernel/cpu.c static int __ref _cpu_down(unsigned int · line 308
Quoted 6 lines
kernel/cpu.c +4
-2-smpboot_unpark_threads (cpu); -cpu_notify_nofail (CPU_DOWN_FAILED | mod, hcpu); +for_each_cpu_mask (cpu, *cpus_to_offline) { + hcpu = (void *)(long )cpu; +smpboot_unpark_threads (cpu); +cpu_notify_nofail (CPU_DOWN_FAILED | mod, hcpu);Is this accurate? What happens if we asked
stop_machine()to down 100 CPUs but it failed at CPU #50? We now tell listeners that we failed to down all 100 CPUs? That's not true.
I'll look closer at that.
Quoted 55 lines
Quoted 36 lines
kernel/cpu.c +15
-12+ } goto out_release; } -BUG_ON (cpu_online (cpu));/* * The migration_call() CPU_DYING callback will have removed all * runnable tasks from the cpu, there's only the idle task left now * that the migration thread is done doing the stop_machine thing. -* -* Wait for the stop thread to go away. */ -while (!idle_cpu (cpu)) -cpu_relax (); - -/* This actually kills the CPU. */ -__cpu_die (cpu); +for_each_cpu_mask (cpu, *cpus_offlined) { +BUG_ON (cpu_online (cpu)); -/* CPU is completely dead: tell everyone. Too late to complain. */ -cpu_notify_nofail (CPU_DEAD | mod, hcpu); - -check_for_tasks (cpu); +/* +* Wait for the stop thread to go away. +*/ +while (!idle_cpu (cpu)) +cpu_relax (); + +/* +* This actually kills the CPU. +*/ +__cpu_die (cpu); + +/* kernel/cpu.c static int __ref _cpu_down(unsigned int · line 342
Quoted 10 lines
kernel/cpu.c +7
-0+* CPU is completely dead: tell everyone. Too late to complain. +*/ + hcpu = (void *)(long )cpu; +cpu_notify_nofail (CPU_DEAD | mod, hcpu); +check_for_tasks (cpu); + } out_release: +free_cpumask_var (cpus_offlined);cpu_hotplug_done ();Swap the above two lines and we reduced the locked region by an unmeasurable amount!
Quoted 5 lines
kernel/cpu.c +0
-0if (!err)cpu_notify_nofail (CPU_POST_DEAD | mod, hcpu);
Signature · 2 lines
Russ Anderson, OS RAS/Partitioning Project Lead SGI - Silicon Graphics Inc [email protected]