CPU hotplug rework - episode I

The stop machine threads are still killed when a cpu goes offline. The reason is that the thread is used to bring the cpu down, so it can't be parked along with the other per cpu threads.

Allow a per cpu thread to be excluded from automatic parking, so it can park itself once it's done

Add a create callback function as well.

Signed-off-by Thomas Gleixner <[email protected]>

---
 include/linux/smpboot.h |    5 +++++
 kernel/smpboot.c        |    5 +++--
 2 files changed, 8 insertions(+), 2 deletions(-)

Index: linux-2.6/include/linux/smpboot.h
===================================================================

include/linux/smpboot.h +5 -0

@@ -14,6 +14,8 @@ struct smpboot_thread_data;
  * @thread_should_run:	Check whether the thread should run or not. Called with
  *			preemption disabled.
  * @thread_fn:		The associated thread function
+ * @create:		Optional setup function, called when the thread gets
+ *			created (Not called from the thread context)
  * @setup:		Optional setup function, called when the thread gets
  *			operational the first time
  * @cleanup:		Optional cleanup function, called when the thread

@@ -22,6 +24,7 @@ struct smpboot_thread_data;
  *			parked (cpu offline)
  * @unpark:		Optional unpark function, called when the thread is
  *			unparked (cpu online)
+ * @selfparking:	Thread is not parked by the park function.
  * @thread_comm:	The base name of the thread
  */
 struct smp_hotplug_thread {

@@ -29,10 +32,12 @@ struct smp_hotplug_thread {
 	struct list_head		list;
 	int				(*thread_should_run)(unsigned int cpu);
 	void				(*thread_fn)(unsigned int cpu);
+	void				(*create)(unsigned int cpu);
 	void				(*setup)(unsigned int cpu);
 	void				(*cleanup)(unsigned int cpu, bool online);
 	void				(*park)(unsigned int cpu);
 	void				(*unpark)(unsigned int cpu);
+	bool				selfparking;
 	const char			*thread_comm;
 };
 

Index: linux-2.6/kernel/smpboot.c
===================================================================

kernel/smpboot.c +3 -2

@@ -183,9 +183,10 @@ __smpboot_create_thread(struct smp_hotpl
 		kfree(td);
 		return PTR_ERR(tsk);
 	}
-
 	get_task_struct(tsk);
 	*per_cpu_ptr(ht->store, cpu) = tsk;
+	if (ht->create)
+		ht->create(cpu);
 	return 0;
 }
 

@@ -225,7 +226,7 @@ static void smpboot_park_thread(struct s
 {
 	struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu);
 
-	if (tsk)
+	if (tsk && !ht->selfparking)
 		kthread_park(tsk);
 }
 

To allow the stopper thread being managed by the smpboot thread infrastructure separate out the task storage from the stopper data structure.

Signed-off-by Thomas Gleixner <[email protected]>

---
 kernel/stop_machine.c |   32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

Index: linux-2.6/kernel/stop_machine.c
===================================================================

kernel/stop_machine.c +16 -16

@@ -37,10 +37,10 @@ struct cpu_stopper {
 	spinlock_t		lock;
 	bool			enabled;	/* is this stopper enabled? */
 	struct list_head	works;		/* list of pending works */
-	struct task_struct	*thread;	/* stopper thread */
 };
 
 static DEFINE_PER_CPU(struct cpu_stopper, cpu_stopper);
+static DEFINE_PER_CPU(struct task_struct *, cpu_stopper_task);
 static bool stop_machine_initialized = false;
 
 static void cpu_stop_init_done(struct cpu_stop_done *done, unsigned int nr_todo)

@@ -62,16 +62,18 @@ static void cpu_stop_signal_done(struct 
 }
 
 /* queue @work to @stopper.  if offline, @work is completed immediately */
-static void cpu_stop_queue_work(struct cpu_stopper *stopper,
-				struct cpu_stop_work *work)
+static void cpu_stop_queue_work(unsigned int cpu, struct cpu_stop_work *work)
 {
+	struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
+	struct task_struct *p = per_cpu(cpu_stopper_task, cpu);
+
 	unsigned long flags;
 
 	spin_lock_irqsave(&stopper->lock, flags);
 
 	if (stopper->enabled) {
 		list_add_tail(&work->list, &stopper->works);
-		wake_up_process(stopper->thread);
+		wake_up_process(p);
 	} else
 		cpu_stop_signal_done(work->done, false);
 

@@ -108,7 +110,7 @@ int stop_one_cpu(unsigned int cpu, cpu_s
 	struct cpu_stop_work work = { .fn = fn, .arg = arg, .done = &done };
 
 	cpu_stop_init_done(&done, 1);
-	cpu_stop_queue_work(&per_cpu(cpu_stopper, cpu), &work);
+	cpu_stop_queue_work(cpu, &work);
 	wait_for_completion(&done.completion);
 	return done.executed ? done.ret : -ENOENT;
 }

@@ -130,7 +132,7 @@ void stop_one_cpu_nowait(unsigned int cp
 			struct cpu_stop_work *work_buf)
 {
 	*work_buf = (struct cpu_stop_work){ .fn = fn, .arg = arg, };
-	cpu_stop_queue_work(&per_cpu(cpu_stopper, cpu), work_buf);
+	cpu_stop_queue_work(cpu, work_buf);
 }
 
 /* static data for stop_cpus */

@@ -159,8 +161,7 @@ static void queue_stop_cpus_work(const s
 	 */
 	preempt_disable();
 	for_each_cpu(cpu, cpumask)
-		cpu_stop_queue_work(&per_cpu(cpu_stopper, cpu),
-				    &per_cpu(stop_cpus_work, cpu));
+		cpu_stop_queue_work(cpu, &per_cpu(stop_cpus_work, cpu));
 	preempt_enable();
 }
 

@@ -304,12 +305,11 @@ static int __cpuinit cpu_stop_cpu_callba
 {
 	unsigned int cpu = (unsigned long)hcpu;
 	struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
-	struct task_struct *p;
+	struct task_struct *p = per_cpu(cpu_stopper_task, cpu);
 
 	switch (action & ~CPU_TASKS_FROZEN) {
 	case CPU_UP_PREPARE:
-		BUG_ON(stopper->thread || stopper->enabled ||
-		       !list_empty(&stopper->works));
+		BUG_ON(p || stopper->enabled || !list_empty(&stopper->works));
 		p = kthread_create_on_node(cpu_stopper_thread,
 					   stopper,
 					   cpu_to_node(cpu),

@@ -319,12 +319,12 @@ static int __cpuinit cpu_stop_cpu_callba
 		get_task_struct(p);
 		kthread_bind(p, cpu);
 		sched_set_stop_task(cpu, p);
-		stopper->thread = p;
+		per_cpu(cpu_stopper_task, cpu) = p;
 		break;
 
 	case CPU_ONLINE:
 		/* strictly unnecessary, as first user will wake it */
-		wake_up_process(stopper->thread);
+		wake_up_process(p);
 		/* mark enabled */
 		spin_lock_irq(&stopper->lock);
 		stopper->enabled = true;

@@ -339,7 +339,7 @@ static int __cpuinit cpu_stop_cpu_callba
 
 		sched_set_stop_task(cpu, NULL);
 		/* kill the stopper */
-		kthread_stop(stopper->thread);
+		kthread_stop(p);
 		/* drain remaining works */
 		spin_lock_irq(&stopper->lock);
 		list_for_each_entry(work, &stopper->works, list)

@@ -347,8 +347,8 @@ static int __cpuinit cpu_stop_cpu_callba
 		stopper->enabled = false;
 		spin_unlock_irq(&stopper->lock);
 		/* release the stopper */
-		put_task_struct(stopper->thread);
-		stopper->thread = NULL;
+		put_task_struct(p);
+		per_cpu(cpu_stopper_task, cpu) = NULL;
 		break;
 	}
 #endif

Use the smpboot thread infrastructure. Mark the stopper thread selfparking and park it after it has finished the take_cpu_down() work.

Signed-off-by Thomas Gleixner <[email protected]>

---
 kernel/cpu.c          |    2 
 kernel/stop_machine.c |  134 ++++++++++++++++++--------------------------------
 2 files changed, 51 insertions(+), 85 deletions(-)

Index: linux-2.6/kernel/cpu.c
===================================================================

kernel/cpu.c +2 -0

@@ -254,6 +254,8 @@ static int __ref take_cpu_down(void *_pa
 		return err;
 
 	cpu_notify(CPU_DYING | param->mod, param->hcpu);
+	/* Park the stopper thread */
+	kthread_park(current);
 	return 0;
 }
 

Index: linux-2.6/kernel/stop_machine.c
===================================================================

kernel/stop_machine.c +25 -26

@@ -18,7 +18,7 @@
 #include <linux/stop_machine.h>
 #include <linux/interrupt.h>
 #include <linux/kallsyms.h>
-
+#include <linux/smpboot.h>
 #include <linux/atomic.h>
 
 /*

@@ -245,20 +245,25 @@ int try_stop_cpus(const struct cpumask *
 	return ret;
 }
 
-static int cpu_stopper_thread(void *data)
+static int cpu_stop_should_run(unsigned int cpu)
+{
+	struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
+	unsigned long flags;
+	int run;
+
+	spin_lock_irqsave(&stopper->lock, flags);
+	run = !list_empty(&stopper->works);
+	spin_unlock_irqrestore(&stopper->lock, flags);
+	return run;
+}
+
+static void cpu_stopper_thread(unsigned int cpu)
 {
-	struct cpu_stopper *stopper = data;
+	struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
 	struct cpu_stop_work *work;
 	int ret;
 
 repeat:
-	set_current_state(TASK_INTERRUPTIBLE);	/* mb paired w/ kthread_stop */
-
-	if (kthread_should_stop()) {
-		__set_current_state(TASK_RUNNING);
-		return 0;
-	}
-
 	work = NULL;
 	spin_lock_irq(&stopper->lock);
 	if (!list_empty(&stopper->works)) {

@@ -274,8 +279,6 @@ repeat:
 		struct cpu_stop_done *done = work->done;
 		char ksym_buf[KSYM_NAME_LEN] __maybe_unused;
 
-		__set_current_state(TASK_RUNNING);
-
 		/* cpu stop callbacks are not allowed to sleep */
 		preempt_disable();
 

@@ -291,87 +294,55 @@ repeat:
 					  ksym_buf), arg);
 
 		cpu_stop_signal_done(done, true);
-	} else
-		schedule();
-
-	goto repeat;
+		goto repeat;
+	}
 }
 
 extern void sched_set_stop_task(int cpu, struct task_struct *stop);
 
-/* manage stopper for a cpu, mostly lifted from sched migration thread mgmt */
-static int __cpuinit cpu_stop_cpu_callback(struct notifier_block *nfb,
-					   unsigned long action, void *hcpu)
+static void cpu_stop_create(unsigned int cpu)
+{
+	sched_set_stop_task(cpu, per_cpu(cpu_stopper_task, cpu));
+}
+
+static void cpu_stop_park(unsigned int cpu)
 {
-	unsigned int cpu = (unsigned long)hcpu;
 	struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
-	struct task_struct *p = per_cpu(cpu_stopper_task, cpu);
+	struct cpu_stop_work *work;
+	unsigned long flags;
 
-	switch (action & ~CPU_TASKS_FROZEN) {
-	case CPU_UP_PREPARE:
-		BUG_ON(p || stopper->enabled || !list_empty(&stopper->works));
-		p = kthread_create_on_node(cpu_stopper_thread,
-					   stopper,

kernel/stop_machine.c +23 -50

+	spin_lock_irqsave(&stopper->lock, flags);
+	list_for_each_entry(work, &stopper->works, list)
+		cpu_stop_signal_done(work->done, false);
+	stopper->enabled = false;
+	spin_unlock_irqrestore(&stopper->lock, flags);
+}
 
-	case CPU_ONLINE:
-		/* strictly unnecessary, as first user will wake it */
-		wake_up_process(p);
-		/* mark enabled */
-		spin_lock_irq(&stopper->lock);
-		stopper->enabled = true;
-		spin_unlock_irq(&stopper->lock);
-		break;
-
-#ifdef CONFIG_HOTPLUG_CPU
-	case CPU_UP_CANCELED:
-	case CPU_POST_DEAD:
-	{
-		struct cpu_stop_work *work;
-
-		sched_set_stop_task(cpu, NULL);
-		/* kill the stopper */
-		kthread_stop(p);
-		/* drain remaining works */
-		spin_lock_irq(&stopper->lock);
-		list_for_each_entry(work, &stopper->works, list)
-			cpu_stop_signal_done(work->done, false);
-		stopper->enabled = false;
-		spin_unlock_irq(&stopper->lock);
-		/* release the stopper */
-		put_task_struct(p);
-		per_cpu(cpu_stopper_task, cpu) = NULL;
-		break;
-	}
-#endif
-	}
+static void cpu_stop_unpark(unsigned int cpu)
+{
+	struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
 
-	return NOTIFY_OK;
+	spin_lock_irq(&stopper->lock);
+	stopper->enabled = true;
+	spin_unlock_irq(&stopper->lock);
 }
 
-/*
- * Give it a higher priority so that cpu stopper is available to other
- * cpu notifiers.  It currently shares the same priority as sched
- * migration_notifier.
- */
-static struct notifier_block __cpuinitdata cpu_stop_cpu_notifier = {
-	.notifier_call	= cpu_stop_cpu_callback,
-	.priority	= 10,
+static struct smp_hotplug_thread cpu_stop_threads = {
+	.store			= &cpu_stopper_task,
+	.thread_should_run	= cpu_stop_should_run,
+	.thread_fn		= cpu_stopper_thread,
+	.thread_comm		= "migration/%u",
+	.create			= cpu_stop_create,
+	.setup			= cpu_stop_unpark,
+	.park			= cpu_stop_park,
+	.unpark			= cpu_stop_unpark,
+	.selfparking		= true,
 };
 
 static int __init cpu_stop_init(void)
 {
-	void *bcpu = (void *)(long)smp_processor_id();
 	unsigned int cpu;
-	int err;
 
 	for_each_possible_cpu(cpu) {
 		struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);

@@ -380,15 +351,8 @@ static int __init cpu_stop_init(void)
 		INIT_LIST_HEAD(&stopper->works);
 	}
 
-	/* start one for the boot cpu */
-	err = cpu_stop_cpu_callback(&cpu_stop_cpu_notifier, CPU_UP_PREPARE,
-				    bcpu);
-	BUG_ON(err != NOTIFY_OK);
-	cpu_stop_cpu_callback(&cpu_stop_cpu_notifier, CPU_ONLINE, bcpu);
-	register_cpu_notifier(&cpu_stop_cpu_notifier);
-
+	BUG_ON(smpboot_register_percpu_thread(&cpu_stop_threads));
 	stop_machine_initialized = true;
-
 	return 0;
 }
 early_initcall(cpu_stop_init);

There are only a few callbacks which really care about FROZEN vs. !FROZEN. No need to have extra states for this.

Publish the frozen state in an extra variable which is updated under the hotplug lock and let the users interested deal with it w/o imposing that extra state checks on everyone.

Signed-off-by Thomas Gleixner <[email protected]>

---
 kernel/cpu.c |   66 ++++++++++++++++++++++++-----------------------------------
 1 file changed, 27 insertions(+), 39 deletions(-)

Index: linux-2.6/kernel/cpu.c
===================================================================

kernel/cpu.c +27 -39

@@ -25,6 +25,7 @@
 #ifdef CONFIG_SMP
 /* Serializes the updates to cpu_online_mask, cpu_present_mask */
 static DEFINE_MUTEX(cpu_add_remove_lock);
+static bool cpuhp_tasks_frozen;
 
 /*
  * The following two API's must be used when attempting

@@ -148,27 +149,30 @@ int __ref register_cpu_notifier(struct n
 	return ret;
 }
 
-static int __cpu_notify(unsigned long val, void *v, int nr_to_call,
+static int __cpu_notify(unsigned long val, unsigned int cpu, int nr_to_call,
 			int *nr_calls)
 {
+	unsigned long mod = cpuhp_tasks_frozen ? CPU_TASKS_FROZEN : 0;
+	void *hcpu = (void *)(long)cpu;
+
 	int ret;
 
-	ret = __raw_notifier_call_chain(&cpu_chain, val, v, nr_to_call,
+	ret = __raw_notifier_call_chain(&cpu_chain, val | mod, hcpu, nr_to_call,
 					nr_calls);
 
 	return notifier_to_errno(ret);
 }
 
-static int cpu_notify(unsigned long val, void *v)
+static int cpu_notify(unsigned long val, unsigned int cpu)
 {
-	return __cpu_notify(val, v, -1, NULL);
+	return __cpu_notify(val, cpu, -1, NULL);
 }
 
 #ifdef CONFIG_HOTPLUG_CPU
 
-static void cpu_notify_nofail(unsigned long val, void *v)
+static void cpu_notify_nofail(unsigned long val, unsigned int cpu)
 {
-	BUG_ON(cpu_notify(val, v));
+	BUG_ON(cpu_notify(val, cpu));
 }
 EXPORT_SYMBOL(register_cpu_notifier);
 

@@ -237,23 +241,17 @@ static inline void check_for_tasks(int c
 	write_unlock_irq(&tasklist_lock);
 }
 
-struct take_cpu_down_param {
-	unsigned long mod;
-	void *hcpu;
-};
-
 /* Take this CPU down. */
 static int __ref take_cpu_down(void *_param)
 {
-	struct take_cpu_down_param *param = _param;
-	int err;
+	int err, cpu = smp_processor_id();
 
 	/* Ensure this CPU doesn't handle any more interrupts. */
 	err = __cpu_disable();
 	if (err < 0)
 		return err;
 
-	cpu_notify(CPU_DYING | param->mod, param->hcpu);
+	cpu_notify(CPU_DYING, cpu);
 	/* Park the stopper thread */
 	kthread_park(current);
 	return 0;

@@ -263,12 +261,6 @@ static int __ref take_cpu_down(void *_pa
 static int __ref _cpu_down(unsigned int cpu, int tasks_frozen)
 {
 	int err, nr_calls = 0;
-	void *hcpu = (void *)(long)cpu;
-	unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0;
-	struct take_cpu_down_param tcd_param = {
-		.mod = mod,
-		.hcpu = hcpu,
-	};
 
 	if (num_online_cpus() == 1)
 		return -EBUSY;

@@ -278,21 +270,23 @@ static int __ref _cpu_down(unsigned int 
 
 	cpu_hotplug_begin();
 
-	err = __cpu_notify(CPU_DOWN_PREPARE | mod, hcpu, -1, &nr_calls);
+	cpuhp_tasks_frozen = tasks_frozen;
+
+	err = __cpu_notify(CPU_DOWN_PREPARE, cpu, -1, &nr_calls);
 	if (err) {
 		nr_calls--;
-		__cpu_notify(CPU_DOWN_FAILED | mod, hcpu, nr_calls, NULL);
+		__cpu_notify(CPU_DOWN_FAILED, cpu, nr_calls, NULL);
 		printk("%s: attempt to take down CPU %u failed\n",
 				__func__, cpu);
 		goto out_release;
 	}
 	smpboot_park_threads(cpu);
 
-	err = __stop_machine(take_cpu_down, &tcd_param, cpumask_of(cpu));
+	err = __stop_machine(take_cpu_down, NULL, cpumask_of(cpu));
 	if (err) {
 		/* CPU didn't die: tell everyone.  Can't complain. */
 		smpboot_unpark_threads(cpu);
-		cpu_notify_nofail(CPU_DOWN_FAILED | mod, hcpu);
+		cpu_notify_nofail(CPU_DOWN_FAILED, cpu);
 		goto out_release;
 	}
 	BUG_ON(cpu_online(cpu));

@@ -311,14 +305,14 @@ static int __ref _cpu_down(unsigned int 
 	__cpu_die(cpu);
 
 	/* CPU is completely dead: tell everyone.  Too late to complain. */
-	cpu_notify_nofail(CPU_DEAD | mod, hcpu);
+	cpu_notify_nofail(CPU_DEAD, cpu);
 
 	check_for_tasks(cpu);
 
 out_release:
 	cpu_hotplug_done();
 	if (!err)
-		cpu_notify_nofail(CPU_POST_DEAD | mod, hcpu);
+		cpu_notify_nofail(CPU_POST_DEAD, cpu);
 	return err;
 }
 

@@ -345,10 +339,8 @@ EXPORT_SYMBOL(cpu_down);
 /* Requires cpu_add_remove_lock to be held */
 static int __cpuinit _cpu_up(unsigned int cpu, int tasks_frozen)
 {
-	int ret, nr_calls = 0;
-	void *hcpu = (void *)(long)cpu;
-	unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0;
 	struct task_struct *idle;
+	int ret, nr_calls = 0;
 
 	cpu_hotplug_begin();
 

@@ -367,7 +359,9 @@ static int __cpuinit _cpu_up(unsigned in
 	if (ret)
 		goto out;
 
-	ret = __cpu_notify(CPU_UP_PREPARE | mod, hcpu, -1, &nr_calls);
+	cpuhp_tasks_frozen = tasks_frozen;
+
+	ret = __cpu_notify(CPU_UP_PREPARE, cpu, -1, &nr_calls);
 	if (ret) {
 		nr_calls--;
 		printk(KERN_WARNING "%s: attempt to bring up CPU %u failed\n",

@@ -385,11 +379,11 @@ static int __cpuinit _cpu_up(unsigned in
 	smpboot_unpark_threads(cpu);
 
 	/* Now call notifier in preparation. */
-	cpu_notify(CPU_ONLINE | mod, hcpu);
+	cpu_notify(CPU_ONLINE, cpu);
 
 out_notify:
 	if (ret != 0)
-		__cpu_notify(CPU_UP_CANCELED | mod, hcpu, nr_calls, NULL);
+		__cpu_notify(CPU_UP_CANCELED, cpu, nr_calls, NULL);
 out:
 	cpu_hotplug_done();
 

@@ -627,13 +621,7 @@ core_initcall(cpu_hotplug_pm_sync_init);
  */
 void __cpuinit notify_cpu_starting(unsigned int cpu)
 {
-	unsigned long val = CPU_STARTING;
-
-#ifdef CONFIG_PM_SLEEP_SMP
-	if (frozen_cpus != NULL && cpumask_test_cpu(cpu, frozen_cpus))
-		val = CPU_STARTING_FROZEN;
-#endif /* CONFIG_PM_SLEEP_SMP */
-	cpu_notify(val, (void *)(long)cpu);
+	cpu_notify(CPU_STARTING, cpu);
 }
 
 #endif /* CONFIG_SMP */

There are only a few callbacks which really care about FROZEN vs. !FROZEN. No need to have extra states for this.

Publish the frozen state in an extra variable which is updated under the hotplug lock and let the users interested deal with it w/o imposing that extra state checks on everyone.

Signed-off-by Thomas Gleixner <[email protected]>

---
 kernel/cpu.c |   66 ++++++++++++++++++++++++-----------------------------------
 1 file changed, 27 insertions(+), 39 deletions(-)

From [email protected] Thu Jan 31 13:13:23 2013

Split out into separate functions, so we can convert it to a state machine.

Signed-off-by Thomas Gleixner <[email protected]>

---
 kernel/cpu.c |   69 ++++++++++++++++++++++++++++++++++++++++-------------------
 1 file changed, 47 insertions(+), 22 deletions(-)

Index: linux-2.6/kernel/cpu.c
===================================================================

kernel/cpu.c +47 -22

@@ -168,6 +168,43 @@ static int cpu_notify(unsigned long val,
 	return __cpu_notify(val, cpu, -1, NULL);
 }
 
+/* Notifier wrappers for transitioning to state machine */
+static int notify_prepare(unsigned int cpu)
+{
+	int nr_calls = 0;
+	int ret;
+
+	ret = __cpu_notify(CPU_UP_PREPARE, cpu, -1, &nr_calls);
+	if (ret) {
+		nr_calls--;
+		printk(KERN_WARNING "%s: attempt to bring up CPU %u failed\n",
+				__func__, cpu);
+		__cpu_notify(CPU_UP_CANCELED, cpu, nr_calls, NULL);
+	}
+	return ret;
+}
+
+static int notify_online(unsigned int cpu)
+{
+	cpu_notify(CPU_ONLINE, cpu);
+	return 0;
+}
+
+static int bringup_cpu(unsigned int cpu)
+{
+	struct task_struct *idle = idle_thread_get(cpu);
+	int ret;
+
+	/* Arch-specific enabling code. */
+	ret = __cpu_up(cpu, idle);
+	if (ret) {
+		cpu_notify(CPU_UP_CANCELED, cpu);
+		return ret;
+	}
+	BUG_ON(!cpu_online(cpu));
+	return 0;
+}
+
 #ifdef CONFIG_HOTPLUG_CPU
 
 static void cpu_notify_nofail(unsigned long val, unsigned int cpu)

@@ -340,7 +377,7 @@ EXPORT_SYMBOL(cpu_down);
 static int __cpuinit _cpu_up(unsigned int cpu, int tasks_frozen)
 {
 	struct task_struct *idle;
-	int ret, nr_calls = 0;
+	int ret;
 
 	cpu_hotplug_begin();
 

@@ -355,35 +392,23 @@ static int __cpuinit _cpu_up(unsigned in
 		goto out;
 	}
 
+	cpuhp_tasks_frozen = tasks_frozen;
+
 	ret = smpboot_create_threads(cpu);
 	if (ret)
 		goto out;
 
-	cpuhp_tasks_frozen = tasks_frozen;
-
-	ret = __cpu_notify(CPU_UP_PREPARE, cpu, -1, &nr_calls);
-	if (ret) {
-		nr_calls--;
-		printk(KERN_WARNING "%s: attempt to bring up CPU %u failed\n",
-				__func__, cpu);
-		goto out_notify;
-	}
+	ret = notify_prepare(cpu);
+	if (ret)
+		goto out;
 
-	/* Arch-specific enabling code. */
-	ret = __cpu_up(cpu, idle);
-	if (ret != 0)
-		goto out_notify;
-	BUG_ON(!cpu_online(cpu));
+	ret = bringup_cpu(cpu);
+	if (ret)
+		goto out;
 
 	/* Wake the per cpu threads */
 	smpboot_unpark_threads(cpu);
-
-	/* Now call notifier in preparation. */
-	cpu_notify(CPU_ONLINE, cpu);
-
-out_notify:
-	if (ret != 0)
-		__cpu_notify(CPU_UP_CANCELED, cpu, nr_calls, NULL);
+	notify_online(cpu);
 out:
 	cpu_hotplug_done();
 

Split out into separate functions, so we can convert it to a state machine.

Signed-off-by Thomas Gleixner <[email protected]>

From [email protected] Thu Jan 31 13:11:32 2013

Split cpu_down in separate functions in preparation for state machine conversion.

Signed-off-by Thomas Gleixner <[email protected]>

---
 kernel/cpu.c |   83 +++++++++++++++++++++++++++++++++++++----------------------
 1 file changed, 53 insertions(+), 30 deletions(-)

Index: linux-2.6/kernel/cpu.c
===================================================================

kernel/cpu.c +53 -30

@@ -206,11 +206,6 @@ static int bringup_cpu(unsigned int cpu)
 }
 
 #ifdef CONFIG_HOTPLUG_CPU
-
-static void cpu_notify_nofail(unsigned long val, unsigned int cpu)
-{
-	BUG_ON(cpu_notify(val, cpu));
-}
 EXPORT_SYMBOL(register_cpu_notifier);
 
 void __ref unregister_cpu_notifier(struct notifier_block *nb)

@@ -278,6 +273,25 @@ static inline void check_for_tasks(int c
 	write_unlock_irq(&tasklist_lock);
 }
 
+static void cpu_notify_nofail(unsigned long val, unsigned int cpu)
+{
+	BUG_ON(cpu_notify(val, cpu));
+}
+
+static int notify_down_prepare(unsigned int cpu)
+{
+	int err, nr_calls = 0;
+
+	err = __cpu_notify(CPU_DOWN_PREPARE, cpu, -1, &nr_calls);
+	if (err) {
+		nr_calls--;
+		__cpu_notify(CPU_DOWN_FAILED, cpu, nr_calls, NULL);
+		printk("%s: attempt to take down CPU %u failed\n",
+				__func__, cpu);
+	}
+	return err;
+}
+
 /* Take this CPU down. */
 static int __ref take_cpu_down(void *_param)
 {

@@ -294,37 +308,17 @@ static int __ref take_cpu_down(void *_pa
 	return 0;
 }
 
-/* Requires cpu_add_remove_lock to be held */
-static int __ref _cpu_down(unsigned int cpu, int tasks_frozen)
+static int takedown_cpu(unsigned int cpu)
 {
-	int err, nr_calls = 0;
-
-	if (num_online_cpus() == 1)
-		return -EBUSY;
-
-	if (!cpu_online(cpu))
-		return -EINVAL;
-
-	cpu_hotplug_begin();
-
-	cpuhp_tasks_frozen = tasks_frozen;
+	int err;
 
-	err = __cpu_notify(CPU_DOWN_PREPARE, cpu, -1, &nr_calls);
-	if (err) {
-		nr_calls--;
-		__cpu_notify(CPU_DOWN_FAILED, cpu, nr_calls, NULL);
-		printk("%s: attempt to take down CPU %u failed\n",
-				__func__, cpu);
-		goto out_release;
-	}
 	smpboot_park_threads(cpu);
-
 	err = __stop_machine(take_cpu_down, NULL, cpumask_of(cpu));
 	if (err) {
 		/* CPU didn't die: tell everyone.  Can't complain. */
 		smpboot_unpark_threads(cpu);
 		cpu_notify_nofail(CPU_DOWN_FAILED, cpu);
-		goto out_release;
+		return err;
 	}
 	BUG_ON(cpu_online(cpu));
 

@@ -341,10 +335,39 @@ static int __ref _cpu_down(unsigned int 
 	/* This actually kills the CPU. */
 	__cpu_die(cpu);
 
-	/* CPU is completely dead: tell everyone.  Too late to complain. */
-	cpu_notify_nofail(CPU_DEAD, cpu);
+	return 0;
+}
 
+static int notify_dead(unsigned int cpu)
+{
+	cpu_notify_nofail(CPU_DEAD, cpu);
 	check_for_tasks(cpu);
+	return 0;
+}
+
+/* Requires cpu_add_remove_lock to be held */
+static int __ref _cpu_down(unsigned int cpu, int tasks_frozen)
+{
+	int err;
+
+	if (num_online_cpus() == 1)
+		return -EBUSY;
+
+	if (!cpu_online(cpu))
+		return -EINVAL;
+
+	cpu_hotplug_begin();
+
+	cpuhp_tasks_frozen = tasks_frozen;
+
+	err = notify_down_prepare(cpu);
+	if (err)
+		goto out_release;
+	err = takedown_cpu(cpu);
+	if (err)
+		goto out_release;
+
+	notify_dead(cpu);
 
 out_release:
 	cpu_hotplug_done();

Move the split out steps into a callback array and let the cpu_up/down code iterate through the array functions. For now most of the callbacks are asymetric to resemble the current hotplug maze.

Signed-off-by Thomas Gleixner <[email protected]>

---
 include/linux/cpu.h        |    4 +
 include/linux/cpuhotplug.h |   16 ++++
 init/main.c                |   15 ---
 kernel/cpu.c               |  180 ++++++++++++++++++++++++++++++++++++---------
 kernel/smpboot.c           |    6 +
 kernel/smpboot.h           |    4 -
 6 files changed, 173 insertions(+), 52 deletions(-)

Index: linux-2.6/include/linux/cpu.h
===================================================================

include/linux/cpu.h +4 -0

@@ -26,6 +26,9 @@ struct cpu {
 	struct device dev;
 };
 
+extern void boot_cpu_init(void);
+extern void boot_cpu_state_init(void);
+
 extern int register_cpu(struct cpu *cpu, int num);
 extern struct device *get_cpu_device(unsigned cpu);
 extern bool cpu_is_hotpluggable(unsigned cpu);

@@ -112,6 +115,7 @@ enum {
 
 
 #ifdef CONFIG_SMP
+extern bool cpuhp_tasks_frozen;
 /* Need to know about CPUs going up/down? */
 #if defined(CONFIG_HOTPLUG_CPU) || !defined(MODULE)
 #define cpu_notifier(fn, pri) {					\

Index: linux-2.6/include/linux/cpuhotplug.h
===================================================================

include/linux/cpuhotplug.h new file +16 -0

@@ -0,0 +1,16 @@
+#ifndef __CPUHOTPLUG_H
+#define __CPUHOTPLUG_H
+
+enum cpuhp_states {
+	CPUHP_OFFLINE,
+	CPUHP_CREATE_THREADS,
+	CPUHP_NOTIFY_PREPARE,
+	CPUHP_NOTIFY_DEAD,
+	CPUHP_BRINGUP_CPU,
+	CPUHP_TEARDOWN_CPU,
+	CPUHP_PERCPU_THREADS,
+	CPUHP_NOTIFY_ONLINE,
+	CPUHP_NOTIFY_DOWN_PREPARE,
+	CPUHP_MAX,
+};
+#endif

Index: linux-2.6/init/main.c
===================================================================

init/main.c +1 -14

@@ -424,20 +424,6 @@ void __init parse_early_param(void)
 	done = 1;
 }
 
-/*
- *	Activate the first processor.
- */
-
-static void __init boot_cpu_init(void)
-{
-	int cpu = smp_processor_id();
-	/* Mark the boot cpu "present", "online" etc for SMP and UP case */
-	set_cpu_online(cpu, true);
-	set_cpu_active(cpu, true);
-	set_cpu_present(cpu, true);
-	set_cpu_possible(cpu, true);
-}
-
 void __init __weak smp_setup_processor_id(void)
 {
 }

@@ -502,6 +488,7 @@ asmlinkage void __init start_kernel(void
 	setup_command_line(command_line);
 	setup_nr_cpu_ids();
 	setup_per_cpu_areas();
+	boot_cpu_state_init();
 	smp_prepare_boot_cpu();	/* arch-specific boot-cpu hooks */
 
 	build_all_zonelists(NULL, NULL);

Index: linux-2.6/kernel/cpu.c
===================================================================

kernel/cpu.c +146 -34

@@ -19,13 +19,24 @@
 #include <linux/mutex.h>
 #include <linux/gfp.h>
 #include <linux/suspend.h>
+#include <linux/cpuhotplug.h>
 
 #include "smpboot.h"
 
+/* CPU state */
+static DEFINE_PER_CPU(enum cpuhp_states, cpuhp_state);
+
+struct cpuhp_step {
+	int (*startup)(unsigned int cpu);
+	int (*teardown)(unsigned int cpu);
+};
+
+static struct cpuhp_step cpuhp_bp_states[];
+
 #ifdef CONFIG_SMP
 /* Serializes the updates to cpu_online_mask, cpu_present_mask */
 static DEFINE_MUTEX(cpu_add_remove_lock);
-static bool cpuhp_tasks_frozen;
+bool cpuhp_tasks_frozen;
 
 /*
  * The following two API's must be used when attempting

@@ -310,13 +321,10 @@ static int __ref take_cpu_down(void *_pa
 
 static int takedown_cpu(unsigned int cpu)
 {
-	int err;
+	int err = __stop_machine(take_cpu_down, NULL, cpumask_of(cpu));
 
-	smpboot_park_threads(cpu);
-	err = __stop_machine(take_cpu_down, NULL, cpumask_of(cpu));
 	if (err) {
 		/* CPU didn't die: tell everyone.  Can't complain. */
-		smpboot_unpark_threads(cpu);
 		cpu_notify_nofail(CPU_DOWN_FAILED, cpu);
 		return err;
 	}

@@ -345,10 +353,32 @@ static int notify_dead(unsigned int cpu)
 	return 0;
 }
 
+#else
+#define notify_down_prepare	NULL
+#define takedown_cpu		NULL
+#define notify_dead		NULL
+#endif
+
+#ifdef CONFIG_HOTPLUG_CPU
+static void undo_cpu_down(unsigned int cpu, int step)
+{
+	while (step++ < CPUHP_MAX) {
+		/*
+		 * Transitional check. Will be removed when we have a
+		 * fully symetric mechanism
+		 */
+		if (!cpuhp_bp_states[step].teardown)
+			continue;
+
+		if (cpuhp_bp_states[step].startup)
+			cpuhp_bp_states[step].startup(cpu);
+	}
+}
+
 /* Requires cpu_add_remove_lock to be held */
 static int __ref _cpu_down(unsigned int cpu, int tasks_frozen)
 {
-	int err;
+	int ret = 0, step;
 
 	if (num_online_cpus() == 1)
 		return -EBUSY;

@@ -360,20 +390,23 @@ static int __ref _cpu_down(unsigned int 
 
 	cpuhp_tasks_frozen = tasks_frozen;
 
-	err = notify_down_prepare(cpu);
-	if (err)
-		goto out_release;
-	err = takedown_cpu(cpu);
-	if (err)
-		goto out_release;
-
-	notify_dead(cpu);
+	for (step = per_cpu(cpuhp_state, cpu); step > 0; step--) {
+		if (cpuhp_bp_states[step].teardown) {
+			ret = cpuhp_bp_states[step].teardown(cpu);
+			if (ret) {
+				undo_cpu_down(cpu, step + 1);
+				step = CPUHP_MAX;
+				break;
+			}
+		}
+	}
+	/* Store the current cpu state */
+	per_cpu(cpuhp_state, cpu) = step;
 
-out_release:
 	cpu_hotplug_done();
-	if (!err)
+	if (!ret)
 		cpu_notify_nofail(CPU_POST_DEAD, cpu);
-	return err;
+	return ret;
 }
 
 int __ref cpu_down(unsigned int cpu)

@@ -396,11 +429,25 @@ out:
 EXPORT_SYMBOL(cpu_down);
 #endif /*CONFIG_HOTPLUG_CPU*/
 
+static void undo_cpu_up(unsigned int cpu, int step)
+{
+	while (step--) {
+		/*
+		 * Transitional check. Will be removed when we have a
+		 * fully symetric mechanism
+		 */
+		if (!cpuhp_bp_states[step].startup)
+			continue;
+		if (cpuhp_bp_states[step].teardown)
+			cpuhp_bp_states[step].teardown(cpu);
+	}
+}
+
 /* Requires cpu_add_remove_lock to be held */
 static int __cpuinit _cpu_up(unsigned int cpu, int tasks_frozen)
 {
+	int ret = 0, step;
 	struct task_struct *idle;
-	int ret;
 
 	cpu_hotplug_begin();
 

@@ -409,6 +456,7 @@ static int __cpuinit _cpu_up(unsigned in
 		goto out;
 	}
 
+	/* Let it fail before we try to bring the cpu up */
 	idle = idle_thread_get(cpu);
 	if (IS_ERR(idle)) {
 		ret = PTR_ERR(idle);

@@ -417,24 +465,20 @@ static int __cpuinit _cpu_up(unsigned in
 
 	cpuhp_tasks_frozen = tasks_frozen;
 
-	ret = smpboot_create_threads(cpu);
-	if (ret)
-		goto out;
-
-	ret = notify_prepare(cpu);
-	if (ret)
-		goto out;
-
-	ret = bringup_cpu(cpu);
-	if (ret)
-		goto out;
-
-	/* Wake the per cpu threads */
-	smpboot_unpark_threads(cpu);
-	notify_online(cpu);
+	for (step = per_cpu(cpuhp_state, cpu); step < CPUHP_MAX; step++) {
+		if (cpuhp_bp_states[step].startup) {
+			ret = cpuhp_bp_states[step].startup(cpu);
+			if (ret) {
+				undo_cpu_up(cpu, step - 1);
+				step = 0;
+				break;
+			}
+		}
+	}
+	/* Store the current cpu state */
+	per_cpu(cpuhp_state, cpu) = step;
 out:
 	cpu_hotplug_done();
-
 	return ret;
 }
 

@@ -674,6 +718,52 @@ void __cpuinit notify_cpu_starting(unsig
 
 #endif /* CONFIG_SMP */
 
+/* Boot processor state steps */
+static struct cpuhp_step cpuhp_bp_states[] = {
+	[CPUHP_OFFLINE] = {
+		.startup = NULL,
+		.teardown = NULL,
+	},
+#ifdef CONFIG_SMP
+	[CPUHP_CREATE_THREADS] = {
+		.startup = smpboot_create_threads,
+		.teardown = NULL,
+	},
+	[CPUHP_NOTIFY_PREPARE] = {
+		.startup = notify_prepare,
+		.teardown = NULL,
+	},
+	[CPUHP_NOTIFY_DEAD] = {
+		.startup = NULL,
+		.teardown = notify_dead,
+	},
+	[CPUHP_BRINGUP_CPU] = {
+		.startup = bringup_cpu,
+		.teardown = NULL,
+	},
+	[CPUHP_TEARDOWN_CPU] = {
+		.startup = NULL,
+		.teardown = takedown_cpu,
+	},
+	[CPUHP_PERCPU_THREADS] = {
+		.startup = smpboot_unpark_threads,
+		.teardown = smpboot_park_threads,
+	},
+	[CPUHP_NOTIFY_ONLINE] = {
+		.startup = notify_online,
+		.teardown = NULL,
+	},
+	[CPUHP_NOTIFY_DOWN_PREPARE] = {
+		.startup = NULL,
+		.teardown = notify_down_prepare,
+	},
+#endif
+	[CPUHP_MAX] = {
+		.startup = NULL,
+		.teardown = NULL,
+	},
+};
+
 /*
  * cpu_bit_bitmap[] is a special, "compressed" data structure that
  * represents all NR_CPUS bits binary values of 1<<nr.

@@ -769,3 +859,25 @@ void init_cpu_online(const struct cpumas
 {
 	cpumask_copy(to_cpumask(cpu_online_bits), src);
 }
+
+/*
+ * Activate the first processor.
+ */
+void __init boot_cpu_init(void)
+{
+	int cpu = smp_processor_id();
+
+	/* Mark the boot cpu "present", "online" etc for SMP and UP case */
+	set_cpu_online(cpu, true);
+	set_cpu_active(cpu, true);
+	set_cpu_present(cpu, true);
+	set_cpu_possible(cpu, true);
+}
+
+/*
+ * Must be called _AFTER_ setting up the per_cpu areas
+ */
+void __init boot_cpu_state_init(void)
+{
+	per_cpu(cpuhp_state, smp_processor_id()) = CPUHP_MAX;
+}

Index: linux-2.6/kernel/smpboot.c
===================================================================

kernel/smpboot.c +4 -2

@@ -212,7 +212,7 @@ static void smpboot_unpark_thread(struct
 	kthread_unpark(tsk);
 }
 
-void smpboot_unpark_threads(unsigned int cpu)
+int smpboot_unpark_threads(unsigned int cpu)
 {
 	struct smp_hotplug_thread *cur;
 

@@ -220,6 +220,7 @@ void smpboot_unpark_threads(unsigned int
 	list_for_each_entry(cur, &hotplug_threads, list)
 		smpboot_unpark_thread(cur, cpu);
 	mutex_unlock(&smpboot_threads_lock);
+	return 0;
 }
 
 static void smpboot_park_thread(struct smp_hotplug_thread *ht, unsigned int cpu)

@@ -230,7 +231,7 @@ static void smpboot_park_thread(struct s
 		kthread_park(tsk);
 }
 
-void smpboot_park_threads(unsigned int cpu)
+int smpboot_park_threads(unsigned int cpu)
 {
 	struct smp_hotplug_thread *cur;
 

@@ -238,6 +239,7 @@ void smpboot_park_threads(unsigned int c
 	list_for_each_entry_reverse(cur, &hotplug_threads, list)
 		smpboot_park_thread(cur, cpu);
 	mutex_unlock(&smpboot_threads_lock);
+	return 0;
 }
 
 static void smpboot_destroy_threads(struct smp_hotplug_thread *ht)

Index: linux-2.6/kernel/smpboot.h
===================================================================

kernel/smpboot.h +2 -2

@@ -14,7 +14,7 @@ static inline void idle_threads_init(voi
 #endif
 
 int smpboot_create_threads(unsigned int cpu);
-void smpboot_park_threads(unsigned int cpu);
-void smpboot_unpark_threads(unsigned int cpu);
+int smpboot_park_threads(unsigned int cpu);
+int smpboot_unpark_threads(unsigned int cpu);
 
 #endif

Move the functions which need to run on the hotplugged processor into a state machine array and let the code iterate through these functions.

In a later state, this will grow synchronization points between the control processor and the hotplugged processor, so we can move the various architecture implementations of the synchronizations to the core.

Signed-off-by Thomas Gleixner <[email protected]>

---
 include/linux/cpuhotplug.h |    4 ++
 kernel/cpu.c               |   70 +++++++++++++++++++++++++++++++++++----------
 2 files changed, 59 insertions(+), 15 deletions(-)

Index: linux-2.6/include/linux/cpuhotplug.h
===================================================================

include/linux/cpuhotplug.h +4 -0

@@ -7,6 +7,10 @@ enum cpuhp_states {
 	CPUHP_NOTIFY_PREPARE,
 	CPUHP_NOTIFY_DEAD,
 	CPUHP_BRINGUP_CPU,
+	CPUHP_AP_OFFLINE,
+	CPUHP_AP_NOTIFY_STARTING,
+	CPUHP_AP_NOTIFY_DYING,
+	CPUHP_AP_MAX,
 	CPUHP_TEARDOWN_CPU,
 	CPUHP_PERCPU_THREADS,
 	CPUHP_NOTIFY_ONLINE,

Index: linux-2.6/kernel/cpu.c
===================================================================

kernel/cpu.c +55 -15

@@ -32,6 +32,7 @@ struct cpuhp_step {
 };
 
 static struct cpuhp_step cpuhp_bp_states[];
+static struct cpuhp_step cpuhp_ap_states[];
 
 #ifdef CONFIG_SMP
 /* Serializes the updates to cpu_online_mask, cpu_present_mask */

@@ -216,6 +217,12 @@ static int bringup_cpu(unsigned int cpu)
 	return 0;
 }
 
+static int notify_starting(unsigned int cpu)
+{
+	cpu_notify(CPU_STARTING, cpu);
+	return 0;
+}
+
 #ifdef CONFIG_HOTPLUG_CPU
 EXPORT_SYMBOL(register_cpu_notifier);
 

@@ -303,17 +310,26 @@ static int notify_down_prepare(unsigned 
 	return err;
 }
 
+static int notify_dying(unsigned int cpu)
+{
+	cpu_notify(CPU_DYING, cpu);
+	return 0;
+}
+
 /* Take this CPU down. */
 static int __ref take_cpu_down(void *_param)
 {
-	int err, cpu = smp_processor_id();
+	int step, err, cpu = smp_processor_id();
 
 	/* Ensure this CPU doesn't handle any more interrupts. */
 	err = __cpu_disable();
 	if (err < 0)
 		return err;
 
-	cpu_notify(CPU_DYING, cpu);
+	for (step = CPUHP_AP_MAX; step >= CPUHP_AP_OFFLINE; step--) {
+		if (cpuhp_ap_states[step].teardown)
+			cpuhp_ap_states[step].teardown(cpu);
+	}
 	/* Park the stopper thread */
 	kthread_park(current);
 	return 0;

@@ -357,6 +373,7 @@ static int notify_dead(unsigned int cpu)
 #define notify_down_prepare	NULL
 #define takedown_cpu		NULL
 #define notify_dead		NULL
+#define notify_dying		NULL
 #endif
 
 #ifdef CONFIG_HOTPLUG_CPU

@@ -429,6 +446,24 @@ out:
 EXPORT_SYMBOL(cpu_down);
 #endif /*CONFIG_HOTPLUG_CPU*/
 
+/**
+ * notify_cpu_starting(cpu) - call the CPU_STARTING notifiers
+ * @cpu: cpu that just started
+ *
+ * This function calls the cpu_chain notifiers with CPU_STARTING.
+ * It must be called by the arch code on the new cpu, before the new cpu
+ * enables interrupts and before the "boot" cpu returns from __cpu_up().
+ */
+void notify_cpu_starting(unsigned int cpu)
+{
+	int step;
+
+	for (step = CPUHP_AP_OFFLINE; step <  CPUHP_AP_MAX; step++) {
+		if (cpuhp_ap_states[step].startup)
+			cpuhp_ap_states[step].startup(cpu);
+	}
+}
+
 static void undo_cpu_up(unsigned int cpu, int step)
 {
 	while (step--) {

@@ -703,19 +738,6 @@ core_initcall(cpu_hotplug_pm_sync_init);
 
 #endif /* CONFIG_PM_SLEEP_SMP */
 
-/**
- * notify_cpu_starting(cpu) - call the CPU_STARTING notifiers
- * @cpu: cpu that just started
- *
- * This function calls the cpu_chain notifiers with CPU_STARTING.
- * It must be called by the arch code on the new cpu, before the new cpu
- * enables interrupts and before the "boot" cpu returns from __cpu_up().
- */
-void __cpuinit notify_cpu_starting(unsigned int cpu)
-{
-	cpu_notify(CPU_STARTING, cpu);
-}
-
 #endif /* CONFIG_SMP */
 
 /* Boot processor state steps */

@@ -764,6 +786,24 @@ static struct cpuhp_step cpuhp_bp_states
 	},
 };
 
+/* Application processor state steps */
+static struct cpuhp_step cpuhp_ap_states[] = {
+#ifdef CONFIG_SMP
+	[CPUHP_AP_NOTIFY_STARTING] = {
+		.startup = notify_starting,
+		.teardown = NULL,
+	},
+	[CPUHP_AP_NOTIFY_DYING] = {
+		.startup = NULL,
+		.teardown = notify_dying,
+	},
+#endif
+	[CPUHP_MAX] = {
+		.startup = NULL,
+		.teardown = NULL,
+	},
+};
+
 /*
  * cpu_bit_bitmap[] is a special, "compressed" data structure that
  * represents all NR_CPUS bits binary values of 1<<nr.

kernel/cpu.c +20 -0

+/**
+ * __cpuhp_setup_state - Setup the callbacks for an hotplug machine state
+ * @state:     The state to setup
+ * @invoke:    If true, the startup function is invoked for cpus where
+ *             cpu state >= @state
+ * @startup:   startup callback function
+ * @teardown:  teardown callback function
+ *
+ * Returns 0 if successful, otherwise a proper error code
+ */
+int __cpuhp_setup_state(enum cpuhp_states state, bool invoke,
+                       int (*startup)(unsigned int cpu),
+                       int (*teardown)(unsigned int cpu))
+{
+       int cpu, ret = 0;
+
+       if (cpuhp_cb_check(state))
+               return -EINVAL;
+
+       get_online_cpus();

kernel/cpu.c +17 -0

+
+               if (cpustate < state)
+                       continue;
+
+               ret = cpuhp_issue_call(cpu, state, startup, true);
+               if (ret) {
+                       cpuhp_rollback_install(cpu, state, teardown);
+                       goto out;
+               }
+       }
+install:
+       cpuhp_store_callbacks(state, startup, teardown);
+out:
+       put_online_cpus();
+       return ret;
+}
+EXPORT_SYMBOL(__cpuhp_setup_state);

kernel/cpu.c static struct cpuhp_step cpuhp_ap_states · line 933

Quoted 3 lines · Thomas Gleixner

kernel/cpu.c +3 -0

+/**
+       for_each_present_cpu(cpu) {
+               int ret, cpustate = per_cpu(cpuhp_state, cpu);

s/ret,//

The scheduler sports quite a bunch of hotplug notifiers. One reason for multiple notifiers is the fact, that the startup and teardown process are asymetric. Now the scheduler wants to be called early on startup and late on teardown. That requires to install two different notifiers for the same issue.

With the state machine implementation we can register a callback pair for startup and teardown at the appropriate spot.

This patch converts the notifiers which are setup with special priorities and combines CPU_PRI_SCHED and CPU_PRI_CPUSET notifiers to a single callback. They run back to back anyway and we can make sure in the callbacks that the ordering inside the scheduler is correct. These notifiers are installed in sched_init_smp() as we can't run them during the bringup of the non boot cpus because the smp scheduler is setup after that. It would be nice if we just could compile them in, but that needs a larger surgery to the scheduler code and is beyond the scope of this patch.

Signed-off-by Thomas Gleixner <[email protected]>

---
 include/linux/cpu.h        |   16 ----
 include/linux/cpuhotplug.h |    6 +
 kernel/cpu.c               |    4 +
 kernel/sched/core.c        |  154 +++++++++++++++++----------------------------
 4 files changed, 69 insertions(+), 111 deletions(-)

Index: linux-2.6/include/linux/cpu.h
===================================================================

include/linux/cpu.h +0 -16

@@ -58,22 +58,6 @@ extern ssize_t arch_print_cpu_modalias(s
  * CPU notifier priorities.
  */
 enum {
-	/*
-	 * SCHED_ACTIVE marks a cpu which is coming up active during
-	 * CPU_ONLINE and CPU_DOWN_FAILED and must be the first
-	 * notifier.  CPUSET_ACTIVE adjusts cpuset according to
-	 * cpu_active mask right after SCHED_ACTIVE.  During
-	 * CPU_DOWN_PREPARE, SCHED_INACTIVE and CPUSET_INACTIVE are
-	 * ordered in the similar way.
-	 *
-	 * This ordering guarantees consistent cpu_active mask and
-	 * migration behavior to all cpu notifiers.
-	 */
-	CPU_PRI_SCHED_ACTIVE	= INT_MAX,
-	CPU_PRI_CPUSET_ACTIVE	= INT_MAX - 1,
-	CPU_PRI_SCHED_INACTIVE	= INT_MIN + 1,
-	CPU_PRI_CPUSET_INACTIVE	= INT_MIN,
-
 	/* migration should happen before other stuff but after perf */
 	CPU_PRI_PERF		= 20,
 	CPU_PRI_MIGRATION	= 10,

Index: linux-2.6/include/linux/cpuhotplug.h
===================================================================

include/linux/cpuhotplug.h +6 -0

@@ -6,13 +6,16 @@ enum cpuhp_states {
 	CPUHP_CREATE_THREADS,
 	CPUHP_NOTIFY_PREPARE,
 	CPUHP_NOTIFY_DEAD,
+	CPUHP_SCHED_DEAD,
 	CPUHP_BRINGUP_CPU,
 	CPUHP_AP_OFFLINE,
+	CPUHP_AP_SCHED_STARTING,
 	CPUHP_AP_NOTIFY_STARTING,
 	CPUHP_AP_NOTIFY_DYING,
 	CPUHP_AP_MAX,
 	CPUHP_TEARDOWN_CPU,
 	CPUHP_PERCPU_THREADS,
+	CPUHP_SCHED_ONLINE,
 	CPUHP_NOTIFY_ONLINE,
 	CPUHP_NOTIFY_DOWN_PREPARE,
 	CPUHP_MAX,

@@ -87,4 +90,7 @@ static inline void cpuhp_remove_state_no
 	__cpuhp_remove_state(state, false);
 }
 
+/* Compiled in scheduler hotplug functions */
+int sched_starting_cpu(unsigned int cpu);
+
 #endif

Index: linux-2.6/kernel/cpu.c
===================================================================

kernel/cpu.c +4 -0

@@ -788,6 +788,10 @@ static struct cpuhp_step cpuhp_bp_states
 /* Application processor state steps */
 static struct cpuhp_step cpuhp_ap_states[] = {
 #ifdef CONFIG_SMP
+	[CPUHP_AP_SCHED_STARTING] = {
+		.startup = sched_starting_cpu,
+		.teardown = NULL,
+	},
 	[CPUHP_AP_NOTIFY_STARTING] = {
 		.startup = notify_starting,
 		.teardown = NULL,

Index: linux-2.6/kernel/sched/core.c
===================================================================

kernel/sched/core.c +59 -95

@@ -5167,31 +5167,6 @@ static struct notifier_block __cpuinitda
 	.priority = CPU_PRI_MIGRATION,
 };
 
-static int __cpuinit sched_cpu_active(struct notifier_block *nfb,
-				      unsigned long action, void *hcpu)
-{
-	switch (action & ~CPU_TASKS_FROZEN) {
-	case CPU_STARTING:
-	case CPU_DOWN_FAILED:
-		set_cpu_active((long)hcpu, true);
-		return NOTIFY_OK;
-	default:
-		return NOTIFY_DONE;
-	}
-}
-
-static int __cpuinit sched_cpu_inactive(struct notifier_block *nfb,
-					unsigned long action, void *hcpu)
-{
-	switch (action & ~CPU_TASKS_FROZEN) {
-	case CPU_DOWN_PREPARE:
-		set_cpu_active((long)hcpu, false);
-		return NOTIFY_OK;
-	default:
-		return NOTIFY_DONE;
-	}
-}
-
 static int __init migration_init(void)
 {
 	void *cpu = (void *)(long)smp_processor_id();

@@ -5203,10 +5178,6 @@ static int __init migration_init(void)
 	migration_call(&migration_notifier, CPU_ONLINE, cpu);
 	register_cpu_notifier(&migration_notifier);
 
-	/* Register cpu active notifiers */
-	cpu_notifier(sched_cpu_active, CPU_PRI_SCHED_ACTIVE);
-	cpu_notifier(sched_cpu_inactive, CPU_PRI_SCHED_INACTIVE);
-
 	return 0;
 }
 early_initcall(migration_init);

@@ -6292,42 +6263,12 @@ static void sched_domains_numa_masks_cle
 	}
 }
 
-/*
- * Update sched_domains_numa_masks[level][node] array when new cpus
- * are onlined.
- */
-static int sched_domains_numa_masks_update(struct notifier_block *nfb,
-					   unsigned long action,
-					   void *hcpu)
-{
-	int cpu = (long)hcpu;
-
-	switch (action & ~CPU_TASKS_FROZEN) {
-	case CPU_ONLINE:
-		sched_domains_numa_masks_set(cpu);
-		break;
-
-	case CPU_DEAD:
-		sched_domains_numa_masks_clear(cpu);
-		break;
-
-	default:
-		return NOTIFY_DONE;
-	}
-
-	return NOTIFY_OK;
-}
 #else
-static inline void sched_init_numa(void)
-{
-}
-
-static int sched_domains_numa_masks_update(struct notifier_block *nfb,
-					   unsigned long action,
-					   void *hcpu)
-{
-	return 0;
-}
+static inline void sched_init_numa(void) { }
+#ifdef CONFIG_HOTPLUG_CPU
+static void sched_domains_numa_masks_set(int cpu) { }
+static void sched_domains_numa_masks_clear(int cpu) { }
+#endif
 #endif /* CONFIG_NUMA */
 
 static int __sdt_alloc(const struct cpumask *cpu_map)

@@ -6696,6 +6637,7 @@ match2:
 	mutex_unlock(&sched_domains_mutex);
 }
 
+#ifdef CONFIG_HOTPLUG_CPU
 static int num_cpus_frozen;	/* used to mark begin/end of suspend/resume */
 
 /*

@@ -6706,13 +6648,9 @@ static int num_cpus_frozen;	/* used to m
  * If we come here as part of a suspend/resume, don't touch cpusets because we
  * want to restore it back to its original state upon resume anyway.
  */
-static int cpuset_cpu_active(struct notifier_block *nfb, unsigned long action,
-			     void *hcpu)
+static void cpuset_cpu_active(void)
 {
-	switch (action) {
-	case CPU_ONLINE_FROZEN:
-	case CPU_DOWN_FAILED_FROZEN:
-
+	if (cpuhp_tasks_frozen) {
 		/*
 		 * num_cpus_frozen tracks how many CPUs are involved in suspend
 		 * resume sequence. As long as this is not the last online

@@ -6722,40 +6660,62 @@ static int cpuset_cpu_active(struct noti
 		num_cpus_frozen--;
 		if (likely(num_cpus_frozen)) {
 			partition_sched_domains(1, NULL, NULL);
-			break;
+			return;
 		}
-
 		/*
 		 * This is the last CPU online operation. So fall through and
 		 * restore the original sched domains by considering the
 		 * cpuset configurations.
 		 */
-
-	case CPU_ONLINE:
-	case CPU_DOWN_FAILED:
-		cpuset_update_active_cpus(true);
-		break;
-	default:
-		return NOTIFY_DONE;
 	}
-	return NOTIFY_OK;
+	cpuset_update_active_cpus(true);
 }
 
-static int cpuset_cpu_inactive(struct notifier_block *nfb, unsigned long action,
-			       void *hcpu)
+static void cpuset_cpu_inactive(void)
 {
-	switch (action) {
-	case CPU_DOWN_PREPARE:
-		cpuset_update_active_cpus(false);
-		break;
-	case CPU_DOWN_PREPARE_FROZEN:
+	if (cpuhp_tasks_frozen) {
 		num_cpus_frozen++;
 		partition_sched_domains(1, NULL, NULL);
-		break;
-	default:
-		return NOTIFY_DONE;
-	}
-	return NOTIFY_OK;
+	} else
+		cpuset_update_active_cpus(false);
+}
+
+static int sched_dead_cpu(unsigned int cpu)
+{
+	sched_domains_numa_masks_clear(cpu);
+	return 0;
+}
+
+static int sched_online_cpu(unsigned int cpu)
+{
+	/* Looks redundant, but we need it in case of down canceled */
+	set_cpu_active(cpu, true);
+	/*
+	 * Asymetric to sched_dead_cpu, but this just fiddles with
+	 * bits. Sigh
+	 */
+	sched_domains_numa_masks_set(cpu);
+	/* This is actually symetric */
+	cpuset_cpu_active();
+	return 0;
+}
+
+static int sched_offline_cpu(unsigned int cpu)
+{
+	set_cpu_active(cpu, false);
+	cpuset_cpu_inactive();
+	return 0;
+}
+#else
+#define sched_dead_cpu		NULL
+#define sched_online_cpu	NULL
+#define sched_offline_cpu	NULL
+#endif
+
+int __cpuinit sched_starting_cpu(unsigned int cpu)
+{
+	set_cpu_active(cpu, true);
+	return 0;
 }
 
 void __init sched_init_smp(void)

@@ -6776,9 +6736,13 @@ void __init sched_init_smp(void)
 	mutex_unlock(&sched_domains_mutex);
 	put_online_cpus();
 
-	hotcpu_notifier(sched_domains_numa_masks_update, CPU_PRI_SCHED_ACTIVE);
-	hotcpu_notifier(cpuset_cpu_active, CPU_PRI_CPUSET_ACTIVE);
-	hotcpu_notifier(cpuset_cpu_inactive, CPU_PRI_CPUSET_INACTIVE);
+	/*
+	 * Note: These callbacks are installed late because we init
+	 * numa and sched domains after we brought up the cpus.
+	 */
+	cpuhp_setup_state_nocalls(CPUHP_SCHED_DEAD, NULL, sched_dead_cpu);
+	cpuhp_setup_state_nocalls(CPUHP_SCHED_ONLINE, sched_online_cpu,
+				  sched_offline_cpu);
 
 	/* RT runtime code needs to handle some hotplug events */
 	hotcpu_notifier(update_runtime, 0);