Skip to contentSkip to conversation

PPC64: Generic hotplug cpu support

PATCH In review Zwane Mwaikambo 5 participants 7 messages Feb 1, 2005 – Mar 7, 2005 PowerPC

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 PPC64: Generic hotplug cpu support

Patch provides a generic hotplug cpu implementation, with the only current user being pmac. This doesn't replace real hotplug code as is currently used by LPAR systems. Ben i can add the additional pmac specific code to put the processor into a sleeping state seperately. Thanks to Nathan for testing.

 arch/ppc64/Kconfig                |    2
 arch/ppc64/kernel/idle.c          |    4 +
 arch/ppc64/kernel/irq.c           |   29 +++++++++++++
 arch/ppc64/kernel/pSeries_setup.c |    5 +-
 arch/ppc64/kernel/pmac_setup.c    |    3 +
 arch/ppc64/kernel/pmac_smp.c      |    5 ++
 arch/ppc64/kernel/setup.c         |    3 -
 arch/ppc64/kernel/smp.c           |   80 ++++++++++++++++++++++++++++++++++++++
 arch/ppc64/kernel/sysfs.c         |    6 --
 include/asm-ppc64/machdep.h       |    1
 include/asm-ppc64/smp.h           |    9 +++-
 11 files changed, 136 insertions(+), 11 deletions(-)

Index: linux-2.6.11-rc2-mm2-ppc64/arch/ppc64/Kconfig
===================================================================
RCS file: /home/cvsroot/linux-2.6.11-rc2-mm2/arch/ppc64/Kconfig,v retrieving revision 1.1.1.1

arch/ppc64/Kconfig +1 -1
diff -u -p -B -r1.1.1.1 Kconfig
@@ -313,7 +313,7 @@ source "drivers/pci/Kconfig"
 
 config HOTPLUG_CPU
 	bool "Support for hot-pluggable CPUs"
-	depends on SMP && EXPERIMENTAL && PPC_PSERIES
+	depends on SMP && EXPERIMENTAL && (PPC_PSERIES || PPC_PMAC)
 	select HOTPLUG
 	---help---
 	  Say Y here to be able to turn CPUs off and on.

Index: linux-2.6.11-rc2-mm2-ppc64/arch/ppc64/kernel/idle.c
===================================================================
RCS file: /home/cvsroot/linux-2.6.11-rc2-mm2/arch/ppc64/kernel/idle.c,v retrieving revision 1.1.1.1

arch/ppc64/kernel/idle.c +4 -0
diff -u -p -B -r1.1.1.1 idle.c
@@ -293,6 +293,10 @@ static int native_idle(void)
 			power4_idle();
 		if (need_resched())
 			schedule();
+
+		if (cpu_is_offline(smp_processor_id()) &&
+		    system_state == SYSTEM_RUNNING)
+			cpu_die();
 	}
 	return 0;
 }

Index: linux-2.6.11-rc2-mm2-ppc64/arch/ppc64/kernel/irq.c
===================================================================
RCS file: /home/cvsroot/linux-2.6.11-rc2-mm2/arch/ppc64/kernel/irq.c,v retrieving revision 1.1.1.1

arch/ppc64/kernel/irq.c +29 -0
diff -u -p -B -r1.1.1.1 irq.c
@@ -115,6 +115,35 @@ skip:
 	return 0;
 }
 
+#ifdef CONFIG_HOTPLUG_CPU
+void fixup_irqs(cpumask_t map)
+{
+	unsigned int irq;
+	static int warned;
+
+	for_each_irq(irq) {
+		cpumask_t mask;
+
+		if (irq_desc[irq].status & IRQ_PER_CPU)
+			continue;
+
+		cpus_and(mask, irq_affinity[irq], map);
+		if (any_online_cpu(mask) == NR_CPUS) {
+			printk("Breaking affinity for irq %i\n", irq);
+			mask = map;
+		}
+		if (irq_desc[irq].handler->set_affinity)
+			irq_desc[irq].handler->set_affinity(irq, mask);
+		else if (irq_desc[irq].action && !(warned++))
+			printk("Cannot set affinity for irq %i\n", irq);
+	}
+
+	local_irq_enable();
+	mdelay(1);
+	local_irq_disable();
+}
+#endif
+
 extern int noirqdebug;
 
 /*

Index: linux-2.6.11-rc2-mm2-ppc64/arch/ppc64/kernel/pSeries_setup.c
===================================================================
RCS file: /home/cvsroot/linux-2.6.11-rc2-mm2/arch/ppc64/kernel/pSeries_setup.c,v retrieving revision 1.1.1.1

arch/ppc64/kernel/pSeries_setup.c +3 -2
diff -u -p -B -r1.1.1.1 pSeries_setup.c
@@ -320,8 +320,9 @@ static  void __init pSeries_discover_pic
 	}
 }
 
-static void pSeries_cpu_die(void)
+static void pSeries_mach_cpu_die(void)
 {
+	idle_task_exit();
 	local_irq_disable();
 	/* Some hardware requires clearing the CPPR, while other hardware does not
 	 * it is safe either way
@@ -599,7 +600,7 @@ struct machdep_calls __initdata pSeries_
 	.power_off		= rtas_power_off,
 	.halt			= rtas_halt,
 	.panic			= rtas_os_term,
-	.cpu_die		= pSeries_cpu_die,
+	.cpu_die		= pSeries_mach_cpu_die,
 	.get_boot_time		= pSeries_get_boot_time,
 	.get_rtc_time		= pSeries_get_rtc_time,
 	.set_rtc_time		= pSeries_set_rtc_time,

Index: linux-2.6.11-rc2-mm2-ppc64/arch/ppc64/kernel/pmac_setup.c
===================================================================
RCS file: /home/cvsroot/linux-2.6.11-rc2-mm2/arch/ppc64/kernel/pmac_setup.c,v retrieving revision 1.1.1.1

arch/ppc64/kernel/pmac_setup.c +3 -0
diff -u -p -B -r1.1.1.1 pmac_setup.c
@@ -439,6 +439,9 @@ static int __init pmac_probe(int platfor
 }
 
 struct machdep_calls __initdata pmac_md = {
+#ifdef CONFIG_HOTPLUG_CPU
+	.cpu_die		= generic_mach_cpu_die,
+#endif
 	.probe			= pmac_probe,
 	.setup_arch		= pmac_setup_arch,
 	.init_early		= pmac_init_early,

Index: linux-2.6.11-rc2-mm2-ppc64/arch/ppc64/kernel/pmac_smp.c
===================================================================
RCS file: /home/cvsroot/linux-2.6.11-rc2-mm2/arch/ppc64/kernel/pmac_smp.c,v retrieving revision 1.1.1.1

arch/ppc64/kernel/pmac_smp.c +5 -0
diff -u -p -B -r1.1.1.1 pmac_smp.c
@@ -308,4 +308,9 @@ struct smp_ops_t core99_smp_ops __pmacda
 void __init pmac_setup_smp(void)
 {
 	smp_ops = &core99_smp_ops;
+#ifdef CONFIG_HOTPLUG_CPU
+	smp_ops->cpu_enable = generic_cpu_enable;
+	smp_ops->cpu_disable = generic_cpu_disable;
+	smp_ops->cpu_die = generic_cpu_die;
+#endif
 }

Index: linux-2.6.11-rc2-mm2-ppc64/arch/ppc64/kernel/setup.c
===================================================================
RCS file: /home/cvsroot/linux-2.6.11-rc2-mm2/arch/ppc64/kernel/setup.c,v retrieving revision 1.1.1.1

arch/ppc64/kernel/setup.c +0 -3
diff -u -p -B -r1.1.1.1 setup.c
@@ -1345,9 +1345,6 @@ early_param("xmon", early_xmon);
 
 void cpu_die(void)
 {
-	idle_task_exit();
 	if (ppc_md.cpu_die)
 		ppc_md.cpu_die();
-	local_irq_disable();
-	for (;;);
 }

Index: linux-2.6.11-rc2-mm2-ppc64/arch/ppc64/kernel/smp.c
===================================================================
RCS file: /home/cvsroot/linux-2.6.11-rc2-mm2/arch/ppc64/kernel/smp.c,v retrieving revision 1.1.1.1

arch/ppc64/kernel/smp.c +80 -0

96 lines not on this page · raw message

Index: linux-2.6.11-rc2-mm2-ppc64/arch/ppc64/kernel/sysfs.c
===================================================================
RCS file: /home/cvsroot/linux-2.6.11-rc2-mm2/arch/ppc64/kernel/sysfs.c,v retrieving revision 1.1.1.1

arch/ppc64/kernel/sysfs.c +2 -4
diff -u -p -B -r1.1.1.1 sysfs.c
@@ -18,7 +18,7 @@
 #include <asm/systemcfg.h>
 #include <asm/paca.h>
 #include <asm/lppaca.h>
-
+#include <asm/machdep.h>
 
 static DEFINE_PER_CPU(struct cpu, cpu_devices);
 
@@ -413,9 +413,7 @@ static int __init topology_init(void)
 		 * CPU.  For instance, the boot cpu might never be valid
 		 * for hotplugging.
 		 */
-#ifdef CONFIG_HOTPLUG_CPU
-		if (systemcfg->platform != PLATFORM_PSERIES_LPAR)
-#endif
+		if (!ppc_md.cpu_die)
 			c->no_control = 1;
 
 		if (cpu_online(cpu) || (c->no_control == 0)) {

Index: linux-2.6.11-rc2-mm2-ppc64/include/asm-ppc64/machdep.h
===================================================================
RCS file: /home/cvsroot/linux-2.6.11-rc2-mm2/include/asm-ppc64/machdep.h,v retrieving revision 1.1.1.1

include/asm-ppc64/machdep.h +1 -0
diff -u -p -B -r1.1.1.1 machdep.h
@@ -30,6 +30,7 @@ struct smp_ops_t {
 	void  (*setup_cpu)(int nr);
 	void  (*take_timebase)(void);
 	void  (*give_timebase)(void);
+	int   (*cpu_enable)(unsigned int nr);
 	int   (*cpu_disable)(void);
 	void  (*cpu_die)(unsigned int nr);
 };

Index: linux-2.6.11-rc2-mm2-ppc64/include/asm-ppc64/smp.h
===================================================================
RCS file: /home/cvsroot/linux-2.6.11-rc2-mm2/include/asm-ppc64/smp.h,v retrieving revision 1.1.1.1

include/asm-ppc64/smp.h +8 -1
diff -u -p -B -r1.1.1.1 smp.h
@@ -29,7 +29,7 @@
 extern int boot_cpuid;
 extern int boot_cpuid_phys;
 
-extern void cpu_die(void) __attribute__((noreturn));
+extern void cpu_die(void);
 
 #ifdef CONFIG_SMP
 
@@ -37,6 +37,13 @@ extern void smp_send_debugger_break(int 
 struct pt_regs;
 extern void smp_message_recv(int, struct pt_regs *);
 
+#ifdef CONFIG_HOTPLUG_CPU
+extern void fixup_irqs(cpumask_t map);
+int generic_cpu_disable(void);
+int generic_cpu_enable(unsigned int cpu);
+void generic_cpu_die(unsigned int cpu);
+void generic_mach_cpu_die(void);
+#endif
 
 #define __smp_processor_id() (get_paca()->paca_index)
 #define hard_smp_processor_id() (get_paca()->hw_cpu_id)
#1 lore raw
Quoted 3 lines · Zwane Mwaikambo

Patch provides a generic hotplug cpu implementation, with the only current
user being pmac.

BUG: using smp_processor_id() in preemptible [00000001] code: swapper/0
caller is .native_idle+0x30/0x60
arch/ppc64/kernel/idle.c renamed from arch/ppc64/kernel/idle.c~ppc64-generic-hotplug-cpu-support-fix +1 -1
@@ -294,7 +294,7 @@ static int native_idle(void)
 		if (need_resched())
 			schedule();
 
-		if (cpu_is_offline(smp_processor_id()) &&
+		if (cpu_is_offline(_smp_processor_id()) &&
 		    system_state == SYSTEM_RUNNING)
 			cpu_die();
 	}

_

#2 lore raw
Quoted 19 lines · Andrew Morton
Quoted 3 lines · Zwane Mwaikambo

Patch provides a generic hotplug cpu implementation, with the only current
user being pmac.

BUG: using smp_processor_id() in preemptible [00000001] code: swapper/0
caller is .native_idle+0x30/0x60
arch/ppc64/kernel/idle.c renamed from arch/ppc64/kernel/idle.c~ppc64-generic-hotplug-cpu-support-fix +1 -1
@@ -294,7 +294,7 @@ static int native_idle(void)
 		if (need_resched())
 			schedule();
 
-		if (cpu_is_offline(smp_processor_id()) &&
+		if (cpu_is_offline(_smp_processor_id()) &&
 		    system_state == SYSTEM_RUNNING)
 			cpu_die();
 	}
Quoted 1 line

_

This is the idle loop. Is that ever supposed to be preempted ?

Ben.

#3 lore raw
Quoted 9 lines · Benjamin Herrenschmidt
Quoted 7 lines
arch/ppc64/kernel/idle.c renamed from arch/ppc64/kernel/idle.c~ppc64-generic-hotplug-cpu-support-fix +1 -1
-		if (cpu_is_offline(smp_processor_id()) &&
+		if (cpu_is_offline(_smp_processor_id()) &&
 		    system_state == SYSTEM_RUNNING)
 			cpu_die();
 	}

_

This is the idle loop. Is that ever supposed to be preempted ?

Nope, it's a false positive. We had to do the same in x86's idle loop and probably others will hit it.

#4 lore raw

PATCH 1/1 explicitly bind idle tasks

Quoted 9 lines · Andrew Morton
Quoted 8 lines · Benjamin Herrenschmidt
Quoted 7 lines
(quoted diff) +1 -1
-		if (cpu_is_offline(smp_processor_id()) &&
+		if (cpu_is_offline(_smp_processor_id()) &&
 		    system_state == SYSTEM_RUNNING)
 			cpu_die();
 	}

_

Quoted 4 lines
Quoted 1 line

This is the idle loop. Is that ever supposed to be preempted ?

Nope, it's a false positive. We had to do the same in x86's idle loop and probably others will hit it.

Perhaps I'm missing something, but is there any reason we can't do the following? I've tested it on ppc64, doesn't seem to break anything.

With hotplug cpu and preempt, we tend to see smp_processor_id warnings from idle loop code because it's always checking whether its cpu has gone offline. Replacing every use of smp_processor_id with _smp_processor_id in all idle loop code is one solution; another way is explicitly binding idle threads to their cpus (the smp_processor_id warning does not fire if the caller is bound only to the calling cpu). This has the (admittedly slight) advantage of letting us know if an idle thread ever runs on the wrong cpu.

init/main.c +4 -0
Index: linux-2.6.11-rc5-mm1/init/main.c
===================================================================
@@ -638,6 +638,10 @@
 {
 	lock_kernel();
 	/*
+	 * init can run on any cpu.
+	 */
+	set_cpus_allowed(current, CPU_MASK_ALL);
+	/*
 	 * Tell the world that we're going to be the grim
 	 * reaper of innocent orphaned children.
 	 *
kernel/sched.c +1 -0
Index: linux-2.6.11-rc5-mm1/kernel/sched.c
===================================================================
@@ -4092,6 +4092,7 @@
 	idle->array = NULL;
 	idle->prio = MAX_PRIO;
 	idle->state = TASK_RUNNING;
+	idle->cpus_allowed = cpumask_of_cpu(cpu);
 	set_task_cpu(idle, cpu);
 
 	spin_lock_irqsave(&rq->lock, flags);
#5 lore raw

Re: [PATCH] explicitly bind idle tasks

Quoted 18 lines · Nathan Lynch
Quoted 9 lines · Andrew Morton
Quoted 8 lines · Benjamin Herrenschmidt
Quoted 7 lines
(quoted diff) +1 -1
-		if (cpu_is_offline(smp_processor_id()) &&
+		if (cpu_is_offline(_smp_processor_id()) &&
 		    system_state == SYSTEM_RUNNING)
 			cpu_die();
 	}

_

Quoted 4 lines
Quoted 1 line

This is the idle loop. Is that ever supposed to be preempted ?

Nope, it's a false positive. We had to do the same in x86's idle loop and probably others will hit it.

Perhaps I'm missing something, but is there any reason we can't do the following? I've tested it on ppc64, doesn't seem to break anything.

Quoted 8 lines

With hotplug cpu and preempt, we tend to see smp_processor_id warnings from idle loop code because it's always checking whether its cpu has gone offline. Replacing every use of smp_processor_id with _smp_processor_id in all idle loop code is one solution; another way is explicitly binding idle threads to their cpus (the smp_processor_id warning does not fire if the caller is bound only to the calling cpu). This has the (admittedly slight) advantage of letting us know if an idle thread ever runs on the wrong cpu.

Makes sense to me, for some reason i thought the smp_processor_id() function did a cpu_rq->idle check of some sort.

Thanks,
Zwane

Acked-by on v1 1/1
#6 lore raw

Re: [PATCH] explicitly bind idle tasks

In reply to Nathan Lynch
Quoted 8 lines · Nathan Lynch

With hotplug cpu and preempt, we tend to see smp_processor_id warnings from idle loop code because it's always checking whether its cpu has gone offline. Replacing every use of smp_processor_id with _smp_processor_id in all idle loop code is one solution; another way is explicitly binding idle threads to their cpus (the smp_processor_id warning does not fire if the caller is bound only to the calling cpu). This has the (admittedly slight) advantage of letting us know if an idle thread ever runs on the wrong cpu.

I also prefer explicitly binding idle threads to their cpus instead of replacing use of smp_processor_id with _smp_processor_id.

Quoted 3 lines
Quoted 34 lines

Index: linux-2.6.11-rc5-mm1/init/main.c
===================================================================

init/main.c +4 -0
@@ -638,6 +638,10 @@
 {
 	lock_kernel();
 	/*
+	 * init can run on any cpu.
+	 */
+	set_cpus_allowed(current, CPU_MASK_ALL);
+	/*
 	 * Tell the world that we're going to be the grim
 	 * reaper of innocent orphaned children.
 	 *

Index: linux-2.6.11-rc5-mm1/kernel/sched.c
===================================================================

kernel/sched.c +1 -1
@@ -4092,6 +4092,7 @@
 	idle->array = NULL;
 	idle->prio = MAX_PRIO;
 	idle->state = TASK_RUNNING;
+	idle->cpus_allowed = cpumask_of_cpu(cpu);
 	set_task_cpu(idle, cpu);
 
 	spin_lock_irqsave(&rq->lock, flags);
-

To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/