PPC64: Generic hotplug cpu support
PATCH 1/1 PPC64: Generic hotplug cpu support
Patch provides a generic hotplug cpu implementation, with the only current user being pmac. This doesn't replace real hotplug code as is currently used by LPAR systems. Ben i can add the additional pmac specific code to put the processor into a sleeping state seperately. Thanks to Nathan for testing.
arch/ppc64/Kconfig | 2 arch/ppc64/kernel/idle.c | 4 + arch/ppc64/kernel/irq.c | 29 +++++++++++++ arch/ppc64/kernel/pSeries_setup.c | 5 +- arch/ppc64/kernel/pmac_setup.c | 3 + arch/ppc64/kernel/pmac_smp.c | 5 ++ arch/ppc64/kernel/setup.c | 3 - arch/ppc64/kernel/smp.c | 80 ++++++++++++++++++++++++++++++++++++++ arch/ppc64/kernel/sysfs.c | 6 -- include/asm-ppc64/machdep.h | 1 include/asm-ppc64/smp.h | 9 +++- 11 files changed, 136 insertions(+), 11 deletions(-)
Index: linux-2.6.11-rc2-mm2-ppc64/arch/ppc64/Kconfig
===================================================================
RCS file: /home/cvsroot/linux-2.6.11-rc2-mm2/arch/ppc64/Kconfig,v retrieving revision 1.1.1.1
arch/ppc64/Kconfig +1 -1
@@ -313,7 +313,7 @@ source "drivers/pci/Kconfig" config HOTPLUG_CPU bool "Support for hot-pluggable CPUs" - depends on SMP && EXPERIMENTAL && PPC_PSERIES + depends on SMP && EXPERIMENTAL && (PPC_PSERIES || PPC_PMAC) select HOTPLUG ---help--- Say Y here to be able to turn CPUs off and on.
Index: linux-2.6.11-rc2-mm2-ppc64/arch/ppc64/kernel/idle.c
===================================================================
RCS file: /home/cvsroot/linux-2.6.11-rc2-mm2/arch/ppc64/kernel/idle.c,v retrieving revision 1.1.1.1
arch/ppc64/kernel/idle.c +4 -0
@@ -293,6 +293,10 @@ static int native_idle(void)power4_idle ();if (need_resched ())schedule (); + +if (cpu_is_offline (smp_processor_id ()) && + system_state ==SYSTEM_RUNNING ) +cpu_die (); }return 0 ; }
Index: linux-2.6.11-rc2-mm2-ppc64/arch/ppc64/kernel/irq.c
===================================================================
RCS file: /home/cvsroot/linux-2.6.11-rc2-mm2/arch/ppc64/kernel/irq.c,v retrieving revision 1.1.1.1
arch/ppc64/kernel/irq.c +29 -0
@@ -115,6 +115,35 @@ skip:return 0 ; } +#ifdef CONFIG_HOTPLUG_CPU +void fixup_irqs (cpumask_t map) +{ +unsigned int irq; +static int warned; + +for_each_irq (irq) { +cpumask_t mask; + +if (irq_desc[irq].status &IRQ_PER_CPU ) +continue ; + +cpus_and (mask, irq_affinity[irq], map); +if (any_online_cpu (mask) ==NR_CPUS ) { +printk ("Breaking affinity for irq %i\n" , irq); + mask = map; + } +if (irq_desc[irq].handler->set_affinity) + irq_desc[irq].handler->set_affinity (irq, mask); +else if (irq_desc[irq].action && !(warned++)) +printk ("Cannot set affinity for irq %i\n" , irq); + } + +local_irq_enable (); +mdelay (1 ); +local_irq_disable (); +} +#endif +extern int noirqdebug;/*
Index: linux-2.6.11-rc2-mm2-ppc64/arch/ppc64/kernel/pSeries_setup.c
===================================================================
RCS file: /home/cvsroot/linux-2.6.11-rc2-mm2/arch/ppc64/kernel/pSeries_setup.c,v retrieving revision 1.1.1.1
arch/ppc64/kernel/pSeries_setup.c +3 -2
@@ -320,8 +320,9 @@ static void __init pSeries_discover_pic } } -static void pSeries_cpu_die (void ) +static void pSeries_mach_cpu_die (void ) { +idle_task_exit ();local_irq_disable ();/* Some hardware requires clearing the CPPR, while other hardware does not * it is safe either way
@@ -599,7 +600,7 @@ struct machdep_calls __initdata pSeries_ .power_off = rtas_power_off, .halt = rtas_halt, .panic = rtas_os_term, - .cpu_die = pSeries_cpu_die, + .cpu_die = pSeries_mach_cpu_die, .get_boot_time = pSeries_get_boot_time, .get_rtc_time = pSeries_get_rtc_time, .set_rtc_time = pSeries_set_rtc_time,
Index: linux-2.6.11-rc2-mm2-ppc64/arch/ppc64/kernel/pmac_setup.c
===================================================================
RCS file: /home/cvsroot/linux-2.6.11-rc2-mm2/arch/ppc64/kernel/pmac_setup.c,v retrieving revision 1.1.1.1
arch/ppc64/kernel/pmac_setup.c +3 -0
@@ -439,6 +439,9 @@ static int __init pmac_probe(int platfor }struct machdep_calls __initdata pmac_md = { +#ifdef CONFIG_HOTPLUG_CPU + .cpu_die = generic_mach_cpu_die, +#endif .probe = pmac_probe, .setup_arch = pmac_setup_arch, .init_early = pmac_init_early,
Index: linux-2.6.11-rc2-mm2-ppc64/arch/ppc64/kernel/pmac_smp.c
===================================================================
RCS file: /home/cvsroot/linux-2.6.11-rc2-mm2/arch/ppc64/kernel/pmac_smp.c,v retrieving revision 1.1.1.1
arch/ppc64/kernel/pmac_smp.c +5 -0
@@ -308,4 +308,9 @@ struct smp_ops_t core99_smp_ops __pmacdavoid __init pmac_setup_smp (void ) { smp_ops = &core99_smp_ops; +#ifdef CONFIG_HOTPLUG_CPU + smp_ops->cpu_enable = generic_cpu_enable; + smp_ops->cpu_disable = generic_cpu_disable; + smp_ops->cpu_die = generic_cpu_die; +#endif }
Index: linux-2.6.11-rc2-mm2-ppc64/arch/ppc64/kernel/setup.c
===================================================================
RCS file: /home/cvsroot/linux-2.6.11-rc2-mm2/arch/ppc64/kernel/setup.c,v retrieving revision 1.1.1.1
arch/ppc64/kernel/setup.c +0 -3
@@ -1345,9 +1345,6 @@ early_param("xmon", early_xmon);void cpu_die (void ) { -idle_task_exit ();if (ppc_md.cpu_die) ppc_md.cpu_die (); -local_irq_disable (); -for (;;); }
Index: linux-2.6.11-rc2-mm2-ppc64/arch/ppc64/kernel/smp.c
===================================================================
RCS file: /home/cvsroot/linux-2.6.11-rc2-mm2/arch/ppc64/kernel/smp.c,v retrieving revision 1.1.1.1
arch/ppc64/kernel/smp.c +80 -0
96 lines not on this page · raw message
Index: linux-2.6.11-rc2-mm2-ppc64/arch/ppc64/kernel/sysfs.c
===================================================================
RCS file: /home/cvsroot/linux-2.6.11-rc2-mm2/arch/ppc64/kernel/sysfs.c,v retrieving revision 1.1.1.1
arch/ppc64/kernel/sysfs.c +2 -4
@@ -18,7 +18,7 @@#include <asm/systemcfg.h> #include <asm/paca.h> #include <asm/lppaca.h> - +#include <asm/machdep.h> static DEFINE_PER_CPU(struct cpu, cpu_devices);
@@ -413,9 +413,7 @@ static int __init topology_init(void)* CPU. For instance, the boot cpu might never be valid * for hotplugging. */ -#ifdef CONFIG_HOTPLUG_CPU - if (systemcfg->platform != PLATFORM_PSERIES_LPAR) -#endif +if (!ppc_md.cpu_die) c->no_control =1 ;if (cpu_online (cpu) || (c->no_control ==0 )) {
Index: linux-2.6.11-rc2-mm2-ppc64/include/asm-ppc64/machdep.h
===================================================================
RCS file: /home/cvsroot/linux-2.6.11-rc2-mm2/include/asm-ppc64/machdep.h,v retrieving revision 1.1.1.1
include/asm-ppc64/machdep.h +1 -0
@@ -30,6 +30,7 @@ struct smp_ops_t {void (*setup_cpu)(int nr);void (*take_timebase)(void );void (*give_timebase)(void ); +int (*cpu_enable)(unsigned int nr);int (*cpu_disable)(void );void (*cpu_die)(unsigned int nr); };
Index: linux-2.6.11-rc2-mm2-ppc64/include/asm-ppc64/smp.h
===================================================================
RCS file: /home/cvsroot/linux-2.6.11-rc2-mm2/include/asm-ppc64/smp.h,v retrieving revision 1.1.1.1
include/asm-ppc64/smp.h +8 -1
@@ -29,7 +29,7 @@extern int boot_cpuid;extern int boot_cpuid_phys; -extern void cpu_die (void ) __attribute__((noreturn)); +extern void cpu_die (void );#ifdef CONFIG_SMP
@@ -37,6 +37,13 @@ extern void smp_send_debugger_break(intstruct pt_regs ;extern void smp_message_recv (int ,struct pt_regs *); +#ifdef CONFIG_HOTPLUG_CPU +extern void fixup_irqs (cpumask_t map); +int generic_cpu_disable (void ); +int generic_cpu_enable (unsigned int cpu); +void generic_cpu_die (unsigned int cpu); +void generic_mach_cpu_die (void ); +#endif #define __smp_processor_id () (get_paca()->paca_index)#define hard_smp_processor_id () (get_paca()->hw_cpu_id)
Quoted 3 lines · Zwane Mwaikambo
Patch provides a generic hotplug cpu implementation, with the only current
user being pmac.
BUG: using smp_processor_id() in preemptible [00000001] code: swapper/0 caller is .native_idle+0x30/0x60
arch/ppc64/kernel/idle.c renamed from arch/ppc64/kernel/idle.c~ppc64-generic-hotplug-cpu-support-fix +1 -1
@@ -294,7 +294,7 @@ static int native_idle(void)if (need_resched ())schedule (); -if (cpu_is_offline (smp_processor_id ()) && +if (cpu_is_offline (_smp_processor_id ()) && system_state ==SYSTEM_RUNNING )cpu_die (); }
_
Quoted 19 lines · Andrew Morton
Quoted 3 lines · Zwane Mwaikambo
Patch provides a generic hotplug cpu implementation, with the only current
user being pmac.BUG: using smp_processor_id() in preemptible [00000001] code: swapper/0 caller is .native_idle+0x30/0x60arch/ppc64/kernel/idle.c renamed from arch/ppc64/kernel/idle.c~ppc64-generic-hotplug-cpu-support-fix +1
-1@@ -294,7 +294,7 @@ static int native_idle(void)if (need_resched ())schedule (); -if (cpu_is_offline (smp_processor_id ()) && +if (cpu_is_offline (_smp_processor_id ()) && system_state ==SYSTEM_RUNNING )cpu_die (); }
Quoted 1 line
_
This is the idle loop. Is that ever supposed to be preempted ?
Ben.
Quoted 9 lines · Benjamin Herrenschmidt
Quoted 7 lines
arch/ppc64/kernel/idle.c renamed from arch/ppc64/kernel/idle.c~ppc64-generic-hotplug-cpu-support-fix +1
-1-if (cpu_is_offline (smp_processor_id ()) && +if (cpu_is_offline (_smp_processor_id ()) && system_state ==SYSTEM_RUNNING )cpu_die (); }_
This is the idle loop. Is that ever supposed to be preempted ?
Nope, it's a false positive. We had to do the same in x86's idle loop and probably others will hit it.
PATCH 1/1 explicitly bind idle tasks
Quoted 9 lines · Andrew Morton
Quoted 8 lines · Benjamin Herrenschmidt
Quoted 7 lines
(quoted diff) +1
-1- if (cpu_is_offline(smp_processor_id()) && + if (cpu_is_offline(_smp_processor_id()) && system_state == SYSTEM_RUNNING) cpu_die(); }_
Quoted 4 lines
Quoted 1 line
This is the idle loop. Is that ever supposed to be preempted ?
Nope, it's a false positive. We had to do the same in x86's idle loop and probably others will hit it.
Perhaps I'm missing something, but is there any reason we can't do the following? I've tested it on ppc64, doesn't seem to break anything.
With hotplug cpu and preempt, we tend to see smp_processor_id warnings from idle loop code because it's always checking whether its cpu has gone offline. Replacing every use of smp_processor_id with _smp_processor_id in all idle loop code is one solution; another way is explicitly binding idle threads to their cpus (the smp_processor_id warning does not fire if the caller is bound only to the calling cpu). This has the (admittedly slight) advantage of letting us know if an idle thread ever runs on the wrong cpu.
init/main.c +4 -0
@@ -638,6 +638,10 @@ {lock_kernel ();/* +* init can run on any cpu. +*/ +set_cpus_allowed (current,CPU_MASK_ALL ); +/* * Tell the world that we're going to be the grim * reaper of innocent orphaned children. *
kernel/sched.c +1 -0
@@ -4092,6 +4092,7 @@ idle->array =NULL ; idle->prio =MAX_PRIO ; idle->state =TASK_RUNNING ; + idle->cpus_allowed =cpumask_of_cpu (cpu);set_task_cpu (idle, cpu);spin_lock_irqsave (&rq->lock, flags);
Re: [PATCH] explicitly bind idle tasks
Quoted 18 lines · Nathan Lynch
Quoted 9 lines · Andrew Morton
Quoted 8 lines · Benjamin Herrenschmidt
Quoted 7 lines
(quoted diff) +1
-1- if (cpu_is_offline(smp_processor_id()) && + if (cpu_is_offline(_smp_processor_id()) && system_state == SYSTEM_RUNNING) cpu_die(); }_
Quoted 4 lines
Quoted 1 line
This is the idle loop. Is that ever supposed to be preempted ?
Nope, it's a false positive. We had to do the same in x86's idle loop and probably others will hit it.
Perhaps I'm missing something, but is there any reason we can't do the following? I've tested it on ppc64, doesn't seem to break anything.
Quoted 8 lines
With hotplug cpu and preempt, we tend to see
smp_processor_idwarnings from idle loop code because it's always checking whether its cpu has gone offline. Replacing every use ofsmp_processor_idwith_smp_processor_idin all idle loop code is one solution; another way is explicitly binding idle threads to their cpus (thesmp_processor_idwarning does not fire if the caller is bound only to the calling cpu). This has the (admittedly slight) advantage of letting us know if an idle thread ever runs on the wrong cpu.
Makes sense to me, for some reason i thought the smp_processor_id() function did a cpu_rq->idle check of some sort.
Thanks,
Zwane
Re: [PATCH] explicitly bind idle tasks
In reply to Nathan LynchQuoted 8 lines · Nathan Lynch
With hotplug cpu and preempt, we tend to see
smp_processor_idwarnings from idle loop code because it's always checking whether its cpu has gone offline. Replacing every use ofsmp_processor_idwith_smp_processor_idin all idle loop code is one solution; another way is explicitly binding idle threads to their cpus (thesmp_processor_idwarning does not fire if the caller is bound only to the calling cpu). This has the (admittedly slight) advantage of letting us know if an idle thread ever runs on the wrong cpu.
I also prefer explicitly binding idle threads to their cpus instead of replacing use of smp_processor_id with _smp_processor_id.
Quoted 3 lines
Signed-off-by Nathan Lynch <[email protected]>
Quoted 34 lines
Index:
linux-2.6.11-rc5-mm1/init/main.c
===================================================================init/main.c +4
-0@@ -638,6 +638,10 @@ {lock_kernel ();/* +* init can run on any cpu. +*/ +set_cpus_allowed (current,CPU_MASK_ALL ); +/* * Tell the world that we're going to be the grim * reaper of innocent orphaned children. * Index:
linux-2.6.11-rc5-mm1/kernel/sched.c
===================================================================kernel/sched.c +1
-1@@ -4092,6 +4092,7 @@ idle->array =NULL ; idle->prio =MAX_PRIO ; idle->state =TASK_RUNNING ; + idle->cpus_allowed =cpumask_of_cpu (cpu);set_task_cpu (idle, cpu);spin_lock_irqsave (&rq->lock, flags); -To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/