cpuidle: (POWER) Replace pseries_notify_cpuidle_add call with a elegant notifier to fix lockdep problem in start_secondary
PATCH 1/1 cpuidle: (POWER) Replace pseries_notify_cpuidle_add call with a elegant notifier to fix lockdep problem in start_secondary
The following patch is to remove the pseries_notify_add_cpu() call and replace it by a hot plug notifier.
This would prevent cpuidle resources being
released and allocated each time cpu comes online on pseries. The earlier design was causing a lockdep problem
in start_secondary as reported on this thread
-https://lkml.org/lkml/2012/5/17/2
This applies on 3.4-rc7
--- arch/powerpc/include/asm/processor.h | 2 -- arch/powerpc/platforms/pseries/processor_idle.c | 25 +++++++++++++++++------ arch/powerpc/platforms/pseries/smp.c | 1 - 3 files changed, 19 insertions(+), 9 deletions(-)
(quoted diff) +0 -0
b/arch/powerpc/include/asm/processor.h
index 8e2d037..c6bc22b 100644
arch/powerpc/include/asm/processor.h +0 -2
@@ -390,10 +390,8 @@ void cpu_idle_wait(void);#ifdef CONFIG_PSERIES_IDLE extern void update_smt_snooze_delay (int snooze); -extern int pseries_notify_cpuidle_add_cpu (int cpu);#else static inline void update_smt_snooze_delay (int snooze) {} -static inline int pseries_notify_cpuidle_add_cpu (int cpu) {return 0 ; }#endif extern void flush_instruction_cache (void );
(quoted diff) +0 -0
b/arch/powerpc/platforms/pseries/processor_idle.c
index 41a34bc..d1a7dc0 100644
arch/powerpc/platforms/pseries/processor_idle.c +1 -0
@@ -11,6 +11,7 @@#include <linux/moduleparam.h> #include <linux/cpuidle.h> #include <linux/cpu.h> +#include <linux/notifier.h> #include <asm/paca.h> #include <asm/reg.h>
@@ -186,17 +187,28 @@ static struct cpuidle_state
shared_states[MAX_IDLE_STATE_COUNT] = {
.enter = &shared_cede_loop },
};
-int pseries_notify_cpuidle_add_cpu(int cpu)
+static int pseries_cpuidle_add_cpu_notifier(struct notifier_block *n,
+ unsigned long action, void *hcpu)
{
+ int hotcpu = (unsigned long)hcpu;
struct cpuidle_device *dev =
- per_cpu_ptr(pseries_cpuidle_devices, cpu);
- if (dev && cpuidle_get_driver()) {
- cpuidle_disable_device(dev);
- cpuidle_enable_device(dev);
+ per_cpu_ptr(pseries_cpuidle_devices, hotcpu);
+
+ switch (action & 0xf) {
+ case CPU_ONLINE:
+ if (dev && cpuidle_get_driver()) {
+ cpuidle_disable_device(dev);
+ cpuidle_enable_device(dev);
+ }
+ break;
}
- return 0;
+ return NOTIFY_OK;
}
+static struct notifier_block setup_hotplug_notifier = {
+ .notifier_call = pseries_cpuidle_add_cpu_notifier,
+};
+
/*
* pseries_cpuidle_driver_init()
*/
@@ -321,6 +333,7 @@ static int __init pseries_processor_idle_init(void)
return retval;
}
+ register_cpu_notifier(&setup_hotplug_notifier);
printk(KERN_DEBUG "pseries_idle_driver registered\n");
return 0;
(quoted diff) +0 -0
b/arch/powerpc/platforms/pseries/smp.c
index e16bb8d..71706bc 100644
arch/powerpc/platforms/pseries/smp.c +0 -1
@@ -147,7 +147,6 @@ static void __devinit smp_xics_setup_cpu(int cpu)set_cpu_current_state (cpu,CPU_STATE_ONLINE );set_default_offline_state (cpu);#endif -pseries_notify_cpuidle_add_cpu (cpu); }static int __devinit smp_pSeries_kick_cpu (int nr)
Quoted 6 lines · Deepthi Dharwar
The following patch is to remove the
pseries_notify_add_cpu()call and replace it by a hot plug notifier.
This would prevent cpuidle resources being
released and allocated each time cpu comes online on pseries. The earlier design was causing a lockdep problem
instart_secondaryas reported on this thread
Quoted 6 lines
-https://lkml.org/lkml/2012/5/17/2This applies on 3.4-rc7
Signed-off-by Deepthi Dharwar <[email protected]>---
Any reason why you don't do cpuidle_disable_device() when the CPU is going offline and cpuidle_enable_device() when it's coming back ?
I'm applying the patch for now since it fixes a real problem but if the above makes sense, please send a followup fix.
Cheers,
Ben.
Quoted 92 lines
arch/powerpc/include/asm/processor.h | 2 -- arch/powerpc/platforms/pseries/processor_idle.c | 25 +++++++++++++++++------ arch/powerpc/platforms/pseries/smp.c | 1 - 3 files changed, 19 insertions(+), 9 deletions(-)arch/powerpc/include/asm/processor.h +0
-2@@ -390,10 +390,8 @@ void cpu_idle_wait(void);#ifdef CONFIG_PSERIES_IDLE extern void update_smt_snooze_delay (int snooze); -extern int pseries_notify_cpuidle_add_cpu (int cpu);#else static inline void update_smt_snooze_delay (int snooze) {} -static inline int pseries_notify_cpuidle_add_cpu (int cpu) {return 0 ; }#endif extern void flush_instruction_cache (void );arch/powerpc/platforms/pseries/processor_idle.c +18
-6@@ -11,6 +11,7 @@#include <linux/moduleparam.h> #include <linux/cpuidle.h> #include <linux/cpu.h> +#include <linux/notifier.h> #include <asm/paca.h> #include <asm/reg.h> @@ -186,17 +187,28 @@ static struct cpuidle_state shared_states[MAX_IDLE_STATE_COUNT] = { .enter = &shared_cede_loop }, }; -int pseries_notify_cpuidle_add_cpu (int cpu) +static int pseries_cpuidle_add_cpu_notifier (struct notifier_block *n, +unsigned long action,void *hcpu) { +int hotcpu = (unsigned long )hcpu;struct cpuidle_device *dev = -per_cpu_ptr (pseries_cpuidle_devices, cpu); -if (dev &&cpuidle_get_driver ()) { -cpuidle_disable_device (dev); -cpuidle_enable_device (dev); +per_cpu_ptr (pseries_cpuidle_devices, hotcpu); + +switch (action &0xf ) { +case CPU_ONLINE : +if (dev &&cpuidle_get_driver ()) { +cpuidle_disable_device (dev); +cpuidle_enable_device (dev); + } +break ; } -return 0 ; +return NOTIFY_OK ; } +static struct notifier_block setup_hotplug_notifier = { + .notifier_call = pseries_cpuidle_add_cpu_notifier, +}; +/* * pseries_cpuidle_driver_init() */
arch/powerpc/platforms/pseries/processor_idle.c +1
-0@@ -321,6 +333,7 @@ static int __init pseries_processor_idle_init(void)return retval; } +register_cpu_notifier (&setup_hotplug_notifier);printk (KERN_DEBUG "pseries_idle_driver registered\n" );return 0 ;arch/powerpc/platforms/pseries/smp.c +0
-1@@ -147,7 +147,6 @@ static void __devinit smp_xics_setup_cpu(int cpu)set_cpu_current_state (cpu,CPU_STATE_ONLINE );set_default_offline_state (cpu);#endif -pseries_notify_cpuidle_add_cpu (cpu); }static int __devinit smp_pSeries_kick_cpu (int nr)
Quoted 6 lines · Benjamin Herrenschmidt
Any reason why you don't do
cpuidle_disable_device()when the CPU is going offline andcpuidle_enable_device()when it's coming back ?I'm applying the patch for now since it fixes a real problem but if the above makes sense, please send a followup fix.
No I'm not ... it's damaged in some odd ways (some whitespace missing here or there among others). Please resend a fixed one.
Cheers,
Ben.
The following patch is to remove the pseries_notify_add_cpu() call and replace it by a hot plug notifier.
This would prevent cpuidle resources being
released and allocated each time cpu comes online on pseries. The earlier design was causing a lockdep problem
in start_secondary as reported on this thread
-https://lkml.org/lkml/2012/5/17/2
This applies on 3.4-rc7
--- arch/powerpc/include/asm/processor.h | 2 -- arch/powerpc/platforms/pseries/processor_idle.c | 25 +++++++++++++++++------ arch/powerpc/platforms/pseries/smp.c | 1 - 3 files changed, 19 insertions(+), 9 deletions(-)
arch/powerpc/include/asm/processor.h +0 -2
@@ -390,10 +390,8 @@ void cpu_idle_wait(void);#ifdef CONFIG_PSERIES_IDLE extern void update_smt_snooze_delay (int snooze); -extern int pseries_notify_cpuidle_add_cpu (int cpu);#else static inline void update_smt_snooze_delay (int snooze) {} -static inline int pseries_notify_cpuidle_add_cpu (int cpu) {return 0 ; }#endif extern void flush_instruction_cache (void );
arch/powerpc/platforms/pseries/processor_idle.c +19 -6
@@ -11,6 +11,7 @@#include <linux/moduleparam.h> #include <linux/cpuidle.h> #include <linux/cpu.h> +#include <linux/notifier.h> #include <asm/paca.h> #include <asm/reg.h>
@@ -186,17 +187,28 @@ static struct cpuidle_state shared_states[MAX_IDLE_STATE_COUNT] = { .enter = &shared_cede_loop }, }; -int pseries_notify_cpuidle_add_cpu (int cpu) +static int pseries_cpuidle_add_cpu_notifier (struct notifier_block *n, +unsigned long action,void *hcpu) { +int hotcpu = (unsigned long )hcpu;struct cpuidle_device *dev = -per_cpu_ptr (pseries_cpuidle_devices, cpu); -if (dev &&cpuidle_get_driver ()) { -cpuidle_disable_device (dev); -cpuidle_enable_device (dev); +per_cpu_ptr (pseries_cpuidle_devices, hotcpu); + +switch (action &0xf ) { +case CPU_ONLINE : +if (dev &&cpuidle_get_driver ()) { +cpuidle_disable_device (dev); +cpuidle_enable_device (dev); + } +break ; } -return 0 ; +return NOTIFY_OK ; } +static struct notifier_block setup_hotplug_notifier = { + .notifier_call = pseries_cpuidle_add_cpu_notifier, +}; +/* * pseries_cpuidle_driver_init() */
@@ -321,6 +333,7 @@ static int __init pseries_processor_idle_init(void)return retval; } +register_cpu_notifier (&setup_hotplug_notifier);printk (KERN_DEBUG "pseries_idle_driver registered\n" );return 0 ;
arch/powerpc/platforms/pseries/smp.c +0 -1
@@ -147,7 +147,6 @@ static void __devinit smp_xics_setup_cpu(int cpu)set_cpu_current_state (cpu,CPU_STATE_ONLINE );set_default_offline_state (cpu);#endif -pseries_notify_cpuidle_add_cpu (cpu); }static int __devinit smp_pSeries_kick_cpu (int nr)
Hi Ben,
Quoted 9 lines · Benjamin Herrenschmidt
Quoted 8 lines · Deepthi Dharwar
The following patch is to remove the
pseries_notify_add_cpu()call and replace it by a hot plug notifier.
This would prevent cpuidle resources being
released and allocated each time cpu comes online on pseries. The earlier design was causing a lockdep problem
instart_secondaryas reported on this thread-https://lkml.org/lkml/2012/5/17/2
Quoted 8 lines
Quoted 4 lines
This applies on 3.4-rc7
Signed-off-by Deepthi Dharwar <[email protected]>---
Any reason why you don't do
cpuidle_disable_device()when the CPU is going offline andcpuidle_enable_device()when it's coming back ?
In the current design disable and enable device are called when the cpu comes online. This is to make sure that we clean up and re-register again. All the counters are reset.
Not calling cpu disable when cpu goes offline currently, would only retain the counters right now.
But I could add a offline check and disable the device there, if that results in cleaner design. I will test and send across the patch with couple more pseries-idle fixes soon.
Thanks for your review comments !
Quoted 105 lines
I'm applying the patch for now since it fixes a real problem but if the above makes sense, please send a followup fix.
Cheers,
Ben.Quoted 85 lines
arch/powerpc/include/asm/processor.h | 2 -- arch/powerpc/platforms/pseries/processor_idle.c | 25 +++++++++++++++++------ arch/powerpc/platforms/pseries/smp.c | 1 - 3 files changed, 19 insertions(+), 9 deletions(-)arch/powerpc/include/asm/processor.h +0
-2@@ -390,10 +390,8 @@ void cpu_idle_wait(void);#ifdef CONFIG_PSERIES_IDLE extern void update_smt_snooze_delay (int snooze); -extern int pseries_notify_cpuidle_add_cpu (int cpu);#else static inline void update_smt_snooze_delay (int snooze) {} -static inline int pseries_notify_cpuidle_add_cpu (int cpu) {return 0 ; }#endif extern void flush_instruction_cache (void );arch/powerpc/platforms/pseries/processor_idle.c +18
-6@@ -11,6 +11,7 @@#include <linux/moduleparam.h> #include <linux/cpuidle.h> #include <linux/cpu.h> +#include <linux/notifier.h> #include <asm/paca.h> #include <asm/reg.h> @@ -186,17 +187,28 @@ static struct cpuidle_state shared_states[MAX_IDLE_STATE_COUNT] = { .enter = &shared_cede_loop }, }; -int pseries_notify_cpuidle_add_cpu (int cpu) +static int pseries_cpuidle_add_cpu_notifier (struct notifier_block *n, +unsigned long action,void *hcpu) { +int hotcpu = (unsigned long )hcpu;struct cpuidle_device *dev = -per_cpu_ptr (pseries_cpuidle_devices, cpu); -if (dev &&cpuidle_get_driver ()) { -cpuidle_disable_device (dev); -cpuidle_enable_device (dev); +per_cpu_ptr (pseries_cpuidle_devices, hotcpu); + +switch (action &0xf ) { +case CPU_ONLINE : +if (dev &&cpuidle_get_driver ()) { +cpuidle_disable_device (dev); +cpuidle_enable_device (dev); + } +break ; } -return 0 ; +return NOTIFY_OK ; } +static struct notifier_block setup_hotplug_notifier = { + .notifier_call = pseries_cpuidle_add_cpu_notifier, +}; +/* * pseries_cpuidle_driver_init() */
arch/powerpc/platforms/pseries/processor_idle.c +1
-0@@ -321,6 +333,7 @@ static int __init pseries_processor_idle_init(void)return retval; } +register_cpu_notifier (&setup_hotplug_notifier);printk (KERN_DEBUG "pseries_idle_driver registered\n" );return 0 ;arch/powerpc/platforms/pseries/smp.c +0
-0arch/powerpc/platforms/pseries/smp.c static void __devinit smp_xics_setup_cpu(int cpu) · line 152
Quoted 7 lines
arch/powerpc/platforms/pseries/smp.c +0
-1@@ -147,7 +147,6 @@ static void __devinit smp_xics_setup_cpu(int cpu)set_cpu_current_state (cpu,CPU_STATE_ONLINE );set_default_offline_state (cpu);#endif -pseries_notify_cpuidle_add_cpu (cpu); }static int __devinit smp_pSeries_kick_cpu (int nr)_______________________________________________
Linuxppc-dev mailing list
[email protected]
https://lists.ozlabs.org/listinfo/linuxppc-dev