cpuhp: Expedite RCU when toggling system-wide SMT mode
Subsystems (2)
v5 cover cpuhp: Expedite RCU when toggling system-wide SMT mode
Hello All,
SMT mode switch operation on a large CPU count system takes close to an hour to complete. Initial debugging root caused the delay to the CPU hotplug subsystem being blocked on numerous synchronize_rcu() calls. Simply enabling system-wide RCU expediting reduced the switch time to 5-6 minutes. Since then, different approaches have been explored, of which some had their own side effects and others didn't work as expected.
Approaches explored:
1. Expedited individual CPU hotplug operations by wrapping _cpu_up()/_cpu_down() with rcu_expedite_gp()/rcu_unexpedite_gp() [0]. Peter suggested expediting only when SMT switch is triggered via the sysfs control interface, not for individual hotplug operations [1].
2. Replacing synchronize_rcu() calls in the CPU hotplug codepath with their expedited variants. This is not viable because one synchronize_rcu() is invoked inside cpus_write_lock(), which is shared with other kernel subsystems [5].
3. Hoisting cpus_write_lock() to be taken once for the entire SMT switch operation instead of per-CPU [3][4]. On large systems where the SMT switch can still take 5-6 minutes, holding the lock for that duration causes hung task splats and starves other subsystems depending on the read lock.
4. Peter has also suggested using rcu_sync_{enter|exit}() which as is doesn't help as is, but can be paired the approach 2 from above.
Current approach: expedite RCU grace periods around the SMT switch operation in the sysfs control interface path, per Peter's suggestion [1], with Aboorva's analysis confirming synchronize_rcu() as the bottleneck [2].
[0] https://lore.kernel.org/all/[email protected]
[1] https://lore.kernel.org/all/[email protected]/
[2] https://lore.kernel.org/all/5f2ab8a44d685701fe36cdaa8042a1aef215d10d.camel@linux.vnet.ibm.com
[3] https://lore.kernel.org/all/[email protected]/
[4] https://lore.kernel.org/all/[email protected]/
[5] https://lore.kernel.org/all/[email protected]/
[6] https://lore.kernel.org/all/[email protected]/
v4 -> v5: collected tags
Vishal Chourasia (1):
cpuhp: Expedite RCU when toggling system-wide SMT mode
include/linux/rcupdate.h | 8 ++++++++ kernel/cpu.c | 4 ++++ kernel/rcu/rcu.h | 4 ---- 3 files changed, 12 insertions(+), 4 deletions(-)
Signature · 1 line
2.54.0
PATCH v5 1/1 cpuhp: Expedite RCU when toggling system-wide SMT mode
On large idle systems, changing system-wide SMT level using the sysfs control interface still takes approximately 40-55 minutes. Changing SMT levels is an user triggered operation generally done when systems are fairly idle. This causes users to be blocked from doing other subsequent work.
Analyzing the collected profile data during SMT level switch, showed that CPU hotplug machinery was blocked on synchronize_rcu() calls.
Expedite RCU grace periods for the entire duration of triggered SMT switch operation via the sysfs control interface. Individual CPU hotplug operations via the online/offline interface are not affected.
On a PPC64 system with 400 CPUs:
SMT8 to SMT1:
before: 1m14s
after: 3.2s (~23x faster)SMT1 to SMT8:
before: 2m27s
after: 2.5s (~58x faster)On a large config system with 1920 CPUs, completion time improves from ~1 hour to 5-6 minutes.
--- include/linux/rcupdate.h | 8 ++++++++ kernel/cpu.c | 4 ++++ kernel/rcu/rcu.h | 4 ---- 3 files changed, 12 insertions(+), 4 deletions(-)
include/linux/rcupdate.h +8 -0
@@ -1178,6 +1178,14 @@ rcu_head_after_call_rcu(struct rcu_head *rhp, rcu_callback_t f)extern int rcu_expedited;extern int rcu_normal; +#ifdef CONFIG_TINY_RCU +static inline void rcu_expedite_gp (void ) { } +static inline void rcu_unexpedite_gp (void ) { } +#else +void rcu_expedite_gp (void ); +void rcu_unexpedite_gp (void ); +#endif + DEFINE_LOCK_GUARD_0(rcu, rcu_read_lock(), rcu_read_unlock())DECLARE_LOCK_GUARD_0_ATTRS (rcu,__acquires_shared (RCU ),__releases_shared (RCU ))
kernel/cpu.c +4 -0
@@ -2658,6 +2658,7 @@ int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)int cpu, ret =0 ;cpu_maps_update_begin (); +rcu_expedite_gp ();for_each_online_cpu (cpu) {if (topology_is_primary_thread (cpu))continue ;
@@ -2687,6 +2688,7 @@ int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval) }if (!ret) cpu_smt_control = ctrlval; +rcu_unexpedite_gp ();cpu_maps_update_done ();return ret; }
@@ -2704,6 +2706,7 @@ int cpuhp_smt_enable(void)int cpu, ret =0 ;cpu_maps_update_begin (); +rcu_expedite_gp (); cpu_smt_control =CPU_SMT_ENABLED ;for_each_present_cpu (cpu) {/* Skip online CPUs and CPUs on offline nodes */
@@ -2717,6 +2720,7 @@ int cpuhp_smt_enable(void)/* See comment in cpuhp_smt_disable() */ cpuhp_online_cpu_device (cpu); } +rcu_unexpedite_gp ();cpu_maps_update_done ();return ret; }
kernel/rcu/rcu.h +0 -4
@@ -521,8 +521,6 @@ do { \static inline bool rcu_gp_is_normal (void ) {return true; }static inline bool rcu_gp_is_expedited (void ) {return false; }static inline bool rcu_async_should_hurry (void ) {return false; } -static inline void rcu_expedite_gp (void ) { } -static inline void rcu_unexpedite_gp (void ) { }static inline void rcu_async_hurry (void ) { }static inline void rcu_async_relax (void ) { }static inline bool rcu_cpu_online (int cpu) {return true; }
@@ -530,8 +528,6 @@ static inline bool rcu_cpu_online(int cpu) { return true; }bool rcu_gp_is_normal (void );/* Internal RCU use. */ bool rcu_gp_is_expedited (void );/* Internal RCU use. */ bool rcu_async_should_hurry (void );/* Internal RCU use. */ -void rcu_expedite_gp (void ); -void rcu_unexpedite_gp (void );void rcu_async_hurry (void );void rcu_async_relax (void );void rcupdate_announce_bootup_oddness (void );
Signature · 1 line
2.54.0
Quoted 23 lines · Vishal Chourasia
On large idle systems, changing system-wide SMT level using the sysfs control interface still takes approximately 40-55 minutes. Changing SMT levels is an user triggered operation generally done when systems are fairly idle. This causes users to be blocked from doing other subsequent work.
Analyzing the collected profile data during SMT level switch, showed that CPU hotplug machinery was blocked on
synchronize_rcu()calls.Expedite RCU grace periods for the entire duration of triggered SMT switch operation via the sysfs control interface. Individual CPU hotplug operations via the online/offline interface are not affected.
On a PPC64 system with 400 CPUs:
SMT8 to SMT1:
before: 1m14s after: 3.2s (~23x faster)SMT1 to SMT8:
before: 2m27s after: 2.5s (~58x faster)
Quoted 6 lines
On a large config system with 1920 CPUs, completion time improves from ~1 hour to 5-6 minutes.
Tested-by Samir <[email protected]>Reviewed-by Shrikanth Hegde <[email protected]>Signed-off-by Vishal Chourasia <[email protected]>
From an RCU viewpoint:
I am guessing that you will be taking this up some other path, but if you would like it to to via RCU, please let me know.
Thanx, Paul
Quoted 86 lines
--- include/linux/rcupdate.h | 8 ++++++++ kernel/cpu.c | 4 ++++ kernel/rcu/rcu.h | 4 ---- 3 files changed, 12 insertions(+), 4 deletions(-)include/linux/rcupdate.h +8
-0@@ -1178,6 +1178,14 @@ rcu_head_after_call_rcu(struct rcu_head *rhp, rcu_callback_t f)extern int rcu_expedited;extern int rcu_normal; +#ifdef CONFIG_TINY_RCU +static inline void rcu_expedite_gp (void ) { } +static inline void rcu_unexpedite_gp (void ) { } +#else +void rcu_expedite_gp (void ); +void rcu_unexpedite_gp (void ); +#endif + DEFINE_LOCK_GUARD_0(rcu, rcu_read_lock(), rcu_read_unlock())DECLARE_LOCK_GUARD_0_ATTRS (rcu,__acquires_shared (RCU ),__releases_shared (RCU ))kernel/cpu.c +4
-0@@ -2658,6 +2658,7 @@ int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)int cpu, ret =0 ;cpu_maps_update_begin (); +rcu_expedite_gp ();for_each_online_cpu (cpu) {if (topology_is_primary_thread (cpu))continue ;@@ -2687,6 +2688,7 @@ int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval) }if (!ret) cpu_smt_control = ctrlval; +rcu_unexpedite_gp ();cpu_maps_update_done ();return ret; }@@ -2704,6 +2706,7 @@ int cpuhp_smt_enable(void)int cpu, ret =0 ;cpu_maps_update_begin (); +rcu_expedite_gp (); cpu_smt_control =CPU_SMT_ENABLED ;for_each_present_cpu (cpu) {/* Skip online CPUs and CPUs on offline nodes */ @@ -2717,6 +2720,7 @@ int cpuhp_smt_enable(void)/* See comment in cpuhp_smt_disable() */ cpuhp_online_cpu_device (cpu); } +rcu_unexpedite_gp ();cpu_maps_update_done ();return ret; }kernel/rcu/rcu.h +0
-4@@ -521,8 +521,6 @@ do { \static inline bool rcu_gp_is_normal (void ) {return true; }static inline bool rcu_gp_is_expedited (void ) {return false; }static inline bool rcu_async_should_hurry (void ) {return false; } -static inline void rcu_expedite_gp (void ) { } -static inline void rcu_unexpedite_gp (void ) { }static inline void rcu_async_hurry (void ) { }static inline void rcu_async_relax (void ) { }static inline bool rcu_cpu_online (int cpu) {return true; }@@ -530,8 +528,6 @@ static inline bool rcu_cpu_online(int cpu) { return true; }bool rcu_gp_is_normal (void );/* Internal RCU use. */ bool rcu_gp_is_expedited (void );/* Internal RCU use. */ bool rcu_async_should_hurry (void );/* Internal RCU use. */ -void rcu_expedite_gp (void ); -void rcu_unexpedite_gp (void );void rcu_async_hurry (void );void rcu_async_relax (void );void rcupdate_announce_bootup_oddness (void );Signature · 1 line
2.54.0
Quoted 27 lines · Paul E. McKenney
Quoted 26 lines · Vishal Chourasia
On large idle systems, changing system-wide SMT level using the sysfs control interface still takes approximately 40-55 minutes. Changing SMT levels is an user triggered operation generally done when systems are fairly idle. This causes users to be blocked from doing other subsequent work.
Analyzing the collected profile data during SMT level switch, showed that CPU hotplug machinery was blocked on
synchronize_rcu()calls.Expedite RCU grace periods for the entire duration of triggered SMT switch operation via the sysfs control interface. Individual CPU hotplug operations via the online/offline interface are not affected.
On a PPC64 system with 400 CPUs:
SMT8 to SMT1: before: 1m14s after: 3.2s (~23x faster) SMT1 to SMT8: before: 2m27s after: 2.5s (~58x faster)On a large config system with 1920 CPUs, completion time improves from ~1 hour to 5-6 minutes.
Quoted 7 lines
Quoted 3 lines
Tested-by Samir <[email protected]>Reviewed-by Shrikanth Hegde <[email protected]>Signed-off-by Vishal Chourasia <[email protected]>From an RCU viewpoint:
Reviewed-by Paul E. McKenney <[email protected]>
Thank you Paul for reviewing this.
Quoted 6 lines
I am guessing that you will be taking this up some other path, but if you would like it to to via RCU, please let me know.
Thanx, Paul
I would prefer this to go via tip tree since main change is in cpu.c
Thomas, peter,
What do you guys suggest? can this go via tip tree?
Also, do you have any other review comments on this or test to be done?
Quoted 86 lines
Quoted 86 lines
--- include/linux/rcupdate.h | 8 ++++++++ kernel/cpu.c | 4 ++++ kernel/rcu/rcu.h | 4 ---- 3 files changed, 12 insertions(+), 4 deletions(-)include/linux/rcupdate.h +8
-0@@ -1178,6 +1178,14 @@ rcu_head_after_call_rcu(struct rcu_head *rhp, rcu_callback_t f)extern int rcu_expedited;extern int rcu_normal; +#ifdef CONFIG_TINY_RCU +static inline void rcu_expedite_gp (void ) { } +static inline void rcu_unexpedite_gp (void ) { } +#else +void rcu_expedite_gp (void ); +void rcu_unexpedite_gp (void ); +#endif + DEFINE_LOCK_GUARD_0(rcu, rcu_read_lock(), rcu_read_unlock())DECLARE_LOCK_GUARD_0_ATTRS (rcu,__acquires_shared (RCU ),__releases_shared (RCU ))kernel/cpu.c +4
-0@@ -2658,6 +2658,7 @@ int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)int cpu, ret =0 ;cpu_maps_update_begin (); +rcu_expedite_gp ();for_each_online_cpu (cpu) {if (topology_is_primary_thread (cpu))continue ;@@ -2687,6 +2688,7 @@ int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval) }if (!ret) cpu_smt_control = ctrlval; +rcu_unexpedite_gp ();cpu_maps_update_done ();return ret; }@@ -2704,6 +2706,7 @@ int cpuhp_smt_enable(void)int cpu, ret =0 ;cpu_maps_update_begin (); +rcu_expedite_gp (); cpu_smt_control =CPU_SMT_ENABLED ;for_each_present_cpu (cpu) {/* Skip online CPUs and CPUs on offline nodes */ @@ -2717,6 +2720,7 @@ int cpuhp_smt_enable(void)/* See comment in cpuhp_smt_disable() */ cpuhp_online_cpu_device (cpu); } +rcu_unexpedite_gp ();cpu_maps_update_done ();return ret; }kernel/rcu/rcu.h +0
-4@@ -521,8 +521,6 @@ do { \static inline bool rcu_gp_is_normal (void ) {return true; }static inline bool rcu_gp_is_expedited (void ) {return false; }static inline bool rcu_async_should_hurry (void ) {return false; } -static inline void rcu_expedite_gp (void ) { } -static inline void rcu_unexpedite_gp (void ) { }static inline void rcu_async_hurry (void ) { }static inline void rcu_async_relax (void ) { }static inline bool rcu_cpu_online (int cpu) {return true; }@@ -530,8 +528,6 @@ static inline bool rcu_cpu_online(int cpu) { return true; }bool rcu_gp_is_normal (void );/* Internal RCU use. */ bool rcu_gp_is_expedited (void );/* Internal RCU use. */ bool rcu_async_should_hurry (void );/* Internal RCU use. */ -void rcu_expedite_gp (void ); -void rcu_unexpedite_gp (void );void rcu_async_hurry (void );void rcu_async_relax (void );void rcupdate_announce_bootup_oddness (void );Signature · 1 line
2.54.0
Quoted 24 lines · Shrikanth Hegde
Quoted 7 lines
Quoted 7 lines
On a large config system with 1920 CPUs, completion time improves from ~1 hour to 5-6 minutes.
Quoted 7 lines
Quoted 3 lines
Tested-by Samir <[email protected]>Reviewed-by Shrikanth Hegde <[email protected]>Signed-off-by Vishal Chourasia <[email protected]>From an RCU viewpoint:
Reviewed-by Paul E. McKenney <[email protected]>Thank you Paul for reviewing this.
Quoted 6 lines
I am guessing that you will be taking this up some other path, but if you would like it to to via RCU, please let me know.
Thanx, Paul
Quoted 7 lines
I would prefer this to go via tip tree since main change is in
cpu.cThomas, peter,
What do you guys suggest? can this go via tip tree?Also, do you have any other review comments on this or test to be done?
Gentle Ping.
Thomas, Peter, Ingo, Can this go via tip tree?
Do you guys have any comments to be addressed?
Or Do you want it to go via RCU tree?
Quoted 35 lines · Shrikanth Hegde
Quoted 24 lines · Shrikanth Hegde
Quoted 7 lines
Quoted 7 lines
On a large config system with 1920 CPUs, completion time improves from ~1 hour to 5-6 minutes.
Quoted 7 lines
Quoted 3 lines
Tested-by Samir <[email protected]>Reviewed-by Shrikanth Hegde <[email protected]>Signed-off-by Vishal Chourasia <[email protected]>From an RCU viewpoint:
Reviewed-by Paul E. McKenney <[email protected]>Thank you Paul for reviewing this.
Quoted 6 lines
I am guessing that you will be taking this up some other path, but if you would like it to to via RCU, please let me know.
Thanx, Paul
Quoted 7 lines
I would prefer this to go via tip tree since main change is in
cpu.cThomas, peter,
What do you guys suggest? can this go via tip tree?Also, do you have any other review comments on this or test to be done?
Quoted 7 lines
Gentle Ping.
Thomas, Peter, Ingo, Can this go via tip tree?
Do you guys have any comments to be addressed?Or Do you want it to go via RCU tree?
Gentle Reminder.
This might have fell through the cracks.
Are there any concerns still to be addressed?
Hi Paul, Peter, Ingo, Thomas,
Still seeing need to explicitly expedite during benchmarking while reading this patch series a while ago, reminded me again that we still don't have this patch in upstream.
https://lore.kernel.org/all/[email protected]/
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
I am guessing that you will be taking this up some other path, but if you would like it to to via RCU, please let me know.
Thanx, Paul
Peter, Ingo, Thomas,
Can this go via RCU tree??
Quoted 14 lines
Quoted 14 lines
Quoted 8 lines
I would prefer this to go via tip tree since main change is in
cpu.cThomas, peter,
What do you guys suggest? can this go via tip tree?Also, do you have any other review comments on this or test to be done?
Gentle Ping.
Thomas, Peter, Ingo, Can this go via tip tree?
Do you guys have any comments to be addressed?
Quoted 7 lines
Quoted 2 lines
Or Do you want it to go via RCU tree?
Gentle Reminder.
This might have fell through the cracks.
Are there any concerns still to be addressed?
PS:
I checked it still applies cleanly to tip/sched/core.
Joel Fernandes
maintainerQuoted 35 lines · Paul E. McKenney
Quoted 23 lines · Vishal Chourasia
On large idle systems, changing system-wide SMT level using the sysfs control interface still takes approximately 40-55 minutes. Changing SMT levels is an user triggered operation generally done when systems are fairly idle. This causes users to be blocked from doing other subsequent work.
Analyzing the collected profile data during SMT level switch, showed that CPU hotplug machinery was blocked on
synchronize_rcu()calls.Expedite RCU grace periods for the entire duration of triggered SMT switch operation via the sysfs control interface. Individual CPU hotplug operations via the online/offline interface are not affected.
On a PPC64 system with 400 CPUs:
SMT8 to SMT1:
before: 1m14s after: 3.2s (~23x faster)SMT1 to SMT8:
before: 2m27s after: 2.5s (~58x faster)Quoted 6 lines
On a large config system with 1920 CPUs, completion time improves from ~1 hour to 5-6 minutes.
Tested-by Samir <[email protected]>Reviewed-by Shrikanth Hegde <[email protected]>Signed-off-by Vishal Chourasia <[email protected]>
Quoted 6 lines
From an RCU viewpoint:
Reviewed-by Paul E. McKenney <[email protected]>I am guessing that you will be taking this up some other path, but if you would like it to to via RCU, please let me know.
I do feel there is a strong case, as the change is simple and the performance issue is kind of ridiculous, and the authors have done due diligence.
Since Paul has provided his tag and I am helping with 7.5 merge window for RCU, I will run some tests on this (no promises) but if someone does not want me to apply it - maybe now would be a great time to yell at me.
thanks,
Joel Fernandes
Quoted 89 lines
Thanx, Paul
Quoted 86 lines
---
include/linux/rcupdate.h | 8 ++++++++ kernel/cpu.c | 4 ++++ kernel/rcu/rcu.h | 4 ---- 3 files changed, 12 insertions(+), 4 deletions(-)include/linux/rcupdate.h +8
-0@@ -1178,6 +1178,14 @@ rcu_head_after_call_rcu(struct rcu_head *rhp, rcu_callback_t f)extern int rcu_expedited;extern int rcu_normal; +#ifdef CONFIG_TINY_RCU +static inline void rcu_expedite_gp (void ) { } +static inline void rcu_unexpedite_gp (void ) { } +#else +void rcu_expedite_gp (void ); +void rcu_unexpedite_gp (void ); +#endif + DEFINE_LOCK_GUARD_0(rcu, rcu_read_lock(), rcu_read_unlock())DECLARE_LOCK_GUARD_0_ATTRS (rcu,__acquires_shared (RCU ),__releases_shared (RCU ))kernel/cpu.c +4
-0@@ -2658,6 +2658,7 @@ int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)int cpu, ret =0 ;cpu_maps_update_begin (); +rcu_expedite_gp ();for_each_online_cpu (cpu) {if (topology_is_primary_thread (cpu))continue ;@@ -2687,6 +2688,7 @@ int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval) }if (!ret) cpu_smt_control = ctrlval; +rcu_unexpedite_gp ();cpu_maps_update_done ();return ret; }@@ -2704,6 +2706,7 @@ int cpuhp_smt_enable(void)int cpu, ret =0 ;cpu_maps_update_begin (); +rcu_expedite_gp (); cpu_smt_control =CPU_SMT_ENABLED ;for_each_present_cpu (cpu) {/* Skip online CPUs and CPUs on offline nodes */ @@ -2717,6 +2720,7 @@ int cpuhp_smt_enable(void)/* See comment in cpuhp_smt_disable() */ cpuhp_online_cpu_device (cpu); } +rcu_unexpedite_gp ();cpu_maps_update_done ();return ret; }kernel/rcu/rcu.h +0
-2@@ -521,8 +521,6 @@ do { \static inline bool rcu_gp_is_normal (void ) {return true; }static inline bool rcu_gp_is_expedited (void ) {return false; }static inline bool rcu_async_should_hurry (void ) {return false; } -static inline void rcu_expedite_gp (void ) { } -static inline void rcu_unexpedite_gp (void ) { }static inline void rcu_async_hurry (void ) { }static inline void rcu_async_relax (void ) { }static inline bool rcu_cpu_online (int cpu) {return true; }@@ -530,8 +528,6 @@ static inline bool rcu_cpu_online(int cpu) { return true; }bool
rcu_gp_is_normal(void); /* Internal RCU use. */
boolrcu_gp_is_expedited(void); /* Internal RCU use. */ boolrcu_async_should_hurry(void); /* Internal RCU use. */kernel/rcu/rcu.h +0
-2-void rcu_expedite_gp (void ); -void rcu_unexpedite_gp (void );void
rcu_async_hurry(void);
voidrcu_async_relax(void);
voidrcupdate_announce_bootup_oddness(void);
--
2.54.0