cpuhp: Improve SMT switch time via lock batching and RCU expedition
Subsystems (2)
v2 cover cpuhp: Improve SMT switch time via lock batching and RCU expedition
On large systems with high core counts, toggling SMT modes via sysfs (/sys/devices/system/cpu/smt/control) incurs significant latency. For instance, on ~2000 CPUs, switching SMT modes can take close to an hour as the system hotplugs each hardware thread individually. This series reduces this time to minutes.
Analysis of the hotplug path [1] identifies synchronize_rcu() as the primary bottleneck. During a bulk SMT switch, the kernel repeatedly enters RCU grace periods for each CPU being brought online or offline.
This series optimizes the SMT transition in two ways:
1. Lock Batching [1]: Instead of repeatedly acquiring and releasing the CPU hotplug write lock for every individual CPU, we now hold cpus_write_lock across the entire SMT toggle operation.
2. Expedite RCU grace period [2]: It utilizes rcu_expedite_gp() to force expedite grace periods specifically for the duration of the SMT switch. The trade-off is justified here to prevent the administrative task of SMT switching from stalling for an unacceptable duration on large systems.
v1: https://lore.kernel.org/all/[email protected]/
[1] https://lore.kernel.org/all/5f2ab8a44d685701fe36cdaa8042a1aef215d10d.camel@linux.vnet.ibm.com
[2] https://lore.kernel.org/all/[email protected]/
Vishal Chourasia (2):
cpuhp: Optimize SMT switch operation by batching lock acquisition
cpuhp: Expedite RCU grace periods during SMT operations
include/linux/rcupdate.h | 3 ++ kernel/cpu.c | 77 +++++++++++++++++++++++++++++----------- 2 files changed, 60 insertions(+), 20 deletions(-)
Signature · 1 line
2.53.0
PATCH v2 1/2 cpuhp: Optimize SMT switch operation by batching lock acquisition
Bulk CPU hotplug operations, such as an SMT switch operation, requires hotplugging multiple CPUs. The current implementation takes cpus_write_lock() for each individual CPU, causing multiple slow grace period requests.
Introduce cpu_up_locked() and cpu_down_locked() that assume the caller already holds cpus_write_lock(). The cpuhp_smt_enable() and cpuhp_smt_disable() functions are updated to hold the lock once around the entire loop, rather than for each individual CPU.
--- kernel/cpu.c | 73 ++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 53 insertions(+), 20 deletions(-)
kernel/cpu.c +53 -20
@@ -1400,8 +1400,8 @@ static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,return ret; } -/* Requires cpu_add_remove_lock to be held */ -static int __ref _cpu_down (unsigned int cpu,int tasks_frozen, +/* Requires cpu_add_remove_lock and cpus_write_lock to be held */ +static int __ref cpu_down_locked (unsigned int cpu,int tasks_frozen,enum cpuhp_state target) {struct cpuhp_cpu_state *st =per_cpu_ptr (&cpuhp_state, cpu);
@@ -1413,7 +1413,7 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,if (!cpu_present (cpu))return -EINVAL ; -cpus_write_lock (); +lockdep_assert_cpus_held ();/* * Keep at least one housekeeping cpu onlined to avoid generating
@@ -1421,8 +1421,7 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,*/ if (cpumask_any_and (cpu_online_mask,housekeeping_cpumask (HK_TYPE_DOMAIN )) >= nr_cpu_ids) { - ret = -EBUSY ; - goto out; +return -EBUSY ; } cpuhp_tasks_frozen = tasks_frozen;
@@ -1440,14 +1439,14 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,* return the error code.. */ if (ret) - goto out; +return ret;/* * We might have stopped still in the range of the AP hotplug * thread. Nothing to do anymore. */ if (st->state >CPUHP_TEARDOWN_CPU ) - goto out; +return 0 ; st->target = target; }
@@ -1464,8 +1463,17 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,WARN (1 ,"DEAD callback error for CPU%d" , cpu); } } +return ret; +} -out: +static int __ref _cpu_down (unsigned int cpu,int tasks_frozen, +enum cpuhp_state target) +{ + +int ret; + +cpus_write_lock (); + ret =cpu_down_locked (cpu, tasks_frozen, target);cpus_write_unlock ();arch_smt_update ();return ret;
@@ -1613,18 +1621,18 @@ void cpuhp_online_idle(enum cpuhp_state state)complete_ap_thread (st, true); } -/* Requires cpu_add_remove_lock to be held */ -static int _cpu_up (unsigned int cpu,int tasks_frozen,enum cpuhp_state target) +/* Requires cpu_add_remove_lock and cpus_write_lock to be held. */ +static int cpu_up_locked (unsigned int cpu,int tasks_frozen, +enum cpuhp_state target) {struct cpuhp_cpu_state *st =per_cpu_ptr (&cpuhp_state, cpu);struct task_struct *idle;int ret =0 ; -cpus_write_lock (); +lockdep_assert_cpus_held ();if (!cpu_present (cpu)) { - ret = -EINVAL ; - goto out; +return -EINVAL ; }/*
@@ -1632,14 +1640,13 @@ static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)* caller. Nothing to do. */ if (st->state >= target) - goto out; +return 0 ;if (st->state ==CPUHP_OFFLINE ) {/* Let it fail before we try to bring the cpu up */ idle =idle_thread_get (cpu);if (IS_ERR (idle)) { - ret =PTR_ERR (idle); - goto out; +return PTR_ERR (idle); }/*
@@ -1663,7 +1670,7 @@ static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)* return the error code.. */ if (ret) - goto out; +return ret; }/*
@@ -1673,7 +1680,16 @@ static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)*/ target =min ((int )target,CPUHP_BRINGUP_CPU ); ret =cpuhp_up_callbacks (cpu, st, target); -out: +return ret; +} + +/* Requires cpu_add_remove_lock to be held */ +static int _cpu_up (unsigned int cpu,int tasks_frozen,enum cpuhp_state target) +{ +int ret; + +cpus_write_lock (); + ret =cpu_up_locked (cpu, tasks_frozen, target);cpus_write_unlock ();arch_smt_update ();return ret;
@@ -2659,6 +2675,16 @@ int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)int cpu, ret =0 ;cpu_maps_update_begin (); +if (cpu_hotplug_offline_disabled) { + ret = -EOPNOTSUPP ; + goto out; + } +if (cpu_hotplug_disabled) { + ret = -EBUSY ; + goto out; + } +/* Hold cpus_write_lock() for entire batch operation. */ +cpus_write_lock ();for_each_online_cpu (cpu) {if (topology_is_primary_thread (cpu))continue ;
@@ -2668,7 +2694,7 @@ int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)*/ if (ctrlval ==CPU_SMT_ENABLED &&cpu_smt_thread_allowed (cpu))continue ; - ret =cpu_down_maps_locked (cpu,CPUHP_OFFLINE ); + ret =cpu_down_locked (cpu,0 ,CPUHP_OFFLINE );if (ret)break ;/*
@@ -2688,6 +2714,9 @@ int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval) }if (!ret) cpu_smt_control = ctrlval; +cpus_write_unlock (); +arch_smt_update (); +out:cpu_maps_update_done ();return ret; }
@@ -2705,6 +2734,8 @@ int cpuhp_smt_enable(void)int cpu, ret =0 ;cpu_maps_update_begin (); +/* Hold cpus_write_lock() for entire batch operation. */ +cpus_write_lock (); cpu_smt_control =CPU_SMT_ENABLED ;for_each_present_cpu (cpu) {/* Skip online CPUs and CPUs on offline nodes */
@@ -2712,12 +2743,14 @@ int cpuhp_smt_enable(void)continue ;if (!cpu_smt_thread_allowed (cpu) || !topology_is_core_online (cpu))continue ; - ret =_cpu_up (cpu,0 ,CPUHP_ONLINE ); + ret =cpu_up_locked (cpu,0 ,CPUHP_ONLINE );if (ret)break ;/* See comment in cpuhp_smt_disable() */ cpuhp_online_cpu_device (cpu); } +cpus_write_unlock (); +arch_smt_update ();cpu_maps_update_done ();return ret; }
Signature · 1 line
2.53.0
Re: [PATCH v2 1/2] cpuhp: Optimize SMT switch operation by batching lock acquisition
Quoted 11 lines · Vishal Chourasia
Bulk CPU hotplug operations, such as an SMT switch operation, requires hotplugging multiple CPUs. The current implementation takes
cpus_write_lock()for each individual CPU, causing multiple slow grace period requests.Introduce
cpu_up_locked()andcpu_down_locked()that assume the caller already holdscpus_write_lock(). Thecpuhp_smt_enable()andcpuhp_smt_disable()functions are updated to hold the lock once around the entire loop, rather than for each individual CPU.Suggested-by Peter Zijlstra <[email protected]>
The code hositing the cpus_write_lock up in the cpuhp_smt_enable() was provided by Joel [1]. Thanks Joel.
I missed adding an appropriate tag for it.
[1] https://lore.kernel.org/all/20260119051835.GA696111@joelbox2/
Quoted 207 lines
Signed-off-by Vishal Chourasia <[email protected]>--- kernel/cpu.c | 73 ++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 53 insertions(+), 20 deletions(-)kernel/cpu.c +53
-20@@ -1400,8 +1400,8 @@ static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,return ret; } -/* Requires cpu_add_remove_lock to be held */ -static int __ref _cpu_down (unsigned int cpu,int tasks_frozen, +/* Requires cpu_add_remove_lock and cpus_write_lock to be held */ +static int __ref cpu_down_locked (unsigned int cpu,int tasks_frozen,enum cpuhp_state target) {struct cpuhp_cpu_state *st =per_cpu_ptr (&cpuhp_state, cpu);@@ -1413,7 +1413,7 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,if (!cpu_present (cpu))return -EINVAL ; -cpus_write_lock (); +lockdep_assert_cpus_held ();/* * Keep at least one housekeeping cpu onlined to avoid generating @@ -1421,8 +1421,7 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,*/ if (cpumask_any_and (cpu_online_mask,housekeeping_cpumask (HK_TYPE_DOMAIN )) >= nr_cpu_ids) { - ret = -EBUSY ; - goto out; +return -EBUSY ; } cpuhp_tasks_frozen = tasks_frozen;@@ -1440,14 +1439,14 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,* return the error code.. */ if (ret) - goto out; +return ret;/* * We might have stopped still in the range of the AP hotplug * thread. Nothing to do anymore. */ if (st->state >CPUHP_TEARDOWN_CPU ) - goto out; +return 0 ; st->target = target; }@@ -1464,8 +1463,17 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,WARN (1 ,"DEAD callback error for CPU%d" , cpu); } } +return ret; +} -out: +static int __ref _cpu_down (unsigned int cpu,int tasks_frozen, +enum cpuhp_state target) +{ + +int ret; + +cpus_write_lock (); + ret =cpu_down_locked (cpu, tasks_frozen, target);cpus_write_unlock ();arch_smt_update ();return ret;@@ -1613,18 +1621,18 @@ void cpuhp_online_idle(enum cpuhp_state state)complete_ap_thread (st, true); } -/* Requires cpu_add_remove_lock to be held */ -static int _cpu_up (unsigned int cpu,int tasks_frozen,enum cpuhp_state target) +/* Requires cpu_add_remove_lock and cpus_write_lock to be held. */ +static int cpu_up_locked (unsigned int cpu,int tasks_frozen, +enum cpuhp_state target) {struct cpuhp_cpu_state *st =per_cpu_ptr (&cpuhp_state, cpu);struct task_struct *idle;int ret =0 ; -cpus_write_lock (); +lockdep_assert_cpus_held ();if (!cpu_present (cpu)) { - ret = -EINVAL ; - goto out; +return -EINVAL ; }/* @@ -1632,14 +1640,13 @@ static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)* caller. Nothing to do. */ if (st->state >= target) - goto out; +return 0 ;if (st->state ==CPUHP_OFFLINE ) {/* Let it fail before we try to bring the cpu up */ idle =idle_thread_get (cpu);if (IS_ERR (idle)) { - ret =PTR_ERR (idle); - goto out; +return PTR_ERR (idle); }/* @@ -1663,7 +1670,7 @@ static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)* return the error code.. */ if (ret) - goto out; +return ret; }/* @@ -1673,7 +1680,16 @@ static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)*/ target =min ((int )target,CPUHP_BRINGUP_CPU ); ret =cpuhp_up_callbacks (cpu, st, target); -out: +return ret; +} + +/* Requires cpu_add_remove_lock to be held */ +static int _cpu_up (unsigned int cpu,int tasks_frozen,enum cpuhp_state target) +{ +int ret; + +cpus_write_lock (); + ret =cpu_up_locked (cpu, tasks_frozen, target);cpus_write_unlock ();arch_smt_update ();return ret;@@ -2659,6 +2675,16 @@ int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)int cpu, ret =0 ;cpu_maps_update_begin (); +if (cpu_hotplug_offline_disabled) { + ret = -EOPNOTSUPP ; + goto out; + } +if (cpu_hotplug_disabled) { + ret = -EBUSY ; + goto out; + } +/* Hold cpus_write_lock() for entire batch operation. */ +cpus_write_lock ();for_each_online_cpu (cpu) {if (topology_is_primary_thread (cpu))continue ;@@ -2668,7 +2694,7 @@ int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)*/ if (ctrlval ==CPU_SMT_ENABLED &&cpu_smt_thread_allowed (cpu))continue ; - ret =cpu_down_maps_locked (cpu,CPUHP_OFFLINE ); + ret =cpu_down_locked (cpu,0 ,CPUHP_OFFLINE );if (ret)break ;/* @@ -2688,6 +2714,9 @@ int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval) }if (!ret) cpu_smt_control = ctrlval; +cpus_write_unlock (); +arch_smt_update (); +out:cpu_maps_update_done ();return ret; }@@ -2705,6 +2734,8 @@ int cpuhp_smt_enable(void)int cpu, ret =0 ;cpu_maps_update_begin (); +/* Hold cpus_write_lock() for entire batch operation. */ +cpus_write_lock (); cpu_smt_control =CPU_SMT_ENABLED ;for_each_present_cpu (cpu) {/* Skip online CPUs and CPUs on offline nodes */ @@ -2712,12 +2743,14 @@ int cpuhp_smt_enable(void)continue ;if (!cpu_smt_thread_allowed (cpu) || !topology_is_core_online (cpu))continue ; - ret =_cpu_up (cpu,0 ,CPUHP_ONLINE ); + ret =cpu_up_locked (cpu,0 ,CPUHP_ONLINE );if (ret)break ;/* See comment in cpuhp_smt_disable() */ cpuhp_online_cpu_device (cpu); } +cpus_write_unlock (); +arch_smt_update ();cpu_maps_update_done ();return ret; }Signature · 1 line
2.53.0
Re: [PATCH v2 1/2] cpuhp: Optimize SMT switch operation by batching lock acquisition
In reply to Vishal ChourasiaQuoted 69 lines · Vishal Chourasia
Bulk CPU hotplug operations, such as an SMT switch operation, requires hotplugging multiple CPUs. The current implementation takes
cpus_write_lock()for each individual CPU, causing multiple slow grace period requests.Introduce
cpu_up_locked()andcpu_down_locked()that assume the caller already holdscpus_write_lock(). Thecpuhp_smt_enable()andcpuhp_smt_disable()functions are updated to hold the lock once around the entire loop, rather than for each individual CPU.Suggested-by Peter Zijlstra <[email protected]>Signed-off-by Vishal Chourasia <[email protected]>--- kernel/cpu.c | 73 ++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 53 insertions(+), 20 deletions(-)kernel/cpu.c +6
-7@@ -1400,8 +1400,8 @@ static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,return ret; } -/* Requires cpu_add_remove_lock to be held */ -static int __ref _cpu_down (unsigned int cpu,int tasks_frozen, +/* Requires cpu_add_remove_lock and cpus_write_lock to be held */ +static int __ref cpu_down_locked (unsigned int cpu,int tasks_frozen,enum cpuhp_state target) {struct cpuhp_cpu_state *st =per_cpu_ptr (&cpuhp_state, cpu);@@ -1413,7 +1413,7 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,if (!cpu_present (cpu))return -EINVAL ; -cpus_write_lock (); +lockdep_assert_cpus_held ();/* * Keep at least one housekeeping cpu onlined to avoid generating @@ -1421,8 +1421,7 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,*/ if (cpumask_any_and (cpu_online_mask,housekeeping_cpumask (HK_TYPE_DOMAIN )) >= nr_cpu_ids) { - ret = -EBUSY ; - goto out; +return -EBUSY ; } cpuhp_tasks_frozen = tasks_frozen;@@ -1440,14 +1439,14 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,* return the error code.. */ if (ret) - goto out; +return ret;/* * We might have stopped still in the range of the AP hotplug * thread. Nothing to do anymore. */ if (st->state >CPUHP_TEARDOWN_CPU ) - goto out; +return 0 ; st->target = target; }@@ -1464,8 +1463,17 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,WARN (1 ,"DEAD callback error for CPU%d" , cpu);
kernel/cpu.c static int __ref _cpu_down(unsigned int cpu, int tasks_frozen, · line 1472
Quoted 10 lines
kernel/cpu.c +6
-1} } +return ret; +} -out: +static int __ref _cpu_down (unsigned int cpu,int tasks_frozen, +enum cpuhp_state target) +{ +
Extra line isn't needed.
Please run checkpatch.pl --strict to fix some of these checks.
Quoted 93 lines
kernel/cpu.c +32
-10+int ret; + +cpus_write_lock (); + ret =cpu_down_locked (cpu, tasks_frozen, target);cpus_write_unlock ();arch_smt_update ();return ret;@@ -1613,18 +1621,18 @@ void cpuhp_online_idle(enum cpuhp_state state)complete_ap_thread (st, true); } -/* Requires cpu_add_remove_lock to be held */ -static int _cpu_up (unsigned int cpu,int tasks_frozen,enum cpuhp_state target) +/* Requires cpu_add_remove_lock and cpus_write_lock to be held. */ +static int cpu_up_locked (unsigned int cpu,int tasks_frozen, +enum cpuhp_state target) {struct cpuhp_cpu_state *st =per_cpu_ptr (&cpuhp_state, cpu);struct task_struct *idle;int ret =0 ; -cpus_write_lock (); +lockdep_assert_cpus_held ();if (!cpu_present (cpu)) { - ret = -EINVAL ; - goto out; +return -EINVAL ; }/* @@ -1632,14 +1640,13 @@ static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)* caller. Nothing to do. */ if (st->state >= target) - goto out; +return 0 ;if (st->state ==CPUHP_OFFLINE ) {/* Let it fail before we try to bring the cpu up */ idle =idle_thread_get (cpu);if (IS_ERR (idle)) { - ret =PTR_ERR (idle); - goto out; +return PTR_ERR (idle); }/* @@ -1663,7 +1670,7 @@ static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)* return the error code.. */ if (ret) - goto out; +return ret; }/* @@ -1673,7 +1680,16 @@ static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)*/ target =min ((int )target,CPUHP_BRINGUP_CPU ); ret =cpuhp_up_callbacks (cpu, st, target); -out: +return ret; +} + +/* Requires cpu_add_remove_lock to be held */ +static int _cpu_up (unsigned int cpu,int tasks_frozen,enum cpuhp_state target) +{ +int ret; + +cpus_write_lock (); + ret =cpu_up_locked (cpu, tasks_frozen, target);cpus_write_unlock ();arch_smt_update ();return ret;@@ -2659,6 +2675,16 @@ int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)int cpu, ret =0 ;cpu_maps_update_begin (); +if (cpu_hotplug_offline_disabled) { + ret = -EOPNOTSUPP ; + goto out; + } +if (cpu_hotplug_disabled) { + ret = -EBUSY ; + goto out; + } +/* Hold cpus_write_lock() for entire batch operation. */ +cpus_write_lock ();for_each_online_cpu (cpu) {if (topology_is_primary_thread (cpu))continue ;
kernel/cpu.c int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval) · line 2671
Quoted 4 lines
kernel/cpu.c +0
-1@@ -2668,7 +2694,7 @@ int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)*/ if (ctrlval ==CPU_SMT_ENABLED &&cpu_smt_thread_allowed (cpu))continue ; - ret =cpu_down_maps_locked (cpu,CPUHP_OFFLINE );
If you see comment in cpu_down_maps_locked, It says.
* Ensure that the control task does not run on the to be offlined
* CPU to prevent a deadlock against cfs_b->period_timer.
* Also keep at least one housekeeping cpu onlined to avoid generating
* an empty sched_domain span
Now, instead of running on the online_CPU you are now doing the _cpu_down functionality in the cpu which is going offline.
How are you preventing that deadlock?
Quoted 39 lines
kernel/cpu.c +9
-1+ ret =cpu_down_locked (cpu,0 ,CPUHP_OFFLINE );if (ret)break ;/* @@ -2688,6 +2714,9 @@ int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval) }if (!ret) cpu_smt_control = ctrlval; +cpus_write_unlock (); +arch_smt_update (); +out:cpu_maps_update_done ();return ret; }@@ -2705,6 +2734,8 @@ int cpuhp_smt_enable(void)int cpu, ret =0 ;cpu_maps_update_begin (); +/* Hold cpus_write_lock() for entire batch operation. */ +cpus_write_lock (); cpu_smt_control =CPU_SMT_ENABLED ;for_each_present_cpu (cpu) {/* Skip online CPUs and CPUs on offline nodes */ @@ -2712,12 +2743,14 @@ int cpuhp_smt_enable(void)continue ;if (!cpu_smt_thread_allowed (cpu) || !topology_is_core_online (cpu))continue ; - ret =_cpu_up (cpu,0 ,CPUHP_ONLINE ); + ret =cpu_up_locked (cpu,0 ,CPUHP_ONLINE );if (ret)break ;/* See comment in cpuhp_smt_disable() */ cpuhp_online_cpu_device (cpu); } +cpus_write_unlock (); +arch_smt_update ();cpu_maps_update_done ();return ret; }
Re: [PATCH v2 1/2] cpuhp: Optimize SMT switch operation by batching lock acquisition
Quoted 15 lines · Shrikanth Hegde
Quoted 6 lines
*/
if (ctrlval ==CPU_SMT_ENABLED&&cpu_smt_thread_allowed(cpu)) continue;
- ret =cpu_down_maps_locked(cpu, CPUHP_OFFLINE);If you see comment in
cpu_down_maps_locked, It says.* Ensure that the control task does not run on the to be offlined
* CPU to prevent a deadlock againstcfs_b->period_timer.
* Also keep at least one housekeeping cpu onlined to avoid
generating
* an emptysched_domainspan
Quoted 4 lines
Now, instead of running on the
online_CPUyou are now doing the_cpu_downfunctionality in the cpu which is going offline.How are you preventing that deadlock?
Ok. I should have looked at updated code. My bad.
This is no longer true after below commit.de715325cc47 cpu: Revert "cpu/hotplug: Prevent self deadlock on CPU hot-unplug"
Quoted 41 lines
Quoted 40 lines
+ ret =
cpu_down_locked(cpu, 0, CPUHP_OFFLINE); if (ret)
break;
/*kernel/cpu.c +3
-0@@ -2688,6 +2714,9 @@ int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval) } if (!ret) cpu_smt_control = ctrlval; + cpus_write_unlock(); + arch_smt_update(); +out: cpu_maps_update_done(); return ret;}
kernel/cpu.c +5
-1@@ -2705,6 +2734,8 @@ int cpuhp_smt_enable(void) int cpu, ret = 0; cpu_maps_update_begin(); + /* Hold cpus_write_lock() for entire batch operation. */ + cpus_write_lock(); cpu_smt_control = CPU_SMT_ENABLED; for_each_present_cpu(cpu) { /* Skip online CPUs and CPUs on offline nodes */@@ -2712,12 +2743,14 @@ int cpuhp_smt_enable(void) continue; if (!cpu_smt_thread_allowed(cpu) || ! topology_is_core_online(cpu)) continue; - ret = _cpu_up(cpu, 0, CPUHP_ONLINE); + ret = cpu_up_locked(cpu, 0, CPUHP_ONLINE); if (ret) break; /* See comment in cpuhp_smt_disable() */ cpuhp_online_cpu_device(cpu); } + cpus_write_unlock(); + arch_smt_update(); cpu_maps_update_done(); return ret; }
PATCH v2 2/2 cpuhp: Expedite RCU grace periods during SMT operations
In reply to Vishal ChourasiaExpedite synchronize_rcu during the SMT mode switch operation when initiated via /sys/devices/system/cpu/smt/control interface
SMT mode switch operation i.e. between SMT 8 to SMT 1 or vice versa and others, are user driven operations and therefore should complete as soon as possible. Switching SMT states involves iterating over a list of CPUs and performing hotplug operations. It was found these transitions took significantly large amount of time to complete particularly on high-core-count systems because system was blocked on synchronize_rcu calls.
Below is one of the call-stacks that accounted for most of the blocking time overhead as reported by offcputime bcc script for CPU offline operation,
finish_task_switch
__schedule
schedule
schedule_timeout
wait_for_completion
__wait_rcu_gp
synchronize_rcu
cpuidle_uninstall_idle_handler
powernv_cpuidle_cpu_dead
cpuhp_invoke_callback
__cpuhp_invoke_callback_range
_cpu_down
cpu_device_down
cpu_subsys_offline
device_offline
online_store
dev_attr_store
sysfs_kf_write
kernfs_fop_write_iter
vfs_write
ksys_write
system_call_exception
system_call_common
- bash (29705)
5771569 ------------------------> Duration (us)--- include/linux/rcupdate.h | 3 +++ kernel/cpu.c | 4 ++++ 2 files changed, 7 insertions(+)
include/linux/rcupdate.h +3 -0
@@ -1190,6 +1190,9 @@ rcu_head_after_call_rcu(struct rcu_head *rhp, rcu_callback_t f)extern int rcu_expedited;extern int rcu_normal; +extern void rcu_expedite_gp (void ); +extern void rcu_unexpedite_gp (void ); + DEFINE_LOCK_GUARD_0(rcu, rcu_read_lock(), rcu_read_unlock())DECLARE_LOCK_GUARD_0_ATTRS (rcu,__acquires_shared (RCU ),__releases_shared (RCU ))
kernel/cpu.c +4 -0
@@ -2683,6 +2683,7 @@ int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval) ret = -EBUSY ; goto out; } +rcu_expedite_gp ();/* Hold cpus_write_lock() for entire batch operation. */ cpus_write_lock ();for_each_online_cpu (cpu) {
@@ -2715,6 +2716,7 @@ int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)if (!ret) cpu_smt_control = ctrlval;cpus_write_unlock (); +rcu_unexpedite_gp ();arch_smt_update (); out:cpu_maps_update_done ();
@@ -2734,6 +2736,7 @@ int cpuhp_smt_enable(void)int cpu, ret =0 ;cpu_maps_update_begin (); +rcu_expedite_gp ();/* Hold cpus_write_lock() for entire batch operation. */ cpus_write_lock (); cpu_smt_control =CPU_SMT_ENABLED ;
@@ -2750,6 +2753,7 @@ int cpuhp_smt_enable(void)cpuhp_online_cpu_device (cpu); }cpus_write_unlock (); +rcu_unexpedite_gp ();arch_smt_update ();cpu_maps_update_done ();return ret;
Signature · 1 line
2.53.0
kernel test robot
build botRe: [PATCH v2 2/2] cpuhp: Expedite RCU grace periods during SMT operations
kernel test robot noticed the following build errors:
Hi Vishal,
kernel test robot noticed the following build errors:
[auto build test ERROR on next-20260213]
[also build test ERROR on linus/master]
[cannot apply to tip/smp/core rcu/rcu/dev v6.19 v6.19-rc8 v6.19-rc7 v6.19] [If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Vishal-Chourasia/cpuhp-Optimize-SMT-switch-operation-by-batching-lock-acquisition/20260216-202247 base: next-20260213 patch link: https://lore.kernel.org/r/20260216121927.489062-6-vishalc%40linux.ibm.com patch subject: [PATCH v2 2/2] cpuhp: Expedite RCU grace periods during SMT operations config: alpha-allnoconfig (https://download.01.org/0day-ci/archive/20260217/[email protected]/config) compiler: alpha-linux-gcc (GCC) 15.2.0 reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20260217/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All errors (new ones prefixed by >>):
In file included from kernel/rcu/update.c:50:
>> kernel/rcu/rcu.h:515:20: error: static declaration of 'rcu_expedite_gp' follows non-static declaration
515 | static inline void rcu_expedite_gp(void) { }
| ^~~~~~~~~~~~~~~
In file included from include/linux/rbtree.h:24,
from include/linux/mm_types.h:11,
from include/linux/mmzone.h:22,
from include/linux/topology.h:34,
from include/linux/irq.h:19,
from include/asm-generic/hardirq.h:17,
from arch/alpha/include/asm/hardirq.h:8,
from include/linux/hardirq.h:11,
from include/linux/interrupt.h:11,
from kernel/rcu/update.c:25:
include/linux/rcupdate.h:1193:13: note: previous declaration of 'rcu_expedite_gp' with type 'void(void)'
1193 | extern void rcu_expedite_gp(void);
| ^~~~~~~~~~~~~~~
>> kernel/rcu/rcu.h:516:20: error: static declaration of 'rcu_unexpedite_gp' follows non-static declaration
516 | static inline void rcu_unexpedite_gp(void) { }
| ^~~~~~~~~~~~~~~~~
include/linux/rcupdate.h:1194:13: note: previous declaration of 'rcu_unexpedite_gp' with type 'void(void)'
1194 | extern void rcu_unexpedite_gp(void);
| ^~~~~~~~~~~~~~~~~vim +/rcu_expedite_gp +515 kernel/rcu/rcu.h
2b34c43cc1671c Paul E. McKenney 2017-03-14 509
25c36329a30c8c Paul E. McKenney 2017-05-03 510 #ifdef CONFIG_TINY_RCU
25c36329a30c8c Paul E. McKenney 2017-05-03 511 /* Tiny RCU doesn't expedite, as its purpose in life is instead to be tiny. */
7414fac050d5e0 Paul E. McKenney 2017-06-12 512 static inline bool rcu_gp_is_normal(void) { return true; }
7414fac050d5e0 Paul E. McKenney 2017-06-12 513 static inline bool rcu_gp_is_expedited(void) { return false; }
6efdda8bec2900 Joel Fernandes (Google 2023-01-12 514) static inline bool rcu_async_should_hurry(void) { return false; }
7414fac050d5e0 Paul E. McKenney 2017-06-12 @515 static inline void rcu_expedite_gp(void) { }
7414fac050d5e0 Paul E. McKenney 2017-06-12 @516 static inline void rcu_unexpedite_gp(void) { }
6efdda8bec2900 Joel Fernandes (Google 2023-01-12 517) static inline void rcu_async_hurry(void) { }
6efdda8bec2900 Joel Fernandes (Google 2023-01-12 518) static inline void rcu_async_relax(void) { }
2be4686d866ad5 Frederic Weisbecker 2023-10-27 519 static inline bool rcu_cpu_online(int cpu) { return true; }
25c36329a30c8c Paul E. McKenney 2017-05-03 520 #else /* #ifdef CONFIG_TINY_RCU */
25c36329a30c8c Paul E. McKenney 2017-05-03 521 bool rcu_gp_is_normal(void); /* Internal RCU use. */
25c36329a30c8c Paul E. McKenney 2017-05-03 522 bool rcu_gp_is_expedited(void); /* Internal RCU use. */
6efdda8bec2900 Joel Fernandes (Google 2023-01-12 523) bool rcu_async_should_hurry(void); /* Internal RCU use. */
25c36329a30c8c Paul E. McKenney 2017-05-03 524 void rcu_expedite_gp(void);
25c36329a30c8c Paul E. McKenney 2017-05-03 525 void rcu_unexpedite_gp(void);
6efdda8bec2900 Joel Fernandes (Google 2023-01-12 526) void rcu_async_hurry(void);
6efdda8bec2900 Joel Fernandes (Google 2023-01-12 527) void rcu_async_relax(void);
25c36329a30c8c Paul E. McKenney 2017-05-03 528 void rcupdate_announce_bootup_oddness(void);
2be4686d866ad5 Frederic Weisbecker 2023-10-27 529 bool rcu_cpu_online(int cpu);
474d0997361c07 Paul E. McKenney 2021-04-20 530 #ifdef CONFIG_TASKS_RCU_GENERIC
e21408ceec2de5 Paul E. McKenney 2020-03-16 531 void show_rcu_tasks_gp_kthreads(void);
e0a34641eb551e Arnd Bergmann 2023-06-09 532 #else /* #ifdef CONFIG_TASKS_RCU_GENERIC */
e0a34641eb551e Arnd Bergmann 2023-06-09 533 static inline void show_rcu_tasks_gp_kthreads(void) {}
e0a34641eb551e Arnd Bergmann 2023-06-09 534 #endif /* #else #ifdef CONFIG_TASKS_RCU_GENERIC */
e0a34641eb551e Arnd Bergmann 2023-06-09 535 #endif /* #else #ifdef CONFIG_TINY_RCU */
e0a34641eb551e Arnd Bergmann 2023-06-09 536 Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki
kernel test robot
build botRe: [PATCH v2 2/2] cpuhp: Expedite RCU grace periods during SMT operations
In reply to Vishal Chourasiakernel test robot noticed the following build errors:
Hi Vishal,
kernel test robot noticed the following build errors:
[auto build test ERROR on next-20260213]
[also build test ERROR on linus/master]
[cannot apply to tip/smp/core rcu/rcu/dev v6.19 v6.19-rc8 v6.19-rc7 v6.19] [If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Vishal-Chourasia/cpuhp-Optimize-SMT-switch-operation-by-batching-lock-acquisition/20260216-202247 base: next-20260213 patch link: https://lore.kernel.org/r/20260216121927.489062-6-vishalc%40linux.ibm.com patch subject: [PATCH v2 2/2] cpuhp: Expedite RCU grace periods during SMT operations config: arm-allnoconfig (https://download.01.org/0day-ci/archive/20260217/[email protected]/config) compiler: clang version 23.0.0git (https://github.com/llvm/llvm-project e86750b29fa0ff207cd43213d66dabe565417638) reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20260217/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All errors (new ones prefixed by >>):
>> ld.lld: error: duplicate symbol: rcu_expedite_gp >>> defined at update.c >>> kernel/rcu/update.o:(rcu_expedite_gp) in archive vmlinux.a >>> defined at srcutiny.c >>> kernel/rcu/srcutiny.o:(.text+0x1) in archive vmlinux.a -- >> ld.lld: error: duplicate symbol: rcu_unexpedite_gp >>> defined at update.c >>> kernel/rcu/update.o:(rcu_unexpedite_gp) in archive vmlinux.a >>> defined at srcutiny.c >>> kernel/rcu/srcutiny.o:(.text+0x41) in archive vmlinux.a -- >> ld.lld: error: duplicate symbol: rcu_expedite_gp >>> defined at update.c >>> kernel/rcu/update.o:(rcu_expedite_gp) in archive vmlinux.a >>> defined at tiny.c >>> kernel/rcu/tiny.o:(.text+0x1) in archive vmlinux.a -- >> ld.lld: error: duplicate symbol: rcu_unexpedite_gp >>> defined at update.c >>> kernel/rcu/update.o:(rcu_unexpedite_gp) in archive vmlinux.a >>> defined at tiny.c >>> kernel/rcu/tiny.o:(.text+0x41) in archive vmlinux.a -- >> ld.lld: error: duplicate symbol: rcu_expedite_gp >>> defined at update.c >>> kernel/rcu/update.o:(rcu_expedite_gp) in archive vmlinux.a >>> defined at slab_common.c >>> mm/slab_common.o:(.text+0x1) in archive vmlinux.a -- >> ld.lld: error: duplicate symbol: rcu_unexpedite_gp >>> defined at update.c >>> kernel/rcu/update.o:(rcu_unexpedite_gp) in archive vmlinux.a >>> defined at slab_common.c >>> mm/slab_common.o:(.text+0x41) in archive vmlinux.a
Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki
Re: [PATCH v2 2/2] cpuhp: Expedite RCU grace periods during SMT operations
In reply to Vishal ChourasiaQuoted 52 lines · Vishal Chourasia
Expedite
synchronize_rcuduring the SMT mode switch operation when initiated via/sys/devices/system/cpu/smt/controlinterfaceSMT mode switch operation i.e. between SMT 8 to SMT 1 or vice versa and others, are user driven operations and therefore should complete as soon as possible. Switching SMT states involves iterating over a list of CPUs and performing hotplug operations. It was found these transitions took significantly large amount of time to complete particularly on high-core-count systems because system was blocked on
synchronize_rcucalls.Below is one of the call-stacks that accounted for most of the blocking time overhead as reported by offcputime bcc script for CPU offline operation,
finish_task_switch __schedule schedule schedule_timeout wait_for_completion __wait_rcu_gp synchronize_rcu cpuidle_uninstall_idle_handler powernv_cpuidle_cpu_dead cpuhp_invoke_callback __cpuhp_invoke_callback_range _cpu_down cpu_device_down cpu_subsys_offline device_offline online_store dev_attr_store sysfs_kf_write kernfs_fop_write_iter vfs_write ksys_write system_call_exception system_call_common - bash (29705) 5771569 ------------------------> Duration (us)Signed-off-by Vishal Chourasia <[email protected]>--- include/linux/rcupdate.h | 3 +++ kernel/cpu.c | 4 ++++ 2 files changed, 7 insertions(+)include/linux/rcupdate.h +0
-0
include/linux/rcupdate.h rcu_head_after_call_rcu(struct rcu_head *rhp, rcu_callback_t f) · line 1198
Quoted 9 lines
include/linux/rcupdate.h +3
-0@@ -1190,6 +1190,9 @@ rcu_head_after_call_rcu(struct rcu_head *rhp, rcu_callback_t f)extern int rcu_expedited;extern int rcu_normal; +extern void rcu_expedite_gp (void ); +extern void rcu_unexpedite_gp (void ); + DEFINE_LOCK_GUARD_0(rcu, rcu_read_lock(), rcu_read_unlock())DECLARE_LOCK_GUARD_0_ATTRS (rcu,__acquires_shared (RCU ),__releases_shared (RCU ))
IMHO, below maybe better.
include/linux/rcupdate.h +7 -2
@@ -1190,8 +1190,13 @@ rcu_head_after_call_rcu(struct rcu_head *rhp, rcu_callback_t f)extern int rcu_expedited;extern int rcu_normal; -extern void rcu_expedite_gp (void ); -extern void rcu_unexpedite_gp (void ); +#ifdef CONFIG_TINY_RCU +static inline void rcu_expedite_gp (void ) { } +static inline void rcu_unexpedite_gp (void ) { } +#else +void rcu_expedite_gp (void ); +void rcu_unexpedite_gp (void ); +#endif DEFINE_LOCK_GUARD_0(rcu, rcu_read_lock(), rcu_read_unlock()) DECLARE_LOCK_GUARD_0_ATTRS(rcu, __acquires_shared(RCU),
__releases_shared(RCU))
kernel/rcu/rcu.h +0 -2
@@ -512,8 +512,6 @@ do { \static inline bool rcu_gp_is_normal (void ) {return true; }static inline bool rcu_gp_is_expedited (void ) {return false; }static inline bool rcu_async_should_hurry (void ) {return false; } -static inline void rcu_expedite_gp (void ) { } -static inline void rcu_unexpedite_gp (void ) { }static inline void rcu_async_hurry (void ) { }static inline void rcu_async_relax (void ) { }
static inline bool rcu_cpu_online(int cpu) { return true; }
@@ -521,8 +519,6 @@ static inline bool rcu_cpu_online(int cpu) { return true; }
bool rcu_gp_is_normal(void); /* Internal RCU use. */
bool rcu_gp_is_expedited(void); /* Internal RCU use. */
bool rcu_async_should_hurry(void); /* Internal RCU use. */
-void rcu_expedite_gp(void);
-void rcu_unexpedite_gp(void);
void rcu_async_hurry(void);
void rcu_async_relax(void);
void rcupdate_announce_bootup_oddness(void);