Skip to contentSkip to conversation

cpuhp: Improve SMT switch time via lock batching and RCU expedition

PATCH Superseded Vishal Chourasia 3 participants 9 messages Feb 16

Subsystems (2)

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

v2 cover cpuhp: Improve SMT switch time via lock batching and RCU expedition

On large systems with high core counts, toggling SMT modes via sysfs (/sys/devices/system/cpu/smt/control) incurs significant latency. For instance, on ~2000 CPUs, switching SMT modes can take close to an hour as the system hotplugs each hardware thread individually. This series reduces this time to minutes.

Analysis of the hotplug path [1] identifies synchronize_rcu() as the primary bottleneck. During a bulk SMT switch, the kernel repeatedly enters RCU grace periods for each CPU being brought online or offline.

This series optimizes the SMT transition in two ways:

1. Lock Batching [1]: Instead of repeatedly acquiring and releasing the CPU hotplug write lock for every individual CPU, we now hold cpus_write_lock across the entire SMT toggle operation.

2. Expedite RCU grace period [2]: It utilizes rcu_expedite_gp() to force expedite grace periods specifically for the duration of the SMT switch. The trade-off is justified here to prevent the administrative task of SMT switching from stalling for an unacceptable duration on large systems.

v1: https://lore.kernel.org/all/[email protected]/

[1] https://lore.kernel.org/all/5f2ab8a44d685701fe36cdaa8042a1aef215d10d.camel@linux.vnet.ibm.com
[2] https://lore.kernel.org/all/[email protected]/

Vishal Chourasia (2):
cpuhp: Optimize SMT switch operation by batching lock acquisition
cpuhp: Expedite RCU grace periods during SMT operations

 include/linux/rcupdate.h |  3 ++
 kernel/cpu.c             | 77 +++++++++++++++++++++++++++++-----------
 2 files changed, 60 insertions(+), 20 deletions(-)
Signature · 1 line
2.53.0
#1 lore raw

PATCH v2 1/2 cpuhp: Optimize SMT switch operation by batching lock acquisition

Bulk CPU hotplug operations, such as an SMT switch operation, requires hotplugging multiple CPUs. The current implementation takes cpus_write_lock() for each individual CPU, causing multiple slow grace period requests.

Introduce cpu_up_locked() and cpu_down_locked() that assume the caller already holds cpus_write_lock(). The cpuhp_smt_enable() and cpuhp_smt_disable() functions are updated to hold the lock once around the entire loop, rather than for each individual CPU.

---
 kernel/cpu.c | 73 ++++++++++++++++++++++++++++++++++++++--------------
 1 file changed, 53 insertions(+), 20 deletions(-)
kernel/cpu.c +53 -20
index 01968a5c4a16..edaa37419036 100644
@@ -1400,8 +1400,8 @@ static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
 	return ret;
 }
 
-/* Requires cpu_add_remove_lock to be held */
-static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
+/* Requires cpu_add_remove_lock and cpus_write_lock to be held */
+static int __ref cpu_down_locked(unsigned int cpu, int tasks_frozen,
 			   enum cpuhp_state target)
 {
 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
@@ -1413,7 +1413,7 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
 	if (!cpu_present(cpu))
 		return -EINVAL;
 
-	cpus_write_lock();
+	lockdep_assert_cpus_held();
 
 	/*
 	 * Keep at least one housekeeping cpu onlined to avoid generating
@@ -1421,8 +1421,7 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
 	 */
 	if (cpumask_any_and(cpu_online_mask,
 			    housekeeping_cpumask(HK_TYPE_DOMAIN)) >= nr_cpu_ids) {
-		ret = -EBUSY;
-		goto out;
+		return -EBUSY;
 	}
 
 	cpuhp_tasks_frozen = tasks_frozen;
@@ -1440,14 +1439,14 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
 		 * return the error code..
 		 */
 		if (ret)
-			goto out;
+			return ret;
 
 		/*
 		 * We might have stopped still in the range of the AP hotplug
 		 * thread. Nothing to do anymore.
 		 */
 		if (st->state > CPUHP_TEARDOWN_CPU)
-			goto out;
+			return 0;
 
 		st->target = target;
 	}
@@ -1464,8 +1463,17 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
 			WARN(1, "DEAD callback error for CPU%d", cpu);
 		}
 	}
+	return ret;
+}
 
-out:
+static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
+			   enum cpuhp_state target)
+{
+
+	int ret;
+
+	cpus_write_lock();
+	ret = cpu_down_locked(cpu, tasks_frozen, target);
 	cpus_write_unlock();
 	arch_smt_update();
 	return ret;
@@ -1613,18 +1621,18 @@ void cpuhp_online_idle(enum cpuhp_state state)
 	complete_ap_thread(st, true);
 }
 
-/* Requires cpu_add_remove_lock to be held */
-static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
+/* Requires cpu_add_remove_lock and cpus_write_lock to be held. */
+static int cpu_up_locked(unsigned int cpu, int tasks_frozen,
+			 enum cpuhp_state target)
 {
 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 	struct task_struct *idle;
 	int ret = 0;
 
-	cpus_write_lock();
+	lockdep_assert_cpus_held();
 
 	if (!cpu_present(cpu)) {
-		ret = -EINVAL;
-		goto out;
+		return -EINVAL;
 	}
 
 	/*
@@ -1632,14 +1640,13 @@ static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
 	 * caller. Nothing to do.
 	 */
 	if (st->state >= target)
-		goto out;
+		return 0;
 
 	if (st->state == CPUHP_OFFLINE) {
 		/* Let it fail before we try to bring the cpu up */
 		idle = idle_thread_get(cpu);
 		if (IS_ERR(idle)) {
-			ret = PTR_ERR(idle);
-			goto out;
+			return PTR_ERR(idle);
 		}
 
 		/*
@@ -1663,7 +1670,7 @@ static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
 		 * return the error code..
 		 */
 		if (ret)
-			goto out;
+			return ret;
 	}
 
 	/*
@@ -1673,7 +1680,16 @@ static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
 	 */
 	target = min((int)target, CPUHP_BRINGUP_CPU);
 	ret = cpuhp_up_callbacks(cpu, st, target);
-out:
+	return ret;
+}
+
+/* Requires cpu_add_remove_lock to be held */
+static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
+{
+	int ret;
+
+	cpus_write_lock();
+	ret = cpu_up_locked(cpu, tasks_frozen, target);
 	cpus_write_unlock();
 	arch_smt_update();
 	return ret;
@@ -2659,6 +2675,16 @@ int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)
 	int cpu, ret = 0;
 
 	cpu_maps_update_begin();
+	if (cpu_hotplug_offline_disabled) {
+		ret = -EOPNOTSUPP;
+		goto out;
+	}
+	if (cpu_hotplug_disabled) {
+		ret = -EBUSY;
+		goto out;
+	}
+	/* Hold cpus_write_lock() for entire batch operation. */
+	cpus_write_lock();
 	for_each_online_cpu(cpu) {
 		if (topology_is_primary_thread(cpu))
 			continue;
@@ -2668,7 +2694,7 @@ int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)
 		 */
 		if (ctrlval == CPU_SMT_ENABLED && cpu_smt_thread_allowed(cpu))
 			continue;
-		ret = cpu_down_maps_locked(cpu, CPUHP_OFFLINE);
+		ret = cpu_down_locked(cpu, 0, CPUHP_OFFLINE);
 		if (ret)
 			break;
 		/*
@@ -2688,6 +2714,9 @@ int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)
 	}
 	if (!ret)
 		cpu_smt_control = ctrlval;
+	cpus_write_unlock();
+	arch_smt_update();
+out:
 	cpu_maps_update_done();
 	return ret;
 }
@@ -2705,6 +2734,8 @@ int cpuhp_smt_enable(void)
 	int cpu, ret = 0;
 
 	cpu_maps_update_begin();
+	/* Hold cpus_write_lock() for entire batch operation. */
+	cpus_write_lock();
 	cpu_smt_control = CPU_SMT_ENABLED;
 	for_each_present_cpu(cpu) {
 		/* Skip online CPUs and CPUs on offline nodes */
@@ -2712,12 +2743,14 @@ int cpuhp_smt_enable(void)
 			continue;
 		if (!cpu_smt_thread_allowed(cpu) || !topology_is_core_online(cpu))
 			continue;
-		ret = _cpu_up(cpu, 0, CPUHP_ONLINE);
+		ret = cpu_up_locked(cpu, 0, CPUHP_ONLINE);
 		if (ret)
 			break;
 		/* See comment in cpuhp_smt_disable() */
 		cpuhp_online_cpu_device(cpu);
 	}
+	cpus_write_unlock();
+	arch_smt_update();
 	cpu_maps_update_done();
 	return ret;
 }
Signature · 1 line
2.53.0
#2 lore raw

Re: [PATCH v2 1/2] cpuhp: Optimize SMT switch operation by batching lock acquisition

Quoted 11 lines · Vishal Chourasia

Bulk CPU hotplug operations, such as an SMT switch operation, requires hotplugging multiple CPUs. The current implementation takes cpus_write_lock() for each individual CPU, causing multiple slow grace period requests.

Introduce cpu_up_locked() and cpu_down_locked() that assume the caller already holds cpus_write_lock(). The cpuhp_smt_enable() and cpuhp_smt_disable() functions are updated to hold the lock once around the entire loop, rather than for each individual CPU.

The code hositing the cpus_write_lock up in the cpuhp_smt_enable() was provided by Joel [1]. Thanks Joel.

I missed adding an appropriate tag for it.

[1] https://lore.kernel.org/all/20260119051835.GA696111@joelbox2/

Quoted 207 lines
---
 kernel/cpu.c | 73 ++++++++++++++++++++++++++++++++++++++--------------
 1 file changed, 53 insertions(+), 20 deletions(-)
kernel/cpu.c +53 -20
index 01968a5c4a16..edaa37419036 100644
@@ -1400,8 +1400,8 @@ static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
 	return ret;
 }
 
-/* Requires cpu_add_remove_lock to be held */
-static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
+/* Requires cpu_add_remove_lock and cpus_write_lock to be held */
+static int __ref cpu_down_locked(unsigned int cpu, int tasks_frozen,
 			   enum cpuhp_state target)
 {
 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
@@ -1413,7 +1413,7 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
 	if (!cpu_present(cpu))
 		return -EINVAL;
 
-	cpus_write_lock();
+	lockdep_assert_cpus_held();
 
 	/*
 	 * Keep at least one housekeeping cpu onlined to avoid generating
@@ -1421,8 +1421,7 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
 	 */
 	if (cpumask_any_and(cpu_online_mask,
 			    housekeeping_cpumask(HK_TYPE_DOMAIN)) >= nr_cpu_ids) {
-		ret = -EBUSY;
-		goto out;
+		return -EBUSY;
 	}
 
 	cpuhp_tasks_frozen = tasks_frozen;
@@ -1440,14 +1439,14 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
 		 * return the error code..
 		 */
 		if (ret)
-			goto out;
+			return ret;
 
 		/*
 		 * We might have stopped still in the range of the AP hotplug
 		 * thread. Nothing to do anymore.
 		 */
 		if (st->state > CPUHP_TEARDOWN_CPU)
-			goto out;
+			return 0;
 
 		st->target = target;
 	}
@@ -1464,8 +1463,17 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
 			WARN(1, "DEAD callback error for CPU%d", cpu);
 		}
 	}
+	return ret;
+}
 
-out:
+static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
+			   enum cpuhp_state target)
+{
+
+	int ret;
+
+	cpus_write_lock();
+	ret = cpu_down_locked(cpu, tasks_frozen, target);
 	cpus_write_unlock();
 	arch_smt_update();
 	return ret;
@@ -1613,18 +1621,18 @@ void cpuhp_online_idle(enum cpuhp_state state)
 	complete_ap_thread(st, true);
 }
 
-/* Requires cpu_add_remove_lock to be held */
-static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
+/* Requires cpu_add_remove_lock and cpus_write_lock to be held. */
+static int cpu_up_locked(unsigned int cpu, int tasks_frozen,
+			 enum cpuhp_state target)
 {
 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 	struct task_struct *idle;
 	int ret = 0;
 
-	cpus_write_lock();
+	lockdep_assert_cpus_held();
 
 	if (!cpu_present(cpu)) {
-		ret = -EINVAL;
-		goto out;
+		return -EINVAL;
 	}
 
 	/*
@@ -1632,14 +1640,13 @@ static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
 	 * caller. Nothing to do.
 	 */
 	if (st->state >= target)
-		goto out;
+		return 0;
 
 	if (st->state == CPUHP_OFFLINE) {
 		/* Let it fail before we try to bring the cpu up */
 		idle = idle_thread_get(cpu);
 		if (IS_ERR(idle)) {
-			ret = PTR_ERR(idle);
-			goto out;
+			return PTR_ERR(idle);
 		}
 
 		/*
@@ -1663,7 +1670,7 @@ static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
 		 * return the error code..
 		 */
 		if (ret)
-			goto out;
+			return ret;
 	}
 
 	/*
@@ -1673,7 +1680,16 @@ static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
 	 */
 	target = min((int)target, CPUHP_BRINGUP_CPU);
 	ret = cpuhp_up_callbacks(cpu, st, target);
-out:
+	return ret;
+}
+
+/* Requires cpu_add_remove_lock to be held */
+static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
+{
+	int ret;
+
+	cpus_write_lock();
+	ret = cpu_up_locked(cpu, tasks_frozen, target);
 	cpus_write_unlock();
 	arch_smt_update();
 	return ret;
@@ -2659,6 +2675,16 @@ int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)
 	int cpu, ret = 0;
 
 	cpu_maps_update_begin();
+	if (cpu_hotplug_offline_disabled) {
+		ret = -EOPNOTSUPP;
+		goto out;
+	}
+	if (cpu_hotplug_disabled) {
+		ret = -EBUSY;
+		goto out;
+	}
+	/* Hold cpus_write_lock() for entire batch operation. */
+	cpus_write_lock();
 	for_each_online_cpu(cpu) {
 		if (topology_is_primary_thread(cpu))
 			continue;
@@ -2668,7 +2694,7 @@ int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)
 		 */
 		if (ctrlval == CPU_SMT_ENABLED && cpu_smt_thread_allowed(cpu))
 			continue;
-		ret = cpu_down_maps_locked(cpu, CPUHP_OFFLINE);
+		ret = cpu_down_locked(cpu, 0, CPUHP_OFFLINE);
 		if (ret)
 			break;
 		/*
@@ -2688,6 +2714,9 @@ int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)
 	}
 	if (!ret)
 		cpu_smt_control = ctrlval;
+	cpus_write_unlock();
+	arch_smt_update();
+out:
 	cpu_maps_update_done();
 	return ret;
 }
@@ -2705,6 +2734,8 @@ int cpuhp_smt_enable(void)
 	int cpu, ret = 0;
 
 	cpu_maps_update_begin();
+	/* Hold cpus_write_lock() for entire batch operation. */
+	cpus_write_lock();
 	cpu_smt_control = CPU_SMT_ENABLED;
 	for_each_present_cpu(cpu) {
 		/* Skip online CPUs and CPUs on offline nodes */
@@ -2712,12 +2743,14 @@ int cpuhp_smt_enable(void)
 			continue;
 		if (!cpu_smt_thread_allowed(cpu) || !topology_is_core_online(cpu))
 			continue;
-		ret = _cpu_up(cpu, 0, CPUHP_ONLINE);
+		ret = cpu_up_locked(cpu, 0, CPUHP_ONLINE);
 		if (ret)
 			break;
 		/* See comment in cpuhp_smt_disable() */
 		cpuhp_online_cpu_device(cpu);
 	}
+	cpus_write_unlock();
+	arch_smt_update();
 	cpu_maps_update_done();
 	return ret;
 }
Signature · 1 line
2.53.0
#3 lore raw

Re: [PATCH v2 1/2] cpuhp: Optimize SMT switch operation by batching lock acquisition

In reply to Vishal Chourasia
Quoted 69 lines · Vishal Chourasia

Bulk CPU hotplug operations, such as an SMT switch operation, requires hotplugging multiple CPUs. The current implementation takes cpus_write_lock() for each individual CPU, causing multiple slow grace period requests.

Introduce cpu_up_locked() and cpu_down_locked() that assume the caller already holds cpus_write_lock(). The cpuhp_smt_enable() and cpuhp_smt_disable() functions are updated to hold the lock once around the entire loop, rather than for each individual CPU.

---
  kernel/cpu.c | 73 ++++++++++++++++++++++++++++++++++++++--------------
  1 file changed, 53 insertions(+), 20 deletions(-)
kernel/cpu.c +6 -7
index 01968a5c4a16..edaa37419036 100644
@@ -1400,8 +1400,8 @@ static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
  	return ret;
  }
 
-/* Requires cpu_add_remove_lock to be held */
-static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
+/* Requires cpu_add_remove_lock and cpus_write_lock to be held */
+static int __ref cpu_down_locked(unsigned int cpu, int tasks_frozen,
  			   enum cpuhp_state target)
  {
  	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
@@ -1413,7 +1413,7 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
  	if (!cpu_present(cpu))
  		return -EINVAL;
 
-	cpus_write_lock();
+	lockdep_assert_cpus_held();
 
  	/*
  	 * Keep at least one housekeeping cpu onlined to avoid generating
@@ -1421,8 +1421,7 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
  	 */
  	if (cpumask_any_and(cpu_online_mask,
  			    housekeeping_cpumask(HK_TYPE_DOMAIN)) >= nr_cpu_ids) {
-		ret = -EBUSY;
-		goto out;
+		return -EBUSY;
  	}
 
  	cpuhp_tasks_frozen = tasks_frozen;
@@ -1440,14 +1439,14 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
  		 * return the error code..
  		 */
  		if (ret)
-			goto out;
+			return ret;
 
  		/*
  		 * We might have stopped still in the range of the AP hotplug
  		 * thread. Nothing to do anymore.
  		 */
  		if (st->state > CPUHP_TEARDOWN_CPU)
-			goto out;
+			return 0;
 
  		st->target = target;
  	}
@@ -1464,8 +1463,17 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
  			WARN(1, "DEAD callback error for CPU%d", cpu);

kernel/cpu.c static int __ref _cpu_down(unsigned int cpu, int tasks_frozen, · line 1472

Quoted 10 lines
kernel/cpu.c +6 -1
  		}
  	}
+	return ret;
+}
 
-out:
+static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
+			   enum cpuhp_state target)
+{
+

Extra line isn't needed.
Please run checkpatch.pl --strict to fix some of these checks.

Quoted 93 lines
kernel/cpu.c +32 -10
+	int ret;
+
+	cpus_write_lock();
+	ret = cpu_down_locked(cpu, tasks_frozen, target);
  	cpus_write_unlock();
  	arch_smt_update();
  	return ret;
@@ -1613,18 +1621,18 @@ void cpuhp_online_idle(enum cpuhp_state state)
  	complete_ap_thread(st, true);
  }
 
-/* Requires cpu_add_remove_lock to be held */
-static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
+/* Requires cpu_add_remove_lock and cpus_write_lock to be held. */
+static int cpu_up_locked(unsigned int cpu, int tasks_frozen,
+			 enum cpuhp_state target)
  {
  	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
  	struct task_struct *idle;
  	int ret = 0;
 
-	cpus_write_lock();
+	lockdep_assert_cpus_held();
 
  	if (!cpu_present(cpu)) {
-		ret = -EINVAL;
-		goto out;
+		return -EINVAL;
  	}
 
  	/*
@@ -1632,14 +1640,13 @@ static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
  	 * caller. Nothing to do.
  	 */
  	if (st->state >= target)
-		goto out;
+		return 0;
 
  	if (st->state == CPUHP_OFFLINE) {
  		/* Let it fail before we try to bring the cpu up */
  		idle = idle_thread_get(cpu);
  		if (IS_ERR(idle)) {
-			ret = PTR_ERR(idle);
-			goto out;
+			return PTR_ERR(idle);
  		}
 
  		/*
@@ -1663,7 +1670,7 @@ static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
  		 * return the error code..
  		 */
  		if (ret)
-			goto out;
+			return ret;
  	}
 
  	/*
@@ -1673,7 +1680,16 @@ static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
  	 */
  	target = min((int)target, CPUHP_BRINGUP_CPU);
  	ret = cpuhp_up_callbacks(cpu, st, target);
-out:
+	return ret;
+}
+
+/* Requires cpu_add_remove_lock to be held */
+static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
+{
+	int ret;
+
+	cpus_write_lock();
+	ret = cpu_up_locked(cpu, tasks_frozen, target);
  	cpus_write_unlock();
  	arch_smt_update();
  	return ret;
@@ -2659,6 +2675,16 @@ int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)
  	int cpu, ret = 0;
 
  	cpu_maps_update_begin();
+	if (cpu_hotplug_offline_disabled) {
+		ret = -EOPNOTSUPP;
+		goto out;
+	}
+	if (cpu_hotplug_disabled) {
+		ret = -EBUSY;
+		goto out;
+	}
+	/* Hold cpus_write_lock() for entire batch operation. */
+	cpus_write_lock();
  	for_each_online_cpu(cpu) {
  		if (topology_is_primary_thread(cpu))
  			continue;

kernel/cpu.c int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval) · line 2671

Quoted 4 lines
kernel/cpu.c +0 -1
@@ -2668,7 +2694,7 @@ int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)
  		 */
  		if (ctrlval == CPU_SMT_ENABLED && cpu_smt_thread_allowed(cpu))
  			continue;
-		ret = cpu_down_maps_locked(cpu, CPUHP_OFFLINE);

If you see comment in cpu_down_maps_locked, It says.

* Ensure that the control task does not run on the to be offlined
* CPU to prevent a deadlock against cfs_b->period_timer.
* Also keep at least one housekeeping cpu onlined to avoid generating
* an empty sched_domain span

Now, instead of running on the online_CPU you are now doing the _cpu_down functionality in the cpu which is going offline.

How are you preventing that deadlock?

Quoted 39 lines
kernel/cpu.c +9 -1
+		ret = cpu_down_locked(cpu, 0, CPUHP_OFFLINE);
  		if (ret)
  			break;
  		/*
@@ -2688,6 +2714,9 @@ int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)
  	}
  	if (!ret)
  		cpu_smt_control = ctrlval;
+	cpus_write_unlock();
+	arch_smt_update();
+out:
  	cpu_maps_update_done();
  	return ret;
  }
@@ -2705,6 +2734,8 @@ int cpuhp_smt_enable(void)
  	int cpu, ret = 0;
 
  	cpu_maps_update_begin();
+	/* Hold cpus_write_lock() for entire batch operation. */
+	cpus_write_lock();
  	cpu_smt_control = CPU_SMT_ENABLED;
  	for_each_present_cpu(cpu) {
  		/* Skip online CPUs and CPUs on offline nodes */
@@ -2712,12 +2743,14 @@ int cpuhp_smt_enable(void)
  			continue;
  		if (!cpu_smt_thread_allowed(cpu) || !topology_is_core_online(cpu))
  			continue;
-		ret = _cpu_up(cpu, 0, CPUHP_ONLINE);
+		ret = cpu_up_locked(cpu, 0, CPUHP_ONLINE);
  		if (ret)
  			break;
  		/* See comment in cpuhp_smt_disable() */
  		cpuhp_online_cpu_device(cpu);
  	}
+	cpus_write_unlock();
+	arch_smt_update();
  	cpu_maps_update_done();
  	return ret;
  }
#4 lore raw

Re: [PATCH v2 1/2] cpuhp: Optimize SMT switch operation by batching lock acquisition

Quoted 15 lines · Shrikanth Hegde
Quoted 6 lines

           */
          if (ctrlval == CPU_SMT_ENABLED && cpu_smt_thread_allowed(cpu)) continue;
-        ret = cpu_down_maps_locked(cpu, CPUHP_OFFLINE);

If you see comment in cpu_down_maps_locked, It says.

         * Ensure that the control task does not run on the to be offlined
         * CPU to prevent a deadlock against cfs_b->period_timer.
         * Also keep at least one housekeeping cpu onlined to avoid
generating
         * an empty sched_domain span

Quoted 4 lines

Now, instead of running on the online_CPU you are now doing the _cpu_down functionality in the cpu which is going offline.

How are you preventing that deadlock?

Ok. I should have looked at updated code. My bad.

This is no longer true after below commit.
de715325cc47 cpu: Revert "cpu/hotplug: Prevent self deadlock on CPU hot-unplug"

Quoted 41 lines
Quoted 40 lines

+        ret = cpu_down_locked(cpu, 0, CPUHP_OFFLINE); if (ret)
              break;
          /*

kernel/cpu.c +3 -0
@@ -2688,6 +2714,9 @@ int cpuhp_smt_disable(enum cpuhp_smt_control 
 ctrlval)
       }
       if (!ret)
           cpu_smt_control = ctrlval;
+    cpus_write_unlock();
+    arch_smt_update();
+out:
       cpu_maps_update_done();
       return ret;

  }

kernel/cpu.c +5 -1
@@ -2705,6 +2734,8 @@ int cpuhp_smt_enable(void)
       int cpu, ret = 0;
       cpu_maps_update_begin();
+    /* Hold cpus_write_lock() for entire batch operation. */
+    cpus_write_lock();
       cpu_smt_control = CPU_SMT_ENABLED;
       for_each_present_cpu(cpu) {
           /* Skip online CPUs and CPUs on offline nodes */
@@ -2712,12 +2743,14 @@ int cpuhp_smt_enable(void)
               continue;
           if (!cpu_smt_thread_allowed(cpu) || ! topology_is_core_online(cpu))
               continue;
-        ret = _cpu_up(cpu, 0, CPUHP_ONLINE);
+        ret = cpu_up_locked(cpu, 0, CPUHP_ONLINE);
           if (ret)
               break;
           /* See comment in cpuhp_smt_disable() */
           cpuhp_online_cpu_device(cpu);
       }
+    cpus_write_unlock();
+    arch_smt_update();
       cpu_maps_update_done();
       return ret;
   }
#5 lore raw

PATCH v2 2/2 cpuhp: Expedite RCU grace periods during SMT operations

In reply to Vishal Chourasia

Expedite synchronize_rcu during the SMT mode switch operation when initiated via /sys/devices/system/cpu/smt/control interface

SMT mode switch operation i.e. between SMT 8 to SMT 1 or vice versa and others, are user driven operations and therefore should complete as soon as possible. Switching SMT states involves iterating over a list of CPUs and performing hotplug operations. It was found these transitions took significantly large amount of time to complete particularly on high-core-count systems because system was blocked on synchronize_rcu calls.

Below is one of the call-stacks that accounted for most of the blocking time overhead as reported by offcputime bcc script for CPU offline operation,

    finish_task_switch
    __schedule
    schedule
    schedule_timeout
    wait_for_completion
    __wait_rcu_gp
    synchronize_rcu
    cpuidle_uninstall_idle_handler
    powernv_cpuidle_cpu_dead
    cpuhp_invoke_callback
    __cpuhp_invoke_callback_range
    _cpu_down
    cpu_device_down
    cpu_subsys_offline
    device_offline
    online_store
    dev_attr_store
    sysfs_kf_write
    kernfs_fop_write_iter
    vfs_write
    ksys_write
    system_call_exception
    system_call_common
   -                bash (29705)
        5771569  ------------------------>  Duration (us)
---
 include/linux/rcupdate.h | 3 +++
 kernel/cpu.c             | 4 ++++
 2 files changed, 7 insertions(+)
include/linux/rcupdate.h +3 -0
index 7729fef249e1..f12d0d0f008d 100644
@@ -1190,6 +1190,9 @@ rcu_head_after_call_rcu(struct rcu_head *rhp, rcu_callback_t f)
 extern int rcu_expedited;
 extern int rcu_normal;
 
+extern void rcu_expedite_gp(void);
+extern void rcu_unexpedite_gp(void);
+
 DEFINE_LOCK_GUARD_0(rcu, rcu_read_lock(), rcu_read_unlock())
 DECLARE_LOCK_GUARD_0_ATTRS(rcu, __acquires_shared(RCU), __releases_shared(RCU))
 
kernel/cpu.c +4 -0
index edaa37419036..f5517d64d3f3 100644
@@ -2683,6 +2683,7 @@ int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)
 		ret = -EBUSY;
 		goto out;
 	}
+	rcu_expedite_gp();
 	/* Hold cpus_write_lock() for entire batch operation. */
 	cpus_write_lock();
 	for_each_online_cpu(cpu) {
@@ -2715,6 +2716,7 @@ int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)
 	if (!ret)
 		cpu_smt_control = ctrlval;
 	cpus_write_unlock();
+	rcu_unexpedite_gp();
 	arch_smt_update();
 out:
 	cpu_maps_update_done();
@@ -2734,6 +2736,7 @@ int cpuhp_smt_enable(void)
 	int cpu, ret = 0;
 
 	cpu_maps_update_begin();
+	rcu_expedite_gp();
 	/* Hold cpus_write_lock() for entire batch operation. */
 	cpus_write_lock();
 	cpu_smt_control = CPU_SMT_ENABLED;
@@ -2750,6 +2753,7 @@ int cpuhp_smt_enable(void)
 		cpuhp_online_cpu_device(cpu);
 	}
 	cpus_write_unlock();
+	rcu_unexpedite_gp();
 	arch_smt_update();
 	cpu_maps_update_done();
 	return ret;
Signature · 1 line
2.53.0
#6 lore raw

Re: [PATCH v2 2/2] cpuhp: Expedite RCU grace periods during SMT operations

kernel test robot noticed the following build errors:

Hi Vishal,

kernel test robot noticed the following build errors:

[auto build test ERROR on next-20260213]
[also build test ERROR on linus/master]
[cannot apply to tip/smp/core rcu/rcu/dev v6.19 v6.19-rc8 v6.19-rc7 v6.19] [If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]

url:    https://github.com/intel-lab-lkp/linux/commits/Vishal-Chourasia/cpuhp-Optimize-SMT-switch-operation-by-batching-lock-acquisition/20260216-202247
base:   next-20260213
patch link:    https://lore.kernel.org/r/20260216121927.489062-6-vishalc%40linux.ibm.com
patch subject: [PATCH v2 2/2] cpuhp: Expedite RCU grace periods during SMT operations
config: alpha-allnoconfig (https://download.01.org/0day-ci/archive/20260217/[email protected]/config)
compiler: alpha-linux-gcc (GCC) 15.2.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20260217/[email protected]/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/

All errors (new ones prefixed by >>):

In file included from kernel/rcu/update.c:50:

>> kernel/rcu/rcu.h:515:20: error: static declaration of 'rcu_expedite_gp' follows non-static declaration
     515 | static inline void rcu_expedite_gp(void) { }
         |                    ^~~~~~~~~~~~~~~
   In file included from include/linux/rbtree.h:24,
                    from include/linux/mm_types.h:11,
                    from include/linux/mmzone.h:22,
                    from include/linux/topology.h:34,
                    from include/linux/irq.h:19,
                    from include/asm-generic/hardirq.h:17,
                    from arch/alpha/include/asm/hardirq.h:8,
                    from include/linux/hardirq.h:11,
                    from include/linux/interrupt.h:11,
                    from kernel/rcu/update.c:25:
   include/linux/rcupdate.h:1193:13: note: previous declaration of 'rcu_expedite_gp' with type 'void(void)'
    1193 | extern void rcu_expedite_gp(void);
         |             ^~~~~~~~~~~~~~~
>> kernel/rcu/rcu.h:516:20: error: static declaration of 'rcu_unexpedite_gp' follows non-static declaration
     516 | static inline void rcu_unexpedite_gp(void) { }
         |                    ^~~~~~~~~~~~~~~~~
   include/linux/rcupdate.h:1194:13: note: previous declaration of 'rcu_unexpedite_gp' with type 'void(void)'
    1194 | extern void rcu_unexpedite_gp(void);
         |             ^~~~~~~~~~~~~~~~~

vim +/rcu_expedite_gp +515 kernel/rcu/rcu.h

2b34c43cc1671c Paul E. McKenney        2017-03-14  509  
25c36329a30c8c Paul E. McKenney        2017-05-03  510  #ifdef CONFIG_TINY_RCU
25c36329a30c8c Paul E. McKenney        2017-05-03  511  /* Tiny RCU doesn't expedite, as its purpose in life is instead to be tiny. */
7414fac050d5e0 Paul E. McKenney        2017-06-12  512  static inline bool rcu_gp_is_normal(void) { return true; }
7414fac050d5e0 Paul E. McKenney        2017-06-12  513  static inline bool rcu_gp_is_expedited(void) { return false; }
6efdda8bec2900 Joel Fernandes (Google  2023-01-12  514) static inline bool rcu_async_should_hurry(void) { return false; }
7414fac050d5e0 Paul E. McKenney        2017-06-12 @515  static inline void rcu_expedite_gp(void) { }
7414fac050d5e0 Paul E. McKenney        2017-06-12 @516  static inline void rcu_unexpedite_gp(void) { }
6efdda8bec2900 Joel Fernandes (Google  2023-01-12  517) static inline void rcu_async_hurry(void) { }
6efdda8bec2900 Joel Fernandes (Google  2023-01-12  518) static inline void rcu_async_relax(void) { }
2be4686d866ad5 Frederic Weisbecker     2023-10-27  519  static inline bool rcu_cpu_online(int cpu) { return true; }
25c36329a30c8c Paul E. McKenney        2017-05-03  520  #else /* #ifdef CONFIG_TINY_RCU */
25c36329a30c8c Paul E. McKenney        2017-05-03  521  bool rcu_gp_is_normal(void);     /* Internal RCU use. */
25c36329a30c8c Paul E. McKenney        2017-05-03  522  bool rcu_gp_is_expedited(void);  /* Internal RCU use. */
6efdda8bec2900 Joel Fernandes (Google  2023-01-12  523) bool rcu_async_should_hurry(void);  /* Internal RCU use. */
25c36329a30c8c Paul E. McKenney        2017-05-03  524  void rcu_expedite_gp(void);
25c36329a30c8c Paul E. McKenney        2017-05-03  525  void rcu_unexpedite_gp(void);
6efdda8bec2900 Joel Fernandes (Google  2023-01-12  526) void rcu_async_hurry(void);
6efdda8bec2900 Joel Fernandes (Google  2023-01-12  527) void rcu_async_relax(void);
25c36329a30c8c Paul E. McKenney        2017-05-03  528  void rcupdate_announce_bootup_oddness(void);
2be4686d866ad5 Frederic Weisbecker     2023-10-27  529  bool rcu_cpu_online(int cpu);
474d0997361c07 Paul E. McKenney        2021-04-20  530  #ifdef CONFIG_TASKS_RCU_GENERIC
e21408ceec2de5 Paul E. McKenney        2020-03-16  531  void show_rcu_tasks_gp_kthreads(void);
e0a34641eb551e Arnd Bergmann           2023-06-09  532  #else /* #ifdef CONFIG_TASKS_RCU_GENERIC */
e0a34641eb551e Arnd Bergmann           2023-06-09  533  static inline void show_rcu_tasks_gp_kthreads(void) {}
e0a34641eb551e Arnd Bergmann           2023-06-09  534  #endif /* #else #ifdef CONFIG_TASKS_RCU_GENERIC */
e0a34641eb551e Arnd Bergmann           2023-06-09  535  #endif /* #else #ifdef CONFIG_TINY_RCU */
e0a34641eb551e Arnd Bergmann           2023-06-09  536  
Signature · 2 lines
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
#7 lore raw

Re: [PATCH v2 2/2] cpuhp: Expedite RCU grace periods during SMT operations

In reply to Vishal Chourasia
kernel test robot noticed the following build errors:

Hi Vishal,

kernel test robot noticed the following build errors:

[auto build test ERROR on next-20260213]
[also build test ERROR on linus/master]
[cannot apply to tip/smp/core rcu/rcu/dev v6.19 v6.19-rc8 v6.19-rc7 v6.19] [If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]

url:    https://github.com/intel-lab-lkp/linux/commits/Vishal-Chourasia/cpuhp-Optimize-SMT-switch-operation-by-batching-lock-acquisition/20260216-202247
base:   next-20260213
patch link:    https://lore.kernel.org/r/20260216121927.489062-6-vishalc%40linux.ibm.com
patch subject: [PATCH v2 2/2] cpuhp: Expedite RCU grace periods during SMT operations
config: arm-allnoconfig (https://download.01.org/0day-ci/archive/20260217/[email protected]/config)
compiler: clang version 23.0.0git (https://github.com/llvm/llvm-project e86750b29fa0ff207cd43213d66dabe565417638)
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20260217/[email protected]/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/

All errors (new ones prefixed by >>):

>> ld.lld: error: duplicate symbol: rcu_expedite_gp
   >>> defined at update.c
   >>>            kernel/rcu/update.o:(rcu_expedite_gp) in archive vmlinux.a
   >>> defined at srcutiny.c
   >>>            kernel/rcu/srcutiny.o:(.text+0x1) in archive vmlinux.a
--
>> ld.lld: error: duplicate symbol: rcu_unexpedite_gp
   >>> defined at update.c
   >>>            kernel/rcu/update.o:(rcu_unexpedite_gp) in archive vmlinux.a
   >>> defined at srcutiny.c
   >>>            kernel/rcu/srcutiny.o:(.text+0x41) in archive vmlinux.a
--
>> ld.lld: error: duplicate symbol: rcu_expedite_gp
   >>> defined at update.c
   >>>            kernel/rcu/update.o:(rcu_expedite_gp) in archive vmlinux.a
   >>> defined at tiny.c
   >>>            kernel/rcu/tiny.o:(.text+0x1) in archive vmlinux.a
--
>> ld.lld: error: duplicate symbol: rcu_unexpedite_gp
   >>> defined at update.c
   >>>            kernel/rcu/update.o:(rcu_unexpedite_gp) in archive vmlinux.a
   >>> defined at tiny.c
   >>>            kernel/rcu/tiny.o:(.text+0x41) in archive vmlinux.a
--
>> ld.lld: error: duplicate symbol: rcu_expedite_gp
   >>> defined at update.c
   >>>            kernel/rcu/update.o:(rcu_expedite_gp) in archive vmlinux.a
   >>> defined at slab_common.c
   >>>            mm/slab_common.o:(.text+0x1) in archive vmlinux.a
--
>> ld.lld: error: duplicate symbol: rcu_unexpedite_gp
   >>> defined at update.c
   >>>            kernel/rcu/update.o:(rcu_unexpedite_gp) in archive vmlinux.a
   >>> defined at slab_common.c
   >>>            mm/slab_common.o:(.text+0x41) in archive vmlinux.a
Signature · 2 lines
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
#8 lore raw

Re: [PATCH v2 2/2] cpuhp: Expedite RCU grace periods during SMT operations

In reply to Vishal Chourasia
Quoted 52 lines · Vishal Chourasia

Expedite synchronize_rcu during the SMT mode switch operation when initiated via /sys/devices/system/cpu/smt/control interface

SMT mode switch operation i.e. between SMT 8 to SMT 1 or vice versa and others, are user driven operations and therefore should complete as soon as possible. Switching SMT states involves iterating over a list of CPUs and performing hotplug operations. It was found these transitions took significantly large amount of time to complete particularly on high-core-count systems because system was blocked on synchronize_rcu calls.

Below is one of the call-stacks that accounted for most of the blocking time overhead as reported by offcputime bcc script for CPU offline operation,

     finish_task_switch
     __schedule
     schedule
     schedule_timeout
     wait_for_completion
     __wait_rcu_gp
     synchronize_rcu
     cpuidle_uninstall_idle_handler
     powernv_cpuidle_cpu_dead
     cpuhp_invoke_callback
     __cpuhp_invoke_callback_range
     _cpu_down
     cpu_device_down
     cpu_subsys_offline
     device_offline
     online_store
     dev_attr_store
     sysfs_kf_write
     kernfs_fop_write_iter
     vfs_write
     ksys_write
     system_call_exception
     system_call_common
    -                bash (29705)
         5771569  ------------------------>  Duration (us)
---
  include/linux/rcupdate.h | 3 +++
  kernel/cpu.c             | 4 ++++
  2 files changed, 7 insertions(+)
include/linux/rcupdate.h +0 -0
index 7729fef249e1..f12d0d0f008d 100644

include/linux/rcupdate.h rcu_head_after_call_rcu(struct rcu_head *rhp, rcu_callback_t f) · line 1198

Quoted 9 lines
include/linux/rcupdate.h +3 -0
@@ -1190,6 +1190,9 @@ rcu_head_after_call_rcu(struct rcu_head *rhp, rcu_callback_t f)
  extern int rcu_expedited;
  extern int rcu_normal;
 
+extern void rcu_expedite_gp(void);
+extern void rcu_unexpedite_gp(void);
+
  DEFINE_LOCK_GUARD_0(rcu, rcu_read_lock(), rcu_read_unlock())
  DECLARE_LOCK_GUARD_0_ATTRS(rcu, __acquires_shared(RCU), __releases_shared(RCU))
 

IMHO, below maybe better.

include/linux/rcupdate.h +7 -2
index f12d0d0f008d..61b80c29d53b 100644
@@ -1190,8 +1190,13 @@ rcu_head_after_call_rcu(struct rcu_head *rhp, rcu_callback_t f)
 extern int rcu_expedited;
 extern int rcu_normal;
 
-extern void rcu_expedite_gp(void);
-extern void rcu_unexpedite_gp(void);
+#ifdef CONFIG_TINY_RCU
+static inline void rcu_expedite_gp(void) { }
+static inline void rcu_unexpedite_gp(void) { }
+#else
+void rcu_expedite_gp(void);
+void rcu_unexpedite_gp(void);
+#endif
 
 DEFINE_LOCK_GUARD_0(rcu, rcu_read_lock(), rcu_read_unlock())
 DECLARE_LOCK_GUARD_0_ATTRS(rcu, __acquires_shared(RCU), 

__releases_shared(RCU))

kernel/rcu/rcu.h +0 -2
index dc5d614b372c..41a0d262e964 100644
@@ -512,8 +512,6 @@ do { 
                       \
 static inline bool rcu_gp_is_normal(void) { return true; }
 static inline bool rcu_gp_is_expedited(void) { return false; }
 static inline bool rcu_async_should_hurry(void) { return false; }
-static inline void rcu_expedite_gp(void) { }
-static inline void rcu_unexpedite_gp(void) { }
 static inline void rcu_async_hurry(void) { }
 static inline void rcu_async_relax(void) { }

static inline bool rcu_cpu_online(int cpu) { return true; }
@@ -521,8 +519,6 @@ static inline bool rcu_cpu_online(int cpu) { return true; }
bool rcu_gp_is_normal(void); /* Internal RCU use. */
bool rcu_gp_is_expedited(void); /* Internal RCU use. */
bool rcu_async_should_hurry(void); /* Internal RCU use. */
-void rcu_expedite_gp(void);
-void rcu_unexpedite_gp(void);
void rcu_async_hurry(void);
void rcu_async_relax(void);
void rcupdate_announce_bootup_oddness(void);