Skip to contentSkip to conversation

drivers/hv: introduce vmbus_channel_set_cpu()

PATCH Superseded Hamza Mahfooz 2 participants 3 messages Jan 16, 2025

Subsystems (2)

v4 of 5 Superseded Latest is v5 →

lore
Share & tools
lore
#0 lore raw

PATCH v4 1/2 drivers/hv: introduce vmbus_channel_set_cpu()

The core functionality in target_cpu_store() is also needed in a subsequent patch for automatically changing the CPU when taking a CPU offline. As such, factor out the body of target_cpu_store() into new function vmbus_channel_set_cpu() that can also be used elsewhere.

No functional change is intended.

Cc Boqun Feng <[email protected]>
Cc Michael Kelley <[email protected]>
Cc Wei Liu <[email protected]>

---
v2: separate vmbus_channel_set_cpu() changes from
cpu offlining changes.

v3: address comments from Michael.

---
 drivers/hv/vmbus_drv.c | 50 +++++++++++++++++++++++++-----------------
 include/linux/hyperv.h |  1 +
 2 files changed, 31 insertions(+), 20 deletions(-)
drivers/hv/vmbus_drv.c +30 -20
index 2892b8da20a5..0ca0e85e6edd 100644
@@ -1611,16 +1611,16 @@ static ssize_t target_cpu_show(struct vmbus_channel *channel, char *buf)
 {
 	return sprintf(buf, "%u\n", channel->target_cpu);
 }
-static ssize_t target_cpu_store(struct vmbus_channel *channel,
-				const char *buf, size_t count)
+
+int vmbus_channel_set_cpu(struct vmbus_channel *channel, u32 target_cpu)
 {
-	u32 target_cpu, origin_cpu;
-	ssize_t ret = count;
+	u32 origin_cpu;
+	int ret = 0;
 
-	if (vmbus_proto_version < VERSION_WIN10_V4_1)
-		return -EIO;
+	lockdep_assert_cpus_held();
+	lockdep_assert_held(&vmbus_connection.channel_mutex);
 
-	if (sscanf(buf, "%uu", &target_cpu) != 1)
+	if (vmbus_proto_version < VERSION_WIN10_V4_1)
 		return -EIO;
 
 	/* Validate target_cpu for the cpumask_test_cpu() operation below. */
@@ -1630,22 +1630,17 @@ static ssize_t target_cpu_store(struct vmbus_channel *channel,
 	if (!cpumask_test_cpu(target_cpu, housekeeping_cpumask(HK_TYPE_MANAGED_IRQ)))
 		return -EINVAL;
 
-	/* No CPUs should come up or down during this. */
-	cpus_read_lock();
-
-	if (!cpu_online(target_cpu)) {
-		cpus_read_unlock();
+	if (!cpu_online(target_cpu))
 		return -EINVAL;
-	}
 
 	/*
-	 * Synchronizes target_cpu_store() and channel closure:
+	 * Synchronizes vmbus_channel_set_cpu() and channel closure:
 	 *
 	 * { Initially: state = CHANNEL_OPENED }
 	 *
 	 * CPU1				CPU2
 	 *
-	 * [target_cpu_store()]		[vmbus_disconnect_ring()]
+	 * [vmbus_channel_set_cpu()]	[vmbus_disconnect_ring()]
 	 *
 	 * LOCK channel_mutex		LOCK channel_mutex
 	 * LOAD r1 = state		LOAD r2 = state
@@ -1660,7 +1655,6 @@ static ssize_t target_cpu_store(struct vmbus_channel *channel,
 	 * Note.  The host processes the channel messages "sequentially", in
 	 * the order in which they are received on a per-partition basis.
 	 */
-	mutex_lock(&vmbus_connection.channel_mutex);
 
 	/*
 	 * Hyper-V will ignore MODIFYCHANNEL messages for "non-open" channels;
@@ -1668,17 +1662,17 @@ static ssize_t target_cpu_store(struct vmbus_channel *channel,
 	 */
 	if (channel->state != CHANNEL_OPENED_STATE) {
 		ret = -EIO;
-		goto cpu_store_unlock;
+		goto end;
 	}
 
 	origin_cpu = channel->target_cpu;
 	if (target_cpu == origin_cpu)
-		goto cpu_store_unlock;
+		goto end;
 
 	if (vmbus_send_modifychannel(channel,
 				     hv_cpu_number_to_vp_number(target_cpu))) {
 		ret = -EIO;
-		goto cpu_store_unlock;
+		goto end;
 	}
 
 	/*
@@ -1708,9 +1702,25 @@ static ssize_t target_cpu_store(struct vmbus_channel *channel,
 				origin_cpu, target_cpu);
 	}
 
-cpu_store_unlock:
+end:
+	return ret;
+}
+
+static ssize_t target_cpu_store(struct vmbus_channel *channel,
+				const char *buf, size_t count)
+{
+	ssize_t ret = count;
+	u32 target_cpu;
+
+	if (sscanf(buf, "%uu", &target_cpu) != 1)
+		return -EIO;
+
+	cpus_read_lock();
+	mutex_lock(&vmbus_connection.channel_mutex);
+	ret = vmbus_channel_set_cpu(channel, target_cpu);
 	mutex_unlock(&vmbus_connection.channel_mutex);
 	cpus_read_unlock();
+
 	return ret;
 }
 static VMBUS_CHAN_ATTR(cpu, 0644, target_cpu_show, target_cpu_store);
include/linux/hyperv.h +1 -0
index 02a226bcf0ed..25e9e982f1b0 100644
@@ -1670,6 +1670,7 @@ int vmbus_send_tl_connect_request(const guid_t *shv_guest_servie_id,
 				  const guid_t *shv_host_servie_id);
 int vmbus_send_modifychannel(struct vmbus_channel *channel, u32 target_vp);
 void vmbus_set_event(struct vmbus_channel *channel);
+int vmbus_channel_set_cpu(struct vmbus_channel *channel, u32 target_cpu);
 
 /* Get the start of the ring buffer. */
 static inline void *
Signature · 1 line
2.47.1
#1 lore raw

PATCH v4 2/2 drivers/hv: add CPU offlining support

Currently, it is tedious to offline CPUs. Since, most CPUs will have vmbus channels attached to them that a user would have to manually rebind elsewhere. So, as made mention of in
commit d570aec0f2154 ("Drivers: hv: vmbus: Synchronize init_vp_index() vs. CPU hotplug"), rebind channels associated with CPUs that a user is trying to offline to a new "randomly" selected CPU.

Cc Boqun Feng <[email protected]>
Cc Michael Kelley <[email protected]>
Cc Wei Liu <[email protected]>

---
v2: remove cpus_read_{un,}lock() from hv_pick_new_cpu() and add
lockdep_assert_cpus_held().

v3: use for_each_cpu_wrap() in hv_pick_new_cpu().

v4: store get_random_u32_below() in start.

---
 drivers/hv/hv.c | 65 +++++++++++++++++++++++++++++++++++++------------
 1 file changed, 50 insertions(+), 15 deletions(-)
drivers/hv/hv.c +50 -15
index 36d9ba097ff5..4388c0030a59 100644
@@ -433,13 +433,48 @@ static bool hv_synic_event_pending(void)
 	return pending;
 }
 
+static int hv_pick_new_cpu(struct vmbus_channel *channel,
+			   unsigned int current_cpu)
+{
+	int ret = 0;
+	int start;
+	int cpu;
+
+	lockdep_assert_cpus_held();
+	lockdep_assert_held(&vmbus_connection.channel_mutex);
+
+	/*
+	 * We can't assume that the relevant interrupts will be sent before
+	 * the cpu is offlined on older versions of hyperv.
+	 */
+	if (vmbus_proto_version < VERSION_WIN10_V5_3)
+		return -EBUSY;
+
+	start = get_random_u32_below(nr_cpu_ids);
+
+	for_each_cpu_wrap(cpu, cpu_online_mask, start) {
+		if (cpu == current_cpu || cpu == VMBUS_CONNECT_CPU)
+			continue;
+
+		ret = vmbus_channel_set_cpu(channel, cpu);
+
+		if (!ret)
+			break;
+	}
+
+	if (ret)
+		ret = vmbus_channel_set_cpu(channel, VMBUS_CONNECT_CPU);
+
+	return ret;
+}
+
 /*
  * hv_synic_cleanup - Cleanup routine for hv_synic_init().
  */
 int hv_synic_cleanup(unsigned int cpu)
 {
 	struct vmbus_channel *channel, *sc;
-	bool channel_found = false;
+	int ret = 0;
 
 	if (vmbus_connection.conn_state != CONNECTED)
 		goto always_cleanup;
@@ -456,31 +491,31 @@ int hv_synic_cleanup(unsigned int cpu)
 
 	/*
 	 * Search for channels which are bound to the CPU we're about to
-	 * cleanup.  In case we find one and vmbus is still connected, we
-	 * fail; this will effectively prevent CPU offlining.
-	 *
-	 * TODO: Re-bind the channels to different CPUs.
+	 * cleanup.
 	 */
 	mutex_lock(&vmbus_connection.channel_mutex);
 	list_for_each_entry(channel, &vmbus_connection.chn_list, listentry) {
 		if (channel->target_cpu == cpu) {
-			channel_found = true;
-			break;
+			ret = hv_pick_new_cpu(channel, cpu);
+
+			if (ret) {
+				mutex_unlock(&vmbus_connection.channel_mutex);
+				return ret;
+			}
 		}
 		list_for_each_entry(sc, &channel->sc_list, sc_list) {
 			if (sc->target_cpu == cpu) {
-				channel_found = true;
-				break;
+				ret = hv_pick_new_cpu(channel, cpu);
+
+				if (ret) {
+					mutex_unlock(&vmbus_connection.channel_mutex);
+					return ret;
+				}
 			}
 		}
-		if (channel_found)
-			break;
 	}
 	mutex_unlock(&vmbus_connection.channel_mutex);
 
-	if (channel_found)
-		return -EBUSY;
-
 	/*
 	 * channel_found == false means that any channels that were previously
 	 * assigned to the CPU have been reassigned elsewhere with a call of
@@ -497,5 +532,5 @@ int hv_synic_cleanup(unsigned int cpu)
 
 	hv_synic_disable_regs(cpu);
 
-	return 0;
+	return ret;
 }
Signature · 1 line
2.47.1
#2 lore raw

RE: [PATCH v4 2/2] drivers/hv: add CPU offlining support

From: Hamza Mahfooz <[email protected]> Sent: Thursday, January 16, 2025 5:42 AM

Quoted 4 lines

Currently, it is tedious to offline CPUs. Since, most CPUs will have vmbus channels attached to them that a user would have to manually rebind elsewhere.

Cleaning up the punctuation and wording a bit:

Currently, it is tedious to offline CPUs in a Hyper-V VM since CPUs may have VMBus channels attached to them that a user would have to manually rebind elsewhere.

Quoted 43 lines

So, as made mention of in
commit d570aec0f2154 ("Drivers: hv: vmbus: Synchronize init_vp_index() vs. CPU hotplug"), rebind channels associated with CPUs that a user is trying to offline to a new "randomly" selected CPU.

Cc Boqun Feng <[email protected]>
Cc Michael Kelley <[email protected]>
Cc Wei Liu <[email protected]>

---
v2: remove cpus_read_{un,}lock() from hv_pick_new_cpu() and add
lockdep_assert_cpus_held().

v3: use for_each_cpu_wrap() in hv_pick_new_cpu().

v4: store get_random_u32_below() in start.

---
 drivers/hv/hv.c | 65 +++++++++++++++++++++++++++++++++++++------------
 1 file changed, 50 insertions(+), 15 deletions(-)
drivers/hv/hv.c +15 -0
index 36d9ba097ff5..4388c0030a59 100644
@@ -433,13 +433,48 @@ static bool hv_synic_event_pending(void)
 	return pending;
 }
 
+static int hv_pick_new_cpu(struct vmbus_channel *channel,
+			   unsigned int current_cpu)
+{
+	int ret = 0;
+	int start;
+	int cpu;
+
+	lockdep_assert_cpus_held();
+	lockdep_assert_held(&vmbus_connection.channel_mutex);
+
+	/*
+	 * We can't assume that the relevant interrupts will be sent before
+	 * the cpu is offlined on older versions of hyperv.
+	 */
+	if (vmbus_proto_version < VERSION_WIN10_V5_3)

drivers/hv/hv.c static bool hv_synic_event_pending(void) · line 460

Quoted 10 lines
drivers/hv/hv.c +10 -0
+		return -EBUSY;
+
+	start = get_random_u32_below(nr_cpu_ids);
+
+	for_each_cpu_wrap(cpu, cpu_online_mask, start) {
+		if (cpu == current_cpu || cpu == VMBUS_CONNECT_CPU)
+			continue;
+
+		ret = vmbus_channel_set_cpu(channel, cpu);
+

Avoid the blank line between setting "ret" and testing it. If you look through other Linux kernel code, you'll see that the set and test are almost always grouped together with no intervening blank line.

drivers/hv/hv.c static bool hv_synic_event_pending(void) · line 463

Quoted 3 lines
drivers/hv/hv.c +3 -0
+		if (!ret)
+			break;
+	}

The loop looks good as a solution to filtering isolated CPUs! But there's a subtle bug. Consider a VM with two CPUs. One of the CPUs is the VMBUS_CONNECT_CPU, and the other CPU is being taken offline. The loop will exit without ever having called vmbus_channel_set_cpu(), so "ret" will still be zero. The check below will *not* call vmbus_channel_set_cpu(), and the CPU will be offlined with the channel interrupt still assigned.

I think the easy fix is to initialize "ret" to -EBUSY, but double-check my thinking.

Quoted 25 lines
drivers/hv/hv.c +8 -4
+
+	if (ret)
+		ret = vmbus_channel_set_cpu(channel, VMBUS_CONNECT_CPU);
+
+	return ret;
+}
+
 /*
  * hv_synic_cleanup - Cleanup routine for hv_synic_init().
  */
 int hv_synic_cleanup(unsigned int cpu)
 {
 	struct vmbus_channel *channel, *sc;
-	bool channel_found = false;
+	int ret = 0;
 
 	if (vmbus_connection.conn_state != CONNECTED)
 		goto always_cleanup;
@@ -456,31 +491,31 @@ int hv_synic_cleanup(unsigned int cpu)
 
 	/*
 	 * Search for channels which are bound to the CPU we're about to
-	 * cleanup.  In case we find one and vmbus is still connected, we
-	 * fail; this will effectively prevent CPU offlining.
-	 *

drivers/hv/hv.c int hv_synic_cleanup(unsigned int cpu) · line 500

Quoted 10 lines
drivers/hv/hv.c +3 -3
-	 * TODO: Re-bind the channels to different CPUs.
+	 * cleanup.
 	 */
 	mutex_lock(&vmbus_connection.channel_mutex);
 	list_for_each_entry(channel, &vmbus_connection.chn_list, listentry) {
 		if (channel->target_cpu == cpu) {
-			channel_found = true;
-			break;
+			ret = hv_pick_new_cpu(channel, cpu);
+

As above, avoid the blank line here.

drivers/hv/hv.c int hv_synic_cleanup(unsigned int cpu) · line 508

Quoted 10 lines
drivers/hv/hv.c +5 -2
+			if (ret) {
+				mutex_unlock(&vmbus_connection.channel_mutex);
+				return ret;
+			}
 		}
 		list_for_each_entry(sc, &channel->sc_list, sc_list) {
 			if (sc->target_cpu == cpu) {
-				channel_found = true;
-				break;
+				ret = hv_pick_new_cpu(channel, cpu);

The first argument to hv_pick_new_cpu() should be "sc", not "channel". Currently you'll be updating the primary channel, not the intended sub-channel.

Also, you don't really need to pass the "cpu" as an argument to hv_pick_new_cpu(). Just have that function take the channel as its only argument. It can get the cpu as the target_cpu field in the channel.

drivers/hv/hv.c int hv_synic_cleanup(unsigned int cpu) · line 509

Quoted 1 line
drivers/hv/hv.c +1 -0
+

Avoid the blank line here too.

Quoted 6 lines
drivers/hv/hv.c +4 -0
+				if (ret) {
+					mutex_unlock(&vmbus_connection.channel_mutex);
+					return ret;
+				}
 			}
 		}

drivers/hv/hv.c int hv_synic_cleanup(unsigned int cpu) · line 520

Quoted 10 lines
drivers/hv/hv.c +0 -5
-		if (channel_found)
-			break;
 	}
 	mutex_unlock(&vmbus_connection.channel_mutex);
 
-	if (channel_found)
-		return -EBUSY;
-
 	/*
 	 * channel_found == false means that any channels that were previously

Since the "channel_found" local variable has been removed, this comment needs some cleanup.

Quoted 10 lines
drivers/hv/hv.c +1 -2
 	 * assigned to the CPU have been reassigned elsewhere with a call of
@@ -497,5 +532,5 @@ int hv_synic_cleanup(unsigned int cpu)
 
 	hv_synic_disable_regs(cpu);
 
-	return 0;
+	return ret;
 }
--

2.47.1