Skip to contentSkip to conversation

Drivers: hv

PATCH RESEND Superseded K. Y. Srinivasan 2 participants 9 messages Mar 12, 2013 – Mar 13, 2013 Hyper-V/Azure core and drivers

lore
Share & tools
lore
#0 lore raw

cover Drivers: hv

This patch set enhances the balloon driver functionality to add support for memory "hot-add". Windows hosts use a combination of ballooning and hot-add to dynamically balance the available memory across competing virtual machines. With this, Linux guests can fully participate in the Windows Dynamic Memory protocol.

Additionally, I have also fixed an issue in the Vmbus driver in the way the offer rescind message is handled.

K. Y. Srinivasan (6):
Drivers: hv: balloon: Do not request completion notification
Drivers: hv: balloon: Execute balloon inflation in a separate context
Drivers: hv: balloon: Execute hot-add code in a separate context
Drivers: hv: balloon: Make the balloon driver not unloadable
Drivers: hv: balloon: Implement hot-add functionality
Drivers: hv: vmbus: Handle channel rescind message correctly

 drivers/hv/Kconfig        |    2 +-
 drivers/hv/channel_mgmt.c |   11 +
 drivers/hv/hv_balloon.c   |  467 +++++++++++++++++++++++++++++++++++++++-----
 3 files changed, 426 insertions(+), 54 deletions(-)
Signature · 1 line
1.7.4.1
#1 lore raw

PATCH RESEND 1/6 Drivers: hv: balloon: Do not request completion notification

There is no need to request completion notification; get rid of it.

---
 drivers/hv/hv_balloon.c |    6 ++----
 1 files changed, 2 insertions(+), 4 deletions(-)
drivers/hv/hv_balloon.c +2 -4
index 3787321..7fb72dd 100644
@@ -962,8 +962,7 @@ static int balloon_probe(struct hv_device *dev,
 	ret = vmbus_sendpacket(dev->channel, &version_req,
 				sizeof(struct dm_version_request),
 				(unsigned long)NULL,
-				VM_PKT_DATA_INBAND,
-				VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
+				VM_PKT_DATA_INBAND, 0);
 	if (ret)
 		goto probe_error2;
 
@@ -1009,8 +1008,7 @@ static int balloon_probe(struct hv_device *dev,
 	ret = vmbus_sendpacket(dev->channel, &cap_msg,
 				sizeof(struct dm_capabilities),
 				(unsigned long)NULL,
-				VM_PKT_DATA_INBAND,
-				VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
+				VM_PKT_DATA_INBAND, 0);
 	if (ret)
 		goto probe_error2;
 
Signature · 1 line
1.7.4.1
#2 lore raw

PATCH RESEND 2/6 Drivers: hv: balloon: Execute balloon inflation in a separate context

Execute the balloon inflation operation in a separate work context. This allows us to decouple the pressure reporting activity from the ballooning activity. Testing has shown that this decoupling makes the guest more reponsive.

---
 drivers/hv/hv_balloon.c |   34 ++++++++++++++++++++++++----------
 1 files changed, 24 insertions(+), 10 deletions(-)
drivers/hv/hv_balloon.c +24 -10
index 7fb72dd..8dc406c 100644
@@ -412,6 +412,11 @@ struct dm_info_msg {
  * End protocol definitions.
  */
 
+struct balloon_state {
+	__u32 num_pages;
+	struct work_struct wrk;
+};
+
 static bool hot_add;
 static bool do_hot_add;
 /*
@@ -459,7 +464,12 @@ struct hv_dynmem_device {
 	unsigned int num_pages_ballooned;
 
 	/*
-	 * This thread handles both balloon/hot-add
+	 * State to manage the ballooning (up) operation.
+	 */
+	struct balloon_state balloon_wrk;
+
+	/*
+	 * This thread handles hot-add
 	 * requests from the host as well as notifying
 	 * the host with regards to memory pressure in
 	 * the guest.
@@ -657,9 +667,9 @@ static int  alloc_balloon_pages(struct hv_dynmem_device *dm, int num_pages,
 
 
 
-static void balloon_up(struct hv_dynmem_device *dm, struct dm_balloon *req)
+static void balloon_up(struct work_struct *dummy)
 {
-	int num_pages = req->num_pages;
+	int num_pages = dm_device.balloon_wrk.num_pages;
 	int num_ballooned = 0;
 	struct dm_balloon_response *bl_resp;
 	int alloc_unit;
@@ -684,14 +694,14 @@ static void balloon_up(struct hv_dynmem_device *dm, struct dm_balloon *req)
 
 
 		num_pages -= num_ballooned;
-		num_ballooned = alloc_balloon_pages(dm, num_pages,
+		num_ballooned = alloc_balloon_pages(&dm_device, num_pages,
 						bl_resp, alloc_unit,
 						 &alloc_error);
 
 		if ((alloc_error) || (num_ballooned == num_pages)) {
 			bl_resp->more_pages = 0;
 			done = true;
-			dm->state = DM_INITIALIZED;
+			dm_device.state = DM_INITIALIZED;
 		}
 
 		/*
@@ -719,7 +729,7 @@ static void balloon_up(struct hv_dynmem_device *dm, struct dm_balloon *req)
 			pr_info("Balloon response failed\n");
 
 			for (i = 0; i < bl_resp->range_count; i++)
-				free_balloon_pages(dm,
+				free_balloon_pages(&dm_device,
 						 &bl_resp->range_array[i]);
 
 			done = true;
@@ -775,9 +785,6 @@ static int dm_thread_func(void *dm_dev)
 
 		scan_start = jiffies;
 		switch (dm->state) {
-		case DM_BALLOON_UP:
-			balloon_up(dm, (struct dm_balloon *)recv_buffer);
-			break;
 
 		case DM_HOT_ADD:
 			hot_add_req(dm, (struct dm_hot_add *)recv_buffer);
@@ -861,6 +868,7 @@ static void balloon_onchannelcallback(void *context)
 	struct dm_message *dm_msg;
 	struct dm_header *dm_hdr;
 	struct hv_dynmem_device *dm = hv_get_drvdata(dev);
+	struct dm_balloon *bal_msg;
 
 	memset(recv_buffer, 0, sizeof(recv_buffer));
 	vmbus_recvpacket(dev->channel, recv_buffer,
@@ -882,8 +890,12 @@ static void balloon_onchannelcallback(void *context)
 			break;
 
 		case DM_BALLOON_REQUEST:
+			if (dm->state == DM_BALLOON_UP)
+				pr_warn("Currently ballooning\n");
+			bal_msg = (struct dm_balloon *)recv_buffer;
 			dm->state = DM_BALLOON_UP;
-			complete(&dm->config_event);
+			dm_device.balloon_wrk.num_pages = bal_msg->num_pages;
+			schedule_work(&dm_device.balloon_wrk.wrk);
 			break;
 
 		case DM_UNBALLOON_REQUEST:
@@ -937,6 +949,7 @@ static int balloon_probe(struct hv_device *dev,
 	dm_device.next_version = DYNMEM_PROTOCOL_VERSION_WIN7;
 	init_completion(&dm_device.host_event);
 	init_completion(&dm_device.config_event);
+	INIT_WORK(&dm_device.balloon_wrk.wrk, balloon_up);
 
 	dm_device.thread =
 		 kthread_run(dm_thread_func, &dm_device, "hv_balloon");
@@ -1048,6 +1061,7 @@ static int balloon_remove(struct hv_device *dev)
 	if (dm->num_pages_ballooned != 0)
 		pr_warn("Ballooned pages: %d\n", dm->num_pages_ballooned);
 
+	cancel_work_sync(&dm->balloon_wrk.wrk);
 	vmbus_close(dev->channel);
 	kthread_stop(dm->thread);
 	kfree(send_buffer);
Signature · 1 line
1.7.4.1
#3 lore raw

PATCH RESEND 3/6 Drivers: hv: balloon: Execute hot-add code in a separate context

In reply to K. Y. Srinivasan

Execute the hot-add operation in a separate work context. This allows us to decouple the pressure reporting activity from the "hot-add" activity. Testing has shown that this makes the guest more responsive to hot add requests.

---
 drivers/hv/hv_balloon.c |   41 +++++++++++++++++++++++------------------
 1 files changed, 23 insertions(+), 18 deletions(-)
drivers/hv/hv_balloon.c +23 -18
index 8dc406c..13fda38 100644
@@ -417,6 +417,11 @@ struct balloon_state {
 	struct work_struct wrk;
 };
 
+struct hot_add_wrk {
+	union dm_mem_page_range ha_page_range;
+	struct work_struct wrk;
+};
+
 static bool hot_add;
 static bool do_hot_add;
 /*
@@ -469,6 +474,11 @@ struct hv_dynmem_device {
 	struct balloon_state balloon_wrk;
 
 	/*
+	 * State to execute the "hot-add" operation.
+	 */
+	struct hot_add_wrk ha_wrk;
+
+	/*
 	 * This thread handles hot-add
 	 * requests from the host as well as notifying
 	 * the host with regards to memory pressure in
@@ -486,7 +496,7 @@ struct hv_dynmem_device {
 
 static struct hv_dynmem_device dm_device;
 
-static void hot_add_req(struct hv_dynmem_device *dm, struct dm_hot_add *msg)
+static void hot_add_req(struct work_struct *dummy)
 {
 
 	struct dm_hot_add_response resp;
@@ -509,8 +519,8 @@ static void hot_add_req(struct hv_dynmem_device *dm, struct dm_hot_add *msg)
 	resp.page_count = 0;
 	resp.result = 0;
 
-	dm->state = DM_INITIALIZED;
-	vmbus_sendpacket(dm->dev->channel, &resp,
+	dm_device.state = DM_INITIALIZED;
+	vmbus_sendpacket(dm_device.dev->channel, &resp,
 			sizeof(struct dm_hot_add_response),
 			(unsigned long)NULL,
 			VM_PKT_DATA_INBAND, 0);
@@ -771,7 +781,6 @@ static int dm_thread_func(void *dm_dev)
 {
 	struct hv_dynmem_device *dm = dm_dev;
 	int t;
-	unsigned long  scan_start;
 
 	while (!kthread_should_stop()) {
 		t = wait_for_completion_timeout(&dm_device.config_event, 1*HZ);
@@ -783,19 +792,6 @@ static int dm_thread_func(void *dm_dev)
 		if (t == 0)
 			post_status(dm);
 
-		scan_start = jiffies;
-		switch (dm->state) {
-
-		case DM_HOT_ADD:
-			hot_add_req(dm, (struct dm_hot_add *)recv_buffer);
-			break;
-		default:
-			break;
-		}
-
-		if (!time_in_range(jiffies, scan_start, scan_start + HZ))
-			post_status(dm);
-
 	}
 
 	return 0;
@@ -869,6 +865,8 @@ static void balloon_onchannelcallback(void *context)
 	struct dm_header *dm_hdr;
 	struct hv_dynmem_device *dm = hv_get_drvdata(dev);
 	struct dm_balloon *bal_msg;
+	struct dm_hot_add *ha_msg;
+	union dm_mem_page_range *ha_pg_range;
 
 	memset(recv_buffer, 0, sizeof(recv_buffer));
 	vmbus_recvpacket(dev->channel, recv_buffer,
@@ -905,8 +903,13 @@ static void balloon_onchannelcallback(void *context)
 			break;
 
 		case DM_MEM_HOT_ADD_REQUEST:
+			if (dm->state == DM_HOT_ADD)
+				pr_warn("Currently hot-adding\n");
 			dm->state = DM_HOT_ADD;
-			complete(&dm->config_event);
+			ha_msg = (struct dm_hot_add *)recv_buffer;
+			ha_pg_range = &ha_msg->range;
+			dm_device.ha_wrk.ha_page_range = *ha_pg_range;
+			schedule_work(&dm_device.ha_wrk.wrk);
 			break;
 
 		case DM_INFO_MESSAGE:
@@ -950,6 +953,7 @@ static int balloon_probe(struct hv_device *dev,
 	init_completion(&dm_device.host_event);
 	init_completion(&dm_device.config_event);
 	INIT_WORK(&dm_device.balloon_wrk.wrk, balloon_up);
+	INIT_WORK(&dm_device.ha_wrk.wrk, hot_add_req);
 
 	dm_device.thread =
 		 kthread_run(dm_thread_func, &dm_device, "hv_balloon");
@@ -1062,6 +1066,7 @@ static int balloon_remove(struct hv_device *dev)
 		pr_warn("Ballooned pages: %d\n", dm->num_pages_ballooned);
 
 	cancel_work_sync(&dm->balloon_wrk.wrk);
+	cancel_work_sync(&dm->ha_wrk.wrk);
 	vmbus_close(dev->channel);
 	kthread_stop(dm->thread);
 	kfree(send_buffer);
Signature · 1 line
1.7.4.1
#4 lore raw

PATCH RESEND 4/6 Drivers: hv: balloon: Make the balloon driver not unloadable

In reply to K. Y. Srinivasan

The balloon driver is stateful. For instance, it needs to keep track of pages that have been ballooned out to properly post pressure reports. This state cannot be re-constructed if the driver were to be unloaded and subsequently loaded. Furthermore, as we support memory hot-add as part of this driver, this driver becomes even more stateful and this state cannot be re-created. Make the balloon driver unloadable to deal with this issue.

---
 drivers/hv/hv_balloon.c |    7 -------
 1 files changed, 0 insertions(+), 7 deletions(-)
drivers/hv/hv_balloon.c +0 -7
index 13fda38..4743db9 100644
@@ -1096,14 +1096,7 @@ static int __init init_balloon_drv(void)
 	return vmbus_driver_register(&balloon_drv);
 }
 
-static void exit_balloon_drv(void)
-{
-
-	vmbus_driver_unregister(&balloon_drv);
-}
-
 module_init(init_balloon_drv);
-module_exit(exit_balloon_drv);
 
 MODULE_DESCRIPTION("Hyper-V Balloon");
 MODULE_VERSION(HV_DRV_VERSION);
Signature · 1 line
1.7.4.1
#5 lore raw

PATCH RESEND 5/6 Drivers: hv: balloon: Implement hot-add functionality

In reply to K. Y. Srinivasan

Implement the memory hot-add functionality. With this, Linux guests can fully participate in the Dynamic Memory protocol implemented in the Windows hosts.

---
 drivers/hv/Kconfig      |    2 +-
 drivers/hv/hv_balloon.c |  393 ++++++++++++++++++++++++++++++++++++++++++++---
 2 files changed, 373 insertions(+), 22 deletions(-)
drivers/hv/Kconfig +1 -1
index 64630f1..5415fac 100644
@@ -15,7 +15,7 @@ config HYPERV_UTILS
 
 config HYPERV_BALLOON
 	tristate "Microsoft Hyper-V Balloon driver"
-	depends on HYPERV
+	depends on HYPERV && MEMORY_HOTPLUG
 	help
 	  Select this option to enable Hyper-V Balloon driver.
 
drivers/hv/hv_balloon.c +372 -21

502 lines not on this page · raw message

Signature · 1 line
1.7.4.1
#6 lore raw

Re: [PATCH RESEND 5/6] Drivers: hv: balloon: Implement hot-add functionality

drivers/hv/Kconfig config HYPERV_UTILS · line 20

Quoted 12 lines · K. Y. Srinivasan

Implement the memory hot-add functionality. With this, Linux guests can fully participate in the Dynamic Memory protocol implemented in the Windows hosts.

+++ b/drivers/hv/Kconfig

drivers/hv/Kconfig +1 -1
@@ -15,7 +15,7 @@ config HYPERV_UTILS
 
 config HYPERV_BALLOON
 	tristate "Microsoft Hyper-V Balloon driver"
-	depends on HYPERV
+	depends on HYPERV && MEMORY_HOTPLUG
 	help
 	  Select this option to enable Hyper-V Balloon driver.

This driver now depends on CONFIG_MEMORY_HOTPLUG, which is disabled in openSuSE i386 kernel. As a result, the hv_balloon driver disappears for 32bit kernels.

Any chance that the Kconfig dependency can be moved inside the code so that a 32bit guest can still do balloon operations?

Olaf

#7 lore raw

RE: [PATCH RESEND 5/6] Drivers: hv: balloon: Implement hot-add functionality

Quoted 31 lines

-----Original Message-----
From: Olaf Hering [mailto:[email protected]]
Sent: Wednesday, March 13, 2013 12:50 PM
To: KY Srinivasan
Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: Re: [PATCH RESEND 5/6] Drivers: hv: balloon: Implement hot-add functionality

Quoted 2 lines · K. Y. Srinivasan

Implement the memory hot-add functionality. With this, Linux guests can fully participate in the Dynamic Memory protocol implemented in the Windows

hosts.

drivers/hv/Kconfig config HYPERV_UTILS · line 20

Quoted 9 lines

+++ b/drivers/hv/Kconfig

drivers/hv/Kconfig +1 -1
@@ -15,7 +15,7 @@ config HYPERV_UTILS
 
 config HYPERV_BALLOON
 	tristate "Microsoft Hyper-V Balloon driver"
-	depends on HYPERV
+	depends on HYPERV && MEMORY_HOTPLUG
 	help
 	  Select this option to enable Hyper-V Balloon driver.

This driver now depends on CONFIG_MEMORY_HOTPLUG, which is disabled in openSuSE i386 kernel. As a result, the hv_balloon driver disappears for 32bit kernels.

Quoted 2 lines

Any chance that the Kconfig dependency can be moved inside the code so that a 32bit guest can still do balloon operations?

I can preserve the old behavior by getting rid of the config dependency and adding in the MEMORY_HOTPLUG dependency around the affected code. Greg, shall I just re-send this one patch.

Regards,

K. Y

Quoted 4 lines

Olaf

���{.n�+�������+%�����ݶ��w��{.n�+����{��G�����{ay�ʇڙ�,j��f���h��������z_�(�階�ݢj"���m������G������?����&���~��iO��z��v�^�m���� ������?�I�

#8 lore raw

PATCH RESEND 6/6 Drivers: hv: vmbus: Handle channel rescind message correctly

In reply to K. Y. Srinivasan

Properly cleanup the channel state on receipt of the "offer rescind" message. Starting with ws2012, the host requires that the channel "relid" be properly cleaned up when the offer is rescinded.

---
 drivers/hv/channel_mgmt.c |   11 +++++++++++
 1 files changed, 11 insertions(+), 0 deletions(-)
drivers/hv/channel_mgmt.c +11 -0
index ff1be16..bad8128 100644
@@ -165,8 +165,19 @@ static void vmbus_process_rescind_offer(struct work_struct *work)
 	struct vmbus_channel *channel = container_of(work,
 						     struct vmbus_channel,
 						     work);
+	unsigned long flags;
+	struct vmbus_channel_relid_released msg;
 
 	vmbus_device_unregister(channel->device_obj);
+	memset(&msg, 0, sizeof(struct vmbus_channel_relid_released));
+	msg.child_relid = channel->offermsg.child_relid;
+	msg.header.msgtype = CHANNELMSG_RELID_RELEASED;
+	vmbus_post_msg(&msg, sizeof(struct vmbus_channel_relid_released));
+
+	spin_lock_irqsave(&vmbus_connection.channel_lock, flags);
+	list_del(&channel->listentry);
+	spin_unlock_irqrestore(&vmbus_connection.channel_lock, flags);
+	free_channel(channel);
 }
 
 void vmbus_free_channels(void)
Signature · 1 line
1.7.4.1