KVM virtio balloon driver

Hi Rusty,

It was agreed that the balloon driver should be merged through the virtio tree, so here it goes. It depends on the config_changed patch posted earlier.

-----

Following patch adds the KVM balloon driver.

Changes from last version:
- Get rid of global variables/structure
- Use page->lru to link ballooned pages
- Use dev_dbg/dev_printk
- Proper kthread_should_stop handling
- Move shared definitions to separate header
- Use ->config_changed method for notification

This depends on Rusty's config_changed patch.

Signed-off-by Marcelo Tosatti <[email protected]>

Index: linux-2.6-nv/drivers/virtio/Kconfig
===================================================================

drivers/virtio/Kconfig +0 -0

@@ -23,3 +23,12 @@ config VIRTIO_PCI
 
 	  If unsure, say M.

Index: linux-2.6-nv/drivers/virtio/Makefile
===================================================================

drivers/virtio/Makefile +1 -0

@@ -1,3 +1,4 @@
 obj-$(CONFIG_VIRTIO) += virtio.o
 obj-$(CONFIG_VIRTIO_RING) += virtio_ring.o
 obj-$(CONFIG_VIRTIO_PCI) += virtio_pci.o
+obj-$(CONFIG_KVM_BALLOON) += kvm_balloon.o

Index: linux-2.6-nv/drivers/virtio/kvm_balloon.c
===================================================================

drivers/virtio/kvm_balloon.c new file +19 -0

@@ -0,0 +1,537 @@
+/*
+ * KVM guest balloon driver
+ *
+ * Copyright (C) 2007, Qumranet, Inc., Dor Laor <[email protected]>
+ * Copyright (C) 2007, Red Hat, Inc., Marcelo Tosatti <[email protected]>
+ * 
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ */
+
+#define DEBUG
+#include <asm/uaccess.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/percpu.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/mm.h>
+#include <linux/swap.h>

drivers/virtio/kvm_balloon.c new file +498 -0

+#define BALLOON_DATA_SIZE 200
+
+struct balloon_buf {
+	struct virtio_balloon_hdr hdr;
+	u8 data[BALLOON_DATA_SIZE];
+};
+
+struct balloon_work {
+	struct balloon_buf *buf;
+	struct list_head list;
+};
+
+#define VIRTIO_MAX_SG 2
+
+struct virtballoon {
+	struct virtio_device *vdev;
+	struct virtqueue *vq;
+	struct task_struct *balloon_thread;
+	wait_queue_head_t balloon_wait;
+	wait_queue_head_t rmmod_wait;
+	uint32_t target_nrpages;
+	atomic_t inflight_bufs;
+	int balloon_size;
+	struct list_head balloon_plist;
+	struct list_head balloon_work;
+	spinlock_t plist_lock;
+	spinlock_t queue_lock;
+	struct list_head list;
+};
+
+struct balloon_buf *alloc_balloon_buf(struct virtio_device *vdev, gfp_t flags)
+{
+	struct balloon_buf *buf;
+
+	buf = kzalloc(sizeof(struct balloon_buf), flags);
+	if (!buf)
+		dev_printk(KERN_ERR, &vdev->dev, "%s: alloc fail\n", __func__);
+
+	return buf;
+}
+
+static int send_balloon_buf(struct virtballoon *v, uint8_t cmd,
+			    struct balloon_buf *buf)
+{
+	struct scatterlist sg[VIRTIO_MAX_SG];
+	int err = 0;
+
+	buf->hdr.cmd = cmd;
+
+	sg_init_table(sg, VIRTIO_MAX_SG);
+	sg_set_buf(&sg[0], &buf->hdr, sizeof(buf->hdr));
+	sg_set_buf(&sg[1], &buf->data, sizeof(buf->data));
+
+	spin_lock_irq(&v->queue_lock);
+	err = v->vq->vq_ops->add_buf(v->vq, sg, 0, 2, buf);
+	if (err) {
+		dev_printk(KERN_ERR, &v->vq->vdev->dev, "%s: add_buf err\n",
+			   __func__);
+		goto out;
+	}
+
+	/* TODO: kick several balloon buffers at once */
+	v->vq->vq_ops->kick(v->vq);
+out:
+	spin_unlock_irq(&v->queue_lock);
+	atomic_inc(&v->inflight_bufs);
+	return err;
+}
+
+static int kvm_balloon_inflate(struct virtballoon *v, int32_t npages)
+{
+	LIST_HEAD(tmp_list);
+	struct page *page, *tmp;
+	struct balloon_buf *buf;
+	u32 *pfn;
+	int allocated = 0;
+	int i, r = -ENOMEM;
+
+	buf = alloc_balloon_buf(v->vdev, GFP_KERNEL);
+	if (!buf)
+		return r;
+
+	pfn = (u32 *)&buf->data;
+	*pfn++ = (u32)npages;
+
+	for (i = 0; i < npages; i++) {
+		page = alloc_page(GFP_HIGHUSER | __GFP_NORETRY);
+		if (!page)
+			goto out_free;
+		list_add(&page->lru, &tmp_list);
+		allocated++;
+		*pfn = page_to_pfn(page);
+		pfn++;
+	}
+
+	r = send_balloon_buf(v, CMD_BALLOON_INFLATE, buf);
+	if (r)
+		goto out_free;
+
+	spin_lock(&v->plist_lock);
+	list_splice(&tmp_list, &v->balloon_plist);
+	v->balloon_size += allocated;
+	totalram_pages -= allocated;
+	dprintk(&v->vdev->dev, "%s: current balloon size=%d\n", __func__,
+		v->balloon_size);
+	spin_unlock(&v->plist_lock);
+	return allocated;
+
+out_free:
+	list_for_each_entry_safe(page, tmp, &tmp_list, lru) {
+		list_del(&page->lru);
+		__free_page(page);
+	}
+	return r;
+}
+
+static int kvm_balloon_deflate(struct virtballoon *v, int32_t npages)
+{
+	LIST_HEAD(tmp_list);
+	struct page *page, *tmp;
+	struct balloon_buf *buf;
+	u32 *pfn;
+	int deallocated = 0;
+	int r = 0;
+
+	buf = alloc_balloon_buf(v->vdev, GFP_KERNEL);
+	if (!buf)
+		return r;
+
+	spin_lock(&v->plist_lock);
+
+	if (v->balloon_size < npages) {
+		dev_printk(KERN_INFO, &v->vdev->dev,
+			   "%s: balloon=%d with deflate rq=%d\n",
+			   __func__, v->balloon_size, npages);
+		npages = v->balloon_size;
+		if (!npages)
+			goto out;
+	}
+
+	pfn = (u32 *)&buf->data;
+	*pfn++ = (u32)-npages;
+
+	/*
+	 * Move the balloon pages to tmp list before issuing 
+	 * the virtio buffer
+	 */
+	list_for_each_entry_safe(page, tmp, &v->balloon_plist, lru) {
+		*pfn++ = page_to_pfn(page);
+		list_move(&page->lru, &tmp_list);
+		if (++deallocated == npages)
+			break;
+	}
+
+	r = send_balloon_buf(v, CMD_BALLOON_DEFLATE, buf);
+	if (r)
+		goto out;
+
+	list_for_each_entry_safe(page, tmp, &tmp_list, lru)
+		list_del_init(&page->lru);
+
+	v->balloon_size -= npages;
+	totalram_pages += npages;
+	dprintk(&v->vdev->dev, "%s: current balloon size=%d\n", __func__,
+	       v->balloon_size);
+
+	spin_unlock(&v->plist_lock);
+	return deallocated;
+
+out:
+	list_splice(&tmp_list, &v->balloon_plist);
+	spin_unlock(&v->plist_lock);
+	return r;
+}
+
+#define MAX_BALLOON_PAGES_PER_OP (BALLOON_DATA_SIZE/sizeof(u32)) \
+				 - sizeof(int32_t)
+#define MAX_BALLOON_XFLATE_OP 1000000
+
+static int kvm_balloon_xflate(struct virtballoon *v, int32_t npages)
+{
+	int r = -EINVAL, i;
+	int iterations;
+	int abspages;
+	int curr_pages = 0;
+	int gfns_per_buf;
+
+	abspages = abs(npages);
+
+	if (abspages > MAX_BALLOON_XFLATE_OP) {
+		dev_printk(KERN_ERR, &v->vdev->dev,
+			   "%s: bad npages=%d\n", __func__, npages);
+		return -EINVAL;
+	}
+
+	dprintk(&v->vdev->dev, "%s: got %s, npages=%d\n", __func__,
+	       (npages > 0)? "inflate":"deflate", npages);
+
+	gfns_per_buf = MAX_BALLOON_PAGES_PER_OP;
+
+	/*
+	 * Call the balloon in PAGE_SIZE*pfns-per-buf
+	 * iterations
+	 */
+	iterations = DIV_ROUND_UP(abspages, gfns_per_buf);
+	dprintk(&v->vdev->dev, "%s: iterations=%d\n", __func__, iterations);
+
+	for (i = 0; i < iterations; i++) {
+		int32_t pages_in_iteration = 
+			min(abspages - curr_pages, gfns_per_buf);
+
+		if (npages > 0)
+			r = kvm_balloon_inflate(v, pages_in_iteration);
+		else
+			r = kvm_balloon_deflate(v, pages_in_iteration);
+						
+		if (r < 0)
+			return r;
+		curr_pages += r;
+		if (r != pages_in_iteration)
+			break;
+		cond_resched();
+	}
+
+	return curr_pages;
+}
+
+static void inflate_done(struct virtballoon *v, struct balloon_buf *buf)
+{
+	uint8_t status = buf->hdr.status;
+
+	/* error inflating, return pages to the system */
+	if (status) {
+	 	struct page *page;
+		u32 *pfn = (u32 *)&buf->data;
+		int npages = (int)*pfn++;
+		int i;
+
+		spin_lock(&v->plist_lock);
+		for (i=0;i<npages;i++) { 	
+			page = pfn_to_page(*pfn);
+			list_del_init(&page->lru);
+			__free_page(page);
+			v->balloon_size--;
+			totalram_pages++;
+			v->target_nrpages++;
+			pfn++;
+		}
+		spin_unlock(&v->plist_lock);
+	}
+}
+
+static void deflate_done(struct virtballoon *v, struct balloon_buf *buf)
+{
+	uint8_t status = buf->hdr.status;
+
+	/* deflate OK, return pages to the system */
+	if (!status) { 
+		u32 *pfn = (u32 *)&buf->data;
+		int npages, i;
+
+		npages = (int)*pfn++;
+		npages = abs(npages);
+
+		for (i = 0; i<npages; i++) {
+			__free_page(pfn_to_page(*pfn));
+			pfn++;
+		}
+	/* deflate error, add pages back to ballooned list */
+	} else {
+		u32 *pfn = (u32 *)&buf->data;
+		int npages, i;
+		struct page *page;
+
+		npages = (int)*pfn++;
+		npages = abs(npages);
+
+		spin_lock(&v->plist_lock);
+		for (i = 0; i < npages; i++) {
+			page = pfn_to_page(*pfn++);
+			list_add(&page->lru, &v->balloon_plist);
+			v->balloon_size++;
+			totalram_pages--;
+			v->target_nrpages--;
+		}
+		spin_unlock(&v->plist_lock);
+	}
+	return;
+}
+
+static int balloon_thread(void *p)
+{
+	struct virtballoon *v = p;
+	DEFINE_WAIT(wait);
+	int rmmod = 0;
+
+	set_freezable();
+	while (!kthread_should_stop()) {
+		int delta;
+
+		prepare_to_wait(&v->balloon_wait, &wait, TASK_INTERRUPTIBLE);
+		schedule();
+		finish_wait(&v->balloon_wait, &wait);
+
+		try_to_freeze();
+
+		/* wait for kthread_stop() if rmmod has been called */
+		if (rmmod)
+			continue;
+
+		spin_lock_irq(&v->plist_lock);
+		delta = totalram_pages - v->target_nrpages;
+		spin_unlock_irq(&v->plist_lock);
+
+		if (delta)
+			kvm_balloon_xflate(v, delta);
+
+		spin_lock_irq(&v->queue_lock);
+		while (!list_empty(&v->balloon_work)) {
+			struct balloon_work *work;
+			struct balloon_buf *buf;
+
+			work = list_entry(v->balloon_work.next,
+					 struct balloon_work, list);
+			list_del(&work->list);
+			spin_unlock_irq(&v->queue_lock);
+			buf = work->buf;
+			kfree(work);
+
+			switch(buf->hdr.cmd) {
+			case CMD_BALLOON_DEFLATE:
+				deflate_done(v, buf);
+				break;
+			case CMD_BALLOON_INFLATE:
+				inflate_done(v, buf);
+				break;
+			default:
+				printk("%s: unknown cmd 0x%x\n", __func__,
+					buf->hdr.cmd);
+			}
+			kfree(buf);
+			if (atomic_dec_and_test(&v->inflight_bufs)) {
+				if (waitqueue_active(&v->rmmod_wait)) {
+					wake_up(&v->rmmod_wait);
+					rmmod = 1;
+				}
+			}
+			cond_resched();
+			spin_lock_irq(&v->queue_lock);
+		}
+		spin_unlock_irq(&v->queue_lock);
+	}
+	return 0;
+}
+
+static bool balloon_tx_done(struct virtqueue *vq)
+{
+	struct balloon_buf *buf;
+	struct virtballoon *v = vq->vdev->priv;
+	unsigned int len;
+
+	spin_lock(&v->queue_lock);
+	while ((buf = vq->vq_ops->get_buf(vq, &len)) != NULL) {
+		struct balloon_work *work;
+
+		work = kzalloc(sizeof(struct balloon_work), GFP_ATOMIC);
+		if (!work)
+			continue;
+		INIT_LIST_HEAD(&work->list);
+		work->buf = buf;
+
+		list_add(&work->list, &v->balloon_work);
+	}
+	spin_unlock(&v->queue_lock);
+	wake_up(&v->balloon_wait);
+
+	return true;
+}
+
+static struct virtio_device_id id_table[] = {
+	{ VIRTIO_ID_BALLOON, VIRTIO_DEV_ANY_ID},
+	{ 0 },
+};
+
+static LIST_HEAD(balloon_devices);
+
+static int balloon_probe(struct virtio_device *vdev)
+{
+	int err = -EINVAL;
+	struct virtballoon *v;
+
+	v = kzalloc(GFP_KERNEL, sizeof(struct virtballoon));
+	if (!v)
+		return -ENOMEM;
+
+	v->vq = vdev->config->find_vq(vdev, 0, balloon_tx_done);
+	if (IS_ERR(v->vq))
+		goto out_free;
+	
+	v->vdev = vdev;
+
+	init_waitqueue_head(&v->balloon_wait);
+	init_waitqueue_head(&v->rmmod_wait);
+	spin_lock_init(&v->plist_lock);
+	spin_lock_init(&v->queue_lock);
+	INIT_LIST_HEAD(&v->balloon_plist);
+	INIT_LIST_HEAD(&v->balloon_work);
+	INIT_LIST_HEAD(&v->list);
+	atomic_set(&v->inflight_bufs, 0);
+
+	vdev->priv = v;
+
+	v->balloon_thread = kthread_run(balloon_thread, v, "kvm_balloond");
+	if (IS_ERR(v->balloon_thread)) 
+		goto out_free_vq;
+
+	list_add(&v->list, &balloon_devices);
+
+	dev_printk(KERN_INFO, &v->vdev->dev, "registered\n");
+
+	return 0;
+
+out_free_vq:
+	vdev->config->del_vq(v->vq);
+out_free:
+	kfree(v);
+	return err;
+}
+
+static void balloon_remove(struct virtio_device *vdev)
+{
+	struct virtballoon *v = vdev->priv;
+
+	kthread_stop(v->balloon_thread);
+	vdev->config->del_vq(v->vq);
+	list_del(&v->list);
+	kfree(v);
+}
+
+static void balloon_config_changed(struct virtio_device *vdev)
+{
+	struct virtballoon *v = vdev->priv;
+
+	spin_lock(&v->plist_lock);
+	__virtio_config_val(v->vdev, 0, &v->target_nrpages);
+	spin_unlock(&v->plist_lock);
+	wake_up(&v->balloon_wait);
+	dprintk(&vdev->dev, "%s\n", __func__);
+}
+
+static struct virtio_driver virtio_balloon = {
+	.driver.name =  KBUILD_MODNAME,
+	.driver.owner = THIS_MODULE,
+	.id_table =     id_table,
+	.probe =        balloon_probe,
+	.remove =	__devexit_p(balloon_remove),
+	.config_changed	= balloon_config_changed,
+};
+
+module_param(kvm_balloon_debug, int, 0);
+
+static int __init kvm_balloon_init(void)
+{
+	return register_virtio_driver(&virtio_balloon);
+}
+
+static void __exit kvm_balloon_exit(void)
+{
+	struct virtballoon *v;
+	
+	list_for_each_entry(v, &balloon_devices, list) { 
+		spin_lock(&v->plist_lock);
+		if (v->balloon_size) {
+			DEFINE_WAIT(wait);
+
+			v->target_nrpages += v->balloon_size;
+			spin_unlock(&v->plist_lock);
+			wake_up(&v->balloon_wait);
+			prepare_to_wait(&v->rmmod_wait, &wait,
+					TASK_INTERRUPTIBLE);
+			schedule();
+			finish_wait(&v->rmmod_wait, &wait);
+			spin_lock(&v->plist_lock);
+		}
+
+		if (v->balloon_size)
+			dev_printk(KERN_ERR, &v->vdev->dev,
+				   "%s: exit while balloon not empty!\n",
+				    __func__);
+
+		spin_unlock(&v->plist_lock);
+	}
+
+	unregister_virtio_driver(&virtio_balloon);
+}
+
+module_init(kvm_balloon_init);
+module_exit(kvm_balloon_exit);

Index: linux-2.6-nv/drivers/virtio/virtio_pci.c
===================================================================

drivers/virtio/virtio_pci.c +1 -0

@@ -67,6 +67,7 @@ static struct pci_device_id virtio_pci_i
 	{ 0x1AF4, 0x1000, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, /* Dummy entry */
 	{ 0x1AF4, 0x1001, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, /* Dummy entry */
 	{ 0x1AF4, 0x1002, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, /* Dummy entry */
+	{ 0x1AF4, 0x1003, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, /* Balloon */
 	{ 0 },
 };
 

Index: linux-2.6-nv/include/linux/virtio_balloon.h
===================================================================

include/linux/virtio_balloon.h new file +20 -0

@@ -0,0 +1,20 @@
+#ifndef _LINUX_VIRTIO_BALLOON_H
+#define _LINUX_VIRTIO_BALLOON_H
+#include <linux/virtio_config.h>
+
+#define VIRTIO_ID_BALLOON 3
+
+#define CMD_BALLOON_INFLATE 0x1
+#define CMD_BALLOON_DEFLATE 0x2
+
+struct virtio_balloon_hdr {
+	uint8_t cmd;
+	uint8_t status;
+};
+
+struct virtio_balloon_config
+{
+	uint32_t target_nrpages;
+};
+
+#endif /* _LINUX_VIRTIO_BALLOON_H */

Quoted 31 lines · Marcelo Tosatti

Hi Rusty,

It was agreed that the balloon driver should be merged through the virtio tree, so here it goes. It depends on the config_changed patch posted earlier.

-----

Following patch adds the KVM balloon driver.

Changes from last version:
- Get rid of global variables/structure
- Use page->lru to link ballooned pages
- Use dev_dbg/dev_printk
- Proper kthread_should_stop handling
- Move shared definitions to separate header
- Use ->config_changed method for notification

This depends on Rusty's config_changed patch.

Signed-off-by Marcelo Tosatti <[email protected]>

Index: linux-2.6-nv/drivers/virtio/Kconfig
===================================================================

drivers/virtio/Kconfig +0 -0

@@ -23,3 +23,12 @@ config VIRTIO_PCI
 
 	  If unsure, say M.

drivers/virtio/Kconfig config VIRTIO_PCI · line 34

Quoted 10 lines

drivers/virtio/Kconfig +9 -0

 
+config KVM_BALLOON
+	tristate "KVM balloon driver (EXPERIMENTAL)"
+	depends on VIRTIO_PCI
+	---help---
+	 This driver provides support for ballooning memory in/out of a
+	 KVM paravirt guest.	
+
+	 If unsure, say M.
+

Please rename from KVM_BALLOON to VIRTIO_BALLOON. Also, it doesn't depend on VIRTIO_PCI. It should select VIRTIO and VIRTIO_RING.

Quoted 34 lines

Index: linux-2.6-nv/drivers/virtio/Makefile
===================================================================

drivers/virtio/Makefile +0 -0

@@ -1,3 +1,4 @@

obj-$(CONFIG_VIRTIO) += virtio.o
obj-$(CONFIG_VIRTIO_RING) += virtio_ring.o
obj-$(CONFIG_VIRTIO_PCI) += virtio_pci.o
+obj-$(CONFIG_KVM_BALLOON) += kvm_balloon.o
Index: linux-2.6-nv/drivers/virtio/kvm_balloon.c
===================================================================

drivers/virtio/kvm_balloon.c new file +5 -0

@@ -0,0 +1,537 @@
+/*
+ * KVM guest balloon driver
+ *
+ * Copyright (C) 2007, Qumranet, Inc., Dor Laor <[email protected]>
+ * Copyright (C) 2007, Red Hat, Inc., Marcelo Tosatti 

<[email protected]>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2. See
+ * the COPYING file in the top-level directory.
+ */

drivers/virtio/kvm_balloon.c new file +10 -0

+
+#define DEBUG
+#include <asm/uaccess.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/percpu.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/mm.h>
+#include <linux/swap.h>

drivers/virtio/kvm_balloon.c line 29

Quoted 10 lines

drivers/virtio/kvm_balloon.c new file +10 -0

+#include <linux/wait.h>
+#include <linux/kthread.h>
+#include <linux/freezer.h>
+#include <linux/version.h>
+#include <linux/virtio.h>
+#include <linux/virtio_config.h>
+#include <linux/virtio_balloon.h>
+#include <linux/preempt.h>
+#include <linux/kvm_types.h>
+#include <linux/kvm_host.h>

Please don't include kvm_types or kvm_host.

drivers/virtio/kvm_balloon.c line 38

Quoted 9 lines

drivers/virtio/kvm_balloon.c new file +9 -0

+
+MODULE_AUTHOR ("Dor Laor");
+MODULE_DESCRIPTION ("Implements guest ballooning support");
+MODULE_LICENSE("GPL");
+MODULE_VERSION("1");
+
+static int kvm_balloon_debug;
+
+#define dprintk(dev, str...) if (kvm_balloon_debug) dev_dbg(dev, str)

This can go away.

Hi Rusty,

It was agreed that the balloon driver should be merged through the virtio tree, so here it goes. It depends on the config_changed patch posted earlier.

drivers/virtio/kvm_balloon.c new file +5 -0

+static int kvm_balloon_inflate(struct virtballoon *v, int32_t npages)
+{
+	LIST_HEAD(tmp_list);
+	struct page *page, *tmp;
+	struct balloon_buf *buf;

Quoted 5 lines · On Tuesday 15 January 2008 07:03:57 Marcelo Tosatti wrote:

Hi Rusty,

It was agreed that the balloon driver should be merged through the virtio tree, so here it goes. It depends on the config_changed patch posted earlier.

Hi Marcelo,

Excellent! Although the main user will be kvm, it'd be nice to have a
demonstration in-tree using lguest; any chance of you conjuring up an appropriate patch?

If not, I can whip something up.

Review

Quoted 8 lines

(quoted diff) +8 -0

+config KVM_BALLOON
+	tristate "KVM balloon driver (EXPERIMENTAL)"
+	depends on VIRTIO_PCI
+	---help---
+	 This driver provides support for ballooning memory in/out of a
+	 KVM paravirt guest.
+
+	 If unsure, say M.

Please don't define "balloon" in terms of "ballooning". How about "This driver supports increasing and decreasing the amount of memory within a KVM guest." ?

Review

Quoted 1 line

(quoted diff) +1 -0

+	uint32_t target_nrpages;

I prefer u32 within the kernel, but no big deal.

Review

Quoted 11 lines

(quoted diff) +11 -0

+static int send_balloon_buf(struct virtballoon *v, uint8_t cmd,
+			    struct balloon_buf *buf)
+{
+	struct scatterlist sg[VIRTIO_MAX_SG];
+	int err = 0;
+
+	buf->hdr.cmd = cmd;
+
+	sg_init_table(sg, VIRTIO_MAX_SG);
+	sg_set_buf(&sg[0], &buf->hdr, sizeof(buf->hdr));
+	sg_set_buf(&sg[1], &buf->data, sizeof(buf->data));

Since these are adjacent, can't you just combine them into one sg element? Or does the kvm code rely on the sg as a boundary between header and data?

Quoted 5 lines

(quoted diff) +5 -0

+static int kvm_balloon_inflate(struct virtballoon *v, int32_t npages)
+{
+	LIST_HEAD(tmp_list);
+	struct page *page, *tmp;
+	struct balloon_buf *buf;

Review

Quoted 10 lines

(quoted diff) +10 -0

+	u32 *pfn;
+	int allocated = 0;
+	int i, r = -ENOMEM;
+
+	buf = alloc_balloon_buf(v->vdev, GFP_KERNEL);
+	if (!buf)
+		return r;
+
+	pfn = (u32 *)&buf->data;
+	*pfn++ = (u32)npages;

OK, this seems strange. You always use the data portion as an array of u32s, yet you declare it as char[200]. You have a perfectly good header, but you put the number of pages in the first element of the data array: and you hand the entire data array even though you normally only use part of it.

You can intuit the number from the sg len, I suggest you do that instead.

Looking at this driver, I just don't think this needs to be so complicated:
it's not a high speed device, after all. Perhaps allocate one buffer up front, and just reuse that. One simple thread loop, less locking needed.

Review

Quoted 9 lines

(quoted diff) +9 -0

+	for (i = 0; i < npages; i++) {
+		page = alloc_page(GFP_HIGHUSER | __GFP_NORETRY);
+		if (!page)
+			goto out_free;
+		list_add(&page->lru, &tmp_list);
+		allocated++;
+		*pfn = page_to_pfn(page);
+		pfn++;
+	}

I think it might be simpler to defer adding to any list until after the callback is done? Helpers to add an array to the balloon array (on successful inflate, or unsuccessful deflate) and to free them (unsuccessful inflate, or successful deflate) would also make the code more readable.

Review

Quoted 10 lines

(quoted diff) +10 -0

+	gfns_per_buf = MAX_BALLOON_PAGES_PER_OP;
+
+	/*
+	 * Call the balloon in PAGE_SIZE*pfns-per-buf
+	 * iterations
+	 */
+	iterations = DIV_ROUND_UP(abspages, gfns_per_buf);
+	dprintk(&v->vdev->dev, "%s: iterations=%d\n", __func__, iterations);
+
+	for (i = 0; i < iterations; i++) {

This logic seems overly complex. How about in the main thread:

	/* We're woken when target changes, in config_changed() */
	if (wait_event_interruptible(&v->wq,
		(diff = atomic_read(&v->target_pages) - total_pages)) == 0) {

		/* If we submit inflate/deflate request, wait for it to finish. */
		if (xflate(v, diff) == 0)
			wait_for_completion(&v->complete);
	}

xflate just does as much as it can (up to "diff"), and then the interrupt handler adds/removes from the linked list, frees pages, etc and fires the completion.

No need for locking, since there's only one pending request at any time.

OK, thats simpler. How about this:

[PATCH] Virtio balloon driver

Add a balloon driver for KVM, host<->guest communication is performed via virtio.

Signed-off-by Marcelo Tosatti <[email protected]>

drivers/virtio/kvm_balloon.c new file +29 -0

+	}
+}
+
+static void add_page_array(struct virtballoon *v, struct balloon_buf *buf,
+			   unsigned int npages)
+{
+	struct page *page;
+	u32 *pfn = (u32 *)&buf->data;
+	int i;
+
+	for (i=0; i<npages; i++) {
+		page = pfn_to_page(*pfn);
+		v->balloon_size++;
+		totalram_pages--;
+		list_add(&page->lru, &v->balloon_plist);
+		pfn++;
+	}
+}
+
+static void inflate_done(struct virtballoon *v, struct balloon_buf *buf,
+			 unsigned int npages)
+{
+	u8 status = buf->hdr.status;
+
+	/* inflate OK */
+	if (!status)
+		add_page_array(v, buf, npages);
+	else 
+		free_page_array(buf, npages);

drivers/virtio/kvm_balloon.c new file +6 -0

+		totalram_pages += npages;
+		v->balloon_size -= npages;
+	}
+	return;
+}
+

drivers/virtio/kvm_balloon.c new file +44 -0

+	__virtio_config_val(v->vdev, 0, &target_nrpages);
+	atomic_set(&v->target_nrpages, target_nrpages);
+	wake_up(&v->balloon_wait);
+	dprintk(&vdev->dev, "%s\n", __func__);
+}
+
+static struct virtio_driver virtio_balloon = {
+	.driver.name =  KBUILD_MODNAME,
+	.driver.owner = THIS_MODULE,
+	.id_table =     id_table,
+	.probe =        balloon_probe,
+	.remove =	__devexit_p(balloon_remove),
+	.config_changed	= balloon_config_changed,
+};
+
+module_param(kvm_balloon_debug, int, 0);
+
+static int __init kvm_balloon_init(void)
+{
+	return register_virtio_driver(&virtio_balloon);
+}
+
+static void __exit kvm_balloon_exit(void)
+{
+	struct virtballoon *v;
+	
+	list_for_each_entry(v, &balloon_devices, list) { 
+		while (v->balloon_size) {
+			DEFINE_WAIT(wait);
+
+			atomic_add(v->balloon_size, &v->target_nrpages);
+			wake_up(&v->balloon_wait);
+			prepare_to_wait(&v->rmmod_wait, &wait,
+					TASK_INTERRUPTIBLE);
+			schedule_timeout(HZ*10);
+			finish_wait(&v->rmmod_wait, &wait);
+		}
+	}
+
+	unregister_virtio_driver(&virtio_balloon);
+}
+
+module_init(kvm_balloon_init);
+module_exit(kvm_balloon_exit);

Index: linux-2.6-nv/drivers/virtio/virtio_pci.c
===================================================================

drivers/virtio/virtio_pci.c +1 -0

@@ -67,6 +67,7 @@ static struct pci_device_id virtio_pci_i
 	{ 0x1AF4, 0x1000, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, /* Dummy entry */
 	{ 0x1AF4, 0x1001, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, /* Dummy entry */
 	{ 0x1AF4, 0x1002, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, /* Dummy entry */
+	{ 0x1AF4, 0x1003, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, /* Balloon */
 	{ 0 },
 };
 

Index: linux-2.6-nv/include/linux/virtio_balloon.h
===================================================================

include/linux/virtio_balloon.h new file +20 -0

@@ -0,0 +1,20 @@
+#ifndef _LINUX_VIRTIO_BALLOON_H
+#define _LINUX_VIRTIO_BALLOON_H
+#include <linux/virtio_config.h>
+
+#define VIRTIO_ID_BALLOON 3
+
+#define CMD_BALLOON_INFLATE 0x1
+#define CMD_BALLOON_DEFLATE 0x2
+
+struct virtio_balloon_hdr {
+	__u8 cmd;
+	__u8 status;
+};
+
+struct virtio_balloon_config
+{
+	__u32 target_nrpages;
+};
+
+#endif /* _LINUX_VIRTIO_BALLOON_H */

_______________________________________________
Virtualization mailing list Virtualization-cunTk1MwBs9QetFLy7KEm3xJsTq8ys+cHZ5vskTnxNA@public.gmane.org
https://lists.linux-foundation.org/mailman/listinfo/virtualization

After discussions with Anthony Liguori, it seems that the virtio balloon can be made even simpler. Here's my attempt.

Since the balloon requires Guest cooperation anyway, there seems little reason to force it to tell the Host when it wants to reuse a page. It can simply fault it in.

Moreover, the target is best expressed in balloon size, since there is no portable way of getting the total RAM in the system. The host can do the math.

Tested with a (fairly hacky) lguest patch.

Signed-off-by Rusty Russell <rusty-8n+1lVoiYb80n/[email protected]>

---
 drivers/virtio/Kconfig          |   10 +
 drivers/virtio/Makefile         |    1
 drivers/virtio/virtio_balloon.c |  230 ++++++++++++++++++++++++++++++++++++++++
 include/linux/virtio_balloon.h  |   13 ++
 4 files changed, 254 insertions(+)

diff -r c4762959de25 drivers/virtio/Kconfig

drivers/virtio/Kconfig +10 -0

@@ -23,3 +23,13 @@ config VIRTIO_PCI
 
 	  If unsure, say M.
 
+config VIRTIO_BALLOON
+	tristate "Virtio balloon driver (EXPERIMENTAL)"
+	select VIRTIO
+	select VIRTIO_RING
+	---help---
+	 This driver supports increasing and decreasing the amount 
+	 of memory within a KVM guest.
+
+	 If unsure, say M.
+

diff -r c4762959de25 drivers/virtio/Makefile

drivers/virtio/Makefile +1 -0

@@ -1,3 +1,4 @@ obj-$(CONFIG_VIRTIO) += virtio.o
 obj-$(CONFIG_VIRTIO) += virtio.o
 obj-$(CONFIG_VIRTIO_RING) += virtio_ring.o
 obj-$(CONFIG_VIRTIO_PCI) += virtio_pci.o
+obj-$(CONFIG_VIRTIO_BALLOON) += virtio_balloon.o

diff -r c4762959de25 drivers/virtio/virtio_balloon.c

drivers/virtio/virtio_balloon.c new file +41 -0

@@ -0,0 +1,235 @@
+/* Virtio balloon implementation, inspired by Dor Loar and Marcelo
+ * Tosatti's implementations.
+ *
+ *  Copyright 2008 Rusty Russell IBM Corporation
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+#define DEBUG
+#include <linux/virtio.h>
+#include <linux/virtio_balloon.h>
+#include <linux/swap.h>
+#include <linux/kthread.h>
+#include <linux/freezer.h>
+
+struct virtio_balloon
+{
+	struct virtio_device *vdev;
+	struct virtqueue *vq;
+
+	/* Where the ballooning thread waits for config to change. */
+	wait_queue_head_t config_change;
+
+	/* The thread servicing the balloon. */
+	struct task_struct *thread;
+
+	/* Waiting for host to ack the pages we released. */
+	struct completion acked;
+
+	/* The pages we've told the Host we're not using. */

drivers/virtio/virtio_balloon.c new file +5 -0

+	{ 0 },
+};
+
+static void leak_balloon(struct virtio_balloon *vb, unsigned int num)
+{

drivers/virtio/virtio_balloon.c new file +169 -0

+	}
+}
+
+static void balloon_ack(struct virtqueue *vq)
+{
+	struct virtio_balloon *vb;
+	unsigned int len;
+
+	vb = vq->vq_ops->get_buf(vq, &len);
+	if (vb)
+		complete(&vb->acked);
+}
+
+static void fill_balloon(struct virtio_balloon *vb, unsigned int num)
+{
+	struct scatterlist sg;
+
+	/* We can only do one array worth at a time. */
+	num = min(num, ARRAY_SIZE(vb->pfns));
+
+	for (vb->num_pfns = 0; vb->num_pfns < num; vb->num_pfns++) {
+		struct page *page = alloc_page(GFP_HIGHUSER | __GFP_NORETRY);
+		if (!page) {
+			if (printk_ratelimit())
+				dev_printk(KERN_INFO, &vb->vdev->dev,
+					   "Out of puff! Can't get %u pages\n",
+					   num);
+			/* Sleep for at least 1/5 of a second before retry. */
+			msleep(200);
+			break;
+		}
+		vb->pfns[vb->num_pfns] = page_to_pfn(page);
+		totalram_pages--;
+		vb->num_pages++;
+		list_add(&page->lru, &vb->pages);
+	}
+
+	/* Didn't get any?  Oh well. */
+	if (vb->num_pfns == 0)
+		return;
+	
+	init_completion(&vb->acked);
+	sg_init_one(&sg, vb->pfns, sizeof(vb->pfns[0]) * vb->num_pfns);
+
+	/* We should always be able to add one buffer to an empty queue. */
+	if (vb->vq->vq_ops->add_buf(vb->vq, &sg, 1, 0, vb) != 0)
+		BUG();
+	vb->vq->vq_ops->kick(vb->vq);
+
+	/* When host has read buffer, this completes via balloon_ack */
+	wait_for_completion(&vb->acked);
+}
+
+static void virtballoon_changed(struct virtio_device *vdev)
+{
+	struct virtio_balloon *vb = vdev->priv;
+
+	wake_up(&vb->config_change);
+}
+
+static inline int towards_target(struct virtio_balloon *vb)
+{
+	u32 v;
+	__virtio_config_val(vb->vdev,
+			    offsetof(struct virtio_balloon_config, num_pages),
+			    &v);
+	return v - vb->num_pages;
+}
+
+static int balloon(void *_vballoon)
+{
+	struct virtio_balloon *vb = _vballoon;
+
+	set_freezable();
+	while (!kthread_should_stop()) {
+		int diff;
+
+		try_to_freeze();
+		wait_event_interruptible(vb->config_change,
+					 (diff = towards_target(vb)) != 0
+					 || kthread_should_stop());
+		if (diff > 0)
+			fill_balloon(vb, diff);
+		else if (diff < 0)
+			leak_balloon(vb, -diff);
+	}
+	return 0;
+}
+
+static int virtballoon_probe(struct virtio_device *vdev)
+{
+	struct virtio_balloon *vb;
+	int err;
+
+	vdev->priv = vb = kmalloc(sizeof(*vb), GFP_KERNEL);
+	if (!vb) {
+		err = -ENOMEM;
+		goto out;
+	}
+
+	INIT_LIST_HEAD(&vb->pages);
+	vb->num_pages = 0;
+	init_waitqueue_head(&vb->config_change);
+	vb->vdev = vdev;
+
+	/* We expect one virtqueue, for output. */
+	vb->vq = vdev->config->find_vq(vdev, 0, balloon_ack);
+	if (IS_ERR(vb->vq)) {
+		err = PTR_ERR(vb->vq);
+		goto out_free_vb;
+	}
+
+	vb->thread = kthread_run(balloon, vb, "vballoon");
+	if (IS_ERR(vb->thread)) {
+		err = PTR_ERR(vb->thread);
+		goto out_del_vq;
+	}
+	return 0;
+
+out_del_vq:
+	vdev->config->del_vq(vb->vq);
+out_free_vb:
+	kfree(vb);
+out:
+	return err;
+}
+
+static void virtballoon_remove(struct virtio_device *vdev)
+{
+	struct virtio_balloon *vb = vdev->priv;
+	struct page *i, *next;
+
+	kthread_stop(vb->thread);
+
+	/* There might be pages left in the balloon: free them. */
+	list_for_each_entry_safe(i, next, &vb->pages, lru) {
+		vb->num_pages--;
+		__free_page(i);
+	}
+	BUG_ON(vb->num_pages);
+
+	vdev->config->del_vq(vb->vq);
+	kfree(vb);
+}
+
+static struct virtio_driver virtio_balloon = {
+	.driver.name =	KBUILD_MODNAME,
+	.driver.owner =	THIS_MODULE,
+	.id_table =	id_table,
+	.probe =	virtballoon_probe,
+	.remove =	__devexit_p(virtballoon_remove),
+	.config_changed = virtballoon_changed,
+};
+
+static int __init init(void)
+{
+	return register_virtio_driver(&virtio_balloon);
+}
+
+static void __exit fini(void)
+{
+	unregister_virtio_driver(&virtio_balloon);
+}
+module_init(init);
+module_exit(fini);
+
+MODULE_DEVICE_TABLE(virtio, id_table);
+MODULE_DESCRIPTION("Virtio balloon driver");
+MODULE_LICENSE("GPL");

diff -r c4762959de25 include/linux/virtio_balloon.h

include/linux/virtio_balloon.h new file +13 -1

@@ -0,0 +1,13 @@
+#ifndef _LINUX_VIRTIO_BALLOON_H
+#define _LINUX_VIRTIO_BALLOON_H
+#include <linux/virtio_config.h>
+
+/* The ID for virtio_balloon */
+#define VIRTIO_ID_BALLOON	5
+
+struct virtio_balloon_config
+{
+	/* Number of pages host wants Guest to give up. */
+	__le32 num_pages;
+};
+#endif /* _LINUX_VIRTIO_BALLOON_H */
 
-------------------------------------------------------------------------

This SF.net email is sponsored by: Microsoft
Defy all challenges. Microsoft(R) Visual Studio 2008.
http://clk.atdmt.com/MRT/go/vse0120000070mrt/direct/01/
_______________________________________________
kvm-devel mailing list [email protected]
https://lists.sourceforge.net/lists/listinfo/kvm-devel

drivers/virtio/virtio_balloon.c line 51

Quoted 2 lines · Rusty Russell

drivers/virtio/virtio_balloon.c new file +2 -0

+static struct virtio_device_id id_table[] = {
+	{ VIRTIO_ID_BALLOON, VIRTIO_DEV_ANY_ID},

Could use a space after VIRTIO_DEV_ANY_ID

drivers/virtio/virtio_balloon.c line 66

Quoted 3 lines

drivers/virtio/virtio_balloon.c new file +3 -0

+		__free_page(page);
+		vb->num_pages--;
+		totalram_pages++;

Do we really want to modify totalram_pages in this driver? The only other place that I see that modifies it is in mm/memory_hotplug and it also modifies other things (like num_physpages). The cmm driver doesn't touch totalram_pages.

It would be very useful too to write vb->num_pages into the config space whenever it was updated. This way, the host can easily keep track of where the guest is at in terms of ballooning.

Quoted 7 lines · On Thursday 17 January 2008 13:14:58 Anthony Liguori wrote:

drivers/virtio/virtio_balloon.c line 51

Quoted 3 lines · Rusty Russell

drivers/virtio/virtio_balloon.c new file +2 -0

+static struct virtio_device_id id_table[] = {
+	{ VIRTIO_ID_BALLOON, VIRTIO_DEV_ANY_ID},

Could use a space after VIRTIO_DEV_ANY_ID

Thanks, fixed.

Quoted 9 lines

drivers/virtio/virtio_balloon.c line 66

Quoted 4 lines

drivers/virtio/virtio_balloon.c new file +3 -0

+		__free_page(page);
+		vb->num_pages--;
+		totalram_pages++;

Do we really want to modify totalram_pages in this driver? The only other place that I see that modifies it is in mm/memory_hotplug and it also modifies other things (like num_physpages). The cmm driver doesn't touch totalram_pages.

I don't think there's a standard here, they're all ad-hoc (eg. no locking) Modifying totalram_pages has the nice effect of showing up in "free" in the guest.

We should probably not modify num_physpages, because some places seem to use it as an address space limit. But we should probably fix all those networking size heuristics to use totalram_pages instead of num_physpages.

Quoted 4 lines

It would be very useful too to write vb->num_pages into the config space whenever it was updated. This way, the host can easily keep track of where the guest is at in terms of ballooning.

OTOH it's currently pretty obvious (and usually fatal) if the guest has trouble meeting the balloon requirements. A serious host needs a way of detecting stress in the guest anyway, which this doesn't offer until it's too late...

Rusty.

Quoted 4 lines · Rusty Russell

OTOH it's currently pretty obvious (and usually fatal) if the guest has trouble meeting the balloon requirements. A serious host needs a way of detecting stress in the guest anyway, which this doesn't offer until it's too late...

The question I'm interested in answering though is not if but when. I would like to know when the guest has reached it's target.

And while we do get the madvise call outs, it's possible that pages have been faulted in since then.

Quoted 7 lines

Quoted 7 lines

It would be very useful too to write vb->num_pages into the config space whenever it was updated. This way, the host can easily keep track of where the guest is at in terms of ballooning.

Quoted 6 lines

OTOH it's currently pretty obvious (and usually fatal) if the guest has trouble meeting the balloon requirements. A serious host needs a way of detecting stress in the guest anyway, which this doesn't offer until it's too late...

The question I'm interested in answering though is not if but when. I would like to know when the guest has reached it's target.

Quoted 4 lines · Rusty Russell

OTOH it's currently pretty obvious (and usually fatal) if the guest has trouble meeting the balloon requirements. A serious host needs a way of detecting stress in the guest anyway, which this doesn't offer until it's too late...

The question I'm interested in answering though is not if but when. I would like to know when the guest has reached it's target.

And while we do get the madvise call outs, it's possible that pages have been faulted in since then.

Quoted 7 lines

Quoted 7 lines

It would be very useful too to write vb->num_pages into the config space whenever it was updated. This way, the host can easily keep track of where the guest is at in terms of ballooning.

Quoted 3 lines

OTOH it's currently pretty obvious (and usually fatal) if the guest has trouble meeting the balloon requirements. A serious host needs a way of detecting stress in the guest anyway, which this doesn't offer until it's too late...

The question I'm interested in answering though is not if but when. I would like to know when the guest has reached it's target.

Quoted 7 lines · On Thursday 17 January 2008 13:14:58 Anthony Liguori wrote:

drivers/virtio/virtio_balloon.c line 51

Quoted 3 lines · Rusty Russell

drivers/virtio/virtio_balloon.c new file +2 -0

+static struct virtio_device_id id_table[] = {
+	{ VIRTIO_ID_BALLOON, VIRTIO_DEV_ANY_ID},

Could use a space after VIRTIO_DEV_ANY_ID

Thanks, fixed.

Quoted 9 lines

drivers/virtio/virtio_balloon.c line 66

Quoted 4 lines

drivers/virtio/virtio_balloon.c new file +3 -0

+		__free_page(page);
+		vb->num_pages--;
+		totalram_pages++;

Do we really want to modify totalram_pages in this driver? The only other place that I see that modifies it is in mm/memory_hotplug and it also modifies other things (like num_physpages). The cmm driver doesn't touch totalram_pages.

I don't think there's a standard here, they're all ad-hoc (eg. no locking) Modifying totalram_pages has the nice effect of showing up in "free" in the guest.

We should probably not modify num_physpages, because some places seem to use it as an address space limit. But we should probably fix all those networking size heuristics to use totalram_pages instead of num_physpages.

Quoted 4 lines

It would be very useful too to write vb->num_pages into the config space whenever it was updated. This way, the host can easily keep track of where the guest is at in terms of ballooning.

OTOH it's currently pretty obvious (and usually fatal) if the guest has trouble meeting the balloon requirements. A serious host needs a way of detecting stress in the guest anyway, which this doesn't offer until it's too late...

Rusty.

drivers/virtio/virtio_balloon.c line 51

Quoted 2 lines · Rusty Russell

drivers/virtio/virtio_balloon.c new file +2 -0

+static struct virtio_device_id id_table[] = {
+	{ VIRTIO_ID_BALLOON, VIRTIO_DEV_ANY_ID},

Could use a space after VIRTIO_DEV_ANY_ID

drivers/virtio/virtio_balloon.c line 66

Quoted 3 lines

drivers/virtio/virtio_balloon.c new file +3 -0

+		__free_page(page);
+		vb->num_pages--;
+		totalram_pages++;

Do we really want to modify totalram_pages in this driver? The only other place that I see that modifies it is in mm/memory_hotplug and it also modifies other things (like num_physpages). The cmm driver doesn't touch totalram_pages.

It would be very useful too to write vb->num_pages into the config space whenever it was updated. This way, the host can easily keep track of where the guest is at in terms of ballooning.

Since the balloon requires Guest cooperation anyway, there seems little reason to force it to tell the Host when it wants to reuse a page. It can simply fault it in.

Am Donnerstag, 17. Januar 2008 schrieb Rusty Russell:

Quoted 3 lines

Since the balloon requires Guest cooperation anyway, there seems little reason to force it to tell the Host when it wants to reuse a page. It can simply fault it in.

Yes, thats what we do in the s390 cmm driver.

All in all the driver has similarities with cmm. I dont know, if we can consolidate some code.
Besides the hypervisor, we have a additional user interface: /proc/sys/vm/. The root user can specify the amount of pages in the balloon via /proc/sys/vm/cmm_pages.

Another idea: Martin added an oom notifier to the cmm driver. Before the oom-killer kicks in cmm will try to free 256 pages. I think your virtio balloon driver should do the same - it seems to be the correct tradeoff.

Quoted 4 lines

Another idea: Martin added an oom notifier to the cmm driver. Before the oom-killer kicks in cmm will try to free 256 pages. I think your virtio balloon driver should do the same - it seems to be the correct tradeoff.

Nod, you definitly want to add an oom notifier. If 256 pages is the correct number of pages to free is debatable. We have seen long delays for a process that quickly eats up memory if there are lots of pages in the balloon. The problem is that the memory management tries hard to find memory until it decides to oom kill a process, only to be stopped in the last second by the oom notifier. The 1MB is quickly eaten up again so the whole things starts again. The profile of such a scenario shows that almost all cpu is burned in the page reclaim code.

Quoted 12 lines · Martin Schwidefsky

Quoted 2 lines

Another idea: Martin added an oom notifier to the cmm driver. Before the oom-killer kicks in cmm will try to free 256 pages. I think your virtio balloon driver should do the same - it seems to be the correct tradeoff.

Nod, you definitly want to add an oom notifier. If 256 pages is the correct number of pages to free is debatable. We have seen long delays for a process that quickly eats up memory if there are lots of pages in the balloon. The problem is that the memory management tries hard to find memory until it decides to oom kill a process, only to be stopped in the last second by the oom notifier. The 1MB is quickly eaten up again so the whole things starts again. The profile of such a scenario shows that almost all cpu is burned in the page reclaim code.

Seconded, in that case we can add a config space notification from the guest to the host that will be triggered by the oom.
The host will get this notification and will decide whether to allow the guest to deflate the balloon or to keep the current balloon size because the whole host is over committed.

Regards,
Dor.

Quoted 17 lines · Dor Laor

Quoted 4 lines · Martin Schwidefsky

Quoted 4 lines

Another idea: Martin added an oom notifier to the cmm driver. Before the oom-killer kicks in cmm will try to free 256 pages. I think your virtio balloon driver should do the same - it seems to be the correct tradeoff.

Quoted 10 lines

Nod, you definitly want to add an oom notifier. If 256 pages is the correct number of pages to free is debatable. We have seen long delays for a process that quickly eats up memory if there are lots of pages in the balloon. The problem is that the memory management tries hard to find memory until it decides to oom kill a process, only to be stopped in the last second by the oom notifier. The 1MB is quickly eaten up again so the whole things starts again. The profile of such a scenario shows that almost all cpu is burned in the page reclaim code.

Quoted 6 lines

Seconded, in that case we can add a config space notification from the guest to the host that will be triggered by the oom.
The host will get this notification and will decide whether to allow the guest to deflate the balloon or to keep the current balloon size because the whole host is over committed.

The host doesn't decide whether to allow the guest to deflate. Virtual memory size and resident memory side are independent in KVM. The host decides what the RSS is but the guest is free to determine it's VSS. The host provides hints to the guest about it's RSS size (via ballooning) so the guest can optimize it's VSS size. If guest chooses to make it's VSS size to large, it will only hurt itself (by being forced to swap by the host).

Quoted 28 lines · Anthony Liguori

Quoted 19 lines · Dor Laor

Quoted 5 lines · Martin Schwidefsky

Quoted 5 lines

Another idea: Martin added an oom notifier to the cmm driver. Before the oom-killer kicks in cmm will try to free 256 pages. I think your virtio balloon driver should do the same - it seems to be the correct tradeoff.

Quoted 11 lines

Nod, you definitly want to add an oom notifier. If 256 pages is the correct number of pages to free is debatable. We have seen long delays for a process that quickly eats up memory if there are lots of pages in the balloon. The problem is that the memory management tries hard to find memory until it decides to oom kill a process, only to be stopped in the last second by the oom notifier. The 1MB is quickly eaten up again so the whole things starts again. The profile of such a scenario shows that almost all cpu is burned in the page reclaim code.

Quoted 7 lines

Seconded, in that case we can add a config space notification from the guest to the host that will be triggered by the oom.
The host will get this notification and will decide whether to allow the guest to deflate the balloon or to keep the current balloon size because the whole host is over committed.

Quoted 8 lines

The host doesn't decide whether to allow the guest to deflate. Virtual memory size and resident memory side are independent in KVM. The host decides what the RSS is but the guest is free to determine it's VSS. The host provides hints to the guest about it's RSS size (via ballooning) so the guest can optimize it's VSS size. If guest chooses to make it's VSS size to large, it will only hurt itself (by being forced to swap by the host).

ok, this is true when the new memory controller targeted for the next kernel.In that case I more then agree.
Nevertheless, the guest should pass the host data about it's current memory usage (working set size, oom triggering, etc) so the host would change the rss limit dynamically.

Thanks,
Dor

Quoted 26 lines · Anthony Liguori

Quoted 24 lines · Dor Laor

Quoted 14 lines · Martin Schwidefsky

Quoted 3 lines

Another idea: Martin added an oom notifier to the cmm driver. Before the oom-killer kicks in cmm will try to free 256 pages. I think your virtio balloon driver should do the same - it seems to be the correct tradeoff.

Nod, you definitly want to add an oom notifier. If 256 pages is the correct number of pages to free is debatable. We have seen long delays for a process that quickly eats up memory if there are lots of pages in the balloon. The problem is that the memory management tries hard to find memory until it decides to oom kill a process, only to be stopped in the last second by the oom notifier. The 1MB is quickly eaten up again so the whole things starts again. The profile of such a scenario shows that almost all cpu is burned in the page reclaim code.

Seconded, in that case we can add a config space notification from the guest to the host that will be triggered by the oom.
The host will get this notification and will decide whether to allow the guest to deflate the balloon or to keep the current balloon size because the whole host is over committed.

Quoted 2 lines

The host doesn't decide whether to allow the guest to deflate. Virtual memory size and resident memory side are independent in KVM. The host decides what the RSS is but the guest is free to determine it's VSS. The host provides hints to the guest about it's RSS size (via ballooning) so the guest can optimize it's VSS size. If guest chooses to make it's VSS size to large, it will only hurt itself (by being forced to swap by the host).

ok, this is true when the new memory controller targeted for the next
kernel.In that case I more then agree.
Nevertheless, the guest should pass the host data about it's current
memory usage (working set size, oom triggering, etc) so the host would change the rss limit dynamically.

Thanks,
Dor

Quoted 17 lines · Dor Laor

Quoted 4 lines · Martin Schwidefsky

Quoted 4 lines

Another idea: Martin added an oom notifier to the cmm driver. Before the oom-killer kicks in cmm will try to free 256 pages. I think your virtio balloon driver should do the same - it seems to be the correct tradeoff.

Quoted 10 lines

Nod, you definitly want to add an oom notifier. If 256 pages is the correct number of pages to free is debatable. We have seen long delays for a process that quickly eats up memory if there are lots of pages in the balloon. The problem is that the memory management tries hard to find memory until it decides to oom kill a process, only to be stopped in the last second by the oom notifier. The 1MB is quickly eaten up again so the whole things starts again. The profile of such a scenario shows that almost all cpu is burned in the page reclaim code.

Quoted 6 lines

Seconded, in that case we can add a config space notification from the guest to the host that will be triggered by the oom.
The host will get this notification and will decide whether to allow the guest to deflate the balloon or to keep the current balloon size because the whole host is over committed.

The host doesn't decide whether to allow the guest to deflate. Virtual memory size and resident memory side are independent in KVM. The host decides what the RSS is but the guest is free to determine it's VSS. The host provides hints to the guest about it's RSS size (via ballooning) so the guest can optimize it's VSS size. If guest chooses to make it's VSS size to large, it will only hurt itself (by being forced to swap by the host).

Quoted 4 lines

Another idea: Martin added an oom notifier to the cmm driver. Before the oom-killer kicks in cmm will try to free 256 pages. I think your virtio balloon driver should do the same - it seems to be the correct tradeoff.

Nod, you definitly want to add an oom notifier. If 256 pages is the correct number of pages to free is debatable. We have seen long delays for a process that quickly eats up memory if there are lots of pages in the balloon. The problem is that the memory management tries hard to find memory until it decides to oom kill a process, only to be stopped in the last second by the oom notifier. The 1MB is quickly eaten up again so the whole things starts again. The profile of such a scenario shows that almost all cpu is burned in the page reclaim code.

Am Donnerstag, 17. Januar 2008 schrieb Rusty Russell:

Quoted 3 lines

Since the balloon requires Guest cooperation anyway, there seems little reason to force it to tell the Host when it wants to reuse a page. It can simply fault it in.

Yes, thats what we do in the s390 cmm driver.

All in all the driver has similarities with cmm. I dont know, if we can consolidate some code.
Besides the hypervisor, we have a additional user interface: /proc/sys/vm/. The root user can specify the amount of pages in the balloon via /proc/sys/vm/cmm_pages.

Another idea: Martin added an oom notifier to the cmm driver. Before the oom-killer kicks in cmm will try to free 256 pages. I think your virtio balloon driver should do the same - it seems to be the correct tradeoff.

After discussions with Anthony Liguori, it seems that the virtio balloon can be made even simpler. Here's my attempt.

Since the balloon requires Guest cooperation anyway, there seems little reason to force it to tell the Host when it wants to reuse a page. It can simply fault it in.

After discussions with Anthony Liguori, it seems that the virtio balloon can be made even simpler. Here's my attempt.

Since the balloon requires Guest cooperation anyway, there seems little reason to force it to tell the Host when it wants to reuse a page. It can simply fault it in.

Moreover, the target is best expressed in balloon size, since there is no portable way of getting the total RAM in the system. The host can do the math.

Tested with a (fairly hacky) lguest patch.

Signed-off-by Rusty Russell <[email protected]>

---
 drivers/virtio/Kconfig          |   10 +
 drivers/virtio/Makefile         |    1
 drivers/virtio/virtio_balloon.c |  230 ++++++++++++++++++++++++++++++++++++++++
 include/linux/virtio_balloon.h  |   13 ++
 4 files changed, 254 insertions(+)

diff -r c4762959de25 drivers/virtio/Kconfig

drivers/virtio/Kconfig +10 -0

@@ -23,3 +23,13 @@ config VIRTIO_PCI
 
 	  If unsure, say M.
 
+config VIRTIO_BALLOON
+	tristate "Virtio balloon driver (EXPERIMENTAL)"
+	select VIRTIO
+	select VIRTIO_RING
+	---help---
+	 This driver supports increasing and decreasing the amount 
+	 of memory within a KVM guest.
+
+	 If unsure, say M.
+

diff -r c4762959de25 drivers/virtio/Makefile

drivers/virtio/Makefile +1 -0

@@ -1,3 +1,4 @@ obj-$(CONFIG_VIRTIO) += virtio.o
 obj-$(CONFIG_VIRTIO) += virtio.o
 obj-$(CONFIG_VIRTIO_RING) += virtio_ring.o
 obj-$(CONFIG_VIRTIO_PCI) += virtio_pci.o
+obj-$(CONFIG_VIRTIO_BALLOON) += virtio_balloon.o

diff -r c4762959de25 drivers/virtio/virtio_balloon.c

drivers/virtio/virtio_balloon.c new file +41 -0

@@ -0,0 +1,235 @@
+/* Virtio balloon implementation, inspired by Dor Loar and Marcelo
+ * Tosatti's implementations.
+ *
+ *  Copyright 2008 Rusty Russell IBM Corporation
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+#define DEBUG
+#include <linux/virtio.h>
+#include <linux/virtio_balloon.h>
+#include <linux/swap.h>
+#include <linux/kthread.h>
+#include <linux/freezer.h>
+
+struct virtio_balloon
+{
+	struct virtio_device *vdev;
+	struct virtqueue *vq;
+
+	/* Where the ballooning thread waits for config to change. */
+	wait_queue_head_t config_change;
+
+	/* The thread servicing the balloon. */
+	struct task_struct *thread;
+
+	/* Waiting for host to ack the pages we released. */
+	struct completion acked;
+
+	/* The pages we've told the Host we're not using. */

drivers/virtio/virtio_balloon.c new file +5 -0

+	{ 0 },
+};
+
+static void leak_balloon(struct virtio_balloon *vb, unsigned int num)
+{

drivers/virtio/virtio_balloon.c new file +169 -0

+	}
+}
+
+static void balloon_ack(struct virtqueue *vq)
+{
+	struct virtio_balloon *vb;
+	unsigned int len;
+
+	vb = vq->vq_ops->get_buf(vq, &len);
+	if (vb)
+		complete(&vb->acked);
+}
+
+static void fill_balloon(struct virtio_balloon *vb, unsigned int num)
+{
+	struct scatterlist sg;
+
+	/* We can only do one array worth at a time. */
+	num = min(num, ARRAY_SIZE(vb->pfns));
+
+	for (vb->num_pfns = 0; vb->num_pfns < num; vb->num_pfns++) {
+		struct page *page = alloc_page(GFP_HIGHUSER | __GFP_NORETRY);
+		if (!page) {
+			if (printk_ratelimit())
+				dev_printk(KERN_INFO, &vb->vdev->dev,
+					   "Out of puff! Can't get %u pages\n",
+					   num);
+			/* Sleep for at least 1/5 of a second before retry. */
+			msleep(200);
+			break;
+		}
+		vb->pfns[vb->num_pfns] = page_to_pfn(page);
+		totalram_pages--;
+		vb->num_pages++;
+		list_add(&page->lru, &vb->pages);
+	}
+
+	/* Didn't get any?  Oh well. */
+	if (vb->num_pfns == 0)
+		return;
+	
+	init_completion(&vb->acked);
+	sg_init_one(&sg, vb->pfns, sizeof(vb->pfns[0]) * vb->num_pfns);
+
+	/* We should always be able to add one buffer to an empty queue. */
+	if (vb->vq->vq_ops->add_buf(vb->vq, &sg, 1, 0, vb) != 0)
+		BUG();
+	vb->vq->vq_ops->kick(vb->vq);
+
+	/* When host has read buffer, this completes via balloon_ack */
+	wait_for_completion(&vb->acked);
+}
+
+static void virtballoon_changed(struct virtio_device *vdev)
+{
+	struct virtio_balloon *vb = vdev->priv;
+
+	wake_up(&vb->config_change);
+}
+
+static inline int towards_target(struct virtio_balloon *vb)
+{
+	u32 v;
+	__virtio_config_val(vb->vdev,
+			    offsetof(struct virtio_balloon_config, num_pages),
+			    &v);
+	return v - vb->num_pages;
+}
+
+static int balloon(void *_vballoon)
+{
+	struct virtio_balloon *vb = _vballoon;
+
+	set_freezable();
+	while (!kthread_should_stop()) {
+		int diff;
+
+		try_to_freeze();
+		wait_event_interruptible(vb->config_change,
+					 (diff = towards_target(vb)) != 0
+					 || kthread_should_stop());
+		if (diff > 0)
+			fill_balloon(vb, diff);
+		else if (diff < 0)
+			leak_balloon(vb, -diff);
+	}
+	return 0;
+}
+
+static int virtballoon_probe(struct virtio_device *vdev)
+{
+	struct virtio_balloon *vb;
+	int err;
+
+	vdev->priv = vb = kmalloc(sizeof(*vb), GFP_KERNEL);
+	if (!vb) {
+		err = -ENOMEM;
+		goto out;
+	}
+
+	INIT_LIST_HEAD(&vb->pages);
+	vb->num_pages = 0;
+	init_waitqueue_head(&vb->config_change);
+	vb->vdev = vdev;
+
+	/* We expect one virtqueue, for output. */
+	vb->vq = vdev->config->find_vq(vdev, 0, balloon_ack);
+	if (IS_ERR(vb->vq)) {
+		err = PTR_ERR(vb->vq);
+		goto out_free_vb;
+	}
+
+	vb->thread = kthread_run(balloon, vb, "vballoon");
+	if (IS_ERR(vb->thread)) {
+		err = PTR_ERR(vb->thread);
+		goto out_del_vq;
+	}
+	return 0;
+
+out_del_vq:
+	vdev->config->del_vq(vb->vq);
+out_free_vb:
+	kfree(vb);
+out:
+	return err;
+}
+
+static void virtballoon_remove(struct virtio_device *vdev)
+{
+	struct virtio_balloon *vb = vdev->priv;
+	struct page *i, *next;
+
+	kthread_stop(vb->thread);
+
+	/* There might be pages left in the balloon: free them. */
+	list_for_each_entry_safe(i, next, &vb->pages, lru) {
+		vb->num_pages--;
+		__free_page(i);
+	}
+	BUG_ON(vb->num_pages);
+
+	vdev->config->del_vq(vb->vq);
+	kfree(vb);
+}
+
+static struct virtio_driver virtio_balloon = {
+	.driver.name =	KBUILD_MODNAME,
+	.driver.owner =	THIS_MODULE,
+	.id_table =	id_table,
+	.probe =	virtballoon_probe,
+	.remove =	__devexit_p(virtballoon_remove),
+	.config_changed = virtballoon_changed,
+};
+
+static int __init init(void)
+{
+	return register_virtio_driver(&virtio_balloon);
+}
+
+static void __exit fini(void)
+{
+	unregister_virtio_driver(&virtio_balloon);
+}
+module_init(init);
+module_exit(fini);
+
+MODULE_DEVICE_TABLE(virtio, id_table);
+MODULE_DESCRIPTION("Virtio balloon driver");
+MODULE_LICENSE("GPL");

diff -r c4762959de25 include/linux/virtio_balloon.h

include/linux/virtio_balloon.h new file +13 -1

@@ -0,0 +1,13 @@
+#ifndef _LINUX_VIRTIO_BALLOON_H
+#define _LINUX_VIRTIO_BALLOON_H
+#include <linux/virtio_config.h>
+
+/* The ID for virtio_balloon */
+#define VIRTIO_ID_BALLOON	5
+
+struct virtio_balloon_config
+{
+	/* Number of pages host wants Guest to give up. */
+	__le32 num_pages;
+};
+#endif /* _LINUX_VIRTIO_BALLOON_H */
 
-------------------------------------------------------------------------

This SF.net email is sponsored by: Microsoft Defy all challenges. Microsoft(R) Visual Studio 2008.
http://clk.atdmt.com/MRT/go/vse0120000070mrt/direct/01/
_______________________________________________
kvm-devel mailing list [email protected]
https://lists.sourceforge.net/lists/listinfo/kvm-devel

Since the balloon requires Guest cooperation anyway, there seems little reason to force it to tell the Host when it wants to reuse a page. It can simply fault it in.

After discussions with Anthony Liguori, it seems that the virtio balloon can be made even simpler. Here's my attempt.

Since the balloon requires Guest cooperation anyway, there seems little reason to force it to tell the Host when it wants to reuse a page. It can simply fault it in.

Quoted 8 lines · Rusty Russell

After discussions with Anthony Liguori, it seems that the virtio balloon can be made even simpler. Here's my attempt.

Since the balloon requires Guest cooperation anyway, there seems little reason to force it to tell the Host when it wants to reuse a page. It can simply fault it in.

Faulting is synchronous, while deflating is (or can be made) asynchronous. If the host needs to do some work to get the memory, the guest will be slowed down considerably.

If we have explicit deflate, the host can call madvise(MADV_WILLNEED) or actually touch the pages before the guest accesses them.

Quoted 10 lines · Avi Kivity

Quoted 8 lines · Rusty Russell

After discussions with Anthony Liguori, it seems that the virtio balloon can be made even simpler. Here's my attempt.

Since the balloon requires Guest cooperation anyway, there seems little reason to force it to tell the Host when it wants to reuse a page. It can simply fault it in.

Quoted 3 lines

Faulting is synchronous, while deflating is (or can be made) asynchronous. If the host needs to do some work to get the memory, the guest will be slowed down considerably.

Good point. Basically, we have two page hinting operations which roughly correspond to madvise(MADV_DONTNEED) and madvise(MADV_WILLNEED).

Quoted 17 lines · Anthony Liguori

Quoted 15 lines · Avi Kivity

Quoted 9 lines · Rusty Russell

After discussions with Anthony Liguori, it seems that the virtio balloon can be made even simpler. Here's my attempt.

Since the balloon requires Guest cooperation anyway, there seems little reason to force it to tell the Host when it wants to reuse a page. It can simply fault it in.

Faulting is synchronous, while deflating is (or can be made) asynchronous. If the host needs to do some work to get the memory, the guest will be slowed down considerably.

Quoted 3 lines

Good point. Basically, we have two page hinting operations which roughly correspond to madvise(MADV_DONTNEED) and madvise(MADV_WILLNEED).

Also, the simplified driver does not handle errors at all.

I don't think that assuming madvise() can't fail is a good thing.

Quoted 13 lines · Anthony Liguori

Quoted 11 lines · Avi Kivity

Quoted 9 lines · Rusty Russell

After discussions with Anthony Liguori, it seems that the virtio balloon can be made even simpler. Here's my attempt.

Since the balloon requires Guest cooperation anyway, there seems little reason to force it to tell the Host when it wants to reuse a page. It can simply fault it in.

Quoted 4 lines

Quoted 2 lines

Faulting is synchronous, while deflating is (or can be made) asynchronous. If the host needs to do some work to get the memory, the guest will be slowed down considerably.

Good point. Basically, we have two page hinting operations which roughly correspond to madvise(MADV_DONTNEED) and madvise(MADV_WILLNEED).

Also, the simplified driver does not handle errors at all.

I don't think that assuming madvise() can't fail is a good thing.

Quoted 10 lines · Avi Kivity

Quoted 8 lines · Rusty Russell

After discussions with Anthony Liguori, it seems that the virtio balloon can be made even simpler. Here's my attempt.

Since the balloon requires Guest cooperation anyway, there seems little reason to force it to tell the Host when it wants to reuse a page. It can simply fault it in.

Quoted 3 lines

Faulting is synchronous, while deflating is (or can be made) asynchronous. If the host needs to do some work to get the memory, the guest will be slowed down considerably.

Good point. Basically, we have two page hinting operations which roughly correspond to madvise(MADV_DONTNEED) and madvise(MADV_WILLNEED).

Quoted 8 lines · Rusty Russell

After discussions with Anthony Liguori, it seems that the virtio balloon can be made even simpler. Here's my attempt.

Since the balloon requires Guest cooperation anyway, there seems little reason to force it to tell the Host when it wants to reuse a page. It can simply fault it in.

Faulting is synchronous, while deflating is (or can be made) asynchronous. If the host needs to do some work to get the memory, the guest will be slowed down considerably.

If we have explicit deflate, the host can call madvise(MADV_WILLNEED) or actually touch the pages before the guest accesses them.

Quoted 8 lines · Rusty Russell

After discussions with Anthony Liguori, it seems that the virtio balloon can be made even simpler. Here's my attempt.

Since the balloon requires Guest cooperation anyway, there seems little reason to force it to tell the Host when it wants to reuse a page. It can simply fault it in.

Faulting is synchronous, while deflating is (or can be made) asynchronous. If the host needs to do some work to get the memory, the guest will be slowed down considerably.

If we have explicit deflate, the host can call madvise(MADV_WILLNEED) or actually touch the pages before the guest accesses them.

OK, thats simpler. How about this:

[PATCH] Virtio balloon driver

Add a balloon driver for KVM, host<->guest communication is performed via virtio.

Signed-off-by Marcelo Tosatti <[email protected]>

drivers/virtio/kvm_balloon.c new file +29 -0

+	}
+}
+
+static void add_page_array(struct virtballoon *v, struct balloon_buf *buf,
+			   unsigned int npages)
+{
+	struct page *page;
+	u32 *pfn = (u32 *)&buf->data;
+	int i;
+
+	for (i=0; i<npages; i++) {
+		page = pfn_to_page(*pfn);
+		v->balloon_size++;
+		totalram_pages--;
+		list_add(&page->lru, &v->balloon_plist);
+		pfn++;
+	}
+}
+
+static void inflate_done(struct virtballoon *v, struct balloon_buf *buf,
+			 unsigned int npages)
+{
+	u8 status = buf->hdr.status;
+
+	/* inflate OK */
+	if (!status)
+		add_page_array(v, buf, npages);
+	else 
+		free_page_array(buf, npages);

drivers/virtio/kvm_balloon.c new file +6 -0

+		totalram_pages += npages;
+		v->balloon_size -= npages;
+	}
+	return;
+}
+

drivers/virtio/kvm_balloon.c new file +44 -0

+	__virtio_config_val(v->vdev, 0, &target_nrpages);
+	atomic_set(&v->target_nrpages, target_nrpages);
+	wake_up(&v->balloon_wait);
+	dprintk(&vdev->dev, "%s\n", __func__);
+}
+
+static struct virtio_driver virtio_balloon = {
+	.driver.name =  KBUILD_MODNAME,
+	.driver.owner = THIS_MODULE,
+	.id_table =     id_table,
+	.probe =        balloon_probe,
+	.remove =	__devexit_p(balloon_remove),
+	.config_changed	= balloon_config_changed,
+};
+
+module_param(kvm_balloon_debug, int, 0);
+
+static int __init kvm_balloon_init(void)
+{
+	return register_virtio_driver(&virtio_balloon);
+}
+
+static void __exit kvm_balloon_exit(void)
+{
+	struct virtballoon *v;
+	
+	list_for_each_entry(v, &balloon_devices, list) { 
+		while (v->balloon_size) {
+			DEFINE_WAIT(wait);
+
+			atomic_add(v->balloon_size, &v->target_nrpages);
+			wake_up(&v->balloon_wait);
+			prepare_to_wait(&v->rmmod_wait, &wait,
+					TASK_INTERRUPTIBLE);
+			schedule_timeout(HZ*10);
+			finish_wait(&v->rmmod_wait, &wait);
+		}
+	}
+
+	unregister_virtio_driver(&virtio_balloon);
+}
+
+module_init(kvm_balloon_init);
+module_exit(kvm_balloon_exit);

Index: linux-2.6-nv/drivers/virtio/virtio_pci.c
===================================================================

drivers/virtio/virtio_pci.c +1 -0

@@ -67,6 +67,7 @@ static struct pci_device_id virtio_pci_i
 	{ 0x1AF4, 0x1000, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, /* Dummy entry */
 	{ 0x1AF4, 0x1001, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, /* Dummy entry */
 	{ 0x1AF4, 0x1002, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, /* Dummy entry */
+	{ 0x1AF4, 0x1003, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, /* Balloon */
 	{ 0 },
 };
 

Index: linux-2.6-nv/include/linux/virtio_balloon.h
===================================================================

include/linux/virtio_balloon.h new file +20 -0

@@ -0,0 +1,20 @@
+#ifndef _LINUX_VIRTIO_BALLOON_H
+#define _LINUX_VIRTIO_BALLOON_H
+#include <linux/virtio_config.h>
+
+#define VIRTIO_ID_BALLOON 3
+
+#define CMD_BALLOON_INFLATE 0x1
+#define CMD_BALLOON_DEFLATE 0x2
+
+struct virtio_balloon_hdr {
+	__u8 cmd;
+	__u8 status;
+};
+
+struct virtio_balloon_config
+{
+	__u32 target_nrpages;
+};
+
+#endif /* _LINUX_VIRTIO_BALLOON_H */

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linux-foundation.org/mailman/listinfo/virtualization