Skip to contentSkip to conversation

kvm guest balloon driver

PATCH In review Marcelo Tosatti 10 participants 19 messages Jan 8, 2008 – Jan 9, 2008 5 messages not in the archive

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 kvm guest balloon driver

Following patch introduces a KVM guest balloon driver. Communication to/from the host is performed via virtio.

Next patch implements the QEMU driver and handling.

drivers/virtio/Kconfig +9 -0
Index: linux-2.6-nv/drivers/virtio/Kconfig
===================================================================
@@ -23,3 +23,12 @@ config VIRTIO_PCI
 
 	  If unsure, say M.
 
+config KVM_BALLOON
+	tristate "KVM balloon driver (EXPERIMENTAL)"
+	depends on VIRTIO_PCI
+	---help---
+	 This driver provides support for ballooning memory in/out of a
+	 KVM paravirt guest.	
+
+	 If unsure, say M.
+
drivers/virtio/Makefile +1 -0
Index: linux-2.6-nv/drivers/virtio/Makefile
===================================================================
@@ -1,3 +1,4 @@
 obj-$(CONFIG_VIRTIO) += virtio.o
 obj-$(CONFIG_VIRTIO_RING) += virtio_ring.o
 obj-$(CONFIG_VIRTIO_PCI) += virtio_pci.o
+obj-$(CONFIG_KVM_BALLOON) += kvm_balloon.o
drivers/virtio/kvm_balloon.c new file +577 -0

577 lines not on this page · raw message

drivers/virtio/virtio_pci.c +1 -20
Index: linux-2.6-nv/drivers/virtio/virtio_pci.c
===================================================================
@@ -30,20 +30,6 @@ MODULE_DESCRIPTION("virtio-pci");
 MODULE_LICENSE("GPL");
 MODULE_VERSION("1");
 
-/* Our device structure */
-struct virtio_pci_device
-{
-	struct virtio_device vdev;
-	struct pci_dev *pci_dev;
-
-	/* the IO mapping for the PCI config space */
-	void *ioaddr;
-
-	/* a list of queues so we can dispatch IRQs */
-	spinlock_t lock;
-	struct list_head virtqueues;
-};
-
 struct virtio_pci_vq_info
 {
 	/* the actual virtqueue */
@@ -67,6 +53,7 @@ static struct pci_device_id virtio_pci_i
 	{ 0x1AF4, 0x1000, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, /* Dummy entry */
 	{ 0x1AF4, 0x1001, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, /* Dummy entry */
 	{ 0x1AF4, 0x1002, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, /* Dummy entry */
+	{ 0x1AF4, 0x1003, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, /* Balloon */
 	{ 0 },
 };
 
@@ -89,12 +76,6 @@ static struct device virtio_pci_root = {
 /* Unique numbering for devices under the kvm root */
 static unsigned int dev_index;
 
-/* Convert a generic virtio device to our structure */
-static struct virtio_pci_device *to_vp_device(struct virtio_device *vdev)
-{
-	return container_of(vdev, struct virtio_pci_device, vdev);
-}
-
 /* virtio config->feature() implementation */
 static bool vp_feature(struct virtio_device *vdev, unsigned bit)
 {
include/linux/virtio_pci.h +20 -0
Index: linux-2.6-nv/include/linux/virtio_pci.h
===================================================================
@@ -19,6 +19,26 @@
 
 #include <linux/virtio_config.h>
 
+/* Our device structure */
+struct virtio_pci_device
+{
+	struct virtio_device vdev;
+	struct pci_dev *pci_dev;
+
+	/* the IO mapping for the PCI config space */
+	void *ioaddr;
+
+	/* a list of queues so we can dispatch IRQs */
+	spinlock_t lock;
+	struct list_head virtqueues;
+};
+
+/* Convert a generic virtio device to our structure */
+struct virtio_pci_device *to_vp_device(struct virtio_device *vdev)
+{
+	return container_of(vdev, struct virtio_pci_device, vdev);
+}
+
 /* A 32-bit r/o bitmask of the features supported by the host */
 #define VIRTIO_PCI_HOST_FEATURES	0
 

-------------------------------------------------------------------------
Check out the new SourceForge.net Marketplace.
It's the best place to buy or sell services for
just about anything Open Source.
http://ad.doubleclick.net/clk;164216239;13503038;w?http://sf.net/marketplace

#1 lore raw
Quoted 66 lines · Marcelo Tosatti

Following patch introduces a KVM guest balloon driver. Communication to/from the host is performed via virtio.

Next patch implements the QEMU driver and handling.

Index: linux-2.6-nv/drivers/virtio/Kconfig
===================================================================

drivers/virtio/Kconfig +9 -0
@@ -23,3 +23,12 @@ config VIRTIO_PCI
 
 	  If unsure, say M.
 
+config KVM_BALLOON
+	tristate "KVM balloon driver (EXPERIMENTAL)"
+	depends on VIRTIO_PCI
+	---help---
+	 This driver provides support for ballooning memory in/out of a
+	 KVM paravirt guest.	
+
+	 If unsure, say M.
+

Index: linux-2.6-nv/drivers/virtio/Makefile
===================================================================

drivers/virtio/Makefile +1 -0
@@ -1,3 +1,4 @@
 obj-$(CONFIG_VIRTIO) += virtio.o
 obj-$(CONFIG_VIRTIO_RING) += virtio_ring.o
 obj-$(CONFIG_VIRTIO_PCI) += virtio_pci.o
+obj-$(CONFIG_KVM_BALLOON) += kvm_balloon.o

Index: linux-2.6-nv/drivers/virtio/kvm_balloon.c
===================================================================

drivers/virtio/kvm_balloon.c new file +27 -0
@@ -0,0 +1,577 @@
+/*
+ * KVM guest balloon driver
+ *
+ * Copyright (C) 2007, Qumranet, Inc., Dor Laor <dor.laor-atKUWr5tajBWk0Htik3J/[email protected]>
+ * Copyright (C) 2007, Red Hat, Inc., Marcelo Tosatti <[email protected]>
+ * 
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ */
+
+#include <asm/uaccess.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/percpu.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/pci.h>
+#include <linux/mm.h>
+#include <linux/swap.h>
+#include <linux/wait.h>
+#include <linux/kthread.h>
+#include <linux/freezer.h>
+#include <linux/version.h>
+#include <linux/virtio.h>
+#include <linux/virtio_config.h>
+#include <linux/virtio_pci.h>
+#include <linux/preempt.h>

drivers/virtio/kvm_balloon.c line 37

Quoted 10 lines
drivers/virtio/kvm_balloon.c new file +10 -0
+#include <linux/kvm_types.h>
+#include <linux/kvm_host.h>
+
+MODULE_AUTHOR ("Dor Laor");
+MODULE_DESCRIPTION ("Implements guest ballooning support");
+MODULE_LICENSE("GPL");
+MODULE_VERSION("1");
+
+#define CMD_BALLOON_INFLATE 0x1
+#define CMD_BALLOON_DEFLATE 0x2

Anything that's part of the ABI needs to be in a header file. This is how the rest of the virtio devices work.

drivers/virtio/kvm_balloon.c line 46

Quoted 9 lines
drivers/virtio/kvm_balloon.c new file +8 -0
+static int kvm_balloon_debug;
+
+#define DEBUG
+#ifdef DEBUG
+#define dprintk(str...) if (kvm_balloon_debug) printk(str)
+#else
+#define dprintk(str...)
+#endif

I think pr_debug is the more appropriate thing to use.

drivers/virtio/kvm_balloon.c line 52

Quoted 6 lines
drivers/virtio/kvm_balloon.c new file +5 -0
+static LIST_HEAD(balloon_plist);
+static LIST_HEAD(balloon_work);
+static int balloon_size = 0;
+static DEFINE_SPINLOCK(balloon_plist_lock);
+static DEFINE_SPINLOCK(balloon_queue_lock);

So far, all the drivers hang their state off of the virtio_device instead of using statics. While it's probably arguable that you will never have multiple balloon drivers, I think it's a good idea to at least be consistent and not use all of these globals.

Quoted 85 lines
drivers/virtio/kvm_balloon.c new file +84 -0
+struct virtio_balloon_hdr {
+	uint8_t cmd;
+	uint8_t status;
+};
+
+#define BALLOON_DATA_SIZE 200
+
+struct balloon_buf {
+	struct virtio_balloon_hdr hdr;
+	u8 data[BALLOON_DATA_SIZE];
+};
+
+struct balloon_work {
+	struct balloon_buf *buf;
+	struct list_head list;
+};
+
+#define VIRTIO_MAX_SG 2
+
+struct virtballoon {
+	struct virtio_device *dev;
+	struct virtqueue *vq;
+	struct task_struct *balloon_thread;
+	wait_queue_head_t balloon_wait;
+	wait_queue_head_t rmmod_wait;
+	uint32_t target_nrpages;
+	atomic_t inflight_bufs;
+};
+
+struct balloon_page {
+	struct page *bpage;
+	struct list_head bp_list;
+};
+
+struct virtballoon virtballoon;
+
+struct balloon_buf *alloc_balloon_buf(void)
+{
+	struct balloon_buf *buf;
+
+	buf = kzalloc(sizeof(struct balloon_buf), GFP_KERNEL);
+	if (!buf)
+		printk(KERN_ERR "%s: allocation failed\n", __func__);
+
+	return buf;
+}
+
+static int send_balloon_buf(uint8_t cmd, struct balloon_buf *buf)
+{
+	struct scatterlist sg[VIRTIO_MAX_SG];
+	struct virtqueue *vq = virtballoon.vq;
+	int err = 0;
+
+	buf->hdr.cmd = cmd;
+
+	sg_init_table(sg, VIRTIO_MAX_SG);
+	sg_set_buf(&sg[0], &buf->hdr, sizeof(buf->hdr));
+	sg_set_buf(&sg[1], &buf->data, sizeof(buf->data));
+
+	spin_lock_irq(&balloon_queue_lock);
+	err = vq->vq_ops->add_buf(vq, sg, 0, 2, buf);
+	if (err) {
+		printk("%s: add_buf err\n", __func__);
+		goto out;
+	}
+	atomic_inc(&virtballoon.inflight_bufs);
+
+	/* TODO: kick several balloon buffers at once */
+	vq->vq_ops->kick(vq);
+out:
+	spin_unlock_irq(&balloon_queue_lock);
+	return err;
+}
+
+static int kvm_balloon_inflate(int32_t npages)
+{
+	LIST_HEAD(tmp_list);
+	struct balloon_page *node, *tmp;
+	struct balloon_buf *buf;
+	u32 *pfn;
+	int allocated = 0;
+	int i, r = -ENOMEM;
+
+	buf = alloc_balloon_buf();

drivers/virtio/kvm_balloon.c line 147

Quoted 10 lines
drivers/virtio/kvm_balloon.c new file +10 -0
+	if (!buf)
+		return r;
+
+	pfn = (u32 *)&buf->data;
+	*pfn++ = (u32)npages;
+
+	for (i = 0; i < npages; i++) {
+		node = kzalloc(sizeof(struct balloon_page), GFP_KERNEL);
+		if (!node)
+			goto out_free;

Instead of allocating a node for each page, you could use page->private to create a linked list. You're potentially burning a lot of memory when ballooning down by a large amount otherwise.

Quoted 34 lines
drivers/virtio/kvm_balloon.c new file +33 -0
+		node->bpage = alloc_page(GFP_HIGHUSER | __GFP_NORETRY);
+		if (!node->bpage) {
+			kfree(node);
+			goto out_free;
+		}
+
+		list_add(&node->bp_list, &tmp_list);
+		allocated++;
+		*pfn = page_to_pfn(node->bpage);
+		pfn++;
+	}
+
+	r = send_balloon_buf(CMD_BALLOON_INFLATE, buf);
+	if (r)
+		goto out_free;
+
+	spin_lock(&balloon_plist_lock);
+	list_splice(&tmp_list, &balloon_plist);
+	balloon_size += allocated;
+	totalram_pages -= allocated;
+	dprintk("%s: current balloon size=%d\n", __FUNCTION__,
+	       balloon_size);
+	spin_unlock(&balloon_plist_lock);
+	return allocated;
+
+out_free:
+	list_for_each_entry_safe(node, tmp, &tmp_list, bp_list) {
+		__free_page(node->bpage);
+		list_del(&node->bp_list);
+		kfree(node);
+	}
+	return r;
+}

<snip>

Quoted 17 lines
drivers/virtio/kvm_balloon.c new file +16 -0
+
+static int balloon_probe(struct virtio_device *vdev)
+{
+	struct virtio_pci_device *pvdev = to_vp_device(vdev);
+	int err = -EBUSY;
+
+	if (virtballoon.dev) { 
+		printk("kvm_ballon: error, already registered\n");
+		return -EBUSY;
+	}
+
+	virtballoon.vq = vdev->config->find_vq(vdev, 0, balloon_tx_done);
+	if (IS_ERR(virtballoon.vq)) {
+		printk("%s: error finding vq\n", __func__);
+		return -EINVAL;
+	}

drivers/virtio/kvm_balloon.c line 510

Quoted 10 lines
drivers/virtio/kvm_balloon.c new file +10 -0
+
+	virtballoon.dev = vdev;
+	init_waitqueue_head(&virtballoon.balloon_wait);
+	init_waitqueue_head(&virtballoon.rmmod_wait);
+	atomic_set(&virtballoon.inflight_bufs, 0);
+
+	err = request_irq(pvdev->pci_dev->irq, balloon_irq, IRQF_SHARED,
+			  pvdev->vdev.dev.bus_id, &virtballoon);
+	if (err)
+		goto out_free_vq;

Why is it taking over the irq? This is very, very wrong. A virtio device cannot be dependent on being used on top of the virtio-pci backend.

Quoted 132 lines
drivers/virtio/kvm_balloon.c new file +67 -0
+
+	virtballoon.balloon_thread = kthread_run(balloon_thread,
+						 &virtballoon,
+						 "kvm_balloond");
+	printk("kvm_balloon: registered\n");
+
+	return 0;
+
+out_free_vq:
+	vdev->config->del_vq(virtballoon.vq);
+	virtballoon.dev = NULL;
+	return err;
+}
+
+static void balloon_remove(struct virtio_device *vdev)
+{
+	kthread_stop(virtballoon.balloon_thread);
+	vdev->config->del_vq(virtballoon.vq);
+}
+
+static struct virtio_driver virtio_balloon = {
+	.driver.name =  KBUILD_MODNAME,
+	.driver.owner = THIS_MODULE,
+	.id_table =     id_table,
+	.probe =        balloon_probe,
+	.remove =	__devexit_p(balloon_remove),
+};
+
+module_param(kvm_balloon_debug, int, 0);
+
+static int __init kvm_balloon_init(void)
+{
+	virtballoon.dev = NULL;
+
+	return register_virtio_driver(&virtio_balloon);
+}
+
+static void __exit kvm_balloon_exit(void)
+{
+	spin_lock(&balloon_plist_lock);
+	if (balloon_size) {
+		DEFINE_WAIT(wait);
+
+		virtballoon.target_nrpages += balloon_size;
+		spin_unlock(&balloon_plist_lock);
+		virtballoon.dev->config->set(virtballoon.dev, 0, 
+					   &virtballoon.target_nrpages,
+					   sizeof(virtballoon.target_nrpages));
+		wake_up(&virtballoon.balloon_wait);
+		prepare_to_wait(&virtballoon.rmmod_wait, &wait,
+				TASK_UNINTERRUPTIBLE);
+		schedule();
+		finish_wait(&virtballoon.rmmod_wait, &wait);
+		spin_lock(&balloon_plist_lock);
+	}
+
+	if (balloon_size)
+		printk(KERN_ERR "%s: exit while balloon not empty!\n",
+			__FUNCTION__);
+
+	spin_unlock(&balloon_plist_lock);
+
+	unregister_virtio_driver(&virtio_balloon);
+}
+
+module_init(kvm_balloon_init);
+module_exit(kvm_balloon_exit);

Index: linux-2.6-nv/drivers/virtio/virtio_pci.c
===================================================================

drivers/virtio/virtio_pci.c +1 -20
@@ -30,20 +30,6 @@ MODULE_DESCRIPTION("virtio-pci");
 MODULE_LICENSE("GPL");
 MODULE_VERSION("1");
 
-/* Our device structure */
-struct virtio_pci_device
-{
-	struct virtio_device vdev;
-	struct pci_dev *pci_dev;
-
-	/* the IO mapping for the PCI config space */
-	void *ioaddr;
-
-	/* a list of queues so we can dispatch IRQs */
-	spinlock_t lock;
-	struct list_head virtqueues;
-};
-
 struct virtio_pci_vq_info
 {
 	/* the actual virtqueue */
@@ -67,6 +53,7 @@ static struct pci_device_id virtio_pci_i
 	{ 0x1AF4, 0x1000, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, /* Dummy entry */
 	{ 0x1AF4, 0x1001, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, /* Dummy entry */
 	{ 0x1AF4, 0x1002, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, /* Dummy entry */
+	{ 0x1AF4, 0x1003, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, /* Balloon */
 	{ 0 },
 };
 
@@ -89,12 +76,6 @@ static struct device virtio_pci_root = {
 /* Unique numbering for devices under the kvm root */
 static unsigned int dev_index;
 
-/* Convert a generic virtio device to our structure */
-static struct virtio_pci_device *to_vp_device(struct virtio_device *vdev)
-{
-	return container_of(vdev, struct virtio_pci_device, vdev);
-}
-
 /* virtio config->feature() implementation */
 static bool vp_feature(struct virtio_device *vdev, unsigned bit)
 {

Index: linux-2.6-nv/include/linux/virtio_pci.h
===================================================================

include/linux/virtio_pci.h +11 -0
@@ -19,6 +19,26 @@
 
 #include <linux/virtio_config.h>
 
+/* Our device structure */
+struct virtio_pci_device
+{
+	struct virtio_device vdev;
+	struct pci_dev *pci_dev;
+
+	/* the IO mapping for the PCI config space */
+	void *ioaddr;
+
+	/* a list of queues so we can dispatch IRQs */
+	spinlock_t lock;

include/linux/virtio_pci.h line 42

Quoted 10 lines
include/linux/virtio_pci.h +9 -0
+	struct list_head virtqueues;
+};
+
+/* Convert a generic virtio device to our structure */
+struct virtio_pci_device *to_vp_device(struct virtio_device *vdev)
+{
+	return container_of(vdev, struct virtio_pci_device, vdev);
+}
+
 

This should be a big indicator that something is wrong. You should not have to change the encapsulation of the virtio-pci backend.

Regards,

Anthony Liguori

Quoted 5 lines
include/linux/virtio_pci.h +0 -1
 /* A 32-bit r/o bitmask of the features supported by the host */
 #define VIRTIO_PCI_HOST_FEATURES	0
 
 
-------------------------------------------------------------------------
Quoted 9 lines

Check out the new SourceForge.net Marketplace. It's the best place to buy or sell services for just about anything Open Source.
http://ad.doubleclick.net/clk;164216239;13503038;w?http://sf.net/marketplace
_______________________________________________
kvm-devel mailing list [email protected]
https://lists.sourceforge.net/lists/listinfo/kvm-devel

-------------------------------------------------------------------------
Check out the new SourceForge.net Marketplace.
It's the best place to buy or sell services for
just about anything Open Source.
http://ad.doubleclick.net/clk;164216239;13503038;w?http://sf.net/marketplace

A message that is not in the archive (sent off-list or not yet received): [email protected]

#3 lore raw
Quoted 4 lines · Anthony Liguori
Quoted 3 lines · Marcelo Tosatti

Following patch introduces a KVM guest balloon driver. Communication to/from the host is performed via virtio.

I'll address the other comments.

Quoted 5 lines
Quoted 5 lines
drivers/virtio/kvm_balloon.c new file +5 -0
+	virtballoon.dev = vdev;
+	init_waitqueue_head(&virtballoon.balloon_wait);
+	init_waitqueue_head(&virtballoon.rmmod_wait);
+	atomic_set(&virtballoon.inflight_bufs, 0);
+
Quoted 8 lines

drivers/virtio/kvm_balloon.c line 510

Quoted 5 lines
drivers/virtio/kvm_balloon.c new file +4 -0
+	err = request_irq(pvdev->pci_dev->irq, balloon_irq, IRQF_SHARED,
+			  pvdev->vdev.dev.bus_id, &virtballoon);
+	if (err)
+		goto out_free_vq;

Why is it taking over the irq? This is very, very wrong. A virtio device cannot be dependent on being used on top of the virtio-pci backend.

A notification is necessary whenever the host changes the target value in the config space. So right now this notification is sharing the same IRQ as the main PCI device.

Do you have any suggestion on how to retrieve the IRQ of the virtio device, or some other notification mechanism?

-------------------------------------------------------------------------
Check out the new SourceForge.net Marketplace.
It's the best place to buy or sell services for
just about anything Open Source.
http://ad.doubleclick.net/clk;164216239;13503038;w?http://sf.net/marketplace

#4 lore raw
Quoted 27 lines · Marcelo Tosatti
Quoted 6 lines · Anthony Liguori
Quoted 4 lines · Marcelo Tosatti

Following patch introduces a KVM guest balloon driver. Communication to/from the host is performed via virtio.

I'll address the other comments.

Quoted 5 lines
Quoted 5 lines
drivers/virtio/kvm_balloon.c new file +5 -0
+	virtballoon.dev = vdev;
+	init_waitqueue_head(&virtballoon.balloon_wait);
+	init_waitqueue_head(&virtballoon.rmmod_wait);
+	atomic_set(&virtballoon.inflight_bufs, 0);
+
Quoted 9 lines

drivers/virtio/kvm_balloon.c line 510

Quoted 6 lines
drivers/virtio/kvm_balloon.c new file +4 -0
+	err = request_irq(pvdev->pci_dev->irq, balloon_irq, IRQF_SHARED,
+			  pvdev->vdev.dev.bus_id, &virtballoon);
+	if (err)
+		goto out_free_vq;

Why is it taking over the irq? This is very, very wrong. A virtio device cannot be dependent on being used on top of the virtio-pci backend.

Quoted 4 lines

A notification is necessary whenever the host changes the target value in the config space. So right now this notification is sharing the same IRQ as the main PCI device.

A message should be sent over a virtqueue to indicate that the other need needs to reread a config value.

You really need two virtqueues. The virtqueue you have now for the guest to send messages to the host, and another virtqueue that the guest adds buffers too that the host can use to send messages to the guest. A good example of using two queues for bidirectional communication would be the virtio_console driver.

BTW, I don't think the target should be a config value. You don't gain anything from it being in the config space and it's somewhat unnatural for a virtio device. It makes more sense as a message to the guest.

The PCI config space is not automatically saved/restored during migration.

Quoted 3 lines

Do you have any suggestion on how to retrieve the IRQ of the virtio device, or some other notification mechanism?

A virtio device does not necessarily have an interrupt or it may have multiple interrupts.

Regards,

Anthony Liguori

-------------------------------------------------------------------------
Check out the new SourceForge.net Marketplace.
It's the best place to buy or sell services for
just about anything Open Source.
http://ad.doubleclick.net/clk;164216239;13503038;w?http://sf.net/marketplace

A message that is not in the archive (sent off-list or not yet received): [email protected]

#6 lore raw
Quoted 4 lines · Anthony Liguori

BTW, I don't think the target should be a config value. You don't gain anything from it being in the config space and it's somewhat unnatural for a virtio device. It makes more sense as a message to the guest.

I disagree. The target is state, not an individual item that needs to be acted on. Having it as a single variable means that multiple changes are collapsed automatically, and that the the setting survives module reload.

It's like a volume control, it doesn't send messages when you turn it, it just sets a value.

(maybe a thermostat knob is a better analogy, with the driver being the circuitry around the knob that tries to control the temperature to match the setting)

I believe state-like controls will be useful for other settings, like ethernet link state.

Signature · 8 lines
error compiling committee.c: too many arguments to function


-------------------------------------------------------------------------
Check out the new SourceForge.net Marketplace.
It's the best place to buy or sell services for
just about anything Open Source.
http://ad.doubleclick.net/clk;164216239;13503038;w?http://sf.net/marketplace

A message that is not in the archive (sent off-list or not yet received): 4783A68E.80901-atKUWr5tajBWk0Htik3J/[email protected]

#8 lore raw
Quoted 7 lines · Avi Kivity
Quoted 5 lines · Anthony Liguori

BTW, I don't think the target should be a config value. You don't gain anything from it being in the config space and it's somewhat unnatural for a virtio device. It makes more sense as a message to the guest.

Quoted 7 lines

I disagree. The target is state, not an individual item that needs to be acted on. Having it as a single variable means that multiple changes are collapsed automatically, and that the the setting survives module reload.

It's like a volume control, it doesn't send messages when you turn it, it just sets a value.

It's a value that is meant to be acted upon though, not something to be polled. You want to tell the driver that it should now try to balloon to a certain value. Or maybe not. Maybe the driver should read the target from the config space but then have a "kick" message that tells it, hey, something's probably changed.

Quoted 3 lines

(maybe a thermostat knob is a better analogy, with the driver being the circuitry around the knob that tries to control the temperature to match the setting)

FWIW, I'm pretty sure that most modern volume controls are actually button presses for either direction instead of a variable resistor but point taken ;-)

Quoted 2 lines

I believe state-like controls will be useful for other settings, like ethernet link state.

It's difficult with the current config interface since it treats everything as a discrete blob that is updated all at once. I'm not really sure how to proceed.

Rusty: do you have an opinion here?

Regards,

Anthony Liguori

-------------------------------------------------------------------------
Check out the new SourceForge.net Marketplace.
It's the best place to buy or sell services for
just about anything Open Source.
http://ad.doubleclick.net/clk;164216239;13503038;w?http://sf.net/marketplace

#9 lore raw
In reply to Avi Kivity
Quoted 7 lines · Avi Kivity
Quoted 2 lines · Anthony Liguori

BTW, I don't think the target should be a config value. You don't gain anything from it being in the config space and it's somewhat unnatural for a virtio device. It makes more sense as a message to the guest.

I disagree. The target is state, not an individual item that needs to be acted on. Having it as a single variable means that multiple changes are collapsed automatically, and that the the setting survives module reload.

It's like a volume control, it doesn't send messages when you turn it, it just sets a value.

It's a value that is meant to be acted upon though, not something to be polled. You want to tell the driver that it should now try to balloon to a certain value. Or maybe not. Maybe the driver should read the target from the config space but then have a "kick" message that tells it, hey, something's probably changed.

Quoted 1 line

(maybe a thermostat knob is a better analogy, with the driver being the circuitry around the knob that tries to control the temperature to match the setting)

FWIW, I'm pretty sure that most modern volume controls are actually button presses for either direction instead of a variable resistor but point taken ;-)

Quoted 1 line

I believe state-like controls will be useful for other settings, like ethernet link state.

It's difficult with the current config interface since it treats everything as a discrete blob that is updated all at once. I'm not really sure how to proceed.

Rusty: do you have an opinion here?

Regards,

Anthony Liguori

#10 lore raw
In reply to Anthony Liguori
Quoted 2 lines · Anthony Liguori

BTW, I don't think the target should be a config value. You don't gain anything from it being in the config space and it's somewhat unnatural for a virtio device. It makes more sense as a message to the guest.

I disagree. The target is state, not an individual item that needs to be acted on. Having it as a single variable means that multiple changes are collapsed automatically, and that the the setting survives module reload.

It's like a volume control, it doesn't send messages when you turn it, it just sets a value.

(maybe a thermostat knob is a better analogy, with the driver being the circuitry around the knob that tries to control the temperature to match the setting)

I believe state-like controls will be useful for other settings, like ethernet link state.

Signature · 1 line
error compiling committee.c: too many arguments to function
#11 lore raw
In reply to Marcelo Tosatti
Quoted 26 lines · Marcelo Tosatti
Quoted 6 lines · Anthony Liguori
Quoted 4 lines · Marcelo Tosatti

Following patch introduces a KVM guest balloon driver. Communication to/from the host is performed via virtio.

I'll address the other comments.

Quoted 4 lines
Quoted 4 lines
drivers/virtio/kvm_balloon.c new file +4 -0
+	virtballoon.dev = vdev;
+	init_waitqueue_head(&virtballoon.balloon_wait);
+	init_waitqueue_head(&virtballoon.rmmod_wait);
+	atomic_set(&virtballoon.inflight_bufs, 0);
Quoted 9 lines

drivers/virtio/kvm_balloon.c line 510

Quoted 7 lines
drivers/virtio/kvm_balloon.c new file +5 -0
+
+	err = request_irq(pvdev->pci_dev->irq, balloon_irq, IRQF_SHARED,
+			  pvdev->vdev.dev.bus_id, &virtballoon);
+	if (err)
+		goto out_free_vq;

Why is it taking over the irq? This is very, very wrong. A virtio device cannot be dependent on being used on top of the virtio-pci backend.

Quoted 4 lines

A notification is necessary whenever the host changes the target value in the config space. So right now this notification is sharing the same IRQ as the main PCI device.

A message should be sent over a virtqueue to indicate that the other need needs to reread a config value.

You really need two virtqueues. The virtqueue you have now for the guest to send messages to the host, and another virtqueue that the guest adds buffers too that the host can use to send messages to the guest. A good example of using two queues for bidirectional communication would be the virtio_console driver.

BTW, I don't think the target should be a config value. You don't gain anything from it being in the config space and it's somewhat unnatural for a virtio device. It makes more sense as a message to the guest.

The PCI config space is not automatically saved/restored during migration.

Quoted 3 lines

Do you have any suggestion on how to retrieve the IRQ of the virtio device, or some other notification mechanism?

A virtio device does not necessarily have an interrupt or it may have multiple interrupts.

Regards,

Anthony Liguori

#12 lore raw
In reply to Marcelo Tosatti
Quoted 11 lines · Marcelo Tosatti
Quoted 3 lines

Why is it taking over the irq? This is very, very wrong. A virtio device cannot be dependent on being used on top of the virtio-pci backend.

A notification is necessary whenever the host changes the target value in the config space. So right now this notification is sharing the same IRQ as the main PCI device.

Do you have any suggestion on how to retrieve the IRQ of the virtio device, or some other notification mechanism?

One way would be to send a "look at config" queue message, but it seems that it is a fairly generic operation. Maybe virtio can add support for it, with a new callback.

Signature · 8 lines
error compiling committee.c: too many arguments to function


-------------------------------------------------------------------------
Check out the new SourceForge.net Marketplace.
It's the best place to buy or sell services for
just about anything Open Source.
http://ad.doubleclick.net/clk;164216239;13503038;w?http://sf.net/marketplace

A message that is not in the archive (sent off-list or not yet received): 4783A245.90002-atKUWr5tajBWk0Htik3J/[email protected]

#14 lore raw
Quoted 3 lines · On Wednesday 09 January 2008 03:18:13 Avi Kivity wrote:
Quoted 2 lines · Marcelo Tosatti

Do you have any suggestion on how to retrieve the IRQ of the virtio device, or some other notification mechanism?

Unfortunately, irqs are logically assigned to virtio queues, not devices. Yet configuration info is per-device.

Quoted 3 lines

One way would be to send a "look at config" queue message, but it seems that it is a fairly generic operation. Maybe virtio can add support for it, with a new callback.

This is the first time we've had to do this, but I don't think it's the last, so we should consider it carefully.

Chatted with Anthony, and this is what we came up with:
===
virtio: configuration change callback

Various drivers want to know when their configuration information
changes: the balloon driver is the immediate user, but the network driver may one day have a "carrier" status as well.

This introduces that callback, and adds it to the virtio PCI implementation.

---
 drivers/virtio/virtio_pci.c |   10 ++++++++++
 include/linux/virtio.h      |    3 +++
 include/linux/virtio_pci.h  |    3 +++
 3 files changed, 16 insertions(+)
drivers/virtio/virtio_pci.c +10 -0
diff -r 7494c7702462 drivers/virtio/virtio_pci.c
@@ -184,6 +184,16 @@ static irqreturn_t vp_interrupt(int irq,
 	/* It's definitely not us if the ISR was not high */
 	if (!isr)
 		return IRQ_NONE;
+
+	/* Configuration change?  Tell driver if it wants to know. */
+	if (isr & VIRTIO_PCI_ISR_CONFIG) {
+		struct virtio_driver *drv;
+		drv = container_of(vp_dev->vdev.dev.driver,
+				   struct virtio_driver, driver);
+
+		if (drv->config_changed)
+			drv->config_changed(&vp_dev->vdev);
+	}
 
 	spin_lock(&vp_dev->lock);
 	list_for_each_entry(info, &vp_dev->virtqueues, node) {
include/linux/virtio.h +3 -0
diff -r 7494c7702462 include/linux/virtio.h
@@ -98,12 +98,15 @@ void unregister_virtio_device(struct vir
  * @probe: the function to call when a device is found.  Returns a token for
  *    remove, or PTR_ERR().
  * @remove: the function when a device is removed.
+ * @config_changed: optional function to call when the device configuration
+ *    changes; may be called in interrupt context.
  */
 struct virtio_driver {
 	struct device_driver driver;
 	const struct virtio_device_id *id_table;
 	int (*probe)(struct virtio_device *dev);
 	void (*remove)(struct virtio_device *dev);
+	void (*config_changed)(struct virtio_device *dev);
 };
 
 int register_virtio_driver(struct virtio_driver *drv);
include/linux/virtio_pci.h +3 -0
diff -r 7494c7702462 include/linux/virtio_pci.h
@@ -45,6 +45,9 @@
  * a read-and-acknowledge. */
 #define VIRTIO_PCI_ISR			19
 
+/* The bit of the ISR which indicates a device configuration change. */
+#define VIRTIO_PCI_ISR_CONFIG		0x2
+
 /* The remaining space is defined by each driver as the per-driver
  * configuration space */
 #define VIRTIO_PCI_CONFIG		20

-------------------------------------------------------------------------
Check out the new SourceForge.net Marketplace.
It's the best place to buy or sell services for
just about anything Open Source.
http://ad.doubleclick.net/clk;164216239;13503038;w?http://sf.net/marketplace

#15 lore raw
In reply to Avi Kivity
Quoted 3 lines · On Wednesday 09 January 2008 03:18:13 Avi Kivity wrote:
Quoted 2 lines · Marcelo Tosatti

Do you have any suggestion on how to retrieve the IRQ of the virtio device, or some other notification mechanism?

Unfortunately, irqs are logically assigned to virtio queues, not devices. Yet configuration info is per-device.

Quoted 3 lines

One way would be to send a "look at config" queue message, but it seems that it is a fairly generic operation. Maybe virtio can add support for it, with a new callback.

This is the first time we've had to do this, but I don't think it's the last, so we should consider it carefully.

Chatted with Anthony, and this is what we came up with:
===
virtio: configuration change callback

Various drivers want to know when their configuration information
changes: the balloon driver is the immediate user, but the network driver may one day have a "carrier" status as well.

This introduces that callback, and adds it to the virtio PCI implementation.

---
 drivers/virtio/virtio_pci.c |   10 ++++++++++
 include/linux/virtio.h      |    3 +++
 include/linux/virtio_pci.h  |    3 +++
 3 files changed, 16 insertions(+)
drivers/virtio/virtio_pci.c +10 -0
diff -r 7494c7702462 drivers/virtio/virtio_pci.c
@@ -184,6 +184,16 @@ static irqreturn_t vp_interrupt(int irq,
 	/* It's definitely not us if the ISR was not high */
 	if (!isr)
 		return IRQ_NONE;
+
+	/* Configuration change?  Tell driver if it wants to know. */
+	if (isr & VIRTIO_PCI_ISR_CONFIG) {
+		struct virtio_driver *drv;
+		drv = container_of(vp_dev->vdev.dev.driver,
+				   struct virtio_driver, driver);
+
+		if (drv->config_changed)
+			drv->config_changed(&vp_dev->vdev);
+	}
 
 	spin_lock(&vp_dev->lock);
 	list_for_each_entry(info, &vp_dev->virtqueues, node) {
include/linux/virtio.h +3 -0
diff -r 7494c7702462 include/linux/virtio.h
@@ -98,12 +98,15 @@ void unregister_virtio_device(struct vir
  * @probe: the function to call when a device is found.  Returns a token for
  *    remove, or PTR_ERR().
  * @remove: the function when a device is removed.
+ * @config_changed: optional function to call when the device configuration
+ *    changes; may be called in interrupt context.
  */
 struct virtio_driver {
 	struct device_driver driver;
 	const struct virtio_device_id *id_table;
 	int (*probe)(struct virtio_device *dev);
 	void (*remove)(struct virtio_device *dev);
+	void (*config_changed)(struct virtio_device *dev);
 };
 
 int register_virtio_driver(struct virtio_driver *drv);
include/linux/virtio_pci.h +3 -0
diff -r 7494c7702462 include/linux/virtio_pci.h
@@ -45,6 +45,9 @@
  * a read-and-acknowledge. */
 #define VIRTIO_PCI_ISR			19
 
+/* The bit of the ISR which indicates a device configuration change. */
+#define VIRTIO_PCI_ISR_CONFIG		0x2
+
 /* The remaining space is defined by each driver as the per-driver
  * configuration space */
 #define VIRTIO_PCI_CONFIG		20
#16 lore raw
In reply to Marcelo Tosatti
Quoted 10 lines · Marcelo Tosatti
Quoted 2 lines

Why is it taking over the irq? This is very, very wrong. A virtio device cannot be dependent on being used on top of the virtio-pci backend.

A notification is necessary whenever the host changes the target value in the config space. So right now this notification is sharing the same IRQ as the main PCI device.

Do you have any suggestion on how to retrieve the IRQ of the virtio device, or some other notification mechanism?

One way would be to send a "look at config" queue message, but it seems that it is a fairly generic operation. Maybe virtio can add support for it, with a new callback.

Signature · 1 line
error compiling committee.c: too many arguments to function
#17 lore raw
In reply to a message not in the archive
Quoted 1 line · Anthony Liguori

Instead of allocating a node for each page, you could use page->private

page->lru is probably better for this so splice still works etc... (the struct page isn't visible to the guest VM so it's free to use)

-------------------------------------------------------------------------
Check out the new SourceForge.net Marketplace. It's the best place to buy or sell services for just about anything Open Source.
http://ad.doubleclick.net/clk;164216239;13503038;w?http://sf.net/marketplace

A message that is not in the archive (sent off-list or not yet received): [email protected]

#19 lore raw
Quoted 6 lines · Andrea Arcangeli
Quoted 1 line · Anthony Liguori

Instead of allocating a node for each page, you could use page->private

page->lru is probably better for this so splice still works etc... (the struct page isn't visible to the guest VM so it's free to use)

Yeap, that works.

-------------------------------------------------------------------------
Check out the new SourceForge.net Marketplace.
It's the best place to buy or sell services for
just about anything Open Source.
http://ad.doubleclick.net/clk;164216239;13503038;w?http://sf.net/marketplace

#20 lore raw
In reply to Andrea Arcangeli
Quoted 6 lines · Andrea Arcangeli
Quoted 1 line · Anthony Liguori

Instead of allocating a node for each page, you could use page->private

page->lru is probably better for this so splice still works etc... (the struct page isn't visible to the guest VM so it's free to use)

Yeap, that works.

#21 lore raw
In reply to Anthony Liguori
Quoted 4 lines · Anthony Liguori
Quoted 3 lines · Marcelo Tosatti

Following patch introduces a KVM guest balloon driver. Communication to/from the host is performed via virtio.

I'll address the other comments.

Quoted 5 lines
Quoted 5 lines
drivers/virtio/kvm_balloon.c new file +5 -0
+	virtballoon.dev = vdev;
+	init_waitqueue_head(&virtballoon.balloon_wait);
+	init_waitqueue_head(&virtballoon.rmmod_wait);
+	atomic_set(&virtballoon.inflight_bufs, 0);
+
Quoted 8 lines

drivers/virtio/kvm_balloon.c line 510

Quoted 5 lines
drivers/virtio/kvm_balloon.c new file +4 -0
+	err = request_irq(pvdev->pci_dev->irq, balloon_irq, IRQF_SHARED,
+			  pvdev->vdev.dev.bus_id, &virtballoon);
+	if (err)
+		goto out_free_vq;

Why is it taking over the irq? This is very, very wrong. A virtio device cannot be dependent on being used on top of the virtio-pci backend.

A notification is necessary whenever the host changes the target value in the config space. So right now this notification is sharing the same IRQ as the main PCI device.

Do you have any suggestion on how to retrieve the IRQ of the virtio device, or some other notification mechanism?

#22 lore raw
In reply to Anthony Liguori
Quoted 1 line · Anthony Liguori

Instead of allocating a node for each page, you could use page->private

page->lru is probably better for this so splice still works etc... (the struct page isn't visible to the guest VM so it's free to use)

#23 lore raw
In reply to Marcelo Tosatti
Quoted 66 lines · Marcelo Tosatti

Following patch introduces a KVM guest balloon driver. Communication to/from the host is performed via virtio.

Next patch implements the QEMU driver and handling.

Index: linux-2.6-nv/drivers/virtio/Kconfig
===================================================================

drivers/virtio/Kconfig +9 -0
@@ -23,3 +23,12 @@ config VIRTIO_PCI
 
 	  If unsure, say M.
 
+config KVM_BALLOON
+	tristate "KVM balloon driver (EXPERIMENTAL)"
+	depends on VIRTIO_PCI
+	---help---
+	 This driver provides support for ballooning memory in/out of a
+	 KVM paravirt guest.	
+
+	 If unsure, say M.
+

Index: linux-2.6-nv/drivers/virtio/Makefile
===================================================================

drivers/virtio/Makefile +1 -0
@@ -1,3 +1,4 @@
 obj-$(CONFIG_VIRTIO) += virtio.o
 obj-$(CONFIG_VIRTIO_RING) += virtio_ring.o
 obj-$(CONFIG_VIRTIO_PCI) += virtio_pci.o
+obj-$(CONFIG_KVM_BALLOON) += kvm_balloon.o

Index: linux-2.6-nv/drivers/virtio/kvm_balloon.c
===================================================================

drivers/virtio/kvm_balloon.c new file +27 -0
@@ -0,0 +1,577 @@
+/*
+ * KVM guest balloon driver
+ *
+ * Copyright (C) 2007, Qumranet, Inc., Dor Laor <[email protected]>
+ * Copyright (C) 2007, Red Hat, Inc., Marcelo Tosatti <[email protected]>
+ * 
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ */
+
+#include <asm/uaccess.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/percpu.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/pci.h>
+#include <linux/mm.h>
+#include <linux/swap.h>
+#include <linux/wait.h>
+#include <linux/kthread.h>
+#include <linux/freezer.h>
+#include <linux/version.h>
+#include <linux/virtio.h>
+#include <linux/virtio_config.h>
+#include <linux/virtio_pci.h>
+#include <linux/preempt.h>

drivers/virtio/kvm_balloon.c line 37

Quoted 10 lines
drivers/virtio/kvm_balloon.c new file +10 -0
+#include <linux/kvm_types.h>
+#include <linux/kvm_host.h>
+
+MODULE_AUTHOR ("Dor Laor");
+MODULE_DESCRIPTION ("Implements guest ballooning support");
+MODULE_LICENSE("GPL");
+MODULE_VERSION("1");
+
+#define CMD_BALLOON_INFLATE 0x1
+#define CMD_BALLOON_DEFLATE 0x2

Anything that's part of the ABI needs to be in a header file. This is how the rest of the virtio devices work.

drivers/virtio/kvm_balloon.c line 46

Quoted 9 lines
drivers/virtio/kvm_balloon.c new file +8 -0
+static int kvm_balloon_debug;
+
+#define DEBUG
+#ifdef DEBUG
+#define dprintk(str...) if (kvm_balloon_debug) printk(str)
+#else
+#define dprintk(str...)
+#endif

I think pr_debug is the more appropriate thing to use.

drivers/virtio/kvm_balloon.c line 52

Quoted 6 lines
drivers/virtio/kvm_balloon.c new file +5 -0
+static LIST_HEAD(balloon_plist);
+static LIST_HEAD(balloon_work);
+static int balloon_size = 0;
+static DEFINE_SPINLOCK(balloon_plist_lock);
+static DEFINE_SPINLOCK(balloon_queue_lock);

So far, all the drivers hang their state off of the virtio_device instead of using statics. While it's probably arguable that you will never have multiple balloon drivers, I think it's a good idea to at least be consistent and not use all of these globals.

Quoted 85 lines
drivers/virtio/kvm_balloon.c new file +84 -0
+struct virtio_balloon_hdr {
+	uint8_t cmd;
+	uint8_t status;
+};
+
+#define BALLOON_DATA_SIZE 200
+
+struct balloon_buf {
+	struct virtio_balloon_hdr hdr;
+	u8 data[BALLOON_DATA_SIZE];
+};
+
+struct balloon_work {
+	struct balloon_buf *buf;
+	struct list_head list;
+};
+
+#define VIRTIO_MAX_SG 2
+
+struct virtballoon {
+	struct virtio_device *dev;
+	struct virtqueue *vq;
+	struct task_struct *balloon_thread;
+	wait_queue_head_t balloon_wait;
+	wait_queue_head_t rmmod_wait;
+	uint32_t target_nrpages;
+	atomic_t inflight_bufs;
+};
+
+struct balloon_page {
+	struct page *bpage;
+	struct list_head bp_list;
+};
+
+struct virtballoon virtballoon;
+
+struct balloon_buf *alloc_balloon_buf(void)
+{
+	struct balloon_buf *buf;
+
+	buf = kzalloc(sizeof(struct balloon_buf), GFP_KERNEL);
+	if (!buf)
+		printk(KERN_ERR "%s: allocation failed\n", __func__);
+
+	return buf;
+}
+
+static int send_balloon_buf(uint8_t cmd, struct balloon_buf *buf)
+{
+	struct scatterlist sg[VIRTIO_MAX_SG];
+	struct virtqueue *vq = virtballoon.vq;
+	int err = 0;
+
+	buf->hdr.cmd = cmd;
+
+	sg_init_table(sg, VIRTIO_MAX_SG);
+	sg_set_buf(&sg[0], &buf->hdr, sizeof(buf->hdr));
+	sg_set_buf(&sg[1], &buf->data, sizeof(buf->data));
+
+	spin_lock_irq(&balloon_queue_lock);
+	err = vq->vq_ops->add_buf(vq, sg, 0, 2, buf);
+	if (err) {
+		printk("%s: add_buf err\n", __func__);
+		goto out;
+	}
+	atomic_inc(&virtballoon.inflight_bufs);
+
+	/* TODO: kick several balloon buffers at once */
+	vq->vq_ops->kick(vq);
+out:
+	spin_unlock_irq(&balloon_queue_lock);
+	return err;
+}
+
+static int kvm_balloon_inflate(int32_t npages)
+{
+	LIST_HEAD(tmp_list);
+	struct balloon_page *node, *tmp;
+	struct balloon_buf *buf;
+	u32 *pfn;
+	int allocated = 0;
+	int i, r = -ENOMEM;
+
+	buf = alloc_balloon_buf();

drivers/virtio/kvm_balloon.c line 147

Quoted 10 lines
drivers/virtio/kvm_balloon.c new file +10 -0
+	if (!buf)
+		return r;
+
+	pfn = (u32 *)&buf->data;
+	*pfn++ = (u32)npages;
+
+	for (i = 0; i < npages; i++) {
+		node = kzalloc(sizeof(struct balloon_page), GFP_KERNEL);
+		if (!node)
+			goto out_free;

Instead of allocating a node for each page, you could use page->private to create a linked list. You're potentially burning a lot of memory when ballooning down by a large amount otherwise.

Quoted 34 lines
drivers/virtio/kvm_balloon.c new file +33 -0
+		node->bpage = alloc_page(GFP_HIGHUSER | __GFP_NORETRY);
+		if (!node->bpage) {
+			kfree(node);
+			goto out_free;
+		}
+
+		list_add(&node->bp_list, &tmp_list);
+		allocated++;
+		*pfn = page_to_pfn(node->bpage);
+		pfn++;
+	}
+
+	r = send_balloon_buf(CMD_BALLOON_INFLATE, buf);
+	if (r)
+		goto out_free;
+
+	spin_lock(&balloon_plist_lock);
+	list_splice(&tmp_list, &balloon_plist);
+	balloon_size += allocated;
+	totalram_pages -= allocated;
+	dprintk("%s: current balloon size=%d\n", __FUNCTION__,
+	       balloon_size);
+	spin_unlock(&balloon_plist_lock);
+	return allocated;
+
+out_free:
+	list_for_each_entry_safe(node, tmp, &tmp_list, bp_list) {
+		__free_page(node->bpage);
+		list_del(&node->bp_list);
+		kfree(node);
+	}
+	return r;
+}

<snip>

Quoted 17 lines
drivers/virtio/kvm_balloon.c new file +16 -0
+
+static int balloon_probe(struct virtio_device *vdev)
+{
+	struct virtio_pci_device *pvdev = to_vp_device(vdev);
+	int err = -EBUSY;
+
+	if (virtballoon.dev) { 
+		printk("kvm_ballon: error, already registered\n");
+		return -EBUSY;
+	}
+
+	virtballoon.vq = vdev->config->find_vq(vdev, 0, balloon_tx_done);
+	if (IS_ERR(virtballoon.vq)) {
+		printk("%s: error finding vq\n", __func__);
+		return -EINVAL;
+	}

drivers/virtio/kvm_balloon.c line 510

Quoted 10 lines
drivers/virtio/kvm_balloon.c new file +10 -0
+
+	virtballoon.dev = vdev;
+	init_waitqueue_head(&virtballoon.balloon_wait);
+	init_waitqueue_head(&virtballoon.rmmod_wait);
+	atomic_set(&virtballoon.inflight_bufs, 0);
+
+	err = request_irq(pvdev->pci_dev->irq, balloon_irq, IRQF_SHARED,
+			  pvdev->vdev.dev.bus_id, &virtballoon);
+	if (err)
+		goto out_free_vq;

Why is it taking over the irq? This is very, very wrong. A virtio device cannot be dependent on being used on top of the virtio-pci backend.

Quoted 132 lines
drivers/virtio/kvm_balloon.c new file +67 -0
+
+	virtballoon.balloon_thread = kthread_run(balloon_thread,
+						 &virtballoon,
+						 "kvm_balloond");
+	printk("kvm_balloon: registered\n");
+
+	return 0;
+
+out_free_vq:
+	vdev->config->del_vq(virtballoon.vq);
+	virtballoon.dev = NULL;
+	return err;
+}
+
+static void balloon_remove(struct virtio_device *vdev)
+{
+	kthread_stop(virtballoon.balloon_thread);
+	vdev->config->del_vq(virtballoon.vq);
+}
+
+static struct virtio_driver virtio_balloon = {
+	.driver.name =  KBUILD_MODNAME,
+	.driver.owner = THIS_MODULE,
+	.id_table =     id_table,
+	.probe =        balloon_probe,
+	.remove =	__devexit_p(balloon_remove),
+};
+
+module_param(kvm_balloon_debug, int, 0);
+
+static int __init kvm_balloon_init(void)
+{
+	virtballoon.dev = NULL;
+
+	return register_virtio_driver(&virtio_balloon);
+}
+
+static void __exit kvm_balloon_exit(void)
+{
+	spin_lock(&balloon_plist_lock);
+	if (balloon_size) {
+		DEFINE_WAIT(wait);
+
+		virtballoon.target_nrpages += balloon_size;
+		spin_unlock(&balloon_plist_lock);
+		virtballoon.dev->config->set(virtballoon.dev, 0, 
+					   &virtballoon.target_nrpages,
+					   sizeof(virtballoon.target_nrpages));
+		wake_up(&virtballoon.balloon_wait);
+		prepare_to_wait(&virtballoon.rmmod_wait, &wait,
+				TASK_UNINTERRUPTIBLE);
+		schedule();
+		finish_wait(&virtballoon.rmmod_wait, &wait);
+		spin_lock(&balloon_plist_lock);
+	}
+
+	if (balloon_size)
+		printk(KERN_ERR "%s: exit while balloon not empty!\n",
+			__FUNCTION__);
+
+	spin_unlock(&balloon_plist_lock);
+
+	unregister_virtio_driver(&virtio_balloon);
+}
+
+module_init(kvm_balloon_init);
+module_exit(kvm_balloon_exit);

Index: linux-2.6-nv/drivers/virtio/virtio_pci.c
===================================================================

drivers/virtio/virtio_pci.c +1 -20
@@ -30,20 +30,6 @@ MODULE_DESCRIPTION("virtio-pci");
 MODULE_LICENSE("GPL");
 MODULE_VERSION("1");
 
-/* Our device structure */
-struct virtio_pci_device
-{
-	struct virtio_device vdev;
-	struct pci_dev *pci_dev;
-
-	/* the IO mapping for the PCI config space */
-	void *ioaddr;
-
-	/* a list of queues so we can dispatch IRQs */
-	spinlock_t lock;
-	struct list_head virtqueues;
-};
-
 struct virtio_pci_vq_info
 {
 	/* the actual virtqueue */
@@ -67,6 +53,7 @@ static struct pci_device_id virtio_pci_i
 	{ 0x1AF4, 0x1000, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, /* Dummy entry */
 	{ 0x1AF4, 0x1001, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, /* Dummy entry */
 	{ 0x1AF4, 0x1002, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, /* Dummy entry */
+	{ 0x1AF4, 0x1003, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, /* Balloon */
 	{ 0 },
 };
 
@@ -89,12 +76,6 @@ static struct device virtio_pci_root = {
 /* Unique numbering for devices under the kvm root */
 static unsigned int dev_index;
 
-/* Convert a generic virtio device to our structure */
-static struct virtio_pci_device *to_vp_device(struct virtio_device *vdev)
-{
-	return container_of(vdev, struct virtio_pci_device, vdev);
-}
-
 /* virtio config->feature() implementation */
 static bool vp_feature(struct virtio_device *vdev, unsigned bit)
 {

Index: linux-2.6-nv/include/linux/virtio_pci.h
===================================================================

include/linux/virtio_pci.h +11 -0
@@ -19,6 +19,26 @@
 
 #include <linux/virtio_config.h>
 
+/* Our device structure */
+struct virtio_pci_device
+{
+	struct virtio_device vdev;
+	struct pci_dev *pci_dev;
+
+	/* the IO mapping for the PCI config space */
+	void *ioaddr;
+
+	/* a list of queues so we can dispatch IRQs */
+	spinlock_t lock;

include/linux/virtio_pci.h line 42

Quoted 10 lines
include/linux/virtio_pci.h +9 -0
+	struct list_head virtqueues;
+};
+
+/* Convert a generic virtio device to our structure */
+struct virtio_pci_device *to_vp_device(struct virtio_device *vdev)
+{
+	return container_of(vdev, struct virtio_pci_device, vdev);
+}
+
 

This should be a big indicator that something is wrong. You should not have to change the encapsulation of the virtio-pci backend.

Regards,

Anthony Liguori

Quoted 14 lines
include/linux/virtio_pci.h +0 -1
 /* A 32-bit r/o bitmask of the features supported by the host */
 #define VIRTIO_PCI_HOST_FEATURES	0
 
 
-------------------------------------------------------------------------

Check out the new SourceForge.net Marketplace.
It's the best place to buy or sell services for
just about anything Open Source.
http://ad.doubleclick.net/clk;164216239;13503038;w?http://sf.net/marketplace
_______________________________________________
kvm-devel mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/kvm-devel