CPU hotplug virtio driver

I'm sending a first draft of my proposed cpu hotplug driver for kvm/virtio The first patch is the kernel module, while the second, the userspace pci device.

The host boots with the maximum cpus it should ever use, through the -smp parameter. Due to real machine constraints (which qemu copies), i386 does not allow for any addition of cpus after boot, so this is the most general way.

I do however, include an "attempt_buffer" in the userspace part. It's purpose is to allow tools like virt-manager to set a max_cpus (-smp), and a desired number of cpus in their configuration files. (and AFAICT, there's is no easy way for them to tell when the backend driver is up and running)

Other than that, it should be pretty much straightforward.

Looking forward for your comments

-------------------------------------------------------------------------
Check out the new SourceForge.net Marketplace.
It's the best place to buy or sell services for
just about anything Open Source.
http://ad.doubleclick.net/clk;164216239;13503038;w?http://sf.net/marketplace

---
 drivers/virtio/Kconfig      |    6 +
 drivers/virtio/Makefile     |    1 +
 drivers/virtio/virtio_cpu.c |  226 +++++++++++++++++++++++++++++++++++++++++++
 drivers/virtio/virtio_pci.c |    1 +
 kernel/cpu.c                |    2 +
 5 files changed, 236 insertions(+), 0 deletions(-)
 create mode 100644 drivers/virtio/virtio_cpu.c

index 73a414c..6665d4d 100644
@@ -23,3 +23,9 @@ config VIRTIO_PCI
 
 	  If unsure, say M.
 
+config VIRTIO_CPU
+	tristate "virtio-cpu"
+	depends on VIRTIO_PCI && EXPERIMENTAL
+	---help---
+	  I'll obviously put a better description later
+

index cc84999..057ebcc 100644
@@ -1,3 +1,4 @@
 obj-$(CONFIG_VIRTIO) += virtio.o
 obj-$(CONFIG_VIRTIO_RING) += virtio_ring.o
 obj-$(CONFIG_VIRTIO_PCI) += virtio_pci.o
+obj-$(CONFIG_VIRTIO_CPU) += virtio_cpu.o

index e1fa68d..8d5a719 100644
@@ -67,6 +67,7 @@ static struct pci_device_id virtio_pci_id_table[] = {
 	{ 0x1AF4, 0x1000, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, /* Dummy entry */
 	{ 0x1AF4, 0x1001, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, /* Dummy entry */
 	{ 0x1AF4, 0x1002, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, /* Dummy entry */
+	{ 0x2523, 0x2325, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, /* cpu hotplug driver */
 	{ 0 },
 };
 

index 6b3a0c1..c432c28 100644
@@ -219,6 +219,7 @@ int cpu_down(unsigned int cpu)
 	mutex_unlock(&cpu_add_remove_lock);
 	return err;
 }
+EXPORT_SYMBOL_GPL(cpu_down);
 #endif /*CONFIG_HOTPLUG_CPU*/
 
 /* Requires cpu_add_remove_lock to be held */

@@ -284,6 +285,7 @@ int __cpuinit cpu_up(unsigned int cpu)
 	mutex_unlock(&cpu_add_remove_lock);
 	return err;
 }
+EXPORT_SYMBOL_GPL(cpu_up);
 
 #ifdef CONFIG_PM_SLEEP_SMP
 static cpumask_t frozen_cpus;

1.5.0.6


-------------------------------------------------------------------------
Check out the new SourceForge.net Marketplace.
It's the best place to buy or sell services for
just about anything Open Source.
http://ad.doubleclick.net/clk;164216239;13503038;w?http://sf.net/marketplace

---
 qemu/Makefile.target     |    2 +-
 qemu/hw/pc.c             |    4 +-
 qemu/hw/pc.h             |    3 +
 qemu/hw/virtio-hotplug.c |  111 ++++++++++++++++++++++++++++++++++++++++++++++
 qemu/monitor.c           |   11 +++++
 qemu/qemu-kvm.h          |    2 +
 6 files changed, 131 insertions(+), 2 deletions(-)
 create mode 100644 qemu/hw/virtio-hotplug.c

index bb7be0f..4d5679a 100644
@@ -464,7 +464,7 @@ VL_OBJS += rtl8139.o
 VL_OBJS+= hypercall.o
 
 # virtio devices
-VL_OBJS += virtio.o virtio-net.o virtio-blk.o
+VL_OBJS += virtio.o virtio-net.o virtio-blk.o virtio-hotplug.o
 
 ifeq ($(TARGET_BASE_ARCH), i386)
 # Hardware support

index 3972ab4..c9d8f89 100644
@@ -1029,7 +1029,9 @@ static void pc_init1(ram_addr_t ram_size, int vga_ram_size,
         }
     }
 
-#define USE_HYPERCALL
+    virtio_hotplug_init(pci_bus);
+
+#undef USE_HYPERCALL
 #ifdef USE_HYPERCALL
     pci_hypercall_init(pci_bus);
 #endif

index 7d1832f..3e2cfb4 100644
@@ -151,6 +151,9 @@ void virtio_net_poll(void);
 void *virtio_blk_init(PCIBus *bus, uint16_t vendor, uint16_t device,
 		      BlockDriverState *bs);
 
+/* virtio-hotplug.c */
+void *virtio_hotplug_init(PCIBus *bus);
+
 /* extboot.c */
 
 void extboot_init(BlockDriverState *bs, int cmd);

index e03c473..f0c55c6 100644
@@ -346,6 +346,16 @@ static void do_cpu_set(int index)
         term_printf("Invalid CPU index\n");
 }
 
+static void do_cpu_set_nr(int value)
+{
+    if ((value < 1)) {
+       term_printf("value out of range\n");
+       return;
+    }
+
+    hotplug_send_cmd(value);
+}
+
 static void do_info_jit(void)
 {
     dump_exec_info(NULL, monitor_fprintf);

@@ -1339,6 +1349,7 @@ static term_cmd_t term_cmds[] = {
       "", "cancel the current VM migration" },
     { "migrate_set_speed", "s", do_migrate_set_speed,
       "value", "set maximum speed (in bytes) for migrations" },
+    { "cpu_set", "i", do_cpu_set_nr, "value", "number of cpus in the guest" },
     { NULL, NULL, },
 };
 

index e4aeb3a..6bf234d 100644
@@ -48,6 +48,8 @@ void kvm_tpr_access_report(CPUState *env, uint64_t rip, int is_write);
 int handle_tpr_access(void *opaque, int vcpu,
 			     uint64_t rip, int is_write);
 
+int hotplug_send_smd(int cpus);
+
 #define ALIGN(x, y)  (((x)+(y)-1) & ~((y)-1))
 #define BITMAP_SIZE(m) (ALIGN(((m)>>TARGET_PAGE_BITS), HOST_LONG_BITS) / 8)
 

1.5.0.6


-------------------------------------------------------------------------
Check out the new SourceForge.net Marketplace.
It's the best place to buy or sell services for
just about anything Open Source.
http://ad.doubleclick.net/clk;164216239;13503038;w?http://sf.net/marketplace

---
 qemu/Makefile.target     |    2 +-
 qemu/hw/pc.c             |    4 +-
 qemu/hw/pc.h             |    3 +
 qemu/hw/virtio-hotplug.c |  111 ++++++++++++++++++++++++++++++++++++++++++++++
 qemu/monitor.c           |   11 +++++
 qemu/qemu-kvm.h          |    2 +
 6 files changed, 131 insertions(+), 2 deletions(-)
 create mode 100644 qemu/hw/virtio-hotplug.c

index bb7be0f..4d5679a 100644
@@ -464,7 +464,7 @@ VL_OBJS += rtl8139.o
 VL_OBJS+= hypercall.o
 
 # virtio devices
-VL_OBJS += virtio.o virtio-net.o virtio-blk.o
+VL_OBJS += virtio.o virtio-net.o virtio-blk.o virtio-hotplug.o
 
 ifeq ($(TARGET_BASE_ARCH), i386)
 # Hardware support

index 3972ab4..c9d8f89 100644
@@ -1029,7 +1029,9 @@ static void pc_init1(ram_addr_t ram_size, int vga_ram_size,
         }
     }
 
-#define USE_HYPERCALL
+    virtio_hotplug_init(pci_bus);
+
+#undef USE_HYPERCALL
 #ifdef USE_HYPERCALL
     pci_hypercall_init(pci_bus);
 #endif

index 7d1832f..3e2cfb4 100644
@@ -151,6 +151,9 @@ void virtio_net_poll(void);
 void *virtio_blk_init(PCIBus *bus, uint16_t vendor, uint16_t device,
 		      BlockDriverState *bs);
 
+/* virtio-hotplug.c */
+void *virtio_hotplug_init(PCIBus *bus);
+
 /* extboot.c */
 
 void extboot_init(BlockDriverState *bs, int cmd);

index e03c473..f0c55c6 100644
@@ -346,6 +346,16 @@ static void do_cpu_set(int index)
         term_printf("Invalid CPU index\n");
 }
 
+static void do_cpu_set_nr(int value)
+{
+    if ((value < 1)) {
+       term_printf("value out of range\n");
+       return;
+    }
+
+    hotplug_send_cmd(value);
+}
+
 static void do_info_jit(void)
 {
     dump_exec_info(NULL, monitor_fprintf);

@@ -1339,6 +1349,7 @@ static term_cmd_t term_cmds[] = {
       "", "cancel the current VM migration" },
     { "migrate_set_speed", "s", do_migrate_set_speed,
       "value", "set maximum speed (in bytes) for migrations" },
+    { "cpu_set", "i", do_cpu_set_nr, "value", "number of cpus in the guest" },
     { NULL, NULL, },
 };
 

index e4aeb3a..6bf234d 100644
@@ -48,6 +48,8 @@ void kvm_tpr_access_report(CPUState *env, uint64_t rip, int is_write);
 int handle_tpr_access(void *opaque, int vcpu,
 			     uint64_t rip, int is_write);
 
+int hotplug_send_smd(int cpus);
+
 #define ALIGN(x, y)  (((x)+(y)-1) & ~((y)-1))
 #define BITMAP_SIZE(m) (ALIGN(((m)>>TARGET_PAGE_BITS), HOST_LONG_BITS) / 8)
 

1.5.0.6

I would much prefer to see cpu hotplug implemented via acpi. Such an implementation would work on older kernels without change, and will also work with other operating systems. It isn't a high-speed interface so virtio doesn't buy us anything.

Linux appears to support it (CONFIG_ACPI_HOTPLUG_CPU) so all that's needed is the host side support (likely qemu/bios only). Of course hacking on acpi is fun, if you're the kind of person than enjoys dental surgery.

error compiling committee.c: too many arguments to function


-------------------------------------------------------------------------
Check out the new SourceForge.net Marketplace.
It's the best place to buy or sell services for
just about anything Open Source.
http://ad.doubleclick.net/clk;164216239;13503038;w?http://sf.net/marketplace

Being acpi so that fun, how can it be that virtio don't by us _anything_ ?

-------------------------------------------------------------------------
Check out the new SourceForge.net Marketplace.
It's the best place to buy or sell services for
just about anything Open Source.
http://ad.doubleclick.net/clk;164216239;13503038;w?http://sf.net/marketplace

Being acpi so that fun, how can it be that virtio don't by us _anything_ ?

Am Mittwoch, 9. Januar 2008 schrieb Glauber de Oliveira Costa:

device.

I personally prefer to use non paravirtualized cpu hotplug and implement the existing interfaces instead in the hypervisor. (sigp/sclp for s390 and maybe acpi for x86 etc.). This is not a performance critical operation.

Christian

-------------------------------------------------------------------------
Check out the new SourceForge.net Marketplace.
It's the best place to buy or sell services for
just about anything Open Source.
http://ad.doubleclick.net/clk;164216239;13503038;w?http://sf.net/marketplace

It's barely "paravirtualized". It's a device driver talking to a hardware from the guest POV, not much different than acpi itself is, but sanner. The only difference is that acpi support "already exists", but virtio is a lot easier, and it wouldn't take long to write a driver like that for windows or anything else (not that I am volunteering ;-)).

Also, in the future the interface can be extended in ways acpi can't handle. (and just because nobody will change the hardware to embrace new things that fast)

That said, if acpi is really the preference here, and this patches have chance, no problem. But it will take me a little more time to implement them ;-)

The power button support that was recently added at least proves that the host->guest notification path works, so that's one tooth that does not need pulling out. The real trouble with acpi is understanding the spec which is very badly written.

_AVI

error compiling committee.c: too many arguments to function


-------------------------------------------------------------------------
Check out the new SourceForge.net Marketplace.
It's the best place to buy or sell services for
just about anything Open Source.
http://ad.doubleclick.net/clk;164216239;13503038;w?http://sf.net/marketplace

Edwin implemented device hotplug support for Xen/Qemu, probably we can reuse the effort.

Edwin:
Can u share the device hotplug patch with Glauber?
thx,eddie

-------------------------------------------------------------------------
This SF.net email is sponsored by: Microsoft Defy all challenges. Microsoft(R) Visual Studio 2008.
http://clk.atdmt.com/MRT/go/vse0120000070mrt/direct/01/

Edwin implemented device hotplug support for Xen/Qemu, probably we can reuse the effort.

Edwin:
Can u share the device hotplug patch with Glauber?
thx,eddie

The power button support that was recently added at least proves that the host->guest notification path works, so that's one tooth that does not need pulling out. The real trouble with acpi is understanding the spec which is very badly written.

_AVI

error compiling committee.c: too many arguments to function

---
 drivers/virtio/Kconfig      |    6 +
 drivers/virtio/Makefile     |    1 +
 drivers/virtio/virtio_cpu.c |  226 +++++++++++++++++++++++++++++++++++++++++++
 drivers/virtio/virtio_pci.c |    1 +
 kernel/cpu.c                |    2 +
 5 files changed, 236 insertions(+), 0 deletions(-)
 create mode 100644 drivers/virtio/virtio_cpu.c

index 73a414c..6665d4d 100644
@@ -23,3 +23,9 @@ config VIRTIO_PCI
 
 	  If unsure, say M.
 
+config VIRTIO_CPU
+	tristate "virtio-cpu"
+	depends on VIRTIO_PCI && EXPERIMENTAL
+	---help---
+	  I'll obviously put a better description later
+

index cc84999..057ebcc 100644
@@ -1,3 +1,4 @@
 obj-$(CONFIG_VIRTIO) += virtio.o
 obj-$(CONFIG_VIRTIO_RING) += virtio_ring.o
 obj-$(CONFIG_VIRTIO_PCI) += virtio_pci.o
+obj-$(CONFIG_VIRTIO_CPU) += virtio_cpu.o

index e1fa68d..8d5a719 100644
@@ -67,6 +67,7 @@ static struct pci_device_id virtio_pci_id_table[] = {
 	{ 0x1AF4, 0x1000, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, /* Dummy entry */
 	{ 0x1AF4, 0x1001, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, /* Dummy entry */
 	{ 0x1AF4, 0x1002, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, /* Dummy entry */
+	{ 0x2523, 0x2325, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, /* cpu hotplug driver */
 	{ 0 },
 };
 

index 6b3a0c1..c432c28 100644
@@ -219,6 +219,7 @@ int cpu_down(unsigned int cpu)
 	mutex_unlock(&cpu_add_remove_lock);
 	return err;
 }
+EXPORT_SYMBOL_GPL(cpu_down);
 #endif /*CONFIG_HOTPLUG_CPU*/
 
 /* Requires cpu_add_remove_lock to be held */

@@ -284,6 +285,7 @@ int __cpuinit cpu_up(unsigned int cpu)
 	mutex_unlock(&cpu_add_remove_lock);
 	return err;
 }
+EXPORT_SYMBOL_GPL(cpu_up);
 
 #ifdef CONFIG_PM_SLEEP_SMP
 static cpumask_t frozen_cpus;

1.5.0.6

I would much prefer to see cpu hotplug implemented via acpi. Such an implementation would work on older kernels without change, and will also work with other operating systems. It isn't a high-speed interface so virtio doesn't buy us anything.

Linux appears to support it (CONFIG_ACPI_HOTPLUG_CPU) so all that's needed is the host side support (likely qemu/bios only). Of course hacking on acpi is fun, if you're the kind of person than enjoys dental surgery.

error compiling committee.c: too many arguments to function

Am Mittwoch, 9. Januar 2008 schrieb Glauber de Oliveira Costa:

device.

I personally prefer to use non paravirtualized cpu hotplug and implement the existing interfaces instead in the hypervisor. (sigp/sclp for s390 and maybe acpi for x86 etc.). This is not a performance critical operation.

Christian