Skip to contentSkip to conversation

Virtio network device migration support

PATCH In review Dor Laor 2 participants 3 messages Mar 3, 2008 – Mar 4, 2008 KVM

lore
Share & tools
lore

Dor Laor

author
#0 lore raw

PATCH 1/1 Virtio network device migration support

Virtio network device migration support

It is composed of state saving and dirty bit tracking.

Added dirty bit tracking for the rx packets. There is no need to add dirty bits for the outgoing packets since we do not write over guest memory. As for the descriptor ring (guest memory), I rather copy the entire ring (3 pages) when saving state than touching the dirty bits every time in fast path.

Besides that the virtio device, pci bus state and the network device states are saved.

qemu/hw/virtio-net.c +52 -1
index eb2a441..612cf6b 100644
@@ -128,6 +128,7 @@ static void virtio_net_receive(void *opaque, const uint8_t *buf, int size)
     hdr = (void *)elem.in_sg[0].iov_base;
     hdr->flags = 0;
     hdr->gso_type = VIRTIO_NET_HDR_GSO_NONE;
+    cpu_physical_memory_set_dirty((ram_addr_t)elem.in_sg[0].iov_base - (ram_addr_t)phys_ram_base);
 
     /* copy in packet.  ugh */
     offset = 0;
@@ -136,6 +137,7 @@ static void virtio_net_receive(void *opaque, const uint8_t *buf, int size)
 	int len = MIN(elem.in_sg[i].iov_len, size - offset);
 	memcpy(elem.in_sg[i].iov_base, buf + offset, len);
 	offset += len;
+        cpu_physical_memory_set_dirty((ram_addr_t)elem.in_sg[i].iov_base - (ram_addr_t)phys_ram_base);
 	i++;
     }
 
@@ -210,6 +212,8 @@ again:
                 else
                     fprintf(stderr, "reading network error %d", len);
             }
+            cpu_physical_memory_set_dirty((ram_addr_t)elem.in_sg[1].iov_base - (ram_addr_t)phys_ram_base);
+            cpu_physical_memory_set_dirty((ram_addr_t)elem.in_sg[0].iov_base - (ram_addr_t)phys_ram_base);
             virtqueue_push(vnet->rx_vq, &elem, sizeof(*hdr) + len);
             vnet->do_notify = 1;
         }
@@ -281,11 +285,52 @@ static void virtio_net_tx_timer(void *opaque)
     virtio_net_flush_tx(n, n->tx_vq);
 }
 
+
+static void virtio_net_save(QEMUFile *f, void *opaque)
+{
+    VirtIONet *n = opaque;
+
+    pci_device_save(&n->vdev.pci_dev, f);
+    qemu_put_buffer(f, n->mac, sizeof n->mac);
+    qemu_put_be32s(f, &n->can_receive);
+
+    virtio_dev_save(f, &n->vdev);
+}
+
+static int virtio_net_load(QEMUFile *f, void *opaque, int version_id)
+{
+    VirtIONet *n = opaque;
+    int ret;
+
+    if (version_id > 1) {
+        fprintf(stderr, "%s: not supporting version > 1\n", __FUNCTION__);
+        return -1;
+    }
+
+    if ((ret = pci_device_load(&n->vdev.pci_dev, f)) < 0)
+        return ret;
+
+    qemu_get_buffer(f, n->mac, sizeof n->mac);
+    qemu_get_be32s(f, &n->can_receive);
+
+
+    if ((ret = virtio_dev_load(f, &n->vdev, version_id)) < 0)
+        return ret;
+
+    /* Make sure we kick the tx */ 
+    qemu_mod_timer(n->tx_timer,
+                   qemu_get_clock(vm_clock) + TX_TIMER_INTERVAL);
+    n->tx_timer_active = 1;
+
+    return 0;
+}
+
 void *virtio_net_init(PCIBus *bus, NICInfo *nd, int devfn)
 {
     VirtIONet *n;
+    const char *info_str = "virtio-net";
 
-    n = (VirtIONet *)virtio_init_pci(bus, "virtio-net", 6900, 0x1000,
+    n = (VirtIONet *)virtio_init_pci(bus, info_str, 6900, 0x1000,
 				     0, VIRTIO_ID_NET,
 				     0x02, 0x00, 0x00,
 				     6, sizeof(VirtIONet));
@@ -308,5 +353,11 @@ void *virtio_net_init(PCIBus *bus, NICInfo *nd, int devfn)
     n->tx_timer = qemu_new_timer(vm_clock, virtio_net_tx_timer, n);
     n->tx_timer_active = 0;
 
+    snprintf(n->vc->info_str, sizeof(n->vc->info_str),
+             "%s macaddr=%02x:%02x:%02x:%02x:%02x:%02x", info_str,
+             nd->macaddr[0], nd->macaddr[1], nd->macaddr[2],
+             nd->macaddr[3], nd->macaddr[4], nd->macaddr[5]);
+    register_savevm(info_str, 1, 1, virtio_net_save, virtio_net_load, n);
+
     return &n->vdev;
 }
qemu/hw/virtio.c +53 -0

59 lines not on this page · raw message

qemu/hw/virtio.h +3 -0
index dee97ba..54632ba 100644
@@ -140,4 +140,7 @@ int virtqueue_pop(VirtQueue *vq, VirtQueueElement *elem);
 
 void virtio_notify(VirtIODevice *vdev, VirtQueue *vq);
 
+void virtio_dev_save(QEMUFile *f, VirtIODevice *vdev);
+int virtio_dev_load(QEMUFile *f, VirtIODevice *vdev, int version_id);
+
 #endif

-------------------------------------------------------------------------
This SF.net email is sponsored by: Microsoft Defy all challenges. Microsoft(R) Visual Studio 2008.
http://clk.atdmt.com/MRT/go/vse0120000070mrt/direct/01/

#1 lore raw

Hi Dor,

Quoted 11 lines · Dor Laor

void *virtio_net_init(PCIBus *bus, NICInfo *nd, int devfn)

 {
     VirtIONet *n;
qemu/hw/virtio-net.c +2 -1
+    const char *info_str = "virtio-net";
 
-    n = (VirtIONet *)virtio_init_pci(bus, "virtio-net", 6900, 0x1000,
+    n = (VirtIONet *)virtio_init_pci(bus, info_str, 6900, 0x1000,
 				     0, VIRTIO_ID_NET,
 				     0x02, 0x00, 0x00,
 				     6, sizeof(VirtIONet));

qemu/hw/virtio-net.c void *virtio_net_init(PCIBus *bus, NICInfo *nd, int devfn) · line 361

Quoted 9 lines
qemu/hw/virtio-net.c +5 -0
@@ -308,5 +353,11 @@ void *virtio_net_init(PCIBus *bus, NICInfo *nd, int devfn)
     n->tx_timer = qemu_new_timer(vm_clock, virtio_net_tx_timer, n);
     n->tx_timer_active = 0;
 
+    snprintf(n->vc->info_str, sizeof(n->vc->info_str),
+             "%s macaddr=%02x:%02x:%02x:%02x:%02x:%02x", info_str,
+             nd->macaddr[0], nd->macaddr[1], nd->macaddr[2],
+             nd->macaddr[3], nd->macaddr[4], nd->macaddr[5]);
+    register_savevm(info_str, 1, 1, virtio_net_save, virtio_net_load, n);
 

I think we need to maintain an instance id and increment it here like we do for the rest of the network cards.

Quoted 21 lines
qemu/hw/virtio-net.c +0 -0
     return &n->vdev;
 }
qemu/hw/virtio.c +11 -0
index 634f869..69fe810 100644
@@ -180,6 +180,59 @@ void virtio_reset(void *opaque)
     }
 }
 
+void virtio_dev_save(QEMUFile *f, VirtIODevice *vdev)
+{
+    int i;
+
+    qemu_put_be32s(f, &vdev->features);
+    qemu_put_be16s(f, &vdev->queue_sel);
+    qemu_put_8s(f, &vdev->status);
+    qemu_put_8s(f, &vdev->isr);
+
+    for(i = 0; i < VIRTIO_PCI_QUEUE_MAX; i++) {
+        if (!vdev->vq[i].vring.num)

qemu/hw/virtio.c void virtio_reset(void *opaque) · line 203

Quoted 10 lines
qemu/hw/virtio.c +9 -0
+            continue;
+        qemu_put_be32s(f, &vdev->vq[i].pfn);
+        qemu_put_be16s(f, &vdev->vq[i].last_avail_idx);
+        qemu_put_be32s(f, &vdev->vq[i].index);
+
+        /* Save the descriptor ring instead of constantly mark them dirty */
+        qemu_put_buffer(f, (uint8_t*)vdev->vq[i].vring.desc, vdev->vq[i].vring.num * sizeof(VRingDesc));
+        qemu_put_buffer(f, (uint8_t*)vdev->vq[i].vring.avail, TARGET_PAGE_SIZE);
+        qemu_put_buffer(f, (uint8_t*)vdev->vq[i].vring.used, TARGET_PAGE_SIZE);
 

I think these two need to be sizeof(VRingAvail) * vring.num and sizeof(VringUsed) * vring.num

Regards,

Anthony Liguori

-------------------------------------------------------------------------
This SF.net email is sponsored by: Microsoft Defy all challenges. Microsoft(R) Visual Studio 2008.
http://clk.atdmt.com/MRT/go/vse0120000070mrt/direct/01/

author
#2 lore raw
Quoted 17 lines · Anthony Liguori

Hi Dor,

Quoted 14 lines · Dor Laor

void *virtio_net_init(PCIBus *bus, NICInfo *nd, int devfn)

 {
     VirtIONet *n;
qemu/hw/virtio-net.c +2 -1
+    const char *info_str = "virtio-net";
 
-    n = (VirtIONet *)virtio_init_pci(bus, "virtio-net", 6900, 0x1000,
+    n = (VirtIONet *)virtio_init_pci(bus, info_str, 6900, 0x1000,
 				     0, VIRTIO_ID_NET,
 				     0x02, 0x00, 0x00,
 				     6, sizeof(VirtIONet));
@@ -308,5 +353,11 @@ void *virtio_net_init(PCIBus *bus, NICInfo *nd, int devfn)
     n->tx_timer = qemu_new_timer(vm_clock, virtio_net_tx_timer, n);
     n->tx_timer_active = 0;
 
Quoted 10 lines

qemu/hw/virtio-net.c void *virtio_net_init(PCIBus *bus, NICInfo *nd, int devfn)

Quoted 6 lines
qemu/hw/virtio-net.c +5 -0
+    snprintf(n->vc->info_str, sizeof(n->vc->info_str),
+             "%s macaddr=%02x:%02x:%02x:%02x:%02x:%02x", info_str,
+             nd->macaddr[0], nd->macaddr[1], nd->macaddr[2],
+             nd->macaddr[3], nd->macaddr[4], nd->macaddr[5]);
+    register_savevm(info_str, 1, 1, virtio_net_save, virtio_net_load, n);
 

I think we need to maintain an instance id and increment it here like we do for the rest of the network cards.

Sure

Quoted 26 lines
Quoted 26 lines
qemu/hw/virtio-net.c +0 -0
     return &n->vdev;
 }
qemu/hw/virtio.c +16 -0
index 634f869..69fe810 100644
@@ -180,6 +180,59 @@ void virtio_reset(void *opaque)
     }
 }
 
+void virtio_dev_save(QEMUFile *f, VirtIODevice *vdev)
+{
+    int i;
+
+    qemu_put_be32s(f, &vdev->features);
+    qemu_put_be16s(f, &vdev->queue_sel);
+    qemu_put_8s(f, &vdev->status);
+    qemu_put_8s(f, &vdev->isr);
+
+    for(i = 0; i < VIRTIO_PCI_QUEUE_MAX; i++) {
+        if (!vdev->vq[i].vring.num)
+            continue;
+        qemu_put_be32s(f, &vdev->vq[i].pfn);
+        qemu_put_be16s(f, &vdev->vq[i].last_avail_idx);
+        qemu_put_be32s(f, &vdev->vq[i].index);
+
Quoted 9 lines

qemu/hw/virtio.c void virtio_reset(void *opaque)

Quoted 5 lines
qemu/hw/virtio.c +4 -0
+        /* Save the descriptor ring instead of constantly mark them dirty */
+        qemu_put_buffer(f, (uint8_t*)vdev->vq[i].vring.desc, vdev->vq[i].vring.num * sizeof(VRingDesc));
+        qemu_put_buffer(f, (uint8_t*)vdev->vq[i].vring.avail, TARGET_PAGE_SIZE);
+        qemu_put_buffer(f, (uint8_t*)vdev->vq[i].vring.used, TARGET_PAGE_SIZE);
 

I think these two need to be sizeof(VRingAvail) * vring.num and sizeof(VringUsed) * vring.num

Right, actually it should be sizeof(VRingAvail) + 2 * vring.num and sizeof(VRingUsedElem) * vring.num + sizeof(VRingUsed) respectively.

Thanks,
Dor

Quoted 4 lines

Regards,

Anthony Liguori

-------------------------------------------------------------------------
This SF.net email is sponsored by: Microsoft Defy all challenges. Microsoft(R) Visual Studio 2008.
http://clk.atdmt.com/MRT/go/vse0120000070mrt/direct/01/