Skip to contentSkip to conversation

vfio: assign idstr for VFIO's mmaped regions for migration

PATCH New Zhao Yan 3 participants 5 messages Jan 8, 2019 – Feb 20, 2019

lore
Share & tools
lore

Zhao Yan

author
#0 lore raw

PATCH 1/1 vfio: assign idstr for VFIO's mmaped regions for migration

if multiple regions in vfio are mmaped, their corresponding ramblocks are like below, i.e. their idstrs are "".

(qemu) info ramblock
Block Name PSize Offset Used Total

pc.ram      4 KiB  0x0000000000000000 0x0000000020000000 0x0000000020000000
            4 KiB  0x0000000021100000 0x0000000020000000 0x0000000020000000
            4 KiB  0x0000000020900000 0x0000000000800000 0x0000000000800000
            4 KiB  0x0000000020240000 0x0000000000687000 0x0000000000687000
            4 KiB  0x00000000200c0000 0x0000000000178000 0x0000000000178000
pc.bios     4 KiB  0x0000000020000000 0x0000000000040000 0x0000000000040000
pc.rom      4 KiB  0x0000000020040000 0x0000000000020000 0x0000000000020000

This is because ramblocks' idstr are assigned by calling vmstate_register_ram(), but memory region of type ram device ptr does not call vmstate_register_ram().
vfio_region_mmap

        |->memory_region_init_ram_device_ptr
               |-> memory_region_init_ram_ptr

Without empty idstrs will cause problem to snapshot copying during migration, because it uses ramblocks' idstr to identify ramblocks. ram_save_setup {
…

  RAMBLOCK_FOREACH(block) {
      qemu_put_byte(f, strlen(block->idstr));
      qemu_put_buffer(f, (uint8_t *)block->idstr,strlen(block->idstr));
      qemu_put_be64(f, block->used_length);
  }
  …
}
ram_load() {
    block = qemu_ram_block_by_name(id);
    if (block) {
        if (length != block->used_length) {
            qemu_ram_resize(block, length, &local_err);
        }
     ….

}
}

Therefore, in this patch,
vmstate_register_ram() is called for memory region of type ram ptr, also a unique vfioid is assigned to vfio devices across source and target vms.
e.g. in source vm, use qemu parameter
-device vfio-pci,sysfsdev=/sys/bus/pci/devices/0000:00:02.0/ 882cc4da-dede-11e7-9180-078a62063ab1,vfioid=igd

and in target vm, use qemu paramter
-device vfio-pci,sysfsdev=/sys/bus/pci/devices/0000:00:02.0/ 5ac1fb20-2bbf-4842-bb7e-36c58c3be9cd,vfioid=igd

---
 hw/vfio/pci.c                 | 8 +++++++-
 include/hw/vfio/vfio-common.h | 1 +
 memory.c                      | 4 ++++
 3 files changed, 12 insertions(+), 1 deletion(-)
hw/vfio/pci.c +7 -1
index c0cb1ec289..7bc2ed0752 100644
@@ -2533,7 +2533,12 @@ static void vfio_populate_device(VFIOPCIDevice *vdev, Error **errp)
     }
 
     for (i = VFIO_PCI_BAR0_REGION_INDEX; i < VFIO_PCI_ROM_REGION_INDEX; i++) {
-        char *name = g_strdup_printf("%s BAR %d", vbasedev->name, i);
+        char *name;
+        if (vbasedev->vfioid) {
+            name = g_strdup_printf("%s BAR %d", vbasedev->vfioid, i);
+        } else {
+            name = g_strdup_printf("%s BAR %d", vbasedev->name, i);
+        }
 
         ret = vfio_region_setup(OBJECT(vdev), vbasedev,
                                 &vdev->bars[i].region, i, name);
@@ -3180,6 +3185,7 @@ static void vfio_instance_init(Object *obj)
 static Property vfio_pci_dev_properties[] = {
     DEFINE_PROP_PCI_HOST_DEVADDR("host", VFIOPCIDevice, host),
     DEFINE_PROP_STRING("sysfsdev", VFIOPCIDevice, vbasedev.sysfsdev),
+    DEFINE_PROP_STRING("vfioid", VFIOPCIDevice, vbasedev.vfioid),
     DEFINE_PROP_ON_OFF_AUTO("display", VFIOPCIDevice,
                             display, ON_OFF_AUTO_OFF),
     DEFINE_PROP_UINT32("x-intx-mmap-timeout-ms", VFIOPCIDevice,
include/hw/vfio/vfio-common.h +1 -0
index 1b434d02f6..84bab94f52 100644
@@ -108,6 +108,7 @@ typedef struct VFIODevice {
     struct VFIOGroup *group;
     char *sysfsdev;
     char *name;
+    char *vfioid;
     DeviceState *dev;
     int fd;
     int type;
memory.c +4 -0
index d14c6dec1d..dbb29fa989 100644
@@ -1588,6 +1588,7 @@ void memory_region_init_ram_ptr(MemoryRegion *mr,
                                 uint64_t size,
                                 void *ptr)
 {
+    DeviceState *owner_dev;
     memory_region_init(mr, owner, name, size);
     mr->ram = true;
     mr->terminates = true;
@@ -1597,6 +1598,9 @@ void memory_region_init_ram_ptr(MemoryRegion *mr,
     /* qemu_ram_alloc_from_ptr cannot fail with ptr != NULL.  */
     assert(ptr != NULL);
     mr->ram_block = qemu_ram_alloc_from_ptr(size, ptr, mr, &error_fatal);
+
+    owner_dev = DEVICE(owner);
+    vmstate_register_ram(mr, owner_dev);
 }
 
 void memory_region_init_ram_device_ptr(MemoryRegion *mr,
Signature · 1 line
2.17.1
author
#1 lore raw

PATCH 1/1 vfio: assign idstr for VFIO's mmaped regions for migration

if multiple regions in vfio are mmaped, their corresponding ramblocks are like below, i.e. their idstrs are "".

(qemu) info ramblock
Block Name PSize Offset Used Total

pc.ram      4 KiB  0x0000000000000000 0x0000000020000000 0x0000000020000000
            4 KiB  0x0000000021100000 0x0000000020000000 0x0000000020000000
            4 KiB  0x0000000020900000 0x0000000000800000 0x0000000000800000
            4 KiB  0x0000000020240000 0x0000000000687000 0x0000000000687000
            4 KiB  0x00000000200c0000 0x0000000000178000 0x0000000000178000
pc.bios     4 KiB  0x0000000020000000 0x0000000000040000 0x0000000000040000
pc.rom      4 KiB  0x0000000020040000 0x0000000000020000 0x0000000000020000

This is because ramblocks' idstr are assigned by calling vmstate_register_ram(), but memory region of type ram device ptr does not call vmstate_register_ram().
vfio_region_mmap

        |->memory_region_init_ram_device_ptr
               |-> memory_region_init_ram_ptr

Without empty idstrs will cause problem to snapshot copying during migration, because it uses ramblocks' idstr to identify ramblocks. ram_save_setup {
…

  RAMBLOCK_FOREACH(block) {
      qemu_put_byte(f, strlen(block->idstr));
      qemu_put_buffer(f, (uint8_t *)block->idstr,strlen(block->idstr));
      qemu_put_be64(f, block->used_length);
  }
  …
}
ram_load() {
    block = qemu_ram_block_by_name(id);
    if (block) {
        if (length != block->used_length) {
            qemu_ram_resize(block, length, &local_err);
        }
     ….

}
}

Therefore, in this patch,
vmstate_register_ram() is called for memory region of type ram ptr, also a unique vfioid is assigned to vfio devices across source and target vms.
e.g. in source vm, use qemu parameter
-device vfio-pci,sysfsdev=/sys/bus/pci/devices/0000:00:02.0/ 882cc4da-dede-11e7-9180-078a62063ab1,vfioid=igd

and in target vm, use qemu paramter
-device vfio-pci,sysfsdev=/sys/bus/pci/devices/0000:00:02.0/ 5ac1fb20-2bbf-4842-bb7e-36c58c3be9cd,vfioid=igd

---
 hw/vfio/pci.c                 | 8 +++++++-
 include/hw/vfio/vfio-common.h | 1 +
 memory.c                      | 4 ++++
 3 files changed, 12 insertions(+), 1 deletion(-)
hw/vfio/pci.c +7 -1
index c0cb1ec289..7bc2ed0752 100644
@@ -2533,7 +2533,12 @@ static void vfio_populate_device(VFIOPCIDevice *vdev, Error **errp)
     }
 
     for (i = VFIO_PCI_BAR0_REGION_INDEX; i < VFIO_PCI_ROM_REGION_INDEX; i++) {
-        char *name = g_strdup_printf("%s BAR %d", vbasedev->name, i);
+        char *name;
+        if (vbasedev->vfioid) {
+            name = g_strdup_printf("%s BAR %d", vbasedev->vfioid, i);
+        } else {
+            name = g_strdup_printf("%s BAR %d", vbasedev->name, i);
+        }
 
         ret = vfio_region_setup(OBJECT(vdev), vbasedev,
                                 &vdev->bars[i].region, i, name);
@@ -3180,6 +3185,7 @@ static void vfio_instance_init(Object *obj)
 static Property vfio_pci_dev_properties[] = {
     DEFINE_PROP_PCI_HOST_DEVADDR("host", VFIOPCIDevice, host),
     DEFINE_PROP_STRING("sysfsdev", VFIOPCIDevice, vbasedev.sysfsdev),
+    DEFINE_PROP_STRING("vfioid", VFIOPCIDevice, vbasedev.vfioid),
     DEFINE_PROP_ON_OFF_AUTO("display", VFIOPCIDevice,
                             display, ON_OFF_AUTO_OFF),
     DEFINE_PROP_UINT32("x-intx-mmap-timeout-ms", VFIOPCIDevice,
include/hw/vfio/vfio-common.h +1 -0
index 1b434d02f6..84bab94f52 100644
@@ -108,6 +108,7 @@ typedef struct VFIODevice {
     struct VFIOGroup *group;
     char *sysfsdev;
     char *name;
+    char *vfioid;
     DeviceState *dev;
     int fd;
     int type;
memory.c +4 -0
index d14c6dec1d..dbb29fa989 100644
@@ -1588,6 +1588,7 @@ void memory_region_init_ram_ptr(MemoryRegion *mr,
                                 uint64_t size,
                                 void *ptr)
 {
+    DeviceState *owner_dev;
     memory_region_init(mr, owner, name, size);
     mr->ram = true;
     mr->terminates = true;
@@ -1597,6 +1598,9 @@ void memory_region_init_ram_ptr(MemoryRegion *mr,
     /* qemu_ram_alloc_from_ptr cannot fail with ptr != NULL.  */
     assert(ptr != NULL);
     mr->ram_block = qemu_ram_alloc_from_ptr(size, ptr, mr, &error_fatal);
+
+    owner_dev = DEVICE(owner);
+    vmstate_register_ram(mr, owner_dev);
 }
 
 void memory_region_init_ram_device_ptr(MemoryRegion *mr,
Signature · 1 line
2.17.1
#2 lore raw
Quoted 50 lines · On Tue, 8 Jan 2019 01:03:48 -0500 Zhao Yan <[email protected]> wrote:

if multiple regions in vfio are mmaped, their corresponding ramblocks are like below, i.e. their idstrs are "".

(qemu) info ramblock
Block Name PSize Offset Used Total

pc.ram      4 KiB  0x0000000000000000 0x0000000020000000 0x0000000020000000
            4 KiB  0x0000000021100000 0x0000000020000000 0x0000000020000000
            4 KiB  0x0000000020900000 0x0000000000800000 0x0000000000800000
            4 KiB  0x0000000020240000 0x0000000000687000 0x0000000000687000
            4 KiB  0x00000000200c0000 0x0000000000178000 0x0000000000178000
pc.bios     4 KiB  0x0000000020000000 0x0000000000040000 0x0000000000040000
pc.rom      4 KiB  0x0000000020040000 0x0000000000020000 0x0000000000020000

This is because ramblocks' idstr are assigned by calling vmstate_register_ram(), but memory region of type ram device ptr does not call vmstate_register_ram().
vfio_region_mmap

        |->memory_region_init_ram_device_ptr
               |-> memory_region_init_ram_ptr

Without empty idstrs will cause problem to snapshot copying during migration, because it uses ramblocks' idstr to identify ramblocks. ram_save_setup {
…

  RAMBLOCK_FOREACH(block) {
      qemu_put_byte(f, strlen(block->idstr));
      qemu_put_buffer(f, (uint8_t *)block->idstr,strlen(block->idstr));
      qemu_put_be64(f, block->used_length);
  }
  …
}
ram_load() {
    block = qemu_ram_block_by_name(id);
    if (block) {
        if (length != block->used_length) {
            qemu_ram_resize(block, length, &local_err);
        }
     ….

}
}

Therefore, in this patch,
vmstate_register_ram() is called for memory region of type ram ptr, also a unique vfioid is assigned to vfio devices across source and target vms.
e.g. in source vm, use qemu parameter
-device vfio-pci,sysfsdev=/sys/bus/pci/devices/0000:00:02.0/ 882cc4da-dede-11e7-9180-078a62063ab1,vfioid=igd

Quoted 4 lines

and in target vm, use qemu paramter
-device vfio-pci,sysfsdev=/sys/bus/pci/devices/0000:00:02.0/ 5ac1fb20-2bbf-4842-bb7e-36c58c3be9cd,vfioid=igd

Why wouldn't we just use the id= (DeviceState.id) value instead of adding yet another one? I can't imagine anyone, especially libvirt, wants to deal with a vfio specific id for a device.

Quoted 58 lines
---
 hw/vfio/pci.c                 | 8 +++++++-
 include/hw/vfio/vfio-common.h | 1 +
 memory.c                      | 4 ++++
 3 files changed, 12 insertions(+), 1 deletion(-)
hw/vfio/pci.c +7 -1
index c0cb1ec289..7bc2ed0752 100644
@@ -2533,7 +2533,12 @@ static void vfio_populate_device(VFIOPCIDevice *vdev, Error **errp)
     }
 
     for (i = VFIO_PCI_BAR0_REGION_INDEX; i < VFIO_PCI_ROM_REGION_INDEX; i++) {
-        char *name = g_strdup_printf("%s BAR %d", vbasedev->name, i);
+        char *name;
+        if (vbasedev->vfioid) {
+            name = g_strdup_printf("%s BAR %d", vbasedev->vfioid, i);
+        } else {
+            name = g_strdup_printf("%s BAR %d", vbasedev->name, i);
+        }
 
         ret = vfio_region_setup(OBJECT(vdev), vbasedev,
                                 &vdev->bars[i].region, i, name);
@@ -3180,6 +3185,7 @@ static void vfio_instance_init(Object *obj)
 static Property vfio_pci_dev_properties[] = {
     DEFINE_PROP_PCI_HOST_DEVADDR("host", VFIOPCIDevice, host),
     DEFINE_PROP_STRING("sysfsdev", VFIOPCIDevice, vbasedev.sysfsdev),
+    DEFINE_PROP_STRING("vfioid", VFIOPCIDevice, vbasedev.vfioid),
     DEFINE_PROP_ON_OFF_AUTO("display", VFIOPCIDevice,
                             display, ON_OFF_AUTO_OFF),
     DEFINE_PROP_UINT32("x-intx-mmap-timeout-ms", VFIOPCIDevice,
include/hw/vfio/vfio-common.h +1 -0
index 1b434d02f6..84bab94f52 100644
@@ -108,6 +108,7 @@ typedef struct VFIODevice {
     struct VFIOGroup *group;
     char *sysfsdev;
     char *name;
+    char *vfioid;
     DeviceState *dev;
     int fd;
     int type;
memory.c +1 -0
index d14c6dec1d..dbb29fa989 100644
@@ -1588,6 +1588,7 @@ void memory_region_init_ram_ptr(MemoryRegion *mr,
                                 uint64_t size,
                                 void *ptr)
 {
+    DeviceState *owner_dev;
     memory_region_init(mr, owner, name, size);
     mr->ram = true;
     mr->terminates = true;

memory.c void memory_region_init_ram_ptr(MemoryRegion *mr, · line 1603

Quoted 6 lines
memory.c +3 -0
@@ -1597,6 +1598,9 @@ void memory_region_init_ram_ptr(MemoryRegion *mr,
     /* qemu_ram_alloc_from_ptr cannot fail with ptr != NULL.  */
     assert(ptr != NULL);
     mr->ram_block = qemu_ram_alloc_from_ptr(size, ptr, mr, &error_fatal);
+
+    owner_dev = DEVICE(owner);
+    vmstate_register_ram(mr, owner_dev);

Where does the corresponding vmstate_unregister_ram() call occur when unplugged? Thanks,

Alex

Quoted 3 lines
memory.c +0 -0
 }
 
 void memory_region_init_ram_device_ptr(MemoryRegion *mr,
author
#3 lore raw
Quoted 53 lines · Alex Williamson
Quoted 50 lines · On Tue, 8 Jan 2019 01:03:48 -0500 Zhao Yan <[email protected]> wrote:

if multiple regions in vfio are mmaped, their corresponding ramblocks are like below, i.e. their idstrs are "".

(qemu) info ramblock
Block Name PSize Offset Used Total

pc.ram      4 KiB  0x0000000000000000 0x0000000020000000 0x0000000020000000
            4 KiB  0x0000000021100000 0x0000000020000000 0x0000000020000000
            4 KiB  0x0000000020900000 0x0000000000800000 0x0000000000800000
            4 KiB  0x0000000020240000 0x0000000000687000 0x0000000000687000
            4 KiB  0x00000000200c0000 0x0000000000178000 0x0000000000178000
pc.bios     4 KiB  0x0000000020000000 0x0000000000040000 0x0000000000040000
pc.rom      4 KiB  0x0000000020040000 0x0000000000020000 0x0000000000020000

This is because ramblocks' idstr are assigned by calling vmstate_register_ram(), but memory region of type ram device ptr does not call vmstate_register_ram().
vfio_region_mmap

        |->memory_region_init_ram_device_ptr
               |-> memory_region_init_ram_ptr

Without empty idstrs will cause problem to snapshot copying during migration, because it uses ramblocks' idstr to identify ramblocks. ram_save_setup {
…

  RAMBLOCK_FOREACH(block) {
      qemu_put_byte(f, strlen(block->idstr));
      qemu_put_buffer(f, (uint8_t *)block->idstr,strlen(block->idstr));
      qemu_put_be64(f, block->used_length);
  }
  …
}
ram_load() {
    block = qemu_ram_block_by_name(id);
    if (block) {
        if (length != block->used_length) {
            qemu_ram_resize(block, length, &local_err);
        }
     ….

}
}

Therefore, in this patch,
vmstate_register_ram() is called for memory region of type ram ptr, also a unique vfioid is assigned to vfio devices across source and target vms.
e.g. in source vm, use qemu parameter
-device vfio-pci,sysfsdev=/sys/bus/pci/devices/0000:00:02.0/ 882cc4da-dede-11e7-9180-078a62063ab1,vfioid=igd

Quoted 9 lines
Quoted 4 lines

and in target vm, use qemu paramter
-device vfio-pci,sysfsdev=/sys/bus/pci/devices/0000:00:02.0/ 5ac1fb20-2bbf-4842-bb7e-36c58c3be9cd,vfioid=igd

Why wouldn't we just use the id= (DeviceState.id) value instead of adding yet another one? I can't imagine anyone, especially libvirt, wants to deal with a vfio specific id for a device.

hi Alex
You are right! DeviceState.id can be used here. Thanks for your suggestion.

Quoted 59 lines
Quoted 59 lines
---
 hw/vfio/pci.c                 | 8 +++++++-
 include/hw/vfio/vfio-common.h | 1 +
 memory.c                      | 4 ++++
 3 files changed, 12 insertions(+), 1 deletion(-)
hw/vfio/pci.c +7 -1
index c0cb1ec289..7bc2ed0752 100644
@@ -2533,7 +2533,12 @@ static void vfio_populate_device(VFIOPCIDevice *vdev, Error **errp)
     }
 
     for (i = VFIO_PCI_BAR0_REGION_INDEX; i < VFIO_PCI_ROM_REGION_INDEX; i++) {
-        char *name = g_strdup_printf("%s BAR %d", vbasedev->name, i);
+        char *name;
+        if (vbasedev->vfioid) {
+            name = g_strdup_printf("%s BAR %d", vbasedev->vfioid, i);
+        } else {
+            name = g_strdup_printf("%s BAR %d", vbasedev->name, i);
+        }
 
         ret = vfio_region_setup(OBJECT(vdev), vbasedev,
                                 &vdev->bars[i].region, i, name);
@@ -3180,6 +3185,7 @@ static void vfio_instance_init(Object *obj)
 static Property vfio_pci_dev_properties[] = {
     DEFINE_PROP_PCI_HOST_DEVADDR("host", VFIOPCIDevice, host),
     DEFINE_PROP_STRING("sysfsdev", VFIOPCIDevice, vbasedev.sysfsdev),
+    DEFINE_PROP_STRING("vfioid", VFIOPCIDevice, vbasedev.vfioid),
     DEFINE_PROP_ON_OFF_AUTO("display", VFIOPCIDevice,
                             display, ON_OFF_AUTO_OFF),
     DEFINE_PROP_UINT32("x-intx-mmap-timeout-ms", VFIOPCIDevice,
include/hw/vfio/vfio-common.h +1 -0
index 1b434d02f6..84bab94f52 100644
@@ -108,6 +108,7 @@ typedef struct VFIODevice {
     struct VFIOGroup *group;
     char *sysfsdev;
     char *name;
+    char *vfioid;
     DeviceState *dev;
     int fd;
     int type;
memory.c +1 -0
index d14c6dec1d..dbb29fa989 100644
@@ -1588,6 +1588,7 @@ void memory_region_init_ram_ptr(MemoryRegion *mr,
                                 uint64_t size,
                                 void *ptr)
 {
+    DeviceState *owner_dev;
     memory_region_init(mr, owner, name, size);
     mr->ram = true;
     mr->terminates = true;
@@ -1597,6 +1598,9 @@ void memory_region_init_ram_ptr(MemoryRegion *mr,
     /* qemu_ram_alloc_from_ptr cannot fail with ptr != NULL.  */
Quoted 9 lines

memory.c void memory_region_init_ram_ptr(MemoryRegion *mr, · line 1603

Quoted 5 lines
memory.c +3 -0
     assert(ptr != NULL);
     mr->ram_block = qemu_ram_alloc_from_ptr(size, ptr, mr, &error_fatal);
+
+    owner_dev = DEVICE(owner);
+    vmstate_register_ram(mr, owner_dev);

Where does the corresponding vmstate_unregister_ram() call occur when unplugged? Thanks,

sorry, I just updated my qemu code base and found that in migration/ram.c now it will not save/restore ramblocks who do not call vmstate_regitser_ram().
therefore, the vmstate_register_ram() may not be necessary for memory region mapped to device resources, as it's better to save/restore that part of memory from vendor driver side.
So, do you think it's ok to just call qemu_ram_set_idstr() to set idstr for ramblocks of mmaped region?

Thanks
Yan

Quoted 6 lines

Alex

Quoted 3 lines
memory.c +0 -0
 }
 
 void memory_region_init_ram_device_ptr(MemoryRegion *mr,
#4 lore raw
Quoted 84 lines

-----Original Message-----
From: Qemu-devel [mailto:[email protected]] On Behalf Of Zhao Yan
Sent: Thursday, January 10, 2019 9:19 AM
To: Alex Williamson <[email protected]>
Cc: [email protected]; [email protected]
Subject: Re: [Qemu-devel] [PATCH] vfio: assign idstr for VFIO's mmaped regions for migration

Quoted 8 lines · Alex Williamson
Quoted 5 lines · On Tue, 8 Jan 2019 01:03:48 -0500 Zhao Yan <[email protected]> wrote:

if multiple regions in vfio are mmaped, their corresponding ramblocks are like below, i.e. their idstrs are "".

(qemu) info ramblock
Block Name PSize Offset Used

Total

Quoted 1 line
Quoted 1 line

pc.ram 4 KiB 0x0000000000000000 0x0000000020000000

0x0000000020000000

Quoted 1 line
Quoted 1 line
            4 KiB  0x0000000021100000 0x0000000020000000

0x0000000020000000

Quoted 1 line
Quoted 1 line
            4 KiB  0x0000000020900000 0x0000000000800000

0x0000000000800000

Quoted 1 line
Quoted 1 line
            4 KiB  0x0000000020240000 0x0000000000687000

0x0000000000687000

Quoted 1 line
Quoted 1 line
            4 KiB  0x00000000200c0000 0x0000000000178000

0x0000000000178000

Quoted 1 line
Quoted 1 line

pc.bios 4 KiB 0x0000000020000000 0x0000000000040000

0x0000000000040000

Quoted 1 line
Quoted 1 line

pc.rom 4 KiB 0x0000000020040000 0x0000000000020000

0x0000000000020000

Quoted 3 lines
Quoted 3 lines

This is because ramblocks' idstr are assigned by calling vmstate_register_ram(), but memory region of type ram device ptr does

not

Quoted 35 lines
Quoted 35 lines

call vmstate_register_ram().
vfio_region_mmap

        |->memory_region_init_ram_device_ptr
               |-> memory_region_init_ram_ptr

Without empty idstrs will cause problem to snapshot copying during migration, because it uses ramblocks' idstr to identify ramblocks. ram_save_setup {
…

  RAMBLOCK_FOREACH(block) {
      qemu_put_byte(f, strlen(block->idstr));
      qemu_put_buffer(f, (uint8_t *)block->idstr,strlen(block->idstr));
      qemu_put_be64(f, block->used_length);
  }
  …
}
ram_load() {
    block = qemu_ram_block_by_name(id);
    if (block) {
        if (length != block->used_length) {
            qemu_ram_resize(block, length, &local_err);
        }
     ….

}
}

Therefore, in this patch,
vmstate_register_ram() is called for memory region of type ram ptr, also a unique vfioid is assigned to vfio devices across source and target vms.
e.g. in source vm, use qemu parameter
-device vfio-pci,sysfsdev=/sys/bus/pci/devices/0000:00:02.0/ 882cc4da-dede-11e7-9180-078a62063ab1,vfioid=igd

Quoted 9 lines
Quoted 4 lines

and in target vm, use qemu paramter
-device vfio-pci,sysfsdev=/sys/bus/pci/devices/0000:00:02.0/ 5ac1fb20-2bbf-4842-bb7e-36c58c3be9cd,vfioid=igd

Why wouldn't we just use the id= (DeviceState.id) value instead of adding yet another one? I can't imagine anyone, especially libvirt, wants to deal with a vfio specific id for a device.

Quoted 3 lines

hi Alex
You are right! DeviceState.id can be used here. Thanks for your suggestion.

Then Libvirt and/or Nova need to keep the device id unchanged.

Quoted 84 lines
Quoted 8 lines
Quoted 8 lines
---
 hw/vfio/pci.c                 | 8 +++++++-
 include/hw/vfio/vfio-common.h | 1 +
 memory.c                      | 4 ++++
 3 files changed, 12 insertions(+), 1 deletion(-)
Quoted 5 lines

hw/vfio/pci.c static void vfio_populate_device(VFIOPCIDevice

Quoted 5 lines
hw/vfio/pci.c +0 -0
index c0cb1ec289..7bc2ed0752 100644
@@ -2533,7 +2533,12 @@ static void vfio_populate_device(VFIOPCIDevice

*vdev, Error **errp)

Quoted 3 lines

hw/vfio/pci.c static void vfio_populate_device(VFIOPCIDevice · line 2535

Quoted 3 lines
hw/vfio/pci.c +0 -0
     }
 
     for (i = VFIO_PCI_BAR0_REGION_INDEX; i <

VFIO_PCI_ROM_REGION_INDEX; i++) {

Quoted 8 lines
Quoted 8 lines
hw/vfio/pci.c +6 -1
-        char *name = g_strdup_printf("%s BAR %d", vbasedev->name, i);
+        char *name;
+        if (vbasedev->vfioid) {
+            name = g_strdup_printf("%s BAR %d", vbasedev->vfioid, i);
+        } else {
+            name = g_strdup_printf("%s BAR %d", vbasedev->name, i);
+        }
 
Quoted 6 lines

hw/vfio/pci.c static void vfio_instance_init(Object *obj) · line 3187

Quoted 6 lines
hw/vfio/pci.c +0 -0
         ret = vfio_region_setup(OBJECT(vdev), vbasedev,
                                 &vdev->bars[i].region, i, name);
@@ -3180,6 +3185,7 @@ static void vfio_instance_init(Object *obj)
 static Property vfio_pci_dev_properties[] = {
     DEFINE_PROP_PCI_HOST_DEVADDR("host", VFIOPCIDevice, host),
     DEFINE_PROP_STRING("sysfsdev", VFIOPCIDevice,

vbasedev.sysfsdev),

Quoted 13 lines
Quoted 13 lines
hw/vfio/pci.c +1 -0
+    DEFINE_PROP_STRING("vfioid", VFIOPCIDevice, vbasedev.vfioid),
     DEFINE_PROP_ON_OFF_AUTO("display", VFIOPCIDevice,
                             display, ON_OFF_AUTO_OFF),
     DEFINE_PROP_UINT32("x-intx-mmap-timeout-ms", VFIOPCIDevice,
include/hw/vfio/vfio-common.h +1 -0
index 1b434d02f6..84bab94f52 100644
@@ -108,6 +108,7 @@ typedef struct VFIODevice {
     struct VFIOGroup *group;
     char *sysfsdev;
     char *name;
+    char *vfioid;
Quoted 8 lines

memory.c void

Quoted 8 lines
include/hw/vfio/vfio-common.h +0 -0
     DeviceState *dev;
     int fd;
     int type;
memory.c +0 -0
index d14c6dec1d..dbb29fa989 100644
@@ -1588,6 +1588,7 @@ void

memory_region_init_ram_ptr(MemoryRegion *mr,

Quoted 8 lines

memory.c void

Quoted 8 lines
memory.c +1 -0
                                 uint64_t size,
                                 void *ptr)
 {
+    DeviceState *owner_dev;
     memory_region_init(mr, owner, name, size);
     mr->ram = true;
     mr->terminates = true;
@@ -1597,6 +1598,9 @@ void

memory_region_init_ram_ptr(MemoryRegion *mr,

Quoted 3 lines

memory.c void memory_region_init_ram_ptr(MemoryRegion *mr,

Quoted 3 lines
memory.c +0 -0
     /* qemu_ram_alloc_from_ptr cannot fail with ptr != NULL.  */
     assert(ptr != NULL);
     mr->ram_block = qemu_ram_alloc_from_ptr(size, ptr, mr,

&error_fatal);

Quoted 7 lines

memory.c void memory_region_init_ram_ptr(MemoryRegion *mr, · line 1603

Quoted 3 lines
memory.c +3 -0
+
+    owner_dev = DEVICE(owner);
+    vmstate_register_ram(mr, owner_dev);

Where does the corresponding vmstate_unregister_ram() call occur when unplugged? Thanks,

sorry, I just updated my qemu code base and found that in migration/ram.c now it will not save/restore ramblocks who do not call vmstate_regitser_ram().
therefore, the vmstate_register_ram() may not be necessary for memory region mapped to device resources, as it's better to save/restore that part of memory from vendor driver side.
So, do you think it's ok to just call qemu_ram_set_idstr() to set idstr for ramblocks of mmaped region?

Quoted 3 lines

Thanks
Yan

Why not invoking vmstate_register_ram() in vfio_region_mmap and Invoking vmstate_unregister_ram() in vfio_region_exit?

Regards,
-Gonglei