Skip to contentSkip to conversation

Enable live migration of vfio-ap devices

PATCH In review Anthony Krowiak 2 participants 12 messages Apr 9 – Apr 16

lore
Share & tools
lore
#0 lore raw

v2 cover Enable live migration of vfio-ap devices

This patch series enables live guest migration for s390 guests using AP (Adjunct Processor) crypto devices passed through via the VFIO framework.

The s390 AP architecture provides cryptographic coprocessor and accelerator functions to guests. The vfio_ap device driver allows these AP devices to be passed through to KVM guests, giving them access to the cryptographic capabilities of CEXxS crypto cards.

Previously, guests using vfio-ap devices could not be migrated because the device was marked as unmigratable. This series implements the necessary QEMU-side support to enable migration by:

* Removing the migration blocker and adding proper VMState infrastructure
* Implementing the VFIO migration callbacks that coordinate with the kernel vfio_ap driver to save and restore device state

The actual AP device state (queue configurations, crypto adapter assignments, etc.) is managed entirely by the kernel vfio_ap driver and transferred via the VFIO migration protocol. QEMU's role is to coordinate the migration process and ensure proper sequencing of state transfers.

This implementation follows the standard VFIO migration framework used by other VFIO device types (GPUs, network adapters, etc.) and integrates cleanly with QEMU's existing migration infrastructure.

Dependencies:
------------
* Requires corresponding kernel patches that implement the VFIO migration protocol in the vfio_ap driver
- see kernel patch series at https://lkml.org/lkml/2026/4/7/1991
* Kernel must support VFIO_DEVICE_FEATURE_MIGRATION for vfio-ap devices

Anthony Krowiak (1):
hw/vfio/ap: Callbacks for migration of guests with pass-through access
to AP devices

Anthony Krowiak (1):
hw/vfio/ap: Extend vfio device state description to facilitate live
migration

 hw/vfio/ap.c | 60 ++++++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 49 insertions(+), 11 deletions(-)
Signature · 1 line
2.52.0
#1 lore raw

PATCH v2 1/2 hw/vfio/ap: Extend vfio device state description to facilitate live migration

Make the changes needed to the VFIOAPDevice state description to facilitate live guest migration.

---
 hw/vfio/ap.c | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)
hw/vfio/ap.c +11 -8
index 5c8f305653..fd68983702 100644
@@ -41,6 +41,16 @@ struct VFIOAPDevice {
     EventNotifier req_notifier;
     EventNotifier cfg_notifier;
 };
+OBJECT_DECLARE_SIMPLE_TYPE(VFIOAPDevice, VFIO_AP_DEVICE)
+
+static const VMStateDescription vmstate_ap_device = {
+    .name = "vfio-ap-device",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (const VMStateField[]) {
+        VMSTATE_END_OF_LIST()
+    },
+};
 
 typedef struct APConfigChgEvent {
     QTAILQ_ENTRY(APConfigChgEvent) next;
@@ -56,8 +66,6 @@ static void __attribute__((constructor)) vfio_ap_global_init(void)
     qemu_mutex_init(&cfg_chg_events_lock);
 }
 
-OBJECT_DECLARE_SIMPLE_TYPE(VFIOAPDevice, VFIO_AP_DEVICE)
-
 static void vfio_ap_compute_needs_reset(VFIODevice *vdev)
 {
     vdev->needs_reset = false;
@@ -297,11 +305,6 @@ static void vfio_ap_reset(DeviceState *dev)
     }
 }
 
-static const VMStateDescription vfio_ap_vmstate = {
-    .name = "vfio-ap",
-    .unmigratable = 1,
-};
-
 static void vfio_ap_instance_init(Object *obj)
 {
     VFIOAPDevice *vapdev = VFIO_AP_DEVICE(obj);
@@ -335,7 +338,7 @@ static void vfio_ap_class_init(ObjectClass *klass, const void *data)
 #ifdef CONFIG_IOMMUFD
     object_class_property_add_str(klass, "fd", NULL, vfio_ap_set_fd);
 #endif
-    dc->vmsd = &vfio_ap_vmstate;
+    dc->vmsd = &vmstate_ap_device;
     dc->desc = "VFIO-based AP device assignment";
     set_bit(DEVICE_CATEGORY_MISC, dc->categories);
     dc->realize = vfio_ap_realize;
Signature · 1 line
2.52.0
#2 lore raw

Re: [PATCH v2 1/2] hw/vfio/ap: Extend vfio device state description to facilitate live migration

Quoted 52 lines · Anthony Krowiak

Make the changes needed to the VFIOAPDevice state description to facilitate live guest migration.

---
  hw/vfio/ap.c | 19 +++++++++++--------
  1 file changed, 11 insertions(+), 8 deletions(-)
hw/vfio/ap.c +10 -7
index 5c8f305653..fd68983702 100644
@@ -41,6 +41,16 @@ struct VFIOAPDevice {
      EventNotifier req_notifier;
      EventNotifier cfg_notifier;
  };
+OBJECT_DECLARE_SIMPLE_TYPE(VFIOAPDevice, VFIO_AP_DEVICE)
+
+static const VMStateDescription vmstate_ap_device = {
+    .name = "vfio-ap-device",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (const VMStateField[]) {
+        VMSTATE_END_OF_LIST()
+    },
+};
 
  typedef struct APConfigChgEvent {
      QTAILQ_ENTRY(APConfigChgEvent) next;
@@ -56,8 +66,6 @@ static void __attribute__((constructor)) vfio_ap_global_init(void)
      qemu_mutex_init(&cfg_chg_events_lock);
  }
 
-OBJECT_DECLARE_SIMPLE_TYPE(VFIOAPDevice, VFIO_AP_DEVICE)
-
  static void vfio_ap_compute_needs_reset(VFIODevice *vdev)
  {
      vdev->needs_reset = false;
@@ -297,11 +305,6 @@ static void vfio_ap_reset(DeviceState *dev)
      }
  }
 
-static const VMStateDescription vfio_ap_vmstate = {
-    .name = "vfio-ap",
-    .unmigratable = 1,
-};
-
  static void vfio_ap_instance_init(Object *obj)
  {
      VFIOAPDevice *vapdev = VFIO_AP_DEVICE(obj);

hw/vfio/ap.c static void vfio_ap_class_init(ObjectClass *klass, const void *data) · line 340

Quoted 3 lines
hw/vfio/ap.c +0 -0
@@ -335,7 +338,7 @@ static void vfio_ap_class_init(ObjectClass *klass, const void *data)
  #ifdef CONFIG_IOMMUFD
      object_class_property_add_str(klass, "fd", NULL, vfio_ap_set_fd);
  #endif

You will need to rebase on 'vfio-next' or 'master' when the changes pending in the vfio queue are merged.

Quoted 5 lines
hw/vfio/ap.c +1 -1
-    dc->vmsd = &vfio_ap_vmstate;
+    dc->vmsd = &vmstate_ap_device;
      dc->desc = "VFIO-based AP device assignment";
      set_bit(DEVICE_CATEGORY_MISC, dc->categories);
      dc->realize = vfio_ap_realize;
#3 lore raw

Re: [PATCH v2 1/2] hw/vfio/ap: Extend vfio device state description to facilitate live migration

Quoted 50 lines · Cédric Le Goater
Quoted 49 lines · Anthony Krowiak

Make the changes needed to the VFIOAPDevice state description to facilitate live guest migration.

---
  hw/vfio/ap.c | 19 +++++++++++--------

  1 file changed, 11 insertions(+), 8 deletions(-)
hw/vfio/ap.c +10 -1
index 5c8f305653..fd68983702 100644
@@ -41,6 +41,16 @@ struct VFIOAPDevice {
       EventNotifier req_notifier;
       EventNotifier cfg_notifier;
   };
+OBJECT_DECLARE_SIMPLE_TYPE(VFIOAPDevice, VFIO_AP_DEVICE)
+
+static const VMStateDescription vmstate_ap_device = {
+    .name = "vfio-ap-device",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (const VMStateField[]) {
+        VMSTATE_END_OF_LIST()
+    },
+};
     typedef struct APConfigChgEvent {
       QTAILQ_ENTRY(APConfigChgEvent) next;
@@ -56,8 +66,6 @@ static void __attribute__((constructor)) 
 vfio_ap_global_init(void)
       qemu_mutex_init(&cfg_chg_events_lock);
   }
   -OBJECT_DECLARE_SIMPLE_TYPE(VFIOAPDevice, VFIO_AP_DEVICE)
-
   static void vfio_ap_compute_needs_reset(VFIODevice *vdev)
   {

      vdev->needs_reset = false;

hw/vfio/ap.c +0 -4
@@ -297,11 +305,6 @@ static void vfio_ap_reset(DeviceState *dev)
       }
   }
   -static const VMStateDescription vfio_ap_vmstate = {
-    .name = "vfio-ap",
-    .unmigratable = 1,
-};
-
   static void vfio_ap_instance_init(Object *obj)
   {
       VFIOAPDevice *vapdev = VFIO_AP_DEVICE(obj);
Quoted 8 lines

hw/vfio/ap.c static void vfio_ap_class_init(ObjectClass · line 341

Quoted 5 lines
hw/vfio/ap.c +0 -0
@@ -335,7 +338,7 @@ static void vfio_ap_class_init(ObjectClass 
 *klass, const void *data)
   #ifdef CONFIG_IOMMUFD
       object_class_property_add_str(klass, "fd", NULL, vfio_ap_set_fd);
   #endif

You will need to rebase on 'vfio-next' or 'master' when the changes pending in the vfio queue are merged.

Will do. Thanks for the review.

Quoted 8 lines
Quoted 7 lines
hw/vfio/ap.c +1 -1
-    dc->vmsd = &vfio_ap_vmstate;
+    dc->vmsd = &vmstate_ap_device;
       dc->desc = "VFIO-based AP device assignment";
       set_bit(DEVICE_CATEGORY_MISC, dc->categories);

      dc->realize = vfio_ap_realize;

#4 lore raw

PATCH v2 2/2 hw/vfio/ap: Callbacks for migration of guests with pass-through access to AP devices

In reply to Anthony Krowiak

From: Anthony Krowiak <[email protected]>

Implements the callbacks used by the VFIO migration framework to migrate guests with pass-through access to s390 crypto devices.

---
 hw/vfio/ap.c | 43 +++++++++++++++++++++++++++++++++++++++----
 1 file changed, 39 insertions(+), 4 deletions(-)
hw/vfio/ap.c +39 -4
index fd68983702..49947b18c3 100644
@@ -32,6 +32,7 @@
 #include "hw/s390x/ap-bridge.h"
 #include "system/address-spaces.h"
 #include "qom/object.h"
+#include "vfio-migration-internal.h"
 
 #define TYPE_VFIO_AP_DEVICE      "vfio-ap"
 
@@ -47,6 +48,7 @@ static const VMStateDescription vmstate_ap_device = {
     .name = "vfio-ap-device",
     .version_id = 1,
     .minimum_version_id = 1,
+    .unmigratable = 0,
     .fields = (const VMStateField[]) {
         VMSTATE_END_OF_LIST()
     },
@@ -66,17 +68,44 @@ static void __attribute__((constructor)) vfio_ap_global_init(void)
     qemu_mutex_init(&cfg_chg_events_lock);
 }
 
+static int vfio_ap_save_config(VFIODevice *vdev, QEMUFile *f, Error **errp)
+{
+    VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
+
+    return vmstate_save_state(f, &vmstate_ap_device, vapdev, NULL, errp);
+}
+
+static int vfio_ap_load_config(VFIODevice *vdev, QEMUFile *f)
+{
+    VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
+    Error *err = NULL;
+    int ret;
+
+    ret = vmstate_load_state(f, &vmstate_ap_device, vapdev, 1, &err);
+    if (ret) {
+        error_report_err(err);
+    }
+
+    return ret;
+}
+
+static Object *vfio_ap_get_object(VFIODevice *vbasedev)
+{
+    VFIOAPDevice *vdev = container_of(vbasedev, VFIOAPDevice, vdev);
+
+    return OBJECT(vdev);
+}
+
 static void vfio_ap_compute_needs_reset(VFIODevice *vdev)
 {
     vdev->needs_reset = false;
 }
 
-/*
- * We don't need vfio_hot_reset_multi and vfio_eoi operations for
- * vfio-ap device now.
- */
 struct VFIODeviceOps vfio_ap_ops = {
     .vfio_compute_needs_reset = vfio_ap_compute_needs_reset,
+    .vfio_save_config = vfio_ap_save_config,
+    .vfio_load_config = vfio_ap_load_config,
+    .vfio_get_object = vfio_ap_get_object,
 };
 
 static void vfio_ap_req_notifier_handler(void *opaque)
@@ -251,6 +280,10 @@ static void vfio_ap_realize(DeviceState *dev, Error **errp)
         goto error;
     }
 
+    if (!vfio_migration_realize(vbasedev, errp)) {
+        goto error;
+    }
+
     if (!vfio_ap_register_irq_notifier(vapdev, VFIO_AP_REQ_IRQ_INDEX, &err)) {
         /*
          * Report this error, but do not make it a failing condition.
@@ -287,6 +320,8 @@ static void vfio_ap_unrealize(DeviceState *dev)
 
 static const Property vfio_ap_properties[] = {
     DEFINE_PROP_STRING("sysfsdev", VFIOAPDevice, vdev.sysfsdev),
+    DEFINE_PROP_ON_OFF_AUTO("enable-migration", VFIOAPDevice,
+                            vdev.enable_migration, ON_OFF_AUTO_ON),
 #ifdef CONFIG_IOMMUFD
     DEFINE_PROP_LINK("iommufd", VFIOAPDevice, vdev.iommufd,
                      TYPE_IOMMUFD_BACKEND, IOMMUFDBackend *),
Signature · 1 line
2.52.0
#5 lore raw

Re: [PATCH v2 2/2] hw/vfio/ap: Callbacks for migration of guests with pass-through access to AP devices

Quoted 23 lines · Anthony Krowiak

From: Anthony Krowiak <[email protected]>

Implements the callbacks used by the VFIO migration framework to migrate guests with pass-through access to s390 crypto devices.

---
  hw/vfio/ap.c | 43 +++++++++++++++++++++++++++++++++++++++----
  1 file changed, 39 insertions(+), 4 deletions(-)
hw/vfio/ap.c +1 -0
index fd68983702..49947b18c3 100644
@@ -32,6 +32,7 @@
  #include "hw/s390x/ap-bridge.h"
  #include "system/address-spaces.h"
  #include "qom/object.h"
+#include "vfio-migration-internal.h"
 
  #define TYPE_VFIO_AP_DEVICE      "vfio-ap"
 

hw/vfio/ap.c static const VMStateDescription vmstate_ap_device = { · line 51

Quoted 4 lines
hw/vfio/ap.c +1 -0
@@ -47,6 +48,7 @@ static const VMStateDescription vmstate_ap_device = {
      .name = "vfio-ap-device",
      .version_id = 1,
      .minimum_version_id = 1,
+    .unmigratable = 0,

unneeded.

Quoted 64 lines
hw/vfio/ap.c +35 -4
      .fields = (const VMStateField[]) {
          VMSTATE_END_OF_LIST()
      },
@@ -66,17 +68,44 @@ static void __attribute__((constructor)) vfio_ap_global_init(void)
      qemu_mutex_init(&cfg_chg_events_lock);
  }
 
+static int vfio_ap_save_config(VFIODevice *vdev, QEMUFile *f, Error **errp)
+{
+    VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
+
+    return vmstate_save_state(f, &vmstate_ap_device, vapdev, NULL, errp);
+}
+
+static int vfio_ap_load_config(VFIODevice *vdev, QEMUFile *f)
+{
+    VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
+    Error *err = NULL;
+    int ret;
+
+    ret = vmstate_load_state(f, &vmstate_ap_device, vapdev, 1, &err);
+    if (ret) {
+        error_report_err(err);
+    }
+
+    return ret;
+}
+
+static Object *vfio_ap_get_object(VFIODevice *vbasedev)
+{
+    VFIOAPDevice *vdev = container_of(vbasedev, VFIOAPDevice, vdev);
+
+    return OBJECT(vdev);
+}
+
  static void vfio_ap_compute_needs_reset(VFIODevice *vdev)
  {
      vdev->needs_reset = false;
  }
 
-/*
- * We don't need vfio_hot_reset_multi and vfio_eoi operations for
- * vfio-ap device now.
- */
  struct VFIODeviceOps vfio_ap_ops = {
      .vfio_compute_needs_reset = vfio_ap_compute_needs_reset,
+    .vfio_save_config = vfio_ap_save_config,
+    .vfio_load_config = vfio_ap_load_config,
+    .vfio_get_object = vfio_ap_get_object,
  };
 
  static void vfio_ap_req_notifier_handler(void *opaque)
@@ -251,6 +280,10 @@ static void vfio_ap_realize(DeviceState *dev, Error **errp)
          goto error;
      }
 
+    if (!vfio_migration_realize(vbasedev, errp)) {
+        goto error;
+    }
+
      if (!vfio_ap_register_irq_notifier(vapdev, VFIO_AP_REQ_IRQ_INDEX, &err)) {
          /*
           * Report this error, but do not make it a failing condition.

hw/vfio/ap.c static void vfio_ap_unrealize(DeviceState *dev) · line 324

Quoted 5 lines
hw/vfio/ap.c +2 -0
@@ -287,6 +320,8 @@ static void vfio_ap_unrealize(DeviceState *dev)
 
  static const Property vfio_ap_properties[] = {
      DEFINE_PROP_STRING("sysfsdev", VFIOAPDevice, vdev.sysfsdev),
+    DEFINE_PROP_ON_OFF_AUTO("enable-migration", VFIOAPDevice,
+                            vdev.enable_migration, ON_OFF_AUTO_ON),

Why not ON_OFF_AUTO_AUTO ? This would enable migration only if kernel has migration support. If not, the VM would still start but migration would be blocked.

thanks,

C.

Quoted 3 lines
hw/vfio/ap.c +0 -0
  #ifdef CONFIG_IOMMUFD
      DEFINE_PROP_LINK("iommufd", VFIOAPDevice, vdev.iommufd,
                       TYPE_IOMMUFD_BACKEND, IOMMUFDBackend *),
#6 lore raw

Re: [PATCH v2 2/2] hw/vfio/ap: Callbacks for migration of guests with pass-through access to AP devices

Quoted 20 lines · Cédric Le Goater
Quoted 19 lines · Anthony Krowiak

From: Anthony  Krowiak <[email protected]>

Implements the callbacks used by the VFIO migration framework to migrate guests with pass-through access to s390 crypto devices.

---
  hw/vfio/ap.c | 43 +++++++++++++++++++++++++++++++++++++++----

  1 file changed, 39 insertions(+), 4 deletions(-)
hw/vfio/ap.c +1 -0
index fd68983702..49947b18c3 100644
@@ -32,6 +32,7 @@
   #include "hw/s390x/ap-bridge.h"
   #include "system/address-spaces.h"
   #include "qom/object.h"
+#include "vfio-migration-internal.h"
Quoted 8 lines
Quoted 6 lines
hw/vfio/ap.c +0 -0
     #define TYPE_VFIO_AP_DEVICE      "vfio-ap"
   @@ -47,6 +48,7 @@ static const VMStateDescription vmstate_ap_device = {
       .name = "vfio-ap-device",

      .version_id = 1,
      .minimum_version_id = 1,
+    .unmigratable = 0,

unneeded.

I'll remove it

Quoted 64 lines
Quoted 64 lines

      .fields = (const VMStateField[]) {
          VMSTATE_END_OF_LIST()
      },

hw/vfio/ap.c +27 -0
@@ -66,17 +68,44 @@ static void __attribute__((constructor)) 
 vfio_ap_global_init(void)
       qemu_mutex_init(&cfg_chg_events_lock);
   }
   +static int vfio_ap_save_config(VFIODevice *vdev, QEMUFile *f, Error **errp)
+{
+    VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
+
+    return vmstate_save_state(f, &vmstate_ap_device, vapdev, NULL, 
 errp);
+}
+
+static int vfio_ap_load_config(VFIODevice *vdev, QEMUFile *f)
+{
+    VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
+    Error *err = NULL;
+    int ret;
+
+    ret = vmstate_load_state(f, &vmstate_ap_device, vapdev, 1, &err);
+    if (ret) {
+        error_report_err(err);
+    }
+
+    return ret;
+}
+
+static Object *vfio_ap_get_object(VFIODevice *vbasedev)
+{
+    VFIOAPDevice *vdev = container_of(vbasedev, VFIOAPDevice, vdev);
+
+    return OBJECT(vdev);
+}
+
   static void vfio_ap_compute_needs_reset(VFIODevice *vdev)
   {
       vdev->needs_reset = false;
   }

  -/*
- * We don't need vfio_hot_reset_multi and vfio_eoi operations for
- * vfio-ap device now.
- */
  struct VFIODeviceOps vfio_ap_ops = { .vfio_compute_needs_reset = vfio_ap_compute_needs_reset, +    .vfio_save_config = vfio_ap_save_config,
+    .vfio_load_config = vfio_ap_load_config,
+    .vfio_get_object = vfio_ap_get_object,
  };
    static void vfio_ap_req_notifier_handler(void *opaque)

hw/vfio/ap.c +3 -0
@@ -251,6 +280,10 @@ static void vfio_ap_realize(DeviceState *dev, 
 Error **errp)
           goto error;
       }
   +    if (!vfio_migration_realize(vbasedev, errp)) {
+        goto error;
+    }
+
       if (!vfio_ap_register_irq_notifier(vapdev, VFIO_AP_REQ_IRQ_INDEX, &err)) {
           /*

           * Report this error, but do not make it a failing condition.

hw/vfio/ap.c +0 -0
@@ -287,6 +320,8 @@ static void vfio_ap_unrealize(DeviceState *dev)
Quoted 8 lines

hw/vfio/ap.c static void vfio_ap_unrealize(DeviceState *dev) · line 324

Quoted 4 lines
hw/vfio/ap.c +2 -0
     static const Property vfio_ap_properties[] = {
       DEFINE_PROP_STRING("sysfsdev", VFIOAPDevice, vdev.sysfsdev),
+    DEFINE_PROP_ON_OFF_AUTO("enable-migration", VFIOAPDevice,
+                            vdev.enable_migration, ON_OFF_AUTO_ON),

Why not ON_OFF_AUTO_AUTO ? This would enable migration only if kernel has migration support. If not, the VM would still start but migration would be blocked.

I will make that change but I'm wondering; how does QEMU know whether the kernel has support for migration of vfio-ap devices or not?

Quoted 11 lines

thanks,

C.

Quoted 3 lines
hw/vfio/ap.c +0 -0
   #ifdef CONFIG_IOMMUFD
       DEFINE_PROP_LINK("iommufd", VFIOAPDevice, vdev.iommufd,
                        TYPE_IOMMUFD_BACKEND, IOMMUFDBackend *),
#7 lore raw

Re: [PATCH v2 2/2] hw/vfio/ap: Callbacks for migration of guests with pass-through access to AP devices

Quoted 104 lines · Anthony Krowiak
Quoted 20 lines · Cédric Le Goater
Quoted 19 lines · Anthony Krowiak

From: Anthony  Krowiak <[email protected]>

Implements the callbacks used by the VFIO migration framework to migrate guests with pass-through access to s390 crypto devices.

---
  hw/vfio/ap.c | 43 +++++++++++++++++++++++++++++++++++++++----

  1 file changed, 39 insertions(+), 4 deletions(-)
hw/vfio/ap.c +1 -0
index fd68983702..49947b18c3 100644
@@ -32,6 +32,7 @@
   #include "hw/s390x/ap-bridge.h"
   #include "system/address-spaces.h"
   #include "qom/object.h"
+#include "vfio-migration-internal.h"
Quoted 8 lines
Quoted 6 lines
hw/vfio/ap.c +0 -0
     #define TYPE_VFIO_AP_DEVICE      "vfio-ap"
   @@ -47,6 +48,7 @@ static const VMStateDescription vmstate_ap_device = {
       .name = "vfio-ap-device",

      .version_id = 1,
      .minimum_version_id = 1,
+    .unmigratable = 0,

unneeded.

I'll remove it

Quoted 61 lines
Quoted 61 lines

      .fields = (const VMStateField[]) {
          VMSTATE_END_OF_LIST()
      },

hw/vfio/ap.c +27 -0
@@ -66,17 +68,44 @@ static void __attribute__((constructor)) vfio_ap_global_init(void)
       qemu_mutex_init(&cfg_chg_events_lock);
   }
   +static int vfio_ap_save_config(VFIODevice *vdev, QEMUFile *f, Error **errp)
+{
+    VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
+
+    return vmstate_save_state(f, &vmstate_ap_device, vapdev, NULL, errp);
+}
+
+static int vfio_ap_load_config(VFIODevice *vdev, QEMUFile *f)
+{
+    VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
+    Error *err = NULL;
+    int ret;
+
+    ret = vmstate_load_state(f, &vmstate_ap_device, vapdev, 1, &err);
+    if (ret) {
+        error_report_err(err);
+    }
+
+    return ret;
+}
+
+static Object *vfio_ap_get_object(VFIODevice *vbasedev)
+{
+    VFIOAPDevice *vdev = container_of(vbasedev, VFIOAPDevice, vdev);
+
+    return OBJECT(vdev);
+}
+
   static void vfio_ap_compute_needs_reset(VFIODevice *vdev)
   {
       vdev->needs_reset = false;
   }

  -/*
- * We don't need vfio_hot_reset_multi and vfio_eoi operations for
- * vfio-ap device now.
- */
  struct VFIODeviceOps vfio_ap_ops = { .vfio_compute_needs_reset = vfio_ap_compute_needs_reset, +    .vfio_save_config = vfio_ap_save_config,
+    .vfio_load_config = vfio_ap_load_config,
+    .vfio_get_object = vfio_ap_get_object,
  };
    static void vfio_ap_req_notifier_handler(void *opaque)

hw/vfio/ap.c +3 -0
@@ -251,6 +280,10 @@ static void vfio_ap_realize(DeviceState *dev, Error **errp)
           goto error;
       }
   +    if (!vfio_migration_realize(vbasedev, errp)) {
+        goto error;
+    }
+
       if (!vfio_ap_register_irq_notifier(vapdev, VFIO_AP_REQ_IRQ_INDEX, &err)) {
           /*
            * Report this error, but do not make it a failing condition.
@@ -287,6 +320,8 @@ static void vfio_ap_unrealize(DeviceState *dev)
Quoted 8 lines

hw/vfio/ap.c static void vfio_ap_unrealize(DeviceState *dev) · line 324

Quoted 4 lines
hw/vfio/ap.c +2 -0
     static const Property vfio_ap_properties[] = {
       DEFINE_PROP_STRING("sysfsdev", VFIOAPDevice, vdev.sysfsdev),
+    DEFINE_PROP_ON_OFF_AUTO("enable-migration", VFIOAPDevice,
+                            vdev.enable_migration, ON_OFF_AUTO_ON),

Why not ON_OFF_AUTO_AUTO ? This would enable migration only if kernel has migration support. If not, the VM would still start but migration would be blocked.

Quoted 2 lines

I will make that change but I'm wondering; how does QEMU know whether the kernel has support for migration of vfio-ap devices or not?

vfio_migration_init() will probe the kernel for migration support and install a migration blocker in case of failure.

You should see a message like :

VFIO migration init failed: migration is not supported in kernel

Thanks,

C.

#8 lore raw

Re: [PATCH v2 2/2] hw/vfio/ap: Callbacks for migration of guests with pass-through access to AP devices

In reply to Cédric Le Goater
Quoted 92 lines · Cédric Le Goater
Quoted 22 lines · Anthony Krowiak

From: Anthony  Krowiak <[email protected]>

Implements the callbacks used by the VFIO migration framework to migrate guests with pass-through access to s390 crypto devices.

---
  hw/vfio/ap.c | 43 +++++++++++++++++++++++++++++++++++++++----

  1 file changed, 39 insertions(+), 4 deletions(-)
hw/vfio/ap.c +1 -0
index fd68983702..49947b18c3 100644
@@ -32,6 +32,7 @@
   #include "hw/s390x/ap-bridge.h"
   #include "system/address-spaces.h"
   #include "qom/object.h"
+#include "vfio-migration-internal.h"
     #define TYPE_VFIO_AP_DEVICE      "vfio-ap"
   @@ -47,6 +48,7 @@ static const VMStateDescription vmstate_ap_device = {
       .name = "vfio-ap-device",
Quoted 3 lines

      .version_id = 1,
      .minimum_version_id = 1,
+    .unmigratable = 0,

unneeded.

Quoted 63 lines

      .fields = (const VMStateField[]) {
          VMSTATE_END_OF_LIST()
      },

hw/vfio/ap.c +27 -0
@@ -66,17 +68,44 @@ static void __attribute__((constructor)) 
 vfio_ap_global_init(void)
       qemu_mutex_init(&cfg_chg_events_lock);
   }
   +static int vfio_ap_save_config(VFIODevice *vdev, QEMUFile *f, Error **errp)
+{
+    VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
+
+    return vmstate_save_state(f, &vmstate_ap_device, vapdev, NULL, 
 errp);
+}
+
+static int vfio_ap_load_config(VFIODevice *vdev, QEMUFile *f)
+{
+    VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
+    Error *err = NULL;
+    int ret;
+
+    ret = vmstate_load_state(f, &vmstate_ap_device, vapdev, 1, &err);
+    if (ret) {
+        error_report_err(err);
+    }
+
+    return ret;
+}
+
+static Object *vfio_ap_get_object(VFIODevice *vbasedev)
+{
+    VFIOAPDevice *vdev = container_of(vbasedev, VFIOAPDevice, vdev);
+
+    return OBJECT(vdev);
+}
+
   static void vfio_ap_compute_needs_reset(VFIODevice *vdev)
   {
       vdev->needs_reset = false;
   }

  -/*
- * We don't need vfio_hot_reset_multi and vfio_eoi operations for
- * vfio-ap device now.
- */
  struct VFIODeviceOps vfio_ap_ops = { .vfio_compute_needs_reset = vfio_ap_compute_needs_reset, +    .vfio_save_config = vfio_ap_save_config,
+    .vfio_load_config = vfio_ap_load_config,
+    .vfio_get_object = vfio_ap_get_object,
  };
    static void vfio_ap_req_notifier_handler(void *opaque)

hw/vfio/ap.c +3 -0
@@ -251,6 +280,10 @@ static void vfio_ap_realize(DeviceState *dev, 
 Error **errp)
           goto error;
       }
   +    if (!vfio_migration_realize(vbasedev, errp)) {
+        goto error;
+    }
+
       if (!vfio_ap_register_irq_notifier(vapdev, VFIO_AP_REQ_IRQ_INDEX, &err)) {
           /*

           * Report this error, but do not make it a failing condition.

hw/vfio/ap.c +0 -0
@@ -287,6 +320,8 @@ static void vfio_ap_unrealize(DeviceState *dev)
Quoted 8 lines

hw/vfio/ap.c static void vfio_ap_unrealize(DeviceState *dev) · line 324

Quoted 4 lines
hw/vfio/ap.c +2 -0
     static const Property vfio_ap_properties[] = {
       DEFINE_PROP_STRING("sysfsdev", VFIOAPDevice, vdev.sysfsdev),
+    DEFINE_PROP_ON_OFF_AUTO("enable-migration", VFIOAPDevice,
+                            vdev.enable_migration, ON_OFF_AUTO_ON),

Why not ON_OFF_AUTO_AUTO ? This would enable migration only if kernel has migration support. If not, the VM would still start but migration would be blocked.

Will this allow the vfio-ap device migration patches to get merged before the kernel/KVM patches?

Quoted 10 lines

thanks,

C.

Quoted 3 lines
hw/vfio/ap.c +0 -0
   #ifdef CONFIG_IOMMUFD
       DEFINE_PROP_LINK("iommufd", VFIOAPDevice, vdev.iommufd,
                        TYPE_IOMMUFD_BACKEND, IOMMUFDBackend *),
#9 lore raw

Re: [PATCH v2 2/2] hw/vfio/ap: Callbacks for migration of guests with pass-through access to AP devices

Quoted 101 lines · Anthony Krowiak
Quoted 89 lines · Cédric Le Goater
Quoted 22 lines · Anthony Krowiak

From: Anthony  Krowiak <[email protected]>

Implements the callbacks used by the VFIO migration framework to migrate guests with pass-through access to s390 crypto devices.

---
  hw/vfio/ap.c | 43 +++++++++++++++++++++++++++++++++++++++----

  1 file changed, 39 insertions(+), 4 deletions(-)
hw/vfio/ap.c +1 -0
index fd68983702..49947b18c3 100644
@@ -32,6 +32,7 @@
   #include "hw/s390x/ap-bridge.h"
   #include "system/address-spaces.h"
   #include "qom/object.h"
+#include "vfio-migration-internal.h"
     #define TYPE_VFIO_AP_DEVICE      "vfio-ap"
   @@ -47,6 +48,7 @@ static const VMStateDescription vmstate_ap_device = {
       .name = "vfio-ap-device",
Quoted 3 lines

      .version_id = 1,
      .minimum_version_id = 1,
+    .unmigratable = 0,

unneeded.

Quoted 60 lines

      .fields = (const VMStateField[]) {
          VMSTATE_END_OF_LIST()
      },

hw/vfio/ap.c +27 -0
@@ -66,17 +68,44 @@ static void __attribute__((constructor)) vfio_ap_global_init(void)
       qemu_mutex_init(&cfg_chg_events_lock);
   }
   +static int vfio_ap_save_config(VFIODevice *vdev, QEMUFile *f, Error **errp)
+{
+    VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
+
+    return vmstate_save_state(f, &vmstate_ap_device, vapdev, NULL, errp);
+}
+
+static int vfio_ap_load_config(VFIODevice *vdev, QEMUFile *f)
+{
+    VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
+    Error *err = NULL;
+    int ret;
+
+    ret = vmstate_load_state(f, &vmstate_ap_device, vapdev, 1, &err);
+    if (ret) {
+        error_report_err(err);
+    }
+
+    return ret;
+}
+
+static Object *vfio_ap_get_object(VFIODevice *vbasedev)
+{
+    VFIOAPDevice *vdev = container_of(vbasedev, VFIOAPDevice, vdev);
+
+    return OBJECT(vdev);
+}
+
   static void vfio_ap_compute_needs_reset(VFIODevice *vdev)
   {
       vdev->needs_reset = false;
   }

  -/*
- * We don't need vfio_hot_reset_multi and vfio_eoi operations for
- * vfio-ap device now.
- */
  struct VFIODeviceOps vfio_ap_ops = { .vfio_compute_needs_reset = vfio_ap_compute_needs_reset, +    .vfio_save_config = vfio_ap_save_config,
+    .vfio_load_config = vfio_ap_load_config,
+    .vfio_get_object = vfio_ap_get_object,
  };
    static void vfio_ap_req_notifier_handler(void *opaque)

hw/vfio/ap.c +3 -0
@@ -251,6 +280,10 @@ static void vfio_ap_realize(DeviceState *dev, Error **errp)
           goto error;
       }
   +    if (!vfio_migration_realize(vbasedev, errp)) {
+        goto error;
+    }
+
       if (!vfio_ap_register_irq_notifier(vapdev, VFIO_AP_REQ_IRQ_INDEX, &err)) {
           /*
            * Report this error, but do not make it a failing condition.
@@ -287,6 +320,8 @@ static void vfio_ap_unrealize(DeviceState *dev)
Quoted 8 lines

hw/vfio/ap.c static void vfio_ap_unrealize(DeviceState *dev) · line 324

Quoted 4 lines
hw/vfio/ap.c +2 -0
     static const Property vfio_ap_properties[] = {
       DEFINE_PROP_STRING("sysfsdev", VFIOAPDevice, vdev.sysfsdev),
+    DEFINE_PROP_ON_OFF_AUTO("enable-migration", VFIOAPDevice,
+                            vdev.enable_migration, ON_OFF_AUTO_ON),

Why not ON_OFF_AUTO_AUTO ? This would enable migration only if kernel has migration support. If not, the VM would still start but migration would be blocked.

Quoted 2 lines

Will this allow the vfio-ap device migration patches to get merged before the kernel/KVM patches?

It's possible. Several reasons could explain the lack of migration support. The most obvious is a lack of kernel support, but it could also be a firmware support issue, an incompatible hardware state of the adapter, adapter model, etc.

Do all CEXxS crypto cards support migration ?

Do you expect kernel support [*] to be merged in v7.1 ?

Also, I think the QEMU support lacks vfio_migration_exit().

Thanks,

C.

[*] https://lore.kernel.org/r/[email protected]

#10 lore raw

Re: [PATCH v2 2/2] hw/vfio/ap: Callbacks for migration of guests with pass-through access to AP devices

In reply to Cédric Le Goater
Quoted 92 lines · Cédric Le Goater
Quoted 22 lines · Anthony Krowiak

From: Anthony  Krowiak <[email protected]>

Implements the callbacks used by the VFIO migration framework to migrate guests with pass-through access to s390 crypto devices.

---
  hw/vfio/ap.c | 43 +++++++++++++++++++++++++++++++++++++++----

  1 file changed, 39 insertions(+), 4 deletions(-)
hw/vfio/ap.c +1 -0
index fd68983702..49947b18c3 100644
@@ -32,6 +32,7 @@
   #include "hw/s390x/ap-bridge.h"
   #include "system/address-spaces.h"
   #include "qom/object.h"
+#include "vfio-migration-internal.h"
     #define TYPE_VFIO_AP_DEVICE      "vfio-ap"
   @@ -47,6 +48,7 @@ static const VMStateDescription vmstate_ap_device = {
       .name = "vfio-ap-device",
Quoted 3 lines

      .version_id = 1,
      .minimum_version_id = 1,
+    .unmigratable = 0,

unneeded.

Quoted 63 lines

      .fields = (const VMStateField[]) {
          VMSTATE_END_OF_LIST()
      },

hw/vfio/ap.c +27 -0
@@ -66,17 +68,44 @@ static void __attribute__((constructor)) 
 vfio_ap_global_init(void)
       qemu_mutex_init(&cfg_chg_events_lock);
   }
   +static int vfio_ap_save_config(VFIODevice *vdev, QEMUFile *f, Error **errp)
+{
+    VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
+
+    return vmstate_save_state(f, &vmstate_ap_device, vapdev, NULL, 
 errp);
+}
+
+static int vfio_ap_load_config(VFIODevice *vdev, QEMUFile *f)
+{
+    VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
+    Error *err = NULL;
+    int ret;
+
+    ret = vmstate_load_state(f, &vmstate_ap_device, vapdev, 1, &err);
+    if (ret) {
+        error_report_err(err);
+    }
+
+    return ret;
+}
+
+static Object *vfio_ap_get_object(VFIODevice *vbasedev)
+{
+    VFIOAPDevice *vdev = container_of(vbasedev, VFIOAPDevice, vdev);
+
+    return OBJECT(vdev);
+}
+
   static void vfio_ap_compute_needs_reset(VFIODevice *vdev)
   {
       vdev->needs_reset = false;
   }

  -/*
- * We don't need vfio_hot_reset_multi and vfio_eoi operations for
- * vfio-ap device now.
- */
  struct VFIODeviceOps vfio_ap_ops = { .vfio_compute_needs_reset = vfio_ap_compute_needs_reset, +    .vfio_save_config = vfio_ap_save_config,
+    .vfio_load_config = vfio_ap_load_config,
+    .vfio_get_object = vfio_ap_get_object,
  };
    static void vfio_ap_req_notifier_handler(void *opaque)

hw/vfio/ap.c +3 -0
@@ -251,6 +280,10 @@ static void vfio_ap_realize(DeviceState *dev, 
 Error **errp)
           goto error;
       }
   +    if (!vfio_migration_realize(vbasedev, errp)) {
+        goto error;
+    }
+
       if (!vfio_ap_register_irq_notifier(vapdev, VFIO_AP_REQ_IRQ_INDEX, &err)) {
           /*

           * Report this error, but do not make it a failing condition.

hw/vfio/ap.c +0 -0
@@ -287,6 +320,8 @@ static void vfio_ap_unrealize(DeviceState *dev)
Quoted 8 lines

hw/vfio/ap.c static void vfio_ap_unrealize(DeviceState *dev) · line 324

Quoted 4 lines
hw/vfio/ap.c +2 -0
     static const Property vfio_ap_properties[] = {
       DEFINE_PROP_STRING("sysfsdev", VFIOAPDevice, vdev.sysfsdev),
+    DEFINE_PROP_ON_OFF_AUTO("enable-migration", VFIOAPDevice,
+                            vdev.enable_migration, ON_OFF_AUTO_ON),

Why not ON_OFF_AUTO_AUTO ? This would enable migration only if kernel has migration support. If not, the VM would still start but migration would be blocked.

I made this change and migration failed on the destination host with the following error message:

error: Requested operation is not valid: cannot migrate domain: 62177883-f1bb-47f0-914d-32a22e3a8804: VFIO device doesn't support device and IOMMU dirty tracking

This despite having the following property defined:

DEFINE_PROP_ON_OFF_AUTO("x-device-dirty-page-tracking", VFIOAPDevice, vdev.device_dirty_page_tracking, ON_OFF_AUTO_ON)

Quoted 10 lines

thanks,

C.

Quoted 3 lines
hw/vfio/ap.c +0 -0
   #ifdef CONFIG_IOMMUFD
       DEFINE_PROP_LINK("iommufd", VFIOAPDevice, vdev.iommufd,
                        TYPE_IOMMUFD_BACKEND, IOMMUFDBackend *),
#11 lore raw

Re: [PATCH v2 2/2] hw/vfio/ap: Callbacks for migration of guests with pass-through access to AP devices

Quoted 101 lines · Anthony Krowiak
Quoted 89 lines · Cédric Le Goater
Quoted 22 lines · Anthony Krowiak

From: Anthony  Krowiak <[email protected]>

Implements the callbacks used by the VFIO migration framework to migrate guests with pass-through access to s390 crypto devices.

---
  hw/vfio/ap.c | 43 +++++++++++++++++++++++++++++++++++++++----

  1 file changed, 39 insertions(+), 4 deletions(-)
hw/vfio/ap.c +1 -0
index fd68983702..49947b18c3 100644
@@ -32,6 +32,7 @@
   #include "hw/s390x/ap-bridge.h"
   #include "system/address-spaces.h"
   #include "qom/object.h"
+#include "vfio-migration-internal.h"
     #define TYPE_VFIO_AP_DEVICE      "vfio-ap"
   @@ -47,6 +48,7 @@ static const VMStateDescription vmstate_ap_device = {
       .name = "vfio-ap-device",
Quoted 3 lines

      .version_id = 1,
      .minimum_version_id = 1,
+    .unmigratable = 0,

unneeded.

Quoted 60 lines

      .fields = (const VMStateField[]) {
          VMSTATE_END_OF_LIST()
      },

hw/vfio/ap.c +27 -0
@@ -66,17 +68,44 @@ static void __attribute__((constructor)) vfio_ap_global_init(void)
       qemu_mutex_init(&cfg_chg_events_lock);
   }
   +static int vfio_ap_save_config(VFIODevice *vdev, QEMUFile *f, Error **errp)
+{
+    VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
+
+    return vmstate_save_state(f, &vmstate_ap_device, vapdev, NULL, errp);
+}
+
+static int vfio_ap_load_config(VFIODevice *vdev, QEMUFile *f)
+{
+    VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
+    Error *err = NULL;
+    int ret;
+
+    ret = vmstate_load_state(f, &vmstate_ap_device, vapdev, 1, &err);
+    if (ret) {
+        error_report_err(err);
+    }
+
+    return ret;
+}
+
+static Object *vfio_ap_get_object(VFIODevice *vbasedev)
+{
+    VFIOAPDevice *vdev = container_of(vbasedev, VFIOAPDevice, vdev);
+
+    return OBJECT(vdev);
+}
+
   static void vfio_ap_compute_needs_reset(VFIODevice *vdev)
   {
       vdev->needs_reset = false;
   }

  -/*
- * We don't need vfio_hot_reset_multi and vfio_eoi operations for
- * vfio-ap device now.
- */
  struct VFIODeviceOps vfio_ap_ops = { .vfio_compute_needs_reset = vfio_ap_compute_needs_reset, +    .vfio_save_config = vfio_ap_save_config,
+    .vfio_load_config = vfio_ap_load_config,
+    .vfio_get_object = vfio_ap_get_object,
  };
    static void vfio_ap_req_notifier_handler(void *opaque)

hw/vfio/ap.c +3 -0
@@ -251,6 +280,10 @@ static void vfio_ap_realize(DeviceState *dev, Error **errp)
           goto error;
       }
   +    if (!vfio_migration_realize(vbasedev, errp)) {
+        goto error;
+    }
+
       if (!vfio_ap_register_irq_notifier(vapdev, VFIO_AP_REQ_IRQ_INDEX, &err)) {
           /*
            * Report this error, but do not make it a failing condition.
@@ -287,6 +320,8 @@ static void vfio_ap_unrealize(DeviceState *dev)
Quoted 8 lines

hw/vfio/ap.c static void vfio_ap_unrealize(DeviceState *dev) · line 324

Quoted 4 lines
hw/vfio/ap.c +2 -0
     static const Property vfio_ap_properties[] = {
       DEFINE_PROP_STRING("sysfsdev", VFIOAPDevice, vdev.sysfsdev),
+    DEFINE_PROP_ON_OFF_AUTO("enable-migration", VFIOAPDevice,
+                            vdev.enable_migration, ON_OFF_AUTO_ON),

Why not ON_OFF_AUTO_AUTO ? This would enable migration only if kernel has migration support. If not, the VM would still start but migration would be blocked.

Quoted 8 lines

I made this change and migration failed on the destination host with the following error message:

error: Requested operation is not valid: cannot migrate domain: 62177883-f1bb-47f0-914d-32a22e3a8804: VFIO device doesn't support device and IOMMU dirty tracking

This despite having the following property defined:

DEFINE_PROP_ON_OFF_AUTO("x-device-dirty-page-tracking", VFIOAPDevice, vdev.device_dirty_page_tracking, ON_OFF_AUTO_ON)

try adding dummy vfio_log_ops handlers in the driver.

static int mtty_log_start(struct vfio_device *vdev,

			  struct rb_root_cached *ranges,
			  u32 nnodes, u64 *page_size)
{
	return 0;
}

static int mtty_log_stop(struct vfio_device *vdev)

{
	return 0;
}

static int mtty_log_read_and_clear(struct vfio_device *vdev,
unsigned long iova, unsigned long length, struct iova_bitmap *dirty)

{
	return 0;
}

static const struct vfio_log_ops mtty_log_ops = {
.log_start = mtty_log_start, .log_stop = mtty_log_stop, .log_read_and_clear = mtty_log_read_and_clear,
};

See samples/vfio-mdev/mtty.c.

C.