Enable live migration of vfio-ap devices
v2 cover Enable live migration of vfio-ap devices
This patch series enables live guest migration for s390 guests using AP (Adjunct Processor) crypto devices passed through via the VFIO framework.
The s390 AP architecture provides cryptographic coprocessor and accelerator functions to guests. The vfio_ap device driver allows these AP devices to be passed through to KVM guests, giving them access to the cryptographic capabilities of CEXxS crypto cards.
Previously, guests using vfio-ap devices could not be migrated because the device was marked as unmigratable. This series implements the necessary QEMU-side support to enable migration by:
* Removing the migration blocker and adding proper VMState infrastructure
* Implementing the VFIO migration callbacks that coordinate with the kernel vfio_ap driver to save and restore device state
The actual AP device state (queue configurations, crypto adapter assignments, etc.) is managed entirely by the kernel vfio_ap driver and transferred via the VFIO migration protocol. QEMU's role is to coordinate the migration process and ensure proper sequencing of state transfers.
This implementation follows the standard VFIO migration framework used by other VFIO device types (GPUs, network adapters, etc.) and integrates cleanly with QEMU's existing migration infrastructure.
Dependencies:
------------
* Requires corresponding kernel patches that implement the VFIO migration protocol in the vfio_ap driver
- see kernel patch series at https://lkml.org/lkml/2026/4/7/1991
* Kernel must support VFIO_DEVICE_FEATURE_MIGRATION for vfio-ap devices
Anthony Krowiak (1):
hw/vfio/ap: Callbacks for migration of guests with pass-through access
to AP devices
Anthony Krowiak (1):
hw/vfio/ap: Extend vfio device state description to facilitate live
migration
hw/vfio/ap.c | 60 ++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 49 insertions(+), 11 deletions(-)
Signature · 1 line
2.52.0
PATCH v2 1/2 hw/vfio/ap: Extend vfio device state description to facilitate live migration
Make the changes needed to the VFIOAPDevice state description to facilitate live guest migration.
--- hw/vfio/ap.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-)
hw/vfio/ap.c +11 -8
@@ -41,6 +41,16 @@ struct VFIOAPDevice {EventNotifier req_notifier;EventNotifier cfg_notifier; }; +OBJECT_DECLARE_SIMPLE_TYPE(VFIOAPDevice, VFIO_AP_DEVICE) + +static const VMStateDescription vmstate_ap_device = { + .name ="vfio-ap-device" , + .version_id =1 , + .minimum_version_id =1 , + .fields = (const VMStateField []) { +VMSTATE_END_OF_LIST () + }, +}; typedef struct APConfigChgEvent { QTAILQ_ENTRY(APConfigChgEvent) next;
@@ -56,8 +66,6 @@ static void __attribute__((constructor)) vfio_ap_global_init(void)qemu_mutex_init (&cfg_chg_events_lock); } -OBJECT_DECLARE_SIMPLE_TYPE(VFIOAPDevice, VFIO_AP_DEVICE) -static void vfio_ap_compute_needs_reset (VFIODevice *vdev) { vdev->needs_reset = false;
@@ -297,11 +305,6 @@ static void vfio_ap_reset(DeviceState *dev) } } -static const VMStateDescription vfio_ap_vmstate = { - .name ="vfio-ap" , - .unmigratable =1 , -}; -static void vfio_ap_instance_init (Object *obj) {VFIOAPDevice *vapdev =VFIO_AP_DEVICE (obj);
@@ -335,7 +338,7 @@ static void vfio_ap_class_init(ObjectClass *klass, const void *data)#ifdef CONFIG_IOMMUFD object_class_property_add_str (klass,"fd" ,NULL , vfio_ap_set_fd);#endif - dc->vmsd = &vfio_ap_vmstate; + dc->vmsd = &vmstate_ap_device; dc->desc ="VFIO-based AP device assignment" ;set_bit (DEVICE_CATEGORY_MISC , dc->categories); dc->realize = vfio_ap_realize;
Signature · 1 line
2.52.0
Re: [PATCH v2 1/2] hw/vfio/ap: Extend vfio device state description to facilitate live migration
Quoted 52 lines · Anthony Krowiak
Make the changes needed to the VFIOAPDevice state description to facilitate live guest migration.
Signed-off-by Anthony Krowiak <[email protected]>Reviewed-by Jason J. Herne <[email protected]>--- hw/vfio/ap.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-)hw/vfio/ap.c +10
-7@@ -41,6 +41,16 @@ struct VFIOAPDevice {EventNotifier req_notifier;EventNotifier cfg_notifier; }; +OBJECT_DECLARE_SIMPLE_TYPE(VFIOAPDevice, VFIO_AP_DEVICE) + +static const VMStateDescription vmstate_ap_device = { + .name ="vfio-ap-device" , + .version_id =1 , + .minimum_version_id =1 , + .fields = (const VMStateField []) { +VMSTATE_END_OF_LIST () + }, +}; typedef struct APConfigChgEvent { QTAILQ_ENTRY(APConfigChgEvent) next;@@ -56,8 +66,6 @@ static void __attribute__((constructor)) vfio_ap_global_init(void)qemu_mutex_init (&cfg_chg_events_lock); } -OBJECT_DECLARE_SIMPLE_TYPE(VFIOAPDevice, VFIO_AP_DEVICE) -static void vfio_ap_compute_needs_reset (VFIODevice *vdev) { vdev->needs_reset = false;@@ -297,11 +305,6 @@ static void vfio_ap_reset(DeviceState *dev) } } -static const VMStateDescription vfio_ap_vmstate = { - .name ="vfio-ap" , - .unmigratable =1 , -}; -static void vfio_ap_instance_init (Object *obj) {VFIOAPDevice *vapdev =VFIO_AP_DEVICE (obj);
hw/vfio/ap.c static void vfio_ap_class_init(ObjectClass *klass, const void *data) · line 340
Quoted 3 lines
hw/vfio/ap.c +0
-0@@ -335,7 +338,7 @@ static void vfio_ap_class_init(ObjectClass *klass, const void *data)#ifdef CONFIG_IOMMUFD object_class_property_add_str (klass,"fd" ,NULL , vfio_ap_set_fd);#endif
You will need to rebase on 'vfio-next' or 'master' when the changes pending in the vfio queue are merged.
Quoted 5 lines
hw/vfio/ap.c +1
-1- dc->vmsd = &vfio_ap_vmstate; + dc->vmsd = &vmstate_ap_device; dc->desc ="VFIO-based AP device assignment" ;set_bit (DEVICE_CATEGORY_MISC , dc->categories); dc->realize = vfio_ap_realize;
Re: [PATCH v2 1/2] hw/vfio/ap: Extend vfio device state description to facilitate live migration
Quoted 50 lines · Cédric Le Goater
Quoted 49 lines · Anthony Krowiak
Make the changes needed to the VFIOAPDevice state description to facilitate live guest migration.
Signed-off-by Anthony Krowiak <[email protected]>Reviewed-by Jason J. Herne <[email protected]>---
hw/vfio/ap.c| 19 +++++++++++--------1 file changed, 11 insertions(+), 8 deletions(-)hw/vfio/ap.c +10
-1@@ -41,6 +41,16 @@ struct VFIOAPDevice { EventNotifier req_notifier; EventNotifier cfg_notifier; }; +OBJECT_DECLARE_SIMPLE_TYPE(VFIOAPDevice, VFIO_AP_DEVICE) + +static const VMStateDescription vmstate_ap_device = { + .name = "vfio-ap-device", + .version_id = 1, + .minimum_version_id = 1, + .fields = (const VMStateField[]) { + VMSTATE_END_OF_LIST() + }, +}; typedef struct APConfigChgEvent { QTAILQ_ENTRY(APConfigChgEvent) next;@@ -56,8 +66,6 @@ static void __attribute__((constructor))vfio_ap_global_init (void ) qemu_mutex_init(&cfg_chg_events_lock); } -OBJECT_DECLARE_SIMPLE_TYPE(VFIOAPDevice, VFIO_AP_DEVICE) - static void vfio_ap_compute_needs_reset(VFIODevice *vdev) {
vdev->needs_reset= false;hw/vfio/ap.c +0
-4@@ -297,11 +305,6 @@ static void vfio_ap_reset(DeviceState *dev) } } -static const VMStateDescription vfio_ap_vmstate = { - .name = "vfio-ap", - .unmigratable = 1, -}; - static void vfio_ap_instance_init(Object *obj) { VFIOAPDevice *vapdev = VFIO_AP_DEVICE(obj);
Quoted 8 lines
hw/vfio/ap.c static void vfio_ap_class_init(ObjectClass · line 341
Quoted 5 lines
hw/vfio/ap.c +0
-0@@ -335,7 +338,7 @@ static void vfio_ap_class_init(ObjectClass *klass, const void *data) #ifdef CONFIG_IOMMUFD object_class_property_add_str(klass, "fd", NULL, vfio_ap_set_fd); #endifYou will need to rebase on 'vfio-next' or 'master' when the changes pending in the vfio queue are merged.
Will do. Thanks for the review.
Quoted 8 lines
Quoted 7 lines
hw/vfio/ap.c +1
-1- dc->vmsd = &vfio_ap_vmstate; + dc->vmsd = &vmstate_ap_device; dc->desc = "VFIO-based AP device assignment"; set_bit(DEVICE_CATEGORY_MISC, dc->categories);
dc->realize=vfio_ap_realize;
PATCH v2 2/2 hw/vfio/ap: Callbacks for migration of guests with pass-through access to AP devices
In reply to Anthony KrowiakFrom: Anthony Krowiak <[email protected]>
Implements the callbacks used by the VFIO migration framework to migrate guests with pass-through access to s390 crypto devices.
--- hw/vfio/ap.c | 43 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 4 deletions(-)
hw/vfio/ap.c +39 -4
@@ -32,6 +32,7 @@#include "hw/s390x/ap-bridge.h" #include "system/address-spaces.h" #include "qom/object.h" +#include "vfio-migration-internal.h" #define TYPE_VFIO_AP_DEVICE "vfio-ap"
@@ -47,6 +48,7 @@ static const VMStateDescription vmstate_ap_device = { .name = "vfio-ap-device", .version_id = 1, .minimum_version_id = 1, + .unmigratable = 0, .fields = (const VMStateField []) {VMSTATE_END_OF_LIST () },
@@ -66,17 +68,44 @@ static void __attribute__((constructor)) vfio_ap_global_init(void)qemu_mutex_init (&cfg_chg_events_lock); } +static int vfio_ap_save_config (VFIODevice *vdev,QEMUFile *f,Error **errp) +{ +VFIOAPDevice *vapdev =container_of (vdev, VFIOAPDevice, vdev); + +return vmstate_save_state (f, &vmstate_ap_device, vapdev,NULL , errp); +} + +static int vfio_ap_load_config (VFIODevice *vdev,QEMUFile *f) +{ +VFIOAPDevice *vapdev =container_of (vdev, VFIOAPDevice, vdev); +Error *err =NULL ; +int ret; + + ret =vmstate_load_state (f, &vmstate_ap_device, vapdev,1 , &err); +if (ret) { +error_report_err (err); + } + +return ret; +} + +static Object *vfio_ap_get_object (VFIODevice *vbasedev) +{ +VFIOAPDevice *vdev =container_of (vbasedev, VFIOAPDevice, vdev); + +return OBJECT (vdev); +} +static void vfio_ap_compute_needs_reset (VFIODevice *vdev) { vdev->needs_reset = false; } -/* -* We don't need vfio_hot_reset_multi and vfio_eoi operations for -* vfio-ap device now. -*/ struct VFIODeviceOps vfio_ap_ops = { .vfio_compute_needs_reset = vfio_ap_compute_needs_reset, + .vfio_save_config = vfio_ap_save_config, + .vfio_load_config = vfio_ap_load_config, + .vfio_get_object = vfio_ap_get_object, };static void vfio_ap_req_notifier_handler (void *opaque)
@@ -251,6 +280,10 @@ static void vfio_ap_realize(DeviceState *dev, Error **errp) goto error; } +if (!vfio_migration_realize (vbasedev, errp)) { + goto error; + } +if (!vfio_ap_register_irq_notifier (vapdev,VFIO_AP_REQ_IRQ_INDEX , &err)) {/* * Report this error, but do not make it a failing condition.
@@ -287,6 +320,8 @@ static void vfio_ap_unrealize(DeviceState *dev)static const Property vfio_ap_properties[] = {DEFINE_PROP_STRING ("sysfsdev" , VFIOAPDevice, vdev.sysfsdev), +DEFINE_PROP_ON_OFF_AUTO ("enable-migration" , VFIOAPDevice, + vdev.enable_migration,ON_OFF_AUTO_ON ), #ifdef CONFIG_IOMMUFDDEFINE_PROP_LINK ("iommufd" , VFIOAPDevice, vdev.iommufd, TYPE_IOMMUFD_BACKEND, IOMMUFDBackend *),
Signature · 1 line
2.52.0
Re: [PATCH v2 2/2] hw/vfio/ap: Callbacks for migration of guests with pass-through access to AP devices
Quoted 23 lines · Anthony Krowiak
From: Anthony Krowiak <[email protected]>
Implements the callbacks used by the VFIO migration framework to migrate guests with pass-through access to s390 crypto devices.
Signed-off-by Anthony Krowiak <[email protected]>--- hw/vfio/ap.c | 43 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 4 deletions(-)hw/vfio/ap.c +1
-0@@ -32,6 +32,7 @@#include "hw/s390x/ap-bridge.h" #include "system/address-spaces.h" #include "qom/object.h" +#include "vfio-migration-internal.h" #define TYPE_VFIO_AP_DEVICE "vfio-ap"
hw/vfio/ap.c static const VMStateDescription vmstate_ap_device = { · line 51
Quoted 4 lines
hw/vfio/ap.c +1
-0@@ -47,6 +48,7 @@ static const VMStateDescription vmstate_ap_device = { .name = "vfio-ap-device", .version_id = 1, .minimum_version_id = 1, + .unmigratable = 0,
unneeded.
Quoted 64 lines
hw/vfio/ap.c +35
-4.fields = (const VMStateField[]) {VMSTATE_END_OF_LIST () },@@ -66,17 +68,44 @@ static void __attribute__((constructor)) vfio_ap_global_init(void)qemu_mutex_init (&cfg_chg_events_lock); } +static int vfio_ap_save_config (VFIODevice *vdev,QEMUFile *f,Error **errp) +{ +VFIOAPDevice *vapdev =container_of (vdev, VFIOAPDevice, vdev); + +return vmstate_save_state (f, &vmstate_ap_device, vapdev,NULL , errp); +} + +static int vfio_ap_load_config (VFIODevice *vdev,QEMUFile *f) +{ +VFIOAPDevice *vapdev =container_of (vdev, VFIOAPDevice, vdev); +Error *err =NULL ; +int ret; + + ret =vmstate_load_state (f, &vmstate_ap_device, vapdev,1 , &err); +if (ret) { +error_report_err (err); + } + +return ret; +} + +static Object *vfio_ap_get_object (VFIODevice *vbasedev) +{ +VFIOAPDevice *vdev =container_of (vbasedev, VFIOAPDevice, vdev); + +return OBJECT (vdev); +} +static void vfio_ap_compute_needs_reset (VFIODevice *vdev) { vdev->needs_reset = false; } -/* -* We don't need vfio_hot_reset_multi and vfio_eoi operations for -* vfio-ap device now. -*/ struct VFIODeviceOps vfio_ap_ops = { .vfio_compute_needs_reset = vfio_ap_compute_needs_reset, + .vfio_save_config = vfio_ap_save_config, + .vfio_load_config = vfio_ap_load_config, + .vfio_get_object = vfio_ap_get_object, };static void vfio_ap_req_notifier_handler (void *opaque)@@ -251,6 +280,10 @@ static void vfio_ap_realize(DeviceState *dev, Error **errp) goto error; } +if (!vfio_migration_realize (vbasedev, errp)) { + goto error; + } +if (!vfio_ap_register_irq_notifier (vapdev,VFIO_AP_REQ_IRQ_INDEX , &err)) {/* * Report this error, but do not make it a failing condition.
hw/vfio/ap.c static void vfio_ap_unrealize(DeviceState *dev) · line 324
Quoted 5 lines
hw/vfio/ap.c +2
-0@@ -287,6 +320,8 @@ static void vfio_ap_unrealize(DeviceState *dev)static const Property vfio_ap_properties[] = {DEFINE_PROP_STRING ("sysfsdev" , VFIOAPDevice, vdev.sysfsdev), +DEFINE_PROP_ON_OFF_AUTO ("enable-migration" , VFIOAPDevice, + vdev.enable_migration,ON_OFF_AUTO_ON ),
Why not ON_OFF_AUTO_AUTO ? This would enable migration only if kernel has migration support. If not, the VM would still start but migration would be blocked.
thanks,
C.
Quoted 3 lines
hw/vfio/ap.c +0
-0#ifdef CONFIG_IOMMUFD DEFINE_PROP_LINK ("iommufd" , VFIOAPDevice, vdev.iommufd, TYPE_IOMMUFD_BACKEND, IOMMUFDBackend *),
Re: [PATCH v2 2/2] hw/vfio/ap: Callbacks for migration of guests with pass-through access to AP devices
Quoted 20 lines · Cédric Le Goater
Quoted 19 lines · Anthony Krowiak
From: Anthony Krowiak <[email protected]>
Implements the callbacks used by the VFIO migration framework to migrate guests with pass-through access to s390 crypto devices.
Signed-off-by Anthony Krowiak <[email protected]>---
hw/vfio/ap.c| 43 +++++++++++++++++++++++++++++++++++++++----1 file changed, 39 insertions(+), 4 deletions(-)hw/vfio/ap.c +1
-0@@ -32,6 +32,7 @@ #include "hw/s390x/ap-bridge.h" #include "system/address-spaces.h" #include "qom/object.h" +#include "vfio-migration-internal.h"
Quoted 8 lines
Quoted 6 lines
hw/vfio/ap.c +0
-0#define TYPE_VFIO_AP_DEVICE "vfio-ap" @@ -47,6 +48,7 @@ static const VMStateDescription vmstate_ap_device = { .name = "vfio-ap-device",.version_id = 1,
.minimum_version_id = 1,
+ .unmigratable = 0,unneeded.
I'll remove it
Quoted 64 lines
Quoted 64 lines
.fields = (const VMStateField[]) {
VMSTATE_END_OF_LIST()
},hw/vfio/ap.c +27
-0@@ -66,17 +68,44 @@ static void __attribute__((constructor))vfio_ap_global_init (void ) qemu_mutex_init(&cfg_chg_events_lock); } +static int vfio_ap_save_config(VFIODevice *vdev, QEMUFile *f, Error **errp) +{ + VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev); + + return vmstate_save_state(f, &vmstate_ap_device, vapdev, NULL, errp); +} + +static int vfio_ap_load_config (VFIODevice *vdev,QEMUFile *f) +{ + VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev); + Error *err = NULL; + int ret; + + ret = vmstate_load_state(f, &vmstate_ap_device, vapdev, 1, &err); + if (ret) { + error_report_err(err); + } + + return ret; +} + +static Object *vfio_ap_get_object (VFIODevice *vbasedev) +{ + VFIOAPDevice *vdev = container_of(vbasedev, VFIOAPDevice, vdev); + + return OBJECT(vdev); +} + static void vfio_ap_compute_needs_reset(VFIODevice *vdev) { vdev->needs_reset = false; }-/*
- * We don't needvfio_hot_reset_multiandvfio_eoioperations for
- * vfio-ap device now.
- */
struct VFIODeviceOpsvfio_ap_ops= { .vfio_compute_needs_reset =vfio_ap_compute_needs_reset, + .vfio_save_config =vfio_ap_save_config,
+ .vfio_load_config =vfio_ap_load_config,
+ .vfio_get_object =vfio_ap_get_object,
};
static voidvfio_ap_req_notifier_handler(void *opaque)hw/vfio/ap.c +3
-0@@ -251,6 +280,10 @@ static void vfio_ap_realize(DeviceState *dev, Error **errp) goto error; } + if (!vfio_migration_realize(vbasedev, errp)) { + goto error; + } + if (!vfio_ap_register_irq_notifier(vapdev, VFIO_AP_REQ_IRQ_INDEX, &err)) { /** Report this error, but do not make it a failing condition.
hw/vfio/ap.c +0
-0@@ -287,6 +320,8 @@ static void vfio_ap_unrealize(DeviceState *dev)
Quoted 8 lines
hw/vfio/ap.c static void vfio_ap_unrealize(DeviceState *dev) · line 324
Quoted 4 lines
hw/vfio/ap.c +2
-0static const Property vfio_ap_properties[] = { DEFINE_PROP_STRING("sysfsdev", VFIOAPDevice, vdev.sysfsdev), + DEFINE_PROP_ON_OFF_AUTO("enable-migration", VFIOAPDevice, + vdev.enable_migration, ON_OFF_AUTO_ON),Why not
ON_OFF_AUTO_AUTO? This would enable migration only if kernel has migration support. If not, the VM would still start but migration would be blocked.
I will make that change but I'm wondering; how does QEMU know whether the kernel has support for migration of vfio-ap devices or not?
Quoted 11 lines
thanks,
C.
Quoted 3 lines
hw/vfio/ap.c +0
-0#ifdef CONFIG_IOMMUFD DEFINE_PROP_LINK("iommufd", VFIOAPDevice, vdev.iommufd, TYPE_IOMMUFD_BACKEND, IOMMUFDBackend *),
Re: [PATCH v2 2/2] hw/vfio/ap: Callbacks for migration of guests with pass-through access to AP devices
Quoted 104 lines · Anthony Krowiak
Quoted 20 lines · Cédric Le Goater
Quoted 19 lines · Anthony Krowiak
From: Anthony Krowiak <[email protected]>
Implements the callbacks used by the VFIO migration framework to migrate guests with pass-through access to s390 crypto devices.
Signed-off-by Anthony Krowiak <[email protected]>---
hw/vfio/ap.c| 43 +++++++++++++++++++++++++++++++++++++++----1 file changed, 39 insertions(+), 4 deletions(-)hw/vfio/ap.c +1
-0@@ -32,6 +32,7 @@ #include "hw/s390x/ap-bridge.h" #include "system/address-spaces.h" #include "qom/object.h" +#include "vfio-migration-internal.h" Quoted 8 lines
Quoted 6 lines
hw/vfio/ap.c +0
-0#define TYPE_VFIO_AP_DEVICE "vfio-ap" @@ -47,6 +48,7 @@ static const VMStateDescription vmstate_ap_device = { .name = "vfio-ap-device",.version_id = 1,
.minimum_version_id = 1,
+ .unmigratable = 0,unneeded.
I'll remove it
Quoted 61 lines
Quoted 61 lines
.fields = (const VMStateField[]) {
VMSTATE_END_OF_LIST()
},hw/vfio/ap.c +27
-0@@ -66,17 +68,44 @@ static void __attribute__((constructor)) vfio_ap_global_init(void) qemu_mutex_init(&cfg_chg_events_lock); } +static int vfio_ap_save_config(VFIODevice *vdev, QEMUFile *f, Error **errp) +{ + VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev); + + return vmstate_save_state(f, &vmstate_ap_device, vapdev, NULL, errp); +} + +static int vfio_ap_load_config (VFIODevice *vdev,QEMUFile *f) +{ + VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev); + Error *err = NULL; + int ret; + + ret = vmstate_load_state(f, &vmstate_ap_device, vapdev, 1, &err); + if (ret) { + error_report_err(err); + } + + return ret; +} + +static Object *vfio_ap_get_object (VFIODevice *vbasedev) +{ + VFIOAPDevice *vdev = container_of(vbasedev, VFIOAPDevice, vdev); + + return OBJECT(vdev); +} + static void vfio_ap_compute_needs_reset(VFIODevice *vdev) { vdev->needs_reset = false; }-/*
- * We don't needvfio_hot_reset_multiandvfio_eoioperations for
- * vfio-ap device now.
- */
struct VFIODeviceOpsvfio_ap_ops= { .vfio_compute_needs_reset =vfio_ap_compute_needs_reset, + .vfio_save_config =vfio_ap_save_config,
+ .vfio_load_config =vfio_ap_load_config,
+ .vfio_get_object =vfio_ap_get_object,
};
static voidvfio_ap_req_notifier_handler(void *opaque)hw/vfio/ap.c +3
-0@@ -251,6 +280,10 @@ static void vfio_ap_realize(DeviceState *dev, Error **errp) goto error; } + if (!vfio_migration_realize(vbasedev, errp)) { + goto error; + } + if (!vfio_ap_register_irq_notifier(vapdev, VFIO_AP_REQ_IRQ_INDEX, &err)) { /** Report this error, but do not make it a failing condition. @@ -287,6 +320,8 @@ static void vfio_ap_unrealize(DeviceState *dev)Quoted 8 lines
hw/vfio/ap.c static void vfio_ap_unrealize(DeviceState *dev) · line 324
Quoted 4 lines
hw/vfio/ap.c +2
-0static const Property vfio_ap_properties[] = { DEFINE_PROP_STRING("sysfsdev", VFIOAPDevice, vdev.sysfsdev), + DEFINE_PROP_ON_OFF_AUTO("enable-migration", VFIOAPDevice, + vdev.enable_migration, ON_OFF_AUTO_ON),Why not
ON_OFF_AUTO_AUTO? This would enable migration only if kernel has migration support. If not, the VM would still start but migration would be blocked.
Quoted 2 lines
I will make that change but I'm wondering; how does QEMU know whether the kernel has support for migration of vfio-ap devices or not?
vfio_migration_init() will probe the kernel for migration support and install a migration blocker in case of failure.
You should see a message like :
VFIO migration init failed: migration is not supported in kernel
Thanks,
C.
Re: [PATCH v2 2/2] hw/vfio/ap: Callbacks for migration of guests with pass-through access to AP devices
In reply to Cédric Le GoaterQuoted 92 lines · Cédric Le Goater
Quoted 22 lines · Anthony Krowiak
From: Anthony Krowiak <[email protected]>
Implements the callbacks used by the VFIO migration framework to migrate guests with pass-through access to s390 crypto devices.
Signed-off-by Anthony Krowiak <[email protected]>---
hw/vfio/ap.c| 43 +++++++++++++++++++++++++++++++++++++++----1 file changed, 39 insertions(+), 4 deletions(-)hw/vfio/ap.c +1
-0@@ -32,6 +32,7 @@ #include "hw/s390x/ap-bridge.h" #include "system/address-spaces.h" #include "qom/object.h" +#include "vfio-migration-internal.h" #define TYPE_VFIO_AP_DEVICE "vfio-ap" @@ -47,6 +48,7 @@ static const VMStateDescription vmstate_ap_device = { .name = "vfio-ap-device",Quoted 3 lines
.version_id = 1,
.minimum_version_id = 1,
+ .unmigratable = 0,unneeded.
Quoted 63 lines
.fields = (const VMStateField[]) {
VMSTATE_END_OF_LIST()
},hw/vfio/ap.c +27
-0@@ -66,17 +68,44 @@ static void __attribute__((constructor))vfio_ap_global_init (void ) qemu_mutex_init(&cfg_chg_events_lock); } +static int vfio_ap_save_config(VFIODevice *vdev, QEMUFile *f, Error **errp) +{ + VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev); + + return vmstate_save_state(f, &vmstate_ap_device, vapdev, NULL, errp); +} + +static int vfio_ap_load_config (VFIODevice *vdev,QEMUFile *f) +{ + VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev); + Error *err = NULL; + int ret; + + ret = vmstate_load_state(f, &vmstate_ap_device, vapdev, 1, &err); + if (ret) { + error_report_err(err); + } + + return ret; +} + +static Object *vfio_ap_get_object (VFIODevice *vbasedev) +{ + VFIOAPDevice *vdev = container_of(vbasedev, VFIOAPDevice, vdev); + + return OBJECT(vdev); +} + static void vfio_ap_compute_needs_reset(VFIODevice *vdev) { vdev->needs_reset = false; }-/*
- * We don't needvfio_hot_reset_multiandvfio_eoioperations for
- * vfio-ap device now.
- */
struct VFIODeviceOpsvfio_ap_ops= { .vfio_compute_needs_reset =vfio_ap_compute_needs_reset, + .vfio_save_config =vfio_ap_save_config,
+ .vfio_load_config =vfio_ap_load_config,
+ .vfio_get_object =vfio_ap_get_object,
};
static voidvfio_ap_req_notifier_handler(void *opaque)hw/vfio/ap.c +3
-0@@ -251,6 +280,10 @@ static void vfio_ap_realize(DeviceState *dev, Error **errp) goto error; } + if (!vfio_migration_realize(vbasedev, errp)) { + goto error; + } + if (!vfio_ap_register_irq_notifier(vapdev, VFIO_AP_REQ_IRQ_INDEX, &err)) { /** Report this error, but do not make it a failing condition.
hw/vfio/ap.c +0
-0@@ -287,6 +320,8 @@ static void vfio_ap_unrealize(DeviceState *dev)
Quoted 8 lines
hw/vfio/ap.c static void vfio_ap_unrealize(DeviceState *dev) · line 324
Quoted 4 lines
hw/vfio/ap.c +2
-0static const Property vfio_ap_properties[] = { DEFINE_PROP_STRING("sysfsdev", VFIOAPDevice, vdev.sysfsdev), + DEFINE_PROP_ON_OFF_AUTO("enable-migration", VFIOAPDevice, + vdev.enable_migration, ON_OFF_AUTO_ON),Why not
ON_OFF_AUTO_AUTO? This would enable migration only if kernel has migration support. If not, the VM would still start but migration would be blocked.
Will this allow the vfio-ap device migration patches to get merged before the kernel/KVM patches?
Quoted 10 lines
thanks,
C.
Quoted 3 lines
hw/vfio/ap.c +0
-0#ifdef CONFIG_IOMMUFD DEFINE_PROP_LINK("iommufd", VFIOAPDevice, vdev.iommufd, TYPE_IOMMUFD_BACKEND, IOMMUFDBackend *),
Re: [PATCH v2 2/2] hw/vfio/ap: Callbacks for migration of guests with pass-through access to AP devices
Quoted 101 lines · Anthony Krowiak
Quoted 89 lines · Cédric Le Goater
Quoted 22 lines · Anthony Krowiak
From: Anthony Krowiak <[email protected]>
Implements the callbacks used by the VFIO migration framework to migrate guests with pass-through access to s390 crypto devices.
Signed-off-by Anthony Krowiak <[email protected]>---
hw/vfio/ap.c| 43 +++++++++++++++++++++++++++++++++++++++----1 file changed, 39 insertions(+), 4 deletions(-)hw/vfio/ap.c +1
-0@@ -32,6 +32,7 @@ #include "hw/s390x/ap-bridge.h" #include "system/address-spaces.h" #include "qom/object.h" +#include "vfio-migration-internal.h" #define TYPE_VFIO_AP_DEVICE "vfio-ap" @@ -47,6 +48,7 @@ static const VMStateDescription vmstate_ap_device = { .name = "vfio-ap-device",Quoted 3 lines
.version_id = 1,
.minimum_version_id = 1,
+ .unmigratable = 0,unneeded.
Quoted 60 lines
.fields = (const VMStateField[]) {
VMSTATE_END_OF_LIST()
},hw/vfio/ap.c +27
-0@@ -66,17 +68,44 @@ static void __attribute__((constructor)) vfio_ap_global_init(void) qemu_mutex_init(&cfg_chg_events_lock); } +static int vfio_ap_save_config(VFIODevice *vdev, QEMUFile *f, Error **errp) +{ + VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev); + + return vmstate_save_state(f, &vmstate_ap_device, vapdev, NULL, errp); +} + +static int vfio_ap_load_config (VFIODevice *vdev,QEMUFile *f) +{ + VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev); + Error *err = NULL; + int ret; + + ret = vmstate_load_state(f, &vmstate_ap_device, vapdev, 1, &err); + if (ret) { + error_report_err(err); + } + + return ret; +} + +static Object *vfio_ap_get_object (VFIODevice *vbasedev) +{ + VFIOAPDevice *vdev = container_of(vbasedev, VFIOAPDevice, vdev); + + return OBJECT(vdev); +} + static void vfio_ap_compute_needs_reset(VFIODevice *vdev) { vdev->needs_reset = false; }-/*
- * We don't needvfio_hot_reset_multiandvfio_eoioperations for
- * vfio-ap device now.
- */
struct VFIODeviceOpsvfio_ap_ops= { .vfio_compute_needs_reset =vfio_ap_compute_needs_reset, + .vfio_save_config =vfio_ap_save_config,
+ .vfio_load_config =vfio_ap_load_config,
+ .vfio_get_object =vfio_ap_get_object,
};
static voidvfio_ap_req_notifier_handler(void *opaque)hw/vfio/ap.c +3
-0@@ -251,6 +280,10 @@ static void vfio_ap_realize(DeviceState *dev, Error **errp) goto error; } + if (!vfio_migration_realize(vbasedev, errp)) { + goto error; + } + if (!vfio_ap_register_irq_notifier(vapdev, VFIO_AP_REQ_IRQ_INDEX, &err)) { /** Report this error, but do not make it a failing condition. @@ -287,6 +320,8 @@ static void vfio_ap_unrealize(DeviceState *dev)Quoted 8 lines
hw/vfio/ap.c static void vfio_ap_unrealize(DeviceState *dev) · line 324
Quoted 4 lines
hw/vfio/ap.c +2
-0static const Property vfio_ap_properties[] = { DEFINE_PROP_STRING("sysfsdev", VFIOAPDevice, vdev.sysfsdev), + DEFINE_PROP_ON_OFF_AUTO("enable-migration", VFIOAPDevice, + vdev.enable_migration, ON_OFF_AUTO_ON),Why not
ON_OFF_AUTO_AUTO? This would enable migration only if kernel has migration support. If not, the VM would still start but migration would be blocked.
Quoted 2 lines
Will this allow the vfio-ap device migration patches to get merged before the kernel/KVM patches?
It's possible. Several reasons could explain the lack of migration support. The most obvious is a lack of kernel support, but it could also be a firmware support issue, an incompatible hardware state of the adapter, adapter model, etc.
Do all CEXxS crypto cards support migration ?
Do you expect kernel support [*] to be merged in v7.1 ?
Also, I think the QEMU support lacks vfio_migration_exit().
Thanks,
C.
Re: [PATCH v2 2/2] hw/vfio/ap: Callbacks for migration of guests with pass-through access to AP devices
In reply to Cédric Le GoaterQuoted 92 lines · Cédric Le Goater
Quoted 22 lines · Anthony Krowiak
From: Anthony Krowiak <[email protected]>
Implements the callbacks used by the VFIO migration framework to migrate guests with pass-through access to s390 crypto devices.
Signed-off-by Anthony Krowiak <[email protected]>---
hw/vfio/ap.c| 43 +++++++++++++++++++++++++++++++++++++++----1 file changed, 39 insertions(+), 4 deletions(-)hw/vfio/ap.c +1
-0@@ -32,6 +32,7 @@ #include "hw/s390x/ap-bridge.h" #include "system/address-spaces.h" #include "qom/object.h" +#include "vfio-migration-internal.h" #define TYPE_VFIO_AP_DEVICE "vfio-ap" @@ -47,6 +48,7 @@ static const VMStateDescription vmstate_ap_device = { .name = "vfio-ap-device",Quoted 3 lines
.version_id = 1,
.minimum_version_id = 1,
+ .unmigratable = 0,unneeded.
Quoted 63 lines
.fields = (const VMStateField[]) {
VMSTATE_END_OF_LIST()
},hw/vfio/ap.c +27
-0@@ -66,17 +68,44 @@ static void __attribute__((constructor))vfio_ap_global_init (void ) qemu_mutex_init(&cfg_chg_events_lock); } +static int vfio_ap_save_config(VFIODevice *vdev, QEMUFile *f, Error **errp) +{ + VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev); + + return vmstate_save_state(f, &vmstate_ap_device, vapdev, NULL, errp); +} + +static int vfio_ap_load_config (VFIODevice *vdev,QEMUFile *f) +{ + VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev); + Error *err = NULL; + int ret; + + ret = vmstate_load_state(f, &vmstate_ap_device, vapdev, 1, &err); + if (ret) { + error_report_err(err); + } + + return ret; +} + +static Object *vfio_ap_get_object (VFIODevice *vbasedev) +{ + VFIOAPDevice *vdev = container_of(vbasedev, VFIOAPDevice, vdev); + + return OBJECT(vdev); +} + static void vfio_ap_compute_needs_reset(VFIODevice *vdev) { vdev->needs_reset = false; }-/*
- * We don't needvfio_hot_reset_multiandvfio_eoioperations for
- * vfio-ap device now.
- */
struct VFIODeviceOpsvfio_ap_ops= { .vfio_compute_needs_reset =vfio_ap_compute_needs_reset, + .vfio_save_config =vfio_ap_save_config,
+ .vfio_load_config =vfio_ap_load_config,
+ .vfio_get_object =vfio_ap_get_object,
};
static voidvfio_ap_req_notifier_handler(void *opaque)hw/vfio/ap.c +3
-0@@ -251,6 +280,10 @@ static void vfio_ap_realize(DeviceState *dev, Error **errp) goto error; } + if (!vfio_migration_realize(vbasedev, errp)) { + goto error; + } + if (!vfio_ap_register_irq_notifier(vapdev, VFIO_AP_REQ_IRQ_INDEX, &err)) { /** Report this error, but do not make it a failing condition.
hw/vfio/ap.c +0
-0@@ -287,6 +320,8 @@ static void vfio_ap_unrealize(DeviceState *dev)
Quoted 8 lines
hw/vfio/ap.c static void vfio_ap_unrealize(DeviceState *dev) · line 324
Quoted 4 lines
hw/vfio/ap.c +2
-0static const Property vfio_ap_properties[] = { DEFINE_PROP_STRING("sysfsdev", VFIOAPDevice, vdev.sysfsdev), + DEFINE_PROP_ON_OFF_AUTO("enable-migration", VFIOAPDevice, + vdev.enable_migration, ON_OFF_AUTO_ON),Why not
ON_OFF_AUTO_AUTO? This would enable migration only if kernel has migration support. If not, the VM would still start but migration would be blocked.
I made this change and migration failed on the destination host with the following error message:
error: Requested operation is not valid: cannot migrate domain: 62177883-f1bb-47f0-914d-32a22e3a8804: VFIO device doesn't support device and IOMMU dirty tracking
This despite having the following property defined:
DEFINE_PROP_ON_OFF_AUTO("x-device-dirty-page-tracking", VFIOAPDevice, vdev.device_dirty_page_tracking, ON_OFF_AUTO_ON)
Quoted 10 lines
thanks,
C.
Quoted 3 lines
hw/vfio/ap.c +0
-0#ifdef CONFIG_IOMMUFD DEFINE_PROP_LINK("iommufd", VFIOAPDevice, vdev.iommufd, TYPE_IOMMUFD_BACKEND, IOMMUFDBackend *),
Re: [PATCH v2 2/2] hw/vfio/ap: Callbacks for migration of guests with pass-through access to AP devices
Quoted 101 lines · Anthony Krowiak
Quoted 89 lines · Cédric Le Goater
Quoted 22 lines · Anthony Krowiak
From: Anthony Krowiak <[email protected]>
Implements the callbacks used by the VFIO migration framework to migrate guests with pass-through access to s390 crypto devices.
Signed-off-by Anthony Krowiak <[email protected]>---
hw/vfio/ap.c| 43 +++++++++++++++++++++++++++++++++++++++----1 file changed, 39 insertions(+), 4 deletions(-)hw/vfio/ap.c +1
-0@@ -32,6 +32,7 @@ #include "hw/s390x/ap-bridge.h" #include "system/address-spaces.h" #include "qom/object.h" +#include "vfio-migration-internal.h" #define TYPE_VFIO_AP_DEVICE "vfio-ap" @@ -47,6 +48,7 @@ static const VMStateDescription vmstate_ap_device = { .name = "vfio-ap-device",Quoted 3 lines
.version_id = 1,
.minimum_version_id = 1,
+ .unmigratable = 0,unneeded.
Quoted 60 lines
.fields = (const VMStateField[]) {
VMSTATE_END_OF_LIST()
},hw/vfio/ap.c +27
-0@@ -66,17 +68,44 @@ static void __attribute__((constructor)) vfio_ap_global_init(void) qemu_mutex_init(&cfg_chg_events_lock); } +static int vfio_ap_save_config(VFIODevice *vdev, QEMUFile *f, Error **errp) +{ + VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev); + + return vmstate_save_state(f, &vmstate_ap_device, vapdev, NULL, errp); +} + +static int vfio_ap_load_config (VFIODevice *vdev,QEMUFile *f) +{ + VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev); + Error *err = NULL; + int ret; + + ret = vmstate_load_state(f, &vmstate_ap_device, vapdev, 1, &err); + if (ret) { + error_report_err(err); + } + + return ret; +} + +static Object *vfio_ap_get_object (VFIODevice *vbasedev) +{ + VFIOAPDevice *vdev = container_of(vbasedev, VFIOAPDevice, vdev); + + return OBJECT(vdev); +} + static void vfio_ap_compute_needs_reset(VFIODevice *vdev) { vdev->needs_reset = false; }-/*
- * We don't needvfio_hot_reset_multiandvfio_eoioperations for
- * vfio-ap device now.
- */
struct VFIODeviceOpsvfio_ap_ops= { .vfio_compute_needs_reset =vfio_ap_compute_needs_reset, + .vfio_save_config =vfio_ap_save_config,
+ .vfio_load_config =vfio_ap_load_config,
+ .vfio_get_object =vfio_ap_get_object,
};
static voidvfio_ap_req_notifier_handler(void *opaque)hw/vfio/ap.c +3
-0@@ -251,6 +280,10 @@ static void vfio_ap_realize(DeviceState *dev, Error **errp) goto error; } + if (!vfio_migration_realize(vbasedev, errp)) { + goto error; + } + if (!vfio_ap_register_irq_notifier(vapdev, VFIO_AP_REQ_IRQ_INDEX, &err)) { /** Report this error, but do not make it a failing condition. @@ -287,6 +320,8 @@ static void vfio_ap_unrealize(DeviceState *dev)Quoted 8 lines
hw/vfio/ap.c static void vfio_ap_unrealize(DeviceState *dev) · line 324
Quoted 4 lines
hw/vfio/ap.c +2
-0static const Property vfio_ap_properties[] = { DEFINE_PROP_STRING("sysfsdev", VFIOAPDevice, vdev.sysfsdev), + DEFINE_PROP_ON_OFF_AUTO("enable-migration", VFIOAPDevice, + vdev.enable_migration, ON_OFF_AUTO_ON),Why not
ON_OFF_AUTO_AUTO? This would enable migration only if kernel has migration support. If not, the VM would still start but migration would be blocked.
Quoted 8 lines
I made this change and migration failed on the destination host with the following error message:
error: Requested operation is not valid: cannot migrate domain: 62177883-f1bb-47f0-914d-32a22e3a8804: VFIO device doesn't support device and IOMMU dirty tracking
This despite having the following property defined:
DEFINE_PROP_ON_OFF_AUTO("x-device-dirty-page-tracking", VFIOAPDevice, vdev.device_dirty_page_tracking, ON_OFF_AUTO_ON)
try adding dummy vfio_log_ops handlers in the driver.
static int mtty_log_start(struct vfio_device *vdev,
struct rb_root_cached *ranges,
u32 nnodes, u64 *page_size)
{
return 0;
}static int mtty_log_stop(struct vfio_device *vdev)
{
return 0;
}static int mtty_log_read_and_clear(struct vfio_device *vdev,
unsigned long iova, unsigned long length, struct iova_bitmap *dirty)
{
return 0;
}static const struct vfio_log_ops mtty_log_ops = {
.log_start = mtty_log_start, .log_stop = mtty_log_stop, .log_read_and_clear = mtty_log_read_and_clear,
};
See samples/vfio-mdev/mtty.c.
C.