vfio-user server in QEMU
PATCH v4 5/14 vfio-user: instantiate vfio-user context
In reply to Jagannathan Ramancreate a context with the vfio-user library to run a PCI device
--- hw/remote/vfio-user-obj.c | 69 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+)
hw/remote/vfio-user-obj.c +69 -0
@@ -41,6 +41,9 @@#include "hw/remote/machine.h" #include "qapi/error.h" #include "qapi/qapi-visit-sockets.h" +#include "qemu/notify.h" +#include "sysemu/sysemu.h" +#include "libvfio-user.h" #define TYPE_VFU_OBJECT "x-vfio-user-server"OBJECT_DECLARE_TYPE (VfuObject, VfuObjectClass,VFU_OBJECT )
@@ -62,8 +65,14 @@ struct VfuObject {char *device;Error *err; + +Notifier machine_done; + +vfu_ctx_t *vfu_ctx; }; +static void vfu_object_init_ctx (VfuObject *o,Error **errp); +static void vfu_object_set_socket (Object *obj,Visitor *v,const char *name,void *opaque,Error **errp) {
@@ -84,6 +93,8 @@ static void vfu_object_set_socket(Object *obj, Visitor *v, const char *name, }trace_vfu_prop ("socket" , o->socket->u.q_unix.path); + +vfu_object_init_ctx (o, errp); }static void vfu_object_set_device (Object *obj,const char *str,Error **errp)
@@ -95,6 +106,50 @@ static void vfu_object_set_device(Object *obj, const char *str, Error **errp) o->device =g_strdup (str);trace_vfu_prop ("device" , str); + +vfu_object_init_ctx (o, errp); +} + +/* +* TYPE_VFU_OBJECT depends on the availability of the 'socket' and 'device' +* properties. It also depends on devices instantiated in QEMU. These +* dependencies are not available during the instance_init phase of this +* object's life-cycle. As such, the server is initialized after the +* machine is setup. machine_init_done_notifier notifies TYPE_VFU_OBJECT +* when the machine is setup, and the dependencies are available. +*/ +static void vfu_object_machine_done (Notifier *notifier,void *data) +{ +VfuObject *o =container_of (notifier, VfuObject, machine_done); +Error *err =NULL ; + +vfu_object_init_ctx (o, &err); + +if (err) { +error_propagate (&error_abort, err); + } +} + +static void vfu_object_init_ctx (VfuObject *o,Error **errp) +{ +ERRP_GUARD (); + +if (o->vfu_ctx || !o->socket || !o->device || + !phase_check (PHASE_MACHINE_READY )) { +return ; + } + +if (o->err) { +error_propagate (errp, o->err); +return ; + } + + o->vfu_ctx =vfu_create_ctx (VFU_TRANS_SOCK , o->socket->u.q_unix.path,0 , + o,VFU_DEV_TYPE_PCI ); +if (o->vfu_ctx ==NULL ) { +error_setg (errp,"vfu: Failed to create context - %s" ,strerror (errno)); +return ; + } }static void vfu_object_init (Object *obj)
@@ -104,6 +159,11 @@ static void vfu_object_init(Object *obj) k->nr_devs++; +if (!phase_check (PHASE_MACHINE_READY )) { + o->machine_done.notify = vfu_object_machine_done; +qemu_add_machine_init_done_notifier (&o->machine_done); + } +if (!object_dynamic_cast (OBJECT (current_machine),TYPE_REMOTE_MACHINE )) {error_setg (&o->err,"vfu: %s only compatible with %s machine" ,TYPE_VFU_OBJECT ,TYPE_REMOTE_MACHINE );
@@ -122,6 +182,10 @@ static void vfu_object_finalize(Object *obj) o->socket =NULL ; +if (o->vfu_ctx) { +vfu_destroy_ctx (o->vfu_ctx); + } +g_free (o->device); o->device =NULL ;
@@ -129,6 +193,11 @@ static void vfu_object_finalize(Object *obj)if (!k->nr_devs && !k->daemon) {qemu_system_shutdown_request (SHUTDOWN_CAUSE_GUEST_SHUTDOWN ); } + +if (o->machine_done.notify) { +qemu_remove_machine_init_done_notifier (&o->machine_done); + o->machine_done.notify =NULL ; + } }static void vfu_object_class_init (ObjectClass *klass,void *data)
Signature · 1 line
2.20.1
PATCH v4 13/14 vfio-user: register handlers to facilitate migration
In reply to Jagannathan RamanStore and load the device's state during migration. use libvfio-user's handlers for this purpose
--- include/migration/vmstate.h | 2 + migration/savevm.h | 2 + hw/remote/vfio-user-obj.c | 323 ++++++++++++++++++++++++++++++++++++ migration/savevm.c | 73 ++++++++ migration/vmstate.c | 19 +++ 5 files changed, 419 insertions(+)
include/migration/vmstate.h +2 -0
@@ -1165,6 +1165,8 @@ extern const VMStateInfo vmstate_info_qlist;#define VMSTATE_END_OF_LIST () \ {} +uint64_t vmstate_vmsd_size (PCIDevice *pci_dev); +int vmstate_load_state (QEMUFile *f,const VMStateDescription *vmsd,void *opaque,int version_id); int vmstate_save_state(QEMUFile *f, const VMStateDescription *vmsd,
migration/savevm.h +2 -0
@@ -67,5 +67,7 @@ int qemu_loadvm_state_main(QEMUFile *f, MigrationIncomingState *mis);int qemu_load_device_state (QEMUFile *f);int qemu_savevm_state_complete_precopy_non_iterable (QEMUFile *f,bool in_postcopy,bool inactivate_disks); +int qemu_remote_savevm (QEMUFile *f,DeviceState *dev); +int qemu_remote_loadvm (QEMUFile *f);#endif
hw/remote/vfio-user-obj.c +323 -0
359 lines not on this page · raw message
migration/savevm.c +73 -0
85 lines not on this page · raw message
migration/vmstate.c +19 -0
@@ -63,6 +63,25 @@ static int vmstate_size(void *opaque, const VMStateField *field)return size; } +uint64_t vmstate_vmsd_size (PCIDevice *pci_dev) +{ +DeviceClass *dc =DEVICE_GET_CLASS (DEVICE (pci_dev)); +const VMStateField *field =NULL ; +uint64_t size =0 ; + +if (!dc->vmsd) { +return 0 ; + } + + field = dc->vmsd->fields; +while (field && field->name) { + size +=vmstate_size (pci_dev, field); + field++; + } + +return size; +} +static void vmstate_handle_alloc (void *ptr,const VMStateField *field,void *opaque) {
Signature · 1 line
2.20.1
PATCH v4 8/14 vfio-user: handle PCI config space accesses
In reply to Jagannathan RamanDefine and register handlers for PCI config space accesses
--- hw/remote/vfio-user-obj.c | 45 +++++++++++++++++++++++++++++++++++++++ hw/remote/trace-events | 2 ++ 2 files changed, 47 insertions(+)
hw/remote/vfio-user-obj.c +45 -0
@@ -43,6 +43,7 @@#include "qapi/qapi-visit-sockets.h" #include "qemu/notify.h" #include "qemu/thread.h" +#include "qemu/main-loop.h" #include "sysemu/sysemu.h" #include "libvfio-user.h" #include "hw/qdev-core.h"
@@ -174,6 +175,39 @@ retry_attach:qemu_set_fd_handler (o->vfu_poll_fd, vfu_object_ctx_run,NULL , o); } +static ssize_t vfu_object_cfg_access (vfu_ctx_t *vfu_ctx,char *const buf, +size_t count,loff_t offset, +const bool is_write) +{ +VfuObject *o =vfu_get_private (vfu_ctx); +uint32_t pci_access_width =sizeof (uint32_t ); +size_t bytes = count; +uint32_t val =0 ; +char *ptr = buf; +int len; + +while (bytes >0 ) { + len = (bytes > pci_access_width) ? pci_access_width : bytes; +if (is_write) { +memcpy (&val, ptr, len); +pci_host_config_write_common (o->pci_dev, offset, +pci_config_size (o->pci_dev), + val, len); +trace_vfu_cfg_write (offset, val); + }else { + val =pci_host_config_read_common (o->pci_dev, offset, +pci_config_size (o->pci_dev), len); +memcpy (ptr, &val, len); +trace_vfu_cfg_read (offset, val); + } + offset += len; + ptr += len; + bytes -= len; + } + +return count; +} +/* * TYPE_VFU_OBJECT depends on the availability of the 'socket' and 'device' * properties. It also depends on devices instantiated in QEMU. These
@@ -244,6 +278,17 @@ static void vfu_object_init_ctx(VfuObject *o, Error **errp) goto fail; } + ret =vfu_setup_region (o->vfu_ctx,VFU_PCI_DEV_CFG_REGION_IDX , +pci_config_size (o->pci_dev), &vfu_object_cfg_access, +VFU_REGION_FLAG_RW |VFU_REGION_FLAG_ALWAYS_CB , +NULL ,0 ,-1 ,0 ); +if (ret <0 ) { +error_setg (errp, +"vfu: Failed to setup config space handlers for %s- %s" , + o->device,strerror (errno)); + goto fail; + } + ret =vfu_realize_ctx (o->vfu_ctx);if (ret <0 ) { error_setg(errp, "vfu: Failed to realize device %s- %s",
hw/remote/trace-events +2 -0
@@ -5,3 +5,5 @@ mpqemu_recv_io_error(int cmd, int size, int nfds) "failed to receive %d size %d, # vfio-user-obj.c vfu_prop(const char *prop, const char *val) "vfu: setting %s as %s" +vfu_cfg_read(uint32_t offset, uint32_t val) "vfu: cfg: 0x%u -> 0x%x" +vfu_cfg_write(uint32_t offset, uint32_t val) "vfu: cfg: 0x%u <- 0x%x"
Signature · 1 line
2.20.1
PATCH v4 11/14 vfio-user: IOMMU support for remote device
In reply to Jagannathan RamanAssign separate address space for each device in the remote processes.
--- include/hw/pci/pci.h | 2 + include/hw/remote/iommu.h | 24 ++++++++ hw/pci/pci.c | 2 +- hw/remote/iommu.c | 117 ++++++++++++++++++++++++++++++++++++++ hw/remote/machine.c | 5 ++ hw/remote/vfio-user-obj.c | 20 ++++++- MAINTAINERS | 2 + hw/remote/meson.build | 1 + 8 files changed, 169 insertions(+), 4 deletions(-) create mode 100644 include/hw/remote/iommu.h create mode 100644 hw/remote/iommu.c
include/hw/pci/pci.h +2 -0
@@ -734,6 +734,8 @@ void lsi53c8xx_handle_legacy_cmdline(DeviceState *lsi_dev);qemu_irq pci_allocate_irq (PCIDevice *pci_dev);void pci_set_irq (PCIDevice *pci_dev,int level); +void pci_init_bus_master (PCIDevice *pci_dev); +static inline void pci_irq_assert (PCIDevice *pci_dev) {pci_set_irq (pci_dev,1 );
include/hw/remote/iommu.h new file +24 -0
@@ -0,0 +1,24 @@ +/* +* IOMMU for remote device +* +* Copyright © 2021 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later. +* See the COPYING file in the top-level directory. +* +*/ + +#ifndef REMOTE_IOMMU_H +#define REMOTE_IOMMU_H + +#include "hw/pci/pci_bus.h" + +void remote_iommu_free (PCIDevice *pci_dev); + +void remote_iommu_init (void ); + +void remote_iommu_set (PCIBus *bus); + +MemoryRegion *remote_iommu_get_ram (PCIDevice *pci_dev); + +#endif
hw/pci/pci.c +1 -1
@@ -95,7 +95,7 @@ static const VMStateDescription vmstate_pcibus = { } }; -static void pci_init_bus_master (PCIDevice *pci_dev) +void pci_init_bus_master (PCIDevice *pci_dev) {AddressSpace *dma_as =pci_device_iommu_address_space (pci_dev);
hw/remote/iommu.c new file +117 -0
117 lines not on this page · raw message
hw/remote/machine.c +5 -0
@@ -21,6 +21,7 @@#include "qapi/error.h" #include "hw/pci/pci_host.h" #include "hw/remote/iohub.h" +#include "hw/remote/iommu.h" static void remote_machine_init (MachineState *machine) {
@@ -52,6 +53,10 @@ static void remote_machine_init(MachineState *machine)remote_iohub_init (&s->iohub); +remote_iommu_init (); + +remote_iommu_set (pci_host->bus); +pci_bus_irqs (pci_host->bus, remote_iohub_set_irq, remote_iohub_map_irq, &s->iohub,REMOTE_IOHUB_NB_PIRQS ); }
hw/remote/vfio-user-obj.c +17 -3
@@ -49,6 +49,7 @@#include "hw/qdev-core.h" #include "hw/pci/pci.h" #include "qemu/timer.h" +#include "hw/remote/iommu.h" #define TYPE_VFU_OBJECT "x-vfio-user-server"OBJECT_DECLARE_TYPE (VfuObject, VfuObjectClass,VFU_OBJECT )
@@ -210,6 +211,7 @@ static ssize_t vfu_object_cfg_access(vfu_ctx_t *vfu_ctx, char * const buf,static void dma_register (vfu_ctx_t *vfu_ctx,vfu_dma_info_t *info) { +VfuObject *o =vfu_get_private (vfu_ctx);MemoryRegion *subregion =NULL ; g_autofree char *name = NULL;static unsigned int suffix;
@@ -226,14 +228,15 @@ static void dma_register(vfu_ctx_t *vfu_ctx, vfu_dma_info_t *info)memory_region_init_ram_ptr (subregion,NULL , name, iov->iov_len, info->vaddr); -memory_region_add_subregion (get_system_memory (), (hwaddr )iov->iov_base, - subregion); +memory_region_add_subregion (remote_iommu_get_ram (o->pci_dev), + (hwaddr )iov->iov_base, subregion);trace_vfu_dma_register ((uint64_t )iov->iov_base, iov->iov_len); }static void dma_unregister (vfu_ctx_t *vfu_ctx,vfu_dma_info_t *info) { +VfuObject *o =vfu_get_private (vfu_ctx);MemoryRegion *mr =NULL ;ram_addr_t offset;
@@ -242,7 +245,7 @@ static void dma_unregister(vfu_ctx_t *vfu_ctx, vfu_dma_info_t *info)return ; } -memory_region_del_subregion (get_system_memory (), mr); +memory_region_del_subregion (remote_iommu_get_ram (o->pci_dev), mr);object_unparent ((OBJECT (mr)));
@@ -320,6 +323,7 @@ static vfu_region_access_cb_t *vfu_object_bar_handlers[PCI_NUM_REGIONS] = {*/ static void vfu_object_register_bars (vfu_ctx_t *vfu_ctx,PCIDevice *pdev) { +VfuObject *o =vfu_get_private (vfu_ctx);int i;for (i =0 ; i <PCI_NUM_REGIONS ; i++) {
@@ -332,6 +336,12 @@ static void vfu_object_register_bars(vfu_ctx_t *vfu_ctx, PCIDevice *pdev) vfu_object_bar_handlers[i], VFU_REGION_FLAG_RW, NULL, 0, -1, 0); +if ((o->pci_dev->io_regions[i].type &PCI_BASE_ADDRESS_SPACE ) ==0 ) { +memory_region_unref (o->pci_dev->io_regions[i].address_space); + o->pci_dev->io_regions[i].address_space = +remote_iommu_get_ram (o->pci_dev); + } +trace_vfu_bar_register (i, pdev->io_regions[i].addr, pdev->io_regions[i].size); }
@@ -490,6 +500,10 @@ static void vfu_object_finalize(Object *obj) o->device =NULL ; +if (o->pci_dev) { +remote_iommu_free (o->pci_dev); + } + o->pci_dev =NULL ;if (!k->nr_devs && !k->daemon) {
MAINTAINERS +2 -0
@@ -3466,6 +3466,8 @@ F: hw/remote/iohub.c F: include/hw/remote/iohub.h F: subprojects/libvfio-user F: hw/remote/vfio-user-obj.c +F: include/hw/remote/iommu.h +F: hw/remote/iommu.c EBPF: M: Jason Wang <[email protected]>
hw/remote/meson.build +1 -0
@@ -6,6 +6,7 @@ remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('message.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('remote-obj.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('proxy.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('iohub.c')) +remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('iommu.c')) remote_ss.add(when: 'CONFIG_VFIO_USER_SERVER', if_true: files('vfio-user-obj.c')) remote_ss.add(when: 'CONFIG_VFIO_USER_SERVER', if_true: vfiouser)
Signature · 1 line
2.20.1
PATCH v4 7/14 vfio-user: run vfio-user context
In reply to Jagannathan RamanSetup a handler to run vfio-user context. The context is driven by messages to the file descriptor associated with it - get the fd for the context and hook up the handler with it
--- hw/remote/vfio-user-obj.c | 80 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 1 deletion(-)
hw/remote/vfio-user-obj.c +79 -1
@@ -42,10 +42,12 @@#include "qapi/error.h" #include "qapi/qapi-visit-sockets.h" #include "qemu/notify.h" +#include "qemu/thread.h" #include "sysemu/sysemu.h" #include "libvfio-user.h" #include "hw/qdev-core.h" #include "hw/pci/pci.h" +#include "qemu/timer.h" #define TYPE_VFU_OBJECT "x-vfio-user-server"OBJECT_DECLARE_TYPE (VfuObject, VfuObjectClass,VFU_OBJECT )
@@ -73,6 +75,8 @@ struct VfuObject {vfu_ctx_t *vfu_ctx;PCIDevice *pci_dev; + +int vfu_poll_fd; };static void vfu_object_init_ctx (VfuObject *o,Error **errp);
@@ -114,6 +118,62 @@ static void vfu_object_set_device(Object *obj, const char *str, Error **errp)vfu_object_init_ctx (o, errp); } +static void vfu_object_ctx_run (void *opaque) +{ +VfuObject *o = opaque; +int ret =-1 ; + +while (ret !=0 ) { + ret =vfu_run_ctx (o->vfu_ctx); +if (ret <0 ) { +if (errno ==EINTR ) { +continue ; + }else if (errno ==ENOTCONN ) { +qemu_set_fd_handler (o->vfu_poll_fd,NULL ,NULL ,NULL ); + o->vfu_poll_fd =-1 ; +object_unparent (OBJECT (o)); +break ; + }else { +error_setg (&error_abort,"vfu: Failed to run device %s - %s" , + o->device,strerror (errno)); +break ; + } + } + } +} + +static void vfu_object_attach_ctx (void *opaque) +{ +VfuObject *o = opaque; +GPollFD pfds[1 ]; +int ret; + +qemu_set_fd_handler (o->vfu_poll_fd,NULL ,NULL ,NULL ); + + pfds[0 ].fd = o->vfu_poll_fd; + pfds[0 ].events =G_IO_IN |G_IO_HUP |G_IO_ERR ; + +retry_attach: + ret =vfu_attach_ctx (o->vfu_ctx); +if (ret <0 && (errno ==EAGAIN || errno ==EWOULDBLOCK )) { +qemu_poll_ns (pfds,1 ,500 * (int64_t )SCALE_MS ); + goto retry_attach; + }else if (ret <0 ) { +error_setg (&error_abort, +"vfu: Failed to attach device %s to context - %s" , + o->device,strerror (errno)); +return ; + } + + o->vfu_poll_fd =vfu_get_poll_fd (o->vfu_ctx); +if (o->vfu_poll_fd <0 ) { +error_setg (&error_abort,"vfu: Failed to get poll fd %s" , o->device); +return ; + } + +qemu_set_fd_handler (o->vfu_poll_fd, vfu_object_ctx_run,NULL , o); +} +/* * TYPE_VFU_OBJECT depends on the availability of the 'socket' and 'device' * properties. It also depends on devices instantiated in QEMU. These
@@ -151,7 +211,8 @@ static void vfu_object_init_ctx(VfuObject *o, Error **errp)return ; } - o->vfu_ctx =vfu_create_ctx (VFU_TRANS_SOCK , o->socket->u.q_unix.path,0 , + o->vfu_ctx =vfu_create_ctx (VFU_TRANS_SOCK , o->socket->u.q_unix.path, +LIBVFIO_USER_FLAG_ATTACH_NB , o,VFU_DEV_TYPE_PCI );if (o->vfu_ctx ==NULL ) {error_setg (errp,"vfu: Failed to create context - %s" ,strerror (errno));
@@ -183,6 +244,21 @@ static void vfu_object_init_ctx(VfuObject *o, Error **errp) goto fail; } + ret =vfu_realize_ctx (o->vfu_ctx); +if (ret <0 ) { +error_setg (errp,"vfu: Failed to realize device %s- %s" , + o->device,strerror (errno)); + goto fail; + } + + o->vfu_poll_fd =vfu_get_poll_fd (o->vfu_ctx); +if (o->vfu_poll_fd <0 ) { +error_setg (errp,"vfu: Failed to get poll fd %s" , o->device); + goto fail; + } + +qemu_set_fd_handler (o->vfu_poll_fd, vfu_object_attach_ctx,NULL , o); +return ; fail:
@@ -208,6 +284,8 @@ static void vfu_object_init(Object *obj) TYPE_VFU_OBJECT, TYPE_REMOTE_MACHINE);return ; } + + o->vfu_poll_fd =-1 ; }static void vfu_object_finalize (Object *obj)
Signature · 1 line
2.20.1
PATCH v4 12/14 vfio-user: handle device interrupts
In reply to Jagannathan RamanForward remote device's interrupts to the guest
--- include/hw/pci/pci.h | 6 ++++ include/hw/remote/iohub.h | 1 + hw/pci/msi.c | 13 ++++++- hw/pci/msix.c | 12 ++++++- hw/remote/iohub.c | 7 ++++ hw/remote/vfio-user-obj.c | 74 +++++++++++++++++++++++++++++++++++++++ hw/remote/trace-events | 1 + 7 files changed, 112 insertions(+), 2 deletions(-)
include/hw/pci/pci.h +6 -0
@@ -128,6 +128,8 @@ typedef uint32_t PCIConfigReadFunc(PCIDevice *pci_dev,typedef void PCIMapIORegionFunc (PCIDevice *pci_dev,int region_num,pcibus_t addr,pcibus_t size,int type);typedef void PCIUnregisterFunc (PCIDevice *pci_dev); +typedef void PCIMSINotify (PCIDevice *pci_dev,unsigned vector); +typedef void PCIMSIxNotify (PCIDevice *pci_dev,unsigned vector); typedef struct PCIIORegion { pcibus_t addr; /* current PCI mapping address. -1 means not mapped */
@@ -321,6 +323,10 @@ struct PCIDevice {/* Space to store MSIX table & pending bit array */ uint8_t *msix_table;uint8_t *msix_pba; + +PCIMSINotify *msi_notify; +PCIMSIxNotify *msix_notify; +/* MemoryRegion container for msix exclusive BAR setup */ MemoryRegion msix_exclusive_bar;/* Memory Regions for MSIX table and pending bit entries. */
include/hw/remote/iohub.h +1 -0
@@ -30,6 +30,7 @@ typedef struct RemoteIOHubState {unsigned int irq_level[REMOTE_IOHUB_NB_PIRQS ];ResampleToken token[REMOTE_IOHUB_NB_PIRQS ];QemuMutex irq_level_lock[REMOTE_IOHUB_NB_PIRQS ]; +void (*intx_notify)(int pirq,unsigned vector); } RemoteIOHubState;int remote_iohub_map_irq (PCIDevice *pci_dev,int intx);
hw/pci/msi.c +12 -1
@@ -51,6 +51,8 @@*/ bool msi_nonbroken; +static void pci_msi_notify (PCIDevice *dev,unsigned int vector); +/* If we get rid of cap allocator, we won't need this. */ static inline uint8_t msi_cap_sizeof (uint16_t flags) {
@@ -225,6 +227,8 @@ int msi_init(struct PCIDevice *dev, uint8_t offset, dev->msi_cap = config_offset; dev->cap_present |=QEMU_PCI_CAP_MSI ; + dev->msi_notify = pci_msi_notify; +pci_set_word (dev->config +msi_flags_off (dev), flags);pci_set_word (dev->wmask +msi_flags_off (dev),PCI_MSI_FLAGS_QSIZE |PCI_MSI_FLAGS_ENABLE );
@@ -307,7 +311,7 @@ bool msi_is_masked(const PCIDevice *dev, unsigned int vector)return mask & (1U << vector); } -void msi_notify (PCIDevice *dev,unsigned int vector) +static void pci_msi_notify (PCIDevice *dev,unsigned int vector) {uint16_t flags =pci_get_word (dev->config +msi_flags_off (dev));bool msi64bit = flags &PCI_MSI_FLAGS_64BIT ;
@@ -332,6 +336,13 @@ void msi_notify(PCIDevice *dev, unsigned int vector)msi_send_message (dev, msg); } +void msi_notify (PCIDevice *dev,unsigned int vector) +{ +if (dev->msi_notify) { + dev->msi_notify (dev, vector); + } +} +void msi_send_message (PCIDevice *dev,MSIMessage msg) {MemTxAttrs attrs = {};
hw/pci/msix.c +11 -1
@@ -31,6 +31,8 @@#define MSIX_ENABLE_MASK (PCI_MSIX_FLAGS_ENABLE >> 8)#define MSIX_MASKALL_MASK (PCI_MSIX_FLAGS_MASKALL >> 8) +static void pci_msix_notify (PCIDevice *dev,unsigned vector); +MSIMessage msix_get_message (PCIDevice *dev,unsigned vector) {uint8_t *table_entry = dev->msix_table + vector *PCI_MSIX_ENTRY_SIZE ;
@@ -334,6 +336,7 @@ int msix_init(struct PCIDevice *dev, unsigned short nentries, dev->msix_table =g_malloc0 (table_size); dev->msix_pba =g_malloc0 (pba_size); dev->msix_entry_used =g_malloc0 (nentries *sizeof *dev->msix_entry_used); + dev->msix_notify = pci_msix_notify;msix_mask_all (dev, nentries);
@@ -485,7 +488,7 @@ int msix_enabled(PCIDevice *dev) }/* Send an MSI-X message */ -void msix_notify (PCIDevice *dev,unsigned vector) +static void pci_msix_notify (PCIDevice *dev,unsigned vector) {MSIMessage msg;
@@ -503,6 +506,13 @@ void msix_notify(PCIDevice *dev, unsigned vector)msi_send_message (dev, msg); } +void msix_notify (PCIDevice *dev,unsigned vector) +{ +if (dev->msix_notify) { + dev->msix_notify (dev, vector); + } +} +void msix_reset (PCIDevice *dev) {if (!msix_present (dev)) {
hw/remote/iohub.c +7 -0
@@ -17,7 +17,9 @@#include "qemu/thread.h" #include "hw/remote/machine.h" #include "hw/remote/iohub.h" +#include "hw/pci/msi.h" #include "qemu/main-loop.h" +#include "trace.h" void remote_iohub_init (RemoteIOHubState *iohub) {
@@ -32,6 +34,8 @@ void remote_iohub_init(RemoteIOHubState *iohub)event_notifier_init_fd (&iohub->irqfds[pirq],-1 );event_notifier_init_fd (&iohub->resamplefds[pirq],-1 ); } + + msi_nonbroken = true; }void remote_iohub_finalize (RemoteIOHubState *iohub)
@@ -62,6 +66,9 @@ void remote_iohub_set_irq(void *opaque, int pirq, int level)QEMU_LOCK_GUARD (&iohub->irq_level_lock[pirq]);if (level) { +if (iohub->intx_notify) { + iohub->intx_notify (pirq,0 ); + }if (++iohub->irq_level[pirq] ==1 ) {event_notifier_set (&iohub->irqfds[pirq]); }
hw/remote/vfio-user-obj.c +74 -0
119 lines not on this page · raw message
hw/remote/trace-events +1 -0
@@ -12,3 +12,4 @@ vfu_dma_unregister(uint64_t gpa) "vfu: unregistering GPA 0x%"PRIx64"" vfu_bar_register(int i, uint64_t addr, uint64_t size) "vfu: BAR %d: addr 0x%"PRIx64" size 0x%"PRIx64"" vfu_bar_rw_enter(const char *op, uint64_t addr) "vfu: %s request for BAR address 0x%"PRIx64"" vfu_bar_rw_exit(const char *op, uint64_t addr) "vfu: Finished %s of BAR address 0x%"PRIx64"" +vfu_interrupt(int pirq) "vfu: sending interrupt to device - PIRQ %d"
Signature · 1 line
2.20.1
PATCH v4 14/14 vfio-user: avocado tests for vfio-user
In reply to Jagannathan RamanAvocado tests for libvfio-user in QEMU - tests startup, hotplug and migration of the server object
--- MAINTAINERS | 1 + tests/avocado/vfio-user.py | 225 +++++++++++++++++++++++++++++++++++++ 2 files changed, 226 insertions(+) create mode 100644 tests/avocado/vfio-user.py
MAINTAINERS +1 -0
@@ -3468,6 +3468,7 @@ F: subprojects/libvfio-user F: hw/remote/vfio-user-obj.c F: include/hw/remote/iommu.h F: hw/remote/iommu.c +F: tests/avocado/vfio-user.py EBPF: M: Jason Wang <[email protected]>
tests/avocado/vfio-user.py new file +225 -0
225 lines not on this page · raw message
Signature · 1 line
2.20.1
PATCH v4 2/14 tests/avocado: Specify target VM argument to helper routines
In reply to Jagannathan RamanSpecify target VM for exec_command andexec_command_and_wait_for_pattern routines
--- tests/avocado/avocado_qemu/__init__.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-)
tests/avocado/avocado_qemu/__init__.py +6 -4
@@ -198,7 +198,7 @@ def wait_for_console_pattern(test, success_message, failure_message=None, """ _console_interaction(test, success_message, failure_message, None, vm=vm) -def exec_command(test, command): +def exec_command(test, command, vm=None): """ Send a command to a console (appending CRLF characters), while logging the content.
@@ -208,10 +208,11 @@ def exec_command(test, command): :param command: the command to send :type command: str """ - _console_interaction(test, None, None, command + '\r') + _console_interaction(test, None, None, command + '\r', vm=vm) def exec_command_and_wait_for_pattern(test, command, - success_message, failure_message=None): + success_message, failure_message=None, + vm=None): """ Send a command to a console (appending CRLF characters), then wait for success_message to appear on the console, while logging the.
@@ -224,7 +225,8 @@ def exec_command_and_wait_for_pattern(test, command, :param success_message: if this message appears, test succeeds :param failure_message: if this message appears, test fails """ - _console_interaction(test, success_message, failure_message, command + '\r') + _console_interaction(test, success_message, failure_message, command + '\r', + vm=vm) class QemuBaseTest(avocado.Test): def _get_unique_tag_val(self, tag_name):
Signature · 1 line
2.20.1
PATCH v4 1/14 configure, meson: override C compiler for cmake
In reply to Jagannathan RamanThe compiler path that cmake gets from meson is corrupted. It results in the following error:
| -- The C compiler identification is unknown | CMake Error at CMakeLists.txt:35 (project): | The CMAKE_C_COMPILER: | /opt/rh/devtoolset-9/root/bin/cc;-m64;-mcx16 | is not a full path to an existing compiler tool.
Explicitly specify the C compiler for cmake to avoid this error
--- configure | 2 ++ 1 file changed, 2 insertions(+)
configure +2 -0
@@ -3919,6 +3919,8 @@ if test "$skip_meson" = no; then echo "cpp_args = [$(meson_quote $CXXFLAGS $EXTRA_CXXFLAGS)]" >> $cross echo "c_link_args = [$(meson_quote $CFLAGS $LDFLAGS $EXTRA_CFLAGS $EXTRA_LDFLAGS)]" >> $cross echo "cpp_link_args = [$(meson_quote $CXXFLAGS $LDFLAGS $EXTRA_CXXFLAGS $EXTRA_LDFLAGS)]" >> $cross + echo "[cmake]" >> $cross + echo "CMAKE_C_COMPILER = [$(meson_quote $cc $CPU_CFLAGS)]" >> $cross echo "[binaries]" >> $cross echo "c = [$(meson_quote $cc $CPU_CFLAGS)]" >> $cross test -n "$cxx" && echo "cpp = [$(meson_quote $cxx $CPU_CFLAGS)]" >> $cross
Signature · 1 line
2.20.1
PATCH v4 4/14 vfio-user: define vfio-user-server object
In reply to Jagannathan RamanDefine vfio-user object which is remote process server for QEMU. Setup object initialization functions and properties necessary to instantiate the object
--- qapi/qom.json | 20 ++++- hw/remote/vfio-user-obj.c | 175 ++++++++++++++++++++++++++++++++++++++ MAINTAINERS | 1 + hw/remote/meson.build | 1 + hw/remote/trace-events | 3 + 5 files changed, 198 insertions(+), 2 deletions(-) create mode 100644 hw/remote/vfio-user-obj.c
qapi/qom.json +18 -2
@@ -703,6 +703,20 @@ { 'struct': 'RemoteObjectProperties', 'data': { 'fd': 'str', 'devid': 'str' } } +## +# @VfioUserServerProperties: +# +# Properties for x-vfio-user-server objects. +# +# @socket: socket to be used by the libvfiouser library +# +# @device: the id of the device to be emulated at the server +# +# Since: 6.2 +## +{ 'struct': 'VfioUserServerProperties', + 'data': { 'socket': 'SocketAddress', 'device': 'str' } } + ## # @RngProperties: #
@@ -837,7 +851,8 @@ 'tls-creds-psk', 'tls-creds-x509', 'tls-cipher-suites', - { 'name': 'x-remote-object', 'features': [ 'unstable' ] } + { 'name': 'x-remote-object', 'features': [ 'unstable' ] }, + { 'name': 'x-vfio-user-server', 'features': [ 'unstable' ] } ] } ##
@@ -900,7 +915,8 @@ 'tls-creds-psk': 'TlsCredsPskProperties', 'tls-creds-x509': 'TlsCredsX509Properties', 'tls-cipher-suites': 'TlsCredsProperties', - 'x-remote-object': 'RemoteObjectProperties' + 'x-remote-object': 'RemoteObjectProperties', + 'x-vfio-user-server': 'VfioUserServerProperties' } } ##
hw/remote/vfio-user-obj.c new file +175 -0
175 lines not on this page · raw message
MAINTAINERS +1 -0
@@ -3465,6 +3465,7 @@ F: include/hw/remote/proxy-memory-listener.h F: hw/remote/iohub.c F: include/hw/remote/iohub.h F: subprojects/libvfio-user +F: hw/remote/vfio-user-obj.c EBPF: M: Jason Wang <[email protected]>
hw/remote/meson.build +1 -0
@@ -6,6 +6,7 @@ remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('message.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('remote-obj.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('proxy.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('iohub.c')) +remote_ss.add(when: 'CONFIG_VFIO_USER_SERVER', if_true: files('vfio-user-obj.c')) remote_ss.add(when: 'CONFIG_VFIO_USER_SERVER', if_true: vfiouser)
hw/remote/trace-events +3 -0
@@ -2,3 +2,6 @@ mpqemu_send_io_error(int cmd, int size, int nfds) "send command %d size %d, %d file descriptors to remote process" mpqemu_recv_io_error(int cmd, int size, int nfds) "failed to receive %d size %d, %d file descriptors to remote process" + +# vfio-user-obj.c +vfu_prop(const char *prop, const char *val) "vfu: setting %s as %s"
Signature · 1 line
2.20.1
PATCH v4 10/14 vfio-user: handle PCI BAR accesses
In reply to Jagannathan RamanDetermine the BARs used by the PCI device and register handlers to manage the access to the same.
--- hw/remote/vfio-user-obj.c | 90 +++++++++++++++++++++++++++++++++++++++ hw/remote/trace-events | 3 ++ 2 files changed, 93 insertions(+)
hw/remote/vfio-user-obj.c +90 -0
@@ -249,6 +249,94 @@ static void dma_unregister(vfu_ctx_t *vfu_ctx, vfu_dma_info_t *info)trace_vfu_dma_unregister ((uint64_t )info->iova.iov_base); } +static ssize_t vfu_object_bar_rw (PCIDevice *pci_dev,hwaddr addr,size_t count, +char *const buf,const bool is_write, +bool is_io) +{ +AddressSpace *as =NULL ; +MemTxResult res; + +if (is_io) { + as = &address_space_io; + }else { + as =pci_device_iommu_address_space (pci_dev); + } + +trace_vfu_bar_rw_enter (is_write ?"Write" :"Read" , (uint64_t )addr); + + res =address_space_rw (as, addr,MEMTXATTRS_UNSPECIFIED , (void *)buf, + (hwaddr )count, is_write); +if (res !=MEMTX_OK ) { +warn_report ("vfu: failed to %s 0x%" PRIx64"" , + is_write ?"write to" :"read from" , + addr); +return -1 ; + } + +trace_vfu_bar_rw_exit (is_write ?"Write" :"Read" , (uint64_t )addr); + +return count; +} + +/** +* VFU_OBJECT_BAR_HANDLER - macro for defining handlers for PCI BARs. +* +* To create handler for BAR number 2, VFU_OBJECT_BAR_HANDLER(2) would +* define vfu_object_bar2_handler +*/ +#define VFU_OBJECT_BAR_HANDLER (BAR_NO ) \ + static ssize_t vfu_object_bar##BAR_NO##_handler(vfu_ctx_t *vfu_ctx, \ + char * const buf, size_t count, \ + loff_t offset, const bool is_write) \ + { \ + VfuObject *o = vfu_get_private(vfu_ctx); \ + PCIDevice *pci_dev = o->pci_dev; \ + hwaddr addr = (hwaddr)(pci_get_bar_addr(pci_dev, BAR_NO) + offset); \ + bool is_io = !!(pci_dev->io_regions[BAR_NO].type & \ + PCI_BASE_ADDRESS_SPACE); \ + \ + return vfu_object_bar_rw(pci_dev, addr, count, buf, is_write, is_io); \ + } \ + +VFU_OBJECT_BAR_HANDLER(0) +VFU_OBJECT_BAR_HANDLER(1) +VFU_OBJECT_BAR_HANDLER(2) +VFU_OBJECT_BAR_HANDLER(3) +VFU_OBJECT_BAR_HANDLER(4) +VFU_OBJECT_BAR_HANDLER(5) + +static vfu_region_access_cb_t *vfu_object_bar_handlers[PCI_NUM_REGIONS ] = { + &vfu_object_bar0_handler, + &vfu_object_bar1_handler, + &vfu_object_bar2_handler, + &vfu_object_bar3_handler, + &vfu_object_bar4_handler, + &vfu_object_bar5_handler, +}; + +/** +* vfu_object_register_bars - Identify active BAR regions of pdev and setup +* callbacks to handle read/write accesses +*/ +static void vfu_object_register_bars (vfu_ctx_t *vfu_ctx,PCIDevice *pdev) +{ +int i; + +for (i =0 ; i <PCI_NUM_REGIONS ; i++) { +if (!pdev->io_regions[i].size) { +continue ; + } + +vfu_setup_region (vfu_ctx,VFU_PCI_DEV_BAR0_REGION_IDX + i, + (size_t )pdev->io_regions[i].size, + vfu_object_bar_handlers[i], +VFU_REGION_FLAG_RW ,NULL ,0 ,-1 ,0 ); + +trace_vfu_bar_register (i, pdev->io_regions[i].addr, + pdev->io_regions[i].size); + } +} +/* * TYPE_VFU_OBJECT depends on the availability of the 'socket' and 'device' * properties. It also depends on devices instantiated in QEMU. These
@@ -337,6 +425,8 @@ static void vfu_object_init_ctx(VfuObject *o, Error **errp) goto fail; } +vfu_object_register_bars (o->vfu_ctx, o->pci_dev); + ret =vfu_realize_ctx (o->vfu_ctx);if (ret <0 ) { error_setg(errp, "vfu: Failed to realize device %s- %s",
hw/remote/trace-events +3 -0
@@ -9,3 +9,6 @@ vfu_cfg_read(uint32_t offset, uint32_t val) "vfu: cfg: 0x%u -> 0x%x" vfu_cfg_write(uint32_t offset, uint32_t val) "vfu: cfg: 0x%u <- 0x%x" vfu_dma_register(uint64_t gpa, size_t len) "vfu: registering GPA 0x%"PRIx64", %zu bytes" vfu_dma_unregister(uint64_t gpa) "vfu: unregistering GPA 0x%"PRIx64"" +vfu_bar_register(int i, uint64_t addr, uint64_t size) "vfu: BAR %d: addr 0x%"PRIx64" size 0x%"PRIx64"" +vfu_bar_rw_enter(const char *op, uint64_t addr) "vfu: %s request for BAR address 0x%"PRIx64"" +vfu_bar_rw_exit(const char *op, uint64_t addr) "vfu: Finished %s of BAR address 0x%"PRIx64""
Signature · 1 line
2.20.1
PATCH v4 6/14 vfio-user: find and init PCI device
In reply to Jagannathan RamanFind the PCI device with specified id. Initialize the device context with the QEMU PCI device
--- hw/remote/vfio-user-obj.c | 41 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+)
hw/remote/vfio-user-obj.c +41 -0
@@ -44,6 +44,8 @@#include "qemu/notify.h" #include "sysemu/sysemu.h" #include "libvfio-user.h" +#include "hw/qdev-core.h" +#include "hw/pci/pci.h" #define TYPE_VFU_OBJECT "x-vfio-user-server"OBJECT_DECLARE_TYPE (VfuObject, VfuObjectClass,VFU_OBJECT )
@@ -69,6 +71,8 @@ struct VfuObject {Notifier machine_done;vfu_ctx_t *vfu_ctx; + +PCIDevice *pci_dev; };static void vfu_object_init_ctx (VfuObject *o,Error **errp);
@@ -133,6 +137,9 @@ static void vfu_object_machine_done(Notifier *notifier, void *data)static void vfu_object_init_ctx (VfuObject *o,Error **errp) {ERRP_GUARD (); +DeviceState *dev =NULL ; +vfu_pci_type_t pci_type =VFU_PCI_TYPE_CONVENTIONAL ; +int ret;if (o->vfu_ctx || !o->socket || !o->device || !phase_check (PHASE_MACHINE_READY )) {
@@ -150,6 +157,38 @@ static void vfu_object_init_ctx(VfuObject *o, Error **errp)error_setg (errp,"vfu: Failed to create context - %s" ,strerror (errno));return ; } + + dev =qdev_find_recursive (sysbus_get_default (), o->device); +if (dev ==NULL ) { +error_setg (errp,"vfu: Device %s not found" , o->device); + goto fail; + } + +if (!object_dynamic_cast (OBJECT (dev),TYPE_PCI_DEVICE )) { +error_setg (errp,"vfu: %s not a PCI device" , o->device); + goto fail; + } + + o->pci_dev =PCI_DEVICE (dev); + +if (pci_is_express (o->pci_dev)) { + pci_type =VFU_PCI_TYPE_EXPRESS ; + } + + ret =vfu_pci_init (o->vfu_ctx, pci_type,PCI_HEADER_TYPE_NORMAL ,0 ); +if (ret <0 ) { +error_setg (errp, +"vfu: Failed to attach PCI device %s to context - %s" , + o->device,strerror (errno)); + goto fail; + } + +return ; + +fail: +vfu_destroy_ctx (o->vfu_ctx); + o->vfu_ctx =NULL ; + o->pci_dev =NULL ; }static void vfu_object_init (Object *obj)
@@ -190,6 +229,8 @@ static void vfu_object_finalize(Object *obj) o->device =NULL ; + o->pci_dev =NULL ; +if (!k->nr_devs && !k->daemon) {qemu_system_shutdown_request (SHUTDOWN_CAUSE_GUEST_SHUTDOWN ); }
Signature · 1 line
2.20.1
v4 cover vfio-user server in QEMU
Based-on: <[email protected]>
Hi,
Thank you very much for reviewing the patches in the previous revision!
We have addressed your comments. We added the support for MSI/x and IOMMU in this series.
Please share your feedback for the latest series - appreciate your time.
Please the list below for changes since last revision:
[PATCH v4 02/14] tests/avocado: Specify target VM argument to helper routines
- New in this series
[PATCH v4 03/14] vfio-user: build library
- Added VFIO_USER_SERVER Kconfig option and vfio_user_server option to the configure script. VFIO_USER_SERVER depends on MULTIPROCESS
- Reverted to enabling multiprocess by default. As such, tagging distros in the multiprocess acceptance test for multiprocess is not necessary - reverted them.
[PATCH v4 04/14] vfio-user: define vfio-user-server object
- Using qapi_free_SocketAddress to free SocketAddress
- Changed object name to x-vfio-user-server
- Removed max_devs class variable. This is not needed anymore as we added IOMMU support in this series
[PATCH v4 05/14] vfio-user: instantiate vfio-user context
- Redesigned init code to abort QEMU during startup upon error, whereas synchronously report the error and continue running QEMU during hotplug
- Added qemu_remove_machine_init_done_notifier() in object finalize
- Removed NULL initialization of o->vfu_ctx
[PATCH v4 07/14] vfio-user: run vfio-user context
- Addressed comment about hot plug from this patch, in patch 4
- poll for fd to become readable instead of busy spinning alone
[PATCH v4 11/14] vfio-user: IOMMU support for remote device
- New in this series
[PATCH v4 12/14] handle device interrupts
- Added support for MSI/MSI-x interrupts
- refactored INTx handling code to leverage the hash table used by the MSI code
- Added a hash table to lookup vfio-user context by PCI device function index
[PATCH v4 13/14] vfio-user: register handlers to facilitate migration
- Compute the size of the migration data when registering for migration BAR. This alleviates the need to hard code a sufficiently large value for the migration BAR.
[PATCH v4 14/14] vfio-user: avocado tests for vfio-user
- Corrected the object’s name in the test
- Added tests for hotplug and migration cases
Thank you very much!
Jagannathan Raman (14):
configure, meson: override C compiler for cmake tests/avocado: Specify target VM argument to helper routines
vfio-user: build library
vfio-user: define vfio-user-server object
vfio-user: instantiate vfio-user context
vfio-user: find and init PCI device
vfio-user: run vfio-user context
vfio-user: handle PCI config space accesses
vfio-user: handle DMA mappings
vfio-user: handle PCI BAR accesses
vfio-user: IOMMU support for remote device
vfio-user: handle device interrupts
vfio-user: register handlers to facilitate migration
vfio-user: avocado tests for vfio-user
configure | 21 +- meson.build | 44 +- qapi/qom.json | 20 +- include/hw/pci/pci.h | 8 + include/hw/remote/iohub.h | 1 + include/hw/remote/iommu.h | 24 + include/migration/vmstate.h | 2 + migration/savevm.h | 2 + hw/pci/msi.c | 13 +- hw/pci/msix.c | 12 +- hw/pci/pci.c | 2 +- hw/remote/iohub.c | 7 + hw/remote/iommu.c | 117 +++ hw/remote/machine.c | 5 + hw/remote/vfio-user-obj.c | 957 +++++++++++++++++++++ migration/savevm.c | 73 ++ migration/vmstate.c | 19 + .gitlab-ci.d/buildtest.yml | 2 + .gitmodules | 3 + Kconfig.host | 4 + MAINTAINERS | 5 + hw/remote/Kconfig | 4 + hw/remote/meson.build | 4 + hw/remote/trace-events | 11 + meson_options.txt | 2 + subprojects/libvfio-user | 1 + tests/avocado/avocado_qemu/__init__.py | 10 +- tests/avocado/vfio-user.py | 225 +++++ tests/docker/dockerfiles/centos8.docker | 2 + tests/docker/dockerfiles/ubuntu2004.docker | 2 + 30 files changed, 1591 insertions(+), 11 deletions(-) create mode 100644 include/hw/remote/iommu.h create mode 100644 hw/remote/iommu.c create mode 100644 hw/remote/vfio-user-obj.c create mode 160000 subprojects/libvfio-user create mode 100644 tests/avocado/vfio-user.py
Signature · 1 line
2.20.1
Re: [PATCH v4 02/14] tests/avocado: Specify target VM argument to helper routines
In reply to Jagannathan RamanQuoted 9 lines · Jagannathan Raman
Specify target VM for
exec_commandandexec_command_and_wait_for_patternroutinesSigned-off-by Elena Ufimtseva <[email protected]>Signed-off-by John G Johnson <[email protected]>Signed-off-by Jagannathan Raman <[email protected]>--- tests/avocado/avocado_qemu/__init__.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-)
PATCH v4 3/14 vfio-user: build library
In reply to Jagannathan Ramanadd the libvfio-user library as a submodule. build it as a cmake subproject.
--- configure | 19 +++++++++- meson.build | 44 +++++++++++++++++++++- .gitlab-ci.d/buildtest.yml | 2 + .gitmodules | 3 ++ Kconfig.host | 4 ++ MAINTAINERS | 1 + hw/remote/Kconfig | 4 ++ hw/remote/meson.build | 2 + meson_options.txt | 2 + subprojects/libvfio-user | 1 + tests/docker/dockerfiles/centos8.docker | 2 + tests/docker/dockerfiles/ubuntu2004.docker | 2 + 12 files changed, 84 insertions(+), 2 deletions(-) create mode 160000 subprojects/libvfio-user
configure +18 -1
@@ -366,6 +366,7 @@ ninja="" gio="$default_feature" skip_meson=no slirp_smbd="$default_feature" +vfio_user_server="disabled" # The following Meson options are handled manually (still they # are included in the automatically generated help message)
@@ -1190,6 +1191,10 @@ for opt do ;; --disable-blobs) meson_option_parse --disable-install-blobs "" ;; + --enable-vfio-user-server) vfio_user_server="enabled" + ;; + --disable-vfio-user-server) vfio_user_server="disabled" + ;; --enable-tcmalloc) meson_option_parse --enable-malloc=tcmalloc tcmalloc ;; --enable-jemalloc) meson_option_parse --enable-malloc=jemalloc jemalloc
@@ -1470,6 +1475,7 @@ cat << EOF rng-none dummy RNG, avoid using /dev/(u)random and getrandom() gio libgio support slirp-smbd use smbd (at path --smbd=*) in slirp networking + vfio-user-server vfio-user server support NOTE: The object files are built at the place where configure is launched EOF
@@ -3250,6 +3256,17 @@ but not implemented on your system" fi fi +########################################## +# check for vfio_user_server + +case "$vfio_user_server" in + auto | enabled ) + if test "$git_submodules_action" != "ignore"; then + git_submodules="${git_submodules} subprojects/libvfio-user" + fi + ;; +esac + ########################################## # End of CC checks # After here, no more $cc or $ld runs
@@ -3998,7 +4015,7 @@ if test "$skip_meson" = no; then -Db_pie=$(if test "$pie" = yes; then echo true; else echo false; fi) \ -Db_coverage=$(if test "$gcov" = yes; then echo true; else echo false; fi) \ -Db_lto=$lto -Dcfi=$cfi -Dtcg=$tcg -Dxen=$xen \ - -Dcapstone=$capstone -Dfdt=$fdt -Dslirp=$slirp \ + -Dcapstone=$capstone -Dfdt=$fdt -Dslirp=$slirp -Dvfio_user_server=$vfio_user_server \ $(test -n "${LIB_FUZZING_ENGINE+xxx}" && echo "-Dfuzzing_engine=$LIB_FUZZING_ENGINE") \ $(if test "$default_feature" = no; then echo "-Dauto_features=disabled"; fi) \ "$@" $cross_arg "$PWD" "$source_path"
meson.build +43 -1
68 lines not on this page · raw message
.gitlab-ci.d/buildtest.yml +2 -0
@@ -42,6 +42,7 @@ build-system-ubuntu: variables: IMAGE: ubuntu2004 CONFIGURE_ARGS: --enable-docs --enable-fdt=system --enable-slirp=system + --enable-vfio-user-server TARGETS: aarch64-softmmu alpha-softmmu cris-softmmu hppa-softmmu microblazeel-softmmu mips64el-softmmu MAKE_CHECK_ARGS: check-build
@@ -142,6 +143,7 @@ build-system-centos: IMAGE: centos8 CONFIGURE_ARGS: --disable-nettle --enable-gcrypt --enable-fdt=system --enable-modules --enable-trace-backends=dtrace + --enable-vfio-user-server TARGETS: ppc64-softmmu or1k-softmmu s390x-softmmu x86_64-softmmu rx-softmmu sh4-softmmu nios2-softmmu MAKE_CHECK_ARGS: check-build
.gitmodules +3 -0
@@ -64,3 +64,6 @@ [submodule "roms/vbootrom"] path = roms/vbootrom url = https://gitlab.com/qemu-project/vbootrom.git +[submodule "subprojects/libvfio-user"] + path = subprojects/libvfio-user + url = https://github.com/nutanix/libvfio-user.git
Kconfig.host +4 -0
@@ -45,3 +45,7 @@ config MULTIPROCESS_ALLOWED config FUZZ bool select SPARSE_MEM + +config VFIO_USER_SERVER_ALLOWED + bool + imply VFIO_USER_SERVER
MAINTAINERS +1 -0
@@ -3464,6 +3464,7 @@ F: hw/remote/proxy-memory-listener.c F: include/hw/remote/proxy-memory-listener.h F: hw/remote/iohub.c F: include/hw/remote/iohub.h +F: subprojects/libvfio-user EBPF: M: Jason Wang <[email protected]>
hw/remote/Kconfig +4 -0
@@ -2,3 +2,7 @@ config MULTIPROCESS bool depends on PCI && PCI_EXPRESS && KVM select REMOTE_PCIHOST + +config VFIO_USER_SERVER + bool + depends on MULTIPROCESS
hw/remote/meson.build +2 -0
@@ -7,6 +7,8 @@ remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('remote-obj.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('proxy.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('iohub.c')) +remote_ss.add(when: 'CONFIG_VFIO_USER_SERVER', if_true: vfiouser) + specific_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('memory.c')) specific_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('proxy-memory-listener.c'))
meson_options.txt +2 -0
@@ -66,6 +66,8 @@ option('cfi_debug', type: 'boolean', value: 'false', description: 'Verbose errors in case of CFI violation') option('multiprocess', type: 'feature', value: 'auto', description: 'Out of process device emulation support') +option('vfio_user_server', type: 'feature', value: 'auto', + description: 'vfio-user server support') option('attr', type : 'feature', value : 'auto', description: 'attr/xattr support')
subprojects/libvfio-user new file +1 -0
@@ -0,0 +1 @@ +Subproject commit 7056525da5399d00831e90bed4aedb4b8442c9b2
tests/docker/dockerfiles/centos8.docker +2 -0
@@ -12,6 +12,7 @@ ENV PACKAGES \ capstone-devel \ ccache \ clang \ + cmake \ ctags \ cyrus-sasl-devel \ daxctl-devel \
@@ -32,6 +33,7 @@ ENV PACKAGES \ gtk3-devel \ hostname \ jemalloc-devel \ + json-c-devel \ libaio-devel \ libasan \ libattr-devel \
tests/docker/dockerfiles/ubuntu2004.docker +2 -0
@@ -6,6 +6,7 @@ ENV PACKAGES \ ca-certificates \ ccache \ clang \ + cmake \ dbus \ debianutils \ diffutils \
@@ -44,6 +45,7 @@ ENV PACKAGES \ libiscsi-dev \ libjemalloc-dev \ libjpeg-turbo8-dev \ + libjson-c-dev \ liblttng-ust-dev \ liblzo2-dev \ libncursesw5-dev \
Signature · 1 line
2.20.1
PATCH v4 9/14 vfio-user: handle DMA mappings
In reply to Jagannathan RamanDefine and register callbacks to manage the RAM regions used for device DMA
--- hw/remote/vfio-user-obj.c | 48 +++++++++++++++++++++++++++++++++++++++ hw/remote/trace-events | 2 ++ 2 files changed, 50 insertions(+)
hw/remote/vfio-user-obj.c +48 -0
@@ -208,6 +208,47 @@ static ssize_t vfu_object_cfg_access(vfu_ctx_t *vfu_ctx, char * const buf,return count; } +static void dma_register (vfu_ctx_t *vfu_ctx,vfu_dma_info_t *info) +{ +MemoryRegion *subregion =NULL ; + g_autofree char *name = NULL; +static unsigned int suffix; +struct iovec *iov = &info->iova; + +if (!info->vaddr) { +return ; + } + + name =g_strdup_printf ("remote-mem-%u" , suffix++); + + subregion =g_new0 (MemoryRegion,1 ); + +memory_region_init_ram_ptr (subregion,NULL , name, + iov->iov_len, info->vaddr); + +memory_region_add_subregion (get_system_memory (), (hwaddr )iov->iov_base, + subregion); + +trace_vfu_dma_register ((uint64_t )iov->iov_base, iov->iov_len); +} + +static void dma_unregister (vfu_ctx_t *vfu_ctx,vfu_dma_info_t *info) +{ +MemoryRegion *mr =NULL ; +ram_addr_t offset; + + mr =memory_region_from_host (info->vaddr, &offset); +if (!mr) { +return ; + } + +memory_region_del_subregion (get_system_memory (), mr); + +object_unparent ((OBJECT (mr))); + +trace_vfu_dma_unregister ((uint64_t )info->iova.iov_base); +} +/* * TYPE_VFU_OBJECT depends on the availability of the 'socket' and 'device' * properties. It also depends on devices instantiated in QEMU. These
@@ -289,6 +330,13 @@ static void vfu_object_init_ctx(VfuObject *o, Error **errp) goto fail; } + ret =vfu_setup_device_dma (o->vfu_ctx, &dma_register, &dma_unregister); +if (ret <0 ) { +error_setg (errp,"vfu: Failed to setup DMA handlers for %s" , + o->device); + goto fail; + } + ret =vfu_realize_ctx (o->vfu_ctx);if (ret <0 ) { error_setg(errp, "vfu: Failed to realize device %s- %s",
hw/remote/trace-events +2 -0
@@ -7,3 +7,5 @@ mpqemu_recv_io_error(int cmd, int size, int nfds) "failed to receive %d size %d, vfu_prop(const char *prop, const char *val) "vfu: setting %s as %s" vfu_cfg_read(uint32_t offset, uint32_t val) "vfu: cfg: 0x%u -> 0x%x" vfu_cfg_write(uint32_t offset, uint32_t val) "vfu: cfg: 0x%u <- 0x%x" +vfu_dma_register(uint64_t gpa, size_t len) "vfu: registering GPA 0x%"PRIx64", %zu bytes" +vfu_dma_unregister(uint64_t gpa) "vfu: unregistering GPA 0x%"PRIx64""
Signature · 1 line
2.20.1
Re: [PATCH v4 02/14] tests/avocado: Specify target VM argument to helper routines
In reply to Jagannathan RamanQuoted 15 lines · Jagannathan Raman
Specify target VM for
exec_commandandexec_command_and_wait_for_patternroutinesSigned-off-by Elena Ufimtseva <[email protected]>Signed-off-by John G Johnson <[email protected]>Signed-off-by Jagannathan Raman <[email protected]>--- tests/avocado/avocado_qemu/__init__.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-)tests/avocado/avocado_qemu/__init__.py +0
-0
tests/avocado/avocado_qemu/__init__.py def wait_for_console_pattern(test, success_message, failure_message=None, · line 201
Quoted 5 lines
tests/avocado/avocado_qemu/__init__.py +1
-1@@ -198,7 +198,7 @@ def wait_for_console_pattern(test, success_message, failure_message=None, """ _console_interaction(test, success_message, failure_message, None, vm=vm) -def exec_command(test, command): +def exec_command(test, command, vm=None):
nitpick: if possible, it would be nice to update the docstring, by adding this new argument.
Quoted 4 lines
tests/avocado/avocado_qemu/__init__.py +0
-0""" Send a command to a console (appending CRLF characters), while logging the content.
tests/avocado/avocado_qemu/__init__.py def exec_command(test, command): · line 215
Quoted 10 lines
tests/avocado/avocado_qemu/__init__.py +3
-2@@ -208,10 +208,11 @@ def exec_command(test, command): :param command: the command to send :type command: str """ - _console_interaction(test, None, None, command + '\r') + _console_interaction(test, None, None, command + '\r', vm=vm) def exec_command_and_wait_for_pattern(test, command, - success_message, failure_message=None): + success_message, failure_message=None, + vm=None):
Same here.
Other than that, lgtm.
--
Beraldo
Re: [PATCH v4 04/14] vfio-user: define vfio-user-server object
In reply to Jagannathan RamanQuoted 8 lines · Jagannathan Raman
qapi/qom.json +0
-0@@ -703,6 +703,20 @@ { 'struct': 'RemoteObjectProperties', 'data': { 'fd': 'str', 'devid': 'str' } }
qapi/qom.json line 715
Quoted 10 lines
qapi/qom.json +10
-0+## +# @VfioUserServerProperties: +# +# Properties for x-vfio-user-server objects. +# +# @socket: socket to be used by the libvfiouser library +# +# @device: the id of the device to be emulated at the server +# +# Since: 6.2
6.2 has been released so the version number needs to be incremented.
qapi/qom.json line 721
Quoted 6 lines
qapi/qom.json +6
-0+struct VfuObjectClass { + ObjectClass parent_class; + + unsigned int nr_devs; + + bool daemon;
I was wondering what this means. auto_shutdown might be a clearer name.
Quoted 6 lines
qapi/qom.json +6
-0+static void vfu_object_set_socket(Object *obj, Visitor *v, const char *name, + void *opaque, Error **errp) +{ + VfuObject *o = VFU_OBJECT(obj); + + qapi_free_SocketAddress(o->socket);
qapi/qom.json
Quoted 10 lines
qapi/qom.json +10
-0+ + o->socket = NULL; + + visit_type_SocketAddress(v, name, &o->socket, errp); + + if (o->socket->type != SOCKET_ADDRESS_TYPE_UNIX) { + qapi_free_SocketAddress(o->socket); + o->socket = NULL; + error_setg(errp, "vfu: Unsupported socket type - %s", + o->socket->u.q_unix.path);
s/o->socket->u.q_unix.path/SocketAddressType_str(o->socket->type)/
Re: [PATCH v4 05/14] vfio-user: instantiate vfio-user context
In reply to Jagannathan Ramanhw/remote/vfio-user-obj.c static void vfu_object_set_device(Object *obj, const char *str, Error **errp) · line 143
Quoted 11 lines · Jagannathan Raman
hw/remote/vfio-user-obj.c +11
-0+static void vfu_object_init_ctx (VfuObject *o,Error **errp) +{ +ERRP_GUARD (); + +if (o->vfu_ctx || !o->socket || !o->device || + !phase_check (PHASE_MACHINE_READY )) { +return ; + } + +if (o->err) { +error_propagate (errp, o->err);
Missing o->err = NULL because ownership has been passed to errp.
Re: [PATCH v4 04/14] vfio-user: define vfio-user-server object
In reply to Jagannathan RamanQuoted 22 lines · Jagannathan Raman
hw/remote/vfio-user-obj.c new file +22
-0+static void vfu_object_set_socket (Object *obj,Visitor *v,const char *name, +void *opaque,Error **errp) +{ +VfuObject *o =VFU_OBJECT (obj); + +qapi_free_SocketAddress (o->socket); + + o->socket =NULL ; + +visit_type_SocketAddress (v, name, &o->socket, errp); + +if (o->socket->type !=SOCKET_ADDRESS_TYPE_UNIX ) { +qapi_free_SocketAddress (o->socket); + o->socket =NULL ; +error_setg (errp,"vfu: Unsupported socket type - %s" , + o->socket->u.q_unix.path); +return ; + } + +trace_vfu_prop ("socket" , o->socket->u.q_unix.path); +} +
hw/remote/vfio-user-obj.c line 98
Quoted 10 lines
hw/remote/vfio-user-obj.c new file +10
-0+static void vfu_object_set_device (Object *obj,const char *str,Error **errp) +{ +VfuObject *o =VFU_OBJECT (obj); + +g_free (o->device); + + o->device =g_strdup (str); + +trace_vfu_prop ("device" , str); +}
It appears "socket" and "device" can be changed after the vfio-user server has started. In the best case it just means the properties contain values that do not reflect the actual socket/device currently in use, which is confusing.
It's safer to refuse changing these properties once the vfio-user server has started.
Stefan
Re: [PATCH v4 06/14] vfio-user: find and init PCI device
In reply to Jagannathan RamanQuoted 2 lines · Jagannathan Raman
hw/remote/vfio-user-obj.c +1
-0@@ -150,6 +157,38 @@ static void vfu_object_init_ctx(VfuObject *o, Error **errp) + o->pci_dev =PCI_DEVICE (dev);
...
hw/remote/vfio-user-obj.c static void vfu_object_finalize(Object *obj) · line 232
Quoted 5 lines
hw/remote/vfio-user-obj.c +1
-0@@ -190,6 +229,8 @@ static void vfu_object_finalize(Object *obj) o->device =NULL ; + o->pci_dev =NULL ;
We need to consider how this interacts with device_del hot unplug. o->pci_dev is a pointer and we don't hold a refcount, so I think o->pci_dev could disappear at any moment.
A pair of object_ref/unref(OBJECT(o->pci_dev)) calls would not be enough because device_del will still unrealize the device that's in use by the vfio-user server.
I suggest adding a check to qdev_unplug() that prevents unplug when the device is in use by the vfio-user server. That's similar to the code in that function for preventing unplug during migration.
One way to do that is by adding a new API:
/*
* Register an Error that is raised when unplug is attempted on a
* device. If another blocker has already been registered then that
* Error may be raised during unplug instead.
*
* qdev_del_unplug_blocker() must be called to remove this blocker. */
void qdev_add_unplug_blocker(DeviceState *dev, Error *blocker);
/*
* Deregister an Error that was previously registered with
* qdev_add_unplug_blocker().
*/
void qdev_del_unplug_blocker(DeviceState *dev, Error *blocker);
The vfio-user server then needs to add an Error *unplug_blocker field to VfuObject and call qdev_add/del_unplug_blocker() on the PCI device.
From a user perspective this means that device_del fails with "Device currently in use by vfio-user server '%s'".
Stefan
Re: [PATCH v4 07/14] vfio-user: run vfio-user context
In reply to Jagannathan RamanQuoted 9 lines · Jagannathan Raman
hw/remote/vfio-user-obj.c +5
-0@@ -114,6 +118,62 @@ static void vfu_object_set_device(Object *obj, const char *str, Error **errp)vfu_object_init_ctx (o, errp); } +static void vfu_object_ctx_run (void *opaque) +{ +VfuObject *o = opaque; +int ret =-1 ; +
hw/remote/vfio-user-obj.c static void vfu_object_set_device(Object *obj, const char *str, Error **errp) · line 135
Quoted 10 lines
hw/remote/vfio-user-obj.c +10
-0+while (ret !=0 ) { + ret =vfu_run_ctx (o->vfu_ctx); +if (ret <0 ) { +if (errno ==EINTR ) { +continue ; + }else if (errno ==ENOTCONN ) { +qemu_set_fd_handler (o->vfu_poll_fd,NULL ,NULL ,NULL ); + o->vfu_poll_fd =-1 ; +object_unparent (OBJECT (o)); +break ;
If nothing else logs a message then I think that should be done here so users know why their vfio-user server object disappeared.
hw/remote/vfio-user-obj.c static void vfu_object_set_device(Object *obj, const char *str, Error **errp) · line 138
Quoted 3 lines
hw/remote/vfio-user-obj.c +3
-0+ }else { +error_setg (&error_abort,"vfu: Failed to run device %s - %s" , + o->device,strerror (errno));
error_abort is equivalent to assuming !o->daemon. In the case where the user doesn't want to automatically shut down the process we need to log a message without aborting.
Review static void vfu_object_set_device(Object *obj, const char *str, Error **errp) · line 139
Quoted 1 line
(quoted diff) +1
-0+ break;
Indentation is off.
Quoted 11 lines
hw/remote/vfio-user-obj.c +11
-0+ } + } + } +} + +static void vfu_object_attach_ctx (void *opaque) +{ +VfuObject *o = opaque; +GPollFD pfds[1 ]; +int ret; +
hw/remote/vfio-user-obj.c static void vfu_object_set_device(Object *obj, const char *str, Error **errp) · line 160
Quoted 10 lines
hw/remote/vfio-user-obj.c +10
-0+qemu_set_fd_handler (o->vfu_poll_fd,NULL ,NULL ,NULL ); + + pfds[0 ].fd = o->vfu_poll_fd; + pfds[0 ].events =G_IO_IN |G_IO_HUP |G_IO_ERR ; + +retry_attach: + ret =vfu_attach_ctx (o->vfu_ctx); +if (ret <0 && (errno ==EAGAIN || errno ==EWOULDBLOCK )) { +qemu_poll_ns (pfds,1 ,500 * (int64_t )SCALE_MS ); + goto retry_attach;
This can block the thread indefinitely. Other events like monitor commands are not handled in this loop. Please make this asynchronous (set an fd handler and return from this function so we can try again later).
The vfu_attach_ctx() implementation synchronously negotiates the vfio-user connection :(. That's a shame because even if accept(2) is handled asynchronously, the negotiation can still block. It would be cleanest to have a fully async libvfio-user's vfu_attach_ctx() API to avoid blocking. Is that possible?
If libvfio-user can't make vfu_attach_ctx() fully async then it may be possible to spawn a thread just for the blocking vfu_attach_ctx() call and report the result back to the event loop (e.g. using EventNotifier). That adds a bunch of code to work around a blocking API though, so I guess we can leave the blocking part if necessary.
At the very minimum, please make EAGAIN/EWOULDBLOCK async as mentioned above and add a comment explaining the situation with the partially-async vfu_attach_ctx() API so it's clear that this can block (that way it's clear that you're aware of the issue and this isn't by accident).
hw/remote/vfio-user-obj.c static void vfu_object_set_device(Object *obj, const char *str, Error **errp) · line 164
Quoted 4 lines
hw/remote/vfio-user-obj.c +4
-0+ }else if (ret <0 ) { +error_setg (&error_abort, +"vfu: Failed to attach device %s to context - %s" , + o->device,strerror (errno));
error_abort assumes !o->daemon. Please handle the o->daemon == true case by logging an error without aborting.
hw/remote/vfio-user-obj.c static void vfu_object_set_device(Object *obj, const char *str, Error **errp) · line 170
Quoted 6 lines
hw/remote/vfio-user-obj.c +6
-0+return ; + } + + o->vfu_poll_fd =vfu_get_poll_fd (o->vfu_ctx); +if (o->vfu_poll_fd <0 ) { +error_setg (&error_abort,"vfu: Failed to get poll fd %s" , o->device);
Same here.
hw/remote/vfio-user-obj.c static void vfu_object_init(Object *obj) · line 289
Quoted 7 lines
hw/remote/vfio-user-obj.c +2
-0@@ -208,6 +284,8 @@ static void vfu_object_init(Object *obj) TYPE_VFU_OBJECT, TYPE_REMOTE_MACHINE);return ; } + + o->vfu_poll_fd =-1 ; }
This must call qemu_set_fd_handler(o->vfu_poll_fd, NULL, NULL, NULL) when o->vfu_poll_fd != -1 to avoid leaving a dangling fd handler callback registered.
Re: [PATCH v4 08/14] vfio-user: handle PCI config space accesses
In reply to Jagannathan RamanQuoted 63 lines · Jagannathan Raman
Define and register handlers for PCI config space accesses
Signed-off-by Elena Ufimtseva <[email protected]>Signed-off-by John G Johnson <[email protected]>Signed-off-by Jagannathan Raman <[email protected]>--- hw/remote/vfio-user-obj.c | 45 +++++++++++++++++++++++++++++++++++++++ hw/remote/trace-events | 2 ++ 2 files changed, 47 insertions(+)hw/remote/vfio-user-obj.c +34
-0@@ -43,6 +43,7 @@#include "qapi/qapi-visit-sockets.h" #include "qemu/notify.h" #include "qemu/thread.h" +#include "qemu/main-loop.h" #include "sysemu/sysemu.h" #include "libvfio-user.h" #include "hw/qdev-core.h" @@ -174,6 +175,39 @@ retry_attach:qemu_set_fd_handler (o->vfu_poll_fd, vfu_object_ctx_run,NULL , o); } +static ssize_t vfu_object_cfg_access (vfu_ctx_t *vfu_ctx,char *const buf, +size_t count,loff_t offset, +const bool is_write) +{ +VfuObject *o =vfu_get_private (vfu_ctx); +uint32_t pci_access_width =sizeof (uint32_t ); +size_t bytes = count; +uint32_t val =0 ; +char *ptr = buf; +int len; + +while (bytes >0 ) { + len = (bytes > pci_access_width) ? pci_access_width : bytes; +if (is_write) { +memcpy (&val, ptr, len); +pci_host_config_write_common (o->pci_dev, offset, +pci_config_size (o->pci_dev), + val, len); +trace_vfu_cfg_write (offset, val); + }else { + val =pci_host_config_read_common (o->pci_dev, offset, +pci_config_size (o->pci_dev), len); +memcpy (ptr, &val, len); +trace_vfu_cfg_read (offset, val); + } + offset += len; + ptr += len; + bytes -= len; + } + +return count; +} +/* * TYPE_VFU_OBJECT depends on the availability of the 'socket' and 'device' * properties. It also depends on devices instantiated in QEMU. These
hw/remote/vfio-user-obj.c static void vfu_object_init_ctx(VfuObject *o, Error **errp) · line 284
Quoted 7 lines
hw/remote/vfio-user-obj.c +4
-0@@ -244,6 +278,17 @@ static void vfu_object_init_ctx(VfuObject *o, Error **errp) goto fail; } + ret =vfu_setup_region (o->vfu_ctx,VFU_PCI_DEV_CFG_REGION_IDX , +pci_config_size (o->pci_dev), &vfu_object_cfg_access, +VFU_REGION_FLAG_RW |VFU_REGION_FLAG_ALWAYS_CB , +NULL ,0 ,-1 ,0 );
Thanos or John: QEMU's pci_host_config_read/write_common() works with host-endian values. I don't know which endianness libvfio-user's region callbacks expect. Does this patch look endian-safe to you (e.g. will it work on a big-endian host)?
Thanks,
Stefan
Re: [PATCH v4 08/14] vfio-user: handle PCI config space accesses
Quoted 9 lines · Stefan Hajnoczi
hw/remote/vfio-user-obj.c static void vfu_object_init_ctx(VfuObject *o, Error **errp) · line 284
Quoted 4 lines
hw/remote/vfio-user-obj.c +4
-0+ ret =vfu_setup_region (o->vfu_ctx,VFU_PCI_DEV_CFG_REGION_IDX , +pci_config_size (o->pci_dev), &vfu_object_cfg_access, +VFU_REGION_FLAG_RW |VFU_REGION_FLAG_ALWAYS_CB , +NULL ,0 ,-1 ,0 );Thanos or John: QEMU's pci_host_config_read/write_common() works with host-endian values. I don't know which endianness libvfio-user's region callbacks expect. Does this patch look endian-safe to you (e.g. will it work on a big-endian host)?
https://github.com/nutanix/libvfio-user/issues/615
regards
john
Re: [PATCH v4 09/14] vfio-user: handle DMA mappings
In reply to Jagannathan RamanQuoted 50 lines · Jagannathan Raman
Define and register callbacks to manage the RAM regions used for device DMA
Signed-off-by Elena Ufimtseva <[email protected]>Signed-off-by John G Johnson <[email protected]>Signed-off-by Jagannathan Raman <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>--- hw/remote/vfio-user-obj.c | 48 +++++++++++++++++++++++++++++++++++++++ hw/remote/trace-events | 2 ++ 2 files changed, 50 insertions(+)hw/remote/vfio-user-obj.c +30
-0@@ -208,6 +208,47 @@ static ssize_t vfu_object_cfg_access(vfu_ctx_t *vfu_ctx, char * const buf,return count; } +static void dma_register (vfu_ctx_t *vfu_ctx,vfu_dma_info_t *info) +{ +MemoryRegion *subregion =NULL ; + g_autofree char *name = NULL; +static unsigned int suffix; +struct iovec *iov = &info->iova; + +if (!info->vaddr) { +return ; + } + + name =g_strdup_printf ("remote-mem-%u" , suffix++); + + subregion =g_new0 (MemoryRegion,1 ); + +memory_region_init_ram_ptr (subregion,NULL , name, + iov->iov_len, info->vaddr); + +memory_region_add_subregion (get_system_memory (), (hwaddr )iov->iov_base, + subregion); + +trace_vfu_dma_register ((uint64_t )iov->iov_base, iov->iov_len); +} + +static void dma_unregister (vfu_ctx_t *vfu_ctx,vfu_dma_info_t *info) +{ +MemoryRegion *mr =NULL ; +ram_addr_t offset; + + mr =memory_region_from_host (info->vaddr, &offset);
hw/remote/vfio-user-obj.c static ssize_t vfu_object_cfg_access(vfu_ctx_t *vfu_ctx, char * const buf, · line 250
Quoted 10 lines
hw/remote/vfio-user-obj.c +10
-0+if (!mr) { +return ; + } + +memory_region_del_subregion (get_system_memory (), mr); + +object_unparent ((OBJECT (mr))); + +trace_vfu_dma_unregister ((uint64_t )info->iova.iov_base); +}
This does not support hot unplug (memory regions pointing to memory mapped by libvfio-user are left registered). The code should keep a list (e.g. https://docs.gtk.org/glib/struct.SList.html) of memory regions and automatically remove them before destroying the vfu context.
It also doesn't support multiple vfio-user server instances running in the same QEMU process. get_system_memory() is global but the memory regions provided by vfio-user are per-client (i.e. VM). If multiple VMs are connected to one vfio-user server process then they conflict.
I don't know the best way to support multiple vfio-user server instances, it would be straightforward if QEMU supported multiple MachineStates and didn't use global get_system_memory()/get_io_memory() APIs. It would be nice to solve that in the future.
Maybe it's too hard to change that, I haven't looked. An alternative is to make the x-remote machine empty (it doesn't create any devices) and instead create a new PCI bus, interrupt controller, memory MemoryRegion, and io MemoryRegion in VfuObject. Stop using get_system_memory() and instead use the per-VfuObject memory MemoryRegion.
In either of those approaches it's probably necessary to specify the PCI bus ID in --device and device_add so it's clear which vfio-user server the PCI device is associated with.
The multiple vfio-user server instance limitation doesn't need to be solved now, but I wanted to share some ideas around it. Maybe someone has better ideas or is aware of limitations preventing what I described.
Stefan
Re: [PATCH v4 10/14] vfio-user: handle PCI BAR accesses
In reply to Jagannathan Ramanhw/remote/vfio-user-obj.c static void dma_unregister(vfu_ctx_t *vfu_ctx, vfu_dma_info_t *info) · line 262
Quoted 11 lines · Jagannathan Raman
hw/remote/vfio-user-obj.c +11
-0+static ssize_t vfu_object_bar_rw (PCIDevice *pci_dev,hwaddr addr,size_t count, +char *const buf,const bool is_write, +bool is_io) +{ +AddressSpace *as =NULL ; +MemTxResult res; + +if (is_io) { + as = &address_space_io; + }else { + as =pci_device_iommu_address_space (pci_dev);
This access is not initiated by the device, it's coming from the CPU. It shouldn't go through the IOMMU address space.
Re: [PATCH v4 11/14] vfio-user: IOMMU support for remote device
In reply to Jagannathan RamanQuoted 1 line · Jagannathan Raman
Assign separate address space for each device in the remote processes.
If I understand correctly this isn't really an IOMMU. It's abusing the IOMMU APIs to create isolated address spaces for each device. This way memory regions added by the vfio-user client do not conflict when there are multiple vfio-user servers.
Calling pci_root_bus_new() and keeping one PCI bus per VfuObject might be a cleaner approach:
- Lets you isolate both PCI Memory Space and IO Space.
- Isolates the PCIDevices and their addresses on the bus.
- Isolates irqs.
- No more need to abuse the IOMMU API.
I might be missing something because I haven't investigated how to do this myself.
Quoted 21 lines
Signed-off-by Elena Ufimtseva <[email protected]>Signed-off-by John G Johnson <[email protected]>Signed-off-by Jagannathan Raman <[email protected]>--- include/hw/pci/pci.h | 2 + include/hw/remote/iommu.h | 24 ++++++++ hw/pci/pci.c | 2 +- hw/remote/iommu.c | 117 ++++++++++++++++++++++++++++++++++++++ hw/remote/machine.c | 5 ++ hw/remote/vfio-user-obj.c | 20 ++++++- MAINTAINERS | 2 + hw/remote/meson.build | 1 + 8 files changed, 169 insertions(+), 4 deletions(-) create mode 100644 include/hw/remote/iommu.h create mode 100644 hw/remote/iommu.cinclude/hw/pci/pci.h +0
-0
include/hw/pci/pci.h void lsi53c8xx_handle_legacy_cmdline(DeviceState *lsi_dev); · line 737
Quoted 4 lines
include/hw/pci/pci.h +1
-0@@ -734,6 +734,8 @@ void lsi53c8xx_handle_legacy_cmdline(DeviceState *lsi_dev);qemu_irq pci_allocate_irq (PCIDevice *pci_dev);void pci_set_irq (PCIDevice *pci_dev,int level); +void pci_init_bus_master (PCIDevice *pci_dev);
This function isn't used in this patch. Why make it public?
Quoted 65 lines
include/hw/pci/pci.h +1
-0+static inline void pci_irq_assert (PCIDevice *pci_dev) {pci_set_irq (pci_dev,1 );include/hw/remote/iommu.h new file +24
-0@@ -0,0 +1,24 @@ +/* +* IOMMU for remote device +* +* Copyright © 2021 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later. +* See the COPYING file in the top-level directory. +* +*/ + +#ifndef REMOTE_IOMMU_H +#define REMOTE_IOMMU_H + +#include "hw/pci/pci_bus.h" + +void remote_iommu_free (PCIDevice *pci_dev); + +void remote_iommu_init (void ); + +void remote_iommu_set (PCIBus *bus); + +MemoryRegion *remote_iommu_get_ram (PCIDevice *pci_dev); + +#endif hw/pci/pci.c +1
-1@@ -95,7 +95,7 @@ static const VMStateDescription vmstate_pcibus = { } }; -static void pci_init_bus_master (PCIDevice *pci_dev) +void pci_init_bus_master (PCIDevice *pci_dev) {AddressSpace *dma_as =pci_device_iommu_address_space (pci_dev);hw/remote/iommu.c new file +12
-0@@ -0,0 +1,117 @@ +/* +* Remote IOMMU +* +* Copyright © 2021 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later. +* See the COPYING file in the top-level directory. +* +*/ + +#include "qemu/osdep.h" +#include "qemu-common.h"
hw/remote/iommu.c line 22
Quoted 10 lines
hw/remote/iommu.c new file +10
-0+ +#include "hw/remote/iommu.h" +#include "hw/pci/pci_bus.h" +#include "exec/memory.h" +#include "exec/address-spaces.h" +#include "trace.h" + +struct VFUIOMMU { +AddressSpace as; +MemoryRegion mr;
I guess this is the root MemoryRegion container? Calling it "root" or "root_mr" instead of "mr" would make that clearer.
hw/remote/iommu.c line 25
Quoted 3 lines
hw/remote/iommu.c new file +3
-0+}; + +typedef struct VFUPciBus {
There is no uppercase/lowercase consistency between VfuObject vs VFUIOMMU vs VFUPciBus. Although the coding standard doesn't dictate ABC vs Abc, please be consistent. I suggest following the VfuObject convention started in the previous patches. The names would be VfuIommu and VfuPciBus.
Quoted 171 lines
hw/remote/iommu.c new file +92
-0+PCIBus *bus; +struct VFUIOMMU *iommu[]; +} VFUPciBus; + +GHashTable *remote_as_table; + +static AddressSpace *remote_iommu_get_as (PCIBus *bus,void *opaque,int devfn) +{ +VFUPciBus *vfu_pci_bus =NULL ; +struct VFUIOMMU *iommu =NULL ; + +if (!remote_as_table) { +return &address_space_memory; + } + + vfu_pci_bus =g_hash_table_lookup (remote_as_table, bus); + +if (!vfu_pci_bus) { + vfu_pci_bus =g_malloc0 (sizeof (VFUPciBus)); + vfu_pci_bus->bus = bus; +g_hash_table_insert (remote_as_table, bus, vfu_pci_bus); + } + + iommu = vfu_pci_bus->iommu[devfn]; + +if (!iommu) { + g_autofree char *mr_name = g_strdup_printf("vfu-ram-%d", devfn); + g_autofree char *as_name = g_strdup_printf("vfu-as-%d", devfn); + + iommu =g_malloc0 (sizeof (struct VFUIOMMU )); + +memory_region_init (&iommu->mr,NULL , mr_name,UINT64_MAX ); +address_space_init (&iommu->as, &iommu->mr, as_name); + + vfu_pci_bus->iommu[devfn] = iommu; + } + +return &iommu->as; +} + +void remote_iommu_free (PCIDevice *pci_dev) +{ +VFUPciBus *vfu_pci_bus =NULL ; +struct VFUIOMMU *iommu =NULL ; + +if (!remote_as_table) { +return ; + } + + vfu_pci_bus =g_hash_table_lookup (remote_as_table,pci_get_bus (pci_dev)); + +if (!vfu_pci_bus) { +return ; + } + + iommu = vfu_pci_bus->iommu[pci_dev->devfn]; + + vfu_pci_bus->iommu[pci_dev->devfn] =NULL ; + +if (iommu) { +memory_region_unref (&iommu->mr); +address_space_destroy (&iommu->as); +g_free (iommu); + } +} + +void remote_iommu_init (void ) +{ + remote_as_table =g_hash_table_new_full (NULL ,NULL ,NULL ,NULL ); +} + +void remote_iommu_set (PCIBus *bus) +{ +pci_setup_iommu (bus, remote_iommu_get_as,NULL ); +} + +MemoryRegion *remote_iommu_get_ram (PCIDevice *pci_dev) +{ +PCIBus *bus =pci_get_bus (pci_dev); +VFUPciBus *vfu_pci_bus; + +if (!remote_as_table) { +return get_system_memory (); + } + + vfu_pci_bus =g_hash_table_lookup (remote_as_table, bus); +if (!vfu_pci_bus) { +return get_system_memory (); + } + +return &vfu_pci_bus->iommu[pci_dev->devfn]->mr; +}hw/remote/machine.c +5
-0@@ -21,6 +21,7 @@#include "qapi/error.h" #include "hw/pci/pci_host.h" #include "hw/remote/iohub.h" +#include "hw/remote/iommu.h" static void remote_machine_init (MachineState *machine) {@@ -52,6 +53,10 @@ static void remote_machine_init(MachineState *machine)remote_iohub_init (&s->iohub); +remote_iommu_init (); + +remote_iommu_set (pci_host->bus); +pci_bus_irqs (pci_host->bus, remote_iohub_set_irq, remote_iohub_map_irq, &s->iohub,REMOTE_IOHUB_NB_PIRQS ); }hw/remote/vfio-user-obj.c +7
-3@@ -49,6 +49,7 @@#include "hw/qdev-core.h" #include "hw/pci/pci.h" #include "qemu/timer.h" +#include "hw/remote/iommu.h" #define TYPE_VFU_OBJECT "x-vfio-user-server"OBJECT_DECLARE_TYPE (VfuObject, VfuObjectClass,VFU_OBJECT )@@ -210,6 +211,7 @@ static ssize_t vfu_object_cfg_access(vfu_ctx_t *vfu_ctx, char * const buf,static void dma_register (vfu_ctx_t *vfu_ctx,vfu_dma_info_t *info) { +VfuObject *o =vfu_get_private (vfu_ctx);MemoryRegion *subregion =NULL ; g_autofree char *name = NULL;static unsigned int suffix;@@ -226,14 +228,15 @@ static void dma_register(vfu_ctx_t *vfu_ctx, vfu_dma_info_t *info)memory_region_init_ram_ptr (subregion,NULL , name, iov->iov_len, info->vaddr); -memory_region_add_subregion (get_system_memory (), (hwaddr )iov->iov_base, - subregion); +memory_region_add_subregion (remote_iommu_get_ram (o->pci_dev), + (hwaddr )iov->iov_base, subregion);trace_vfu_dma_register ((uint64_t )iov->iov_base, iov->iov_len); }static void dma_unregister (vfu_ctx_t *vfu_ctx,vfu_dma_info_t *info) { +VfuObject *o =vfu_get_private (vfu_ctx);MemoryRegion *mr =NULL ;ram_addr_t offset;@@ -242,7 +245,7 @@ static void dma_unregister(vfu_ctx_t *vfu_ctx, vfu_dma_info_t *info)return ; } -memory_region_del_subregion (get_system_memory (), mr); +memory_region_del_subregion (remote_iommu_get_ram (o->pci_dev), mr);object_unparent ((OBJECT (mr)));@@ -320,6 +323,7 @@ static vfu_region_access_cb_t *vfu_object_bar_handlers[PCI_NUM_REGIONS] = {*/ static void vfu_object_register_bars (vfu_ctx_t *vfu_ctx,PCIDevice *pdev) { +VfuObject *o =vfu_get_private (vfu_ctx);int i;for (i =0 ; i <PCI_NUM_REGIONS ; i++) {
hw/remote/vfio-user-obj.c static void vfu_object_register_bars(vfu_ctx_t *vfu_ctx, PCIDevice *pdev) · line 343
Quoted 8 lines
hw/remote/vfio-user-obj.c +5
-0@@ -332,6 +336,12 @@ static void vfu_object_register_bars(vfu_ctx_t *vfu_ctx, PCIDevice *pdev) vfu_object_bar_handlers[i], VFU_REGION_FLAG_RW, NULL, 0, -1, 0); +if ((o->pci_dev->io_regions[i].type &PCI_BASE_ADDRESS_SPACE ) ==0 ) { +memory_region_unref (o->pci_dev->io_regions[i].address_space); + o->pci_dev->io_regions[i].address_space = +remote_iommu_get_ram (o->pci_dev); + }
This looks hacky. If you create a separate PCIHost for each device instead then the BARs will be created in the MemoryRegion (confusingly named "address_space" in the PCI code) of your choosing.
Also, why is PCI Memory Space isolated via VFUIOMMU but PCI IO Space is not?
Re: [PATCH v4 12/14] vfio-user: handle device interrupts
In reply to Jagannathan Ramanhw/remote/iohub.c void remote_iohub_set_irq(void *opaque, int pirq, int level) · line 74
Quoted 10 lines · Jagannathan Raman
hw/remote/iohub.c +3
-0@@ -62,6 +66,9 @@ void remote_iohub_set_irq(void *opaque, int pirq, int level)QEMU_LOCK_GUARD (&iohub->irq_level_lock[pirq]);if (level) { +if (iohub->intx_notify) { + iohub->intx_notify (pirq,0 ); + }if (++iohub->irq_level[pirq] ==1 ) {event_notifier_set (&iohub->irqfds[pirq]); }
Does it make sense to use iohub.c in vfio-user since these irqfds are unused?
Instead of adding iohub->intx_notify(), can vfio-user register its own set_irq() callback for the PCI bus?
Quoted 24 lines
hw/remote/vfio-user-obj.c +5
-0@@ -50,6 +50,9 @@#include "hw/pci/pci.h" #include "qemu/timer.h" #include "hw/remote/iommu.h" +#include "hw/pci/msi.h" +#include "hw/pci/msix.h" +#include "hw/remote/iohub.h" #define TYPE_VFU_OBJECT "x-vfio-user-server"OBJECT_DECLARE_TYPE (VfuObject, VfuObjectClass,VFU_OBJECT )@@ -81,6 +84,8 @@ struct VfuObject {int vfu_poll_fd; }; +static GHashTable *vfu_object_dev_table; +static void vfu_object_init_ctx (VfuObject *o,Error **errp); static void vfu_object_set_socket(Object *obj, Visitor *v, const char *name,
hw/remote/vfio-user-obj.c static void vfu_object_register_bars(vfu_ctx_t *vfu_ctx, PCIDevice *pdev) · line 358
Quoted 7 lines
hw/remote/vfio-user-obj.c +4
-0@@ -347,6 +352,54 @@ static void vfu_object_register_bars(vfu_ctx_t *vfu_ctx, PCIDevice *pdev) } } +static void vfu_object_intx_notify (int pci_devfn,unsigned vector) +{ +vfu_ctx_t *vfu_ctx =g_hash_table_lookup (vfu_object_dev_table, + (void *)(uint64_t )pci_devfn);
I'm not sure the hash table is necessary. The function arguments currently don't contain the information we need, but that's easy to fix because these functions are added by this patch. Add an opaque pointer argument to ->intx_notify, ->msix_notify, and ->msi_notify in order to pass along the VfuObject we need.
Quoted 62 lines
hw/remote/vfio-user-obj.c +51
-0+ +if (vfu_ctx) { +vfu_irq_trigger (vfu_ctx, vector); + } +} + +static void vfu_object_msi_notify (PCIDevice *pci_dev,unsigned vector) +{ +vfu_object_intx_notify (pci_dev->devfn, vector); +} + +static int vfu_object_setup_irqs (vfu_ctx_t *vfu_ctx,PCIDevice *pci_dev) +{ +RemoteMachineState *machine =REMOTE_MACHINE (current_machine); +RemoteIOHubState *iohub = &machine->iohub; +int ret; + + ret =vfu_setup_device_nr_irqs (vfu_ctx,VFU_DEV_INTX_IRQ ,1 ); +if (ret <0 ) { +return ret; + } + + iohub->intx_notify = vfu_object_intx_notify; + + ret =0 ; +if (msix_nr_vectors_allocated (pci_dev)) { + ret =vfu_setup_device_nr_irqs (vfu_ctx,VFU_DEV_MSIX_IRQ , +msix_nr_vectors_allocated (pci_dev)); + + pci_dev->msix_notify = vfu_object_msi_notify; + }else if (msi_nr_vectors_allocated (pci_dev)) { + ret =vfu_setup_device_nr_irqs (vfu_ctx,VFU_DEV_MSI_IRQ , +msi_nr_vectors_allocated (pci_dev)); + + pci_dev->msi_notify = vfu_object_msi_notify; + } + +if (ret <0 ) { +return ret; + } + +return 0 ; +} +/* * TYPE_VFU_OBJECT depends on the availability of the 'socket' and 'device' * properties. It also depends on devices instantiated in QEMU. These @@ -437,6 +490,13 @@ static void vfu_object_init_ctx(VfuObject *o, Error **errp)vfu_object_register_bars (o->vfu_ctx, o->pci_dev); + ret =vfu_object_setup_irqs (o->vfu_ctx, o->pci_dev); +if (ret <0 ) { +error_setg (errp,"vfu: Failed to setup interrupts for %s" , + o->device); + goto fail; + } + ret =vfu_realize_ctx (o->vfu_ctx);if (ret <0 ) { error_setg(errp, "vfu: Failed to realize device %s- %s",
hw/remote/vfio-user-obj.c static void vfu_object_init_ctx(VfuObject *o, Error **errp) · line 514
Quoted 5 lines
hw/remote/vfio-user-obj.c +2
-0@@ -450,6 +510,9 @@ static void vfu_object_init_ctx(VfuObject *o, Error **errp) goto fail; } +g_hash_table_insert (vfu_object_dev_table, + (void *)(uint64_t )o->pci_dev->devfn, o->vfu_ctx);
vfu_object_dev_table seems like a misnomer since the values stored are actually vfu_ctx_t, not VfuObjects. vfu_devfn_to_vfu_ctx_table?
hw/remote/vfio-user-obj.c static void vfu_object_finalize(Object *obj) · line 572
Quoted 11 lines
hw/remote/vfio-user-obj.c +4
-0+qemu_set_fd_handler (o->vfu_poll_fd, vfu_object_attach_ctx,NULL , o);return ;@@ -504,9 +567,18 @@ static void vfu_object_finalize(Object *obj)remote_iommu_free (o->pci_dev); } +if (o->pci_dev && +g_hash_table_lookup (vfu_object_dev_table, + (void *)(uint64_t )o->pci_dev->devfn)) {
lookup() is unnecessary since remove() is a nop if the key does not exist.
Re: [PATCH v4 08/14] vfio-user: handle PCI config space accesses
In reply to John LevonQuoted 7 lines · John Levon
Quoted 5 lines · Stefan Hajnoczi
Quoted 4 lines
hw/remote/vfio-user-obj.c +4
-0+ ret =vfu_setup_region (o->vfu_ctx,VFU_PCI_DEV_CFG_REGION_IDX , +pci_config_size (o->pci_dev), &vfu_object_cfg_access, +VFU_REGION_FLAG_RW |VFU_REGION_FLAG_ALWAYS_CB , +NULL ,0 ,-1 ,0 );
Quoted 6 lines
Quoted 4 lines
Thanos or John: QEMU's pci_host_config_read/write_common() works with host-endian values. I don't know which endianness libvfio-user's region callbacks expect. Does this patch look endian-safe to you (e.g. will it work on a big-endian host)?
Thanks!
Stefan
Re: [PATCH v4 02/14] tests/avocado: Specify target VM argument to helper routines
In reply to Beraldo LealQuoted 39 lines · Beraldo Leal
Quoted 15 lines · Jagannathan Raman
Specify target VM for
exec_commandandexec_command_and_wait_for_patternroutinesSigned-off-by Elena Ufimtseva <[email protected]>Signed-off-by John G Johnson <[email protected]>Signed-off-by Jagannathan Raman <[email protected]>---
tests/avocado/avocado_qemu/__init__.py| 10 ++++++----1 file changed, 6 insertions(+), 4 deletions(-)tests/avocado/avocado_qemu/__init__.py +0
-0tests/avocado/avocado_qemu/__init__.py def wait_for_console_pattern(test, success_message, failure_message=None, · line 201
Quoted 5 lines
tests/avocado/avocado_qemu/__init__.py +1
-1@@ -198,7 +198,7 @@ def wait_for_console_pattern(test, success_message, failure_message=None, """ _console_interaction(test, success_message, failure_message, None, vm=vm) -def exec_command(test, command): +def exec_command(test, command, vm=None):nitpick: if possible, it would be nice to update the docstring, by adding this new argument.
Quoted 10 lines
tests/avocado/avocado_qemu/__init__.py +1
-1""" Send a command to a console (appending CRLF characters), while logging the content.@@ -208,10 +208,11 @@ def exec_command(test, command): :param command: the command to send :type command: str """ - _console_interaction(test, None, None, command + '\r') + _console_interaction(test, None, None, command + '\r', vm=vm)
Quoted 6 lines
tests/avocado/avocado_qemu/__init__.py def exec_command(test, command): · line 215
Quoted 4 lines
tests/avocado/avocado_qemu/__init__.py +2
-1def exec_command_and_wait_for_pattern(test, command, - success_message, failure_message=None): + success_message, failure_message=None, + vm=None):Same here.
OK sure, will do. Thank you!
--
Jag
Quoted 8 lines
Other than that, lgtm.
Reviewed-by Beraldo Leal <[email protected]>--
Beraldo
Re: [PATCH v4 05/14] vfio-user: instantiate vfio-user context
In reply to Stefan HajnocziQuoted 18 lines · Stefan Hajnoczi
hw/remote/vfio-user-obj.c static void vfu_object_set_device(Object *obj, const char *str, Error **errp) · line 143
Quoted 11 lines · Jagannathan Raman
hw/remote/vfio-user-obj.c +11
-0+static void vfu_object_init_ctx (VfuObject *o,Error **errp) +{ +ERRP_GUARD (); + +if (o->vfu_ctx || !o->socket || !o->device || + !phase_check (PHASE_MACHINE_READY )) { +return ; + } + +if (o->err) { +error_propagate (errp, o->err);Missing
o->err= NULL because ownership has been passed to errp.
OK, will do!
Thank you!
--
Jag
Re: [PATCH v4 04/14] vfio-user: define vfio-user-server object
In reply to Stefan HajnocziQuoted 13 lines · Stefan Hajnoczi
Quoted 8 lines · Jagannathan Raman
qapi/qom.json +0
-0@@ -703,6 +703,20 @@ { 'struct': 'RemoteObjectProperties', 'data': { 'fd': 'str', 'devid': 'str' } }
Quoted 12 lines
qapi/qom.json line 715
Quoted 10 lines
qapi/qom.json +10
-0+## +# @VfioUserServerProperties: +# +# Properties for x-vfio-user-server objects. +# +# @socket: socket to be used by the libvfiouser library +# +# @device: the id of the device to be emulated at the server +# +# Since: 6.26.2 has been released so the version number needs to be incremented.
OK, thanks!
Quoted 9 lines
qapi/qom.json line 721
Quoted 7 lines
qapi/qom.json +6
-0+struct VfuObjectClass { + ObjectClass parent_class; + + unsigned int nr_devs; + + bool daemon;I was wondering what this means.
auto_shutdownmight be a clearer name.
Sure, will do.
Quoted 11 lines
Quoted 11 lines
qapi/qom.json +10
-0+static void vfu_object_set_socket(Object *obj, Visitor *v, const char *name, + void *opaque, Error **errp) +{ + VfuObject *o = VFU_OBJECT(obj); + + qapi_free_SocketAddress(o->socket); + + o->socket = NULL; + + visit_type_SocketAddress(v, name, &o->socket, errp);
Quoted 8 lines
qapi/qom.json
Quoted 6 lines
qapi/qom.json +6
-0+ + if (o->socket->type != SOCKET_ADDRESS_TYPE_UNIX) { + qapi_free_SocketAddress(o->socket); + o->socket = NULL; + error_setg(errp, "vfu: Unsupported socket type - %s", + o->socket->u.q_unix.path);s/o->socket->u.q_unix.path/SocketAddressType_str(o->socket->type)/
Ok, got it.
Thanks!
--
Jag
Re: [PATCH v4 04/14] vfio-user: define vfio-user-server object
In reply to Stefan HajnocziQuoted 34 lines · Stefan Hajnoczi
Quoted 29 lines · Jagannathan Raman
hw/remote/vfio-user-obj.c new file +29
-0+static void vfu_object_set_socket (Object *obj,Visitor *v,const char *name, +void *opaque,Error **errp) +{ +VfuObject *o =VFU_OBJECT (obj); + +qapi_free_SocketAddress (o->socket); + + o->socket =NULL ; + +visit_type_SocketAddress (v, name, &o->socket, errp); + +if (o->socket->type !=SOCKET_ADDRESS_TYPE_UNIX ) { +qapi_free_SocketAddress (o->socket); + o->socket =NULL ; +error_setg (errp,"vfu: Unsupported socket type - %s" , + o->socket->u.q_unix.path); +return ; + } + +trace_vfu_prop ("socket" , o->socket->u.q_unix.path); +} + +static void vfu_object_set_device (Object *obj,const char *str,Error **errp) +{ +VfuObject *o =VFU_OBJECT (obj); + +g_free (o->device); + + o->device =g_strdup (str);
Quoted 8 lines
hw/remote/vfio-user-obj.c line 98
Quoted 3 lines
hw/remote/vfio-user-obj.c new file +3
-0+ +trace_vfu_prop ("device" , str); +}It appears "socket" and "device" can be changed after the vfio-user server has started. In the best case it just means the properties contain values that do not reflect the actual socket/device currently in use, which is confusing.
I’m not sure about the scenario in which that would occur, but will add that check.
Thank you!
--
Jag
Quoted 5 lines
It's safer to refuse changing these properties once the vfio-user server has started.
Stefan
Re: [PATCH v4 06/14] vfio-user: find and init PCI device
In reply to Stefan HajnocziQuoted 43 lines · Stefan Hajnoczi
Quoted 2 lines · Jagannathan Raman
hw/remote/vfio-user-obj.c +1
-0@@ -150,6 +157,38 @@ static void vfu_object_init_ctx(VfuObject *o, Error **errp) + o->pci_dev =PCI_DEVICE (dev);...
hw/remote/vfio-user-obj.c static void vfu_object_finalize(Object *obj) · line 232
Quoted 5 lines
hw/remote/vfio-user-obj.c +1
-0@@ -190,6 +229,8 @@ static void vfu_object_finalize(Object *obj) o->device =NULL ; + o->pci_dev =NULL ;We need to consider how this interacts with
device_delhot unplug.o->pci_devis a pointer and we don't hold a refcount, so I thinko->pci_devcould disappear at any moment.A pair of object_ref/unref(OBJECT(o->pci_dev)) calls would not be enough because
device_delwill still unrealize the device that's in use by the vfio-user server.I suggest adding a check to
qdev_unplug()that prevents unplug when the device is in use by the vfio-user server. That's similar to the code in that function for preventing unplug during migration.One way to do that is by adding a new API:
/*
* Register an Error that is raised when unplug is attempted on a
* device. If another blocker has already been registered then that
* Error may be raised during unplug instead.
*
*qdev_del_unplug_blocker()must be called to remove this blocker. */
voidqdev_add_unplug_blocker(DeviceState *dev, Error *blocker);/*
* Deregister an Error that was previously registered with
*qdev_add_unplug_blocker().
*/
voidqdev_del_unplug_blocker(DeviceState *dev, Error *blocker);
Quoted 5 lines
The vfio-user server then needs to add an Error
*unplug_blockerfield to VfuObject and call qdev_add/del_unplug_blocker() on the PCI device.From a user perspective this means that
device_delfails with "Device currently in use by vfio-user server '%s'".
OK sounds good, will add the above unplug blocker API for PCI devices.
Thank you!
--
Jag
Quoted 2 lines
Stefan
Re: [PATCH v4 04/14] vfio-user: define vfio-user-server object
In reply to Jag RamanQuoted 43 lines · Jag Raman
Quoted 34 lines · Stefan Hajnoczi
Quoted 29 lines · Jagannathan Raman
hw/remote/vfio-user-obj.c new file +29
-0+static void vfu_object_set_socket (Object *obj,Visitor *v,const char *name, +void *opaque,Error **errp) +{ +VfuObject *o =VFU_OBJECT (obj); + +qapi_free_SocketAddress (o->socket); + + o->socket =NULL ; + +visit_type_SocketAddress (v, name, &o->socket, errp); + +if (o->socket->type !=SOCKET_ADDRESS_TYPE_UNIX ) { +qapi_free_SocketAddress (o->socket); + o->socket =NULL ; +error_setg (errp,"vfu: Unsupported socket type - %s" , + o->socket->u.q_unix.path); +return ; + } + +trace_vfu_prop ("socket" , o->socket->u.q_unix.path); +} + +static void vfu_object_set_device (Object *obj,const char *str,Error **errp) +{ +VfuObject *o =VFU_OBJECT (obj); + +g_free (o->device); + + o->device =g_strdup (str);Quoted 8 lines
hw/remote/vfio-user-obj.c line 98
Quoted 3 lines
hw/remote/vfio-user-obj.c new file +3
-0+ +trace_vfu_prop ("device" , str); +}It appears "socket" and "device" can be changed after the vfio-user server has started. In the best case it just means the properties contain values that do not reflect the actual socket/device currently in use, which is confusing.
Quoted 1 line
I’m not sure about the scenario in which that would occur, but will add that check.
Properties can be modified using the qom-set monitor command. This can happen any time.
Stefan
Re: [PATCH v4 07/14] vfio-user: run vfio-user context
In reply to Stefan HajnocziQuoted 19 lines · Stefan Hajnoczi
Quoted 14 lines · Jagannathan Raman
hw/remote/vfio-user-obj.c +10
-0@@ -114,6 +118,62 @@ static void vfu_object_set_device(Object *obj, const char *str, Error **errp)vfu_object_init_ctx (o, errp); } +static void vfu_object_ctx_run (void *opaque) +{ +VfuObject *o = opaque; +int ret =-1 ; + +while (ret !=0 ) { + ret =vfu_run_ctx (o->vfu_ctx); +if (ret <0 ) { +if (errno ==EINTR ) { +continue ;
Quoted 8 lines
hw/remote/vfio-user-obj.c static void vfu_object_set_device(Object *obj, const char *str, Error **errp) · line 135
Quoted 5 lines
hw/remote/vfio-user-obj.c +5
-0+ }else if (errno ==ENOTCONN ) { +qemu_set_fd_handler (o->vfu_poll_fd,NULL ,NULL ,NULL ); + o->vfu_poll_fd =-1 ; +object_unparent (OBJECT (o)); +break ;If nothing else logs a message then I think that should be done here so users know why their vfio-user server object disappeared.
Sure will do.
Do you prefer a trace, or a message to the console? Trace makes sense to me. Presently, the client could unplug the vfio-user device which would trigger the deletion of this object. This process could happen quietly.
Quoted 8 lines
hw/remote/vfio-user-obj.c static void vfu_object_set_device(Object *obj, const char *str, Error **errp) · line 138
Quoted 4 lines
hw/remote/vfio-user-obj.c +3
-0+ }else { +error_setg (&error_abort,"vfu: Failed to run device %s - %s" , + o->device,strerror (errno));
error_abortis equivalent to assuming !o->daemon. In the case where the user doesn't want to automatically shut down the process we need to log a message without aborting.
OK, makes sense.
Quoted 32 lines
Review static void vfu_object_set_device(Object *obj, const char *str, Error **errp) · line 139
Quoted 2 lines
(quoted diff) +1
-0+ break;Indentation is off.
Quoted 11 lines
hw/remote/vfio-user-obj.c +11
-0+ } + } + } +} + +static void vfu_object_attach_ctx (void *opaque) +{ +VfuObject *o = opaque; +GPollFD pfds[1 ]; +int ret; +hw/remote/vfio-user-obj.c static void vfu_object_set_device(Object *obj, const char *str, Error **errp) · line 160
Quoted 10 lines
hw/remote/vfio-user-obj.c +10
-0+qemu_set_fd_handler (o->vfu_poll_fd,NULL ,NULL ,NULL ); + + pfds[0 ].fd = o->vfu_poll_fd; + pfds[0 ].events =G_IO_IN |G_IO_HUP |G_IO_ERR ; + +retry_attach: + ret =vfu_attach_ctx (o->vfu_ctx); +if (ret <0 && (errno ==EAGAIN || errno ==EWOULDBLOCK )) { +qemu_poll_ns (pfds,1 ,500 * (int64_t )SCALE_MS ); + goto retry_attach;This can block the thread indefinitely. Other events like monitor commands are not handled in this loop. Please make this asynchronous (set an fd handler and return from this function so we can try again later).
Quoted 5 lines
The
vfu_attach_ctx()implementation synchronously negotiates the vfio-user connection :(. That's a shame because even if accept(2) is handled asynchronously, the negotiation can still block. It would be cleanest to have a fully async libvfio-user'svfu_attach_ctx()API to avoid blocking. Is that possible?
Thanos / John,
Any thoughts on this?
Quoted 7 lines
If libvfio-user can't make
vfu_attach_ctx()fully async then it may be possible to spawn a thread just for the blockingvfu_attach_ctx()call and report the result back to the event loop (e.g. using EventNotifier). That adds a bunch of code to work around a blocking API though, so I guess we can leave the blocking part if necessary.
Quoted 5 lines
At the very minimum, please make EAGAIN/EWOULDBLOCK async as mentioned above and add a comment explaining the situation with the partially-async
vfu_attach_ctx()API so it's clear that this can block (that way it's clear that you're aware of the issue and this isn't by accident).
Sure, we could make the attach async at QEMU depending on how the library prefers to do this.
Quoted 18 lines
hw/remote/vfio-user-obj.c static void vfu_object_set_device(Object *obj, const char *str, Error **errp) · line 164
Quoted 5 lines
hw/remote/vfio-user-obj.c +4
-0+ }else if (ret <0 ) { +error_setg (&error_abort, +"vfu: Failed to attach device %s to context - %s" , + o->device,strerror (errno));
error_abortassumes !o->daemon. Please handle theo->daemon== true case by logging an error without aborting.hw/remote/vfio-user-obj.c static void vfu_object_set_device(Object *obj, const char *str, Error **errp) · line 170
Quoted 6 lines
hw/remote/vfio-user-obj.c +6
-0+return ; + } + + o->vfu_poll_fd =vfu_get_poll_fd (o->vfu_ctx); +if (o->vfu_poll_fd <0 ) { +error_setg (&error_abort,"vfu: Failed to get poll fd %s" , o->device);Same here.
Quoted 11 lines
hw/remote/vfio-user-obj.c static void vfu_object_init(Object *obj) · line 289
Quoted 7 lines
hw/remote/vfio-user-obj.c +2
-0@@ -208,6 +284,8 @@ static void vfu_object_init(Object *obj) TYPE_VFU_OBJECT, TYPE_REMOTE_MACHINE);return ; } + + o->vfu_poll_fd =-1 ; }This must call
qemu_set_fd_handler(o->vfu_poll_fd, NULL, NULL, NULL)wheno->vfu_poll_fd!= -1 to avoid leaving a dangling fd handler callback registered.
This is during the init phase, and the FD handlers are not set. Do you mean to add this at finalize?
I agree it’s good to explicitly add this at finalize. But vfu_destroy_ctx() should trigger a ENOTCONN, which would do it anyway.
Thank you!
--
Jag
Re: [PATCH v4 09/14] vfio-user: handle DMA mappings
In reply to Stefan HajnocziQuoted 76 lines · Stefan Hajnoczi
Quoted 50 lines · Jagannathan Raman
Define and register callbacks to manage the RAM regions used for device DMA
Signed-off-by Elena Ufimtseva <[email protected]>Signed-off-by John G Johnson <[email protected]>Signed-off-by Jagannathan Raman <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>---
hw/remote/vfio-user-obj.c| 48 +++++++++++++++++++++++++++++++++++++++
hw/remote/trace-events | 2 ++2 files changed, 50 insertions(+)hw/remote/vfio-user-obj.c +30
-0@@ -208,6 +208,47 @@ static ssize_t vfu_object_cfg_access(vfu_ctx_t *vfu_ctx, char * const buf,return count; } +static void dma_register (vfu_ctx_t *vfu_ctx,vfu_dma_info_t *info) +{ +MemoryRegion *subregion =NULL ; + g_autofree char *name = NULL; +static unsigned int suffix; +struct iovec *iov = &info->iova; + +if (!info->vaddr) { +return ; + } + + name =g_strdup_printf ("remote-mem-%u" , suffix++); + + subregion =g_new0 (MemoryRegion,1 ); + +memory_region_init_ram_ptr (subregion,NULL , name, + iov->iov_len, info->vaddr); + +memory_region_add_subregion (get_system_memory (), (hwaddr )iov->iov_base, + subregion); + +trace_vfu_dma_register ((uint64_t )iov->iov_base, iov->iov_len); +} + +static void dma_unregister (vfu_ctx_t *vfu_ctx,vfu_dma_info_t *info) +{ +MemoryRegion *mr =NULL ; +ram_addr_t offset; + + mr =memory_region_from_host (info->vaddr, &offset);hw/remote/vfio-user-obj.c static ssize_t vfu_object_cfg_access(vfu_ctx_t *vfu_ctx, char * const buf, · line 250
Quoted 10 lines
hw/remote/vfio-user-obj.c +10
-0+if (!mr) { +return ; + } + +memory_region_del_subregion (get_system_memory (), mr); + +object_unparent ((OBJECT (mr))); + +trace_vfu_dma_unregister ((uint64_t )info->iova.iov_base); +}This does not support hot unplug (memory regions pointing to memory mapped by libvfio-user are left registered). The code should keep a list (e.g. https://docs.gtk.org/glib/struct.SList.html) of memory regions and automatically remove them before destroying the vfu context.
It also doesn't support multiple vfio-user server instances running in the same QEMU process.
get_system_memory()is global but the memory regions provided by vfio-user are per-client (i.e. VM). If multiple VMs are connected to one vfio-user server process then they conflict.
Quoted 4 lines
I don't know the best way to support multiple vfio-user server instances, it would be straightforward if QEMU supported multiple MachineStates and didn't use global get_system_memory()/get_io_memory() APIs. It would be nice to solve that in the future.
We’ve addressed the multiple vfio-user-server instances in "[PATCH v4 11/14] vfio-user: IOMMU support for remote device” patch down the line. I see your comments there, will address them.
Thank you!
--
Jag
Quoted 16 lines
Maybe it's too hard to change that, I haven't looked. An alternative is to make the x-remote machine empty (it doesn't create any devices) and instead create a new PCI bus, interrupt controller, memory MemoryRegion, and io MemoryRegion in VfuObject. Stop using
get_system_memory()and instead use the per-VfuObject memory MemoryRegion.In either of those approaches it's probably necessary to specify the PCI bus ID in --device and
device_addso it's clear which vfio-user server the PCI device is associated with.The multiple vfio-user server instance limitation doesn't need to be solved now, but I wanted to share some ideas around it. Maybe someone has better ideas or is aware of limitations preventing what I described.
Stefan
Re: [PATCH v4 10/14] vfio-user: handle PCI BAR accesses
In reply to Stefan HajnocziQuoted 19 lines · Stefan Hajnoczi
hw/remote/vfio-user-obj.c static void dma_unregister(vfu_ctx_t *vfu_ctx, vfu_dma_info_t *info) · line 262
Quoted 11 lines · Jagannathan Raman
hw/remote/vfio-user-obj.c +11
-0+static ssize_t vfu_object_bar_rw (PCIDevice *pci_dev,hwaddr addr,size_t count, +char *const buf,const bool is_write, +bool is_io) +{ +AddressSpace *as =NULL ; +MemTxResult res; + +if (is_io) { + as = &address_space_io; + }else { + as =pci_device_iommu_address_space (pci_dev);This access is not initiated by the device, it's coming from the CPU. It shouldn't go through the IOMMU address space.
Got it, thank you!
--
Jag
Re: [PATCH v4 11/14] vfio-user: IOMMU support for remote device
In reply to Stefan HajnocziQuoted 11 lines · Stefan Hajnoczi
Quoted 1 line · Jagannathan Raman
Assign separate address space for each device in the remote processes.
If I understand correctly this isn't really an IOMMU. It's abusing the IOMMU APIs to create isolated address spaces for each device. This way memory regions added by the vfio-user client do not conflict when there are multiple vfio-user servers.
Like you already figured out, having isolated DMA address space alone is not sufficient for this application, we also needed to isolate the sysmem/RAM address space. As such, the available IOMMU APIs alone were not sufficient, so we had to improvise.
Quoted 7 lines
Calling
pci_root_bus_new()and keeping one PCI bus per VfuObject might be a cleaner approach:
- Lets you isolate both PCI Memory Space and IO Space.
- Isolates the PCIDevices and their addresses on the bus.
- Isolates irqs.
- No more need to abuse the IOMMU API.
I believe we would still need to have an IOMMU. It’s because, devices use the pci_dma_read()/_write() functions. These functions look up the address in DMA address space (via pci_get_address_space() -> PCIDevice->bus_master_as -> PCIDevice->bus_master_enable_region -> PCIDevice->bus_master_container_region).
bus_master_enable_region and bus_master_container_region are effectively aliases
to the DMA address space - without an IOMMU, the dma_as would be the shared global sysmem/RAM space (address_space_mem, please see pci_init_bus_master())
Quoted 28 lines
I might be missing something because I haven't investigated how to do this myself.
Quoted 24 lines
Signed-off-by Elena Ufimtseva <[email protected]>Signed-off-by John G Johnson <[email protected]>Signed-off-by Jagannathan Raman <[email protected]>---
include/hw/pci/pci.h | 2 + include/hw/remote/iommu.h | 24 ++++++++ hw/pci/pci.c | 2 +- hw/remote/iommu.c | 117 ++++++++++++++++++++++++++++++++++++++ hw/remote/machine.c | 5 ++ hw/remote/vfio-user-obj.c | 20 ++++++- MAINTAINERS | 2 + hw/remote/meson.build | 1 + 8 files changed, 169 insertions(+), 4 deletions(-)create mode 100644
include/hw/remote/iommu.h
create mode 100644hw/remote/iommu.cinclude/hw/pci/pci.h +0
-0@@ -734,6 +734,8 @@ void lsi53c8xx_handle_legacy_cmdline(DeviceState *lsi_dev);qemu_irq pci_allocate_irq (PCIDevice *pci_dev);void pci_set_irq (PCIDevice *pci_dev,int level);
Quoted 3 lines
include/hw/pci/pci.h void lsi53c8xx_handle_legacy_cmdline(DeviceState *lsi_dev); · line 737
Quoted 1 line
include/hw/pci/pci.h +1
-0+void pci_init_bus_master (PCIDevice *pci_dev);This function isn't used in this patch. Why make it public?
We were investigating updating the bus’s address space before the PCI device initialized, but we dropped it as that would be an invasive change. This is cruft from that effort, sorry - will remove.
Quoted 71 lines
Quoted 71 lines
include/hw/pci/pci.h +1
-0+static inline void pci_irq_assert (PCIDevice *pci_dev) {pci_set_irq (pci_dev,1 );include/hw/remote/iommu.h new file +24
-0@@ -0,0 +1,24 @@ +/* +* IOMMU for remote device +* +* Copyright © 2021 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later. +* See the COPYING file in the top-level directory. +* +*/ + +#ifndef REMOTE_IOMMU_H +#define REMOTE_IOMMU_H + +#include "hw/pci/pci_bus.h" + +void remote_iommu_free (PCIDevice *pci_dev); + +void remote_iommu_init (void ); + +void remote_iommu_set (PCIBus *bus); + +MemoryRegion *remote_iommu_get_ram (PCIDevice *pci_dev); + +#endif hw/pci/pci.c +1
-1@@ -95,7 +95,7 @@ static const VMStateDescription vmstate_pcibus = { } }; -static void pci_init_bus_master (PCIDevice *pci_dev) +void pci_init_bus_master (PCIDevice *pci_dev) {AddressSpace *dma_as =pci_device_iommu_address_space (pci_dev);hw/remote/iommu.c new file +17
-0@@ -0,0 +1,117 @@ +/* +* Remote IOMMU +* +* Copyright © 2021 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later. +* See the COPYING file in the top-level directory. +* +*/ + +#include "qemu/osdep.h" +#include "qemu-common.h" + +#include "hw/remote/iommu.h" +#include "hw/pci/pci_bus.h" +#include "exec/memory.h" +#include "exec/address-spaces.h"
Quoted 8 lines
hw/remote/iommu.c line 22
Quoted 5 lines
hw/remote/iommu.c new file +5
-0+#include "trace.h" + +struct VFUIOMMU { +AddressSpace as; +MemoryRegion mr;I guess this is the root MemoryRegion container? Calling it "root" or "
root_mr" instead of "mr" would make that clearer.
OK
Quoted 10 lines
hw/remote/iommu.c line 25
Quoted 4 lines
hw/remote/iommu.c new file +3
-0+}; + +typedef struct VFUPciBus {There is no uppercase/lowercase consistency between VfuObject vs VFUIOMMU vs VFUPciBus. Although the coding standard doesn't dictate ABC vs Abc, please be consistent. I suggest following the VfuObject convention started in the previous patches. The names would be VfuIommu and VfuPciBus.
Sounds good, thank you!
Quoted 175 lines
Quoted 175 lines
hw/remote/iommu.c new file +92
-0+PCIBus *bus; +struct VFUIOMMU *iommu[]; +} VFUPciBus; + +GHashTable *remote_as_table; + +static AddressSpace *remote_iommu_get_as (PCIBus *bus,void *opaque,int devfn) +{ +VFUPciBus *vfu_pci_bus =NULL ; +struct VFUIOMMU *iommu =NULL ; + +if (!remote_as_table) { +return &address_space_memory; + } + + vfu_pci_bus =g_hash_table_lookup (remote_as_table, bus); + +if (!vfu_pci_bus) { + vfu_pci_bus =g_malloc0 (sizeof (VFUPciBus)); + vfu_pci_bus->bus = bus; +g_hash_table_insert (remote_as_table, bus, vfu_pci_bus); + } + + iommu = vfu_pci_bus->iommu[devfn]; + +if (!iommu) { + g_autofree char *mr_name = g_strdup_printf("vfu-ram-%d", devfn); + g_autofree char *as_name = g_strdup_printf("vfu-as-%d", devfn); + + iommu =g_malloc0 (sizeof (struct VFUIOMMU )); + +memory_region_init (&iommu->mr,NULL , mr_name,UINT64_MAX ); +address_space_init (&iommu->as, &iommu->mr, as_name); + + vfu_pci_bus->iommu[devfn] = iommu; + } + +return &iommu->as; +} + +void remote_iommu_free (PCIDevice *pci_dev) +{ +VFUPciBus *vfu_pci_bus =NULL ; +struct VFUIOMMU *iommu =NULL ; + +if (!remote_as_table) { +return ; + } + + vfu_pci_bus =g_hash_table_lookup (remote_as_table,pci_get_bus (pci_dev)); + +if (!vfu_pci_bus) { +return ; + } + + iommu = vfu_pci_bus->iommu[pci_dev->devfn]; + + vfu_pci_bus->iommu[pci_dev->devfn] =NULL ; + +if (iommu) { +memory_region_unref (&iommu->mr); +address_space_destroy (&iommu->as); +g_free (iommu); + } +} + +void remote_iommu_init (void ) +{ + remote_as_table =g_hash_table_new_full (NULL ,NULL ,NULL ,NULL ); +} + +void remote_iommu_set (PCIBus *bus) +{ +pci_setup_iommu (bus, remote_iommu_get_as,NULL ); +} + +MemoryRegion *remote_iommu_get_ram (PCIDevice *pci_dev) +{ +PCIBus *bus =pci_get_bus (pci_dev); +VFUPciBus *vfu_pci_bus; + +if (!remote_as_table) { +return get_system_memory (); + } + + vfu_pci_bus =g_hash_table_lookup (remote_as_table, bus); +if (!vfu_pci_bus) { +return get_system_memory (); + } + +return &vfu_pci_bus->iommu[pci_dev->devfn]->mr; +}hw/remote/machine.c +5
-0@@ -21,6 +21,7 @@#include "qapi/error.h" #include "hw/pci/pci_host.h" #include "hw/remote/iohub.h" +#include "hw/remote/iommu.h" static void remote_machine_init (MachineState *machine) {@@ -52,6 +53,10 @@ static void remote_machine_init(MachineState *machine)remote_iohub_init (&s->iohub); +remote_iommu_init (); + +remote_iommu_set (pci_host->bus); +pci_bus_irqs (pci_host->bus, remote_iohub_set_irq, remote_iohub_map_irq, &s->iohub,REMOTE_IOHUB_NB_PIRQS ); }hw/remote/vfio-user-obj.c +7
-3@@ -49,6 +49,7 @@#include "hw/qdev-core.h" #include "hw/pci/pci.h" #include "qemu/timer.h" +#include "hw/remote/iommu.h" #define TYPE_VFU_OBJECT "x-vfio-user-server"OBJECT_DECLARE_TYPE (VfuObject, VfuObjectClass,VFU_OBJECT )@@ -210,6 +211,7 @@ static ssize_t vfu_object_cfg_access(vfu_ctx_t *vfu_ctx, char * const buf,static void dma_register (vfu_ctx_t *vfu_ctx,vfu_dma_info_t *info) { +VfuObject *o =vfu_get_private (vfu_ctx);MemoryRegion *subregion =NULL ; g_autofree char *name = NULL;static unsigned int suffix;@@ -226,14 +228,15 @@ static void dma_register(vfu_ctx_t *vfu_ctx, vfu_dma_info_t *info)memory_region_init_ram_ptr (subregion,NULL , name, iov->iov_len, info->vaddr); -memory_region_add_subregion (get_system_memory (), (hwaddr )iov->iov_base, - subregion); +memory_region_add_subregion (remote_iommu_get_ram (o->pci_dev), + (hwaddr )iov->iov_base, subregion);trace_vfu_dma_register ((uint64_t )iov->iov_base, iov->iov_len); }static void dma_unregister (vfu_ctx_t *vfu_ctx,vfu_dma_info_t *info) { +VfuObject *o =vfu_get_private (vfu_ctx);MemoryRegion *mr =NULL ;ram_addr_t offset;@@ -242,7 +245,7 @@ static void dma_unregister(vfu_ctx_t *vfu_ctx, vfu_dma_info_t *info)return ; } -memory_region_del_subregion (get_system_memory (), mr); +memory_region_del_subregion (remote_iommu_get_ram (o->pci_dev), mr);object_unparent ((OBJECT (mr)));@@ -320,6 +323,7 @@ static vfu_region_access_cb_t *vfu_object_bar_handlers[PCI_NUM_REGIONS] = {*/ static void vfu_object_register_bars (vfu_ctx_t *vfu_ctx,PCIDevice *pdev) { +VfuObject *o =vfu_get_private (vfu_ctx);int i;for (i =0 ; i <PCI_NUM_REGIONS ; i++) {@@ -332,6 +336,12 @@ static void vfu_object_register_bars(vfu_ctx_t *vfu_ctx, PCIDevice *pdev) vfu_object_bar_handlers[i], VFU_REGION_FLAG_RW, NULL, 0, -1, 0);
Quoted 9 lines
hw/remote/vfio-user-obj.c static void vfu_object_register_bars(vfu_ctx_t *vfu_ctx, PCIDevice *pdev) · line 343
Quoted 5 lines
hw/remote/vfio-user-obj.c +5
-0+if ((o->pci_dev->io_regions[i].type &PCI_BASE_ADDRESS_SPACE ) ==0 ) { +memory_region_unref (o->pci_dev->io_regions[i].address_space); + o->pci_dev->io_regions[i].address_space = +remote_iommu_get_ram (o->pci_dev); + }This looks hacky. If you create a separate PCIHost for each device instead then the BARs will be created in the MemoryRegion (confusingly named "
address_space" in the PCI code) of your choosing.
I was also not very comfortable with this - added it very grudgingly out of necessity. Thank god this can go away with separate bus for each device.
Quoted 3 lines
Also, why is PCI Memory Space isolated via VFUIOMMU but PCI IO Space is not?
If I understand correctly, the IO address space translates sysmem address to direct device access (such as I2C). Once we are inside a device, we already have access to all parts of the device (unlike RAM which sits outside the device). So didn’t think device would go via IOMMU to access IO. Also didn’t see any other IOMMU translating IO address space accesses.
Thank you!
--
Jag
Re: [PATCH v4 07/14] vfio-user: run vfio-user context
In reply to Jag RamanQuoted 28 lines · Jag Raman
Quoted 19 lines · Stefan Hajnoczi
Quoted 14 lines · Jagannathan Raman
hw/remote/vfio-user-obj.c +10
-0@@ -114,6 +118,62 @@ static void vfu_object_set_device(Object *obj, const char *str, Error **errp)vfu_object_init_ctx (o, errp); } +static void vfu_object_ctx_run (void *opaque) +{ +VfuObject *o = opaque; +int ret =-1 ; + +while (ret !=0 ) { + ret =vfu_run_ctx (o->vfu_ctx); +if (ret <0 ) { +if (errno ==EINTR ) { +continue ;Quoted 8 lines
hw/remote/vfio-user-obj.c static void vfu_object_set_device(Object *obj, const char *str, Error **errp) · line 135
Quoted 5 lines
hw/remote/vfio-user-obj.c +5
-0+ }else if (errno ==ENOTCONN ) { +qemu_set_fd_handler (o->vfu_poll_fd,NULL ,NULL ,NULL ); + o->vfu_poll_fd =-1 ; +object_unparent (OBJECT (o)); +break ;If nothing else logs a message then I think that should be done here so users know why their vfio-user server object disappeared.
Quoted 5 lines
Sure will do.
Do you prefer a trace, or a message to the console? Trace makes sense to me. Presently, the client could unplug the vfio-user device which would trigger the deletion of this object. This process could happen quietly.
If there is no way to differentiate graceful disconnect from unexpected disconnect then logging might be too noisy.
Regarding the automatic deletion of the object, that might not be desirable for two reasons:
1. It prevents reconnection or another client connecting.
2. Management tools are in the dark about it.
For #2 there are monitor events that QEMU emits to notify management tools about state changes like disconnections.
It's worth thinking about current and future use cases before baking in a policy like automatically deleting VfuObject on disconnect because it's inflexible and would require a QEMU update in the future to support a different policy.
One approach is to emit a disconnect event but leave the VfuObject in a disconnected state. The management tool can then restart or clean up, depending on its policy.
I'm not sure what's best because it depends on the use cases, but maybe you and others have ideas here.
Quoted 12 lines
Quoted 11 lines
hw/remote/vfio-user-obj.c static void vfu_object_init(Object *obj) · line 289
Quoted 7 lines
hw/remote/vfio-user-obj.c +2
-0@@ -208,6 +284,8 @@ static void vfu_object_init(Object *obj) TYPE_VFU_OBJECT, TYPE_REMOTE_MACHINE);return ; } + + o->vfu_poll_fd =-1 ; }This must call
qemu_set_fd_handler(o->vfu_poll_fd, NULL, NULL, NULL)wheno->vfu_poll_fd!= -1 to avoid leaving a dangling fd handler callback registered.
Quoted 5 lines
This is during the init phase, and the FD handlers are not set. Do you mean to add this at finalize?
I agree it’s good to explicitly add this at finalize. But
vfu_destroy_ctx()should trigger a ENOTCONN, which would do it anyway.
I'm not sure my comment makes sense since this is the init function, not finalize.
However, it's not clear to me that the o->vfu_poll_fd fd handler is unregistered from the event loop when VfuObject is finalized (e.g. by the object-del monitor command). You say vfu_destroy_ctx() triggers ENOTCONN, but the VfuObject is freed after finalize returns so when is the fd handler deregistered?
Stefan
Re: [PATCH v4 11/14] vfio-user: IOMMU support for remote device
In reply to Jag RamanQuoted 23 lines · Jag Raman
Quoted 9 lines · Stefan Hajnoczi
Quoted 1 line · Jagannathan Raman
Assign separate address space for each device in the remote processes.
If I understand correctly this isn't really an IOMMU. It's abusing the IOMMU APIs to create isolated address spaces for each device. This way memory regions added by the vfio-user client do not conflict when there are multiple vfio-user servers.
Like you already figured out, having isolated DMA address space alone is not sufficient for this application, we also needed to isolate the sysmem/RAM address space. As such, the available IOMMU APIs alone were not sufficient, so we had to improvise.
Quoted 7 lines
Calling
pci_root_bus_new()and keeping one PCI bus per VfuObject might be a cleaner approach:
- Lets you isolate both PCI Memory Space and IO Space.
- Isolates the PCIDevices and their addresses on the bus.
- Isolates irqs.
- No more need to abuse the IOMMU API.
Quoted 7 lines
I believe we would still need to have an IOMMU. It’s because, devices use the pci_dma_read()/_write() functions. These functions look up the address in DMA address space (via
pci_get_address_space()->PCIDevice->bus_master_as->PCIDevice->bus_master_enable_region->PCIDevice->bus_master_container_region).
bus_master_enable_regionandbus_master_container_regionare effectively aliases
to the DMA address space - without an IOMMU, thedma_aswould be the shared global sysmem/RAM space (address_space_mem, please seepci_init_bus_master())
Good point, that code assumes there is a global address space. Creating a fake IOMMU works around that assumption but it seems cleaner to eliminate it:
AddressSpace *pci_device_iommu_address_space(PCIDevice *dev)
{
...
if (!pci_bus_bypass_iommu(bus) && iommu_bus && iommu_bus->iommu_fn) {
return iommu_bus->iommu_fn(bus, iommu_bus->iommu_opaque, devfn);
}
return &address_space_memory;
^^^^^^^^^^^^^^^^^^^^When creating a PCI root bus an AddressSpace argument could be provided, just like it already does for the address_space_memory and address_space_io MemoryRegions. Then the hardcoded return can be changed to something like:
return bus->dma_address_space;
Quoted 14 lines
Quoted 4 lines
Quoted 4 lines
hw/remote/vfio-user-obj.c +0
-0@@ -332,6 +336,12 @@ static void vfu_object_register_bars(vfu_ctx_t *vfu_ctx, PCIDevice *pdev) vfu_object_bar_handlers[i], VFU_REGION_FLAG_RW, NULL, 0, -1, 0);Quoted 9 lines
hw/remote/vfio-user-obj.c static void vfu_object_register_bars(vfu_ctx_t *vfu_ctx, PCIDevice *pdev) · line 343
Quoted 5 lines
hw/remote/vfio-user-obj.c +5
-0+if ((o->pci_dev->io_regions[i].type &PCI_BASE_ADDRESS_SPACE ) ==0 ) { +memory_region_unref (o->pci_dev->io_regions[i].address_space); + o->pci_dev->io_regions[i].address_space = +remote_iommu_get_ram (o->pci_dev); + }This looks hacky. If you create a separate PCIHost for each device instead then the BARs will be created in the MemoryRegion (confusingly named "
address_space" in the PCI code) of your choosing.
Quoted 2 lines
I was also not very comfortable with this - added it very grudgingly out of necessity. Thank god this can go away with separate bus for each device.
I talked to Kevin Wolf about having separate busses. qdev currently requires each DeviceState to have a parent bus and each bus must have a parent DeviceState. There is only one exception: a special check that allows the global system bus (sysbus_get_default()) to be created without a parent DeviceState.
This restriction probably needs to be loosened in order to support an isolated PCIHost for each vfio-user server. The challenge is that qdev_find_recursive() and monitor commands like device_add currently only search the global system bus. Maybe new syntax is needed for the multiple root bus case or the behavior of existing monitor commands needs to be understood and extended without breaking anything.
Quoted 9 lines
Quoted 3 lines
Also, why is PCI Memory Space isolated via VFUIOMMU but PCI IO Space is not?
If I understand correctly, the IO address space translates sysmem address to direct device access (such as I2C). Once we are inside a device, we already have access to all parts of the device (unlike RAM which sits outside the device). So didn’t think device would go via IOMMU to access IO. Also didn’t see any other IOMMU translating IO address space accesses.
I reviewed how BARs are configured with VFIO:
1. When the guest writes to the vfio-pci PCIDevice's Configuration Space the write is forwarded to the VFIO device (i.e. vfio-user or VFIO kernel ioctl).
2. The vfio-user server receives the Configuration Space write and forwards it to pci_dev (the PCIDevice we're serving up). BAR mappings are updated in the vfio-user server so the BAR MemoryRegions are mapped/unmapped at the locations given by the guest.
This applies for both Memory and IO Space accesses.
Because this patch series does not isolate IO Space between VfuObject instances the MemoryRegions will collide when two guests map IO Space BARs of different devices at the same IO Space address. In other words, vfu_object_bar_rw() uses the global address_space_io and that means collisions can occur.
Stefan
Re: [PATCH v4 07/14] vfio-user: run vfio-user context
In reply to Stefan HajnocziQuoted 33 lines · Stefan Hajnoczi
Quoted 28 lines · Jag Raman
Quoted 27 lines · Stefan Hajnoczi
Quoted 9 lines · Jagannathan Raman
hw/remote/vfio-user-obj.c +5
-0@@ -114,6 +118,62 @@ static void vfu_object_set_device(Object *obj, const char *str, Error **errp)vfu_object_init_ctx (o, errp); } +static void vfu_object_ctx_run (void *opaque) +{ +VfuObject *o = opaque; +int ret =-1 ; +hw/remote/vfio-user-obj.c static void vfu_object_set_device(Object *obj, const char *str, Error **errp) · line 135
Quoted 10 lines
hw/remote/vfio-user-obj.c +10
-0+while (ret !=0 ) { + ret =vfu_run_ctx (o->vfu_ctx); +if (ret <0 ) { +if (errno ==EINTR ) { +continue ; + }else if (errno ==ENOTCONN ) { +qemu_set_fd_handler (o->vfu_poll_fd,NULL ,NULL ,NULL ); + o->vfu_poll_fd =-1 ; +object_unparent (OBJECT (o)); +break ;If nothing else logs a message then I think that should be done here so users know why their vfio-user server object disappeared.
Quoted 8 lines
Quoted 5 lines
Sure will do.
Do you prefer a trace, or a message to the console? Trace makes sense to me. Presently, the client could unplug the vfio-user device which would trigger the deletion of this object. This process could happen quietly.
If there is no way to differentiate graceful disconnect from unexpected disconnect then logging might be too noisy.
I think that’s what happens in the regular VFIO case also. vfio_put_base_device() closes the FD used for ioctls.
Quoted 8 lines
Regarding the automatic deletion of the object, that might not be desirable for two reasons:
1. It prevents reconnection or another client connecting.
2. Management tools are in the dark about it.For #2 there are monitor events that QEMU emits to notify management tools about state changes like disconnections.
This is very interesting. I suppose you’re referring to something like ‘BLOCK_JOB_COMPLETED’ event.
It’d be good to inform the management tools about disconnection. Not used this before, will check it out to gather ideas on how to use it.
Quoted 31 lines
It's worth thinking about current and future use cases before baking in a policy like automatically deleting VfuObject on disconnect because it's inflexible and would require a QEMU update in the future to support a different policy.
One approach is to emit a disconnect event but leave the VfuObject in a disconnected state. The management tool can then restart or clean up, depending on its policy.
I'm not sure what's best because it depends on the use cases, but maybe you and others have ideas here.
Quoted 12 lines
Quoted 11 lines
hw/remote/vfio-user-obj.c static void vfu_object_init(Object *obj) · line 289
Quoted 7 lines
hw/remote/vfio-user-obj.c +2
-0@@ -208,6 +284,8 @@ static void vfu_object_init(Object *obj) TYPE_VFU_OBJECT, TYPE_REMOTE_MACHINE);return ; } + + o->vfu_poll_fd =-1 ; }This must call
qemu_set_fd_handler(o->vfu_poll_fd, NULL, NULL, NULL)wheno->vfu_poll_fd!= -1 to avoid leaving a dangling fd handler callback registered.Quoted 5 lines
This is during the init phase, and the FD handlers are not set. Do you mean to add this at finalize?
I agree it’s good to explicitly add this at finalize. But
vfu_destroy_ctx()should trigger a ENOTCONN, which would do it anyway.
Quoted 8 lines
I'm not sure my comment makes sense since this is the init function, not finalize.
However, it's not clear to me that the
o->vfu_poll_fdfd handler is unregistered from the event loop when VfuObject is finalized (e.g. by the object-del monitor command). You sayvfu_destroy_ctx()triggers ENOTCONN, but the VfuObject is freed after finalize returns so when is the fd handler deregistered?
That is correct - will remove the FD handler in finalize also.
Thank you!
--
Jag
Quoted 2 lines
Stefan
Re: [PATCH v4 11/14] vfio-user: IOMMU support for remote device
In reply to Stefan HajnocziQuoted 49 lines · Stefan Hajnoczi
Quoted 23 lines · Jag Raman
Quoted 9 lines · Stefan Hajnoczi
Quoted 1 line · Jagannathan Raman
Assign separate address space for each device in the remote processes.
If I understand correctly this isn't really an IOMMU. It's abusing the IOMMU APIs to create isolated address spaces for each device. This way memory regions added by the vfio-user client do not conflict when there are multiple vfio-user servers.
Like you already figured out, having isolated DMA address space alone is not sufficient for this application, we also needed to isolate the sysmem/RAM address space. As such, the available IOMMU APIs alone were not sufficient, so we had to improvise.
Quoted 7 lines
Calling
pci_root_bus_new()and keeping one PCI bus per VfuObject might be a cleaner approach:
- Lets you isolate both PCI Memory Space and IO Space.
- Isolates the PCIDevices and their addresses on the bus.
- Isolates irqs.
- No more need to abuse the IOMMU API.Quoted 7 lines
I believe we would still need to have an IOMMU. It’s because, devices use the pci_dma_read()/_write() functions. These functions look up the address in DMA address space (via
pci_get_address_space()->PCIDevice->bus_master_as->PCIDevice->bus_master_enable_region->PCIDevice->bus_master_container_region).bus_master_enable_regionandbus_master_container_regionare effectively aliases to the DMA address space - without an IOMMU, thedma_aswould be the shared global sysmem/RAM space (address_space_mem, please seepci_init_bus_master())Good point, that code assumes there is a global address space. Creating a fake IOMMU works around that assumption but it seems cleaner to eliminate it:
AddressSpace *pci_device_iommu_address_space(PCIDevice *dev)
{ ... if (!pci_bus_bypass_iommu(bus) && iommu_bus && iommu_bus->iommu_fn) { return iommu_bus->iommu_fn(bus, iommu_bus->iommu_opaque, devfn); } return &address_space_memory; ^^^^^^^^^^^^^^^^^^^^
Quoted 6 lines
When creating a PCI root bus an AddressSpace argument could be provided, just like it already does for the
address_space_memoryandaddress_space_ioMemoryRegions. Then the hardcoded return can be changed to something like:return
bus->dma_address_space;
This approach should work when we are using separate PCIBus for each PCIDevice.
Quoted 24 lines
Quoted 15 lines
Quoted 5 lines
Quoted 5 lines
hw/remote/vfio-user-obj.c +0
-0@@ -332,6 +336,12 @@ static void vfu_object_register_bars(vfu_ctx_t *vfu_ctx, PCIDevice *pdev) vfu_object_bar_handlers[i], VFU_REGION_FLAG_RW, NULL, 0, -1, 0);Quoted 9 lines
hw/remote/vfio-user-obj.c static void vfu_object_register_bars(vfu_ctx_t *vfu_ctx, PCIDevice *pdev) · line 343
Quoted 5 lines
hw/remote/vfio-user-obj.c +5
-0+if ((o->pci_dev->io_regions[i].type &PCI_BASE_ADDRESS_SPACE ) ==0 ) { +memory_region_unref (o->pci_dev->io_regions[i].address_space); + o->pci_dev->io_regions[i].address_space = +remote_iommu_get_ram (o->pci_dev); + }This looks hacky. If you create a separate PCIHost for each device instead then the BARs will be created in the MemoryRegion (confusingly named "
address_space" in the PCI code) of your choosing.Quoted 2 lines
I was also not very comfortable with this - added it very grudgingly out of necessity. Thank god this can go away with separate bus for each device.
I talked to Kevin Wolf about having separate busses. qdev currently requires each DeviceState to have a parent bus and each bus must have a parent DeviceState. There is only one exception: a special check that allows the global system bus (
sysbus_get_default()) to be created without a parent DeviceState.
Quoted 6 lines
This restriction probably needs to be loosened in order to support an isolated PCIHost for each vfio-user server. The challenge is that
qdev_find_recursive()and monitor commands likedevice_addcurrently only search the global system bus. Maybe new syntax is needed for the multiple root bus case or the behavior of existing monitor commands needs to be understood and extended without breaking anything.
Lemme check if it’s possible to create multiple PCIBuses within the global system bus, something similar to what PCI expansion cards are doing. That would help avoid the complexities you just mentioned.
Quoted 22 lines
Quoted 10 lines
Quoted 4 lines
Also, why is PCI Memory Space isolated via VFUIOMMU but PCI IO Space is not?
If I understand correctly, the IO address space translates sysmem address to direct device access (such as I2C). Once we are inside a device, we already have access to all parts of the device (unlike RAM which sits outside the device). So didn’t think device would go via IOMMU to access IO. Also didn’t see any other IOMMU translating IO address space accesses.
I reviewed how BARs are configured with VFIO:
1. When the guest writes to the vfio-pci PCIDevice's Configuration Space the write is forwarded to the VFIO device (i.e. vfio-user or VFIO kernel ioctl).
2. The vfio-user server receives the Configuration Space write and forwards it to
pci_dev(the PCIDevice we're serving up). BAR mappings are updated in the vfio-user server so the BAR MemoryRegions are mapped/unmapped at the locations given by the guest.
Quoted 7 lines
This applies for both Memory and IO Space accesses.
Because this patch series does not isolate IO Space between VfuObject instances the MemoryRegions will collide when two guests map IO Space BARs of different devices at the same IO Space address. In other words,
vfu_object_bar_rw()uses the globaladdress_space_ioand that means collisions can occur.
I agree that collision could occur from the CPU end. But I'm not if IOMMU needs to translate IO space.
Thank you!
--
Jag
Quoted 2 lines
Stefan
RE: [PATCH v4 07/14] vfio-user: run vfio-user context
In reply to Jag RamanQuoted 102 lines
-----Original Message-----
From: Jag Raman <[email protected]>
Sent: 17 December 2021 18:00
To: Stefan Hajnoczi <[email protected]>; John Levon <[email protected]>; Thanos Makatos <[email protected]>
Cc: qemu-devel <[email protected]>; Alex Williamson <[email protected]>; Marc-André Lureau <[email protected]>; Philippe Mathieu-Daudé <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; Elena Ufimtseva <[email protected]>; John Levon <[email protected]>; John Johnson <[email protected]>; Thanos Makatos <[email protected]>; Swapnil Ingle <[email protected]>
Subject: Re: [PATCH v4 07/14] vfio-user: run vfio-user contextQuoted 4 lines · Stefan Hajnoczi
Review static void vfu_object_set_device(Object *obj,
Quoted 1 line · Jagannathan Raman
(quoted diff) +0
-0@@ -114,6 +118,62 @@ static void vfu_object_set_device(Object *obj,const char *str, Error **errp)
Quoted 13 lines
Quoted 13 lines
hw/remote/vfio-user-obj.c +10
-0vfu_object_init_ctx (o, errp); } +static void vfu_object_ctx_run (void *opaque) +{ +VfuObject *o = opaque; +int ret =-1 ; + +while (ret !=0 ) { + ret =vfu_run_ctx (o->vfu_ctx); +if (ret <0 ) { +if (errno ==EINTR ) { +continue ;Quoted 8 lines
hw/remote/vfio-user-obj.c static void vfu_object_set_device(Object *obj, const char *str, Error **errp) · line 135
Quoted 5 lines
hw/remote/vfio-user-obj.c +5
-0+ }else if (errno ==ENOTCONN ) { +qemu_set_fd_handler (o->vfu_poll_fd,NULL ,NULL ,NULL ); + o->vfu_poll_fd =-1 ; +object_unparent (OBJECT (o)); +break ;If nothing else logs a message then I think that should be done here so users know why their vfio-user server object disappeared.
Sure will do.
Do you prefer a trace, or a message to the console? Trace makes sense to me. Presently, the client could unplug the vfio-user device which would trigger the deletion of this object. This process could happen quietly.
Quoted 8 lines
hw/remote/vfio-user-obj.c static void vfu_object_set_device(Object *obj, const char *str, Error **errp) · line 138
Quoted 4 lines
hw/remote/vfio-user-obj.c +3
-0+ }else { +error_setg (&error_abort,"vfu: Failed to run device %s - %s" , + o->device,strerror (errno));
error_abortis equivalent to assuming !o->daemon. In the case where the user doesn't want to automatically shut down the process we need to log a message without aborting.OK, makes sense.
Quoted 32 lines
Review static void vfu_object_set_device(Object *obj, · line 139
Quoted 2 lines
(quoted diff) +1
-0+ break;Indentation is off.
Quoted 11 lines
hw/remote/vfio-user-obj.c +11
-0+ } + } + } +} + +static void vfu_object_attach_ctx (void *opaque) +{ +VfuObject *o = opaque; +GPollFD pfds[1 ]; +int ret; +hw/remote/vfio-user-obj.c static void vfu_object_set_device(Object *obj, const char *str, Error **errp) · line 160
Quoted 10 lines
hw/remote/vfio-user-obj.c +10
-0+qemu_set_fd_handler (o->vfu_poll_fd,NULL ,NULL ,NULL ); + + pfds[0 ].fd = o->vfu_poll_fd; + pfds[0 ].events =G_IO_IN |G_IO_HUP |G_IO_ERR ; + +retry_attach: + ret =vfu_attach_ctx (o->vfu_ctx); +if (ret <0 && (errno ==EAGAIN || errno ==EWOULDBLOCK )) { +qemu_poll_ns (pfds,1 ,500 * (int64_t )SCALE_MS ); + goto retry_attach;This can block the thread indefinitely. Other events like monitor commands are not handled in this loop. Please make this asynchronous (set an fd handler and return from this function so we can try again later).
Quoted 5 lines
The
vfu_attach_ctx()implementation synchronously negotiates the vfio-user connection :(. That's a shame because even if accept(2) is handled asynchronously, the negotiation can still block. It would be cleanest to have a fully async libvfio-user'svfu_attach_ctx()API to avoid blocking. Is that possible?
Quoted 3 lines
Thanos / John,
Any thoughts on this?
I'm discussing this with John and FYI there are other places where libvfio-user can block, e.g. sending a response or receiving a command. Is it just the negotiation you want it to be asynchronous or _all_ libvfio-user operations? Making libvfio-user fully asynchronous might require a substantial API rewrite.
Quoted 57 lines
Quoted 9 lines
If libvfio-user can't make
vfu_attach_ctx()fully async then it may be possible to spawn a thread just for the blockingvfu_attach_ctx()call and report the result back to the event loop (e.g. using EventNotifier). That adds a bunch of code to work around a blocking API though, so I guess we can leave the blocking part if necessary.At the very minimum, please make EAGAIN/EWOULDBLOCK async as
mentioned
Quoted 4 lines
above and add a comment explaining the situation with the partially-async
vfu_attach_ctx()API so it's clear that this can block (that way it's clear that you're aware of the issue and this isn't by accident).Sure, we could make the attach async at QEMU depending on how the library prefers to do this.
Quoted 18 lines
hw/remote/vfio-user-obj.c static void vfu_object_set_device(Object *obj, const char *str, Error **errp) · line 164
Quoted 5 lines
hw/remote/vfio-user-obj.c +4
-0+ }else if (ret <0 ) { +error_setg (&error_abort, +"vfu: Failed to attach device %s to context - %s" , + o->device,strerror (errno));
error_abortassumes !o->daemon. Please handle theo->daemon== true case by logging an error without aborting.hw/remote/vfio-user-obj.c static void vfu_object_set_device(Object *obj, const char *str, Error **errp) · line 170
Quoted 6 lines
hw/remote/vfio-user-obj.c +6
-0+return ; + } + + o->vfu_poll_fd =vfu_get_poll_fd (o->vfu_ctx); +if (o->vfu_poll_fd <0 ) { +error_setg (&error_abort,"vfu: Failed to get poll fd %s" , o->device);Same here.
Quoted 11 lines
hw/remote/vfio-user-obj.c static void vfu_object_init(Object *obj) · line 289
Quoted 7 lines
hw/remote/vfio-user-obj.c +2
-0@@ -208,6 +284,8 @@ static void vfu_object_init(Object *obj) TYPE_VFU_OBJECT, TYPE_REMOTE_MACHINE);return ; } + + o->vfu_poll_fd =-1 ; }This must call
qemu_set_fd_handler(o->vfu_poll_fd, NULL, NULL, NULL)wheno->vfu_poll_fd!= -1 to avoid leaving a dangling fd handler callback registered.This is during the init phase, and the FD handlers are not set. Do you mean to add this at finalize?
I agree it’s good to explicitly add this at finalize. But
vfu_destroy_ctx()should trigger a ENOTCONN, which would do it anyway.Thank you!
--
Jag
Re: [PATCH v4 11/14] vfio-user: IOMMU support for remote device
In reply to Jag RamanQuoted 26 lines · Jag Raman
Quoted 26 lines · Stefan Hajnoczi
Quoted 10 lines · Jag Raman
Quoted 4 lines · Stefan Hajnoczi
On Wed, Dec 15, 2021 at 10:35:35AM -0500, Jagannathan Raman wrote:
Also, why is PCI Memory Space isolated via VFUIOMMU but PCI IO Space is not?If I understand correctly, the IO address space translates sysmem address to direct device access (such as I2C). Once we are inside a device, we already have access to all parts of the device (unlike RAM which sits outside the device). So didn’t think device would go via IOMMU to access IO. Also didn’t see any other IOMMU translating IO address space accesses.
I reviewed how BARs are configured with VFIO:
1. When the guest writes to the vfio-pci PCIDevice's Configuration Space the write is forwarded to the VFIO device (i.e. vfio-user or VFIO kernel ioctl).
2. The vfio-user server receives the Configuration Space write and forwards it to
pci_dev(the PCIDevice we're serving up). BAR mappings are updated in the vfio-user server so the BAR MemoryRegions are mapped/unmapped at the locations given by the guest.This applies for both Memory and IO Space accesses.
Quoted 8 lines
Quoted 5 lines
Because this patch series does not isolate IO Space between VfuObject instances the MemoryRegions will collide when two guests map IO Space BARs of different devices at the same IO Space address. In other words,
vfu_object_bar_rw()uses the globaladdress_space_ioand that means collisions can occur.I agree that collision could occur from the CPU end. But I'm not if IOMMU needs to translate IO space.
QEMU's IOMMUs do not translate IO Space addresses AFAIK.
IO Space just needs to be isolated between vfio-user server instances so there is no collision when one client maps an IO Space BAR to the same address as another client.
I think the cleanest way of achieving that is by creating a per-vfio-user server PCI bus with an address_space_io MemoryRegion.
Stefan
Re: [PATCH v4 07/14] vfio-user: run vfio-user context
In reply to Thanos MakatosQuoted 102 lines · Thanos Makatos
Quoted 102 lines
-----Original Message-----
From: Jag Raman <[email protected]>
Sent: 17 December 2021 18:00
To: Stefan Hajnoczi <[email protected]>; John Levon <[email protected]>; Thanos Makatos <[email protected]>
Cc: qemu-devel <[email protected]>; Alex Williamson <[email protected]>; Marc-André Lureau <[email protected]>; Philippe Mathieu-Daudé <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; Elena Ufimtseva <[email protected]>; John Levon <[email protected]>; John Johnson <[email protected]>; Thanos Makatos <[email protected]>; Swapnil Ingle <[email protected]>
Subject: Re: [PATCH v4 07/14] vfio-user: run vfio-user contextQuoted 4 lines · Stefan Hajnoczi
Review static void vfu_object_set_device(Object *obj,
Quoted 1 line · Jagannathan Raman
(quoted diff) +0
-0@@ -114,6 +118,62 @@ static void vfu_object_set_device(Object *obj,const char *str, Error **errp)
Quoted 13 lines
Quoted 13 lines
hw/remote/vfio-user-obj.c +10
-0vfu_object_init_ctx (o, errp); } +static void vfu_object_ctx_run (void *opaque) +{ +VfuObject *o = opaque; +int ret =-1 ; + +while (ret !=0 ) { + ret =vfu_run_ctx (o->vfu_ctx); +if (ret <0 ) { +if (errno ==EINTR ) { +continue ;Quoted 8 lines
hw/remote/vfio-user-obj.c static void vfu_object_set_device(Object *obj, const char *str, Error **errp) · line 135
Quoted 5 lines
hw/remote/vfio-user-obj.c +5
-0+ }else if (errno ==ENOTCONN ) { +qemu_set_fd_handler (o->vfu_poll_fd,NULL ,NULL ,NULL ); + o->vfu_poll_fd =-1 ; +object_unparent (OBJECT (o)); +break ;If nothing else logs a message then I think that should be done here so users know why their vfio-user server object disappeared.
Sure will do.
Do you prefer a trace, or a message to the console? Trace makes sense to me. Presently, the client could unplug the vfio-user device which would trigger the deletion of this object. This process could happen quietly.
Quoted 8 lines
hw/remote/vfio-user-obj.c static void vfu_object_set_device(Object *obj, const char *str, Error **errp) · line 138
Quoted 4 lines
hw/remote/vfio-user-obj.c +3
-0+ }else { +error_setg (&error_abort,"vfu: Failed to run device %s - %s" , + o->device,strerror (errno));
error_abortis equivalent to assuming !o->daemon. In the case where the user doesn't want to automatically shut down the process we need to log a message without aborting.OK, makes sense.
Quoted 37 lines
Review static void vfu_object_set_device(Object *obj, · line 139
Quoted 2 lines
(quoted diff) +1
-0+ break;Indentation is off.
Quoted 11 lines
hw/remote/vfio-user-obj.c +11
-0+ } + } + } +} + +static void vfu_object_attach_ctx (void *opaque) +{ +VfuObject *o = opaque; +GPollFD pfds[1 ]; +int ret; +hw/remote/vfio-user-obj.c static void vfu_object_set_device(Object *obj, const char *str, Error **errp) · line 160
Quoted 10 lines
hw/remote/vfio-user-obj.c +10
-0+qemu_set_fd_handler (o->vfu_poll_fd,NULL ,NULL ,NULL ); + + pfds[0 ].fd = o->vfu_poll_fd; + pfds[0 ].events =G_IO_IN |G_IO_HUP |G_IO_ERR ; + +retry_attach: + ret =vfu_attach_ctx (o->vfu_ctx); +if (ret <0 && (errno ==EAGAIN || errno ==EWOULDBLOCK )) { +qemu_poll_ns (pfds,1 ,500 * (int64_t )SCALE_MS ); + goto retry_attach;This can block the thread indefinitely. Other events like monitor commands are not handled in this loop. Please make this asynchronous (set an fd handler and return from this function so we can try again later).
The
vfu_attach_ctx()implementation synchronously negotiates the vfio-user connection :(. That's a shame because even if accept(2) is handled asynchronously, the negotiation can still block. It would be cleanest to have a fully async libvfio-user'svfu_attach_ctx()API to avoid blocking. Is that possible?
Quoted 5 lines
Quoted 3 lines
Thanos / John,
Any thoughts on this?
I'm discussing this with John and FYI there are other places where libvfio-user can block, e.g. sending a response or receiving a command. Is it just the negotiation you want it to be asynchronous or _all_ libvfio-user operations? Making libvfio-user fully asynchronous might require a substantial API rewrite.
I see at least two reasons for a fully async API:
1. The program wants to handle other events (e.g. a management REST API) from the same event loop thread that invokes libvfio-user. If libvfio-user blocks then the other events cannot be handled within a reasonable time frame.
The workaround for this is to use multi-threading and ignore the event-driven architecture implied by vfu_get_poll_fd().
2. The program handles multiple clients that do not trust each other. This could be a software-defined network switch or storage appliance. A malicious client can cause a denial-of-service by making a libvfio-user call block.
Again, the program needs separate threads instead of an event loop to work around this.
The downside to a sync approach is that programs that already have an event loop require extra code to set up dedicated threads for libvfio-user. That's a library integration/usability issue.
In some cases it's okay to block: when the program doesn't need to handle other events. If most users of libvfio-user are expected to fall into this category then there's no need to change the API.
Either way, the doc comments in libvfio-user.h aren't very clear. Someone integrating this library may think vfu_get_poll_fd() allows for fully async operation.
Stefan
Re: [PATCH v4 07/14] vfio-user: run vfio-user context
Quoted 44 lines · Stefan Hajnoczi
Quoted 28 lines
Quoted 28 lines
Quoted 27 lines
Quoted 6 lines
hw/remote/vfio-user-obj.c +6
-0+static void vfu_object_attach_ctx (void *opaque) +{ +VfuObject *o = opaque; +GPollFD pfds[1 ]; +int ret; +hw/remote/vfio-user-obj.c static void vfu_object_set_device(Object *obj, const char *str, Error **errp) · line 160
Quoted 10 lines
hw/remote/vfio-user-obj.c +10
-0+qemu_set_fd_handler (o->vfu_poll_fd,NULL ,NULL ,NULL ); + + pfds[0 ].fd = o->vfu_poll_fd; + pfds[0 ].events =G_IO_IN |G_IO_HUP |G_IO_ERR ; + +retry_attach: + ret =vfu_attach_ctx (o->vfu_ctx); +if (ret <0 && (errno ==EAGAIN || errno ==EWOULDBLOCK )) { +qemu_poll_ns (pfds,1 ,500 * (int64_t )SCALE_MS ); + goto retry_attach;This can block the thread indefinitely. Other events like monitor commands are not handled in this loop. Please make this asynchronous (set an fd handler and return from this function so we can try again later).
The
vfu_attach_ctx()implementation synchronously negotiates the vfio-user connection :(. That's a shame because even if accept(2) is handled asynchronously, the negotiation can still block. It would be cleanest to have a fully async libvfio-user'svfu_attach_ctx()API to avoid blocking. Is that possible?Quoted 5 lines
Quoted 3 lines
Thanos / John,
Any thoughts on this?
I'm discussing this with John and FYI there are other places where libvfio-user can block, e.g. sending a response or receiving a command. Is it just the negotiation you want it to be asynchronous or _all_ libvfio-user operations? Making libvfio-user fully asynchronous might require a substantial API rewrite.
I see at least two reasons for a fully async API:
1. The program wants to handle other events (e.g. a management REST API) from the same event loop thread that invokes libvfio-user. If libvfio-user blocks then the other events cannot be handled within a reasonable time frame.
The workaround for this is to use multi-threading and ignore the event-driven architecture implied by
vfu_get_poll_fd().
Quoted 7 lines
2. The program handles multiple clients that do not trust each other. This could be a software-defined network switch or storage appliance. A malicious client can cause a denial-of-service by making a libvfio-user call block.
Again, the program needs separate threads instead of an event loop to work around this.
Hi Stefan, we're certainly aware of the rationale. Ultimately we just haven't yet found resources to fix this: in practice, we don't really hit problems from the parts that are still synchronous. Of course, that's not a good argument when it comes to your second issue, and indeed the library is not currently suitable for multiple untrusted clients.
For Jag but: patches are welcome. But it's not just negotiate(): all sorts of things are potentially synchronous - for example, it's expected that the region read/write callbacks are done synchronously. Any other client reply, or server->client message, is also synchronous.
It's partly why we haven't yet stabilised the API.
regards
john
Re: [PATCH v4 07/14] vfio-user: run vfio-user context
Quoted 60 lines · John Levon
Quoted 44 lines · Stefan Hajnoczi
Quoted 28 lines
Quoted 28 lines
Quoted 27 lines
Quoted 6 lines
hw/remote/vfio-user-obj.c +6
-0+static void vfu_object_attach_ctx (void *opaque) +{ +VfuObject *o = opaque; +GPollFD pfds[1 ]; +int ret; +hw/remote/vfio-user-obj.c static void vfu_object_set_device(Object *obj, const char *str, Error **errp) · line 160
Quoted 10 lines
hw/remote/vfio-user-obj.c +10
-0+qemu_set_fd_handler (o->vfu_poll_fd,NULL ,NULL ,NULL ); + + pfds[0 ].fd = o->vfu_poll_fd; + pfds[0 ].events =G_IO_IN |G_IO_HUP |G_IO_ERR ; + +retry_attach: + ret =vfu_attach_ctx (o->vfu_ctx); +if (ret <0 && (errno ==EAGAIN || errno ==EWOULDBLOCK )) { +qemu_poll_ns (pfds,1 ,500 * (int64_t )SCALE_MS ); + goto retry_attach;This can block the thread indefinitely. Other events like monitor commands are not handled in this loop. Please make this asynchronous (set an fd handler and return from this function so we can try again later).
The
vfu_attach_ctx()implementation synchronously negotiates the vfio-user connection :(. That's a shame because even if accept(2) is handled asynchronously, the negotiation can still block. It would be cleanest to have a fully async libvfio-user'svfu_attach_ctx()API to avoid blocking. Is that possible?Quoted 5 lines
Quoted 3 lines
Thanos / John,
Any thoughts on this?
I'm discussing this with John and FYI there are other places where libvfio-user can block, e.g. sending a response or receiving a command. Is it just the negotiation you want it to be asynchronous or _all_ libvfio-user operations? Making libvfio-user fully asynchronous might require a substantial API rewrite.
I see at least two reasons for a fully async API:
1. The program wants to handle other events (e.g. a management REST API) from the same event loop thread that invokes libvfio-user. If libvfio-user blocks then the other events cannot be handled within a reasonable time frame.
The workaround for this is to use multi-threading and ignore the event-driven architecture implied by
vfu_get_poll_fd().Quoted 7 lines
2. The program handles multiple clients that do not trust each other. This could be a software-defined network switch or storage appliance. A malicious client can cause a denial-of-service by making a libvfio-user call block.
Again, the program needs separate threads instead of an event loop to work around this.
Hi Stefan, we're certainly aware of the rationale. Ultimately we just haven't yet found resources to fix this: in practice, we don't really hit problems from the parts that are still synchronous. Of course, that's not a good argument when it comes to your second issue, and indeed the library is not currently suitable for multiple untrusted clients.
Quoted 6 lines
For Jag but: patches are welcome. But it's not just
negotiate(): all sorts of things are potentially synchronous - for example, it's expected that the region read/write callbacks are done synchronously. Any other client reply, orserver->clientmessage, is also synchronous.It's partly why we haven't yet stabilised the API.
From the QEMU side it's okay to have blocking code in this experimental feature if users are aware of the limitation (e.g. the monitor is unresponsive if vfio-user blocks) and multiple untrusted clients are not supported. The QEMU code should also make its limitations clear so readers are aware of them and know the author chose this approach intentionally rather than due to an oversight.
Jag: Please mention this in the documentation and add a comment to vfu_object_attach_ctx() explaining that this code currently blocks.
I think in the long run it will be necessary to address it, e.g. in the multi-client case. That can either be done with threads or by making libvfio-user fully async.
Stefan
Re: [PATCH v4 07/14] vfio-user: run vfio-user context
Quoted 79 lines · Stefan Hajnoczi
Quoted 60 lines · John Levon
Quoted 44 lines · Stefan Hajnoczi
Quoted 28 lines
Quoted 28 lines
Quoted 27 lines
Quoted 6 lines
(quoted diff) +6
-0+static void vfu_object_attach_ctx(void *opaque) +{ + VfuObject *o = opaque; + GPollFD pfds[1]; + int ret; +Review
Quoted 10 lines
(quoted diff) +10
-0+ qemu_set_fd_handler(o->vfu_poll_fd, NULL, NULL, NULL); + + pfds[0].fd = o->vfu_poll_fd; + pfds[0].events = G_IO_IN | G_IO_HUP | G_IO_ERR; + +retry_attach: + ret = vfu_attach_ctx(o->vfu_ctx); + if (ret < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) { + qemu_poll_ns(pfds, 1, 500 * (int64_t)SCALE_MS); + goto retry_attach;This can block the thread indefinitely. Other events like monitor commands are not handled in this loop. Please make this asynchronous (set an fd handler and return from this function so we can try again later).
The
vfu_attach_ctx()implementation synchronously negotiates the vfio-user connection :(. That's a shame because even if accept(2) is handled asynchronously, the negotiation can still block. It would be cleanest to have a fully async libvfio-user'svfu_attach_ctx()API to avoid blocking. Is that possible?Quoted 5 lines
Quoted 3 lines
Thanos / John,
Any thoughts on this?
I'm discussing this with John and FYI there are other places where libvfio-user can block, e.g. sending a response or receiving a command. Is it just the negotiation you want it to be asynchronous or _all_ libvfio-user operations? Making libvfio-user fully asynchronous might require a substantial API rewrite.
I see at least two reasons for a fully async API:
1. The program wants to handle other events (e.g. a management REST API) from the same event loop thread that invokes libvfio-user. If libvfio-user blocks then the other events cannot be handled within a reasonable time frame.
The workaround for this is to use multi-threading and ignore the event-driven architecture implied by
vfu_get_poll_fd().Quoted 7 lines
2. The program handles multiple clients that do not trust each other. This could be a software-defined network switch or storage appliance. A malicious client can cause a denial-of-service by making a libvfio-user call block.
Again, the program needs separate threads instead of an event loop to work around this.
Hi Stefan, we're certainly aware of the rationale. Ultimately we just haven't yet found resources to fix this: in practice, we don't really hit problems from the parts that are still synchronous. Of course, that's not a good argument when it comes to your second issue, and indeed the library is not currently suitable for multiple untrusted clients.
Quoted 6 lines
For Jag but: patches are welcome. But it's not just
negotiate(): all sorts of things are potentially synchronous - for example, it's expected that the region read/write callbacks are done synchronously. Any other client reply, orserver->clientmessage, is also synchronous.It's partly why we haven't yet stabilised the API.
From the QEMU side it's okay to have blocking code in this experimental feature if users are aware of the limitation (e.g. the monitor is unresponsive if vfio-user blocks) and multiple untrusted clients are not supported. The QEMU code should also make its limitations clear so readers are aware of them and know the author chose this approach intentionally rather than due to an oversight.
Quoted 2 lines
Jag: Please mention this in the documentation and add a comment to
vfu_object_attach_ctx()explaining that this code currently blocks.
Sure, will do.
Thank you!
--
Jag
Quoted 6 lines
I think in the long run it will be necessary to address it, e.g. in the multi-client case. That can either be done with threads or by making libvfio-user fully async.
Stefan