vfio/pci: Device request support
Subsystems (2)
cover vfio/pci: Device request support
Currently when the host kernel attempts to unbind an in-use device from a vfio driver, we simply block until userspace releases the device. With a cooperative userspace client, we can do better (we can potentially do better with a non-cooperative client too, but this series does not attempt to address that). Instead of idly waiting, we can poke the vfio bus driver using a new device request callback. For vfio-pci, this is exposed to the user as a new device management interrupt, like the error reporting interrupt. With a userspace client like QEMU, this can be tied to an unplug request, allowing for automatic removal of a device from a VM when an unbind request is made on the host. This of course presumes that the host operation takes precedence over VM policy or else the VM would not choose to be cooperative to the request.
A couple issues are fixed to make this reliable, the first being to close a race where a driver stalled on an unbind can create a small window where the vfio group is non-viable. This can interfere with attempts to acquire a group reference to match and release an existing reference. The other problem is that our IOMMU group ref is tied to the device, so if the device is removed, the IOMMU group can be removed, putting the vfio group in an awkward state. The reference is moved to the vfio group to prevent this. The rest is a straight forward addition of a callback and eventfd notifier path. Thanks,
Alex
---
Alex Williamson (5):
vfio-pci: Add device request interface
vfio-pci: Generalize setup of simple eventfds
vfio: Add and use device request op for vfio bus drivers
vfio: Tie IOMMU group reference to vfio group
vfio: Add device tracking during unbind
drivers/vfio/pci/vfio_pci.c | 21 ++++++ drivers/vfio/pci/vfio_pci_intrs.c | 60 +++++++++++++----- drivers/vfio/pci/vfio_pci_private.h | 1 drivers/vfio/vfio.c | 119 +++++++++++++++++++++++++++++++---- include/linux/vfio.h | 2 + include/uapi/linux/vfio.h | 1 6 files changed, 173 insertions(+), 31 deletions(-)
PATCH 1/5 vfio: Add device tracking during unbind
There's a small window between the vfio bus driver calling vfio_del_group_dev() and the device being completely unbound where the vfio group appears to be non-viable. This creates a race for users like QEMU/KVM where the kvm-vfio module tries to get an external reference to the group in order to match and release an existing reference, while the device is potentially being removed from the vfio bus driver. If the group is momentarily non-viable, kvm-vfio may not be able to release the group reference until VM shutdown, making the group unusable until that point.
Bridge the gap between device removal from the group and completion of the driver unbind by tracking it in a list. The device is added to the list before the bus driver reference is released and removed using the existing unbind notifier.
---
drivers/vfio/vfio.c | 76 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 71 insertions(+), 5 deletions(-)
drivers/vfio/vfio.c +71 -5
@@ -63,6 +63,11 @@ struct vfio_container {void *iommu_data; }; +struct vfio_unbound_dev { +struct device *dev; +struct list_head unbound_next; +}; +struct vfio_group {struct kref kref;int minor;
@@ -75,6 +80,8 @@ struct vfio_group {struct notifier_block nb;struct list_head vfio_next;struct list_head container_next; +struct list_head unbound_list; +struct mutex unbound_lock;atomic_t opened; };
@@ -204,6 +211,8 @@ static struct vfio_group *vfio_create_group(struct iommu_group *iommu_group)kref_init (&group->kref);INIT_LIST_HEAD (&group->device_list);mutex_init (&group->device_lock); +INIT_LIST_HEAD (&group->unbound_list); +mutex_init (&group->unbound_lock);atomic_set (&group->container_users,0 );atomic_set (&group->opened,0 ); group->iommu_group = iommu_group;
@@ -264,9 +273,16 @@ static struct vfio_group *vfio_create_group(struct iommu_group *iommu_group)static void vfio_group_release (struct kref *kref) { struct vfio_group *group = container_of(kref, struct vfio_group, kref); +struct vfio_unbound_dev *unbound, *tmp;WARN_ON (!list_empty (&group->device_list)); +list_for_each_entry_safe (unbound, tmp, + &group->unbound_list, unbound_next) { +list_del (&unbound->unbound_next); +kfree (unbound); + } +device_destroy (vfio.class,MKDEV (MAJOR (vfio.group_devt), group->minor));list_del (&group->vfio_next);vfio_free_group_minor (group->minor);
@@ -440,17 +456,36 @@ static bool vfio_whitelisted_driver(struct device_driver *drv) }/* -* A vfio group is viable for use by userspace if all devices are either -* driver-less or bound to a vfio or whitelisted driver. We test the -* latter by the existence of a struct vfio_device matching the dev. +* A vfio group is viable for use by userspace if all devices are in +* one of the following states: +* - driver-less +* - bound to a vfio driver +* - bound to a whitelisted driver +* +* We use two methods to determine whether a device is bound to a vfio +* driver. The first is to test whether the device exists in the vfio +* group. The second is to test if the device exists on the group +* unbound_list, indicating it's in the middle of transitioning from +* a vfio driver to driver-less. */ static int vfio_dev_viable (struct device *dev,void *data) {struct vfio_group *group = data;struct vfio_device *device;struct device_driver *drv =ACCESS_ONCE (dev->driver); +struct vfio_unbound_dev *unbound; +int ret = -EINVAL ; -if (!drv ||vfio_whitelisted_driver (drv)) +mutex_lock (&group->unbound_lock); +list_for_each_entry (unbound, &group->unbound_list, unbound_next) { +if (dev == unbound->dev) { + ret =0 ; +break ; + } + } +mutex_unlock (&group->unbound_lock); + +if (!ret || !drv ||vfio_whitelisted_driver (drv))return 0 ; device =vfio_group_get_device (group, dev);
@@ -459,7 +494,7 @@ static int vfio_dev_viable(struct device *dev, void *data)return 0 ; } -return -EINVAL ; +return ret; }/**
@@ -501,6 +536,7 @@ static int vfio_iommu_group_notifier(struct notifier_block *nb, { struct vfio_group *group = container_of(nb, struct vfio_group, nb);struct device *dev = data; +struct vfio_unbound_dev *unbound;/* * Need to go through a group_lock lookup to get a reference or we
@@ -550,6 +586,17 @@ static int vfio_iommu_group_notifier(struct notifier_block *nb,* stop the system to maintain isolation. At a minimum, we'd * want a toggle to disable driver auto probe for this device. */ + +mutex_lock (&group->unbound_lock); +list_for_each_entry (unbound, + &group->unbound_list, unbound_next) { +if (dev == unbound->dev) { +list_del (&unbound->unbound_next); +kfree (unbound); +break ; + } + } +mutex_unlock (&group->unbound_lock);break ; }
@@ -657,6 +704,7 @@ void *vfio_del_group_dev(struct device *dev)struct vfio_group *group = device->group;struct iommu_group *iommu_group = group->iommu_group;void *device_data = device->device_data; +struct vfio_unbound_dev *unbound;/* * The group exists so long as we have a device reference. Get
@@ -664,6 +712,24 @@ void *vfio_del_group_dev(struct device *dev)*/ vfio_group_get (group); +/* +* When the device is removed from the group, the group suddenly +* becomes non-viable; the device has a driver (until the unbind +* completes), but it's not present in the group. This is bad news +* for any external users that need to re-acquire a group reference +* in order to match and release their existing reference. To +* solve this, we track such devices on the unbound_list to bridge +* the gap until they're fully unbound. +*/ + unbound =kzalloc (sizeof (*unbound),GFP_KERNEL ); +if (unbound) { + unbound->dev = dev; +mutex_lock (&group->unbound_lock); +list_add (&unbound->unbound_next, &group->unbound_list); +mutex_unlock (&group->unbound_lock); + } +WARN_ON (!unbound); +vfio_device_put (device);/* TODO send a signal to encourage this to be released */
PATCH 2/5 vfio: Tie IOMMU group reference to vfio group
In reply to Alex WilliamsonMove the iommu_group reference from the device to the vfio_group. This ensures that the iommu_group persists as long as the vfio_group remains. This can be important if all of the device from an iommu_group are removed, but we still have an outstanding vfio_group reference; we can still walk the empty list of devices.
---
drivers/vfio/vfio.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-)
drivers/vfio/vfio.c +11 -8
@@ -274,6 +274,7 @@ static void vfio_group_release(struct kref *kref) { struct vfio_group *group = container_of(kref, struct vfio_group, kref);struct vfio_unbound_dev *unbound, *tmp; +struct iommu_group *iommu_group = group->iommu_group;WARN_ON (!list_empty (&group->device_list));
@@ -287,6 +288,7 @@ static void vfio_group_release(struct kref *kref)list_del (&group->vfio_next);vfio_free_group_minor (group->minor);vfio_group_unlock_and_free (group); +iommu_group_put (iommu_group); }static void vfio_group_put (struct vfio_group *group)
@@ -625,6 +627,12 @@ int vfio_add_group_dev(struct device *dev,iommu_group_put (iommu_group);return PTR_ERR (group); } + }else { +/* +* A found vfio_group already holds a reference to the +* iommu_group. A created vfio_group keeps the reference. +*/ +iommu_group_put (iommu_group); } device =vfio_group_get_device (group, dev);
@@ -633,21 +641,19 @@ int vfio_add_group_dev(struct device *dev, dev_name(dev), iommu_group_id(iommu_group));vfio_device_put (device);vfio_group_put (group); -iommu_group_put (iommu_group);return -EBUSY ; } device =vfio_group_create_device (group, dev, ops, device_data);if (IS_ERR (device)) {vfio_group_put (group); -iommu_group_put (iommu_group);return PTR_ERR (device); }/* -* Added device holds reference to iommu_group and vfio_device -* (which in turn holds reference to vfio_group). Drop extra -* group reference used while acquiring device. +* Drop all but the vfio_device reference. The vfio_device holds +* a reference to the vfio_group, which holds a reference to the +* iommu_group. */ vfio_group_put (group);
@@ -702,7 +708,6 @@ void *vfio_del_group_dev(struct device *dev) {struct vfio_device *device =dev_get_drvdata (dev);struct vfio_group *group = device->group; -struct iommu_group *iommu_group = group->iommu_group;void *device_data = device->device_data;struct vfio_unbound_dev *unbound;
@@ -737,8 +742,6 @@ void *vfio_del_group_dev(struct device *dev)vfio_group_put (group); -iommu_group_put (iommu_group); -return device_data; }EXPORT_SYMBOL_GPL (vfio_del_group_dev);
PATCH 3/5 vfio: Add and use device request op for vfio bus drivers
In reply to Alex WilliamsonWhen a request is made to unbind a device from a vfio bus driver, we need to wait for the device to become unused, ie. for userspace to release the device. However, we have a long standing TODO in the code to do something proactive to make that happen. To enable this, we add a request callback on the vfio bus driver struct, which is intended to signal the user through the vfio device interface to release the device. Instead of passively waiting for the device to become unused, we can now pester the user to give it up.
---
drivers/vfio/vfio.c | 24 ++++++++++++++++++++++-- include/linux/vfio.h | 2 ++ 2 files changed, 24 insertions(+), 2 deletions(-)
drivers/vfio/vfio.c +22 -2
@@ -710,6 +710,7 @@ void *vfio_del_group_dev(struct device *dev)struct vfio_group *group = device->group;void *device_data = device->device_data;struct vfio_unbound_dev *unbound; +unsigned int i =0 ;/* * The group exists so long as we have a device reference. Get
@@ -737,8 +738,27 @@ void *vfio_del_group_dev(struct device *dev)vfio_device_put (device); -/* TODO send a signal to encourage this to be released */ -wait_event (vfio.release_q, !vfio_dev_present (group, dev)); +/* +* If the device is still present in the group after the above +* 'put', then it is in use and we need to request it from the +* bus driver. The driver may in turn need to request the +* device from the user. We send the request on an arbitrary +* interval with counter to allow the driver to take escalating +* measures to release the device if it has the ability to do so. +*/ +do { + device =vfio_group_get_device (group, dev); +if (!device) +break ; + +if (device->ops->request) + device->ops->request (device_data, i++); + +vfio_device_put (device); + + }while (wait_event_interruptible_timeout (vfio.release_q, + !vfio_dev_present (group, dev), +HZ *10 ) <=0 );vfio_group_put (group);
include/linux/vfio.h +2 -0
@@ -26,6 +26,7 @@* @ioctl: Perform ioctl(2) on device file descriptor, supporting VFIO_DEVICE_* * operations documented below * @mmap: Perform mmap(2) on a region of the device file descriptor +* @request: Request for the bus driver to release the device */ struct vfio_device_ops {char *name;
@@ -38,6 +39,7 @@ struct vfio_device_ops {long (*ioctl)(void *device_data,unsigned int cmd,unsigned long arg);int (*mmap)(void *device_data,struct vm_area_struct *vma); +void (*request)(void *device_data,unsigned int count); }; extern int vfio_add_group_dev(struct device *dev,
PATCH 4/5 vfio-pci: Generalize setup of simple eventfds
In reply to Alex WilliamsonWe want another single vector IRQ index to support signaling of the device request to userspace. Generalize the error reporting IRQ index to avoid code duplication.
---
drivers/vfio/pci/vfio_pci_intrs.c | 44 ++++++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 15 deletions(-)
drivers/vfio/pci/vfio_pci_intrs.c +29 -15
@@ -763,46 +763,60 @@ static int vfio_pci_set_msi_trigger(struct vfio_pci_device *vdev,return 0 ; } -static int vfio_pci_set_err_trigger (struct vfio_pci_device *vdev, -unsigned index,unsigned start, -unsigned count,uint32_t flags,void *data) +static int vfio_pci_set_ctx_trigger_single (struct eventfd_ctx **ctx, +uint32_t flags,void *data) {int32_t fd = *(int32_t *)data; -if ((index !=VFIO_PCI_ERR_IRQ_INDEX ) || - !(flags &VFIO_IRQ_SET_DATA_TYPE_MASK )) +if (!(flags &VFIO_IRQ_SET_DATA_TYPE_MASK ))return -EINVAL ;/* DATA_NONE/DATA_BOOL enables loopback testing */ if (flags &VFIO_IRQ_SET_DATA_NONE ) { -if (vdev->err_trigger) -eventfd_signal (vdev->err_trigger,1 ); +if (*ctx) +eventfd_signal (*ctx,1 );return 0 ; }else if (flags &VFIO_IRQ_SET_DATA_BOOL ) {uint8_t trigger = *(uint8_t *)data; -if (trigger && vdev->err_trigger) -eventfd_signal (vdev->err_trigger,1 ); +if (trigger && *ctx) +eventfd_signal (*ctx,1 );return 0 ; }/* Handle SET_DATA_EVENTFD */ if (fd ==-1 ) { -if (vdev->err_trigger) -eventfd_ctx_put (vdev->err_trigger); - vdev->err_trigger =NULL ; +if (*ctx) +eventfd_ctx_put (*ctx); + *ctx =NULL ;return 0 ; }else if (fd >=0 ) {struct eventfd_ctx *efdctx; efdctx =eventfd_ctx_fdget (fd);if (IS_ERR (efdctx))return PTR_ERR (efdctx); -if (vdev->err_trigger) -eventfd_ctx_put (vdev->err_trigger); - vdev->err_trigger = efdctx; +if (*ctx) +eventfd_ctx_put (*ctx); + *ctx = efdctx;return 0 ; }else return -EINVAL ; } + +static int vfio_pci_set_err_trigger (struct vfio_pci_device *vdev, +unsigned index,unsigned start, +unsigned count,uint32_t flags,void *data) +{ +if (index !=VFIO_PCI_ERR_IRQ_INDEX ) +return -EINVAL ; + +/* +* We should sanitize start & count, but that wasn't caught +* originally, so this IRQ index must forever ignore them :-( +*/ + +return vfio_pci_set_ctx_trigger_single (&vdev->err_trigger, flags, data); +} +int vfio_pci_set_irqs_ioctl (struct vfio_pci_device *vdev,uint32_t flags,unsigned index,unsigned start,unsigned count,void *data)
PATCH 5/5 vfio-pci: Add device request interface
In reply to Alex WilliamsonUserspace can opt to receive a device request notification, indicating that the device should be released. This is setup the same way as the error IRQ and also supports eventfd signaling. Future support may forcefully remove the device from the user if the request is ignored.
---
drivers/vfio/pci/vfio_pci.c | 21 ++++++++++++++++++++- drivers/vfio/pci/vfio_pci_intrs.c | 16 ++++++++++++++++ drivers/vfio/pci/vfio_pci_private.h | 1 + include/uapi/linux/vfio.h | 1 + 4 files changed, 38 insertions(+), 1 deletion(-)
drivers/vfio/pci/vfio_pci.c +20 -1
@@ -239,9 +239,12 @@ static int vfio_pci_get_irq_count(struct vfio_pci_device *vdev, int irq_type)return (flags &PCI_MSIX_FLAGS_QSIZE ) +1 ; } - }else if (irq_type ==VFIO_PCI_ERR_IRQ_INDEX ) + }else if (irq_type ==VFIO_PCI_ERR_IRQ_INDEX ) {if (pci_is_pcie (vdev->pdev))return 1 ; + }else if (irq_type ==VFIO_PCI_REQ_IRQ_INDEX ) { +return 1 ; + }return 0 ; }
@@ -464,6 +467,7 @@ static long vfio_pci_ioctl(void *device_data,switch (info.index) { case VFIO_PCI_INTX_IRQ_INDEX ... VFIO_PCI_MSIX_IRQ_INDEX: +case VFIO_PCI_REQ_IRQ_INDEX :break ;case VFIO_PCI_ERR_IRQ_INDEX :if (pci_is_pcie (vdev->pdev))
@@ -828,6 +832,20 @@ static int vfio_pci_mmap(void *device_data, struct vm_area_struct *vma) req_len, vma->vm_page_prot); } +static void vfio_pci_request (void *device_data,unsigned int count) +{ +struct vfio_pci_device *vdev = device_data; + +mutex_lock (&vdev->igate); + +if (vdev->req_trigger) { +dev_dbg (&vdev->pdev->dev,"Requesting device from user\n" ); +eventfd_signal (vdev->req_trigger,1 ); + } + +mutex_unlock (&vdev->igate); +} +static const struct vfio_device_ops vfio_pci_ops = { .name ="vfio-pci" , .open = vfio_pci_open,
@@ -836,6 +854,7 @@ static const struct vfio_device_ops vfio_pci_ops = { .read = vfio_pci_read, .write = vfio_pci_write, .mmap = vfio_pci_mmap, + .request = vfio_pci_request, };static int vfio_pci_probe (struct pci_dev *pdev,const struct pci_device_id *id)
drivers/vfio/pci/vfio_pci_intrs.c +16 -0
@@ -817,6 +817,16 @@ static int vfio_pci_set_err_trigger(struct vfio_pci_device *vdev,return vfio_pci_set_ctx_trigger_single (&vdev->err_trigger, flags, data); } +static int vfio_pci_set_req_trigger (struct vfio_pci_device *vdev, +unsigned index,unsigned start, +unsigned count,uint32_t flags,void *data) +{ +if (index !=VFIO_PCI_REQ_IRQ_INDEX || start !=0 || count !=1 ) +return -EINVAL ; + +return vfio_pci_set_ctx_trigger_single (&vdev->req_trigger, flags, data); +} +int vfio_pci_set_irqs_ioctl (struct vfio_pci_device *vdev,uint32_t flags,unsigned index,unsigned start,unsigned count,void *data)
@@ -858,6 +868,12 @@ int vfio_pci_set_irqs_ioctl(struct vfio_pci_device *vdev, uint32_t flags, func = vfio_pci_set_err_trigger;break ; } +case VFIO_PCI_REQ_IRQ_INDEX : +switch (flags &VFIO_IRQ_SET_ACTION_TYPE_MASK ) { +case VFIO_IRQ_SET_ACTION_TRIGGER : + func = vfio_pci_set_req_trigger; +break ; + } }if (!func)
drivers/vfio/pci/vfio_pci_private.h +1 -0
@@ -58,6 +58,7 @@ struct vfio_pci_device {struct pci_saved_state *pci_saved_state;int refcnt;struct eventfd_ctx *err_trigger; +struct eventfd_ctx *req_trigger; };#define is_intx (vdev) (vdev->irq_type == VFIO_PCI_INTX_IRQ_INDEX)
include/uapi/linux/vfio.h +1 -0
@@ -333,6 +333,7 @@ enum {VFIO_PCI_MSI_IRQ_INDEX ,VFIO_PCI_MSIX_IRQ_INDEX ,VFIO_PCI_ERR_IRQ_INDEX , +VFIO_PCI_REQ_IRQ_INDEX , VFIO_PCI_NUM_IRQS };