Connect request callback to mdev and vfio-ccw
Subsystems (6)
Eric Farman
maintainerv3 cover Connect request callback to mdev and vfio-ccw
Alex,
As promised, here is a v3 with a message indicating a request is blocked until a user releases the device in question and a corresponding tweak to the commit message describing this change (and Conny's r-b). The remainder of the patches are otherwise identical.
v2 : https://lore.kernel.org/kvm/[email protected]/
v1(RFC) : https://lore.kernel.org/kvm/[email protected]/
Eric Farman (2):
vfio-mdev: Wire in a request handler for mdev parent
vfio-ccw: Wire in the request callback
drivers/s390/cio/vfio_ccw_ops.c | 26 ++++++++++++++++++++++++++ drivers/s390/cio/vfio_ccw_private.h | 4 ++++ drivers/vfio/mdev/mdev_core.c | 4 ++++ drivers/vfio/mdev/vfio_mdev.c | 13 +++++++++++++ include/linux/mdev.h | 4 ++++ include/uapi/linux/vfio.h | 1 + 6 files changed, 52 insertions(+)
Signature · 1 line
2.17.1
Eric Farman
maintainerPATCH v3 2/2 vfio-ccw: Wire in the request callback
The device is being unplugged, so pass the request to userspace to ask for a graceful cleanup. This should free up the thread that would otherwise loop waiting for the device to be fully released.
--- drivers/s390/cio/vfio_ccw_ops.c | 26 ++++++++++++++++++++++++++ drivers/s390/cio/vfio_ccw_private.h | 4 ++++ include/uapi/linux/vfio.h | 1 + 3 files changed, 31 insertions(+)
drivers/s390/cio/vfio_ccw_ops.c +26 -0
@@ -394,6 +394,7 @@ static int vfio_ccw_mdev_get_irq_info(struct vfio_irq_info *info)switch (info->index) {case VFIO_CCW_IO_IRQ_INDEX :case VFIO_CCW_CRW_IRQ_INDEX : +case VFIO_CCW_REQ_IRQ_INDEX : info->count =1 ; info->flags =VFIO_IRQ_INFO_EVENTFD ;break ;
@@ -424,6 +425,9 @@ static int vfio_ccw_mdev_set_irqs(struct mdev_device *mdev,case VFIO_CCW_CRW_IRQ_INDEX : ctx = &private->crw_trigger;break ; +case VFIO_CCW_REQ_IRQ_INDEX : + ctx = &private->req_trigger; +break ;default :return -EINVAL ; }
@@ -607,6 +611,27 @@ static ssize_t vfio_ccw_mdev_ioctl(struct mdev_device *mdev, } } +/* Request removal of the device*/ +static void vfio_ccw_mdev_request (struct mdev_device *mdev,unsigned int count) +{ +struct vfio_ccw_private *private =dev_get_drvdata (mdev_parent_dev (mdev)); + +if (!private) +return ; + +if (private->req_trigger) { +if (!(count %10 )) +dev_notice_ratelimited (mdev_dev (private->mdev), +"Relaying device request to user (#%u)\n" , + count); + +eventfd_signal (private->req_trigger,1 ); + }else if (count ==0 ) { +dev_notice (mdev_dev (private->mdev), +"No device request channel registered, blocked until released by user\n" ); + } +} +static const struct mdev_parent_ops vfio_ccw_mdev_ops = { .owner =THIS_MODULE , .supported_type_groups = mdev_type_groups,
@@ -617,6 +642,7 @@ static const struct mdev_parent_ops vfio_ccw_mdev_ops = { .read = vfio_ccw_mdev_read, .write = vfio_ccw_mdev_write, .ioctl = vfio_ccw_mdev_ioctl, + .request = vfio_ccw_mdev_request, };int vfio_ccw_mdev_reg (struct subchannel *sch)
drivers/s390/cio/vfio_ccw_private.h +4 -0
@@ -84,7 +84,10 @@ struct vfio_ccw_crw {* @irb: irb info received from interrupt * @scsw: scsw info * @io_trigger: eventfd ctx for signaling userspace I/O results +* @crw_trigger: eventfd ctx for signaling userspace CRW information +* @req_trigger: eventfd ctx for signaling userspace to return device * @io_work: work for deferral process of I/O handling +* @crw_work: work for deferral process of CRW handling */ struct vfio_ccw_private {struct subchannel *sch;
@@ -108,6 +111,7 @@ struct vfio_ccw_private {struct eventfd_ctx *io_trigger;struct eventfd_ctx *crw_trigger; +struct eventfd_ctx *req_trigger;struct work_struct io_work;struct work_struct crw_work; }__aligned(8) ;
include/uapi/linux/vfio.h +1 -0
@@ -820,6 +820,7 @@ enum {enum {VFIO_CCW_IO_IRQ_INDEX ,VFIO_CCW_CRW_IRQ_INDEX , +VFIO_CCW_REQ_IRQ_INDEX ,VFIO_CCW_NUM_IRQS };
Signature · 1 line
2.17.1
Eric Farman
maintainerPATCH v3 1/2 vfio-mdev: Wire in a request handler for mdev parent
In reply to Eric FarmanWhile performing some destructive tests with vfio-ccw, where the paths to a device are forcible removed and thus the device itself is unreachable, it is rather easy to end up in an endless loop in vfio_del_group_dev() due to the lack of a request callback for the associated device.
In this example, one MDEV (77c) is used by a guest, while another (77b) is not. The symptom is that the iommu is detached from the mdev for 77b, but not 77c, until that guest is shutdown:
[ 238.794867] vfio_ccw 0.0.077b: MDEV: Unregistering
[ 238.794996] vfio_mdev 11f2d2bc-4083-431d-a023-eff72715c4f0: Removing from iommu group 2
[ 238.795001] vfio_mdev 11f2d2bc-4083-431d-a023-eff72715c4f0: MDEV: detaching iommu
[ 238.795036] vfio_ccw 0.0.077c: MDEV: Unregistering
...silence...Let's wire in the request call back to the mdev device, so that a device being physically removed from the host can be (gracefully?) handled by the parent device at the time the device is removed.
Add a message when registering the device if a driver doesn't provide this callback, so a clue is given that this same loop may be encountered in a similar situation, and a message when this occurs instead of the awkward silence noted above.
--- drivers/vfio/mdev/mdev_core.c | 4 ++++ drivers/vfio/mdev/vfio_mdev.c | 13 +++++++++++++ include/linux/mdev.h | 4 ++++ 3 files changed, 21 insertions(+)
drivers/vfio/mdev/mdev_core.c +4 -0
@@ -154,6 +154,10 @@ int mdev_register_device(struct device *dev, const struct mdev_parent_ops *ops)if (!dev)return -EINVAL ; +/* Not mandatory, but its absence could be a problem */ +if (!ops->request) +dev_info (dev,"Driver cannot be asked to release device\n" ); +mutex_lock (&parent_list_lock);/* Check for duplicate */
drivers/vfio/mdev/vfio_mdev.c +13 -0
@@ -98,6 +98,18 @@ static int vfio_mdev_mmap(void *device_data, struct vm_area_struct *vma)return parent->ops->mmap (mdev, vma); } +static void vfio_mdev_request (void *device_data,unsigned int count) +{ +struct mdev_device *mdev = device_data; +struct mdev_parent *parent = mdev->parent; + +if (parent->ops->request) + parent->ops->request (mdev, count); +else if (count ==0 ) +dev_notice (mdev_dev (mdev), +"No mdev vendor driver request callback support, blocked until released by user\n" ); +} +static const struct vfio_device_ops vfio_mdev_dev_ops = { .name ="vfio-mdev" , .open = vfio_mdev_open,
@@ -106,6 +118,7 @@ static const struct vfio_device_ops vfio_mdev_dev_ops = { .read = vfio_mdev_read, .write = vfio_mdev_write, .mmap = vfio_mdev_mmap, + .request = vfio_mdev_request, };static int vfio_mdev_probe (struct device *dev)
include/linux/mdev.h +4 -0
@@ -72,6 +72,9 @@ struct device *mdev_get_iommu_device(struct device *dev);* @mmap: mmap callback * @mdev: mediated device structure * @vma: vma structure +* @request: request callback to release device +* @mdev: mediated device structure +* @count: request sequence number * Parent device that support mediated device should be registered with mdev * module with mdev_parent_ops structure. **/
@@ -92,6 +95,7 @@ struct mdev_parent_ops {long (*ioctl)(struct mdev_device *mdev,unsigned int cmd,unsigned long arg);int (*mmap)(struct mdev_device *mdev,struct vm_area_struct *vma); +void (*request)(struct mdev_device *mdev,unsigned int count); };/* interface for exporting mdev supported type attributes */
Signature · 1 line
2.17.1
Quoted 14 lines · On Thu, 3 Dec 2020 22:35:10 +0100 Eric Farman <[email protected]> wrote:
Alex,
As promised, here is a v3 with a message indicating a request is blocked until a user releases the device in question and a corresponding tweak to the commit message describing this change (and Conny's r-b). The remainder of the patches are otherwise identical.
v2 : https://lore.kernel.org/kvm/[email protected]/
v1(RFC) : https://lore.kernel.org/kvm/[email protected]/Eric Farman (2):
vfio-mdev: Wire in a request handler for mdev parent
vfio-ccw: Wire in the request callback
Quoted 8 lines
drivers/s390/cio/vfio_ccw_ops.c | 26 ++++++++++++++++++++++++++ drivers/s390/cio/vfio_ccw_private.h | 4 ++++ drivers/vfio/mdev/mdev_core.c | 4 ++++ drivers/vfio/mdev/vfio_mdev.c | 13 +++++++++++++ include/linux/mdev.h | 4 ++++ include/uapi/linux/vfio.h | 1 + 6 files changed, 52 insertions(+)
Thanks! Applied to vfio next branch for v5.11.
Alex