Fixes, non-PCI-2.3 support, EOI enhancements

index 88deb19..8af995d 100644
@@ -797,7 +797,7 @@ static void vfio_bar_fixup(struct vfio_dev *vdev)
 
 	if (pci_resource_start(pdev, PCI_ROM_RESOURCE)) {
 		mask = ~(pci_resource_len(pdev, PCI_ROM_RESOURCE) - 1);
-		mask = PCI_ROM_ADDRESS_ENABLE;
+		mask |= PCI_ROM_ADDRESS_ENABLE;
 	} else
 		mask = 0;
 	lp = (u32 *)(vdev->vconfig + PCI_ROM_ADDRESS);

index 3b7d352..f671795 100644
@@ -316,6 +316,18 @@ static int vfio_mmap(struct file *filep, struct vm_area_struct *vma)
 	if (requested > actual || actual == 0)
 		return -EINVAL;
 
+	/*
+	 * Even though we don't make use of the barmap for the mmap,
+	 * we need to request the region and the barmap tracks that.
+	 */
+	if (!vdev->barmap[pci_space]) {
+		ret = pci_request_selected_regions(pdev, (1 << pci_space),
+						   vdev->name);
+		if (ret)
+			return ret;
+		vdev->barmap[pci_space] = pci_iomap(pdev, pci_space, 0);
+	}
+
 	start = vma->vm_pgoff << PAGE_SHIFT;
 	len = vma->vm_end - vma->vm_start;
 	if (allow_unsafe_intrs && (vma->vm_flags & VM_WRITE)) {

index b75bf92..83ae38a 100644
@@ -67,7 +67,7 @@ ssize_t vfio_io_readwrite(
 		int ret;
 
 		ret = pci_request_selected_regions(pdev,
-			(1 << pci_space), "vfio");
+			(1 << pci_space), vdev->name);
 		if (ret)
 			return ret;
 		vdev->barmap[pci_space] = pci_iomap(pdev, pci_space, 0);

@@ -164,7 +164,7 @@ ssize_t vfio_mem_readwrite(
 			int ret;
 
 			ret = pci_request_selected_regions(pdev,
-				(1 << pci_space), "vfio");
+				(1 << pci_space), vdev->name);
 			if (ret)
 				return ret;
 			vdev->barmap[pci_space] =

index aaee175..a57d5aa 100644
@@ -83,6 +83,23 @@ done:
 	return ret;
 }
 
+int vfio_irq_eoi(struct vfio_dev *vdev)
+{
+	struct pci_dev *pdev = vdev->pdev;
+	u16 cmd;
+
+	spin_lock_irq(&vdev->irqlock);
+	pci_block_user_cfg_access(pdev);
+
+	pci_read_config_word(pdev, PCI_COMMAND, &cmd);
+	cmd &= ~PCI_COMMAND_INTX_DISABLE;
+	pci_write_config_word(pdev, PCI_COMMAND, cmd);
+
+	pci_unblock_user_cfg_access(pdev);
+	spin_unlock_irq(&vdev->irqlock);
+	return 0;
+}
+
 /*
  * MSI and MSI-X Interrupt handler.
  * Just signal an event

index f671795..72db79c 100644
@@ -464,6 +464,10 @@ static long vfio_unl_ioctl(struct file *filep,
 		ret = vfio_domain_unset(vdev);
 		break;
 
+	case VFIO_IRQ_EOI:
+		ret = vfio_irq_eoi(vdev);
+		break;
+
 	default:
 		return -EINVAL;
 	}

index c491180..73d7e84 100644
@@ -155,6 +155,7 @@ void vfio_error_resume(struct pci_dev *);
 #define VFIO_SUSPEND_REPLY_TIMEOUT	(5*HZ)
 
 irqreturn_t vfio_interrupt(int, void *);
+int vfio_irq_eoi(struct vfio_dev *);
 
 #endif	/* __KERNEL__ */
 

@@ -198,6 +199,8 @@ struct vfio_dma_map {
 /* Unset the IOMMU domain */
 #define	VFIO_DOMAIN_UNSET	_IO(';', 108)
 
+/* Re-enable INTx */
+#define	VFIO_IRQ_EOI		_IO(';', 109)
 /*
  * Reads, writes, and mmaps determine which PCI BAR (or config space)
  * from the high level bits of the file offset

index a57d5aa..4d5a7f8 100644
@@ -52,32 +52,44 @@ irqreturn_t vfio_interrupt(int irq, void *dev_id)
 	u16 origcmd, newcmd, status;
 
 	spin_lock_irq(&vdev->irqlock);
-	pci_block_user_cfg_access(pdev);
-
-	/* Read both command and status registers in a single 32-bit operation.
-	 * Note: we could cache the value for command and move the status read
-	 * out of the lock if there was a way to get notified of user changes
-	 * to command register through sysfs. Should be good for shared irqs. */
-	pci_read_config_dword(pdev, PCI_COMMAND, &cmd_status_dword);
-	origcmd = cmd_status_dword;
-	status = cmd_status_dword >> 16;
-
-	/* Check interrupt status register to see whether our device
-	 * triggered the interrupt. */
-	if (!(status & PCI_STATUS_INTERRUPT))
-		goto done;
-
-	/* We triggered the interrupt, disable it. */
-	newcmd = origcmd | PCI_COMMAND_INTX_DISABLE;
-	if (newcmd != origcmd)
-		pci_write_config_word(pdev, PCI_COMMAND, newcmd);
-
-	ret = IRQ_HANDLED;
+
+	if (vdev->pci_2_3) {
+		pci_block_user_cfg_access(pdev);
+
+		/* Read both command and status registers in a single 32-bit
+		 * operation. Note: we could cache the value for command and
+		 * move the status read out of the lock if there was a way to
+		 * get notified of user changes to command register through
+		 * sysfs. Should be good for shared irqs. */
+		pci_read_config_dword(pdev, PCI_COMMAND, &cmd_status_dword);
+		origcmd = cmd_status_dword;
+		status = cmd_status_dword >> 16;
+
+		/* Check interrupt status register to see whether our device
+	 	* triggered the interrupt. */
+		if (!(status & PCI_STATUS_INTERRUPT))
+			goto done;
+
+		/* We triggered the interrupt, disable it. */
+		newcmd = origcmd | PCI_COMMAND_INTX_DISABLE;
+		if (newcmd != origcmd)
+			pci_write_config_word(pdev, PCI_COMMAND, newcmd);
+
+		ret = IRQ_HANDLED;
 done:
-	pci_unblock_user_cfg_access(pdev);
+		pci_unblock_user_cfg_access(pdev);
+	} else {
+		disable_irq_nosync(pdev->irq);
+		ret = IRQ_HANDLED;
+	}
+
+	vdev->irq_disabled = (ret == IRQ_HANDLED);
+
 	spin_unlock_irq(&vdev->irqlock);
+
 	if (ret != IRQ_HANDLED)
 		return ret;
+
 	if (vdev->ev_irq)
 		eventfd_signal(vdev->ev_irq, 1);
 	return ret;

@@ -86,16 +98,25 @@ done:
 int vfio_irq_eoi(struct vfio_dev *vdev)
 {
 	struct pci_dev *pdev = vdev->pdev;
-	u16 cmd;
 
 	spin_lock_irq(&vdev->irqlock);
-	pci_block_user_cfg_access(pdev);
 
-	pci_read_config_word(pdev, PCI_COMMAND, &cmd);
-	cmd &= ~PCI_COMMAND_INTX_DISABLE;
-	pci_write_config_word(pdev, PCI_COMMAND, cmd);
+	if (vdev->irq_disabled) {
+		if (vdev->pci_2_3) {
+			u16 cmd;
+			pci_block_user_cfg_access(pdev);
+
+			pci_read_config_word(pdev, PCI_COMMAND, &cmd);
+			cmd &= ~PCI_COMMAND_INTX_DISABLE;
+			pci_write_config_word(pdev, PCI_COMMAND, cmd);
+
+			pci_unblock_user_cfg_access(pdev);
+		} else
+			enable_irq(pdev->irq);
+
+		vdev->irq_disabled = false;
+	}
 
-	pci_unblock_user_cfg_access(pdev);
 	spin_unlock_irq(&vdev->irqlock);
 	return 0;
 }

index 72db79c..cf2e671 100644
@@ -401,7 +401,8 @@ static long vfio_unl_ioctl(struct file *filep,
 				if (vdev->ev_irq)
 					ret = request_irq(pdev->irq,
 						vfio_interrupt,
-						IRQF_SHARED, vdev->name, vdev);
+						vdev->pci_2_3 ? IRQF_SHARED : 0,
+						vdev->name, vdev);
 				else
 					ret = -EINVAL;
 			}

@@ -567,8 +568,8 @@ static int verify_pci_2_3(struct pci_dev *pdev)
 		return -EBUSY;
 	}
 	if (!((new ^ orig) & PCI_COMMAND_INTX_DISABLE)) {
-		dev_warn(&pdev->dev, "Device does not support "
-			 "disabling interrupts: unable to bind.\n");
+		dev_warn(&pdev->dev, "Device does not support disabling "
+			 "interrupts, exclusive interrupt required.\n");
 		return -ENODEV;
 	}
 	/* Now restore the original value. */

@@ -589,15 +590,13 @@ static int vfio_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	if ((type & 0x7F) != PCI_HEADER_TYPE_NORMAL)
 		return -EINVAL;
 
-	err = verify_pci_2_3(pdev);
-	if (err)
-		return err;
-
 	vdev = kzalloc(sizeof(struct vfio_dev), GFP_KERNEL);
 	if (!vdev)
 		return -ENOMEM;
 	vdev->pdev = pdev;
 
+	vdev->pci_2_3 = (verify_pci_2_3(pdev) == 0);
+
 	mutex_init(&vdev->lgate);
 	mutex_init(&vdev->dgate);
 	mutex_init(&vdev->igate);

index 73d7e84..f7e51ff 100644
@@ -77,6 +77,8 @@ struct vfio_dev {
 	u8		msi_qmax;
 	u8		bardirty;
 	struct perm_bits	*msi_perm;
+	bool		pci_2_3;
+	bool		irq_disabled;
 };
 
 struct vfio_listener {

index 4d5a7f8..604082c 100644
@@ -36,6 +36,10 @@
 #include <linux/eventfd.h>
 #include <linux/pci.h>
 #include <linux/mmu_notifier.h>
+#include <linux/file.h>
+#include <linux/poll.h>
+#include <linux/wait.h>
+#include <linux/workqueue.h>
 
 #include <linux/vfio.h>
 

@@ -121,6 +125,174 @@ int vfio_irq_eoi(struct vfio_dev *vdev)
 	return 0;
 }
 
+struct eoi_eventfd {
+	struct vfio_dev		*vdev;
+	struct eventfd_ctx	*eventfd;
+	poll_table		pt;
+	wait_queue_t		wait;
+	struct work_struct	inject;
+	struct work_struct	shutdown;
+};
+
+static struct workqueue_struct *eoi_cleanup_wq;
+
+static void inject_eoi(struct work_struct *work)
+{
+	struct eoi_eventfd *ev_eoi = container_of(work, struct eoi_eventfd,
+						  inject);
+	vfio_irq_eoi(ev_eoi->vdev);
+}
+
+static void shutdown_eoi(struct work_struct *work)
+{
+	u64 cnt;
+	struct eoi_eventfd *ev_eoi = container_of(work, struct eoi_eventfd,
+						  shutdown);
+	struct vfio_dev *vdev = ev_eoi->vdev;
+
+	eventfd_ctx_remove_wait_queue(ev_eoi->eventfd, &ev_eoi->wait, &cnt);
+	flush_work(&ev_eoi->inject);
+	eventfd_ctx_put(ev_eoi->eventfd);
+	kfree(vdev->ev_eoi);
+	vdev->ev_eoi = NULL;
+}
+
+static void deactivate_eoi(struct eoi_eventfd *ev_eoi)
+{
+	queue_work(eoi_cleanup_wq, &ev_eoi->shutdown);
+}
+
+static int wakeup_eoi(wait_queue_t *wait, unsigned mode, int sync, void *key)
+{
+	struct eoi_eventfd *ev_eoi = container_of(wait, struct eoi_eventfd,
+						  wait);
+	unsigned long flags = (unsigned long)key;
+
+	if (flags & POLLIN)
+		/* An event has been signaled, inject an interrupt */
+		schedule_work(&ev_eoi->inject);
+
+	if (flags & POLLHUP)
+		/* The eventfd is closing, detach from VFIO */
+		deactivate_eoi(ev_eoi);
+
+	return 0;
+}
+
+static void
+eoi_ptable_queue_proc(struct file *file, wait_queue_head_t *wqh, poll_table *pt)
+{
+	struct eoi_eventfd *ev_eoi = container_of(pt, struct eoi_eventfd, pt);
+	add_wait_queue(wqh, &ev_eoi->wait);
+}
+
+static int vfio_irq_eoi_eventfd_enable(struct vfio_dev *vdev, int fd)
+{
+	struct file *file = NULL;
+	struct eventfd_ctx *eventfd = NULL;
+	struct eoi_eventfd *ev_eoi;
+	int ret = 0;
+	unsigned int events;
+
+	if (vdev->ev_eoi)
+		return -EBUSY;
+
+	ev_eoi = kzalloc(sizeof(struct eoi_eventfd), GFP_KERNEL);
+	if (!ev_eoi)
+		return -ENOMEM;
+
+	vdev->ev_eoi = ev_eoi;
+	ev_eoi->vdev = vdev;
+
+	INIT_WORK(&ev_eoi->inject, inject_eoi);
+	INIT_WORK(&ev_eoi->shutdown, shutdown_eoi);
+
+	file = eventfd_fget(fd);
+	if (IS_ERR(eventfd)) {
+		ret = PTR_ERR(eventfd);
+		goto fail;
+	}
+
+	eventfd = eventfd_ctx_fileget(file);
+	if (IS_ERR(eventfd)) {
+		ret = PTR_ERR(eventfd);
+		goto fail;
+	}
+
+	ev_eoi->eventfd = eventfd;
+
+	/*
+	 * Install our own custom wake-up handling so we are notified via
+	 * a callback whenever someone signals the underlying eventfd
+	 */
+	init_waitqueue_func_entry(&ev_eoi->wait, wakeup_eoi);
+	init_poll_funcptr(&ev_eoi->pt, eoi_ptable_queue_proc);
+
+	events = file->f_op->poll(file, &ev_eoi->pt);
+
+	/*
+	 * Check if there was an event already pending on the eventfd
+	 * before we registered, and trigger it as if we didn't miss it.
+	 */
+	if (events & POLLIN)
+		schedule_work(&ev_eoi->inject);
+
+	/*
+	 * do not drop the file until the irqfd is fully initialized, otherwise
+	 * we might race against the POLLHUP
+	 */
+	fput(file);
+
+	return 0;
+
+fail:
+	if (eventfd && !IS_ERR(eventfd))
+		eventfd_ctx_put(eventfd);
+
+	if (!IS_ERR(file))
+		fput(file);
+
+	return ret;
+}
+
+static int vfio_irq_eoi_eventfd_disable(struct vfio_dev *vdev, int fd)
+{
+	if (!vdev->ev_eoi)
+		return -ENODEV;
+
+	deactivate_eoi(vdev->ev_eoi);
+
+	/*
+	 * Block until we know all outstanding shutdown jobs have completed
+	 * so that we guarantee there will not be any more interrupts on this
+	 * gsi once this deassign function returns.
+	 */
+	flush_workqueue(eoi_cleanup_wq);
+
+	return 0;
+}
+
+int vfio_irq_eoi_eventfd(struct vfio_dev *vdev, int fd)
+{
+	if (fd < 0)
+		return vfio_irq_eoi_eventfd_disable(vdev, fd);
+	return vfio_irq_eoi_eventfd_enable(vdev, fd);
+}
+
+int __init vfio_eoi_module_init(void)
+{
+	eoi_cleanup_wq = create_singlethread_workqueue("vfio-eoi-cleanup");
+	if (!eoi_cleanup_wq)
+		return -ENOMEM;
+
+	return 0;
+}
+
+void __exit vfio_eoi_module_exit(void)
+{
+	destroy_workqueue(eoi_cleanup_wq);
+}
+
 /*
  * MSI and MSI-X Interrupt handler.
  * Just signal an event

index cf2e671..3cd3cb8 100644
@@ -469,6 +469,12 @@ static long vfio_unl_ioctl(struct file *filep,
 		ret = vfio_irq_eoi(vdev);
 		break;
 
+	case VFIO_IRQ_EOI_EVENTFD:
+		if (copy_from_user(&fd, uarg, sizeof fd))
+			return -EFAULT;
+		ret = vfio_irq_eoi_eventfd(vdev, fd);
+		break;
+
 	default:
 		return -EINVAL;
 	}

@@ -774,6 +780,7 @@ static int __init init(void)
 	vfio_class_init();
 	vfio_nl_init();
 	register_pm_notifier(&vfio_pm_nb);
+	vfio_eoi_module_init();
 	return pci_register_driver(&driver);
 }
 

@@ -782,6 +789,7 @@ static void __exit cleanup(void)
 	if (vfio_major >= 0)
 		unregister_chrdev(vfio_major, "vfio");
 	pci_unregister_driver(&driver);
+	vfio_eoi_module_exit();
 	unregister_pm_notifier(&vfio_pm_nb);
 	unregister_pm_notifier(&vfio_pm_nb);
 	vfio_nl_exit();

index f7e51ff..c26f3b3 100644
@@ -43,6 +43,7 @@ struct vfio_nl_client {
 };
 
 struct perm_bits;
+struct eoi_eventfd;
 struct vfio_dev {
 	struct device	*dev;
 	struct pci_dev	*pdev;

@@ -79,6 +80,7 @@ struct vfio_dev {
 	struct perm_bits	*msi_perm;
 	bool		pci_2_3;
 	bool		irq_disabled;
+	struct eoi_eventfd	*ev_eoi;
 };
 
 struct vfio_listener {

@@ -158,6 +160,9 @@ void vfio_error_resume(struct pci_dev *);
 
 irqreturn_t vfio_interrupt(int, void *);
 int vfio_irq_eoi(struct vfio_dev *);
+int vfio_irq_eoi_eventfd(struct vfio_dev *, int);
+int vfio_eoi_module_init(void);
+void vfio_eoi_module_exit(void);
 
 #endif	/* __KERNEL__ */
 

@@ -203,6 +208,10 @@ struct vfio_dma_map {
 
 /* Re-enable INTx */
 #define	VFIO_IRQ_EOI		_IO(';', 109)
+
+/* Re-enable INTx via eventfd */
+#define	VFIO_IRQ_EOI_EVENTFD	_IOW(';', 110, int)
+
 /*
  * Reads, writes, and mmaps determine which PCI BAR (or config space)
  * from the high level bits of the file offset