Skip to contentSkip to conversation

introduce update_irq_devid and optimize VFIO irq ops

PATCH Superseded Ben Luo 3 participants 11 messages Aug 8, 2019 – Aug 12, 2019

Subsystems (2)

v1 of 6 Superseded Latest is v6 →

lore
Share & tools
lore

Ben Luo

author
#0 lore raw

cover introduce update_irq_devid and optimize VFIO irq ops

Currently, VFIO takes a lot of free-then-request-irq actions whenever a VM (with device passthru via VFIO) sets irq affinity or mask/unmask irq. Those actions only change the cookie data of irqaction or even change nothing. The free-then-request-irq not only add more latency, but also increases the risk of losing interrupt, this may lead to a VM hung forever in waiting IO completion

This patchset solved this issue by:
patch 1 introduces update_irq_devid to only update dev_id of irqaction patch 2 make use of update_irq_devid and optimize irq operations in VFIO

Ben Luo (2):
genirq: introduce update_irq_devid()
vfio_pci: make use of update_irq_devid and optimize irq ops

 drivers/vfio/pci/vfio_pci_intrs.c | 100 +++++++++++++++++++++++---------------
 include/linux/interrupt.h         |   3 ++
 kernel/irq/manage.c               |  74 ++++++++++++++++++++++++++++
 3 files changed, 139 insertions(+), 38 deletions(-)
Signature · 1 line
1.8.3.1
author
#4 lore raw

PATCH 2/2 vfio_pci: make use of update_irq_devid and optimize irq ops

When userspace (e.g. qemu) triggers a switch between KVM irqfd and userspace eventfd, only dev_id of irq action (i.e. the "trigger" in this patch's context) will be changed, but a free-then-request-irq action is taken in current code. And, irq affinity setting in VM will also trigger a free-then-request-irq action, which actully changes nothing, but only fires a producer re-registration to update irte in case that posted-interrupt is in use.

This patch makes use of update_irq_devid() and optimize both cases above, which reduces the risk of losing interrupt and also cuts some overhead.

---
 drivers/vfio/pci/vfio_pci_intrs.c | 100 +++++++++++++++++++++++---------------
 1 file changed, 62 insertions(+), 38 deletions(-)
drivers/vfio/pci/vfio_pci_intrs.c +62 -38
index 3fa3f72..1323dc8 100644
@@ -285,69 +285,93 @@ static int vfio_msi_set_vector_signal(struct vfio_pci_device *vdev,
 				      int vector, int fd, bool msix)
 {
 	struct pci_dev *pdev = vdev->pdev;
-	struct eventfd_ctx *trigger;
+	struct eventfd_ctx *trigger = NULL;
 	int irq, ret;
 
 	if (vector < 0 || vector >= vdev->num_ctx)
 		return -EINVAL;
 
+	if (fd >= 0) {
+		trigger = eventfd_ctx_fdget(fd);
+		if (IS_ERR(trigger))
+			return PTR_ERR(trigger);
+	}
+
 	irq = pci_irq_vector(pdev, vector);
 
 	if (vdev->ctx[vector].trigger) {
-		free_irq(irq, vdev->ctx[vector].trigger);
-		irq_bypass_unregister_producer(&vdev->ctx[vector].producer);
-		kfree(vdev->ctx[vector].name);
-		eventfd_ctx_put(vdev->ctx[vector].trigger);
-		vdev->ctx[vector].trigger = NULL;
+		if (vdev->ctx[vector].trigger != trigger) {
+			if (trigger) {
+				ret = update_irq_devid(irq,
+						vdev->ctx[vector].trigger, trigger);
+				if (unlikely(ret)) {
+					dev_info(&pdev->dev,
+							"update_irq_devid %d (token %p) fails: %d\n",
+							irq, vdev->ctx[vector].trigger, ret);
+					eventfd_ctx_put(trigger);
+					return ret;
+				}
+				irq_bypass_unregister_producer(&vdev->ctx[vector].producer);
+				eventfd_ctx_put(vdev->ctx[vector].trigger);
+				vdev->ctx[vector].producer.token = trigger;
+				vdev->ctx[vector].trigger = trigger;
+			} else {
+				free_irq(irq, vdev->ctx[vector].trigger);
+				irq_bypass_unregister_producer(&vdev->ctx[vector].producer);
+				kfree(vdev->ctx[vector].name);
+				eventfd_ctx_put(vdev->ctx[vector].trigger);
+				vdev->ctx[vector].trigger = NULL;
+			}
+		} else
+			irq_bypass_unregister_producer(&vdev->ctx[vector].producer);
 	}
 
 	if (fd < 0)
 		return 0;
 
-	vdev->ctx[vector].name = kasprintf(GFP_KERNEL, "vfio-msi%s[%d](%s)",
-					   msix ? "x" : "", vector,
-					   pci_name(pdev));
-	if (!vdev->ctx[vector].name)
-		return -ENOMEM;
+	if (vdev->ctx[vector].trigger == NULL) {
+		vdev->ctx[vector].name = kasprintf(GFP_KERNEL, "vfio-msi%s[%d](%s)",
+						   msix ? "x" : "", vector,
+						   pci_name(pdev));
+		if (!vdev->ctx[vector].name) {
+			eventfd_ctx_put(trigger);
+			return -ENOMEM;
+		}
 
-	trigger = eventfd_ctx_fdget(fd);
-	if (IS_ERR(trigger)) {
-		kfree(vdev->ctx[vector].name);
-		return PTR_ERR(trigger);
-	}
+		/*
+		 * The MSIx vector table resides in device memory which may be cleared
+		 * via backdoor resets. We don't allow direct access to the vector
+		 * table so even if a userspace driver attempts to save/restore around
+		 * such a reset it would be unsuccessful. To avoid this, restore the
+		 * cached value of the message prior to enabling.
+		 */
+		if (msix) {
+			struct msi_msg msg;
 
-	/*
-	 * The MSIx vector table resides in device memory which may be cleared
-	 * via backdoor resets. We don't allow direct access to the vector
-	 * table so even if a userspace driver attempts to save/restore around
-	 * such a reset it would be unsuccessful. To avoid this, restore the
-	 * cached value of the message prior to enabling.
-	 */
-	if (msix) {
-		struct msi_msg msg;
+			get_cached_msi_msg(irq, &msg);
+			pci_write_msi_msg(irq, &msg);
+		}
 
-		get_cached_msi_msg(irq, &msg);
-		pci_write_msi_msg(irq, &msg);
-	}
+		ret = request_irq(irq, vfio_msihandler, 0,
+				  vdev->ctx[vector].name, trigger);
+		if (ret) {
+			kfree(vdev->ctx[vector].name);
+			eventfd_ctx_put(trigger);
+			return ret;
+		}
 
-	ret = request_irq(irq, vfio_msihandler, 0,
-			  vdev->ctx[vector].name, trigger);
-	if (ret) {
-		kfree(vdev->ctx[vector].name);
-		eventfd_ctx_put(trigger);
-		return ret;
+		vdev->ctx[vector].producer.token = trigger;
+		vdev->ctx[vector].producer.irq = irq;
+		vdev->ctx[vector].trigger = trigger;
 	}
 
-	vdev->ctx[vector].producer.token = trigger;
-	vdev->ctx[vector].producer.irq = irq;
+	/* always update irte for posted mode */
 	ret = irq_bypass_register_producer(&vdev->ctx[vector].producer);
 	if (unlikely(ret))
 		dev_info(&pdev->dev,
 		"irq bypass producer (token %p) registration fails: %d\n",
 		vdev->ctx[vector].producer.token, ret);
 
-	vdev->ctx[vector].trigger = trigger;
-
 	return 0;
 }
 
Signature · 1 line
1.8.3.1
author
#1 lore raw

PATCH 1/2 genirq: introduce update_irq_devid()

In reply to Ben Luo

Sometimes, only the dev_id field of irqaction need to be changed. E.g. KVM VM with device passthru via VFIO may switch irq injection path between KVM irqfd and userspace eventfd. These two paths share the same irq num and handler for the same vector of a device, only with different dev_id referencing to different fds' contexts.

So, instead of free/request irq, only update dev_id of irqaction. This narrows the gap for setting up new irq (and irte, if has) and also gains some performance benefit.

---
 include/linux/interrupt.h |  3 ++
 kernel/irq/manage.c       | 74 +++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 77 insertions(+)
include/linux/interrupt.h +3 -0
index 5b8328a..2838113 100644
@@ -172,6 +172,9 @@ struct irqaction {
 request_percpu_nmi(unsigned int irq, irq_handler_t handler,
 		   const char *devname, void __percpu *dev);
 
+extern int update_irq_devid(unsigned int irq, void *dev_id,
+		void *new_dev_id);
+
 extern const void *free_irq(unsigned int, void *);
 extern void free_percpu_irq(unsigned int, void __percpu *);
 
kernel/irq/manage.c +74 -0

80 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#2 lore raw

Re: [PATCH 1/2] genirq: introduce update_irq_devid()

kernel/irq/manage.c int request_threaded_irq(unsigned int irq, irq_handler_t handler, · line 2091

Quoted 8 lines · Ben Luo
kernel/irq/manage.c +8 -0
+int update_irq_devid(unsigned int irq, void *dev_id, void *new_dev_id)
+{
+	struct irq_desc *desc = irq_to_desc(irq);
+	struct irqaction *action, **action_ptr;
+	unsigned long flags;
+
+	WARN(in_interrupt(),
+			"Trying to update IRQ %d from IRQ context!\n", irq);

This is broken. The function needs to return on that condition. Actually it cannot even be called from non-preemptible code.

What's worse is that if the interrupt in question is handled concurrently, then it will either see the old or the new dev_id and because the interrupt handler loop runs with desc->lock dropped even more crap can happen because dev_id can be subject to load and store tearing.

Staring at that, I see that there is the same issue in setup_irq() and free_irq(). It's actually worse there. I'll have a look.

kernel/irq/manage.c int request_threaded_irq(unsigned int irq, irq_handler_t handler, · line 2108

Quoted 10 lines
kernel/irq/manage.c +10 -0
+	/*
+	 * There can be multiple actions per IRQ descriptor, find the right
+	 * one based on the dev_id:
+	 */
+	action_ptr = &desc->action;
+	for (;;) {
+		action = *action_ptr;
+
+		if (!action) {
+			WARN(1, "Trying to update already-free IRQ %d\n", irq);

That's wrong in two aspects:

1) The warn should be outside of the locked region.

2) Just having the irq number is not useful for debugging either when the interrupt is shared.

Quoted 15 lines
kernel/irq/manage.c +15 -0
+			raw_spin_unlock_irqrestore(&desc->lock, flags);
+			chip_bus_sync_unlock(desc);
+			return -ENXIO;
+		}
+
+		if (action->dev_id == dev_id) {
+			action->dev_id = new_dev_id;
+			break;
+		}
+		action_ptr = &action->next;
+	}
+
+	raw_spin_unlock_irqrestore(&desc->lock, flags);
+	chip_bus_sync_unlock(desc);
+

kernel/irq/manage.c int request_threaded_irq(unsigned int irq, irq_handler_t handler, · line 2133

Quoted 10 lines
kernel/irq/manage.c +10 -0
+	/*
+	 * Make sure it's not being used on another CPU:
+	 * There is a risk of UAF for old *dev_id, if it is
+	 * freed in a short time after this func returns
+	 */
+	synchronize_irq(irq);
+
+	return 0;
+}
+EXPORT_SYMBOL(update_irq_devid);

EXPORT_SYMBOL_GPL() please.

Thanks,

tglx

author
#3 lore raw

Re: [PATCH 1/2] genirq: introduce update_irq_devid()

Quoted 12 lines · Thomas Gleixner

kernel/irq/manage.c int request_threaded_irq(unsigned int irq, irq_handler_t handler, · line 2091

Quoted 8 lines · Ben Luo
kernel/irq/manage.c +8 -0
+int update_irq_devid(unsigned int irq, void *dev_id, void *new_dev_id)
+{
+	struct irq_desc *desc = irq_to_desc(irq);
+	struct irqaction *action, **action_ptr;
+	unsigned long flags;
+
+	WARN(in_interrupt(),
+			"Trying to update IRQ %d from IRQ context!\n", irq);

This is broken. The function needs to return on that condition. Actually it cannot even be called from non-preemptible code.

Quoted 7 lines

What's worse is that if the interrupt in question is handled concurrently, then it will either see the old or the new dev_id and because the interrupt handler loop runs with desc->lock dropped even more crap can happen because dev_id can be subject to load and store tearing.

Staring at that, I see that there is the same issue in setup_irq() and free_irq(). It's actually worse there. I'll have a look.

ok,  will return with an error code in v2 in this case

Quoted 13 lines

kernel/irq/manage.c int request_threaded_irq(unsigned int irq, irq_handler_t handler, · line 2108

Quoted 11 lines
kernel/irq/manage.c +10 -0
+	/*
+	 * There can be multiple actions per IRQ descriptor, find the right
+	 * one based on the dev_id:
+	 */
+	action_ptr = &desc->action;
+	for (;;) {
+		action = *action_ptr;
+
+		if (!action) {
+			WARN(1, "Trying to update already-free IRQ %d\n", irq);

That's wrong in two aspects:

Quoted 4 lines

1) The warn should be outside of the locked region.

2) Just having the irq number is not useful for debugging either when the interrupt is shared.

will take care in v2

Quoted 18 lines
Quoted 18 lines
kernel/irq/manage.c +18 -0
+			raw_spin_unlock_irqrestore(&desc->lock, flags);
+			chip_bus_sync_unlock(desc);
+			return -ENXIO;
+		}
+
+		if (action->dev_id == dev_id) {
+			action->dev_id = new_dev_id;
+			break;
+		}
+		action_ptr = &action->next;
+	}
+
+	raw_spin_unlock_irqrestore(&desc->lock, flags);
+	chip_bus_sync_unlock(desc);
+
+	/*
+	 * Make sure it's not being used on another CPU:
+	 * There is a risk of UAF for old *dev_id, if it is
Quoted 8 lines

kernel/irq/manage.c int request_threaded_irq(unsigned int irq, irq_handler_t handler, · line 2133

Quoted 7 lines
kernel/irq/manage.c +7 -0
+	 * freed in a short time after this func returns
+	 */
+	synchronize_irq(irq);
+
+	return 0;
+}
+EXPORT_SYMBOL(update_irq_devid);

EXPORT_SYMBOL_GPL() please.

thanks, will use EXPORT_SYMBOL_GPL in v2

Quoted 4 lines

Thanks,

tglx

Thanks,

     Ben

#5 lore raw

Re: [PATCH 2/2] vfio_pci: make use of update_irq_devid and optimize irq ops

In reply to Ben Luo
Quoted 24 lines · Ben Luo

When userspace (e.g. qemu) triggers a switch between KVM irqfd and userspace eventfd, only dev_id of irq action (i.e. the "trigger" in this patch's context) will be changed, but a free-then-request-irq action is taken in current code. And, irq affinity setting in VM will also trigger a free-then-request-irq action, which actully changes nothing, but only fires a producer re-registration to update irte in case that posted-interrupt is in use.

This patch makes use of update_irq_devid() and optimize both cases above, which reduces the risk of losing interrupt and also cuts some overhead.

---
 drivers/vfio/pci/vfio_pci_intrs.c | 100 +++++++++++++++++++++++---------------
 1 file changed, 62 insertions(+), 38 deletions(-)
drivers/vfio/pci/vfio_pci_intrs.c +0 -0
index 3fa3f72..1323dc8 100644

drivers/vfio/pci/vfio_pci_intrs.c static int vfio_msi_set_vector_signal(struct vfio_pci_device *vdev, · line 288

Quoted 5 lines
drivers/vfio/pci/vfio_pci_intrs.c +1 -1
@@ -285,69 +285,93 @@ static int vfio_msi_set_vector_signal(struct vfio_pci_device *vdev,
 				      int vector, int fd, bool msix)
 {
 	struct pci_dev *pdev = vdev->pdev;
-	struct eventfd_ctx *trigger;
+	struct eventfd_ctx *trigger = NULL;

struct eventfd_ctx *trigger = NULL;
struct pci_dev *pdev = vdev->pdev;

to maintain reverse christmas tree?

drivers/vfio/pci/vfio_pci_intrs.c static int vfio_msi_set_vector_signal(struct vfio_pci_device *vdev, · line 297

Quoted 9 lines
drivers/vfio/pci/vfio_pci_intrs.c +4 -0
 	int irq, ret;
 
 	if (vector < 0 || vector >= vdev->num_ctx)
 		return -EINVAL;
 
+	if (fd >= 0) {
+		trigger = eventfd_ctx_fdget(fd);
+		if (IS_ERR(trigger))
+			return PTR_ERR(trigger);

It seems vdev->ctx[vector].trigger is freed even if fd < 0 before this patch. If it return here, vdev->ctx[vector].trigger is not free?

Quoted 25 lines
drivers/vfio/pci/vfio_pci_intrs.c +17 -5
+	}
+
 	irq = pci_irq_vector(pdev, vector);
 
 	if (vdev->ctx[vector].trigger) {
-		free_irq(irq, vdev->ctx[vector].trigger);
-		irq_bypass_unregister_producer(&vdev->ctx[vector].producer);
-		kfree(vdev->ctx[vector].name);
-		eventfd_ctx_put(vdev->ctx[vector].trigger);
-		vdev->ctx[vector].trigger = NULL;
+		if (vdev->ctx[vector].trigger != trigger) {
+			if (trigger) {
+				ret = update_irq_devid(irq,
+						vdev->ctx[vector].trigger, trigger);
+				if (unlikely(ret)) {
+					dev_info(&pdev->dev,
+							"update_irq_devid %d (token %p) fails: %d\n",
+							irq, vdev->ctx[vector].trigger, ret);
+					eventfd_ctx_put(trigger);
+					return ret;
+				}
+				irq_bypass_unregister_producer(&vdev->ctx[vector].producer);
+				eventfd_ctx_put(vdev->ctx[vector].trigger);
+				vdev->ctx[vector].producer.token = trigger;
+				vdev->ctx[vector].trigger = trigger;

drivers/vfio/pci/vfio_pci_intrs.c static int vfio_msi_set_vector_signal(struct vfio_pci_device *vdev, · line 327

Quoted 10 lines
drivers/vfio/pci/vfio_pci_intrs.c +9 -0
+			} else {
+				free_irq(irq, vdev->ctx[vector].trigger);
+				irq_bypass_unregister_producer(&vdev->ctx[vector].producer);
+				kfree(vdev->ctx[vector].name);
+				eventfd_ctx_put(vdev->ctx[vector].trigger);
+				vdev->ctx[vector].trigger = NULL;
+			}
+		} else
+			irq_bypass_unregister_producer(&vdev->ctx[vector].producer);
 	}

Maybe adjust it a litte to reduce indent and to improve readability?

	if (vdev->ctx[vector].trigger && vdev->ctx[vector].trigger == trigger) {
		irq_bypass_unregister_producer(&vdev->ctx[vector].producer);
	} else if (vdev->ctx[vector].trigger && !trigger) {
		free_irq(irq, vdev->ctx[vector].trigger);
		irq_bypass_unregister_producer(&vdev->ctx[vector].producer);
		kfree(vdev->ctx[vector].name);
		eventfd_ctx_put(vdev->ctx[vector].trigger);
		vdev->ctx[vector].trigger = NULL;
	} else if (vdev->ctx[vector].trigger) {
		ret = update_irq_devid(irq, vdev->ctx[vector].trigger, trigger);
		if (unlikely(ret)) {
			dev_info(&pdev->dev,
				 "update_irq_devid %d (token %p) fails: %d\n",
				 irq, vdev->ctx[vector].trigger, ret);
				 eventfd_ctx_put(trigger);
				 return ret;
		}
		irq_bypass_unregister_producer(&vdev->ctx[vector].producer);
		eventfd_ctx_put(vdev->ctx[vector].trigger);
		vdev->ctx[vector].producer.token = trigger;
		vdev->ctx[vector].trigger = trigger;
	}

drivers/vfio/pci/vfio_pci_intrs.c static int vfio_msi_set_vector_signal(struct vfio_pci_device *vdev, · line 332

Quoted 10 lines
drivers/vfio/pci/vfio_pci_intrs.c +1 -5
 	if (fd < 0)
 		return 0;
 
-	vdev->ctx[vector].name = kasprintf(GFP_KERNEL, "vfio-msi%s[%d](%s)",
-					   msix ? "x" : "", vector,
-					   pci_name(pdev));
-	if (!vdev->ctx[vector].name)
-		return -ENOMEM;
+	if (vdev->ctx[vector].trigger == NULL) {

It may be common to use the below check to do NULL checking:
If (!vdev->ctx[vector].trigger)

Quoted 73 lines
drivers/vfio/pci/vfio_pci_intrs.c +30 -27
+		vdev->ctx[vector].name = kasprintf(GFP_KERNEL, "vfio-msi%s[%d](%s)",
+						   msix ? "x" : "", vector,
+						   pci_name(pdev));
+		if (!vdev->ctx[vector].name) {
+			eventfd_ctx_put(trigger);
+			return -ENOMEM;
+		}
 
-	trigger = eventfd_ctx_fdget(fd);
-	if (IS_ERR(trigger)) {
-		kfree(vdev->ctx[vector].name);
-		return PTR_ERR(trigger);
-	}
+		/*
+		 * The MSIx vector table resides in device memory which may be cleared
+		 * via backdoor resets. We don't allow direct access to the vector
+		 * table so even if a userspace driver attempts to save/restore around
+		 * such a reset it would be unsuccessful. To avoid this, restore the
+		 * cached value of the message prior to enabling.
+		 */
+		if (msix) {
+			struct msi_msg msg;
 
-	/*
-	 * The MSIx vector table resides in device memory which may be cleared
-	 * via backdoor resets. We don't allow direct access to the vector
-	 * table so even if a userspace driver attempts to save/restore around
-	 * such a reset it would be unsuccessful. To avoid this, restore the
-	 * cached value of the message prior to enabling.
-	 */
-	if (msix) {
-		struct msi_msg msg;
+			get_cached_msi_msg(irq, &msg);
+			pci_write_msi_msg(irq, &msg);
+		}
 
-		get_cached_msi_msg(irq, &msg);
-		pci_write_msi_msg(irq, &msg);
-	}
+		ret = request_irq(irq, vfio_msihandler, 0,
+				  vdev->ctx[vector].name, trigger);
+		if (ret) {
+			kfree(vdev->ctx[vector].name);
+			eventfd_ctx_put(trigger);
+			return ret;
+		}
 
-	ret = request_irq(irq, vfio_msihandler, 0,
-			  vdev->ctx[vector].name, trigger);
-	if (ret) {
-		kfree(vdev->ctx[vector].name);
-		eventfd_ctx_put(trigger);
-		return ret;
+		vdev->ctx[vector].producer.token = trigger;
+		vdev->ctx[vector].producer.irq = irq;
+		vdev->ctx[vector].trigger = trigger;
 	}
 
-	vdev->ctx[vector].producer.token = trigger;
-	vdev->ctx[vector].producer.irq = irq;
+	/* always update irte for posted mode */
 	ret = irq_bypass_register_producer(&vdev->ctx[vector].producer);
 	if (unlikely(ret))
 		dev_info(&pdev->dev,
 		"irq bypass producer (token %p) registration fails: %d\n",
 		vdev->ctx[vector].producer.token, ret);
 
-	vdev->ctx[vector].trigger = trigger;
-
 	return 0;
 }
 
 
author
#6 lore raw

Re: [PATCH 2/2] vfio_pci: make use of update_irq_devid and optimize irq ops

Quoted 33 lines · Yunsheng Lin
Quoted 24 lines · Ben Luo

When userspace (e.g. qemu) triggers a switch between KVM irqfd and userspace eventfd, only dev_id of irq action (i.e. the "trigger" in this patch's context) will be changed, but a free-then-request-irq action is taken in current code. And, irq affinity setting in VM will also trigger a free-then-request-irq action, which actully changes nothing, but only fires a producer re-registration to update irte in case that posted-interrupt is in use.

This patch makes use of update_irq_devid() and optimize both cases above, which reduces the risk of losing interrupt and also cuts some overhead.

---
  drivers/vfio/pci/vfio_pci_intrs.c | 100 +++++++++++++++++++++++---------------
  1 file changed, 62 insertions(+), 38 deletions(-)
drivers/vfio/pci/vfio_pci_intrs.c +0 -0
index 3fa3f72..1323dc8 100644

drivers/vfio/pci/vfio_pci_intrs.c static int vfio_msi_set_vector_signal(struct vfio_pci_device *vdev, · line 288

Quoted 5 lines
drivers/vfio/pci/vfio_pci_intrs.c +1 -1
@@ -285,69 +285,93 @@ static int vfio_msi_set_vector_signal(struct vfio_pci_device *vdev,
  				      int vector, int fd, bool msix)
  {
  	struct pci_dev *pdev = vdev->pdev;
-	struct eventfd_ctx *trigger;
+	struct eventfd_ctx *trigger = NULL;

struct eventfd_ctx *trigger = NULL;
struct pci_dev *pdev = vdev->pdev;

Quoted 1 line

to maintain reverse christmas tree?

ok,  keep reverse christmas tree in v2

Quoted 6 lines
Quoted 6 lines
drivers/vfio/pci/vfio_pci_intrs.c +0 -0
  	int irq, ret;
 
  	if (vector < 0 || vector >= vdev->num_ctx)
  		return -EINVAL;
 
Quoted 6 lines

drivers/vfio/pci/vfio_pci_intrs.c static int vfio_msi_set_vector_signal(struct vfio_pci_device *vdev, · line 297

Quoted 4 lines
drivers/vfio/pci/vfio_pci_intrs.c +4 -0
+	if (fd >= 0) {
+		trigger = eventfd_ctx_fdget(fd);
+		if (IS_ERR(trigger))
+			return PTR_ERR(trigger);

It seems vdev->ctx[vector].trigger is freed even if fd < 0 before this patch. If it return here, vdev->ctx[vector].trigger is not free?

if fd < 0, it won't enter here

if fd >= 0,  I think it's better to return and leave everything as it was, and

let the caller to deal with this bad fd case and disable msi to free the resouces if it wants

Quoted 52 lines
Quoted 26 lines
drivers/vfio/pci/vfio_pci_intrs.c +17 -5
+	}
+
  	irq = pci_irq_vector(pdev, vector);
 
  	if (vdev->ctx[vector].trigger) {
-		free_irq(irq, vdev->ctx[vector].trigger);
-		irq_bypass_unregister_producer(&vdev->ctx[vector].producer);
-		kfree(vdev->ctx[vector].name);
-		eventfd_ctx_put(vdev->ctx[vector].trigger);
-		vdev->ctx[vector].trigger = NULL;
+		if (vdev->ctx[vector].trigger != trigger) {
+			if (trigger) {
+				ret = update_irq_devid(irq,
+						vdev->ctx[vector].trigger, trigger);
+				if (unlikely(ret)) {
+					dev_info(&pdev->dev,
+							"update_irq_devid %d (token %p) fails: %d\n",
+							irq, vdev->ctx[vector].trigger, ret);
+					eventfd_ctx_put(trigger);
+					return ret;
+				}
+				irq_bypass_unregister_producer(&vdev->ctx[vector].producer);
+				eventfd_ctx_put(vdev->ctx[vector].trigger);
+				vdev->ctx[vector].producer.token = trigger;
+				vdev->ctx[vector].trigger = trigger;

drivers/vfio/pci/vfio_pci_intrs.c static int vfio_msi_set_vector_signal(struct vfio_pci_device *vdev, · line 327

Quoted 10 lines
drivers/vfio/pci/vfio_pci_intrs.c +9 -0
+			} else {
+				free_irq(irq, vdev->ctx[vector].trigger);
+				irq_bypass_unregister_producer(&vdev->ctx[vector].producer);
+				kfree(vdev->ctx[vector].name);
+				eventfd_ctx_put(vdev->ctx[vector].trigger);
+				vdev->ctx[vector].trigger = NULL;
+			}
+		} else
+			irq_bypass_unregister_producer(&vdev->ctx[vector].producer);
  	}

Maybe adjust it a litte to reduce indent and to improve readability?

	if (vdev->ctx[vector].trigger && vdev->ctx[vector].trigger == trigger) {
		irq_bypass_unregister_producer(&vdev->ctx[vector].producer);
	} else if (vdev->ctx[vector].trigger && !trigger) {
		free_irq(irq, vdev->ctx[vector].trigger);
		irq_bypass_unregister_producer(&vdev->ctx[vector].producer);
		kfree(vdev->ctx[vector].name);
		eventfd_ctx_put(vdev->ctx[vector].trigger);
		vdev->ctx[vector].trigger = NULL;
	} else if (vdev->ctx[vector].trigger) {
		ret = update_irq_devid(irq, vdev->ctx[vector].trigger, trigger);
		if (unlikely(ret)) {
			dev_info(&pdev->dev,
				 "update_irq_devid %d (token %p) fails: %d\n",
				 irq, vdev->ctx[vector].trigger, ret);
Quoted 9 lines
				 eventfd_ctx_put(trigger);
				 return ret;
		}
		irq_bypass_unregister_producer(&vdev->ctx[vector].producer);
		eventfd_ctx_put(vdev->ctx[vector].trigger);
		vdev->ctx[vector].producer.token = trigger;
		vdev->ctx[vector].trigger = trigger;
	}

I will reformat this chunk in v2

Quoted 4 lines
Quoted 4 lines
drivers/vfio/pci/vfio_pci_intrs.c +0 -0
  	if (fd < 0)
  		return 0;
 
Quoted 8 lines

drivers/vfio/pci/vfio_pci_intrs.c static int vfio_msi_set_vector_signal(struct vfio_pci_device *vdev, · line 332

Quoted 6 lines
drivers/vfio/pci/vfio_pci_intrs.c +1 -5
-	vdev->ctx[vector].name = kasprintf(GFP_KERNEL, "vfio-msi%s[%d](%s)",
-					   msix ? "x" : "", vector,
-					   pci_name(pdev));
-	if (!vdev->ctx[vector].name)
-		return -ENOMEM;
+	if (vdev->ctx[vector].trigger == NULL) {

It may be common to use the below check to do NULL checking:
If (!vdev->ctx[vector].trigger)

ok, make it this way in v2

Quoted 68 lines
Quoted 68 lines
drivers/vfio/pci/vfio_pci_intrs.c +30 -25
+		vdev->ctx[vector].name = kasprintf(GFP_KERNEL, "vfio-msi%s[%d](%s)",
+						   msix ? "x" : "", vector,
+						   pci_name(pdev));
+		if (!vdev->ctx[vector].name) {
+			eventfd_ctx_put(trigger);
+			return -ENOMEM;
+		}
 
-	trigger = eventfd_ctx_fdget(fd);
-	if (IS_ERR(trigger)) {
-		kfree(vdev->ctx[vector].name);
-		return PTR_ERR(trigger);
-	}
+		/*
+		 * The MSIx vector table resides in device memory which may be cleared
+		 * via backdoor resets. We don't allow direct access to the vector
+		 * table so even if a userspace driver attempts to save/restore around
+		 * such a reset it would be unsuccessful. To avoid this, restore the
+		 * cached value of the message prior to enabling.
+		 */
+		if (msix) {
+			struct msi_msg msg;
 
-	/*
-	 * The MSIx vector table resides in device memory which may be cleared
-	 * via backdoor resets. We don't allow direct access to the vector
-	 * table so even if a userspace driver attempts to save/restore around
-	 * such a reset it would be unsuccessful. To avoid this, restore the
-	 * cached value of the message prior to enabling.
-	 */
-	if (msix) {
-		struct msi_msg msg;
+			get_cached_msi_msg(irq, &msg);
+			pci_write_msi_msg(irq, &msg);
+		}
 
-		get_cached_msi_msg(irq, &msg);
-		pci_write_msi_msg(irq, &msg);
-	}
+		ret = request_irq(irq, vfio_msihandler, 0,
+				  vdev->ctx[vector].name, trigger);
+		if (ret) {
+			kfree(vdev->ctx[vector].name);
+			eventfd_ctx_put(trigger);
+			return ret;
+		}
 
-	ret = request_irq(irq, vfio_msihandler, 0,
-			  vdev->ctx[vector].name, trigger);
-	if (ret) {
-		kfree(vdev->ctx[vector].name);
-		eventfd_ctx_put(trigger);
-		return ret;
+		vdev->ctx[vector].producer.token = trigger;
+		vdev->ctx[vector].producer.irq = irq;
+		vdev->ctx[vector].trigger = trigger;
  	}
 
-	vdev->ctx[vector].producer.token = trigger;
-	vdev->ctx[vector].producer.irq = irq;
+	/* always update irte for posted mode */
  	ret = irq_bypass_register_producer(&vdev->ctx[vector].producer);
  	if (unlikely(ret))
  		dev_info(&pdev->dev,
  		"irq bypass producer (token %p) registration fails: %d\n",
  		vdev->ctx[vector].producer.token, ret);
 
Quoted 5 lines

drivers/vfio/pci/vfio_pci_intrs.c static int vfio_msi_set_vector_signal(struct vfio_pci_device *vdev, · line 377

Quoted 5 lines
drivers/vfio/pci/vfio_pci_intrs.c +0 -2
-	vdev->ctx[vector].trigger = trigger;
-
  	return 0;
  }
 

Thanks,

    Ben

author
#7 lore raw

v2 cover introduce update_irq_devid and optimize VFIO irq ops

In reply to Ben Luo

Currently, VFIO takes a lot of free-then-request-irq actions whenever a VM (with device passthru via VFIO) sets irq affinity or mask/unmask irq. Those actions only change the cookie data of irqaction or even change nothing. The free-then-request-irq not only add more latency, but also increases the risk of losing interrupt, this may lead to a VM hung forever in waiting IO completion

This patchset solved this issue by:
Patch 2 introduces update_irq_devid to only update dev_id of irqaction Patch 3 make use of update_irq_devid and optimize irq operations in VFIO

changes from v1:
- add Patch 1 to enhance error recovery etc. in free irq per tglx's comments
- enhance error recovery code and debugging info in update_irq_devid
- use __must_check in external referencing of update_irq_devid
- use EXPORT_SYMBOL_GPL for update_irq_devid
- reformat code of patch 3 for better readability

Ben Luo (3):
genirq: enhance error recovery code in free irq
genirq: introduce update_irq_devid()
vfio_pci: make use of update_irq_devid and optimize irq ops

 drivers/vfio/pci/vfio_pci_intrs.c |  99 +++++++++++++++++++++-------------
 include/linux/interrupt.h         |   3 ++
 kernel/irq/manage.c               | 111 +++++++++++++++++++++++++++++++++-----
 3 files changed, 163 insertions(+), 50 deletions(-)
Signature · 1 line
1.8.3.1
author
#8 lore raw

PATCH v2 1/3 genirq: enhance error recovery code in free irq

Per Thomas Gleixner's comments:
1) free_irq/free_percpu_irq returns if called from IRQ context
2) move WARN out of the locked region and print out dev_id

---
 kernel/irq/manage.c | 32 ++++++++++++++++++++------------
 1 file changed, 20 insertions(+), 12 deletions(-)
kernel/irq/manage.c +20 -12
index e8f7f17..b97ee5e 100644
@@ -1690,7 +1690,11 @@ static struct irqaction *__free_irq(struct irq_desc *desc, void *dev_id)
 	struct irqaction *action, **action_ptr;
 	unsigned long flags;
 
-	WARN(in_interrupt(), "Trying to free IRQ %d from IRQ context!\n", irq);
+	if (in_interrupt()) {
+		WARN(1, "Trying to free IRQ %d (dev_id %p) from IRQ context!\n",
+				irq, dev_id);
+		return NULL;
+	}
 
 	mutex_lock(&desc->request_mutex);
 	chip_bus_lock(desc);
@@ -1705,10 +1709,11 @@ static struct irqaction *__free_irq(struct irq_desc *desc, void *dev_id)
 		action = *action_ptr;
 
 		if (!action) {
-			WARN(1, "Trying to free already-free IRQ %d\n", irq);
 			raw_spin_unlock_irqrestore(&desc->lock, flags);
 			chip_bus_sync_unlock(desc);
 			mutex_unlock(&desc->request_mutex);
+			WARN(1, "Trying to free already-free IRQ %d (dev_id %p)\n",
+					irq, dev_id);
 			return NULL;
 		}
 
@@ -2286,7 +2291,11 @@ static struct irqaction *__free_percpu_irq(unsigned int irq, void __percpu *dev_
 	struct irqaction *action;
 	unsigned long flags;
 
-	WARN(in_interrupt(), "Trying to free IRQ %d from IRQ context!\n", irq);
+	if (in_interrupt()) {
+		WARN(1, "Trying to free IRQ %d (dev_id %p) from IRQ context!\n",
+				irq, dev_id);
+		return NULL;
+	}
 
 	if (!desc)
 		return NULL;
@@ -2295,14 +2304,17 @@ static struct irqaction *__free_percpu_irq(unsigned int irq, void __percpu *dev_
 
 	action = desc->action;
 	if (!action || action->percpu_dev_id != dev_id) {
-		WARN(1, "Trying to free already-free IRQ %d\n", irq);
-		goto bad;
+		raw_spin_unlock_irqrestore(&desc->lock, flags);
+		WARN(1, "Trying to free already-free IRQ (dev_id %p) %d\n",
+				irq, dev_id);
+		return NULL;
 	}
 
 	if (!cpumask_empty(desc->percpu_enabled)) {
-		WARN(1, "percpu IRQ %d still enabled on CPU%d!\n",
-		     irq, cpumask_first(desc->percpu_enabled));
-		goto bad;
+		raw_spin_unlock_irqrestore(&desc->lock, flags);
+		WARN(1, "percpu IRQ %d (dev_id %p) still enabled on CPU%d!\n",
+		     irq, dev_id, cpumask_first(desc->percpu_enabled));
+		return NULL;
 	}
 
 	/* Found it - now remove it from the list of entries: */
@@ -2317,10 +2329,6 @@ static struct irqaction *__free_percpu_irq(unsigned int irq, void __percpu *dev_
 	irq_chip_pm_put(&desc->irq_data);
 	module_put(desc->owner);
 	return action;
-
-bad:
-	raw_spin_unlock_irqrestore(&desc->lock, flags);
-	return NULL;
 }
 
 /**
Signature · 1 line
1.8.3.1
author
#9 lore raw

PATCH v2 2/3 genirq: introduce update_irq_devid()

In reply to Ben Luo

Sometimes, only the dev_id field of irqaction need to be changed. E.g. KVM VM with device passthru via VFIO may switch irq injection path between KVM irqfd and userspace eventfd. These two paths share the same irq num and handler for the same vector of a device, only with different dev_id referencing to different fds' contexts.

So, instead of free/request irq, only update dev_id of irqaction. This narrows the gap for setting up new irq (and irte, if has) and also gains some performance benefit.

---
 include/linux/interrupt.h |  3 ++
 kernel/irq/manage.c       | 79 +++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 82 insertions(+)
include/linux/interrupt.h +3 -0
index 5b8328a..6060c5a 100644
@@ -172,6 +172,9 @@ struct irqaction {
 request_percpu_nmi(unsigned int irq, irq_handler_t handler,
 		   const char *devname, void __percpu *dev);
 
+extern int __must_check
+update_irq_devid(unsigned int irq, void *dev_id, void *new_dev_id);
+
 extern const void *free_irq(unsigned int, void *);
 extern void free_percpu_irq(unsigned int, void __percpu *);
 
kernel/irq/manage.c +79 -0

85 lines not on this page · raw message

Signature · 1 line
1.8.3.1
author
#10 lore raw

PATCH v2 3/3 vfio_pci: make use of update_irq_devid and optimize irq ops

In reply to Ben Luo

When userspace (e.g. qemu) triggers a switch between KVM irqfd and userspace eventfd, only dev_id of irq action (i.e. the "trigger" in this patch's context) will be changed, but a free-then-request-irq action is taken in current code. And, irq affinity setting in VM will also trigger a free-then-request-irq action, which actully changes nothing, but only fires a producer re-registration to update irte in case that posted-interrupt is in use.

This patch makes use of update_irq_devid() and optimize both cases above, which reduces the risk of losing interrupt and also cuts some overhead.

---
 drivers/vfio/pci/vfio_pci_intrs.c | 99 ++++++++++++++++++++++++---------------
 1 file changed, 61 insertions(+), 38 deletions(-)
drivers/vfio/pci/vfio_pci_intrs.c +61 -38
index 3fa3f72..541153d 100644
@@ -284,70 +284,93 @@ static int vfio_msi_enable(struct vfio_pci_device *vdev, int nvec, bool msix)
 static int vfio_msi_set_vector_signal(struct vfio_pci_device *vdev,
 				      int vector, int fd, bool msix)
 {
+	struct eventfd_ctx *trigger = NULL;
 	struct pci_dev *pdev = vdev->pdev;
-	struct eventfd_ctx *trigger;
 	int irq, ret;
 
 	if (vector < 0 || vector >= vdev->num_ctx)
 		return -EINVAL;
 
+	if (fd >= 0) {
+		trigger = eventfd_ctx_fdget(fd);
+		if (IS_ERR(trigger))
+			return PTR_ERR(trigger);
+	}
+
 	irq = pci_irq_vector(pdev, vector);
 
 	if (vdev->ctx[vector].trigger) {
-		free_irq(irq, vdev->ctx[vector].trigger);
-		irq_bypass_unregister_producer(&vdev->ctx[vector].producer);
-		kfree(vdev->ctx[vector].name);
-		eventfd_ctx_put(vdev->ctx[vector].trigger);
-		vdev->ctx[vector].trigger = NULL;
+		if (vdev->ctx[vector].trigger == trigger) {
+			irq_bypass_unregister_producer(&vdev->ctx[vector].producer);
+		} else if (trigger) {
+			ret = update_irq_devid(irq,
+					vdev->ctx[vector].trigger, trigger);
+			if (unlikely(ret)) {
+				dev_info(&pdev->dev,
+						"update_irq_devid %d (token %p) fails: %d\n",
+						irq, vdev->ctx[vector].trigger, ret);
+				eventfd_ctx_put(trigger);
+				return ret;
+			}
+			irq_bypass_unregister_producer(&vdev->ctx[vector].producer);
+			eventfd_ctx_put(vdev->ctx[vector].trigger);
+			vdev->ctx[vector].producer.token = trigger;
+			vdev->ctx[vector].trigger = trigger;
+		} else {
+			free_irq(irq, vdev->ctx[vector].trigger);
+			irq_bypass_unregister_producer(&vdev->ctx[vector].producer);
+			kfree(vdev->ctx[vector].name);
+			eventfd_ctx_put(vdev->ctx[vector].trigger);
+			vdev->ctx[vector].trigger = NULL;
+		}
 	}
 
 	if (fd < 0)
 		return 0;
 
-	vdev->ctx[vector].name = kasprintf(GFP_KERNEL, "vfio-msi%s[%d](%s)",
-					   msix ? "x" : "", vector,
-					   pci_name(pdev));
-	if (!vdev->ctx[vector].name)
-		return -ENOMEM;
+	if (!vdev->ctx[vector].trigger) {
+		vdev->ctx[vector].name = kasprintf(GFP_KERNEL, "vfio-msi%s[%d](%s)",
+						   msix ? "x" : "", vector,
+						   pci_name(pdev));
+		if (!vdev->ctx[vector].name) {
+			eventfd_ctx_put(trigger);
+			return -ENOMEM;
+		}
 
-	trigger = eventfd_ctx_fdget(fd);
-	if (IS_ERR(trigger)) {
-		kfree(vdev->ctx[vector].name);
-		return PTR_ERR(trigger);
-	}
+		/*
+		 * The MSIx vector table resides in device memory which may be cleared
+		 * via backdoor resets. We don't allow direct access to the vector
+		 * table so even if a userspace driver attempts to save/restore around
+		 * such a reset it would be unsuccessful. To avoid this, restore the
+		 * cached value of the message prior to enabling.
+		 */
+		if (msix) {
+			struct msi_msg msg;
 
-	/*
-	 * The MSIx vector table resides in device memory which may be cleared
-	 * via backdoor resets. We don't allow direct access to the vector
-	 * table so even if a userspace driver attempts to save/restore around
-	 * such a reset it would be unsuccessful. To avoid this, restore the
-	 * cached value of the message prior to enabling.
-	 */
-	if (msix) {
-		struct msi_msg msg;
+			get_cached_msi_msg(irq, &msg);
+			pci_write_msi_msg(irq, &msg);
+		}
 
-		get_cached_msi_msg(irq, &msg);
-		pci_write_msi_msg(irq, &msg);
-	}
+		ret = request_irq(irq, vfio_msihandler, 0,
+				  vdev->ctx[vector].name, trigger);
+		if (ret) {
+			kfree(vdev->ctx[vector].name);
+			eventfd_ctx_put(trigger);
+			return ret;
+		}
 
-	ret = request_irq(irq, vfio_msihandler, 0,
-			  vdev->ctx[vector].name, trigger);
-	if (ret) {
-		kfree(vdev->ctx[vector].name);
-		eventfd_ctx_put(trigger);
-		return ret;
+		vdev->ctx[vector].producer.token = trigger;
+		vdev->ctx[vector].producer.irq = irq;
+		vdev->ctx[vector].trigger = trigger;
 	}
 
-	vdev->ctx[vector].producer.token = trigger;
-	vdev->ctx[vector].producer.irq = irq;
+	/* always update irte for posted mode */
 	ret = irq_bypass_register_producer(&vdev->ctx[vector].producer);
 	if (unlikely(ret))
 		dev_info(&pdev->dev,
 		"irq bypass producer (token %p) registration fails: %d\n",
 		vdev->ctx[vector].producer.token, ret);
 
-	vdev->ctx[vector].trigger = trigger;
-
 	return 0;
 }
 
Signature · 1 line
1.8.3.1