Skip to contentSkip to conversation

msi support for platform devices

RFC In review Vikas Gupta 4 participants 55 messages Nov 5, 2020 – Feb 8, 2021

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

cover msi support for platform devices

This RFC adds support for MSI for platform devices.
a) MSI(s) is/are added in addition to the normal interrupts.
b) The vendor specific MSI configuration can be done using callbacks which is implemented as msi module.
c) Adds a msi handling module for the Broadcom platform devices.

Vikas Gupta (3):
vfio/platform: add support for msi
vfio/platform: change cleanup order
vfio/platform: add Broadcom msi module

 drivers/vfio/platform/Kconfig                 |   1 +
 drivers/vfio/platform/Makefile                |   1 +
 drivers/vfio/platform/msi/Kconfig             |   9 +
 drivers/vfio/platform/msi/Makefile            |   2 +
 .../vfio/platform/msi/vfio_platform_bcmplt.c  |  74 ++++++
 drivers/vfio/platform/vfio_platform_common.c  |  86 ++++++-
 drivers/vfio/platform/vfio_platform_irq.c     | 235 +++++++++++++++++-
 drivers/vfio/platform/vfio_platform_private.h |  23 ++
 include/uapi/linux/vfio.h                     |   1 +
 9 files changed, 417 insertions(+), 15 deletions(-)
 create mode 100644 drivers/vfio/platform/msi/Kconfig
 create mode 100644 drivers/vfio/platform/msi/Makefile
 create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
Signature · 1 line
2.17.1
#1 lore raw

RFC 1/3 vfio/platform: add support for msi

MSI support for platform devices.

---
 drivers/vfio/platform/vfio_platform_common.c  |  84 ++++++-
 drivers/vfio/platform/vfio_platform_irq.c     | 235 +++++++++++++++++-
 drivers/vfio/platform/vfio_platform_private.h |  23 ++
 include/uapi/linux/vfio.h                     |   1 +
 4 files changed, 329 insertions(+), 14 deletions(-)
drivers/vfio/platform/vfio_platform_common.c +83 -1
index c0771a9567fb..c713f4e4c552 100644
@@ -26,6 +26,10 @@
 #define VFIO_PLATFORM_IS_ACPI(vdev) ((vdev)->acpihid != NULL)
 
 static LIST_HEAD(reset_list);
+
+/* devices having MSI support */
+static LIST_HEAD(msi_list);
+
 static DEFINE_MUTEX(driver_lock);
 
 static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,
@@ -47,6 +51,26 @@ static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,
 	return reset_fn;
 }
 
+static bool vfio_platform_lookup_msi(struct vfio_platform_device *vdev)
+{
+	bool has_msi = false;
+	struct vfio_platform_msi_node *iter;
+
+	mutex_lock(&driver_lock);
+	list_for_each_entry(iter, &msi_list, link) {
+		if (!strcmp(iter->compat, vdev->compat) &&
+		    try_module_get(iter->owner)) {
+			vdev->msi_module = iter->owner;
+			vdev->of_get_msi = iter->of_get_msi;
+			vdev->of_msi_write = iter->of_msi_write;
+			has_msi = true;
+			break;
+		}
+	}
+	mutex_unlock(&driver_lock);
+	return has_msi;
+}
+
 static int vfio_platform_acpi_probe(struct vfio_platform_device *vdev,
 				    struct device *dev)
 {
@@ -110,6 +134,11 @@ static bool vfio_platform_has_reset(struct vfio_platform_device *vdev)
 	return vdev->of_reset ? true : false;
 }
 
+static bool vfio_platform_has_msi(struct vfio_platform_device *vdev)
+{
+	return vdev->of_get_msi ? true : false;
+}
+
 static int vfio_platform_get_reset(struct vfio_platform_device *vdev)
 {
 	if (VFIO_PLATFORM_IS_ACPI(vdev))
@@ -126,6 +155,19 @@ static int vfio_platform_get_reset(struct vfio_platform_device *vdev)
 	return vdev->of_reset ? 0 : -ENOENT;
 }
 
+static int vfio_platform_get_msi(struct vfio_platform_device *vdev)
+{
+	bool has_msi;
+
+	has_msi = vfio_platform_lookup_msi(vdev);
+	if (!has_msi) {
+		request_module("vfio-msi:%s", vdev->compat);
+		has_msi = vfio_platform_lookup_msi(vdev);
+	}
+
+	return has_msi ? 0 : -ENOENT;
+}
+
 static void vfio_platform_put_reset(struct vfio_platform_device *vdev)
 {
 	if (VFIO_PLATFORM_IS_ACPI(vdev))
@@ -135,6 +177,12 @@ static void vfio_platform_put_reset(struct vfio_platform_device *vdev)
 		module_put(vdev->reset_module);
 }
 
+static void vfio_platform_put_msi(struct vfio_platform_device *vdev)
+{
+	if (vdev->of_get_msi)
+		module_put(vdev->msi_module);
+}
+
 static int vfio_platform_regions_init(struct vfio_platform_device *vdev)
 {
 	int cnt = 0, i;
@@ -313,6 +361,10 @@ static long vfio_platform_ioctl(void *device_data,
 
 		if (vfio_platform_has_reset(vdev))
 			vdev->flags |= VFIO_DEVICE_FLAGS_RESET;
+
+		if (vfio_platform_has_msi(vdev))
+			vdev->flags |= VFIO_DEVICE_FLAGS_MSI;
+
 		info.flags = vdev->flags;
 		info.num_regions = vdev->num_regions;
 		info.num_irqs = vdev->num_irqs;
@@ -679,11 +731,15 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,
 		return ret;
 	}
 
+	ret = vfio_platform_get_msi(vdev);
+	if (ret)
+		dev_info(vdev->device, "msi not supported\n");
+
 	group = vfio_iommu_group_get(dev);
 	if (!group) {
 		dev_err(dev, "No IOMMU group for device %s\n", vdev->name);
 		ret = -EINVAL;
-		goto put_reset;
+		goto put_msi;
 	}
 
 	ret = vfio_add_group_dev(dev, &vfio_platform_ops, vdev);
@@ -697,6 +753,8 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,
 
 put_iommu:
 	vfio_iommu_group_put(group, dev);
+put_msi:
+	vfio_platform_put_msi(vdev);
 put_reset:
 	vfio_platform_put_reset(vdev);
 	return ret;
@@ -745,6 +803,30 @@ void vfio_platform_unregister_reset(const char *compat,
 }
 EXPORT_SYMBOL_GPL(vfio_platform_unregister_reset);
 
+void __vfio_platform_register_msi(struct vfio_platform_msi_node *node)
+{
+	mutex_lock(&driver_lock);
+	list_add(&node->link, &msi_list);
+	mutex_unlock(&driver_lock);
+}
+EXPORT_SYMBOL_GPL(__vfio_platform_register_msi);
+
+void vfio_platform_unregister_msi(const char *compat)
+{
+	struct vfio_platform_msi_node *iter, *temp;
+
+	mutex_lock(&driver_lock);
+	list_for_each_entry_safe(iter, temp, &msi_list, link) {
+		if (!strcmp(iter->compat, compat)) {
+			list_del(&iter->link);
+			break;
+		}
+	}
+
+	mutex_unlock(&driver_lock);
+}
+EXPORT_SYMBOL_GPL(vfio_platform_unregister_msi);
+
 MODULE_VERSION(DRIVER_VERSION);
 MODULE_LICENSE("GPL v2");
 MODULE_AUTHOR(DRIVER_AUTHOR);
drivers/vfio/platform/vfio_platform_irq.c +222 -13

282 lines not on this page · raw message

drivers/vfio/platform/vfio_platform_private.h +23 -0
index 289089910643..2aea445e1071 100644
@@ -9,6 +9,7 @@
 
 #include <linux/types.h>
 #include <linux/interrupt.h>
+#include <linux/msi.h>
 
 #define VFIO_PLATFORM_OFFSET_SHIFT   40
 #define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
@@ -26,6 +27,7 @@ struct vfio_platform_irq {
 	char			*name;
 	struct eventfd_ctx	*trigger;
 	bool			masked;
+	bool			is_msi;
 	spinlock_t		lock;
 	struct virqfd		*unmask;
 	struct virqfd		*mask;
@@ -46,13 +48,16 @@ struct vfio_platform_device {
 	u32				num_regions;
 	struct vfio_platform_irq	*irqs;
 	u32				num_irqs;
+	u32				num_msis;
 	int				refcnt;
 	struct mutex			igate;
 	struct module			*parent_module;
 	const char			*compat;
 	const char			*acpihid;
 	struct module			*reset_module;
+	struct module			*msi_module;
 	struct device			*device;
+	int				config_msi;
 
 	/*
 	 * These fields should be filled by the bus specific binder
@@ -65,11 +70,19 @@ struct vfio_platform_device {
 		(*get_resource)(struct vfio_platform_device *vdev, int i);
 	int	(*get_irq)(struct vfio_platform_device *vdev, int i);
 	int	(*of_reset)(struct vfio_platform_device *vdev);
+	u32	(*of_get_msi)(struct vfio_platform_device *vdev);
+	int	(*of_msi_write)(struct vfio_platform_device *vdev,
+				struct msi_desc *desc,
+				struct msi_msg *msg);
 
 	bool				reset_required;
 };
 
 typedef int (*vfio_platform_reset_fn_t)(struct vfio_platform_device *vdev);
+typedef u32 (*vfio_platform_get_msi_fn_t)(struct vfio_platform_device *vdev);
+typedef int (*vfio_platform_msi_write_fn_t)(struct vfio_platform_device *vdev,
+					    struct msi_desc *desc,
+					    struct msi_msg *msg);
 
 struct vfio_platform_reset_node {
 	struct list_head link;
@@ -78,6 +91,14 @@ struct vfio_platform_reset_node {
 	vfio_platform_reset_fn_t of_reset;
 };
 
+struct vfio_platform_msi_node {
+	struct list_head link;
+	char *compat;
+	struct module *owner;
+	vfio_platform_get_msi_fn_t of_get_msi;
+	vfio_platform_msi_write_fn_t of_msi_write;
+};
+
 extern int vfio_platform_probe_common(struct vfio_platform_device *vdev,
 				      struct device *dev);
 extern struct vfio_platform_device *vfio_platform_remove_common
@@ -94,6 +115,8 @@ extern int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 extern void __vfio_platform_register_reset(struct vfio_platform_reset_node *n);
 extern void vfio_platform_unregister_reset(const char *compat,
 					   vfio_platform_reset_fn_t fn);
+void __vfio_platform_register_msi(struct vfio_platform_msi_node *n);
+void vfio_platform_unregister_msi(const char *compat);
 #define vfio_platform_register_reset(__compat, __reset)		\
 static struct vfio_platform_reset_node __reset ## _node = {	\
 	.owner = THIS_MODULE,					\
include/uapi/linux/vfio.h +1 -0
index 2f313a238a8f..aab051e8338d 100644
@@ -203,6 +203,7 @@ struct vfio_device_info {
 #define VFIO_DEVICE_FLAGS_AP	(1 << 5)	/* vfio-ap device */
 #define VFIO_DEVICE_FLAGS_FSL_MC (1 << 6)	/* vfio-fsl-mc device */
 #define VFIO_DEVICE_FLAGS_CAPS	(1 << 7)	/* Info supports caps */
+#define VFIO_DEVICE_FLAGS_MSI	(1 << 8)	/* Device supports msi */
 	__u32	num_regions;	/* Max region index + 1 */
 	__u32	num_irqs;	/* Max IRQ index + 1 */
 	__u32   cap_offset;	/* Offset within info struct of first cap */
Signature · 1 line
2.17.1
#2 lore raw

Re: [RFC, v0 1/3] vfio/platform: add support for msi

include/uapi/linux/vfio.h struct vfio_device_info { · line 209

Quoted 12 lines · On Thu, 5 Nov 2020 11:32:55 +0530 Vikas Gupta <[email protected]> wrote:
include/uapi/linux/vfio.h +1 -0
index 2f313a238a8f..aab051e8338d 100644
@@ -203,6 +203,7 @@ struct vfio_device_info {
 #define VFIO_DEVICE_FLAGS_AP	(1 << 5)	/* vfio-ap device */
 #define VFIO_DEVICE_FLAGS_FSL_MC (1 << 6)	/* vfio-fsl-mc device */
 #define VFIO_DEVICE_FLAGS_CAPS	(1 << 7)	/* Info supports caps */
+#define VFIO_DEVICE_FLAGS_MSI	(1 << 8)	/* Device supports msi */
 	__u32	num_regions;	/* Max region index + 1 */
 	__u32	num_irqs;	/* Max IRQ index + 1 */
 	__u32   cap_offset;	/* Offset within info struct of first cap */

This doesn't make any sense to me, MSIs are just edge triggered interrupts to userspace, so why isn't this fully described via VFIO_DEVICE_GET_IRQ_INFO? If we do need something new to describe it, this seems incomplete, which indexes are MSI (IRQ_INFO can describe that)? We also already support MSI with vfio-pci, so a global flag for the device advertising this still seems wrong. Thanks,

Alex

#3 lore raw

Re: [RFC, v0 1/3] vfio/platform: add support for msi

Hi Alex,

Quoted 24 lines · Alex Williamson

include/uapi/linux/vfio.h struct vfio_device_info { · line 209

Quoted 12 lines · On Thu, 5 Nov 2020 11:32:55 +0530 Vikas Gupta <[email protected]> wrote:
include/uapi/linux/vfio.h +1 -0
index 2f313a238a8f..aab051e8338d 100644
@@ -203,6 +203,7 @@ struct vfio_device_info {
 #define VFIO_DEVICE_FLAGS_AP (1 << 5)        /* vfio-ap device */
 #define VFIO_DEVICE_FLAGS_FSL_MC (1 << 6)    /* vfio-fsl-mc device */
 #define VFIO_DEVICE_FLAGS_CAPS       (1 << 7)        /* Info supports caps */
+#define VFIO_DEVICE_FLAGS_MSI        (1 << 8)        /* Device supports msi */
      __u32   num_regions;    /* Max region index + 1 */
      __u32   num_irqs;       /* Max IRQ index + 1 */
      __u32   cap_offset;     /* Offset within info struct of first cap */

This doesn't make any sense to me, MSIs are just edge triggered interrupts to userspace, so why isn't this fully described via VFIO_DEVICE_GET_IRQ_INFO? If we do need something new to describe it, this seems incomplete, which indexes are MSI (IRQ_INFO can describe that)? We also already support MSI with vfio-pci, so a global flag for the device advertising this still seems wrong. Thanks,

Quoted 2 lines

Alex

Since VFIO platform uses indexes for IRQ numbers so I think MSI(s) cannot be described using indexes.
In the patch set there is no difference between MSI and normal interrupt for VFIO_DEVICE_GET_IRQ_INFO.
The patch set adds MSI(s), say as an extension, to the normal interrupts and handled accordingly. Do you see this is a violation? If yes, then we`ll think of other possible ways to support MSI for the platform devices.
Macro VFIO_DEVICE_FLAGS_MSI can be changed to any other name if it collides with an already supported vfio-pci or if not necessary, we can remove this flag.

Thanks,
Vikas

#4 lore raw

Re: [RFC, v0 1/3] vfio/platform: add support for msi

Quoted 28 lines · On Fri, 6 Nov 2020 08:24:26 +0530 Vikas Gupta <[email protected]> wrote:

Hi Alex,

Quoted 24 lines · Alex Williamson

include/uapi/linux/vfio.h struct vfio_device_info { · line 209

Quoted 12 lines · On Thu, 5 Nov 2020 11:32:55 +0530 Vikas Gupta <[email protected]> wrote:
include/uapi/linux/vfio.h +1 -0
index 2f313a238a8f..aab051e8338d 100644
@@ -203,6 +203,7 @@ struct vfio_device_info {
 #define VFIO_DEVICE_FLAGS_AP (1 << 5)        /* vfio-ap device */
 #define VFIO_DEVICE_FLAGS_FSL_MC (1 << 6)    /* vfio-fsl-mc device */
 #define VFIO_DEVICE_FLAGS_CAPS       (1 << 7)        /* Info supports caps */
+#define VFIO_DEVICE_FLAGS_MSI        (1 << 8)        /* Device supports msi */
      __u32   num_regions;    /* Max region index + 1 */
      __u32   num_irqs;       /* Max IRQ index + 1 */
      __u32   cap_offset;     /* Offset within info struct of first cap */  

This doesn't make any sense to me, MSIs are just edge triggered interrupts to userspace, so why isn't this fully described via VFIO_DEVICE_GET_IRQ_INFO? If we do need something new to describe it, this seems incomplete, which indexes are MSI (IRQ_INFO can describe that)? We also already support MSI with vfio-pci, so a global flag for the device advertising this still seems wrong. Thanks,

Quoted 4 lines
Quoted 2 lines

Alex

Since VFIO platform uses indexes for IRQ numbers so I think MSI(s) cannot be described using indexes.

That would be news for vfio-pci which has been describing MSIs with sub-indexes within indexes since vfio started.

Quoted 2 lines

In the patch set there is no difference between MSI and normal interrupt for VFIO_DEVICE_GET_IRQ_INFO.

Then what exactly is a global device flag indicating? Does it indicate all IRQs are MSI?

Quoted 2 lines

The patch set adds MSI(s), say as an extension, to the normal interrupts and handled accordingly.

So we have both "normal" IRQs and MSIs? How does the user know which indexes are which?

Quoted 1 line

Do you see this is a violation? If

Seems pretty unclear and dubious use of a global device flag.

Quoted 5 lines

yes, then we`ll think of other possible ways to support MSI for the platform devices.
Macro VFIO_DEVICE_FLAGS_MSI can be changed to any other name if it collides with an already supported vfio-pci or if not necessary, we can remove this flag.

If nothing else you're using a global flag to describe a platform device specific augmentation. We've recently added capabilities on the device info return that would be more appropriate for this, but fundamentally I don't understand why the irq info isn't sufficient. Thanks,

Alex

#5 lore raw

Re: [RFC, v0 1/3] vfio/platform: add support for msi

Hi Alex,

Quoted 37 lines · Alex Williamson
Quoted 28 lines · On Fri, 6 Nov 2020 08:24:26 +0530 Vikas Gupta <[email protected]> wrote:

Hi Alex,

Quoted 24 lines · Alex Williamson

include/uapi/linux/vfio.h struct vfio_device_info { · line 209

Quoted 12 lines · On Thu, 5 Nov 2020 11:32:55 +0530 Vikas Gupta <[email protected]> wrote:
include/uapi/linux/vfio.h +1 -0
index 2f313a238a8f..aab051e8338d 100644
@@ -203,6 +203,7 @@ struct vfio_device_info {
 #define VFIO_DEVICE_FLAGS_AP (1 << 5)        /* vfio-ap device */
 #define VFIO_DEVICE_FLAGS_FSL_MC (1 << 6)    /* vfio-fsl-mc device */
 #define VFIO_DEVICE_FLAGS_CAPS       (1 << 7)        /* Info supports caps */
+#define VFIO_DEVICE_FLAGS_MSI        (1 << 8)        /* Device supports msi */
      __u32   num_regions;    /* Max region index + 1 */
      __u32   num_irqs;       /* Max IRQ index + 1 */
      __u32   cap_offset;     /* Offset within info struct of first cap */

This doesn't make any sense to me, MSIs are just edge triggered interrupts to userspace, so why isn't this fully described via VFIO_DEVICE_GET_IRQ_INFO? If we do need something new to describe it, this seems incomplete, which indexes are MSI (IRQ_INFO can describe that)? We also already support MSI with vfio-pci, so a global flag for the device advertising this still seems wrong. Thanks,

Quoted 4 lines
Quoted 2 lines

Alex

Since VFIO platform uses indexes for IRQ numbers so I think MSI(s) cannot be described using indexes.

Quoted 8 lines

That would be news for vfio-pci which has been describing MSIs with sub-indexes within indexes since vfio started.

Quoted 2 lines

In the patch set there is no difference between MSI and normal interrupt for VFIO_DEVICE_GET_IRQ_INFO.

Then what exactly is a global device flag indicating? Does it indicate all IRQs are MSI?

No, it's not indicating that all are MSI.
The rationale behind adding the flag to tell user-space that platform device supports MSI as well. As you mentioned recently added capabilities can help on this, I`ll go through that.

Quoted 6 lines
Quoted 3 lines

The patch set adds MSI(s), say as an extension, to the normal interrupts and handled accordingly.

So we have both "normal" IRQs and MSIs? How does the user know which indexes are which?

With this patch set, I think this is missing and user space cannot know that particular index is MSI interrupt.
For platform devices there is no such mechanism, like index and sub-indexes to differentiate between legacy, MSI or MSIX as it’s there in PCI.
I believe for a particular IRQ index if the flag VFIO_IRQ_INFO_NORESIZE is used then user space can know which IRQ index has MSI(s). Does it make sense?
Suggestions on this would be helpful.

Thanks,
Vikas

Quoted 19 lines
Quoted 2 lines

Do you see this is a violation? If

Seems pretty unclear and dubious use of a global device flag.

Quoted 5 lines

yes, then we`ll think of other possible ways to support MSI for the platform devices.
Macro VFIO_DEVICE_FLAGS_MSI can be changed to any other name if it collides with an already supported vfio-pci or if not necessary, we can remove this flag.

If nothing else you're using a global flag to describe a platform device specific augmentation. We've recently added capabilities on the device info return that would be more appropriate for this, but fundamentally I don't understand why the irq info isn't sufficient. Thanks,

Alex

maintainer
#6 lore raw

Re: [RFC, v0 1/3] vfio/platform: add support for msi

Hi Vikas,

Quoted 59 lines · Vikas Gupta

Hi Alex,

Quoted 37 lines · Alex Williamson
Quoted 28 lines · On Fri, 6 Nov 2020 08:24:26 +0530 Vikas Gupta <[email protected]> wrote:

Hi Alex,

Quoted 24 lines · Alex Williamson

include/uapi/linux/vfio.h struct vfio_device_info { · line 209

Quoted 12 lines · On Thu, 5 Nov 2020 11:32:55 +0530 Vikas Gupta <[email protected]> wrote:
include/uapi/linux/vfio.h +1 -0
index 2f313a238a8f..aab051e8338d 100644
@@ -203,6 +203,7 @@ struct vfio_device_info {
 #define VFIO_DEVICE_FLAGS_AP (1 << 5)        /* vfio-ap device */
 #define VFIO_DEVICE_FLAGS_FSL_MC (1 << 6)    /* vfio-fsl-mc device */
 #define VFIO_DEVICE_FLAGS_CAPS       (1 << 7)        /* Info supports caps */
+#define VFIO_DEVICE_FLAGS_MSI        (1 << 8)        /* Device supports msi */
      __u32   num_regions;    /* Max region index + 1 */
      __u32   num_irqs;       /* Max IRQ index + 1 */
      __u32   cap_offset;     /* Offset within info struct of first cap */

This doesn't make any sense to me, MSIs are just edge triggered interrupts to userspace, so why isn't this fully described via VFIO_DEVICE_GET_IRQ_INFO? If we do need something new to describe it, this seems incomplete, which indexes are MSI (IRQ_INFO can describe that)? We also already support MSI with vfio-pci, so a global flag for the device advertising this still seems wrong. Thanks,

Quoted 4 lines
Quoted 2 lines

Alex

Since VFIO platform uses indexes for IRQ numbers so I think MSI(s) cannot be described using indexes.

Quoted 8 lines

That would be news for vfio-pci which has been describing MSIs with sub-indexes within indexes since vfio started.

Quoted 2 lines

In the patch set there is no difference between MSI and normal interrupt for VFIO_DEVICE_GET_IRQ_INFO.

Then what exactly is a global device flag indicating? Does it indicate all IRQs are MSI?

No, it's not indicating that all are MSI.
The rationale behind adding the flag to tell user-space that platform device supports MSI as well. As you mentioned recently added capabilities can help on this, I`ll go through that.

Quoted 4 lines
Quoted 3 lines

The patch set adds MSI(s), say as an extension, to the normal interrupts and handled accordingly.

Quoted 8 lines
Quoted 2 lines

So we have both "normal" IRQs and MSIs? How does the user know which indexes are which?

With this patch set, I think this is missing and user space cannot know that particular index is MSI interrupt.
For platform devices there is no such mechanism, like index and sub-indexes to differentiate between legacy, MSI or MSIX as it’s there in PCI.

Wht can't you use the count field (as per vfio_pci_get_irq_count())?

Quoted 3 lines

I believe for a particular IRQ index if the flag VFIO_IRQ_INFO_NORESIZE is used then user space can know which IRQ index has MSI(s). Does it make sense?

I don't think it is the same semantics.

Thanks

Eric

Quoted 23 lines

Suggestions on this would be helpful.

Thanks,
Vikas

Quoted 19 lines
Quoted 2 lines

Do you see this is a violation? If

Seems pretty unclear and dubious use of a global device flag.

Quoted 5 lines

yes, then we`ll think of other possible ways to support MSI for the platform devices.
Macro VFIO_DEVICE_FLAGS_MSI can be changed to any other name if it collides with an already supported vfio-pci or if not necessary, we can remove this flag.

If nothing else you're using a global flag to describe a platform device specific augmentation. We've recently added capabilities on the device info return that would be more appropriate for this, but fundamentally I don't understand why the irq info isn't sufficient. Thanks,

Alex

#7 lore raw

Re: [RFC, v0 1/3] vfio/platform: add support for msi

In reply to Vikas Gupta
Quoted 47 lines · On Mon, 9 Nov 2020 12:11:15 +0530 Vikas Gupta <[email protected]> wrote:

Hi Alex,

Quoted 43 lines · Alex Williamson
Quoted 28 lines · On Fri, 6 Nov 2020 08:24:26 +0530 Vikas Gupta <[email protected]> wrote:

Hi Alex,

Quoted 24 lines · Alex Williamson

include/uapi/linux/vfio.h struct vfio_device_info { · line 209

Quoted 12 lines · On Thu, 5 Nov 2020 11:32:55 +0530 Vikas Gupta <[email protected]> wrote:
include/uapi/linux/vfio.h +1 -0
index 2f313a238a8f..aab051e8338d 100644
@@ -203,6 +203,7 @@ struct vfio_device_info {
 #define VFIO_DEVICE_FLAGS_AP (1 << 5)        /* vfio-ap device */
 #define VFIO_DEVICE_FLAGS_FSL_MC (1 << 6)    /* vfio-fsl-mc device */
 #define VFIO_DEVICE_FLAGS_CAPS       (1 << 7)        /* Info supports caps */
+#define VFIO_DEVICE_FLAGS_MSI        (1 << 8)        /* Device supports msi */
      __u32   num_regions;    /* Max region index + 1 */
      __u32   num_irqs;       /* Max IRQ index + 1 */
      __u32   cap_offset;     /* Offset within info struct of first cap */  

This doesn't make any sense to me, MSIs are just edge triggered interrupts to userspace, so why isn't this fully described via VFIO_DEVICE_GET_IRQ_INFO? If we do need something new to describe it, this seems incomplete, which indexes are MSI (IRQ_INFO can describe that)? We also already support MSI with vfio-pci, so a global flag for the device advertising this still seems wrong. Thanks,

Quoted 4 lines
Quoted 2 lines

Alex

Since VFIO platform uses indexes for IRQ numbers so I think MSI(s) cannot be described using indexes.

That would be news for vfio-pci which has been describing MSIs with sub-indexes within indexes since vfio started.

Quoted 2 lines

In the patch set there is no difference between MSI and normal interrupt for VFIO_DEVICE_GET_IRQ_INFO.

Quoted 7 lines
Quoted 2 lines

Then what exactly is a global device flag indicating? Does it indicate all IRQs are MSI?

No, it's not indicating that all are MSI.
The rationale behind adding the flag to tell user-space that platform device supports MSI as well. As you mentioned recently added capabilities can help on this, I`ll go through that.

It still seems questionable to me to use a device info capability to describe an interrupt index specific feature. The scope seems wrong. Why does userspace need to know that this IRQ is MSI rather than indicating it's simply an edge triggered interrupt? That can be done using only vfio_irq_info.flags.

Quoted 11 lines
Quoted 5 lines
Quoted 2 lines

The patch set adds MSI(s), say as an extension, to the normal interrupts and handled accordingly.

So we have both "normal" IRQs and MSIs? How does the user know which indexes are which?

With this patch set, I think this is missing and user space cannot know that particular index is MSI interrupt.
For platform devices there is no such mechanism, like index and sub-indexes to differentiate between legacy, MSI or MSIX as it’s there in PCI.

Indexes and sub-indexes are a grouping mechanism of vfio to describe related interrupts. That terminology doesn't exist on PCI either, it's meant to be used generically. It's left to the vfio bus driver how userspace associates a given index to a device feature.

Quoted 3 lines

I believe for a particular IRQ index if the flag VFIO_IRQ_INFO_NORESIZE is used then user space can know which IRQ index has MSI(s). Does it make sense?

No, no-resize is an implementation detail, not an indication of the interrupt mechanism. It's still not clear to me why it's important to expose to userspace that a given interrupt is MSI versus simply exposing it as an edge interrupt (ie. automasked = false). If it is necessary, the most direct approach might be to expose a capability extension in the vfio_irq_info structure to describe it. Even then though, I don't think simply exposing a index as MSI is very meaningful. What is userspace intended to do differently based on this information? Thanks,

Alex

#8 lore raw

Re: [RFC, v0 1/3] vfio/platform: add support for msi

Hi Alex,

Quoted 60 lines · Alex Williamson
Quoted 47 lines · On Mon, 9 Nov 2020 12:11:15 +0530 Vikas Gupta <[email protected]> wrote:

Hi Alex,

Quoted 43 lines · Alex Williamson
Quoted 28 lines · On Fri, 6 Nov 2020 08:24:26 +0530 Vikas Gupta <[email protected]> wrote:

Hi Alex,

Quoted 24 lines · Alex Williamson

include/uapi/linux/vfio.h struct vfio_device_info { · line 209

Quoted 12 lines · On Thu, 5 Nov 2020 11:32:55 +0530 Vikas Gupta <[email protected]> wrote:
include/uapi/linux/vfio.h +1 -0
index 2f313a238a8f..aab051e8338d 100644
@@ -203,6 +203,7 @@ struct vfio_device_info {
 #define VFIO_DEVICE_FLAGS_AP (1 << 5)        /* vfio-ap device */
 #define VFIO_DEVICE_FLAGS_FSL_MC (1 << 6)    /* vfio-fsl-mc device */
 #define VFIO_DEVICE_FLAGS_CAPS       (1 << 7)        /* Info supports caps */
+#define VFIO_DEVICE_FLAGS_MSI        (1 << 8)        /* Device supports msi */
      __u32   num_regions;    /* Max region index + 1 */
      __u32   num_irqs;       /* Max IRQ index + 1 */
      __u32   cap_offset;     /* Offset within info struct of first cap */

This doesn't make any sense to me, MSIs are just edge triggered interrupts to userspace, so why isn't this fully described via VFIO_DEVICE_GET_IRQ_INFO? If we do need something new to describe it, this seems incomplete, which indexes are MSI (IRQ_INFO can describe that)? We also already support MSI with vfio-pci, so a global flag for the device advertising this still seems wrong. Thanks,

Quoted 4 lines
Quoted 2 lines

Alex

Since VFIO platform uses indexes for IRQ numbers so I think MSI(s) cannot be described using indexes.

That would be news for vfio-pci which has been describing MSIs with sub-indexes within indexes since vfio started.

Quoted 2 lines

In the patch set there is no difference between MSI and normal interrupt for VFIO_DEVICE_GET_IRQ_INFO.

Quoted 7 lines
Quoted 2 lines

Then what exactly is a global device flag indicating? Does it indicate all IRQs are MSI?

No, it's not indicating that all are MSI.
The rationale behind adding the flag to tell user-space that platform device supports MSI as well. As you mentioned recently added capabilities can help on this, I`ll go through that.

Quoted 5 lines

It still seems questionable to me to use a device info capability to describe an interrupt index specific feature. The scope seems wrong. Why does userspace need to know that this IRQ is MSI rather than indicating it's simply an edge triggered interrupt? That can be done using only vfio_irq_info.flags.

Ok. In the next patch set I`ll remove the device flag (VFIO_DEVICE_FLAGS_MSI) as vfio_irq_info.flags should have enough information for edge triggered interrupt.

Quoted 24 lines
Quoted 5 lines
Quoted 5 lines
Quoted 4 lines

The patch set adds MSI(s), say as an extension, to the normal interrupts and handled accordingly.

Quoted 8 lines
Quoted 2 lines

So we have both "normal" IRQs and MSIs? How does the user know which indexes are which?

With this patch set, I think this is missing and user space cannot know that particular index is MSI interrupt.
For platform devices there is no such mechanism, like index and sub-indexes to differentiate between legacy, MSI or MSIX as it’s there in PCI.

Indexes and sub-indexes are a grouping mechanism of vfio to describe related interrupts. That terminology doesn't exist on PCI either, it's meant to be used generically. It's left to the vfio bus driver how userspace associates a given index to a device feature.

Quoted 3 lines

I believe for a particular IRQ index if the flag VFIO_IRQ_INFO_NORESIZE is used then user space can know which IRQ index has MSI(s). Does it make sense?

Quoted 9 lines

No, no-resize is an implementation detail, not an indication of the interrupt mechanism. It's still not clear to me why it's important to expose to userspace that a given interrupt is MSI versus simply exposing it as an edge interrupt (ie. automasked = false). If it is necessary, the most direct approach might be to expose a capability extension in the vfio_irq_info structure to describe it. Even then though, I don't think simply exposing a index as MSI is very meaningful. What is userspace intended to do differently based on this information? Thanks,

The current patch set is not setting VFIO_IRQ_INFO_AUTOMASKED (automasked=false) for MSIs so I believe this much is information enough for user space to know that this is an edge triggered interrupt.
I agree that exposing an index as MSI is not meaningful as user space
has nothing special to do with this information.

Quoted 3 lines

Alex

maintainer
#9 lore raw

Re: [RFC, v0 1/3] vfio/platform: add support for msi

In reply to Vikas Gupta

Hi Vikas,

Quoted 28 lines · Vikas Gupta

Hi Alex,

Quoted 24 lines · Alex Williamson

include/uapi/linux/vfio.h struct vfio_device_info { · line 209

Quoted 12 lines · On Thu, 5 Nov 2020 11:32:55 +0530 Vikas Gupta <[email protected]> wrote:
include/uapi/linux/vfio.h +1 -0
index 2f313a238a8f..aab051e8338d 100644
@@ -203,6 +203,7 @@ struct vfio_device_info {
 #define VFIO_DEVICE_FLAGS_AP (1 << 5)        /* vfio-ap device */
 #define VFIO_DEVICE_FLAGS_FSL_MC (1 << 6)    /* vfio-fsl-mc device */
 #define VFIO_DEVICE_FLAGS_CAPS       (1 << 7)        /* Info supports caps */
+#define VFIO_DEVICE_FLAGS_MSI        (1 << 8)        /* Device supports msi */
      __u32   num_regions;    /* Max region index + 1 */
      __u32   num_irqs;       /* Max IRQ index + 1 */
      __u32   cap_offset;     /* Offset within info struct of first cap */

This doesn't make any sense to me, MSIs are just edge triggered interrupts to userspace, so why isn't this fully described via VFIO_DEVICE_GET_IRQ_INFO? If we do need something new to describe it, this seems incomplete, which indexes are MSI (IRQ_INFO can describe that)? We also already support MSI with vfio-pci, so a global flag for the device advertising this still seems wrong. Thanks,

Quoted 6 lines
Quoted 2 lines

Alex

Since VFIO platform uses indexes for IRQ numbers so I think MSI(s) cannot be described using indexes.
In the patch set there is no difference between MSI and normal interrupt for VFIO_DEVICE_GET_IRQ_INFO.

in vfio_platform_irq_init() we first iterate on normal interrupts using get_irq(). Can't we add an MSI index at the end of this list with vdev->irqs[i].count > 1 and set vdev->num_irqs accordingly?

Thanks

Eric

Quoted 11 lines

The patch set adds MSI(s), say as an extension, to the normal interrupts and handled accordingly. Do you see this is a violation? If yes, then we`ll think of other possible ways to support MSI for the platform devices.
Macro VFIO_DEVICE_FLAGS_MSI can be changed to any other name if it collides with an already supported vfio-pci or if not necessary, we can remove this flag.

Thanks,
Vikas

#10 lore raw

Re: [RFC, v0 1/3] vfio/platform: add support for msi

Hi Eric,

Quoted 34 lines · Auger Eric

Hi Vikas,

Quoted 30 lines · Vikas Gupta

Hi Alex,

Quoted 26 lines · Alex Williamson

include/uapi/linux/vfio.h struct vfio_device_info { · line 209

Quoted 12 lines · On Thu, 5 Nov 2020 11:32:55 +0530 Vikas Gupta <[email protected]> wrote:
include/uapi/linux/vfio.h +1 -0
index 2f313a238a8f..aab051e8338d 100644
@@ -203,6 +203,7 @@ struct vfio_device_info {
 #define VFIO_DEVICE_FLAGS_AP (1 << 5)        /* vfio-ap device */
 #define VFIO_DEVICE_FLAGS_FSL_MC (1 << 6)    /* vfio-fsl-mc device */
 #define VFIO_DEVICE_FLAGS_CAPS       (1 << 7)        /* Info supports caps */
+#define VFIO_DEVICE_FLAGS_MSI        (1 << 8)        /* Device supports msi */
      __u32   num_regions;    /* Max region index + 1 */
      __u32   num_irqs;       /* Max IRQ index + 1 */
      __u32   cap_offset;     /* Offset within info struct of first cap */

This doesn't make any sense to me, MSIs are just edge triggered interrupts to userspace, so why isn't this fully described via VFIO_DEVICE_GET_IRQ_INFO? If we do need something new to describe it, this seems incomplete, which indexes are MSI (IRQ_INFO can describe that)? We also already support MSI with vfio-pci, so a global flag for the device advertising this still seems wrong. Thanks,

Alex

Quoted 7 lines
Quoted 4 lines

Since VFIO platform uses indexes for IRQ numbers so I think MSI(s) cannot be described using indexes.
In the patch set there is no difference between MSI and normal interrupt for VFIO_DEVICE_GET_IRQ_INFO.

in vfio_platform_irq_init() we first iterate on normal interrupts using get_irq(). Can't we add an MSI index at the end of this list with vdev->irqs[i].count > 1 and set vdev->num_irqs accordingly?

Yes, I think MSI can be added to the end of list with setting vdev->irqs[i].count > 1.
I`ll consider changing in the next patch set. Thanks,
Vikas

Quoted 16 lines

Thanks

Eric

Quoted 11 lines

The patch set adds MSI(s), say as an extension, to the normal interrupts and handled accordingly. Do you see this is a violation? If yes, then we`ll think of other possible ways to support MSI for the platform devices.
Macro VFIO_DEVICE_FLAGS_MSI can be changed to any other name if it collides with an already supported vfio-pci or if not necessary, we can remove this flag.

Thanks,
Vikas

#11 lore raw

RFC 2/3 vfio/platform: change cleanup order

In reply to Vikas Gupta

In the case of msi, vendor specific msi module may require region access to handle msi cleanup so we need to cleanup region after irq cleanup only.

---
 drivers/vfio/platform/vfio_platform_common.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/vfio/platform/vfio_platform_common.c +1 -1
index c713f4e4c552..b9d68ccdf92f 100644
@@ -283,8 +283,8 @@ static void vfio_platform_release(void *device_data)
 			WARN_ON(1);
 		}
 		pm_runtime_put(vdev->device);
-		vfio_platform_regions_cleanup(vdev);
 		vfio_platform_irq_cleanup(vdev);
+		vfio_platform_regions_cleanup(vdev);
 	}
 
 	mutex_unlock(&driver_lock);
Signature · 1 line
2.17.1
#12 lore raw

RFC 3/3 vfio/platform: add Broadcom msi module

In reply to Vikas Gupta

Add Broadcom msi module for platform devices.

---
 drivers/vfio/platform/Kconfig                 |  1 +
 drivers/vfio/platform/Makefile                |  1 +
 drivers/vfio/platform/msi/Kconfig             |  9 +++
 drivers/vfio/platform/msi/Makefile            |  2 +
 .../vfio/platform/msi/vfio_platform_bcmplt.c  | 74 +++++++++++++++++++
 5 files changed, 87 insertions(+)
 create mode 100644 drivers/vfio/platform/msi/Kconfig
 create mode 100644 drivers/vfio/platform/msi/Makefile
 create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
drivers/vfio/platform/Kconfig +1 -0
index dc1a3c44f2c6..7b8696febe61 100644
@@ -21,3 +21,4 @@ config VFIO_AMBA
 	  If you don't know what to do here, say N.
 
 source "drivers/vfio/platform/reset/Kconfig"
+source "drivers/vfio/platform/msi/Kconfig"
drivers/vfio/platform/Makefile +1 -0
index 3f3a24e7c4ef..9ccdcdbf0e7e 100644
@@ -5,6 +5,7 @@ vfio-platform-y := vfio_platform.o
 obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform.o
 obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform-base.o
 obj-$(CONFIG_VFIO_PLATFORM) += reset/
+obj-$(CONFIG_VFIO_PLATFORM) += msi/
 
 vfio-amba-y := vfio_amba.o
 
drivers/vfio/platform/msi/Kconfig new file +9 -0
new file mode 100644
index 000000000000..54d6b70e1e32
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0-only
+config VFIO_PLATFORM_BCMPLT_MSI
+	tristate "MSI support for Broadcom platform devices"
+	depends on VFIO_PLATFORM && (ARCH_BCM_IPROC || COMPILE_TEST)
+	default ARCH_BCM_IPROC
+	help
+	  Enables the VFIO platform driver to handle msi for Broadcom devices
+
+	  If you don't know what to do here, say N.
drivers/vfio/platform/msi/Makefile new file +2 -0
new file mode 100644
index 000000000000..27422d45cecb
@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_VFIO_PLATFORM_BCMPLT_MSI) += vfio_platform_bcmplt.o
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +74 -0

74 lines not on this page · raw message

Signature · 1 line
2.17.1
#13 lore raw

cover msi support for platform devices

In reply to Vikas Gupta

This RFC adds support for MSI for platform devices.
a) MSI(s) is/are added in addition to the normal interrupts.
b) The vendor specific MSI configuration can be done using callbacks which is implemented as msi module.
c) Adds a msi handling module for the Broadcom platform devices.

Changes from:
-------------
v0 to v1:
i) Removed MSI device flag VFIO_DEVICE_FLAGS_MSI. ii) Add MSI(s) at the end of the irq list of platform IRQs. MSI(s) with first entry of MSI block has count and flag information.
IRQ list: Allocation for IRQs + MSIs are allocated as below
Example: if there are 'n' IRQs and 'k' MSIs
-------------------------------------------------------
|IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
-------------------------------------------------------
MSI-0 will have count=k set and flags set accordingly.

Vikas Gupta (3):
vfio/platform: add support for msi
vfio/platform: change cleanup order
vfio/platform: add Broadcom msi module

 drivers/vfio/platform/Kconfig                 |   1 +
 drivers/vfio/platform/Makefile                |   1 +
 drivers/vfio/platform/msi/Kconfig             |   9 +
 drivers/vfio/platform/msi/Makefile            |   2 +
 .../vfio/platform/msi/vfio_platform_bcmplt.c  |  74 ++++++
 drivers/vfio/platform/vfio_platform_common.c  |  86 ++++++-
 drivers/vfio/platform/vfio_platform_irq.c     | 238 +++++++++++++++++-
 drivers/vfio/platform/vfio_platform_private.h |  23 ++
 8 files changed, 419 insertions(+), 15 deletions(-)
 create mode 100644 drivers/vfio/platform/msi/Kconfig
 create mode 100644 drivers/vfio/platform/msi/Makefile
 create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
Signature · 1 line
2.17.1
#14 lore raw

RFC 1/3 vfio/platform: add support for msi

MSI support for platform devices.

---
 drivers/vfio/platform/vfio_platform_common.c  |  84 ++++++-
 drivers/vfio/platform/vfio_platform_irq.c     | 238 +++++++++++++++++-
 drivers/vfio/platform/vfio_platform_private.h |  23 ++
 3 files changed, 331 insertions(+), 14 deletions(-)
drivers/vfio/platform/vfio_platform_common.c +82 -2
index c0771a9567fb..226aa8083751 100644
@@ -26,6 +26,10 @@
 #define VFIO_PLATFORM_IS_ACPI(vdev) ((vdev)->acpihid != NULL)
 
 static LIST_HEAD(reset_list);
+
+/* devices having MSI support */
+static LIST_HEAD(msi_list);
+
 static DEFINE_MUTEX(driver_lock);
 
 static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,
@@ -47,6 +51,26 @@ static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,
 	return reset_fn;
 }
 
+static bool vfio_platform_lookup_msi(struct vfio_platform_device *vdev)
+{
+	bool has_msi = false;
+	struct vfio_platform_msi_node *iter;
+
+	mutex_lock(&driver_lock);
+	list_for_each_entry(iter, &msi_list, link) {
+		if (!strcmp(iter->compat, vdev->compat) &&
+		    try_module_get(iter->owner)) {
+			vdev->msi_module = iter->owner;
+			vdev->of_get_msi = iter->of_get_msi;
+			vdev->of_msi_write = iter->of_msi_write;
+			has_msi = true;
+			break;
+		}
+	}
+	mutex_unlock(&driver_lock);
+	return has_msi;
+}
+
 static int vfio_platform_acpi_probe(struct vfio_platform_device *vdev,
 				    struct device *dev)
 {
@@ -126,6 +150,19 @@ static int vfio_platform_get_reset(struct vfio_platform_device *vdev)
 	return vdev->of_reset ? 0 : -ENOENT;
 }
 
+static int vfio_platform_get_msi(struct vfio_platform_device *vdev)
+{
+	bool has_msi;
+
+	has_msi = vfio_platform_lookup_msi(vdev);
+	if (!has_msi) {
+		request_module("vfio-msi:%s", vdev->compat);
+		has_msi = vfio_platform_lookup_msi(vdev);
+	}
+
+	return has_msi ? 0 : -ENOENT;
+}
+
 static void vfio_platform_put_reset(struct vfio_platform_device *vdev)
 {
 	if (VFIO_PLATFORM_IS_ACPI(vdev))
@@ -135,6 +172,12 @@ static void vfio_platform_put_reset(struct vfio_platform_device *vdev)
 		module_put(vdev->reset_module);
 }
 
+static void vfio_platform_put_msi(struct vfio_platform_device *vdev)
+{
+	if (vdev->of_get_msi)
+		module_put(vdev->msi_module);
+}
+
 static int vfio_platform_regions_init(struct vfio_platform_device *vdev)
 {
 	int cnt = 0, i;
@@ -313,6 +356,7 @@ static long vfio_platform_ioctl(void *device_data,
 
 		if (vfio_platform_has_reset(vdev))
 			vdev->flags |= VFIO_DEVICE_FLAGS_RESET;
+
 		info.flags = vdev->flags;
 		info.num_regions = vdev->num_regions;
 		info.num_irqs = vdev->num_irqs;
@@ -366,6 +410,7 @@ static long vfio_platform_ioctl(void *device_data,
 		struct vfio_irq_set hdr;
 		u8 *data = NULL;
 		int ret = 0;
+		int max;
 		size_t data_size = 0;
 
 		minsz = offsetofend(struct vfio_irq_set, count);
@@ -373,7 +418,12 @@ static long vfio_platform_ioctl(void *device_data,
 		if (copy_from_user(&hdr, (void __user *)arg, minsz))
 			return -EFAULT;
 
-		ret = vfio_set_irqs_validate_and_prepare(&hdr, vdev->num_irqs,
+		if (hdr.index >= vdev->num_irqs)
+			return -EINVAL;
+
+		max = vdev->irqs[hdr.index].count;
+
+		ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
 						 vdev->num_irqs, &data_size);
 		if (ret)
 			return ret;
@@ -679,11 +729,15 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,
 		return ret;
 	}
 
+	ret = vfio_platform_get_msi(vdev);
+	if (ret)
+		dev_info(vdev->device, "msi not supported\n");
+
 	group = vfio_iommu_group_get(dev);
 	if (!group) {
 		dev_err(dev, "No IOMMU group for device %s\n", vdev->name);
 		ret = -EINVAL;
-		goto put_reset;
+		goto put_msi;
 	}
 
 	ret = vfio_add_group_dev(dev, &vfio_platform_ops, vdev);
@@ -697,6 +751,8 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,
 
 put_iommu:
 	vfio_iommu_group_put(group, dev);
+put_msi:
+	vfio_platform_put_msi(vdev);
 put_reset:
 	vfio_platform_put_reset(vdev);
 	return ret;
@@ -745,6 +801,30 @@ void vfio_platform_unregister_reset(const char *compat,
 }
 EXPORT_SYMBOL_GPL(vfio_platform_unregister_reset);
 
+void __vfio_platform_register_msi(struct vfio_platform_msi_node *node)
+{
+	mutex_lock(&driver_lock);
+	list_add(&node->link, &msi_list);
+	mutex_unlock(&driver_lock);
+}
+EXPORT_SYMBOL_GPL(__vfio_platform_register_msi);
+
+void vfio_platform_unregister_msi(const char *compat)
+{
+	struct vfio_platform_msi_node *iter, *temp;
+
+	mutex_lock(&driver_lock);
+	list_for_each_entry_safe(iter, temp, &msi_list, link) {
+		if (!strcmp(iter->compat, compat)) {
+			list_del(&iter->link);
+			break;
+		}
+	}
+
+	mutex_unlock(&driver_lock);
+}
+EXPORT_SYMBOL_GPL(vfio_platform_unregister_msi);
+
 MODULE_VERSION(DRIVER_VERSION);
 MODULE_LICENSE("GPL v2");
 MODULE_AUTHOR(DRIVER_AUTHOR);
drivers/vfio/platform/vfio_platform_irq.c +226 -12

285 lines not on this page · raw message

drivers/vfio/platform/vfio_platform_private.h +23 -0
index 289089910643..2aea445e1071 100644
@@ -9,6 +9,7 @@
 
 #include <linux/types.h>
 #include <linux/interrupt.h>
+#include <linux/msi.h>
 
 #define VFIO_PLATFORM_OFFSET_SHIFT   40
 #define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
@@ -26,6 +27,7 @@ struct vfio_platform_irq {
 	char			*name;
 	struct eventfd_ctx	*trigger;
 	bool			masked;
+	bool			is_msi;
 	spinlock_t		lock;
 	struct virqfd		*unmask;
 	struct virqfd		*mask;
@@ -46,13 +48,16 @@ struct vfio_platform_device {
 	u32				num_regions;
 	struct vfio_platform_irq	*irqs;
 	u32				num_irqs;
+	u32				num_msis;
 	int				refcnt;
 	struct mutex			igate;
 	struct module			*parent_module;
 	const char			*compat;
 	const char			*acpihid;
 	struct module			*reset_module;
+	struct module			*msi_module;
 	struct device			*device;
+	int				config_msi;
 
 	/*
 	 * These fields should be filled by the bus specific binder
@@ -65,11 +70,19 @@ struct vfio_platform_device {
 		(*get_resource)(struct vfio_platform_device *vdev, int i);
 	int	(*get_irq)(struct vfio_platform_device *vdev, int i);
 	int	(*of_reset)(struct vfio_platform_device *vdev);
+	u32	(*of_get_msi)(struct vfio_platform_device *vdev);
+	int	(*of_msi_write)(struct vfio_platform_device *vdev,
+				struct msi_desc *desc,
+				struct msi_msg *msg);
 
 	bool				reset_required;
 };
 
 typedef int (*vfio_platform_reset_fn_t)(struct vfio_platform_device *vdev);
+typedef u32 (*vfio_platform_get_msi_fn_t)(struct vfio_platform_device *vdev);
+typedef int (*vfio_platform_msi_write_fn_t)(struct vfio_platform_device *vdev,
+					    struct msi_desc *desc,
+					    struct msi_msg *msg);
 
 struct vfio_platform_reset_node {
 	struct list_head link;
@@ -78,6 +91,14 @@ struct vfio_platform_reset_node {
 	vfio_platform_reset_fn_t of_reset;
 };
 
+struct vfio_platform_msi_node {
+	struct list_head link;
+	char *compat;
+	struct module *owner;
+	vfio_platform_get_msi_fn_t of_get_msi;
+	vfio_platform_msi_write_fn_t of_msi_write;
+};
+
 extern int vfio_platform_probe_common(struct vfio_platform_device *vdev,
 				      struct device *dev);
 extern struct vfio_platform_device *vfio_platform_remove_common
@@ -94,6 +115,8 @@ extern int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 extern void __vfio_platform_register_reset(struct vfio_platform_reset_node *n);
 extern void vfio_platform_unregister_reset(const char *compat,
 					   vfio_platform_reset_fn_t fn);
+void __vfio_platform_register_msi(struct vfio_platform_msi_node *n);
+void vfio_platform_unregister_msi(const char *compat);
 #define vfio_platform_register_reset(__compat, __reset)		\
 static struct vfio_platform_reset_node __reset ## _node = {	\
 	.owner = THIS_MODULE,					\
Signature · 1 line
2.17.1
#15 lore raw

RFC 2/3 vfio/platform: change cleanup order

In reply to Vikas Gupta

In the case of msi, vendor specific msi module may require region access to handle msi cleanup so we need to cleanup region after irq cleanup only.

---
 drivers/vfio/platform/vfio_platform_common.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/vfio/platform/vfio_platform_common.c +1 -1
index 226aa8083751..ca4a01c488b8 100644
@@ -278,8 +278,8 @@ static void vfio_platform_release(void *device_data)
 			WARN_ON(1);
 		}
 		pm_runtime_put(vdev->device);
-		vfio_platform_regions_cleanup(vdev);
 		vfio_platform_irq_cleanup(vdev);
+		vfio_platform_regions_cleanup(vdev);
 	}
 
 	mutex_unlock(&driver_lock);
Signature · 1 line
2.17.1
#16 lore raw

RFC 3/3 vfio/platform: add Broadcom msi module

In reply to Vikas Gupta

Add Broadcom msi module for platform devices.

---
 drivers/vfio/platform/Kconfig                 |  1 +
 drivers/vfio/platform/Makefile                |  1 +
 drivers/vfio/platform/msi/Kconfig             |  9 +++
 drivers/vfio/platform/msi/Makefile            |  2 +
 .../vfio/platform/msi/vfio_platform_bcmplt.c  | 74 +++++++++++++++++++
 5 files changed, 87 insertions(+)
 create mode 100644 drivers/vfio/platform/msi/Kconfig
 create mode 100644 drivers/vfio/platform/msi/Makefile
 create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
drivers/vfio/platform/Kconfig +1 -0
index dc1a3c44f2c6..7b8696febe61 100644
@@ -21,3 +21,4 @@ config VFIO_AMBA
 	  If you don't know what to do here, say N.
 
 source "drivers/vfio/platform/reset/Kconfig"
+source "drivers/vfio/platform/msi/Kconfig"
drivers/vfio/platform/Makefile +1 -0
index 3f3a24e7c4ef..9ccdcdbf0e7e 100644
@@ -5,6 +5,7 @@ vfio-platform-y := vfio_platform.o
 obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform.o
 obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform-base.o
 obj-$(CONFIG_VFIO_PLATFORM) += reset/
+obj-$(CONFIG_VFIO_PLATFORM) += msi/
 
 vfio-amba-y := vfio_amba.o
 
drivers/vfio/platform/msi/Kconfig new file +9 -0
new file mode 100644
index 000000000000..54d6b70e1e32
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0-only
+config VFIO_PLATFORM_BCMPLT_MSI
+	tristate "MSI support for Broadcom platform devices"
+	depends on VFIO_PLATFORM && (ARCH_BCM_IPROC || COMPILE_TEST)
+	default ARCH_BCM_IPROC
+	help
+	  Enables the VFIO platform driver to handle msi for Broadcom devices
+
+	  If you don't know what to do here, say N.
drivers/vfio/platform/msi/Makefile new file +2 -0
new file mode 100644
index 000000000000..27422d45cecb
@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_VFIO_PLATFORM_BCMPLT_MSI) += vfio_platform_bcmplt.o
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +74 -0

74 lines not on this page · raw message

Signature · 1 line
2.17.1
maintainer
#17 lore raw
In reply to Vikas Gupta

Hi Vikas,

Quoted 6 lines · Vikas Gupta

This RFC adds support for MSI for platform devices.
a) MSI(s) is/are added in addition to the normal interrupts.
b) The vendor specific MSI configuration can be done using callbacks which is implemented as msi module.
c) Adds a msi handling module for the Broadcom platform devices.

Quoted 12 lines

Changes from:
-------------
v0 to v1:
i) Removed MSI device flag VFIO_DEVICE_FLAGS_MSI. ii) Add MSI(s) at the end of the irq list of platform IRQs. MSI(s) with first entry of MSI block has count and flag information.
IRQ list: Allocation for IRQs + MSIs are allocated as below
Example: if there are 'n' IRQs and 'k' MSIs
-------------------------------------------------------
|IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
-------------------------------------------------------

I have not taken time yet to look at your series, but to me you should have |IRQ-0|IRQ-1|....|IRQ-n|MSI|MSIX
then for setting a given MSIX (i) you would select the MSIx index and then set start=i count=1.
to me individual MSIs are encoded in the subindex and not in the index. The index just selects the "type" of interrupt.

For PCI you just have:

        VFIO_PCI_INTX_IRQ_INDEX,
        VFIO_PCI_MSI_IRQ_INDEX, -> MSI index and then you play with

start/count
VFIO_PCI_MSIX_IRQ_INDEX,
VFIO_PCI_ERR_IRQ_INDEX,
VFIO_PCI_REQ_IRQ_INDEX,

(include/uapi/linux/vfio.h)

Thanks

Eric

Quoted 20 lines

MSI-0 will have count=k set and flags set accordingly.

Vikas Gupta (3):
vfio/platform: add support for msi
vfio/platform: change cleanup order
vfio/platform: add Broadcom msi module

 drivers/vfio/platform/Kconfig                 |   1 +
 drivers/vfio/platform/Makefile                |   1 +
 drivers/vfio/platform/msi/Kconfig             |   9 +
 drivers/vfio/platform/msi/Makefile            |   2 +
 .../vfio/platform/msi/vfio_platform_bcmplt.c  |  74 ++++++
 drivers/vfio/platform/vfio_platform_common.c  |  86 ++++++-
 drivers/vfio/platform/vfio_platform_irq.c     | 238 +++++++++++++++++-
 drivers/vfio/platform/vfio_platform_private.h |  23 ++
 8 files changed, 419 insertions(+), 15 deletions(-)
 create mode 100644 drivers/vfio/platform/msi/Kconfig
 create mode 100644 drivers/vfio/platform/msi/Makefile
 create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
#18 lore raw

Hi Eric,

Quoted 22 lines · Auger Eric

Hi Vikas,

Quoted 6 lines · Vikas Gupta

This RFC adds support for MSI for platform devices.
a) MSI(s) is/are added in addition to the normal interrupts.
b) The vendor specific MSI configuration can be done using callbacks which is implemented as msi module.
c) Adds a msi handling module for the Broadcom platform devices.

Quoted 12 lines

Changes from:
-------------
v0 to v1:
i) Removed MSI device flag VFIO_DEVICE_FLAGS_MSI. ii) Add MSI(s) at the end of the irq list of platform IRQs. MSI(s) with first entry of MSI block has count and flag information.
IRQ list: Allocation for IRQs + MSIs are allocated as below
Example: if there are 'n' IRQs and 'k' MSIs
-------------------------------------------------------
|IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
-------------------------------------------------------

Quoted 4 lines

I have not taken time yet to look at your series, but to me you should have |IRQ-0|IRQ-1|....|IRQ-n|MSI|MSIX
then for setting a given MSIX (i) you would select the MSIx index and then set start=i count=1.

As per your suggestion, we should have, if there are n-IRQs, k-MSIXs and m-MSIs, allocation of IRQs should be done as below

|IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX|
                                             |        |
                                             |
|MSIX0||MSIX1||MSXI2|....|MSIX-(k-1)|
                                             |MSI0||MSI1||MSI2|....|MSI-(m-1)|

With this implementation user space can know that, at indexes n and n+1, edge triggered interrupts are present.
We may add an element in vfio_platform_irq itself to allocate MSIs/MSIXs struct vfio_platform_irq{
.....
.....
struct vfio_platform_irq *block; => this points to the block
allocation for MSIs/MSIXs and all msi/msix are type of IRQs.
};
OR
Another structure can be defined in 'vfio_pci_private.h'

struct vfio_msi_ctx {
        struct eventfd_ctx      *trigger;
        char                    *name;
};

and
struct vfio_platform_irq {
.....
.....
struct vfio_msi_ctx *block; => this points to the block allocation
for MSIs/MSIXs
};
Which of the above two options sounds OK to you? Please suggest.

Quoted 4 lines

to me individual MSIs are encoded in the subindex and not in the index. The index just selects the "type" of interrupt.

For PCI you just have:

Quoted 8 lines
        VFIO_PCI_INTX_IRQ_INDEX,
        VFIO_PCI_MSI_IRQ_INDEX, -> MSI index and then you play with

start/count
VFIO_PCI_MSIX_IRQ_INDEX,
VFIO_PCI_ERR_IRQ_INDEX,
VFIO_PCI_REQ_IRQ_INDEX,

(include/uapi/linux/vfio.h)

In pci case, type of interrupts is fixed so they can be 'indexed' by these enums but for VFIO platform user space will need to iterate all (num_irqs) indexes to know at which indexes edge triggered interrupts are present.

Thanks,
Vikas

Quoted 25 lines

Thanks

Eric

Quoted 20 lines

MSI-0 will have count=k set and flags set accordingly.

Vikas Gupta (3):
vfio/platform: add support for msi
vfio/platform: change cleanup order
vfio/platform: add Broadcom msi module

 drivers/vfio/platform/Kconfig                 |   1 +
 drivers/vfio/platform/Makefile                |   1 +
 drivers/vfio/platform/msi/Kconfig             |   9 +
 drivers/vfio/platform/msi/Makefile            |   2 +
 .../vfio/platform/msi/vfio_platform_bcmplt.c  |  74 ++++++
 drivers/vfio/platform/vfio_platform_common.c  |  86 ++++++-
 drivers/vfio/platform/vfio_platform_irq.c     | 238 +++++++++++++++++-
 drivers/vfio/platform/vfio_platform_private.h |  23 ++
 8 files changed, 419 insertions(+), 15 deletions(-)
 create mode 100644 drivers/vfio/platform/msi/Kconfig
 create mode 100644 drivers/vfio/platform/msi/Makefile
 create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
maintainer
#19 lore raw

Hi Vikas,

Quoted 30 lines · Vikas Gupta

Hi Eric,

Quoted 22 lines · Auger Eric

Hi Vikas,

Quoted 6 lines · Vikas Gupta

This RFC adds support for MSI for platform devices.
a) MSI(s) is/are added in addition to the normal interrupts.
b) The vendor specific MSI configuration can be done using callbacks which is implemented as msi module.
c) Adds a msi handling module for the Broadcom platform devices.

Quoted 12 lines

Changes from:
-------------
v0 to v1:
i) Removed MSI device flag VFIO_DEVICE_FLAGS_MSI. ii) Add MSI(s) at the end of the irq list of platform IRQs. MSI(s) with first entry of MSI block has count and flag information.
IRQ list: Allocation for IRQs + MSIs are allocated as below
Example: if there are 'n' IRQs and 'k' MSIs
-------------------------------------------------------
|IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
-------------------------------------------------------

Quoted 4 lines

I have not taken time yet to look at your series, but to me you should have |IRQ-0|IRQ-1|....|IRQ-n|MSI|MSIX
then for setting a given MSIX (i) you would select the MSIx index and then set start=i count=1.

Quoted 8 lines

As per your suggestion, we should have, if there are n-IRQs, k-MSIXs and m-MSIs, allocation of IRQs should be done as below

|IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX|
                                             |        |
                                             |
|MSIX0||MSIX1||MSXI2|....|MSIX-(k-1)|
                                             |MSI0||MSI1||MSI2|....|MSI-(m-1)|

No I really meant this list of indices: IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX| and potentially later on IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX| ERR| REQ if ERR/REQ were to be added.

I think the userspace could query the total number of indices using VFIO_DEVICE_GET_INFO and retrieve num_irqs (corresponding to the n wire interrupts + MSI index + MSIX index)

Then userspace can loop on all the indices using VFIO_DEVICE_GET_IRQ_INFO. For each index it uses count to determine the first indices related to wire interrupts (count = 1). Then comes the MSI index, and after the MSI index. If any of those is supported, count >1, otherwise count=0. The only thing I am dubious about is can the device use a single MSI/MSIX? Because my hypothesis here is we use count to discriminate between wire first indices and other indices.

Quoted 2 lines

With this implementation user space can know that, at indexes n and n+1, edge triggered interrupts are present.

note wired interrupts can also be edge ones.

Quoted 6 lines
   We may add an element in vfio_platform_irq itself to allocate MSIs/MSIXs
   struct vfio_platform_irq{
   .....
   .....
   struct vfio_platform_irq *block; => this points to the block

allocation for MSIs/MSIXs and all msi/msix are type of IRQs.As wired interrupts and MSI interrupts coexist, I would store in vdev an

array of wired interrupts (the existing vdev->irqs) and a new array for MSI(x) as done in the PCI code.

vdev->ctx = kcalloc(nvec, sizeof(struct vfio_pci_irq_ctx), GFP_KERNEL);

Does it make sense?

Quoted 27 lines
   };
                         OR
Another structure can be defined in 'vfio_pci_private.h'
struct vfio_msi_ctx {
        struct eventfd_ctx      *trigger;
        char                    *name;
};

and
struct vfio_platform_irq {
.....
.....
struct vfio_msi_ctx *block; => this points to the block allocation
for MSIs/MSIXs
};
Which of the above two options sounds OK to you? Please suggest.

Quoted 11 lines

to me individual MSIs are encoded in the subindex and not in the index. The index just selects the "type" of interrupt.

For PCI you just have:

        VFIO_PCI_INTX_IRQ_INDEX,
        VFIO_PCI_MSI_IRQ_INDEX, -> MSI index and then you play with

start/count
VFIO_PCI_MSIX_IRQ_INDEX,
VFIO_PCI_ERR_IRQ_INDEX,
VFIO_PCI_REQ_IRQ_INDEX,

Quoted 6 lines
Quoted 1 line

(include/uapi/linux/vfio.h)

In pci case, type of interrupts is fixed so they can be 'indexed' by these enums but for VFIO platform user space will need to iterate all (num_irqs) indexes to know at which indexes edge triggered interrupts are present.

indeed, but can't you loop over all indices looking until count !=1? At this point you know if have finished emurating the wires. Holds if MSI(x) count !=1 of course.

Thanks

Eric

Quoted 28 lines

Thanks,
Vikas

Quoted 25 lines

Thanks

Eric

Quoted 20 lines

MSI-0 will have count=k set and flags set accordingly.

Vikas Gupta (3):
vfio/platform: add support for msi
vfio/platform: change cleanup order
vfio/platform: add Broadcom msi module

 drivers/vfio/platform/Kconfig                 |   1 +
 drivers/vfio/platform/Makefile                |   1 +
 drivers/vfio/platform/msi/Kconfig             |   9 +
 drivers/vfio/platform/msi/Makefile            |   2 +
 .../vfio/platform/msi/vfio_platform_bcmplt.c  |  74 ++++++
 drivers/vfio/platform/vfio_platform_common.c  |  86 ++++++-
 drivers/vfio/platform/vfio_platform_irq.c     | 238 +++++++++++++++++-
 drivers/vfio/platform/vfio_platform_private.h |  23 ++
 8 files changed, 419 insertions(+), 15 deletions(-)
 create mode 100644 drivers/vfio/platform/msi/Kconfig
 create mode 100644 drivers/vfio/platform/msi/Makefile
 create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
#20 lore raw

Hi Eric,

Quoted 37 lines · Auger Eric

Hi Vikas,

Quoted 33 lines · Vikas Gupta

Hi Eric,

Quoted 22 lines · Auger Eric

Hi Vikas,

Quoted 6 lines · Vikas Gupta

This RFC adds support for MSI for platform devices.
a) MSI(s) is/are added in addition to the normal interrupts.
b) The vendor specific MSI configuration can be done using callbacks which is implemented as msi module.
c) Adds a msi handling module for the Broadcom platform devices.

Quoted 12 lines

Changes from:
-------------
v0 to v1:
i) Removed MSI device flag VFIO_DEVICE_FLAGS_MSI. ii) Add MSI(s) at the end of the irq list of platform IRQs. MSI(s) with first entry of MSI block has count and flag information.
IRQ list: Allocation for IRQs + MSIs are allocated as below
Example: if there are 'n' IRQs and 'k' MSIs
-------------------------------------------------------
|IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
-------------------------------------------------------

Quoted 4 lines

I have not taken time yet to look at your series, but to me you should have |IRQ-0|IRQ-1|....|IRQ-n|MSI|MSIX
then for setting a given MSIX (i) you would select the MSIx index and then set start=i count=1.

As per your suggestion, we should have, if there are n-IRQs, k-MSIXs and m-MSIs, allocation of IRQs should be done as below

Quoted 8 lines
Quoted 5 lines
|IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX|
                                             |        |
                                             |
|MSIX0||MSIX1||MSXI2|....|MSIX-(k-1)|
                                             |MSI0||MSI1||MSI2|....|MSI-(m-1)|

No I really meant this list of indices: IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX| and potentially later on IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX| ERR| REQ if ERR/REQ were to be added.

I agree on this. Actually the map I drew incorrectly above but wanted to demonstrate the same. It was a child-parent relationship for MSI and its members and similarly for MSIX as well.

Quoted 5 lines

I think the userspace could query the total number of indices using VFIO_DEVICE_GET_INFO and retrieve num_irqs (corresponding to the n wire interrupts + MSI index + MSIX index)

Quoted 7 lines

Then userspace can loop on all the indices using VFIO_DEVICE_GET_IRQ_INFO. For each index it uses count to determine the first indices related to wire interrupts (count = 1). Then comes the MSI index, and after the MSI index. If any of those is supported, count >1, otherwise count=0. The only thing I am dubious about is can the device use a single MSI/MSIX? Because my hypothesis here is we use count to discriminate between wire first indices and other indices.

I believe count can be one as well, especially for ERR/REQ as you mentioned above.I think we can not rely on the count > 1. Now, this is blocking and we are not left with options unless we consider adding more enums in flags in vfio_irq_info to tell userspace that particular index is wired, MSI, MSIX etc. for the platform device. What do you think?

Quoted 12 lines
Quoted 5 lines

With this implementation user space can know that, at indexes n and n+1, edge triggered interrupts are present.

note wired interrupts can also be edge ones.

Quoted 6 lines
   We may add an element in vfio_platform_irq itself to allocate MSIs/MSIXs
   struct vfio_platform_irq{
   .....
   .....
   struct vfio_platform_irq *block; => this points to the block

allocation for MSIs/MSIXs and all msi/msix are type of IRQs.As wired interrupts and MSI interrupts coexist, I would store in vdev an

Quoted 6 lines

array of wired interrupts (the existing vdev->irqs) and a new array for MSI(x) as done in the PCI code.

vdev->ctx = kcalloc(nvec, sizeof(struct vfio_pci_irq_ctx), GFP_KERNEL);

Does it make sense?

Yes, we can use similar kinds of allocations.

Thanks,
Vikas

Quoted 71 lines
Quoted 28 lines
   };
                         OR
Another structure can be defined in 'vfio_pci_private.h'
struct vfio_msi_ctx {
        struct eventfd_ctx      *trigger;
        char                    *name;
};

and
struct vfio_platform_irq {
.....
.....
struct vfio_msi_ctx *block; => this points to the block allocation
for MSIs/MSIXs
};
Which of the above two options sounds OK to you? Please suggest.

Quoted 11 lines

to me individual MSIs are encoded in the subindex and not in the index. The index just selects the "type" of interrupt.

For PCI you just have:

        VFIO_PCI_INTX_IRQ_INDEX,
        VFIO_PCI_MSI_IRQ_INDEX, -> MSI index and then you play with

start/count
VFIO_PCI_MSIX_IRQ_INDEX,
VFIO_PCI_ERR_IRQ_INDEX,
VFIO_PCI_REQ_IRQ_INDEX,

Quoted 6 lines
Quoted 1 line

(include/uapi/linux/vfio.h)

In pci case, type of interrupts is fixed so they can be 'indexed' by these enums but for VFIO platform user space will need to iterate all (num_irqs) indexes to know at which indexes edge triggered interrupts are present.

indeed, but can't you loop over all indices looking until count !=1? At this point you know if have finished emurating the wires. Holds if MSI(x) count !=1 of course.

Thanks

Eric

Quoted 28 lines

Thanks,
Vikas

Quoted 25 lines

Thanks

Eric

Quoted 20 lines

MSI-0 will have count=k set and flags set accordingly.

Vikas Gupta (3):
vfio/platform: add support for msi
vfio/platform: change cleanup order
vfio/platform: add Broadcom msi module

 drivers/vfio/platform/Kconfig                 |   1 +
 drivers/vfio/platform/Makefile                |   1 +
 drivers/vfio/platform/msi/Kconfig             |   9 +
 drivers/vfio/platform/msi/Makefile            |   2 +
 .../vfio/platform/msi/vfio_platform_bcmplt.c  |  74 ++++++
 drivers/vfio/platform/vfio_platform_common.c  |  86 ++++++-
 drivers/vfio/platform/vfio_platform_irq.c     | 238 +++++++++++++++++-
 drivers/vfio/platform/vfio_platform_private.h |  23 ++
 8 files changed, 419 insertions(+), 15 deletions(-)
 create mode 100644 drivers/vfio/platform/msi/Kconfig
 create mode 100644 drivers/vfio/platform/msi/Makefile
 create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
maintainer
#21 lore raw

Hi Vikas,

Quoted 63 lines · Vikas Gupta

Hi Eric,

Quoted 37 lines · Auger Eric

Hi Vikas,

Quoted 33 lines · Vikas Gupta

Hi Eric,

Quoted 22 lines · Auger Eric

Hi Vikas,

Quoted 6 lines · Vikas Gupta

This RFC adds support for MSI for platform devices.
a) MSI(s) is/are added in addition to the normal interrupts.
b) The vendor specific MSI configuration can be done using callbacks which is implemented as msi module.
c) Adds a msi handling module for the Broadcom platform devices.

Quoted 12 lines

Changes from:
-------------
v0 to v1:
i) Removed MSI device flag VFIO_DEVICE_FLAGS_MSI. ii) Add MSI(s) at the end of the irq list of platform IRQs. MSI(s) with first entry of MSI block has count and flag information.
IRQ list: Allocation for IRQs + MSIs are allocated as below
Example: if there are 'n' IRQs and 'k' MSIs
-------------------------------------------------------
|IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
-------------------------------------------------------

Quoted 4 lines

I have not taken time yet to look at your series, but to me you should have |IRQ-0|IRQ-1|....|IRQ-n|MSI|MSIX
then for setting a given MSIX (i) you would select the MSIx index and then set start=i count=1.

As per your suggestion, we should have, if there are n-IRQs, k-MSIXs and m-MSIs, allocation of IRQs should be done as below

Quoted 8 lines
Quoted 5 lines
|IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX|
                                             |        |
                                             |
|MSIX0||MSIX1||MSXI2|....|MSIX-(k-1)|
                                             |MSI0||MSI1||MSI2|....|MSI-(m-1)|

No I really meant this list of indices: IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX| and potentially later on IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX| ERR| REQ if ERR/REQ were to be added.

I agree on this. Actually the map I drew incorrectly above but wanted to demonstrate the same. It was a child-parent relationship for MSI and its members and similarly for MSIX as well.

Quoted 5 lines

I think the userspace could query the total number of indices using VFIO_DEVICE_GET_INFO and retrieve num_irqs (corresponding to the n wire interrupts + MSI index + MSIX index)

Quoted 7 lines

Then userspace can loop on all the indices using VFIO_DEVICE_GET_IRQ_INFO. For each index it uses count to determine the first indices related to wire interrupts (count = 1). Then comes the MSI index, and after the MSI index. If any of those is supported, count >1, otherwise count=0. The only thing I am dubious about is can the device use a single MSI/MSIX? Because my hypothesis here is we use count to discriminate between wire first indices and other indices.

Quoted 2 lines

I believe count can be one as well, especially for ERR/REQ as you mentioned above.

Given ERR and REQ indices would follow MSI and MSIX ones, MSI index could be recognized by the first index whose count != 1. But indeed I am not sure the number of supported vectors cannot be 1. In your case it is induced by the size of the ring so it is OK but for other devices this may be different.

I think we can not rely on the count > 1. Now, this is

Quoted 4 lines

blocking and we are not left with options unless we consider adding more enums in flags in vfio_irq_info to tell userspace that particular index is wired, MSI, MSIX etc. for the platform device. What do you think?

If count is not reliable to discriminate the first n wired interrupts from the subsequen MSI and MSIx index, Alex suggested to add a capability extension in the vfio_irq_info structure. Something similar to what was done for vfio_region_info.

Such kind of thing was attempted in
https://lore.kernel.org/kvmarm/[email protected]/T/#u

` [PATCH v11 07/13] vfio: Use capability chains to handle device specific irq
[PATCH v11 08/13] vfio/pci: Add framework for custom interrupt indices [PATCH v11 09/13] vfio: Add new IRQ for DMA fault reporting

Note this has not been reviewed yet.

Thanks

Eric

Quoted 93 lines
Quoted 12 lines
Quoted 5 lines

With this implementation user space can know that, at indexes n and n+1, edge triggered interrupts are present.

note wired interrupts can also be edge ones.

Quoted 6 lines
   We may add an element in vfio_platform_irq itself to allocate MSIs/MSIXs
   struct vfio_platform_irq{
   .....
   .....
   struct vfio_platform_irq *block; => this points to the block

allocation for MSIs/MSIXs and all msi/msix are type of IRQs.As wired interrupts and MSI interrupts coexist, I would store in vdev an

Quoted 6 lines

array of wired interrupts (the existing vdev->irqs) and a new array for MSI(x) as done in the PCI code.

vdev->ctx = kcalloc(nvec, sizeof(struct vfio_pci_irq_ctx), GFP_KERNEL);

Does it make sense?

Yes, we can use similar kinds of allocations.

Thanks,
Vikas

Quoted 71 lines
Quoted 28 lines
   };
                         OR
Another structure can be defined in 'vfio_pci_private.h'
struct vfio_msi_ctx {
        struct eventfd_ctx      *trigger;
        char                    *name;
};

and
struct vfio_platform_irq {
.....
.....
struct vfio_msi_ctx *block; => this points to the block allocation
for MSIs/MSIXs
};
Which of the above two options sounds OK to you? Please suggest.

Quoted 11 lines

to me individual MSIs are encoded in the subindex and not in the index. The index just selects the "type" of interrupt.

For PCI you just have:

        VFIO_PCI_INTX_IRQ_INDEX,
        VFIO_PCI_MSI_IRQ_INDEX, -> MSI index and then you play with

start/count
VFIO_PCI_MSIX_IRQ_INDEX,
VFIO_PCI_ERR_IRQ_INDEX,
VFIO_PCI_REQ_IRQ_INDEX,

Quoted 6 lines
Quoted 1 line

(include/uapi/linux/vfio.h)

In pci case, type of interrupts is fixed so they can be 'indexed' by these enums but for VFIO platform user space will need to iterate all (num_irqs) indexes to know at which indexes edge triggered interrupts are present.

indeed, but can't you loop over all indices looking until count !=1? At this point you know if have finished emurating the wires. Holds if MSI(x) count !=1 of course.

Thanks

Eric

Quoted 28 lines

Thanks,
Vikas

Quoted 25 lines

Thanks

Eric

Quoted 20 lines

MSI-0 will have count=k set and flags set accordingly.

Vikas Gupta (3):
vfio/platform: add support for msi
vfio/platform: change cleanup order
vfio/platform: add Broadcom msi module

 drivers/vfio/platform/Kconfig                 |   1 +
 drivers/vfio/platform/Makefile                |   1 +
 drivers/vfio/platform/msi/Kconfig             |   9 +
 drivers/vfio/platform/msi/Makefile            |   2 +
 .../vfio/platform/msi/vfio_platform_bcmplt.c  |  74 ++++++
 drivers/vfio/platform/vfio_platform_common.c  |  86 ++++++-
 drivers/vfio/platform/vfio_platform_irq.c     | 238 +++++++++++++++++-
 drivers/vfio/platform/vfio_platform_private.h |  23 ++
 8 files changed, 419 insertions(+), 15 deletions(-)
 create mode 100644 drivers/vfio/platform/msi/Kconfig
 create mode 100644 drivers/vfio/platform/msi/Makefile
 create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
maintainer
#22 lore raw

Hi Vikas,

Quoted 84 lines · Auger Eric

Hi Vikas,

Quoted 63 lines · Vikas Gupta

Hi Eric,

Quoted 37 lines · Auger Eric

Hi Vikas,

Quoted 33 lines · Vikas Gupta

Hi Eric,

Quoted 22 lines · Auger Eric

Hi Vikas,

Quoted 6 lines · Vikas Gupta

This RFC adds support for MSI for platform devices.
a) MSI(s) is/are added in addition to the normal interrupts.
b) The vendor specific MSI configuration can be done using callbacks which is implemented as msi module.
c) Adds a msi handling module for the Broadcom platform devices.

Quoted 12 lines

Changes from:
-------------
v0 to v1:
i) Removed MSI device flag VFIO_DEVICE_FLAGS_MSI. ii) Add MSI(s) at the end of the irq list of platform IRQs. MSI(s) with first entry of MSI block has count and flag information.
IRQ list: Allocation for IRQs + MSIs are allocated as below
Example: if there are 'n' IRQs and 'k' MSIs
-------------------------------------------------------
|IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
-------------------------------------------------------

Quoted 4 lines

I have not taken time yet to look at your series, but to me you should have |IRQ-0|IRQ-1|....|IRQ-n|MSI|MSIX
then for setting a given MSIX (i) you would select the MSIx index and then set start=i count=1.

As per your suggestion, we should have, if there are n-IRQs, k-MSIXs and m-MSIs, allocation of IRQs should be done as below

Quoted 8 lines
Quoted 5 lines
|IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX|
                                             |        |
                                             |
|MSIX0||MSIX1||MSXI2|....|MSIX-(k-1)|
                                             |MSI0||MSI1||MSI2|....|MSI-(m-1)|

No I really meant this list of indices: IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX| and potentially later on IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX| ERR| REQ if ERR/REQ were to be added.

I agree on this. Actually the map I drew incorrectly above but wanted to demonstrate the same. It was a child-parent relationship for MSI and its members and similarly for MSIX as well.

Quoted 5 lines

I think the userspace could query the total number of indices using VFIO_DEVICE_GET_INFO and retrieve num_irqs (corresponding to the n wire interrupts + MSI index + MSIX index)

Quoted 7 lines

Then userspace can loop on all the indices using VFIO_DEVICE_GET_IRQ_INFO. For each index it uses count to determine the first indices related to wire interrupts (count = 1). Then comes the MSI index, and after the MSI index. If any of those is supported, count >1, otherwise count=0. The only thing I am dubious about is can the device use a single MSI/MSIX? Because my hypothesis here is we use count to discriminate between wire first indices and other indices.

Quoted 2 lines

I believe count can be one as well, especially for ERR/REQ as you mentioned above.

Given ERR and REQ indices would follow MSI and MSIX ones, MSI index could be recognized by the first index whose count != 1. But indeed I am not sure the number of supported vectors cannot be 1. In your case it is induced by the size of the ring so it is OK but for other devices this may be different.

I think we can not rely on the count > 1. Now, this is

Quoted 4 lines

blocking and we are not left with options unless we consider adding more enums in flags in vfio_irq_info to tell userspace that particular index is wired, MSI, MSIX etc. for the platform device. What do you think?

If count is not reliable to discriminate the first n wired interrupts from the subsequen MSI and MSIx index, Alex suggested to add a capability extension in the vfio_irq_info structure. Something similar to what was done for vfio_region_info.

Quoted 7 lines

Such kind of thing was attempted in
https://lore.kernel.org/kvmarm/[email protected]/T/#u

` [PATCH v11 07/13] vfio: Use capability chains to handle device specific irq
[PATCH v11 08/13] vfio/pci: Add framework for custom interrupt indices [PATCH v11 09/13] vfio: Add new IRQ for DMA fault reporting

By the way I was mentionning MSI/MSIx in my previous reply but, as Alex pointed out, with platform device only a single MSI index does make sense, no?

Thanks

Eric

Quoted 100 lines

Note this has not been reviewed yet.

Thanks

Eric

Quoted 93 lines
Quoted 12 lines
Quoted 5 lines

With this implementation user space can know that, at indexes n and n+1, edge triggered interrupts are present.

note wired interrupts can also be edge ones.

Quoted 6 lines
   We may add an element in vfio_platform_irq itself to allocate MSIs/MSIXs
   struct vfio_platform_irq{
   .....
   .....
   struct vfio_platform_irq *block; => this points to the block

allocation for MSIs/MSIXs and all msi/msix are type of IRQs.As wired interrupts and MSI interrupts coexist, I would store in vdev an

Quoted 6 lines

array of wired interrupts (the existing vdev->irqs) and a new array for MSI(x) as done in the PCI code.

vdev->ctx = kcalloc(nvec, sizeof(struct vfio_pci_irq_ctx), GFP_KERNEL);

Does it make sense?

Yes, we can use similar kinds of allocations.

Thanks,
Vikas

Quoted 71 lines
Quoted 28 lines
   };
                         OR
Another structure can be defined in 'vfio_pci_private.h'
struct vfio_msi_ctx {
        struct eventfd_ctx      *trigger;
        char                    *name;
};

and
struct vfio_platform_irq {
.....
.....
struct vfio_msi_ctx *block; => this points to the block allocation
for MSIs/MSIXs
};
Which of the above two options sounds OK to you? Please suggest.

Quoted 11 lines

to me individual MSIs are encoded in the subindex and not in the index. The index just selects the "type" of interrupt.

For PCI you just have:

        VFIO_PCI_INTX_IRQ_INDEX,
        VFIO_PCI_MSI_IRQ_INDEX, -> MSI index and then you play with

start/count
VFIO_PCI_MSIX_IRQ_INDEX,
VFIO_PCI_ERR_IRQ_INDEX,
VFIO_PCI_REQ_IRQ_INDEX,

Quoted 6 lines
Quoted 1 line

(include/uapi/linux/vfio.h)

In pci case, type of interrupts is fixed so they can be 'indexed' by these enums but for VFIO platform user space will need to iterate all (num_irqs) indexes to know at which indexes edge triggered interrupts are present.

indeed, but can't you loop over all indices looking until count !=1? At this point you know if have finished emurating the wires. Holds if MSI(x) count !=1 of course.

Thanks

Eric

Quoted 28 lines

Thanks,
Vikas

Quoted 25 lines

Thanks

Eric

Quoted 20 lines

MSI-0 will have count=k set and flags set accordingly.

Vikas Gupta (3):
vfio/platform: add support for msi
vfio/platform: change cleanup order
vfio/platform: add Broadcom msi module

 drivers/vfio/platform/Kconfig                 |   1 +
 drivers/vfio/platform/Makefile                |   1 +
 drivers/vfio/platform/msi/Kconfig             |   9 +
 drivers/vfio/platform/msi/Makefile            |   2 +
 .../vfio/platform/msi/vfio_platform_bcmplt.c  |  74 ++++++
 drivers/vfio/platform/vfio_platform_common.c  |  86 ++++++-
 drivers/vfio/platform/vfio_platform_irq.c     | 238 +++++++++++++++++-
 drivers/vfio/platform/vfio_platform_private.h |  23 ++
 8 files changed, 419 insertions(+), 15 deletions(-)
 create mode 100644 drivers/vfio/platform/msi/Kconfig
 create mode 100644 drivers/vfio/platform/msi/Makefile
 create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
#23 lore raw

Hi Eric,

Quoted 91 lines · Auger Eric

Hi Vikas,

Quoted 87 lines · Auger Eric

Hi Vikas,

Quoted 63 lines · Vikas Gupta

Hi Eric,

Quoted 37 lines · Auger Eric

Hi Vikas,

Quoted 33 lines · Vikas Gupta

Hi Eric,

Quoted 22 lines · Auger Eric

Hi Vikas,

Quoted 6 lines · Vikas Gupta

This RFC adds support for MSI for platform devices.
a) MSI(s) is/are added in addition to the normal interrupts.
b) The vendor specific MSI configuration can be done using callbacks which is implemented as msi module.
c) Adds a msi handling module for the Broadcom platform devices.

Quoted 12 lines

Changes from:
-------------
v0 to v1:
i) Removed MSI device flag VFIO_DEVICE_FLAGS_MSI. ii) Add MSI(s) at the end of the irq list of platform IRQs. MSI(s) with first entry of MSI block has count and flag information.
IRQ list: Allocation for IRQs + MSIs are allocated as below
Example: if there are 'n' IRQs and 'k' MSIs
-------------------------------------------------------
|IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
-------------------------------------------------------

Quoted 4 lines

I have not taken time yet to look at your series, but to me you should have |IRQ-0|IRQ-1|....|IRQ-n|MSI|MSIX
then for setting a given MSIX (i) you would select the MSIx index and then set start=i count=1.

As per your suggestion, we should have, if there are n-IRQs, k-MSIXs and m-MSIs, allocation of IRQs should be done as below

Quoted 8 lines
Quoted 5 lines
|IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX|
                                             |        |
                                             |
|MSIX0||MSIX1||MSXI2|....|MSIX-(k-1)|
                                             |MSI0||MSI1||MSI2|....|MSI-(m-1)|

No I really meant this list of indices: IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX| and potentially later on IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX| ERR| REQ if ERR/REQ were to be added.

I agree on this. Actually the map I drew incorrectly above but wanted to demonstrate the same. It was a child-parent relationship for MSI and its members and similarly for MSIX as well.

Quoted 5 lines

I think the userspace could query the total number of indices using VFIO_DEVICE_GET_INFO and retrieve num_irqs (corresponding to the n wire interrupts + MSI index + MSIX index)

Quoted 7 lines

Then userspace can loop on all the indices using VFIO_DEVICE_GET_IRQ_INFO. For each index it uses count to determine the first indices related to wire interrupts (count = 1). Then comes the MSI index, and after the MSI index. If any of those is supported, count >1, otherwise count=0. The only thing I am dubious about is can the device use a single MSI/MSIX? Because my hypothesis here is we use count to discriminate between wire first indices and other indices.

Quoted 2 lines

I believe count can be one as well, especially for ERR/REQ as you mentioned above.

Given ERR and REQ indices would follow MSI and MSIX ones, MSI index could be recognized by the first index whose count != 1. But indeed I am not sure the number of supported vectors cannot be 1. In your case it is induced by the size of the ring so it is OK but for other devices this may be different.

I think we can not rely on the count > 1. Now, this is

Quoted 4 lines

blocking and we are not left with options unless we consider adding more enums in flags in vfio_irq_info to tell userspace that particular index is wired, MSI, MSIX etc. for the platform device. What do you think?

If count is not reliable to discriminate the first n wired interrupts from the subsequen MSI and MSIx index, Alex suggested to add a capability extension in the vfio_irq_info structure. Something similar to what was done for vfio_region_info.

Such kind of thing was attempted in
https://lore.kernel.org/kvmarm/[email protected]/T/#u

Quoted 8 lines
Quoted 4 lines

` [PATCH v11 07/13] vfio: Use capability chains to handle device specific irq
[PATCH v11 08/13] vfio/pci: Add framework for custom interrupt indices [PATCH v11 09/13] vfio: Add new IRQ for DMA fault reporting

By the way I was mentionning MSI/MSIx in my previous reply but, as Alex pointed out, with platform device only a single MSI index does make sense, no?

Yes, I think single MSI should be OK.
This single MSI index should be implemented as ext_irqs, similar to, as you implemented in the mentioned patch. Is my understanding correct?
Thanks,
Vikas

Quoted 105 lines

Thanks

Eric

Quoted 100 lines

Note this has not been reviewed yet.

Thanks

Eric

Quoted 93 lines
Quoted 12 lines
Quoted 5 lines

With this implementation user space can know that, at indexes n and n+1, edge triggered interrupts are present.

note wired interrupts can also be edge ones.

Quoted 6 lines
   We may add an element in vfio_platform_irq itself to allocate MSIs/MSIXs
   struct vfio_platform_irq{
   .....
   .....
   struct vfio_platform_irq *block; => this points to the block

allocation for MSIs/MSIXs and all msi/msix are type of IRQs.As wired interrupts and MSI interrupts coexist, I would store in vdev an

Quoted 6 lines

array of wired interrupts (the existing vdev->irqs) and a new array for MSI(x) as done in the PCI code.

vdev->ctx = kcalloc(nvec, sizeof(struct vfio_pci_irq_ctx), GFP_KERNEL);

Does it make sense?

Yes, we can use similar kinds of allocations.

Thanks,
Vikas

Quoted 71 lines
Quoted 28 lines
   };
                         OR
Another structure can be defined in 'vfio_pci_private.h'
struct vfio_msi_ctx {
        struct eventfd_ctx      *trigger;
        char                    *name;
};

and
struct vfio_platform_irq {
.....
.....
struct vfio_msi_ctx *block; => this points to the block allocation
for MSIs/MSIXs
};
Which of the above two options sounds OK to you? Please suggest.

Quoted 11 lines

to me individual MSIs are encoded in the subindex and not in the index. The index just selects the "type" of interrupt.

For PCI you just have:

        VFIO_PCI_INTX_IRQ_INDEX,
        VFIO_PCI_MSI_IRQ_INDEX, -> MSI index and then you play with

start/count
VFIO_PCI_MSIX_IRQ_INDEX,
VFIO_PCI_ERR_IRQ_INDEX,
VFIO_PCI_REQ_IRQ_INDEX,

Quoted 6 lines
Quoted 1 line

(include/uapi/linux/vfio.h)

In pci case, type of interrupts is fixed so they can be 'indexed' by these enums but for VFIO platform user space will need to iterate all (num_irqs) indexes to know at which indexes edge triggered interrupts are present.

indeed, but can't you loop over all indices looking until count !=1? At this point you know if have finished emurating the wires. Holds if MSI(x) count !=1 of course.

Thanks

Eric

Quoted 28 lines

Thanks,
Vikas

Quoted 25 lines

Thanks

Eric

Quoted 20 lines

MSI-0 will have count=k set and flags set accordingly.

Vikas Gupta (3):
vfio/platform: add support for msi
vfio/platform: change cleanup order
vfio/platform: add Broadcom msi module

 drivers/vfio/platform/Kconfig                 |   1 +
 drivers/vfio/platform/Makefile                |   1 +
 drivers/vfio/platform/msi/Kconfig             |   9 +
 drivers/vfio/platform/msi/Makefile            |   2 +
 .../vfio/platform/msi/vfio_platform_bcmplt.c  |  74 ++++++
 drivers/vfio/platform/vfio_platform_common.c  |  86 ++++++-
 drivers/vfio/platform/vfio_platform_irq.c     | 238 +++++++++++++++++-
 drivers/vfio/platform/vfio_platform_private.h |  23 ++
 8 files changed, 419 insertions(+), 15 deletions(-)
 create mode 100644 drivers/vfio/platform/msi/Kconfig
 create mode 100644 drivers/vfio/platform/msi/Makefile
 create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
maintainer
#24 lore raw

Hi Vikas,

Quoted 99 lines · Vikas Gupta

Hi Eric,

Quoted 96 lines · Auger Eric

Hi Vikas,

Quoted 91 lines · Auger Eric

Hi Vikas,

Quoted 63 lines · Vikas Gupta

Hi Eric,

Quoted 37 lines · Auger Eric

Hi Vikas,

Quoted 33 lines · Vikas Gupta

Hi Eric,

Quoted 22 lines · Auger Eric

Hi Vikas,

Quoted 6 lines · Vikas Gupta

This RFC adds support for MSI for platform devices.
a) MSI(s) is/are added in addition to the normal interrupts.
b) The vendor specific MSI configuration can be done using callbacks which is implemented as msi module.
c) Adds a msi handling module for the Broadcom platform devices.

Quoted 12 lines

Changes from:
-------------
v0 to v1:
i) Removed MSI device flag VFIO_DEVICE_FLAGS_MSI. ii) Add MSI(s) at the end of the irq list of platform IRQs. MSI(s) with first entry of MSI block has count and flag information.
IRQ list: Allocation for IRQs + MSIs are allocated as below
Example: if there are 'n' IRQs and 'k' MSIs
-------------------------------------------------------
|IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
-------------------------------------------------------

Quoted 4 lines

I have not taken time yet to look at your series, but to me you should have |IRQ-0|IRQ-1|....|IRQ-n|MSI|MSIX
then for setting a given MSIX (i) you would select the MSIx index and then set start=i count=1.

As per your suggestion, we should have, if there are n-IRQs, k-MSIXs and m-MSIs, allocation of IRQs should be done as below

Quoted 8 lines
Quoted 5 lines
|IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX|
                                             |        |
                                             |
|MSIX0||MSIX1||MSXI2|....|MSIX-(k-1)|
                                             |MSI0||MSI1||MSI2|....|MSI-(m-1)|

No I really meant this list of indices: IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX| and potentially later on IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX| ERR| REQ if ERR/REQ were to be added.

I agree on this. Actually the map I drew incorrectly above but wanted to demonstrate the same. It was a child-parent relationship for MSI and its members and similarly for MSIX as well.

Quoted 5 lines

I think the userspace could query the total number of indices using VFIO_DEVICE_GET_INFO and retrieve num_irqs (corresponding to the n wire interrupts + MSI index + MSIX index)

Quoted 7 lines

Then userspace can loop on all the indices using VFIO_DEVICE_GET_IRQ_INFO. For each index it uses count to determine the first indices related to wire interrupts (count = 1). Then comes the MSI index, and after the MSI index. If any of those is supported, count >1, otherwise count=0. The only thing I am dubious about is can the device use a single MSI/MSIX? Because my hypothesis here is we use count to discriminate between wire first indices and other indices.

Quoted 2 lines

I believe count can be one as well, especially for ERR/REQ as you mentioned above.

Given ERR and REQ indices would follow MSI and MSIX ones, MSI index could be recognized by the first index whose count != 1. But indeed I am not sure the number of supported vectors cannot be 1. In your case it is induced by the size of the ring so it is OK but for other devices this may be different.

I think we can not rely on the count > 1. Now, this is

Quoted 4 lines

blocking and we are not left with options unless we consider adding more enums in flags in vfio_irq_info to tell userspace that particular index is wired, MSI, MSIX etc. for the platform device. What do you think?

If count is not reliable to discriminate the first n wired interrupts from the subsequen MSI and MSIx index, Alex suggested to add a capability extension in the vfio_irq_info structure. Something similar to what was done for vfio_region_info.

Such kind of thing was attempted in
https://lore.kernel.org/kvmarm/[email protected]/T/#u

` [PATCH v11 07/13] vfio: Use capability chains to handle device specific irq
[PATCH v11 08/13] vfio/pci: Add framework for custom interrupt indices [PATCH v11 09/13] vfio: Add new IRQ for DMA fault reporting

Quoted 7 lines
Quoted 3 lines

By the way I was mentionning MSI/MSIx in my previous reply but, as Alex pointed out, with platform device only a single MSI index does make sense, no?

Yes, I think single MSI should be OK.
This single MSI index should be implemented as ext_irqs, similar to, as you implemented in the mentioned patch. Is my understanding correct?

Yes, if count !=1 cannot be used to detect the MSI index, I think using a capability would do the job and this is aligned with last Alex' suggestion.

Thanks

Eric

Quoted 107 lines

Thanks,
Vikas

Quoted 105 lines

Thanks

Eric

Quoted 100 lines

Note this has not been reviewed yet.

Thanks

Eric

Quoted 93 lines
Quoted 12 lines
Quoted 5 lines

With this implementation user space can know that, at indexes n and n+1, edge triggered interrupts are present.

note wired interrupts can also be edge ones.

Quoted 6 lines
   We may add an element in vfio_platform_irq itself to allocate MSIs/MSIXs
   struct vfio_platform_irq{
   .....
   .....
   struct vfio_platform_irq *block; => this points to the block

allocation for MSIs/MSIXs and all msi/msix are type of IRQs.As wired interrupts and MSI interrupts coexist, I would store in vdev an

Quoted 6 lines

array of wired interrupts (the existing vdev->irqs) and a new array for MSI(x) as done in the PCI code.

vdev->ctx = kcalloc(nvec, sizeof(struct vfio_pci_irq_ctx), GFP_KERNEL);

Does it make sense?

Yes, we can use similar kinds of allocations.

Thanks,
Vikas

Quoted 71 lines
Quoted 28 lines
   };
                         OR
Another structure can be defined in 'vfio_pci_private.h'
struct vfio_msi_ctx {
        struct eventfd_ctx      *trigger;
        char                    *name;
};

and
struct vfio_platform_irq {
.....
.....
struct vfio_msi_ctx *block; => this points to the block allocation
for MSIs/MSIXs
};
Which of the above two options sounds OK to you? Please suggest.

Quoted 11 lines

to me individual MSIs are encoded in the subindex and not in the index. The index just selects the "type" of interrupt.

For PCI you just have:

        VFIO_PCI_INTX_IRQ_INDEX,
        VFIO_PCI_MSI_IRQ_INDEX, -> MSI index and then you play with

start/count
VFIO_PCI_MSIX_IRQ_INDEX,
VFIO_PCI_ERR_IRQ_INDEX,
VFIO_PCI_REQ_IRQ_INDEX,

Quoted 6 lines
Quoted 1 line

(include/uapi/linux/vfio.h)

In pci case, type of interrupts is fixed so they can be 'indexed' by these enums but for VFIO platform user space will need to iterate all (num_irqs) indexes to know at which indexes edge triggered interrupts are present.

indeed, but can't you loop over all indices looking until count !=1? At this point you know if have finished emurating the wires. Holds if MSI(x) count !=1 of course.

Thanks

Eric

Quoted 28 lines

Thanks,
Vikas

Quoted 25 lines

Thanks

Eric

Quoted 20 lines

MSI-0 will have count=k set and flags set accordingly.

Vikas Gupta (3):
vfio/platform: add support for msi
vfio/platform: change cleanup order
vfio/platform: add Broadcom msi module

 drivers/vfio/platform/Kconfig                 |   1 +
 drivers/vfio/platform/Makefile                |   1 +
 drivers/vfio/platform/msi/Kconfig             |   9 +
 drivers/vfio/platform/msi/Makefile            |   2 +
 .../vfio/platform/msi/vfio_platform_bcmplt.c  |  74 ++++++
 drivers/vfio/platform/vfio_platform_common.c  |  86 ++++++-
 drivers/vfio/platform/vfio_platform_irq.c     | 238 +++++++++++++++++-
 drivers/vfio/platform/vfio_platform_private.h |  23 ++
 8 files changed, 419 insertions(+), 15 deletions(-)
 create mode 100644 drivers/vfio/platform/msi/Kconfig
 create mode 100644 drivers/vfio/platform/msi/Makefile
 create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
#25 lore raw

v2 cover msi support for platform devices

In reply to Vikas Gupta

This RFC adds support for MSI for platform devices. MSI block is added as an ext irq along with the existing wired interrupt implementation.

Changes from:
-------------
v1 to v2:
1) IRQ allocation has been implemented as below:
----------------------------
|IRQ-0|IRQ-1|....|IRQ-n|MSI|
----------------------------
MSI block has msi contexts and its implemneted as ext irq.

2) Removed vendor specific module for msi handling so previously patch2 and patch3 are not required.

3) MSI related data is exported to userspace using 'caps'.
Please note VFIO_IRQ_INFO_CAP_TYPE in include/uapi/linux/vfio.h implementation
is taken from the Eric`s patch
https://patchwork.kernel.org/project/kvm/patch/[email protected]/

v0 to v1:
i) Removed MSI device flag VFIO_DEVICE_FLAGS_MSI. ii) Add MSI(s) at the end of the irq list of platform IRQs. MSI(s) with first entry of MSI block has count and flag information.
IRQ list: Allocation for IRQs + MSIs are allocated as below
Example: if there are 'n' IRQs and 'k' MSIs
-------------------------------------------------------
|IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
-------------------------------------------------------
MSI-0 will have count=k set and flags set accordingly.

Vikas Gupta (1):
vfio/platform: add support for msi

 drivers/vfio/platform/vfio_platform_common.c  |  99 ++++++-
 drivers/vfio/platform/vfio_platform_irq.c     | 260 +++++++++++++++++-
 drivers/vfio/platform/vfio_platform_private.h |  16 ++
 include/uapi/linux/vfio.h                     |  43 +++
 4 files changed, 401 insertions(+), 17 deletions(-)
Signature · 1 line
2.17.1
#26 lore raw

RFC v2 1/1 vfio/platform: add support for msi

MSI support for platform devices.

---
 drivers/vfio/platform/vfio_platform_common.c  |  99 ++++++-
 drivers/vfio/platform/vfio_platform_irq.c     | 260 +++++++++++++++++-
 drivers/vfio/platform/vfio_platform_private.h |  16 ++
 include/uapi/linux/vfio.h                     |  43 +++
 4 files changed, 401 insertions(+), 17 deletions(-)
drivers/vfio/platform/vfio_platform_common.c +97 -2
index c0771a9567fb..b0bfc0f4ee1f 100644
@@ -16,6 +16,7 @@
 #include <linux/types.h>
 #include <linux/uaccess.h>
 #include <linux/vfio.h>
+#include <linux/nospec.h>
 
 #include "vfio_platform_private.h"
 
@@ -344,9 +345,16 @@ static long vfio_platform_ioctl(void *device_data,
 
 	} else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
 		struct vfio_irq_info info;
+		struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
+		struct vfio_irq_info_cap_msi *msi_info = NULL;
+		unsigned long capsz;
+		int ext_irq_index = vdev->num_irqs - vdev->num_ext_irqs;
 
 		minsz = offsetofend(struct vfio_irq_info, count);
 
+		/* For backward compatibility, cannot require this */
+		capsz = offsetofend(struct vfio_irq_info, cap_offset);
+
 		if (copy_from_user(&info, (void __user *)arg, minsz))
 			return -EFAULT;
 
@@ -356,9 +364,89 @@ static long vfio_platform_ioctl(void *device_data,
 		if (info.index >= vdev->num_irqs)
 			return -EINVAL;
 
+		if (info.argsz >= capsz)
+			minsz = capsz;
+
+		if (info.index == ext_irq_index) {
+			struct vfio_irq_info_cap_type cap_type = {
+				.header.id = VFIO_IRQ_INFO_CAP_TYPE,
+				.header.version = 1 };
+			int i;
+			int ret;
+			int num_msgs;
+			size_t msi_info_size;
+			struct vfio_platform_irq *irq;
+
+			info.index = array_index_nospec(info.index,
+							vdev->num_irqs);
+
+			irq = &vdev->irqs[info.index];
+
+			info.flags = irq->flags;
+			cap_type.type = irq->type;
+			cap_type.subtype = irq->subtype;
+
+			ret = vfio_info_add_capability(&caps,
+						       &cap_type.header,
+						       sizeof(cap_type));
+			if (ret)
+				return ret;
+
+			num_msgs = irq->num_ctx;
+
+			msi_info_size = struct_size(msi_info, msgs, num_msgs);
+
+			msi_info = kzalloc(msi_info_size, GFP_KERNEL);
+			if (!msi_info) {
+				kfree(caps.buf);
+				return -ENOMEM;
+			}
+
+			msi_info->header.id = VFIO_IRQ_INFO_CAP_MSI_DESCS;
+			msi_info->header.version = 1;
+			msi_info->nr_msgs = num_msgs;
+
+			for (i = 0; i < num_msgs; i++) {
+				struct vfio_irq_ctx *ctx = &irq->ctx[i];
+
+				msi_info->msgs[i].addr_lo = ctx->msg.address_lo;
+				msi_info->msgs[i].addr_hi = ctx->msg.address_hi;
+				msi_info->msgs[i].data = ctx->msg.data;
+			}
+
+			ret = vfio_info_add_capability(&caps, &msi_info->header,
+						       msi_info_size);
+			if (ret) {
+				kfree(msi_info);
+				kfree(caps.buf);
+				return ret;
+			}
+		}
+
 		info.flags = vdev->irqs[info.index].flags;
 		info.count = vdev->irqs[info.index].count;
 
+		if (caps.size) {
+			info.flags |= VFIO_IRQ_INFO_FLAG_CAPS;
+			if (info.argsz < sizeof(info) + caps.size) {
+				info.argsz = sizeof(info) + caps.size;
+				info.cap_offset = 0;
+			} else {
+				vfio_info_cap_shift(&caps, sizeof(info));
+				if (copy_to_user((void __user *)arg +
+						  sizeof(info), caps.buf,
+						  caps.size)) {
+					kfree(msi_info);
+					kfree(caps.buf);
+					return -EFAULT;
+				}
+				info.cap_offset = sizeof(info);
+			}
+
+			kfree(msi_info);
+			kfree(caps.buf);
+		}
+
 		return copy_to_user((void __user *)arg, &info, minsz) ?
 			-EFAULT : 0;
 
@@ -366,6 +454,7 @@ static long vfio_platform_ioctl(void *device_data,
 		struct vfio_irq_set hdr;
 		u8 *data = NULL;
 		int ret = 0;
+		int max;
 		size_t data_size = 0;
 
 		minsz = offsetofend(struct vfio_irq_set, count);
@@ -373,8 +462,14 @@ static long vfio_platform_ioctl(void *device_data,
 		if (copy_from_user(&hdr, (void __user *)arg, minsz))
 			return -EFAULT;
 
-		ret = vfio_set_irqs_validate_and_prepare(&hdr, vdev->num_irqs,
-						 vdev->num_irqs, &data_size);
+		if (hdr.index >= vdev->num_irqs)
+			return -EINVAL;
+
+		max = vdev->irqs[hdr.index].count;
+
+		ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
+							 vdev->num_irqs,
+							 &data_size);
 		if (ret)
 			return ret;
 
drivers/vfio/platform/vfio_platform_irq.c +245 -15

304 lines not on this page · raw message

drivers/vfio/platform/vfio_platform_private.h +16 -0
index 289089910643..7bbb05988705 100644
@@ -9,6 +9,7 @@
 
 #include <linux/types.h>
 #include <linux/interrupt.h>
+#include <linux/msi.h>
 
 #define VFIO_PLATFORM_OFFSET_SHIFT   40
 #define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
@@ -19,9 +20,18 @@
 #define VFIO_PLATFORM_INDEX_TO_OFFSET(index)	\
 	((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT)
 
+struct vfio_irq_ctx {
+	int			hwirq;
+	char			*name;
+	struct msi_msg		msg;
+	struct eventfd_ctx	*trigger;
+};
+
 struct vfio_platform_irq {
 	u32			flags;
 	u32			count;
+	int			num_ctx;
+	struct vfio_irq_ctx	*ctx;
 	int			hwirq;
 	char			*name;
 	struct eventfd_ctx	*trigger;
@@ -29,6 +39,11 @@ struct vfio_platform_irq {
 	spinlock_t		lock;
 	struct virqfd		*unmask;
 	struct virqfd		*mask;
+
+	/* for extended irqs */
+	u32			type;
+	u32			subtype;
+	int			config_msi;
 };
 
 struct vfio_platform_region {
@@ -46,6 +61,7 @@ struct vfio_platform_device {
 	u32				num_regions;
 	struct vfio_platform_irq	*irqs;
 	u32				num_irqs;
+	int				num_ext_irqs;
 	int				refcnt;
 	struct mutex			igate;
 	struct module			*parent_module;
include/uapi/linux/vfio.h +43 -0

54 lines not on this page · raw message

Signature · 1 line
2.17.1
maintainer
#27 lore raw

Re: [RFC v2 1/1] vfio/platform: add support for msi

Hi Vikas,

Quoted 40 lines · Vikas Gupta

MSI support for platform devices.

---
 drivers/vfio/platform/vfio_platform_common.c  |  99 ++++++-
 drivers/vfio/platform/vfio_platform_irq.c     | 260 +++++++++++++++++-
 drivers/vfio/platform/vfio_platform_private.h |  16 ++
 include/uapi/linux/vfio.h                     |  43 +++
 4 files changed, 401 insertions(+), 17 deletions(-)
drivers/vfio/platform/vfio_platform_common.c +8 -0
index c0771a9567fb..b0bfc0f4ee1f 100644
@@ -16,6 +16,7 @@
 #include <linux/types.h>
 #include <linux/uaccess.h>
 #include <linux/vfio.h>
+#include <linux/nospec.h>
 
 #include "vfio_platform_private.h"
 
@@ -344,9 +345,16 @@ static long vfio_platform_ioctl(void *device_data,
 
 	} else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
 		struct vfio_irq_info info;
+		struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
+		struct vfio_irq_info_cap_msi *msi_info = NULL;
+		unsigned long capsz;
+		int ext_irq_index = vdev->num_irqs - vdev->num_ext_irqs;
 
 		minsz = offsetofend(struct vfio_irq_info, count);
 
+		/* For backward compatibility, cannot require this */
+		capsz = offsetofend(struct vfio_irq_info, cap_offset);
+
 		if (copy_from_user(&info, (void __user *)arg, minsz))
 			return -EFAULT;
 

drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 370

Quoted 7 lines
drivers/vfio/platform/vfio_platform_common.c +4 -0
@@ -356,9 +364,89 @@ static long vfio_platform_ioctl(void *device_data,
 		if (info.index >= vdev->num_irqs)
 			return -EINVAL;
 
+		if (info.argsz >= capsz)
+			minsz = capsz;
+
+		if (info.index == ext_irq_index) {

nit: n case we add new ext indices afterwards, I would check info.index
- ext_irq_index against an VFIO_EXT_IRQ_MSI define.

drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 378

Quoted 8 lines
drivers/vfio/platform/vfio_platform_common.c +8 -0
+			struct vfio_irq_info_cap_type cap_type = {
+				.header.id = VFIO_IRQ_INFO_CAP_TYPE,
+				.header.version = 1 };
+			int i;
+			int ret;
+			int num_msgs;
+			size_t msi_info_size;
+			struct vfio_platform_irq *irq;

nit: I think generally the opposite order (length) is chosen. This also would better match the existing style in this file

drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 385

Quoted 7 lines
drivers/vfio/platform/vfio_platform_common.c +7 -0
+
+			info.index = array_index_nospec(info.index,
+							vdev->num_irqs);
+
+			irq = &vdev->irqs[info.index];
+
+			info.flags = irq->flags;

I think this can be removed given [*]

drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 395

Quoted 10 lines
drivers/vfio/platform/vfio_platform_common.c +10 -0
+			cap_type.type = irq->type;
+			cap_type.subtype = irq->subtype;
+
+			ret = vfio_info_add_capability(&caps,
+						       &cap_type.header,
+						       sizeof(cap_type));
+			if (ret)
+				return ret;
+
+			num_msgs = irq->num_ctx;

do would want to return the cap even if !num_ctx?

Quoted 21 lines
drivers/vfio/platform/vfio_platform_common.c +21 -0
+
+			msi_info_size = struct_size(msi_info, msgs, num_msgs);
+
+			msi_info = kzalloc(msi_info_size, GFP_KERNEL);
+			if (!msi_info) {
+				kfree(caps.buf);
+				return -ENOMEM;
+			}
+
+			msi_info->header.id = VFIO_IRQ_INFO_CAP_MSI_DESCS;
+			msi_info->header.version = 1;
+			msi_info->nr_msgs = num_msgs;
+
+			for (i = 0; i < num_msgs; i++) {
+				struct vfio_irq_ctx *ctx = &irq->ctx[i];
+
+				msi_info->msgs[i].addr_lo = ctx->msg.address_lo;
+				msi_info->msgs[i].addr_hi = ctx->msg.address_hi;
+				msi_info->msgs[i].data = ctx->msg.data;
+			}
+

drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 426

Quoted 10 lines
drivers/vfio/platform/vfio_platform_common.c +9 -0
+			ret = vfio_info_add_capability(&caps, &msi_info->header,
+						       msi_info_size);
+			if (ret) {
+				kfree(msi_info);
+				kfree(caps.buf);
+				return ret;
+			}
+		}
+
 		info.flags = vdev->irqs[info.index].flags;

[*]

Quoted 329 lines
drivers/vfio/platform/vfio_platform_common.c +30 -2
 		info.count = vdev->irqs[info.index].count;
 
+		if (caps.size) {
+			info.flags |= VFIO_IRQ_INFO_FLAG_CAPS;
+			if (info.argsz < sizeof(info) + caps.size) {
+				info.argsz = sizeof(info) + caps.size;
+				info.cap_offset = 0;
+			} else {
+				vfio_info_cap_shift(&caps, sizeof(info));
+				if (copy_to_user((void __user *)arg +
+						  sizeof(info), caps.buf,
+						  caps.size)) {
+					kfree(msi_info);
+					kfree(caps.buf);
+					return -EFAULT;
+				}
+				info.cap_offset = sizeof(info);
+			}
+
+			kfree(msi_info);
+			kfree(caps.buf);
+		}
+
 		return copy_to_user((void __user *)arg, &info, minsz) ?
 			-EFAULT : 0;
 
@@ -366,6 +454,7 @@ static long vfio_platform_ioctl(void *device_data,
 		struct vfio_irq_set hdr;
 		u8 *data = NULL;
 		int ret = 0;
+		int max;
 		size_t data_size = 0;
 
 		minsz = offsetofend(struct vfio_irq_set, count);
@@ -373,8 +462,14 @@ static long vfio_platform_ioctl(void *device_data,
 		if (copy_from_user(&hdr, (void __user *)arg, minsz))
 			return -EFAULT;
 
-		ret = vfio_set_irqs_validate_and_prepare(&hdr, vdev->num_irqs,
-						 vdev->num_irqs, &data_size);
+		if (hdr.index >= vdev->num_irqs)
+			return -EINVAL;
+
+		max = vdev->irqs[hdr.index].count;
+
+		ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
+							 vdev->num_irqs,
+							 &data_size);
 		if (ret)
 			return ret;
 
drivers/vfio/platform/vfio_platform_irq.c +225 -12
index c5b09ec0a3c9..4066223e5b2e 100644
@@ -8,10 +8,12 @@
 
 #include <linux/eventfd.h>
 #include <linux/interrupt.h>
+#include <linux/eventfd.h>
 #include <linux/slab.h>
 #include <linux/types.h>
 #include <linux/vfio.h>
 #include <linux/irq.h>
+#include <linux/msi.h>
 
 #include "vfio_platform_private.h"
 
@@ -253,6 +255,195 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,
 	return 0;
 }
 
+/* MSI/MSIX */
+static irqreturn_t vfio_msihandler(int irq, void *arg)
+{
+	struct eventfd_ctx *trigger = arg;
+
+	eventfd_signal(trigger, 1);
+	return IRQ_HANDLED;
+}
+
+static void msi_write(struct msi_desc *desc, struct msi_msg *msg)
+{
+	int i;
+	struct vfio_platform_irq *irq;
+	u16 index = desc->platform.msi_index;
+	struct device *dev = msi_desc_to_dev(desc);
+	struct vfio_device *device = dev_get_drvdata(dev);
+	struct vfio_platform_device *vdev = (struct vfio_platform_device *)
+						vfio_device_data(device);
+
+	for (i = 0; i < vdev->num_irqs; i++)
+		if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
+			irq = &vdev->irqs[i];
+
+	irq->ctx[index].msg.address_lo = msg->address_lo;
+	irq->ctx[index].msg.address_hi = msg->address_hi;
+	irq->ctx[index].msg.data = msg->data;
+}
+
+static int vfio_msi_enable(struct vfio_platform_device *vdev,
+			   struct vfio_platform_irq *irq, int nvec)
+{
+	int ret;
+	int msi_idx = 0;
+	struct msi_desc *desc;
+	struct device *dev = vdev->device;
+
+	irq->ctx = kcalloc(nvec, sizeof(struct vfio_irq_ctx), GFP_KERNEL);
+	if (!irq->ctx)
+		return -ENOMEM;
+
+	/* Allocate platform MSIs */
+	ret = platform_msi_domain_alloc_irqs(dev, nvec, msi_write);
+	if (ret < 0) {
+		kfree(irq->ctx);
+		return ret;
+	}
+
+	for_each_msi_entry(desc, dev) {
+		irq->ctx[msi_idx].hwirq = desc->irq;
+		msi_idx++;
+	}
+
+	irq->num_ctx = nvec;
+	irq->config_msi = 1;
+
+	return 0;
+}
+
+static int vfio_msi_set_vector_signal(struct vfio_platform_irq *irq,
+				      int vector, int fd)
+{
+	struct eventfd_ctx *trigger;
+	int irq_num, ret;
+
+	if (vector < 0 || vector >= irq->num_ctx)
+		return -EINVAL;
+
+	irq_num = irq->ctx[vector].hwirq;
+
+	if (irq->ctx[vector].trigger) {
+		free_irq(irq_num, irq->ctx[vector].trigger);
+		kfree(irq->ctx[vector].name);
+		eventfd_ctx_put(irq->ctx[vector].trigger);
+		irq->ctx[vector].trigger = NULL;
+	}
+
+	if (fd < 0)
+		return 0;
+
+	irq->ctx[vector].name = kasprintf(GFP_KERNEL,
+					  "vfio-msi[%d]", vector);
+	if (!irq->ctx[vector].name)
+		return -ENOMEM;
+
+	trigger = eventfd_ctx_fdget(fd);
+	if (IS_ERR(trigger)) {
+		kfree(irq->ctx[vector].name);
+		return PTR_ERR(trigger);
+	}
+
+	ret = request_irq(irq_num, vfio_msihandler, 0,
+			  irq->ctx[vector].name, trigger);
+	if (ret) {
+		kfree(irq->ctx[vector].name);
+		eventfd_ctx_put(trigger);
+		return ret;
+	}
+
+	irq->ctx[vector].trigger = trigger;
+
+	return 0;
+}
+
+static int vfio_msi_set_block(struct vfio_platform_irq *irq, unsigned int start,
+			      unsigned int count, int32_t *fds)
+{
+	int i, j, ret = 0;
+
+	if (start >= irq->num_ctx || start + count > irq->num_ctx)
+		return -EINVAL;
+
+	for (i = 0, j = start; i < count && !ret; i++, j++) {
+		int fd = fds ? fds[i] : -1;
+
+		ret = vfio_msi_set_vector_signal(irq, j, fd);
+	}
+
+	if (ret) {
+		for (--j; j >= (int)start; j--)
+			vfio_msi_set_vector_signal(irq, j, -1);
+	}
+
+	return ret;
+}
+
+static void vfio_msi_disable(struct vfio_platform_device *vdev,
+			     struct vfio_platform_irq *irq)
+{
+	struct device *dev = vdev->device;
+
+	vfio_msi_set_block(irq, 0, irq->num_ctx, NULL);
+
+	platform_msi_domain_free_irqs(dev);
+
+	irq->config_msi = 0;
+	irq->num_ctx = 0;
+
+	kfree(irq->ctx);
+}
+
+static int vfio_set_msi_trigger(struct vfio_platform_device *vdev,
+				unsigned int index, unsigned int start,
+				unsigned int count, uint32_t flags, void *data)
+{
+	int i;
+	struct vfio_platform_irq *irq = &vdev->irqs[index];
+
+	if (start + count > irq->count)
+		return -EINVAL;
+
+	if (!count && (flags & VFIO_IRQ_SET_DATA_NONE)) {
+		vfio_msi_disable(vdev, irq);
+		return 0;
+	}
+
+	if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
+		s32 *fds = data;
+		int ret;
+
+		if (irq->config_msi)
+			return vfio_msi_set_block(irq, start, count,
+						  fds);
+		ret = vfio_msi_enable(vdev, irq, start + count);
+		if (ret)
+			return ret;
+
+		ret = vfio_msi_set_block(irq, start, count, fds);
+		if (ret)
+			vfio_msi_disable(vdev, irq);
+
+		return ret;
+	}
+
+	for (i = start; i < start + count; i++) {
+		if (!irq->ctx[i].trigger)
+			continue;
+		if (flags & VFIO_IRQ_SET_DATA_NONE) {
+			eventfd_signal(irq->ctx[i].trigger, 1);
+		} else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
+			u8 *bools = data;
+
+			if (bools[i - start])
+				eventfd_signal(irq->ctx[i].trigger, 1);
+		}
+	}
+
+	return 0;
+}
+
 int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 				 uint32_t flags, unsigned index, unsigned start,
 				 unsigned count, void *data)
@@ -261,16 +452,29 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 		    unsigned start, unsigned count, uint32_t flags,
 		    void *data) = NULL;
 
-	switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
-	case VFIO_IRQ_SET_ACTION_MASK:
-		func = vfio_platform_set_irq_mask;
-		break;
-	case VFIO_IRQ_SET_ACTION_UNMASK:
-		func = vfio_platform_set_irq_unmask;
-		break;
-	case VFIO_IRQ_SET_ACTION_TRIGGER:
-		func = vfio_platform_set_irq_trigger;
-		break;
+	struct vfio_platform_irq *irq = &vdev->irqs[index];
+
+	if (irq->type == VFIO_IRQ_TYPE_MSI) {
+		switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
+		case VFIO_IRQ_SET_ACTION_MASK:
+		case VFIO_IRQ_SET_ACTION_UNMASK:
+			break;
+		case VFIO_IRQ_SET_ACTION_TRIGGER:
+			func = vfio_set_msi_trigger;
+			break;
+		}
+	} else {
+		switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
+		case VFIO_IRQ_SET_ACTION_MASK:
+			func = vfio_platform_set_irq_mask;
+			break;
+		case VFIO_IRQ_SET_ACTION_UNMASK:
+			func = vfio_platform_set_irq_unmask;
+			break;
+		case VFIO_IRQ_SET_ACTION_TRIGGER:
+			func = vfio_platform_set_irq_trigger;
+			break;
+		}
 	}
 
 	if (!func)
@@ -281,12 +485,21 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 
 int vfio_platform_irq_init(struct vfio_platform_device *vdev)
 {
-	int cnt = 0, i;
+	int i;
+	int cnt = 0;
+	int num_irqs;
+	struct device *dev = vdev->device;
 
 	while (vdev->get_irq(vdev, cnt) >= 0)
 		cnt++;
 
-	vdev->irqs = kcalloc(cnt, sizeof(struct vfio_platform_irq), GFP_KERNEL);
+	num_irqs = cnt;
+
+	if (dev->msi_domain)
+		num_irqs++;
+
+	vdev->irqs = kcalloc(num_irqs, sizeof(struct vfio_platform_irq),
+			     GFP_KERNEL);
 	if (!vdev->irqs)
 		return -ENOMEM;
 

drivers/vfio/platform/vfio_platform_irq.c int vfio_platform_irq_init(struct vfio_platform_device *vdev) · line 526

Quoted 6 lines
drivers/vfio/platform/vfio_platform_irq.c +2 -1
@@ -309,7 +522,19 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev)
 		vdev->irqs[i].masked = false;
 	}
 
-	vdev->num_irqs = cnt;
+	/*
+	 * MSI block is added at last index and its an ext irq

it is

drivers/vfio/platform/vfio_platform_irq.c int vfio_platform_irq_init(struct vfio_platform_device *vdev) · line 530

Quoted 4 lines
drivers/vfio/platform/vfio_platform_irq.c +4 -0
+	 */
+	if (dev->msi_domain) {
+		vdev->irqs[i].flags = VFIO_IRQ_INFO_EVENTFD;
+		vdev->irqs[i].count = NR_IRQS;

why NR_IRQS?

Quoted 120 lines
drivers/vfio/platform/vfio_platform_irq.c +14 -2
+		vdev->irqs[i].hwirq = 0;
+		vdev->irqs[i].masked = false;
+		vdev->irqs[i].type = VFIO_IRQ_TYPE_MSI;
+		vdev->num_ext_irqs = 1;
+	}
+
+	vdev->num_irqs = num_irqs;
 
 	return 0;
 err:
@@ -321,8 +546,13 @@ void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev)
 {
 	int i;
 
-	for (i = 0; i < vdev->num_irqs; i++)
-		vfio_set_trigger(vdev, i, -1, NULL);
+	for (i = 0; i < vdev->num_irqs; i++) {
+		if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
+			vfio_set_msi_trigger(vdev, i, 0, 0,
+					     VFIO_IRQ_SET_DATA_NONE, NULL);
+		else
+			vfio_set_trigger(vdev, i, -1, NULL);
+	}
 
 	vdev->num_irqs = 0;
 	kfree(vdev->irqs);
drivers/vfio/platform/vfio_platform_private.h +16 -0
index 289089910643..7bbb05988705 100644
@@ -9,6 +9,7 @@
 
 #include <linux/types.h>
 #include <linux/interrupt.h>
+#include <linux/msi.h>
 
 #define VFIO_PLATFORM_OFFSET_SHIFT   40
 #define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
@@ -19,9 +20,18 @@
 #define VFIO_PLATFORM_INDEX_TO_OFFSET(index)	\
 	((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT)
 
+struct vfio_irq_ctx {
+	int			hwirq;
+	char			*name;
+	struct msi_msg		msg;
+	struct eventfd_ctx	*trigger;
+};
+
 struct vfio_platform_irq {
 	u32			flags;
 	u32			count;
+	int			num_ctx;
+	struct vfio_irq_ctx	*ctx;
 	int			hwirq;
 	char			*name;
 	struct eventfd_ctx	*trigger;
@@ -29,6 +39,11 @@ struct vfio_platform_irq {
 	spinlock_t		lock;
 	struct virqfd		*unmask;
 	struct virqfd		*mask;
+
+	/* for extended irqs */
+	u32			type;
+	u32			subtype;
+	int			config_msi;
 };
 
 struct vfio_platform_region {
@@ -46,6 +61,7 @@ struct vfio_platform_device {
 	u32				num_regions;
 	struct vfio_platform_irq	*irqs;
 	u32				num_irqs;
+	int				num_ext_irqs;
 	int				refcnt;
 	struct mutex			igate;
 	struct module			*parent_module;
include/uapi/linux/vfio.h +30 -0
index 2f313a238a8f..598d1c944283 100644
@@ -697,11 +697,54 @@ struct vfio_irq_info {
 #define VFIO_IRQ_INFO_MASKABLE		(1 << 1)
 #define VFIO_IRQ_INFO_AUTOMASKED	(1 << 2)
 #define VFIO_IRQ_INFO_NORESIZE		(1 << 3)
+#define VFIO_IRQ_INFO_FLAG_CAPS		(1 << 4) /* Info supports caps */
 	__u32	index;		/* IRQ index */
 	__u32	count;		/* Number of IRQs within this index */
+	__u32	cap_offset;	/* Offset within info struct of first cap */
 };
 #define VFIO_DEVICE_GET_IRQ_INFO	_IO(VFIO_TYPE, VFIO_BASE + 9)
 
+/*
+ * The irq type capability allows IRQs unique to a specific device or
+ * class of devices to be exposed.
+ *
+ * The structures below define version 1 of this capability.
+ */
+#define VFIO_IRQ_INFO_CAP_TYPE		3
+
+struct vfio_irq_info_cap_type {
+	struct vfio_info_cap_header header;
+	__u32 type;     /* global per bus driver */
+	__u32 subtype;  /* type specific */
+};
+
+/*
+ * List of IRQ types, global per bus driver.
+ * If you introduce a new type, please add it here.
+ */
+
+/* Non PCI devices having MSI(s) support */
+#define VFIO_IRQ_TYPE_MSI		(1)
+
+/*
+ * The msi capability allows the user to use the msi msg to
+ * configure the iova for the msi configuration.
+ * The structures below define version 1 of this capability.
+ */
+#define VFIO_IRQ_INFO_CAP_MSI_DESCS	4

include/uapi/linux/vfio.h struct vfio_irq_info { · line 744

Quoted 10 lines
include/uapi/linux/vfio.h +10 -0
+
+struct vfio_irq_msi_msg {
+	__u32	addr_lo;
+	__u32	addr_hi;
+	__u32	data;
+};
+
+struct vfio_irq_info_cap_msi {
+	struct vfio_info_cap_header header;
+	__u32 nr_msgs;

I think you should align a __u32 reserved field to have a 64b alignment

include/uapi/linux/vfio.h struct vfio_irq_info { · line 745

Quoted 1 line
include/uapi/linux/vfio.h +1 -0
+	struct vfio_irq_msi_msg msgs[];

Please can you clarify why this cap is needed versus your prior approach.

include/uapi/linux/vfio.h struct vfio_irq_info { · line 750

Quoted 6 lines
include/uapi/linux/vfio.h +2 -0
+};
+
 /**
  * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set)
  *

Thanks

Eric

#28 lore raw

Re: [RFC v2 1/1] vfio/platform: add support for msi

Hi Eric,

Quoted 61 lines · Auger Eric

Hi Vikas,

Quoted 40 lines · Vikas Gupta

MSI support for platform devices.

---
 drivers/vfio/platform/vfio_platform_common.c  |  99 ++++++-
 drivers/vfio/platform/vfio_platform_irq.c     | 260 +++++++++++++++++-
 drivers/vfio/platform/vfio_platform_private.h |  16 ++
 include/uapi/linux/vfio.h                     |  43 +++
 4 files changed, 401 insertions(+), 17 deletions(-)
drivers/vfio/platform/vfio_platform_common.c +8 -0
index c0771a9567fb..b0bfc0f4ee1f 100644
@@ -16,6 +16,7 @@
 #include <linux/types.h>
 #include <linux/uaccess.h>
 #include <linux/vfio.h>
+#include <linux/nospec.h>
 
 #include "vfio_platform_private.h"
 
@@ -344,9 +345,16 @@ static long vfio_platform_ioctl(void *device_data,
 
      } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
              struct vfio_irq_info info;
+             struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
+             struct vfio_irq_info_cap_msi *msi_info = NULL;
+             unsigned long capsz;
+             int ext_irq_index = vdev->num_irqs - vdev->num_ext_irqs;
 
              minsz = offsetofend(struct vfio_irq_info, count);
 
+             /* For backward compatibility, cannot require this */
+             capsz = offsetofend(struct vfio_irq_info, cap_offset);
+
              if (copy_from_user(&info, (void __user *)arg, minsz))
                      return -EFAULT;
 

drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 370

Quoted 7 lines
drivers/vfio/platform/vfio_platform_common.c +4 -0
@@ -356,9 +364,89 @@ static long vfio_platform_ioctl(void *device_data,
              if (info.index >= vdev->num_irqs)
                      return -EINVAL;
 
+             if (info.argsz >= capsz)
+                     minsz = capsz;
+
+             if (info.index == ext_irq_index) {

nit: n case we add new ext indices afterwards, I would check info.index
- ext_irq_index against an VFIO_EXT_IRQ_MSI define.

drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 378

Quoted 8 lines
drivers/vfio/platform/vfio_platform_common.c +8 -0
+                     struct vfio_irq_info_cap_type cap_type = {
+                             .header.id = VFIO_IRQ_INFO_CAP_TYPE,
+                             .header.version = 1 };
+                     int i;
+                     int ret;
+                     int num_msgs;
+                     size_t msi_info_size;
+                     struct vfio_platform_irq *irq;
Quoted 2 lines

nit: I think generally the opposite order (length) is chosen. This also would better match the existing style in this file

Ok will fix it

Quoted 8 lines

drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 385

Quoted 7 lines
drivers/vfio/platform/vfio_platform_common.c +7 -0
+
+                     info.index = array_index_nospec(info.index,
+                                                     vdev->num_irqs);
+
+                     irq = &vdev->irqs[info.index];
+
+                     info.flags = irq->flags;

I think this can be removed given [*]

Sure.

Quoted 11 lines

drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 395

Quoted 10 lines
drivers/vfio/platform/vfio_platform_common.c +10 -0
+                     cap_type.type = irq->type;
+                     cap_type.subtype = irq->subtype;
+
+                     ret = vfio_info_add_capability(&caps,
+                                                    &cap_type.header,
+                                                    sizeof(cap_type));
+                     if (ret)
+                             return ret;
+
+                     num_msgs = irq->num_ctx;

do would want to return the cap even if !num_ctx?

If num_ctx = 0 then VFIO_IRQ_INFO_CAP_MSI_DESCS should not be written. I`ll take care of same.

Quoted 24 lines
Quoted 24 lines
drivers/vfio/platform/vfio_platform_common.c +24 -0
+
+                     msi_info_size = struct_size(msi_info, msgs, num_msgs);
+
+                     msi_info = kzalloc(msi_info_size, GFP_KERNEL);
+                     if (!msi_info) {
+                             kfree(caps.buf);
+                             return -ENOMEM;
+                     }
+
+                     msi_info->header.id = VFIO_IRQ_INFO_CAP_MSI_DESCS;
+                     msi_info->header.version = 1;
+                     msi_info->nr_msgs = num_msgs;
+
+                     for (i = 0; i < num_msgs; i++) {
+                             struct vfio_irq_ctx *ctx = &irq->ctx[i];
+
+                             msi_info->msgs[i].addr_lo = ctx->msg.address_lo;
+                             msi_info->msgs[i].addr_hi = ctx->msg.address_hi;
+                             msi_info->msgs[i].data = ctx->msg.data;
+                     }
+
+                     ret = vfio_info_add_capability(&caps, &msi_info->header,
+                                                    msi_info_size);
+                     if (ret) {
Quoted 8 lines

drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 426

Quoted 7 lines
drivers/vfio/platform/vfio_platform_common.c +6 -0
+                             kfree(msi_info);
+                             kfree(caps.buf);
+                             return ret;
+                     }
+             }
+
              info.flags = vdev->irqs[info.index].flags;

[*]

Will fix it.

Quoted 336 lines
Quoted 329 lines
drivers/vfio/platform/vfio_platform_common.c +30 -2
              info.count = vdev->irqs[info.index].count;
 
+             if (caps.size) {
+                     info.flags |= VFIO_IRQ_INFO_FLAG_CAPS;
+                     if (info.argsz < sizeof(info) + caps.size) {
+                             info.argsz = sizeof(info) + caps.size;
+                             info.cap_offset = 0;
+                     } else {
+                             vfio_info_cap_shift(&caps, sizeof(info));
+                             if (copy_to_user((void __user *)arg +
+                                               sizeof(info), caps.buf,
+                                               caps.size)) {
+                                     kfree(msi_info);
+                                     kfree(caps.buf);
+                                     return -EFAULT;
+                             }
+                             info.cap_offset = sizeof(info);
+                     }
+
+                     kfree(msi_info);
+                     kfree(caps.buf);
+             }
+
              return copy_to_user((void __user *)arg, &info, minsz) ?
                      -EFAULT : 0;
 
@@ -366,6 +454,7 @@ static long vfio_platform_ioctl(void *device_data,
              struct vfio_irq_set hdr;
              u8 *data = NULL;
              int ret = 0;
+             int max;
              size_t data_size = 0;
 
              minsz = offsetofend(struct vfio_irq_set, count);
@@ -373,8 +462,14 @@ static long vfio_platform_ioctl(void *device_data,
              if (copy_from_user(&hdr, (void __user *)arg, minsz))
                      return -EFAULT;
 
-             ret = vfio_set_irqs_validate_and_prepare(&hdr, vdev->num_irqs,
-                                              vdev->num_irqs, &data_size);
+             if (hdr.index >= vdev->num_irqs)
+                     return -EINVAL;
+
+             max = vdev->irqs[hdr.index].count;
+
+             ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
+                                                      vdev->num_irqs,
+                                                      &data_size);
              if (ret)
                      return ret;
 
drivers/vfio/platform/vfio_platform_irq.c +225 -12
index c5b09ec0a3c9..4066223e5b2e 100644
@@ -8,10 +8,12 @@
 
 #include <linux/eventfd.h>
 #include <linux/interrupt.h>
+#include <linux/eventfd.h>
 #include <linux/slab.h>
 #include <linux/types.h>
 #include <linux/vfio.h>
 #include <linux/irq.h>
+#include <linux/msi.h>
 
 #include "vfio_platform_private.h"
 
@@ -253,6 +255,195 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,
      return 0;
 }
 
+/* MSI/MSIX */
+static irqreturn_t vfio_msihandler(int irq, void *arg)
+{
+     struct eventfd_ctx *trigger = arg;
+
+     eventfd_signal(trigger, 1);
+     return IRQ_HANDLED;
+}
+
+static void msi_write(struct msi_desc *desc, struct msi_msg *msg)
+{
+     int i;
+     struct vfio_platform_irq *irq;
+     u16 index = desc->platform.msi_index;
+     struct device *dev = msi_desc_to_dev(desc);
+     struct vfio_device *device = dev_get_drvdata(dev);
+     struct vfio_platform_device *vdev = (struct vfio_platform_device *)
+                                             vfio_device_data(device);
+
+     for (i = 0; i < vdev->num_irqs; i++)
+             if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
+                     irq = &vdev->irqs[i];
+
+     irq->ctx[index].msg.address_lo = msg->address_lo;
+     irq->ctx[index].msg.address_hi = msg->address_hi;
+     irq->ctx[index].msg.data = msg->data;
+}
+
+static int vfio_msi_enable(struct vfio_platform_device *vdev,
+                        struct vfio_platform_irq *irq, int nvec)
+{
+     int ret;
+     int msi_idx = 0;
+     struct msi_desc *desc;
+     struct device *dev = vdev->device;
+
+     irq->ctx = kcalloc(nvec, sizeof(struct vfio_irq_ctx), GFP_KERNEL);
+     if (!irq->ctx)
+             return -ENOMEM;
+
+     /* Allocate platform MSIs */
+     ret = platform_msi_domain_alloc_irqs(dev, nvec, msi_write);
+     if (ret < 0) {
+             kfree(irq->ctx);
+             return ret;
+     }
+
+     for_each_msi_entry(desc, dev) {
+             irq->ctx[msi_idx].hwirq = desc->irq;
+             msi_idx++;
+     }
+
+     irq->num_ctx = nvec;
+     irq->config_msi = 1;
+
+     return 0;
+}
+
+static int vfio_msi_set_vector_signal(struct vfio_platform_irq *irq,
+                                   int vector, int fd)
+{
+     struct eventfd_ctx *trigger;
+     int irq_num, ret;
+
+     if (vector < 0 || vector >= irq->num_ctx)
+             return -EINVAL;
+
+     irq_num = irq->ctx[vector].hwirq;
+
+     if (irq->ctx[vector].trigger) {
+             free_irq(irq_num, irq->ctx[vector].trigger);
+             kfree(irq->ctx[vector].name);
+             eventfd_ctx_put(irq->ctx[vector].trigger);
+             irq->ctx[vector].trigger = NULL;
+     }
+
+     if (fd < 0)
+             return 0;
+
+     irq->ctx[vector].name = kasprintf(GFP_KERNEL,
+                                       "vfio-msi[%d]", vector);
+     if (!irq->ctx[vector].name)
+             return -ENOMEM;
+
+     trigger = eventfd_ctx_fdget(fd);
+     if (IS_ERR(trigger)) {
+             kfree(irq->ctx[vector].name);
+             return PTR_ERR(trigger);
+     }
+
+     ret = request_irq(irq_num, vfio_msihandler, 0,
+                       irq->ctx[vector].name, trigger);
+     if (ret) {
+             kfree(irq->ctx[vector].name);
+             eventfd_ctx_put(trigger);
+             return ret;
+     }
+
+     irq->ctx[vector].trigger = trigger;
+
+     return 0;
+}
+
+static int vfio_msi_set_block(struct vfio_platform_irq *irq, unsigned int start,
+                           unsigned int count, int32_t *fds)
+{
+     int i, j, ret = 0;
+
+     if (start >= irq->num_ctx || start + count > irq->num_ctx)
+             return -EINVAL;
+
+     for (i = 0, j = start; i < count && !ret; i++, j++) {
+             int fd = fds ? fds[i] : -1;
+
+             ret = vfio_msi_set_vector_signal(irq, j, fd);
+     }
+
+     if (ret) {
+             for (--j; j >= (int)start; j--)
+                     vfio_msi_set_vector_signal(irq, j, -1);
+     }
+
+     return ret;
+}
+
+static void vfio_msi_disable(struct vfio_platform_device *vdev,
+                          struct vfio_platform_irq *irq)
+{
+     struct device *dev = vdev->device;
+
+     vfio_msi_set_block(irq, 0, irq->num_ctx, NULL);
+
+     platform_msi_domain_free_irqs(dev);
+
+     irq->config_msi = 0;
+     irq->num_ctx = 0;
+
+     kfree(irq->ctx);
+}
+
+static int vfio_set_msi_trigger(struct vfio_platform_device *vdev,
+                             unsigned int index, unsigned int start,
+                             unsigned int count, uint32_t flags, void *data)
+{
+     int i;
+     struct vfio_platform_irq *irq = &vdev->irqs[index];
+
+     if (start + count > irq->count)
+             return -EINVAL;
+
+     if (!count && (flags & VFIO_IRQ_SET_DATA_NONE)) {
+             vfio_msi_disable(vdev, irq);
+             return 0;
+     }
+
+     if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
+             s32 *fds = data;
+             int ret;
+
+             if (irq->config_msi)
+                     return vfio_msi_set_block(irq, start, count,
+                                               fds);
+             ret = vfio_msi_enable(vdev, irq, start + count);
+             if (ret)
+                     return ret;
+
+             ret = vfio_msi_set_block(irq, start, count, fds);
+             if (ret)
+                     vfio_msi_disable(vdev, irq);
+
+             return ret;
+     }
+
+     for (i = start; i < start + count; i++) {
+             if (!irq->ctx[i].trigger)
+                     continue;
+             if (flags & VFIO_IRQ_SET_DATA_NONE) {
+                     eventfd_signal(irq->ctx[i].trigger, 1);
+             } else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
+                     u8 *bools = data;
+
+                     if (bools[i - start])
+                             eventfd_signal(irq->ctx[i].trigger, 1);
+             }
+     }
+
+     return 0;
+}
+
 int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
                               uint32_t flags, unsigned index, unsigned start,
                               unsigned count, void *data)
@@ -261,16 +452,29 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
                  unsigned start, unsigned count, uint32_t flags,
                  void *data) = NULL;
 
-     switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
-     case VFIO_IRQ_SET_ACTION_MASK:
-             func = vfio_platform_set_irq_mask;
-             break;
-     case VFIO_IRQ_SET_ACTION_UNMASK:
-             func = vfio_platform_set_irq_unmask;
-             break;
-     case VFIO_IRQ_SET_ACTION_TRIGGER:
-             func = vfio_platform_set_irq_trigger;
-             break;
+     struct vfio_platform_irq *irq = &vdev->irqs[index];
+
+     if (irq->type == VFIO_IRQ_TYPE_MSI) {
+             switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
+             case VFIO_IRQ_SET_ACTION_MASK:
+             case VFIO_IRQ_SET_ACTION_UNMASK:
+                     break;
+             case VFIO_IRQ_SET_ACTION_TRIGGER:
+                     func = vfio_set_msi_trigger;
+                     break;
+             }
+     } else {
+             switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
+             case VFIO_IRQ_SET_ACTION_MASK:
+                     func = vfio_platform_set_irq_mask;
+                     break;
+             case VFIO_IRQ_SET_ACTION_UNMASK:
+                     func = vfio_platform_set_irq_unmask;
+                     break;
+             case VFIO_IRQ_SET_ACTION_TRIGGER:
+                     func = vfio_platform_set_irq_trigger;
+                     break;
+             }
      }
 
      if (!func)
@@ -281,12 +485,21 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 
 int vfio_platform_irq_init(struct vfio_platform_device *vdev)
 {
-     int cnt = 0, i;
+     int i;
+     int cnt = 0;
+     int num_irqs;
+     struct device *dev = vdev->device;
 
      while (vdev->get_irq(vdev, cnt) >= 0)
              cnt++;
 
-     vdev->irqs = kcalloc(cnt, sizeof(struct vfio_platform_irq), GFP_KERNEL);
+     num_irqs = cnt;
+
+     if (dev->msi_domain)
+             num_irqs++;
+
+     vdev->irqs = kcalloc(num_irqs, sizeof(struct vfio_platform_irq),
+                          GFP_KERNEL);
      if (!vdev->irqs)
              return -ENOMEM;
 

drivers/vfio/platform/vfio_platform_irq.c int vfio_platform_irq_init(struct vfio_platform_device *vdev) · line 526

Quoted 6 lines
drivers/vfio/platform/vfio_platform_irq.c +2 -1
@@ -309,7 +522,19 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev)
              vdev->irqs[i].masked = false;
      }
 
-     vdev->num_irqs = cnt;
+     /*
+      * MSI block is added at last index and its an ext irq

it is

Quoted 5 lines

drivers/vfio/platform/vfio_platform_irq.c int vfio_platform_irq_init(struct vfio_platform_device *vdev) · line 530

Quoted 4 lines
drivers/vfio/platform/vfio_platform_irq.c +4 -0
+      */
+     if (dev->msi_domain) {
+             vdev->irqs[i].flags = VFIO_IRQ_INFO_EVENTFD;
+             vdev->irqs[i].count = NR_IRQS;

why NR_IRQS?

Since different devices can have different numbers of MSI(s) so we
need to initialize with max possible values. Can you please suggest if this value does not seem appropriate?

Quoted 123 lines
Quoted 123 lines
drivers/vfio/platform/vfio_platform_irq.c +14 -2
+             vdev->irqs[i].hwirq = 0;
+             vdev->irqs[i].masked = false;
+             vdev->irqs[i].type = VFIO_IRQ_TYPE_MSI;
+             vdev->num_ext_irqs = 1;
+     }
+
+     vdev->num_irqs = num_irqs;
 
      return 0;
 err:
@@ -321,8 +546,13 @@ void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev)
 {
      int i;
 
-     for (i = 0; i < vdev->num_irqs; i++)
-             vfio_set_trigger(vdev, i, -1, NULL);
+     for (i = 0; i < vdev->num_irqs; i++) {
+             if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
+                     vfio_set_msi_trigger(vdev, i, 0, 0,
+                                          VFIO_IRQ_SET_DATA_NONE, NULL);
+             else
+                     vfio_set_trigger(vdev, i, -1, NULL);
+     }
 
      vdev->num_irqs = 0;
      kfree(vdev->irqs);
drivers/vfio/platform/vfio_platform_private.h +16 -0
index 289089910643..7bbb05988705 100644
@@ -9,6 +9,7 @@
 
 #include <linux/types.h>
 #include <linux/interrupt.h>
+#include <linux/msi.h>
 
 #define VFIO_PLATFORM_OFFSET_SHIFT   40
 #define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
@@ -19,9 +20,18 @@
 #define VFIO_PLATFORM_INDEX_TO_OFFSET(index) \
      ((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT)
 
+struct vfio_irq_ctx {
+     int                     hwirq;
+     char                    *name;
+     struct msi_msg          msg;
+     struct eventfd_ctx      *trigger;
+};
+
 struct vfio_platform_irq {
      u32                     flags;
      u32                     count;
+     int                     num_ctx;
+     struct vfio_irq_ctx     *ctx;
      int                     hwirq;
      char                    *name;
      struct eventfd_ctx      *trigger;
@@ -29,6 +39,11 @@ struct vfio_platform_irq {
      spinlock_t              lock;
      struct virqfd           *unmask;
      struct virqfd           *mask;
+
+     /* for extended irqs */
+     u32                     type;
+     u32                     subtype;
+     int                     config_msi;
 };
 
 struct vfio_platform_region {
@@ -46,6 +61,7 @@ struct vfio_platform_device {
      u32                             num_regions;
      struct vfio_platform_irq        *irqs;
      u32                             num_irqs;
+     int                             num_ext_irqs;
      int                             refcnt;
      struct mutex                    igate;
      struct module                   *parent_module;
include/uapi/linux/vfio.h +33 -0
index 2f313a238a8f..598d1c944283 100644
@@ -697,11 +697,54 @@ struct vfio_irq_info {
 #define VFIO_IRQ_INFO_MASKABLE               (1 << 1)
 #define VFIO_IRQ_INFO_AUTOMASKED     (1 << 2)
 #define VFIO_IRQ_INFO_NORESIZE               (1 << 3)
+#define VFIO_IRQ_INFO_FLAG_CAPS              (1 << 4) /* Info supports caps */
      __u32   index;          /* IRQ index */
      __u32   count;          /* Number of IRQs within this index */
+     __u32   cap_offset;     /* Offset within info struct of first cap */
 };
 #define VFIO_DEVICE_GET_IRQ_INFO     _IO(VFIO_TYPE, VFIO_BASE + 9)
 
+/*
+ * The irq type capability allows IRQs unique to a specific device or
+ * class of devices to be exposed.
+ *
+ * The structures below define version 1 of this capability.
+ */
+#define VFIO_IRQ_INFO_CAP_TYPE               3
+
+struct vfio_irq_info_cap_type {
+     struct vfio_info_cap_header header;
+     __u32 type;     /* global per bus driver */
+     __u32 subtype;  /* type specific */
+};
+
+/*
+ * List of IRQ types, global per bus driver.
+ * If you introduce a new type, please add it here.
+ */
+
+/* Non PCI devices having MSI(s) support */
+#define VFIO_IRQ_TYPE_MSI            (1)
+
+/*
+ * The msi capability allows the user to use the msi msg to
+ * configure the iova for the msi configuration.
+ * The structures below define version 1 of this capability.
+ */
+#define VFIO_IRQ_INFO_CAP_MSI_DESCS  4
+
+struct vfio_irq_msi_msg {
+     __u32   addr_lo;
Quoted 8 lines

include/uapi/linux/vfio.h struct vfio_irq_info { · line 744

Quoted 7 lines
include/uapi/linux/vfio.h +7 -0
+     __u32   addr_hi;
+     __u32   data;
+};
+
+struct vfio_irq_info_cap_msi {
+     struct vfio_info_cap_header header;
+     __u32 nr_msgs;

I think you should align a __u32 reserved field to have a 64b alignment

Sure.

Quoted 2 lines

include/uapi/linux/vfio.h struct vfio_irq_info { · line 745

Quoted 1 line
include/uapi/linux/vfio.h +1 -0
+     struct vfio_irq_msi_msg msgs[];

Please can you clarify why this cap is needed versus your prior approach.

In the previous patch, the reset module was configuring the device with MSI msg/data now since the module is not available, user space needs to have this data.
With this approach userspace just needs the pairs <msg and ctx > associated with the MSI(s) and it can choose to configure the MSI(s) sources accordingly.
Let me know if this approach does not look appropriate.

Thanks,
Vikas

Quoted 10 lines

include/uapi/linux/vfio.h struct vfio_irq_info { · line 750

Quoted 6 lines
include/uapi/linux/vfio.h +2 -0
+};
+
 /**
  * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set)
  *

Thanks

Eric

maintainer
#29 lore raw

Re: [RFC v2 1/1] vfio/platform: add support for msi

Hi Vikas,

Quoted 458 lines · Vikas Gupta

Hi Eric,

Quoted 61 lines · Auger Eric

Hi Vikas,

Quoted 40 lines · Vikas Gupta

MSI support for platform devices.

---
 drivers/vfio/platform/vfio_platform_common.c  |  99 ++++++-
 drivers/vfio/platform/vfio_platform_irq.c     | 260 +++++++++++++++++-
 drivers/vfio/platform/vfio_platform_private.h |  16 ++
 include/uapi/linux/vfio.h                     |  43 +++
 4 files changed, 401 insertions(+), 17 deletions(-)
drivers/vfio/platform/vfio_platform_common.c +8 -0
index c0771a9567fb..b0bfc0f4ee1f 100644
@@ -16,6 +16,7 @@
 #include <linux/types.h>
 #include <linux/uaccess.h>
 #include <linux/vfio.h>
+#include <linux/nospec.h>
 
 #include "vfio_platform_private.h"
 
@@ -344,9 +345,16 @@ static long vfio_platform_ioctl(void *device_data,
 
      } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
              struct vfio_irq_info info;
+             struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
+             struct vfio_irq_info_cap_msi *msi_info = NULL;
+             unsigned long capsz;
+             int ext_irq_index = vdev->num_irqs - vdev->num_ext_irqs;
 
              minsz = offsetofend(struct vfio_irq_info, count);
 
+             /* For backward compatibility, cannot require this */
+             capsz = offsetofend(struct vfio_irq_info, cap_offset);
+
              if (copy_from_user(&info, (void __user *)arg, minsz))
                      return -EFAULT;
 

drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 370

Quoted 7 lines
drivers/vfio/platform/vfio_platform_common.c +4 -0
@@ -356,9 +364,89 @@ static long vfio_platform_ioctl(void *device_data,
              if (info.index >= vdev->num_irqs)
                      return -EINVAL;
 
+             if (info.argsz >= capsz)
+                     minsz = capsz;
+
+             if (info.index == ext_irq_index) {

nit: n case we add new ext indices afterwards, I would check info.index
- ext_irq_index against an VFIO_EXT_IRQ_MSI define.

drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 378

Quoted 8 lines
drivers/vfio/platform/vfio_platform_common.c +8 -0
+                     struct vfio_irq_info_cap_type cap_type = {
+                             .header.id = VFIO_IRQ_INFO_CAP_TYPE,
+                             .header.version = 1 };
+                     int i;
+                     int ret;
+                     int num_msgs;
+                     size_t msi_info_size;
+                     struct vfio_platform_irq *irq;
Quoted 2 lines

nit: I think generally the opposite order (length) is chosen. This also would better match the existing style in this file

Ok will fix it

Quoted 8 lines

drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 385

Quoted 7 lines
drivers/vfio/platform/vfio_platform_common.c +7 -0
+
+                     info.index = array_index_nospec(info.index,
+                                                     vdev->num_irqs);
+
+                     irq = &vdev->irqs[info.index];
+
+                     info.flags = irq->flags;

I think this can be removed given [*]

Sure.

Quoted 11 lines

drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 395

Quoted 10 lines
drivers/vfio/platform/vfio_platform_common.c +10 -0
+                     cap_type.type = irq->type;
+                     cap_type.subtype = irq->subtype;
+
+                     ret = vfio_info_add_capability(&caps,
+                                                    &cap_type.header,
+                                                    sizeof(cap_type));
+                     if (ret)
+                             return ret;
+
+                     num_msgs = irq->num_ctx;

do would want to return the cap even if !num_ctx?

If num_ctx = 0 then VFIO_IRQ_INFO_CAP_MSI_DESCS should not be written. I`ll take care of same.

Quoted 24 lines
Quoted 24 lines
drivers/vfio/platform/vfio_platform_common.c +24 -0
+
+                     msi_info_size = struct_size(msi_info, msgs, num_msgs);
+
+                     msi_info = kzalloc(msi_info_size, GFP_KERNEL);
+                     if (!msi_info) {
+                             kfree(caps.buf);
+                             return -ENOMEM;
+                     }
+
+                     msi_info->header.id = VFIO_IRQ_INFO_CAP_MSI_DESCS;
+                     msi_info->header.version = 1;
+                     msi_info->nr_msgs = num_msgs;
+
+                     for (i = 0; i < num_msgs; i++) {
+                             struct vfio_irq_ctx *ctx = &irq->ctx[i];
+
+                             msi_info->msgs[i].addr_lo = ctx->msg.address_lo;
+                             msi_info->msgs[i].addr_hi = ctx->msg.address_hi;
+                             msi_info->msgs[i].data = ctx->msg.data;
+                     }
+
+                     ret = vfio_info_add_capability(&caps, &msi_info->header,
+                                                    msi_info_size);
+                     if (ret) {
Quoted 8 lines

drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 426

Quoted 7 lines
drivers/vfio/platform/vfio_platform_common.c +6 -0
+                             kfree(msi_info);
+                             kfree(caps.buf);
+                             return ret;
+                     }
+             }
+
              info.flags = vdev->irqs[info.index].flags;

[*]

Will fix it.

Quoted 336 lines
Quoted 329 lines
drivers/vfio/platform/vfio_platform_common.c +30 -2
              info.count = vdev->irqs[info.index].count;
 
+             if (caps.size) {
+                     info.flags |= VFIO_IRQ_INFO_FLAG_CAPS;
+                     if (info.argsz < sizeof(info) + caps.size) {
+                             info.argsz = sizeof(info) + caps.size;
+                             info.cap_offset = 0;
+                     } else {
+                             vfio_info_cap_shift(&caps, sizeof(info));
+                             if (copy_to_user((void __user *)arg +
+                                               sizeof(info), caps.buf,
+                                               caps.size)) {
+                                     kfree(msi_info);
+                                     kfree(caps.buf);
+                                     return -EFAULT;
+                             }
+                             info.cap_offset = sizeof(info);
+                     }
+
+                     kfree(msi_info);
+                     kfree(caps.buf);
+             }
+
              return copy_to_user((void __user *)arg, &info, minsz) ?
                      -EFAULT : 0;
 
@@ -366,6 +454,7 @@ static long vfio_platform_ioctl(void *device_data,
              struct vfio_irq_set hdr;
              u8 *data = NULL;
              int ret = 0;
+             int max;
              size_t data_size = 0;
 
              minsz = offsetofend(struct vfio_irq_set, count);
@@ -373,8 +462,14 @@ static long vfio_platform_ioctl(void *device_data,
              if (copy_from_user(&hdr, (void __user *)arg, minsz))
                      return -EFAULT;
 
-             ret = vfio_set_irqs_validate_and_prepare(&hdr, vdev->num_irqs,
-                                              vdev->num_irqs, &data_size);
+             if (hdr.index >= vdev->num_irqs)
+                     return -EINVAL;
+
+             max = vdev->irqs[hdr.index].count;
+
+             ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
+                                                      vdev->num_irqs,
+                                                      &data_size);
              if (ret)
                      return ret;
 
drivers/vfio/platform/vfio_platform_irq.c +225 -12
index c5b09ec0a3c9..4066223e5b2e 100644
@@ -8,10 +8,12 @@
 
 #include <linux/eventfd.h>
 #include <linux/interrupt.h>
+#include <linux/eventfd.h>
 #include <linux/slab.h>
 #include <linux/types.h>
 #include <linux/vfio.h>
 #include <linux/irq.h>
+#include <linux/msi.h>
 
 #include "vfio_platform_private.h"
 
@@ -253,6 +255,195 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,
      return 0;
 }
 
+/* MSI/MSIX */
+static irqreturn_t vfio_msihandler(int irq, void *arg)
+{
+     struct eventfd_ctx *trigger = arg;
+
+     eventfd_signal(trigger, 1);
+     return IRQ_HANDLED;
+}
+
+static void msi_write(struct msi_desc *desc, struct msi_msg *msg)
+{
+     int i;
+     struct vfio_platform_irq *irq;
+     u16 index = desc->platform.msi_index;
+     struct device *dev = msi_desc_to_dev(desc);
+     struct vfio_device *device = dev_get_drvdata(dev);
+     struct vfio_platform_device *vdev = (struct vfio_platform_device *)
+                                             vfio_device_data(device);
+
+     for (i = 0; i < vdev->num_irqs; i++)
+             if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
+                     irq = &vdev->irqs[i];
+
+     irq->ctx[index].msg.address_lo = msg->address_lo;
+     irq->ctx[index].msg.address_hi = msg->address_hi;
+     irq->ctx[index].msg.data = msg->data;
+}
+
+static int vfio_msi_enable(struct vfio_platform_device *vdev,
+                        struct vfio_platform_irq *irq, int nvec)
+{
+     int ret;
+     int msi_idx = 0;
+     struct msi_desc *desc;
+     struct device *dev = vdev->device;
+
+     irq->ctx = kcalloc(nvec, sizeof(struct vfio_irq_ctx), GFP_KERNEL);
+     if (!irq->ctx)
+             return -ENOMEM;
+
+     /* Allocate platform MSIs */
+     ret = platform_msi_domain_alloc_irqs(dev, nvec, msi_write);
+     if (ret < 0) {
+             kfree(irq->ctx);
+             return ret;
+     }
+
+     for_each_msi_entry(desc, dev) {
+             irq->ctx[msi_idx].hwirq = desc->irq;
+             msi_idx++;
+     }
+
+     irq->num_ctx = nvec;
+     irq->config_msi = 1;
+
+     return 0;
+}
+
+static int vfio_msi_set_vector_signal(struct vfio_platform_irq *irq,
+                                   int vector, int fd)
+{
+     struct eventfd_ctx *trigger;
+     int irq_num, ret;
+
+     if (vector < 0 || vector >= irq->num_ctx)
+             return -EINVAL;
+
+     irq_num = irq->ctx[vector].hwirq;
+
+     if (irq->ctx[vector].trigger) {
+             free_irq(irq_num, irq->ctx[vector].trigger);
+             kfree(irq->ctx[vector].name);
+             eventfd_ctx_put(irq->ctx[vector].trigger);
+             irq->ctx[vector].trigger = NULL;
+     }
+
+     if (fd < 0)
+             return 0;
+
+     irq->ctx[vector].name = kasprintf(GFP_KERNEL,
+                                       "vfio-msi[%d]", vector);
+     if (!irq->ctx[vector].name)
+             return -ENOMEM;
+
+     trigger = eventfd_ctx_fdget(fd);
+     if (IS_ERR(trigger)) {
+             kfree(irq->ctx[vector].name);
+             return PTR_ERR(trigger);
+     }
+
+     ret = request_irq(irq_num, vfio_msihandler, 0,
+                       irq->ctx[vector].name, trigger);
+     if (ret) {
+             kfree(irq->ctx[vector].name);
+             eventfd_ctx_put(trigger);
+             return ret;
+     }
+
+     irq->ctx[vector].trigger = trigger;
+
+     return 0;
+}
+
+static int vfio_msi_set_block(struct vfio_platform_irq *irq, unsigned int start,
+                           unsigned int count, int32_t *fds)
+{
+     int i, j, ret = 0;
+
+     if (start >= irq->num_ctx || start + count > irq->num_ctx)
+             return -EINVAL;
+
+     for (i = 0, j = start; i < count && !ret; i++, j++) {
+             int fd = fds ? fds[i] : -1;
+
+             ret = vfio_msi_set_vector_signal(irq, j, fd);
+     }
+
+     if (ret) {
+             for (--j; j >= (int)start; j--)
+                     vfio_msi_set_vector_signal(irq, j, -1);
+     }
+
+     return ret;
+}
+
+static void vfio_msi_disable(struct vfio_platform_device *vdev,
+                          struct vfio_platform_irq *irq)
+{
+     struct device *dev = vdev->device;
+
+     vfio_msi_set_block(irq, 0, irq->num_ctx, NULL);
+
+     platform_msi_domain_free_irqs(dev);
+
+     irq->config_msi = 0;
+     irq->num_ctx = 0;
+
+     kfree(irq->ctx);
+}
+
+static int vfio_set_msi_trigger(struct vfio_platform_device *vdev,
+                             unsigned int index, unsigned int start,
+                             unsigned int count, uint32_t flags, void *data)
+{
+     int i;
+     struct vfio_platform_irq *irq = &vdev->irqs[index];
+
+     if (start + count > irq->count)
+             return -EINVAL;
+
+     if (!count && (flags & VFIO_IRQ_SET_DATA_NONE)) {
+             vfio_msi_disable(vdev, irq);
+             return 0;
+     }
+
+     if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
+             s32 *fds = data;
+             int ret;
+
+             if (irq->config_msi)
+                     return vfio_msi_set_block(irq, start, count,
+                                               fds);
+             ret = vfio_msi_enable(vdev, irq, start + count);
+             if (ret)
+                     return ret;
+
+             ret = vfio_msi_set_block(irq, start, count, fds);
+             if (ret)
+                     vfio_msi_disable(vdev, irq);
+
+             return ret;
+     }
+
+     for (i = start; i < start + count; i++) {
+             if (!irq->ctx[i].trigger)
+                     continue;
+             if (flags & VFIO_IRQ_SET_DATA_NONE) {
+                     eventfd_signal(irq->ctx[i].trigger, 1);
+             } else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
+                     u8 *bools = data;
+
+                     if (bools[i - start])
+                             eventfd_signal(irq->ctx[i].trigger, 1);
+             }
+     }
+
+     return 0;
+}
+
 int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
                               uint32_t flags, unsigned index, unsigned start,
                               unsigned count, void *data)
@@ -261,16 +452,29 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
                  unsigned start, unsigned count, uint32_t flags,
                  void *data) = NULL;
 
-     switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
-     case VFIO_IRQ_SET_ACTION_MASK:
-             func = vfio_platform_set_irq_mask;
-             break;
-     case VFIO_IRQ_SET_ACTION_UNMASK:
-             func = vfio_platform_set_irq_unmask;
-             break;
-     case VFIO_IRQ_SET_ACTION_TRIGGER:
-             func = vfio_platform_set_irq_trigger;
-             break;
+     struct vfio_platform_irq *irq = &vdev->irqs[index];
+
+     if (irq->type == VFIO_IRQ_TYPE_MSI) {
+             switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
+             case VFIO_IRQ_SET_ACTION_MASK:
+             case VFIO_IRQ_SET_ACTION_UNMASK:
+                     break;
+             case VFIO_IRQ_SET_ACTION_TRIGGER:
+                     func = vfio_set_msi_trigger;
+                     break;
+             }
+     } else {
+             switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
+             case VFIO_IRQ_SET_ACTION_MASK:
+                     func = vfio_platform_set_irq_mask;
+                     break;
+             case VFIO_IRQ_SET_ACTION_UNMASK:
+                     func = vfio_platform_set_irq_unmask;
+                     break;
+             case VFIO_IRQ_SET_ACTION_TRIGGER:
+                     func = vfio_platform_set_irq_trigger;
+                     break;
+             }
      }
 
      if (!func)
@@ -281,12 +485,21 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 
 int vfio_platform_irq_init(struct vfio_platform_device *vdev)
 {
-     int cnt = 0, i;
+     int i;
+     int cnt = 0;
+     int num_irqs;
+     struct device *dev = vdev->device;
 
      while (vdev->get_irq(vdev, cnt) >= 0)
              cnt++;
 
-     vdev->irqs = kcalloc(cnt, sizeof(struct vfio_platform_irq), GFP_KERNEL);
+     num_irqs = cnt;
+
+     if (dev->msi_domain)
+             num_irqs++;
+
+     vdev->irqs = kcalloc(num_irqs, sizeof(struct vfio_platform_irq),
+                          GFP_KERNEL);
      if (!vdev->irqs)
              return -ENOMEM;
 

drivers/vfio/platform/vfio_platform_irq.c int vfio_platform_irq_init(struct vfio_platform_device *vdev) · line 526

Quoted 6 lines
drivers/vfio/platform/vfio_platform_irq.c +2 -1
@@ -309,7 +522,19 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev)
              vdev->irqs[i].masked = false;
      }
 
-     vdev->num_irqs = cnt;
+     /*
+      * MSI block is added at last index and its an ext irq

it is

Quoted 8 lines
Quoted 5 lines

drivers/vfio/platform/vfio_platform_irq.c int vfio_platform_irq_init(struct vfio_platform_device *vdev) · line 530

Quoted 4 lines
drivers/vfio/platform/vfio_platform_irq.c +4 -0
+      */
+     if (dev->msi_domain) {
+             vdev->irqs[i].flags = VFIO_IRQ_INFO_EVENTFD;
+             vdev->irqs[i].count = NR_IRQS;

why NR_IRQS?

Since different devices can have different numbers of MSI(s) so we
need to initialize with max possible values. Can you please suggest if this value does not seem appropriate?

As opposed to PCIe, the userspace has no real way to guess how many vectors can be set (what vfio_pci_get_irq_count does). This also means we do not fully implement the original API as we are not able to report an accurate value for .count. How will the user determine how many vectors he can use?

Quoted 133 lines
Quoted 123 lines
Quoted 123 lines
drivers/vfio/platform/vfio_platform_irq.c +14 -2
+             vdev->irqs[i].hwirq = 0;
+             vdev->irqs[i].masked = false;
+             vdev->irqs[i].type = VFIO_IRQ_TYPE_MSI;
+             vdev->num_ext_irqs = 1;
+     }
+
+     vdev->num_irqs = num_irqs;
 
      return 0;
 err:
@@ -321,8 +546,13 @@ void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev)
 {
      int i;
 
-     for (i = 0; i < vdev->num_irqs; i++)
-             vfio_set_trigger(vdev, i, -1, NULL);
+     for (i = 0; i < vdev->num_irqs; i++) {
+             if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
+                     vfio_set_msi_trigger(vdev, i, 0, 0,
+                                          VFIO_IRQ_SET_DATA_NONE, NULL);
+             else
+                     vfio_set_trigger(vdev, i, -1, NULL);
+     }
 
      vdev->num_irqs = 0;
      kfree(vdev->irqs);
drivers/vfio/platform/vfio_platform_private.h +16 -0
index 289089910643..7bbb05988705 100644
@@ -9,6 +9,7 @@
 
 #include <linux/types.h>
 #include <linux/interrupt.h>
+#include <linux/msi.h>
 
 #define VFIO_PLATFORM_OFFSET_SHIFT   40
 #define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
@@ -19,9 +20,18 @@
 #define VFIO_PLATFORM_INDEX_TO_OFFSET(index) \
      ((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT)
 
+struct vfio_irq_ctx {
+     int                     hwirq;
+     char                    *name;
+     struct msi_msg          msg;
+     struct eventfd_ctx      *trigger;
+};
+
 struct vfio_platform_irq {
      u32                     flags;
      u32                     count;
+     int                     num_ctx;
+     struct vfio_irq_ctx     *ctx;
      int                     hwirq;
      char                    *name;
      struct eventfd_ctx      *trigger;
@@ -29,6 +39,11 @@ struct vfio_platform_irq {
      spinlock_t              lock;
      struct virqfd           *unmask;
      struct virqfd           *mask;
+
+     /* for extended irqs */
+     u32                     type;
+     u32                     subtype;
+     int                     config_msi;
 };
 
 struct vfio_platform_region {
@@ -46,6 +61,7 @@ struct vfio_platform_device {
      u32                             num_regions;
      struct vfio_platform_irq        *irqs;
      u32                             num_irqs;
+     int                             num_ext_irqs;
      int                             refcnt;
      struct mutex                    igate;
      struct module                   *parent_module;
include/uapi/linux/vfio.h +33 -0
index 2f313a238a8f..598d1c944283 100644
@@ -697,11 +697,54 @@ struct vfio_irq_info {
 #define VFIO_IRQ_INFO_MASKABLE               (1 << 1)
 #define VFIO_IRQ_INFO_AUTOMASKED     (1 << 2)
 #define VFIO_IRQ_INFO_NORESIZE               (1 << 3)
+#define VFIO_IRQ_INFO_FLAG_CAPS              (1 << 4) /* Info supports caps */
      __u32   index;          /* IRQ index */
      __u32   count;          /* Number of IRQs within this index */
+     __u32   cap_offset;     /* Offset within info struct of first cap */
 };
 #define VFIO_DEVICE_GET_IRQ_INFO     _IO(VFIO_TYPE, VFIO_BASE + 9)
 
+/*
+ * The irq type capability allows IRQs unique to a specific device or
+ * class of devices to be exposed.
+ *
+ * The structures below define version 1 of this capability.
+ */
+#define VFIO_IRQ_INFO_CAP_TYPE               3
+
+struct vfio_irq_info_cap_type {
+     struct vfio_info_cap_header header;
+     __u32 type;     /* global per bus driver */
+     __u32 subtype;  /* type specific */
+};
+
+/*
+ * List of IRQ types, global per bus driver.
+ * If you introduce a new type, please add it here.
+ */
+
+/* Non PCI devices having MSI(s) support */
+#define VFIO_IRQ_TYPE_MSI            (1)
+
+/*
+ * The msi capability allows the user to use the msi msg to
+ * configure the iova for the msi configuration.
+ * The structures below define version 1 of this capability.
+ */
+#define VFIO_IRQ_INFO_CAP_MSI_DESCS  4
+
+struct vfio_irq_msi_msg {
+     __u32   addr_lo;
Quoted 8 lines

include/uapi/linux/vfio.h struct vfio_irq_info { · line 744

Quoted 7 lines
include/uapi/linux/vfio.h +7 -0
+     __u32   addr_hi;
+     __u32   data;
+};
+
+struct vfio_irq_info_cap_msi {
+     struct vfio_info_cap_header header;
+     __u32 nr_msgs;

I think you should align a __u32 reserved field to have a 64b alignment

Sure.

Quoted 1 line
Quoted 1 line
include/uapi/linux/vfio.h +1 -0
+     struct vfio_irq_msi_msg msgs[];
Quoted 8 lines
Quoted 1 line

Please can you clarify why this cap is needed versus your prior approach.

In the previous patch, the reset module was configuring the device with MSI msg/data now since the module is not available, user space needs to have this data.
With this approach userspace just needs the pairs <msg and ctx > associated with the MSI(s) and it can choose to configure the MSI(s) sources accordingly.
Let me know if this approach does not look appropriate.

This comes to the question I asked in my previous email, ie. could you give some more info about the expected MSI setup sequence? May be the opportunity to enhance the commit message ;-)

Thanks

Eric

Quoted 13 lines

Thanks,
Vikas

Quoted 10 lines

include/uapi/linux/vfio.h struct vfio_irq_info { · line 750

Quoted 6 lines
include/uapi/linux/vfio.h +2 -0
+};
+
 /**
  * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set)
  *

Thanks

Eric

#30 lore raw

Re: [RFC v2 1/1] vfio/platform: add support for msi

HI Eric,

Quoted 467 lines · Auger Eric

Hi Vikas,

Quoted 463 lines · Vikas Gupta

Hi Eric,

Quoted 61 lines · Auger Eric

Hi Vikas,

Quoted 40 lines · Vikas Gupta

MSI support for platform devices.

---
 drivers/vfio/platform/vfio_platform_common.c  |  99 ++++++-
 drivers/vfio/platform/vfio_platform_irq.c     | 260 +++++++++++++++++-
 drivers/vfio/platform/vfio_platform_private.h |  16 ++
 include/uapi/linux/vfio.h                     |  43 +++
 4 files changed, 401 insertions(+), 17 deletions(-)
drivers/vfio/platform/vfio_platform_common.c +8 -0
index c0771a9567fb..b0bfc0f4ee1f 100644
@@ -16,6 +16,7 @@
 #include <linux/types.h>
 #include <linux/uaccess.h>
 #include <linux/vfio.h>
+#include <linux/nospec.h>
 
 #include "vfio_platform_private.h"
 
@@ -344,9 +345,16 @@ static long vfio_platform_ioctl(void *device_data,
 
      } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
              struct vfio_irq_info info;
+             struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
+             struct vfio_irq_info_cap_msi *msi_info = NULL;
+             unsigned long capsz;
+             int ext_irq_index = vdev->num_irqs - vdev->num_ext_irqs;
 
              minsz = offsetofend(struct vfio_irq_info, count);
 
+             /* For backward compatibility, cannot require this */
+             capsz = offsetofend(struct vfio_irq_info, cap_offset);
+
              if (copy_from_user(&info, (void __user *)arg, minsz))
                      return -EFAULT;
 

drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 370

Quoted 7 lines
drivers/vfio/platform/vfio_platform_common.c +4 -0
@@ -356,9 +364,89 @@ static long vfio_platform_ioctl(void *device_data,
              if (info.index >= vdev->num_irqs)
                      return -EINVAL;
 
+             if (info.argsz >= capsz)
+                     minsz = capsz;
+
+             if (info.index == ext_irq_index) {

nit: n case we add new ext indices afterwards, I would check info.index
- ext_irq_index against an VFIO_EXT_IRQ_MSI define.

drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 378

Quoted 8 lines
drivers/vfio/platform/vfio_platform_common.c +8 -0
+                     struct vfio_irq_info_cap_type cap_type = {
+                             .header.id = VFIO_IRQ_INFO_CAP_TYPE,
+                             .header.version = 1 };
+                     int i;
+                     int ret;
+                     int num_msgs;
+                     size_t msi_info_size;
+                     struct vfio_platform_irq *irq;
Quoted 2 lines

nit: I think generally the opposite order (length) is chosen. This also would better match the existing style in this file

Ok will fix it

Quoted 8 lines

drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 385

Quoted 7 lines
drivers/vfio/platform/vfio_platform_common.c +7 -0
+
+                     info.index = array_index_nospec(info.index,
+                                                     vdev->num_irqs);
+
+                     irq = &vdev->irqs[info.index];
+
+                     info.flags = irq->flags;

I think this can be removed given [*]

Sure.

Quoted 11 lines

drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 395

Quoted 10 lines
drivers/vfio/platform/vfio_platform_common.c +10 -0
+                     cap_type.type = irq->type;
+                     cap_type.subtype = irq->subtype;
+
+                     ret = vfio_info_add_capability(&caps,
+                                                    &cap_type.header,
+                                                    sizeof(cap_type));
+                     if (ret)
+                             return ret;
+
+                     num_msgs = irq->num_ctx;

do would want to return the cap even if !num_ctx?

If num_ctx = 0 then VFIO_IRQ_INFO_CAP_MSI_DESCS should not be written. I`ll take care of same.

Quoted 24 lines
Quoted 24 lines
drivers/vfio/platform/vfio_platform_common.c +24 -0
+
+                     msi_info_size = struct_size(msi_info, msgs, num_msgs);
+
+                     msi_info = kzalloc(msi_info_size, GFP_KERNEL);
+                     if (!msi_info) {
+                             kfree(caps.buf);
+                             return -ENOMEM;
+                     }
+
+                     msi_info->header.id = VFIO_IRQ_INFO_CAP_MSI_DESCS;
+                     msi_info->header.version = 1;
+                     msi_info->nr_msgs = num_msgs;
+
+                     for (i = 0; i < num_msgs; i++) {
+                             struct vfio_irq_ctx *ctx = &irq->ctx[i];
+
+                             msi_info->msgs[i].addr_lo = ctx->msg.address_lo;
+                             msi_info->msgs[i].addr_hi = ctx->msg.address_hi;
+                             msi_info->msgs[i].data = ctx->msg.data;
+                     }
+
+                     ret = vfio_info_add_capability(&caps, &msi_info->header,
+                                                    msi_info_size);
+                     if (ret) {
Quoted 8 lines

drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 426

Quoted 7 lines
drivers/vfio/platform/vfio_platform_common.c +6 -0
+                             kfree(msi_info);
+                             kfree(caps.buf);
+                             return ret;
+                     }
+             }
+
              info.flags = vdev->irqs[info.index].flags;

[*]

Will fix it.

Quoted 341 lines
Quoted 329 lines
drivers/vfio/platform/vfio_platform_common.c +30 -2
              info.count = vdev->irqs[info.index].count;
 
+             if (caps.size) {
+                     info.flags |= VFIO_IRQ_INFO_FLAG_CAPS;
+                     if (info.argsz < sizeof(info) + caps.size) {
+                             info.argsz = sizeof(info) + caps.size;
+                             info.cap_offset = 0;
+                     } else {
+                             vfio_info_cap_shift(&caps, sizeof(info));
+                             if (copy_to_user((void __user *)arg +
+                                               sizeof(info), caps.buf,
+                                               caps.size)) {
+                                     kfree(msi_info);
+                                     kfree(caps.buf);
+                                     return -EFAULT;
+                             }
+                             info.cap_offset = sizeof(info);
+                     }
+
+                     kfree(msi_info);
+                     kfree(caps.buf);
+             }
+
              return copy_to_user((void __user *)arg, &info, minsz) ?
                      -EFAULT : 0;
 
@@ -366,6 +454,7 @@ static long vfio_platform_ioctl(void *device_data,
              struct vfio_irq_set hdr;
              u8 *data = NULL;
              int ret = 0;
+             int max;
              size_t data_size = 0;
 
              minsz = offsetofend(struct vfio_irq_set, count);
@@ -373,8 +462,14 @@ static long vfio_platform_ioctl(void *device_data,
              if (copy_from_user(&hdr, (void __user *)arg, minsz))
                      return -EFAULT;
 
-             ret = vfio_set_irqs_validate_and_prepare(&hdr, vdev->num_irqs,
-                                              vdev->num_irqs, &data_size);
+             if (hdr.index >= vdev->num_irqs)
+                     return -EINVAL;
+
+             max = vdev->irqs[hdr.index].count;
+
+             ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
+                                                      vdev->num_irqs,
+                                                      &data_size);
              if (ret)
                      return ret;
 
drivers/vfio/platform/vfio_platform_irq.c +225 -12
index c5b09ec0a3c9..4066223e5b2e 100644
@@ -8,10 +8,12 @@
 
 #include <linux/eventfd.h>
 #include <linux/interrupt.h>
+#include <linux/eventfd.h>
 #include <linux/slab.h>
 #include <linux/types.h>
 #include <linux/vfio.h>
 #include <linux/irq.h>
+#include <linux/msi.h>
 
 #include "vfio_platform_private.h"
 
@@ -253,6 +255,195 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,
      return 0;
 }
 
+/* MSI/MSIX */
+static irqreturn_t vfio_msihandler(int irq, void *arg)
+{
+     struct eventfd_ctx *trigger = arg;
+
+     eventfd_signal(trigger, 1);
+     return IRQ_HANDLED;
+}
+
+static void msi_write(struct msi_desc *desc, struct msi_msg *msg)
+{
+     int i;
+     struct vfio_platform_irq *irq;
+     u16 index = desc->platform.msi_index;
+     struct device *dev = msi_desc_to_dev(desc);
+     struct vfio_device *device = dev_get_drvdata(dev);
+     struct vfio_platform_device *vdev = (struct vfio_platform_device *)
+                                             vfio_device_data(device);
+
+     for (i = 0; i < vdev->num_irqs; i++)
+             if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
+                     irq = &vdev->irqs[i];
+
+     irq->ctx[index].msg.address_lo = msg->address_lo;
+     irq->ctx[index].msg.address_hi = msg->address_hi;
+     irq->ctx[index].msg.data = msg->data;
+}
+
+static int vfio_msi_enable(struct vfio_platform_device *vdev,
+                        struct vfio_platform_irq *irq, int nvec)
+{
+     int ret;
+     int msi_idx = 0;
+     struct msi_desc *desc;
+     struct device *dev = vdev->device;
+
+     irq->ctx = kcalloc(nvec, sizeof(struct vfio_irq_ctx), GFP_KERNEL);
+     if (!irq->ctx)
+             return -ENOMEM;
+
+     /* Allocate platform MSIs */
+     ret = platform_msi_domain_alloc_irqs(dev, nvec, msi_write);
+     if (ret < 0) {
+             kfree(irq->ctx);
+             return ret;
+     }
+
+     for_each_msi_entry(desc, dev) {
+             irq->ctx[msi_idx].hwirq = desc->irq;
+             msi_idx++;
+     }
+
+     irq->num_ctx = nvec;
+     irq->config_msi = 1;
+
+     return 0;
+}
+
+static int vfio_msi_set_vector_signal(struct vfio_platform_irq *irq,
+                                   int vector, int fd)
+{
+     struct eventfd_ctx *trigger;
+     int irq_num, ret;
+
+     if (vector < 0 || vector >= irq->num_ctx)
+             return -EINVAL;
+
+     irq_num = irq->ctx[vector].hwirq;
+
+     if (irq->ctx[vector].trigger) {
+             free_irq(irq_num, irq->ctx[vector].trigger);
+             kfree(irq->ctx[vector].name);
+             eventfd_ctx_put(irq->ctx[vector].trigger);
+             irq->ctx[vector].trigger = NULL;
+     }
+
+     if (fd < 0)
+             return 0;
+
+     irq->ctx[vector].name = kasprintf(GFP_KERNEL,
+                                       "vfio-msi[%d]", vector);
+     if (!irq->ctx[vector].name)
+             return -ENOMEM;
+
+     trigger = eventfd_ctx_fdget(fd);
+     if (IS_ERR(trigger)) {
+             kfree(irq->ctx[vector].name);
+             return PTR_ERR(trigger);
+     }
+
+     ret = request_irq(irq_num, vfio_msihandler, 0,
+                       irq->ctx[vector].name, trigger);
+     if (ret) {
+             kfree(irq->ctx[vector].name);
+             eventfd_ctx_put(trigger);
+             return ret;
+     }
+
+     irq->ctx[vector].trigger = trigger;
+
+     return 0;
+}
+
+static int vfio_msi_set_block(struct vfio_platform_irq *irq, unsigned int start,
+                           unsigned int count, int32_t *fds)
+{
+     int i, j, ret = 0;
+
+     if (start >= irq->num_ctx || start + count > irq->num_ctx)
+             return -EINVAL;
+
+     for (i = 0, j = start; i < count && !ret; i++, j++) {
+             int fd = fds ? fds[i] : -1;
+
+             ret = vfio_msi_set_vector_signal(irq, j, fd);
+     }
+
+     if (ret) {
+             for (--j; j >= (int)start; j--)
+                     vfio_msi_set_vector_signal(irq, j, -1);
+     }
+
+     return ret;
+}
+
+static void vfio_msi_disable(struct vfio_platform_device *vdev,
+                          struct vfio_platform_irq *irq)
+{
+     struct device *dev = vdev->device;
+
+     vfio_msi_set_block(irq, 0, irq->num_ctx, NULL);
+
+     platform_msi_domain_free_irqs(dev);
+
+     irq->config_msi = 0;
+     irq->num_ctx = 0;
+
+     kfree(irq->ctx);
+}
+
+static int vfio_set_msi_trigger(struct vfio_platform_device *vdev,
+                             unsigned int index, unsigned int start,
+                             unsigned int count, uint32_t flags, void *data)
+{
+     int i;
+     struct vfio_platform_irq *irq = &vdev->irqs[index];
+
+     if (start + count > irq->count)
+             return -EINVAL;
+
+     if (!count && (flags & VFIO_IRQ_SET_DATA_NONE)) {
+             vfio_msi_disable(vdev, irq);
+             return 0;
+     }
+
+     if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
+             s32 *fds = data;
+             int ret;
+
+             if (irq->config_msi)
+                     return vfio_msi_set_block(irq, start, count,
+                                               fds);
+             ret = vfio_msi_enable(vdev, irq, start + count);
+             if (ret)
+                     return ret;
+
+             ret = vfio_msi_set_block(irq, start, count, fds);
+             if (ret)
+                     vfio_msi_disable(vdev, irq);
+
+             return ret;
+     }
+
+     for (i = start; i < start + count; i++) {
+             if (!irq->ctx[i].trigger)
+                     continue;
+             if (flags & VFIO_IRQ_SET_DATA_NONE) {
+                     eventfd_signal(irq->ctx[i].trigger, 1);
+             } else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
+                     u8 *bools = data;
+
+                     if (bools[i - start])
+                             eventfd_signal(irq->ctx[i].trigger, 1);
+             }
+     }
+
+     return 0;
+}
+
 int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
                               uint32_t flags, unsigned index, unsigned start,
                               unsigned count, void *data)
@@ -261,16 +452,29 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
                  unsigned start, unsigned count, uint32_t flags,
                  void *data) = NULL;
 
-     switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
-     case VFIO_IRQ_SET_ACTION_MASK:
-             func = vfio_platform_set_irq_mask;
-             break;
-     case VFIO_IRQ_SET_ACTION_UNMASK:
-             func = vfio_platform_set_irq_unmask;
-             break;
-     case VFIO_IRQ_SET_ACTION_TRIGGER:
-             func = vfio_platform_set_irq_trigger;
-             break;
+     struct vfio_platform_irq *irq = &vdev->irqs[index];
+
+     if (irq->type == VFIO_IRQ_TYPE_MSI) {
+             switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
+             case VFIO_IRQ_SET_ACTION_MASK:
+             case VFIO_IRQ_SET_ACTION_UNMASK:
+                     break;
+             case VFIO_IRQ_SET_ACTION_TRIGGER:
+                     func = vfio_set_msi_trigger;
+                     break;
+             }
+     } else {
+             switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
+             case VFIO_IRQ_SET_ACTION_MASK:
+                     func = vfio_platform_set_irq_mask;
+                     break;
+             case VFIO_IRQ_SET_ACTION_UNMASK:
+                     func = vfio_platform_set_irq_unmask;
+                     break;
+             case VFIO_IRQ_SET_ACTION_TRIGGER:
+                     func = vfio_platform_set_irq_trigger;
+                     break;
+             }
      }
 
      if (!func)
@@ -281,12 +485,21 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 
 int vfio_platform_irq_init(struct vfio_platform_device *vdev)
 {
-     int cnt = 0, i;
+     int i;
+     int cnt = 0;
+     int num_irqs;
+     struct device *dev = vdev->device;
 
      while (vdev->get_irq(vdev, cnt) >= 0)
              cnt++;
 
-     vdev->irqs = kcalloc(cnt, sizeof(struct vfio_platform_irq), GFP_KERNEL);
+     num_irqs = cnt;
+
+     if (dev->msi_domain)
+             num_irqs++;
+
+     vdev->irqs = kcalloc(num_irqs, sizeof(struct vfio_platform_irq),
+                          GFP_KERNEL);
      if (!vdev->irqs)
              return -ENOMEM;
 

drivers/vfio/platform/vfio_platform_irq.c int vfio_platform_irq_init(struct vfio_platform_device *vdev) · line 526

Quoted 6 lines
drivers/vfio/platform/vfio_platform_irq.c +2 -1
@@ -309,7 +522,19 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev)
              vdev->irqs[i].masked = false;
      }
 
-     vdev->num_irqs = cnt;
+     /*
+      * MSI block is added at last index and its an ext irq

it is

drivers/vfio/platform/vfio_platform_irq.c int vfio_platform_irq_init(struct vfio_platform_device *vdev) · line 530

Quoted 4 lines
drivers/vfio/platform/vfio_platform_irq.c +4 -0
+      */
+     if (dev->msi_domain) {
+             vdev->irqs[i].flags = VFIO_IRQ_INFO_EVENTFD;
+             vdev->irqs[i].count = NR_IRQS;

why NR_IRQS?

Quoted 8 lines
Quoted 3 lines

Since different devices can have different numbers of MSI(s) so we
need to initialize with max possible values. Can you please suggest if this value does not seem appropriate?

As opposed to PCIe, the userspace has no real way to guess how many vectors can be set (what vfio_pci_get_irq_count does). This also means we do not fully implement the original API as we are not able to report an accurate value for .count. How will the user determine how many vectors he can use?

I believe user space will know how many MSIs platform device can have. We have assigned NR_IRQS because that’s what maximum number of interrupt can be supported on a specific platform. In case user space errantly tries to allocate any number of MSIs in kernel platform_msi_domain_alloc_irqs should fail.
Do you think this approach can create an issue as .count is not exactly the same as MSIs are existing with device? Is it necessary to have a PCIe kind of approach as PCIe is standard but platform devices do not have standards?

Quoted 141 lines
Quoted 133 lines
Quoted 123 lines
Quoted 123 lines
drivers/vfio/platform/vfio_platform_irq.c +14 -2
+             vdev->irqs[i].hwirq = 0;
+             vdev->irqs[i].masked = false;
+             vdev->irqs[i].type = VFIO_IRQ_TYPE_MSI;
+             vdev->num_ext_irqs = 1;
+     }
+
+     vdev->num_irqs = num_irqs;
 
      return 0;
 err:
@@ -321,8 +546,13 @@ void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev)
 {
      int i;
 
-     for (i = 0; i < vdev->num_irqs; i++)
-             vfio_set_trigger(vdev, i, -1, NULL);
+     for (i = 0; i < vdev->num_irqs; i++) {
+             if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
+                     vfio_set_msi_trigger(vdev, i, 0, 0,
+                                          VFIO_IRQ_SET_DATA_NONE, NULL);
+             else
+                     vfio_set_trigger(vdev, i, -1, NULL);
+     }
 
      vdev->num_irqs = 0;
      kfree(vdev->irqs);
drivers/vfio/platform/vfio_platform_private.h +16 -0
index 289089910643..7bbb05988705 100644
@@ -9,6 +9,7 @@
 
 #include <linux/types.h>
 #include <linux/interrupt.h>
+#include <linux/msi.h>
 
 #define VFIO_PLATFORM_OFFSET_SHIFT   40
 #define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
@@ -19,9 +20,18 @@
 #define VFIO_PLATFORM_INDEX_TO_OFFSET(index) \
      ((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT)
 
+struct vfio_irq_ctx {
+     int                     hwirq;
+     char                    *name;
+     struct msi_msg          msg;
+     struct eventfd_ctx      *trigger;
+};
+
 struct vfio_platform_irq {
      u32                     flags;
      u32                     count;
+     int                     num_ctx;
+     struct vfio_irq_ctx     *ctx;
      int                     hwirq;
      char                    *name;
      struct eventfd_ctx      *trigger;
@@ -29,6 +39,11 @@ struct vfio_platform_irq {
      spinlock_t              lock;
      struct virqfd           *unmask;
      struct virqfd           *mask;
+
+     /* for extended irqs */
+     u32                     type;
+     u32                     subtype;
+     int                     config_msi;
 };
 
 struct vfio_platform_region {
@@ -46,6 +61,7 @@ struct vfio_platform_device {
      u32                             num_regions;
      struct vfio_platform_irq        *irqs;
      u32                             num_irqs;
+     int                             num_ext_irqs;
      int                             refcnt;
      struct mutex                    igate;
      struct module                   *parent_module;
include/uapi/linux/vfio.h +33 -0
index 2f313a238a8f..598d1c944283 100644
@@ -697,11 +697,54 @@ struct vfio_irq_info {
 #define VFIO_IRQ_INFO_MASKABLE               (1 << 1)
 #define VFIO_IRQ_INFO_AUTOMASKED     (1 << 2)
 #define VFIO_IRQ_INFO_NORESIZE               (1 << 3)
+#define VFIO_IRQ_INFO_FLAG_CAPS              (1 << 4) /* Info supports caps */
      __u32   index;          /* IRQ index */
      __u32   count;          /* Number of IRQs within this index */
+     __u32   cap_offset;     /* Offset within info struct of first cap */
 };
 #define VFIO_DEVICE_GET_IRQ_INFO     _IO(VFIO_TYPE, VFIO_BASE + 9)
 
+/*
+ * The irq type capability allows IRQs unique to a specific device or
+ * class of devices to be exposed.
+ *
+ * The structures below define version 1 of this capability.
+ */
+#define VFIO_IRQ_INFO_CAP_TYPE               3
+
+struct vfio_irq_info_cap_type {
+     struct vfio_info_cap_header header;
+     __u32 type;     /* global per bus driver */
+     __u32 subtype;  /* type specific */
+};
+
+/*
+ * List of IRQ types, global per bus driver.
+ * If you introduce a new type, please add it here.
+ */
+
+/* Non PCI devices having MSI(s) support */
+#define VFIO_IRQ_TYPE_MSI            (1)
+
+/*
+ * The msi capability allows the user to use the msi msg to
+ * configure the iova for the msi configuration.
+ * The structures below define version 1 of this capability.
+ */
+#define VFIO_IRQ_INFO_CAP_MSI_DESCS  4
+
+struct vfio_irq_msi_msg {
+     __u32   addr_lo;
Quoted 8 lines

include/uapi/linux/vfio.h struct vfio_irq_info { · line 744

Quoted 7 lines
include/uapi/linux/vfio.h +7 -0
+     __u32   addr_hi;
+     __u32   data;
+};
+
+struct vfio_irq_info_cap_msi {
+     struct vfio_info_cap_header header;
+     __u32 nr_msgs;

I think you should align a __u32 reserved field to have a 64b alignment

Sure.

Quoted 1 line
Quoted 1 line
include/uapi/linux/vfio.h +1 -0
+     struct vfio_irq_msi_msg msgs[];
Quoted 8 lines
Quoted 1 line

Please can you clarify why this cap is needed versus your prior approach.

In the previous patch, the reset module was configuring the device with MSI msg/data now since the module is not available, user space needs to have this data.
With this approach userspace just needs the pairs <msg and ctx > associated with the MSI(s) and it can choose to configure the MSI(s) sources accordingly.
Let me know if this approach does not look appropriate.

Quoted 3 lines

This comes to the question I asked in my previous email, ie. could you give some more info about the expected MSI setup sequence? May be the opportunity to enhance the commit message ;-)

With our proposal, user space needs to know how many MSI sources there are as we want user space to map msi-msg(s) to specific MSI sources. Let us assume there are max ‘n’ MSI sources a platform device have, user-space requests ‘n’ msi(s) allocation in the kernel which in turn gets ‘n’ msi_msg with help of cap (VFIO_IRQ_INFO_CAP_MSI_DESCS). User space is free to map/configure msi_msg->data value to any particular source. Since msi_msg->data is one to one mapped to eventfd, user-space knows that particular eventfd corresponds to the same MSI source which it has configured with the msi_msg->data value.

Steps:

a)   User space allocates nvec in kernel. Ioctl VFIO_DEVICE_SET_IRQS
b)   Kernel allocates nvec and does associated IRQ allocation, eventfd etc.
c)   User space gets the msi_msgs VFIO_DEVICE_GET_IRQ_INFO.
d)   User space configures the particular MSI source with msi_msg info.

Do these steps look OK?
We can add more information where this cap is introduced in the file uapi/linux/vfio.h.

Thanks,
Vikas

Quoted 18 lines

Thanks

Eric

Quoted 13 lines

Thanks,
Vikas

Quoted 10 lines

include/uapi/linux/vfio.h struct vfio_irq_info { · line 750

Quoted 6 lines
include/uapi/linux/vfio.h +2 -0
+};
+
 /**
  * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set)
  *

Thanks

Eric

Signature · 11 lines
This electronic communication and the information and any files transmitted 
with it, or attached to it, are confidential and are intended solely for 
the use of the individual or entity to whom it is addressed and may contain 
information that is confidential, legally privileged, protected by privacy 
laws, or otherwise restricted from disclosure to anyone else. If you are 
not the intended recipient or the person responsible for delivering the 
e-mail to the intended recipient, you are hereby notified that any use, 
copying, distributing, dissemination, forwarding, printing, or copying of 
this e-mail is strictly prohibited. If you received this e-mail in error, 
please return the e-mail to the sender, delete it from your computer, and 
destroy any printed copy of it.
maintainer
#31 lore raw

Re: [RFC v2 1/1] vfio/platform: add support for msi

Hi Vikas,

Quoted 479 lines · Vikas Gupta

HI Eric,

Quoted 467 lines · Auger Eric

Hi Vikas,

Quoted 463 lines · Vikas Gupta

Hi Eric,

Quoted 61 lines · Auger Eric

Hi Vikas,

Quoted 40 lines · Vikas Gupta

MSI support for platform devices.

---
 drivers/vfio/platform/vfio_platform_common.c  |  99 ++++++-
 drivers/vfio/platform/vfio_platform_irq.c     | 260 +++++++++++++++++-
 drivers/vfio/platform/vfio_platform_private.h |  16 ++
 include/uapi/linux/vfio.h                     |  43 +++
 4 files changed, 401 insertions(+), 17 deletions(-)
drivers/vfio/platform/vfio_platform_common.c +8 -0
index c0771a9567fb..b0bfc0f4ee1f 100644
@@ -16,6 +16,7 @@
 #include <linux/types.h>
 #include <linux/uaccess.h>
 #include <linux/vfio.h>
+#include <linux/nospec.h>
 
 #include "vfio_platform_private.h"
 
@@ -344,9 +345,16 @@ static long vfio_platform_ioctl(void *device_data,
 
      } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
              struct vfio_irq_info info;
+             struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
+             struct vfio_irq_info_cap_msi *msi_info = NULL;
+             unsigned long capsz;
+             int ext_irq_index = vdev->num_irqs - vdev->num_ext_irqs;
 
              minsz = offsetofend(struct vfio_irq_info, count);
 
+             /* For backward compatibility, cannot require this */
+             capsz = offsetofend(struct vfio_irq_info, cap_offset);
+
              if (copy_from_user(&info, (void __user *)arg, minsz))
                      return -EFAULT;
 

drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 370

Quoted 7 lines
drivers/vfio/platform/vfio_platform_common.c +4 -0
@@ -356,9 +364,89 @@ static long vfio_platform_ioctl(void *device_data,
              if (info.index >= vdev->num_irqs)
                      return -EINVAL;
 
+             if (info.argsz >= capsz)
+                     minsz = capsz;
+
+             if (info.index == ext_irq_index) {

nit: n case we add new ext indices afterwards, I would check info.index
- ext_irq_index against an VFIO_EXT_IRQ_MSI define.

drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 378

Quoted 8 lines
drivers/vfio/platform/vfio_platform_common.c +8 -0
+                     struct vfio_irq_info_cap_type cap_type = {
+                             .header.id = VFIO_IRQ_INFO_CAP_TYPE,
+                             .header.version = 1 };
+                     int i;
+                     int ret;
+                     int num_msgs;
+                     size_t msi_info_size;
+                     struct vfio_platform_irq *irq;
Quoted 2 lines

nit: I think generally the opposite order (length) is chosen. This also would better match the existing style in this file

Ok will fix it

Quoted 8 lines

drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 385

Quoted 7 lines
drivers/vfio/platform/vfio_platform_common.c +7 -0
+
+                     info.index = array_index_nospec(info.index,
+                                                     vdev->num_irqs);
+
+                     irq = &vdev->irqs[info.index];
+
+                     info.flags = irq->flags;

I think this can be removed given [*]

Sure.

Quoted 11 lines

drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 395

Quoted 10 lines
drivers/vfio/platform/vfio_platform_common.c +10 -0
+                     cap_type.type = irq->type;
+                     cap_type.subtype = irq->subtype;
+
+                     ret = vfio_info_add_capability(&caps,
+                                                    &cap_type.header,
+                                                    sizeof(cap_type));
+                     if (ret)
+                             return ret;
+
+                     num_msgs = irq->num_ctx;

do would want to return the cap even if !num_ctx?

If num_ctx = 0 then VFIO_IRQ_INFO_CAP_MSI_DESCS should not be written. I`ll take care of same.

Quoted 24 lines
Quoted 24 lines
drivers/vfio/platform/vfio_platform_common.c +24 -0
+
+                     msi_info_size = struct_size(msi_info, msgs, num_msgs);
+
+                     msi_info = kzalloc(msi_info_size, GFP_KERNEL);
+                     if (!msi_info) {
+                             kfree(caps.buf);
+                             return -ENOMEM;
+                     }
+
+                     msi_info->header.id = VFIO_IRQ_INFO_CAP_MSI_DESCS;
+                     msi_info->header.version = 1;
+                     msi_info->nr_msgs = num_msgs;
+
+                     for (i = 0; i < num_msgs; i++) {
+                             struct vfio_irq_ctx *ctx = &irq->ctx[i];
+
+                             msi_info->msgs[i].addr_lo = ctx->msg.address_lo;
+                             msi_info->msgs[i].addr_hi = ctx->msg.address_hi;
+                             msi_info->msgs[i].data = ctx->msg.data;
+                     }
+
+                     ret = vfio_info_add_capability(&caps, &msi_info->header,
+                                                    msi_info_size);
+                     if (ret) {
Quoted 8 lines

drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 426

Quoted 7 lines
drivers/vfio/platform/vfio_platform_common.c +6 -0
+                             kfree(msi_info);
+                             kfree(caps.buf);
+                             return ret;
+                     }
+             }
+
              info.flags = vdev->irqs[info.index].flags;

[*]

Will fix it.

Quoted 341 lines
Quoted 329 lines
drivers/vfio/platform/vfio_platform_common.c +30 -2
              info.count = vdev->irqs[info.index].count;
 
+             if (caps.size) {
+                     info.flags |= VFIO_IRQ_INFO_FLAG_CAPS;
+                     if (info.argsz < sizeof(info) + caps.size) {
+                             info.argsz = sizeof(info) + caps.size;
+                             info.cap_offset = 0;
+                     } else {
+                             vfio_info_cap_shift(&caps, sizeof(info));
+                             if (copy_to_user((void __user *)arg +
+                                               sizeof(info), caps.buf,
+                                               caps.size)) {
+                                     kfree(msi_info);
+                                     kfree(caps.buf);
+                                     return -EFAULT;
+                             }
+                             info.cap_offset = sizeof(info);
+                     }
+
+                     kfree(msi_info);
+                     kfree(caps.buf);
+             }
+
              return copy_to_user((void __user *)arg, &info, minsz) ?
                      -EFAULT : 0;
 
@@ -366,6 +454,7 @@ static long vfio_platform_ioctl(void *device_data,
              struct vfio_irq_set hdr;
              u8 *data = NULL;
              int ret = 0;
+             int max;
              size_t data_size = 0;
 
              minsz = offsetofend(struct vfio_irq_set, count);
@@ -373,8 +462,14 @@ static long vfio_platform_ioctl(void *device_data,
              if (copy_from_user(&hdr, (void __user *)arg, minsz))
                      return -EFAULT;
 
-             ret = vfio_set_irqs_validate_and_prepare(&hdr, vdev->num_irqs,
-                                              vdev->num_irqs, &data_size);
+             if (hdr.index >= vdev->num_irqs)
+                     return -EINVAL;
+
+             max = vdev->irqs[hdr.index].count;
+
+             ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
+                                                      vdev->num_irqs,
+                                                      &data_size);
              if (ret)
                      return ret;
 
drivers/vfio/platform/vfio_platform_irq.c +225 -12
index c5b09ec0a3c9..4066223e5b2e 100644
@@ -8,10 +8,12 @@
 
 #include <linux/eventfd.h>
 #include <linux/interrupt.h>
+#include <linux/eventfd.h>
 #include <linux/slab.h>
 #include <linux/types.h>
 #include <linux/vfio.h>
 #include <linux/irq.h>
+#include <linux/msi.h>
 
 #include "vfio_platform_private.h"
 
@@ -253,6 +255,195 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,
      return 0;
 }
 
+/* MSI/MSIX */
+static irqreturn_t vfio_msihandler(int irq, void *arg)
+{
+     struct eventfd_ctx *trigger = arg;
+
+     eventfd_signal(trigger, 1);
+     return IRQ_HANDLED;
+}
+
+static void msi_write(struct msi_desc *desc, struct msi_msg *msg)
+{
+     int i;
+     struct vfio_platform_irq *irq;
+     u16 index = desc->platform.msi_index;
+     struct device *dev = msi_desc_to_dev(desc);
+     struct vfio_device *device = dev_get_drvdata(dev);
+     struct vfio_platform_device *vdev = (struct vfio_platform_device *)
+                                             vfio_device_data(device);
+
+     for (i = 0; i < vdev->num_irqs; i++)
+             if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
+                     irq = &vdev->irqs[i];
+
+     irq->ctx[index].msg.address_lo = msg->address_lo;
+     irq->ctx[index].msg.address_hi = msg->address_hi;
+     irq->ctx[index].msg.data = msg->data;
+}
+
+static int vfio_msi_enable(struct vfio_platform_device *vdev,
+                        struct vfio_platform_irq *irq, int nvec)
+{
+     int ret;
+     int msi_idx = 0;
+     struct msi_desc *desc;
+     struct device *dev = vdev->device;
+
+     irq->ctx = kcalloc(nvec, sizeof(struct vfio_irq_ctx), GFP_KERNEL);
+     if (!irq->ctx)
+             return -ENOMEM;
+
+     /* Allocate platform MSIs */
+     ret = platform_msi_domain_alloc_irqs(dev, nvec, msi_write);
+     if (ret < 0) {
+             kfree(irq->ctx);
+             return ret;
+     }
+
+     for_each_msi_entry(desc, dev) {
+             irq->ctx[msi_idx].hwirq = desc->irq;
+             msi_idx++;
+     }
+
+     irq->num_ctx = nvec;
+     irq->config_msi = 1;
+
+     return 0;
+}
+
+static int vfio_msi_set_vector_signal(struct vfio_platform_irq *irq,
+                                   int vector, int fd)
+{
+     struct eventfd_ctx *trigger;
+     int irq_num, ret;
+
+     if (vector < 0 || vector >= irq->num_ctx)
+             return -EINVAL;
+
+     irq_num = irq->ctx[vector].hwirq;
+
+     if (irq->ctx[vector].trigger) {
+             free_irq(irq_num, irq->ctx[vector].trigger);
+             kfree(irq->ctx[vector].name);
+             eventfd_ctx_put(irq->ctx[vector].trigger);
+             irq->ctx[vector].trigger = NULL;
+     }
+
+     if (fd < 0)
+             return 0;
+
+     irq->ctx[vector].name = kasprintf(GFP_KERNEL,
+                                       "vfio-msi[%d]", vector);
+     if (!irq->ctx[vector].name)
+             return -ENOMEM;
+
+     trigger = eventfd_ctx_fdget(fd);
+     if (IS_ERR(trigger)) {
+             kfree(irq->ctx[vector].name);
+             return PTR_ERR(trigger);
+     }
+
+     ret = request_irq(irq_num, vfio_msihandler, 0,
+                       irq->ctx[vector].name, trigger);
+     if (ret) {
+             kfree(irq->ctx[vector].name);
+             eventfd_ctx_put(trigger);
+             return ret;
+     }
+
+     irq->ctx[vector].trigger = trigger;
+
+     return 0;
+}
+
+static int vfio_msi_set_block(struct vfio_platform_irq *irq, unsigned int start,
+                           unsigned int count, int32_t *fds)
+{
+     int i, j, ret = 0;
+
+     if (start >= irq->num_ctx || start + count > irq->num_ctx)
+             return -EINVAL;
+
+     for (i = 0, j = start; i < count && !ret; i++, j++) {
+             int fd = fds ? fds[i] : -1;
+
+             ret = vfio_msi_set_vector_signal(irq, j, fd);
+     }
+
+     if (ret) {
+             for (--j; j >= (int)start; j--)
+                     vfio_msi_set_vector_signal(irq, j, -1);
+     }
+
+     return ret;
+}
+
+static void vfio_msi_disable(struct vfio_platform_device *vdev,
+                          struct vfio_platform_irq *irq)
+{
+     struct device *dev = vdev->device;
+
+     vfio_msi_set_block(irq, 0, irq->num_ctx, NULL);
+
+     platform_msi_domain_free_irqs(dev);
+
+     irq->config_msi = 0;
+     irq->num_ctx = 0;
+
+     kfree(irq->ctx);
+}
+
+static int vfio_set_msi_trigger(struct vfio_platform_device *vdev,
+                             unsigned int index, unsigned int start,
+                             unsigned int count, uint32_t flags, void *data)
+{
+     int i;
+     struct vfio_platform_irq *irq = &vdev->irqs[index];
+
+     if (start + count > irq->count)
+             return -EINVAL;
+
+     if (!count && (flags & VFIO_IRQ_SET_DATA_NONE)) {
+             vfio_msi_disable(vdev, irq);
+             return 0;
+     }
+
+     if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
+             s32 *fds = data;
+             int ret;
+
+             if (irq->config_msi)
+                     return vfio_msi_set_block(irq, start, count,
+                                               fds);
+             ret = vfio_msi_enable(vdev, irq, start + count);
+             if (ret)
+                     return ret;
+
+             ret = vfio_msi_set_block(irq, start, count, fds);
+             if (ret)
+                     vfio_msi_disable(vdev, irq);
+
+             return ret;
+     }
+
+     for (i = start; i < start + count; i++) {
+             if (!irq->ctx[i].trigger)
+                     continue;
+             if (flags & VFIO_IRQ_SET_DATA_NONE) {
+                     eventfd_signal(irq->ctx[i].trigger, 1);
+             } else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
+                     u8 *bools = data;
+
+                     if (bools[i - start])
+                             eventfd_signal(irq->ctx[i].trigger, 1);
+             }
+     }
+
+     return 0;
+}
+
 int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
                               uint32_t flags, unsigned index, unsigned start,
                               unsigned count, void *data)
@@ -261,16 +452,29 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
                  unsigned start, unsigned count, uint32_t flags,
                  void *data) = NULL;
 
-     switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
-     case VFIO_IRQ_SET_ACTION_MASK:
-             func = vfio_platform_set_irq_mask;
-             break;
-     case VFIO_IRQ_SET_ACTION_UNMASK:
-             func = vfio_platform_set_irq_unmask;
-             break;
-     case VFIO_IRQ_SET_ACTION_TRIGGER:
-             func = vfio_platform_set_irq_trigger;
-             break;
+     struct vfio_platform_irq *irq = &vdev->irqs[index];
+
+     if (irq->type == VFIO_IRQ_TYPE_MSI) {
+             switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
+             case VFIO_IRQ_SET_ACTION_MASK:
+             case VFIO_IRQ_SET_ACTION_UNMASK:
+                     break;
+             case VFIO_IRQ_SET_ACTION_TRIGGER:
+                     func = vfio_set_msi_trigger;
+                     break;
+             }
+     } else {
+             switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
+             case VFIO_IRQ_SET_ACTION_MASK:
+                     func = vfio_platform_set_irq_mask;
+                     break;
+             case VFIO_IRQ_SET_ACTION_UNMASK:
+                     func = vfio_platform_set_irq_unmask;
+                     break;
+             case VFIO_IRQ_SET_ACTION_TRIGGER:
+                     func = vfio_platform_set_irq_trigger;
+                     break;
+             }
      }
 
      if (!func)
@@ -281,12 +485,21 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 
 int vfio_platform_irq_init(struct vfio_platform_device *vdev)
 {
-     int cnt = 0, i;
+     int i;
+     int cnt = 0;
+     int num_irqs;
+     struct device *dev = vdev->device;
 
      while (vdev->get_irq(vdev, cnt) >= 0)
              cnt++;
 
-     vdev->irqs = kcalloc(cnt, sizeof(struct vfio_platform_irq), GFP_KERNEL);
+     num_irqs = cnt;
+
+     if (dev->msi_domain)
+             num_irqs++;
+
+     vdev->irqs = kcalloc(num_irqs, sizeof(struct vfio_platform_irq),
+                          GFP_KERNEL);
      if (!vdev->irqs)
              return -ENOMEM;
 

drivers/vfio/platform/vfio_platform_irq.c int vfio_platform_irq_init(struct vfio_platform_device *vdev) · line 526

Quoted 6 lines
drivers/vfio/platform/vfio_platform_irq.c +2 -1
@@ -309,7 +522,19 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev)
              vdev->irqs[i].masked = false;
      }
 
-     vdev->num_irqs = cnt;
+     /*
+      * MSI block is added at last index and its an ext irq

it is

drivers/vfio/platform/vfio_platform_irq.c int vfio_platform_irq_init(struct vfio_platform_device *vdev) · line 530

Quoted 4 lines
drivers/vfio/platform/vfio_platform_irq.c +4 -0
+      */
+     if (dev->msi_domain) {
+             vdev->irqs[i].flags = VFIO_IRQ_INFO_EVENTFD;
+             vdev->irqs[i].count = NR_IRQS;

why NR_IRQS?

Quoted 8 lines
Quoted 3 lines

Since different devices can have different numbers of MSI(s) so we
need to initialize with max possible values. Can you please suggest if this value does not seem appropriate?

As opposed to PCIe, the userspace has no real way to guess how many vectors can be set (what vfio_pci_get_irq_count does). This also means we do not fully implement the original API as we are not able to report an accurate value for .count. How will the user determine how many vectors he can use?

Quoted 9 lines

I believe user space will know how many MSIs platform device can have. We have assigned NR_IRQS because that’s what maximum number of interrupt can be supported on a specific platform. In case user space errantly tries to allocate any number of MSIs in kernel platform_msi_domain_alloc_irqs should fail.
Do you think this approach can create an issue as .count is not exactly the same as MSIs are existing with device? Is it necessary to have a PCIe kind of approach as PCIe is standard but platform devices do not have standards?

I had a short discussion about this with Alex and I think we came to the conclusion that is an issue not to be able to report accurately the number of supported vectors. Assuming that the userspace knows the number of vectors does not really look as a valid assumption and does not really fit the vfio spirit.

The previous approach beased on a kernel module allowed to report their actual number. Even if it is heavier it allows to fill the info supposed to be returned by the GET_IRQ ioctl and we think it is better.

Thanks

Eric

Quoted 186 lines
Quoted 142 lines
Quoted 134 lines
Quoted 124 lines
Quoted 124 lines
drivers/vfio/platform/vfio_platform_irq.c +14 -2
+             vdev->irqs[i].hwirq = 0;
+             vdev->irqs[i].masked = false;
+             vdev->irqs[i].type = VFIO_IRQ_TYPE_MSI;
+             vdev->num_ext_irqs = 1;
+     }
+
+     vdev->num_irqs = num_irqs;
 
      return 0;
 err:
@@ -321,8 +546,13 @@ void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev)
 {
      int i;
 
-     for (i = 0; i < vdev->num_irqs; i++)
-             vfio_set_trigger(vdev, i, -1, NULL);
+     for (i = 0; i < vdev->num_irqs; i++) {
+             if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
+                     vfio_set_msi_trigger(vdev, i, 0, 0,
+                                          VFIO_IRQ_SET_DATA_NONE, NULL);
+             else
+                     vfio_set_trigger(vdev, i, -1, NULL);
+     }
 
      vdev->num_irqs = 0;
      kfree(vdev->irqs);
drivers/vfio/platform/vfio_platform_private.h +16 -0
index 289089910643..7bbb05988705 100644
@@ -9,6 +9,7 @@
 
 #include <linux/types.h>
 #include <linux/interrupt.h>
+#include <linux/msi.h>
 
 #define VFIO_PLATFORM_OFFSET_SHIFT   40
 #define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
@@ -19,9 +20,18 @@
 #define VFIO_PLATFORM_INDEX_TO_OFFSET(index) \
      ((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT)
 
+struct vfio_irq_ctx {
+     int                     hwirq;
+     char                    *name;
+     struct msi_msg          msg;
+     struct eventfd_ctx      *trigger;
+};
+
 struct vfio_platform_irq {
      u32                     flags;
      u32                     count;
+     int                     num_ctx;
+     struct vfio_irq_ctx     *ctx;
      int                     hwirq;
      char                    *name;
      struct eventfd_ctx      *trigger;
@@ -29,6 +39,11 @@ struct vfio_platform_irq {
      spinlock_t              lock;
      struct virqfd           *unmask;
      struct virqfd           *mask;
+
+     /* for extended irqs */
+     u32                     type;
+     u32                     subtype;
+     int                     config_msi;
 };
 
 struct vfio_platform_region {
@@ -46,6 +61,7 @@ struct vfio_platform_device {
      u32                             num_regions;
      struct vfio_platform_irq        *irqs;
      u32                             num_irqs;
+     int                             num_ext_irqs;
      int                             refcnt;
      struct mutex                    igate;
      struct module                   *parent_module;
include/uapi/linux/vfio.h +33 -0
index 2f313a238a8f..598d1c944283 100644
@@ -697,11 +697,54 @@ struct vfio_irq_info {
 #define VFIO_IRQ_INFO_MASKABLE               (1 << 1)
 #define VFIO_IRQ_INFO_AUTOMASKED     (1 << 2)
 #define VFIO_IRQ_INFO_NORESIZE               (1 << 3)
+#define VFIO_IRQ_INFO_FLAG_CAPS              (1 << 4) /* Info supports caps */
      __u32   index;          /* IRQ index */
      __u32   count;          /* Number of IRQs within this index */
+     __u32   cap_offset;     /* Offset within info struct of first cap */
 };
 #define VFIO_DEVICE_GET_IRQ_INFO     _IO(VFIO_TYPE, VFIO_BASE + 9)
 
+/*
+ * The irq type capability allows IRQs unique to a specific device or
+ * class of devices to be exposed.
+ *
+ * The structures below define version 1 of this capability.
+ */
+#define VFIO_IRQ_INFO_CAP_TYPE               3
+
+struct vfio_irq_info_cap_type {
+     struct vfio_info_cap_header header;
+     __u32 type;     /* global per bus driver */
+     __u32 subtype;  /* type specific */
+};
+
+/*
+ * List of IRQ types, global per bus driver.
+ * If you introduce a new type, please add it here.
+ */
+
+/* Non PCI devices having MSI(s) support */
+#define VFIO_IRQ_TYPE_MSI            (1)
+
+/*
+ * The msi capability allows the user to use the msi msg to
+ * configure the iova for the msi configuration.
+ * The structures below define version 1 of this capability.
+ */
+#define VFIO_IRQ_INFO_CAP_MSI_DESCS  4
+
+struct vfio_irq_msi_msg {
+     __u32   addr_lo;
Quoted 8 lines

include/uapi/linux/vfio.h struct vfio_irq_info { · line 744

Quoted 7 lines
include/uapi/linux/vfio.h +7 -0
+     __u32   addr_hi;
+     __u32   data;
+};
+
+struct vfio_irq_info_cap_msi {
+     struct vfio_info_cap_header header;
+     __u32 nr_msgs;

I think you should align a __u32 reserved field to have a 64b alignment

Sure.

Quoted 1 line
Quoted 1 line
include/uapi/linux/vfio.h +1 -0
+     struct vfio_irq_msi_msg msgs[];
Quoted 8 lines
Quoted 1 line

Please can you clarify why this cap is needed versus your prior approach.

In the previous patch, the reset module was configuring the device with MSI msg/data now since the module is not available, user space needs to have this data.
With this approach userspace just needs the pairs <msg and ctx > associated with the MSI(s) and it can choose to configure the MSI(s) sources accordingly.
Let me know if this approach does not look appropriate.

Quoted 3 lines

This comes to the question I asked in my previous email, ie. could you give some more info about the expected MSI setup sequence? May be the opportunity to enhance the commit message ;-)

With our proposal, user space needs to know how many MSI sources there are as we want user space to map msi-msg(s) to specific MSI sources. Let us assume there are max ‘n’ MSI sources a platform device have, user-space requests ‘n’ msi(s) allocation in the kernel which in turn gets ‘n’ msi_msg with help of cap (VFIO_IRQ_INFO_CAP_MSI_DESCS). User space is free to map/configure msi_msg->data value to any particular source. Since msi_msg->data is one to one mapped to eventfd, user-space knows that particular eventfd corresponds to the same MSI source which it has configured with the msi_msg->data value.

Steps:

a)   User space allocates nvec in kernel. Ioctl VFIO_DEVICE_SET_IRQS
b)   Kernel allocates nvec and does associated IRQ allocation, eventfd etc.
c)   User space gets the msi_msgs VFIO_DEVICE_GET_IRQ_INFO.
d)   User space configures the particular MSI source with msi_msg info.

Do these steps look OK?
We can add more information where this cap is introduced in the file uapi/linux/vfio.h.

Thanks,
Vikas

Quoted 18 lines

Thanks

Eric

Quoted 13 lines

Thanks,
Vikas

Quoted 10 lines

include/uapi/linux/vfio.h struct vfio_irq_info { · line 750

Quoted 6 lines
include/uapi/linux/vfio.h +2 -0
+};
+
 /**
  * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set)
  *

Thanks

Eric

maintainer
#32 lore raw
In reply to Vikas Gupta

Hi Vikas,

Quoted 4 lines · Vikas Gupta

This RFC adds support for MSI for platform devices. MSI block is added as an ext irq along with the existing wired interrupt implementation.

Quoted 8 lines

Changes from:
-------------
v1 to v2:

	1) IRQ allocation has been implemented as below:
	       ----------------------------
	       |IRQ-0|IRQ-1|....|IRQ-n|MSI|
       	       ----------------------------
		MSI block has msi contexts and its implemneted

it is implemented

Quoted 9 lines

as ext irq.

2) Removed vendor specific module for msi handling so previously patch2 and patch3 are not required.

3) MSI related data is exported to userspace using 'caps'.
Please note VFIO_IRQ_INFO_CAP_TYPE in include/uapi/linux/vfio.h implementation
is taken from the Eric`s patch
https://patchwork.kernel.org/project/kvm/patch/[email protected]/

So do you mean that by exposing the vectors, now you do not need the msi module anymore?

Thanks

Eric

Quoted 23 lines

v0 to v1:
i) Removed MSI device flag VFIO_DEVICE_FLAGS_MSI. ii) Add MSI(s) at the end of the irq list of platform IRQs. MSI(s) with first entry of MSI block has count and flag information.
IRQ list: Allocation for IRQs + MSIs are allocated as below
Example: if there are 'n' IRQs and 'k' MSIs
-------------------------------------------------------
|IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
-------------------------------------------------------
MSI-0 will have count=k set and flags set accordingly.

Vikas Gupta (1):
vfio/platform: add support for msi

 drivers/vfio/platform/vfio_platform_common.c  |  99 ++++++-
 drivers/vfio/platform/vfio_platform_irq.c     | 260 +++++++++++++++++-
 drivers/vfio/platform/vfio_platform_private.h |  16 ++
 include/uapi/linux/vfio.h                     |  43 +++
 4 files changed, 401 insertions(+), 17 deletions(-)
#33 lore raw

HI Eric,

Quoted 21 lines · Auger Eric

Hi Vikas,

Quoted 4 lines · Vikas Gupta

This RFC adds support for MSI for platform devices. MSI block is added as an ext irq along with the existing wired interrupt implementation.

Quoted 8 lines

Changes from:
-------------
v1 to v2:

      1) IRQ allocation has been implemented as below:
             ----------------------------
             |IRQ-0|IRQ-1|....|IRQ-n|MSI|
                     ----------------------------
              MSI block has msi contexts and its implemneted

it is implemented

Quoted 5 lines

as ext irq.

2) Removed vendor specific module for msi handling so previously patch2 and patch3 are not required.

Quoted 6 lines
Quoted 4 lines

3) MSI related data is exported to userspace using 'caps'.
Please note VFIO_IRQ_INFO_CAP_TYPE in include/uapi/linux/vfio.h implementation
is taken from the Eric`s patch
https://patchwork.kernel.org/project/kvm/patch/[email protected]/

So do you mean that by exposing the vectors, now you do not need the msi module anymore?

Yes, with the support of caps we can expose the MSI related data to userspace and userspace can configure the device, which previous patches were doing in the kernel module.

Thanks,
Vikas

Quoted 29 lines

Thanks

Eric

Quoted 23 lines

v0 to v1:
i) Removed MSI device flag VFIO_DEVICE_FLAGS_MSI. ii) Add MSI(s) at the end of the irq list of platform IRQs. MSI(s) with first entry of MSI block has count and flag information.
IRQ list: Allocation for IRQs + MSIs are allocated as below
Example: if there are 'n' IRQs and 'k' MSIs
-------------------------------------------------------
|IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
-------------------------------------------------------
MSI-0 will have count=k set and flags set accordingly.

Vikas Gupta (1):
vfio/platform: add support for msi

 drivers/vfio/platform/vfio_platform_common.c  |  99 ++++++-
 drivers/vfio/platform/vfio_platform_irq.c     | 260 +++++++++++++++++-
 drivers/vfio/platform/vfio_platform_private.h |  16 ++
 include/uapi/linux/vfio.h                     |  43 +++
 4 files changed, 401 insertions(+), 17 deletions(-)
#34 lore raw

v3 cover msi support for platform devices

In reply to Vikas Gupta

This RFC adds support for MSI for platform devices. MSI block is added as an ext irq along with the existing wired interrupt implementation. The patchset exports two caps for MSI and related data to configure MSI source device.

Changes from:
-------------
v2 to v3:
1) Restored the vendor specific module to get max number of MSIs supported and .count value initialized.
2) Comments from Eric addressed.

v1 to v2:
1) IRQ allocation has been implemented as below:
----------------------------
|IRQ-0|IRQ-1|....|IRQ-n|MSI|
----------------------------
MSI block has msi contexts and its implemneted as ext irq.

2) Removed vendor specific module for msi handling so previously patch2 and patch3 are not required.

3) MSI related data is exported to userspace using 'caps'.
Please note VFIO_IRQ_INFO_CAP_TYPE in include/uapi/linux/vfio.h implementation
is taken from the Eric`s patch
https://patchwork.kernel.org/project/kvm/patch/[email protected]/

v0 to v1:
i) Removed MSI device flag VFIO_DEVICE_FLAGS_MSI. ii) Add MSI(s) at the end of the irq list of platform IRQs. MSI(s) with first entry of MSI block has count and flag information.
IRQ list: Allocation for IRQs + MSIs are allocated as below
Example: if there are 'n' IRQs and 'k' MSIs
-------------------------------------------------------
|IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
-------------------------------------------------------
MSI-0 will have count=k set and flags set accordingly.

Vikas Gupta (2):
vfio/platform: add support for msi vfio/platform: msi: add Broadcom platform devices

 drivers/vfio/platform/Kconfig                 |   1 +
 drivers/vfio/platform/Makefile                |   1 +
 drivers/vfio/platform/msi/Kconfig             |   9 +
 drivers/vfio/platform/msi/Makefile            |   2 +
 .../vfio/platform/msi/vfio_platform_bcmplt.c  |  49 ++++
 drivers/vfio/platform/vfio_platform_common.c  | 179 +++++++++++-
 drivers/vfio/platform/vfio_platform_irq.c     | 260 +++++++++++++++++-
 drivers/vfio/platform/vfio_platform_private.h |  32 +++
 include/uapi/linux/vfio.h                     |  44 +++
 9 files changed, 558 insertions(+), 19 deletions(-)
 create mode 100644 drivers/vfio/platform/msi/Kconfig
 create mode 100644 drivers/vfio/platform/msi/Makefile
 create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
Signature · 15 lines
2.17.1


-- 
This electronic communication and the information and any files transmitted 
with it, or attached to it, are confidential and are intended solely for 
the use of the individual or entity to whom it is addressed and may contain 
information that is confidential, legally privileged, protected by privacy 
laws, or otherwise restricted from disclosure to anyone else. If you are 
not the intended recipient or the person responsible for delivering the 
e-mail to the intended recipient, you are hereby notified that any use, 
copying, distributing, dissemination, forwarding, printing, or copying of 
this e-mail is strictly prohibited. If you received this e-mail in error, 
please return the e-mail to the sender, delete it from your computer, and 
destroy any printed copy of it.
#35 lore raw

RFC v3 1/2 vfio/platform: add support for msi

MSI support for platform devices.The MSI block is added as an extended IRQ which exports caps VFIO_IRQ_INFO_CAP_TYPE and VFIO_IRQ_INFO_CAP_MSI_DESCS.

---
 drivers/vfio/platform/vfio_platform_common.c  | 179 +++++++++++-
 drivers/vfio/platform/vfio_platform_irq.c     | 260 +++++++++++++++++-
 drivers/vfio/platform/vfio_platform_private.h |  32 +++
 include/uapi/linux/vfio.h                     |  44 +++
 4 files changed, 496 insertions(+), 19 deletions(-)
drivers/vfio/platform/vfio_platform_common.c +175 -4
index fb4b385191f2..c936852f35d7 100644
@@ -16,6 +16,7 @@
 #include <linux/types.h>
 #include <linux/uaccess.h>
 #include <linux/vfio.h>
+#include <linux/nospec.h>
 
 #include "vfio_platform_private.h"
 
@@ -26,6 +27,8 @@
 #define VFIO_PLATFORM_IS_ACPI(vdev) ((vdev)->acpihid != NULL)
 
 static LIST_HEAD(reset_list);
+/* devices having MSI support */
+static LIST_HEAD(msi_list);
 static DEFINE_MUTEX(driver_lock);
 
 static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,
@@ -47,6 +50,25 @@ static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,
 	return reset_fn;
 }
 
+static bool vfio_platform_lookup_msi(struct vfio_platform_device *vdev)
+{
+	bool has_msi = false;
+	struct vfio_platform_msi_node *iter;
+
+	mutex_lock(&driver_lock);
+	list_for_each_entry(iter, &msi_list, link) {
+		if (!strcmp(iter->compat, vdev->compat) &&
+		    try_module_get(iter->owner)) {
+			vdev->msi_module = iter->owner;
+			vdev->of_get_msi = iter->of_get_msi;
+			has_msi = true;
+			break;
+		}
+	}
+	mutex_unlock(&driver_lock);
+	return has_msi;
+}
+
 static int vfio_platform_acpi_probe(struct vfio_platform_device *vdev,
 				    struct device *dev)
 {
@@ -126,6 +148,19 @@ static int vfio_platform_get_reset(struct vfio_platform_device *vdev)
 	return vdev->of_reset ? 0 : -ENOENT;
 }
 
+static int vfio_platform_get_msi(struct vfio_platform_device *vdev)
+{
+	bool has_msi;
+
+	has_msi = vfio_platform_lookup_msi(vdev);
+	if (!has_msi) {
+		request_module("vfio-msi:%s", vdev->compat);
+		has_msi = vfio_platform_lookup_msi(vdev);
+	}
+
+	return has_msi ? 0 : -ENOENT;
+}
+
 static void vfio_platform_put_reset(struct vfio_platform_device *vdev)
 {
 	if (VFIO_PLATFORM_IS_ACPI(vdev))
@@ -135,6 +170,12 @@ static void vfio_platform_put_reset(struct vfio_platform_device *vdev)
 		module_put(vdev->reset_module);
 }
 
+static void vfio_platform_put_msi(struct vfio_platform_device *vdev)
+{
+	if (vdev->of_get_msi)
+		module_put(vdev->msi_module);
+}
+
 static int vfio_platform_regions_init(struct vfio_platform_device *vdev)
 {
 	int cnt = 0, i;
@@ -343,9 +384,17 @@ static long vfio_platform_ioctl(void *device_data,
 
 	} else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
 		struct vfio_irq_info info;
+		struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
+		struct vfio_irq_info_cap_msi *msi_info = NULL;
+		int ext_irq_index = vdev->num_irqs - vdev->num_ext_irqs;
+		unsigned long capsz;
+		u32 index;
 
 		minsz = offsetofend(struct vfio_irq_info, count);
 
+		/* For backward compatibility, cannot require this */
+		capsz = offsetofend(struct vfio_irq_info, cap_offset);
+
 		if (copy_from_user(&info, (void __user *)arg, minsz))
 			return -EFAULT;
 
@@ -355,8 +404,94 @@ static long vfio_platform_ioctl(void *device_data,
 		if (info.index >= vdev->num_irqs)
 			return -EINVAL;
 
-		info.flags = vdev->irqs[info.index].flags;
-		info.count = vdev->irqs[info.index].count;
+		if (info.argsz >= capsz)
+			minsz = capsz;
+
+		index = info.index;
+
+		info.flags = vdev->irqs[index].flags;
+		info.count = vdev->irqs[index].count;
+
+		if (ext_irq_index - index == VFIO_EXT_IRQ_MSI) {
+			struct vfio_irq_info_cap_type cap_type = {
+				.header.id = VFIO_IRQ_INFO_CAP_TYPE,
+				.header.version = 1 };
+			struct vfio_platform_irq *irq;
+			size_t msi_info_size;
+			int num_msgs;
+			int ret;
+			int i;
+
+			index = array_index_nospec(index,
+						   vdev->num_irqs);
+			irq = &vdev->irqs[index];
+
+			cap_type.type = irq->type;
+			cap_type.subtype = irq->subtype;
+
+			ret = vfio_info_add_capability(&caps,
+						       &cap_type.header,
+						       sizeof(cap_type));
+			if (ret)
+				return ret;
+
+			num_msgs = irq->num_ctx;
+			if (num_msgs) {
+				msi_info_size = struct_size(msi_info,
+							    msgs, num_msgs);
+
+				msi_info = kzalloc(msi_info_size, GFP_KERNEL);
+				if (!msi_info) {
+					kfree(caps.buf);
+					return -ENOMEM;
+				}
+
+				msi_info->header.id =
+						VFIO_IRQ_INFO_CAP_MSI_DESCS;
+				msi_info->header.version = 1;
+				msi_info->nr_msgs = num_msgs;
+
+				for (i = 0; i < num_msgs; i++) {
+					struct vfio_irq_ctx *ctx = &irq->ctx[i];
+
+					msi_info->msgs[i].addr_lo =
+							ctx->msg.address_lo;
+					msi_info->msgs[i].addr_hi =
+							ctx->msg.address_hi;
+					msi_info->msgs[i].data = ctx->msg.data;
+				}
+
+				ret = vfio_info_add_capability(&caps,
+							&msi_info->header,
+							msi_info_size);
+				if (ret) {
+					kfree(msi_info);
+					kfree(caps.buf);
+					return ret;
+				}
+			}
+		}
+
+		if (caps.size) {
+			info.flags |= VFIO_IRQ_INFO_FLAG_CAPS;
+			if (info.argsz < sizeof(info) + caps.size) {
+				info.argsz = sizeof(info) + caps.size;
+				info.cap_offset = 0;
+			} else {
+				vfio_info_cap_shift(&caps, sizeof(info));
+				if (copy_to_user((void __user *)arg +
+						  sizeof(info), caps.buf,
+						  caps.size)) {
+					kfree(msi_info);
+					kfree(caps.buf);
+					return -EFAULT;
+				}
+				info.cap_offset = sizeof(info);
+			}
+
+			kfree(msi_info);
+			kfree(caps.buf);
+		}
 
 		return copy_to_user((void __user *)arg, &info, minsz) ?
 			-EFAULT : 0;
@@ -365,6 +500,7 @@ static long vfio_platform_ioctl(void *device_data,
 		struct vfio_irq_set hdr;
 		u8 *data = NULL;
 		int ret = 0;
+		int max;
 		size_t data_size = 0;
 
 		minsz = offsetofend(struct vfio_irq_set, count);
@@ -372,8 +508,14 @@ static long vfio_platform_ioctl(void *device_data,
 		if (copy_from_user(&hdr, (void __user *)arg, minsz))
 			return -EFAULT;
 
-		ret = vfio_set_irqs_validate_and_prepare(&hdr, vdev->num_irqs,
-						 vdev->num_irqs, &data_size);
+		if (hdr.index >= vdev->num_irqs)
+			return -EINVAL;
+
+		max = vdev->irqs[hdr.index].count;
+
+		ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
+							 vdev->num_irqs,
+							 &data_size);
 		if (ret)
 			return ret;
 
@@ -678,6 +820,10 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,
 		return ret;
 	}
 
+	ret = vfio_platform_get_msi(vdev);
+	if (ret)
+		dev_info(vdev->device, "msi not supported\n");
+
 	group = vfio_iommu_group_get(dev);
 	if (!group) {
 		dev_err(dev, "No IOMMU group for device %s\n", vdev->name);
@@ -697,6 +843,7 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,
 put_iommu:
 	vfio_iommu_group_put(group, dev);
 put_reset:
+	vfio_platform_put_msi(vdev);
 	vfio_platform_put_reset(vdev);
 	return ret;
 }
@@ -744,6 +891,30 @@ void vfio_platform_unregister_reset(const char *compat,
 }
 EXPORT_SYMBOL_GPL(vfio_platform_unregister_reset);
 
+void __vfio_platform_register_msi(struct vfio_platform_msi_node *node)
+{
+	mutex_lock(&driver_lock);
+	list_add(&node->link, &msi_list);
+	mutex_unlock(&driver_lock);
+}
+EXPORT_SYMBOL_GPL(__vfio_platform_register_msi);
+
+void vfio_platform_unregister_msi(const char *compat)
+{
+	struct vfio_platform_msi_node *iter, *temp;
+
+	mutex_lock(&driver_lock);
+	list_for_each_entry_safe(iter, temp, &msi_list, link) {
+		if (!strcmp(iter->compat, compat)) {
+			list_del(&iter->link);
+			break;
+		}
+	}
+
+	mutex_unlock(&driver_lock);
+}
+EXPORT_SYMBOL_GPL(vfio_platform_unregister_msi);
+
 MODULE_VERSION(DRIVER_VERSION);
 MODULE_LICENSE("GPL v2");
 MODULE_AUTHOR(DRIVER_AUTHOR);
drivers/vfio/platform/vfio_platform_irq.c +245 -15

304 lines not on this page · raw message

drivers/vfio/platform/vfio_platform_private.h +32 -0
index 289089910643..1307feddda21 100644
@@ -9,6 +9,7 @@
 
 #include <linux/types.h>
 #include <linux/interrupt.h>
+#include <linux/msi.h>
 
 #define VFIO_PLATFORM_OFFSET_SHIFT   40
 #define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
@@ -19,9 +20,21 @@
 #define VFIO_PLATFORM_INDEX_TO_OFFSET(index)	\
 	((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT)
 
+/* IRQ index for MSI in ext IRQs */
+#define VFIO_EXT_IRQ_MSI	0
+
+struct vfio_irq_ctx {
+	int			hwirq;
+	char			*name;
+	struct msi_msg		msg;
+	struct eventfd_ctx	*trigger;
+};
+
 struct vfio_platform_irq {
 	u32			flags;
 	u32			count;
+	int			num_ctx;
+	struct vfio_irq_ctx	*ctx;
 	int			hwirq;
 	char			*name;
 	struct eventfd_ctx	*trigger;
@@ -29,6 +42,11 @@ struct vfio_platform_irq {
 	spinlock_t		lock;
 	struct virqfd		*unmask;
 	struct virqfd		*mask;
+
+	/* for extended irqs */
+	u32			type;
+	u32			subtype;
+	int			config_msi;
 };
 
 struct vfio_platform_region {
@@ -46,12 +64,14 @@ struct vfio_platform_device {
 	u32				num_regions;
 	struct vfio_platform_irq	*irqs;
 	u32				num_irqs;
+	int				num_ext_irqs;
 	int				refcnt;
 	struct mutex			igate;
 	struct module			*parent_module;
 	const char			*compat;
 	const char			*acpihid;
 	struct module			*reset_module;
+	struct module			*msi_module;
 	struct device			*device;
 
 	/*
@@ -65,11 +85,13 @@ struct vfio_platform_device {
 		(*get_resource)(struct vfio_platform_device *vdev, int i);
 	int	(*get_irq)(struct vfio_platform_device *vdev, int i);
 	int	(*of_reset)(struct vfio_platform_device *vdev);
+	u32	(*of_get_msi)(struct vfio_platform_device *vdev);
 
 	bool				reset_required;
 };
 
 typedef int (*vfio_platform_reset_fn_t)(struct vfio_platform_device *vdev);
+typedef u32 (*vfio_platform_get_msi_fn_t)(struct vfio_platform_device *vdev);
 
 struct vfio_platform_reset_node {
 	struct list_head link;
@@ -78,6 +100,13 @@ struct vfio_platform_reset_node {
 	vfio_platform_reset_fn_t of_reset;
 };
 
+struct vfio_platform_msi_node {
+	struct list_head link;
+	char *compat;
+	struct module *owner;
+	vfio_platform_get_msi_fn_t of_get_msi;
+};
+
 extern int vfio_platform_probe_common(struct vfio_platform_device *vdev,
 				      struct device *dev);
 extern struct vfio_platform_device *vfio_platform_remove_common
@@ -94,6 +123,9 @@ extern int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 extern void __vfio_platform_register_reset(struct vfio_platform_reset_node *n);
 extern void vfio_platform_unregister_reset(const char *compat,
 					   vfio_platform_reset_fn_t fn);
+void __vfio_platform_register_msi(struct vfio_platform_msi_node *n);
+void vfio_platform_unregister_msi(const char *compat);
+
 #define vfio_platform_register_reset(__compat, __reset)		\
 static struct vfio_platform_reset_node __reset ## _node = {	\
 	.owner = THIS_MODULE,					\
include/uapi/linux/vfio.h +44 -0

55 lines not on this page · raw message

Signature · 15 lines
2.17.1


-- 
This electronic communication and the information and any files transmitted 
with it, or attached to it, are confidential and are intended solely for 
the use of the individual or entity to whom it is addressed and may contain 
information that is confidential, legally privileged, protected by privacy 
laws, or otherwise restricted from disclosure to anyone else. If you are 
not the intended recipient or the person responsible for delivering the 
e-mail to the intended recipient, you are hereby notified that any use, 
copying, distributing, dissemination, forwarding, printing, or copying of 
this e-mail is strictly prohibited. If you received this e-mail in error, 
please return the e-mail to the sender, delete it from your computer, and 
destroy any printed copy of it.
maintainer
#36 lore raw

Re: [RFC v3 1/2] vfio/platform: add support for msi

Hi Vikas,

Quoted 25 lines · Vikas Gupta

MSI support for platform devices.The MSI block is added as an extended IRQ which exports caps VFIO_IRQ_INFO_CAP_TYPE and VFIO_IRQ_INFO_CAP_MSI_DESCS.

---
 drivers/vfio/platform/vfio_platform_common.c  | 179 +++++++++++-
 drivers/vfio/platform/vfio_platform_irq.c     | 260 +++++++++++++++++-
 drivers/vfio/platform/vfio_platform_private.h |  32 +++
 include/uapi/linux/vfio.h                     |  44 +++
 4 files changed, 496 insertions(+), 19 deletions(-)
drivers/vfio/platform/vfio_platform_common.c +1 -0
index fb4b385191f2..c936852f35d7 100644
@@ -16,6 +16,7 @@
 #include <linux/types.h>
 #include <linux/uaccess.h>
 #include <linux/vfio.h>
+#include <linux/nospec.h>
 
 #include "vfio_platform_private.h"
 

drivers/vfio/platform/vfio_platform_common.c line 30

Quoted 4 lines
drivers/vfio/platform/vfio_platform_common.c +1 -0
@@ -26,6 +27,8 @@
 #define VFIO_PLATFORM_IS_ACPI(vdev) ((vdev)->acpihid != NULL)
 
 static LIST_HEAD(reset_list);
+/* devices having MSI support */

nit: for devices using MSIs?

Quoted 121 lines
drivers/vfio/platform/vfio_platform_common.c +81 -2
+static LIST_HEAD(msi_list);
 static DEFINE_MUTEX(driver_lock);
 
 static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,
@@ -47,6 +50,25 @@ static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,
 	return reset_fn;
 }
 
+static bool vfio_platform_lookup_msi(struct vfio_platform_device *vdev)
+{
+	bool has_msi = false;
+	struct vfio_platform_msi_node *iter;
+
+	mutex_lock(&driver_lock);
+	list_for_each_entry(iter, &msi_list, link) {
+		if (!strcmp(iter->compat, vdev->compat) &&
+		    try_module_get(iter->owner)) {
+			vdev->msi_module = iter->owner;
+			vdev->of_get_msi = iter->of_get_msi;
+			has_msi = true;
+			break;
+		}
+	}
+	mutex_unlock(&driver_lock);
+	return has_msi;
+}
+
 static int vfio_platform_acpi_probe(struct vfio_platform_device *vdev,
 				    struct device *dev)
 {
@@ -126,6 +148,19 @@ static int vfio_platform_get_reset(struct vfio_platform_device *vdev)
 	return vdev->of_reset ? 0 : -ENOENT;
 }
 
+static int vfio_platform_get_msi(struct vfio_platform_device *vdev)
+{
+	bool has_msi;
+
+	has_msi = vfio_platform_lookup_msi(vdev);
+	if (!has_msi) {
+		request_module("vfio-msi:%s", vdev->compat);
+		has_msi = vfio_platform_lookup_msi(vdev);
+	}
+
+	return has_msi ? 0 : -ENOENT;
+}
+
 static void vfio_platform_put_reset(struct vfio_platform_device *vdev)
 {
 	if (VFIO_PLATFORM_IS_ACPI(vdev))
@@ -135,6 +170,12 @@ static void vfio_platform_put_reset(struct vfio_platform_device *vdev)
 		module_put(vdev->reset_module);
 }
 
+static void vfio_platform_put_msi(struct vfio_platform_device *vdev)
+{
+	if (vdev->of_get_msi)
+		module_put(vdev->msi_module);
+}
+
 static int vfio_platform_regions_init(struct vfio_platform_device *vdev)
 {
 	int cnt = 0, i;
@@ -343,9 +384,17 @@ static long vfio_platform_ioctl(void *device_data,
 
 	} else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
 		struct vfio_irq_info info;
+		struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
+		struct vfio_irq_info_cap_msi *msi_info = NULL;
+		int ext_irq_index = vdev->num_irqs - vdev->num_ext_irqs;
+		unsigned long capsz;
+		u32 index;
 
 		minsz = offsetofend(struct vfio_irq_info, count);
 
+		/* For backward compatibility, cannot require this */
+		capsz = offsetofend(struct vfio_irq_info, cap_offset);
+
 		if (copy_from_user(&info, (void __user *)arg, minsz))
 			return -EFAULT;
 
@@ -355,8 +404,94 @@ static long vfio_platform_ioctl(void *device_data,
 		if (info.index >= vdev->num_irqs)
 			return -EINVAL;
 
-		info.flags = vdev->irqs[info.index].flags;
-		info.count = vdev->irqs[info.index].count;
+		if (info.argsz >= capsz)
+			minsz = capsz;
+
+		index = info.index;
+
+		info.flags = vdev->irqs[index].flags;
+		info.count = vdev->irqs[index].count;
+
+		if (ext_irq_index - index == VFIO_EXT_IRQ_MSI) {
+			struct vfio_irq_info_cap_type cap_type = {
+				.header.id = VFIO_IRQ_INFO_CAP_TYPE,
+				.header.version = 1 };
+			struct vfio_platform_irq *irq;
+			size_t msi_info_size;
+			int num_msgs;
+			int ret;
+			int i;
+
+			index = array_index_nospec(index,
+						   vdev->num_irqs);
+			irq = &vdev->irqs[index];
+
+			cap_type.type = irq->type;
+			cap_type.subtype = irq->subtype;
+
+			ret = vfio_info_add_capability(&caps,
+						       &cap_type.header,
+						       sizeof(cap_type));
+			if (ret)
+				return ret;
+
+			num_msgs = irq->num_ctx;
+			if (num_msgs) {
+				msi_info_size = struct_size(msi_info,

drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 450

Quoted 10 lines
drivers/vfio/platform/vfio_platform_common.c +10 -0
+							    msgs, num_msgs);
+
+				msi_info = kzalloc(msi_info_size, GFP_KERNEL);
+				if (!msi_info) {
+					kfree(caps.buf);
+					return -ENOMEM;
+				}
+
+				msi_info->header.id =
+						VFIO_IRQ_INFO_CAP_MSI_DESCS;

I thought you would remove this cap now the module is back. What is the motivation to keep it?

Thanks

Eric

Quoted 73 lines
drivers/vfio/platform/vfio_platform_common.c +53 -2
+				msi_info->header.version = 1;
+				msi_info->nr_msgs = num_msgs;
+
+				for (i = 0; i < num_msgs; i++) {
+					struct vfio_irq_ctx *ctx = &irq->ctx[i];
+
+					msi_info->msgs[i].addr_lo =
+							ctx->msg.address_lo;
+					msi_info->msgs[i].addr_hi =
+							ctx->msg.address_hi;
+					msi_info->msgs[i].data = ctx->msg.data;
+				}
+
+				ret = vfio_info_add_capability(&caps,
+							&msi_info->header,
+							msi_info_size);
+				if (ret) {
+					kfree(msi_info);
+					kfree(caps.buf);
+					return ret;
+				}
+			}
+		}
+
+		if (caps.size) {
+			info.flags |= VFIO_IRQ_INFO_FLAG_CAPS;
+			if (info.argsz < sizeof(info) + caps.size) {
+				info.argsz = sizeof(info) + caps.size;
+				info.cap_offset = 0;
+			} else {
+				vfio_info_cap_shift(&caps, sizeof(info));
+				if (copy_to_user((void __user *)arg +
+						  sizeof(info), caps.buf,
+						  caps.size)) {
+					kfree(msi_info);
+					kfree(caps.buf);
+					return -EFAULT;
+				}
+				info.cap_offset = sizeof(info);
+			}
+
+			kfree(msi_info);
+			kfree(caps.buf);
+		}
 
 		return copy_to_user((void __user *)arg, &info, minsz) ?
 			-EFAULT : 0;
@@ -365,6 +500,7 @@ static long vfio_platform_ioctl(void *device_data,
 		struct vfio_irq_set hdr;
 		u8 *data = NULL;
 		int ret = 0;
+		int max;
 		size_t data_size = 0;
 
 		minsz = offsetofend(struct vfio_irq_set, count);
@@ -372,8 +508,14 @@ static long vfio_platform_ioctl(void *device_data,
 		if (copy_from_user(&hdr, (void __user *)arg, minsz))
 			return -EFAULT;
 
-		ret = vfio_set_irqs_validate_and_prepare(&hdr, vdev->num_irqs,
-						 vdev->num_irqs, &data_size);
+		if (hdr.index >= vdev->num_irqs)
+			return -EINVAL;
+
+		max = vdev->irqs[hdr.index].count;
+
+		ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
+							 vdev->num_irqs,
+							 &data_size);
 		if (ret)
 			return ret;
 

drivers/vfio/platform/vfio_platform_common.c int vfio_platform_probe_common(struct vfio_platform_device *vdev, · line 825

Quoted 6 lines
drivers/vfio/platform/vfio_platform_common.c +3 -0
@@ -678,6 +820,10 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,
 		return ret;
 	}
 
+	ret = vfio_platform_get_msi(vdev);
+	if (ret)
+		dev_info(vdev->device, "msi not supported\n");

I don't think we should output this message. This is printed for amd-xgbe which does not have msi so this is misleading. I would say either the vfio_platform_get_msi() can return an actual error or we return a void?

Thanks

Eric

Quoted 517 lines
drivers/vfio/platform/vfio_platform_common.c +26 -0
+
 	group = vfio_iommu_group_get(dev);
 	if (!group) {
 		dev_err(dev, "No IOMMU group for device %s\n", vdev->name);
@@ -697,6 +843,7 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,
 put_iommu:
 	vfio_iommu_group_put(group, dev);
 put_reset:
+	vfio_platform_put_msi(vdev);
 	vfio_platform_put_reset(vdev);
 	return ret;
 }
@@ -744,6 +891,30 @@ void vfio_platform_unregister_reset(const char *compat,
 }
 EXPORT_SYMBOL_GPL(vfio_platform_unregister_reset);
 
+void __vfio_platform_register_msi(struct vfio_platform_msi_node *node)
+{
+	mutex_lock(&driver_lock);
+	list_add(&node->link, &msi_list);
+	mutex_unlock(&driver_lock);
+}
+EXPORT_SYMBOL_GPL(__vfio_platform_register_msi);
+
+void vfio_platform_unregister_msi(const char *compat)
+{
+	struct vfio_platform_msi_node *iter, *temp;
+
+	mutex_lock(&driver_lock);
+	list_for_each_entry_safe(iter, temp, &msi_list, link) {
+		if (!strcmp(iter->compat, compat)) {
+			list_del(&iter->link);
+			break;
+		}
+	}
+
+	mutex_unlock(&driver_lock);
+}
+EXPORT_SYMBOL_GPL(vfio_platform_unregister_msi);
+
 MODULE_VERSION(DRIVER_VERSION);
 MODULE_LICENSE("GPL v2");
 MODULE_AUTHOR(DRIVER_AUTHOR);
drivers/vfio/platform/vfio_platform_irq.c +245 -15
index c5b09ec0a3c9..e0f4696afedd 100644
@@ -8,10 +8,12 @@
 
 #include <linux/eventfd.h>
 #include <linux/interrupt.h>
+#include <linux/eventfd.h>
 #include <linux/slab.h>
 #include <linux/types.h>
 #include <linux/vfio.h>
 #include <linux/irq.h>
+#include <linux/msi.h>
 
 #include "vfio_platform_private.h"
 
@@ -253,6 +255,195 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,
 	return 0;
 }
 
+/* MSI/MSIX */
+static irqreturn_t vfio_msihandler(int irq, void *arg)
+{
+	struct eventfd_ctx *trigger = arg;
+
+	eventfd_signal(trigger, 1);
+	return IRQ_HANDLED;
+}
+
+static void msi_write(struct msi_desc *desc, struct msi_msg *msg)
+{
+	int i;
+	struct vfio_platform_irq *irq;
+	u16 index = desc->platform.msi_index;
+	struct device *dev = msi_desc_to_dev(desc);
+	struct vfio_device *device = dev_get_drvdata(dev);
+	struct vfio_platform_device *vdev = (struct vfio_platform_device *)
+						vfio_device_data(device);
+
+	for (i = 0; i < vdev->num_irqs; i++)
+		if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
+			irq = &vdev->irqs[i];
+
+	irq->ctx[index].msg.address_lo = msg->address_lo;
+	irq->ctx[index].msg.address_hi = msg->address_hi;
+	irq->ctx[index].msg.data = msg->data;
+}
+
+static int vfio_msi_enable(struct vfio_platform_device *vdev,
+			   struct vfio_platform_irq *irq, int nvec)
+{
+	int ret;
+	int msi_idx = 0;
+	struct msi_desc *desc;
+	struct device *dev = vdev->device;
+
+	irq->ctx = kcalloc(nvec, sizeof(struct vfio_irq_ctx), GFP_KERNEL);
+	if (!irq->ctx)
+		return -ENOMEM;
+
+	/* Allocate platform MSIs */
+	ret = platform_msi_domain_alloc_irqs(dev, nvec, msi_write);
+	if (ret < 0) {
+		kfree(irq->ctx);
+		return ret;
+	}
+
+	for_each_msi_entry(desc, dev) {
+		irq->ctx[msi_idx].hwirq = desc->irq;
+		msi_idx++;
+	}
+
+	irq->num_ctx = nvec;
+	irq->config_msi = 1;
+
+	return 0;
+}
+
+static int vfio_msi_set_vector_signal(struct vfio_platform_irq *irq,
+				      int vector, int fd)
+{
+	struct eventfd_ctx *trigger;
+	int irq_num, ret;
+
+	if (vector < 0 || vector >= irq->num_ctx)
+		return -EINVAL;
+
+	irq_num = irq->ctx[vector].hwirq;
+
+	if (irq->ctx[vector].trigger) {
+		free_irq(irq_num, irq->ctx[vector].trigger);
+		kfree(irq->ctx[vector].name);
+		eventfd_ctx_put(irq->ctx[vector].trigger);
+		irq->ctx[vector].trigger = NULL;
+	}
+
+	if (fd < 0)
+		return 0;
+
+	irq->ctx[vector].name = kasprintf(GFP_KERNEL,
+					  "vfio-msi[%d]", vector);
+	if (!irq->ctx[vector].name)
+		return -ENOMEM;
+
+	trigger = eventfd_ctx_fdget(fd);
+	if (IS_ERR(trigger)) {
+		kfree(irq->ctx[vector].name);
+		return PTR_ERR(trigger);
+	}
+
+	ret = request_irq(irq_num, vfio_msihandler, 0,
+			  irq->ctx[vector].name, trigger);
+	if (ret) {
+		kfree(irq->ctx[vector].name);
+		eventfd_ctx_put(trigger);
+		return ret;
+	}
+
+	irq->ctx[vector].trigger = trigger;
+
+	return 0;
+}
+
+static int vfio_msi_set_block(struct vfio_platform_irq *irq, unsigned int start,
+			      unsigned int count, int32_t *fds)
+{
+	int i, j, ret = 0;
+
+	if (start >= irq->num_ctx || start + count > irq->num_ctx)
+		return -EINVAL;
+
+	for (i = 0, j = start; i < count && !ret; i++, j++) {
+		int fd = fds ? fds[i] : -1;
+
+		ret = vfio_msi_set_vector_signal(irq, j, fd);
+	}
+
+	if (ret) {
+		for (--j; j >= (int)start; j--)
+			vfio_msi_set_vector_signal(irq, j, -1);
+	}
+
+	return ret;
+}
+
+static void vfio_msi_disable(struct vfio_platform_device *vdev,
+			     struct vfio_platform_irq *irq)
+{
+	struct device *dev = vdev->device;
+
+	vfio_msi_set_block(irq, 0, irq->num_ctx, NULL);
+
+	platform_msi_domain_free_irqs(dev);
+
+	irq->config_msi = 0;
+	irq->num_ctx = 0;
+
+	kfree(irq->ctx);
+}
+
+static int vfio_set_msi_trigger(struct vfio_platform_device *vdev,
+				unsigned int index, unsigned int start,
+				unsigned int count, uint32_t flags, void *data)
+{
+	int i;
+	struct vfio_platform_irq *irq = &vdev->irqs[index];
+
+	if (start + count > irq->count)
+		return -EINVAL;
+
+	if (!count && (flags & VFIO_IRQ_SET_DATA_NONE)) {
+		vfio_msi_disable(vdev, irq);
+		return 0;
+	}
+
+	if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
+		s32 *fds = data;
+		int ret;
+
+		if (irq->config_msi)
+			return vfio_msi_set_block(irq, start, count,
+						  fds);
+		ret = vfio_msi_enable(vdev, irq, start + count);
+		if (ret)
+			return ret;
+
+		ret = vfio_msi_set_block(irq, start, count, fds);
+		if (ret)
+			vfio_msi_disable(vdev, irq);
+
+		return ret;
+	}
+
+	for (i = start; i < start + count; i++) {
+		if (!irq->ctx[i].trigger)
+			continue;
+		if (flags & VFIO_IRQ_SET_DATA_NONE) {
+			eventfd_signal(irq->ctx[i].trigger, 1);
+		} else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
+			u8 *bools = data;
+
+			if (bools[i - start])
+				eventfd_signal(irq->ctx[i].trigger, 1);
+		}
+	}
+
+	return 0;
+}
+
 int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 				 uint32_t flags, unsigned index, unsigned start,
 				 unsigned count, void *data)
@@ -261,16 +452,29 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 		    unsigned start, unsigned count, uint32_t flags,
 		    void *data) = NULL;
 
-	switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
-	case VFIO_IRQ_SET_ACTION_MASK:
-		func = vfio_platform_set_irq_mask;
-		break;
-	case VFIO_IRQ_SET_ACTION_UNMASK:
-		func = vfio_platform_set_irq_unmask;
-		break;
-	case VFIO_IRQ_SET_ACTION_TRIGGER:
-		func = vfio_platform_set_irq_trigger;
-		break;
+	struct vfio_platform_irq *irq = &vdev->irqs[index];
+
+	if (irq->type == VFIO_IRQ_TYPE_MSI) {
+		switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
+		case VFIO_IRQ_SET_ACTION_MASK:
+		case VFIO_IRQ_SET_ACTION_UNMASK:
+			break;
+		case VFIO_IRQ_SET_ACTION_TRIGGER:
+			func = vfio_set_msi_trigger;
+			break;
+		}
+	} else {
+		switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
+		case VFIO_IRQ_SET_ACTION_MASK:
+			func = vfio_platform_set_irq_mask;
+			break;
+		case VFIO_IRQ_SET_ACTION_UNMASK:
+			func = vfio_platform_set_irq_unmask;
+			break;
+		case VFIO_IRQ_SET_ACTION_TRIGGER:
+			func = vfio_platform_set_irq_trigger;
+			break;
+		}
 	}
 
 	if (!func)
@@ -281,12 +485,21 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 
 int vfio_platform_irq_init(struct vfio_platform_device *vdev)
 {
-	int cnt = 0, i;
+	int i;
+	int cnt = 0;
+	int num_irqs;
+	int msi_cnt = 0;
 
 	while (vdev->get_irq(vdev, cnt) >= 0)
 		cnt++;
 
-	vdev->irqs = kcalloc(cnt, sizeof(struct vfio_platform_irq), GFP_KERNEL);
+	if (vdev->of_get_msi) {
+		msi_cnt = vdev->of_get_msi(vdev);
+		num_irqs++;
+	}
+
+	vdev->irqs = kcalloc(num_irqs, sizeof(struct vfio_platform_irq),
+			     GFP_KERNEL);
 	if (!vdev->irqs)
 		return -ENOMEM;
 
@@ -309,7 +522,19 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev)
 		vdev->irqs[i].masked = false;
 	}
 
-	vdev->num_irqs = cnt;
+	/*
+	 * MSI block is added at last index and it is an ext irq
+	 */
+	if (msi_cnt > 0) {
+		vdev->irqs[i].flags = VFIO_IRQ_INFO_EVENTFD;
+		vdev->irqs[i].count = msi_cnt;
+		vdev->irqs[i].hwirq = 0;
+		vdev->irqs[i].masked = false;
+		vdev->irqs[i].type = VFIO_IRQ_TYPE_MSI;
+		vdev->num_ext_irqs = 1;
+	}
+
+	vdev->num_irqs = num_irqs;
 
 	return 0;
 err:
@@ -321,8 +546,13 @@ void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev)
 {
 	int i;
 
-	for (i = 0; i < vdev->num_irqs; i++)
-		vfio_set_trigger(vdev, i, -1, NULL);
+	for (i = 0; i < vdev->num_irqs; i++) {
+		if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
+			vfio_set_msi_trigger(vdev, i, 0, 0,
+					     VFIO_IRQ_SET_DATA_NONE, NULL);
+		else
+			vfio_set_trigger(vdev, i, -1, NULL);
+	}
 
 	vdev->num_irqs = 0;
 	kfree(vdev->irqs);
drivers/vfio/platform/vfio_platform_private.h +32 -0
index 289089910643..1307feddda21 100644
@@ -9,6 +9,7 @@
 
 #include <linux/types.h>
 #include <linux/interrupt.h>
+#include <linux/msi.h>
 
 #define VFIO_PLATFORM_OFFSET_SHIFT   40
 #define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
@@ -19,9 +20,21 @@
 #define VFIO_PLATFORM_INDEX_TO_OFFSET(index)	\
 	((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT)
 
+/* IRQ index for MSI in ext IRQs */
+#define VFIO_EXT_IRQ_MSI	0
+
+struct vfio_irq_ctx {
+	int			hwirq;
+	char			*name;
+	struct msi_msg		msg;
+	struct eventfd_ctx	*trigger;
+};
+
 struct vfio_platform_irq {
 	u32			flags;
 	u32			count;
+	int			num_ctx;
+	struct vfio_irq_ctx	*ctx;
 	int			hwirq;
 	char			*name;
 	struct eventfd_ctx	*trigger;
@@ -29,6 +42,11 @@ struct vfio_platform_irq {
 	spinlock_t		lock;
 	struct virqfd		*unmask;
 	struct virqfd		*mask;
+
+	/* for extended irqs */
+	u32			type;
+	u32			subtype;
+	int			config_msi;
 };
 
 struct vfio_platform_region {
@@ -46,12 +64,14 @@ struct vfio_platform_device {
 	u32				num_regions;
 	struct vfio_platform_irq	*irqs;
 	u32				num_irqs;
+	int				num_ext_irqs;
 	int				refcnt;
 	struct mutex			igate;
 	struct module			*parent_module;
 	const char			*compat;
 	const char			*acpihid;
 	struct module			*reset_module;
+	struct module			*msi_module;
 	struct device			*device;
 
 	/*
@@ -65,11 +85,13 @@ struct vfio_platform_device {
 		(*get_resource)(struct vfio_platform_device *vdev, int i);
 	int	(*get_irq)(struct vfio_platform_device *vdev, int i);
 	int	(*of_reset)(struct vfio_platform_device *vdev);
+	u32	(*of_get_msi)(struct vfio_platform_device *vdev);
 
 	bool				reset_required;
 };
 
 typedef int (*vfio_platform_reset_fn_t)(struct vfio_platform_device *vdev);
+typedef u32 (*vfio_platform_get_msi_fn_t)(struct vfio_platform_device *vdev);
 
 struct vfio_platform_reset_node {
 	struct list_head link;
@@ -78,6 +100,13 @@ struct vfio_platform_reset_node {
 	vfio_platform_reset_fn_t of_reset;
 };
 
+struct vfio_platform_msi_node {
+	struct list_head link;
+	char *compat;
+	struct module *owner;
+	vfio_platform_get_msi_fn_t of_get_msi;
+};
+
 extern int vfio_platform_probe_common(struct vfio_platform_device *vdev,
 				      struct device *dev);
 extern struct vfio_platform_device *vfio_platform_remove_common
@@ -94,6 +123,9 @@ extern int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 extern void __vfio_platform_register_reset(struct vfio_platform_reset_node *n);
 extern void vfio_platform_unregister_reset(const char *compat,
 					   vfio_platform_reset_fn_t fn);
+void __vfio_platform_register_msi(struct vfio_platform_msi_node *n);
+void vfio_platform_unregister_msi(const char *compat);
+
 #define vfio_platform_register_reset(__compat, __reset)		\
 static struct vfio_platform_reset_node __reset ## _node = {	\
 	.owner = THIS_MODULE,					\
include/uapi/linux/vfio.h +44 -0
index d1812777139f..53a7ff2b524e 100644
@@ -697,11 +697,55 @@ struct vfio_irq_info {
 #define VFIO_IRQ_INFO_MASKABLE		(1 << 1)
 #define VFIO_IRQ_INFO_AUTOMASKED	(1 << 2)
 #define VFIO_IRQ_INFO_NORESIZE		(1 << 3)
+#define VFIO_IRQ_INFO_FLAG_CAPS		(1 << 4) /* Info supports caps */
 	__u32	index;		/* IRQ index */
 	__u32	count;		/* Number of IRQs within this index */
+	__u32	cap_offset;	/* Offset within info struct of first cap */
 };
 #define VFIO_DEVICE_GET_IRQ_INFO	_IO(VFIO_TYPE, VFIO_BASE + 9)
 
+/*
+ * The irq type capability allows IRQs unique to a specific device or
+ * class of devices to be exposed.
+ *
+ * The structures below define version 1 of this capability.
+ */
+#define VFIO_IRQ_INFO_CAP_TYPE		3
+
+struct vfio_irq_info_cap_type {
+	struct vfio_info_cap_header header;
+	__u32 type;     /* global per bus driver */
+	__u32 subtype;  /* type specific */
+};
+
+/*
+ * List of IRQ types, global per bus driver.
+ * If you introduce a new type, please add it here.
+ */
+
+/* Non PCI devices having MSI(s) support */
+#define VFIO_IRQ_TYPE_MSI		(1)
+
+/*
+ * The msi capability allows the user to use the msi msg to
+ * configure the iova for the msi configuration.
+ * The structures below define version 1 of this capability.
+ */
+#define VFIO_IRQ_INFO_CAP_MSI_DESCS	4
+
+struct vfio_irq_msi_msg {
+	__u32	addr_lo;
+	__u32	addr_hi;
+	__u32	data;
+};
+
+struct vfio_irq_info_cap_msi {
+	struct vfio_info_cap_header header;
+	__u32	nr_msgs;
+	__u32	reserved;
+	struct vfio_irq_msi_msg msgs[];
+};
+
 /**
  * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set)
  *
#37 lore raw

Re: [RFC v3 1/2] vfio/platform: add support for msi

Quoted 159 lines · Auger Eric

Hi Vikas,

Quoted 25 lines · Vikas Gupta

MSI support for platform devices.The MSI block is added as an extended IRQ which exports caps VFIO_IRQ_INFO_CAP_TYPE and VFIO_IRQ_INFO_CAP_MSI_DESCS.

---
 drivers/vfio/platform/vfio_platform_common.c  | 179 +++++++++++-
 drivers/vfio/platform/vfio_platform_irq.c     | 260 +++++++++++++++++-
 drivers/vfio/platform/vfio_platform_private.h |  32 +++
 include/uapi/linux/vfio.h                     |  44 +++
 4 files changed, 496 insertions(+), 19 deletions(-)
drivers/vfio/platform/vfio_platform_common.c +1 -0
index fb4b385191f2..c936852f35d7 100644
@@ -16,6 +16,7 @@
 #include <linux/types.h>
 #include <linux/uaccess.h>
 #include <linux/vfio.h>
+#include <linux/nospec.h>
 
 #include "vfio_platform_private.h"
 

drivers/vfio/platform/vfio_platform_common.c line 30

Quoted 4 lines
drivers/vfio/platform/vfio_platform_common.c +1 -0
@@ -26,6 +27,8 @@
 #define VFIO_PLATFORM_IS_ACPI(vdev) ((vdev)->acpihid != NULL)
 
 static LIST_HEAD(reset_list);
+/* devices having MSI support */

nit: for devices using MSIs?

Quoted 125 lines
drivers/vfio/platform/vfio_platform_common.c +85 -2
+static LIST_HEAD(msi_list);
 static DEFINE_MUTEX(driver_lock);
 
 static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,
@@ -47,6 +50,25 @@ static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,
      return reset_fn;
 }
 
+static bool vfio_platform_lookup_msi(struct vfio_platform_device *vdev)
+{
+     bool has_msi = false;
+     struct vfio_platform_msi_node *iter;
+
+     mutex_lock(&driver_lock);
+     list_for_each_entry(iter, &msi_list, link) {
+             if (!strcmp(iter->compat, vdev->compat) &&
+                 try_module_get(iter->owner)) {
+                     vdev->msi_module = iter->owner;
+                     vdev->of_get_msi = iter->of_get_msi;
+                     has_msi = true;
+                     break;
+             }
+     }
+     mutex_unlock(&driver_lock);
+     return has_msi;
+}
+
 static int vfio_platform_acpi_probe(struct vfio_platform_device *vdev,
                                  struct device *dev)
 {
@@ -126,6 +148,19 @@ static int vfio_platform_get_reset(struct vfio_platform_device *vdev)
      return vdev->of_reset ? 0 : -ENOENT;
 }
 
+static int vfio_platform_get_msi(struct vfio_platform_device *vdev)
+{
+     bool has_msi;
+
+     has_msi = vfio_platform_lookup_msi(vdev);
+     if (!has_msi) {
+             request_module("vfio-msi:%s", vdev->compat);
+             has_msi = vfio_platform_lookup_msi(vdev);
+     }
+
+     return has_msi ? 0 : -ENOENT;
+}
+
 static void vfio_platform_put_reset(struct vfio_platform_device *vdev)
 {
      if (VFIO_PLATFORM_IS_ACPI(vdev))
@@ -135,6 +170,12 @@ static void vfio_platform_put_reset(struct vfio_platform_device *vdev)
              module_put(vdev->reset_module);
 }
 
+static void vfio_platform_put_msi(struct vfio_platform_device *vdev)
+{
+     if (vdev->of_get_msi)
+             module_put(vdev->msi_module);
+}
+
 static int vfio_platform_regions_init(struct vfio_platform_device *vdev)
 {
      int cnt = 0, i;
@@ -343,9 +384,17 @@ static long vfio_platform_ioctl(void *device_data,
 
      } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
              struct vfio_irq_info info;
+             struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
+             struct vfio_irq_info_cap_msi *msi_info = NULL;
+             int ext_irq_index = vdev->num_irqs - vdev->num_ext_irqs;
+             unsigned long capsz;
+             u32 index;
 
              minsz = offsetofend(struct vfio_irq_info, count);
 
+             /* For backward compatibility, cannot require this */
+             capsz = offsetofend(struct vfio_irq_info, cap_offset);
+
              if (copy_from_user(&info, (void __user *)arg, minsz))
                      return -EFAULT;
 
@@ -355,8 +404,94 @@ static long vfio_platform_ioctl(void *device_data,
              if (info.index >= vdev->num_irqs)
                      return -EINVAL;
 
-             info.flags = vdev->irqs[info.index].flags;
-             info.count = vdev->irqs[info.index].count;
+             if (info.argsz >= capsz)
+                     minsz = capsz;
+
+             index = info.index;
+
+             info.flags = vdev->irqs[index].flags;
+             info.count = vdev->irqs[index].count;
+
+             if (ext_irq_index - index == VFIO_EXT_IRQ_MSI) {
+                     struct vfio_irq_info_cap_type cap_type = {
+                             .header.id = VFIO_IRQ_INFO_CAP_TYPE,
+                             .header.version = 1 };
+                     struct vfio_platform_irq *irq;
+                     size_t msi_info_size;
+                     int num_msgs;
+                     int ret;
+                     int i;
+
+                     index = array_index_nospec(index,
+                                                vdev->num_irqs);
+                     irq = &vdev->irqs[index];
+
+                     cap_type.type = irq->type;
+                     cap_type.subtype = irq->subtype;
+
+                     ret = vfio_info_add_capability(&caps,
+                                                    &cap_type.header,
+                                                    sizeof(cap_type));
+                     if (ret)
+                             return ret;
+
+                     num_msgs = irq->num_ctx;
+                     if (num_msgs) {
+                             msi_info_size = struct_size(msi_info,
+                                                         msgs, num_msgs);
+
+                             msi_info = kzalloc(msi_info_size, GFP_KERNEL);
+                             if (!msi_info) {
Quoted 8 lines

drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 450

Quoted 6 lines
drivers/vfio/platform/vfio_platform_common.c +6 -0
+                                     kfree(caps.buf);
+                                     return -ENOMEM;
+                             }
+
+                             msi_info->header.id =
+                                             VFIO_IRQ_INFO_CAP_MSI_DESCS;

I thought you would remove this cap now the module is back. What is the motivation to keep it?

Hi Eric,
The earlier module was serving two purposes
a) Max number of MSI(s) supported by the device. Since the module is
back this information can be filled in .count.
b) Writing msi_msg(s) to devices for MSI sources.
We want b) to be handled in the user space so that they need not be
dependent on the kernel. VFIO_IRQ_INFO_CAP_MSI_DESCS is helping user space to get MSI configuration data and thus they can independently configure MSI sources. For example, user space can just ask for ‘N’ MSI vectors and related msi_msg for ‘N’ vectors is exported to user space using this CAP. User space can now use ‘N’ vectors to configure any ‘N’ MSI sources and these sources might be in any order. However, this design is different from vfio-pci as MSI configuration is being handled in kernel only.
Let us know if implementing VFIO_IRQ_INFO_CAP_MSI_DESCS does not fit with the CAP framework.

Thanks,
Vikas

Quoted 80 lines

Thanks

Eric

Quoted 76 lines
drivers/vfio/platform/vfio_platform_common.c +53 -2
+                             msi_info->header.version = 1;
+                             msi_info->nr_msgs = num_msgs;
+
+                             for (i = 0; i < num_msgs; i++) {
+                                     struct vfio_irq_ctx *ctx = &irq->ctx[i];
+
+                                     msi_info->msgs[i].addr_lo =
+                                                     ctx->msg.address_lo;
+                                     msi_info->msgs[i].addr_hi =
+                                                     ctx->msg.address_hi;
+                                     msi_info->msgs[i].data = ctx->msg.data;
+                             }
+
+                             ret = vfio_info_add_capability(&caps,
+                                                     &msi_info->header,
+                                                     msi_info_size);
+                             if (ret) {
+                                     kfree(msi_info);
+                                     kfree(caps.buf);
+                                     return ret;
+                             }
+                     }
+             }
+
+             if (caps.size) {
+                     info.flags |= VFIO_IRQ_INFO_FLAG_CAPS;
+                     if (info.argsz < sizeof(info) + caps.size) {
+                             info.argsz = sizeof(info) + caps.size;
+                             info.cap_offset = 0;
+                     } else {
+                             vfio_info_cap_shift(&caps, sizeof(info));
+                             if (copy_to_user((void __user *)arg +
+                                               sizeof(info), caps.buf,
+                                               caps.size)) {
+                                     kfree(msi_info);
+                                     kfree(caps.buf);
+                                     return -EFAULT;
+                             }
+                             info.cap_offset = sizeof(info);
+                     }
+
+                     kfree(msi_info);
+                     kfree(caps.buf);
+             }
 
              return copy_to_user((void __user *)arg, &info, minsz) ?
                      -EFAULT : 0;
@@ -365,6 +500,7 @@ static long vfio_platform_ioctl(void *device_data,
              struct vfio_irq_set hdr;
              u8 *data = NULL;
              int ret = 0;
+             int max;
              size_t data_size = 0;
 
              minsz = offsetofend(struct vfio_irq_set, count);
@@ -372,8 +508,14 @@ static long vfio_platform_ioctl(void *device_data,
              if (copy_from_user(&hdr, (void __user *)arg, minsz))
                      return -EFAULT;
 
-             ret = vfio_set_irqs_validate_and_prepare(&hdr, vdev->num_irqs,
-                                              vdev->num_irqs, &data_size);
+             if (hdr.index >= vdev->num_irqs)
+                     return -EINVAL;
+
+             max = vdev->irqs[hdr.index].count;
+
+             ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
+                                                      vdev->num_irqs,
+                                                      &data_size);
              if (ret)
                      return ret;
 
@@ -678,6 +820,10 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,
              return ret;
      }
 
Quoted 7 lines

drivers/vfio/platform/vfio_platform_common.c int vfio_platform_probe_common(struct vfio_platform_device *vdev, · line 825

Quoted 3 lines
drivers/vfio/platform/vfio_platform_common.c +3 -0
+     ret = vfio_platform_get_msi(vdev);
+     if (ret)
+             dev_info(vdev->device, "msi not supported\n");

I don't think we should output this message. This is printed for amd-xgbe which does not have msi so this is misleading. I would say either the vfio_platform_get_msi() can return an actual error or we return a void?

Will check on this what

Quoted 522 lines

Thanks

Eric

Quoted 517 lines
drivers/vfio/platform/vfio_platform_common.c +26 -0
+
      group = vfio_iommu_group_get(dev);
      if (!group) {
              dev_err(dev, "No IOMMU group for device %s\n", vdev->name);
@@ -697,6 +843,7 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,
 put_iommu:
      vfio_iommu_group_put(group, dev);
 put_reset:
+     vfio_platform_put_msi(vdev);
      vfio_platform_put_reset(vdev);
      return ret;
 }
@@ -744,6 +891,30 @@ void vfio_platform_unregister_reset(const char *compat,
 }
 EXPORT_SYMBOL_GPL(vfio_platform_unregister_reset);
 
+void __vfio_platform_register_msi(struct vfio_platform_msi_node *node)
+{
+     mutex_lock(&driver_lock);
+     list_add(&node->link, &msi_list);
+     mutex_unlock(&driver_lock);
+}
+EXPORT_SYMBOL_GPL(__vfio_platform_register_msi);
+
+void vfio_platform_unregister_msi(const char *compat)
+{
+     struct vfio_platform_msi_node *iter, *temp;
+
+     mutex_lock(&driver_lock);
+     list_for_each_entry_safe(iter, temp, &msi_list, link) {
+             if (!strcmp(iter->compat, compat)) {
+                     list_del(&iter->link);
+                     break;
+             }
+     }
+
+     mutex_unlock(&driver_lock);
+}
+EXPORT_SYMBOL_GPL(vfio_platform_unregister_msi);
+
 MODULE_VERSION(DRIVER_VERSION);
 MODULE_LICENSE("GPL v2");
 MODULE_AUTHOR(DRIVER_AUTHOR);
drivers/vfio/platform/vfio_platform_irq.c +245 -15
index c5b09ec0a3c9..e0f4696afedd 100644
@@ -8,10 +8,12 @@
 
 #include <linux/eventfd.h>
 #include <linux/interrupt.h>
+#include <linux/eventfd.h>
 #include <linux/slab.h>
 #include <linux/types.h>
 #include <linux/vfio.h>
 #include <linux/irq.h>
+#include <linux/msi.h>
 
 #include "vfio_platform_private.h"
 
@@ -253,6 +255,195 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,
      return 0;
 }
 
+/* MSI/MSIX */
+static irqreturn_t vfio_msihandler(int irq, void *arg)
+{
+     struct eventfd_ctx *trigger = arg;
+
+     eventfd_signal(trigger, 1);
+     return IRQ_HANDLED;
+}
+
+static void msi_write(struct msi_desc *desc, struct msi_msg *msg)
+{
+     int i;
+     struct vfio_platform_irq *irq;
+     u16 index = desc->platform.msi_index;
+     struct device *dev = msi_desc_to_dev(desc);
+     struct vfio_device *device = dev_get_drvdata(dev);
+     struct vfio_platform_device *vdev = (struct vfio_platform_device *)
+                                             vfio_device_data(device);
+
+     for (i = 0; i < vdev->num_irqs; i++)
+             if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
+                     irq = &vdev->irqs[i];
+
+     irq->ctx[index].msg.address_lo = msg->address_lo;
+     irq->ctx[index].msg.address_hi = msg->address_hi;
+     irq->ctx[index].msg.data = msg->data;
+}
+
+static int vfio_msi_enable(struct vfio_platform_device *vdev,
+                        struct vfio_platform_irq *irq, int nvec)
+{
+     int ret;
+     int msi_idx = 0;
+     struct msi_desc *desc;
+     struct device *dev = vdev->device;
+
+     irq->ctx = kcalloc(nvec, sizeof(struct vfio_irq_ctx), GFP_KERNEL);
+     if (!irq->ctx)
+             return -ENOMEM;
+
+     /* Allocate platform MSIs */
+     ret = platform_msi_domain_alloc_irqs(dev, nvec, msi_write);
+     if (ret < 0) {
+             kfree(irq->ctx);
+             return ret;
+     }
+
+     for_each_msi_entry(desc, dev) {
+             irq->ctx[msi_idx].hwirq = desc->irq;
+             msi_idx++;
+     }
+
+     irq->num_ctx = nvec;
+     irq->config_msi = 1;
+
+     return 0;
+}
+
+static int vfio_msi_set_vector_signal(struct vfio_platform_irq *irq,
+                                   int vector, int fd)
+{
+     struct eventfd_ctx *trigger;
+     int irq_num, ret;
+
+     if (vector < 0 || vector >= irq->num_ctx)
+             return -EINVAL;
+
+     irq_num = irq->ctx[vector].hwirq;
+
+     if (irq->ctx[vector].trigger) {
+             free_irq(irq_num, irq->ctx[vector].trigger);
+             kfree(irq->ctx[vector].name);
+             eventfd_ctx_put(irq->ctx[vector].trigger);
+             irq->ctx[vector].trigger = NULL;
+     }
+
+     if (fd < 0)
+             return 0;
+
+     irq->ctx[vector].name = kasprintf(GFP_KERNEL,
+                                       "vfio-msi[%d]", vector);
+     if (!irq->ctx[vector].name)
+             return -ENOMEM;
+
+     trigger = eventfd_ctx_fdget(fd);
+     if (IS_ERR(trigger)) {
+             kfree(irq->ctx[vector].name);
+             return PTR_ERR(trigger);
+     }
+
+     ret = request_irq(irq_num, vfio_msihandler, 0,
+                       irq->ctx[vector].name, trigger);
+     if (ret) {
+             kfree(irq->ctx[vector].name);
+             eventfd_ctx_put(trigger);
+             return ret;
+     }
+
+     irq->ctx[vector].trigger = trigger;
+
+     return 0;
+}
+
+static int vfio_msi_set_block(struct vfio_platform_irq *irq, unsigned int start,
+                           unsigned int count, int32_t *fds)
+{
+     int i, j, ret = 0;
+
+     if (start >= irq->num_ctx || start + count > irq->num_ctx)
+             return -EINVAL;
+
+     for (i = 0, j = start; i < count && !ret; i++, j++) {
+             int fd = fds ? fds[i] : -1;
+
+             ret = vfio_msi_set_vector_signal(irq, j, fd);
+     }
+
+     if (ret) {
+             for (--j; j >= (int)start; j--)
+                     vfio_msi_set_vector_signal(irq, j, -1);
+     }
+
+     return ret;
+}
+
+static void vfio_msi_disable(struct vfio_platform_device *vdev,
+                          struct vfio_platform_irq *irq)
+{
+     struct device *dev = vdev->device;
+
+     vfio_msi_set_block(irq, 0, irq->num_ctx, NULL);
+
+     platform_msi_domain_free_irqs(dev);
+
+     irq->config_msi = 0;
+     irq->num_ctx = 0;
+
+     kfree(irq->ctx);
+}
+
+static int vfio_set_msi_trigger(struct vfio_platform_device *vdev,
+                             unsigned int index, unsigned int start,
+                             unsigned int count, uint32_t flags, void *data)
+{
+     int i;
+     struct vfio_platform_irq *irq = &vdev->irqs[index];
+
+     if (start + count > irq->count)
+             return -EINVAL;
+
+     if (!count && (flags & VFIO_IRQ_SET_DATA_NONE)) {
+             vfio_msi_disable(vdev, irq);
+             return 0;
+     }
+
+     if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
+             s32 *fds = data;
+             int ret;
+
+             if (irq->config_msi)
+                     return vfio_msi_set_block(irq, start, count,
+                                               fds);
+             ret = vfio_msi_enable(vdev, irq, start + count);
+             if (ret)
+                     return ret;
+
+             ret = vfio_msi_set_block(irq, start, count, fds);
+             if (ret)
+                     vfio_msi_disable(vdev, irq);
+
+             return ret;
+     }
+
+     for (i = start; i < start + count; i++) {
+             if (!irq->ctx[i].trigger)
+                     continue;
+             if (flags & VFIO_IRQ_SET_DATA_NONE) {
+                     eventfd_signal(irq->ctx[i].trigger, 1);
+             } else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
+                     u8 *bools = data;
+
+                     if (bools[i - start])
+                             eventfd_signal(irq->ctx[i].trigger, 1);
+             }
+     }
+
+     return 0;
+}
+
 int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
                               uint32_t flags, unsigned index, unsigned start,
                               unsigned count, void *data)
@@ -261,16 +452,29 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
                  unsigned start, unsigned count, uint32_t flags,
                  void *data) = NULL;
 
-     switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
-     case VFIO_IRQ_SET_ACTION_MASK:
-             func = vfio_platform_set_irq_mask;
-             break;
-     case VFIO_IRQ_SET_ACTION_UNMASK:
-             func = vfio_platform_set_irq_unmask;
-             break;
-     case VFIO_IRQ_SET_ACTION_TRIGGER:
-             func = vfio_platform_set_irq_trigger;
-             break;
+     struct vfio_platform_irq *irq = &vdev->irqs[index];
+
+     if (irq->type == VFIO_IRQ_TYPE_MSI) {
+             switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
+             case VFIO_IRQ_SET_ACTION_MASK:
+             case VFIO_IRQ_SET_ACTION_UNMASK:
+                     break;
+             case VFIO_IRQ_SET_ACTION_TRIGGER:
+                     func = vfio_set_msi_trigger;
+                     break;
+             }
+     } else {
+             switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
+             case VFIO_IRQ_SET_ACTION_MASK:
+                     func = vfio_platform_set_irq_mask;
+                     break;
+             case VFIO_IRQ_SET_ACTION_UNMASK:
+                     func = vfio_platform_set_irq_unmask;
+                     break;
+             case VFIO_IRQ_SET_ACTION_TRIGGER:
+                     func = vfio_platform_set_irq_trigger;
+                     break;
+             }
      }
 
      if (!func)
@@ -281,12 +485,21 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 
 int vfio_platform_irq_init(struct vfio_platform_device *vdev)
 {
-     int cnt = 0, i;
+     int i;
+     int cnt = 0;
+     int num_irqs;
+     int msi_cnt = 0;
 
      while (vdev->get_irq(vdev, cnt) >= 0)
              cnt++;
 
-     vdev->irqs = kcalloc(cnt, sizeof(struct vfio_platform_irq), GFP_KERNEL);
+     if (vdev->of_get_msi) {
+             msi_cnt = vdev->of_get_msi(vdev);
+             num_irqs++;
+     }
+
+     vdev->irqs = kcalloc(num_irqs, sizeof(struct vfio_platform_irq),
+                          GFP_KERNEL);
      if (!vdev->irqs)
              return -ENOMEM;
 
@@ -309,7 +522,19 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev)
              vdev->irqs[i].masked = false;
      }
 
-     vdev->num_irqs = cnt;
+     /*
+      * MSI block is added at last index and it is an ext irq
+      */
+     if (msi_cnt > 0) {
+             vdev->irqs[i].flags = VFIO_IRQ_INFO_EVENTFD;
+             vdev->irqs[i].count = msi_cnt;
+             vdev->irqs[i].hwirq = 0;
+             vdev->irqs[i].masked = false;
+             vdev->irqs[i].type = VFIO_IRQ_TYPE_MSI;
+             vdev->num_ext_irqs = 1;
+     }
+
+     vdev->num_irqs = num_irqs;
 
      return 0;
 err:
@@ -321,8 +546,13 @@ void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev)
 {
      int i;
 
-     for (i = 0; i < vdev->num_irqs; i++)
-             vfio_set_trigger(vdev, i, -1, NULL);
+     for (i = 0; i < vdev->num_irqs; i++) {
+             if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
+                     vfio_set_msi_trigger(vdev, i, 0, 0,
+                                          VFIO_IRQ_SET_DATA_NONE, NULL);
+             else
+                     vfio_set_trigger(vdev, i, -1, NULL);
+     }
 
      vdev->num_irqs = 0;
      kfree(vdev->irqs);
drivers/vfio/platform/vfio_platform_private.h +32 -0
index 289089910643..1307feddda21 100644
@@ -9,6 +9,7 @@
 
 #include <linux/types.h>
 #include <linux/interrupt.h>
+#include <linux/msi.h>
 
 #define VFIO_PLATFORM_OFFSET_SHIFT   40
 #define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
@@ -19,9 +20,21 @@
 #define VFIO_PLATFORM_INDEX_TO_OFFSET(index) \
      ((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT)
 
+/* IRQ index for MSI in ext IRQs */
+#define VFIO_EXT_IRQ_MSI     0
+
+struct vfio_irq_ctx {
+     int                     hwirq;
+     char                    *name;
+     struct msi_msg          msg;
+     struct eventfd_ctx      *trigger;
+};
+
 struct vfio_platform_irq {
      u32                     flags;
      u32                     count;
+     int                     num_ctx;
+     struct vfio_irq_ctx     *ctx;
      int                     hwirq;
      char                    *name;
      struct eventfd_ctx      *trigger;
@@ -29,6 +42,11 @@ struct vfio_platform_irq {
      spinlock_t              lock;
      struct virqfd           *unmask;
      struct virqfd           *mask;
+
+     /* for extended irqs */
+     u32                     type;
+     u32                     subtype;
+     int                     config_msi;
 };
 
 struct vfio_platform_region {
@@ -46,12 +64,14 @@ struct vfio_platform_device {
      u32                             num_regions;
      struct vfio_platform_irq        *irqs;
      u32                             num_irqs;
+     int                             num_ext_irqs;
      int                             refcnt;
      struct mutex                    igate;
      struct module                   *parent_module;
      const char                      *compat;
      const char                      *acpihid;
      struct module                   *reset_module;
+     struct module                   *msi_module;
      struct device                   *device;
 
      /*
@@ -65,11 +85,13 @@ struct vfio_platform_device {
              (*get_resource)(struct vfio_platform_device *vdev, int i);
      int     (*get_irq)(struct vfio_platform_device *vdev, int i);
      int     (*of_reset)(struct vfio_platform_device *vdev);
+     u32     (*of_get_msi)(struct vfio_platform_device *vdev);
 
      bool                            reset_required;
 };
 
 typedef int (*vfio_platform_reset_fn_t)(struct vfio_platform_device *vdev);
+typedef u32 (*vfio_platform_get_msi_fn_t)(struct vfio_platform_device *vdev);
 
 struct vfio_platform_reset_node {
      struct list_head link;
@@ -78,6 +100,13 @@ struct vfio_platform_reset_node {
      vfio_platform_reset_fn_t of_reset;
 };
 
+struct vfio_platform_msi_node {
+     struct list_head link;
+     char *compat;
+     struct module *owner;
+     vfio_platform_get_msi_fn_t of_get_msi;
+};
+
 extern int vfio_platform_probe_common(struct vfio_platform_device *vdev,
                                    struct device *dev);
 extern struct vfio_platform_device *vfio_platform_remove_common
@@ -94,6 +123,9 @@ extern int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 extern void __vfio_platform_register_reset(struct vfio_platform_reset_node *n);
 extern void vfio_platform_unregister_reset(const char *compat,
                                         vfio_platform_reset_fn_t fn);
+void __vfio_platform_register_msi(struct vfio_platform_msi_node *n);
+void vfio_platform_unregister_msi(const char *compat);
+
 #define vfio_platform_register_reset(__compat, __reset)              \
 static struct vfio_platform_reset_node __reset ## _node = {  \
      .owner = THIS_MODULE,                                   \
include/uapi/linux/vfio.h +44 -0
index d1812777139f..53a7ff2b524e 100644
@@ -697,11 +697,55 @@ struct vfio_irq_info {
 #define VFIO_IRQ_INFO_MASKABLE               (1 << 1)
 #define VFIO_IRQ_INFO_AUTOMASKED     (1 << 2)
 #define VFIO_IRQ_INFO_NORESIZE               (1 << 3)
+#define VFIO_IRQ_INFO_FLAG_CAPS              (1 << 4) /* Info supports caps */
      __u32   index;          /* IRQ index */
      __u32   count;          /* Number of IRQs within this index */
+     __u32   cap_offset;     /* Offset within info struct of first cap */
 };
 #define VFIO_DEVICE_GET_IRQ_INFO     _IO(VFIO_TYPE, VFIO_BASE + 9)
 
+/*
+ * The irq type capability allows IRQs unique to a specific device or
+ * class of devices to be exposed.
+ *
+ * The structures below define version 1 of this capability.
+ */
+#define VFIO_IRQ_INFO_CAP_TYPE               3
+
+struct vfio_irq_info_cap_type {
+     struct vfio_info_cap_header header;
+     __u32 type;     /* global per bus driver */
+     __u32 subtype;  /* type specific */
+};
+
+/*
+ * List of IRQ types, global per bus driver.
+ * If you introduce a new type, please add it here.
+ */
+
+/* Non PCI devices having MSI(s) support */
+#define VFIO_IRQ_TYPE_MSI            (1)
+
+/*
+ * The msi capability allows the user to use the msi msg to
+ * configure the iova for the msi configuration.
+ * The structures below define version 1 of this capability.
+ */
+#define VFIO_IRQ_INFO_CAP_MSI_DESCS  4
+
+struct vfio_irq_msi_msg {
+     __u32   addr_lo;
+     __u32   addr_hi;
+     __u32   data;
+};
+
+struct vfio_irq_info_cap_msi {
+     struct vfio_info_cap_header header;
+     __u32   nr_msgs;
+     __u32   reserved;
+     struct vfio_irq_msi_msg msgs[];
+};
+
 /**
  * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set)
  *
Signature · 11 lines
This electronic communication and the information and any files transmitted 
with it, or attached to it, are confidential and are intended solely for 
the use of the individual or entity to whom it is addressed and may contain 
information that is confidential, legally privileged, protected by privacy 
laws, or otherwise restricted from disclosure to anyone else. If you are 
not the intended recipient or the person responsible for delivering the 
e-mail to the intended recipient, you are hereby notified that any use, 
copying, distributing, dissemination, forwarding, printing, or copying of 
this e-mail is strictly prohibited. If you received this e-mail in error, 
please return the e-mail to the sender, delete it from your computer, and 
destroy any printed copy of it.
maintainer
#38 lore raw

Re: [RFC v3 1/2] vfio/platform: add support for msi

Hi Vikas,

Quoted 169 lines · Vikas Gupta
Quoted 159 lines · Auger Eric

Hi Vikas,

Quoted 25 lines · Vikas Gupta

MSI support for platform devices.The MSI block is added as an extended IRQ which exports caps VFIO_IRQ_INFO_CAP_TYPE and VFIO_IRQ_INFO_CAP_MSI_DESCS.

---
 drivers/vfio/platform/vfio_platform_common.c  | 179 +++++++++++-
 drivers/vfio/platform/vfio_platform_irq.c     | 260 +++++++++++++++++-
 drivers/vfio/platform/vfio_platform_private.h |  32 +++
 include/uapi/linux/vfio.h                     |  44 +++
 4 files changed, 496 insertions(+), 19 deletions(-)
drivers/vfio/platform/vfio_platform_common.c +1 -0
index fb4b385191f2..c936852f35d7 100644
@@ -16,6 +16,7 @@
 #include <linux/types.h>
 #include <linux/uaccess.h>
 #include <linux/vfio.h>
+#include <linux/nospec.h>
 
 #include "vfio_platform_private.h"
 

drivers/vfio/platform/vfio_platform_common.c line 30

Quoted 4 lines
drivers/vfio/platform/vfio_platform_common.c +1 -0
@@ -26,6 +27,8 @@
 #define VFIO_PLATFORM_IS_ACPI(vdev) ((vdev)->acpihid != NULL)
 
 static LIST_HEAD(reset_list);
+/* devices having MSI support */

nit: for devices using MSIs?

Quoted 125 lines
drivers/vfio/platform/vfio_platform_common.c +85 -2
+static LIST_HEAD(msi_list);
 static DEFINE_MUTEX(driver_lock);
 
 static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,
@@ -47,6 +50,25 @@ static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,
      return reset_fn;
 }
 
+static bool vfio_platform_lookup_msi(struct vfio_platform_device *vdev)
+{
+     bool has_msi = false;
+     struct vfio_platform_msi_node *iter;
+
+     mutex_lock(&driver_lock);
+     list_for_each_entry(iter, &msi_list, link) {
+             if (!strcmp(iter->compat, vdev->compat) &&
+                 try_module_get(iter->owner)) {
+                     vdev->msi_module = iter->owner;
+                     vdev->of_get_msi = iter->of_get_msi;
+                     has_msi = true;
+                     break;
+             }
+     }
+     mutex_unlock(&driver_lock);
+     return has_msi;
+}
+
 static int vfio_platform_acpi_probe(struct vfio_platform_device *vdev,
                                  struct device *dev)
 {
@@ -126,6 +148,19 @@ static int vfio_platform_get_reset(struct vfio_platform_device *vdev)
      return vdev->of_reset ? 0 : -ENOENT;
 }
 
+static int vfio_platform_get_msi(struct vfio_platform_device *vdev)
+{
+     bool has_msi;
+
+     has_msi = vfio_platform_lookup_msi(vdev);
+     if (!has_msi) {
+             request_module("vfio-msi:%s", vdev->compat);
+             has_msi = vfio_platform_lookup_msi(vdev);
+     }
+
+     return has_msi ? 0 : -ENOENT;
+}
+
 static void vfio_platform_put_reset(struct vfio_platform_device *vdev)
 {
      if (VFIO_PLATFORM_IS_ACPI(vdev))
@@ -135,6 +170,12 @@ static void vfio_platform_put_reset(struct vfio_platform_device *vdev)
              module_put(vdev->reset_module);
 }
 
+static void vfio_platform_put_msi(struct vfio_platform_device *vdev)
+{
+     if (vdev->of_get_msi)
+             module_put(vdev->msi_module);
+}
+
 static int vfio_platform_regions_init(struct vfio_platform_device *vdev)
 {
      int cnt = 0, i;
@@ -343,9 +384,17 @@ static long vfio_platform_ioctl(void *device_data,
 
      } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
              struct vfio_irq_info info;
+             struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
+             struct vfio_irq_info_cap_msi *msi_info = NULL;
+             int ext_irq_index = vdev->num_irqs - vdev->num_ext_irqs;
+             unsigned long capsz;
+             u32 index;
 
              minsz = offsetofend(struct vfio_irq_info, count);
 
+             /* For backward compatibility, cannot require this */
+             capsz = offsetofend(struct vfio_irq_info, cap_offset);
+
              if (copy_from_user(&info, (void __user *)arg, minsz))
                      return -EFAULT;
 
@@ -355,8 +404,94 @@ static long vfio_platform_ioctl(void *device_data,
              if (info.index >= vdev->num_irqs)
                      return -EINVAL;
 
-             info.flags = vdev->irqs[info.index].flags;
-             info.count = vdev->irqs[info.index].count;
+             if (info.argsz >= capsz)
+                     minsz = capsz;
+
+             index = info.index;
+
+             info.flags = vdev->irqs[index].flags;
+             info.count = vdev->irqs[index].count;
+
+             if (ext_irq_index - index == VFIO_EXT_IRQ_MSI) {
+                     struct vfio_irq_info_cap_type cap_type = {
+                             .header.id = VFIO_IRQ_INFO_CAP_TYPE,
+                             .header.version = 1 };
+                     struct vfio_platform_irq *irq;
+                     size_t msi_info_size;
+                     int num_msgs;
+                     int ret;
+                     int i;
+
+                     index = array_index_nospec(index,
+                                                vdev->num_irqs);
+                     irq = &vdev->irqs[index];
+
+                     cap_type.type = irq->type;
+                     cap_type.subtype = irq->subtype;
+
+                     ret = vfio_info_add_capability(&caps,
+                                                    &cap_type.header,
+                                                    sizeof(cap_type));
+                     if (ret)
+                             return ret;
+
+                     num_msgs = irq->num_ctx;
+                     if (num_msgs) {
+                             msi_info_size = struct_size(msi_info,
+                                                         msgs, num_msgs);
+
+                             msi_info = kzalloc(msi_info_size, GFP_KERNEL);
+                             if (!msi_info) {
Quoted 8 lines

drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 450

Quoted 6 lines
drivers/vfio/platform/vfio_platform_common.c +6 -0
+                                     kfree(caps.buf);
+                                     return -ENOMEM;
+                             }
+
+                             msi_info->header.id =
+                                             VFIO_IRQ_INFO_CAP_MSI_DESCS;

I thought you would remove this cap now the module is back. What is the motivation to keep it?

Quoted 4 lines

Hi Eric,
The earlier module was serving two purposes
a) Max number of MSI(s) supported by the device. Since the module is
back this information can be filled in .count.

understood

Quoted 3 lines

b) Writing msi_msg(s) to devices for MSI sources.
We want b) to be handled in the user space so that they need not be
dependent on the kernel.

I fail to understand the problem with the "dependency on the kernel". What is the functional difference when exposing this to the userspace, please could you highlight the benefits?

VFIO_IRQ_INFO_CAP_MSI_DESCS is helping user

Quoted 3 lines

space to get MSI configuration data and thus they can independently configure MSI sources. For example, user space can just ask for ‘N’ MSI vectors and related msi_msg for ‘N’ vectors

this is induced by vfio_msi_enable(), right?

is exported to user

Quoted 6 lines

space using this CAP. User space can now use ‘N’ vectors to configure any ‘N’ MSI sources and these sources might be in any order. However, this design is different from vfio-pci as MSI configuration is being handled in kernel only.
Let us know if implementing VFIO_IRQ_INFO_CAP_MSI_DESCS does not fit with the CAP framework.

I think it is OK reporting them through the cap but I fail to understand the benefits.

Thanks

Eric

Quoted 615 lines

Thanks,
Vikas

Quoted 80 lines

Thanks

Eric

Quoted 76 lines
drivers/vfio/platform/vfio_platform_common.c +53 -2
+                             msi_info->header.version = 1;
+                             msi_info->nr_msgs = num_msgs;
+
+                             for (i = 0; i < num_msgs; i++) {
+                                     struct vfio_irq_ctx *ctx = &irq->ctx[i];
+
+                                     msi_info->msgs[i].addr_lo =
+                                                     ctx->msg.address_lo;
+                                     msi_info->msgs[i].addr_hi =
+                                                     ctx->msg.address_hi;
+                                     msi_info->msgs[i].data = ctx->msg.data;
+                             }
+
+                             ret = vfio_info_add_capability(&caps,
+                                                     &msi_info->header,
+                                                     msi_info_size);
+                             if (ret) {
+                                     kfree(msi_info);
+                                     kfree(caps.buf);
+                                     return ret;
+                             }
+                     }
+             }
+
+             if (caps.size) {
+                     info.flags |= VFIO_IRQ_INFO_FLAG_CAPS;
+                     if (info.argsz < sizeof(info) + caps.size) {
+                             info.argsz = sizeof(info) + caps.size;
+                             info.cap_offset = 0;
+                     } else {
+                             vfio_info_cap_shift(&caps, sizeof(info));
+                             if (copy_to_user((void __user *)arg +
+                                               sizeof(info), caps.buf,
+                                               caps.size)) {
+                                     kfree(msi_info);
+                                     kfree(caps.buf);
+                                     return -EFAULT;
+                             }
+                             info.cap_offset = sizeof(info);
+                     }
+
+                     kfree(msi_info);
+                     kfree(caps.buf);
+             }
 
              return copy_to_user((void __user *)arg, &info, minsz) ?
                      -EFAULT : 0;
@@ -365,6 +500,7 @@ static long vfio_platform_ioctl(void *device_data,
              struct vfio_irq_set hdr;
              u8 *data = NULL;
              int ret = 0;
+             int max;
              size_t data_size = 0;
 
              minsz = offsetofend(struct vfio_irq_set, count);
@@ -372,8 +508,14 @@ static long vfio_platform_ioctl(void *device_data,
              if (copy_from_user(&hdr, (void __user *)arg, minsz))
                      return -EFAULT;
 
-             ret = vfio_set_irqs_validate_and_prepare(&hdr, vdev->num_irqs,
-                                              vdev->num_irqs, &data_size);
+             if (hdr.index >= vdev->num_irqs)
+                     return -EINVAL;
+
+             max = vdev->irqs[hdr.index].count;
+
+             ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
+                                                      vdev->num_irqs,
+                                                      &data_size);
              if (ret)
                      return ret;
 
@@ -678,6 +820,10 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,
              return ret;
      }
 
Quoted 7 lines

drivers/vfio/platform/vfio_platform_common.c int vfio_platform_probe_common(struct vfio_platform_device *vdev, · line 825

Quoted 3 lines
drivers/vfio/platform/vfio_platform_common.c +3 -0
+     ret = vfio_platform_get_msi(vdev);
+     if (ret)
+             dev_info(vdev->device, "msi not supported\n");

I don't think we should output this message. This is printed for amd-xgbe which does not have msi so this is misleading. I would say either the vfio_platform_get_msi() can return an actual error or we return a void?

Will check on this what

Quoted 522 lines

Thanks

Eric

Quoted 517 lines
drivers/vfio/platform/vfio_platform_common.c +26 -0
+
      group = vfio_iommu_group_get(dev);
      if (!group) {
              dev_err(dev, "No IOMMU group for device %s\n", vdev->name);
@@ -697,6 +843,7 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,
 put_iommu:
      vfio_iommu_group_put(group, dev);
 put_reset:
+     vfio_platform_put_msi(vdev);
      vfio_platform_put_reset(vdev);
      return ret;
 }
@@ -744,6 +891,30 @@ void vfio_platform_unregister_reset(const char *compat,
 }
 EXPORT_SYMBOL_GPL(vfio_platform_unregister_reset);
 
+void __vfio_platform_register_msi(struct vfio_platform_msi_node *node)
+{
+     mutex_lock(&driver_lock);
+     list_add(&node->link, &msi_list);
+     mutex_unlock(&driver_lock);
+}
+EXPORT_SYMBOL_GPL(__vfio_platform_register_msi);
+
+void vfio_platform_unregister_msi(const char *compat)
+{
+     struct vfio_platform_msi_node *iter, *temp;
+
+     mutex_lock(&driver_lock);
+     list_for_each_entry_safe(iter, temp, &msi_list, link) {
+             if (!strcmp(iter->compat, compat)) {
+                     list_del(&iter->link);
+                     break;
+             }
+     }
+
+     mutex_unlock(&driver_lock);
+}
+EXPORT_SYMBOL_GPL(vfio_platform_unregister_msi);
+
 MODULE_VERSION(DRIVER_VERSION);
 MODULE_LICENSE("GPL v2");
 MODULE_AUTHOR(DRIVER_AUTHOR);
drivers/vfio/platform/vfio_platform_irq.c +245 -15
index c5b09ec0a3c9..e0f4696afedd 100644
@@ -8,10 +8,12 @@
 
 #include <linux/eventfd.h>
 #include <linux/interrupt.h>
+#include <linux/eventfd.h>
 #include <linux/slab.h>
 #include <linux/types.h>
 #include <linux/vfio.h>
 #include <linux/irq.h>
+#include <linux/msi.h>
 
 #include "vfio_platform_private.h"
 
@@ -253,6 +255,195 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,
      return 0;
 }
 
+/* MSI/MSIX */
+static irqreturn_t vfio_msihandler(int irq, void *arg)
+{
+     struct eventfd_ctx *trigger = arg;
+
+     eventfd_signal(trigger, 1);
+     return IRQ_HANDLED;
+}
+
+static void msi_write(struct msi_desc *desc, struct msi_msg *msg)
+{
+     int i;
+     struct vfio_platform_irq *irq;
+     u16 index = desc->platform.msi_index;
+     struct device *dev = msi_desc_to_dev(desc);
+     struct vfio_device *device = dev_get_drvdata(dev);
+     struct vfio_platform_device *vdev = (struct vfio_platform_device *)
+                                             vfio_device_data(device);
+
+     for (i = 0; i < vdev->num_irqs; i++)
+             if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
+                     irq = &vdev->irqs[i];
+
+     irq->ctx[index].msg.address_lo = msg->address_lo;
+     irq->ctx[index].msg.address_hi = msg->address_hi;
+     irq->ctx[index].msg.data = msg->data;
+}
+
+static int vfio_msi_enable(struct vfio_platform_device *vdev,
+                        struct vfio_platform_irq *irq, int nvec)
+{
+     int ret;
+     int msi_idx = 0;
+     struct msi_desc *desc;
+     struct device *dev = vdev->device;
+
+     irq->ctx = kcalloc(nvec, sizeof(struct vfio_irq_ctx), GFP_KERNEL);
+     if (!irq->ctx)
+             return -ENOMEM;
+
+     /* Allocate platform MSIs */
+     ret = platform_msi_domain_alloc_irqs(dev, nvec, msi_write);
+     if (ret < 0) {
+             kfree(irq->ctx);
+             return ret;
+     }
+
+     for_each_msi_entry(desc, dev) {
+             irq->ctx[msi_idx].hwirq = desc->irq;
+             msi_idx++;
+     }
+
+     irq->num_ctx = nvec;
+     irq->config_msi = 1;
+
+     return 0;
+}
+
+static int vfio_msi_set_vector_signal(struct vfio_platform_irq *irq,
+                                   int vector, int fd)
+{
+     struct eventfd_ctx *trigger;
+     int irq_num, ret;
+
+     if (vector < 0 || vector >= irq->num_ctx)
+             return -EINVAL;
+
+     irq_num = irq->ctx[vector].hwirq;
+
+     if (irq->ctx[vector].trigger) {
+             free_irq(irq_num, irq->ctx[vector].trigger);
+             kfree(irq->ctx[vector].name);
+             eventfd_ctx_put(irq->ctx[vector].trigger);
+             irq->ctx[vector].trigger = NULL;
+     }
+
+     if (fd < 0)
+             return 0;
+
+     irq->ctx[vector].name = kasprintf(GFP_KERNEL,
+                                       "vfio-msi[%d]", vector);
+     if (!irq->ctx[vector].name)
+             return -ENOMEM;
+
+     trigger = eventfd_ctx_fdget(fd);
+     if (IS_ERR(trigger)) {
+             kfree(irq->ctx[vector].name);
+             return PTR_ERR(trigger);
+     }
+
+     ret = request_irq(irq_num, vfio_msihandler, 0,
+                       irq->ctx[vector].name, trigger);
+     if (ret) {
+             kfree(irq->ctx[vector].name);
+             eventfd_ctx_put(trigger);
+             return ret;
+     }
+
+     irq->ctx[vector].trigger = trigger;
+
+     return 0;
+}
+
+static int vfio_msi_set_block(struct vfio_platform_irq *irq, unsigned int start,
+                           unsigned int count, int32_t *fds)
+{
+     int i, j, ret = 0;
+
+     if (start >= irq->num_ctx || start + count > irq->num_ctx)
+             return -EINVAL;
+
+     for (i = 0, j = start; i < count && !ret; i++, j++) {
+             int fd = fds ? fds[i] : -1;
+
+             ret = vfio_msi_set_vector_signal(irq, j, fd);
+     }
+
+     if (ret) {
+             for (--j; j >= (int)start; j--)
+                     vfio_msi_set_vector_signal(irq, j, -1);
+     }
+
+     return ret;
+}
+
+static void vfio_msi_disable(struct vfio_platform_device *vdev,
+                          struct vfio_platform_irq *irq)
+{
+     struct device *dev = vdev->device;
+
+     vfio_msi_set_block(irq, 0, irq->num_ctx, NULL);
+
+     platform_msi_domain_free_irqs(dev);
+
+     irq->config_msi = 0;
+     irq->num_ctx = 0;
+
+     kfree(irq->ctx);
+}
+
+static int vfio_set_msi_trigger(struct vfio_platform_device *vdev,
+                             unsigned int index, unsigned int start,
+                             unsigned int count, uint32_t flags, void *data)
+{
+     int i;
+     struct vfio_platform_irq *irq = &vdev->irqs[index];
+
+     if (start + count > irq->count)
+             return -EINVAL;
+
+     if (!count && (flags & VFIO_IRQ_SET_DATA_NONE)) {
+             vfio_msi_disable(vdev, irq);
+             return 0;
+     }
+
+     if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
+             s32 *fds = data;
+             int ret;
+
+             if (irq->config_msi)
+                     return vfio_msi_set_block(irq, start, count,
+                                               fds);
+             ret = vfio_msi_enable(vdev, irq, start + count);
+             if (ret)
+                     return ret;
+
+             ret = vfio_msi_set_block(irq, start, count, fds);
+             if (ret)
+                     vfio_msi_disable(vdev, irq);
+
+             return ret;
+     }
+
+     for (i = start; i < start + count; i++) {
+             if (!irq->ctx[i].trigger)
+                     continue;
+             if (flags & VFIO_IRQ_SET_DATA_NONE) {
+                     eventfd_signal(irq->ctx[i].trigger, 1);
+             } else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
+                     u8 *bools = data;
+
+                     if (bools[i - start])
+                             eventfd_signal(irq->ctx[i].trigger, 1);
+             }
+     }
+
+     return 0;
+}
+
 int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
                               uint32_t flags, unsigned index, unsigned start,
                               unsigned count, void *data)
@@ -261,16 +452,29 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
                  unsigned start, unsigned count, uint32_t flags,
                  void *data) = NULL;
 
-     switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
-     case VFIO_IRQ_SET_ACTION_MASK:
-             func = vfio_platform_set_irq_mask;
-             break;
-     case VFIO_IRQ_SET_ACTION_UNMASK:
-             func = vfio_platform_set_irq_unmask;
-             break;
-     case VFIO_IRQ_SET_ACTION_TRIGGER:
-             func = vfio_platform_set_irq_trigger;
-             break;
+     struct vfio_platform_irq *irq = &vdev->irqs[index];
+
+     if (irq->type == VFIO_IRQ_TYPE_MSI) {
+             switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
+             case VFIO_IRQ_SET_ACTION_MASK:
+             case VFIO_IRQ_SET_ACTION_UNMASK:
+                     break;
+             case VFIO_IRQ_SET_ACTION_TRIGGER:
+                     func = vfio_set_msi_trigger;
+                     break;
+             }
+     } else {
+             switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
+             case VFIO_IRQ_SET_ACTION_MASK:
+                     func = vfio_platform_set_irq_mask;
+                     break;
+             case VFIO_IRQ_SET_ACTION_UNMASK:
+                     func = vfio_platform_set_irq_unmask;
+                     break;
+             case VFIO_IRQ_SET_ACTION_TRIGGER:
+                     func = vfio_platform_set_irq_trigger;
+                     break;
+             }
      }
 
      if (!func)
@@ -281,12 +485,21 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 
 int vfio_platform_irq_init(struct vfio_platform_device *vdev)
 {
-     int cnt = 0, i;
+     int i;
+     int cnt = 0;
+     int num_irqs;
+     int msi_cnt = 0;
 
      while (vdev->get_irq(vdev, cnt) >= 0)
              cnt++;
 
-     vdev->irqs = kcalloc(cnt, sizeof(struct vfio_platform_irq), GFP_KERNEL);
+     if (vdev->of_get_msi) {
+             msi_cnt = vdev->of_get_msi(vdev);
+             num_irqs++;
+     }
+
+     vdev->irqs = kcalloc(num_irqs, sizeof(struct vfio_platform_irq),
+                          GFP_KERNEL);
      if (!vdev->irqs)
              return -ENOMEM;
 
@@ -309,7 +522,19 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev)
              vdev->irqs[i].masked = false;
      }
 
-     vdev->num_irqs = cnt;
+     /*
+      * MSI block is added at last index and it is an ext irq
+      */
+     if (msi_cnt > 0) {
+             vdev->irqs[i].flags = VFIO_IRQ_INFO_EVENTFD;
+             vdev->irqs[i].count = msi_cnt;
+             vdev->irqs[i].hwirq = 0;
+             vdev->irqs[i].masked = false;
+             vdev->irqs[i].type = VFIO_IRQ_TYPE_MSI;
+             vdev->num_ext_irqs = 1;
+     }
+
+     vdev->num_irqs = num_irqs;
 
      return 0;
 err:
@@ -321,8 +546,13 @@ void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev)
 {
      int i;
 
-     for (i = 0; i < vdev->num_irqs; i++)
-             vfio_set_trigger(vdev, i, -1, NULL);
+     for (i = 0; i < vdev->num_irqs; i++) {
+             if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
+                     vfio_set_msi_trigger(vdev, i, 0, 0,
+                                          VFIO_IRQ_SET_DATA_NONE, NULL);
+             else
+                     vfio_set_trigger(vdev, i, -1, NULL);
+     }
 
      vdev->num_irqs = 0;
      kfree(vdev->irqs);
drivers/vfio/platform/vfio_platform_private.h +32 -0
index 289089910643..1307feddda21 100644
@@ -9,6 +9,7 @@
 
 #include <linux/types.h>
 #include <linux/interrupt.h>
+#include <linux/msi.h>
 
 #define VFIO_PLATFORM_OFFSET_SHIFT   40
 #define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
@@ -19,9 +20,21 @@
 #define VFIO_PLATFORM_INDEX_TO_OFFSET(index) \
      ((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT)
 
+/* IRQ index for MSI in ext IRQs */
+#define VFIO_EXT_IRQ_MSI     0
+
+struct vfio_irq_ctx {
+     int                     hwirq;
+     char                    *name;
+     struct msi_msg          msg;
+     struct eventfd_ctx      *trigger;
+};
+
 struct vfio_platform_irq {
      u32                     flags;
      u32                     count;
+     int                     num_ctx;
+     struct vfio_irq_ctx     *ctx;
      int                     hwirq;
      char                    *name;
      struct eventfd_ctx      *trigger;
@@ -29,6 +42,11 @@ struct vfio_platform_irq {
      spinlock_t              lock;
      struct virqfd           *unmask;
      struct virqfd           *mask;
+
+     /* for extended irqs */
+     u32                     type;
+     u32                     subtype;
+     int                     config_msi;
 };
 
 struct vfio_platform_region {
@@ -46,12 +64,14 @@ struct vfio_platform_device {
      u32                             num_regions;
      struct vfio_platform_irq        *irqs;
      u32                             num_irqs;
+     int                             num_ext_irqs;
      int                             refcnt;
      struct mutex                    igate;
      struct module                   *parent_module;
      const char                      *compat;
      const char                      *acpihid;
      struct module                   *reset_module;
+     struct module                   *msi_module;
      struct device                   *device;
 
      /*
@@ -65,11 +85,13 @@ struct vfio_platform_device {
              (*get_resource)(struct vfio_platform_device *vdev, int i);
      int     (*get_irq)(struct vfio_platform_device *vdev, int i);
      int     (*of_reset)(struct vfio_platform_device *vdev);
+     u32     (*of_get_msi)(struct vfio_platform_device *vdev);
 
      bool                            reset_required;
 };
 
 typedef int (*vfio_platform_reset_fn_t)(struct vfio_platform_device *vdev);
+typedef u32 (*vfio_platform_get_msi_fn_t)(struct vfio_platform_device *vdev);
 
 struct vfio_platform_reset_node {
      struct list_head link;
@@ -78,6 +100,13 @@ struct vfio_platform_reset_node {
      vfio_platform_reset_fn_t of_reset;
 };
 
+struct vfio_platform_msi_node {
+     struct list_head link;
+     char *compat;
+     struct module *owner;
+     vfio_platform_get_msi_fn_t of_get_msi;
+};
+
 extern int vfio_platform_probe_common(struct vfio_platform_device *vdev,
                                    struct device *dev);
 extern struct vfio_platform_device *vfio_platform_remove_common
@@ -94,6 +123,9 @@ extern int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 extern void __vfio_platform_register_reset(struct vfio_platform_reset_node *n);
 extern void vfio_platform_unregister_reset(const char *compat,
                                         vfio_platform_reset_fn_t fn);
+void __vfio_platform_register_msi(struct vfio_platform_msi_node *n);
+void vfio_platform_unregister_msi(const char *compat);
+
 #define vfio_platform_register_reset(__compat, __reset)              \
 static struct vfio_platform_reset_node __reset ## _node = {  \
      .owner = THIS_MODULE,                                   \
include/uapi/linux/vfio.h +44 -0
index d1812777139f..53a7ff2b524e 100644
@@ -697,11 +697,55 @@ struct vfio_irq_info {
 #define VFIO_IRQ_INFO_MASKABLE               (1 << 1)
 #define VFIO_IRQ_INFO_AUTOMASKED     (1 << 2)
 #define VFIO_IRQ_INFO_NORESIZE               (1 << 3)
+#define VFIO_IRQ_INFO_FLAG_CAPS              (1 << 4) /* Info supports caps */
      __u32   index;          /* IRQ index */
      __u32   count;          /* Number of IRQs within this index */
+     __u32   cap_offset;     /* Offset within info struct of first cap */
 };
 #define VFIO_DEVICE_GET_IRQ_INFO     _IO(VFIO_TYPE, VFIO_BASE + 9)
 
+/*
+ * The irq type capability allows IRQs unique to a specific device or
+ * class of devices to be exposed.
+ *
+ * The structures below define version 1 of this capability.
+ */
+#define VFIO_IRQ_INFO_CAP_TYPE               3
+
+struct vfio_irq_info_cap_type {
+     struct vfio_info_cap_header header;
+     __u32 type;     /* global per bus driver */
+     __u32 subtype;  /* type specific */
+};
+
+/*
+ * List of IRQ types, global per bus driver.
+ * If you introduce a new type, please add it here.
+ */
+
+/* Non PCI devices having MSI(s) support */
+#define VFIO_IRQ_TYPE_MSI            (1)
+
+/*
+ * The msi capability allows the user to use the msi msg to
+ * configure the iova for the msi configuration.
+ * The structures below define version 1 of this capability.
+ */
+#define VFIO_IRQ_INFO_CAP_MSI_DESCS  4
+
+struct vfio_irq_msi_msg {
+     __u32   addr_lo;
+     __u32   addr_hi;
+     __u32   data;
+};
+
+struct vfio_irq_info_cap_msi {
+     struct vfio_info_cap_header header;
+     __u32   nr_msgs;
+     __u32   reserved;
+     struct vfio_irq_msi_msg msgs[];
+};
+
 /**
  * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set)
  *
#39 lore raw

Re: [RFC v3 1/2] vfio/platform: add support for msi

Hi Eric,

Quoted 177 lines · Auger Eric

Hi Vikas,

Quoted 169 lines · Vikas Gupta
Quoted 159 lines · Auger Eric

Hi Vikas,

Quoted 25 lines · Vikas Gupta

MSI support for platform devices.The MSI block is added as an extended IRQ which exports caps VFIO_IRQ_INFO_CAP_TYPE and VFIO_IRQ_INFO_CAP_MSI_DESCS.

---
 drivers/vfio/platform/vfio_platform_common.c  | 179 +++++++++++-
 drivers/vfio/platform/vfio_platform_irq.c     | 260 +++++++++++++++++-
 drivers/vfio/platform/vfio_platform_private.h |  32 +++
 include/uapi/linux/vfio.h                     |  44 +++
 4 files changed, 496 insertions(+), 19 deletions(-)
drivers/vfio/platform/vfio_platform_common.c +1 -0
index fb4b385191f2..c936852f35d7 100644
@@ -16,6 +16,7 @@
 #include <linux/types.h>
 #include <linux/uaccess.h>
 #include <linux/vfio.h>
+#include <linux/nospec.h>
 
 #include "vfio_platform_private.h"
 

drivers/vfio/platform/vfio_platform_common.c line 30

Quoted 4 lines
drivers/vfio/platform/vfio_platform_common.c +1 -0
@@ -26,6 +27,8 @@
 #define VFIO_PLATFORM_IS_ACPI(vdev) ((vdev)->acpihid != NULL)
 
 static LIST_HEAD(reset_list);
+/* devices having MSI support */

nit: for devices using MSIs?

Quoted 125 lines
drivers/vfio/platform/vfio_platform_common.c +85 -2
+static LIST_HEAD(msi_list);
 static DEFINE_MUTEX(driver_lock);
 
 static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,
@@ -47,6 +50,25 @@ static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,
      return reset_fn;
 }
 
+static bool vfio_platform_lookup_msi(struct vfio_platform_device *vdev)
+{
+     bool has_msi = false;
+     struct vfio_platform_msi_node *iter;
+
+     mutex_lock(&driver_lock);
+     list_for_each_entry(iter, &msi_list, link) {
+             if (!strcmp(iter->compat, vdev->compat) &&
+                 try_module_get(iter->owner)) {
+                     vdev->msi_module = iter->owner;
+                     vdev->of_get_msi = iter->of_get_msi;
+                     has_msi = true;
+                     break;
+             }
+     }
+     mutex_unlock(&driver_lock);
+     return has_msi;
+}
+
 static int vfio_platform_acpi_probe(struct vfio_platform_device *vdev,
                                  struct device *dev)
 {
@@ -126,6 +148,19 @@ static int vfio_platform_get_reset(struct vfio_platform_device *vdev)
      return vdev->of_reset ? 0 : -ENOENT;
 }
 
+static int vfio_platform_get_msi(struct vfio_platform_device *vdev)
+{
+     bool has_msi;
+
+     has_msi = vfio_platform_lookup_msi(vdev);
+     if (!has_msi) {
+             request_module("vfio-msi:%s", vdev->compat);
+             has_msi = vfio_platform_lookup_msi(vdev);
+     }
+
+     return has_msi ? 0 : -ENOENT;
+}
+
 static void vfio_platform_put_reset(struct vfio_platform_device *vdev)
 {
      if (VFIO_PLATFORM_IS_ACPI(vdev))
@@ -135,6 +170,12 @@ static void vfio_platform_put_reset(struct vfio_platform_device *vdev)
              module_put(vdev->reset_module);
 }
 
+static void vfio_platform_put_msi(struct vfio_platform_device *vdev)
+{
+     if (vdev->of_get_msi)
+             module_put(vdev->msi_module);
+}
+
 static int vfio_platform_regions_init(struct vfio_platform_device *vdev)
 {
      int cnt = 0, i;
@@ -343,9 +384,17 @@ static long vfio_platform_ioctl(void *device_data,
 
      } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
              struct vfio_irq_info info;
+             struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
+             struct vfio_irq_info_cap_msi *msi_info = NULL;
+             int ext_irq_index = vdev->num_irqs - vdev->num_ext_irqs;
+             unsigned long capsz;
+             u32 index;
 
              minsz = offsetofend(struct vfio_irq_info, count);
 
+             /* For backward compatibility, cannot require this */
+             capsz = offsetofend(struct vfio_irq_info, cap_offset);
+
              if (copy_from_user(&info, (void __user *)arg, minsz))
                      return -EFAULT;
 
@@ -355,8 +404,94 @@ static long vfio_platform_ioctl(void *device_data,
              if (info.index >= vdev->num_irqs)
                      return -EINVAL;
 
-             info.flags = vdev->irqs[info.index].flags;
-             info.count = vdev->irqs[info.index].count;
+             if (info.argsz >= capsz)
+                     minsz = capsz;
+
+             index = info.index;
+
+             info.flags = vdev->irqs[index].flags;
+             info.count = vdev->irqs[index].count;
+
+             if (ext_irq_index - index == VFIO_EXT_IRQ_MSI) {
+                     struct vfio_irq_info_cap_type cap_type = {
+                             .header.id = VFIO_IRQ_INFO_CAP_TYPE,
+                             .header.version = 1 };
+                     struct vfio_platform_irq *irq;
+                     size_t msi_info_size;
+                     int num_msgs;
+                     int ret;
+                     int i;
+
+                     index = array_index_nospec(index,
+                                                vdev->num_irqs);
+                     irq = &vdev->irqs[index];
+
+                     cap_type.type = irq->type;
+                     cap_type.subtype = irq->subtype;
+
+                     ret = vfio_info_add_capability(&caps,
+                                                    &cap_type.header,
+                                                    sizeof(cap_type));
+                     if (ret)
+                             return ret;
+
+                     num_msgs = irq->num_ctx;
+                     if (num_msgs) {
+                             msi_info_size = struct_size(msi_info,
+                                                         msgs, num_msgs);
+
+                             msi_info = kzalloc(msi_info_size, GFP_KERNEL);
+                             if (!msi_info) {
Quoted 8 lines

drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 450

Quoted 6 lines
drivers/vfio/platform/vfio_platform_common.c +6 -0
+                                     kfree(caps.buf);
+                                     return -ENOMEM;
+                             }
+
+                             msi_info->header.id =
+                                             VFIO_IRQ_INFO_CAP_MSI_DESCS;

I thought you would remove this cap now the module is back. What is the motivation to keep it?

Quoted 4 lines

Hi Eric,
The earlier module was serving two purposes
a) Max number of MSI(s) supported by the device. Since the module is
back this information can be filled in .count.

Quoted 7 lines

understood

Quoted 3 lines

b) Writing msi_msg(s) to devices for MSI sources.
We want b) to be handled in the user space so that they need not be
dependent on the kernel.

I fail to understand the problem with the "dependency on the kernel". What is the functional difference when exposing this to the userspace, please could you highlight the benefits?

We do not see any functional difference here, the benefit we see is that user space can configure MSI sources with msi_msg rather than doing it in the kernel.

Quoted 7 lines

VFIO_IRQ_INFO_CAP_MSI_DESCS is helping user

Quoted 3 lines

space to get MSI configuration data and thus they can independently configure MSI sources. For example, user space can just ask for ‘N’ MSI vectors and related msi_msg for ‘N’ vectors

this is induced by vfio_msi_enable(), right?

Yes

Quoted 10 lines

is exported to user

Quoted 6 lines

space using this CAP. User space can now use ‘N’ vectors to configure any ‘N’ MSI sources and these sources might be in any order. However, this design is different from vfio-pci as MSI configuration is being handled in kernel only.
Let us know if implementing VFIO_IRQ_INFO_CAP_MSI_DESCS does not fit with the CAP framework.

I think it is OK reporting them through the cap but I fail to understand the benefits.

I believe, since there is no significant benefit this CAP should be avoided. I plan to remove this CAP in the next patch set.

Thanks,
Vikas

Quoted 620 lines

Thanks

Eric

Quoted 615 lines

Thanks,
Vikas

Quoted 80 lines

Thanks

Eric

Quoted 76 lines
drivers/vfio/platform/vfio_platform_common.c +53 -2
+                             msi_info->header.version = 1;
+                             msi_info->nr_msgs = num_msgs;
+
+                             for (i = 0; i < num_msgs; i++) {
+                                     struct vfio_irq_ctx *ctx = &irq->ctx[i];
+
+                                     msi_info->msgs[i].addr_lo =
+                                                     ctx->msg.address_lo;
+                                     msi_info->msgs[i].addr_hi =
+                                                     ctx->msg.address_hi;
+                                     msi_info->msgs[i].data = ctx->msg.data;
+                             }
+
+                             ret = vfio_info_add_capability(&caps,
+                                                     &msi_info->header,
+                                                     msi_info_size);
+                             if (ret) {
+                                     kfree(msi_info);
+                                     kfree(caps.buf);
+                                     return ret;
+                             }
+                     }
+             }
+
+             if (caps.size) {
+                     info.flags |= VFIO_IRQ_INFO_FLAG_CAPS;
+                     if (info.argsz < sizeof(info) + caps.size) {
+                             info.argsz = sizeof(info) + caps.size;
+                             info.cap_offset = 0;
+                     } else {
+                             vfio_info_cap_shift(&caps, sizeof(info));
+                             if (copy_to_user((void __user *)arg +
+                                               sizeof(info), caps.buf,
+                                               caps.size)) {
+                                     kfree(msi_info);
+                                     kfree(caps.buf);
+                                     return -EFAULT;
+                             }
+                             info.cap_offset = sizeof(info);
+                     }
+
+                     kfree(msi_info);
+                     kfree(caps.buf);
+             }
 
              return copy_to_user((void __user *)arg, &info, minsz) ?
                      -EFAULT : 0;
@@ -365,6 +500,7 @@ static long vfio_platform_ioctl(void *device_data,
              struct vfio_irq_set hdr;
              u8 *data = NULL;
              int ret = 0;
+             int max;
              size_t data_size = 0;
 
              minsz = offsetofend(struct vfio_irq_set, count);
@@ -372,8 +508,14 @@ static long vfio_platform_ioctl(void *device_data,
              if (copy_from_user(&hdr, (void __user *)arg, minsz))
                      return -EFAULT;
 
-             ret = vfio_set_irqs_validate_and_prepare(&hdr, vdev->num_irqs,
-                                              vdev->num_irqs, &data_size);
+             if (hdr.index >= vdev->num_irqs)
+                     return -EINVAL;
+
+             max = vdev->irqs[hdr.index].count;
+
+             ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
+                                                      vdev->num_irqs,
+                                                      &data_size);
              if (ret)
                      return ret;
 
@@ -678,6 +820,10 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,
              return ret;
      }
 
Quoted 7 lines

drivers/vfio/platform/vfio_platform_common.c int vfio_platform_probe_common(struct vfio_platform_device *vdev, · line 825

Quoted 3 lines
drivers/vfio/platform/vfio_platform_common.c +3 -0
+     ret = vfio_platform_get_msi(vdev);
+     if (ret)
+             dev_info(vdev->device, "msi not supported\n");

I don't think we should output this message. This is printed for amd-xgbe which does not have msi so this is misleading. I would say either the vfio_platform_get_msi() can return an actual error or we return a void?

Will check on this what

Quoted 522 lines

Thanks

Eric

Quoted 517 lines
drivers/vfio/platform/vfio_platform_common.c +26 -0
+
      group = vfio_iommu_group_get(dev);
      if (!group) {
              dev_err(dev, "No IOMMU group for device %s\n", vdev->name);
@@ -697,6 +843,7 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,
 put_iommu:
      vfio_iommu_group_put(group, dev);
 put_reset:
+     vfio_platform_put_msi(vdev);
      vfio_platform_put_reset(vdev);
      return ret;
 }
@@ -744,6 +891,30 @@ void vfio_platform_unregister_reset(const char *compat,
 }
 EXPORT_SYMBOL_GPL(vfio_platform_unregister_reset);
 
+void __vfio_platform_register_msi(struct vfio_platform_msi_node *node)
+{
+     mutex_lock(&driver_lock);
+     list_add(&node->link, &msi_list);
+     mutex_unlock(&driver_lock);
+}
+EXPORT_SYMBOL_GPL(__vfio_platform_register_msi);
+
+void vfio_platform_unregister_msi(const char *compat)
+{
+     struct vfio_platform_msi_node *iter, *temp;
+
+     mutex_lock(&driver_lock);
+     list_for_each_entry_safe(iter, temp, &msi_list, link) {
+             if (!strcmp(iter->compat, compat)) {
+                     list_del(&iter->link);
+                     break;
+             }
+     }
+
+     mutex_unlock(&driver_lock);
+}
+EXPORT_SYMBOL_GPL(vfio_platform_unregister_msi);
+
 MODULE_VERSION(DRIVER_VERSION);
 MODULE_LICENSE("GPL v2");
 MODULE_AUTHOR(DRIVER_AUTHOR);
drivers/vfio/platform/vfio_platform_irq.c +245 -15
index c5b09ec0a3c9..e0f4696afedd 100644
@@ -8,10 +8,12 @@
 
 #include <linux/eventfd.h>
 #include <linux/interrupt.h>
+#include <linux/eventfd.h>
 #include <linux/slab.h>
 #include <linux/types.h>
 #include <linux/vfio.h>
 #include <linux/irq.h>
+#include <linux/msi.h>
 
 #include "vfio_platform_private.h"
 
@@ -253,6 +255,195 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,
      return 0;
 }
 
+/* MSI/MSIX */
+static irqreturn_t vfio_msihandler(int irq, void *arg)
+{
+     struct eventfd_ctx *trigger = arg;
+
+     eventfd_signal(trigger, 1);
+     return IRQ_HANDLED;
+}
+
+static void msi_write(struct msi_desc *desc, struct msi_msg *msg)
+{
+     int i;
+     struct vfio_platform_irq *irq;
+     u16 index = desc->platform.msi_index;
+     struct device *dev = msi_desc_to_dev(desc);
+     struct vfio_device *device = dev_get_drvdata(dev);
+     struct vfio_platform_device *vdev = (struct vfio_platform_device *)
+                                             vfio_device_data(device);
+
+     for (i = 0; i < vdev->num_irqs; i++)
+             if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
+                     irq = &vdev->irqs[i];
+
+     irq->ctx[index].msg.address_lo = msg->address_lo;
+     irq->ctx[index].msg.address_hi = msg->address_hi;
+     irq->ctx[index].msg.data = msg->data;
+}
+
+static int vfio_msi_enable(struct vfio_platform_device *vdev,
+                        struct vfio_platform_irq *irq, int nvec)
+{
+     int ret;
+     int msi_idx = 0;
+     struct msi_desc *desc;
+     struct device *dev = vdev->device;
+
+     irq->ctx = kcalloc(nvec, sizeof(struct vfio_irq_ctx), GFP_KERNEL);
+     if (!irq->ctx)
+             return -ENOMEM;
+
+     /* Allocate platform MSIs */
+     ret = platform_msi_domain_alloc_irqs(dev, nvec, msi_write);
+     if (ret < 0) {
+             kfree(irq->ctx);
+             return ret;
+     }
+
+     for_each_msi_entry(desc, dev) {
+             irq->ctx[msi_idx].hwirq = desc->irq;
+             msi_idx++;
+     }
+
+     irq->num_ctx = nvec;
+     irq->config_msi = 1;
+
+     return 0;
+}
+
+static int vfio_msi_set_vector_signal(struct vfio_platform_irq *irq,
+                                   int vector, int fd)
+{
+     struct eventfd_ctx *trigger;
+     int irq_num, ret;
+
+     if (vector < 0 || vector >= irq->num_ctx)
+             return -EINVAL;
+
+     irq_num = irq->ctx[vector].hwirq;
+
+     if (irq->ctx[vector].trigger) {
+             free_irq(irq_num, irq->ctx[vector].trigger);
+             kfree(irq->ctx[vector].name);
+             eventfd_ctx_put(irq->ctx[vector].trigger);
+             irq->ctx[vector].trigger = NULL;
+     }
+
+     if (fd < 0)
+             return 0;
+
+     irq->ctx[vector].name = kasprintf(GFP_KERNEL,
+                                       "vfio-msi[%d]", vector);
+     if (!irq->ctx[vector].name)
+             return -ENOMEM;
+
+     trigger = eventfd_ctx_fdget(fd);
+     if (IS_ERR(trigger)) {
+             kfree(irq->ctx[vector].name);
+             return PTR_ERR(trigger);
+     }
+
+     ret = request_irq(irq_num, vfio_msihandler, 0,
+                       irq->ctx[vector].name, trigger);
+     if (ret) {
+             kfree(irq->ctx[vector].name);
+             eventfd_ctx_put(trigger);
+             return ret;
+     }
+
+     irq->ctx[vector].trigger = trigger;
+
+     return 0;
+}
+
+static int vfio_msi_set_block(struct vfio_platform_irq *irq, unsigned int start,
+                           unsigned int count, int32_t *fds)
+{
+     int i, j, ret = 0;
+
+     if (start >= irq->num_ctx || start + count > irq->num_ctx)
+             return -EINVAL;
+
+     for (i = 0, j = start; i < count && !ret; i++, j++) {
+             int fd = fds ? fds[i] : -1;
+
+             ret = vfio_msi_set_vector_signal(irq, j, fd);
+     }
+
+     if (ret) {
+             for (--j; j >= (int)start; j--)
+                     vfio_msi_set_vector_signal(irq, j, -1);
+     }
+
+     return ret;
+}
+
+static void vfio_msi_disable(struct vfio_platform_device *vdev,
+                          struct vfio_platform_irq *irq)
+{
+     struct device *dev = vdev->device;
+
+     vfio_msi_set_block(irq, 0, irq->num_ctx, NULL);
+
+     platform_msi_domain_free_irqs(dev);
+
+     irq->config_msi = 0;
+     irq->num_ctx = 0;
+
+     kfree(irq->ctx);
+}
+
+static int vfio_set_msi_trigger(struct vfio_platform_device *vdev,
+                             unsigned int index, unsigned int start,
+                             unsigned int count, uint32_t flags, void *data)
+{
+     int i;
+     struct vfio_platform_irq *irq = &vdev->irqs[index];
+
+     if (start + count > irq->count)
+             return -EINVAL;
+
+     if (!count && (flags & VFIO_IRQ_SET_DATA_NONE)) {
+             vfio_msi_disable(vdev, irq);
+             return 0;
+     }
+
+     if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
+             s32 *fds = data;
+             int ret;
+
+             if (irq->config_msi)
+                     return vfio_msi_set_block(irq, start, count,
+                                               fds);
+             ret = vfio_msi_enable(vdev, irq, start + count);
+             if (ret)
+                     return ret;
+
+             ret = vfio_msi_set_block(irq, start, count, fds);
+             if (ret)
+                     vfio_msi_disable(vdev, irq);
+
+             return ret;
+     }
+
+     for (i = start; i < start + count; i++) {
+             if (!irq->ctx[i].trigger)
+                     continue;
+             if (flags & VFIO_IRQ_SET_DATA_NONE) {
+                     eventfd_signal(irq->ctx[i].trigger, 1);
+             } else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
+                     u8 *bools = data;
+
+                     if (bools[i - start])
+                             eventfd_signal(irq->ctx[i].trigger, 1);
+             }
+     }
+
+     return 0;
+}
+
 int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
                               uint32_t flags, unsigned index, unsigned start,
                               unsigned count, void *data)
@@ -261,16 +452,29 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
                  unsigned start, unsigned count, uint32_t flags,
                  void *data) = NULL;
 
-     switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
-     case VFIO_IRQ_SET_ACTION_MASK:
-             func = vfio_platform_set_irq_mask;
-             break;
-     case VFIO_IRQ_SET_ACTION_UNMASK:
-             func = vfio_platform_set_irq_unmask;
-             break;
-     case VFIO_IRQ_SET_ACTION_TRIGGER:
-             func = vfio_platform_set_irq_trigger;
-             break;
+     struct vfio_platform_irq *irq = &vdev->irqs[index];
+
+     if (irq->type == VFIO_IRQ_TYPE_MSI) {
+             switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
+             case VFIO_IRQ_SET_ACTION_MASK:
+             case VFIO_IRQ_SET_ACTION_UNMASK:
+                     break;
+             case VFIO_IRQ_SET_ACTION_TRIGGER:
+                     func = vfio_set_msi_trigger;
+                     break;
+             }
+     } else {
+             switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
+             case VFIO_IRQ_SET_ACTION_MASK:
+                     func = vfio_platform_set_irq_mask;
+                     break;
+             case VFIO_IRQ_SET_ACTION_UNMASK:
+                     func = vfio_platform_set_irq_unmask;
+                     break;
+             case VFIO_IRQ_SET_ACTION_TRIGGER:
+                     func = vfio_platform_set_irq_trigger;
+                     break;
+             }
      }
 
      if (!func)
@@ -281,12 +485,21 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 
 int vfio_platform_irq_init(struct vfio_platform_device *vdev)
 {
-     int cnt = 0, i;
+     int i;
+     int cnt = 0;
+     int num_irqs;
+     int msi_cnt = 0;
 
      while (vdev->get_irq(vdev, cnt) >= 0)
              cnt++;
 
-     vdev->irqs = kcalloc(cnt, sizeof(struct vfio_platform_irq), GFP_KERNEL);
+     if (vdev->of_get_msi) {
+             msi_cnt = vdev->of_get_msi(vdev);
+             num_irqs++;
+     }
+
+     vdev->irqs = kcalloc(num_irqs, sizeof(struct vfio_platform_irq),
+                          GFP_KERNEL);
      if (!vdev->irqs)
              return -ENOMEM;
 
@@ -309,7 +522,19 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev)
              vdev->irqs[i].masked = false;
      }
 
-     vdev->num_irqs = cnt;
+     /*
+      * MSI block is added at last index and it is an ext irq
+      */
+     if (msi_cnt > 0) {
+             vdev->irqs[i].flags = VFIO_IRQ_INFO_EVENTFD;
+             vdev->irqs[i].count = msi_cnt;
+             vdev->irqs[i].hwirq = 0;
+             vdev->irqs[i].masked = false;
+             vdev->irqs[i].type = VFIO_IRQ_TYPE_MSI;
+             vdev->num_ext_irqs = 1;
+     }
+
+     vdev->num_irqs = num_irqs;
 
      return 0;
 err:
@@ -321,8 +546,13 @@ void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev)
 {
      int i;
 
-     for (i = 0; i < vdev->num_irqs; i++)
-             vfio_set_trigger(vdev, i, -1, NULL);
+     for (i = 0; i < vdev->num_irqs; i++) {
+             if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
+                     vfio_set_msi_trigger(vdev, i, 0, 0,
+                                          VFIO_IRQ_SET_DATA_NONE, NULL);
+             else
+                     vfio_set_trigger(vdev, i, -1, NULL);
+     }
 
      vdev->num_irqs = 0;
      kfree(vdev->irqs);
drivers/vfio/platform/vfio_platform_private.h +32 -0
index 289089910643..1307feddda21 100644
@@ -9,6 +9,7 @@
 
 #include <linux/types.h>
 #include <linux/interrupt.h>
+#include <linux/msi.h>
 
 #define VFIO_PLATFORM_OFFSET_SHIFT   40
 #define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
@@ -19,9 +20,21 @@
 #define VFIO_PLATFORM_INDEX_TO_OFFSET(index) \
      ((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT)
 
+/* IRQ index for MSI in ext IRQs */
+#define VFIO_EXT_IRQ_MSI     0
+
+struct vfio_irq_ctx {
+     int                     hwirq;
+     char                    *name;
+     struct msi_msg          msg;
+     struct eventfd_ctx      *trigger;
+};
+
 struct vfio_platform_irq {
      u32                     flags;
      u32                     count;
+     int                     num_ctx;
+     struct vfio_irq_ctx     *ctx;
      int                     hwirq;
      char                    *name;
      struct eventfd_ctx      *trigger;
@@ -29,6 +42,11 @@ struct vfio_platform_irq {
      spinlock_t              lock;
      struct virqfd           *unmask;
      struct virqfd           *mask;
+
+     /* for extended irqs */
+     u32                     type;
+     u32                     subtype;
+     int                     config_msi;
 };
 
 struct vfio_platform_region {
@@ -46,12 +64,14 @@ struct vfio_platform_device {
      u32                             num_regions;
      struct vfio_platform_irq        *irqs;
      u32                             num_irqs;
+     int                             num_ext_irqs;
      int                             refcnt;
      struct mutex                    igate;
      struct module                   *parent_module;
      const char                      *compat;
      const char                      *acpihid;
      struct module                   *reset_module;
+     struct module                   *msi_module;
      struct device                   *device;
 
      /*
@@ -65,11 +85,13 @@ struct vfio_platform_device {
              (*get_resource)(struct vfio_platform_device *vdev, int i);
      int     (*get_irq)(struct vfio_platform_device *vdev, int i);
      int     (*of_reset)(struct vfio_platform_device *vdev);
+     u32     (*of_get_msi)(struct vfio_platform_device *vdev);
 
      bool                            reset_required;
 };
 
 typedef int (*vfio_platform_reset_fn_t)(struct vfio_platform_device *vdev);
+typedef u32 (*vfio_platform_get_msi_fn_t)(struct vfio_platform_device *vdev);
 
 struct vfio_platform_reset_node {
      struct list_head link;
@@ -78,6 +100,13 @@ struct vfio_platform_reset_node {
      vfio_platform_reset_fn_t of_reset;
 };
 
+struct vfio_platform_msi_node {
+     struct list_head link;
+     char *compat;
+     struct module *owner;
+     vfio_platform_get_msi_fn_t of_get_msi;
+};
+
 extern int vfio_platform_probe_common(struct vfio_platform_device *vdev,
                                    struct device *dev);
 extern struct vfio_platform_device *vfio_platform_remove_common
@@ -94,6 +123,9 @@ extern int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 extern void __vfio_platform_register_reset(struct vfio_platform_reset_node *n);
 extern void vfio_platform_unregister_reset(const char *compat,
                                         vfio_platform_reset_fn_t fn);
+void __vfio_platform_register_msi(struct vfio_platform_msi_node *n);
+void vfio_platform_unregister_msi(const char *compat);
+
 #define vfio_platform_register_reset(__compat, __reset)              \
 static struct vfio_platform_reset_node __reset ## _node = {  \
      .owner = THIS_MODULE,                                   \
include/uapi/linux/vfio.h +44 -0
index d1812777139f..53a7ff2b524e 100644
@@ -697,11 +697,55 @@ struct vfio_irq_info {
 #define VFIO_IRQ_INFO_MASKABLE               (1 << 1)
 #define VFIO_IRQ_INFO_AUTOMASKED     (1 << 2)
 #define VFIO_IRQ_INFO_NORESIZE               (1 << 3)
+#define VFIO_IRQ_INFO_FLAG_CAPS              (1 << 4) /* Info supports caps */
      __u32   index;          /* IRQ index */
      __u32   count;          /* Number of IRQs within this index */
+     __u32   cap_offset;     /* Offset within info struct of first cap */
 };
 #define VFIO_DEVICE_GET_IRQ_INFO     _IO(VFIO_TYPE, VFIO_BASE + 9)
 
+/*
+ * The irq type capability allows IRQs unique to a specific device or
+ * class of devices to be exposed.
+ *
+ * The structures below define version 1 of this capability.
+ */
+#define VFIO_IRQ_INFO_CAP_TYPE               3
+
+struct vfio_irq_info_cap_type {
+     struct vfio_info_cap_header header;
+     __u32 type;     /* global per bus driver */
+     __u32 subtype;  /* type specific */
+};
+
+/*
+ * List of IRQ types, global per bus driver.
+ * If you introduce a new type, please add it here.
+ */
+
+/* Non PCI devices having MSI(s) support */
+#define VFIO_IRQ_TYPE_MSI            (1)
+
+/*
+ * The msi capability allows the user to use the msi msg to
+ * configure the iova for the msi configuration.
+ * The structures below define version 1 of this capability.
+ */
+#define VFIO_IRQ_INFO_CAP_MSI_DESCS  4
+
+struct vfio_irq_msi_msg {
+     __u32   addr_lo;
+     __u32   addr_hi;
+     __u32   data;
+};
+
+struct vfio_irq_info_cap_msi {
+     struct vfio_info_cap_header header;
+     __u32   nr_msgs;
+     __u32   reserved;
+     struct vfio_irq_msi_msg msgs[];
+};
+
 /**
  * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set)
  *
maintainer
#40 lore raw

Re: [RFC v3 1/2] vfio/platform: add support for msi

Hi Vikas,

Quoted 207 lines · Vikas Gupta

Hi Eric,

Quoted 177 lines · Auger Eric

Hi Vikas,

Quoted 169 lines · Vikas Gupta
Quoted 159 lines · Auger Eric

Hi Vikas,

Quoted 25 lines · Vikas Gupta

MSI support for platform devices.The MSI block is added as an extended IRQ which exports caps VFIO_IRQ_INFO_CAP_TYPE and VFIO_IRQ_INFO_CAP_MSI_DESCS.

---
 drivers/vfio/platform/vfio_platform_common.c  | 179 +++++++++++-
 drivers/vfio/platform/vfio_platform_irq.c     | 260 +++++++++++++++++-
 drivers/vfio/platform/vfio_platform_private.h |  32 +++
 include/uapi/linux/vfio.h                     |  44 +++
 4 files changed, 496 insertions(+), 19 deletions(-)
drivers/vfio/platform/vfio_platform_common.c +1 -0
index fb4b385191f2..c936852f35d7 100644
@@ -16,6 +16,7 @@
 #include <linux/types.h>
 #include <linux/uaccess.h>
 #include <linux/vfio.h>
+#include <linux/nospec.h>
 
 #include "vfio_platform_private.h"
 

drivers/vfio/platform/vfio_platform_common.c line 30

Quoted 4 lines
drivers/vfio/platform/vfio_platform_common.c +1 -0
@@ -26,6 +27,8 @@
 #define VFIO_PLATFORM_IS_ACPI(vdev) ((vdev)->acpihid != NULL)
 
 static LIST_HEAD(reset_list);
+/* devices having MSI support */

nit: for devices using MSIs?

Quoted 125 lines
drivers/vfio/platform/vfio_platform_common.c +85 -2
+static LIST_HEAD(msi_list);
 static DEFINE_MUTEX(driver_lock);
 
 static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,
@@ -47,6 +50,25 @@ static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,
      return reset_fn;
 }
 
+static bool vfio_platform_lookup_msi(struct vfio_platform_device *vdev)
+{
+     bool has_msi = false;
+     struct vfio_platform_msi_node *iter;
+
+     mutex_lock(&driver_lock);
+     list_for_each_entry(iter, &msi_list, link) {
+             if (!strcmp(iter->compat, vdev->compat) &&
+                 try_module_get(iter->owner)) {
+                     vdev->msi_module = iter->owner;
+                     vdev->of_get_msi = iter->of_get_msi;
+                     has_msi = true;
+                     break;
+             }
+     }
+     mutex_unlock(&driver_lock);
+     return has_msi;
+}
+
 static int vfio_platform_acpi_probe(struct vfio_platform_device *vdev,
                                  struct device *dev)
 {
@@ -126,6 +148,19 @@ static int vfio_platform_get_reset(struct vfio_platform_device *vdev)
      return vdev->of_reset ? 0 : -ENOENT;
 }
 
+static int vfio_platform_get_msi(struct vfio_platform_device *vdev)
+{
+     bool has_msi;
+
+     has_msi = vfio_platform_lookup_msi(vdev);
+     if (!has_msi) {
+             request_module("vfio-msi:%s", vdev->compat);
+             has_msi = vfio_platform_lookup_msi(vdev);
+     }
+
+     return has_msi ? 0 : -ENOENT;
+}
+
 static void vfio_platform_put_reset(struct vfio_platform_device *vdev)
 {
      if (VFIO_PLATFORM_IS_ACPI(vdev))
@@ -135,6 +170,12 @@ static void vfio_platform_put_reset(struct vfio_platform_device *vdev)
              module_put(vdev->reset_module);
 }
 
+static void vfio_platform_put_msi(struct vfio_platform_device *vdev)
+{
+     if (vdev->of_get_msi)
+             module_put(vdev->msi_module);
+}
+
 static int vfio_platform_regions_init(struct vfio_platform_device *vdev)
 {
      int cnt = 0, i;
@@ -343,9 +384,17 @@ static long vfio_platform_ioctl(void *device_data,
 
      } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
              struct vfio_irq_info info;
+             struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
+             struct vfio_irq_info_cap_msi *msi_info = NULL;
+             int ext_irq_index = vdev->num_irqs - vdev->num_ext_irqs;
+             unsigned long capsz;
+             u32 index;
 
              minsz = offsetofend(struct vfio_irq_info, count);
 
+             /* For backward compatibility, cannot require this */
+             capsz = offsetofend(struct vfio_irq_info, cap_offset);
+
              if (copy_from_user(&info, (void __user *)arg, minsz))
                      return -EFAULT;
 
@@ -355,8 +404,94 @@ static long vfio_platform_ioctl(void *device_data,
              if (info.index >= vdev->num_irqs)
                      return -EINVAL;
 
-             info.flags = vdev->irqs[info.index].flags;
-             info.count = vdev->irqs[info.index].count;
+             if (info.argsz >= capsz)
+                     minsz = capsz;
+
+             index = info.index;
+
+             info.flags = vdev->irqs[index].flags;
+             info.count = vdev->irqs[index].count;
+
+             if (ext_irq_index - index == VFIO_EXT_IRQ_MSI) {
+                     struct vfio_irq_info_cap_type cap_type = {
+                             .header.id = VFIO_IRQ_INFO_CAP_TYPE,
+                             .header.version = 1 };
+                     struct vfio_platform_irq *irq;
+                     size_t msi_info_size;
+                     int num_msgs;
+                     int ret;
+                     int i;
+
+                     index = array_index_nospec(index,
+                                                vdev->num_irqs);
+                     irq = &vdev->irqs[index];
+
+                     cap_type.type = irq->type;
+                     cap_type.subtype = irq->subtype;
+
+                     ret = vfio_info_add_capability(&caps,
+                                                    &cap_type.header,
+                                                    sizeof(cap_type));
+                     if (ret)
+                             return ret;
+
+                     num_msgs = irq->num_ctx;
+                     if (num_msgs) {
+                             msi_info_size = struct_size(msi_info,
+                                                         msgs, num_msgs);
+
+                             msi_info = kzalloc(msi_info_size, GFP_KERNEL);
+                             if (!msi_info) {
Quoted 8 lines

drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 450

Quoted 6 lines
drivers/vfio/platform/vfio_platform_common.c +6 -0
+                                     kfree(caps.buf);
+                                     return -ENOMEM;
+                             }
+
+                             msi_info->header.id =
+                                             VFIO_IRQ_INFO_CAP_MSI_DESCS;

I thought you would remove this cap now the module is back. What is the motivation to keep it?

Quoted 4 lines

Hi Eric,
The earlier module was serving two purposes
a) Max number of MSI(s) supported by the device. Since the module is
back this information can be filled in .count.

Quoted 7 lines

understood

Quoted 3 lines

b) Writing msi_msg(s) to devices for MSI sources.
We want b) to be handled in the user space so that they need not be
dependent on the kernel.

I fail to understand the problem with the "dependency on the kernel". What is the functional difference when exposing this to the userspace, please could you highlight the benefits?

We do not see any functional difference here, the benefit we see is that user space can configure MSI sources with msi_msg rather than doing it in the kernel.

Quoted 7 lines

VFIO_IRQ_INFO_CAP_MSI_DESCS is helping user

Quoted 3 lines

space to get MSI configuration data and thus they can independently configure MSI sources. For example, user space can just ask for ‘N’ MSI vectors and related msi_msg for ‘N’ vectors

this is induced by vfio_msi_enable(), right?

Yes

Quoted 8 lines

is exported to user

Quoted 6 lines

space using this CAP. User space can now use ‘N’ vectors to configure any ‘N’ MSI sources and these sources might be in any order. However, this design is different from vfio-pci as MSI configuration is being handled in kernel only.
Let us know if implementing VFIO_IRQ_INFO_CAP_MSI_DESCS does not fit with the CAP framework.

Quoted 5 lines
Quoted 2 lines

I think it is OK reporting them through the cap but I fail to understand the benefits.

I believe, since there is no significant benefit this CAP should be avoided. I plan to remove this CAP in the next patch set.

OK thank you for the confirmation.

Thanks

Eric

Quoted 623 lines

Thanks,
Vikas

Quoted 620 lines

Thanks

Eric

Quoted 615 lines

Thanks,
Vikas

Quoted 80 lines

Thanks

Eric

Quoted 76 lines
drivers/vfio/platform/vfio_platform_common.c +53 -2
+                             msi_info->header.version = 1;
+                             msi_info->nr_msgs = num_msgs;
+
+                             for (i = 0; i < num_msgs; i++) {
+                                     struct vfio_irq_ctx *ctx = &irq->ctx[i];
+
+                                     msi_info->msgs[i].addr_lo =
+                                                     ctx->msg.address_lo;
+                                     msi_info->msgs[i].addr_hi =
+                                                     ctx->msg.address_hi;
+                                     msi_info->msgs[i].data = ctx->msg.data;
+                             }
+
+                             ret = vfio_info_add_capability(&caps,
+                                                     &msi_info->header,
+                                                     msi_info_size);
+                             if (ret) {
+                                     kfree(msi_info);
+                                     kfree(caps.buf);
+                                     return ret;
+                             }
+                     }
+             }
+
+             if (caps.size) {
+                     info.flags |= VFIO_IRQ_INFO_FLAG_CAPS;
+                     if (info.argsz < sizeof(info) + caps.size) {
+                             info.argsz = sizeof(info) + caps.size;
+                             info.cap_offset = 0;
+                     } else {
+                             vfio_info_cap_shift(&caps, sizeof(info));
+                             if (copy_to_user((void __user *)arg +
+                                               sizeof(info), caps.buf,
+                                               caps.size)) {
+                                     kfree(msi_info);
+                                     kfree(caps.buf);
+                                     return -EFAULT;
+                             }
+                             info.cap_offset = sizeof(info);
+                     }
+
+                     kfree(msi_info);
+                     kfree(caps.buf);
+             }
 
              return copy_to_user((void __user *)arg, &info, minsz) ?
                      -EFAULT : 0;
@@ -365,6 +500,7 @@ static long vfio_platform_ioctl(void *device_data,
              struct vfio_irq_set hdr;
              u8 *data = NULL;
              int ret = 0;
+             int max;
              size_t data_size = 0;
 
              minsz = offsetofend(struct vfio_irq_set, count);
@@ -372,8 +508,14 @@ static long vfio_platform_ioctl(void *device_data,
              if (copy_from_user(&hdr, (void __user *)arg, minsz))
                      return -EFAULT;
 
-             ret = vfio_set_irqs_validate_and_prepare(&hdr, vdev->num_irqs,
-                                              vdev->num_irqs, &data_size);
+             if (hdr.index >= vdev->num_irqs)
+                     return -EINVAL;
+
+             max = vdev->irqs[hdr.index].count;
+
+             ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
+                                                      vdev->num_irqs,
+                                                      &data_size);
              if (ret)
                      return ret;
 
@@ -678,6 +820,10 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,
              return ret;
      }
 
Quoted 7 lines

drivers/vfio/platform/vfio_platform_common.c int vfio_platform_probe_common(struct vfio_platform_device *vdev, · line 825

Quoted 3 lines
drivers/vfio/platform/vfio_platform_common.c +3 -0
+     ret = vfio_platform_get_msi(vdev);
+     if (ret)
+             dev_info(vdev->device, "msi not supported\n");

I don't think we should output this message. This is printed for amd-xgbe which does not have msi so this is misleading. I would say either the vfio_platform_get_msi() can return an actual error or we return a void?

Will check on this what

Quoted 522 lines

Thanks

Eric

Quoted 517 lines
drivers/vfio/platform/vfio_platform_common.c +26 -0
+
      group = vfio_iommu_group_get(dev);
      if (!group) {
              dev_err(dev, "No IOMMU group for device %s\n", vdev->name);
@@ -697,6 +843,7 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,
 put_iommu:
      vfio_iommu_group_put(group, dev);
 put_reset:
+     vfio_platform_put_msi(vdev);
      vfio_platform_put_reset(vdev);
      return ret;
 }
@@ -744,6 +891,30 @@ void vfio_platform_unregister_reset(const char *compat,
 }
 EXPORT_SYMBOL_GPL(vfio_platform_unregister_reset);
 
+void __vfio_platform_register_msi(struct vfio_platform_msi_node *node)
+{
+     mutex_lock(&driver_lock);
+     list_add(&node->link, &msi_list);
+     mutex_unlock(&driver_lock);
+}
+EXPORT_SYMBOL_GPL(__vfio_platform_register_msi);
+
+void vfio_platform_unregister_msi(const char *compat)
+{
+     struct vfio_platform_msi_node *iter, *temp;
+
+     mutex_lock(&driver_lock);
+     list_for_each_entry_safe(iter, temp, &msi_list, link) {
+             if (!strcmp(iter->compat, compat)) {
+                     list_del(&iter->link);
+                     break;
+             }
+     }
+
+     mutex_unlock(&driver_lock);
+}
+EXPORT_SYMBOL_GPL(vfio_platform_unregister_msi);
+
 MODULE_VERSION(DRIVER_VERSION);
 MODULE_LICENSE("GPL v2");
 MODULE_AUTHOR(DRIVER_AUTHOR);
drivers/vfio/platform/vfio_platform_irq.c +245 -15
index c5b09ec0a3c9..e0f4696afedd 100644
@@ -8,10 +8,12 @@
 
 #include <linux/eventfd.h>
 #include <linux/interrupt.h>
+#include <linux/eventfd.h>
 #include <linux/slab.h>
 #include <linux/types.h>
 #include <linux/vfio.h>
 #include <linux/irq.h>
+#include <linux/msi.h>
 
 #include "vfio_platform_private.h"
 
@@ -253,6 +255,195 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,
      return 0;
 }
 
+/* MSI/MSIX */
+static irqreturn_t vfio_msihandler(int irq, void *arg)
+{
+     struct eventfd_ctx *trigger = arg;
+
+     eventfd_signal(trigger, 1);
+     return IRQ_HANDLED;
+}
+
+static void msi_write(struct msi_desc *desc, struct msi_msg *msg)
+{
+     int i;
+     struct vfio_platform_irq *irq;
+     u16 index = desc->platform.msi_index;
+     struct device *dev = msi_desc_to_dev(desc);
+     struct vfio_device *device = dev_get_drvdata(dev);
+     struct vfio_platform_device *vdev = (struct vfio_platform_device *)
+                                             vfio_device_data(device);
+
+     for (i = 0; i < vdev->num_irqs; i++)
+             if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
+                     irq = &vdev->irqs[i];
+
+     irq->ctx[index].msg.address_lo = msg->address_lo;
+     irq->ctx[index].msg.address_hi = msg->address_hi;
+     irq->ctx[index].msg.data = msg->data;
+}
+
+static int vfio_msi_enable(struct vfio_platform_device *vdev,
+                        struct vfio_platform_irq *irq, int nvec)
+{
+     int ret;
+     int msi_idx = 0;
+     struct msi_desc *desc;
+     struct device *dev = vdev->device;
+
+     irq->ctx = kcalloc(nvec, sizeof(struct vfio_irq_ctx), GFP_KERNEL);
+     if (!irq->ctx)
+             return -ENOMEM;
+
+     /* Allocate platform MSIs */
+     ret = platform_msi_domain_alloc_irqs(dev, nvec, msi_write);
+     if (ret < 0) {
+             kfree(irq->ctx);
+             return ret;
+     }
+
+     for_each_msi_entry(desc, dev) {
+             irq->ctx[msi_idx].hwirq = desc->irq;
+             msi_idx++;
+     }
+
+     irq->num_ctx = nvec;
+     irq->config_msi = 1;
+
+     return 0;
+}
+
+static int vfio_msi_set_vector_signal(struct vfio_platform_irq *irq,
+                                   int vector, int fd)
+{
+     struct eventfd_ctx *trigger;
+     int irq_num, ret;
+
+     if (vector < 0 || vector >= irq->num_ctx)
+             return -EINVAL;
+
+     irq_num = irq->ctx[vector].hwirq;
+
+     if (irq->ctx[vector].trigger) {
+             free_irq(irq_num, irq->ctx[vector].trigger);
+             kfree(irq->ctx[vector].name);
+             eventfd_ctx_put(irq->ctx[vector].trigger);
+             irq->ctx[vector].trigger = NULL;
+     }
+
+     if (fd < 0)
+             return 0;
+
+     irq->ctx[vector].name = kasprintf(GFP_KERNEL,
+                                       "vfio-msi[%d]", vector);
+     if (!irq->ctx[vector].name)
+             return -ENOMEM;
+
+     trigger = eventfd_ctx_fdget(fd);
+     if (IS_ERR(trigger)) {
+             kfree(irq->ctx[vector].name);
+             return PTR_ERR(trigger);
+     }
+
+     ret = request_irq(irq_num, vfio_msihandler, 0,
+                       irq->ctx[vector].name, trigger);
+     if (ret) {
+             kfree(irq->ctx[vector].name);
+             eventfd_ctx_put(trigger);
+             return ret;
+     }
+
+     irq->ctx[vector].trigger = trigger;
+
+     return 0;
+}
+
+static int vfio_msi_set_block(struct vfio_platform_irq *irq, unsigned int start,
+                           unsigned int count, int32_t *fds)
+{
+     int i, j, ret = 0;
+
+     if (start >= irq->num_ctx || start + count > irq->num_ctx)
+             return -EINVAL;
+
+     for (i = 0, j = start; i < count && !ret; i++, j++) {
+             int fd = fds ? fds[i] : -1;
+
+             ret = vfio_msi_set_vector_signal(irq, j, fd);
+     }
+
+     if (ret) {
+             for (--j; j >= (int)start; j--)
+                     vfio_msi_set_vector_signal(irq, j, -1);
+     }
+
+     return ret;
+}
+
+static void vfio_msi_disable(struct vfio_platform_device *vdev,
+                          struct vfio_platform_irq *irq)
+{
+     struct device *dev = vdev->device;
+
+     vfio_msi_set_block(irq, 0, irq->num_ctx, NULL);
+
+     platform_msi_domain_free_irqs(dev);
+
+     irq->config_msi = 0;
+     irq->num_ctx = 0;
+
+     kfree(irq->ctx);
+}
+
+static int vfio_set_msi_trigger(struct vfio_platform_device *vdev,
+                             unsigned int index, unsigned int start,
+                             unsigned int count, uint32_t flags, void *data)
+{
+     int i;
+     struct vfio_platform_irq *irq = &vdev->irqs[index];
+
+     if (start + count > irq->count)
+             return -EINVAL;
+
+     if (!count && (flags & VFIO_IRQ_SET_DATA_NONE)) {
+             vfio_msi_disable(vdev, irq);
+             return 0;
+     }
+
+     if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
+             s32 *fds = data;
+             int ret;
+
+             if (irq->config_msi)
+                     return vfio_msi_set_block(irq, start, count,
+                                               fds);
+             ret = vfio_msi_enable(vdev, irq, start + count);
+             if (ret)
+                     return ret;
+
+             ret = vfio_msi_set_block(irq, start, count, fds);
+             if (ret)
+                     vfio_msi_disable(vdev, irq);
+
+             return ret;
+     }
+
+     for (i = start; i < start + count; i++) {
+             if (!irq->ctx[i].trigger)
+                     continue;
+             if (flags & VFIO_IRQ_SET_DATA_NONE) {
+                     eventfd_signal(irq->ctx[i].trigger, 1);
+             } else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
+                     u8 *bools = data;
+
+                     if (bools[i - start])
+                             eventfd_signal(irq->ctx[i].trigger, 1);
+             }
+     }
+
+     return 0;
+}
+
 int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
                               uint32_t flags, unsigned index, unsigned start,
                               unsigned count, void *data)
@@ -261,16 +452,29 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
                  unsigned start, unsigned count, uint32_t flags,
                  void *data) = NULL;
 
-     switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
-     case VFIO_IRQ_SET_ACTION_MASK:
-             func = vfio_platform_set_irq_mask;
-             break;
-     case VFIO_IRQ_SET_ACTION_UNMASK:
-             func = vfio_platform_set_irq_unmask;
-             break;
-     case VFIO_IRQ_SET_ACTION_TRIGGER:
-             func = vfio_platform_set_irq_trigger;
-             break;
+     struct vfio_platform_irq *irq = &vdev->irqs[index];
+
+     if (irq->type == VFIO_IRQ_TYPE_MSI) {
+             switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
+             case VFIO_IRQ_SET_ACTION_MASK:
+             case VFIO_IRQ_SET_ACTION_UNMASK:
+                     break;
+             case VFIO_IRQ_SET_ACTION_TRIGGER:
+                     func = vfio_set_msi_trigger;
+                     break;
+             }
+     } else {
+             switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
+             case VFIO_IRQ_SET_ACTION_MASK:
+                     func = vfio_platform_set_irq_mask;
+                     break;
+             case VFIO_IRQ_SET_ACTION_UNMASK:
+                     func = vfio_platform_set_irq_unmask;
+                     break;
+             case VFIO_IRQ_SET_ACTION_TRIGGER:
+                     func = vfio_platform_set_irq_trigger;
+                     break;
+             }
      }
 
      if (!func)
@@ -281,12 +485,21 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 
 int vfio_platform_irq_init(struct vfio_platform_device *vdev)
 {
-     int cnt = 0, i;
+     int i;
+     int cnt = 0;
+     int num_irqs;
+     int msi_cnt = 0;
 
      while (vdev->get_irq(vdev, cnt) >= 0)
              cnt++;
 
-     vdev->irqs = kcalloc(cnt, sizeof(struct vfio_platform_irq), GFP_KERNEL);
+     if (vdev->of_get_msi) {
+             msi_cnt = vdev->of_get_msi(vdev);
+             num_irqs++;
+     }
+
+     vdev->irqs = kcalloc(num_irqs, sizeof(struct vfio_platform_irq),
+                          GFP_KERNEL);
      if (!vdev->irqs)
              return -ENOMEM;
 
@@ -309,7 +522,19 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev)
              vdev->irqs[i].masked = false;
      }
 
-     vdev->num_irqs = cnt;
+     /*
+      * MSI block is added at last index and it is an ext irq
+      */
+     if (msi_cnt > 0) {
+             vdev->irqs[i].flags = VFIO_IRQ_INFO_EVENTFD;
+             vdev->irqs[i].count = msi_cnt;
+             vdev->irqs[i].hwirq = 0;
+             vdev->irqs[i].masked = false;
+             vdev->irqs[i].type = VFIO_IRQ_TYPE_MSI;
+             vdev->num_ext_irqs = 1;
+     }
+
+     vdev->num_irqs = num_irqs;
 
      return 0;
 err:
@@ -321,8 +546,13 @@ void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev)
 {
      int i;
 
-     for (i = 0; i < vdev->num_irqs; i++)
-             vfio_set_trigger(vdev, i, -1, NULL);
+     for (i = 0; i < vdev->num_irqs; i++) {
+             if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
+                     vfio_set_msi_trigger(vdev, i, 0, 0,
+                                          VFIO_IRQ_SET_DATA_NONE, NULL);
+             else
+                     vfio_set_trigger(vdev, i, -1, NULL);
+     }
 
      vdev->num_irqs = 0;
      kfree(vdev->irqs);
drivers/vfio/platform/vfio_platform_private.h +32 -0
index 289089910643..1307feddda21 100644
@@ -9,6 +9,7 @@
 
 #include <linux/types.h>
 #include <linux/interrupt.h>
+#include <linux/msi.h>
 
 #define VFIO_PLATFORM_OFFSET_SHIFT   40
 #define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
@@ -19,9 +20,21 @@
 #define VFIO_PLATFORM_INDEX_TO_OFFSET(index) \
      ((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT)
 
+/* IRQ index for MSI in ext IRQs */
+#define VFIO_EXT_IRQ_MSI     0
+
+struct vfio_irq_ctx {
+     int                     hwirq;
+     char                    *name;
+     struct msi_msg          msg;
+     struct eventfd_ctx      *trigger;
+};
+
 struct vfio_platform_irq {
      u32                     flags;
      u32                     count;
+     int                     num_ctx;
+     struct vfio_irq_ctx     *ctx;
      int                     hwirq;
      char                    *name;
      struct eventfd_ctx      *trigger;
@@ -29,6 +42,11 @@ struct vfio_platform_irq {
      spinlock_t              lock;
      struct virqfd           *unmask;
      struct virqfd           *mask;
+
+     /* for extended irqs */
+     u32                     type;
+     u32                     subtype;
+     int                     config_msi;
 };
 
 struct vfio_platform_region {
@@ -46,12 +64,14 @@ struct vfio_platform_device {
      u32                             num_regions;
      struct vfio_platform_irq        *irqs;
      u32                             num_irqs;
+     int                             num_ext_irqs;
      int                             refcnt;
      struct mutex                    igate;
      struct module                   *parent_module;
      const char                      *compat;
      const char                      *acpihid;
      struct module                   *reset_module;
+     struct module                   *msi_module;
      struct device                   *device;
 
      /*
@@ -65,11 +85,13 @@ struct vfio_platform_device {
              (*get_resource)(struct vfio_platform_device *vdev, int i);
      int     (*get_irq)(struct vfio_platform_device *vdev, int i);
      int     (*of_reset)(struct vfio_platform_device *vdev);
+     u32     (*of_get_msi)(struct vfio_platform_device *vdev);
 
      bool                            reset_required;
 };
 
 typedef int (*vfio_platform_reset_fn_t)(struct vfio_platform_device *vdev);
+typedef u32 (*vfio_platform_get_msi_fn_t)(struct vfio_platform_device *vdev);
 
 struct vfio_platform_reset_node {
      struct list_head link;
@@ -78,6 +100,13 @@ struct vfio_platform_reset_node {
      vfio_platform_reset_fn_t of_reset;
 };
 
+struct vfio_platform_msi_node {
+     struct list_head link;
+     char *compat;
+     struct module *owner;
+     vfio_platform_get_msi_fn_t of_get_msi;
+};
+
 extern int vfio_platform_probe_common(struct vfio_platform_device *vdev,
                                    struct device *dev);
 extern struct vfio_platform_device *vfio_platform_remove_common
@@ -94,6 +123,9 @@ extern int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 extern void __vfio_platform_register_reset(struct vfio_platform_reset_node *n);
 extern void vfio_platform_unregister_reset(const char *compat,
                                         vfio_platform_reset_fn_t fn);
+void __vfio_platform_register_msi(struct vfio_platform_msi_node *n);
+void vfio_platform_unregister_msi(const char *compat);
+
 #define vfio_platform_register_reset(__compat, __reset)              \
 static struct vfio_platform_reset_node __reset ## _node = {  \
      .owner = THIS_MODULE,                                   \
include/uapi/linux/vfio.h +44 -0
index d1812777139f..53a7ff2b524e 100644
@@ -697,11 +697,55 @@ struct vfio_irq_info {
 #define VFIO_IRQ_INFO_MASKABLE               (1 << 1)
 #define VFIO_IRQ_INFO_AUTOMASKED     (1 << 2)
 #define VFIO_IRQ_INFO_NORESIZE               (1 << 3)
+#define VFIO_IRQ_INFO_FLAG_CAPS              (1 << 4) /* Info supports caps */
      __u32   index;          /* IRQ index */
      __u32   count;          /* Number of IRQs within this index */
+     __u32   cap_offset;     /* Offset within info struct of first cap */
 };
 #define VFIO_DEVICE_GET_IRQ_INFO     _IO(VFIO_TYPE, VFIO_BASE + 9)
 
+/*
+ * The irq type capability allows IRQs unique to a specific device or
+ * class of devices to be exposed.
+ *
+ * The structures below define version 1 of this capability.
+ */
+#define VFIO_IRQ_INFO_CAP_TYPE               3
+
+struct vfio_irq_info_cap_type {
+     struct vfio_info_cap_header header;
+     __u32 type;     /* global per bus driver */
+     __u32 subtype;  /* type specific */
+};
+
+/*
+ * List of IRQ types, global per bus driver.
+ * If you introduce a new type, please add it here.
+ */
+
+/* Non PCI devices having MSI(s) support */
+#define VFIO_IRQ_TYPE_MSI            (1)
+
+/*
+ * The msi capability allows the user to use the msi msg to
+ * configure the iova for the msi configuration.
+ * The structures below define version 1 of this capability.
+ */
+#define VFIO_IRQ_INFO_CAP_MSI_DESCS  4
+
+struct vfio_irq_msi_msg {
+     __u32   addr_lo;
+     __u32   addr_hi;
+     __u32   data;
+};
+
+struct vfio_irq_info_cap_msi {
+     struct vfio_info_cap_header header;
+     __u32   nr_msgs;
+     __u32   reserved;
+     struct vfio_irq_msi_msg msgs[];
+};
+
 /**
  * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set)
  *
#41 lore raw

Re: [RFC v3 1/2] vfio/platform: add support for msi

In reply to Vikas Gupta
[If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in https://git-scm.com/docs/git-format-patch]

Hi Vikas,

[FYI, it's a private test report for your RFC patch.]
[auto build test ERROR on vfio/next]
[also build test ERROR on v5.10 next-20201223]
[If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]

url:    https://github.com/0day-ci/linux/commits/Vikas-Gupta/msi-support-for-platform-devices/20201215-015115
base:   https://github.com/awilliam/linux-vfio.git next
config: arm-randconfig-r021-20201221 (attached as .config)
compiler: arm-linux-gnueabi-gcc (GCC) 9.3.0

reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# https://github.com/0day-ci/linux/commit/029a6fee02b683d688d7e6e685c78127ac29df8a
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Vikas-Gupta/msi-support-for-platform-devices/20201215-015115 git checkout 029a6fee02b683d688d7e6e685c78127ac29df8a
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=arm

If you fix the issue, kindly add following tag as appropriate

All errors (new ones prefixed by >>):

drivers/vfio/platform/vfio_platform_irq.c: In function 'vfio_msi_enable':

>> drivers/vfio/platform/vfio_platform_irq.c:299:8: error: implicit declaration of function 'platform_msi_domain_alloc_irqs' [-Werror=implicit-function-declaration]
     299 |  ret = platform_msi_domain_alloc_irqs(dev, nvec, msi_write);
         |        ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   In file included from include/linux/list.h:9,
                    from include/linux/wait.h:7,
                    from include/linux/eventfd.h:13,
                    from drivers/vfio/platform/vfio_platform_irq.c:9:
>> include/linux/msi.h:135:38: error: 'struct device' has no member named 'msi_list'
     135 | #define dev_to_msi_list(dev)  (&(dev)->msi_list)
         |                                      ^~
   include/linux/kernel.h:852:26: note: in definition of macro 'container_of'
     852 |  void *__mptr = (void *)(ptr);     \
         |                          ^~~
   include/linux/list.h:522:2: note: in expansion of macro 'list_entry'
     522 |  list_entry((ptr)->next, type, member)
         |  ^~~~~~~~~~
   include/linux/list.h:628:13: note: in expansion of macro 'list_first_entry'
     628 |  for (pos = list_first_entry(head, typeof(*pos), member); \
         |             ^~~~~~~~~~~~~~~~
   include/linux/msi.h:139:2: note: in expansion of macro 'list_for_each_entry'
     139 |  list_for_each_entry((desc), dev_to_msi_list((dev)), list)
         |  ^~~~~~~~~~~~~~~~~~~
   include/linux/msi.h:139:30: note: in expansion of macro 'dev_to_msi_list'
     139 |  list_for_each_entry((desc), dev_to_msi_list((dev)), list)
         |                              ^~~~~~~~~~~~~~~
   drivers/vfio/platform/vfio_platform_irq.c:305:2: note: in expansion of macro 'for_each_msi_entry'
     305 |  for_each_msi_entry(desc, dev) {
         |  ^~~~~~~~~~~~~~~~~~
   In file included from <command-line>:
>> include/linux/msi.h:135:38: error: 'struct device' has no member named 'msi_list'
     135 | #define dev_to_msi_list(dev)  (&(dev)->msi_list)
         |                                      ^~
   include/linux/compiler_types.h:295:9: note: in definition of macro '__compiletime_assert'
     295 |   if (!(condition))     \
         |         ^~~~~~~~~
   include/linux/compiler_types.h:315:2: note: in expansion of macro '_compiletime_assert'
     315 |  _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__)
         |  ^~~~~~~~~~~~~~~~~~~
   include/linux/build_bug.h:39:37: note: in expansion of macro 'compiletime_assert'
      39 | #define BUILD_BUG_ON_MSG(cond, msg) compiletime_assert(!(cond), msg)
         |                                     ^~~~~~~~~~~~~~~~~~
   include/linux/kernel.h:853:2: note: in expansion of macro 'BUILD_BUG_ON_MSG'
     853 |  BUILD_BUG_ON_MSG(!__same_type(*(ptr), ((type *)0)->member) && \
         |  ^~~~~~~~~~~~~~~~
   include/linux/kernel.h:853:20: note: in expansion of macro '__same_type'
     853 |  BUILD_BUG_ON_MSG(!__same_type(*(ptr), ((type *)0)->member) && \
         |                    ^~~~~~~~~~~
   include/linux/list.h:511:2: note: in expansion of macro 'container_of'
     511 |  container_of(ptr, type, member)
         |  ^~~~~~~~~~~~
   include/linux/list.h:522:2: note: in expansion of macro 'list_entry'
     522 |  list_entry((ptr)->next, type, member)
         |  ^~~~~~~~~~
   include/linux/list.h:628:13: note: in expansion of macro 'list_first_entry'
     628 |  for (pos = list_first_entry(head, typeof(*pos), member); \
         |             ^~~~~~~~~~~~~~~~
   include/linux/msi.h:139:2: note: in expansion of macro 'list_for_each_entry'
     139 |  list_for_each_entry((desc), dev_to_msi_list((dev)), list)
         |  ^~~~~~~~~~~~~~~~~~~
   include/linux/msi.h:139:30: note: in expansion of macro 'dev_to_msi_list'
     139 |  list_for_each_entry((desc), dev_to_msi_list((dev)), list)
         |                              ^~~~~~~~~~~~~~~
   drivers/vfio/platform/vfio_platform_irq.c:305:2: note: in expansion of macro 'for_each_msi_entry'
     305 |  for_each_msi_entry(desc, dev) {
         |  ^~~~~~~~~~~~~~~~~~
>> include/linux/msi.h:135:38: error: 'struct device' has no member named 'msi_list'
     135 | #define dev_to_msi_list(dev)  (&(dev)->msi_list)
         |                                      ^~
   include/linux/compiler_types.h:295:9: note: in definition of macro '__compiletime_assert'
     295 |   if (!(condition))     \
         |         ^~~~~~~~~
   include/linux/compiler_types.h:315:2: note: in expansion of macro '_compiletime_assert'
     315 |  _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__)
         |  ^~~~~~~~~~~~~~~~~~~
   include/linux/build_bug.h:39:37: note: in expansion of macro 'compiletime_assert'
      39 | #define BUILD_BUG_ON_MSG(cond, msg) compiletime_assert(!(cond), msg)
         |                                     ^~~~~~~~~~~~~~~~~~
   include/linux/kernel.h:853:2: note: in expansion of macro 'BUILD_BUG_ON_MSG'
     853 |  BUILD_BUG_ON_MSG(!__same_type(*(ptr), ((type *)0)->member) && \
         |  ^~~~~~~~~~~~~~~~
   include/linux/kernel.h:854:6: note: in expansion of macro '__same_type'
     854 |     !__same_type(*(ptr), void),   \
         |      ^~~~~~~~~~~
   include/linux/list.h:511:2: note: in expansion of macro 'container_of'
     511 |  container_of(ptr, type, member)
         |  ^~~~~~~~~~~~
   include/linux/list.h:522:2: note: in expansion of macro 'list_entry'
     522 |  list_entry((ptr)->next, type, member)
         |  ^~~~~~~~~~
   include/linux/list.h:628:13: note: in expansion of macro 'list_first_entry'
     628 |  for (pos = list_first_entry(head, typeof(*pos), member); \
         |             ^~~~~~~~~~~~~~~~
   include/linux/msi.h:139:2: note: in expansion of macro 'list_for_each_entry'
     139 |  list_for_each_entry((desc), dev_to_msi_list((dev)), list)
         |  ^~~~~~~~~~~~~~~~~~~
   include/linux/msi.h:139:30: note: in expansion of macro 'dev_to_msi_list'
     139 |  list_for_each_entry((desc), dev_to_msi_list((dev)), list)
         |                              ^~~~~~~~~~~~~~~
   drivers/vfio/platform/vfio_platform_irq.c:305:2: note: in expansion of macro 'for_each_msi_entry'
     305 |  for_each_msi_entry(desc, dev) {
         |  ^~~~~~~~~~~~~~~~~~
   In file included from include/linux/wait.h:7,
                    from include/linux/eventfd.h:13,
                    from drivers/vfio/platform/vfio_platform_irq.c:9:
>> include/linux/msi.h:135:38: error: 'struct device' has no member named 'msi_list'
     135 | #define dev_to_msi_list(dev)  (&(dev)->msi_list)
         |                                      ^~
   include/linux/list.h:619:20: note: in definition of macro 'list_entry_is_head'
     619 |  (&pos->member == (head))
         |                    ^~~~
   include/linux/msi.h:139:2: note: in expansion of macro 'list_for_each_entry'
     139 |  list_for_each_entry((desc), dev_to_msi_list((dev)), list)
         |  ^~~~~~~~~~~~~~~~~~~
   include/linux/msi.h:139:30: note: in expansion of macro 'dev_to_msi_list'
     139 |  list_for_each_entry((desc), dev_to_msi_list((dev)), list)
         |                              ^~~~~~~~~~~~~~~
   drivers/vfio/platform/vfio_platform_irq.c:305:2: note: in expansion of macro 'for_each_msi_entry'
     305 |  for_each_msi_entry(desc, dev) {
         |  ^~~~~~~~~~~~~~~~~~
   drivers/vfio/platform/vfio_platform_irq.c: In function 'vfio_msi_disable':
>> drivers/vfio/platform/vfio_platform_irq.c:390:2: error: implicit declaration of function 'platform_msi_domain_free_irqs' [-Werror=implicit-function-declaration]
     390 |  platform_msi_domain_free_irqs(dev);
         |  ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~

cc1: some warnings being treated as errors

vim +/platform_msi_domain_alloc_irqs +299 drivers/vfio/platform/vfio_platform_irq.c

285
286 static int vfio_msi_enable(struct vfio_platform_device *vdev,
287 struct vfio_platform_irq *irq, int nvec)
288 {
289 int ret;
290 int msi_idx = 0;
291 struct msi_desc *desc;
292 struct device *dev = vdev->device;
293
294 irq->ctx = kcalloc(nvec, sizeof(struct vfio_irq_ctx), GFP_KERNEL);
295 if (!irq->ctx)
296 return -ENOMEM;
297
298 /* Allocate platform MSIs */

 > 299		ret = platform_msi_domain_alloc_irqs(dev, nvec, msi_write);
   300		if (ret < 0) {
   301			kfree(irq->ctx);
   302			return ret;
   303		}
   304	
   305		for_each_msi_entry(desc, dev) {
   306			irq->ctx[msi_idx].hwirq = desc->irq;
   307			msi_idx++;
   308		}
   309	
   310		irq->num_ctx = nvec;
   311		irq->config_msi = 1;
   312	
   313		return 0;
   314	}
   315	
   316	static int vfio_msi_set_vector_signal(struct vfio_platform_irq *irq,
   317					      int vector, int fd)
   318	{
   319		struct eventfd_ctx *trigger;
   320		int irq_num, ret;
   321	
   322		if (vector < 0 || vector >= irq->num_ctx)
   323			return -EINVAL;
   324	
   325		irq_num = irq->ctx[vector].hwirq;
   326	
   327		if (irq->ctx[vector].trigger) {
   328			free_irq(irq_num, irq->ctx[vector].trigger);
   329			kfree(irq->ctx[vector].name);
   330			eventfd_ctx_put(irq->ctx[vector].trigger);
   331			irq->ctx[vector].trigger = NULL;
   332		}
   333	
   334		if (fd < 0)
   335			return 0;
   336	
   337		irq->ctx[vector].name = kasprintf(GFP_KERNEL,
   338						  "vfio-msi[%d]", vector);
   339		if (!irq->ctx[vector].name)
   340			return -ENOMEM;
   341	
   342		trigger = eventfd_ctx_fdget(fd);
   343		if (IS_ERR(trigger)) {
   344			kfree(irq->ctx[vector].name);
   345			return PTR_ERR(trigger);
   346		}
   347	
   348		ret = request_irq(irq_num, vfio_msihandler, 0,
   349				  irq->ctx[vector].name, trigger);
   350		if (ret) {
   351			kfree(irq->ctx[vector].name);
   352			eventfd_ctx_put(trigger);
   353			return ret;
   354		}
   355	
   356		irq->ctx[vector].trigger = trigger;
   357	
   358		return 0;
   359	}
   360	
   361	static int vfio_msi_set_block(struct vfio_platform_irq *irq, unsigned int start,
   362				      unsigned int count, int32_t *fds)
   363	{
   364		int i, j, ret = 0;
   365	
   366		if (start >= irq->num_ctx || start + count > irq->num_ctx)
   367			return -EINVAL;
   368	
   369		for (i = 0, j = start; i < count && !ret; i++, j++) {
   370			int fd = fds ? fds[i] : -1;
   371	
   372			ret = vfio_msi_set_vector_signal(irq, j, fd);
   373		}
   374	
   375		if (ret) {
   376			for (--j; j >= (int)start; j--)
   377				vfio_msi_set_vector_signal(irq, j, -1);
   378		}
   379	
   380		return ret;
   381	}
   382	
   383	static void vfio_msi_disable(struct vfio_platform_device *vdev,
   384				     struct vfio_platform_irq *irq)
   385	{
   386		struct device *dev = vdev->device;
   387	
   388		vfio_msi_set_block(irq, 0, irq->num_ctx, NULL);
   389	
 > 390		platform_msi_domain_free_irqs(dev);

391
392 irq->config_msi = 0;
393 irq->num_ctx = 0;
394
395 kfree(irq->ctx);
396 }
397

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org

#42 lore raw

RFC v3 2/2 vfio/platform: msi: add Broadcom platform devices

In reply to Vikas Gupta

Add msi support for Broadcom platform devices

---
 drivers/vfio/platform/Kconfig                 |  1 +
 drivers/vfio/platform/Makefile                |  1 +
 drivers/vfio/platform/msi/Kconfig             |  9 ++++
 drivers/vfio/platform/msi/Makefile            |  2 +
 .../vfio/platform/msi/vfio_platform_bcmplt.c  | 49 +++++++++++++++++++
 5 files changed, 62 insertions(+)
 create mode 100644 drivers/vfio/platform/msi/Kconfig
 create mode 100644 drivers/vfio/platform/msi/Makefile
 create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
drivers/vfio/platform/Kconfig +1 -0
index dc1a3c44f2c6..7b8696febe61 100644
@@ -21,3 +21,4 @@ config VFIO_AMBA
 	  If you don't know what to do here, say N.
 
 source "drivers/vfio/platform/reset/Kconfig"
+source "drivers/vfio/platform/msi/Kconfig"
drivers/vfio/platform/Makefile +1 -0
index 3f3a24e7c4ef..9ccdcdbf0e7e 100644
@@ -5,6 +5,7 @@ vfio-platform-y := vfio_platform.o
 obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform.o
 obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform-base.o
 obj-$(CONFIG_VFIO_PLATFORM) += reset/
+obj-$(CONFIG_VFIO_PLATFORM) += msi/
 
 vfio-amba-y := vfio_amba.o
 
drivers/vfio/platform/msi/Kconfig new file +9 -0
new file mode 100644
index 000000000000..54d6b70e1e32
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0-only
+config VFIO_PLATFORM_BCMPLT_MSI
+	tristate "MSI support for Broadcom platform devices"
+	depends on VFIO_PLATFORM && (ARCH_BCM_IPROC || COMPILE_TEST)
+	default ARCH_BCM_IPROC
+	help
+	  Enables the VFIO platform driver to handle msi for Broadcom devices
+
+	  If you don't know what to do here, say N.
drivers/vfio/platform/msi/Makefile new file +2 -0
new file mode 100644
index 000000000000..27422d45cecb
@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_VFIO_PLATFORM_BCMPLT_MSI) += vfio_platform_bcmplt.o
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +49 -0

49 lines not on this page · raw message

Signature · 15 lines
2.17.1


-- 
This electronic communication and the information and any files transmitted 
with it, or attached to it, are confidential and are intended solely for 
the use of the individual or entity to whom it is addressed and may contain 
information that is confidential, legally privileged, protected by privacy 
laws, or otherwise restricted from disclosure to anyone else. If you are 
not the intended recipient or the person responsible for delivering the 
e-mail to the intended recipient, you are hereby notified that any use, 
copying, distributing, dissemination, forwarding, printing, or copying of 
this e-mail is strictly prohibited. If you received this e-mail in error, 
please return the e-mail to the sender, delete it from your computer, and 
destroy any printed copy of it.
maintainer
#43 lore raw

Re: [RFC v3 2/2] vfio/platform: msi: add Broadcom platform devices

Hi Vikas,

Quoted 13 lines · Vikas Gupta

Add msi support for Broadcom platform devices

---
 drivers/vfio/platform/Kconfig                 |  1 +
 drivers/vfio/platform/Makefile                |  1 +
 drivers/vfio/platform/msi/Kconfig             |  9 ++++
 drivers/vfio/platform/msi/Makefile            |  2 +
 .../vfio/platform/msi/vfio_platform_bcmplt.c  | 49 +++++++++++++++++++
 5 files changed, 62 insertions(+)
 create mode 100644 drivers/vfio/platform/msi/Kconfig
 create mode 100644 drivers/vfio/platform/msi/Makefile
 create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c

what does plt mean?

Quoted 59 lines
drivers/vfio/platform/Kconfig +1 -0
index dc1a3c44f2c6..7b8696febe61 100644
@@ -21,3 +21,4 @@ config VFIO_AMBA
 	  If you don't know what to do here, say N.
 
 source "drivers/vfio/platform/reset/Kconfig"
+source "drivers/vfio/platform/msi/Kconfig"
drivers/vfio/platform/Makefile +1 -0
index 3f3a24e7c4ef..9ccdcdbf0e7e 100644
@@ -5,6 +5,7 @@ vfio-platform-y := vfio_platform.o
 obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform.o
 obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform-base.o
 obj-$(CONFIG_VFIO_PLATFORM) += reset/
+obj-$(CONFIG_VFIO_PLATFORM) += msi/
 
 vfio-amba-y := vfio_amba.o
 
drivers/vfio/platform/msi/Kconfig new file +9 -0
new file mode 100644
index 000000000000..54d6b70e1e32
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0-only
+config VFIO_PLATFORM_BCMPLT_MSI
+	tristate "MSI support for Broadcom platform devices"
+	depends on VFIO_PLATFORM && (ARCH_BCM_IPROC || COMPILE_TEST)
+	default ARCH_BCM_IPROC
+	help
+	  Enables the VFIO platform driver to handle msi for Broadcom devices
+
+	  If you don't know what to do here, say N.
drivers/vfio/platform/msi/Makefile new file +2 -0
new file mode 100644
index 000000000000..27422d45cecb
@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_VFIO_PLATFORM_BCMPLT_MSI) += vfio_platform_bcmplt.o
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +8 -0
new file mode 100644
index 000000000000..a074b5e92d77
@@ -0,0 +1,49 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2020 Broadcom.
+ */
+
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/interrupt.h>

drivers/vfio/platform/msi/vfio_platform_bcmplt.c line 18

Quoted 10 lines
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +10 -0
+#include <linux/msi.h>
+#include <linux/vfio.h>
+
+#include "../vfio_platform_private.h"
+
+#define RING_SIZE		(64 << 10)
+
+#define RING_MSI_ADDR_LS	0x03c
+#define RING_MSI_ADDR_MS	0x040
+#define RING_MSI_DATA_VALUE	0x064

Those 3 defines would not be needed anymore with that implementation option.

Quoted 18 lines
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +18 -0
+
+static u32 bcm_num_msi(struct vfio_platform_device *vdev)
+{
+	struct vfio_platform_region *reg = &vdev->regions[0];
+
+	return (reg->size / RING_SIZE);
+}
+
+static struct vfio_platform_msi_node vfio_platform_bcmflexrm_msi_node = {
+	.owner = THIS_MODULE,
+	.compat = "brcm,iproc-flexrm-mbox",
+	.of_get_msi = bcm_num_msi,
+};
+
+static int __init vfio_platform_bcmflexrm_msi_module_init(void)
+{
+	__vfio_platform_register_msi(&vfio_platform_bcmflexrm_msi_node);
+

drivers/vfio/platform/msi/vfio_platform_bcmplt.c line 46

Quoted 10 lines
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +10 -0
+	return 0;
+}
+
+static void __exit vfio_platform_bcmflexrm_msi_module_exit(void)
+{
+	vfio_platform_unregister_msi("brcm,iproc-flexrm-mbox");
+}
+
+module_init(vfio_platform_bcmflexrm_msi_module_init);
+module_exit(vfio_platform_bcmflexrm_msi_module_exit);

One thing I would like to discuss with Alex.

As the reset module is mandated (except if reset_required is forced to 0), I am wondering if we shouldn't try to turn the reset module into a "specialization" module and put the msi hooks there. I am afraid we may end up having modules for each and every vfio platform feature specialization. At the moment that's fully bearable but I can't predict what's next.

As the mandated feature is the reset capability maybe we could just keep the config/module name terminology, tune the kconfig help message to mention the msi support in case of flex-rm?

What do you think?

Thanks

Eric

Quoted 4 lines
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +3 -0
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Broadcom");
#44 lore raw

Re: [RFC v3 2/2] vfio/platform: msi: add Broadcom platform devices

Hi Eric,

Quoted 17 lines · Auger Eric

Hi Vikas,

Quoted 13 lines · Vikas Gupta

Add msi support for Broadcom platform devices

---
 drivers/vfio/platform/Kconfig                 |  1 +
 drivers/vfio/platform/Makefile                |  1 +
 drivers/vfio/platform/msi/Kconfig             |  9 ++++
 drivers/vfio/platform/msi/Makefile            |  2 +
 .../vfio/platform/msi/vfio_platform_bcmplt.c  | 49 +++++++++++++++++++
 5 files changed, 62 insertions(+)
 create mode 100644 drivers/vfio/platform/msi/Kconfig
 create mode 100644 drivers/vfio/platform/msi/Makefile
 create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
Quoted 1 line

what does plt mean?

This(plt) is a generic name for Broadcom platform devices, which we`ll
plan to add in this file. Currently we have only one in this file.
Do you think this name does not sound good here?

Quoted 107 lines
Quoted 59 lines
drivers/vfio/platform/Kconfig +1 -0
index dc1a3c44f2c6..7b8696febe61 100644
@@ -21,3 +21,4 @@ config VFIO_AMBA
        If you don't know what to do here, say N.
 
 source "drivers/vfio/platform/reset/Kconfig"
+source "drivers/vfio/platform/msi/Kconfig"
drivers/vfio/platform/Makefile +1 -0
index 3f3a24e7c4ef..9ccdcdbf0e7e 100644
@@ -5,6 +5,7 @@ vfio-platform-y := vfio_platform.o
 obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform.o
 obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform-base.o
 obj-$(CONFIG_VFIO_PLATFORM) += reset/
+obj-$(CONFIG_VFIO_PLATFORM) += msi/
 
 vfio-amba-y := vfio_amba.o
 
drivers/vfio/platform/msi/Kconfig new file +9 -0
new file mode 100644
index 000000000000..54d6b70e1e32
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0-only
+config VFIO_PLATFORM_BCMPLT_MSI
+     tristate "MSI support for Broadcom platform devices"
+     depends on VFIO_PLATFORM && (ARCH_BCM_IPROC || COMPILE_TEST)
+     default ARCH_BCM_IPROC
+     help
+       Enables the VFIO platform driver to handle msi for Broadcom devices
+
+       If you don't know what to do here, say N.
drivers/vfio/platform/msi/Makefile new file +2 -0
new file mode 100644
index 000000000000..27422d45cecb
@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_VFIO_PLATFORM_BCMPLT_MSI) += vfio_platform_bcmplt.o
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +8 -0
new file mode 100644
index 000000000000..a074b5e92d77
@@ -0,0 +1,49 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2020 Broadcom.
+ */
+
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/interrupt.h>

drivers/vfio/platform/msi/vfio_platform_bcmplt.c line 18

Quoted 10 lines
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +10 -0
+#include <linux/msi.h>
+#include <linux/vfio.h>
+
+#include "../vfio_platform_private.h"
+
+#define RING_SIZE            (64 << 10)
+
+#define RING_MSI_ADDR_LS     0x03c
+#define RING_MSI_ADDR_MS     0x040
+#define RING_MSI_DATA_VALUE  0x064

Those 3 defines would not be needed anymore with that implementation option.

Quoted 18 lines
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +18 -0
+
+static u32 bcm_num_msi(struct vfio_platform_device *vdev)
+{
+     struct vfio_platform_region *reg = &vdev->regions[0];
+
+     return (reg->size / RING_SIZE);
+}
+
+static struct vfio_platform_msi_node vfio_platform_bcmflexrm_msi_node = {
+     .owner = THIS_MODULE,
+     .compat = "brcm,iproc-flexrm-mbox",
+     .of_get_msi = bcm_num_msi,
+};
+
+static int __init vfio_platform_bcmflexrm_msi_module_init(void)
+{
+     __vfio_platform_register_msi(&vfio_platform_bcmflexrm_msi_node);
+

drivers/vfio/platform/msi/vfio_platform_bcmplt.c line 46

Quoted 10 lines
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +10 -0
+     return 0;
+}
+
+static void __exit vfio_platform_bcmflexrm_msi_module_exit(void)
+{
+     vfio_platform_unregister_msi("brcm,iproc-flexrm-mbox");
+}
+
+module_init(vfio_platform_bcmflexrm_msi_module_init);
+module_exit(vfio_platform_bcmflexrm_msi_module_exit);

One thing I would like to discuss with Alex.

As the reset module is mandated (except if reset_required is forced to 0), I am wondering if we shouldn't try to turn the reset module into a "specialization" module and put the msi hooks there. I am afraid we may end up having modules for each and every vfio platform feature specialization. At the moment that's fully bearable but I can't predict what's next.

Quoted 4 lines

As the mandated feature is the reset capability maybe we could just keep the config/module name terminology, tune the kconfig help message to mention the msi support in case of flex-rm?

As I understand, your proposal is that we should not have a separate module for MSI, rather we add in the existing reset module for flex-rm. Thus, this way reset modules do not seem to be specialized just for reset functionality only but for MSI as well. Apart from this we need not to load the proposed msi module in this patch series. Is my understanding correct?
For me it looks OK to consolidate MSI in the existing 'reset' module. Let me know your views so that I can work for the next patch set accordingly.

Thanks,
Vikas

Quoted 14 lines

What do you think?

Thanks

Eric

Quoted 4 lines
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +3 -0
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Broadcom");
maintainer
#45 lore raw

Re: [RFC v3 2/2] vfio/platform: msi: add Broadcom platform devices

Hi Vikas,

Quoted 20 lines · Vikas Gupta

Hi Eric,

Quoted 17 lines · Auger Eric

Hi Vikas,

Quoted 13 lines · Vikas Gupta

Add msi support for Broadcom platform devices

---
 drivers/vfio/platform/Kconfig                 |  1 +
 drivers/vfio/platform/Makefile                |  1 +
 drivers/vfio/platform/msi/Kconfig             |  9 ++++
 drivers/vfio/platform/msi/Makefile            |  2 +
 .../vfio/platform/msi/vfio_platform_bcmplt.c  | 49 +++++++++++++++++++
 5 files changed, 62 insertions(+)
 create mode 100644 drivers/vfio/platform/msi/Kconfig
 create mode 100644 drivers/vfio/platform/msi/Makefile
 create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
Quoted 4 lines
Quoted 1 line

what does plt mean?

This(plt) is a generic name for Broadcom platform devices, which we`ll
plan to add in this file. Currently we have only one in this file.
Do you think this name does not sound good here?

we have VFIO_PLATFORM_BCMFLEXRM_RESET config which also applied to vfio flex-rm platform device.

I think it would be more homegenous to have VFIO_PLATFORM_BCMFLEXRM_MSI in case we keep a separate msi module.

also in reset dir we have vfio_platform_bcmflexrm.c

Quoted 111 lines
Quoted 107 lines
Quoted 59 lines
drivers/vfio/platform/Kconfig +1 -0
index dc1a3c44f2c6..7b8696febe61 100644
@@ -21,3 +21,4 @@ config VFIO_AMBA
        If you don't know what to do here, say N.
 
 source "drivers/vfio/platform/reset/Kconfig"
+source "drivers/vfio/platform/msi/Kconfig"
drivers/vfio/platform/Makefile +1 -0
index 3f3a24e7c4ef..9ccdcdbf0e7e 100644
@@ -5,6 +5,7 @@ vfio-platform-y := vfio_platform.o
 obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform.o
 obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform-base.o
 obj-$(CONFIG_VFIO_PLATFORM) += reset/
+obj-$(CONFIG_VFIO_PLATFORM) += msi/
 
 vfio-amba-y := vfio_amba.o
 
drivers/vfio/platform/msi/Kconfig new file +9 -0
new file mode 100644
index 000000000000..54d6b70e1e32
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0-only
+config VFIO_PLATFORM_BCMPLT_MSI
+     tristate "MSI support for Broadcom platform devices"
+     depends on VFIO_PLATFORM && (ARCH_BCM_IPROC || COMPILE_TEST)
+     default ARCH_BCM_IPROC
+     help
+       Enables the VFIO platform driver to handle msi for Broadcom devices
+
+       If you don't know what to do here, say N.
drivers/vfio/platform/msi/Makefile new file +2 -0
new file mode 100644
index 000000000000..27422d45cecb
@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_VFIO_PLATFORM_BCMPLT_MSI) += vfio_platform_bcmplt.o
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +8 -0
new file mode 100644
index 000000000000..a074b5e92d77
@@ -0,0 +1,49 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2020 Broadcom.
+ */
+
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/interrupt.h>

drivers/vfio/platform/msi/vfio_platform_bcmplt.c line 18

Quoted 10 lines
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +10 -0
+#include <linux/msi.h>
+#include <linux/vfio.h>
+
+#include "../vfio_platform_private.h"
+
+#define RING_SIZE            (64 << 10)
+
+#define RING_MSI_ADDR_LS     0x03c
+#define RING_MSI_ADDR_MS     0x040
+#define RING_MSI_DATA_VALUE  0x064

Those 3 defines would not be needed anymore with that implementation option.

Quoted 18 lines
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +18 -0
+
+static u32 bcm_num_msi(struct vfio_platform_device *vdev)
+{
+     struct vfio_platform_region *reg = &vdev->regions[0];
+
+     return (reg->size / RING_SIZE);
+}
+
+static struct vfio_platform_msi_node vfio_platform_bcmflexrm_msi_node = {
+     .owner = THIS_MODULE,
+     .compat = "brcm,iproc-flexrm-mbox",
+     .of_get_msi = bcm_num_msi,
+};
+
+static int __init vfio_platform_bcmflexrm_msi_module_init(void)
+{
+     __vfio_platform_register_msi(&vfio_platform_bcmflexrm_msi_node);
+

drivers/vfio/platform/msi/vfio_platform_bcmplt.c line 46

Quoted 10 lines
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +10 -0
+     return 0;
+}
+
+static void __exit vfio_platform_bcmflexrm_msi_module_exit(void)
+{
+     vfio_platform_unregister_msi("brcm,iproc-flexrm-mbox");
+}
+
+module_init(vfio_platform_bcmflexrm_msi_module_init);
+module_exit(vfio_platform_bcmflexrm_msi_module_exit);

One thing I would like to discuss with Alex.

As the reset module is mandated (except if reset_required is forced to 0), I am wondering if we shouldn't try to turn the reset module into a "specialization" module and put the msi hooks there. I am afraid we may end up having modules for each and every vfio platform feature specialization. At the moment that's fully bearable but I can't predict what's next.

Quoted 4 lines

As the mandated feature is the reset capability maybe we could just keep the config/module name terminology, tune the kconfig help message to mention the msi support in case of flex-rm?

Quoted 6 lines

As I understand, your proposal is that we should not have a separate module for MSI, rather we add in the existing reset module for flex-rm. Thus, this way reset modules do not seem to be specialized just for reset functionality only but for MSI as well. Apart from this we need not to load the proposed msi module in this patch series. Is my understanding correct?

yes it is.

Quoted 2 lines

For me it looks OK to consolidate MSI in the existing 'reset' module. Let me know your views so that I can work for the next patch set accordingly.

Before you launch into the rewriting I would like to get the confirmation Alex is OK or if he prefers to keep separate modules.

Thanks

Eric

Quoted 18 lines

Thanks,
Vikas

Quoted 14 lines

What do you think?

Thanks

Eric

Quoted 4 lines
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +3 -0
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Broadcom");
#46 lore raw

Re: [RFC v3 2/2] vfio/platform: msi: add Broadcom platform devices

Quoted 154 lines · On Fri, 15 Jan 2021 10:24:33 +0100 Auger Eric <[email protected]> wrote:

Hi Vikas,

Quoted 20 lines · Vikas Gupta

Hi Eric,

Quoted 17 lines · Auger Eric

Hi Vikas,

Quoted 13 lines · Vikas Gupta

Add msi support for Broadcom platform devices

---
 drivers/vfio/platform/Kconfig                 |  1 +
 drivers/vfio/platform/Makefile                |  1 +
 drivers/vfio/platform/msi/Kconfig             |  9 ++++
 drivers/vfio/platform/msi/Makefile            |  2 +
 .../vfio/platform/msi/vfio_platform_bcmplt.c  | 49 +++++++++++++++++++
 5 files changed, 62 insertions(+)
 create mode 100644 drivers/vfio/platform/msi/Kconfig
 create mode 100644 drivers/vfio/platform/msi/Makefile
 create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c  
Quoted 4 lines
Quoted 1 line

what does plt mean?

This(plt) is a generic name for Broadcom platform devices, which we`ll
plan to add in this file. Currently we have only one in this file.
Do you think this name does not sound good here?

we have VFIO_PLATFORM_BCMFLEXRM_RESET config which also applied to vfio flex-rm platform device.

I think it would be more homegenous to have VFIO_PLATFORM_BCMFLEXRM_MSI in case we keep a separate msi module.

also in reset dir we have vfio_platform_bcmflexrm.c

Quoted 111 lines
Quoted 107 lines
Quoted 59 lines
drivers/vfio/platform/Kconfig +1 -0
index dc1a3c44f2c6..7b8696febe61 100644
@@ -21,3 +21,4 @@ config VFIO_AMBA
        If you don't know what to do here, say N.
 
 source "drivers/vfio/platform/reset/Kconfig"
+source "drivers/vfio/platform/msi/Kconfig"
drivers/vfio/platform/Makefile +1 -0
index 3f3a24e7c4ef..9ccdcdbf0e7e 100644
@@ -5,6 +5,7 @@ vfio-platform-y := vfio_platform.o
 obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform.o
 obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform-base.o
 obj-$(CONFIG_VFIO_PLATFORM) += reset/
+obj-$(CONFIG_VFIO_PLATFORM) += msi/
 
 vfio-amba-y := vfio_amba.o
 
drivers/vfio/platform/msi/Kconfig new file +9 -0
new file mode 100644
index 000000000000..54d6b70e1e32
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0-only
+config VFIO_PLATFORM_BCMPLT_MSI
+     tristate "MSI support for Broadcom platform devices"
+     depends on VFIO_PLATFORM && (ARCH_BCM_IPROC || COMPILE_TEST)
+     default ARCH_BCM_IPROC
+     help
+       Enables the VFIO platform driver to handle msi for Broadcom devices
+
+       If you don't know what to do here, say N.
drivers/vfio/platform/msi/Makefile new file +2 -0
new file mode 100644
index 000000000000..27422d45cecb
@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_VFIO_PLATFORM_BCMPLT_MSI) += vfio_platform_bcmplt.o
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +8 -0
new file mode 100644
index 000000000000..a074b5e92d77
@@ -0,0 +1,49 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2020 Broadcom.
+ */
+
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/interrupt.h>

drivers/vfio/platform/msi/vfio_platform_bcmplt.c line 18

Quoted 10 lines
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +10 -0
+#include <linux/msi.h>
+#include <linux/vfio.h>
+
+#include "../vfio_platform_private.h"
+
+#define RING_SIZE            (64 << 10)
+
+#define RING_MSI_ADDR_LS     0x03c
+#define RING_MSI_ADDR_MS     0x040
+#define RING_MSI_DATA_VALUE  0x064  

Those 3 defines would not be needed anymore with that implementation option.

Quoted 18 lines
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +18 -0
+
+static u32 bcm_num_msi(struct vfio_platform_device *vdev)
+{
+     struct vfio_platform_region *reg = &vdev->regions[0];
+
+     return (reg->size / RING_SIZE);
+}
+
+static struct vfio_platform_msi_node vfio_platform_bcmflexrm_msi_node = {
+     .owner = THIS_MODULE,
+     .compat = "brcm,iproc-flexrm-mbox",
+     .of_get_msi = bcm_num_msi,
+};
+
+static int __init vfio_platform_bcmflexrm_msi_module_init(void)
+{
+     __vfio_platform_register_msi(&vfio_platform_bcmflexrm_msi_node);
+

drivers/vfio/platform/msi/vfio_platform_bcmplt.c line 46

Quoted 10 lines
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +10 -0
+     return 0;
+}
+
+static void __exit vfio_platform_bcmflexrm_msi_module_exit(void)
+{
+     vfio_platform_unregister_msi("brcm,iproc-flexrm-mbox");
+}
+
+module_init(vfio_platform_bcmflexrm_msi_module_init);
+module_exit(vfio_platform_bcmflexrm_msi_module_exit);  

One thing I would like to discuss with Alex.

As the reset module is mandated (except if reset_required is forced to 0), I am wondering if we shouldn't try to turn the reset module into a "specialization" module and put the msi hooks there. I am afraid we may end up having modules for each and every vfio platform feature specialization. At the moment that's fully bearable but I can't predict what's next.

Quoted 4 lines

As the mandated feature is the reset capability maybe we could just keep the config/module name terminology, tune the kconfig help message to mention the msi support in case of flex-rm?

Quoted 6 lines

As I understand, your proposal is that we should not have a separate module for MSI, rather we add in the existing reset module for flex-rm. Thus, this way reset modules do not seem to be specialized just for reset functionality only but for MSI as well. Apart from this we need not to load the proposed msi module in this patch series. Is my understanding correct?

Quoted 6 lines

yes it is.

Quoted 2 lines

For me it looks OK to consolidate MSI in the existing 'reset' module. Let me know your views so that I can work for the next patch set accordingly.

Before you launch into the rewriting I would like to get the confirmation Alex is OK or if he prefers to keep separate modules.

If I understand correctly, the proposal here creates an entirely parallel vfio-msi request module interface like we have for vfio-reset, so the question is whether we should simplify vfio-platform-core to do a single module request per compat string and the device specific module would register multiple features rather than one per module. Is that right?

It seems the submodules are pretty simple, there's not a lot to be gained from duplicate boilerplate code in the modules themselves. The core code would clearly be simplified slightly to avoid multiple module requests, but for a more grand benefit is seems the registration interfaces would also need to be consolidated, perhaps providing a feature "ops" structure. As you indicate, having only two features at this point with a fairly small number of modules each, it's not yet too burdensome, but I could imagine it being a useful project.

More importantly in the short term, I'd expect modules handling the same compat string to be named similarly and enabled by a common Kconfig option. Thanks,

Alex

maintainer
#47 lore raw

Re: [RFC v3 2/2] vfio/platform: msi: add Broadcom platform devices

Hi Alex,

Quoted 164 lines · Alex Williamson
Quoted 154 lines · On Fri, 15 Jan 2021 10:24:33 +0100 Auger Eric <[email protected]> wrote:

Hi Vikas,

Quoted 20 lines · Vikas Gupta

Hi Eric,

Quoted 17 lines · Auger Eric

Hi Vikas,

Quoted 13 lines · Vikas Gupta

Add msi support for Broadcom platform devices

---
 drivers/vfio/platform/Kconfig                 |  1 +
 drivers/vfio/platform/Makefile                |  1 +
 drivers/vfio/platform/msi/Kconfig             |  9 ++++
 drivers/vfio/platform/msi/Makefile            |  2 +
 .../vfio/platform/msi/vfio_platform_bcmplt.c  | 49 +++++++++++++++++++
 5 files changed, 62 insertions(+)
 create mode 100644 drivers/vfio/platform/msi/Kconfig
 create mode 100644 drivers/vfio/platform/msi/Makefile
 create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c  
Quoted 4 lines
Quoted 1 line

what does plt mean?

This(plt) is a generic name for Broadcom platform devices, which we`ll
plan to add in this file. Currently we have only one in this file.
Do you think this name does not sound good here?

we have VFIO_PLATFORM_BCMFLEXRM_RESET config which also applied to vfio flex-rm platform device.

I think it would be more homegenous to have VFIO_PLATFORM_BCMFLEXRM_MSI in case we keep a separate msi module.

also in reset dir we have vfio_platform_bcmflexrm.c

Quoted 111 lines
Quoted 107 lines
Quoted 59 lines
drivers/vfio/platform/Kconfig +1 -0
index dc1a3c44f2c6..7b8696febe61 100644
@@ -21,3 +21,4 @@ config VFIO_AMBA
        If you don't know what to do here, say N.
 
 source "drivers/vfio/platform/reset/Kconfig"
+source "drivers/vfio/platform/msi/Kconfig"
drivers/vfio/platform/Makefile +1 -0
index 3f3a24e7c4ef..9ccdcdbf0e7e 100644
@@ -5,6 +5,7 @@ vfio-platform-y := vfio_platform.o
 obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform.o
 obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform-base.o
 obj-$(CONFIG_VFIO_PLATFORM) += reset/
+obj-$(CONFIG_VFIO_PLATFORM) += msi/
 
 vfio-amba-y := vfio_amba.o
 
drivers/vfio/platform/msi/Kconfig new file +9 -0
new file mode 100644
index 000000000000..54d6b70e1e32
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0-only
+config VFIO_PLATFORM_BCMPLT_MSI
+     tristate "MSI support for Broadcom platform devices"
+     depends on VFIO_PLATFORM && (ARCH_BCM_IPROC || COMPILE_TEST)
+     default ARCH_BCM_IPROC
+     help
+       Enables the VFIO platform driver to handle msi for Broadcom devices
+
+       If you don't know what to do here, say N.
drivers/vfio/platform/msi/Makefile new file +2 -0
new file mode 100644
index 000000000000..27422d45cecb
@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_VFIO_PLATFORM_BCMPLT_MSI) += vfio_platform_bcmplt.o
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +8 -0
new file mode 100644
index 000000000000..a074b5e92d77
@@ -0,0 +1,49 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2020 Broadcom.
+ */
+
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/interrupt.h>

drivers/vfio/platform/msi/vfio_platform_bcmplt.c line 18

Quoted 10 lines
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +10 -0
+#include <linux/msi.h>
+#include <linux/vfio.h>
+
+#include "../vfio_platform_private.h"
+
+#define RING_SIZE            (64 << 10)
+
+#define RING_MSI_ADDR_LS     0x03c
+#define RING_MSI_ADDR_MS     0x040
+#define RING_MSI_DATA_VALUE  0x064  

Those 3 defines would not be needed anymore with that implementation option.

Quoted 18 lines
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +18 -0
+
+static u32 bcm_num_msi(struct vfio_platform_device *vdev)
+{
+     struct vfio_platform_region *reg = &vdev->regions[0];
+
+     return (reg->size / RING_SIZE);
+}
+
+static struct vfio_platform_msi_node vfio_platform_bcmflexrm_msi_node = {
+     .owner = THIS_MODULE,
+     .compat = "brcm,iproc-flexrm-mbox",
+     .of_get_msi = bcm_num_msi,
+};
+
+static int __init vfio_platform_bcmflexrm_msi_module_init(void)
+{
+     __vfio_platform_register_msi(&vfio_platform_bcmflexrm_msi_node);
+

drivers/vfio/platform/msi/vfio_platform_bcmplt.c line 46

Quoted 10 lines
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +10 -0
+     return 0;
+}
+
+static void __exit vfio_platform_bcmflexrm_msi_module_exit(void)
+{
+     vfio_platform_unregister_msi("brcm,iproc-flexrm-mbox");
+}
+
+module_init(vfio_platform_bcmflexrm_msi_module_init);
+module_exit(vfio_platform_bcmflexrm_msi_module_exit);  

One thing I would like to discuss with Alex.

As the reset module is mandated (except if reset_required is forced to 0), I am wondering if we shouldn't try to turn the reset module into a "specialization" module and put the msi hooks there. I am afraid we may end up having modules for each and every vfio platform feature specialization. At the moment that's fully bearable but I can't predict what's next.

Quoted 4 lines

As the mandated feature is the reset capability maybe we could just keep the config/module name terminology, tune the kconfig help message to mention the msi support in case of flex-rm?

Quoted 6 lines

As I understand, your proposal is that we should not have a separate module for MSI, rather we add in the existing reset module for flex-rm. Thus, this way reset modules do not seem to be specialized just for reset functionality only but for MSI as well. Apart from this we need not to load the proposed msi module in this patch series. Is my understanding correct?

Quoted 6 lines

yes it is.

Quoted 2 lines

For me it looks OK to consolidate MSI in the existing 'reset' module. Let me know your views so that I can work for the next patch set accordingly.

Before you launch into the rewriting I would like to get the confirmation Alex is OK or if he prefers to keep separate modules.

Quoted 6 lines

If I understand correctly, the proposal here creates an entirely parallel vfio-msi request module interface like we have for vfio-reset, so the question is whether we should simplify vfio-platform-core to do a single module request per compat string and the device specific module would register multiple features rather than one per module. Is that right?

Yes that's correct, the so-called "reset" module would also implement msi hooks and if new specialization are needed in the future they also could be put there.

Quoted 9 lines

It seems the submodules are pretty simple, there's not a lot to be gained from duplicate boilerplate code in the modules themselves. The core code would clearly be simplified slightly to avoid multiple module requests, but for a more grand benefit is seems the registration interfaces would also need to be consolidated, perhaps providing a feature "ops" structure. As you indicate, having only two features at this point with a fairly small number of modules each, it's not yet too burdensome, but I could imagine it being a useful project.

Yes the registration must be reworked anyway.

Quoted 4 lines

More importantly in the short term, I'd expect modules handling the same compat string to be named similarly and enabled by a common Kconfig option. Thanks,

I agree with you.

Thanks

Eric

Quoted 3 lines

Alex

#48 lore raw

v4 cover msi support for platform devices

In reply to Vikas Gupta

This RFC adds support for MSI for platform devices. MSI block is added as an ext irq along with the existing wired interrupt implementation. The patchset exports two caps for MSI and related data to configure MSI source device.

Changes from:
-------------
v3 to v4:
1) Removed the 'cap' for exporting MSI info to userspace and restored into vedor specific module.
2) Enable GENERIC_MSI_IRQ_DOMAIN in Kconfig.
3) Removed the vendor specific, Broadcom, 'msi' module and integrated the MSI relates ops into the 'reset' module for MSI support.

v2 to v3:
1) Restored the vendor specific module to get max number of MSIs supported and .count value initialized.
2) Comments from Eric addressed.

v1 to v2:
1) IRQ allocation has been implemented as below:
----------------------------
|IRQ-0|IRQ-1|....|IRQ-n|MSI|
----------------------------
MSI block has msi contexts and its implemneted as ext irq.

2) Removed vendor specific module for msi handling so previously patch2 and patch3 are not required.

3) MSI related data is exported to userspace using 'caps'.
Please note VFIO_IRQ_INFO_CAP_TYPE in include/uapi/linux/vfio.h implementation
is taken from the Eric`s patch
https://patchwork.kernel.org/project/kvm/patch/[email protected]/

v0 to v1:
i) Removed MSI device flag VFIO_DEVICE_FLAGS_MSI. ii) Add MSI(s) at the end of the irq list of platform IRQs. MSI(s) with first entry of MSI block has count and flag information.
IRQ list: Allocation for IRQs + MSIs are allocated as below
Example: if there are 'n' IRQs and 'k' MSIs
-------------------------------------------------------
|IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
-------------------------------------------------------
MSI-0 will have count=k set and flags set accordingly.

Vikas Gupta (3):
vfio/platform: add support for msi
vfio/platform: change cleanup order
vfio: platform: reset: add msi support

 drivers/vfio/platform/Kconfig                 |   1 +
 .../platform/reset/vfio_platform_bcmflexrm.c  |  72 ++++-
 drivers/vfio/platform/vfio_platform_common.c  |  97 +++++--
 drivers/vfio/platform/vfio_platform_irq.c     | 253 ++++++++++++++++--
 drivers/vfio/platform/vfio_platform_private.h |  29 ++
 include/uapi/linux/vfio.h                     |  24 ++
 6 files changed, 444 insertions(+), 32 deletions(-)
Signature · 1 line
2.17.1
#49 lore raw

RFC v4 1/3 vfio/platform: add support for msi

MSI support for platform devices. MSI is added
as a single 'index' with 'count' as the number of
MSI(s) supported by the devices.

---
 drivers/vfio/platform/Kconfig                 |   1 +
 drivers/vfio/platform/vfio_platform_common.c  |  95 ++++++-
 drivers/vfio/platform/vfio_platform_irq.c     | 253 ++++++++++++++++--
 drivers/vfio/platform/vfio_platform_private.h |  29 ++
 include/uapi/linux/vfio.h                     |  24 ++
 5 files changed, 373 insertions(+), 29 deletions(-)
drivers/vfio/platform/Kconfig +1 -0
index dc1a3c44f2c6..d4bbc9f27763 100644
@@ -3,6 +3,7 @@ config VFIO_PLATFORM
 	tristate "VFIO support for platform devices"
 	depends on VFIO && EVENTFD && (ARM || ARM64)
 	select VFIO_VIRQFD
+	select GENERIC_MSI_IRQ_DOMAIN
 	help
 	  Support for platform devices with VFIO. This is required to make
 	  use of platform devices present on the system using the VFIO
drivers/vfio/platform/vfio_platform_common.c +81 -14

156 lines not on this page · raw message

drivers/vfio/platform/vfio_platform_irq.c +238 -15

297 lines not on this page · raw message

drivers/vfio/platform/vfio_platform_private.h +29 -0
index 289089910643..b8dd892aec53 100644
@@ -9,6 +9,7 @@
 
 #include <linux/types.h>
 #include <linux/interrupt.h>
+#include <linux/msi.h>
 
 #define VFIO_PLATFORM_OFFSET_SHIFT   40
 #define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
@@ -19,9 +20,21 @@
 #define VFIO_PLATFORM_INDEX_TO_OFFSET(index)	\
 	((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT)
 
+/* IRQ index for MSI in ext IRQs */
+#define VFIO_EXT_IRQ_MSI	0
+
+struct vfio_irq_ctx {
+	int			hwirq;
+	char			*name;
+	struct msi_msg		msg;
+	struct eventfd_ctx	*trigger;
+};
+
 struct vfio_platform_irq {
 	u32			flags;
 	u32			count;
+	int			num_ctx;
+	struct vfio_irq_ctx	*ctx;
 	int			hwirq;
 	char			*name;
 	struct eventfd_ctx	*trigger;
@@ -29,6 +42,11 @@ struct vfio_platform_irq {
 	spinlock_t		lock;
 	struct virqfd		*unmask;
 	struct virqfd		*mask;
+
+	/* for extended irqs */
+	u32			type;
+	u32			subtype;
+	int			config_msi;
 };
 
 struct vfio_platform_region {
@@ -46,6 +64,7 @@ struct vfio_platform_device {
 	u32				num_regions;
 	struct vfio_platform_irq	*irqs;
 	u32				num_irqs;
+	int				num_ext_irqs;
 	int				refcnt;
 	struct mutex			igate;
 	struct module			*parent_module;
@@ -65,17 +84,27 @@ struct vfio_platform_device {
 		(*get_resource)(struct vfio_platform_device *vdev, int i);
 	int	(*get_irq)(struct vfio_platform_device *vdev, int i);
 	int	(*of_reset)(struct vfio_platform_device *vdev);
+	u32	(*of_get_msi)(struct vfio_platform_device *vdev);
+	void	(*of_msi_write)(struct vfio_platform_device *vdev,
+				struct msi_desc *desc,
+				struct msi_msg *msg);
 
 	bool				reset_required;
 };
 
 typedef int (*vfio_platform_reset_fn_t)(struct vfio_platform_device *vdev);
+typedef u32 (*vfio_platform_get_msi_fn_t)(struct vfio_platform_device *vdev);
+typedef void (*vfio_platform_msi_write_fn_t)(struct vfio_platform_device *vdev,
+					     struct msi_desc *desc,
+					     struct msi_msg *msg);
 
 struct vfio_platform_reset_node {
 	struct list_head link;
 	char *compat;
 	struct module *owner;
 	vfio_platform_reset_fn_t of_reset;
+	vfio_platform_get_msi_fn_t of_get_msi;
+	vfio_platform_msi_write_fn_t of_msi_write;
 };
 
 extern int vfio_platform_probe_common(struct vfio_platform_device *vdev,
include/uapi/linux/vfio.h +24 -0
index d1812777139f..8e2c0131781d 100644
@@ -697,11 +697,35 @@ struct vfio_irq_info {
 #define VFIO_IRQ_INFO_MASKABLE		(1 << 1)
 #define VFIO_IRQ_INFO_AUTOMASKED	(1 << 2)
 #define VFIO_IRQ_INFO_NORESIZE		(1 << 3)
+#define VFIO_IRQ_INFO_FLAG_CAPS		(1 << 4) /* Info supports caps */
 	__u32	index;		/* IRQ index */
 	__u32	count;		/* Number of IRQs within this index */
+	__u32	cap_offset;	/* Offset within info struct of first cap */
 };
 #define VFIO_DEVICE_GET_IRQ_INFO	_IO(VFIO_TYPE, VFIO_BASE + 9)
 
+/*
+ * The irq type capability allows IRQs unique to a specific device or
+ * class of devices to be exposed.
+ *
+ * The structures below define version 1 of this capability.
+ */
+#define VFIO_IRQ_INFO_CAP_TYPE		3
+
+struct vfio_irq_info_cap_type {
+	struct vfio_info_cap_header header;
+	__u32 type;     /* global per bus driver */
+	__u32 subtype;  /* type specific */
+};
+
+/*
+ * List of IRQ types, global per bus driver.
+ * If you introduce a new type, please add it here.
+ */
+
+/* Non PCI devices having MSI(s) support */
+#define VFIO_IRQ_TYPE_MSI		(1)
+
 /**
  * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set)
  *
Signature · 1 line
2.17.1
maintainer
#50 lore raw

Re: [RFC v4 1/3] vfio/platform: add support for msi

Hi Vikas,

Quoted 1 line · Vikas Gupta

MSI support for platform devices. MSI is added

s/MSI support/ Add MSI support

Quoted 2 lines

as a single 'index' with 'count' as the number of
MSI(s) supported by the devices.

as a single 'index' following the last wired irq index index, with count.

It allows to associate eventfds to MSIs.

If MSI is supported, specialization callbacks need to be implemented in the reset module (of_get_msi and of_msi_write).

Quoted 35 lines
---
 drivers/vfio/platform/Kconfig                 |   1 +
 drivers/vfio/platform/vfio_platform_common.c  |  95 ++++++-
 drivers/vfio/platform/vfio_platform_irq.c     | 253 ++++++++++++++++--
 drivers/vfio/platform/vfio_platform_private.h |  29 ++
 include/uapi/linux/vfio.h                     |  24 ++
 5 files changed, 373 insertions(+), 29 deletions(-)
drivers/vfio/platform/Kconfig +1 -0
index dc1a3c44f2c6..d4bbc9f27763 100644
@@ -3,6 +3,7 @@ config VFIO_PLATFORM
 	tristate "VFIO support for platform devices"
 	depends on VFIO && EVENTFD && (ARM || ARM64)
 	select VFIO_VIRQFD
+	select GENERIC_MSI_IRQ_DOMAIN
 	help
 	  Support for platform devices with VFIO. This is required to make
 	  use of platform devices present on the system using the VFIO
drivers/vfio/platform/vfio_platform_common.c +1 -0
index fb4b385191f2..f2b1f0c3bfcc 100644
@@ -16,6 +16,7 @@
 #include <linux/types.h>
 #include <linux/uaccess.h>
 #include <linux/vfio.h>
+#include <linux/nospec.h>
 
 #include "vfio_platform_private.h"
 

drivers/vfio/platform/vfio_platform_common.c line 34

Quoted 8 lines
drivers/vfio/platform/vfio_platform_common.c +3 -2
@@ -28,23 +29,22 @@
 static LIST_HEAD(reset_list);
 static DEFINE_MUTEX(driver_lock);
 
-static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,
-					struct module **module)
+static void vfio_platform_lookup_reset(const char *compat,
+				       struct module **module,
+				       struct vfio_platform_reset_node **node)

nit: I would prefer this function directly returns a struct vfio_platform_reset_node *

Quoted 9 lines
drivers/vfio/platform/vfio_platform_common.c +0 -1
 {
 	struct vfio_platform_reset_node *iter;
-	vfio_platform_reset_fn_t reset_fn = NULL;
 
 	mutex_lock(&driver_lock);
 	list_for_each_entry(iter, &reset_list, link) {
 		if (!strcmp(iter->compat, compat) &&
 			try_module_get(iter->owner)) {
 			*module = iter->owner;

drivers/vfio/platform/vfio_platform_common.c

Quoted 10 lines
drivers/vfio/platform/vfio_platform_common.c +1 -2
-			reset_fn = iter->of_reset;
+			*node = iter;
 			break;
 		}
 	}
 	mutex_unlock(&driver_lock);
-	return reset_fn;
 }
 
 static int vfio_platform_acpi_probe(struct vfio_platform_device *vdev,> @@ -112,15 +112,23 @@ static bool vfio_platform_has_reset(struct

vfio_platform_device *vdev)

Quoted 289 lines

static int vfio_platform_get_reset(struct vfio_platform_device *vdev) {

drivers/vfio/platform/vfio_platform_common.c +76 -9
+	struct vfio_platform_reset_node *node = NULL;
+
 	if (VFIO_PLATFORM_IS_ACPI(vdev))
 		return vfio_platform_acpi_has_reset(vdev) ? 0 : -ENOENT;
 
-	vdev->of_reset = vfio_platform_lookup_reset(vdev->compat,
-						    &vdev->reset_module);
-	if (!vdev->of_reset) {
+	vfio_platform_lookup_reset(vdev->compat, &vdev->reset_module,
+				   &node);
+	if (!node) {
 		request_module("vfio-reset:%s", vdev->compat);
-		vdev->of_reset = vfio_platform_lookup_reset(vdev->compat,
-							&vdev->reset_module);
+		vfio_platform_lookup_reset(vdev->compat, &vdev->reset_module,
+					   &node);
+	}
+
+	if (node) {
+		vdev->of_reset = node->of_reset;
+		vdev->of_get_msi = node->of_get_msi;
+		vdev->of_msi_write = node->of_msi_write;>  	}
 
 	return vdev->of_reset ? 0 : -ENOENT;
@@ -343,9 +351,16 @@ static long vfio_platform_ioctl(void *device_data,
 
 	} else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
 		struct vfio_irq_info info;
+		struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
+		int ext_irq_index = vdev->num_irqs - vdev->num_ext_irqs;
+		unsigned long capsz;
+		u32 index;
 
 		minsz = offsetofend(struct vfio_irq_info, count);
 
+		/* For backward compatibility, cannot require this */
+		capsz = offsetofend(struct vfio_irq_info, cap_offset);
+
 		if (copy_from_user(&info, (void __user *)arg, minsz))
 			return -EFAULT;
 
@@ -355,8 +370,53 @@ static long vfio_platform_ioctl(void *device_data,
 		if (info.index >= vdev->num_irqs)
 			return -EINVAL;
 
-		info.flags = vdev->irqs[info.index].flags;
-		info.count = vdev->irqs[info.index].count;
+		if (info.argsz >= capsz)
+			minsz = capsz;
+
+		index = info.index;
+
+		info.flags = vdev->irqs[index].flags;
+		info.count = vdev->irqs[index].count;
+
+		if (ext_irq_index - index == VFIO_EXT_IRQ_MSI) {
+			struct vfio_irq_info_cap_type cap_type = {
+				.header.id = VFIO_IRQ_INFO_CAP_TYPE,
+				.header.version = 1 };
+			struct vfio_platform_irq *irq;
+			int ret;
+
+			index = array_index_nospec(index,
+						   vdev->num_irqs);
+			irq = &vdev->irqs[index];
+
+			cap_type.type = irq->type;
+			cap_type.subtype = irq->subtype;
+
+			ret = vfio_info_add_capability(&caps,
+						       &cap_type.header,
+						       sizeof(cap_type));
+			if (ret)
+				return ret;
+		}
+
+		if (caps.size) {
+			info.flags |= VFIO_IRQ_INFO_FLAG_CAPS;
+			if (info.argsz < sizeof(info) + caps.size) {
+				info.argsz = sizeof(info) + caps.size;
+				info.cap_offset = 0;
+			} else {
+				vfio_info_cap_shift(&caps, sizeof(info));
+				if (copy_to_user((void __user *)arg +
+						  sizeof(info), caps.buf,
+						  caps.size)) {
+					kfree(caps.buf);
+					return -EFAULT;
+				}
+				info.cap_offset = sizeof(info);
+			}
+
+			kfree(caps.buf);
+		}
 
 		return copy_to_user((void __user *)arg, &info, minsz) ?
 			-EFAULT : 0;
@@ -365,6 +425,7 @@ static long vfio_platform_ioctl(void *device_data,
 		struct vfio_irq_set hdr;
 		u8 *data = NULL;
 		int ret = 0;
+		int max;
 		size_t data_size = 0;
 
 		minsz = offsetofend(struct vfio_irq_set, count);
@@ -372,8 +433,14 @@ static long vfio_platform_ioctl(void *device_data,
 		if (copy_from_user(&hdr, (void __user *)arg, minsz))
 			return -EFAULT;
 
-		ret = vfio_set_irqs_validate_and_prepare(&hdr, vdev->num_irqs,
-						 vdev->num_irqs, &data_size);
+		if (hdr.index >= vdev->num_irqs)
+			return -EINVAL;
+
+		max = vdev->irqs[hdr.index].count;
+
+		ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
+							 vdev->num_irqs,
+							 &data_size);
 		if (ret)
 			return ret;
 
drivers/vfio/platform/vfio_platform_irq.c +145 -0
index c5b09ec0a3c9..db60240d27ca 100644
@@ -8,10 +8,12 @@
 
 #include <linux/eventfd.h>
 #include <linux/interrupt.h>
+#include <linux/eventfd.h>
 #include <linux/slab.h>
 #include <linux/types.h>
 #include <linux/vfio.h>
 #include <linux/irq.h>
+#include <linux/msi.h>
 
 #include "vfio_platform_private.h"
 
@@ -253,6 +255,186 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,
 	return 0;
 }
 
+/* MSI/MSIX */
+static irqreturn_t vfio_msihandler(int irq, void *arg)
+{
+	struct eventfd_ctx *trigger = arg;
+
+	eventfd_signal(trigger, 1);
+	return IRQ_HANDLED;
+}
+
+static void msi_write(struct msi_desc *desc, struct msi_msg *msg)
+{
+	struct device *dev = msi_desc_to_dev(desc);
+	struct vfio_device *device = dev_get_drvdata(dev);
+	struct vfio_platform_device *vdev = (struct vfio_platform_device *)
+						vfio_device_data(device);
+
+	vdev->of_msi_write(vdev, desc, msg);
+}
+
+static int vfio_msi_enable(struct vfio_platform_device *vdev,
+			   struct vfio_platform_irq *irq, int nvec)
+{
+	int ret;
+	int msi_idx = 0;
+	struct msi_desc *desc;
+	struct device *dev = vdev->device;
+
+	irq->ctx = kcalloc(nvec, sizeof(struct vfio_irq_ctx), GFP_KERNEL);
+	if (!irq->ctx)
+		return -ENOMEM;
+
+	/* Allocate platform MSIs */
+	ret = platform_msi_domain_alloc_irqs(dev, nvec, msi_write);
+	if (ret < 0) {
+		kfree(irq->ctx);
+		return ret;
+	}
+
+	for_each_msi_entry(desc, dev) {
+		irq->ctx[msi_idx].hwirq = desc->irq;
+		msi_idx++;
+	}
+
+	irq->num_ctx = nvec;
+	irq->config_msi = 1;
+
+	return 0;
+}
+
+static int vfio_msi_set_vector_signal(struct vfio_platform_irq *irq,
+				      int vector, int fd)
+{
+	struct eventfd_ctx *trigger;
+	int irq_num, ret;
+
+	if (vector < 0 || vector >= irq->num_ctx)
+		return -EINVAL;
+
+	irq_num = irq->ctx[vector].hwirq;
+
+	if (irq->ctx[vector].trigger) {
+		free_irq(irq_num, irq->ctx[vector].trigger);
+		kfree(irq->ctx[vector].name);
+		eventfd_ctx_put(irq->ctx[vector].trigger);
+		irq->ctx[vector].trigger = NULL;
+	}
+
+	if (fd < 0)
+		return 0;
+
+	irq->ctx[vector].name = kasprintf(GFP_KERNEL,
+					  "vfio-msi[%d]", vector);
+	if (!irq->ctx[vector].name)
+		return -ENOMEM;
+
+	trigger = eventfd_ctx_fdget(fd);
+	if (IS_ERR(trigger)) {
+		kfree(irq->ctx[vector].name);
+		return PTR_ERR(trigger);
+	}
+
+	ret = request_irq(irq_num, vfio_msihandler, 0,
+			  irq->ctx[vector].name, trigger);
+	if (ret) {
+		kfree(irq->ctx[vector].name);
+		eventfd_ctx_put(trigger);
+		return ret;
+	}
+
+	irq->ctx[vector].trigger = trigger;
+
+	return 0;
+}
+
+static int vfio_msi_set_block(struct vfio_platform_irq *irq, unsigned int start,
+			      unsigned int count, int32_t *fds)
+{
+	int i, j, ret = 0;
+
+	if (start >= irq->num_ctx || start + count > irq->num_ctx)
+		return -EINVAL;
+
+	for (i = 0, j = start; i < count && !ret; i++, j++) {
+		int fd = fds ? fds[i] : -1;
+
+		ret = vfio_msi_set_vector_signal(irq, j, fd);
+	}
+
+	if (ret) {
+		for (--j; j >= (int)start; j--)
+			vfio_msi_set_vector_signal(irq, j, -1);
+	}
+
+	return ret;
+}
+
+static void vfio_msi_disable(struct vfio_platform_device *vdev,
+			     struct vfio_platform_irq *irq)
+{
+	struct device *dev = vdev->device;
+
+	vfio_msi_set_block(irq, 0, irq->num_ctx, NULL);
+
+	platform_msi_domain_free_irqs(dev);
+
+	irq->config_msi = 0;
+	irq->num_ctx = 0;
+
+	kfree(irq->ctx);
+}
+
+static int vfio_set_msi_trigger(struct vfio_platform_device *vdev,
+				unsigned int index, unsigned int start,
+				unsigned int count, uint32_t flags, void *data)
+{
+	int i;
+	struct vfio_platform_irq *irq = &vdev->irqs[index];
+
+	if (start + count > irq->count)
+		return -EINVAL;
+
+	if (!count && (flags & VFIO_IRQ_SET_DATA_NONE)) {
+		vfio_msi_disable(vdev, irq);

drivers/vfio/platform/vfio_platform_irq.c static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev, · line 410

Quoted 10 lines
drivers/vfio/platform/vfio_platform_irq.c +10 -0
+		return 0;
+	}
+
+	if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
+		s32 *fds = data;
+		int ret;
+
+		if (irq->config_msi)
+			return vfio_msi_set_block(irq, start, count,
+						  fds);

looks a bit weird to me that in case of failure we do not do the same as below (ie. vfio_msi_disable). The pci code does the same but I fail to understand the logic.

Quoted 77 lines
drivers/vfio/platform/vfio_platform_irq.c +52 -11
+		ret = vfio_msi_enable(vdev, irq, start + count);
+		if (ret)
+			return ret;
+
+		ret = vfio_msi_set_block(irq, start, count, fds);
+		if (ret)
+			vfio_msi_disable(vdev, irq);
+
+		return ret;
+	}
+
+	for (i = start; i < start + count; i++) {
+		if (!irq->ctx[i].trigger)
+			continue;
+		if (flags & VFIO_IRQ_SET_DATA_NONE) {
+			eventfd_signal(irq->ctx[i].trigger, 1);
+		} else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
+			u8 *bools = data;
+
+			if (bools[i - start])
+				eventfd_signal(irq->ctx[i].trigger, 1);
+		}
+	}
+
+	return 0;
+}
+
 int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 				 uint32_t flags, unsigned index, unsigned start,
 				 unsigned count, void *data)
@@ -261,16 +443,29 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 		    unsigned start, unsigned count, uint32_t flags,
 		    void *data) = NULL;
 
-	switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
-	case VFIO_IRQ_SET_ACTION_MASK:
-		func = vfio_platform_set_irq_mask;
-		break;
-	case VFIO_IRQ_SET_ACTION_UNMASK:
-		func = vfio_platform_set_irq_unmask;
-		break;
-	case VFIO_IRQ_SET_ACTION_TRIGGER:
-		func = vfio_platform_set_irq_trigger;
-		break;
+	struct vfio_platform_irq *irq = &vdev->irqs[index];
+
+	if (irq->type == VFIO_IRQ_TYPE_MSI) {
+		switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
+		case VFIO_IRQ_SET_ACTION_MASK:
+		case VFIO_IRQ_SET_ACTION_UNMASK:
+			break;
+		case VFIO_IRQ_SET_ACTION_TRIGGER:
+			func = vfio_set_msi_trigger;
+			break;
+		}
+	} else {
+		switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
+		case VFIO_IRQ_SET_ACTION_MASK:
+			func = vfio_platform_set_irq_mask;
+			break;
+		case VFIO_IRQ_SET_ACTION_UNMASK:
+			func = vfio_platform_set_irq_unmask;
+			break;
+		case VFIO_IRQ_SET_ACTION_TRIGGER:
+			func = vfio_platform_set_irq_trigger;
+			break;
+		}
 	}
 
 	if (!func)
@@ -281,12 +476,23 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 
 int vfio_platform_irq_init(struct vfio_platform_device *vdev)
 {
-	int cnt = 0, i;
+	int i;
+	int cnt = 0;

drivers/vfio/platform/vfio_platform_irq.c int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev, · line 489

Quoted 10 lines
drivers/vfio/platform/vfio_platform_irq.c +5 -1
+	int num_irqs = 0;
+	int msi_cnt = 0;
 
 	while (vdev->get_irq(vdev, cnt) >= 0)
 		cnt++;
 
-	vdev->irqs = kcalloc(cnt, sizeof(struct vfio_platform_irq), GFP_KERNEL);
+	num_irqs = cnt;
+
+	if (vdev->of_get_msi) {

I think you also need to test that of_msi_write also is non null?

Quoted 81 lines
drivers/vfio/platform/vfio_platform_irq.c +26 -3
+		msi_cnt = vdev->of_get_msi(vdev);
+		num_irqs++;
+	}
+
+	vdev->irqs = kcalloc(num_irqs, sizeof(struct vfio_platform_irq),
+			     GFP_KERNEL);
 	if (!vdev->irqs)
 		return -ENOMEM;
 
@@ -309,7 +515,19 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev)
 		vdev->irqs[i].masked = false;
 	}
 
-	vdev->num_irqs = cnt;
+	/*
+	 * MSI block is added at last index and it is an ext irq
+	 */
+	if (msi_cnt > 0) {
+		vdev->irqs[i].flags = VFIO_IRQ_INFO_EVENTFD;
+		vdev->irqs[i].count = msi_cnt;
+		vdev->irqs[i].hwirq = 0;
+		vdev->irqs[i].masked = false;
+		vdev->irqs[i].type = VFIO_IRQ_TYPE_MSI;
+		vdev->num_ext_irqs = 1;
+	}
+
+	vdev->num_irqs = num_irqs;
 
 	return 0;
 err:
@@ -321,8 +539,13 @@ void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev)
 {
 	int i;
 
-	for (i = 0; i < vdev->num_irqs; i++)
-		vfio_set_trigger(vdev, i, -1, NULL);
+	for (i = 0; i < vdev->num_irqs; i++) {
+		if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
+			vfio_set_msi_trigger(vdev, i, 0, 0,
+					     VFIO_IRQ_SET_DATA_NONE, NULL);
+		else
+			vfio_set_trigger(vdev, i, -1, NULL);
+	}
 
 	vdev->num_irqs = 0;
 	kfree(vdev->irqs);
drivers/vfio/platform/vfio_platform_private.h +13 -0
index 289089910643..b8dd892aec53 100644
@@ -9,6 +9,7 @@
 
 #include <linux/types.h>
 #include <linux/interrupt.h>
+#include <linux/msi.h>
 
 #define VFIO_PLATFORM_OFFSET_SHIFT   40
 #define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
@@ -19,9 +20,21 @@
 #define VFIO_PLATFORM_INDEX_TO_OFFSET(index)	\
 	((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT)
 
+/* IRQ index for MSI in ext IRQs */
+#define VFIO_EXT_IRQ_MSI	0
+
+struct vfio_irq_ctx {
+	int			hwirq;
+	char			*name;
+	struct msi_msg		msg;
+	struct eventfd_ctx	*trigger;
+};
+
 struct vfio_platform_irq {
 	u32			flags;
 	u32			count;
+	int			num_ctx;
+	struct vfio_irq_ctx	*ctx;
 	int			hwirq;
 	char			*name;
 	struct eventfd_ctx	*trigger;

drivers/vfio/platform/vfio_platform_private.h struct vfio_platform_irq { · line 49

Quoted 8 lines
drivers/vfio/platform/vfio_platform_private.h +5 -0
@@ -29,6 +42,11 @@ struct vfio_platform_irq {
 	spinlock_t		lock;
 	struct virqfd		*unmask;
 	struct virqfd		*mask;
+
+	/* for extended irqs */
+	u32			type;
+	u32			subtype;
+	int			config_msi;

bool?

Quoted 70 lines
drivers/vfio/platform/vfio_platform_private.h +11 -0
 };
 
 struct vfio_platform_region {
@@ -46,6 +64,7 @@ struct vfio_platform_device {
 	u32				num_regions;
 	struct vfio_platform_irq	*irqs;
 	u32				num_irqs;
+	int				num_ext_irqs;
 	int				refcnt;
 	struct mutex			igate;
 	struct module			*parent_module;
@@ -65,17 +84,27 @@ struct vfio_platform_device {
 		(*get_resource)(struct vfio_platform_device *vdev, int i);
 	int	(*get_irq)(struct vfio_platform_device *vdev, int i);
 	int	(*of_reset)(struct vfio_platform_device *vdev);
+	u32	(*of_get_msi)(struct vfio_platform_device *vdev);
+	void	(*of_msi_write)(struct vfio_platform_device *vdev,
+				struct msi_desc *desc,
+				struct msi_msg *msg);
 
 	bool				reset_required;
 };
 
 typedef int (*vfio_platform_reset_fn_t)(struct vfio_platform_device *vdev);
+typedef u32 (*vfio_platform_get_msi_fn_t)(struct vfio_platform_device *vdev);
+typedef void (*vfio_platform_msi_write_fn_t)(struct vfio_platform_device *vdev,
+					     struct msi_desc *desc,
+					     struct msi_msg *msg);
 
 struct vfio_platform_reset_node {
 	struct list_head link;
 	char *compat;
 	struct module *owner;
 	vfio_platform_reset_fn_t of_reset;
+	vfio_platform_get_msi_fn_t of_get_msi;
+	vfio_platform_msi_write_fn_t of_msi_write;
 };
 
 extern int vfio_platform_probe_common(struct vfio_platform_device *vdev,
include/uapi/linux/vfio.h +17 -0
index d1812777139f..8e2c0131781d 100644
@@ -697,11 +697,35 @@ struct vfio_irq_info {
 #define VFIO_IRQ_INFO_MASKABLE		(1 << 1)
 #define VFIO_IRQ_INFO_AUTOMASKED	(1 << 2)
 #define VFIO_IRQ_INFO_NORESIZE		(1 << 3)
+#define VFIO_IRQ_INFO_FLAG_CAPS		(1 << 4) /* Info supports caps */
 	__u32	index;		/* IRQ index */
 	__u32	count;		/* Number of IRQs within this index */
+	__u32	cap_offset;	/* Offset within info struct of first cap */
 };
 #define VFIO_DEVICE_GET_IRQ_INFO	_IO(VFIO_TYPE, VFIO_BASE + 9)
 
+/*
+ * The irq type capability allows IRQs unique to a specific device or
+ * class of devices to be exposed.
+ *
+ * The structures below define version 1 of this capability.
+ */
+#define VFIO_IRQ_INFO_CAP_TYPE		3
+
+struct vfio_irq_info_cap_type {
+	struct vfio_info_cap_header header;
+	__u32 type;     /* global per bus driver */
+	__u32 subtype;  /* type specific */
+};
+
+/*

include/uapi/linux/vfio.h struct vfio_irq_info { · line 731

Quoted 10 lines
include/uapi/linux/vfio.h +7 -0
+ * List of IRQ types, global per bus driver.
+ * If you introduce a new type, please add it here.
+ */
+
+/* Non PCI devices having MSI(s) support */
+#define VFIO_IRQ_TYPE_MSI		(1)
+
 /**
  * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set)
  *

Besides looks good to me.

Thanks

Eric

#51 lore raw

RFC v4 2/3 vfio/platform: change cleanup order

In reply to Vikas Gupta

In the case of msi, vendor specific msi module may require region access to handle msi cleanup so we need to cleanup region after irq cleanup only.

---
 drivers/vfio/platform/vfio_platform_common.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/vfio/platform/vfio_platform_common.c +1 -1
index f2b1f0c3bfcc..1cc040e3ed1f 100644
@@ -243,8 +243,8 @@ static void vfio_platform_release(void *device_data)
 			WARN_ON(1);
 		}
 		pm_runtime_put(vdev->device);
-		vfio_platform_regions_cleanup(vdev);
 		vfio_platform_irq_cleanup(vdev);
+		vfio_platform_regions_cleanup(vdev);
 	}
 
 	mutex_unlock(&driver_lock);
Signature · 1 line
2.17.1
maintainer Acked-by on v4 2/3
#52 lore raw

Re: [RFC v4 2/3] vfio/platform: change cleanup order

Hi Vikas,

Quoted 5 lines · Vikas Gupta

In the case of msi, vendor specific msi module may require region access to handle msi cleanup so we need to cleanup region after irq cleanup only.

Thanks

Eric

Quoted 19 lines
---
 drivers/vfio/platform/vfio_platform_common.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/vfio/platform/vfio_platform_common.c +1 -1
index f2b1f0c3bfcc..1cc040e3ed1f 100644
@@ -243,8 +243,8 @@ static void vfio_platform_release(void *device_data)
 			WARN_ON(1);
 		}
 		pm_runtime_put(vdev->device);
-		vfio_platform_regions_cleanup(vdev);
 		vfio_platform_irq_cleanup(vdev);
+		vfio_platform_regions_cleanup(vdev);
 	}
 
 	mutex_unlock(&driver_lock);
#53 lore raw

RFC v4 3/3 vfio: platform: reset: add msi support

In reply to Vikas Gupta

Add msi support for Broadcom FlexRm device.

---
 .../platform/reset/vfio_platform_bcmflexrm.c  | 72 ++++++++++++++++++-
 1 file changed, 70 insertions(+), 2 deletions(-)
drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c +70 -2
index 96064ef8f629..6ca4ca12575b 100644
@@ -21,7 +21,9 @@
 #include <linux/init.h>
 #include <linux/io.h>
 #include <linux/kernel.h>
+#include <linux/msi.h>
 #include <linux/module.h>
+#include <linux/vfio.h>
 
 #include "../vfio_platform_private.h"
 
@@ -33,6 +35,9 @@
 #define RING_VER					0x000
 #define RING_CONTROL					0x034
 #define RING_FLUSH_DONE					0x038
+#define RING_MSI_ADDR_LS				0x03c
+#define RING_MSI_ADDR_MS				0x040
+#define RING_MSI_DATA_VALUE				0x064
 
 /* Register RING_CONTROL fields */
 #define CONTROL_FLUSH_SHIFT				5
@@ -105,8 +110,71 @@ static int vfio_platform_bcmflexrm_reset(struct vfio_platform_device *vdev)
 	return ret;
 }
 
-module_vfio_reset_handler("brcm,iproc-flexrm-mbox",
-			  vfio_platform_bcmflexrm_reset);
+static u32 bcm_num_msi(struct vfio_platform_device *vdev)
+{
+	struct vfio_platform_region *reg = &vdev->regions[0];
+
+	return (reg->size / RING_REGS_SIZE);
+}
+
+static void bcm_write_msi(struct vfio_platform_device *vdev,
+		struct msi_desc *desc,
+		struct msi_msg *msg)
+{
+	int i;
+	int hwirq = -1;
+	int msi_src;
+	void __iomem *ring;
+	struct vfio_platform_region *reg = &vdev->regions[0];
+
+	if (!reg)
+		return;
+
+	for (i = 0; i < vdev->num_irqs; i++)
+		if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
+			hwirq = vdev->irqs[i].ctx[0].hwirq;
+
+	if (hwirq < 0)
+		return;
+
+	msi_src = desc->irq - hwirq;
+
+	if (!reg->ioaddr) {
+		reg->ioaddr = ioremap(reg->addr, reg->size);
+		if (!reg->ioaddr)
+			return;
+	}
+
+	ring = reg->ioaddr + msi_src * RING_REGS_SIZE;
+
+	writel_relaxed(msg->address_lo, ring + RING_MSI_ADDR_LS);
+	writel_relaxed(msg->address_hi, ring + RING_MSI_ADDR_MS);
+	writel_relaxed(msg->data, ring + RING_MSI_DATA_VALUE);
+}
+
+static struct vfio_platform_reset_node vfio_platform_bcmflexrm_reset_node = {
+	.owner = THIS_MODULE,
+	.compat = "brcm,iproc-flexrm-mbox",
+	.of_reset = vfio_platform_bcmflexrm_reset,
+	.of_get_msi = bcm_num_msi,
+	.of_msi_write = bcm_write_msi
+};
+
+static int __init vfio_platform_bcmflexrm_reset_module_init(void)
+{
+	__vfio_platform_register_reset(&vfio_platform_bcmflexrm_reset_node);
+
+	return 0;
+}
+
+static void __exit vfio_platform_bcmflexrm_reset_module_exit(void)
+{
+	vfio_platform_unregister_reset("brcm,iproc-flexrm-mbox",
+				       vfio_platform_bcmflexrm_reset);
+}
+
+module_init(vfio_platform_bcmflexrm_reset_module_init);
+module_exit(vfio_platform_bcmflexrm_reset_module_exit);
 
 MODULE_LICENSE("GPL v2");
 MODULE_AUTHOR("Anup Patel <[email protected]>");
Signature · 1 line
2.17.1
maintainer
#54 lore raw

Re: [RFC v4 3/3] vfio: platform: reset: add msi support

Hi Vikas,

Quoted 32 lines · Vikas Gupta

Add msi support for Broadcom FlexRm device.

---
 .../platform/reset/vfio_platform_bcmflexrm.c  | 72 ++++++++++++++++++-
 1 file changed, 70 insertions(+), 2 deletions(-)
drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c +5 -0
index 96064ef8f629..6ca4ca12575b 100644
@@ -21,7 +21,9 @@
 #include <linux/init.h>
 #include <linux/io.h>
 #include <linux/kernel.h>
+#include <linux/msi.h>
 #include <linux/module.h>
+#include <linux/vfio.h>
 
 #include "../vfio_platform_private.h"
 
@@ -33,6 +35,9 @@
 #define RING_VER					0x000
 #define RING_CONTROL					0x034
 #define RING_FLUSH_DONE					0x038
+#define RING_MSI_ADDR_LS				0x03c
+#define RING_MSI_ADDR_MS				0x040
+#define RING_MSI_DATA_VALUE				0x064
 
 /* Register RING_CONTROL fields */
 #define CONTROL_FLUSH_SHIFT				5

drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c static int vfio_platform_bcmflexrm_reset(struct vfio_platform_device *vdev) · line 113

Quoted 6 lines
drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c +1 -2
@@ -105,8 +110,71 @@ static int vfio_platform_bcmflexrm_reset(struct vfio_platform_device *vdev)
 	return ret;
 }
 
-module_vfio_reset_handler("brcm,iproc-flexrm-mbox",
-			  vfio_platform_bcmflexrm_reset);
+static u32 bcm_num_msi(struct vfio_platform_device *vdev)

Please use the same prefix as for reset, ie. vfio_platform_bcmflexrm_get_msi

drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c static int vfio_platform_bcmflexrm_reset(struct vfio_platform_device *vdev) · line 120

Quoted 6 lines
drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c +6 -0
+{
+	struct vfio_platform_region *reg = &vdev->regions[0];> +
+	return (reg->size / RING_REGS_SIZE);
+}
+
+static void bcm_write_msi(struct vfio_platform_device *vdev,

vfio_platform_bcmflexrm_msi_write?

drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c static int vfio_platform_bcmflexrm_reset(struct vfio_platform_device *vdev) · line 131

Quoted 11 lines
drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c +11 -0
+		struct msi_desc *desc,
+		struct msi_msg *msg)
+{
+	int i;
+	int hwirq = -1;
+	int msi_src;
+	void __iomem *ring;
+	struct vfio_platform_region *reg = &vdev->regions[0];
+
+	if (!reg)
+		return;

why do you need this check? For this to be called, SET_IRQ IOCTL must have been called. This can only happen after vfio_platform_open which first calls vfio_platform_regions_init and then vfio_platform_irq_init

drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c static int vfio_platform_bcmflexrm_reset(struct vfio_platform_device *vdev) · line 135

Quoted 4 lines
drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c +4 -0
+
+	for (i = 0; i < vdev->num_irqs; i++)
+		if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
+			hwirq = vdev->irqs[i].ctx[0].hwirq;

nit: It would have been easier to record in vdev the last index of wired interrupts and just add the index of the MSI

drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c static int vfio_platform_bcmflexrm_reset(struct vfio_platform_device *vdev) · line 144

Quoted 9 lines
drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c +9 -0
+
+	if (hwirq < 0)
+		return;
+
+	msi_src = desc->irq - hwirq;
+
+	if (!reg->ioaddr) {
+		reg->ioaddr = ioremap(reg->addr, reg->size);
+		if (!reg->ioaddr)

pr_warn_once("")?

Quoted 27 lines
drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c +26 -0
+			return;
+	}
+
+	ring = reg->ioaddr + msi_src * RING_REGS_SIZE;
+
+	writel_relaxed(msg->address_lo, ring + RING_MSI_ADDR_LS);
+	writel_relaxed(msg->address_hi, ring + RING_MSI_ADDR_MS);
+	writel_relaxed(msg->data, ring + RING_MSI_DATA_VALUE);
+}
+
+static struct vfio_platform_reset_node vfio_platform_bcmflexrm_reset_node = {
+	.owner = THIS_MODULE,
+	.compat = "brcm,iproc-flexrm-mbox",
+	.of_reset = vfio_platform_bcmflexrm_reset,
+	.of_get_msi = bcm_num_msi,
+	.of_msi_write = bcm_write_msi
+};
+
+static int __init vfio_platform_bcmflexrm_reset_module_init(void)
+{
+	__vfio_platform_register_reset(&vfio_platform_bcmflexrm_reset_node);
+
+	return 0;
+}
+
+static void __exit vfio_platform_bcmflexrm_reset_module_exit(void)

drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c static int vfio_platform_bcmflexrm_reset(struct vfio_platform_device *vdev) · line 180

Quoted 10 lines
drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c +7 -0
+{
+	vfio_platform_unregister_reset("brcm,iproc-flexrm-mbox",
+				       vfio_platform_bcmflexrm_reset);
+}
+
+module_init(vfio_platform_bcmflexrm_reset_module_init);
+module_exit(vfio_platform_bcmflexrm_reset_module_exit);
 
 MODULE_LICENSE("GPL v2");
 MODULE_AUTHOR("Anup Patel <[email protected]>");

I think you should move the whole series in PATCH now.

Thanks

Eric