msi support for platform devices
Subsystems (3)
cover msi support for platform devices
This RFC adds support for MSI for platform devices.
a) MSI(s) is/are added in addition to the normal interrupts.
b) The vendor specific MSI configuration can be done using callbacks which is implemented as msi module.
c) Adds a msi handling module for the Broadcom platform devices.
Vikas Gupta (3):
vfio/platform: add support for msi
vfio/platform: change cleanup order
vfio/platform: add Broadcom msi module
drivers/vfio/platform/Kconfig | 1 + drivers/vfio/platform/Makefile | 1 + drivers/vfio/platform/msi/Kconfig | 9 + drivers/vfio/platform/msi/Makefile | 2 + .../vfio/platform/msi/vfio_platform_bcmplt.c | 74 ++++++ drivers/vfio/platform/vfio_platform_common.c | 86 ++++++- drivers/vfio/platform/vfio_platform_irq.c | 235 +++++++++++++++++- drivers/vfio/platform/vfio_platform_private.h | 23 ++ include/uapi/linux/vfio.h | 1 + 9 files changed, 417 insertions(+), 15 deletions(-) create mode 100644 drivers/vfio/platform/msi/Kconfig create mode 100644 drivers/vfio/platform/msi/Makefile create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
Signature · 1 line
2.17.1
RFC 1/3 vfio/platform: add support for msi
MSI support for platform devices.
--- drivers/vfio/platform/vfio_platform_common.c | 84 ++++++- drivers/vfio/platform/vfio_platform_irq.c | 235 +++++++++++++++++- drivers/vfio/platform/vfio_platform_private.h | 23 ++ include/uapi/linux/vfio.h | 1 + 4 files changed, 329 insertions(+), 14 deletions(-)
drivers/vfio/platform/vfio_platform_common.c +83 -1
@@ -26,6 +26,10 @@#define VFIO_PLATFORM_IS_ACPI (vdev) ((vdev)->acpihid != NULL)static LIST_HEAD (reset_list); + +/* devices having MSI support */ +static LIST_HEAD (msi_list); +static DEFINE_MUTEX (driver_lock); static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,
@@ -47,6 +51,26 @@ static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,return reset_fn; } +static bool vfio_platform_lookup_msi (struct vfio_platform_device *vdev) +{ +bool has_msi = false; +struct vfio_platform_msi_node *iter; + +mutex_lock (&driver_lock); +list_for_each_entry (iter, &msi_list, link) { +if (!strcmp (iter->compat, vdev->compat) && +try_module_get (iter->owner)) { + vdev->msi_module = iter->owner; + vdev->of_get_msi = iter->of_get_msi; + vdev->of_msi_write = iter->of_msi_write; + has_msi = true; +break ; + } + } +mutex_unlock (&driver_lock); +return has_msi; +} +static int vfio_platform_acpi_probe (struct vfio_platform_device *vdev,struct device *dev) {
@@ -110,6 +134,11 @@ static bool vfio_platform_has_reset(struct vfio_platform_device *vdev)return vdev->of_reset ? true : false; } +static bool vfio_platform_has_msi (struct vfio_platform_device *vdev) +{ +return vdev->of_get_msi ? true : false; +} +static int vfio_platform_get_reset (struct vfio_platform_device *vdev) {if (VFIO_PLATFORM_IS_ACPI (vdev))
@@ -126,6 +155,19 @@ static int vfio_platform_get_reset(struct vfio_platform_device *vdev)return vdev->of_reset ?0 : -ENOENT ; } +static int vfio_platform_get_msi (struct vfio_platform_device *vdev) +{ +bool has_msi; + + has_msi =vfio_platform_lookup_msi (vdev); +if (!has_msi) { +request_module ("vfio-msi:%s" , vdev->compat); + has_msi =vfio_platform_lookup_msi (vdev); + } + +return has_msi ?0 : -ENOENT ; +} +static void vfio_platform_put_reset (struct vfio_platform_device *vdev) {if (VFIO_PLATFORM_IS_ACPI (vdev))
@@ -135,6 +177,12 @@ static void vfio_platform_put_reset(struct vfio_platform_device *vdev)module_put (vdev->reset_module); } +static void vfio_platform_put_msi (struct vfio_platform_device *vdev) +{ +if (vdev->of_get_msi) +module_put (vdev->msi_module); +} +static int vfio_platform_regions_init (struct vfio_platform_device *vdev) {int cnt =0 , i;
@@ -313,6 +361,10 @@ static long vfio_platform_ioctl(void *device_data,if (vfio_platform_has_reset (vdev)) vdev->flags |=VFIO_DEVICE_FLAGS_RESET ; + +if (vfio_platform_has_msi (vdev)) + vdev->flags |=VFIO_DEVICE_FLAGS_MSI ; + info.flags = vdev->flags; info.num_regions = vdev->num_regions; info.num_irqs = vdev->num_irqs;
@@ -679,11 +731,15 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,return ret; } + ret =vfio_platform_get_msi (vdev); +if (ret) +dev_info (vdev->device,"msi not supported\n" ); + group =vfio_iommu_group_get (dev);if (!group) {dev_err (dev,"No IOMMU group for device %s\n" , vdev->name); ret = -EINVAL ; - goto put_reset; + goto put_msi; } ret =vfio_add_group_dev (dev, &vfio_platform_ops, vdev);
@@ -697,6 +753,8 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev, put_iommu:vfio_iommu_group_put (group, dev); +put_msi: +vfio_platform_put_msi (vdev); put_reset:vfio_platform_put_reset (vdev);return ret;
@@ -745,6 +803,30 @@ void vfio_platform_unregister_reset(const char *compat, }EXPORT_SYMBOL_GPL (vfio_platform_unregister_reset); +void __vfio_platform_register_msi (struct vfio_platform_msi_node *node) +{ +mutex_lock (&driver_lock); +list_add (&node->link, &msi_list); +mutex_unlock (&driver_lock); +} +EXPORT_SYMBOL_GPL (__vfio_platform_register_msi); + +void vfio_platform_unregister_msi (const char *compat) +{ +struct vfio_platform_msi_node *iter, *temp; + +mutex_lock (&driver_lock); +list_for_each_entry_safe (iter, temp, &msi_list, link) { +if (!strcmp (iter->compat, compat)) { +list_del (&iter->link); +break ; + } + } + +mutex_unlock (&driver_lock); +} +EXPORT_SYMBOL_GPL (vfio_platform_unregister_msi); +MODULE_VERSION (DRIVER_VERSION );MODULE_LICENSE ("GPL v2" );MODULE_AUTHOR (DRIVER_AUTHOR );
drivers/vfio/platform/vfio_platform_irq.c +222 -13
282 lines not on this page · raw message
drivers/vfio/platform/vfio_platform_private.h +23 -0
@@ -9,6 +9,7 @@#include <linux/types.h> #include <linux/interrupt.h> +#include <linux/msi.h> #define VFIO_PLATFORM_OFFSET_SHIFT 40#define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
@@ -26,6 +27,7 @@ struct vfio_platform_irq {char *name;struct eventfd_ctx *trigger;bool masked; +bool is_msi;spinlock_t lock;struct virqfd *unmask;struct virqfd *mask;
@@ -46,13 +48,16 @@ struct vfio_platform_device {u32 num_regions;struct vfio_platform_irq *irqs;u32 num_irqs; +u32 num_msis;int refcnt;struct mutex igate;struct module *parent_module;const char *compat;const char *acpihid;struct module *reset_module; +struct module *msi_module;struct device *device; +int config_msi;/* * These fields should be filled by the bus specific binder
@@ -65,11 +70,19 @@ struct vfio_platform_device { (*get_resource)(struct vfio_platform_device *vdev, int i);int (*get_irq)(struct vfio_platform_device *vdev,int i);int (*of_reset)(struct vfio_platform_device *vdev); +u32 (*of_get_msi)(struct vfio_platform_device *vdev); +int (*of_msi_write)(struct vfio_platform_device *vdev, +struct msi_desc *desc, +struct msi_msg *msg);bool reset_required; };typedef int (*vfio_platform_reset_fn_t )(struct vfio_platform_device *vdev); +typedef u32 (*vfio_platform_get_msi_fn_t )(struct vfio_platform_device *vdev); +typedef int (*vfio_platform_msi_write_fn_t )(struct vfio_platform_device *vdev, +struct msi_desc *desc, +struct msi_msg *msg);struct vfio_platform_reset_node {struct list_head link;
@@ -78,6 +91,14 @@ struct vfio_platform_reset_node {vfio_platform_reset_fn_t of_reset; }; +struct vfio_platform_msi_node { +struct list_head link; +char *compat; +struct module *owner; +vfio_platform_get_msi_fn_t of_get_msi; +vfio_platform_msi_write_fn_t of_msi_write; +}; +extern int vfio_platform_probe_common (struct vfio_platform_device *vdev,struct device *dev);extern struct vfio_platform_device *vfio_platform_remove_common
@@ -94,6 +115,8 @@ extern int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,extern void __vfio_platform_register_reset (struct vfio_platform_reset_node *n);extern void vfio_platform_unregister_reset (const char *compat,vfio_platform_reset_fn_t fn); +void __vfio_platform_register_msi (struct vfio_platform_msi_node *n); +void vfio_platform_unregister_msi (const char *compat);#define vfio_platform_register_reset (__compat, __reset) \ static struct vfio_platform_reset_node __reset ## _node = { \ .owner = THIS_MODULE, \
include/uapi/linux/vfio.h +1 -0
@@ -203,6 +203,7 @@ struct vfio_device_info {#define VFIO_DEVICE_FLAGS_AP (1 << 5)/* vfio-ap device */ #define VFIO_DEVICE_FLAGS_FSL_MC (1 << 6)/* vfio-fsl-mc device */ #define VFIO_DEVICE_FLAGS_CAPS (1 << 7)/* Info supports caps */ +#define VFIO_DEVICE_FLAGS_MSI (1 << 8)/* Device supports msi */ __u32 num_regions;/* Max region index + 1 */ __u32 num_irqs;/* Max IRQ index + 1 */ __u32 cap_offset;/* Offset within info struct of first cap */
Signature · 1 line
2.17.1
Re: [RFC, v0 1/3] vfio/platform: add support for msi
include/uapi/linux/vfio.h struct vfio_device_info { · line 209
Quoted 12 lines · On Thu, 5 Nov 2020 11:32:55 +0530 Vikas Gupta <[email protected]> wrote:
include/uapi/linux/vfio.h +1
-0@@ -203,6 +203,7 @@ struct vfio_device_info {#define VFIO_DEVICE_FLAGS_AP (1 << 5)/* vfio-ap device */ #define VFIO_DEVICE_FLAGS_FSL_MC (1 << 6)/* vfio-fsl-mc device */ #define VFIO_DEVICE_FLAGS_CAPS (1 << 7)/* Info supports caps */ +#define VFIO_DEVICE_FLAGS_MSI (1 << 8)/* Device supports msi */ __u32 num_regions;/* Max region index + 1 */ __u32 num_irqs;/* Max IRQ index + 1 */ __u32 cap_offset;/* Offset within info struct of first cap */
This doesn't make any sense to me, MSIs are just edge triggered interrupts to userspace, so why isn't this fully described via VFIO_DEVICE_GET_IRQ_INFO? If we do need something new to describe it, this seems incomplete, which indexes are MSI (IRQ_INFO can describe that)? We also already support MSI with vfio-pci, so a global flag for the device advertising this still seems wrong. Thanks,
Alex
Re: [RFC, v0 1/3] vfio/platform: add support for msi
Hi Alex,
Quoted 24 lines · Alex Williamson
include/uapi/linux/vfio.h struct vfio_device_info { · line 209
Quoted 12 lines · On Thu, 5 Nov 2020 11:32:55 +0530 Vikas Gupta <[email protected]> wrote:
include/uapi/linux/vfio.h +1
-0@@ -203,6 +203,7 @@ struct vfio_device_info {#define VFIO_DEVICE_FLAGS_AP (1 << 5)/* vfio-ap device */ #define VFIO_DEVICE_FLAGS_FSL_MC (1 << 6)/* vfio-fsl-mc device */ #define VFIO_DEVICE_FLAGS_CAPS (1 << 7)/* Info supports caps */ +#define VFIO_DEVICE_FLAGS_MSI (1 << 8)/* Device supports msi */ __u32 num_regions;/* Max region index + 1 */ __u32 num_irqs;/* Max IRQ index + 1 */ __u32 cap_offset;/* Offset within info struct of first cap */ This doesn't make any sense to me, MSIs are just edge triggered interrupts to userspace, so why isn't this fully described via
VFIO_DEVICE_GET_IRQ_INFO? If we do need something new to describe it, this seems incomplete, which indexes are MSI (IRQ_INFOcan describe that)? We also already support MSI with vfio-pci, so a global flag for the device advertising this still seems wrong. Thanks,
Quoted 2 lines
Alex
Since VFIO platform uses indexes for IRQ numbers so I think MSI(s) cannot be described using indexes.
In the patch set there is no difference between MSI and normal interrupt for VFIO_DEVICE_GET_IRQ_INFO.
The patch set adds MSI(s), say as an extension, to the normal interrupts and handled accordingly. Do you see this is a violation? If yes, then we`ll think of other possible ways to support MSI for the platform devices.
Macro VFIO_DEVICE_FLAGS_MSI can be changed to any other name if it collides with an already supported vfio-pci or if not necessary, we can remove this flag.
Thanks,
Vikas
Re: [RFC, v0 1/3] vfio/platform: add support for msi
Quoted 28 lines · On Fri, 6 Nov 2020 08:24:26 +0530 Vikas Gupta <[email protected]> wrote:
Hi Alex,
Quoted 24 lines · Alex Williamson
include/uapi/linux/vfio.h struct vfio_device_info { · line 209
Quoted 12 lines · On Thu, 5 Nov 2020 11:32:55 +0530 Vikas Gupta <[email protected]> wrote:
include/uapi/linux/vfio.h +1
-0@@ -203,6 +203,7 @@ struct vfio_device_info {#define VFIO_DEVICE_FLAGS_AP (1 << 5)/* vfio-ap device */ #define VFIO_DEVICE_FLAGS_FSL_MC (1 << 6)/* vfio-fsl-mc device */ #define VFIO_DEVICE_FLAGS_CAPS (1 << 7)/* Info supports caps */ +#define VFIO_DEVICE_FLAGS_MSI (1 << 8)/* Device supports msi */ __u32 num_regions;/* Max region index + 1 */ __u32 num_irqs;/* Max IRQ index + 1 */ __u32 cap_offset;/* Offset within info struct of first cap */ This doesn't make any sense to me, MSIs are just edge triggered interrupts to userspace, so why isn't this fully described via
VFIO_DEVICE_GET_IRQ_INFO? If we do need something new to describe it, this seems incomplete, which indexes are MSI (IRQ_INFOcan describe that)? We also already support MSI with vfio-pci, so a global flag for the device advertising this still seems wrong. Thanks,
Quoted 4 lines
Quoted 2 lines
Alex
Since VFIO platform uses indexes for IRQ numbers so I think MSI(s) cannot be described using indexes.
That would be news for vfio-pci which has been describing MSIs with sub-indexes within indexes since vfio started.
Quoted 2 lines
In the patch set there is no difference between MSI and normal interrupt for
VFIO_DEVICE_GET_IRQ_INFO.
Then what exactly is a global device flag indicating? Does it indicate all IRQs are MSI?
Quoted 2 lines
The patch set adds MSI(s), say as an extension, to the normal interrupts and handled accordingly.
So we have both "normal" IRQs and MSIs? How does the user know which indexes are which?
Quoted 1 line
Do you see this is a violation? If
Seems pretty unclear and dubious use of a global device flag.
Quoted 5 lines
yes, then we`ll think of other possible ways to support MSI for the platform devices.
MacroVFIO_DEVICE_FLAGS_MSIcan be changed to any other name if it collides with an already supported vfio-pci or if not necessary, we can remove this flag.
If nothing else you're using a global flag to describe a platform device specific augmentation. We've recently added capabilities on the device info return that would be more appropriate for this, but fundamentally I don't understand why the irq info isn't sufficient. Thanks,
Alex
Re: [RFC, v0 1/3] vfio/platform: add support for msi
Hi Alex,
Quoted 37 lines · Alex Williamson
Quoted 28 lines · On Fri, 6 Nov 2020 08:24:26 +0530 Vikas Gupta <[email protected]> wrote:
Hi Alex,
Quoted 24 lines · Alex Williamson
include/uapi/linux/vfio.h struct vfio_device_info { · line 209
Quoted 12 lines · On Thu, 5 Nov 2020 11:32:55 +0530 Vikas Gupta <[email protected]> wrote:
include/uapi/linux/vfio.h +1
-0@@ -203,6 +203,7 @@ struct vfio_device_info {#define VFIO_DEVICE_FLAGS_AP (1 << 5)/* vfio-ap device */ #define VFIO_DEVICE_FLAGS_FSL_MC (1 << 6)/* vfio-fsl-mc device */ #define VFIO_DEVICE_FLAGS_CAPS (1 << 7)/* Info supports caps */ +#define VFIO_DEVICE_FLAGS_MSI (1 << 8)/* Device supports msi */ __u32 num_regions;/* Max region index + 1 */ __u32 num_irqs;/* Max IRQ index + 1 */ __u32 cap_offset;/* Offset within info struct of first cap */ This doesn't make any sense to me, MSIs are just edge triggered interrupts to userspace, so why isn't this fully described via
VFIO_DEVICE_GET_IRQ_INFO? If we do need something new to describe it, this seems incomplete, which indexes are MSI (IRQ_INFOcan describe that)? We also already support MSI with vfio-pci, so a global flag for the device advertising this still seems wrong. Thanks,Quoted 4 lines
Quoted 2 lines
Alex
Since VFIO platform uses indexes for IRQ numbers so I think MSI(s) cannot be described using indexes.
Quoted 8 lines
That would be news for vfio-pci which has been describing MSIs with sub-indexes within indexes since vfio started.
Quoted 2 lines
In the patch set there is no difference between MSI and normal interrupt for
VFIO_DEVICE_GET_IRQ_INFO.Then what exactly is a global device flag indicating? Does it indicate all IRQs are MSI?
No, it's not indicating that all are MSI.
The rationale behind adding the flag to tell user-space that platform device supports MSI as well. As you mentioned recently added capabilities can help on this, I`ll go through that.
Quoted 6 lines
Quoted 3 lines
The patch set adds MSI(s), say as an extension, to the normal interrupts and handled accordingly.
So we have both "normal" IRQs and MSIs? How does the user know which indexes are which?
With this patch set, I think this is missing and user space cannot know that particular index is MSI interrupt.
For platform devices there is no such mechanism, like index and sub-indexes to differentiate between legacy, MSI or MSIX as it’s there in PCI.
I believe for a particular IRQ index if the flag VFIO_IRQ_INFO_NORESIZE is used then user space can know which IRQ index has MSI(s). Does it make sense?
Suggestions on this would be helpful.
Thanks,
Vikas
Quoted 19 lines
Quoted 2 lines
Do you see this is a violation? If
Seems pretty unclear and dubious use of a global device flag.
Quoted 5 lines
yes, then we`ll think of other possible ways to support MSI for the platform devices.
MacroVFIO_DEVICE_FLAGS_MSIcan be changed to any other name if it collides with an already supported vfio-pci or if not necessary, we can remove this flag.If nothing else you're using a global flag to describe a platform device specific augmentation. We've recently added capabilities on the device info return that would be more appropriate for this, but fundamentally I don't understand why the irq info isn't sufficient. Thanks,
Alex
Auger Eric
maintainerRe: [RFC, v0 1/3] vfio/platform: add support for msi
Hi Vikas,
Quoted 59 lines · Vikas Gupta
Hi Alex,
Quoted 37 lines · Alex Williamson
Quoted 28 lines · On Fri, 6 Nov 2020 08:24:26 +0530 Vikas Gupta <[email protected]> wrote:
Hi Alex,
Quoted 24 lines · Alex Williamson
include/uapi/linux/vfio.h struct vfio_device_info { · line 209
Quoted 12 lines · On Thu, 5 Nov 2020 11:32:55 +0530 Vikas Gupta <[email protected]> wrote:
include/uapi/linux/vfio.h +1
-0@@ -203,6 +203,7 @@ struct vfio_device_info {#define VFIO_DEVICE_FLAGS_AP (1 << 5)/* vfio-ap device */ #define VFIO_DEVICE_FLAGS_FSL_MC (1 << 6)/* vfio-fsl-mc device */ #define VFIO_DEVICE_FLAGS_CAPS (1 << 7)/* Info supports caps */ +#define VFIO_DEVICE_FLAGS_MSI (1 << 8)/* Device supports msi */ __u32 num_regions;/* Max region index + 1 */ __u32 num_irqs;/* Max IRQ index + 1 */ __u32 cap_offset;/* Offset within info struct of first cap */ This doesn't make any sense to me, MSIs are just edge triggered interrupts to userspace, so why isn't this fully described via
VFIO_DEVICE_GET_IRQ_INFO? If we do need something new to describe it, this seems incomplete, which indexes are MSI (IRQ_INFOcan describe that)? We also already support MSI with vfio-pci, so a global flag for the device advertising this still seems wrong. Thanks,Quoted 4 lines
Quoted 2 lines
Alex
Since VFIO platform uses indexes for IRQ numbers so I think MSI(s) cannot be described using indexes.
Quoted 8 lines
That would be news for vfio-pci which has been describing MSIs with sub-indexes within indexes since vfio started.
Quoted 2 lines
In the patch set there is no difference between MSI and normal interrupt for
VFIO_DEVICE_GET_IRQ_INFO.Then what exactly is a global device flag indicating? Does it indicate all IRQs are MSI?
No, it's not indicating that all are MSI.
The rationale behind adding the flag to tell user-space that platform device supports MSI as well. As you mentioned recently added capabilities can help on this, I`ll go through that.Quoted 4 lines
Quoted 3 lines
The patch set adds MSI(s), say as an extension, to the normal interrupts and handled accordingly.
Quoted 8 lines
Quoted 2 lines
So we have both "normal" IRQs and MSIs? How does the user know which indexes are which?
With this patch set, I think this is missing and user space cannot know that particular index is MSI interrupt.
For platform devices there is no such mechanism, like index and sub-indexes to differentiate between legacy, MSI or MSIX as it’s there in PCI.
Wht can't you use the count field (as per vfio_pci_get_irq_count())?
Quoted 3 lines
I believe for a particular IRQ index if the flag
VFIO_IRQ_INFO_NORESIZEis used then user space can know which IRQ index has MSI(s). Does it make sense?
I don't think it is the same semantics.
Thanks
Eric
Quoted 23 lines
Suggestions on this would be helpful.
Thanks,
VikasQuoted 19 lines
Quoted 2 lines
Do you see this is a violation? If
Seems pretty unclear and dubious use of a global device flag.
Quoted 5 lines
yes, then we`ll think of other possible ways to support MSI for the platform devices.
MacroVFIO_DEVICE_FLAGS_MSIcan be changed to any other name if it collides with an already supported vfio-pci or if not necessary, we can remove this flag.If nothing else you're using a global flag to describe a platform device specific augmentation. We've recently added capabilities on the device info return that would be more appropriate for this, but fundamentally I don't understand why the irq info isn't sufficient. Thanks,
Alex
Re: [RFC, v0 1/3] vfio/platform: add support for msi
In reply to Vikas GuptaQuoted 47 lines · On Mon, 9 Nov 2020 12:11:15 +0530 Vikas Gupta <[email protected]> wrote:
Hi Alex,
Quoted 43 lines · Alex Williamson
Quoted 28 lines · On Fri, 6 Nov 2020 08:24:26 +0530 Vikas Gupta <[email protected]> wrote:
Hi Alex,
Quoted 24 lines · Alex Williamson
include/uapi/linux/vfio.h struct vfio_device_info { · line 209
Quoted 12 lines · On Thu, 5 Nov 2020 11:32:55 +0530 Vikas Gupta <[email protected]> wrote:
include/uapi/linux/vfio.h +1
-0@@ -203,6 +203,7 @@ struct vfio_device_info {#define VFIO_DEVICE_FLAGS_AP (1 << 5)/* vfio-ap device */ #define VFIO_DEVICE_FLAGS_FSL_MC (1 << 6)/* vfio-fsl-mc device */ #define VFIO_DEVICE_FLAGS_CAPS (1 << 7)/* Info supports caps */ +#define VFIO_DEVICE_FLAGS_MSI (1 << 8)/* Device supports msi */ __u32 num_regions;/* Max region index + 1 */ __u32 num_irqs;/* Max IRQ index + 1 */ __u32 cap_offset;/* Offset within info struct of first cap */ This doesn't make any sense to me, MSIs are just edge triggered interrupts to userspace, so why isn't this fully described via
VFIO_DEVICE_GET_IRQ_INFO? If we do need something new to describe it, this seems incomplete, which indexes are MSI (IRQ_INFOcan describe that)? We also already support MSI with vfio-pci, so a global flag for the device advertising this still seems wrong. Thanks,Quoted 4 lines
Quoted 2 lines
Alex
Since VFIO platform uses indexes for IRQ numbers so I think MSI(s) cannot be described using indexes.
That would be news for vfio-pci which has been describing MSIs with sub-indexes within indexes since vfio started.
Quoted 2 lines
In the patch set there is no difference between MSI and normal interrupt for
VFIO_DEVICE_GET_IRQ_INFO.
Quoted 7 lines
Quoted 2 lines
Then what exactly is a global device flag indicating? Does it indicate all IRQs are MSI?
No, it's not indicating that all are MSI.
The rationale behind adding the flag to tell user-space that platform device supports MSI as well. As you mentioned recently added capabilities can help on this, I`ll go through that.
It still seems questionable to me to use a device info capability to describe an interrupt index specific feature. The scope seems wrong. Why does userspace need to know that this IRQ is MSI rather than indicating it's simply an edge triggered interrupt? That can be done using only vfio_irq_info.flags.
Quoted 11 lines
Quoted 5 lines
Quoted 2 lines
The patch set adds MSI(s), say as an extension, to the normal interrupts and handled accordingly.
So we have both "normal" IRQs and MSIs? How does the user know which indexes are which?
With this patch set, I think this is missing and user space cannot know that particular index is MSI interrupt.
For platform devices there is no such mechanism, like index and sub-indexes to differentiate between legacy, MSI or MSIX as it’s there in PCI.
Indexes and sub-indexes are a grouping mechanism of vfio to describe related interrupts. That terminology doesn't exist on PCI either, it's meant to be used generically. It's left to the vfio bus driver how userspace associates a given index to a device feature.
Quoted 3 lines
I believe for a particular IRQ index if the flag
VFIO_IRQ_INFO_NORESIZEis used then user space can know which IRQ index has MSI(s). Does it make sense?
No, no-resize is an implementation detail, not an indication of the interrupt mechanism. It's still not clear to me why it's important to expose to userspace that a given interrupt is MSI versus simply exposing it as an edge interrupt (ie. automasked = false). If it is necessary, the most direct approach might be to expose a capability extension in the vfio_irq_info structure to describe it. Even then though, I don't think simply exposing a index as MSI is very meaningful. What is userspace intended to do differently based on this information? Thanks,
Alex
Re: [RFC, v0 1/3] vfio/platform: add support for msi
Hi Alex,
Quoted 60 lines · Alex Williamson
Quoted 47 lines · On Mon, 9 Nov 2020 12:11:15 +0530 Vikas Gupta <[email protected]> wrote:
Hi Alex,
Quoted 43 lines · Alex Williamson
Quoted 28 lines · On Fri, 6 Nov 2020 08:24:26 +0530 Vikas Gupta <[email protected]> wrote:
Hi Alex,
Quoted 24 lines · Alex Williamson
include/uapi/linux/vfio.h struct vfio_device_info { · line 209
Quoted 12 lines · On Thu, 5 Nov 2020 11:32:55 +0530 Vikas Gupta <[email protected]> wrote:
include/uapi/linux/vfio.h +1
-0@@ -203,6 +203,7 @@ struct vfio_device_info {#define VFIO_DEVICE_FLAGS_AP (1 << 5)/* vfio-ap device */ #define VFIO_DEVICE_FLAGS_FSL_MC (1 << 6)/* vfio-fsl-mc device */ #define VFIO_DEVICE_FLAGS_CAPS (1 << 7)/* Info supports caps */ +#define VFIO_DEVICE_FLAGS_MSI (1 << 8)/* Device supports msi */ __u32 num_regions;/* Max region index + 1 */ __u32 num_irqs;/* Max IRQ index + 1 */ __u32 cap_offset;/* Offset within info struct of first cap */ This doesn't make any sense to me, MSIs are just edge triggered interrupts to userspace, so why isn't this fully described via
VFIO_DEVICE_GET_IRQ_INFO? If we do need something new to describe it, this seems incomplete, which indexes are MSI (IRQ_INFOcan describe that)? We also already support MSI with vfio-pci, so a global flag for the device advertising this still seems wrong. Thanks,Quoted 4 lines
Quoted 2 lines
Alex
Since VFIO platform uses indexes for IRQ numbers so I think MSI(s) cannot be described using indexes.
That would be news for vfio-pci which has been describing MSIs with sub-indexes within indexes since vfio started.
Quoted 2 lines
In the patch set there is no difference between MSI and normal interrupt for
VFIO_DEVICE_GET_IRQ_INFO.Quoted 7 lines
Quoted 2 lines
Then what exactly is a global device flag indicating? Does it indicate all IRQs are MSI?
No, it's not indicating that all are MSI.
The rationale behind adding the flag to tell user-space that platform device supports MSI as well. As you mentioned recently added capabilities can help on this, I`ll go through that.
Quoted 5 lines
It still seems questionable to me to use a device info capability to describe an interrupt index specific feature. The scope seems wrong. Why does userspace need to know that this IRQ is MSI rather than indicating it's simply an edge triggered interrupt? That can be done using only vfio_irq_info.flags.
Ok. In the next patch set I`ll remove the device flag (VFIO_DEVICE_FLAGS_MSI) as vfio_irq_info.flags should have enough information for edge triggered interrupt.
Quoted 24 lines
Quoted 5 lines
Quoted 5 lines
Quoted 4 lines
The patch set adds MSI(s), say as an extension, to the normal interrupts and handled accordingly.
Quoted 8 lines
Quoted 2 lines
So we have both "normal" IRQs and MSIs? How does the user know which indexes are which?
With this patch set, I think this is missing and user space cannot know that particular index is MSI interrupt.
For platform devices there is no such mechanism, like index and sub-indexes to differentiate between legacy, MSI or MSIX as it’s there in PCI.Indexes and sub-indexes are a grouping mechanism of vfio to describe related interrupts. That terminology doesn't exist on PCI either, it's meant to be used generically. It's left to the vfio bus driver how userspace associates a given index to a device feature.
Quoted 3 lines
I believe for a particular IRQ index if the flag
VFIO_IRQ_INFO_NORESIZEis used then user space can know which IRQ index has MSI(s). Does it make sense?
Quoted 9 lines
No, no-resize is an implementation detail, not an indication of the interrupt mechanism. It's still not clear to me why it's important to expose to userspace that a given interrupt is MSI versus simply exposing it as an edge interrupt (ie. automasked = false). If it is necessary, the most direct approach might be to expose a capability extension in the
vfio_irq_infostructure to describe it. Even then though, I don't think simply exposing a index as MSI is very meaningful. What is userspace intended to do differently based on this information? Thanks,
The current patch set is not setting VFIO_IRQ_INFO_AUTOMASKED (automasked=false) for MSIs so I believe this much is information enough for user space to know that this is an edge triggered interrupt.
I agree that exposing an index as MSI is not meaningful as user space
has nothing special to do with this information.
Quoted 3 lines
Alex
Auger Eric
maintainerRe: [RFC, v0 1/3] vfio/platform: add support for msi
In reply to Vikas GuptaHi Vikas,
Quoted 28 lines · Vikas Gupta
Hi Alex,
Quoted 24 lines · Alex Williamson
include/uapi/linux/vfio.h struct vfio_device_info { · line 209
Quoted 12 lines · On Thu, 5 Nov 2020 11:32:55 +0530 Vikas Gupta <[email protected]> wrote:
include/uapi/linux/vfio.h +1
-0@@ -203,6 +203,7 @@ struct vfio_device_info {#define VFIO_DEVICE_FLAGS_AP (1 << 5)/* vfio-ap device */ #define VFIO_DEVICE_FLAGS_FSL_MC (1 << 6)/* vfio-fsl-mc device */ #define VFIO_DEVICE_FLAGS_CAPS (1 << 7)/* Info supports caps */ +#define VFIO_DEVICE_FLAGS_MSI (1 << 8)/* Device supports msi */ __u32 num_regions;/* Max region index + 1 */ __u32 num_irqs;/* Max IRQ index + 1 */ __u32 cap_offset;/* Offset within info struct of first cap */ This doesn't make any sense to me, MSIs are just edge triggered interrupts to userspace, so why isn't this fully described via
VFIO_DEVICE_GET_IRQ_INFO? If we do need something new to describe it, this seems incomplete, which indexes are MSI (IRQ_INFOcan describe that)? We also already support MSI with vfio-pci, so a global flag for the device advertising this still seems wrong. Thanks,
Quoted 6 lines
Quoted 2 lines
Alex
Since VFIO platform uses indexes for IRQ numbers so I think MSI(s) cannot be described using indexes.
In the patch set there is no difference between MSI and normal interrupt forVFIO_DEVICE_GET_IRQ_INFO.
in vfio_platform_irq_init() we first iterate on normal interrupts using get_irq(). Can't we add an MSI index at the end of this list with vdev->irqs[i].count > 1 and set vdev->num_irqs accordingly?
Thanks
Eric
Quoted 11 lines
The patch set adds MSI(s), say as an extension, to the normal interrupts and handled accordingly. Do you see this is a violation? If yes, then we`ll think of other possible ways to support MSI for the platform devices.
MacroVFIO_DEVICE_FLAGS_MSIcan be changed to any other name if it collides with an already supported vfio-pci or if not necessary, we can remove this flag.Thanks,
Vikas
Re: [RFC, v0 1/3] vfio/platform: add support for msi
Hi Eric,
Quoted 34 lines · Auger Eric
Hi Vikas,
Quoted 30 lines · Vikas Gupta
Hi Alex,
Quoted 26 lines · Alex Williamson
include/uapi/linux/vfio.h struct vfio_device_info { · line 209
Quoted 12 lines · On Thu, 5 Nov 2020 11:32:55 +0530 Vikas Gupta <[email protected]> wrote:
include/uapi/linux/vfio.h +1
-0@@ -203,6 +203,7 @@ struct vfio_device_info {#define VFIO_DEVICE_FLAGS_AP (1 << 5)/* vfio-ap device */ #define VFIO_DEVICE_FLAGS_FSL_MC (1 << 6)/* vfio-fsl-mc device */ #define VFIO_DEVICE_FLAGS_CAPS (1 << 7)/* Info supports caps */ +#define VFIO_DEVICE_FLAGS_MSI (1 << 8)/* Device supports msi */ __u32 num_regions;/* Max region index + 1 */ __u32 num_irqs;/* Max IRQ index + 1 */ __u32 cap_offset;/* Offset within info struct of first cap */ This doesn't make any sense to me, MSIs are just edge triggered interrupts to userspace, so why isn't this fully described via
VFIO_DEVICE_GET_IRQ_INFO? If we do need something new to describe it, this seems incomplete, which indexes are MSI (IRQ_INFOcan describe that)? We also already support MSI with vfio-pci, so a global flag for the device advertising this still seems wrong. Thanks,Alex
Quoted 7 lines
Quoted 4 lines
Since VFIO platform uses indexes for IRQ numbers so I think MSI(s) cannot be described using indexes.
In the patch set there is no difference between MSI and normal interrupt forVFIO_DEVICE_GET_IRQ_INFO.in
vfio_platform_irq_init()we first iterate on normal interrupts usingget_irq(). Can't we add an MSI index at the end of this list with vdev->irqs[i].count > 1 and setvdev->num_irqsaccordingly?
Yes, I think MSI can be added to the end of list with setting vdev->irqs[i].count > 1.
I`ll consider changing in the next patch set. Thanks,
Vikas
Quoted 16 lines
Thanks
Eric
Quoted 11 lines
The patch set adds MSI(s), say as an extension, to the normal interrupts and handled accordingly. Do you see this is a violation? If yes, then we`ll think of other possible ways to support MSI for the platform devices.
MacroVFIO_DEVICE_FLAGS_MSIcan be changed to any other name if it collides with an already supported vfio-pci or if not necessary, we can remove this flag.Thanks,
Vikas
RFC 2/3 vfio/platform: change cleanup order
In reply to Vikas GuptaIn the case of msi, vendor specific msi module may require region access to handle msi cleanup so we need to cleanup region after irq cleanup only.
--- drivers/vfio/platform/vfio_platform_common.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/vfio/platform/vfio_platform_common.c +1 -1
@@ -283,8 +283,8 @@ static void vfio_platform_release(void *device_data)WARN_ON (1 ); }pm_runtime_put (vdev->device); -vfio_platform_regions_cleanup (vdev);vfio_platform_irq_cleanup (vdev); +vfio_platform_regions_cleanup (vdev); }mutex_unlock (&driver_lock);
Signature · 1 line
2.17.1
RFC 3/3 vfio/platform: add Broadcom msi module
In reply to Vikas GuptaAdd Broadcom msi module for platform devices.
--- drivers/vfio/platform/Kconfig | 1 + drivers/vfio/platform/Makefile | 1 + drivers/vfio/platform/msi/Kconfig | 9 +++ drivers/vfio/platform/msi/Makefile | 2 + .../vfio/platform/msi/vfio_platform_bcmplt.c | 74 +++++++++++++++++++ 5 files changed, 87 insertions(+) create mode 100644 drivers/vfio/platform/msi/Kconfig create mode 100644 drivers/vfio/platform/msi/Makefile create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
drivers/vfio/platform/Kconfig +1 -0
@@ -21,3 +21,4 @@ config VFIO_AMBA If you don't know what to do here, say N. source "drivers/vfio/platform/reset/Kconfig" +source "drivers/vfio/platform/msi/Kconfig"
drivers/vfio/platform/Makefile +1 -0
@@ -5,6 +5,7 @@ vfio-platform-y := vfio_platform.o obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform.o obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform-base.o obj-$(CONFIG_VFIO_PLATFORM) += reset/ +obj-$(CONFIG_VFIO_PLATFORM) += msi/ vfio-amba-y := vfio_amba.o
drivers/vfio/platform/msi/Kconfig new file +9 -0
@@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0-only +config VFIO_PLATFORM_BCMPLT_MSI + tristate "MSI support for Broadcom platform devices" + depends on VFIO_PLATFORM && (ARCH_BCM_IPROC || COMPILE_TEST) + default ARCH_BCM_IPROC + help + Enables the VFIO platform driver to handle msi for Broadcom devices + + If you don't know what to do here, say N.
drivers/vfio/platform/msi/Makefile new file +2 -0
@@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_VFIO_PLATFORM_BCMPLT_MSI) += vfio_platform_bcmplt.o
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +74 -0
74 lines not on this page · raw message
Signature · 1 line
2.17.1
cover msi support for platform devices
In reply to Vikas GuptaThis RFC adds support for MSI for platform devices.
a) MSI(s) is/are added in addition to the normal interrupts.
b) The vendor specific MSI configuration can be done using callbacks which is implemented as msi module.
c) Adds a msi handling module for the Broadcom platform devices.
Changes from:
-------------
v0 to v1:
i) Removed MSI device flag VFIO_DEVICE_FLAGS_MSI. ii) Add MSI(s) at the end of the irq list of platform IRQs. MSI(s) with first entry of MSI block has count and flag information.
IRQ list: Allocation for IRQs + MSIs are allocated as below
Example: if there are 'n' IRQs and 'k' MSIs
-------------------------------------------------------
|IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
-------------------------------------------------------
MSI-0 will have count=k set and flags set accordingly.
Vikas Gupta (3):
vfio/platform: add support for msi
vfio/platform: change cleanup order
vfio/platform: add Broadcom msi module
drivers/vfio/platform/Kconfig | 1 + drivers/vfio/platform/Makefile | 1 + drivers/vfio/platform/msi/Kconfig | 9 + drivers/vfio/platform/msi/Makefile | 2 + .../vfio/platform/msi/vfio_platform_bcmplt.c | 74 ++++++ drivers/vfio/platform/vfio_platform_common.c | 86 ++++++- drivers/vfio/platform/vfio_platform_irq.c | 238 +++++++++++++++++- drivers/vfio/platform/vfio_platform_private.h | 23 ++ 8 files changed, 419 insertions(+), 15 deletions(-) create mode 100644 drivers/vfio/platform/msi/Kconfig create mode 100644 drivers/vfio/platform/msi/Makefile create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
Signature · 1 line
2.17.1
RFC 1/3 vfio/platform: add support for msi
MSI support for platform devices.
--- drivers/vfio/platform/vfio_platform_common.c | 84 ++++++- drivers/vfio/platform/vfio_platform_irq.c | 238 +++++++++++++++++- drivers/vfio/platform/vfio_platform_private.h | 23 ++ 3 files changed, 331 insertions(+), 14 deletions(-)
drivers/vfio/platform/vfio_platform_common.c +82 -2
@@ -26,6 +26,10 @@#define VFIO_PLATFORM_IS_ACPI (vdev) ((vdev)->acpihid != NULL)static LIST_HEAD (reset_list); + +/* devices having MSI support */ +static LIST_HEAD (msi_list); +static DEFINE_MUTEX (driver_lock); static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,
@@ -47,6 +51,26 @@ static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,return reset_fn; } +static bool vfio_platform_lookup_msi (struct vfio_platform_device *vdev) +{ +bool has_msi = false; +struct vfio_platform_msi_node *iter; + +mutex_lock (&driver_lock); +list_for_each_entry (iter, &msi_list, link) { +if (!strcmp (iter->compat, vdev->compat) && +try_module_get (iter->owner)) { + vdev->msi_module = iter->owner; + vdev->of_get_msi = iter->of_get_msi; + vdev->of_msi_write = iter->of_msi_write; + has_msi = true; +break ; + } + } +mutex_unlock (&driver_lock); +return has_msi; +} +static int vfio_platform_acpi_probe (struct vfio_platform_device *vdev,struct device *dev) {
@@ -126,6 +150,19 @@ static int vfio_platform_get_reset(struct vfio_platform_device *vdev)return vdev->of_reset ?0 : -ENOENT ; } +static int vfio_platform_get_msi (struct vfio_platform_device *vdev) +{ +bool has_msi; + + has_msi =vfio_platform_lookup_msi (vdev); +if (!has_msi) { +request_module ("vfio-msi:%s" , vdev->compat); + has_msi =vfio_platform_lookup_msi (vdev); + } + +return has_msi ?0 : -ENOENT ; +} +static void vfio_platform_put_reset (struct vfio_platform_device *vdev) {if (VFIO_PLATFORM_IS_ACPI (vdev))
@@ -135,6 +172,12 @@ static void vfio_platform_put_reset(struct vfio_platform_device *vdev)module_put (vdev->reset_module); } +static void vfio_platform_put_msi (struct vfio_platform_device *vdev) +{ +if (vdev->of_get_msi) +module_put (vdev->msi_module); +} +static int vfio_platform_regions_init (struct vfio_platform_device *vdev) {int cnt =0 , i;
@@ -313,6 +356,7 @@ static long vfio_platform_ioctl(void *device_data,if (vfio_platform_has_reset (vdev)) vdev->flags |=VFIO_DEVICE_FLAGS_RESET ; + info.flags = vdev->flags; info.num_regions = vdev->num_regions; info.num_irqs = vdev->num_irqs;
@@ -366,6 +410,7 @@ static long vfio_platform_ioctl(void *device_data,struct vfio_irq_set hdr;u8 *data =NULL ;int ret =0 ; +int max;size_t data_size =0 ; minsz = offsetofend(struct vfio_irq_set, count);
@@ -373,7 +418,12 @@ static long vfio_platform_ioctl(void *device_data,if (copy_from_user (&hdr, (void __user *)arg, minsz))return -EFAULT ; - ret =vfio_set_irqs_validate_and_prepare (&hdr, vdev->num_irqs, +if (hdr.index >= vdev->num_irqs) +return -EINVAL ; + + max = vdev->irqs[hdr.index].count; + + ret =vfio_set_irqs_validate_and_prepare (&hdr, max, vdev->num_irqs, &data_size);if (ret)return ret;
@@ -679,11 +729,15 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,return ret; } + ret =vfio_platform_get_msi (vdev); +if (ret) +dev_info (vdev->device,"msi not supported\n" ); + group =vfio_iommu_group_get (dev);if (!group) {dev_err (dev,"No IOMMU group for device %s\n" , vdev->name); ret = -EINVAL ; - goto put_reset; + goto put_msi; } ret =vfio_add_group_dev (dev, &vfio_platform_ops, vdev);
@@ -697,6 +751,8 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev, put_iommu:vfio_iommu_group_put (group, dev); +put_msi: +vfio_platform_put_msi (vdev); put_reset:vfio_platform_put_reset (vdev);return ret;
@@ -745,6 +801,30 @@ void vfio_platform_unregister_reset(const char *compat, }EXPORT_SYMBOL_GPL (vfio_platform_unregister_reset); +void __vfio_platform_register_msi (struct vfio_platform_msi_node *node) +{ +mutex_lock (&driver_lock); +list_add (&node->link, &msi_list); +mutex_unlock (&driver_lock); +} +EXPORT_SYMBOL_GPL (__vfio_platform_register_msi); + +void vfio_platform_unregister_msi (const char *compat) +{ +struct vfio_platform_msi_node *iter, *temp; + +mutex_lock (&driver_lock); +list_for_each_entry_safe (iter, temp, &msi_list, link) { +if (!strcmp (iter->compat, compat)) { +list_del (&iter->link); +break ; + } + } + +mutex_unlock (&driver_lock); +} +EXPORT_SYMBOL_GPL (vfio_platform_unregister_msi); +MODULE_VERSION (DRIVER_VERSION );MODULE_LICENSE ("GPL v2" );MODULE_AUTHOR (DRIVER_AUTHOR );
drivers/vfio/platform/vfio_platform_irq.c +226 -12
285 lines not on this page · raw message
drivers/vfio/platform/vfio_platform_private.h +23 -0
@@ -9,6 +9,7 @@#include <linux/types.h> #include <linux/interrupt.h> +#include <linux/msi.h> #define VFIO_PLATFORM_OFFSET_SHIFT 40#define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
@@ -26,6 +27,7 @@ struct vfio_platform_irq {char *name;struct eventfd_ctx *trigger;bool masked; +bool is_msi;spinlock_t lock;struct virqfd *unmask;struct virqfd *mask;
@@ -46,13 +48,16 @@ struct vfio_platform_device {u32 num_regions;struct vfio_platform_irq *irqs;u32 num_irqs; +u32 num_msis;int refcnt;struct mutex igate;struct module *parent_module;const char *compat;const char *acpihid;struct module *reset_module; +struct module *msi_module;struct device *device; +int config_msi;/* * These fields should be filled by the bus specific binder
@@ -65,11 +70,19 @@ struct vfio_platform_device { (*get_resource)(struct vfio_platform_device *vdev, int i);int (*get_irq)(struct vfio_platform_device *vdev,int i);int (*of_reset)(struct vfio_platform_device *vdev); +u32 (*of_get_msi)(struct vfio_platform_device *vdev); +int (*of_msi_write)(struct vfio_platform_device *vdev, +struct msi_desc *desc, +struct msi_msg *msg);bool reset_required; };typedef int (*vfio_platform_reset_fn_t )(struct vfio_platform_device *vdev); +typedef u32 (*vfio_platform_get_msi_fn_t )(struct vfio_platform_device *vdev); +typedef int (*vfio_platform_msi_write_fn_t )(struct vfio_platform_device *vdev, +struct msi_desc *desc, +struct msi_msg *msg);struct vfio_platform_reset_node {struct list_head link;
@@ -78,6 +91,14 @@ struct vfio_platform_reset_node {vfio_platform_reset_fn_t of_reset; }; +struct vfio_platform_msi_node { +struct list_head link; +char *compat; +struct module *owner; +vfio_platform_get_msi_fn_t of_get_msi; +vfio_platform_msi_write_fn_t of_msi_write; +}; +extern int vfio_platform_probe_common (struct vfio_platform_device *vdev,struct device *dev);extern struct vfio_platform_device *vfio_platform_remove_common
@@ -94,6 +115,8 @@ extern int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,extern void __vfio_platform_register_reset (struct vfio_platform_reset_node *n);extern void vfio_platform_unregister_reset (const char *compat,vfio_platform_reset_fn_t fn); +void __vfio_platform_register_msi (struct vfio_platform_msi_node *n); +void vfio_platform_unregister_msi (const char *compat);#define vfio_platform_register_reset (__compat, __reset) \ static struct vfio_platform_reset_node __reset ## _node = { \ .owner = THIS_MODULE, \
Signature · 1 line
2.17.1
RFC 2/3 vfio/platform: change cleanup order
In reply to Vikas GuptaIn the case of msi, vendor specific msi module may require region access to handle msi cleanup so we need to cleanup region after irq cleanup only.
--- drivers/vfio/platform/vfio_platform_common.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/vfio/platform/vfio_platform_common.c +1 -1
@@ -278,8 +278,8 @@ static void vfio_platform_release(void *device_data)WARN_ON (1 ); }pm_runtime_put (vdev->device); -vfio_platform_regions_cleanup (vdev);vfio_platform_irq_cleanup (vdev); +vfio_platform_regions_cleanup (vdev); }mutex_unlock (&driver_lock);
Signature · 1 line
2.17.1
RFC 3/3 vfio/platform: add Broadcom msi module
In reply to Vikas GuptaAdd Broadcom msi module for platform devices.
--- drivers/vfio/platform/Kconfig | 1 + drivers/vfio/platform/Makefile | 1 + drivers/vfio/platform/msi/Kconfig | 9 +++ drivers/vfio/platform/msi/Makefile | 2 + .../vfio/platform/msi/vfio_platform_bcmplt.c | 74 +++++++++++++++++++ 5 files changed, 87 insertions(+) create mode 100644 drivers/vfio/platform/msi/Kconfig create mode 100644 drivers/vfio/platform/msi/Makefile create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
drivers/vfio/platform/Kconfig +1 -0
@@ -21,3 +21,4 @@ config VFIO_AMBA If you don't know what to do here, say N. source "drivers/vfio/platform/reset/Kconfig" +source "drivers/vfio/platform/msi/Kconfig"
drivers/vfio/platform/Makefile +1 -0
@@ -5,6 +5,7 @@ vfio-platform-y := vfio_platform.o obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform.o obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform-base.o obj-$(CONFIG_VFIO_PLATFORM) += reset/ +obj-$(CONFIG_VFIO_PLATFORM) += msi/ vfio-amba-y := vfio_amba.o
drivers/vfio/platform/msi/Kconfig new file +9 -0
@@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0-only +config VFIO_PLATFORM_BCMPLT_MSI + tristate "MSI support for Broadcom platform devices" + depends on VFIO_PLATFORM && (ARCH_BCM_IPROC || COMPILE_TEST) + default ARCH_BCM_IPROC + help + Enables the VFIO platform driver to handle msi for Broadcom devices + + If you don't know what to do here, say N.
drivers/vfio/platform/msi/Makefile new file +2 -0
@@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_VFIO_PLATFORM_BCMPLT_MSI) += vfio_platform_bcmplt.o
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +74 -0
74 lines not on this page · raw message
Signature · 1 line
2.17.1
Auger Eric
maintainerHi Vikas,
Quoted 6 lines · Vikas Gupta
This RFC adds support for MSI for platform devices.
a) MSI(s) is/are added in addition to the normal interrupts.
b) The vendor specific MSI configuration can be done using callbacks which is implemented as msi module.
c) Adds a msi handling module for the Broadcom platform devices.
Quoted 12 lines
Changes from:
-------------
v0 to v1:
i) Removed MSI device flagVFIO_DEVICE_FLAGS_MSI. ii) Add MSI(s) at the end of the irq list of platform IRQs. MSI(s) with first entry of MSI block has count and flag information.
IRQ list: Allocation for IRQs + MSIs are allocated as below
Example: if there are 'n' IRQs and 'k' MSIs
-------------------------------------------------------
|IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
-------------------------------------------------------
I have not taken time yet to look at your series, but to me you should have |IRQ-0|IRQ-1|....|IRQ-n|MSI|MSIX
then for setting a given MSIX (i) you would select the MSIx index and then set start=i count=1.
to me individual MSIs are encoded in the subindex and not in the index. The index just selects the "type" of interrupt.
For PCI you just have:
VFIO_PCI_INTX_IRQ_INDEX,
VFIO_PCI_MSI_IRQ_INDEX, -> MSI index and then you play withstart/count
VFIO_PCI_MSIX_IRQ_INDEX,
VFIO_PCI_ERR_IRQ_INDEX,
VFIO_PCI_REQ_IRQ_INDEX,
(include/uapi/linux/vfio.h)
Thanks
Eric
Quoted 20 lines
MSI-0 will have count=k set and flags set accordingly.
Vikas Gupta (3):
vfio/platform: add support for msi
vfio/platform: change cleanup order
vfio/platform: add Broadcom msi moduledrivers/vfio/platform/Kconfig | 1 + drivers/vfio/platform/Makefile | 1 + drivers/vfio/platform/msi/Kconfig | 9 + drivers/vfio/platform/msi/Makefile | 2 + .../vfio/platform/msi/vfio_platform_bcmplt.c | 74 ++++++ drivers/vfio/platform/vfio_platform_common.c | 86 ++++++- drivers/vfio/platform/vfio_platform_irq.c | 238 +++++++++++++++++- drivers/vfio/platform/vfio_platform_private.h | 23 ++ 8 files changed, 419 insertions(+), 15 deletions(-) create mode 100644 drivers/vfio/platform/msi/Kconfig create mode 100644 drivers/vfio/platform/msi/Makefile create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
Hi Eric,
Quoted 22 lines · Auger Eric
Hi Vikas,
Quoted 6 lines · Vikas Gupta
This RFC adds support for MSI for platform devices.
a) MSI(s) is/are added in addition to the normal interrupts.
b) The vendor specific MSI configuration can be done using callbacks which is implemented as msi module.
c) Adds a msi handling module for the Broadcom platform devices.Quoted 12 lines
Changes from:
-------------
v0 to v1:
i) Removed MSI device flagVFIO_DEVICE_FLAGS_MSI. ii) Add MSI(s) at the end of the irq list of platform IRQs. MSI(s) with first entry of MSI block has count and flag information.
IRQ list: Allocation for IRQs + MSIs are allocated as below
Example: if there are 'n' IRQs and 'k' MSIs
-------------------------------------------------------
|IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
-------------------------------------------------------
Quoted 4 lines
I have not taken time yet to look at your series, but to me you should have |IRQ-0|IRQ-1|....|IRQ-n|MSI|MSIX
then for setting a given MSIX (i) you would select the MSIx index and then set start=i count=1.
As per your suggestion, we should have, if there are n-IRQs, k-MSIXs and m-MSIs, allocation of IRQs should be done as below
|IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX|
| |
|
|MSIX0||MSIX1||MSXI2|....|MSIX-(k-1)|
|MSI0||MSI1||MSI2|....|MSI-(m-1)|With this implementation user space can know that, at indexes n and n+1, edge triggered interrupts are present.
We may add an element in vfio_platform_irq itself to allocate MSIs/MSIXs struct vfio_platform_irq{
.....
.....
struct vfio_platform_irq *block; => this points to the block
allocation for MSIs/MSIXs and all msi/msix are type of IRQs.
};
OR
Another structure can be defined in 'vfio_pci_private.h'
struct vfio_msi_ctx {
struct eventfd_ctx *trigger;
char *name;
};and
struct vfio_platform_irq {
.....
.....
struct vfio_msi_ctx *block; => this points to the block allocation
for MSIs/MSIXs
};
Which of the above two options sounds OK to you? Please suggest.
Quoted 4 lines
to me individual MSIs are encoded in the subindex and not in the index. The index just selects the "type" of interrupt.
For PCI you just have:
Quoted 8 lines
VFIO_PCI_INTX_IRQ_INDEX, VFIO_PCI_MSI_IRQ_INDEX, -> MSI index and then you play withstart/count
VFIO_PCI_MSIX_IRQ_INDEX,
VFIO_PCI_ERR_IRQ_INDEX,
VFIO_PCI_REQ_IRQ_INDEX,(
include/uapi/linux/vfio.h)
In pci case, type of interrupts is fixed so they can be 'indexed' by these enums but for VFIO platform user space will need to iterate all (num_irqs) indexes to know at which indexes edge triggered interrupts are present.
Thanks,
Vikas
Quoted 25 lines
Thanks
Eric
Quoted 20 lines
MSI-0 will have count=k set and flags set accordingly.
Vikas Gupta (3):
vfio/platform: add support for msi
vfio/platform: change cleanup order
vfio/platform: add Broadcom msi moduledrivers/vfio/platform/Kconfig | 1 + drivers/vfio/platform/Makefile | 1 + drivers/vfio/platform/msi/Kconfig | 9 + drivers/vfio/platform/msi/Makefile | 2 + .../vfio/platform/msi/vfio_platform_bcmplt.c | 74 ++++++ drivers/vfio/platform/vfio_platform_common.c | 86 ++++++- drivers/vfio/platform/vfio_platform_irq.c | 238 +++++++++++++++++- drivers/vfio/platform/vfio_platform_private.h | 23 ++ 8 files changed, 419 insertions(+), 15 deletions(-) create mode 100644 drivers/vfio/platform/msi/Kconfig create mode 100644 drivers/vfio/platform/msi/Makefile create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
Auger Eric
maintainerHi Vikas,
Quoted 30 lines · Vikas Gupta
Hi Eric,
Quoted 22 lines · Auger Eric
Hi Vikas,
Quoted 6 lines · Vikas Gupta
This RFC adds support for MSI for platform devices.
a) MSI(s) is/are added in addition to the normal interrupts.
b) The vendor specific MSI configuration can be done using callbacks which is implemented as msi module.
c) Adds a msi handling module for the Broadcom platform devices.Quoted 12 lines
Changes from:
-------------
v0 to v1:
i) Removed MSI device flagVFIO_DEVICE_FLAGS_MSI. ii) Add MSI(s) at the end of the irq list of platform IRQs. MSI(s) with first entry of MSI block has count and flag information.
IRQ list: Allocation for IRQs + MSIs are allocated as below
Example: if there are 'n' IRQs and 'k' MSIs
-------------------------------------------------------
|IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
-------------------------------------------------------Quoted 4 lines
I have not taken time yet to look at your series, but to me you should have |IRQ-0|IRQ-1|....|IRQ-n|MSI|MSIX
then for setting a given MSIX (i) you would select the MSIx index and then set start=i count=1.
Quoted 8 lines
As per your suggestion, we should have, if there are n-IRQs, k-MSIXs and m-MSIs, allocation of IRQs should be done as below
|IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX| | | | |MSIX0||MSIX1||MSXI2|....|MSIX-(k-1)| |MSI0||MSI1||MSI2|....|MSI-(m-1)|
No I really meant this list of indices: IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX| and potentially later on IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX| ERR| REQ if ERR/REQ were to be added.
I think the userspace could query the total number of indices using VFIO_DEVICE_GET_INFO and retrieve num_irqs (corresponding to the n wire interrupts + MSI index + MSIX index)
Then userspace can loop on all the indices using VFIO_DEVICE_GET_IRQ_INFO. For each index it uses count to determine the first indices related to wire interrupts (count = 1). Then comes the MSI index, and after the MSI index. If any of those is supported, count >1, otherwise count=0. The only thing I am dubious about is can the device use a single MSI/MSIX? Because my hypothesis here is we use count to discriminate between wire first indices and other indices.
Quoted 2 lines
With this implementation user space can know that, at indexes n and n+1, edge triggered interrupts are present.
note wired interrupts can also be edge ones.
Quoted 6 lines
We may add an element in vfio_platform_irq itself to allocate MSIs/MSIXs struct vfio_platform_irq{ ..... ..... struct vfio_platform_irq *block; => this points to the blockallocation for MSIs/MSIXs and all msi/msix are type of IRQs.As wired interrupts and MSI interrupts coexist, I would store in vdev an
array of wired interrupts (the existing vdev->irqs) and a new array for MSI(x) as done in the PCI code.
vdev->ctx = kcalloc(nvec, sizeof(struct vfio_pci_irq_ctx), GFP_KERNEL);
Does it make sense?
Quoted 27 lines
}; OR Another structure can be defined in 'vfio_pci_private.h' struct vfio_msi_ctx { struct eventfd_ctx *trigger; char *name; };and
structvfio_platform_irq{
.....
.....
structvfio_msi_ctx*block; => this points to the block allocation
for MSIs/MSIXs
};
Which of the above two options sounds OK to you? Please suggest.Quoted 11 lines
to me individual MSIs are encoded in the subindex and not in the index. The index just selects the "type" of interrupt.
For PCI you just have:
VFIO_PCI_INTX_IRQ_INDEX, VFIO_PCI_MSI_IRQ_INDEX, -> MSI index and then you play withstart/count
VFIO_PCI_MSIX_IRQ_INDEX,
VFIO_PCI_ERR_IRQ_INDEX,
VFIO_PCI_REQ_IRQ_INDEX,
Quoted 6 lines
Quoted 1 line
(
include/uapi/linux/vfio.h)In pci case, type of interrupts is fixed so they can be 'indexed' by these enums but for VFIO platform user space will need to iterate all (
num_irqs) indexes to know at which indexes edge triggered interrupts are present.
indeed, but can't you loop over all indices looking until count !=1? At this point you know if have finished emurating the wires. Holds if MSI(x) count !=1 of course.
Thanks
Eric
Quoted 28 lines
Thanks,
VikasQuoted 25 lines
Thanks
Eric
Quoted 20 lines
MSI-0 will have count=k set and flags set accordingly.
Vikas Gupta (3):
vfio/platform: add support for msi
vfio/platform: change cleanup order
vfio/platform: add Broadcom msi moduledrivers/vfio/platform/Kconfig | 1 + drivers/vfio/platform/Makefile | 1 + drivers/vfio/platform/msi/Kconfig | 9 + drivers/vfio/platform/msi/Makefile | 2 + .../vfio/platform/msi/vfio_platform_bcmplt.c | 74 ++++++ drivers/vfio/platform/vfio_platform_common.c | 86 ++++++- drivers/vfio/platform/vfio_platform_irq.c | 238 +++++++++++++++++- drivers/vfio/platform/vfio_platform_private.h | 23 ++ 8 files changed, 419 insertions(+), 15 deletions(-) create mode 100644 drivers/vfio/platform/msi/Kconfig create mode 100644 drivers/vfio/platform/msi/Makefile create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
Hi Eric,
Quoted 37 lines · Auger Eric
Hi Vikas,
Quoted 33 lines · Vikas Gupta
Hi Eric,
Quoted 22 lines · Auger Eric
Hi Vikas,
Quoted 6 lines · Vikas Gupta
This RFC adds support for MSI for platform devices.
a) MSI(s) is/are added in addition to the normal interrupts.
b) The vendor specific MSI configuration can be done using callbacks which is implemented as msi module.
c) Adds a msi handling module for the Broadcom platform devices.Quoted 12 lines
Changes from:
-------------
v0 to v1:
i) Removed MSI device flagVFIO_DEVICE_FLAGS_MSI. ii) Add MSI(s) at the end of the irq list of platform IRQs. MSI(s) with first entry of MSI block has count and flag information.
IRQ list: Allocation for IRQs + MSIs are allocated as below
Example: if there are 'n' IRQs and 'k' MSIs
-------------------------------------------------------
|IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
-------------------------------------------------------Quoted 4 lines
I have not taken time yet to look at your series, but to me you should have |IRQ-0|IRQ-1|....|IRQ-n|MSI|MSIX
then for setting a given MSIX (i) you would select the MSIx index and then set start=i count=1.As per your suggestion, we should have, if there are n-IRQs, k-MSIXs and m-MSIs, allocation of IRQs should be done as below
Quoted 8 lines
Quoted 5 lines
|IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX| | | | |MSIX0||MSIX1||MSXI2|....|MSIX-(k-1)| |MSI0||MSI1||MSI2|....|MSI-(m-1)|No I really meant this list of indices: IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX| and potentially later on IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX| ERR| REQ if ERR/REQ were to be added.
I agree on this. Actually the map I drew incorrectly above but wanted to demonstrate the same. It was a child-parent relationship for MSI and its members and similarly for MSIX as well.
Quoted 5 lines
I think the userspace could query the total number of indices using
VFIO_DEVICE_GET_INFOand retrievenum_irqs(corresponding to the n wire interrupts + MSI index + MSIX index)
Quoted 7 lines
Then userspace can loop on all the indices using
VFIO_DEVICE_GET_IRQ_INFO. For each index it uses count to determine the first indices related to wire interrupts (count = 1). Then comes the MSI index, and after the MSI index. If any of those is supported, count >1, otherwise count=0. The only thing I am dubious about is can the device use a single MSI/MSIX? Because my hypothesis here is we use count to discriminate between wire first indices and other indices.
I believe count can be one as well, especially for ERR/REQ as you mentioned above.I think we can not rely on the count > 1. Now, this is blocking and we are not left with options unless we consider adding more enums in flags in vfio_irq_info to tell userspace that particular index is wired, MSI, MSIX etc. for the platform device. What do you think?
Quoted 12 lines
Quoted 5 lines
With this implementation user space can know that, at indexes n and n+1, edge triggered interrupts are present.
note wired interrupts can also be edge ones.
Quoted 6 lines
We may add an element in vfio_platform_irq itself to allocate MSIs/MSIXs struct vfio_platform_irq{ ..... ..... struct vfio_platform_irq *block; => this points to the blockallocation for MSIs/MSIXs and all msi/msix are type of IRQs.As wired interrupts and MSI interrupts coexist, I would store in vdev an
Quoted 6 lines
array of wired interrupts (the existing
vdev->irqs) and a new array for MSI(x) as done in the PCI code.
vdev->ctx=kcalloc(nvec, sizeof(struct vfio_pci_irq_ctx), GFP_KERNEL);Does it make sense?
Yes, we can use similar kinds of allocations.
Thanks,
Vikas
Quoted 71 lines
Quoted 28 lines
}; OR Another structure can be defined in 'vfio_pci_private.h' struct vfio_msi_ctx { struct eventfd_ctx *trigger; char *name; };and
structvfio_platform_irq{
.....
.....
structvfio_msi_ctx*block; => this points to the block allocation
for MSIs/MSIXs
};
Which of the above two options sounds OK to you? Please suggest.Quoted 11 lines
to me individual MSIs are encoded in the subindex and not in the index. The index just selects the "type" of interrupt.
For PCI you just have:
VFIO_PCI_INTX_IRQ_INDEX, VFIO_PCI_MSI_IRQ_INDEX, -> MSI index and then you play withstart/count
VFIO_PCI_MSIX_IRQ_INDEX,
VFIO_PCI_ERR_IRQ_INDEX,
VFIO_PCI_REQ_IRQ_INDEX,Quoted 6 lines
Quoted 1 line
(
include/uapi/linux/vfio.h)In pci case, type of interrupts is fixed so they can be 'indexed' by these enums but for VFIO platform user space will need to iterate all (
num_irqs) indexes to know at which indexes edge triggered interrupts are present.indeed, but can't you loop over all indices looking until count !=1? At this point you know if have finished emurating the wires. Holds if MSI(x) count !=1 of course.
Thanks
Eric
Quoted 28 lines
Thanks,
VikasQuoted 25 lines
Thanks
Eric
Quoted 20 lines
MSI-0 will have count=k set and flags set accordingly.
Vikas Gupta (3):
vfio/platform: add support for msi
vfio/platform: change cleanup order
vfio/platform: add Broadcom msi moduledrivers/vfio/platform/Kconfig | 1 + drivers/vfio/platform/Makefile | 1 + drivers/vfio/platform/msi/Kconfig | 9 + drivers/vfio/platform/msi/Makefile | 2 + .../vfio/platform/msi/vfio_platform_bcmplt.c | 74 ++++++ drivers/vfio/platform/vfio_platform_common.c | 86 ++++++- drivers/vfio/platform/vfio_platform_irq.c | 238 +++++++++++++++++- drivers/vfio/platform/vfio_platform_private.h | 23 ++ 8 files changed, 419 insertions(+), 15 deletions(-) create mode 100644 drivers/vfio/platform/msi/Kconfig create mode 100644 drivers/vfio/platform/msi/Makefile create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
Auger Eric
maintainerHi Vikas,
Quoted 63 lines · Vikas Gupta
Hi Eric,
Quoted 37 lines · Auger Eric
Hi Vikas,
Quoted 33 lines · Vikas Gupta
Hi Eric,
Quoted 22 lines · Auger Eric
Hi Vikas,
Quoted 6 lines · Vikas Gupta
This RFC adds support for MSI for platform devices.
a) MSI(s) is/are added in addition to the normal interrupts.
b) The vendor specific MSI configuration can be done using callbacks which is implemented as msi module.
c) Adds a msi handling module for the Broadcom platform devices.Quoted 12 lines
Changes from:
-------------
v0 to v1:
i) Removed MSI device flagVFIO_DEVICE_FLAGS_MSI. ii) Add MSI(s) at the end of the irq list of platform IRQs. MSI(s) with first entry of MSI block has count and flag information.
IRQ list: Allocation for IRQs + MSIs are allocated as below
Example: if there are 'n' IRQs and 'k' MSIs
-------------------------------------------------------
|IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
-------------------------------------------------------Quoted 4 lines
I have not taken time yet to look at your series, but to me you should have |IRQ-0|IRQ-1|....|IRQ-n|MSI|MSIX
then for setting a given MSIX (i) you would select the MSIx index and then set start=i count=1.As per your suggestion, we should have, if there are n-IRQs, k-MSIXs and m-MSIs, allocation of IRQs should be done as below
Quoted 8 lines
Quoted 5 lines
|IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX| | | | |MSIX0||MSIX1||MSXI2|....|MSIX-(k-1)| |MSI0||MSI1||MSI2|....|MSI-(m-1)|No I really meant this list of indices: IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX| and potentially later on IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX| ERR| REQ if ERR/REQ were to be added.
I agree on this. Actually the map I drew incorrectly above but wanted to demonstrate the same. It was a child-parent relationship for MSI and its members and similarly for MSIX as well.
Quoted 5 lines
I think the userspace could query the total number of indices using
VFIO_DEVICE_GET_INFOand retrievenum_irqs(corresponding to the n wire interrupts + MSI index + MSIX index)Quoted 7 lines
Then userspace can loop on all the indices using
VFIO_DEVICE_GET_IRQ_INFO. For each index it uses count to determine the first indices related to wire interrupts (count = 1). Then comes the MSI index, and after the MSI index. If any of those is supported, count >1, otherwise count=0. The only thing I am dubious about is can the device use a single MSI/MSIX? Because my hypothesis here is we use count to discriminate between wire first indices and other indices.
Quoted 2 lines
I believe count can be one as well, especially for ERR/REQ as you mentioned above.
Given ERR and REQ indices would follow MSI and MSIX ones, MSI index could be recognized by the first index whose count != 1. But indeed I am not sure the number of supported vectors cannot be 1. In your case it is induced by the size of the ring so it is OK but for other devices this may be different.
I think we can not rely on the count > 1. Now, this is
Quoted 4 lines
blocking and we are not left with options unless we consider adding more enums in flags in
vfio_irq_infoto tell userspace that particular index is wired, MSI, MSIX etc. for the platform device. What do you think?
If count is not reliable to discriminate the first n wired interrupts from the subsequen MSI and MSIx index, Alex suggested to add a capability extension in the vfio_irq_info structure. Something similar to what was done for vfio_region_info.
Such kind of thing was attempted in
https://lore.kernel.org/kvmarm/[email protected]/T/#u
` [PATCH v11 07/13] vfio: Use capability chains to handle device specific irq [PATCH v11 08/13] vfio/pci: Add framework for custom interrupt indices [PATCH v11 09/13] vfio: Add new IRQ for DMA fault reporting
Note this has not been reviewed yet.
Thanks
Eric
Quoted 93 lines
Quoted 12 lines
Quoted 5 lines
With this implementation user space can know that, at indexes n and n+1, edge triggered interrupts are present.
note wired interrupts can also be edge ones.
Quoted 6 lines
We may add an element in vfio_platform_irq itself to allocate MSIs/MSIXs struct vfio_platform_irq{ ..... ..... struct vfio_platform_irq *block; => this points to the blockallocation for MSIs/MSIXs and all msi/msix are type of IRQs.As wired interrupts and MSI interrupts coexist, I would store in vdev an
Quoted 6 lines
array of wired interrupts (the existing
vdev->irqs) and a new array for MSI(x) as done in the PCI code.
vdev->ctx=kcalloc(nvec, sizeof(struct vfio_pci_irq_ctx), GFP_KERNEL);Does it make sense?
Yes, we can use similar kinds of allocations.
Thanks,
VikasQuoted 71 lines
Quoted 28 lines
}; OR Another structure can be defined in 'vfio_pci_private.h' struct vfio_msi_ctx { struct eventfd_ctx *trigger; char *name; };and
structvfio_platform_irq{
.....
.....
structvfio_msi_ctx*block; => this points to the block allocation
for MSIs/MSIXs
};
Which of the above two options sounds OK to you? Please suggest.Quoted 11 lines
to me individual MSIs are encoded in the subindex and not in the index. The index just selects the "type" of interrupt.
For PCI you just have:
VFIO_PCI_INTX_IRQ_INDEX, VFIO_PCI_MSI_IRQ_INDEX, -> MSI index and then you play withstart/count
VFIO_PCI_MSIX_IRQ_INDEX,
VFIO_PCI_ERR_IRQ_INDEX,
VFIO_PCI_REQ_IRQ_INDEX,Quoted 6 lines
Quoted 1 line
(
include/uapi/linux/vfio.h)In pci case, type of interrupts is fixed so they can be 'indexed' by these enums but for VFIO platform user space will need to iterate all (
num_irqs) indexes to know at which indexes edge triggered interrupts are present.indeed, but can't you loop over all indices looking until count !=1? At this point you know if have finished emurating the wires. Holds if MSI(x) count !=1 of course.
Thanks
Eric
Quoted 28 lines
Thanks,
VikasQuoted 25 lines
Thanks
Eric
Quoted 20 lines
MSI-0 will have count=k set and flags set accordingly.
Vikas Gupta (3):
vfio/platform: add support for msi
vfio/platform: change cleanup order
vfio/platform: add Broadcom msi moduledrivers/vfio/platform/Kconfig | 1 + drivers/vfio/platform/Makefile | 1 + drivers/vfio/platform/msi/Kconfig | 9 + drivers/vfio/platform/msi/Makefile | 2 + .../vfio/platform/msi/vfio_platform_bcmplt.c | 74 ++++++ drivers/vfio/platform/vfio_platform_common.c | 86 ++++++- drivers/vfio/platform/vfio_platform_irq.c | 238 +++++++++++++++++- drivers/vfio/platform/vfio_platform_private.h | 23 ++ 8 files changed, 419 insertions(+), 15 deletions(-) create mode 100644 drivers/vfio/platform/msi/Kconfig create mode 100644 drivers/vfio/platform/msi/Makefile create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
Auger Eric
maintainerHi Vikas,
Quoted 84 lines · Auger Eric
Hi Vikas,
Quoted 63 lines · Vikas Gupta
Hi Eric,
Quoted 37 lines · Auger Eric
Hi Vikas,
Quoted 33 lines · Vikas Gupta
Hi Eric,
Quoted 22 lines · Auger Eric
Hi Vikas,
Quoted 6 lines · Vikas Gupta
This RFC adds support for MSI for platform devices.
a) MSI(s) is/are added in addition to the normal interrupts.
b) The vendor specific MSI configuration can be done using callbacks which is implemented as msi module.
c) Adds a msi handling module for the Broadcom platform devices.Quoted 12 lines
Changes from:
-------------
v0 to v1:
i) Removed MSI device flagVFIO_DEVICE_FLAGS_MSI. ii) Add MSI(s) at the end of the irq list of platform IRQs. MSI(s) with first entry of MSI block has count and flag information.
IRQ list: Allocation for IRQs + MSIs are allocated as below
Example: if there are 'n' IRQs and 'k' MSIs
-------------------------------------------------------
|IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
-------------------------------------------------------Quoted 4 lines
I have not taken time yet to look at your series, but to me you should have |IRQ-0|IRQ-1|....|IRQ-n|MSI|MSIX
then for setting a given MSIX (i) you would select the MSIx index and then set start=i count=1.As per your suggestion, we should have, if there are n-IRQs, k-MSIXs and m-MSIs, allocation of IRQs should be done as below
Quoted 8 lines
Quoted 5 lines
|IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX| | | | |MSIX0||MSIX1||MSXI2|....|MSIX-(k-1)| |MSI0||MSI1||MSI2|....|MSI-(m-1)|No I really meant this list of indices: IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX| and potentially later on IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX| ERR| REQ if ERR/REQ were to be added.
I agree on this. Actually the map I drew incorrectly above but wanted to demonstrate the same. It was a child-parent relationship for MSI and its members and similarly for MSIX as well.
Quoted 5 lines
I think the userspace could query the total number of indices using
VFIO_DEVICE_GET_INFOand retrievenum_irqs(corresponding to the n wire interrupts + MSI index + MSIX index)Quoted 7 lines
Then userspace can loop on all the indices using
VFIO_DEVICE_GET_IRQ_INFO. For each index it uses count to determine the first indices related to wire interrupts (count = 1). Then comes the MSI index, and after the MSI index. If any of those is supported, count >1, otherwise count=0. The only thing I am dubious about is can the device use a single MSI/MSIX? Because my hypothesis here is we use count to discriminate between wire first indices and other indices.Quoted 2 lines
I believe count can be one as well, especially for ERR/REQ as you mentioned above.
Given ERR and REQ indices would follow MSI and MSIX ones, MSI index could be recognized by the first index whose count != 1. But indeed I am not sure the number of supported vectors cannot be 1. In your case it is induced by the size of the ring so it is OK but for other devices this may be different.
I think we can not rely on the count > 1. Now, this is
Quoted 4 lines
blocking and we are not left with options unless we consider adding more enums in flags in
vfio_irq_infoto tell userspace that particular index is wired, MSI, MSIX etc. for the platform device. What do you think?If count is not reliable to discriminate the first n wired interrupts from the subsequen MSI and MSIx index, Alex suggested to add a capability extension in the
vfio_irq_infostructure. Something similar to what was done forvfio_region_info.
Quoted 7 lines
Such kind of thing was attempted in
https://lore.kernel.org/kvmarm/[email protected]/T/#u` [PATCH v11 07/13] vfio: Use capability chains to handle device specific irq
[PATCH v11 08/13] vfio/pci: Add framework for custom interrupt indices[PATCH v11 09/13] vfio: Add new IRQ for DMA fault reporting
By the way I was mentionning MSI/MSIx in my previous reply but, as Alex pointed out, with platform device only a single MSI index does make sense, no?
Thanks
Eric
Quoted 100 lines
Note this has not been reviewed yet.
Thanks
Eric
Quoted 93 lines
Quoted 12 lines
Quoted 5 lines
With this implementation user space can know that, at indexes n and n+1, edge triggered interrupts are present.
note wired interrupts can also be edge ones.
Quoted 6 lines
We may add an element in vfio_platform_irq itself to allocate MSIs/MSIXs struct vfio_platform_irq{ ..... ..... struct vfio_platform_irq *block; => this points to the blockallocation for MSIs/MSIXs and all msi/msix are type of IRQs.As wired interrupts and MSI interrupts coexist, I would store in vdev an
Quoted 6 lines
array of wired interrupts (the existing
vdev->irqs) and a new array for MSI(x) as done in the PCI code.
vdev->ctx=kcalloc(nvec, sizeof(struct vfio_pci_irq_ctx), GFP_KERNEL);Does it make sense?
Yes, we can use similar kinds of allocations.
Thanks,
VikasQuoted 71 lines
Quoted 28 lines
}; OR Another structure can be defined in 'vfio_pci_private.h' struct vfio_msi_ctx { struct eventfd_ctx *trigger; char *name; };and
structvfio_platform_irq{
.....
.....
structvfio_msi_ctx*block; => this points to the block allocation
for MSIs/MSIXs
};
Which of the above two options sounds OK to you? Please suggest.Quoted 11 lines
to me individual MSIs are encoded in the subindex and not in the index. The index just selects the "type" of interrupt.
For PCI you just have:
VFIO_PCI_INTX_IRQ_INDEX, VFIO_PCI_MSI_IRQ_INDEX, -> MSI index and then you play withstart/count
VFIO_PCI_MSIX_IRQ_INDEX,
VFIO_PCI_ERR_IRQ_INDEX,
VFIO_PCI_REQ_IRQ_INDEX,Quoted 6 lines
Quoted 1 line
(
include/uapi/linux/vfio.h)In pci case, type of interrupts is fixed so they can be 'indexed' by these enums but for VFIO platform user space will need to iterate all (
num_irqs) indexes to know at which indexes edge triggered interrupts are present.indeed, but can't you loop over all indices looking until count !=1? At this point you know if have finished emurating the wires. Holds if MSI(x) count !=1 of course.
Thanks
Eric
Quoted 28 lines
Thanks,
VikasQuoted 25 lines
Thanks
Eric
Quoted 20 lines
MSI-0 will have count=k set and flags set accordingly.
Vikas Gupta (3):
vfio/platform: add support for msi
vfio/platform: change cleanup order
vfio/platform: add Broadcom msi moduledrivers/vfio/platform/Kconfig | 1 + drivers/vfio/platform/Makefile | 1 + drivers/vfio/platform/msi/Kconfig | 9 + drivers/vfio/platform/msi/Makefile | 2 + .../vfio/platform/msi/vfio_platform_bcmplt.c | 74 ++++++ drivers/vfio/platform/vfio_platform_common.c | 86 ++++++- drivers/vfio/platform/vfio_platform_irq.c | 238 +++++++++++++++++- drivers/vfio/platform/vfio_platform_private.h | 23 ++ 8 files changed, 419 insertions(+), 15 deletions(-) create mode 100644 drivers/vfio/platform/msi/Kconfig create mode 100644 drivers/vfio/platform/msi/Makefile create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
Hi Eric,
Quoted 91 lines · Auger Eric
Hi Vikas,
Quoted 87 lines · Auger Eric
Hi Vikas,
Quoted 63 lines · Vikas Gupta
Hi Eric,
Quoted 37 lines · Auger Eric
Hi Vikas,
Quoted 33 lines · Vikas Gupta
Hi Eric,
Quoted 22 lines · Auger Eric
Hi Vikas,
Quoted 6 lines · Vikas Gupta
This RFC adds support for MSI for platform devices.
a) MSI(s) is/are added in addition to the normal interrupts.
b) The vendor specific MSI configuration can be done using callbacks which is implemented as msi module.
c) Adds a msi handling module for the Broadcom platform devices.Quoted 12 lines
Changes from:
-------------
v0 to v1:
i) Removed MSI device flagVFIO_DEVICE_FLAGS_MSI. ii) Add MSI(s) at the end of the irq list of platform IRQs. MSI(s) with first entry of MSI block has count and flag information.
IRQ list: Allocation for IRQs + MSIs are allocated as below
Example: if there are 'n' IRQs and 'k' MSIs
-------------------------------------------------------
|IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
-------------------------------------------------------Quoted 4 lines
I have not taken time yet to look at your series, but to me you should have |IRQ-0|IRQ-1|....|IRQ-n|MSI|MSIX
then for setting a given MSIX (i) you would select the MSIx index and then set start=i count=1.As per your suggestion, we should have, if there are n-IRQs, k-MSIXs and m-MSIs, allocation of IRQs should be done as below
Quoted 8 lines
Quoted 5 lines
|IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX| | | | |MSIX0||MSIX1||MSXI2|....|MSIX-(k-1)| |MSI0||MSI1||MSI2|....|MSI-(m-1)|No I really meant this list of indices: IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX| and potentially later on IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX| ERR| REQ if ERR/REQ were to be added.
I agree on this. Actually the map I drew incorrectly above but wanted to demonstrate the same. It was a child-parent relationship for MSI and its members and similarly for MSIX as well.
Quoted 5 lines
I think the userspace could query the total number of indices using
VFIO_DEVICE_GET_INFOand retrievenum_irqs(corresponding to the n wire interrupts + MSI index + MSIX index)Quoted 7 lines
Then userspace can loop on all the indices using
VFIO_DEVICE_GET_IRQ_INFO. For each index it uses count to determine the first indices related to wire interrupts (count = 1). Then comes the MSI index, and after the MSI index. If any of those is supported, count >1, otherwise count=0. The only thing I am dubious about is can the device use a single MSI/MSIX? Because my hypothesis here is we use count to discriminate between wire first indices and other indices.Quoted 2 lines
I believe count can be one as well, especially for ERR/REQ as you mentioned above.
Given ERR and REQ indices would follow MSI and MSIX ones, MSI index could be recognized by the first index whose count != 1. But indeed I am not sure the number of supported vectors cannot be 1. In your case it is induced by the size of the ring so it is OK but for other devices this may be different.
I think we can not rely on the count > 1. Now, this is
Quoted 4 lines
blocking and we are not left with options unless we consider adding more enums in flags in
vfio_irq_infoto tell userspace that particular index is wired, MSI, MSIX etc. for the platform device. What do you think?If count is not reliable to discriminate the first n wired interrupts from the subsequen MSI and MSIx index, Alex suggested to add a capability extension in the
vfio_irq_infostructure. Something similar to what was done forvfio_region_info.Such kind of thing was attempted in
https://lore.kernel.org/kvmarm/[email protected]/T/#u
Quoted 8 lines
Quoted 4 lines
` [PATCH v11 07/13] vfio: Use capability chains to handle device specific irq
[PATCH v11 08/13] vfio/pci: Add framework for custom interrupt indices[PATCH v11 09/13] vfio: Add new IRQ for DMA fault reportingBy the way I was mentionning MSI/MSIx in my previous reply but, as Alex pointed out, with platform device only a single MSI index does make sense, no?
Yes, I think single MSI should be OK.
This single MSI index should be implemented as ext_irqs, similar to, as you implemented in the mentioned patch. Is my understanding correct?
Thanks,
Vikas
Quoted 105 lines
Thanks
Eric
Quoted 100 lines
Note this has not been reviewed yet.
Thanks
Eric
Quoted 93 lines
Quoted 12 lines
Quoted 5 lines
With this implementation user space can know that, at indexes n and n+1, edge triggered interrupts are present.
note wired interrupts can also be edge ones.
Quoted 6 lines
We may add an element in vfio_platform_irq itself to allocate MSIs/MSIXs struct vfio_platform_irq{ ..... ..... struct vfio_platform_irq *block; => this points to the blockallocation for MSIs/MSIXs and all msi/msix are type of IRQs.As wired interrupts and MSI interrupts coexist, I would store in vdev an
Quoted 6 lines
array of wired interrupts (the existing
vdev->irqs) and a new array for MSI(x) as done in the PCI code.
vdev->ctx=kcalloc(nvec, sizeof(struct vfio_pci_irq_ctx), GFP_KERNEL);Does it make sense?
Yes, we can use similar kinds of allocations.
Thanks,
VikasQuoted 71 lines
Quoted 28 lines
}; OR Another structure can be defined in 'vfio_pci_private.h' struct vfio_msi_ctx { struct eventfd_ctx *trigger; char *name; };and
structvfio_platform_irq{
.....
.....
structvfio_msi_ctx*block; => this points to the block allocation
for MSIs/MSIXs
};
Which of the above two options sounds OK to you? Please suggest.Quoted 11 lines
to me individual MSIs are encoded in the subindex and not in the index. The index just selects the "type" of interrupt.
For PCI you just have:
VFIO_PCI_INTX_IRQ_INDEX, VFIO_PCI_MSI_IRQ_INDEX, -> MSI index and then you play withstart/count
VFIO_PCI_MSIX_IRQ_INDEX,
VFIO_PCI_ERR_IRQ_INDEX,
VFIO_PCI_REQ_IRQ_INDEX,Quoted 6 lines
Quoted 1 line
(
include/uapi/linux/vfio.h)In pci case, type of interrupts is fixed so they can be 'indexed' by these enums but for VFIO platform user space will need to iterate all (
num_irqs) indexes to know at which indexes edge triggered interrupts are present.indeed, but can't you loop over all indices looking until count !=1? At this point you know if have finished emurating the wires. Holds if MSI(x) count !=1 of course.
Thanks
Eric
Quoted 28 lines
Thanks,
VikasQuoted 25 lines
Thanks
Eric
Quoted 20 lines
MSI-0 will have count=k set and flags set accordingly.
Vikas Gupta (3):
vfio/platform: add support for msi
vfio/platform: change cleanup order
vfio/platform: add Broadcom msi moduledrivers/vfio/platform/Kconfig | 1 + drivers/vfio/platform/Makefile | 1 + drivers/vfio/platform/msi/Kconfig | 9 + drivers/vfio/platform/msi/Makefile | 2 + .../vfio/platform/msi/vfio_platform_bcmplt.c | 74 ++++++ drivers/vfio/platform/vfio_platform_common.c | 86 ++++++- drivers/vfio/platform/vfio_platform_irq.c | 238 +++++++++++++++++- drivers/vfio/platform/vfio_platform_private.h | 23 ++ 8 files changed, 419 insertions(+), 15 deletions(-) create mode 100644 drivers/vfio/platform/msi/Kconfig create mode 100644 drivers/vfio/platform/msi/Makefile create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
Auger Eric
maintainerHi Vikas,
Quoted 99 lines · Vikas Gupta
Hi Eric,
Quoted 96 lines · Auger Eric
Hi Vikas,
Quoted 91 lines · Auger Eric
Hi Vikas,
Quoted 63 lines · Vikas Gupta
Hi Eric,
Quoted 37 lines · Auger Eric
Hi Vikas,
Quoted 33 lines · Vikas Gupta
Hi Eric,
Quoted 22 lines · Auger Eric
Hi Vikas,
Quoted 6 lines · Vikas Gupta
This RFC adds support for MSI for platform devices.
a) MSI(s) is/are added in addition to the normal interrupts.
b) The vendor specific MSI configuration can be done using callbacks which is implemented as msi module.
c) Adds a msi handling module for the Broadcom platform devices.Quoted 12 lines
Changes from:
-------------
v0 to v1:
i) Removed MSI device flagVFIO_DEVICE_FLAGS_MSI. ii) Add MSI(s) at the end of the irq list of platform IRQs. MSI(s) with first entry of MSI block has count and flag information.
IRQ list: Allocation for IRQs + MSIs are allocated as below
Example: if there are 'n' IRQs and 'k' MSIs
-------------------------------------------------------
|IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
-------------------------------------------------------Quoted 4 lines
I have not taken time yet to look at your series, but to me you should have |IRQ-0|IRQ-1|....|IRQ-n|MSI|MSIX
then for setting a given MSIX (i) you would select the MSIx index and then set start=i count=1.As per your suggestion, we should have, if there are n-IRQs, k-MSIXs and m-MSIs, allocation of IRQs should be done as below
Quoted 8 lines
Quoted 5 lines
|IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX| | | | |MSIX0||MSIX1||MSXI2|....|MSIX-(k-1)| |MSI0||MSI1||MSI2|....|MSI-(m-1)|No I really meant this list of indices: IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX| and potentially later on IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX| ERR| REQ if ERR/REQ were to be added.
I agree on this. Actually the map I drew incorrectly above but wanted to demonstrate the same. It was a child-parent relationship for MSI and its members and similarly for MSIX as well.
Quoted 5 lines
I think the userspace could query the total number of indices using
VFIO_DEVICE_GET_INFOand retrievenum_irqs(corresponding to the n wire interrupts + MSI index + MSIX index)Quoted 7 lines
Then userspace can loop on all the indices using
VFIO_DEVICE_GET_IRQ_INFO. For each index it uses count to determine the first indices related to wire interrupts (count = 1). Then comes the MSI index, and after the MSI index. If any of those is supported, count >1, otherwise count=0. The only thing I am dubious about is can the device use a single MSI/MSIX? Because my hypothesis here is we use count to discriminate between wire first indices and other indices.Quoted 2 lines
I believe count can be one as well, especially for ERR/REQ as you mentioned above.
Given ERR and REQ indices would follow MSI and MSIX ones, MSI index could be recognized by the first index whose count != 1. But indeed I am not sure the number of supported vectors cannot be 1. In your case it is induced by the size of the ring so it is OK but for other devices this may be different.
I think we can not rely on the count > 1. Now, this is
Quoted 4 lines
blocking and we are not left with options unless we consider adding more enums in flags in
vfio_irq_infoto tell userspace that particular index is wired, MSI, MSIX etc. for the platform device. What do you think?If count is not reliable to discriminate the first n wired interrupts from the subsequen MSI and MSIx index, Alex suggested to add a capability extension in the
vfio_irq_infostructure. Something similar to what was done forvfio_region_info.Such kind of thing was attempted in
https://lore.kernel.org/kvmarm/[email protected]/T/#u` [PATCH v11 07/13] vfio: Use capability chains to handle device specific irq
[PATCH v11 08/13] vfio/pci: Add framework for custom interrupt indices[PATCH v11 09/13] vfio: Add new IRQ for DMA fault reporting
Quoted 7 lines
Quoted 3 lines
By the way I was mentionning MSI/MSIx in my previous reply but, as Alex pointed out, with platform device only a single MSI index does make sense, no?
Yes, I think single MSI should be OK.
This single MSI index should be implemented asext_irqs, similar to, as you implemented in the mentioned patch. Is my understanding correct?
Yes, if count !=1 cannot be used to detect the MSI index, I think using a capability would do the job and this is aligned with last Alex' suggestion.
Thanks
Eric
Quoted 107 lines
Thanks,
VikasQuoted 105 lines
Thanks
Eric
Quoted 100 lines
Note this has not been reviewed yet.
Thanks
Eric
Quoted 93 lines
Quoted 12 lines
Quoted 5 lines
With this implementation user space can know that, at indexes n and n+1, edge triggered interrupts are present.
note wired interrupts can also be edge ones.
Quoted 6 lines
We may add an element in vfio_platform_irq itself to allocate MSIs/MSIXs struct vfio_platform_irq{ ..... ..... struct vfio_platform_irq *block; => this points to the blockallocation for MSIs/MSIXs and all msi/msix are type of IRQs.As wired interrupts and MSI interrupts coexist, I would store in vdev an
Quoted 6 lines
array of wired interrupts (the existing
vdev->irqs) and a new array for MSI(x) as done in the PCI code.
vdev->ctx=kcalloc(nvec, sizeof(struct vfio_pci_irq_ctx), GFP_KERNEL);Does it make sense?
Yes, we can use similar kinds of allocations.
Thanks,
VikasQuoted 71 lines
Quoted 28 lines
}; OR Another structure can be defined in 'vfio_pci_private.h' struct vfio_msi_ctx { struct eventfd_ctx *trigger; char *name; };and
structvfio_platform_irq{
.....
.....
structvfio_msi_ctx*block; => this points to the block allocation
for MSIs/MSIXs
};
Which of the above two options sounds OK to you? Please suggest.Quoted 11 lines
to me individual MSIs are encoded in the subindex and not in the index. The index just selects the "type" of interrupt.
For PCI you just have:
VFIO_PCI_INTX_IRQ_INDEX, VFIO_PCI_MSI_IRQ_INDEX, -> MSI index and then you play withstart/count
VFIO_PCI_MSIX_IRQ_INDEX,
VFIO_PCI_ERR_IRQ_INDEX,
VFIO_PCI_REQ_IRQ_INDEX,Quoted 6 lines
Quoted 1 line
(
include/uapi/linux/vfio.h)In pci case, type of interrupts is fixed so they can be 'indexed' by these enums but for VFIO platform user space will need to iterate all (
num_irqs) indexes to know at which indexes edge triggered interrupts are present.indeed, but can't you loop over all indices looking until count !=1? At this point you know if have finished emurating the wires. Holds if MSI(x) count !=1 of course.
Thanks
Eric
Quoted 28 lines
Thanks,
VikasQuoted 25 lines
Thanks
Eric
Quoted 20 lines
MSI-0 will have count=k set and flags set accordingly.
Vikas Gupta (3):
vfio/platform: add support for msi
vfio/platform: change cleanup order
vfio/platform: add Broadcom msi moduledrivers/vfio/platform/Kconfig | 1 + drivers/vfio/platform/Makefile | 1 + drivers/vfio/platform/msi/Kconfig | 9 + drivers/vfio/platform/msi/Makefile | 2 + .../vfio/platform/msi/vfio_platform_bcmplt.c | 74 ++++++ drivers/vfio/platform/vfio_platform_common.c | 86 ++++++- drivers/vfio/platform/vfio_platform_irq.c | 238 +++++++++++++++++- drivers/vfio/platform/vfio_platform_private.h | 23 ++ 8 files changed, 419 insertions(+), 15 deletions(-) create mode 100644 drivers/vfio/platform/msi/Kconfig create mode 100644 drivers/vfio/platform/msi/Makefile create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
v2 cover msi support for platform devices
In reply to Vikas GuptaThis RFC adds support for MSI for platform devices. MSI block is added as an ext irq along with the existing wired interrupt implementation.
Changes from:
-------------
v1 to v2:
1) IRQ allocation has been implemented as below:
----------------------------
|IRQ-0|IRQ-1|....|IRQ-n|MSI|
----------------------------
MSI block has msi contexts and its implemneted as ext irq.
2) Removed vendor specific module for msi handling so previously patch2 and patch3 are not required.
3) MSI related data is exported to userspace using 'caps'.
Please note VFIO_IRQ_INFO_CAP_TYPE in include/uapi/linux/vfio.h implementation
is taken from the Eric`s patch
https://patchwork.kernel.org/project/kvm/patch/[email protected]/
v0 to v1:
i) Removed MSI device flag VFIO_DEVICE_FLAGS_MSI. ii) Add MSI(s) at the end of the irq list of platform IRQs. MSI(s) with first entry of MSI block has count and flag information.
IRQ list: Allocation for IRQs + MSIs are allocated as below
Example: if there are 'n' IRQs and 'k' MSIs
-------------------------------------------------------
|IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
-------------------------------------------------------
MSI-0 will have count=k set and flags set accordingly.
Vikas Gupta (1):
vfio/platform: add support for msi
drivers/vfio/platform/vfio_platform_common.c | 99 ++++++- drivers/vfio/platform/vfio_platform_irq.c | 260 +++++++++++++++++- drivers/vfio/platform/vfio_platform_private.h | 16 ++ include/uapi/linux/vfio.h | 43 +++ 4 files changed, 401 insertions(+), 17 deletions(-)
Signature · 1 line
2.17.1
RFC v2 1/1 vfio/platform: add support for msi
MSI support for platform devices.
--- drivers/vfio/platform/vfio_platform_common.c | 99 ++++++- drivers/vfio/platform/vfio_platform_irq.c | 260 +++++++++++++++++- drivers/vfio/platform/vfio_platform_private.h | 16 ++ include/uapi/linux/vfio.h | 43 +++ 4 files changed, 401 insertions(+), 17 deletions(-)
drivers/vfio/platform/vfio_platform_common.c +97 -2
@@ -16,6 +16,7 @@#include <linux/types.h> #include <linux/uaccess.h> #include <linux/vfio.h> +#include <linux/nospec.h> #include "vfio_platform_private.h"
@@ -344,9 +345,16 @@ static long vfio_platform_ioctl(void *device_data, }else if (cmd ==VFIO_DEVICE_GET_IRQ_INFO ) {struct vfio_irq_info info; +struct vfio_info_cap caps = { .buf =NULL , .size =0 }; +struct vfio_irq_info_cap_msi *msi_info =NULL ; +unsigned long capsz; +int ext_irq_index = vdev->num_irqs - vdev->num_ext_irqs; minsz = offsetofend(struct vfio_irq_info, count); +/* For backward compatibility, cannot require this */ + capsz = offsetofend(struct vfio_irq_info, cap_offset); +if (copy_from_user (&info, (void __user *)arg, minsz))return -EFAULT ;
@@ -356,9 +364,89 @@ static long vfio_platform_ioctl(void *device_data,if (info.index >= vdev->num_irqs)return -EINVAL ; +if (info.argsz >= capsz) + minsz = capsz; + +if (info.index == ext_irq_index) { +struct vfio_irq_info_cap_type cap_type = { + .header.id =VFIO_IRQ_INFO_CAP_TYPE , + .header.version =1 }; +int i; +int ret; +int num_msgs; +size_t msi_info_size; +struct vfio_platform_irq *irq; + + info.index =array_index_nospec (info.index, + vdev->num_irqs); + + irq = &vdev->irqs[info.index]; + + info.flags = irq->flags; + cap_type.type = irq->type; + cap_type.subtype = irq->subtype; + + ret =vfio_info_add_capability (&caps, + &cap_type.header, +sizeof (cap_type)); +if (ret) +return ret; + + num_msgs = irq->num_ctx; + + msi_info_size =struct_size (msi_info, msgs, num_msgs); + + msi_info =kzalloc (msi_info_size,GFP_KERNEL ); +if (!msi_info) { +kfree (caps.buf); +return -ENOMEM ; + } + + msi_info->header.id =VFIO_IRQ_INFO_CAP_MSI_DESCS ; + msi_info->header.version =1 ; + msi_info->nr_msgs = num_msgs; + +for (i =0 ; i < num_msgs; i++) { +struct vfio_irq_ctx *ctx = &irq->ctx[i]; + + msi_info->msgs[i].addr_lo = ctx->msg.address_lo; + msi_info->msgs[i].addr_hi = ctx->msg.address_hi; + msi_info->msgs[i].data = ctx->msg.data; + } + + ret =vfio_info_add_capability (&caps, &msi_info->header, + msi_info_size); +if (ret) { +kfree (msi_info); +kfree (caps.buf); +return ret; + } + } + info.flags = vdev->irqs[info.index].flags; info.count = vdev->irqs[info.index].count; +if (caps.size) { + info.flags |=VFIO_IRQ_INFO_FLAG_CAPS ; +if (info.argsz <sizeof (info) + caps.size) { + info.argsz =sizeof (info) + caps.size; + info.cap_offset =0 ; + }else { +vfio_info_cap_shift (&caps,sizeof (info)); +if (copy_to_user ((void __user *)arg + +sizeof (info), caps.buf, + caps.size)) { +kfree (msi_info); +kfree (caps.buf); +return -EFAULT ; + } + info.cap_offset =sizeof (info); + } + +kfree (msi_info); +kfree (caps.buf); + } +return copy_to_user ((void __user *)arg, &info, minsz) ? -EFAULT :0 ;
@@ -366,6 +454,7 @@ static long vfio_platform_ioctl(void *device_data,struct vfio_irq_set hdr;u8 *data =NULL ;int ret =0 ; +int max;size_t data_size =0 ; minsz = offsetofend(struct vfio_irq_set, count);
@@ -373,8 +462,14 @@ static long vfio_platform_ioctl(void *device_data,if (copy_from_user (&hdr, (void __user *)arg, minsz))return -EFAULT ; - ret =vfio_set_irqs_validate_and_prepare (&hdr, vdev->num_irqs, - vdev->num_irqs, &data_size); +if (hdr.index >= vdev->num_irqs) +return -EINVAL ; + + max = vdev->irqs[hdr.index].count; + + ret =vfio_set_irqs_validate_and_prepare (&hdr, max, + vdev->num_irqs, + &data_size);if (ret)return ret;
drivers/vfio/platform/vfio_platform_irq.c +245 -15
304 lines not on this page · raw message
drivers/vfio/platform/vfio_platform_private.h +16 -0
@@ -9,6 +9,7 @@#include <linux/types.h> #include <linux/interrupt.h> +#include <linux/msi.h> #define VFIO_PLATFORM_OFFSET_SHIFT 40#define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
@@ -19,9 +20,18 @@#define VFIO_PLATFORM_INDEX_TO_OFFSET (index) \ ((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT) +struct vfio_irq_ctx { +int hwirq; +char *name; +struct msi_msg msg; +struct eventfd_ctx *trigger; +}; +struct vfio_platform_irq {u32 flags;u32 count; +int num_ctx; +struct vfio_irq_ctx *ctx;int hwirq;char *name;struct eventfd_ctx *trigger;
@@ -29,6 +39,11 @@ struct vfio_platform_irq {spinlock_t lock;struct virqfd *unmask;struct virqfd *mask; + +/* for extended irqs */ +u32 type; +u32 subtype; +int config_msi; };struct vfio_platform_region {
@@ -46,6 +61,7 @@ struct vfio_platform_device {u32 num_regions;struct vfio_platform_irq *irqs;u32 num_irqs; +int num_ext_irqs;int refcnt;struct mutex igate;struct module *parent_module;
include/uapi/linux/vfio.h +43 -0
54 lines not on this page · raw message
Signature · 1 line
2.17.1
Auger Eric
maintainerRe: [RFC v2 1/1] vfio/platform: add support for msi
Hi Vikas,
Quoted 40 lines · Vikas Gupta
MSI support for platform devices.
Signed-off-by Vikas Gupta <[email protected]>--- drivers/vfio/platform/vfio_platform_common.c | 99 ++++++- drivers/vfio/platform/vfio_platform_irq.c | 260 +++++++++++++++++- drivers/vfio/platform/vfio_platform_private.h | 16 ++ include/uapi/linux/vfio.h | 43 +++ 4 files changed, 401 insertions(+), 17 deletions(-)drivers/vfio/platform/vfio_platform_common.c +8
-0@@ -16,6 +16,7 @@#include <linux/types.h> #include <linux/uaccess.h> #include <linux/vfio.h> +#include <linux/nospec.h> #include "vfio_platform_private.h" @@ -344,9 +345,16 @@ static long vfio_platform_ioctl(void *device_data, }else if (cmd ==VFIO_DEVICE_GET_IRQ_INFO ) {struct vfio_irq_info info; +struct vfio_info_cap caps = { .buf =NULL , .size =0 }; +struct vfio_irq_info_cap_msi *msi_info =NULL ; +unsigned long capsz; +int ext_irq_index = vdev->num_irqs - vdev->num_ext_irqs; minsz = offsetofend(struct vfio_irq_info, count); +/* For backward compatibility, cannot require this */ + capsz = offsetofend(struct vfio_irq_info, cap_offset); +if (copy_from_user (&info, (void __user *)arg, minsz))return -EFAULT ;
drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 370
Quoted 7 lines
drivers/vfio/platform/vfio_platform_common.c +4
-0@@ -356,9 +364,89 @@ static long vfio_platform_ioctl(void *device_data,if (info.index >= vdev->num_irqs)return -EINVAL ; +if (info.argsz >= capsz) + minsz = capsz; + +if (info.index == ext_irq_index) {
nit: n case we add new ext indices afterwards, I would check info.index
- ext_irq_index against an VFIO_EXT_IRQ_MSI define.
drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 378
Quoted 8 lines
drivers/vfio/platform/vfio_platform_common.c +8
-0+struct vfio_irq_info_cap_type cap_type = { + .header.id =VFIO_IRQ_INFO_CAP_TYPE , + .header.version =1 }; +int i; +int ret; +int num_msgs; +size_t msi_info_size; +struct vfio_platform_irq *irq;
nit: I think generally the opposite order (length) is chosen. This also would better match the existing style in this file
drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 385
Quoted 7 lines
drivers/vfio/platform/vfio_platform_common.c +7
-0+ + info.index =array_index_nospec (info.index, + vdev->num_irqs); + + irq = &vdev->irqs[info.index]; + + info.flags = irq->flags;
I think this can be removed given [*]
drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 395
Quoted 10 lines
drivers/vfio/platform/vfio_platform_common.c +10
-0+ cap_type.type = irq->type; + cap_type.subtype = irq->subtype; + + ret =vfio_info_add_capability (&caps, + &cap_type.header, +sizeof (cap_type)); +if (ret) +return ret; + + num_msgs = irq->num_ctx;
do would want to return the cap even if !num_ctx?
Quoted 21 lines
drivers/vfio/platform/vfio_platform_common.c +21
-0+ + msi_info_size =struct_size (msi_info, msgs, num_msgs); + + msi_info =kzalloc (msi_info_size,GFP_KERNEL ); +if (!msi_info) { +kfree (caps.buf); +return -ENOMEM ; + } + + msi_info->header.id =VFIO_IRQ_INFO_CAP_MSI_DESCS ; + msi_info->header.version =1 ; + msi_info->nr_msgs = num_msgs; + +for (i =0 ; i < num_msgs; i++) { +struct vfio_irq_ctx *ctx = &irq->ctx[i]; + + msi_info->msgs[i].addr_lo = ctx->msg.address_lo; + msi_info->msgs[i].addr_hi = ctx->msg.address_hi; + msi_info->msgs[i].data = ctx->msg.data; + } +
drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 426
Quoted 10 lines
drivers/vfio/platform/vfio_platform_common.c +9
-0+ ret =vfio_info_add_capability (&caps, &msi_info->header, + msi_info_size); +if (ret) { +kfree (msi_info); +kfree (caps.buf); +return ret; + } + } + info.flags = vdev->irqs[info.index].flags;
[*]
Quoted 329 lines
drivers/vfio/platform/vfio_platform_common.c +30
-2info.count = vdev->irqs[info.index].count; +if (caps.size) { + info.flags |=VFIO_IRQ_INFO_FLAG_CAPS ; +if (info.argsz <sizeof (info) + caps.size) { + info.argsz =sizeof (info) + caps.size; + info.cap_offset =0 ; + }else { +vfio_info_cap_shift (&caps,sizeof (info)); +if (copy_to_user ((void __user *)arg + +sizeof (info), caps.buf, + caps.size)) { +kfree (msi_info); +kfree (caps.buf); +return -EFAULT ; + } + info.cap_offset =sizeof (info); + } + +kfree (msi_info); +kfree (caps.buf); + } +return copy_to_user ((void __user *)arg, &info, minsz) ? -EFAULT :0 ;@@ -366,6 +454,7 @@ static long vfio_platform_ioctl(void *device_data,struct vfio_irq_set hdr;u8 *data =NULL ;int ret =0 ; +int max;size_t data_size =0 ; minsz = offsetofend(struct vfio_irq_set, count);@@ -373,8 +462,14 @@ static long vfio_platform_ioctl(void *device_data,if (copy_from_user (&hdr, (void __user *)arg, minsz))return -EFAULT ; - ret =vfio_set_irqs_validate_and_prepare (&hdr, vdev->num_irqs, - vdev->num_irqs, &data_size); +if (hdr.index >= vdev->num_irqs) +return -EINVAL ; + + max = vdev->irqs[hdr.index].count; + + ret =vfio_set_irqs_validate_and_prepare (&hdr, max, + vdev->num_irqs, + &data_size);if (ret)return ret;drivers/vfio/platform/vfio_platform_irq.c +225
-12@@ -8,10 +8,12 @@#include <linux/eventfd.h> #include <linux/interrupt.h> +#include <linux/eventfd.h> #include <linux/slab.h> #include <linux/types.h> #include <linux/vfio.h> #include <linux/irq.h> +#include <linux/msi.h> #include "vfio_platform_private.h" @@ -253,6 +255,195 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,return 0 ; } +/* MSI/MSIX */ +static irqreturn_t vfio_msihandler (int irq,void *arg) +{ +struct eventfd_ctx *trigger = arg; + +eventfd_signal (trigger,1 ); +return IRQ_HANDLED ; +} + +static void msi_write (struct msi_desc *desc,struct msi_msg *msg) +{ +int i; +struct vfio_platform_irq *irq; +u16 index = desc->platform.msi_index; +struct device *dev =msi_desc_to_dev (desc); +struct vfio_device *device =dev_get_drvdata (dev); +struct vfio_platform_device *vdev = (struct vfio_platform_device *) +vfio_device_data (device); + +for (i =0 ; i < vdev->num_irqs; i++) +if (vdev->irqs[i].type ==VFIO_IRQ_TYPE_MSI ) + irq = &vdev->irqs[i]; + + irq->ctx[index].msg.address_lo = msg->address_lo; + irq->ctx[index].msg.address_hi = msg->address_hi; + irq->ctx[index].msg.data = msg->data; +} + +static int vfio_msi_enable (struct vfio_platform_device *vdev, +struct vfio_platform_irq *irq,int nvec) +{ +int ret; +int msi_idx =0 ; +struct msi_desc *desc; +struct device *dev = vdev->device; + + irq->ctx =kcalloc (nvec,sizeof (struct vfio_irq_ctx ),GFP_KERNEL ); +if (!irq->ctx) +return -ENOMEM ; + +/* Allocate platform MSIs */ + ret =platform_msi_domain_alloc_irqs (dev, nvec, msi_write); +if (ret <0 ) { +kfree (irq->ctx); +return ret; + } + +for_each_msi_entry (desc, dev) { + irq->ctx[msi_idx].hwirq = desc->irq; + msi_idx++; + } + + irq->num_ctx = nvec; + irq->config_msi =1 ; + +return 0 ; +} + +static int vfio_msi_set_vector_signal (struct vfio_platform_irq *irq, +int vector,int fd) +{ +struct eventfd_ctx *trigger; +int irq_num, ret; + +if (vector <0 || vector >= irq->num_ctx) +return -EINVAL ; + + irq_num = irq->ctx[vector].hwirq; + +if (irq->ctx[vector].trigger) { +free_irq (irq_num, irq->ctx[vector].trigger); +kfree (irq->ctx[vector].name); +eventfd_ctx_put (irq->ctx[vector].trigger); + irq->ctx[vector].trigger =NULL ; + } + +if (fd <0 ) +return 0 ; + + irq->ctx[vector].name =kasprintf (GFP_KERNEL , +"vfio-msi[%d]" , vector); +if (!irq->ctx[vector].name) +return -ENOMEM ; + + trigger =eventfd_ctx_fdget (fd); +if (IS_ERR (trigger)) { +kfree (irq->ctx[vector].name); +return PTR_ERR (trigger); + } + + ret =request_irq (irq_num, vfio_msihandler,0 , + irq->ctx[vector].name, trigger); +if (ret) { +kfree (irq->ctx[vector].name); +eventfd_ctx_put (trigger); +return ret; + } + + irq->ctx[vector].trigger = trigger; + +return 0 ; +} + +static int vfio_msi_set_block (struct vfio_platform_irq *irq,unsigned int start, +unsigned int count,int32_t *fds) +{ +int i, j, ret =0 ; + +if (start >= irq->num_ctx || start + count > irq->num_ctx) +return -EINVAL ; + +for (i =0 , j = start; i < count && !ret; i++, j++) { +int fd = fds ? fds[i] :-1 ; + + ret =vfio_msi_set_vector_signal (irq, j, fd); + } + +if (ret) { +for (--j; j >= (int )start; j--) +vfio_msi_set_vector_signal (irq, j,-1 ); + } + +return ret; +} + +static void vfio_msi_disable (struct vfio_platform_device *vdev, +struct vfio_platform_irq *irq) +{ +struct device *dev = vdev->device; + +vfio_msi_set_block (irq,0 , irq->num_ctx,NULL ); + +platform_msi_domain_free_irqs (dev); + + irq->config_msi =0 ; + irq->num_ctx =0 ; + +kfree (irq->ctx); +} + +static int vfio_set_msi_trigger (struct vfio_platform_device *vdev, +unsigned int index,unsigned int start, +unsigned int count,uint32_t flags,void *data) +{ +int i; +struct vfio_platform_irq *irq = &vdev->irqs[index]; + +if (start + count > irq->count) +return -EINVAL ; + +if (!count && (flags &VFIO_IRQ_SET_DATA_NONE )) { +vfio_msi_disable (vdev, irq); +return 0 ; + } + +if (flags &VFIO_IRQ_SET_DATA_EVENTFD ) { +s32 *fds = data; +int ret; + +if (irq->config_msi) +return vfio_msi_set_block (irq, start, count, + fds); + ret =vfio_msi_enable (vdev, irq, start + count); +if (ret) +return ret; + + ret =vfio_msi_set_block (irq, start, count, fds); +if (ret) +vfio_msi_disable (vdev, irq); + +return ret; + } + +for (i = start; i < start + count; i++) { +if (!irq->ctx[i].trigger) +continue ; +if (flags &VFIO_IRQ_SET_DATA_NONE ) { +eventfd_signal (irq->ctx[i].trigger,1 ); + }else if (flags &VFIO_IRQ_SET_DATA_BOOL ) { +u8 *bools = data; + +if (bools[i - start]) +eventfd_signal (irq->ctx[i].trigger,1 ); + } + } + +return 0 ; +} +int vfio_platform_set_irqs_ioctl (struct vfio_platform_device *vdev,uint32_t flags,unsigned index,unsigned start,unsigned count,void *data)@@ -261,16 +452,29 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev, unsigned start, unsigned count, uint32_t flags, void *data) = NULL; -switch (flags &VFIO_IRQ_SET_ACTION_TYPE_MASK ) { -case VFIO_IRQ_SET_ACTION_MASK : - func = vfio_platform_set_irq_mask; -break ; -case VFIO_IRQ_SET_ACTION_UNMASK : - func = vfio_platform_set_irq_unmask; -break ; -case VFIO_IRQ_SET_ACTION_TRIGGER : - func = vfio_platform_set_irq_trigger; -break ; +struct vfio_platform_irq *irq = &vdev->irqs[index]; + +if (irq->type ==VFIO_IRQ_TYPE_MSI ) { +switch (flags &VFIO_IRQ_SET_ACTION_TYPE_MASK ) { +case VFIO_IRQ_SET_ACTION_MASK : +case VFIO_IRQ_SET_ACTION_UNMASK : +break ; +case VFIO_IRQ_SET_ACTION_TRIGGER : + func = vfio_set_msi_trigger; +break ; + } + }else { +switch (flags &VFIO_IRQ_SET_ACTION_TYPE_MASK ) { +case VFIO_IRQ_SET_ACTION_MASK : + func = vfio_platform_set_irq_mask; +break ; +case VFIO_IRQ_SET_ACTION_UNMASK : + func = vfio_platform_set_irq_unmask; +break ; +case VFIO_IRQ_SET_ACTION_TRIGGER : + func = vfio_platform_set_irq_trigger; +break ; + } }if (!func)@@ -281,12 +485,21 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,int vfio_platform_irq_init (struct vfio_platform_device *vdev) { -int cnt =0 , i; +int i; +int cnt =0 ; +int num_irqs; +struct device *dev = vdev->device;while (vdev->get_irq (vdev, cnt) >=0 ) cnt++; - vdev->irqs =kcalloc (cnt,sizeof (struct vfio_platform_irq ),GFP_KERNEL ); + num_irqs = cnt; + +if (dev->msi_domain) + num_irqs++; + + vdev->irqs =kcalloc (num_irqs,sizeof (struct vfio_platform_irq ), +GFP_KERNEL );if (!vdev->irqs)return -ENOMEM ;
drivers/vfio/platform/vfio_platform_irq.c int vfio_platform_irq_init(struct vfio_platform_device *vdev) · line 526
Quoted 6 lines
drivers/vfio/platform/vfio_platform_irq.c +2
-1@@ -309,7 +522,19 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev) vdev->irqs[i].masked = false; } - vdev->num_irqs = cnt; +/* +* MSI block is added at last index and its an ext irq
it is
drivers/vfio/platform/vfio_platform_irq.c int vfio_platform_irq_init(struct vfio_platform_device *vdev) · line 530
Quoted 4 lines
drivers/vfio/platform/vfio_platform_irq.c +4
-0+*/ +if (dev->msi_domain) { + vdev->irqs[i].flags =VFIO_IRQ_INFO_EVENTFD ; + vdev->irqs[i].count =NR_IRQS ;
why NR_IRQS?
Quoted 120 lines
drivers/vfio/platform/vfio_platform_irq.c +14
-2+ vdev->irqs[i].hwirq =0 ; + vdev->irqs[i].masked = false; + vdev->irqs[i].type =VFIO_IRQ_TYPE_MSI ; + vdev->num_ext_irqs =1 ; + } + + vdev->num_irqs = num_irqs;return 0 ; err:@@ -321,8 +546,13 @@ void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev) {int i; -for (i =0 ; i < vdev->num_irqs; i++) -vfio_set_trigger (vdev, i,-1 ,NULL ); +for (i =0 ; i < vdev->num_irqs; i++) { +if (vdev->irqs[i].type ==VFIO_IRQ_TYPE_MSI ) +vfio_set_msi_trigger (vdev, i,0 ,0 , +VFIO_IRQ_SET_DATA_NONE ,NULL ); +else +vfio_set_trigger (vdev, i,-1 ,NULL ); + } vdev->num_irqs =0 ;kfree (vdev->irqs);drivers/vfio/platform/vfio_platform_private.h +16
-0@@ -9,6 +9,7 @@#include <linux/types.h> #include <linux/interrupt.h> +#include <linux/msi.h> #define VFIO_PLATFORM_OFFSET_SHIFT 40#define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)@@ -19,9 +20,18 @@#define VFIO_PLATFORM_INDEX_TO_OFFSET (index) \ ((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT) +struct vfio_irq_ctx { +int hwirq; +char *name; +struct msi_msg msg; +struct eventfd_ctx *trigger; +}; +struct vfio_platform_irq {u32 flags;u32 count; +int num_ctx; +struct vfio_irq_ctx *ctx;int hwirq;char *name;struct eventfd_ctx *trigger;@@ -29,6 +39,11 @@ struct vfio_platform_irq {spinlock_t lock;struct virqfd *unmask;struct virqfd *mask; + +/* for extended irqs */ +u32 type; +u32 subtype; +int config_msi; };struct vfio_platform_region {@@ -46,6 +61,7 @@ struct vfio_platform_device {u32 num_regions;struct vfio_platform_irq *irqs;u32 num_irqs; +int num_ext_irqs;int refcnt;struct mutex igate;struct module *parent_module;include/uapi/linux/vfio.h +30
-0@@ -697,11 +697,54 @@ struct vfio_irq_info {#define VFIO_IRQ_INFO_MASKABLE (1 << 1)#define VFIO_IRQ_INFO_AUTOMASKED (1 << 2)#define VFIO_IRQ_INFO_NORESIZE (1 << 3) +#define VFIO_IRQ_INFO_FLAG_CAPS (1 << 4)/* Info supports caps */ __u32 index;/* IRQ index */ __u32 count;/* Number of IRQs within this index */ +__u32 cap_offset;/* Offset within info struct of first cap */ };#define VFIO_DEVICE_GET_IRQ_INFO _IO(VFIO_TYPE, VFIO_BASE + 9) +/* +* The irq type capability allows IRQs unique to a specific device or +* class of devices to be exposed. +* +* The structures below define version 1 of this capability. +*/ +#define VFIO_IRQ_INFO_CAP_TYPE 3 + +struct vfio_irq_info_cap_type { +struct vfio_info_cap_header header; +__u32 type;/* global per bus driver */ +__u32 subtype;/* type specific */ +}; + +/* +* List of IRQ types, global per bus driver. +* If you introduce a new type, please add it here. +*/ + +/* Non PCI devices having MSI(s) support */ +#define VFIO_IRQ_TYPE_MSI (1) + +/* +* The msi capability allows the user to use the msi msg to +* configure the iova for the msi configuration. +* The structures below define version 1 of this capability. +*/ +#define VFIO_IRQ_INFO_CAP_MSI_DESCS 4
include/uapi/linux/vfio.h struct vfio_irq_info { · line 744
Quoted 10 lines
include/uapi/linux/vfio.h +10
-0+ +struct vfio_irq_msi_msg { +__u32 addr_lo; +__u32 addr_hi; +__u32 data; +}; + +struct vfio_irq_info_cap_msi { +struct vfio_info_cap_header header; +__u32 nr_msgs;
I think you should align a __u32 reserved field to have a 64b alignment
include/uapi/linux/vfio.h struct vfio_irq_info { · line 745
Quoted 1 line
include/uapi/linux/vfio.h +1
-0+struct vfio_irq_msi_msg msgs[];
Please can you clarify why this cap is needed versus your prior approach.
include/uapi/linux/vfio.h struct vfio_irq_info { · line 750
Quoted 6 lines
include/uapi/linux/vfio.h +2
-0+}; +/** * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set) *
Thanks
Eric
Re: [RFC v2 1/1] vfio/platform: add support for msi
Hi Eric,
Quoted 61 lines · Auger Eric
Hi Vikas,
Quoted 40 lines · Vikas Gupta
MSI support for platform devices.
Signed-off-by Vikas Gupta <[email protected]>--- drivers/vfio/platform/vfio_platform_common.c | 99 ++++++- drivers/vfio/platform/vfio_platform_irq.c | 260 +++++++++++++++++- drivers/vfio/platform/vfio_platform_private.h | 16 ++ include/uapi/linux/vfio.h | 43 +++ 4 files changed, 401 insertions(+), 17 deletions(-)drivers/vfio/platform/vfio_platform_common.c +8
-0@@ -16,6 +16,7 @@#include <linux/types.h> #include <linux/uaccess.h> #include <linux/vfio.h> +#include <linux/nospec.h> #include "vfio_platform_private.h" @@ -344,9 +345,16 @@ static long vfio_platform_ioctl(void *device_data, }else if (cmd ==VFIO_DEVICE_GET_IRQ_INFO ) {struct vfio_irq_info info; +struct vfio_info_cap caps = { .buf =NULL , .size =0 }; +struct vfio_irq_info_cap_msi *msi_info =NULL ; +unsigned long capsz; +int ext_irq_index = vdev->num_irqs - vdev->num_ext_irqs; minsz = offsetofend(struct vfio_irq_info, count); +/* For backward compatibility, cannot require this */ + capsz = offsetofend(struct vfio_irq_info, cap_offset); +if (copy_from_user (&info, (void __user *)arg, minsz))return -EFAULT ;drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 370
Quoted 7 lines
drivers/vfio/platform/vfio_platform_common.c +4
-0@@ -356,9 +364,89 @@ static long vfio_platform_ioctl(void *device_data,if (info.index >= vdev->num_irqs)return -EINVAL ; +if (info.argsz >= capsz) + minsz = capsz; + +if (info.index == ext_irq_index) {nit: n case we add new ext indices afterwards, I would check info.index
-ext_irq_indexagainst anVFIO_EXT_IRQ_MSIdefine.drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 378
Quoted 8 lines
drivers/vfio/platform/vfio_platform_common.c +8
-0+struct vfio_irq_info_cap_type cap_type = { + .header.id =VFIO_IRQ_INFO_CAP_TYPE , + .header.version =1 }; +int i; +int ret; +int num_msgs; +size_t msi_info_size; +struct vfio_platform_irq *irq;
Quoted 2 lines
nit: I think generally the opposite order (length) is chosen. This also would better match the existing style in this file
Ok will fix it
Quoted 8 lines
drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 385
Quoted 7 lines
drivers/vfio/platform/vfio_platform_common.c +7
-0+ + info.index =array_index_nospec (info.index, + vdev->num_irqs); + + irq = &vdev->irqs[info.index]; + + info.flags = irq->flags;I think this can be removed given [*]
Sure.
Quoted 11 lines
drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 395
Quoted 10 lines
drivers/vfio/platform/vfio_platform_common.c +10
-0+ cap_type.type = irq->type; + cap_type.subtype = irq->subtype; + + ret =vfio_info_add_capability (&caps, + &cap_type.header, +sizeof (cap_type)); +if (ret) +return ret; + + num_msgs = irq->num_ctx;do would want to return the cap even if !num_ctx?
If num_ctx = 0 then VFIO_IRQ_INFO_CAP_MSI_DESCS should not be written. I`ll take care of same.
Quoted 24 lines
Quoted 24 lines
drivers/vfio/platform/vfio_platform_common.c +24
-0+ + msi_info_size =struct_size (msi_info, msgs, num_msgs); + + msi_info =kzalloc (msi_info_size,GFP_KERNEL ); +if (!msi_info) { +kfree (caps.buf); +return -ENOMEM ; + } + + msi_info->header.id =VFIO_IRQ_INFO_CAP_MSI_DESCS ; + msi_info->header.version =1 ; + msi_info->nr_msgs = num_msgs; + +for (i =0 ; i < num_msgs; i++) { +struct vfio_irq_ctx *ctx = &irq->ctx[i]; + + msi_info->msgs[i].addr_lo = ctx->msg.address_lo; + msi_info->msgs[i].addr_hi = ctx->msg.address_hi; + msi_info->msgs[i].data = ctx->msg.data; + } + + ret =vfio_info_add_capability (&caps, &msi_info->header, + msi_info_size); +if (ret) {
Quoted 8 lines
drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 426
Quoted 7 lines
drivers/vfio/platform/vfio_platform_common.c +6
-0+kfree (msi_info); +kfree (caps.buf); +return ret; + } + } + info.flags = vdev->irqs[info.index].flags;[*]
Will fix it.
Quoted 336 lines
Quoted 329 lines
drivers/vfio/platform/vfio_platform_common.c +30
-2info.count = vdev->irqs[info.index].count; +if (caps.size) { + info.flags |=VFIO_IRQ_INFO_FLAG_CAPS ; +if (info.argsz <sizeof (info) + caps.size) { + info.argsz =sizeof (info) + caps.size; + info.cap_offset =0 ; + }else { +vfio_info_cap_shift (&caps,sizeof (info)); +if (copy_to_user ((void __user *)arg + +sizeof (info), caps.buf, + caps.size)) { +kfree (msi_info); +kfree (caps.buf); +return -EFAULT ; + } + info.cap_offset =sizeof (info); + } + +kfree (msi_info); +kfree (caps.buf); + } +return copy_to_user ((void __user *)arg, &info, minsz) ? -EFAULT :0 ;@@ -366,6 +454,7 @@ static long vfio_platform_ioctl(void *device_data,struct vfio_irq_set hdr;u8 *data =NULL ;int ret =0 ; +int max;size_t data_size =0 ; minsz = offsetofend(struct vfio_irq_set, count);@@ -373,8 +462,14 @@ static long vfio_platform_ioctl(void *device_data,if (copy_from_user (&hdr, (void __user *)arg, minsz))return -EFAULT ; - ret =vfio_set_irqs_validate_and_prepare (&hdr, vdev->num_irqs, - vdev->num_irqs, &data_size); +if (hdr.index >= vdev->num_irqs) +return -EINVAL ; + + max = vdev->irqs[hdr.index].count; + + ret =vfio_set_irqs_validate_and_prepare (&hdr, max, + vdev->num_irqs, + &data_size);if (ret)return ret;drivers/vfio/platform/vfio_platform_irq.c +225
-12@@ -8,10 +8,12 @@#include <linux/eventfd.h> #include <linux/interrupt.h> +#include <linux/eventfd.h> #include <linux/slab.h> #include <linux/types.h> #include <linux/vfio.h> #include <linux/irq.h> +#include <linux/msi.h> #include "vfio_platform_private.h" @@ -253,6 +255,195 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,return 0 ; } +/* MSI/MSIX */ +static irqreturn_t vfio_msihandler (int irq,void *arg) +{ +struct eventfd_ctx *trigger = arg; + +eventfd_signal (trigger,1 ); +return IRQ_HANDLED ; +} + +static void msi_write (struct msi_desc *desc,struct msi_msg *msg) +{ +int i; +struct vfio_platform_irq *irq; +u16 index = desc->platform.msi_index; +struct device *dev =msi_desc_to_dev (desc); +struct vfio_device *device =dev_get_drvdata (dev); +struct vfio_platform_device *vdev = (struct vfio_platform_device *) +vfio_device_data (device); + +for (i =0 ; i < vdev->num_irqs; i++) +if (vdev->irqs[i].type ==VFIO_IRQ_TYPE_MSI ) + irq = &vdev->irqs[i]; + + irq->ctx[index].msg.address_lo = msg->address_lo; + irq->ctx[index].msg.address_hi = msg->address_hi; + irq->ctx[index].msg.data = msg->data; +} + +static int vfio_msi_enable (struct vfio_platform_device *vdev, +struct vfio_platform_irq *irq,int nvec) +{ +int ret; +int msi_idx =0 ; +struct msi_desc *desc; +struct device *dev = vdev->device; + + irq->ctx =kcalloc (nvec,sizeof (struct vfio_irq_ctx ),GFP_KERNEL ); +if (!irq->ctx) +return -ENOMEM ; + +/* Allocate platform MSIs */ + ret =platform_msi_domain_alloc_irqs (dev, nvec, msi_write); +if (ret <0 ) { +kfree (irq->ctx); +return ret; + } + +for_each_msi_entry (desc, dev) { + irq->ctx[msi_idx].hwirq = desc->irq; + msi_idx++; + } + + irq->num_ctx = nvec; + irq->config_msi =1 ; + +return 0 ; +} + +static int vfio_msi_set_vector_signal (struct vfio_platform_irq *irq, +int vector,int fd) +{ +struct eventfd_ctx *trigger; +int irq_num, ret; + +if (vector <0 || vector >= irq->num_ctx) +return -EINVAL ; + + irq_num = irq->ctx[vector].hwirq; + +if (irq->ctx[vector].trigger) { +free_irq (irq_num, irq->ctx[vector].trigger); +kfree (irq->ctx[vector].name); +eventfd_ctx_put (irq->ctx[vector].trigger); + irq->ctx[vector].trigger =NULL ; + } + +if (fd <0 ) +return 0 ; + + irq->ctx[vector].name =kasprintf (GFP_KERNEL , +"vfio-msi[%d]" , vector); +if (!irq->ctx[vector].name) +return -ENOMEM ; + + trigger =eventfd_ctx_fdget (fd); +if (IS_ERR (trigger)) { +kfree (irq->ctx[vector].name); +return PTR_ERR (trigger); + } + + ret =request_irq (irq_num, vfio_msihandler,0 , + irq->ctx[vector].name, trigger); +if (ret) { +kfree (irq->ctx[vector].name); +eventfd_ctx_put (trigger); +return ret; + } + + irq->ctx[vector].trigger = trigger; + +return 0 ; +} + +static int vfio_msi_set_block (struct vfio_platform_irq *irq,unsigned int start, +unsigned int count,int32_t *fds) +{ +int i, j, ret =0 ; + +if (start >= irq->num_ctx || start + count > irq->num_ctx) +return -EINVAL ; + +for (i =0 , j = start; i < count && !ret; i++, j++) { +int fd = fds ? fds[i] :-1 ; + + ret =vfio_msi_set_vector_signal (irq, j, fd); + } + +if (ret) { +for (--j; j >= (int )start; j--) +vfio_msi_set_vector_signal (irq, j,-1 ); + } + +return ret; +} + +static void vfio_msi_disable (struct vfio_platform_device *vdev, +struct vfio_platform_irq *irq) +{ +struct device *dev = vdev->device; + +vfio_msi_set_block (irq,0 , irq->num_ctx,NULL ); + +platform_msi_domain_free_irqs (dev); + + irq->config_msi =0 ; + irq->num_ctx =0 ; + +kfree (irq->ctx); +} + +static int vfio_set_msi_trigger (struct vfio_platform_device *vdev, +unsigned int index,unsigned int start, +unsigned int count,uint32_t flags,void *data) +{ +int i; +struct vfio_platform_irq *irq = &vdev->irqs[index]; + +if (start + count > irq->count) +return -EINVAL ; + +if (!count && (flags &VFIO_IRQ_SET_DATA_NONE )) { +vfio_msi_disable (vdev, irq); +return 0 ; + } + +if (flags &VFIO_IRQ_SET_DATA_EVENTFD ) { +s32 *fds = data; +int ret; + +if (irq->config_msi) +return vfio_msi_set_block (irq, start, count, + fds); + ret =vfio_msi_enable (vdev, irq, start + count); +if (ret) +return ret; + + ret =vfio_msi_set_block (irq, start, count, fds); +if (ret) +vfio_msi_disable (vdev, irq); + +return ret; + } + +for (i = start; i < start + count; i++) { +if (!irq->ctx[i].trigger) +continue ; +if (flags &VFIO_IRQ_SET_DATA_NONE ) { +eventfd_signal (irq->ctx[i].trigger,1 ); + }else if (flags &VFIO_IRQ_SET_DATA_BOOL ) { +u8 *bools = data; + +if (bools[i - start]) +eventfd_signal (irq->ctx[i].trigger,1 ); + } + } + +return 0 ; +} +int vfio_platform_set_irqs_ioctl (struct vfio_platform_device *vdev,uint32_t flags,unsigned index,unsigned start,unsigned count,void *data)@@ -261,16 +452,29 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev, unsigned start, unsigned count, uint32_t flags, void *data) = NULL; -switch (flags &VFIO_IRQ_SET_ACTION_TYPE_MASK ) { -case VFIO_IRQ_SET_ACTION_MASK : - func = vfio_platform_set_irq_mask; -break ; -case VFIO_IRQ_SET_ACTION_UNMASK : - func = vfio_platform_set_irq_unmask; -break ; -case VFIO_IRQ_SET_ACTION_TRIGGER : - func = vfio_platform_set_irq_trigger; -break ; +struct vfio_platform_irq *irq = &vdev->irqs[index]; + +if (irq->type ==VFIO_IRQ_TYPE_MSI ) { +switch (flags &VFIO_IRQ_SET_ACTION_TYPE_MASK ) { +case VFIO_IRQ_SET_ACTION_MASK : +case VFIO_IRQ_SET_ACTION_UNMASK : +break ; +case VFIO_IRQ_SET_ACTION_TRIGGER : + func = vfio_set_msi_trigger; +break ; + } + }else { +switch (flags &VFIO_IRQ_SET_ACTION_TYPE_MASK ) { +case VFIO_IRQ_SET_ACTION_MASK : + func = vfio_platform_set_irq_mask; +break ; +case VFIO_IRQ_SET_ACTION_UNMASK : + func = vfio_platform_set_irq_unmask; +break ; +case VFIO_IRQ_SET_ACTION_TRIGGER : + func = vfio_platform_set_irq_trigger; +break ; + } }if (!func)@@ -281,12 +485,21 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,int vfio_platform_irq_init (struct vfio_platform_device *vdev) { -int cnt =0 , i; +int i; +int cnt =0 ; +int num_irqs; +struct device *dev = vdev->device;while (vdev->get_irq (vdev, cnt) >=0 ) cnt++; - vdev->irqs =kcalloc (cnt,sizeof (struct vfio_platform_irq ),GFP_KERNEL ); + num_irqs = cnt; + +if (dev->msi_domain) + num_irqs++; + + vdev->irqs =kcalloc (num_irqs,sizeof (struct vfio_platform_irq ), +GFP_KERNEL );if (!vdev->irqs)return -ENOMEM ;drivers/vfio/platform/vfio_platform_irq.c int vfio_platform_irq_init(struct vfio_platform_device *vdev) · line 526
Quoted 6 lines
drivers/vfio/platform/vfio_platform_irq.c +2
-1@@ -309,7 +522,19 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev) vdev->irqs[i].masked = false; } - vdev->num_irqs = cnt; +/* +* MSI block is added at last index and its an ext irq it is
Quoted 5 lines
drivers/vfio/platform/vfio_platform_irq.c int vfio_platform_irq_init(struct vfio_platform_device *vdev) · line 530
Quoted 4 lines
drivers/vfio/platform/vfio_platform_irq.c +4
-0+*/ +if (dev->msi_domain) { + vdev->irqs[i].flags =VFIO_IRQ_INFO_EVENTFD ; + vdev->irqs[i].count =NR_IRQS ;why
NR_IRQS?
Since different devices can have different numbers of MSI(s) so we
need to initialize with max possible values. Can you please suggest if this value does not seem appropriate?
Quoted 123 lines
Quoted 123 lines
drivers/vfio/platform/vfio_platform_irq.c +14
-2+ vdev->irqs[i].hwirq =0 ; + vdev->irqs[i].masked = false; + vdev->irqs[i].type =VFIO_IRQ_TYPE_MSI ; + vdev->num_ext_irqs =1 ; + } + + vdev->num_irqs = num_irqs;return 0 ; err:@@ -321,8 +546,13 @@ void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev) {int i; -for (i =0 ; i < vdev->num_irqs; i++) -vfio_set_trigger (vdev, i,-1 ,NULL ); +for (i =0 ; i < vdev->num_irqs; i++) { +if (vdev->irqs[i].type ==VFIO_IRQ_TYPE_MSI ) +vfio_set_msi_trigger (vdev, i,0 ,0 , +VFIO_IRQ_SET_DATA_NONE ,NULL ); +else +vfio_set_trigger (vdev, i,-1 ,NULL ); + } vdev->num_irqs =0 ;kfree (vdev->irqs);drivers/vfio/platform/vfio_platform_private.h +16
-0@@ -9,6 +9,7 @@#include <linux/types.h> #include <linux/interrupt.h> +#include <linux/msi.h> #define VFIO_PLATFORM_OFFSET_SHIFT 40#define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)@@ -19,9 +20,18 @@#define VFIO_PLATFORM_INDEX_TO_OFFSET (index) \ ((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT) +struct vfio_irq_ctx { +int hwirq; +char *name; +struct msi_msg msg; +struct eventfd_ctx *trigger; +}; +struct vfio_platform_irq {u32 flags;u32 count; +int num_ctx; +struct vfio_irq_ctx *ctx;int hwirq;char *name;struct eventfd_ctx *trigger;@@ -29,6 +39,11 @@ struct vfio_platform_irq {spinlock_t lock;struct virqfd *unmask;struct virqfd *mask; + +/* for extended irqs */ +u32 type; +u32 subtype; +int config_msi; };struct vfio_platform_region {@@ -46,6 +61,7 @@ struct vfio_platform_device {u32 num_regions;struct vfio_platform_irq *irqs;u32 num_irqs; +int num_ext_irqs;int refcnt;struct mutex igate;struct module *parent_module;include/uapi/linux/vfio.h +33
-0@@ -697,11 +697,54 @@ struct vfio_irq_info {#define VFIO_IRQ_INFO_MASKABLE (1 << 1)#define VFIO_IRQ_INFO_AUTOMASKED (1 << 2)#define VFIO_IRQ_INFO_NORESIZE (1 << 3) +#define VFIO_IRQ_INFO_FLAG_CAPS (1 << 4)/* Info supports caps */ __u32 index;/* IRQ index */ __u32 count;/* Number of IRQs within this index */ +__u32 cap_offset;/* Offset within info struct of first cap */ };#define VFIO_DEVICE_GET_IRQ_INFO _IO(VFIO_TYPE, VFIO_BASE + 9) +/* +* The irq type capability allows IRQs unique to a specific device or +* class of devices to be exposed. +* +* The structures below define version 1 of this capability. +*/ +#define VFIO_IRQ_INFO_CAP_TYPE 3 + +struct vfio_irq_info_cap_type { +struct vfio_info_cap_header header; +__u32 type;/* global per bus driver */ +__u32 subtype;/* type specific */ +}; + +/* +* List of IRQ types, global per bus driver. +* If you introduce a new type, please add it here. +*/ + +/* Non PCI devices having MSI(s) support */ +#define VFIO_IRQ_TYPE_MSI (1) + +/* +* The msi capability allows the user to use the msi msg to +* configure the iova for the msi configuration. +* The structures below define version 1 of this capability. +*/ +#define VFIO_IRQ_INFO_CAP_MSI_DESCS 4 + +struct vfio_irq_msi_msg { +__u32 addr_lo;
Quoted 8 lines
include/uapi/linux/vfio.h struct vfio_irq_info { · line 744
Quoted 7 lines
include/uapi/linux/vfio.h +7
-0+__u32 addr_hi; +__u32 data; +}; + +struct vfio_irq_info_cap_msi { +struct vfio_info_cap_header header; +__u32 nr_msgs;I think you should align a
__u32reserved field to have a 64b alignment
Sure.
Quoted 2 lines
include/uapi/linux/vfio.h struct vfio_irq_info { · line 745
Quoted 1 line
include/uapi/linux/vfio.h +1
-0+struct vfio_irq_msi_msg msgs[];Please can you clarify why this cap is needed versus your prior approach.
In the previous patch, the reset module was configuring the device with MSI msg/data now since the module is not available, user space needs to have this data.
With this approach userspace just needs the pairs <msg and ctx > associated with the MSI(s) and it can choose to configure the MSI(s) sources accordingly.
Let me know if this approach does not look appropriate.
Thanks,
Vikas
Quoted 10 lines
include/uapi/linux/vfio.h struct vfio_irq_info { · line 750
Quoted 6 lines
include/uapi/linux/vfio.h +2
-0+}; +/** * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set) * Thanks
Eric
Auger Eric
maintainerRe: [RFC v2 1/1] vfio/platform: add support for msi
Hi Vikas,
Quoted 458 lines · Vikas Gupta
Hi Eric,
Quoted 61 lines · Auger Eric
Hi Vikas,
Quoted 40 lines · Vikas Gupta
MSI support for platform devices.
Signed-off-by Vikas Gupta <[email protected]>--- drivers/vfio/platform/vfio_platform_common.c | 99 ++++++- drivers/vfio/platform/vfio_platform_irq.c | 260 +++++++++++++++++- drivers/vfio/platform/vfio_platform_private.h | 16 ++ include/uapi/linux/vfio.h | 43 +++ 4 files changed, 401 insertions(+), 17 deletions(-)drivers/vfio/platform/vfio_platform_common.c +8
-0@@ -16,6 +16,7 @@#include <linux/types.h> #include <linux/uaccess.h> #include <linux/vfio.h> +#include <linux/nospec.h> #include "vfio_platform_private.h" @@ -344,9 +345,16 @@ static long vfio_platform_ioctl(void *device_data, }else if (cmd ==VFIO_DEVICE_GET_IRQ_INFO ) {struct vfio_irq_info info; +struct vfio_info_cap caps = { .buf =NULL , .size =0 }; +struct vfio_irq_info_cap_msi *msi_info =NULL ; +unsigned long capsz; +int ext_irq_index = vdev->num_irqs - vdev->num_ext_irqs; minsz = offsetofend(struct vfio_irq_info, count); +/* For backward compatibility, cannot require this */ + capsz = offsetofend(struct vfio_irq_info, cap_offset); +if (copy_from_user (&info, (void __user *)arg, minsz))return -EFAULT ;drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 370
Quoted 7 lines
drivers/vfio/platform/vfio_platform_common.c +4
-0@@ -356,9 +364,89 @@ static long vfio_platform_ioctl(void *device_data,if (info.index >= vdev->num_irqs)return -EINVAL ; +if (info.argsz >= capsz) + minsz = capsz; + +if (info.index == ext_irq_index) {nit: n case we add new ext indices afterwards, I would check info.index
-ext_irq_indexagainst anVFIO_EXT_IRQ_MSIdefine.drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 378
Quoted 8 lines
drivers/vfio/platform/vfio_platform_common.c +8
-0+struct vfio_irq_info_cap_type cap_type = { + .header.id =VFIO_IRQ_INFO_CAP_TYPE , + .header.version =1 }; +int i; +int ret; +int num_msgs; +size_t msi_info_size; +struct vfio_platform_irq *irq;Quoted 2 lines
nit: I think generally the opposite order (length) is chosen. This also would better match the existing style in this file
Ok will fix it
Quoted 8 lines
drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 385
Quoted 7 lines
drivers/vfio/platform/vfio_platform_common.c +7
-0+ + info.index =array_index_nospec (info.index, + vdev->num_irqs); + + irq = &vdev->irqs[info.index]; + + info.flags = irq->flags;I think this can be removed given [*]
Sure.
Quoted 11 lines
drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 395
Quoted 10 lines
drivers/vfio/platform/vfio_platform_common.c +10
-0+ cap_type.type = irq->type; + cap_type.subtype = irq->subtype; + + ret =vfio_info_add_capability (&caps, + &cap_type.header, +sizeof (cap_type)); +if (ret) +return ret; + + num_msgs = irq->num_ctx;do would want to return the cap even if !num_ctx?
If
num_ctx= 0 thenVFIO_IRQ_INFO_CAP_MSI_DESCSshould not be written. I`ll take care of same.Quoted 24 lines
Quoted 24 lines
drivers/vfio/platform/vfio_platform_common.c +24
-0+ + msi_info_size =struct_size (msi_info, msgs, num_msgs); + + msi_info =kzalloc (msi_info_size,GFP_KERNEL ); +if (!msi_info) { +kfree (caps.buf); +return -ENOMEM ; + } + + msi_info->header.id =VFIO_IRQ_INFO_CAP_MSI_DESCS ; + msi_info->header.version =1 ; + msi_info->nr_msgs = num_msgs; + +for (i =0 ; i < num_msgs; i++) { +struct vfio_irq_ctx *ctx = &irq->ctx[i]; + + msi_info->msgs[i].addr_lo = ctx->msg.address_lo; + msi_info->msgs[i].addr_hi = ctx->msg.address_hi; + msi_info->msgs[i].data = ctx->msg.data; + } + + ret =vfio_info_add_capability (&caps, &msi_info->header, + msi_info_size); +if (ret) {Quoted 8 lines
drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 426
Quoted 7 lines
drivers/vfio/platform/vfio_platform_common.c +6
-0+kfree (msi_info); +kfree (caps.buf); +return ret; + } + } + info.flags = vdev->irqs[info.index].flags;[*]
Will fix it.
Quoted 336 lines
Quoted 329 lines
drivers/vfio/platform/vfio_platform_common.c +30
-2info.count = vdev->irqs[info.index].count; +if (caps.size) { + info.flags |=VFIO_IRQ_INFO_FLAG_CAPS ; +if (info.argsz <sizeof (info) + caps.size) { + info.argsz =sizeof (info) + caps.size; + info.cap_offset =0 ; + }else { +vfio_info_cap_shift (&caps,sizeof (info)); +if (copy_to_user ((void __user *)arg + +sizeof (info), caps.buf, + caps.size)) { +kfree (msi_info); +kfree (caps.buf); +return -EFAULT ; + } + info.cap_offset =sizeof (info); + } + +kfree (msi_info); +kfree (caps.buf); + } +return copy_to_user ((void __user *)arg, &info, minsz) ? -EFAULT :0 ;@@ -366,6 +454,7 @@ static long vfio_platform_ioctl(void *device_data,struct vfio_irq_set hdr;u8 *data =NULL ;int ret =0 ; +int max;size_t data_size =0 ; minsz = offsetofend(struct vfio_irq_set, count);@@ -373,8 +462,14 @@ static long vfio_platform_ioctl(void *device_data,if (copy_from_user (&hdr, (void __user *)arg, minsz))return -EFAULT ; - ret =vfio_set_irqs_validate_and_prepare (&hdr, vdev->num_irqs, - vdev->num_irqs, &data_size); +if (hdr.index >= vdev->num_irqs) +return -EINVAL ; + + max = vdev->irqs[hdr.index].count; + + ret =vfio_set_irqs_validate_and_prepare (&hdr, max, + vdev->num_irqs, + &data_size);if (ret)return ret;drivers/vfio/platform/vfio_platform_irq.c +225
-12@@ -8,10 +8,12 @@#include <linux/eventfd.h> #include <linux/interrupt.h> +#include <linux/eventfd.h> #include <linux/slab.h> #include <linux/types.h> #include <linux/vfio.h> #include <linux/irq.h> +#include <linux/msi.h> #include "vfio_platform_private.h" @@ -253,6 +255,195 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,return 0 ; } +/* MSI/MSIX */ +static irqreturn_t vfio_msihandler (int irq,void *arg) +{ +struct eventfd_ctx *trigger = arg; + +eventfd_signal (trigger,1 ); +return IRQ_HANDLED ; +} + +static void msi_write (struct msi_desc *desc,struct msi_msg *msg) +{ +int i; +struct vfio_platform_irq *irq; +u16 index = desc->platform.msi_index; +struct device *dev =msi_desc_to_dev (desc); +struct vfio_device *device =dev_get_drvdata (dev); +struct vfio_platform_device *vdev = (struct vfio_platform_device *) +vfio_device_data (device); + +for (i =0 ; i < vdev->num_irqs; i++) +if (vdev->irqs[i].type ==VFIO_IRQ_TYPE_MSI ) + irq = &vdev->irqs[i]; + + irq->ctx[index].msg.address_lo = msg->address_lo; + irq->ctx[index].msg.address_hi = msg->address_hi; + irq->ctx[index].msg.data = msg->data; +} + +static int vfio_msi_enable (struct vfio_platform_device *vdev, +struct vfio_platform_irq *irq,int nvec) +{ +int ret; +int msi_idx =0 ; +struct msi_desc *desc; +struct device *dev = vdev->device; + + irq->ctx =kcalloc (nvec,sizeof (struct vfio_irq_ctx ),GFP_KERNEL ); +if (!irq->ctx) +return -ENOMEM ; + +/* Allocate platform MSIs */ + ret =platform_msi_domain_alloc_irqs (dev, nvec, msi_write); +if (ret <0 ) { +kfree (irq->ctx); +return ret; + } + +for_each_msi_entry (desc, dev) { + irq->ctx[msi_idx].hwirq = desc->irq; + msi_idx++; + } + + irq->num_ctx = nvec; + irq->config_msi =1 ; + +return 0 ; +} + +static int vfio_msi_set_vector_signal (struct vfio_platform_irq *irq, +int vector,int fd) +{ +struct eventfd_ctx *trigger; +int irq_num, ret; + +if (vector <0 || vector >= irq->num_ctx) +return -EINVAL ; + + irq_num = irq->ctx[vector].hwirq; + +if (irq->ctx[vector].trigger) { +free_irq (irq_num, irq->ctx[vector].trigger); +kfree (irq->ctx[vector].name); +eventfd_ctx_put (irq->ctx[vector].trigger); + irq->ctx[vector].trigger =NULL ; + } + +if (fd <0 ) +return 0 ; + + irq->ctx[vector].name =kasprintf (GFP_KERNEL , +"vfio-msi[%d]" , vector); +if (!irq->ctx[vector].name) +return -ENOMEM ; + + trigger =eventfd_ctx_fdget (fd); +if (IS_ERR (trigger)) { +kfree (irq->ctx[vector].name); +return PTR_ERR (trigger); + } + + ret =request_irq (irq_num, vfio_msihandler,0 , + irq->ctx[vector].name, trigger); +if (ret) { +kfree (irq->ctx[vector].name); +eventfd_ctx_put (trigger); +return ret; + } + + irq->ctx[vector].trigger = trigger; + +return 0 ; +} + +static int vfio_msi_set_block (struct vfio_platform_irq *irq,unsigned int start, +unsigned int count,int32_t *fds) +{ +int i, j, ret =0 ; + +if (start >= irq->num_ctx || start + count > irq->num_ctx) +return -EINVAL ; + +for (i =0 , j = start; i < count && !ret; i++, j++) { +int fd = fds ? fds[i] :-1 ; + + ret =vfio_msi_set_vector_signal (irq, j, fd); + } + +if (ret) { +for (--j; j >= (int )start; j--) +vfio_msi_set_vector_signal (irq, j,-1 ); + } + +return ret; +} + +static void vfio_msi_disable (struct vfio_platform_device *vdev, +struct vfio_platform_irq *irq) +{ +struct device *dev = vdev->device; + +vfio_msi_set_block (irq,0 , irq->num_ctx,NULL ); + +platform_msi_domain_free_irqs (dev); + + irq->config_msi =0 ; + irq->num_ctx =0 ; + +kfree (irq->ctx); +} + +static int vfio_set_msi_trigger (struct vfio_platform_device *vdev, +unsigned int index,unsigned int start, +unsigned int count,uint32_t flags,void *data) +{ +int i; +struct vfio_platform_irq *irq = &vdev->irqs[index]; + +if (start + count > irq->count) +return -EINVAL ; + +if (!count && (flags &VFIO_IRQ_SET_DATA_NONE )) { +vfio_msi_disable (vdev, irq); +return 0 ; + } + +if (flags &VFIO_IRQ_SET_DATA_EVENTFD ) { +s32 *fds = data; +int ret; + +if (irq->config_msi) +return vfio_msi_set_block (irq, start, count, + fds); + ret =vfio_msi_enable (vdev, irq, start + count); +if (ret) +return ret; + + ret =vfio_msi_set_block (irq, start, count, fds); +if (ret) +vfio_msi_disable (vdev, irq); + +return ret; + } + +for (i = start; i < start + count; i++) { +if (!irq->ctx[i].trigger) +continue ; +if (flags &VFIO_IRQ_SET_DATA_NONE ) { +eventfd_signal (irq->ctx[i].trigger,1 ); + }else if (flags &VFIO_IRQ_SET_DATA_BOOL ) { +u8 *bools = data; + +if (bools[i - start]) +eventfd_signal (irq->ctx[i].trigger,1 ); + } + } + +return 0 ; +} +int vfio_platform_set_irqs_ioctl (struct vfio_platform_device *vdev,uint32_t flags,unsigned index,unsigned start,unsigned count,void *data)@@ -261,16 +452,29 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev, unsigned start, unsigned count, uint32_t flags, void *data) = NULL; -switch (flags &VFIO_IRQ_SET_ACTION_TYPE_MASK ) { -case VFIO_IRQ_SET_ACTION_MASK : - func = vfio_platform_set_irq_mask; -break ; -case VFIO_IRQ_SET_ACTION_UNMASK : - func = vfio_platform_set_irq_unmask; -break ; -case VFIO_IRQ_SET_ACTION_TRIGGER : - func = vfio_platform_set_irq_trigger; -break ; +struct vfio_platform_irq *irq = &vdev->irqs[index]; + +if (irq->type ==VFIO_IRQ_TYPE_MSI ) { +switch (flags &VFIO_IRQ_SET_ACTION_TYPE_MASK ) { +case VFIO_IRQ_SET_ACTION_MASK : +case VFIO_IRQ_SET_ACTION_UNMASK : +break ; +case VFIO_IRQ_SET_ACTION_TRIGGER : + func = vfio_set_msi_trigger; +break ; + } + }else { +switch (flags &VFIO_IRQ_SET_ACTION_TYPE_MASK ) { +case VFIO_IRQ_SET_ACTION_MASK : + func = vfio_platform_set_irq_mask; +break ; +case VFIO_IRQ_SET_ACTION_UNMASK : + func = vfio_platform_set_irq_unmask; +break ; +case VFIO_IRQ_SET_ACTION_TRIGGER : + func = vfio_platform_set_irq_trigger; +break ; + } }if (!func)@@ -281,12 +485,21 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,int vfio_platform_irq_init (struct vfio_platform_device *vdev) { -int cnt =0 , i; +int i; +int cnt =0 ; +int num_irqs; +struct device *dev = vdev->device;while (vdev->get_irq (vdev, cnt) >=0 ) cnt++; - vdev->irqs =kcalloc (cnt,sizeof (struct vfio_platform_irq ),GFP_KERNEL ); + num_irqs = cnt; + +if (dev->msi_domain) + num_irqs++; + + vdev->irqs =kcalloc (num_irqs,sizeof (struct vfio_platform_irq ), +GFP_KERNEL );if (!vdev->irqs)return -ENOMEM ;drivers/vfio/platform/vfio_platform_irq.c int vfio_platform_irq_init(struct vfio_platform_device *vdev) · line 526
Quoted 6 lines
drivers/vfio/platform/vfio_platform_irq.c +2
-1@@ -309,7 +522,19 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev) vdev->irqs[i].masked = false; } - vdev->num_irqs = cnt; +/* +* MSI block is added at last index and its an ext irq it is
Quoted 8 lines
Quoted 5 lines
drivers/vfio/platform/vfio_platform_irq.c int vfio_platform_irq_init(struct vfio_platform_device *vdev) · line 530
Quoted 4 lines
drivers/vfio/platform/vfio_platform_irq.c +4
-0+*/ +if (dev->msi_domain) { + vdev->irqs[i].flags =VFIO_IRQ_INFO_EVENTFD ; + vdev->irqs[i].count =NR_IRQS ;why
NR_IRQS?Since different devices can have different numbers of MSI(s) so we
need to initialize with max possible values. Can you please suggest if this value does not seem appropriate?
As opposed to PCIe, the userspace has no real way to guess how many vectors can be set (what vfio_pci_get_irq_count does). This also means we do not fully implement the original API as we are not able to report an accurate value for .count. How will the user determine how many vectors he can use?
Quoted 133 lines
Quoted 123 lines
Quoted 123 lines
drivers/vfio/platform/vfio_platform_irq.c +14
-2+ vdev->irqs[i].hwirq =0 ; + vdev->irqs[i].masked = false; + vdev->irqs[i].type =VFIO_IRQ_TYPE_MSI ; + vdev->num_ext_irqs =1 ; + } + + vdev->num_irqs = num_irqs;return 0 ; err:@@ -321,8 +546,13 @@ void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev) {int i; -for (i =0 ; i < vdev->num_irqs; i++) -vfio_set_trigger (vdev, i,-1 ,NULL ); +for (i =0 ; i < vdev->num_irqs; i++) { +if (vdev->irqs[i].type ==VFIO_IRQ_TYPE_MSI ) +vfio_set_msi_trigger (vdev, i,0 ,0 , +VFIO_IRQ_SET_DATA_NONE ,NULL ); +else +vfio_set_trigger (vdev, i,-1 ,NULL ); + } vdev->num_irqs =0 ;kfree (vdev->irqs);drivers/vfio/platform/vfio_platform_private.h +16
-0@@ -9,6 +9,7 @@#include <linux/types.h> #include <linux/interrupt.h> +#include <linux/msi.h> #define VFIO_PLATFORM_OFFSET_SHIFT 40#define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)@@ -19,9 +20,18 @@#define VFIO_PLATFORM_INDEX_TO_OFFSET (index) \ ((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT) +struct vfio_irq_ctx { +int hwirq; +char *name; +struct msi_msg msg; +struct eventfd_ctx *trigger; +}; +struct vfio_platform_irq {u32 flags;u32 count; +int num_ctx; +struct vfio_irq_ctx *ctx;int hwirq;char *name;struct eventfd_ctx *trigger;@@ -29,6 +39,11 @@ struct vfio_platform_irq {spinlock_t lock;struct virqfd *unmask;struct virqfd *mask; + +/* for extended irqs */ +u32 type; +u32 subtype; +int config_msi; };struct vfio_platform_region {@@ -46,6 +61,7 @@ struct vfio_platform_device {u32 num_regions;struct vfio_platform_irq *irqs;u32 num_irqs; +int num_ext_irqs;int refcnt;struct mutex igate;struct module *parent_module;include/uapi/linux/vfio.h +33
-0@@ -697,11 +697,54 @@ struct vfio_irq_info {#define VFIO_IRQ_INFO_MASKABLE (1 << 1)#define VFIO_IRQ_INFO_AUTOMASKED (1 << 2)#define VFIO_IRQ_INFO_NORESIZE (1 << 3) +#define VFIO_IRQ_INFO_FLAG_CAPS (1 << 4)/* Info supports caps */ __u32 index;/* IRQ index */ __u32 count;/* Number of IRQs within this index */ +__u32 cap_offset;/* Offset within info struct of first cap */ };#define VFIO_DEVICE_GET_IRQ_INFO _IO(VFIO_TYPE, VFIO_BASE + 9) +/* +* The irq type capability allows IRQs unique to a specific device or +* class of devices to be exposed. +* +* The structures below define version 1 of this capability. +*/ +#define VFIO_IRQ_INFO_CAP_TYPE 3 + +struct vfio_irq_info_cap_type { +struct vfio_info_cap_header header; +__u32 type;/* global per bus driver */ +__u32 subtype;/* type specific */ +}; + +/* +* List of IRQ types, global per bus driver. +* If you introduce a new type, please add it here. +*/ + +/* Non PCI devices having MSI(s) support */ +#define VFIO_IRQ_TYPE_MSI (1) + +/* +* The msi capability allows the user to use the msi msg to +* configure the iova for the msi configuration. +* The structures below define version 1 of this capability. +*/ +#define VFIO_IRQ_INFO_CAP_MSI_DESCS 4 + +struct vfio_irq_msi_msg { +__u32 addr_lo;Quoted 8 lines
include/uapi/linux/vfio.h struct vfio_irq_info { · line 744
Quoted 7 lines
include/uapi/linux/vfio.h +7
-0+__u32 addr_hi; +__u32 data; +}; + +struct vfio_irq_info_cap_msi { +struct vfio_info_cap_header header; +__u32 nr_msgs;I think you should align a
__u32reserved field to have a 64b alignmentSure.
Quoted 1 line
Quoted 1 line
include/uapi/linux/vfio.h +1
-0+struct vfio_irq_msi_msg msgs[];
Quoted 8 lines
Quoted 1 line
Please can you clarify why this cap is needed versus your prior approach.
In the previous patch, the reset module was configuring the device with MSI msg/data now since the module is not available, user space needs to have this data.
With this approach userspace just needs the pairs <msg and ctx > associated with the MSI(s) and it can choose to configure the MSI(s) sources accordingly.
Let me know if this approach does not look appropriate.
This comes to the question I asked in my previous email, ie. could you give some more info about the expected MSI setup sequence? May be the opportunity to enhance the commit message ;-)
Thanks
Eric
Quoted 13 lines
Thanks,
VikasQuoted 10 lines
include/uapi/linux/vfio.h struct vfio_irq_info { · line 750
Quoted 6 lines
include/uapi/linux/vfio.h +2
-0+}; +/** * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set) * Thanks
Eric
Re: [RFC v2 1/1] vfio/platform: add support for msi
HI Eric,
Quoted 467 lines · Auger Eric
Hi Vikas,
Quoted 463 lines · Vikas Gupta
Hi Eric,
Quoted 61 lines · Auger Eric
Hi Vikas,
Quoted 40 lines · Vikas Gupta
MSI support for platform devices.
Signed-off-by Vikas Gupta <[email protected]>--- drivers/vfio/platform/vfio_platform_common.c | 99 ++++++- drivers/vfio/platform/vfio_platform_irq.c | 260 +++++++++++++++++- drivers/vfio/platform/vfio_platform_private.h | 16 ++ include/uapi/linux/vfio.h | 43 +++ 4 files changed, 401 insertions(+), 17 deletions(-)drivers/vfio/platform/vfio_platform_common.c +8
-0@@ -16,6 +16,7 @@#include <linux/types.h> #include <linux/uaccess.h> #include <linux/vfio.h> +#include <linux/nospec.h> #include "vfio_platform_private.h" @@ -344,9 +345,16 @@ static long vfio_platform_ioctl(void *device_data, }else if (cmd ==VFIO_DEVICE_GET_IRQ_INFO ) {struct vfio_irq_info info; +struct vfio_info_cap caps = { .buf =NULL , .size =0 }; +struct vfio_irq_info_cap_msi *msi_info =NULL ; +unsigned long capsz; +int ext_irq_index = vdev->num_irqs - vdev->num_ext_irqs; minsz = offsetofend(struct vfio_irq_info, count); +/* For backward compatibility, cannot require this */ + capsz = offsetofend(struct vfio_irq_info, cap_offset); +if (copy_from_user (&info, (void __user *)arg, minsz))return -EFAULT ;drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 370
Quoted 7 lines
drivers/vfio/platform/vfio_platform_common.c +4
-0@@ -356,9 +364,89 @@ static long vfio_platform_ioctl(void *device_data,if (info.index >= vdev->num_irqs)return -EINVAL ; +if (info.argsz >= capsz) + minsz = capsz; + +if (info.index == ext_irq_index) {nit: n case we add new ext indices afterwards, I would check info.index
-ext_irq_indexagainst anVFIO_EXT_IRQ_MSIdefine.drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 378
Quoted 8 lines
drivers/vfio/platform/vfio_platform_common.c +8
-0+struct vfio_irq_info_cap_type cap_type = { + .header.id =VFIO_IRQ_INFO_CAP_TYPE , + .header.version =1 }; +int i; +int ret; +int num_msgs; +size_t msi_info_size; +struct vfio_platform_irq *irq;Quoted 2 lines
nit: I think generally the opposite order (length) is chosen. This also would better match the existing style in this file
Ok will fix it
Quoted 8 lines
drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 385
Quoted 7 lines
drivers/vfio/platform/vfio_platform_common.c +7
-0+ + info.index =array_index_nospec (info.index, + vdev->num_irqs); + + irq = &vdev->irqs[info.index]; + + info.flags = irq->flags;I think this can be removed given [*]
Sure.
Quoted 11 lines
drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 395
Quoted 10 lines
drivers/vfio/platform/vfio_platform_common.c +10
-0+ cap_type.type = irq->type; + cap_type.subtype = irq->subtype; + + ret =vfio_info_add_capability (&caps, + &cap_type.header, +sizeof (cap_type)); +if (ret) +return ret; + + num_msgs = irq->num_ctx;do would want to return the cap even if !num_ctx?
If
num_ctx= 0 thenVFIO_IRQ_INFO_CAP_MSI_DESCSshould not be written. I`ll take care of same.Quoted 24 lines
Quoted 24 lines
drivers/vfio/platform/vfio_platform_common.c +24
-0+ + msi_info_size =struct_size (msi_info, msgs, num_msgs); + + msi_info =kzalloc (msi_info_size,GFP_KERNEL ); +if (!msi_info) { +kfree (caps.buf); +return -ENOMEM ; + } + + msi_info->header.id =VFIO_IRQ_INFO_CAP_MSI_DESCS ; + msi_info->header.version =1 ; + msi_info->nr_msgs = num_msgs; + +for (i =0 ; i < num_msgs; i++) { +struct vfio_irq_ctx *ctx = &irq->ctx[i]; + + msi_info->msgs[i].addr_lo = ctx->msg.address_lo; + msi_info->msgs[i].addr_hi = ctx->msg.address_hi; + msi_info->msgs[i].data = ctx->msg.data; + } + + ret =vfio_info_add_capability (&caps, &msi_info->header, + msi_info_size); +if (ret) {Quoted 8 lines
drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 426
Quoted 7 lines
drivers/vfio/platform/vfio_platform_common.c +6
-0+kfree (msi_info); +kfree (caps.buf); +return ret; + } + } + info.flags = vdev->irqs[info.index].flags;[*]
Will fix it.
Quoted 341 lines
Quoted 329 lines
drivers/vfio/platform/vfio_platform_common.c +30
-2info.count = vdev->irqs[info.index].count; +if (caps.size) { + info.flags |=VFIO_IRQ_INFO_FLAG_CAPS ; +if (info.argsz <sizeof (info) + caps.size) { + info.argsz =sizeof (info) + caps.size; + info.cap_offset =0 ; + }else { +vfio_info_cap_shift (&caps,sizeof (info)); +if (copy_to_user ((void __user *)arg + +sizeof (info), caps.buf, + caps.size)) { +kfree (msi_info); +kfree (caps.buf); +return -EFAULT ; + } + info.cap_offset =sizeof (info); + } + +kfree (msi_info); +kfree (caps.buf); + } +return copy_to_user ((void __user *)arg, &info, minsz) ? -EFAULT :0 ;@@ -366,6 +454,7 @@ static long vfio_platform_ioctl(void *device_data,struct vfio_irq_set hdr;u8 *data =NULL ;int ret =0 ; +int max;size_t data_size =0 ; minsz = offsetofend(struct vfio_irq_set, count);@@ -373,8 +462,14 @@ static long vfio_platform_ioctl(void *device_data,if (copy_from_user (&hdr, (void __user *)arg, minsz))return -EFAULT ; - ret =vfio_set_irqs_validate_and_prepare (&hdr, vdev->num_irqs, - vdev->num_irqs, &data_size); +if (hdr.index >= vdev->num_irqs) +return -EINVAL ; + + max = vdev->irqs[hdr.index].count; + + ret =vfio_set_irqs_validate_and_prepare (&hdr, max, + vdev->num_irqs, + &data_size);if (ret)return ret;drivers/vfio/platform/vfio_platform_irq.c +225
-12@@ -8,10 +8,12 @@#include <linux/eventfd.h> #include <linux/interrupt.h> +#include <linux/eventfd.h> #include <linux/slab.h> #include <linux/types.h> #include <linux/vfio.h> #include <linux/irq.h> +#include <linux/msi.h> #include "vfio_platform_private.h" @@ -253,6 +255,195 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,return 0 ; } +/* MSI/MSIX */ +static irqreturn_t vfio_msihandler (int irq,void *arg) +{ +struct eventfd_ctx *trigger = arg; + +eventfd_signal (trigger,1 ); +return IRQ_HANDLED ; +} + +static void msi_write (struct msi_desc *desc,struct msi_msg *msg) +{ +int i; +struct vfio_platform_irq *irq; +u16 index = desc->platform.msi_index; +struct device *dev =msi_desc_to_dev (desc); +struct vfio_device *device =dev_get_drvdata (dev); +struct vfio_platform_device *vdev = (struct vfio_platform_device *) +vfio_device_data (device); + +for (i =0 ; i < vdev->num_irqs; i++) +if (vdev->irqs[i].type ==VFIO_IRQ_TYPE_MSI ) + irq = &vdev->irqs[i]; + + irq->ctx[index].msg.address_lo = msg->address_lo; + irq->ctx[index].msg.address_hi = msg->address_hi; + irq->ctx[index].msg.data = msg->data; +} + +static int vfio_msi_enable (struct vfio_platform_device *vdev, +struct vfio_platform_irq *irq,int nvec) +{ +int ret; +int msi_idx =0 ; +struct msi_desc *desc; +struct device *dev = vdev->device; + + irq->ctx =kcalloc (nvec,sizeof (struct vfio_irq_ctx ),GFP_KERNEL ); +if (!irq->ctx) +return -ENOMEM ; + +/* Allocate platform MSIs */ + ret =platform_msi_domain_alloc_irqs (dev, nvec, msi_write); +if (ret <0 ) { +kfree (irq->ctx); +return ret; + } + +for_each_msi_entry (desc, dev) { + irq->ctx[msi_idx].hwirq = desc->irq; + msi_idx++; + } + + irq->num_ctx = nvec; + irq->config_msi =1 ; + +return 0 ; +} + +static int vfio_msi_set_vector_signal (struct vfio_platform_irq *irq, +int vector,int fd) +{ +struct eventfd_ctx *trigger; +int irq_num, ret; + +if (vector <0 || vector >= irq->num_ctx) +return -EINVAL ; + + irq_num = irq->ctx[vector].hwirq; + +if (irq->ctx[vector].trigger) { +free_irq (irq_num, irq->ctx[vector].trigger); +kfree (irq->ctx[vector].name); +eventfd_ctx_put (irq->ctx[vector].trigger); + irq->ctx[vector].trigger =NULL ; + } + +if (fd <0 ) +return 0 ; + + irq->ctx[vector].name =kasprintf (GFP_KERNEL , +"vfio-msi[%d]" , vector); +if (!irq->ctx[vector].name) +return -ENOMEM ; + + trigger =eventfd_ctx_fdget (fd); +if (IS_ERR (trigger)) { +kfree (irq->ctx[vector].name); +return PTR_ERR (trigger); + } + + ret =request_irq (irq_num, vfio_msihandler,0 , + irq->ctx[vector].name, trigger); +if (ret) { +kfree (irq->ctx[vector].name); +eventfd_ctx_put (trigger); +return ret; + } + + irq->ctx[vector].trigger = trigger; + +return 0 ; +} + +static int vfio_msi_set_block (struct vfio_platform_irq *irq,unsigned int start, +unsigned int count,int32_t *fds) +{ +int i, j, ret =0 ; + +if (start >= irq->num_ctx || start + count > irq->num_ctx) +return -EINVAL ; + +for (i =0 , j = start; i < count && !ret; i++, j++) { +int fd = fds ? fds[i] :-1 ; + + ret =vfio_msi_set_vector_signal (irq, j, fd); + } + +if (ret) { +for (--j; j >= (int )start; j--) +vfio_msi_set_vector_signal (irq, j,-1 ); + } + +return ret; +} + +static void vfio_msi_disable (struct vfio_platform_device *vdev, +struct vfio_platform_irq *irq) +{ +struct device *dev = vdev->device; + +vfio_msi_set_block (irq,0 , irq->num_ctx,NULL ); + +platform_msi_domain_free_irqs (dev); + + irq->config_msi =0 ; + irq->num_ctx =0 ; + +kfree (irq->ctx); +} + +static int vfio_set_msi_trigger (struct vfio_platform_device *vdev, +unsigned int index,unsigned int start, +unsigned int count,uint32_t flags,void *data) +{ +int i; +struct vfio_platform_irq *irq = &vdev->irqs[index]; + +if (start + count > irq->count) +return -EINVAL ; + +if (!count && (flags &VFIO_IRQ_SET_DATA_NONE )) { +vfio_msi_disable (vdev, irq); +return 0 ; + } + +if (flags &VFIO_IRQ_SET_DATA_EVENTFD ) { +s32 *fds = data; +int ret; + +if (irq->config_msi) +return vfio_msi_set_block (irq, start, count, + fds); + ret =vfio_msi_enable (vdev, irq, start + count); +if (ret) +return ret; + + ret =vfio_msi_set_block (irq, start, count, fds); +if (ret) +vfio_msi_disable (vdev, irq); + +return ret; + } + +for (i = start; i < start + count; i++) { +if (!irq->ctx[i].trigger) +continue ; +if (flags &VFIO_IRQ_SET_DATA_NONE ) { +eventfd_signal (irq->ctx[i].trigger,1 ); + }else if (flags &VFIO_IRQ_SET_DATA_BOOL ) { +u8 *bools = data; + +if (bools[i - start]) +eventfd_signal (irq->ctx[i].trigger,1 ); + } + } + +return 0 ; +} +int vfio_platform_set_irqs_ioctl (struct vfio_platform_device *vdev,uint32_t flags,unsigned index,unsigned start,unsigned count,void *data)@@ -261,16 +452,29 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev, unsigned start, unsigned count, uint32_t flags, void *data) = NULL; -switch (flags &VFIO_IRQ_SET_ACTION_TYPE_MASK ) { -case VFIO_IRQ_SET_ACTION_MASK : - func = vfio_platform_set_irq_mask; -break ; -case VFIO_IRQ_SET_ACTION_UNMASK : - func = vfio_platform_set_irq_unmask; -break ; -case VFIO_IRQ_SET_ACTION_TRIGGER : - func = vfio_platform_set_irq_trigger; -break ; +struct vfio_platform_irq *irq = &vdev->irqs[index]; + +if (irq->type ==VFIO_IRQ_TYPE_MSI ) { +switch (flags &VFIO_IRQ_SET_ACTION_TYPE_MASK ) { +case VFIO_IRQ_SET_ACTION_MASK : +case VFIO_IRQ_SET_ACTION_UNMASK : +break ; +case VFIO_IRQ_SET_ACTION_TRIGGER : + func = vfio_set_msi_trigger; +break ; + } + }else { +switch (flags &VFIO_IRQ_SET_ACTION_TYPE_MASK ) { +case VFIO_IRQ_SET_ACTION_MASK : + func = vfio_platform_set_irq_mask; +break ; +case VFIO_IRQ_SET_ACTION_UNMASK : + func = vfio_platform_set_irq_unmask; +break ; +case VFIO_IRQ_SET_ACTION_TRIGGER : + func = vfio_platform_set_irq_trigger; +break ; + } }if (!func)@@ -281,12 +485,21 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,int vfio_platform_irq_init (struct vfio_platform_device *vdev) { -int cnt =0 , i; +int i; +int cnt =0 ; +int num_irqs; +struct device *dev = vdev->device;while (vdev->get_irq (vdev, cnt) >=0 ) cnt++; - vdev->irqs =kcalloc (cnt,sizeof (struct vfio_platform_irq ),GFP_KERNEL ); + num_irqs = cnt; + +if (dev->msi_domain) + num_irqs++; + + vdev->irqs =kcalloc (num_irqs,sizeof (struct vfio_platform_irq ), +GFP_KERNEL );if (!vdev->irqs)return -ENOMEM ;drivers/vfio/platform/vfio_platform_irq.c int vfio_platform_irq_init(struct vfio_platform_device *vdev) · line 526
Quoted 6 lines
drivers/vfio/platform/vfio_platform_irq.c +2
-1@@ -309,7 +522,19 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev) vdev->irqs[i].masked = false; } - vdev->num_irqs = cnt; +/* +* MSI block is added at last index and its an ext irq it is
drivers/vfio/platform/vfio_platform_irq.c int vfio_platform_irq_init(struct vfio_platform_device *vdev) · line 530
Quoted 4 lines
drivers/vfio/platform/vfio_platform_irq.c +4
-0+*/ +if (dev->msi_domain) { + vdev->irqs[i].flags =VFIO_IRQ_INFO_EVENTFD ; + vdev->irqs[i].count =NR_IRQS ;why
NR_IRQS?
Quoted 8 lines
Quoted 3 lines
Since different devices can have different numbers of MSI(s) so we
need to initialize with max possible values. Can you please suggest if this value does not seem appropriate?As opposed to PCIe, the userspace has no real way to guess how many vectors can be set (what
vfio_pci_get_irq_countdoes). This also means we do not fully implement the original API as we are not able to report an accurate value for .count. How will the user determine how many vectors he can use?
I believe user space will know how many MSIs platform device can have. We have assigned NR_IRQS because that’s what maximum number of interrupt can be supported on a specific platform. In case user space errantly tries to allocate any number of MSIs in kernel platform_msi_domain_alloc_irqs should fail.
Do you think this approach can create an issue as .count is not exactly the same as MSIs are existing with device? Is it necessary to have a PCIe kind of approach as PCIe is standard but platform devices do not have standards?
Quoted 141 lines
Quoted 133 lines
Quoted 123 lines
Quoted 123 lines
drivers/vfio/platform/vfio_platform_irq.c +14
-2+ vdev->irqs[i].hwirq =0 ; + vdev->irqs[i].masked = false; + vdev->irqs[i].type =VFIO_IRQ_TYPE_MSI ; + vdev->num_ext_irqs =1 ; + } + + vdev->num_irqs = num_irqs;return 0 ; err:@@ -321,8 +546,13 @@ void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev) {int i; -for (i =0 ; i < vdev->num_irqs; i++) -vfio_set_trigger (vdev, i,-1 ,NULL ); +for (i =0 ; i < vdev->num_irqs; i++) { +if (vdev->irqs[i].type ==VFIO_IRQ_TYPE_MSI ) +vfio_set_msi_trigger (vdev, i,0 ,0 , +VFIO_IRQ_SET_DATA_NONE ,NULL ); +else +vfio_set_trigger (vdev, i,-1 ,NULL ); + } vdev->num_irqs =0 ;kfree (vdev->irqs);drivers/vfio/platform/vfio_platform_private.h +16
-0@@ -9,6 +9,7 @@#include <linux/types.h> #include <linux/interrupt.h> +#include <linux/msi.h> #define VFIO_PLATFORM_OFFSET_SHIFT 40#define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)@@ -19,9 +20,18 @@#define VFIO_PLATFORM_INDEX_TO_OFFSET (index) \ ((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT) +struct vfio_irq_ctx { +int hwirq; +char *name; +struct msi_msg msg; +struct eventfd_ctx *trigger; +}; +struct vfio_platform_irq {u32 flags;u32 count; +int num_ctx; +struct vfio_irq_ctx *ctx;int hwirq;char *name;struct eventfd_ctx *trigger;@@ -29,6 +39,11 @@ struct vfio_platform_irq {spinlock_t lock;struct virqfd *unmask;struct virqfd *mask; + +/* for extended irqs */ +u32 type; +u32 subtype; +int config_msi; };struct vfio_platform_region {@@ -46,6 +61,7 @@ struct vfio_platform_device {u32 num_regions;struct vfio_platform_irq *irqs;u32 num_irqs; +int num_ext_irqs;int refcnt;struct mutex igate;struct module *parent_module;include/uapi/linux/vfio.h +33
-0@@ -697,11 +697,54 @@ struct vfio_irq_info {#define VFIO_IRQ_INFO_MASKABLE (1 << 1)#define VFIO_IRQ_INFO_AUTOMASKED (1 << 2)#define VFIO_IRQ_INFO_NORESIZE (1 << 3) +#define VFIO_IRQ_INFO_FLAG_CAPS (1 << 4)/* Info supports caps */ __u32 index;/* IRQ index */ __u32 count;/* Number of IRQs within this index */ +__u32 cap_offset;/* Offset within info struct of first cap */ };#define VFIO_DEVICE_GET_IRQ_INFO _IO(VFIO_TYPE, VFIO_BASE + 9) +/* +* The irq type capability allows IRQs unique to a specific device or +* class of devices to be exposed. +* +* The structures below define version 1 of this capability. +*/ +#define VFIO_IRQ_INFO_CAP_TYPE 3 + +struct vfio_irq_info_cap_type { +struct vfio_info_cap_header header; +__u32 type;/* global per bus driver */ +__u32 subtype;/* type specific */ +}; + +/* +* List of IRQ types, global per bus driver. +* If you introduce a new type, please add it here. +*/ + +/* Non PCI devices having MSI(s) support */ +#define VFIO_IRQ_TYPE_MSI (1) + +/* +* The msi capability allows the user to use the msi msg to +* configure the iova for the msi configuration. +* The structures below define version 1 of this capability. +*/ +#define VFIO_IRQ_INFO_CAP_MSI_DESCS 4 + +struct vfio_irq_msi_msg { +__u32 addr_lo;Quoted 8 lines
include/uapi/linux/vfio.h struct vfio_irq_info { · line 744
Quoted 7 lines
include/uapi/linux/vfio.h +7
-0+__u32 addr_hi; +__u32 data; +}; + +struct vfio_irq_info_cap_msi { +struct vfio_info_cap_header header; +__u32 nr_msgs;I think you should align a
__u32reserved field to have a 64b alignmentSure.
Quoted 1 line
Quoted 1 line
include/uapi/linux/vfio.h +1
-0+struct vfio_irq_msi_msg msgs[];Quoted 8 lines
Quoted 1 line
Please can you clarify why this cap is needed versus your prior approach.
In the previous patch, the reset module was configuring the device with MSI msg/data now since the module is not available, user space needs to have this data.
With this approach userspace just needs the pairs <msg and ctx > associated with the MSI(s) and it can choose to configure the MSI(s) sources accordingly.
Let me know if this approach does not look appropriate.
Quoted 3 lines
This comes to the question I asked in my previous email, ie. could you give some more info about the expected MSI setup sequence? May be the opportunity to enhance the commit message ;-)
With our proposal, user space needs to know how many MSI sources there are as we want user space to map msi-msg(s) to specific MSI sources. Let us assume there are max ‘n’ MSI sources a platform device have, user-space requests ‘n’ msi(s) allocation in the kernel which in turn gets ‘n’ msi_msg with help of cap (VFIO_IRQ_INFO_CAP_MSI_DESCS). User space is free to map/configure msi_msg->data value to any particular source. Since msi_msg->data is one to one mapped to eventfd, user-space knows that particular eventfd corresponds to the same MSI source which it has configured with the msi_msg->data value.
Steps:
a) User space allocates nvec in kernel. Ioctl VFIO_DEVICE_SET_IRQS b) Kernel allocates nvec and does associated IRQ allocation, eventfd etc. c) User space gets the msi_msgs VFIO_DEVICE_GET_IRQ_INFO. d) User space configures the particular MSI source with msi_msg info.
Do these steps look OK?
We can add more information where this cap is introduced in the file uapi/linux/vfio.h.
Thanks,
Vikas
Quoted 18 lines
Thanks
Eric
Quoted 13 lines
Thanks,
VikasQuoted 10 lines
include/uapi/linux/vfio.h struct vfio_irq_info { · line 750
Quoted 6 lines
include/uapi/linux/vfio.h +2
-0+}; +/** * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set) * Thanks
Eric
Signature · 11 lines
This electronic communication and the information and any files transmitted with it, or attached to it, are confidential and are intended solely for the use of the individual or entity to whom it is addressed and may contain information that is confidential, legally privileged, protected by privacy laws, or otherwise restricted from disclosure to anyone else. If you are not the intended recipient or the person responsible for delivering the e-mail to the intended recipient, you are hereby notified that any use, copying, distributing, dissemination, forwarding, printing, or copying of this e-mail is strictly prohibited. If you received this e-mail in error, please return the e-mail to the sender, delete it from your computer, and destroy any printed copy of it.
Auger Eric
maintainerRe: [RFC v2 1/1] vfio/platform: add support for msi
Hi Vikas,
Quoted 479 lines · Vikas Gupta
HI Eric,
Quoted 467 lines · Auger Eric
Hi Vikas,
Quoted 463 lines · Vikas Gupta
Hi Eric,
Quoted 61 lines · Auger Eric
Hi Vikas,
Quoted 40 lines · Vikas Gupta
MSI support for platform devices.
Signed-off-by Vikas Gupta <[email protected]>--- drivers/vfio/platform/vfio_platform_common.c | 99 ++++++- drivers/vfio/platform/vfio_platform_irq.c | 260 +++++++++++++++++- drivers/vfio/platform/vfio_platform_private.h | 16 ++ include/uapi/linux/vfio.h | 43 +++ 4 files changed, 401 insertions(+), 17 deletions(-)drivers/vfio/platform/vfio_platform_common.c +8
-0@@ -16,6 +16,7 @@#include <linux/types.h> #include <linux/uaccess.h> #include <linux/vfio.h> +#include <linux/nospec.h> #include "vfio_platform_private.h" @@ -344,9 +345,16 @@ static long vfio_platform_ioctl(void *device_data, }else if (cmd ==VFIO_DEVICE_GET_IRQ_INFO ) {struct vfio_irq_info info; +struct vfio_info_cap caps = { .buf =NULL , .size =0 }; +struct vfio_irq_info_cap_msi *msi_info =NULL ; +unsigned long capsz; +int ext_irq_index = vdev->num_irqs - vdev->num_ext_irqs; minsz = offsetofend(struct vfio_irq_info, count); +/* For backward compatibility, cannot require this */ + capsz = offsetofend(struct vfio_irq_info, cap_offset); +if (copy_from_user (&info, (void __user *)arg, minsz))return -EFAULT ;drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 370
Quoted 7 lines
drivers/vfio/platform/vfio_platform_common.c +4
-0@@ -356,9 +364,89 @@ static long vfio_platform_ioctl(void *device_data,if (info.index >= vdev->num_irqs)return -EINVAL ; +if (info.argsz >= capsz) + minsz = capsz; + +if (info.index == ext_irq_index) {nit: n case we add new ext indices afterwards, I would check info.index
-ext_irq_indexagainst anVFIO_EXT_IRQ_MSIdefine.drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 378
Quoted 8 lines
drivers/vfio/platform/vfio_platform_common.c +8
-0+struct vfio_irq_info_cap_type cap_type = { + .header.id =VFIO_IRQ_INFO_CAP_TYPE , + .header.version =1 }; +int i; +int ret; +int num_msgs; +size_t msi_info_size; +struct vfio_platform_irq *irq;Quoted 2 lines
nit: I think generally the opposite order (length) is chosen. This also would better match the existing style in this file
Ok will fix it
Quoted 8 lines
drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 385
Quoted 7 lines
drivers/vfio/platform/vfio_platform_common.c +7
-0+ + info.index =array_index_nospec (info.index, + vdev->num_irqs); + + irq = &vdev->irqs[info.index]; + + info.flags = irq->flags;I think this can be removed given [*]
Sure.
Quoted 11 lines
drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 395
Quoted 10 lines
drivers/vfio/platform/vfio_platform_common.c +10
-0+ cap_type.type = irq->type; + cap_type.subtype = irq->subtype; + + ret =vfio_info_add_capability (&caps, + &cap_type.header, +sizeof (cap_type)); +if (ret) +return ret; + + num_msgs = irq->num_ctx;do would want to return the cap even if !num_ctx?
If
num_ctx= 0 thenVFIO_IRQ_INFO_CAP_MSI_DESCSshould not be written. I`ll take care of same.Quoted 24 lines
Quoted 24 lines
drivers/vfio/platform/vfio_platform_common.c +24
-0+ + msi_info_size =struct_size (msi_info, msgs, num_msgs); + + msi_info =kzalloc (msi_info_size,GFP_KERNEL ); +if (!msi_info) { +kfree (caps.buf); +return -ENOMEM ; + } + + msi_info->header.id =VFIO_IRQ_INFO_CAP_MSI_DESCS ; + msi_info->header.version =1 ; + msi_info->nr_msgs = num_msgs; + +for (i =0 ; i < num_msgs; i++) { +struct vfio_irq_ctx *ctx = &irq->ctx[i]; + + msi_info->msgs[i].addr_lo = ctx->msg.address_lo; + msi_info->msgs[i].addr_hi = ctx->msg.address_hi; + msi_info->msgs[i].data = ctx->msg.data; + } + + ret =vfio_info_add_capability (&caps, &msi_info->header, + msi_info_size); +if (ret) {Quoted 8 lines
drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 426
Quoted 7 lines
drivers/vfio/platform/vfio_platform_common.c +6
-0+kfree (msi_info); +kfree (caps.buf); +return ret; + } + } + info.flags = vdev->irqs[info.index].flags;[*]
Will fix it.
Quoted 341 lines
Quoted 329 lines
drivers/vfio/platform/vfio_platform_common.c +30
-2info.count = vdev->irqs[info.index].count; +if (caps.size) { + info.flags |=VFIO_IRQ_INFO_FLAG_CAPS ; +if (info.argsz <sizeof (info) + caps.size) { + info.argsz =sizeof (info) + caps.size; + info.cap_offset =0 ; + }else { +vfio_info_cap_shift (&caps,sizeof (info)); +if (copy_to_user ((void __user *)arg + +sizeof (info), caps.buf, + caps.size)) { +kfree (msi_info); +kfree (caps.buf); +return -EFAULT ; + } + info.cap_offset =sizeof (info); + } + +kfree (msi_info); +kfree (caps.buf); + } +return copy_to_user ((void __user *)arg, &info, minsz) ? -EFAULT :0 ;@@ -366,6 +454,7 @@ static long vfio_platform_ioctl(void *device_data,struct vfio_irq_set hdr;u8 *data =NULL ;int ret =0 ; +int max;size_t data_size =0 ; minsz = offsetofend(struct vfio_irq_set, count);@@ -373,8 +462,14 @@ static long vfio_platform_ioctl(void *device_data,if (copy_from_user (&hdr, (void __user *)arg, minsz))return -EFAULT ; - ret =vfio_set_irqs_validate_and_prepare (&hdr, vdev->num_irqs, - vdev->num_irqs, &data_size); +if (hdr.index >= vdev->num_irqs) +return -EINVAL ; + + max = vdev->irqs[hdr.index].count; + + ret =vfio_set_irqs_validate_and_prepare (&hdr, max, + vdev->num_irqs, + &data_size);if (ret)return ret;drivers/vfio/platform/vfio_platform_irq.c +225
-12@@ -8,10 +8,12 @@#include <linux/eventfd.h> #include <linux/interrupt.h> +#include <linux/eventfd.h> #include <linux/slab.h> #include <linux/types.h> #include <linux/vfio.h> #include <linux/irq.h> +#include <linux/msi.h> #include "vfio_platform_private.h" @@ -253,6 +255,195 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,return 0 ; } +/* MSI/MSIX */ +static irqreturn_t vfio_msihandler (int irq,void *arg) +{ +struct eventfd_ctx *trigger = arg; + +eventfd_signal (trigger,1 ); +return IRQ_HANDLED ; +} + +static void msi_write (struct msi_desc *desc,struct msi_msg *msg) +{ +int i; +struct vfio_platform_irq *irq; +u16 index = desc->platform.msi_index; +struct device *dev =msi_desc_to_dev (desc); +struct vfio_device *device =dev_get_drvdata (dev); +struct vfio_platform_device *vdev = (struct vfio_platform_device *) +vfio_device_data (device); + +for (i =0 ; i < vdev->num_irqs; i++) +if (vdev->irqs[i].type ==VFIO_IRQ_TYPE_MSI ) + irq = &vdev->irqs[i]; + + irq->ctx[index].msg.address_lo = msg->address_lo; + irq->ctx[index].msg.address_hi = msg->address_hi; + irq->ctx[index].msg.data = msg->data; +} + +static int vfio_msi_enable (struct vfio_platform_device *vdev, +struct vfio_platform_irq *irq,int nvec) +{ +int ret; +int msi_idx =0 ; +struct msi_desc *desc; +struct device *dev = vdev->device; + + irq->ctx =kcalloc (nvec,sizeof (struct vfio_irq_ctx ),GFP_KERNEL ); +if (!irq->ctx) +return -ENOMEM ; + +/* Allocate platform MSIs */ + ret =platform_msi_domain_alloc_irqs (dev, nvec, msi_write); +if (ret <0 ) { +kfree (irq->ctx); +return ret; + } + +for_each_msi_entry (desc, dev) { + irq->ctx[msi_idx].hwirq = desc->irq; + msi_idx++; + } + + irq->num_ctx = nvec; + irq->config_msi =1 ; + +return 0 ; +} + +static int vfio_msi_set_vector_signal (struct vfio_platform_irq *irq, +int vector,int fd) +{ +struct eventfd_ctx *trigger; +int irq_num, ret; + +if (vector <0 || vector >= irq->num_ctx) +return -EINVAL ; + + irq_num = irq->ctx[vector].hwirq; + +if (irq->ctx[vector].trigger) { +free_irq (irq_num, irq->ctx[vector].trigger); +kfree (irq->ctx[vector].name); +eventfd_ctx_put (irq->ctx[vector].trigger); + irq->ctx[vector].trigger =NULL ; + } + +if (fd <0 ) +return 0 ; + + irq->ctx[vector].name =kasprintf (GFP_KERNEL , +"vfio-msi[%d]" , vector); +if (!irq->ctx[vector].name) +return -ENOMEM ; + + trigger =eventfd_ctx_fdget (fd); +if (IS_ERR (trigger)) { +kfree (irq->ctx[vector].name); +return PTR_ERR (trigger); + } + + ret =request_irq (irq_num, vfio_msihandler,0 , + irq->ctx[vector].name, trigger); +if (ret) { +kfree (irq->ctx[vector].name); +eventfd_ctx_put (trigger); +return ret; + } + + irq->ctx[vector].trigger = trigger; + +return 0 ; +} + +static int vfio_msi_set_block (struct vfio_platform_irq *irq,unsigned int start, +unsigned int count,int32_t *fds) +{ +int i, j, ret =0 ; + +if (start >= irq->num_ctx || start + count > irq->num_ctx) +return -EINVAL ; + +for (i =0 , j = start; i < count && !ret; i++, j++) { +int fd = fds ? fds[i] :-1 ; + + ret =vfio_msi_set_vector_signal (irq, j, fd); + } + +if (ret) { +for (--j; j >= (int )start; j--) +vfio_msi_set_vector_signal (irq, j,-1 ); + } + +return ret; +} + +static void vfio_msi_disable (struct vfio_platform_device *vdev, +struct vfio_platform_irq *irq) +{ +struct device *dev = vdev->device; + +vfio_msi_set_block (irq,0 , irq->num_ctx,NULL ); + +platform_msi_domain_free_irqs (dev); + + irq->config_msi =0 ; + irq->num_ctx =0 ; + +kfree (irq->ctx); +} + +static int vfio_set_msi_trigger (struct vfio_platform_device *vdev, +unsigned int index,unsigned int start, +unsigned int count,uint32_t flags,void *data) +{ +int i; +struct vfio_platform_irq *irq = &vdev->irqs[index]; + +if (start + count > irq->count) +return -EINVAL ; + +if (!count && (flags &VFIO_IRQ_SET_DATA_NONE )) { +vfio_msi_disable (vdev, irq); +return 0 ; + } + +if (flags &VFIO_IRQ_SET_DATA_EVENTFD ) { +s32 *fds = data; +int ret; + +if (irq->config_msi) +return vfio_msi_set_block (irq, start, count, + fds); + ret =vfio_msi_enable (vdev, irq, start + count); +if (ret) +return ret; + + ret =vfio_msi_set_block (irq, start, count, fds); +if (ret) +vfio_msi_disable (vdev, irq); + +return ret; + } + +for (i = start; i < start + count; i++) { +if (!irq->ctx[i].trigger) +continue ; +if (flags &VFIO_IRQ_SET_DATA_NONE ) { +eventfd_signal (irq->ctx[i].trigger,1 ); + }else if (flags &VFIO_IRQ_SET_DATA_BOOL ) { +u8 *bools = data; + +if (bools[i - start]) +eventfd_signal (irq->ctx[i].trigger,1 ); + } + } + +return 0 ; +} +int vfio_platform_set_irqs_ioctl (struct vfio_platform_device *vdev,uint32_t flags,unsigned index,unsigned start,unsigned count,void *data)@@ -261,16 +452,29 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev, unsigned start, unsigned count, uint32_t flags, void *data) = NULL; -switch (flags &VFIO_IRQ_SET_ACTION_TYPE_MASK ) { -case VFIO_IRQ_SET_ACTION_MASK : - func = vfio_platform_set_irq_mask; -break ; -case VFIO_IRQ_SET_ACTION_UNMASK : - func = vfio_platform_set_irq_unmask; -break ; -case VFIO_IRQ_SET_ACTION_TRIGGER : - func = vfio_platform_set_irq_trigger; -break ; +struct vfio_platform_irq *irq = &vdev->irqs[index]; + +if (irq->type ==VFIO_IRQ_TYPE_MSI ) { +switch (flags &VFIO_IRQ_SET_ACTION_TYPE_MASK ) { +case VFIO_IRQ_SET_ACTION_MASK : +case VFIO_IRQ_SET_ACTION_UNMASK : +break ; +case VFIO_IRQ_SET_ACTION_TRIGGER : + func = vfio_set_msi_trigger; +break ; + } + }else { +switch (flags &VFIO_IRQ_SET_ACTION_TYPE_MASK ) { +case VFIO_IRQ_SET_ACTION_MASK : + func = vfio_platform_set_irq_mask; +break ; +case VFIO_IRQ_SET_ACTION_UNMASK : + func = vfio_platform_set_irq_unmask; +break ; +case VFIO_IRQ_SET_ACTION_TRIGGER : + func = vfio_platform_set_irq_trigger; +break ; + } }if (!func)@@ -281,12 +485,21 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,int vfio_platform_irq_init (struct vfio_platform_device *vdev) { -int cnt =0 , i; +int i; +int cnt =0 ; +int num_irqs; +struct device *dev = vdev->device;while (vdev->get_irq (vdev, cnt) >=0 ) cnt++; - vdev->irqs =kcalloc (cnt,sizeof (struct vfio_platform_irq ),GFP_KERNEL ); + num_irqs = cnt; + +if (dev->msi_domain) + num_irqs++; + + vdev->irqs =kcalloc (num_irqs,sizeof (struct vfio_platform_irq ), +GFP_KERNEL );if (!vdev->irqs)return -ENOMEM ;drivers/vfio/platform/vfio_platform_irq.c int vfio_platform_irq_init(struct vfio_platform_device *vdev) · line 526
Quoted 6 lines
drivers/vfio/platform/vfio_platform_irq.c +2
-1@@ -309,7 +522,19 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev) vdev->irqs[i].masked = false; } - vdev->num_irqs = cnt; +/* +* MSI block is added at last index and its an ext irq it is
drivers/vfio/platform/vfio_platform_irq.c int vfio_platform_irq_init(struct vfio_platform_device *vdev) · line 530
Quoted 4 lines
drivers/vfio/platform/vfio_platform_irq.c +4
-0+*/ +if (dev->msi_domain) { + vdev->irqs[i].flags =VFIO_IRQ_INFO_EVENTFD ; + vdev->irqs[i].count =NR_IRQS ;why
NR_IRQS?Quoted 8 lines
Quoted 3 lines
Since different devices can have different numbers of MSI(s) so we
need to initialize with max possible values. Can you please suggest if this value does not seem appropriate?As opposed to PCIe, the userspace has no real way to guess how many vectors can be set (what
vfio_pci_get_irq_countdoes). This also means we do not fully implement the original API as we are not able to report an accurate value for .count. How will the user determine how many vectors he can use?
Quoted 9 lines
I believe user space will know how many MSIs platform device can have. We have assigned
NR_IRQSbecause that’s what maximum number of interrupt can be supported on a specific platform. In case user space errantly tries to allocate any number of MSIs in kernelplatform_msi_domain_alloc_irqsshould fail.
Do you think this approach can create an issue as .count is not exactly the same as MSIs are existing with device? Is it necessary to have a PCIe kind of approach as PCIe is standard but platform devices do not have standards?
I had a short discussion about this with Alex and I think we came to the conclusion that is an issue not to be able to report accurately the number of supported vectors. Assuming that the userspace knows the number of vectors does not really look as a valid assumption and does not really fit the vfio spirit.
The previous approach beased on a kernel module allowed to report their actual number. Even if it is heavier it allows to fill the info supposed to be returned by the GET_IRQ ioctl and we think it is better.
Thanks
Eric
Quoted 186 lines
Quoted 142 lines
Quoted 134 lines
Quoted 124 lines
Quoted 124 lines
drivers/vfio/platform/vfio_platform_irq.c +14
-2+ vdev->irqs[i].hwirq =0 ; + vdev->irqs[i].masked = false; + vdev->irqs[i].type =VFIO_IRQ_TYPE_MSI ; + vdev->num_ext_irqs =1 ; + } + + vdev->num_irqs = num_irqs;return 0 ; err:@@ -321,8 +546,13 @@ void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev) {int i; -for (i =0 ; i < vdev->num_irqs; i++) -vfio_set_trigger (vdev, i,-1 ,NULL ); +for (i =0 ; i < vdev->num_irqs; i++) { +if (vdev->irqs[i].type ==VFIO_IRQ_TYPE_MSI ) +vfio_set_msi_trigger (vdev, i,0 ,0 , +VFIO_IRQ_SET_DATA_NONE ,NULL ); +else +vfio_set_trigger (vdev, i,-1 ,NULL ); + } vdev->num_irqs =0 ;kfree (vdev->irqs);drivers/vfio/platform/vfio_platform_private.h +16
-0@@ -9,6 +9,7 @@#include <linux/types.h> #include <linux/interrupt.h> +#include <linux/msi.h> #define VFIO_PLATFORM_OFFSET_SHIFT 40#define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)@@ -19,9 +20,18 @@#define VFIO_PLATFORM_INDEX_TO_OFFSET (index) \ ((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT) +struct vfio_irq_ctx { +int hwirq; +char *name; +struct msi_msg msg; +struct eventfd_ctx *trigger; +}; +struct vfio_platform_irq {u32 flags;u32 count; +int num_ctx; +struct vfio_irq_ctx *ctx;int hwirq;char *name;struct eventfd_ctx *trigger;@@ -29,6 +39,11 @@ struct vfio_platform_irq {spinlock_t lock;struct virqfd *unmask;struct virqfd *mask; + +/* for extended irqs */ +u32 type; +u32 subtype; +int config_msi; };struct vfio_platform_region {@@ -46,6 +61,7 @@ struct vfio_platform_device {u32 num_regions;struct vfio_platform_irq *irqs;u32 num_irqs; +int num_ext_irqs;int refcnt;struct mutex igate;struct module *parent_module;include/uapi/linux/vfio.h +33
-0@@ -697,11 +697,54 @@ struct vfio_irq_info {#define VFIO_IRQ_INFO_MASKABLE (1 << 1)#define VFIO_IRQ_INFO_AUTOMASKED (1 << 2)#define VFIO_IRQ_INFO_NORESIZE (1 << 3) +#define VFIO_IRQ_INFO_FLAG_CAPS (1 << 4)/* Info supports caps */ __u32 index;/* IRQ index */ __u32 count;/* Number of IRQs within this index */ +__u32 cap_offset;/* Offset within info struct of first cap */ };#define VFIO_DEVICE_GET_IRQ_INFO _IO(VFIO_TYPE, VFIO_BASE + 9) +/* +* The irq type capability allows IRQs unique to a specific device or +* class of devices to be exposed. +* +* The structures below define version 1 of this capability. +*/ +#define VFIO_IRQ_INFO_CAP_TYPE 3 + +struct vfio_irq_info_cap_type { +struct vfio_info_cap_header header; +__u32 type;/* global per bus driver */ +__u32 subtype;/* type specific */ +}; + +/* +* List of IRQ types, global per bus driver. +* If you introduce a new type, please add it here. +*/ + +/* Non PCI devices having MSI(s) support */ +#define VFIO_IRQ_TYPE_MSI (1) + +/* +* The msi capability allows the user to use the msi msg to +* configure the iova for the msi configuration. +* The structures below define version 1 of this capability. +*/ +#define VFIO_IRQ_INFO_CAP_MSI_DESCS 4 + +struct vfio_irq_msi_msg { +__u32 addr_lo;Quoted 8 lines
include/uapi/linux/vfio.h struct vfio_irq_info { · line 744
Quoted 7 lines
include/uapi/linux/vfio.h +7
-0+__u32 addr_hi; +__u32 data; +}; + +struct vfio_irq_info_cap_msi { +struct vfio_info_cap_header header; +__u32 nr_msgs;I think you should align a
__u32reserved field to have a 64b alignmentSure.
Quoted 1 line
Quoted 1 line
include/uapi/linux/vfio.h +1
-0+struct vfio_irq_msi_msg msgs[];Quoted 8 lines
Quoted 1 line
Please can you clarify why this cap is needed versus your prior approach.
In the previous patch, the reset module was configuring the device with MSI msg/data now since the module is not available, user space needs to have this data.
With this approach userspace just needs the pairs <msg and ctx > associated with the MSI(s) and it can choose to configure the MSI(s) sources accordingly.
Let me know if this approach does not look appropriate.Quoted 3 lines
This comes to the question I asked in my previous email, ie. could you give some more info about the expected MSI setup sequence? May be the opportunity to enhance the commit message ;-)
With our proposal, user space needs to know how many MSI sources there are as we want user space to map msi-msg(s) to specific MSI sources. Let us assume there are max ‘n’ MSI sources a platform device have, user-space requests ‘n’ msi(s) allocation in the kernel which in turn gets ‘n’
msi_msgwith help of cap (VFIO_IRQ_INFO_CAP_MSI_DESCS). User space is free to map/configuremsi_msg->datavalue to any particular source. Sincemsi_msg->datais one to one mapped to eventfd, user-space knows that particular eventfd corresponds to the same MSI source which it has configured with themsi_msg->datavalue.Steps:
a) User space allocates nvec in kernel. Ioctl VFIO_DEVICE_SET_IRQS b) Kernel allocates nvec and does associated IRQ allocation, eventfd etc. c) User space gets the msi_msgs VFIO_DEVICE_GET_IRQ_INFO. d) User space configures the particular MSI source with msi_msg info.Do these steps look OK?
We can add more information where this cap is introduced in the fileuapi/linux/vfio.h.Thanks,
VikasQuoted 18 lines
Thanks
Eric
Quoted 13 lines
Thanks,
VikasQuoted 10 lines
include/uapi/linux/vfio.h struct vfio_irq_info { · line 750
Quoted 6 lines
include/uapi/linux/vfio.h +2
-0+}; +/** * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set) * Thanks
Eric
Auger Eric
maintainerHi Vikas,
Quoted 4 lines · Vikas Gupta
This RFC adds support for MSI for platform devices. MSI block is added as an ext irq along with the existing wired interrupt implementation.
Quoted 8 lines
Changes from:
-------------
v1 to v2:1) IRQ allocation has been implemented as below: ---------------------------- |IRQ-0|IRQ-1|....|IRQ-n|MSI| ---------------------------- MSI block has msi contexts and its implemneted
it is implemented
Quoted 9 lines
as ext irq.
2) Removed vendor specific module for msi handling so previously patch2 and patch3 are not required.
3) MSI related data is exported to userspace using 'caps'.
Please noteVFIO_IRQ_INFO_CAP_TYPEininclude/uapi/linux/vfio.himplementation
is taken from the Eric`s patch
https://patchwork.kernel.org/project/kvm/patch/[email protected]/
So do you mean that by exposing the vectors, now you do not need the msi module anymore?
Thanks
Eric
Quoted 23 lines
v0 to v1:
i) Removed MSI device flagVFIO_DEVICE_FLAGS_MSI. ii) Add MSI(s) at the end of the irq list of platform IRQs. MSI(s) with first entry of MSI block has count and flag information.
IRQ list: Allocation for IRQs + MSIs are allocated as below
Example: if there are 'n' IRQs and 'k' MSIs
-------------------------------------------------------
|IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
-------------------------------------------------------
MSI-0 will have count=k set and flags set accordingly.Vikas Gupta (1):
vfio/platform: add support for msidrivers/vfio/platform/vfio_platform_common.c | 99 ++++++- drivers/vfio/platform/vfio_platform_irq.c | 260 +++++++++++++++++- drivers/vfio/platform/vfio_platform_private.h | 16 ++ include/uapi/linux/vfio.h | 43 +++ 4 files changed, 401 insertions(+), 17 deletions(-)
HI Eric,
Quoted 21 lines · Auger Eric
Hi Vikas,
Quoted 4 lines · Vikas Gupta
This RFC adds support for MSI for platform devices. MSI block is added as an ext irq along with the existing wired interrupt implementation.
Quoted 8 lines
Changes from:
-------------
v1 to v2:1) IRQ allocation has been implemented as below: ---------------------------- |IRQ-0|IRQ-1|....|IRQ-n|MSI| ---------------------------- MSI block has msi contexts and its implemnetedit is implemented
Quoted 5 lines
as ext irq.
2) Removed vendor specific module for msi handling so previously patch2 and patch3 are not required.
Quoted 6 lines
Quoted 4 lines
3) MSI related data is exported to userspace using 'caps'.
Please noteVFIO_IRQ_INFO_CAP_TYPEininclude/uapi/linux/vfio.himplementation
is taken from the Eric`s patch
https://patchwork.kernel.org/project/kvm/patch/[email protected]/So do you mean that by exposing the vectors, now you do not need the msi module anymore?
Yes, with the support of caps we can expose the MSI related data to userspace and userspace can configure the device, which previous patches were doing in the kernel module.
Thanks,
Vikas
Quoted 29 lines
Thanks
Eric
Quoted 23 lines
v0 to v1:
i) Removed MSI device flagVFIO_DEVICE_FLAGS_MSI. ii) Add MSI(s) at the end of the irq list of platform IRQs. MSI(s) with first entry of MSI block has count and flag information.
IRQ list: Allocation for IRQs + MSIs are allocated as below
Example: if there are 'n' IRQs and 'k' MSIs
-------------------------------------------------------
|IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
-------------------------------------------------------
MSI-0 will have count=k set and flags set accordingly.Vikas Gupta (1):
vfio/platform: add support for msidrivers/vfio/platform/vfio_platform_common.c | 99 ++++++- drivers/vfio/platform/vfio_platform_irq.c | 260 +++++++++++++++++- drivers/vfio/platform/vfio_platform_private.h | 16 ++ include/uapi/linux/vfio.h | 43 +++ 4 files changed, 401 insertions(+), 17 deletions(-)
v3 cover msi support for platform devices
In reply to Vikas GuptaThis RFC adds support for MSI for platform devices. MSI block is added as an ext irq along with the existing wired interrupt implementation. The patchset exports two caps for MSI and related data to configure MSI source device.
Changes from:
-------------
v2 to v3:
1) Restored the vendor specific module to get max number of MSIs supported and .count value initialized.
2) Comments from Eric addressed.
v1 to v2:
1) IRQ allocation has been implemented as below:
----------------------------
|IRQ-0|IRQ-1|....|IRQ-n|MSI|
----------------------------
MSI block has msi contexts and its implemneted as ext irq.
2) Removed vendor specific module for msi handling so previously patch2 and patch3 are not required.
3) MSI related data is exported to userspace using 'caps'.
Please note VFIO_IRQ_INFO_CAP_TYPE in include/uapi/linux/vfio.h implementation
is taken from the Eric`s patch
https://patchwork.kernel.org/project/kvm/patch/[email protected]/
v0 to v1:
i) Removed MSI device flag VFIO_DEVICE_FLAGS_MSI. ii) Add MSI(s) at the end of the irq list of platform IRQs. MSI(s) with first entry of MSI block has count and flag information.
IRQ list: Allocation for IRQs + MSIs are allocated as below
Example: if there are 'n' IRQs and 'k' MSIs
-------------------------------------------------------
|IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
-------------------------------------------------------
MSI-0 will have count=k set and flags set accordingly.
Vikas Gupta (2):
vfio/platform: add support for msi vfio/platform: msi: add Broadcom platform devices
drivers/vfio/platform/Kconfig | 1 + drivers/vfio/platform/Makefile | 1 + drivers/vfio/platform/msi/Kconfig | 9 + drivers/vfio/platform/msi/Makefile | 2 + .../vfio/platform/msi/vfio_platform_bcmplt.c | 49 ++++ drivers/vfio/platform/vfio_platform_common.c | 179 +++++++++++- drivers/vfio/platform/vfio_platform_irq.c | 260 +++++++++++++++++- drivers/vfio/platform/vfio_platform_private.h | 32 +++ include/uapi/linux/vfio.h | 44 +++ 9 files changed, 558 insertions(+), 19 deletions(-) create mode 100644 drivers/vfio/platform/msi/Kconfig create mode 100644 drivers/vfio/platform/msi/Makefile create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
Signature · 15 lines
2.17.1 -- This electronic communication and the information and any files transmitted with it, or attached to it, are confidential and are intended solely for the use of the individual or entity to whom it is addressed and may contain information that is confidential, legally privileged, protected by privacy laws, or otherwise restricted from disclosure to anyone else. If you are not the intended recipient or the person responsible for delivering the e-mail to the intended recipient, you are hereby notified that any use, copying, distributing, dissemination, forwarding, printing, or copying of this e-mail is strictly prohibited. If you received this e-mail in error, please return the e-mail to the sender, delete it from your computer, and destroy any printed copy of it.
RFC v3 1/2 vfio/platform: add support for msi
MSI support for platform devices.The MSI block is added as an extended IRQ which exports caps VFIO_IRQ_INFO_CAP_TYPE and VFIO_IRQ_INFO_CAP_MSI_DESCS.
--- drivers/vfio/platform/vfio_platform_common.c | 179 +++++++++++- drivers/vfio/platform/vfio_platform_irq.c | 260 +++++++++++++++++- drivers/vfio/platform/vfio_platform_private.h | 32 +++ include/uapi/linux/vfio.h | 44 +++ 4 files changed, 496 insertions(+), 19 deletions(-)
drivers/vfio/platform/vfio_platform_common.c +175 -4
@@ -16,6 +16,7 @@#include <linux/types.h> #include <linux/uaccess.h> #include <linux/vfio.h> +#include <linux/nospec.h> #include "vfio_platform_private.h"
@@ -26,6 +27,8 @@#define VFIO_PLATFORM_IS_ACPI (vdev) ((vdev)->acpihid != NULL)static LIST_HEAD (reset_list); +/* devices having MSI support */ +static LIST_HEAD (msi_list);static DEFINE_MUTEX (driver_lock); static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,
@@ -47,6 +50,25 @@ static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,return reset_fn; } +static bool vfio_platform_lookup_msi (struct vfio_platform_device *vdev) +{ +bool has_msi = false; +struct vfio_platform_msi_node *iter; + +mutex_lock (&driver_lock); +list_for_each_entry (iter, &msi_list, link) { +if (!strcmp (iter->compat, vdev->compat) && +try_module_get (iter->owner)) { + vdev->msi_module = iter->owner; + vdev->of_get_msi = iter->of_get_msi; + has_msi = true; +break ; + } + } +mutex_unlock (&driver_lock); +return has_msi; +} +static int vfio_platform_acpi_probe (struct vfio_platform_device *vdev,struct device *dev) {
@@ -126,6 +148,19 @@ static int vfio_platform_get_reset(struct vfio_platform_device *vdev)return vdev->of_reset ?0 : -ENOENT ; } +static int vfio_platform_get_msi (struct vfio_platform_device *vdev) +{ +bool has_msi; + + has_msi =vfio_platform_lookup_msi (vdev); +if (!has_msi) { +request_module ("vfio-msi:%s" , vdev->compat); + has_msi =vfio_platform_lookup_msi (vdev); + } + +return has_msi ?0 : -ENOENT ; +} +static void vfio_platform_put_reset (struct vfio_platform_device *vdev) {if (VFIO_PLATFORM_IS_ACPI (vdev))
@@ -135,6 +170,12 @@ static void vfio_platform_put_reset(struct vfio_platform_device *vdev)module_put (vdev->reset_module); } +static void vfio_platform_put_msi (struct vfio_platform_device *vdev) +{ +if (vdev->of_get_msi) +module_put (vdev->msi_module); +} +static int vfio_platform_regions_init (struct vfio_platform_device *vdev) {int cnt =0 , i;
@@ -343,9 +384,17 @@ static long vfio_platform_ioctl(void *device_data, }else if (cmd ==VFIO_DEVICE_GET_IRQ_INFO ) {struct vfio_irq_info info; +struct vfio_info_cap caps = { .buf =NULL , .size =0 }; +struct vfio_irq_info_cap_msi *msi_info =NULL ; +int ext_irq_index = vdev->num_irqs - vdev->num_ext_irqs; +unsigned long capsz; +u32 index; minsz = offsetofend(struct vfio_irq_info, count); +/* For backward compatibility, cannot require this */ + capsz = offsetofend(struct vfio_irq_info, cap_offset); +if (copy_from_user (&info, (void __user *)arg, minsz))return -EFAULT ;
@@ -355,8 +404,94 @@ static long vfio_platform_ioctl(void *device_data,if (info.index >= vdev->num_irqs)return -EINVAL ; - info.flags = vdev->irqs[info.index].flags; - info.count = vdev->irqs[info.index].count; +if (info.argsz >= capsz) + minsz = capsz; + + index = info.index; + + info.flags = vdev->irqs[index].flags; + info.count = vdev->irqs[index].count; + +if (ext_irq_index - index ==VFIO_EXT_IRQ_MSI ) { +struct vfio_irq_info_cap_type cap_type = { + .header.id =VFIO_IRQ_INFO_CAP_TYPE , + .header.version =1 }; +struct vfio_platform_irq *irq; +size_t msi_info_size; +int num_msgs; +int ret; +int i; + + index =array_index_nospec (index, + vdev->num_irqs); + irq = &vdev->irqs[index]; + + cap_type.type = irq->type; + cap_type.subtype = irq->subtype; + + ret =vfio_info_add_capability (&caps, + &cap_type.header, +sizeof (cap_type)); +if (ret) +return ret; + + num_msgs = irq->num_ctx; +if (num_msgs) { + msi_info_size =struct_size (msi_info, + msgs, num_msgs); + + msi_info =kzalloc (msi_info_size,GFP_KERNEL ); +if (!msi_info) { +kfree (caps.buf); +return -ENOMEM ; + } + + msi_info->header.id = +VFIO_IRQ_INFO_CAP_MSI_DESCS ; + msi_info->header.version =1 ; + msi_info->nr_msgs = num_msgs; + +for (i =0 ; i < num_msgs; i++) { +struct vfio_irq_ctx *ctx = &irq->ctx[i]; + + msi_info->msgs[i].addr_lo = + ctx->msg.address_lo; + msi_info->msgs[i].addr_hi = + ctx->msg.address_hi; + msi_info->msgs[i].data = ctx->msg.data; + } + + ret =vfio_info_add_capability (&caps, + &msi_info->header, + msi_info_size); +if (ret) { +kfree (msi_info); +kfree (caps.buf); +return ret; + } + } + } + +if (caps.size) { + info.flags |=VFIO_IRQ_INFO_FLAG_CAPS ; +if (info.argsz <sizeof (info) + caps.size) { + info.argsz =sizeof (info) + caps.size; + info.cap_offset =0 ; + }else { +vfio_info_cap_shift (&caps,sizeof (info)); +if (copy_to_user ((void __user *)arg + +sizeof (info), caps.buf, + caps.size)) { +kfree (msi_info); +kfree (caps.buf); +return -EFAULT ; + } + info.cap_offset =sizeof (info); + } + +kfree (msi_info); +kfree (caps.buf); + }return copy_to_user ((void __user *)arg, &info, minsz) ? -EFAULT :0 ;
@@ -365,6 +500,7 @@ static long vfio_platform_ioctl(void *device_data,struct vfio_irq_set hdr;u8 *data =NULL ;int ret =0 ; +int max;size_t data_size =0 ; minsz = offsetofend(struct vfio_irq_set, count);
@@ -372,8 +508,14 @@ static long vfio_platform_ioctl(void *device_data,if (copy_from_user (&hdr, (void __user *)arg, minsz))return -EFAULT ; - ret =vfio_set_irqs_validate_and_prepare (&hdr, vdev->num_irqs, - vdev->num_irqs, &data_size); +if (hdr.index >= vdev->num_irqs) +return -EINVAL ; + + max = vdev->irqs[hdr.index].count; + + ret =vfio_set_irqs_validate_and_prepare (&hdr, max, + vdev->num_irqs, + &data_size);if (ret)return ret;
@@ -678,6 +820,10 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,return ret; } + ret =vfio_platform_get_msi (vdev); +if (ret) +dev_info (vdev->device,"msi not supported\n" ); + group =vfio_iommu_group_get (dev);if (!group) {dev_err (dev,"No IOMMU group for device %s\n" , vdev->name);
@@ -697,6 +843,7 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev, put_iommu:vfio_iommu_group_put (group, dev); put_reset: +vfio_platform_put_msi (vdev);vfio_platform_put_reset (vdev);return ret; }
@@ -744,6 +891,30 @@ void vfio_platform_unregister_reset(const char *compat, }EXPORT_SYMBOL_GPL (vfio_platform_unregister_reset); +void __vfio_platform_register_msi (struct vfio_platform_msi_node *node) +{ +mutex_lock (&driver_lock); +list_add (&node->link, &msi_list); +mutex_unlock (&driver_lock); +} +EXPORT_SYMBOL_GPL (__vfio_platform_register_msi); + +void vfio_platform_unregister_msi (const char *compat) +{ +struct vfio_platform_msi_node *iter, *temp; + +mutex_lock (&driver_lock); +list_for_each_entry_safe (iter, temp, &msi_list, link) { +if (!strcmp (iter->compat, compat)) { +list_del (&iter->link); +break ; + } + } + +mutex_unlock (&driver_lock); +} +EXPORT_SYMBOL_GPL (vfio_platform_unregister_msi); +MODULE_VERSION (DRIVER_VERSION );MODULE_LICENSE ("GPL v2" );MODULE_AUTHOR (DRIVER_AUTHOR );
drivers/vfio/platform/vfio_platform_irq.c +245 -15
304 lines not on this page · raw message
drivers/vfio/platform/vfio_platform_private.h +32 -0
@@ -9,6 +9,7 @@#include <linux/types.h> #include <linux/interrupt.h> +#include <linux/msi.h> #define VFIO_PLATFORM_OFFSET_SHIFT 40#define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
@@ -19,9 +20,21 @@#define VFIO_PLATFORM_INDEX_TO_OFFSET (index) \ ((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT) +/* IRQ index for MSI in ext IRQs */ +#define VFIO_EXT_IRQ_MSI 0 + +struct vfio_irq_ctx { +int hwirq; +char *name; +struct msi_msg msg; +struct eventfd_ctx *trigger; +}; +struct vfio_platform_irq {u32 flags;u32 count; +int num_ctx; +struct vfio_irq_ctx *ctx;int hwirq;char *name;struct eventfd_ctx *trigger;
@@ -29,6 +42,11 @@ struct vfio_platform_irq {spinlock_t lock;struct virqfd *unmask;struct virqfd *mask; + +/* for extended irqs */ +u32 type; +u32 subtype; +int config_msi; };struct vfio_platform_region {
@@ -46,12 +64,14 @@ struct vfio_platform_device {u32 num_regions;struct vfio_platform_irq *irqs;u32 num_irqs; +int num_ext_irqs;int refcnt;struct mutex igate;struct module *parent_module;const char *compat;const char *acpihid;struct module *reset_module; +struct module *msi_module;struct device *device;/*
@@ -65,11 +85,13 @@ struct vfio_platform_device { (*get_resource)(struct vfio_platform_device *vdev, int i);int (*get_irq)(struct vfio_platform_device *vdev,int i);int (*of_reset)(struct vfio_platform_device *vdev); +u32 (*of_get_msi)(struct vfio_platform_device *vdev);bool reset_required; };typedef int (*vfio_platform_reset_fn_t )(struct vfio_platform_device *vdev); +typedef u32 (*vfio_platform_get_msi_fn_t )(struct vfio_platform_device *vdev);struct vfio_platform_reset_node {struct list_head link;
@@ -78,6 +100,13 @@ struct vfio_platform_reset_node {vfio_platform_reset_fn_t of_reset; }; +struct vfio_platform_msi_node { +struct list_head link; +char *compat; +struct module *owner; +vfio_platform_get_msi_fn_t of_get_msi; +}; +extern int vfio_platform_probe_common (struct vfio_platform_device *vdev,struct device *dev);extern struct vfio_platform_device *vfio_platform_remove_common
@@ -94,6 +123,9 @@ extern int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,extern void __vfio_platform_register_reset (struct vfio_platform_reset_node *n);extern void vfio_platform_unregister_reset (const char *compat,vfio_platform_reset_fn_t fn); +void __vfio_platform_register_msi (struct vfio_platform_msi_node *n); +void vfio_platform_unregister_msi (const char *compat); +#define vfio_platform_register_reset (__compat, __reset) \ static struct vfio_platform_reset_node __reset ## _node = { \ .owner = THIS_MODULE, \
include/uapi/linux/vfio.h +44 -0
55 lines not on this page · raw message
Signature · 15 lines
2.17.1 -- This electronic communication and the information and any files transmitted with it, or attached to it, are confidential and are intended solely for the use of the individual or entity to whom it is addressed and may contain information that is confidential, legally privileged, protected by privacy laws, or otherwise restricted from disclosure to anyone else. If you are not the intended recipient or the person responsible for delivering the e-mail to the intended recipient, you are hereby notified that any use, copying, distributing, dissemination, forwarding, printing, or copying of this e-mail is strictly prohibited. If you received this e-mail in error, please return the e-mail to the sender, delete it from your computer, and destroy any printed copy of it.
Auger Eric
maintainerRe: [RFC v3 1/2] vfio/platform: add support for msi
Hi Vikas,
Quoted 25 lines · Vikas Gupta
MSI support for platform devices.The MSI block is added as an extended IRQ which exports caps
VFIO_IRQ_INFO_CAP_TYPEandVFIO_IRQ_INFO_CAP_MSI_DESCS.Signed-off-by Vikas Gupta <[email protected]>--- drivers/vfio/platform/vfio_platform_common.c | 179 +++++++++++- drivers/vfio/platform/vfio_platform_irq.c | 260 +++++++++++++++++- drivers/vfio/platform/vfio_platform_private.h | 32 +++ include/uapi/linux/vfio.h | 44 +++ 4 files changed, 496 insertions(+), 19 deletions(-)drivers/vfio/platform/vfio_platform_common.c +1
-0@@ -16,6 +16,7 @@#include <linux/types.h> #include <linux/uaccess.h> #include <linux/vfio.h> +#include <linux/nospec.h> #include "vfio_platform_private.h"
drivers/vfio/platform/vfio_platform_common.c line 30
Quoted 4 lines
drivers/vfio/platform/vfio_platform_common.c +1
-0@@ -26,6 +27,8 @@#define VFIO_PLATFORM_IS_ACPI (vdev) ((vdev)->acpihid != NULL)static LIST_HEAD (reset_list); +/* devices having MSI support */
nit: for devices using MSIs?
Quoted 121 lines
drivers/vfio/platform/vfio_platform_common.c +81
-2+static LIST_HEAD (msi_list);static DEFINE_MUTEX (driver_lock); static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,@@ -47,6 +50,25 @@ static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,return reset_fn; } +static bool vfio_platform_lookup_msi (struct vfio_platform_device *vdev) +{ +bool has_msi = false; +struct vfio_platform_msi_node *iter; + +mutex_lock (&driver_lock); +list_for_each_entry (iter, &msi_list, link) { +if (!strcmp (iter->compat, vdev->compat) && +try_module_get (iter->owner)) { + vdev->msi_module = iter->owner; + vdev->of_get_msi = iter->of_get_msi; + has_msi = true; +break ; + } + } +mutex_unlock (&driver_lock); +return has_msi; +} +static int vfio_platform_acpi_probe (struct vfio_platform_device *vdev,struct device *dev) {@@ -126,6 +148,19 @@ static int vfio_platform_get_reset(struct vfio_platform_device *vdev)return vdev->of_reset ?0 : -ENOENT ; } +static int vfio_platform_get_msi (struct vfio_platform_device *vdev) +{ +bool has_msi; + + has_msi =vfio_platform_lookup_msi (vdev); +if (!has_msi) { +request_module ("vfio-msi:%s" , vdev->compat); + has_msi =vfio_platform_lookup_msi (vdev); + } + +return has_msi ?0 : -ENOENT ; +} +static void vfio_platform_put_reset (struct vfio_platform_device *vdev) {if (VFIO_PLATFORM_IS_ACPI (vdev))@@ -135,6 +170,12 @@ static void vfio_platform_put_reset(struct vfio_platform_device *vdev)module_put (vdev->reset_module); } +static void vfio_platform_put_msi (struct vfio_platform_device *vdev) +{ +if (vdev->of_get_msi) +module_put (vdev->msi_module); +} +static int vfio_platform_regions_init (struct vfio_platform_device *vdev) {int cnt =0 , i;@@ -343,9 +384,17 @@ static long vfio_platform_ioctl(void *device_data, }else if (cmd ==VFIO_DEVICE_GET_IRQ_INFO ) {struct vfio_irq_info info; +struct vfio_info_cap caps = { .buf =NULL , .size =0 }; +struct vfio_irq_info_cap_msi *msi_info =NULL ; +int ext_irq_index = vdev->num_irqs - vdev->num_ext_irqs; +unsigned long capsz; +u32 index; minsz = offsetofend(struct vfio_irq_info, count); +/* For backward compatibility, cannot require this */ + capsz = offsetofend(struct vfio_irq_info, cap_offset); +if (copy_from_user (&info, (void __user *)arg, minsz))return -EFAULT ;@@ -355,8 +404,94 @@ static long vfio_platform_ioctl(void *device_data,if (info.index >= vdev->num_irqs)return -EINVAL ; - info.flags = vdev->irqs[info.index].flags; - info.count = vdev->irqs[info.index].count; +if (info.argsz >= capsz) + minsz = capsz; + + index = info.index; + + info.flags = vdev->irqs[index].flags; + info.count = vdev->irqs[index].count; + +if (ext_irq_index - index ==VFIO_EXT_IRQ_MSI ) { +struct vfio_irq_info_cap_type cap_type = { + .header.id =VFIO_IRQ_INFO_CAP_TYPE , + .header.version =1 }; +struct vfio_platform_irq *irq; +size_t msi_info_size; +int num_msgs; +int ret; +int i; + + index =array_index_nospec (index, + vdev->num_irqs); + irq = &vdev->irqs[index]; + + cap_type.type = irq->type; + cap_type.subtype = irq->subtype; + + ret =vfio_info_add_capability (&caps, + &cap_type.header, +sizeof (cap_type)); +if (ret) +return ret; + + num_msgs = irq->num_ctx; +if (num_msgs) { + msi_info_size = struct_size(msi_info,
drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 450
Quoted 10 lines
drivers/vfio/platform/vfio_platform_common.c +10
-0+ msgs, num_msgs); + + msi_info =kzalloc (msi_info_size,GFP_KERNEL ); +if (!msi_info) { +kfree (caps.buf); +return -ENOMEM ; + } + + msi_info->header.id = +VFIO_IRQ_INFO_CAP_MSI_DESCS ;
I thought you would remove this cap now the module is back. What is the motivation to keep it?
Thanks
Eric
Quoted 73 lines
drivers/vfio/platform/vfio_platform_common.c +53
-2+ msi_info->header.version =1 ; + msi_info->nr_msgs = num_msgs; + +for (i =0 ; i < num_msgs; i++) { +struct vfio_irq_ctx *ctx = &irq->ctx[i]; + + msi_info->msgs[i].addr_lo = + ctx->msg.address_lo; + msi_info->msgs[i].addr_hi = + ctx->msg.address_hi; + msi_info->msgs[i].data = ctx->msg.data; + } + + ret =vfio_info_add_capability (&caps, + &msi_info->header, + msi_info_size); +if (ret) { +kfree (msi_info); +kfree (caps.buf); +return ret; + } + } + } + +if (caps.size) { + info.flags |=VFIO_IRQ_INFO_FLAG_CAPS ; +if (info.argsz <sizeof (info) + caps.size) { + info.argsz =sizeof (info) + caps.size; + info.cap_offset =0 ; + }else { +vfio_info_cap_shift (&caps,sizeof (info)); +if (copy_to_user ((void __user *)arg + +sizeof (info), caps.buf, + caps.size)) { +kfree (msi_info); +kfree (caps.buf); +return -EFAULT ; + } + info.cap_offset =sizeof (info); + } + +kfree (msi_info); +kfree (caps.buf); + }return copy_to_user ((void __user *)arg, &info, minsz) ? -EFAULT :0 ;@@ -365,6 +500,7 @@ static long vfio_platform_ioctl(void *device_data,struct vfio_irq_set hdr;u8 *data =NULL ;int ret =0 ; +int max;size_t data_size =0 ; minsz = offsetofend(struct vfio_irq_set, count);@@ -372,8 +508,14 @@ static long vfio_platform_ioctl(void *device_data,if (copy_from_user (&hdr, (void __user *)arg, minsz))return -EFAULT ; - ret =vfio_set_irqs_validate_and_prepare (&hdr, vdev->num_irqs, - vdev->num_irqs, &data_size); +if (hdr.index >= vdev->num_irqs) +return -EINVAL ; + + max = vdev->irqs[hdr.index].count; + + ret =vfio_set_irqs_validate_and_prepare (&hdr, max, + vdev->num_irqs, + &data_size);if (ret)return ret;
drivers/vfio/platform/vfio_platform_common.c int vfio_platform_probe_common(struct vfio_platform_device *vdev, · line 825
Quoted 6 lines
drivers/vfio/platform/vfio_platform_common.c +3
-0@@ -678,6 +820,10 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,return ret; } + ret =vfio_platform_get_msi (vdev); +if (ret) +dev_info (vdev->device,"msi not supported\n" );
I don't think we should output this message. This is printed for amd-xgbe which does not have msi so this is misleading. I would say either the vfio_platform_get_msi() can return an actual error or we return a void?
Thanks
Eric
Quoted 517 lines
drivers/vfio/platform/vfio_platform_common.c +26
-0+ group =vfio_iommu_group_get (dev);if (!group) {dev_err (dev,"No IOMMU group for device %s\n" , vdev->name);@@ -697,6 +843,7 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev, put_iommu:vfio_iommu_group_put (group, dev); put_reset: +vfio_platform_put_msi (vdev);vfio_platform_put_reset (vdev);return ret; }@@ -744,6 +891,30 @@ void vfio_platform_unregister_reset(const char *compat, }EXPORT_SYMBOL_GPL (vfio_platform_unregister_reset); +void __vfio_platform_register_msi (struct vfio_platform_msi_node *node) +{ +mutex_lock (&driver_lock); +list_add (&node->link, &msi_list); +mutex_unlock (&driver_lock); +} +EXPORT_SYMBOL_GPL (__vfio_platform_register_msi); + +void vfio_platform_unregister_msi (const char *compat) +{ +struct vfio_platform_msi_node *iter, *temp; + +mutex_lock (&driver_lock); +list_for_each_entry_safe (iter, temp, &msi_list, link) { +if (!strcmp (iter->compat, compat)) { +list_del (&iter->link); +break ; + } + } + +mutex_unlock (&driver_lock); +} +EXPORT_SYMBOL_GPL (vfio_platform_unregister_msi); +MODULE_VERSION (DRIVER_VERSION );MODULE_LICENSE ("GPL v2" );MODULE_AUTHOR (DRIVER_AUTHOR );drivers/vfio/platform/vfio_platform_irq.c +245
-15@@ -8,10 +8,12 @@#include <linux/eventfd.h> #include <linux/interrupt.h> +#include <linux/eventfd.h> #include <linux/slab.h> #include <linux/types.h> #include <linux/vfio.h> #include <linux/irq.h> +#include <linux/msi.h> #include "vfio_platform_private.h" @@ -253,6 +255,195 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,return 0 ; } +/* MSI/MSIX */ +static irqreturn_t vfio_msihandler (int irq,void *arg) +{ +struct eventfd_ctx *trigger = arg; + +eventfd_signal (trigger,1 ); +return IRQ_HANDLED ; +} + +static void msi_write (struct msi_desc *desc,struct msi_msg *msg) +{ +int i; +struct vfio_platform_irq *irq; +u16 index = desc->platform.msi_index; +struct device *dev =msi_desc_to_dev (desc); +struct vfio_device *device =dev_get_drvdata (dev); +struct vfio_platform_device *vdev = (struct vfio_platform_device *) +vfio_device_data (device); + +for (i =0 ; i < vdev->num_irqs; i++) +if (vdev->irqs[i].type ==VFIO_IRQ_TYPE_MSI ) + irq = &vdev->irqs[i]; + + irq->ctx[index].msg.address_lo = msg->address_lo; + irq->ctx[index].msg.address_hi = msg->address_hi; + irq->ctx[index].msg.data = msg->data; +} + +static int vfio_msi_enable (struct vfio_platform_device *vdev, +struct vfio_platform_irq *irq,int nvec) +{ +int ret; +int msi_idx =0 ; +struct msi_desc *desc; +struct device *dev = vdev->device; + + irq->ctx =kcalloc (nvec,sizeof (struct vfio_irq_ctx ),GFP_KERNEL ); +if (!irq->ctx) +return -ENOMEM ; + +/* Allocate platform MSIs */ + ret =platform_msi_domain_alloc_irqs (dev, nvec, msi_write); +if (ret <0 ) { +kfree (irq->ctx); +return ret; + } + +for_each_msi_entry (desc, dev) { + irq->ctx[msi_idx].hwirq = desc->irq; + msi_idx++; + } + + irq->num_ctx = nvec; + irq->config_msi =1 ; + +return 0 ; +} + +static int vfio_msi_set_vector_signal (struct vfio_platform_irq *irq, +int vector,int fd) +{ +struct eventfd_ctx *trigger; +int irq_num, ret; + +if (vector <0 || vector >= irq->num_ctx) +return -EINVAL ; + + irq_num = irq->ctx[vector].hwirq; + +if (irq->ctx[vector].trigger) { +free_irq (irq_num, irq->ctx[vector].trigger); +kfree (irq->ctx[vector].name); +eventfd_ctx_put (irq->ctx[vector].trigger); + irq->ctx[vector].trigger =NULL ; + } + +if (fd <0 ) +return 0 ; + + irq->ctx[vector].name =kasprintf (GFP_KERNEL , +"vfio-msi[%d]" , vector); +if (!irq->ctx[vector].name) +return -ENOMEM ; + + trigger =eventfd_ctx_fdget (fd); +if (IS_ERR (trigger)) { +kfree (irq->ctx[vector].name); +return PTR_ERR (trigger); + } + + ret =request_irq (irq_num, vfio_msihandler,0 , + irq->ctx[vector].name, trigger); +if (ret) { +kfree (irq->ctx[vector].name); +eventfd_ctx_put (trigger); +return ret; + } + + irq->ctx[vector].trigger = trigger; + +return 0 ; +} + +static int vfio_msi_set_block (struct vfio_platform_irq *irq,unsigned int start, +unsigned int count,int32_t *fds) +{ +int i, j, ret =0 ; + +if (start >= irq->num_ctx || start + count > irq->num_ctx) +return -EINVAL ; + +for (i =0 , j = start; i < count && !ret; i++, j++) { +int fd = fds ? fds[i] :-1 ; + + ret =vfio_msi_set_vector_signal (irq, j, fd); + } + +if (ret) { +for (--j; j >= (int )start; j--) +vfio_msi_set_vector_signal (irq, j,-1 ); + } + +return ret; +} + +static void vfio_msi_disable (struct vfio_platform_device *vdev, +struct vfio_platform_irq *irq) +{ +struct device *dev = vdev->device; + +vfio_msi_set_block (irq,0 , irq->num_ctx,NULL ); + +platform_msi_domain_free_irqs (dev); + + irq->config_msi =0 ; + irq->num_ctx =0 ; + +kfree (irq->ctx); +} + +static int vfio_set_msi_trigger (struct vfio_platform_device *vdev, +unsigned int index,unsigned int start, +unsigned int count,uint32_t flags,void *data) +{ +int i; +struct vfio_platform_irq *irq = &vdev->irqs[index]; + +if (start + count > irq->count) +return -EINVAL ; + +if (!count && (flags &VFIO_IRQ_SET_DATA_NONE )) { +vfio_msi_disable (vdev, irq); +return 0 ; + } + +if (flags &VFIO_IRQ_SET_DATA_EVENTFD ) { +s32 *fds = data; +int ret; + +if (irq->config_msi) +return vfio_msi_set_block (irq, start, count, + fds); + ret =vfio_msi_enable (vdev, irq, start + count); +if (ret) +return ret; + + ret =vfio_msi_set_block (irq, start, count, fds); +if (ret) +vfio_msi_disable (vdev, irq); + +return ret; + } + +for (i = start; i < start + count; i++) { +if (!irq->ctx[i].trigger) +continue ; +if (flags &VFIO_IRQ_SET_DATA_NONE ) { +eventfd_signal (irq->ctx[i].trigger,1 ); + }else if (flags &VFIO_IRQ_SET_DATA_BOOL ) { +u8 *bools = data; + +if (bools[i - start]) +eventfd_signal (irq->ctx[i].trigger,1 ); + } + } + +return 0 ; +} +int vfio_platform_set_irqs_ioctl (struct vfio_platform_device *vdev,uint32_t flags,unsigned index,unsigned start,unsigned count,void *data)@@ -261,16 +452,29 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev, unsigned start, unsigned count, uint32_t flags, void *data) = NULL; -switch (flags &VFIO_IRQ_SET_ACTION_TYPE_MASK ) { -case VFIO_IRQ_SET_ACTION_MASK : - func = vfio_platform_set_irq_mask; -break ; -case VFIO_IRQ_SET_ACTION_UNMASK : - func = vfio_platform_set_irq_unmask; -break ; -case VFIO_IRQ_SET_ACTION_TRIGGER : - func = vfio_platform_set_irq_trigger; -break ; +struct vfio_platform_irq *irq = &vdev->irqs[index]; + +if (irq->type ==VFIO_IRQ_TYPE_MSI ) { +switch (flags &VFIO_IRQ_SET_ACTION_TYPE_MASK ) { +case VFIO_IRQ_SET_ACTION_MASK : +case VFIO_IRQ_SET_ACTION_UNMASK : +break ; +case VFIO_IRQ_SET_ACTION_TRIGGER : + func = vfio_set_msi_trigger; +break ; + } + }else { +switch (flags &VFIO_IRQ_SET_ACTION_TYPE_MASK ) { +case VFIO_IRQ_SET_ACTION_MASK : + func = vfio_platform_set_irq_mask; +break ; +case VFIO_IRQ_SET_ACTION_UNMASK : + func = vfio_platform_set_irq_unmask; +break ; +case VFIO_IRQ_SET_ACTION_TRIGGER : + func = vfio_platform_set_irq_trigger; +break ; + } }if (!func)@@ -281,12 +485,21 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,int vfio_platform_irq_init (struct vfio_platform_device *vdev) { -int cnt =0 , i; +int i; +int cnt =0 ; +int num_irqs; +int msi_cnt =0 ;while (vdev->get_irq (vdev, cnt) >=0 ) cnt++; - vdev->irqs =kcalloc (cnt,sizeof (struct vfio_platform_irq ),GFP_KERNEL ); +if (vdev->of_get_msi) { + msi_cnt = vdev->of_get_msi (vdev); + num_irqs++; + } + + vdev->irqs =kcalloc (num_irqs,sizeof (struct vfio_platform_irq ), +GFP_KERNEL );if (!vdev->irqs)return -ENOMEM ;@@ -309,7 +522,19 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev) vdev->irqs[i].masked = false; } - vdev->num_irqs = cnt; +/* +* MSI block is added at last index and it is an ext irq +*/ +if (msi_cnt >0 ) { + vdev->irqs[i].flags =VFIO_IRQ_INFO_EVENTFD ; + vdev->irqs[i].count = msi_cnt; + vdev->irqs[i].hwirq =0 ; + vdev->irqs[i].masked = false; + vdev->irqs[i].type =VFIO_IRQ_TYPE_MSI ; + vdev->num_ext_irqs =1 ; + } + + vdev->num_irqs = num_irqs;return 0 ; err:@@ -321,8 +546,13 @@ void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev) {int i; -for (i =0 ; i < vdev->num_irqs; i++) -vfio_set_trigger (vdev, i,-1 ,NULL ); +for (i =0 ; i < vdev->num_irqs; i++) { +if (vdev->irqs[i].type ==VFIO_IRQ_TYPE_MSI ) +vfio_set_msi_trigger (vdev, i,0 ,0 , +VFIO_IRQ_SET_DATA_NONE ,NULL ); +else +vfio_set_trigger (vdev, i,-1 ,NULL ); + } vdev->num_irqs =0 ;kfree (vdev->irqs);drivers/vfio/platform/vfio_platform_private.h +32
-0@@ -9,6 +9,7 @@#include <linux/types.h> #include <linux/interrupt.h> +#include <linux/msi.h> #define VFIO_PLATFORM_OFFSET_SHIFT 40#define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)@@ -19,9 +20,21 @@#define VFIO_PLATFORM_INDEX_TO_OFFSET (index) \ ((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT) +/* IRQ index for MSI in ext IRQs */ +#define VFIO_EXT_IRQ_MSI 0 + +struct vfio_irq_ctx { +int hwirq; +char *name; +struct msi_msg msg; +struct eventfd_ctx *trigger; +}; +struct vfio_platform_irq {u32 flags;u32 count; +int num_ctx; +struct vfio_irq_ctx *ctx;int hwirq;char *name;struct eventfd_ctx *trigger;@@ -29,6 +42,11 @@ struct vfio_platform_irq {spinlock_t lock;struct virqfd *unmask;struct virqfd *mask; + +/* for extended irqs */ +u32 type; +u32 subtype; +int config_msi; };struct vfio_platform_region {@@ -46,12 +64,14 @@ struct vfio_platform_device {u32 num_regions;struct vfio_platform_irq *irqs;u32 num_irqs; +int num_ext_irqs;int refcnt;struct mutex igate;struct module *parent_module;const char *compat;const char *acpihid;struct module *reset_module; +struct module *msi_module;struct device *device;/* @@ -65,11 +85,13 @@ struct vfio_platform_device { (*get_resource)(struct vfio_platform_device *vdev, int i);int (*get_irq)(struct vfio_platform_device *vdev,int i);int (*of_reset)(struct vfio_platform_device *vdev); +u32 (*of_get_msi)(struct vfio_platform_device *vdev);bool reset_required; };typedef int (*vfio_platform_reset_fn_t )(struct vfio_platform_device *vdev); +typedef u32 (*vfio_platform_get_msi_fn_t )(struct vfio_platform_device *vdev);struct vfio_platform_reset_node {struct list_head link;@@ -78,6 +100,13 @@ struct vfio_platform_reset_node {vfio_platform_reset_fn_t of_reset; }; +struct vfio_platform_msi_node { +struct list_head link; +char *compat; +struct module *owner; +vfio_platform_get_msi_fn_t of_get_msi; +}; +extern int vfio_platform_probe_common (struct vfio_platform_device *vdev,struct device *dev);extern struct vfio_platform_device *vfio_platform_remove_common@@ -94,6 +123,9 @@ extern int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,extern void __vfio_platform_register_reset (struct vfio_platform_reset_node *n);extern void vfio_platform_unregister_reset (const char *compat,vfio_platform_reset_fn_t fn); +void __vfio_platform_register_msi (struct vfio_platform_msi_node *n); +void vfio_platform_unregister_msi (const char *compat); +#define vfio_platform_register_reset (__compat, __reset) \ static struct vfio_platform_reset_node __reset ## _node = { \ .owner = THIS_MODULE, \include/uapi/linux/vfio.h +44
-0@@ -697,11 +697,55 @@ struct vfio_irq_info {#define VFIO_IRQ_INFO_MASKABLE (1 << 1)#define VFIO_IRQ_INFO_AUTOMASKED (1 << 2)#define VFIO_IRQ_INFO_NORESIZE (1 << 3) +#define VFIO_IRQ_INFO_FLAG_CAPS (1 << 4)/* Info supports caps */ __u32 index;/* IRQ index */ __u32 count;/* Number of IRQs within this index */ +__u32 cap_offset;/* Offset within info struct of first cap */ };#define VFIO_DEVICE_GET_IRQ_INFO _IO(VFIO_TYPE, VFIO_BASE + 9) +/* +* The irq type capability allows IRQs unique to a specific device or +* class of devices to be exposed. +* +* The structures below define version 1 of this capability. +*/ +#define VFIO_IRQ_INFO_CAP_TYPE 3 + +struct vfio_irq_info_cap_type { +struct vfio_info_cap_header header; +__u32 type;/* global per bus driver */ +__u32 subtype;/* type specific */ +}; + +/* +* List of IRQ types, global per bus driver. +* If you introduce a new type, please add it here. +*/ + +/* Non PCI devices having MSI(s) support */ +#define VFIO_IRQ_TYPE_MSI (1) + +/* +* The msi capability allows the user to use the msi msg to +* configure the iova for the msi configuration. +* The structures below define version 1 of this capability. +*/ +#define VFIO_IRQ_INFO_CAP_MSI_DESCS 4 + +struct vfio_irq_msi_msg { +__u32 addr_lo; +__u32 addr_hi; +__u32 data; +}; + +struct vfio_irq_info_cap_msi { +struct vfio_info_cap_header header; +__u32 nr_msgs; +__u32 reserved; +struct vfio_irq_msi_msg msgs[]; +}; +/** * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set) *
Re: [RFC v3 1/2] vfio/platform: add support for msi
Quoted 159 lines · Auger Eric
Hi Vikas,
Quoted 25 lines · Vikas Gupta
MSI support for platform devices.The MSI block is added as an extended IRQ which exports caps
VFIO_IRQ_INFO_CAP_TYPEandVFIO_IRQ_INFO_CAP_MSI_DESCS.Signed-off-by Vikas Gupta <[email protected]>--- drivers/vfio/platform/vfio_platform_common.c | 179 +++++++++++- drivers/vfio/platform/vfio_platform_irq.c | 260 +++++++++++++++++- drivers/vfio/platform/vfio_platform_private.h | 32 +++ include/uapi/linux/vfio.h | 44 +++ 4 files changed, 496 insertions(+), 19 deletions(-)drivers/vfio/platform/vfio_platform_common.c +1
-0@@ -16,6 +16,7 @@#include <linux/types.h> #include <linux/uaccess.h> #include <linux/vfio.h> +#include <linux/nospec.h> #include "vfio_platform_private.h" drivers/vfio/platform/vfio_platform_common.c line 30
Quoted 4 lines
drivers/vfio/platform/vfio_platform_common.c +1
-0@@ -26,6 +27,8 @@#define VFIO_PLATFORM_IS_ACPI (vdev) ((vdev)->acpihid != NULL)static LIST_HEAD (reset_list); +/* devices having MSI support */ nit: for devices using MSIs?
Quoted 125 lines
drivers/vfio/platform/vfio_platform_common.c +85
-2+static LIST_HEAD (msi_list);static DEFINE_MUTEX (driver_lock); static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,@@ -47,6 +50,25 @@ static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,return reset_fn; } +static bool vfio_platform_lookup_msi (struct vfio_platform_device *vdev) +{ +bool has_msi = false; +struct vfio_platform_msi_node *iter; + +mutex_lock (&driver_lock); +list_for_each_entry (iter, &msi_list, link) { +if (!strcmp (iter->compat, vdev->compat) && +try_module_get (iter->owner)) { + vdev->msi_module = iter->owner; + vdev->of_get_msi = iter->of_get_msi; + has_msi = true; +break ; + } + } +mutex_unlock (&driver_lock); +return has_msi; +} +static int vfio_platform_acpi_probe (struct vfio_platform_device *vdev,struct device *dev) {@@ -126,6 +148,19 @@ static int vfio_platform_get_reset(struct vfio_platform_device *vdev)return vdev->of_reset ?0 : -ENOENT ; } +static int vfio_platform_get_msi (struct vfio_platform_device *vdev) +{ +bool has_msi; + + has_msi =vfio_platform_lookup_msi (vdev); +if (!has_msi) { +request_module ("vfio-msi:%s" , vdev->compat); + has_msi =vfio_platform_lookup_msi (vdev); + } + +return has_msi ?0 : -ENOENT ; +} +static void vfio_platform_put_reset (struct vfio_platform_device *vdev) {if (VFIO_PLATFORM_IS_ACPI (vdev))@@ -135,6 +170,12 @@ static void vfio_platform_put_reset(struct vfio_platform_device *vdev)module_put (vdev->reset_module); } +static void vfio_platform_put_msi (struct vfio_platform_device *vdev) +{ +if (vdev->of_get_msi) +module_put (vdev->msi_module); +} +static int vfio_platform_regions_init (struct vfio_platform_device *vdev) {int cnt =0 , i;@@ -343,9 +384,17 @@ static long vfio_platform_ioctl(void *device_data, }else if (cmd ==VFIO_DEVICE_GET_IRQ_INFO ) {struct vfio_irq_info info; +struct vfio_info_cap caps = { .buf =NULL , .size =0 }; +struct vfio_irq_info_cap_msi *msi_info =NULL ; +int ext_irq_index = vdev->num_irqs - vdev->num_ext_irqs; +unsigned long capsz; +u32 index; minsz = offsetofend(struct vfio_irq_info, count); +/* For backward compatibility, cannot require this */ + capsz = offsetofend(struct vfio_irq_info, cap_offset); +if (copy_from_user (&info, (void __user *)arg, minsz))return -EFAULT ;@@ -355,8 +404,94 @@ static long vfio_platform_ioctl(void *device_data,if (info.index >= vdev->num_irqs)return -EINVAL ; - info.flags = vdev->irqs[info.index].flags; - info.count = vdev->irqs[info.index].count; +if (info.argsz >= capsz) + minsz = capsz; + + index = info.index; + + info.flags = vdev->irqs[index].flags; + info.count = vdev->irqs[index].count; + +if (ext_irq_index - index ==VFIO_EXT_IRQ_MSI ) { +struct vfio_irq_info_cap_type cap_type = { + .header.id =VFIO_IRQ_INFO_CAP_TYPE , + .header.version =1 }; +struct vfio_platform_irq *irq; +size_t msi_info_size; +int num_msgs; +int ret; +int i; + + index =array_index_nospec (index, + vdev->num_irqs); + irq = &vdev->irqs[index]; + + cap_type.type = irq->type; + cap_type.subtype = irq->subtype; + + ret =vfio_info_add_capability (&caps, + &cap_type.header, +sizeof (cap_type)); +if (ret) +return ret; + + num_msgs = irq->num_ctx; +if (num_msgs) { + msi_info_size =struct_size (msi_info, + msgs, num_msgs); + + msi_info =kzalloc (msi_info_size,GFP_KERNEL ); +if (!msi_info) {
Quoted 8 lines
drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 450
Quoted 6 lines
drivers/vfio/platform/vfio_platform_common.c +6
-0+kfree (caps.buf); +return -ENOMEM ; + } + + msi_info->header.id = +VFIO_IRQ_INFO_CAP_MSI_DESCS ;I thought you would remove this cap now the module is back. What is the motivation to keep it?
Hi Eric,
The earlier module was serving two purposes
a) Max number of MSI(s) supported by the device. Since the module is
back this information can be filled in .count.
b) Writing msi_msg(s) to devices for MSI sources.
We want b) to be handled in the user space so that they need not be
dependent on the kernel. VFIO_IRQ_INFO_CAP_MSI_DESCS is helping user space to get MSI configuration data and thus they can independently configure MSI sources. For example, user space can just ask for ‘N’ MSI vectors and related msi_msg for ‘N’ vectors is exported to user space using this CAP. User space can now use ‘N’ vectors to configure any ‘N’ MSI sources and these sources might be in any order. However, this design is different from vfio-pci as MSI configuration is being handled in kernel only.
Let us know if implementing VFIO_IRQ_INFO_CAP_MSI_DESCS does not fit with the CAP framework.
Thanks,
Vikas
Quoted 80 lines
Thanks
Eric
Quoted 76 lines
drivers/vfio/platform/vfio_platform_common.c +53
-2+ msi_info->header.version =1 ; + msi_info->nr_msgs = num_msgs; + +for (i =0 ; i < num_msgs; i++) { +struct vfio_irq_ctx *ctx = &irq->ctx[i]; + + msi_info->msgs[i].addr_lo = + ctx->msg.address_lo; + msi_info->msgs[i].addr_hi = + ctx->msg.address_hi; + msi_info->msgs[i].data = ctx->msg.data; + } + + ret =vfio_info_add_capability (&caps, + &msi_info->header, + msi_info_size); +if (ret) { +kfree (msi_info); +kfree (caps.buf); +return ret; + } + } + } + +if (caps.size) { + info.flags |=VFIO_IRQ_INFO_FLAG_CAPS ; +if (info.argsz <sizeof (info) + caps.size) { + info.argsz =sizeof (info) + caps.size; + info.cap_offset =0 ; + }else { +vfio_info_cap_shift (&caps,sizeof (info)); +if (copy_to_user ((void __user *)arg + +sizeof (info), caps.buf, + caps.size)) { +kfree (msi_info); +kfree (caps.buf); +return -EFAULT ; + } + info.cap_offset =sizeof (info); + } + +kfree (msi_info); +kfree (caps.buf); + }return copy_to_user ((void __user *)arg, &info, minsz) ? -EFAULT :0 ;@@ -365,6 +500,7 @@ static long vfio_platform_ioctl(void *device_data,struct vfio_irq_set hdr;u8 *data =NULL ;int ret =0 ; +int max;size_t data_size =0 ; minsz = offsetofend(struct vfio_irq_set, count);@@ -372,8 +508,14 @@ static long vfio_platform_ioctl(void *device_data,if (copy_from_user (&hdr, (void __user *)arg, minsz))return -EFAULT ; - ret =vfio_set_irqs_validate_and_prepare (&hdr, vdev->num_irqs, - vdev->num_irqs, &data_size); +if (hdr.index >= vdev->num_irqs) +return -EINVAL ; + + max = vdev->irqs[hdr.index].count; + + ret =vfio_set_irqs_validate_and_prepare (&hdr, max, + vdev->num_irqs, + &data_size);if (ret)return ret;@@ -678,6 +820,10 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,return ret; }
Quoted 7 lines
drivers/vfio/platform/vfio_platform_common.c int vfio_platform_probe_common(struct vfio_platform_device *vdev, · line 825
Quoted 3 lines
drivers/vfio/platform/vfio_platform_common.c +3
-0+ ret =vfio_platform_get_msi (vdev); +if (ret) +dev_info (vdev->device,"msi not supported\n" );I don't think we should output this message. This is printed for amd-xgbe which does not have msi so this is misleading. I would say either the
vfio_platform_get_msi()can return an actual error or we return a void?
Will check on this what
Quoted 522 lines
Thanks
Eric
Quoted 517 lines
drivers/vfio/platform/vfio_platform_common.c +26
-0+ group =vfio_iommu_group_get (dev);if (!group) {dev_err (dev,"No IOMMU group for device %s\n" , vdev->name);@@ -697,6 +843,7 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev, put_iommu:vfio_iommu_group_put (group, dev); put_reset: +vfio_platform_put_msi (vdev);vfio_platform_put_reset (vdev);return ret; }@@ -744,6 +891,30 @@ void vfio_platform_unregister_reset(const char *compat, }EXPORT_SYMBOL_GPL (vfio_platform_unregister_reset); +void __vfio_platform_register_msi (struct vfio_platform_msi_node *node) +{ +mutex_lock (&driver_lock); +list_add (&node->link, &msi_list); +mutex_unlock (&driver_lock); +} +EXPORT_SYMBOL_GPL (__vfio_platform_register_msi); + +void vfio_platform_unregister_msi (const char *compat) +{ +struct vfio_platform_msi_node *iter, *temp; + +mutex_lock (&driver_lock); +list_for_each_entry_safe (iter, temp, &msi_list, link) { +if (!strcmp (iter->compat, compat)) { +list_del (&iter->link); +break ; + } + } + +mutex_unlock (&driver_lock); +} +EXPORT_SYMBOL_GPL (vfio_platform_unregister_msi); +MODULE_VERSION (DRIVER_VERSION );MODULE_LICENSE ("GPL v2" );MODULE_AUTHOR (DRIVER_AUTHOR );drivers/vfio/platform/vfio_platform_irq.c +245
-15@@ -8,10 +8,12 @@#include <linux/eventfd.h> #include <linux/interrupt.h> +#include <linux/eventfd.h> #include <linux/slab.h> #include <linux/types.h> #include <linux/vfio.h> #include <linux/irq.h> +#include <linux/msi.h> #include "vfio_platform_private.h" @@ -253,6 +255,195 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,return 0 ; } +/* MSI/MSIX */ +static irqreturn_t vfio_msihandler (int irq,void *arg) +{ +struct eventfd_ctx *trigger = arg; + +eventfd_signal (trigger,1 ); +return IRQ_HANDLED ; +} + +static void msi_write (struct msi_desc *desc,struct msi_msg *msg) +{ +int i; +struct vfio_platform_irq *irq; +u16 index = desc->platform.msi_index; +struct device *dev =msi_desc_to_dev (desc); +struct vfio_device *device =dev_get_drvdata (dev); +struct vfio_platform_device *vdev = (struct vfio_platform_device *) +vfio_device_data (device); + +for (i =0 ; i < vdev->num_irqs; i++) +if (vdev->irqs[i].type ==VFIO_IRQ_TYPE_MSI ) + irq = &vdev->irqs[i]; + + irq->ctx[index].msg.address_lo = msg->address_lo; + irq->ctx[index].msg.address_hi = msg->address_hi; + irq->ctx[index].msg.data = msg->data; +} + +static int vfio_msi_enable (struct vfio_platform_device *vdev, +struct vfio_platform_irq *irq,int nvec) +{ +int ret; +int msi_idx =0 ; +struct msi_desc *desc; +struct device *dev = vdev->device; + + irq->ctx =kcalloc (nvec,sizeof (struct vfio_irq_ctx ),GFP_KERNEL ); +if (!irq->ctx) +return -ENOMEM ; + +/* Allocate platform MSIs */ + ret =platform_msi_domain_alloc_irqs (dev, nvec, msi_write); +if (ret <0 ) { +kfree (irq->ctx); +return ret; + } + +for_each_msi_entry (desc, dev) { + irq->ctx[msi_idx].hwirq = desc->irq; + msi_idx++; + } + + irq->num_ctx = nvec; + irq->config_msi =1 ; + +return 0 ; +} + +static int vfio_msi_set_vector_signal (struct vfio_platform_irq *irq, +int vector,int fd) +{ +struct eventfd_ctx *trigger; +int irq_num, ret; + +if (vector <0 || vector >= irq->num_ctx) +return -EINVAL ; + + irq_num = irq->ctx[vector].hwirq; + +if (irq->ctx[vector].trigger) { +free_irq (irq_num, irq->ctx[vector].trigger); +kfree (irq->ctx[vector].name); +eventfd_ctx_put (irq->ctx[vector].trigger); + irq->ctx[vector].trigger =NULL ; + } + +if (fd <0 ) +return 0 ; + + irq->ctx[vector].name =kasprintf (GFP_KERNEL , +"vfio-msi[%d]" , vector); +if (!irq->ctx[vector].name) +return -ENOMEM ; + + trigger =eventfd_ctx_fdget (fd); +if (IS_ERR (trigger)) { +kfree (irq->ctx[vector].name); +return PTR_ERR (trigger); + } + + ret =request_irq (irq_num, vfio_msihandler,0 , + irq->ctx[vector].name, trigger); +if (ret) { +kfree (irq->ctx[vector].name); +eventfd_ctx_put (trigger); +return ret; + } + + irq->ctx[vector].trigger = trigger; + +return 0 ; +} + +static int vfio_msi_set_block (struct vfio_platform_irq *irq,unsigned int start, +unsigned int count,int32_t *fds) +{ +int i, j, ret =0 ; + +if (start >= irq->num_ctx || start + count > irq->num_ctx) +return -EINVAL ; + +for (i =0 , j = start; i < count && !ret; i++, j++) { +int fd = fds ? fds[i] :-1 ; + + ret =vfio_msi_set_vector_signal (irq, j, fd); + } + +if (ret) { +for (--j; j >= (int )start; j--) +vfio_msi_set_vector_signal (irq, j,-1 ); + } + +return ret; +} + +static void vfio_msi_disable (struct vfio_platform_device *vdev, +struct vfio_platform_irq *irq) +{ +struct device *dev = vdev->device; + +vfio_msi_set_block (irq,0 , irq->num_ctx,NULL ); + +platform_msi_domain_free_irqs (dev); + + irq->config_msi =0 ; + irq->num_ctx =0 ; + +kfree (irq->ctx); +} + +static int vfio_set_msi_trigger (struct vfio_platform_device *vdev, +unsigned int index,unsigned int start, +unsigned int count,uint32_t flags,void *data) +{ +int i; +struct vfio_platform_irq *irq = &vdev->irqs[index]; + +if (start + count > irq->count) +return -EINVAL ; + +if (!count && (flags &VFIO_IRQ_SET_DATA_NONE )) { +vfio_msi_disable (vdev, irq); +return 0 ; + } + +if (flags &VFIO_IRQ_SET_DATA_EVENTFD ) { +s32 *fds = data; +int ret; + +if (irq->config_msi) +return vfio_msi_set_block (irq, start, count, + fds); + ret =vfio_msi_enable (vdev, irq, start + count); +if (ret) +return ret; + + ret =vfio_msi_set_block (irq, start, count, fds); +if (ret) +vfio_msi_disable (vdev, irq); + +return ret; + } + +for (i = start; i < start + count; i++) { +if (!irq->ctx[i].trigger) +continue ; +if (flags &VFIO_IRQ_SET_DATA_NONE ) { +eventfd_signal (irq->ctx[i].trigger,1 ); + }else if (flags &VFIO_IRQ_SET_DATA_BOOL ) { +u8 *bools = data; + +if (bools[i - start]) +eventfd_signal (irq->ctx[i].trigger,1 ); + } + } + +return 0 ; +} +int vfio_platform_set_irqs_ioctl (struct vfio_platform_device *vdev,uint32_t flags,unsigned index,unsigned start,unsigned count,void *data)@@ -261,16 +452,29 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev, unsigned start, unsigned count, uint32_t flags, void *data) = NULL; -switch (flags &VFIO_IRQ_SET_ACTION_TYPE_MASK ) { -case VFIO_IRQ_SET_ACTION_MASK : - func = vfio_platform_set_irq_mask; -break ; -case VFIO_IRQ_SET_ACTION_UNMASK : - func = vfio_platform_set_irq_unmask; -break ; -case VFIO_IRQ_SET_ACTION_TRIGGER : - func = vfio_platform_set_irq_trigger; -break ; +struct vfio_platform_irq *irq = &vdev->irqs[index]; + +if (irq->type ==VFIO_IRQ_TYPE_MSI ) { +switch (flags &VFIO_IRQ_SET_ACTION_TYPE_MASK ) { +case VFIO_IRQ_SET_ACTION_MASK : +case VFIO_IRQ_SET_ACTION_UNMASK : +break ; +case VFIO_IRQ_SET_ACTION_TRIGGER : + func = vfio_set_msi_trigger; +break ; + } + }else { +switch (flags &VFIO_IRQ_SET_ACTION_TYPE_MASK ) { +case VFIO_IRQ_SET_ACTION_MASK : + func = vfio_platform_set_irq_mask; +break ; +case VFIO_IRQ_SET_ACTION_UNMASK : + func = vfio_platform_set_irq_unmask; +break ; +case VFIO_IRQ_SET_ACTION_TRIGGER : + func = vfio_platform_set_irq_trigger; +break ; + } }if (!func)@@ -281,12 +485,21 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,int vfio_platform_irq_init (struct vfio_platform_device *vdev) { -int cnt =0 , i; +int i; +int cnt =0 ; +int num_irqs; +int msi_cnt =0 ;while (vdev->get_irq (vdev, cnt) >=0 ) cnt++; - vdev->irqs =kcalloc (cnt,sizeof (struct vfio_platform_irq ),GFP_KERNEL ); +if (vdev->of_get_msi) { + msi_cnt = vdev->of_get_msi (vdev); + num_irqs++; + } + + vdev->irqs =kcalloc (num_irqs,sizeof (struct vfio_platform_irq ), +GFP_KERNEL );if (!vdev->irqs)return -ENOMEM ;@@ -309,7 +522,19 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev) vdev->irqs[i].masked = false; } - vdev->num_irqs = cnt; +/* +* MSI block is added at last index and it is an ext irq +*/ +if (msi_cnt >0 ) { + vdev->irqs[i].flags =VFIO_IRQ_INFO_EVENTFD ; + vdev->irqs[i].count = msi_cnt; + vdev->irqs[i].hwirq =0 ; + vdev->irqs[i].masked = false; + vdev->irqs[i].type =VFIO_IRQ_TYPE_MSI ; + vdev->num_ext_irqs =1 ; + } + + vdev->num_irqs = num_irqs;return 0 ; err:@@ -321,8 +546,13 @@ void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev) {int i; -for (i =0 ; i < vdev->num_irqs; i++) -vfio_set_trigger (vdev, i,-1 ,NULL ); +for (i =0 ; i < vdev->num_irqs; i++) { +if (vdev->irqs[i].type ==VFIO_IRQ_TYPE_MSI ) +vfio_set_msi_trigger (vdev, i,0 ,0 , +VFIO_IRQ_SET_DATA_NONE ,NULL ); +else +vfio_set_trigger (vdev, i,-1 ,NULL ); + } vdev->num_irqs =0 ;kfree (vdev->irqs);drivers/vfio/platform/vfio_platform_private.h +32
-0@@ -9,6 +9,7 @@#include <linux/types.h> #include <linux/interrupt.h> +#include <linux/msi.h> #define VFIO_PLATFORM_OFFSET_SHIFT 40#define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)@@ -19,9 +20,21 @@#define VFIO_PLATFORM_INDEX_TO_OFFSET (index) \ ((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT) +/* IRQ index for MSI in ext IRQs */ +#define VFIO_EXT_IRQ_MSI 0 + +struct vfio_irq_ctx { +int hwirq; +char *name; +struct msi_msg msg; +struct eventfd_ctx *trigger; +}; +struct vfio_platform_irq {u32 flags;u32 count; +int num_ctx; +struct vfio_irq_ctx *ctx;int hwirq;char *name;struct eventfd_ctx *trigger;@@ -29,6 +42,11 @@ struct vfio_platform_irq {spinlock_t lock;struct virqfd *unmask;struct virqfd *mask; + +/* for extended irqs */ +u32 type; +u32 subtype; +int config_msi; };struct vfio_platform_region {@@ -46,12 +64,14 @@ struct vfio_platform_device {u32 num_regions;struct vfio_platform_irq *irqs;u32 num_irqs; +int num_ext_irqs;int refcnt;struct mutex igate;struct module *parent_module;const char *compat;const char *acpihid;struct module *reset_module; +struct module *msi_module;struct device *device;/* @@ -65,11 +85,13 @@ struct vfio_platform_device { (*get_resource)(struct vfio_platform_device *vdev, int i);int (*get_irq)(struct vfio_platform_device *vdev,int i);int (*of_reset)(struct vfio_platform_device *vdev); +u32 (*of_get_msi)(struct vfio_platform_device *vdev);bool reset_required; };typedef int (*vfio_platform_reset_fn_t )(struct vfio_platform_device *vdev); +typedef u32 (*vfio_platform_get_msi_fn_t )(struct vfio_platform_device *vdev);struct vfio_platform_reset_node {struct list_head link;@@ -78,6 +100,13 @@ struct vfio_platform_reset_node {vfio_platform_reset_fn_t of_reset; }; +struct vfio_platform_msi_node { +struct list_head link; +char *compat; +struct module *owner; +vfio_platform_get_msi_fn_t of_get_msi; +}; +extern int vfio_platform_probe_common (struct vfio_platform_device *vdev,struct device *dev);extern struct vfio_platform_device *vfio_platform_remove_common@@ -94,6 +123,9 @@ extern int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,extern void __vfio_platform_register_reset (struct vfio_platform_reset_node *n);extern void vfio_platform_unregister_reset (const char *compat,vfio_platform_reset_fn_t fn); +void __vfio_platform_register_msi (struct vfio_platform_msi_node *n); +void vfio_platform_unregister_msi (const char *compat); +#define vfio_platform_register_reset (__compat, __reset) \ static struct vfio_platform_reset_node __reset ## _node = { \ .owner = THIS_MODULE, \include/uapi/linux/vfio.h +44
-0@@ -697,11 +697,55 @@ struct vfio_irq_info {#define VFIO_IRQ_INFO_MASKABLE (1 << 1)#define VFIO_IRQ_INFO_AUTOMASKED (1 << 2)#define VFIO_IRQ_INFO_NORESIZE (1 << 3) +#define VFIO_IRQ_INFO_FLAG_CAPS (1 << 4)/* Info supports caps */ __u32 index;/* IRQ index */ __u32 count;/* Number of IRQs within this index */ +__u32 cap_offset;/* Offset within info struct of first cap */ };#define VFIO_DEVICE_GET_IRQ_INFO _IO(VFIO_TYPE, VFIO_BASE + 9) +/* +* The irq type capability allows IRQs unique to a specific device or +* class of devices to be exposed. +* +* The structures below define version 1 of this capability. +*/ +#define VFIO_IRQ_INFO_CAP_TYPE 3 + +struct vfio_irq_info_cap_type { +struct vfio_info_cap_header header; +__u32 type;/* global per bus driver */ +__u32 subtype;/* type specific */ +}; + +/* +* List of IRQ types, global per bus driver. +* If you introduce a new type, please add it here. +*/ + +/* Non PCI devices having MSI(s) support */ +#define VFIO_IRQ_TYPE_MSI (1) + +/* +* The msi capability allows the user to use the msi msg to +* configure the iova for the msi configuration. +* The structures below define version 1 of this capability. +*/ +#define VFIO_IRQ_INFO_CAP_MSI_DESCS 4 + +struct vfio_irq_msi_msg { +__u32 addr_lo; +__u32 addr_hi; +__u32 data; +}; + +struct vfio_irq_info_cap_msi { +struct vfio_info_cap_header header; +__u32 nr_msgs; +__u32 reserved; +struct vfio_irq_msi_msg msgs[]; +}; +/** * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set) *
Signature · 11 lines
This electronic communication and the information and any files transmitted with it, or attached to it, are confidential and are intended solely for the use of the individual or entity to whom it is addressed and may contain information that is confidential, legally privileged, protected by privacy laws, or otherwise restricted from disclosure to anyone else. If you are not the intended recipient or the person responsible for delivering the e-mail to the intended recipient, you are hereby notified that any use, copying, distributing, dissemination, forwarding, printing, or copying of this e-mail is strictly prohibited. If you received this e-mail in error, please return the e-mail to the sender, delete it from your computer, and destroy any printed copy of it.
Auger Eric
maintainerRe: [RFC v3 1/2] vfio/platform: add support for msi
Hi Vikas,
Quoted 169 lines · Vikas Gupta
Quoted 159 lines · Auger Eric
Hi Vikas,
Quoted 25 lines · Vikas Gupta
MSI support for platform devices.The MSI block is added as an extended IRQ which exports caps
VFIO_IRQ_INFO_CAP_TYPEandVFIO_IRQ_INFO_CAP_MSI_DESCS.Signed-off-by Vikas Gupta <[email protected]>--- drivers/vfio/platform/vfio_platform_common.c | 179 +++++++++++- drivers/vfio/platform/vfio_platform_irq.c | 260 +++++++++++++++++- drivers/vfio/platform/vfio_platform_private.h | 32 +++ include/uapi/linux/vfio.h | 44 +++ 4 files changed, 496 insertions(+), 19 deletions(-)drivers/vfio/platform/vfio_platform_common.c +1
-0@@ -16,6 +16,7 @@#include <linux/types.h> #include <linux/uaccess.h> #include <linux/vfio.h> +#include <linux/nospec.h> #include "vfio_platform_private.h" drivers/vfio/platform/vfio_platform_common.c line 30
Quoted 4 lines
drivers/vfio/platform/vfio_platform_common.c +1
-0@@ -26,6 +27,8 @@#define VFIO_PLATFORM_IS_ACPI (vdev) ((vdev)->acpihid != NULL)static LIST_HEAD (reset_list); +/* devices having MSI support */ nit: for devices using MSIs?
Quoted 125 lines
drivers/vfio/platform/vfio_platform_common.c +85
-2+static LIST_HEAD (msi_list);static DEFINE_MUTEX (driver_lock); static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,@@ -47,6 +50,25 @@ static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,return reset_fn; } +static bool vfio_platform_lookup_msi (struct vfio_platform_device *vdev) +{ +bool has_msi = false; +struct vfio_platform_msi_node *iter; + +mutex_lock (&driver_lock); +list_for_each_entry (iter, &msi_list, link) { +if (!strcmp (iter->compat, vdev->compat) && +try_module_get (iter->owner)) { + vdev->msi_module = iter->owner; + vdev->of_get_msi = iter->of_get_msi; + has_msi = true; +break ; + } + } +mutex_unlock (&driver_lock); +return has_msi; +} +static int vfio_platform_acpi_probe (struct vfio_platform_device *vdev,struct device *dev) {@@ -126,6 +148,19 @@ static int vfio_platform_get_reset(struct vfio_platform_device *vdev)return vdev->of_reset ?0 : -ENOENT ; } +static int vfio_platform_get_msi (struct vfio_platform_device *vdev) +{ +bool has_msi; + + has_msi =vfio_platform_lookup_msi (vdev); +if (!has_msi) { +request_module ("vfio-msi:%s" , vdev->compat); + has_msi =vfio_platform_lookup_msi (vdev); + } + +return has_msi ?0 : -ENOENT ; +} +static void vfio_platform_put_reset (struct vfio_platform_device *vdev) {if (VFIO_PLATFORM_IS_ACPI (vdev))@@ -135,6 +170,12 @@ static void vfio_platform_put_reset(struct vfio_platform_device *vdev)module_put (vdev->reset_module); } +static void vfio_platform_put_msi (struct vfio_platform_device *vdev) +{ +if (vdev->of_get_msi) +module_put (vdev->msi_module); +} +static int vfio_platform_regions_init (struct vfio_platform_device *vdev) {int cnt =0 , i;@@ -343,9 +384,17 @@ static long vfio_platform_ioctl(void *device_data, }else if (cmd ==VFIO_DEVICE_GET_IRQ_INFO ) {struct vfio_irq_info info; +struct vfio_info_cap caps = { .buf =NULL , .size =0 }; +struct vfio_irq_info_cap_msi *msi_info =NULL ; +int ext_irq_index = vdev->num_irqs - vdev->num_ext_irqs; +unsigned long capsz; +u32 index; minsz = offsetofend(struct vfio_irq_info, count); +/* For backward compatibility, cannot require this */ + capsz = offsetofend(struct vfio_irq_info, cap_offset); +if (copy_from_user (&info, (void __user *)arg, minsz))return -EFAULT ;@@ -355,8 +404,94 @@ static long vfio_platform_ioctl(void *device_data,if (info.index >= vdev->num_irqs)return -EINVAL ; - info.flags = vdev->irqs[info.index].flags; - info.count = vdev->irqs[info.index].count; +if (info.argsz >= capsz) + minsz = capsz; + + index = info.index; + + info.flags = vdev->irqs[index].flags; + info.count = vdev->irqs[index].count; + +if (ext_irq_index - index ==VFIO_EXT_IRQ_MSI ) { +struct vfio_irq_info_cap_type cap_type = { + .header.id =VFIO_IRQ_INFO_CAP_TYPE , + .header.version =1 }; +struct vfio_platform_irq *irq; +size_t msi_info_size; +int num_msgs; +int ret; +int i; + + index =array_index_nospec (index, + vdev->num_irqs); + irq = &vdev->irqs[index]; + + cap_type.type = irq->type; + cap_type.subtype = irq->subtype; + + ret =vfio_info_add_capability (&caps, + &cap_type.header, +sizeof (cap_type)); +if (ret) +return ret; + + num_msgs = irq->num_ctx; +if (num_msgs) { + msi_info_size =struct_size (msi_info, + msgs, num_msgs); + + msi_info =kzalloc (msi_info_size,GFP_KERNEL ); +if (!msi_info) {Quoted 8 lines
drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 450
Quoted 6 lines
drivers/vfio/platform/vfio_platform_common.c +6
-0+kfree (caps.buf); +return -ENOMEM ; + } + + msi_info->header.id = +VFIO_IRQ_INFO_CAP_MSI_DESCS ;I thought you would remove this cap now the module is back. What is the motivation to keep it?
Quoted 4 lines
Hi Eric,
The earlier module was serving two purposes
a) Max number of MSI(s) supported by the device. Since the module is
back this information can be filled in .count.
understood
Quoted 3 lines
b) Writing
msi_msg(s)to devices for MSI sources.
We want b) to be handled in the user space so that they need not be
dependent on the kernel.
I fail to understand the problem with the "dependency on the kernel". What is the functional difference when exposing this to the userspace, please could you highlight the benefits?
VFIO_IRQ_INFO_CAP_MSI_DESCS is helping user
Quoted 3 lines
space to get MSI configuration data and thus they can independently configure MSI sources. For example, user space can just ask for ‘N’ MSI vectors and related
msi_msgfor ‘N’ vectors
this is induced by vfio_msi_enable(), right?
is exported to user
Quoted 6 lines
space using this CAP. User space can now use ‘N’ vectors to configure any ‘N’ MSI sources and these sources might be in any order. However, this design is different from vfio-pci as MSI configuration is being handled in kernel only.
Let us know if implementingVFIO_IRQ_INFO_CAP_MSI_DESCSdoes not fit with the CAP framework.
I think it is OK reporting them through the cap but I fail to understand the benefits.
Thanks
Eric
Quoted 615 lines
Thanks,
VikasQuoted 80 lines
Thanks
Eric
Quoted 76 lines
drivers/vfio/platform/vfio_platform_common.c +53
-2+ msi_info->header.version =1 ; + msi_info->nr_msgs = num_msgs; + +for (i =0 ; i < num_msgs; i++) { +struct vfio_irq_ctx *ctx = &irq->ctx[i]; + + msi_info->msgs[i].addr_lo = + ctx->msg.address_lo; + msi_info->msgs[i].addr_hi = + ctx->msg.address_hi; + msi_info->msgs[i].data = ctx->msg.data; + } + + ret =vfio_info_add_capability (&caps, + &msi_info->header, + msi_info_size); +if (ret) { +kfree (msi_info); +kfree (caps.buf); +return ret; + } + } + } + +if (caps.size) { + info.flags |=VFIO_IRQ_INFO_FLAG_CAPS ; +if (info.argsz <sizeof (info) + caps.size) { + info.argsz =sizeof (info) + caps.size; + info.cap_offset =0 ; + }else { +vfio_info_cap_shift (&caps,sizeof (info)); +if (copy_to_user ((void __user *)arg + +sizeof (info), caps.buf, + caps.size)) { +kfree (msi_info); +kfree (caps.buf); +return -EFAULT ; + } + info.cap_offset =sizeof (info); + } + +kfree (msi_info); +kfree (caps.buf); + }return copy_to_user ((void __user *)arg, &info, minsz) ? -EFAULT :0 ;@@ -365,6 +500,7 @@ static long vfio_platform_ioctl(void *device_data,struct vfio_irq_set hdr;u8 *data =NULL ;int ret =0 ; +int max;size_t data_size =0 ; minsz = offsetofend(struct vfio_irq_set, count);@@ -372,8 +508,14 @@ static long vfio_platform_ioctl(void *device_data,if (copy_from_user (&hdr, (void __user *)arg, minsz))return -EFAULT ; - ret =vfio_set_irqs_validate_and_prepare (&hdr, vdev->num_irqs, - vdev->num_irqs, &data_size); +if (hdr.index >= vdev->num_irqs) +return -EINVAL ; + + max = vdev->irqs[hdr.index].count; + + ret =vfio_set_irqs_validate_and_prepare (&hdr, max, + vdev->num_irqs, + &data_size);if (ret)return ret;@@ -678,6 +820,10 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,return ret; }Quoted 7 lines
drivers/vfio/platform/vfio_platform_common.c int vfio_platform_probe_common(struct vfio_platform_device *vdev, · line 825
Quoted 3 lines
drivers/vfio/platform/vfio_platform_common.c +3
-0+ ret =vfio_platform_get_msi (vdev); +if (ret) +dev_info (vdev->device,"msi not supported\n" );I don't think we should output this message. This is printed for amd-xgbe which does not have msi so this is misleading. I would say either the
vfio_platform_get_msi()can return an actual error or we return a void?Will check on this what
Quoted 522 lines
Thanks
Eric
Quoted 517 lines
drivers/vfio/platform/vfio_platform_common.c +26
-0+ group =vfio_iommu_group_get (dev);if (!group) {dev_err (dev,"No IOMMU group for device %s\n" , vdev->name);@@ -697,6 +843,7 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev, put_iommu:vfio_iommu_group_put (group, dev); put_reset: +vfio_platform_put_msi (vdev);vfio_platform_put_reset (vdev);return ret; }@@ -744,6 +891,30 @@ void vfio_platform_unregister_reset(const char *compat, }EXPORT_SYMBOL_GPL (vfio_platform_unregister_reset); +void __vfio_platform_register_msi (struct vfio_platform_msi_node *node) +{ +mutex_lock (&driver_lock); +list_add (&node->link, &msi_list); +mutex_unlock (&driver_lock); +} +EXPORT_SYMBOL_GPL (__vfio_platform_register_msi); + +void vfio_platform_unregister_msi (const char *compat) +{ +struct vfio_platform_msi_node *iter, *temp; + +mutex_lock (&driver_lock); +list_for_each_entry_safe (iter, temp, &msi_list, link) { +if (!strcmp (iter->compat, compat)) { +list_del (&iter->link); +break ; + } + } + +mutex_unlock (&driver_lock); +} +EXPORT_SYMBOL_GPL (vfio_platform_unregister_msi); +MODULE_VERSION (DRIVER_VERSION );MODULE_LICENSE ("GPL v2" );MODULE_AUTHOR (DRIVER_AUTHOR );drivers/vfio/platform/vfio_platform_irq.c +245
-15@@ -8,10 +8,12 @@#include <linux/eventfd.h> #include <linux/interrupt.h> +#include <linux/eventfd.h> #include <linux/slab.h> #include <linux/types.h> #include <linux/vfio.h> #include <linux/irq.h> +#include <linux/msi.h> #include "vfio_platform_private.h" @@ -253,6 +255,195 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,return 0 ; } +/* MSI/MSIX */ +static irqreturn_t vfio_msihandler (int irq,void *arg) +{ +struct eventfd_ctx *trigger = arg; + +eventfd_signal (trigger,1 ); +return IRQ_HANDLED ; +} + +static void msi_write (struct msi_desc *desc,struct msi_msg *msg) +{ +int i; +struct vfio_platform_irq *irq; +u16 index = desc->platform.msi_index; +struct device *dev =msi_desc_to_dev (desc); +struct vfio_device *device =dev_get_drvdata (dev); +struct vfio_platform_device *vdev = (struct vfio_platform_device *) +vfio_device_data (device); + +for (i =0 ; i < vdev->num_irqs; i++) +if (vdev->irqs[i].type ==VFIO_IRQ_TYPE_MSI ) + irq = &vdev->irqs[i]; + + irq->ctx[index].msg.address_lo = msg->address_lo; + irq->ctx[index].msg.address_hi = msg->address_hi; + irq->ctx[index].msg.data = msg->data; +} + +static int vfio_msi_enable (struct vfio_platform_device *vdev, +struct vfio_platform_irq *irq,int nvec) +{ +int ret; +int msi_idx =0 ; +struct msi_desc *desc; +struct device *dev = vdev->device; + + irq->ctx =kcalloc (nvec,sizeof (struct vfio_irq_ctx ),GFP_KERNEL ); +if (!irq->ctx) +return -ENOMEM ; + +/* Allocate platform MSIs */ + ret =platform_msi_domain_alloc_irqs (dev, nvec, msi_write); +if (ret <0 ) { +kfree (irq->ctx); +return ret; + } + +for_each_msi_entry (desc, dev) { + irq->ctx[msi_idx].hwirq = desc->irq; + msi_idx++; + } + + irq->num_ctx = nvec; + irq->config_msi =1 ; + +return 0 ; +} + +static int vfio_msi_set_vector_signal (struct vfio_platform_irq *irq, +int vector,int fd) +{ +struct eventfd_ctx *trigger; +int irq_num, ret; + +if (vector <0 || vector >= irq->num_ctx) +return -EINVAL ; + + irq_num = irq->ctx[vector].hwirq; + +if (irq->ctx[vector].trigger) { +free_irq (irq_num, irq->ctx[vector].trigger); +kfree (irq->ctx[vector].name); +eventfd_ctx_put (irq->ctx[vector].trigger); + irq->ctx[vector].trigger =NULL ; + } + +if (fd <0 ) +return 0 ; + + irq->ctx[vector].name =kasprintf (GFP_KERNEL , +"vfio-msi[%d]" , vector); +if (!irq->ctx[vector].name) +return -ENOMEM ; + + trigger =eventfd_ctx_fdget (fd); +if (IS_ERR (trigger)) { +kfree (irq->ctx[vector].name); +return PTR_ERR (trigger); + } + + ret =request_irq (irq_num, vfio_msihandler,0 , + irq->ctx[vector].name, trigger); +if (ret) { +kfree (irq->ctx[vector].name); +eventfd_ctx_put (trigger); +return ret; + } + + irq->ctx[vector].trigger = trigger; + +return 0 ; +} + +static int vfio_msi_set_block (struct vfio_platform_irq *irq,unsigned int start, +unsigned int count,int32_t *fds) +{ +int i, j, ret =0 ; + +if (start >= irq->num_ctx || start + count > irq->num_ctx) +return -EINVAL ; + +for (i =0 , j = start; i < count && !ret; i++, j++) { +int fd = fds ? fds[i] :-1 ; + + ret =vfio_msi_set_vector_signal (irq, j, fd); + } + +if (ret) { +for (--j; j >= (int )start; j--) +vfio_msi_set_vector_signal (irq, j,-1 ); + } + +return ret; +} + +static void vfio_msi_disable (struct vfio_platform_device *vdev, +struct vfio_platform_irq *irq) +{ +struct device *dev = vdev->device; + +vfio_msi_set_block (irq,0 , irq->num_ctx,NULL ); + +platform_msi_domain_free_irqs (dev); + + irq->config_msi =0 ; + irq->num_ctx =0 ; + +kfree (irq->ctx); +} + +static int vfio_set_msi_trigger (struct vfio_platform_device *vdev, +unsigned int index,unsigned int start, +unsigned int count,uint32_t flags,void *data) +{ +int i; +struct vfio_platform_irq *irq = &vdev->irqs[index]; + +if (start + count > irq->count) +return -EINVAL ; + +if (!count && (flags &VFIO_IRQ_SET_DATA_NONE )) { +vfio_msi_disable (vdev, irq); +return 0 ; + } + +if (flags &VFIO_IRQ_SET_DATA_EVENTFD ) { +s32 *fds = data; +int ret; + +if (irq->config_msi) +return vfio_msi_set_block (irq, start, count, + fds); + ret =vfio_msi_enable (vdev, irq, start + count); +if (ret) +return ret; + + ret =vfio_msi_set_block (irq, start, count, fds); +if (ret) +vfio_msi_disable (vdev, irq); + +return ret; + } + +for (i = start; i < start + count; i++) { +if (!irq->ctx[i].trigger) +continue ; +if (flags &VFIO_IRQ_SET_DATA_NONE ) { +eventfd_signal (irq->ctx[i].trigger,1 ); + }else if (flags &VFIO_IRQ_SET_DATA_BOOL ) { +u8 *bools = data; + +if (bools[i - start]) +eventfd_signal (irq->ctx[i].trigger,1 ); + } + } + +return 0 ; +} +int vfio_platform_set_irqs_ioctl (struct vfio_platform_device *vdev,uint32_t flags,unsigned index,unsigned start,unsigned count,void *data)@@ -261,16 +452,29 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev, unsigned start, unsigned count, uint32_t flags, void *data) = NULL; -switch (flags &VFIO_IRQ_SET_ACTION_TYPE_MASK ) { -case VFIO_IRQ_SET_ACTION_MASK : - func = vfio_platform_set_irq_mask; -break ; -case VFIO_IRQ_SET_ACTION_UNMASK : - func = vfio_platform_set_irq_unmask; -break ; -case VFIO_IRQ_SET_ACTION_TRIGGER : - func = vfio_platform_set_irq_trigger; -break ; +struct vfio_platform_irq *irq = &vdev->irqs[index]; + +if (irq->type ==VFIO_IRQ_TYPE_MSI ) { +switch (flags &VFIO_IRQ_SET_ACTION_TYPE_MASK ) { +case VFIO_IRQ_SET_ACTION_MASK : +case VFIO_IRQ_SET_ACTION_UNMASK : +break ; +case VFIO_IRQ_SET_ACTION_TRIGGER : + func = vfio_set_msi_trigger; +break ; + } + }else { +switch (flags &VFIO_IRQ_SET_ACTION_TYPE_MASK ) { +case VFIO_IRQ_SET_ACTION_MASK : + func = vfio_platform_set_irq_mask; +break ; +case VFIO_IRQ_SET_ACTION_UNMASK : + func = vfio_platform_set_irq_unmask; +break ; +case VFIO_IRQ_SET_ACTION_TRIGGER : + func = vfio_platform_set_irq_trigger; +break ; + } }if (!func)@@ -281,12 +485,21 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,int vfio_platform_irq_init (struct vfio_platform_device *vdev) { -int cnt =0 , i; +int i; +int cnt =0 ; +int num_irqs; +int msi_cnt =0 ;while (vdev->get_irq (vdev, cnt) >=0 ) cnt++; - vdev->irqs =kcalloc (cnt,sizeof (struct vfio_platform_irq ),GFP_KERNEL ); +if (vdev->of_get_msi) { + msi_cnt = vdev->of_get_msi (vdev); + num_irqs++; + } + + vdev->irqs =kcalloc (num_irqs,sizeof (struct vfio_platform_irq ), +GFP_KERNEL );if (!vdev->irqs)return -ENOMEM ;@@ -309,7 +522,19 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev) vdev->irqs[i].masked = false; } - vdev->num_irqs = cnt; +/* +* MSI block is added at last index and it is an ext irq +*/ +if (msi_cnt >0 ) { + vdev->irqs[i].flags =VFIO_IRQ_INFO_EVENTFD ; + vdev->irqs[i].count = msi_cnt; + vdev->irqs[i].hwirq =0 ; + vdev->irqs[i].masked = false; + vdev->irqs[i].type =VFIO_IRQ_TYPE_MSI ; + vdev->num_ext_irqs =1 ; + } + + vdev->num_irqs = num_irqs;return 0 ; err:@@ -321,8 +546,13 @@ void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev) {int i; -for (i =0 ; i < vdev->num_irqs; i++) -vfio_set_trigger (vdev, i,-1 ,NULL ); +for (i =0 ; i < vdev->num_irqs; i++) { +if (vdev->irqs[i].type ==VFIO_IRQ_TYPE_MSI ) +vfio_set_msi_trigger (vdev, i,0 ,0 , +VFIO_IRQ_SET_DATA_NONE ,NULL ); +else +vfio_set_trigger (vdev, i,-1 ,NULL ); + } vdev->num_irqs =0 ;kfree (vdev->irqs);drivers/vfio/platform/vfio_platform_private.h +32
-0@@ -9,6 +9,7 @@#include <linux/types.h> #include <linux/interrupt.h> +#include <linux/msi.h> #define VFIO_PLATFORM_OFFSET_SHIFT 40#define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)@@ -19,9 +20,21 @@#define VFIO_PLATFORM_INDEX_TO_OFFSET (index) \ ((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT) +/* IRQ index for MSI in ext IRQs */ +#define VFIO_EXT_IRQ_MSI 0 + +struct vfio_irq_ctx { +int hwirq; +char *name; +struct msi_msg msg; +struct eventfd_ctx *trigger; +}; +struct vfio_platform_irq {u32 flags;u32 count; +int num_ctx; +struct vfio_irq_ctx *ctx;int hwirq;char *name;struct eventfd_ctx *trigger;@@ -29,6 +42,11 @@ struct vfio_platform_irq {spinlock_t lock;struct virqfd *unmask;struct virqfd *mask; + +/* for extended irqs */ +u32 type; +u32 subtype; +int config_msi; };struct vfio_platform_region {@@ -46,12 +64,14 @@ struct vfio_platform_device {u32 num_regions;struct vfio_platform_irq *irqs;u32 num_irqs; +int num_ext_irqs;int refcnt;struct mutex igate;struct module *parent_module;const char *compat;const char *acpihid;struct module *reset_module; +struct module *msi_module;struct device *device;/* @@ -65,11 +85,13 @@ struct vfio_platform_device { (*get_resource)(struct vfio_platform_device *vdev, int i);int (*get_irq)(struct vfio_platform_device *vdev,int i);int (*of_reset)(struct vfio_platform_device *vdev); +u32 (*of_get_msi)(struct vfio_platform_device *vdev);bool reset_required; };typedef int (*vfio_platform_reset_fn_t )(struct vfio_platform_device *vdev); +typedef u32 (*vfio_platform_get_msi_fn_t )(struct vfio_platform_device *vdev);struct vfio_platform_reset_node {struct list_head link;@@ -78,6 +100,13 @@ struct vfio_platform_reset_node {vfio_platform_reset_fn_t of_reset; }; +struct vfio_platform_msi_node { +struct list_head link; +char *compat; +struct module *owner; +vfio_platform_get_msi_fn_t of_get_msi; +}; +extern int vfio_platform_probe_common (struct vfio_platform_device *vdev,struct device *dev);extern struct vfio_platform_device *vfio_platform_remove_common@@ -94,6 +123,9 @@ extern int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,extern void __vfio_platform_register_reset (struct vfio_platform_reset_node *n);extern void vfio_platform_unregister_reset (const char *compat,vfio_platform_reset_fn_t fn); +void __vfio_platform_register_msi (struct vfio_platform_msi_node *n); +void vfio_platform_unregister_msi (const char *compat); +#define vfio_platform_register_reset (__compat, __reset) \ static struct vfio_platform_reset_node __reset ## _node = { \ .owner = THIS_MODULE, \include/uapi/linux/vfio.h +44
-0@@ -697,11 +697,55 @@ struct vfio_irq_info {#define VFIO_IRQ_INFO_MASKABLE (1 << 1)#define VFIO_IRQ_INFO_AUTOMASKED (1 << 2)#define VFIO_IRQ_INFO_NORESIZE (1 << 3) +#define VFIO_IRQ_INFO_FLAG_CAPS (1 << 4)/* Info supports caps */ __u32 index;/* IRQ index */ __u32 count;/* Number of IRQs within this index */ +__u32 cap_offset;/* Offset within info struct of first cap */ };#define VFIO_DEVICE_GET_IRQ_INFO _IO(VFIO_TYPE, VFIO_BASE + 9) +/* +* The irq type capability allows IRQs unique to a specific device or +* class of devices to be exposed. +* +* The structures below define version 1 of this capability. +*/ +#define VFIO_IRQ_INFO_CAP_TYPE 3 + +struct vfio_irq_info_cap_type { +struct vfio_info_cap_header header; +__u32 type;/* global per bus driver */ +__u32 subtype;/* type specific */ +}; + +/* +* List of IRQ types, global per bus driver. +* If you introduce a new type, please add it here. +*/ + +/* Non PCI devices having MSI(s) support */ +#define VFIO_IRQ_TYPE_MSI (1) + +/* +* The msi capability allows the user to use the msi msg to +* configure the iova for the msi configuration. +* The structures below define version 1 of this capability. +*/ +#define VFIO_IRQ_INFO_CAP_MSI_DESCS 4 + +struct vfio_irq_msi_msg { +__u32 addr_lo; +__u32 addr_hi; +__u32 data; +}; + +struct vfio_irq_info_cap_msi { +struct vfio_info_cap_header header; +__u32 nr_msgs; +__u32 reserved; +struct vfio_irq_msi_msg msgs[]; +}; +/** * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set) *
Re: [RFC v3 1/2] vfio/platform: add support for msi
Hi Eric,
Quoted 177 lines · Auger Eric
Hi Vikas,
Quoted 169 lines · Vikas Gupta
Quoted 159 lines · Auger Eric
Hi Vikas,
Quoted 25 lines · Vikas Gupta
MSI support for platform devices.The MSI block is added as an extended IRQ which exports caps
VFIO_IRQ_INFO_CAP_TYPEandVFIO_IRQ_INFO_CAP_MSI_DESCS.Signed-off-by Vikas Gupta <[email protected]>--- drivers/vfio/platform/vfio_platform_common.c | 179 +++++++++++- drivers/vfio/platform/vfio_platform_irq.c | 260 +++++++++++++++++- drivers/vfio/platform/vfio_platform_private.h | 32 +++ include/uapi/linux/vfio.h | 44 +++ 4 files changed, 496 insertions(+), 19 deletions(-)drivers/vfio/platform/vfio_platform_common.c +1
-0@@ -16,6 +16,7 @@#include <linux/types.h> #include <linux/uaccess.h> #include <linux/vfio.h> +#include <linux/nospec.h> #include "vfio_platform_private.h" drivers/vfio/platform/vfio_platform_common.c line 30
Quoted 4 lines
drivers/vfio/platform/vfio_platform_common.c +1
-0@@ -26,6 +27,8 @@#define VFIO_PLATFORM_IS_ACPI (vdev) ((vdev)->acpihid != NULL)static LIST_HEAD (reset_list); +/* devices having MSI support */ nit: for devices using MSIs?
Quoted 125 lines
drivers/vfio/platform/vfio_platform_common.c +85
-2+static LIST_HEAD (msi_list);static DEFINE_MUTEX (driver_lock); static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,@@ -47,6 +50,25 @@ static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,return reset_fn; } +static bool vfio_platform_lookup_msi (struct vfio_platform_device *vdev) +{ +bool has_msi = false; +struct vfio_platform_msi_node *iter; + +mutex_lock (&driver_lock); +list_for_each_entry (iter, &msi_list, link) { +if (!strcmp (iter->compat, vdev->compat) && +try_module_get (iter->owner)) { + vdev->msi_module = iter->owner; + vdev->of_get_msi = iter->of_get_msi; + has_msi = true; +break ; + } + } +mutex_unlock (&driver_lock); +return has_msi; +} +static int vfio_platform_acpi_probe (struct vfio_platform_device *vdev,struct device *dev) {@@ -126,6 +148,19 @@ static int vfio_platform_get_reset(struct vfio_platform_device *vdev)return vdev->of_reset ?0 : -ENOENT ; } +static int vfio_platform_get_msi (struct vfio_platform_device *vdev) +{ +bool has_msi; + + has_msi =vfio_platform_lookup_msi (vdev); +if (!has_msi) { +request_module ("vfio-msi:%s" , vdev->compat); + has_msi =vfio_platform_lookup_msi (vdev); + } + +return has_msi ?0 : -ENOENT ; +} +static void vfio_platform_put_reset (struct vfio_platform_device *vdev) {if (VFIO_PLATFORM_IS_ACPI (vdev))@@ -135,6 +170,12 @@ static void vfio_platform_put_reset(struct vfio_platform_device *vdev)module_put (vdev->reset_module); } +static void vfio_platform_put_msi (struct vfio_platform_device *vdev) +{ +if (vdev->of_get_msi) +module_put (vdev->msi_module); +} +static int vfio_platform_regions_init (struct vfio_platform_device *vdev) {int cnt =0 , i;@@ -343,9 +384,17 @@ static long vfio_platform_ioctl(void *device_data, }else if (cmd ==VFIO_DEVICE_GET_IRQ_INFO ) {struct vfio_irq_info info; +struct vfio_info_cap caps = { .buf =NULL , .size =0 }; +struct vfio_irq_info_cap_msi *msi_info =NULL ; +int ext_irq_index = vdev->num_irqs - vdev->num_ext_irqs; +unsigned long capsz; +u32 index; minsz = offsetofend(struct vfio_irq_info, count); +/* For backward compatibility, cannot require this */ + capsz = offsetofend(struct vfio_irq_info, cap_offset); +if (copy_from_user (&info, (void __user *)arg, minsz))return -EFAULT ;@@ -355,8 +404,94 @@ static long vfio_platform_ioctl(void *device_data,if (info.index >= vdev->num_irqs)return -EINVAL ; - info.flags = vdev->irqs[info.index].flags; - info.count = vdev->irqs[info.index].count; +if (info.argsz >= capsz) + minsz = capsz; + + index = info.index; + + info.flags = vdev->irqs[index].flags; + info.count = vdev->irqs[index].count; + +if (ext_irq_index - index ==VFIO_EXT_IRQ_MSI ) { +struct vfio_irq_info_cap_type cap_type = { + .header.id =VFIO_IRQ_INFO_CAP_TYPE , + .header.version =1 }; +struct vfio_platform_irq *irq; +size_t msi_info_size; +int num_msgs; +int ret; +int i; + + index =array_index_nospec (index, + vdev->num_irqs); + irq = &vdev->irqs[index]; + + cap_type.type = irq->type; + cap_type.subtype = irq->subtype; + + ret =vfio_info_add_capability (&caps, + &cap_type.header, +sizeof (cap_type)); +if (ret) +return ret; + + num_msgs = irq->num_ctx; +if (num_msgs) { + msi_info_size =struct_size (msi_info, + msgs, num_msgs); + + msi_info =kzalloc (msi_info_size,GFP_KERNEL ); +if (!msi_info) {Quoted 8 lines
drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 450
Quoted 6 lines
drivers/vfio/platform/vfio_platform_common.c +6
-0+kfree (caps.buf); +return -ENOMEM ; + } + + msi_info->header.id = +VFIO_IRQ_INFO_CAP_MSI_DESCS ;I thought you would remove this cap now the module is back. What is the motivation to keep it?
Quoted 4 lines
Hi Eric,
The earlier module was serving two purposes
a) Max number of MSI(s) supported by the device. Since the module is
back this information can be filled in .count.
Quoted 7 lines
understood
Quoted 3 lines
b) Writing
msi_msg(s)to devices for MSI sources.
We want b) to be handled in the user space so that they need not be
dependent on the kernel.I fail to understand the problem with the "dependency on the kernel". What is the functional difference when exposing this to the userspace, please could you highlight the benefits?
We do not see any functional difference here, the benefit we see is that user space can configure MSI sources with msi_msg rather than doing it in the kernel.
Quoted 7 lines
VFIO_IRQ_INFO_CAP_MSI_DESCSis helping userQuoted 3 lines
space to get MSI configuration data and thus they can independently configure MSI sources. For example, user space can just ask for ‘N’ MSI vectors and related
msi_msgfor ‘N’ vectorsthis is induced by
vfio_msi_enable(), right?
Yes
Quoted 10 lines
is exported to user
Quoted 6 lines
space using this CAP. User space can now use ‘N’ vectors to configure any ‘N’ MSI sources and these sources might be in any order. However, this design is different from vfio-pci as MSI configuration is being handled in kernel only.
Let us know if implementingVFIO_IRQ_INFO_CAP_MSI_DESCSdoes not fit with the CAP framework.I think it is OK reporting them through the cap but I fail to understand the benefits.
I believe, since there is no significant benefit this CAP should be avoided. I plan to remove this CAP in the next patch set.
Thanks,
Vikas
Quoted 620 lines
Thanks
Eric
Quoted 615 lines
Thanks,
VikasQuoted 80 lines
Thanks
Eric
Quoted 76 lines
drivers/vfio/platform/vfio_platform_common.c +53
-2+ msi_info->header.version =1 ; + msi_info->nr_msgs = num_msgs; + +for (i =0 ; i < num_msgs; i++) { +struct vfio_irq_ctx *ctx = &irq->ctx[i]; + + msi_info->msgs[i].addr_lo = + ctx->msg.address_lo; + msi_info->msgs[i].addr_hi = + ctx->msg.address_hi; + msi_info->msgs[i].data = ctx->msg.data; + } + + ret =vfio_info_add_capability (&caps, + &msi_info->header, + msi_info_size); +if (ret) { +kfree (msi_info); +kfree (caps.buf); +return ret; + } + } + } + +if (caps.size) { + info.flags |=VFIO_IRQ_INFO_FLAG_CAPS ; +if (info.argsz <sizeof (info) + caps.size) { + info.argsz =sizeof (info) + caps.size; + info.cap_offset =0 ; + }else { +vfio_info_cap_shift (&caps,sizeof (info)); +if (copy_to_user ((void __user *)arg + +sizeof (info), caps.buf, + caps.size)) { +kfree (msi_info); +kfree (caps.buf); +return -EFAULT ; + } + info.cap_offset =sizeof (info); + } + +kfree (msi_info); +kfree (caps.buf); + }return copy_to_user ((void __user *)arg, &info, minsz) ? -EFAULT :0 ;@@ -365,6 +500,7 @@ static long vfio_platform_ioctl(void *device_data,struct vfio_irq_set hdr;u8 *data =NULL ;int ret =0 ; +int max;size_t data_size =0 ; minsz = offsetofend(struct vfio_irq_set, count);@@ -372,8 +508,14 @@ static long vfio_platform_ioctl(void *device_data,if (copy_from_user (&hdr, (void __user *)arg, minsz))return -EFAULT ; - ret =vfio_set_irqs_validate_and_prepare (&hdr, vdev->num_irqs, - vdev->num_irqs, &data_size); +if (hdr.index >= vdev->num_irqs) +return -EINVAL ; + + max = vdev->irqs[hdr.index].count; + + ret =vfio_set_irqs_validate_and_prepare (&hdr, max, + vdev->num_irqs, + &data_size);if (ret)return ret;@@ -678,6 +820,10 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,return ret; }Quoted 7 lines
drivers/vfio/platform/vfio_platform_common.c int vfio_platform_probe_common(struct vfio_platform_device *vdev, · line 825
Quoted 3 lines
drivers/vfio/platform/vfio_platform_common.c +3
-0+ ret =vfio_platform_get_msi (vdev); +if (ret) +dev_info (vdev->device,"msi not supported\n" );I don't think we should output this message. This is printed for amd-xgbe which does not have msi so this is misleading. I would say either the
vfio_platform_get_msi()can return an actual error or we return a void?Will check on this what
Quoted 522 lines
Thanks
Eric
Quoted 517 lines
drivers/vfio/platform/vfio_platform_common.c +26
-0+ group =vfio_iommu_group_get (dev);if (!group) {dev_err (dev,"No IOMMU group for device %s\n" , vdev->name);@@ -697,6 +843,7 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev, put_iommu:vfio_iommu_group_put (group, dev); put_reset: +vfio_platform_put_msi (vdev);vfio_platform_put_reset (vdev);return ret; }@@ -744,6 +891,30 @@ void vfio_platform_unregister_reset(const char *compat, }EXPORT_SYMBOL_GPL (vfio_platform_unregister_reset); +void __vfio_platform_register_msi (struct vfio_platform_msi_node *node) +{ +mutex_lock (&driver_lock); +list_add (&node->link, &msi_list); +mutex_unlock (&driver_lock); +} +EXPORT_SYMBOL_GPL (__vfio_platform_register_msi); + +void vfio_platform_unregister_msi (const char *compat) +{ +struct vfio_platform_msi_node *iter, *temp; + +mutex_lock (&driver_lock); +list_for_each_entry_safe (iter, temp, &msi_list, link) { +if (!strcmp (iter->compat, compat)) { +list_del (&iter->link); +break ; + } + } + +mutex_unlock (&driver_lock); +} +EXPORT_SYMBOL_GPL (vfio_platform_unregister_msi); +MODULE_VERSION (DRIVER_VERSION );MODULE_LICENSE ("GPL v2" );MODULE_AUTHOR (DRIVER_AUTHOR );drivers/vfio/platform/vfio_platform_irq.c +245
-15@@ -8,10 +8,12 @@#include <linux/eventfd.h> #include <linux/interrupt.h> +#include <linux/eventfd.h> #include <linux/slab.h> #include <linux/types.h> #include <linux/vfio.h> #include <linux/irq.h> +#include <linux/msi.h> #include "vfio_platform_private.h" @@ -253,6 +255,195 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,return 0 ; } +/* MSI/MSIX */ +static irqreturn_t vfio_msihandler (int irq,void *arg) +{ +struct eventfd_ctx *trigger = arg; + +eventfd_signal (trigger,1 ); +return IRQ_HANDLED ; +} + +static void msi_write (struct msi_desc *desc,struct msi_msg *msg) +{ +int i; +struct vfio_platform_irq *irq; +u16 index = desc->platform.msi_index; +struct device *dev =msi_desc_to_dev (desc); +struct vfio_device *device =dev_get_drvdata (dev); +struct vfio_platform_device *vdev = (struct vfio_platform_device *) +vfio_device_data (device); + +for (i =0 ; i < vdev->num_irqs; i++) +if (vdev->irqs[i].type ==VFIO_IRQ_TYPE_MSI ) + irq = &vdev->irqs[i]; + + irq->ctx[index].msg.address_lo = msg->address_lo; + irq->ctx[index].msg.address_hi = msg->address_hi; + irq->ctx[index].msg.data = msg->data; +} + +static int vfio_msi_enable (struct vfio_platform_device *vdev, +struct vfio_platform_irq *irq,int nvec) +{ +int ret; +int msi_idx =0 ; +struct msi_desc *desc; +struct device *dev = vdev->device; + + irq->ctx =kcalloc (nvec,sizeof (struct vfio_irq_ctx ),GFP_KERNEL ); +if (!irq->ctx) +return -ENOMEM ; + +/* Allocate platform MSIs */ + ret =platform_msi_domain_alloc_irqs (dev, nvec, msi_write); +if (ret <0 ) { +kfree (irq->ctx); +return ret; + } + +for_each_msi_entry (desc, dev) { + irq->ctx[msi_idx].hwirq = desc->irq; + msi_idx++; + } + + irq->num_ctx = nvec; + irq->config_msi =1 ; + +return 0 ; +} + +static int vfio_msi_set_vector_signal (struct vfio_platform_irq *irq, +int vector,int fd) +{ +struct eventfd_ctx *trigger; +int irq_num, ret; + +if (vector <0 || vector >= irq->num_ctx) +return -EINVAL ; + + irq_num = irq->ctx[vector].hwirq; + +if (irq->ctx[vector].trigger) { +free_irq (irq_num, irq->ctx[vector].trigger); +kfree (irq->ctx[vector].name); +eventfd_ctx_put (irq->ctx[vector].trigger); + irq->ctx[vector].trigger =NULL ; + } + +if (fd <0 ) +return 0 ; + + irq->ctx[vector].name =kasprintf (GFP_KERNEL , +"vfio-msi[%d]" , vector); +if (!irq->ctx[vector].name) +return -ENOMEM ; + + trigger =eventfd_ctx_fdget (fd); +if (IS_ERR (trigger)) { +kfree (irq->ctx[vector].name); +return PTR_ERR (trigger); + } + + ret =request_irq (irq_num, vfio_msihandler,0 , + irq->ctx[vector].name, trigger); +if (ret) { +kfree (irq->ctx[vector].name); +eventfd_ctx_put (trigger); +return ret; + } + + irq->ctx[vector].trigger = trigger; + +return 0 ; +} + +static int vfio_msi_set_block (struct vfio_platform_irq *irq,unsigned int start, +unsigned int count,int32_t *fds) +{ +int i, j, ret =0 ; + +if (start >= irq->num_ctx || start + count > irq->num_ctx) +return -EINVAL ; + +for (i =0 , j = start; i < count && !ret; i++, j++) { +int fd = fds ? fds[i] :-1 ; + + ret =vfio_msi_set_vector_signal (irq, j, fd); + } + +if (ret) { +for (--j; j >= (int )start; j--) +vfio_msi_set_vector_signal (irq, j,-1 ); + } + +return ret; +} + +static void vfio_msi_disable (struct vfio_platform_device *vdev, +struct vfio_platform_irq *irq) +{ +struct device *dev = vdev->device; + +vfio_msi_set_block (irq,0 , irq->num_ctx,NULL ); + +platform_msi_domain_free_irqs (dev); + + irq->config_msi =0 ; + irq->num_ctx =0 ; + +kfree (irq->ctx); +} + +static int vfio_set_msi_trigger (struct vfio_platform_device *vdev, +unsigned int index,unsigned int start, +unsigned int count,uint32_t flags,void *data) +{ +int i; +struct vfio_platform_irq *irq = &vdev->irqs[index]; + +if (start + count > irq->count) +return -EINVAL ; + +if (!count && (flags &VFIO_IRQ_SET_DATA_NONE )) { +vfio_msi_disable (vdev, irq); +return 0 ; + } + +if (flags &VFIO_IRQ_SET_DATA_EVENTFD ) { +s32 *fds = data; +int ret; + +if (irq->config_msi) +return vfio_msi_set_block (irq, start, count, + fds); + ret =vfio_msi_enable (vdev, irq, start + count); +if (ret) +return ret; + + ret =vfio_msi_set_block (irq, start, count, fds); +if (ret) +vfio_msi_disable (vdev, irq); + +return ret; + } + +for (i = start; i < start + count; i++) { +if (!irq->ctx[i].trigger) +continue ; +if (flags &VFIO_IRQ_SET_DATA_NONE ) { +eventfd_signal (irq->ctx[i].trigger,1 ); + }else if (flags &VFIO_IRQ_SET_DATA_BOOL ) { +u8 *bools = data; + +if (bools[i - start]) +eventfd_signal (irq->ctx[i].trigger,1 ); + } + } + +return 0 ; +} +int vfio_platform_set_irqs_ioctl (struct vfio_platform_device *vdev,uint32_t flags,unsigned index,unsigned start,unsigned count,void *data)@@ -261,16 +452,29 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev, unsigned start, unsigned count, uint32_t flags, void *data) = NULL; -switch (flags &VFIO_IRQ_SET_ACTION_TYPE_MASK ) { -case VFIO_IRQ_SET_ACTION_MASK : - func = vfio_platform_set_irq_mask; -break ; -case VFIO_IRQ_SET_ACTION_UNMASK : - func = vfio_platform_set_irq_unmask; -break ; -case VFIO_IRQ_SET_ACTION_TRIGGER : - func = vfio_platform_set_irq_trigger; -break ; +struct vfio_platform_irq *irq = &vdev->irqs[index]; + +if (irq->type ==VFIO_IRQ_TYPE_MSI ) { +switch (flags &VFIO_IRQ_SET_ACTION_TYPE_MASK ) { +case VFIO_IRQ_SET_ACTION_MASK : +case VFIO_IRQ_SET_ACTION_UNMASK : +break ; +case VFIO_IRQ_SET_ACTION_TRIGGER : + func = vfio_set_msi_trigger; +break ; + } + }else { +switch (flags &VFIO_IRQ_SET_ACTION_TYPE_MASK ) { +case VFIO_IRQ_SET_ACTION_MASK : + func = vfio_platform_set_irq_mask; +break ; +case VFIO_IRQ_SET_ACTION_UNMASK : + func = vfio_platform_set_irq_unmask; +break ; +case VFIO_IRQ_SET_ACTION_TRIGGER : + func = vfio_platform_set_irq_trigger; +break ; + } }if (!func)@@ -281,12 +485,21 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,int vfio_platform_irq_init (struct vfio_platform_device *vdev) { -int cnt =0 , i; +int i; +int cnt =0 ; +int num_irqs; +int msi_cnt =0 ;while (vdev->get_irq (vdev, cnt) >=0 ) cnt++; - vdev->irqs =kcalloc (cnt,sizeof (struct vfio_platform_irq ),GFP_KERNEL ); +if (vdev->of_get_msi) { + msi_cnt = vdev->of_get_msi (vdev); + num_irqs++; + } + + vdev->irqs =kcalloc (num_irqs,sizeof (struct vfio_platform_irq ), +GFP_KERNEL );if (!vdev->irqs)return -ENOMEM ;@@ -309,7 +522,19 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev) vdev->irqs[i].masked = false; } - vdev->num_irqs = cnt; +/* +* MSI block is added at last index and it is an ext irq +*/ +if (msi_cnt >0 ) { + vdev->irqs[i].flags =VFIO_IRQ_INFO_EVENTFD ; + vdev->irqs[i].count = msi_cnt; + vdev->irqs[i].hwirq =0 ; + vdev->irqs[i].masked = false; + vdev->irqs[i].type =VFIO_IRQ_TYPE_MSI ; + vdev->num_ext_irqs =1 ; + } + + vdev->num_irqs = num_irqs;return 0 ; err:@@ -321,8 +546,13 @@ void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev) {int i; -for (i =0 ; i < vdev->num_irqs; i++) -vfio_set_trigger (vdev, i,-1 ,NULL ); +for (i =0 ; i < vdev->num_irqs; i++) { +if (vdev->irqs[i].type ==VFIO_IRQ_TYPE_MSI ) +vfio_set_msi_trigger (vdev, i,0 ,0 , +VFIO_IRQ_SET_DATA_NONE ,NULL ); +else +vfio_set_trigger (vdev, i,-1 ,NULL ); + } vdev->num_irqs =0 ;kfree (vdev->irqs);drivers/vfio/platform/vfio_platform_private.h +32
-0@@ -9,6 +9,7 @@#include <linux/types.h> #include <linux/interrupt.h> +#include <linux/msi.h> #define VFIO_PLATFORM_OFFSET_SHIFT 40#define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)@@ -19,9 +20,21 @@#define VFIO_PLATFORM_INDEX_TO_OFFSET (index) \ ((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT) +/* IRQ index for MSI in ext IRQs */ +#define VFIO_EXT_IRQ_MSI 0 + +struct vfio_irq_ctx { +int hwirq; +char *name; +struct msi_msg msg; +struct eventfd_ctx *trigger; +}; +struct vfio_platform_irq {u32 flags;u32 count; +int num_ctx; +struct vfio_irq_ctx *ctx;int hwirq;char *name;struct eventfd_ctx *trigger;@@ -29,6 +42,11 @@ struct vfio_platform_irq {spinlock_t lock;struct virqfd *unmask;struct virqfd *mask; + +/* for extended irqs */ +u32 type; +u32 subtype; +int config_msi; };struct vfio_platform_region {@@ -46,12 +64,14 @@ struct vfio_platform_device {u32 num_regions;struct vfio_platform_irq *irqs;u32 num_irqs; +int num_ext_irqs;int refcnt;struct mutex igate;struct module *parent_module;const char *compat;const char *acpihid;struct module *reset_module; +struct module *msi_module;struct device *device;/* @@ -65,11 +85,13 @@ struct vfio_platform_device { (*get_resource)(struct vfio_platform_device *vdev, int i);int (*get_irq)(struct vfio_platform_device *vdev,int i);int (*of_reset)(struct vfio_platform_device *vdev); +u32 (*of_get_msi)(struct vfio_platform_device *vdev);bool reset_required; };typedef int (*vfio_platform_reset_fn_t )(struct vfio_platform_device *vdev); +typedef u32 (*vfio_platform_get_msi_fn_t )(struct vfio_platform_device *vdev);struct vfio_platform_reset_node {struct list_head link;@@ -78,6 +100,13 @@ struct vfio_platform_reset_node {vfio_platform_reset_fn_t of_reset; }; +struct vfio_platform_msi_node { +struct list_head link; +char *compat; +struct module *owner; +vfio_platform_get_msi_fn_t of_get_msi; +}; +extern int vfio_platform_probe_common (struct vfio_platform_device *vdev,struct device *dev);extern struct vfio_platform_device *vfio_platform_remove_common@@ -94,6 +123,9 @@ extern int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,extern void __vfio_platform_register_reset (struct vfio_platform_reset_node *n);extern void vfio_platform_unregister_reset (const char *compat,vfio_platform_reset_fn_t fn); +void __vfio_platform_register_msi (struct vfio_platform_msi_node *n); +void vfio_platform_unregister_msi (const char *compat); +#define vfio_platform_register_reset (__compat, __reset) \ static struct vfio_platform_reset_node __reset ## _node = { \ .owner = THIS_MODULE, \include/uapi/linux/vfio.h +44
-0@@ -697,11 +697,55 @@ struct vfio_irq_info {#define VFIO_IRQ_INFO_MASKABLE (1 << 1)#define VFIO_IRQ_INFO_AUTOMASKED (1 << 2)#define VFIO_IRQ_INFO_NORESIZE (1 << 3) +#define VFIO_IRQ_INFO_FLAG_CAPS (1 << 4)/* Info supports caps */ __u32 index;/* IRQ index */ __u32 count;/* Number of IRQs within this index */ +__u32 cap_offset;/* Offset within info struct of first cap */ };#define VFIO_DEVICE_GET_IRQ_INFO _IO(VFIO_TYPE, VFIO_BASE + 9) +/* +* The irq type capability allows IRQs unique to a specific device or +* class of devices to be exposed. +* +* The structures below define version 1 of this capability. +*/ +#define VFIO_IRQ_INFO_CAP_TYPE 3 + +struct vfio_irq_info_cap_type { +struct vfio_info_cap_header header; +__u32 type;/* global per bus driver */ +__u32 subtype;/* type specific */ +}; + +/* +* List of IRQ types, global per bus driver. +* If you introduce a new type, please add it here. +*/ + +/* Non PCI devices having MSI(s) support */ +#define VFIO_IRQ_TYPE_MSI (1) + +/* +* The msi capability allows the user to use the msi msg to +* configure the iova for the msi configuration. +* The structures below define version 1 of this capability. +*/ +#define VFIO_IRQ_INFO_CAP_MSI_DESCS 4 + +struct vfio_irq_msi_msg { +__u32 addr_lo; +__u32 addr_hi; +__u32 data; +}; + +struct vfio_irq_info_cap_msi { +struct vfio_info_cap_header header; +__u32 nr_msgs; +__u32 reserved; +struct vfio_irq_msi_msg msgs[]; +}; +/** * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set) *
Auger Eric
maintainerRe: [RFC v3 1/2] vfio/platform: add support for msi
Hi Vikas,
Quoted 207 lines · Vikas Gupta
Hi Eric,
Quoted 177 lines · Auger Eric
Hi Vikas,
Quoted 169 lines · Vikas Gupta
Quoted 159 lines · Auger Eric
Hi Vikas,
Quoted 25 lines · Vikas Gupta
MSI support for platform devices.The MSI block is added as an extended IRQ which exports caps
VFIO_IRQ_INFO_CAP_TYPEandVFIO_IRQ_INFO_CAP_MSI_DESCS.Signed-off-by Vikas Gupta <[email protected]>--- drivers/vfio/platform/vfio_platform_common.c | 179 +++++++++++- drivers/vfio/platform/vfio_platform_irq.c | 260 +++++++++++++++++- drivers/vfio/platform/vfio_platform_private.h | 32 +++ include/uapi/linux/vfio.h | 44 +++ 4 files changed, 496 insertions(+), 19 deletions(-)drivers/vfio/platform/vfio_platform_common.c +1
-0@@ -16,6 +16,7 @@#include <linux/types.h> #include <linux/uaccess.h> #include <linux/vfio.h> +#include <linux/nospec.h> #include "vfio_platform_private.h" drivers/vfio/platform/vfio_platform_common.c line 30
Quoted 4 lines
drivers/vfio/platform/vfio_platform_common.c +1
-0@@ -26,6 +27,8 @@#define VFIO_PLATFORM_IS_ACPI (vdev) ((vdev)->acpihid != NULL)static LIST_HEAD (reset_list); +/* devices having MSI support */ nit: for devices using MSIs?
Quoted 125 lines
drivers/vfio/platform/vfio_platform_common.c +85
-2+static LIST_HEAD (msi_list);static DEFINE_MUTEX (driver_lock); static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,@@ -47,6 +50,25 @@ static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,return reset_fn; } +static bool vfio_platform_lookup_msi (struct vfio_platform_device *vdev) +{ +bool has_msi = false; +struct vfio_platform_msi_node *iter; + +mutex_lock (&driver_lock); +list_for_each_entry (iter, &msi_list, link) { +if (!strcmp (iter->compat, vdev->compat) && +try_module_get (iter->owner)) { + vdev->msi_module = iter->owner; + vdev->of_get_msi = iter->of_get_msi; + has_msi = true; +break ; + } + } +mutex_unlock (&driver_lock); +return has_msi; +} +static int vfio_platform_acpi_probe (struct vfio_platform_device *vdev,struct device *dev) {@@ -126,6 +148,19 @@ static int vfio_platform_get_reset(struct vfio_platform_device *vdev)return vdev->of_reset ?0 : -ENOENT ; } +static int vfio_platform_get_msi (struct vfio_platform_device *vdev) +{ +bool has_msi; + + has_msi =vfio_platform_lookup_msi (vdev); +if (!has_msi) { +request_module ("vfio-msi:%s" , vdev->compat); + has_msi =vfio_platform_lookup_msi (vdev); + } + +return has_msi ?0 : -ENOENT ; +} +static void vfio_platform_put_reset (struct vfio_platform_device *vdev) {if (VFIO_PLATFORM_IS_ACPI (vdev))@@ -135,6 +170,12 @@ static void vfio_platform_put_reset(struct vfio_platform_device *vdev)module_put (vdev->reset_module); } +static void vfio_platform_put_msi (struct vfio_platform_device *vdev) +{ +if (vdev->of_get_msi) +module_put (vdev->msi_module); +} +static int vfio_platform_regions_init (struct vfio_platform_device *vdev) {int cnt =0 , i;@@ -343,9 +384,17 @@ static long vfio_platform_ioctl(void *device_data, }else if (cmd ==VFIO_DEVICE_GET_IRQ_INFO ) {struct vfio_irq_info info; +struct vfio_info_cap caps = { .buf =NULL , .size =0 }; +struct vfio_irq_info_cap_msi *msi_info =NULL ; +int ext_irq_index = vdev->num_irqs - vdev->num_ext_irqs; +unsigned long capsz; +u32 index; minsz = offsetofend(struct vfio_irq_info, count); +/* For backward compatibility, cannot require this */ + capsz = offsetofend(struct vfio_irq_info, cap_offset); +if (copy_from_user (&info, (void __user *)arg, minsz))return -EFAULT ;@@ -355,8 +404,94 @@ static long vfio_platform_ioctl(void *device_data,if (info.index >= vdev->num_irqs)return -EINVAL ; - info.flags = vdev->irqs[info.index].flags; - info.count = vdev->irqs[info.index].count; +if (info.argsz >= capsz) + minsz = capsz; + + index = info.index; + + info.flags = vdev->irqs[index].flags; + info.count = vdev->irqs[index].count; + +if (ext_irq_index - index ==VFIO_EXT_IRQ_MSI ) { +struct vfio_irq_info_cap_type cap_type = { + .header.id =VFIO_IRQ_INFO_CAP_TYPE , + .header.version =1 }; +struct vfio_platform_irq *irq; +size_t msi_info_size; +int num_msgs; +int ret; +int i; + + index =array_index_nospec (index, + vdev->num_irqs); + irq = &vdev->irqs[index]; + + cap_type.type = irq->type; + cap_type.subtype = irq->subtype; + + ret =vfio_info_add_capability (&caps, + &cap_type.header, +sizeof (cap_type)); +if (ret) +return ret; + + num_msgs = irq->num_ctx; +if (num_msgs) { + msi_info_size =struct_size (msi_info, + msgs, num_msgs); + + msi_info =kzalloc (msi_info_size,GFP_KERNEL ); +if (!msi_info) {Quoted 8 lines
drivers/vfio/platform/vfio_platform_common.c static long vfio_platform_ioctl(void *device_data, · line 450
Quoted 6 lines
drivers/vfio/platform/vfio_platform_common.c +6
-0+kfree (caps.buf); +return -ENOMEM ; + } + + msi_info->header.id = +VFIO_IRQ_INFO_CAP_MSI_DESCS ;I thought you would remove this cap now the module is back. What is the motivation to keep it?
Quoted 4 lines
Hi Eric,
The earlier module was serving two purposes
a) Max number of MSI(s) supported by the device. Since the module is
back this information can be filled in .count.Quoted 7 lines
understood
Quoted 3 lines
b) Writing
msi_msg(s)to devices for MSI sources.
We want b) to be handled in the user space so that they need not be
dependent on the kernel.I fail to understand the problem with the "dependency on the kernel". What is the functional difference when exposing this to the userspace, please could you highlight the benefits?
We do not see any functional difference here, the benefit we see is that user space can configure MSI sources with
msi_msgrather than doing it in the kernel.Quoted 7 lines
VFIO_IRQ_INFO_CAP_MSI_DESCSis helping userQuoted 3 lines
space to get MSI configuration data and thus they can independently configure MSI sources. For example, user space can just ask for ‘N’ MSI vectors and related
msi_msgfor ‘N’ vectorsthis is induced by
vfio_msi_enable(), right?Yes
Quoted 8 lines
is exported to user
Quoted 6 lines
space using this CAP. User space can now use ‘N’ vectors to configure any ‘N’ MSI sources and these sources might be in any order. However, this design is different from vfio-pci as MSI configuration is being handled in kernel only.
Let us know if implementingVFIO_IRQ_INFO_CAP_MSI_DESCSdoes not fit with the CAP framework.
Quoted 5 lines
Quoted 2 lines
I think it is OK reporting them through the cap but I fail to understand the benefits.
I believe, since there is no significant benefit this CAP should be avoided. I plan to remove this CAP in the next patch set.
OK thank you for the confirmation.
Thanks
Eric
Quoted 623 lines
Thanks,
VikasQuoted 620 lines
Thanks
Eric
Quoted 615 lines
Thanks,
VikasQuoted 80 lines
Thanks
Eric
Quoted 76 lines
drivers/vfio/platform/vfio_platform_common.c +53
-2+ msi_info->header.version =1 ; + msi_info->nr_msgs = num_msgs; + +for (i =0 ; i < num_msgs; i++) { +struct vfio_irq_ctx *ctx = &irq->ctx[i]; + + msi_info->msgs[i].addr_lo = + ctx->msg.address_lo; + msi_info->msgs[i].addr_hi = + ctx->msg.address_hi; + msi_info->msgs[i].data = ctx->msg.data; + } + + ret =vfio_info_add_capability (&caps, + &msi_info->header, + msi_info_size); +if (ret) { +kfree (msi_info); +kfree (caps.buf); +return ret; + } + } + } + +if (caps.size) { + info.flags |=VFIO_IRQ_INFO_FLAG_CAPS ; +if (info.argsz <sizeof (info) + caps.size) { + info.argsz =sizeof (info) + caps.size; + info.cap_offset =0 ; + }else { +vfio_info_cap_shift (&caps,sizeof (info)); +if (copy_to_user ((void __user *)arg + +sizeof (info), caps.buf, + caps.size)) { +kfree (msi_info); +kfree (caps.buf); +return -EFAULT ; + } + info.cap_offset =sizeof (info); + } + +kfree (msi_info); +kfree (caps.buf); + }return copy_to_user ((void __user *)arg, &info, minsz) ? -EFAULT :0 ;@@ -365,6 +500,7 @@ static long vfio_platform_ioctl(void *device_data,struct vfio_irq_set hdr;u8 *data =NULL ;int ret =0 ; +int max;size_t data_size =0 ; minsz = offsetofend(struct vfio_irq_set, count);@@ -372,8 +508,14 @@ static long vfio_platform_ioctl(void *device_data,if (copy_from_user (&hdr, (void __user *)arg, minsz))return -EFAULT ; - ret =vfio_set_irqs_validate_and_prepare (&hdr, vdev->num_irqs, - vdev->num_irqs, &data_size); +if (hdr.index >= vdev->num_irqs) +return -EINVAL ; + + max = vdev->irqs[hdr.index].count; + + ret =vfio_set_irqs_validate_and_prepare (&hdr, max, + vdev->num_irqs, + &data_size);if (ret)return ret;@@ -678,6 +820,10 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,return ret; }Quoted 7 lines
drivers/vfio/platform/vfio_platform_common.c int vfio_platform_probe_common(struct vfio_platform_device *vdev, · line 825
Quoted 3 lines
drivers/vfio/platform/vfio_platform_common.c +3
-0+ ret =vfio_platform_get_msi (vdev); +if (ret) +dev_info (vdev->device,"msi not supported\n" );I don't think we should output this message. This is printed for amd-xgbe which does not have msi so this is misleading. I would say either the
vfio_platform_get_msi()can return an actual error or we return a void?Will check on this what
Quoted 522 lines
Thanks
Eric
Quoted 517 lines
drivers/vfio/platform/vfio_platform_common.c +26
-0+ group =vfio_iommu_group_get (dev);if (!group) {dev_err (dev,"No IOMMU group for device %s\n" , vdev->name);@@ -697,6 +843,7 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev, put_iommu:vfio_iommu_group_put (group, dev); put_reset: +vfio_platform_put_msi (vdev);vfio_platform_put_reset (vdev);return ret; }@@ -744,6 +891,30 @@ void vfio_platform_unregister_reset(const char *compat, }EXPORT_SYMBOL_GPL (vfio_platform_unregister_reset); +void __vfio_platform_register_msi (struct vfio_platform_msi_node *node) +{ +mutex_lock (&driver_lock); +list_add (&node->link, &msi_list); +mutex_unlock (&driver_lock); +} +EXPORT_SYMBOL_GPL (__vfio_platform_register_msi); + +void vfio_platform_unregister_msi (const char *compat) +{ +struct vfio_platform_msi_node *iter, *temp; + +mutex_lock (&driver_lock); +list_for_each_entry_safe (iter, temp, &msi_list, link) { +if (!strcmp (iter->compat, compat)) { +list_del (&iter->link); +break ; + } + } + +mutex_unlock (&driver_lock); +} +EXPORT_SYMBOL_GPL (vfio_platform_unregister_msi); +MODULE_VERSION (DRIVER_VERSION );MODULE_LICENSE ("GPL v2" );MODULE_AUTHOR (DRIVER_AUTHOR );drivers/vfio/platform/vfio_platform_irq.c +245
-15@@ -8,10 +8,12 @@#include <linux/eventfd.h> #include <linux/interrupt.h> +#include <linux/eventfd.h> #include <linux/slab.h> #include <linux/types.h> #include <linux/vfio.h> #include <linux/irq.h> +#include <linux/msi.h> #include "vfio_platform_private.h" @@ -253,6 +255,195 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,return 0 ; } +/* MSI/MSIX */ +static irqreturn_t vfio_msihandler (int irq,void *arg) +{ +struct eventfd_ctx *trigger = arg; + +eventfd_signal (trigger,1 ); +return IRQ_HANDLED ; +} + +static void msi_write (struct msi_desc *desc,struct msi_msg *msg) +{ +int i; +struct vfio_platform_irq *irq; +u16 index = desc->platform.msi_index; +struct device *dev =msi_desc_to_dev (desc); +struct vfio_device *device =dev_get_drvdata (dev); +struct vfio_platform_device *vdev = (struct vfio_platform_device *) +vfio_device_data (device); + +for (i =0 ; i < vdev->num_irqs; i++) +if (vdev->irqs[i].type ==VFIO_IRQ_TYPE_MSI ) + irq = &vdev->irqs[i]; + + irq->ctx[index].msg.address_lo = msg->address_lo; + irq->ctx[index].msg.address_hi = msg->address_hi; + irq->ctx[index].msg.data = msg->data; +} + +static int vfio_msi_enable (struct vfio_platform_device *vdev, +struct vfio_platform_irq *irq,int nvec) +{ +int ret; +int msi_idx =0 ; +struct msi_desc *desc; +struct device *dev = vdev->device; + + irq->ctx =kcalloc (nvec,sizeof (struct vfio_irq_ctx ),GFP_KERNEL ); +if (!irq->ctx) +return -ENOMEM ; + +/* Allocate platform MSIs */ + ret =platform_msi_domain_alloc_irqs (dev, nvec, msi_write); +if (ret <0 ) { +kfree (irq->ctx); +return ret; + } + +for_each_msi_entry (desc, dev) { + irq->ctx[msi_idx].hwirq = desc->irq; + msi_idx++; + } + + irq->num_ctx = nvec; + irq->config_msi =1 ; + +return 0 ; +} + +static int vfio_msi_set_vector_signal (struct vfio_platform_irq *irq, +int vector,int fd) +{ +struct eventfd_ctx *trigger; +int irq_num, ret; + +if (vector <0 || vector >= irq->num_ctx) +return -EINVAL ; + + irq_num = irq->ctx[vector].hwirq; + +if (irq->ctx[vector].trigger) { +free_irq (irq_num, irq->ctx[vector].trigger); +kfree (irq->ctx[vector].name); +eventfd_ctx_put (irq->ctx[vector].trigger); + irq->ctx[vector].trigger =NULL ; + } + +if (fd <0 ) +return 0 ; + + irq->ctx[vector].name =kasprintf (GFP_KERNEL , +"vfio-msi[%d]" , vector); +if (!irq->ctx[vector].name) +return -ENOMEM ; + + trigger =eventfd_ctx_fdget (fd); +if (IS_ERR (trigger)) { +kfree (irq->ctx[vector].name); +return PTR_ERR (trigger); + } + + ret =request_irq (irq_num, vfio_msihandler,0 , + irq->ctx[vector].name, trigger); +if (ret) { +kfree (irq->ctx[vector].name); +eventfd_ctx_put (trigger); +return ret; + } + + irq->ctx[vector].trigger = trigger; + +return 0 ; +} + +static int vfio_msi_set_block (struct vfio_platform_irq *irq,unsigned int start, +unsigned int count,int32_t *fds) +{ +int i, j, ret =0 ; + +if (start >= irq->num_ctx || start + count > irq->num_ctx) +return -EINVAL ; + +for (i =0 , j = start; i < count && !ret; i++, j++) { +int fd = fds ? fds[i] :-1 ; + + ret =vfio_msi_set_vector_signal (irq, j, fd); + } + +if (ret) { +for (--j; j >= (int )start; j--) +vfio_msi_set_vector_signal (irq, j,-1 ); + } + +return ret; +} + +static void vfio_msi_disable (struct vfio_platform_device *vdev, +struct vfio_platform_irq *irq) +{ +struct device *dev = vdev->device; + +vfio_msi_set_block (irq,0 , irq->num_ctx,NULL ); + +platform_msi_domain_free_irqs (dev); + + irq->config_msi =0 ; + irq->num_ctx =0 ; + +kfree (irq->ctx); +} + +static int vfio_set_msi_trigger (struct vfio_platform_device *vdev, +unsigned int index,unsigned int start, +unsigned int count,uint32_t flags,void *data) +{ +int i; +struct vfio_platform_irq *irq = &vdev->irqs[index]; + +if (start + count > irq->count) +return -EINVAL ; + +if (!count && (flags &VFIO_IRQ_SET_DATA_NONE )) { +vfio_msi_disable (vdev, irq); +return 0 ; + } + +if (flags &VFIO_IRQ_SET_DATA_EVENTFD ) { +s32 *fds = data; +int ret; + +if (irq->config_msi) +return vfio_msi_set_block (irq, start, count, + fds); + ret =vfio_msi_enable (vdev, irq, start + count); +if (ret) +return ret; + + ret =vfio_msi_set_block (irq, start, count, fds); +if (ret) +vfio_msi_disable (vdev, irq); + +return ret; + } + +for (i = start; i < start + count; i++) { +if (!irq->ctx[i].trigger) +continue ; +if (flags &VFIO_IRQ_SET_DATA_NONE ) { +eventfd_signal (irq->ctx[i].trigger,1 ); + }else if (flags &VFIO_IRQ_SET_DATA_BOOL ) { +u8 *bools = data; + +if (bools[i - start]) +eventfd_signal (irq->ctx[i].trigger,1 ); + } + } + +return 0 ; +} +int vfio_platform_set_irqs_ioctl (struct vfio_platform_device *vdev,uint32_t flags,unsigned index,unsigned start,unsigned count,void *data)@@ -261,16 +452,29 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev, unsigned start, unsigned count, uint32_t flags, void *data) = NULL; -switch (flags &VFIO_IRQ_SET_ACTION_TYPE_MASK ) { -case VFIO_IRQ_SET_ACTION_MASK : - func = vfio_platform_set_irq_mask; -break ; -case VFIO_IRQ_SET_ACTION_UNMASK : - func = vfio_platform_set_irq_unmask; -break ; -case VFIO_IRQ_SET_ACTION_TRIGGER : - func = vfio_platform_set_irq_trigger; -break ; +struct vfio_platform_irq *irq = &vdev->irqs[index]; + +if (irq->type ==VFIO_IRQ_TYPE_MSI ) { +switch (flags &VFIO_IRQ_SET_ACTION_TYPE_MASK ) { +case VFIO_IRQ_SET_ACTION_MASK : +case VFIO_IRQ_SET_ACTION_UNMASK : +break ; +case VFIO_IRQ_SET_ACTION_TRIGGER : + func = vfio_set_msi_trigger; +break ; + } + }else { +switch (flags &VFIO_IRQ_SET_ACTION_TYPE_MASK ) { +case VFIO_IRQ_SET_ACTION_MASK : + func = vfio_platform_set_irq_mask; +break ; +case VFIO_IRQ_SET_ACTION_UNMASK : + func = vfio_platform_set_irq_unmask; +break ; +case VFIO_IRQ_SET_ACTION_TRIGGER : + func = vfio_platform_set_irq_trigger; +break ; + } }if (!func)@@ -281,12 +485,21 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,int vfio_platform_irq_init (struct vfio_platform_device *vdev) { -int cnt =0 , i; +int i; +int cnt =0 ; +int num_irqs; +int msi_cnt =0 ;while (vdev->get_irq (vdev, cnt) >=0 ) cnt++; - vdev->irqs =kcalloc (cnt,sizeof (struct vfio_platform_irq ),GFP_KERNEL ); +if (vdev->of_get_msi) { + msi_cnt = vdev->of_get_msi (vdev); + num_irqs++; + } + + vdev->irqs =kcalloc (num_irqs,sizeof (struct vfio_platform_irq ), +GFP_KERNEL );if (!vdev->irqs)return -ENOMEM ;@@ -309,7 +522,19 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev) vdev->irqs[i].masked = false; } - vdev->num_irqs = cnt; +/* +* MSI block is added at last index and it is an ext irq +*/ +if (msi_cnt >0 ) { + vdev->irqs[i].flags =VFIO_IRQ_INFO_EVENTFD ; + vdev->irqs[i].count = msi_cnt; + vdev->irqs[i].hwirq =0 ; + vdev->irqs[i].masked = false; + vdev->irqs[i].type =VFIO_IRQ_TYPE_MSI ; + vdev->num_ext_irqs =1 ; + } + + vdev->num_irqs = num_irqs;return 0 ; err:@@ -321,8 +546,13 @@ void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev) {int i; -for (i =0 ; i < vdev->num_irqs; i++) -vfio_set_trigger (vdev, i,-1 ,NULL ); +for (i =0 ; i < vdev->num_irqs; i++) { +if (vdev->irqs[i].type ==VFIO_IRQ_TYPE_MSI ) +vfio_set_msi_trigger (vdev, i,0 ,0 , +VFIO_IRQ_SET_DATA_NONE ,NULL ); +else +vfio_set_trigger (vdev, i,-1 ,NULL ); + } vdev->num_irqs =0 ;kfree (vdev->irqs);drivers/vfio/platform/vfio_platform_private.h +32
-0@@ -9,6 +9,7 @@#include <linux/types.h> #include <linux/interrupt.h> +#include <linux/msi.h> #define VFIO_PLATFORM_OFFSET_SHIFT 40#define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)@@ -19,9 +20,21 @@#define VFIO_PLATFORM_INDEX_TO_OFFSET (index) \ ((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT) +/* IRQ index for MSI in ext IRQs */ +#define VFIO_EXT_IRQ_MSI 0 + +struct vfio_irq_ctx { +int hwirq; +char *name; +struct msi_msg msg; +struct eventfd_ctx *trigger; +}; +struct vfio_platform_irq {u32 flags;u32 count; +int num_ctx; +struct vfio_irq_ctx *ctx;int hwirq;char *name;struct eventfd_ctx *trigger;@@ -29,6 +42,11 @@ struct vfio_platform_irq {spinlock_t lock;struct virqfd *unmask;struct virqfd *mask; + +/* for extended irqs */ +u32 type; +u32 subtype; +int config_msi; };struct vfio_platform_region {@@ -46,12 +64,14 @@ struct vfio_platform_device {u32 num_regions;struct vfio_platform_irq *irqs;u32 num_irqs; +int num_ext_irqs;int refcnt;struct mutex igate;struct module *parent_module;const char *compat;const char *acpihid;struct module *reset_module; +struct module *msi_module;struct device *device;/* @@ -65,11 +85,13 @@ struct vfio_platform_device { (*get_resource)(struct vfio_platform_device *vdev, int i);int (*get_irq)(struct vfio_platform_device *vdev,int i);int (*of_reset)(struct vfio_platform_device *vdev); +u32 (*of_get_msi)(struct vfio_platform_device *vdev);bool reset_required; };typedef int (*vfio_platform_reset_fn_t )(struct vfio_platform_device *vdev); +typedef u32 (*vfio_platform_get_msi_fn_t )(struct vfio_platform_device *vdev);struct vfio_platform_reset_node {struct list_head link;@@ -78,6 +100,13 @@ struct vfio_platform_reset_node {vfio_platform_reset_fn_t of_reset; }; +struct vfio_platform_msi_node { +struct list_head link; +char *compat; +struct module *owner; +vfio_platform_get_msi_fn_t of_get_msi; +}; +extern int vfio_platform_probe_common (struct vfio_platform_device *vdev,struct device *dev);extern struct vfio_platform_device *vfio_platform_remove_common@@ -94,6 +123,9 @@ extern int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,extern void __vfio_platform_register_reset (struct vfio_platform_reset_node *n);extern void vfio_platform_unregister_reset (const char *compat,vfio_platform_reset_fn_t fn); +void __vfio_platform_register_msi (struct vfio_platform_msi_node *n); +void vfio_platform_unregister_msi (const char *compat); +#define vfio_platform_register_reset (__compat, __reset) \ static struct vfio_platform_reset_node __reset ## _node = { \ .owner = THIS_MODULE, \include/uapi/linux/vfio.h +44
-0@@ -697,11 +697,55 @@ struct vfio_irq_info {#define VFIO_IRQ_INFO_MASKABLE (1 << 1)#define VFIO_IRQ_INFO_AUTOMASKED (1 << 2)#define VFIO_IRQ_INFO_NORESIZE (1 << 3) +#define VFIO_IRQ_INFO_FLAG_CAPS (1 << 4)/* Info supports caps */ __u32 index;/* IRQ index */ __u32 count;/* Number of IRQs within this index */ +__u32 cap_offset;/* Offset within info struct of first cap */ };#define VFIO_DEVICE_GET_IRQ_INFO _IO(VFIO_TYPE, VFIO_BASE + 9) +/* +* The irq type capability allows IRQs unique to a specific device or +* class of devices to be exposed. +* +* The structures below define version 1 of this capability. +*/ +#define VFIO_IRQ_INFO_CAP_TYPE 3 + +struct vfio_irq_info_cap_type { +struct vfio_info_cap_header header; +__u32 type;/* global per bus driver */ +__u32 subtype;/* type specific */ +}; + +/* +* List of IRQ types, global per bus driver. +* If you introduce a new type, please add it here. +*/ + +/* Non PCI devices having MSI(s) support */ +#define VFIO_IRQ_TYPE_MSI (1) + +/* +* The msi capability allows the user to use the msi msg to +* configure the iova for the msi configuration. +* The structures below define version 1 of this capability. +*/ +#define VFIO_IRQ_INFO_CAP_MSI_DESCS 4 + +struct vfio_irq_msi_msg { +__u32 addr_lo; +__u32 addr_hi; +__u32 data; +}; + +struct vfio_irq_info_cap_msi { +struct vfio_info_cap_header header; +__u32 nr_msgs; +__u32 reserved; +struct vfio_irq_msi_msg msgs[]; +}; +/** * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set) *
kernel test robot
build botRe: [RFC v3 1/2] vfio/platform: add support for msi
In reply to Vikas Gupta[If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in https://git-scm.com/docs/git-format-patch]
Hi Vikas,
[FYI, it's a private test report for your RFC patch.]
[auto build test ERROR on vfio/next]
[also build test ERROR on v5.10 next-20201223]
[If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]
url: https://github.com/0day-ci/linux/commits/Vikas-Gupta/msi-support-for-platform-devices/20201215-015115 base: https://github.com/awilliam/linux-vfio.git next config: arm-randconfig-r021-20201221 (attached as .config) compiler: arm-linux-gnueabi-gcc (GCC) 9.3.0
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# https://github.com/0day-ci/linux/commit/029a6fee02b683d688d7e6e685c78127ac29df8a
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Vikas-Gupta/msi-support-for-platform-devices/20201215-015115 git checkout 029a6fee02b683d688d7e6e685c78127ac29df8a
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=arm
If you fix the issue, kindly add following tag as appropriate
All errors (new ones prefixed by >>):
drivers/vfio/platform/vfio_platform_irq.c: In function 'vfio_msi_enable':
>> drivers/vfio/platform/vfio_platform_irq.c:299:8: error: implicit declaration of function 'platform_msi_domain_alloc_irqs' [-Werror=implicit-function-declaration]
299 | ret = platform_msi_domain_alloc_irqs(dev, nvec, msi_write);
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
In file included from include/linux/list.h:9,
from include/linux/wait.h:7,
from include/linux/eventfd.h:13,
from drivers/vfio/platform/vfio_platform_irq.c:9:
>> include/linux/msi.h:135:38: error: 'struct device' has no member named 'msi_list'
135 | #define dev_to_msi_list(dev) (&(dev)->msi_list)
| ^~
include/linux/kernel.h:852:26: note: in definition of macro 'container_of'
852 | void *__mptr = (void *)(ptr); \
| ^~~
include/linux/list.h:522:2: note: in expansion of macro 'list_entry'
522 | list_entry((ptr)->next, type, member)
| ^~~~~~~~~~
include/linux/list.h:628:13: note: in expansion of macro 'list_first_entry'
628 | for (pos = list_first_entry(head, typeof(*pos), member); \
| ^~~~~~~~~~~~~~~~
include/linux/msi.h:139:2: note: in expansion of macro 'list_for_each_entry'
139 | list_for_each_entry((desc), dev_to_msi_list((dev)), list)
| ^~~~~~~~~~~~~~~~~~~
include/linux/msi.h:139:30: note: in expansion of macro 'dev_to_msi_list'
139 | list_for_each_entry((desc), dev_to_msi_list((dev)), list)
| ^~~~~~~~~~~~~~~
drivers/vfio/platform/vfio_platform_irq.c:305:2: note: in expansion of macro 'for_each_msi_entry'
305 | for_each_msi_entry(desc, dev) {
| ^~~~~~~~~~~~~~~~~~
In file included from <command-line>:
>> include/linux/msi.h:135:38: error: 'struct device' has no member named 'msi_list'
135 | #define dev_to_msi_list(dev) (&(dev)->msi_list)
| ^~
include/linux/compiler_types.h:295:9: note: in definition of macro '__compiletime_assert'
295 | if (!(condition)) \
| ^~~~~~~~~
include/linux/compiler_types.h:315:2: note: in expansion of macro '_compiletime_assert'
315 | _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__)
| ^~~~~~~~~~~~~~~~~~~
include/linux/build_bug.h:39:37: note: in expansion of macro 'compiletime_assert'
39 | #define BUILD_BUG_ON_MSG(cond, msg) compiletime_assert(!(cond), msg)
| ^~~~~~~~~~~~~~~~~~
include/linux/kernel.h:853:2: note: in expansion of macro 'BUILD_BUG_ON_MSG'
853 | BUILD_BUG_ON_MSG(!__same_type(*(ptr), ((type *)0)->member) && \
| ^~~~~~~~~~~~~~~~
include/linux/kernel.h:853:20: note: in expansion of macro '__same_type'
853 | BUILD_BUG_ON_MSG(!__same_type(*(ptr), ((type *)0)->member) && \
| ^~~~~~~~~~~
include/linux/list.h:511:2: note: in expansion of macro 'container_of'
511 | container_of(ptr, type, member)
| ^~~~~~~~~~~~
include/linux/list.h:522:2: note: in expansion of macro 'list_entry'
522 | list_entry((ptr)->next, type, member)
| ^~~~~~~~~~
include/linux/list.h:628:13: note: in expansion of macro 'list_first_entry'
628 | for (pos = list_first_entry(head, typeof(*pos), member); \
| ^~~~~~~~~~~~~~~~
include/linux/msi.h:139:2: note: in expansion of macro 'list_for_each_entry'
139 | list_for_each_entry((desc), dev_to_msi_list((dev)), list)
| ^~~~~~~~~~~~~~~~~~~
include/linux/msi.h:139:30: note: in expansion of macro 'dev_to_msi_list'
139 | list_for_each_entry((desc), dev_to_msi_list((dev)), list)
| ^~~~~~~~~~~~~~~
drivers/vfio/platform/vfio_platform_irq.c:305:2: note: in expansion of macro 'for_each_msi_entry'
305 | for_each_msi_entry(desc, dev) {
| ^~~~~~~~~~~~~~~~~~
>> include/linux/msi.h:135:38: error: 'struct device' has no member named 'msi_list'
135 | #define dev_to_msi_list(dev) (&(dev)->msi_list)
| ^~
include/linux/compiler_types.h:295:9: note: in definition of macro '__compiletime_assert'
295 | if (!(condition)) \
| ^~~~~~~~~
include/linux/compiler_types.h:315:2: note: in expansion of macro '_compiletime_assert'
315 | _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__)
| ^~~~~~~~~~~~~~~~~~~
include/linux/build_bug.h:39:37: note: in expansion of macro 'compiletime_assert'
39 | #define BUILD_BUG_ON_MSG(cond, msg) compiletime_assert(!(cond), msg)
| ^~~~~~~~~~~~~~~~~~
include/linux/kernel.h:853:2: note: in expansion of macro 'BUILD_BUG_ON_MSG'
853 | BUILD_BUG_ON_MSG(!__same_type(*(ptr), ((type *)0)->member) && \
| ^~~~~~~~~~~~~~~~
include/linux/kernel.h:854:6: note: in expansion of macro '__same_type'
854 | !__same_type(*(ptr), void), \
| ^~~~~~~~~~~
include/linux/list.h:511:2: note: in expansion of macro 'container_of'
511 | container_of(ptr, type, member)
| ^~~~~~~~~~~~
include/linux/list.h:522:2: note: in expansion of macro 'list_entry'
522 | list_entry((ptr)->next, type, member)
| ^~~~~~~~~~
include/linux/list.h:628:13: note: in expansion of macro 'list_first_entry'
628 | for (pos = list_first_entry(head, typeof(*pos), member); \
| ^~~~~~~~~~~~~~~~
include/linux/msi.h:139:2: note: in expansion of macro 'list_for_each_entry'
139 | list_for_each_entry((desc), dev_to_msi_list((dev)), list)
| ^~~~~~~~~~~~~~~~~~~
include/linux/msi.h:139:30: note: in expansion of macro 'dev_to_msi_list'
139 | list_for_each_entry((desc), dev_to_msi_list((dev)), list)
| ^~~~~~~~~~~~~~~
drivers/vfio/platform/vfio_platform_irq.c:305:2: note: in expansion of macro 'for_each_msi_entry'
305 | for_each_msi_entry(desc, dev) {
| ^~~~~~~~~~~~~~~~~~
In file included from include/linux/wait.h:7,
from include/linux/eventfd.h:13,
from drivers/vfio/platform/vfio_platform_irq.c:9:
>> include/linux/msi.h:135:38: error: 'struct device' has no member named 'msi_list'
135 | #define dev_to_msi_list(dev) (&(dev)->msi_list)
| ^~
include/linux/list.h:619:20: note: in definition of macro 'list_entry_is_head'
619 | (&pos->member == (head))
| ^~~~
include/linux/msi.h:139:2: note: in expansion of macro 'list_for_each_entry'
139 | list_for_each_entry((desc), dev_to_msi_list((dev)), list)
| ^~~~~~~~~~~~~~~~~~~
include/linux/msi.h:139:30: note: in expansion of macro 'dev_to_msi_list'
139 | list_for_each_entry((desc), dev_to_msi_list((dev)), list)
| ^~~~~~~~~~~~~~~
drivers/vfio/platform/vfio_platform_irq.c:305:2: note: in expansion of macro 'for_each_msi_entry'
305 | for_each_msi_entry(desc, dev) {
| ^~~~~~~~~~~~~~~~~~
drivers/vfio/platform/vfio_platform_irq.c: In function 'vfio_msi_disable':
>> drivers/vfio/platform/vfio_platform_irq.c:390:2: error: implicit declaration of function 'platform_msi_domain_free_irqs' [-Werror=implicit-function-declaration]
390 | platform_msi_domain_free_irqs(dev);
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~cc1: some warnings being treated as errors
vim +/platform_msi_domain_alloc_irqs +299 drivers/vfio/platform/vfio_platform_irq.c
285
286 static int vfio_msi_enable(struct vfio_platform_device *vdev,
287 struct vfio_platform_irq *irq, int nvec)
288 {
289 int ret;
290 int msi_idx = 0;
291 struct msi_desc *desc;
292 struct device *dev = vdev->device;
293
294 irq->ctx = kcalloc(nvec, sizeof(struct vfio_irq_ctx), GFP_KERNEL);
295 if (!irq->ctx)
296 return -ENOMEM;
297
298 /* Allocate platform MSIs */
> 299 ret = platform_msi_domain_alloc_irqs(dev, nvec, msi_write);
300 if (ret < 0) {
301 kfree(irq->ctx);
302 return ret;
303 }
304
305 for_each_msi_entry(desc, dev) {
306 irq->ctx[msi_idx].hwirq = desc->irq;
307 msi_idx++;
308 }
309
310 irq->num_ctx = nvec;
311 irq->config_msi = 1;
312
313 return 0;
314 }
315
316 static int vfio_msi_set_vector_signal(struct vfio_platform_irq *irq,
317 int vector, int fd)
318 {
319 struct eventfd_ctx *trigger;
320 int irq_num, ret;
321
322 if (vector < 0 || vector >= irq->num_ctx)
323 return -EINVAL;
324
325 irq_num = irq->ctx[vector].hwirq;
326
327 if (irq->ctx[vector].trigger) {
328 free_irq(irq_num, irq->ctx[vector].trigger);
329 kfree(irq->ctx[vector].name);
330 eventfd_ctx_put(irq->ctx[vector].trigger);
331 irq->ctx[vector].trigger = NULL;
332 }
333
334 if (fd < 0)
335 return 0;
336
337 irq->ctx[vector].name = kasprintf(GFP_KERNEL,
338 "vfio-msi[%d]", vector);
339 if (!irq->ctx[vector].name)
340 return -ENOMEM;
341
342 trigger = eventfd_ctx_fdget(fd);
343 if (IS_ERR(trigger)) {
344 kfree(irq->ctx[vector].name);
345 return PTR_ERR(trigger);
346 }
347
348 ret = request_irq(irq_num, vfio_msihandler, 0,
349 irq->ctx[vector].name, trigger);
350 if (ret) {
351 kfree(irq->ctx[vector].name);
352 eventfd_ctx_put(trigger);
353 return ret;
354 }
355
356 irq->ctx[vector].trigger = trigger;
357
358 return 0;
359 }
360
361 static int vfio_msi_set_block(struct vfio_platform_irq *irq, unsigned int start,
362 unsigned int count, int32_t *fds)
363 {
364 int i, j, ret = 0;
365
366 if (start >= irq->num_ctx || start + count > irq->num_ctx)
367 return -EINVAL;
368
369 for (i = 0, j = start; i < count && !ret; i++, j++) {
370 int fd = fds ? fds[i] : -1;
371
372 ret = vfio_msi_set_vector_signal(irq, j, fd);
373 }
374
375 if (ret) {
376 for (--j; j >= (int)start; j--)
377 vfio_msi_set_vector_signal(irq, j, -1);
378 }
379
380 return ret;
381 }
382
383 static void vfio_msi_disable(struct vfio_platform_device *vdev,
384 struct vfio_platform_irq *irq)
385 {
386 struct device *dev = vdev->device;
387
388 vfio_msi_set_block(irq, 0, irq->num_ctx, NULL);
389
> 390 platform_msi_domain_free_irqs(dev); 391
392 irq->config_msi = 0;
393 irq->num_ctx = 0;
394
395 kfree(irq->ctx);
396 }
397
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org
RFC v3 2/2 vfio/platform: msi: add Broadcom platform devices
In reply to Vikas GuptaAdd msi support for Broadcom platform devices
--- drivers/vfio/platform/Kconfig | 1 + drivers/vfio/platform/Makefile | 1 + drivers/vfio/platform/msi/Kconfig | 9 ++++ drivers/vfio/platform/msi/Makefile | 2 + .../vfio/platform/msi/vfio_platform_bcmplt.c | 49 +++++++++++++++++++ 5 files changed, 62 insertions(+) create mode 100644 drivers/vfio/platform/msi/Kconfig create mode 100644 drivers/vfio/platform/msi/Makefile create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
drivers/vfio/platform/Kconfig +1 -0
@@ -21,3 +21,4 @@ config VFIO_AMBA If you don't know what to do here, say N. source "drivers/vfio/platform/reset/Kconfig" +source "drivers/vfio/platform/msi/Kconfig"
drivers/vfio/platform/Makefile +1 -0
@@ -5,6 +5,7 @@ vfio-platform-y := vfio_platform.o obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform.o obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform-base.o obj-$(CONFIG_VFIO_PLATFORM) += reset/ +obj-$(CONFIG_VFIO_PLATFORM) += msi/ vfio-amba-y := vfio_amba.o
drivers/vfio/platform/msi/Kconfig new file +9 -0
@@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0-only +config VFIO_PLATFORM_BCMPLT_MSI + tristate "MSI support for Broadcom platform devices" + depends on VFIO_PLATFORM && (ARCH_BCM_IPROC || COMPILE_TEST) + default ARCH_BCM_IPROC + help + Enables the VFIO platform driver to handle msi for Broadcom devices + + If you don't know what to do here, say N.
drivers/vfio/platform/msi/Makefile new file +2 -0
@@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_VFIO_PLATFORM_BCMPLT_MSI) += vfio_platform_bcmplt.o
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +49 -0
49 lines not on this page · raw message
Signature · 15 lines
2.17.1 -- This electronic communication and the information and any files transmitted with it, or attached to it, are confidential and are intended solely for the use of the individual or entity to whom it is addressed and may contain information that is confidential, legally privileged, protected by privacy laws, or otherwise restricted from disclosure to anyone else. If you are not the intended recipient or the person responsible for delivering the e-mail to the intended recipient, you are hereby notified that any use, copying, distributing, dissemination, forwarding, printing, or copying of this e-mail is strictly prohibited. If you received this e-mail in error, please return the e-mail to the sender, delete it from your computer, and destroy any printed copy of it.
Auger Eric
maintainerRe: [RFC v3 2/2] vfio/platform: msi: add Broadcom platform devices
Hi Vikas,
Quoted 13 lines · Vikas Gupta
Add msi support for Broadcom platform devices
Signed-off-by Vikas Gupta <[email protected]>--- drivers/vfio/platform/Kconfig | 1 + drivers/vfio/platform/Makefile | 1 + drivers/vfio/platform/msi/Kconfig | 9 ++++ drivers/vfio/platform/msi/Makefile | 2 + .../vfio/platform/msi/vfio_platform_bcmplt.c | 49 +++++++++++++++++++ 5 files changed, 62 insertions(+) create mode 100644 drivers/vfio/platform/msi/Kconfig create mode 100644 drivers/vfio/platform/msi/Makefile create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
what does plt mean?
Quoted 59 lines
drivers/vfio/platform/Kconfig +1
-0@@ -21,3 +21,4 @@ config VFIO_AMBA If you don't know what to do here, say N. source "drivers/vfio/platform/reset/Kconfig" +source "drivers/vfio/platform/msi/Kconfig"drivers/vfio/platform/Makefile +1
-0@@ -5,6 +5,7 @@ vfio-platform-y := vfio_platform.o obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform.o obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform-base.o obj-$(CONFIG_VFIO_PLATFORM) += reset/ +obj-$(CONFIG_VFIO_PLATFORM) += msi/ vfio-amba-y := vfio_amba.odrivers/vfio/platform/msi/Kconfig new file +9
-0@@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0-only +config VFIO_PLATFORM_BCMPLT_MSI + tristate "MSI support for Broadcom platform devices" + depends on VFIO_PLATFORM && (ARCH_BCM_IPROC || COMPILE_TEST) + default ARCH_BCM_IPROC + help + Enables the VFIO platform driver to handle msi for Broadcom devices + + If you don't know what to do here, say N.drivers/vfio/platform/msi/Makefile new file +2
-0@@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_VFIO_PLATFORM_BCMPLT_MSI) += vfio_platform_bcmplt.odrivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +8
-0@@ -0,0 +1,49 @@ +// SPDX-License-Identifier: GPL-2.0 +/* +* Copyright 2020 Broadcom. +*/ + +#include <linux/module.h> +#include <linux/device.h> +#include <linux/interrupt.h>
drivers/vfio/platform/msi/vfio_platform_bcmplt.c line 18
Quoted 10 lines
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +10
-0+#include <linux/msi.h> +#include <linux/vfio.h> + +#include "../vfio_platform_private.h" + +#define RING_SIZE (64 << 10) + +#define RING_MSI_ADDR_LS 0x03c +#define RING_MSI_ADDR_MS 0x040 +#define RING_MSI_DATA_VALUE 0x064
Those 3 defines would not be needed anymore with that implementation option.
Quoted 18 lines
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +18
-0+ +static u32 bcm_num_msi (struct vfio_platform_device *vdev) +{ +struct vfio_platform_region *reg = &vdev->regions[0 ]; + +return (reg->size /RING_SIZE ); +} + +static struct vfio_platform_msi_node vfio_platform_bcmflexrm_msi_node = { + .owner =THIS_MODULE , + .compat ="brcm,iproc-flexrm-mbox" , + .of_get_msi = bcm_num_msi, +}; + +static int __init vfio_platform_bcmflexrm_msi_module_init (void ) +{ +__vfio_platform_register_msi (&vfio_platform_bcmflexrm_msi_node); +
drivers/vfio/platform/msi/vfio_platform_bcmplt.c line 46
Quoted 10 lines
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +10
-0+return 0 ; +} + +static void __exit vfio_platform_bcmflexrm_msi_module_exit(void) +{ +vfio_platform_unregister_msi ("brcm,iproc-flexrm-mbox" ); +} + +module_init (vfio_platform_bcmflexrm_msi_module_init); +module_exit (vfio_platform_bcmflexrm_msi_module_exit);
One thing I would like to discuss with Alex.
As the reset module is mandated (except if reset_required is forced to 0), I am wondering if we shouldn't try to turn the reset module into a "specialization" module and put the msi hooks there. I am afraid we may end up having modules for each and every vfio platform feature specialization. At the moment that's fully bearable but I can't predict what's next.
As the mandated feature is the reset capability maybe we could just keep the config/module name terminology, tune the kconfig help message to mention the msi support in case of flex-rm?
What do you think?
Thanks
Eric
Quoted 4 lines
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +3
-0+ +MODULE_LICENSE ("GPL v2" ); +MODULE_AUTHOR ("Broadcom" );
Re: [RFC v3 2/2] vfio/platform: msi: add Broadcom platform devices
Hi Eric,
Quoted 17 lines · Auger Eric
Hi Vikas,
Quoted 13 lines · Vikas Gupta
Add msi support for Broadcom platform devices
Signed-off-by Vikas Gupta <[email protected]>--- drivers/vfio/platform/Kconfig | 1 + drivers/vfio/platform/Makefile | 1 + drivers/vfio/platform/msi/Kconfig | 9 ++++ drivers/vfio/platform/msi/Makefile | 2 + .../vfio/platform/msi/vfio_platform_bcmplt.c | 49 +++++++++++++++++++ 5 files changed, 62 insertions(+) create mode 100644 drivers/vfio/platform/msi/Kconfig create mode 100644 drivers/vfio/platform/msi/Makefile create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
Quoted 1 line
what does plt mean?
This(plt) is a generic name for Broadcom platform devices, which we`ll
plan to add in this file. Currently we have only one in this file.
Do you think this name does not sound good here?
Quoted 107 lines
Quoted 59 lines
drivers/vfio/platform/Kconfig +1
-0@@ -21,3 +21,4 @@ config VFIO_AMBA If you don't know what to do here, say N. source "drivers/vfio/platform/reset/Kconfig" +source "drivers/vfio/platform/msi/Kconfig"drivers/vfio/platform/Makefile +1
-0@@ -5,6 +5,7 @@ vfio-platform-y := vfio_platform.o obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform.o obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform-base.o obj-$(CONFIG_VFIO_PLATFORM) += reset/ +obj-$(CONFIG_VFIO_PLATFORM) += msi/ vfio-amba-y := vfio_amba.odrivers/vfio/platform/msi/Kconfig new file +9
-0@@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0-only +config VFIO_PLATFORM_BCMPLT_MSI + tristate "MSI support for Broadcom platform devices" + depends on VFIO_PLATFORM && (ARCH_BCM_IPROC || COMPILE_TEST) + default ARCH_BCM_IPROC + help + Enables the VFIO platform driver to handle msi for Broadcom devices + + If you don't know what to do here, say N.drivers/vfio/platform/msi/Makefile new file +2
-0@@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_VFIO_PLATFORM_BCMPLT_MSI) += vfio_platform_bcmplt.odrivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +8
-0@@ -0,0 +1,49 @@ +// SPDX-License-Identifier: GPL-2.0 +/* +* Copyright 2020 Broadcom. +*/ + +#include <linux/module.h> +#include <linux/device.h> +#include <linux/interrupt.h> drivers/vfio/platform/msi/vfio_platform_bcmplt.c line 18
Quoted 10 lines
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +10
-0+#include <linux/msi.h> +#include <linux/vfio.h> + +#include "../vfio_platform_private.h" + +#define RING_SIZE (64 << 10) + +#define RING_MSI_ADDR_LS 0x03c +#define RING_MSI_ADDR_MS 0x040 +#define RING_MSI_DATA_VALUE 0x064Those 3 defines would not be needed anymore with that implementation option.
Quoted 18 lines
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +18
-0+ +static u32 bcm_num_msi (struct vfio_platform_device *vdev) +{ +struct vfio_platform_region *reg = &vdev->regions[0 ]; + +return (reg->size /RING_SIZE ); +} + +static struct vfio_platform_msi_node vfio_platform_bcmflexrm_msi_node = { + .owner =THIS_MODULE , + .compat ="brcm,iproc-flexrm-mbox" , + .of_get_msi = bcm_num_msi, +}; + +static int __init vfio_platform_bcmflexrm_msi_module_init (void ) +{ +__vfio_platform_register_msi (&vfio_platform_bcmflexrm_msi_node); +drivers/vfio/platform/msi/vfio_platform_bcmplt.c line 46
Quoted 10 lines
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +10
-0+return 0 ; +} + +static void __exit vfio_platform_bcmflexrm_msi_module_exit(void) +{ +vfio_platform_unregister_msi ("brcm,iproc-flexrm-mbox" ); +} + +module_init (vfio_platform_bcmflexrm_msi_module_init); +module_exit (vfio_platform_bcmflexrm_msi_module_exit);One thing I would like to discuss with Alex.
As the reset module is mandated (except if
reset_requiredis forced to 0), I am wondering if we shouldn't try to turn the reset module into a "specialization" module and put the msi hooks there. I am afraid we may end up having modules for each and every vfio platform feature specialization. At the moment that's fully bearable but I can't predict what's next.
Quoted 4 lines
As the mandated feature is the reset capability maybe we could just keep the config/module name terminology, tune the kconfig help message to mention the msi support in case of flex-rm?
As I understand, your proposal is that we should not have a separate module for MSI, rather we add in the existing reset module for flex-rm. Thus, this way reset modules do not seem to be specialized just for reset functionality only but for MSI as well. Apart from this we need not to load the proposed msi module in this patch series. Is my understanding correct?
For me it looks OK to consolidate MSI in the existing 'reset' module. Let me know your views so that I can work for the next patch set accordingly.
Thanks,
Vikas
Quoted 14 lines
What do you think?
Thanks
Eric
Quoted 4 lines
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +3
-0+ +MODULE_LICENSE ("GPL v2" ); +MODULE_AUTHOR ("Broadcom" );
Auger Eric
maintainerRe: [RFC v3 2/2] vfio/platform: msi: add Broadcom platform devices
Hi Vikas,
Quoted 20 lines · Vikas Gupta
Hi Eric,
Quoted 17 lines · Auger Eric
Hi Vikas,
Quoted 13 lines · Vikas Gupta
Add msi support for Broadcom platform devices
Signed-off-by Vikas Gupta <[email protected]>--- drivers/vfio/platform/Kconfig | 1 + drivers/vfio/platform/Makefile | 1 + drivers/vfio/platform/msi/Kconfig | 9 ++++ drivers/vfio/platform/msi/Makefile | 2 + .../vfio/platform/msi/vfio_platform_bcmplt.c | 49 +++++++++++++++++++ 5 files changed, 62 insertions(+) create mode 100644 drivers/vfio/platform/msi/Kconfig create mode 100644 drivers/vfio/platform/msi/Makefile create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
Quoted 4 lines
Quoted 1 line
what does plt mean?
This(plt) is a generic name for Broadcom platform devices, which we`ll
plan to add in this file. Currently we have only one in this file.
Do you think this name does not sound good here?
we have VFIO_PLATFORM_BCMFLEXRM_RESET config which also applied to vfio flex-rm platform device.
I think it would be more homegenous to have VFIO_PLATFORM_BCMFLEXRM_MSI in case we keep a separate msi module.
also in reset dir we have vfio_platform_bcmflexrm.c
Quoted 111 lines
Quoted 107 lines
Quoted 59 lines
drivers/vfio/platform/Kconfig +1
-0@@ -21,3 +21,4 @@ config VFIO_AMBA If you don't know what to do here, say N. source "drivers/vfio/platform/reset/Kconfig" +source "drivers/vfio/platform/msi/Kconfig"drivers/vfio/platform/Makefile +1
-0@@ -5,6 +5,7 @@ vfio-platform-y := vfio_platform.o obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform.o obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform-base.o obj-$(CONFIG_VFIO_PLATFORM) += reset/ +obj-$(CONFIG_VFIO_PLATFORM) += msi/ vfio-amba-y := vfio_amba.odrivers/vfio/platform/msi/Kconfig new file +9
-0@@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0-only +config VFIO_PLATFORM_BCMPLT_MSI + tristate "MSI support for Broadcom platform devices" + depends on VFIO_PLATFORM && (ARCH_BCM_IPROC || COMPILE_TEST) + default ARCH_BCM_IPROC + help + Enables the VFIO platform driver to handle msi for Broadcom devices + + If you don't know what to do here, say N.drivers/vfio/platform/msi/Makefile new file +2
-0@@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_VFIO_PLATFORM_BCMPLT_MSI) += vfio_platform_bcmplt.odrivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +8
-0@@ -0,0 +1,49 @@ +// SPDX-License-Identifier: GPL-2.0 +/* +* Copyright 2020 Broadcom. +*/ + +#include <linux/module.h> +#include <linux/device.h> +#include <linux/interrupt.h> drivers/vfio/platform/msi/vfio_platform_bcmplt.c line 18
Quoted 10 lines
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +10
-0+#include <linux/msi.h> +#include <linux/vfio.h> + +#include "../vfio_platform_private.h" + +#define RING_SIZE (64 << 10) + +#define RING_MSI_ADDR_LS 0x03c +#define RING_MSI_ADDR_MS 0x040 +#define RING_MSI_DATA_VALUE 0x064Those 3 defines would not be needed anymore with that implementation option.
Quoted 18 lines
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +18
-0+ +static u32 bcm_num_msi (struct vfio_platform_device *vdev) +{ +struct vfio_platform_region *reg = &vdev->regions[0 ]; + +return (reg->size /RING_SIZE ); +} + +static struct vfio_platform_msi_node vfio_platform_bcmflexrm_msi_node = { + .owner =THIS_MODULE , + .compat ="brcm,iproc-flexrm-mbox" , + .of_get_msi = bcm_num_msi, +}; + +static int __init vfio_platform_bcmflexrm_msi_module_init (void ) +{ +__vfio_platform_register_msi (&vfio_platform_bcmflexrm_msi_node); +drivers/vfio/platform/msi/vfio_platform_bcmplt.c line 46
Quoted 10 lines
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +10
-0+return 0 ; +} + +static void __exit vfio_platform_bcmflexrm_msi_module_exit(void) +{ +vfio_platform_unregister_msi ("brcm,iproc-flexrm-mbox" ); +} + +module_init (vfio_platform_bcmflexrm_msi_module_init); +module_exit (vfio_platform_bcmflexrm_msi_module_exit);One thing I would like to discuss with Alex.
As the reset module is mandated (except if
reset_requiredis forced to 0), I am wondering if we shouldn't try to turn the reset module into a "specialization" module and put the msi hooks there. I am afraid we may end up having modules for each and every vfio platform feature specialization. At the moment that's fully bearable but I can't predict what's next.Quoted 4 lines
As the mandated feature is the reset capability maybe we could just keep the config/module name terminology, tune the kconfig help message to mention the msi support in case of flex-rm?
Quoted 6 lines
As I understand, your proposal is that we should not have a separate module for MSI, rather we add in the existing reset module for flex-rm. Thus, this way reset modules do not seem to be specialized just for reset functionality only but for MSI as well. Apart from this we need not to load the proposed msi module in this patch series. Is my understanding correct?
yes it is.
Quoted 2 lines
For me it looks OK to consolidate MSI in the existing 'reset' module. Let me know your views so that I can work for the next patch set accordingly.
Before you launch into the rewriting I would like to get the confirmation Alex is OK or if he prefers to keep separate modules.
Thanks
Eric
Quoted 18 lines
Thanks,
VikasQuoted 14 lines
What do you think?
Thanks
Eric
Quoted 4 lines
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +3
-0+ +MODULE_LICENSE ("GPL v2" ); +MODULE_AUTHOR ("Broadcom" );
Re: [RFC v3 2/2] vfio/platform: msi: add Broadcom platform devices
Quoted 154 lines · On Fri, 15 Jan 2021 10:24:33 +0100 Auger Eric <[email protected]> wrote:
Hi Vikas,
Quoted 20 lines · Vikas Gupta
Hi Eric,
Quoted 17 lines · Auger Eric
Hi Vikas,
Quoted 13 lines · Vikas Gupta
Add msi support for Broadcom platform devices
Signed-off-by Vikas Gupta <[email protected]>--- drivers/vfio/platform/Kconfig | 1 + drivers/vfio/platform/Makefile | 1 + drivers/vfio/platform/msi/Kconfig | 9 ++++ drivers/vfio/platform/msi/Makefile | 2 + .../vfio/platform/msi/vfio_platform_bcmplt.c | 49 +++++++++++++++++++ 5 files changed, 62 insertions(+) create mode 100644 drivers/vfio/platform/msi/Kconfig create mode 100644 drivers/vfio/platform/msi/Makefile create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.cQuoted 4 lines
Quoted 1 line
what does plt mean?
This(plt) is a generic name for Broadcom platform devices, which we`ll
plan to add in this file. Currently we have only one in this file.
Do you think this name does not sound good here?we have
VFIO_PLATFORM_BCMFLEXRM_RESETconfig which also applied to vfio flex-rm platform device.I think it would be more homegenous to have
VFIO_PLATFORM_BCMFLEXRM_MSIin case we keep a separate msi module.also in reset dir we have
vfio_platform_bcmflexrm.cQuoted 111 lines
Quoted 107 lines
Quoted 59 lines
drivers/vfio/platform/Kconfig +1
-0@@ -21,3 +21,4 @@ config VFIO_AMBA If you don't know what to do here, say N. source "drivers/vfio/platform/reset/Kconfig" +source "drivers/vfio/platform/msi/Kconfig"drivers/vfio/platform/Makefile +1
-0@@ -5,6 +5,7 @@ vfio-platform-y := vfio_platform.o obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform.o obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform-base.o obj-$(CONFIG_VFIO_PLATFORM) += reset/ +obj-$(CONFIG_VFIO_PLATFORM) += msi/ vfio-amba-y := vfio_amba.odrivers/vfio/platform/msi/Kconfig new file +9
-0@@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0-only +config VFIO_PLATFORM_BCMPLT_MSI + tristate "MSI support for Broadcom platform devices" + depends on VFIO_PLATFORM && (ARCH_BCM_IPROC || COMPILE_TEST) + default ARCH_BCM_IPROC + help + Enables the VFIO platform driver to handle msi for Broadcom devices + + If you don't know what to do here, say N.drivers/vfio/platform/msi/Makefile new file +2
-0@@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_VFIO_PLATFORM_BCMPLT_MSI) += vfio_platform_bcmplt.odrivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +8
-0@@ -0,0 +1,49 @@ +// SPDX-License-Identifier: GPL-2.0 +/* +* Copyright 2020 Broadcom. +*/ + +#include <linux/module.h> +#include <linux/device.h> +#include <linux/interrupt.h> drivers/vfio/platform/msi/vfio_platform_bcmplt.c line 18
Quoted 10 lines
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +10
-0+#include <linux/msi.h> +#include <linux/vfio.h> + +#include "../vfio_platform_private.h" + +#define RING_SIZE (64 << 10) + +#define RING_MSI_ADDR_LS 0x03c +#define RING_MSI_ADDR_MS 0x040 +#define RING_MSI_DATA_VALUE 0x064Those 3 defines would not be needed anymore with that implementation option.
Quoted 18 lines
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +18
-0+ +static u32 bcm_num_msi (struct vfio_platform_device *vdev) +{ +struct vfio_platform_region *reg = &vdev->regions[0 ]; + +return (reg->size /RING_SIZE ); +} + +static struct vfio_platform_msi_node vfio_platform_bcmflexrm_msi_node = { + .owner =THIS_MODULE , + .compat ="brcm,iproc-flexrm-mbox" , + .of_get_msi = bcm_num_msi, +}; + +static int __init vfio_platform_bcmflexrm_msi_module_init (void ) +{ +__vfio_platform_register_msi (&vfio_platform_bcmflexrm_msi_node); +drivers/vfio/platform/msi/vfio_platform_bcmplt.c line 46
Quoted 10 lines
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +10
-0+return 0 ; +} + +static void __exit vfio_platform_bcmflexrm_msi_module_exit(void) +{ +vfio_platform_unregister_msi ("brcm,iproc-flexrm-mbox" ); +} + +module_init (vfio_platform_bcmflexrm_msi_module_init); +module_exit (vfio_platform_bcmflexrm_msi_module_exit);One thing I would like to discuss with Alex.
As the reset module is mandated (except if
reset_requiredis forced to 0), I am wondering if we shouldn't try to turn the reset module into a "specialization" module and put the msi hooks there. I am afraid we may end up having modules for each and every vfio platform feature specialization. At the moment that's fully bearable but I can't predict what's next.Quoted 4 lines
As the mandated feature is the reset capability maybe we could just keep the config/module name terminology, tune the kconfig help message to mention the msi support in case of flex-rm?
Quoted 6 lines
As I understand, your proposal is that we should not have a separate module for MSI, rather we add in the existing reset module for flex-rm. Thus, this way reset modules do not seem to be specialized just for reset functionality only but for MSI as well. Apart from this we need not to load the proposed msi module in this patch series. Is my understanding correct?
Quoted 6 lines
yes it is.
Quoted 2 lines
For me it looks OK to consolidate MSI in the existing 'reset' module. Let me know your views so that I can work for the next patch set accordingly.
Before you launch into the rewriting I would like to get the confirmation Alex is OK or if he prefers to keep separate modules.
If I understand correctly, the proposal here creates an entirely parallel vfio-msi request module interface like we have for vfio-reset, so the question is whether we should simplify vfio-platform-core to do a single module request per compat string and the device specific module would register multiple features rather than one per module. Is that right?
It seems the submodules are pretty simple, there's not a lot to be gained from duplicate boilerplate code in the modules themselves. The core code would clearly be simplified slightly to avoid multiple module requests, but for a more grand benefit is seems the registration interfaces would also need to be consolidated, perhaps providing a feature "ops" structure. As you indicate, having only two features at this point with a fairly small number of modules each, it's not yet too burdensome, but I could imagine it being a useful project.
More importantly in the short term, I'd expect modules handling the same compat string to be named similarly and enabled by a common Kconfig option. Thanks,
Alex
Auger Eric
maintainerRe: [RFC v3 2/2] vfio/platform: msi: add Broadcom platform devices
Hi Alex,
Quoted 164 lines · Alex Williamson
Quoted 154 lines · On Fri, 15 Jan 2021 10:24:33 +0100 Auger Eric <[email protected]> wrote:
Hi Vikas,
Quoted 20 lines · Vikas Gupta
Hi Eric,
Quoted 17 lines · Auger Eric
Hi Vikas,
Quoted 13 lines · Vikas Gupta
Add msi support for Broadcom platform devices
Signed-off-by Vikas Gupta <[email protected]>--- drivers/vfio/platform/Kconfig | 1 + drivers/vfio/platform/Makefile | 1 + drivers/vfio/platform/msi/Kconfig | 9 ++++ drivers/vfio/platform/msi/Makefile | 2 + .../vfio/platform/msi/vfio_platform_bcmplt.c | 49 +++++++++++++++++++ 5 files changed, 62 insertions(+) create mode 100644 drivers/vfio/platform/msi/Kconfig create mode 100644 drivers/vfio/platform/msi/Makefile create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.cQuoted 4 lines
Quoted 1 line
what does plt mean?
This(plt) is a generic name for Broadcom platform devices, which we`ll
plan to add in this file. Currently we have only one in this file.
Do you think this name does not sound good here?we have
VFIO_PLATFORM_BCMFLEXRM_RESETconfig which also applied to vfio flex-rm platform device.I think it would be more homegenous to have
VFIO_PLATFORM_BCMFLEXRM_MSIin case we keep a separate msi module.also in reset dir we have
vfio_platform_bcmflexrm.cQuoted 111 lines
Quoted 107 lines
Quoted 59 lines
drivers/vfio/platform/Kconfig +1
-0@@ -21,3 +21,4 @@ config VFIO_AMBA If you don't know what to do here, say N. source "drivers/vfio/platform/reset/Kconfig" +source "drivers/vfio/platform/msi/Kconfig"drivers/vfio/platform/Makefile +1
-0@@ -5,6 +5,7 @@ vfio-platform-y := vfio_platform.o obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform.o obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform-base.o obj-$(CONFIG_VFIO_PLATFORM) += reset/ +obj-$(CONFIG_VFIO_PLATFORM) += msi/ vfio-amba-y := vfio_amba.odrivers/vfio/platform/msi/Kconfig new file +9
-0@@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0-only +config VFIO_PLATFORM_BCMPLT_MSI + tristate "MSI support for Broadcom platform devices" + depends on VFIO_PLATFORM && (ARCH_BCM_IPROC || COMPILE_TEST) + default ARCH_BCM_IPROC + help + Enables the VFIO platform driver to handle msi for Broadcom devices + + If you don't know what to do here, say N.drivers/vfio/platform/msi/Makefile new file +2
-0@@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_VFIO_PLATFORM_BCMPLT_MSI) += vfio_platform_bcmplt.odrivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +8
-0@@ -0,0 +1,49 @@ +// SPDX-License-Identifier: GPL-2.0 +/* +* Copyright 2020 Broadcom. +*/ + +#include <linux/module.h> +#include <linux/device.h> +#include <linux/interrupt.h> drivers/vfio/platform/msi/vfio_platform_bcmplt.c line 18
Quoted 10 lines
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +10
-0+#include <linux/msi.h> +#include <linux/vfio.h> + +#include "../vfio_platform_private.h" + +#define RING_SIZE (64 << 10) + +#define RING_MSI_ADDR_LS 0x03c +#define RING_MSI_ADDR_MS 0x040 +#define RING_MSI_DATA_VALUE 0x064Those 3 defines would not be needed anymore with that implementation option.
Quoted 18 lines
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +18
-0+ +static u32 bcm_num_msi (struct vfio_platform_device *vdev) +{ +struct vfio_platform_region *reg = &vdev->regions[0 ]; + +return (reg->size /RING_SIZE ); +} + +static struct vfio_platform_msi_node vfio_platform_bcmflexrm_msi_node = { + .owner =THIS_MODULE , + .compat ="brcm,iproc-flexrm-mbox" , + .of_get_msi = bcm_num_msi, +}; + +static int __init vfio_platform_bcmflexrm_msi_module_init (void ) +{ +__vfio_platform_register_msi (&vfio_platform_bcmflexrm_msi_node); +drivers/vfio/platform/msi/vfio_platform_bcmplt.c line 46
Quoted 10 lines
drivers/vfio/platform/msi/vfio_platform_bcmplt.c new file +10
-0+return 0 ; +} + +static void __exit vfio_platform_bcmflexrm_msi_module_exit(void) +{ +vfio_platform_unregister_msi ("brcm,iproc-flexrm-mbox" ); +} + +module_init (vfio_platform_bcmflexrm_msi_module_init); +module_exit (vfio_platform_bcmflexrm_msi_module_exit);One thing I would like to discuss with Alex.
As the reset module is mandated (except if
reset_requiredis forced to 0), I am wondering if we shouldn't try to turn the reset module into a "specialization" module and put the msi hooks there. I am afraid we may end up having modules for each and every vfio platform feature specialization. At the moment that's fully bearable but I can't predict what's next.Quoted 4 lines
As the mandated feature is the reset capability maybe we could just keep the config/module name terminology, tune the kconfig help message to mention the msi support in case of flex-rm?
Quoted 6 lines
As I understand, your proposal is that we should not have a separate module for MSI, rather we add in the existing reset module for flex-rm. Thus, this way reset modules do not seem to be specialized just for reset functionality only but for MSI as well. Apart from this we need not to load the proposed msi module in this patch series. Is my understanding correct?
Quoted 6 lines
yes it is.
Quoted 2 lines
For me it looks OK to consolidate MSI in the existing 'reset' module. Let me know your views so that I can work for the next patch set accordingly.
Before you launch into the rewriting I would like to get the confirmation Alex is OK or if he prefers to keep separate modules.
Quoted 6 lines
If I understand correctly, the proposal here creates an entirely parallel vfio-msi request module interface like we have for vfio-reset, so the question is whether we should simplify vfio-platform-core to do a single module request per compat string and the device specific module would register multiple features rather than one per module. Is that right?
Yes that's correct, the so-called "reset" module would also implement msi hooks and if new specialization are needed in the future they also could be put there.
Quoted 9 lines
It seems the submodules are pretty simple, there's not a lot to be gained from duplicate boilerplate code in the modules themselves. The core code would clearly be simplified slightly to avoid multiple module requests, but for a more grand benefit is seems the registration interfaces would also need to be consolidated, perhaps providing a feature "ops" structure. As you indicate, having only two features at this point with a fairly small number of modules each, it's not yet too burdensome, but I could imagine it being a useful project.
Yes the registration must be reworked anyway.
Quoted 4 lines
More importantly in the short term, I'd expect modules handling the same compat string to be named similarly and enabled by a common
Kconfigoption. Thanks,
I agree with you.
Thanks
Eric
Quoted 3 lines
Alex
v4 cover msi support for platform devices
In reply to Vikas GuptaThis RFC adds support for MSI for platform devices. MSI block is added as an ext irq along with the existing wired interrupt implementation. The patchset exports two caps for MSI and related data to configure MSI source device.
Changes from:
-------------
v3 to v4:
1) Removed the 'cap' for exporting MSI info to userspace and restored into vedor specific module.
2) Enable GENERIC_MSI_IRQ_DOMAIN in Kconfig.
3) Removed the vendor specific, Broadcom, 'msi' module and integrated the MSI relates ops into the 'reset' module for MSI support.
v2 to v3:
1) Restored the vendor specific module to get max number of MSIs supported and .count value initialized.
2) Comments from Eric addressed.
v1 to v2:
1) IRQ allocation has been implemented as below:
----------------------------
|IRQ-0|IRQ-1|....|IRQ-n|MSI|
----------------------------
MSI block has msi contexts and its implemneted as ext irq.
2) Removed vendor specific module for msi handling so previously patch2 and patch3 are not required.
3) MSI related data is exported to userspace using 'caps'.
Please note VFIO_IRQ_INFO_CAP_TYPE in include/uapi/linux/vfio.h implementation
is taken from the Eric`s patch
https://patchwork.kernel.org/project/kvm/patch/[email protected]/
v0 to v1:
i) Removed MSI device flag VFIO_DEVICE_FLAGS_MSI. ii) Add MSI(s) at the end of the irq list of platform IRQs. MSI(s) with first entry of MSI block has count and flag information.
IRQ list: Allocation for IRQs + MSIs are allocated as below
Example: if there are 'n' IRQs and 'k' MSIs
-------------------------------------------------------
|IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
-------------------------------------------------------
MSI-0 will have count=k set and flags set accordingly.
Vikas Gupta (3):
vfio/platform: add support for msi
vfio/platform: change cleanup order
vfio: platform: reset: add msi support
drivers/vfio/platform/Kconfig | 1 + .../platform/reset/vfio_platform_bcmflexrm.c | 72 ++++- drivers/vfio/platform/vfio_platform_common.c | 97 +++++-- drivers/vfio/platform/vfio_platform_irq.c | 253 ++++++++++++++++-- drivers/vfio/platform/vfio_platform_private.h | 29 ++ include/uapi/linux/vfio.h | 24 ++ 6 files changed, 444 insertions(+), 32 deletions(-)
Signature · 1 line
2.17.1
RFC v4 1/3 vfio/platform: add support for msi
MSI support for platform devices. MSI is added
as a single 'index' with 'count' as the number of
MSI(s) supported by the devices.
--- drivers/vfio/platform/Kconfig | 1 + drivers/vfio/platform/vfio_platform_common.c | 95 ++++++- drivers/vfio/platform/vfio_platform_irq.c | 253 ++++++++++++++++-- drivers/vfio/platform/vfio_platform_private.h | 29 ++ include/uapi/linux/vfio.h | 24 ++ 5 files changed, 373 insertions(+), 29 deletions(-)
drivers/vfio/platform/Kconfig +1 -0
@@ -3,6 +3,7 @@ config VFIO_PLATFORM tristate "VFIO support for platform devices" depends on VFIO && EVENTFD && (ARM || ARM64) select VFIO_VIRQFD + select GENERIC_MSI_IRQ_DOMAIN help Support for platform devices with VFIO. This is required to make use of platform devices present on the system using the VFIO
drivers/vfio/platform/vfio_platform_common.c +81 -14
156 lines not on this page · raw message
drivers/vfio/platform/vfio_platform_irq.c +238 -15
297 lines not on this page · raw message
drivers/vfio/platform/vfio_platform_private.h +29 -0
@@ -9,6 +9,7 @@#include <linux/types.h> #include <linux/interrupt.h> +#include <linux/msi.h> #define VFIO_PLATFORM_OFFSET_SHIFT 40#define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
@@ -19,9 +20,21 @@#define VFIO_PLATFORM_INDEX_TO_OFFSET (index) \ ((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT) +/* IRQ index for MSI in ext IRQs */ +#define VFIO_EXT_IRQ_MSI 0 + +struct vfio_irq_ctx { +int hwirq; +char *name; +struct msi_msg msg; +struct eventfd_ctx *trigger; +}; +struct vfio_platform_irq {u32 flags;u32 count; +int num_ctx; +struct vfio_irq_ctx *ctx;int hwirq;char *name;struct eventfd_ctx *trigger;
@@ -29,6 +42,11 @@ struct vfio_platform_irq {spinlock_t lock;struct virqfd *unmask;struct virqfd *mask; + +/* for extended irqs */ +u32 type; +u32 subtype; +int config_msi; };struct vfio_platform_region {
@@ -46,6 +64,7 @@ struct vfio_platform_device {u32 num_regions;struct vfio_platform_irq *irqs;u32 num_irqs; +int num_ext_irqs;int refcnt;struct mutex igate;struct module *parent_module;
@@ -65,17 +84,27 @@ struct vfio_platform_device { (*get_resource)(struct vfio_platform_device *vdev, int i);int (*get_irq)(struct vfio_platform_device *vdev,int i);int (*of_reset)(struct vfio_platform_device *vdev); +u32 (*of_get_msi)(struct vfio_platform_device *vdev); +void (*of_msi_write)(struct vfio_platform_device *vdev, +struct msi_desc *desc, +struct msi_msg *msg);bool reset_required; };typedef int (*vfio_platform_reset_fn_t )(struct vfio_platform_device *vdev); +typedef u32 (*vfio_platform_get_msi_fn_t )(struct vfio_platform_device *vdev); +typedef void (*vfio_platform_msi_write_fn_t )(struct vfio_platform_device *vdev, +struct msi_desc *desc, +struct msi_msg *msg);struct vfio_platform_reset_node {struct list_head link;char *compat;struct module *owner;vfio_platform_reset_fn_t of_reset; +vfio_platform_get_msi_fn_t of_get_msi; +vfio_platform_msi_write_fn_t of_msi_write; }; extern int vfio_platform_probe_common(struct vfio_platform_device *vdev,
include/uapi/linux/vfio.h +24 -0
@@ -697,11 +697,35 @@ struct vfio_irq_info {#define VFIO_IRQ_INFO_MASKABLE (1 << 1)#define VFIO_IRQ_INFO_AUTOMASKED (1 << 2)#define VFIO_IRQ_INFO_NORESIZE (1 << 3) +#define VFIO_IRQ_INFO_FLAG_CAPS (1 << 4)/* Info supports caps */ __u32 index;/* IRQ index */ __u32 count;/* Number of IRQs within this index */ +__u32 cap_offset;/* Offset within info struct of first cap */ };#define VFIO_DEVICE_GET_IRQ_INFO _IO(VFIO_TYPE, VFIO_BASE + 9) +/* +* The irq type capability allows IRQs unique to a specific device or +* class of devices to be exposed. +* +* The structures below define version 1 of this capability. +*/ +#define VFIO_IRQ_INFO_CAP_TYPE 3 + +struct vfio_irq_info_cap_type { +struct vfio_info_cap_header header; +__u32 type;/* global per bus driver */ +__u32 subtype;/* type specific */ +}; + +/* +* List of IRQ types, global per bus driver. +* If you introduce a new type, please add it here. +*/ + +/* Non PCI devices having MSI(s) support */ +#define VFIO_IRQ_TYPE_MSI (1) +/** * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set) *
Signature · 1 line
2.17.1
Auger Eric
maintainerRe: [RFC v4 1/3] vfio/platform: add support for msi
Hi Vikas,
Quoted 1 line · Vikas Gupta
MSI support for platform devices. MSI is added
s/MSI support/ Add MSI support
Quoted 2 lines
as a single 'index' with 'count' as the number of
MSI(s) supported by the devices.
as a single 'index' following the last wired irq index index, with count.
It allows to associate eventfds to MSIs.
If MSI is supported, specialization callbacks need to be implemented in the reset module (of_get_msi and of_msi_write).
Quoted 35 lines
Signed-off-by Vikas Gupta <[email protected]>--- drivers/vfio/platform/Kconfig | 1 + drivers/vfio/platform/vfio_platform_common.c | 95 ++++++- drivers/vfio/platform/vfio_platform_irq.c | 253 ++++++++++++++++-- drivers/vfio/platform/vfio_platform_private.h | 29 ++ include/uapi/linux/vfio.h | 24 ++ 5 files changed, 373 insertions(+), 29 deletions(-)drivers/vfio/platform/Kconfig +1
-0@@ -3,6 +3,7 @@ config VFIO_PLATFORM tristate "VFIO support for platform devices" depends on VFIO && EVENTFD && (ARM || ARM64) select VFIO_VIRQFD + select GENERIC_MSI_IRQ_DOMAIN help Support for platform devices with VFIO. This is required to make use of platform devices present on the system using the VFIOdrivers/vfio/platform/vfio_platform_common.c +1
-0@@ -16,6 +16,7 @@#include <linux/types.h> #include <linux/uaccess.h> #include <linux/vfio.h> +#include <linux/nospec.h> #include "vfio_platform_private.h"
drivers/vfio/platform/vfio_platform_common.c line 34
Quoted 8 lines
drivers/vfio/platform/vfio_platform_common.c +3
-2@@ -28,23 +29,22 @@static LIST_HEAD (reset_list);static DEFINE_MUTEX (driver_lock); -static vfio_platform_reset_fn_t vfio_platform_lookup_reset (const char *compat, -struct module **module) +static void vfio_platform_lookup_reset (const char *compat, +struct module **module, +struct vfio_platform_reset_node **node)
nit: I would prefer this function directly returns a struct vfio_platform_reset_node *
Quoted 9 lines
drivers/vfio/platform/vfio_platform_common.c +0
-1{struct vfio_platform_reset_node *iter; -vfio_platform_reset_fn_t reset_fn =NULL ;mutex_lock (&driver_lock);list_for_each_entry (iter, &reset_list, link) {if (!strcmp (iter->compat, compat) &&try_module_get (iter->owner)) { *module = iter->owner;
drivers/vfio/platform/vfio_platform_common.c
Quoted 10 lines
drivers/vfio/platform/vfio_platform_common.c +1
-2- reset_fn = iter->of_reset; + *node = iter;break ; } }mutex_unlock (&driver_lock); -return reset_fn; } static int vfio_platform_acpi_probe(struct vfio_platform_device *vdev,> @@ -112,15 +112,23 @@ static bool vfio_platform_has_reset(struct
vfio_platform_device *vdev)
Quoted 289 lines
static int
vfio_platform_get_reset(struct vfio_platform_device *vdev){drivers/vfio/platform/vfio_platform_common.c +76
-9+struct vfio_platform_reset_node *node =NULL ; +if (VFIO_PLATFORM_IS_ACPI (vdev))return vfio_platform_acpi_has_reset (vdev) ?0 : -ENOENT ; - vdev->of_reset =vfio_platform_lookup_reset (vdev->compat, - &vdev->reset_module); -if (!vdev->of_reset) { +vfio_platform_lookup_reset (vdev->compat, &vdev->reset_module, + &node); +if (!node) {request_module ("vfio-reset:%s" , vdev->compat); - vdev->of_reset =vfio_platform_lookup_reset (vdev->compat, - &vdev->reset_module); +vfio_platform_lookup_reset (vdev->compat, &vdev->reset_module, + &node); + } + +if (node) { + vdev->of_reset = node->of_reset; + vdev->of_get_msi = node->of_get_msi; + vdev->of_msi_write = node->of_msi_write;> }return vdev->of_reset ?0 : -ENOENT ;@@ -343,9 +351,16 @@ static long vfio_platform_ioctl(void *device_data, }else if (cmd ==VFIO_DEVICE_GET_IRQ_INFO ) {struct vfio_irq_info info; +struct vfio_info_cap caps = { .buf =NULL , .size =0 }; +int ext_irq_index = vdev->num_irqs - vdev->num_ext_irqs; +unsigned long capsz; +u32 index; minsz = offsetofend(struct vfio_irq_info, count); +/* For backward compatibility, cannot require this */ + capsz = offsetofend(struct vfio_irq_info, cap_offset); +if (copy_from_user (&info, (void __user *)arg, minsz))return -EFAULT ;@@ -355,8 +370,53 @@ static long vfio_platform_ioctl(void *device_data,if (info.index >= vdev->num_irqs)return -EINVAL ; - info.flags = vdev->irqs[info.index].flags; - info.count = vdev->irqs[info.index].count; +if (info.argsz >= capsz) + minsz = capsz; + + index = info.index; + + info.flags = vdev->irqs[index].flags; + info.count = vdev->irqs[index].count; + +if (ext_irq_index - index ==VFIO_EXT_IRQ_MSI ) { +struct vfio_irq_info_cap_type cap_type = { + .header.id =VFIO_IRQ_INFO_CAP_TYPE , + .header.version =1 }; +struct vfio_platform_irq *irq; +int ret; + + index =array_index_nospec (index, + vdev->num_irqs); + irq = &vdev->irqs[index]; + + cap_type.type = irq->type; + cap_type.subtype = irq->subtype; + + ret =vfio_info_add_capability (&caps, + &cap_type.header, +sizeof (cap_type)); +if (ret) +return ret; + } + +if (caps.size) { + info.flags |=VFIO_IRQ_INFO_FLAG_CAPS ; +if (info.argsz <sizeof (info) + caps.size) { + info.argsz =sizeof (info) + caps.size; + info.cap_offset =0 ; + }else { +vfio_info_cap_shift (&caps,sizeof (info)); +if (copy_to_user ((void __user *)arg + +sizeof (info), caps.buf, + caps.size)) { +kfree (caps.buf); +return -EFAULT ; + } + info.cap_offset =sizeof (info); + } + +kfree (caps.buf); + }return copy_to_user ((void __user *)arg, &info, minsz) ? -EFAULT :0 ;@@ -365,6 +425,7 @@ static long vfio_platform_ioctl(void *device_data,struct vfio_irq_set hdr;u8 *data =NULL ;int ret =0 ; +int max;size_t data_size =0 ; minsz = offsetofend(struct vfio_irq_set, count);@@ -372,8 +433,14 @@ static long vfio_platform_ioctl(void *device_data,if (copy_from_user (&hdr, (void __user *)arg, minsz))return -EFAULT ; - ret =vfio_set_irqs_validate_and_prepare (&hdr, vdev->num_irqs, - vdev->num_irqs, &data_size); +if (hdr.index >= vdev->num_irqs) +return -EINVAL ; + + max = vdev->irqs[hdr.index].count; + + ret =vfio_set_irqs_validate_and_prepare (&hdr, max, + vdev->num_irqs, + &data_size);if (ret)return ret;drivers/vfio/platform/vfio_platform_irq.c +145
-0@@ -8,10 +8,12 @@#include <linux/eventfd.h> #include <linux/interrupt.h> +#include <linux/eventfd.h> #include <linux/slab.h> #include <linux/types.h> #include <linux/vfio.h> #include <linux/irq.h> +#include <linux/msi.h> #include "vfio_platform_private.h" @@ -253,6 +255,186 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,return 0 ; } +/* MSI/MSIX */ +static irqreturn_t vfio_msihandler (int irq,void *arg) +{ +struct eventfd_ctx *trigger = arg; + +eventfd_signal (trigger,1 ); +return IRQ_HANDLED ; +} + +static void msi_write (struct msi_desc *desc,struct msi_msg *msg) +{ +struct device *dev =msi_desc_to_dev (desc); +struct vfio_device *device =dev_get_drvdata (dev); +struct vfio_platform_device *vdev = (struct vfio_platform_device *) +vfio_device_data (device); + + vdev->of_msi_write (vdev, desc, msg); +} + +static int vfio_msi_enable (struct vfio_platform_device *vdev, +struct vfio_platform_irq *irq,int nvec) +{ +int ret; +int msi_idx =0 ; +struct msi_desc *desc; +struct device *dev = vdev->device; + + irq->ctx =kcalloc (nvec,sizeof (struct vfio_irq_ctx ),GFP_KERNEL ); +if (!irq->ctx) +return -ENOMEM ; + +/* Allocate platform MSIs */ + ret =platform_msi_domain_alloc_irqs (dev, nvec, msi_write); +if (ret <0 ) { +kfree (irq->ctx); +return ret; + } + +for_each_msi_entry (desc, dev) { + irq->ctx[msi_idx].hwirq = desc->irq; + msi_idx++; + } + + irq->num_ctx = nvec; + irq->config_msi =1 ; + +return 0 ; +} + +static int vfio_msi_set_vector_signal (struct vfio_platform_irq *irq, +int vector,int fd) +{ +struct eventfd_ctx *trigger; +int irq_num, ret; + +if (vector <0 || vector >= irq->num_ctx) +return -EINVAL ; + + irq_num = irq->ctx[vector].hwirq; + +if (irq->ctx[vector].trigger) { +free_irq (irq_num, irq->ctx[vector].trigger); +kfree (irq->ctx[vector].name); +eventfd_ctx_put (irq->ctx[vector].trigger); + irq->ctx[vector].trigger =NULL ; + } + +if (fd <0 ) +return 0 ; + + irq->ctx[vector].name =kasprintf (GFP_KERNEL , +"vfio-msi[%d]" , vector); +if (!irq->ctx[vector].name) +return -ENOMEM ; + + trigger =eventfd_ctx_fdget (fd); +if (IS_ERR (trigger)) { +kfree (irq->ctx[vector].name); +return PTR_ERR (trigger); + } + + ret =request_irq (irq_num, vfio_msihandler,0 , + irq->ctx[vector].name, trigger); +if (ret) { +kfree (irq->ctx[vector].name); +eventfd_ctx_put (trigger); +return ret; + } + + irq->ctx[vector].trigger = trigger; + +return 0 ; +} + +static int vfio_msi_set_block (struct vfio_platform_irq *irq,unsigned int start, +unsigned int count,int32_t *fds) +{ +int i, j, ret =0 ; + +if (start >= irq->num_ctx || start + count > irq->num_ctx) +return -EINVAL ; + +for (i =0 , j = start; i < count && !ret; i++, j++) { +int fd = fds ? fds[i] :-1 ; + + ret =vfio_msi_set_vector_signal (irq, j, fd); + } + +if (ret) { +for (--j; j >= (int )start; j--) +vfio_msi_set_vector_signal (irq, j,-1 ); + } + +return ret; +} + +static void vfio_msi_disable (struct vfio_platform_device *vdev, +struct vfio_platform_irq *irq) +{ +struct device *dev = vdev->device; + +vfio_msi_set_block (irq,0 , irq->num_ctx,NULL ); + +platform_msi_domain_free_irqs (dev); + + irq->config_msi =0 ; + irq->num_ctx =0 ; + +kfree (irq->ctx); +} + +static int vfio_set_msi_trigger (struct vfio_platform_device *vdev, +unsigned int index,unsigned int start, +unsigned int count,uint32_t flags,void *data) +{ +int i; +struct vfio_platform_irq *irq = &vdev->irqs[index]; + +if (start + count > irq->count) +return -EINVAL ; + +if (!count && (flags &VFIO_IRQ_SET_DATA_NONE )) { +vfio_msi_disable (vdev, irq);
drivers/vfio/platform/vfio_platform_irq.c static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev, · line 410
Quoted 10 lines
drivers/vfio/platform/vfio_platform_irq.c +10
-0+return 0 ; + } + +if (flags &VFIO_IRQ_SET_DATA_EVENTFD ) { +s32 *fds = data; +int ret; + +if (irq->config_msi) +return vfio_msi_set_block (irq, start, count, + fds);
looks a bit weird to me that in case of failure we do not do the same as below (ie. vfio_msi_disable). The pci code does the same but I fail to understand the logic.
Quoted 77 lines
drivers/vfio/platform/vfio_platform_irq.c +52
-11+ ret =vfio_msi_enable (vdev, irq, start + count); +if (ret) +return ret; + + ret =vfio_msi_set_block (irq, start, count, fds); +if (ret) +vfio_msi_disable (vdev, irq); + +return ret; + } + +for (i = start; i < start + count; i++) { +if (!irq->ctx[i].trigger) +continue ; +if (flags &VFIO_IRQ_SET_DATA_NONE ) { +eventfd_signal (irq->ctx[i].trigger,1 ); + }else if (flags &VFIO_IRQ_SET_DATA_BOOL ) { +u8 *bools = data; + +if (bools[i - start]) +eventfd_signal (irq->ctx[i].trigger,1 ); + } + } + +return 0 ; +} +int vfio_platform_set_irqs_ioctl (struct vfio_platform_device *vdev,uint32_t flags,unsigned index,unsigned start,unsigned count,void *data)@@ -261,16 +443,29 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev, unsigned start, unsigned count, uint32_t flags, void *data) = NULL; -switch (flags &VFIO_IRQ_SET_ACTION_TYPE_MASK ) { -case VFIO_IRQ_SET_ACTION_MASK : - func = vfio_platform_set_irq_mask; -break ; -case VFIO_IRQ_SET_ACTION_UNMASK : - func = vfio_platform_set_irq_unmask; -break ; -case VFIO_IRQ_SET_ACTION_TRIGGER : - func = vfio_platform_set_irq_trigger; -break ; +struct vfio_platform_irq *irq = &vdev->irqs[index]; + +if (irq->type ==VFIO_IRQ_TYPE_MSI ) { +switch (flags &VFIO_IRQ_SET_ACTION_TYPE_MASK ) { +case VFIO_IRQ_SET_ACTION_MASK : +case VFIO_IRQ_SET_ACTION_UNMASK : +break ; +case VFIO_IRQ_SET_ACTION_TRIGGER : + func = vfio_set_msi_trigger; +break ; + } + }else { +switch (flags &VFIO_IRQ_SET_ACTION_TYPE_MASK ) { +case VFIO_IRQ_SET_ACTION_MASK : + func = vfio_platform_set_irq_mask; +break ; +case VFIO_IRQ_SET_ACTION_UNMASK : + func = vfio_platform_set_irq_unmask; +break ; +case VFIO_IRQ_SET_ACTION_TRIGGER : + func = vfio_platform_set_irq_trigger; +break ; + } }if (!func)@@ -281,12 +476,23 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,int vfio_platform_irq_init (struct vfio_platform_device *vdev) { -int cnt =0 , i; +int i; +int cnt =0 ;
drivers/vfio/platform/vfio_platform_irq.c int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev, · line 489
Quoted 10 lines
drivers/vfio/platform/vfio_platform_irq.c +5
-1+int num_irqs =0 ; +int msi_cnt =0 ;while (vdev->get_irq (vdev, cnt) >=0 ) cnt++; - vdev->irqs =kcalloc (cnt,sizeof (struct vfio_platform_irq ),GFP_KERNEL ); + num_irqs = cnt; + +if (vdev->of_get_msi) {
I think you also need to test that of_msi_write also is non null?
Quoted 81 lines
drivers/vfio/platform/vfio_platform_irq.c +26
-3+ msi_cnt = vdev->of_get_msi (vdev); + num_irqs++; + } + + vdev->irqs =kcalloc (num_irqs,sizeof (struct vfio_platform_irq ), +GFP_KERNEL );if (!vdev->irqs)return -ENOMEM ;@@ -309,7 +515,19 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev) vdev->irqs[i].masked = false; } - vdev->num_irqs = cnt; +/* +* MSI block is added at last index and it is an ext irq +*/ +if (msi_cnt >0 ) { + vdev->irqs[i].flags =VFIO_IRQ_INFO_EVENTFD ; + vdev->irqs[i].count = msi_cnt; + vdev->irqs[i].hwirq =0 ; + vdev->irqs[i].masked = false; + vdev->irqs[i].type =VFIO_IRQ_TYPE_MSI ; + vdev->num_ext_irqs =1 ; + } + + vdev->num_irqs = num_irqs;return 0 ; err:@@ -321,8 +539,13 @@ void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev) {int i; -for (i =0 ; i < vdev->num_irqs; i++) -vfio_set_trigger (vdev, i,-1 ,NULL ); +for (i =0 ; i < vdev->num_irqs; i++) { +if (vdev->irqs[i].type ==VFIO_IRQ_TYPE_MSI ) +vfio_set_msi_trigger (vdev, i,0 ,0 , +VFIO_IRQ_SET_DATA_NONE ,NULL ); +else +vfio_set_trigger (vdev, i,-1 ,NULL ); + } vdev->num_irqs =0 ;kfree (vdev->irqs);drivers/vfio/platform/vfio_platform_private.h +13
-0@@ -9,6 +9,7 @@#include <linux/types.h> #include <linux/interrupt.h> +#include <linux/msi.h> #define VFIO_PLATFORM_OFFSET_SHIFT 40#define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)@@ -19,9 +20,21 @@#define VFIO_PLATFORM_INDEX_TO_OFFSET (index) \ ((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT) +/* IRQ index for MSI in ext IRQs */ +#define VFIO_EXT_IRQ_MSI 0 + +struct vfio_irq_ctx { +int hwirq; +char *name; +struct msi_msg msg; +struct eventfd_ctx *trigger; +}; +struct vfio_platform_irq {u32 flags;u32 count; +int num_ctx; +struct vfio_irq_ctx *ctx;int hwirq;char *name;struct eventfd_ctx *trigger;
drivers/vfio/platform/vfio_platform_private.h struct vfio_platform_irq { · line 49
Quoted 8 lines
drivers/vfio/platform/vfio_platform_private.h +5
-0@@ -29,6 +42,11 @@ struct vfio_platform_irq {spinlock_t lock;struct virqfd *unmask;struct virqfd *mask; + +/* for extended irqs */ +u32 type; +u32 subtype; +int config_msi;
bool?
Quoted 70 lines
drivers/vfio/platform/vfio_platform_private.h +11
-0};struct vfio_platform_region {@@ -46,6 +64,7 @@ struct vfio_platform_device {u32 num_regions;struct vfio_platform_irq *irqs;u32 num_irqs; +int num_ext_irqs;int refcnt;struct mutex igate;struct module *parent_module;@@ -65,17 +84,27 @@ struct vfio_platform_device { (*get_resource)(struct vfio_platform_device *vdev, int i);int (*get_irq)(struct vfio_platform_device *vdev,int i);int (*of_reset)(struct vfio_platform_device *vdev); +u32 (*of_get_msi)(struct vfio_platform_device *vdev); +void (*of_msi_write)(struct vfio_platform_device *vdev, +struct msi_desc *desc, +struct msi_msg *msg);bool reset_required; };typedef int (*vfio_platform_reset_fn_t )(struct vfio_platform_device *vdev); +typedef u32 (*vfio_platform_get_msi_fn_t )(struct vfio_platform_device *vdev); +typedef void (*vfio_platform_msi_write_fn_t )(struct vfio_platform_device *vdev, +struct msi_desc *desc, +struct msi_msg *msg);struct vfio_platform_reset_node {struct list_head link;char *compat;struct module *owner;vfio_platform_reset_fn_t of_reset; +vfio_platform_get_msi_fn_t of_get_msi; +vfio_platform_msi_write_fn_t of_msi_write; }; extern int vfio_platform_probe_common(struct vfio_platform_device *vdev,include/uapi/linux/vfio.h +17
-0@@ -697,11 +697,35 @@ struct vfio_irq_info {#define VFIO_IRQ_INFO_MASKABLE (1 << 1)#define VFIO_IRQ_INFO_AUTOMASKED (1 << 2)#define VFIO_IRQ_INFO_NORESIZE (1 << 3) +#define VFIO_IRQ_INFO_FLAG_CAPS (1 << 4)/* Info supports caps */ __u32 index;/* IRQ index */ __u32 count;/* Number of IRQs within this index */ +__u32 cap_offset;/* Offset within info struct of first cap */ };#define VFIO_DEVICE_GET_IRQ_INFO _IO(VFIO_TYPE, VFIO_BASE + 9) +/* +* The irq type capability allows IRQs unique to a specific device or +* class of devices to be exposed. +* +* The structures below define version 1 of this capability. +*/ +#define VFIO_IRQ_INFO_CAP_TYPE 3 + +struct vfio_irq_info_cap_type { +struct vfio_info_cap_header header; +__u32 type;/* global per bus driver */ +__u32 subtype;/* type specific */ +}; + +/*
include/uapi/linux/vfio.h struct vfio_irq_info { · line 731
Quoted 10 lines
include/uapi/linux/vfio.h +7
-0+* List of IRQ types, global per bus driver. +* If you introduce a new type, please add it here. +*/ + +/* Non PCI devices having MSI(s) support */ +#define VFIO_IRQ_TYPE_MSI (1) +/** * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set) *
Besides looks good to me.
Thanks
Eric
RFC v4 2/3 vfio/platform: change cleanup order
In reply to Vikas GuptaIn the case of msi, vendor specific msi module may require region access to handle msi cleanup so we need to cleanup region after irq cleanup only.
--- drivers/vfio/platform/vfio_platform_common.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/vfio/platform/vfio_platform_common.c +1 -1
@@ -243,8 +243,8 @@ static void vfio_platform_release(void *device_data)WARN_ON (1 ); }pm_runtime_put (vdev->device); -vfio_platform_regions_cleanup (vdev);vfio_platform_irq_cleanup (vdev); +vfio_platform_regions_cleanup (vdev); }mutex_unlock (&driver_lock);
Signature · 1 line
2.17.1
Re: [RFC v4 2/3] vfio/platform: change cleanup order
Hi Vikas,
Quoted 5 lines · Vikas Gupta
In the case of msi, vendor specific msi module may require region access to handle msi cleanup so we need to cleanup region after irq cleanup only.
Signed-off-by Vikas Gupta <[email protected]>
Thanks
Eric
Quoted 19 lines
--- drivers/vfio/platform/vfio_platform_common.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)drivers/vfio/platform/vfio_platform_common.c +1
-1@@ -243,8 +243,8 @@ static void vfio_platform_release(void *device_data)WARN_ON (1 ); }pm_runtime_put (vdev->device); -vfio_platform_regions_cleanup (vdev);vfio_platform_irq_cleanup (vdev); +vfio_platform_regions_cleanup (vdev); }mutex_unlock (&driver_lock);
RFC v4 3/3 vfio: platform: reset: add msi support
In reply to Vikas GuptaAdd msi support for Broadcom FlexRm device.
--- .../platform/reset/vfio_platform_bcmflexrm.c | 72 ++++++++++++++++++- 1 file changed, 70 insertions(+), 2 deletions(-)
drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c +70 -2
@@ -21,7 +21,9 @@#include <linux/init.h> #include <linux/io.h> #include <linux/kernel.h> +#include <linux/msi.h> #include <linux/module.h> +#include <linux/vfio.h> #include "../vfio_platform_private.h"
@@ -33,6 +35,9 @@#define RING_VER 0x000#define RING_CONTROL 0x034#define RING_FLUSH_DONE 0x038 +#define RING_MSI_ADDR_LS 0x03c +#define RING_MSI_ADDR_MS 0x040 +#define RING_MSI_DATA_VALUE 0x064/* Register RING_CONTROL fields */ #define CONTROL_FLUSH_SHIFT 5
@@ -105,8 +110,71 @@ static int vfio_platform_bcmflexrm_reset(struct vfio_platform_device *vdev)return ret; } -module_vfio_reset_handler ("brcm,iproc-flexrm-mbox" , - vfio_platform_bcmflexrm_reset); +static u32 bcm_num_msi (struct vfio_platform_device *vdev) +{ +struct vfio_platform_region *reg = &vdev->regions[0 ]; + +return (reg->size /RING_REGS_SIZE ); +} + +static void bcm_write_msi (struct vfio_platform_device *vdev, +struct msi_desc *desc, +struct msi_msg *msg) +{ +int i; +int hwirq =-1 ; +int msi_src; +void __iomem *ring; +struct vfio_platform_region *reg = &vdev->regions[0 ]; + +if (!reg) +return ; + +for (i =0 ; i < vdev->num_irqs; i++) +if (vdev->irqs[i].type ==VFIO_IRQ_TYPE_MSI ) + hwirq = vdev->irqs[i].ctx[0 ].hwirq; + +if (hwirq <0 ) +return ; + + msi_src = desc->irq - hwirq; + +if (!reg->ioaddr) { + reg->ioaddr =ioremap (reg->addr, reg->size); +if (!reg->ioaddr) +return ; + } + + ring = reg->ioaddr + msi_src *RING_REGS_SIZE ; + +writel_relaxed (msg->address_lo, ring +RING_MSI_ADDR_LS ); +writel_relaxed (msg->address_hi, ring +RING_MSI_ADDR_MS ); +writel_relaxed (msg->data, ring +RING_MSI_DATA_VALUE ); +} + +static struct vfio_platform_reset_node vfio_platform_bcmflexrm_reset_node = { + .owner =THIS_MODULE , + .compat ="brcm,iproc-flexrm-mbox" , + .of_reset = vfio_platform_bcmflexrm_reset, + .of_get_msi = bcm_num_msi, + .of_msi_write = bcm_write_msi +}; + +static int __init vfio_platform_bcmflexrm_reset_module_init (void ) +{ +__vfio_platform_register_reset (&vfio_platform_bcmflexrm_reset_node); + +return 0 ; +} + +static void __exit vfio_platform_bcmflexrm_reset_module_exit(void) +{ +vfio_platform_unregister_reset ("brcm,iproc-flexrm-mbox" , + vfio_platform_bcmflexrm_reset); +} + +module_init (vfio_platform_bcmflexrm_reset_module_init); +module_exit (vfio_platform_bcmflexrm_reset_module_exit);MODULE_LICENSE ("GPL v2" );MODULE_AUTHOR ("Anup Patel <[email protected]>" );
Signature · 1 line
2.17.1
Auger Eric
maintainerRe: [RFC v4 3/3] vfio: platform: reset: add msi support
Hi Vikas,
Quoted 32 lines · Vikas Gupta
Add msi support for Broadcom FlexRm device.
Signed-off-by Vikas Gupta <[email protected]>--- .../platform/reset/vfio_platform_bcmflexrm.c | 72 ++++++++++++++++++- 1 file changed, 70 insertions(+), 2 deletions(-)drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c +5
-0@@ -21,7 +21,9 @@#include <linux/init.h> #include <linux/io.h> #include <linux/kernel.h> +#include <linux/msi.h> #include <linux/module.h> +#include <linux/vfio.h> #include "../vfio_platform_private.h" @@ -33,6 +35,9 @@#define RING_VER 0x000#define RING_CONTROL 0x034#define RING_FLUSH_DONE 0x038 +#define RING_MSI_ADDR_LS 0x03c +#define RING_MSI_ADDR_MS 0x040 +#define RING_MSI_DATA_VALUE 0x064/* Register RING_CONTROL fields */ #define CONTROL_FLUSH_SHIFT 5
drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c static int vfio_platform_bcmflexrm_reset(struct vfio_platform_device *vdev) · line 113
Quoted 6 lines
drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c +1
-2@@ -105,8 +110,71 @@ static int vfio_platform_bcmflexrm_reset(struct vfio_platform_device *vdev)return ret; } -module_vfio_reset_handler ("brcm,iproc-flexrm-mbox" , - vfio_platform_bcmflexrm_reset); +static u32 bcm_num_msi (struct vfio_platform_device *vdev)
Please use the same prefix as for reset, ie. vfio_platform_bcmflexrm_get_msi
drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c static int vfio_platform_bcmflexrm_reset(struct vfio_platform_device *vdev) · line 120
Quoted 6 lines
drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c +6
-0+{ + struct vfio_platform_region *reg = &vdev->regions[0];> + + return (reg->size / RING_REGS_SIZE); +} + +static void bcm_write_msi(struct vfio_platform_device *vdev,
vfio_platform_bcmflexrm_msi_write?
drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c static int vfio_platform_bcmflexrm_reset(struct vfio_platform_device *vdev) · line 131
Quoted 11 lines
drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c +11
-0+struct msi_desc *desc, + struct msi_msg *msg) +{ + int i; +int hwirq =-1 ; +int msi_src; +void __iomem *ring; +struct vfio_platform_region *reg = &vdev->regions[0 ]; + +if (!reg) +return ;
why do you need this check? For this to be called, SET_IRQ IOCTL must have been called. This can only happen after vfio_platform_open which first calls vfio_platform_regions_init and then vfio_platform_irq_init
drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c static int vfio_platform_bcmflexrm_reset(struct vfio_platform_device *vdev) · line 135
Quoted 4 lines
drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c +4
-0+ +for (i =0 ; i < vdev->num_irqs; i++) +if (vdev->irqs[i].type ==VFIO_IRQ_TYPE_MSI ) + hwirq = vdev->irqs[i].ctx[0 ].hwirq;
nit: It would have been easier to record in vdev the last index of wired interrupts and just add the index of the MSI
drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c static int vfio_platform_bcmflexrm_reset(struct vfio_platform_device *vdev) · line 144
Quoted 9 lines
drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c +9
-0+ +if (hwirq <0 ) +return ; + + msi_src = desc->irq - hwirq; + +if (!reg->ioaddr) { + reg->ioaddr =ioremap (reg->addr, reg->size); +if (!reg->ioaddr)
pr_warn_once("")?
Quoted 27 lines
drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c +26
-0+return ; + } + + ring = reg->ioaddr + msi_src *RING_REGS_SIZE ; + +writel_relaxed (msg->address_lo, ring +RING_MSI_ADDR_LS ); +writel_relaxed (msg->address_hi, ring +RING_MSI_ADDR_MS ); +writel_relaxed (msg->data, ring +RING_MSI_DATA_VALUE ); +} + +static struct vfio_platform_reset_node vfio_platform_bcmflexrm_reset_node = { + .owner =THIS_MODULE , + .compat ="brcm,iproc-flexrm-mbox" , + .of_reset = vfio_platform_bcmflexrm_reset, + .of_get_msi = bcm_num_msi, + .of_msi_write = bcm_write_msi +}; + +static int __init vfio_platform_bcmflexrm_reset_module_init (void ) +{ +__vfio_platform_register_reset (&vfio_platform_bcmflexrm_reset_node); + +return 0 ; +} + +static void __exit vfio_platform_bcmflexrm_reset_module_exit(void)
drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c static int vfio_platform_bcmflexrm_reset(struct vfio_platform_device *vdev) · line 180
Quoted 10 lines
drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c +7
-0+{ +vfio_platform_unregister_reset ("brcm,iproc-flexrm-mbox" , + vfio_platform_bcmflexrm_reset); +} + +module_init (vfio_platform_bcmflexrm_reset_module_init); +module_exit (vfio_platform_bcmflexrm_reset_module_exit);MODULE_LICENSE ("GPL v2" );MODULE_AUTHOR ("Anup Patel <[email protected]>" );
I think you should move the whole series in PATCH now.
Thanks
Eric