util/vfio-helpers: Add support for multiple IRQs
v5 cover util/vfio-helpers: Add support for multiple IRQs
This series intends to setup the VFIO helper to allow binding notifiers on different IRQs.
For the NVMe use case, we only care about MSIX interrupts. To not disrupt other users, introduce the qemu_vfio_pci_init_msix_irqs function to initialize multiple MSIX IRQs and attach eventfd to them.
Since RFC v4:
- addressed Alex review comment:
check ioctl(VFIO_DEVICE_SET_IRQS) return value
Since RFC v3:
- addressed Alex and Stefan review comments
Since RFC v2:
- new patch to report vfio-helpers is not supported on AA64/POWER
(NVMe block driver series will follow).
Based-on: <[email protected]> (Block layer pending pull request)
Philippe Mathieu-Daudé (4):
util/vfio-helpers: Improve reporting unsupported IOMMU type util/vfio-helpers: Report error when IOMMU page size is not supported util/vfio-helpers: Introduce qemu_vfio_pci_init_msix_irqs() block/nvme: Use qemu_vfio_pci_init_msix_irqs() to initialize our IRQ
include/qemu/vfio-helpers.h | 6 ++- block/nvme.c | 9 +++- util/vfio-helpers.c | 87 ++++++++++++++++++++++++++++++++++++- 3 files changed, 97 insertions(+), 5 deletions(-)
Signature · 1 line
2.26.2
RFC PATCH v5 1/4 util/vfio-helpers: Improve reporting unsupported IOMMU type
Change the confuse "VFIO IOMMU check failed" error message by the explicit "VFIO IOMMU Type1 is not supported" once.
Example on POWER:
$ qemu-system-ppc64 -drive if=none,id=nvme0,file=nvme://0001:01:00.0/1,format=raw qemu-system-ppc64: -drive if=none,id=nvme0,file=nvme://0001:01:00.0/1,format=raw: VFIO IOMMU Type1 is not supported
--- util/vfio-helpers.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
util/vfio-helpers.c +1 -1
@@ -261,7 +261,7 @@ static int qemu_vfio_init_pci(QEMUVFIOState *s, const char *device, }if (!ioctl (s->container,VFIO_CHECK_EXTENSION ,VFIO_TYPE1_IOMMU )) { -error_setg_errno (errp, errno,"VFIO IOMMU check failed" ); +error_setg_errno (errp, errno,"VFIO IOMMU Type1 is not supported" ); ret = -EINVAL ; goto fail_container; }
Signature · 1 line
2.26.2
RFC PATCH v5 2/4 util/vfio-helpers: Report error when IOMMU page size is not supported
In reply to Philippe Mathieu-DaudéThis driver uses the host page size to align its memory regions, but this size is not always compatible with the IOMMU. Add a check if the size matches, and bails out with listing the sizes the IOMMU supports.
Example on Aarch64:
$ qemu-system-aarch64 -M virt -drive if=none,id=nvme0,file=nvme://0006:90:00.0/1,format=raw qemu-system-aarch64: -drive if=none,id=nvme0,file=nvme://0006:90:00.0/1,format=raw: Unsupported IOMMU page size: 4 KiB Available page size: 64 KiB 512 MiB
--- util/vfio-helpers.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+)
util/vfio-helpers.c +20 -0
@@ -11,6 +11,7 @@*/ #include "qemu/osdep.h" +#include "qemu/cutils.h" #include <sys/ioctl.h> #include <linux/vfio.h> #include "qapi/error.h"
@@ -316,6 +317,25 @@ static int qemu_vfio_init_pci(QEMUVFIOState *s, const char *device, ret = -errno; goto fail; } +if (!(iommu_info.flags &VFIO_IOMMU_INFO_PGSIZES )) { +error_setg (errp,"Failed to get IOMMU page size info" ); + ret = -errno; + goto fail; + } +if (!extract64 (iommu_info.iova_pgsizes, +ctz64 (qemu_real_host_page_size),1 )) { + g_autofree char *host_page_size = size_to_str(qemu_real_host_page_size); +error_setg (errp,"Unsupported IOMMU page size: %s" , host_page_size); +error_append_hint (errp,"Available page size:\n" ); +for (int i =0 ; i <64 ; i++) { +if (extract64 (iommu_info.iova_pgsizes, i,1 )) { + g_autofree char *iova_pgsizes = size_to_str(1UL << i); +error_append_hint (errp," %s\n" , iova_pgsizes); + } + } + ret = -EINVAL ; + goto fail; + } s->device =ioctl (s->group,VFIO_GROUP_GET_DEVICE_FD , device);
Signature · 1 line
2.26.2
Re: [RFC PATCH v5 2/4] util/vfio-helpers: Report error when IOMMU page size is not supported
Quoted 32 lines · Philippe Mathieu-Daudé
This driver uses the host page size to align its memory regions, but this size is not always compatible with the IOMMU. Add a check if the size matches, and bails out with listing the sizes the IOMMU supports.
Example on Aarch64:
$ qemu-system-aarch64 -M virt -drive if=none,id=nvme0,file=nvme://0006:90:00.0/1,format=raw qemu-system-aarch64: -drive if=none,id=nvme0,file=nvme://0006:90:00.0/1,format=raw: Unsupported IOMMU page size: 4 KiB Available page size: 64 KiB 512 MiBSuggested-by Alex Williamson <[email protected]>Signed-off-by Philippe Mathieu-Daudé <[email protected]>--- util/vfio-helpers.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+)util/vfio-helpers.c +1
-0@@ -11,6 +11,7 @@*/ #include "qemu/osdep.h" +#include "qemu/cutils.h" #include <sys/ioctl.h> #include <linux/vfio.h> #include "qapi/error.h"
util/vfio-helpers.c static int qemu_vfio_init_pci(QEMUVFIOState *s, const char *device, · line 322
Quoted 6 lines
util/vfio-helpers.c +3
-0@@ -316,6 +317,25 @@ static int qemu_vfio_init_pci(QEMUVFIOState *s, const char *device, ret = -errno; goto fail; } +if (!(iommu_info.flags &VFIO_IOMMU_INFO_PGSIZES )) { +error_setg (errp,"Failed to get IOMMU page size info" ); + ret = -errno;
We don't have errno here, do we?
util/vfio-helpers.c static int qemu_vfio_init_pci(QEMUVFIOState *s, const char *device, · line 333
Quoted 11 lines
util/vfio-helpers.c +11
-0+ goto fail; + } +if (!extract64 (iommu_info.iova_pgsizes, +ctz64 (qemu_real_host_page_size),1 )) { + g_autofree char *host_page_size = size_to_str(qemu_real_host_page_size); +error_setg (errp,"Unsupported IOMMU page size: %s" , host_page_size); +error_append_hint (errp,"Available page size:\n" ); +for (int i =0 ; i <64 ; i++) { +if (extract64 (iommu_info.iova_pgsizes, i,1 )) { + g_autofree char *iova_pgsizes = size_to_str(1UL << i); +error_append_hint (errp," %s\n" , iova_pgsizes);
Interesting... Since it's printing page size which is fairly low level, why not just plain (hex) numbers?
Fam
Quoted 12 lines
util/vfio-helpers.c +5
-0+ } + } + ret = -EINVAL ; + goto fail; + } s->device =ioctl (s->group,VFIO_GROUP_GET_DEVICE_FD , device);Signature · 1 line
2.26.2
Re: [RFC PATCH v5 2/4] util/vfio-helpers: Report error when IOMMU page size is not supported
Quoted 27 lines · Fam Zheng
Quoted 26 lines · Philippe Mathieu-Daudé
This driver uses the host page size to align its memory regions, but this size is not always compatible with the IOMMU. Add a check if the size matches, and bails out with listing the sizes the IOMMU supports.
Example on Aarch64:
$ qemu-system-aarch64 -M virt -drive if=none,id=nvme0,file=nvme://0006:90:00.0/1,format=raw qemu-system-aarch64: -drive if=none,id=nvme0,file=nvme://0006:90:00.0/1,format=raw: Unsupported IOMMU page size: 4 KiB Available page size: 64 KiB 512 MiBSuggested-by Alex Williamson <[email protected]>Signed-off-by Philippe Mathieu-Daudé <[email protected]>--- util/vfio-helpers.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+)util/vfio-helpers.c +0
-0@@ -11,6 +11,7 @@*/
Quoted 14 lines
util/vfio-helpers.c static int qemu_vfio_init_pci(QEMUVFIOState *s, const char *device, · line 322
Quoted 12 lines
util/vfio-helpers.c +4
-0#include "qemu/osdep.h" +#include "qemu/cutils.h" #include <sys/ioctl.h> #include <linux/vfio.h> #include "qapi/error.h" @@ -316,6 +317,25 @@ static int qemu_vfio_init_pci(QEMUVFIOState *s, const char *device, ret = -errno; goto fail; } +if (!(iommu_info.flags &VFIO_IOMMU_INFO_PGSIZES )) { +error_setg (errp,"Failed to get IOMMU page size info" ); + ret = -errno;We don't have errno here, do we?
Oops thanks, I'll replace by "ret = -EINVAL;".
Quoted 13 lines
util/vfio-helpers.c static int qemu_vfio_init_pci(QEMUVFIOState *s, const char *device, · line 333
Quoted 12 lines
util/vfio-helpers.c +11
-0+ goto fail; + } +if (!extract64 (iommu_info.iova_pgsizes, +ctz64 (qemu_real_host_page_size),1 )) { + g_autofree char *host_page_size = size_to_str(qemu_real_host_page_size); +error_setg (errp,"Unsupported IOMMU page size: %s" , host_page_size); +error_append_hint (errp,"Available page size:\n" ); +for (int i =0 ; i <64 ; i++) { +if (extract64 (iommu_info.iova_pgsizes, i,1 )) { + g_autofree char *iova_pgsizes = size_to_str(1UL << i); +error_append_hint (errp," %s\n" , iova_pgsizes);
Quoted 2 lines
Interesting... Since it's printing page size which is fairly low level, why not just plain (hex) numbers?
Because this error is reported to the user (not the developer) and depends of the host/iommu.
If you don't mind I'll keep it as it (as the code is written).
Thank you for reviewing the series!
Phil.
Quoted 16 lines
Fam
Quoted 12 lines
util/vfio-helpers.c +5
-0+ } + } + ret = -EINVAL ; + goto fail; + } s->device =ioctl (s->group,VFIO_GROUP_GET_DEVICE_FD , device);Signature · 1 line
2.26.2
RFC PATCH v5 3/4 util/vfio-helpers: Introduce qemu_vfio_pci_init_msix_irqs()
In reply to Philippe Mathieu-Daudéqemu_vfio_pci_init_irq() allows us to initialize any type of IRQ, but only one. Introduce qemu_vfio_pci_init_msix_irqs() which is specific to MSIX IRQ type, and allow us to use multiple IRQs (thus passing multiple eventfd notifiers).
--- include/qemu/vfio-helpers.h | 6 +++- util/vfio-helpers.c | 65 ++++++++++++++++++++++++++++++++++++- 2 files changed, 69 insertions(+), 2 deletions(-)
include/qemu/vfio-helpers.h +5 -1
@@ -1,11 +1,13 @@/* * QEMU VFIO helpers * -* Copyright 2016 - 2018 Red Hat, Inc. +* Copyright 2016 - 2020 Red Hat, Inc. * * Authors: * Fam Zheng <[email protected]> +* Philippe Mathieu-Daudé <[email protected]> * +* SPDX-License-Identifier: GPL-2.0-or-later * This work is licensed under the terms of the GNU GPL, version 2 or later. * See the COPYING file in the top-level directory. */
@@ -28,5 +30,7 @@ void qemu_vfio_pci_unmap_bar(QEMUVFIOState *s, int index, void *bar, uint64_t offset, uint64_t size);int qemu_vfio_pci_init_irq (QEMUVFIOState *s,EventNotifier *e,int irq_type,Error **errp); +int qemu_vfio_pci_init_msix_irqs (QEMUVFIOState *s,EventNotifier *e, +unsigned *irq_count,Error **errp);#endif
util/vfio-helpers.c +64 -1
81 lines not on this page · raw message
Signature · 1 line
2.26.2
RFC PATCH v5 4/4 block/nvme: Use qemu_vfio_pci_init_msix_irqs() to initialize our IRQ
In reply to Philippe Mathieu-DaudéInstead of initializing one MSIX IRQ with the generic qemu_vfio_pci_init_irq() function, use the MSIX specific one which ill allow us to use multiple IRQs. For now we provide an array of a single IRQ.
--- block/nvme.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-)
block/nvme.c +7 -2
@@ -694,6 +694,7 @@ static int nvme_init(BlockDriverState *bs, const char *device, int namespace,uint64_t timeout_ms;uint64_t deadline, now;Error *local_err =NULL ; +unsigned irq_count =MSIX_IRQ_COUNT ;qemu_co_mutex_init (&s->dma_map_lock);qemu_co_queue_init (&s->dma_flush_queue);
@@ -779,9 +780,13 @@ static int nvme_init(BlockDriverState *bs, const char *device, int namespace, } } - ret =qemu_vfio_pci_init_irq (s->vfio, s->irq_notifier, -VFIO_PCI_MSIX_IRQ_INDEX , errp); + ret =qemu_vfio_pci_init_msix_irqs (s->vfio, s->irq_notifier, + &irq_count, errp);if (ret) { +if (ret == -EOVERFLOW ) { +error_append_hint (errp,"%u IRQs requested but only %u available\n" , +MSIX_IRQ_COUNT , irq_count); + } goto out; } aio_set_event_notifier(bdrv_get_aio_context(bs),
Signature · 1 line
2.26.2
Re: [RFC PATCH v5 4/4] block/nvme: Use qemu_vfio_pci_init_msix_irqs() to initialize our IRQ
Quoted 3 lines · Philippe Mathieu-Daudé
Instead of initializing one MSIX IRQ with the generic
qemu_vfio_pci_init_irq()function, use the MSIX specific one which ill allow us to use multiple IRQs. For now we provide an array of
s/ill/will/
Quoted 3 lines
a single IRQ.
Signed-off-by Philippe Mathieu-Daudé <[email protected]>
Fam
Quoted 11 lines · Philippe Mathieu-Daudé
This series intends to setup the VFIO helper to allow binding notifiers on different IRQs.
For the NVMe use case, we only care about MSIX interrupts. To not disrupt other users, introduce the
qemu_vfio_pci_init_msix_irqsfunction to initialize multiple MSIX IRQs and attach eventfd to them.Since RFC v4:
- addressed Alex review comment:
checkioctl(VFIO_DEVICE_SET_IRQS)return value