util/vfio-helpers: Add support for multiple IRQs
v2 cover util/vfio-helpers: Add support for multiple IRQs
This series intends to setup the VFIO helper to allow binding notifiers on different IRQs. Only MSIX IRQ type considered so far.
Stefan suggested me to publish earlier to discuss. If not too bad feedbacks I'll add some documentation in "qemu/vfio-helpers.h" before reposting.
(NVMe block driver series will follow).
Based-on: <[email protected]> "block/nvme: Various cleanups required to use multiple queues"
https://www.mail-archive.com/[email protected]/msg729395.html
Supersedes: <[email protected]>
Philippe Mathieu-Daudé (7):
util/vfio-helpers: Store eventfd using int32_t type util/vfio-helpers: Move IRQ 'type' from pci_init_irq() to open_pci() util/vfio-helpers: Introduce 'irq_count' variable util/vfio-helpers: Check the device allow up to 'irq_count' IRQs util/vfio-helpers: Support multiple eventfd util/vfio-helpers: Allow to set EventNotifier to particular IRQ util/vfio-helpers: Allow opening device requesting for multiple IRQs
include/qemu/vfio-helpers.h | 5 +++-- block/nvme.c | 9 +++++--- util/vfio-helpers.c | 42 ++++++++++++++++++++++++++++++------- util/trace-events | 1 + 4 files changed, 45 insertions(+), 12 deletions(-)
Signature · 1 line
2.21.3
RFC PATCH v2 1/7 util/vfio-helpers: Store eventfd using int32_t type
Per the documentation in linux-headers/linux/vfio.h:
VFIO_DEVICE_SET_IRQS
* DATA_EVENTFD binds the specified ACTION to the provided __s32 eventfd.
Replace the 'int' by an 'int32_t' to match the documentation.
--- util/vfio-helpers.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
util/vfio-helpers.c +2 -2
@@ -193,7 +193,7 @@ int qemu_vfio_pci_init_irq(QEMUVFIOState *s, EventNotifier *e,return -EINVAL ; } - irq_set_size =sizeof (*irq_set) +sizeof (int ); + irq_set_size =sizeof (*irq_set) +sizeof (int32_t ); irq_set =g_malloc0 (irq_set_size);/* Get to a known IRQ state */
@@ -205,7 +205,7 @@ int qemu_vfio_pci_init_irq(QEMUVFIOState *s, EventNotifier *e, .count = 1, }; - *(int *)&irq_set->data =event_notifier_get_fd (e); + *(int32_t *)&irq_set->data =event_notifier_get_fd (e); r =ioctl (s->device,VFIO_DEVICE_SET_IRQS , irq_set);g_free (irq_set);if (r) {
Signature · 1 line
2.21.3
RFC PATCH v2 2/7 util/vfio-helpers: Move IRQ 'type' from pci_init_irq() to open_pci()
In reply to Philippe Mathieu-DaudéOnce opened, we will used the same IRQ type for all our event notifiers, so pass the argument when we open the PCI device, store the IRQ type in the driver state, and directly use the value saved in the state each time we call qemu_vfio_pci_init_irq.
--- include/qemu/vfio-helpers.h | 5 +++-- block/nvme.c | 6 +++--- util/vfio-helpers.c | 13 +++++++++---- 3 files changed, 15 insertions(+), 9 deletions(-)
include/qemu/vfio-helpers.h +3 -2
@@ -15,7 +15,8 @@typedef struct QEMUVFIOState QEMUVFIOState ; -QEMUVFIOState *qemu_vfio_open_pci (const char *device,Error **errp); +QEMUVFIOState *qemu_vfio_open_pci (const char *device,int irq_type, +Error **errp);void qemu_vfio_close (QEMUVFIOState *s);int qemu_vfio_dma_map (QEMUVFIOState *s,void *host,size_t size,bool temporary,uint64_t *iova_list);
@@ -27,6 +28,6 @@ void *qemu_vfio_pci_map_bar(QEMUVFIOState *s, int index,void qemu_vfio_pci_unmap_bar (QEMUVFIOState *s,int index,void *bar,uint64_t offset,uint64_t size);int qemu_vfio_pci_init_irq (QEMUVFIOState *s,EventNotifier *e, -int irq_type,Error **errp); +Error **errp);#endif
block/nvme.c +3 -3
@@ -711,7 +711,8 @@ static int nvme_init(BlockDriverState *bs, const char *device, int namespace,return ret; } - s->vfio =qemu_vfio_open_pci (device, errp); + s->vfio =qemu_vfio_open_pci (device,VFIO_PCI_MSIX_IRQ_INDEX , + errp);if (!s->vfio) { ret = -EINVAL ; goto out;
@@ -784,8 +785,7 @@ static int nvme_init(BlockDriverState *bs, const char *device, int namespace, } } - ret =qemu_vfio_pci_init_irq (s->vfio, &s->irq_notifier, -VFIO_PCI_MSIX_IRQ_INDEX , errp); + ret =qemu_vfio_pci_init_irq (s->vfio, &s->irq_notifier, errp);if (ret) { goto out; }
util/vfio-helpers.c +9 -4
@@ -43,6 +43,8 @@ typedef struct {struct QEMUVFIOState {QemuMutex lock; +int irq_type;/* vfio index */ +/* These fields are protected by BQL */ int container;int group;
@@ -176,14 +178,14 @@ void qemu_vfio_pci_unmap_bar(QEMUVFIOState *s, int index, void *bar,* Initialize device IRQ with @irq_type and and register an event notifier. */ int qemu_vfio_pci_init_irq (QEMUVFIOState *s,EventNotifier *e, -int irq_type,Error **errp) +Error **errp) {int r;struct vfio_irq_set *irq_set;size_t irq_set_size;struct vfio_irq_info irq_info = { .argsz =sizeof (irq_info) }; - irq_info.index = irq_type; + irq_info.index = s->irq_type;if (ioctl (s->device,VFIO_DEVICE_GET_IRQ_INFO , &irq_info)) {error_setg_errno (errp, errno,"Failed to get device interrupt info" );return -errno;
@@ -237,6 +239,7 @@ static int qemu_vfio_pci_write_config(QEMUVFIOState *s, void *buf, int size, int }static int qemu_vfio_init_pci (QEMUVFIOState *s,const char *device, +int irq_type,Error **errp) {int ret;
@@ -331,6 +334,7 @@ static int qemu_vfio_init_pci(QEMUVFIOState *s, const char *device, ret = -errno; goto fail; } + s->irq_type = irq_type;if (device_info.num_regions <VFIO_PCI_CONFIG_REGION_INDEX ) {error_setg (errp,"Invalid device regions" );
@@ -423,12 +427,13 @@ static void qemu_vfio_open_common(QEMUVFIOState *s)/** * Open a PCI device, e.g. "0000:00:01.0". */ -QEMUVFIOState *qemu_vfio_open_pci (const char *device,Error **errp) +QEMUVFIOState *qemu_vfio_open_pci (const char *device,int irq_type, +Error **errp) {int r;QEMUVFIOState *s =g_new0 (QEMUVFIOState,1 ); - r =qemu_vfio_init_pci (s, device, errp); + r =qemu_vfio_init_pci (s, device, irq_type, errp);if (r) {g_free (s);return NULL ;
Signature · 1 line
2.21.3
Re: [RFC PATCH v2 2/7] util/vfio-helpers: Move IRQ 'type' from pci_init_irq() to open_pci()
Quoted 7 lines · On Thu, 13 Aug 2020 19:29:52 +0200 Philippe Mathieu-Daudé <[email protected]>…
Once opened, we will used the same IRQ type for all our event notifiers, so pass the argument when we open the PCI device, store the IRQ type in the driver state, and directly use the value saved in the state each time we call
qemu_vfio_pci_init_irq.Signed-off-by Philippe Mathieu-Daudé <[email protected]>---
This feels quite a bit strange to me, a PCI device can operate in one of several interrupt modes, or without interrupts at all. Why would we force a user of this interface to define the interrupt type they'll use in advance and then not even verify if the device supports that type? A driver might want to fall back to a different interrupt type if the one they want is not supported. If we want to abstract this from the driver then we should at least have an interface separate from the initial open function that tells us to preconfigure some specified number of vectors. We could then have a preference policy that would attempt to use MSI-X, followed by MSI, followed by INTx (assuming request is for a single vector), based on what the device supports. Then a driver could fallback to fewer interrupts if the device does not support, or the host system cannot provide, the desired number of interrupts. Thanks,
Alex
Quoted 113 lines
include/qemu/vfio-helpers.h | 5 +++-- block/nvme.c | 6 +++--- util/vfio-helpers.c | 13 +++++++++---- 3 files changed, 15 insertions(+), 9 deletions(-)include/qemu/vfio-helpers.h +3
-2@@ -15,7 +15,8 @@typedef struct QEMUVFIOState QEMUVFIOState ; -QEMUVFIOState *qemu_vfio_open_pci (const char *device,Error **errp); +QEMUVFIOState *qemu_vfio_open_pci (const char *device,int irq_type, +Error **errp);void qemu_vfio_close (QEMUVFIOState *s);int qemu_vfio_dma_map (QEMUVFIOState *s,void *host,size_t size,bool temporary,uint64_t *iova_list);@@ -27,6 +28,6 @@ void *qemu_vfio_pci_map_bar(QEMUVFIOState *s, int index,void qemu_vfio_pci_unmap_bar (QEMUVFIOState *s,int index,void *bar,uint64_t offset,uint64_t size);int qemu_vfio_pci_init_irq (QEMUVFIOState *s,EventNotifier *e, -int irq_type,Error **errp); +Error **errp);#endif block/nvme.c +3
-3@@ -711,7 +711,8 @@ static int nvme_init(BlockDriverState *bs, const char *device, int namespace,return ret; } - s->vfio =qemu_vfio_open_pci (device, errp); + s->vfio =qemu_vfio_open_pci (device,VFIO_PCI_MSIX_IRQ_INDEX , + errp);if (!s->vfio) { ret = -EINVAL ; goto out;@@ -784,8 +785,7 @@ static int nvme_init(BlockDriverState *bs, const char *device, int namespace, } } - ret =qemu_vfio_pci_init_irq (s->vfio, &s->irq_notifier, -VFIO_PCI_MSIX_IRQ_INDEX , errp); + ret =qemu_vfio_pci_init_irq (s->vfio, &s->irq_notifier, errp);if (ret) { goto out; }util/vfio-helpers.c +9
-4@@ -43,6 +43,8 @@ typedef struct {struct QEMUVFIOState {QemuMutex lock; +int irq_type;/* vfio index */ +/* These fields are protected by BQL */ int container;int group;@@ -176,14 +178,14 @@ void qemu_vfio_pci_unmap_bar(QEMUVFIOState *s, int index, void *bar,* Initialize device IRQ with @irq_type and and register an event notifier. */ int qemu_vfio_pci_init_irq (QEMUVFIOState *s,EventNotifier *e, -int irq_type,Error **errp) +Error **errp) {int r;struct vfio_irq_set *irq_set;size_t irq_set_size;struct vfio_irq_info irq_info = { .argsz =sizeof (irq_info) }; - irq_info.index = irq_type; + irq_info.index = s->irq_type;if (ioctl (s->device,VFIO_DEVICE_GET_IRQ_INFO , &irq_info)) {error_setg_errno (errp, errno,"Failed to get device interrupt info" );return -errno;@@ -237,6 +239,7 @@ static int qemu_vfio_pci_write_config(QEMUVFIOState *s, void *buf, int size, int }static int qemu_vfio_init_pci (QEMUVFIOState *s,const char *device, +int irq_type,Error **errp) {int ret;@@ -331,6 +334,7 @@ static int qemu_vfio_init_pci(QEMUVFIOState *s, const char *device, ret = -errno; goto fail; } + s->irq_type = irq_type;if (device_info.num_regions <VFIO_PCI_CONFIG_REGION_INDEX ) {error_setg (errp,"Invalid device regions" );@@ -423,12 +427,13 @@ static void qemu_vfio_open_common(QEMUVFIOState *s)/** * Open a PCI device, e.g. "0000:00:01.0". */ -QEMUVFIOState *qemu_vfio_open_pci (const char *device,Error **errp) +QEMUVFIOState *qemu_vfio_open_pci (const char *device,int irq_type, +Error **errp) {int r;QEMUVFIOState *s =g_new0 (QEMUVFIOState,1 ); - r =qemu_vfio_init_pci (s, device, errp); + r =qemu_vfio_init_pci (s, device, irq_type, errp);if (r) {g_free (s);return NULL ;
RFC PATCH v2 3/7 util/vfio-helpers: Introduce 'irq_count' variable
In reply to Philippe Mathieu-DaudéCurrently this helper is restricted to a single VFIO (MSIX) IRQ. As we will slowly make it support multiple IRQs, introduce the 'irq_count' variable which contains the total number of IRQs we initialized the device with.
Set the variable in qemu_vfio_init_pci().
Hardcode the current single IRQ used.
--- util/vfio-helpers.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-)
util/vfio-helpers.c +6 -4
@@ -44,6 +44,7 @@ struct QEMUVFIOState {QemuMutex lock;int irq_type;/* vfio index */ +size_t irq_count;/* vfio subindex (vector) */ /* These fields are protected by BQL */ int container;
@@ -195,7 +196,7 @@ int qemu_vfio_pci_init_irq(QEMUVFIOState *s, EventNotifier *e,return -EINVAL ; } - irq_set_size =sizeof (*irq_set) +sizeof (int32_t ); + irq_set_size =sizeof (*irq_set) + s->irq_count *sizeof (int32_t ); irq_set =g_malloc0 (irq_set_size);/* Get to a known IRQ state */
@@ -204,7 +205,7 @@ int qemu_vfio_pci_init_irq(QEMUVFIOState *s, EventNotifier *e, .flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER, .index = irq_info.index, .start = 0, - .count = 1, + .count = s->irq_count, }; *(int32_t *)&irq_set->data =event_notifier_get_fd (e);
@@ -239,7 +240,7 @@ static int qemu_vfio_pci_write_config(QEMUVFIOState *s, void *buf, int size, int }static int qemu_vfio_init_pci (QEMUVFIOState *s,const char *device, -int irq_type, +int irq_type,unsigned irq_count,Error **errp) {int ret;
@@ -335,6 +336,7 @@ static int qemu_vfio_init_pci(QEMUVFIOState *s, const char *device, goto fail; } s->irq_type = irq_type; + s->irq_count = irq_count;if (device_info.num_regions <VFIO_PCI_CONFIG_REGION_INDEX ) {error_setg (errp,"Invalid device regions" );
@@ -433,7 +435,7 @@ QEMUVFIOState *qemu_vfio_open_pci(const char *device, int irq_type,int r;QEMUVFIOState *s =g_new0 (QEMUVFIOState,1 ); - r =qemu_vfio_init_pci (s, device, irq_type, errp); + r =qemu_vfio_init_pci (s, device, irq_type,1 , errp);if (r) {g_free (s);return NULL ;
Signature · 1 line
2.21.3
RFC PATCH v2 4/7 util/vfio-helpers: Check the device allow up to 'irq_count' IRQs
In reply to Philippe Mathieu-DaudéAs we want to use more than one single IRQ, add a check that the device accept our request to use multiple IRQs.
--- util/vfio-helpers.c | 6 ++++++ util/trace-events | 1 + 2 files changed, 7 insertions(+)
util/vfio-helpers.c +6 -0
@@ -335,6 +335,12 @@ static int qemu_vfio_init_pci(QEMUVFIOState *s, const char *device, ret = -errno; goto fail; } +trace_qemu_vfio_init_pci (device_info.num_irqs); +if (device_info.num_irqs < irq_count) { +error_setg (errp,"Invalid device IRQ count" ); + ret = -EINVAL ; + goto fail; + } s->irq_type = irq_type; s->irq_count = irq_count;
util/trace-events +1 -0
@@ -83,3 +83,4 @@ qemu_vfio_new_mapping(void *s, void *host, size_t size, int index, uint64_t iova qemu_vfio_do_mapping(void *s, void *host, size_t size, uint64_t iova) "s %p host %p size %zu iova 0x%"PRIx64 qemu_vfio_dma_map(void *s, void *host, size_t size, bool temporary, uint64_t *iova) "s %p host %p size %zu temporary %d iova %p" qemu_vfio_dma_unmap(void *s, void *host) "s %p host %p" +qemu_vfio_init_pci(uint32_t count) "device interrupt count: %"PRIu32
Signature · 1 line
2.21.3
Re: [RFC PATCH v2 4/7] util/vfio-helpers: Check the device allow up to 'irq_count' IRQs
Quoted 14 lines · On Thu, 13 Aug 2020 19:29:54 +0200 Philippe Mathieu-Daudé <[email protected]>…
As we want to use more than one single IRQ, add a check that the device accept our request to use multiple IRQs.
Signed-off-by Philippe Mathieu-Daudé <[email protected]>--- util/vfio-helpers.c | 6 ++++++ util/trace-events | 1 + 2 files changed, 7 insertions(+)util/vfio-helpers.c +0
-0
util/vfio-helpers.c static int qemu_vfio_init_pci(QEMUVFIOState *s, const char *device, · line 343
Quoted 9 lines
util/vfio-helpers.c +6
-0@@ -335,6 +335,12 @@ static int qemu_vfio_init_pci(QEMUVFIOState *s, const char *device, ret = -errno; goto fail; } +trace_qemu_vfio_init_pci (device_info.num_irqs); +if (device_info.num_irqs < irq_count) { +error_setg (errp,"Invalid device IRQ count" ); + ret = -EINVAL ; + goto fail; + }
This is confusing the number of IRQ indexes (ie. IRQ types - INTx/MSI/MSIx plus virtual interrupts like error reporting and device request) with the number of sub-indexes available for a given type again. You actually need to look at VFIO_DEVICE_GET_IRQ_INFO for the specified irq_type to see if it supports irq_count sub-indexes.
Maybe think of interrupts as a 2-dimensional array, we have:
INDEX \ SUBINDEX
\ 0 1 2 3 4 ... N ========================== INTx [0]| MSI [1]| MSI-X [2]| ... [M]|
VFIO_DEVICE_GET_INFO only tells us essentially the last INDEX that the device supports. In order to learn about the number of SUBINDEXes, or vectors, if any, that each INDEX provides, we need to look at VFIO_DEVICE_GET_IRQ_INFO. When we're wanting to probe support for some number of concurrent device interrupt vectors, we need to look at the vfio_irq_info.count value for the desired index, ie. the extent of the entries in the row associated with our column index type. Thanks,
Alex
Quoted 12 lines
util/vfio-helpers.c +0
-0s->irq_type = irq_type; s->irq_count = irq_count;util/trace-events +1
-0@@ -83,3 +83,4 @@ qemu_vfio_new_mapping(void *s, void *host, size_t size, int index, uint64_t iova qemu_vfio_do_mapping(void *s, void *host, size_t size, uint64_t iova) "s %p host %p size %zu iova 0x%"PRIx64 qemu_vfio_dma_map(void *s, void *host, size_t size, bool temporary, uint64_t *iova) "s %p host %p size %zu temporary %d iova %p" qemu_vfio_dma_unmap(void *s, void *host) "s %p host %p" +qemu_vfio_init_pci(uint32_t count) "device interrupt count: %"PRIu32
RFC PATCH v2 5/7 util/vfio-helpers: Support multiple eventfd
In reply to Philippe Mathieu-DaudéWhen using multiple IRQs, we'll assign an eventfd to each IRQ. Be ready by holding an array of eventfd file descriptors in the instance state, so when we assign new IRQs we will still use the previous eventfds for the already assigned IRQs.
--- util/vfio-helpers.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-)
util/vfio-helpers.c +9 -1
@@ -45,6 +45,7 @@ struct QEMUVFIOState {int irq_type;/* vfio index */ size_t irq_count;/* vfio subindex (vector) */ +int32_t *eventfd;/* These fields are protected by BQL */ int container;
@@ -195,6 +196,7 @@ int qemu_vfio_pci_init_irq(QEMUVFIOState *s, EventNotifier *e,error_setg (errp,"Device interrupt doesn't support eventfd" );return -EINVAL ; } + s->eventfd[0 ] =event_notifier_get_fd (e); irq_set_size =sizeof (*irq_set) + s->irq_count *sizeof (int32_t ); irq_set =g_malloc0 (irq_set_size);
@@ -207,8 +209,8 @@ int qemu_vfio_pci_init_irq(QEMUVFIOState *s, EventNotifier *e, .start = 0, .count = s->irq_count, }; +memcpy (&irq_set->data, &s->eventfd, s->irq_count *sizeof (int32_t )); - *(int32_t *)&irq_set->data =event_notifier_get_fd (e); r =ioctl (s->device,VFIO_DEVICE_SET_IRQS , irq_set);g_free (irq_set);if (r) {
@@ -343,6 +345,10 @@ static int qemu_vfio_init_pci(QEMUVFIOState *s, const char *device, } s->irq_type = irq_type; s->irq_count = irq_count; + s->eventfd =g_new (int32_t, irq_count); +for (i =0 ; i < irq_count; i++) { + s->eventfd[i] =-1 ; + }if (device_info.num_regions <VFIO_PCI_CONFIG_REGION_INDEX ) {error_setg (errp,"Invalid device regions" );
@@ -379,6 +385,7 @@ static int qemu_vfio_init_pci(QEMUVFIOState *s, const char *device, }return 0 ; fail: +g_free (s->eventfd);close (s->group); fail_container:close (s->container);
@@ -730,6 +737,7 @@ void qemu_vfio_close(QEMUVFIOState *s) }ram_block_notifier_remove (&s->ram_notifier);qemu_vfio_reset (s); +g_free (s->eventfd);close (s->device);close (s->group);close (s->container);
Signature · 1 line
2.21.3
RFC PATCH v2 6/7 util/vfio-helpers: Allow to set EventNotifier to particular IRQ
In reply to Philippe Mathieu-DaudéLet qemu_vfio_pci_init_irq() take an 'index' argument, so we can set the EventNotifier to a specific IRQ.
Add a safety check. Since our helper is limited to one single IRQ we are safe.
Our only user is the NVMe block driver, update it (also safe because it only uses the first IRQ).
--- include/qemu/vfio-helpers.h | 2 +- block/nvme.c | 2 +- util/vfio-helpers.c | 11 +++++++++-- 3 files changed, 11 insertions(+), 4 deletions(-)
include/qemu/vfio-helpers.h +1 -1
@@ -28,6 +28,6 @@ void *qemu_vfio_pci_map_bar(QEMUVFIOState *s, int index,void qemu_vfio_pci_unmap_bar (QEMUVFIOState *s,int index,void *bar,uint64_t offset,uint64_t size);int qemu_vfio_pci_init_irq (QEMUVFIOState *s,EventNotifier *e, -Error **errp); +int irq_index,Error **errp);#endif
block/nvme.c +1 -1
@@ -785,7 +785,7 @@ static int nvme_init(BlockDriverState *bs, const char *device, int namespace, } } - ret =qemu_vfio_pci_init_irq (s->vfio, &s->irq_notifier, errp); + ret =qemu_vfio_pci_init_irq (s->vfio, &s->irq_notifier,INDEX_ADMIN , errp);if (ret) { goto out; }
util/vfio-helpers.c +9 -2
@@ -180,13 +180,20 @@ void qemu_vfio_pci_unmap_bar(QEMUVFIOState *s, int index, void *bar,* Initialize device IRQ with @irq_type and and register an event notifier. */ int qemu_vfio_pci_init_irq (QEMUVFIOState *s,EventNotifier *e, -Error **errp) +int irq_index,Error **errp) {int r;struct vfio_irq_set *irq_set;size_t irq_set_size;struct vfio_irq_info irq_info = { .argsz =sizeof (irq_info) }; +if (irq_index >= s->irq_count) { +error_setg (errp, +"Illegal interrupt %d (device initialized for %zu in total)" , + irq_index, s->irq_count); +return -EINVAL ; + } + irq_info.index = s->irq_type;if (ioctl (s->device,VFIO_DEVICE_GET_IRQ_INFO , &irq_info)) {error_setg_errno (errp, errno,"Failed to get device interrupt info" );
@@ -196,7 +203,7 @@ int qemu_vfio_pci_init_irq(QEMUVFIOState *s, EventNotifier *e,error_setg (errp,"Device interrupt doesn't support eventfd" );return -EINVAL ; } - s->eventfd[0 ] =event_notifier_get_fd (e); + s->eventfd[irq_index] =event_notifier_get_fd (e); irq_set_size =sizeof (*irq_set) + s->irq_count *sizeof (int32_t ); irq_set =g_malloc0 (irq_set_size);
Signature · 1 line
2.21.3
Re: [RFC PATCH v2 6/7] util/vfio-helpers: Allow to set EventNotifier to particular IRQ
Quoted 67 lines · On Thu, 13 Aug 2020 19:29:56 +0200 Philippe Mathieu-Daudé <[email protected]>…
Let
qemu_vfio_pci_init_irq()take an 'index' argument, so we can set the EventNotifier to a specific IRQ.
Add a safety check. Since our helper is limited to one single IRQ we are safe.Our only user is the NVMe block driver, update it (also safe because it only uses the first IRQ).
Signed-off-by Philippe Mathieu-Daudé <[email protected]>--- include/qemu/vfio-helpers.h | 2 +- block/nvme.c | 2 +- util/vfio-helpers.c | 11 +++++++++-- 3 files changed, 11 insertions(+), 4 deletions(-)include/qemu/vfio-helpers.h +1
-1@@ -28,6 +28,6 @@ void *qemu_vfio_pci_map_bar(QEMUVFIOState *s, int index,void qemu_vfio_pci_unmap_bar (QEMUVFIOState *s,int index,void *bar,uint64_t offset,uint64_t size);int qemu_vfio_pci_init_irq (QEMUVFIOState *s,EventNotifier *e, -Error **errp); +int irq_index,Error **errp);#endif block/nvme.c +1
-1@@ -785,7 +785,7 @@ static int nvme_init(BlockDriverState *bs, const char *device, int namespace, } } - ret =qemu_vfio_pci_init_irq (s->vfio, &s->irq_notifier, errp); + ret =qemu_vfio_pci_init_irq (s->vfio, &s->irq_notifier,INDEX_ADMIN , errp);if (ret) { goto out; }util/vfio-helpers.c +8
-1@@ -180,13 +180,20 @@ void qemu_vfio_pci_unmap_bar(QEMUVFIOState *s, int index, void *bar,* Initialize device IRQ with @irq_type and and register an event notifier. */ int qemu_vfio_pci_init_irq (QEMUVFIOState *s,EventNotifier *e, -Error **errp) +int irq_index,Error **errp) {int r;struct vfio_irq_set *irq_set;size_t irq_set_size;struct vfio_irq_info irq_info = { .argsz =sizeof (irq_info) }; +if (irq_index >= s->irq_count) { +error_setg (errp, +"Illegal interrupt %d (device initialized for %zu in total)" , + irq_index, s->irq_count); +return -EINVAL ; + } + irq_info.index = s->irq_type;if (ioctl (s->device,VFIO_DEVICE_GET_IRQ_INFO , &irq_info)) {error_setg_errno (errp, errno,"Failed to get device interrupt info" );
util/vfio-helpers.c int qemu_vfio_pci_init_irq(QEMUVFIOState *s, EventNotifier *e, · line 206
Quoted 5 lines
util/vfio-helpers.c +1
-1@@ -196,7 +203,7 @@ int qemu_vfio_pci_init_irq(QEMUVFIOState *s, EventNotifier *e,error_setg (errp,"Device interrupt doesn't support eventfd" );return -EINVAL ; } - s->eventfd[0 ] =event_notifier_get_fd (e); + s->eventfd[irq_index] =event_notifier_get_fd (e);
This can't work. For each fd in the array provided the kernel is going to try to get that fd and configure it as an eventfd. For each call until we set all eventfd index {0..irq_count}, this SET_IRQS ioctl will fail. I would probably make that pre-configure function I referred to earlier and create a single spurious interrupt eventfd and configure all of the vectors to signal that one eventfd. You could then have this per vector callback swap the eventfd with the caller provided one for the given vector.
NB, I don't know if you're going to run into trouble with this scheme with the fact that devices can behave differently based on the number of vectors they have enabled. You're creating an interface for a driver, so presumably that driver knows, for example, that as soon as vector N is enabled, signaling for event foo moves from vector 0 to vector N. Thanks,
Alex
Quoted 3 lines
util/vfio-helpers.c +0
-0irq_set_size =sizeof (*irq_set) + s->irq_count *sizeof (int32_t ); irq_set =g_malloc0 (irq_set_size);
RFC PATCH v2 7/7 util/vfio-helpers: Allow opening device requesting for multiple IRQs
In reply to Philippe Mathieu-DaudéNow that our helper is ready for handling multiple IRQs, let qemu_vfio_open_pci() take an 'irq_count' argument.
--- include/qemu/vfio-helpers.h | 2 +- block/nvme.c | 5 ++++- util/vfio-helpers.c | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-)
include/qemu/vfio-helpers.h +1 -1
@@ -16,7 +16,7 @@typedef struct QEMUVFIOState QEMUVFIOState ;QEMUVFIOState *qemu_vfio_open_pci (const char *device,int irq_type, -Error **errp); +unsigned irq_count,Error **errp);void qemu_vfio_close (QEMUVFIOState *s);int qemu_vfio_dma_map (QEMUVFIOState *s,void *host,size_t size,bool temporary,uint64_t *iova_list);
block/nvme.c +4 -1
@@ -106,6 +106,9 @@ QEMU_BUILD_BUG_ON(offsetof(NVMeRegs, doorbells) != 0x1000);#define INDEX_ADMIN 0#define INDEX_IO (n) (1 + n) +/* This driver shares a single MSIX IRQ for the admin and I/O queues */ +#define MSIX_IRQ_COUNT 1 +struct BDRVNVMeState {AioContext *aio_context;QEMUVFIOState *vfio;
@@ -712,7 +715,7 @@ static int nvme_init(BlockDriverState *bs, const char *device, int namespace, } s->vfio =qemu_vfio_open_pci (device,VFIO_PCI_MSIX_IRQ_INDEX , - errp); +MSIX_IRQ_COUNT , errp);if (!s->vfio) { ret = -EINVAL ; goto out;
util/vfio-helpers.c +2 -2
@@ -450,12 +450,12 @@ static void qemu_vfio_open_common(QEMUVFIOState *s)* Open a PCI device, e.g. "0000:00:01.0". */ QEMUVFIOState *qemu_vfio_open_pci (const char *device,int irq_type, -Error **errp) +unsigned irq_count,Error **errp) {int r;QEMUVFIOState *s =g_new0 (QEMUVFIOState,1 ); - r =qemu_vfio_init_pci (s, device, irq_type,1 , errp); + r =qemu_vfio_init_pci (s, device, irq_type, irq_count, errp);if (r) {g_free (s);return NULL ;
Signature · 1 line
2.21.3
Re: [RFC PATCH v2 7/7] util/vfio-helpers: Allow opening device requesting for multiple IRQs
Quoted 5 lines · On Thu, 13 Aug 2020 19:29:57 +0200 Philippe Mathieu-Daudé <[email protected]>…
Now that our helper is ready for handling multiple IRQs, let
qemu_vfio_open_pci()take an 'irq_count' argument.Signed-off-by Philippe Mathieu-Daudé <[email protected]>---
As with patch 2/ tying IRQ setup with the opening of a device seems wrong. Get the device open, then create an interface to configure the interrupt. Thanks,
Alex
Quoted 60 lines
include/qemu/vfio-helpers.h | 2 +- block/nvme.c | 5 ++++- util/vfio-helpers.c | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-)include/qemu/vfio-helpers.h +1
-1@@ -16,7 +16,7 @@typedef struct QEMUVFIOState QEMUVFIOState ;QEMUVFIOState *qemu_vfio_open_pci (const char *device,int irq_type, -Error **errp); +unsigned irq_count,Error **errp);void qemu_vfio_close (QEMUVFIOState *s);int qemu_vfio_dma_map (QEMUVFIOState *s,void *host,size_t size,bool temporary,uint64_t *iova_list);block/nvme.c +4
-1@@ -106,6 +106,9 @@ QEMU_BUILD_BUG_ON(offsetof(NVMeRegs, doorbells) != 0x1000);#define INDEX_ADMIN 0#define INDEX_IO (n) (1 + n) +/* This driver shares a single MSIX IRQ for the admin and I/O queues */ +#define MSIX_IRQ_COUNT 1 +struct BDRVNVMeState {AioContext *aio_context;QEMUVFIOState *vfio;@@ -712,7 +715,7 @@ static int nvme_init(BlockDriverState *bs, const char *device, int namespace, } s->vfio =qemu_vfio_open_pci (device,VFIO_PCI_MSIX_IRQ_INDEX , - errp); +MSIX_IRQ_COUNT , errp);if (!s->vfio) { ret = -EINVAL ; goto out;util/vfio-helpers.c +2
-2@@ -450,12 +450,12 @@ static void qemu_vfio_open_common(QEMUVFIOState *s)* Open a PCI device, e.g. "0000:00:01.0". */ QEMUVFIOState *qemu_vfio_open_pci (const char *device,int irq_type, -Error **errp) +unsigned irq_count,Error **errp) {int r;QEMUVFIOState *s =g_new0 (QEMUVFIOState,1 ); - r =qemu_vfio_init_pci (s, device, irq_type,1 , errp); + r =qemu_vfio_init_pci (s, device, irq_type, irq_count, errp);if (r) {g_free (s);return NULL ;