msix: fixes for 1.0
cover msix: fixes for 1.0
This fixes bugs dealing with msi-x mask bits pointed out by Jan Kiszka.
Jan Kiszka (1):
msix: Prevent bogus mask updates on MMIO accesses
Michael S. Tsirkin (2):
msix: track function masked in pci device state
msix: avoid mask updates if mask is unchanged
hw/msix.c | 48 ++++++++++++++++++++++++++++++++++++------------ hw/pci.h | 2 ++ 2 files changed, 38 insertions(+), 12 deletions(-)
Signature · 1 line
1.7.5.53.gc233e
PATCH 1/3 msix: track function masked in pci device state
Only go over the table when function is masked. This is not really important for qemu.git but helps fix a bug in qemu-kvm.git.
--- hw/msix.c | 21 ++++++++++++++------- hw/pci.h | 2 ++ 2 files changed, 16 insertions(+), 7 deletions(-)
hw/msix.c +14 -7
@@ -79,6 +79,7 @@ static int msix_add_config(struct PCIDevice *pdev, unsigned short nentries,/* Make flags bit writable. */ pdev->wmask[config_offset +MSIX_CONTROL_OFFSET ] |=MSIX_ENABLE_MASK |MSIX_MASKALL_MASK ; + pdev->msix_function_masked = true;return 0 ; }
@@ -117,16 +118,11 @@ static void msix_clr_pending(PCIDevice *dev, int vector) *msix_pending_byte (dev, vector) &= ~msix_pending_mask (vector); } -static int msix_function_masked (PCIDevice *dev) -{ -return dev->config[dev->msix_cap +MSIX_CONTROL_OFFSET ] &MSIX_MASKALL_MASK ; -} -static int msix_is_masked (PCIDevice *dev,int vector) {unsigned offset = vector *PCI_MSIX_ENTRY_SIZE +PCI_MSIX_ENTRY_VECTOR_CTRL ; -return msix_function_masked (dev) || +return dev->msix_function_masked || dev->msix_table_page[offset] &PCI_MSIX_ENTRY_CTRL_MASKBIT ; }
@@ -138,24 +134,34 @@ static void msix_handle_mask_update(PCIDevice *dev, int vector) } } +static void msix_update_function_masked (PCIDevice *dev) +{ + dev->msix_function_masked = !msix_enabled (dev) || + (dev->config[dev->msix_cap +MSIX_CONTROL_OFFSET ] &MSIX_MASKALL_MASK ); +} +/* Handle MSI-X capability config write. */ void msix_write_config (PCIDevice *dev,uint32_t addr,uint32_t val,int len) {unsigned enable_pos = dev->msix_cap +MSIX_CONTROL_OFFSET ;int vector; +bool was_masked;if (!range_covers_byte (addr, len, enable_pos)) {return ; } + was_masked = dev->msix_function_masked; +msix_update_function_masked (dev); +if (!msix_enabled (dev)) {return ; }pci_device_deassert_intx (dev); -if (msix_function_masked (dev)) { +if (dev->msix_function_masked == was_masked) {return ; }
@@ -300,6 +306,7 @@ void msix_load(PCIDevice *dev, QEMUFile *f)msix_free_irq_entries (dev);qemu_get_buffer (f, dev->msix_table_page, n *PCI_MSIX_ENTRY_SIZE );qemu_get_buffer (f, dev->msix_table_page +MSIX_PAGE_PENDING , (n +7 ) /8 ); +msix_update_function_masked (dev); }/* Does device support MSI-X? */
hw/pci.h +2 -0
@@ -178,6 +178,8 @@ struct PCIDevice {unsigned *msix_entry_used;/* Region including the MSI-X table */ uint32_t msix_bar_size; +/* MSIX function mask set or MSIX disabled */ +bool msix_function_masked;/* Version id needed for VMState */ int32_t version_id;
Signature · 1 line
1.7.5.53.gc233e
Re: [Qemu-devel] [PATCH for v1.0 1/3] msix: track function masked in pci device state
Based on a git bisect, this patch breaks msi-x interrupt delivery in the ivshmem device.
Quoted 16 lines · Michael S. Tsirkin
Only go over the table when function is masked. This is not really important for qemu.git but helps fix a bug in qemu-kvm.git.
Signed-off-by Michael S. Tsirkin <[email protected]>---
hw/msix.c | 21 ++++++++++++++-------
hw/pci.h | 2 ++2 files changed, 16 insertions(+), 7 deletions(-)hw/msix.c +0
-0@@ -79,6 +79,7 @@ static int msix_add_config(struct PCIDevice *pdev, unsigned short nentries, /* Make flags bit writable. */
Quoted 4 lines
pdev->wmask[config_offset +
MSIX_CONTROL_OFFSET] |=MSIX_ENABLE_MASK|MSIX_MASKALL_MASK;
+ pdev->msix_function_masked = true;
return 0;
iiuc, this masks the msix by default.
Quoted 45 lines
}
hw/msix.c +0
-5@@ -117,16 +118,11 @@ static void msix_clr_pending(PCIDevice *dev, int vector) *msix_pending_byte(dev, vector) &= ~msix_pending_mask(vector); } -static int msix_function_masked (PCIDevice *dev) -{ - return dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] & MSIX_MASKALL_MASK; -} - static int msix_is_masked(PCIDevice *dev, int vector) {unsigned offset =
vector *PCI_MSIX_ENTRY_SIZE+PCI_MSIX_ENTRY_VECTOR_CTRL;- return msix_function_masked(dev) || + return dev->msix_function_masked ||dev->msix_table_page[offset] &
PCI_MSIX_ENTRY_CTRL_MASKBIT; }hw/msix.c +10
-0@@ -138,24 +134,34 @@ static void msix_handle_mask_update(PCIDevice *dev, int vector) } } +static void msix_update_function_masked (PCIDevice *dev) +{ + dev->msix_function_masked = !msix_enabled(dev) || + (dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] & MSIX_MASKALL_MASK); +} + /* Handle MSI-X capability config write. */ void msix_write_config(PCIDevice *dev, uint32_t addr, uint32_t val, int len) { unsigned enable_pos = dev->msix_cap + MSIX_CONTROL_OFFSET; int vector; + bool was_masked; if (!range_covers_byte(addr, len, enable_pos)) { return; } + was_masked = dev->msix_function_masked; + msix_update_function_masked(dev); +
hw/msix.c static void msix_handle_mask_update(PCIDevice *dev, int vector) · line 166
Quoted 10 lines
hw/msix.c +1
-1if (!msix_enabled(dev)) { return; } pci_device_deassert_intx(dev); - if (msix_function_masked(dev)) { + if (dev->msix_function_masked == was_masked) { return; }
So I believe my bug is due to the fact the new logic included in this patch requires msix_write_config() to be called to unmask the vectors.
Virtio-pci calls msix_write_config(), but ivshmem does not (nor does
PCIe so I'm not sure if it's also affected).
I haven't been able to fix the bug yet, but I wanted to make sure I was looking in the correct place. Any help of further explanation of this patch would be greatly appreciated.
Sincerely,
Cam
Quoted 26 lines
hw/msix.c +1
-0@@ -300,6 +306,7 @@ void msix_load(PCIDevice *dev, QEMUFile *f) msix_free_irq_entries(dev); qemu_get_buffer(f, dev->msix_table_page, n * PCI_MSIX_ENTRY_SIZE); qemu_get_buffer(f, dev->msix_table_page + MSIX_PAGE_PENDING, (n + 7) / 8); + msix_update_function_masked(dev); } /* Does device support MSI-X? */hw/pci.h +2
-1@@ -178,6 +178,8 @@ struct PCIDevice { unsigned *msix_entry_used; /* Region including the MSI-X table */ uint32_t msix_bar_size; + /* MSIX function mask set or MSIX disabled */ + bool msix_function_masked; /* Version id needed for VMState */ int32_t version_id; --1.7.5.53.gc233e
Re: [Qemu-devel] [PATCH for v1.0 1/3] msix: track function masked in pci device state
Quoted 4 lines · Cam Macdonell
So I believe my bug is due to the fact the new logic included in this patch requires
msix_write_config()to be called to unmask the vectors.
Virtio-pci callsmsix_write_config(), but ivshmem does not (nor does
PCIe so I'm not sure if it's also affected).
msi[x]_write_config should not have to be called from device models but it must be right now. I proposed a patch a while ago to improve this. Same for msix_reset.
Jan
Re: [Qemu-devel] [PATCH for v1.0 1/3] msix: track function masked in pci device state
In reply to Cam MacdonellQuoted 14 lines · Cam Macdonell
Based on a git bisect, this patch breaks msi-x interrupt delivery in the ivshmem device.
Quoted 10 lines · Michael S. Tsirkin
Only go over the table when function is masked. This is not really important for qemu.git but helps fix a bug in qemu-kvm.git.
Signed-off-by Michael S. Tsirkin <[email protected]>---
hw/msix.c | 21 ++++++++++++++-------
hw/pci.h | 2 ++2 files changed, 16 insertions(+), 7 deletions(-)
Quoted 12 lines
Quoted 10 lines
hw/msix.c +0
-0@@ -79,6 +79,7 @@ static int msix_add_config(struct PCIDevice *pdev, unsigned short nentries, /* Make flags bit writable. */pdev->wmask[config_offset +
MSIX_CONTROL_OFFSET] |=MSIX_ENABLE_MASK|MSIX_MASKALL_MASK;
+ pdev->msix_function_masked = true;
return 0;iiuc, this masks the msix by default.
Yes, because msi-x is disabled by default, that's
in the pci spec.
Quoted 51 lines
Quoted 51 lines
}
hw/msix.c +0
-5@@ -117,16 +118,11 @@ static void msix_clr_pending(PCIDevice *dev, int vector) *msix_pending_byte(dev, vector) &= ~msix_pending_mask(vector); } -static int msix_function_masked (PCIDevice *dev) -{ - return dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] & MSIX_MASKALL_MASK; -} - static int msix_is_masked(PCIDevice *dev, int vector) {unsigned offset =
vector *PCI_MSIX_ENTRY_SIZE+PCI_MSIX_ENTRY_VECTOR_CTRL;- return msix_function_masked(dev) || + return dev->msix_function_masked ||dev->msix_table_page[offset] &
PCI_MSIX_ENTRY_CTRL_MASKBIT; }hw/msix.c +10
-0@@ -138,24 +134,34 @@ static void msix_handle_mask_update(PCIDevice *dev, int vector) } } +static void msix_update_function_masked (PCIDevice *dev) +{ + dev->msix_function_masked = !msix_enabled(dev) || + (dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] & MSIX_MASKALL_MASK); +} + /* Handle MSI-X capability config write. */ void msix_write_config(PCIDevice *dev, uint32_t addr, uint32_t val, int len) { unsigned enable_pos = dev->msix_cap + MSIX_CONTROL_OFFSET; int vector; + bool was_masked; if (!range_covers_byte(addr, len, enable_pos)) { return; } + was_masked = dev->msix_function_masked; + msix_update_function_masked(dev); + if (!msix_enabled(dev)) { return; } pci_device_deassert_intx(dev);
Quoted 7 lines
hw/msix.c static void msix_handle_mask_update(PCIDevice *dev, int vector) · line 166
Quoted 4 lines
hw/msix.c +1
-1- if (msix_function_masked(dev)) { + if (dev->msix_function_masked == was_masked) { return; }So I believe my bug is due to the fact the new logic included in this patch requires
msix_write_config()to be called to unmask the vectors.
Not exactly, to enable msi-x really.
Quoted 2 lines
Virtio-pci calls
msix_write_config(), but ivshmem does not (nor does
PCIe so I'm not sure if it's also affected).
At this point PCIe is a stub.
Quoted 6 lines
I haven't been able to fix the bug yet, but I wanted to make sure I was looking in the correct place. Any help of further explanation of this patch would be greatly appreciated.
Sincerely,
Cam
So I think you just need to call msix_write_config, otherwise msix is not getting enabled.
BTW looking at the ivshmem code, this bit looks wrong:
pci_conf[PCI_COMMAND] = PCI_COMMAND_IO | PCI_COMMAND_MEMORY;
I think the spec says IO/MEMORY must be disabled at init time since BARs are not yet set to anything reasonable.
Quoted 26 lines
Quoted 26 lines
hw/msix.c +1
-0@@ -300,6 +306,7 @@ void msix_load(PCIDevice *dev, QEMUFile *f) msix_free_irq_entries(dev); qemu_get_buffer(f, dev->msix_table_page, n * PCI_MSIX_ENTRY_SIZE); qemu_get_buffer(f, dev->msix_table_page + MSIX_PAGE_PENDING, (n + 7) / 8); + msix_update_function_masked(dev); } /* Does device support MSI-X? */hw/pci.h +2
-1@@ -178,6 +178,8 @@ struct PCIDevice { unsigned *msix_entry_used; /* Region including the MSI-X table */ uint32_t msix_bar_size; + /* MSIX function mask set or MSIX disabled */ + bool msix_function_masked; /* Version id needed for VMState */ int32_t version_id; --1.7.5.53.gc233e
Re: [Qemu-devel] [PATCH for v1.0 1/3] msix: track function masked in pci device state
In reply to Cam MacdonellQuoted 2 lines · Cam Macdonell
Based on a git bisect, this patch breaks msi-x interrupt delivery in the ivshmem device.
I think the following should fix it. Compiled-only - could you pls check? If yes let's apply to the stable branch.
-->
ivshmem: add missing msix calls
ivshmem used msix but didn't call it on either reset or config write paths. This used to partically work since guests don't use all of msi-x configuration fields,
and reset is rarely used, but the patch 'msix: track function masked in pci device state' broke that. Fix by adding appropriate calls.
--
hw/ivshmem.c +10 -0
@@ -505,6 +505,7 @@ static void ivshmem_reset(DeviceState *d)IVShmemState *s =DO_UPCAST (IVShmemState, dev.qdev, d); s->intrstatus =0 ; +msix_reset (&s->dev);return ; }
@@ -610,6 +611,13 @@ static int ivshmem_load(QEMUFile* f, void *opaque, int version_id)return 0 ; } +static void ivshmem_write_config (PCIDevice *pci_dev,uint32_t address, +uint32_t val,int len) +{ +pci_default_write_config (pci_dev, address, val, len); +msix_write_config (pci_dev, address, val, len); +} +static int pci_ivshmem_init (PCIDevice *dev) {IVShmemState *s =DO_UPCAST (IVShmemState, dev, dev);
@@ -734,6 +742,8 @@ static int pci_ivshmem_init(PCIDevice *dev) } + s->dev.config_write = ivshmem_write_config; +return 0 ; }
Re: [Qemu-devel] [PATCH for v1.0 1/3] msix: track function masked in pci device state
Quoted 50 lines · Michael S. Tsirkin
Quoted 2 lines · Cam Macdonell
Based on a git bisect, this patch breaks msi-x interrupt delivery in the ivshmem device.
I think the following should fix it. Compiled-only - could you pls check? If yes let's apply to the stable branch.
-->
ivshmem: add missing msix calls
ivshmem used msix but didn't call it on either reset or config write paths. This used to partically work since guests don't use all of msi-x configuration fields,
and reset is rarely used, but the patch 'msix: track function masked in pci device state' broke that. Fix by adding appropriate calls.Signed-off-by Michael S. Tsirkin <[email protected]>--
hw/ivshmem.c +8
-0@@ -505,6 +505,7 @@ static void ivshmem_reset(DeviceState *d)IVShmemState *s =DO_UPCAST (IVShmemState, dev.qdev, d); s->intrstatus =0 ; +msix_reset (&s->dev);return ; }@@ -610,6 +611,13 @@ static int ivshmem_load(QEMUFile* f, void *opaque, int version_id)return 0 ; } +static void ivshmem_write_config (PCIDevice *pci_dev,uint32_t address, +uint32_t val,int len) +{ +pci_default_write_config (pci_dev, address, val, len); +msix_write_config (pci_dev, address, val, len); +} +static int pci_ivshmem_init (PCIDevice *dev) {IVShmemState *s =DO_UPCAST (IVShmemState, dev, dev);
hw/ivshmem.c static int pci_ivshmem_init(PCIDevice *dev) · line 749
Quoted 8 lines
hw/ivshmem.c +2
-0@@ -734,6 +742,8 @@ static int pci_ivshmem_init(PCIDevice *dev) } + s->dev.config_write = ivshmem_write_config; +return 0 ; }
But please fix this for real and merge [1]&[2] (with depending patches) into master. The above is just boilerplate code from device POV.
Jan
[1] http://thread.gmane.org/gmane.comp.emulators.kvm.devel/80240
[2] http://thread.gmane.org/gmane.comp.emulators.kvm.devel/80244
Re: [Qemu-devel] [PATCH for v1.0 1/3] msix: track function masked in pci device state
Quoted 63 lines · Jan Kiszka
Quoted 50 lines · Michael S. Tsirkin
Quoted 2 lines · Cam Macdonell
Based on a git bisect, this patch breaks msi-x interrupt delivery in the ivshmem device.
I think the following should fix it. Compiled-only - could you pls check? If yes let's apply to the stable branch.
-->
ivshmem: add missing msix calls
ivshmem used msix but didn't call it on either reset or config write paths. This used to partically work since guests don't use all of msi-x configuration fields,
and reset is rarely used, but the patch 'msix: track function masked in pci device state' broke that. Fix by adding appropriate calls.Signed-off-by Michael S. Tsirkin <[email protected]>--
hw/ivshmem.c +8
-0@@ -505,6 +505,7 @@ static void ivshmem_reset(DeviceState *d)IVShmemState *s =DO_UPCAST (IVShmemState, dev.qdev, d); s->intrstatus =0 ; +msix_reset (&s->dev);return ; }@@ -610,6 +611,13 @@ static int ivshmem_load(QEMUFile* f, void *opaque, int version_id)return 0 ; } +static void ivshmem_write_config (PCIDevice *pci_dev,uint32_t address, +uint32_t val,int len) +{ +pci_default_write_config (pci_dev, address, val, len); +msix_write_config (pci_dev, address, val, len); +} +static int pci_ivshmem_init (PCIDevice *dev) {IVShmemState *s =DO_UPCAST (IVShmemState, dev, dev);hw/ivshmem.c static int pci_ivshmem_init(PCIDevice *dev) · line 749
Quoted 8 lines
hw/ivshmem.c +2
-0@@ -734,6 +742,8 @@ static int pci_ivshmem_init(PCIDevice *dev) } + s->dev.config_write = ivshmem_write_config; +return 0 ; }But please fix this for real and merge [1]&[2] (with depending patches) into master. The above is just boilerplate code from device POV.
Quoted 5 lines
Jan
[1] http://thread.gmane.org/gmane.comp.emulators.kvm.devel/80240
[2] http://thread.gmane.org/gmane.comp.emulators.kvm.devel/80244
Yes, I agree we should make it easier for devices. What annoyed me was the need to put msix in save/load.
And that is because of the need to do this in a specific order. I hope to switch to an unordered format and then this will become straight-forward.
Signature · 1 line
MST
Re: [Qemu-devel] [PATCH for v1.0 1/3] msix: track function masked in pci device state
In reply to Michael S. TsirkinQuoted 6 lines · Michael S. Tsirkin
Quoted 2 lines · Cam Macdonell
Based on a git bisect, this patch breaks msi-x interrupt delivery in the ivshmem device.
I think the following should fix it. Compiled-only - could you pls check? If yes let's apply to the stable branch.
Thanks for the patch Michael.
It addresses the need for msix_write_config() to be called, but the addition of the msix_reset() is causing a reset of the vectors after they've been initialized in pci_ivshmem_init(). So, interrupts still aren't delivered with this patch applied as it is.
In particular, a reset occurs after pci_ivshmem_init runs, so the msix_entry_used array is reset to 0s, which causes the interrupt delivery to fail.
If I comment out the msix_reset(), then interrupts are delivered. Would the reset be caused by a bug in the guest driver? or do I need to reconfigure the msix after reset? I'm unclear as to the proper behaviour after a reset.
Thanks,
Cam
Quoted 51 lines
-->
ivshmem: add missing msix calls
ivshmem used msix but didn't call it on either reset or config write paths. This used to partically work since guests don't use all of msi-x configuration fields,
and reset is rarely used, but the patch 'msix: track function masked in pci device state' broke that. Fix by adding appropriate calls.Signed-off-by Michael S. Tsirkin <[email protected]>--
hw/ivshmem.c +10
-0@@ -505,6 +505,7 @@ static void ivshmem_reset(DeviceState *d) IVShmemState *s = DO_UPCAST(IVShmemState, dev.qdev, d); s->intrstatus = 0; + msix_reset(&s->dev); return; }@@ -610,6 +611,13 @@ static int ivshmem_load(QEMUFile* f, void *opaque, int version_id) return 0; } +static void ivshmem_write_config (PCIDevice *pci_dev,uint32_t address, + uint32_t val, int len) +{ + pci_default_write_config(pci_dev, address, val, len); + msix_write_config(pci_dev, address, val, len); +} + static int pci_ivshmem_init(PCIDevice *dev) { IVShmemState *s = DO_UPCAST(IVShmemState, dev, dev);@@ -734,6 +742,8 @@ static int pci_ivshmem_init(PCIDevice *dev) } + s->dev.config_write = ivshmem_write_config; + return 0; }
Re: [Qemu-devel] [PATCH for v1.0 1/3] msix: track function masked in pci device state
Quoted 19 lines · Cam Macdonell
Quoted 6 lines · Michael S. Tsirkin
Quoted 2 lines · Cam Macdonell
Based on a git bisect, this patch breaks msi-x interrupt delivery in the ivshmem device.
I think the following should fix it. Compiled-only - could you pls check? If yes let's apply to the stable branch.
Thanks for the patch Michael.
It addresses the need for
msix_write_config()to be called, but the addition of themsix_reset()is causing a reset of the vectors after they've been initialized inpci_ivshmem_init(). So, interrupts still aren't delivered with this patch applied as it is.In particular, a reset occurs after
pci_ivshmem_initruns, so themsix_entry_usedarray is reset to 0s, which causes the interrupt delivery to fail.
Quoted 8 lines
If I comment out the
msix_reset(), then interrupts are delivered. Would the reset be caused by a bug in the guest driver? or do I need to reconfigure the msix after reset? I'm unclear as to the proper behaviour after a reset.Thanks,
Cam
I didn't anticipate this, virtio only configures vectors when msi-x is enabled. For 1.0 it's safest to do the same and just re-enable after reset.
So we'll end up with something like the following - does it help?
Side note: exit(1) is not the best way to handle
errors in the init function. I think you should add error_report, then goto err on failures to init, cleanup what you have set up meanwhile and return an error code.
hw/ivshmem.c +31 -11
@@ -500,11 +500,29 @@ static void ivshmem_read(void *opaque, const uint8_t * buf, int flags)return ; } +/* Select the MSI-X vectors used by device. +* ivshmem maps events to vectors statically, so +* we just enable all vectors on init and after reset. */ +static void ivshmem_use_msix (IVShmemState * s) +{ +int i; + +if (!msix_present (&s->dev)) { +return ; + } + +for (i =0 ; i < s->vectors; i++) { +msix_vector_use (&s->dev, i); + } +} +static void ivshmem_reset (DeviceState *d) {IVShmemState *s =DO_UPCAST (IVShmemState, dev.qdev, d); s->intrstatus =0 ; +msix_reset (&s->dev); +ivshmem_use_msix (s);return ; }
@@ -535,12 +553,8 @@ static uint64_t ivshmem_get_size(IVShmemState * s) {return value; } -static void ivshmem_setup_msi (IVShmemState * s) { - -int i; - -/* allocate the MSI-X vectors */ - +static void ivshmem_setup_msi (IVShmemState * s) +{memory_region_init (&s->msix_bar,"ivshmem-msix" ,4096 );if (!msix_init (&s->dev, s->vectors, &s->msix_bar,1 ,0 )) { pci_register_bar(&s->dev, 1, PCI_BASE_ADDRESS_SPACE_MEMORY,
@@ -551,13 +565,10 @@ static void ivshmem_setup_msi(IVShmemState * s) {exit (1 ); } -/* 'activate' the vectors */ -for (i =0 ; i < s->vectors; i++) { -msix_vector_use (&s->dev, i); - } -/* allocate Qemu char devices for receiving interrupts */ s->eventfd_table =g_malloc0 (s->vectors *sizeof (EventfdEntry)); + +ivshmem_use_msix (s); }static void ivshmem_save (QEMUFile * f,void *opaque)
@@ -610,6 +621,13 @@ static int ivshmem_load(QEMUFile* f, void *opaque, int version_id)return 0 ; } +static void ivshmem_write_config (PCIDevice *pci_dev,uint32_t address, +uint32_t val,int len) +{ +pci_default_write_config (pci_dev, address, val, len); +msix_write_config (pci_dev, address, val, len); +} +static int pci_ivshmem_init (PCIDevice *dev) {IVShmemState *s =DO_UPCAST (IVShmemState, dev, dev);
@@ -734,6 +752,8 @@ static int pci_ivshmem_init(PCIDevice *dev) } + s->dev.config_write = ivshmem_write_config; +return 0 ; }
Re: [Qemu-devel] [PATCH for v1.0 1/3] msix: track function masked in pci device state
Quoted 25 lines · Michael S. Tsirkin
Quoted 24 lines · Cam Macdonell
Quoted 6 lines · Michael S. Tsirkin
Quoted 2 lines · Cam Macdonell
Based on a git bisect, this patch breaks msi-x interrupt delivery in the ivshmem device.
I think the following should fix it. Compiled-only - could you pls check? If yes let's apply to the stable branch.
Thanks for the patch Michael.
It addresses the need for
msix_write_config()to be called, but the addition of themsix_reset()is causing a reset of the vectors after they've been initialized inpci_ivshmem_init(). So, interrupts still aren't delivered with this patch applied as it is.In particular, a reset occurs after
pci_ivshmem_initruns, so themsix_entry_usedarray is reset to 0s, which causes the interrupt delivery to fail.If I comment out the
msix_reset(), then interrupts are delivered. Would the reset be caused by a bug in the guest driver? or do I need to reconfigure the msix after reset? I'm unclear as to the proper behaviour after a reset.
Quoted 8 lines
Quoted 3 lines
Thanks,
CamI didn't anticipate this, virtio only configures vectors when msi-x is enabled. For 1.0 it's safest to do the same and just re-enable after reset.
So we'll end up with something like the following - does it help?
Yes, this fixes interrupt delivery.
Quoted 5 lines
Side note: exit(1) is not the best way to handle
errors in the init function. I think you should adderror_report, then goto err on failures to init, cleanup what you have set up meanwhile and return an error code.
Thanks, I'll fix that.
Quoted 90 lines
hw/ivshmem.c +31
-11@@ -500,11 +500,29 @@ static void ivshmem_read(void *opaque, const uint8_t * buf, int flags) return; } +/* Select the MSI-X vectors used by device. +* ivshmem maps events to vectors statically, so +* we just enable all vectors on init and after reset. */ +static void ivshmem_use_msix (IVShmemState * s) +{ + int i; + + if (!msix_present(&s->dev)) { + return; + } + + for (i = 0; i < s->vectors; i++) { + msix_vector_use(&s->dev, i); + } +} + static void ivshmem_reset(DeviceState *d) { IVShmemState *s = DO_UPCAST(IVShmemState, dev.qdev, d); s->intrstatus = 0; + msix_reset(&s->dev); + ivshmem_use_msix(s); return; }@@ -535,12 +553,8 @@ static uint64_t ivshmem_get_size(IVShmemState * s) { return value; } -static void ivshmem_setup_msi (IVShmemState * s) { - - int i; - - /* allocate the MSI-X vectors */ - +static void ivshmem_setup_msi (IVShmemState * s) +{ memory_region_init(&s->msix_bar, "ivshmem-msix", 4096); if (!msix_init(&s->dev, s->vectors, &s->msix_bar, 1, 0)) { pci_register_bar(&s->dev, 1, PCI_BASE_ADDRESS_SPACE_MEMORY,@@ -551,13 +565,10 @@ static void ivshmem_setup_msi(IVShmemState * s) { exit(1); } - /* 'activate' the vectors */ - for (i = 0; i < s->vectors; i++) { - msix_vector_use(&s->dev, i); - } - /* allocate Qemu char devices for receiving interrupts */ s->eventfd_table = g_malloc0(s->vectors * sizeof(EventfdEntry)); + + ivshmem_use_msix(s); } static void ivshmem_save(QEMUFile* f, void *opaque)@@ -610,6 +621,13 @@ static int ivshmem_load(QEMUFile* f, void *opaque, int version_id) return 0; } +static void ivshmem_write_config (PCIDevice *pci_dev,uint32_t address, + uint32_t val, int len) +{ + pci_default_write_config(pci_dev, address, val, len); + msix_write_config(pci_dev, address, val, len); +} + static int pci_ivshmem_init(PCIDevice *dev) { IVShmemState *s = DO_UPCAST(IVShmemState, dev, dev);@@ -734,6 +752,8 @@ static int pci_ivshmem_init(PCIDevice *dev) } + s->dev.config_write = ivshmem_write_config; + return 0; }
PATCH 1/1 ivshmem: add missing msix calls
In reply to Cam Macdonellivshmem used msix but didn't call it on either reset or config write paths. This used to partically work since guests don't use all of msi-x configuration fields,
and reset is rarely used, but the patch 'msix: track function masked in pci device state' broke that. Fix by adding appropriate calls.
---
Please apply the following to both master
and 1.0 stable branch. Thanks!
hw/ivshmem.c +31 -11
@@ -500,11 +500,29 @@ static void ivshmem_read(void *opaque, const uint8_t * buf, int flags)return ; } +/* Select the MSI-X vectors used by device. +* ivshmem maps events to vectors statically, so +* we just enable all vectors on init and after reset. */ +static void ivshmem_use_msix (IVShmemState * s) +{ +int i; + +if (!msix_present (&s->dev)) { +return ; + } + +for (i =0 ; i < s->vectors; i++) { +msix_vector_use (&s->dev, i); + } +} +static void ivshmem_reset (DeviceState *d) {IVShmemState *s =DO_UPCAST (IVShmemState, dev.qdev, d); s->intrstatus =0 ; +msix_reset (&s->dev); +ivshmem_use_msix (s);return ; }
@@ -535,12 +553,8 @@ static uint64_t ivshmem_get_size(IVShmemState * s) {return value; } -static void ivshmem_setup_msi (IVShmemState * s) { - -int i; - -/* allocate the MSI-X vectors */ - +static void ivshmem_setup_msi (IVShmemState * s) +{memory_region_init (&s->msix_bar,"ivshmem-msix" ,4096 );if (!msix_init (&s->dev, s->vectors, &s->msix_bar,1 ,0 )) { pci_register_bar(&s->dev, 1, PCI_BASE_ADDRESS_SPACE_MEMORY,
@@ -551,13 +565,10 @@ static void ivshmem_setup_msi(IVShmemState * s) {exit (1 ); } -/* 'activate' the vectors */ -for (i =0 ; i < s->vectors; i++) { -msix_vector_use (&s->dev, i); - } -/* allocate Qemu char devices for receiving interrupts */ s->eventfd_table =g_malloc0 (s->vectors *sizeof (EventfdEntry)); + +ivshmem_use_msix (s); }static void ivshmem_save (QEMUFile * f,void *opaque)
@@ -610,6 +621,13 @@ static int ivshmem_load(QEMUFile* f, void *opaque, int version_id)return 0 ; } +static void ivshmem_write_config (PCIDevice *pci_dev,uint32_t address, +uint32_t val,int len) +{ +pci_default_write_config (pci_dev, address, val, len); +msix_write_config (pci_dev, address, val, len); +} +static int pci_ivshmem_init (PCIDevice *dev) {IVShmemState *s =DO_UPCAST (IVShmemState, dev, dev);
@@ -734,6 +752,8 @@ static int pci_ivshmem_init(PCIDevice *dev) } + s->dev.config_write = ivshmem_write_config; +return 0 ; }
Re: [Qemu-devel] [PATCH master/v1.0.x] ivshmem: add missing msix calls
Hello,
Can this patch be merged, please?
Thanks,
Cam
Quoted 104 lines · Michael S. Tsirkin
ivshmem used msix but didn't call it on either reset or config write paths. This used to partically work since guests don't use all of msi-x configuration fields,
and reset is rarely used, but the patch 'msix: track function masked in pci device state' broke that. Fix by adding appropriate calls.Signed-off-by Michael S. Tsirkin <[email protected]>Reported-by Cam Macdonell <[email protected]>Tested-by Cam Macdonell <[email protected]>---
Please apply the following to both master
and 1.0 stable branch. Thanks!hw/ivshmem.c +31
-11@@ -500,11 +500,29 @@ static void ivshmem_read(void *opaque, const uint8_t * buf, int flags) return; } +/* Select the MSI-X vectors used by device. +* ivshmem maps events to vectors statically, so +* we just enable all vectors on init and after reset. */ +static void ivshmem_use_msix (IVShmemState * s) +{ + int i; + + if (!msix_present(&s->dev)) { + return; + } + + for (i = 0; i < s->vectors; i++) { + msix_vector_use(&s->dev, i); + } +} + static void ivshmem_reset(DeviceState *d) { IVShmemState *s = DO_UPCAST(IVShmemState, dev.qdev, d); s->intrstatus = 0; + msix_reset(&s->dev); + ivshmem_use_msix(s); return; }@@ -535,12 +553,8 @@ static uint64_t ivshmem_get_size(IVShmemState * s) { return value; } -static void ivshmem_setup_msi (IVShmemState * s) { - - int i; - - /* allocate the MSI-X vectors */ - +static void ivshmem_setup_msi (IVShmemState * s) +{ memory_region_init(&s->msix_bar, "ivshmem-msix", 4096); if (!msix_init(&s->dev, s->vectors, &s->msix_bar, 1, 0)) { pci_register_bar(&s->dev, 1, PCI_BASE_ADDRESS_SPACE_MEMORY,@@ -551,13 +565,10 @@ static void ivshmem_setup_msi(IVShmemState * s) { exit(1); } - /* 'activate' the vectors */ - for (i = 0; i < s->vectors; i++) { - msix_vector_use(&s->dev, i); - } - /* allocate Qemu char devices for receiving interrupts */ s->eventfd_table = g_malloc0(s->vectors * sizeof(EventfdEntry)); + + ivshmem_use_msix(s); } static void ivshmem_save(QEMUFile* f, void *opaque)@@ -610,6 +621,13 @@ static int ivshmem_load(QEMUFile* f, void *opaque, int version_id) return 0; } +static void ivshmem_write_config (PCIDevice *pci_dev,uint32_t address, + uint32_t val, int len) +{ + pci_default_write_config(pci_dev, address, val, len); + msix_write_config(pci_dev, address, val, len); +} + static int pci_ivshmem_init(PCIDevice *dev) { IVShmemState *s = DO_UPCAST(IVShmemState, dev, dev);@@ -734,6 +752,8 @@ static int pci_ivshmem_init(PCIDevice *dev) } + s->dev.config_write = ivshmem_write_config; + return 0; }
Re: [Qemu-devel] [PATCH master/v1.0.x] ivshmem: add missing msix calls
Am 13.01.2012 23:43, schrieb Cam Macdonell:
Quoted 1 line
Can this patch be merged, please?
You need to cc qemu-stable if you want it backported to v1.0.x once applied.
Andreas
Quoted 104 lines · Michael S. Tsirkin
Quoted 103 lines
ivshmem used msix but didn't call it on either reset or config write paths. This used to partically work since guests don't use all of msi-x configuration fields,
and reset is rarely used, but the patch 'msix: track function masked in pci device state' broke that. Fix by adding appropriate calls.Signed-off-by Michael S. Tsirkin <[email protected]>Reported-by Cam Macdonell <[email protected]>Tested-by Cam Macdonell <[email protected]>---
Please apply the following to both master
and 1.0 stable branch. Thanks!hw/ivshmem.c +31
-11@@ -500,11 +500,29 @@ static void ivshmem_read(void *opaque, const uint8_t * buf, int flags)return ; } +/* Select the MSI-X vectors used by device. +* ivshmem maps events to vectors statically, so +* we just enable all vectors on init and after reset. */ +static void ivshmem_use_msix (IVShmemState * s) +{ +int i; + +if (!msix_present (&s->dev)) { +return ; + } + +for (i =0 ; i < s->vectors; i++) { +msix_vector_use (&s->dev, i); + } +} +static void ivshmem_reset (DeviceState *d) {IVShmemState *s =DO_UPCAST (IVShmemState, dev.qdev, d); s->intrstatus =0 ; +msix_reset (&s->dev); +ivshmem_use_msix (s);return ; }@@ -535,12 +553,8 @@ static uint64_t ivshmem_get_size(IVShmemState * s) {return value; } -static void ivshmem_setup_msi (IVShmemState * s) { - -int i; - -/* allocate the MSI-X vectors */ - +static void ivshmem_setup_msi (IVShmemState * s) +{memory_region_init (&s->msix_bar,"ivshmem-msix" ,4096 );if (!msix_init (&s->dev, s->vectors, &s->msix_bar,1 ,0 )) { pci_register_bar(&s->dev, 1, PCI_BASE_ADDRESS_SPACE_MEMORY,@@ -551,13 +565,10 @@ static void ivshmem_setup_msi(IVShmemState * s) {exit (1 ); } -/* 'activate' the vectors */ -for (i =0 ; i < s->vectors; i++) { -msix_vector_use (&s->dev, i); - } -/* allocate Qemu char devices for receiving interrupts */ s->eventfd_table =g_malloc0 (s->vectors *sizeof (EventfdEntry)); + +ivshmem_use_msix (s); }static void ivshmem_save (QEMUFile * f,void *opaque)@@ -610,6 +621,13 @@ static int ivshmem_load(QEMUFile* f, void *opaque, int version_id)return 0 ; } +static void ivshmem_write_config (PCIDevice *pci_dev,uint32_t address, +uint32_t val,int len) +{ +pci_default_write_config (pci_dev, address, val, len); +msix_write_config (pci_dev, address, val, len); +} +static int pci_ivshmem_init (PCIDevice *dev) {IVShmemState *s =DO_UPCAST (IVShmemState, dev, dev);@@ -734,6 +752,8 @@ static int pci_ivshmem_init(PCIDevice *dev) } + s->dev.config_write = ivshmem_write_config; +return 0 ; }
Signature · 2 lines
SUSE LINUX Products GmbH, Maxfeldstr. 5, 90409 Nürnberg, Germany GF: Jeff Hawn, Jennifer Guild, Felix Imendörffer; HRB 16746 AG Nürnberg
PATCH 2/3 msix: Prevent bogus mask updates on MMIO accesses
In reply to Michael S. TsirkinQuoted 1 line
From: Jan Kiszka <[email protected]>
Only accesses to the MSI-X table must trigger a call to msix_handle_mask_update, otherwise the vector value might be out of range.
--- hw/msix.c | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-)
hw/msix.c +6 -0
@@ -176,6 +176,12 @@ static void msix_mmio_write(void *opaque, target_phys_addr_t addr,PCIDevice *dev = opaque;unsigned int offset = addr & (MSIX_PAGE_SIZE -1 ) & ~0x3 ;int vector = offset /PCI_MSIX_ENTRY_SIZE ; + +/* MSI-X page includes a read-only PBA and a writeable Vector Control. */ +if (vector >= dev->msix_entries_nr) { +return ; + } +pci_set_long (dev->msix_table_page + offset, val);msix_handle_mask_update (dev, vector); }
Signature · 1 line
1.7.5.53.gc233e
PATCH 3/3 msix: avoid mask updates if mask is unchanged
In reply to Michael S. TsirkinCheck pending bit only if vector mask status changed. This is not really important for qemu.git but helps fix a bug in qemu-kvm.git.
--- hw/msix.c | 29 ++++++++++++++++++++--------- 1 files changed, 20 insertions(+), 9 deletions(-)
hw/msix.c +20 -9
@@ -118,17 +118,25 @@ static void msix_clr_pending(PCIDevice *dev, int vector) *msix_pending_byte (dev, vector) &= ~msix_pending_mask (vector); } -static int msix_is_masked (PCIDevice *dev,int vector) +static bool msix_vector_masked (PCIDevice *dev,int vector,bool fmask) { -unsigned offset = - vector *PCI_MSIX_ENTRY_SIZE +PCI_MSIX_ENTRY_VECTOR_CTRL ; -return dev->msix_function_masked || - dev->msix_table_page[offset] &PCI_MSIX_ENTRY_CTRL_MASKBIT ; +unsigned offset = vector *PCI_MSIX_ENTRY_SIZE +PCI_MSIX_ENTRY_VECTOR_CTRL ; +return fmask || dev->msix_table_page[offset] &PCI_MSIX_ENTRY_CTRL_MASKBIT ; } -static void msix_handle_mask_update (PCIDevice *dev,int vector) +static bool msix_is_masked (PCIDevice *dev,int vector) { -if (!msix_is_masked (dev, vector) &&msix_is_pending (dev, vector)) { +return msix_vector_masked (dev, vector, dev->msix_function_masked); +} + +static void msix_handle_mask_update (PCIDevice *dev,int vector,bool was_masked) +{ +bool is_masked =msix_is_masked (dev, vector); +if (is_masked == was_masked) { +return ; + } + +if (!is_masked &&msix_is_pending (dev, vector)) {msix_clr_pending (dev, vector);msix_notify (dev, vector); }
@@ -166,7 +174,8 @@ void msix_write_config(PCIDevice *dev, uint32_t addr, }for (vector =0 ; vector < dev->msix_entries_nr; ++vector) { -msix_handle_mask_update (dev, vector); +msix_handle_mask_update (dev, vector, +msix_vector_masked (dev, vector, was_masked)); } }
@@ -176,14 +185,16 @@ static void msix_mmio_write(void *opaque, target_phys_addr_t addr,PCIDevice *dev = opaque;unsigned int offset = addr & (MSIX_PAGE_SIZE -1 ) & ~0x3 ;int vector = offset /PCI_MSIX_ENTRY_SIZE ; +bool was_masked;/* MSI-X page includes a read-only PBA and a writeable Vector Control. */ if (vector >= dev->msix_entries_nr) {return ; } + was_masked =msix_is_masked (dev, vector);pci_set_long (dev->msix_table_page + offset, val); -msix_handle_mask_update (dev, vector); +msix_handle_mask_update (dev, vector, was_masked); }static const MemoryRegionOps msix_mmio_ops = {
Signature · 1 line
1.7.5.53.gc233e
Anthony Liguori
AppliedQuoted 1 line · Michael S. Tsirkin
This fixes bugs dealing with msi-x mask bits pointed out by Jan Kiszka.
Applied. Thanks.
Regards,
Anthony Liguori
Quoted 12 lines
Jan Kiszka (1):
msix: Prevent bogus mask updates on MMIO accessesMichael S. Tsirkin (2):
msix: track function masked in pci device state
msix: avoid mask updates if mask is unchangedhw/msix.c | 48 ++++++++++++++++++++++++++++++++++++------------ hw/pci.h | 2 ++ 2 files changed, 38 insertions(+), 12 deletions(-)