Skip to contentSkip to conversation

msix: fixes for 1.0

PATCH Applied Michael S. Tsirkin 5 participants 17 messages Nov 21, 2011 – Jan 15, 2012

lore
Share & tools
lore
#0 lore raw

cover msix: fixes for 1.0

This fixes bugs dealing with msi-x mask bits pointed out by Jan Kiszka.

Jan Kiszka (1):
msix: Prevent bogus mask updates on MMIO accesses

Michael S. Tsirkin (2):
msix: track function masked in pci device state
msix: avoid mask updates if mask is unchanged

 hw/msix.c |   48 ++++++++++++++++++++++++++++++++++++------------
 hw/pci.h  |    2 ++
 2 files changed, 38 insertions(+), 12 deletions(-)
Signature · 1 line
1.7.5.53.gc233e
#1 lore raw

PATCH 1/3 msix: track function masked in pci device state

Only go over the table when function is masked. This is not really important for qemu.git but helps fix a bug in qemu-kvm.git.

---
 hw/msix.c |   21 ++++++++++++++-------
 hw/pci.h  |    2 ++
 2 files changed, 16 insertions(+), 7 deletions(-)
hw/msix.c +14 -7
index b15bafc..63b41b9 100644
@@ -79,6 +79,7 @@ static int msix_add_config(struct PCIDevice *pdev, unsigned short nentries,
     /* Make flags bit writable. */
     pdev->wmask[config_offset + MSIX_CONTROL_OFFSET] |= MSIX_ENABLE_MASK |
 	    MSIX_MASKALL_MASK;
+    pdev->msix_function_masked = true;
     return 0;
 }
 
@@ -117,16 +118,11 @@ static void msix_clr_pending(PCIDevice *dev, int vector)
     *msix_pending_byte(dev, vector) &= ~msix_pending_mask(vector);
 }
 
-static int msix_function_masked(PCIDevice *dev)
-{
-    return dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] & MSIX_MASKALL_MASK;
-}
-
 static int msix_is_masked(PCIDevice *dev, int vector)
 {
     unsigned offset =
         vector * PCI_MSIX_ENTRY_SIZE + PCI_MSIX_ENTRY_VECTOR_CTRL;
-    return msix_function_masked(dev) ||
+    return dev->msix_function_masked ||
 	   dev->msix_table_page[offset] & PCI_MSIX_ENTRY_CTRL_MASKBIT;
 }
 
@@ -138,24 +134,34 @@ static void msix_handle_mask_update(PCIDevice *dev, int vector)
     }
 }
 
+static void msix_update_function_masked(PCIDevice *dev)
+{
+    dev->msix_function_masked = !msix_enabled(dev) ||
+        (dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] & MSIX_MASKALL_MASK);
+}
+
 /* Handle MSI-X capability config write. */
 void msix_write_config(PCIDevice *dev, uint32_t addr,
                        uint32_t val, int len)
 {
     unsigned enable_pos = dev->msix_cap + MSIX_CONTROL_OFFSET;
     int vector;
+    bool was_masked;
 
     if (!range_covers_byte(addr, len, enable_pos)) {
         return;
     }
 
+    was_masked = dev->msix_function_masked;
+    msix_update_function_masked(dev);
+
     if (!msix_enabled(dev)) {
         return;
     }
 
     pci_device_deassert_intx(dev);
 
-    if (msix_function_masked(dev)) {
+    if (dev->msix_function_masked == was_masked) {
         return;
     }
 
@@ -300,6 +306,7 @@ void msix_load(PCIDevice *dev, QEMUFile *f)
     msix_free_irq_entries(dev);
     qemu_get_buffer(f, dev->msix_table_page, n * PCI_MSIX_ENTRY_SIZE);
     qemu_get_buffer(f, dev->msix_table_page + MSIX_PAGE_PENDING, (n + 7) / 8);
+    msix_update_function_masked(dev);
 }
 
 /* Does device support MSI-X? */
hw/pci.h +2 -0
index 4b2e785..625e717 100644
@@ -178,6 +178,8 @@ struct PCIDevice {
     unsigned *msix_entry_used;
     /* Region including the MSI-X table */
     uint32_t msix_bar_size;
+    /* MSIX function mask set or MSIX disabled */
+    bool msix_function_masked;
     /* Version id needed for VMState */
     int32_t version_id;
 
Signature · 1 line
1.7.5.53.gc233e
#2 lore raw

Re: [Qemu-devel] [PATCH for v1.0 1/3] msix: track function masked in pci device state

Based on a git bisect, this patch breaks msi-x interrupt delivery in the ivshmem device.

Quoted 16 lines · Michael S. Tsirkin

Only go over the table when function is masked. This is not really important for qemu.git but helps fix a bug in qemu-kvm.git.

---
 hw/msix.c |   21 ++++++++++++++-------
 hw/pci.h  |    2 ++

 2 files changed, 16 insertions(+), 7 deletions(-)
hw/msix.c +0 -0
index b15bafc..63b41b9 100644
@@ -79,6 +79,7 @@ static int msix_add_config(struct PCIDevice *pdev, unsigned short nentries,
     /* Make flags bit writable. */
Quoted 4 lines

    pdev->wmask[config_offset + MSIX_CONTROL_OFFSET] |= MSIX_ENABLE_MASK | MSIX_MASKALL_MASK;
+    pdev->msix_function_masked = true;
    return 0;

iiuc, this masks the msix by default.

Quoted 45 lines

 }

hw/msix.c +0 -5
@@ -117,16 +118,11 @@ static void msix_clr_pending(PCIDevice *dev, int vector)
     *msix_pending_byte(dev, vector) &= ~msix_pending_mask(vector);
  }
 
-static int msix_function_masked(PCIDevice *dev)
-{
-    return dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] & MSIX_MASKALL_MASK;
-}
-
  static int msix_is_masked(PCIDevice *dev, int vector)
  {

    unsigned offset =
        vector * PCI_MSIX_ENTRY_SIZE + PCI_MSIX_ENTRY_VECTOR_CTRL;

-    return msix_function_masked(dev) ||
+    return dev->msix_function_masked ||

          dev->msix_table_page[offset] & PCI_MSIX_ENTRY_CTRL_MASKBIT; }

hw/msix.c +10 -0
@@ -138,24 +134,34 @@ static void msix_handle_mask_update(PCIDevice *dev, int vector)
     }
  }
 
+static void msix_update_function_masked(PCIDevice *dev)
+{
+    dev->msix_function_masked = !msix_enabled(dev) ||
+        (dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] & MSIX_MASKALL_MASK);
+}
+
  /* Handle MSI-X capability config write. */
  void msix_write_config(PCIDevice *dev, uint32_t addr,
                        uint32_t val, int len)
  {
     unsigned enable_pos = dev->msix_cap + MSIX_CONTROL_OFFSET;
     int vector;
+    bool was_masked;
 
     if (!range_covers_byte(addr, len, enable_pos)) {
         return;
     }
 
+    was_masked = dev->msix_function_masked;
+    msix_update_function_masked(dev);
+

hw/msix.c static void msix_handle_mask_update(PCIDevice *dev, int vector) · line 166

Quoted 10 lines
hw/msix.c +1 -1
     if (!msix_enabled(dev)) {
         return;
     }
 
     pci_device_deassert_intx(dev);
 
-    if (msix_function_masked(dev)) {
+    if (dev->msix_function_masked == was_masked) {
         return;
     }

So I believe my bug is due to the fact the new logic included in this patch requires msix_write_config() to be called to unmask the vectors.
Virtio-pci calls msix_write_config(), but ivshmem does not (nor does
PCIe so I'm not sure if it's also affected).

I haven't been able to fix the bug yet, but I wanted to make sure I was looking in the correct place. Any help of further explanation of this patch would be greatly appreciated.

Sincerely,
Cam

Quoted 26 lines
hw/msix.c +1 -0
@@ -300,6 +306,7 @@ void msix_load(PCIDevice *dev, QEMUFile *f)
     msix_free_irq_entries(dev);
     qemu_get_buffer(f, dev->msix_table_page, n * PCI_MSIX_ENTRY_SIZE);
     qemu_get_buffer(f, dev->msix_table_page + MSIX_PAGE_PENDING, (n + 7) / 8);
+    msix_update_function_masked(dev);
  }
 
  /* Does device support MSI-X? */
hw/pci.h +2 -1
index 4b2e785..625e717 100644
@@ -178,6 +178,8 @@ struct PCIDevice {
     unsigned *msix_entry_used;
     /* Region including the MSI-X table */
     uint32_t msix_bar_size;
+    /* MSIX function mask set or MSIX disabled */
+    bool msix_function_masked;
     /* Version id needed for VMState */
     int32_t version_id;
 
--

1.7.5.53.gc233e

#3 lore raw

Re: [Qemu-devel] [PATCH for v1.0 1/3] msix: track function masked in pci device state

Quoted 4 lines · Cam Macdonell

So I believe my bug is due to the fact the new logic included in this patch requires msix_write_config() to be called to unmask the vectors.
Virtio-pci calls msix_write_config(), but ivshmem does not (nor does
PCIe so I'm not sure if it's also affected).

msi[x]_write_config should not have to be called from device models but it must be right now. I proposed a patch a while ago to improve this. Same for msix_reset.

Jan

#4 lore raw

Re: [Qemu-devel] [PATCH for v1.0 1/3] msix: track function masked in pci device state

In reply to Cam Macdonell
Quoted 14 lines · Cam Macdonell

Based on a git bisect, this patch breaks msi-x interrupt delivery in the ivshmem device.

Quoted 10 lines · Michael S. Tsirkin

Only go over the table when function is masked. This is not really important for qemu.git but helps fix a bug in qemu-kvm.git.

---
 hw/msix.c |   21 ++++++++++++++-------
 hw/pci.h  |    2 ++

 2 files changed, 16 insertions(+), 7 deletions(-)
Quoted 12 lines
Quoted 10 lines
hw/msix.c +0 -0
index b15bafc..63b41b9 100644
@@ -79,6 +79,7 @@ static int msix_add_config(struct PCIDevice *pdev, unsigned short nentries,
     /* Make flags bit writable. */

    pdev->wmask[config_offset + MSIX_CONTROL_OFFSET] |= MSIX_ENABLE_MASK | MSIX_MASKALL_MASK;
+    pdev->msix_function_masked = true;
    return 0;

iiuc, this masks the msix by default.

Yes, because msi-x is disabled by default, that's
in the pci spec.

Quoted 51 lines
Quoted 51 lines

 }

hw/msix.c +0 -5
@@ -117,16 +118,11 @@ static void msix_clr_pending(PCIDevice *dev, int vector)
     *msix_pending_byte(dev, vector) &= ~msix_pending_mask(vector);
  }
 
-static int msix_function_masked(PCIDevice *dev)
-{
-    return dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] & MSIX_MASKALL_MASK;
-}
-
  static int msix_is_masked(PCIDevice *dev, int vector)
  {

    unsigned offset =
        vector * PCI_MSIX_ENTRY_SIZE + PCI_MSIX_ENTRY_VECTOR_CTRL;

-    return msix_function_masked(dev) ||
+    return dev->msix_function_masked ||

          dev->msix_table_page[offset] & PCI_MSIX_ENTRY_CTRL_MASKBIT; }

hw/msix.c +10 -0
@@ -138,24 +134,34 @@ static void msix_handle_mask_update(PCIDevice *dev, int vector)
     }
  }
 
+static void msix_update_function_masked(PCIDevice *dev)
+{
+    dev->msix_function_masked = !msix_enabled(dev) ||
+        (dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] & MSIX_MASKALL_MASK);
+}
+
  /* Handle MSI-X capability config write. */
  void msix_write_config(PCIDevice *dev, uint32_t addr,
                        uint32_t val, int len)
  {
     unsigned enable_pos = dev->msix_cap + MSIX_CONTROL_OFFSET;
     int vector;
+    bool was_masked;
 
     if (!range_covers_byte(addr, len, enable_pos)) {
         return;
     }
 
+    was_masked = dev->msix_function_masked;
+    msix_update_function_masked(dev);
+
     if (!msix_enabled(dev)) {
         return;
     }
 
     pci_device_deassert_intx(dev);
 
Quoted 7 lines

hw/msix.c static void msix_handle_mask_update(PCIDevice *dev, int vector) · line 166

Quoted 4 lines
hw/msix.c +1 -1
-    if (msix_function_masked(dev)) {
+    if (dev->msix_function_masked == was_masked) {
         return;
     }

So I believe my bug is due to the fact the new logic included in this patch requires msix_write_config() to be called to unmask the vectors.

Not exactly, to enable msi-x really.

Quoted 2 lines

Virtio-pci calls msix_write_config(), but ivshmem does not (nor does
PCIe so I'm not sure if it's also affected).

At this point PCIe is a stub.

Quoted 6 lines

I haven't been able to fix the bug yet, but I wanted to make sure I was looking in the correct place. Any help of further explanation of this patch would be greatly appreciated.

Sincerely,
Cam

So I think you just need to call msix_write_config, otherwise msix is not getting enabled.

BTW looking at the ivshmem code, this bit looks wrong:

    pci_conf[PCI_COMMAND] = PCI_COMMAND_IO | PCI_COMMAND_MEMORY;

I think the spec says IO/MEMORY must be disabled at init time since BARs are not yet set to anything reasonable.

Quoted 26 lines
Quoted 26 lines
hw/msix.c +1 -0
@@ -300,6 +306,7 @@ void msix_load(PCIDevice *dev, QEMUFile *f)
     msix_free_irq_entries(dev);
     qemu_get_buffer(f, dev->msix_table_page, n * PCI_MSIX_ENTRY_SIZE);
     qemu_get_buffer(f, dev->msix_table_page + MSIX_PAGE_PENDING, (n + 7) / 8);
+    msix_update_function_masked(dev);
  }
 
  /* Does device support MSI-X? */
hw/pci.h +2 -1
index 4b2e785..625e717 100644
@@ -178,6 +178,8 @@ struct PCIDevice {
     unsigned *msix_entry_used;
     /* Region including the MSI-X table */
     uint32_t msix_bar_size;
+    /* MSIX function mask set or MSIX disabled */
+    bool msix_function_masked;
     /* Version id needed for VMState */
     int32_t version_id;
 
--

1.7.5.53.gc233e

#5 lore raw

Re: [Qemu-devel] [PATCH for v1.0 1/3] msix: track function masked in pci device state

In reply to Cam Macdonell
Quoted 2 lines · Cam Macdonell

Based on a git bisect, this patch breaks msi-x interrupt delivery in the ivshmem device.

I think the following should fix it. Compiled-only - could you pls check? If yes let's apply to the stable branch.

-->

ivshmem: add missing msix calls

ivshmem used msix but didn't call it on either reset or config write paths. This used to partically work since guests don't use all of msi-x configuration fields,
and reset is rarely used, but the patch 'msix: track function masked in pci device state' broke that. Fix by adding appropriate calls.

--

hw/ivshmem.c +10 -0
index 242fbea..3680c0f 100644
@@ -505,6 +505,7 @@ static void ivshmem_reset(DeviceState *d)
     IVShmemState *s = DO_UPCAST(IVShmemState, dev.qdev, d);
 
     s->intrstatus = 0;
+    msix_reset(&s->dev);
     return;
 }
 
@@ -610,6 +611,13 @@ static int ivshmem_load(QEMUFile* f, void *opaque, int version_id)
     return 0;
 }
 
+static void ivshmem_write_config(PCIDevice *pci_dev, uint32_t address,
+				 uint32_t val, int len)
+{
+    pci_default_write_config(pci_dev, address, val, len);
+    msix_write_config(pci_dev, address, val, len);
+}
+
 static int pci_ivshmem_init(PCIDevice *dev)
 {
     IVShmemState *s = DO_UPCAST(IVShmemState, dev, dev);
@@ -734,6 +742,8 @@ static int pci_ivshmem_init(PCIDevice *dev)
 
     }
 
+    s->dev.config_write = ivshmem_write_config;
+
     return 0;
 }
 
#6 lore raw

Re: [Qemu-devel] [PATCH for v1.0 1/3] msix: track function masked in pci device state

Quoted 50 lines · Michael S. Tsirkin
Quoted 2 lines · Cam Macdonell

Based on a git bisect, this patch breaks msi-x interrupt delivery in the ivshmem device.

I think the following should fix it. Compiled-only - could you pls check? If yes let's apply to the stable branch.

-->

ivshmem: add missing msix calls

ivshmem used msix but didn't call it on either reset or config write paths. This used to partically work since guests don't use all of msi-x configuration fields,
and reset is rarely used, but the patch 'msix: track function masked in pci device state' broke that. Fix by adding appropriate calls.

--

hw/ivshmem.c +8 -0
index 242fbea..3680c0f 100644
@@ -505,6 +505,7 @@ static void ivshmem_reset(DeviceState *d)
     IVShmemState *s = DO_UPCAST(IVShmemState, dev.qdev, d);
 
     s->intrstatus = 0;
+    msix_reset(&s->dev);
     return;
 }
 
@@ -610,6 +611,13 @@ static int ivshmem_load(QEMUFile* f, void *opaque, int version_id)
     return 0;
 }
 
+static void ivshmem_write_config(PCIDevice *pci_dev, uint32_t address,
+				 uint32_t val, int len)
+{
+    pci_default_write_config(pci_dev, address, val, len);
+    msix_write_config(pci_dev, address, val, len);
+}
+
 static int pci_ivshmem_init(PCIDevice *dev)
 {
     IVShmemState *s = DO_UPCAST(IVShmemState, dev, dev);

hw/ivshmem.c static int pci_ivshmem_init(PCIDevice *dev) · line 749

Quoted 8 lines
hw/ivshmem.c +2 -0
@@ -734,6 +742,8 @@ static int pci_ivshmem_init(PCIDevice *dev)
 
     }
 
+    s->dev.config_write = ivshmem_write_config;
+
     return 0;
 }
 

But please fix this for real and merge [1]&[2] (with depending patches) into master. The above is just boilerplate code from device POV.

Jan

[1] http://thread.gmane.org/gmane.comp.emulators.kvm.devel/80240
[2] http://thread.gmane.org/gmane.comp.emulators.kvm.devel/80244

#7 lore raw

Re: [Qemu-devel] [PATCH for v1.0 1/3] msix: track function masked in pci device state

Quoted 63 lines · Jan Kiszka
Quoted 50 lines · Michael S. Tsirkin
Quoted 2 lines · Cam Macdonell

Based on a git bisect, this patch breaks msi-x interrupt delivery in the ivshmem device.

I think the following should fix it. Compiled-only - could you pls check? If yes let's apply to the stable branch.

-->

ivshmem: add missing msix calls

ivshmem used msix but didn't call it on either reset or config write paths. This used to partically work since guests don't use all of msi-x configuration fields,
and reset is rarely used, but the patch 'msix: track function masked in pci device state' broke that. Fix by adding appropriate calls.

--

hw/ivshmem.c +8 -0
index 242fbea..3680c0f 100644
@@ -505,6 +505,7 @@ static void ivshmem_reset(DeviceState *d)
     IVShmemState *s = DO_UPCAST(IVShmemState, dev.qdev, d);
 
     s->intrstatus = 0;
+    msix_reset(&s->dev);
     return;
 }
 
@@ -610,6 +611,13 @@ static int ivshmem_load(QEMUFile* f, void *opaque, int version_id)
     return 0;
 }
 
+static void ivshmem_write_config(PCIDevice *pci_dev, uint32_t address,
+				 uint32_t val, int len)
+{
+    pci_default_write_config(pci_dev, address, val, len);
+    msix_write_config(pci_dev, address, val, len);
+}
+
 static int pci_ivshmem_init(PCIDevice *dev)
 {
     IVShmemState *s = DO_UPCAST(IVShmemState, dev, dev);

hw/ivshmem.c static int pci_ivshmem_init(PCIDevice *dev) · line 749

Quoted 8 lines
hw/ivshmem.c +2 -0
@@ -734,6 +742,8 @@ static int pci_ivshmem_init(PCIDevice *dev)
 
     }
 
+    s->dev.config_write = ivshmem_write_config;
+
     return 0;
 }
 

But please fix this for real and merge [1]&[2] (with depending patches) into master. The above is just boilerplate code from device POV.

Quoted 5 lines

Jan

[1] http://thread.gmane.org/gmane.comp.emulators.kvm.devel/80240
[2] http://thread.gmane.org/gmane.comp.emulators.kvm.devel/80244

Yes, I agree we should make it easier for devices. What annoyed me was the need to put msix in save/load.

And that is because of the need to do this in a specific order. I hope to switch to an unordered format and then this will become straight-forward.

Signature · 1 line
MST
#8 lore raw

Re: [Qemu-devel] [PATCH for v1.0 1/3] msix: track function masked in pci device state

In reply to Michael S. Tsirkin
Quoted 6 lines · Michael S. Tsirkin
Quoted 2 lines · Cam Macdonell

Based on a git bisect, this patch breaks msi-x interrupt delivery in the ivshmem device.

I think the following should fix it. Compiled-only - could you pls check? If yes let's apply to the stable branch.

Thanks for the patch Michael.

It addresses the need for msix_write_config() to be called, but the addition of the msix_reset() is causing a reset of the vectors after they've been initialized in pci_ivshmem_init(). So, interrupts still aren't delivered with this patch applied as it is.

In particular, a reset occurs after pci_ivshmem_init runs, so the msix_entry_used array is reset to 0s, which causes the interrupt delivery to fail.

If I comment out the msix_reset(), then interrupts are delivered. Would the reset be caused by a bug in the guest driver? or do I need to reconfigure the msix after reset? I'm unclear as to the proper behaviour after a reset.

Thanks,
Cam

Quoted 51 lines

-->

ivshmem: add missing msix calls

ivshmem used msix but didn't call it on either reset or config write paths. This used to partically work since guests don't use all of msi-x configuration fields,
and reset is rarely used, but the patch 'msix: track function masked in pci device state' broke that. Fix by adding appropriate calls.

--

hw/ivshmem.c +10 -0
index 242fbea..3680c0f 100644
@@ -505,6 +505,7 @@ static void ivshmem_reset(DeviceState *d)
     IVShmemState *s = DO_UPCAST(IVShmemState, dev.qdev, d);
 
     s->intrstatus = 0;
+    msix_reset(&s->dev);
     return;
  }
 
@@ -610,6 +611,13 @@ static int ivshmem_load(QEMUFile* f, void *opaque, int version_id)
     return 0;
  }
 
+static void ivshmem_write_config(PCIDevice *pci_dev, uint32_t address,
+                                uint32_t val, int len)
+{
+    pci_default_write_config(pci_dev, address, val, len);
+    msix_write_config(pci_dev, address, val, len);
+}
+
  static int pci_ivshmem_init(PCIDevice *dev)
  {
     IVShmemState *s = DO_UPCAST(IVShmemState, dev, dev);
@@ -734,6 +742,8 @@ static int pci_ivshmem_init(PCIDevice *dev)
 
     }
 
+    s->dev.config_write = ivshmem_write_config;
+
     return 0;
  }
 
#9 lore raw

Re: [Qemu-devel] [PATCH for v1.0 1/3] msix: track function masked in pci device state

Quoted 19 lines · Cam Macdonell
Quoted 6 lines · Michael S. Tsirkin
Quoted 2 lines · Cam Macdonell

Based on a git bisect, this patch breaks msi-x interrupt delivery in the ivshmem device.

I think the following should fix it. Compiled-only - could you pls check? If yes let's apply to the stable branch.

Thanks for the patch Michael.

It addresses the need for msix_write_config() to be called, but the addition of the msix_reset() is causing a reset of the vectors after they've been initialized in pci_ivshmem_init(). So, interrupts still aren't delivered with this patch applied as it is.

In particular, a reset occurs after pci_ivshmem_init runs, so the msix_entry_used array is reset to 0s, which causes the interrupt delivery to fail.

Quoted 8 lines

If I comment out the msix_reset(), then interrupts are delivered. Would the reset be caused by a bug in the guest driver? or do I need to reconfigure the msix after reset? I'm unclear as to the proper behaviour after a reset.

Thanks,
Cam

I didn't anticipate this, virtio only configures vectors when msi-x is enabled. For 1.0 it's safest to do the same and just re-enable after reset.

So we'll end up with something like the following - does it help?

Side note: exit(1) is not the best way to handle
errors in the init function. I think you should add error_report, then goto err on failures to init, cleanup what you have set up meanwhile and return an error code.

hw/ivshmem.c +31 -11
index 242fbea..c58f4d3 100644
@@ -500,11 +500,29 @@ static void ivshmem_read(void *opaque, const uint8_t * buf, int flags)
     return;
 }
 
+/* Select the MSI-X vectors used by device.
+ * ivshmem maps events to vectors statically, so
+ * we just enable all vectors on init and after reset. */
+static void ivshmem_use_msix(IVShmemState * s)
+{
+    int i;
+
+    if (!msix_present(&s->dev)) {
+        return;
+    }
+
+    for (i = 0; i < s->vectors; i++) {
+        msix_vector_use(&s->dev, i);
+    }
+}
+
 static void ivshmem_reset(DeviceState *d)
 {
     IVShmemState *s = DO_UPCAST(IVShmemState, dev.qdev, d);
 
     s->intrstatus = 0;
+    msix_reset(&s->dev);
+    ivshmem_use_msix(s);
     return;
 }
 
@@ -535,12 +553,8 @@ static uint64_t ivshmem_get_size(IVShmemState * s) {
     return value;
 }
 
-static void ivshmem_setup_msi(IVShmemState * s) {
-
-    int i;
-
-    /* allocate the MSI-X vectors */
-
+static void ivshmem_setup_msi(IVShmemState * s)
+{
     memory_region_init(&s->msix_bar, "ivshmem-msix", 4096);
     if (!msix_init(&s->dev, s->vectors, &s->msix_bar, 1, 0)) {
         pci_register_bar(&s->dev, 1, PCI_BASE_ADDRESS_SPACE_MEMORY,
@@ -551,13 +565,10 @@ static void ivshmem_setup_msi(IVShmemState * s) {
         exit(1);
     }
 
-    /* 'activate' the vectors */
-    for (i = 0; i < s->vectors; i++) {
-        msix_vector_use(&s->dev, i);
-    }
-
     /* allocate Qemu char devices for receiving interrupts */
     s->eventfd_table = g_malloc0(s->vectors * sizeof(EventfdEntry));
+
+    ivshmem_use_msix(s);
 }
 
 static void ivshmem_save(QEMUFile* f, void *opaque)
@@ -610,6 +621,13 @@ static int ivshmem_load(QEMUFile* f, void *opaque, int version_id)
     return 0;
 }
 
+static void ivshmem_write_config(PCIDevice *pci_dev, uint32_t address,
+				 uint32_t val, int len)
+{
+    pci_default_write_config(pci_dev, address, val, len);
+    msix_write_config(pci_dev, address, val, len);
+}
+
 static int pci_ivshmem_init(PCIDevice *dev)
 {
     IVShmemState *s = DO_UPCAST(IVShmemState, dev, dev);
@@ -734,6 +752,8 @@ static int pci_ivshmem_init(PCIDevice *dev)
 
     }
 
+    s->dev.config_write = ivshmem_write_config;
+
     return 0;
 }
 
#10 lore raw

Re: [Qemu-devel] [PATCH for v1.0 1/3] msix: track function masked in pci device state

Quoted 25 lines · Michael S. Tsirkin
Quoted 24 lines · Cam Macdonell
Quoted 6 lines · Michael S. Tsirkin
Quoted 2 lines · Cam Macdonell

Based on a git bisect, this patch breaks msi-x interrupt delivery in the ivshmem device.

I think the following should fix it. Compiled-only - could you pls check? If yes let's apply to the stable branch.

Thanks for the patch Michael.

It addresses the need for msix_write_config() to be called, but the addition of the msix_reset() is causing a reset of the vectors after they've been initialized in pci_ivshmem_init().  So, interrupts still aren't delivered with this patch applied as it is.

In particular, a reset occurs after pci_ivshmem_init runs, so the msix_entry_used array is reset to 0s, which causes the interrupt delivery to fail.

If I comment out the msix_reset(), then interrupts are delivered. Would the reset be caused by a bug in the guest driver?  or do I need to reconfigure the msix after reset?  I'm unclear as to the proper behaviour after a reset.

Quoted 8 lines
Quoted 3 lines

Thanks,
Cam

I didn't anticipate this, virtio only configures vectors when msi-x is enabled. For 1.0 it's safest to do the same and just re-enable after reset.

So we'll end up with something like the following - does it help?

Yes, this fixes interrupt delivery.

Quoted 5 lines

Side note: exit(1) is not the best way to handle
errors in the init function. I think you should add error_report, then goto err on failures to init, cleanup what you have set up meanwhile and return an error code.

Thanks, I'll fix that.

Quoted 90 lines
hw/ivshmem.c +31 -11
index 242fbea..c58f4d3 100644
@@ -500,11 +500,29 @@ static void ivshmem_read(void *opaque, const uint8_t * buf, int flags)
     return;
  }
 
+/* Select the MSI-X vectors used by device.
+ * ivshmem maps events to vectors statically, so
+ * we just enable all vectors on init and after reset. */
+static void ivshmem_use_msix(IVShmemState * s)
+{
+    int i;
+
+    if (!msix_present(&s->dev)) {
+        return;
+    }
+
+    for (i = 0; i < s->vectors; i++) {
+        msix_vector_use(&s->dev, i);
+    }
+}
+
  static void ivshmem_reset(DeviceState *d)
  {
     IVShmemState *s = DO_UPCAST(IVShmemState, dev.qdev, d);
 
     s->intrstatus = 0;
+    msix_reset(&s->dev);
+    ivshmem_use_msix(s);
     return;
  }
 
@@ -535,12 +553,8 @@ static uint64_t ivshmem_get_size(IVShmemState * s) {
     return value;
  }
 
-static void ivshmem_setup_msi(IVShmemState * s) {
-
-    int i;
-
-    /* allocate the MSI-X vectors */
-
+static void ivshmem_setup_msi(IVShmemState * s)
+{
     memory_region_init(&s->msix_bar, "ivshmem-msix", 4096);
     if (!msix_init(&s->dev, s->vectors, &s->msix_bar, 1, 0)) {
         pci_register_bar(&s->dev, 1, PCI_BASE_ADDRESS_SPACE_MEMORY,
@@ -551,13 +565,10 @@ static void ivshmem_setup_msi(IVShmemState * s) {
         exit(1);
     }
 
-    /* 'activate' the vectors */
-    for (i = 0; i < s->vectors; i++) {
-        msix_vector_use(&s->dev, i);
-    }
-
     /* allocate Qemu char devices for receiving interrupts */
     s->eventfd_table = g_malloc0(s->vectors * sizeof(EventfdEntry));
+
+    ivshmem_use_msix(s);
  }
 
  static void ivshmem_save(QEMUFile* f, void *opaque)
@@ -610,6 +621,13 @@ static int ivshmem_load(QEMUFile* f, void *opaque, int version_id)
     return 0;
  }
 
+static void ivshmem_write_config(PCIDevice *pci_dev, uint32_t address,
+                                uint32_t val, int len)
+{
+    pci_default_write_config(pci_dev, address, val, len);
+    msix_write_config(pci_dev, address, val, len);
+}
+
  static int pci_ivshmem_init(PCIDevice *dev)
  {
     IVShmemState *s = DO_UPCAST(IVShmemState, dev, dev);
@@ -734,6 +752,8 @@ static int pci_ivshmem_init(PCIDevice *dev)
 
     }
 
+    s->dev.config_write = ivshmem_write_config;
+
     return 0;
  }
 
#11 lore raw

PATCH 1/1 ivshmem: add missing msix calls

In reply to Cam Macdonell

ivshmem used msix but didn't call it on either reset or config write paths. This used to partically work since guests don't use all of msi-x configuration fields,
and reset is rarely used, but the patch 'msix: track function masked in pci device state' broke that. Fix by adding appropriate calls.

---

Please apply the following to both master
and 1.0 stable branch. Thanks!

hw/ivshmem.c +31 -11
index 242fbea..c58f4d3 100644
@@ -500,11 +500,29 @@ static void ivshmem_read(void *opaque, const uint8_t * buf, int flags)
     return;
 }
 
+/* Select the MSI-X vectors used by device.
+ * ivshmem maps events to vectors statically, so
+ * we just enable all vectors on init and after reset. */
+static void ivshmem_use_msix(IVShmemState * s)
+{
+    int i;
+
+    if (!msix_present(&s->dev)) {
+        return;
+    }
+
+    for (i = 0; i < s->vectors; i++) {
+        msix_vector_use(&s->dev, i);
+    }
+}
+
 static void ivshmem_reset(DeviceState *d)
 {
     IVShmemState *s = DO_UPCAST(IVShmemState, dev.qdev, d);
 
     s->intrstatus = 0;
+    msix_reset(&s->dev);
+    ivshmem_use_msix(s);
     return;
 }
 
@@ -535,12 +553,8 @@ static uint64_t ivshmem_get_size(IVShmemState * s) {
     return value;
 }
 
-static void ivshmem_setup_msi(IVShmemState * s) {
-
-    int i;
-
-    /* allocate the MSI-X vectors */
-
+static void ivshmem_setup_msi(IVShmemState * s)
+{
     memory_region_init(&s->msix_bar, "ivshmem-msix", 4096);
     if (!msix_init(&s->dev, s->vectors, &s->msix_bar, 1, 0)) {
         pci_register_bar(&s->dev, 1, PCI_BASE_ADDRESS_SPACE_MEMORY,
@@ -551,13 +565,10 @@ static void ivshmem_setup_msi(IVShmemState * s) {
         exit(1);
     }
 
-    /* 'activate' the vectors */
-    for (i = 0; i < s->vectors; i++) {
-        msix_vector_use(&s->dev, i);
-    }
-
     /* allocate Qemu char devices for receiving interrupts */
     s->eventfd_table = g_malloc0(s->vectors * sizeof(EventfdEntry));
+
+    ivshmem_use_msix(s);
 }
 
 static void ivshmem_save(QEMUFile* f, void *opaque)
@@ -610,6 +621,13 @@ static int ivshmem_load(QEMUFile* f, void *opaque, int version_id)
     return 0;
 }
 
+static void ivshmem_write_config(PCIDevice *pci_dev, uint32_t address,
+				 uint32_t val, int len)
+{
+    pci_default_write_config(pci_dev, address, val, len);
+    msix_write_config(pci_dev, address, val, len);
+}
+
 static int pci_ivshmem_init(PCIDevice *dev)
 {
     IVShmemState *s = DO_UPCAST(IVShmemState, dev, dev);
@@ -734,6 +752,8 @@ static int pci_ivshmem_init(PCIDevice *dev)
 
     }
 
+    s->dev.config_write = ivshmem_write_config;
+
     return 0;
 }
 
#12 lore raw

Re: [Qemu-devel] [PATCH master/v1.0.x] ivshmem: add missing msix calls

Hello,

Can this patch be merged, please?

Thanks,
Cam

Quoted 104 lines · Michael S. Tsirkin

ivshmem used msix but didn't call it on either reset or config write paths. This used to partically work since guests don't use all of msi-x configuration fields,
and reset is rarely used, but the patch 'msix: track function masked in pci device state' broke that. Fix by adding appropriate calls.

---

Please apply the following to both master
and 1.0 stable branch. Thanks!

hw/ivshmem.c +31 -11
index 242fbea..c58f4d3 100644
@@ -500,11 +500,29 @@ static void ivshmem_read(void *opaque, const uint8_t * buf, int flags)
     return;
  }
 
+/* Select the MSI-X vectors used by device.
+ * ivshmem maps events to vectors statically, so
+ * we just enable all vectors on init and after reset. */
+static void ivshmem_use_msix(IVShmemState * s)
+{
+    int i;
+
+    if (!msix_present(&s->dev)) {
+        return;
+    }
+
+    for (i = 0; i < s->vectors; i++) {
+        msix_vector_use(&s->dev, i);
+    }
+}
+
  static void ivshmem_reset(DeviceState *d)
  {
     IVShmemState *s = DO_UPCAST(IVShmemState, dev.qdev, d);
 
     s->intrstatus = 0;
+    msix_reset(&s->dev);
+    ivshmem_use_msix(s);
     return;
  }
 
@@ -535,12 +553,8 @@ static uint64_t ivshmem_get_size(IVShmemState * s) {
     return value;
  }
 
-static void ivshmem_setup_msi(IVShmemState * s) {
-
-    int i;
-
-    /* allocate the MSI-X vectors */
-
+static void ivshmem_setup_msi(IVShmemState * s)
+{
     memory_region_init(&s->msix_bar, "ivshmem-msix", 4096);
     if (!msix_init(&s->dev, s->vectors, &s->msix_bar, 1, 0)) {
         pci_register_bar(&s->dev, 1, PCI_BASE_ADDRESS_SPACE_MEMORY,
@@ -551,13 +565,10 @@ static void ivshmem_setup_msi(IVShmemState * s) {
         exit(1);
     }
 
-    /* 'activate' the vectors */
-    for (i = 0; i < s->vectors; i++) {
-        msix_vector_use(&s->dev, i);
-    }
-
     /* allocate Qemu char devices for receiving interrupts */
     s->eventfd_table = g_malloc0(s->vectors * sizeof(EventfdEntry));
+
+    ivshmem_use_msix(s);
  }
 
  static void ivshmem_save(QEMUFile* f, void *opaque)
@@ -610,6 +621,13 @@ static int ivshmem_load(QEMUFile* f, void *opaque, int version_id)
     return 0;
  }
 
+static void ivshmem_write_config(PCIDevice *pci_dev, uint32_t address,
+                                uint32_t val, int len)
+{
+    pci_default_write_config(pci_dev, address, val, len);
+    msix_write_config(pci_dev, address, val, len);
+}
+
  static int pci_ivshmem_init(PCIDevice *dev)
  {
     IVShmemState *s = DO_UPCAST(IVShmemState, dev, dev);
@@ -734,6 +752,8 @@ static int pci_ivshmem_init(PCIDevice *dev)
 
     }
 
+    s->dev.config_write = ivshmem_write_config;
+
     return 0;
  }
 
#13 lore raw

Re: [Qemu-devel] [PATCH master/v1.0.x] ivshmem: add missing msix calls

Am 13.01.2012 23:43, schrieb Cam Macdonell:

Quoted 1 line

Can this patch be merged, please?

You need to cc qemu-stable if you want it backported to v1.0.x once applied.

Andreas

Quoted 104 lines · Michael S. Tsirkin
Quoted 103 lines

ivshmem used msix but didn't call it on either reset or config write paths. This used to partically work since guests don't use all of msi-x configuration fields,
and reset is rarely used, but the patch 'msix: track function masked in pci device state' broke that. Fix by adding appropriate calls.

---

Please apply the following to both master
and 1.0 stable branch. Thanks!

hw/ivshmem.c +31 -11
index 242fbea..c58f4d3 100644
@@ -500,11 +500,29 @@ static void ivshmem_read(void *opaque, const uint8_t * buf, int flags)
    return;
 }
 
+/* Select the MSI-X vectors used by device.
+ * ivshmem maps events to vectors statically, so
+ * we just enable all vectors on init and after reset. */
+static void ivshmem_use_msix(IVShmemState * s)
+{
+    int i;
+
+    if (!msix_present(&s->dev)) {
+        return;
+    }
+
+    for (i = 0; i < s->vectors; i++) {
+        msix_vector_use(&s->dev, i);
+    }
+}
+
 static void ivshmem_reset(DeviceState *d)
 {
    IVShmemState *s = DO_UPCAST(IVShmemState, dev.qdev, d);
 
    s->intrstatus = 0;
+    msix_reset(&s->dev);
+    ivshmem_use_msix(s);
    return;
 }
 
@@ -535,12 +553,8 @@ static uint64_t ivshmem_get_size(IVShmemState * s) {
    return value;
 }
 
-static void ivshmem_setup_msi(IVShmemState * s) {
-
-    int i;
-
-    /* allocate the MSI-X vectors */
-
+static void ivshmem_setup_msi(IVShmemState * s)
+{
    memory_region_init(&s->msix_bar, "ivshmem-msix", 4096);
    if (!msix_init(&s->dev, s->vectors, &s->msix_bar, 1, 0)) {
        pci_register_bar(&s->dev, 1, PCI_BASE_ADDRESS_SPACE_MEMORY,
@@ -551,13 +565,10 @@ static void ivshmem_setup_msi(IVShmemState * s) {
        exit(1);
    }
 
-    /* 'activate' the vectors */
-    for (i = 0; i < s->vectors; i++) {
-        msix_vector_use(&s->dev, i);
-    }
-
    /* allocate Qemu char devices for receiving interrupts */
    s->eventfd_table = g_malloc0(s->vectors * sizeof(EventfdEntry));
+
+    ivshmem_use_msix(s);
 }
 
 static void ivshmem_save(QEMUFile* f, void *opaque)
@@ -610,6 +621,13 @@ static int ivshmem_load(QEMUFile* f, void *opaque, int version_id)
    return 0;
 }
 
+static void ivshmem_write_config(PCIDevice *pci_dev, uint32_t address,
+                                uint32_t val, int len)
+{
+    pci_default_write_config(pci_dev, address, val, len);
+    msix_write_config(pci_dev, address, val, len);
+}
+
 static int pci_ivshmem_init(PCIDevice *dev)
 {
    IVShmemState *s = DO_UPCAST(IVShmemState, dev, dev);
@@ -734,6 +752,8 @@ static int pci_ivshmem_init(PCIDevice *dev)
 
    }
 
+    s->dev.config_write = ivshmem_write_config;
+
    return 0;
 }
 
Signature · 2 lines
SUSE LINUX Products GmbH, Maxfeldstr. 5, 90409 Nürnberg, Germany
GF: Jeff Hawn, Jennifer Guild, Felix Imendörffer; HRB 16746 AG Nürnberg
#14 lore raw

PATCH 2/3 msix: Prevent bogus mask updates on MMIO accesses

In reply to Michael S. Tsirkin
Quoted 1 line

From: Jan Kiszka <[email protected]>

Only accesses to the MSI-X table must trigger a call to msix_handle_mask_update, otherwise the vector value might be out of range.

---
 hw/msix.c |    6 ++++++
 1 files changed, 6 insertions(+), 0 deletions(-)
hw/msix.c +6 -0
index 63b41b9..2969601 100644
@@ -176,6 +176,12 @@ static void msix_mmio_write(void *opaque, target_phys_addr_t addr,
     PCIDevice *dev = opaque;
     unsigned int offset = addr & (MSIX_PAGE_SIZE - 1) & ~0x3;
     int vector = offset / PCI_MSIX_ENTRY_SIZE;
+
+    /* MSI-X page includes a read-only PBA and a writeable Vector Control. */
+    if (vector >= dev->msix_entries_nr) {
+        return;
+    }
+
     pci_set_long(dev->msix_table_page + offset, val);
     msix_handle_mask_update(dev, vector);
 }
Signature · 1 line
1.7.5.53.gc233e
#15 lore raw

PATCH 3/3 msix: avoid mask updates if mask is unchanged

In reply to Michael S. Tsirkin

Check pending bit only if vector mask status changed. This is not really important for qemu.git but helps fix a bug in qemu-kvm.git.

---
 hw/msix.c |   29 ++++++++++++++++++++---------
 1 files changed, 20 insertions(+), 9 deletions(-)
hw/msix.c +20 -9
index 2969601..149eed2 100644
@@ -118,17 +118,25 @@ static void msix_clr_pending(PCIDevice *dev, int vector)
     *msix_pending_byte(dev, vector) &= ~msix_pending_mask(vector);
 }
 
-static int msix_is_masked(PCIDevice *dev, int vector)
+static bool msix_vector_masked(PCIDevice *dev, int vector, bool fmask)
 {
-    unsigned offset =
-        vector * PCI_MSIX_ENTRY_SIZE + PCI_MSIX_ENTRY_VECTOR_CTRL;
-    return dev->msix_function_masked ||
-	   dev->msix_table_page[offset] & PCI_MSIX_ENTRY_CTRL_MASKBIT;
+    unsigned offset = vector * PCI_MSIX_ENTRY_SIZE + PCI_MSIX_ENTRY_VECTOR_CTRL;
+    return fmask || dev->msix_table_page[offset] & PCI_MSIX_ENTRY_CTRL_MASKBIT;
 }
 
-static void msix_handle_mask_update(PCIDevice *dev, int vector)
+static bool msix_is_masked(PCIDevice *dev, int vector)
 {
-    if (!msix_is_masked(dev, vector) && msix_is_pending(dev, vector)) {
+    return msix_vector_masked(dev, vector, dev->msix_function_masked);
+}
+
+static void msix_handle_mask_update(PCIDevice *dev, int vector, bool was_masked)
+{
+    bool is_masked = msix_is_masked(dev, vector);
+    if (is_masked == was_masked) {
+        return;
+    }
+
+    if (!is_masked && msix_is_pending(dev, vector)) {
         msix_clr_pending(dev, vector);
         msix_notify(dev, vector);
     }
@@ -166,7 +174,8 @@ void msix_write_config(PCIDevice *dev, uint32_t addr,
     }
 
     for (vector = 0; vector < dev->msix_entries_nr; ++vector) {
-        msix_handle_mask_update(dev, vector);
+        msix_handle_mask_update(dev, vector,
+                                msix_vector_masked(dev, vector, was_masked));
     }
 }
 
@@ -176,14 +185,16 @@ static void msix_mmio_write(void *opaque, target_phys_addr_t addr,
     PCIDevice *dev = opaque;
     unsigned int offset = addr & (MSIX_PAGE_SIZE - 1) & ~0x3;
     int vector = offset / PCI_MSIX_ENTRY_SIZE;
+    bool was_masked;
 
     /* MSI-X page includes a read-only PBA and a writeable Vector Control. */
     if (vector >= dev->msix_entries_nr) {
         return;
     }
 
+    was_masked = msix_is_masked(dev, vector);
     pci_set_long(dev->msix_table_page + offset, val);
-    msix_handle_mask_update(dev, vector);
+    msix_handle_mask_update(dev, vector, was_masked);
 }
 
 static const MemoryRegionOps msix_mmio_ops = {
Signature · 1 line
1.7.5.53.gc233e
#16 lore raw
In reply to Michael S. Tsirkin
Quoted 1 line · Michael S. Tsirkin

This fixes bugs dealing with msi-x mask bits pointed out by Jan Kiszka.

Applied. Thanks.

Regards,

Anthony Liguori

Quoted 12 lines

Jan Kiszka (1):
msix: Prevent bogus mask updates on MMIO accesses

Michael S. Tsirkin (2):
msix: track function masked in pci device state
msix: avoid mask updates if mask is unchanged

  hw/msix.c |   48 ++++++++++++++++++++++++++++++++++++------------
  hw/pci.h  |    2 ++
  2 files changed, 38 insertions(+), 12 deletions(-)