Skip to contentSkip to conversation

qemu-kvm: pci-assign: Host IRQ sharing suppport

PATCH Superseded Jan Kiszka 2 participants 12 messages Mar 7, 2012 KVM

v1 of 4 Superseded Latest is v4 →

lore
Share & tools
lore

Jan Kiszka

author
#0 lore raw

cover qemu-kvm: pci-assign: Host IRQ sharing suppport

This enables support for the host IRQ sharing support via PCI-2.3 interrupt masking. See the patches for details.

Patch 1 could be considered for uq/master although we have no use case for the changes in upstream yet.

Jan Kiszka (3):
kvm: Update kernel headers against kvm.git
pci-assign: Use PCI-2.3-based shared legacy interrupts
pci_assign: Flip defaults of prefer_msi and share_intx

 hw/device-assignment.c       |   23 ++++++++++++++++++++++-
 hw/device-assignment.h       |   10 ++++++----
 linux-headers/asm-s390/kvm.h |    2 ++
 linux-headers/linux/kvm.h    |    6 ++++++
 qemu-kvm.c                   |    9 +++++++++
 qemu-kvm.h                   |    2 ++
 6 files changed, 47 insertions(+), 5 deletions(-)
Signature · 1 line
1.7.3.4
author
#1 lore raw

PATCH 1/3 kvm: Update kernel headers against kvm.git

Required for INTx masking / host IRQ sharing support of assigned devices.

Corresponding kvm.git hash: 186195928e

Signed-off-by Jan Kiszka <[email protected]>
---
 linux-headers/asm-s390/kvm.h |    2 ++
 linux-headers/linux/kvm.h    |    6 ++++++
 2 files changed, 8 insertions(+), 0 deletions(-)
linux-headers/asm-s390/kvm.h +2 -0
index 9acbde4..9607667 100644
@@ -44,10 +44,12 @@ struct kvm_guest_debug_arch {
 #define KVM_SYNC_PREFIX (1UL << 0)
 #define KVM_SYNC_GPRS   (1UL << 1)
 #define KVM_SYNC_ACRS   (1UL << 2)
+#define KVM_SYNC_CRS    (1UL << 3)
 /* definition of registers in kvm_run */
 struct kvm_sync_regs {
 	__u64 prefix;	/* prefix register */
 	__u64 gprs[16];	/* general purpose registers */
 	__u32 acrs[16];	/* access registers */
+	__u64 crs[16];	/* control registers */
 };
 #endif
linux-headers/linux/kvm.h +6 -0
index f6b5343..cf7b098 100644
@@ -588,6 +588,7 @@ struct kvm_ppc_pvinfo {
 #define KVM_CAP_TSC_DEADLINE_TIMER 72
 #define KVM_CAP_S390_UCONTROL 73
 #define KVM_CAP_SYNC_REGS 74
+#define KVM_CAP_PCI_2_3 75
 
 #ifdef KVM_CAP_IRQ_ROUTING
 

@@ -784,6 +785,9 @@ struct kvm_s390_ucas_mapping {
 /* Available with KVM_CAP_TSC_CONTROL */
 #define KVM_SET_TSC_KHZ           _IO(KVMIO,  0xa2)
 #define KVM_GET_TSC_KHZ           _IO(KVMIO,  0xa3)
+/* Available with KVM_CAP_PCI_2_3 */
+#define KVM_ASSIGN_SET_INTX_MASK  _IOW(KVMIO,  0xa4, \
+				       struct kvm_assigned_pci_dev)
 
 /*
  * ioctls for vcpu fds

@@ -857,6 +861,8 @@ struct kvm_s390_ucas_mapping {
 #define KVM_SET_ONE_REG		  _IOW(KVMIO,  0xac, struct kvm_one_reg)
 
 #define KVM_DEV_ASSIGN_ENABLE_IOMMU	(1 << 0)
+#define KVM_DEV_ASSIGN_PCI_2_3		(1 << 1)
+#define KVM_DEV_ASSIGN_MASK_INTX	(1 << 2)
 
 struct kvm_assigned_pci_dev {
 	__u32 assigned_dev_id;
Signature · 1 line
1.7.3.4
author
#2 lore raw

PATCH 2/3 pci-assign: Use PCI-2.3-based shared legacy interrupts

In reply to Jan Kiszka

Enable the new KVM feature that allows legacy interrupt sharing for PCI-2.3-compliant devices. This requires to synchronize any guest change of the INTx mask bit to the kernel.

The feature is controlled by the property 'share_intx' and is off by default for now.

Signed-off-by Jan Kiszka <[email protected]>
---
 hw/device-assignment.c |   21 +++++++++++++++++++++
 hw/device-assignment.h |   10 ++++++----
 qemu-kvm.c             |    9 +++++++++
 qemu-kvm.h             |    2 ++
 4 files changed, 38 insertions(+), 4 deletions(-)
hw/device-assignment.c +21 -0
index 0e2c7b5..45e7913 100644
@@ -782,6 +782,13 @@ static int assign_device(AssignedDevice *dev)
                 "cause host memory corruption if the device issues DMA write "
                 "requests!\n");
     }
+    if (dev->features & ASSIGNED_DEVICE_SHARE_INTX_MASK) {
+        assigned_dev_data.flags |= KVM_DEV_ASSIGN_PCI_2_3;
+
+        /* hide host-side INTx masking from the guest */
+        dev->emulate_config_read[PCI_COMMAND + 1] |=
+            PCI_COMMAND_INTX_DISABLE >> 8;
+    }
 
     r = kvm_assign_pci_device(kvm_state, &assigned_dev_data);
     if (r < 0) {

@@ -1122,9 +1129,21 @@ static void assigned_dev_pci_write_config(PCIDevice *pci_dev, uint32_t address,
 {
     AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
     uint32_t emulate_mask, full_emulation_mask;
+    int ret;
 
     pci_default_write_config(pci_dev, address, val, len);
 
+    if (range_covers_byte(address, len, PCI_COMMAND + 1)) {
+        bool intx_masked = (pci_get_word(pci_dev->config + PCI_COMMAND) &
+                            PCI_COMMAND_INTX_DISABLE);
+
+        ret = kvm_device_intx_set_mask(kvm_state,
+                                       calc_assigned_dev_id(assigned_dev),
+                                       intx_masked);
+        if (ret) {
+            perror("assigned_dev_pci_write_config: set intx mask");
+        }
+    }
     if (assigned_dev->cap.available & ASSIGNED_DEVICE_CAP_MSI) {
         if (range_covers_byte(address, len,
                               pci_dev->msi_cap + PCI_MSI_FLAGS)) {

@@ -1748,6 +1767,8 @@ static Property da_properties[] =
                    ASSIGNED_DEVICE_USE_IOMMU_BIT, true),
     DEFINE_PROP_BIT("prefer_msi", AssignedDevice, features,
                    ASSIGNED_DEVICE_PREFER_MSI_BIT, true),
+    DEFINE_PROP_BIT("share_intx", AssignedDevice, features,
+                    ASSIGNED_DEVICE_SHARE_INTX_BIT, false),
     DEFINE_PROP_INT32("bootindex", AssignedDevice, bootindex, -1),
     DEFINE_PROP_STRING("configfd", AssignedDevice, configfd_name),
     DEFINE_PROP_END_OF_LIST(),
hw/device-assignment.h +6 -4
index b4bcfa6..5d271d5 100644
@@ -74,11 +74,13 @@ typedef struct {
     PCIRegion *region;
 } AssignedDevRegion;
 
-#define ASSIGNED_DEVICE_USE_IOMMU_BIT	0
-#define ASSIGNED_DEVICE_PREFER_MSI_BIT	1
+#define ASSIGNED_DEVICE_USE_IOMMU_BIT   0
+#define ASSIGNED_DEVICE_PREFER_MSI_BIT  1
+#define ASSIGNED_DEVICE_SHARE_INTX_BIT  2
 
-#define ASSIGNED_DEVICE_USE_IOMMU_MASK	(1 << ASSIGNED_DEVICE_USE_IOMMU_BIT)
-#define ASSIGNED_DEVICE_PREFER_MSI_MASK	(1 << ASSIGNED_DEVICE_PREFER_MSI_BIT)
+#define ASSIGNED_DEVICE_USE_IOMMU_MASK  (1 << ASSIGNED_DEVICE_USE_IOMMU_BIT)
+#define ASSIGNED_DEVICE_PREFER_MSI_MASK (1 << ASSIGNED_DEVICE_PREFER_MSI_BIT)
+#define ASSIGNED_DEVICE_SHARE_INTX_MASK (1 << ASSIGNED_DEVICE_SHARE_INTX_BIT)
 
 typedef struct {
     uint32_t addr_lo;
qemu-kvm.c +9 -0
index b1eee80..520d737 100644
@@ -189,6 +189,15 @@ static int kvm_old_assign_irq(KVMState *s,
     return kvm_vm_ioctl(s, KVM_ASSIGN_IRQ, assigned_irq);
 }
 
+int kvm_device_intx_set_mask(KVMState *s, uint32_t dev_id, bool masked)
+{
+    struct kvm_assigned_pci_dev assigned_dev;
+
+    assigned_dev.assigned_dev_id = dev_id;
+    assigned_dev.flags = masked ? KVM_DEV_ASSIGN_MASK_INTX : 0;
+    return kvm_vm_ioctl(s, KVM_ASSIGN_SET_INTX_MASK, &assigned_dev);
+}
+
 #ifdef KVM_CAP_ASSIGN_DEV_IRQ
 int kvm_assign_irq(KVMState *s, struct kvm_assigned_irq *assigned_irq)
 {
qemu-kvm.h +2 -0
index 975b6fa..63f46a5 100644
@@ -161,6 +161,8 @@ int kvm_assign_irq(KVMState *s, struct kvm_assigned_irq *assigned_irq);
  */
 int kvm_deassign_irq(KVMState *s, struct kvm_assigned_irq *assigned_irq);
 
+int kvm_device_intx_set_mask(KVMState *s, uint32_t dev_id, bool masked);
+
 /*!
  * \brief Notifies host kernel about a PCI device to be deassigned from a guest
  *
Signature · 1 line
1.7.3.4
#3 lore raw

Re: [PATCH 2/3] pci-assign: Use PCI-2.3-based shared legacy interrupts

Quoted 42 lines · Jan Kiszka

Enable the new KVM feature that allows legacy interrupt sharing for PCI-2.3-compliant devices. This requires to synchronize any guest change of the INTx mask bit to the kernel.

The feature is controlled by the property 'share_intx' and is off by default for now.

Signed-off-by Jan Kiszka <[email protected]>
---
 hw/device-assignment.c |   21 +++++++++++++++++++++
 hw/device-assignment.h |   10 ++++++----
 qemu-kvm.c             |    9 +++++++++
 qemu-kvm.h             |    2 ++
 4 files changed, 38 insertions(+), 4 deletions(-)
hw/device-assignment.c +9 -0
index 0e2c7b5..45e7913 100644
@@ -782,6 +782,13 @@ static int assign_device(AssignedDevice *dev)
                 "cause host memory corruption if the device issues DMA write "
                 "requests!\n");
     }
+    if (dev->features & ASSIGNED_DEVICE_SHARE_INTX_MASK) {
+        assigned_dev_data.flags |= KVM_DEV_ASSIGN_PCI_2_3;
+
+        /* hide host-side INTx masking from the guest */
+        dev->emulate_config_read[PCI_COMMAND + 1] |=
+            PCI_COMMAND_INTX_DISABLE >> 8;
+    }
 
     r = kvm_assign_pci_device(kvm_state, &assigned_dev_data);
     if (r < 0) {

@@ -1122,9 +1129,21 @@ static void assigned_dev_pci_write_config(PCIDevice *pci_dev, uint32_t address,
 {
     AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
     uint32_t emulate_mask, full_emulation_mask;
+    int ret;
 
     pci_default_write_config(pci_dev, address, val, len);
 
+    if (range_covers_byte(address, len, PCI_COMMAND + 1)) {

hw/device-assignment.c static void assigned_dev_pci_write_config(PCIDevice *pci_dev, uint32_t address, · line 1146

Quoted 10 lines
hw/device-assignment.c +10 -0
+        bool intx_masked = (pci_get_word(pci_dev->config + PCI_COMMAND) &
+                            PCI_COMMAND_INTX_DISABLE);
+
+        ret = kvm_device_intx_set_mask(kvm_state,
+                                       calc_assigned_dev_id(assigned_dev),
+                                       intx_masked);
+        if (ret) {
+            perror("assigned_dev_pci_write_config: set intx mask");
+        }
+    }

Can we be a little more strategic and only call intx_set_mask when the bit changes? Looks ok otherwise. Thanks,

Alex

author
#4 lore raw

PATCH v2 2/3 pci-assign: Use PCI-2.3-based shared legacy interrupts

Enable the new KVM feature that allows legacy interrupt sharing for PCI-2.3-compliant devices. This requires to synchronize any guest change of the INTx mask bit to the kernel.

The feature is controlled by the property 'share_intx' and is off by default for now.

Signed-off-by Jan Kiszka <[email protected]>

---

Changes in v2:
- only call into kvm_device_intx_set_mask if the mask actually changed

 hw/device-assignment.c |   24 ++++++++++++++++++++++++
 hw/device-assignment.h |   10 ++++++----
 qemu-kvm.c             |    9 +++++++++
 qemu-kvm.h             |    2 ++
 4 files changed, 41 insertions(+), 4 deletions(-)
hw/device-assignment.c +24 -0
index a5f1abb..12e2195 100644
@@ -782,6 +782,13 @@ static int assign_device(AssignedDevice *dev)
                 "cause host memory corruption if the device issues DMA write "
                 "requests!\n");
     }
+    if (dev->features & ASSIGNED_DEVICE_SHARE_INTX_MASK) {
+        assigned_dev_data.flags |= KVM_DEV_ASSIGN_PCI_2_3;
+
+        /* hide host-side INTx masking from the guest */
+        dev->emulate_config_read[PCI_COMMAND + 1] |=
+            PCI_COMMAND_INTX_DISABLE >> 8;
+    }
 
     r = kvm_assign_pci_device(kvm_state, &assigned_dev_data);
     if (r < 0) {

@@ -1121,10 +1128,25 @@ static void assigned_dev_pci_write_config(PCIDevice *pci_dev, uint32_t address,
                                           uint32_t val, int len)
 {
     AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+    uint32_t old_cmd = pci_get_word(pci_dev->config + PCI_COMMAND);
     uint32_t emulate_mask, full_emulation_mask;
+    int ret;
 
     pci_default_write_config(pci_dev, address, val, len);
 
+    if (range_covers_byte(address, len, PCI_COMMAND + 1)) {
+        bool intx_masked = (pci_get_word(pci_dev->config + PCI_COMMAND) &
+                            PCI_COMMAND_INTX_DISABLE);
+
+        if (intx_masked != !!(old_cmd & PCI_COMMAND_INTX_DISABLE)) {
+            ret = kvm_device_intx_set_mask(kvm_state,
+                                           calc_assigned_dev_id(assigned_dev),
+                                           intx_masked);
+            if (ret) {
+                perror("assigned_dev_pci_write_config: set intx mask");
+            }
+        }
+    }
     if (assigned_dev->cap.available & ASSIGNED_DEVICE_CAP_MSI) {
         if (range_covers_byte(address, len,
                               pci_dev->msi_cap + PCI_MSI_FLAGS)) {

@@ -1748,6 +1770,8 @@ static Property da_properties[] =
                    ASSIGNED_DEVICE_USE_IOMMU_BIT, true),
     DEFINE_PROP_BIT("prefer_msi", AssignedDevice, features,
                    ASSIGNED_DEVICE_PREFER_MSI_BIT, true),
+    DEFINE_PROP_BIT("share_intx", AssignedDevice, features,
+                    ASSIGNED_DEVICE_SHARE_INTX_BIT, false),
     DEFINE_PROP_INT32("bootindex", AssignedDevice, bootindex, -1),
     DEFINE_PROP_STRING("configfd", AssignedDevice, configfd_name),
     DEFINE_PROP_END_OF_LIST(),
hw/device-assignment.h +6 -4
index b4bcfa6..5d271d5 100644
@@ -74,11 +74,13 @@ typedef struct {
     PCIRegion *region;
 } AssignedDevRegion;
 
-#define ASSIGNED_DEVICE_USE_IOMMU_BIT	0
-#define ASSIGNED_DEVICE_PREFER_MSI_BIT	1
+#define ASSIGNED_DEVICE_USE_IOMMU_BIT   0
+#define ASSIGNED_DEVICE_PREFER_MSI_BIT  1
+#define ASSIGNED_DEVICE_SHARE_INTX_BIT  2
 
-#define ASSIGNED_DEVICE_USE_IOMMU_MASK	(1 << ASSIGNED_DEVICE_USE_IOMMU_BIT)
-#define ASSIGNED_DEVICE_PREFER_MSI_MASK	(1 << ASSIGNED_DEVICE_PREFER_MSI_BIT)
+#define ASSIGNED_DEVICE_USE_IOMMU_MASK  (1 << ASSIGNED_DEVICE_USE_IOMMU_BIT)
+#define ASSIGNED_DEVICE_PREFER_MSI_MASK (1 << ASSIGNED_DEVICE_PREFER_MSI_BIT)
+#define ASSIGNED_DEVICE_SHARE_INTX_MASK (1 << ASSIGNED_DEVICE_SHARE_INTX_BIT)
 
 typedef struct {
     uint32_t addr_lo;
qemu-kvm.c +9 -0
index 09a35f0..2047ebb 100644
@@ -54,6 +54,15 @@ static int kvm_old_assign_irq(KVMState *s,
     return kvm_vm_ioctl(s, KVM_ASSIGN_IRQ, assigned_irq);
 }
 
+int kvm_device_intx_set_mask(KVMState *s, uint32_t dev_id, bool masked)
+{
+    struct kvm_assigned_pci_dev assigned_dev;
+
+    assigned_dev.assigned_dev_id = dev_id;
+    assigned_dev.flags = masked ? KVM_DEV_ASSIGN_MASK_INTX : 0;
+    return kvm_vm_ioctl(s, KVM_ASSIGN_SET_INTX_MASK, &assigned_dev);
+}
+
 #ifdef KVM_CAP_ASSIGN_DEV_IRQ
 int kvm_assign_irq(KVMState *s, struct kvm_assigned_irq *assigned_irq)
 {
qemu-kvm.h +2 -0
index 3c4f023..2b23daf 100644
@@ -114,6 +114,8 @@ int kvm_assign_irq(KVMState *s, struct kvm_assigned_irq *assigned_irq);
  */
 int kvm_deassign_irq(KVMState *s, struct kvm_assigned_irq *assigned_irq);
 
+int kvm_device_intx_set_mask(KVMState *s, uint32_t dev_id, bool masked);
+
 /*!
  * \brief Notifies host kernel about a PCI device to be deassigned from a guest
  *
Signature · 1 line
1.7.3.4
#5 lore raw

Re: [PATCH v2 2/3] pci-assign: Use PCI-2.3-based shared legacy interrupts

Quoted 38 lines · Jan Kiszka

Enable the new KVM feature that allows legacy interrupt sharing for PCI-2.3-compliant devices. This requires to synchronize any guest change of the INTx mask bit to the kernel.

The feature is controlled by the property 'share_intx' and is off by default for now.

Signed-off-by Jan Kiszka <[email protected]>

---

Changes in v2:
- only call into kvm_device_intx_set_mask if the mask actually changed

 hw/device-assignment.c |   24 ++++++++++++++++++++++++
 hw/device-assignment.h |   10 ++++++----
 qemu-kvm.c             |    9 +++++++++
 qemu-kvm.h             |    2 ++
 4 files changed, 41 insertions(+), 4 deletions(-)
hw/device-assignment.c +7 -0
index a5f1abb..12e2195 100644
@@ -782,6 +782,13 @@ static int assign_device(AssignedDevice *dev)
                 "cause host memory corruption if the device issues DMA write "
                 "requests!\n");
     }
+    if (dev->features & ASSIGNED_DEVICE_SHARE_INTX_MASK) {
+        assigned_dev_data.flags |= KVM_DEV_ASSIGN_PCI_2_3;
+
+        /* hide host-side INTx masking from the guest */
+        dev->emulate_config_read[PCI_COMMAND + 1] |=
+            PCI_COMMAND_INTX_DISABLE >> 8;
+    }
 
     r = kvm_assign_pci_device(kvm_state, &assigned_dev_data);
     if (r < 0) {

hw/device-assignment.c static void assigned_dev_pci_write_config(PCIDevice *pci_dev, uint32_t address, · line 1131

Quoted 4 lines
hw/device-assignment.c +1 -0
@@ -1121,10 +1128,25 @@ static void assigned_dev_pci_write_config(PCIDevice *pci_dev, uint32_t address,
                                           uint32_t val, int len)
 {
     AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+    uint32_t old_cmd = pci_get_word(pci_dev->config + PCI_COMMAND);

nit, uint16_t

author
#6 lore raw

PATCH v3 2/3 pci-assign: Use PCI-2.3-based shared legacy interrupts

Enable the new KVM feature that allows legacy interrupt sharing for PCI-2.3-compliant devices. This requires to synchronize any guest change of the INTx mask bit to the kernel.

The feature is controlled by the property 'share_intx' and is off by default for now.

Signed-off-by Jan Kiszka <[email protected]>

---

Changes in v3:
- fixed nit :)

 hw/device-assignment.c |   24 ++++++++++++++++++++++++
 hw/device-assignment.h |   10 ++++++----
 qemu-kvm.c             |    9 +++++++++
 qemu-kvm.h             |    2 ++
 4 files changed, 41 insertions(+), 4 deletions(-)
hw/device-assignment.c +24 -0
index a5f1abb..e2a8479 100644
@@ -782,6 +782,13 @@ static int assign_device(AssignedDevice *dev)
                 "cause host memory corruption if the device issues DMA write "
                 "requests!\n");
     }
+    if (dev->features & ASSIGNED_DEVICE_SHARE_INTX_MASK) {
+        assigned_dev_data.flags |= KVM_DEV_ASSIGN_PCI_2_3;
+
+        /* hide host-side INTx masking from the guest */
+        dev->emulate_config_read[PCI_COMMAND + 1] |=
+            PCI_COMMAND_INTX_DISABLE >> 8;
+    }
 
     r = kvm_assign_pci_device(kvm_state, &assigned_dev_data);
     if (r < 0) {

@@ -1121,10 +1128,25 @@ static void assigned_dev_pci_write_config(PCIDevice *pci_dev, uint32_t address,
                                           uint32_t val, int len)
 {
     AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+    uint16_t old_cmd = pci_get_word(pci_dev->config + PCI_COMMAND);
     uint32_t emulate_mask, full_emulation_mask;
+    int ret;
 
     pci_default_write_config(pci_dev, address, val, len);
 
+    if (range_covers_byte(address, len, PCI_COMMAND + 1)) {
+        bool intx_masked = (pci_get_word(pci_dev->config + PCI_COMMAND) &
+                            PCI_COMMAND_INTX_DISABLE);
+
+        if (intx_masked != !!(old_cmd & PCI_COMMAND_INTX_DISABLE)) {
+            ret = kvm_device_intx_set_mask(kvm_state,
+                                           calc_assigned_dev_id(assigned_dev),
+                                           intx_masked);
+            if (ret) {
+                perror("assigned_dev_pci_write_config: set intx mask");
+            }
+        }
+    }
     if (assigned_dev->cap.available & ASSIGNED_DEVICE_CAP_MSI) {
         if (range_covers_byte(address, len,
                               pci_dev->msi_cap + PCI_MSI_FLAGS)) {

@@ -1748,6 +1770,8 @@ static Property da_properties[] =
                    ASSIGNED_DEVICE_USE_IOMMU_BIT, true),
     DEFINE_PROP_BIT("prefer_msi", AssignedDevice, features,
                    ASSIGNED_DEVICE_PREFER_MSI_BIT, true),
+    DEFINE_PROP_BIT("share_intx", AssignedDevice, features,
+                    ASSIGNED_DEVICE_SHARE_INTX_BIT, false),
     DEFINE_PROP_INT32("bootindex", AssignedDevice, bootindex, -1),
     DEFINE_PROP_STRING("configfd", AssignedDevice, configfd_name),
     DEFINE_PROP_END_OF_LIST(),
hw/device-assignment.h +6 -4
index b4bcfa6..5d271d5 100644
@@ -74,11 +74,13 @@ typedef struct {
     PCIRegion *region;
 } AssignedDevRegion;
 
-#define ASSIGNED_DEVICE_USE_IOMMU_BIT	0
-#define ASSIGNED_DEVICE_PREFER_MSI_BIT	1
+#define ASSIGNED_DEVICE_USE_IOMMU_BIT   0
+#define ASSIGNED_DEVICE_PREFER_MSI_BIT  1
+#define ASSIGNED_DEVICE_SHARE_INTX_BIT  2
 
-#define ASSIGNED_DEVICE_USE_IOMMU_MASK	(1 << ASSIGNED_DEVICE_USE_IOMMU_BIT)
-#define ASSIGNED_DEVICE_PREFER_MSI_MASK	(1 << ASSIGNED_DEVICE_PREFER_MSI_BIT)
+#define ASSIGNED_DEVICE_USE_IOMMU_MASK  (1 << ASSIGNED_DEVICE_USE_IOMMU_BIT)
+#define ASSIGNED_DEVICE_PREFER_MSI_MASK (1 << ASSIGNED_DEVICE_PREFER_MSI_BIT)
+#define ASSIGNED_DEVICE_SHARE_INTX_MASK (1 << ASSIGNED_DEVICE_SHARE_INTX_BIT)
 
 typedef struct {
     uint32_t addr_lo;
qemu-kvm.c +9 -0
index 09a35f0..2047ebb 100644
@@ -54,6 +54,15 @@ static int kvm_old_assign_irq(KVMState *s,
     return kvm_vm_ioctl(s, KVM_ASSIGN_IRQ, assigned_irq);
 }
 
+int kvm_device_intx_set_mask(KVMState *s, uint32_t dev_id, bool masked)
+{
+    struct kvm_assigned_pci_dev assigned_dev;
+
+    assigned_dev.assigned_dev_id = dev_id;
+    assigned_dev.flags = masked ? KVM_DEV_ASSIGN_MASK_INTX : 0;
+    return kvm_vm_ioctl(s, KVM_ASSIGN_SET_INTX_MASK, &assigned_dev);
+}
+
 #ifdef KVM_CAP_ASSIGN_DEV_IRQ
 int kvm_assign_irq(KVMState *s, struct kvm_assigned_irq *assigned_irq)
 {
qemu-kvm.h +2 -0
index 3c4f023..2b23daf 100644
@@ -114,6 +114,8 @@ int kvm_assign_irq(KVMState *s, struct kvm_assigned_irq *assigned_irq);
  */
 int kvm_deassign_irq(KVMState *s, struct kvm_assigned_irq *assigned_irq);
 
+int kvm_device_intx_set_mask(KVMState *s, uint32_t dev_id, bool masked);
+
 /*!
  * \brief Notifies host kernel about a PCI device to be deassigned from a guest
  *
Signature · 1 line
1.7.3.4
#7 lore raw

Re: [PATCH v3 2/3] pci-assign: Use PCI-2.3-based shared legacy interrupts

Quoted 38 lines · Jan Kiszka

Enable the new KVM feature that allows legacy interrupt sharing for PCI-2.3-compliant devices. This requires to synchronize any guest change of the INTx mask bit to the kernel.

The feature is controlled by the property 'share_intx' and is off by default for now.

Signed-off-by Jan Kiszka <[email protected]>

---

Changes in v3:
- fixed nit :)

 hw/device-assignment.c |   24 ++++++++++++++++++++++++
 hw/device-assignment.h |   10 ++++++----
 qemu-kvm.c             |    9 +++++++++
 qemu-kvm.h             |    2 ++
 4 files changed, 41 insertions(+), 4 deletions(-)
hw/device-assignment.c +7 -0
index a5f1abb..e2a8479 100644
@@ -782,6 +782,13 @@ static int assign_device(AssignedDevice *dev)
                 "cause host memory corruption if the device issues DMA write "
                 "requests!\n");
     }
+    if (dev->features & ASSIGNED_DEVICE_SHARE_INTX_MASK) {
+        assigned_dev_data.flags |= KVM_DEV_ASSIGN_PCI_2_3;
+
+        /* hide host-side INTx masking from the guest */
+        dev->emulate_config_read[PCI_COMMAND + 1] |=
+            PCI_COMMAND_INTX_DISABLE >> 8;
+    }
 
     r = kvm_assign_pci_device(kvm_state, &assigned_dev_data);
     if (r < 0) {

hw/device-assignment.c static void assigned_dev_pci_write_config(PCIDevice *pci_dev, uint32_t address, · line 1137

Quoted 10 lines
hw/device-assignment.c +3 -0
@@ -1121,10 +1128,25 @@ static void assigned_dev_pci_write_config(PCIDevice *pci_dev, uint32_t address,
                                           uint32_t val, int len)
 {
     AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+    uint16_t old_cmd = pci_get_word(pci_dev->config + PCI_COMMAND);
     uint32_t emulate_mask, full_emulation_mask;
+    int ret;
 
     pci_default_write_config(pci_dev, address, val, len);
 
+    if (range_covers_byte(address, len, PCI_COMMAND + 1)) {

Hmm, now that I've acked this... shouldn't we have a feature check here to avoid the ioctl when share_intx=false or when the host kernel doesn't support this? Thanks,

Alex

Quoted 79 lines
hw/device-assignment.c +14 -0
+        bool intx_masked = (pci_get_word(pci_dev->config + PCI_COMMAND) &
+                            PCI_COMMAND_INTX_DISABLE);
+
+        if (intx_masked != !!(old_cmd & PCI_COMMAND_INTX_DISABLE)) {
+            ret = kvm_device_intx_set_mask(kvm_state,
+                                           calc_assigned_dev_id(assigned_dev),
+                                           intx_masked);
+            if (ret) {
+                perror("assigned_dev_pci_write_config: set intx mask");
+            }
+        }
+    }
     if (assigned_dev->cap.available & ASSIGNED_DEVICE_CAP_MSI) {
         if (range_covers_byte(address, len,
                               pci_dev->msi_cap + PCI_MSI_FLAGS)) {

@@ -1748,6 +1770,8 @@ static Property da_properties[] =
                    ASSIGNED_DEVICE_USE_IOMMU_BIT, true),
     DEFINE_PROP_BIT("prefer_msi", AssignedDevice, features,
                    ASSIGNED_DEVICE_PREFER_MSI_BIT, true),
+    DEFINE_PROP_BIT("share_intx", AssignedDevice, features,
+                    ASSIGNED_DEVICE_SHARE_INTX_BIT, false),
     DEFINE_PROP_INT32("bootindex", AssignedDevice, bootindex, -1),
     DEFINE_PROP_STRING("configfd", AssignedDevice, configfd_name),
     DEFINE_PROP_END_OF_LIST(),
hw/device-assignment.h +6 -4
index b4bcfa6..5d271d5 100644
@@ -74,11 +74,13 @@ typedef struct {
     PCIRegion *region;
 } AssignedDevRegion;
 
-#define ASSIGNED_DEVICE_USE_IOMMU_BIT	0
-#define ASSIGNED_DEVICE_PREFER_MSI_BIT	1
+#define ASSIGNED_DEVICE_USE_IOMMU_BIT   0
+#define ASSIGNED_DEVICE_PREFER_MSI_BIT  1
+#define ASSIGNED_DEVICE_SHARE_INTX_BIT  2
 
-#define ASSIGNED_DEVICE_USE_IOMMU_MASK	(1 << ASSIGNED_DEVICE_USE_IOMMU_BIT)
-#define ASSIGNED_DEVICE_PREFER_MSI_MASK	(1 << ASSIGNED_DEVICE_PREFER_MSI_BIT)
+#define ASSIGNED_DEVICE_USE_IOMMU_MASK  (1 << ASSIGNED_DEVICE_USE_IOMMU_BIT)
+#define ASSIGNED_DEVICE_PREFER_MSI_MASK (1 << ASSIGNED_DEVICE_PREFER_MSI_BIT)
+#define ASSIGNED_DEVICE_SHARE_INTX_MASK (1 << ASSIGNED_DEVICE_SHARE_INTX_BIT)
 
 typedef struct {
     uint32_t addr_lo;
qemu-kvm.c +9 -0
index 09a35f0..2047ebb 100644
@@ -54,6 +54,15 @@ static int kvm_old_assign_irq(KVMState *s,
     return kvm_vm_ioctl(s, KVM_ASSIGN_IRQ, assigned_irq);
 }
 
+int kvm_device_intx_set_mask(KVMState *s, uint32_t dev_id, bool masked)
+{
+    struct kvm_assigned_pci_dev assigned_dev;
+
+    assigned_dev.assigned_dev_id = dev_id;
+    assigned_dev.flags = masked ? KVM_DEV_ASSIGN_MASK_INTX : 0;
+    return kvm_vm_ioctl(s, KVM_ASSIGN_SET_INTX_MASK, &assigned_dev);
+}
+
 #ifdef KVM_CAP_ASSIGN_DEV_IRQ
 int kvm_assign_irq(KVMState *s, struct kvm_assigned_irq *assigned_irq)
 {
qemu-kvm.h +2 -0
index 3c4f023..2b23daf 100644
@@ -114,6 +114,8 @@ int kvm_assign_irq(KVMState *s, struct kvm_assigned_irq *assigned_irq);
  */
 int kvm_deassign_irq(KVMState *s, struct kvm_assigned_irq *assigned_irq);
 
+int kvm_device_intx_set_mask(KVMState *s, uint32_t dev_id, bool masked);
+
 /*!
  * \brief Notifies host kernel about a PCI device to be deassigned from a guest
  *
author
#8 lore raw

Re: [PATCH v3 2/3] pci-assign: Use PCI-2.3-based shared legacy interrupts

Quoted 46 lines · Alex Williamson
Quoted 45 lines · Jan Kiszka

Enable the new KVM feature that allows legacy interrupt sharing for PCI-2.3-compliant devices. This requires to synchronize any guest change of the INTx mask bit to the kernel.

The feature is controlled by the property 'share_intx' and is off by default for now.

Signed-off-by Jan Kiszka <[email protected]>

---

Changes in v3:
- fixed nit :)

 hw/device-assignment.c |   24 ++++++++++++++++++++++++
 hw/device-assignment.h |   10 ++++++----
 qemu-kvm.c             |    9 +++++++++
 qemu-kvm.h             |    2 ++
 4 files changed, 41 insertions(+), 4 deletions(-)
hw/device-assignment.c +9 -0
index a5f1abb..e2a8479 100644
@@ -782,6 +782,13 @@ static int assign_device(AssignedDevice *dev)
                 "cause host memory corruption if the device issues DMA write "
                 "requests!\n");
     }
+    if (dev->features & ASSIGNED_DEVICE_SHARE_INTX_MASK) {
+        assigned_dev_data.flags |= KVM_DEV_ASSIGN_PCI_2_3;
+
+        /* hide host-side INTx masking from the guest */
+        dev->emulate_config_read[PCI_COMMAND + 1] |=
+            PCI_COMMAND_INTX_DISABLE >> 8;
+    }
 
     r = kvm_assign_pci_device(kvm_state, &assigned_dev_data);
     if (r < 0) {

@@ -1121,10 +1128,25 @@ static void assigned_dev_pci_write_config(PCIDevice *pci_dev, uint32_t address,
                                           uint32_t val, int len)
 {
     AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+    uint16_t old_cmd = pci_get_word(pci_dev->config + PCI_COMMAND);
     uint32_t emulate_mask, full_emulation_mask;
+    int ret;
Quoted 7 lines

hw/device-assignment.c static void assigned_dev_pci_write_config(PCIDevice *pci_dev, uint32_t address, · line 1137

Quoted 3 lines
hw/device-assignment.c +1 -0
     pci_default_write_config(pci_dev, address, val, len);
 
+    if (range_covers_byte(address, len, PCI_COMMAND + 1)) {

Hmm, now that I've acked this... shouldn't we have a feature check here to avoid the ioctl when share_intx=false or when the host kernel doesn't support this? Thanks,

Hmm, I thought I handled this, somehow. But it's not there. v4 follows...

Thanks,
Jan

Signature · 2 lines
Siemens AG, Corporate Technology, CT T DE IT 1
Corporate Competence Center Embedded Linux
author
#9 lore raw

Re: [PATCH v3 2/3] pci-assign: Use PCI-2.3-based shared legacy interrupts

Quoted 49 lines · Jan Kiszka
Quoted 48 lines · Alex Williamson
Quoted 47 lines · Jan Kiszka

Enable the new KVM feature that allows legacy interrupt sharing for PCI-2.3-compliant devices. This requires to synchronize any guest change of the INTx mask bit to the kernel.

The feature is controlled by the property 'share_intx' and is off by default for now.

Signed-off-by Jan Kiszka <[email protected]>

---

Changes in v3:
- fixed nit :)

 hw/device-assignment.c |   24 ++++++++++++++++++++++++
 hw/device-assignment.h |   10 ++++++----
 qemu-kvm.c             |    9 +++++++++
 qemu-kvm.h             |    2 ++
 4 files changed, 41 insertions(+), 4 deletions(-)
hw/device-assignment.c +9 -0
index a5f1abb..e2a8479 100644
@@ -782,6 +782,13 @@ static int assign_device(AssignedDevice *dev)
                 "cause host memory corruption if the device issues DMA write "
                 "requests!\n");
     }
+    if (dev->features & ASSIGNED_DEVICE_SHARE_INTX_MASK) {
+        assigned_dev_data.flags |= KVM_DEV_ASSIGN_PCI_2_3;
+
+        /* hide host-side INTx masking from the guest */
+        dev->emulate_config_read[PCI_COMMAND + 1] |=
+            PCI_COMMAND_INTX_DISABLE >> 8;
+    }
 
     r = kvm_assign_pci_device(kvm_state, &assigned_dev_data);
     if (r < 0) {

@@ -1121,10 +1128,25 @@ static void assigned_dev_pci_write_config(PCIDevice *pci_dev, uint32_t address,
                                           uint32_t val, int len)
 {
     AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+    uint16_t old_cmd = pci_get_word(pci_dev->config + PCI_COMMAND);
     uint32_t emulate_mask, full_emulation_mask;
+    int ret;
 
     pci_default_write_config(pci_dev, address, val, len);
Quoted 7 lines
Quoted 5 lines

hw/device-assignment.c static void assigned_dev_pci_write_config(PCIDevice *pci_dev, uint32_t address, · line 1137

Quoted 1 line
hw/device-assignment.c +1 -0
+    if (range_covers_byte(address, len, PCI_COMMAND + 1)) {

Hmm, now that I've acked this... shouldn't we have a feature check here to avoid the ioctl when share_intx=false or when the host kernel doesn't support this? Thanks,

Hmm, I thought I handled this, somehow. But it's not there. v4 follows...

To be precise: We only need to worry about false positive error messages in the absence of KVM_CAP_PCI_2_3. Devices not supporting INTx sharing will either refuse to work due to IRQ conflicts, or we will map INTx masking on IRQ line masking.

Jan

Signature · 2 lines
Siemens AG, Corporate Technology, CT T DE IT 1
Corporate Competence Center Embedded Linux
author
#10 lore raw

PATCH 3/3 pci_assign: Flip defaults of prefer_msi and share_intx

In reply to Jan Kiszka

INTx sharing is a bit more expensive than exclusive host interrupts, but this channel is not supposed to be used for high-performance scenarios anyway. Modern devices support MSI/MSI-X and do not depend on using INTx under critical workload, real old devices do not support INTx sharing anyway.

For those in the middle, the user experience is much better if they just work even when IRQ sharing is required. If there is nothing to share, share_intx=off can still be applied as tuning parameter.

With INTx sharing as default, the primary reason for prefer_msi=on is gone. Make it default off, specifically as it is known to cause troubles with devices that have incomplete/broken MSI support or otherwise stumble if host IRQ configuration does not match guest driver expectation.

Signed-off-by Jan Kiszka <[email protected]>
---
 hw/device-assignment.c |    4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)
hw/device-assignment.c +2 -2
index 45e7913..0b2dfba 100644
@@ -1766,9 +1766,9 @@ static Property da_properties[] =
     DEFINE_PROP_BIT("iommu", AssignedDevice, features,
                    ASSIGNED_DEVICE_USE_IOMMU_BIT, true),
     DEFINE_PROP_BIT("prefer_msi", AssignedDevice, features,
-                   ASSIGNED_DEVICE_PREFER_MSI_BIT, true),
+                    ASSIGNED_DEVICE_PREFER_MSI_BIT, false),
     DEFINE_PROP_BIT("share_intx", AssignedDevice, features,
-                    ASSIGNED_DEVICE_SHARE_INTX_BIT, false),
+                    ASSIGNED_DEVICE_SHARE_INTX_BIT, true),
     DEFINE_PROP_INT32("bootindex", AssignedDevice, bootindex, -1),
     DEFINE_PROP_STRING("configfd", AssignedDevice, configfd_name),
     DEFINE_PROP_END_OF_LIST(),
Signature · 1 line
1.7.3.4
Acked-by on the v1 series
#11 lore raw
In reply to Jan Kiszka
Quoted 11 lines · Jan Kiszka

This enables support for the host IRQ sharing support via PCI-2.3 interrupt masking. See the patches for details.

Patch 1 could be considered for uq/master although we have no use case for the changes in upstream yet.

Jan Kiszka (3):
kvm: Update kernel headers against kvm.git
pci-assign: Use PCI-2.3-based shared legacy interrupts
pci_assign: Flip defaults of prefer_msi and share_intx

Quoted 8 lines
 hw/device-assignment.c       |   23 ++++++++++++++++++++++-
 hw/device-assignment.h       |   10 ++++++----
 linux-headers/asm-s390/kvm.h |    2 ++
 linux-headers/linux/kvm.h    |    6 ++++++
 qemu-kvm.c                   |    9 +++++++++
 qemu-kvm.h                   |    2 ++
 6 files changed, 47 insertions(+), 5 deletions(-)

For series w/ v3 2/3:

Acked-by Alex Williamson <[email protected]>