qemu-kvm: pci-assign: Host IRQ sharing suppport
v4 cover qemu-kvm: pci-assign: Host IRQ sharing suppport
This enables support for the host IRQ sharing support via PCI-2.3 interrupt masking. See the patches for details.
Patch 1 & 3 could be considered for uq/master although we have no use case for the changes in upstream yet.
Changes in v4:
- add patch for kvm_has_intx_set_mask
- do not issue unsupported IOCTLs when the host kernel does not support INTx mask control
Jan Kiszka (4):
kvm: Update kernel headers against kvm.git
kvm: Introduce kvm_has_intx_set_mask
pci-assign: Use PCI-2.3-based shared legacy interrupts
pci_assign: Flip defaults of prefer_msi and share_intx
hw/device-assignment.c | 27 ++++++++++++++++++++++++++- hw/device-assignment.h | 10 ++++++---- kvm-all.c | 8 ++++++++ kvm.h | 1 + linux-headers/asm-s390/kvm.h | 2 ++ linux-headers/linux/kvm.h | 6 ++++++ qemu-kvm.c | 9 +++++++++ qemu-kvm.h | 2 ++ 8 files changed, 60 insertions(+), 5 deletions(-)
Signature · 1 line
1.7.3.4
PATCH v4 1/4 kvm: Update kernel headers against kvm.git
Required for INTx masking / host IRQ sharing support of assigned devices.
Corresponding kvm.git hash: 186195928e
--- linux-headers/asm-s390/kvm.h | 2 ++ linux-headers/linux/kvm.h | 6 ++++++ 2 files changed, 8 insertions(+), 0 deletions(-)
linux-headers/asm-s390/kvm.h +2 -0
@@ -44,10 +44,12 @@ struct kvm_guest_debug_arch {#define KVM_SYNC_PREFIX (1UL << 0)#define KVM_SYNC_GPRS (1UL << 1)#define KVM_SYNC_ACRS (1UL << 2) +#define KVM_SYNC_CRS (1UL << 3)/* definition of registers in kvm_run */ struct kvm_sync_regs {__u64 prefix;/* prefix register */ __u64 gprs[16 ];/* general purpose registers */ __u32 acrs[16 ];/* access registers */ +__u64 crs[16 ];/* control registers */ };#endif
linux-headers/linux/kvm.h +6 -0
@@ -588,6 +588,7 @@ struct kvm_ppc_pvinfo {#define KVM_CAP_TSC_DEADLINE_TIMER 72#define KVM_CAP_S390_UCONTROL 73#define KVM_CAP_SYNC_REGS 74 +#define KVM_CAP_PCI_2_3 75#ifdef KVM_CAP_IRQ_ROUTING
@@ -784,6 +785,9 @@ struct kvm_s390_ucas_mapping {/* Available with KVM_CAP_TSC_CONTROL */ #define KVM_SET_TSC_KHZ _IO(KVMIO, 0xa2)#define KVM_GET_TSC_KHZ _IO(KVMIO, 0xa3) +/* Available with KVM_CAP_PCI_2_3 */ +#define KVM_ASSIGN_SET_INTX_MASK _IOW(KVMIO, 0xa4, \ + struct kvm_assigned_pci_dev)/* * ioctls for vcpu fds
@@ -857,6 +861,8 @@ struct kvm_s390_ucas_mapping {#define KVM_SET_ONE_REG _IOW(KVMIO, 0xac, struct kvm_one_reg)#define KVM_DEV_ASSIGN_ENABLE_IOMMU (1 << 0) +#define KVM_DEV_ASSIGN_PCI_2_3 (1 << 1) +#define KVM_DEV_ASSIGN_MASK_INTX (1 << 2)struct kvm_assigned_pci_dev {__u32 assigned_dev_id;
Signature · 1 line
1.7.3.4
Re: [PATCH v4 1/4] kvm: Update kernel headers against kvm.git
Quoted 6 lines · Jan Kiszka
Required for INTx masking / host IRQ sharing support of assigned devices.
Corresponding kvm.git hash:
186195928eSigned-off-by Jan Kiszka <[email protected]>
Quoted 55 lines
--- linux-headers/asm-s390/kvm.h | 2 ++ linux-headers/linux/kvm.h | 6 ++++++ 2 files changed, 8 insertions(+), 0 deletions(-)linux-headers/asm-s390/kvm.h +2
-0@@ -44,10 +44,12 @@ struct kvm_guest_debug_arch {#define KVM_SYNC_PREFIX (1UL << 0)#define KVM_SYNC_GPRS (1UL << 1)#define KVM_SYNC_ACRS (1UL << 2) +#define KVM_SYNC_CRS (1UL << 3)/* definition of registers in kvm_run */ struct kvm_sync_regs {__u64 prefix;/* prefix register */ __u64 gprs[16 ];/* general purpose registers */ __u32 acrs[16 ];/* access registers */ +__u64 crs[16 ];/* control registers */ };#endif linux-headers/linux/kvm.h +6
-0@@ -588,6 +588,7 @@ struct kvm_ppc_pvinfo {#define KVM_CAP_TSC_DEADLINE_TIMER 72#define KVM_CAP_S390_UCONTROL 73#define KVM_CAP_SYNC_REGS 74 +#define KVM_CAP_PCI_2_3 75#ifdef KVM_CAP_IRQ_ROUTING @@ -784,6 +785,9 @@ struct kvm_s390_ucas_mapping {/* Available with KVM_CAP_TSC_CONTROL */ #define KVM_SET_TSC_KHZ _IO(KVMIO, 0xa2)#define KVM_GET_TSC_KHZ _IO(KVMIO, 0xa3) +/* Available with KVM_CAP_PCI_2_3 */ +#define KVM_ASSIGN_SET_INTX_MASK _IOW(KVMIO, 0xa4, \ + struct kvm_assigned_pci_dev)/* * ioctls for vcpu fds @@ -857,6 +861,8 @@ struct kvm_s390_ucas_mapping {#define KVM_SET_ONE_REG _IOW(KVMIO, 0xac, struct kvm_one_reg)#define KVM_DEV_ASSIGN_ENABLE_IOMMU (1 << 0) +#define KVM_DEV_ASSIGN_PCI_2_3 (1 << 1) +#define KVM_DEV_ASSIGN_MASK_INTX (1 << 2)struct kvm_assigned_pci_dev {__u32 assigned_dev_id;Signature · 1 line
1.7.3.4
PATCH v4 2/4 kvm: Introduce kvm_has_intx_set_mask
In reply to Jan KiszkaWill be used by PCI device assignment code.
--- kvm-all.c | 8 ++++++++ kvm.h | 1 + 2 files changed, 9 insertions(+), 0 deletions(-)
kvm-all.c +8 -0
@@ -78,6 +78,7 @@ struct KVMStateint xsave, xcrs;int many_ioeventfds;int pit_state2; +int intx_set_mask;int irqchip_inject_ioctl;#ifdef KVM_CAP_IRQ_ROUTING struct kvm_irq_routing *irq_routes;
@@ -960,6 +961,8 @@ int kvm_init(void) s->pit_state2 =kvm_check_extension (s,KVM_CAP_PIT_STATE2 );#endif + s->intx_set_mask =kvm_check_extension (s,KVM_CAP_PCI_2_3 ); + ret =kvm_arch_init (s);if (ret <0 ) { goto err;
@@ -1319,6 +1322,11 @@ int kvm_has_gsi_routing(void)#endif } +int kvm_has_intx_set_mask (void ) +{ +return kvm_state->intx_set_mask; +} +int kvm_allows_irq0_override (void ) {return !kvm_irqchip_in_kernel () ||kvm_has_gsi_routing ();
kvm.h +1 -0
@@ -57,6 +57,7 @@ int kvm_has_xcrs(void);int kvm_has_many_ioeventfds (void );int kvm_has_pit_state2 (void );int kvm_has_gsi_routing (void ); +int kvm_has_intx_set_mask (void );int kvm_allows_irq0_override (void );
Signature · 1 line
1.7.3.4
Re: [PATCH v4 2/4] kvm: Introduce kvm_has_intx_set_mask
Quoted 3 lines · Jan Kiszka
Will be used by PCI device assignment code.
Signed-off-by Jan Kiszka <[email protected]>
Quoted 52 lines
--- kvm-all.c | 8 ++++++++ kvm.h | 1 + 2 files changed, 9 insertions(+), 0 deletions(-)kvm-all.c +8
-0@@ -78,6 +78,7 @@ struct KVMStateint xsave, xcrs;int many_ioeventfds;int pit_state2; +int intx_set_mask;int irqchip_inject_ioctl;#ifdef KVM_CAP_IRQ_ROUTING struct kvm_irq_routing *irq_routes;@@ -960,6 +961,8 @@ int kvm_init(void) s->pit_state2 =kvm_check_extension (s,KVM_CAP_PIT_STATE2 );#endif + s->intx_set_mask =kvm_check_extension (s,KVM_CAP_PCI_2_3 ); + ret =kvm_arch_init (s);if (ret <0 ) { goto err;@@ -1319,6 +1322,11 @@ int kvm_has_gsi_routing(void)#endif } +int kvm_has_intx_set_mask (void ) +{ +return kvm_state->intx_set_mask; +} +int kvm_allows_irq0_override (void ) {return !kvm_irqchip_in_kernel () ||kvm_has_gsi_routing ();kvm.h +1
-0@@ -57,6 +57,7 @@ int kvm_has_xcrs(void);int kvm_has_many_ioeventfds (void );int kvm_has_pit_state2 (void );int kvm_has_gsi_routing (void ); +int kvm_has_intx_set_mask (void );int kvm_allows_irq0_override (void );Signature · 1 line
1.7.3.4
PATCH v4 3/4 pci-assign: Use PCI-2.3-based shared legacy interrupts
In reply to Jan KiszkaEnable the new KVM feature that allows legacy interrupt sharing for PCI-2.3-compliant devices. This requires to synchronize any guest change of the INTx mask bit to the kernel.
The feature is controlled by the property 'share_intx' and is off by default for now.
--- hw/device-assignment.c | 25 +++++++++++++++++++++++++ hw/device-assignment.h | 10 ++++++---- qemu-kvm.c | 9 +++++++++ qemu-kvm.h | 2 ++ 4 files changed, 42 insertions(+), 4 deletions(-)
hw/device-assignment.c +25 -0
@@ -782,6 +782,13 @@ static int assign_device(AssignedDevice *dev)"cause host memory corruption if the device issues DMA write " "requests!\n"); } +if (dev->features &ASSIGNED_DEVICE_SHARE_INTX_MASK ) { + assigned_dev_data.flags |=KVM_DEV_ASSIGN_PCI_2_3 ; + +/* hide host-side INTx masking from the guest */ + dev->emulate_config_read[PCI_COMMAND +1 ] |= +PCI_COMMAND_INTX_DISABLE >>8 ; + } r =kvm_assign_pci_device (kvm_state, &assigned_dev_data);if (r <0 ) {
@@ -1121,10 +1128,26 @@ static void assigned_dev_pci_write_config(PCIDevice *pci_dev, uint32_t address, uint32_t val, int len) {AssignedDevice *assigned_dev =DO_UPCAST (AssignedDevice, dev, pci_dev); +uint16_t old_cmd =pci_get_word (pci_dev->config +PCI_COMMAND );uint32_t emulate_mask, full_emulation_mask; +int ret;pci_default_write_config (pci_dev, address, val, len); +if (kvm_has_intx_set_mask () && +range_covers_byte (address, len,PCI_COMMAND +1 )) { +bool intx_masked = (pci_get_word (pci_dev->config +PCI_COMMAND ) & +PCI_COMMAND_INTX_DISABLE ); + +if (intx_masked != !!(old_cmd &PCI_COMMAND_INTX_DISABLE )) { + ret =kvm_device_intx_set_mask (kvm_state, +calc_assigned_dev_id (assigned_dev), + intx_masked); +if (ret) { +perror ("assigned_dev_pci_write_config: set intx mask" ); + } + } + }if (assigned_dev->cap.available &ASSIGNED_DEVICE_CAP_MSI ) {if (range_covers_byte (address, len, pci_dev->msi_cap +PCI_MSI_FLAGS )) {
@@ -1748,6 +1771,8 @@ static Property da_properties[] = ASSIGNED_DEVICE_USE_IOMMU_BIT, true),DEFINE_PROP_BIT ("prefer_msi" , AssignedDevice, features,ASSIGNED_DEVICE_PREFER_MSI_BIT , true), +DEFINE_PROP_BIT ("share_intx" , AssignedDevice, features, +ASSIGNED_DEVICE_SHARE_INTX_BIT , false),DEFINE_PROP_INT32 ("bootindex" , AssignedDevice, bootindex,-1 ),DEFINE_PROP_STRING ("configfd" , AssignedDevice, configfd_name), DEFINE_PROP_END_OF_LIST(),
hw/device-assignment.h +6 -4
@@ -74,11 +74,13 @@ typedef struct {PCIRegion *region; } AssignedDevRegion; -#define ASSIGNED_DEVICE_USE_IOMMU_BIT 0 -#define ASSIGNED_DEVICE_PREFER_MSI_BIT 1 +#define ASSIGNED_DEVICE_USE_IOMMU_BIT 0 +#define ASSIGNED_DEVICE_PREFER_MSI_BIT 1 +#define ASSIGNED_DEVICE_SHARE_INTX_BIT 2 -#define ASSIGNED_DEVICE_USE_IOMMU_MASK (1 << ASSIGNED_DEVICE_USE_IOMMU_BIT) -#define ASSIGNED_DEVICE_PREFER_MSI_MASK (1 << ASSIGNED_DEVICE_PREFER_MSI_BIT) +#define ASSIGNED_DEVICE_USE_IOMMU_MASK (1 << ASSIGNED_DEVICE_USE_IOMMU_BIT) +#define ASSIGNED_DEVICE_PREFER_MSI_MASK (1 << ASSIGNED_DEVICE_PREFER_MSI_BIT) +#define ASSIGNED_DEVICE_SHARE_INTX_MASK (1 << ASSIGNED_DEVICE_SHARE_INTX_BIT) typedef struct { uint32_t addr_lo;
qemu-kvm.c +9 -0
@@ -54,6 +54,15 @@ static int kvm_old_assign_irq(KVMState *s,return kvm_vm_ioctl (s,KVM_ASSIGN_IRQ , assigned_irq); } +int kvm_device_intx_set_mask (KVMState *s,uint32_t dev_id,bool masked) +{ +struct kvm_assigned_pci_dev assigned_dev; + + assigned_dev.assigned_dev_id = dev_id; + assigned_dev.flags = masked ?KVM_DEV_ASSIGN_MASK_INTX :0 ; +return kvm_vm_ioctl (s,KVM_ASSIGN_SET_INTX_MASK , &assigned_dev); +} +#ifdef KVM_CAP_ASSIGN_DEV_IRQ int kvm_assign_irq (KVMState *s,struct kvm_assigned_irq *assigned_irq) {
qemu-kvm.h +2 -0
@@ -114,6 +114,8 @@ int kvm_assign_irq(KVMState *s, struct kvm_assigned_irq *assigned_irq);*/ int kvm_deassign_irq (KVMState *s,struct kvm_assigned_irq *assigned_irq); +int kvm_device_intx_set_mask (KVMState *s,uint32_t dev_id,bool masked); +/*! * \brief Notifies host kernel about a PCI device to be deassigned from a guest *
Signature · 1 line
1.7.3.4
Re: [PATCH v4 3/4] pci-assign: Use PCI-2.3-based shared legacy interrupts
Quoted 20 lines · Jan Kiszka
Enable the new KVM feature that allows legacy interrupt sharing for PCI-2.3-compliant devices. This requires to synchronize any guest change of the INTx mask bit to the kernel.
The feature is controlled by the property '
share_intx' and is off by default for now.Signed-off-by Jan Kiszka <[email protected]>--- hw/device-assignment.c | 25 +++++++++++++++++++++++++ hw/device-assignment.h | 10 ++++++---- qemu-kvm.c | 9 +++++++++ qemu-kvm.h | 2 ++ 4 files changed, 42 insertions(+), 4 deletions(-)hw/device-assignment.c +0
-0
hw/device-assignment.c static int assign_device(AssignedDevice *dev) · line 791
Quoted 10 lines
hw/device-assignment.c +7
-0@@ -782,6 +782,13 @@ static int assign_device(AssignedDevice *dev)"cause host memory corruption if the device issues DMA write " "requests!\n"); } +if (dev->features &ASSIGNED_DEVICE_SHARE_INTX_MASK ) { + assigned_dev_data.flags |=KVM_DEV_ASSIGN_PCI_2_3 ; + +/* hide host-side INTx masking from the guest */ + dev->emulate_config_read[PCI_COMMAND +1 ] |= +PCI_COMMAND_INTX_DISABLE >>8 ; + }
I think this also needs a kvm_has_intx_set_mask() check or else we're emulating pci2.3 intx disable support without actually backing it. Thanks,
Alex
Quoted 94 lines
hw/device-assignment.c +18
-0r =kvm_assign_pci_device (kvm_state, &assigned_dev_data);if (r <0 ) {@@ -1121,10 +1128,26 @@ static void assigned_dev_pci_write_config(PCIDevice *pci_dev, uint32_t address, uint32_t val, int len) {AssignedDevice *assigned_dev =DO_UPCAST (AssignedDevice, dev, pci_dev); +uint16_t old_cmd =pci_get_word (pci_dev->config +PCI_COMMAND );uint32_t emulate_mask, full_emulation_mask; +int ret;pci_default_write_config (pci_dev, address, val, len); +if (kvm_has_intx_set_mask () && +range_covers_byte (address, len,PCI_COMMAND +1 )) { +bool intx_masked = (pci_get_word (pci_dev->config +PCI_COMMAND ) & +PCI_COMMAND_INTX_DISABLE ); + +if (intx_masked != !!(old_cmd &PCI_COMMAND_INTX_DISABLE )) { + ret =kvm_device_intx_set_mask (kvm_state, +calc_assigned_dev_id (assigned_dev), + intx_masked); +if (ret) { +perror ("assigned_dev_pci_write_config: set intx mask" ); + } + } + }if (assigned_dev->cap.available &ASSIGNED_DEVICE_CAP_MSI ) {if (range_covers_byte (address, len, pci_dev->msi_cap +PCI_MSI_FLAGS )) {@@ -1748,6 +1771,8 @@ static Property da_properties[] = ASSIGNED_DEVICE_USE_IOMMU_BIT, true),DEFINE_PROP_BIT ("prefer_msi" , AssignedDevice, features,ASSIGNED_DEVICE_PREFER_MSI_BIT , true), +DEFINE_PROP_BIT ("share_intx" , AssignedDevice, features, +ASSIGNED_DEVICE_SHARE_INTX_BIT , false),DEFINE_PROP_INT32 ("bootindex" , AssignedDevice, bootindex,-1 ),DEFINE_PROP_STRING ("configfd" , AssignedDevice, configfd_name), DEFINE_PROP_END_OF_LIST(),hw/device-assignment.h +6
-4@@ -74,11 +74,13 @@ typedef struct {PCIRegion *region; } AssignedDevRegion; -#define ASSIGNED_DEVICE_USE_IOMMU_BIT 0 -#define ASSIGNED_DEVICE_PREFER_MSI_BIT 1 +#define ASSIGNED_DEVICE_USE_IOMMU_BIT 0 +#define ASSIGNED_DEVICE_PREFER_MSI_BIT 1 +#define ASSIGNED_DEVICE_SHARE_INTX_BIT 2 -#define ASSIGNED_DEVICE_USE_IOMMU_MASK (1 << ASSIGNED_DEVICE_USE_IOMMU_BIT) -#define ASSIGNED_DEVICE_PREFER_MSI_MASK (1 << ASSIGNED_DEVICE_PREFER_MSI_BIT) +#define ASSIGNED_DEVICE_USE_IOMMU_MASK (1 << ASSIGNED_DEVICE_USE_IOMMU_BIT) +#define ASSIGNED_DEVICE_PREFER_MSI_MASK (1 << ASSIGNED_DEVICE_PREFER_MSI_BIT) +#define ASSIGNED_DEVICE_SHARE_INTX_MASK (1 << ASSIGNED_DEVICE_SHARE_INTX_BIT) typedef struct { uint32_t addr_lo;qemu-kvm.c +9
-0@@ -54,6 +54,15 @@ static int kvm_old_assign_irq(KVMState *s,return kvm_vm_ioctl (s,KVM_ASSIGN_IRQ , assigned_irq); } +int kvm_device_intx_set_mask (KVMState *s,uint32_t dev_id,bool masked) +{ +struct kvm_assigned_pci_dev assigned_dev; + + assigned_dev.assigned_dev_id = dev_id; + assigned_dev.flags = masked ?KVM_DEV_ASSIGN_MASK_INTX :0 ; +return kvm_vm_ioctl (s,KVM_ASSIGN_SET_INTX_MASK , &assigned_dev); +} +#ifdef KVM_CAP_ASSIGN_DEV_IRQ int kvm_assign_irq (KVMState *s,struct kvm_assigned_irq *assigned_irq) {qemu-kvm.h +2
-0@@ -114,6 +114,8 @@ int kvm_assign_irq(KVMState *s, struct kvm_assigned_irq *assigned_irq);*/ int kvm_deassign_irq (KVMState *s,struct kvm_assigned_irq *assigned_irq); +int kvm_device_intx_set_mask (KVMState *s,uint32_t dev_id,bool masked); +/*! * \brief Notifies host kernel about a PCI device to be deassigned from a guest *
Re: [PATCH v4 3/4] pci-assign: Use PCI-2.3-based shared legacy interrupts
Quoted 26 lines · Alex Williamson
Quoted 25 lines · Jan Kiszka
Enable the new KVM feature that allows legacy interrupt sharing for PCI-2.3-compliant devices. This requires to synchronize any guest change of the INTx mask bit to the kernel.
The feature is controlled by the property '
share_intx' and is off by default for now.Signed-off-by Jan Kiszka <[email protected]>--- hw/device-assignment.c | 25 +++++++++++++++++++++++++ hw/device-assignment.h | 10 ++++++---- qemu-kvm.c | 9 +++++++++ qemu-kvm.h | 2 ++ 4 files changed, 42 insertions(+), 4 deletions(-)hw/device-assignment.c +2
-0@@ -782,6 +782,13 @@ static int assign_device(AssignedDevice *dev)"cause host memory corruption if the device issues DMA write " "requests!\n"); } +if (dev->features &ASSIGNED_DEVICE_SHARE_INTX_MASK ) { + assigned_dev_data.flags |=KVM_DEV_ASSIGN_PCI_2_3 ;
Quoted 8 lines
hw/device-assignment.c static int assign_device(AssignedDevice *dev) · line 791
Quoted 5 lines
hw/device-assignment.c +5
-0+ +/* hide host-side INTx masking from the guest */ + dev->emulate_config_read[PCI_COMMAND +1 ] |= +PCI_COMMAND_INTX_DISABLE >>8 ; + }I think this also needs a
kvm_has_intx_set_mask()check or else we're emulating pci2.3 intx disable support without actually backing it.
Hmm... true.
Jan
Signature · 2 lines
Siemens AG, Corporate Technology, CT T DE IT 1 Corporate Competence Center Embedded Linux
PATCH v5 3/4 pci-assign: Use PCI-2.3-based shared legacy interrupts
Enable the new KVM feature that allows legacy interrupt sharing for PCI-2.3-compliant devices. This requires to synchronize any guest change of the INTx mask bit to the kernel.
The feature is controlled by the property 'share_intx' and is off by default for now.
---
Changes in v5:
- Only emulate INTx disable bit if there is kernel support
hw/device-assignment.c | 26 ++++++++++++++++++++++++++ hw/device-assignment.h | 10 ++++++---- qemu-kvm.c | 9 +++++++++ qemu-kvm.h | 2 ++ 4 files changed, 43 insertions(+), 4 deletions(-)
hw/device-assignment.c +26 -0
@@ -782,6 +782,14 @@ static int assign_device(AssignedDevice *dev)"cause host memory corruption if the device issues DMA write " "requests!\n"); } +if (dev->features &ASSIGNED_DEVICE_SHARE_INTX_MASK && +kvm_has_intx_set_mask ()) { + assigned_dev_data.flags |=KVM_DEV_ASSIGN_PCI_2_3 ; + +/* hide host-side INTx masking from the guest */ + dev->emulate_config_read[PCI_COMMAND +1 ] |= +PCI_COMMAND_INTX_DISABLE >>8 ; + } r =kvm_assign_pci_device (kvm_state, &assigned_dev_data);if (r <0 ) {
@@ -1121,10 +1129,26 @@ static void assigned_dev_pci_write_config(PCIDevice *pci_dev, uint32_t address, uint32_t val, int len) {AssignedDevice *assigned_dev =DO_UPCAST (AssignedDevice, dev, pci_dev); +uint16_t old_cmd =pci_get_word (pci_dev->config +PCI_COMMAND );uint32_t emulate_mask, full_emulation_mask; +int ret;pci_default_write_config (pci_dev, address, val, len); +if (kvm_has_intx_set_mask () && +range_covers_byte (address, len,PCI_COMMAND +1 )) { +bool intx_masked = (pci_get_word (pci_dev->config +PCI_COMMAND ) & +PCI_COMMAND_INTX_DISABLE ); + +if (intx_masked != !!(old_cmd &PCI_COMMAND_INTX_DISABLE )) { + ret =kvm_device_intx_set_mask (kvm_state, +calc_assigned_dev_id (assigned_dev), + intx_masked); +if (ret) { +perror ("assigned_dev_pci_write_config: set intx mask" ); + } + } + }if (assigned_dev->cap.available &ASSIGNED_DEVICE_CAP_MSI ) {if (range_covers_byte (address, len, pci_dev->msi_cap +PCI_MSI_FLAGS )) {
@@ -1748,6 +1772,8 @@ static Property da_properties[] = ASSIGNED_DEVICE_USE_IOMMU_BIT, true),DEFINE_PROP_BIT ("prefer_msi" , AssignedDevice, features,ASSIGNED_DEVICE_PREFER_MSI_BIT , true), +DEFINE_PROP_BIT ("share_intx" , AssignedDevice, features, +ASSIGNED_DEVICE_SHARE_INTX_BIT , false),DEFINE_PROP_INT32 ("bootindex" , AssignedDevice, bootindex,-1 ),DEFINE_PROP_STRING ("configfd" , AssignedDevice, configfd_name), DEFINE_PROP_END_OF_LIST(),
hw/device-assignment.h +6 -4
@@ -74,11 +74,13 @@ typedef struct {PCIRegion *region; } AssignedDevRegion; -#define ASSIGNED_DEVICE_USE_IOMMU_BIT 0 -#define ASSIGNED_DEVICE_PREFER_MSI_BIT 1 +#define ASSIGNED_DEVICE_USE_IOMMU_BIT 0 +#define ASSIGNED_DEVICE_PREFER_MSI_BIT 1 +#define ASSIGNED_DEVICE_SHARE_INTX_BIT 2 -#define ASSIGNED_DEVICE_USE_IOMMU_MASK (1 << ASSIGNED_DEVICE_USE_IOMMU_BIT) -#define ASSIGNED_DEVICE_PREFER_MSI_MASK (1 << ASSIGNED_DEVICE_PREFER_MSI_BIT) +#define ASSIGNED_DEVICE_USE_IOMMU_MASK (1 << ASSIGNED_DEVICE_USE_IOMMU_BIT) +#define ASSIGNED_DEVICE_PREFER_MSI_MASK (1 << ASSIGNED_DEVICE_PREFER_MSI_BIT) +#define ASSIGNED_DEVICE_SHARE_INTX_MASK (1 << ASSIGNED_DEVICE_SHARE_INTX_BIT) typedef struct { uint32_t addr_lo;
qemu-kvm.c +9 -0
@@ -54,6 +54,15 @@ static int kvm_old_assign_irq(KVMState *s,return kvm_vm_ioctl (s,KVM_ASSIGN_IRQ , assigned_irq); } +int kvm_device_intx_set_mask (KVMState *s,uint32_t dev_id,bool masked) +{ +struct kvm_assigned_pci_dev assigned_dev; + + assigned_dev.assigned_dev_id = dev_id; + assigned_dev.flags = masked ?KVM_DEV_ASSIGN_MASK_INTX :0 ; +return kvm_vm_ioctl (s,KVM_ASSIGN_SET_INTX_MASK , &assigned_dev); +} +#ifdef KVM_CAP_ASSIGN_DEV_IRQ int kvm_assign_irq (KVMState *s,struct kvm_assigned_irq *assigned_irq) {
qemu-kvm.h +2 -0
@@ -114,6 +114,8 @@ int kvm_assign_irq(KVMState *s, struct kvm_assigned_irq *assigned_irq);*/ int kvm_deassign_irq (KVMState *s,struct kvm_assigned_irq *assigned_irq); +int kvm_device_intx_set_mask (KVMState *s,uint32_t dev_id,bool masked); +/*! * \brief Notifies host kernel about a PCI device to be deassigned from a guest *
Signature · 1 line
1.7.3.4
Re: [PATCH v4 3/4] pci-assign: Use PCI-2.3-based shared legacy interrupts
In reply to Jan KiszkaQuoted 8 lines · Jan Kiszka
Enable the new KVM feature that allows legacy interrupt sharing for PCI-2.3-compliant devices. This requires to synchronize any guest change of the INTx mask bit to the kernel.
The feature is controlled by the property '
share_intx' and is off by default for now.Signed-off-by Jan Kiszka <[email protected]>
Quoted 118 lines
--- hw/device-assignment.c | 25 +++++++++++++++++++++++++ hw/device-assignment.h | 10 ++++++---- qemu-kvm.c | 9 +++++++++ qemu-kvm.h | 2 ++ 4 files changed, 42 insertions(+), 4 deletions(-)hw/device-assignment.c +25
-0@@ -782,6 +782,13 @@ static int assign_device(AssignedDevice *dev)"cause host memory corruption if the device issues DMA write " "requests!\n"); } +if (dev->features &ASSIGNED_DEVICE_SHARE_INTX_MASK ) { + assigned_dev_data.flags |=KVM_DEV_ASSIGN_PCI_2_3 ; + +/* hide host-side INTx masking from the guest */ + dev->emulate_config_read[PCI_COMMAND +1 ] |= +PCI_COMMAND_INTX_DISABLE >>8 ; + } r =kvm_assign_pci_device (kvm_state, &assigned_dev_data);if (r <0 ) {@@ -1121,10 +1128,26 @@ static void assigned_dev_pci_write_config(PCIDevice *pci_dev, uint32_t address, uint32_t val, int len) {AssignedDevice *assigned_dev =DO_UPCAST (AssignedDevice, dev, pci_dev); +uint16_t old_cmd =pci_get_word (pci_dev->config +PCI_COMMAND );uint32_t emulate_mask, full_emulation_mask; +int ret;pci_default_write_config (pci_dev, address, val, len); +if (kvm_has_intx_set_mask () && +range_covers_byte (address, len,PCI_COMMAND +1 )) { +bool intx_masked = (pci_get_word (pci_dev->config +PCI_COMMAND ) & +PCI_COMMAND_INTX_DISABLE ); + +if (intx_masked != !!(old_cmd &PCI_COMMAND_INTX_DISABLE )) { + ret =kvm_device_intx_set_mask (kvm_state, +calc_assigned_dev_id (assigned_dev), + intx_masked); +if (ret) { +perror ("assigned_dev_pci_write_config: set intx mask" ); + } + } + }if (assigned_dev->cap.available &ASSIGNED_DEVICE_CAP_MSI ) {if (range_covers_byte (address, len, pci_dev->msi_cap +PCI_MSI_FLAGS )) {@@ -1748,6 +1771,8 @@ static Property da_properties[] = ASSIGNED_DEVICE_USE_IOMMU_BIT, true),DEFINE_PROP_BIT ("prefer_msi" , AssignedDevice, features,ASSIGNED_DEVICE_PREFER_MSI_BIT , true), +DEFINE_PROP_BIT ("share_intx" , AssignedDevice, features, +ASSIGNED_DEVICE_SHARE_INTX_BIT , false),DEFINE_PROP_INT32 ("bootindex" , AssignedDevice, bootindex,-1 ),DEFINE_PROP_STRING ("configfd" , AssignedDevice, configfd_name), DEFINE_PROP_END_OF_LIST(),hw/device-assignment.h +6
-4@@ -74,11 +74,13 @@ typedef struct {PCIRegion *region; } AssignedDevRegion; -#define ASSIGNED_DEVICE_USE_IOMMU_BIT 0 -#define ASSIGNED_DEVICE_PREFER_MSI_BIT 1 +#define ASSIGNED_DEVICE_USE_IOMMU_BIT 0 +#define ASSIGNED_DEVICE_PREFER_MSI_BIT 1 +#define ASSIGNED_DEVICE_SHARE_INTX_BIT 2 -#define ASSIGNED_DEVICE_USE_IOMMU_MASK (1 << ASSIGNED_DEVICE_USE_IOMMU_BIT) -#define ASSIGNED_DEVICE_PREFER_MSI_MASK (1 << ASSIGNED_DEVICE_PREFER_MSI_BIT) +#define ASSIGNED_DEVICE_USE_IOMMU_MASK (1 << ASSIGNED_DEVICE_USE_IOMMU_BIT) +#define ASSIGNED_DEVICE_PREFER_MSI_MASK (1 << ASSIGNED_DEVICE_PREFER_MSI_BIT) +#define ASSIGNED_DEVICE_SHARE_INTX_MASK (1 << ASSIGNED_DEVICE_SHARE_INTX_BIT) typedef struct { uint32_t addr_lo;qemu-kvm.c +9
-0@@ -54,6 +54,15 @@ static int kvm_old_assign_irq(KVMState *s,return kvm_vm_ioctl (s,KVM_ASSIGN_IRQ , assigned_irq); } +int kvm_device_intx_set_mask (KVMState *s,uint32_t dev_id,bool masked) +{ +struct kvm_assigned_pci_dev assigned_dev; + + assigned_dev.assigned_dev_id = dev_id; + assigned_dev.flags = masked ?KVM_DEV_ASSIGN_MASK_INTX :0 ; +return kvm_vm_ioctl (s,KVM_ASSIGN_SET_INTX_MASK , &assigned_dev); +} +#ifdef KVM_CAP_ASSIGN_DEV_IRQ int kvm_assign_irq (KVMState *s,struct kvm_assigned_irq *assigned_irq) {qemu-kvm.h +2
-0@@ -114,6 +114,8 @@ int kvm_assign_irq(KVMState *s, struct kvm_assigned_irq *assigned_irq);*/ int kvm_deassign_irq (KVMState *s,struct kvm_assigned_irq *assigned_irq); +int kvm_device_intx_set_mask (KVMState *s,uint32_t dev_id,bool masked); +/*! * \brief Notifies host kernel about a PCI device to be deassigned from a guest * Signature · 1 line
1.7.3.4
PATCH v4 4/4 pci_assign: Flip defaults of prefer_msi and share_intx
In reply to Jan KiszkaINTx sharing is a bit more expensive than exclusive host interrupts, but this channel is not supposed to be used for high-performance scenarios anyway. Modern devices support MSI/MSI-X and do not depend on using INTx under critical workload, real old devices do not support INTx sharing anyway.
For those in the middle, the user experience is much better if they just work even when IRQ sharing is required. If there is nothing to share, share_intx=off can still be applied as tuning parameter.
With INTx sharing as default, the primary reason for prefer_msi=on is gone. Make it default off, specifically as it is known to cause troubles with devices that have incomplete/broken MSI support or otherwise stumble if host IRQ configuration does not match guest driver expectation.
--- hw/device-assignment.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-)
hw/device-assignment.c +2 -2
@@ -1770,9 +1770,9 @@ static Property da_properties[] =DEFINE_PROP_BIT ("iommu" , AssignedDevice, features,ASSIGNED_DEVICE_USE_IOMMU_BIT , true),DEFINE_PROP_BIT ("prefer_msi" , AssignedDevice, features, -ASSIGNED_DEVICE_PREFER_MSI_BIT , true), +ASSIGNED_DEVICE_PREFER_MSI_BIT , false),DEFINE_PROP_BIT ("share_intx" , AssignedDevice, features, -ASSIGNED_DEVICE_SHARE_INTX_BIT , false), +ASSIGNED_DEVICE_SHARE_INTX_BIT , true),DEFINE_PROP_INT32 ("bootindex" , AssignedDevice, bootindex,-1 ),DEFINE_PROP_STRING ("configfd" , AssignedDevice, configfd_name), DEFINE_PROP_END_OF_LIST(),
Signature · 1 line
1.7.3.4
Re: [PATCH v4 4/4] pci_assign: Flip defaults of prefer_msi and share_intx
Quoted 10 lines · Jan Kiszka
INTx sharing is a bit more expensive than exclusive host interrupts, but this channel is not supposed to be used for high-performance scenarios anyway. Modern devices support MSI/MSI-X and do not depend on using INTx under critical workload, real old devices do not support INTx sharing anyway.
For those in the middle, the user experience is much better if they just work even when IRQ sharing is required. If there is nothing to share, share_intx=off can still be applied as tuning parameter.
Quoted 7 lines
With INTx sharing as default, the primary reason for prefer_msi=on is gone. Make it default off, specifically as it is known to cause troubles with devices that have incomplete/broken MSI support or otherwise stumble if host IRQ configuration does not match guest driver expectation.
Signed-off-by Jan Kiszka <[email protected]>
I'm not familiar enough with what prefer_msi does so no comment here. Maybe Alex can ack. If not I'll try to find the time to understand it a bit better but will take some time.
Quoted 21 lines
hw/device-assignment.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-)hw/device-assignment.c +2
-2@@ -1770,9 +1770,9 @@ static Property da_properties[] =DEFINE_PROP_BIT ("iommu" , AssignedDevice, features,ASSIGNED_DEVICE_USE_IOMMU_BIT , true),DEFINE_PROP_BIT ("prefer_msi" , AssignedDevice, features, -ASSIGNED_DEVICE_PREFER_MSI_BIT , true), +ASSIGNED_DEVICE_PREFER_MSI_BIT , false),DEFINE_PROP_BIT ("share_intx" , AssignedDevice, features, -ASSIGNED_DEVICE_SHARE_INTX_BIT , false), +ASSIGNED_DEVICE_SHARE_INTX_BIT , true),DEFINE_PROP_INT32 ("bootindex" , AssignedDevice, bootindex,-1 ),DEFINE_PROP_STRING ("configfd" , AssignedDevice, configfd_name), DEFINE_PROP_END_OF_LIST(),Signature · 1 line
1.7.3.4
Quoted 17 lines · Jan Kiszka
This enables support for the host IRQ sharing support via PCI-2.3 interrupt masking. See the patches for details.
Patch 1 & 3 could be considered for uq/master although we have no use case for the changes in upstream yet.
Changes in v4:
- add patch forkvm_has_intx_set_mask
- do not issue unsupported IOCTLs when the host kernel does not support INTx mask controlJan Kiszka (4):
kvm: Update kernel headers against kvm.git
kvm: Introducekvm_has_intx_set_mask
pci-assign: Use PCI-2.3-based shared legacy interrupts
pci_assign: Flip defaults ofprefer_msiandshare_intx
Quoted 10 lines
hw/device-assignment.c | 27 ++++++++++++++++++++++++++- hw/device-assignment.h | 10 ++++++---- kvm-all.c | 8 ++++++++ kvm.h | 1 + linux-headers/asm-s390/kvm.h | 2 ++ linux-headers/linux/kvm.h | 6 ++++++ qemu-kvm.c | 9 +++++++++ qemu-kvm.h | 2 ++ 8 files changed, 60 insertions(+), 5 deletions(-)
Looks right to me w/ the updated (v5) 3rd patch.
Marcelo Tosatti
AppliedQuoted 17 lines · Jan Kiszka
This enables support for the host IRQ sharing support via PCI-2.3 interrupt masking. See the patches for details.
Patch 1 & 3 could be considered for uq/master although we have no use case for the changes in upstream yet.
Changes in v4:
- add patch forkvm_has_intx_set_mask
- do not issue unsupported IOCTLs when the host kernel does not support INTx mask controlJan Kiszka (4):
kvm: Update kernel headers against kvm.git
kvm: Introducekvm_has_intx_set_mask
pci-assign: Use PCI-2.3-based shared legacy interrupts
pci_assign: Flip defaults ofprefer_msiandshare_intx
Quoted 9 lines
hw/device-assignment.c | 27 ++++++++++++++++++++++++++- hw/device-assignment.h | 10 ++++++---- kvm-all.c | 8 ++++++++ kvm.h | 1 + linux-headers/asm-s390/kvm.h | 2 ++ linux-headers/linux/kvm.h | 6 ++++++ qemu-kvm.c | 9 +++++++++ qemu-kvm.h | 2 ++ 8 files changed, 60 insertions(+), 5 deletions(-)
Applied, thanks.