MSI-X enabling
v4 cover MSI-X enabling
PATCH 1/4 KVM: Fix wrong usage of _IOR in assigned device interface
_IOR for copy_to_user and _IOW for copy_from_user...
Noticed by Avi.
--- include/linux/kvm.h | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-)
include/linux/kvm.h +2 -2
@@ -469,10 +469,10 @@ struct kvm_irq_routing { _IOW(KVMIO, 0x67, struct kvm_coalesced_mmio_zone)#define KVM_UNREGISTER_COALESCED_MMIO \ _IOW(KVMIO, 0x68, struct kvm_coalesced_mmio_zone) -#define KVM_ASSIGN_PCI_DEVICE _IOR(KVMIO, 0x69, \ +#define KVM_ASSIGN_PCI_DEVICE _IOW(KVMIO, 0x69, \ struct kvm_assigned_pci_dev)#define KVM_SET_GSI_ROUTING _IOW(KVMIO, 0x6a, struct kvm_irq_routing) -#define KVM_ASSIGN_IRQ _IOR(KVMIO, 0x70, \ +#define KVM_ASSIGN_IRQ _IOW(KVMIO, 0x70, \ struct kvm_assigned_irq)#define KVM_REINJECT_CONTROL _IO(KVMIO, 0x71)
Signature · 1 line
1.5.4.5
Re: [PATCH 1/4] KVM: Fix wrong usage of _IOR in assigned device interface
Quoted 17 lines · Sheng Yang
_IORforcopy_to_userand_IOWforcopy_from_user...Noticed by Avi.
Signed-off-by Sheng Yang <[email protected]>--- include/linux/kvm.h | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-)include/linux/kvm.h +0
-0@@ -469,10 +469,10 @@ struct kvm_irq_routing { _IOW(KVMIO, 0x67, struct kvm_coalesced_mmio_zone)#define KVM_UNREGISTER_COALESCED_MMIO \
include/linux/kvm.h struct kvm_irq_routing { · line 478
Quoted 10 lines
include/linux/kvm.h +2
-2_IOW(KVMIO, 0x68, struct kvm_coalesced_mmio_zone) -#define KVM_ASSIGN_PCI_DEVICE _IOR(KVMIO, 0x69, \ +#define KVM_ASSIGN_PCI_DEVICE _IOW(KVMIO, 0x69, \ struct kvm_assigned_pci_dev)#define KVM_SET_GSI_ROUTING _IOW(KVMIO, 0x6a, struct kvm_irq_routing) -#define KVM_ASSIGN_IRQ _IOR(KVMIO, 0x70, \ +#define KVM_ASSIGN_IRQ _IOW(KVMIO, 0x70, \ struct kvm_assigned_irq)#define KVM_REINJECT_CONTROL _IO(KVMIO, 0x71)
KVM_ASSIGN_PCI_DEVICE was introduced in 2.6.28. We can't fix it since it's part of the ABI.
Signature · 2 lines
I have a truly marvellous patch that fixes the bug which this signature is too narrow to contain.
Re: [PATCH 1/4] KVM: Fix wrong usage of _IOR in assigned device interface
Quoted 21 lines · On Wednesday 18 February 2009 22:44:15 Avi Kivity wrote:
Quoted 20 lines · Sheng Yang
_IORforcopy_to_userand_IOWforcopy_from_user...Noticed by Avi.
Signed-off-by Sheng Yang <[email protected]>--- include/linux/kvm.h | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-)include/linux/kvm.h +1
-1@@ -469,10 +469,10 @@ struct kvm_irq_routing { _IOW(KVMIO, 0x67, struct kvm_coalesced_mmio_zone)#define KVM_UNREGISTER_COALESCED_MMIO \ _IOW(KVMIO, 0x68, struct kvm_coalesced_mmio_zone) -#define KVM_ASSIGN_PCI_DEVICE _IOR(KVMIO, 0x69, \ +#define KVM_ASSIGN_PCI_DEVICE _IOW(KVMIO, 0x69, \ struct kvm_assigned_pci_dev)
Quoted 8 lines
include/linux/kvm.h struct kvm_irq_routing { · line 477
Quoted 5 lines
include/linux/kvm.h +1
-0#define KVM_SET_GSI_ROUTING _IOW(KVMIO, 0x6a, struct kvm_irq_routing) -#define KVM_ASSIGN_IRQ _IOR(KVMIO, 0x70, \ +#define KVM_ASSIGN_IRQ _IOW(KVMIO, 0x70, \ struct kvm_assigned_irq)#define KVM_REINJECT_CONTROL _IO(KVMIO, 0x71)
KVM_ASSIGN_PCI_DEVICEwas introduced in 2.6.28. We can't fix it since it's part of the ABI.
OK... So please take this one off.
Signature · 2 lines
regards Yang, Sheng
PATCH 2/4 KVM: Ioctls for init MSI-X entry
In reply to Sheng YangIntroduce KVM_SET_MSIX_NR and KVM_SET_MSIX_ENTRY two ioctls.
This two ioctls are used by userspace to specific guest device MSI-X entry number and correlate MSI-X entry with GSI during the initialization stage.
MSI-X should be well initialzed before enabling.
Don't support change MSI-X entry number for now.
--- include/linux/kvm.h | 18 ++++++++ include/linux/kvm_host.h | 10 ++++ virt/kvm/kvm_main.c | 104 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 132 insertions(+), 0 deletions(-)
include/linux/kvm.h +18 -0
@@ -475,6 +475,10 @@ struct kvm_irq_routing {#define KVM_ASSIGN_IRQ _IOW(KVMIO, 0x70, \ struct kvm_assigned_irq)#define KVM_REINJECT_CONTROL _IO(KVMIO, 0x71) +#define KVM_ASSIGN_SET_MSIX_NR \ + _IOW(KVMIO, 0x72, struct kvm_assigned_msix_nr) +#define KVM_ASSIGN_SET_MSIX_ENTRY \ + _IOW(KVMIO, 0x73, struct kvm_assigned_msix_entry)/* * ioctls for vcpu fds
@@ -595,4 +599,18 @@ struct kvm_assigned_irq {#define KVM_DEV_IRQ_ASSIGN_MSI_ACTION KVM_DEV_IRQ_ASSIGN_ENABLE_MSI#define KVM_DEV_IRQ_ASSIGN_ENABLE_MSI (1 << 0) +struct kvm_assigned_msix_nr { +__u32 assigned_dev_id; +__u16 entry_nr; +__u16 padding; +}; + +#define KVM_MAX_MSIX_PER_DEV 512 +struct kvm_assigned_msix_entry { +__u32 assigned_dev_id; +__u32 gsi; +__u16 entry;/* The index of entry in the MSI-X table */ +__u16 padding[3 ]; +}; +#endif
include/linux/kvm_host.h +10 -0
@@ -319,6 +319,12 @@ struct kvm_irq_ack_notifier {void (*irq_acked)(struct kvm_irq_ack_notifier *kian); }; +struct kvm_guest_msix_entry { +u32 vector; +u16 entry; +u16 flags; +}; +struct kvm_assigned_dev_kernel {struct kvm_irq_ack_notifier ack_notifier;struct work_struct interrupt_work;
@@ -326,13 +332,17 @@ struct kvm_assigned_dev_kernel {int assigned_dev_id;int host_busnr;int host_devfn; +unsigned int entries_nr;int host_irq;bool host_irq_disabled; +struct msix_entry *host_msix_entries;int guest_irq; +struct kvm_guest_msix_entry *guest_msix_entries;#define KVM_ASSIGNED_DEV_GUEST_INTX (1 << 0)#define KVM_ASSIGNED_DEV_GUEST_MSI (1 << 1)#define KVM_ASSIGNED_DEV_HOST_INTX (1 << 8)#define KVM_ASSIGNED_DEV_HOST_MSI (1 << 9) +#define KVM_ASSIGNED_DEV_MSIX ((1 << 2) | (1 << 10))unsigned long irq_requested_type;int irq_source_id;int flags;
virt/kvm/kvm_main.c +104 -0
117 lines not on this page · raw message
Signature · 1 line
1.5.4.5
PATCH 3/4 KVM: Add MSI-X interrupt injection logic
In reply to Sheng YangWe have to handle more than one interrupt with one handler for MSI-X. Avi suggested to use a flag to indicate the pending. So here is it.
--- include/linux/kvm_host.h | 1 + virt/kvm/kvm_main.c | 66 +++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 60 insertions(+), 7 deletions(-)
include/linux/kvm_host.h +1 -0
@@ -319,6 +319,7 @@ struct kvm_irq_ack_notifier {void (*irq_acked)(struct kvm_irq_ack_notifier *kian); }; +#define KVM_ASSIGNED_MSIX_PENDING 0x1struct kvm_guest_msix_entry {u32 vector;u16 entry;
virt/kvm/kvm_main.c +59 -7
90 lines not on this page · raw message
Signature · 1 line
1.5.4.5
PATCH 4/4 KVM: Enable MSI-X for KVM assigned device
In reply to Sheng YangThis patch finally enable MSI-X.
What we need for MSI-X:
1. Intercept one page in MMIO region of device. So that we can get guest desired MSI-X table and set up the real one. Now this have been done by guest, and transfer to kernel using ioctl KVM_SET_MSIX_NR and KVM_SET_MSIX_ENTRY.
2. Information for incoming interrupt. Now one device can have more than one interrupt, and they are all handled by one workqueue structure. So we need to identify them. The previous patch enable gsi_msg_pending_bitmap get this done.
3. Mapping from host IRQ to guest gsi as well as guest gsi to real MSI/MSI-X message address/data. We used same entry number for the host and guest here, so that it's easy to find the correlated guest gsi.
What we lack for now:
1. The PCI spec said nothing can existed with MSI-X table in the same page of MMIO region, except pending bits. The patch ignore pending bits as the first step (so they are always 0 - no pending).
2. The PCI spec allowed to change MSI-X table dynamically. That means, the OS can enable MSI-X, then mask one MSI-X entry, modify it, and unmask it. The patch didn't support this, and Linux also don't work in this way.
3. The patch didn't implement MSI-X mask all and mask single entry. I would implement the former in driver/pci/msi.c later. And for single entry, userspace should have reposibility to handle it.
--- arch/x86/include/asm/kvm.h | 1 + include/linux/kvm.h | 8 ++++ virt/kvm/kvm_main.c | 98 +++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 101 insertions(+), 6 deletions(-)
arch/x86/include/asm/kvm.h +1 -0
@@ -16,6 +16,7 @@#define __KVM_HAVE_MSI#define __KVM_HAVE_USER_NMI#define __KVM_HAVE_GUEST_DEBUG +#define __KVM_HAVE_MSIX/* Architectural interrupt line count. */ #define KVM_NR_INTERRUPTS 256
include/linux/kvm.h +8 -0
@@ -402,6 +402,9 @@ struct kvm_trace_rec {#ifdef __KVM_HAVE_IOAPIC#define KVM_CAP_IRQ_ROUTING 25#endif +#ifdef __KVM_HAVE_MSIX +#define KVM_CAP_DEVICE_MSIX 26 +#endif #ifdef KVM_CAP_IRQ_ROUTING
@@ -599,6 +602,11 @@ struct kvm_assigned_irq {#define KVM_DEV_IRQ_ASSIGN_MSI_ACTION KVM_DEV_IRQ_ASSIGN_ENABLE_MSI#define KVM_DEV_IRQ_ASSIGN_ENABLE_MSI (1 << 0) +#define KVM_DEV_IRQ_ASSIGN_MSIX_ACTION (KVM_DEV_IRQ_ASSIGN_ENABLE_MSIX |\ + KVM_DEV_IRQ_ASSIGN_MASK_MSIX) +#define KVM_DEV_IRQ_ASSIGN_ENABLE_MSIX (1 << 1) +#define KVM_DEV_IRQ_ASSIGN_MASK_MSIX (1 << 2) +struct kvm_assigned_msix_nr {__u32 assigned_dev_id;__u16 entry_nr;
virt/kvm/kvm_main.c +92 -6
123 lines not on this page · raw message
Signature · 1 line
1.5.4.5