Make KVM/MSI code device-ID-aware
v3 cover Make KVM/MSI code device-ID-aware
On ARM architecture ITS (Interrupt Translation Service), additionally to normal MSI data, uses also side-band device IDs. This series prepares the infrastructure to handling them.
This small series is actually an extraction from vITS support RFC
(https://lists.gnu.org/archive/html/qemu-devel/2015-09/msg07074.html).
Nobody reviewed it so far, however some bits of it could actually be applied early. This series consists only of those parts, which do not depend on any new unreleased kernel APIs.
v2 => v3:
- Use official "requester ID" denotion everywhere. Renamed getter function and MemTxAttrs field.
v1 => v2:
- Improved commit messages, more explanations
- msi_device_id() function made architecture-specific
Pavel Fedin (3):
kvm: Make KVM_CAP_SIGNAL_MSI globally available hw/pci: Introduce msi_requester_id()
kvm: Pass PCI device pointer to MSI routing functions
hw/i386/kvm/pci-assign.c | 9 +++++---- hw/pci/msi.c | 2 +- hw/vfio/pci.c | 11 ++++++----- hw/virtio/virtio-pci.c | 5 +++-- include/exec/memattrs.h | 4 ++-- include/hw/pci/msi.h | 1 + include/sysemu/kvm.h | 14 ++++++++++++-- kvm-all.c | 15 ++++++++------- kvm-stub.c | 5 +++-- stubs/Makefile.objs | 1 + stubs/msi.c | 16 ++++++++++++++++ target-arm/Makefile.objs | 1 + target-arm/msi.c | 16 ++++++++++++++++ 13 files changed, 75 insertions(+), 25 deletions(-) create mode 100644 stubs/msi.c create mode 100644 target-arm/msi.c
Signature · 1 line
2.4.4
PATCH v3 3/3 kvm: Pass PCI device pointer to MSI routing functions
In-kernel ITS emulation on ARM64 will require to supply requester IDs. These IDs can now be retrieved from the device pointer using new msi_requester_id() function.
This patch adds pci_dev pointer to KVM GSI routing functions and makes callers passing it.
x86 architecture does not use requester IDs, but hw/i386/kvm/pci-assign.c also made passing PCI device pointer instead of NULL for consistency with the rest of the code.
--- hw/i386/kvm/pci-assign.c | 9 +++++---- hw/vfio/pci.c | 11 ++++++----- hw/virtio/virtio-pci.c | 5 +++-- include/sysemu/kvm.h | 5 +++-- kvm-all.c | 5 +++-- kvm-stub.c | 5 +++-- 6 files changed, 23 insertions(+), 17 deletions(-)
hw/i386/kvm/pci-assign.c +5 -4
@@ -979,7 +979,7 @@ static void assigned_dev_update_msi(PCIDevice *pci_dev)MSIMessage msg =msi_get_message (pci_dev,0 );int virq; - virq =kvm_irqchip_add_msi_route (kvm_state, msg); + virq =kvm_irqchip_add_msi_route (kvm_state, msg, pci_dev);if (virq <0 ) {perror ("assigned_dev_update_msi: kvm_irqchip_add_msi_route" );return ;
@@ -1017,7 +1017,7 @@ static void assigned_dev_update_msi_msg(PCIDevice *pci_dev) }kvm_irqchip_update_msi_route (kvm_state, assigned_dev->msi_virq[0 ], -msi_get_message (pci_dev,0 )); +msi_get_message (pci_dev,0 ), pci_dev); }static bool assigned_dev_msix_masked (MSIXTableEntry *entry)
@@ -1083,7 +1083,7 @@ static int assigned_dev_update_msix_mmio(PCIDevice *pci_dev) msg.address = entry->addr_lo | ((uint64_t )entry->addr_hi <<32 ); msg.data = entry->data; - r =kvm_irqchip_add_msi_route (kvm_state, msg); + r =kvm_irqchip_add_msi_route (kvm_state, msg, pci_dev);if (r <0 ) {return r; }
@@ -1602,7 +1602,8 @@ static void assigned_dev_msix_mmio_write(void *opaque, hwaddr addr, msg.data = entry->data; ret =kvm_irqchip_update_msi_route (kvm_state, - adev->msi_virq[i], msg); + adev->msi_virq[i], msg, + pci_dev);if (ret) {error_report ("Error updating irq routing entry (%d)" , ret); }
hw/vfio/pci.c +6 -5
@@ -424,7 +424,7 @@ static void vfio_add_kvm_msi_virq(VFIOPCIDevice *vdev, VFIOMSIVector *vector,return ; } - virq =kvm_irqchip_add_msi_route (kvm_state, *msg); + virq =kvm_irqchip_add_msi_route (kvm_state, *msg, &vdev->pdev);if (virq <0 ) {event_notifier_cleanup (&vector->kvm_interrupt);return ;
@@ -449,9 +449,10 @@ static void vfio_remove_kvm_msi_virq(VFIOMSIVector *vector)event_notifier_cleanup (&vector->kvm_interrupt); } -static void vfio_update_kvm_msi_virq (VFIOMSIVector *vector,MSIMessage msg) +static void vfio_update_kvm_msi_virq (VFIOMSIVector *vector,MSIMessage msg, +PCIDevice *pdev) { -kvm_irqchip_update_msi_route (kvm_state, vector->virq, msg); +kvm_irqchip_update_msi_route (kvm_state, vector->virq, msg, pdev); } static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr,
@@ -486,7 +487,7 @@ static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr,if (!msg) {vfio_remove_kvm_msi_virq (vector); }else { -vfio_update_kvm_msi_virq (vector, *msg); +vfio_update_kvm_msi_virq (vector, *msg, pdev); } }else {vfio_add_kvm_msi_virq (vdev, vector, msg, true);
@@ -760,7 +761,7 @@ static void vfio_update_msi(VFIOPCIDevice *vdev) } msg =msi_get_message (&vdev->pdev, i); -vfio_update_kvm_msi_virq (vector, msg); +vfio_update_kvm_msi_virq (vector, msg, &vdev->pdev); } }
hw/virtio/virtio-pci.c +3 -2
@@ -590,7 +590,7 @@ static int kvm_virtio_pci_vq_vector_use(VirtIOPCIProxy *proxy,int ret;if (irqfd->users ==0 ) { - ret =kvm_irqchip_add_msi_route (kvm_state, msg); + ret =kvm_irqchip_add_msi_route (kvm_state, msg, &proxy->pci_dev);if (ret <0 ) {return ret; }
@@ -726,7 +726,8 @@ static int virtio_pci_vq_vector_unmask(VirtIOPCIProxy *proxy,if (proxy->vector_irqfd) { irqfd = &proxy->vector_irqfd[vector];if (irqfd->msg.data != msg.data || irqfd->msg.address != msg.address) { - ret =kvm_irqchip_update_msi_route (kvm_state, irqfd->virq, msg); + ret =kvm_irqchip_update_msi_route (kvm_state, irqfd->virq, msg, + &proxy->pci_dev);if (ret <0 ) {return ret; }
include/sysemu/kvm.h +3 -2
@@ -447,8 +447,9 @@ static inline void cpu_clean_state(CPUState *cpu) } } -int kvm_irqchip_add_msi_route (KVMState *s,MSIMessage msg); -int kvm_irqchip_update_msi_route (KVMState *s,int virq,MSIMessage msg); +int kvm_irqchip_add_msi_route (KVMState *s,MSIMessage msg,PCIDevice *dev); +int kvm_irqchip_update_msi_route (KVMState *s,int virq,MSIMessage msg, +PCIDevice *dev);void kvm_irqchip_release_virq (KVMState *s,int virq);int kvm_irqchip_add_adapter_route (KVMState *s,AdapterInfo *adapter);
kvm-all.c +3 -2
@@ -1189,7 +1189,7 @@ int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)return kvm_set_irq (s, route->kroute.gsi,1 ); } -int kvm_irqchip_add_msi_route (KVMState *s,MSIMessage msg) +int kvm_irqchip_add_msi_route (KVMState *s,MSIMessage msg,PCIDevice *dev) {struct kvm_irq_routing_entry kroute = {};int virq;
@@ -1224,7 +1224,8 @@ int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg)return virq; } -int kvm_irqchip_update_msi_route (KVMState *s,int virq,MSIMessage msg) +int kvm_irqchip_update_msi_route (KVMState *s,int virq,MSIMessage msg, +PCIDevice *dev) {struct kvm_irq_routing_entry kroute = {};
kvm-stub.c +3 -2
@@ -115,7 +115,7 @@ int kvm_on_sigbus(int code, void *addr) }#ifndef CONFIG_USER_ONLY -int kvm_irqchip_add_msi_route (KVMState *s,MSIMessage msg) +int kvm_irqchip_add_msi_route (KVMState *s,MSIMessage msg,PCIDevice *dev) {return -ENOSYS ; }
@@ -128,7 +128,8 @@ void kvm_irqchip_release_virq(KVMState *s, int virq) { } -int kvm_irqchip_update_msi_route (KVMState *s,int virq,MSIMessage msg) +int kvm_irqchip_update_msi_route (KVMState *s,int virq,MSIMessage msg, +PCIDevice *dev) {return -ENOSYS ; }
Signature · 1 line
2.4.4
PATCH v3 2/3 hw/pci: Introduce msi_requester_id()
In reply to Pavel FedinFor GICv3 ITS implementation we are going to use requester IDs in KVM IRQ routing code. This patch introduces reusable convenient way to obtain this ID from the device pointer.
Since requester ID is an architecture-specific thing, the function can be overridden on per-architecture basis. Default stub just returns 0.
MemTxAttrs.stream_id also renamed to requester_id in order to better reflect semantics of the field.
--- hw/pci/msi.c | 2 +- include/exec/memattrs.h | 4 ++-- include/hw/pci/msi.h | 1 + stubs/Makefile.objs | 1 + stubs/msi.c | 16 ++++++++++++++++ target-arm/Makefile.objs | 1 + target-arm/msi.c | 16 ++++++++++++++++ 7 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 stubs/msi.c create mode 100644 target-arm/msi.c
hw/pci/msi.c +1 -1
@@ -294,7 +294,7 @@ void msi_send_message(PCIDevice *dev, MSIMessage msg) {MemTxAttrs attrs = {}; - attrs.stream_id = (pci_bus_num (dev->bus) <<8 ) | dev->devfn; + attrs.requester_id =msi_requester_id (dev);address_space_stl_le (&dev->bus_master_as, msg.address, msg.data, attrs,NULL ); }
include/exec/memattrs.h +2 -2
@@ -35,8 +35,8 @@ typedef struct MemTxAttrs { unsigned int secure:1;/* Memory access is usermode (unprivileged) */ unsigned int user:1; -/* Stream ID (for MSI for example) */ - unsigned int stream_id:16; +/* Requester ID (for MSI for example) */ + unsigned int requester_id:16; } MemTxAttrs;/* Bus masters which don't specify any attributes will get this,
include/hw/pci/msi.h +1 -0
@@ -42,6 +42,7 @@ void msi_notify(PCIDevice *dev, unsigned int vector);void msi_send_message (PCIDevice *dev,MSIMessage msg);void msi_write_config (PCIDevice *dev,uint32_t addr,uint32_t val,int len);unsigned int msi_nr_vectors_allocated (const PCIDevice *dev); +uint16_t msi_requester_id (PCIDevice *dev);static inline bool msi_present (const PCIDevice *dev) {
stubs/Makefile.objs +1 -0
@@ -22,6 +22,7 @@ stub-obj-y += migr-blocker.o stub-obj-y += mon-is-qmp.o stub-obj-y += mon-printf.o stub-obj-y += monitor-init.o +stub-obj-$(CONFIG_PCI) += msi.o stub-obj-y += notify-event.o stub-obj-$(CONFIG_SPICE) += qemu-chr-open-spice.o stub-obj-y += qtest.o
stubs/msi.c new file +16 -0
@@ -0,0 +1,16 @@ +/* +* QEMU architecture-specific MSI functions +* +* Copyright (c) 2015 Samsung Electronics Co., Ltd. +* Written by Pavel Fedin +* +* This work is licensed under the terms of the GNU GPL, version 2 or later. +* See the COPYING file in the top-level directory. +* +*/ +#include "hw/pci/msi.h" + +uint16_t msi_requester_id (PCIDevice *dev) +{ +return 0 ; +}
target-arm/Makefile.objs +1 -0
@@ -1,5 +1,6 @@ obj-y += arm-semi.o obj-$(CONFIG_SOFTMMU) += machine.o +obj-$(CONFIG_PCI) += msi.o obj-$(CONFIG_KVM) += kvm.o obj-$(call land,$(CONFIG_KVM),$(call lnot,$(TARGET_AARCH64))) += kvm32.o obj-$(call land,$(CONFIG_KVM),$(TARGET_AARCH64)) += kvm64.o
target-arm/msi.c new file +16 -0
@@ -0,0 +1,16 @@ +/* +* QEMU ARM specific MSI functions +* +* Copyright (c) 2015 Samsung Electronics Co., Ltd. +* Written by Pavel Fedin +* +* This work is licensed under the terms of the GNU GPL, version 2 or later. +* See the COPYING file in the top-level directory. +* +*/ +#include "hw/pci/msi.h" + +uint16_t msi_requester_id (PCIDevice *dev) +{ +return (pci_bus_num (dev->bus) <<8 ) | dev->devfn; +}
Signature · 1 line
2.4.4
PATCH v3 1/3 kvm: Make KVM_CAP_SIGNAL_MSI globally available
In reply to Pavel FedinThis capability is useful to determine whether we can use KVM ITS emulation on ARM
--- include/sysemu/kvm.h | 9 +++++++++ kvm-all.c | 10 +++++----- 2 files changed, 14 insertions(+), 5 deletions(-)
include/sysemu/kvm.h +9 -0
@@ -52,6 +52,7 @@ extern bool kvm_msi_via_irqfd_allowed;extern bool kvm_gsi_routing_allowed;extern bool kvm_gsi_direct_mapping;extern bool kvm_readonly_mem_allowed; +extern bool kvm_direct_msi_allowed;#if definedCONFIG_KVM || !definedNEED_CPU_H #define kvm_enabled () (kvm_allowed)
@@ -145,6 +146,13 @@ extern bool kvm_readonly_mem_allowed;*/ #define kvm_readonly_mem_enabled () (kvm_readonly_mem_allowed) +/** +* kvm_direct_msi_enabled: +* +* Returns: true if KVM allows direct MSI injection. +*/ +#define kvm_direct_msi_enabled () (kvm_direct_msi_allowed) +#else #define kvm_enabled () (0)#define kvm_irqchip_in_kernel () (false)
@@ -157,6 +165,7 @@ extern bool kvm_readonly_mem_allowed;#define kvm_gsi_routing_allowed () (false)#define kvm_gsi_direct_mapping () (false)#define kvm_readonly_mem_enabled () (false) +#define kvm_direct_msi_enabled () (false)#endif struct kvm_run ;
kvm-all.c +5 -5
@@ -93,7 +93,6 @@ struct KVMStateuint32_t *used_gsi_bitmap;unsigned int gsi_count; QTAILQ_HEAD(msi_hashtab, KVMMSIRoute) msi_hashtab[KVM_MSI_HASHTAB_SIZE]; -bool direct_msi;#endif KVMMemoryListener memory_listener; };
@@ -111,6 +110,7 @@ bool kvm_gsi_direct_mapping;bool kvm_allowed;bool kvm_readonly_mem_allowed;bool kvm_vm_attributes_allowed; +bool kvm_direct_msi_allowed;static const KVMCapabilityInfo kvm_required_capabilites[] = {KVM_CAP_INFO (USER_MEMORY ),
@@ -979,7 +979,7 @@ void kvm_init_irq_routing(KVMState *s) s->irq_routes =g_malloc0 (sizeof (*s->irq_routes)); s->nr_allocated_irq_routes =0 ; -if (!s->direct_msi) { +if (!kvm_direct_msi_allowed) {for (i =0 ; i <KVM_MSI_HASHTAB_SIZE ; i++) {QTAILQ_INIT (&s->msi_hashtab[i]); }
@@ -1113,7 +1113,7 @@ static int kvm_irqchip_get_virq(KVMState *s)* number can succeed even though a new route entry cannot be added. * When this happens, flush dynamic MSI entries to free IRQ route entries. */ -if (!s->direct_msi && s->irq_routes->nr == s->gsi_count) { +if (!kvm_direct_msi_allowed && s->irq_routes->nr == s->gsi_count) {kvm_flush_dynamic_msi_routes (s); }
@@ -1150,7 +1150,7 @@ int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)struct kvm_msi msi;KVMMSIRoute *route; -if (s->direct_msi) { +if (kvm_direct_msi_allowed) { msi.address_lo = (uint32_t )msg.address; msi.address_hi = msg.address >>32 ; msi.data =le32_to_cpu (msg.data);
@@ -1598,7 +1598,7 @@ static int kvm_init(MachineState *ms)#endif #ifdef KVM_CAP_IRQ_ROUTING - s->direct_msi = (kvm_check_extension (s,KVM_CAP_SIGNAL_MSI ) >0 ); + kvm_direct_msi_allowed = (kvm_check_extension (s,KVM_CAP_SIGNAL_MSI ) >0 );#endif s->intx_set_mask =kvm_check_extension (s,KVM_CAP_PCI_2_3 );
Signature · 1 line
2.4.4
Re: [Qemu-devel] [PATCH v3 2/3] hw/pci: Introduce msi_requester_id()
In reply to Pavel FedinQuoted 11 lines · Pavel Fedin
For GICv3 ITS implementation we are going to use requester IDs in KVM IRQ routing code. This patch introduces reusable convenient way to obtain this ID from the device pointer.
Since requester ID is an architecture-specific thing, the function can be overridden on per-architecture basis. Default stub just returns 0.
MemTxAttrs.stream_id also renamed to
requester_idin order to better reflect semantics of the field.Signed-off-by Pavel Fedin <[email protected]>
This is a pci requester id, not an msi requester id.
Quoted 121 lines
--- hw/pci/msi.c | 2 +- include/exec/memattrs.h | 4 ++-- include/hw/pci/msi.h | 1 + stubs/Makefile.objs | 1 + stubs/msi.c | 16 ++++++++++++++++ target-arm/Makefile.objs | 1 + target-arm/msi.c | 16 ++++++++++++++++ 7 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 stubs/msi.c create mode 100644 target-arm/msi.chw/pci/msi.c +1
-1@@ -294,7 +294,7 @@ void msi_send_message(PCIDevice *dev, MSIMessage msg) {MemTxAttrs attrs = {}; - attrs.stream_id = (pci_bus_num (dev->bus) <<8 ) | dev->devfn; + attrs.requester_id =msi_requester_id (dev);address_space_stl_le (&dev->bus_master_as, msg.address, msg.data, attrs,NULL ); }include/exec/memattrs.h +2
-2@@ -35,8 +35,8 @@ typedef struct MemTxAttrs { unsigned int secure:1;/* Memory access is usermode (unprivileged) */ unsigned int user:1; -/* Stream ID (for MSI for example) */ - unsigned int stream_id:16; +/* Requester ID (for MSI for example) */ + unsigned int requester_id:16; } MemTxAttrs;/* Bus masters which don't specify any attributes will get this, include/hw/pci/msi.h +1
-0@@ -42,6 +42,7 @@ void msi_notify(PCIDevice *dev, unsigned int vector);void msi_send_message (PCIDevice *dev,MSIMessage msg);void msi_write_config (PCIDevice *dev,uint32_t addr,uint32_t val,int len);unsigned int msi_nr_vectors_allocated (const PCIDevice *dev); +uint16_t msi_requester_id (PCIDevice *dev);static inline bool msi_present (const PCIDevice *dev) {stubs/Makefile.objs +1
-0@@ -22,6 +22,7 @@ stub-obj-y += migr-blocker.o stub-obj-y += mon-is-qmp.o stub-obj-y += mon-printf.o stub-obj-y += monitor-init.o +stub-obj-$(CONFIG_PCI) += msi.o stub-obj-y += notify-event.o stub-obj-$(CONFIG_SPICE) += qemu-chr-open-spice.o stub-obj-y += qtest.ostubs/msi.c new file +16
-0@@ -0,0 +1,16 @@ +/* +* QEMU architecture-specific MSI functions +* +* Copyright (c) 2015 Samsung Electronics Co., Ltd. +* Written by Pavel Fedin +* +* This work is licensed under the terms of the GNU GPL, version 2 or later. +* See the COPYING file in the top-level directory. +* +*/ +#include "hw/pci/msi.h" + +uint16_t msi_requester_id (PCIDevice *dev) +{ +return 0 ; +}target-arm/Makefile.objs +1
-0@@ -1,5 +1,6 @@ obj-y += arm-semi.o obj-$(CONFIG_SOFTMMU) += machine.o +obj-$(CONFIG_PCI) += msi.o obj-$(CONFIG_KVM) += kvm.o obj-$(call land,$(CONFIG_KVM),$(call lnot,$(TARGET_AARCH64))) += kvm32.o obj-$(call land,$(CONFIG_KVM),$(TARGET_AARCH64)) += kvm64.otarget-arm/msi.c new file +16
-0@@ -0,0 +1,16 @@ +/* +* QEMU ARM specific MSI functions +* +* Copyright (c) 2015 Samsung Electronics Co., Ltd. +* Written by Pavel Fedin +* +* This work is licensed under the terms of the GNU GPL, version 2 or later. +* See the COPYING file in the top-level directory. +* +*/ +#include "hw/pci/msi.h" + +uint16_t msi_requester_id (PCIDevice *dev) +{ +return (pci_bus_num (dev->bus) <<8 ) | dev->devfn; +}Signature · 1 line
2.4.4