KVM noncoherent DMA registration and VFIO pseudo device
Subsystems (7)
v2 cover KVM noncoherent DMA registration and VFIO pseudo device
v2:
- Drop KVM device changes
- Add kfree(dev) in destroy function
- Add atomic_set(,0) in kvm_arch init
As described in v1 and discussion, this adds a VFIO pseudo device to KVM through which userspace can register VFIO groups. The initial use of this interface is to enable emulation of coherency instructions on x86 when a VFIO group is attached. Future patches will allow KVM to query the coherency state of the IOMMU domain used by the group to avoid unnecessarily enabling this emulation. Thanks,
Alex
---
Alex Williamson (3):
kvm: Add VFIO device
kvm/x86: Convert iommu_flags to iommu_noncoherent
kvm: Create non-coherent DMA registeration
Documentation/virtual/kvm/devices/vfio.txt | 22 ++ arch/ia64/include/asm/kvm_host.h | 2 arch/x86/include/asm/kvm_host.h | 4 arch/x86/kvm/Kconfig | 1 arch/x86/kvm/Makefile | 2 arch/x86/kvm/vmx.c | 3 arch/x86/kvm/x86.c | 22 ++ include/linux/kvm_host.h | 23 ++ include/uapi/linux/kvm.h | 4 virt/kvm/Kconfig | 3 virt/kvm/iommu.c | 22 +- virt/kvm/kvm_main.c | 5 + virt/kvm/vfio.c | 264 ++++++++++++++++++++++++++++ 13 files changed, 359 insertions(+), 18 deletions(-) create mode 100644 Documentation/virtual/kvm/devices/vfio.txt create mode 100644 virt/kvm/vfio.c
PATCH v2 1/3 kvm: Add VFIO device
So far we've succeeded at making KVM and VFIO mostly unaware of each other, but areas are cropping up where a connection beyond eventfds and irqfds needs to be made. This patch introduces a KVM-VFIO device that is meant to be a gateway for such interaction. The user creates the device and can add and remove VFIO groups to it via file descriptors. When a group is added, KVM verifies the group is valid and gets a reference to it via the VFIO external user interface.
--- Documentation/virtual/kvm/devices/vfio.txt | 22 +++ arch/x86/kvm/Kconfig | 1 arch/x86/kvm/Makefile | 2 include/linux/kvm_host.h | 1 include/uapi/linux/kvm.h | 4 + virt/kvm/Kconfig | 3 virt/kvm/kvm_main.c | 5 + virt/kvm/vfio.c | 220 ++++++++++++++++++++++++++++ 8 files changed, 257 insertions(+), 1 deletion(-) create mode 100644 Documentation/virtual/kvm/devices/vfio.txt create mode 100644 virt/kvm/vfio.c
Documentation/virtual/kvm/devices/vfio.txt new file +22 -0
@@ -0,0 +1,22 @@ +VFIO virtual device +=================== + +Device types supported: + KVM_DEV_TYPE_VFIO + +Only one VFIO instance may be created per VM. The created device +tracks VFIO groups in use by the VM and features of those groups +important to the correctness and acceleration of the VM. As groups +are enabled and disabled for use by the VM, KVM should be updated +about their presence. When registered with KVM, a reference to the +VFIO-group is held by KVM. + +Groups: + KVM_DEV_VFIO_GROUP + +KVM_DEV_VFIO_GROUP attributes: + KVM_DEV_VFIO_GROUP_ADD: Add a VFIO group to VFIO-KVM device tracking + KVM_DEV_VFIO_GROUP_DEL: Remove a VFIO group from VFIO-KVM device tracking + +For each, kvm_device_attr.addr points to an int32_t file descriptor +for the VFIO group.
arch/x86/kvm/Kconfig +1 -0
@@ -38,6 +38,7 @@ config KVM select PERF_EVENTS select HAVE_KVM_MSI select HAVE_KVM_CPU_RELAX_INTERCEPT + select KVM_VFIO ---help--- Support hosting fully virtualized guest machines using hardware virtualization extensions. You will need a fairly recent
arch/x86/kvm/Makefile +1 -1
@@ -9,7 +9,7 @@ KVM := ../../../virt/kvm kvm-y += $(KVM)/kvm_main.o $(KVM)/ioapic.o \ $(KVM)/coalesced_mmio.o $(KVM)/irq_comm.o \ - $(KVM)/eventfd.o $(KVM)/irqchip.o + $(KVM)/eventfd.o $(KVM)/irqchip.o $(KVM)/vfio.o kvm-$(CONFIG_KVM_DEVICE_ASSIGNMENT) += $(KVM)/assigned-dev.o $(KVM)/iommu.o kvm-$(CONFIG_KVM_ASYNC_PF) += $(KVM)/async_pf.o
include/linux/kvm_host.h +1 -0
@@ -1066,6 +1066,7 @@ struct kvm_device *kvm_device_from_filp(struct file *filp);extern struct kvm_device_ops kvm_mpic_ops;extern struct kvm_device_ops kvm_xics_ops; +extern struct kvm_device_ops kvm_vfio_ops;#ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
include/uapi/linux/kvm.h +4 -0
@@ -843,6 +843,10 @@ struct kvm_device_attr {#define KVM_DEV_TYPE_FSL_MPIC_20 1#define KVM_DEV_TYPE_FSL_MPIC_42 2#define KVM_DEV_TYPE_XICS 3 +#define KVM_DEV_TYPE_VFIO 4 +#define KVM_DEV_VFIO_GROUP 1 +#define KVM_DEV_VFIO_GROUP_ADD 1 +#define KVM_DEV_VFIO_GROUP_DEL 2/* * ioctls for VM fds
virt/kvm/Kconfig +3 -0
@@ -27,3 +27,6 @@ config HAVE_KVM_MSI config HAVE_KVM_CPU_RELAX_INTERCEPT bool + +config KVM_VFIO + bool
virt/kvm/kvm_main.c +5 -0
@@ -2271,6 +2271,11 @@ static int kvm_ioctl_create_device(struct kvm *kvm, ops = &kvm_xics_ops;break ;#endif +#ifdef CONFIG_KVM_VFIO +case KVM_DEV_TYPE_VFIO : + ops = &kvm_vfio_ops; +break ; +#endif default :return -ENODEV ; }
virt/kvm/vfio.c new file +220 -0
220 lines not on this page · raw message
PATCH v2 2/3 kvm/x86: Convert iommu_flags to iommu_noncoherent
In reply to Alex WilliamsonDefault to operating in coherent mode. This simplifies the logic when we switch to a model of registering and unregistering noncoherent I/O with KVM.
--- arch/ia64/include/asm/kvm_host.h | 2 +- arch/x86/include/asm/kvm_host.h | 2 +- arch/x86/kvm/vmx.c | 2 +- arch/x86/kvm/x86.c | 2 +- include/linux/kvm_host.h | 3 --- virt/kvm/iommu.c | 16 ++++++++-------- 6 files changed, 12 insertions(+), 15 deletions(-)
arch/ia64/include/asm/kvm_host.h +1 -1
@@ -480,7 +480,7 @@ struct kvm_arch {struct list_head assigned_dev_head;struct iommu_domain *iommu_domain; -int iommu_flags; +bool iommu_noncoherent;unsigned long irq_sources_bitmap;unsigned long irq_states[KVM_IOAPIC_NUM_PINS ];
arch/x86/include/asm/kvm_host.h +1 -1
@@ -557,7 +557,7 @@ struct kvm_arch {struct list_head assigned_dev_head;struct iommu_domain *iommu_domain; -int iommu_flags; +bool iommu_noncoherent;struct kvm_pic *vpic;struct kvm_ioapic *vioapic;struct kvm_pit *vpit;
arch/x86/kvm/vmx.c +1 -1
@@ -7411,7 +7411,7 @@ static u64 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)if (is_mmio) ret =MTRR_TYPE_UNCACHABLE <<VMX_EPT_MT_EPTE_SHIFT ;else if (vcpu->kvm->arch.iommu_domain && - !(vcpu->kvm->arch.iommu_flags &KVM_IOMMU_CACHE_COHERENCY )) + vcpu->kvm->arch.iommu_noncoherent) ret =kvm_get_guest_memory_type (vcpu, gfn) <<VMX_EPT_MT_EPTE_SHIFT ;else
arch/x86/kvm/x86.c +1 -1
@@ -2716,7 +2716,7 @@ static void wbinvd_ipi(void *garbage)static bool need_emulate_wbinvd (struct kvm_vcpu *vcpu) {return vcpu->kvm->arch.iommu_domain && - !(vcpu->kvm->arch.iommu_flags &KVM_IOMMU_CACHE_COHERENCY ); + vcpu->kvm->arch.iommu_noncoherent; }void kvm_arch_vcpu_load (struct kvm_vcpu *vcpu,int cpu)
include/linux/kvm_host.h +0 -3
@@ -747,9 +747,6 @@ void kvm_unregister_irq_ack_notifier(struct kvm *kvm,int kvm_request_irq_source_id (struct kvm *kvm);void kvm_free_irq_source_id (struct kvm *kvm,int irq_source_id); -/* For vcpu->arch.iommu_flags */ -#define KVM_IOMMU_CACHE_COHERENCY 0x1 -#ifdef CONFIG_KVM_DEVICE_ASSIGNMENT int kvm_iommu_map_pages (struct kvm *kvm,struct kvm_memory_slot *slot);void kvm_iommu_unmap_pages (struct kvm *kvm,struct kvm_memory_slot *slot);
virt/kvm/iommu.c +8 -8
@@ -79,7 +79,7 @@ int kvm_iommu_map_pages(struct kvm *kvm, struct kvm_memory_slot *slot) flags =IOMMU_READ ;if (!(slot->flags &KVM_MEM_READONLY )) flags |=IOMMU_WRITE ; -if (kvm->arch.iommu_flags &KVM_IOMMU_CACHE_COHERENCY ) +if (!kvm->arch.iommu_noncoherent) flags |=IOMMU_CACHE ;
@@ -158,7 +158,8 @@ int kvm_assign_device(struct kvm *kvm, {struct pci_dev *pdev =NULL ;struct iommu_domain *domain = kvm->arch.iommu_domain; -int r, last_flags; +int r; +bool noncoherent;/* check if iommu exists and in use */ if (!domain)
@@ -174,15 +175,13 @@ int kvm_assign_device(struct kvm *kvm,return r; } - last_flags = kvm->arch.iommu_flags; -if (iommu_domain_has_cap (kvm->arch.iommu_domain, -IOMMU_CAP_CACHE_COHERENCY )) - kvm->arch.iommu_flags |=KVM_IOMMU_CACHE_COHERENCY ; + noncoherent = !iommu_domain_has_cap (kvm->arch.iommu_domain, +IOMMU_CAP_CACHE_COHERENCY );/* Check if need to update IOMMU page table for guest memory */ -if ((last_flags ^ kvm->arch.iommu_flags) == -KVM_IOMMU_CACHE_COHERENCY ) { +if (noncoherent != kvm->arch.iommu_noncoherent) {kvm_iommu_unmap_memslots (kvm); + kvm->arch.iommu_noncoherent = noncoherent; r =kvm_iommu_map_memslots (kvm);if (r) goto out_unmap;
@@ -350,6 +349,7 @@ int kvm_iommu_unmap_guest(struct kvm *kvm)mutex_lock (&kvm->slots_lock);kvm_iommu_unmap_memslots (kvm); kvm->arch.iommu_domain =NULL ; + kvm->arch.iommu_noncoherent = false;mutex_unlock (&kvm->slots_lock);iommu_domain_free (domain);
PATCH v2 3/3 kvm: Create non-coherent DMA registeration
In reply to Alex WilliamsonWe currently use some ad-hoc arch variables tied to legacy KVM device assignment to manage emulation of instructions that depend on whether non-coherent DMA is present. Create an interface for this, adapting legacy KVM device assignment and adding VFIO via the KVM-VFIO device. For now we assume that non-coherent DMA is possible any time we have a VFIO group. Eventually an interface can be developed as part of the VFIO external user interface to query the coherency of a group.
--- arch/x86/include/asm/kvm_host.h | 2 ++ arch/x86/kvm/vmx.c | 3 +-- arch/x86/kvm/x86.c | 22 ++++++++++++++++++-- include/linux/kvm_host.h | 19 +++++++++++++++++ virt/kvm/iommu.c | 6 +++++ virt/kvm/vfio.c | 44 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 92 insertions(+), 4 deletions(-)
arch/x86/include/asm/kvm_host.h +2 -0
@@ -558,6 +558,8 @@ struct kvm_arch {struct list_head assigned_dev_head;struct iommu_domain *iommu_domain;bool iommu_noncoherent; +#define __KVM_HAVE_ARCH_NONCOHERENT_DMA +atomic_t noncoherent_dma_count;struct kvm_pic *vpic;struct kvm_ioapic *vioapic;struct kvm_pit *vpit;
arch/x86/kvm/vmx.c +1 -2
@@ -7410,8 +7410,7 @@ static u64 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)*/ if (is_mmio) ret =MTRR_TYPE_UNCACHABLE <<VMX_EPT_MT_EPTE_SHIFT ; -else if (vcpu->kvm->arch.iommu_domain && - vcpu->kvm->arch.iommu_noncoherent) +else if (kvm_arch_has_noncoherent_dma (vcpu->kvm)) ret =kvm_get_guest_memory_type (vcpu, gfn) <<VMX_EPT_MT_EPTE_SHIFT ;else
arch/x86/kvm/x86.c +20 -2
@@ -2715,8 +2715,7 @@ static void wbinvd_ipi(void *garbage)static bool need_emulate_wbinvd (struct kvm_vcpu *vcpu) { -return vcpu->kvm->arch.iommu_domain && - vcpu->kvm->arch.iommu_noncoherent; +return kvm_arch_has_noncoherent_dma (vcpu->kvm); }void kvm_arch_vcpu_load (struct kvm_vcpu *vcpu,int cpu)
@@ -6981,6 +6980,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)INIT_LIST_HEAD (&kvm->arch.active_mmu_pages);INIT_LIST_HEAD (&kvm->arch.zapped_obsolete_pages);INIT_LIST_HEAD (&kvm->arch.assigned_dev_head); +atomic_set (&kvm->arch.noncoherent_dma_count,0 );/* Reserve bit 0 of irq_sources_bitmap for userspace irq source */ set_bit (KVM_USERSPACE_IRQ_SOURCE_ID , &kvm->arch.irq_sources_bitmap);
@@ -7420,6 +7420,24 @@ bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu) kvm_x86_ops->interrupt_allowed (vcpu); } +void kvm_arch_register_noncoherent_dma (struct kvm *kvm) +{ +atomic_inc (&kvm->arch.noncoherent_dma_count); +} +EXPORT_SYMBOL_GPL (kvm_arch_register_noncoherent_dma); + +void kvm_arch_unregister_noncoherent_dma (struct kvm *kvm) +{ +atomic_dec (&kvm->arch.noncoherent_dma_count); +} +EXPORT_SYMBOL_GPL (kvm_arch_unregister_noncoherent_dma); + +bool kvm_arch_has_noncoherent_dma (struct kvm *kvm) +{ +return atomic_read (&kvm->arch.noncoherent_dma_count); +} +EXPORT_SYMBOL_GPL (kvm_arch_has_noncoherent_dma); +EXPORT_TRACEPOINT_SYMBOL_GPL (kvm_exit);EXPORT_TRACEPOINT_SYMBOL_GPL (kvm_inj_virq);EXPORT_TRACEPOINT_SYMBOL_GPL (kvm_page_fault);
include/linux/kvm_host.h +19 -0
@@ -671,6 +671,25 @@ static inline void kvm_arch_free_vm(struct kvm *kvm) }#endif +#ifdef __KVM_HAVE_ARCH_NONCOHERENT_DMA +void kvm_arch_register_noncoherent_dma (struct kvm *kvm); +void kvm_arch_unregister_noncoherent_dma (struct kvm *kvm); +bool kvm_arch_has_noncoherent_dma (struct kvm *kvm); +#else +static inline void kvm_arch_register_noncoherent_dma (struct kvm *kvm) +{ +} + +static inline void kvm_arch_unregister_noncoherent_dma (struct kvm *kvm) +{ +} + +static inline bool kvm_arch_has_noncoherent_dma (struct kvm *kvm) +{ +return false; +} +#endif +static inline wait_queue_head_t *kvm_arch_vcpu_wq (struct kvm_vcpu *vcpu) {#ifdef __KVM_HAVE_ARCH_WQP
virt/kvm/iommu.c +6 -0
@@ -140,6 +140,9 @@ static int kvm_iommu_map_memslots(struct kvm *kvm)struct kvm_memslots *slots;struct kvm_memory_slot *memslot; +if (kvm->arch.iommu_noncoherent) +kvm_arch_register_noncoherent_dma (kvm); + idx =srcu_read_lock (&kvm->srcu); slots =kvm_memslots (kvm);
@@ -335,6 +338,9 @@ static int kvm_iommu_unmap_memslots(struct kvm *kvm)srcu_read_unlock (&kvm->srcu, idx); +if (kvm->arch.iommu_noncoherent) +kvm_arch_unregister_noncoherent_dma (kvm); +return 0 ; }
virt/kvm/vfio.c +44 -0
74 lines not on this page · raw message
Il 30/10/2013 18:02, Alex Williamson ha scritto:
Quoted 22 lines
v2:
- Drop KVM device changes
- Add kfree(dev) in destroy function
- Addatomic_set(,0)inkvm_archinitAs described in v1 and discussion, this adds a VFIO pseudo device to KVM through which userspace can register VFIO groups. The initial use of this interface is to enable emulation of coherency instructions on x86 when a VFIO group is attached. Future patches will allow KVM to query the coherency state of the IOMMU domain used by the group to avoid unnecessarily enabling this emulation. Thanks,
Alex
---
Alex Williamson (3):
kvm: Add VFIO device
kvm/x86: Convertiommu_flagstoiommu_noncoherent
kvm: Create non-coherent DMA registeration
Quoted 17 lines
Documentation/virtual/kvm/devices/vfio.txt | 22 ++ arch/ia64/include/asm/kvm_host.h | 2 arch/x86/include/asm/kvm_host.h | 4 arch/x86/kvm/Kconfig | 1 arch/x86/kvm/Makefile | 2 arch/x86/kvm/vmx.c | 3 arch/x86/kvm/x86.c | 22 ++ include/linux/kvm_host.h | 23 ++ include/uapi/linux/kvm.h | 4 virt/kvm/Kconfig | 3 virt/kvm/iommu.c | 22 +- virt/kvm/kvm_main.c | 5 + virt/kvm/vfio.c | 264 ++++++++++++++++++++++++++++ 13 files changed, 359 insertions(+), 18 deletions(-) create mode 100644 Documentation/virtual/kvm/devices/vfio.txt create mode 100644 virt/kvm/vfio.c
Applied to kvm/queue, thanks.
Paolo
RFC PATCH v2 1/1 KVM: PPC: vfio kvm device: support spapr tce
In reply to Alex Williamson---
Changes:
v2:
* it does not try to introduce a realmode search function. Instead, liobn-to-iommu-group lookup is done by VFIO KVM device in virtual mode and the result (iommu_group pointer) is cached in kvm_arch so the realmode handlers do not use VFIO KVM device for that. And the iommu groups get released on KVM termination.
I tried this, seems viable.
Did not I miss anything? Thanks.
--- arch/powerpc/include/asm/kvm_host.h | 3 ++ arch/powerpc/kvm/Kconfig | 1 + arch/powerpc/kvm/Makefile | 3 ++ include/linux/vfio.h | 3 ++ include/uapi/linux/kvm.h | 1 + virt/kvm/vfio.c | 74 +++++++++++++++++++++++++++++++++++++ 6 files changed, 85 insertions(+)
arch/powerpc/include/asm/kvm_host.h +3 -0
@@ -293,6 +293,9 @@ struct kvm_arch {#ifdef CONFIG_KVM_XICS struct kvmppc_xics *xics;#endif +#ifdef CONFIG_KVM_VFIO +struct kvm_vfio *vfio; +#endif };/*
arch/powerpc/kvm/Kconfig +1 -0
@@ -60,6 +60,7 @@ config KVM_BOOK3S_64 select KVM_BOOK3S_64_HANDLER select KVM select SPAPR_TCE_IOMMU + select KVM_VFIO ---help--- Support running unmodified book3s_64 and book3s_32 guest kernels in virtual machines on book3s_64 host processors.
arch/powerpc/kvm/Makefile +3 -0
@@ -87,6 +87,9 @@ kvm-book3s_64-builtin-objs-$(CONFIG_KVM_BOOK3S_64_HV) := \ kvm-book3s_64-objs-$(CONFIG_KVM_XICS) += \ book3s_xics.o +kvm-book3s_64-objs-$(CONFIG_KVM_VFIO) += \ + $(KVM)/vfio.o \ + kvm-book3s_64-module-objs := \ $(KVM)/kvm_main.o \ $(KVM)/eventfd.o \
include/linux/vfio.h +3 -0
@@ -97,4 +97,7 @@ extern struct vfio_group *vfio_group_get_external_user(struct file *filep);extern void vfio_group_put_external_user (struct vfio_group *group);extern int vfio_external_user_iommu_id (struct vfio_group *group); +extern struct iommu_group *vfio_find_group_by_liobn (struct kvm *kvm, +unsigned long liobn); +#endif /* VFIO_H */
include/uapi/linux/kvm.h +1 -0
@@ -847,6 +847,7 @@ struct kvm_device_attr {#define KVM_DEV_VFIO_GROUP 1#define KVM_DEV_VFIO_GROUP_ADD 1#define KVM_DEV_VFIO_GROUP_DEL 2 +#define KVM_DEV_VFIO_SPAPR_TCE_LIOBN 2/* * ioctls for VM fds
virt/kvm/vfio.c +73 -0
103 lines not on this page · raw message
Signature · 1 line
1.8.4.rc4
Re: [RFC PATCH v2] KVM: PPC: vfio kvm device: support spapr tce
Quoted 11 lines · Alexey Kardashevskiy
Signed-off-by Alexey Kardashevskiy <[email protected]>---
Changes:
v2:
* it does not try to introduce a realmode search function. Instead, liobn-to-iommu-group lookup is done by VFIO KVM device in virtual mode and the result (iommu_grouppointer) is cached inkvm_archso the realmode handlers do not use VFIO KVM device for that. And the iommu groups get released on KVM termination.
Quoted 3 lines
I tried this, seems viable.
Did not I miss anything? Thanks.
A commit message ;)
Quoted 54 lines
--- arch/powerpc/include/asm/kvm_host.h | 3 ++ arch/powerpc/kvm/Kconfig | 1 + arch/powerpc/kvm/Makefile | 3 ++ include/linux/vfio.h | 3 ++ include/uapi/linux/kvm.h | 1 + virt/kvm/vfio.c | 74 +++++++++++++++++++++++++++++++++++++ 6 files changed, 85 insertions(+)arch/powerpc/include/asm/kvm_host.h +3
-0@@ -293,6 +293,9 @@ struct kvm_arch {#ifdef CONFIG_KVM_XICS struct kvmppc_xics *xics;#endif +#ifdef CONFIG_KVM_VFIO +struct kvm_vfio *vfio; +#endif };/* arch/powerpc/kvm/Kconfig +1
-0@@ -60,6 +60,7 @@ config KVM_BOOK3S_64 select KVM_BOOK3S_64_HANDLER select KVM select SPAPR_TCE_IOMMU + select KVM_VFIO ---help--- Support running unmodified book3s_64 and book3s_32 guest kernels in virtual machines on book3s_64 host processors.arch/powerpc/kvm/Makefile +3
-0@@ -87,6 +87,9 @@ kvm-book3s_64-builtin-objs-$(CONFIG_KVM_BOOK3S_64_HV) := \ kvm-book3s_64-objs-$(CONFIG_KVM_XICS) += \ book3s_xics.o +kvm-book3s_64-objs-$(CONFIG_KVM_VFIO) += \ + $(KVM)/vfio.o \ + kvm-book3s_64-module-objs := \ $(KVM)/kvm_main.o \ $(KVM)/eventfd.o \include/linux/vfio.h +0
-0
include/linux/vfio.h extern struct vfio_group *vfio_group_get_external_user(struct file *filep); · line 102
Quoted 6 lines
include/linux/vfio.h +3
-0@@ -97,4 +97,7 @@ extern struct vfio_group *vfio_group_get_external_user(struct file *filep);extern void vfio_group_put_external_user (struct vfio_group *group);extern int vfio_external_user_iommu_id (struct vfio_group *group); +extern struct iommu_group *vfio_find_group_by_liobn (struct kvm *kvm, +unsigned long liobn); +
Nope, this doesn't go in vfio.h, it's a function provided by kvm. It should be named as such too, kvm_vfio_... It also depends on both CONFIG_KVM_VFIO and CONFIG_SPAPR_TCE_IOMMU and needs stub version otherwise. Is just _liobn specific enough or does it need a spapr_tce thrown in to avoid confusion with embedded ppc folks?
include/uapi/linux/kvm.h struct kvm_device_attr { · line 850
Quoted 10 lines
include/linux/vfio.h +0
-0#endif /* VFIO_H */ include/uapi/linux/kvm.h +1
-0@@ -847,6 +847,7 @@ struct kvm_device_attr {#define KVM_DEV_VFIO_GROUP 1#define KVM_DEV_VFIO_GROUP_ADD 1#define KVM_DEV_VFIO_GROUP_DEL 2 +#define KVM_DEV_VFIO_SPAPR_TCE_LIOBN 2
I wonder if it would be better architecturally if this was an attribute rather than a new group, ex:
#define KVM_DEV_VFIO_GROUP_SET_SPAPR_TCE_LIOBN 3
It's a mouthful, but we are setting an attribute of a VFIO group, so it makes sense. kvm_device_attr.addr would then need to point to a struct containing both the fd and liobn.
Whatever we come up with need a documentation addition in Documentation/virtual/kvm/devices/vfio.txt.
virt/kvm/vfio.c line 26
Quoted 13 lines
include/uapi/linux/kvm.h +0
-0/* * ioctls for VM fds virt/kvm/vfio.c +2
-0@@ -22,6 +22,9 @@struct kvm_vfio_group {struct list_head node;struct vfio_group *vfio_group; +#ifdef CONFIG_SPAPR_TCE_IOMMU +uint64_t liobn;
Why is liobn an unsigned long in the exported function but a uint64_t here?
Quoted 19 lines
virt/kvm/vfio.c +12
-0+#endif };struct kvm_vfio {@@ -188,12 +191,76 @@ static int kvm_vfio_set_group(struct kvm_device *dev, long attr, u64 arg)return -ENXIO ; } +#ifdef CONFIG_SPAPR_TCE_IOMMU +static int kvm_vfio_set_spapr_tce_liobn (struct kvm_device *dev, +long attr,u64 arg) +{ +struct kvm_vfio *kv = dev->private; +struct vfio_group *vfio_group; +struct kvm_vfio_group *kvg; +void __user *argp = (void __user *)arg; +struct fd f; +int32_t fd; +uint64_t liobn = attr;
virt/kvm/vfio.c static int kvm_vfio_set_group(struct kvm_device *dev, long attr, u64 arg) · line 214
Quoted 10 lines
virt/kvm/vfio.c +10
-0+ +if (get_user (fd, (int32_t __user *)argp)) +return -EFAULT ; + + f =fdget (fd); +if (!f.file) +return -EBADF ; + + vfio_group =kvm_vfio_group_get_external_user (f.file); +fdput (f);
Not sure why you dropped this from the example of kvm_vfio_set_group:
if (IS_ERR(vfio_group))
return PTR_ERR(vfio_group);You're also ignoring kv->lock.
virt/kvm/vfio.c static int kvm_vfio_set_group(struct kvm_device *dev, long attr, u64 arg) · line 218
Quoted 4 lines
virt/kvm/vfio.c +4
-0+ +list_for_each_entry (kvg, &kv->group_list, node) { +if (kvg->vfio_group == vfio_group) { +WARN_ON (kvg->liobn);
Userspace should not be able to trigger a WARN this easily. Return EBUSY if it's an error, otherwise let it go.
virt/kvm/vfio.c static int kvm_vfio_set_group(struct kvm_device *dev, long attr, u64 arg) · line 231
Quoted 13 lines
virt/kvm/vfio.c +13
-0+ kvg->liobn = liobn; +kvm_vfio_group_put_external_user (vfio_group); +return 0 ; + } + } + +kvm_vfio_group_put_external_user (vfio_group); + +return -ENXIO ; +} + +struct iommu_group *vfio_find_group_by_liobn (struct kvm *kvm, +unsigned long liobn)
As mentioned, kvm_vfio_...
virt/kvm/vfio.c static int kvm_vfio_set_group(struct kvm_device *dev, long attr, u64 arg) · line 236
Quoted 5 lines
virt/kvm/vfio.c +5
-0+{ +struct kvm_vfio_group *kvg; + +if (!kvm->arch.vfio) +return NULL ;
If this is already a slow path you can avoid stashing this pointer and search kvm->devices for the matching ops struct.
kv->lock...
virt/kvm/vfio.c static int kvm_vfio_set_group(struct kvm_device *dev, long attr, u64 arg)
Quoted 7 lines
virt/kvm/vfio.c +7
-0+ +list_for_each_entry (kvg, &kvm->arch.vfio->group_list, node) { +if (kvg->liobn == liobn) { +int group_id =vfio_external_user_iommu_id ( + kvg->vfio_group); +struct iommu_group *grp = +iommu_group_get_by_id (group_id);
nit, ugly line wrapping. Where's the iommu_group_put() done?
virt/kvm/vfio.c static int kvm_vfio_set_group(struct kvm_device *dev, long attr, u64 arg) · line 243
Quoted 1 line
virt/kvm/vfio.c +1
-0+return grp;
So you've now got an liobn to iommu_group mapping cached away somewhere, what happens when a group is removed? Would it be invalid for userspace to re-use the liobn? Do we need a way to invalidate your cached entry and perhaps do an iommu_group_put()?
This version is actually plausible, so big improvement from v1! Thanks,
Alex
Quoted 43 lines
virt/kvm/vfio.c +20
-0+ } + } + +return NULL ; +} +EXPORT_SYMBOL_GPL (vfio_find_group_by_liobn); +#endif +static int kvm_vfio_set_attr (struct kvm_device *dev,struct kvm_device_attr *attr) {switch (attr->group) {case KVM_DEV_VFIO_GROUP :return kvm_vfio_set_group (dev, attr->attr, attr->addr); +#ifdef CONFIG_SPAPR_TCE_IOMMU +case KVM_DEV_VFIO_SPAPR_TCE_LIOBN : +return kvm_vfio_set_spapr_tce_liobn (dev, attr->attr, + attr->addr); +#endif }return -ENXIO ;@@ -211,6 +278,10 @@ static int kvm_vfio_has_attr(struct kvm_device *dev, }break ; +#ifdef CONFIG_SPAPR_TCE_IOMMU +case KVM_DEV_VFIO_SPAPR_TCE_LIOBN : +return 0 ; +#endif }return -ENXIO ;@@ -251,6 +322,9 @@ static int kvm_vfio_create(struct kvm_device *dev, u32 type)mutex_init (&kv->lock); dev->private = kv; +#ifdef CONFIG_SPAPR_TCE_IOMMU + dev->kvm->arch.vfio = kv; +#endif return 0 ; }