Skip to contentSkip to conversation

KVM noncoherent DMA registration and VFIO pseudo device

PATCH Applied Alex Williamson 3 participants 7 messages Oct 30, 2013 – Nov 5, 2013

Subsystems (7)
Documentation x86 KVM PowerPC +3 more subsystems
lore
Share & tools
lore
#0 lore raw

v2 cover KVM noncoherent DMA registration and VFIO pseudo device

v2:
- Drop KVM device changes
- Add kfree(dev) in destroy function
- Add atomic_set(,0) in kvm_arch init

As described in v1 and discussion, this adds a VFIO pseudo device to KVM through which userspace can register VFIO groups. The initial use of this interface is to enable emulation of coherency instructions on x86 when a VFIO group is attached. Future patches will allow KVM to query the coherency state of the IOMMU domain used by the group to avoid unnecessarily enabling this emulation. Thanks,

Alex

---

Alex Williamson (3):
kvm: Add VFIO device
kvm/x86: Convert iommu_flags to iommu_noncoherent
kvm: Create non-coherent DMA registeration

 Documentation/virtual/kvm/devices/vfio.txt |   22 ++
 arch/ia64/include/asm/kvm_host.h           |    2 
 arch/x86/include/asm/kvm_host.h            |    4 
 arch/x86/kvm/Kconfig                       |    1 
 arch/x86/kvm/Makefile                      |    2 
 arch/x86/kvm/vmx.c                         |    3 
 arch/x86/kvm/x86.c                         |   22 ++
 include/linux/kvm_host.h                   |   23 ++
 include/uapi/linux/kvm.h                   |    4 
 virt/kvm/Kconfig                           |    3 
 virt/kvm/iommu.c                           |   22 +-
 virt/kvm/kvm_main.c                        |    5 +
 virt/kvm/vfio.c                            |  264 ++++++++++++++++++++++++++++
 13 files changed, 359 insertions(+), 18 deletions(-)
 create mode 100644 Documentation/virtual/kvm/devices/vfio.txt
 create mode 100644 virt/kvm/vfio.c
#1 lore raw

PATCH v2 1/3 kvm: Add VFIO device

So far we've succeeded at making KVM and VFIO mostly unaware of each other, but areas are cropping up where a connection beyond eventfds and irqfds needs to be made. This patch introduces a KVM-VFIO device that is meant to be a gateway for such interaction. The user creates the device and can add and remove VFIO groups to it via file descriptors. When a group is added, KVM verifies the group is valid and gets a reference to it via the VFIO external user interface.

---
 Documentation/virtual/kvm/devices/vfio.txt |   22 +++
 arch/x86/kvm/Kconfig                       |    1 
 arch/x86/kvm/Makefile                      |    2 
 include/linux/kvm_host.h                   |    1 
 include/uapi/linux/kvm.h                   |    4 +
 virt/kvm/Kconfig                           |    3 
 virt/kvm/kvm_main.c                        |    5 +
 virt/kvm/vfio.c                            |  220 ++++++++++++++++++++++++++++
 8 files changed, 257 insertions(+), 1 deletion(-)
 create mode 100644 Documentation/virtual/kvm/devices/vfio.txt
 create mode 100644 virt/kvm/vfio.c
Documentation/virtual/kvm/devices/vfio.txt new file +22 -0
new file mode 100644
index 0000000..ef51740
@@ -0,0 +1,22 @@
+VFIO virtual device
+===================
+
+Device types supported:
+  KVM_DEV_TYPE_VFIO
+
+Only one VFIO instance may be created per VM.  The created device
+tracks VFIO groups in use by the VM and features of those groups
+important to the correctness and acceleration of the VM.  As groups
+are enabled and disabled for use by the VM, KVM should be updated
+about their presence.  When registered with KVM, a reference to the
+VFIO-group is held by KVM.
+
+Groups:
+  KVM_DEV_VFIO_GROUP
+
+KVM_DEV_VFIO_GROUP attributes:
+  KVM_DEV_VFIO_GROUP_ADD: Add a VFIO group to VFIO-KVM device tracking
+  KVM_DEV_VFIO_GROUP_DEL: Remove a VFIO group from VFIO-KVM device tracking
+
+For each, kvm_device_attr.addr points to an int32_t file descriptor
+for the VFIO group.
arch/x86/kvm/Kconfig +1 -0
index a47a3e5..b89c5db 100644
@@ -38,6 +38,7 @@ config KVM
 	select PERF_EVENTS
 	select HAVE_KVM_MSI
 	select HAVE_KVM_CPU_RELAX_INTERCEPT
+	select KVM_VFIO
 	---help---
 	  Support hosting fully virtualized guest machines using hardware
 	  virtualization extensions.  You will need a fairly recent
arch/x86/kvm/Makefile +1 -1
index bf4fb04..25d22b2 100644
@@ -9,7 +9,7 @@ KVM := ../../../virt/kvm
 
 kvm-y			+= $(KVM)/kvm_main.o $(KVM)/ioapic.o \
 				$(KVM)/coalesced_mmio.o $(KVM)/irq_comm.o \
-				$(KVM)/eventfd.o $(KVM)/irqchip.o
+				$(KVM)/eventfd.o $(KVM)/irqchip.o $(KVM)/vfio.o
 kvm-$(CONFIG_KVM_DEVICE_ASSIGNMENT)	+= $(KVM)/assigned-dev.o $(KVM)/iommu.o
 kvm-$(CONFIG_KVM_ASYNC_PF)	+= $(KVM)/async_pf.o
 
include/linux/kvm_host.h +1 -0
index 0fbbc7a..279002b 100644
@@ -1066,6 +1066,7 @@ struct kvm_device *kvm_device_from_filp(struct file *filp);
 
 extern struct kvm_device_ops kvm_mpic_ops;
 extern struct kvm_device_ops kvm_xics_ops;
+extern struct kvm_device_ops kvm_vfio_ops;
 
 #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
 
include/uapi/linux/kvm.h +4 -0
index 99c2533..7c1a349 100644
@@ -843,6 +843,10 @@ struct kvm_device_attr {
 #define KVM_DEV_TYPE_FSL_MPIC_20	1
 #define KVM_DEV_TYPE_FSL_MPIC_42	2
 #define KVM_DEV_TYPE_XICS		3
+#define KVM_DEV_TYPE_VFIO		4
+#define  KVM_DEV_VFIO_GROUP			1
+#define   KVM_DEV_VFIO_GROUP_ADD			1
+#define   KVM_DEV_VFIO_GROUP_DEL			2
 
 /*
  * ioctls for VM fds
virt/kvm/Kconfig +3 -0
index 779262f..fbe1a48 100644
@@ -27,3 +27,6 @@ config HAVE_KVM_MSI
 
 config HAVE_KVM_CPU_RELAX_INTERCEPT
        bool
+
+config KVM_VFIO
+       bool
virt/kvm/kvm_main.c +5 -0
index a9dd682..1053fe6 100644
@@ -2271,6 +2271,11 @@ static int kvm_ioctl_create_device(struct kvm *kvm,
 		ops = &kvm_xics_ops;
 		break;
 #endif
+#ifdef CONFIG_KVM_VFIO
+	case KVM_DEV_TYPE_VFIO:
+		ops = &kvm_vfio_ops;
+		break;
+#endif
 	default:
 		return -ENODEV;
 	}
virt/kvm/vfio.c new file +220 -0

220 lines not on this page · raw message

#2 lore raw

PATCH v2 2/3 kvm/x86: Convert iommu_flags to iommu_noncoherent

In reply to Alex Williamson

Default to operating in coherent mode. This simplifies the logic when we switch to a model of registering and unregistering noncoherent I/O with KVM.

---
 arch/ia64/include/asm/kvm_host.h |    2 +-
 arch/x86/include/asm/kvm_host.h  |    2 +-
 arch/x86/kvm/vmx.c               |    2 +-
 arch/x86/kvm/x86.c               |    2 +-
 include/linux/kvm_host.h         |    3 ---
 virt/kvm/iommu.c                 |   16 ++++++++--------
 6 files changed, 12 insertions(+), 15 deletions(-)
arch/ia64/include/asm/kvm_host.h +1 -1
index 989dd3f..1933b7a 100644
@@ -480,7 +480,7 @@ struct kvm_arch {
 
 	struct list_head assigned_dev_head;
 	struct iommu_domain *iommu_domain;
-	int iommu_flags;
+	bool iommu_noncoherent;
 
 	unsigned long irq_sources_bitmap;
 	unsigned long irq_states[KVM_IOAPIC_NUM_PINS];
arch/x86/include/asm/kvm_host.h +1 -1
index c76ff74..1b6b5f9 100644
@@ -557,7 +557,7 @@ struct kvm_arch {
 
 	struct list_head assigned_dev_head;
 	struct iommu_domain *iommu_domain;
-	int iommu_flags;
+	bool iommu_noncoherent;
 	struct kvm_pic *vpic;
 	struct kvm_ioapic *vioapic;
 	struct kvm_pit *vpit;
arch/x86/kvm/vmx.c +1 -1
index 2b2fce1..5d0e3ab 100644
@@ -7411,7 +7411,7 @@ static u64 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
 	if (is_mmio)
 		ret = MTRR_TYPE_UNCACHABLE << VMX_EPT_MT_EPTE_SHIFT;
 	else if (vcpu->kvm->arch.iommu_domain &&
-		!(vcpu->kvm->arch.iommu_flags & KVM_IOMMU_CACHE_COHERENCY))
+		 vcpu->kvm->arch.iommu_noncoherent)
 		ret = kvm_get_guest_memory_type(vcpu, gfn) <<
 		      VMX_EPT_MT_EPTE_SHIFT;
 	else
arch/x86/kvm/x86.c +1 -1
index e5ca72a..b1231b0 100644
@@ -2716,7 +2716,7 @@ static void wbinvd_ipi(void *garbage)
 static bool need_emulate_wbinvd(struct kvm_vcpu *vcpu)
 {
 	return vcpu->kvm->arch.iommu_domain &&
-		!(vcpu->kvm->arch.iommu_flags & KVM_IOMMU_CACHE_COHERENCY);
+	       vcpu->kvm->arch.iommu_noncoherent;
 }
 
 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
include/linux/kvm_host.h +0 -3
index 279002b..46ce3d1 100644
@@ -747,9 +747,6 @@ void kvm_unregister_irq_ack_notifier(struct kvm *kvm,
 int kvm_request_irq_source_id(struct kvm *kvm);
 void kvm_free_irq_source_id(struct kvm *kvm, int irq_source_id);
 
-/* For vcpu->arch.iommu_flags */
-#define KVM_IOMMU_CACHE_COHERENCY	0x1
-
 #ifdef CONFIG_KVM_DEVICE_ASSIGNMENT
 int kvm_iommu_map_pages(struct kvm *kvm, struct kvm_memory_slot *slot);
 void kvm_iommu_unmap_pages(struct kvm *kvm, struct kvm_memory_slot *slot);
virt/kvm/iommu.c +8 -8
index 72a130b..9cde444 100644
@@ -79,7 +79,7 @@ int kvm_iommu_map_pages(struct kvm *kvm, struct kvm_memory_slot *slot)
 	flags = IOMMU_READ;
 	if (!(slot->flags & KVM_MEM_READONLY))
 		flags |= IOMMU_WRITE;
-	if (kvm->arch.iommu_flags & KVM_IOMMU_CACHE_COHERENCY)
+	if (!kvm->arch.iommu_noncoherent)
 		flags |= IOMMU_CACHE;
 
 
@@ -158,7 +158,8 @@ int kvm_assign_device(struct kvm *kvm,
 {
 	struct pci_dev *pdev = NULL;
 	struct iommu_domain *domain = kvm->arch.iommu_domain;
-	int r, last_flags;
+	int r;
+	bool noncoherent;
 
 	/* check if iommu exists and in use */
 	if (!domain)
@@ -174,15 +175,13 @@ int kvm_assign_device(struct kvm *kvm,
 		return r;
 	}
 
-	last_flags = kvm->arch.iommu_flags;
-	if (iommu_domain_has_cap(kvm->arch.iommu_domain,
-				 IOMMU_CAP_CACHE_COHERENCY))
-		kvm->arch.iommu_flags |= KVM_IOMMU_CACHE_COHERENCY;
+	noncoherent = !iommu_domain_has_cap(kvm->arch.iommu_domain,
+					    IOMMU_CAP_CACHE_COHERENCY);
 
 	/* Check if need to update IOMMU page table for guest memory */
-	if ((last_flags ^ kvm->arch.iommu_flags) ==
-			KVM_IOMMU_CACHE_COHERENCY) {
+	if (noncoherent != kvm->arch.iommu_noncoherent) {
 		kvm_iommu_unmap_memslots(kvm);
+		kvm->arch.iommu_noncoherent = noncoherent;
 		r = kvm_iommu_map_memslots(kvm);
 		if (r)
 			goto out_unmap;
@@ -350,6 +349,7 @@ int kvm_iommu_unmap_guest(struct kvm *kvm)
 	mutex_lock(&kvm->slots_lock);
 	kvm_iommu_unmap_memslots(kvm);
 	kvm->arch.iommu_domain = NULL;
+	kvm->arch.iommu_noncoherent = false;
 	mutex_unlock(&kvm->slots_lock);
 
 	iommu_domain_free(domain);
#3 lore raw

PATCH v2 3/3 kvm: Create non-coherent DMA registeration

In reply to Alex Williamson

We currently use some ad-hoc arch variables tied to legacy KVM device assignment to manage emulation of instructions that depend on whether non-coherent DMA is present. Create an interface for this, adapting legacy KVM device assignment and adding VFIO via the KVM-VFIO device. For now we assume that non-coherent DMA is possible any time we have a VFIO group. Eventually an interface can be developed as part of the VFIO external user interface to query the coherency of a group.

---
 arch/x86/include/asm/kvm_host.h |    2 ++
 arch/x86/kvm/vmx.c              |    3 +--
 arch/x86/kvm/x86.c              |   22 ++++++++++++++++++--
 include/linux/kvm_host.h        |   19 +++++++++++++++++
 virt/kvm/iommu.c                |    6 +++++
 virt/kvm/vfio.c                 |   44 +++++++++++++++++++++++++++++++++++++++
 6 files changed, 92 insertions(+), 4 deletions(-)
arch/x86/include/asm/kvm_host.h +2 -0
index 1b6b5f9..50c1e9c1 100644
@@ -558,6 +558,8 @@ struct kvm_arch {
 	struct list_head assigned_dev_head;
 	struct iommu_domain *iommu_domain;
 	bool iommu_noncoherent;
+#define __KVM_HAVE_ARCH_NONCOHERENT_DMA
+	atomic_t noncoherent_dma_count;
 	struct kvm_pic *vpic;
 	struct kvm_ioapic *vioapic;
 	struct kvm_pit *vpit;
arch/x86/kvm/vmx.c +1 -2
index 5d0e3ab..f7fe08a 100644
@@ -7410,8 +7410,7 @@ static u64 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
 	 */
 	if (is_mmio)
 		ret = MTRR_TYPE_UNCACHABLE << VMX_EPT_MT_EPTE_SHIFT;
-	else if (vcpu->kvm->arch.iommu_domain &&
-		 vcpu->kvm->arch.iommu_noncoherent)
+	else if (kvm_arch_has_noncoherent_dma(vcpu->kvm))
 		ret = kvm_get_guest_memory_type(vcpu, gfn) <<
 		      VMX_EPT_MT_EPTE_SHIFT;
 	else
arch/x86/kvm/x86.c +20 -2
index b1231b0..92e1722 100644
@@ -2715,8 +2715,7 @@ static void wbinvd_ipi(void *garbage)
 
 static bool need_emulate_wbinvd(struct kvm_vcpu *vcpu)
 {
-	return vcpu->kvm->arch.iommu_domain &&
-	       vcpu->kvm->arch.iommu_noncoherent;
+	return kvm_arch_has_noncoherent_dma(vcpu->kvm);
 }
 
 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
@@ -6981,6 +6980,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 	INIT_LIST_HEAD(&kvm->arch.active_mmu_pages);
 	INIT_LIST_HEAD(&kvm->arch.zapped_obsolete_pages);
 	INIT_LIST_HEAD(&kvm->arch.assigned_dev_head);
+	atomic_set(&kvm->arch.noncoherent_dma_count, 0);
 
 	/* Reserve bit 0 of irq_sources_bitmap for userspace irq source */
 	set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap);
@@ -7420,6 +7420,24 @@ bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
 			kvm_x86_ops->interrupt_allowed(vcpu);
 }
 
+void kvm_arch_register_noncoherent_dma(struct kvm *kvm)
+{
+	atomic_inc(&kvm->arch.noncoherent_dma_count);
+}
+EXPORT_SYMBOL_GPL(kvm_arch_register_noncoherent_dma);
+
+void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm)
+{
+	atomic_dec(&kvm->arch.noncoherent_dma_count);
+}
+EXPORT_SYMBOL_GPL(kvm_arch_unregister_noncoherent_dma);
+
+bool kvm_arch_has_noncoherent_dma(struct kvm *kvm)
+{
+	return atomic_read(&kvm->arch.noncoherent_dma_count);
+}
+EXPORT_SYMBOL_GPL(kvm_arch_has_noncoherent_dma);
+
 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_exit);
 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_inj_virq);
 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_page_fault);
include/linux/kvm_host.h +19 -0
index 46ce3d1..a225349 100644
@@ -671,6 +671,25 @@ static inline void kvm_arch_free_vm(struct kvm *kvm)
 }
 #endif
 
+#ifdef __KVM_HAVE_ARCH_NONCOHERENT_DMA
+void kvm_arch_register_noncoherent_dma(struct kvm *kvm);
+void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm);
+bool kvm_arch_has_noncoherent_dma(struct kvm *kvm);
+#else
+static inline void kvm_arch_register_noncoherent_dma(struct kvm *kvm)
+{
+}
+
+static inline void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm)
+{
+}
+
+static inline bool kvm_arch_has_noncoherent_dma(struct kvm *kvm)
+{
+	return false;
+}
+#endif
+
 static inline wait_queue_head_t *kvm_arch_vcpu_wq(struct kvm_vcpu *vcpu)
 {
 #ifdef __KVM_HAVE_ARCH_WQP
virt/kvm/iommu.c +6 -0
index 9cde444..0a54456 100644
@@ -140,6 +140,9 @@ static int kvm_iommu_map_memslots(struct kvm *kvm)
 	struct kvm_memslots *slots;
 	struct kvm_memory_slot *memslot;
 
+	if (kvm->arch.iommu_noncoherent)
+		kvm_arch_register_noncoherent_dma(kvm);
+
 	idx = srcu_read_lock(&kvm->srcu);
 	slots = kvm_memslots(kvm);
 
@@ -335,6 +338,9 @@ static int kvm_iommu_unmap_memslots(struct kvm *kvm)
 
 	srcu_read_unlock(&kvm->srcu, idx);
 
+	if (kvm->arch.iommu_noncoherent)
+		kvm_arch_unregister_noncoherent_dma(kvm);
+
 	return 0;
 }
 
virt/kvm/vfio.c +44 -0

74 lines not on this page · raw message

maintainer Applied
#4 lore raw
In reply to Alex Williamson

Il 30/10/2013 18:02, Alex Williamson ha scritto:

Quoted 22 lines

v2:
- Drop KVM device changes
- Add kfree(dev) in destroy function
- Add atomic_set(,0) in kvm_arch init

As described in v1 and discussion, this adds a VFIO pseudo device to KVM through which userspace can register VFIO groups. The initial use of this interface is to enable emulation of coherency instructions on x86 when a VFIO group is attached. Future patches will allow KVM to query the coherency state of the IOMMU domain used by the group to avoid unnecessarily enabling this emulation. Thanks,

Alex

---

Alex Williamson (3):
kvm: Add VFIO device
kvm/x86: Convert iommu_flags to iommu_noncoherent
kvm: Create non-coherent DMA registeration

Quoted 17 lines
 Documentation/virtual/kvm/devices/vfio.txt |   22 ++
 arch/ia64/include/asm/kvm_host.h           |    2 
 arch/x86/include/asm/kvm_host.h            |    4 
 arch/x86/kvm/Kconfig                       |    1 
 arch/x86/kvm/Makefile                      |    2 
 arch/x86/kvm/vmx.c                         |    3 
 arch/x86/kvm/x86.c                         |   22 ++
 include/linux/kvm_host.h                   |   23 ++
 include/uapi/linux/kvm.h                   |    4 
 virt/kvm/Kconfig                           |    3 
 virt/kvm/iommu.c                           |   22 +-
 virt/kvm/kvm_main.c                        |    5 +
 virt/kvm/vfio.c                            |  264 ++++++++++++++++++++++++++++
 13 files changed, 359 insertions(+), 18 deletions(-)
 create mode 100644 Documentation/virtual/kvm/devices/vfio.txt
 create mode 100644 virt/kvm/vfio.c

Applied to kvm/queue, thanks.

Paolo

#5 lore raw

RFC PATCH v2 1/1 KVM: PPC: vfio kvm device: support spapr tce

In reply to Alex Williamson

---

Changes:
v2:
* it does not try to introduce a realmode search function. Instead, liobn-to-iommu-group lookup is done by VFIO KVM device in virtual mode and the result (iommu_group pointer) is cached in kvm_arch so the realmode handlers do not use VFIO KVM device for that. And the iommu groups get released on KVM termination.

I tried this, seems viable.

Did not I miss anything? Thanks.

---
 arch/powerpc/include/asm/kvm_host.h |  3 ++
 arch/powerpc/kvm/Kconfig            |  1 +
 arch/powerpc/kvm/Makefile           |  3 ++
 include/linux/vfio.h                |  3 ++
 include/uapi/linux/kvm.h            |  1 +
 virt/kvm/vfio.c                     | 74 +++++++++++++++++++++++++++++++++++++
 6 files changed, 85 insertions(+)
arch/powerpc/include/asm/kvm_host.h +3 -0
index 48dbe8b..e1163d7 100644
@@ -293,6 +293,9 @@ struct kvm_arch {
 #ifdef CONFIG_KVM_XICS
 	struct kvmppc_xics *xics;
 #endif
+#ifdef CONFIG_KVM_VFIO
+	struct kvm_vfio *vfio;
+#endif
 };
 
 /*
arch/powerpc/kvm/Kconfig +1 -0
index 61b3535..d1b7f64 100644
@@ -60,6 +60,7 @@ config KVM_BOOK3S_64
 	select KVM_BOOK3S_64_HANDLER
 	select KVM
 	select SPAPR_TCE_IOMMU
+	select KVM_VFIO
 	---help---
 	  Support running unmodified book3s_64 and book3s_32 guest kernels
 	  in virtual machines on book3s_64 host processors.
arch/powerpc/kvm/Makefile +3 -0
index 6646c95..2438d2e 100644
@@ -87,6 +87,9 @@ kvm-book3s_64-builtin-objs-$(CONFIG_KVM_BOOK3S_64_HV) := \
 kvm-book3s_64-objs-$(CONFIG_KVM_XICS) += \
 	book3s_xics.o
 
+kvm-book3s_64-objs-$(CONFIG_KVM_VFIO) += \
+	$(KVM)/vfio.o \
+
 kvm-book3s_64-module-objs := \
 	$(KVM)/kvm_main.o \
 	$(KVM)/eventfd.o \
include/linux/vfio.h +3 -0
index 24579a0..681e19b 100644
@@ -97,4 +97,7 @@ extern struct vfio_group *vfio_group_get_external_user(struct file *filep);
 extern void vfio_group_put_external_user(struct vfio_group *group);
 extern int vfio_external_user_iommu_id(struct vfio_group *group);
 
+extern struct iommu_group *vfio_find_group_by_liobn(struct kvm *kvm,
+		unsigned long liobn);
+
 #endif /* VFIO_H */
include/uapi/linux/kvm.h +1 -0
index 7c1a349..a74ad16 100644
@@ -847,6 +847,7 @@ struct kvm_device_attr {
 #define  KVM_DEV_VFIO_GROUP			1
 #define   KVM_DEV_VFIO_GROUP_ADD			1
 #define   KVM_DEV_VFIO_GROUP_DEL			2
+#define  KVM_DEV_VFIO_SPAPR_TCE_LIOBN		2
 
 /*
  * ioctls for VM fds
virt/kvm/vfio.c +73 -0

103 lines not on this page · raw message

Signature · 1 line
1.8.4.rc4
#6 lore raw

Re: [RFC PATCH v2] KVM: PPC: vfio kvm device: support spapr tce

Quoted 11 lines · Alexey Kardashevskiy

---

Changes:
v2:
* it does not try to introduce a realmode search function. Instead, liobn-to-iommu-group lookup is done by VFIO KVM device in virtual mode and the result (iommu_group pointer) is cached in kvm_arch so the realmode handlers do not use VFIO KVM device for that. And the iommu groups get released on KVM termination.

Quoted 3 lines

I tried this, seems viable.

Did not I miss anything? Thanks.

A commit message ;)

Quoted 54 lines
---
 arch/powerpc/include/asm/kvm_host.h |  3 ++
 arch/powerpc/kvm/Kconfig            |  1 +
 arch/powerpc/kvm/Makefile           |  3 ++
 include/linux/vfio.h                |  3 ++
 include/uapi/linux/kvm.h            |  1 +
 virt/kvm/vfio.c                     | 74 +++++++++++++++++++++++++++++++++++++
 6 files changed, 85 insertions(+)
arch/powerpc/include/asm/kvm_host.h +3 -0
index 48dbe8b..e1163d7 100644
@@ -293,6 +293,9 @@ struct kvm_arch {
 #ifdef CONFIG_KVM_XICS
 	struct kvmppc_xics *xics;
 #endif
+#ifdef CONFIG_KVM_VFIO
+	struct kvm_vfio *vfio;
+#endif
 };
 
 /*
arch/powerpc/kvm/Kconfig +1 -0
index 61b3535..d1b7f64 100644
@@ -60,6 +60,7 @@ config KVM_BOOK3S_64
 	select KVM_BOOK3S_64_HANDLER
 	select KVM
 	select SPAPR_TCE_IOMMU
+	select KVM_VFIO
 	---help---
 	  Support running unmodified book3s_64 and book3s_32 guest kernels
 	  in virtual machines on book3s_64 host processors.
arch/powerpc/kvm/Makefile +3 -0
index 6646c95..2438d2e 100644
@@ -87,6 +87,9 @@ kvm-book3s_64-builtin-objs-$(CONFIG_KVM_BOOK3S_64_HV) := \
 kvm-book3s_64-objs-$(CONFIG_KVM_XICS) += \
 	book3s_xics.o
 
+kvm-book3s_64-objs-$(CONFIG_KVM_VFIO) += \
+	$(KVM)/vfio.o \
+
 kvm-book3s_64-module-objs := \
 	$(KVM)/kvm_main.o \
 	$(KVM)/eventfd.o \
include/linux/vfio.h +0 -0
index 24579a0..681e19b 100644

include/linux/vfio.h extern struct vfio_group *vfio_group_get_external_user(struct file *filep); · line 102

Quoted 6 lines
include/linux/vfio.h +3 -0
@@ -97,4 +97,7 @@ extern struct vfio_group *vfio_group_get_external_user(struct file *filep);
 extern void vfio_group_put_external_user(struct vfio_group *group);
 extern int vfio_external_user_iommu_id(struct vfio_group *group);
 
+extern struct iommu_group *vfio_find_group_by_liobn(struct kvm *kvm,
+		unsigned long liobn);
+

Nope, this doesn't go in vfio.h, it's a function provided by kvm. It should be named as such too, kvm_vfio_... It also depends on both CONFIG_KVM_VFIO and CONFIG_SPAPR_TCE_IOMMU and needs stub version otherwise. Is just _liobn specific enough or does it need a spapr_tce thrown in to avoid confusion with embedded ppc folks?

include/uapi/linux/kvm.h struct kvm_device_attr { · line 850

Quoted 10 lines
include/linux/vfio.h +0 -0
 #endif /* VFIO_H */
include/uapi/linux/kvm.h +1 -0
index 7c1a349..a74ad16 100644
@@ -847,6 +847,7 @@ struct kvm_device_attr {
 #define  KVM_DEV_VFIO_GROUP			1
 #define   KVM_DEV_VFIO_GROUP_ADD			1
 #define   KVM_DEV_VFIO_GROUP_DEL			2
+#define  KVM_DEV_VFIO_SPAPR_TCE_LIOBN		2

I wonder if it would be better architecturally if this was an attribute rather than a new group, ex:

#define KVM_DEV_VFIO_GROUP_SET_SPAPR_TCE_LIOBN 3

It's a mouthful, but we are setting an attribute of a VFIO group, so it makes sense. kvm_device_attr.addr would then need to point to a struct containing both the fd and liobn.

Whatever we come up with need a documentation addition in Documentation/virtual/kvm/devices/vfio.txt.

virt/kvm/vfio.c line 26

Quoted 13 lines
include/uapi/linux/kvm.h +0 -0
 /*
  * ioctls for VM fds
virt/kvm/vfio.c +2 -0
index ca4260e..f9271d5 100644
@@ -22,6 +22,9 @@
 struct kvm_vfio_group {
 	struct list_head node;
 	struct vfio_group *vfio_group;
+#ifdef CONFIG_SPAPR_TCE_IOMMU
+	uint64_t liobn;

Why is liobn an unsigned long in the exported function but a uint64_t here?

Quoted 19 lines
virt/kvm/vfio.c +12 -0
+#endif
 };
 
 struct kvm_vfio {
@@ -188,12 +191,76 @@ static int kvm_vfio_set_group(struct kvm_device *dev, long attr, u64 arg)
 	return -ENXIO;
 }
 
+#ifdef CONFIG_SPAPR_TCE_IOMMU
+static int kvm_vfio_set_spapr_tce_liobn(struct kvm_device *dev,
+		long attr, u64 arg)
+{
+	struct kvm_vfio *kv = dev->private;
+	struct vfio_group *vfio_group;
+	struct kvm_vfio_group *kvg;
+	void __user *argp = (void __user *)arg;
+	struct fd f;
+	int32_t fd;
+	uint64_t liobn = attr;

virt/kvm/vfio.c static int kvm_vfio_set_group(struct kvm_device *dev, long attr, u64 arg) · line 214

Quoted 10 lines
virt/kvm/vfio.c +10 -0
+
+	if (get_user(fd, (int32_t __user *)argp))
+		return -EFAULT;
+
+	f = fdget(fd);
+	if (!f.file)
+		return -EBADF;
+
+	vfio_group = kvm_vfio_group_get_external_user(f.file);
+	fdput(f);

Not sure why you dropped this from the example of kvm_vfio_set_group:

        if (IS_ERR(vfio_group))
                return PTR_ERR(vfio_group);

You're also ignoring kv->lock.

virt/kvm/vfio.c static int kvm_vfio_set_group(struct kvm_device *dev, long attr, u64 arg) · line 218

Quoted 4 lines
virt/kvm/vfio.c +4 -0
+
+	list_for_each_entry(kvg, &kv->group_list, node) {
+		if (kvg->vfio_group == vfio_group) {
+			WARN_ON(kvg->liobn);

Userspace should not be able to trigger a WARN this easily. Return EBUSY if it's an error, otherwise let it go.

virt/kvm/vfio.c static int kvm_vfio_set_group(struct kvm_device *dev, long attr, u64 arg) · line 231

Quoted 13 lines
virt/kvm/vfio.c +13 -0
+			kvg->liobn = liobn;
+			kvm_vfio_group_put_external_user(vfio_group);
+			return 0;
+		}
+	}
+
+	kvm_vfio_group_put_external_user(vfio_group);
+
+	return -ENXIO;
+}
+
+struct iommu_group *vfio_find_group_by_liobn(struct kvm *kvm,
+		unsigned long liobn)

As mentioned, kvm_vfio_...

virt/kvm/vfio.c static int kvm_vfio_set_group(struct kvm_device *dev, long attr, u64 arg) · line 236

Quoted 5 lines
virt/kvm/vfio.c +5 -0
+{
+	struct kvm_vfio_group *kvg;
+
+	if (!kvm->arch.vfio)
+		return NULL;

If this is already a slow path you can avoid stashing this pointer and search kvm->devices for the matching ops struct.

kv->lock...

virt/kvm/vfio.c static int kvm_vfio_set_group(struct kvm_device *dev, long attr, u64 arg)

Quoted 7 lines
virt/kvm/vfio.c +7 -0
+
+	list_for_each_entry(kvg, &kvm->arch.vfio->group_list, node) {
+		if (kvg->liobn == liobn) {
+			int group_id = vfio_external_user_iommu_id(
+					kvg->vfio_group);
+			struct iommu_group *grp =
+					iommu_group_get_by_id(group_id);

nit, ugly line wrapping. Where's the iommu_group_put() done?

virt/kvm/vfio.c static int kvm_vfio_set_group(struct kvm_device *dev, long attr, u64 arg) · line 243

Quoted 1 line
virt/kvm/vfio.c +1 -0
+			return grp;

So you've now got an liobn to iommu_group mapping cached away somewhere, what happens when a group is removed? Would it be invalid for userspace to re-use the liobn? Do we need a way to invalidate your cached entry and perhaps do an iommu_group_put()?

This version is actually plausible, so big improvement from v1! Thanks,

Alex

Quoted 43 lines
virt/kvm/vfio.c +20 -0
+		}
+	}
+
+	return NULL;
+}
+EXPORT_SYMBOL_GPL(vfio_find_group_by_liobn);
+#endif
+
 static int kvm_vfio_set_attr(struct kvm_device *dev,
 			     struct kvm_device_attr *attr)
 {
 	switch (attr->group) {
 	case KVM_DEV_VFIO_GROUP:
 		return kvm_vfio_set_group(dev, attr->attr, attr->addr);
+#ifdef CONFIG_SPAPR_TCE_IOMMU
+	case KVM_DEV_VFIO_SPAPR_TCE_LIOBN:
+		return kvm_vfio_set_spapr_tce_liobn(dev, attr->attr,
+				attr->addr);
+#endif
 	}
 
 	return -ENXIO;
@@ -211,6 +278,10 @@ static int kvm_vfio_has_attr(struct kvm_device *dev,
 		}
 
 		break;
+#ifdef CONFIG_SPAPR_TCE_IOMMU
+	case KVM_DEV_VFIO_SPAPR_TCE_LIOBN:
+		return 0;
+#endif
 	}
 
 	return -ENXIO;
@@ -251,6 +322,9 @@ static int kvm_vfio_create(struct kvm_device *dev, u32 type)
 	mutex_init(&kv->lock);
 
 	dev->private = kv;
+#ifdef CONFIG_SPAPR_TCE_IOMMU
+	dev->kvm->arch.vfio = kv;
+#endif
 
 	return 0;
 }