Skip to contentSkip to conversation

KVM: s390: pci: Hook to access KVM lowlevel from VFIO

PATCH In review Pierre Morel 4 participants 5 messages Aug 19, 2022 – Aug 23, 2022

Subsystems (6)
KVM s390 VFIO driver KVM s390 +2 more subsystems
lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 KVM: s390: pci: Hook to access KVM lowlevel from VFIO

We have a cross dependency between KVM and VFIO when using s390 vfio_pci_zdev extensions for PCI passthrough
To be able to keep both subsystem modular we add a registering hook inside the S390 core code.

This fixes a build problem when VFIO is built-in and KVM is built as a module.

Fixes 09340b2fca007 ("KVM: s390: pci: add routines to start/stop interpretive execution")
---
 arch/s390/include/asm/kvm_host.h | 17 ++++++-----------
 arch/s390/kvm/pci.c              | 12 ++++++++----
 arch/s390/pci/Makefile           |  2 +-
 arch/s390/pci/pci_kvm_hook.c     | 11 +++++++++++
 drivers/vfio/pci/vfio_pci_zdev.c |  8 ++++++--
 5 files changed, 32 insertions(+), 18 deletions(-)
 create mode 100644 arch/s390/pci/pci_kvm_hook.c
arch/s390/include/asm/kvm_host.h +6 -11
index f39092e0ceaa..b1e98a9ed152 100644
@@ -1038,16 +1038,11 @@ static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {}
 #define __KVM_HAVE_ARCH_VM_FREE
 void kvm_arch_free_vm(struct kvm *kvm);
 
-#ifdef CONFIG_VFIO_PCI_ZDEV_KVM
-int kvm_s390_pci_register_kvm(struct zpci_dev *zdev, struct kvm *kvm);
-void kvm_s390_pci_unregister_kvm(struct zpci_dev *zdev);
-#else
-static inline int kvm_s390_pci_register_kvm(struct zpci_dev *dev,
-					    struct kvm *kvm)
-{
-	return -EPERM;
-}
-static inline void kvm_s390_pci_unregister_kvm(struct zpci_dev *dev) {}
-#endif
+struct zpci_kvm_hook {
+	int (*kvm_register)(void *opaque, struct kvm *kvm);
+	void (*kvm_unregister)(void *opaque);
+};
+
+extern struct zpci_kvm_hook zpci_kvm_hook;
 
 #endif
arch/s390/kvm/pci.c +8 -4
index 4946fb7757d6..bb8c335d17b9 100644
@@ -431,8 +431,9 @@ static void kvm_s390_pci_dev_release(struct zpci_dev *zdev)
  * available, enable them and let userspace indicate whether or not they will
  * be used (specify SHM bit to disable).
  */
-int kvm_s390_pci_register_kvm(struct zpci_dev *zdev, struct kvm *kvm)
+static int kvm_s390_pci_register_kvm(void *opaque, struct kvm *kvm)
 {
+	struct zpci_dev *zdev = opaque;
 	int rc;
 
 	if (!zdev)
@@ -510,10 +511,10 @@ int kvm_s390_pci_register_kvm(struct zpci_dev *zdev, struct kvm *kvm)
 	kvm_put_kvm(kvm);
 	return rc;
 }
-EXPORT_SYMBOL_GPL(kvm_s390_pci_register_kvm);
 
-void kvm_s390_pci_unregister_kvm(struct zpci_dev *zdev)
+static void kvm_s390_pci_unregister_kvm(void *opaque)
 {
+	struct zpci_dev *zdev = opaque;
 	struct kvm *kvm;
 
 	if (!zdev)
@@ -566,7 +567,6 @@ void kvm_s390_pci_unregister_kvm(struct zpci_dev *zdev)
 
 	kvm_put_kvm(kvm);
 }
-EXPORT_SYMBOL_GPL(kvm_s390_pci_unregister_kvm);
 
 void kvm_s390_pci_init_list(struct kvm *kvm)
 {
@@ -678,6 +678,8 @@ int kvm_s390_pci_init(void)
 
 	spin_lock_init(&aift->gait_lock);
 	mutex_init(&aift->aift_lock);
+	zpci_kvm_hook.kvm_register = kvm_s390_pci_register_kvm;
+	zpci_kvm_hook.kvm_unregister = kvm_s390_pci_unregister_kvm;
 
 	return 0;
 }
@@ -685,6 +687,8 @@ int kvm_s390_pci_init(void)
 void kvm_s390_pci_exit(void)
 {
 	mutex_destroy(&aift->aift_lock);
+	zpci_kvm_hook.kvm_register = NULL;
+	zpci_kvm_hook.kvm_unregister = NULL;
 
 	kfree(aift);
 }
arch/s390/pci/Makefile +1 -1
index bf557a1b789c..5ae31ca9dd44 100644
@@ -5,5 +5,5 @@
 
 obj-$(CONFIG_PCI)	+= pci.o pci_irq.o pci_dma.o pci_clp.o pci_sysfs.o \
 			   pci_event.o pci_debug.o pci_insn.o pci_mmio.o \
-			   pci_bus.o
+			   pci_bus.o pci_kvm_hook.o
 obj-$(CONFIG_PCI_IOV)	+= pci_iov.o
arch/s390/pci/pci_kvm_hook.c new file +11 -0
new file mode 100644
index 000000000000..ff34baf50a3e
@@ -0,0 +1,11 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * VFIO ZPCI devices support
+ *
+ * Copyright (C) IBM Corp. 2022.  All rights reserved.
+ *	Author(s): Pierre Morel <[email protected]>
+ */
+#include <linux/kvm_host.h>
+
+struct zpci_kvm_hook zpci_kvm_hook;
+EXPORT_SYMBOL_GPL(zpci_kvm_hook);
drivers/vfio/pci/vfio_pci_zdev.c +6 -2
index e163aa9f6144..0cbdcd14f1c8 100644
@@ -151,7 +151,10 @@ int vfio_pci_zdev_open_device(struct vfio_pci_core_device *vdev)
 	if (!vdev->vdev.kvm)
 		return 0;
 
-	return kvm_s390_pci_register_kvm(zdev, vdev->vdev.kvm);
+	if (zpci_kvm_hook.kvm_register)
+		return zpci_kvm_hook.kvm_register(zdev, vdev->vdev.kvm);
+
+	return -ENOENT;
 }
 
 void vfio_pci_zdev_close_device(struct vfio_pci_core_device *vdev)
@@ -161,5 +164,6 @@ void vfio_pci_zdev_close_device(struct vfio_pci_core_device *vdev)
 	if (!zdev || !vdev->vdev.kvm)
 		return;
 
-	kvm_s390_pci_unregister_kvm(zdev);
+	if (zpci_kvm_hook.kvm_unregister)
+		zpci_kvm_hook.kvm_unregister(zdev);
 }
Signature · 1 line
2.31.1
Acked-by on v2 1/1
#1 lore raw
Quoted 8 lines · Pierre Morel

We have a cross dependency between KVM and VFIO when using s390 vfio_pci_zdev extensions for PCI passthrough
To be able to keep both subsystem modular we add a registering hook inside the S390 core code.

This fixes a build problem when VFIO is built-in and KVM is built as a module.

Quoted 7 lines
Fixes 09340b2fca007 ("KVM: s390: pci: add routines to start/stop interpretive execution")

Thanks.

Quoted 142 lines
---
 arch/s390/include/asm/kvm_host.h | 17 ++++++-----------
 arch/s390/kvm/pci.c              | 12 ++++++++----
 arch/s390/pci/Makefile           |  2 +-
 arch/s390/pci/pci_kvm_hook.c     | 11 +++++++++++
 drivers/vfio/pci/vfio_pci_zdev.c |  8 ++++++--
 5 files changed, 32 insertions(+), 18 deletions(-)
 create mode 100644 arch/s390/pci/pci_kvm_hook.c
arch/s390/include/asm/kvm_host.h +6 -11
index f39092e0ceaa..b1e98a9ed152 100644
@@ -1038,16 +1038,11 @@ static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {}
 #define __KVM_HAVE_ARCH_VM_FREE
 void kvm_arch_free_vm(struct kvm *kvm);
 
-#ifdef CONFIG_VFIO_PCI_ZDEV_KVM
-int kvm_s390_pci_register_kvm(struct zpci_dev *zdev, struct kvm *kvm);
-void kvm_s390_pci_unregister_kvm(struct zpci_dev *zdev);
-#else
-static inline int kvm_s390_pci_register_kvm(struct zpci_dev *dev,
-					    struct kvm *kvm)
-{
-	return -EPERM;
-}
-static inline void kvm_s390_pci_unregister_kvm(struct zpci_dev *dev) {}
-#endif
+struct zpci_kvm_hook {
+	int (*kvm_register)(void *opaque, struct kvm *kvm);
+	void (*kvm_unregister)(void *opaque);
+};
+
+extern struct zpci_kvm_hook zpci_kvm_hook;
 
 #endif
arch/s390/kvm/pci.c +8 -4
index 4946fb7757d6..bb8c335d17b9 100644
@@ -431,8 +431,9 @@ static void kvm_s390_pci_dev_release(struct zpci_dev *zdev)
  * available, enable them and let userspace indicate whether or not they will
  * be used (specify SHM bit to disable).
  */
-int kvm_s390_pci_register_kvm(struct zpci_dev *zdev, struct kvm *kvm)
+static int kvm_s390_pci_register_kvm(void *opaque, struct kvm *kvm)
 {
+	struct zpci_dev *zdev = opaque;
 	int rc;
 
 	if (!zdev)
@@ -510,10 +511,10 @@ int kvm_s390_pci_register_kvm(struct zpci_dev *zdev, struct kvm *kvm)
 	kvm_put_kvm(kvm);
 	return rc;
 }
-EXPORT_SYMBOL_GPL(kvm_s390_pci_register_kvm);
 
-void kvm_s390_pci_unregister_kvm(struct zpci_dev *zdev)
+static void kvm_s390_pci_unregister_kvm(void *opaque)
 {
+	struct zpci_dev *zdev = opaque;
 	struct kvm *kvm;
 
 	if (!zdev)
@@ -566,7 +567,6 @@ void kvm_s390_pci_unregister_kvm(struct zpci_dev *zdev)
 
 	kvm_put_kvm(kvm);
 }
-EXPORT_SYMBOL_GPL(kvm_s390_pci_unregister_kvm);
 
 void kvm_s390_pci_init_list(struct kvm *kvm)
 {
@@ -678,6 +678,8 @@ int kvm_s390_pci_init(void)
 
 	spin_lock_init(&aift->gait_lock);
 	mutex_init(&aift->aift_lock);
+	zpci_kvm_hook.kvm_register = kvm_s390_pci_register_kvm;
+	zpci_kvm_hook.kvm_unregister = kvm_s390_pci_unregister_kvm;
 
 	return 0;
 }
@@ -685,6 +687,8 @@ int kvm_s390_pci_init(void)
 void kvm_s390_pci_exit(void)
 {
 	mutex_destroy(&aift->aift_lock);
+	zpci_kvm_hook.kvm_register = NULL;
+	zpci_kvm_hook.kvm_unregister = NULL;
 
 	kfree(aift);
 }
arch/s390/pci/Makefile +1 -1
index bf557a1b789c..5ae31ca9dd44 100644
@@ -5,5 +5,5 @@
 
 obj-$(CONFIG_PCI)	+= pci.o pci_irq.o pci_dma.o pci_clp.o pci_sysfs.o \
 			   pci_event.o pci_debug.o pci_insn.o pci_mmio.o \
-			   pci_bus.o
+			   pci_bus.o pci_kvm_hook.o
 obj-$(CONFIG_PCI_IOV)	+= pci_iov.o
arch/s390/pci/pci_kvm_hook.c new file +11 -0
new file mode 100644
index 000000000000..ff34baf50a3e
@@ -0,0 +1,11 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * VFIO ZPCI devices support
+ *
+ * Copyright (C) IBM Corp. 2022.  All rights reserved.
+ *	Author(s): Pierre Morel <[email protected]>
+ */
+#include <linux/kvm_host.h>
+
+struct zpci_kvm_hook zpci_kvm_hook;
+EXPORT_SYMBOL_GPL(zpci_kvm_hook);
drivers/vfio/pci/vfio_pci_zdev.c +6 -2
index e163aa9f6144..0cbdcd14f1c8 100644
@@ -151,7 +151,10 @@ int vfio_pci_zdev_open_device(struct vfio_pci_core_device *vdev)
 	if (!vdev->vdev.kvm)
 		return 0;
 
-	return kvm_s390_pci_register_kvm(zdev, vdev->vdev.kvm);
+	if (zpci_kvm_hook.kvm_register)
+		return zpci_kvm_hook.kvm_register(zdev, vdev->vdev.kvm);
+
+	return -ENOENT;
 }
 
 void vfio_pci_zdev_close_device(struct vfio_pci_core_device *vdev)
@@ -161,5 +164,6 @@ void vfio_pci_zdev_close_device(struct vfio_pci_core_device *vdev)
 	if (!zdev || !vdev->vdev.kvm)
 		return;
 
-	kvm_s390_pci_unregister_kvm(zdev);
+	if (zpci_kvm_hook.kvm_unregister)
+		zpci_kvm_hook.kvm_unregister(zdev);
 }
Signature · 1 line
~Randy
#2 lore raw
Quoted 19 lines · Randy Dunlap
Quoted 8 lines · Pierre Morel

We have a cross dependency between KVM and VFIO when using s390 vfio_pci_zdev extensions for PCI passthrough
To be able to keep both subsystem modular we add a registering hook inside the S390 core code.

This fixes a build problem when VFIO is built-in and KVM is built as a module.

Quoted 7 lines
Fixes 09340b2fca007 ("KVM: s390: pci: add routines to start/stop interpretive execution")
Quoted 3 lines

Thanks.

Thanks Randy,

Regards,
Pierre

Quoted 144 lines
Quoted 143 lines
---
  arch/s390/include/asm/kvm_host.h | 17 ++++++-----------
  arch/s390/kvm/pci.c              | 12 ++++++++----
  arch/s390/pci/Makefile           |  2 +-
  arch/s390/pci/pci_kvm_hook.c     | 11 +++++++++++
  drivers/vfio/pci/vfio_pci_zdev.c |  8 ++++++--
  5 files changed, 32 insertions(+), 18 deletions(-)

create mode 100644 arch/s390/pci/pci_kvm_hook.c

arch/s390/include/asm/kvm_host.h +6 -11
index f39092e0ceaa..b1e98a9ed152 100644
@@ -1038,16 +1038,11 @@ static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {}
  #define __KVM_HAVE_ARCH_VM_FREE
  void kvm_arch_free_vm(struct kvm *kvm);
 
-#ifdef CONFIG_VFIO_PCI_ZDEV_KVM
-int kvm_s390_pci_register_kvm(struct zpci_dev *zdev, struct kvm *kvm);
-void kvm_s390_pci_unregister_kvm(struct zpci_dev *zdev);
-#else
-static inline int kvm_s390_pci_register_kvm(struct zpci_dev *dev,
-					    struct kvm *kvm)
-{
-	return -EPERM;
-}
-static inline void kvm_s390_pci_unregister_kvm(struct zpci_dev *dev) {}
-#endif
+struct zpci_kvm_hook {
+	int (*kvm_register)(void *opaque, struct kvm *kvm);
+	void (*kvm_unregister)(void *opaque);
+};
+
+extern struct zpci_kvm_hook zpci_kvm_hook;
 
  #endif
arch/s390/kvm/pci.c +8 -4
index 4946fb7757d6..bb8c335d17b9 100644
@@ -431,8 +431,9 @@ static void kvm_s390_pci_dev_release(struct zpci_dev *zdev)
   * available, enable them and let userspace indicate whether or not they will
   * be used (specify SHM bit to disable).
   */
-int kvm_s390_pci_register_kvm(struct zpci_dev *zdev, struct kvm *kvm)
+static int kvm_s390_pci_register_kvm(void *opaque, struct kvm *kvm)
  {
+	struct zpci_dev *zdev = opaque;
  	int rc;
 
  	if (!zdev)
@@ -510,10 +511,10 @@ int kvm_s390_pci_register_kvm(struct zpci_dev *zdev, struct kvm *kvm)
  	kvm_put_kvm(kvm);
  	return rc;
  }
-EXPORT_SYMBOL_GPL(kvm_s390_pci_register_kvm);
 
-void kvm_s390_pci_unregister_kvm(struct zpci_dev *zdev)
+static void kvm_s390_pci_unregister_kvm(void *opaque)
  {
+	struct zpci_dev *zdev = opaque;
  	struct kvm *kvm;
 
  	if (!zdev)
@@ -566,7 +567,6 @@ void kvm_s390_pci_unregister_kvm(struct zpci_dev *zdev)
 
  	kvm_put_kvm(kvm);
  }
-EXPORT_SYMBOL_GPL(kvm_s390_pci_unregister_kvm);
 
  void kvm_s390_pci_init_list(struct kvm *kvm)
  {
@@ -678,6 +678,8 @@ int kvm_s390_pci_init(void)
 
  	spin_lock_init(&aift->gait_lock);
  	mutex_init(&aift->aift_lock);
+	zpci_kvm_hook.kvm_register = kvm_s390_pci_register_kvm;
+	zpci_kvm_hook.kvm_unregister = kvm_s390_pci_unregister_kvm;
 
  	return 0;
  }
@@ -685,6 +687,8 @@ int kvm_s390_pci_init(void)
  void kvm_s390_pci_exit(void)
  {
  	mutex_destroy(&aift->aift_lock);
+	zpci_kvm_hook.kvm_register = NULL;
+	zpci_kvm_hook.kvm_unregister = NULL;
 
  	kfree(aift);
  }
arch/s390/pci/Makefile +1 -1
index bf557a1b789c..5ae31ca9dd44 100644
@@ -5,5 +5,5 @@
 
  obj-$(CONFIG_PCI)	+= pci.o pci_irq.o pci_dma.o pci_clp.o pci_sysfs.o \
  			   pci_event.o pci_debug.o pci_insn.o pci_mmio.o \
-			   pci_bus.o
+			   pci_bus.o pci_kvm_hook.o
  obj-$(CONFIG_PCI_IOV)	+= pci_iov.o
arch/s390/pci/pci_kvm_hook.c new file +11 -0
new file mode 100644
index 000000000000..ff34baf50a3e
@@ -0,0 +1,11 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * VFIO ZPCI devices support
+ *
+ * Copyright (C) IBM Corp. 2022.  All rights reserved.
+ *	Author(s): Pierre Morel <[email protected]>
+ */
+#include <linux/kvm_host.h>
+
+struct zpci_kvm_hook zpci_kvm_hook;
+EXPORT_SYMBOL_GPL(zpci_kvm_hook);
drivers/vfio/pci/vfio_pci_zdev.c +6 -2
index e163aa9f6144..0cbdcd14f1c8 100644
@@ -151,7 +151,10 @@ int vfio_pci_zdev_open_device(struct vfio_pci_core_device *vdev)
  	if (!vdev->vdev.kvm)
  		return 0;
 
-	return kvm_s390_pci_register_kvm(zdev, vdev->vdev.kvm);
+	if (zpci_kvm_hook.kvm_register)
+		return zpci_kvm_hook.kvm_register(zdev, vdev->vdev.kvm);
+
+	return -ENOENT;
  }
 
  void vfio_pci_zdev_close_device(struct vfio_pci_core_device *vdev)
@@ -161,5 +164,6 @@ void vfio_pci_zdev_close_device(struct vfio_pci_core_device *vdev)
  	if (!zdev || !vdev->vdev.kvm)
  		return;
 
-	kvm_s390_pci_unregister_kvm(zdev);
+	if (zpci_kvm_hook.kvm_unregister)
+		zpci_kvm_hook.kvm_unregister(zdev);
  }
Signature · 2 lines
Pierre Morel
IBM Lab Boeblingen
maintainer Acked-by on v2 1/1
#3 lore raw
In reply to Pierre Morel
Quoted 8 lines · Pierre Morel

We have a cross dependency between KVM and VFIO when using s390 vfio_pci_zdev extensions for PCI passthrough
To be able to keep both subsystem modular we add a registering hook inside the S390 core code.

This fixes a build problem when VFIO is built-in and KVM is built as a module.

Quoted 7 lines
Fixes 09340b2fca007 ("KVM: s390: pci: add routines to start/stop interpretive execution")

@Niklas @Matt: Since the patches that introduced the PCI interpretation went via the KVM tree I'll also move this patch via the KVM tree.

maintainer
#4 lore raw
Quoted 17 lines · Janosch Frank
Quoted 8 lines · Pierre Morel

We have a cross dependency between KVM and VFIO when using s390 vfio_pci_zdev extensions for PCI passthrough
To be able to keep both subsystem modular we add a registering hook inside the S390 core code.

This fixes a build problem when VFIO is built-in and KVM is built as a module.

Quoted 7 lines
Fixes 09340b2fca007 ("KVM: s390: pci: add routines to start/stop interpretive execution")
Quoted 4 lines

@Niklas @Matt: Since the patches that introduced the PCI interpretation went via the KVM tree I'll also move this patch via the KVM tree.

Sounds good, thanks.