Skip to contentSkip to conversation

s390x/pci: zPCI interpretation support

PATCH Superseded Matthew Rosato 1 participant 12 messages Mar 14, 2022

Subsystems (2)

v4 of 8 Superseded Latest is v8 →

lore
Share & tools
lore
#0 lore raw

v4 cover s390x/pci: zPCI interpretation support

For QEMU, the majority of the work in enabling instruction interpretation is handled via a new KVM ioctls to enable interpretation, interrupt forwarding and registration of the guest IOAT tables. In order to make use of the KVM-managed IOMMU domain operations on the host, we also add some code to vfio to indicate that a given device wishes to register the alternate domain type for its group.

This series also adds a new, optional 'interpret' parameter to zpci which can be used to disable interpretation support (interpret=off) as well as an 'forwarding_assist' parameter to determine whether or not the firmware assist will be used for interrupt delivery (default when interpretation is in use) or whether the host will be responsible for delivering all interrupts (forwarding_assist=off).

The ZPCI_INTERP CPU feature is added beginning with the z14 model to enable this support.

As a consequence of implementing zPCI interpretation, ISM devices now become eligible for passthrough (but only when zPCI interpretation is available).

From the perspective of guest configuration, you passthrough zPCI devices in the same manner as before, with intepretation support being used by default if available in kernel+qemu.

Associated kernel series:
https://lore.kernel.org/kvm/[email protected]/

Changelog v3->v4
- Unfortunately I had to remove some Review tags because the userspace API moved from vfio device feature ioctls to KVM ioctls in response to feedback from the kernel series. The vast majority of the QEMU logic remains intact however, with most changes being to the way we issue ioctls.
- Additional logic was added to test for availability of the KVM ioctl, this replaces the probe logic done for the vfio ioctls
- Add code to issue indicate on VFIO_SET_IOMMU that a KVM-managed IOMMU domain is to be allocated.

Matthew Rosato (11):
Update linux headers
vfio: handle KVM-owned IOMMU requests
target/s390x: add zpci-interp to cpu models
s390x/pci: add routine to get host function handle from CLP info s390x/pci: enable for load/store intepretation
s390x/pci: don't fence interpreted devices without MSI-X s390x/pci: enable adapter event notification for interpreted devices s390x/pci: use KVM-managed IOMMU for interpretation s390x/pci: use I/O Address Translation assist when interpreting s390x/pci: use dtsm provided from vfio capabilities for interpreted
devices
s390x/pci: let intercept devices have separate PCI groups

 hw/s390x/meson.build                |   1 +
 hw/s390x/s390-pci-bus.c             | 125 ++++++++++++++++++++--
 hw/s390x/s390-pci-inst.c            | 136 +++++++++++++++++++++--
 hw/s390x/s390-pci-kvm.c             | 160 ++++++++++++++++++++++++++++
 hw/s390x/s390-pci-vfio.c            | 151 ++++++++++++++++++++++----
 hw/s390x/s390-virtio-ccw.c          |   1 +
 hw/vfio/ap.c                        |   2 +-
 hw/vfio/ccw.c                       |   2 +-
 hw/vfio/common.c                    |  26 ++++-
 hw/vfio/pci.c                       |   3 +-
 hw/vfio/pci.h                       |   1 +
 hw/vfio/platform.c                  |   2 +-
 include/hw/s390x/s390-pci-bus.h     |   8 +-
 include/hw/s390x/s390-pci-inst.h    |   2 +-
 include/hw/s390x/s390-pci-kvm.h     |  68 ++++++++++++
 include/hw/s390x/s390-pci-vfio.h    |  11 ++
 include/hw/vfio/vfio-common.h       |   4 +-
 linux-headers/asm-s390/kvm.h        |   1 +
 linux-headers/asm-x86/kvm.h         |   3 +
 linux-headers/linux/kvm.h           |  51 ++++++++-
 linux-headers/linux/vfio.h          |   6 ++
 linux-headers/linux/vfio_zdev.h     |   6 ++
 target/s390x/cpu_features_def.h.inc |   1 +
 target/s390x/gen-features.c         |   2 +
 target/s390x/kvm/kvm.c              |   8 ++
 target/s390x/kvm/kvm_s390x.h        |   1 +
 26 files changed, 731 insertions(+), 51 deletions(-)
 create mode 100644 hw/s390x/s390-pci-kvm.c
 create mode 100644 include/hw/s390x/s390-pci-kvm.h
Signature · 1 line
2.27.0
#1 lore raw

PATCH v4 1/11 Update linux headers

This is a placeholder that pulls in 5.17-rc7 + unmerged kernel changes required by this item. A proper header sync can be done once the associated kernel code merges.

---
 linux-headers/asm-s390/kvm.h    |  1 +
 linux-headers/asm-x86/kvm.h     |  3 ++
 linux-headers/linux/kvm.h       | 51 +++++++++++++++++++++++++++++++--
 linux-headers/linux/vfio.h      |  6 ++++
 linux-headers/linux/vfio_zdev.h |  6 ++++
 5 files changed, 64 insertions(+), 3 deletions(-)
linux-headers/asm-s390/kvm.h +1 -0
index f053b8304a..d8259ff9a1 100644
@@ -130,6 +130,7 @@ struct kvm_s390_vm_cpu_machine {
 #define KVM_S390_VM_CPU_FEAT_PFMFI	11
 #define KVM_S390_VM_CPU_FEAT_SIGPIF	12
 #define KVM_S390_VM_CPU_FEAT_KSS	13
+#define KVM_S390_VM_CPU_FEAT_ZPCI_INTERP 14
 struct kvm_s390_vm_cpu_feat {
 	__u64 feat[16];
 };
linux-headers/asm-x86/kvm.h +3 -0
index 2da3316bb5..bf6e96011d 100644
@@ -452,6 +452,9 @@ struct kvm_sync_regs {
 
 #define KVM_STATE_VMX_PREEMPTION_TIMER_DEADLINE	0x00000001
 
+/* attributes for system fd (group 0) */
+#define KVM_X86_XCOMP_GUEST_SUPP	0
+
 struct kvm_vmx_nested_state_data {
 	__u8 vmcs12[KVM_STATE_NESTED_VMX_VMCS_SIZE];
 	__u8 shadow_vmcs12[KVM_STATE_NESTED_VMX_VMCS_SIZE];
linux-headers/linux/kvm.h +48 -3

67 lines not on this page · raw message

linux-headers/linux/vfio.h +6 -0
index e680594f27..38d43e1205 100644
@@ -52,6 +52,12 @@
 /* Supports the vaddr flag for DMA map and unmap */
 #define VFIO_UPDATE_VADDR		10
 
+/*
+ * The KVM_IOMMU type implies that the hypervisor will control the mappings
+ * rather than userspace
+ */
+#define VFIO_KVM_IOMMU			11
+
 /*
  * The IOCTL interface is designed for extensibility by embedding the
  * structure length (argsz) and flags into structures passed between
linux-headers/linux/vfio_zdev.h +6 -0
index b4309397b6..29351687e9 100644
@@ -29,6 +29,9 @@ struct vfio_device_info_cap_zpci_base {
 	__u16 fmb_length;	/* Measurement Block Length (in bytes) */
 	__u8 pft;		/* PCI Function Type */
 	__u8 gid;		/* PCI function group ID */
+	/* End of version 1 */
+	__u32 fh;		/* PCI function handle */
+	/* End of version 2 */
 };
 
 /**
@@ -47,6 +50,9 @@ struct vfio_device_info_cap_zpci_group {
 	__u16 noi;		/* Maximum number of MSIs */
 	__u16 maxstbl;		/* Maximum Store Block Length */
 	__u8 version;		/* Supported PCI Version */
+	/* End of version 1 */
+	__u8 dtsm;		/* Supported IOAT Designations */
+	/* End of version 2 */
 };
 
 /**
Signature · 1 line
2.27.0
#2 lore raw

PATCH v4 2/11 vfio: handle KVM-owned IOMMU requests

In reply to Matthew Rosato

s390x PCI devices need to use a special KVM-managed IOMMU domain as part of zPCI interpretation. To facilitate this, let a vfio device indicate that it wishes to use a KVM-managed IOMMU so that it can be reflected by the group and, ultimately, trigger a KVM-managed argument for the VFIO_SET_IOMMU ioctl.

This patch sets up the framework to allow a device to trigger the VFIO_SET_IOMMU with the new KVM-owend type. A subsequent patch will add exploitation by s390x PCI.

---
 hw/vfio/ap.c                  |  2 +-
 hw/vfio/ccw.c                 |  2 +-
 hw/vfio/common.c              | 26 +++++++++++++++++++++-----
 hw/vfio/pci.c                 |  3 ++-
 hw/vfio/pci.h                 |  1 +
 hw/vfio/platform.c            |  2 +-
 include/hw/vfio/vfio-common.h |  4 +++-
 7 files changed, 30 insertions(+), 10 deletions(-)
hw/vfio/ap.c +1 -1
index e0dd561e85..22c402771a 100644
@@ -81,7 +81,7 @@ static VFIOGroup *vfio_ap_get_group(VFIOAPDevice *vapdev, Error **errp)
 
     g_free(group_path);
 
-    return vfio_get_group(groupid, &address_space_memory, errp);
+    return vfio_get_group(groupid, &address_space_memory, false, errp);
 }
 
 static void vfio_ap_realize(DeviceState *dev, Error **errp)
hw/vfio/ccw.c +1 -1
index 0354737666..08b0af5897 100644
@@ -650,7 +650,7 @@ static VFIOGroup *vfio_ccw_get_group(S390CCWDevice *cdev, Error **errp)
         return NULL;
     }
 
-    return vfio_get_group(groupid, &address_space_memory, errp);
+    return vfio_get_group(groupid, &address_space_memory, false, errp);
 }
 
 static void vfio_ccw_realize(DeviceState *dev, Error **errp)
hw/vfio/common.c +21 -5
index 080046e3f5..227880bf84 100644
@@ -1873,7 +1873,7 @@ static int vfio_get_iommu_type(VFIOContainer *container,
     return -EINVAL;
 }
 
-static int vfio_init_container(VFIOContainer *container, int group_fd,
+static int vfio_init_container(VFIOContainer *container, VFIOGroup *group,
                                Error **errp)
 {
     int iommu_type, ret;
@@ -1883,12 +1883,20 @@ static int vfio_init_container(VFIOContainer *container, int group_fd,
         return iommu_type;
     }
 
-    ret = ioctl(group_fd, VFIO_GROUP_SET_CONTAINER, &container->fd);
+    ret = ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &container->fd);
     if (ret) {
         error_setg_errno(errp, errno, "Failed to set group container");
         return -errno;
     }
 
+    /*
+     * In the case where KVM will manage the IOMMU, we must instruct the host
+     * IOMMU to use the appropriate domain ops
+     */
+    if (group->kvm_managed_iommu) {
+        iommu_type = VFIO_KVM_IOMMU;
+    }
+
     while (ioctl(container->fd, VFIO_SET_IOMMU, iommu_type)) {
         if (iommu_type == VFIO_SPAPR_TCE_v2_IOMMU) {
             /*
@@ -2062,7 +2070,7 @@ static int vfio_connect_container(VFIOGroup *group, AddressSpace *as,
     QLIST_INIT(&container->hostwin_list);
     QLIST_INIT(&container->vrdl_list);
 
-    ret = vfio_init_container(container, group->fd, errp);
+    ret = vfio_init_container(container, group, errp);
     if (ret) {
         goto free_container_exit;
     }
@@ -2265,7 +2273,8 @@ static void vfio_disconnect_container(VFIOGroup *group)
     }
 }
 
-VFIOGroup *vfio_get_group(int groupid, AddressSpace *as, Error **errp)
+VFIOGroup *vfio_get_group(int groupid, AddressSpace *as, bool kvm_managed_iommu,
+                          Error **errp)
 {
     VFIOGroup *group;
     char path[32];
@@ -2273,7 +2282,13 @@ VFIOGroup *vfio_get_group(int groupid, AddressSpace *as, Error **errp)
 
     QLIST_FOREACH(group, &vfio_group_list, next) {
         if (group->groupid == groupid) {
-            /* Found it.  Now is it already in the right context? */
+            /* Found it.  Ensure using same IOMMU type */
+            if (group->kvm_managed_iommu != kvm_managed_iommu) {
+                error_setg(errp, "group %d using conflicting iommu ops",
+                           group->groupid);
+                return NULL;
+            }
+            /* Is it already in the right context? */
             if (group->container->space->as == as) {
                 return group;
             } else {
@@ -2307,6 +2322,7 @@ VFIOGroup *vfio_get_group(int groupid, AddressSpace *as, Error **errp)
     }
 
     group->groupid = groupid;
+    group->kvm_managed_iommu = kvm_managed_iommu;
     QLIST_INIT(&group->device_list);
 
     if (vfio_connect_container(group, as, errp)) {
hw/vfio/pci.c +2 -1
index 7b45353ce2..80f7e2880a 100644
@@ -2855,7 +2855,8 @@ static void vfio_realize(PCIDevice *pdev, Error **errp)
 
     trace_vfio_realize(vdev->vbasedev.name, groupid);
 
-    group = vfio_get_group(groupid, pci_device_iommu_address_space(pdev), errp);
+    group = vfio_get_group(groupid, pci_device_iommu_address_space(pdev),
+                           vdev->kvm_managed_iommu, errp);
     if (!group) {
         goto error;
     }
hw/vfio/pci.h +1 -0
index 64777516d1..f74524384c 100644
@@ -171,6 +171,7 @@ struct VFIOPCIDevice {
     bool no_kvm_ioeventfd;
     bool no_vfio_ioeventfd;
     bool enable_ramfb;
+    bool kvm_managed_iommu;
     VFIODisplay *dpy;
     Notifier irqchip_change_notifier;
 };
hw/vfio/platform.c +1 -1
index f8f08a0f36..08793401dd 100644
@@ -577,7 +577,7 @@ static int vfio_base_device_init(VFIODevice *vbasedev, Error **errp)
 
     trace_vfio_platform_base_device_init(vbasedev->name, groupid);
 
-    group = vfio_get_group(groupid, &address_space_memory, errp);
+    group = vfio_get_group(groupid, &address_space_memory, false, errp);
     if (!group) {
         return -ENOENT;
     }
include/hw/vfio/vfio-common.h +3 -1
index 8af11b0a76..37aa6ca162 100644
@@ -162,6 +162,7 @@ typedef struct VFIOGroup {
     QLIST_ENTRY(VFIOGroup) next;
     QLIST_ENTRY(VFIOGroup) container_next;
     bool ram_block_discard_allowed;
+    bool kvm_managed_iommu;
 } VFIOGroup;
 
 typedef struct VFIODMABuf {
@@ -208,7 +209,8 @@ void vfio_region_unmap(VFIORegion *region);
 void vfio_region_exit(VFIORegion *region);
 void vfio_region_finalize(VFIORegion *region);
 void vfio_reset_handler(void *opaque);
-VFIOGroup *vfio_get_group(int groupid, AddressSpace *as, Error **errp);
+VFIOGroup *vfio_get_group(int groupid, AddressSpace *as, bool kvm_managed_iommu,
+                          Error **errp);
 void vfio_put_group(VFIOGroup *group);
 int vfio_get_device(VFIOGroup *group, const char *name,
                     VFIODevice *vbasedev, Error **errp);
Signature · 1 line
2.27.0
#3 lore raw

PATCH v4 3/11 target/s390x: add zpci-interp to cpu models

In reply to Matthew Rosato

The zpci-interp feature is used to specify whether zPCI interpretation is to be used for this guest.

---
 hw/s390x/s390-virtio-ccw.c          | 1 +
 target/s390x/cpu_features_def.h.inc | 1 +
 target/s390x/gen-features.c         | 2 ++
 target/s390x/kvm/kvm.c              | 1 +
 4 files changed, 5 insertions(+)
hw/s390x/s390-virtio-ccw.c +1 -0
index 90480e7cf9..b190234308 100644
@@ -805,6 +805,7 @@ static void ccw_machine_6_2_instance_options(MachineState *machine)
     static const S390FeatInit qemu_cpu_feat = { S390_FEAT_LIST_QEMU_V6_2 };
 
     ccw_machine_7_0_instance_options(machine);
+    s390_cpudef_featoff_greater(14, 1, S390_FEAT_ZPCI_INTERP);
     s390_set_qemu_cpu_model(0x3906, 14, 2, qemu_cpu_feat);
 }
 
target/s390x/cpu_features_def.h.inc +1 -0
index e86662bb3b..4ade3182aa 100644
@@ -146,6 +146,7 @@ DEF_FEAT(SIE_CEI, "cei", SCLP_CPU, 43, "SIE: Conditional-external-interception f
 DEF_FEAT(DAT_ENH_2, "dateh2", MISC, 0, "DAT-enhancement facility 2")
 DEF_FEAT(CMM, "cmm", MISC, 0, "Collaborative-memory-management facility")
 DEF_FEAT(AP, "ap", MISC, 0, "AP instructions installed")
+DEF_FEAT(ZPCI_INTERP, "zpci-interp", MISC, 0, "zPCI interpretation")
 
 /* Features exposed via the PLO instruction. */
 DEF_FEAT(PLO_CL, "plo-cl", PLO, 0, "PLO Compare and load (32 bit in general registers)")
target/s390x/gen-features.c +2 -0
index 22846121c4..9db6bd545e 100644
@@ -554,6 +554,7 @@ static uint16_t full_GEN14_GA1[] = {
     S390_FEAT_HPMA2,
     S390_FEAT_SIE_KSS,
     S390_FEAT_GROUP_MULTIPLE_EPOCH_PTFF,
+    S390_FEAT_ZPCI_INTERP,
 };
 
 #define full_GEN14_GA2 EmptyFeat
@@ -650,6 +651,7 @@ static uint16_t default_GEN14_GA1[] = {
     S390_FEAT_GROUP_MSA_EXT_8,
     S390_FEAT_MULTIPLE_EPOCH,
     S390_FEAT_GROUP_MULTIPLE_EPOCH_PTFF,
+    S390_FEAT_ZPCI_INTERP,
 };
 
 #define default_GEN14_GA2 EmptyFeat
target/s390x/kvm/kvm.c +1 -0
index 6acf14d5ec..0357bfda89 100644
@@ -2294,6 +2294,7 @@ static int kvm_to_feat[][2] = {
     { KVM_S390_VM_CPU_FEAT_PFMFI, S390_FEAT_SIE_PFMFI},
     { KVM_S390_VM_CPU_FEAT_SIGPIF, S390_FEAT_SIE_SIGPIF},
     { KVM_S390_VM_CPU_FEAT_KSS, S390_FEAT_SIE_KSS},
+    { KVM_S390_VM_CPU_FEAT_ZPCI_INTERP, S390_FEAT_ZPCI_INTERP },
 };
 
 static int query_cpu_feat(S390FeatBitmap features)
Signature · 1 line
2.27.0
#4 lore raw

PATCH v4 4/11 s390x/pci: add routine to get host function handle from CLP info

In reply to Matthew Rosato

In order to interface with the underlying host zPCI device, we need to know it's function handle. Add a routine to grab this from the vfio CLP capabilities chain.

---
 hw/s390x/s390-pci-vfio.c         | 83 ++++++++++++++++++++++++++------
 include/hw/s390x/s390-pci-vfio.h |  6 +++
 2 files changed, 73 insertions(+), 16 deletions(-)
hw/s390x/s390-pci-vfio.c +67 -16
index 6f80a47e29..4bf0a7e22d 100644
@@ -124,6 +124,27 @@ static void s390_pci_read_base(S390PCIBusDevice *pbdev,
     pbdev->zpci_fn.pft = 0;
 }
 
+static bool get_host_fh(S390PCIBusDevice *pbdev, struct vfio_device_info *info,
+                        uint32_t *fh)
+{
+    struct vfio_info_cap_header *hdr;
+    struct vfio_device_info_cap_zpci_base *cap;
+    VFIOPCIDevice *vpci =  container_of(pbdev->pdev, VFIOPCIDevice, pdev);
+
+    hdr = vfio_get_device_info_cap(info, VFIO_DEVICE_INFO_CAP_ZPCI_BASE);
+
+    /* Can only get the host fh with version 2 or greater */
+    if (hdr == NULL || hdr->version < 2) {
+        trace_s390_pci_clp_cap(vpci->vbasedev.name,
+                               VFIO_DEVICE_INFO_CAP_ZPCI_BASE);
+        return false;
+    }
+    cap = (void *) hdr;
+
+    *fh = cap->fh;
+    return true;
+}
+
 static void s390_pci_read_group(S390PCIBusDevice *pbdev,
                                 struct vfio_device_info *info)
 {
@@ -217,25 +238,13 @@ static void s390_pci_read_pfip(S390PCIBusDevice *pbdev,
     memcpy(pbdev->zpci_fn.pfip, cap->pfip, CLP_PFIP_NR_SEGMENTS);
 }
 
-/*
- * This function will issue the VFIO_DEVICE_GET_INFO ioctl and look for
- * capabilities that contain information about CLP features provided by the
- * underlying host.
- * On entry, defaults have already been placed into the guest CLP response
- * buffers.  On exit, defaults will have been overwritten for any CLP features
- * found in the capability chain; defaults will remain for any CLP features not
- * found in the chain.
- */
-void s390_pci_get_clp_info(S390PCIBusDevice *pbdev)
+static struct vfio_device_info *get_device_info(S390PCIBusDevice *pbdev,
+                                                uint32_t argsz)
 {
-    g_autofree struct vfio_device_info *info = NULL;
+    struct vfio_device_info *info = g_malloc0(argsz);
     VFIOPCIDevice *vfio_pci;
-    uint32_t argsz;
     int fd;
 
-    argsz = sizeof(*info);
-    info = g_malloc0(argsz);
-
     vfio_pci = container_of(pbdev->pdev, VFIOPCIDevice, pdev);
     fd = vfio_pci->vbasedev.fd;
 
@@ -250,7 +259,8 @@ retry:
 
     if (ioctl(fd, VFIO_DEVICE_GET_INFO, info)) {
         trace_s390_pci_clp_dev_info(vfio_pci->vbasedev.name);
-        return;
+        free(info);
+        return NULL;
     }
 
     if (info->argsz > argsz) {
@@ -259,6 +269,47 @@ retry:
         goto retry;
     }
 
+    return info;
+}
+
+/*
+ * Get the host function handle from the vfio CLP capabilities chain.  Returns
+ * true if a fh value was placed into the provided buffer.  Returns false
+ * if a fh could not be obtained (ioctl failed or capabilitiy version does
+ * not include the fh)
+ */
+bool s390_pci_get_host_fh(S390PCIBusDevice *pbdev, uint32_t *fh)
+{
+    g_autofree struct vfio_device_info *info = NULL;
+
+    assert(fh);
+
+    info = get_device_info(pbdev, sizeof(*info));
+    if (!info) {
+        return false;
+    }
+
+    return get_host_fh(pbdev, info, fh);
+}
+
+/*
+ * This function will issue the VFIO_DEVICE_GET_INFO ioctl and look for
+ * capabilities that contain information about CLP features provided by the
+ * underlying host.
+ * On entry, defaults have already been placed into the guest CLP response
+ * buffers.  On exit, defaults will have been overwritten for any CLP features
+ * found in the capability chain; defaults will remain for any CLP features not
+ * found in the chain.
+ */
+void s390_pci_get_clp_info(S390PCIBusDevice *pbdev)
+{
+    g_autofree struct vfio_device_info *info = NULL;
+
+    info = get_device_info(pbdev, sizeof(*info));
+    if (!info) {
+        return;
+    }
+
     /*
      * Find the CLP features provided and fill in the guest CLP responses.
      * Always call s390_pci_read_base first as information from this could
include/hw/s390x/s390-pci-vfio.h +6 -0
index ff708aef50..0c2e4b5175 100644
@@ -20,6 +20,7 @@ bool s390_pci_update_dma_avail(int fd, unsigned int *avail);
 S390PCIDMACount *s390_pci_start_dma_count(S390pciState *s,
                                           S390PCIBusDevice *pbdev);
 void s390_pci_end_dma_count(S390pciState *s, S390PCIDMACount *cnt);
+bool s390_pci_get_host_fh(S390PCIBusDevice *pbdev, uint32_t *fh);
 void s390_pci_get_clp_info(S390PCIBusDevice *pbdev);
 #else
 static inline bool s390_pci_update_dma_avail(int fd, unsigned int *avail)
@@ -33,6 +34,11 @@ static inline S390PCIDMACount *s390_pci_start_dma_count(S390pciState *s,
 }
 static inline void s390_pci_end_dma_count(S390pciState *s,
                                           S390PCIDMACount *cnt) { }
+static inline bool s390_pci_get_host_fh(S390PCIBusDevice *pbdev,
+                                        unsigned int *fh)
+{
+    return false;
+}
 static inline void s390_pci_get_clp_info(S390PCIBusDevice *pbdev) { }
 #endif
 
Signature · 1 line
2.27.0
#5 lore raw

PATCH v4 5/11 s390x/pci: enable for load/store intepretation

In reply to Matthew Rosato

Use the associated kvm ioctl to enable interpretation for devices when requested. As part of this process, we must use the host function handle rather than a QEMU-generated one -- we use an initial value from vfio CLP and maintain an updated fh value from kvm ioctl response info.

By default, unless interpret=off is specified, interpretation support will always be assumed and exploited if the necessary ioctl and features are available on the host kernel. When these are unavailable, we will silently revert to the interception model; this allows existing guest configurations to work unmodified on hosts with and without zPCI interpretation support, allowing QEMU to choose the best support model available.

---
 hw/s390x/meson.build            |   1 +
 hw/s390x/s390-pci-bus.c         |  67 ++++++++++++++++++-
 hw/s390x/s390-pci-inst.c        |  54 ++++++++++++++-
 hw/s390x/s390-pci-kvm.c         | 112 ++++++++++++++++++++++++++++++++
 include/hw/s390x/s390-pci-bus.h |   1 +
 include/hw/s390x/s390-pci-kvm.h |  46 +++++++++++++
 target/s390x/kvm/kvm.c          |   7 ++
 target/s390x/kvm/kvm_s390x.h    |   1 +
 8 files changed, 287 insertions(+), 2 deletions(-)
 create mode 100644 hw/s390x/s390-pci-kvm.c
 create mode 100644 include/hw/s390x/s390-pci-kvm.h
hw/s390x/meson.build +1 -0
index 28484256ec..6e6e47fcda 100644
@@ -23,6 +23,7 @@ s390x_ss.add(when: 'CONFIG_KVM', if_true: files(
   's390-skeys-kvm.c',
   's390-stattrib-kvm.c',
   'pv.c',
+  's390-pci-kvm.c',
 ))
 s390x_ss.add(when: 'CONFIG_TCG', if_true: files(
   'tod-tcg.c',
hw/s390x/s390-pci-bus.c +66 -1

114 lines not on this page · raw message

hw/s390x/s390-pci-inst.c +53 -1

78 lines not on this page · raw message

hw/s390x/s390-pci-kvm.c new file +112 -0

112 lines not on this page · raw message

include/hw/s390x/s390-pci-bus.h +1 -0
index da3cde2bb4..a9843dfe97 100644
@@ -350,6 +350,7 @@ struct S390PCIBusDevice {
     IndAddr *indicator;
     bool pci_unplug_request_processed;
     bool unplug_requested;
+    bool interp;
     QTAILQ_ENTRY(S390PCIBusDevice) link;
 };
 
include/hw/s390x/s390-pci-kvm.h new file +46 -0

46 lines not on this page · raw message

target/s390x/kvm/kvm.c +7 -0
index 0357bfda89..288fbd1d75 100644
@@ -157,6 +157,7 @@ static int cap_ri;
 static int cap_hpage_1m;
 static int cap_vcpu_resets;
 static int cap_protected;
+static int cap_zpci_op;
 
 static int active_cmma;
 
@@ -358,6 +359,7 @@ int kvm_arch_init(MachineState *ms, KVMState *s)
     cap_s390_irq = kvm_check_extension(s, KVM_CAP_S390_INJECT_IRQ);
     cap_vcpu_resets = kvm_check_extension(s, KVM_CAP_S390_VCPU_RESETS);
     cap_protected = kvm_check_extension(s, KVM_CAP_S390_PROTECTED);
+    cap_zpci_op = kvm_check_extension(s, KVM_CAP_S390_ZPCI_OP);
 
     kvm_vm_enable_cap(s, KVM_CAP_S390_USER_SIGP, 0);
     kvm_vm_enable_cap(s, KVM_CAP_S390_VECTOR_REGISTERS, 0);
@@ -2567,3 +2569,8 @@ bool kvm_arch_cpu_check_are_resettable(void)
 {
     return true;
 }
+
+int kvm_s390_get_zpci_op(void)
+{
+    return cap_zpci_op;
+}
target/s390x/kvm/kvm_s390x.h +1 -0
index 05a5e1e6f4..aaae8570de 100644
@@ -27,6 +27,7 @@ void kvm_s390_vcpu_interrupt_pre_save(S390CPU *cpu);
 int kvm_s390_vcpu_interrupt_post_load(S390CPU *cpu);
 int kvm_s390_get_hpage_1m(void);
 int kvm_s390_get_ri(void);
+int kvm_s390_get_zpci_op(void);
 int kvm_s390_get_clock(uint8_t *tod_high, uint64_t *tod_clock);
 int kvm_s390_get_clock_ext(uint8_t *tod_high, uint64_t *tod_clock);
 int kvm_s390_set_clock(uint8_t tod_high, uint64_t tod_clock);
Signature · 1 line
2.27.0
#6 lore raw

PATCH v4 6/11 s390x/pci: don't fence interpreted devices without MSI-X

In reply to Matthew Rosato

Lack of MSI-X support is not an issue for interpreted passthrough devices, so let's let these in. This will allow, for example, ISM devices to be passed through -- but only when interpretation is available and being used.

---
 hw/s390x/s390-pci-bus.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
hw/s390x/s390-pci-bus.c +1 -1
index 7ce7bda26d..acc91af64c 100644
@@ -1087,7 +1087,7 @@ static void s390_pcihost_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
             pbdev->interp = false;
         }
 
-        if (s390_pci_msix_init(pbdev)) {
+        if (s390_pci_msix_init(pbdev) && !pbdev->interp) {
             error_setg(errp, "MSI-X support is mandatory "
                        "in the S390 architecture");
             return;
Signature · 1 line
2.27.0
#7 lore raw

PATCH v4 7/11 s390x/pci: enable adapter event notification for interpreted devices

In reply to Matthew Rosato

Use the associated kvm ioctl operation to enable adapter event notification and forwarding for devices when requested. This feature will be set up with or without firmware assist based upon the 'forwarding_assist' setting.

---
 hw/s390x/s390-pci-bus.c         | 20 ++++++++++++++---
 hw/s390x/s390-pci-inst.c        | 40 +++++++++++++++++++++++++++++++--
 hw/s390x/s390-pci-kvm.c         | 27 ++++++++++++++++++++++
 include/hw/s390x/s390-pci-bus.h |  1 +
 include/hw/s390x/s390-pci-kvm.h | 12 ++++++++++
 5 files changed, 95 insertions(+), 5 deletions(-)
hw/s390x/s390-pci-bus.c +17 -3
index acc91af64c..5043b8c85c 100644
@@ -190,7 +190,10 @@ void s390_pci_sclp_deconfigure(SCCB *sccb)
         rc = SCLP_RC_NO_ACTION_REQUIRED;
         break;
     default:
-        if (pbdev->summary_ind) {
+        if (pbdev->interp && (pbdev->fh & FH_MASK_ENABLE)) {
+            /* Interpreted devices were using interrupt forwarding */
+            s390_pci_kvm_aif_disable(pbdev);
+        } else if (pbdev->summary_ind) {
             pci_dereg_irqs(pbdev);
         }
         if (pbdev->iommu->enabled) {
@@ -1072,6 +1075,7 @@ static void s390_pcihost_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
                 } else {
                     DPRINTF("zPCI interpretation facilities missing.\n");
                     pbdev->interp = false;
+                    pbdev->forwarding_assist = false;
                 }
             }
             pbdev->iommu->dma_limit = s390_pci_start_dma_count(s, pbdev);
@@ -1080,11 +1084,13 @@ static void s390_pcihost_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
             if (!pbdev->interp) {
                 /* Do vfio passthrough but intercept for I/O */
                 pbdev->fh |= FH_SHM_VFIO;
+                pbdev->forwarding_assist = false;
             }
         } else {
             pbdev->fh |= FH_SHM_EMUL;
             /* Always intercept emulated devices */
             pbdev->interp = false;
+            pbdev->forwarding_assist = false;
         }
 
         if (s390_pci_msix_init(pbdev) && !pbdev->interp) {
@@ -1241,7 +1247,10 @@ static void s390_pcihost_reset(DeviceState *dev)
     /* Process all pending unplug requests */
     QTAILQ_FOREACH_SAFE(pbdev, &s->zpci_devs, link, next) {
         if (pbdev->unplug_requested) {
-            if (pbdev->summary_ind) {
+            if (pbdev->interp && (pbdev->fh & FH_MASK_ENABLE)) {
+                /* Interpreted devices were using interrupt forwarding */
+                s390_pci_kvm_aif_disable(pbdev);
+            } else if (pbdev->summary_ind) {
                 pci_dereg_irqs(pbdev);
             }
             if (pbdev->iommu->enabled) {
@@ -1379,7 +1388,10 @@ static void s390_pci_device_reset(DeviceState *dev)
         break;
     }
 
-    if (pbdev->summary_ind) {
+    if (pbdev->interp && (pbdev->fh & FH_MASK_ENABLE)) {
+        /* Interpreted devices were using interrupt forwarding */
+        s390_pci_kvm_aif_disable(pbdev);
+    } else if (pbdev->summary_ind) {
         pci_dereg_irqs(pbdev);
     }
     if (pbdev->iommu->enabled) {
@@ -1425,6 +1437,8 @@ static Property s390_pci_device_properties[] = {
     DEFINE_PROP_S390_PCI_FID("fid", S390PCIBusDevice, fid),
     DEFINE_PROP_STRING("target", S390PCIBusDevice, target),
     DEFINE_PROP_BOOL("interpret", S390PCIBusDevice, interp, true),
+    DEFINE_PROP_BOOL("forwarding_assist", S390PCIBusDevice, forwarding_assist,
+                     true),
     DEFINE_PROP_END_OF_LIST(),
 };
 
hw/s390x/s390-pci-inst.c +38 -2
index 92ea7b73e4..f7b01e2059 100644
@@ -1102,6 +1102,32 @@ static void fmb_update(void *opaque)
     timer_mod(pbdev->fmb_timer, t + pbdev->pci_group->zpci_group.mui);
 }
 
+static int mpcifc_reg_int_interp(S390PCIBusDevice *pbdev, ZpciFib *fib)
+{
+    int rc;
+
+    rc = s390_pci_kvm_aif_enable(pbdev, fib, pbdev->forwarding_assist);
+    if (rc) {
+        DPRINTF("Failed to enable interrupt forwarding\n");
+        return rc;
+    }
+
+    return 0;
+}
+
+static int mpcifc_dereg_int_interp(S390PCIBusDevice *pbdev, ZpciFib *fib)
+{
+    int rc;
+
+    rc = s390_pci_kvm_aif_disable(pbdev);
+    if (rc) {
+        DPRINTF("Failed to disable interrupt forwarding\n");
+        return rc;
+    }
+
+    return 0;
+}
+
 int mpcifc_service_call(S390CPU *cpu, uint8_t r1, uint64_t fiba, uint8_t ar,
                         uintptr_t ra)
 {
@@ -1156,7 +1182,12 @@ int mpcifc_service_call(S390CPU *cpu, uint8_t r1, uint64_t fiba, uint8_t ar,
 
     switch (oc) {
     case ZPCI_MOD_FC_REG_INT:
-        if (pbdev->summary_ind) {
+        if (pbdev->interp) {
+            if (mpcifc_reg_int_interp(pbdev, &fib)) {
+                cc = ZPCI_PCI_LS_ERR;
+                s390_set_status_code(env, r1, ZPCI_MOD_ST_SEQUENCE);
+            }
+        } else if (pbdev->summary_ind) {
             cc = ZPCI_PCI_LS_ERR;
             s390_set_status_code(env, r1, ZPCI_MOD_ST_SEQUENCE);
         } else if (reg_irqs(env, pbdev, fib)) {
@@ -1165,7 +1196,12 @@ int mpcifc_service_call(S390CPU *cpu, uint8_t r1, uint64_t fiba, uint8_t ar,
         }
         break;
     case ZPCI_MOD_FC_DEREG_INT:
-        if (!pbdev->summary_ind) {
+        if (pbdev->interp) {
+            if (mpcifc_dereg_int_interp(pbdev, &fib)) {
+                cc = ZPCI_PCI_LS_ERR;
+                s390_set_status_code(env, r1, ZPCI_MOD_ST_SEQUENCE);
+            }
+        } else if (!pbdev->summary_ind) {
             cc = ZPCI_PCI_LS_ERR;
             s390_set_status_code(env, r1, ZPCI_MOD_ST_SEQUENCE);
         } else {
hw/s390x/s390-pci-kvm.c +27 -0
index 755ea0618a..ebb862abd0 100644
@@ -16,6 +16,7 @@
 #include "kvm/kvm_s390x.h"
 #include "hw/s390x/s390-pci-bus.h"
 #include "hw/s390x/s390-pci-kvm.h"
+#include "hw/s390x/s390-pci-inst.h"
 #include "hw/s390x/s390-pci-vfio.h"
 
 bool s390_pci_kvm_zpciop_allowed(void)
@@ -110,3 +111,29 @@ int s390_pci_kvm_interp_disable(S390PCIBusDevice *pbdev)
 
     return rc;
 }
+
+int s390_pci_kvm_aif_enable(S390PCIBusDevice *pbdev, ZpciFib *fib, bool assist)
+{
+    struct kvm_s390_zpci_op args = {
+        .fh = pbdev->fh,
+        .op = KVM_S390_ZPCIOP_REG_INT,
+        .u.reg_int.ibv = fib->aibv,
+        .u.reg_int.sb = fib->aisb,
+        .u.reg_int.noi = FIB_DATA_NOI(fib->data),
+        .u.reg_int.isc = FIB_DATA_ISC(fib->data),
+        .u.reg_int.sbo = FIB_DATA_AISBO(fib->data),
+        .u.reg_int.flags = (assist) ? 0 : KVM_S390_ZPCIOP_REGINT_HOST
+    };
+
+    return kvm_vm_ioctl(kvm_state, KVM_S390_ZPCI_OP, &args);
+}
+
+int s390_pci_kvm_aif_disable(S390PCIBusDevice *pbdev)
+{
+    struct kvm_s390_zpci_op args = {
+        .fh = pbdev->fh,
+        .op = KVM_S390_ZPCIOP_DEREG_INT
+    };
+
+    return kvm_vm_ioctl(kvm_state, KVM_S390_ZPCI_OP, &args);
+}
include/hw/s390x/s390-pci-bus.h +1 -0
index a9843dfe97..5b09f0cf2f 100644
@@ -351,6 +351,7 @@ struct S390PCIBusDevice {
     bool pci_unplug_request_processed;
     bool unplug_requested;
     bool interp;
+    bool forwarding_assist;
     QTAILQ_ENTRY(S390PCIBusDevice) link;
 };
 
include/hw/s390x/s390-pci-kvm.h +12 -0
index 6b2528cf82..09004d3f6c 100644
@@ -13,6 +13,7 @@
 #define HW_S390_PCI_KVM_H
 
 #include "hw/s390x/s390-pci-bus.h"
+#include "hw/s390x/s390-pci-inst.h"
 
 #ifdef CONFIG_KVM
 bool s390_pci_kvm_zpciop_allowed(void);
@@ -20,6 +21,8 @@ int s390_pci_kvm_plug(S390PCIBusDevice *pbdev);
 int s390_pci_kvm_unplug(S390PCIBusDevice *pbdev);
 int s390_pci_kvm_interp_enable(S390PCIBusDevice *pbdev);
 int s390_pci_kvm_interp_disable(S390PCIBusDevice *pbdev);
+int s390_pci_kvm_aif_enable(S390PCIBusDevice *pbdev, ZpciFib *fib, bool assist);
+int s390_pci_kvm_aif_disable(S390PCIBusDevice *pbdev);
 #else
 static inline bool s390_pci_kvm_zpciop_allowed(void)
 {
@@ -41,6 +44,15 @@ static inline int s390_pci_kvm_interp_enable(S390PCIBusDevice *pbdev)
 {
     return -EINVAL;
 }
+static inline int s390_pci_kvm_aif_enable(S390PCIBusDevice *pbdev, ZpciFib *fib,
+                                          bool assist)
+{
+    return -EINVAL;
+}
+static inline int s390_pci_kvm_aif_disable(S390PCIBusDevice *pbdev)
+{
+    return -EINVAL;
+}
 #endif
 
 #endif
Signature · 1 line
2.27.0
#8 lore raw

PATCH v4 8/11 s390x/pci: use KVM-managed IOMMU for interpretation

In reply to Matthew Rosato

When interpreting zPCI instructions, KVM will control the IOMMU mappings in response to RPCIT instructions rather than relying on mapping ioctls from userspace. Mark the vfio device in pre_plug so that the appropriate iommu domain will be allocated on the host during VFIO_SET_IOMMU.

---
 hw/s390x/s390-pci-bus.c          | 11 +++++++++++
 hw/s390x/s390-pci-vfio.c         | 22 ++++++++++++++++++++++
 include/hw/s390x/s390-pci-vfio.h |  5 +++++
 3 files changed, 38 insertions(+)
hw/s390x/s390-pci-bus.c +11 -0
index 5043b8c85c..513a276711 100644
@@ -950,6 +950,17 @@ static void s390_pcihost_pre_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
             error_setg(errp, "multifunction not supported in s390");
             return;
         }
+        /*
+         * If we have a vfio-pci device that wishes to use interpretation
+         * we must update the host IOMMU domain ops.
+         */
+        if (s390_pci_kvm_zpciop_allowed() &&
+            object_dynamic_cast(OBJECT(dev), "vfio-pci")) {
+            if (s390_pci_set_kvm_iommu(s, dev)) {
+                error_setg(errp, "KVM IOMMU not available for interpretation");
+                return;
+            }
+        }
     } else if (object_dynamic_cast(OBJECT(dev), TYPE_S390_PCI_DEVICE)) {
         S390PCIBusDevice *pbdev = S390_PCI_DEVICE(dev);
 
hw/s390x/s390-pci-vfio.c +22 -0
index 4bf0a7e22d..7808e8d939 100644
@@ -324,3 +324,25 @@ void s390_pci_get_clp_info(S390PCIBusDevice *pbdev)
 
     return;
 }
+
+/*
+ * This function will determine if the specified VFIOPCIDevice is linked to a
+ * zPCI device that requests interpretation support.  In this case, we must
+ * inform vfio that the KVM-managed IOMMU should be requested when the
+ * VFIO_SET_IOMMU ioctl is issued.
+ */
+int s390_pci_set_kvm_iommu(S390pciState *s, DeviceState *dev)
+{
+    VFIOPCIDevice *vdev = VFIO_PCI(dev);
+    S390PCIBusDevice *pbdev = s390_pci_find_dev_by_target(s, dev->id);
+
+    if (!pbdev) {
+        return -ENODEV;
+    }
+
+    if (pbdev->interp) {
+        vdev->kvm_managed_iommu = true;
+    }
+
+    return 0;
+}
include/hw/s390x/s390-pci-vfio.h +5 -0
index 0c2e4b5175..5026f978c2 100644
@@ -22,6 +22,7 @@ S390PCIDMACount *s390_pci_start_dma_count(S390pciState *s,
 void s390_pci_end_dma_count(S390pciState *s, S390PCIDMACount *cnt);
 bool s390_pci_get_host_fh(S390PCIBusDevice *pbdev, uint32_t *fh);
 void s390_pci_get_clp_info(S390PCIBusDevice *pbdev);
+int s390_pci_set_kvm_iommu(S390pciState *s, DeviceState *dev);
 #else
 static inline bool s390_pci_update_dma_avail(int fd, unsigned int *avail)
 {
@@ -40,6 +41,10 @@ static inline bool s390_pci_get_host_fh(S390PCIBusDevice *pbdev,
     return false;
 }
 static inline void s390_pci_get_clp_info(S390PCIBusDevice *pbdev) { }
+static inline int s390_pci_set_kvm_iommu(S390pciState *s, DeviceState *dev)
+{
+    return -EINVAL;
+}
 #endif
 
 #endif
Signature · 1 line
2.27.0
#9 lore raw

PATCH v4 9/11 s390x/pci: use I/O Address Translation assist when interpreting

In reply to Matthew Rosato

Allow the underlying kvm host to handle the Refresh PCI Translation instruction intercepts.

---
 hw/s390x/s390-pci-bus.c          |  6 ++---
 hw/s390x/s390-pci-inst.c         | 42 +++++++++++++++++++++++++++++---
 hw/s390x/s390-pci-kvm.c          | 21 ++++++++++++++++
 include/hw/s390x/s390-pci-inst.h |  2 +-
 include/hw/s390x/s390-pci-kvm.h  | 10 ++++++++
 5 files changed, 74 insertions(+), 7 deletions(-)
hw/s390x/s390-pci-bus.c +3 -3
index 513a276711..32894fbe9c 100644
@@ -197,7 +197,7 @@ void s390_pci_sclp_deconfigure(SCCB *sccb)
             pci_dereg_irqs(pbdev);
         }
         if (pbdev->iommu->enabled) {
-            pci_dereg_ioat(pbdev->iommu);
+            pci_dereg_ioat(pbdev);
         }
         pbdev->state = ZPCI_FS_STANDBY;
         rc = SCLP_RC_NORMAL_COMPLETION;
@@ -1265,7 +1265,7 @@ static void s390_pcihost_reset(DeviceState *dev)
                 pci_dereg_irqs(pbdev);
             }
             if (pbdev->iommu->enabled) {
-                pci_dereg_ioat(pbdev->iommu);
+                pci_dereg_ioat(pbdev);
             }
             pbdev->state = ZPCI_FS_STANDBY;
             s390_pci_perform_unplug(pbdev);
@@ -1406,7 +1406,7 @@ static void s390_pci_device_reset(DeviceState *dev)
         pci_dereg_irqs(pbdev);
     }
     if (pbdev->iommu->enabled) {
-        pci_dereg_ioat(pbdev->iommu);
+        pci_dereg_ioat(pbdev);
     }
 
     fmb_timer_free(pbdev);
hw/s390x/s390-pci-inst.c +39 -3

73 lines not on this page · raw message

hw/s390x/s390-pci-kvm.c +21 -0
index ebb862abd0..2332efd676 100644
@@ -137,3 +137,24 @@ int s390_pci_kvm_aif_disable(S390PCIBusDevice *pbdev)
 
     return kvm_vm_ioctl(kvm_state, KVM_S390_ZPCI_OP, &args);
 }
+
+int s390_pci_kvm_ioat_enable(S390PCIBusDevice *pbdev, uint64_t iota)
+{
+    struct kvm_s390_zpci_op args = {
+        .fh = pbdev->fh,
+        .op = KVM_S390_ZPCIOP_REG_IOAT,
+        .u.reg_ioat.iota = iota
+    };
+
+    return kvm_vm_ioctl(kvm_state, KVM_S390_ZPCI_OP, &args);
+}
+
+int s390_pci_kvm_ioat_disable(S390PCIBusDevice *pbdev)
+{
+    struct kvm_s390_zpci_op args = {
+        .fh = pbdev->fh,
+        .op = KVM_S390_ZPCIOP_DEREG_IOAT
+    };
+
+    return kvm_vm_ioctl(kvm_state, KVM_S390_ZPCI_OP, &args);
+}
include/hw/s390x/s390-pci-inst.h +1 -1
index a55c448aad..13566fb7b4 100644
@@ -99,7 +99,7 @@ typedef struct ZpciFib {
 } QEMU_PACKED ZpciFib;
 
 int pci_dereg_irqs(S390PCIBusDevice *pbdev);
-void pci_dereg_ioat(S390PCIIOMMU *iommu);
+void pci_dereg_ioat(S390PCIBusDevice *pbdev);
 int clp_service_call(S390CPU *cpu, uint8_t r2, uintptr_t ra);
 int pcilg_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2, uintptr_t ra);
 int pcistg_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2, uintptr_t ra);
include/hw/s390x/s390-pci-kvm.h +10 -0
index 09004d3f6c..e52e92d0f0 100644
@@ -23,6 +23,8 @@ int s390_pci_kvm_interp_enable(S390PCIBusDevice *pbdev);
 int s390_pci_kvm_interp_disable(S390PCIBusDevice *pbdev);
 int s390_pci_kvm_aif_enable(S390PCIBusDevice *pbdev, ZpciFib *fib, bool assist);
 int s390_pci_kvm_aif_disable(S390PCIBusDevice *pbdev);
+int s390_pci_kvm_ioat_enable(S390PCIBusDevice *pbdev, uint64_t iota);
+int s390_pci_kvm_ioat_disable(S390PCIBusDevice *pbdev);
 #else
 static inline bool s390_pci_kvm_zpciop_allowed(void)
 {
@@ -53,6 +55,14 @@ static inline int s390_pci_kvm_aif_disable(S390PCIBusDevice *pbdev)
 {
     return -EINVAL;
 }
+int s390_pci_kvm_ioat_enable(S390PCIBusDevice *pbdev, uint64_t iota)
+{
+    return -EINVAL;
+}
+int s390_pci_kvm_ioat_disable(S390PCIBusDevice *pbdev)
+{
+    return -EINVAL;
+}
 #endif
 
 #endif
Signature · 1 line
2.27.0
#10 lore raw

PATCH v4 10/11 s390x/pci: use dtsm provided from vfio capabilities for interpreted devices

In reply to Matthew Rosato

When using the IOAT assist via interpretation, we should advertise what the host driver supports, not QEMU.

---
 hw/s390x/s390-pci-vfio.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)
hw/s390x/s390-pci-vfio.c +5 -1
index 7808e8d939..def4e90a40 100644
@@ -181,7 +181,11 @@ static void s390_pci_read_group(S390PCIBusDevice *pbdev,
         resgrp->i = cap->noi;
         resgrp->maxstbl = cap->maxstbl;
         resgrp->version = cap->version;
-        resgrp->dtsm = ZPCI_DTSM;
+        if (hdr->version >= 2 && pbdev->interp) {
+            resgrp->dtsm = cap->dtsm;
+        } else {
+            resgrp->dtsm = ZPCI_DTSM;
+        }
     }
 }
 
Signature · 1 line
2.27.0
#11 lore raw

PATCH v4 11/11 s390x/pci: let intercept devices have separate PCI groups

In reply to Matthew Rosato

Let's use the reserved pool of simulated PCI groups to allow intercept devices to have separate groups from interpreted devices as some group values may be different. If we run out of simulated PCI groups, subsequent intercept devices just get the default group.
Furthermore, if we encounter any PCI groups from hostdevs that are marked as simulated, let's just assign them to the default group to avoid conflicts between host simulated groups and our own simulated groups.

---
 hw/s390x/s390-pci-bus.c         | 19 ++++++++++++++--
 hw/s390x/s390-pci-vfio.c        | 40 ++++++++++++++++++++++++++++++---
 include/hw/s390x/s390-pci-bus.h |  6 ++++-
 3 files changed, 59 insertions(+), 6 deletions(-)
hw/s390x/s390-pci-bus.c +17 -2
index 32894fbe9c..e932f72e89 100644
@@ -748,13 +748,14 @@ static void s390_pci_iommu_free(S390pciState *s, PCIBus *bus, int32_t devfn)
     object_unref(OBJECT(iommu));
 }
 
-S390PCIGroup *s390_group_create(int id)
+S390PCIGroup *s390_group_create(int id, int host_id)
 {
     S390PCIGroup *group;
     S390pciState *s = s390_get_phb();
 
     group = g_new0(S390PCIGroup, 1);
     group->id = id;
+    group->host_id = host_id;
     QTAILQ_INSERT_TAIL(&s->zpci_groups, group, link);
     return group;
 }
@@ -772,12 +773,25 @@ S390PCIGroup *s390_group_find(int id)
     return NULL;
 }
 
+S390PCIGroup *s390_group_find_host_sim(int host_id)
+{
+    S390PCIGroup *group;
+    S390pciState *s = s390_get_phb();
+
+    QTAILQ_FOREACH(group, &s->zpci_groups, link) {
+        if (group->id >= ZPCI_SIM_GRP_START && group->host_id == host_id) {
+            return group;
+        }
+    }
+    return NULL;
+}
+
 static void s390_pci_init_default_group(void)
 {
     S390PCIGroup *group;
     ClpRspQueryPciGrp *resgrp;
 
-    group = s390_group_create(ZPCI_DEFAULT_FN_GRP);
+    group = s390_group_create(ZPCI_DEFAULT_FN_GRP, ZPCI_DEFAULT_FN_GRP);
     resgrp = &group->zpci_group;
     resgrp->fr = 1;
     resgrp->dasm = 0;
@@ -825,6 +839,7 @@ static void s390_pcihost_realize(DeviceState *dev, Error **errp)
                                            NULL, g_free);
     s->zpci_table = g_hash_table_new_full(g_int_hash, g_int_equal, NULL, NULL);
     s->bus_no = 0;
+    s->next_sim_grp = ZPCI_SIM_GRP_START;
     QTAILQ_INIT(&s->pending_sei);
     QTAILQ_INIT(&s->zpci_devs);
     QTAILQ_INIT(&s->zpci_dma_limit);
hw/s390x/s390-pci-vfio.c +37 -3
index def4e90a40..de24c4ddf0 100644
@@ -150,13 +150,17 @@ static void s390_pci_read_group(S390PCIBusDevice *pbdev,
 {
     struct vfio_info_cap_header *hdr;
     struct vfio_device_info_cap_zpci_group *cap;
+    S390pciState *s = s390_get_phb();
     ClpRspQueryPciGrp *resgrp;
     VFIOPCIDevice *vpci =  container_of(pbdev->pdev, VFIOPCIDevice, pdev);
 
     hdr = vfio_get_device_info_cap(info, VFIO_DEVICE_INFO_CAP_ZPCI_GROUP);
 
-    /* If capability not provided, just use the default group */
-    if (hdr == NULL) {
+    /*
+     * If capability not provided or the underlying hostdev is simulated, just
+     * use the default group.
+     */
+    if (hdr == NULL || pbdev->zpci_fn.pfgid >= ZPCI_SIM_GRP_START) {
         trace_s390_pci_clp_cap(vpci->vbasedev.name,
                                VFIO_DEVICE_INFO_CAP_ZPCI_GROUP);
         pbdev->zpci_fn.pfgid = ZPCI_DEFAULT_FN_GRP;
@@ -165,11 +169,41 @@ static void s390_pci_read_group(S390PCIBusDevice *pbdev,
     }
     cap = (void *) hdr;
 
+    /*
+     * For an intercept device, let's use an existing simulated group if one
+     * one was already created for other intercept devices in this group.
+     * If not, create a new simulated group if any are still available.
+     * If all else fails, just fall back on the default group.
+     */
+    if (!pbdev->interp) {
+        pbdev->pci_group = s390_group_find_host_sim(pbdev->zpci_fn.pfgid);
+        if (pbdev->pci_group) {
+            /* Use existing simulated group */
+            pbdev->zpci_fn.pfgid = pbdev->pci_group->id;
+            return;
+        } else {
+            if (s->next_sim_grp == ZPCI_DEFAULT_FN_GRP) {
+                /* All out of simulated groups, use default */
+                trace_s390_pci_clp_cap(vpci->vbasedev.name,
+                                       VFIO_DEVICE_INFO_CAP_ZPCI_GROUP);
+                pbdev->zpci_fn.pfgid = ZPCI_DEFAULT_FN_GRP;
+                pbdev->pci_group = s390_group_find(ZPCI_DEFAULT_FN_GRP);
+                return;
+            } else {
+                /* We can assign a new simulated group */
+                pbdev->zpci_fn.pfgid = s->next_sim_grp;
+                s->next_sim_grp++;
+                /* Fall through to create the new sim group using CLP info */
+            }
+        }
+    }
+
     /* See if the PCI group is already defined, create if not */
     pbdev->pci_group = s390_group_find(pbdev->zpci_fn.pfgid);
 
     if (!pbdev->pci_group) {
-        pbdev->pci_group = s390_group_create(pbdev->zpci_fn.pfgid);
+        pbdev->pci_group = s390_group_create(pbdev->zpci_fn.pfgid,
+                                             pbdev->zpci_fn.pfgid);
 
         resgrp = &pbdev->pci_group->zpci_group;
         if (cap->flags & VFIO_DEVICE_INFO_ZPCI_FLAG_REFRESH) {
include/hw/s390x/s390-pci-bus.h +5 -1
index 5b09f0cf2f..0605fcea24 100644
@@ -315,13 +315,16 @@ typedef struct ZpciFmb {
 QEMU_BUILD_BUG_MSG(offsetof(ZpciFmb, fmt0) != 48, "padding in ZpciFmb");
 
 #define ZPCI_DEFAULT_FN_GRP 0xFF
+#define ZPCI_SIM_GRP_START 0xF0
 typedef struct S390PCIGroup {
     ClpRspQueryPciGrp zpci_group;
     int id;
+    int host_id;
     QTAILQ_ENTRY(S390PCIGroup) link;
 } S390PCIGroup;
-S390PCIGroup *s390_group_create(int id);
+S390PCIGroup *s390_group_create(int id, int host_id);
 S390PCIGroup *s390_group_find(int id);
+S390PCIGroup *s390_group_find_host_sim(int host_id);
 
 struct S390PCIBusDevice {
     DeviceState qdev;
@@ -370,6 +373,7 @@ struct S390pciState {
     QTAILQ_HEAD(, S390PCIBusDevice) zpci_devs;
     QTAILQ_HEAD(, S390PCIDMACount) zpci_dma_limit;
     QTAILQ_HEAD(, S390PCIGroup) zpci_groups;
+    uint8_t next_sim_grp;
 };
 
 S390pciState *s390_get_phb(void);
Signature · 1 line
2.27.0