Skip to contentSkip to conversation

Retrieving zPCI specific info from QEMU

PATCH Superseded Pierre Morel 3 participants 10 messages May 10, 2019 – May 16, 2019

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

cover Retrieving zPCI specific info from QEMU

This patch implement the QEMU part to retrieve ZPCI specific information from the host.
The Linux part has been posted on a separate patch on the LKML.

Subject: [PATCH 0/4] Retrieving zPCI specific info with VFIO
Message-Id: <[email protected]>

We use the PCI VFIO interface to retrieve ZPCI specific information from the host.

The Linux patch enhance the VFIO_IOMMU_GET_INFO ioctl with a new Z specific capability and we use it to access the information stored in the zPCI function.

The retrieval is done only once per function and function group during the plugging of the device.
The guest's requests are filled with shadow values we keep for the duration of the device remains plugged.

Still there are some values we need to virtualize, like the UID and FID of the zPCI function, and we currently only allow the refresh bit for the zPCI group flags.

Note that we export the CLP specific definitions in a dedicated file for clarity.

Pierre Morel (5):
vfio: vfio_iommu_type1: linux header place holder
s390: PCI: Creation a header dedicated to PCI CLP
s390: vfio_pci: Use a PCI Group structure
s390: vfio_pci: Use a PCI Function structure
s390: vfio_pci: Get zPCI function info from host

 hw/s390x/s390-pci-bus.c    | 142 ++++++++++++++++++++++++++++--
 hw/s390x/s390-pci-bus.h    |  13 ++-
 hw/s390x/s390-pci-clp.h    | 211 +++++++++++++++++++++++++++++++++++++++++++++
 hw/s390x/s390-pci-inst.c   |  28 +++---
 hw/s390x/s390-pci-inst.h   | 196 -----------------------------------------
 linux-headers/linux/vfio.h |  16 +++-
 6 files changed, 386 insertions(+), 220 deletions(-)
 create mode 100644 hw/s390x/s390-pci-clp.h
Signature · 1 line
2.7.4
#1 lore raw

PATCH 1/5 vfio: vfio_iommu_type1: linux header place holder

This should be copied from Linux kernel UAPI includes.

---
 linux-headers/linux/vfio.h | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)
linux-headers/linux/vfio.h +13 -3
index 12a7b1d..eaecaef 100644
@@ -9,8 +9,8 @@
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  */
-#ifndef VFIO_H
-#define VFIO_H
+#ifndef _UAPIVFIO_H
+#define _UAPIVFIO_H
 
 #include <linux/types.h>
 #include <linux/ioctl.h>
@@ -711,6 +711,16 @@ struct vfio_iommu_type1_info {
 	__u32	flags;
 #define VFIO_IOMMU_INFO_PGSIZES (1 << 0)	/* supported page sizes info */
 	__u64	iova_pgsizes;		/* Bitmap of supported page sizes */
+#define VFIO_IOMMU_INFO_CAPABILITIES (1 << 1)  /* support capabilities info */
+	__u64   cap_offset;     /* Offset within info struct of first cap */
+};
+
+#define VFIO_IOMMU_INFO_CAP_QFN		1
+#define VFIO_IOMMU_INFO_CAP_QGRP	2
+
+struct vfio_iommu_type1_info_block {
+	struct vfio_info_cap_header header;
+	__u32 data[];
 };
 
 #define VFIO_IOMMU_GET_INFO _IO(VFIO_TYPE, VFIO_BASE + 12)
@@ -910,4 +920,4 @@ struct vfio_iommu_spapr_tce_remove {
 
 /* ***************************************************************** */
 
-#endif /* VFIO_H */
+#endif /* _UAPIVFIO_H */
Signature · 1 line
2.7.4
#2 lore raw

Re: [Qemu-devel] [PATCH 1/5] vfio: vfio_iommu_type1: linux header place holder

Quoted 3 lines · Pierre Morel

This should be copied from Linux kernel UAPI includes.

pls add a note which linux version did you sync with.

Quoted 44 lines
---
 linux-headers/linux/vfio.h | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)
linux-headers/linux/vfio.h +13 -3
index 12a7b1d..eaecaef 100644
@@ -9,8 +9,8 @@
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  */
-#ifndef VFIO_H
-#define VFIO_H
+#ifndef _UAPIVFIO_H
+#define _UAPIVFIO_H
 
 #include <linux/types.h>
 #include <linux/ioctl.h>
@@ -711,6 +711,16 @@ struct vfio_iommu_type1_info {
 	__u32	flags;
 #define VFIO_IOMMU_INFO_PGSIZES (1 << 0)	/* supported page sizes info */
 	__u64	iova_pgsizes;		/* Bitmap of supported page sizes */
+#define VFIO_IOMMU_INFO_CAPABILITIES (1 << 1)  /* support capabilities info */
+	__u64   cap_offset;     /* Offset within info struct of first cap */
+};
+
+#define VFIO_IOMMU_INFO_CAP_QFN		1
+#define VFIO_IOMMU_INFO_CAP_QGRP	2
+
+struct vfio_iommu_type1_info_block {
+	struct vfio_info_cap_header header;
+	__u32 data[];
 };
 
 #define VFIO_IOMMU_GET_INFO _IO(VFIO_TYPE, VFIO_BASE + 12)
@@ -910,4 +920,4 @@ struct vfio_iommu_spapr_tce_remove {
 
 /* ***************************************************************** */
 
-#endif /* VFIO_H */
+#endif /* _UAPIVFIO_H */
Signature · 1 line
2.7.4
#3 lore raw

Re: [Qemu-devel] [PATCH 1/5] vfio: vfio_iommu_type1: linux header place holder

Quoted 6 lines · Michael S. Tsirkin
Quoted 3 lines · Pierre Morel

This should be copied from Linux kernel UAPI includes.

pls add a note which linux version did you sync with.

I will, thanks.
Pierre

Quoted 46 lines
Quoted 45 lines
---
  linux-headers/linux/vfio.h | 16 +++++++++++++---
  1 file changed, 13 insertions(+), 3 deletions(-)
linux-headers/linux/vfio.h +13 -3
index 12a7b1d..eaecaef 100644
@@ -9,8 +9,8 @@
   * it under the terms of the GNU General Public License version 2 as
   * published by the Free Software Foundation.
   */
-#ifndef VFIO_H
-#define VFIO_H
+#ifndef _UAPIVFIO_H
+#define _UAPIVFIO_H
 
  #include <linux/types.h>
  #include <linux/ioctl.h>
@@ -711,6 +711,16 @@ struct vfio_iommu_type1_info {
  	__u32	flags;
  #define VFIO_IOMMU_INFO_PGSIZES (1 << 0)	/* supported page sizes info */
  	__u64	iova_pgsizes;		/* Bitmap of supported page sizes */
+#define VFIO_IOMMU_INFO_CAPABILITIES (1 << 1)  /* support capabilities info */
+	__u64   cap_offset;     /* Offset within info struct of first cap */
+};
+
+#define VFIO_IOMMU_INFO_CAP_QFN		1
+#define VFIO_IOMMU_INFO_CAP_QGRP	2
+
+struct vfio_iommu_type1_info_block {
+	struct vfio_info_cap_header header;
+	__u32 data[];
  };
 
  #define VFIO_IOMMU_GET_INFO _IO(VFIO_TYPE, VFIO_BASE + 12)
@@ -910,4 +920,4 @@ struct vfio_iommu_spapr_tce_remove {
 
  /* ***************************************************************** */
 
-#endif /* VFIO_H */
+#endif /* _UAPIVFIO_H */
Signature · 1 line
2.7.4
Signature · 2 lines
Pierre Morel
Linux/KVM/QEMU in Böblingen - Germany
#4 lore raw

PATCH 2/5 s390: PCI: Creation a header dedicated to PCI CLP

In reply to Pierre Morel

To have a clean separation between s390-pci-bus.h
and s390-pci-inst.h headers we export the PCI CLP
instructions in a dedicated header.

---
 hw/s390x/s390-pci-bus.h  |   1 +
 hw/s390x/s390-pci-clp.h  | 211 +++++++++++++++++++++++++++++++++++++++++++++++
 hw/s390x/s390-pci-inst.h | 196 -------------------------------------------
 3 files changed, 212 insertions(+), 196 deletions(-)
 create mode 100644 hw/s390x/s390-pci-clp.h
hw/s390x/s390-pci-bus.h +1 -0
index 550f3cc..a5d2049 100644
@@ -19,6 +19,7 @@
 #include "hw/s390x/sclp.h"
 #include "hw/s390x/s390_flic.h"
 #include "hw/s390x/css.h"
+#include "s390-pci-clp.h"
 
 #define TYPE_S390_PCI_HOST_BRIDGE "s390-pcihost"
 #define TYPE_S390_PCI_BUS "s390-pcibus"
hw/s390x/s390-pci-clp.h new file +211 -0

211 lines not on this page · raw message

hw/s390x/s390-pci-inst.h +0 -196

202 lines not on this page · raw message

Signature · 1 line
2.7.4
#5 lore raw

PATCH 3/5 s390: vfio_pci: Use a PCI Group structure

In reply to Pierre Morel

We use a S390PCIGroup structure to hold the information related to zPCI Function group.

This allows us to be ready to support multiple groups and to retrieve the group information from the host.

---
 hw/s390x/s390-pci-bus.c  | 42 ++++++++++++++++++++++++++++++++++++++++++
 hw/s390x/s390-pci-bus.h  | 11 ++++++++++-
 hw/s390x/s390-pci-inst.c | 22 +++++++++++++---------
 3 files changed, 65 insertions(+), 10 deletions(-)
hw/s390x/s390-pci-bus.c +42 -0
index 2d0a28d..175ea8c 100644
@@ -729,6 +729,46 @@ static void s390_pci_iommu_free(S390pciState *s, PCIBus *bus, int32_t devfn)
     object_unref(OBJECT(iommu));
 }
 
+static S390PCIGroup *s390_grp_create(int ug)
+{
+    S390PCIGroup *grp;
+    S390pciState *s = s390_get_phb();
+
+    grp = g_new0(S390PCIGroup, 1);
+    grp->ug = ug;
+    QTAILQ_INSERT_TAIL(&s->zpci_grps, grp, link);
+    return grp;
+}
+
+S390PCIGroup *s390_grp_find(int ug)
+{
+    S390PCIGroup *grp;
+    S390pciState *s = s390_get_phb();
+
+    QTAILQ_FOREACH(grp, &s->zpci_grps, link) {
+        if ((grp->ug & CLP_REQ_QPCIG_MASK_PFGID) == ug) {
+            return grp;
+        }
+    }
+    return NULL;
+}
+
+static void s390_pci_init_default_group(void)
+{
+    S390PCIGroup *grp;
+    ClpRspQueryPciGrp *resgrp;
+
+    grp = s390_grp_create(ZPCI_DEFAULT_FN_GRP);
+    resgrp = &grp->zpci_grp;
+    resgrp->fr = 1;
+    stq_p(&resgrp->dasm, 0);
+    stq_p(&resgrp->msia, ZPCI_MSI_ADDR);
+    stw_p(&resgrp->mui, DEFAULT_MUI);
+    stw_p(&resgrp->i, 128);
+    stw_p(&resgrp->maxstbl, 128);
+    resgrp->version = 0;
+}
+
 static void s390_pcihost_realize(DeviceState *dev, Error **errp)
 {
     PCIBus *b;
@@ -765,7 +805,9 @@ static void s390_pcihost_realize(DeviceState *dev, Error **errp)
     s->bus_no = 0;
     QTAILQ_INIT(&s->pending_sei);
     QTAILQ_INIT(&s->zpci_devs);
+    QTAILQ_INIT(&s->zpci_grps);
 
+    s390_pci_init_default_group();
     css_register_io_adapters(CSS_IO_ADAPTER_PCI, true, false,
                              S390_ADAPTER_SUPPRESSIBLE, &local_err);
     error_propagate(errp, local_err);
hw/s390x/s390-pci-bus.h +10 -1
index a5d2049..e95a797 100644
@@ -312,6 +312,14 @@ typedef struct ZpciFmb {
 } ZpciFmb;
 QEMU_BUILD_BUG_MSG(offsetof(ZpciFmb, fmt0) != 48, "padding in ZpciFmb");
 
+#define ZPCI_DEFAULT_FN_GRP 0x20
+typedef struct S390PCIGroup {
+    ClpRspQueryPciGrp zpci_grp;
+    int ug;
+    QTAILQ_ENTRY(S390PCIGroup) link;
+} S390PCIGroup;
+S390PCIGroup *s390_grp_find(int ug);
+
 struct S390PCIBusDevice {
     DeviceState qdev;
     PCIDevice *pdev;
@@ -327,8 +335,8 @@ struct S390PCIBusDevice {
     QEMUTimer *fmb_timer;
     uint8_t isc;
     uint16_t noi;
-    uint16_t maxstbl;
     uint8_t sum;
+    S390PCIGroup *pci_grp;
     S390MsixInfo msix;
     AdapterRoutes routes;
     S390PCIIOMMU *iommu;
@@ -353,6 +361,7 @@ typedef struct S390pciState {
     GHashTable *zpci_table;
     QTAILQ_HEAD(, SeiContainer) pending_sei;
     QTAILQ_HEAD(, S390PCIBusDevice) zpci_devs;
+    QTAILQ_HEAD(, S390PCIGroup) zpci_grps;
 } S390pciState;
 
 S390pciState *s390_get_phb(void);
hw/s390x/s390-pci-inst.c +13 -9
index be28962..8147847 100644
@@ -284,21 +284,25 @@ int clp_service_call(S390CPU *cpu, uint8_t r2, uintptr_t ra)
         stq_p(&resquery->edma, ZPCI_EDMA_ADDR);
         stl_p(&resquery->fid, pbdev->fid);
         stw_p(&resquery->pchid, 0);
-        stw_p(&resquery->ug, 1);
+        stw_p(&resquery->ug, ZPCI_DEFAULT_FN_GRP);
         stl_p(&resquery->uid, pbdev->uid);
         stw_p(&resquery->hdr.rsp, CLP_RC_OK);
         break;
     }
     case CLP_QUERY_PCI_FNGRP: {
         ClpRspQueryPciGrp *resgrp = (ClpRspQueryPciGrp *)resh;
-        resgrp->fr = 1;
-        stq_p(&resgrp->dasm, 0);
-        stq_p(&resgrp->msia, ZPCI_MSI_ADDR);
-        stw_p(&resgrp->mui, DEFAULT_MUI);
-        stw_p(&resgrp->i, 128);
-        stw_p(&resgrp->maxstbl, 128);
-        resgrp->version = 0;
 
+        ClpReqQueryPciGrp *reqgrp = (ClpReqQueryPciGrp *)reqh;
+        S390PCIGroup *grp;
+
+        grp = s390_grp_find(reqgrp->g);
+        if (!grp) {
+            /* We do not allow access to unknown groups */
+            /* The group must have been obtained with a vfio device */
+            stw_p(&resgrp->hdr.rsp, CLP_RC_QUERYPCIFG_PFGID);
+            goto out;
+        }
+        memcpy(resgrp, &grp->zpci_grp, sizeof(ClpRspQueryPciGrp));
         stw_p(&resgrp->hdr.rsp, CLP_RC_OK);
         break;
     }
@@ -752,7 +756,7 @@ int pcistb_service_call(S390CPU *cpu, uint8_t r1, uint8_t r3, uint64_t gaddr,
     }
     /* Length must be greater than 8, a multiple of 8 */
     /* and not greater than maxstbl */
-    if ((len <= 8) || (len % 8) || (len > pbdev->maxstbl)) {
+    if ((len <= 8) || (len % 8) || (len > pbdev->pci_grp->zpci_grp.maxstbl)) {
         goto specification_error;
     }
     /* Do not cross a 4K-byte boundary */
Signature · 1 line
2.7.4
#6 lore raw

Re: [Qemu-devel] [PATCH 3/5] s390: vfio_pci: Use a PCI Group structure

Quoted 5 lines · On Fri, 10 May 2019 16:38:51 +0200 Pierre Morel <[email protected]> wrote:

We use a S390PCIGroup structure to hold the information related to zPCI Function group.

This allows us to be ready to support multiple groups and to retrieve the group information from the host.

What if there is no host to retrieve information from?

Quoted 29 lines
---
 hw/s390x/s390-pci-bus.c  | 42 ++++++++++++++++++++++++++++++++++++++++++
 hw/s390x/s390-pci-bus.h  | 11 ++++++++++-
 hw/s390x/s390-pci-inst.c | 22 +++++++++++++---------
 3 files changed, 65 insertions(+), 10 deletions(-)
hw/s390x/s390-pci-inst.c +1 -6
index be28962..8147847 100644
@@ -284,21 +284,25 @@ int clp_service_call(S390CPU *cpu, uint8_t r2, uintptr_t ra)
         stq_p(&resquery->edma, ZPCI_EDMA_ADDR);
         stl_p(&resquery->fid, pbdev->fid);
         stw_p(&resquery->pchid, 0);
-        stw_p(&resquery->ug, 1);
+        stw_p(&resquery->ug, ZPCI_DEFAULT_FN_GRP);
         stl_p(&resquery->uid, pbdev->uid);
         stw_p(&resquery->hdr.rsp, CLP_RC_OK);
         break;
     }
     case CLP_QUERY_PCI_FNGRP: {
         ClpRspQueryPciGrp *resgrp = (ClpRspQueryPciGrp *)resh;
-        resgrp->fr = 1;
-        stq_p(&resgrp->dasm, 0);
-        stq_p(&resgrp->msia, ZPCI_MSI_ADDR);
-        stw_p(&resgrp->mui, DEFAULT_MUI);
-        stw_p(&resgrp->i, 128);

hw/s390x/s390-pci-inst.c int clp_service_call(S390CPU *cpu, uint8_t r2, uintptr_t ra) · line 301

Quoted 10 lines
hw/s390x/s390-pci-inst.c +7 -2
-        stw_p(&resgrp->maxstbl, 128);
-        resgrp->version = 0;
 
+        ClpReqQueryPciGrp *reqgrp = (ClpReqQueryPciGrp *)reqh;
+        S390PCIGroup *grp;
+
+        grp = s390_grp_find(reqgrp->g);
+        if (!grp) {
+            /* We do not allow access to unknown groups */
+            /* The group must have been obtained with a vfio device */

What about non-vfio devices? How does this whole feature work for emulated devices?

Quoted 16 lines
hw/s390x/s390-pci-inst.c +5 -1
+            stw_p(&resgrp->hdr.rsp, CLP_RC_QUERYPCIFG_PFGID);
+            goto out;
+        }
+        memcpy(resgrp, &grp->zpci_grp, sizeof(ClpRspQueryPciGrp));
         stw_p(&resgrp->hdr.rsp, CLP_RC_OK);
         break;
     }
@@ -752,7 +756,7 @@ int pcistb_service_call(S390CPU *cpu, uint8_t r1, uint8_t r3, uint64_t gaddr,
     }
     /* Length must be greater than 8, a multiple of 8 */
     /* and not greater than maxstbl */
-    if ((len <= 8) || (len % 8) || (len > pbdev->maxstbl)) {
+    if ((len <= 8) || (len % 8) || (len > pbdev->pci_grp->zpci_grp.maxstbl)) {
         goto specification_error;
     }
     /* Do not cross a 4K-byte boundary */
#7 lore raw

Re: [Qemu-devel] [PATCH 3/5] s390: vfio_pci: Use a PCI Group structure

Quoted 10 lines · Cornelia Huck
Quoted 5 lines · On Fri, 10 May 2019 16:38:51 +0200 Pierre Morel <[email protected]> wrote:

We use a S390PCIGroup structure to hold the information related to zPCI Function group.

This allows us to be ready to support multiple groups and to retrieve the group information from the host.

What if there is no host to retrieve information from?

There is a default group for emulate devices.
I will enhance the comment.

Thanks

Quoted 33 lines
Quoted 33 lines
---
  hw/s390x/s390-pci-bus.c  | 42 ++++++++++++++++++++++++++++++++++++++++++
  hw/s390x/s390-pci-bus.h  | 11 ++++++++++-
  hw/s390x/s390-pci-inst.c | 22 +++++++++++++---------
  3 files changed, 65 insertions(+), 10 deletions(-)
hw/s390x/s390-pci-inst.c +1 -8
index be28962..8147847 100644
@@ -284,21 +284,25 @@ int clp_service_call(S390CPU *cpu, uint8_t r2, uintptr_t ra)
          stq_p(&resquery->edma, ZPCI_EDMA_ADDR);
          stl_p(&resquery->fid, pbdev->fid);
          stw_p(&resquery->pchid, 0);
-        stw_p(&resquery->ug, 1);
+        stw_p(&resquery->ug, ZPCI_DEFAULT_FN_GRP);
          stl_p(&resquery->uid, pbdev->uid);
          stw_p(&resquery->hdr.rsp, CLP_RC_OK);
          break;
      }
      case CLP_QUERY_PCI_FNGRP: {
          ClpRspQueryPciGrp *resgrp = (ClpRspQueryPciGrp *)resh;
-        resgrp->fr = 1;
-        stq_p(&resgrp->dasm, 0);
-        stq_p(&resgrp->msia, ZPCI_MSI_ADDR);
-        stw_p(&resgrp->mui, DEFAULT_MUI);
-        stw_p(&resgrp->i, 128);
-        stw_p(&resgrp->maxstbl, 128);
-        resgrp->version = 0;
 
Quoted 10 lines

hw/s390x/s390-pci-inst.c int clp_service_call(S390CPU *cpu, uint8_t r2, uintptr_t ra) · line 301

Quoted 7 lines
hw/s390x/s390-pci-inst.c +7 -0
+        ClpReqQueryPciGrp *reqgrp = (ClpReqQueryPciGrp *)reqh;
+        S390PCIGroup *grp;
+
+        grp = s390_grp_find(reqgrp->g);
+        if (!grp) {
+            /* We do not allow access to unknown groups */
+            /* The group must have been obtained with a vfio device */

What about non-vfio devices? How does this whole feature work for emulated devices?

Emulated devices get a default group with predefined values. The predefined values we used before this series.
I will modify the patch comment to explain the emulated devices case. Thanks for the comments.

Regards,
Pierre

Signature · 2 lines
Pierre Morel
Linux/KVM/QEMU in Böblingen - Germany
#8 lore raw

PATCH 4/5 s390: vfio_pci: Use a PCI Function structure

In reply to Pierre Morel

We use a ClpRspQueryPci structure to hold the information
related to zPCI Function.

This allows us to be ready to support different zPCI functions and to retrieve the zPCI function information from the host.

---
 hw/s390x/s390-pci-bus.c  | 22 +++++++++++++++++-----
 hw/s390x/s390-pci-bus.h  |  1 +
 hw/s390x/s390-pci-inst.c |  8 ++------
 3 files changed, 20 insertions(+), 11 deletions(-)
hw/s390x/s390-pci-bus.c +17 -5
index 175ea8c..6df80aa 100644
@@ -769,6 +769,17 @@ static void s390_pci_init_default_group(void)
     resgrp->version = 0;
 }
 
+static void set_pbdev_info(S390PCIBusDevice *pbdev)
+{
+    pbdev->zpci_fn.sdma = ZPCI_SDMA_ADDR;
+    pbdev->zpci_fn.edma = ZPCI_EDMA_ADDR;
+    pbdev->zpci_fn.pchid = 0;
+    pbdev->zpci_fn.ug = ZPCI_DEFAULT_FN_GRP;
+    pbdev->zpci_fn.fid = pbdev->fid;
+    pbdev->zpci_fn.uid = pbdev->uid;
+    pbdev->pci_grp = s390_grp_find(ZPCI_DEFAULT_FN_GRP);
+}
+
 static void s390_pcihost_realize(DeviceState *dev, Error **errp)
 {
     PCIBus *b;
@@ -987,17 +998,18 @@ static void s390_pcihost_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
             }
         }
 
+        pbdev->pdev = pdev;
+        pbdev->iommu = s390_pci_get_iommu(s, pci_get_bus(pdev), pdev->devfn);
+        pbdev->iommu->pbdev = pbdev;
+        pbdev->state = ZPCI_FS_DISABLED;
+        set_pbdev_info(pbdev);
+
         if (object_dynamic_cast(OBJECT(dev), "vfio-pci")) {
             pbdev->fh |= FH_SHM_VFIO;
         } else {
             pbdev->fh |= FH_SHM_EMUL;
         }
 
-        pbdev->pdev = pdev;
-        pbdev->iommu = s390_pci_get_iommu(s, pci_get_bus(pdev), pdev->devfn);
-        pbdev->iommu->pbdev = pbdev;
-        pbdev->state = ZPCI_FS_DISABLED;
-
         if (s390_pci_msix_init(pbdev)) {
             error_setg(errp, "MSI-X support is mandatory "
                        "in the S390 architecture");
hw/s390x/s390-pci-bus.h +1 -0
index e95a797..8c969d1 100644
@@ -337,6 +337,7 @@ struct S390PCIBusDevice {
     uint16_t noi;
     uint8_t sum;
     S390PCIGroup *pci_grp;
+    ClpRspQueryPci zpci_fn;
     S390MsixInfo msix;
     AdapterRoutes routes;
     S390PCIIOMMU *iommu;
hw/s390x/s390-pci-inst.c +2 -6
index 8147847..68ca240 100644
@@ -267,6 +267,8 @@ int clp_service_call(S390CPU *cpu, uint8_t r2, uintptr_t ra)
             goto out;
         }
 
+        memcpy(resquery, &pbdev->zpci_fn, sizeof(*resquery));
+
         for (i = 0; i < PCI_BAR_COUNT; i++) {
             uint32_t data = pci_get_long(pbdev->pdev->config +
                 PCI_BASE_ADDRESS_0 + (i * 4));
@@ -280,12 +282,6 @@ int clp_service_call(S390CPU *cpu, uint8_t r2, uintptr_t ra)
                     resquery->bar_size[i]);
         }
 
-        stq_p(&resquery->sdma, ZPCI_SDMA_ADDR);
-        stq_p(&resquery->edma, ZPCI_EDMA_ADDR);
-        stl_p(&resquery->fid, pbdev->fid);
-        stw_p(&resquery->pchid, 0);
-        stw_p(&resquery->ug, ZPCI_DEFAULT_FN_GRP);
-        stl_p(&resquery->uid, pbdev->uid);
         stw_p(&resquery->hdr.rsp, CLP_RC_OK);
         break;
     }
Signature · 1 line
2.7.4
#9 lore raw

PATCH 5/5 s390: vfio_pci: Get zPCI function info from host

In reply to Pierre Morel

The VFIO_IOMMU_INFO_CAPABILITIES flag allows to retrieve IOMMU specific informations from the iommu associated with the zPCI VFIO device.

When we retrieve the host device information, we take care to
- use the virtual UID and FID
- Disable all the IOMMU flags we do not support yet. Just keeping the refresh bit.

---
 hw/s390x/s390-pci-bus.c | 80 +++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 78 insertions(+), 2 deletions(-)
hw/s390x/s390-pci-bus.c +78 -2
index 6df80aa..3b7539c 100644
@@ -23,6 +23,9 @@
 #include "hw/pci/msi.h"
 #include "qemu/error-report.h"
 
+#include "hw/vfio/pci.h"
+#include <sys/ioctl.h>
+
 #ifndef DEBUG_S390PCI_BUS
 #define DEBUG_S390PCI_BUS  0
 #endif
@@ -780,6 +783,75 @@ static void set_pbdev_info(S390PCIBusDevice *pbdev)
     pbdev->pci_grp = s390_grp_find(ZPCI_DEFAULT_FN_GRP);
 }
 
+static int s390_fill_zpci(S390PCIBusDevice *pbdev,
+                          struct vfio_info_cap_header *cap)
+{
+    ClpRspQueryPci *rsp_fn;
+    ClpRspQueryPciGrp *rsp_grp;
+    S390PCIGroup *pci_grp;
+
+    /* We expect the function response first */
+    if (cap->id != VFIO_IOMMU_INFO_CAP_QFN) {
+        return -ENODEV;
+    }
+    rsp_fn = (struct ClpRspQueryPci *)(cap + 1);
+    memcpy(&pbdev->zpci_fn, rsp_fn, sizeof(*rsp_fn));
+    /* We use the virtualized FID and UID */
+    pbdev->zpci_fn.fid = pbdev->fid;
+    pbdev->zpci_fn.uid = pbdev->uid;
+
+    cap = (struct vfio_info_cap_header *)((char *)cap + cap->next);
+    if (cap->id != VFIO_IOMMU_INFO_CAP_QGRP) {
+        return -ENODEV;
+    }
+    pci_grp = s390_grp_find(rsp_fn->ug);
+    if (!pci_grp) {
+        pci_grp = s390_grp_create(rsp_fn->ug);
+    }
+
+    rsp_grp = (struct ClpRspQueryPciGrp *)(cap + 1);
+
+    memcpy(&pci_grp->zpci_grp, rsp_grp, sizeof(*rsp_grp));
+    /* We only support the refresh bit */
+    pci_grp->zpci_grp.fr = 1;
+
+    pbdev->pci_grp = pci_grp;
+
+    return 0;
+}
+
+static int get_pbdev_info(S390PCIBusDevice *pbdev)
+{
+    VFIOPCIDevice *vfio_pci;
+    int fd;
+    int ret;
+    struct vfio_iommu_type1_info *info;
+    int size;
+
+    vfio_pci = container_of(pbdev->pdev, VFIOPCIDevice, pdev);
+    fd = vfio_pci->vbasedev.group->container->fd;
+    info = g_malloc0(sizeof(*info));
+    info->flags = VFIO_IOMMU_INFO_CAPABILITIES;
+    info->argsz = sizeof(*info);
+    ret = ioctl(fd, VFIO_IOMMU_GET_INFO, info);
+    if (ret) {
+        return ret;
+    }
+    size = info->argsz;
+    info = g_realloc(info, size);
+    info->flags = VFIO_IOMMU_INFO_CAPABILITIES;
+    info->argsz = size;
+    ret = ioctl(fd, VFIO_IOMMU_GET_INFO, info);
+    if (ret) {
+        return ret;
+    }
+    /* Fill zPCI parameters maxstbl, start dma, end dma...*/
+    /* using the caps */
+    ret = s390_fill_zpci(pbdev, (struct vfio_info_cap_header *)(info + 1));
+    g_free(info);
+    return ret;
+}
+
 static void s390_pcihost_realize(DeviceState *dev, Error **errp)
 {
     PCIBus *b;
@@ -852,7 +924,8 @@ static int s390_pci_msix_init(S390PCIBusDevice *pbdev)
     name = g_strdup_printf("msix-s390-%04x", pbdev->uid);
     memory_region_init_io(&pbdev->msix_notify_mr, OBJECT(pbdev),
                           &s390_msi_ctrl_ops, pbdev, name, PAGE_SIZE);
-    memory_region_add_subregion(&pbdev->iommu->mr, ZPCI_MSI_ADDR,
+    memory_region_add_subregion(&pbdev->iommu->mr,
+                                pbdev->pci_grp->zpci_grp.msia,
                                 &pbdev->msix_notify_mr);
     g_free(name);
 
@@ -1002,12 +1075,15 @@ static void s390_pcihost_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
         pbdev->iommu = s390_pci_get_iommu(s, pci_get_bus(pdev), pdev->devfn);
         pbdev->iommu->pbdev = pbdev;
         pbdev->state = ZPCI_FS_DISABLED;
-        set_pbdev_info(pbdev);
 
         if (object_dynamic_cast(OBJECT(dev), "vfio-pci")) {
             pbdev->fh |= FH_SHM_VFIO;
+            if (get_pbdev_info(pbdev) != 0) {
+                set_pbdev_info(pbdev);
+            }
         } else {
             pbdev->fh |= FH_SHM_EMUL;
+            set_pbdev_info(pbdev);
         }
 
         if (s390_pci_msix_init(pbdev)) {
Signature · 1 line
2.7.4