Retrieving zPCI specific info from QEMU
v2 cover Retrieving zPCI specific info from QEMU
This patch implement the QEMU part to retrieve ZPCI specific information from the host.
The Linux part has been posted on a separate patch on the LKML.
Subject: [PATCH 0/4] Retrieving zPCI specific info with VFIO
Message-Id: <[email protected]>
We use the PCI VFIO interface to retrieve ZPCI specific information from the host with a ZPCI specific device region.
The retrieval is done only once per function and function group during the plugging of the device.
The guest's requests are filled with shadow values we keep for the duration of the device remains plugged.
Still there are some values we need to virtualize, like the UID and FID of the zPCI function, and we currently only allow the refresh bit for the zPCI group flags.
Note that we export the CLP specific definitions in a dedicated file for clarity.
Pierre Morel (5):
vfio: vfio_iommu_type1: linux header place holder
s390: PCI: Creation a header dedicated to PCI CLP
s390: vfio_pci: Use a PCI Group structure
s390: vfio_pci: Use a PCI Function structure
s390: vfio_pci: Get zPCI function info from host
hw/s390x/s390-pci-bus.c | 144 +++++++++++++++++++++++++-- hw/s390x/s390-pci-bus.h | 15 ++- hw/s390x/s390-pci-clp.h | 211 ++++++++++++++++++++++++++++++++++++++++ hw/s390x/s390-pci-inst.c | 28 +++--- hw/s390x/s390-pci-inst.h | 196 ------------------------------------- linux-headers/linux/vfio.h | 14 ++- linux-headers/linux/vfio_zdev.h | 34 +++++++ 7 files changed, 422 insertions(+), 220 deletions(-) create mode 100644 hw/s390x/s390-pci-clp.h create mode 100644 linux-headers/linux/vfio_zdev.h
Signature · 1 line
2.7.4
PATCH v2 1/5 vfio: vfio_iommu_type1: linux header place holder
This should be copied from Linux kernel UAPI includes. The version used here is Linux 5.1.0
We define a new device region in vfio.h to be able to get the ZPCI CLP information by reading this region from userland.
We create a new file, vfio_zdev.h to define the structure of the new region we defined in vfio.h
--- linux-headers/linux/vfio.h | 14 +++++++++++--- linux-headers/linux/vfio_zdev.h | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) create mode 100644 linux-headers/linux/vfio_zdev.h
linux-headers/linux/vfio.h +11 -3
@@ -9,8 +9,8 @@* it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. */ -#ifndef VFIO_H -#define VFIO_H +#ifndef _UAPIVFIO_H +#define _UAPIVFIO_H#include <linux/types.h> #include <linux/ioctl.h>
@@ -306,6 +306,10 @@ struct vfio_region_info_cap_type {#define VFIO_REGION_TYPE_GFX (1)#define VFIO_REGION_SUBTYPE_GFX_EDID (1) +/* IBM Subtypes */ +#define VFIO_REGION_TYPE_IBM_ZDEV (1) +#define VFIO_REGION_SUBTYPE_ZDEV_CLP (1) +/** * struct vfio_region_gfx_edid - EDID region layout. *
@@ -353,6 +357,10 @@ struct vfio_region_gfx_edid {#define VFIO_DEVICE_GFX_LINK_STATE_DOWN 2 }; +#define VFIO_REGION_TYPE_CCW (2) +/* ccw sub-types */ +#define VFIO_REGION_SUBTYPE_CCW_ASYNC_CMD (1) +/* * 10de vendor sub-type *
@@ -910,4 +918,4 @@ struct vfio_iommu_spapr_tce_remove {/* ***************************************************************** */ -#endif /* VFIO_H */ +#endif /* _UAPIVFIO_H */
linux-headers/linux/vfio_zdev.h new file +34 -0
@@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* +* Region definition for ZPCI devices +* +* Copyright IBM Corp. 2019 +* +* Author(s): Pierre Morel <[email protected]> +*/ + +#ifndef _VFIO_ZDEV_H_ +#define _VFIO_ZDEV_H_ + +#include <linux/types.h> + +/** +* struct vfio_region_zpci_info - ZPCI information. +* +*/ +struct vfio_region_zpci_info { +__u64 dasm; +__u64 start_dma; +__u64 end_dma; +__u64 msi_addr; +__u64 flags; +__u16 pchid; +__u16 mui; +__u16 noi; +__u8 gid; +__u8 version; +#define VFIO_PCI_ZDEV_FLAGS_REFRESH 1 +__u8 util_str[CLP_UTIL_STR_LEN ]; +} __packed; + +#endif
Signature · 1 line
2.7.4
PATCH v2 2/5 s390: PCI: Creation a header dedicated to PCI CLP
In reply to Pierre MorelTo have a clean separation between s390-pci-bus.h
and s390-pci-inst.h headers we export the PCI CLP
instructions in a dedicated header.
--- hw/s390x/s390-pci-bus.h | 1 + hw/s390x/s390-pci-clp.h | 211 +++++++++++++++++++++++++++++++++++++++++++++++ hw/s390x/s390-pci-inst.h | 196 ------------------------------------------- 3 files changed, 212 insertions(+), 196 deletions(-) create mode 100644 hw/s390x/s390-pci-clp.h
hw/s390x/s390-pci-bus.h +1 -0
@@ -19,6 +19,7 @@#include "hw/s390x/sclp.h" #include "hw/s390x/s390_flic.h" #include "hw/s390x/css.h" +#include "s390-pci-clp.h" #define TYPE_S390_PCI_HOST_BRIDGE "s390-pcihost"#define TYPE_S390_PCI_BUS "s390-pcibus"
hw/s390x/s390-pci-clp.h new file +211 -0
211 lines not on this page · raw message
hw/s390x/s390-pci-inst.h +0 -196
202 lines not on this page · raw message
Signature · 1 line
2.7.4
PATCH v2 3/5 s390: vfio_pci: Use a PCI Group structure
In reply to Pierre MorelWe use a S390PCIGroup structure to hold the information related to zPCI Function group.
This allows us to be ready to support multiple groups and to retrieve the group information from the host.
--- hw/s390x/s390-pci-bus.c | 42 ++++++++++++++++++++++++++++++++++++++++++ hw/s390x/s390-pci-bus.h | 11 ++++++++++- hw/s390x/s390-pci-inst.c | 22 +++++++++++++--------- 3 files changed, 65 insertions(+), 10 deletions(-)
hw/s390x/s390-pci-bus.c +42 -0
@@ -729,6 +729,46 @@ static void s390_pci_iommu_free(S390pciState *s, PCIBus *bus, int32_t devfn)object_unref (OBJECT (iommu)); } +static S390PCIGroup *s390_grp_create (int ug) +{ +S390PCIGroup *grp; +S390pciState *s =s390_get_phb (); + + grp =g_new0 (S390PCIGroup,1 ); + grp->ug = ug; +QTAILQ_INSERT_TAIL (&s->zpci_grps, grp, link); +return grp; +} + +S390PCIGroup *s390_grp_find (int ug) +{ +S390PCIGroup *grp; +S390pciState *s =s390_get_phb (); + + QTAILQ_FOREACH(grp, &s->zpci_grps, link) { +if ((grp->ug &CLP_REQ_QPCIG_MASK_PFGID ) == ug) { +return grp; + } + } +return NULL ; +} + +static void s390_pci_init_default_group (void ) +{ +S390PCIGroup *grp; +ClpRspQueryPciGrp *resgrp; + + grp =s390_grp_create (ZPCI_DEFAULT_FN_GRP ); + resgrp = &grp->zpci_grp; + resgrp->fr =1 ; +stq_p (&resgrp->dasm,0 ); +stq_p (&resgrp->msia,ZPCI_MSI_ADDR ); +stw_p (&resgrp->mui,DEFAULT_MUI ); +stw_p (&resgrp->i,128 ); +stw_p (&resgrp->maxstbl,128 ); + resgrp->version =0 ; +} +static void s390_pcihost_realize (DeviceState *dev,Error **errp) {PCIBus *b;
@@ -765,7 +805,9 @@ static void s390_pcihost_realize(DeviceState *dev, Error **errp) s->bus_no =0 ;QTAILQ_INIT (&s->pending_sei);QTAILQ_INIT (&s->zpci_devs); +QTAILQ_INIT (&s->zpci_grps); +s390_pci_init_default_group ();css_register_io_adapters (CSS_IO_ADAPTER_PCI , true, false,S390_ADAPTER_SUPPRESSIBLE , &local_err);error_propagate (errp, local_err);
hw/s390x/s390-pci-bus.h +10 -1
@@ -312,6 +312,14 @@ typedef struct ZpciFmb { } ZpciFmb;QEMU_BUILD_BUG_MSG (offsetof(ZpciFmb , fmt0) !=48 ,"padding in ZpciFmb" ); +#define ZPCI_DEFAULT_FN_GRP 0x20 +typedef struct S390PCIGroup { +ClpRspQueryPciGrp zpci_grp; +int ug; +QTAILQ_ENTRY (S390PCIGroup) link; +}S390PCIGroup ; +S390PCIGroup *s390_grp_find (int ug); +struct S390PCIBusDevice {DeviceState qdev;PCIDevice *pdev;
@@ -327,8 +335,8 @@ struct S390PCIBusDevice {QEMUTimer *fmb_timer;uint8_t isc;uint16_t noi; -uint16_t maxstbl;uint8_t sum; +S390PCIGroup *pci_grp;S390MsixInfo msix;AdapterRoutes routes;S390PCIIOMMU *iommu;
@@ -353,6 +361,7 @@ typedef struct S390pciState {GHashTable *zpci_table; QTAILQ_HEAD(, SeiContainer) pending_sei; QTAILQ_HEAD(, S390PCIBusDevice) zpci_devs; + QTAILQ_HEAD(, S390PCIGroup) zpci_grps; } S390pciState;S390pciState *s390_get_phb (void );
hw/s390x/s390-pci-inst.c +13 -9
@@ -284,21 +284,25 @@ int clp_service_call(S390CPU *cpu, uint8_t r2, uintptr_t ra)stq_p (&resquery->edma,ZPCI_EDMA_ADDR );stl_p (&resquery->fid, pbdev->fid);stw_p (&resquery->pchid,0 ); -stw_p (&resquery->ug,1 ); +stw_p (&resquery->ug,ZPCI_DEFAULT_FN_GRP );stl_p (&resquery->uid, pbdev->uid);stw_p (&resquery->hdr.rsp,CLP_RC_OK );break ; }case CLP_QUERY_PCI_FNGRP : {ClpRspQueryPciGrp *resgrp = (ClpRspQueryPciGrp *)resh; - resgrp->fr =1 ; -stq_p (&resgrp->dasm,0 ); -stq_p (&resgrp->msia,ZPCI_MSI_ADDR ); -stw_p (&resgrp->mui,DEFAULT_MUI ); -stw_p (&resgrp->i,128 ); -stw_p (&resgrp->maxstbl,128 ); - resgrp->version =0 ; +ClpReqQueryPciGrp *reqgrp = (ClpReqQueryPciGrp *)reqh; +S390PCIGroup *grp; + + grp =s390_grp_find (reqgrp->g); +if (!grp) { +/* We do not allow access to unknown groups */ +/* The group must have been obtained with a vfio device */ +stw_p (&resgrp->hdr.rsp,CLP_RC_QUERYPCIFG_PFGID ); + goto out; + } +memcpy (resgrp, &grp->zpci_grp,sizeof (ClpRspQueryPciGrp));stw_p (&resgrp->hdr.rsp,CLP_RC_OK );break ; }
@@ -752,7 +756,7 @@ int pcistb_service_call(S390CPU *cpu, uint8_t r1, uint8_t r3, uint64_t gaddr, }/* Length must be greater than 8, a multiple of 8 */ /* and not greater than maxstbl */ -if ((len <=8 ) || (len %8 ) || (len > pbdev->maxstbl)) { +if ((len <=8 ) || (len %8 ) || (len > pbdev->pci_grp->zpci_grp.maxstbl)) { goto specification_error; }/* Do not cross a 4K-byte boundary */
Signature · 1 line
2.7.4
PATCH v2 4/5 s390: vfio_pci: Use a PCI Function structure
In reply to Pierre MorelWe use a ClpRspQueryPci structure to hold the information related to zPCI Function.
This allows us to be ready to support different zPCI functions and to retrieve the zPCI function information from the host.
--- hw/s390x/s390-pci-bus.c | 22 +++++++++++++++++----- hw/s390x/s390-pci-bus.h | 1 + hw/s390x/s390-pci-inst.c | 8 ++------ 3 files changed, 20 insertions(+), 11 deletions(-)
hw/s390x/s390-pci-bus.c +17 -5
@@ -769,6 +769,17 @@ static void s390_pci_init_default_group(void) resgrp->version =0 ; } +static void set_pbdev_info (S390PCIBusDevice *pbdev) +{ + pbdev->zpci_fn.sdma =ZPCI_SDMA_ADDR ; + pbdev->zpci_fn.edma =ZPCI_EDMA_ADDR ; + pbdev->zpci_fn.pchid =0 ; + pbdev->zpci_fn.ug =ZPCI_DEFAULT_FN_GRP ; + pbdev->zpci_fn.fid = pbdev->fid; + pbdev->zpci_fn.uid = pbdev->uid; + pbdev->pci_grp =s390_grp_find (ZPCI_DEFAULT_FN_GRP ); +} +static void s390_pcihost_realize (DeviceState *dev,Error **errp) {PCIBus *b;
@@ -987,17 +998,18 @@ static void s390_pcihost_plug(HotplugHandler *hotplug_dev, DeviceState *dev, } } + pbdev->pdev = pdev; + pbdev->iommu =s390_pci_get_iommu (s,pci_get_bus (pdev), pdev->devfn); + pbdev->iommu->pbdev = pbdev; + pbdev->state =ZPCI_FS_DISABLED ; +set_pbdev_info (pbdev); +if (object_dynamic_cast (OBJECT (dev),"vfio-pci" )) { pbdev->fh |=FH_SHM_VFIO ; }else { pbdev->fh |=FH_SHM_EMUL ; } - pbdev->pdev = pdev; - pbdev->iommu =s390_pci_get_iommu (s,pci_get_bus (pdev), pdev->devfn); - pbdev->iommu->pbdev = pbdev; - pbdev->state =ZPCI_FS_DISABLED ; -if (s390_pci_msix_init (pbdev)) {error_setg (errp,"MSI-X support is mandatory " "in the S390 architecture" );
hw/s390x/s390-pci-bus.h +1 -0
@@ -337,6 +337,7 @@ struct S390PCIBusDevice {uint16_t noi;uint8_t sum;S390PCIGroup *pci_grp; +ClpRspQueryPci zpci_fn;S390MsixInfo msix;AdapterRoutes routes;S390PCIIOMMU *iommu;
hw/s390x/s390-pci-inst.c +2 -6
@@ -267,6 +267,8 @@ int clp_service_call(S390CPU *cpu, uint8_t r2, uintptr_t ra) goto out; } +memcpy (resquery, &pbdev->zpci_fn,sizeof (*resquery)); +for (i =0 ; i <PCI_BAR_COUNT ; i++) {uint32_t data =pci_get_long (pbdev->pdev->config +PCI_BASE_ADDRESS_0 + (i *4 ));
@@ -280,12 +282,6 @@ int clp_service_call(S390CPU *cpu, uint8_t r2, uintptr_t ra) resquery->bar_size[i]); } -stq_p (&resquery->sdma,ZPCI_SDMA_ADDR ); -stq_p (&resquery->edma,ZPCI_EDMA_ADDR ); -stl_p (&resquery->fid, pbdev->fid); -stw_p (&resquery->pchid,0 ); -stw_p (&resquery->ug,ZPCI_DEFAULT_FN_GRP ); -stl_p (&resquery->uid, pbdev->uid);stw_p (&resquery->hdr.rsp,CLP_RC_OK );break ; }
Signature · 1 line
2.7.4
PATCH v2 5/5 s390: vfio_pci: Get zPCI function info from host
In reply to Pierre MorelWe use the VFIO_REGION_SUBTYPE_ZDEV_CLP subregion of PCI_VENDOR_ID_IBM to retrieve the CLP information the kernel exports.
To be compatible with previous kernel versions we fall back on previous predefined values, same as the emulation values, when the region is not found or when any problem happens during the search for the information.
Once we retrieved the host device information, we take care to
- use the virtual UID and FID
- Disable all the IOMMU flags we do not support yet. Just keeping the refresh bit.
--- hw/s390x/s390-pci-bus.c | 82 +++++++++++++++++++++++++++++++++++++++++++++++-- hw/s390x/s390-pci-bus.h | 2 ++ 2 files changed, 82 insertions(+), 2 deletions(-)
hw/s390x/s390-pci-bus.c +80 -2
@@ -16,6 +16,8 @@#include "qapi/visitor.h" #include "qemu-common.h" #include "cpu.h" +#include "s390-pci-clp.h" +#include <linux/vfio_zdev.h> #include "s390-pci-bus.h" #include "s390-pci-inst.h" #include "hw/pci/pci_bus.h"
@@ -23,6 +25,9 @@#include "hw/pci/msi.h" #include "qemu/error-report.h" +#include "hw/vfio/pci.h" +#include <sys/ioctl.h> +#ifndef DEBUG_S390PCI_BUS #define DEBUG_S390PCI_BUS 0#endif
@@ -780,6 +785,75 @@ static void set_pbdev_info(S390PCIBusDevice *pbdev) pbdev->pci_grp =s390_grp_find (ZPCI_DEFAULT_FN_GRP ); } +static int get_pbdev_info (S390PCIBusDevice *pbdev) +{ +VFIOPCIDevice *vfio_pci; +VFIODevice *vdev; +struct vfio_region_info *info; +CLPRegion *clp_region; +int size; +int ret; + + vfio_pci =container_of (pbdev->pdev, VFIOPCIDevice, pdev); + vdev = &vfio_pci->vbasedev; + +if (vdev->num_regions <VFIO_PCI_NUM_REGIONS +1 ) { +/* Fall back to old handling */ +return -ENODEV ; + } + + ret =vfio_get_dev_region_info (vdev, +PCI_VENDOR_ID_IBM | +VFIO_REGION_TYPE_PCI_VENDOR_TYPE , +VFIO_REGION_SUBTYPE_ZDEV_CLP , &info); +if (ret) { +/* Fall back to old handling */ +return -EIO ; + } + +if (info->size !=sizeof (CLPRegion)) { +/* Fall back to old handling */ +g_free (info); +return -ENOMEM ; + } + clp_region =g_malloc0 (sizeof (*clp_region)); + size =pread (vdev->fd, clp_region,sizeof (*clp_region), info->offset); +if (size !=sizeof (*clp_region)) { + goto end; + } + + pbdev->zpci_fn.fid = pbdev->fid; + pbdev->zpci_fn.uid = pbdev->uid; + pbdev->zpci_fn.sdma = clp_region->start_dma; + pbdev->zpci_fn.edma = clp_region->end_dma; + pbdev->zpci_fn.pchid = clp_region->pchid; + pbdev->zpci_fn.ug = clp_region->gid; + pbdev->pci_grp =s390_grp_find (clp_region->gid); + +if (!pbdev->pci_grp) { +ClpRspQueryPciGrp *resgrp; + + pbdev->pci_grp =s390_grp_create (clp_region->gid); + + resgrp = &pbdev->pci_grp->zpci_grp; +if (clp_region->flags &VFIO_PCI_ZDEV_FLAGS_REFRESH ) { + resgrp->fr =1 ; + } +stq_p (&resgrp->dasm, clp_region->dasm); +stq_p (&resgrp->msia, clp_region->msi_addr); +stw_p (&resgrp->mui, clp_region->mui); +stw_p (&resgrp->i, clp_region->noi); +/* These two must be queried in a next iteration */ +stw_p (&resgrp->maxstbl,128 ); + resgrp->version =0 ; + } + +end: +g_free (info); +g_free (clp_region); +return ret; +} +static void s390_pcihost_realize (DeviceState *dev,Error **errp) {PCIBus *b;
@@ -852,7 +926,8 @@ static int s390_pci_msix_init(S390PCIBusDevice *pbdev) name =g_strdup_printf ("msix-s390-%04x" , pbdev->uid);memory_region_init_io (&pbdev->msix_notify_mr,OBJECT (pbdev), &s390_msi_ctrl_ops, pbdev, name,PAGE_SIZE ); -memory_region_add_subregion (&pbdev->iommu->mr,ZPCI_MSI_ADDR , +memory_region_add_subregion (&pbdev->iommu->mr, + pbdev->pci_grp->zpci_grp.msia, &pbdev->msix_notify_mr);g_free (name);
@@ -1002,12 +1077,15 @@ static void s390_pcihost_plug(HotplugHandler *hotplug_dev, DeviceState *dev, pbdev->iommu =s390_pci_get_iommu (s,pci_get_bus (pdev), pdev->devfn); pbdev->iommu->pbdev = pbdev; pbdev->state =ZPCI_FS_DISABLED ; -set_pbdev_info (pbdev);if (object_dynamic_cast (OBJECT (dev),"vfio-pci" )) { pbdev->fh |=FH_SHM_VFIO ; +if (get_pbdev_info (pbdev) !=0 ) { +set_pbdev_info (pbdev); + } }else { pbdev->fh |=FH_SHM_EMUL ; +set_pbdev_info (pbdev); }if (s390_pci_msix_init (pbdev)) {
hw/s390x/s390-pci-bus.h +2 -0
@@ -320,6 +320,8 @@ typedef struct S390PCIGroup { } S390PCIGroup;S390PCIGroup *s390_grp_find (int ug); +typedef struct vfio_region_zpci_info CLPRegion ; +struct S390PCIBusDevice {DeviceState qdev;PCIDevice *pdev;
Signature · 1 line
2.7.4