get real IOMMU information from VFIO IOMMU
cover get real IOMMU information from VFIO IOMMU
In the Z-Architecture, the IOMMU information may change on a device basis.
until this patch series we used hardcoded values to define the IOMMU aperture we provide to the guest during PCI enumeration.
Let's ask the host about the IOMMU specific configuration using VFIO capabilities.
NOTES:
1- You will need the according Linux patch to test this.
2- I am really not happy to add S390 dedicated code in the
VFIO common code, as do SPAPR, but I did not find a better solution.
Any idea?
Pierre Morel (3):
vfio: Linux header placeholder
vfio/pci: Get real IOMMU information from container s390x/pci: Reporting the host aperture to the guest
hw/s390x/s390-pci-bus.c | 2 +- hw/s390x/s390-pci-bus.h | 3 ++ hw/s390x/s390-pci-inst.c | 20 ++++++++++-- hw/vfio/common.c | 75 +++++++++++++++++++++++++++++++++---------- include/hw/vfio/vfio-common.h | 3 ++ linux-headers/linux/vfio.h | 65 +++++++++++++++++++++++++++++++++++-- 6 files changed, 144 insertions(+), 24 deletions(-)
Signature · 1 line
2.7.4
PATCH 1/3 vfio: Linux header placeholder
This is a place holder for VFIO.h as changed by the Linux patch associated with this QEMU series.
--- linux-headers/linux/vfio.h | 65 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 62 insertions(+), 3 deletions(-)
linux-headers/linux/vfio.h +62 -3
@@ -9,8 +9,8 @@* it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. */ -#ifndef VFIO_H -#define VFIO_H +#ifndef _UAPIVFIO_H +#define _UAPIVFIO_H#include <linux/types.h> #include <linux/ioctl.h>
@@ -303,6 +303,56 @@ struct vfio_region_info_cap_type {#define VFIO_REGION_SUBTYPE_INTEL_IGD_HOST_CFG (2)#define VFIO_REGION_SUBTYPE_INTEL_IGD_LPC_CFG (3) +#define VFIO_REGION_TYPE_GFX (1) +#define VFIO_REGION_SUBTYPE_GFX_EDID (1) + +/** +* struct vfio_region_gfx_edid - EDID region layout. +* +* Set display link state and EDID blob. +* +* The EDID blob has monitor information such as brand, name, serial +* number, physical size, supported video modes and more. +* +* This special region allows userspace (typically qemu) set a virtual +* EDID for the virtual monitor, which allows a flexible display +* configuration. +* +* For the edid blob spec look here: +* https://en.wikipedia.org/wiki/Extended_Display_Identification_Data +* +* On linux systems you can find the EDID blob in sysfs: +* /sys/class/drm/${card}/${connector}/edid +* +* You can use the edid-decode ulility (comes with xorg-x11-utils) to +* decode the EDID blob. +* +* @edid_offset: location of the edid blob, relative to the +* start of the region (readonly). +* @edid_max_size: max size of the edid blob (readonly). +* @edid_size: actual edid size (read/write). +* @link_state: display link state (read/write). +* VFIO_DEVICE_GFX_LINK_STATE_UP: Monitor is turned on. +* VFIO_DEVICE_GFX_LINK_STATE_DOWN: Monitor is turned off. +* @max_xres: max display width (0 == no limitation, readonly). +* @max_yres: max display height (0 == no limitation, readonly). +* +* EDID update protocol: +* (1) set link-state to down. +* (2) update edid blob and size. +* (3) set link-state to up. +*/ +struct vfio_region_gfx_edid { +__u32 edid_offset; +__u32 edid_max_size; +__u32 edid_size; +__u32 max_xres; +__u32 max_yres; +__u32 link_state; +#define VFIO_DEVICE_GFX_LINK_STATE_UP 1 +#define VFIO_DEVICE_GFX_LINK_STATE_DOWN 2 +}; +/* * The MSIX mappable capability informs that MSIX data of a BAR can be mmapped * which allows direct access to non-MSIX registers which happened to be within
@@ -619,6 +669,15 @@ struct vfio_iommu_type1_info {__u32 flags;#define VFIO_IOMMU_INFO_PGSIZES (1 << 0)/* supported page sizes info */ __u64 iova_pgsizes;/* Bitmap of supported page sizes */ +#define VFIO_IOMMU_INFO_CAPABILITIES (1 << 1)/* support capabilities info */ +__u64 cap_offset;/* Offset within info struct of first cap */ +}; + +#define VFIO_IOMMU_INFO_CAP_DMA 1 +struct vfio_iommu_cap_dma { +struct vfio_info_cap_header header; +__u64 dma_start; +__u64 dma_end; };#define VFIO_IOMMU_GET_INFO _IO(VFIO_TYPE, VFIO_BASE + 12)
@@ -818,4 +877,4 @@ struct vfio_iommu_spapr_tce_remove {/* ***************************************************************** */ -#endif /* VFIO_H */ +#endif /* _UAPIVFIO_H */
Signature · 1 line
2.7.4
PATCH 2/3 vfio/pci: Get real IOMMU information from container
In reply to Pierre MorelWhen the vfio_iommu_type1 supports the VFIO_IOMMU_INFO_CAPABILITIES and the capability ID VFIO_IOMMU_INFO_CAP_DMA we can use an ioctl to retrieve this information from the real IOMMU device.
Let's use this information to add the host window associated with the container.
--- hw/vfio/common.c | 73 +++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 56 insertions(+), 17 deletions(-)
hw/vfio/common.c +56 -17
@@ -1036,6 +1036,61 @@ static void vfio_put_address_space(VFIOAddressSpace *space) } } +static void vfio_iommu_type1_get_info (int fd,VFIOContainer *container) +{ +struct vfio_iommu_type1_info *info; +size_t argsz =sizeof (*info); +struct vfio_info_cap_header *hdr; +struct vfio_iommu_cap_dma *dma_info; +int ret =0 ; + + info =g_malloc0 (argsz); + +retry: + info->argsz = argsz; + info->flags =VFIO_IOMMU_INFO_CAPABILITIES ; + + ret =ioctl (fd,VFIO_IOMMU_GET_INFO , info); +/* Ignore errors */ +if (ret) { + goto out; + } +if (!(info->flags &VFIO_IOMMU_INFO_PGSIZES )) { + info->iova_pgsizes =4096 ; + } + +if (info->argsz > argsz) { + argsz = info->argsz; + info =g_realloc (info, argsz); + + goto retry; + } +if (info->argsz != argsz) { + goto out; + } +/* Now we have the capabilities */ + hdr = (struct vfio_info_cap_header *)((unsigned char *)info + +sizeof (struct vfio_iommu_type1_info ) + + info->cap_offset); +do { + dma_info = (struct vfio_iommu_cap_dma *) (hdr); +if (hdr->id ==VFIO_IOMMU_INFO_CAP_DMA ) { +vfio_host_win_add (container,0 , + dma_info->dma_end - dma_info->dma_start, + info->iova_pgsizes); + container->pgsizes = info->iova_pgsizes; +return ; + } + hdr = (struct vfio_info_cap_header *)((unsigned char *) dma_info + + hdr->next); + }while (hdr->next); +out: +/* Assume 4k IOVA page size */ +vfio_host_win_add (container,0 , (hwaddr)-1 ,4096 ); + container->pgsizes =4096 ; +return ; +} +static int vfio_connect_container (VFIOGroup *group,AddressSpace *as,Error **errp) {
@@ -1104,7 +1159,6 @@ static int vfio_connect_container(VFIOGroup *group, AddressSpace *as,if (ioctl (fd,VFIO_CHECK_EXTENSION ,VFIO_TYPE1_IOMMU ) ||ioctl (fd,VFIO_CHECK_EXTENSION , VFIO_TYPE1v2_IOMMU)) {bool v2 = !!ioctl (fd,VFIO_CHECK_EXTENSION , VFIO_TYPE1v2_IOMMU); -struct vfio_iommu_type1_info info; ret =ioctl (group->fd,VFIO_GROUP_SET_CONTAINER , &fd);if (ret) {
@@ -1121,22 +1175,7 @@ static int vfio_connect_container(VFIOGroup *group, AddressSpace *as, goto free_container_exit; } -/* -* FIXME: This assumes that a Type1 IOMMU can map any 64-bit -* IOVA whatsoever. That's not actually true, but the current -* kernel interface doesn't tell us what it can map, and the -* existing Type1 IOMMUs generally support any IOVA we're -* going to actually try in practice. -*/ - info.argsz =sizeof (info); - ret =ioctl (fd,VFIO_IOMMU_GET_INFO , &info); -/* Ignore errors */ -if (ret || !(info.flags &VFIO_IOMMU_INFO_PGSIZES )) { -/* Assume 4k IOVA page size */ - info.iova_pgsizes =4096 ; - } -vfio_host_win_add (container,0 , (hwaddr)-1 , info.iova_pgsizes); - container->pgsizes = info.iova_pgsizes; +vfio_iommu_type1_get_info (fd, container); }else if (ioctl (fd,VFIO_CHECK_EXTENSION ,VFIO_SPAPR_TCE_IOMMU ) ||ioctl (fd,VFIO_CHECK_EXTENSION , VFIO_SPAPR_TCE_v2_IOMMU)) {struct vfio_iommu_spapr_tce_info info;
Signature · 1 line
2.7.4
PATCH 3/3 s390x/pci: Reporting the host aperture to the guest
In reply to Pierre MorelThe S390 PCI driver in the guest needs to know the IOMMU aperture associated with the zPCI real device in the host through Start DMA and End DMA value in the response to the Get Function Group information query.
Let's report these SDMA and EDMA using the value we got from the host through the real IOMMU associated with the VFIO container.
--- hw/s390x/s390-pci-bus.c | 2 +- hw/s390x/s390-pci-bus.h | 3 +++ hw/s390x/s390-pci-inst.c | 20 +++++++++++++++++--- hw/vfio/common.c | 4 +++- include/hw/vfio/vfio-common.h | 3 +++ 5 files changed, 27 insertions(+), 5 deletions(-)
hw/s390x/s390-pci-bus.c +1 -1
@@ -551,7 +551,7 @@ static void s390_pci_iommu_replay(IOMMUMemoryRegion *iommu,return ; } -static S390PCIIOMMU *s390_pci_get_iommu (S390pciState *s,PCIBus *bus, +S390PCIIOMMU *s390_pci_get_iommu (S390pciState *s,PCIBus *bus,int devfn) {uint64_t key = (uintptr_t )bus;
hw/s390x/s390-pci-bus.h +3 -0
@@ -276,6 +276,8 @@ typedef struct S390PCIIOMMU {IOMMUMemoryRegion iommu_mr;bool enabled;uint64_t g_iota; +uint64_t sdma; +uint64_t edma;uint64_t pba;uint64_t pal;GHashTable *iotlb;
@@ -344,5 +346,6 @@ S390PCIBusDevice *s390_pci_find_dev_by_target(S390pciState *s, const char *target);S390PCIBusDevice *s390_pci_find_next_avail_dev (S390pciState *s,S390PCIBusDevice *pbdev); +S390PCIIOMMU *s390_pci_get_iommu (S390pciState *s,PCIBus *bus,int devfn);#endif
hw/s390x/s390-pci-inst.c +17 -3
@@ -19,6 +19,7 @@#include "exec/memory-internal.h" #include "qemu/error-report.h" #include "sysemu/hw_accel.h" +#include "hw/vfio/vfio-common.h" #ifndef DEBUG_S390PCI_INST #define DEBUG_S390PCI_INST 0
@@ -31,6 +32,16 @@ } \ }while (0 ) +void vfio_s390_iommu_setup(VFIOContainer *container, uint64_t min, + uint64_t max, uint64_t pgsize) +{ +S390PCIIOMMU *iommu; + + iommu =container_of (container->space->as,S390PCIIOMMU , as); + iommu->sdma = min; + iommu->edma = max; +} +static void s390_set_status_code (CPUS390XState *env,uint8_t r,uint64_t status_code) {
@@ -153,6 +164,7 @@ int clp_service_call(S390CPU *cpu, uint8_t r2, uintptr_t ra)uint8_t cc =0 ;CPUS390XState *env = &cpu->env;S390pciState *s =s390_get_phb (); +S390PCIIOMMU *iommu;int i;if (env->psw.mask &PSW_MASK_PSTATE ) {
@@ -279,8 +291,10 @@ int clp_service_call(S390CPU *cpu, uint8_t r2, uintptr_t ra) resquery->bar_size[i]); } -stq_p (&resquery->sdma,ZPCI_SDMA_ADDR ); -stq_p (&resquery->edma,ZPCI_EDMA_ADDR ); + iommu =s390_pci_get_iommu (s,pci_get_bus (pbdev->pdev), +PCI_FUNC (pbdev->pdev->devfn)); +stq_p (&resquery->sdma, iommu->sdma); +stq_p (&resquery->edma, iommu->edma);stl_p (&resquery->fid, pbdev->fid);stw_p (&resquery->pchid,0 );stw_p (&resquery->ug,1 );
@@ -860,7 +874,7 @@ static int reg_ioat(CPUS390XState *env, S390PCIIOMMU *iommu, ZpciFib fib, pba &= ~0xfff ; pal |=0xfff ; -if (pba > pal || pba <ZPCI_SDMA_ADDR || pal >ZPCI_EDMA_ADDR ) { +if (pba > pal || pba < iommu->sdma || pal > iommu->edma) {s390_program_interrupt (env,PGM_OPERAND ,6 , ra);return -EINVAL ; }
hw/vfio/common.c +3 -1
@@ -1045,7 +1045,6 @@ static void vfio_iommu_type1_get_info(int fd, VFIOContainer *container)int ret =0 ; info =g_malloc0 (argsz); - retry: info->argsz = argsz; info->flags =VFIO_IOMMU_INFO_CAPABILITIES ;
@@ -1079,6 +1078,8 @@ retry: dma_info->dma_end - dma_info->dma_start, info->iova_pgsizes); container->pgsizes = info->iova_pgsizes; +vfio_s390_iommu_setup (container, dma_info->dma_start, + dma_info->dma_end, info->iova_pgsizes);return ; } hdr = (struct vfio_info_cap_header *)((unsigned char *) dma_info +
@@ -1088,6 +1089,7 @@ out:/* Assume 4k IOVA page size */ vfio_host_win_add (container,0 , (hwaddr)-1 ,4096 ); container->pgsizes =4096 ; +vfio_s390_iommu_setup (container,0 , (hwaddr)-1 ,4096 );return ; }
include/hw/vfio/vfio-common.h +3 -0
@@ -198,4 +198,7 @@ int vfio_spapr_create_window(VFIOContainer *container,int vfio_spapr_remove_window (VFIOContainer *container,hwaddr offset_within_address_space); +void vfio_s390_iommu_setup (VFIOContainer *container,uint64_t min, +uint64_t max,uint64_t pgsize); +#endif /* HW_VFIO_VFIO_COMMON_H */
Signature · 1 line
2.7.4
Patchew URL: https://patchew.org/QEMU/[email protected]/
Patchew URL: https://patchew.org/QEMU/[email protected]/
Hi,
This series failed the docker-quick@centos7 build test. Please find the testing commands and their output below. If you have Docker installed, you can probably reproduce it locally.
=== TEST SCRIPT BEGIN ===
#!/bin/bash
time make docker-test-quick@centos7 SHOW_ENV=1 J=8 === TEST SCRIPT END ===
The full log is available at
http://patchew.org/logs/[email protected]/testing.docker-quick@centos7/?type=message.
---
Email generated automatically by Patchew [http://patchew.org/].
Please send your feedback to [email protected]